From 80f18fc933cf3f3e829c5455a1023d69f7b86e52 Mon Sep 17 00:00:00 2001 From: Ondřej Surý Date: Tue, 13 Sep 2011 13:11:55 +0200 Subject: Imported Upstream version 60 --- src/pkg/math/atanh.go | 79 --------------------------------------------------- 1 file changed, 79 deletions(-) delete mode 100644 src/pkg/math/atanh.go (limited to 'src/pkg/math/atanh.go') diff --git a/src/pkg/math/atanh.go b/src/pkg/math/atanh.go deleted file mode 100644 index 6aecb7b3b..000000000 --- a/src/pkg/math/atanh.go +++ /dev/null @@ -1,79 +0,0 @@ -// Copyright 2010 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -package math - - -// The original C code, the long comment, and the constants -// below are from FreeBSD's /usr/src/lib/msun/src/e_atanh.c -// and came with this notice. The go code is a simplified -// version of the original C. -// -// ==================================================== -// Copyright (C) 1993 by Sun Microsystems, Inc. All rights reserved. -// -// Developed at SunPro, a Sun Microsystems, Inc. business. -// Permission to use, copy, modify, and distribute this -// software is freely granted, provided that this notice -// is preserved. -// ==================================================== -// -// -// __ieee754_atanh(x) -// Method : -// 1. Reduce x to positive by atanh(-x) = -atanh(x) -// 2. For x>=0.5 -// 1 2x x -// atanh(x) = --- * log(1 + -------) = 0.5 * log1p(2 * --------) -// 2 1 - x 1 - x -// -// For x<0.5 -// atanh(x) = 0.5*log1p(2x+2x*x/(1-x)) -// -// Special cases: -// atanh(x) is NaN if |x| > 1 with signal; -// atanh(NaN) is that NaN with no signal; -// atanh(+-1) is +-INF with signal. -// - -// Atanh(x) calculates the inverse hyperbolic tangent of x. -// -// Special cases are: -// Atanh(x) = NaN if x < -1 or x > 1 -// Atanh(1) = +Inf -// Atanh(-1) = -Inf -// Atanh(NaN) = NaN -func Atanh(x float64) float64 { - const NearZero = 1.0 / (1 << 28) // 2**-28 - // TODO(rsc): Remove manual inlining of IsNaN - // when compiler does it for us - // special cases - switch { - case x < -1 || x > 1 || x != x: // x < -1 || x > 1 || IsNaN(x): - return NaN() - case x == 1: - return Inf(1) - case x == -1: - return Inf(-1) - } - sign := false - if x < 0 { - x = -x - sign = true - } - var temp float64 - switch { - case x < NearZero: - temp = x - case x < 0.5: - temp = x + x - temp = 0.5 * Log1p(temp+temp*x/(1-x)) - default: - temp = 0.5 * Log1p((x+x)/(1-x)) - } - if sign { - temp = -temp - } - return temp -} -- cgit v1.2.3 From 5ff4c17907d5b19510a62e08fd8d3b11e62b431d Mon Sep 17 00:00:00 2001 From: Ondřej Surý Date: Tue, 13 Sep 2011 13:13:40 +0200 Subject: Imported Upstream version 60 --- AUTHORS | 158 + CONTRIBUTORS | 238 + LICENSE | 27 + PATENTS | 22 + README | 31 + VERSION | 1 + doc/ExpressivenessOfGo.pdf | Bin 0 -> 859406 bytes doc/GoCourseDay1.pdf | Bin 0 -> 287552 bytes doc/GoCourseDay2.pdf | Bin 0 -> 268707 bytes doc/GoCourseDay3.pdf | Bin 0 -> 178003 bytes doc/Makefile | 11 + doc/all.css | 204 + doc/button_background.png | Bin 0 -> 126 bytes doc/code.html | 368 + doc/codelab/wiki/Makefile | 25 + doc/codelab/wiki/edit.html | 6 + doc/codelab/wiki/final-noclosure.go | 100 + doc/codelab/wiki/final-noerror.go | 52 + doc/codelab/wiki/final-parsetemplate.go | 90 + doc/codelab/wiki/final-template.go | 64 + doc/codelab/wiki/final.go | 94 + doc/codelab/wiki/get.go | 50 + doc/codelab/wiki/htmlify.go | 12 + doc/codelab/wiki/http-sample.go | 15 + doc/codelab/wiki/index.html | 1007 + doc/codelab/wiki/notemplate.go | 55 + doc/codelab/wiki/part1-noerror.go | 30 + doc/codelab/wiki/part1.go | 33 + doc/codelab/wiki/part2.go | 40 + doc/codelab/wiki/srcextract.go | 72 + doc/codelab/wiki/test.sh | 27 + doc/codelab/wiki/test_Test.txt.good | 1 + doc/codelab/wiki/test_edit.good | 6 + doc/codelab/wiki/test_view.good | 5 + doc/codelab/wiki/view.html | 5 + doc/codelab/wiki/wiki.html | 784 + doc/codereview_with_mq.html | 113 + doc/codewalk/codewalk.css | 234 + doc/codewalk/codewalk.js | 305 + doc/codewalk/codewalk.xml | 124 + doc/codewalk/functions.xml | 115 + doc/codewalk/markov.go | 130 + doc/codewalk/markov.xml | 308 + doc/codewalk/pig.go | 124 + doc/codewalk/popout.png | Bin 0 -> 213 bytes doc/codewalk/sharemem.xml | 181 + doc/codewalk/urlpoll.go | 117 + doc/community.html | 53 + doc/contrib.html | 69 + doc/contribute.html | 529 + doc/devel/index.html | 11 + doc/devel/release.html | 436 + doc/devel/roadmap.html | 135 + doc/devel/weekly.html | 3970 ++++ doc/docs.html | 236 + doc/effective_go.html | 3041 +++ doc/frontpage.css | 143 + doc/gccgo_contribute.html | 99 + doc/gccgo_install.html | 409 + doc/go-logo-black.png | Bin 0 -> 8855 bytes doc/go-logo-blue.png | Bin 0 -> 10874 bytes doc/go-logo-white.png | Bin 0 -> 25371 bytes doc/go_faq.html | 1496 ++ doc/go_for_cpp_programmers.html | 707 + doc/go_mem.html | 510 + doc/go_spec.html | 5272 +++++ doc/go_tutorial.html | 1455 ++ doc/go_tutorial.tmpl | 990 + doc/godocs.js | 190 + doc/gopher/appenginegopher.jpg | Bin 0 -> 135882 bytes doc/gopher/appenginegophercolor.jpg | Bin 0 -> 162023 bytes doc/gopher/appenginelogo.gif | Bin 0 -> 2105 bytes doc/gopher/bumper.png | Bin 0 -> 380345 bytes doc/gopher/bumper192x108.png | Bin 0 -> 10781 bytes doc/gopher/bumper320x180.png | Bin 0 -> 20193 bytes doc/gopher/bumper480x270.png | Bin 0 -> 36347 bytes doc/gopher/bumper640x360.png | Bin 0 -> 57726 bytes doc/gopher/gopherbw.png | Bin 0 -> 210660 bytes doc/gopher/gophercolor.png | Bin 0 -> 231683 bytes doc/gopher/gophercolor16x16.png | Bin 0 -> 785 bytes doc/ie.css | 1 + doc/install.html | 474 + doc/logo-153x55.png | Bin 0 -> 4055 bytes doc/logo.png | Bin 0 -> 1938 bytes doc/makehtml | 17 + doc/play_overlay.png | Bin 0 -> 1703 bytes doc/playground.html | 27 + doc/popups.js | 24 + doc/progs/cat.go | 47 + doc/progs/cat_rot13.go | 90 + doc/progs/echo.go | 32 + doc/progs/file.go | 89 + doc/progs/file_windows.go | 89 + doc/progs/helloworld.go | 11 + doc/progs/helloworld3.go | 21 + doc/progs/print.go | 23 + doc/progs/print_string.go | 21 + doc/progs/run | 85 + doc/progs/server.go | 51 + doc/progs/server1.go | 56 + doc/progs/sieve.go | 38 + doc/progs/sieve1.go | 51 + doc/progs/sort.go | 59 + doc/progs/sortmain.go | 69 + doc/progs/strings.go | 17 + doc/progs/sum.go | 20 + doc/root.html | 98 + doc/sieve.gif | Bin 0 -> 11019 bytes doc/talks/go_talk-20100112.html | 411 + doc/talks/go_talk-20100121.html | 453 + doc/talks/go_talk-20100323.html | 395 + doc/talks/gofrontend-gcc-summit-2010.pdf | Bin 0 -> 125185 bytes doc/talks/io2010/balance.go | 168 + doc/talks/io2010/decrypt.go | 85 + doc/talks/io2010/encrypt.go | 52 + doc/talks/io2010/eval1.go | 229 + doc/talks/io2010/eval2.go | 261 + doc/talks/io2010/talk.pdf | Bin 0 -> 1094941 bytes doc/talks/io2011/Real_World_Go.pdf | Bin 0 -> 609477 bytes doc/talks/io2011/Writing_Web_Apps_in_Go.pdf | Bin 0 -> 729499 bytes doc/talks/java-typing.png | Bin 0 -> 19328 bytes doc/talks/slidy.css | 277 + doc/talks/slidy.js | 2772 +++ doc/tmpltohtml.go | 176 + doc/video-001.png | Bin 0 -> 29228 bytes doc/video-002.png | Bin 0 -> 22027 bytes doc/video-003.png | Bin 0 -> 11189 bytes doc/video-004.png | Bin 0 -> 22713 bytes doc/video-005.jpg | Bin 0 -> 6783 bytes favicon.ico | Bin 0 -> 785 bytes include/ar.h | 47 + include/bio.h | 116 + include/bootexec.h | 169 + include/fmt.h | 116 + include/libc.h | 386 + include/mach.h | 410 + include/u.h | 230 + include/ureg_amd64.h | 58 + include/ureg_arm.h | 49 + include/ureg_x86.h | 53 + include/utf.h | 1 + lib/codereview/codereview.cfg | 1 + lib/codereview/codereview.py | 3292 +++ lib/godoc/codewalk.html | 58 + lib/godoc/codewalkdir.html | 16 + lib/godoc/dirlist.html | 31 + lib/godoc/error.html | 9 + lib/godoc/godoc.html | 69 + lib/godoc/package.html | 122 + lib/godoc/package.txt | 82 + lib/godoc/search.html | 98 + lib/godoc/search.txt | 40 + misc/IntelliJIDEA/Go.xml | 98 + misc/arm/a | 58 + misc/bash/go | 6 + misc/bbedit/Go.plist | 101 + misc/cgo/gmp/Makefile | 38 + misc/cgo/gmp/fib.go | 43 + misc/cgo/gmp/gmp.go | 368 + misc/cgo/gmp/pi.go | 104 + misc/cgo/life/Makefile | 21 + misc/cgo/life/c-life.c | 56 + misc/cgo/life/golden.out | 17 + misc/cgo/life/life.go | 39 + misc/cgo/life/life.h | 7 + misc/cgo/life/main.go | 44 + misc/cgo/life/test.bash | 11 + misc/cgo/stdio/Makefile | 17 + misc/cgo/stdio/chain.go | 43 + misc/cgo/stdio/fib.go | 47 + misc/cgo/stdio/file.go | 45 + misc/cgo/stdio/golden.out | 150 + misc/cgo/stdio/hello.go | 11 + misc/cgo/stdio/test.bash | 15 + misc/cgo/test/Makefile | 26 + misc/cgo/test/align.go | 72 + misc/cgo/test/basic.go | 134 + misc/cgo/test/callback.go | 136 + misc/cgo/test/callback_c.c | 12 + misc/cgo/test/cgo_test.go | 28 + misc/cgo/test/duplicate_symbol.go | 21 + misc/cgo/test/env.go | 32 + misc/cgo/test/exports.go | 12 + misc/cgo/test/issue1222.go | 29 + misc/cgo/test/issue1328.go | 30 + misc/cgo/test/issue1560.go | 46 + misc/cgo/test/runtime.c | 21 + misc/chrome/gophertool/README.txt | 8 + misc/chrome/gophertool/background.html | 24 + misc/chrome/gophertool/gopher.js | 34 + misc/chrome/gophertool/gopher.png | Bin 0 -> 5588 bytes misc/chrome/gophertool/manifest.json | 17 + misc/chrome/gophertool/popup.html | 54 + misc/dashboard/README | 26 + misc/dashboard/builder/Makefile | 14 + misc/dashboard/builder/doc.go | 58 + misc/dashboard/builder/exec.go | 74 + misc/dashboard/builder/http.go | 148 + misc/dashboard/builder/main.go | 624 + misc/dashboard/builder/package.go | 94 + misc/dashboard/godashboard/_multiprocessing.py | 5 + misc/dashboard/godashboard/app.yaml | 25 + misc/dashboard/godashboard/auth.py | 13 + misc/dashboard/godashboard/const.py | 13 + misc/dashboard/godashboard/cron.yaml | 4 + misc/dashboard/godashboard/fail-notify.txt | 6 + misc/dashboard/godashboard/gobuild.py | 558 + misc/dashboard/godashboard/index.yaml | 51 + misc/dashboard/godashboard/key.py.dummy | 10 + misc/dashboard/godashboard/main.html | 62 + misc/dashboard/godashboard/package.html | 77 + misc/dashboard/godashboard/package.py | 429 + misc/dashboard/godashboard/project-edit.html | 47 + misc/dashboard/godashboard/project-notify.txt | 9 + misc/dashboard/godashboard/project.html | 85 + misc/dashboard/godashboard/static/favicon.ico | Bin 0 -> 785 bytes misc/dashboard/godashboard/static/style.css | 118 + misc/dashboard/godashboard/toutf8.py | 14 + misc/dashboard/googlecode_upload.py | 248 + misc/emacs/go-mode-load.el | 50 + misc/emacs/go-mode.el | 544 + misc/fraise/go.plist | 92 + misc/fraise/readme.txt | 16 + misc/godoc/README | 22 + misc/godoc/app.yaml | 12 + misc/godoc/init.go | 35 + misc/goplay/Makefile | 13 + misc/goplay/README | 1 + misc/goplay/doc.go | 25 + misc/goplay/goplay.go | 280 + misc/kate/go.xml | 147 + misc/notepadplus/README | 8 + misc/notepadplus/go.xml | 66 + misc/notepadplus/userDefineLang.xml | 36 + misc/swig/callback/Makefile | 17 + misc/swig/callback/callback.h | 24 + misc/swig/callback/callback.swigcxx | 18 + misc/swig/callback/run | Bin 0 -> 1179384 bytes misc/swig/callback/run.go | 39 + misc/swig/stdio/Makefile | 17 + misc/swig/stdio/file.swig | 11 + misc/swig/stdio/hello | Bin 0 -> 231270 bytes misc/swig/stdio/hello.go | 11 + misc/vim/autoload/go/complete.vim | 49 + misc/vim/ftdetect/gofiletype.vim | 1 + misc/vim/ftplugin/go/fmt.vim | 30 + misc/vim/ftplugin/go/godoc.vim | 13 + misc/vim/ftplugin/go/import.vim | 201 + misc/vim/indent/go.vim | 65 + misc/vim/plugin/godoc.vim | 85 + misc/vim/readme.txt | 76 + misc/vim/syntax/go.vim | 208 + misc/vim/syntax/godoc.vim | 20 + misc/xcode/go.pbfilespec | 31 + misc/xcode/go.xclangspec | 293 + misc/zsh/go | 14 + robots.txt | 2 + src/Make.ccmd | 48 + src/Make.clib | 37 + src/Make.cmd | 50 + src/Make.common | 22 + src/Make.inc | 169 + src/Make.pkg | 249 + src/all-qemu.bash | 16 + src/all.bash | 14 + src/clean.bash | 29 + src/cmd/5a/Makefile | 25 + src/cmd/5a/a.h | 200 + src/cmd/5a/a.y | 710 + src/cmd/5a/doc.go | 14 + src/cmd/5a/lex.c | 704 + src/cmd/5c/Makefile | 34 + src/cmd/5c/cgen.c | 1199 ++ src/cmd/5c/doc.go | 14 + src/cmd/5c/gc.h | 384 + src/cmd/5c/list.c | 340 + src/cmd/5c/mul.c | 640 + src/cmd/5c/peep.c | 1468 ++ src/cmd/5c/reg.c | 1195 ++ src/cmd/5c/sgen.c | 267 + src/cmd/5c/swt.c | 694 + src/cmd/5c/txt.c | 1298 ++ src/cmd/5g/Makefile | 36 + src/cmd/5g/cgen.c | 1328 ++ src/cmd/5g/cgen64.c | 716 + src/cmd/5g/doc.go | 15 + src/cmd/5g/galign.c | 39 + src/cmd/5g/gg.h | 171 + src/cmd/5g/ggen.c | 1030 + src/cmd/5g/gobj.c | 623 + src/cmd/5g/gsubr.c | 2010 ++ src/cmd/5g/list.c | 331 + src/cmd/5g/opt.h | 165 + src/cmd/5g/peep.c | 1521 ++ src/cmd/5g/reg.c | 1607 ++ src/cmd/5l/5.out.h | 270 + src/cmd/5l/Makefile | 43 + src/cmd/5l/asm.c | 1878 ++ src/cmd/5l/doc.go | 39 + src/cmd/5l/l.h | 426 + src/cmd/5l/list.c | 487 + src/cmd/5l/mkenam | 45 + src/cmd/5l/noop.c | 539 + src/cmd/5l/obj.c | 744 + src/cmd/5l/optab.c | 236 + src/cmd/5l/pass.c | 332 + src/cmd/5l/prof.c | 211 + src/cmd/5l/softfloat.c | 89 + src/cmd/5l/span.c | 885 + src/cmd/6a/Makefile | 25 + src/cmd/6a/a.h | 214 + src/cmd/6a/a.y | 647 + src/cmd/6a/doc.go | 14 + src/cmd/6a/lex.c | 1326 ++ src/cmd/6c/Makefile | 36 + src/cmd/6c/cgen.c | 1976 ++ src/cmd/6c/div.c | 236 + src/cmd/6c/doc.go | 14 + src/cmd/6c/gc.h | 408 + src/cmd/6c/list.c | 396 + src/cmd/6c/machcap.c | 107 + src/cmd/6c/mul.c | 458 + src/cmd/6c/peep.c | 890 + src/cmd/6c/reg.c | 1386 ++ src/cmd/6c/sgen.c | 485 + src/cmd/6c/swt.c | 587 + src/cmd/6c/txt.c | 1564 ++ src/cmd/6g/Makefile | 35 + src/cmd/6g/cgen.c | 1299 ++ src/cmd/6g/doc.go | 13 + src/cmd/6g/galign.c | 37 + src/cmd/6g/gg.h | 161 + src/cmd/6g/ggen.c | 1412 ++ src/cmd/6g/gobj.c | 644 + src/cmd/6g/gsubr.c | 2159 ++ src/cmd/6g/list.c | 359 + src/cmd/6g/opt.h | 166 + src/cmd/6g/peep.c | 999 + src/cmd/6g/reg.c | 1690 ++ src/cmd/6l/6.out.h | 866 + src/cmd/6l/Makefile | 48 + src/cmd/6l/asm.c | 1179 ++ src/cmd/6l/doc.go | 53 + src/cmd/6l/l.h | 421 + src/cmd/6l/list.c | 458 + src/cmd/6l/mkenam | 45 + src/cmd/6l/obj.c | 765 + src/cmd/6l/optab.c | 1279 ++ src/cmd/6l/pass.c | 722 + src/cmd/6l/prof.c | 171 + src/cmd/6l/span.c | 1747 ++ src/cmd/8a/Makefile | 25 + src/cmd/8a/a.h | 215 + src/cmd/8a/a.y | 614 + src/cmd/8a/doc.go | 14 + src/cmd/8a/lex.c | 972 + src/cmd/8c/Makefile | 37 + src/cmd/8c/cgen.c | 1857 ++ src/cmd/8c/cgen64.c | 2657 +++ src/cmd/8c/div.c | 236 + src/cmd/8c/doc.go | 14 + src/cmd/8c/gc.h | 411 + src/cmd/8c/list.c | 328 + src/cmd/8c/machcap.c | 116 + src/cmd/8c/mul.c | 458 + src/cmd/8c/peep.c | 801 + src/cmd/8c/reg.c | 1287 ++ src/cmd/8c/sgen.c | 485 + src/cmd/8c/swt.c | 588 + src/cmd/8c/txt.c | 1458 ++ src/cmd/8g/Makefile | 36 + src/cmd/8g/cgen.c | 1233 ++ src/cmd/8g/cgen64.c | 512 + src/cmd/8g/doc.go | 15 + src/cmd/8g/galign.c | 37 + src/cmd/8g/gg.h | 187 + src/cmd/8g/ggen.c | 1155 ++ src/cmd/8g/gobj.c | 651 + src/cmd/8g/gsubr.c | 1959 ++ src/cmd/8g/list.c | 302 + src/cmd/8g/opt.h | 164 + src/cmd/8g/peep.c | 890 + src/cmd/8g/reg.c | 1544 ++ src/cmd/8l/8.out.h | 543 + src/cmd/8l/Makefile | 49 + src/cmd/8l/asm.c | 1253 ++ src/cmd/8l/doc.go | 50 + src/cmd/8l/l.h | 374 + src/cmd/8l/list.c | 369 + src/cmd/8l/mkenam | 45 + src/cmd/8l/obj.c | 739 + src/cmd/8l/optab.c | 755 + src/cmd/8l/pass.c | 657 + src/cmd/8l/prof.c | 173 + src/cmd/8l/span.c | 1325 ++ src/cmd/Makefile | 69 + src/cmd/cc/Makefile | 36 + src/cmd/cc/acid.c | 344 + src/cmd/cc/bits.c | 120 + src/cmd/cc/cc.h | 831 + src/cmd/cc/cc.y | 1215 ++ src/cmd/cc/com.c | 1385 ++ src/cmd/cc/com64.c | 644 + src/cmd/cc/dcl.c | 1669 ++ src/cmd/cc/doc.go | 11 + src/cmd/cc/dpchk.c | 723 + src/cmd/cc/funct.c | 431 + src/cmd/cc/godefs.c | 388 + src/cmd/cc/lex.c | 1561 ++ src/cmd/cc/lexbody | 769 + src/cmd/cc/mac.c | 34 + src/cmd/cc/macbody | 852 + src/cmd/cc/omachcap.c | 40 + src/cmd/cc/pgen.c | 594 + src/cmd/cc/pswt.c | 168 + src/cmd/cc/scon.c | 637 + src/cmd/cc/sub.c | 2056 ++ src/cmd/cgo/Makefile | 15 + src/cmd/cgo/ast.go | 414 + src/cmd/cgo/doc.go | 96 + src/cmd/cgo/gcc.go | 1375 ++ src/cmd/cgo/main.go | 268 + src/cmd/cgo/out.go | 745 + src/cmd/cgo/util.go | 110 + src/cmd/cov/Makefile | 39 + src/cmd/cov/doc.go | 33 + src/cmd/cov/main.c | 480 + src/cmd/cov/tree.c | 246 + src/cmd/cov/tree.h | 47 + src/cmd/ebnflint/Makefile | 15 + src/cmd/ebnflint/doc.go | 22 + src/cmd/ebnflint/ebnflint.go | 109 + src/cmd/gc/Makefile | 71 + src/cmd/gc/align.c | 653 + src/cmd/gc/bisonerrors | 124 + src/cmd/gc/bits.c | 161 + src/cmd/gc/builtin.c.boot | 114 + src/cmd/gc/closure.c | 252 + src/cmd/gc/const.c | 1299 ++ src/cmd/gc/cplx.c | 479 + src/cmd/gc/dcl.c | 1254 ++ src/cmd/gc/doc.go | 55 + src/cmd/gc/export.c | 428 + src/cmd/gc/gen.c | 790 + src/cmd/gc/go.errors | 70 + src/cmd/gc/go.h | 1279 ++ src/cmd/gc/go.y | 1985 ++ src/cmd/gc/init.c | 195 + src/cmd/gc/lex.c | 1956 ++ src/cmd/gc/md5.c | 290 + src/cmd/gc/md5.h | 16 + src/cmd/gc/mkbuiltin | 31 + src/cmd/gc/mkbuiltin1.c | 84 + src/cmd/gc/mkopnames | 24 + src/cmd/gc/mparith1.c | 509 + src/cmd/gc/mparith2.c | 683 + src/cmd/gc/mparith3.c | 313 + src/cmd/gc/obj.c | 301 + src/cmd/gc/pgen.c | 210 + src/cmd/gc/print.c | 480 + src/cmd/gc/range.c | 256 + src/cmd/gc/reflect.c | 939 + src/cmd/gc/runtime.go | 130 + src/cmd/gc/select.c | 351 + src/cmd/gc/sinit.c | 971 + src/cmd/gc/subr.c | 3885 ++++ src/cmd/gc/swt.c | 896 + src/cmd/gc/typecheck.c | 2827 +++ src/cmd/gc/unsafe.c | 97 + src/cmd/gc/unsafe.go | 22 + src/cmd/gc/walk.c | 2166 ++ src/cmd/godefs/Makefile | 19 + src/cmd/godefs/a.h | 104 + src/cmd/godefs/doc.go | 99 + src/cmd/godefs/main.c | 606 + src/cmd/godefs/stabs.c | 456 + src/cmd/godefs/test.sh | 45 + src/cmd/godefs/testdata.c | 41 + src/cmd/godefs/testdata_darwin_386.golden | 31 + src/cmd/godefs/testdata_darwin_amd64.golden | 31 + src/cmd/godefs/util.c | 36 + src/cmd/godoc/Makefile | 24 + src/cmd/godoc/appconfig.go | 19 + src/cmd/godoc/appinit.go | 86 + src/cmd/godoc/codewalk.go | 487 + src/cmd/godoc/dirtrees.go | 342 + src/cmd/godoc/doc.go | 130 + src/cmd/godoc/filesystem.go | 104 + src/cmd/godoc/format.go | 358 + src/cmd/godoc/godoc.go | 1159 ++ src/cmd/godoc/httpzip.go | 181 + src/cmd/godoc/index.go | 986 + src/cmd/godoc/main.go | 423 + src/cmd/godoc/mapping.go | 200 + src/cmd/godoc/parser.go | 68 + src/cmd/godoc/snippet.go | 100 + src/cmd/godoc/spec.go | 198 + src/cmd/godoc/utils.go | 168 + src/cmd/godoc/zip.go | 207 + src/cmd/gofix/Makefile | 34 + src/cmd/gofix/doc.go | 36 + src/cmd/gofix/filepath.go | 53 + src/cmd/gofix/filepath_test.go | 33 + src/cmd/gofix/fix.go | 582 + src/cmd/gofix/httpfinalurl.go | 56 + src/cmd/gofix/httpfinalurl_test.go | 37 + src/cmd/gofix/httpfs.go | 63 + src/cmd/gofix/httpfs_test.go | 47 + src/cmd/gofix/httpheaders.go | 66 + src/cmd/gofix/httpheaders_test.go | 73 + src/cmd/gofix/httpserver.go | 140 + src/cmd/gofix/httpserver_test.go | 53 + src/cmd/gofix/main.go | 258 + src/cmd/gofix/main_test.go | 126 + src/cmd/gofix/netdial.go | 114 + src/cmd/gofix/netdial_test.go | 51 + src/cmd/gofix/oserrorstring.go | 74 + src/cmd/gofix/oserrorstring_test.go | 57 + src/cmd/gofix/osopen.go | 123 + src/cmd/gofix/osopen_test.go | 82 + src/cmd/gofix/procattr.go | 61 + src/cmd/gofix/procattr_test.go | 74 + src/cmd/gofix/reflect.go | 861 + src/cmd/gofix/reflect_test.go | 31 + src/cmd/gofix/signal.go | 49 + src/cmd/gofix/signal_test.go | 94 + src/cmd/gofix/sorthelpers.go | 46 + src/cmd/gofix/sorthelpers_test.go | 45 + src/cmd/gofix/sortslice.go | 49 + src/cmd/gofix/sortslice_test.go | 35 + src/cmd/gofix/stringssplit.go | 71 + src/cmd/gofix/stringssplit_test.go | 51 + src/cmd/gofix/testdata/reflect.asn1.go.in | 814 + src/cmd/gofix/testdata/reflect.asn1.go.out | 814 + src/cmd/gofix/testdata/reflect.datafmt.go.in | 710 + src/cmd/gofix/testdata/reflect.datafmt.go.out | 710 + src/cmd/gofix/testdata/reflect.decode.go.in | 905 + src/cmd/gofix/testdata/reflect.decode.go.out | 908 + src/cmd/gofix/testdata/reflect.decoder.go.in | 196 + src/cmd/gofix/testdata/reflect.decoder.go.out | 196 + src/cmd/gofix/testdata/reflect.dnsmsg.go.in | 777 + src/cmd/gofix/testdata/reflect.dnsmsg.go.out | 777 + src/cmd/gofix/testdata/reflect.encode.go.in | 367 + src/cmd/gofix/testdata/reflect.encode.go.out | 367 + src/cmd/gofix/testdata/reflect.encoder.go.in | 240 + src/cmd/gofix/testdata/reflect.encoder.go.out | 240 + src/cmd/gofix/testdata/reflect.export.go.in | 400 + src/cmd/gofix/testdata/reflect.export.go.out | 400 + src/cmd/gofix/testdata/reflect.print.go.in | 944 + src/cmd/gofix/testdata/reflect.print.go.out | 944 + src/cmd/gofix/testdata/reflect.quick.go.in | 364 + src/cmd/gofix/testdata/reflect.quick.go.out | 365 + src/cmd/gofix/testdata/reflect.read.go.in | 620 + src/cmd/gofix/testdata/reflect.read.go.out | 620 + src/cmd/gofix/testdata/reflect.scan.go.in | 1082 + src/cmd/gofix/testdata/reflect.scan.go.out | 1082 + src/cmd/gofix/testdata/reflect.script.go.in | 359 + src/cmd/gofix/testdata/reflect.script.go.out | 359 + src/cmd/gofix/testdata/reflect.template.go.in | 1043 + src/cmd/gofix/testdata/reflect.template.go.out | 1044 + src/cmd/gofix/testdata/reflect.type.go.in | 789 + src/cmd/gofix/testdata/reflect.type.go.out | 789 + src/cmd/gofix/typecheck.go | 584 + src/cmd/gofix/url.go | 118 + src/cmd/gofix/url_test.go | 159 + src/cmd/gofmt/Makefile | 19 + src/cmd/gofmt/doc.go | 73 + src/cmd/gofmt/gofmt.go | 261 + src/cmd/gofmt/gofmt_test.go | 79 + src/cmd/gofmt/rewrite.go | 291 + src/cmd/gofmt/simplify.go | 65 + src/cmd/gofmt/test.sh | 162 + src/cmd/gofmt/testdata/composites.golden | 104 + src/cmd/gofmt/testdata/composites.input | 104 + src/cmd/gofmt/testdata/rewrite1.golden | 12 + src/cmd/gofmt/testdata/rewrite1.input | 12 + src/cmd/gofmt/testdata/rewrite2.golden | 10 + src/cmd/gofmt/testdata/rewrite2.input | 10 + src/cmd/goinstall/Makefile | 19 + src/cmd/goinstall/doc.go | 196 + src/cmd/goinstall/download.go | 353 + src/cmd/goinstall/main.go | 334 + src/cmd/goinstall/make.go | 175 + src/cmd/goinstall/tag_test.go | 73 + src/cmd/gomake/doc.go | 36 + src/cmd/gopack/Makefile | 12 + src/cmd/gopack/ar.c | 1717 ++ src/cmd/gopack/doc.go | 26 + src/cmd/gotest/Makefile | 12 + src/cmd/gotest/doc.go | 113 + src/cmd/gotest/flag.go | 159 + src/cmd/gotest/gotest.go | 435 + src/cmd/gotry/Makefile | 18 + src/cmd/gotry/gotry | 168 + src/cmd/gotype/Makefile | 17 + src/cmd/gotype/doc.go | 61 + src/cmd/gotype/gotype.go | 192 + src/cmd/gotype/gotype_test.go | 49 + src/cmd/gotype/testdata/test1.go | 6 + src/cmd/govet/Makefile | 14 + src/cmd/govet/doc.go | 38 + src/cmd/govet/govet.go | 404 + src/cmd/goyacc/Makefile | 17 + src/cmd/goyacc/doc.go | 46 + src/cmd/goyacc/goyacc.go | 3311 +++ src/cmd/goyacc/units.txt | 576 + src/cmd/goyacc/units.y | 765 + src/cmd/hgpatch/Makefile | 11 + src/cmd/hgpatch/doc.go | 18 + src/cmd/hgpatch/main.go | 358 + src/cmd/ld/data.c | 1026 + src/cmd/ld/doc.go | 11 + src/cmd/ld/dwarf.c | 2588 +++ src/cmd/ld/dwarf.h | 30 + src/cmd/ld/dwarf_defs.h | 503 + src/cmd/ld/elf.c | 564 + src/cmd/ld/elf.h | 989 + src/cmd/ld/go.c | 869 + src/cmd/ld/ldelf.c | 816 + src/cmd/ld/ldmacho.c | 821 + src/cmd/ld/ldpe.c | 451 + src/cmd/ld/lib.c | 1417 ++ src/cmd/ld/lib.h | 308 + src/cmd/ld/macho.c | 518 + src/cmd/ld/macho.h | 94 + src/cmd/ld/pe.c | 593 + src/cmd/ld/pe.h | 179 + src/cmd/ld/symtab.c | 378 + src/cmd/nm/Makefile | 15 + src/cmd/nm/doc.go | 21 + src/cmd/nm/nm.c | 368 + src/cmd/prof/Makefile | 38 + src/cmd/prof/doc.go | 48 + src/cmd/prof/gopprof | 4975 +++++ src/cmd/prof/main.c | 895 + src/env.bash | 106 + src/lib9/Makefile | 121 + src/lib9/_exits.c | 35 + src/lib9/_p9dir.c | 179 + src/lib9/argv0.c | 35 + src/lib9/atoi.c | 45 + src/lib9/await.c | 179 + src/lib9/cleanname.c | 78 + src/lib9/create.c | 83 + src/lib9/dirfstat.c | 54 + src/lib9/dirfwstat.c | 80 + src/lib9/dirstat.c | 63 + src/lib9/dirwstat.c | 43 + src/lib9/dup.c | 36 + src/lib9/errstr.c | 106 + src/lib9/exec.c | 33 + src/lib9/execl.c | 53 + src/lib9/exitcode.c | 34 + src/lib9/exits.c | 34 + src/lib9/fmt/charstod.c | 88 + src/lib9/fmt/dofmt.c | 630 + src/lib9/fmt/dorfmt.c | 65 + src/lib9/fmt/errfmt.c | 30 + src/lib9/fmt/fltfmt.c | 679 + src/lib9/fmt/fmt.c | 235 + src/lib9/fmt/fmtdef.h | 119 + src/lib9/fmt/fmtfd.c | 51 + src/lib9/fmt/fmtfdflush.c | 37 + src/lib9/fmt/fmtlocale.c | 69 + src/lib9/fmt/fmtlock.c | 31 + src/lib9/fmt/fmtnull.c | 48 + src/lib9/fmt/fmtprint.c | 51 + src/lib9/fmt/fmtquote.c | 274 + src/lib9/fmt/fmtrune.c | 43 + src/lib9/fmt/fmtstr.c | 31 + src/lib9/fmt/fmtvprint.c | 52 + src/lib9/fmt/fprint.c | 33 + src/lib9/fmt/nan64.c | 93 + src/lib9/fmt/pow10.c | 60 + src/lib9/fmt/print.c | 33 + src/lib9/fmt/seprint.c | 33 + src/lib9/fmt/smprint.c | 33 + src/lib9/fmt/snprint.c | 34 + src/lib9/fmt/sprint.c | 45 + src/lib9/fmt/strtod.c | 532 + src/lib9/fmt/test.c | 65 + src/lib9/fmt/vfprint.c | 37 + src/lib9/fmt/vseprint.c | 44 + src/lib9/fmt/vsmprint.c | 87 + src/lib9/fmt/vsnprint.c | 43 + src/lib9/fmtlock2.c | 38 + src/lib9/fork.c | 46 + src/lib9/getenv.c | 50 + src/lib9/getfields.c | 63 + src/lib9/getuser.c | 41 + src/lib9/getwd.c | 55 + src/lib9/goos.c | 41 + src/lib9/jmp.c | 42 + src/lib9/main.c | 38 + src/lib9/nan.c | 52 + src/lib9/notify.c | 297 + src/lib9/nulldir.c | 35 + src/lib9/open.c | 68 + src/lib9/readn.c | 48 + src/lib9/rfork.c | 153 + src/lib9/seek.c | 33 + src/lib9/strecpy.c | 43 + src/lib9/sysfatal.c | 47 + src/lib9/time.c | 66 + src/lib9/tokenize.c | 133 + src/lib9/utf/Makefile | 32 + src/lib9/utf/mkrunetype.c | 732 + src/lib9/utf/rune.c | 351 + src/lib9/utf/runetype.c | 38 + src/lib9/utf/runetypebody-5.0.0.c | 1361 ++ src/lib9/utf/runetypebody-5.2.0.c | 1541 ++ src/lib9/utf/runetypebody-6.0.0.c | 1565 ++ src/lib9/utf/utf.h | 242 + src/lib9/utf/utfdef.h | 28 + src/lib9/utf/utfecpy.c | 36 + src/lib9/utf/utflen.c | 38 + src/lib9/utf/utfnlen.c | 41 + src/lib9/utf/utfrrune.c | 47 + src/lib9/utf/utfrune.c | 46 + src/lib9/utf/utfutf.c | 42 + src/lib9/win32.c | 26 + src/libbio/Makefile | 51 + src/libbio/bbuffered.c | 46 + src/libbio/bfildes.c | 35 + src/libbio/bflush.c | 59 + src/libbio/bgetc.c | 79 + src/libbio/bgetd.c | 62 + src/libbio/bgetrune.c | 73 + src/libbio/binit.c | 179 + src/libbio/boffset.c | 51 + src/libbio/bprint.c | 82 + src/libbio/bputc.c | 46 + src/libbio/bputrune.c | 49 + src/libbio/brdline.c | 120 + src/libbio/brdstr.c | 60 + src/libbio/bread.c | 71 + src/libbio/bseek.c | 93 + src/libbio/bwrite.c | 64 + src/libmach/5.c | 92 + src/libmach/5db.c | 1095 + src/libmach/5obj.c | 174 + src/libmach/6.c | 145 + src/libmach/6obj.c | 179 + src/libmach/8.c | 107 + src/libmach/8db.c | 2395 +++ src/libmach/8obj.c | 176 + src/libmach/Makefile | 64 + src/libmach/access.c | 241 + src/libmach/darwin.c | 887 + src/libmach/elf.h | 182 + src/libmach/executable.c | 1280 ++ src/libmach/fakeobj.c | 29 + src/libmach/freebsd.c | 46 + src/libmach/linux.c | 1010 + src/libmach/machdata.c | 477 + src/libmach/macho.h | 99 + src/libmach/map.c | 181 + src/libmach/obj.c | 391 + src/libmach/obj.h | 53 + src/libmach/openbsd.c | 46 + src/libmach/setmach.c | 203 + src/libmach/swap.c | 107 + src/libmach/sym.c | 1443 ++ src/libmach/windows.c | 67 + src/make.bash | 110 + src/pkg/Makefile | 312 + src/pkg/archive/tar/Makefile | 13 + src/pkg/archive/tar/common.go | 75 + src/pkg/archive/tar/reader.go | 221 + src/pkg/archive/tar/reader_test.go | 273 + src/pkg/archive/tar/testdata/gnu.tar | Bin 0 -> 3072 bytes src/pkg/archive/tar/testdata/small.txt | 1 + src/pkg/archive/tar/testdata/small2.txt | 1 + src/pkg/archive/tar/testdata/star.tar | Bin 0 -> 3072 bytes src/pkg/archive/tar/testdata/v7.tar | Bin 0 -> 3584 bytes src/pkg/archive/tar/testdata/writer-big.tar | Bin 0 -> 4096 bytes src/pkg/archive/tar/testdata/writer.tar | Bin 0 -> 3072 bytes src/pkg/archive/tar/writer.go | 205 + src/pkg/archive/tar/writer_test.go | 157 + src/pkg/archive/zip/Makefile | 13 + src/pkg/archive/zip/reader.go | 311 + src/pkg/archive/zip/reader_test.go | 233 + src/pkg/archive/zip/struct.go | 91 + src/pkg/archive/zip/testdata/dd.zip | Bin 0 -> 154 bytes src/pkg/archive/zip/testdata/gophercolor16x16.png | Bin 0 -> 785 bytes src/pkg/archive/zip/testdata/r.zip | Bin 0 -> 440 bytes src/pkg/archive/zip/testdata/readme.notzip | Bin 0 -> 1905 bytes src/pkg/archive/zip/testdata/readme.zip | Bin 0 -> 1885 bytes src/pkg/archive/zip/testdata/test.zip | Bin 0 -> 1170 bytes src/pkg/archive/zip/writer.go | 244 + src/pkg/archive/zip/writer_test.go | 73 + src/pkg/archive/zip/zip_test.go | 57 + src/pkg/asn1/Makefile | 13 + src/pkg/asn1/asn1.go | 840 + src/pkg/asn1/asn1_test.go | 689 + src/pkg/asn1/common.go | 158 + src/pkg/asn1/marshal.go | 544 + src/pkg/asn1/marshal_test.go | 129 + src/pkg/big/Makefile | 18 + src/pkg/big/arith.go | 231 + src/pkg/big/arith_386.s | 265 + src/pkg/big/arith_amd64.s | 263 + src/pkg/big/arith_arm.s | 312 + src/pkg/big/arith_decl.go | 18 + src/pkg/big/arith_test.go | 335 + src/pkg/big/calibrate_test.go | 88 + src/pkg/big/hilbert_test.go | 160 + src/pkg/big/int.go | 868 + src/pkg/big/int_test.go | 1391 ++ src/pkg/big/nat.go | 1272 ++ src/pkg/big/nat_test.go | 669 + src/pkg/big/rat.go | 371 + src/pkg/big/rat_test.go | 332 + src/pkg/bufio/Makefile | 11 + src/pkg/bufio/bufio.go | 549 + src/pkg/bufio/bufio_test.go | 699 + src/pkg/builtin/builtin.go | 135 + src/pkg/bytes/Makefile | 16 + src/pkg/bytes/asm_386.s | 17 + src/pkg/bytes/asm_amd64.s | 92 + src/pkg/bytes/asm_arm.s | 8 + src/pkg/bytes/buffer.go | 348 + src/pkg/bytes/buffer_test.go | 376 + src/pkg/bytes/bytes.go | 605 + src/pkg/bytes/bytes_decl.go | 8 + src/pkg/bytes/bytes_test.go | 858 + src/pkg/bytes/export_test.go | 8 + src/pkg/cmath/Makefile | 25 + src/pkg/cmath/abs.go | 12 + src/pkg/cmath/asin.go | 170 + src/pkg/cmath/cmath_test.go | 853 + src/pkg/cmath/conj.go | 8 + src/pkg/cmath/exp.go | 55 + src/pkg/cmath/isinf.go | 21 + src/pkg/cmath/isnan.go | 25 + src/pkg/cmath/log.go | 64 + src/pkg/cmath/phase.go | 11 + src/pkg/cmath/polar.go | 12 + src/pkg/cmath/pow.go | 60 + src/pkg/cmath/rect.go | 13 + src/pkg/cmath/sin.go | 132 + src/pkg/cmath/sqrt.go | 103 + src/pkg/cmath/tan.go | 184 + src/pkg/compress/bzip2/Makefile | 14 + src/pkg/compress/bzip2/bit_reader.go | 88 + src/pkg/compress/bzip2/bzip2.go | 390 + src/pkg/compress/bzip2/bzip2_test.go | 158 + src/pkg/compress/bzip2/huffman.go | 223 + src/pkg/compress/bzip2/move_to_front.go | 105 + src/pkg/compress/flate/Makefile | 17 + src/pkg/compress/flate/deflate.go | 493 + src/pkg/compress/flate/deflate_test.go | 321 + src/pkg/compress/flate/flate_test.go | 139 + src/pkg/compress/flate/huffman_bit_writer.go | 494 + src/pkg/compress/flate/huffman_code.go | 378 + src/pkg/compress/flate/inflate.go | 708 + src/pkg/compress/flate/reverse_bits.go | 48 + src/pkg/compress/flate/token.go | 103 + src/pkg/compress/flate/util.go | 72 + src/pkg/compress/gzip/Makefile | 12 + src/pkg/compress/gzip/gunzip.go | 230 + src/pkg/compress/gzip/gunzip_test.go | 305 + src/pkg/compress/gzip/gzip.go | 187 + src/pkg/compress/gzip/gzip_test.go | 84 + src/pkg/compress/lzw/Makefile | 12 + src/pkg/compress/lzw/reader.go | 253 + src/pkg/compress/lzw/reader_test.go | 145 + src/pkg/compress/lzw/writer.go | 259 + src/pkg/compress/lzw/writer_test.go | 132 + src/pkg/compress/testdata/e.txt | 1 + src/pkg/compress/testdata/pi.txt | 1 + src/pkg/compress/zlib/Makefile | 12 + src/pkg/compress/zlib/reader.go | 126 + src/pkg/compress/zlib/reader_test.go | 128 + src/pkg/compress/zlib/writer.go | 139 + src/pkg/compress/zlib/writer_test.go | 144 + src/pkg/container/heap/Makefile | 11 + src/pkg/container/heap/heap.go | 96 + src/pkg/container/heap/heap_test.go | 158 + src/pkg/container/list/Makefile | 11 + src/pkg/container/list/list.go | 211 + src/pkg/container/list/list_test.go | 209 + src/pkg/container/ring/Makefile | 11 + src/pkg/container/ring/ring.go | 141 + src/pkg/container/ring/ring_test.go | 220 + src/pkg/container/vector/Makefile | 69 + src/pkg/container/vector/defs.go | 43 + src/pkg/container/vector/intvector.go | 188 + src/pkg/container/vector/intvector_test.go | 331 + src/pkg/container/vector/nogen_test.go | 67 + src/pkg/container/vector/numbers_test.go | 123 + src/pkg/container/vector/stringvector.go | 188 + src/pkg/container/vector/stringvector_test.go | 331 + src/pkg/container/vector/vector.go | 188 + src/pkg/container/vector/vector_test.go | 331 + src/pkg/crypto/Makefile | 11 + src/pkg/crypto/aes/Makefile | 13 + src/pkg/crypto/aes/aes_test.go | 350 + src/pkg/crypto/aes/block.go | 176 + src/pkg/crypto/aes/cipher.go | 71 + src/pkg/crypto/aes/const.go | 362 + src/pkg/crypto/blowfish/Makefile | 13 + src/pkg/crypto/blowfish/block.go | 101 + src/pkg/crypto/blowfish/blowfish_test.go | 192 + src/pkg/crypto/blowfish/cipher.go | 79 + src/pkg/crypto/blowfish/const.go | 199 + src/pkg/crypto/cast5/Makefile | 11 + src/pkg/crypto/cast5/cast5.go | 536 + src/pkg/crypto/cast5/cast5_test.go | 104 + src/pkg/crypto/cipher/Makefile | 17 + src/pkg/crypto/cipher/cbc.go | 78 + src/pkg/crypto/cipher/cbc_aes_test.go | 89 + src/pkg/crypto/cipher/cfb.go | 64 + src/pkg/crypto/cipher/cfb_test.go | 35 + src/pkg/crypto/cipher/cipher.go | 63 + src/pkg/crypto/cipher/common_test.go | 28 + src/pkg/crypto/cipher/ctr.go | 55 + src/pkg/crypto/cipher/ctr_aes_test.go | 101 + src/pkg/crypto/cipher/io.go | 57 + src/pkg/crypto/cipher/ocfb.go | 138 + src/pkg/crypto/cipher/ocfb_test.go | 44 + src/pkg/crypto/cipher/ofb.go | 44 + src/pkg/crypto/cipher/ofb_test.go | 101 + src/pkg/crypto/crypto.go | 73 + src/pkg/crypto/des/Makefile | 13 + src/pkg/crypto/des/block.go | 98 + src/pkg/crypto/des/cipher.go | 103 + src/pkg/crypto/des/const.go | 139 + src/pkg/crypto/des/des_test.go | 1497 ++ src/pkg/crypto/dsa/Makefile | 11 + src/pkg/crypto/dsa/dsa.go | 276 + src/pkg/crypto/dsa/dsa_test.go | 84 + src/pkg/crypto/ecdsa/Makefile | 11 + src/pkg/crypto/ecdsa/ecdsa.go | 149 + src/pkg/crypto/ecdsa/ecdsa_test.go | 227 + src/pkg/crypto/elliptic/Makefile | 11 + src/pkg/crypto/elliptic/elliptic.go | 381 + src/pkg/crypto/elliptic/elliptic_test.go | 334 + src/pkg/crypto/hmac/Makefile | 11 + src/pkg/crypto/hmac/hmac.go | 100 + src/pkg/crypto/hmac/hmac_test.go | 205 + src/pkg/crypto/md4/Makefile | 12 + src/pkg/crypto/md4/md4.go | 117 + src/pkg/crypto/md4/md4_test.go | 71 + src/pkg/crypto/md4/md4block.go | 89 + src/pkg/crypto/md5/Makefile | 12 + src/pkg/crypto/md5/md5.go | 117 + src/pkg/crypto/md5/md5_test.go | 71 + src/pkg/crypto/md5/md5block.go | 172 + src/pkg/crypto/ocsp/Makefile | 11 + src/pkg/crypto/ocsp/ocsp.go | 192 + src/pkg/crypto/ocsp/ocsp_test.go | 97 + src/pkg/crypto/openpgp/Makefile | 14 + src/pkg/crypto/openpgp/armor/Makefile | 12 + src/pkg/crypto/openpgp/armor/armor.go | 220 + src/pkg/crypto/openpgp/armor/armor_test.go | 95 + src/pkg/crypto/openpgp/armor/encode.go | 161 + src/pkg/crypto/openpgp/canonical_text.go | 58 + src/pkg/crypto/openpgp/canonical_text_test.go | 49 + src/pkg/crypto/openpgp/elgamal/Makefile | 11 + src/pkg/crypto/openpgp/elgamal/elgamal.go | 122 + src/pkg/crypto/openpgp/elgamal/elgamal_test.go | 49 + src/pkg/crypto/openpgp/error/Makefile | 11 + src/pkg/crypto/openpgp/error/error.go | 64 + src/pkg/crypto/openpgp/keys.go | 545 + src/pkg/crypto/openpgp/packet/Makefile | 22 + src/pkg/crypto/openpgp/packet/compressed.go | 39 + src/pkg/crypto/openpgp/packet/compressed_test.go | 41 + src/pkg/crypto/openpgp/packet/encrypted_key.go | 168 + .../crypto/openpgp/packet/encrypted_key_test.go | 126 + src/pkg/crypto/openpgp/packet/literal.go | 90 + .../crypto/openpgp/packet/one_pass_signature.go | 74 + src/pkg/crypto/openpgp/packet/packet.go | 483 + src/pkg/crypto/openpgp/packet/packet_test.go | 256 + src/pkg/crypto/openpgp/packet/private_key.go | 301 + src/pkg/crypto/openpgp/packet/private_key_test.go | 57 + src/pkg/crypto/openpgp/packet/public_key.go | 393 + src/pkg/crypto/openpgp/packet/public_key_test.go | 98 + src/pkg/crypto/openpgp/packet/reader.go | 63 + src/pkg/crypto/openpgp/packet/signature.go | 558 + src/pkg/crypto/openpgp/packet/signature_test.go | 42 + .../openpgp/packet/symmetric_key_encrypted.go | 162 + .../openpgp/packet/symmetric_key_encrypted_test.go | 101 + .../openpgp/packet/symmetrically_encrypted.go | 291 + .../openpgp/packet/symmetrically_encrypted_test.go | 124 + src/pkg/crypto/openpgp/packet/userid.go | 161 + src/pkg/crypto/openpgp/packet/userid_test.go | 87 + src/pkg/crypto/openpgp/read.go | 415 + src/pkg/crypto/openpgp/read_test.go | 361 + src/pkg/crypto/openpgp/s2k/Makefile | 11 + src/pkg/crypto/openpgp/s2k/s2k.go | 180 + src/pkg/crypto/openpgp/s2k/s2k_test.go | 118 + src/pkg/crypto/openpgp/write.go | 308 + src/pkg/crypto/openpgp/write_test.go | 233 + src/pkg/crypto/rand/Makefile | 30 + src/pkg/crypto/rand/rand.go | 21 + src/pkg/crypto/rand/rand_test.go | 31 + src/pkg/crypto/rand/rand_unix.go | 125 + src/pkg/crypto/rand/rand_windows.go | 43 + src/pkg/crypto/rand/util.go | 80 + src/pkg/crypto/rc4/Makefile | 11 + src/pkg/crypto/rc4/rc4.go | 66 + src/pkg/crypto/rc4/rc4_test.go | 59 + src/pkg/crypto/ripemd160/Makefile | 12 + src/pkg/crypto/ripemd160/ripemd160.go | 118 + src/pkg/crypto/ripemd160/ripemd160_test.go | 64 + src/pkg/crypto/ripemd160/ripemd160block.go | 161 + src/pkg/crypto/rsa/Makefile | 12 + src/pkg/crypto/rsa/pkcs1v15.go | 242 + src/pkg/crypto/rsa/pkcs1v15_test.go | 221 + src/pkg/crypto/rsa/rsa.go | 502 + src/pkg/crypto/rsa/rsa_test.go | 348 + src/pkg/crypto/sha1/Makefile | 12 + src/pkg/crypto/sha1/sha1.go | 119 + src/pkg/crypto/sha1/sha1_test.go | 73 + src/pkg/crypto/sha1/sha1block.go | 81 + src/pkg/crypto/sha256/Makefile | 12 + src/pkg/crypto/sha256/sha256.go | 166 + src/pkg/crypto/sha256/sha256_test.go | 125 + src/pkg/crypto/sha256/sha256block.go | 129 + src/pkg/crypto/sha512/Makefile | 12 + src/pkg/crypto/sha512/sha512.go | 170 + src/pkg/crypto/sha512/sha512_test.go | 125 + src/pkg/crypto/sha512/sha512block.go | 144 + src/pkg/crypto/subtle/Makefile | 11 + src/pkg/crypto/subtle/constant_time.go | 57 + src/pkg/crypto/subtle/constant_time_test.go | 105 + src/pkg/crypto/tls/Makefile | 20 + src/pkg/crypto/tls/alert.go | 73 + src/pkg/crypto/tls/cipher_suites.go | 102 + src/pkg/crypto/tls/common.go | 267 + src/pkg/crypto/tls/conn.go | 799 + src/pkg/crypto/tls/conn_test.go | 52 + src/pkg/crypto/tls/generate_cert.go | 72 + src/pkg/crypto/tls/handshake_client.go | 315 + src/pkg/crypto/tls/handshake_client_test.go | 211 + src/pkg/crypto/tls/handshake_messages.go | 904 + src/pkg/crypto/tls/handshake_messages_test.go | 206 + src/pkg/crypto/tls/handshake_server.go | 298 + src/pkg/crypto/tls/handshake_server_test.go | 517 + src/pkg/crypto/tls/key_agreement.go | 245 + src/pkg/crypto/tls/parse-gnutls-cli-debug-log.py | 55 + src/pkg/crypto/tls/prf.go | 153 + src/pkg/crypto/tls/prf_test.go | 104 + src/pkg/crypto/tls/tls.go | 181 + src/pkg/crypto/twofish/Makefile | 11 + src/pkg/crypto/twofish/twofish.go | 358 + src/pkg/crypto/twofish/twofish_test.go | 129 + src/pkg/crypto/x509/Makefile | 13 + src/pkg/crypto/x509/cert_pool.go | 106 + src/pkg/crypto/x509/pkix/Makefile | 11 + src/pkg/crypto/x509/pkix/pkix.go | 167 + src/pkg/crypto/x509/verify.go | 244 + src/pkg/crypto/x509/verify_test.go | 391 + src/pkg/crypto/x509/x509.go | 1073 + src/pkg/crypto/x509/x509_test.go | 431 + src/pkg/crypto/xtea/Makefile | 12 + src/pkg/crypto/xtea/block.go | 66 + src/pkg/crypto/xtea/cipher.go | 92 + src/pkg/crypto/xtea/xtea_test.go | 246 + src/pkg/csv/Makefile | 12 + src/pkg/csv/reader.go | 372 + src/pkg/csv/reader_test.go | 265 + src/pkg/csv/writer.go | 122 + src/pkg/csv/writer_test.go | 44 + src/pkg/debug/dwarf/Makefile | 16 + src/pkg/debug/dwarf/buf.go | 154 + src/pkg/debug/dwarf/const.go | 433 + src/pkg/debug/dwarf/entry.go | 343 + src/pkg/debug/dwarf/open.go | 80 + src/pkg/debug/dwarf/testdata/typedef.c | 79 + src/pkg/debug/dwarf/testdata/typedef.elf | Bin 0 -> 10837 bytes src/pkg/debug/dwarf/testdata/typedef.macho | Bin 0 -> 5256 bytes src/pkg/debug/dwarf/type.go | 584 + src/pkg/debug/dwarf/type_test.go | 111 + src/pkg/debug/dwarf/unit.go | 62 + src/pkg/debug/elf/Makefile | 12 + src/pkg/debug/elf/elf.go | 1521 ++ src/pkg/debug/elf/elf_test.go | 49 + src/pkg/debug/elf/file.go | 761 + src/pkg/debug/elf/file_test.go | 241 + src/pkg/debug/elf/testdata/gcc-386-freebsd-exec | Bin 0 -> 5742 bytes src/pkg/debug/elf/testdata/gcc-amd64-linux-exec | Bin 0 -> 8844 bytes .../testdata/go-relocation-test-gcc424-x86-64.obj | Bin 0 -> 3088 bytes .../testdata/go-relocation-test-gcc441-x86-64.obj | Bin 0 -> 2936 bytes .../elf/testdata/go-relocation-test-gcc441-x86.obj | Bin 0 -> 1884 bytes src/pkg/debug/gosym/Makefile | 19 + src/pkg/debug/gosym/pclinetest.h | 7 + src/pkg/debug/gosym/pclinetest.s | 58 + src/pkg/debug/gosym/pclntab.go | 82 + src/pkg/debug/gosym/pclntab_test.go | 204 + src/pkg/debug/gosym/symtab.go | 548 + src/pkg/debug/macho/Makefile | 12 + src/pkg/debug/macho/file.go | 517 + src/pkg/debug/macho/file_test.go | 167 + src/pkg/debug/macho/macho.go | 305 + src/pkg/debug/macho/testdata/gcc-386-darwin-exec | Bin 0 -> 12588 bytes src/pkg/debug/macho/testdata/gcc-amd64-darwin-exec | Bin 0 -> 8512 bytes .../macho/testdata/gcc-amd64-darwin-exec-debug | Bin 0 -> 4540 bytes src/pkg/debug/macho/testdata/hello.c | 8 + src/pkg/debug/pe/Makefile | 12 + src/pkg/debug/pe/file.go | 315 + src/pkg/debug/pe/file_test.go | 99 + src/pkg/debug/pe/pe.go | 51 + src/pkg/debug/pe/testdata/gcc-386-mingw-exec | Bin 0 -> 29941 bytes src/pkg/debug/pe/testdata/gcc-386-mingw-obj | Bin 0 -> 2372 bytes src/pkg/debug/pe/testdata/hello.c | 8 + src/pkg/deps.bash | 49 + src/pkg/ebnf/Makefile | 12 + src/pkg/ebnf/ebnf.go | 245 + src/pkg/ebnf/ebnf_test.go | 87 + src/pkg/ebnf/parser.go | 197 + src/pkg/encoding/ascii85/Makefile | 11 + src/pkg/encoding/ascii85/ascii85.go | 300 + src/pkg/encoding/ascii85/ascii85_test.go | 188 + src/pkg/encoding/base32/Makefile | 11 + src/pkg/encoding/base32/base32.go | 368 + src/pkg/encoding/base32/base32_test.go | 193 + src/pkg/encoding/base64/Makefile | 11 + src/pkg/encoding/base64/base64.go | 343 + src/pkg/encoding/base64/base64_test.go | 199 + src/pkg/encoding/binary/Makefile | 11 + src/pkg/encoding/binary/binary.go | 498 + src/pkg/encoding/binary/binary_test.go | 235 + src/pkg/encoding/git85/Makefile | 11 + src/pkg/encoding/git85/git.go | 277 + src/pkg/encoding/git85/git_test.go | 194 + src/pkg/encoding/hex/Makefile | 11 + src/pkg/encoding/hex/hex.go | 216 + src/pkg/encoding/hex/hex_test.go | 192 + src/pkg/encoding/pem/Makefile | 11 + src/pkg/encoding/pem/pem.go | 258 + src/pkg/encoding/pem/pem_test.go | 390 + src/pkg/exec/Makefile | 31 + src/pkg/exec/exec.go | 377 + src/pkg/exec/exec_test.go | 214 + src/pkg/exec/lp_plan9.go | 51 + src/pkg/exec/lp_test.go | 33 + src/pkg/exec/lp_unix.go | 52 + src/pkg/exec/lp_windows.go | 77 + src/pkg/exp/README | 3 + src/pkg/exp/datafmt/Makefile | 12 + src/pkg/exp/datafmt/datafmt.go | 710 + src/pkg/exp/datafmt/datafmt_test.go | 330 + src/pkg/exp/datafmt/parser.go | 368 + src/pkg/exp/gui/Makefile | 11 + src/pkg/exp/gui/gui.go | 58 + src/pkg/exp/gui/x11/Makefile | 12 + src/pkg/exp/gui/x11/auth.go | 93 + src/pkg/exp/gui/x11/conn.go | 627 + src/pkg/exp/norm/Makefile | 44 + src/pkg/exp/norm/composition.go | 344 + src/pkg/exp/norm/composition_test.go | 138 + src/pkg/exp/norm/forminfo.go | 188 + src/pkg/exp/norm/maketables.go | 855 + src/pkg/exp/norm/maketesttables.go | 42 + src/pkg/exp/norm/norm_test.go | 14 + src/pkg/exp/norm/normalize.go | 99 + src/pkg/exp/norm/tables.go | 6580 ++++++ src/pkg/exp/norm/trie.go | 234 + src/pkg/exp/norm/trie_test.go | 107 + src/pkg/exp/norm/triedata_test.go | 63 + src/pkg/exp/norm/triegen.go | 211 + src/pkg/exp/regexp/Makefile | 12 + src/pkg/exp/regexp/all_test.go | 429 + src/pkg/exp/regexp/exec.go | 295 + src/pkg/exp/regexp/find_test.go | 472 + src/pkg/exp/regexp/regexp.go | 795 + src/pkg/exp/regexp/syntax/Makefile | 16 + src/pkg/exp/regexp/syntax/compile.go | 269 + src/pkg/exp/regexp/syntax/make_perl_groups.pl | 103 + src/pkg/exp/regexp/syntax/parse.go | 1797 ++ src/pkg/exp/regexp/syntax/parse_test.go | 350 + src/pkg/exp/regexp/syntax/perl_groups.go | 130 + src/pkg/exp/regexp/syntax/prog.go | 237 + src/pkg/exp/regexp/syntax/prog_test.go | 91 + src/pkg/exp/regexp/syntax/regexp.go | 284 + src/pkg/exp/regexp/syntax/simplify.go | 151 + src/pkg/exp/regexp/syntax/simplify_test.go | 151 + src/pkg/exp/template/html/Makefile | 11 + src/pkg/exp/template/html/context.go | 98 + src/pkg/exp/template/html/escape.go | 105 + src/pkg/exp/template/html/escape_test.go | 75 + src/pkg/exp/wingui/Makefile | 29 + src/pkg/exp/wingui/gui.go | 153 + src/pkg/exp/wingui/winapi.go | 131 + src/pkg/exp/wingui/zwinapi.go | 211 + src/pkg/expvar/Makefile | 11 + src/pkg/expvar/expvar.go | 287 + src/pkg/expvar/expvar_test.go | 128 + src/pkg/flag/Makefile | 11 + src/pkg/flag/export_test.go | 22 + src/pkg/flag/flag.go | 705 + src/pkg/flag/flag_test.go | 255 + src/pkg/fmt/Makefile | 14 + src/pkg/fmt/doc.go | 181 + src/pkg/fmt/fmt_test.go | 746 + src/pkg/fmt/format.go | 452 + src/pkg/fmt/print.go | 996 + src/pkg/fmt/scan.go | 1112 + src/pkg/fmt/scan_test.go | 923 + src/pkg/fmt/stringer_test.go | 61 + src/pkg/go/ast/Makefile | 16 + src/pkg/go/ast/ast.go | 914 + src/pkg/go/ast/filter.go | 475 + src/pkg/go/ast/print.go | 224 + src/pkg/go/ast/print_test.go | 77 + src/pkg/go/ast/resolve.go | 174 + src/pkg/go/ast/scope.go | 156 + src/pkg/go/ast/walk.go | 382 + src/pkg/go/build/Makefile | 22 + src/pkg/go/build/build.go | 444 + src/pkg/go/build/build_test.go | 61 + src/pkg/go/build/cgotest/cgotest.c | 9 + src/pkg/go/build/cgotest/cgotest.go | 19 + src/pkg/go/build/cgotest/cgotest.h | 5 + src/pkg/go/build/cmdtest/main.go | 12 + src/pkg/go/build/dir.go | 172 + src/pkg/go/build/path.go | 182 + src/pkg/go/build/pkgtest/pkgtest.go | 9 + src/pkg/go/build/pkgtest/sqrt_386.s | 10 + src/pkg/go/build/pkgtest/sqrt_amd64.s | 9 + src/pkg/go/build/pkgtest/sqrt_arm.s | 10 + src/pkg/go/build/syslist_test.go | 62 + src/pkg/go/doc/Makefile | 12 + src/pkg/go/doc/comment.go | 345 + src/pkg/go/doc/doc.go | 641 + src/pkg/go/parser/Makefile | 12 + src/pkg/go/parser/interface.go | 214 + src/pkg/go/parser/parser.go | 2161 ++ src/pkg/go/parser/parser_test.go | 130 + src/pkg/go/printer/Makefile | 12 + src/pkg/go/printer/nodes.go | 1510 ++ src/pkg/go/printer/performance_test.go | 58 + src/pkg/go/printer/printer.go | 1012 + src/pkg/go/printer/printer_test.go | 194 + src/pkg/go/printer/testdata/comments.golden | 473 + src/pkg/go/printer/testdata/comments.input | 483 + src/pkg/go/printer/testdata/comments.x | 56 + src/pkg/go/printer/testdata/declarations.golden | 747 + src/pkg/go/printer/testdata/declarations.input | 757 + src/pkg/go/printer/testdata/empty.golden | 5 + src/pkg/go/printer/testdata/empty.input | 5 + src/pkg/go/printer/testdata/expressions.golden | 627 + src/pkg/go/printer/testdata/expressions.input | 656 + src/pkg/go/printer/testdata/expressions.raw | 627 + src/pkg/go/printer/testdata/linebreaks.golden | 223 + src/pkg/go/printer/testdata/linebreaks.input | 223 + src/pkg/go/printer/testdata/parser.go | 2153 ++ src/pkg/go/printer/testdata/slow.golden | 85 + src/pkg/go/printer/testdata/slow.input | 85 + src/pkg/go/printer/testdata/statements.golden | 412 + src/pkg/go/printer/testdata/statements.input | 351 + src/pkg/go/scanner/Makefile | 12 + src/pkg/go/scanner/errors.go | 168 + src/pkg/go/scanner/scanner.go | 670 + src/pkg/go/scanner/scanner_test.go | 672 + src/pkg/go/token/Makefile | 12 + src/pkg/go/token/position.go | 424 + src/pkg/go/token/position_test.go | 180 + src/pkg/go/token/token.go | 310 + src/pkg/go/typechecker/Makefile | 14 + src/pkg/go/typechecker/scope.go | 69 + src/pkg/go/typechecker/testdata/test0.src | 94 + src/pkg/go/typechecker/testdata/test1.src | 13 + src/pkg/go/typechecker/testdata/test3.src | 41 + src/pkg/go/typechecker/testdata/test4.src | 11 + src/pkg/go/typechecker/type.go | 118 + src/pkg/go/typechecker/typechecker.go | 468 + src/pkg/go/typechecker/typechecker_test.go | 163 + src/pkg/go/typechecker/universe.go | 36 + src/pkg/go/types/Makefile | 16 + src/pkg/go/types/check.go | 226 + src/pkg/go/types/check_test.go | 215 + src/pkg/go/types/const.go | 332 + src/pkg/go/types/exportdata.go | 132 + src/pkg/go/types/gcimporter.go | 799 + src/pkg/go/types/gcimporter_test.go | 101 + src/pkg/go/types/testdata/exports.go | 84 + src/pkg/go/types/testdata/test0.src | 154 + src/pkg/go/types/types.go | 255 + src/pkg/go/types/universe.go | 108 + src/pkg/gob/Makefile | 25 + src/pkg/gob/codec_test.go | 1406 ++ src/pkg/gob/debug.go | 686 + src/pkg/gob/decode.go | 1275 ++ src/pkg/gob/decoder.go | 202 + src/pkg/gob/doc.go | 360 + src/pkg/gob/dump.go | 22 + src/pkg/gob/encode.go | 717 + src/pkg/gob/encoder.go | 243 + src/pkg/gob/encoder_test.go | 580 + src/pkg/gob/error.go | 42 + src/pkg/gob/gobencdec_test.go | 490 + src/pkg/gob/timing_test.go | 94 + src/pkg/gob/type.go | 786 + src/pkg/gob/type_test.go | 153 + src/pkg/hash/Makefile | 11 + src/pkg/hash/adler32/Makefile | 11 + src/pkg/hash/adler32/adler32.go | 88 + src/pkg/hash/adler32/adler32_test.go | 77 + src/pkg/hash/crc32/Makefile | 20 + src/pkg/hash/crc32/crc32.go | 139 + src/pkg/hash/crc32/crc32_amd64.go | 25 + src/pkg/hash/crc32/crc32_amd64.s | 62 + src/pkg/hash/crc32/crc32_generic.go | 12 + src/pkg/hash/crc32/crc32_test.go | 97 + src/pkg/hash/crc64/Makefile | 11 + src/pkg/hash/crc64/crc64.go | 97 + src/pkg/hash/crc64/crc64_test.go | 78 + src/pkg/hash/fnv/Makefile | 11 + src/pkg/hash/fnv/fnv.go | 131 + src/pkg/hash/fnv/fnv_test.go | 167 + src/pkg/hash/hash.go | 37 + src/pkg/hash/test_cases.txt | 31 + src/pkg/hash/test_gen.awk | 14 + src/pkg/html/Makefile | 17 + src/pkg/html/const.go | 90 + src/pkg/html/doc.go | 110 + src/pkg/html/entity.go | 2253 +++ src/pkg/html/entity_test.go | 29 + src/pkg/html/escape.go | 238 + src/pkg/html/node.go | 147 + src/pkg/html/parse.go | 800 + src/pkg/html/parse_test.go | 156 + src/pkg/html/testdata/webkit/README | 28 + src/pkg/html/testdata/webkit/adoption01.dat | 194 + src/pkg/html/testdata/webkit/adoption02.dat | 31 + src/pkg/html/testdata/webkit/comments01.dat | 135 + src/pkg/html/testdata/webkit/doctype01.dat | 370 + src/pkg/html/testdata/webkit/entities01.dat | 603 + src/pkg/html/testdata/webkit/entities02.dat | 249 + src/pkg/html/testdata/webkit/html5test-com.dat | 246 + src/pkg/html/testdata/webkit/inbody01.dat | 43 + src/pkg/html/testdata/webkit/isindex.dat | 40 + .../pending-spec-changes-plain-text-unsafe.dat | Bin 0 -> 115 bytes .../html/testdata/webkit/pending-spec-changes.dat | 28 + src/pkg/html/testdata/webkit/plain-text-unsafe.dat | 8 + src/pkg/html/testdata/webkit/scriptdata01.dat | 308 + .../html/testdata/webkit/scripted/adoption01.dat | 15 + src/pkg/html/testdata/webkit/scripted/webkit01.dat | 28 + src/pkg/html/testdata/webkit/tables01.dat | 197 + src/pkg/html/testdata/webkit/tests1.dat | 1952 ++ src/pkg/html/testdata/webkit/tests10.dat | 799 + src/pkg/html/testdata/webkit/tests11.dat | 482 + src/pkg/html/testdata/webkit/tests12.dat | 62 + src/pkg/html/testdata/webkit/tests14.dat | 74 + src/pkg/html/testdata/webkit/tests15.dat | 208 + src/pkg/html/testdata/webkit/tests16.dat | 2277 +++ src/pkg/html/testdata/webkit/tests17.dat | 153 + src/pkg/html/testdata/webkit/tests18.dat | 269 + src/pkg/html/testdata/webkit/tests19.dat | 1220 ++ src/pkg/html/testdata/webkit/tests2.dat | 763 + src/pkg/html/testdata/webkit/tests20.dat | 455 + src/pkg/html/testdata/webkit/tests21.dat | 221 + src/pkg/html/testdata/webkit/tests22.dat | 157 + src/pkg/html/testdata/webkit/tests23.dat | 155 + src/pkg/html/testdata/webkit/tests24.dat | 79 + src/pkg/html/testdata/webkit/tests25.dat | 219 + src/pkg/html/testdata/webkit/tests26.dat | 195 + src/pkg/html/testdata/webkit/tests3.dat | 305 + src/pkg/html/testdata/webkit/tests4.dat | 59 + src/pkg/html/testdata/webkit/tests5.dat | 191 + src/pkg/html/testdata/webkit/tests6.dat | 663 + src/pkg/html/testdata/webkit/tests7.dat | 390 + src/pkg/html/testdata/webkit/tests8.dat | 148 + src/pkg/html/testdata/webkit/tests9.dat | 457 + src/pkg/html/testdata/webkit/tests_innerHTML_1.dat | 733 + src/pkg/html/testdata/webkit/tricky01.dat | 261 + src/pkg/html/testdata/webkit/webkit01.dat | 609 + src/pkg/html/testdata/webkit/webkit02.dat | 104 + src/pkg/html/token.go | 575 + src/pkg/html/token_test.go | 340 + src/pkg/http/Makefile | 26 + src/pkg/http/cgi/Makefile | 12 + src/pkg/http/cgi/child.go | 191 + src/pkg/http/cgi/child_test.go | 93 + src/pkg/http/cgi/host.go | 323 + src/pkg/http/cgi/host_test.go | 449 + src/pkg/http/cgi/matryoshka_test.go | 74 + src/pkg/http/cgi/testdata/test.cgi | 96 + src/pkg/http/chunked.go | 77 + src/pkg/http/client.go | 290 + src/pkg/http/client_test.go | 292 + src/pkg/http/cookie.go | 268 + src/pkg/http/cookie_test.go | 201 + src/pkg/http/dump.go | 78 + src/pkg/http/export_test.go | 41 + src/pkg/http/fcgi/Makefile | 12 + src/pkg/http/fcgi/child.go | 258 + src/pkg/http/fcgi/fcgi.go | 271 + src/pkg/http/fcgi/fcgi_test.go | 114 + src/pkg/http/fs.go | 323 + src/pkg/http/fs_test.go | 308 + src/pkg/http/header.go | 78 + src/pkg/http/header_test.go | 81 + src/pkg/http/httptest/Makefile | 12 + src/pkg/http/httptest/recorder.go | 59 + src/pkg/http/httptest/server.go | 151 + src/pkg/http/lex.go | 144 + src/pkg/http/lex_test.go | 70 + src/pkg/http/persist.go | 420 + src/pkg/http/pprof/Makefile | 11 + src/pkg/http/pprof/pprof.go | 132 + src/pkg/http/proxy_test.go | 48 + src/pkg/http/range_test.go | 57 + src/pkg/http/readrequest_test.go | 183 + src/pkg/http/request.go | 745 + src/pkg/http/request_test.go | 336 + src/pkg/http/requestwrite_test.go | 370 + src/pkg/http/response.go | 214 + src/pkg/http/response_test.go | 397 + src/pkg/http/responsewrite_test.go | 109 + src/pkg/http/reverseproxy.go | 159 + src/pkg/http/reverseproxy_test.go | 66 + src/pkg/http/serve_test.go | 959 + src/pkg/http/server.go | 1183 ++ src/pkg/http/sniff.go | 214 + src/pkg/http/sniff_test.go | 80 + src/pkg/http/spdy/Makefile | 13 + src/pkg/http/spdy/read.go | 313 + src/pkg/http/spdy/spdy_test.go | 497 + src/pkg/http/spdy/types.go | 370 + src/pkg/http/spdy/write.go | 286 + src/pkg/http/status.go | 106 + src/pkg/http/testdata/file | 1 + src/pkg/http/testdata/index.html | 1 + src/pkg/http/testdata/style.css | 1 + src/pkg/http/transfer.go | 520 + src/pkg/http/transport.go | 717 + src/pkg/http/transport_test.go | 662 + src/pkg/http/triv.go | 149 + src/pkg/image/Makefile | 15 + src/pkg/image/bmp/Makefile | 11 + src/pkg/image/bmp/reader.go | 151 + src/pkg/image/color.go | 251 + src/pkg/image/decode_test.go | 123 + src/pkg/image/draw/Makefile | 11 + src/pkg/image/draw/bench_test.go | 206 + src/pkg/image/draw/clip_test.go | 193 + src/pkg/image/draw/draw.go | 493 + src/pkg/image/draw/draw_test.go | 354 + src/pkg/image/format.go | 100 + src/pkg/image/geom.go | 234 + src/pkg/image/gif/Makefile | 11 + src/pkg/image/gif/reader.go | 426 + src/pkg/image/image.go | 850 + src/pkg/image/image_test.go | 112 + src/pkg/image/jpeg/Makefile | 15 + src/pkg/image/jpeg/fdct.go | 190 + src/pkg/image/jpeg/huffman.go | 190 + src/pkg/image/jpeg/idct.go | 204 + src/pkg/image/jpeg/reader.go | 476 + src/pkg/image/jpeg/writer.go | 549 + src/pkg/image/jpeg/writer_test.go | 115 + src/pkg/image/names.go | 67 + src/pkg/image/png/Makefile | 12 + src/pkg/image/png/reader.go | 708 + src/pkg/image/png/reader_test.go | 241 + src/pkg/image/png/testdata/pngsuite/README | 21 + .../image/png/testdata/pngsuite/README.original | 85 + .../image/png/testdata/pngsuite/basn0g01-30.png | Bin 0 -> 162 bytes .../image/png/testdata/pngsuite/basn0g01-30.sng | 39 + src/pkg/image/png/testdata/pngsuite/basn0g01.png | Bin 0 -> 164 bytes src/pkg/image/png/testdata/pngsuite/basn0g01.sng | 41 + .../image/png/testdata/pngsuite/basn0g02-29.png | Bin 0 -> 110 bytes .../image/png/testdata/pngsuite/basn0g02-29.sng | 38 + src/pkg/image/png/testdata/pngsuite/basn0g02.png | Bin 0 -> 104 bytes src/pkg/image/png/testdata/pngsuite/basn0g02.sng | 41 + .../image/png/testdata/pngsuite/basn0g04-31.png | Bin 0 -> 153 bytes .../image/png/testdata/pngsuite/basn0g04-31.sng | 40 + src/pkg/image/png/testdata/pngsuite/basn0g04.png | Bin 0 -> 145 bytes src/pkg/image/png/testdata/pngsuite/basn0g04.sng | 41 + src/pkg/image/png/testdata/pngsuite/basn0g08.png | Bin 0 -> 138 bytes src/pkg/image/png/testdata/pngsuite/basn0g08.sng | 41 + src/pkg/image/png/testdata/pngsuite/basn0g16.png | Bin 0 -> 167 bytes src/pkg/image/png/testdata/pngsuite/basn0g16.sng | 41 + src/pkg/image/png/testdata/pngsuite/basn2c08.png | Bin 0 -> 145 bytes src/pkg/image/png/testdata/pngsuite/basn2c08.sng | 41 + src/pkg/image/png/testdata/pngsuite/basn2c16.png | Bin 0 -> 302 bytes src/pkg/image/png/testdata/pngsuite/basn2c16.sng | 41 + src/pkg/image/png/testdata/pngsuite/basn3p01.png | Bin 0 -> 112 bytes src/pkg/image/png/testdata/pngsuite/basn3p01.sng | 45 + src/pkg/image/png/testdata/pngsuite/basn3p02.png | Bin 0 -> 146 bytes src/pkg/image/png/testdata/pngsuite/basn3p02.sng | 47 + src/pkg/image/png/testdata/pngsuite/basn3p04.png | Bin 0 -> 216 bytes src/pkg/image/png/testdata/pngsuite/basn3p04.sng | 58 + .../image/png/testdata/pngsuite/basn3p08-trns.png | Bin 0 -> 1538 bytes .../image/png/testdata/pngsuite/basn3p08-trns.sng | 301 + src/pkg/image/png/testdata/pngsuite/basn3p08.png | Bin 0 -> 1286 bytes src/pkg/image/png/testdata/pngsuite/basn3p08.sng | 299 + src/pkg/image/png/testdata/pngsuite/basn4a08.png | Bin 0 -> 126 bytes src/pkg/image/png/testdata/pngsuite/basn4a08.sng | 41 + src/pkg/image/png/testdata/pngsuite/basn4a16.png | Bin 0 -> 2206 bytes src/pkg/image/png/testdata/pngsuite/basn4a16.sng | 41 + src/pkg/image/png/testdata/pngsuite/basn6a08.png | Bin 0 -> 184 bytes src/pkg/image/png/testdata/pngsuite/basn6a08.sng | 41 + src/pkg/image/png/testdata/pngsuite/basn6a16.png | Bin 0 -> 3435 bytes src/pkg/image/png/testdata/pngsuite/basn6a16.sng | 41 + src/pkg/image/png/writer.go | 468 + src/pkg/image/png/writer_test.go | 149 + src/pkg/image/testdata/video-001.5bpp.gif | Bin 0 -> 6214 bytes src/pkg/image/testdata/video-001.bmp | Bin 0 -> 46610 bytes src/pkg/image/testdata/video-001.gif | Bin 0 -> 13106 bytes src/pkg/image/testdata/video-001.interlaced.gif | Bin 0 -> 14142 bytes src/pkg/image/testdata/video-001.jpeg | Bin 0 -> 21459 bytes src/pkg/image/testdata/video-001.png | Bin 0 -> 29228 bytes src/pkg/image/testdata/video-001.tiff | Bin 0 -> 30810 bytes src/pkg/image/testdata/video-005.gray.jpeg | Bin 0 -> 5618 bytes src/pkg/image/testdata/video-005.gray.png | Bin 0 -> 14974 bytes src/pkg/image/tiff/Makefile | 13 + src/pkg/image/tiff/buffer.go | 57 + src/pkg/image/tiff/buffer_test.go | 36 + src/pkg/image/tiff/consts.go | 103 + src/pkg/image/tiff/reader.go | 423 + src/pkg/image/tiff/reader_test.go | 25 + src/pkg/image/tiff/testdata/no_rps.tiff | Bin 0 -> 1294 bytes src/pkg/image/ycbcr/Makefile | 11 + src/pkg/image/ycbcr/ycbcr.go | 183 + src/pkg/image/ycbcr/ycbcr_test.go | 33 + src/pkg/index/suffixarray/Makefile | 12 + src/pkg/index/suffixarray/qsufsort.go | 166 + src/pkg/index/suffixarray/suffixarray.go | 181 + src/pkg/index/suffixarray/suffixarray_test.go | 230 + src/pkg/io/Makefile | 13 + src/pkg/io/io.go | 439 + src/pkg/io/io_test.go | 179 + src/pkg/io/ioutil/Makefile | 12 + src/pkg/io/ioutil/ioutil.go | 130 + src/pkg/io/ioutil/ioutil_test.go | 91 + src/pkg/io/ioutil/tempfile.go | 91 + src/pkg/io/ioutil/tempfile_test.go | 54 + src/pkg/io/multi.go | 58 + src/pkg/io/multi_test.go | 89 + src/pkg/io/pipe.go | 182 + src/pkg/io/pipe_test.go | 271 + src/pkg/json/Makefile | 15 + src/pkg/json/decode.go | 920 + src/pkg/json/decode_test.go | 552 + src/pkg/json/encode.go | 428 + src/pkg/json/encode_test.go | 44 + src/pkg/json/indent.go | 116 + src/pkg/json/scanner.go | 624 + src/pkg/json/scanner_test.go | 278 + src/pkg/json/stream.go | 185 + src/pkg/json/stream_test.go | 122 + src/pkg/json/tagkey_test.go | 95 + src/pkg/log/Makefile | 11 + src/pkg/log/log.go | 327 + src/pkg/log/log_test.go | 119 + src/pkg/mail/Makefile | 11 + src/pkg/mail/message.go | 524 + src/pkg/mail/message_test.go | 278 + src/pkg/math/Makefile | 102 + src/pkg/math/acosh.go | 61 + src/pkg/math/all_test.go | 2737 +++ src/pkg/math/asin.go | 49 + src/pkg/math/asin_386.s | 28 + src/pkg/math/asin_decl.go | 8 + src/pkg/math/asinh.go | 71 + src/pkg/math/atan.go | 62 + src/pkg/math/atan2.go | 71 + src/pkg/math/atan2_386.s | 11 + src/pkg/math/atan2_decl.go | 7 + src/pkg/math/atan_386.s | 11 + src/pkg/math/atan_decl.go | 7 + src/pkg/math/atanh.go | 78 + src/pkg/math/bits.go | 59 + src/pkg/math/cbrt.go | 79 + src/pkg/math/const.go | 53 + src/pkg/math/copysign.go | 12 + src/pkg/math/erf.go | 339 + src/pkg/math/exp.go | 14 + src/pkg/math/exp2.go | 10 + src/pkg/math/exp2_386.s | 38 + src/pkg/math/exp2_decl.go | 7 + src/pkg/math/exp_386.s | 39 + src/pkg/math/exp_amd64.s | 112 + src/pkg/math/exp_decl.go | 7 + src/pkg/math/exp_port.go | 191 + src/pkg/math/exp_test.go | 10 + src/pkg/math/expm1.go | 237 + src/pkg/math/expm1_386.s | 55 + src/pkg/math/expm1_decl.go | 7 + src/pkg/math/fabs.go | 21 + src/pkg/math/fabs_386.s | 10 + src/pkg/math/fabs_amd64.s | 12 + src/pkg/math/fabs_decl.go | 7 + src/pkg/math/fdim.go | 29 + src/pkg/math/fdim_amd64.s | 26 + src/pkg/math/fdim_decl.go | 9 + src/pkg/math/floor.go | 52 + src/pkg/math/floor_386.s | 44 + src/pkg/math/floor_decl.go | 9 + src/pkg/math/fltasm_amd64.s | 67 + src/pkg/math/fmod.go | 47 + src/pkg/math/fmod_386.s | 15 + src/pkg/math/fmod_decl.go | 7 + src/pkg/math/frexp.go | 33 + src/pkg/math/frexp_386.s | 23 + src/pkg/math/frexp_decl.go | 7 + src/pkg/math/gamma.go | 188 + src/pkg/math/hypot.go | 41 + src/pkg/math/hypot_386.s | 57 + src/pkg/math/hypot_amd64.s | 50 + src/pkg/math/hypot_decl.go | 7 + src/pkg/math/hypot_port.go | 63 + src/pkg/math/hypot_test.go | 9 + src/pkg/math/j0.go | 433 + src/pkg/math/j1.go | 426 + src/pkg/math/jn.go | 310 + src/pkg/math/ldexp.go | 45 + src/pkg/math/ldexp_386.s | 12 + src/pkg/math/ldexp_decl.go | 7 + src/pkg/math/lgamma.go | 350 + src/pkg/math/log.go | 123 + src/pkg/math/log10.go | 13 + src/pkg/math/log10_386.s | 19 + src/pkg/math/log10_decl.go | 8 + src/pkg/math/log1p.go | 199 + src/pkg/math/log1p_386.s | 25 + src/pkg/math/log1p_decl.go | 7 + src/pkg/math/log_386.s | 11 + src/pkg/math/log_amd64.s | 109 + src/pkg/math/log_decl.go | 7 + src/pkg/math/logb.go | 54 + src/pkg/math/modf.go | 33 + src/pkg/math/modf_386.s | 19 + src/pkg/math/modf_decl.go | 7 + src/pkg/math/nextafter.go | 29 + src/pkg/math/pow.go | 139 + src/pkg/math/pow10.go | 30 + src/pkg/math/remainder.go | 85 + src/pkg/math/remainder_386.s | 15 + src/pkg/math/remainder_decl.go | 7 + src/pkg/math/signbit.go | 10 + src/pkg/math/sin.go | 65 + src/pkg/math/sin_386.s | 45 + src/pkg/math/sin_decl.go | 8 + src/pkg/math/sincos.go | 13 + src/pkg/math/sincos_386.s | 26 + src/pkg/math/sincos_amd64.s | 143 + src/pkg/math/sincos_decl.go | 7 + src/pkg/math/sinh.go | 67 + src/pkg/math/sqrt.go | 14 + src/pkg/math/sqrt_386.s | 10 + src/pkg/math/sqrt_amd64.s | 9 + src/pkg/math/sqrt_arm.s | 10 + src/pkg/math/sqrt_decl.go | 7 + src/pkg/math/sqrt_port.go | 147 + src/pkg/math/sqrt_test.go | 9 + src/pkg/math/tan.go | 64 + src/pkg/math/tan_386.s | 26 + src/pkg/math/tan_decl.go | 7 + src/pkg/math/tanh.go | 27 + src/pkg/math/unsafe.go | 21 + src/pkg/mime/Makefile | 13 + src/pkg/mime/grammar.go | 36 + src/pkg/mime/mediatype.go | 295 + src/pkg/mime/mediatype_test.go | 248 + src/pkg/mime/mime_test.go | 27 + src/pkg/mime/multipart/Makefile | 13 + src/pkg/mime/multipart/formdata.go | 166 + src/pkg/mime/multipart/formdata_test.go | 89 + src/pkg/mime/multipart/multipart.go | 268 + src/pkg/mime/multipart/multipart_test.go | 380 + src/pkg/mime/multipart/writer.go | 157 + src/pkg/mime/multipart/writer_test.go | 78 + src/pkg/mime/test.types | 8 + src/pkg/mime/type.go | 104 + src/pkg/net/Makefile | 157 + src/pkg/net/cgo_bsd.go | 14 + src/pkg/net/cgo_linux.go | 14 + src/pkg/net/cgo_stub.go | 25 + src/pkg/net/cgo_unix.go | 148 + src/pkg/net/dial.go | 153 + src/pkg/net/dialgoogle_test.go | 169 + src/pkg/net/dict/Makefile | 7 + src/pkg/net/dict/dict.go | 211 + src/pkg/net/dnsclient.go | 247 + src/pkg/net/dnsclient_unix.go | 261 + src/pkg/net/dnsconfig.go | 119 + src/pkg/net/dnsmsg.go | 779 + src/pkg/net/dnsmsg_test.go | 100 + src/pkg/net/dnsname_test.go | 65 + src/pkg/net/fd.go | 645 + src/pkg/net/fd_darwin.go | 120 + src/pkg/net/fd_freebsd.go | 116 + src/pkg/net/fd_linux.go | 182 + src/pkg/net/fd_openbsd.go | 116 + src/pkg/net/fd_windows.go | 532 + src/pkg/net/file.go | 119 + src/pkg/net/file_plan9.go | 33 + src/pkg/net/file_test.go | 133 + src/pkg/net/file_windows.go | 25 + src/pkg/net/hosts.go | 86 + src/pkg/net/hosts_test.go | 68 + src/pkg/net/hosts_testdata | 12 + src/pkg/net/interface.go | 132 + src/pkg/net/interface_bsd.go | 171 + src/pkg/net/interface_darwin.go | 80 + src/pkg/net/interface_freebsd.go | 80 + src/pkg/net/interface_linux.go | 262 + src/pkg/net/interface_openbsd.go | 16 + src/pkg/net/interface_stub.go | 30 + src/pkg/net/interface_test.go | 73 + src/pkg/net/interface_windows.go | 158 + src/pkg/net/ip.go | 614 + src/pkg/net/ip_test.go | 215 + src/pkg/net/ipraw_test.go | 120 + src/pkg/net/iprawsock.go | 69 + src/pkg/net/iprawsock_plan9.go | 99 + src/pkg/net/iprawsock_posix.go | 305 + src/pkg/net/ipsock.go | 158 + src/pkg/net/ipsock_plan9.go | 305 + src/pkg/net/ipsock_posix.go | 174 + src/pkg/net/lookup_plan9.go | 226 + src/pkg/net/lookup_test.go | 57 + src/pkg/net/lookup_unix.go | 111 + src/pkg/net/lookup_windows.go | 130 + src/pkg/net/multicast_test.go | 73 + src/pkg/net/net.go | 188 + src/pkg/net/net_test.go | 121 + src/pkg/net/newpollserver.go | 43 + src/pkg/net/parse.go | 204 + src/pkg/net/parse_test.go | 50 + src/pkg/net/pipe.go | 62 + src/pkg/net/pipe_test.go | 57 + src/pkg/net/port.go | 70 + src/pkg/net/port_test.go | 53 + src/pkg/net/sendfile_linux.go | 84 + src/pkg/net/sendfile_stub.go | 14 + src/pkg/net/sendfile_windows.go | 68 + src/pkg/net/server_test.go | 243 + src/pkg/net/sock.go | 166 + src/pkg/net/sock_bsd.go | 31 + src/pkg/net/sock_linux.go | 25 + src/pkg/net/sock_windows.go | 25 + src/pkg/net/tcpsock.go | 40 + src/pkg/net/tcpsock_plan9.go | 63 + src/pkg/net/tcpsock_posix.go | 283 + src/pkg/net/textproto/Makefile | 15 + src/pkg/net/textproto/header.go | 43 + src/pkg/net/textproto/pipeline.go | 117 + src/pkg/net/textproto/reader.go | 514 + src/pkg/net/textproto/reader_test.go | 140 + src/pkg/net/textproto/textproto.go | 121 + src/pkg/net/textproto/writer.go | 119 + src/pkg/net/textproto/writer_test.go | 35 + src/pkg/net/timeout_test.go | 57 + src/pkg/net/udpsock.go | 40 + src/pkg/net/udpsock_plan9.go | 187 + src/pkg/net/udpsock_posix.go | 294 + src/pkg/net/unixsock.go | 50 + src/pkg/net/unixsock_plan9.go | 105 + src/pkg/net/unixsock_posix.go | 418 + src/pkg/netchan/Makefile | 13 + src/pkg/netchan/common.go | 336 + src/pkg/netchan/export.go | 400 + src/pkg/netchan/import.go | 287 + src/pkg/netchan/netchan_test.go | 435 + src/pkg/old/template/Makefile | 14 + src/pkg/old/template/doc.go | 91 + src/pkg/old/template/execute.go | 346 + src/pkg/old/template/format.go | 77 + src/pkg/old/template/parse.go | 743 + src/pkg/old/template/template_test.go | 804 + src/pkg/os/Makefile | 100 + src/pkg/os/dir_plan9.go | 300 + src/pkg/os/dir_unix.go | 67 + src/pkg/os/dir_windows.go | 14 + src/pkg/os/env.go | 75 + src/pkg/os/env_plan9.go | 96 + src/pkg/os/env_test.go | 59 + src/pkg/os/env_unix.go | 109 + src/pkg/os/env_windows.go | 127 + src/pkg/os/error.go | 31 + src/pkg/os/error_plan9.go | 61 + src/pkg/os/error_posix.go | 90 + src/pkg/os/exec.go | 58 + src/pkg/os/exec_plan9.go | 153 + src/pkg/os/exec_posix.go | 147 + src/pkg/os/exec_unix.go | 77 + src/pkg/os/exec_windows.go | 66 + src/pkg/os/file.go | 211 + src/pkg/os/file_plan9.go | 331 + src/pkg/os/file_posix.go | 226 + src/pkg/os/file_unix.go | 208 + src/pkg/os/file_windows.go | 317 + src/pkg/os/getwd.go | 92 + src/pkg/os/inotify/Makefile | 14 + src/pkg/os/inotify/inotify_linux.go | 288 + src/pkg/os/inotify/inotify_linux_test.go | 96 + src/pkg/os/mkunixsignals.sh | 24 + src/pkg/os/os_test.go | 1058 + src/pkg/os/path.go | 118 + src/pkg/os/path_plan9.go | 15 + src/pkg/os/path_test.go | 208 + src/pkg/os/path_unix.go | 15 + src/pkg/os/path_windows.go | 16 + src/pkg/os/proc.go | 34 + src/pkg/os/signal/Makefile | 11 + src/pkg/os/signal/signal.go | 33 + src/pkg/os/signal/signal_test.go | 20 + src/pkg/os/stat_darwin.go | 32 + src/pkg/os/stat_freebsd.go | 32 + src/pkg/os/stat_linux.go | 32 + src/pkg/os/stat_openbsd.go | 32 + src/pkg/os/stat_plan9.go | 90 + src/pkg/os/stat_windows.go | 46 + src/pkg/os/str.go | 20 + src/pkg/os/sys_bsd.go | 19 + src/pkg/os/sys_linux.go | 27 + src/pkg/os/sys_plan9.go | 26 + src/pkg/os/sys_windows.go | 15 + src/pkg/os/time.go | 19 + src/pkg/os/types.go | 56 + src/pkg/os/user/Makefile | 26 + src/pkg/os/user/lookup_stubs.go | 19 + src/pkg/os/user/lookup_unix.go | 108 + src/pkg/os/user/user.go | 37 + src/pkg/os/user/user_test.go | 61 + src/pkg/patch/Makefile | 14 + src/pkg/patch/apply.go | 54 + src/pkg/patch/git.go | 121 + src/pkg/patch/patch.go | 322 + src/pkg/patch/patch_test.go | 390 + src/pkg/patch/textdiff.go | 175 + src/pkg/path/Makefile | 14 + src/pkg/path/filepath/Makefile | 32 + src/pkg/path/filepath/match.go | 294 + src/pkg/path/filepath/match_test.go | 134 + src/pkg/path/filepath/path.go | 358 + src/pkg/path/filepath/path_plan9.go | 23 + src/pkg/path/filepath/path_test.go | 576 + src/pkg/path/filepath/path_unix.go | 23 + src/pkg/path/filepath/path_windows.go | 46 + src/pkg/path/match.go | 207 + src/pkg/path/match_test.go | 77 + src/pkg/path/path.go | 162 + src/pkg/path/path_test.go | 196 + src/pkg/rand/Makefile | 15 + src/pkg/rand/exp.go | 223 + src/pkg/rand/normal.go | 158 + src/pkg/rand/rand.go | 179 + src/pkg/rand/rand_test.go | 350 + src/pkg/rand/rng.go | 246 + src/pkg/rand/zipf.go | 73 + src/pkg/reflect/Makefile | 13 + src/pkg/reflect/all_test.go | 1564 ++ src/pkg/reflect/deepequal.go | 126 + src/pkg/reflect/set_test.go | 211 + src/pkg/reflect/tostring_test.go | 96 + src/pkg/reflect/type.go | 1165 ++ src/pkg/reflect/value.go | 1736 ++ src/pkg/regexp/Makefile | 11 + src/pkg/regexp/all_test.go | 426 + src/pkg/regexp/find_test.go | 472 + src/pkg/regexp/regexp.go | 1488 ++ src/pkg/rpc/Makefile | 13 + src/pkg/rpc/client.go | 286 + src/pkg/rpc/debug.go | 90 + src/pkg/rpc/jsonrpc/Makefile | 12 + src/pkg/rpc/jsonrpc/all_test.go | 156 + src/pkg/rpc/jsonrpc/client.go | 124 + src/pkg/rpc/jsonrpc/server.go | 136 + src/pkg/rpc/server.go | 637 + src/pkg/rpc/server_test.go | 501 + src/pkg/runtime/386/arch.h | 3 + src/pkg/runtime/386/asm.s | 549 + src/pkg/runtime/386/atomic.c | 19 + src/pkg/runtime/386/closure.c | 105 + src/pkg/runtime/386/memmove.s | 86 + src/pkg/runtime/386/vlop.s | 48 + src/pkg/runtime/386/vlrt.c | 815 + src/pkg/runtime/Makefile | 168 + src/pkg/runtime/amd64/arch.h | 3 + src/pkg/runtime/amd64/asm.s | 577 + src/pkg/runtime/amd64/atomic.c | 19 + src/pkg/runtime/amd64/closure.c | 123 + src/pkg/runtime/amd64/memmove.s | 88 + src/pkg/runtime/amd64/traceback.c | 295 + src/pkg/runtime/append_test.go | 52 + src/pkg/runtime/arm/arch.h | 3 + src/pkg/runtime/arm/asm.s | 316 + src/pkg/runtime/arm/atomic.c | 83 + src/pkg/runtime/arm/closure.c | 129 + src/pkg/runtime/arm/memmove.s | 255 + src/pkg/runtime/arm/memset.s | 94 + src/pkg/runtime/arm/softfloat.c | 525 + src/pkg/runtime/arm/traceback.c | 213 + src/pkg/runtime/arm/vlop.s | 190 + src/pkg/runtime/arm/vlrt.c | 816 + src/pkg/runtime/cgo/386.S | 67 + src/pkg/runtime/cgo/Makefile | 60 + src/pkg/runtime/cgo/amd64.S | 73 + src/pkg/runtime/cgo/arm.S | 1 + src/pkg/runtime/cgo/callbacks.c | 73 + src/pkg/runtime/cgo/cgo.go | 17 + src/pkg/runtime/cgo/darwin_386.c | 149 + src/pkg/runtime/cgo/darwin_amd64.c | 119 + src/pkg/runtime/cgo/freebsd.c | 13 + src/pkg/runtime/cgo/freebsd_386.c | 64 + src/pkg/runtime/cgo/freebsd_amd64.c | 63 + src/pkg/runtime/cgo/iscgo.c | 14 + src/pkg/runtime/cgo/libcgo.h | 60 + src/pkg/runtime/cgo/linux_386.c | 73 + src/pkg/runtime/cgo/linux_amd64.c | 63 + src/pkg/runtime/cgo/linux_arm.c | 19 + src/pkg/runtime/cgo/setenv.c | 16 + src/pkg/runtime/cgo/util.c | 51 + src/pkg/runtime/cgo/windows_386.c | 62 + src/pkg/runtime/cgo/windows_amd64.c | 60 + src/pkg/runtime/cgocall.c | 249 + src/pkg/runtime/cgocall.h | 12 + src/pkg/runtime/chan.c | 1161 ++ src/pkg/runtime/chan_test.go | 322 + src/pkg/runtime/closure_test.go | 53 + src/pkg/runtime/complex.c | 60 + src/pkg/runtime/cpuprof.c | 425 + src/pkg/runtime/darwin/386/defs.h | 289 + src/pkg/runtime/darwin/386/rt0.s | 8 + src/pkg/runtime/darwin/386/signal.c | 194 + src/pkg/runtime/darwin/386/sys.s | 311 + src/pkg/runtime/darwin/amd64/defs.h | 305 + src/pkg/runtime/darwin/amd64/rt0.s | 10 + src/pkg/runtime/darwin/amd64/signal.c | 204 + src/pkg/runtime/darwin/amd64/sys.s | 295 + src/pkg/runtime/darwin/defs.c | 159 + src/pkg/runtime/darwin/mem.c | 55 + src/pkg/runtime/darwin/os.h | 31 + src/pkg/runtime/darwin/signals.h | 51 + src/pkg/runtime/darwin/thread.c | 484 + src/pkg/runtime/debug.go | 115 + src/pkg/runtime/debug/Makefile | 11 + src/pkg/runtime/debug/stack.go | 90 + src/pkg/runtime/debug/stack_test.go | 55 + src/pkg/runtime/error.go | 138 + src/pkg/runtime/export_test.go | 23 + src/pkg/runtime/extern.go | 192 + src/pkg/runtime/float.c | 173 + src/pkg/runtime/freebsd/386/defs.h | 187 + src/pkg/runtime/freebsd/386/rt0.s | 9 + src/pkg/runtime/freebsd/386/signal.c | 193 + src/pkg/runtime/freebsd/386/sys.s | 239 + src/pkg/runtime/freebsd/amd64/defs.h | 198 + src/pkg/runtime/freebsd/amd64/rt0.s | 9 + src/pkg/runtime/freebsd/amd64/signal.c | 201 + src/pkg/runtime/freebsd/amd64/sys.s | 182 + src/pkg/runtime/freebsd/defs.c | 108 + src/pkg/runtime/freebsd/mem.c | 74 + src/pkg/runtime/freebsd/os.h | 12 + src/pkg/runtime/freebsd/signals.h | 52 + src/pkg/runtime/freebsd/thread.c | 201 + src/pkg/runtime/goc2c.c | 727 + src/pkg/runtime/hashmap.c | 1180 ++ src/pkg/runtime/hashmap.h | 159 + src/pkg/runtime/iface.c | 788 + src/pkg/runtime/linux/386/defs.h | 191 + src/pkg/runtime/linux/386/rt0.s | 9 + src/pkg/runtime/linux/386/signal.c | 184 + src/pkg/runtime/linux/386/sys.s | 344 + src/pkg/runtime/linux/amd64/defs.h | 236 + src/pkg/runtime/linux/amd64/rt0.s | 10 + src/pkg/runtime/linux/amd64/signal.c | 194 + src/pkg/runtime/linux/amd64/sys.s | 252 + src/pkg/runtime/linux/arm/defs.h | 149 + src/pkg/runtime/linux/arm/rt0.s | 6 + src/pkg/runtime/linux/arm/signal.c | 189 + src/pkg/runtime/linux/arm/sys.s | 319 + src/pkg/runtime/linux/defs.c | 95 + src/pkg/runtime/linux/defs1.c | 24 + src/pkg/runtime/linux/defs2.c | 120 + src/pkg/runtime/linux/defs_arm.c | 122 + src/pkg/runtime/linux/mem.c | 113 + src/pkg/runtime/linux/os.h | 19 + src/pkg/runtime/linux/signals.h | 51 + src/pkg/runtime/linux/thread.c | 320 + src/pkg/runtime/malloc.goc | 482 + src/pkg/runtime/malloc.h | 422 + src/pkg/runtime/mcache.c | 133 + src/pkg/runtime/mcentral.c | 200 + src/pkg/runtime/mem.go | 74 + src/pkg/runtime/mfinal.c | 181 + src/pkg/runtime/mfixalloc.c | 62 + src/pkg/runtime/mgc0.c | 910 + src/pkg/runtime/mheap.c | 374 + src/pkg/runtime/mkasmh.sh | 112 + src/pkg/runtime/mkgodefs.sh | 39 + src/pkg/runtime/mkversion.c | 15 + src/pkg/runtime/mprof.goc | 274 + src/pkg/runtime/msize.c | 168 + src/pkg/runtime/openbsd/amd64/defs.h | 149 + src/pkg/runtime/openbsd/amd64/rt0.s | 8 + src/pkg/runtime/openbsd/amd64/signal.c | 199 + src/pkg/runtime/openbsd/amd64/sys.s | 221 + src/pkg/runtime/openbsd/defs.c | 103 + src/pkg/runtime/openbsd/mem.c | 74 + src/pkg/runtime/openbsd/os.h | 12 + src/pkg/runtime/openbsd/signals.h | 52 + src/pkg/runtime/openbsd/thread.c | 156 + src/pkg/runtime/plan9/386/defs.h | 2 + src/pkg/runtime/plan9/386/rt0.s | 32 + src/pkg/runtime/plan9/386/signal.c | 24 + src/pkg/runtime/plan9/386/sys.s | 82 + src/pkg/runtime/plan9/mem.c | 67 + src/pkg/runtime/plan9/os.h | 57 + src/pkg/runtime/plan9/signals.h | 1 + src/pkg/runtime/plan9/thread.c | 174 + src/pkg/runtime/pprof/Makefile | 11 + src/pkg/runtime/pprof/pprof.go | 176 + src/pkg/runtime/pprof/pprof_test.go | 77 + src/pkg/runtime/print.c | 351 + src/pkg/runtime/proc.c | 1568 ++ src/pkg/runtime/proc.p | 526 + src/pkg/runtime/proc_test.go | 125 + src/pkg/runtime/rune.c | 224 + src/pkg/runtime/runtime-gdb.py | 400 + src/pkg/runtime/runtime.c | 728 + src/pkg/runtime/runtime.h | 635 + src/pkg/runtime/runtime1.goc | 10 + src/pkg/runtime/sema.goc | 180 + src/pkg/runtime/sema_test.go | 100 + src/pkg/runtime/sig.go | 16 + src/pkg/runtime/sigqueue.goc | 99 + src/pkg/runtime/slice.c | 330 + src/pkg/runtime/softfloat64.go | 498 + src/pkg/runtime/softfloat64_test.go | 198 + src/pkg/runtime/stack.h | 97 + src/pkg/runtime/string.goc | 360 + src/pkg/runtime/symtab.c | 466 + src/pkg/runtime/symtab_test.go | 47 + src/pkg/runtime/type.go | 208 + src/pkg/runtime/type.h | 131 + src/pkg/runtime/windows/386/defs.h | 81 + src/pkg/runtime/windows/386/rt0.s | 14 + src/pkg/runtime/windows/386/signal.c | 98 + src/pkg/runtime/windows/386/sys.s | 256 + src/pkg/runtime/windows/amd64/defs.h | 40 + src/pkg/runtime/windows/amd64/rt0.s | 13 + src/pkg/runtime/windows/amd64/signal.c | 20 + src/pkg/runtime/windows/amd64/sys.s | 130 + src/pkg/runtime/windows/defs.c | 37 + src/pkg/runtime/windows/mem.c | 70 + src/pkg/runtime/windows/os.h | 30 + src/pkg/runtime/windows/signals.h | 3 + src/pkg/runtime/windows/syscall.goc | 67 + src/pkg/runtime/windows/thread.c | 432 + src/pkg/scanner/Makefile | 11 + src/pkg/scanner/scanner.go | 660 + src/pkg/scanner/scanner_test.go | 566 + src/pkg/smtp/Makefile | 12 + src/pkg/smtp/auth.go | 69 + src/pkg/smtp/smtp.go | 294 + src/pkg/smtp/smtp_test.go | 182 + src/pkg/sort/Makefile | 12 + src/pkg/sort/search.go | 104 + src/pkg/sort/search_test.go | 132 + src/pkg/sort/sort.go | 202 + src/pkg/sort/sort_test.go | 282 + src/pkg/strconv/Makefile | 17 + src/pkg/strconv/atob.go | 28 + src/pkg/strconv/atob_test.go | 58 + src/pkg/strconv/atof.go | 415 + src/pkg/strconv/atof_test.go | 191 + src/pkg/strconv/atoi.go | 202 + src/pkg/strconv/atoi_test.go | 303 + src/pkg/strconv/decimal.go | 371 + src/pkg/strconv/decimal_test.go | 117 + src/pkg/strconv/fp_test.go | 149 + src/pkg/strconv/ftoa.go | 405 + src/pkg/strconv/ftoa_test.go | 150 + src/pkg/strconv/internal_test.go | 15 + src/pkg/strconv/itoa.go | 57 + src/pkg/strconv/itoa_test.go | 174 + src/pkg/strconv/quote.go | 310 + src/pkg/strconv/quote_test.go | 213 + src/pkg/strconv/testfp.txt | 181 + src/pkg/strings/Makefile | 12 + src/pkg/strings/reader.go | 91 + src/pkg/strings/strings.go | 585 + src/pkg/strings/strings_test.go | 927 + src/pkg/sync/Makefile | 15 + src/pkg/sync/atomic/Makefile | 18 + src/pkg/sync/atomic/asm_386.s | 96 + src/pkg/sync/atomic/asm_amd64.s | 69 + src/pkg/sync/atomic/asm_arm.s | 122 + src/pkg/sync/atomic/asm_linux_arm.s | 98 + src/pkg/sync/atomic/atomic_test.go | 641 + src/pkg/sync/atomic/doc.go | 68 + src/pkg/sync/cond.go | 113 + src/pkg/sync/cond_test.go | 126 + src/pkg/sync/mutex.go | 95 + src/pkg/sync/mutex_test.go | 166 + src/pkg/sync/once.go | 43 + src/pkg/sync/once_test.go | 62 + src/pkg/sync/rwmutex.go | 93 + src/pkg/sync/rwmutex_test.go | 237 + src/pkg/sync/waitgroup.go | 97 + src/pkg/sync/waitgroup_test.go | 165 + src/pkg/syscall/Makefile | 58 + src/pkg/syscall/asm_darwin_386.s | 137 + src/pkg/syscall/asm_darwin_amd64.s | 101 + src/pkg/syscall/asm_freebsd_386.s | 137 + src/pkg/syscall/asm_freebsd_amd64.s | 97 + src/pkg/syscall/asm_linux_386.s | 181 + src/pkg/syscall/asm_linux_amd64.s | 129 + src/pkg/syscall/asm_linux_arm.s | 153 + src/pkg/syscall/asm_plan9_386.s | 151 + src/pkg/syscall/asm_windows_386.s | 7 + src/pkg/syscall/asm_windows_amd64.s | 7 + src/pkg/syscall/bpf_bsd.go | 167 + src/pkg/syscall/exec_plan9.go | 525 + src/pkg/syscall/exec_unix.go | 425 + src/pkg/syscall/exec_windows.go | 327 + src/pkg/syscall/lsf_linux.go | 78 + src/pkg/syscall/mkall.sh | 186 + src/pkg/syscall/mkerrors.sh | 235 + src/pkg/syscall/mkerrors_windows.sh | 202 + src/pkg/syscall/mksyscall.pl | 234 + src/pkg/syscall/mksyscall_windows.pl | 294 + src/pkg/syscall/mksysnum_darwin.pl | 32 + src/pkg/syscall/mksysnum_freebsd.pl | 43 + src/pkg/syscall/mksysnum_linux.pl | 38 + src/pkg/syscall/mksysnum_plan9.sh | 25 + src/pkg/syscall/netlink_linux.go | 227 + src/pkg/syscall/route_bsd.go | 152 + src/pkg/syscall/route_darwin.go | 77 + src/pkg/syscall/route_freebsd.go | 77 + src/pkg/syscall/sockcmsg_linux.go | 38 + src/pkg/syscall/sockcmsg_unix.go | 113 + src/pkg/syscall/str.go | 20 + src/pkg/syscall/syscall.go | 30 + src/pkg/syscall/syscall_386.go | 7 + src/pkg/syscall/syscall_amd64.go | 7 + src/pkg/syscall/syscall_arm.go | 7 + src/pkg/syscall/syscall_bsd.go | 627 + src/pkg/syscall/syscall_darwin.go | 377 + src/pkg/syscall/syscall_darwin_386.go | 55 + src/pkg/syscall/syscall_darwin_amd64.go | 53 + src/pkg/syscall/syscall_freebsd.go | 366 + src/pkg/syscall/syscall_freebsd_386.go | 44 + src/pkg/syscall/syscall_freebsd_amd64.go | 42 + src/pkg/syscall/syscall_linux.go | 999 + src/pkg/syscall/syscall_linux_386.go | 218 + src/pkg/syscall/syscall_linux_amd64.go | 92 + src/pkg/syscall/syscall_linux_arm.go | 138 + src/pkg/syscall/syscall_plan9.go | 357 + src/pkg/syscall/syscall_plan9_386.go | 5 + src/pkg/syscall/syscall_unix.go | 90 + src/pkg/syscall/syscall_windows.go | 770 + src/pkg/syscall/syscall_windows_386.go | 5 + src/pkg/syscall/syscall_windows_amd64.go | 5 + src/pkg/syscall/types_darwin.c | 180 + src/pkg/syscall/types_freebsd.c | 190 + src/pkg/syscall/types_linux.c | 266 + src/pkg/syscall/types_plan9.c | 113 + src/pkg/syscall/zerrors_darwin_386.go | 1195 ++ src/pkg/syscall/zerrors_darwin_amd64.go | 1195 ++ src/pkg/syscall/zerrors_freebsd_386.go | 1405 ++ src/pkg/syscall/zerrors_freebsd_amd64.go | 1405 ++ src/pkg/syscall/zerrors_linux_386.go | 1329 ++ src/pkg/syscall/zerrors_linux_amd64.go | 1330 ++ src/pkg/syscall/zerrors_linux_arm.go | 1319 ++ src/pkg/syscall/zerrors_plan9_386.go | 23 + src/pkg/syscall/zerrors_windows.go | 283 + src/pkg/syscall/zerrors_windows_386.go | 5 + src/pkg/syscall/zerrors_windows_amd64.go | 5 + src/pkg/syscall/zsyscall_darwin_386.go | 977 + src/pkg/syscall/zsyscall_darwin_amd64.go | 977 + src/pkg/syscall/zsyscall_freebsd_386.go | 959 + src/pkg/syscall/zsyscall_freebsd_amd64.go | 959 + src/pkg/syscall/zsyscall_linux_386.go | 1128 ++ src/pkg/syscall/zsyscall_linux_amd64.go | 1272 ++ src/pkg/syscall/zsyscall_linux_arm.go | 1210 ++ src/pkg/syscall/zsyscall_plan9_386.go | 267 + src/pkg/syscall/zsyscall_windows_386.go | 1323 ++ src/pkg/syscall/zsyscall_windows_amd64.go | 1323 ++ src/pkg/syscall/zsysnum_darwin_386.go | 355 + src/pkg/syscall/zsysnum_darwin_amd64.go | 355 + src/pkg/syscall/zsysnum_freebsd_386.go | 321 + src/pkg/syscall/zsysnum_freebsd_amd64.go | 321 + src/pkg/syscall/zsysnum_linux_386.go | 341 + src/pkg/syscall/zsysnum_linux_amd64.go | 306 + src/pkg/syscall/zsysnum_linux_arm.go | 336 + src/pkg/syscall/zsysnum_plan9_386.go | 47 + src/pkg/syscall/zsysnum_windows_386.go | 3 + src/pkg/syscall/zsysnum_windows_amd64.go | 3 + src/pkg/syscall/ztypes_darwin_386.go | 406 + src/pkg/syscall/ztypes_darwin_amd64.go | 417 + src/pkg/syscall/ztypes_freebsd_386.go | 401 + src/pkg/syscall/ztypes_freebsd_amd64.go | 405 + src/pkg/syscall/ztypes_linux_386.go | 486 + src/pkg/syscall/ztypes_linux_amd64.go | 502 + src/pkg/syscall/ztypes_linux_arm.go | 476 + src/pkg/syscall/ztypes_plan9_386.go | 75 + src/pkg/syscall/ztypes_windows.go | 603 + src/pkg/syscall/ztypes_windows_386.go | 22 + src/pkg/syscall/ztypes_windows_amd64.go | 22 + src/pkg/syslog/Makefile | 12 + src/pkg/syslog/syslog.go | 149 + src/pkg/syslog/syslog_test.go | 113 + src/pkg/syslog/syslog_unix.go | 31 + src/pkg/tabwriter/Makefile | 11 + src/pkg/tabwriter/tabwriter.go | 561 + src/pkg/tabwriter/tabwriter_test.go | 615 + src/pkg/template/Makefile | 15 + src/pkg/template/doc.go | 313 + src/pkg/template/exec.go | 674 + src/pkg/template/exec_test.go | 613 + src/pkg/template/funcs.go | 368 + src/pkg/template/helper.go | 238 + src/pkg/template/parse.go | 85 + src/pkg/template/parse/Makefile | 14 + src/pkg/template/parse/lex.go | 472 + src/pkg/template/parse/lex_test.go | 223 + src/pkg/template/parse/node.go | 470 + src/pkg/template/parse/parse.go | 436 + src/pkg/template/parse/parse_test.go | 259 + src/pkg/template/parse/set.go | 50 + src/pkg/template/set.go | 108 + src/pkg/template/set_test.go | 239 + src/pkg/template/testdata/file1.tmpl | 2 + src/pkg/template/testdata/file2.tmpl | 2 + src/pkg/template/testdata/tmpl1.tmpl | 1 + src/pkg/template/testdata/tmpl2.tmpl | 1 + src/pkg/testing/Makefile | 12 + src/pkg/testing/benchmark.go | 236 + src/pkg/testing/iotest/Makefile | 13 + src/pkg/testing/iotest/logger.go | 55 + src/pkg/testing/iotest/reader.go | 86 + src/pkg/testing/iotest/writer.go | 38 + src/pkg/testing/quick/Makefile | 11 + src/pkg/testing/quick/quick.go | 362 + src/pkg/testing/quick/quick_test.go | 147 + src/pkg/testing/script/Makefile | 11 + src/pkg/testing/script/script.go | 359 + src/pkg/testing/script/script_test.go | 75 + src/pkg/testing/testing.go | 298 + src/pkg/time/Makefile | 46 + src/pkg/time/format.go | 734 + src/pkg/time/sleep.go | 177 + src/pkg/time/sleep_test.go | 189 + src/pkg/time/sys.go | 51 + src/pkg/time/sys_plan9.go | 18 + src/pkg/time/sys_posix.go | 18 + src/pkg/time/tick.go | 177 + src/pkg/time/tick_test.go | 58 + src/pkg/time/time.go | 230 + src/pkg/time/time_test.go | 506 + src/pkg/time/zoneinfo_plan9.go | 59 + src/pkg/time/zoneinfo_posix.go | 62 + src/pkg/time/zoneinfo_unix.go | 213 + src/pkg/time/zoneinfo_windows.go | 192 + src/pkg/try/Makefile | 11 + src/pkg/try/try.go | 174 + src/pkg/try/try_test.go | 60 + src/pkg/unicode/Makefile | 36 + src/pkg/unicode/casetables.go | 20 + src/pkg/unicode/digit.go | 13 + src/pkg/unicode/digit_test.go | 126 + src/pkg/unicode/graphic.go | 132 + src/pkg/unicode/graphic_test.go | 122 + src/pkg/unicode/letter.go | 326 + src/pkg/unicode/letter_test.go | 428 + src/pkg/unicode/maketables.go | 1276 ++ src/pkg/unicode/script_test.go | 257 + src/pkg/unicode/tables.go | 6026 ++++++ src/pkg/unsafe/unsafe.go | 61 + src/pkg/url/Makefile | 11 + src/pkg/url/url.go | 677 + src/pkg/url/url_test.go | 698 + src/pkg/utf16/Makefile | 11 + src/pkg/utf16/utf16.go | 101 + src/pkg/utf16/utf16_test.go | 118 + src/pkg/utf8/Makefile | 12 + src/pkg/utf8/string.go | 211 + src/pkg/utf8/string_test.go | 114 + src/pkg/utf8/utf8.go | 356 + src/pkg/utf8/utf8_test.go | 315 + src/pkg/websocket/Makefile | 9 + src/pkg/websocket/client.go | 323 + src/pkg/websocket/server.go | 219 + src/pkg/websocket/websocket.go | 192 + src/pkg/websocket/websocket_test.go | 271 + src/pkg/xml/Makefile | 14 + src/pkg/xml/atom_test.go | 50 + src/pkg/xml/embed_test.go | 124 + src/pkg/xml/marshal.go | 228 + src/pkg/xml/marshal_test.go | 305 + src/pkg/xml/read.go | 631 + src/pkg/xml/read_test.go | 371 + src/pkg/xml/xml.go | 1694 ++ src/pkg/xml/xml_test.go | 609 + src/quietgcc.bash | 44 + src/run.bash | 102 + src/sudo.bash | 21 + src/version.bash | 50 + test/235.go | 72 + test/64bit.go | 709 + test/append.go | 227 + test/args.go | 21 + test/assign.go | 53 + test/assign1.go | 343 + test/bench/Makefile | 14 + test/bench/binary-tree-freelist.go | 129 + test/bench/binary-tree-freelist.txt | 8 + test/bench/binary-tree.c | 165 + test/bench/binary-tree.go | 92 + test/bench/binary-tree.txt | 8 + test/bench/chameneosredux.c | 330 + test/bench/chameneosredux.go | 180 + test/bench/chameneosredux.txt | 29 + test/bench/fannkuch-parallel.go | 224 + test/bench/fannkuch-parallel.txt | 31 + test/bench/fannkuch.c | 134 + test/bench/fannkuch.go | 122 + test/bench/fannkuch.txt | 31 + test/bench/fasta-1000.out | 171 + test/bench/fasta.c | 219 + test/bench/fasta.go | 206 + test/bench/fasta.txt | 171 + test/bench/k-nucleotide-parallel.go | 157 + test/bench/k-nucleotide-parallel.txt | 27 + test/bench/k-nucleotide.c | 228 + test/bench/k-nucleotide.go | 140 + test/bench/k-nucleotide.txt | 27 + test/bench/mandelbrot.c | 91 + test/bench/mandelbrot.go | 95 + test/bench/mandelbrot.txt | Bin 0 -> 5011 bytes test/bench/meteor-contest.c | 626 + test/bench/meteor-contest.go | 665 + test/bench/meteor-contest.txt | 24 + test/bench/nbody.c | 170 + test/bench/nbody.go | 177 + test/bench/nbody.txt | 2 + test/bench/pidigits.c | 123 + test/bench/pidigits.go | 135 + test/bench/pidigits.txt | 3 + test/bench/regex-dna-parallel.go | 124 + test/bench/regex-dna-parallel.txt | 13 + test/bench/regex-dna.c | 154 + test/bench/regex-dna.go | 106 + test/bench/regex-dna.txt | 13 + test/bench/reverse-complement.c | 100 + test/bench/reverse-complement.go | 105 + test/bench/reverse-complement.txt | 171 + test/bench/spectral-norm-parallel.go | 111 + test/bench/spectral-norm.c | 82 + test/bench/spectral-norm.go | 93 + test/bench/spectral-norm.txt | 1 + test/bench/threadring.c | 102 + test/bench/threadring.go | 71 + test/bench/threadring.txt | 1 + test/bench/timing.log | 902 + test/bench/timing.sh | 205 + test/bigalg.go | 117 + test/bigmap.go | 34 + test/blank.go | 122 + test/blank1.go | 12 + test/bugs/placeholder | 2 + test/chan/doubleselect.go | 86 + test/chan/fifo.go | 57 + test/chan/goroutines.go | 41 + test/chan/nonblock.go | 283 + test/chan/perm.go | 51 + test/chan/powser1.go | 709 + test/chan/powser2.go | 722 + test/chan/select.go | 56 + test/chan/select2.go | 48 + test/chan/select3.go | 224 + test/chan/select4.go | 25 + test/chan/select5.go | 480 + test/chan/select6.go | 34 + test/chan/sendstmt.go | 37 + test/chan/sieve1.go | 54 + test/chan/sieve2.go | 172 + test/chan/zerosize.go | 16 + test/chancap.go | 29 + test/char_lit.go | 43 + test/char_lit1.go | 25 + test/closedchan.go | 330 + test/closure.go | 112 + test/cmp1.go | 130 + test/cmp2.go | 15 + test/cmp3.go | 15 + test/cmp4.go | 14 + test/cmp5.go | 14 + test/cmp6.go | 49 + test/cmplx.go | 25 + test/cmplxdivide.c | 82 + test/cmplxdivide.go | 48 + test/cmplxdivide1.go | 2406 +++ test/complit.go | 102 + test/compos.go | 23 + test/const.go | 120 + test/const1.go | 82 + test/const2.go | 12 + test/const3.go | 35 + test/convert.go | 44 + test/convert3.go | 26 + test/convlit.go | 66 + test/convlit1.go | 17 + test/copy.go | 346 + test/ddd.go | 218 + test/ddd1.go | 49 + test/ddd2.go | 16 + test/ddd3.go | 28 + test/decl.go | 40 + test/declbad.go | 58 + test/defer.go | 48 + test/deferprint.go | 14 + test/divide.go | 54 + test/empty.go | 12 + test/env.go | 29 + test/eof.go | 9 + test/eof1.go | 9 + test/errchk | 147 + test/escape.go | 207 + test/fixedbugs/bug000.go | 20 + test/fixedbugs/bug002.go | 11 + test/fixedbugs/bug003.go | 14 + test/fixedbugs/bug004.go | 11 + test/fixedbugs/bug005.go | 18 + test/fixedbugs/bug006.go | 24 + test/fixedbugs/bug007.go | 24 + test/fixedbugs/bug008.go | 20 + test/fixedbugs/bug009.go | 16 + test/fixedbugs/bug010.go | 24 + test/fixedbugs/bug011.go | 27 + test/fixedbugs/bug012.go | 26 + test/fixedbugs/bug013.go | 20 + test/fixedbugs/bug014.go | 14 + test/fixedbugs/bug015.go | 13 + test/fixedbugs/bug016.go | 18 + test/fixedbugs/bug017.go | 25 + test/fixedbugs/bug020.go | 22 + test/fixedbugs/bug021.go | 13 + test/fixedbugs/bug022.go | 26 + test/fixedbugs/bug023.go | 30 + test/fixedbugs/bug024.go | 21 + test/fixedbugs/bug026.go | 26 + test/fixedbugs/bug027.go | 61 + test/fixedbugs/bug028.go | 29 + test/fixedbugs/bug030.go | 13 + test/fixedbugs/bug031.go | 29 + test/fixedbugs/bug035.go | 13 + test/fixedbugs/bug036.go | 13 + test/fixedbugs/bug037.go | 11 + test/fixedbugs/bug038.go | 13 + test/fixedbugs/bug039.go | 11 + test/fixedbugs/bug040.go | 11 + test/fixedbugs/bug045.go | 21 + test/fixedbugs/bug046.go | 15 + test/fixedbugs/bug047.go | 23 + test/fixedbugs/bug048.go | 13 + test/fixedbugs/bug049.go | 19 + test/fixedbugs/bug050.go | 8 + test/fixedbugs/bug051.go | 15 + test/fixedbugs/bug052.go | 20 + test/fixedbugs/bug053.go | 12 + test/fixedbugs/bug054.go | 41 + test/fixedbugs/bug055.go | 27 + test/fixedbugs/bug056.go | 22 + test/fixedbugs/bug057.go | 25 + test/fixedbugs/bug058.go | 23 + test/fixedbugs/bug059.go | 36 + test/fixedbugs/bug060.go | 19 + test/fixedbugs/bug061.go | 18 + test/fixedbugs/bug062.go | 11 + test/fixedbugs/bug063.go | 8 + test/fixedbugs/bug064.go | 22 + test/fixedbugs/bug065.go | 12 + test/fixedbugs/bug066.go | 26 + test/fixedbugs/bug067.go | 15 + test/fixedbugs/bug068.go | 19 + test/fixedbugs/bug069.go | 20 + test/fixedbugs/bug070.go | 25 + test/fixedbugs/bug071.go | 23 + test/fixedbugs/bug072.go | 11 + test/fixedbugs/bug073.go | 14 + test/fixedbugs/bug074.go | 12 + test/fixedbugs/bug075.go | 17 + test/fixedbugs/bug076.go | 25 + test/fixedbugs/bug077.go | 14 + test/fixedbugs/bug078.go | 16 + test/fixedbugs/bug080.go | 25 + test/fixedbugs/bug081.go | 14 + test/fixedbugs/bug082.go | 21 + test/fixedbugs/bug083.dir/bug0.go | 10 + test/fixedbugs/bug083.dir/bug1.go | 14 + test/fixedbugs/bug083.go | 7 + test/fixedbugs/bug084.go | 27 + test/fixedbugs/bug085.go | 27 + test/fixedbugs/bug086.go | 23 + test/fixedbugs/bug087.go | 20 + test/fixedbugs/bug088.dir/bug0.go | 9 + test/fixedbugs/bug088.dir/bug1.go | 23 + test/fixedbugs/bug088.go | 7 + test/fixedbugs/bug089.go | 21 + test/fixedbugs/bug090.go | 46 + test/fixedbugs/bug091.go | 25 + test/fixedbugs/bug092.go | 20 + test/fixedbugs/bug093.go | 64 + test/fixedbugs/bug094.go | 32 + test/fixedbugs/bug096.go | 26 + test/fixedbugs/bug097.go | 53 + test/fixedbugs/bug098.go | 23 + test/fixedbugs/bug099.go | 37 + test/fixedbugs/bug101.go | 15 + test/fixedbugs/bug102.go | 26 + test/fixedbugs/bug103.go | 14 + test/fixedbugs/bug104.go | 10 + test/fixedbugs/bug106.dir/bug0.go | 7 + test/fixedbugs/bug106.dir/bug1.go | 8 + test/fixedbugs/bug106.go | 7 + test/fixedbugs/bug107.go | 15 + test/fixedbugs/bug108.go | 10 + test/fixedbugs/bug109.go | 25 + test/fixedbugs/bug110.go | 20 + test/fixedbugs/bug111.go | 32 + test/fixedbugs/bug112.go | 16 + test/fixedbugs/bug113.go | 28 + test/fixedbugs/bug114.go | 26 + test/fixedbugs/bug115.go | 14 + test/fixedbugs/bug116.go | 35 + test/fixedbugs/bug117.go | 29 + test/fixedbugs/bug118.go | 15 + test/fixedbugs/bug119.go | 36 + test/fixedbugs/bug120.go | 60 + test/fixedbugs/bug121.go | 18 + test/fixedbugs/bug122.go | 12 + test/fixedbugs/bug123.go | 14 + test/fixedbugs/bug126.go | 11 + test/fixedbugs/bug127.go | 12 + test/fixedbugs/bug128.go | 23 + test/fixedbugs/bug129.go | 14 + test/fixedbugs/bug130.go | 22 + test/fixedbugs/bug131.go | 12 + test/fixedbugs/bug132.go | 11 + test/fixedbugs/bug133.dir/bug0.go | 7 + test/fixedbugs/bug133.dir/bug1.go | 9 + test/fixedbugs/bug133.dir/bug2.go | 16 + test/fixedbugs/bug133.go | 7 + test/fixedbugs/bug135.go | 19 + test/fixedbugs/bug136.go | 22 + test/fixedbugs/bug137.go | 31 + test/fixedbugs/bug139.go | 17 + test/fixedbugs/bug140.go | 27 + test/fixedbugs/bug141.go | 32 + test/fixedbugs/bug142.go | 31 + test/fixedbugs/bug143.go | 44 + test/fixedbugs/bug144.go | 22 + test/fixedbugs/bug145.go | 18 + test/fixedbugs/bug146.go | 15 + test/fixedbugs/bug147.go | 26 + test/fixedbugs/bug148.go | 39 + test/fixedbugs/bug149.go | 24 + test/fixedbugs/bug150.go | 23 + test/fixedbugs/bug151.go | 21 + test/fixedbugs/bug1515.go | 20 + test/fixedbugs/bug152.go | 17 + test/fixedbugs/bug154.go | 33 + test/fixedbugs/bug155.go | 23 + test/fixedbugs/bug156.go | 21 + test/fixedbugs/bug157.go | 32 + test/fixedbugs/bug158.go | 26 + test/fixedbugs/bug159.go | 39 + test/fixedbugs/bug160.dir/x.go | 8 + test/fixedbugs/bug160.dir/y.go | 19 + test/fixedbugs/bug160.go | 7 + test/fixedbugs/bug161.go | 17 + test/fixedbugs/bug163.go | 11 + test/fixedbugs/bug164.go | 17 + test/fixedbugs/bug165.go | 15 + test/fixedbugs/bug167.go | 30 + test/fixedbugs/bug168.go | 19 + test/fixedbugs/bug169.go | 10 + test/fixedbugs/bug170.go | 14 + test/fixedbugs/bug171.go | 10 + test/fixedbugs/bug172.go | 12 + test/fixedbugs/bug173.go | 21 + test/fixedbugs/bug174.go | 12 + test/fixedbugs/bug175.go | 14 + test/fixedbugs/bug176.go | 14 + test/fixedbugs/bug177.go | 30 + test/fixedbugs/bug178.go | 33 + test/fixedbugs/bug179.go | 27 + test/fixedbugs/bug180.go | 16 + test/fixedbugs/bug181.go | 11 + test/fixedbugs/bug182.go | 13 + test/fixedbugs/bug183.go | 27 + test/fixedbugs/bug184.go | 49 + test/fixedbugs/bug185.go | 35 + test/fixedbugs/bug186.go | 17 + test/fixedbugs/bug187.go | 22 + test/fixedbugs/bug188.go | 15 + test/fixedbugs/bug189.go | 18 + test/fixedbugs/bug190.go | 26 + test/fixedbugs/bug191.dir/a.go | 12 + test/fixedbugs/bug191.dir/b.go | 11 + test/fixedbugs/bug191.go | 16 + test/fixedbugs/bug192.go | 11 + test/fixedbugs/bug193.go | 16 + test/fixedbugs/bug194.go | 35 + test/fixedbugs/bug195.go | 27 + test/fixedbugs/bug196.go | 50 + test/fixedbugs/bug197.go | 33 + test/fixedbugs/bug198.go | 12 + test/fixedbugs/bug199.go | 27 + test/fixedbugs/bug200.go | 19 + test/fixedbugs/bug201.go | 48 + test/fixedbugs/bug202.go | 16 + test/fixedbugs/bug203.go | 18 + test/fixedbugs/bug204.go | 24 + test/fixedbugs/bug205.go | 18 + test/fixedbugs/bug206.go | 48 + test/fixedbugs/bug207.go | 23 + test/fixedbugs/bug208.go | 20 + test/fixedbugs/bug209.go | 18 + test/fixedbugs/bug211.go | 14 + test/fixedbugs/bug212.go | 12 + test/fixedbugs/bug213.go | 16 + test/fixedbugs/bug214.go | 15 + test/fixedbugs/bug215.go | 14 + test/fixedbugs/bug216.go | 20 + test/fixedbugs/bug217.go | 15 + test/fixedbugs/bug218.go | 23 + test/fixedbugs/bug219.go | 38 + test/fixedbugs/bug220.go | 14 + test/fixedbugs/bug221.go | 42 + test/fixedbugs/bug222.dir/chanbug.go | 5 + test/fixedbugs/bug222.dir/chanbug2.go | 2 + test/fixedbugs/bug222.go | 7 + test/fixedbugs/bug223.go | 21 + test/fixedbugs/bug224.go | 10 + test/fixedbugs/bug225.go | 22 + test/fixedbugs/bug226.dir/x.go | 9 + test/fixedbugs/bug226.dir/y.go | 31 + test/fixedbugs/bug226.go | 7 + test/fixedbugs/bug227.go | 36 + test/fixedbugs/bug228.go | 19 + test/fixedbugs/bug229.go | 20 + test/fixedbugs/bug230.go | 28 + test/fixedbugs/bug231.go | 22 + test/fixedbugs/bug232.go | 8 + test/fixedbugs/bug233.go | 10 + test/fixedbugs/bug234.go | 23 + test/fixedbugs/bug235.go | 17 + test/fixedbugs/bug236.go | 53 + test/fixedbugs/bug237.go | 25 + test/fixedbugs/bug238.go | 21 + test/fixedbugs/bug239.go | 21 + test/fixedbugs/bug240.go | 20 + test/fixedbugs/bug241.go | 11 + test/fixedbugs/bug242.go | 132 + test/fixedbugs/bug243.go | 51 + test/fixedbugs/bug244.go | 31 + test/fixedbugs/bug245.go | 16 + test/fixedbugs/bug246.go | 23 + test/fixedbugs/bug247.go | 21 + test/fixedbugs/bug248.dir/bug0.go | 9 + test/fixedbugs/bug248.dir/bug1.go | 9 + test/fixedbugs/bug248.dir/bug2.go | 101 + test/fixedbugs/bug248.dir/bug3.go | 69 + test/fixedbugs/bug248.go | 12 + test/fixedbugs/bug249.go | 39 + test/fixedbugs/bug250.go | 19 + test/fixedbugs/bug251.go | 21 + test/fixedbugs/bug252.go | 15 + test/fixedbugs/bug253.go | 29 + test/fixedbugs/bug254.go | 17 + test/fixedbugs/bug255.go | 15 + test/fixedbugs/bug256.go | 16 + test/fixedbugs/bug257.go | 20070 +++++++++++++++++++ test/fixedbugs/bug258.go | 33 + test/fixedbugs/bug259.go | 16 + test/fixedbugs/bug260.go | 61 + test/fixedbugs/bug261.go | 23 + test/fixedbugs/bug262.go | 54 + test/fixedbugs/bug263.go | 16 + test/fixedbugs/bug264.go | 44 + test/fixedbugs/bug265.go | 22 + test/fixedbugs/bug266.go | 26 + test/fixedbugs/bug267.go | 22 + test/fixedbugs/bug268.go | 53 + test/fixedbugs/bug269.go | 18 + test/fixedbugs/bug270.go | 21 + test/fixedbugs/bug271.go | 20 + test/fixedbugs/bug272.go | 25 + test/fixedbugs/bug273.go | 93 + test/fixedbugs/bug274.go | 31 + test/fixedbugs/bug275.go | 20 + test/fixedbugs/bug276.go | 23 + test/fixedbugs/bug277.go | 72 + test/fixedbugs/bug278.go | 23 + test/fixedbugs/bug279.go | 36 + test/fixedbugs/bug280.go | 13 + test/fixedbugs/bug281.go | 55 + test/fixedbugs/bug282.dir/p1.go | 10 + test/fixedbugs/bug282.dir/p2.go | 8 + test/fixedbugs/bug282.go | 7 + test/fixedbugs/bug283.go | 19 + test/fixedbugs/bug284.go | 191 + test/fixedbugs/bug285.go | 116 + test/fixedbugs/bug286.go | 94 + test/fixedbugs/bug287.go | 11 + test/fixedbugs/bug288.go | 18 + test/fixedbugs/bug289.go | 26 + test/fixedbugs/bug290.go | 15 + test/fixedbugs/bug291.go | 23 + test/fixedbugs/bug292.go | 22 + test/fixedbugs/bug293.go | 37 + test/fixedbugs/bug294.go | 79 + test/fixedbugs/bug295.go | 17 + test/fixedbugs/bug296.go | 88 + test/fixedbugs/bug297.go | 15 + test/fixedbugs/bug298.go | 11 + test/fixedbugs/bug299.go | 27 + test/fixedbugs/bug300.go | 29 + test/fixedbugs/bug301.go | 18 + test/fixedbugs/bug302.dir/main.go | 12 + test/fixedbugs/bug302.dir/p.go | 1011 + test/fixedbugs/bug302.go | 6 + test/fixedbugs/bug303.go | 37 + test/fixedbugs/bug304.go | 18 + test/fixedbugs/bug305.go | 24 + test/fixedbugs/bug306.dir/p1.go | 9 + test/fixedbugs/bug306.dir/p2.go | 8 + test/fixedbugs/bug306.go | 7 + test/fixedbugs/bug307.go | 15 + test/fixedbugs/bug308.go | 19 + test/fixedbugs/bug309.go | 19 + test/fixedbugs/bug310.go | 20 + test/fixedbugs/bug311.go | 20 + test/fixedbugs/bug312.go | 22 + test/fixedbugs/bug313.dir/a.go | 11 + test/fixedbugs/bug313.dir/b.go | 11 + test/fixedbugs/bug313.go | 19 + test/fixedbugs/bug314.go | 31 + test/fixedbugs/bug315.go | 18 + test/fixedbugs/bug316.go | 17 + test/fixedbugs/bug317.go | 16 + test/fixedbugs/bug318.go | 12 + test/fixedbugs/bug319.go | 22 + test/fixedbugs/bug320.go | 45 + test/fixedbugs/bug321.go | 30 + test/fixedbugs/bug322.dir/lib.go | 15 + test/fixedbugs/bug322.dir/main.go | 40 + test/fixedbugs/bug322.go | 8 + test/fixedbugs/bug323.go | 20 + test/fixedbugs/bug324.dir/main.go | 53 + test/fixedbugs/bug324.dir/p.go | 15 + test/fixedbugs/bug324.go | 8 + test/fixedbugs/bug325.go | 15 + test/fixedbugs/bug326.go | 41 + test/fixedbugs/bug327.go | 24 + test/fixedbugs/bug328.go | 14 + test/fixedbugs/bug329.go | 46 + test/fixedbugs/bug330.go | 13 + test/fixedbugs/bug331.go | 36 + test/fixedbugs/bug332.go | 17 + test/fixedbugs/bug333.go | 19 + test/fixedbugs/bug334.go | 31 + test/fixedbugs/bug335.dir/a.go | 9 + test/fixedbugs/bug335.dir/b.go | 11 + test/fixedbugs/bug335.go | 10 + test/fixedbugs/bug336.go | 86 + test/fixedbugs/bug337.go | 19 + test/fixedbugs/bug338.go | 22 + test/fixedbugs/bug339.go | 20 + test/fixedbugs/bug340.go | 17 + test/fixedbugs/bug341.go | 22 + test/fixedbugs/bug342.go | 24 + test/fixedbugs/bug343.go | 33 + test/fixedbugs/bug344.go | 24 + test/fixedbugs/bug345.dir/io.go | 15 + test/fixedbugs/bug345.dir/main.go | 28 + test/fixedbugs/bug345.go | 7 + test/fixedbugs/bug346.go | 19 + test/fixedbugs/bug347.go | 49 + test/fixedbugs/bug348.go | 46 + test/fixedbugs/bug349.go | 13 + test/fixedbugs/bug350.go | 15 + test/fixedbugs/bug351.go | 13 + test/fixedbugs/bug352.go | 19 + test/fixedbugs/bug353.go | 30 + test/fixedbugs/bug354.go | 26 + test/fixedbugs/bug355.go | 18 + test/fixedbugs/bug356.go | 41 + test/fixedbugs/bug357.go | 25 + test/fixedbugs/bug358.go | 26 + test/fixedbugs/bug359.go | 26 + test/fixedbugs/bug361.go | 15 + test/fixedbugs/bug362.go | 16 + test/fixedbugs/bug363.go | 21 + test/fixedbugs/bug364.go | 25 + test/fixedbugs/bug365.go | 22 + test/float_lit.go | 197 + test/floatcmp.go | 88 + test/for.go | 56 + test/func.go | 89 + test/func1.go | 18 + test/func2.go | 31 + test/func3.go | 17 + test/func4.go | 14 + test/func5.go | 89 + test/func6.go | 14 + test/func7.go | 29 + test/garbage/Makefile | 27 + test/garbage/parser.go | 237 + test/garbage/peano.go | 135 + test/garbage/stats.go | 44 + test/garbage/tree.go | 100 + test/gc.go | 24 + test/gc1.go | 14 + test/gc2.go | 42 + test/golden.out | 166 + test/goprint.go | 14 + test/goto.go | 535 + test/hashmap.go | 181 + test/helloworld.go | 11 + test/if.go | 91 + test/import.go | 25 + test/import1.go | 17 + test/import2.go | 42 + test/import3.go | 54 + test/import4.go | 24 + test/index.go | 224 + test/indirect.go | 85 + test/indirect1.go | 68 + test/init.go | 18 + test/initcomma.go | 79 + test/initialize.go | 62 + test/initializerr.go | 25 + test/initsyscall.go | 27 + test/int_lit.go | 24 + test/intcvt.go | 179 + test/interface/bigdata.go | 75 + test/interface/convert.go | 147 + test/interface/convert1.go | 25 + test/interface/convert2.go | 25 + test/interface/embed.go | 61 + test/interface/embed0.go | 29 + test/interface/embed1.go | 45 + test/interface/embed2.go | 70 + test/interface/explicit.go | 75 + test/interface/fail.go | 26 + test/interface/fake.go | 97 + test/interface/pointer.go | 37 + test/interface/private.go | 32 + test/interface/private1.go | 18 + test/interface/receiver.go | 121 + test/interface/receiver1.go | 58 + test/interface/recursive.go | 21 + test/interface/returntype.go | 25 + test/interface/struct.go | 156 + test/iota.go | 120 + test/ken/array.go | 133 + test/ken/chan.go | 329 + test/ken/chan1.go | 53 + test/ken/complit.go | 170 + test/ken/convert.go | 431 + test/ken/cplx0.go | 28 + test/ken/cplx1.go | 97 + test/ken/cplx2.go | 108 + test/ken/cplx3.go | 33 + test/ken/cplx4.go | 44 + test/ken/cplx5.go | 54 + test/ken/divconst.go | 632 + test/ken/divmod.go | 247 + test/ken/embed.go | 317 + test/ken/for.go | 18 + test/ken/interbasic.go | 182 + test/ken/interfun.go | 57 + test/ken/intervar.go | 64 + test/ken/label.go | 36 + test/ken/litfun.go | 22 + test/ken/mfunc.go | 20 + test/ken/modconst.go | 632 + test/ken/ptrfun.go | 44 + test/ken/ptrvar.go | 53 + test/ken/range.go | 119 + test/ken/rob1.go | 67 + test/ken/rob2.go | 272 + test/ken/robfor.go | 56 + test/ken/robfunc.go | 94 + test/ken/shift.go | 119 + test/ken/simparray.go | 48 + test/ken/simpbool.go | 105 + test/ken/simpconv.go | 28 + test/ken/simpfun.go | 25 + test/ken/simpprint.go | 13 + test/ken/simpswitch.go | 24 + test/ken/simpvar.go | 25 + test/ken/slicearray.go | 210 + test/ken/sliceslice.go | 203 + test/ken/string.go | 118 + test/ken/strvar.go | 78 + test/label.go | 60 + test/label1.go | 85 + test/literal.go | 228 + test/malloc1.go | 25 + test/mallocfin.go | 67 + test/mallocrand.go | 92 + test/mallocrep.go | 69 + test/mallocrep1.go | 143 + test/map.go | 491 + test/method.go | 127 + test/method1.go | 17 + test/method2.go | 33 + test/method3.go | 35 + test/named.go | 281 + test/named1.go | 62 + test/nil.go | 178 + test/nilptr/arrayindex.go | 26 + test/nilptr/arrayindex1.go | 31 + test/nilptr/arraytoslice.go | 36 + test/nilptr/arraytoslice1.go | 33 + test/nilptr/arraytoslice2.go | 34 + test/nilptr/slicearray.go | 32 + test/nilptr/structfield.go | 34 + test/nilptr/structfield1.go | 37 + test/nilptr/structfield2.go | 36 + test/nilptr/structfieldaddr.go | 34 + test/nul1.go | 59 + test/parentype.go | 17 + test/peano.go | 126 + test/printbig.go | 12 + test/range.go | 297 + test/recover.go | 246 + test/recover1.go | 141 + test/recover2.go | 87 + test/recover3.go | 76 + test/rename.go | 73 + test/rename1.go | 46 + test/run | 130 + test/runtime.go | 20 + test/shift1.go | 36 + test/shift2.go | 42 + test/sieve.go | 42 + test/sigchld.go | 14 + test/simassign.go | 77 + test/sinit.go | 100 + test/sizeof.go | 23 + test/solitaire.go | 119 + test/stack.go | 99 + test/string_lit.go | 120 + test/stringrange.go | 61 + test/switch.go | 139 + test/switch1.go | 20 + test/syntax/chan.go | 17 + test/syntax/chan1.go | 17 + test/syntax/forvar.go | 10 + test/syntax/if.go | 18 + test/syntax/import.go | 14 + test/syntax/interface.go | 14 + test/syntax/semi1.go | 14 + test/syntax/semi2.go | 14 + test/syntax/semi3.go | 14 + test/syntax/semi4.go | 14 + test/syntax/semi5.go | 13 + test/syntax/semi6.go | 13 + test/syntax/semi7.go | 14 + test/syntax/topexpr.go | 20 + test/syntax/typesw.go | 13 + test/syntax/vareq.go | 10 + test/syntax/vareq1.go | 10 + test/test0.go | 92 + test/turing.go | 52 + test/typeswitch.go | 115 + test/typeswitch1.go | 83 + test/typeswitch2.go | 28 + test/undef.go | 44 + test/utf.go | 54 + test/varerr.go | 14 + test/varinit.go | 29 + test/vectors.go | 64 + test/zerodivide.go | 241 + 2979 files changed, 641545 insertions(+) create mode 100644 AUTHORS create mode 100644 CONTRIBUTORS create mode 100644 LICENSE create mode 100644 PATENTS create mode 100644 README create mode 100644 VERSION create mode 100644 doc/ExpressivenessOfGo.pdf create mode 100644 doc/GoCourseDay1.pdf create mode 100644 doc/GoCourseDay2.pdf create mode 100644 doc/GoCourseDay3.pdf create mode 100644 doc/Makefile create mode 100644 doc/all.css create mode 100644 doc/button_background.png create mode 100644 doc/code.html create mode 100644 doc/codelab/wiki/Makefile create mode 100644 doc/codelab/wiki/edit.html create mode 100644 doc/codelab/wiki/final-noclosure.go create mode 100644 doc/codelab/wiki/final-noerror.go create mode 100644 doc/codelab/wiki/final-parsetemplate.go create mode 100644 doc/codelab/wiki/final-template.go create mode 100644 doc/codelab/wiki/final.go create mode 100644 doc/codelab/wiki/get.go create mode 100644 doc/codelab/wiki/htmlify.go create mode 100644 doc/codelab/wiki/http-sample.go create mode 100644 doc/codelab/wiki/index.html create mode 100644 doc/codelab/wiki/notemplate.go create mode 100644 doc/codelab/wiki/part1-noerror.go create mode 100644 doc/codelab/wiki/part1.go create mode 100644 doc/codelab/wiki/part2.go create mode 100644 doc/codelab/wiki/srcextract.go create mode 100755 doc/codelab/wiki/test.sh create mode 100644 doc/codelab/wiki/test_Test.txt.good create mode 100644 doc/codelab/wiki/test_edit.good create mode 100644 doc/codelab/wiki/test_view.good create mode 100644 doc/codelab/wiki/view.html create mode 100644 doc/codelab/wiki/wiki.html create mode 100644 doc/codereview_with_mq.html create mode 100644 doc/codewalk/codewalk.css create mode 100644 doc/codewalk/codewalk.js create mode 100644 doc/codewalk/codewalk.xml create mode 100644 doc/codewalk/functions.xml create mode 100644 doc/codewalk/markov.go create mode 100644 doc/codewalk/markov.xml create mode 100644 doc/codewalk/pig.go create mode 100644 doc/codewalk/popout.png create mode 100644 doc/codewalk/sharemem.xml create mode 100644 doc/codewalk/urlpoll.go create mode 100644 doc/community.html create mode 100644 doc/contrib.html create mode 100644 doc/contribute.html create mode 100644 doc/devel/index.html create mode 100644 doc/devel/release.html create mode 100644 doc/devel/roadmap.html create mode 100644 doc/devel/weekly.html create mode 100644 doc/docs.html create mode 100644 doc/effective_go.html create mode 100644 doc/frontpage.css create mode 100644 doc/gccgo_contribute.html create mode 100644 doc/gccgo_install.html create mode 100644 doc/go-logo-black.png create mode 100644 doc/go-logo-blue.png create mode 100644 doc/go-logo-white.png create mode 100644 doc/go_faq.html create mode 100644 doc/go_for_cpp_programmers.html create mode 100644 doc/go_mem.html create mode 100644 doc/go_spec.html create mode 100644 doc/go_tutorial.html create mode 100644 doc/go_tutorial.tmpl create mode 100644 doc/godocs.js create mode 100644 doc/gopher/appenginegopher.jpg create mode 100644 doc/gopher/appenginegophercolor.jpg create mode 100644 doc/gopher/appenginelogo.gif create mode 100644 doc/gopher/bumper.png create mode 100644 doc/gopher/bumper192x108.png create mode 100644 doc/gopher/bumper320x180.png create mode 100644 doc/gopher/bumper480x270.png create mode 100644 doc/gopher/bumper640x360.png create mode 100644 doc/gopher/gopherbw.png create mode 100644 doc/gopher/gophercolor.png create mode 100644 doc/gopher/gophercolor16x16.png create mode 100644 doc/ie.css create mode 100644 doc/install.html create mode 100644 doc/logo-153x55.png create mode 100644 doc/logo.png create mode 100755 doc/makehtml create mode 100644 doc/play_overlay.png create mode 100644 doc/playground.html create mode 100644 doc/popups.js create mode 100644 doc/progs/cat.go create mode 100644 doc/progs/cat_rot13.go create mode 100644 doc/progs/echo.go create mode 100644 doc/progs/file.go create mode 100644 doc/progs/file_windows.go create mode 100644 doc/progs/helloworld.go create mode 100644 doc/progs/helloworld3.go create mode 100644 doc/progs/print.go create mode 100644 doc/progs/print_string.go create mode 100755 doc/progs/run create mode 100644 doc/progs/server.go create mode 100644 doc/progs/server1.go create mode 100644 doc/progs/sieve.go create mode 100644 doc/progs/sieve1.go create mode 100644 doc/progs/sort.go create mode 100644 doc/progs/sortmain.go create mode 100644 doc/progs/strings.go create mode 100644 doc/progs/sum.go create mode 100644 doc/root.html create mode 100644 doc/sieve.gif create mode 100644 doc/talks/go_talk-20100112.html create mode 100644 doc/talks/go_talk-20100121.html create mode 100644 doc/talks/go_talk-20100323.html create mode 100644 doc/talks/gofrontend-gcc-summit-2010.pdf create mode 100644 doc/talks/io2010/balance.go create mode 100644 doc/talks/io2010/decrypt.go create mode 100644 doc/talks/io2010/encrypt.go create mode 100644 doc/talks/io2010/eval1.go create mode 100644 doc/talks/io2010/eval2.go create mode 100644 doc/talks/io2010/talk.pdf create mode 100644 doc/talks/io2011/Real_World_Go.pdf create mode 100644 doc/talks/io2011/Writing_Web_Apps_in_Go.pdf create mode 100644 doc/talks/java-typing.png create mode 100644 doc/talks/slidy.css create mode 100644 doc/talks/slidy.js create mode 100644 doc/tmpltohtml.go create mode 100644 doc/video-001.png create mode 100644 doc/video-002.png create mode 100644 doc/video-003.png create mode 100644 doc/video-004.png create mode 100644 doc/video-005.jpg create mode 100644 favicon.ico create mode 100644 include/ar.h create mode 100644 include/bio.h create mode 100644 include/bootexec.h create mode 100644 include/fmt.h create mode 100644 include/libc.h create mode 100644 include/mach.h create mode 100644 include/u.h create mode 100644 include/ureg_amd64.h create mode 100644 include/ureg_arm.h create mode 100644 include/ureg_x86.h create mode 100644 include/utf.h create mode 100644 lib/codereview/codereview.cfg create mode 100644 lib/codereview/codereview.py create mode 100644 lib/godoc/codewalk.html create mode 100644 lib/godoc/codewalkdir.html create mode 100644 lib/godoc/dirlist.html create mode 100644 lib/godoc/error.html create mode 100644 lib/godoc/godoc.html create mode 100644 lib/godoc/package.html create mode 100644 lib/godoc/package.txt create mode 100644 lib/godoc/search.html create mode 100644 lib/godoc/search.txt create mode 100644 misc/IntelliJIDEA/Go.xml create mode 100755 misc/arm/a create mode 100644 misc/bash/go create mode 100755 misc/bbedit/Go.plist create mode 100644 misc/cgo/gmp/Makefile create mode 100644 misc/cgo/gmp/fib.go create mode 100644 misc/cgo/gmp/gmp.go create mode 100644 misc/cgo/gmp/pi.go create mode 100644 misc/cgo/life/Makefile create mode 100644 misc/cgo/life/c-life.c create mode 100644 misc/cgo/life/golden.out create mode 100644 misc/cgo/life/life.go create mode 100644 misc/cgo/life/life.h create mode 100644 misc/cgo/life/main.go create mode 100755 misc/cgo/life/test.bash create mode 100644 misc/cgo/stdio/Makefile create mode 100644 misc/cgo/stdio/chain.go create mode 100644 misc/cgo/stdio/fib.go create mode 100644 misc/cgo/stdio/file.go create mode 100644 misc/cgo/stdio/golden.out create mode 100644 misc/cgo/stdio/hello.go create mode 100755 misc/cgo/stdio/test.bash create mode 100644 misc/cgo/test/Makefile create mode 100644 misc/cgo/test/align.go create mode 100644 misc/cgo/test/basic.go create mode 100644 misc/cgo/test/callback.go create mode 100644 misc/cgo/test/callback_c.c create mode 100644 misc/cgo/test/cgo_test.go create mode 100644 misc/cgo/test/duplicate_symbol.go create mode 100644 misc/cgo/test/env.go create mode 100644 misc/cgo/test/exports.go create mode 100644 misc/cgo/test/issue1222.go create mode 100644 misc/cgo/test/issue1328.go create mode 100644 misc/cgo/test/issue1560.go create mode 100644 misc/cgo/test/runtime.c create mode 100644 misc/chrome/gophertool/README.txt create mode 100644 misc/chrome/gophertool/background.html create mode 100644 misc/chrome/gophertool/gopher.js create mode 100644 misc/chrome/gophertool/gopher.png create mode 100644 misc/chrome/gophertool/manifest.json create mode 100644 misc/chrome/gophertool/popup.html create mode 100644 misc/dashboard/README create mode 100644 misc/dashboard/builder/Makefile create mode 100644 misc/dashboard/builder/doc.go create mode 100644 misc/dashboard/builder/exec.go create mode 100644 misc/dashboard/builder/http.go create mode 100644 misc/dashboard/builder/main.go create mode 100644 misc/dashboard/builder/package.go create mode 100644 misc/dashboard/godashboard/_multiprocessing.py create mode 100644 misc/dashboard/godashboard/app.yaml create mode 100644 misc/dashboard/godashboard/auth.py create mode 100644 misc/dashboard/godashboard/const.py create mode 100644 misc/dashboard/godashboard/cron.yaml create mode 100644 misc/dashboard/godashboard/fail-notify.txt create mode 100644 misc/dashboard/godashboard/gobuild.py create mode 100644 misc/dashboard/godashboard/index.yaml create mode 100644 misc/dashboard/godashboard/key.py.dummy create mode 100644 misc/dashboard/godashboard/main.html create mode 100644 misc/dashboard/godashboard/package.html create mode 100644 misc/dashboard/godashboard/package.py create mode 100644 misc/dashboard/godashboard/project-edit.html create mode 100644 misc/dashboard/godashboard/project-notify.txt create mode 100644 misc/dashboard/godashboard/project.html create mode 100644 misc/dashboard/godashboard/static/favicon.ico create mode 100644 misc/dashboard/godashboard/static/style.css create mode 100644 misc/dashboard/godashboard/toutf8.py create mode 100755 misc/dashboard/googlecode_upload.py create mode 100644 misc/emacs/go-mode-load.el create mode 100644 misc/emacs/go-mode.el create mode 100644 misc/fraise/go.plist create mode 100644 misc/fraise/readme.txt create mode 100644 misc/godoc/README create mode 100644 misc/godoc/app.yaml create mode 100644 misc/godoc/init.go create mode 100644 misc/goplay/Makefile create mode 100644 misc/goplay/README create mode 100644 misc/goplay/doc.go create mode 100644 misc/goplay/goplay.go create mode 100644 misc/kate/go.xml create mode 100755 misc/notepadplus/README create mode 100755 misc/notepadplus/go.xml create mode 100755 misc/notepadplus/userDefineLang.xml create mode 100644 misc/swig/callback/Makefile create mode 100644 misc/swig/callback/callback.h create mode 100644 misc/swig/callback/callback.swigcxx create mode 100755 misc/swig/callback/run create mode 100644 misc/swig/callback/run.go create mode 100644 misc/swig/stdio/Makefile create mode 100644 misc/swig/stdio/file.swig create mode 100755 misc/swig/stdio/hello create mode 100644 misc/swig/stdio/hello.go create mode 100644 misc/vim/autoload/go/complete.vim create mode 100644 misc/vim/ftdetect/gofiletype.vim create mode 100644 misc/vim/ftplugin/go/fmt.vim create mode 100644 misc/vim/ftplugin/go/godoc.vim create mode 100644 misc/vim/ftplugin/go/import.vim create mode 100644 misc/vim/indent/go.vim create mode 100644 misc/vim/plugin/godoc.vim create mode 100644 misc/vim/readme.txt create mode 100644 misc/vim/syntax/go.vim create mode 100644 misc/vim/syntax/godoc.vim create mode 100644 misc/xcode/go.pbfilespec create mode 100644 misc/xcode/go.xclangspec create mode 100644 misc/zsh/go create mode 100644 robots.txt create mode 100644 src/Make.ccmd create mode 100644 src/Make.clib create mode 100644 src/Make.cmd create mode 100644 src/Make.common create mode 100644 src/Make.inc create mode 100644 src/Make.pkg create mode 100755 src/all-qemu.bash create mode 100755 src/all.bash create mode 100755 src/clean.bash create mode 100644 src/cmd/5a/Makefile create mode 100644 src/cmd/5a/a.h create mode 100644 src/cmd/5a/a.y create mode 100644 src/cmd/5a/doc.go create mode 100644 src/cmd/5a/lex.c create mode 100644 src/cmd/5c/Makefile create mode 100644 src/cmd/5c/cgen.c create mode 100644 src/cmd/5c/doc.go create mode 100644 src/cmd/5c/gc.h create mode 100644 src/cmd/5c/list.c create mode 100644 src/cmd/5c/mul.c create mode 100644 src/cmd/5c/peep.c create mode 100644 src/cmd/5c/reg.c create mode 100644 src/cmd/5c/sgen.c create mode 100644 src/cmd/5c/swt.c create mode 100644 src/cmd/5c/txt.c create mode 100644 src/cmd/5g/Makefile create mode 100644 src/cmd/5g/cgen.c create mode 100644 src/cmd/5g/cgen64.c create mode 100644 src/cmd/5g/doc.go create mode 100644 src/cmd/5g/galign.c create mode 100644 src/cmd/5g/gg.h create mode 100644 src/cmd/5g/ggen.c create mode 100644 src/cmd/5g/gobj.c create mode 100644 src/cmd/5g/gsubr.c create mode 100644 src/cmd/5g/list.c create mode 100644 src/cmd/5g/opt.h create mode 100644 src/cmd/5g/peep.c create mode 100644 src/cmd/5g/reg.c create mode 100644 src/cmd/5l/5.out.h create mode 100644 src/cmd/5l/Makefile create mode 100644 src/cmd/5l/asm.c create mode 100644 src/cmd/5l/doc.go create mode 100644 src/cmd/5l/l.h create mode 100644 src/cmd/5l/list.c create mode 100644 src/cmd/5l/mkenam create mode 100644 src/cmd/5l/noop.c create mode 100644 src/cmd/5l/obj.c create mode 100644 src/cmd/5l/optab.c create mode 100644 src/cmd/5l/pass.c create mode 100644 src/cmd/5l/prof.c create mode 100644 src/cmd/5l/softfloat.c create mode 100644 src/cmd/5l/span.c create mode 100644 src/cmd/6a/Makefile create mode 100644 src/cmd/6a/a.h create mode 100644 src/cmd/6a/a.y create mode 100644 src/cmd/6a/doc.go create mode 100644 src/cmd/6a/lex.c create mode 100644 src/cmd/6c/Makefile create mode 100644 src/cmd/6c/cgen.c create mode 100644 src/cmd/6c/div.c create mode 100644 src/cmd/6c/doc.go create mode 100644 src/cmd/6c/gc.h create mode 100644 src/cmd/6c/list.c create mode 100644 src/cmd/6c/machcap.c create mode 100644 src/cmd/6c/mul.c create mode 100644 src/cmd/6c/peep.c create mode 100644 src/cmd/6c/reg.c create mode 100644 src/cmd/6c/sgen.c create mode 100644 src/cmd/6c/swt.c create mode 100644 src/cmd/6c/txt.c create mode 100644 src/cmd/6g/Makefile create mode 100644 src/cmd/6g/cgen.c create mode 100644 src/cmd/6g/doc.go create mode 100644 src/cmd/6g/galign.c create mode 100644 src/cmd/6g/gg.h create mode 100644 src/cmd/6g/ggen.c create mode 100644 src/cmd/6g/gobj.c create mode 100644 src/cmd/6g/gsubr.c create mode 100644 src/cmd/6g/list.c create mode 100644 src/cmd/6g/opt.h create mode 100644 src/cmd/6g/peep.c create mode 100644 src/cmd/6g/reg.c create mode 100644 src/cmd/6l/6.out.h create mode 100644 src/cmd/6l/Makefile create mode 100644 src/cmd/6l/asm.c create mode 100644 src/cmd/6l/doc.go create mode 100644 src/cmd/6l/l.h create mode 100644 src/cmd/6l/list.c create mode 100644 src/cmd/6l/mkenam create mode 100644 src/cmd/6l/obj.c create mode 100644 src/cmd/6l/optab.c create mode 100644 src/cmd/6l/pass.c create mode 100644 src/cmd/6l/prof.c create mode 100644 src/cmd/6l/span.c create mode 100644 src/cmd/8a/Makefile create mode 100644 src/cmd/8a/a.h create mode 100644 src/cmd/8a/a.y create mode 100644 src/cmd/8a/doc.go create mode 100644 src/cmd/8a/lex.c create mode 100644 src/cmd/8c/Makefile create mode 100644 src/cmd/8c/cgen.c create mode 100644 src/cmd/8c/cgen64.c create mode 100644 src/cmd/8c/div.c create mode 100644 src/cmd/8c/doc.go create mode 100644 src/cmd/8c/gc.h create mode 100644 src/cmd/8c/list.c create mode 100644 src/cmd/8c/machcap.c create mode 100644 src/cmd/8c/mul.c create mode 100644 src/cmd/8c/peep.c create mode 100644 src/cmd/8c/reg.c create mode 100644 src/cmd/8c/sgen.c create mode 100644 src/cmd/8c/swt.c create mode 100644 src/cmd/8c/txt.c create mode 100644 src/cmd/8g/Makefile create mode 100644 src/cmd/8g/cgen.c create mode 100644 src/cmd/8g/cgen64.c create mode 100644 src/cmd/8g/doc.go create mode 100644 src/cmd/8g/galign.c create mode 100644 src/cmd/8g/gg.h create mode 100644 src/cmd/8g/ggen.c create mode 100644 src/cmd/8g/gobj.c create mode 100644 src/cmd/8g/gsubr.c create mode 100644 src/cmd/8g/list.c create mode 100644 src/cmd/8g/opt.h create mode 100644 src/cmd/8g/peep.c create mode 100644 src/cmd/8g/reg.c create mode 100644 src/cmd/8l/8.out.h create mode 100644 src/cmd/8l/Makefile create mode 100644 src/cmd/8l/asm.c create mode 100644 src/cmd/8l/doc.go create mode 100644 src/cmd/8l/l.h create mode 100644 src/cmd/8l/list.c create mode 100644 src/cmd/8l/mkenam create mode 100644 src/cmd/8l/obj.c create mode 100644 src/cmd/8l/optab.c create mode 100644 src/cmd/8l/pass.c create mode 100644 src/cmd/8l/prof.c create mode 100644 src/cmd/8l/span.c create mode 100644 src/cmd/Makefile create mode 100644 src/cmd/cc/Makefile create mode 100644 src/cmd/cc/acid.c create mode 100644 src/cmd/cc/bits.c create mode 100644 src/cmd/cc/cc.h create mode 100644 src/cmd/cc/cc.y create mode 100644 src/cmd/cc/com.c create mode 100644 src/cmd/cc/com64.c create mode 100644 src/cmd/cc/dcl.c create mode 100644 src/cmd/cc/doc.go create mode 100644 src/cmd/cc/dpchk.c create mode 100644 src/cmd/cc/funct.c create mode 100644 src/cmd/cc/godefs.c create mode 100644 src/cmd/cc/lex.c create mode 100644 src/cmd/cc/lexbody create mode 100644 src/cmd/cc/mac.c create mode 100644 src/cmd/cc/macbody create mode 100644 src/cmd/cc/omachcap.c create mode 100644 src/cmd/cc/pgen.c create mode 100644 src/cmd/cc/pswt.c create mode 100644 src/cmd/cc/scon.c create mode 100644 src/cmd/cc/sub.c create mode 100644 src/cmd/cgo/Makefile create mode 100644 src/cmd/cgo/ast.go create mode 100644 src/cmd/cgo/doc.go create mode 100644 src/cmd/cgo/gcc.go create mode 100644 src/cmd/cgo/main.go create mode 100644 src/cmd/cgo/out.go create mode 100644 src/cmd/cgo/util.go create mode 100644 src/cmd/cov/Makefile create mode 100644 src/cmd/cov/doc.go create mode 100644 src/cmd/cov/main.c create mode 100644 src/cmd/cov/tree.c create mode 100644 src/cmd/cov/tree.h create mode 100644 src/cmd/ebnflint/Makefile create mode 100644 src/cmd/ebnflint/doc.go create mode 100644 src/cmd/ebnflint/ebnflint.go create mode 100644 src/cmd/gc/Makefile create mode 100644 src/cmd/gc/align.c create mode 100755 src/cmd/gc/bisonerrors create mode 100644 src/cmd/gc/bits.c create mode 100644 src/cmd/gc/builtin.c.boot create mode 100644 src/cmd/gc/closure.c create mode 100644 src/cmd/gc/const.c create mode 100644 src/cmd/gc/cplx.c create mode 100644 src/cmd/gc/dcl.c create mode 100644 src/cmd/gc/doc.go create mode 100644 src/cmd/gc/export.c create mode 100644 src/cmd/gc/gen.c create mode 100644 src/cmd/gc/go.errors create mode 100644 src/cmd/gc/go.h create mode 100644 src/cmd/gc/go.y create mode 100644 src/cmd/gc/init.c create mode 100644 src/cmd/gc/lex.c create mode 100644 src/cmd/gc/md5.c create mode 100644 src/cmd/gc/md5.h create mode 100755 src/cmd/gc/mkbuiltin create mode 100644 src/cmd/gc/mkbuiltin1.c create mode 100755 src/cmd/gc/mkopnames create mode 100644 src/cmd/gc/mparith1.c create mode 100644 src/cmd/gc/mparith2.c create mode 100644 src/cmd/gc/mparith3.c create mode 100644 src/cmd/gc/obj.c create mode 100644 src/cmd/gc/pgen.c create mode 100644 src/cmd/gc/print.c create mode 100644 src/cmd/gc/range.c create mode 100644 src/cmd/gc/reflect.c create mode 100644 src/cmd/gc/runtime.go create mode 100644 src/cmd/gc/select.c create mode 100644 src/cmd/gc/sinit.c create mode 100644 src/cmd/gc/subr.c create mode 100644 src/cmd/gc/swt.c create mode 100644 src/cmd/gc/typecheck.c create mode 100644 src/cmd/gc/unsafe.c create mode 100644 src/cmd/gc/unsafe.go create mode 100644 src/cmd/gc/walk.c create mode 100644 src/cmd/godefs/Makefile create mode 100644 src/cmd/godefs/a.h create mode 100644 src/cmd/godefs/doc.go create mode 100644 src/cmd/godefs/main.c create mode 100644 src/cmd/godefs/stabs.c create mode 100755 src/cmd/godefs/test.sh create mode 100644 src/cmd/godefs/testdata.c create mode 100644 src/cmd/godefs/testdata_darwin_386.golden create mode 100644 src/cmd/godefs/testdata_darwin_amd64.golden create mode 100644 src/cmd/godefs/util.c create mode 100644 src/cmd/godoc/Makefile create mode 100644 src/cmd/godoc/appconfig.go create mode 100644 src/cmd/godoc/appinit.go create mode 100644 src/cmd/godoc/codewalk.go create mode 100644 src/cmd/godoc/dirtrees.go create mode 100644 src/cmd/godoc/doc.go create mode 100644 src/cmd/godoc/filesystem.go create mode 100644 src/cmd/godoc/format.go create mode 100644 src/cmd/godoc/godoc.go create mode 100644 src/cmd/godoc/httpzip.go create mode 100644 src/cmd/godoc/index.go create mode 100644 src/cmd/godoc/main.go create mode 100644 src/cmd/godoc/mapping.go create mode 100644 src/cmd/godoc/parser.go create mode 100755 src/cmd/godoc/snippet.go create mode 100644 src/cmd/godoc/spec.go create mode 100644 src/cmd/godoc/utils.go create mode 100644 src/cmd/godoc/zip.go create mode 100644 src/cmd/gofix/Makefile create mode 100644 src/cmd/gofix/doc.go create mode 100644 src/cmd/gofix/filepath.go create mode 100644 src/cmd/gofix/filepath_test.go create mode 100644 src/cmd/gofix/fix.go create mode 100644 src/cmd/gofix/httpfinalurl.go create mode 100644 src/cmd/gofix/httpfinalurl_test.go create mode 100644 src/cmd/gofix/httpfs.go create mode 100644 src/cmd/gofix/httpfs_test.go create mode 100644 src/cmd/gofix/httpheaders.go create mode 100644 src/cmd/gofix/httpheaders_test.go create mode 100644 src/cmd/gofix/httpserver.go create mode 100644 src/cmd/gofix/httpserver_test.go create mode 100644 src/cmd/gofix/main.go create mode 100644 src/cmd/gofix/main_test.go create mode 100644 src/cmd/gofix/netdial.go create mode 100644 src/cmd/gofix/netdial_test.go create mode 100644 src/cmd/gofix/oserrorstring.go create mode 100644 src/cmd/gofix/oserrorstring_test.go create mode 100644 src/cmd/gofix/osopen.go create mode 100644 src/cmd/gofix/osopen_test.go create mode 100644 src/cmd/gofix/procattr.go create mode 100644 src/cmd/gofix/procattr_test.go create mode 100644 src/cmd/gofix/reflect.go create mode 100644 src/cmd/gofix/reflect_test.go create mode 100644 src/cmd/gofix/signal.go create mode 100644 src/cmd/gofix/signal_test.go create mode 100644 src/cmd/gofix/sorthelpers.go create mode 100644 src/cmd/gofix/sorthelpers_test.go create mode 100644 src/cmd/gofix/sortslice.go create mode 100644 src/cmd/gofix/sortslice_test.go create mode 100644 src/cmd/gofix/stringssplit.go create mode 100644 src/cmd/gofix/stringssplit_test.go create mode 100644 src/cmd/gofix/testdata/reflect.asn1.go.in create mode 100644 src/cmd/gofix/testdata/reflect.asn1.go.out create mode 100644 src/cmd/gofix/testdata/reflect.datafmt.go.in create mode 100644 src/cmd/gofix/testdata/reflect.datafmt.go.out create mode 100644 src/cmd/gofix/testdata/reflect.decode.go.in create mode 100644 src/cmd/gofix/testdata/reflect.decode.go.out create mode 100644 src/cmd/gofix/testdata/reflect.decoder.go.in create mode 100644 src/cmd/gofix/testdata/reflect.decoder.go.out create mode 100644 src/cmd/gofix/testdata/reflect.dnsmsg.go.in create mode 100644 src/cmd/gofix/testdata/reflect.dnsmsg.go.out create mode 100644 src/cmd/gofix/testdata/reflect.encode.go.in create mode 100644 src/cmd/gofix/testdata/reflect.encode.go.out create mode 100644 src/cmd/gofix/testdata/reflect.encoder.go.in create mode 100644 src/cmd/gofix/testdata/reflect.encoder.go.out create mode 100644 src/cmd/gofix/testdata/reflect.export.go.in create mode 100644 src/cmd/gofix/testdata/reflect.export.go.out create mode 100644 src/cmd/gofix/testdata/reflect.print.go.in create mode 100644 src/cmd/gofix/testdata/reflect.print.go.out create mode 100644 src/cmd/gofix/testdata/reflect.quick.go.in create mode 100644 src/cmd/gofix/testdata/reflect.quick.go.out create mode 100644 src/cmd/gofix/testdata/reflect.read.go.in create mode 100644 src/cmd/gofix/testdata/reflect.read.go.out create mode 100644 src/cmd/gofix/testdata/reflect.scan.go.in create mode 100644 src/cmd/gofix/testdata/reflect.scan.go.out create mode 100644 src/cmd/gofix/testdata/reflect.script.go.in create mode 100644 src/cmd/gofix/testdata/reflect.script.go.out create mode 100644 src/cmd/gofix/testdata/reflect.template.go.in create mode 100644 src/cmd/gofix/testdata/reflect.template.go.out create mode 100644 src/cmd/gofix/testdata/reflect.type.go.in create mode 100644 src/cmd/gofix/testdata/reflect.type.go.out create mode 100644 src/cmd/gofix/typecheck.go create mode 100644 src/cmd/gofix/url.go create mode 100644 src/cmd/gofix/url_test.go create mode 100644 src/cmd/gofmt/Makefile create mode 100644 src/cmd/gofmt/doc.go create mode 100644 src/cmd/gofmt/gofmt.go create mode 100644 src/cmd/gofmt/gofmt_test.go create mode 100644 src/cmd/gofmt/rewrite.go create mode 100644 src/cmd/gofmt/simplify.go create mode 100755 src/cmd/gofmt/test.sh create mode 100644 src/cmd/gofmt/testdata/composites.golden create mode 100644 src/cmd/gofmt/testdata/composites.input create mode 100644 src/cmd/gofmt/testdata/rewrite1.golden create mode 100644 src/cmd/gofmt/testdata/rewrite1.input create mode 100644 src/cmd/gofmt/testdata/rewrite2.golden create mode 100644 src/cmd/gofmt/testdata/rewrite2.input create mode 100644 src/cmd/goinstall/Makefile create mode 100644 src/cmd/goinstall/doc.go create mode 100644 src/cmd/goinstall/download.go create mode 100644 src/cmd/goinstall/main.go create mode 100644 src/cmd/goinstall/make.go create mode 100644 src/cmd/goinstall/tag_test.go create mode 100644 src/cmd/gomake/doc.go create mode 100644 src/cmd/gopack/Makefile create mode 100644 src/cmd/gopack/ar.c create mode 100644 src/cmd/gopack/doc.go create mode 100644 src/cmd/gotest/Makefile create mode 100644 src/cmd/gotest/doc.go create mode 100644 src/cmd/gotest/flag.go create mode 100644 src/cmd/gotest/gotest.go create mode 100644 src/cmd/gotry/Makefile create mode 100755 src/cmd/gotry/gotry create mode 100644 src/cmd/gotype/Makefile create mode 100644 src/cmd/gotype/doc.go create mode 100644 src/cmd/gotype/gotype.go create mode 100644 src/cmd/gotype/gotype_test.go create mode 100644 src/cmd/gotype/testdata/test1.go create mode 100644 src/cmd/govet/Makefile create mode 100644 src/cmd/govet/doc.go create mode 100644 src/cmd/govet/govet.go create mode 100644 src/cmd/goyacc/Makefile create mode 100644 src/cmd/goyacc/doc.go create mode 100644 src/cmd/goyacc/goyacc.go create mode 100644 src/cmd/goyacc/units.txt create mode 100644 src/cmd/goyacc/units.y create mode 100644 src/cmd/hgpatch/Makefile create mode 100644 src/cmd/hgpatch/doc.go create mode 100644 src/cmd/hgpatch/main.go create mode 100644 src/cmd/ld/data.c create mode 100644 src/cmd/ld/doc.go create mode 100644 src/cmd/ld/dwarf.c create mode 100644 src/cmd/ld/dwarf.h create mode 100644 src/cmd/ld/dwarf_defs.h create mode 100644 src/cmd/ld/elf.c create mode 100644 src/cmd/ld/elf.h create mode 100644 src/cmd/ld/go.c create mode 100644 src/cmd/ld/ldelf.c create mode 100644 src/cmd/ld/ldmacho.c create mode 100644 src/cmd/ld/ldpe.c create mode 100644 src/cmd/ld/lib.c create mode 100644 src/cmd/ld/lib.h create mode 100644 src/cmd/ld/macho.c create mode 100644 src/cmd/ld/macho.h create mode 100644 src/cmd/ld/pe.c create mode 100644 src/cmd/ld/pe.h create mode 100644 src/cmd/ld/symtab.c create mode 100644 src/cmd/nm/Makefile create mode 100644 src/cmd/nm/doc.go create mode 100644 src/cmd/nm/nm.c create mode 100644 src/cmd/prof/Makefile create mode 100644 src/cmd/prof/doc.go create mode 100755 src/cmd/prof/gopprof create mode 100644 src/cmd/prof/main.c create mode 100644 src/env.bash create mode 100644 src/lib9/Makefile create mode 100644 src/lib9/_exits.c create mode 100644 src/lib9/_p9dir.c create mode 100644 src/lib9/argv0.c create mode 100644 src/lib9/atoi.c create mode 100644 src/lib9/await.c create mode 100644 src/lib9/cleanname.c create mode 100644 src/lib9/create.c create mode 100644 src/lib9/dirfstat.c create mode 100644 src/lib9/dirfwstat.c create mode 100644 src/lib9/dirstat.c create mode 100644 src/lib9/dirwstat.c create mode 100644 src/lib9/dup.c create mode 100644 src/lib9/errstr.c create mode 100644 src/lib9/exec.c create mode 100644 src/lib9/execl.c create mode 100644 src/lib9/exitcode.c create mode 100644 src/lib9/exits.c create mode 100644 src/lib9/fmt/charstod.c create mode 100644 src/lib9/fmt/dofmt.c create mode 100644 src/lib9/fmt/dorfmt.c create mode 100644 src/lib9/fmt/errfmt.c create mode 100644 src/lib9/fmt/fltfmt.c create mode 100644 src/lib9/fmt/fmt.c create mode 100644 src/lib9/fmt/fmtdef.h create mode 100644 src/lib9/fmt/fmtfd.c create mode 100644 src/lib9/fmt/fmtfdflush.c create mode 100644 src/lib9/fmt/fmtlocale.c create mode 100644 src/lib9/fmt/fmtlock.c create mode 100644 src/lib9/fmt/fmtnull.c create mode 100644 src/lib9/fmt/fmtprint.c create mode 100644 src/lib9/fmt/fmtquote.c create mode 100644 src/lib9/fmt/fmtrune.c create mode 100644 src/lib9/fmt/fmtstr.c create mode 100644 src/lib9/fmt/fmtvprint.c create mode 100644 src/lib9/fmt/fprint.c create mode 100644 src/lib9/fmt/nan64.c create mode 100644 src/lib9/fmt/pow10.c create mode 100644 src/lib9/fmt/print.c create mode 100644 src/lib9/fmt/seprint.c create mode 100644 src/lib9/fmt/smprint.c create mode 100644 src/lib9/fmt/snprint.c create mode 100644 src/lib9/fmt/sprint.c create mode 100644 src/lib9/fmt/strtod.c create mode 100644 src/lib9/fmt/test.c create mode 100644 src/lib9/fmt/vfprint.c create mode 100644 src/lib9/fmt/vseprint.c create mode 100644 src/lib9/fmt/vsmprint.c create mode 100644 src/lib9/fmt/vsnprint.c create mode 100644 src/lib9/fmtlock2.c create mode 100644 src/lib9/fork.c create mode 100644 src/lib9/getenv.c create mode 100644 src/lib9/getfields.c create mode 100644 src/lib9/getuser.c create mode 100644 src/lib9/getwd.c create mode 100644 src/lib9/goos.c create mode 100644 src/lib9/jmp.c create mode 100644 src/lib9/main.c create mode 100644 src/lib9/nan.c create mode 100644 src/lib9/notify.c create mode 100644 src/lib9/nulldir.c create mode 100644 src/lib9/open.c create mode 100644 src/lib9/readn.c create mode 100644 src/lib9/rfork.c create mode 100644 src/lib9/seek.c create mode 100644 src/lib9/strecpy.c create mode 100644 src/lib9/sysfatal.c create mode 100644 src/lib9/time.c create mode 100644 src/lib9/tokenize.c create mode 100644 src/lib9/utf/Makefile create mode 100644 src/lib9/utf/mkrunetype.c create mode 100644 src/lib9/utf/rune.c create mode 100644 src/lib9/utf/runetype.c create mode 100644 src/lib9/utf/runetypebody-5.0.0.c create mode 100644 src/lib9/utf/runetypebody-5.2.0.c create mode 100644 src/lib9/utf/runetypebody-6.0.0.c create mode 100644 src/lib9/utf/utf.h create mode 100644 src/lib9/utf/utfdef.h create mode 100644 src/lib9/utf/utfecpy.c create mode 100644 src/lib9/utf/utflen.c create mode 100644 src/lib9/utf/utfnlen.c create mode 100644 src/lib9/utf/utfrrune.c create mode 100644 src/lib9/utf/utfrune.c create mode 100644 src/lib9/utf/utfutf.c create mode 100644 src/lib9/win32.c create mode 100644 src/libbio/Makefile create mode 100644 src/libbio/bbuffered.c create mode 100644 src/libbio/bfildes.c create mode 100644 src/libbio/bflush.c create mode 100644 src/libbio/bgetc.c create mode 100644 src/libbio/bgetd.c create mode 100644 src/libbio/bgetrune.c create mode 100644 src/libbio/binit.c create mode 100644 src/libbio/boffset.c create mode 100644 src/libbio/bprint.c create mode 100644 src/libbio/bputc.c create mode 100644 src/libbio/bputrune.c create mode 100644 src/libbio/brdline.c create mode 100644 src/libbio/brdstr.c create mode 100644 src/libbio/bread.c create mode 100644 src/libbio/bseek.c create mode 100644 src/libbio/bwrite.c create mode 100644 src/libmach/5.c create mode 100644 src/libmach/5db.c create mode 100644 src/libmach/5obj.c create mode 100644 src/libmach/6.c create mode 100644 src/libmach/6obj.c create mode 100644 src/libmach/8.c create mode 100644 src/libmach/8db.c create mode 100644 src/libmach/8obj.c create mode 100644 src/libmach/Makefile create mode 100644 src/libmach/access.c create mode 100644 src/libmach/darwin.c create mode 100644 src/libmach/elf.h create mode 100644 src/libmach/executable.c create mode 100644 src/libmach/fakeobj.c create mode 100644 src/libmach/freebsd.c create mode 100644 src/libmach/linux.c create mode 100644 src/libmach/machdata.c create mode 100644 src/libmach/macho.h create mode 100644 src/libmach/map.c create mode 100644 src/libmach/obj.c create mode 100644 src/libmach/obj.h create mode 100644 src/libmach/openbsd.c create mode 100644 src/libmach/setmach.c create mode 100644 src/libmach/swap.c create mode 100644 src/libmach/sym.c create mode 100644 src/libmach/windows.c create mode 100755 src/make.bash create mode 100644 src/pkg/Makefile create mode 100644 src/pkg/archive/tar/Makefile create mode 100644 src/pkg/archive/tar/common.go create mode 100644 src/pkg/archive/tar/reader.go create mode 100644 src/pkg/archive/tar/reader_test.go create mode 100644 src/pkg/archive/tar/testdata/gnu.tar create mode 100644 src/pkg/archive/tar/testdata/small.txt create mode 100644 src/pkg/archive/tar/testdata/small2.txt create mode 100644 src/pkg/archive/tar/testdata/star.tar create mode 100644 src/pkg/archive/tar/testdata/v7.tar create mode 100644 src/pkg/archive/tar/testdata/writer-big.tar create mode 100644 src/pkg/archive/tar/testdata/writer.tar create mode 100644 src/pkg/archive/tar/writer.go create mode 100644 src/pkg/archive/tar/writer_test.go create mode 100644 src/pkg/archive/zip/Makefile create mode 100644 src/pkg/archive/zip/reader.go create mode 100644 src/pkg/archive/zip/reader_test.go create mode 100644 src/pkg/archive/zip/struct.go create mode 100644 src/pkg/archive/zip/testdata/dd.zip create mode 100644 src/pkg/archive/zip/testdata/gophercolor16x16.png create mode 100644 src/pkg/archive/zip/testdata/r.zip create mode 100644 src/pkg/archive/zip/testdata/readme.notzip create mode 100644 src/pkg/archive/zip/testdata/readme.zip create mode 100644 src/pkg/archive/zip/testdata/test.zip create mode 100644 src/pkg/archive/zip/writer.go create mode 100644 src/pkg/archive/zip/writer_test.go create mode 100644 src/pkg/archive/zip/zip_test.go create mode 100644 src/pkg/asn1/Makefile create mode 100644 src/pkg/asn1/asn1.go create mode 100644 src/pkg/asn1/asn1_test.go create mode 100644 src/pkg/asn1/common.go create mode 100644 src/pkg/asn1/marshal.go create mode 100644 src/pkg/asn1/marshal_test.go create mode 100644 src/pkg/big/Makefile create mode 100644 src/pkg/big/arith.go create mode 100644 src/pkg/big/arith_386.s create mode 100644 src/pkg/big/arith_amd64.s create mode 100644 src/pkg/big/arith_arm.s create mode 100644 src/pkg/big/arith_decl.go create mode 100644 src/pkg/big/arith_test.go create mode 100644 src/pkg/big/calibrate_test.go create mode 100644 src/pkg/big/hilbert_test.go create mode 100755 src/pkg/big/int.go create mode 100755 src/pkg/big/int_test.go create mode 100755 src/pkg/big/nat.go create mode 100755 src/pkg/big/nat_test.go create mode 100644 src/pkg/big/rat.go create mode 100644 src/pkg/big/rat_test.go create mode 100644 src/pkg/bufio/Makefile create mode 100644 src/pkg/bufio/bufio.go create mode 100644 src/pkg/bufio/bufio_test.go create mode 100644 src/pkg/builtin/builtin.go create mode 100644 src/pkg/bytes/Makefile create mode 100644 src/pkg/bytes/asm_386.s create mode 100644 src/pkg/bytes/asm_amd64.s create mode 100644 src/pkg/bytes/asm_arm.s create mode 100644 src/pkg/bytes/buffer.go create mode 100644 src/pkg/bytes/buffer_test.go create mode 100644 src/pkg/bytes/bytes.go create mode 100644 src/pkg/bytes/bytes_decl.go create mode 100644 src/pkg/bytes/bytes_test.go create mode 100644 src/pkg/bytes/export_test.go create mode 100644 src/pkg/cmath/Makefile create mode 100644 src/pkg/cmath/abs.go create mode 100644 src/pkg/cmath/asin.go create mode 100644 src/pkg/cmath/cmath_test.go create mode 100644 src/pkg/cmath/conj.go create mode 100644 src/pkg/cmath/exp.go create mode 100644 src/pkg/cmath/isinf.go create mode 100644 src/pkg/cmath/isnan.go create mode 100644 src/pkg/cmath/log.go create mode 100644 src/pkg/cmath/phase.go create mode 100644 src/pkg/cmath/polar.go create mode 100644 src/pkg/cmath/pow.go create mode 100644 src/pkg/cmath/rect.go create mode 100644 src/pkg/cmath/sin.go create mode 100644 src/pkg/cmath/sqrt.go create mode 100644 src/pkg/cmath/tan.go create mode 100644 src/pkg/compress/bzip2/Makefile create mode 100644 src/pkg/compress/bzip2/bit_reader.go create mode 100644 src/pkg/compress/bzip2/bzip2.go create mode 100644 src/pkg/compress/bzip2/bzip2_test.go create mode 100644 src/pkg/compress/bzip2/huffman.go create mode 100644 src/pkg/compress/bzip2/move_to_front.go create mode 100644 src/pkg/compress/flate/Makefile create mode 100644 src/pkg/compress/flate/deflate.go create mode 100644 src/pkg/compress/flate/deflate_test.go create mode 100644 src/pkg/compress/flate/flate_test.go create mode 100644 src/pkg/compress/flate/huffman_bit_writer.go create mode 100644 src/pkg/compress/flate/huffman_code.go create mode 100644 src/pkg/compress/flate/inflate.go create mode 100644 src/pkg/compress/flate/reverse_bits.go create mode 100644 src/pkg/compress/flate/token.go create mode 100644 src/pkg/compress/flate/util.go create mode 100644 src/pkg/compress/gzip/Makefile create mode 100644 src/pkg/compress/gzip/gunzip.go create mode 100644 src/pkg/compress/gzip/gunzip_test.go create mode 100644 src/pkg/compress/gzip/gzip.go create mode 100644 src/pkg/compress/gzip/gzip_test.go create mode 100644 src/pkg/compress/lzw/Makefile create mode 100644 src/pkg/compress/lzw/reader.go create mode 100644 src/pkg/compress/lzw/reader_test.go create mode 100644 src/pkg/compress/lzw/writer.go create mode 100644 src/pkg/compress/lzw/writer_test.go create mode 100644 src/pkg/compress/testdata/e.txt create mode 100644 src/pkg/compress/testdata/pi.txt create mode 100644 src/pkg/compress/zlib/Makefile create mode 100644 src/pkg/compress/zlib/reader.go create mode 100644 src/pkg/compress/zlib/reader_test.go create mode 100644 src/pkg/compress/zlib/writer.go create mode 100644 src/pkg/compress/zlib/writer_test.go create mode 100644 src/pkg/container/heap/Makefile create mode 100644 src/pkg/container/heap/heap.go create mode 100644 src/pkg/container/heap/heap_test.go create mode 100644 src/pkg/container/list/Makefile create mode 100644 src/pkg/container/list/list.go create mode 100644 src/pkg/container/list/list_test.go create mode 100644 src/pkg/container/ring/Makefile create mode 100644 src/pkg/container/ring/ring.go create mode 100644 src/pkg/container/ring/ring_test.go create mode 100644 src/pkg/container/vector/Makefile create mode 100644 src/pkg/container/vector/defs.go create mode 100644 src/pkg/container/vector/intvector.go create mode 100644 src/pkg/container/vector/intvector_test.go create mode 100644 src/pkg/container/vector/nogen_test.go create mode 100644 src/pkg/container/vector/numbers_test.go create mode 100644 src/pkg/container/vector/stringvector.go create mode 100644 src/pkg/container/vector/stringvector_test.go create mode 100644 src/pkg/container/vector/vector.go create mode 100644 src/pkg/container/vector/vector_test.go create mode 100644 src/pkg/crypto/Makefile create mode 100644 src/pkg/crypto/aes/Makefile create mode 100644 src/pkg/crypto/aes/aes_test.go create mode 100644 src/pkg/crypto/aes/block.go create mode 100644 src/pkg/crypto/aes/cipher.go create mode 100644 src/pkg/crypto/aes/const.go create mode 100644 src/pkg/crypto/blowfish/Makefile create mode 100644 src/pkg/crypto/blowfish/block.go create mode 100644 src/pkg/crypto/blowfish/blowfish_test.go create mode 100644 src/pkg/crypto/blowfish/cipher.go create mode 100644 src/pkg/crypto/blowfish/const.go create mode 100644 src/pkg/crypto/cast5/Makefile create mode 100644 src/pkg/crypto/cast5/cast5.go create mode 100644 src/pkg/crypto/cast5/cast5_test.go create mode 100644 src/pkg/crypto/cipher/Makefile create mode 100644 src/pkg/crypto/cipher/cbc.go create mode 100644 src/pkg/crypto/cipher/cbc_aes_test.go create mode 100644 src/pkg/crypto/cipher/cfb.go create mode 100644 src/pkg/crypto/cipher/cfb_test.go create mode 100644 src/pkg/crypto/cipher/cipher.go create mode 100644 src/pkg/crypto/cipher/common_test.go create mode 100644 src/pkg/crypto/cipher/ctr.go create mode 100644 src/pkg/crypto/cipher/ctr_aes_test.go create mode 100644 src/pkg/crypto/cipher/io.go create mode 100644 src/pkg/crypto/cipher/ocfb.go create mode 100644 src/pkg/crypto/cipher/ocfb_test.go create mode 100644 src/pkg/crypto/cipher/ofb.go create mode 100644 src/pkg/crypto/cipher/ofb_test.go create mode 100644 src/pkg/crypto/crypto.go create mode 100644 src/pkg/crypto/des/Makefile create mode 100644 src/pkg/crypto/des/block.go create mode 100644 src/pkg/crypto/des/cipher.go create mode 100644 src/pkg/crypto/des/const.go create mode 100644 src/pkg/crypto/des/des_test.go create mode 100644 src/pkg/crypto/dsa/Makefile create mode 100644 src/pkg/crypto/dsa/dsa.go create mode 100644 src/pkg/crypto/dsa/dsa_test.go create mode 100644 src/pkg/crypto/ecdsa/Makefile create mode 100644 src/pkg/crypto/ecdsa/ecdsa.go create mode 100644 src/pkg/crypto/ecdsa/ecdsa_test.go create mode 100644 src/pkg/crypto/elliptic/Makefile create mode 100644 src/pkg/crypto/elliptic/elliptic.go create mode 100644 src/pkg/crypto/elliptic/elliptic_test.go create mode 100644 src/pkg/crypto/hmac/Makefile create mode 100644 src/pkg/crypto/hmac/hmac.go create mode 100644 src/pkg/crypto/hmac/hmac_test.go create mode 100644 src/pkg/crypto/md4/Makefile create mode 100644 src/pkg/crypto/md4/md4.go create mode 100644 src/pkg/crypto/md4/md4_test.go create mode 100644 src/pkg/crypto/md4/md4block.go create mode 100644 src/pkg/crypto/md5/Makefile create mode 100644 src/pkg/crypto/md5/md5.go create mode 100644 src/pkg/crypto/md5/md5_test.go create mode 100644 src/pkg/crypto/md5/md5block.go create mode 100644 src/pkg/crypto/ocsp/Makefile create mode 100644 src/pkg/crypto/ocsp/ocsp.go create mode 100644 src/pkg/crypto/ocsp/ocsp_test.go create mode 100644 src/pkg/crypto/openpgp/Makefile create mode 100644 src/pkg/crypto/openpgp/armor/Makefile create mode 100644 src/pkg/crypto/openpgp/armor/armor.go create mode 100644 src/pkg/crypto/openpgp/armor/armor_test.go create mode 100644 src/pkg/crypto/openpgp/armor/encode.go create mode 100644 src/pkg/crypto/openpgp/canonical_text.go create mode 100644 src/pkg/crypto/openpgp/canonical_text_test.go create mode 100644 src/pkg/crypto/openpgp/elgamal/Makefile create mode 100644 src/pkg/crypto/openpgp/elgamal/elgamal.go create mode 100644 src/pkg/crypto/openpgp/elgamal/elgamal_test.go create mode 100644 src/pkg/crypto/openpgp/error/Makefile create mode 100644 src/pkg/crypto/openpgp/error/error.go create mode 100644 src/pkg/crypto/openpgp/keys.go create mode 100644 src/pkg/crypto/openpgp/packet/Makefile create mode 100644 src/pkg/crypto/openpgp/packet/compressed.go create mode 100644 src/pkg/crypto/openpgp/packet/compressed_test.go create mode 100644 src/pkg/crypto/openpgp/packet/encrypted_key.go create mode 100644 src/pkg/crypto/openpgp/packet/encrypted_key_test.go create mode 100644 src/pkg/crypto/openpgp/packet/literal.go create mode 100644 src/pkg/crypto/openpgp/packet/one_pass_signature.go create mode 100644 src/pkg/crypto/openpgp/packet/packet.go create mode 100644 src/pkg/crypto/openpgp/packet/packet_test.go create mode 100644 src/pkg/crypto/openpgp/packet/private_key.go create mode 100644 src/pkg/crypto/openpgp/packet/private_key_test.go create mode 100644 src/pkg/crypto/openpgp/packet/public_key.go create mode 100644 src/pkg/crypto/openpgp/packet/public_key_test.go create mode 100644 src/pkg/crypto/openpgp/packet/reader.go create mode 100644 src/pkg/crypto/openpgp/packet/signature.go create mode 100644 src/pkg/crypto/openpgp/packet/signature_test.go create mode 100644 src/pkg/crypto/openpgp/packet/symmetric_key_encrypted.go create mode 100644 src/pkg/crypto/openpgp/packet/symmetric_key_encrypted_test.go create mode 100644 src/pkg/crypto/openpgp/packet/symmetrically_encrypted.go create mode 100644 src/pkg/crypto/openpgp/packet/symmetrically_encrypted_test.go create mode 100644 src/pkg/crypto/openpgp/packet/userid.go create mode 100644 src/pkg/crypto/openpgp/packet/userid_test.go create mode 100644 src/pkg/crypto/openpgp/read.go create mode 100644 src/pkg/crypto/openpgp/read_test.go create mode 100644 src/pkg/crypto/openpgp/s2k/Makefile create mode 100644 src/pkg/crypto/openpgp/s2k/s2k.go create mode 100644 src/pkg/crypto/openpgp/s2k/s2k_test.go create mode 100644 src/pkg/crypto/openpgp/write.go create mode 100644 src/pkg/crypto/openpgp/write_test.go create mode 100644 src/pkg/crypto/rand/Makefile create mode 100644 src/pkg/crypto/rand/rand.go create mode 100644 src/pkg/crypto/rand/rand_test.go create mode 100644 src/pkg/crypto/rand/rand_unix.go create mode 100755 src/pkg/crypto/rand/rand_windows.go create mode 100644 src/pkg/crypto/rand/util.go create mode 100644 src/pkg/crypto/rc4/Makefile create mode 100644 src/pkg/crypto/rc4/rc4.go create mode 100644 src/pkg/crypto/rc4/rc4_test.go create mode 100644 src/pkg/crypto/ripemd160/Makefile create mode 100644 src/pkg/crypto/ripemd160/ripemd160.go create mode 100644 src/pkg/crypto/ripemd160/ripemd160_test.go create mode 100644 src/pkg/crypto/ripemd160/ripemd160block.go create mode 100644 src/pkg/crypto/rsa/Makefile create mode 100644 src/pkg/crypto/rsa/pkcs1v15.go create mode 100644 src/pkg/crypto/rsa/pkcs1v15_test.go create mode 100644 src/pkg/crypto/rsa/rsa.go create mode 100644 src/pkg/crypto/rsa/rsa_test.go create mode 100644 src/pkg/crypto/sha1/Makefile create mode 100644 src/pkg/crypto/sha1/sha1.go create mode 100644 src/pkg/crypto/sha1/sha1_test.go create mode 100644 src/pkg/crypto/sha1/sha1block.go create mode 100644 src/pkg/crypto/sha256/Makefile create mode 100644 src/pkg/crypto/sha256/sha256.go create mode 100644 src/pkg/crypto/sha256/sha256_test.go create mode 100644 src/pkg/crypto/sha256/sha256block.go create mode 100644 src/pkg/crypto/sha512/Makefile create mode 100644 src/pkg/crypto/sha512/sha512.go create mode 100644 src/pkg/crypto/sha512/sha512_test.go create mode 100644 src/pkg/crypto/sha512/sha512block.go create mode 100644 src/pkg/crypto/subtle/Makefile create mode 100644 src/pkg/crypto/subtle/constant_time.go create mode 100644 src/pkg/crypto/subtle/constant_time_test.go create mode 100644 src/pkg/crypto/tls/Makefile create mode 100644 src/pkg/crypto/tls/alert.go create mode 100644 src/pkg/crypto/tls/cipher_suites.go create mode 100644 src/pkg/crypto/tls/common.go create mode 100644 src/pkg/crypto/tls/conn.go create mode 100644 src/pkg/crypto/tls/conn_test.go create mode 100644 src/pkg/crypto/tls/generate_cert.go create mode 100644 src/pkg/crypto/tls/handshake_client.go create mode 100644 src/pkg/crypto/tls/handshake_client_test.go create mode 100644 src/pkg/crypto/tls/handshake_messages.go create mode 100644 src/pkg/crypto/tls/handshake_messages_test.go create mode 100644 src/pkg/crypto/tls/handshake_server.go create mode 100644 src/pkg/crypto/tls/handshake_server_test.go create mode 100644 src/pkg/crypto/tls/key_agreement.go create mode 100644 src/pkg/crypto/tls/parse-gnutls-cli-debug-log.py create mode 100644 src/pkg/crypto/tls/prf.go create mode 100644 src/pkg/crypto/tls/prf_test.go create mode 100644 src/pkg/crypto/tls/tls.go create mode 100644 src/pkg/crypto/twofish/Makefile create mode 100644 src/pkg/crypto/twofish/twofish.go create mode 100644 src/pkg/crypto/twofish/twofish_test.go create mode 100644 src/pkg/crypto/x509/Makefile create mode 100644 src/pkg/crypto/x509/cert_pool.go create mode 100644 src/pkg/crypto/x509/pkix/Makefile create mode 100644 src/pkg/crypto/x509/pkix/pkix.go create mode 100644 src/pkg/crypto/x509/verify.go create mode 100644 src/pkg/crypto/x509/verify_test.go create mode 100644 src/pkg/crypto/x509/x509.go create mode 100644 src/pkg/crypto/x509/x509_test.go create mode 100644 src/pkg/crypto/xtea/Makefile create mode 100644 src/pkg/crypto/xtea/block.go create mode 100644 src/pkg/crypto/xtea/cipher.go create mode 100644 src/pkg/crypto/xtea/xtea_test.go create mode 100644 src/pkg/csv/Makefile create mode 100644 src/pkg/csv/reader.go create mode 100644 src/pkg/csv/reader_test.go create mode 100644 src/pkg/csv/writer.go create mode 100644 src/pkg/csv/writer_test.go create mode 100644 src/pkg/debug/dwarf/Makefile create mode 100644 src/pkg/debug/dwarf/buf.go create mode 100644 src/pkg/debug/dwarf/const.go create mode 100644 src/pkg/debug/dwarf/entry.go create mode 100644 src/pkg/debug/dwarf/open.go create mode 100644 src/pkg/debug/dwarf/testdata/typedef.c create mode 100755 src/pkg/debug/dwarf/testdata/typedef.elf create mode 100644 src/pkg/debug/dwarf/testdata/typedef.macho create mode 100644 src/pkg/debug/dwarf/type.go create mode 100644 src/pkg/debug/dwarf/type_test.go create mode 100644 src/pkg/debug/dwarf/unit.go create mode 100644 src/pkg/debug/elf/Makefile create mode 100644 src/pkg/debug/elf/elf.go create mode 100644 src/pkg/debug/elf/elf_test.go create mode 100644 src/pkg/debug/elf/file.go create mode 100644 src/pkg/debug/elf/file_test.go create mode 100755 src/pkg/debug/elf/testdata/gcc-386-freebsd-exec create mode 100755 src/pkg/debug/elf/testdata/gcc-amd64-linux-exec create mode 100644 src/pkg/debug/elf/testdata/go-relocation-test-gcc424-x86-64.obj create mode 100644 src/pkg/debug/elf/testdata/go-relocation-test-gcc441-x86-64.obj create mode 100644 src/pkg/debug/elf/testdata/go-relocation-test-gcc441-x86.obj create mode 100644 src/pkg/debug/gosym/Makefile create mode 100644 src/pkg/debug/gosym/pclinetest.h create mode 100644 src/pkg/debug/gosym/pclinetest.s create mode 100644 src/pkg/debug/gosym/pclntab.go create mode 100644 src/pkg/debug/gosym/pclntab_test.go create mode 100644 src/pkg/debug/gosym/symtab.go create mode 100644 src/pkg/debug/macho/Makefile create mode 100644 src/pkg/debug/macho/file.go create mode 100644 src/pkg/debug/macho/file_test.go create mode 100644 src/pkg/debug/macho/macho.go create mode 100755 src/pkg/debug/macho/testdata/gcc-386-darwin-exec create mode 100755 src/pkg/debug/macho/testdata/gcc-amd64-darwin-exec create mode 100644 src/pkg/debug/macho/testdata/gcc-amd64-darwin-exec-debug create mode 100644 src/pkg/debug/macho/testdata/hello.c create mode 100644 src/pkg/debug/pe/Makefile create mode 100644 src/pkg/debug/pe/file.go create mode 100644 src/pkg/debug/pe/file_test.go create mode 100644 src/pkg/debug/pe/pe.go create mode 100644 src/pkg/debug/pe/testdata/gcc-386-mingw-exec create mode 100644 src/pkg/debug/pe/testdata/gcc-386-mingw-obj create mode 100644 src/pkg/debug/pe/testdata/hello.c create mode 100755 src/pkg/deps.bash create mode 100644 src/pkg/ebnf/Makefile create mode 100644 src/pkg/ebnf/ebnf.go create mode 100644 src/pkg/ebnf/ebnf_test.go create mode 100644 src/pkg/ebnf/parser.go create mode 100644 src/pkg/encoding/ascii85/Makefile create mode 100644 src/pkg/encoding/ascii85/ascii85.go create mode 100644 src/pkg/encoding/ascii85/ascii85_test.go create mode 100644 src/pkg/encoding/base32/Makefile create mode 100644 src/pkg/encoding/base32/base32.go create mode 100644 src/pkg/encoding/base32/base32_test.go create mode 100644 src/pkg/encoding/base64/Makefile create mode 100644 src/pkg/encoding/base64/base64.go create mode 100644 src/pkg/encoding/base64/base64_test.go create mode 100644 src/pkg/encoding/binary/Makefile create mode 100644 src/pkg/encoding/binary/binary.go create mode 100644 src/pkg/encoding/binary/binary_test.go create mode 100644 src/pkg/encoding/git85/Makefile create mode 100644 src/pkg/encoding/git85/git.go create mode 100644 src/pkg/encoding/git85/git_test.go create mode 100644 src/pkg/encoding/hex/Makefile create mode 100644 src/pkg/encoding/hex/hex.go create mode 100644 src/pkg/encoding/hex/hex_test.go create mode 100644 src/pkg/encoding/pem/Makefile create mode 100644 src/pkg/encoding/pem/pem.go create mode 100644 src/pkg/encoding/pem/pem_test.go create mode 100644 src/pkg/exec/Makefile create mode 100644 src/pkg/exec/exec.go create mode 100644 src/pkg/exec/exec_test.go create mode 100644 src/pkg/exec/lp_plan9.go create mode 100644 src/pkg/exec/lp_test.go create mode 100644 src/pkg/exec/lp_unix.go create mode 100644 src/pkg/exec/lp_windows.go create mode 100644 src/pkg/exp/README create mode 100644 src/pkg/exp/datafmt/Makefile create mode 100644 src/pkg/exp/datafmt/datafmt.go create mode 100644 src/pkg/exp/datafmt/datafmt_test.go create mode 100644 src/pkg/exp/datafmt/parser.go create mode 100644 src/pkg/exp/gui/Makefile create mode 100644 src/pkg/exp/gui/gui.go create mode 100644 src/pkg/exp/gui/x11/Makefile create mode 100644 src/pkg/exp/gui/x11/auth.go create mode 100644 src/pkg/exp/gui/x11/conn.go create mode 100644 src/pkg/exp/norm/Makefile create mode 100644 src/pkg/exp/norm/composition.go create mode 100644 src/pkg/exp/norm/composition_test.go create mode 100644 src/pkg/exp/norm/forminfo.go create mode 100644 src/pkg/exp/norm/maketables.go create mode 100644 src/pkg/exp/norm/maketesttables.go create mode 100644 src/pkg/exp/norm/norm_test.go create mode 100644 src/pkg/exp/norm/normalize.go create mode 100644 src/pkg/exp/norm/tables.go create mode 100644 src/pkg/exp/norm/trie.go create mode 100644 src/pkg/exp/norm/trie_test.go create mode 100644 src/pkg/exp/norm/triedata_test.go create mode 100644 src/pkg/exp/norm/triegen.go create mode 100644 src/pkg/exp/regexp/Makefile create mode 100644 src/pkg/exp/regexp/all_test.go create mode 100644 src/pkg/exp/regexp/exec.go create mode 100644 src/pkg/exp/regexp/find_test.go create mode 100644 src/pkg/exp/regexp/regexp.go create mode 100644 src/pkg/exp/regexp/syntax/Makefile create mode 100644 src/pkg/exp/regexp/syntax/compile.go create mode 100755 src/pkg/exp/regexp/syntax/make_perl_groups.pl create mode 100644 src/pkg/exp/regexp/syntax/parse.go create mode 100644 src/pkg/exp/regexp/syntax/parse_test.go create mode 100644 src/pkg/exp/regexp/syntax/perl_groups.go create mode 100644 src/pkg/exp/regexp/syntax/prog.go create mode 100644 src/pkg/exp/regexp/syntax/prog_test.go create mode 100644 src/pkg/exp/regexp/syntax/regexp.go create mode 100644 src/pkg/exp/regexp/syntax/simplify.go create mode 100644 src/pkg/exp/regexp/syntax/simplify_test.go create mode 100644 src/pkg/exp/template/html/Makefile create mode 100644 src/pkg/exp/template/html/context.go create mode 100644 src/pkg/exp/template/html/escape.go create mode 100644 src/pkg/exp/template/html/escape_test.go create mode 100644 src/pkg/exp/wingui/Makefile create mode 100644 src/pkg/exp/wingui/gui.go create mode 100644 src/pkg/exp/wingui/winapi.go create mode 100644 src/pkg/exp/wingui/zwinapi.go create mode 100644 src/pkg/expvar/Makefile create mode 100644 src/pkg/expvar/expvar.go create mode 100644 src/pkg/expvar/expvar_test.go create mode 100644 src/pkg/flag/Makefile create mode 100644 src/pkg/flag/export_test.go create mode 100644 src/pkg/flag/flag.go create mode 100644 src/pkg/flag/flag_test.go create mode 100644 src/pkg/fmt/Makefile create mode 100644 src/pkg/fmt/doc.go create mode 100644 src/pkg/fmt/fmt_test.go create mode 100644 src/pkg/fmt/format.go create mode 100644 src/pkg/fmt/print.go create mode 100644 src/pkg/fmt/scan.go create mode 100644 src/pkg/fmt/scan_test.go create mode 100644 src/pkg/fmt/stringer_test.go create mode 100644 src/pkg/go/ast/Makefile create mode 100644 src/pkg/go/ast/ast.go create mode 100644 src/pkg/go/ast/filter.go create mode 100644 src/pkg/go/ast/print.go create mode 100644 src/pkg/go/ast/print_test.go create mode 100644 src/pkg/go/ast/resolve.go create mode 100644 src/pkg/go/ast/scope.go create mode 100644 src/pkg/go/ast/walk.go create mode 100644 src/pkg/go/build/Makefile create mode 100644 src/pkg/go/build/build.go create mode 100644 src/pkg/go/build/build_test.go create mode 100644 src/pkg/go/build/cgotest/cgotest.c create mode 100644 src/pkg/go/build/cgotest/cgotest.go create mode 100644 src/pkg/go/build/cgotest/cgotest.h create mode 100644 src/pkg/go/build/cmdtest/main.go create mode 100644 src/pkg/go/build/dir.go create mode 100644 src/pkg/go/build/path.go create mode 100644 src/pkg/go/build/pkgtest/pkgtest.go create mode 100644 src/pkg/go/build/pkgtest/sqrt_386.s create mode 100644 src/pkg/go/build/pkgtest/sqrt_amd64.s create mode 100644 src/pkg/go/build/pkgtest/sqrt_arm.s create mode 100644 src/pkg/go/build/syslist_test.go create mode 100644 src/pkg/go/doc/Makefile create mode 100644 src/pkg/go/doc/comment.go create mode 100644 src/pkg/go/doc/doc.go create mode 100644 src/pkg/go/parser/Makefile create mode 100644 src/pkg/go/parser/interface.go create mode 100644 src/pkg/go/parser/parser.go create mode 100644 src/pkg/go/parser/parser_test.go create mode 100644 src/pkg/go/printer/Makefile create mode 100644 src/pkg/go/printer/nodes.go create mode 100644 src/pkg/go/printer/performance_test.go create mode 100644 src/pkg/go/printer/printer.go create mode 100644 src/pkg/go/printer/printer_test.go create mode 100644 src/pkg/go/printer/testdata/comments.golden create mode 100644 src/pkg/go/printer/testdata/comments.input create mode 100644 src/pkg/go/printer/testdata/comments.x create mode 100644 src/pkg/go/printer/testdata/declarations.golden create mode 100644 src/pkg/go/printer/testdata/declarations.input create mode 100644 src/pkg/go/printer/testdata/empty.golden create mode 100644 src/pkg/go/printer/testdata/empty.input create mode 100644 src/pkg/go/printer/testdata/expressions.golden create mode 100644 src/pkg/go/printer/testdata/expressions.input create mode 100644 src/pkg/go/printer/testdata/expressions.raw create mode 100644 src/pkg/go/printer/testdata/linebreaks.golden create mode 100644 src/pkg/go/printer/testdata/linebreaks.input create mode 100644 src/pkg/go/printer/testdata/parser.go create mode 100644 src/pkg/go/printer/testdata/slow.golden create mode 100644 src/pkg/go/printer/testdata/slow.input create mode 100644 src/pkg/go/printer/testdata/statements.golden create mode 100644 src/pkg/go/printer/testdata/statements.input create mode 100644 src/pkg/go/scanner/Makefile create mode 100644 src/pkg/go/scanner/errors.go create mode 100644 src/pkg/go/scanner/scanner.go create mode 100644 src/pkg/go/scanner/scanner_test.go create mode 100644 src/pkg/go/token/Makefile create mode 100644 src/pkg/go/token/position.go create mode 100644 src/pkg/go/token/position_test.go create mode 100644 src/pkg/go/token/token.go create mode 100644 src/pkg/go/typechecker/Makefile create mode 100644 src/pkg/go/typechecker/scope.go create mode 100644 src/pkg/go/typechecker/testdata/test0.src create mode 100644 src/pkg/go/typechecker/testdata/test1.src create mode 100644 src/pkg/go/typechecker/testdata/test3.src create mode 100644 src/pkg/go/typechecker/testdata/test4.src create mode 100644 src/pkg/go/typechecker/type.go create mode 100644 src/pkg/go/typechecker/typechecker.go create mode 100644 src/pkg/go/typechecker/typechecker_test.go create mode 100644 src/pkg/go/typechecker/universe.go create mode 100644 src/pkg/go/types/Makefile create mode 100644 src/pkg/go/types/check.go create mode 100644 src/pkg/go/types/check_test.go create mode 100644 src/pkg/go/types/const.go create mode 100644 src/pkg/go/types/exportdata.go create mode 100644 src/pkg/go/types/gcimporter.go create mode 100644 src/pkg/go/types/gcimporter_test.go create mode 100644 src/pkg/go/types/testdata/exports.go create mode 100644 src/pkg/go/types/testdata/test0.src create mode 100644 src/pkg/go/types/types.go create mode 100644 src/pkg/go/types/universe.go create mode 100644 src/pkg/gob/Makefile create mode 100644 src/pkg/gob/codec_test.go create mode 100644 src/pkg/gob/debug.go create mode 100644 src/pkg/gob/decode.go create mode 100644 src/pkg/gob/decoder.go create mode 100644 src/pkg/gob/doc.go create mode 100644 src/pkg/gob/dump.go create mode 100644 src/pkg/gob/encode.go create mode 100644 src/pkg/gob/encoder.go create mode 100644 src/pkg/gob/encoder_test.go create mode 100644 src/pkg/gob/error.go create mode 100644 src/pkg/gob/gobencdec_test.go create mode 100644 src/pkg/gob/timing_test.go create mode 100644 src/pkg/gob/type.go create mode 100644 src/pkg/gob/type_test.go create mode 100644 src/pkg/hash/Makefile create mode 100644 src/pkg/hash/adler32/Makefile create mode 100644 src/pkg/hash/adler32/adler32.go create mode 100644 src/pkg/hash/adler32/adler32_test.go create mode 100644 src/pkg/hash/crc32/Makefile create mode 100644 src/pkg/hash/crc32/crc32.go create mode 100644 src/pkg/hash/crc32/crc32_amd64.go create mode 100644 src/pkg/hash/crc32/crc32_amd64.s create mode 100644 src/pkg/hash/crc32/crc32_generic.go create mode 100644 src/pkg/hash/crc32/crc32_test.go create mode 100644 src/pkg/hash/crc64/Makefile create mode 100644 src/pkg/hash/crc64/crc64.go create mode 100644 src/pkg/hash/crc64/crc64_test.go create mode 100644 src/pkg/hash/fnv/Makefile create mode 100644 src/pkg/hash/fnv/fnv.go create mode 100644 src/pkg/hash/fnv/fnv_test.go create mode 100644 src/pkg/hash/hash.go create mode 100644 src/pkg/hash/test_cases.txt create mode 100644 src/pkg/hash/test_gen.awk create mode 100644 src/pkg/html/Makefile create mode 100644 src/pkg/html/const.go create mode 100644 src/pkg/html/doc.go create mode 100644 src/pkg/html/entity.go create mode 100644 src/pkg/html/entity_test.go create mode 100644 src/pkg/html/escape.go create mode 100644 src/pkg/html/node.go create mode 100644 src/pkg/html/parse.go create mode 100644 src/pkg/html/parse_test.go create mode 100644 src/pkg/html/testdata/webkit/README create mode 100644 src/pkg/html/testdata/webkit/adoption01.dat create mode 100644 src/pkg/html/testdata/webkit/adoption02.dat create mode 100644 src/pkg/html/testdata/webkit/comments01.dat create mode 100644 src/pkg/html/testdata/webkit/doctype01.dat create mode 100644 src/pkg/html/testdata/webkit/entities01.dat create mode 100644 src/pkg/html/testdata/webkit/entities02.dat create mode 100644 src/pkg/html/testdata/webkit/html5test-com.dat create mode 100644 src/pkg/html/testdata/webkit/inbody01.dat create mode 100644 src/pkg/html/testdata/webkit/isindex.dat create mode 100644 src/pkg/html/testdata/webkit/pending-spec-changes-plain-text-unsafe.dat create mode 100644 src/pkg/html/testdata/webkit/pending-spec-changes.dat create mode 100644 src/pkg/html/testdata/webkit/plain-text-unsafe.dat create mode 100644 src/pkg/html/testdata/webkit/scriptdata01.dat create mode 100644 src/pkg/html/testdata/webkit/scripted/adoption01.dat create mode 100644 src/pkg/html/testdata/webkit/scripted/webkit01.dat create mode 100644 src/pkg/html/testdata/webkit/tables01.dat create mode 100644 src/pkg/html/testdata/webkit/tests1.dat create mode 100644 src/pkg/html/testdata/webkit/tests10.dat create mode 100644 src/pkg/html/testdata/webkit/tests11.dat create mode 100644 src/pkg/html/testdata/webkit/tests12.dat create mode 100644 src/pkg/html/testdata/webkit/tests14.dat create mode 100644 src/pkg/html/testdata/webkit/tests15.dat create mode 100644 src/pkg/html/testdata/webkit/tests16.dat create mode 100644 src/pkg/html/testdata/webkit/tests17.dat create mode 100644 src/pkg/html/testdata/webkit/tests18.dat create mode 100644 src/pkg/html/testdata/webkit/tests19.dat create mode 100644 src/pkg/html/testdata/webkit/tests2.dat create mode 100644 src/pkg/html/testdata/webkit/tests20.dat create mode 100644 src/pkg/html/testdata/webkit/tests21.dat create mode 100644 src/pkg/html/testdata/webkit/tests22.dat create mode 100644 src/pkg/html/testdata/webkit/tests23.dat create mode 100644 src/pkg/html/testdata/webkit/tests24.dat create mode 100644 src/pkg/html/testdata/webkit/tests25.dat create mode 100644 src/pkg/html/testdata/webkit/tests26.dat create mode 100644 src/pkg/html/testdata/webkit/tests3.dat create mode 100644 src/pkg/html/testdata/webkit/tests4.dat create mode 100644 src/pkg/html/testdata/webkit/tests5.dat create mode 100644 src/pkg/html/testdata/webkit/tests6.dat create mode 100644 src/pkg/html/testdata/webkit/tests7.dat create mode 100644 src/pkg/html/testdata/webkit/tests8.dat create mode 100644 src/pkg/html/testdata/webkit/tests9.dat create mode 100644 src/pkg/html/testdata/webkit/tests_innerHTML_1.dat create mode 100644 src/pkg/html/testdata/webkit/tricky01.dat create mode 100644 src/pkg/html/testdata/webkit/webkit01.dat create mode 100644 src/pkg/html/testdata/webkit/webkit02.dat create mode 100644 src/pkg/html/token.go create mode 100644 src/pkg/html/token_test.go create mode 100644 src/pkg/http/Makefile create mode 100644 src/pkg/http/cgi/Makefile create mode 100644 src/pkg/http/cgi/child.go create mode 100644 src/pkg/http/cgi/child_test.go create mode 100644 src/pkg/http/cgi/host.go create mode 100644 src/pkg/http/cgi/host_test.go create mode 100644 src/pkg/http/cgi/matryoshka_test.go create mode 100755 src/pkg/http/cgi/testdata/test.cgi create mode 100644 src/pkg/http/chunked.go create mode 100644 src/pkg/http/client.go create mode 100644 src/pkg/http/client_test.go create mode 100644 src/pkg/http/cookie.go create mode 100644 src/pkg/http/cookie_test.go create mode 100644 src/pkg/http/dump.go create mode 100644 src/pkg/http/export_test.go create mode 100644 src/pkg/http/fcgi/Makefile create mode 100644 src/pkg/http/fcgi/child.go create mode 100644 src/pkg/http/fcgi/fcgi.go create mode 100644 src/pkg/http/fcgi/fcgi_test.go create mode 100644 src/pkg/http/fs.go create mode 100644 src/pkg/http/fs_test.go create mode 100644 src/pkg/http/header.go create mode 100644 src/pkg/http/header_test.go create mode 100644 src/pkg/http/httptest/Makefile create mode 100644 src/pkg/http/httptest/recorder.go create mode 100644 src/pkg/http/httptest/server.go create mode 100644 src/pkg/http/lex.go create mode 100644 src/pkg/http/lex_test.go create mode 100644 src/pkg/http/persist.go create mode 100644 src/pkg/http/pprof/Makefile create mode 100644 src/pkg/http/pprof/pprof.go create mode 100644 src/pkg/http/proxy_test.go create mode 100644 src/pkg/http/range_test.go create mode 100644 src/pkg/http/readrequest_test.go create mode 100644 src/pkg/http/request.go create mode 100644 src/pkg/http/request_test.go create mode 100644 src/pkg/http/requestwrite_test.go create mode 100644 src/pkg/http/response.go create mode 100644 src/pkg/http/response_test.go create mode 100644 src/pkg/http/responsewrite_test.go create mode 100644 src/pkg/http/reverseproxy.go create mode 100644 src/pkg/http/reverseproxy_test.go create mode 100644 src/pkg/http/serve_test.go create mode 100644 src/pkg/http/server.go create mode 100644 src/pkg/http/sniff.go create mode 100644 src/pkg/http/sniff_test.go create mode 100644 src/pkg/http/spdy/Makefile create mode 100644 src/pkg/http/spdy/read.go create mode 100644 src/pkg/http/spdy/spdy_test.go create mode 100644 src/pkg/http/spdy/types.go create mode 100644 src/pkg/http/spdy/write.go create mode 100644 src/pkg/http/status.go create mode 100644 src/pkg/http/testdata/file create mode 100644 src/pkg/http/testdata/index.html create mode 100644 src/pkg/http/testdata/style.css create mode 100644 src/pkg/http/transfer.go create mode 100644 src/pkg/http/transport.go create mode 100644 src/pkg/http/transport_test.go create mode 100644 src/pkg/http/triv.go create mode 100644 src/pkg/image/Makefile create mode 100644 src/pkg/image/bmp/Makefile create mode 100644 src/pkg/image/bmp/reader.go create mode 100644 src/pkg/image/color.go create mode 100644 src/pkg/image/decode_test.go create mode 100644 src/pkg/image/draw/Makefile create mode 100644 src/pkg/image/draw/bench_test.go create mode 100644 src/pkg/image/draw/clip_test.go create mode 100644 src/pkg/image/draw/draw.go create mode 100644 src/pkg/image/draw/draw_test.go create mode 100644 src/pkg/image/format.go create mode 100644 src/pkg/image/geom.go create mode 100644 src/pkg/image/gif/Makefile create mode 100644 src/pkg/image/gif/reader.go create mode 100644 src/pkg/image/image.go create mode 100644 src/pkg/image/image_test.go create mode 100644 src/pkg/image/jpeg/Makefile create mode 100644 src/pkg/image/jpeg/fdct.go create mode 100644 src/pkg/image/jpeg/huffman.go create mode 100644 src/pkg/image/jpeg/idct.go create mode 100644 src/pkg/image/jpeg/reader.go create mode 100644 src/pkg/image/jpeg/writer.go create mode 100644 src/pkg/image/jpeg/writer_test.go create mode 100644 src/pkg/image/names.go create mode 100644 src/pkg/image/png/Makefile create mode 100644 src/pkg/image/png/reader.go create mode 100644 src/pkg/image/png/reader_test.go create mode 100644 src/pkg/image/png/testdata/pngsuite/README create mode 100644 src/pkg/image/png/testdata/pngsuite/README.original create mode 100644 src/pkg/image/png/testdata/pngsuite/basn0g01-30.png create mode 100644 src/pkg/image/png/testdata/pngsuite/basn0g01-30.sng create mode 100644 src/pkg/image/png/testdata/pngsuite/basn0g01.png create mode 100644 src/pkg/image/png/testdata/pngsuite/basn0g01.sng create mode 100644 src/pkg/image/png/testdata/pngsuite/basn0g02-29.png create mode 100644 src/pkg/image/png/testdata/pngsuite/basn0g02-29.sng create mode 100644 src/pkg/image/png/testdata/pngsuite/basn0g02.png create mode 100644 src/pkg/image/png/testdata/pngsuite/basn0g02.sng create mode 100644 src/pkg/image/png/testdata/pngsuite/basn0g04-31.png create mode 100644 src/pkg/image/png/testdata/pngsuite/basn0g04-31.sng create mode 100644 src/pkg/image/png/testdata/pngsuite/basn0g04.png create mode 100644 src/pkg/image/png/testdata/pngsuite/basn0g04.sng create mode 100644 src/pkg/image/png/testdata/pngsuite/basn0g08.png create mode 100644 src/pkg/image/png/testdata/pngsuite/basn0g08.sng create mode 100644 src/pkg/image/png/testdata/pngsuite/basn0g16.png create mode 100644 src/pkg/image/png/testdata/pngsuite/basn0g16.sng create mode 100644 src/pkg/image/png/testdata/pngsuite/basn2c08.png create mode 100644 src/pkg/image/png/testdata/pngsuite/basn2c08.sng create mode 100644 src/pkg/image/png/testdata/pngsuite/basn2c16.png create mode 100644 src/pkg/image/png/testdata/pngsuite/basn2c16.sng create mode 100644 src/pkg/image/png/testdata/pngsuite/basn3p01.png create mode 100644 src/pkg/image/png/testdata/pngsuite/basn3p01.sng create mode 100644 src/pkg/image/png/testdata/pngsuite/basn3p02.png create mode 100644 src/pkg/image/png/testdata/pngsuite/basn3p02.sng create mode 100644 src/pkg/image/png/testdata/pngsuite/basn3p04.png create mode 100644 src/pkg/image/png/testdata/pngsuite/basn3p04.sng create mode 100644 src/pkg/image/png/testdata/pngsuite/basn3p08-trns.png create mode 100644 src/pkg/image/png/testdata/pngsuite/basn3p08-trns.sng create mode 100644 src/pkg/image/png/testdata/pngsuite/basn3p08.png create mode 100644 src/pkg/image/png/testdata/pngsuite/basn3p08.sng create mode 100644 src/pkg/image/png/testdata/pngsuite/basn4a08.png create mode 100644 src/pkg/image/png/testdata/pngsuite/basn4a08.sng create mode 100644 src/pkg/image/png/testdata/pngsuite/basn4a16.png create mode 100644 src/pkg/image/png/testdata/pngsuite/basn4a16.sng create mode 100644 src/pkg/image/png/testdata/pngsuite/basn6a08.png create mode 100644 src/pkg/image/png/testdata/pngsuite/basn6a08.sng create mode 100644 src/pkg/image/png/testdata/pngsuite/basn6a16.png create mode 100644 src/pkg/image/png/testdata/pngsuite/basn6a16.sng create mode 100644 src/pkg/image/png/writer.go create mode 100644 src/pkg/image/png/writer_test.go create mode 100644 src/pkg/image/testdata/video-001.5bpp.gif create mode 100644 src/pkg/image/testdata/video-001.bmp create mode 100644 src/pkg/image/testdata/video-001.gif create mode 100644 src/pkg/image/testdata/video-001.interlaced.gif create mode 100644 src/pkg/image/testdata/video-001.jpeg create mode 100644 src/pkg/image/testdata/video-001.png create mode 100644 src/pkg/image/testdata/video-001.tiff create mode 100644 src/pkg/image/testdata/video-005.gray.jpeg create mode 100644 src/pkg/image/testdata/video-005.gray.png create mode 100644 src/pkg/image/tiff/Makefile create mode 100644 src/pkg/image/tiff/buffer.go create mode 100644 src/pkg/image/tiff/buffer_test.go create mode 100644 src/pkg/image/tiff/consts.go create mode 100644 src/pkg/image/tiff/reader.go create mode 100644 src/pkg/image/tiff/reader_test.go create mode 100644 src/pkg/image/tiff/testdata/no_rps.tiff create mode 100644 src/pkg/image/ycbcr/Makefile create mode 100644 src/pkg/image/ycbcr/ycbcr.go create mode 100644 src/pkg/image/ycbcr/ycbcr_test.go create mode 100644 src/pkg/index/suffixarray/Makefile create mode 100644 src/pkg/index/suffixarray/qsufsort.go create mode 100644 src/pkg/index/suffixarray/suffixarray.go create mode 100644 src/pkg/index/suffixarray/suffixarray_test.go create mode 100644 src/pkg/io/Makefile create mode 100644 src/pkg/io/io.go create mode 100644 src/pkg/io/io_test.go create mode 100644 src/pkg/io/ioutil/Makefile create mode 100644 src/pkg/io/ioutil/ioutil.go create mode 100644 src/pkg/io/ioutil/ioutil_test.go create mode 100644 src/pkg/io/ioutil/tempfile.go create mode 100644 src/pkg/io/ioutil/tempfile_test.go create mode 100644 src/pkg/io/multi.go create mode 100644 src/pkg/io/multi_test.go create mode 100644 src/pkg/io/pipe.go create mode 100644 src/pkg/io/pipe_test.go create mode 100644 src/pkg/json/Makefile create mode 100644 src/pkg/json/decode.go create mode 100644 src/pkg/json/decode_test.go create mode 100644 src/pkg/json/encode.go create mode 100644 src/pkg/json/encode_test.go create mode 100644 src/pkg/json/indent.go create mode 100644 src/pkg/json/scanner.go create mode 100644 src/pkg/json/scanner_test.go create mode 100644 src/pkg/json/stream.go create mode 100644 src/pkg/json/stream_test.go create mode 100644 src/pkg/json/tagkey_test.go create mode 100644 src/pkg/log/Makefile create mode 100644 src/pkg/log/log.go create mode 100644 src/pkg/log/log_test.go create mode 100644 src/pkg/mail/Makefile create mode 100644 src/pkg/mail/message.go create mode 100644 src/pkg/mail/message_test.go create mode 100644 src/pkg/math/Makefile create mode 100644 src/pkg/math/acosh.go create mode 100644 src/pkg/math/all_test.go create mode 100644 src/pkg/math/asin.go create mode 100644 src/pkg/math/asin_386.s create mode 100644 src/pkg/math/asin_decl.go create mode 100644 src/pkg/math/asinh.go create mode 100644 src/pkg/math/atan.go create mode 100644 src/pkg/math/atan2.go create mode 100755 src/pkg/math/atan2_386.s create mode 100755 src/pkg/math/atan2_decl.go create mode 100644 src/pkg/math/atan_386.s create mode 100644 src/pkg/math/atan_decl.go create mode 100644 src/pkg/math/atanh.go create mode 100644 src/pkg/math/bits.go create mode 100644 src/pkg/math/cbrt.go create mode 100644 src/pkg/math/const.go create mode 100644 src/pkg/math/copysign.go create mode 100644 src/pkg/math/erf.go create mode 100644 src/pkg/math/exp.go create mode 100644 src/pkg/math/exp2.go create mode 100644 src/pkg/math/exp2_386.s create mode 100644 src/pkg/math/exp2_decl.go create mode 100644 src/pkg/math/exp_386.s create mode 100644 src/pkg/math/exp_amd64.s create mode 100644 src/pkg/math/exp_decl.go create mode 100644 src/pkg/math/exp_port.go create mode 100644 src/pkg/math/exp_test.go create mode 100644 src/pkg/math/expm1.go create mode 100644 src/pkg/math/expm1_386.s create mode 100644 src/pkg/math/expm1_decl.go create mode 100644 src/pkg/math/fabs.go create mode 100644 src/pkg/math/fabs_386.s create mode 100644 src/pkg/math/fabs_amd64.s create mode 100644 src/pkg/math/fabs_decl.go create mode 100644 src/pkg/math/fdim.go create mode 100644 src/pkg/math/fdim_amd64.s create mode 100644 src/pkg/math/fdim_decl.go create mode 100644 src/pkg/math/floor.go create mode 100644 src/pkg/math/floor_386.s create mode 100644 src/pkg/math/floor_decl.go create mode 100644 src/pkg/math/fltasm_amd64.s create mode 100644 src/pkg/math/fmod.go create mode 100644 src/pkg/math/fmod_386.s create mode 100644 src/pkg/math/fmod_decl.go create mode 100644 src/pkg/math/frexp.go create mode 100644 src/pkg/math/frexp_386.s create mode 100644 src/pkg/math/frexp_decl.go create mode 100644 src/pkg/math/gamma.go create mode 100644 src/pkg/math/hypot.go create mode 100644 src/pkg/math/hypot_386.s create mode 100644 src/pkg/math/hypot_amd64.s create mode 100644 src/pkg/math/hypot_decl.go create mode 100644 src/pkg/math/hypot_port.go create mode 100644 src/pkg/math/hypot_test.go create mode 100644 src/pkg/math/j0.go create mode 100644 src/pkg/math/j1.go create mode 100644 src/pkg/math/jn.go create mode 100644 src/pkg/math/ldexp.go create mode 100644 src/pkg/math/ldexp_386.s create mode 100644 src/pkg/math/ldexp_decl.go create mode 100644 src/pkg/math/lgamma.go create mode 100644 src/pkg/math/log.go create mode 100644 src/pkg/math/log10.go create mode 100644 src/pkg/math/log10_386.s create mode 100644 src/pkg/math/log10_decl.go create mode 100644 src/pkg/math/log1p.go create mode 100644 src/pkg/math/log1p_386.s create mode 100644 src/pkg/math/log1p_decl.go create mode 100644 src/pkg/math/log_386.s create mode 100644 src/pkg/math/log_amd64.s create mode 100644 src/pkg/math/log_decl.go create mode 100644 src/pkg/math/logb.go create mode 100644 src/pkg/math/modf.go create mode 100644 src/pkg/math/modf_386.s create mode 100644 src/pkg/math/modf_decl.go create mode 100644 src/pkg/math/nextafter.go create mode 100644 src/pkg/math/pow.go create mode 100644 src/pkg/math/pow10.go create mode 100644 src/pkg/math/remainder.go create mode 100644 src/pkg/math/remainder_386.s create mode 100644 src/pkg/math/remainder_decl.go create mode 100644 src/pkg/math/signbit.go create mode 100644 src/pkg/math/sin.go create mode 100644 src/pkg/math/sin_386.s create mode 100644 src/pkg/math/sin_decl.go create mode 100644 src/pkg/math/sincos.go create mode 100644 src/pkg/math/sincos_386.s create mode 100644 src/pkg/math/sincos_amd64.s create mode 100644 src/pkg/math/sincos_decl.go create mode 100644 src/pkg/math/sinh.go create mode 100644 src/pkg/math/sqrt.go create mode 100644 src/pkg/math/sqrt_386.s create mode 100644 src/pkg/math/sqrt_amd64.s create mode 100644 src/pkg/math/sqrt_arm.s create mode 100644 src/pkg/math/sqrt_decl.go create mode 100644 src/pkg/math/sqrt_port.go create mode 100644 src/pkg/math/sqrt_test.go create mode 100644 src/pkg/math/tan.go create mode 100644 src/pkg/math/tan_386.s create mode 100644 src/pkg/math/tan_decl.go create mode 100644 src/pkg/math/tanh.go create mode 100644 src/pkg/math/unsafe.go create mode 100644 src/pkg/mime/Makefile create mode 100644 src/pkg/mime/grammar.go create mode 100644 src/pkg/mime/mediatype.go create mode 100644 src/pkg/mime/mediatype_test.go create mode 100644 src/pkg/mime/mime_test.go create mode 100644 src/pkg/mime/multipart/Makefile create mode 100644 src/pkg/mime/multipart/formdata.go create mode 100644 src/pkg/mime/multipart/formdata_test.go create mode 100644 src/pkg/mime/multipart/multipart.go create mode 100644 src/pkg/mime/multipart/multipart_test.go create mode 100644 src/pkg/mime/multipart/writer.go create mode 100644 src/pkg/mime/multipart/writer_test.go create mode 100644 src/pkg/mime/test.types create mode 100644 src/pkg/mime/type.go create mode 100644 src/pkg/net/Makefile create mode 100644 src/pkg/net/cgo_bsd.go create mode 100644 src/pkg/net/cgo_linux.go create mode 100644 src/pkg/net/cgo_stub.go create mode 100644 src/pkg/net/cgo_unix.go create mode 100644 src/pkg/net/dial.go create mode 100644 src/pkg/net/dialgoogle_test.go create mode 100644 src/pkg/net/dict/Makefile create mode 100644 src/pkg/net/dict/dict.go create mode 100644 src/pkg/net/dnsclient.go create mode 100644 src/pkg/net/dnsclient_unix.go create mode 100644 src/pkg/net/dnsconfig.go create mode 100644 src/pkg/net/dnsmsg.go create mode 100644 src/pkg/net/dnsmsg_test.go create mode 100644 src/pkg/net/dnsname_test.go create mode 100644 src/pkg/net/fd.go create mode 100644 src/pkg/net/fd_darwin.go create mode 100644 src/pkg/net/fd_freebsd.go create mode 100644 src/pkg/net/fd_linux.go create mode 100644 src/pkg/net/fd_openbsd.go create mode 100644 src/pkg/net/fd_windows.go create mode 100644 src/pkg/net/file.go create mode 100644 src/pkg/net/file_plan9.go create mode 100644 src/pkg/net/file_test.go create mode 100644 src/pkg/net/file_windows.go create mode 100644 src/pkg/net/hosts.go create mode 100644 src/pkg/net/hosts_test.go create mode 100644 src/pkg/net/hosts_testdata create mode 100644 src/pkg/net/interface.go create mode 100644 src/pkg/net/interface_bsd.go create mode 100644 src/pkg/net/interface_darwin.go create mode 100644 src/pkg/net/interface_freebsd.go create mode 100644 src/pkg/net/interface_linux.go create mode 100644 src/pkg/net/interface_openbsd.go create mode 100644 src/pkg/net/interface_stub.go create mode 100644 src/pkg/net/interface_test.go create mode 100644 src/pkg/net/interface_windows.go create mode 100644 src/pkg/net/ip.go create mode 100644 src/pkg/net/ip_test.go create mode 100644 src/pkg/net/ipraw_test.go create mode 100644 src/pkg/net/iprawsock.go create mode 100644 src/pkg/net/iprawsock_plan9.go create mode 100644 src/pkg/net/iprawsock_posix.go create mode 100644 src/pkg/net/ipsock.go create mode 100644 src/pkg/net/ipsock_plan9.go create mode 100644 src/pkg/net/ipsock_posix.go create mode 100644 src/pkg/net/lookup_plan9.go create mode 100644 src/pkg/net/lookup_test.go create mode 100644 src/pkg/net/lookup_unix.go create mode 100644 src/pkg/net/lookup_windows.go create mode 100644 src/pkg/net/multicast_test.go create mode 100644 src/pkg/net/net.go create mode 100644 src/pkg/net/net_test.go create mode 100644 src/pkg/net/newpollserver.go create mode 100644 src/pkg/net/parse.go create mode 100644 src/pkg/net/parse_test.go create mode 100644 src/pkg/net/pipe.go create mode 100644 src/pkg/net/pipe_test.go create mode 100644 src/pkg/net/port.go create mode 100644 src/pkg/net/port_test.go create mode 100644 src/pkg/net/sendfile_linux.go create mode 100644 src/pkg/net/sendfile_stub.go create mode 100644 src/pkg/net/sendfile_windows.go create mode 100644 src/pkg/net/server_test.go create mode 100644 src/pkg/net/sock.go create mode 100644 src/pkg/net/sock_bsd.go create mode 100644 src/pkg/net/sock_linux.go create mode 100644 src/pkg/net/sock_windows.go create mode 100644 src/pkg/net/tcpsock.go create mode 100644 src/pkg/net/tcpsock_plan9.go create mode 100644 src/pkg/net/tcpsock_posix.go create mode 100644 src/pkg/net/textproto/Makefile create mode 100644 src/pkg/net/textproto/header.go create mode 100644 src/pkg/net/textproto/pipeline.go create mode 100644 src/pkg/net/textproto/reader.go create mode 100644 src/pkg/net/textproto/reader_test.go create mode 100644 src/pkg/net/textproto/textproto.go create mode 100644 src/pkg/net/textproto/writer.go create mode 100644 src/pkg/net/textproto/writer_test.go create mode 100644 src/pkg/net/timeout_test.go create mode 100644 src/pkg/net/udpsock.go create mode 100644 src/pkg/net/udpsock_plan9.go create mode 100644 src/pkg/net/udpsock_posix.go create mode 100644 src/pkg/net/unixsock.go create mode 100644 src/pkg/net/unixsock_plan9.go create mode 100644 src/pkg/net/unixsock_posix.go create mode 100644 src/pkg/netchan/Makefile create mode 100644 src/pkg/netchan/common.go create mode 100644 src/pkg/netchan/export.go create mode 100644 src/pkg/netchan/import.go create mode 100644 src/pkg/netchan/netchan_test.go create mode 100644 src/pkg/old/template/Makefile create mode 100644 src/pkg/old/template/doc.go create mode 100644 src/pkg/old/template/execute.go create mode 100644 src/pkg/old/template/format.go create mode 100644 src/pkg/old/template/parse.go create mode 100644 src/pkg/old/template/template_test.go create mode 100644 src/pkg/os/Makefile create mode 100644 src/pkg/os/dir_plan9.go create mode 100644 src/pkg/os/dir_unix.go create mode 100644 src/pkg/os/dir_windows.go create mode 100644 src/pkg/os/env.go create mode 100644 src/pkg/os/env_plan9.go create mode 100644 src/pkg/os/env_test.go create mode 100644 src/pkg/os/env_unix.go create mode 100644 src/pkg/os/env_windows.go create mode 100644 src/pkg/os/error.go create mode 100644 src/pkg/os/error_plan9.go create mode 100644 src/pkg/os/error_posix.go create mode 100644 src/pkg/os/exec.go create mode 100644 src/pkg/os/exec_plan9.go create mode 100644 src/pkg/os/exec_posix.go create mode 100644 src/pkg/os/exec_unix.go create mode 100644 src/pkg/os/exec_windows.go create mode 100644 src/pkg/os/file.go create mode 100644 src/pkg/os/file_plan9.go create mode 100644 src/pkg/os/file_posix.go create mode 100644 src/pkg/os/file_unix.go create mode 100644 src/pkg/os/file_windows.go create mode 100644 src/pkg/os/getwd.go create mode 100644 src/pkg/os/inotify/Makefile create mode 100644 src/pkg/os/inotify/inotify_linux.go create mode 100644 src/pkg/os/inotify/inotify_linux_test.go create mode 100755 src/pkg/os/mkunixsignals.sh create mode 100644 src/pkg/os/os_test.go create mode 100644 src/pkg/os/path.go create mode 100644 src/pkg/os/path_plan9.go create mode 100644 src/pkg/os/path_test.go create mode 100644 src/pkg/os/path_unix.go create mode 100644 src/pkg/os/path_windows.go create mode 100644 src/pkg/os/proc.go create mode 100644 src/pkg/os/signal/Makefile create mode 100644 src/pkg/os/signal/signal.go create mode 100644 src/pkg/os/signal/signal_test.go create mode 100644 src/pkg/os/stat_darwin.go create mode 100644 src/pkg/os/stat_freebsd.go create mode 100644 src/pkg/os/stat_linux.go create mode 100644 src/pkg/os/stat_openbsd.go create mode 100644 src/pkg/os/stat_plan9.go create mode 100644 src/pkg/os/stat_windows.go create mode 100644 src/pkg/os/str.go create mode 100644 src/pkg/os/sys_bsd.go create mode 100644 src/pkg/os/sys_linux.go create mode 100644 src/pkg/os/sys_plan9.go create mode 100644 src/pkg/os/sys_windows.go create mode 100644 src/pkg/os/time.go create mode 100644 src/pkg/os/types.go create mode 100644 src/pkg/os/user/Makefile create mode 100644 src/pkg/os/user/lookup_stubs.go create mode 100644 src/pkg/os/user/lookup_unix.go create mode 100644 src/pkg/os/user/user.go create mode 100644 src/pkg/os/user/user_test.go create mode 100644 src/pkg/patch/Makefile create mode 100644 src/pkg/patch/apply.go create mode 100644 src/pkg/patch/git.go create mode 100644 src/pkg/patch/patch.go create mode 100644 src/pkg/patch/patch_test.go create mode 100644 src/pkg/patch/textdiff.go create mode 100644 src/pkg/path/Makefile create mode 100644 src/pkg/path/filepath/Makefile create mode 100644 src/pkg/path/filepath/match.go create mode 100644 src/pkg/path/filepath/match_test.go create mode 100644 src/pkg/path/filepath/path.go create mode 100644 src/pkg/path/filepath/path_plan9.go create mode 100644 src/pkg/path/filepath/path_test.go create mode 100644 src/pkg/path/filepath/path_unix.go create mode 100644 src/pkg/path/filepath/path_windows.go create mode 100644 src/pkg/path/match.go create mode 100644 src/pkg/path/match_test.go create mode 100644 src/pkg/path/path.go create mode 100644 src/pkg/path/path_test.go create mode 100644 src/pkg/rand/Makefile create mode 100644 src/pkg/rand/exp.go create mode 100644 src/pkg/rand/normal.go create mode 100644 src/pkg/rand/rand.go create mode 100644 src/pkg/rand/rand_test.go create mode 100644 src/pkg/rand/rng.go create mode 100644 src/pkg/rand/zipf.go create mode 100644 src/pkg/reflect/Makefile create mode 100644 src/pkg/reflect/all_test.go create mode 100644 src/pkg/reflect/deepequal.go create mode 100644 src/pkg/reflect/set_test.go create mode 100644 src/pkg/reflect/tostring_test.go create mode 100644 src/pkg/reflect/type.go create mode 100644 src/pkg/reflect/value.go create mode 100644 src/pkg/regexp/Makefile create mode 100644 src/pkg/regexp/all_test.go create mode 100644 src/pkg/regexp/find_test.go create mode 100644 src/pkg/regexp/regexp.go create mode 100644 src/pkg/rpc/Makefile create mode 100644 src/pkg/rpc/client.go create mode 100644 src/pkg/rpc/debug.go create mode 100644 src/pkg/rpc/jsonrpc/Makefile create mode 100644 src/pkg/rpc/jsonrpc/all_test.go create mode 100644 src/pkg/rpc/jsonrpc/client.go create mode 100644 src/pkg/rpc/jsonrpc/server.go create mode 100644 src/pkg/rpc/server.go create mode 100644 src/pkg/rpc/server_test.go create mode 100644 src/pkg/runtime/386/arch.h create mode 100644 src/pkg/runtime/386/asm.s create mode 100644 src/pkg/runtime/386/atomic.c create mode 100644 src/pkg/runtime/386/closure.c create mode 100644 src/pkg/runtime/386/memmove.s create mode 100644 src/pkg/runtime/386/vlop.s create mode 100644 src/pkg/runtime/386/vlrt.c create mode 100644 src/pkg/runtime/Makefile create mode 100644 src/pkg/runtime/amd64/arch.h create mode 100644 src/pkg/runtime/amd64/asm.s create mode 100644 src/pkg/runtime/amd64/atomic.c create mode 100644 src/pkg/runtime/amd64/closure.c create mode 100644 src/pkg/runtime/amd64/memmove.s create mode 100644 src/pkg/runtime/amd64/traceback.c create mode 100644 src/pkg/runtime/append_test.go create mode 100644 src/pkg/runtime/arm/arch.h create mode 100644 src/pkg/runtime/arm/asm.s create mode 100644 src/pkg/runtime/arm/atomic.c create mode 100644 src/pkg/runtime/arm/closure.c create mode 100644 src/pkg/runtime/arm/memmove.s create mode 100644 src/pkg/runtime/arm/memset.s create mode 100644 src/pkg/runtime/arm/softfloat.c create mode 100644 src/pkg/runtime/arm/traceback.c create mode 100644 src/pkg/runtime/arm/vlop.s create mode 100644 src/pkg/runtime/arm/vlrt.c create mode 100755 src/pkg/runtime/cgo/386.S create mode 100644 src/pkg/runtime/cgo/Makefile create mode 100644 src/pkg/runtime/cgo/amd64.S create mode 100644 src/pkg/runtime/cgo/arm.S create mode 100644 src/pkg/runtime/cgo/callbacks.c create mode 100644 src/pkg/runtime/cgo/cgo.go create mode 100644 src/pkg/runtime/cgo/darwin_386.c create mode 100644 src/pkg/runtime/cgo/darwin_amd64.c create mode 100644 src/pkg/runtime/cgo/freebsd.c create mode 100644 src/pkg/runtime/cgo/freebsd_386.c create mode 100644 src/pkg/runtime/cgo/freebsd_amd64.c create mode 100644 src/pkg/runtime/cgo/iscgo.c create mode 100644 src/pkg/runtime/cgo/libcgo.h create mode 100644 src/pkg/runtime/cgo/linux_386.c create mode 100644 src/pkg/runtime/cgo/linux_amd64.c create mode 100644 src/pkg/runtime/cgo/linux_arm.c create mode 100644 src/pkg/runtime/cgo/setenv.c create mode 100644 src/pkg/runtime/cgo/util.c create mode 100755 src/pkg/runtime/cgo/windows_386.c create mode 100755 src/pkg/runtime/cgo/windows_amd64.c create mode 100644 src/pkg/runtime/cgocall.c create mode 100644 src/pkg/runtime/cgocall.h create mode 100644 src/pkg/runtime/chan.c create mode 100644 src/pkg/runtime/chan_test.go create mode 100644 src/pkg/runtime/closure_test.go create mode 100644 src/pkg/runtime/complex.c create mode 100644 src/pkg/runtime/cpuprof.c create mode 100644 src/pkg/runtime/darwin/386/defs.h create mode 100644 src/pkg/runtime/darwin/386/rt0.s create mode 100644 src/pkg/runtime/darwin/386/signal.c create mode 100644 src/pkg/runtime/darwin/386/sys.s create mode 100644 src/pkg/runtime/darwin/amd64/defs.h create mode 100644 src/pkg/runtime/darwin/amd64/rt0.s create mode 100644 src/pkg/runtime/darwin/amd64/signal.c create mode 100644 src/pkg/runtime/darwin/amd64/sys.s create mode 100644 src/pkg/runtime/darwin/defs.c create mode 100644 src/pkg/runtime/darwin/mem.c create mode 100644 src/pkg/runtime/darwin/os.h create mode 100644 src/pkg/runtime/darwin/signals.h create mode 100644 src/pkg/runtime/darwin/thread.c create mode 100644 src/pkg/runtime/debug.go create mode 100644 src/pkg/runtime/debug/Makefile create mode 100644 src/pkg/runtime/debug/stack.go create mode 100644 src/pkg/runtime/debug/stack_test.go create mode 100644 src/pkg/runtime/error.go create mode 100644 src/pkg/runtime/export_test.go create mode 100644 src/pkg/runtime/extern.go create mode 100644 src/pkg/runtime/float.c create mode 100644 src/pkg/runtime/freebsd/386/defs.h create mode 100644 src/pkg/runtime/freebsd/386/rt0.s create mode 100644 src/pkg/runtime/freebsd/386/signal.c create mode 100644 src/pkg/runtime/freebsd/386/sys.s create mode 100644 src/pkg/runtime/freebsd/amd64/defs.h create mode 100644 src/pkg/runtime/freebsd/amd64/rt0.s create mode 100644 src/pkg/runtime/freebsd/amd64/signal.c create mode 100644 src/pkg/runtime/freebsd/amd64/sys.s create mode 100644 src/pkg/runtime/freebsd/defs.c create mode 100644 src/pkg/runtime/freebsd/mem.c create mode 100644 src/pkg/runtime/freebsd/os.h create mode 100644 src/pkg/runtime/freebsd/signals.h create mode 100644 src/pkg/runtime/freebsd/thread.c create mode 100644 src/pkg/runtime/goc2c.c create mode 100644 src/pkg/runtime/hashmap.c create mode 100644 src/pkg/runtime/hashmap.h create mode 100644 src/pkg/runtime/iface.c create mode 100644 src/pkg/runtime/linux/386/defs.h create mode 100644 src/pkg/runtime/linux/386/rt0.s create mode 100644 src/pkg/runtime/linux/386/signal.c create mode 100644 src/pkg/runtime/linux/386/sys.s create mode 100644 src/pkg/runtime/linux/amd64/defs.h create mode 100644 src/pkg/runtime/linux/amd64/rt0.s create mode 100644 src/pkg/runtime/linux/amd64/signal.c create mode 100644 src/pkg/runtime/linux/amd64/sys.s create mode 100644 src/pkg/runtime/linux/arm/defs.h create mode 100644 src/pkg/runtime/linux/arm/rt0.s create mode 100644 src/pkg/runtime/linux/arm/signal.c create mode 100644 src/pkg/runtime/linux/arm/sys.s create mode 100644 src/pkg/runtime/linux/defs.c create mode 100644 src/pkg/runtime/linux/defs1.c create mode 100644 src/pkg/runtime/linux/defs2.c create mode 100644 src/pkg/runtime/linux/defs_arm.c create mode 100644 src/pkg/runtime/linux/mem.c create mode 100644 src/pkg/runtime/linux/os.h create mode 100644 src/pkg/runtime/linux/signals.h create mode 100644 src/pkg/runtime/linux/thread.c create mode 100644 src/pkg/runtime/malloc.goc create mode 100644 src/pkg/runtime/malloc.h create mode 100644 src/pkg/runtime/mcache.c create mode 100644 src/pkg/runtime/mcentral.c create mode 100644 src/pkg/runtime/mem.go create mode 100644 src/pkg/runtime/mfinal.c create mode 100644 src/pkg/runtime/mfixalloc.c create mode 100644 src/pkg/runtime/mgc0.c create mode 100644 src/pkg/runtime/mheap.c create mode 100755 src/pkg/runtime/mkasmh.sh create mode 100755 src/pkg/runtime/mkgodefs.sh create mode 100644 src/pkg/runtime/mkversion.c create mode 100644 src/pkg/runtime/mprof.goc create mode 100644 src/pkg/runtime/msize.c create mode 100644 src/pkg/runtime/openbsd/amd64/defs.h create mode 100644 src/pkg/runtime/openbsd/amd64/rt0.s create mode 100644 src/pkg/runtime/openbsd/amd64/signal.c create mode 100644 src/pkg/runtime/openbsd/amd64/sys.s create mode 100644 src/pkg/runtime/openbsd/defs.c create mode 100644 src/pkg/runtime/openbsd/mem.c create mode 100644 src/pkg/runtime/openbsd/os.h create mode 100644 src/pkg/runtime/openbsd/signals.h create mode 100644 src/pkg/runtime/openbsd/thread.c create mode 100644 src/pkg/runtime/plan9/386/defs.h create mode 100644 src/pkg/runtime/plan9/386/rt0.s create mode 100644 src/pkg/runtime/plan9/386/signal.c create mode 100644 src/pkg/runtime/plan9/386/sys.s create mode 100644 src/pkg/runtime/plan9/mem.c create mode 100644 src/pkg/runtime/plan9/os.h create mode 100644 src/pkg/runtime/plan9/signals.h create mode 100644 src/pkg/runtime/plan9/thread.c create mode 100644 src/pkg/runtime/pprof/Makefile create mode 100644 src/pkg/runtime/pprof/pprof.go create mode 100644 src/pkg/runtime/pprof/pprof_test.go create mode 100644 src/pkg/runtime/print.c create mode 100644 src/pkg/runtime/proc.c create mode 100644 src/pkg/runtime/proc.p create mode 100644 src/pkg/runtime/proc_test.go create mode 100644 src/pkg/runtime/rune.c create mode 100644 src/pkg/runtime/runtime-gdb.py create mode 100644 src/pkg/runtime/runtime.c create mode 100644 src/pkg/runtime/runtime.h create mode 100644 src/pkg/runtime/runtime1.goc create mode 100644 src/pkg/runtime/sema.goc create mode 100644 src/pkg/runtime/sema_test.go create mode 100644 src/pkg/runtime/sig.go create mode 100644 src/pkg/runtime/sigqueue.goc create mode 100644 src/pkg/runtime/slice.c create mode 100644 src/pkg/runtime/softfloat64.go create mode 100644 src/pkg/runtime/softfloat64_test.go create mode 100644 src/pkg/runtime/stack.h create mode 100644 src/pkg/runtime/string.goc create mode 100644 src/pkg/runtime/symtab.c create mode 100644 src/pkg/runtime/symtab_test.go create mode 100644 src/pkg/runtime/type.go create mode 100644 src/pkg/runtime/type.h create mode 100644 src/pkg/runtime/windows/386/defs.h create mode 100644 src/pkg/runtime/windows/386/rt0.s create mode 100644 src/pkg/runtime/windows/386/signal.c create mode 100644 src/pkg/runtime/windows/386/sys.s create mode 100644 src/pkg/runtime/windows/amd64/defs.h create mode 100644 src/pkg/runtime/windows/amd64/rt0.s create mode 100644 src/pkg/runtime/windows/amd64/signal.c create mode 100644 src/pkg/runtime/windows/amd64/sys.s create mode 100644 src/pkg/runtime/windows/defs.c create mode 100644 src/pkg/runtime/windows/mem.c create mode 100644 src/pkg/runtime/windows/os.h create mode 100644 src/pkg/runtime/windows/signals.h create mode 100644 src/pkg/runtime/windows/syscall.goc create mode 100644 src/pkg/runtime/windows/thread.c create mode 100644 src/pkg/scanner/Makefile create mode 100644 src/pkg/scanner/scanner.go create mode 100644 src/pkg/scanner/scanner_test.go create mode 100644 src/pkg/smtp/Makefile create mode 100644 src/pkg/smtp/auth.go create mode 100644 src/pkg/smtp/smtp.go create mode 100644 src/pkg/smtp/smtp_test.go create mode 100644 src/pkg/sort/Makefile create mode 100644 src/pkg/sort/search.go create mode 100644 src/pkg/sort/search_test.go create mode 100644 src/pkg/sort/sort.go create mode 100644 src/pkg/sort/sort_test.go create mode 100644 src/pkg/strconv/Makefile create mode 100644 src/pkg/strconv/atob.go create mode 100644 src/pkg/strconv/atob_test.go create mode 100644 src/pkg/strconv/atof.go create mode 100644 src/pkg/strconv/atof_test.go create mode 100644 src/pkg/strconv/atoi.go create mode 100644 src/pkg/strconv/atoi_test.go create mode 100644 src/pkg/strconv/decimal.go create mode 100644 src/pkg/strconv/decimal_test.go create mode 100644 src/pkg/strconv/fp_test.go create mode 100644 src/pkg/strconv/ftoa.go create mode 100644 src/pkg/strconv/ftoa_test.go create mode 100644 src/pkg/strconv/internal_test.go create mode 100644 src/pkg/strconv/itoa.go create mode 100644 src/pkg/strconv/itoa_test.go create mode 100644 src/pkg/strconv/quote.go create mode 100644 src/pkg/strconv/quote_test.go create mode 100644 src/pkg/strconv/testfp.txt create mode 100644 src/pkg/strings/Makefile create mode 100644 src/pkg/strings/reader.go create mode 100644 src/pkg/strings/strings.go create mode 100644 src/pkg/strings/strings_test.go create mode 100644 src/pkg/sync/Makefile create mode 100644 src/pkg/sync/atomic/Makefile create mode 100644 src/pkg/sync/atomic/asm_386.s create mode 100644 src/pkg/sync/atomic/asm_amd64.s create mode 100644 src/pkg/sync/atomic/asm_arm.s create mode 100644 src/pkg/sync/atomic/asm_linux_arm.s create mode 100644 src/pkg/sync/atomic/atomic_test.go create mode 100644 src/pkg/sync/atomic/doc.go create mode 100644 src/pkg/sync/cond.go create mode 100644 src/pkg/sync/cond_test.go create mode 100644 src/pkg/sync/mutex.go create mode 100644 src/pkg/sync/mutex_test.go create mode 100644 src/pkg/sync/once.go create mode 100644 src/pkg/sync/once_test.go create mode 100644 src/pkg/sync/rwmutex.go create mode 100644 src/pkg/sync/rwmutex_test.go create mode 100644 src/pkg/sync/waitgroup.go create mode 100644 src/pkg/sync/waitgroup_test.go create mode 100644 src/pkg/syscall/Makefile create mode 100644 src/pkg/syscall/asm_darwin_386.s create mode 100644 src/pkg/syscall/asm_darwin_amd64.s create mode 100644 src/pkg/syscall/asm_freebsd_386.s create mode 100644 src/pkg/syscall/asm_freebsd_amd64.s create mode 100644 src/pkg/syscall/asm_linux_386.s create mode 100644 src/pkg/syscall/asm_linux_amd64.s create mode 100644 src/pkg/syscall/asm_linux_arm.s create mode 100644 src/pkg/syscall/asm_plan9_386.s create mode 100644 src/pkg/syscall/asm_windows_386.s create mode 100644 src/pkg/syscall/asm_windows_amd64.s create mode 100644 src/pkg/syscall/bpf_bsd.go create mode 100644 src/pkg/syscall/exec_plan9.go create mode 100644 src/pkg/syscall/exec_unix.go create mode 100644 src/pkg/syscall/exec_windows.go create mode 100644 src/pkg/syscall/lsf_linux.go create mode 100755 src/pkg/syscall/mkall.sh create mode 100755 src/pkg/syscall/mkerrors.sh create mode 100755 src/pkg/syscall/mkerrors_windows.sh create mode 100755 src/pkg/syscall/mksyscall.pl create mode 100755 src/pkg/syscall/mksyscall_windows.pl create mode 100755 src/pkg/syscall/mksysnum_darwin.pl create mode 100755 src/pkg/syscall/mksysnum_freebsd.pl create mode 100755 src/pkg/syscall/mksysnum_linux.pl create mode 100755 src/pkg/syscall/mksysnum_plan9.sh create mode 100644 src/pkg/syscall/netlink_linux.go create mode 100644 src/pkg/syscall/route_bsd.go create mode 100644 src/pkg/syscall/route_darwin.go create mode 100644 src/pkg/syscall/route_freebsd.go create mode 100644 src/pkg/syscall/sockcmsg_linux.go create mode 100644 src/pkg/syscall/sockcmsg_unix.go create mode 100644 src/pkg/syscall/str.go create mode 100644 src/pkg/syscall/syscall.go create mode 100644 src/pkg/syscall/syscall_386.go create mode 100644 src/pkg/syscall/syscall_amd64.go create mode 100644 src/pkg/syscall/syscall_arm.go create mode 100644 src/pkg/syscall/syscall_bsd.go create mode 100644 src/pkg/syscall/syscall_darwin.go create mode 100644 src/pkg/syscall/syscall_darwin_386.go create mode 100644 src/pkg/syscall/syscall_darwin_amd64.go create mode 100644 src/pkg/syscall/syscall_freebsd.go create mode 100644 src/pkg/syscall/syscall_freebsd_386.go create mode 100644 src/pkg/syscall/syscall_freebsd_amd64.go create mode 100644 src/pkg/syscall/syscall_linux.go create mode 100644 src/pkg/syscall/syscall_linux_386.go create mode 100644 src/pkg/syscall/syscall_linux_amd64.go create mode 100644 src/pkg/syscall/syscall_linux_arm.go create mode 100644 src/pkg/syscall/syscall_plan9.go create mode 100644 src/pkg/syscall/syscall_plan9_386.go create mode 100644 src/pkg/syscall/syscall_unix.go create mode 100644 src/pkg/syscall/syscall_windows.go create mode 100644 src/pkg/syscall/syscall_windows_386.go create mode 100644 src/pkg/syscall/syscall_windows_amd64.go create mode 100644 src/pkg/syscall/types_darwin.c create mode 100644 src/pkg/syscall/types_freebsd.c create mode 100644 src/pkg/syscall/types_linux.c create mode 100644 src/pkg/syscall/types_plan9.c create mode 100644 src/pkg/syscall/zerrors_darwin_386.go create mode 100644 src/pkg/syscall/zerrors_darwin_amd64.go create mode 100644 src/pkg/syscall/zerrors_freebsd_386.go create mode 100644 src/pkg/syscall/zerrors_freebsd_amd64.go create mode 100644 src/pkg/syscall/zerrors_linux_386.go create mode 100644 src/pkg/syscall/zerrors_linux_amd64.go create mode 100644 src/pkg/syscall/zerrors_linux_arm.go create mode 100644 src/pkg/syscall/zerrors_plan9_386.go create mode 100644 src/pkg/syscall/zerrors_windows.go create mode 100644 src/pkg/syscall/zerrors_windows_386.go create mode 100644 src/pkg/syscall/zerrors_windows_amd64.go create mode 100644 src/pkg/syscall/zsyscall_darwin_386.go create mode 100644 src/pkg/syscall/zsyscall_darwin_amd64.go create mode 100644 src/pkg/syscall/zsyscall_freebsd_386.go create mode 100644 src/pkg/syscall/zsyscall_freebsd_amd64.go create mode 100644 src/pkg/syscall/zsyscall_linux_386.go create mode 100644 src/pkg/syscall/zsyscall_linux_amd64.go create mode 100644 src/pkg/syscall/zsyscall_linux_arm.go create mode 100644 src/pkg/syscall/zsyscall_plan9_386.go create mode 100644 src/pkg/syscall/zsyscall_windows_386.go create mode 100644 src/pkg/syscall/zsyscall_windows_amd64.go create mode 100644 src/pkg/syscall/zsysnum_darwin_386.go create mode 100644 src/pkg/syscall/zsysnum_darwin_amd64.go create mode 100644 src/pkg/syscall/zsysnum_freebsd_386.go create mode 100644 src/pkg/syscall/zsysnum_freebsd_amd64.go create mode 100644 src/pkg/syscall/zsysnum_linux_386.go create mode 100644 src/pkg/syscall/zsysnum_linux_amd64.go create mode 100644 src/pkg/syscall/zsysnum_linux_arm.go create mode 100644 src/pkg/syscall/zsysnum_plan9_386.go create mode 100644 src/pkg/syscall/zsysnum_windows_386.go create mode 100644 src/pkg/syscall/zsysnum_windows_amd64.go create mode 100644 src/pkg/syscall/ztypes_darwin_386.go create mode 100644 src/pkg/syscall/ztypes_darwin_amd64.go create mode 100644 src/pkg/syscall/ztypes_freebsd_386.go create mode 100644 src/pkg/syscall/ztypes_freebsd_amd64.go create mode 100644 src/pkg/syscall/ztypes_linux_386.go create mode 100644 src/pkg/syscall/ztypes_linux_amd64.go create mode 100644 src/pkg/syscall/ztypes_linux_arm.go create mode 100644 src/pkg/syscall/ztypes_plan9_386.go create mode 100644 src/pkg/syscall/ztypes_windows.go create mode 100644 src/pkg/syscall/ztypes_windows_386.go create mode 100644 src/pkg/syscall/ztypes_windows_amd64.go create mode 100644 src/pkg/syslog/Makefile create mode 100644 src/pkg/syslog/syslog.go create mode 100644 src/pkg/syslog/syslog_test.go create mode 100644 src/pkg/syslog/syslog_unix.go create mode 100644 src/pkg/tabwriter/Makefile create mode 100644 src/pkg/tabwriter/tabwriter.go create mode 100644 src/pkg/tabwriter/tabwriter_test.go create mode 100644 src/pkg/template/Makefile create mode 100644 src/pkg/template/doc.go create mode 100644 src/pkg/template/exec.go create mode 100644 src/pkg/template/exec_test.go create mode 100644 src/pkg/template/funcs.go create mode 100644 src/pkg/template/helper.go create mode 100644 src/pkg/template/parse.go create mode 100644 src/pkg/template/parse/Makefile create mode 100644 src/pkg/template/parse/lex.go create mode 100644 src/pkg/template/parse/lex_test.go create mode 100644 src/pkg/template/parse/node.go create mode 100644 src/pkg/template/parse/parse.go create mode 100644 src/pkg/template/parse/parse_test.go create mode 100644 src/pkg/template/parse/set.go create mode 100644 src/pkg/template/set.go create mode 100644 src/pkg/template/set_test.go create mode 100644 src/pkg/template/testdata/file1.tmpl create mode 100644 src/pkg/template/testdata/file2.tmpl create mode 100644 src/pkg/template/testdata/tmpl1.tmpl create mode 100644 src/pkg/template/testdata/tmpl2.tmpl create mode 100644 src/pkg/testing/Makefile create mode 100644 src/pkg/testing/benchmark.go create mode 100644 src/pkg/testing/iotest/Makefile create mode 100644 src/pkg/testing/iotest/logger.go create mode 100644 src/pkg/testing/iotest/reader.go create mode 100644 src/pkg/testing/iotest/writer.go create mode 100644 src/pkg/testing/quick/Makefile create mode 100644 src/pkg/testing/quick/quick.go create mode 100644 src/pkg/testing/quick/quick_test.go create mode 100644 src/pkg/testing/script/Makefile create mode 100644 src/pkg/testing/script/script.go create mode 100644 src/pkg/testing/script/script_test.go create mode 100644 src/pkg/testing/testing.go create mode 100644 src/pkg/time/Makefile create mode 100644 src/pkg/time/format.go create mode 100644 src/pkg/time/sleep.go create mode 100644 src/pkg/time/sleep_test.go create mode 100644 src/pkg/time/sys.go create mode 100644 src/pkg/time/sys_plan9.go create mode 100644 src/pkg/time/sys_posix.go create mode 100644 src/pkg/time/tick.go create mode 100644 src/pkg/time/tick_test.go create mode 100644 src/pkg/time/time.go create mode 100644 src/pkg/time/time_test.go create mode 100644 src/pkg/time/zoneinfo_plan9.go create mode 100644 src/pkg/time/zoneinfo_posix.go create mode 100644 src/pkg/time/zoneinfo_unix.go create mode 100644 src/pkg/time/zoneinfo_windows.go create mode 100644 src/pkg/try/Makefile create mode 100644 src/pkg/try/try.go create mode 100644 src/pkg/try/try_test.go create mode 100644 src/pkg/unicode/Makefile create mode 100644 src/pkg/unicode/casetables.go create mode 100644 src/pkg/unicode/digit.go create mode 100644 src/pkg/unicode/digit_test.go create mode 100644 src/pkg/unicode/graphic.go create mode 100644 src/pkg/unicode/graphic_test.go create mode 100644 src/pkg/unicode/letter.go create mode 100644 src/pkg/unicode/letter_test.go create mode 100644 src/pkg/unicode/maketables.go create mode 100644 src/pkg/unicode/script_test.go create mode 100644 src/pkg/unicode/tables.go create mode 100644 src/pkg/unsafe/unsafe.go create mode 100644 src/pkg/url/Makefile create mode 100644 src/pkg/url/url.go create mode 100644 src/pkg/url/url_test.go create mode 100644 src/pkg/utf16/Makefile create mode 100644 src/pkg/utf16/utf16.go create mode 100644 src/pkg/utf16/utf16_test.go create mode 100644 src/pkg/utf8/Makefile create mode 100644 src/pkg/utf8/string.go create mode 100644 src/pkg/utf8/string_test.go create mode 100644 src/pkg/utf8/utf8.go create mode 100644 src/pkg/utf8/utf8_test.go create mode 100644 src/pkg/websocket/Makefile create mode 100644 src/pkg/websocket/client.go create mode 100644 src/pkg/websocket/server.go create mode 100644 src/pkg/websocket/websocket.go create mode 100644 src/pkg/websocket/websocket_test.go create mode 100644 src/pkg/xml/Makefile create mode 100644 src/pkg/xml/atom_test.go create mode 100644 src/pkg/xml/embed_test.go create mode 100644 src/pkg/xml/marshal.go create mode 100644 src/pkg/xml/marshal_test.go create mode 100644 src/pkg/xml/read.go create mode 100644 src/pkg/xml/read_test.go create mode 100644 src/pkg/xml/xml.go create mode 100644 src/pkg/xml/xml_test.go create mode 100755 src/quietgcc.bash create mode 100755 src/run.bash create mode 100755 src/sudo.bash create mode 100755 src/version.bash create mode 100644 test/235.go create mode 100644 test/64bit.go create mode 100644 test/append.go create mode 100644 test/args.go create mode 100644 test/assign.go create mode 100644 test/assign1.go create mode 100644 test/bench/Makefile create mode 100644 test/bench/binary-tree-freelist.go create mode 100644 test/bench/binary-tree-freelist.txt create mode 100644 test/bench/binary-tree.c create mode 100644 test/bench/binary-tree.go create mode 100644 test/bench/binary-tree.txt create mode 100644 test/bench/chameneosredux.c create mode 100644 test/bench/chameneosredux.go create mode 100644 test/bench/chameneosredux.txt create mode 100644 test/bench/fannkuch-parallel.go create mode 100644 test/bench/fannkuch-parallel.txt create mode 100644 test/bench/fannkuch.c create mode 100644 test/bench/fannkuch.go create mode 100644 test/bench/fannkuch.txt create mode 100644 test/bench/fasta-1000.out create mode 100644 test/bench/fasta.c create mode 100644 test/bench/fasta.go create mode 100644 test/bench/fasta.txt create mode 100644 test/bench/k-nucleotide-parallel.go create mode 100644 test/bench/k-nucleotide-parallel.txt create mode 100644 test/bench/k-nucleotide.c create mode 100644 test/bench/k-nucleotide.go create mode 100644 test/bench/k-nucleotide.txt create mode 100644 test/bench/mandelbrot.c create mode 100644 test/bench/mandelbrot.go create mode 100644 test/bench/mandelbrot.txt create mode 100644 test/bench/meteor-contest.c create mode 100644 test/bench/meteor-contest.go create mode 100644 test/bench/meteor-contest.txt create mode 100644 test/bench/nbody.c create mode 100644 test/bench/nbody.go create mode 100644 test/bench/nbody.txt create mode 100644 test/bench/pidigits.c create mode 100644 test/bench/pidigits.go create mode 100644 test/bench/pidigits.txt create mode 100644 test/bench/regex-dna-parallel.go create mode 100644 test/bench/regex-dna-parallel.txt create mode 100644 test/bench/regex-dna.c create mode 100644 test/bench/regex-dna.go create mode 100644 test/bench/regex-dna.txt create mode 100644 test/bench/reverse-complement.c create mode 100644 test/bench/reverse-complement.go create mode 100644 test/bench/reverse-complement.txt create mode 100644 test/bench/spectral-norm-parallel.go create mode 100644 test/bench/spectral-norm.c create mode 100644 test/bench/spectral-norm.go create mode 100644 test/bench/spectral-norm.txt create mode 100644 test/bench/threadring.c create mode 100644 test/bench/threadring.go create mode 100644 test/bench/threadring.txt create mode 100644 test/bench/timing.log create mode 100755 test/bench/timing.sh create mode 100644 test/bigalg.go create mode 100644 test/bigmap.go create mode 100644 test/blank.go create mode 100644 test/blank1.go create mode 100644 test/bugs/placeholder create mode 100644 test/chan/doubleselect.go create mode 100644 test/chan/fifo.go create mode 100644 test/chan/goroutines.go create mode 100644 test/chan/nonblock.go create mode 100644 test/chan/perm.go create mode 100644 test/chan/powser1.go create mode 100644 test/chan/powser2.go create mode 100644 test/chan/select.go create mode 100644 test/chan/select2.go create mode 100644 test/chan/select3.go create mode 100644 test/chan/select4.go create mode 100644 test/chan/select5.go create mode 100644 test/chan/select6.go create mode 100644 test/chan/sendstmt.go create mode 100644 test/chan/sieve1.go create mode 100644 test/chan/sieve2.go create mode 100644 test/chan/zerosize.go create mode 100644 test/chancap.go create mode 100644 test/char_lit.go create mode 100644 test/char_lit1.go create mode 100644 test/closedchan.go create mode 100644 test/closure.go create mode 100644 test/cmp1.go create mode 100644 test/cmp2.go create mode 100644 test/cmp3.go create mode 100644 test/cmp4.go create mode 100644 test/cmp5.go create mode 100644 test/cmp6.go create mode 100644 test/cmplx.go create mode 100644 test/cmplxdivide.c create mode 100644 test/cmplxdivide.go create mode 100644 test/cmplxdivide1.go create mode 100644 test/complit.go create mode 100644 test/compos.go create mode 100644 test/const.go create mode 100644 test/const1.go create mode 100644 test/const2.go create mode 100644 test/const3.go create mode 100644 test/convert.go create mode 100644 test/convert3.go create mode 100644 test/convlit.go create mode 100644 test/convlit1.go create mode 100644 test/copy.go create mode 100644 test/ddd.go create mode 100644 test/ddd1.go create mode 100644 test/ddd2.go create mode 100644 test/ddd3.go create mode 100644 test/decl.go create mode 100644 test/declbad.go create mode 100644 test/defer.go create mode 100644 test/deferprint.go create mode 100644 test/divide.go create mode 100644 test/empty.go create mode 100644 test/env.go create mode 100644 test/eof.go create mode 100644 test/eof1.go create mode 100755 test/errchk create mode 100644 test/escape.go create mode 100644 test/fixedbugs/bug000.go create mode 100644 test/fixedbugs/bug002.go create mode 100644 test/fixedbugs/bug003.go create mode 100644 test/fixedbugs/bug004.go create mode 100644 test/fixedbugs/bug005.go create mode 100644 test/fixedbugs/bug006.go create mode 100644 test/fixedbugs/bug007.go create mode 100644 test/fixedbugs/bug008.go create mode 100644 test/fixedbugs/bug009.go create mode 100644 test/fixedbugs/bug010.go create mode 100644 test/fixedbugs/bug011.go create mode 100644 test/fixedbugs/bug012.go create mode 100644 test/fixedbugs/bug013.go create mode 100644 test/fixedbugs/bug014.go create mode 100644 test/fixedbugs/bug015.go create mode 100644 test/fixedbugs/bug016.go create mode 100644 test/fixedbugs/bug017.go create mode 100644 test/fixedbugs/bug020.go create mode 100644 test/fixedbugs/bug021.go create mode 100644 test/fixedbugs/bug022.go create mode 100644 test/fixedbugs/bug023.go create mode 100644 test/fixedbugs/bug024.go create mode 100644 test/fixedbugs/bug026.go create mode 100644 test/fixedbugs/bug027.go create mode 100644 test/fixedbugs/bug028.go create mode 100644 test/fixedbugs/bug030.go create mode 100644 test/fixedbugs/bug031.go create mode 100644 test/fixedbugs/bug035.go create mode 100644 test/fixedbugs/bug036.go create mode 100644 test/fixedbugs/bug037.go create mode 100644 test/fixedbugs/bug038.go create mode 100644 test/fixedbugs/bug039.go create mode 100644 test/fixedbugs/bug040.go create mode 100644 test/fixedbugs/bug045.go create mode 100644 test/fixedbugs/bug046.go create mode 100644 test/fixedbugs/bug047.go create mode 100644 test/fixedbugs/bug048.go create mode 100644 test/fixedbugs/bug049.go create mode 100644 test/fixedbugs/bug050.go create mode 100644 test/fixedbugs/bug051.go create mode 100644 test/fixedbugs/bug052.go create mode 100644 test/fixedbugs/bug053.go create mode 100644 test/fixedbugs/bug054.go create mode 100644 test/fixedbugs/bug055.go create mode 100644 test/fixedbugs/bug056.go create mode 100644 test/fixedbugs/bug057.go create mode 100644 test/fixedbugs/bug058.go create mode 100644 test/fixedbugs/bug059.go create mode 100644 test/fixedbugs/bug060.go create mode 100644 test/fixedbugs/bug061.go create mode 100644 test/fixedbugs/bug062.go create mode 100644 test/fixedbugs/bug063.go create mode 100644 test/fixedbugs/bug064.go create mode 100644 test/fixedbugs/bug065.go create mode 100644 test/fixedbugs/bug066.go create mode 100644 test/fixedbugs/bug067.go create mode 100644 test/fixedbugs/bug068.go create mode 100644 test/fixedbugs/bug069.go create mode 100644 test/fixedbugs/bug070.go create mode 100644 test/fixedbugs/bug071.go create mode 100644 test/fixedbugs/bug072.go create mode 100644 test/fixedbugs/bug073.go create mode 100644 test/fixedbugs/bug074.go create mode 100644 test/fixedbugs/bug075.go create mode 100644 test/fixedbugs/bug076.go create mode 100644 test/fixedbugs/bug077.go create mode 100644 test/fixedbugs/bug078.go create mode 100644 test/fixedbugs/bug080.go create mode 100644 test/fixedbugs/bug081.go create mode 100644 test/fixedbugs/bug082.go create mode 100644 test/fixedbugs/bug083.dir/bug0.go create mode 100644 test/fixedbugs/bug083.dir/bug1.go create mode 100644 test/fixedbugs/bug083.go create mode 100644 test/fixedbugs/bug084.go create mode 100644 test/fixedbugs/bug085.go create mode 100644 test/fixedbugs/bug086.go create mode 100644 test/fixedbugs/bug087.go create mode 100644 test/fixedbugs/bug088.dir/bug0.go create mode 100644 test/fixedbugs/bug088.dir/bug1.go create mode 100644 test/fixedbugs/bug088.go create mode 100644 test/fixedbugs/bug089.go create mode 100644 test/fixedbugs/bug090.go create mode 100644 test/fixedbugs/bug091.go create mode 100644 test/fixedbugs/bug092.go create mode 100644 test/fixedbugs/bug093.go create mode 100644 test/fixedbugs/bug094.go create mode 100644 test/fixedbugs/bug096.go create mode 100644 test/fixedbugs/bug097.go create mode 100644 test/fixedbugs/bug098.go create mode 100644 test/fixedbugs/bug099.go create mode 100644 test/fixedbugs/bug101.go create mode 100644 test/fixedbugs/bug102.go create mode 100644 test/fixedbugs/bug103.go create mode 100644 test/fixedbugs/bug104.go create mode 100644 test/fixedbugs/bug106.dir/bug0.go create mode 100644 test/fixedbugs/bug106.dir/bug1.go create mode 100644 test/fixedbugs/bug106.go create mode 100644 test/fixedbugs/bug107.go create mode 100644 test/fixedbugs/bug108.go create mode 100644 test/fixedbugs/bug109.go create mode 100644 test/fixedbugs/bug110.go create mode 100644 test/fixedbugs/bug111.go create mode 100644 test/fixedbugs/bug112.go create mode 100644 test/fixedbugs/bug113.go create mode 100644 test/fixedbugs/bug114.go create mode 100644 test/fixedbugs/bug115.go create mode 100644 test/fixedbugs/bug116.go create mode 100644 test/fixedbugs/bug117.go create mode 100644 test/fixedbugs/bug118.go create mode 100644 test/fixedbugs/bug119.go create mode 100644 test/fixedbugs/bug120.go create mode 100644 test/fixedbugs/bug121.go create mode 100644 test/fixedbugs/bug122.go create mode 100644 test/fixedbugs/bug123.go create mode 100644 test/fixedbugs/bug126.go create mode 100644 test/fixedbugs/bug127.go create mode 100644 test/fixedbugs/bug128.go create mode 100644 test/fixedbugs/bug129.go create mode 100644 test/fixedbugs/bug130.go create mode 100644 test/fixedbugs/bug131.go create mode 100644 test/fixedbugs/bug132.go create mode 100644 test/fixedbugs/bug133.dir/bug0.go create mode 100644 test/fixedbugs/bug133.dir/bug1.go create mode 100644 test/fixedbugs/bug133.dir/bug2.go create mode 100644 test/fixedbugs/bug133.go create mode 100644 test/fixedbugs/bug135.go create mode 100644 test/fixedbugs/bug136.go create mode 100644 test/fixedbugs/bug137.go create mode 100644 test/fixedbugs/bug139.go create mode 100644 test/fixedbugs/bug140.go create mode 100644 test/fixedbugs/bug141.go create mode 100644 test/fixedbugs/bug142.go create mode 100644 test/fixedbugs/bug143.go create mode 100644 test/fixedbugs/bug144.go create mode 100644 test/fixedbugs/bug145.go create mode 100644 test/fixedbugs/bug146.go create mode 100644 test/fixedbugs/bug147.go create mode 100644 test/fixedbugs/bug148.go create mode 100644 test/fixedbugs/bug149.go create mode 100644 test/fixedbugs/bug150.go create mode 100644 test/fixedbugs/bug151.go create mode 100644 test/fixedbugs/bug1515.go create mode 100644 test/fixedbugs/bug152.go create mode 100644 test/fixedbugs/bug154.go create mode 100644 test/fixedbugs/bug155.go create mode 100644 test/fixedbugs/bug156.go create mode 100644 test/fixedbugs/bug157.go create mode 100644 test/fixedbugs/bug158.go create mode 100644 test/fixedbugs/bug159.go create mode 100644 test/fixedbugs/bug160.dir/x.go create mode 100644 test/fixedbugs/bug160.dir/y.go create mode 100644 test/fixedbugs/bug160.go create mode 100644 test/fixedbugs/bug161.go create mode 100644 test/fixedbugs/bug163.go create mode 100644 test/fixedbugs/bug164.go create mode 100644 test/fixedbugs/bug165.go create mode 100644 test/fixedbugs/bug167.go create mode 100644 test/fixedbugs/bug168.go create mode 100644 test/fixedbugs/bug169.go create mode 100644 test/fixedbugs/bug170.go create mode 100644 test/fixedbugs/bug171.go create mode 100644 test/fixedbugs/bug172.go create mode 100644 test/fixedbugs/bug173.go create mode 100644 test/fixedbugs/bug174.go create mode 100644 test/fixedbugs/bug175.go create mode 100644 test/fixedbugs/bug176.go create mode 100644 test/fixedbugs/bug177.go create mode 100644 test/fixedbugs/bug178.go create mode 100644 test/fixedbugs/bug179.go create mode 100644 test/fixedbugs/bug180.go create mode 100644 test/fixedbugs/bug181.go create mode 100644 test/fixedbugs/bug182.go create mode 100644 test/fixedbugs/bug183.go create mode 100644 test/fixedbugs/bug184.go create mode 100644 test/fixedbugs/bug185.go create mode 100644 test/fixedbugs/bug186.go create mode 100644 test/fixedbugs/bug187.go create mode 100644 test/fixedbugs/bug188.go create mode 100644 test/fixedbugs/bug189.go create mode 100644 test/fixedbugs/bug190.go create mode 100644 test/fixedbugs/bug191.dir/a.go create mode 100644 test/fixedbugs/bug191.dir/b.go create mode 100644 test/fixedbugs/bug191.go create mode 100644 test/fixedbugs/bug192.go create mode 100644 test/fixedbugs/bug193.go create mode 100644 test/fixedbugs/bug194.go create mode 100644 test/fixedbugs/bug195.go create mode 100644 test/fixedbugs/bug196.go create mode 100644 test/fixedbugs/bug197.go create mode 100644 test/fixedbugs/bug198.go create mode 100644 test/fixedbugs/bug199.go create mode 100644 test/fixedbugs/bug200.go create mode 100644 test/fixedbugs/bug201.go create mode 100644 test/fixedbugs/bug202.go create mode 100644 test/fixedbugs/bug203.go create mode 100644 test/fixedbugs/bug204.go create mode 100644 test/fixedbugs/bug205.go create mode 100644 test/fixedbugs/bug206.go create mode 100644 test/fixedbugs/bug207.go create mode 100644 test/fixedbugs/bug208.go create mode 100644 test/fixedbugs/bug209.go create mode 100644 test/fixedbugs/bug211.go create mode 100644 test/fixedbugs/bug212.go create mode 100644 test/fixedbugs/bug213.go create mode 100644 test/fixedbugs/bug214.go create mode 100644 test/fixedbugs/bug215.go create mode 100644 test/fixedbugs/bug216.go create mode 100644 test/fixedbugs/bug217.go create mode 100644 test/fixedbugs/bug218.go create mode 100644 test/fixedbugs/bug219.go create mode 100644 test/fixedbugs/bug220.go create mode 100644 test/fixedbugs/bug221.go create mode 100644 test/fixedbugs/bug222.dir/chanbug.go create mode 100644 test/fixedbugs/bug222.dir/chanbug2.go create mode 100644 test/fixedbugs/bug222.go create mode 100644 test/fixedbugs/bug223.go create mode 100644 test/fixedbugs/bug224.go create mode 100644 test/fixedbugs/bug225.go create mode 100644 test/fixedbugs/bug226.dir/x.go create mode 100644 test/fixedbugs/bug226.dir/y.go create mode 100644 test/fixedbugs/bug226.go create mode 100644 test/fixedbugs/bug227.go create mode 100644 test/fixedbugs/bug228.go create mode 100644 test/fixedbugs/bug229.go create mode 100644 test/fixedbugs/bug230.go create mode 100644 test/fixedbugs/bug231.go create mode 100644 test/fixedbugs/bug232.go create mode 100644 test/fixedbugs/bug233.go create mode 100644 test/fixedbugs/bug234.go create mode 100644 test/fixedbugs/bug235.go create mode 100644 test/fixedbugs/bug236.go create mode 100644 test/fixedbugs/bug237.go create mode 100644 test/fixedbugs/bug238.go create mode 100644 test/fixedbugs/bug239.go create mode 100644 test/fixedbugs/bug240.go create mode 100644 test/fixedbugs/bug241.go create mode 100644 test/fixedbugs/bug242.go create mode 100644 test/fixedbugs/bug243.go create mode 100644 test/fixedbugs/bug244.go create mode 100644 test/fixedbugs/bug245.go create mode 100644 test/fixedbugs/bug246.go create mode 100644 test/fixedbugs/bug247.go create mode 100644 test/fixedbugs/bug248.dir/bug0.go create mode 100644 test/fixedbugs/bug248.dir/bug1.go create mode 100644 test/fixedbugs/bug248.dir/bug2.go create mode 100644 test/fixedbugs/bug248.dir/bug3.go create mode 100644 test/fixedbugs/bug248.go create mode 100644 test/fixedbugs/bug249.go create mode 100644 test/fixedbugs/bug250.go create mode 100644 test/fixedbugs/bug251.go create mode 100644 test/fixedbugs/bug252.go create mode 100644 test/fixedbugs/bug253.go create mode 100644 test/fixedbugs/bug254.go create mode 100644 test/fixedbugs/bug255.go create mode 100644 test/fixedbugs/bug256.go create mode 100644 test/fixedbugs/bug257.go create mode 100644 test/fixedbugs/bug258.go create mode 100644 test/fixedbugs/bug259.go create mode 100644 test/fixedbugs/bug260.go create mode 100644 test/fixedbugs/bug261.go create mode 100644 test/fixedbugs/bug262.go create mode 100644 test/fixedbugs/bug263.go create mode 100644 test/fixedbugs/bug264.go create mode 100644 test/fixedbugs/bug265.go create mode 100644 test/fixedbugs/bug266.go create mode 100644 test/fixedbugs/bug267.go create mode 100644 test/fixedbugs/bug268.go create mode 100644 test/fixedbugs/bug269.go create mode 100644 test/fixedbugs/bug270.go create mode 100644 test/fixedbugs/bug271.go create mode 100644 test/fixedbugs/bug272.go create mode 100644 test/fixedbugs/bug273.go create mode 100644 test/fixedbugs/bug274.go create mode 100644 test/fixedbugs/bug275.go create mode 100644 test/fixedbugs/bug276.go create mode 100644 test/fixedbugs/bug277.go create mode 100644 test/fixedbugs/bug278.go create mode 100644 test/fixedbugs/bug279.go create mode 100644 test/fixedbugs/bug280.go create mode 100644 test/fixedbugs/bug281.go create mode 100644 test/fixedbugs/bug282.dir/p1.go create mode 100644 test/fixedbugs/bug282.dir/p2.go create mode 100644 test/fixedbugs/bug282.go create mode 100644 test/fixedbugs/bug283.go create mode 100644 test/fixedbugs/bug284.go create mode 100644 test/fixedbugs/bug285.go create mode 100644 test/fixedbugs/bug286.go create mode 100644 test/fixedbugs/bug287.go create mode 100644 test/fixedbugs/bug288.go create mode 100644 test/fixedbugs/bug289.go create mode 100644 test/fixedbugs/bug290.go create mode 100644 test/fixedbugs/bug291.go create mode 100644 test/fixedbugs/bug292.go create mode 100644 test/fixedbugs/bug293.go create mode 100644 test/fixedbugs/bug294.go create mode 100644 test/fixedbugs/bug295.go create mode 100644 test/fixedbugs/bug296.go create mode 100644 test/fixedbugs/bug297.go create mode 100644 test/fixedbugs/bug298.go create mode 100644 test/fixedbugs/bug299.go create mode 100644 test/fixedbugs/bug300.go create mode 100644 test/fixedbugs/bug301.go create mode 100644 test/fixedbugs/bug302.dir/main.go create mode 100644 test/fixedbugs/bug302.dir/p.go create mode 100644 test/fixedbugs/bug302.go create mode 100644 test/fixedbugs/bug303.go create mode 100644 test/fixedbugs/bug304.go create mode 100644 test/fixedbugs/bug305.go create mode 100644 test/fixedbugs/bug306.dir/p1.go create mode 100644 test/fixedbugs/bug306.dir/p2.go create mode 100644 test/fixedbugs/bug306.go create mode 100644 test/fixedbugs/bug307.go create mode 100644 test/fixedbugs/bug308.go create mode 100644 test/fixedbugs/bug309.go create mode 100644 test/fixedbugs/bug310.go create mode 100644 test/fixedbugs/bug311.go create mode 100644 test/fixedbugs/bug312.go create mode 100644 test/fixedbugs/bug313.dir/a.go create mode 100644 test/fixedbugs/bug313.dir/b.go create mode 100644 test/fixedbugs/bug313.go create mode 100644 test/fixedbugs/bug314.go create mode 100644 test/fixedbugs/bug315.go create mode 100644 test/fixedbugs/bug316.go create mode 100644 test/fixedbugs/bug317.go create mode 100644 test/fixedbugs/bug318.go create mode 100644 test/fixedbugs/bug319.go create mode 100644 test/fixedbugs/bug320.go create mode 100644 test/fixedbugs/bug321.go create mode 100644 test/fixedbugs/bug322.dir/lib.go create mode 100644 test/fixedbugs/bug322.dir/main.go create mode 100644 test/fixedbugs/bug322.go create mode 100644 test/fixedbugs/bug323.go create mode 100644 test/fixedbugs/bug324.dir/main.go create mode 100644 test/fixedbugs/bug324.dir/p.go create mode 100644 test/fixedbugs/bug324.go create mode 100644 test/fixedbugs/bug325.go create mode 100644 test/fixedbugs/bug326.go create mode 100644 test/fixedbugs/bug327.go create mode 100644 test/fixedbugs/bug328.go create mode 100644 test/fixedbugs/bug329.go create mode 100644 test/fixedbugs/bug330.go create mode 100644 test/fixedbugs/bug331.go create mode 100644 test/fixedbugs/bug332.go create mode 100644 test/fixedbugs/bug333.go create mode 100644 test/fixedbugs/bug334.go create mode 100644 test/fixedbugs/bug335.dir/a.go create mode 100644 test/fixedbugs/bug335.dir/b.go create mode 100644 test/fixedbugs/bug335.go create mode 100644 test/fixedbugs/bug336.go create mode 100644 test/fixedbugs/bug337.go create mode 100644 test/fixedbugs/bug338.go create mode 100644 test/fixedbugs/bug339.go create mode 100644 test/fixedbugs/bug340.go create mode 100644 test/fixedbugs/bug341.go create mode 100644 test/fixedbugs/bug342.go create mode 100644 test/fixedbugs/bug343.go create mode 100644 test/fixedbugs/bug344.go create mode 100644 test/fixedbugs/bug345.dir/io.go create mode 100644 test/fixedbugs/bug345.dir/main.go create mode 100644 test/fixedbugs/bug345.go create mode 100644 test/fixedbugs/bug346.go create mode 100644 test/fixedbugs/bug347.go create mode 100644 test/fixedbugs/bug348.go create mode 100644 test/fixedbugs/bug349.go create mode 100644 test/fixedbugs/bug350.go create mode 100644 test/fixedbugs/bug351.go create mode 100644 test/fixedbugs/bug352.go create mode 100644 test/fixedbugs/bug353.go create mode 100644 test/fixedbugs/bug354.go create mode 100644 test/fixedbugs/bug355.go create mode 100644 test/fixedbugs/bug356.go create mode 100644 test/fixedbugs/bug357.go create mode 100644 test/fixedbugs/bug358.go create mode 100644 test/fixedbugs/bug359.go create mode 100644 test/fixedbugs/bug361.go create mode 100644 test/fixedbugs/bug362.go create mode 100644 test/fixedbugs/bug363.go create mode 100644 test/fixedbugs/bug364.go create mode 100644 test/fixedbugs/bug365.go create mode 100644 test/float_lit.go create mode 100644 test/floatcmp.go create mode 100644 test/for.go create mode 100644 test/func.go create mode 100644 test/func1.go create mode 100644 test/func2.go create mode 100644 test/func3.go create mode 100644 test/func4.go create mode 100644 test/func5.go create mode 100644 test/func6.go create mode 100644 test/func7.go create mode 100644 test/garbage/Makefile create mode 100644 test/garbage/parser.go create mode 100644 test/garbage/peano.go create mode 100644 test/garbage/stats.go create mode 100644 test/garbage/tree.go create mode 100644 test/gc.go create mode 100644 test/gc1.go create mode 100644 test/gc2.go create mode 100644 test/golden.out create mode 100644 test/goprint.go create mode 100644 test/goto.go create mode 100755 test/hashmap.go create mode 100644 test/helloworld.go create mode 100644 test/if.go create mode 100644 test/import.go create mode 100644 test/import1.go create mode 100644 test/import2.go create mode 100644 test/import3.go create mode 100644 test/import4.go create mode 100644 test/index.go create mode 100644 test/indirect.go create mode 100644 test/indirect1.go create mode 100644 test/init.go create mode 100644 test/initcomma.go create mode 100644 test/initialize.go create mode 100644 test/initializerr.go create mode 100644 test/initsyscall.go create mode 100644 test/int_lit.go create mode 100644 test/intcvt.go create mode 100644 test/interface/bigdata.go create mode 100644 test/interface/convert.go create mode 100644 test/interface/convert1.go create mode 100644 test/interface/convert2.go create mode 100644 test/interface/embed.go create mode 100644 test/interface/embed0.go create mode 100644 test/interface/embed1.go create mode 100644 test/interface/embed2.go create mode 100644 test/interface/explicit.go create mode 100644 test/interface/fail.go create mode 100644 test/interface/fake.go create mode 100644 test/interface/pointer.go create mode 100644 test/interface/private.go create mode 100644 test/interface/private1.go create mode 100644 test/interface/receiver.go create mode 100644 test/interface/receiver1.go create mode 100644 test/interface/recursive.go create mode 100644 test/interface/returntype.go create mode 100644 test/interface/struct.go create mode 100644 test/iota.go create mode 100644 test/ken/array.go create mode 100644 test/ken/chan.go create mode 100644 test/ken/chan1.go create mode 100644 test/ken/complit.go create mode 100644 test/ken/convert.go create mode 100644 test/ken/cplx0.go create mode 100644 test/ken/cplx1.go create mode 100644 test/ken/cplx2.go create mode 100644 test/ken/cplx3.go create mode 100644 test/ken/cplx4.go create mode 100644 test/ken/cplx5.go create mode 100644 test/ken/divconst.go create mode 100644 test/ken/divmod.go create mode 100644 test/ken/embed.go create mode 100644 test/ken/for.go create mode 100644 test/ken/interbasic.go create mode 100644 test/ken/interfun.go create mode 100644 test/ken/intervar.go create mode 100644 test/ken/label.go create mode 100644 test/ken/litfun.go create mode 100644 test/ken/mfunc.go create mode 100644 test/ken/modconst.go create mode 100644 test/ken/ptrfun.go create mode 100644 test/ken/ptrvar.go create mode 100644 test/ken/range.go create mode 100644 test/ken/rob1.go create mode 100644 test/ken/rob2.go create mode 100644 test/ken/robfor.go create mode 100644 test/ken/robfunc.go create mode 100644 test/ken/shift.go create mode 100644 test/ken/simparray.go create mode 100644 test/ken/simpbool.go create mode 100644 test/ken/simpconv.go create mode 100644 test/ken/simpfun.go create mode 100644 test/ken/simpprint.go create mode 100644 test/ken/simpswitch.go create mode 100644 test/ken/simpvar.go create mode 100644 test/ken/slicearray.go create mode 100644 test/ken/sliceslice.go create mode 100644 test/ken/string.go create mode 100644 test/ken/strvar.go create mode 100644 test/label.go create mode 100644 test/label1.go create mode 100644 test/literal.go create mode 100644 test/malloc1.go create mode 100644 test/mallocfin.go create mode 100644 test/mallocrand.go create mode 100644 test/mallocrep.go create mode 100644 test/mallocrep1.go create mode 100644 test/map.go create mode 100644 test/method.go create mode 100644 test/method1.go create mode 100644 test/method2.go create mode 100644 test/method3.go create mode 100644 test/named.go create mode 100644 test/named1.go create mode 100644 test/nil.go create mode 100644 test/nilptr/arrayindex.go create mode 100644 test/nilptr/arrayindex1.go create mode 100644 test/nilptr/arraytoslice.go create mode 100644 test/nilptr/arraytoslice1.go create mode 100644 test/nilptr/arraytoslice2.go create mode 100644 test/nilptr/slicearray.go create mode 100644 test/nilptr/structfield.go create mode 100644 test/nilptr/structfield1.go create mode 100644 test/nilptr/structfield2.go create mode 100644 test/nilptr/structfieldaddr.go create mode 100644 test/nul1.go create mode 100644 test/parentype.go create mode 100644 test/peano.go create mode 100644 test/printbig.go create mode 100644 test/range.go create mode 100644 test/recover.go create mode 100644 test/recover1.go create mode 100644 test/recover2.go create mode 100644 test/recover3.go create mode 100644 test/rename.go create mode 100644 test/rename1.go create mode 100755 test/run create mode 100644 test/runtime.go create mode 100644 test/shift1.go create mode 100644 test/shift2.go create mode 100644 test/sieve.go create mode 100644 test/sigchld.go create mode 100644 test/simassign.go create mode 100644 test/sinit.go create mode 100644 test/sizeof.go create mode 100644 test/solitaire.go create mode 100644 test/stack.go create mode 100644 test/string_lit.go create mode 100644 test/stringrange.go create mode 100644 test/switch.go create mode 100644 test/switch1.go create mode 100644 test/syntax/chan.go create mode 100644 test/syntax/chan1.go create mode 100644 test/syntax/forvar.go create mode 100644 test/syntax/if.go create mode 100644 test/syntax/import.go create mode 100644 test/syntax/interface.go create mode 100644 test/syntax/semi1.go create mode 100644 test/syntax/semi2.go create mode 100644 test/syntax/semi3.go create mode 100644 test/syntax/semi4.go create mode 100644 test/syntax/semi5.go create mode 100644 test/syntax/semi6.go create mode 100644 test/syntax/semi7.go create mode 100644 test/syntax/topexpr.go create mode 100644 test/syntax/typesw.go create mode 100644 test/syntax/vareq.go create mode 100644 test/syntax/vareq1.go create mode 100644 test/test0.go create mode 100644 test/turing.go create mode 100644 test/typeswitch.go create mode 100644 test/typeswitch1.go create mode 100644 test/typeswitch2.go create mode 100644 test/undef.go create mode 100644 test/utf.go create mode 100644 test/varerr.go create mode 100644 test/varinit.go create mode 100644 test/vectors.go create mode 100644 test/zerodivide.go (limited to 'src/pkg/math/atanh.go') diff --git a/AUTHORS b/AUTHORS new file mode 100644 index 000000000..d8bf4b45c --- /dev/null +++ b/AUTHORS @@ -0,0 +1,158 @@ +# This is the official list of Go authors for copyright purposes. +# This file is distinct from the CONTRIBUTORS files. +# See the latter for an explanation. + +# Names should be added to this file as +# Name or Organization +# The email address is not required for organizations. + +# Please keep the list sorted. + +Abhinav Gupta +Adrian O'Grady +Albert Strasheim +Aleksandar Dezelin +Alex Brainman +Alexander Orlov +Alexey Borzenkov +Amrut Joshi +Andrei Vieru +Andrew Balholm +Andrew Skiba +Andrey Mirtchovski +Andy Davis +Anh Hai Trinh +Anschel Schaffer-Cohen +Anthony Martin +Anthony Starks +Aron Nopanen +Arvindh Rajesh Tamilmani +Ben Olive +Benny Siegert +Bobby Powers +Caine Tighe +Charles L. Dorian +Chris Dollin +Chris Jones +Chris Lennert +Christian Himpel +Christopher Nielsen +Christopher Wedgwood +Clement Skau +Conrad Meyer +Corey Thomasson +Dan Sinclair +Daniel Fleischman +Daniel Theophanes +Dave Cheney +David Forsythe +David G. Andersen +David Jakob Fritz +David Titarenco +Dean Prichard +Devon H. O'Dell +Dmitry Chestnykh +Eden Li +Eivind Uggedal +Eoghan Sherry +Eric Clark +Eric Eisner +Evan Shaw +Fan Hongjian +Fazlul Shahriar +Firmansyah Adiputra +Florian Uekermann +Gary Burd +Gideon Jan-Wessel Redelinghuys +Giles Lean +Google Inc. +Graham Miller +Gustavo Niemeyer +Harley Laue +Hector Chu +Icarus Sparry +Isaac Wagner +James Fysh +James Meneghello +James Toy +James Whitehead +Jan H. Hosang +Jan Mercl +Jeff Hodges +Jeff R. Allen +Jim McGrath +Joe Poirier +John Asmuth +Jonathan Mark +Jonathan Wills +Jose Luis Vázquez González +Josh Goebel +Jukka-Pekka Kekkonen +Julian Phillips +Kai Backman +Kei Son +Keith Rarick +Ken Friedenbach +Ken Rockot +Kevin Ballard +Kyle Consalus +Kyle Lemons +Lorenzo Stoakes +Lucio De Re +Luit van Drongelen +Markus Duft +Martin Neubauer +Mathieu Lonjaret +Matthew Horsnell +Micah Stetson +Michael Elkins +Michael Hoisie +Miek Gieben +Mikael Tillenius +Mikio Hara +Mikkel Krautz +Moriyoshi Koizumi +Môshe van der Sterre +ngmoco, LLC +Nicholas Waples +Nigel Kerr +Olivier Antoine +Padraig Kitterick +Paolo Giarrusso +Pascal S. de Kloe +Patrick Gavlin +Petar Maymounkov +Peter Froehlich +Peter Mundy +Peter Williams +Pieter Droogendijk +Quan Yong Zhai +Raif S. Naffah +Risto Jaakko Saarelma +Robert Hencke +Roger Pau Monné +Roger Peppe +Ross Light +Ryan Hitchman +Scott Lawrence +Sebastien Binet +Sergei Skorobogatov +Sergey 'SnakE' Gromov +Sergio Luis O. B. Correia +Spring Mc +Stefan Nilsson +Stephen Weinberg +Sven Almgren +Tarmigan Casebolt +Timo Savola +Tor Andersson +Vincent Ambo +Vincent Vanackere +Vinu Rajashekhar +Volker Dobler +Wei Guangjing +William Josephson +Yasuhiro Matsumoto +Yongjian Xu +Yuusei Kuwana +Yuval Pavel Zholkover diff --git a/CONTRIBUTORS b/CONTRIBUTORS new file mode 100644 index 000000000..293df0bb2 --- /dev/null +++ b/CONTRIBUTORS @@ -0,0 +1,238 @@ +# This is the official list of people who can contribute +# (and typically have contributed) code to the Go repository. +# The AUTHORS file lists the copyright holders; this file +# lists people. For example, Google employees are listed here +# but not in AUTHORS, because Google holds the copyright. +# +# The submission process automatically checks to make sure +# that people submitting code are listed in this file (by email address). +# +# Names should be added to this file only after verifying that +# the individual or the individual's organization has agreed to +# the appropriate Contributor License Agreement, found here: +# +# http://code.google.com/legal/individual-cla-v1.0.html +# http://code.google.com/legal/corporate-cla-v1.0.html +# +# The agreement for individuals can be filled out on the web. +# +# When adding J Random Contributor's name to this file, +# either J's name or J's organization's name should be +# added to the AUTHORS file, depending on whether the +# individual or corporate CLA was used. + +# Names should be added to this file like so: +# Name +# +# An entry with two email addresses specifies that the +# first address should be used in the submit logs and +# that the second address should be recognized as the +# same person when interacting with Rietveld. + +# Please keep the list sorted. + +Abhinav Gupta +Adam Langley +Adrian O'Grady +Albert Strasheim +Aleksandar Dezelin +Alex Brainman +Alexander Orlov +Alexey Borzenkov +Amrut Joshi +Andrei Vieru +Andrew Balholm +Andrew Gerrand +Andrew Skiba +Andrey Mirtchovski +Andy Davis +Anh Hai Trinh +Anschel Schaffer-Cohen +Anthony Martin +Anthony Starks +Aron Nopanen +Arvindh Rajesh Tamilmani +Austin Clements +Balazs Lecz +Ben Eitzen +Ben Lynn +Ben Olive +Benny Siegert +Berengar Lehr +Bill Neubauer +Bobby Powers +Brad Fitzpatrick +Brendan O'Dea +Caine Tighe +Cary Hull +Charles L. Dorian +Chris Dollin +Chris Jones +Chris Lennert +Christian Himpel +Christopher Nielsen +Christopher Wedgwood +Clement Skau +Conrad Meyer +Corey Thomasson +Dan Sinclair +Daniel Fleischman +Daniel Nadasi +Daniel Theophanes +Dave Cheney +Dave Grijalva +David Anderson +David Crawshaw +David Forsythe +David G. Andersen +David Jakob Fritz +David Symonds +David Titarenco +Dean Prichard +Devon H. O'Dell +Dmitriy Vyukov +Dmitry Chestnykh +Eden Li +Eivind Uggedal +Eoghan Sherry +Eric Clark +Eric Eisner +Evan Martin +Evan Shaw +Fan Hongjian +Fazlul Shahriar +Firmansyah Adiputra +Florian Uekermann +Fumitoshi Ukai +Gary Burd +Gideon Jan-Wessel Redelinghuys +Giles Lean +Graham Miller +Gustavo Niemeyer +Harley Laue +Hector Chu +Ian Lance Taylor +Icarus Sparry +Isaac Wagner +Ivan Krasin +Jacob Baskin +James Aguilar +James Fysh +James Meneghello +James Toy +James Whitehead +Jamie Gennis +Jan H. Hosang +Jan Mercl +Jeff Hodges +Jeff R. Allen +Jim McGrath +Joe Poirier +Joel Sing +Johan Euphrosine +John Asmuth +John DeNero +Jonathan Allie +Jonathan Mark +Jonathan Wills +Jos Visser +Jose Luis Vázquez González +Josh Goebel +Jukka-Pekka Kekkonen +Julian Phillips +Kai Backman +Kei Son +Keith Rarick +Ken Friedenbach +Ken Rockot +Ken Thompson +Kevin Ballard +Kirklin McDonald +Kyle Consalus +Kyle Lemons +Larry Hosken +Lorenzo Stoakes +Lucio De Re +Luit van Drongelen +Luuk van Dijk +Marcel van Lohuizen +Mark Zavislak +Markus Duft +Martin Neubauer +Mathieu Lonjaret +Matt Jones +Matthew Horsnell +Maxim Ushakov +Micah Stetson +Michael Elkins +Michael Hoisie +Michael T. Jones +Miek Gieben +Mikael Tillenius +Mike Samuel +Mike Solomon +Mikio Hara +Mikkel Krautz +Moriyoshi Koizumi +Môshe van der Sterre +Nicholas Waples +Nigel Kerr +Nigel Tao +Olivier Antoine +Padraig Kitterick +Paolo Giarrusso +Pascal S. de Kloe +Patrick Gavlin +Paul Borman +Petar Maymounkov +Peter Froehlich +Peter McKenzie +Peter Mundy +Péter Szabó +Peter Williams +Phil Pennock +Pieter Droogendijk +Quan Yong Zhai +Raif S. Naffah +Raph Levien +Risto Jaakko Saarelma +Rob Pike +Robert Griesemer +Robert Hencke +Roger Pau Monné +Roger Peppe +Ross Light +Russ Cox +Ryan Hitchman +Sam Thorogood +Sameer Ajmani +Scott Lawrence +Scott Schwartz +Sebastien Binet +Sergei Skorobogatov +Sergey 'SnakE' Gromov +Sergio Luis O. B. Correia +Spring Mc +Stefan Nilsson +Stephen Ma +Stephen Weinberg +Sugu Sougoumarane +Sven Almgren +Tarmigan Casebolt +Timo Savola +Tom Szymanski +Tor Andersson +Trevor Strohman +Vincent Ambo +Vincent Vanackere +Vinu Rajashekhar +Vish Subramanian +Volker Dobler +Wei Guangjing +William Chan +William Josephson +Yasuhiro Matsumoto +Yongjian Xu +Yuusei Kuwana +Yuval Pavel Zholkover +Yves Junqueira diff --git a/LICENSE b/LICENSE new file mode 100644 index 000000000..6a66aea5e --- /dev/null +++ b/LICENSE @@ -0,0 +1,27 @@ +Copyright (c) 2009 The Go Authors. All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are +met: + + * Redistributions of source code must retain the above copyright +notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above +copyright notice, this list of conditions and the following disclaimer +in the documentation and/or other materials provided with the +distribution. + * Neither the name of Google Inc. nor the names of its +contributors may be used to endorse or promote products derived from +this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. diff --git a/PATENTS b/PATENTS new file mode 100644 index 000000000..733099041 --- /dev/null +++ b/PATENTS @@ -0,0 +1,22 @@ +Additional IP Rights Grant (Patents) + +"This implementation" means the copyrightable works distributed by +Google as part of the Go project. + +Google hereby grants to You a perpetual, worldwide, non-exclusive, +no-charge, royalty-free, irrevocable (except as stated in this section) +patent license to make, have made, use, offer to sell, sell, import, +transfer and otherwise run, modify and propagate the contents of this +implementation of Go, where such license applies only to those patent +claims, both currently owned or controlled by Google and acquired in +the future, licensable by Google that are necessarily infringed by this +implementation of Go. This grant does not include claims that would be +infringed only as a consequence of further modification of this +implementation. If you or your agent or exclusive licensee institute or +order or agree to the institution of patent litigation against any +entity (including a cross-claim or counterclaim in a lawsuit) alleging +that this implementation of Go or any code incorporated within this +implementation of Go constitutes direct or contributory patent +infringement, or inducement of patent infringement, then any patent +rights granted to you under this License for this implementation of Go +shall terminate as of the date such litigation is filed. diff --git a/README b/README new file mode 100644 index 000000000..8bf9e7b8c --- /dev/null +++ b/README @@ -0,0 +1,31 @@ +This is the source code repository for the Go programming language. + +For documentation about how to install and use Go, +visit http://golang.org/ or load doc/install.html in your web browser. + +After installing Go, you can view a nicely formatted +doc/install.html by running godoc --http=:6060 +and then visiting http://localhost:6060/doc/install.html. + +Unless otherwise noted, the Go source files are distributed +under the BSD-style license found in the LICENSE file. + +-- + +Binary Distribution Notes + +If you have just untarred a binary Go distribution, you need to set +the environment variable $GOROOT to the full path of the go +directory (the one containing this README). You can omit the +variable if you unpack it into /usr/local/go, or if you rebuild +from sources by running all.bash (see doc/install.html). +You should also add the Go binary directory $GOROOT/bin +to your shell's path. + +For example, if you extracted the tar file into $HOME/go, you might +put the following in your .profile: + + export GOROOT=$HOME/go + export PATH=$PATH:$GOROOT/bin + +See doc/install.html for more details. diff --git a/VERSION b/VERSION new file mode 100644 index 000000000..0261a0348 --- /dev/null +++ b/VERSION @@ -0,0 +1 @@ +release.r60 9481 diff --git a/doc/ExpressivenessOfGo.pdf b/doc/ExpressivenessOfGo.pdf new file mode 100644 index 000000000..f1931d081 Binary files /dev/null and b/doc/ExpressivenessOfGo.pdf differ diff --git a/doc/GoCourseDay1.pdf b/doc/GoCourseDay1.pdf new file mode 100644 index 000000000..5a7749c18 Binary files /dev/null and b/doc/GoCourseDay1.pdf differ diff --git a/doc/GoCourseDay2.pdf b/doc/GoCourseDay2.pdf new file mode 100644 index 000000000..0d82ba4d3 Binary files /dev/null and b/doc/GoCourseDay2.pdf differ diff --git a/doc/GoCourseDay3.pdf b/doc/GoCourseDay3.pdf new file mode 100644 index 000000000..5a5463ba2 Binary files /dev/null and b/doc/GoCourseDay3.pdf differ diff --git a/doc/Makefile b/doc/Makefile new file mode 100644 index 000000000..e4e3f83b3 --- /dev/null +++ b/doc/Makefile @@ -0,0 +1,11 @@ +# Copyright 2009 The Go Authors. All rights reserved. +# Use of this source code is governed by a BSD-style +# license that can be found in the LICENSE file. + +include ../src/Make.inc + +TARG=tmpltohtml +GOFILES=\ + tmpltohtml.go\ + +include ../src/Make.cmd diff --git a/doc/all.css b/doc/all.css new file mode 100644 index 000000000..f8f8c653f --- /dev/null +++ b/doc/all.css @@ -0,0 +1,204 @@ +/* General Styles */ +body { + font-family: "Bitstream Vera Sans", Verdana, sans-serif; + font-size: 81.25%; + line-height: 1.23em; + padding: 0; + margin: 1.23em; + background: white; + color: black; +} +a { + color: #04a; + text-decoration: none; +} +a:visited { + color: #04a; +} +a:hover { + color: #a40; + text-decoration: underline; +} +a:active { + color: #c00; +} +code, pre { + font-size: 1.2em; +} +pre { + background: #F0F0F0; + padding: 0.5em 1em; +} + +/* Top bar */ +#container { + width: 100%; + margin: auto; +} +#topnav { + height: 55px; + background: url(/doc/logo.png) no-repeat top left; +} +a#logo-box { + display: block; + height: 55px; +} +h1#title { + display: none; +} +#nav-main { + float: right; + width: 500px; + margin-top: -5px; + text-align: center; +} +#nav-main ul { + padding-left: 0; + margin-left: 0; + margin-bottom: 0.5em; +} +#nav-main li a { + display: inline; + display: inline-block; + padding: .46em .62em .38em .62em; +} +#nav-main li a:link, +#nav-main li a:visited { + color: #000; +} +#nav-main li { + display: inline; + display: inline-block; + background: #e6e6e6 url(/doc/button_background.png) repeat-x; + border: solid 1px #999; + margin-left: -1px; + text-shadow: #fff 0 1px 0; + box-shadow: 0 1px 1px #ccc; + -moz-box-shadow: 0 1px 1px #ccc; + -webkit-box-shadow: 0 1px 1px #ccc; +} +#nav-main li:first-child { + -moz-border-top-left-radius: 4px; + border-top-left-radius: 4px; + -moz-border-bottom-left-radius: 4px; + border-bottom-left-radius: 4px; +} +#nav-main li:last-child { + -moz-border-top-right-radius: 4px; + border-top-right-radius: 4px; + -moz-border-bottom-right-radius: 4px; + border-bottom-right-radius: 4px; +} +#nav-main .quickref { + color: #444; +} +#nav-main .quickref .sep { + color: #999; +} +#search { + width: 120px; + margin-left: 0.5em; +} +#search.inactive { + text-align: center; + color: #444; +} + +/* Footer */ +#site-info { + position: relative; + text-align: center; +} +#site-info, #site-info a:link, #site-info a:visited { + color: #aaa; +} + +/* Content */ +#content { + clear: both; + padding: 0; + position: relative; + margin-top: 1.5em; + margin-bottom: 1.5em; + border-top: solid 1px #aaa; + border-bottom: solid 1px #aaa; +} +.left-column { + width: 49%; + float: left; +} +.right-column { + width: 49%; + float: right; +} +.end-columns { + clear: both; +} +#content h1 { + padding: 0; +} +#content h2 { + border-top: 2px solid #ddd; + padding: 8px 0; + margin: 1.5em 0 0; +} +#content .subtitle { + margin-top: 1em; + display: block; +} +.navtop a { + font-weight: normal; font-size: 7pt; + float: right; color: #999; +} + +/* Content and Code Highlighting */ +pre.ebnf, pre.grammar { + background: #FFFFE0; +} +span.ln { + font-size: 80%; + color: #777777; +} +span.comment { + color: #002090; +} +span.highlight { + background: #FF9900; + font-weight: bold; +} +span.highlight-comment { + background: #FF9900; + font-weight: bold; + color: #002090; +} +span.selection { + background: #FFFF00 +} +span.selection-comment { + color: #002090; + background: #FFFF00 +} +span.selection-highlight { + background: #FF9900; + font-weight: bold; +} +span.selection-highlight-comment { + background: #FF9900; + font-weight: bold; + color: #002090; +} +span.alert { + color: #D00000; +} +#nav table { + width: 100%; +} +.detail { + padding: 0.25em 1em; + background: #F4F4F4; +} +sup.new { + color: red; + font-size: 8px; + line-height: 0; +} diff --git a/doc/button_background.png b/doc/button_background.png new file mode 100644 index 000000000..86a3b3086 Binary files /dev/null and b/doc/button_background.png differ diff --git a/doc/code.html b/doc/code.html new file mode 100644 index 000000000..cdc60b071 --- /dev/null +++ b/doc/code.html @@ -0,0 +1,368 @@ + + +

Introduction

+ +

+This document explains how to write a new package +and how to test code. +It assumes you have installed Go using the +installation instructions. +

+ +

+Before embarking on a change to an existing +package or the creation of a new package, +be sure to send mail to the +mailing list +to let people know what you are thinking of doing. +Doing so helps avoid duplication of effort and +enables discussions about design before any code +has been written. +

+ +

Community resources

+ +

+For real-time help, there may be users or developers on +#go-nuts on the Freenode IRC server. +

+ +

+The official mailing list for discussion of the Go language is +Go Nuts. +

+ +

+Bugs can be reported using the Go issue tracker. +

+ +

+For those who wish to keep up with development, +there is another mailing list, golang-checkins, +that receives a message summarizing each checkin to the Go repository. +

+ + +

Creating a new package

+ +

+The source code for the package with import path +x/y is, by convention, kept in the +directory $GOROOT/src/pkg/x/y. +

+ +

Makefile

+ +

+It would be nice to have Go-specific tools that +inspect the source files to determine what to build and in +what order, but for now, Go uses GNU make. +Thus, the first file to create in a new package directory is +usually the Makefile. +The basic form used in the Go source tree +is illustrated by src/pkg/container/vector/Makefile: +

+ +
+include ../../../Make.inc
+
+TARG=container/vector
+GOFILES=\
+	intvector.go\
+	stringvector.go\
+	vector.go\
+
+include ../../../Make.pkg
+
+ +

+Outside the Go source tree (for personal packages), the standard form is +

+ +
+include $(GOROOT)/src/Make.inc
+
+TARG=mypackage
+GOFILES=\
+	my1.go\
+	my2.go\
+
+include $(GOROOT)/src/Make.pkg
+
+ +

+The first and last lines include standard definitions and rules. +Packages maintained in the standard Go tree use a relative path (instead of +$(GOROOT)/src) so that make will work correctly +even if $(GOROOT) contains spaces. +This makes it easy for programmers to try Go. +

+ +

+If you have not set $GOROOT in your environment, +you must run gomake to use this form of makefile. +Gomake also takes care to invoke GNU Make +even on systems where it is installed as gmake +rather than make. +

+ +

+TARG is the target install path for the package, +the string that clients will use to import it. +Inside the Go tree, this string should be the same as the directory +in which the Makefile appears, with the +$GOROOT/src/pkg/ prefix removed. +Outside the Go tree, you can use any TARG you +want that doesn't conflict with the standard Go package names. +A common convention is to use an identifying top-level name +to group your packages: myname/tree, myname/filter, etc. +Note that even if you keep your package source outside the +Go tree, running make install installs your +package binaries in the standard location—$GOROOT/pkg—to +make it easy to find them. +

+ +

+GOFILES is a list of source files to compile to +create the package. The trailing \ characters +allow the list to be split onto multiple lines +for easy sorting. +

+ +

+If you create a new package directory in the Go tree, add it to the list in +$GOROOT/src/pkg/Makefile so that it +is included in the standard build. Then run: +

+cd $GOROOT/src/pkg
+./deps.bash
+
+

+to update the dependency file Make.deps. +(This happens automatically each time you run all.bash +or make.bash.) +

+ +

+If you change the imports of an existing package, +you do not need to edit $GOROOT/src/pkg/Makefile +but you will still need to run deps.bash as above. +

+ + +

Go source files

+ +

+The first statement in each of the source files listed in the Makefile +should be package name, where name +is the package's default name for imports. +(All files in a package must use the same name.) +Go's convention is that the package name is the last element of the +import path: the package imported as "crypto/rot13" +should be named rot13. +There is no requirement that package names be unique +across all packages linked into a single binary, +only that the import paths (their full file names) be unique. +

+ +

+Go compiles all the source files in a package at once, so one file +can refer to constants, variables, types, and functions in another +file without special arrangement or declarations. +

+ +

+Writing clean, idiomatic Go code is beyond the scope of this document. +Effective Go is an introduction to +that topic. +

+ +

Building programs

+

To build a Go program with gomake, create a Makefile alongside your program's +source files. It should be similar to the example above, but include +Make.cmd instead of Make.pkg: + +

+include $(GOROOT)/src/Make.inc
+
+TARG=helloworld
+GOFILES=\
+	helloworld.go\
+
+include $(GOROOT)/src/Make.cmd
+
+ +

Running gomake will compile helloworld.go +and produce an executable named helloworld in the current +directory. +

+ +

+Running gomake install will build helloworld if +necessary and copy it to the $GOBIN directory +($GOROOT/bin/ is the default). +

+ +

Testing

+ +

+Go has a lightweight test framework known as gotest. +You write a test by creating a file with a name ending in _test.go +that contains functions named TestXXX with signature func (t *testing.T). +The test framework runs each such function; +if the function calls a failure function such as t.Error or t.Fail, the test is considered to have failed. +The gotest command documentation +and the testing package documentation give more detail. +

+ +

+The *_test.go files should not be listed in the Makefile. +

+ +

+To run the test, run either make test or gotest +(they are equivalent). +To run only the tests in a single test file, for instance one_test.go, +run gotest one_test.go. +

+ +

+If your change affects performance, add a Benchmark function +(see the gotest command documentation) +and run it using gotest -test.bench=.. +

+ +

+Once your new code is tested and working, +it's time to get it reviewed and submitted. +

+ +

An example package with tests

+ +

+This example package, numbers, consists of the function +Double, which takes an int and returns that value +multiplied by 2. It consists of three files. +

+ +

+First, the package implementation, numbers.go: +

+ +
+package numbers
+
+func Double(i int) int {
+	return i * 2
+}
+
+ +

+Next, the tests, numbers_test.go: +

+ +
+package numbers
+
+import (
+	"testing"
+)
+
+type doubleTest struct {
+	in, out int
+}
+
+var doubleTests = []doubleTest{
+	doubleTest{1, 2},
+	doubleTest{2, 4},
+	doubleTest{-5, -10},
+}
+
+func TestDouble(t *testing.T) {
+	for _, dt := range doubleTests {
+		v := Double(dt.in)
+		if v != dt.out {
+			t.Errorf("Double(%d) = %d, want %d.", dt.in, v, dt.out)
+		}
+	}
+}
+
+ +

+Finally, the Makefile: +

+ +
+include $(GOROOT)/src/Make.inc
+
+TARG=numbers
+GOFILES=\
+	numbers.go\
+
+include $(GOROOT)/src/Make.pkg
+
+ +

+Running gomake install will build and install the package to +the $GOROOT/pkg/ directory (it can then be used by any +program on the system). +

+ +

+Running gomake test (or just running the command +gotest) will rebuild the package, including the +numbers_test.go file, and then run the TestDouble +function. The output "PASS" indicates that all tests passed +successfully. Breaking the implementation by changing the multiplier from +2 to 3 will allow you to see how failing tests are +reported. +

+ +

+See the gotest documentation and the +testing package for more detail. +

+ +

Architecture- and operating system-specific code

+ +

First, a disclaimer: very few Go packages should need to know about the +hardware and operating system they run on. In the vast majority of cases the +language and standard library handle most portability issues. This section is +a guide for experienced systems programmers who have a good reason to write +platform-specific code, such as assembly-language support for fast +trigonometric functions or code that implements a common interface above +different operating systems.

+ +

To compile such code, use the $GOOS and $GOARCH +environment variables in your +source file names and Makefile.

+ +

For example, this Makefile describes a package that builds on +different operating systems by parameterizing the file name with +$GOOS.

+ +
+include $(GOROOT)/src/Make.inc
+
+TARG=mypackage
+GOFILES=\
+	my.go\
+	my_$(GOOS).go\
+
+include $(GOROOT)/src/Make.pkg
+
+ +

The OS-specific code goes in my_linux.go, +my_darwin.go, and so on.

+ +

If you follow these conventional parameterizations, tools such as +goinstall will work seamlessly with your package: +

+ +
+my_$(GOOS).go
+my_$(GOARCH).go
+my_$(GOOS)_$(GOARCH).go
+
+ +

The same holds for .s (assembly) files.

diff --git a/doc/codelab/wiki/Makefile b/doc/codelab/wiki/Makefile new file mode 100644 index 000000000..09c3291a0 --- /dev/null +++ b/doc/codelab/wiki/Makefile @@ -0,0 +1,25 @@ +# Copyright 2010 The Go Authors. All rights reserved. +# Use of this source code is governed by a BSD-style +# license that can be found in the LICENSE file. + +include ../../../src/Make.inc + +all: index.html + +include ../../../src/Make.common + +CLEANFILES+=index.html srcextract.bin htmlify.bin get.bin + +index.html: srcextract.bin htmlify.bin + PATH=.:$$PATH awk '/^!/{system(substr($$0,2)); next} {print}' < wiki.html | tr -d '\r' > index.html + +test: get.bin + bash ./test.sh + rm -f get.6 get.bin + +%.bin: %.$O + $(LD) -o $@ $< + +%.$O: %.go + $(GC) $*.go + diff --git a/doc/codelab/wiki/edit.html b/doc/codelab/wiki/edit.html new file mode 100644 index 000000000..c14953b17 --- /dev/null +++ b/doc/codelab/wiki/edit.html @@ -0,0 +1,6 @@ +

Editing {{.Title |html}}

+ +
+
+
+
diff --git a/doc/codelab/wiki/final-noclosure.go b/doc/codelab/wiki/final-noclosure.go new file mode 100644 index 000000000..067f502c6 --- /dev/null +++ b/doc/codelab/wiki/final-noclosure.go @@ -0,0 +1,100 @@ +package main + +import ( + "http" + "io/ioutil" + "os" + "regexp" + "template" +) + +type Page struct { + Title string + Body []byte +} + +func (p *Page) save() os.Error { + filename := p.Title + ".txt" + return ioutil.WriteFile(filename, p.Body, 0600) +} + +func loadPage(title string) (*Page, os.Error) { + filename := title + ".txt" + body, err := ioutil.ReadFile(filename) + if err != nil { + return nil, err + } + return &Page{Title: title, Body: body}, nil +} + +func viewHandler(w http.ResponseWriter, r *http.Request) { + title, err := getTitle(w, r) + if err != nil { + return + } + p, err := loadPage(title) + if err != nil { + http.Redirect(w, r, "/edit/"+title, http.StatusFound) + return + } + renderTemplate(w, "view", p) +} + +func editHandler(w http.ResponseWriter, r *http.Request) { + title, err := getTitle(w, r) + if err != nil { + return + } + p, err := loadPage(title) + if err != nil { + p = &Page{Title: title} + } + renderTemplate(w, "edit", p) +} + +func saveHandler(w http.ResponseWriter, r *http.Request) { + title, err := getTitle(w, r) + if err != nil { + return + } + body := r.FormValue("body") + p := &Page{Title: title, Body: []byte(body)} + err = p.save() + if err != nil { + http.Error(w, err.String(), http.StatusInternalServerError) + return + } + http.Redirect(w, r, "/view/"+title, http.StatusFound) +} + +func renderTemplate(w http.ResponseWriter, tmpl string, p *Page) { + t, err := template.ParseFile(tmpl+".html") + if err != nil { + http.Error(w, err.String(), http.StatusInternalServerError) + return + } + err = t.Execute(w, p) + if err != nil { + http.Error(w, err.String(), http.StatusInternalServerError) + } +} + +const lenPath = len("/view/") + +var titleValidator = regexp.MustCompile("^[a-zA-Z0-9]+$") + +func getTitle(w http.ResponseWriter, r *http.Request) (title string, err os.Error) { + title = r.URL.Path[lenPath:] + if !titleValidator.MatchString(title) { + http.NotFound(w, r) + err = os.NewError("Invalid Page Title") + } + return +} + +func main() { + http.HandleFunc("/view/", viewHandler) + http.HandleFunc("/edit/", editHandler) + http.HandleFunc("/save/", saveHandler) + http.ListenAndServe(":8080", nil) +} diff --git a/doc/codelab/wiki/final-noerror.go b/doc/codelab/wiki/final-noerror.go new file mode 100644 index 000000000..b8edbee9b --- /dev/null +++ b/doc/codelab/wiki/final-noerror.go @@ -0,0 +1,52 @@ +package main + +import ( + "http" + "io/ioutil" + "os" + "template" +) + +type Page struct { + Title string + Body []byte +} + +func (p *Page) save() os.Error { + filename := p.Title + ".txt" + return ioutil.WriteFile(filename, p.Body, 0600) +} + +func loadPage(title string) (*Page, os.Error) { + filename := title + ".txt" + body, err := ioutil.ReadFile(filename) + if err != nil { + return nil, err + } + return &Page{Title: title, Body: body}, nil +} + +const lenPath = len("/view/") + +func editHandler(w http.ResponseWriter, r *http.Request) { + title := r.URL.Path[lenPath:] + p, err := loadPage(title) + if err != nil { + p = &Page{Title: title} + } + t, _ := template.ParseFile("edit.html") + t.Execute(w, p) +} + +func viewHandler(w http.ResponseWriter, r *http.Request) { + title := r.URL.Path[lenPath:] + p, _ := loadPage(title) + t, _ := template.ParseFile("view.html") + t.Execute(w, p) +} + +func main() { + http.HandleFunc("/view/", viewHandler) + http.HandleFunc("/edit/", editHandler) + http.ListenAndServe(":8080", nil) +} diff --git a/doc/codelab/wiki/final-parsetemplate.go b/doc/codelab/wiki/final-parsetemplate.go new file mode 100644 index 000000000..f25012eed --- /dev/null +++ b/doc/codelab/wiki/final-parsetemplate.go @@ -0,0 +1,90 @@ +package main + +import ( + "http" + "io/ioutil" + "os" + "regexp" + "template" +) + +type Page struct { + Title string + Body []byte +} + +func (p *Page) save() os.Error { + filename := p.Title + ".txt" + return ioutil.WriteFile(filename, p.Body, 0600) +} + +func loadPage(title string) (*Page, os.Error) { + filename := title + ".txt" + body, err := ioutil.ReadFile(filename) + if err != nil { + return nil, err + } + return &Page{Title: title, Body: body}, nil +} + +func viewHandler(w http.ResponseWriter, r *http.Request, title string) { + p, err := loadPage(title) + if err != nil { + http.Redirect(w, r, "/edit/"+title, http.StatusFound) + return + } + renderTemplate(w, "view", p) +} + +func editHandler(w http.ResponseWriter, r *http.Request, title string) { + p, err := loadPage(title) + if err != nil { + p = &Page{Title: title} + } + renderTemplate(w, "edit", p) +} + +func saveHandler(w http.ResponseWriter, r *http.Request, title string) { + body := r.FormValue("body") + p := &Page{Title: title, Body: []byte(body)} + err := p.save() + if err != nil { + http.Error(w, err.String(), http.StatusInternalServerError) + return + } + http.Redirect(w, r, "/view/"+title, http.StatusFound) +} + +func renderTemplate(w http.ResponseWriter, tmpl string, p *Page) { + t, err := template.ParseFile(tmpl+".html", nil) + if err != nil { + http.Error(w, err.String(), http.StatusInternalServerError) + return + } + err = t.Execute(w, p) + if err != nil { + http.Error(w, err.String(), http.StatusInternalServerError) + } +} + +const lenPath = len("/view/") + +var titleValidator = regexp.MustCompile("^[a-zA-Z0-9]+$") + +func makeHandler(fn func(http.ResponseWriter, *http.Request, string)) http.HandlerFunc { + return func(w http.ResponseWriter, r *http.Request) { + title := r.URL.Path[lenPath:] + if !titleValidator.MatchString(title) { + http.NotFound(w, r) + return + } + fn(w, r, title) + } +} + +func main() { + http.HandleFunc("/view/", makeHandler(viewHandler)) + http.HandleFunc("/edit/", makeHandler(editHandler)) + http.HandleFunc("/save/", makeHandler(saveHandler)) + http.ListenAndServe(":8080", nil) +} diff --git a/doc/codelab/wiki/final-template.go b/doc/codelab/wiki/final-template.go new file mode 100644 index 000000000..aab536ee1 --- /dev/null +++ b/doc/codelab/wiki/final-template.go @@ -0,0 +1,64 @@ +package main + +import ( + "http" + "io/ioutil" + "os" + "template" +) + +type Page struct { + Title string + Body []byte +} + +func (p *Page) save() os.Error { + filename := p.Title + ".txt" + return ioutil.WriteFile(filename, p.Body, 0600) +} + +func loadPage(title string) (*Page, os.Error) { + filename := title + ".txt" + body, err := ioutil.ReadFile(filename) + if err != nil { + return nil, err + } + return &Page{Title: title, Body: body}, nil +} + +const lenPath = len("/view/") + +func editHandler(w http.ResponseWriter, r *http.Request) { + title := r.URL.Path[lenPath:] + p, err := loadPage(title) + if err != nil { + p = &Page{Title: title} + } + renderTemplate(w, "edit", p) +} + +func viewHandler(w http.ResponseWriter, r *http.Request) { + title := r.URL.Path[lenPath:] + p, _ := loadPage(title) + renderTemplate(w, "view", p) +} + +func saveHandler(w http.ResponseWriter, r *http.Request) { + title := r.URL.Path[lenPath:] + body := r.FormValue("body") + p := &Page{Title: title, Body: []byte(body)} + p.save() + http.Redirect(w, r, "/view/"+title, http.StatusFound) +} + +func renderTemplate(w http.ResponseWriter, tmpl string, p *Page) { + t, _ := template.ParseFile(tmpl+".html", nil) + t.Execute(w, p) +} + +func main() { + http.HandleFunc("/view/", viewHandler) + http.HandleFunc("/edit/", editHandler) + http.HandleFunc("/save/", saveHandler) + http.ListenAndServe(":8080", nil) +} diff --git a/doc/codelab/wiki/final.go b/doc/codelab/wiki/final.go new file mode 100644 index 000000000..47a4c3473 --- /dev/null +++ b/doc/codelab/wiki/final.go @@ -0,0 +1,94 @@ +package main + +import ( + "http" + "io/ioutil" + "os" + "regexp" + "template" +) + +type Page struct { + Title string + Body []byte +} + +func (p *Page) save() os.Error { + filename := p.Title + ".txt" + return ioutil.WriteFile(filename, p.Body, 0600) +} + +func loadPage(title string) (*Page, os.Error) { + filename := title + ".txt" + body, err := ioutil.ReadFile(filename) + if err != nil { + return nil, err + } + return &Page{Title: title, Body: body}, nil +} + +func viewHandler(w http.ResponseWriter, r *http.Request, title string) { + p, err := loadPage(title) + if err != nil { + http.Redirect(w, r, "/edit/"+title, http.StatusFound) + return + } + renderTemplate(w, "view", p) +} + +func editHandler(w http.ResponseWriter, r *http.Request, title string) { + p, err := loadPage(title) + if err != nil { + p = &Page{Title: title} + } + renderTemplate(w, "edit", p) +} + +func saveHandler(w http.ResponseWriter, r *http.Request, title string) { + body := r.FormValue("body") + p := &Page{Title: title, Body: []byte(body)} + err := p.save() + if err != nil { + http.Error(w, err.String(), http.StatusInternalServerError) + return + } + http.Redirect(w, r, "/view/"+title, http.StatusFound) +} + +var templates = make(map[string]*template.Template) + +func init() { + for _, tmpl := range []string{"edit", "view"} { + t := template.Must(template.ParseFile(tmpl+".html")) + templates[tmpl] = t + } +} + +func renderTemplate(w http.ResponseWriter, tmpl string, p *Page) { + err := templates[tmpl].Execute(w, p) + if err != nil { + http.Error(w, err.String(), http.StatusInternalServerError) + } +} + +const lenPath = len("/view/") + +var titleValidator = regexp.MustCompile("^[a-zA-Z0-9]+$") + +func makeHandler(fn func(http.ResponseWriter, *http.Request, string)) http.HandlerFunc { + return func(w http.ResponseWriter, r *http.Request) { + title := r.URL.Path[lenPath:] + if !titleValidator.MatchString(title) { + http.NotFound(w, r) + return + } + fn(w, r, title) + } +} + +func main() { + http.HandleFunc("/view/", makeHandler(viewHandler)) + http.HandleFunc("/edit/", makeHandler(editHandler)) + http.HandleFunc("/save/", makeHandler(saveHandler)) + http.ListenAndServe(":8080", nil) +} diff --git a/doc/codelab/wiki/get.go b/doc/codelab/wiki/get.go new file mode 100644 index 000000000..c36684e3e --- /dev/null +++ b/doc/codelab/wiki/get.go @@ -0,0 +1,50 @@ +package main + +import ( + "http" + "flag" + "fmt" + "io" + "log" + "net" + "os" + "strings" +) + +var ( + post = flag.String("post", "", "urlencoded form data to POST") + addr = flag.Bool("addr", false, "find open address and print to stdout") +) + +func main() { + flag.Parse() + if *addr { + l, err := net.Listen("tcp", "127.0.0.1:0") + if err != nil { + log.Fatal(err) + } + defer l.Close() + fmt.Print(l.Addr()) + return + } + url := flag.Arg(0) + if url == "" { + log.Fatal("no url supplied") + } + var r *http.Response + var err os.Error + if *post != "" { + b := strings.NewReader(*post) + r, err = http.Post(url, "application/x-www-form-urlencoded", b) + } else { + r, err = http.Get(url) + } + if err != nil { + log.Fatal(err) + } + defer r.Body.Close() + _, err = io.Copy(os.Stdout, r.Body) + if err != nil { + log.Fatal(err) + } +} diff --git a/doc/codelab/wiki/htmlify.go b/doc/codelab/wiki/htmlify.go new file mode 100644 index 000000000..9e7605b92 --- /dev/null +++ b/doc/codelab/wiki/htmlify.go @@ -0,0 +1,12 @@ +package main + +import ( + "template" + "os" + "io/ioutil" +) + +func main() { + b, _ := ioutil.ReadAll(os.Stdin) + template.HTMLEscape(os.Stdout, b) +} diff --git a/doc/codelab/wiki/http-sample.go b/doc/codelab/wiki/http-sample.go new file mode 100644 index 000000000..33379a1b6 --- /dev/null +++ b/doc/codelab/wiki/http-sample.go @@ -0,0 +1,15 @@ +package main + +import ( + "fmt" + "http" +) + +func handler(w http.ResponseWriter, r *http.Request) { + fmt.Fprintf(w, "Hi there, I love %s!", r.URL.Path[1:]) +} + +func main() { + http.HandleFunc("/", handler) + http.ListenAndServe(":8080", nil) +} diff --git a/doc/codelab/wiki/index.html b/doc/codelab/wiki/index.html new file mode 100644 index 000000000..50e9db5e9 --- /dev/null +++ b/doc/codelab/wiki/index.html @@ -0,0 +1,1007 @@ + +

Introduction

+ +

+Covered in this codelab: +

+
    +
  • Creating a data structure with load and save methods
  • +
  • Using the http package to build web applications +
  • Using the template package to process HTML templates
  • +
  • Using the regexp package to validate user input
  • +
  • Using closures
  • +
+ +

+Assumed knowledge: +

+
    +
  • Programming experience
  • +
  • Understanding of basic web technologies (HTTP, HTML)
  • +
  • Some UNIX command-line knowledge
  • +
+ +

Getting Started

+ +

+At present, you need to have a Linux, OS X, or FreeBSD machine to run Go. If +you don't have access to one, you could set up a Linux Virtual Machine (using +VirtualBox or similar) or a +Virtual +Private Server. +

+ +

+Install Go (see the Installation Instructions). +

+ +

+Make a new directory for this codelab and cd to it: +

+ +
+$ mkdir ~/gowiki
+$ cd ~/gowiki
+
+ +

+Create a file named wiki.go, open it in your favorite editor, and +add the following lines: +

+ +
+package main
+
+import (
+	"fmt"
+	"io/ioutil"
+	"os"
+)
+
+ +

+We import the fmt, ioutil and os +packages from the Go standard library. Later, as we implement additional +functionality, we will add more packages to this import +declaration. +

+ +

Data Structures

+ +

+Let's start by defining the data structures. A wiki consists of a series of +interconnected pages, each of which has a title and a body (the page content). +Here, we define Page as a struct with two fields representing +the title and body. +

+ +
+type Page struct {
+	Title	string
+	Body	[]byte
+}
+
+ +

+The type []byte means "a byte slice". +(See Effective Go +for more on slices.) +The Body element is a []byte rather than +string because that is the type expected by the io +libraries we will use, as you'll see below. +

+ +

+The Page struct describes how page data will be stored in memory. +But what about persistent storage? We can address that by creating a +save method on Page: +

+ +
+func (p *Page) save() os.Error {
+	filename := p.Title + ".txt"
+	return ioutil.WriteFile(filename, p.Body, 0600)
+}
+
+ +

+This method's signature reads: "This is a method named save that +takes as its receiver p, a pointer to Page . It takes +no parameters, and returns a value of type os.Error." +

+ +

+This method will save the Page's Body to a text +file. For simplicity, we will use the Title as the file name. +

+ +

+The save method returns an os.Error value because +that is the return type of WriteFile (a standard library function +that writes a byte slice to a file). The save method returns the +error value, to let the application handle it should anything go wrong while +writing the file. If all goes well, Page.save() will return +nil (the zero-value for pointers, interfaces, and some other +types). +

+ +

+The octal integer constant 0600, passed as the third parameter to +WriteFile, indicates that the file should be created with +read-write permissions for the current user only. (See the Unix man page +open(2) for details.) +

+ +

+We will want to load pages, too: +

+ +
+func loadPage(title string) *Page {
+	filename := title + ".txt"
+	body, _ := ioutil.ReadFile(filename)
+	return &Page{Title: title, Body: body}
+}
+
+ +

+The function loadPage constructs the file name from +Title, reads the file's contents into a new +Page, and returns a pointer to that new page. +

+ +

+Functions can return multiple values. The standard library function +io.ReadFile returns []byte and os.Error. +In loadPage, error isn't being handled yet; the "blank identifier" +represented by the underscore (_) symbol is used to throw away the +error return value (in essence, assigning the value to nothing). +

+ +

+But what happens if ReadFile encounters an error? For example, +the file might not exist. We should not ignore such errors. Let's modify the +function to return *Page and os.Error. +

+ +
+func loadPage(title string) (*Page, os.Error) {
+	filename := title + ".txt"
+	body, err := ioutil.ReadFile(filename)
+	if err != nil {
+		return nil, err
+	}
+	return &Page{Title: title, Body: body}, nil
+}
+
+ +

+Callers of this function can now check the second parameter; if it is +nil then it has successfully loaded a Page. If not, it will be an +os.Error that can be handled by the caller (see the os package documentation for +details). +

+ +

+At this point we have a simple data structure and the ability to save to and +load from a file. Let's write a main function to test what we've +written: +

+ +
+func main() {
+	p1 := &Page{Title: "TestPage", Body: []byte("This is a sample Page.")}
+	p1.save()
+	p2, _ := loadPage("TestPage")
+	fmt.Println(string(p2.Body))
+}
+
+ +

+After compiling and executing this code, a file named TestPage.txt +would be created, containing the contents of p1. The file would +then be read into the struct p2, and its Body element +printed to the screen. +

+ +

+You can compile and run the program like this: +

+ +
+$ 8g wiki.go
+$ 8l wiki.8
+$ ./8.out
+This is a sample page.
+
+ +

+(The 8g and 8l commands are applicable to +GOARCH=386. If you're on an amd64 system, +substitute 6's for the 8's.) +

+ +

+Click here to view the code we've written so far. +

+ +

Introducing the http package (an interlude)

+ +

+Here's a full working example of a simple web server: +

+ +
+package main
+
+import (
+	"fmt"
+	"http"
+)
+
+func handler(w http.ResponseWriter, r *http.Request) {
+	fmt.Fprintf(w, "Hi there, I love %s!", r.URL.Path[1:])
+}
+
+func main() {
+	http.HandleFunc("/", handler)
+	http.ListenAndServe(":8080", nil)
+}
+
+ +

+The main function begins with a call to +http.HandleFunc, which tells the http package to +handle all requests to the web root ("/") with +handler. +

+ +

+It then calls http.ListenAndServe, specifying that it should +listen on port 8080 on any interface (":8080"). (Don't +worry about its second parameter, nil, for now.) +This function will block until the program is terminated. +

+ +

+The function handler is of the type http.HandlerFunc. +It takes an http.ResponseWriter and an http.Request as +its arguments. +

+ +

+An http.ResponseWriter value assembles the HTTP server's response; by writing +to it, we send data to the HTTP client. +

+ +

+An http.Request is a data structure that represents the client +HTTP request. The string r.URL.Path is the path component +of the request URL. The trailing [1:] means +"create a sub-slice of Path from the 1st character to the end." +This drops the leading "/" from the path name. +

+ +

+If you run this program and access the URL: +

+
http://localhost:8080/monkeys
+

+the program would present a page containing: +

+
Hi there, I love monkeys!
+ +

Using http to serve wiki pages

+ +

+To use the http package, it must be imported: +

+ +
+import (
+	"fmt"
+	"http"
+	"io/ioutil"
+	"os"
+)
+
+ +

+Let's create a handler to view a wiki page: +

+ +
+const lenPath = len("/view/")
+
+func viewHandler(w http.ResponseWriter, r *http.Request) {
+	title := r.URL.Path[lenPath:]
+	p, _ := loadPage(title)
+	fmt.Fprintf(w, "<h1>%s</h1><div>%s</div>", p.Title, p.Body)
+}
+
+ +

+First, this function extracts the page title from r.URL.Path, +the path component of the request URL. The global constant +lenPath is the length of the leading "/view/" +component of the request path. +The Path is re-sliced with [lenPath:] to drop the +first 6 characters of the string. This is because the path will invariably +begin with "/view/", which is not part of the page title. +

+ +

+The function then loads the page data, formats the page with a string of simple +HTML, and writes it to w, the http.ResponseWriter. +

+ +

+Again, note the use of _ to ignore the os.Error +return value from loadPage. This is done here for simplicity +and generally considered bad practice. We will attend to this later. +

+ +

+To use this handler, we create a main function that +initializes http using the viewHandler to handle +any requests under the path /view/. +

+ +
+func main() {
+	http.HandleFunc("/view/", viewHandler)
+	http.ListenAndServe(":8080", nil)
+}
+
+ +

+Click here to view the code we've written so far. +

+ +

+Let's create some page data (as test.txt), compile our code, and +try serving a wiki page: +

+ +
+$ echo "Hello world" > test.txt
+$ 8g wiki.go
+$ 8l wiki.8
+$ ./8.out
+
+ +

+With this web server running, a visit to http://localhost:8080/view/test +should show a page titled "test" containing the words "Hello world". +

+ +

Editing Pages

+ +

+A wiki is not a wiki without the ability to edit pages. Let's create two new +handlers: one named editHandler to display an 'edit page' form, +and the other named saveHandler to save the data entered via the +form. +

+ +

+First, we add them to main(): +

+ +
+func main() {
+	http.HandleFunc("/view/", viewHandler)
+	http.HandleFunc("/edit/", editHandler)
+	http.HandleFunc("/save/", saveHandler)
+	http.ListenAndServe(":8080", nil)
+}
+
+ +

+The function editHandler loads the page +(or, if it doesn't exist, create an empty Page struct), +and displays an HTML form. +

+ +
+func editHandler(w http.ResponseWriter, r *http.Request) {
+	title := r.URL.Path[lenPath:]
+	p, err := loadPage(title)
+	if err != nil {
+		p = &Page{Title: title}
+	}
+	fmt.Fprintf(w, "<h1>Editing %s</h1>"+
+		"<form action=\"/save/%s\" method=\"POST\">"+
+		"<textarea name=\"body\">%s</textarea><br>"+
+		"<input type=\"submit\" value=\"Save\">"+
+		"</form>",
+		p.Title, p.Title, p.Body)
+}
+
+ +

+This function will work fine, but all that hard-coded HTML is ugly. +Of course, there is a better way. +

+ +

The template package

+ +

+The template package is part of the Go standard library. +(A new template package is coming; this code lab will be updated soon.) +We can +use template to keep the HTML in a separate file, allowing +us to change the layout of our edit page without modifying the underlying Go +code. +

+ +

+First, we must add template to the list of imports: +

+ +
+import (
+	"http"
+	"io/ioutil"
+	"os"
+	"template"
+)
+
+ +

+Let's create a template file containing the HTML form. +Open a new file named edit.html, and add the following lines: +

+ +
+<h1>Editing {{.Title |html}}</h1>
+
+<form action="/save/{{.Title |html}}" method="POST">
+<div><textarea name="body" rows="20" cols="80">{{printf "%s" .Body |html}}</textarea></div>
+<div><input type="submit" value="Save"></div>
+</form>
+
+ +

+Modify editHandler to use the template, instead of the hard-coded +HTML: +

+ +
+func editHandler(w http.ResponseWriter, r *http.Request) {
+	title := r.URL.Path[lenPath:]
+	p, err := loadPage(title)
+	if err != nil {
+		p = &Page{Title: title}
+	}
+	t, _ := template.ParseFile("edit.html")
+	t.Execute(w, p)
+}
+
+ +

+The function template.ParseFile will read the contents of +edit.html and return a *template.Template. +

+ +

+The method t.Execute executes the template, writing the +generated HTML to the http.ResponseWriter. +The .Title and .Body dotted identifiers refer to +p.Title and p.Body. +

+ +

+Template directives are enclosed in double curly braces. +The printf "%s" .Body instruction is a function call +that outputs .Body as a string instead of a stream of bytes, +the same as a call to fmt.Printf. +The |html part of each directive pipes the value through the +html formatter before outputting it, which escapes HTML +characters (such as replacing > with &gt;), +preventing user data from corrupting the form HTML. +

+ +

+Now that we've removed the fmt.Fprintf statement, we can remove +"fmt" from the import list. +

+ +

+While we're working with templates, let's create a template for our +viewHandler called view.html: +

+ +
+<h1>{{.Title |html}}</h1>
+
+<p>[<a href="/edit/{{.Title |html}}">edit</a>]</p>
+
+<div>{{printf "%s" .Body |html}}</div>
+
+ +

+Modify viewHandler accordingly: +

+ +
+func viewHandler(w http.ResponseWriter, r *http.Request) {
+	title := r.URL.Path[lenPath:]
+	p, _ := loadPage(title)
+	t, _ := template.ParseFile("view.html")
+	t.Execute(w, p)
+}
+
+ +

+Notice that we've used almost exactly the same templating code in both +handlers. Let's remove this duplication by moving the templating code +to its own function: +

+ +
+func viewHandler(w http.ResponseWriter, r *http.Request) {
+	title := r.URL.Path[lenPath:]
+	p, _ := loadPage(title)
+	renderTemplate(w, "view", p)
+}
+
+func editHandler(w http.ResponseWriter, r *http.Request) {
+	title := r.URL.Path[lenPath:]
+	p, err := loadPage(title)
+	if err != nil {
+		p = &Page{Title: title}
+	}
+	renderTemplate(w, "edit", p)
+}
+
+func renderTemplate(w http.ResponseWriter, tmpl string, p *Page) {
+	t, _ := template.ParseFile(tmpl+".html", nil)
+	t.Execute(w, p)
+}
+
+ +

+The handlers are now shorter and simpler. +

+ +

Handling non-existent pages

+ +

+What if you visit /view/APageThatDoesntExist? The program will +crash. This is because it ignores the error return value from +loadPage. Instead, if the requested Page doesn't exist, it should +redirect the client to the edit Page so the content may be created: +

+ +
+func viewHandler(w http.ResponseWriter, r *http.Request) {
+	title, err := getTitle(w, r)
+	if err != nil {
+		return
+	}
+	p, err := loadPage(title)
+	if err != nil {
+		http.Redirect(w, r, "/edit/"+title, http.StatusFound)
+		return
+	}
+	renderTemplate(w, "view", p)
+}
+
+ +

+The http.Redirect function adds an HTTP status code of +http.StatusFound (302) and a Location +header to the HTTP response. +

+ +

Saving Pages

+ +

+The function saveHandler will handle the form submission. +

+ +
+func saveHandler(w http.ResponseWriter, r *http.Request) {
+	title := r.URL.Path[lenPath:]
+	body := r.FormValue("body")
+	p := &Page{Title: title, Body: []byte(body)}
+	p.save()
+	http.Redirect(w, r, "/view/"+title, http.StatusFound)
+}
+
+ +

+The page title (provided in the URL) and the form's only field, +Body, are stored in a new Page. +The save() method is then called to write the data to a file, +and the client is redirected to the /view/ page. +

+ +

+The value returned by FormValue is of type string. +We must convert that value to []byte before it will fit into +the Page struct. We use []byte(body) to perform +the conversion. +

+ +

Error handling

+ +

+There are several places in our program where errors are being ignored. This +is bad practice, not least because when an error does occur the program will +crash. A better solution is to handle the errors and return an error message +to the user. That way if something does go wrong, the server will continue to +function and the user will be notified. +

+ +

+First, let's handle the errors in renderTemplate: +

+ +
+func renderTemplate(w http.ResponseWriter, tmpl string, p *Page) {
+	t, err := template.ParseFile(tmpl+".html", nil)
+	if err != nil {
+		http.Error(w, err.String(), http.StatusInternalServerError)
+		return
+	}
+	err = t.Execute(w, p)
+	if err != nil {
+		http.Error(w, err.String(), http.StatusInternalServerError)
+	}
+}
+
+ +

+The http.Error function sends a specified HTTP response code +(in this case "Internal Server Error") and error message. +Already the decision to put this in a separate function is paying off. +

+ +

+Now let's fix up saveHandler: +

+ +
+func saveHandler(w http.ResponseWriter, r *http.Request) {
+	title, err := getTitle(w, r)
+	if err != nil {
+		return
+	}
+	body := r.FormValue("body")
+	p := &Page{Title: title, Body: []byte(body)}
+	err = p.save()
+	if err != nil {
+		http.Error(w, err.String(), http.StatusInternalServerError)
+		return
+	}
+	http.Redirect(w, r, "/view/"+title, http.StatusFound)
+}
+
+ +

+Any errors that occur during p.save() will be reported +to the user. +

+ +

Template caching

+ +

+There is an inefficiency in this code: renderTemplate calls +ParseFile every time a page is rendered. +A better approach would be to call ParseFile once for each +template at program initialization, and store the resultant +*Template values in a data structure for later use. +

+ +

+First we create a global map named templates in which to store +our *Template values, keyed by string +(the template name): +

+ +
+var templates = make(map[string]*template.Template)
+
+ +

+Then we create an init function, which will be called before +main at program initialization. The function +template.Must is a convenience wrapper that panics when passed a +non-nil os.Error value, and otherwise returns the +*Template unaltered. A panic is appropriate here; if the templates +can't be loaded the only sensible thing to do is exit the program. +

+ +
+func init() {
+	for _, tmpl := range []string{"edit", "view"} {
+		t := template.Must(template.ParseFile(tmpl + ".html"))
+		templates[tmpl] = t
+	}
+}
+
+ +

+A for loop is used with a range statement to iterate +over an array constant containing the names of the templates we want parsed. +If we were to add more templates to our program, we would add their names to +that array. +

+ +

+We then modify our renderTemplate function to call +the Execute method on the appropriate Template from +templates: + +

+func renderTemplate(w http.ResponseWriter, tmpl string, p *Page) {
+	err := templates[tmpl].Execute(w, p)
+	if err != nil {
+		http.Error(w, err.String(), http.StatusInternalServerError)
+	}
+}
+
+ +

Validation

+ +

+As you may have observed, this program has a serious security flaw: a user +can supply an arbitrary path to be read/written on the server. To mitigate +this, we can write a function to validate the title with a regular expression. +

+ +

+First, add "regexp" to the import list. +Then we can create a global variable to store our validation regexp: +

+ +
+var titleValidator = regexp.MustCompile("^[a-zA-Z0-9]+$")
+
+ +

+The function regexp.MustCompile will parse and compile the +regular expression, and return a regexp.Regexp. +MustCompile is distinct from Compile in that it will +panic if the expression compilation fails, while Compile returns +an os.Error as a second parameter. +

+ +

+Now, let's write a function that extracts the title string from the request +URL, and tests it against our TitleValidator expression: +

+ +
+func getTitle(w http.ResponseWriter, r *http.Request) (title string, err os.Error) {
+	title = r.URL.Path[lenPath:]
+	if !titleValidator.MatchString(title) {
+		http.NotFound(w, r)
+		err = os.NewError("Invalid Page Title")
+	}
+	return
+}
+
+ +

+If the title is valid, it will be returned along with a nil +error value. If the title is invalid, the function will write a +"404 Not Found" error to the HTTP connection, and return an error to the +handler. +

+ +

+Let's put a call to getTitle in each of the handlers: +

+ +
+func viewHandler(w http.ResponseWriter, r *http.Request) {
+	title, err := getTitle(w, r)
+	if err != nil {
+		return
+	}
+	p, err := loadPage(title)
+	if err != nil {
+		http.Redirect(w, r, "/edit/"+title, http.StatusFound)
+		return
+	}
+	renderTemplate(w, "view", p)
+}
+
+func editHandler(w http.ResponseWriter, r *http.Request) {
+	title, err := getTitle(w, r)
+	if err != nil {
+		return
+	}
+	p, err := loadPage(title)
+	if err != nil {
+		p = &Page{Title: title}
+	}
+	renderTemplate(w, "edit", p)
+}
+
+func saveHandler(w http.ResponseWriter, r *http.Request) {
+	title, err := getTitle(w, r)
+	if err != nil {
+		return
+	}
+	body := r.FormValue("body")
+	p := &Page{Title: title, Body: []byte(body)}
+	err = p.save()
+	if err != nil {
+		http.Error(w, err.String(), http.StatusInternalServerError)
+		return
+	}
+	http.Redirect(w, r, "/view/"+title, http.StatusFound)
+}
+
+ +

Introducing Function Literals and Closures

+ +

+Catching the error condition in each handler introduces a lot of repeated code. +What if we could wrap each of the handlers in a function that does this +validation and error checking? Go's +function +literals provide a powerful means of abstracting functionality +that can help us here. +

+ +

+First, we re-write the function definition of each of the handlers to accept +a title string: +

+ +
+func viewHandler(w http.ResponseWriter, r *http.Request, title string)
+func editHandler(w http.ResponseWriter, r *http.Request, title string)
+func saveHandler(w http.ResponseWriter, r *http.Request, title string)
+
+ +

+Now let's define a wrapper function that takes a function of the above +type, and returns a function of type http.HandlerFunc +(suitable to be passed to the function http.HandleFunc): +

+ +
+func makeHandler(fn func (http.ResponseWriter, *http.Request, string)) http.HandlerFunc {
+	return func(w http.ResponseWriter, r *http.Request) {
+		// Here we will extract the page title from the Request,
+		// and call the provided handler 'fn'
+	}
+}
+
+ +

+The returned function is called a closure because it encloses values defined +outside of it. In this case, the variable fn (the single argument +to makeHandler) is enclosed by the closure. The variable +fn will be one of our save, edit, or view handlers. +

+ +

+Now we can take the code from getTitle and use it here +(with some minor modifications): +

+ +
+func makeHandler(fn func(http.ResponseWriter, *http.Request, string)) http.HandlerFunc {
+	return func(w http.ResponseWriter, r *http.Request) {
+		title := r.URL.Path[lenPath:]
+		if !titleValidator.MatchString(title) {
+			http.NotFound(w, r)
+			return
+		}
+		fn(w, r, title)
+	}
+}
+
+ +

+The closure returned by makeHandler is a function that takes +an http.ResponseWriter and http.Request (in other +words, an http.HandlerFunc). +The closure extracts the title from the request path, and +validates it with the TitleValidator regexp. If the +title is invalid, an error will be written to the +ResponseWriter using the http.NotFound function. +If the title is valid, the enclosed handler function +fn will be called with the ResponseWriter, +Request, and title as arguments. +

+ +

+Now we can wrap the handler functions with makeHandler in +main, before they are registered with the http +package: +

+ +
+func main() {
+	http.HandleFunc("/view/", makeHandler(viewHandler))
+	http.HandleFunc("/edit/", makeHandler(editHandler))
+	http.HandleFunc("/save/", makeHandler(saveHandler))
+	http.ListenAndServe(":8080", nil)
+}
+
+ +

+Finally we remove the calls to getTitle from the handler functions, +making them much simpler: +

+ +
+func viewHandler(w http.ResponseWriter, r *http.Request, title string) {
+	p, err := loadPage(title)
+	if err != nil {
+		http.Redirect(w, r, "/edit/"+title, http.StatusFound)
+		return
+	}
+	renderTemplate(w, "view", p)
+}
+
+func editHandler(w http.ResponseWriter, r *http.Request, title string) {
+	p, err := loadPage(title)
+	if err != nil {
+		p = &Page{Title: title}
+	}
+	renderTemplate(w, "edit", p)
+}
+
+func saveHandler(w http.ResponseWriter, r *http.Request, title string) {
+	body := r.FormValue("body")
+	p := &Page{Title: title, Body: []byte(body)}
+	err := p.save()
+	if err != nil {
+		http.Error(w, err.String(), http.StatusInternalServerError)
+		return
+	}
+	http.Redirect(w, r, "/view/"+title, http.StatusFound)
+}
+
+ +

Try it out!

+ +

+Click here to view the final code listing. +

+ +

+Recompile the code, and run the app: +

+ +
+$ 8g wiki.go
+$ 8l wiki.8
+$ ./8.out
+
+ +

+Visiting http://localhost:8080/view/ANewPage +should present you with the page edit form. You should then be able to +enter some text, click 'Save', and be redirected to the newly created page. +

+ +

Other tasks

+ +

+Here are some simple tasks you might want to tackle on your own: +

+ +
    +
  • Store templates in tmpl/ and page data in data/. +
  • Add a handler to make the web root redirect to + /view/FrontPage.
  • +
  • Spruce up the page templates by making them valid HTML and adding some + CSS rules.
  • +
  • Implement inter-page linking by converting instances of + [PageName] to
    + <a href="/view/PageName">PageName</a>. + (hint: you could use regexp.ReplaceAllFunc to do this) +
  • +
diff --git a/doc/codelab/wiki/notemplate.go b/doc/codelab/wiki/notemplate.go new file mode 100644 index 000000000..9cbe9ad76 --- /dev/null +++ b/doc/codelab/wiki/notemplate.go @@ -0,0 +1,55 @@ +package main + +import ( + "fmt" + "http" + "io/ioutil" + "os" +) + +type Page struct { + Title string + Body []byte +} + +func (p *Page) save() os.Error { + filename := p.Title + ".txt" + return ioutil.WriteFile(filename, p.Body, 0600) +} + +func loadPage(title string) (*Page, os.Error) { + filename := title + ".txt" + body, err := ioutil.ReadFile(filename) + if err != nil { + return nil, err + } + return &Page{Title: title, Body: body}, nil +} + +const lenPath = len("/view/") + +func viewHandler(w http.ResponseWriter, r *http.Request) { + title := r.URL.Path[lenPath:] + p, _ := loadPage(title) + fmt.Fprintf(w, "

%s

%s
", p.Title, p.Body) +} + +func editHandler(w http.ResponseWriter, r *http.Request) { + title := r.URL.Path[lenPath:] + p, err := loadPage(title) + if err != nil { + p = &Page{Title: title} + } + fmt.Fprintf(w, "

Editing %s

"+ + "
"+ + "
"+ + ""+ + "
", + p.Title, p.Title, p.Body) +} + +func main() { + http.HandleFunc("/view/", viewHandler) + http.HandleFunc("/edit/", editHandler) + http.ListenAndServe(":8080", nil) +} diff --git a/doc/codelab/wiki/part1-noerror.go b/doc/codelab/wiki/part1-noerror.go new file mode 100644 index 000000000..14cfc321a --- /dev/null +++ b/doc/codelab/wiki/part1-noerror.go @@ -0,0 +1,30 @@ +package main + +import ( + "fmt" + "io/ioutil" + "os" +) + +type Page struct { + Title string + Body []byte +} + +func (p *Page) save() os.Error { + filename := p.Title + ".txt" + return ioutil.WriteFile(filename, p.Body, 0600) +} + +func loadPage(title string) *Page { + filename := title + ".txt" + body, _ := ioutil.ReadFile(filename) + return &Page{Title: title, Body: body} +} + +func main() { + p1 := &Page{Title: "TestPage", Body: []byte("This is a sample page.")} + p1.save() + p2 := loadPage("TestPage") + fmt.Println(string(p2.Body)) +} diff --git a/doc/codelab/wiki/part1.go b/doc/codelab/wiki/part1.go new file mode 100644 index 000000000..4b0654f8b --- /dev/null +++ b/doc/codelab/wiki/part1.go @@ -0,0 +1,33 @@ +package main + +import ( + "fmt" + "io/ioutil" + "os" +) + +type Page struct { + Title string + Body []byte +} + +func (p *Page) save() os.Error { + filename := p.Title + ".txt" + return ioutil.WriteFile(filename, p.Body, 0600) +} + +func loadPage(title string) (*Page, os.Error) { + filename := title + ".txt" + body, err := ioutil.ReadFile(filename) + if err != nil { + return nil, err + } + return &Page{Title: title, Body: body}, nil +} + +func main() { + p1 := &Page{Title: "TestPage", Body: []byte("This is a sample Page.")} + p1.save() + p2, _ := loadPage("TestPage") + fmt.Println(string(p2.Body)) +} diff --git a/doc/codelab/wiki/part2.go b/doc/codelab/wiki/part2.go new file mode 100644 index 000000000..d57c3a01f --- /dev/null +++ b/doc/codelab/wiki/part2.go @@ -0,0 +1,40 @@ +package main + +import ( + "fmt" + "http" + "io/ioutil" + "os" +) + +type Page struct { + Title string + Body []byte +} + +func (p *Page) save() os.Error { + filename := p.Title + ".txt" + return ioutil.WriteFile(filename, p.Body, 0600) +} + +func loadPage(title string) (*Page, os.Error) { + filename := title + ".txt" + body, err := ioutil.ReadFile(filename) + if err != nil { + return nil, err + } + return &Page{Title: title, Body: body}, nil +} + +const lenPath = len("/view/") + +func viewHandler(w http.ResponseWriter, r *http.Request) { + title := r.URL.Path[lenPath:] + p, _ := loadPage(title) + fmt.Fprintf(w, "

%s

%s
", p.Title, p.Body) +} + +func main() { + http.HandleFunc("/view/", viewHandler) + http.ListenAndServe(":8080", nil) +} diff --git a/doc/codelab/wiki/srcextract.go b/doc/codelab/wiki/srcextract.go new file mode 100644 index 000000000..6b5fbcb43 --- /dev/null +++ b/doc/codelab/wiki/srcextract.go @@ -0,0 +1,72 @@ +package main + +import ( + "bytes" + "flag" + "go/parser" + "go/printer" + "go/ast" + "go/token" + "log" + "template" + "os" +) + +var ( + srcFn = flag.String("src", "", "source filename") + getName = flag.String("name", "", "func/type name to output") + html = flag.Bool("html", true, "output HTML") + showPkg = flag.Bool("pkg", false, "show package in output") +) + +func main() { + // handle input + flag.Parse() + if *srcFn == "" || *getName == "" { + flag.Usage() + os.Exit(2) + } + // load file + fs := token.NewFileSet() + file, err := parser.ParseFile(fs, *srcFn, nil, 0) + if err != nil { + log.Fatal(err) + } + // create filter + filter := func(name string) bool { + return name == *getName + } + // filter + if !ast.FilterFile(file, filter) { + os.Exit(1) + } + // print the AST + var b bytes.Buffer + printer.Fprint(&b, fs, file) + // drop package declaration + if !*showPkg { + for { + c, err := b.ReadByte() + if c == '\n' || err != nil { + break + } + } + } + // drop leading newlines + for { + b, err := b.ReadByte() + if err != nil { + break + } + if b != '\n' { + os.Stdout.Write([]byte{b}) + break + } + } + // output + if *html { + template.HTMLEscape(os.Stdout, b.Bytes()) + } else { + b.WriteTo(os.Stdout) + } +} diff --git a/doc/codelab/wiki/test.sh b/doc/codelab/wiki/test.sh new file mode 100755 index 000000000..ed63ff20f --- /dev/null +++ b/doc/codelab/wiki/test.sh @@ -0,0 +1,27 @@ +#!/usr/bin/env bash + +set -e +wiki_pid= +cleanup() { + kill $wiki_pid + rm -f test_*.out Test.txt final-test.bin final-test.go +} +trap cleanup 0 INT + +gomake get.bin +addr=$(./get.bin -addr) +sed s/:8080/$addr/ < final.go > final-test.go +gomake final-test.bin +(./final-test.bin) & +wiki_pid=$! + +sleep 1 + +./get.bin http://$addr/edit/Test > test_edit.out +diff -u test_edit.out test_edit.good +./get.bin -post=body=some%20content http://$addr/save/Test +diff -u Test.txt test_Test.txt.good +./get.bin http://$addr/view/Test > test_view.out +diff -u test_view.out test_view.good + +echo PASS diff --git a/doc/codelab/wiki/test_Test.txt.good b/doc/codelab/wiki/test_Test.txt.good new file mode 100644 index 000000000..f0eec86f6 --- /dev/null +++ b/doc/codelab/wiki/test_Test.txt.good @@ -0,0 +1 @@ +some content \ No newline at end of file diff --git a/doc/codelab/wiki/test_edit.good b/doc/codelab/wiki/test_edit.good new file mode 100644 index 000000000..36c6dbb73 --- /dev/null +++ b/doc/codelab/wiki/test_edit.good @@ -0,0 +1,6 @@ +

Editing Test

+ +
+
+
+
diff --git a/doc/codelab/wiki/test_view.good b/doc/codelab/wiki/test_view.good new file mode 100644 index 000000000..07e8edb22 --- /dev/null +++ b/doc/codelab/wiki/test_view.good @@ -0,0 +1,5 @@ +

Test

+ +

[edit]

+ +
some content
diff --git a/doc/codelab/wiki/view.html b/doc/codelab/wiki/view.html new file mode 100644 index 000000000..023391577 --- /dev/null +++ b/doc/codelab/wiki/view.html @@ -0,0 +1,5 @@ +

{{.Title |html}}

+ +

[edit]

+ +
{{printf "%s" .Body |html}}
diff --git a/doc/codelab/wiki/wiki.html b/doc/codelab/wiki/wiki.html new file mode 100644 index 000000000..634babd8b --- /dev/null +++ b/doc/codelab/wiki/wiki.html @@ -0,0 +1,784 @@ + +

Introduction

+ +

+Covered in this codelab: +

+
    +
  • Creating a data structure with load and save methods
  • +
  • Using the http package to build web applications +
  • Using the template package to process HTML templates
  • +
  • Using the regexp package to validate user input
  • +
  • Using closures
  • +
+ +

+Assumed knowledge: +

+
    +
  • Programming experience
  • +
  • Understanding of basic web technologies (HTTP, HTML)
  • +
  • Some UNIX command-line knowledge
  • +
+ +

Getting Started

+ +

+At present, you need to have a Linux, OS X, or FreeBSD machine to run Go. If +you don't have access to one, you could set up a Linux Virtual Machine (using +VirtualBox or similar) or a +Virtual +Private Server. +

+ +

+Install Go (see the Installation Instructions). +

+ +

+Make a new directory for this codelab and cd to it: +

+ +
+$ mkdir ~/gowiki
+$ cd ~/gowiki
+
+ +

+Create a file named wiki.go, open it in your favorite editor, and +add the following lines: +

+ +
+package main
+
+import (
+	"fmt"
+	"io/ioutil"
+	"os"
+)
+
+ +

+We import the fmt, ioutil and os +packages from the Go standard library. Later, as we implement additional +functionality, we will add more packages to this import +declaration. +

+ +

Data Structures

+ +

+Let's start by defining the data structures. A wiki consists of a series of +interconnected pages, each of which has a title and a body (the page content). +Here, we define Page as a struct with two fields representing +the title and body. +

+ +
+!srcextract.bin -src=part1.go -name=Page
+
+ +

+The type []byte means "a byte slice". +(See Effective Go +for more on slices.) +The Body element is a []byte rather than +string because that is the type expected by the io +libraries we will use, as you'll see below. +

+ +

+The Page struct describes how page data will be stored in memory. +But what about persistent storage? We can address that by creating a +save method on Page: +

+ +
+!srcextract.bin -src=part1.go -name=save
+
+ +

+This method's signature reads: "This is a method named save that +takes as its receiver p, a pointer to Page . It takes +no parameters, and returns a value of type os.Error." +

+ +

+This method will save the Page's Body to a text +file. For simplicity, we will use the Title as the file name. +

+ +

+The save method returns an os.Error value because +that is the return type of WriteFile (a standard library function +that writes a byte slice to a file). The save method returns the +error value, to let the application handle it should anything go wrong while +writing the file. If all goes well, Page.save() will return +nil (the zero-value for pointers, interfaces, and some other +types). +

+ +

+The octal integer constant 0600, passed as the third parameter to +WriteFile, indicates that the file should be created with +read-write permissions for the current user only. (See the Unix man page +open(2) for details.) +

+ +

+We will want to load pages, too: +

+ +
+!srcextract.bin -src=part1-noerror.go -name=loadPage
+
+ +

+The function loadPage constructs the file name from +Title, reads the file's contents into a new +Page, and returns a pointer to that new page. +

+ +

+Functions can return multiple values. The standard library function +io.ReadFile returns []byte and os.Error. +In loadPage, error isn't being handled yet; the "blank identifier" +represented by the underscore (_) symbol is used to throw away the +error return value (in essence, assigning the value to nothing). +

+ +

+But what happens if ReadFile encounters an error? For example, +the file might not exist. We should not ignore such errors. Let's modify the +function to return *Page and os.Error. +

+ +
+!srcextract.bin -src=part1.go -name=loadPage
+
+ +

+Callers of this function can now check the second parameter; if it is +nil then it has successfully loaded a Page. If not, it will be an +os.Error that can be handled by the caller (see the os package documentation for +details). +

+ +

+At this point we have a simple data structure and the ability to save to and +load from a file. Let's write a main function to test what we've +written: +

+ +
+!srcextract.bin -src=part1.go -name=main
+
+ +

+After compiling and executing this code, a file named TestPage.txt +would be created, containing the contents of p1. The file would +then be read into the struct p2, and its Body element +printed to the screen. +

+ +

+You can compile and run the program like this: +

+ +
+$ 8g wiki.go
+$ 8l wiki.8
+$ ./8.out
+This is a sample page.
+
+ +

+(The 8g and 8l commands are applicable to +GOARCH=386. If you're on an amd64 system, +substitute 6's for the 8's.) +

+ +

+Click here to view the code we've written so far. +

+ +

Introducing the http package (an interlude)

+ +

+Here's a full working example of a simple web server: +

+ +
+!htmlify.bin < http-sample.go
+
+ +

+The main function begins with a call to +http.HandleFunc, which tells the http package to +handle all requests to the web root ("/") with +handler. +

+ +

+It then calls http.ListenAndServe, specifying that it should +listen on port 8080 on any interface (":8080"). (Don't +worry about its second parameter, nil, for now.) +This function will block until the program is terminated. +

+ +

+The function handler is of the type http.HandlerFunc. +It takes an http.ResponseWriter and an http.Request as +its arguments. +

+ +

+An http.ResponseWriter value assembles the HTTP server's response; by writing +to it, we send data to the HTTP client. +

+ +

+An http.Request is a data structure that represents the client +HTTP request. The string r.URL.Path is the path component +of the request URL. The trailing [1:] means +"create a sub-slice of Path from the 1st character to the end." +This drops the leading "/" from the path name. +

+ +

+If you run this program and access the URL: +

+
http://localhost:8080/monkeys
+

+the program would present a page containing: +

+
Hi there, I love monkeys!
+ +

Using http to serve wiki pages

+ +

+To use the http package, it must be imported: +

+ +
+import (
+	"fmt"
+	"http"
+	"io/ioutil"
+	"os"
+)
+
+ +

+Let's create a handler to view a wiki page: +

+ +
+!srcextract.bin -src=part2.go -name=lenPath
+
+!srcextract.bin -src=part2.go -name=viewHandler
+
+ +

+First, this function extracts the page title from r.URL.Path, +the path component of the request URL. The global constant +lenPath is the length of the leading "/view/" +component of the request path. +The Path is re-sliced with [lenPath:] to drop the +first 6 characters of the string. This is because the path will invariably +begin with "/view/", which is not part of the page title. +

+ +

+The function then loads the page data, formats the page with a string of simple +HTML, and writes it to w, the http.ResponseWriter. +

+ +

+Again, note the use of _ to ignore the os.Error +return value from loadPage. This is done here for simplicity +and generally considered bad practice. We will attend to this later. +

+ +

+To use this handler, we create a main function that +initializes http using the viewHandler to handle +any requests under the path /view/. +

+ +
+!srcextract.bin -src=part2.go -name=main
+
+ +

+Click here to view the code we've written so far. +

+ +

+Let's create some page data (as test.txt), compile our code, and +try serving a wiki page: +

+ +
+$ echo "Hello world" > test.txt
+$ 8g wiki.go
+$ 8l wiki.8
+$ ./8.out
+
+ +

+With this web server running, a visit to http://localhost:8080/view/test +should show a page titled "test" containing the words "Hello world". +

+ +

Editing Pages

+ +

+A wiki is not a wiki without the ability to edit pages. Let's create two new +handlers: one named editHandler to display an 'edit page' form, +and the other named saveHandler to save the data entered via the +form. +

+ +

+First, we add them to main(): +

+ +
+!srcextract.bin -src=final-noclosure.go -name=main
+
+ +

+The function editHandler loads the page +(or, if it doesn't exist, create an empty Page struct), +and displays an HTML form. +

+ +
+!srcextract.bin -src=notemplate.go -name=editHandler
+
+ +

+This function will work fine, but all that hard-coded HTML is ugly. +Of course, there is a better way. +

+ +

The template package

+ +

+The template package is part of the Go standard library. +(A new template package is coming; this code lab will be updated soon.) +We can +use template to keep the HTML in a separate file, allowing +us to change the layout of our edit page without modifying the underlying Go +code. +

+ +

+First, we must add template to the list of imports: +

+ +
+import (
+	"http"
+	"io/ioutil"
+	"os"
+	"template"
+)
+
+ +

+Let's create a template file containing the HTML form. +Open a new file named edit.html, and add the following lines: +

+ +
+!htmlify.bin < edit.html
+
+ +

+Modify editHandler to use the template, instead of the hard-coded +HTML: +

+ +
+!srcextract.bin -src=final-noerror.go -name=editHandler
+
+ +

+The function template.ParseFile will read the contents of +edit.html and return a *template.Template. +

+ +

+The method t.Execute executes the template, writing the +generated HTML to the http.ResponseWriter. +The .Title and .Body dotted identifiers refer to +p.Title and p.Body. +

+ +

+Template directives are enclosed in double curly braces. +The printf "%s" .Body instruction is a function call +that outputs .Body as a string instead of a stream of bytes, +the same as a call to fmt.Printf. +The |html part of each directive pipes the value through the +html formatter before outputting it, which escapes HTML +characters (such as replacing > with &gt;), +preventing user data from corrupting the form HTML. +

+ +

+Now that we've removed the fmt.Fprintf statement, we can remove +"fmt" from the import list. +

+ +

+While we're working with templates, let's create a template for our +viewHandler called view.html: +

+ +
+!htmlify.bin < view.html
+
+ +

+Modify viewHandler accordingly: +

+ +
+!srcextract.bin -src=final-noerror.go -name=viewHandler
+
+ +

+Notice that we've used almost exactly the same templating code in both +handlers. Let's remove this duplication by moving the templating code +to its own function: +

+ +
+!srcextract.bin -src=final-template.go -name=viewHandler
+
+!srcextract.bin -src=final-template.go -name=editHandler
+
+!srcextract.bin -src=final-template.go -name=renderTemplate
+
+ +

+The handlers are now shorter and simpler. +

+ +

Handling non-existent pages

+ +

+What if you visit /view/APageThatDoesntExist? The program will +crash. This is because it ignores the error return value from +loadPage. Instead, if the requested Page doesn't exist, it should +redirect the client to the edit Page so the content may be created: +

+ +
+!srcextract.bin -src=final-noclosure.go -name=viewHandler
+
+ +

+The http.Redirect function adds an HTTP status code of +http.StatusFound (302) and a Location +header to the HTTP response. +

+ +

Saving Pages

+ +

+The function saveHandler will handle the form submission. +

+ +
+!srcextract.bin -src=final-template.go -name=saveHandler
+
+ +

+The page title (provided in the URL) and the form's only field, +Body, are stored in a new Page. +The save() method is then called to write the data to a file, +and the client is redirected to the /view/ page. +

+ +

+The value returned by FormValue is of type string. +We must convert that value to []byte before it will fit into +the Page struct. We use []byte(body) to perform +the conversion. +

+ +

Error handling

+ +

+There are several places in our program where errors are being ignored. This +is bad practice, not least because when an error does occur the program will +crash. A better solution is to handle the errors and return an error message +to the user. That way if something does go wrong, the server will continue to +function and the user will be notified. +

+ +

+First, let's handle the errors in renderTemplate: +

+ +
+!srcextract.bin -src=final-parsetemplate.go -name=renderTemplate
+
+ +

+The http.Error function sends a specified HTTP response code +(in this case "Internal Server Error") and error message. +Already the decision to put this in a separate function is paying off. +

+ +

+Now let's fix up saveHandler: +

+ +
+!srcextract.bin -src=final-noclosure.go -name=saveHandler
+
+ +

+Any errors that occur during p.save() will be reported +to the user. +

+ +

Template caching

+ +

+There is an inefficiency in this code: renderTemplate calls +ParseFile every time a page is rendered. +A better approach would be to call ParseFile once for each +template at program initialization, and store the resultant +*Template values in a data structure for later use. +

+ +

+First we create a global map named templates in which to store +our *Template values, keyed by string +(the template name): +

+ +
+!srcextract.bin -src=final.go -name=templates
+
+ +

+Then we create an init function, which will be called before +main at program initialization. The function +template.Must is a convenience wrapper that panics when passed a +non-nil os.Error value, and otherwise returns the +*Template unaltered. A panic is appropriate here; if the templates +can't be loaded the only sensible thing to do is exit the program. +

+ +
+!srcextract.bin -src=final.go -name=init
+
+ +

+A for loop is used with a range statement to iterate +over an array constant containing the names of the templates we want parsed. +If we were to add more templates to our program, we would add their names to +that array. +

+ +

+We then modify our renderTemplate function to call +the Execute method on the appropriate Template from +templates: + +

+!srcextract.bin -src=final.go -name=renderTemplate
+
+ +

Validation

+ +

+As you may have observed, this program has a serious security flaw: a user +can supply an arbitrary path to be read/written on the server. To mitigate +this, we can write a function to validate the title with a regular expression. +

+ +

+First, add "regexp" to the import list. +Then we can create a global variable to store our validation regexp: +

+ +
+!srcextract.bin -src=final-noclosure.go -name=titleValidator
+
+ +

+The function regexp.MustCompile will parse and compile the +regular expression, and return a regexp.Regexp. +MustCompile is distinct from Compile in that it will +panic if the expression compilation fails, while Compile returns +an os.Error as a second parameter. +

+ +

+Now, let's write a function that extracts the title string from the request +URL, and tests it against our TitleValidator expression: +

+ +
+!srcextract.bin -src=final-noclosure.go -name=getTitle
+
+ +

+If the title is valid, it will be returned along with a nil +error value. If the title is invalid, the function will write a +"404 Not Found" error to the HTTP connection, and return an error to the +handler. +

+ +

+Let's put a call to getTitle in each of the handlers: +

+ +
+!srcextract.bin -src=final-noclosure.go -name=viewHandler
+
+!srcextract.bin -src=final-noclosure.go -name=editHandler
+
+!srcextract.bin -src=final-noclosure.go -name=saveHandler
+
+ +

Introducing Function Literals and Closures

+ +

+Catching the error condition in each handler introduces a lot of repeated code. +What if we could wrap each of the handlers in a function that does this +validation and error checking? Go's +function +literals provide a powerful means of abstracting functionality +that can help us here. +

+ +

+First, we re-write the function definition of each of the handlers to accept +a title string: +

+ +
+func viewHandler(w http.ResponseWriter, r *http.Request, title string)
+func editHandler(w http.ResponseWriter, r *http.Request, title string)
+func saveHandler(w http.ResponseWriter, r *http.Request, title string)
+
+ +

+Now let's define a wrapper function that takes a function of the above +type, and returns a function of type http.HandlerFunc +(suitable to be passed to the function http.HandleFunc): +

+ +
+func makeHandler(fn func (http.ResponseWriter, *http.Request, string)) http.HandlerFunc {
+	return func(w http.ResponseWriter, r *http.Request) {
+		// Here we will extract the page title from the Request,
+		// and call the provided handler 'fn'
+	}
+}
+
+ +

+The returned function is called a closure because it encloses values defined +outside of it. In this case, the variable fn (the single argument +to makeHandler) is enclosed by the closure. The variable +fn will be one of our save, edit, or view handlers. +

+ +

+Now we can take the code from getTitle and use it here +(with some minor modifications): +

+ +
+!srcextract.bin -src=final.go -name=makeHandler
+
+ +

+The closure returned by makeHandler is a function that takes +an http.ResponseWriter and http.Request (in other +words, an http.HandlerFunc). +The closure extracts the title from the request path, and +validates it with the TitleValidator regexp. If the +title is invalid, an error will be written to the +ResponseWriter using the http.NotFound function. +If the title is valid, the enclosed handler function +fn will be called with the ResponseWriter, +Request, and title as arguments. +

+ +

+Now we can wrap the handler functions with makeHandler in +main, before they are registered with the http +package: +

+ +
+!srcextract.bin -src=final.go -name=main
+
+ +

+Finally we remove the calls to getTitle from the handler functions, +making them much simpler: +

+ +
+!srcextract.bin -src=final.go -name=viewHandler
+
+!srcextract.bin -src=final.go -name=editHandler
+
+!srcextract.bin -src=final.go -name=saveHandler
+
+ +

Try it out!

+ +

+Click here to view the final code listing. +

+ +

+Recompile the code, and run the app: +

+ +
+$ 8g wiki.go
+$ 8l wiki.8
+$ ./8.out
+
+ +

+Visiting http://localhost:8080/view/ANewPage +should present you with the page edit form. You should then be able to +enter some text, click 'Save', and be redirected to the newly created page. +

+ +

Other tasks

+ +

+Here are some simple tasks you might want to tackle on your own: +

+ +
    +
  • Store templates in tmpl/ and page data in data/. +
  • Add a handler to make the web root redirect to + /view/FrontPage.
  • +
  • Spruce up the page templates by making them valid HTML and adding some + CSS rules.
  • +
  • Implement inter-page linking by converting instances of + [PageName] to
    + <a href="/view/PageName">PageName</a>. + (hint: you could use regexp.ReplaceAllFunc to do this) +
  • +
diff --git a/doc/codereview_with_mq.html b/doc/codereview_with_mq.html new file mode 100644 index 000000000..33f415f13 --- /dev/null +++ b/doc/codereview_with_mq.html @@ -0,0 +1,113 @@ + + +

Introduction

+ +

+The Mercurial Queues extension (mq) provides a mechanism for +managing patches on top of a Mercurial repository and is described in detail +in Chapters +12 +and 13 +of Mercurial: The Definitive Guide. +This document explains how to use mq in conjunction +with the codereview Mercurial extension described in the +instructions for contributing to the Go project. +It assumes you have read those instructions. +

+ +

Configuration

+ +

+To enable mq edit either $HOME/.hgrc (to enable it +for all of your repositories) or $GOROOT/.hg/hgrc (to enable it for the +repository at $GOROOT) to add:

+ +
+[extensions]
+mq=
+
+ +

+Since pulling, pushing, updating and committing while mq patches +are applied can damage your repository or a remote one, add these lines to +prevent that case: +

+ +
+[hooks]
+# Prevent "hg pull" if MQ patches are applied.
+prechangegroup.mq-no-pull = ! hg qtop > /dev/null 2>&1
+# Prevent "hg push" if MQ patches are applied.
+preoutgoing.mq-no-push = ! hg qtop > /dev/null 2>&1
+# Prevent "hg update" if MQ patches are applied.
+preupdate.mq-no-update = ! hg qtop > /dev/null 2>&1
+
+ +

Making a change

+ +

+The entire checked-out tree is writable and you can use mq, +as documented in Chapter +12 +of "The Guide", +to implement your change as a single patch or a series of patches. + +

+ +

When you are ready to send a change out for review, run

+ +
+$ hg change
+
+ +

from any directory in your Go repository with all of the mq patches relevant to your +change applied and then proceed as instructed in contributing +to the Go project. +

+ +

+The change number reported by hg change, preceded by a +, +can be used as an mq patch guard to assist in controlling which patches +are applied as described in Chapter +13 +of "The Guide". +For example, the command: +

+ +
+for p in $(hg qapplied); do hg qguard $p +99999; done
+
+ +

+will apply the guard +99999 guard to all currently applied mq +patches. +

+ +

Synchronizing your client

+ +

While you were working, others might have submitted changes +to the repository and, as explained in contributing +to the Go project, it is necessary to synchronize your repository using +hg syncbefore sending your change list for review. +Because hg sync runs hg pull -u, +you should not run hg sync while mq patches are +applied. Instead +pop all your patches before running hg sync and reapply them after +it has completed. +

+ +

+When reapplying the patches, you may need to resolve conflicts +as described in contributing to the Go project. +

+ +

Mailing the change for review

+ +

+You should have all of the mq patches relevant to your +change applied when you run hg mail. + +

Submitting the change after the review

+ +If you are a committer, you should have all of the mq patches relevant to your +change applied when you run hg commit. diff --git a/doc/codewalk/codewalk.css b/doc/codewalk/codewalk.css new file mode 100644 index 000000000..a0814e4d2 --- /dev/null +++ b/doc/codewalk/codewalk.css @@ -0,0 +1,234 @@ +/* + Copyright 2010 The Go Authors. All rights reserved. + Use of this source code is governed by a BSD-style + license that can be found in the LICENSE file. +*/ + +#codewalk-main { + text-align: left; + width: 100%; + overflow: auto; +} + +#code-display { + border: 0; + width: 100%; +} + +.setting { + font-size: 8pt; + color: #888888; + padding: 5px; +} + +.hotkey { + text-decoration: underline; +} + +/* Style for Comments (the left-hand column) */ + +#comment-column { + margin: 0pt; + width: 30%; +} + +#comment-column.right { + float: right; +} + +#comment-column.left { + float: left; +} + +#comment-area { + overflow-x: hidden; + overflow-y: auto; +} + +.comment { + cursor: pointer; + font-size: 16px; + border: 2px solid #ba9836; + margin-bottom: 10px; + margin-right: 10px; /* yes, for both .left and .right */ +} + +.comment:last-child { + margin-bottom: 0px; +} + +.right .comment { + margin-left: 10px; +} + +.right .comment.first { +} + +.right .comment.last { +} + +.left .comment.first { +} + +.left .comment.last { +} + +.comment.selected { + border-color: #99b2cb; +} + +.right .comment.selected { + border-left-width: 12px; + margin-left: 0px; +} + +.left .comment.selected { + border-right-width: 12px; + margin-right: 0px; +} + +.comment-link { + display: none; +} + +.comment-title { + font-size: small; + font-weight: bold; + background-color: #fffff0; + padding-right: 10px; + padding-left: 10px; + padding-top: 5px; + padding-bottom: 5px; +} + +.right .comment-title { +} + +.left .comment-title { +} + +.comment.selected .comment-title { + background-color: #f8f8ff; +} + +.comment-text { + overflow: auto; + padding-left: 10px; + padding-right: 10px; + padding-top: 10px; + padding-bottom: 5px; + font-size: small; + line-height: 1.3em; +} + +.comment-text p { + margin-top: 0em; + margin-bottom: 0.5em; +} + +.comment-text p:last-child { + margin-bottom: 0em; +} + +.file-name { + font-size: x-small; + padding-top: 0px; + padding-bottom: 5px; +} + +.hidden-filepaths .file-name { + display: none; +} + +.path-dir { + color: #555; +} + +.path-file { + color: #555; +} + + +/* Style for Code (the right-hand column) */ + +/* Wrapper for the code column to make widths get calculated correctly */ +#code-column { + display: block; + position: relative; + margin: 0pt; + width: 70%; +} + +#code-column.left { + float: left; +} + +#code-column.right { + float: right; +} + +#code-area { + background-color: #f8f8ff; + border: 2px solid #99b2cb; + padding: 5px; +} + +.left #code-area { + margin-right: -1px; +} + +.right #code-area { + margin-left: -1px; +} + +#code-header { + margin-bottom: 5px; +} + +#code { + background-color: white; +} + +code { + font-size: 100%; +} + +.codewalkhighlight { + font-weight: bold; + background-color: #f8f8ff; +} + +#code-display { + margin-top: 0px; + margin-bottom: 0px; +} + +#sizer { + position: absolute; + cursor: col-resize; + left: 0px; + top: 0px; + width: 8px; +} + +/* Style for options (bottom strip) */ + +#code-options { + display: none; +} + +#code-options > span { + padding-right: 20px; +} + +#code-options .selected { + border-bottom: 1px dotted; +} + +#comment-options { + text-align: center; +} + +div#content { + padding-bottom: 0em; +} diff --git a/doc/codewalk/codewalk.js b/doc/codewalk/codewalk.js new file mode 100644 index 000000000..f780bc7a5 --- /dev/null +++ b/doc/codewalk/codewalk.js @@ -0,0 +1,305 @@ +// Copyright 2010 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +/** + * A class to hold information about the Codewalk Viewer. + * @param {jQuery} context The top element in whose context the viewer should + * operate. It will not touch any elements above this one. + * @constructor + */ + var CodewalkViewer = function(context) { + this.context = context; + + /** + * The div that contains all of the comments and their controls. + */ + this.commentColumn = this.context.find('#comment-column'); + + /** + * The div that contains the comments proper. + */ + this.commentArea = this.context.find('#comment-area'); + + /** + * The div that wraps the iframe with the code, as well as the drop down menu + * listing the different files. + * @type {jQuery} + */ + this.codeColumn = this.context.find('#code-column'); + + /** + * The div that contains the code but excludes the options strip. + * @type {jQuery} + */ + this.codeArea = this.context.find('#code-area'); + + /** + * The iframe that holds the code (from Sourcerer). + * @type {jQuery} + */ + this.codeDisplay = this.context.find('#code-display'); + + /** + * The overlaid div used as a grab handle for sizing the code/comment panes. + * @type {jQuery} + */ + this.sizer = this.context.find('#sizer'); + + /** + * The full-screen overlay that ensures we don't lose track of the mouse + * while dragging. + * @type {jQuery} + */ + this.overlay = this.context.find('#overlay'); + + /** + * The hidden input field that we use to hold the focus so that we can detect + * shortcut keypresses. + * @type {jQuery} + */ + this.shortcutInput = this.context.find('#shortcut-input'); + + /** + * The last comment that was selected. + * @type {jQuery} + */ + this.lastSelected = null; +}; + +/** + * Minimum width of the comments or code pane, in pixels. + * @type {number} + */ +CodewalkViewer.MIN_PANE_WIDTH = 200; + +/** + * Navigate the code iframe to the given url and update the code popout link. + * @param {string} url The target URL. + * @param {Object} opt_window Window dependency injection for testing only. + */ +CodewalkViewer.prototype.navigateToCode = function(url, opt_window) { + if (!opt_window) opt_window = window; + // Each iframe is represented by two distinct objects in the DOM: an iframe + // object and a window object. These do not expose the same capabilities. + // Here we need to get the window representation to get the location member, + // so we access it directly through window[] since jQuery returns the iframe + // representation. + // We replace location rather than set so as not to create a history for code + // navigation. + opt_window['code-display'].location.replace(url); + var k = url.indexOf('&'); + if (k != -1) url = url.slice(0, k); + k = url.indexOf('fileprint='); + if (k != -1) url = url.slice(k+10, url.length); + this.context.find('#code-popout-link').attr('href', url); +}; + +/** + * Selects the first comment from the list and forces a refresh of the code + * view. + */ +CodewalkViewer.prototype.selectFirstComment = function() { + // TODO(rsc): handle case where there are no comments + var firstSourcererLink = this.context.find('.comment:first'); + this.changeSelectedComment(firstSourcererLink); +}; + +/** + * Sets the target on all links nested inside comments to be _blank. + */ +CodewalkViewer.prototype.targetCommentLinksAtBlank = function() { + this.context.find('.comment a[href], #description a[href]').each(function() { + if (!this.target) this.target = '_blank'; + }); +}; + +/** + * Installs event handlers for all the events we care about. + */ +CodewalkViewer.prototype.installEventHandlers = function() { + var self = this; + + this.context.find('.comment') + .click(function(event) { + if (jQuery(event.target).is('a[href]')) return true; + self.changeSelectedComment(jQuery(this)); + return false; + }); + + this.context.find('#code-selector') + .change(function() {self.navigateToCode(jQuery(this).val());}); + + this.context.find('#description-table .quote-feet.setting') + .click(function() {self.toggleDescription(jQuery(this)); return false;}); + + this.sizer + .mousedown(function(ev) {self.startSizerDrag(ev); return false;}); + this.overlay + .mouseup(function(ev) {self.endSizerDrag(ev); return false;}) + .mousemove(function(ev) {self.handleSizerDrag(ev); return false;}); + + this.context.find('#prev-comment') + .click(function() { + self.changeSelectedComment(self.lastSelected.prev()); return false; + }); + + this.context.find('#next-comment') + .click(function() { + self.changeSelectedComment(self.lastSelected.next()); return false; + }); + + // Workaround for Firefox 2 and 3, which steal focus from the main document + // whenever the iframe content is (re)loaded. The input field is not shown, + // but is a way for us to bring focus back to a place where we can detect + // keypresses. + this.context.find('#code-display') + .load(function(ev) {self.shortcutInput.focus();}); + + jQuery(document).keypress(function(ev) { + switch(ev.which) { + case 110: // 'n' + self.changeSelectedComment(self.lastSelected.next()); + return false; + case 112: // 'p' + self.changeSelectedComment(self.lastSelected.prev()); + return false; + default: // ignore + } + }); + + window.onresize = function() {self.updateHeight();}; +}; + +/** + * Starts dragging the pane sizer. + * @param {Object} ev The mousedown event that started us dragging. + */ +CodewalkViewer.prototype.startSizerDrag = function(ev) { + this.initialCodeWidth = this.codeColumn.width(); + this.initialCommentsWidth = this.commentColumn.width(); + this.initialMouseX = ev.pageX; + this.overlay.show(); +}; + +/** + * Handles dragging the pane sizer. + * @param {Object} ev The mousemove event updating dragging position. + */ +CodewalkViewer.prototype.handleSizerDrag = function(ev) { + var delta = ev.pageX - this.initialMouseX; + if (this.codeColumn.is('.right')) delta = -delta; + var proposedCodeWidth = this.initialCodeWidth + delta; + var proposedCommentWidth = this.initialCommentsWidth - delta; + var mw = CodewalkViewer.MIN_PANE_WIDTH; + if (proposedCodeWidth < mw) delta = mw - this.initialCodeWidth; + if (proposedCommentWidth < mw) delta = this.initialCommentsWidth - mw; + proposedCodeWidth = this.initialCodeWidth + delta; + proposedCommentWidth = this.initialCommentsWidth - delta; + // If window is too small, don't even try to resize. + if (proposedCodeWidth < mw || proposedCommentWidth < mw) return; + this.codeColumn.width(proposedCodeWidth); + this.commentColumn.width(proposedCommentWidth); + this.options.codeWidth = parseInt( + this.codeColumn.width() / + (this.codeColumn.width() + this.commentColumn.width()) * 100); + this.context.find('#code-column-width').text(this.options.codeWidth + '%'); +}; + +/** + * Ends dragging the pane sizer. + * @param {Object} ev The mouseup event that caused us to stop dragging. + */ +CodewalkViewer.prototype.endSizerDrag = function(ev) { + this.overlay.hide(); + this.updateHeight(); +}; + +/** + * Toggles the Codewalk description between being shown and hidden. + * @param {jQuery} target The target that was clicked to trigger this function. + */ +CodewalkViewer.prototype.toggleDescription = function(target) { + var description = this.context.find('#description'); + description.toggle(); + target.find('span').text(description.is(':hidden') ? 'show' : 'hide'); + this.updateHeight(); +}; + +/** + * Changes the side of the window on which the code is shown and saves the + * setting in a cookie. + * @param {string?} codeSide The side on which the code should be, either + * 'left' or 'right'. + */ +CodewalkViewer.prototype.changeCodeSide = function(codeSide) { + var commentSide = codeSide == 'left' ? 'right' : 'left'; + this.context.find('#set-code-' + codeSide).addClass('selected'); + this.context.find('#set-code-' + commentSide).removeClass('selected'); + // Remove previous side class and add new one. + this.codeColumn.addClass(codeSide).removeClass(commentSide); + this.commentColumn.addClass(commentSide).removeClass(codeSide); + this.sizer.css(codeSide, 'auto').css(commentSide, 0); + this.options.codeSide = codeSide; +}; + +/** + * Adds selected class to newly selected comment, removes selected style from + * previously selected comment, changes drop down options so that the correct + * file is selected, and updates the code popout link. + * @param {jQuery} target The target that was clicked to trigger this function. + */ +CodewalkViewer.prototype.changeSelectedComment = function(target) { + var currentFile = target.find('.comment-link').attr('href'); + if (!currentFile) return; + + if (!(this.lastSelected && this.lastSelected.get(0) === target.get(0))) { + if (this.lastSelected) this.lastSelected.removeClass('selected'); + target.addClass('selected'); + this.lastSelected = target; + var targetTop = target.position().top; + var parentTop = target.parent().position().top; + if (targetTop + target.height() > parentTop + target.parent().height() || + targetTop < parentTop) { + var delta = targetTop - parentTop; + target.parent().animate( + {'scrollTop': target.parent().scrollTop() + delta}, + Math.max(delta / 2, 200), 'swing'); + } + var fname = currentFile.match(/(?:select=|fileprint=)\/[^&]+/)[0]; + fname = fname.slice(fname.indexOf('=')+2, fname.length); + this.context.find('#code-selector').val(fname); + this.context.find('#prev-comment').toggleClass( + 'disabled', !target.prev().length); + this.context.find('#next-comment').toggleClass( + 'disabled', !target.next().length); + } + + // Force original file even if user hasn't changed comments since they may + // have nagivated away from it within the iframe without us knowing. + this.navigateToCode(currentFile); +}; + +/** + * Updates the viewer by changing the height of the comments and code so that + * they fit within the height of the window. The function is typically called + * after the user changes the window size. + */ +CodewalkViewer.prototype.updateHeight = function() { + var windowHeight = jQuery(window).height() - 5 // GOK + var areaHeight = windowHeight - this.codeArea.offset().top + var footerHeight = this.context.find('#footer').outerHeight(true) + this.commentArea.height(areaHeight - footerHeight - this.context.find('#comment-options').outerHeight(true)) + var codeHeight = areaHeight - footerHeight - 15 // GOK + this.codeArea.height(codeHeight) + this.codeDisplay.height(codeHeight - this.codeDisplay.offset().top + this.codeArea.offset().top); + this.sizer.height(codeHeight); +}; + +jQuery(document).ready(function() { + var viewer = new CodewalkViewer(jQuery()); + viewer.selectFirstComment(); + viewer.targetCommentLinksAtBlank(); + viewer.installEventHandlers(); + viewer.updateHeight(); +}); diff --git a/doc/codewalk/codewalk.xml b/doc/codewalk/codewalk.xml new file mode 100644 index 000000000..9cd8361e8 --- /dev/null +++ b/doc/codewalk/codewalk.xml @@ -0,0 +1,124 @@ + + + + A codewalk is a guided tour through a piece of code. + It consists of a sequence of steps, each typically explaining + a highlighted section of code. +

+ + The godoc web server translates + an XML file like the one in the main window pane into the HTML + page that you're viewing now. +

+ + The codewalk with URL path /doc/codewalk/name + is loaded from the input file $GOROOT/doc/codewalk/name.xml. +

+ + This codewalk explains how to write a codewalk by examining + its own source code, + $GOROOT/doc/codewalk/codewalk.xml, + shown in the main window pane to the left. +
+ + + The codewalk input file is an XML file containing a single + <codewalk> element. + That element's title attribute gives the title + that is used both on the codewalk page and in the codewalk list. + + + + Each step in the codewalk is a <step> element + nested inside the main <codewalk>. + The step element's title attribute gives the step's title, + which is shown in a shaded bar above the main step text. + The element's src attribute specifies the source + code to show in the main window pane and, optionally, a range of + lines to highlight. +

+ + The first step in this codewalk does not highlight any lines: + its src is just a file name. +
+ + + The most complex part of the codewalk specification is + saying what lines to highlight. + Instead of ordinary line numbers, + the codewalk uses an address syntax that makes it possible + to describe the match by its content. + As the file gets edited, this descriptive address has a better + chance to continue to refer to the right section of the file. +

+ + To specify a source line, use a src attribute of the form + filename:address, + where address is an address in the syntax used by the text editors sam and acme. +

+ + The simplest address is a single regular expression. + The highlighted line in the main window pane shows that the + address for the “Title” step was /title=/, + which matches the first instance of that regular expression (title=) in the file. +
+ + + To highlight a range of source lines, the simplest address to use is + a pair of regular expressions + /regexp1/,/regexp2/. + The highlight begins with the line containing the first match for regexp1 + and ends with the line containing the first match for regexp2 + after the end of the match for regexp1. + Ignoring the HTML quoting, + The line containing the first match for regexp1 will be the first one highlighted, + and the line containing the first match for regexp2. +

+ + The address /<step/,/step>/ looks for the first instance of + <step in the file, and then starting after that point, + looks for the first instance of step>. + (Click on the “Steps” step above to see the highlight in action.) + Note that the < and > had to be written + using XML escapes in order to be valid XML. +
+ + + The /regexp/ + and /regexp1/,/regexp2/ + forms suffice for most highlighting. +

+ + The full address syntax is summarized in this table + (an excerpt of Table II from + The text editor sam): +

+ + + + + + + + + + + + + + + + + + + + +
Simple addresses
#nThe empty string after character n
nLine n
/regexp/The first following match of the regular expression
$The null string at the end of the file
Compound addresses
a1+a2The address a2 evaluated starting at the right of a1
a1-a2The address a2 evaluated in the reverse direction starting at the left of a1
a1,a2From the left of a1 to the right of a2 (default 0,$).
+
+ + + +
diff --git a/doc/codewalk/functions.xml b/doc/codewalk/functions.xml new file mode 100644 index 000000000..986a017e1 --- /dev/null +++ b/doc/codewalk/functions.xml @@ -0,0 +1,115 @@ + + + + Go supports first class functions, higher-order functions, user-defined + function types, function literals, closures, and multiple return values. +

+ + This rich feature set supports a functional programming style in a strongly + typed language. +

+ + In this codewalk we will look at a simple program that simulates a dice game + called Pig and evaluates + basic strategies. +
+ + + Pig is a two-player game played with a 6-sided die. Each turn, you may roll or stay. +
    +
  • If you roll a 1, you lose all points for your turn and play passes to + your opponent. Any other roll adds its value to your turn score.
  • +
  • If you stay, your turn score is added to your total score, and play passes + to your opponent.
  • +
+ + The first person to reach 100 total points wins. +

+ + The score type stores the scores of the current and opposing + players, in addition to the points accumulated during the current turn. +
+ + + In Go, functions can be passed around just like any other value. A function's + type signature describes the types of its arguments and return values. +

+ + The action type is a function that takes a score + and returns the resulting score and whether the current turn is + over. +

+ + If the turn is over, the player and opponent fields + in the resulting score should be swapped, as it is now the other player's + turn. +
+ + + Go functions can return multiple values. +

+ + The functions roll and stay each return a pair of + values. They also match the action type signature. These + action functions define the rules of Pig. +
+ + + A function can use other functions as arguments and return values. +

+ + A strategy is a function that takes a score as input + and returns an action to perform.
+ (Remember, an action is itself a function.) +
+ + + Anonymous functions can be declared in Go, as in this example. Function + literals are closures: they inherit the scope of the function in which they + are declared. +

+ + One basic strategy in Pig is to continue rolling until you have accumulated at + least k points in a turn, and then stay. The argument k is + enclosed by this function literal, which matches the strategy type + signature. +
+ + + We simulate a game of Pig by calling an action to update the + score until one player reaches 100 points. Each + action is selected by calling the strategy function + associated with the current player. + + + + Functions can be compared for equality in Go. From the + language specification: + Function values are equal if they refer to the same function or if both are nil. +

+ + We enforce that a strategy function can only return a legal + action: either roll or stay. +
+ + + The roundRobin function simulates a tournament and tallies wins. + Each strategy plays each other strategy gamesPerSeries times. + + + + Variadic functions like ratioString take a variable number of + arguments. These arguments are available as a slice inside the function. + + + + The main function defines 100 basic strategies, simulates a round + robin tournament, and then prints the win/loss record of each strategy. +

+ + Among these strategies, staying at 25 is best, but the optimal strategy for + Pig is much more complex. +
+ +
diff --git a/doc/codewalk/markov.go b/doc/codewalk/markov.go new file mode 100644 index 000000000..959c2b158 --- /dev/null +++ b/doc/codewalk/markov.go @@ -0,0 +1,130 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +/* +Generating random text: a Markov chain algorithm + +Based on the program presented in the "Design and Implementation" chapter +of The Practice of Programming (Kernighan and Pike, Addison-Wesley 1999). +See also Computer Recreations, Scientific American 260, 122 - 125 (1989). + +A Markov chain algorithm generates text by creating a statistical model of +potential textual suffixes for a given prefix. Consider this text: + + I am not a number! I am a free man! + +Our Markov chain algorithm would arrange this text into this set of prefixes +and suffixes, or "chain": (This table assumes a prefix length of two words.) + + Prefix Suffix + + "" "" I + "" I am + I am a + I am not + a free man! + am a free + am not a + a number! I + number! I am + not a number! + +To generate text using this table we select an initial prefix ("I am", for +example), choose one of the suffixes associated with that prefix at random +with probability determined by the input statistics ("a"), +and then create a new prefix by removing the first word from the prefix +and appending the suffix (making the new prefix is "am a"). Repeat this process +until we can't find any suffixes for the current prefix or we exceed the word +limit. (The word limit is necessary as the chain table may contain cycles.) + +Our version of this program reads text from standard input, parsing it into a +Markov chain, and writes generated text to standard output. +The prefix and output lengths can be specified using the -prefix and -words +flags on the command-line. +*/ +package main + +import ( + "bufio" + "flag" + "fmt" + "io" + "os" + "rand" + "strings" + "time" +) + +// Prefix is a Markov chain prefix of one or more words. +type Prefix []string + +// String returns the Prefix as a string (for use as a map key). +func (p Prefix) String() string { + return strings.Join(p, " ") +} + +// Shift removes the first word from the Prefix and appends the given word. +func (p Prefix) Shift(word string) { + copy(p, p[1:]) + p[len(p)-1] = word +} + +// Chain contains a map ("chain") of prefixes to a list of suffixes. +// A prefix is a string of prefixLen words joined with spaces. +// A suffix is a single word. A prefix can have multiple suffixes. +type Chain struct { + chain map[string][]string + prefixLen int +} + +// NewChain returns a new Chain with prefixes of prefixLen words. +func NewChain(prefixLen int) *Chain { + return &Chain{make(map[string][]string), prefixLen} +} + +// Build reads text from the provided Reader and +// parses it into prefixes and suffixes that are stored in Chain. +func (c *Chain) Build(r io.Reader) { + br := bufio.NewReader(r) + p := make(Prefix, c.prefixLen) + for { + var s string + if _, err := fmt.Fscan(br, &s); err != nil { + break + } + key := p.String() + c.chain[key] = append(c.chain[key], s) + p.Shift(s) + } +} + +// Generate returns a string of at most n words generated from Chain. +func (c *Chain) Generate(n int) string { + p := make(Prefix, c.prefixLen) + var words []string + for i := 0; i < n; i++ { + choices := c.chain[p.String()] + if len(choices) == 0 { + break + } + next := choices[rand.Intn(len(choices))] + words = append(words, next) + p.Shift(next) + } + return strings.Join(words, " ") +} + +func main() { + // Register command-line flags. + numWords := flag.Int("words", 100, "maximum number of words to print") + prefixLen := flag.Int("prefix", 2, "prefix length in words") + + flag.Parse() // Parse command-line flags. + rand.Seed(time.Nanoseconds()) // Seed the random number generator. + + c := NewChain(*prefixLen) // Initialize a new Chain. + c.Build(os.Stdin) // Build chains from standard input. + text := c.Generate(*numWords) // Generate text. + fmt.Println(text) // Write text to standard output. +} diff --git a/doc/codewalk/markov.xml b/doc/codewalk/markov.xml new file mode 100644 index 000000000..a89b4d0ce --- /dev/null +++ b/doc/codewalk/markov.xml @@ -0,0 +1,308 @@ + + + + + + This codewalk describes a program that generates random text using + a Markov chain algorithm. The package comment describes the algorithm + and the operation of the program. Please read it before continuing. + + + + A chain consists of a prefix and a suffix. Each prefix is a set + number of words, while a suffix is a single word. + A prefix can have an arbitrary number of suffixes. + To model this data, we use a map[string][]string. + Each map key is a prefix (a string) and its values are + lists of suffixes (a slice of strings, []string). +

+ Here is the example table from the package comment + as modeled by this data structure: +
+map[string][]string{
+	" ":          {"I"},
+	" I":         {"am"},
+	"I am":       {"a", "not"},
+	"a free":     {"man!"},
+	"am a":       {"free"},
+	"am not":     {"a"},
+	"a number!":  {"I"},
+	"number! I":  {"am"},
+	"not a":      {"number!"},
+}
+ While each prefix consists of multiple words, we + store prefixes in the map as a single string. + It would seem more natural to store the prefix as a + []string, but we can't do this with a map because the + key type of a map must implement equality (and slices do not). +

+ Therefore, in most of our code we will model prefixes as a + []string and join the strings together with a space + to generate the map key: +
+Prefix               Map key
+
+[]string{"", ""}     " "
+[]string{"", "I"}    " I"
+[]string{"I", "am"}  "I am"
+
+
+ + + The complete state of the chain table consists of the table itself and + the word length of the prefixes. The Chain struct stores + this data. + + + + The Chain struct has two unexported fields (those that + do not begin with an upper case character), and so we write a + NewChain constructor function that initializes the + chain map with make and sets the + prefixLen field. +

+ This is constructor function is not strictly necessary as this entire + program is within a single package (main) and therefore + there is little practical difference between exported and unexported + fields. We could just as easily write out the contents of this function + when we want to construct a new Chain. + But using these unexported fields is good practice; it clearly denotes + that only methods of Chain and its constructor function should access + those fields. Also, structuring Chain like this means we + could easily move it into its own package at some later date. +
+ + + Since we'll be working with prefixes often, we define a + Prefix type with the concrete type []string. + Defining a named type clearly allows us to be explicit when we are + working with a prefix instead of just a []string. + Also, in Go we can define methods on any named type (not just structs), + so we can add methods that operate on Prefix if we need to. + + + + The first method we define on Prefix is + String. It returns a string representation + of a Prefix by joining the slice elements together with + spaces. We will use this method to generate keys when working with + the chain map. + + + + The Build method reads text from an io.Reader + and parses it into prefixes and suffixes that are stored in the + Chain. +

+ The io.Reader is an + interface type that is widely used by the standard library and + other Go code. Our code uses the + fmt.Fscan function, which + reads space-separated values from an io.Reader. +

+ The Build method returns once the Reader's + Read method returns os.EOF (end of file) + or some other read error occurs. +
+ + + This function does many small reads, which can be inefficient for some + Readers. For efficiency we wrap the provided + io.Reader with + bufio.NewReader to create a + new io.Reader that provides buffering. + + + + At the top of the function we make a Prefix slice + p using the Chain's prefixLen + field as its length. + We'll use this variable to hold the current prefix and mutate it with + each new word we encounter. + + + + In our loop we read words from the Reader into a + string variable s using + fmt.Fscan. Since Fscan uses space to + separate each input value, each call will yield just one word + (including punctuation), which is exactly what we need. +

+ Fscan returns an error if it encounters a read error + (os.EOF, for example) or if it can't scan the requested + value (in our case, a single string). In either case we just want to + stop scanning, so we break out of the loop. +
+ + + The word stored in s is a new suffix. We add the new + prefix/suffix combination to the chain map by computing + the map key with p.String and appending the suffix + to the slice stored under that key. +

+ The built-in append function appends elements to a slice + and allocates new storage when necessary. When the provided slice is + nil, append allocates a new slice. + This behavior conveniently ties in with the semantics of our map: + retrieving an unset key returns the zero value of the value type and + the zero value of []string is nil. + When our program encounters a new prefix (yielding a nil + value in the map) append will allocate a new slice. +

+ For more information about the append function and slices + in general see the + Slices: usage and internals article. +
+ + + Before reading the next word our algorithm requires us to drop the + first word from the prefix and push the current suffix onto the prefix. +

+ When in this state +
+p == Prefix{"I", "am"}
+s == "not" 
+ the new value for p would be +
+p == Prefix{"am", "not"}
+ This operation is also required during text generation so we put + the code to perform this mutation of the slice inside a method on + Prefix named Shift. +
+ + + The Shift method uses the built-in copy + function to copy the last len(p)-1 elements of p to + the start of the slice, effectively moving the elements + one index to the left (if you consider zero as the leftmost index). +
+p := Prefix{"I", "am"}
+copy(p, p[:1])
+// p == Prefix{"am", "am"}
+ We then assign the provided word to the last index + of the slice: +
+// suffix == "not"
+p[len(p)-1] = suffix
+// p == Prefix{"am", "not"}
+
+ + + The Generate method is similar to Build + except that instead of reading words from a Reader + and storing them in a map, it reads words from the map and + appends them to a slice (words). +

+ Generate uses a conditional for loop to generate + up to n words. +
+ + + At each iteration of the loop we retrieve a list of potential suffixes + for the current prefix. We access the chain map at key + p.String() and assign its contents to choices. +

+ If len(choices) is zero we break out of the loop as there + are no potential suffixes for that prefix. + This test also works if the key isn't present in the map at all: + in that case, choices will be nil and the + length of a nil slice is zero. +
+ + + To choose a suffix we use the + rand.Intn function. + It returns a random integer up to (but not including) the provided + value. Passing in len(choices) gives us a random index + into the full length of the list. +

+ We use that index to pick our new suffix, assign it to + next and append it to the words slice. +

+ Next, we Shift the new suffix onto the prefix just as + we did in the Build method. +
+ + + Before returning the generated text as a string, we use the + strings.Join function to join the elements of + the words slice together, separated by spaces. + + + + To make it easy to tweak the prefix and generated text lengths we + use the flag package to parse + command-line flags. +

+ These calls to flag.Int register new flags with the + flag package. The arguments to Int are the + flag name, its default value, and a description. The Int + function returns a pointer to an integer that will contain the + user-supplied value (or the default value if the flag was omitted on + the command-line). +
+ + + The main function begins by parsing the command-line + flags with flag.Parse and seeding the rand + package's random number generator with the current time. +

+ If the command-line flags provided by the user are invalid the + flag.Parse function will print an informative usage + message and terminate the program. +
+ + + To create the new Chain we call NewChain + with the value of the prefix flag. +

+ To build the chain we call Build with + os.Stdin (which implements io.Reader) so + that it will read its input from standard input. +
+ + + Finally, to generate text we call Generate with + the value of the words flag and assigning the result + to the variable text. +

+ Then we call fmt.Println to write the text to standard + output, followed by a carriage return. +
+ + + To use this program, first compile and link it. + If you are using 6g as your compiler, the command + would look something like this: +
+$ 6g markov.go && 6l -o markov markov.6
+ And then execute it while piping in some input text: +
+$ echo "a man a plan a canal panama" | ./markov -prefix=1
+a plan a man a plan a canal panama
+	
+ Here's a transcript of generating some text using the Go distribution's + README file as source material: +
+$ ./markov -words=10 < $GOROOT/go/README
+This is the source code repository for the Go source
+$ ./markov -prefix=1 -words=10 < $GOROOT/go/README
+This is the go directory (the one containing this README).
+$ ./markov -prefix=1 -words=10 < $GOROOT/go/README
+This is the variable if you have just untarred a
+
+ + + The Generate function does a lot of allocations when it + builds the words slice. As an exercise, modify it to + take an io.Writer to which it incrementally writes the + generated text with Fprint. + Aside from being more efficient this makes Generate + more symmetrical to Build. + + +
diff --git a/doc/codewalk/pig.go b/doc/codewalk/pig.go new file mode 100644 index 000000000..9e415f589 --- /dev/null +++ b/doc/codewalk/pig.go @@ -0,0 +1,124 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package main + +import ( + "fmt" + "rand" +) + +const ( + win = 100 // The winning score in a game of Pig + gamesPerSeries = 10 // The number of games per series to simulate +) + +// A score includes scores accumulated in previous turns for each player, +// as well as the points scored by the current player in this turn. +type score struct { + player, opponent, thisTurn int +} + +// An action transitions stochastically to a resulting score. +type action func(current score) (result score, turnIsOver bool) + +// roll returns the (result, turnIsOver) outcome of simulating a die roll. +// If the roll value is 1, then thisTurn score is abandoned, and the players' +// roles swap. Otherwise, the roll value is added to thisTurn. +func roll(s score) (score, bool) { + outcome := rand.Intn(6) + 1 // A random int in [1, 6] + if outcome == 1 { + return score{s.opponent, s.player, 0}, true + } + return score{s.player, s.opponent, outcome + s.thisTurn}, false +} + +// stay returns the (result, turnIsOver) outcome of staying. +// thisTurn score is added to the player's score, and the players' roles swap. +func stay(s score) (score, bool) { + return score{s.opponent, s.player + s.thisTurn, 0}, true +} + +// A strategy chooses an action for any given score. +type strategy func(score) action + +// stayAtK returns a strategy that rolls until thisTurn is at least k, then stays. +func stayAtK(k int) strategy { + return func(s score) action { + if s.thisTurn >= k { + return stay + } + return roll + } +} + +// play simulates a Pig game and returns the winner (0 or 1). +func play(strategy0, strategy1 strategy) int { + strategies := []strategy{strategy0, strategy1} + var s score + var turnIsOver bool + currentPlayer := rand.Intn(2) // Randomly decide who plays first + for s.player+s.thisTurn < win { + action := strategies[currentPlayer](s) + if action != roll && action != stay { + panic(fmt.Sprintf("Player %d is cheating", currentPlayer)) + } + s, turnIsOver = action(s) + if turnIsOver { + currentPlayer = (currentPlayer + 1) % 2 + } + } + return currentPlayer +} + +// roundRobin simulates a series of games between every pair of strategies. +func roundRobin(strategies []strategy) ([]int, int) { + wins := make([]int, len(strategies)) + for i := 0; i < len(strategies); i++ { + for j := i + 1; j < len(strategies); j++ { + for k := 0; k < gamesPerSeries; k++ { + winner := play(strategies[i], strategies[j]) + if winner == 0 { + wins[i]++ + } else { + wins[j]++ + } + } + } + } + gamesPerStrategy := gamesPerSeries * (len(strategies) - 1) // no self play + return wins, gamesPerStrategy +} + +// ratioString takes a list of integer values and returns a string that lists +// each value and its percentage of the sum of all values. +// e.g., ratios(1, 2, 3) = "1/6 (16.7%), 2/6 (33.3%), 3/6 (50.0%)" +func ratioString(vals ...int) string { + total := 0 + for _, val := range vals { + total += val + } + s := "" + for _, val := range vals { + if s != "" { + s += ", " + } + pct := 100 * float64(val) / float64(total) + s += fmt.Sprintf("%d/%d (%0.1f%%)", val, total, pct) + } + return s +} + +func main() { + strategies := make([]strategy, win) + for k := range strategies { + strategies[k] = stayAtK(k + 1) + } + wins, games := roundRobin(strategies) + + for k := range strategies { + fmt.Printf("Wins, losses staying at k =% 4d: %s\n", + k+1, ratioString(wins[k], games-wins[k])) + } +} diff --git a/doc/codewalk/popout.png b/doc/codewalk/popout.png new file mode 100644 index 000000000..9c0c23638 Binary files /dev/null and b/doc/codewalk/popout.png differ diff --git a/doc/codewalk/sharemem.xml b/doc/codewalk/sharemem.xml new file mode 100644 index 000000000..1a669f7b5 --- /dev/null +++ b/doc/codewalk/sharemem.xml @@ -0,0 +1,181 @@ + + + +Go's approach to concurrency differs from the traditional use of +threads and shared memory. Philosophically, it can be summarized: +

+Don't communicate by sharing memory; share memory by communicating. +

+Channels allow you to pass references to data structures between goroutines. +If you consider this as passing around ownership of the data (the ability to +read and write it), they become a powerful and expressive synchronization +mechanism. +

+In this codewalk we will look at a simple program that polls a list of +URLs, checking their HTTP response codes and periodically printing their state. +
+ + +The State type represents the state of a URL. +

+The Pollers send State values to the StateMonitor, +which maintains a map of the current state of each URL. +
+ + +A Resource represents the state of a URL to be polled: the URL itself +and the number of errors encountered since the last successful poll. +

+When the program starts, it allocates one Resource for each URL. +The main goroutine and the Poller goroutines send the Resources to +each other on channels. +
+ + +Each Poller receives Resource pointers from an input channel. +In this program, the convention is that sending a Resource pointer on +a channel passes ownership of the underlying data from the sender +to the receiver. Because of this convention, we know that +no two goroutines will access this Resource at the same time. +This means we don't have to worry about locking to prevent concurrent +access to these data structures. +

+The Poller processes the Resource by calling its Poll method. +

+It sends a State value to the status channel, to inform the StateMonitor +of the result of the Poll. +

+Finally, it sends the Resource pointer to the out channel. This can be +interpreted as the Poller saying "I'm done with this Resource" and +returning ownership of it to the main goroutine. +

+Several goroutines run Pollers, processing Resources in parallel. +
+ + +The Poll method (of the Resource type) performs an HTTP HEAD request +for the Resource's URL and returns the HTTP response's status code. +If an error occurs, Poll logs the message to standard error and returns the +error string instead. + + + +The main function starts the Poller and StateMonitor goroutines +and then loops passing completed Resources back to the pending +channel after appropriate delays. + + + +First, main makes two channels of *Resource, pending and complete. +

+Inside main, a new goroutine sends one Resource per URL to pending +and the main goroutine receives completed Resources from complete. +

+The pending and complete channels are passed to each of the Poller +goroutines, within which they are known as in and out. +
+ + +StateMonitor will initialize and launch a goroutine that stores the state +of each Resource. We will look at this function in detail later. +

+For now, the important thing to note is that it returns a channel of State, +which is saved as status and passed to the Poller goroutines. +
+ + +Now that it has the necessary channels, main launches a number of +Poller goroutines, passing the channels as arguments. +The channels provide the means of communication between the main, Poller, and +StateMonitor goroutines. + + + +To add the initial work to the system, main starts a new goroutine +that allocates and sends one Resource per URL to pending. +

+The new goroutine is necessary because unbuffered channel sends and +receives are synchronous. That means these channel sends will block until +the Pollers are ready to read from pending. +

+Were these sends performed in the main goroutine with fewer Pollers than +channel sends, the program would reach a deadlock situation, because +main would not yet be receiving from complete. +

+Exercise for the reader: modify this part of the program to read a list of +URLs from a file. (You may want to move this goroutine into its own +named function.) +
+ + +When a Poller is done with a Resource, it sends it on the complete channel. +This loop receives those Resource pointers from complete. +For each received Resource, it starts a new goroutine calling +the Resource's Sleep method. Using a new goroutine for each +ensures that the sleeps can happen in parallel. +

+Note that any single Resource pointer may only be sent on either pending or +complete at any one time. This ensures that a Resource is either being +handled by a Poller goroutine or sleeping, but never both simultaneously. +In this way, we share our Resource data by communicating. +
+ + +Sleep calls time.Sleep to pause before sending the Resource to done. +The pause will either be of a fixed length (pollInterval) plus an +additional delay proportional to the number of sequential errors (r.errCount). +

+This is an example of a typical Go idiom: a function intended to run inside +a goroutine takes a channel, upon which it sends its return value +(or other indication of completed state). +
+ + +The StateMonitor receives State values on a channel and periodically +outputs the state of all Resources being polled by the program. + + + +The variable updates is a channel of State, on which the Poller goroutines +send State values. +

+This channel is returned by the function. +
+ + +The variable urlStatus is a map of URLs to their most recent status. + + + +A time.Ticker is an object that repeatedly sends a value on a channel at a +specified interval. +

+In this case, ticker triggers the printing of the current state to +standard output every updateInterval nanoseconds. +
+ + +StateMonitor will loop forever, selecting on two channels: +ticker.C and update. The select statement blocks until one of its +communications is ready to proceed. +

+When StateMonitor receives a tick from ticker.C, it calls logState to +print the current state. When it receives a State update from updates, +it records the new status in the urlStatus map. +

+Notice that this goroutine owns the urlStatus data structure, +ensuring that it can only be accessed sequentially. +This prevents memory corruption issues that might arise from parallel reads +and/or writes to a shared map. +
+ + +In this codewalk we have explored a simple example of using Go's concurrency +primitives to share memory through commmunication. +

+This should provide a starting point from which to explore the ways in which +goroutines and channels can be used to write expressive and concise concurrent +programs. +
+ +
diff --git a/doc/codewalk/urlpoll.go b/doc/codewalk/urlpoll.go new file mode 100644 index 000000000..b51be9502 --- /dev/null +++ b/doc/codewalk/urlpoll.go @@ -0,0 +1,117 @@ +// Copyright 2010 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package main + +import ( + "http" + "log" + "time" +) + +const ( + numPollers = 2 // number of Poller goroutines to launch + second = 1e9 // one second is 1e9 nanoseconds + pollInterval = 60 * second // how often to poll each URL + statusInterval = 10 * second // how often to log status to stdout + errTimeout = 10 * second // back-off timeout on error +) + +var urls = []string{ + "http://www.google.com/", + "http://golang.org/", + "http://blog.golang.org/", +} + +// State represents the last-known state of a URL. +type State struct { + url string + status string +} + +// StateMonitor maintains a map that stores the state of the URLs being +// polled, and prints the current state every updateInterval nanoseconds. +// It returns a chan State to which resource state should be sent. +func StateMonitor(updateInterval int64) chan<- State { + updates := make(chan State) + urlStatus := make(map[string]string) + ticker := time.NewTicker(updateInterval) + go func() { + for { + select { + case <-ticker.C: + logState(urlStatus) + case s := <-updates: + urlStatus[s.url] = s.status + } + } + }() + return updates +} + +// logState prints a state map. +func logState(s map[string]string) { + log.Println("Current state:") + for k, v := range s { + log.Printf(" %s %s", k, v) + } +} + +// Resource represents an HTTP URL to be polled by this program. +type Resource struct { + url string + errCount int64 +} + +// Poll executes an HTTP HEAD request for url +// and returns the HTTP status string or an error string. +func (r *Resource) Poll() string { + resp, err := http.Head(r.url) + if err != nil { + log.Println("Error", r.url, err) + r.errCount++ + return err.String() + } + r.errCount = 0 + return resp.Status +} + +// Sleep sleeps for an appropriate interval (dependant on error state) +// before sending the Resource to done. +func (r *Resource) Sleep(done chan *Resource) { + time.Sleep(pollInterval + errTimeout*r.errCount) + done <- r +} + +func Poller(in <-chan *Resource, out chan<- *Resource, status chan<- State) { + for r := range in { + s := r.Poll() + status <- State{r.url, s} + out <- r + } +} + +func main() { + // create our input and output channels + pending, complete := make(chan *Resource), make(chan *Resource) + + // launch the StateMonitor + status := StateMonitor(statusInterval) + + // launch some Poller goroutines + for i := 0; i < numPollers; i++ { + go Poller(pending, complete, status) + } + + // send some Resources to the pending queue + go func() { + for _, url := range urls { + pending <- &Resource{url: url} + } + }() + + for r := range complete { + go r.Sleep(pending) + } +} diff --git a/doc/community.html b/doc/community.html new file mode 100644 index 000000000..c3b16cacb --- /dev/null +++ b/doc/community.html @@ -0,0 +1,53 @@ + + +
+ +

The Go Community

+ +

Go Nuts Mailing List

+

The golang-nuts +mailing list is for general Go discussion.

+ +

Go Packages Dashboard

+

A list of the most popular goinstall'd +Go libraries.

+ +

Go Project Dashboard

+

A list of external Go projects including programs and libraries.

+ +

Go IRC Channel

+

#go-nuts on irc.freenode.net is the official Go IRC channel.

+ +

@go_nuts at Twitter

+

The Go project's official Twitter account.

+ +
+ +
+ +

Blogs

+ +

The Go Blog

+

+The Go project's official blog, maintained by the core Go developers. +

+ +

research!rsc

+

+Posts labelled 'Go' by Russ Cox, one of the core Go developers. +

+ +

Airs

+

+Posts labelled 'Programming' by Ian Lance Taylor, one of the core Go developers. +

+ +

nf.id.au

+

+Posts labelled 'Go' by Andrew Gerrand, one of the core Go developers. +

+ +
+ +
+ diff --git a/doc/contrib.html b/doc/contrib.html new file mode 100644 index 000000000..84d2cda6a --- /dev/null +++ b/doc/contrib.html @@ -0,0 +1,69 @@ + + +
+ +

How you can help

+ +

Reporting issues

+ +

+If you spot bugs, mistakes, or inconsistencies in the Go project's code or +documentation, please let us know by +filing a ticket +on our issue tracker. +(Of course, you should check it's not an existing issue before creating +a new one.) +

+ +

+We pride ourselves on being meticulous; no issue is too small. +

+ +

Contributing code

+ +

+Go is an open source project and we welcome contributions from the community. +

+

+To get started, read these contribution +guidelines for information on design, testing, and our code review process. +

+

+Check the tracker for +open issues that interest you. Those labeled +HelpWanted +are particularly in need of outside help. +

+ +
+ +
+ +

The Go Project

+ +

Build Status

+

View the status of Go builds across the supported operating +systems and architectures.

+ +

Roadmap

+

Features and ideas being developed or discussed by the Go team.

+ +

Release History

+

A summary of the changes between Go releases.

+ +

Weekly Snapshot History

+

A summary of the changes between weekly snapshots of Go.

+ +

Developer Mailing List

+

The golang-dev +mailing list is for discussing and reviewing code for the Go project.

+

For general discussion of Go programming, see golang-nuts.

+ +

Checkins Mailing List

+

A mailing list that receives a message summarizing each checkin to the Go repository.

+ +
+ +
+ diff --git a/doc/contribute.html b/doc/contribute.html new file mode 100644 index 000000000..c4b1ce2b5 --- /dev/null +++ b/doc/contribute.html @@ -0,0 +1,529 @@ + + +

Introduction

+ +

+This document explains how to contribute changes to the Go project. +It assumes you have installed Go using the +installation instructions and +have written and tested your code. +(Note that the gccgo frontend lives elsewhere; +see Contributing to gccgo.) +

+ +

Discuss your design

+ +

+The project welcomes submissions but please let everyone know what +you're working on if you want it to become part of the main repository. +

+ +

+Before undertaking to write something new for the Go project, send +mail to the mailing +list to discuss what you plan to do. This gives everyone a +chance to validate the design, helps prevent duplication of effort, +and ensures that the idea fits inside the goals for the language +and tools. It also guarantees that the design is sound before code +is written; the code review tool is not the place for high-level +discussions. +

+ +

+In short, send mail before you code. +And don't start the discussion by mailing a change list! +

+ +

Testing redux

+ +

+You've written and tested your code, but +before sending code out for review, run all the tests for the whole +tree to make sure the changes don't break other packages or programs: +

+ +
+cd $GOROOT/src
+./all.bash
+
+ +

+The final line printed by make all should be of the form: +

+ +
+N known bugs; 0 unexpected bugs
+
+ +

+The value of N varies over time, but the line must +say “0 unexpected bugs” and must not +add “test output differs.” +

+ + +

Code review

+ +

+Changes to Go must be reviewed before they are submitted, +no matter who makes the change. +(In exceptional cases, such as fixing a build, the review can +follow shortly after submitting.) +A Mercurial extension helps manage the code review process. +The extension is included in the Go source tree but needs +to be added to your Mercurial configuration. +

+ +

Caveat for Mercurial aficionados

+ +

+Using Mercurial with the code review extension is not the same +as using standard Mercurial. +

+ +

+The Go repository is maintained as a single line of reviewed changes; +we prefer to avoid the complexity of Mercurial's arbitrary change graph. +The code review extension helps here: its hg submit command +automatically checks for and warns about the local repository +being out of date compared to the remote one. +The hg submit command also verifies other +properties about the Go repository. +For example, +it checks that Go code being checked in is formatted in the standard style, +as defined by gofmt, +and it checks that the author of the code is properly recorded for +copyright purposes. +

+ +

+To help ensure changes are only created by hg submit, +the code review extension disables the standard hg commit +command. +

+ +

+Mercurial power users: if you prefer to use the Mercurial Queues extension, see +Using Mercurial Queues with Codereview. +

+ +

Configure the extension

+ +

Edit $GOROOT/.hg/hgrc to add:

+ +
+[extensions]
+codereview = YOUR_GO_ROOT/lib/codereview/codereview.py
+
+[ui]
+username = Your Name <you@server.dom>
+
+ +

Replace YOUR_GO_ROOT with the value of $GOROOT. +The Mercurial configuration file format does not allow environment variable substitution. +The username information will not be used unless +you are a committer (see below), but Mercurial complains if it is missing. +

+ +

Log in to the code review site.

+ +

+The code review server uses a Google Account to authenticate. +(If you can use the account to +sign in at google.com, +you can use it to sign in to the code review server. +The email address you use on the Code Review site +will be recorded in the Mercurial change log +and in the CONTRIBUTORS file. +You can create a Google Account +associated with any address where you receive email. +

+ +
+$ cd $GOROOT
+$ hg code-login
+Email (login for uploading to codereview.appspot.com): rsc@golang.org
+Password for rsc@golang.org:
+
+Saving authentication cookies to /Users/rsc/.codereview_upload_cookies_codereview.appspot.com
+
+ +

Configure your account settings.

+ +

Edit your code review settings. +Grab a nickname. +Many people prefer to set the Context option to +“Whole file” to see more context when reviewing changes. +

+ +

Once you have chosen a nickname in the settings page, others +can use that nickname as a shorthand for naming reviewers and the CC list. +For example, rsc is an alias for rsc@golang.org. +

+ +

Make a change

+ +

+The entire checked-out tree is writable. +If you need to edit files, just edit them: Mercurial will figure out which ones changed. +You do need to inform Mercurial of added, removed, copied, or renamed files, +by running +hg add, +hg rm, +hg cp, +or +hg mv. +

+ +

When you are ready to send a change out for review, run

+ +
+$ hg change
+
+ +

from any directory in your Go repository. +Mercurial will open a change description file in your editor. +(It uses the editor named by the $EDITOR environment variable, vi by default.) +The file will look like: +

+ +
+# Change list.
+# Lines beginning with # are ignored.
+# Multi-line values should be indented.
+
+Reviewer:
+CC:
+
+Description:
+	<enter description here>
+
+Files:
+	src/pkg/math/sin.go
+	src/pkg/math/tan.go
+	src/pkg/regexp/regexp.go
+
+ +

+The Reviewer line lists the reviewers assigned +to this change, and the CC line lists people to +notify about the change. +These can be code review nicknames or arbitrary email addresses. +Unless explicitly told otherwise, such as in the discussion leading +up to sending in the change list, set the +reviewer field to the +golang-dev@googlegroups.com +mailing list. +

+ +

+Replace “<enter description here>” +with a description of your change. +The first line of the change description is conventionally a one-line +summary of the change, prefixed by the primary affected package, +and is used as the subject for code review mail; the rest of the +description elaborates. +

+ +

+The Files section lists all the modified files +in your client. +It is best to keep unrelated changes in different change lists. +In this example, we can include just the changes to package math +by deleting the line mentioning regexp.go. +

+ +

+After editing, the template might now read: +

+ +
+# Change list.
+# Lines beginning with # are ignored.
+# Multi-line values should be indented.
+
+Reviewer: golang-dev@googlegroups.com
+CC: math-nuts@swtch.com
+
+Description:
+	math: improved Sin, Cos and Tan precision for very large arguments.
+
+	See Bimmler and Shaney, ``Extreme sinusoids,'' J. Math 3(14).
+	Fixes issue 159.
+
+Files:
+	src/pkg/math/sin.go
+	src/pkg/math/tan.go
+
+ +

+The special sentence “Fixes issue 159.” associates +the change with issue 159 in the Go issue tracker. +When this change is eventually submitted, the issue +tracker will automatically mark the issue as fixed. +

+ +

+Save the file and exit the editor.

+ +

+The code review server assigns your change an issue number and URL, +which hg change will print, something like: +

+ +
+CL created: http://codereview.appspot.com/99999
+
+ +

+If you need to re-edit the change description, +run hg change 99999. +

+ +

+You can see a list of your pending changes by running hg pending (hg p for short). +

+ + +

Synchronize your client

+ +

While you were working, others might have submitted changes +to the repository. To update your client, run

+ +
+$ hg sync
+
+ +

(For Mercurial fans, hg sync runs hg pull -u +but then also synchronizes the local change list state against the new data.)

+ +

+If files you were editing have changed, Mercurial does its best to merge the +remote changes into your local changes. It may leave some files to merge by hand. +

+ +

+For example, suppose you have edited flag_test.go but +someone else has committed an independent change. +When you run hg sync, you will get the (scary-looking) output +(emphasis added): + +

+$ hg sync
+adding changesets
+adding manifests
+adding file changes
+added 1 changeset with 2 changes to 2 files
+getting src/pkg/flag/flag.go
+couldn't find merge tool hgmerge
+merging src/pkg/flag/flag_test.go
+warning: conflicts during merge.
+merging src/pkg/flag/flag_test.go failed!
+1 file updated, 0 files merged, 0 files removed, 1 file unresolved
+use 'hg resolve' to retry unresolved file merges
+$
+
+ +

+The only important part in that transcript is the italicized line: +Mercurial failed to merge your changes with the independent change. +When this happens, Mercurial leaves both edits in the file, +marked by <<<<<<< and +>>>>>>>. +it is now your job to edit the file to combine them. +Continuing the example, searching for those strings in flag_test.go +might turn up: +

+ +
+	VisitAll(visitor);
+<<<<<<< local
+	if len(m) != 7 {
+=======
+	if len(m) != 8 {
+>>>>>>> other
+		t.Error("VisitAll misses some flags");
+
+ +

+Mercurial doesn't show it, but suppose the original text that both edits +started with was 6; you added 1 and the other change added 2, +so the correct answer might now be 9. First, edit the section +to remove the markers and leave the correct code: +

+ +
+	VisitAll(visitor);
+	if len(m) != 9 {
+		t.Error("VisitAll misses some flags");
+
+ +

+Then ask Mercurial to mark the conflict as resolved: +

+ +
+$ hg resolve -m flag_test.go
+
+ +

+If you had been editing the file, say for debugging, but do not +care to preserve your changes, you can run +hg revert flag_test.go to abandon your +changes, but you may still need to run +hg resolve -m to mark the conflict resolved. +

+ +

Mail the change for review

+ +

To send out a change for review, run hg mail using the change list number +assigned during hg change:

+ +
+$ hg mail 99999
+
+ +

You can add to the Reviewer: and CC: lines +using the -r or --cc options. +In the above example, we could have left the Reviewer and CC +lines blank and then run: +

+ +
+$ hg mail -r golang-dev@googlegroups.com --cc math-nuts@swtch.com 99999
+
+ +

to achieve the same effect.

+ +

Note that -r and --cc cannot be spelled --r or -cc.

+ + +

Reviewing code

+ +

+Running hg mail will send an email to you and the reviewers +asking them to visit the issue's URL and make coments on the change. +When done, the reviewer clicks “Publish and Mail comments” +to send comments back. +

+ + +

Revise and upload

+ +

You will probably revise your code in response to the reviewer comments. +When you have revised the code and are ready for another round of review, run +

+ +
+$ hg mail 99999
+
+ +

again to upload the latest copy and send mail asking the reviewers to please take another look +(PTAL). +You might also visit the code review web page and reply to the comments, +letting the reviewer know that you've addressed them or explain why you +haven't. When you're done replying, click “Publish and Mail comments” +to send the line-by-line replies and any other comments. +

+

+The reviewer can comment on the new copy, and the process repeats. +The reviewer approves the change by replying with a mail that says +LGTM: looks good to me. +

+ +

Submit the change after the review

+ +

+After the code has been LGTM'ed, it is time to submit +it to the Mercurial repository. +If you are a committer, you can run: +

+ +
+$ hg submit 99999
+
+ +

+This checks the change into the repository. +The change description will include a link to the code review, +and the code review will be updated with a link to the change +in the repository. +

+ +

+If your local copy of the repository is out of date, +hg submit +will refuse the change: +

+ +
+$ hg submit 99999
+local repository out of date; must sync before submit
+
+ +

+If you are not a committer, you cannot submit the change directly. +Instead, a committer, usually the reviewer who said LGTM, +will run: +

+ +
+$ hg clpatch 99999
+$ hg submit 99999
+
+ +

The clpatch command imports your change 99999 into +the committer's local Mercurial client, at which point the committer +can check or test the code more. +(Anyone can run clpatch to try a change that +has been uploaded to the code review server.) +The submit command submits the code. You will be listed as the +author, but the change message will also indicate who the committer was. +Your local client will notice that the change has been submitted +when you next run hg sync. +

+ + + + +

Files in the Go repository don't list author names, +both to avoid clutter and to avoid having to keep the lists up to date. +Instead, your name will appear in the Mercurial change log +and in the CONTRIBUTORS file +and perhaps the AUTHORS file. +

+ +

The CONTRIBUTORS file +defines who the Go contributors—the people—are; +the AUTHORS file, which defines +who “The Go Authors”—the copyright holders—are. +The Go developers at Google will update these files when submitting +your first change. +In order for them to do that, you need to have completed one of the +contributor license agreements: +

    +
  • +If you are the copyright holder, you will need to agree to +the individual +contributor license agreement, which can be completed online. +
  • +
  • +If your organization is the copyright holder, the organization +will need to agree to the corporate contributor license agreement. +(If the copyright holder for your code has already completed the +agreement in connection with another Google open source project, +it does not need to be completed again.) +
  • +
+ +

+This rigmarole needs to be done only for your first submission. +

+ +

Code that you contribute should use the standard copyright header:

+ +
+// Copyright 2011 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
diff --git a/doc/devel/index.html b/doc/devel/index.html new file mode 100644 index 000000000..ae405bf52 --- /dev/null +++ b/doc/devel/index.html @@ -0,0 +1,11 @@ + + + + diff --git a/doc/devel/release.html b/doc/devel/release.html new file mode 100644 index 000000000..458a116de --- /dev/null +++ b/doc/devel/release.html @@ -0,0 +1,436 @@ + + +

This page summarizes the changes between official stable releases of Go. +Between releases we issue less stable +weekly snapshots. +The weekly snapshot history contains more detail, +and the Mercurial change log +has full details.

+ +

To update to a specific release, use:

+ +
+hg pull
+hg update release.rNN
+
+ +

r60 (released 2011/09/07)

+ +

+The r60 release corresponds to +weekly.2011-08-17. +This section highlights the most significant changes in this release. +For a more detailed summary, see the +weekly release notes. +For complete information, see the +Mercurial change list. +

+ +

Language

+ +

+An "else" block is now required to have braces except if the body of the "else" +is another "if". Since gofmt always puts those braces in anyway, +gofmt-formatted programs will not be affected. +To fix other programs, run gofmt. +

+ +

Packages

+ +

+Package http's URL parsing and query escaping code +(such as ParseURL and URLEscape) has been moved to +the new url package, with several simplifications to +the names. Client code can be updated automatically with gofix. +

+ +

+Package image has had significant changes made to the +Pix field of struct types such as +image.RGBA and +image.NRGBA. +The image.Image interface type has not changed, +though, and you should not need to change your code if you don't explicitly +refer to Pix fields. For example, if you decode a number of images +using the image/jpeg package, compose them using +image/draw, and then encode the result using +image/png, then your code should still work as +before. +If your code does refer to Pix fields see the +weekly.2011-07-19 +snapshot notes for how to update your code. +

+ +

+Package template has been replaced with a new +templating package (formerly exp/template). The original template +package is still available as old/template. +The old/template package is deprecated and will be removed. +The Go tree has been updated to use the new template package. We encourage +users of the old template package to switch to the new one. Code that uses +template or exp/template will need to change its +import lines to "old/template" or "template", +respectively. +

+ +

Tools

+ +

+Goinstall now uses a new tag selection scheme. +When downloading or updating, goinstall looks for a tag or branch with the +"go." prefix that corresponds to the local Go version. For Go +release.r58 it looks for go.r58. For +weekly.2011-06-03 it looks for go.weekly.2011-06-03. +If the specific go.X tag or branch is not found, it chooses the +closest earlier version. If an appropriate tag or branch is found, goinstall +uses that version of the code. Otherwise it uses the default version selected +by the version control system. Library authors are encouraged to use the +appropriate tag or branch names in their repositories to make their libraries +more accessible. +

+ +

r59 (released 2011/08/01)

+ +

+The r59 release corresponds to +weekly.2011-07-07. +This section highlights the most significant changes in this release. +For a more detailed summary, see the +weekly release notes. +For complete information, see the +Mercurial change list. +

+ +

Language

+ +

+This release includes a language change that restricts the use of +goto. In essence, a goto statement outside a block +cannot jump to a label inside that block. Your code may require changes if it +uses goto. +See this +changeset for how the new rule affected the Go tree. +

+ +

Packages

+ +

+As usual, gofix will handle the bulk of the rewrites +necessary for these changes to package APIs. +

+ +

+Package http has a new +FileSystem interface that provides access +to files. The FileServer helper now takes a +FileSystem argument instead of an explicit file system root. By +implementing your own FileSystem you can use the +FileServer to serve arbitrary data. +

+ +

+Package os's ErrorString type has been +hidden. Most uses of os.ErrorString can be replaced with +os.NewError. +

+ +

+Package reflect supports a new struct tag scheme +that enables sharing of struct tags between multiple packages. +In this scheme, the tags must be of the form: +

+
+	`key:"value" key2:"value2"`
+
+

+The StructField type's Tag field now +has type StructTag, which has a +Get method. Clients of json and +xml will need to be updated. Code that says +

+
+	type T struct {
+		X int "name"
+	}
+
+

+should become +

+
+	type T struct {
+		X int `json:"name"`  // or `xml:"name"`
+	}
+
+

+Use govet to identify struct tags that need to be +changed to use the new syntax. +

+ +

+Package sort's IntArray type has been +renamed to IntSlice, and similarly for +Float64Slice and +StringSlice. +

+ +

+Package strings's Split function has +itself been split into Split and +SplitN. +SplitN is the same as the old Split. +The new Split is equivalent to SplitN with a final +argument of -1. +

+ +Package image/draw's +Draw function now takes an additional +argument, a compositing operator. +If in doubt, use draw.Over. +

+ +

Tools

+ +

+Goinstall now installs packages and commands from +arbitrary remote repositories (not just Google Code, Github, and so on). +See the goinstall documentation for details. +

+ +

r58 (released 2011/06/29)

+ +

+The r58 release corresponds to +weekly.2011-06-09 +with additional bug fixes. +This section highlights the most significant changes in this release. +For a more detailed summary, see the +weekly release notes. +For complete information, see the +Mercurial change list. +

+ +

Language

+ +

+This release fixes a use of uninitialized memory in programs that misuse goto. +

+ +

Packages

+ +

+As usual, gofix will handle the bulk of the rewrites +necessary for these changes to package APIs. +

+ +

+Package http drops the finalURL return +value from the Client.Get method. The value +is now available via the new Request field on http.Response. +Most instances of the type map[string][]string in have been +replaced with the new Values type. +

+ +

+Package exec has been redesigned with a more +convenient and succinct API. +

+ +

+Package strconv's Quote +function now escapes only those Unicode code points not classified as printable +by unicode.IsPrint. +Previously Quote would escape all non-ASCII characters. +This also affects the fmt package's "%q" +formatting directive. The previous quoting behavior is still available via +strconv's new QuoteToASCII function. +

+ +

+Package os/signal's +Signal and +UnixSignal types have been moved to the +os package. +

+ +

+Package image/draw is the new name for +exp/draw. The GUI-related code from exp/draw is now +located in the exp/gui package. +

+ +

Tools

+ +

+Goinstall now observes the GOPATH environment +variable to build and install your own code and external libraries outside of +the Go tree (and avoid writing Makefiles). +

+ + +

Minor revisions

+ +

r58.1 adds +build and +runtime +changes to make Go run on OS X 10.7 Lion. +

+ +

r57 (released 2011/05/03)

+ +

+The r57 release corresponds to +weekly.2011-04-27 +with additional bug fixes. +This section highlights the most significant changes in this release. +For a more detailed summary, see the +weekly release notes. +For complete information, see the +Mercurial change list. +

+ +

The new gofix tool finds Go programs that use old APIs and rewrites them to use +newer ones. After you update to a new Go release, gofix helps make the +necessary changes to your programs. Gofix will handle the http, os, and syscall +package changes described below, and we will update the program to keep up with +future changes to the libraries. +Gofix can’t +handle all situations perfectly, so read and test the changes it makes before +committing them. +See the gofix blog post for more +information.

+ +

Language

+ +

+Multiple assignment syntax replaces the closed function. +The syntax for channel +receives allows an optional second assigned value, a boolean value +indicating whether the channel is closed. This code: +

+ +
+	v := <-ch
+	if closed(ch) {
+		// channel is closed
+	}
+
+ +

should now be written as:

+ +
+	v, ok := <-ch
+	if !ok {
+		// channel is closed
+	}
+
+ +

Unused labels are now illegal, just as unused local variables are.

+ +

Packages

+ +

+Package gob will now encode and decode values of types that implement the +GobEncoder and +GobDecoder interfaces. This allows types with unexported +fields to transmit self-consistent descriptions; examples include +big.Int and big.Rat. +

+ +

+Package http has been redesigned. +For clients, there are new +Client and Transport +abstractions that give more control over HTTP details such as headers sent +and redirections followed. These abstractions make it easy to implement +custom clients that add functionality such as OAuth2. +For servers, ResponseWriter +has dropped its non-essential methods. +The Hijack and Flush methods are no longer required; +code can test for them by checking whether a specific value implements +Hijacker or Flusher. +The RemoteAddr and UsingTLS methods are replaced by Request's +RemoteAddr and TLS fields. +The SetHeader method is replaced by a Header method; +its result, of type Header, +implements Set and other methods. +

+ +

+Package net +drops the laddr argument from Dial +and drops the cname return value +from LookupHost. +The implementation now uses cgo to implement +network name lookups using the C library getaddrinfo(3) +function when possible. This ensures that Go and C programs +resolve names the same way and also avoids the OS X +application-level firewall. +

+ +

+Package os +introduces simplified Open +and Create functions. +The original Open is now available as OpenFile. +The final three arguments to StartProcess +have been replaced by a pointer to a ProcAttr. +

+ +

+Package reflect has been redesigned. +Type is now an interface that implements +all the possible type methods. +Instead of a type switch on a Type t, switch on t.Kind(). +Value is now a struct value that +implements all the possible value methods. +Instead of a type switch on a Value v, switch on v.Kind(). +Typeof and NewValue are now called TypeOf and ValueOf +To create a writable Value, use New(t).Elem() instead of Zero(t). +See the change description +for the full details. +The new API allows a more efficient implementation of Value +that avoids many of the allocations required by the previous API. +

+ +

+Remember that gofix will handle the bulk of the rewrites +necessary for these changes to package APIs. +

+ +

Tools

+ +

Gofix, a new command, is described above.

+ +

+Gotest is now a Go program instead of a shell script. +The new -test.short flag in combination with package testing's Short function +allows you to write tests that can be run in normal or “short” mode; +all.bash runs tests in short mode to reduce installation time. +The Makefiles know about the flag: use make testshort. +

+ +

+The run-time support now implements CPU and memory profiling. +Gotest's new +-test.cpuprofile and +-test.memprofile flags make it easy to +profile tests. +To add profiling to your web server, see the http/pprof +documentation. +For other uses, see the runtime/pprof documentation. +

+ +

Minor revisions

+ +

r57.1 fixes a nil pointer dereference in http.FormFile.

+

r57.2 fixes a use of uninitialized memory in programs that misuse goto.

+ +

r56 (released 2011/03/16)

+ +

+The r56 release was the first stable release and corresponds to +weekly.2011-03-07.1. +The numbering starts at 56 because before this release, +what we now consider weekly snapshots were called releases. +

diff --git a/doc/devel/roadmap.html b/doc/devel/roadmap.html new file mode 100644 index 000000000..d3c494715 --- /dev/null +++ b/doc/devel/roadmap.html @@ -0,0 +1,135 @@ + + +

Go Roadmap

+ +

+This page lists features and ideas being developed or discussed by the +Go team. This list will be updated as work continues. + +

+The roadmap should be discussed on +the golang-nuts +mailing list. + +

+Language roadmap

+ +

+This is a list of language changes that are being considered. +Appearance on this list is no guarantee that the change will be +accepted. + +

    +
  • +Possibly rewrite restriction on goto across variable declarations. +
  • +Variant types. A way to define a type as being the union of some set +of types. +
  • +Generics. An active topic of discussion. +
  • +Methods for operators, to allow a type to use arithmetic notation for +expressions. +
  • +Possibly allow top-level packages to be given names other than main. +
+ +

+Implementation roadmap

+ +
    +
  • +Improved garbage collector. +
  • +Debugger. +
  • +Improved implementation documentation. +
+ +

+Gc compiler roadmap

+ +
    +
  • +Implement goto restrictions. +
  • +Improved optimization. +
  • +Use escape analysis to keep more data on stack. +
+ +

+Gccgo compiler roadmap

+ +
    +
  • +Implement goto restrictions. +
  • +Use goroutines rather than threads. +
  • +Separate gcc interface from frontend proper. +
  • +Use escape analysis to keep more data on stack. +
+ +

+Tools roadmap

+ +
    +
  • +Strengthen goinstall until it can displace make for most builds. +
+ +

+Packages roadmap

+ +
    +
  • +Faster, RE2-like regular expressions. +
  • +Comprehensive support for international text. +
  • +Support for international dates, times, etc. +
  • +Support for multilingual messages. +
+ + +

Done

+ +
    +
  • +gc: Generate DWARF debug info. +
  • +gc: Provide gdb support for runtime facilities. +
  • +Safe compilation mode: generate code that is guaranteed not to obtain an invalid memory address other than via import "unsafe". +
  • +Gccgo: garbage collection. +
  • +SWIG support. +
  • +Simpler semicolon rules. +
  • +A more general definition of ... in parameter lists. +
  • +Explicit conversions from string +to []byte and []int. +
  • +A function that will be run by the garbage collector when an item is freed +(runtime.SetFinalizer). +
  • +Public continuous build and benchmark infrastructure (gobuilder). +
  • +Package manager (goinstall). +
  • +A means of recovering from a panic (recover). +
  • +5g: Better floating point support. +
  • +Improved CGO including some mechanism for calling back from C to Go. +
  • +Faster, allocation-light reflection. +
  • +App Engine support. +
diff --git a/doc/devel/weekly.html b/doc/devel/weekly.html new file mode 100644 index 000000000..d984d3b1b --- /dev/null +++ b/doc/devel/weekly.html @@ -0,0 +1,3970 @@ + + +

This page summarizes the changes between tagged weekly snapshots of Go. +For full details, see the Mercurial change log.

+ +

Weekly snapshots occur often and may not be stable. +If stability of API and code is more important than having the +latest features, use the official releases instead.

+ +

To update to a specific snapshot, use:

+ +
+hg pull
+hg update weekly.YYYY-MM-DD
+
+ +

2011-09-07

+ +
+This weekly snapshot consists of improvements and bug fixes, including fixes
+for issues introduced by escape analysis changes in the gc compiler.
+
+* build: clear execute bit from Go files (thanks Mike Rosset),
+	error out if problem with sudo.bash /usr/local/bin (thanks Mike Rosset).
+* exp/norm: add Reader and Writer,
+	performance improvements of quickSpan.
+* exp/regexp: bug fixes and RE2 tests.
+* exp/template/html: string replacement refactoring,
+	tweaks to js{,_test}.go.
+* gc: add -p flag to catch import cycles earlier,
+	fix label recursion bugs,
+	fix zero-length struct eval,
+	zero stack-allocated slice backing arrays,
+* gc, ld: fix Windows file paths (thanks Hector Chu).
+* go/parser: accept corner cases of signature syntax.
+* gobuilder: ignore _test.go files when looking for docs, more logging.
+* godoc: minor tweaks for App Engine use.
+* gofix: do not convert url in field names (thanks Gustavo Niemeyer).
+* gofmt: indent multi-line signatures.
+* gopprof: regexp fixes (thanks Hector Chu).
+* image/png: check zlib checksum during Decode.
+* libmach: fix incorrect use of memset (thanks Dave Cheney).
+* misc/goplay: fix template output.
+* net: ParseCIDR returns IPNet instead of IPMask (thanks Mikio Hara),
+	sync CIDRMask code, doc.
+* os: use GetFileAttributesEx to implement Stat on windows (thanks Alex Brainman).
+* runtime: fix openbsd 386 raisesigpipe,
+	implement exception handling on windows/amd64 (thanks Hector Chu),
+	test for concurrent channel consumers (thanks Christopher Wedgwood).
+* sort: use heapsort to bail out quicksort (thanks Ziad Hatahet).
+* sync/atomic: add LoadUintptr, add Store functions.
+* syscall: update routing message attributes handling (thanks Mikio Hara).
+* template: fix deadlock,
+	indirect or dereference function arguments if necessary,
+	slightly simplify the test for assignability of arguments.
+* url: handle ; in ParseQuery.
+* websocket: fix incorrect prints found by govet (thanks Robert Hencke).
+
+ +

2011-09-01

+ +
+This weekly contains performance improvements and bug fixes.
+
+The gc compiler now does escape analysis, which improves program performance
+by placing variables on the call stack instead of the heap when it is safe to
+do so.
+
+The container/vector package is deprecated and will be removed at some point
+in the future.
+
+Other changes:
+* archive/tar: support symlinks. (thanks Mike Rosset)
+* big: fix nat.scan bug. (thanks Evan Shaw)
+* bufio: handle a "\r\n" that straddles the buffer.
+	add openbsd.
+	avoid redundant bss declarations.
+	fix unused parameters.
+	fix windows/amd64 build with newest mingw-w64. (thanks Hector Chu)
+* bytes: clarify that NewBuffer is not for beginners.
+* cgo: explain how to free something.
+	fix GoBytes. (thanks Gustavo Niemeyer)
+	fixes callback for windows amd64. (thanks Wei Guangjing)
+	note that CString result must be freed. (thanks Gustavo Niemeyer)
+* cov: remove tautological #defines. (thanks Lucio De Re)
+* dashboard: yet another utf-8 fix.
+* doc/codelab/wiki: fix Makefile.
+* doc/progs: fix windows/amd64. (thanks Jaroslavas Počepko)
+* doc/tmpltohtml: update to new template package.
+* doc: emphasize that environment variables are optional.
+* effective_go: convert to use tmpltohtml.
+* exp/norm: reduced the size of the byte buffer used by reorderBuffer by half by reusing space when combining.
+	a few minor fixes to support the implementation of norm.
+	added implementation for []byte versions of methods.
+* exp/template/html: add some tests for ">" attributes.
+	added handling for URL attributes.
+	differentiate URL-valued attributes (such as href).
+	reworked escapeText to recognize attr boundaries.
+* exp/wingui: made compatible with windows/amd64. (thanks Jaroslavas Počepko)
+* flag: add Parsed, restore Usage.
+* gc: add openbsd.
+	escape analysis.
+	fix build on Plan 9. (thanks Lucio De Re)
+	fix div bug.
+	fix pc/line table. (thanks Julian Phillips)
+	fix some spurious leaks.
+	make static initialization more static.
+	remove JCXZ; add JCXZW, JCXZL, and JCXZQ instructions. (thanks Jaroslavas Počepko)
+	shuffle #includes.
+	simplify escape analysis recursion.
+	tweak and enable escape analysis.
+* go/ast cleanup: base File/PackageExports on FilterFile/FilterPackage code.
+	adjustments to filter function.
+	fix ast.MergePackageFiles to collect infos about imports. (thanks Sebastien Binet)
+	generalize ast.FilterFile.
+* go/build: add test support & use in gotest.
+	separate test imports out when scanning. (thanks Gustavo Niemeyer)
+* go/parser: fix type switch scoping.
+	fix type switch scoping.
+* gob: explain that Debug isn't useful unless it's compiled in.
+* gobuilder: increase log limit.
+* godashboard: fix utf-8 in user names.
+* godoc: first step towards reducing index size.
+	add dummy playground.js to silence godoc warning at start-up.
+	added systematic throttling to indexing goroutine.
+	fix bug in zip.go.
+	support for reading/writing (splitted) index files.
+	use virtual file system when generating package synopses.
+* gofix: forgot to rename the URL type.
+	osopen: fixed=true when changing O_CREAT. (thanks Tarmigan Casebolt)
+* goinstall: error out with paths that end with '/'. (thanks Tarmigan Casebolt)
+	report lack of $GOPATH on errors. (thanks Gustavo Niemeyer)
+	select the tag that is closest to runtime.Version.
+* gotry: add missing $. (thanks Tarmigan Casebolt)
+* http: add MaxBytesReader to limit request body size.
+	add file protocol transport.
+	adjust test threshold for larger suse buffers.
+	delete error kludge.
+	on invalid request, send 400 response.
+	return 413 instead of 400 when the request body is too large. (thanks Dave Cheney)
+	support setting Transport's TLS client config.
+* image/tiff: add a decode benchmark. (thanks Benny Siegert)
+	decoder optimization. (thanks Benny Siegert)
+* image: add PalettedImage interface, and make image/png recognize it. (thanks Jaroslavas Počepko)
+* io: add TeeReader. (thanks Hector Chu)
+* json: add struct tag option to wrap literals in strings.
+	calculate Offset for Indent correctly. (thanks Jeff Hodges)
+	fix decode bug with struct tag names with ,opts being ignored.
+* ld: handle Plan 9 ar format. (thanks Lucio De Re)
+	remove duplicate bss definitions.
+* libmach: support reading symbols from Windows .exe for nm. (thanks Mateusz Czapliński)
+* math: fix Pow10 loop. (thanks Volker Dobler)
+* mime: ParseMediaType returns os.Error now, not a nil map.
+	media type formatter. (thanks Pascal S. de Kloe)
+	text charset defaults. (thanks Pascal S. de Kloe)
+* misc/dashboard: remove limit for json package list.
+* misc/emacs: refine label detection.
+* net: add ParseMAC function. (thanks Paul Borman)
+	change the internal form of IPMask for IPv4. (thanks Mikio Hara)
+	disable "tcp" test on openbsd.
+	fix windows build. (thanks Alex Brainman)
+	join and leave a IPv6 group address, on a specific interface. (thanks Mikio Hara)
+	make use of IPv4len, IPv6len. (thanks Mikio Hara)
+	move internal string manipulation routines to parse.go. (thanks Mikio Hara)
+* os: disable Hostname test on OpenBSD.
+	fix WNOHANG Waitmsg. (thanks Gustavo Niemeyer)
+* reflect: add Value.Bytes, Value.SetBytes methods.
+* rpc: add benchmark for async rpc calls.
+* runtime: add openbsd 386 defs.h.
+	add runtime support for openbsd 386.
+	add runtime· prefix to showframe.
+	ctrlhandler for windows amd64. (thanks Wei Guangjing)
+	fix stack cleanup on windows/amd64. (thanks Hector Chu)
+	fix void warnings.
+	go interface to cdecl calbacks. (thanks Jaroslavas Počepko)
+	handle string + char literals in goc2c.
+	make arm work on Ubuntu Natty qemu.
+	openbsd thread tweaks.
+	simplify stack traces.
+	speed up cgo calls. (thanks Alex Brainman)
+	use cgo runtime functions to call windows syscalls. (thanks Alex Brainman)
+	windows/amd64 callbacks fixed and syscall fixed to allow using it in callbacks. (thanks Jaroslavas Počepko)
+* strconv: put decimal on stack.
+* spec: update section on Implementation Differences.
+* syscall: SOMAXCONN should be 0x7fffffff at winsock2. (thanks Yasuhiro Matsumoto)
+	add openbsd 386.
+	handle RTM_NEWROUTE in ParseNetlinkRouteAttr on Linux. (thanks Albert Strasheim)
+	handle routing entry in ParseRoutingSockaddr on BSD variants. (thanks Mikio Hara)
+	openbsd amd64 syscall support.
+	use the vdso page on linux x86 for faster syscalls instead of int $0x80. (thanks Yuval Pavel Zholkover)
+* template/parse: give if, range, and with a common representation.
+* template: grammar fix for template documentation. (thanks Bill Neubauer)
+	range over channel.
+	remove else and end nodes from public view.
+* test: put GOROOT/bin before all others in run.
+* time: fix Plan 9 build. (thanks Fazlul Shahriar)
+	fix zone during windows test.
+* type switches: test for pathological case.
+* version.bash: update VERSION on -save if already present. (thanks Gustavo Niemeyer)
+* websocket: implements new version of WebSocket protocol. (thanks Fumitoshi Ukai)
+* windows/386: clean stack after syscall. (thanks Jaroslavas Počepko)
+* xml: marshal "parent>child" tags correctly. (thanks Ross Light)
+
+ +

2011-08-17 (base for r60)

+ +
+This weekly contains some package re-shuffling. Users of the http and
+template packages may be affected.
+
+This weekly replaces the template package with exp/template.
+The original template package is still available as old/template.
+The old/template package is deprecated and will be removed at some point
+in the future. The Go tree has been updated to use the new template package.
+We encourage users of the old template package to switch to the new one.
+Code that uses template or exp/template will need to change
+its import lines to "old/template" or "template", respectively.
+
+The http package's URL parsing and query escaping code (such as ParseURL and
+URLEscape) has been moved to the new url package, with several simplifications
+to the names. Client code can be updated automatically with gofix.
+
+* asn1: support unmarshalling structs with int32 members (thanks Dave Cheney).
+* build: allow builds without cgo or hg,
+	support versioning without hg (thanks Gustavo Niemeyer).
+* builtin: add documentation for builtins.
+* cgo: omit duplicate symbols in writeDefs (thanks Julian Phillips).
+* misc: add support for OpenBSD.
+* doc/codewalk: new Markov chain codewalk.
+* exp/norm: added trie lookup code and associated tests,
+	generate trie struct in triegen.go for better encapsulation,
+	implementation of decomposition and composing functionality.
+* exp/template/html: new experimental package for auto-escaping HTML templates.
+* exp/template: don't panic on range of nil interface,
+	rename Parse*File and Parse*Files for clarity,
+	support field syntax on maps (thanks Gustavo Niemeyer), and
+	many other fixes and changes.
+* gc: implement nil chan and nil map support.
+* go/parser: range clause and type literal fixes.
+* godoc: show all top-level decls for (fake) package builtin.
+* goinstall: really report all newly-installed public packages.
+* html: parse more malformed tags.
+* http: fix ParseMultipartForm after MultipartReader error,
+	fix side effects in DefaultTransport's RoundTrip method (thanks Dave Grijalva).
+* json: fix []unmarshaler case.
+* ld: make addaddrplus4 static (thanks Lucio De Re).
+* syscall: move multicast address handling to the net package.
+* net: Plan 9 support (thanks Fazlul Shahriar),
+	add SetTimeout to Listener interface (thanks Aleksandar Dezelin),
+	add multicast stubs for OpenBSD,
+	return correct local address for an accepted TCP connection (thanks Mikio Hara).
+* reflect: panic on Invalid Interface call (thanks Gustavo Niemeyer).
+* rpc: implement ServeRequest to synchronously serve a single request,
+	make Server.Mutex unexported.
+* runtime: better checks for syscall.NewCallback parameter (thanks Alex Brainman),
+	correct SEH installation during callbacks (thanks Alex Brainman),
+	fix GC bitmap corruption,
+	fix pseudo-randomness on some selects (thanks Gustavo Niemeyer).
+* syscall: make LazyDLL/LazyProc.Mutex unexported.
+* test: allow multiple patterns in errchk,
+	new nil semantics.
+* time: take fractional seconds even if not in the format string.
+* url: new package.
+* utf8: rename some internal constants to remove leading underscores.
+* xml: escape string chardata in xml.Marshal.
+
+ +

2011-08-10

+ +
+This weekly contains performance improvements and bug fixes.
+
+There are no outward-facing changes, but imports of the old-style
+container/vector package have also been removed from the core library (thanks
+John Asmuth, Kyle Consalus).
+
+Other changes:
+
+* 5g: fix set but not used error (thanks Dave Cheney).
+* cmd/ld: Corrected mismatched print formats and variables (thanks Lucio De Re).
+* errchk: add -0 flag.
+* exp/norm: fix build by adding a test placeholder,
+	maketables tool for generating tables for normalization.
+* exp/template: bug fixes,
+	ensure that a valid Set is returned even on error (thanks Roger Peppe),
+	make index on maps return zero when key not present (thanks Roger Peppe),
+	split the parse tree into a separate package exp/template/parse,
+	add url query formatting filter.
+* faq: lots of small tweaks plus a couple of new discussions,
+	variant types, unions.
+* fmt: call UpdateMemStats in malloc counter.
+* go/build: use GOBIN as binary path for GOROOT.
+* gob: add UpdateMemStats calls to malloc counter,
+	avoid a couple of init-time allocations,
+	don't invoke GobEncoder on zero values.
+* gofmt: update test script so 'make test' succeeds.
+* html: parse doctype tokens; merge adjacent text nodes.
+* http: add more MPEG-4 MIME types to sniffer, and disable MP4 sniffing,
+	add test to serve content in index.html (thanks Yasuhiro Matsumoto),
+	configurable and default request header size limit,
+	correct format flags when printing errors in tests (thanks Alex Brainman),
+	correct path to serve index.html (thanks Yasuhiro Matsumoto),
+* ld: add one empty symbol into pe to make dumpbin works (thanks Wei Guangjing),
+	fail linking if the top-level package is not main.
+* misc/vim: godoc command (thanks Yasuhiro Matsumoto).
+* net: add support for openbsd (thanks Joel Sing),
+	fix /proc/net/igmp,igmp6 reading bug on linux (thanks Mikio Hara),
+	implement windows LookupMX and LookupAddr (thanks Mikio Hara),
+	sort SRV records before returning from LookupSRV (thanks Alex Brainman),
+* os: add support for openbsd (thanks Joel Sing).
+* runtime: add more specialized type algorithms,
+	correct Note documentation,
+	faster chan creation on Linux/FreeBSD/Plan9,
+	openbsd amd64 runtime support (thanks Joel Sing),
+	remove unnecessary locking (thanks Hector Chu).
+* scanner: correct error position for illegal UTF-8 encodings.
+* syscall: delay load of dll functions on Windows (thanks Alex Brainman),
+	move BSD mmap syscall (thanks Joel Sing),
+	update routing message support for BSD variants (thanks Mikio Hara).
+* test/bench: note changes after recent improvements to locking and runtime.
+* time: add nanoseconds to the Time structure,
+	parse and format fractional seconds.
+
+ +

2011-07-29

+ +
+This weekly contains performance improvements and many bug fixes.
+
+* 6l: OpenBSD support.
+* archive/zip: handle zip files with more than 65535 files,
+	more efficient reader and bug fix.
+* big: refine printf formatting and optimize string conversion.
+* build: fixes for mingw-w64 (thanks Wei Guangjing),
+	miscellaneous fixes.
+* cgo: add GoBytes, fix gmp example.
+* exp/norm: API for normalization library.
+* exp/regexp: implement regexp API using exp/regexp/syntax.
+* exp/template: more tweaks and fixes, convert the tree to use exp/template.
+* fmt: handle precision 0 format strings in standard way.
+* gc: a raft of bug fixes.
+* go/parser: report illegal label declarations at ':'.
+* gob: send empty but non-nil maps.
+* godoc: allow form feed in text files,
+	app engine configuration and updated documentation.
+* goinstall: abort and warn when using any url scheme, not just 'http://',
+	write to goinstall.log in respective GOPATH.
+* html: handle character entities without semicolons (thanks Andrew Balholm),
+	parse misnested formatting tags according to the HTML5 spec,
+	sync html/testdata/webkit with upstream WebKit.
+* http: content-type sniffing,
+	make serveFile redirects relative (thanks Andrew Balholm),
+	other fixes.
+* image/tiff: Do not panic when RowsPerStrip is missing (thanks Benny Siegert).
+* io/ioutil: improve performance of ioutil.Discard (thanks Mike Solomon).
+* ld: detect all import cycles,
+	ldpe fixes (thanks Wei Guangjing),
+	remove cseekend and redo pe writing (thanks Alex Brainman),
+	remove overlap of ELF sections on dynamic binaries (thanks Gustavo Niemeyer).
+* net/textproto: avoid 1 copy in ReadLine, ReadContinuedLine.
+* net: fix memory corruption in windows *netFD.ReadFrom (thanks Alex Brainman).
+* runtime: faster entersyscall/exitsyscall,
+	fix scheduler races (thanks Hector Chu),
+	higher goroutine arg limit, clearer error,
+	parallelism-related performance optimizations and fixes,
+	replace byte-at-a-time zeroing loop with memclr (thanks Quan Yong Zhai).
+* sort: fix Float64Slice sort; NaN smallest value (thanks Florian Uekermann).
+* src: removed some uses of container/vector (thanks John Asmuth).
+* sync: improve Once fast path.
+* unicode: fix case-mapping for roman numerals.
+
+ +

2011-07-19

+ +
+This weekly snapshot includes a language change and a change to the image
+package that may require changes to client code.
+
+The language change is that an "else" block is now required to have braces
+except if the body of the "else" is another "if". Since gofmt always puts those
+braces in anyway, programs will not be affected unless they contain "else for",
+"else switch", or "else select". Run gofmt to fix any such programs.
+
+The image package has had significant changes made to the Pix field of struct
+types such as image.RGBA and image.NRGBA. The image.Image interface type has
+not changed, though, and you should not need to change your code if you don't
+explicitly refer to Pix fields. For example, if you decode a number of images
+using the image/jpeg package, compose them using image/draw, and then encode
+the result using image/png, then your code should still work as before.
+
+If you do explicitly refer to Pix fields, there are two changes.  First, Pix[0]
+now refers to the pixel at Bounds().Min instead of the pixel at (0, 0). Second,
+the element type of the Pix slice is now uint8 instead of image.FooColor. For
+example, for an image.RGBA, the channel values will be packed R, G, B, A, R, G,
+B, A, etc. For 16-bits-per-channel color types, the pixel data will be stored
+as big-endian uint8s.
+
+Most Pix field types have changed, and so if your code still compiles after
+this change, then you probably don't need to make any further changes (unless
+you use an image.Paletted's Pix field). If you do get compiler errors, code
+that used to look like this:
+
+	// Get the R, G, B, A values for the pixel at (x, y).
+	var m *image.RGBA = loadAnImage()
+	c := m.Pix[y*m.Stride + x]
+	r, g, b, a := c.R, c.G, c.B, c.A
+
+should now look like this:
+
+	// Get the R, G, B, A values for the pixel at (x, y).
+	var m *image.RGBA = loadAnImage()
+	i := (y-m.Rect.Min.Y)*m.Stride + (x-m.Rect.Min.X)*4
+	r := m.Pix[i+0]
+	g := m.Pix[i+1]
+	b := m.Pix[i+2]
+	a := m.Pix[i+3]
+
+This image package change will not be fixed by gofix: how best to translate
+code into something efficient and idiomatic depends on the surrounding context,
+and is not easily automatable. Examples of what to do can be found in the
+changes to image/draw/draw.go in http://codereview.appspot.com/4675076/
+
+Other changes:
+* 6l: change default output name to 6.out.exe on windows (thanks Alex Brainman).
+* archive/zip: add Writer,
+	add Mtime_ns function to get modified time in sensible format.
+* cc, ld, gc: fixes for Plan 9 build (thanks Lucio De Re).
+* cgi: close stdout reader pipe when finished.
+* cgo: add missing semicolon in generated struct,
+	windows amd64 port (thanks Wei Guangjing).
+* codereview: fix for Mercurial 1.9.
+* dashboard: list "most installed this week" with rolling count.
+* debug/elf: read ELF Program headers (thanks Matthew Horsnell).
+* debug/pe: fixes ImportedSymbols for Win64 (thanks Wei Guangjing).
+* debug/proc: remove unused package.
+* doc/talks/io2010: update with gofix and handle the errors.
+* exp/eval, exp/ogle: remove packages eval and ogle.
+* exp/regexp/syntax: add Prog.NumCap.
+* exp/template: API changes, bug fixes, and tweaks.
+* flag: make -help nicer.
+* fmt: Scan(&int) was mishandling a lone digit.
+* gc: fix closure bug,
+	fix to build with clang (thanks Dave Cheney),
+	make size of struct{} and [0]byte 0 bytes (thanks Robert Hencke),
+	some enhancements to printing debug info.
+* gif: fix local color map and coordinates.
+* go/build: fixes for windows (thanks Alex Brainman),
+	include processing of .c files for cgo packages (thanks Alex Brainman),
+	less aggressive failure when GOROOT not found.
+* go/printer: changed max. number of newlines from 3 to 2.
+* gob: register more slice types (thanks Bobby Powers).
+* godoc: support for file systems stored in .zip files.
+* goinstall, dashboard: Google Code now supports git (thanks Tarmigan Casebolt).
+* hash/crc32: add SSE4.2 support.
+* html: update section references in comments to the latest HTML5 spec.
+* http: drain the pipe output in TestHandlerPanic to avoid logging deadlock,
+	fix Content-Type of file extension (thanks Yasuhiro Matsumoto),
+	implement http.FileSystem for zip files,
+	let FileServer work when path doesn't begin with a slash,
+	support for periodic flushing in ReverseProxy.
+* image/draw: add benchmarks.
+* json: add omitempty struct tag option,
+	allow using '$' and '-' as the struct field's tag (thanks Mikio Hara),
+	encode \r and \n in strings as e.g. "\n", not "\u000A" (thanks Evan Martin),
+	escape < and > in any JSON string for XSS prevention.
+* ld: allow seek within write buffer<
+	add a PT_LOAD PHDR entry for the PHDR (thanks David Anderson).
+* net: windows/amd64 port (thanks Wei Guangjing).
+* os: plan9: add Process.Signal as a way to send notes (thanks Yuval Pavel Zholkover).
+* os: don't permit Process.Signal after a successful Wait.
+* path/filepath: fixes for windows paths (thanks Alex Brainman).
+* reflect: add Value.NumMethod,
+	panic if Method index is out of range for a type.
+* runtime: faster entersyscall, exitsyscall,
+	fix panic for make(chan [0]byte),
+	fix subtle select bug (thanks Hector Chu),
+	make goc2c build on Plan 9 (thanks Lucio De Re),
+	make TestSideEffectOrder work twice,
+	several parallelism-related optimizations and fixes,
+	stdcall_raw stack 16byte align for Win64 (thanks Wei Guangjing),
+	string-related optimizations (thanks Quan Yong Zhai),
+	track running goroutine count.
+* strconv: handle [-+]Infinity in atof.
+* sync: add fast paths to WaitGroup,
+	improve RWMutex performance.
+* syscall: add Flock on Linux,
+	parse and encode SCM_RIGHTS and SCM_CREDENTIALS (thanks Albert Strasheim).
+
+ +

2011-07-07 (base for r59)

+ +
+This weekly snapshot includes changes to the strings, http, reflect, json, and
+xml packages. Code that uses these packages will need changes. Most of these
+changes can be made automatically with gofix.
+
+The strings package's Split function has itself been split into Split and
+SplitN. SplitN is the same as the old Split. The new Split is equivalent to
+SplitN with a final argument of -1.
+
+The http package has a new FileSystem interface that provides access to files.
+The FileServer helper now takes a FileSystem argument instead of an explicit
+file system root. By implementing your own FileSystem you can use the
+FileServer to serve arbitrary data.
+
+The reflect package supports a new struct tag scheme that enables sharing of
+struct tags between multiple packages.
+In this scheme, the tags must be of the form:
+        key:"value" key2:"value2"
+reflect.StructField's Tag field now has type StructTag (a string type), which
+has method Get(key string) string that returns the associated value.
+Clients of json and xml will need to be updated. Code that says
+        type T struct {
+                X int "name"
+        }
+should become
+        type T struct {
+                X int `json:"name"`  // or `xml:"name"`
+        }
+Use govet to identify struct tags that need to be changed to use the new syntax.
+
+Other changes:
+* 5l, 6l, 8l: drop use of ed during build.
+* asn1: support T61 and UTF8 string.
+* bufio: do not cache Read errors (thanks Graham Miller).
+* build: make version.bash aware of branches.
+* cgi: don't depend on CGI.pm for tests.
+* codereview: make --ignore_hgpatch_failure work again,
+	restrict sync to default branch.
+* crypto/openpgp: add ability to reserialize keys,
+	bug fix (thanks Gideon Jan-Wessel Redelinghuys).
+* crypto/tls: fix generate_cert.go.
+* crypto/x509: prevent chain cycles in Verify.
+* csv: new package.
+* doc: remove ed from apt-get package list.
+* docs: fold the prog.sh scripting from makehtml into htmlgen itself.
+* ebnflint: better handling of stdin.
+* exp/regexp/syntax: new experimental RE2-based regexp implementation.
+* exp/template: a new experimental templating package.
+* fmt: add SkipSpace to fmt's ScanState interface.
+* fmt: rename errno and error to err for doc consistency.
+* gc: avoid package name ambiguity in error messages,
+	fix package quoting logic,
+	fixes for Plan 9 (thanks Lucio De Re).
+* go/build: evaluate symlinks before comparing path to GOPATH.
+* gob: use exported fields in structs in the package documentation.
+* godoc: ignore directories that begin with '.',
+	search GOPATH for documentation.
+* gofix: os/signal, path/filepath, and sort fixes (thanks Robert Hencke),
+* goinstall: add support for generic hosts (thanks Julian Phillips),
+	only report successfully-installed packages to the dashboard,
+	try to access via https (thanks Yasuhiro Matsumoto).
+* gotest: add -test.benchtime and -test.cpu flags.
+* html: fixes and improvements (thanks Yasuhiro Matsumoto).
+* http/cgi: add Handler.Dir to specify working directory (thanks Yasuhiro Matsumoto).
+* http: add StripPrefix handler wrapper,
+	assume ContentLength 0 on GET requests,
+	better handling of 0-length Request.Body,
+	do TLS handshake explicitly before copying TLS state,
+	document that ServerConn and ClientConn are low-level,
+	make NewChunkedReader public (thanks Andrew Balholm),
+	respect Handlers setting Connection: close in their response.
+* image: more tests, Paletted.Opaque optimization.
+* io.WriteString: if the object has a WriteString method, use it (thanks Evan Shaw).
+* ld: elide the Go symbol table when using -s (thanks Anthony Martin).
+* ld: fix ELF strip by removing overlap of sections (thanks Gustavo Niemeyer).
+* mime/multipart: parse LF-delimited messages, not just CRLF.
+* mime: permit lower-case media type parameters (thanks Pascal S. de Kloe).
+* misc/dashboard: new features and improvements (not yet deployed).
+* misc/emacs: update list of builtins (thanks Quan Yong Zhai).
+* misc/vim: allow only utf-8 for file encoding (thanks Yasuhiro Matsumoto).
+* os: fix documentation for FileInfo.Name,
+	simplify WriteString,
+	use a different symbol from syscall in mkunixsignals.sh.
+* path/filepath: enable TestWalk to run on windows (thanks Alex Brainman).
+* reflect: add MethodByName,
+	allow Len on String values.
+* regexp: document that Regexp is thread-safe.
+* runtime/cgo: check for errors from pthread_create (thanks Albert Strasheim).
+* runtime: add Semacquire/Semrelease benchmarks,
+	improved Semacquire/Semrelease implementation,
+	windows/amd64 port (thanks Wei Guangjing).
+* sync: add fast path to Once,
+	improve Mutex to allow successive acquisitions,
+	new and improved benchmarks.
+* syscall: regenerate zerrors for darwin/linux/freebsd,
+	support for tty options in StartProcess (thanks Ken Rockot).
+* testing: make ResetTimer not start/stop the timer,
+	scale benchmark precision to 0.01ns if needed.
+* time: zero-pad two-digit years.
+* unicode/maketables: update debugging data.
+* windows: define and use syscall.Handle (thanks Wei Guangjing).
+* xml: add Marshal and MarshalIndent.
+
+ +

2011-06-23

+ +
+This release includes a language change that restricts the use of goto.
+In essence, a "goto" statement outside a block cannot jump to a label inside
+that block. Your code may require changes if it uses goto.
+This changeset shows how the new rule affected the Go tree:
+	http://code.google.com/p/go/source/detail?r=dc6d3cf9279d
+
+The os.ErrorString type has been hidden. If your code uses os.ErrorString it
+must be changed. Most uses of os.ErrorString can be replaced with os.NewError.
+
+Other changes:
+* 5c: do not use R9 and R10.
+* 8l: more fixes for Plan 9 (thanks Lucio De Re).
+* build: Make.ccmd: link with mach lib (thanks Joe Poirier).
+* build: exclude packages that fail on Plan 9 (thanks Anthony Martin).
+* cc: nit: silence comment warnings (thanks Dave Cheney).
+* codereview.py: note that hg change -d abandons a change list (thanks Robert Hencke).
+* crypto/openpgp: add ElGamal support.
+* doc/faq: add question about converting from []T to []interface{}.
+* doc: Effective Go: fix variadic function example (thanks Ben Lynn).
+* exec: LookPath should not search %PATH% for files like c:cmd.exe (thanks Alex Brainman),
+        add support for Plan 9 (thanks Anthony Martin),
+        better error message for windows LookPath (thanks Alex Brainman).
+* fmt: catch panics from calls to String etc.
+* gc: descriptive panic for nil pointer -> value method call,
+        implement goto restriction,
+        unsafe.Alignof, unsafe.Offsetof, unsafe.Sizeof now return uintptr.
+* go/build: include Import objects in Script Inputs.
+* godefs: rudimentary tests (thanks Robert Hencke).
+* goinstall: refactor and generalize repo handling code (thanks Julian Phillips),
+        temporarily use Makefiles by default (override with -make=false).
+* gopprof: update list of memory allocators.
+* http: add Server.ListenAndServeTLS,
+        buffer request.Write,
+        fix req.Cookie(name) with cookies in one header,
+        permit handlers to explicitly remove the Date header,
+        write Header keys with empty values.
+* image: basic test for the 16-bits-per-color-channel types.
+* io: clarify Read, ReadAt, Copy, Copyn EOF behavior.
+* ld: don't attempt to build dynamic sections unnecessarily (thanks Gustavo Niemeyer).
+* libmach: fix disassembly of FCMOVcc and FCOMI (thanks Anthony Martin),
+        fix tracing on linux (for cov) (thanks Anthony Martin).
+* mime: fix RFC references (thanks Pascal S. de Kloe).
+* misc/gobuilder: run make single-threaded on windows (thanks Alex Brainman).
+* misc/godashboard: Accept sub-directories for goinstall's report (thanks Yasuhiro Matsumoto).
+* nacl, tiny: remove vestiges (thanks Robert Hencke).
+* net, syscall: interface for windows (thanks Yasuhiro Matsumoto).
+* os: change Waitmsg String method to use pointer receiver (thanks Graham Miller).
+* runtime: don't use twice the memory with grsec-like kernels (thanks Gustavo Niemeyer),
+* spec: disallow goto into blocks.
+* sync: restore GOMAXPROCS during benchmarks.
+* syscall: add LSF support for linux (thanks Mikio Hara),
+        add socket control message support for darwin, freebsd, linux (thanks Mikio Hara),
+        add tty support to StartProcess (thanks Ken Rockot),
+        fix build for Sizeof change.
+* test: test of goto restrictions.
+* time: add support for Plan 9 (thanks Anthony Martin).
+
+ +

2011-06-16

+ +
+This release includes changes to the sort and image/draw packages that will
+require changes to client code.
+
+The sort.IntArray type has been renamed to IntSlice, and similarly for
+StringArray and Float64Array.
+
+The image/draw package's Draw function now takes an additional argument,
+a compositing operator. If in doubt, use draw.Over.
+
+Other changes:
+* build: fix header files for Plan 9 (thanks Lucio De Re).
+* cgo: handle new Apple LLVM-based gcc from Xcode 4.2.
+* crypto/openpgp: add ability to encrypt and sign messages.
+* doc/gopher: add goggled gopher logo for App Engine.
+* doc: Update notes for 3-day Go course.
+* exec: make LookPath work when PATHEXT var not set on Windows (thanks Alex Brainman).
+* exp/regexp/syntax: syntax data structures, parser, escapes, character classes.
+* exp/template: lexical scanner for new template package.
+* fmt: debugging formats for characters: %+q %#U.
+* gc: frame compaction for arm,
+        handle go print() and go println(),
+        work around goto bug.
+* go/build: fixes, self-contained tests.
+* go/printer, gofmt: print "select {}" on one line.
+* godoc: replace OS file system accesses in favor of a FileSystem interface.
+* gofix: fix inconsistent indentation in help output (thanks Scott Lawrence).
+* goinstall: use go/build package to scan and build packages.
+* http/spdy: improve error handling (thanks William Chan).
+* http: use runtime/debug.Stack() to dump stack trace on panic.
+* ld: dwarf emit filenames in debug_line header instead of as extended opcodes,
+        fix link Windows PE __declspec(dllimport) symbol (thanks Wei Guangjing),
+        make .rodata section read-only (thanks Gustavo Niemeyer).
+* mail: decode RFC 2047 "B" encoding.
+* mime/multipart: remove temp files after tests on Windows (thanks Alex Brainman).
+* net: export all fields in Interface (thanks Mikio Hara),
+        rearrange source to run more tests on Windows (thanks Alex Brainman),
+        sendfile for win32 (thanks Yasuhiro Matsumoto).
+* os: Plan 9, fix OpenFile & Chmod, add Process.Kill (thanks Yuval Pavel Zholkover).
+* runtime: fix Plan 9 "lingering goroutines bug" (thanks Yuval Pavel Zholkover).
+* spec: clarify rules for append, scope rules for :=,
+        specify constant conversions,
+        unsafe.Alignof/Offsetof/Sizeof return uintptr.
+* syscall, os, exec: add *syscall.SysProcAttr field to os.ProcAttr and exec.Cmd.
+* syscall: add ptrace on darwin (thanks Jeff Hodges),
+        mksyscall_windows.pl should output unix newline (thanks Yasuhiro Matsumoto).
+        update BPF support for BSD variants (thanks Mikio Hara),
+        use strict in perl scripts (thanks Yasuhiro Matsumoto).
+* xml: handle non-string attribute fields (thanks Maxim Ushakov).
+
+ +

2011-06-09 (base for r58)

+ +
+This release includes changes to the strconv, http, and exp/draw packages.
+Client code that uses the http or exp/draw packages will need to be changed,
+and code that uses strconv or fmt's "%q" formatting directive merits checking.
+
+The strconv package's Quote function now escapes only those Unicode code points
+not classified as printable by unicode.IsPrint. Previously Quote would escape
+all non-ASCII characters. This also affects the fmt package's "%q" formatting
+directive. The previous quoting behavior is still available via strconv's new
+QuoteToASCII function.   
+
+Most instances of the type map[string][]string in the http package have been
+replaced with the new Values type. The http.Values type has the Get, Set, Add,
+and Del helper methods to make working with query parameters and form values
+more convenient.
+
+The exp/draw package has been split into the image/draw and exp/gui packages.
+
+Other changes:
+* 8l, ld: initial adjustments for Plan 9 native compilation of 8l (thanks Lucio De Re).
+* arm: floating point improvements (thanks Fan Hongjian).
+* big: Improved speed of nat-to-string conversion (thanks Michael T. Jones),
+        Rat outputs the requested precision from FloatString (thanks Graham Miller),
+        gobs for big.Rats.
+* cgo: support non intel gcc machine flags (thanks Dave Cheney).
+* compress/lzw: do not use background goroutines,
+        reduce decoder buffer size from 3*4096 to 2*4096.
+* crypto/twofish: fix Reset index overflow bug.
+* crypto: reorg, cleanup and add function for generating CRLs.
+* exec: export the underlying *os.Process in Cmd.
+* gc: enable building under clang/2.9 (thanks Dave Cheney),
+        preparatory work toward escape analysis, compact stack frames.
+* go/build: new incomplete package for building go programs.
+* godefs: do not assume forward type references are enums (thanks Robert Hencke).
+* gofix, gofmt: fix diff regression from exec change.
+* html: improve attribute parsing, note package status.
+* http: don't fail on accept hitting EMFILE,
+        fix handling of 0-length HTTP requests.
+* image/draw: fix clipping bug where sp/mp were not shifted when r.Min was.
+* image/gif: fix buglet in graphics extension.
+* image/tiff: support for bit depths other than 8 (thanks Benny Siegert).
+* ld: fix and simplify ELF symbol generation (thanks Anthony Martin)
+* libmach: use the standardized format for designated initializers (thanks Jeff Hodges)
+* mail: address list parsing.
+* net: add network interface identification API (thanks Mikio Hara),
+        fix bug in net.Interfaces: handle elastic sdl_data size correctly (thanks Mikio Hara).
+* netchan: added drain method to importer (thanks David Jakob Fritz).
+* os: add Process.Kill and Process.Signal (thanks Evan Shaw),
+        fix Getenv for Plan 9 (thanks Yuval Pavel Zholkover).
+* runtime: improve memmove by checking memory overlap (thanks Quan Yong Zhai),
+        support for Linux grsecurity systems (thanks Jonathan Mark).
+* spec: handle a corner case for shifts.
+* testing: check that tests and benchmarks do not affect GOMAXPROCS (thanks Dmitriy Vyukov).
+* unicode: add IsPrint and related properties, general categories.
+
+ +

2011-06-02

+ +
+This release includes changes to the exec package that will require changes
+to client code.
+
+The exec package has been re-designed with a more convenient and succinct API.
+This code:
+	args := []string{"diff", "-u", "file1.txt", "file2.txt"}
+	p, err := exec.Run("/usr/bin/diff", args, os.Environ(), "",
+		exec.DevNull, exec.Pipe, exec.DevNull)
+	if err != nil {
+		return nil, err
+	}
+	var buf bytes.Buffer
+	io.Copy(&buf, p.Stdout)
+	w, err := p.Wait(0)
+	p.Close()
+	if err != nil {
+		return nil, err
+	}
+	return buf.Bytes(), err
+can be rewritten as:
+	return exec.Command("diff", "-u", "file1.txt", "file2.txt").Output()
+See the exec package documentation for the details ("godoc exec").
+
+By setting the GOPATH environment variable you can use goinstall to build and
+install your own code and external libraries outside of the Go tree (and avoid
+writing Makefiles).
+See the goinstall command documentation for the details ("godoc goinstall").
+
+Other changes:
+* 5g: alignment fixes.
+* 6l, 8l: fix Mach-O binaries with many dynamic libraries.
+* 8l: emit resources (.rsrc) in Windows PE.  (thanks Wei Guangjing).
+* asn1: fix marshalling of empty optional RawValues (thanks Mikkel Krautz).
+* big: make Int and Rat implement fmt.Scanner (thanks Evan Shaw),
+	~8x faster number scanning,
+	remove some unnecessary conversions.
+* cgo: restrict #cgo directives to prevent shell expansion (thanks Gustavo Niemeyer),
+	support pkg-config for flags and libs (thanks Gustavo Niemeyer).
+* compress/flate: fix Huffman tree bug,
+	do not use background goroutines.
+* crypto/openpgp: add support for symmetrically encrypting files.
+* crypto/tls/generate_cert.go: fix misspelling of O_CREATE.
+* dashboard: send notification emails when the build breaks.
+* doc: mention go/printer instead of container/vector in effective go,
+	put Release History link on 'Documentation' page,
+	put Weekly Snapshot History link on 'Contributing' page.
+* encoding/base64: add DecodeString and EncodeToString.
+* encoding/binary: add a non-reflect fast path for Read,
+	add a non-reflect fast path for Write.
+* encoding/hex: add hex dumping.
+* encoding/line: delete package. Its functionality is now in bufio.
+* filepath: Abs must always return a clean path (thanks Gustavo Niemeyer).
+* fmt: fix bug in UnreadRune,
+	make %q work for integers, printing a quoted character literal,
+	return EOF when out of input in Scan*.
+* gc: check parameter declarations in interface fields (thanks Anthony Martin),
+	disallow ... in type conversions (thanks Anthony Martin),
+	do not force heap allocation on referencing outer variable in a closure,
+	fix m[x], _ = y.(T),
+	implement new shift rules,
+	patch y.tab.c to fix build when using Bison 2.5,
+	relax assignability of method receivers (thanks Anthony Martin),
+	typecheck the whole tree before walking.
+* go/scanner: don't allow "0x" and "0X" as integers (thanks Evan Shaw).
+* gobuilder: fixes for windows (thanks Alex Brainman).
+* godoc: basic setup for running godoc on local app engine emulator,
+	display advert for the package dashboard on package list page.
+* goinstall: fixes for windows (thanks Alex Brainman),
+	more verbose logging with -v.
+* gotest, pkg/exec: use bash to run shell scripts on windows (thanks Alex Brainman).
+* http/spdy: redo interfaces, flesh out implementation & frame types (thanks William Chan).
+* http: Transport hook to register non-http(s) protocols,
+	add client+server benchmark,
+	catch Handler goroutine panics,
+	fix Set-Cookie date parsing,
+	have client set Content-Length when possible,
+	let Transport use a custom net.Dial function,
+	propagate Set-Cookie in reverse proxy,
+	ServeFile shouldn't send Content-Length when Content-Encoding is set.
+* image: add a SubImage method.
+* image/gif: simplify blockReader.Read.
+* image/png: fix encoding of images that don't start at (0, 0).
+* io, net, http: sendfile support.
+* io: add ByteScanner, RuneScanner interfaces.
+* ld: add -w to disable dwarf, make errors obviously from dwarf.
+* mail: new package.
+* mime/multipart: misc code/doc fixes.
+* misc/cgo: remove reference to 'destroy' function.
+* misc/emacs: don't select the mark after gofmt (thanks Eric Eisner).
+* misc/gophertool: Chrome extension to aid in Go development
+* misc/vim: limit Fmt command to Go buffers (thanks Yasuhiro Matsumoto).
+* net: if we stop polling, remove any pending events for the socket,
+	update IP multicast socket options (thanks Mikio Hara).
+* os: Fix test to work on Solaris,
+	fix Readdir(0) on EOF,
+	fix Readdir, Readdirnames (thanks Yuval Pavel Zholkover),
+	fix os.MkdirAll with backslash path separator (thanks Yasuhiro Matsumoto),
+	handle OpenFile flag parameter properly on Windows (thanks Alex Brainman).
+* path/filepath: remove string constants.
+* pkg: spelling tweaks, I-Z (thanks Robert Hencke).
+* quietgcc: fix typo, respect $TMPDIR.
+* runtime: do not garbage collect windows callbacks (thanks Alex Brainman),
+	fix mmap error return on linux (thanks Dmitry Chestnykh),
+	reset GOMAXPROCS during tests,
+	save cdecl registers in Windows SEH handler (thanks Alexey Borzenkov).
+* spec: be precise with the use of the informal ellipsis and the Go token,
+	clarify rules for shifts.
+* strconv: add QuoteRune; analogous to Quote but for runes rather than strings.
+* strings: implement UnreadByte, UnreadRune.
+* sync: always wake up sleeping goroutines on Cond.Signal (thanks Gustavo Niemeyer).
+* sync/atomic: fix check64.
+* syscall: add ProcAttr field to pass an unescaped command line on windows (thanks Vincent Vanackere),
+	add routing messages support for Linux and BSD (thanks Mikio Hara).
+* template: fixes and clean-ups (thanks Gustavo Niemeyer).
+* time: fix Format bug: midnight/noon are 12AM/PM not 0AM/PM.
+* unicode: make the tables smaller.
+
+ +

2011-05-22

+ +
+This release includes changes to the http package that will require changes to
+client code.
+
+The finalURL return value of the Client.Get method has been removed.
+This value is now accessible via the new Request field on http.Response.
+For example, this code:
+
+	res, finalURL, err := http.Get(...)
+
+should be rewritten as:
+
+	res, err := http.Get(...)
+	if err != nil {
+		// ...
+	}
+	finalURL := res.Request.URL.String()
+
+Uses of http.Get that assign the finalURL value to _ can be rewritten
+automatically with gofix.
+
+This release also includes an optimization to the append function that makes it
+between 2 and 5 times faster in typical use cases.
+
+Other changes:
+* 5a, 6a, 8a, cc: remove old environment variables.
+* 5c, 5g: fix build with too-smart gcc.
+* 5l, 8l: add ELF symbol table to binary.
+* 5l: delete pre-ARMv4 instruction implementations, other fixes.
+* 6l, 8l: emit windows dwarf sections like other platforms (thanks Alex Brainman).
+* 6l: fix emit windows dwarf sections (thanks Wei Guangjing).
+* 8g: fix conversion from float to uint64 (thanks Anthony Martin).
+* Make.cmd: create TARGDIR if necessary (thanks Gustavo Niemeyer).
+* asn1: add big support.
+* big: add Int methods to act on numbered bits (thanks Roger Peppe),
+	better support for string conversions,
+	support %v and # modifier, better handling of unknown formats.
+* cgi: export RequestFromMap (thanks Evan Shaw),
+	set Request.TLS and Request.RemoteAddr for children.
+* cgo: use packed struct to fix Windows behavior.
+* codereview: add release branch support,
+	fetch metadata using JSON API, not XML scraping,
+	handle 'null as missing field' in rietveld json.
+* compress/lzw: silently drop implied codes that are too large.
+* compress/zlib: actually use provided dictionary in NewWriterDict
+* crypto/openpgp: add key generation support,
+	change PublicKey.Serialize to include the header.
+* crypto/rand: add utility functions for number generation (thanks Anthony Martin).
+* crypto/tls: export the verified chains.
+* crypto/x509/crl: add package.
+* crypto/x509: export raw SubjectPublicKeyInfo,
+	support DSA public keys in X.509 certs,
+	support parsing and verifying DSA signatures (thanks Jonathan Allie).
+* doc/roadmap: put "App Engine support" under "Done".
+* doc: add I/O 2011 talks to talks/, docs.html, and front page.
+* effective go: explain about values/pointers in String() example,
+	update to new Open signature.
+* exp/draw: fast paths for drawing a YCbCr or an NRGBA onto an RGBA.
+* filepath: make EvalSymlinks work on Windows (thanks Alex Brainman).
+* flag: allow distinct sets of flags.
+* gc: fix type switch error message for invalid cases (thanks Lorenzo Stoakes),
+	fix unsafe.Sizeof,
+	preserve original expression for errors.
+* go/ast, go/doc, godoc: consider struct fields and interface methods when filtering ASTs.
+* go/ast: consider anonymous fields and set Incomplete bit when filtering ASTs,
+	properly maintain map of package global imports.
+* go/doc, godoc: when filtering for godoc, don't remove elements of a declaration.
+* go/parser: accept parenthesized receive operations in select statements,
+	always introduce an ast.Object when declaring an identifier.
+* go/printer, gofmt: fix alignment of "=" in const/var declarations,
+	fix formatting of expression lists (missing blank).
+* go/printer: added simple performance benchmark,
+	make tests follow syntactic restrictions,
+	more accurate comment for incomplete structs/interfaces,
+* go/token: faster FileSet.Position implementation.
+* go/types: type checker API + testing infrastructure.
+* godoc: added -index flag to enable/disable search index,
+	if there is no search box, don't run the respective JS code.
+* gofmt: update test.sh (exclude a file w/ incorrect syntax).
+* html: parse empty, unquoted, and single-quoted attribute values.
+* http/cgi: correctly set request Content-Type (thanks Evan Shaw),
+	pass down environment variables for IRIX and Solaris.
+* http/pprof: fix POST reading bug.
+* http/spdy: new incomplete package (thanks Ross Light).
+* http: Client.Do should follow redirects for GET and HEAD,
+	add Header.Write method (thanks Evan Shaw),
+	add Request.SetBasicAuth method,
+	add Transport.ProxySelector,
+	add http.SetCookie(ResponseWriter, *Cookie),
+	don't Clean query string in relative redirects,
+	fix FormFile nil pointer dereference on missing multipart form,
+	fix racy test with a simpler version,
+	fix two Transport gzip+persist crashes,
+	include Host header in requests,
+	make HEAD client request follow redirects (thanks Eivind Uggedal).
+	update cookie doc to reference new RFC 6265,
+	write cookies according to RFC 6265 (thanks Christian Himpel).
+* image/bmp: implement a BMP decoder.
+* image/gif: new package provides a GIF decoder.
+* image/jpeg: decode grayscale images, not just color images.
+	optimizations and tweaks.
+* image/png: encode paletted images with alpha channel (thanks Dmitry Chestnykh),
+	speed up opaque RGBA encoding.
+* image/tiff: implement a decoder (thanks Benny Siegert).
+* image: add type-specific Set methods and use them when decoding PNG,
+	make AlphaColor.Set conform to usual signature (thanks Roger Peppe),
+	png & jpeg encoding benchmarks.
+* ld: do not emit reference to dynamic library named "",
+	fix alignment of rodata section on Plan 9 (thanks Anthony Martin),
+	make ELF binaries with no shared library dependencies static binaries.
+* make.bash: remove old bash version of gotest on Windows (thanks Alex Brainman).
+* make: add nuke target for C commands and libs (thanks Anthony Martin).
+* mime/multipart: add FileName accessor on Part,
+	add Writer,
+	return an error on Reader EOF, not (nil, nil).
+* misc/cgo/test: run tests.
+* misc/emacs: use UTF-8 when invoking gofmt as a subprocess (thanks Sameer Ajmani).
+* misc/vim: new Vim indentation script.
+* net, http: add and make use of IP address scope identification API (thanks Mikio Hara).
+* net: default to 127.0.0.1, not localhost, in TestICMP,
+	don't crash on unexpected DNS SRV responses,
+	enable SO_REUSEPORT on BSD variants (thanks Mikio Hara),
+	protocol family adaptive address family selection (thanks Mikio Hara),
+	re-enable wildcard listening (thanks Mikio Hara),
+	sort records returned by LookupSRV (thanks Gary Burd).
+* os: make Readdir & Readdirnames return os.EOF at end,
+	make Setenv update C environment variables.
+* reflect: allow unexported key in Value.MapIndex.
+* runtime, sync/atomic: fix arm cas.
+* runtime: add newline to "finalizer already set" error (thanks Albert Strasheim),
+	handle out-of-threads on Linux gracefully (thanks Albert Strasheim),
+	fix function args not checked warning on ARM (thanks Dave Cheney),
+	make StackSystem part of StackGuard (thanks Alexey Borzenkov),
+	maybe fix Windows build broken by cgo setenv CL.
+* spec: clarify semantics of integer division,
+	clarify semantics of range clause,
+	fix error in production syntax,
+	narrow syntax for expression and select statements,
+	newlines cannot be used inside a char or "" string literal,
+	restricted expressions may still be parenthesized.
+* strings: make Reader.Read use copy instead of an explicit loop.
+* syscall: add Windows file mapping functions and constants (thanks Evan Shaw),
+	add IPv6 scope zone ID support (thanks Mikio Hara),
+	add netlink support for linux/386, linux/amd64, linux/arm (thanks Mikio Hara),
+	add Sendfile,
+	adjust freebsd syscalls.master URL properly (thanks Mikio Hara),
+	change Overlapped.HEvent type, it is a handle (thanks Alex Brainman).
+* syslog: fix skipping of net tests (thanks Gustavo Niemeyer).
+* template: support string, int and float literals (thanks Gustavo Niemeyer).
+* xml: fix reflect error.
+
+ +

2011-04-27 (base for r57)

+ +
+This release includes revisions to the reflect package to make it more
+efficient, after the last weekly's major API update. If your code uses reflect
+it may require further changes, not all of which can be made automatically by
+gofix. For the full details of the change, see
+	http://codereview.appspot.com/4435042
+Also, the Typeof and NewValue functions have been renamed to TypeOf and ValueOf.
+
+Other changes:
+* 5c: make alignment rules match 5g, just like 6c matches 6g.
+* 8g, 8l: fix "set but not used" gcc error (thanks Fazlul Shahriar).
+* all-qemu.bash: remove DISABLE_NET_TESTS.
+* build: remove DISABLE_NET_TESTS.
+* builder: build multiple targets in parallel.
+* cgo: avoid "incompatible pointer type" warning (thanks Albert Strasheim).
+* codereview: add 'hg undo' command, various other fixes.
+* compress/flate: dictionary support.
+* compress/zlib: add FDICT flag in Reader/Writer (thanks Ross Light).
+* container/heap: fix circular dependency in test.
+* crypto/openpgp: better handling of keyrings.
+* crypto/rsa: support > 3 primes.
+* crypto/tls: add server-side OCSP stapling support.
+* crypto/x509: memorize chain building.
+* crypto: move certificate verification into x509.
+* dashboard: build most recent revision first.
+* doc: mention make version in install.html.
+* expvar: add Func for functions that return values that are JSON marshalable.
+* fmt: decrease recursion depth in tests to permit them to run under gccgo,
+	tweak the doc for %U.
+* gc: allow complex types to be receiver types (thanks Robert Hencke),
+	correct handling of unexported method names in embedded interfaces,
+	explain why invalid receiver types are invalid,
+	fix copy([]int, string) error message (thanks Quan Yong Zhai),
+	fix 'invalid recursive type' error (thanks Lorenzo Stoakes),
+	many bug fixes.
+* go spec: attempt at clarifying language for "append",
+	for map types, mention indexing operations.
+* go/types: update for export data format change.
+* gob: fix handling of indirect receivers for GobDecoders,
+	fix trivial bug in map marshaling,
+	have errorf always prefix the message with "gob: ",
+	test case for indirection to large field,
+	use new Implements and AssignableTo methods in reflect,
+	when decoding a string, allocate a string, not a []byte.
+* gobuilder: permit builders of the form goos-goarch-foo,
+	respect MAKEFLAGS if provided (thanks Dave Cheney).
+* godoc: use "search" input type for search box (thanks Dmitry Chestnykh).
+* gofix: add support for reflect rename.
+* gofmt: add -d (diff) (thanks David Crawshaw),
+	don't crash when rewriting nil interfaces in AST,
+	exclude test case that doesn't compile w/o errors,
+	gofmt test harness bug fix.
+* goinstall: support GOPATH; building and installing outside the Go tree,
+	support building executable commands.
+* gopack: fix prefix bug,
+	preserve safe flag when not adding unsafe objects to archive.
+* gotest: add timing, respect $GOARCH,
+	generate gofmt-compliant code.
+* http/cgi: copy some PATH environment variables to child,
+	improve Location response handling,
+	pass some default environment variables.
+* http/fcgi: new package (thanks Evan Shaw).
+* http: add NewRequest helper,
+	add MultipartForm, ParseMultipartForm, and FormFile to Request,
+	be clear when failing to connect to a proxy,
+	bug fixes and new tests,
+	consume request bodies before replying,
+	don't quote Set-Cookie Domain and Path (thanks Petar Maymounkov),
+	fix IP confusion in TestServerTimeouts,
+	handler timeout support,
+	ServerConn, ClientConn: add real Close (thanks Petar Maymounkov),
+	make Client redirect policy configurable,
+	put a limit on POST size,
+	reverse proxy handler.
+* image/jpeg: add an encoder,
+	decode to a YCbCr image instead of an RGBA image.
+* ioutil: add Discard.
+* json: keep track of error offset in SyntaxError.
+* ld: defend against some broken object files,
+	do not emit empty dwarf pe sections (thanks Alex Brainman),
+	fix 6l -d on Mac, diagnose invalid use of -d,
+	fix Plan 9 symbol table (thanks Anthony Martin),
+	remove MachoLoad limit.
+* make: prevent rm provoking 'text file busy' errors (thanks Lorenzo Stoakes).
+* mime/multipart: add ReadForm for parsing multipart forms,
+	limit line length to prevent abuse.
+* mime: RFC 2231 continuation / non-ASCII support,
+	bunch more tests, few minor parsing fixes.
+* misc/goplay: fix Tab and Shift+Enter in Firefox (thanks Dmitry Chestnykh).
+* net: disable one more external network test,
+	fix EAI_BADFLAGS error on freebsd (thanks Mikio Hara),
+	fix ParseIP (thanks Quan Yong Zhai),
+	fix dialgoogle_test.go (thanks Quan Yong Zhai),
+	try /etc/hosts before loading DNS config (thanks Dmitry Chestnykh),
+	use C library resolver on FreeBSD, Linux, OS X / amd64, 386.
+* os/user: new package to look up users.
+* os: Open with O_APPEND|O_CREATE to append on Windows (thanks Alex Brainman),
+	fix race in ReadAt/WriteAt on Windows (thanks Alex Brainman),
+	turn EPIPE exit into panic.
+* rc/env.bash: fix to build on windows under msys (thanks Joe Poirier).
+* reflect: allow Slice of arrays,
+	fix Copy of arrays (thanks Gustavo Niemeyer),
+	require package qualifiers to match during interface check,
+	add Type.Implements, Type.AssignableTo, Value.CallSlice,
+	make Set match Go.
+* rpc: allow the first argument of a method to be a value rather than a pointer,
+	run benchmarks over HTTP as well as direct network connections.
+* run.bash: remove redundant rebuilds.
+* runtime/plan9: warning remediation for Plan 9 (thanks Lucio De Re),
+* runtime: many bug fixes,
+	fix GOMAXPROCS vs garbage collection bug (thanks Dmitriy Vyukov),
+	fix mkversion to output valid path separators (thanks Peter Mundy),
+	more graceful out-of-memory crash,
+	require package qualifiers to match during interface check,
+	skip functions with no lines when building src line table,
+	turn "too many EPIPE" into real SIGPIPE.
+* src/pkg: make package doc comments consistently start with "Package foo".
+* syscall: Madvise and Mprotect for Linux (thanks Albert Strasheim),
+	Mlock, Munlock, Mlockall, Munlockall on Linux (thanks Albert Strasheim),
+	add BPF support for darwin/386, darwin/amd64 (thanks Mikio Hara),
+	correct Windows CreateProcess input parameters (thanks Alex Brainman),
+	fix Ftruncate under linux/arm5 (thanks Dave Cheney),
+	permit StartProcess to hide the executed program on windows (thanks Vincent Vanackere).
+* test/bench: update timings; moving to new machine.
+* time: support Irix 6 location for zoneinfo files.
+* tutorial: modernize the definition and use of Open,
+	replace the forever loops with finite counts in sieve programs.
+* websocket: include *http.Request in websocket.Conn.
+* xml: Parser hook for non-UTF-8 charset converters.
+
+ +

2011-04-13

+ +
+weekly.2011-04-13
+
+This weekly snapshot includes major changes to the reflect package and the
+os.Open function.  Code that uses reflect or os.Open will require updating,
+which can be done mechanically using the gofix tool.
+
+The reflect package's Type and Value types have changed.  Type is now an
+interface that implements all the possible type methods.  Instead of a type
+switch on a reflect.Type t, switch on t.Kind().  Value is now a struct value
+that implements all the possible value methods.  Instead of a type switch on a
+reflect.Value v, switch on v.Kind().  See the change for the full details:
+        http://code.google.com/p/go/source/detail?r=843855f3c026
+
+The os package's Open function has been replaced by three functions:
+        OpenFile(name, flag, perm) // same as old Open
+        Open(name) // same as old Open(name, O_RDONLY, 0)
+        Create(name) // same as old Open(name, O_RDWR|O_TRUNC|O_CREAT, 0666)
+
+To update your code to use the new APIs, run "gofix path/to/code".  Gofix can't
+handle all situations perfectly, so read and test the changes it makes before
+committing them.
+
+Other changes:
+* archive/zip: add func OpenReader, type ReadCloser (thanks Dmitry Chestnykh).
+* asn1: Implement correct marshaling of length octets (thanks Luit van Drongelen).
+* big: don't crash when printing nil ints.
+* bufio: add ReadLine, to replace encoding/line.
+* build: make the build faster, quieter.
+* codereview: automatically port old diffs forward,
+        drop Author: line on self-clpatch,
+        recognize code URL without trailing slash.
+* crypto/block: remove deprecated package.
+* crypto/des: new package implementating DES and TDEA (thanks Yasuhiro Matsumoto).
+* crypto/ecdsa, crypto/rsa: use io.ReadFull to read from random source (thanks Dmitry Chestnykh).
+* crypto/rsa: add 3-prime support,
+        add support for precomputing CRT values,
+        flip the CRT code over so that it matches PKCS#1.
+* crypto/x509: expose complete DER data (thanks Mikkel Krautz).
+* doc: new "Functions" codewalk (thanks John DeNero).
+* doc/roadmap: add sections on tools, packages.
+* fmt: allow %U for unsigned integers.
+* gc: fixes and optimizations.
+* go/printer, gofmt: use blank to separate import rename from import path.
+* go/scanner: better TokenString output.
+* go/types: new Go type hierarchy implementation for AST.
+* godashboard: show packages at launchpad.net (thanks Gustavo Niemeyer).
+* gofix: add -diff, various fixes and helpers.
+* gotest: fix a bug in error handling,
+        fixes for [^.]_test file pattern (thanks Peter Mundy),
+        handle \r\n returned by gomake on Windows (thanks Alex Brainman).
+* gotype: use go/types GcImporter.
+* govet: make name-matching for printf etc. case-insensitive.
+* http: allow override of Content-Type for ServeFile,
+        client gzip support,
+        do not listen on 0.0.0.0 during test,
+        flesh out server Expect handling + tests.
+* image/ycbcr: new package.
+* image: allow "?" wildcards when registering image formats.
+* io: fixes for Read with n > 0, os.EOF (thanks Robert Hencke).
+* ld: correct Plan 9 compiler warnings (thanks Lucio De Re),
+        ELF header function declarations (thanks Lucio De Re),
+        fix Mach-O X86_64_RELOC_SIGNED relocations (thanks Mikkel Krautz),
+        fix Mach-O bss bug (thanks Mikkel Krautz),
+        fix dwarf decoding of strings for struct's fieldnames (thanks Luuk van Dijk),
+        fixes and optimizations (25% faster).
+* log: generalize getting and setting flags and prefix.
+* misc/cgo/life: enable build and test on Windows (thanks Alex Brainman).
+* misc/vim: add plugin with Fmt command (thanks Dmitry Chestnykh),
+        update type highlighting for new reflect package.
+* net: disable multicast tests by default (thanks Dave Cheney),
+        sort records returned by LookupMX (thanks Corey Thomasson).
+* openpgp: Fix improper := shadowing (thanks Gustavo Niemeyer).
+* os: rename Open to OpenFile, add new Open, Create,
+        fix Readdir in Plan 9 (thanks Fazlul Shahriar).
+* os/inotify: use _test for test files, not _obj.
+* pkg/path: enable tests on Windows (thanks Alex Brainman).
+* reflect: new Type and Value API.
+* src/pkg/Makefile: trim per-directory make output except on failure.
+* syscall: Add DT_* and MADV_* constants on Linux (thanks Albert Strasheim),
+        add Mmap, Munmap on Linux, FreeBSD, OS X,
+        fix StartProcess in Plan 9 (thanks Fazlul Shahriar),
+        fix Windows Signaled (thanks Alex Brainman).
+* test/bench: enable build and test on Windows (thanks Alex Brainman).
+
+ +

2011-04-04

+ +
+This release includes changes to the net package. Your code will require
+changes if it uses the Dial or LookupHost functions.
+
+The laddr argument has been removed from net.Dial, and the cname return value
+has been removed from net.LookupHost. The new net.LookupCNAME function can be
+used  to find the canonical host for a given name.  You can update your
+networking code with gofix.
+
+The gotest shell script has been replaced by a Go program, making testing
+significantly faster.
+
+Other changes:
+* asn1: extensions needed for parsing Kerberos.
+* bufio: Write and WriteString cleanup (thanks Evan Shaw).
+* bytes, strings: simplify Join (thanks Evan Shaw).
+* crypto/cipher: bad CTR IV length now triggers panic.
+* crypto/tls: extend NPN support to the client,
+	added X509KeyPair function to parse a Certificate from memory.
+* crypto/x509: parse Extended Key Usage extension (thanks Mikkel Krautz).
+* debug/gosym: remove need for gotest to run preparatory commands.
+* fmt: implement precision (length of input) values for %q: %.20q.
+* go/parser: fix scoping for local type declarations (thanks Roger Peppe),
+	package name must not be the blank identifier.
+* go/printer, gofmt: remove special case for multi-line raw strings.
+* gopack: add P flag to remove prefix from filename information.
+* gotest: add -test.timeout option,
+	replace the shell script with the compiled program written in go,
+	execute gomake properly on Windows (thanks Alex Brainman).
+* gotry: move into its own directory, separate from gotest.
+* gotype: support for more tests, added one new test.
+* http: add Transport.MaxIdleConnsPerHost,
+	use upper case hex in URL escaping (thanks Matt Jones).
+* httptest: add NewTLSServer.
+* misc/kate: reorganize, remove closed() (thanks Evan Shaw).
+* misc/notepadplus: support for notepad++ (thanks Anthony Starks).
+* net: implement non-blocking connect (thanks Alexey Borzenkov).
+* os: fix MkdirAll("/thisdoesnotexist") (thanks Albert Strasheim),
+	Plan 9 support (thanks Yuval Pavel Zholkover),
+	add a few missing Plan 9 errors (thanks Andrey Mirtchovski),
+	fix FileInfo.Name returned by Stat (thanks David Forsythe).
+* path/filepath.Glob: add an error return,
+	don't drop known matches on error.
+* path/filepath: add support for Plan 9 (thanks Andrey Mirtchovski).
+* scanner: treat line comments like in Go.
+* syscall: Plan 9 support (thanks Yuval Pavel Zholkover),
+	StartProcess Chroot and Credential (thanks Albert Strasheim),
+	add BPF support for freebsd/386, freebsd/amd64 (thanks Mikio Hara),
+	make [Raw]Syscall6 pass 6th arg on linux/386 (thanks Evan Shaw).
+
+ +

2011-03-28

+ +
+This weekly release includes improved support for testing.
+
+Memory and CPU profiling is now available via the gotest tool. Gotest will
+produce memory and CPU profiling data when invoked with the -test.memprofile
+and -test.cpuprofile flags. Run "godoc gotest" for details.
+
+We have also introduced a way for tests to run quickly when an exhaustive test
+is unnecessary. Gotest's new -test.short flag in combination with the testing
+package's new Short function allows you to write tests that can be run in
+normal or "short" mode; short mode is now used by all.bash to reduce
+installation time.
+The Makefiles know about the flag - you can just run "make testshort".
+
+Other changes:
+* .hgignore: Ignore all goinstalled packages (thanks Evan Shaw).
+* build: add all-qemu.bash, handful of arm fixes,
+        add support for SWIG, and add two SWIG examples,
+        diagnose Ubuntu's buggy copy of gold,
+        handle broken awk in version.bash (thanks Dave Cheney),
+        reenable clean.bash without gomake (thanks Gustavo Niemeyer).
+* cgo: fix index-out-of-bounds bug.
+* codereview: permit CLs of the form weekly.DATE
+* crypto/ecdsa: truncate hash values.
+* crypto/openpgp: add DSA signature support.
+* dashboard: remove old python/bash builder, update README.
+* doc: explain release and weekly tags in install.html.
+* exec: document dir option for Run (thanks Gustavo Niemeyer).
+* flag: document Nflag function (thanks Fazlul Shahriar).
+* gc: remove interim ... error which rejects valid code.
+* go/ast: implemented NewPackage,
+        merge CaseClause and TypeCaseClause.
+* go/parser: fix memory leak by making a copy of token literals,
+        resolve identifiers properly.
+* go/printer, gofmt: avoid exponential layout algorithm,
+        gofmt: simplify struct formatting and respect line breaks.
+* go/scanner: to interpret line comments with Windows filenames (thanks Alex Brainman).
+* go/token: use array instead of map for token->string table.
+* gob: optimizations to reduce allocations,
+        use pointers in bootstrapType so interfaces behave properly.
+* gobuilder: recognize CLs of the form weekly.DATE.
+* godefs: handle volatile.
+* godoc: add -template flag to specify custom templates,
+        fix path problem for windows (thanks Yasuhiro Matsumoto).
+* gofix: httpserver - rewrite rw.SetHeader to rw.Header.Set.
+* gofmt: add profiling flag.
+* gopprof: fix bug: do not rotate 180 degrees for large scrolls,
+        update list of memory allocation functions.
+* gotest: fix gofmt issue in generated _testmain.go.
+* http: add NewProxyClientConn,
+        avoid crash when asked for multiple file ranges,
+        don't chunk 304 responses,
+        export Transport, add keep-alive support.
+* ld: return > 0 exit code on unsafe import.
+* misc/bbedit: remove closed keyword (thanks Anthony Starks).
+* misc/emacs: gofmt: don't clobber the current buffer on failure.
+* misc/vim: remove 'closed' as a builtin function.
+* net: add FileConn, FilePacketConn, FileListener (thanks Albert Strasheim),
+        don't force epoll/kqueue to wake up in order to add new events,
+        let OS-specific AddFD routine wake up polling thread,
+        use preallocated buffer for epoll and kqueue/kevent.
+* path/filepath: add EvalSymlinks function,
+        fix TestEvalSymlinks when run under symlinked GOROOT.
+* path: work for windows (thanks Yasuhiro Matsumoto).
+* rpc: increase server_test timeout (thanks Gustavo Niemeyer),
+        optimizations to reduce allocations.
+* runtime: fix darwin/amd64 thread VM footprint (thanks Alexey Borzenkov),
+        fix gdb support for goroutines,
+        more stack split fixes,
+        os-specific types and code for setitimer,
+        update defs.h for freebsd-386 (thanks Devon H. O'Dell).
+* strings: Map: avoid allocation when string is unchanged.
+* syscall: GetsockoptInt (thanks Albert Strasheim),
+        StartProcess fixes for windows (thanks Alex Brainman),
+        permit non-blocking syscalls,
+        rename from .sh to .pl, because these files are in Perl.
+* test: enable tests using v, ok := <-ch syntax (thanks Robert Hencke).
+* time: give a helpful message when we can't set the time zone for testing.
+        isolate syscall reference in sys.go.
+
+ +

2011-03-15

+ +
+This week's release introduces a new release tagging scheme. We intend to
+continue with our weekly releases, but have renamed the existing tags from
+"release" to "weekly". The "release" tag will now be applied to one hand-picked
+stable release each month or two.
+
+The revision formerly tagged "release.2011-03-07.1" (now "weekly.2011-03-07.1")
+has been nominated our first stable release, and has been given the tag
+"release.r56". As we tag each stable release we will post an announcement to
+the new golang-announce mailing list:
+  http://groups.google.com/group/golang-announce
+
+You can continue to keep your Go installation updated using "hg update
+release", but now you should only need to update once we tag a new stable
+release, which we will announce here. If you wish to stay at the leading edge,
+you should switch to the weekly tag with "hg update weekly".
+
+
+This weekly release includes significant changes to the language spec and the
+http, os, and syscall packages. Your code may need to be changed. It also
+introduces the new gofix tool.
+
+The closed function has been removed from the language. The syntax for channel
+receives has been changed to return an optional second value, a boolean value
+indicating whether the channel is closed. This code:
+	v := <-ch
+	if closed(ch) {
+		// channel is closed
+	}
+should now be written as:
+	v, ok := <-ch
+	if !ok {
+		// channel is closed
+	}
+
+It is now illegal to declare unused labels, just as it is illegal to declare
+unused local variables.
+
+The new gofix tool finds Go programs that use old APIs and rewrites them to use
+newer ones.  After you update to a new Go release, gofix helps make the
+necessary changes to your programs. Gofix will handle the http, os, and syscall
+package changes described below, and we will update the program to keep up with
+future changes to the libraries.
+
+The Hijack and Flush methods have been removed from the http.ResponseWriter
+interface and are accessible via the new http.Hijacker and http.Flusher
+interfaces. The RemoteAddr and UsingTLS methods have been moved from
+http.ResponseWriter to http.Request.
+
+The http.ResponseWriter interface's SetHeader method has been replaced by a
+Header() method that returns the response's http.Header. Caller code needs to
+change. This code:
+	rw.SetHeader("Content-Type", "text/plain")
+should now be written as:
+	rw.Header().Set("Content-Type", "text/plain")
+The os and syscall packages' StartProcess functions now take their final three
+arguments as an *os.ProcAttr and *syscall.ProcAttr values, respectively. This
+code:
+	os.StartProcess(bin, args, env, dir, fds)
+should now be written as:
+	os.StartProcess(bin, args, &os.ProcAttr{Files: fds, Dir: dir, Env: env})
+
+The gob package will now encode and decode values of types that implement the
+gob.GobEncoder and gob.GobDecoder interfaces. This allows types with unexported
+fields to transmit self-consistent descriptions; one instance is big.Int and
+big.Rat.
+
+Other changes:
+* 5l, 6l, 8l: reduce binary size about 40% by omitting symbols for type, string, go.string.
+* 5l, 8l: output missing section symbols (thanks Anthony Martin).
+* 6l, 8l: fix gdb crash.
+* Make.cmd: also clean _test* (thanks Gustavo Niemeyer).
+* big: implemented custom Gob(En/De)coder for Int type.
+* build: remove duplicate dependency in Make.cmd (thanks Robert Hencke),
+        run gotest in misc/cgo/test.
+* codereview.py: don't suggest change -d if user is not CL author (thanks Robert Hencke).
+* compress/lzw: benchmark a range of input sizes.
+* crypto/ecdsa: add package.
+* crypto/elliptic: add the N value of each curve.
+* crypto/openpgp: bug fixes and fix misnamed function.
+* crypto/tls: fix compile error (thanks Dave Cheney).
+* doc: Effective Go: some small cleanups,
+        update FAQ. hello, world is now 1.1MB, down from 1.8MB,
+        update codelab wiki to fix template.Execute argument order.
+* flag: visit the flags in sorted order, for nicer messages.
+* fmt: do not export EOF = -1.
+* fmt: make ScanState.Token more general (thanks Roger Peppe).
+* gc: diagnose unused labels,
+        fix handling of return values named _,
+        include all dependencies in export metadata,
+        make unsafe.Pointer its own kind of type, instead of an equivalent to *any.
+* go/ast, go/parser: populate identifier scopes at parse time.
+* go/ast: add FileSet parameter to ast.Print and ast.Fprint.
+* go/parser: first constant in a constant declaration must have a value.
+* gob: efficiency and reliability fixes.
+* gofmt: remove -trace and -ast flags.
+* goinstall: handle $(GOOS) and $(GOARCH) in filenames,
+        handle .c files with gc when cgo isn't used, and
+        handle .s files with gc (thanks Gustavo Niemeyer).
+* gopack: omit time stamps, makes output deterministic.
+* gotype: commandline tool to typecheck go programs.
+* govet: handle '*' in print format strings.
+* hash: new FNV-1a implementation (thanks Pascal S. de Kloe).
+* http/cgi: child support (e.g. Go CGI under Apache).
+* http: adapt Cookie code to follow IETF draft (thanks Petar Maymounkov),
+        add test for fixed HTTP/1.0 keep-alive issue,
+        don't hit external network in client_test.go,
+        fix transport crash when request URL is nil,
+        rename interface Transport to RoundTripper,
+        run tests even with DISABLE_NET_TESTS=1.
+* httptest: default the Recorder status code to 200 on a Write.
+* io/ioutil: clean-up of ReadAll and ReadFile.
+* ioutil: add NopCloser.
+* ld: preserve symbol sizes during data layout.
+* lib9, libmach: Change GOOS references to GOHOSTOS (thanks Evan Shaw).
+* libmach: correct string comparison to revive 6cov on darwin (thanks Dave Cheney).
+* misc/vim: Add indent script for Vim (thanks Ross Light).
+* net, os, syslog: fixes for Solaris support.
+* net: don't loop to drain wakeup pipe.
+* nm: document -S flag.
+* openpgp: add PublicKey KeyId string accessors.
+* rpc: optimizations, add benchmarks and memory profiling,
+        use httptest.Server for tests (thanks Robert Hencke).
+* runtime: reduce lock contention via wakeup on scheduler unlock,
+        scheduler, cgo reorganization,
+        split non-debugging malloc interface out of debug.go into mem.go.
+* spec: clarify return statement rules.
+* strings: add IndexRune tests, ASCII fast path,
+        better benchmark names; add BenchmarkIndex.
+* syscall: implement Mount and Unmount for linux,
+        implement Reboot for linux.
+* time: fix Time.ZoneOffset documentation (thanks Peter Mundy).
+* tls: move PeerCertificates to ConnectionState.
+
+ +

2011-03-07 (base for r56)

+ +
+This release includes changes to the reflect and path packages.
+Code that uses reflect or path may need to be updated.
+
+The reflect package's Value.Addr method has been renamed to Value.UnsafeAddr.
+Code that uses the Addr method will have to call UnsafeAddr instead.
+
+The path package has been split into two packages: path and path/filepath.
+Package path manipulates slash-separated paths, regardless of operating system.
+Package filepath implements the local operating system's native file paths.
+OS-specific functioanlity in pacakge path, such as Walk, moved to filepath.
+
+Other changes:
+* build: fixes and simplifications (thanks Dave Cheney),
+        move $GOBIN ahead of /bin, /usr/bin in build $PATH.
+* bzip2: speed up decompression.
+* cgo: fix dwarf type parsing (thanks Gustavo Niemeyer),
+        put temporary source files in _obj (thanks Roger Peppe),
+        fix bug involving 0-argument callbacks.
+* compress/lzw: optimizations.
+* doc: add FAQ about "implements",
+        add FAQ about large binaries ,
+        add FAQ about stack vs heap allocation,
+        add internationalization to roadmap,
+        describe platform-specific conventions in code.html.
+* fmt: allow recursive calls to Fscan etc (thanks Roger Peppe),
+        make %#p suppress leading 0x.
+* gc, gopack: add some missing flags to the docs.
+* gc: fix init of packages named main (thanks Gustavo Niemeyer),
+* gob: make recursive map and slice types work, and other fixes.
+        tentative support for GobEncoder/GobDecoder interfaces.
+* gobuilder: add -package flag to build external packages and -v for verbose.
+* gofmt: exclude test file that is not legal Go.
+* goinstall: protect against malicious filenames (thanks Roger Peppe).
+* goyacc: provide -p flag to set prefix for names, documentation update.
+* http: add cookie support (thanks Petar Maymounkov),
+        allow handlers to send non-chunked responses,
+        export ParseHTTPVersion,
+        expose Client's Transport,
+        use WriteProxy,
+        rename ClientTransport to Transport.
+* http/cgi: new package.
+* http/httptest: new package.
+* image: add a decoding test for common file formats.
+* io/ioutil: add TempDir.
+* mime/multipart: Header changed from map to MIMEHeader
+* path/filepath: new OS-specific path support (thanks Gustavo Niemeyer).
+* reflect: add PtrTo, add Value.Addr (old Addr is now UnsafeAddr).
+* runtime: use kernel-supplied compare-and-swap on linux/arm.
+* spec: minor clarification of scope rule for functions.
+* sync/atomic: new package to expose atomic operations.
+* syscall: regenerate zerrors_freebsd_amd64.go (thanks Mikio Hara),
+        work around FreeBSD execve kernel bug (thanks Devon H. O'Dell).
+* template: document the delimiters.
+* testing: run GC before each benchmark run (thanks Roger Peppe).
+* unsafe: fix the documentation.
+* websocket: use httptest.Server for tests (thanks Robert Hencke).
+* xml: permit nested directives (thanks Chris Dollin).
+
+ +

2011-02-24

+ +
+This release includes changes to the http package and a small language change.
+Your code will require changes if it manipulates http Headers or omits the
+condition in if statements.
+
+The new http.Header type replaces map[string]string in the Header and Trailer
+fields of http.Request and http.Response.
+A Header value can be manipulated via its Get, Set, Add, and Del methods.
+See http://golang.org/pkg/http/#Header
+
+The condition is now mandatory in if statements.
+Previously it would default to true, as in switch and for statements.
+This code is now illegal:
+	if x := foo(); {
+		// code that is always executed
+	}
+The same effect can be achieved like this:
+	if x := foo(); true {
+		// code
+	}
+Or, in a simpler form:
+	{
+		x := foo()
+		// code
+	}
+
+Other changes:
+* 6l: new -Hwindowsgui flag allows to build windows gui pe (thanks Alex Brainman),
+	pe fixes (thanks Wei Guangjing).
+* 8l, 6l: allow for more os threads to be created on Windows (thanks Alex Brainman),
+* build: reduce the use of subshells in recursive make, and
+	remove unused NaCl conditional from make.bash (thanks Dave Cheney).
+* codereview: fix clpatch with empty diffs (thanks Gustavo Niemeyer).
+* compress/bzip2: add package.
+* compress/lzw: implement a decoder.
+* crypto/openpgp: add package.
+* crypto/rand: add read buffer to speed up small requests (thanks Albert Strasheim).
+* crypto/rsa: left-pad OAEP results when needed.
+* crypto/tls: make protocol negotiation failure fatal.
+* fmt: stop giving characters to the Scan method of Scanner when we hit a newline in Scanln.
+* gc: interface error message fixes,
+	make string const comparison unsigned (thanks Jeff R. Allen).
+* go spec: minor clarification on channel types.
+* go/ast, parser: condition in if statement is mandatory.
+* gob: compute information about a user's type once.
+	protect against pure recursive types.
+* godoc: accept symbolic links as path names provided to -path,
+	add robots.txt, log errors when reading filter files.
+* html: tokenize HTML comments.
+* http: add proxy support (thanks Yasuhiro Matsumoto),
+	implement with net/textproto (thanks Petar Maymounkov),
+	send full URL in proxy requests,
+	introduce start of Client and ClientTransport.
+* image/png: support for more formats (thanks Mikael Tillenius).
+* json: only use alphanumeric tags,
+	use base64 to encode []byte (thanks Roger Peppe).
+* ld: detect stack overflow due to NOSPLIT, drop rpath, support weak symbols.
+* misc/dashboard/builder: talk to hg with utf-8 encoding.
+* misc/dashboard: notify golang-dev on build failure.
+* net: *netFD.Read to return os.EOF on eof under windows (thanks Alex Brainman),
+	add IPv4 multicast to UDPConn (thanks Dave Cheney),
+	more accurate IPv4-in-IPv6 API test (thanks Mikio Hara),
+	reject invalid net:proto network names (thanks Olivier Antoine).
+* netchan: allow use of arbitrary connections (thanks Roger Peppe).
+* os: add ENODATA and ENOTCONN (thanks Albert Strasheim).
+* reflect: add a couple of sentences explaining how Methods operate,
+	add a secret method to ArrayOrSliceType to ensure it's only implemented by arrays and slices,
+	add pointer word to CommonType (placeholder for future work).
+* runtime-gdb.py: gdb pretty printer for go strings properly handles length.
+* runtime: various bug fixes, more complete stack traces,
+	record $GOROOT_FINAL for runtime.GOROOT.
+* spec: delete incorrect mention of selector working on pointer to interface type.
+* sync: add Cond (thanks Gustavo Niemeyer).
+* syscall: add MCL_* flags for mlockall (thanks Albert Strasheim),
+	implement chmod() for win32 (thanks Yasuhiro Matsumoto).
+* test/bench: update timings for new GC.
+* testing: rename cmdline flags to avoid conflicts (thanks Gustavo Niemeyer).
+* textproto: introduce Header type (thanks Petar Maymounkov).
+* websocket: use new interface to access Header.
+
+ +

2011-02-15

+ +
+This release includes changes to the io, os, and template packages.
+You may need to update your code.
+
+The io.ReadByter and io.ReadRuner interface types have been renamed to
+io.ByteReader and io.RuneReader respectively.
+
+The os package's ForkExec function has been superseded by the new StartProcess
+function and an API built around the Process type:
+	http://golang.org/pkg/os/#Process
+
+The order of arguments to template.Execute has been reversed to be consistent
+the notion of "destination first", as with io.Copy, fmt.Fprint, and others.
+
+Gotest now works for package main in directories using Make.cmd-based makefiles.
+
+The memory allocation runtime problems from the last release are not completely
+fixed.  The virtual memory exhaustion problems encountered by people using
+ulimit -v have been fixed, but there remain known garbage collector problems
+when using GOMAXPROCS > 1.
+
+Other changes:
+* 5l: stopped generating 64-bit eor.
+* 8l: more work on plan9 support (thanks Yuval Pavel Zholkover).
+* archive/zip: handle files with data descriptors.
+* arm: working peep-hole optimizer.
+* asn1: marshal true as 255, not 1.
+* buffer.go: minor optimization, expanded comment.
+* build: drop syslog on DISABLE_NET_TESTS=1 (thanks Gustavo Niemeyer),
+       allow clean.bash to work on fresh checkout,
+       change "all tests pass" message to be more obvious,
+       fix spaces in GOROOT (thanks Christopher Nielsen).
+* bytes: fix bug in buffer.ReadBytes (thanks Evan Shaw).
+* 5g: better int64 code,
+       don't use MVN instruction.
+* cgo: don't run cgo when not compiling (thanks Gustavo Niemeyer),
+       fix _cgo_run timestamp file order (thanks Gustavo Niemeyer),
+       fix handling of signed enumerations (thanks Gustavo Niemeyer),
+       os/arch dependent #cgo directives (thanks Gustavo Niemeyer),
+       rename internal f to avoid conflict with possible C global named f.
+* codereview: fix hgpatch on windows (thanks Yasuhiro Matsumoto),
+       record repository, base revision,
+       use cmd.communicate (thanks Yasuhiro Matsumoto).
+* container/ring: replace Iter() with Do().
+* crypto/cipher: add resync open to OCFB mode.
+* crypto/openpgp/armor: bug fixes.
+* crypto/openpgp/packet: new subpackage.
+* crypto/tls: load a chain of certificates from a file,
+       select best cipher suite, not worst.
+* crypto/x509: add support for name constraints.
+* debug/pe: ImportedSymbols fixes (thanks Wei Guangjing).
+* doc/code: update to reflect that package names need not be unique.
+* doc/codelab/wiki: a bunch of fixes (thanks Andrey Mirtchovski).
+* doc/install: update for new versions of Mercurial.
+* encoding/line: fix line returned after EOF.
+* flag: allow hexadecimal (0xFF) and octal (0377) input for integer flags.
+* fmt.Scan: scan binary-exponent floating format, 2.4p-3,
+       hexadecimal (0xFF) and octal (0377) integers.
+* fmt: document %%; also %b for floating point.
+* gc, ld: detect stale or incompatible object files,
+       package name main no longer reserved.
+* gc: correct receiver in method missing error (thanks Lorenzo Stoakes),
+       correct rounding of denormal constants (thanks Eoghan Sherry),
+       select receive bug fix.
+* go/printer, gofmt: smarter handling of multi-line raw strings.
+* go/printer: line comments must always end in a newline,
+       remove notion of "Styler", remove HTML mode.
+* gob: allow Decode(nil) and have it just discard the next value.
+* godoc: use IsAbs to test for absolute paths (fix for win32) (thanks Yasuhiro Matsumoto),
+       don't hide package lookup error if there's no command with the same name.
+* gotest: enable unit tests for main programs.
+* http: add Server type supporting timeouts,
+       add pipelining to ClientConn, ServerConn (thanks Petar Maymounkov),
+       handle unchunked, un-lengthed HTTP/1.1 responses.
+* io: add RuneReader.
+* json: correct Marshal documentation.
+* netchan: graceful handling of closed connection (thanks Graham Miller).
+* os: implement new Process API (thanks Alex Brainman).
+* regexp tests: make some benchmarks more meaningful.
+* regexp: add support for matching against text read from RuneReader interface.
+* rpc: make more tolerant of errors, properly discard values (thanks Roger Peppe).
+* runtime: detect failed thread creation on Windows,
+       faster allocator, garbage collector,
+       fix virtual memory exhaustion,
+       implemented windows console ctrl handler (SIGINT) (thanks Hector Chu),
+       more detailed panic traces, line number work,
+       improved Windows callback handling (thanks Hector Chu).
+* spec: adjust notion of Assignability,
+       allow import of packages named main,
+       clarification re: method sets of newly declared pointer types,
+       fix a few typos (thanks Anthony Martin),
+       fix Typeof() return type (thanks Gustavo Niemeyer),
+       move to Unicode 6.0.
+* sync: diagnose Unlock of unlocked Mutex,
+       new Waitgroup type (thanks Gustavo Niemeyer).
+* syscall: add SetsockoptIpMreq (thanks Dave Cheney),
+       add sockaddr_dl, sysctl with routing message support for darwin, freebsd (thanks Mikio Hara),
+       do not use NULL for zero-length read, write,
+       implement windows version of Fsync (thanks Alex Brainman),
+       make ForkExec acquire the ForkLock under windows (thanks Hector Chu),
+       make windows API return errno instead of bool (thanks Alex Brainman),
+       remove obsolete socket IO control (thanks Mikio Hara).
+* template: add simple formatter chaining (thanks Kyle Consalus),
+       allow a leading '*' to indirect through a pointer.
+* testing: include elapsed time in test output
+* windows: replace remaining __MINGW32__ instances with _WIN32 (thanks Joe Poirier).
+
+ +

2011-02-01

+ +
+This release includes significant changes to channel operations and minor
+changes to the log package. Your code will require modification if it uses
+channels in non-blocking communications or the log package's Exit functions.
+
+Non-blocking channel operations have been removed from the language.
+The equivalent operations have always been possible using a select statement
+with a default clause.  If a default clause is present in a select, that clause
+will execute (only) if no other is ready, which allows one to avoid blocking on
+a communication.
+
+For example, the old non-blocking send operation,
+
+	if ch <- v {
+		// sent
+	} else {
+		// not sent
+	}
+
+should be rewritten as,
+
+	select {
+	case ch <- v:
+		// sent
+	default:
+		// not sent
+	}
+
+Similarly, this receive,
+
+	v, ok := <-ch
+	if ok {
+		// received
+	} else {
+		// not received
+	}
+
+should be rewritten as,
+
+	select {
+	case v := <-ch:
+		// received
+	default:
+		// not received
+	}
+
+This change is a prelude to redefining the 'comma-ok' syntax for a receive.
+In a later release, a receive expression will return the received value and an
+optional boolean indicating whether the channel has been closed. These changes
+are being made in two stages to prevent this semantic change from silently
+breaking code that uses 'comma-ok' with receives.
+There are no plans to have a boolean expression form for sends.
+
+Sends to a closed channel will panic immediately. Previously, an unspecified
+number of sends would fail silently before causing a panic.
+
+The log package's Exit, Exitf, and Exitln functions have been renamed Fatal,
+Fatalf, and Fatalln respectively. This brings them in line with the naming of
+the testing package. 
+
+The port to the "tiny" operating system has been removed. It is unmaintained
+and untested. It was a toy to show that Go can run on raw hardware and it
+served its purpose. The source code will of course remain in the repository
+history, so it could be brought back if needed later.
+
+This release also changes some of the internal structure of the memory
+allocator in preparation for other garbage collector changes. 
+If you run into problems, please let us know.
+There is one known issue that we are aware of but have not debugged yet:
+	http://code.google.com/p/go/issues/detail?id=1464&.
+
+Other changes in this release:
+* 5l: document -F, force it on old ARMs (software floating point emulation)
+* 6g: fix registerization of temporaries (thanks Eoghan Sherry),
+        fix uint64(uintptr(unsafe.Pointer(&x))).
+* 6l: Relocate CMOV* instructions (thanks Gustavo Niemeyer),
+        windows/amd64 port (thanks Wei Guangjing).
+* 8l: add PE dynexport, emit DWARF in Windows PE, and
+        code generation fixes (thanks Wei Guangjing).
+* bufio: make Flush a no-op when the buffer is empty.
+* bytes: Add Buffer.ReadBytes, Buffer.ReadString (thanks Evan Shaw).
+* cc: mode to generate go-code for types and variables.
+* cgo: define CGO_CFLAGS and CGO_LDFLAGS in Go files (thanks Gustavo Niemeyer),
+        windows/386 port (thanks Wei Guangjing).
+* codereview: fix windows (thanks Hector Chu),
+        handle file patterns better,
+        more ASCII vs. Unicode nonsense.
+* crypto/dsa: add support for DSA.
+* crypto/openpgp: add s2k.
+* crypto/rand: use defer to unlock mutex (thanks Anschel Schaffer-Cohen).
+* crypto/rsa: correct docstring for SignPKCS1v15.
+* crypto: add package, a common place to store identifiers for hash functions.
+* doc/codelab/wiki: update to work with template changes, add to run.bash.
+* doc/spec: clarify address operators.
+* ebnflint: exit with non-zero status on error.
+* encoding/base32: new package (thanks Miek Gieben).
+* encoding/line: make it an io.Reader too.
+* exec: use custom error for LookPath (thanks Gustavo Niemeyer).
+* fmt/doc: define width and precision for strings.
+* gc: clearer error for struct == struct,
+        fix send precedence,
+        handle invalid name in type switch,
+        special case code for single-op blocking and non-blocking selects.
+* go/scanner: fix build (adjust scanner EOF linecount).
+* gob: better debugging, commentary,
+        make nested interfaces work,
+        report an error when encoding a non-empty struct with no public fields.
+* godoc: full text index for whitelisted non-Go files,
+        show line numbers for non-go files (bug fix).
+* gofmt -r: match(...) arguments may be nil; add missing guards.
+* govet: add Panic to the list of functions.
+* http: add host patterns (thanks Jose Luis Vázquez González),
+        follow relative redirect in Get.
+* json: handle capital floating point exponent (1E100) (thanks Pieter Droogendijk).
+* ld: add -I option to set ELF interpreter,
+        more robust decoding of reflection type info in generating dwarf.
+* lib9: update to Unicode 6.0.0.
+* make.bash: stricter selinux test (don't complain unless it is enabled).
+* misc/vim: Import/Drop commands (thanks Gustavo Niemeyer),
+        set 'syntax sync' to a large value (thanks Yasuhiro Matsumoto).
+* net: fix race condition in test,
+        return cname in LookupHost.
+* netchan: avoid race condition in test,
+        fixed documentation for import (thanks Anschel Schaffer-Cohen).
+* os: add ETIMEDOUT (thanks Albert Strasheim).
+* runtime: generate Go defs for C types,
+        implementation of callback functions for windows (thanks Alex Brainman),
+        make Walk web browser example work (thanks Hector Chu),
+        make select fairer,
+        prefer fixed stack allocator over general memory allocator,
+        simpler heap map, memory allocation.
+* scanner: fix Position returned by Scan, Pos,
+        don't read ahead in Init.
+* suffixarray: use binary search for both ends of Lookup (thanks Eric Eisner).
+* syscall: add missing network interface constants (thanks Mikio Hara).
+* template: treat map keys as zero, not non-existent (thanks Roger Peppe).
+* time: allow cancelling of After events (thanks Roger Peppe),
+        support Solaris zoneinfo directory.
+* token/position: added SetLinesForContent.
+* unicode: update to unicode 6.0.0.
+* unsafe: add missing case to doc for Pointer.
+
+ +

2011-01-20

+ +
+This release removes the float and complex types from the language.
+
+The default type for a floating point literal is now float64, and
+the default type for a complex literal is now complex128.
+
+Existing code that uses float or complex must be rewritten to
+use explicitly sized types.
+
+The two-argument constructor cmplx is now spelled complex.
+
+ +

2011-01-19

+ +
+The 5g (ARM) compiler now has registerization enabled.  If you discover it
+causes bugs, use 5g -N to disable the registerizer and please let us know.
+
+The xml package now allows the extraction of nested XML tags by specifying
+struct tags of the form "parent>child". See the XML documentation for an
+example: http://golang.org/pkg/xml/
+
+* 5a, 5l, 6a, 6l, 8a, 8l: handle out of memory, large allocations (thanks Jeff R. Allen).
+* 8l: pe changes (thanks Alex Brainman).
+* arm: fixes and improvements.
+* cc: fix vlong condition.
+* cgo: add complex float, complex double (thanks Sebastien Binet),
+        in _cgo_main.c define all provided symbols as functions.
+* codereview: don't mail change lists with no files (thanks Ryan Hitchman).
+* crypto/cipher: add OFB mode.
+* expvar: add Float.
+* fmt: document %X of string, []byte.
+* gc, runtime: make range on channel safe for multiple goroutines.
+* gc: fix typed constant declarations (thanks Anthony Martin).
+* go spec: adjust language for constant typing.
+* go/scanner: Make Init take a *token.File instead of a *token.FileSet.
+* godoc: bring back "indexing in progress" message,
+        don't double HTML-escape search result snippets,
+        enable qualified identifiers ("math.Sin") as query strings again,
+        peephole optimization for generated HTML,
+        remove tab before formatted section.
+* gofmt, go/printer: do not insert extra line breaks where they may break the code.
+* http: fix Content-Range and Content-Length in response (thanks Clement Skau),
+        fix scheme-relative URL parsing; add ParseRequestURL,
+        handle HEAD requests correctly,
+        support for relative URLs.
+* math: handle denormalized numbers in Frexp, Ilogb, Ldexp, and Logb (thanks Eoghan Sherry).
+* net, syscall: return source address in Recvmsg (thanks Albert Strasheim).
+* net: add LookupAddr (thanks Kyle Lemons),
+        add unixpacket (thanks Albert Strasheim),
+        avoid nil dereference if /etc/services can't be opened (thanks Corey Thomasson),
+        implement windows timeout (thanks Wei Guangjing).
+* netchan: do not block sends; implement flow control (thanks Roger Peppe).
+* regexp: reject bare '?'. (thanks Ben Lynn)
+* runtime/cgo: don't define crosscall2 in dummy _cgo_main.c.
+* runtime/debug: new package for printing stack traces from a running goroutine.
+* runtime: add per-pause gc stats,
+        fix arm reflect.call boundary case,
+        print signal information during panic.
+* spec: specify that int and uint have the same size.
+* syscall: correct WSTOPPED on OS X,
+        correct length of GNU/Linux abstract Unix domain sockaddr,
+        correct length of SockaddrUnix.
+* tutorial: make stdin, stdout, stderr work on Windows.
+* windows: implement exception handling (thanks Hector Chu).
+
+ +

2011-01-12

+ +
+The json, gob, and template packages have changed, and code that uses them
+may need to be updated after this release. They will no longer read or write
+unexported struct fields. When marshalling a struct with json or gob the
+unexported fields will be silently ignored. Attempting to unmarshal json or
+gob data into an unexported field will generate an error. Accessing an
+unexported field from a template will cause the Execute function to return
+an error.
+
+Godoc now supports regular expression full text search, and this
+functionality is now available on golang.org.
+
+Other changes:
+* arm: initial cut at arm optimizer.
+* bytes.Buffer: Fix bug in UnreadByte.
+* cgo: export unsafe.Pointer as void*, fix enum const conflict,
+        output alignment fix (thanks Gustavo Niemeyer).
+* crypto/block: mark as deprecated.
+* crypto/openpgp: add error and armor.
+* crypto: add twofish package (thanks Berengar Lehr).
+* doc/spec: remove Maxalign from spec.
+* encoding/line: new package for reading lines from an io.Reader.
+* go/ast: correct end position for Index and TypeAssert expressions.
+* gob: make (en|dec)code(Ui|I)nt methods rather than functions.
+* godefs: better handling of enums.
+* gofmt: don't attempt certain illegal rewrites,
+        rewriter matches apply to expressions only.
+* goinstall: preliminary support for cgo packages (thanks Gustavo Niemeyer).
+* hg: add cgo/_cgo_* to .hgignore.
+* http: fix text displayed in Redirect.
+* ld: fix exported dynamic symbols on Mach-O,
+        permit a Mach-O symbol to be exported in the dynamic symbol table.
+* log: add methods for exit and panic.
+* net: use closesocket api instead of CloseHandle on Windows (thanks Alex Brainman).
+* netchan: make fields exported for gob change.
+* os: add Sync to *File, wraps syscall.Fsync.
+* runtime/cgo: Add callbacks to support SWIG.
+* runtime: Restore scheduler stack position if cgo callback panics.
+* suffixarray: faster creation algorithm (thanks Eric Eisner).
+* syscall: fix mksysnum_linux.sh (thanks Anthony Martin).
+* time.NewTicker: panic for intervals <= 0.
+* time: add AfterFunc to call a function after a duration (thanks Roger Peppe),
+        fix tick accuracy when using multiple Tickers (thanks Eoghan Sherry).
+ +

2011-01-06

+ +
+This release includes several fixes and changes:
+
+* build: Make.pkg: use installed runtime.h for cgo.
+* cgo: disallow use of C.errno.
+* crypto/cipher: fix OCFB,
+        make NewCBCEncrypter return BlockMode.
+* doc: 6l: fix documentation of -L flag,
+        add golanguage.ru to foreign-language doc list,
+        effective go: explain the effect of repanicking better,
+        update Effective Go for template API change,
+        update contribution guidelines to prefix the change description.
+* encoding/binary: reject types with implementation-dependent sizes (thanks Patrick Gavlin).
+* exp/evalsimple fix handling of slices like s[:2] (thanks Sebastien Binet).
+* fmt: made format string handling more efficient,
+        normalize processing of format string.
+* gc: return constant floats for parts of complex constants (thanks Anthony Martin),
+        rewrite complex /= to l = l / r (thanks Patrick Gavlin),
+        fix &^=.
+* go/ast: provide complete node text range info.
+* gob: generate a better error message in one confusing place.
+* godoc: fix godoc -src (thanks Icarus Sparry).
+* goinstall: add -clean flag (thanks Kyle Lemons),
+        add checkout concept (thanks Caine Tighe),
+        fix -u for bzr (thanks Gustavo Niemeyer).
+* http: permit empty Reason-Phrase in response Status-Line.
+* io: fix Copyn EOF handling.
+* net: fix close of Listener (thanks Michael Hoisie).
+* regexp: fix performance bug, make anchored searches fail fast,
+        fix prefix bug.
+* runtime/cgo: fix stackguard on FreeBSD/amd64 (thanks Anthony Martin).
+* strconv: atof: added 'E' as valid token for exponent (thanks Stefan Nilsson),
+        update ftoa comment for 'E' and 'G'.
+* strings: fix description of FieldsFunc (thanks Roger Peppe).
+* syscall: correct Linux Splice definition,
+        make Access second argument consistently uint32.
+
+ +

2010-12-22

+ +
+A small release this week. The most significant change is that some 
+outstanding cgo issues were resolved.
+
+* cgo: handle references to symbols in shared libraries.
+* crypto/elliptic: add serialisation and key pair generation.
+* crypto/hmac: add HMAC-SHA256 (thanks Anthony Martin).
+* crypto/tls: add ECDHE support ("Elliptic Curve Diffie Hellman Ephemeral"),
+        add support code for generating handshake scripts for testing.
+* darwin, freebsd: ignore write failure (during print, panic).
+* exp/draw: remove Border function.
+* expvar: quote StringFunc output, same as String output.
+* hash/crc64: fix typo in Sum.
+* ld: allow relocations pointing at ELF .bss symbols, ignore stab symbols.
+* misc/cgo/life: fix, add to build.
+* regexp: add HasMeta, HasOperator, and String methods to Regexp.
+* suffixarray: implemented FindAllIndex regexp search.
+* test/bench: update numbers for regex-dna after speedup to regexp.
+* time: explain the formats a little better.
+
+ +

2010-12-15

+ +
+Package crypto/cipher has been started, to replace crypto/block.
+As part of the changes, rc4.Cipher's XORKeyStream method signature has changed from
+        XORKeyStream(buf []byte)
+to
+        XORKeyStream(dst, src []byte)
+to implement the cipher.Stream interface.  If you use crypto/block, you'll need
+to switch to crypto/cipher once it is complete.
+
+Package smtp's StartTLS now takes a *tls.Config argument.
+
+Package reflect's ArrayCopy has been renamed to Copy.  There are new functions
+Append and AppendSlice.
+
+The print/println bootstrapping functions now write to standard error.
+To write to standard output, use fmt.Print[ln].
+
+A new tool, govet, has been added to the Go distribution. Govet is a static
+checker for Go programs. At the moment, and for the forseeable future,
+it only checks arguments to print calls.
+
+The cgo tool for writing Go bindings for C code has changed so that it no
+longer uses stub .so files (like cgo_stdio.so).  Cgo-based packages using the
+standard Makefiles should build without any changes.  Any alternate build
+mechanisms will need to be updated.
+
+The C and Go compilers (6g, 6c, 8g, 8c, 5g, 5c) now align structs according to
+the maximum alignment of the fields they contain; previously they aligned
+structs to word boundaries.  This may break non-cgo-based code that attempts to
+mix C and Go.
+
+NaCl support has been removed. The recent linker changes broke NaCl support
+a month ago, and there are no known users of it.
+If necessary, the NaCl code can be recovered from the repository history.
+
+* 5g/8g, 8l, ld, prof: fix output of 32-bit values (thanks Eoghan Sherry).
+* [68]l and runtime: GDB support for interfaces and goroutines.
+* 6l, 8l: support for linking ELF and Mach-O .o files.
+* all: simplify two-variable ranges with unused second variable (thanks Ryan Hitchman).
+* arm: updated soft float support.
+* codereview: keep quiet when not in use (thanks Eoghan Sherry).
+* compress/flate: implement Flush, equivalent to zlib's Z_SYNC_FLUSH.
+* crypto/tls: use rand.Reader in cert generation example (thanks Anthony Martin).
+* dashboard: fix project tag filter.
+* debug/elf, debug/macho: add ImportedLibraries, ImportedSymbols.
+* doc/go_mem: goroutine exit is not special.
+* event.go: another print glitch from gocheck.
+* gc: bug fixes,
+        syntax error for incomplete chan type (thanks Ryan Hitchman).
+* go/ast: fix ast.Walk.
+* gob: document the byte count used in the encoding of values,
+        fix bug sending zero-length top-level slices and maps,
+        Register should use the original type, not the indirected one.
+* godashboard: support submitting projects with non-ascii names (thanks Ryan Hitchman)
+* godefs: guard against structs with pad fields
+* godoc: added textual search, to enable use -fulltext flag.
+* gofmt: simplify "x, _ = range y" to "x = range y".
+* gopack: allow ELF/Mach-O objects in .a files without clearing allobj.
+* go/token,scanner: fix comments so godoc aligns properly.
+* govet: on error continue to the next file (thanks Christopher Wedgwood).
+* html: improved parsing.
+* http: ServeFile handles Range header for partial requests.
+* json: check for invalid UTF-8.
+* ld: allow .o files with no symbols,
+        reading of ELF object files,
+        reading of Mach-O object files.
+* math: change float64 bias constant from 1022 to 1023 (thanks Eoghan Sherry),
+        rename the MinFloat constant to SmallestNonzeroFloat.
+* nm: silently ignore .o files in .a files.
+* os: fix test of RemoveAll.
+* os/inotify: new package (thanks Balazs Lecz).
+* os: make MkdirAll work with symlinks (thanks Ryan Hitchman).
+* regexp: speed up by about 30%; also simplify code for brackets.
+* runtime/linux/386: set FPU to 64-bit precision.
+* runtime: remove paranoid mapping at 0.
+* suffixarray: add Bytes function.
+* syscall: add network interface constants for linux/386, linux/amd64 (thanks Mikio Hara).
+* syscall/windows: restrict access rights param of OpenProcess(),
+        remove \r and \n from error messages (thanks Alex Brainman).
+* test/bench: fixes to timing.sh (thanks Anthony Martin).
+* time: fix bug in Ticker: shutdown using channel rather than memory.
+* token/position: provide FileSet.File, provide files iterator.
+* xml: disallow invalid Unicode code points (thanks Nigel Kerr).
+
+ +

2010-12-08

+ +
+This release includes some package changes. If you use the crypto/tls or
+go/parser packages your code may require changes.
+
+The crypto/tls package's Dial function now takes an additional *Config
+argument.  Most uses will pass nil to get the same default behavior as before.
+See the documentation for details:
+        http://golang.org/pkg/crypto/tls/#Config
+        http://golang.org/pkg/crypto/tls/#Dial
+
+The go/parser package's ParseFile function now takes a *token.FileSet as its
+first argument. This is a pointer to a data structure used to store
+position information. If you don't care about position information you
+can pass "token.NewFileSet()". See the documentation for details:
+        http://golang.org/pkg/go/parser/#ParseFile
+
+This release also splits the patent grant text out of the LICENSE file into a
+separate PATENTS file and changes it to be more like the WebM grant.
+These clarifications were made at the request of the Fedora project.
+
+Other changes:
+* [68]l: generate debug info for builtin structured types, prettyprinting in gdb.
+* 8l: add dynimport to import table in Windows PE (thanks Wei Guangjing).
+* 8l, runtime: fix Plan 9 386 build (thanks Yuval Pavel Zholkover).
+* all: fix broken calls to Printf etc.
+* bufio: make Reader.Read implement io.Reader semantics (thanks Roger Peppe).
+* build: allow archiver to be specified by HOST_AR (thanks Albert Strasheim).
+* bytes: add Buffer.UnreadRune, Buffer.UnreadByte (thanks Roger Peppe).
+* crypto/tls: fix build of certificate generation example (thanks Christian Himpel).
+* doc/install: describe GOHOSTOS and GOHOSTARCH.
+* errchk: accept multiple source files (thanks Eoghan Sherry).
+* exec.LookPath: return os.PathError instad of os.ENOENT (thanks Michael Hoisie)..
+* flag: fix format error in boolean error report,
+        handle multiple calls to flag.Parse.
+* fmt: add %U format for standard Unicode representation of code point values.
+* gc: fix method offsets of anonymous interfaces (thanks Eoghan Sherry),
+        skip undefined symbols in import . (thanks Eoghan Sherry).
+* go/scanner: remove Tokenize - was only used in tests
+* gobuilder: add buildroot command-line flag (thanks Devon H. O'Dell).
+* html: unescape numeric entities (thanks Ryan Hitchman).
+* http: Add EncodeQuery, helper for constructing query strings.
+* ld: fix dwarf decoding of 64-bit reflect values (thanks Eoghan Sherry).
+* math: improve accuracy of Exp2 (thanks Eoghan Sherry).
+* runtime: add Goroutines (thanks Keith Rarick).
+* sync: small naming fix for armv5 (thanks Dean Prichard).
+* syscall, net: Add Recvmsg and Sendmsg on Linux (thanks Albert Strasheim).
+* time: make After use fewer goroutines and host processes (thanks Roger Peppe).
+
+ +

2010-12-02

+ +
+Several package changes in this release may require you to update your code if
+you use the bytes, template, or utf8 packages. In all cases, any outdated code
+will fail to compile rather than behave erroneously.
+
+The bytes package has changed. Its Add and AddByte functions have been removed,
+as their functionality is provided by the recently-introduced built-in function
+"append". Any code that uses them will need to be changed:
+s = bytes.Add(s, b)    ->    s = append(s, b...)
+s = bytes.AddByte(b, c)    ->    s = append(s, b)
+s = bytes.Add(nil, c)    ->    append([]byte(nil), c)
+
+The template package has changed. Your code will need to be updated if it calls
+the HTMLFormatter or StringFormatter functions, or implements its own formatter
+functions. The function signature for formatter types has changed to:
+        func(wr io.Writer, formatter string, data ...interface{})
+to allow multiple arguments to the formatter.  No templates will need updating.
+See the change for examples:
+        http://code.google.com/p/go/source/detail?r=2c2be793120e
+
+The template change permits the implementation of multi-word variable
+instantiation for formatters. Before one could say
+        {field}
+or
+        {field|formatter}
+Now one can also say
+        {field1 field2 field3}
+or
+        {field1 field2 field3|formatter}
+and the fields are passed as successive arguments to the formatter,
+by analogy to fmt.Print.
+
+The utf8 package has changed. The order of EncodeRune's arguments has been
+reversed to satisfy the convention of "destination first".
+Any code that uses EncodeRune will need to be updated.
+
+Other changes:
+* [68]l: correct dwarf location for globals and ranges for arrays.
+* big: fix (*Rat) SetFrac64(a, b) when b < 0 (thanks Eoghan Sherry).
+* compress/flate: fix typo in comment (thanks Mathieu Lonjaret).
+* crypto/elliptic: use a Jacobian transform for better performance.
+* doc/code.html: fix reference to "gomake build" (thanks Anschel Schaffer-Cohen).
+* doc/roadmap: update gdb status.
+* doc/spec: fixed some omissions and type errors.
+* doc: some typo fixes (thanks Peter Mundy).
+* exp/eval: build fix for parser.ParseFile API change (thanks Anschel Schaffer-Cohen).
+* fmt: Scan accepts Inf and NaN,
+        allow "% X" as well as "% x".
+* go/printer: preserve newlines in func parameter lists (thanks Jamie Gennis).
+* http: consume request body before next request.
+* log: ensure writes are atomic (thanks Roger Peppe).
+* path: Windows support for Split (thanks Benny Siegert).
+* runtime: fix SysFree to really free memory on Windows (thanks Alex Brainman),
+        parallel definitions in Go for all C structs.
+* sort: avoid overflow in pivot calculation,
+        reduced stack depth to lg(n) in quickSort (thanks Stefan Nilsson).
+* strconv: Atof on Infs and NaNs.
+
+ +

2010-11-23

+ +
+This release includes a backwards-incompatible package change to the
+sort.Search function (introduced in the last release).
+See the change for details and examples of how you might change your code:
+        http://code.google.com/p/go/source/detail?r=102866c369
+
+* build: automatically #define _64BIT in 6c.
+* cgo: print required space after parameter name in wrapper function.
+* crypto/cipher: new package to replace crypto/block (thanks Adam Langley).
+* crypto/elliptic: new package, implements elliptic curves over prime fields (thanks Adam Langley).
+* crypto/x509: policy OID support and fixes (thanks Adam Langley).
+* doc: add link to codewalks,
+        fix recover() documentation (thanks Anschel Schaffer-Cohen),
+        explain how to write Makefiles for commands.
+* exec: enable more tests on windows (thanks Alex Brainman).
+* gc: adjustable hash code in typecheck of composite literals
+        (thanks to vskrap, Andrey Mirtchovski, and Eoghan Sherry).
+* gc: better error message for bad type in channel send (thanks Anthony Martin).
+* godoc: bug fix in relativePath,
+        compute search index for all file systems under godoc's observation,
+        use correct time stamp to indicate accuracy of search result.
+* index/suffixarray: use sort.Search.
+* net: add ReadFrom and WriteTo windows version (thanks Wei Guangjing).
+* reflect: remove unnecessary casts in Get methods.
+* rpc: add RegisterName to allow override of default type name.
+* runtime: free memory allocated by windows CommandLineToArgv (thanks Alex Brainman).
+* sort: simplify Search (thanks Roger Peppe).
+* strings: add LastIndexAny (thanks Benny Siegert).
+
+ +

2010-11-10

+ +
+The birthday release includes a new Search capability inside the sort package.
+It takes an unusual but very general and easy-to-use approach to searching
+arbitrary indexable sorted data.  See the documentation for details:
+    http://golang.org/pkg/sort/#Search
+
+The ARM port now uses the hardware floating point unit (VFP).  It still has a
+few bugs, mostly around conversions between unsigned integer and floating-point
+values, but it's stabilizing.
+
+In addition, there have been many smaller fixes and updates: 
+
+* 6l: generate dwarf variable names with disambiguating suffix.
+* container/list: make Remove return Value of removed element.
+    makes it easier to remove first or last item.
+* crypto: add cast5 (default PGP cipher),
+    switch block cipher methods to be destination first.
+* crypto/tls: use pool building for certificate checking
+* go/ast: change embedded token.Position fields to named fields
+    (preparation for a different position representation)
+* net: provide public access to file descriptors (thanks Keith Rarick)
+* os: add Expand function to evaluate environment variables.
+* path: add Glob (thanks Benny Siegert)
+* runtime: memequal optimization (thanks Graham Miller)
+    prefix all external symbols with "runtime·" to avoid
+    conflicts linking with external C libraries.
+
+ +

2010-11-02

+ +
+This release includes a language change: the new built-in function, append.
+Append makes growing slices much simpler. See the spec for details:
+        http://golang.org/doc/go_spec.html#Appending_and_copying_slices
+
+Other changes:
+* 8l: pe generation fixes (thanks Alex Brainman).
+* doc: Effective Go: append and a few words about "..." args.
+* build: fiddle with make variables.
+* codereview: fix sync and download in Python 2.7 (thanks Fazlul Shahriar).
+* debug/pe, cgo: add windows support (thanks Wei Guangjing).
+* go/ast: add Inspect function for easy AST inspection w/o a visitor.
+* go/printer: do not remove parens around composite literals starting with
+        a type name in control clauses.
+* go/scanner: bug fixes, revisions, and more tests.
+* gob: several fixes and documentation updates.
+* godoc: bug fix (bug introduced with revision 3ee58453e961).
+* gotest: print empty benchmark list in a way that gofmt will leave alone.
+* http server: correctly respond with 304 NotModified (thanks Michael Hoisie).
+* kate: update list of builtins (thanks Evan Shaw).
+* libutf: update to Unicode 5.2.0 to match pkg/unicode (thanks Anthony Martin).
+* misc/bbedit: update list of builtins (thanks Anthony Starks).
+* misc/vim: update list of builtins.
+* mkrunetype: install a Makefile and tweak it slightly so it can be built.
+* netchan: fix locking bug.
+* pidigits: minor improvements (thanks Evan Shaw).
+* rpc: fix client deadlock bug.
+* src: use append where appropriate (often instead of vector).
+* strings: add Contains helper function (thanks Brad Fitzpatrick).
+* syscall: SIO constants for Linux (thanks Albert Strasheim),
+        Stat(path) on windows (thanks Alex Brainman).
+* test/ken/convert.go: add conversion torture test.
+* testing: add Benchmark (thanks Roger Peppe).
+
+ +

2010-10-27

+ +
+*** This release changes the encoding used by package gob. 
+    If you store gobs on disk, see below. ***
+
+The ARM port (5g) now passes all tests. The optimizer is not yet enabled, and
+floating point arithmetic is performed entirely in software. Work is underway
+to address both of these deficiencies.
+
+The syntax for arrays, slices, and maps of composite literals has been
+simplified. Within a composite literal of array, slice, or map type, elements
+that are themselves composite literals may elide the type if it is identical to
+the outer literal's element type. For example, these expressions:
+	[][]int{[]int{1, 2, 3}, []int{4, 5}}
+	map[string]Point{"x": Point{1.5, -3.5}, "y": Point{0, 0}}
+can be simplified to:
+	[][]int{{1, 2, 3}, {4, 5}}
+	map[string]Point{"x": {1.5, -3.5}, "y": {0, 0}}
+Gofmt can make these simplifications mechanically when invoked with the 
+new -s flag.
+
+The built-in copy function can now copy bytes from a string value to a []byte.
+Code like this (for []byte b and string s): 
+	for i := 0; i < len(s); i++ {
+		b[i] = s[i]
+	}
+can be rewritten as:
+	copy(b, s)
+
+The gob package can now encode and decode interface values containing types
+registered ahead of time with the new Register function. These changes required
+a backwards-incompatible change to the wire format.  Data written with the old
+version of the package will not be readable with the new one, and vice versa.
+(Steps were made in this change to make sure this doesn't happen again.) 
+We don't know of anyone using gobs to create permanent data, but if you do this
+and need help converting, please let us know, and do not update to this release
+yet.  We will help you convert your data.
+
+Other changes:
+* 5g, 6g, 8g: generate code for string index instead of calling function.
+* 5l, 6l, 8l: introduce sub-symbols.
+* 6l/8l: global and local variables and type info.
+* Make.inc: delete unnecessary -fno-inline flag to quietgcc.
+* arm: precise float64 software floating point, bug fixes.
+* big: arm assembly, faster software mulWW, divWW.
+* build: only print "You need to add foo to PATH" when needed.
+* container/list: fix Remove bug and use pointer to self as identifier.
+* doc: show page title in browser title bar,
+        update roadmap.
+* encoding/binary: give LittleEndian, BigEndian specific types.
+* go/parser: consume auto-inserted semi when calling ParseExpr().
+* gobuilder: pass GOHOSTOS and GOHOSTARCH to build,
+        write build and benchmarking logs to disk.
+* goinstall: display helpful message when encountering a cgo package,
+        fix test for multiple package names (thanks Fazlul Shahriar).
+* gotest: generate correct gofmt-formatted _testmain.go.
+* image/png: speed up paletted encoding ~25% (thanks Brad Fitzpatrick).
+* misc: update python scripts to specify python2 as python3 is now "python".
+* net: fix comment on Dial to mention unix/unixgram.
+* rpc: expose Server type to allow multiple RPC Server instances.
+* runtime: print unknown types in panic.
+* spec: append built-in (not yet implemented).
+* src: gofmt -s -w src misc.
+        update code to use copy-from-string.
+* test/bench: update numbers.
+* websocket: fix short Read.
+
+ +

2010-10-20

+ +
+This release removes the log package's deprecated functions.
+Code that has not been updated to use the new interface will break.
+See the previous release notes for details:
+	http://golang.org/doc/devel/release.html#2010-10-13
+
+Also included are major improvements to the linker. It is now faster, 
+uses less memory, and more parallelizable (but not yet parallel).
+
+The nntp package has been removed from the standard library.
+Its new home is the nntp-go project at Google Code:
+	http://code.google.com/p/nntp-go
+You can install it with goinstall:
+	goinstall nntp-go.googlecode.com/hg/nntp
+And import it in your code like so:
+	import "nntp-go.googlecode.com/hg/nntp"
+
+Other changes:
+* 6g: avoid too-large immediate constants.
+* 8l, runtime: initial support for Plan 9 (thanks Yuval Pavel Zholkover).
+* 6l, 8l: more improvements on exporting debug information (DWARF).
+* arm: code gen fixes. Most tests now pass, except for floating point code.
+* big: add random number generation (thanks Florian Uekermann).
+* gc: keep track of real actual type of identifiers,
+	report that shift must be unsigned integer,
+	select receive with implicit conversion.
+* goplay: fix to run under windows (thanks Yasuhiro Matsumoto).
+* http: do not close connection after sending HTTP/1.0 request.
+* netchan: add new method Hangup to terminate transmission on a channel.
+* os: change TestForkExec so it can run on windows (thanks Yasuhiro Matsumoto).
+* runtime: don't let select split stack.
+* syscall/arm: correct 64-bit system call arguments.
+
+ +

2010-10-13

+ +
+This release includes changes to the log package, the removal of exp/iterable,
+two new tools (gotry and goplay), one small language change, and many other
+changes and fixes.  If you use the log or iterable packages, you need to make
+changes to your code.
+
+The log package has changed.  Loggers now have only one output, and output to
+standard error by default.  The names have also changed, although the old names
+are still supported.  They will be deleted in the next release, though, so it
+would be good to update now if you can.  For most purposes all you need to do
+is make these substitutions:
+        log.Stderr -> log.Println or log.Print
+        log.Stderrf -> log.Printf
+        log.Crash -> log.Panicln or log.Panic
+        log.Crashf -> log.Panicf
+        log.Exit -> log.Exitln or log.Exit
+        log.Exitf -> log.Exitf (no change)
+Calls to log.New() must drop the second argument.
+Also, custom loggers with exit or panic properties will need to be reworked.
+For full details, see the change description:
+        http://code.google.com/p/go/source/detail?r=d8a3c7563d
+
+The language change is that uses of pointers to interface values no longer
+automatically dereference the pointer.  A pointer to an interface value is more
+often a beginner's bug than correct code.
+
+The package exp/iterable has been removed. It was an interesting experiment,
+but it encourages writing inefficient code and has outlived its utility.
+
+The new tools:
+* gotry: an exercise in reflection and an unusual tool. Run 'gotry' for details.
+* goplay: a stand-alone version of the Go Playground. See misc/goplay.
+
+Other changes:
+* 6l: Mach-O fixes, and fix to work with OS X nm/otool (thanks Jim McGrath).
+* [568]a: correct line numbers for statements.
+* arm: code generation and runtime fixes,
+	adjust recover for new reflect.call,
+	enable 6 more tests after net fix.
+* big: fix panic and round correctly in Rat.FloatString (thanks Anthony Martin).
+* build: Make.cmd: remove $(OFILES) (thanks Eric Clark),
+        Make.pkg: remove .so before installing new one,
+        add GOHOSTOS and GOHOSTARCH environment variables.
+* crypto/tls: better error messages for certificate issues,
+        make SetReadTimeout work.
+* doc: add Sydney University video,
+	add The Expressiveness of Go talk.
+* exp/draw/x11: support X11 vendors other than "The X.Org Foundation".
+* expvar: add (*Int).Set (thanks Sam Thorogood).
+* fmt: add Errorf helper function,
+        allow %d on []byte.
+* gc: O(1) string comparison when lengths differ,
+        various bug fixes.
+* http: return the correct error if a header line is too long.
+* image: add image.Tiled type, the Go equivalent of Plan 9's repl bit.
+* ld: be less picky about bad line number info.
+* misc/cgo/life: fix for new slice rules (thanks Graham Miller).
+* net: allow _ in DNS names.
+* netchan: export before import when testing, and
+        zero out request to ensure correct gob decoding. (thanks Roger Peppe).
+* os: make tests work on windows (thanks Alex Brainman).
+* runtime: bug fix: serialize mcache allocation,
+        correct iteration of large map values,
+        faster strequal, memequal (thanks Graham Miller),
+        fix argument dump in traceback,
+        fix tiny build.
+* smtp: new package (thanks Evan Shaw).
+* syscall: add sockaddr_ll support for linux/386, linux/amd64 (thanks Mikio Hara),
+        add ucred structure for SCM_CREDENTIALS over UNIX sockets. (thanks Albert Strasheim).
+* syscall: implement WaitStatus and Wait4() for windows (thanks Wei Guangjing).
+* time: add After.
+* websocket: enable tests on windows (thanks Alex Brainman).
+
+ +

2010-09-29

+ +
+This release includes some minor language changes and some significant package
+changes. You may need to change your code if you use ...T parameters or the
+http package.
+
+The semantics and syntax of forwarding ...T parameters have changed.
+        func message(f string, s ...interface{}) { fmt.Printf(f, s) }
+Here, s has type []interface{} and contains the parameters passed to message.
+Before this language change, the compiler recognized when a function call
+passed a ... parameter to another ... parameter of the same type, and just
+passed it as though it was a list of arguments.  But this meant that you
+couldn't control whether to pass the slice as a single argument and you
+couldn't pass a regular slice as a ... parameter, which can be handy.  This
+change gives you that control at the cost of a few characters in the call.
+If you want the promotion to ...,  append ... to the argument:
+        func message(f string, s ...interface{}) { fmt.Printf(f, s...) }
+Without the ..., s would be passed to Printf as a single argument of type
+[]interface{}.  The bad news is you might need to fix up some of your code, 
+but the compiler will detect the situation and warn you.
+
+Also, the http.Handler and http.HandlerFunc types have changed. Where http
+handler functions previously accepted an *http.Conn, they now take an interface
+type http.ResponseWriter. ResponseWriter implements the same methods as *Conn,
+so in most cases the only change required will be changing the type signature
+of your handler function's first parameter. See:
+  http://golang.org/pkg/http/#Handler
+
+The utf8 package has a new type, String, that provides efficient indexing 
+into utf8 strings by rune (previously an expensive conversion to []int 
+was required). See:
+  http://golang.org/pkg/utf8/#String
+
+The compiler will now automatically insert a semicolon at the end of a file if
+one is not found. This effect of this is that Go source files are no longer
+required to have a trailing newline.
+
+Other changes:
+* 6prof: more accurate usage message.
+* archive/zip: new package for reading Zip files.
+* arm: fix code generation, 10 more package tests pass.
+* asn1: make interface consistent with json.
+* bufio.UnreadRune: fix bug at EOF.
+* build: clear custom variables like GREP_OPTIONS,
+        silence warnings generated by ubuntu gcc,
+        use full path when compiling libraries.
+* bytes, strings: change lastIndexFunc to use DecodeLastRune (thanks Roger Peppe).
+* doc: add to and consolidate non-english doc references,
+        consolidate FAQs into a single file, go_faq.html,
+        updates for new http interface.
+* fmt/Printf: document and tweak error messages produced for bad formats.
+* gc: allow select case expr = <-c,
+        eliminate duplicates in method table,
+        fix reflect table method receiver,
+        improve error message for x \= 0.
+* go/scanner: treat EOF like a newline for purposes of semicolon insertion.
+* gofmt: stability improvements.
+* gotest: leave _testmain.go for "make clean" to clean up.
+* http: correct escaping of different parts of URL,
+        support HTTP/1.0 Keep-Alive.
+* json: do not write to unexported fields.
+* libcgo: don't build for NaCl,
+        set g, m in thread local storage for windows 386 (thanks Wei Guangjing).
+* math: Fix off-by-one error in Ilogb and Logb.  (thanks Charles L. Dorian).
+* misc/dashboard/builder: remove build files after benchmarking.
+* nacl: update instructions for new SDK.
+* net: enable v4-over-v6 on ip sockets,
+        fix crash in DialIP.
+* os: check for valid arguments in windows Readdir (thanks Peter Mundy).
+* runtime: add mmap of null page just in case,
+        correct stats in SysFree,
+        fix unwindstack crash.
+* syscall: add IPPROTO_IPV6 and IPV6_V6ONLY const to fix nacl and windows build,
+        add inotify on Linux (thanks Balazs Lecz),
+        fix socketpair in syscall_bsd,
+        fix windows value of IPV6_V6ONLY (thanks Alex Brainman),
+        implement windows version of Utimes (thanks Alex Brainman),
+        make mkall.sh work for nacl.
+* test: Add test that causes incorrect error from gccgo.
+* utf8: add DecodeLastRune and DecodeLastRuneInString (thanks Roger Peppe).
+* xml: Allow entities inside CDATA tags (thanks Dan Sinclair).
+
+ +

2010-09-22

+ +
+This release includes new package functionality, and many bug fixes and changes.
+It also improves support for the arm and nacl platforms.
+
+* 5l: avoid fixed buffers in list.
+* 6l, 8l: clean up ELF code, fix NaCl.
+* 6l/8l: emit DWARF frame info.
+* Make.inc: make GOOS detection work on windows (thanks Alex Brainman).
+* build: fixes for native arn build,
+        make all.bash run on Ubuntu ARM.
+* cgo: bug fixes,
+        show preamble gcc errors (thanks Eric Clark).
+* crypto/x509, crypto/tls: improve root matching and observe CA flag.
+* crypto: Fix certificate validation.
+* doc: variable-width layout.
+* env.bash: fix building in directory with spaces in the path (thanks Alex Brainman).
+* exp/4s, exp/nacl/av: sync to recent exp/draw changes.
+* exp/draw/x11: mouse location is a signed integer.
+* exp/nacl/av: update color to max out at 1<<16-1 instead of 1<<32-1.
+* fmt: support '*' for width or precision (thanks Anthony Martin).
+* gc: improvements to static initialization,
+        make sure path names are canonical.
+* gob: make robust when decoding a struct with non-struct data.
+* gobuilder: add -cmd for user-specified build command,
+        add -rev= flag to build specific revision and exit,
+        fix bug that caused old revisions to be rebuilt.
+* godoc: change default filter file name to "",
+        don't use quadratic algorithm to filter paths,
+        show "Last update" info for directory listings.
+* http: new redirect test,
+        URLEscape now escapes all reserved characters as per the RFC.
+* nacl: fix zero-length writes.
+* net/dict: parse response correctly (thanks Fazlul Shahriar).
+* netchan: add a cross-connect test,
+        handle closing of channels,
+        provide a method (Importer.Errors()) to recover protocol errors.
+* os: make Open() O_APPEND flag work on windows (thanks Alex Brainman),
+        make RemoveAll() work on windows (thanks Alex Brainman).
+* pkg/Makefile: disable netchan test to fix windows build (thanks Alex Brainman).
+* regexp: delete Iter methods.
+* runtime: better panic for send to nil channel.
+* strings: fix minor bug in LastIndexFunc (thanks Roger Peppe).
+* suffixarray: a package for creating suffixarray-based indexes.
+* syscall: Use vsyscall for syscall.Gettimeofday and .Time on linux amd64.
+* test: fix NaCl build.
+* windows: fix netchan test by using 127.0.0.1.
+
+ +

2010-09-15

+ +
+This release includes a language change: the lower bound of a subslice may
+now be omitted, in which case the value will default to 0.
+For example, s[0:10] may now be written as s[:10], and s[0:] as s[:].
+
+The release also includes important bug fixes for the ARM architecture,
+as well as the following fixes and changes:
+
+* 5g: register allocation bugs
+* 6c, 8c: show line numbers in -S output
+* 6g, 6l, 8g, 8l: move read-only data to text segment
+* 6l, 8l: make etext accurate; introduce rodata, erodata.
+* arm: fix build bugs.
+        make libcgo build during OS X cross-compile
+        remove reference to deleted file syntax/slice.go
+        use the correct stat syscalls
+        work around reg allocator bug in 5g
+* bufio: add UnreadRune.
+* build: avoid bad environment interactions
+        fix build for tiny
+        generate, clean .exe files on Windows (thanks Joe Poirier)
+        test for _WIN32, not _MINGW32 (thanks Joe Poirier)
+        work with GNU Make 3.82 (thanks Jukka-Pekka Kekkonen)
+* cgo: add typedef for uintptr in generated headers
+        silence warning for C call returning const pointer
+* codereview: convert email address to lower case before checking CONTRIBUTORS
+* crypto/tls: don't return an error from Close()
+* doc/tutorial: update for slice changes.
+* exec: separate LookPath implementations for unix/windows (thanks Joe Poirier)
+* exp/draw/x11: allow clean shutdown when the user closes the window.
+* exp/draw: clip destination rectangle to the image bounds.
+        fast path for drawing overlapping image.RGBAs.
+        fix double-counting of pt.Min for the src and mask points.
+        reintroduce the MouseEvent.Nsec timestamp.
+        rename Context to Window, and add a Close method.
+* exp/debug: preliminary support for 'copy' function (thanks Sebastien Binet)
+* fmt.Fscan: use UnreadRune to preserve data across calls.
+* gc: better printing of named constants, func literals in errors
+        many bug fixes
+        fix line number printing with //line directives
+        fix symbol table generation on windows (thanks Alex Brainman)
+        implement comparison rule from spec change 33abb649cb63
+        implement new slice spec (thanks Scott Lawrence)
+        make string x + y + z + ... + w efficient
+        more accurate line numbers for ATEXT
+        remove &[10]int -> []int conversion
+* go-mode.el: fix highlighting for 'chan' type (thanks Scott Lawrence)
+* godoc: better support for directory trees for user-supplied paths
+        use correct delay time (bug fix)
+* gofmt, go/printer: update internal estimated position correctly
+* goinstall: warn when package name starts with http:// (thanks Scott Lawrence)
+* http: check https certificate against host name
+        do not cache CanonicalHeaderKey (thanks Jukka-Pekka Kekkonen)
+* image: change a ColorImage's minimum point from (0, 0) to (-1e9, -1e9).
+        introduce Intersect and Union rectangle methods.
+* ld: handle quoted spaces in package path (thanks Dan Sinclair)
+* libcgo: fix NaCl build.
+* libmach: fix build on arm host
+        fix new thread race with Linux
+* math: make portable Tan(Pi/2) return NaN
+* misc/dashboard/builder: gobuilder, a continuous build client
+* net: disable tests for functions not available on windows (thanks Alex Brainman)
+* netchan: make -1 unlimited, as advertised.
+* os, exec: rename argv0 to name
+* path: add IsAbs (thanks Ivan Krasin)
+* runtime: fix bug in tracebacks
+        fix crash trace on amd64
+        fix windows build (thanks Alex Brainman)
+        use manual stack for garbage collection
+* spec: add examples for slices with omitted index expressions.
+        allow omission of low slice bound (thanks Scott Lawrence)
+* syscall: fix windows Gettimeofday (thanks Alex Brainman)
+* test(arm): disable zerodivide.go because compilation fails.
+* test(windows): disable tests that cause the build to fail (thanks Joe Poirier)
+* test/garbage/parser: sync with recent parser changes
+* test: Add test for //line
+        Make gccgo believe that the variables can change.
+        Recognize gccgo error messages.
+        Reduce race conditions in chan/nonblock.go.
+        Run garbage collector before testing malloc numbers.
+* websocket: Add support for secure WebSockets (thanks Jukka-Pekka Kekkonen)
+* windows: disable unimplemented tests (thanks Joe Poirier)
+
+ +

2010-09-06

+ +
+This release includes the syntactic modernization of more than 100 files in /test,
+and these additions, changes, and fixes: 
+* 6l/8l: emit DWARF in macho.
+* 8g: use FCHS, not FMUL, for minus float.
+* 8l: emit DWARF in ELF,
+        suppress emitting DWARF in Windows PE (thanks Alex Brainman).
+* big: added RatString, some simplifications.
+* build: create bin and pkg directories as needed; drop from hg,
+        delete Make.386 Make.amd64 Make.arm (obsoleted by Make.inc),
+        fix cgo with -j2,
+        let pkg/Makefile coordinate building of Go commands,
+        never use quietgcc in Make.pkg,
+        remove more references to GOBIN and GOROOT (thanks Christian Himpel).
+* codereview: Fix uploading for Mercurial 1.6.3 (thanks Evan Shaw),
+        consistent indent, cut dead code,
+        fix hang on standard hg commands,
+        print status when tasks take longer than 30 seconds,
+        really disable codereview when not available,
+        upload files in parallel (5x improvement on large CLs).
+* crypto/hmac: make Sum idempotent (thanks Jukka-Pekka Kekkonen).
+* doc: add links to more German docs,
+        add round-robin flag to io2010 balance example,
+        fix a bug in the example in Constants subsection (thanks James Fysh),
+        various changes for validating HTML (thanks Scott Lawrence).
+* fmt: delete erroneous sentence about return value for Sprint*.
+* gc: appease bison version running on FreeBSD builder,
+        fix spurious syntax error.
+* go/doc: use correct escaper for URL.
+* go/printer: align ImportPaths in ImportDecls (thanks Scott Lawrence).
+* go/typechecker: 2nd step towards augmenting AST with full type information.
+* gofmt: permit omission of first index in slice expression.
+* goinstall: added -a flag to mean "all remote packages" (thanks Scott Lawrence),
+        assume go binaries are in path (following new convention),
+        use https for Google Code checkouts.
+* gotest: allow make test of cgo packages (without make install).
+* http: add Date to server, Last-Modified and If-Modified-Since to file server,
+        add PostForm function to post url-encoded key/value data,
+        obscure passwords in return value of URL.String (thanks Scott Lawrence).
+* image: introduce Config type and DecodeConfig function.
+* libcgo: update Makefile to use Make.inc.
+* list: update comment to state that the zero value is ready to use.
+* math: amd64 version of Sincos (thanks Charles L. Dorian).
+* misc/bash: add *.go completion for gofmt (thanks Scott Lawrence).
+* misc/emacs: make _ a word symbol (thanks Scott Lawrence).
+* misc: add zsh completion (using compctl),
+        syntax highlighting for Fraise.app (OS X) (thanks Vincent Ambo).
+* net/textproto: Handle multi-line responses (thanks Evan Shaw).
+* net: add LookupMX (thanks Corey Thomasson).
+* netchan: Fix race condition in test,
+        rather than 0, make -1 mean infinite (a la strings.Split et al),
+        use acknowledgements on export send.
+        new methods Sync and Drain for clean teardown.
+* regexp: interpret all Go characer escapes \a \b \f \n \r \t \v.
+* rpc: fix bug that caused private methods to attempt to be registered.
+* runtime: Correct commonType.kind values to match compiler,
+        add GOOS, GOARCH; fix FuncLine,
+        special case copy, equal for one-word interface values (thanks Kyle Consalus).
+* scanner: fix incorrect reporting of error in Next (thanks Kyle Consalus).
+* spec: clarify that arrays must be addressable to be sliceable.
+* template: fix space handling around actions.
+* test/solitaire: an exercise in backtracking and string conversions.
+* test: Recognize gccgo error messages and other fixes.
+* time: do not crash in String on nil Time.
+* tutorial: regenerate HTML to pick up change to progs/file.go.
+* websocket: fix missing Sec-WebSocket-Protocol on server response (thanks Jukka-Pekka Kekkonen).
+
+ +

2010-08-25

+ +
+This release includes changes to the build system that will likely require you
+to make changes to your environment variables and Makefiles.
+
+All environment variables are now optional:
+ - $GOOS and $GOARCH are now optional; their values should now be inferred 
+   automatically by the build system,
+ - $GOROOT is now optional, but if you choose not to set it you must run
+   'gomake' instead of 'make' or 'gmake' when developing Go programs
+   using the conventional Makefiles,
+ - $GOBIN remains optional and now defaults to $GOROOT/bin;
+   if you wish to use this new default, make sure it is in your $PATH
+   and that you have removed the existing binaries from $HOME/bin.
+
+As a result of these changes, the Go Makefiles have changed. If your Makefiles
+inherit from the Go Makefiles, you must change this line:
+    include ../../Make.$(GOARCH)
+to this:
+    include ../../Make.inc
+
+This release also removes the deprecated functions in regexp and the 
+once package. Any code that still uses them will break.
+See the notes from the last release for details:
+    http://golang.org/doc/devel/release.html#2010-08-11
+
+Other changes:
+* 6g: better registerization for slices, strings, interface values
+* 6l: line number information in DWARF format
+* build: $GOBIN defaults to $GOROOT/bin,
+        no required environment variables
+* cgo: add C.GoStringN (thanks Eric Clark).
+* codereview: fix issues with leading tabs in CL descriptions,
+        do not send "Abandoned" mail if the CL has not been mailed.
+* crypto/ocsp: add missing Makefile.
+* crypto/tls: client certificate support (thanks Mikkel Krautz).
+* doc: update gccgo information for recent changes.
+        fix errors in Effective Go.
+* fmt/print: give %p priority, analogous to %T,
+        honor Formatter in Print, Println.
+* gc: fix parenthesization check.
+* go/ast: facility for printing AST nodes,
+        first step towards augmenting AST with full type information.
+* go/printer: do not modify tabwriter.Escape'd text.
+* gofmt: do not modify multi-line string literals,
+        print AST nodes by setting -ast flag.
+* http: fix typo in http.Request documentation (thanks Scott Lawrence)
+        parse query string always, not just in GET
+* image/png: support 16-bit color.
+* io: ReadAtLeast now errors if min > len(buf).
+* jsonrpc: use `error: null` for success, not `error: ""`.
+* libmach: implement register fetch for 32-bit x86 kernel.
+* net: make IPv6 String method standards-compliant (thanks Mikio Hara).
+* os: FileInfo.Permission() now returns uint32 (thanks Scott Lawrence),
+        implement env using native Windows API (thanks Alex Brainman).
+* reflect: allow PtrValue.PointTo(nil).
+* runtime: correct line numbers for .goc files,
+        fix another stack split bug,
+        fix freebsd/386 mmap.
+* syscall: regenerate syscall/z* files for linux/386, linux/amd64, linux/arm.
+* tabwriter: Introduce a new flag StripEscape.
+* template: fix handling of space around actions,
+        vars preceded by white space parse correctly (thanks Roger Peppe).
+* test: add test case that crashes gccgo.
+* time: parse no longer requires minutes for time zone (thanks Jan H. Hosang)
+* yacc: fix bounds check in error recovery.
+
+ +

2010-08-11

+ +
+This release introduces some package changes. You may need to change your
+code if you use the once, regexp, image, or exp/draw packages.
+
+The type Once has been added to the sync package. The new sync.Once will
+supersede the functionality provided by the once package. We intend to remove
+the once package after this release. See:
+    http://golang.org/pkg/sync/#Once
+All instances of once in the standard library have been replaced with
+sync.Once. Reviewing these changes may help you modify your existing code. 
+The relevant changeset:
+    http://code.google.com/p/go/source/detail?r=fa2c43595119
+
+A new set of methods has been added to the regular expression package, regexp.
+These provide a uniformly named approach to discovering the matches of an
+expression within a piece of text; see the package documentation for details: 
+    http://golang.org/pkg/regexp/
+These new methods will, in a later release, replace the old methods for
+matching substrings.  The following methods are deprecated:
+    Execute (use FindSubmatchIndex)
+    ExecuteString (use FindStringSubmatchIndex)
+    MatchStrings(use FindStringSubmatch)
+    MatchSlices (use FindSubmatch)
+    AllMatches (use FindAll; note that n<0 means 'all matches'; was n<=0)
+    AllMatchesString (use FindAllString; note that n<0 means 'all matches'; was n<=0)
+(Plus there are ten new methods you didn't know you wanted.) 
+Please update your code to use the new routines before the next release.
+
+An image.Image now has a Bounds rectangle, where previously it ranged 
+from (0, 0) to (Width, Height). Loops that previously looked like:
+    for y := 0; y < img.Height(); y++ {
+        for x := 0; x < img.Width(); x++ {
+            // Do something with img.At(x, y)
+        }
+    }
+should instead be:
+    b := img.Bounds()
+    for y := b.Min.Y; y < b.Max.Y; y++ {
+        for x := b.Min.X; x < b.Max.X; x++ {
+            // Do something with img.At(x, y)
+        }
+    }
+The Point and Rectangle types have also moved from exp/draw to image.
+
+Other changes:
+* arm: bugfixes and syscall (thanks Kai Backman).
+* asn1: fix incorrect encoding of signed integers (thanks Nicholas Waples).
+* big: fixes to bitwise functions (thanks Evan Shaw).
+* bytes: add IndexRune, FieldsFunc and To*Special (thanks Christian Himpel).
+* encoding/binary: add complex (thanks Roger Peppe).
+* exp/iterable: add UintArray (thanks Anschel Schaffer-Cohen).
+* godoc: report Status 404 if a pkg or file is not found.
+* gofmt: better reporting for unexpected semicolon errors.
+* html: new package, an HTML tokenizer.
+* image: change image representation from slice-of-slices to linear buffer,
+        introduce Decode and RegisterFormat,
+        introduce Transparent and Opaque,
+        replace Width and Height by Bounds, add the Point and Rect types.
+* libbio: fix Bprint to address 6g issues with large data structures.
+* math: fix amd64 Hypot (thanks Charles L. Dorian).
+* net/textproto: new package, with example net/dict.
+* os: fix ForkExec() handling of envv == nil (thanks Alex Brainman).
+* png: grayscale support (thanks Mathieu Lonjaret).
+* regexp: document that backslashes are the escape character.
+* rpc: catch errors from ReadResponseBody.
+* runtime: memory free fix (thanks Alex Brainman).
+* template: add ParseFile method to template.Template.
+* test/peano: use directly recursive type def.
+
+ +

2010-08-04

+ +
+This release includes a change to os.Open (and co.). The file permission
+argument has been changed to a uint32. Your code may require changes - a simple
+conversion operation at most.
+
+Other changes:
+* amd64: use segment memory for thread-local storage.
+* arm: add gdb support to android launcher script,
+        bugfixes (stack clobbering, indices),
+        disable another flaky test,
+        remove old qemu dependency from gotest.
+* bufio: introduce Peek.
+* bytes: added test case for explode with blank string (thanks Scott Lawrence).
+* cgo: correct multiple return value function invocations (thanks Christian Himpel).
+* crypto/x509: unwrap Subject Key Identifier (thanks Adam Langley).
+* gc: index bounds tests and other fixes.
+* gofmt/go/parser: strengthen syntax checks.
+* goinstall: check for error from exec.*Cmd.Wait() (thanks Alex Brainman).
+* image/png: use image-specific methods for checking opacity.
+* image: introduce Gray and Gray16 types,
+        remove the named colors except for Black and White.
+* json: object members must have a value (thanks Anthony Martin).
+* misc/vim: highlight misspelled words only in comments (thanks Christian Himpel).
+* os: Null device (thanks Peter Mundy).
+* runtime: do not fall through in SIGBUS/SIGSEGV.
+* strings: fix Split("", "", -1) (thanks Scott Lawrence).
+* syscall: make go errors not clash with windows errors (thanks Alex Brainman).
+* test/run: diff old new,
+* websocket: correct challenge response (thanks Tarmigan Casebolt),
+        fix bug involving spaces in header keys (thanks Bill Neubauer). 
+
+ +

2010-07-29

+ +
+* 5g: more soft float support and several bugfixes.
+* asn1: Enumerated, Flag and GeneralizedTime support.
+* build: clean.bash to check that GOOS and GOARCH are set.
+* bytes: add IndexFunc and LastIndexFunc (thanks Fazlul Shahriar),
+	add Title.
+* cgo: If CC is set in environment, use it rather than "gcc",
+	use new command line syntax: -- separates cgo flags from gcc flags.
+* codereview: avoid crash if no config,
+	don't run gofmt with an empty file list,
+	make 'hg submit' work with Mercurial 1.6.
+* crypto/ocsp: add package to parse OCSP responses.
+* crypto/tls: add client-side SNI support and PeerCertificates.
+* exp/bignum: delete package - functionality subsumed by package big.
+* fmt.Print: fix bug in placement of spaces introduced when ...T went in.
+* fmt.Scanf: handle trailing spaces.
+* gc: fix smaller-than-pointer-sized receivers in interfaces,
+	floating point precision/normalization fixes,
+	graceful exit on seg fault,
+	import dot shadowing bug,
+	many fixes including better handling of invalid input,
+	print error detail about failure to open import.
+* gccgo_install.html: add description of the port to RTEMS (thanks Vinu Rajashekhar).
+* gobs: fix bug in singleton arrays.
+* godoc: display synopses for all packages that have some kind of documentation..
+* gofmt: fix some linebreak issues.
+* http: add https client support (thanks Fazlul Shahriar),
+	write body when content length unknown (thanks James Whitehead).
+* io: MultiReader and MultiWriter (thanks Brad Fitzpatrick),
+	fix another race condition in Pipes.
+* ld: many fixes including better handling of invalid input.
+* libmach: correct handling of .5 files with D_REGREG addresses.
+* linux/386: use Xen-friendly ELF TLS instruction sequence.
+* mime: add AddExtensionType (thanks Yuusei Kuwana).
+* misc/vim: syntax file recognizes constants like 1e9 (thanks Petar Maymounkov).
+* net: TCPConn.SetNoDelay, back by popular demand.
+* net(windows): fix crashing Read/Write when passed empty slice on (thanks Alex Brainman),
+	implement LookupHost/Port/SRV (thanks Wei Guangjing),
+	properly handle EOF in (*netFD).Read() (thanks Alex Brainman).
+* runtime: fix bug introduced in revision 4a01b8d28570 (thanks Alex Brainman),
+	rename cgo2c, *.cgo to goc2c, *.goc (thanks Peter Mundy).
+* scanner: better comment.
+* strings: add Title.
+* syscall: add ForkExec, Syscall12 on Windows (thanks Daniel Theophanes),
+	improve windows errno handling (thanks Alex Brainman).
+* syscall(windows): fix FormatMessage (thanks Peter Mundy),
+	implement Pipe() (thanks Wei Guangjing).
+* time: fix parsing of minutes in time zones.
+* utf16(windows): fix cyclic dependency when testing (thanks Peter Mundy).
+
+ +

2010-07-14

+ +
+This release includes a package change. In container/vector, the Iter method
+has been removed from the Vector, IntVector, and StringVector types. Also, the
+Data method has been renamed to Copy to better express its actual behavior.
+Now that Vector is just a slice, any for loops ranging over v.Iter() or
+v.Data() can be changed to range over v instead.
+
+Other changes:
+* big: Improvements to Rat.SetString (thanks Evan Shaw),
+        add sign, abs, Rat.IsInt.
+* cgo: various bug fixes.
+* codereview: Fix for Mercurial >= 1.6 (thanks Evan Shaw).
+* crypto/rand: add Windows implementation (thanks Peter Mundy).
+* crypto/tls: make HTTPS servers easier,
+        add client OCSP stapling support.
+* exp/eval: converted from bignum to big (thanks Evan Shaw).
+* gc: implement new len spec, range bug fix, optimization.
+* go/parser: require that '...' parameters are followed by a type.
+* http: fix ParseURL to handle //relative_path properly.
+* io: fix SectionReader Seek to seek backwards (thanks Peter Mundy).
+* json: Add HTMLEscape (thanks Micah Stetson).
+* ld: bug fixes.
+* math: amd64 version of log (thanks Charles L. Dorian).
+* mime/multipart: new package to parse multipart MIME messages
+        and HTTP multipart/form-data support.
+* os: use TempFile with default TempDir for test files (thanks Peter Mundy).
+* runtime/tiny: add docs for additional VMs, fix build (thanks Markus Duft).
+* runtime: better error for send/recv on nil channel.
+* spec: clarification of channel close(),
+        lock down some details about channels and select,
+        restrict when len(x) is constant,
+        specify len/cap for nil slices, maps, and channels.
+* windows: append .exe to binary names (thanks Joe Poirier).
+
+ +

2010-07-01

+ +
+This release includes some package changes that may require changes to 
+client code.
+
+The Split function in the bytes and strings packages has been changed.
+The count argument, which limits the size of the return, previously treated
+zero as unbounded. It now treats 0 as 0, and will return an empty slice.  
+To request unbounded results, use -1 (or some other negative value).
+The new Replace functions in bytes and strings share this behavior.
+This may require you change your existing code.
+
+The gob package now allows the transmission of non-struct values at the
+top-level. As a result, the rpc and netchan packages have fewer restrictions
+on the types they can handle.  For example, netchan can now share a chan int.
+
+The release also includes a Code Walk: "Share Memory By Communicating".
+It describes an idiomatic Go program that uses goroutines and channels:
+	http://golang.org/doc/codewalk/sharemem/
+
+There is now a Projects page on the Go Dashboard that lists Go programs, 
+tools, and libraries:
+	http://godashboard.appspot.com/project
+
+Other changes:
+* 6a, 6l: bug fixes.
+* bytes, strings: add Replace.
+* cgo: use slash-free relative paths for .so references.
+* cmath: correct IsNaN for argument cmplx(Inf, NaN) (thanks Charles L. Dorian).
+* codereview: allow multiple email addresses in CONTRIBUTORS.
+* doc/codewalk: add Share Memory By Communicating.
+* exp/draw/x11: implement the mapping from keycodes to keysyms.
+* fmt: Printf: fix bug in handling of %#v, allow other verbs for slices
+        Scan: fix handling of EOFs.
+* gc: bug fixes and optimizations.
+* gob: add DecodeValue and EncodeValue,
+        add support for complex numbers.
+* goinstall: support for Bazaar+Launchpad (thanks Gustavo Niemeyer).
+* io/ioutil: add TempFile for Windows (thanks Peter Mundy).
+* ld: add -u flag to check safe bits; discard old -u, -x flags.
+* math: amd64 versions of Exp and Fabs (thanks Charles L. Dorian).
+* misc/vim: always override filetype detection for .go files.
+* net: add support for DNS SRV requests (thanks Kirklin McDonald),
+        initial attempt to implement Windows version (thanks Alex Brainman).
+* netchan: allow chan of basic types now that gob can handle such,
+        eliminate the need for a pointer value in Import and Export.
+* os/signal: only catch all signals if os/signal package imported.
+* regexp: bug fix: need to track whether match begins with fixed prefix.
+* rpc: allow non-struct args and reply (they must still be pointers).
+* runtime: bug fixes and reorganization.
+* strconv: fix bugs in floating-point and base 2 conversions
+* syscall: add syscall_bsd.go to zsycall_freebsd_386.go (thanks Peter Mundy),
+        add socketpair (thanks Ivan Krasin).
+* time: implement time zones for Windows (thanks Alex Brainman).
+* x509: support non-self-signed certs. 
+
+ +

2010-06-21

+ +
+This release includes a language change. The "..." function parameter form is
+gone; "...T" remains. Typically, "...interface{}" can be used instead of "...".
+
+The implementation of Printf has changed in a way that subtly affects its
+handling of the fmt.Stringer interface. You may need to make changes to your
+code. For details, see:
+        https://groups.google.com/group/golang-nuts/msg/6fffba90a3e3dc06
+
+The reflect package has been changed. If you have code that uses reflect, 
+it will need to be updated. For details, see:
+        https://groups.google.com/group/golang-nuts/msg/7a93d07c590e7beb
+
+Other changes:
+* 8l: correct test for sp == top of stack in 8l -K code.
+* asn1: allow '*' in PrintableString.
+* bytes.Buffer.ReadFrom: fix bug.
+* codereview: avoid exception in match (thanks Paolo Giarrusso).
+* complex divide: match C99 implementation.
+* exp/draw: small draw.drawGlyphOver optimization.
+* fmt: Print*: reimplement to switch on type first,
+        Scanf: improve error message when input does not match format.
+* gc: better error messages for interface failures, conversions, undefined symbols.
+* go/scanner: report illegal escape sequences.
+* gob: substitute slice for map.
+* goinstall: process dependencies for package main (thanks Roger Peppe).
+* gopack: add S flag to force marking a package as safe,
+        simplify go metadata code.
+* html: sync testdata/webkit to match WebKit tip.
+* http: reply to Expect 100-continue requests automatically (thanks Brad Fitzpatrick).
+* image: add an Alpha16 type.
+* ld: pad Go symbol table out to page boundary (fixes cgo crash).
+* misc/vim: reorganize plugin to be easier to use (thanks James Whitehead).
+* path: add Base, analogous to Unix basename.
+* pkg/Makefile: allow DISABLE_NET_TESTS=1 to disable network tests.
+* reflect: add Kind, Type.Bits, remove Int8Type, Int8Value, etc.
+* runtime: additional Windows support (thanks Alex Brainman),
+        correct fault for 16-bit divide on Leopard,
+        fix 386 signal handler bug.
+* strconv: add AtofN, FtoaN.
+* string: add IndexFunc and LastIndexFunc (thanks Roger Peppe).
+* syslog: use local network for tests. 
+
+ +

2010-06-09

+ +
+This release contains many fixes and improvements, including several
+clarifications and consolidations to the Language Specification.
+
+The type checking rules around assignments and conversions are simpler but more
+restrictive: assignments no longer convert implicitly from *[10]int to []int
+(write x[0:] instead of &x), and conversions can no longer change the names of
+types inside composite types.
+
+The fmt package now includes flexible type-driven (fmt.Scan) and 
+format-driven (fmt.Scanf) scanners for all basic types.
+
+* big: bug fix for Quo aliasing problem.
+* bufio: change ReadSlice to match description.
+* cgo: bug fixes.
+* doc: add Google I/O talk and programs,
+        codereview + Mercurial Queues info (thanks Peter Williams).
+* exp/draw: Draw fast paths for the Over operator,
+        add Rectangle.Eq and Point.In, fix Rectangle.Clip (thanks Roger Peppe).
+* fmt: Scan fixes and improvements.
+* gc: backslash newline is not a legal escape sequence in strings,
+        better error message when ~ operator is found,
+        fix export of complex types,
+        new typechecking rules.
+* go/parser: correct position of empty statement ';'.
+* gofmt: fix test script.
+* goinstall: use 'git pull' instead of 'git checkout' (thanks Michael Hoisie).
+* http: add Head function for making HTTP HEAD requests,
+        handle status 304 correctly.
+* image: add Opaque method to the image types.
+        make Color.RGBA return 16 bit color instead of 32 bit color.
+* io/ioutil: add TempFile.
+* math: Pow special cases and additional tests (thanks Charles L. Dorian).
+* netchan: improve closing and shutdown.
+* os: implement os.FileInfo.*time_ns for windows (thanks Alex Brainman).
+* os/signal: correct the regexp for finding Unix signal names (thanks Vinu Rajashekhar).
+* regexp: optimizations (thanks Kyle Consalus).
+* runtime: fix printing -Inf (thanks Evan Shaw),
+        finish pchw -> tiny, added gettime for tiny (thanks Daniel Theophanes).
+* spec: clean-ups and consolidation.
+* syscall: additional Windows compatibility fixes (thanks Alex Brainman).
+* test/bench: added regex-dna-parallel.go (thanks Kyle Consalus).
+* vector: type-specific Do functions now take f(type) (thanks Michael Hoisie). 
+
+ +

2010-05-27

+ +
+A sizeable release, including standard library improvements and a slew of
+compiler bug fixes. The three-week interval was largely caused by the team
+preparing for Google I/O. 
+
+* big: add Rat type (thanks Evan Shaw),
+        new features, much performance tuning, cleanups, and more tests.
+* bignum: deprecate by moving into exp directory.
+* build: allow MAKEFLAGS to be set outside the build scripts (thanks Christopher Wedgwood).
+* bytes: add Trim, TrimLeft, TrimRight, and generic functions (thanks Michael Hoisie).
+* cgo: fix to permit cgo callbacks from init code.
+* cmath: update range of Phase and Polar due to signed zero (thanks Charles L. Dorian).
+* codereview: work better with mq (thanks Peter Williams).
+* compress: renamings
+	NewDeflater -> NewWriter
+	NewInflater -> NewReader
+	Deflater -> Compressor
+	Inflater -> Decompressor
+* exp/draw/x11: respect $XAUTHORITY,
+        treat $DISPLAY the same way x-go-bindings does.
+* exp/draw: fast path for glyph images, other optimizations,
+        fix Rectangle.Canon (thanks Roger Peppe).
+* fmt: Scan, Scanln: Start of a simple scanning API in the fmt package,
+        fix Printf crash when given an extra nil argument (thanks Roger Peppe).
+* gc: better error when computing remainder of non-int (thanks Evan Shaw),
+        disallow middot in Go programs,
+        distinguish array, slice literal in error messages,
+        fix shift/reduce conflict in go.y export syntax,
+        fix unsafe.Sizeof on ideal constants,
+        handle use of builtin function outside function call,
+        many other bug fixes.
+* gob: add support for maps,
+        add test for indirect maps, slices, arrays.
+* godoc: collect package comments from all package files.
+* gofmt: don't lose mandatory semicolons,
+        exclude test w/ illegal syntax from test cases,
+        fix printing of labels.
+* http: prevent crash if remote server is not responding with "HTTP/".
+* json: accept escaped slash in string scanner (thanks Michael Hoisie),
+        fix array -> non-array decoding.
+* libmach: skip __nl_symbol_ptr section on OS X.
+* math: amd64 versions of Fdim, Fmax, Fmin,
+        signed zero Sqrt special case (thanks Charles L. Dorian).
+* misc/kate: convert isn't a built in function (thanks Evan Shaw).
+* net: implement BindToDevice,
+        implement raw sockets (thanks Christopher Wedgwood).
+* netFD: fix race between Close and Read/Write (thanks Michael Hoisie).
+* os: add Chtimes function (thanks Brad Fitzpatrick).
+* pkg/Makefile: add netchan to standard package list.
+* runtime: GOMAXPROCS returns previous value,
+        allow large map values,
+        avoid allocation for fixed strings,
+        correct tracebacks for nascent goroutines, even closures,
+        free old hashmap pieces during resizing.
+* spec: added imaginary literal to semicolon rules (was missing),
+        fix and clarify syntax of conversions,
+        simplify section on channel types,
+        other minor tweaks.
+* strconv: Btoui64 optimizations (thanks Kyle Consalus).
+* strings: use copy instead of for loop in Map (thanks Kyle Consalus).
+* syscall: implement BindToDevice (thanks Christopher Wedgwood),
+        add Utimes on Darwin/FreeBSD, add Futimes everywhere,
+        regenerate syscalls for some platforms.
+* template: regularize name lookups of interfaces, pointers, and methods.
+
+ +

2010-05-04

+ +
+In this release we renamed the Windows OS target from 'mingw' to 'windows'.
+If you are currently building for 'mingw' you should set GOOS=windows instead.
+
+* 5l, 6l, 8l, runtime: make -s binaries work.
+* 5l, 6l, 8l: change ELF header so that strip doesn't destroy binary.
+* 8l: fix absolute path detection on Windows.
+* big: new functions, optimizations, and cleanups,
+	add bitwise methods for Int (thanks Evan Shaw).
+* bytes: Change IndexAny to look for UTF-8 encoded characters.
+* darwin: bsdthread_create can fail; print good error.
+* fmt: %T missing print <nil> for nil (thanks Christopher Wedgwood).
+* gc: many fixes.
+* misc/cgo/gmp: fix bug in SetString.
+* net: fix resolv.conf EOF without newline bug (thanks Christopher Wedgwood).
+* spec: some small clarifications (no language changes).
+* syscall: add EWOULDBLOCK to sycall_nacl.go,
+	force O_LARGEFILE in Linux open system call,
+	handle EOF on pipe - special case on Windows (thanks Alex Brainman),
+	mingw Sleep (thanks Joe Poirier).
+* test/bench: import new fasta C reference, update Go, optimizations.
+* test: test of static initialization (fails).
+* vector: use correct capacity in call to make.
+* xml: allow text segments to end at EOF.
+
+ +

2010-04-27

+ +
+This release includes a new Codelab that illustrates the construction of a
+simple wiki web application: 
+	http://golang.org/doc/codelab/wiki/
+
+It also includes a Codewalk framework for documenting code. See:
+	http://golang.org/doc/codewalk/
+
+Other changes:
+* 6g: fix need for parens around array index expression.
+* 6l, 8l: include ELF header in PT_LOAD mapping for text segment.
+* arm: add android runner script,
+	support for printing floats.
+* big: implemented Karatsuba multiplication,
+	many fixes and improvements (thanks Evan Shaw).
+* bytes: add Next method to Buffer, simplify Read,
+	shuffle implementation, making WriteByte 50% faster.
+* crypto/tls: simpler implementation of record layer.
+* exp/eval: fixes (thanks Evan Shaw).
+* flag: eliminate unnecessary structs.
+* gc: better windows support,
+	cmplx typecheck bug fix,
+	more specific error for statements at top level.
+* go/parser: don't require unnecessary parens.
+* godoc: exclude duplicate entries (thanks Andrei Vieru),
+	use int64 for timestamps (thanks Christopher Wedgwood).
+* gofmt: fine-tune stripping of parentheses,
+* json: Marshal, Unmarshal using new scanner,
+	preserve field name case by default,
+	scanner, Compact, Indent, and tests,
+	support for streaming.
+* libmach: disassemble MOVLQZX correctly.
+* math: more special cases for signed zero (thanks Charles L. Dorian).
+* net: add Pipe,
+	fix bugs in packStructValue (thanks Michael Hoisie),
+	introduce net.Error interface.
+* os: FileInfo: regularize the types of some fields,
+	create sys_bsd.go (thanks Giles Lean),
+	mingw bug fixes (thanks Alex Brainman).
+* reflect: add FieldByNameFunc (thanks Raif S. Naffah),
+	implement Set(nil), SetValue(nil) for PtrValue and MapValue.
+* regexp: allow escaping of any punctuation.
+* rpc/jsonrpc: support for jsonrpc wire encoding.
+* rpc: abstract client and server encodings,
+	add Close() method to rpc.Client.
+* runtime: closures, defer bug fix for Native Client,
+	rename cgo2c, *.cgo to goc2c, *.goc to avoid confusion with real cgo.
+	several other fixes.
+* scanner: implement Peek() to look at the next char w/o advancing.
+* strings: add ReadRune to Reader, add FieldsFunc (thanks Kyle Consalus).
+* syscall: match linux Setsid function signature to darwin,
+	mingw bug fixes (thanks Alex Brainman).
+* template: fix handling of pointer inside interface.
+* test/bench: add fannkuch-parallel.go (thanks Kyle Consalus),
+	pidigits ~10% performance win by using adds instead of shifts.
+* time: remove incorrect time.ISO8601 and add time.RFC3339 (thanks Micah Stetson).
+* utf16: add DecodeRune, EncodeRune.
+* xml: add support for XML marshalling embedded structs (thanks Raif S. Naffah),
+	new "innerxml" tag to collect inner XML.
+
+ +

2010-04-13

+ +
+This release contains many changes:
+
+* 8l: add DOS stub to PE binaries (thanks Evan Shaw).
+* cgo: add //export.
+* cmath: new complex math library (thanks Charles L. Dorian).
+* docs: update to match current coding style (thanks Christopher Wedgwood).
+* exp/eval: fix example and add target to Makefile (thanks Evan Shaw).
+* fmt: change behaviour of format verb %b to match %x when negative (thanks Andrei Vieru).
+* gc: compile s == "" as len(s) == 0,
+	distinguish fatal compiler bug from error+exit,
+	fix alignment on non-amd64,
+	good syntax error for defer func() {} - missing fina (),
+	implement panic and recover,
+	zero unnamed return values on entry if func has defer.
+* goyacc: change to be reentrant (thanks Roger Peppe).
+* io/ioutil: fix bug in ReadFile when Open succeeds but Stat fails.
+* kate: update for recent language changes (thanks Evan Shaw).
+* libcgo: initial mingw port work - builds but untested (thanks Joe Poirier).
+* math: new functions and special cases (thanks Charles L. Dorian) 
+* net: use chan bool instead of chan *netFD to avoid cycle.
+* netchan: allow client to send as well as receive.
+* nntp: new package, NNTP client (thanks Conrad Meyer).
+* os: rename os.Dir to os.FileInfo.
+* rpc: don't log normal EOF,
+	fix ServeConn to block as documented.
+* runtime: many bug fixes, better ARM support.
+* strings: add IndexRune, Trim, TrimLeft, TrimRight, etc (thanks Michael Hoisie).
+* syscall: implement some mingw syscalls required by os (thanks Alex Brainman).
+* test/bench: add k-nucleotide-parallel (thanks Kyle Consalus).
+* Unicode: add support for Turkish case mapping.
+* xgb: move from the main repository to http://code.google.com/p/x-go-binding/
+
+ +

2010-03-30

+ +
+This release contains three language changes:
+
+1. Accessing a non-existent key in a map is no longer a run-time error.  
+It now evaluates to the zero value for that type.  For example:
+        x := myMap[i]   is now equivalent to:   x, _ := myMap[i]
+
+2. It is now legal to take the address of a function's return value.  
+The return values are copied back to the caller only after deferred
+functions have run.
+
+3. The functions panic and recover, intended for reporting and recovering from
+failure, have been added to the spec:
+	http://golang.org/doc/go_spec.html#Handling_panics 
+In a related change, panicln is gone, and panic is now a single-argument
+function.  Panic and recover are recognized by the gc compilers but the new
+behavior is not yet implemented.
+
+The ARM build is broken in this release; ARM users should stay at release.2010-03-22.
+
+Other changes:
+* bytes, strings: add IndexAny.
+* cc/ld: Add support for #pragma dynexport,
+        Rename dynld to dynimport throughout. Cgo users will need to rerun cgo.
+* expvar: default publishings for cmdline, memstats
+* flag: add user-defined flag types.
+* gc: usual bug fixes
+* go/ast: generalized ast filtering.
+* go/printer: avoid reflect in print.
+* godefs: fix handling of negative constants.
+* godoc: export pprof debug information, exported variables,
+        support for filtering of command-line output in -src mode,
+        use http GET for remote search instead of rpc.
+* gofmt: don't convert multi-line functions into one-liners,
+        preserve newlines in multiline selector expressions (thanks Risto Jaakko Saarelma).
+* goinstall: include command name in error reporting (thanks Andrey Mirtchovski)
+* http: add HandleFunc as shortcut to Handle(path, HandlerFunc(func))
+* make: use actual dependency for install
+* math: add J1, Y1, Jn, Yn, J0, Y0 (Bessel functions) (thanks Charles L. Dorian)
+* prof: add pprof from google-perftools
+* regexp: don't return non-nil *Regexp if there is an error.
+* runtime: add Callers,
+        add malloc sampling, pprof interface,
+        add memory profiling, more statistics to runtime.MemStats,
+        implement missing destroylock() (thanks Alex Brainman),
+        more malloc statistics,
+        run all finalizers in a single goroutine,
+        Goexit runs deferred calls.
+* strconv: add Atob and Btoa,
+        Unquote could wrongly return a nil error on error (thanks Roger Peppe).
+* syscall: add IPV6 constants,
+        add syscall_bsd.go for Darwin and other *BSDs (thanks Giles Lean),
+        implement SetsockoptString (thanks Christopher Wedgwood).
+* websocket: implement new protocol (thanks Fumitoshi Ukai).
+* xgb: fix request length and request size (thanks Firmansyah Adiputra).
+* xml: add CopyToken (thanks Kyle Consalus),
+        add line numbers to syntax errors (thanks Kyle Consalus),
+        use io.ReadByter in place of local readByter (thanks Raif S. Naffah). 
+
+ +

2010-03-22

+ +
+With this release we announce the launch of the Go Blog:
+	http://blog.golang.org/
+The first post is a brief update covering what has happened since the launch.
+
+This release contains some new packages and functionality, and many fixes:
+* 6g/8g: fix issues with complex data types, other bug fixes.
+* Makefiles: refactored to make writing external Makefiles easier.
+* crypto/rand: new package.
+* godoc: implemented command-line search via RPC,
+	improved comment formatting: recognize URLs.
+* gofmt: more consistent formatting of const/var decls.
+* http: add Error helper function,
+	add ParseQuery (thanks Petar Maymounkov),
+	change RawPath to mean raw path, not raw everything-after-scheme.
+* image/jpeg: fix typos.
+* json: add MarshalIndent (accepts user-specified indent string).
+* math: add Gamma function (thanks Charles L. Dorian).
+* misc/bbedit: support for cmplx, real, imag (thanks Anthony Starks).
+* misc/vim: add new complex types, functions and literals.
+* net: fix IPMask.String not to crash on all-0xff mask.
+* os: drop File finalizer after normal Close.
+* runtime: add GOROOT and Version,
+	lock finalizer table accesses.
+* sha512: add sha384 (truncated version) (thanks Conrad Meyer).
+* syscall: add const ARCH, analogous to OS.
+* syscall: further additions to mingw port (thanks Alex Brainman).
+* template: fixed html formatter []byte input bug.
+* utf16: new package.
+* version.bash: cope with ancient Mercurial.
+* websocket: use URL.RawPath to construct WebSocket-Location: header.
+
+ +

2010-03-15

+ +
+This release includes a language change: support for complex numbers.
+	http://golang.org/doc/go_spec.html#Imaginary_literals
+	http://golang.org/doc/go_spec.html#Complex_numbers
+There is no library support as yet.
+
+This release also includes the goinstall command-line tool. 
+	http://golang.org/cmd/goinstall/
+	http://groups.google.com/group/golang-nuts/t/f091704771128e32
+
+* 5g/6g/8g: fix double function call in slice.
+* arm: cleanup build warnings. (thanks Dean Prichard)
+* big: fix mistakes with probablyPrime.
+* bufio: add WriteRune.
+* bytes: add ReadRune and WriteRune to bytes.Buffer.
+* cc: stack split bug fix.
+* crypto: add SHA-224 to sha256, add sha512 package. (thanks Conrad Meyer)
+* crypto/ripemd160: new package. (thanks Raif S. Naffah)
+* crypto/rsa: don't use safe primes.
+* gc: avoid fixed length buffer cleanbuf. (thanks Dean Prichard)
+	better compilation of floating point +=
+	fix crash on complicated arg to make slice.
+	remove duplicate errors, give better error for I.(T)
+* godoc: support for multiple packages in a directory, other fixes.
+* gofmt: bug fixes.
+* hash: add Sum64 interface.
+* hash/crc32: add Update function.
+* hash/crc64: new package implementing 64-bit CRC.
+* math: add ilogb, logb, remainder. (thanks Charles L. Dorian) 
+* regexp: add ReplaceAllFunc, ReplaceAllStringFunc.
+* runtime: clock garbage collection on bytes allocated, not pages in use.
+* strings: make Split(s, "", n) faster. (thanks Spring Mc)
+* syscall: minimal mingw version of syscall. (thanks Alex Brainman)
+* template: add ParseFile, MustParseFile.
+
+ +

2010-03-04

+ +
+There is one language change: the ability to convert a string to []byte or 
+[]int.  This deprecates the strings.Bytes and strings.Runes functions.
+You can convert your existing sources using these gofmt commands:
+	gofmt -r 'strings.Bytes(x) -> []byte(x)' -w file-or-directory-list
+	gofmt -r 'strings.Runes(x) -> []int(x)' -w file-or-directory-list
+After running these you might need to delete unused imports of the "strings" 
+package.
+
+Other changes and fixes:
+* 6l/8l/5l: add -r option
+* 8g: make a[byte(x)] truncate x
+* codereview.py: fix for compatibility with hg >=1.4.3
+* crypto/blowfish: new package (thanks Raif S. Naffah)
+* dashboard: more performance tuning
+* fmt: use String method in %q to get the value to quote.
+* gofmt: several cosmetic changes
+* http: fix handling of Connection: close, bug in http.Post
+* net: correct DNS configuration,
+	fix network timeout boundary condition,
+	put [ ] around IPv6 addresses for Dial.
+* path: add Match,
+	fix bug in Match with non-greedy stars (thanks Kevin Ballard)
+* strings: delete Bytes, Runes (see above)
+* tests: an Eratosthenesque concurrent prime sieve (thanks Anh Hai Trinh) 
+
+ +

2010-02-23

+ +
+This release is mainly bug fixes and a little new code.
+There are no language changes.
+
+6g/5g/8g: bug fixes
+8a/8l: Added FCMOVcc instructions (thanks Evan Shaw and Charles Dorian)
+crypto/x509: support certificate creation
+dashboard: caching to avoid datastore queries
+exec: add dir argument to Run
+godoc: bug fixes and code cleanups
+http: continued implementation and bug fixes (thanks Petar Maymounkov)
+json: fix quoted strings in Marshal (thanks Sergei Skorobogatov)
+math: more functions, test cases, and benchmarks (thanks Charles L. Dorian)
+misc/bbedit: treat predeclared identifiers as "keywords" (thanks Anthony Starks)
+net: disable UDP server test (flaky on various architectures)
+runtime: work around Linux kernel bug in futex,
+	pchw is now tiny
+sync: fix to work on armv5 (thanks Dean Prichard)
+websocket: fix binary frame size decoding (thanks Timo Savola)
+xml: allow unquoted attribute values in non-Strict mode (thanks Amrut Joshi)
+	treat bool as value in Unmarshal (thanks Michael Hoisie) 
+
+ +

2010-02-17

+ +
+There are two small language changes:
+* NUL bytes may be rejected in souce files, and the tools do reject them.
+* Conversions from string to []int and []byte are defined but not yet implemented.
+
+Other changes and fixes:
+* 5a/6a/8a/5c/6c/8c: remove fixed-size arrays for -I and -D options (thanks Dean Prichard)
+* 5c/6c/8c/5l/6l/8l: add -V flag to display version number
+* 5c/6c/8c: use "cpp" not "/bin/cpp" for external preprocessor (thanks Giles Lean)
+* 8a/8l: Added CMOVcc instructions (thanks Evan Shaw)
+* 8l: pe executable building code changed to include import table for kernel32.dll functions (thanks Alex Brainman)
+* 5g/6g/8g: bug fixes
+* asn1: bug fixes and additions (incl marshalling)
+* build: fix build for Native Client, Linux/ARM
+* dashboard: show benchmarks, add garbage collector benchmarks
+* encoding/pem: add marshalling support
+* exp/draw: fast paths for a nil mask
+* godoc: support for directories outside $GOROOT
+* http: sort header keys when writing Response or Request to wire (thanks Petar Maymounkov)
+* math: special cases and new functions (thanks Charles Dorian)
+* mime: new package, used in http (thanks Michael Hoisie)
+* net: dns bug fix - use random request id
+* os: finalize File, to close fd.
+* path: make Join variadic (thanks Stephen Weinberg)
+* regexp: optimization bug fix
+* runtime: misc fixes and optimizations
+* syscall: make signature of Umask on OS X, FreeBSD match Linux. (thanks Giles Lean)
+
+ +

2010-02-04

+ +
+There is one language change: support for ...T parameters:
+	http://golang.org/doc/go_spec.html#Function_types
+
+You can now check build status on various platforms at the Go Dashboard: 
+	http://godashboard.appspot.com
+
+* 5l/6l/8l: several minor fixes
+* 5a/6a/8a/5l/6l/8l: avoid overflow of symb buffer (thanks Dean Prichard)
+* compress/gzip: gzip deflater (i.e., writer)
+* debug/proc: add mingw specific build stubs (thanks Joe Poirier)
+* exp/draw: separate the source-point and mask-point in Draw
+* fmt: handle nils safely in Printf
+* gccgo: error messages now match those of gc
+* godoc: several fixes
+* http: bug fixes, revision of Request/Response (thanks Petar Maymounkov)
+* image: new image.A type to represent anti-aliased font glyphs
+	add named colors (e.g. image.Blue), suitable for exp/draw
+* io: fixed bugs in Pipe
+* malloc: merge into package runtime
+* math: fix tests on FreeBSD (thanks Devon H. O'Dell)
+	add functions; update tests and special cases (thanks Charles L. Dorian)
+* os/signal: send SIGCHLDs to Incoming (thanks Chris Wedgwood)
+* reflect: add StringHeader to reflect
+* runtime: add SetFinalizer
+* time: Sleep through interruptions (thanks Chris Wedgwood)
+	add RFC822 formats
+	experimental implemenation of Ticker using two goroutines for all tickers
+* xml: allow underscores in XML element names (thanks Michael Hoisie)
+	allow any scalar type in xml.Unmarshal
+
+ +

2010-01-27

+ +
+There are two small language changes: the meaning of chan <- chan int
+is now defined, and functions returning functions do not need to 
+parenthesize the result type.
+
+There is one significant implementation change: the compilers can
+handle multiple packages using the same name in a single binary.
+In the gc compilers, this comes at the cost of ensuring that you
+always import a particular package using a consistent import path.
+In the gccgo compiler, the cost is that you must use the -fgo-prefix
+flag to pass a unique prefix (like the eventual import path).
+
+5a/6a/8a: avoid use of fixed-size buffers (thanks Dean Prichard)
+5g, 6g, 8g: many minor bug fixes
+bufio: give Writer.WriteString same signature as bytes.Buffer.WriteString.
+container/list: PushFrontList, PushBackList (thanks Jan Hosang)
+godoc: trim spaces from search query (thanks Christopher Wedgwood)
+hash: document that Sum does not change state, fix crypto hashes
+http: bug fixes, revision of Request/Response (thanks Petar Maymounkov)
+math: more handling of IEEE 754 special cases (thanks Charles Dorian)
+misc/dashboard: new build dashboard
+net: allow UDP broadcast,
+	use /etc/hosts to resolve names (thanks Yves Junqueira, Michael Hoisie)
+netchan: beginnings of new package for connecting channels across a network
+os: allow FQDN in Hostname test (thanks Icarus Sparry)
+reflect: garbage collection bug in Call
+runtime: demo of Go on raw (emulated) hw in runtime/pchw,
+	performance fix on OS X
+spec: clarify meaning of chan <- chan int,
+	func() func() int is allowed now,
+	define ... T (not yet implemented)
+template: can use interface values
+time: fix for +0000 time zone,
+	more robust tick.Stop.
+xgb: support for authenticated connections (thanks Firmansyah Adiputra)
+xml: add Escape (thanks Stephen Weinberg)
+
+ +

2010-01-13

+ +
+This release is mainly bug fixes with a little new code.
+There are no language changes.
+
+build: $GOBIN should no longer be required in $PATH (thanks Devon H. O'Dell),
+	new package target "make bench" to run benchmarks
+8g: faster float -> uint64 conversion (thanks Evan Shaw)
+5g, 6g, 8g:
+	clean opnames.h to avoid stale errors (thanks Yongjian Xu),
+	a handful of small compiler fixes
+5g, 6g, 8g, 5l, 6l, 8l: ignore $GOARCH, which is implied by name of tool
+6prof: support for writing input files for google-perftools's pprof
+asn1: fix a few structure-handling bugs
+cgo: many bug fixes (thanks Devon H. O'Dell)
+codereview: repeated "hg mail" sends "please take another look"
+gob: reserve ids for future expansion
+godoc: distinguish HTML generation from plain text HTML escaping (thanks Roger Peppe)
+gofmt: minor bug fixes, removed -oldprinter flag
+http: add CanonicalPath (thanks Ivan Krasin),
+	avoid header duplication in Response.Write,
+	correctly escape/unescape URL sections
+io: new interface ReadByter
+json: better error, pointer handling in Marshal (thanks Ivan Krasin)
+libmach: disassembly of FUCOMI, etc (thanks Evan Shaw)
+math: special cases for most functions and 386 hardware Sqrt (thanks Charles Dorian)
+misc/dashboard: beginning of a build dashboard at godashboard.appspot.com.
+misc/emacs: handling of new semicolon rules (thanks Austin Clements),
+	empty buffer bug fix (thanks Kevin Ballard)
+misc/kate: highlighting improvements (tahnks Evan Shaw)
+os/signal: add signal names: signal.SIGHUP, etc (thanks David Symonds)
+runtime: preliminary Windows support (thanks Hector Chu),
+	preemption polling to reduce garbage collector pauses
+scanner: new lightweight scanner package
+template: bug fix involving spaces before a delimited block
+test/bench: updated timings
+time: new Format, Parse functions
+
+ +

2010-01-05

+ +
+This release is mainly bug fixes.  There are no language changes.
+
+6prof: now works on 386
+8a, 8l: add FCOMI, FCOMIP, FUCOMI, and FUCOMIP (thanks Evan Shaw)
+big: fix ProbablyPrime on small numbers
+container/vector: faster []-based implementation (thanks Jan Mercl)
+crypto/tls: extensions and Next Protocol Negotiation
+gob: one encoding bug fix, one decoding bug fix
+image/jpeg: support for RST markers
+image/png: support for transparent paletted images
+misc/xcode: improved support (thanks Ken Friedenbach)
+net: return nil Conn on error from Dial (thanks Roger Peppe)
+regexp: add Regexp.NumSubexp (thanks Peter Froehlich)
+syscall: add Nanosleep on FreeBSD (thanks Devon H. O'Dell)
+template: can use map in .repeated section
+
+There is now a public road map, in the repository and online
+at http://golang.org/doc/devel/roadmap.html.
+
+ +

2009-12-22

+ +
+Since the last release there has been one large syntactic change to
+the language, already discussed extensively on this list: semicolons
+are now implied between statement-ending tokens and newline characters.
+See http://groups.google.com/group/golang-nuts/t/5ee32b588d10f2e9 for
+details.
+
+By default, gofmt now parses and prints the new lighter weight syntax.
+To convert programs written in the old syntax, you can use:
+
+	gofmt -oldparser -w *.go
+
+Since everything was being reformatted anyway, we took the opportunity to
+change the way gofmt does alignment.  Now gofmt uses tabs at the start
+of a line for basic code alignment, but it uses spaces for alignment of
+interior columns.  Thus, in an editor with a fixed-width font, you can
+choose your own tab size to change the indentation, and no matter what
+tab size you choose, columns will be aligned properly.
+
+
+In addition to the syntax and formatting changes, there have been many
+smaller fixes and updates:
+
+6g,8g,5g: many bug fixes, better registerization,
+   build process fix involving mkbuiltin (thanks Yongjian Xu),
+   method expressions for concrete types
+8l: support for Windows PE files (thanks Hector Chu)
+bytes: more efficient Buffer handling
+bytes, strings: new function Fields (thanks Andrey Mirtchovski)
+cgo: handling of enums (thanks Moriyoshi Koizumi),
+    handling of structs with bit fields, multiple files (thanks Devon H. O'Dell),
+    installation of .so to non-standard locations
+crypto/sha256: new package for SHA 256 (thanks Andy Davis)
+encoding/binary: support for slices of fixed-size values (thanks Maxim Ushakov)
+exp/vector: experimental alternate vector representation (thanks Jan Mercl)
+fmt: %p for chan, map, slice types
+gob: a couple more bug fixes
+http: support for basic authentication (thanks Ivan Krasin)
+image/jpeg: basic JPEG decoder
+math: correct handling of Inf and NaN in Pow (thanks Charles Dorian)
+misc/bash: completion file for bash (thanks Alex Ray)
+os/signal: support for handling Unix signals (thanks David Symonds)
+rand: Zipf-distributed random values (thanks William Josephson)
+syscall: correct error return bug on 32-bit machines (thanks Christopher Wedgwood)
+syslog: new package for writing to Unix syslog daemon (thanks Yves Junqueira)
+template: will automatically invoke niladic methods
+time: new ISO8601 format generator (thanks Ben Olive)
+xgb: converted generator to new syntax (thanks Tor Andersson)
+xml: better mapping of tag names to Go identifiers (thanks Kei Son),
+    better handling of unexpected EOF (thanks Arvindh Rajesh Tamilmani)
+
+ +

2009-12-09

+ +
+Since the last release there are two changes to the language: 
+
+* new builtin copy(dst, src) copies n = min(len(dst), len(src)) 
+  elements to dst from src and returns n.  It works correctly 
+  even if dst and src overlap.  bytes.Copy is gone. 
+  Convert your programs using: 
+      gofmt -w -r 'bytes.Copy(d, s) -> copy(d, s)' *.go 
+
+* new syntax x[lo:] is shorthand for x[lo:len(x)]. 
+  Convert your programs using: 
+      gofmt -w -r 'a[b:len(a)] -> a[b:]' *.go 
+
+In addition, there have been many smaller fixes and updates: 
+
+* 6g/8g/5g: many bug fixes 
+* 8g: fix 386 floating point stack bug (thanks Charles Dorian) 
+* all.bash: now works even when $GOROOT has spaces (thanks Sergio Luis O. B. Correia), 
+    starting to make build work with mingw (thanks Hector Chu), 
+    FreeBSD support (thanks Devon O'Dell) 
+* big: much faster on 386. 
+* bytes: new function IndexByte, implemented in assembly 
+    new function Runes (thanks Peter Froehlich), 
+    performance tuning in bytes.Buffer. 
+* codereview: various bugs fixed 
+* container/vector: New is gone; just declare a Vector instead. 
+    call Resize to set len and cap. 
+* cgo: many bug fixes (thanks Eden Li) 
+* crypto: added MD4 (thanks Chris Lennert), 
+    added XTEA (thanks Adrian O'Grady). 
+* crypto/tls: basic client 
+* exp/iterable: new functions (thanks Michael Elkins) 
+* exp/nacl: native client tree builds again 
+* fmt: preliminary performance tuning 
+* go/ast: more powerful Visitor (thanks Roger Peppe) 
+* gob: a few bug fixes 
+* gofmt: better handling of standard input, error reporting (thanks Fazlul Shahriar) 
+    new -r flag for rewriting programs 
+* gotest: support for Benchmark functions (thanks Trevor Strohman) 
+* io: ReadFile, WriteFile, ReadDir now in separate package io/ioutil. 
+* json: new Marshal function (thanks Michael Hoisie), 
+    better white space handling (thanks Andrew Skiba), 
+    decoding into native data structures (thanks Sergey Gromov), 
+    handling of nil interface values (thanks Ross Light). 
+* math: correct handling of sin/cos of large angles 
+* net: better handling of Close (thanks Devon O'Dell and Christopher Wedgwood) 
+    support for UDP broadcast (thanks Jonathan Wills), 
+    support for empty packets 
+* rand: top-level functions now safe to call from multiple goroutines 
+(thanks Roger Peppe). 
+* regexp: a few easy optimizations 
+* rpc: better error handling, a few bug fixes 
+* runtime: better signal handling on OS X, malloc fixes, 
+    global channel lock is gone. 
+* sync: RWMutex now allows concurrent readers (thanks Péter Szabó) 
+* template: can use maps as data (thanks James Meneghello) 
+* unicode: updated to Unicode 5.2. 
+* websocket: new package (thanks Fumitoshi Ukai) 
+* xgb: preliminary X Go Bindings (thanks Tor Andersson) 
+* xml: fixed crash (thanks Vish Subramanian) 
+* misc: bbedit config (thanks Anthony Starks), 
+    kate config (thanks Evan Shaw) 
+
diff --git a/doc/docs.html b/doc/docs.html new file mode 100644 index 000000000..9fd3dcebe --- /dev/null +++ b/doc/docs.html @@ -0,0 +1,236 @@ + + +
+ +

Learning Go

+ +

+If you're new to Go, we recommend you work through the +tutorial. The +language specification has all the details should +you want to explore. +

+

+Once you've learned a little about the language, +Effective Go will help you learn the style and +idioms of programming in Go. +

+ +

A Tutorial for the Go Programming Language

+

+The first tutorial. An introductory text that touches upon several core +concepts: syntax, types, allocation, constants, I/O, sorting, printing, +goroutines, and channels. +

+ +

Course Notes

+

+Slides from a 3-day course about the Go programming language. +A more thorough introduction than the tutorial. +

+ + +

Effective Go

+

+A document that gives tips for writing clear, idiomatic Go code. +A must read for any new Go programmer. It augments the tutorial and +the language specification, both of which should be read first. +

+ +

Frequently Asked Questions (FAQ)

+

+Answers to common questions about Go. +

+ +

How to write Go code

+

+How to write a new package and how to test code. +

+ +

Codelab: Writing Web Applications

+

+This codelab takes the reader through the creation of a simple wiki web +application. It touches on structs, methods, file I/O, http, regular expressions, +and closures. +

+ +

Codewalks

+

+Guided tours of Go programs. +

+ +

Go for C++ Programmers

+

+An introduction to Go for C++ programmers. +

+ +

Non-English Documentation

+ +

Belarusian — Беларуская

+ +
    +
  • faq-be - Frequently Asked Questions.
  • +
+ +

Chinese — 中文

+ + + +

German — Deutsch

+ + + +

Japanese — 日本語

+ + +

Korean — 한국어

+ + +

Russian — Русский

+ + +
+ + +
+ +

References

+ +

Keep these under your pillow.

+ +

Package Documentation

+

+The built-in documentation for the Go standard library. +

+ +

Command Documentation

+

+The built-in documentation for the Go tools. +

+ +

Language Specification

+

+The official Go Language specification. +

+ +

Release History

+

A summary of the changes between Go releases.

+ +

The Go Memory Model

+

+A document that specifies the conditions under which reads of a variable in +one goroutine can be guaranteed to observe values produced by writes to the +same variable in a different goroutine. +

+ +

Videos and Talks

+ +

Writing Web Apps in Go

+

+A talk by Rob Pike and Andrew Gerrand presented at Google I/O 2011. +It walks through the construction and deployment of a simple web application +and unveils the Go runtime for App Engine. +See the presentation slides. +

+ +

Real World Go

+

+A talk by Andrew Gerrand presented at Google I/O Bootcamp 2011. +It gives a broad overview of Go's type system and concurrency model +and provides four examples of Go programs that solve real problems. +See the presentation slides. +

+ +

Go Programming

+

+A presentation delivered by Rob Pike and Russ Cox at Google I/O 2010. It +illustrates how programming in Go differs from other languages through a set of +examples demonstrating features particular to Go. These include concurrency, +embedded types, methods on any type, and program construction using interfaces. +

+ +

Practical Go Programming

+

+This talk presents the development of a complete web application in Go. +It looks at design, storage, concurrency, and scaling issues in detail, using +the simple example of an URL shortening service. +See the presentation slides. +

+ +

The Go Tech Talk

+

+An hour-long talk delivered by Rob Pike at Google in October 2009. +The language's first public introduction. (See the slides in PDF format.) The language has changed since it was made, +but it's still a good introduction. +

+ +

gocoding YouTube Channel

+

+A YouTube channel that includes screencasts and other Go-related videos: +

+ + +

The Expressiveness Of Go

+

+A discussion of the qualities that make Go an expressive and comprehensible +language. The talk was presented by Rob Pike at JAOO 2010. +The recording of the event was lost due to a hardware error. +

+ +

Another Go at Language Design

+

+A tour, with some background, of the major features of Go, intended for +an audience new to the language. The talk was presented at OSCON 2010. +See the presentation slides. +

+

+This talk was also delivered at Sydney University in September 2010. A video +of the lecture is available +here. +

+ +

Go Emerging Languages Conference Talk

+

+Rob Pike's Emerging Languages Conference presentation delivered in July 2010. See the presentation slides. Abstract: +

+

+Go’s approach to concurrency differs from that of many languages, even those +(such as Erlang) that make concurrency central, yet it has deep roots. The path +from Hoare’s 1978 paper to Go provides insight into how and why Go works as it +does. +

+ +

The Go frontend for GCC

+

+A description of the Go language frontend for gcc. +Ian Lance Taylor's paper delivered at the GCC Summit 2010. +

+ +

The Go Promo Video

+

+A short promotional video featuring Russ Cox demonstrating Go's fast compiler. +

+ +
+ +
diff --git a/doc/effective_go.html b/doc/effective_go.html new file mode 100644 index 000000000..37cb516b0 --- /dev/null +++ b/doc/effective_go.html @@ -0,0 +1,3041 @@ + + +

Introduction

+ +

+Go is a new language. Although it borrows ideas from +existing languages, +it has unusual properties that make effective Go programs +different in character from programs written in its relatives. +A straightforward translation of a C++ or Java program into Go +is unlikely to produce a satisfactory result—Java programs +are written in Java, not Go. +On the other hand, thinking about the problem from a Go +perspective could produce a successful but quite different +program. +In other words, +to write Go well, it's important to understand its properties +and idioms. +It's also important to know the established conventions for +programming in Go, such as naming, formatting, program +construction, and so on, so that programs you write +will be easy for other Go programmers to understand. +

+ +

+This document gives tips for writing clear, idiomatic Go code. +It augments the language specification +and the tutorial, both of which you +should read first. +

+ +

Examples

+ +

+The Go package sources +are intended to serve not +only as the core library but also as examples of how to +use the language. +If you have a question about how to approach a problem or how something +might be implemented, they can provide answers, ideas and +background. +

+ + +

Formatting

+ +

+Formatting issues are the most contentious +but the least consequential. +People can adapt to different formatting styles +but it's better if they don't have to, and +less time is devoted to the topic +if everyone adheres to the same style. +The problem is how to approach this Utopia without a long +prescriptive style guide. +

+ +

+With Go we take an unusual +approach and let the machine +take care of most formatting issues. +The gofmt tool reads a Go program +and emits the source in a standard style of indentation +and vertical alignment, retaining and if necessary +reformatting comments. +If you want to know how to handle some new layout +situation, run gofmt; if the answer doesn't +seem right, rearrange your program (or file a bug about gofmt), +don't work around it. +

+ +

+As an example, there's no need to spend time lining up +the comments on the fields of a structure. +Gofmt will do that for you. Given the +declaration +

+ +
+type T struct {
+    name string // name of the object
+    value int // its value
+}
+
+ +

+gofmt will line up the columns: +

+ +
+type T struct {
+    name    string // name of the object
+    value   int    // its value
+}
+
+ +

+All Go code in the standard packages has been formatted with gofmt. +

+ + +

+Some formatting details remain. Very briefly, +

+ +
+
Indentation
+
We use tabs for indentation and gofmt emits them by default. + Use spaces only if you must. +
+
Line length
+
+ Go has no line length limit. Don't worry about overflowing a punched card. + If a line feels too long, wrap it and indent with an extra tab. +
+
Parentheses
+
+ Go needs fewer parentheses: control structures (if, + for, switch) do not have parentheses in + their syntax. + Also, the operator precedence hierarchy is shorter and clearer, so +
+x<<8 + y<<16
+
+ means what the spacing implies. +
+
+ +

Commentary

+ +

+Go provides C-style /* */ block comments +and C++-style // line comments. +Line comments are the norm; +block comments appear mostly as package comments and +are also useful to disable large swaths of code. +

+ +

+The program—and web server—godoc processes +Go source files to extract documentation about the contents of the +package. +Comments that appear before top-level declarations, with no intervening newlines, +are extracted along with the declaration to serve as explanatory text for the item. +The nature and style of these comments determines the +quality of the documentation godoc produces. +

+ +

+Every package should have a package comment, a block +comment preceding the package clause. +For multi-file packages, the package comment only needs to be +present in one file, and any one will do. +The package comment should introduce the package and +provide information relevant to the package as a whole. +It will appear first on the godoc page and +should set up the detailed documentation that follows. +

+ +
+/*
+    Package regexp implements a simple library for
+    regular expressions.
+
+    The syntax of the regular expressions accepted is:
+
+    regexp:
+        concatenation { '|' concatenation }
+    concatenation:
+        { closure }
+    closure:
+        term [ '*' | '+' | '?' ]
+    term:
+        '^'
+        '$'
+        '.'
+        character
+        '[' [ '^' ] character-ranges ']'
+        '(' regexp ')'
+*/
+package regexp
+
+ +

+If the package is simple, the package comment can be brief. +

+ +
+// Package path implements utility routines for
+// manipulating slash-separated filename paths.
+
+ +

+Comments do not need extra formatting such as banners of stars. +The generated output may not even be presented in a fixed-width font, so don't depend +on spacing for alignment—godoc, like gofmt, +takes care of that. +The comments are uninterpreted plain text, so HTML and other +annotations such as _this_ will reproduce verbatim and should +not be used. +Depending on the context, godoc might not even +reformat comments, so make sure they look good straight up: +use correct spelling, punctuation, and sentence structure, +fold long lines, and so on. +

+ +

+Inside a package, any comment immediately preceding a top-level declaration +serves as a doc comment for that declaration. +Every exported (capitalized) name in a program should +have a doc comment. +

+ +

+Doc comments work best as complete sentences, which allow +a wide variety of automated presentations. +The first sentence should be a one-sentence summary that +starts with the name being declared. +

+ +
+// Compile parses a regular expression and returns, if successful, a Regexp
+// object that can be used to match against text.
+func Compile(str string) (regexp *Regexp, error os.Error) {
+
+ +

+Go's declaration syntax allows grouping of declarations. +A single doc comment can introduce a group of related constants or variables. +Since the whole declaration is presented, such a comment can often be perfunctory. +

+ +
+// Error codes returned by failures to parse an expression.
+var (
+    ErrInternal      = os.NewError("regexp: internal error")
+    ErrUnmatchedLpar = os.NewError("regexp: unmatched '('")
+    ErrUnmatchedRpar = os.NewError("regexp: unmatched ')'")
+    ...
+)
+
+ +

+Even for private names, grouping can also indicate relationships between items, +such as the fact that a set of variables is protected by a mutex. +

+ +
+var (
+    countLock   sync.Mutex
+    inputCount  uint32
+    outputCount uint32
+    errorCount  uint32
+)
+
+ +

Names

+ +

+Names are as important in Go as in any other language. +In some cases they even have semantic effect: for instance, +the visibility of a name outside a package is determined by whether its +first character is upper case. +It's therefore worth spending a little time talking about naming conventions +in Go programs. +

+ + +

Package names

+ +

+When a package is imported, the package name becomes an accessor for the +contents. After +

+ +
+import "bytes"
+
+ +

+the importing package can talk about bytes.Buffer. It's +helpful if everyone using the package can use the same name to refer to +its contents, which implies that the package name should be good: +short, concise, evocative. By convention, packages are given +lower case, single-word names; there should be no need for underscores +or mixedCaps. +Err on the side of brevity, since everyone using your +package will be typing that name. +And don't worry about collisions a priori. +The package name is only the default name for imports; it need not be unique +across all source code, and in the rare case of a collision the +importing package can choose a different name to use locally. +In any case, confusion is rare because the file name in the import +determines just which package is being used. +

+ +

+Another convention is that the package name is the base name of +its source directory; +the package in src/pkg/encoding/base64 +is imported as "encoding/base64" but has name base64, +not encoding_base64 and not encodingBase64. +

+ +

+The importer of a package will use the name to refer to its contents +(the import . notation is intended mostly for tests and other +unusual situations and should be avoided unless necessary), +so exported names in the package can use that fact +to avoid stutter. +For instance, the buffered reader type in the bufio package is called Reader, +not BufReader, because users see it as bufio.Reader, +which is a clear, concise name. +Moreover, +because imported entities are always addressed with their package name, bufio.Reader +does not conflict with io.Reader. +Similarly, the function to make new instances of ring.Ring—which +is the definition of a constructor in Go—would +normally be called NewRing, but since +Ring is the only type exported by the package, and since the +package is called ring, it's called just New, +which clients of the package see as ring.New. +Use the package structure to help you choose good names. +

+ +

+Another short example is once.Do; +once.Do(setup) reads well and would not be improved by +writing once.DoOrWaitUntilDone(setup). +Long names don't automatically make things more readable. +If the name represents something intricate or subtle, it's usually better +to write a helpful doc comment than to attempt to put all the information +into the name. +

+ +

Getters

+ +

+Go doesn't provide automatic support for getters and setters. +There's nothing wrong with providing getters and setters yourself, +and it's often appropriate to do so, but it's neither idiomatic nor necessary +to put Get into the getter's name. If you have a field called +owner (lower case, unexported), the getter method should be +called Owner (upper case, exported), not GetOwner. +The use of upper-case names for export provides the hook to discriminate +the field from the method. +A setter function, if needed, will likely be called SetOwner. +Both names read well in practice: +

+
+owner := obj.Owner()
+if owner != user {
+    obj.SetOwner(user)
+}
+
+ +

Interface names

+ +

+By convention, one-method interfaces are named by +the method name plus the -er suffix: Reader, +Writer, Formatter etc. +

+ +

+There are a number of such names and it's productive to honor them and the function +names they capture. +Read, Write, Close, Flush, +String and so on have +canonical signatures and meanings. To avoid confusion, +don't give your method one of those names unless it +has the same signature and meaning. +Conversely, if your type implements a method with the +same meaning as a method on a well-known type, +give it the same name and signature; +call your string-converter method String not ToString. +

+ +

MixedCaps

+ +

+Finally, the convention in Go is to use MixedCaps +or mixedCaps rather than underscores to write +multiword names. +

+ +

Semicolons

+ +

+Like C, Go's formal grammar uses semicolons to terminate statements; +unlike C, those semicolons do not appear in the source. +Instead the lexer uses a simple rule to insert semicolons automatically +as it scans, so the input text is mostly free of them. +

+ +

+The rule is this. If the last token before a newline is an identifier +(which includes words like int and float64), +a basic literal such as a number or string constant, or one of the +tokens +

+
+break continue fallthrough return ++ -- ) }
+
+

+the lexer always inserts a semicolon after the token. +This could be summarized as, “if the newline comes +after a token that could end a statement, insert a semicolon”. +

+ +

+A semicolon can also be omitted immediately before a closing brace, +so a statement such as +

+
+    go func() { for { dst <- <-src } }()
+
+

+needs no semicolons. +Idiomatic Go programs have semicolons only in places such as +for loop clauses, to separate the initializer, condition, and +continuation elements. They are also necessary to separate multiple +statements on a line, should you write code that way. +

+ +

+One caveat. You should never put the opening brace of a +control structure (if, for, switch, +or select) on the next line. If you do, a semicolon +will be inserted before the brace, which could cause unwanted +effects. Write them like this +

+ +
+if i < f() {
+    g()
+}
+
+

+not like this +

+
+if i < f()  // wrong!
+{           // wrong!
+    g()
+}
+
+ + +

Control structures

+ +

+The control structures of Go are related to those of C but differ +in important ways. +There is no do or while loop, only a +slightly generalized +for; +switch is more flexible; +if and switch accept an optional +initialization statement like that of for; +and there are new control structures including a type switch and a +multiway communications multiplexer, select. +The syntax is also slightly different: +there are no parentheses +and the bodies must always be brace-delimited. +

+ +

If

+ +

+In Go a simple if looks like this: +

+
+if x > 0 {
+    return y
+}
+
+ +

+Mandatory braces encourage writing simple if statements +on multiple lines. It's good style to do so anyway, +especially when the body contains a control statement such as a +return or break. +

+ +

+Since if and switch accept an initialization +statement, it's common to see one used to set up a local variable. +

+ +
+if err := file.Chmod(0664); err != nil {
+    log.Print(err)
+    return err
+}
+
+ +

+In the Go libraries, you'll find that +when an if statement doesn't flow into the next statement—that is, +the body ends in break, continue, +goto, or return—the unnecessary +else is omitted. +

+ +
+f, err := os.Open(name)
+if err != nil {
+    return err
+}
+codeUsing(f)
+
+ +

+This is an example of a common situation where code must guard against a +sequence of error conditions. The code reads well if the +successful flow of control runs down the page, eliminating error cases +as they arise. Since error cases tend to end in return +statements, the resulting code needs no else statements. +

+ +
+f, err := os.Open(name)
+if err != nil {
+    return err
+}
+d, err := f.Stat()
+if err != nil {
+    return err
+}
+codeUsing(f, d)
+
+ + +

For

+ +

+The Go for loop is similar to—but not the same as—C's. +It unifies for +and while and there is no do-while. +There are three forms, only one of which has semicolons. +

+
+// Like a C for
+for init; condition; post { }
+
+// Like a C while
+for condition { }
+
+// Like a C for(;;)
+for { }
+
+ +

+Short declarations make it easy to declare the index variable right in the loop. +

+
+sum := 0
+for i := 0; i < 10; i++ {
+    sum += i
+}
+
+ +

+If you're looping over an array, slice, string, or map, +or reading from a channel, a range clause can +manage the loop. +

+
+var m map[string]int
+sum := 0
+for _, value := range m {  // key is unused
+    sum += value
+}
+
+ +

+For strings, the range does more work for you, breaking out individual +Unicode characters by parsing the UTF-8. +Erroneous encodings consume one byte and produce the +replacement rune U+FFFD. The loop +

+
+for pos, char := range "日本語" {
+    fmt.Printf("character %c starts at byte position %d\n", char, pos)
+}
+
+

+prints +

+
+character 日 starts at byte position 0
+character 本 starts at byte position 3
+character 語 starts at byte position 6
+
+ +

+Finally, Go has no comma operator and ++ and -- +are statements not expressions. +Thus if you want to run multiple variables in a for +you should use parallel assignment. +

+
+// Reverse a
+for i, j := 0, len(a)-1; i < j; i, j = i+1, j-1 {
+    a[i], a[j] = a[j], a[i]
+}
+
+ +

Switch

+ +

+Go's switch is more general than C's. +The expressions need not be constants or even integers, +the cases are evaluated top to bottom until a match is found, +and if the switch has no expression it switches on +true. +It's therefore possible—and idiomatic—to write an +if-else-if-else +chain as a switch. +

+ +
+func unhex(c byte) byte {
+    switch {
+    case '0' <= c && c <= '9':
+        return c - '0'
+    case 'a' <= c && c <= 'f':
+        return c - 'a' + 10
+    case 'A' <= c && c <= 'F':
+        return c - 'A' + 10
+    }
+    return 0
+}
+
+ +

+There is no automatic fall through, but cases can be presented +in comma-separated lists. +

+func shouldEscape(c byte) bool {
+    switch c {
+    case ' ', '?', '&', '=', '#', '+', '%':
+        return true
+    }
+    return false
+}
+
+ +

+Here's a comparison routine for byte arrays that uses two +switch statements: +

+// Compare returns an integer comparing the two byte arrays
+// lexicographically.
+// The result will be 0 if a == b, -1 if a < b, and +1 if a > b
+func Compare(a, b []byte) int {
+    for i := 0; i < len(a) && i < len(b); i++ {
+        switch {
+        case a[i] > b[i]:
+            return 1
+        case a[i] < b[i]:
+            return -1
+        }
+    }
+    switch {
+    case len(a) < len(b):
+        return -1
+    case len(a) > len(b):
+        return 1
+    }
+    return 0
+}
+
+ +

+A switch can also be used to discover the dynamic type of an interface +variable. Such a type switch uses the syntax of a type +assertion with the keyword type inside the parentheses. +If the switch declares a variable in the expression, the variable will +have the corresponding type in each clause. +

+
+switch t := interfaceValue.(type) {
+default:
+    fmt.Printf("unexpected type %T", t)  // %T prints type
+case bool:
+    fmt.Printf("boolean %t\n", t)
+case int:
+    fmt.Printf("integer %d\n", t)
+case *bool:
+    fmt.Printf("pointer to boolean %t\n", *t)
+case *int:
+    fmt.Printf("pointer to integer %d\n", *t)
+}
+
+ +

Functions

+ +

Multiple return values

+ +

+One of Go's unusual features is that functions and methods +can return multiple values. This form can be used to +improve on a couple of clumsy idioms in C programs: in-band +error returns (such as -1 for EOF) +and modifying an argument. +

+ +

+In C, a write error is signaled by a negative count with the +error code secreted away in a volatile location. +In Go, Write +can return a count and an error: “Yes, you wrote some +bytes but not all of them because you filled the device”. +The signature of *File.Write in package os is: +

+ +
+func (file *File) Write(b []byte) (n int, err Error)
+
+ +

+and as the documentation says, it returns the number of bytes +written and a non-nil Error when n +!= len(b). +This is a common style; see the section on error handling for more examples. +

+ +

+A similar approach obviates the need to pass a pointer to a return +value to simulate a reference parameter. +Here's a simple-minded function to +grab a number from a position in a byte array, returning the number +and the next position. +

+ +
+func nextInt(b []byte, i int) (int, int) {
+    for ; i < len(b) && !isDigit(b[i]); i++ {
+    }
+    x := 0
+    for ; i < len(b) && isDigit(b[i]); i++ {
+        x = x*10 + int(b[i])-'0'
+    }
+    return x, i
+}
+
+ +

+You could use it to scan the numbers in an input array a like this: +

+ +
+    for i := 0; i < len(a); {
+        x, i = nextInt(a, i)
+        fmt.Println(x)
+    }
+
+ +

Named result parameters

+ +

+The return or result "parameters" of a Go function can be given names and +used as regular variables, just like the incoming parameters. +When named, they are initialized to the zero values for their types when +the function begins; if the function executes a return statement +with no arguments, the current values of the result parameters are +used as the returned values. +

+ +

+The names are not mandatory but they can make code shorter and clearer: +they're documentation. +If we name the results of nextInt it becomes +obvious which returned int +is which. +

+ +
+func nextInt(b []byte, pos int) (value, nextPos int) {
+
+ +

+Because named results are initialized and tied to an unadorned return, they can simplify +as well as clarify. Here's a version +of io.ReadFull that uses them well: +

+ +
+func ReadFull(r Reader, buf []byte) (n int, err os.Error) {
+    for len(buf) > 0 && err == nil {
+        var nr int
+        nr, err = r.Read(buf)
+        n += nr
+        buf = buf[nr:len(buf)]
+    }
+    return
+}
+
+ +

Defer

+ +

+Go's defer statement schedules a function call (the +deferred function) to be run immediately before the function +executing the defer returns. It's an unusual but +effective way to deal with situations such as resources that must be +released regardless of which path a function takes to return. The +canonical examples are unlocking a mutex or closing a file. +

+ +
+// Contents returns the file's contents as a string.
+func Contents(filename string) (string, os.Error) {
+    f, err := os.Open(filename)
+    if err != nil {
+        return "", err
+    }
+    defer f.Close()  // f.Close will run when we're finished.
+
+    var result []byte
+    buf := make([]byte, 100)
+    for {
+        n, err := f.Read(buf[0:])
+        result = append(result, buf[0:n]...) // append is discussed later.
+        if err != nil {
+            if err == os.EOF {
+                break
+            }
+            return "", err  // f will be closed if we return here.
+        }
+    }
+    return string(result), nil // f will be closed if we return here.
+}
+
+ +

+Deferring a call to a function such as Close has two advantages. First, it +guarantees that you will never forget to close the file, a mistake +that's easy to make if you later edit the function to add a new return +path. Second, it means that the close sits near the open, +which is much clearer than placing it at the end of the function. +

+ +

+The arguments to the deferred function (which include the receiver if +the function is a method) are evaluated when the defer +executes, not when the call executes. Besides avoiding worries +about variables changing values as the function executes, this means +that a single deferred call site can defer multiple function +executions. Here's a silly example. +

+ +
+for i := 0; i < 5; i++ {
+    defer fmt.Printf("%d ", i)
+}
+
+ +

+Deferred functions are executed in LIFO order, so this code will cause +4 3 2 1 0 to be printed when the function returns. A +more plausible example is a simple way to trace function execution +through the program. We could write a couple of simple tracing +routines like this: +

+ +
+func trace(s string)   { fmt.Println("entering:", s) }
+func untrace(s string) { fmt.Println("leaving:", s) }
+
+// Use them like this:
+func a() {
+    trace("a")
+    defer untrace("a")
+    // do something....
+}
+
+ +

+We can do better by exploiting the fact that arguments to deferred +functions are evaluated when the defer executes. The +tracing routine can set up the argument to the untracing routine. +This example: +

+ +
+func trace(s string) string {
+    fmt.Println("entering:", s)
+    return s
+}
+
+func un(s string) {
+    fmt.Println("leaving:", s)
+}
+
+func a() {
+    defer un(trace("a"))
+    fmt.Println("in a")
+}
+
+func b() {
+    defer un(trace("b"))
+    fmt.Println("in b")
+    a()
+}
+
+func main() {
+    b()
+}
+
+ +

+prints +

+ +
+entering: b
+in b
+entering: a
+in a
+leaving: a
+leaving: b
+
+ +

+For programmers accustomed to block-level resource management from +other languages, defer may seem peculiar, but its most +interesting and powerful applications come precisely from the fact +that it's not block-based but function-based. In the section on +panic and recover we'll see another +example of its possibilities. +

+ +

Data

+ +

Allocation with new

+ +

+Go has two allocation primitives, the built-in functions +new and make. +They do different things and apply to different types, which can be confusing, +but the rules are simple. +Let's talk about new first. +It's a built-in function that allocates memory, but unlike its namesakes +in some other languages it does not initialize the memory, +it only zeroes it. +That is, +new(T) allocates zeroed storage for a new item of type +T and returns its address, a value of type *T. +In Go terminology, it returns a pointer to a newly allocated zero value of type +T. +

+ +

+Since the memory returned by new is zeroed, it's helpful to arrange that the +zeroed object can be used without further initialization. This means a user of +the data structure can create one with new and get right to +work. +For example, the documentation for bytes.Buffer states that +"the zero value for Buffer is an empty buffer ready to use." +Similarly, sync.Mutex does not +have an explicit constructor or Init method. +Instead, the zero value for a sync.Mutex +is defined to be an unlocked mutex. +

+ +

+The zero-value-is-useful property works transitively. Consider this type declaration. +

+ +
+type SyncedBuffer struct {
+    lock    sync.Mutex
+    buffer  bytes.Buffer
+}
+
+ +

+Values of type SyncedBuffer are also ready to use immediately upon allocation +or just declaration. In the next snippet, both p and v will work +correctly without further arrangement. +

+ +
+p := new(SyncedBuffer)  // type *SyncedBuffer
+var v SyncedBuffer      // type  SyncedBuffer
+
+ +

Constructors and composite literals

+ +

+Sometimes the zero value isn't good enough and an initializing +constructor is necessary, as in this example derived from +package os. +

+ +
+func NewFile(fd int, name string) *File {
+    if fd < 0 {
+        return nil
+    }
+    f := new(File)
+    f.fd = fd
+    f.name = name
+    f.dirinfo = nil
+    f.nepipe = 0
+    return f
+}
+
+ +

+There's a lot of boiler plate in there. We can simplify it +using a composite literal, which is +an expression that creates a +new instance each time it is evaluated. +

+ +
+func NewFile(fd int, name string) *File {
+    if fd < 0 {
+        return nil
+    }
+    f := File{fd, name, nil, 0}
+    return &f
+}
+
+ +

+Note that, unlike in C, it's perfectly OK to return the address of a local variable; +the storage associated with the variable survives after the function +returns. +In fact, taking the address of a composite literal +allocates a fresh instance each time it is evaluated, +so we can combine these last two lines. +

+ +
+    return &File{fd, name, nil, 0}
+
+ +

+The fields of a composite literal are laid out in order and must all be present. +However, by labeling the elements explicitly as field:value +pairs, the initializers can appear in any +order, with the missing ones left as their respective zero values. Thus we could say +

+ +
+    return &File{fd: fd, name: name}
+
+ +

+As a limiting case, if a composite literal contains no fields at all, it creates +a zero value for the type. The expressions new(File) and &File{} are equivalent. +

+ +

+Composite literals can also be created for arrays, slices, and maps, +with the field labels being indices or map keys as appropriate. +In these examples, the initializations work regardless of the values of Enone, +Eio, and Einval, as long as they are distinct. +

+ +
+a := [...]string   {Enone: "no error", Eio: "Eio", Einval: "invalid argument"}
+s := []string      {Enone: "no error", Eio: "Eio", Einval: "invalid argument"}
+m := map[int]string{Enone: "no error", Eio: "Eio", Einval: "invalid argument"}
+
+ +

Allocation with make

+ +

+Back to allocation. +The built-in function make(T, args) serves +a purpose different from new(T). +It creates slices, maps, and channels only, and it returns an initialized (not zero) +value of type T, not *T. +The reason for the distinction +is that these three types are, under the covers, references to data structures that +must be initialized before use. +A slice, for example, is a three-item descriptor +containing a pointer to the data (inside an array), the length, and the +capacity, and until those items are initialized, the slice is nil. +For slices, maps, and channels, +make initializes the internal data structure and prepares +the value for use. +For instance, +

+ +
+make([]int, 10, 100)
+
+ +

+allocates an array of 100 ints and then creates a slice +structure with length 10 and a capacity of 100 pointing at the first +10 elements of the array. +(When making a slice, the capacity can be omitted; see the section on slices +for more information.) +In contrast, new([]int) returns a pointer to a newly allocated, zeroed slice +structure, that is, a pointer to a nil slice value. + +

+These examples illustrate the difference between new and +make. +

+ +
+var p *[]int = new([]int)       // allocates slice structure; *p == nil; rarely useful
+var v  []int = make([]int, 100) // the slice v now refers to a new array of 100 ints
+
+// Unnecessarily complex:
+var p *[]int = new([]int)
+*p = make([]int, 100, 100)
+
+// Idiomatic:
+v := make([]int, 100)
+
+ +

+Remember that make applies only to maps, slices and channels +and does not return a pointer. +To obtain an explicit pointer allocate with new. +

+ +

Arrays

+ +

+Arrays are useful when planning the detailed layout of memory and sometimes +can help avoid allocation, but primarily +they are a building block for slices, the subject of the next section. +To lay the foundation for that topic, here are a few words about arrays. +

+ +

+There are major differences between the ways arrays work in Go and C. +In Go, +

+
    +
  • +Arrays are values. Assigning one array to another copies all the elements. +
  • +
  • +In particular, if you pass an array to a function, it +will receive a copy of the array, not a pointer to it. +
  • +The size of an array is part of its type. The types [10]int +and [20]int are distinct. +
  • +
+ +

+The value property can be useful but also expensive; if you want C-like behavior and efficiency, +you can pass a pointer to the array. +

+ +
+func Sum(a *[3]float64) (sum float64) {
+    for _, v := range *a {
+        sum += v
+    }
+    return
+}
+
+array := [...]float64{7.0, 8.5, 9.1}
+x := Sum(&array)  // Note the explicit address-of operator
+
+ +

+But even this style isn't idiomatic Go. Slices are. +

+ +

Slices

+ +

+Slices wrap arrays to give a more general, powerful, and convenient +interface to sequences of data. Except for items with explicit +dimension such as transformation matrices, most array programming in +Go is done with slices rather than simple arrays. +

+

+Slices are reference types, which means that if you assign one +slice to another, both refer to the same underlying array. For +instance, if a function takes a slice argument, changes it makes to +the elements of the slice will be visible to the caller, analogous to +passing a pointer to the underlying array. A Read +function can therefore accept a slice argument rather than a pointer +and a count; the length within the slice sets an upper +limit of how much data to read. Here is the signature of the +Read method of the File type in package +os: +

+
+func (file *File) Read(buf []byte) (n int, err os.Error)
+
+

+The method returns the number of bytes read and an error value, if +any. To read into the first 32 bytes of a larger buffer +b, slice (here used as a verb) the buffer. +

+
+    n, err := f.Read(buf[0:32])
+
+

+Such slicing is common and efficient. In fact, leaving efficiency aside for +the moment, this snippet would also read the first 32 bytes of the buffer. +

+
+    var n int
+    var err os.Error
+    for i := 0; i < 32; i++ {
+        nbytes, e := f.Read(buf[i:i+1])  // Read one byte.
+        if nbytes == 0 || e != nil {
+            err = e
+            break
+        }
+        n += nbytes
+    }
+
+

+The length of a slice may be changed as long as it still fits within +the limits of the underlying array; just assign it to a slice of +itself. The capacity of a slice, accessible by the built-in +function cap, reports the maximum length the slice may +assume. Here is a function to append data to a slice. If the data +exceeds the capacity, the slice is reallocated. The +resulting slice is returned. The function uses the fact that +len and cap are legal when applied to the +nil slice, and return 0. +

+
+func Append(slice, data[]byte) []byte {
+    l := len(slice)
+    if l + len(data) > cap(slice) {  // reallocate
+        // Allocate double what's needed, for future growth.
+        newSlice := make([]byte, (l+len(data))*2)
+        // The copy function is predeclared and works for any slice type.
+        copy(newSlice, slice)
+        slice = newSlice
+    }
+    slice = slice[0:l+len(data)]
+    for i, c := range data {
+        slice[l+i] = c
+    }
+    return slice
+}
+
+

+We must return the slice afterwards because, although Append +can modify the elements of slice, the slice itself (the run-time data +structure holding the pointer, length, and capacity) is passed by value. +

+The idea of appending to a slice is so useful it's captured by the +append built-in function. To understand that function's +design, though, we need a little more information, so we'll return +to it later. +

+ + +

Maps

+ +

+Maps are a convenient and powerful built-in data structure to associate +values of different types. +The key can be of any type for which the equality operator is defined, +such as integers, +floating point and complex numbers, +strings, pointers, and interfaces (as long as the dynamic type +supports equality). Structs, arrays and slices cannot be used as map keys, +because equality is not defined on those types. +Like slices, maps are a reference type. If you pass a map to a function +that changes the contents of the map, the changes will be visible +in the caller. +

+

+Maps can be constructed using the usual composite literal syntax +with colon-separated key-value pairs, +so it's easy to build them during initialization. +

+
+var timeZone = map[string] int {
+    "UTC":  0*60*60,
+    "EST": -5*60*60,
+    "CST": -6*60*60,
+    "MST": -7*60*60,
+    "PST": -8*60*60,
+}
+
+

+Assigning and fetching map values looks syntactically just like +doing the same for arrays except that the index doesn't need to +be an integer. +

+
+offset := timeZone["EST"]
+
+

+An attempt to fetch a map value with a key that +is not present in the map will return the zero value for the type +of the entries +in the map. For instance, if the map contains integers, looking +up a non-existent key will return 0. +A set can be implemented as a map with value type bool. +Set the map entry to true to put the value in the set, and then +test it by simple indexing. +

+
+attended := map[string] bool {
+    "Ann": true,
+    "Joe": true,
+    ...
+}
+
+if attended[person] { // will be false if person is not in the map
+    fmt.Println(person, "was at the meeting")
+}
+
+

+Sometimes you need to distinguish a missing entry from +a zero value. Is there an entry for "UTC" +or is that zero value because it's not in the map at all? +You can discriminate with a form of multiple assignment. +

+
+var seconds int
+var ok bool
+seconds, ok = timeZone[tz]
+
+

+For obvious reasons this is called the “comma ok” idiom. +In this example, if tz is present, seconds +will be set appropriately and ok will be true; if not, +seconds will be set to zero and ok will +be false. +Here's a function that puts it together with a nice error report: +

+
+func offset(tz string) int {
+    if seconds, ok := timeZone[tz]; ok {
+        return seconds
+    }
+    log.Println("unknown time zone:", tz)
+    return 0
+}
+
+

+To test for presence in the map without worrying about the actual value, +you can use the blank identifier, a simple underscore (_). +The blank identifier can be assigned or declared with any value of any type, with the +value discarded harmlessly. For testing just presence in a map, use the blank +identifier in place of the usual variable for the value. +

+
+_, present := timeZone[tz]
+
+

+To delete a map entry, turn the multiple assignment around by placing +an extra boolean on the right; if the boolean is false, the entry +is deleted. It's safe to do this even if the key is already absent +from the map. +

+
+timeZone["PDT"] = 0, false  // Now on Standard Time
+
+ +

Printing

+ +

+Formatted printing in Go uses a style similar to C's printf +family but is richer and more general. The functions live in the fmt +package and have capitalized names: fmt.Printf, fmt.Fprintf, +fmt.Sprintf and so on. The string functions (Sprintf etc.) +return a string rather than filling in a provided buffer. +

+

+You don't need to provide a format string. For each of Printf, +Fprintf and Sprintf there is another pair +of functions, for instance Print and Println. +These functions do not take a format string but instead generate a default +format for each argument. The Println versions also insert a blank +between arguments and append a newline to the output while +the Print versions add blanks only if the operand on neither side is a string. +In this example each line produces the same output. +

+
+fmt.Printf("Hello %d\n", 23)
+fmt.Fprint(os.Stdout, "Hello ", 23, "\n")
+fmt.Println("Hello", 23)
+fmt.Println(fmt.Sprint("Hello ", 23))
+
+

+As mentioned in +the tutorial, fmt.Fprint +and friends take as a first argument any object +that implements the io.Writer interface; the variables os.Stdout +and os.Stderr are familiar instances. +

+

+Here things start to diverge from C. First, the numeric formats such as %d +do not take flags for signedness or size; instead, the printing routines use the +type of the argument to decide these properties. +

+
+var x uint64 = 1<<64 - 1
+fmt.Printf("%d %x; %d %x\n", x, x, int64(x), int64(x))
+
+

+prints +

+
+18446744073709551615 ffffffffffffffff; -1 -1
+
+

+If you just want the default conversion, such as decimal for integers, you can use +the catchall format %v (for “value”); the result is exactly +what Print and Println would produce. +Moreover, that format can print any value, even arrays, structs, and +maps. Here is a print statement for the time zone map defined in the previous section. +

+
+fmt.Printf("%v\n", timeZone)  // or just fmt.Println(timeZone)
+
+

+which gives output +

+
+map[CST:-21600 PST:-28800 EST:-18000 UTC:0 MST:-25200]
+
+

+For maps the keys may be output in any order, of course. +When printing a struct, the modified format %+v annotates the +fields of the structure with their names, and for any value the alternate +format %#v prints the value in full Go syntax. +

+
+type T struct {
+    a int
+    b float
+    c string
+}
+t := &T{ 7, -2.35, "abc\tdef" }
+fmt.Printf("%v\n", t)
+fmt.Printf("%+v\n", t)
+fmt.Printf("%#v\n", t)
+fmt.Printf("%#v\n", timeZone)
+
+

+prints +

+
+&{7 -2.35 abc   def}
+&{a:7 b:-2.35 c:abc     def}
+&main.T{a:7, b:-2.35, c:"abc\tdef"}
+map[string] int{"CST":-21600, "PST":-28800, "EST":-18000, "UTC":0, "MST":-25200}
+
+

+(Note the ampersands.) +That quoted string format is also available through %q when +applied to a value of type string or []byte; +the alternate format %#q will use backquotes instead if possible. +Also, %x works on strings and arrays of bytes as well as on integers, +generating a long hexadecimal string, and with +a space in the format (% x) it puts spaces between the bytes. +

+

+Another handy format is %T, which prints the type of a value. +

+fmt.Printf("%T\n", timeZone)
+
+

+prints +

+
+map[string] int
+
+

+If you want to control the default format for a custom type, all that's required is to define +a method with the signature String() string on the type. +For our simple type T, that might look like this. +

+
+func (t *T) String() string {
+    return fmt.Sprintf("%d/%g/%q", t.a, t.b, t.c)
+}
+fmt.Printf("%v\n", t)
+
+

+to print in the format +

+
+7/-2.35/"abc\tdef"
+
+

+(If you need to print values of type T as well as pointers to T, +the receiver for String must be of value type; this example used a pointer because +that's more efficient and idiomatic for struct types. +See the section below on pointers vs. value receivers for more information.) +

+

+Our String method is able to call Sprintf because the +print routines are fully reentrant and can be used recursively. +We can even go one step further and pass a print routine's arguments directly to another such routine. +The signature of Printf uses the type ...interface{} +for its final argument to specify that an arbitrary number of parameters (of arbitrary type) +can appear after the format. +

+
+func Printf(format string, v ...interface{}) (n int, errno os.Error) {
+
+

+Within the function Printf, v acts like a variable of type +[]interface{} but if it is passed to another variadic function, it acts like +a regular list of arguments. +Here is the implementation of the +function log.Println we used above. It passes its arguments directly to +fmt.Sprintln for the actual formatting. +

+
+// Println prints to the standard logger in the manner of fmt.Println.
+func Println(v ...interface{}) {
+    std.Output(2, fmt.Sprintln(v...))  // Output takes parameters (int, string)
+}
+
+

+We write ... after v in the nested call to Sprintln to tell the +compiler to treat v as a list of arguments; otherwise it would just pass +v as a single slice argument. +

+There's even more to printing than we've covered here. See the godoc documentation +for package fmt for the details. +

+

+By the way, a ... parameter can be of a specific type, for instance ...int +for a min function that chooses the least of a list of integers: +

+
+func Min(a ...int) int {
+    min := int(^uint(0) >> 1)  // largest int
+    for _, i := range a {
+        if i < min {
+            min = i
+        }
+    }
+    return min
+}
+
+ +

Append

+

+Now we have the missing piece we needed to explain the design of +the append built-in function. The signature of append +is different from our custom Append function above. +Schematically, it's like this: +

+func append(slice []T, elements...T) []T
+
+where T is a placeholder for any given type. You can't +actually write a function in Go where the type T +is determined by the caller. +That's why append is built in: it needs support from the +compiler. +

+What append does is append the elements to the end of +the slice and return the result. The result needs to be returned +because, as with our hand-written Append, the underlying +array may change. This simple example +

+x := []int{1,2,3}
+x = append(x, 4, 5, 6)
+fmt.Println(x)
+
+prints [1 2 3 4 5 6]. So append works a +little like Printf, collecting an arbitrary number of +arguments. +

+But what if we wanted to do what our Append does and +append a slice to a slice? Easy: use ... at the call +site, just as we did in the call to Output above. This +snippet produces identical output to the one above. +

+x := []int{1,2,3}
+y := []int{4,5,6}
+x = append(x, y...)
+fmt.Println(x)
+
+Without that ..., it wouldn't compile because the types +would be wrong; y is not of type int. + +

Initialization

+ +

+Although it doesn't look superficially very different from +initialization in C or C++, initialization in Go is more powerful. +Complex structures can be built during initialization and the ordering +issues between initialized objects in different packages are handled +correctly. +

+ +

Constants

+ +

+Constants in Go are just that—constant. +They are created at compile time, even when defined as +locals in functions, +and can only be numbers, strings or booleans. +Because of the compile-time restriction, the expressions +that define them must be constant expressions, +evaluatable by the compiler. For instance, +1<<3 is a constant expression, while +math.Sin(math.Pi/4) is not because +the function call to math.Sin needs +to happen at run time. +

+ +

+In Go, enumerated constants are created using the iota +enumerator. Since iota can be part of an expression and +expressions can be implicitly repeated, it is easy to build intricate +sets of values. +

+
+type ByteSize float64
+const (
+    _ = iota  // ignore first value by assigning to blank identifier
+    KB ByteSize = 1<<(10*iota)
+    MB
+    GB
+    TB
+    PB
+    EB
+    ZB
+    YB
+)
+
+

+The ability to attach a method such as String to a +type makes it possible for such values to format themselves +automatically for printing, even as part of a general type. +

+
+func (b ByteSize) String() string {
+    switch {
+    case b >= YB:
+        return fmt.Sprintf("%.2fYB", float64(b/YB))
+    case b >= ZB:
+        return fmt.Sprintf("%.2fZB", float64(b/ZB))
+    case b >= EB:
+        return fmt.Sprintf("%.2fEB", float64(b/EB))
+    case b >= PB:
+        return fmt.Sprintf("%.2fPB", float64(b/PB))
+    case b >= TB:
+        return fmt.Sprintf("%.2fTB", float64(b/TB))
+    case b >= GB:
+        return fmt.Sprintf("%.2fGB", float64(b/GB))
+    case b >= MB:
+        return fmt.Sprintf("%.2fMB", float64(b/MB))
+    case b >= KB:
+        return fmt.Sprintf("%.2fKB", float64(b/KB))
+    }
+    return fmt.Sprintf("%.2fB", float64(b))
+}
+
+

+(The float64 conversions prevent Sprintf +from recurring back through the String method for +ByteSize.) +The expression YB prints as 1.00YB, +while ByteSize(1e13) prints as 9.09TB. +

+ +

Variables

+ +

+Variables can be initialized just like constants but the +initializer can be a general expression computed at run time. +

+
+var (
+    HOME = os.Getenv("HOME")
+    USER = os.Getenv("USER")
+    GOROOT = os.Getenv("GOROOT")
+)
+
+ +

The init function

+ +

+Finally, each source file can define its own niladic init function to +set up whatever state is required. (Actually each file can have multiple +init functions.) The only restriction is that, although +goroutines can be launched during initialization, they will not begin +execution until it completes; initialization always runs as a single thread +of execution. +And finally means finally: init is called after all the +variable declarations in the package have evaluated their initializers, +and those are evaluated only after all the imported packages have been +initialized. +

+

+Besides initializations that cannot be expressed as declarations, +a common use of init functions is to verify or repair +correctness of the program state before real execution begins. +

+ +
+func init() {
+    if USER == "" {
+        log.Fatal("$USER not set")
+    }
+    if HOME == "" {
+        HOME = "/usr/" + USER
+    }
+    if GOROOT == "" {
+        GOROOT = HOME + "/go"
+    }
+    // GOROOT may be overridden by --goroot flag on command line.
+    flag.StringVar(&GOROOT, "goroot", GOROOT, "Go root directory")
+}
+
+ +

Methods

+ +

Pointers vs. Values

+

+Methods can be defined for any named type that is not a pointer or an interface; +the receiver does not have to be a struct. +

+In the discussion of slices above, we wrote an Append +function. We can define it as a method on slices instead. To do +this, we first declare a named type to which we can bind the method, and +then make the receiver for the method a value of that type. +

+
+type ByteSlice []byte
+
+func (slice ByteSlice) Append(data []byte) []byte {
+    // Body exactly the same as above
+}
+
+

+This still requires the method to return the updated slice. We can +eliminate that clumsiness by redefining the method to take a +pointer to a ByteSlice as its receiver, so the +method can overwrite the caller's slice. +

+
+func (p *ByteSlice) Append(data []byte) {
+    slice := *p
+    // Body as above, without the return.
+    *p = slice
+}
+
+

+In fact, we can do even better. If we modify our function so it looks +like a standard Write method, like this, +

+
+func (p *ByteSlice) Write(data []byte) (n int, err os.Error) {
+    slice := *p
+    // Again as above.
+    *p = slice
+    return len(data), nil
+}
+
+

+then the type *ByteSlice satisfies the standard interface +io.Writer, which is handy. For instance, we can +print into one. +

+
+    var b ByteSlice
+    fmt.Fprintf(&b, "This hour has %d days\n", 7)
+
+

+We pass the address of a ByteSlice +because only *ByteSlice satisfies io.Writer. +The rule about pointers vs. values for receivers is that value methods +can be invoked on pointers and values, but pointer methods can only be +invoked on pointers. This is because pointer methods can modify the +receiver; invoking them on a copy of the value would cause those +modifications to be discarded. +

+

+By the way, the idea of using Write on a slice of bytes +is implemented by bytes.Buffer. +

+ +

Interfaces and other types

+ +

Interfaces

+

+Interfaces in Go provide a way to specify the behavior of an +object: if something can do this, then it can be used +here. We've seen a couple of simple examples already; +custom printers can be implemented by a String method +while Fprintf can generate output to anything +with a Write method. +Interfaces with only one or two methods are common in Go code, and are +usually given a name derived from the method, such as io.Writer +for something that implements Write. +

+

+A type can implement multiple interfaces. +For instance, a collection can be sorted +by the routines in package sort if it implements +sort.Interface, which contains Len(), +Less(i, j int) bool, and Swap(i, j int), +and it could also have a custom formatter. +In this contrived example Sequence satisfies both. +

+
+type Sequence []int
+
+// Methods required by sort.Interface.
+func (s Sequence) Len() int {
+    return len(s)
+}
+func (s Sequence) Less(i, j int) bool {
+    return s[i] < s[j]
+}
+func (s Sequence) Swap(i, j int) {
+    s[i], s[j] = s[j], s[i]
+}
+
+// Method for printing - sorts the elements before printing.
+func (s Sequence) String() string {
+    sort.Sort(s)
+    str := "["
+    for i, elem := range s {
+        if i > 0 {
+            str += " "
+        }
+        str += fmt.Sprint(elem)
+    }
+    return str + "]"
+}
+
+ +

Conversions

+ +

+The String method of Sequence is recreating the +work that Sprint already does for slices. We can share the +effort if we convert the Sequence to a plain +[]int before calling Sprint. +

+
+func (s Sequence) String() string {
+    sort.Sort(s)
+    return fmt.Sprint([]int(s))
+}
+
+

+The conversion causes s to be treated as an ordinary slice +and therefore receive the default formatting. +Without the conversion, Sprint would find the +String method of Sequence and recur indefinitely. +Because the two types (Sequence and []int) +are the same if we ignore the type name, it's legal to convert between them. +The conversion doesn't create a new value, it just temporarily acts +as though the existing value has a new type. +(There are other legal conversions, such as from integer to floating point, that +do create a new value.) +

+

+It's an idiom in Go programs to convert the +type of an expression to access a different +set of methods. As an example, we could use the existing +type sort.IntArray to reduce the entire example +to this: +

+
+type Sequence []int
+
+// Method for printing - sorts the elements before printing
+func (s Sequence) String() string {
+    sort.IntArray(s).Sort()
+    return fmt.Sprint([]int(s))
+}
+
+

+Now, instead of having Sequence implement multiple +interfaces (sorting and printing), we're using the ability of a data item to be +converted to multiple types (Sequence, sort.IntArray +and []int), each of which does some part of the job. +That's more unusual in practice but can be effective. +

+ +

Generality

+

+If a type exists only to implement an interface +and has no exported methods beyond that interface, +there is no need to export the type itself. +Exporting just the interface makes it clear that +it's the behavior that matters, not the implementation, +and that other implementations with different properties +can mirror the behavior of the original type. +It also avoids the need to repeat the documentation +on every instance of a common method. +

+

+In such cases, the constructor should return an interface value +rather than the implementing type. +As an example, in the hash libraries +both crc32.NewIEEE and adler32.New +return the interface type hash.Hash32. +Substituting the CRC-32 algorithm for Adler-32 in a Go program +requires only changing the constructor call; +the rest of the code is unaffected by the change of algorithm. +

+

+A similar approach allows the streaming cipher algorithms +in the crypto/block package to be +separated from the block ciphers they chain together. +By analogy with the bufio package, +they wrap a Cipher interface +and return hash.Hash, +io.Reader, or io.Writer +interface values, not specific implementations. +

+

+The interface to crypto/block includes: +

+
+type Cipher interface {
+    BlockSize() int
+    Encrypt(src, dst []byte)
+    Decrypt(src, dst []byte)
+}
+
+// NewECBDecrypter returns a reader that reads data
+// from r and decrypts it using c in electronic codebook (ECB) mode.
+func NewECBDecrypter(c Cipher, r io.Reader) io.Reader
+
+// NewCBCDecrypter returns a reader that reads data
+// from r and decrypts it using c in cipher block chaining (CBC) mode
+// with the initialization vector iv.
+func NewCBCDecrypter(c Cipher, iv []byte, r io.Reader) io.Reader
+
+

+NewECBDecrypter and NewCBCReader apply not +just to one specific encryption algorithm and data source but to any +implementation of the Cipher interface and any +io.Reader. Because they return io.Reader +interface values, replacing ECB +encryption with CBC encryption is a localized change. The constructor +calls must be edited, but because the surrounding code must treat the result only +as an io.Reader, it won't notice the difference. +

+ +

Interfaces and methods

+

+Since almost anything can have methods attached, almost anything can +satisfy an interface. One illustrative example is in the http +package, which defines the Handler interface. Any object +that implements Handler can serve HTTP requests. +

+
+type Handler interface {
+    ServeHTTP(ResponseWriter, *Request)
+}
+
+

+ResponseWriter is itself an interface that provides access +to the methods needed to return the response to the client. +Those methods include the standard Write method, so an +http.ResponseWriter can be used wherever an io.Writer +can be used. +Request is a struct containing a parsed representation +of the request from the client. +

+For brevity, let's ignore POSTs and assume HTTP requests are always +GETs; that simplification does not affect the way the handlers are +set up. Here's a trivial but complete implementation of a handler to +count the number of times the +page is visited. +

+
+// Simple counter server.
+type Counter struct {
+    n int
+}
+
+func (ctr *Counter) ServeHTTP(w http.ResponseWriter, req *http.Request) {
+    ctr.n++
+    fmt.Fprintf(w, "counter = %d\n", ctr.n)
+}
+
+

+(Keeping with our theme, note how Fprintf can print to an +http.ResponseWriter.) +For reference, here's how to attach such a server to a node on the URL tree. +

+import "http"
+...
+ctr := new(Counter)
+http.Handle("/counter", ctr)
+
+

+But why make Counter a struct? An integer is all that's needed. +(The receiver needs to be a pointer so the increment is visible to the caller.) +

+
+// Simpler counter server.
+type Counter int
+
+func (ctr *Counter) ServeHTTP(w http.ResponseWriter, req *http.Request) {
+    *ctr++
+    fmt.Fprintf(w, "counter = %d\n", *ctr)
+}
+
+

+What if your program has some internal state that needs to be notified that a page +has been visited? Tie a channel to the web page. +

+
+// A channel that sends a notification on each visit.
+// (Probably want the channel to be buffered.)
+type Chan chan *http.Request
+
+func (ch Chan) ServeHTTP(w http.ResponseWriter, req *http.Request) {
+    ch <- req
+    fmt.Fprint(w, "notification sent")
+}
+
+

+Finally, let's say we wanted to present on /args the arguments +used when invoking the server binary. +It's easy to write a function to print the arguments. +

+
+func ArgServer() {
+    for i, s := range os.Args {
+        fmt.Println(s)
+    }
+}
+
+

+How do we turn that into an HTTP server? We could make ArgServer +a method of some type whose value we ignore, but there's a cleaner way. +Since we can define a method for any type except pointers and interfaces, +we can write a method for a function. +The http package contains this code: +

+
+// The HandlerFunc type is an adapter to allow the use of
+// ordinary functions as HTTP handlers.  If f is a function
+// with the appropriate signature, HandlerFunc(f) is a
+// Handler object that calls f.
+type HandlerFunc func(ResponseWriter, *Request)
+
+// ServeHTTP calls f(c, req).
+func (f HandlerFunc) ServeHTTP(w ResponseWriter, req *Request) {
+    f(w, req)
+}
+
+

+HandlerFunc is a type with a method, ServeHTTP, +so values of that type can serve HTTP requests. Look at the implementation +of the method: the receiver is a function, f, and the method +calls f. That may seem odd but it's not that different from, say, +the receiver being a channel and the method sending on the channel. +

+

+To make ArgServer into an HTTP server, we first modify it +to have the right signature. +

+
+// Argument server.
+func ArgServer(w http.ResponseWriter, req *http.Request) {
+    for i, s := range os.Args {
+        fmt.Fprintln(w, s)
+    }
+}
+
+

+ArgServer now has same signature as HandlerFunc, +so it can be converted to that type to access its methods, +just as we converted Sequence to IntArray +to access IntArray.Sort. +The code to set it up is concise: +

+
+http.Handle("/args", http.HandlerFunc(ArgServer))
+
+

+When someone visits the page /args, +the handler installed at that page has value ArgServer +and type HandlerFunc. +The HTTP server will invoke the method ServeHTTP +of that type, with ArgServer as the receiver, which will in turn call +ArgServer (via the invocation f(c, req) +inside HandlerFunc.ServeHTTP). +The arguments will then be displayed. +

+

+In this section we have made an HTTP server from a struct, an integer, +a channel, and a function, all because interfaces are just sets of +methods, which can be defined for (almost) any type. +

+ +

Embedding

+ +

+Go does not provide the typical, type-driven notion of subclassing, +but it does have the ability to “borrow” pieces of an +implementation by embedding types within a struct or +interface. +

+

+Interface embedding is very simple. +We've mentioned the io.Reader and io.Writer interfaces before; +here are their definitions. +

+
+type Reader interface {
+    Read(p []byte) (n int, err os.Error)
+}
+
+type Writer interface {
+    Write(p []byte) (n int, err os.Error)
+}
+
+

+The io package also exports several other interfaces +that specify objects that can implement several such methods. +For instance, there is io.ReadWriter, an interface +containing both Read and Write. +We could specify io.ReadWriter by listing the +two methods explicitly, but it's easier and more evocative +to embed the two interfaces to form the new one, like this: +

+
+// ReadWriter is the interface that combines the Reader and Writer interfaces.
+type ReadWriter interface {
+    Reader
+    Writer
+}
+
+

+This says just what it looks like: A ReadWriter can do +what a Reader does and what a Writer +does; it is a union of the embedded interfaces (which must be disjoint +sets of methods). +Only interfaces can be embedded within interfaces. +

+The same basic idea applies to structs, but with more far-reaching +implications. The bufio package has two struct types, +bufio.Reader and bufio.Writer, each of +which of course implements the analogous interfaces from package +io. +And bufio also implements a buffered reader/writer, +which it does by combining a reader and a writer into one struct +using embedding: it lists the types within the struct +but does not give them field names. +

+
+// ReadWriter stores pointers to a Reader and a Writer.
+// It implements io.ReadWriter.
+type ReadWriter struct {
+    *Reader  // *bufio.Reader
+    *Writer  // *bufio.Writer
+}
+
+

+The embedded elements are pointers to structs and of course +must be initialized to point to valid structs before they +can be used. +The ReadWriter struct could be written as +

+
+type ReadWriter struct {
+    reader *Reader
+    writer *Writer
+}
+
+

+but then to promote the methods of the fields and to +satisfy the io interfaces, we would also need +to provide forwarding methods, like this: +

+
+func (rw *ReadWriter) Read(p []byte) (n int, err os.Error) {
+    return rw.reader.Read(p)
+}
+
+

+By embedding the structs directly, we avoid this bookkeeping. +The methods of embedded types come along for free, which means that bufio.ReadWriter +not only has the methods of bufio.Reader and bufio.Writer, +it also satisfies all three interfaces: +io.Reader, +io.Writer, and +io.ReadWriter. +

+

+There's an important way in which embedding differs from subclassing. When we embed a type, +the methods of that type become methods of the outer type, +but when they are invoked the receiver of the method is the inner type, not the outer one. +In our example, when the Read method of a bufio.ReadWriter is +invoked, it has exactly the same effect as the forwarding method written out above; +the receiver is the reader field of the ReadWriter, not the +ReadWriter itself. +

+

+Embedding can also be a simple convenience. +This example shows an embedded field alongside a regular, named field. +

+
+type Job struct {
+    Command string
+    *log.Logger
+}
+
+

+The Job type now has the Log, Logf +and other +methods of *log.Logger. We could have given the Logger +a field name, of course, but it's not necessary to do so. And now, once +initialized, we can +log to the Job: +

+
+job.Log("starting now...")
+
+

+The Logger is a regular field of the struct and we can initialize +it in the usual way with a constructor, +

+
+func NewJob(command string, logger *log.Logger) *Job {
+    return &Job{command, logger}
+}
+
+

+or with a composite literal, +

+
+job := &Job{command, log.New(os.Stderr, "Job: ", log.Ldate)}
+
+

+If we need to refer to an embedded field directly, the type name of the field, +ignoring the package qualifier, serves as a field name. If we needed to access the +*log.Logger of a Job variable job, +we would write job.Logger. +This would be useful if we wanted to refine the methods of Logger. +

+
+func (job *Job) Logf(format string, args ...interface{}) {
+    job.Logger.Logf("%q: %s", job.Command, fmt.Sprintf(format, args))
+}
+
+

+Embedding types introduces the problem of name conflicts but the rules to resolve +them are simple. +First, a field or method X hides any other item X in a more deeply +nested part of the type. +If log.Logger contained a field or method called Command, the Command field +of Job would dominate it. +

+

+Second, if the same name appears at the same nesting level, it is usually an error; +it would be erroneous to embed log.Logger if the Job struct +contained another field or method called Logger. +However, if the duplicate name is never mentioned in the program outside the type definition, it is OK. +This qualification provides some protection against changes made to types embedded from outside; there +is no problem if a field is added that conflicts with another field in another subtype if neither field +is ever used. +

+ + +

Concurrency

+ +

Share by communicating

+ +

+Concurrent programming is a large topic and there is space only for some +Go-specific highlights here. +

+

+Concurrent programming in many environments is made difficult by the +subtleties required to implement correct access to shared variables. Go encourages +a different approach in which shared values are passed around on channels +and, in fact, never actively shared by separate threads of execution. +Only one goroutine has access to the value at any given time. +Data races cannot occur, by design. +To encourage this way of thinking we have reduced it to a slogan: +

+
+Do not communicate by sharing memory; +instead, share memory by communicating. +
+

+This approach can be taken too far. Reference counts may be best done +by putting a mutex around an integer variable, for instance. But as a +high-level approach, using channels to control access makes it easier +to write clear, correct programs. +

+

+One way to think about this model is to consider a typical single-threaded +program running on one CPU. It has no need for synchronization primitives. +Now run another such instance; it too needs no synchronization. Now let those +two communicate; if the communication is the synchronizer, there's still no need +for other synchronization. Unix pipelines, for example, fit this model +perfectly. Although Go's approach to concurrency originates in Hoare's +Communicating Sequential Processes (CSP), +it can also be seen as a type-safe generalization of Unix pipes. +

+ +

Goroutines

+ +

+They're called goroutines because the existing +terms—threads, coroutines, processes, and so on—convey +inaccurate connotations. A goroutine has a simple model: it is a +function executing in parallel with other goroutines in the same +address space. It is lightweight, costing little more than the +allocation of stack space. +And the stacks start small, so they are cheap, and grow +by allocating (and freeing) heap storage as required. +

+

+Goroutines are multiplexed onto multiple OS threads so if one should +block, such as while waiting for I/O, others continue to run. Their +design hides many of the complexities of thread creation and +management. +

+

+Prefix a function or method call with the go +keyword to run the call in a new goroutine. +When the call completes, the goroutine +exits, silently. (The effect is similar to the Unix shell's +& notation for running a command in the +background.) +

+
+go list.Sort()  // run list.Sort in parallel; don't wait for it. 
+
+

+A function literal can be handy in a goroutine invocation. +

+func Announce(message string, delay int64) {
+    go func() {
+        time.Sleep(delay)
+        fmt.Println(message)
+    }()  // Note the parentheses - must call the function.
+}
+
+

+In Go, function literals are closures: the implementation makes +sure the variables referred to by the function survive as long as they are active. +

+These examples aren't too practical because the functions have no way of signaling +completion. For that, we need channels. +

+ +

Channels

+ +

+Like maps, channels are a reference type and are allocated with make. +If an optional integer parameter is provided, it sets the buffer size for the channel. +The default is zero, for an unbuffered or synchronous channel. +

+
+ci := make(chan int)            // unbuffered channel of integers
+cj := make(chan int, 0)         // unbuffered channel of integers
+cs := make(chan *os.File, 100)  // buffered channel of pointers to Files
+
+

+Channels combine communication—the exchange of a value—with +synchronization—guaranteeing that two calculations (goroutines) are in +a known state. +

+

+There are lots of nice idioms using channels. Here's one to get us started. +In the previous section we launched a sort in the background. A channel +can allow the launching goroutine to wait for the sort to complete. +

+
+c := make(chan int)  // Allocate a channel.
+// Start the sort in a goroutine; when it completes, signal on the channel.
+go func() {
+    list.Sort()
+    c <- 1  // Send a signal; value does not matter. 
+}()
+doSomethingForAWhile()
+<-c   // Wait for sort to finish; discard sent value.
+
+

+Receivers always block until there is data to receive. +If the channel is unbuffered, the sender blocks until the receiver has +received the value. +If the channel has a buffer, the sender blocks only until the +value has been copied to the buffer; if the buffer is full, this +means waiting until some receiver has retrieved a value. +

+

+A buffered channel can be used like a semaphore, for instance to +limit throughput. In this example, incoming requests are passed +to handle, which sends a value into the channel, processes +the request, and then receives a value from the channel. +The capacity of the channel buffer limits the number of +simultaneous calls to process. +

+
+var sem = make(chan int, MaxOutstanding)
+
+func handle(r *Request) {
+    sem <- 1    // Wait for active queue to drain.
+    process(r)  // May take a long time.
+    <-sem       // Done; enable next request to run.
+}
+
+func Serve(queue chan *Request) {
+    for {
+        req := <-queue
+        go handle(req)  // Don't wait for handle to finish.
+    }
+}
+
+

+Here's the same idea implemented by starting a fixed +number of handle goroutines all reading from the request +channel. +The number of goroutines limits the number of simultaneous +calls to process. +This Serve function also accepts a channel on which +it will be told to exit; after launching the goroutines it blocks +receiving from that channel. +

+
+func handle(queue chan *Request) {
+    for r := range queue {
+        process(r)
+    }
+}
+
+func Serve(clientRequests chan *clientRequests, quit chan bool) {
+    // Start handlers
+    for i := 0; i < MaxOutstanding; i++ {
+        go handle(clientRequests)
+    }
+    <-quit  // Wait to be told to exit.
+}
+
+ +

Channels of channels

+

+One of the most important properties of Go is that +a channel is a first-class value that can be allocated and passed +around like any other. A common use of this property is +to implement safe, parallel demultiplexing. +

+In the example in the previous section, handle was +an idealized handler for a request but we didn't define the +type it was handling. If that type includes a channel on which +to reply, each client can provide its own path for the answer. +Here's a schematic definition of type Request. +

+
+type Request struct {
+    args        []int
+    f           func([]int) int
+    resultChan  chan int
+}
+
+

+The client provides a function and its arguments, as well as +a channel inside the request object on which to receive the answer. +

+
+func sum(a []int) (s int) {
+    for _, v := range a {
+        s += v
+    }
+    return
+}
+
+request := &Request{[]int{3, 4, 5}, sum, make(chan int)}
+// Send request
+clientRequests <- request
+// Wait for response.
+fmt.Printf("answer: %d\n", <-request.resultChan)
+
+

+On the server side, the handler function is the only thing that changes. +

+
+func handle(queue chan *Request) {
+    for req := range queue {
+        req.resultChan <- req.f(req.args)
+    }
+}
+
+

+There's clearly a lot more to do to make it realistic, but this +code is a framework for a rate-limited, parallel, non-blocking RPC +system, and there's not a mutex in sight. +

+ +

Parallelization

+

+Another application of these ideas is to parallelize a calculation +across multiple CPU cores. If the calculation can be broken into +separate pieces, it can be parallelized, with a channel to signal +when each piece completes. +

+

+Let's say we have an expensive operation to perform on a vector of items, +and that the value of the operation on each item is independent, +as in this idealized example. +

+
+type Vector []float64
+
+// Apply the operation to v[i], v[i+1] ... up to v[n-1].
+func (v Vector) DoSome(i, n int, u Vector, c chan int) {
+    for ; i < n; i++ {
+        v[i] += u.Op(v[i])
+    }
+    c <- 1    // signal that this piece is done
+}
+
+

+We launch the pieces independently in a loop, one per CPU. +They can complete in any order but it doesn't matter; we just +count the completion signals by draining the channel after +launching all the goroutines. +

+
+const NCPU = 4  // number of CPU cores
+
+func (v Vector) DoAll(u Vector) {
+    c := make(chan int, NCPU)  // Buffering optional but sensible.
+    for i := 0; i < NCPU; i++ {
+        go v.DoSome(i*len(v)/NCPU, (i+1)*len(v)/NCPU, u, c)
+    }
+    // Drain the channel.
+    for i := 0; i < NCPU; i++ {
+        <-c    // wait for one task to complete
+    }
+    // All done.
+}
+
+
+ +

+The current implementation of gc (6g, etc.) +will not parallelize this code by default. +It dedicates only a single core to user-level processing. An +arbitrary number of goroutines can be blocked in system calls, but +by default only one can be executing user-level code at any time. +It should be smarter and one day it will be smarter, but until it +is if you want CPU parallelism you must tell the run-time +how many goroutines you want executing code simultaneously. There +are two related ways to do this. Either run your job with environment +variable GOMAXPROCS set to the number of cores to use +(default 1); or import the runtime package and call +runtime.GOMAXPROCS(NCPU). +Again, this requirement is expected to be retired as the scheduling and run-time improve. +

+ +

A leaky buffer

+ +

+The tools of concurrent programming can even make non-concurrent +ideas easier to express. Here's an example abstracted from an RPC +package. The client goroutine loops receiving data from some source, +perhaps a network. To avoid allocating and freeing buffers, it keeps +a free list, and uses a buffered channel to represent it. If the +channel is empty, a new buffer gets allocated. +Once the message buffer is ready, it's sent to the server on +serverChan. +

+
+var freeList = make(chan *Buffer, 100)
+var serverChan = make(chan *Buffer)
+
+func client() {
+    for {
+        var b *Buffer
+        // Grab a buffer if available; allocate if not.
+        select {
+        case b = <-freeList:
+            // Got one; nothing more to do.
+        default:
+            // None free, so allocate a new one.
+            b = new(Buffer)
+        }
+        load(b)              // Read next message from the net.
+        serverChan <- b      // Send to server.
+    }
+}
+
+

+The server loop receives each message from the client, processes it, +and returns the buffer to the free list. +

+
+func server() {
+    for {
+        b := <-serverChan    // Wait for work.
+        process(b)
+        // Reuse buffer if there's room.
+        select {
+        case freeList <- b:
+            // Buffer on free list; nothing more to do.
+        default:
+            // Free list full, just carry on.
+        }
+    }
+}
+
+

+The client attempts to retrieve a buffer from freeList; +if none is available, it allocates a fresh one. +The server's send to freeList puts b back +on the free list unless the list is full, in which case the +buffer is dropped on the floor to be reclaimed by +the garbage collector. +(The default clauses in the select +statements execute when no other case is ready, +meaning that the selects never block.) +This implementation builds a leaky bucket free list +in just a few lines, relying on the buffered channel and +the garbage collector for bookkeeping. +

+ +

Errors

+ +

+Library routines must often return some sort of error indication to +the caller. As mentioned earlier, Go's multivalue return makes it +easy to return a detailed error description alongside the normal +return value. By convention, errors have type os.Error, +a simple interface. +

+
+type Error interface {
+    String() string
+}
+
+

+A library writer is free to implement this interface with a +richer model under the covers, making it possible not only +to see the error but also to provide some context. +For example, os.Open returns an os.PathError. +

+
+// PathError records an error and the operation and
+// file path that caused it.
+type PathError struct {
+    Op string    // "open", "unlink", etc.
+    Path string  // The associated file.
+    Error Error  // Returned by the system call.
+}
+
+func (e *PathError) String() string {
+    return e.Op + " " + e.Path + ": " + e.Error.String()
+}
+
+

+PathError's String generates +a string like this: +

+
+open /etc/passwx: no such file or directory
+
+

+Such an error, which includes the problematic file name, the +operation, and the operating system error it triggered, is useful even +if printed far from the call that caused it; +it is much more informative than the plain +"no such file or directory". +

+ +

+When feasible, error strings should identify their origin, such as by having +a prefix naming the package that generated the error. For example, in package +image, the string representation for a decoding error due to an unknown format +is "image: unknown format". +

+ +

+Callers that care about the precise error details can +use a type switch or a type assertion to look for specific +errors and extract details. For PathErrors +this might include examining the internal Error +field for recoverable failures. +

+ +
+for try := 0; try < 2; try++ {
+    file, err = os.Open(filename)
+    if err == nil {
+        return
+    }
+    if e, ok := err.(*os.PathError); ok && e.Error == os.ENOSPC {
+        deleteTempFiles()  // Recover some space.
+        continue
+    }
+    return
+}
+
+ +

Panic

+ +

+The usual way to report an error to a caller is to return an +os.Error as an extra return value. The canonical +Read method is a well-known instance; it returns a byte +count and an os.Error. But what if the error is +unrecoverable? Sometimes the program simply cannot continue. +

+ +

+For this purpose, there is a built-in function panic +that in effect creates a run-time error that will stop the program +(but see the next section). The function takes a single argument +of arbitrary type—often a string—to be printed as the +program dies. It's also a way to indicate that something impossible has +happened, such as exiting an infinite loop. In fact, the compiler +recognizes a panic at the end of a function and +suppresses the usual check for a return statement. +

+ + +
+// A toy implementation of cube root using Newton's method.
+func CubeRoot(x float64) float64 {
+    z := x/3   // Arbitrary initial value
+    for i := 0; i < 1e6; i++ {
+        prevz := z
+        z -= (z*z*z-x) / (3*z*z)
+        if veryClose(z, prevz) {
+            return z
+        }
+    }
+    // A million iterations has not converged; something is wrong.
+    panic(fmt.Sprintf("CubeRoot(%g) did not converge", x))
+}
+
+ +

+This is only an example but real library functions should +avoid panic. If the problem can be masked or worked +around, it's always better to let things continue to run rather +than taking down the whole program. One possible counterexample +is during initialization: if the library truly cannot set itself up, +it might be reasonable to panic, so to speak. +

+ +
+var user = os.Getenv("USER")
+
+func init() {
+    if user == "" {
+        panic("no value for $USER")
+    }
+}
+
+ +

Recover

+ +

+When panic is called, including implicitly for run-time +errors such as indexing an array out of bounds or failing a type +assertion, it immediately stops execution of the current function +and begins unwinding the stack of the goroutine, running any deferred +functions along the way. If that unwinding reaches the top of the +goroutine's stack, the program dies. However, it is possible to +use the built-in function recover to regain control +of the goroutine and resume normal execution. +

+ +

+A call to recover stops the unwinding and returns the +argument passed to panic. Because the only code that +runs while unwinding is inside deferred functions, recover +is only useful inside deferred functions. +

+ +

+One application of recover is to shut down a failing goroutine +inside a server without killing the other executing goroutines. +

+ +
+func server(workChan <-chan *Work) {
+    for work := range workChan {
+        go safelyDo(work)
+    }
+}
+
+func safelyDo(work *Work) {
+    defer func() {
+        if err := recover(); err != nil {
+            log.Println("work failed:", err)
+        }
+    }()
+    do(work)
+}
+
+ +

+In this example, if do(work) panics, the result will be +logged and the goroutine will exit cleanly without disturbing the +others. There's no need to do anything else in the deferred closure; +calling recover handles the condition completely. +

+ +

+Because recover always returns nil unless called directly +from a deferred function, deferred code can call library routines that themselves +use panic and recover without failing. As an example, +the deferred function in safelyDo might call a logging function before +calling recover, and that logging code would run unaffected +by the panicking state. +

+ +

+With our recovery pattern in place, the do +function (and anything it calls) can get out of any bad situation +cleanly by calling panic. We can use that idea to +simplify error handling in complex software. Let's look at an +idealized excerpt from the regexp package, which reports +parsing errors by calling panic with a local +Error type. Here's the definition of Error, +an error method, and the Compile function. +

+ +
+// Error is the type of a parse error; it satisfies os.Error.
+type Error string
+func (e Error) String() string {
+    return string(e)
+}
+
+// error is a method of *Regexp that reports parsing errors by
+// panicking with an Error.
+func (regexp *Regexp) error(err string) {
+    panic(Error(err))
+}
+
+// Compile returns a parsed representation of the regular expression.
+func Compile(str string) (regexp *Regexp, err os.Error) {
+    regexp = new(Regexp)
+    // doParse will panic if there is a parse error.
+    defer func() {
+        if e := recover(); e != nil {
+            regexp = nil    // Clear return value.
+            err = e.(Error) // Will re-panic if not a parse error.
+        }
+    }()
+    return regexp.doParse(str), nil
+}
+
+ +

+If doParse panics, the recovery block will set the +return value to nil—deferred functions can modify +named return values. It then will then check, in the assignment +to err, that the problem was a parse error by asserting +that it has type Error. +If it does not, the type assertion will fail, causing a run-time error +that continues the stack unwinding as though nothing had interrupted +it. This check means that if something unexpected happens, such +as an array index out of bounds, the code will fail even though we +are using panic and recover to handle +user-triggered errors. +

+ +

+With error handling in place, the error method +makes it easy to report parse errors without worrying about unwinding +the parse stack by hand. +

+ +

+Useful though this pattern is, it should be used only within a package. +Parse turns its internal panic calls into +os.Error values; it does not expose panics +to its client. That is a good rule to follow. +

+ +

+By the way, this re-panic idiom changes the panic value if an actual +error occurs. However, both the original and new failures will be +presented in the crash report, so the root cause of the problem will +still be visible. Thus this simple re-panic approach is usually +sufficient—it's a crash after all—but if you want to +display only the original value, you can write a little more code to +filter unexpected problems and re-panic with the original error. +That's left as an exercise for the reader. +

+ + +

A web server

+ +

+Let's finish with a complete Go program, a web server. +This one is actually a kind of web re-server. +Google provides a service at +http://chart.apis.google.com +that does automatic formatting of data into charts and graphs. +It's hard to use interactively, though, +because you need to put the data into the URL as a query. +The program here provides a nicer interface to one form of data: given a short piece of text, +it calls on the chart server to produce a QR code, a matrix of boxes that encode the +text. +That image can be grabbed with your cell phone's camera and interpreted as, +for instance, a URL, saving you typing the URL into the phone's tiny keyboard. +

+

+Here's the complete program. +An explanation follows. +

+ +
+package main
+
+import (
+    "flag"
+    "http"
+    "io"
+    "log"
+    "old/template" // New template package coming soon...
+)
+
+var addr = flag.String("addr", ":1718", "http service address") // Q=17, R=18
+var fmap = template.FormatterMap{
+    "html": template.HTMLFormatter,
+    "url+html": UrlHtmlFormatter,
+}
+var templ = template.MustParse(templateStr, fmap)
+
+func main() {
+    flag.Parse()
+    http.Handle("/", http.HandlerFunc(QR))
+    err := http.ListenAndServe(*addr, nil)
+    if err != nil {
+        log.Fatal("ListenAndServe:", err)
+    }
+}
+
+func QR(w http.ResponseWriter, req *http.Request) {
+    templ.Execute(w, req.FormValue("s"))
+}
+
+func UrlHtmlFormatter(w io.Writer, fmt string, v ...interface{}) {
+    template.HTMLEscape(w, []byte(http.URLEscape(v[0].(string))))
+}
+
+
+const templateStr = `
+<html>
+<head>
+<title>QR Link Generator</title>
+</head>
+<body>
+{.section @}
+<img src="http://chart.apis.google.com/chart?chs=300x300&cht=qr&choe=UTF-8&chl={@|url+html}"
+/>
+<br>
+{@|html}
+<br>
+<br>
+{.end}
+<form action="/" name=f method="GET"><input maxLength=1024 size=70
+name=s value="" title="Text to QR Encode"><input type=submit
+value="Show QR" name=qr>
+</form>
+</body>
+</html>
+`
+
+ +

+The pieces up to main should be easy to follow. +The one flag sets a default HTTP port for our server. The template +variable templ is where the fun happens. It builds an HTML template +that will be executed by the server to display the page; more about +that in a moment. +

+

+The main function parses the flags and, using the mechanism +we talked about above, binds the function QR to the root path +for the server. Then http.ListenAndServe is called to start the +server; it blocks while the server runs. +

+

+QR just receives the request, which contains form data, and +executes the template on the data in the form value named s. +

+

+The template package, inspired by json-template, is +powerful; +this program just touches on its capabilities. +In essence, it rewrites a piece of text on the fly by substituting elements derived +from data items passed to templ.Execute, in this case the +form value. +Within the template text (templateStr), +brace-delimited pieces denote template actions. +The piece from the {.section @} +to {.end} executes with the value of the data item @, +which is a shorthand for “the current item”, which is the form value. +(When the string is empty, this piece of the template is suppressed.) +

+

+The snippet {@|url+html} says to run the data through the formatter +installed in the formatter map (fmap) +under the name "url+html". +That is the function UrlHtmlFormatter, which sanitizes the string +for safe display on the web page. +

+

+The rest of the template string is just the HTML to show when the page loads. +If this is too quick an explanation, see the documentation +for the template package for a more thorough discussion. +

+

+And there you have it: a useful webserver in a few lines of code plus some +data-driven HTML text. +Go is powerful enough to make a lot happen in a few lines. +

+ + + diff --git a/doc/frontpage.css b/doc/frontpage.css new file mode 100644 index 000000000..299855ce9 --- /dev/null +++ b/doc/frontpage.css @@ -0,0 +1,143 @@ +/* Overloads to all.css */ +#container { width: 76em } +.left-column { width: 48%; } +.right-column { width: 48%; } + +/* Frontpage styles */ +#content-introductory code { + font-family: "Bitstream Vera Sans Mono", "Andale Mono", monospace; +} +#content-introductory input, select, textarea { + font-family: "Bitstream Vera Sans", Verdana, sans-serif; + font-size: 1em; +} +span.keyword { + font-family: Cambria, Georgia, Times, "Times New Roman", serif; + font-size: 1.15em; + font-style: italic; +} +#content h3, #content h2 { + margin: 0; + font-size: 1em; + background: none; + border: none; + padding: 0; +} +#content .more { + color: #999; + font-weight: normal; +} +#frontpage h2#branding-tagline { + font-weight: normal; + font-style: italic; +} +#resources { + position: relative; + margin-top: 1em; +} +#resources h3 { + margin-top: 0; + margin-bottom: -.5em; + font-size: 1em; + font-weight: normal; +} +#resources-users { + float: left; + width: 48%; +} +#resources-contributors { + float: right; + width: 50%; +} +#resources ul { + padding-left: 2em; +} +#resources li { + margin-bottom: 0.5em; +} +#content-rotating { + height: 200px; +} +#content-videos { + float: left; + width: 170px; +} +#content-videos .thumbnail { + width: 150px; + height: 103px; + background-repeat: no-repeat; + border: none; +} +#content-videos .thumbnail._001 { + background: url(/doc/video-001.png); +} +#content-videos .thumbnail._002 { + background: url(/doc/video-002.png); +} +#content-videos .thumbnail._003 { + background: url(/doc/video-003.png); +} +#content-videos .thumbnail._004 { + background: url(/doc/video-004.png); +} +#content-videos .thumbnail._005 { + background: url(/doc/video-005.jpg); +} +#content-videos a.video { + display: inline-block; + width: 150px; + margin-right: .30em; + margin-top: 1.2em; +} +#content-videos a.video .caption { + display: block; + text-align: center; +} +#content-videos a.video .caption.title { + margin-top: .31em; + font-weight: bold; +} +#content-blog ul { + margin-top: 1em; + margin-left: 0; + padding-left: 0; +} +#content-blog li { + list-style: none; + margin-bottom: 1em; +} +#content-blog li a { + color: #999; + text-decoration: none; +} +#content-blog .date { + color: #999; + font-size: 0.8em; + display: inline-block; + margin-left: 0.5em; +} +#content-blog li a:link .title { + color: #04a; +} +#content-blog li a:visited .title { + color: #04a; +} +#content-blog li a:hover .title { + color: #a40; + text-decoration: underline; +} +#content-blog li a:active .title { + color: #c00; +} +.navtop { + display: none !important; +} +.how { + float: right; + font-size: 75%; +} +.unsupported { + font-weight: bold; + color: red; +} + diff --git a/doc/gccgo_contribute.html b/doc/gccgo_contribute.html new file mode 100644 index 000000000..8eeb3a5c5 --- /dev/null +++ b/doc/gccgo_contribute.html @@ -0,0 +1,99 @@ + + +

Introduction

+ +

+These are some notes on contributing to the gccgo +frontend for GCC. For information on contributing to parts of Go other +than gccgo, see Contributing to +the Go project. For information on building gccgo +for yourself, see Setting up and using +gccgo. +

+ +

Legal Prerequisites

+ +

+You must follow the Go copyright +rules. +

+ +

Code

+ +

+The source code for the gccgo frontend may be found at +http://code.google.com/p/gofrontend. +Changes made to that project are routinely merged into the source code +hosted at gcc.gnu.org. The gofrontend +project includes only the Go frontend proper. These are the files +which in the gcc sources may be found in the +directories gcc/go and libgo. +The gcc sources also include a copy of +the test directory +from the main Go repository. + +

+The frontend is written in C++ and as such the GNU coding standards do +not entirely apply; in writing code for the frontend, follow the +formatting of the surrounding code. Although the frontend is +currently closely tied to the rest of the gcc codebase, +we plan to make it more independent. Any new code that uses other +parts of gcc should be placed in an appropriate file, +such as gogo-tree.cc. Eventually +all gcc-specific code should migrate to +a gcc-interface subdirectory. +

+ +

+The run-time library for gccgo is mostly the same as the +library in the main Go +repository. The library code in the Go repository is periodically +copied into the gofrontend and the gcc +repositories. Accordingly, most library changes should be made in the +main Go repository. Changes to the few gccgo-specific +parts of the library should follow the process described here. +The gccgo-specific parts of the library are everything in +the libgo directory except for the libgo/go +subdirectory. +

+ +

Testing

+ +

+All patches must be tested. There are two test suites. A patch that +introduces new failures is not acceptable. +

+ +

+To run the compiler test suite, run make check-go in the +gcc subdirectory of your build directory. This will run +various tests underneath gcc/testsuite/go.*. This +includes a copy of the tests in the main Go repository, which are run +using the DejaGNU script found in +in gcc/testsuite/go.test/go-test.exp. Many of the +compiler tests may be run without the Go library, but some do require +the library to built first. +

+ +

+To run the library test suite, run make +check-target-libgo in the top level of your build directory. +

+ +

+Most new tests should be submitted to the main Go repository for +copying into the gccgo repository. If there is a need +for specific tests for gccgo, they should go in +the gcc/testsuite/go.go-torture +or gcc/testsuite/go.dg directories in +the gcc.gnu.org repository. +

+ +

Submitting Changes

+ +

+Changes to the Go frontend should follow the same process as for the +main Go repository, only for the gofrontend project +rather than the go project. Those changes will then be +merged into the gcc sources. +

diff --git a/doc/gccgo_install.html b/doc/gccgo_install.html new file mode 100644 index 000000000..159fab7bb --- /dev/null +++ b/doc/gccgo_install.html @@ -0,0 +1,409 @@ + + +

+This document explains how to use gccgo, a compiler for +the Go language. The gccgo compiler is a new frontend +for gcc, the widely used GNU compiler. Although the +frontend itself is under a BSD-style license, gccgo is +normally used as part of gcc and is then covered by +the GNU General Public +License. +

+ +

+Note that gccgo is not the 6g compiler; see +the Installing Go instructions for that +compiler. +

+ +

Source code

+ +

+The gccgo source code is accessible via Subversion. The +gcc web site +has instructions for getting the +gcc source code. The gccgo source code +is a branch of the main gcc code +repository: svn://gcc.gnu.org/svn/gcc/branches/gccgo. +

+ +

+Note that although gcc.gnu.org is the most convenient way +to get the source code for the compiler, that is not where the master +sources live. If you want to contribute changes to the gccgo +compiler, see Contributing to +gccgo. +

+ + +

Building

+ +

+Building gccgo is just like building gcc +with one or two additional options. See +the instructions on the gcc web +site. When you run configure, add the +option --enable-languages=c,c++,go (along with other +languages you may want to build). If you are targeting a 32-bit x86, +then you will want to build gccgo to default to +supporting locked compare and exchange instructions; do this by also +using the configure option --with-arch=i586 +(or a newer architecture, depending on where you need your programs to +run). +

+ +

+On x86 GNU/Linux systems the gccgo compiler is able to +use a small discontiguous stack for goroutines. This permits programs +to run many more goroutines, since each goroutine can use a relatively +small stack. Doing this requires using a development version of +the gold linker. The easiest way to do this is to build +the GNU binutils, using --enable-gold when you run +the configure script, and to +use --with-ld=GOLD_BINARY when you +configure gccgo. A typical sequence would look like +this (you can replace /opt/gold with any directory to +which you have write access): +

+ +
+cvs -z 9 -d :pserver:anoncvs@sourceware.org:/cvs/src login
+[password is "anoncvs"]
+cvs -z 9 -d :pserver:anoncvs@sourceware.org:/cvs/src co binutils
+mkdir binutils-objdir
+cd binutils-objdir
+../src/configure --enable-gold --prefix=/opt/gold
+make
+make install
+
+ +

+A number of prerequisites are required to build gcc, as +described on the gcc web site. If +those are all available, then a typical build and install sequence +would look like this (only use the --with-ld option if +you built and installed the gold linker as described above): +

+ +
+svn checkout svn://gcc.gnu.org/svn/gcc/branches/gccgo gccgo
+mkdir objdir
+cd objdir
+../gccgo/configure --enable-languages=c,c++,go --with-ld=/opt/gold/bin/ld
+make
+make install
+
+ +

Using gccgo

+ +

+The gccgo compiler works like other gcc frontends. + +

+To compile a file: + +

+gccgo -c file.go
+
+ +

+That produces file.o. To link files together to form an +executable: + +

+gccgo -o file file.o
+
+ +

+To run the resulting file, you will need to tell the program where to +find the compiled Go packages. This can be done either by setting +LD_LIBRARY_PATH in your environment: + +

+LD_LIBRARY_PATH=/usr/lib/gcc/MACHINE/VERSION
+
+ +

+or by passing a -Wl,-R option when you link: + +

+gccgo -o file file.o -Wl,-R,/usr/lib/gcc/MACHINE/VERSION
+
+ +

+or you can use the -static-libgo link-time option to link +statically against libgo, or you can do a fully static link (static +linking is the default for the 6l Go linker). On most +systems, a static link will look something like: + +

+gccgo -o file file.o -static -L /usr/lib/nptl -lgobegin -lgo -lpthread
+
+ +

+You may get a warning about not creating an .eh_frame_hdr +section; this has nothing to do with Go, and may be ignored. In the +future the requirement of explicitly specifying +-L /usr/lib/nptl -lgobegin -lgo -lpthread +may be removed. + + +

Options

+ +

+The gccgo compiler supports all gcc options +that are language independent, notably the -O +and -g options. + +

+The -fgo-prefix=PREFIX option may be used to set a unique +prefix for the package being compiled. This option is intended for +use with large programs that contain many packages, in order to allow +multiple packages to use the same identifier as the package name. +The PREFIX may be any string; a good choice for the +string is the directory where the package will be installed. + +

+The -fno-require-return-statement option may be used to +disable the compiler error about functions missing return statements. +Note that there is no way to disable this error in 6g. + +

+The -I and -L options, which are synonyms +for the compiler, may be used to set the search path for finding +imports. + + +

Imports

+ +

+When you compile a file which exports something, the export +information will be stored directly in the object file. When +you import a package, you must tell gccgo how to +find the file. + +

+When you import the package FILE with gccgo, +it will look for the import data in the following files, and use the +first one that it finds. + +

    +
  • FILE.gox +
  • FILE.o +
  • libFILE.so +
  • libFILE.a +
+ +

+FILE.gox, when used, will typically contain +nothing but export data. This can be generated from +FILE.o via + +

+objcopy -j .go_export FILE.o FILE.gox
+
+ +

+The gccgo compiler will look in the current +directory for import files. In more complex scenarios you +may pass the -I or -L option to +gccgo. Both options take directories to search. The +-L option is also passed to the linker. + +The gccgo compiler does not currently (2009-11-06) record +the file name of imported packages in the object file. You must +arrange for the imported data to be linked into the program. + +

+gccgo -c mypackage.go              # Exports mypackage
+gccgo -c main.go                   # Imports mypackage
+gccgo -o main main.o mypackage.o   # Explicitly links with mypackage.o
+
+ +

Unimplemented

+ +

+Some Go features are not yet implemented in gccgo. As of +2010-08-23, the following are not implemented: + +

    +
  • goroutines are implemented as NPTL threads. If you can not use + the gold linker as described above, they are created with a fixed + stack size, and the number of goroutines that may be created at + one time is limited. +
+ +

Debugging

+ +

+If you use the -g option when you compile, you can run +gdb on your executable. The debugger doesn't (yet) +know anything about Go. However, you can set breakpoints, single-step, +etc. You can print variables, but they will be printed as though they +had C/C++ types. For numeric types this doesn't matter. Go strings +will show up as pointers to structures; to see the value +print *stringvar. In general Go strings, maps, channels +and interfaces are always represented as C pointers. + +

C Interoperability

+ +

+When using gccgo there is limited interoperability with C, +or with C++ code compiled using extern "C". + +

Types

+ +

+Basic types map directly: an int in Go is an int +in C, etc. Go byte is equivalent to C unsigned char. +Pointers in Go are pointers in C. A Go struct is the same as C +struct with the same fields and types. + +

+The Go string type is currently defined as a two-element +structure (this is subject to change): + +

+struct __go_string {
+  const unsigned char *__data;
+  int __length;
+};
+
+ +

+You can't pass arrays between C and Go. However, a pointer to an +array in Go is equivalent to a C pointer to the +equivalent of the element type. +For example, Go *[10]int is equivalent to C int*, +assuming that the C pointer does point to 10 elements. + +

+A slice in Go is a structure. The current definition is +(this is subject to change): + +

+struct __go_slice {
+  void *__values;
+  int __count;
+  int __capacity;
+};
+
+ +

+The type of a Go function with no receiver is equivalent to a C function +whose parameter types are equivalent. When a Go function returns more +than one value, the C function returns a struct. For example, these +functions have equivalent types: + +

+func GoFunction(int) (int, float64)
+struct { int i; float64 f; } CFunction(int)
+
+ +

+A pointer to a Go function is equivalent to a pointer to a C function +when the functions have equivalent types. + +

+Go interface, channel, and map +types have no corresponding C type (interface is a +two-element struct and channel and map are +pointers to structs in C, but the structs are deliberately undocumented). C +enum types correspond to some integer type, but precisely +which one is difficult to predict in general; use a cast. C union +types have no corresponding Go type. C struct types containing +bitfields have no corresponding Go type. C++ class types have +no corresponding Go type. + +

+Memory allocation is completely different between C and Go, as Go uses +garbage collection. The exact guidelines in this area are undetermined, +but it is likely that it will be permitted to pass a pointer to allocated +memory from C to Go. The responsibility of eventually freeing the pointer +will remain with C side, and of course if the C side frees the pointer +while the Go side still has a copy the program will fail. When passing a +pointer from Go to C, the Go function must retain a visible copy of it in +some Go variable. Otherwise the Go garbage collector may delete the +pointer while the C function is still using it. + +

Function names

+ +

+Go code can call C functions directly using a Go extension implemented +in gccgo: a function declaration may be followed by +__asm__("NAME"). For example, here is how the C function +open can be declared in Go: + +

+func c_open(name *byte, mode int, perm int) int __asm__ ("open");
+
+ +

+The C function naturally expects a nul terminated string, which in +Go is equivalent to a pointer to an array (not a slice!) of +byte with a terminating zero byte. So a sample call +from Go would look like (after importing the os package): + +

+var name = [4]byte{'f', 'o', 'o', 0};
+i := c_open(&name[0], os.O_RDONLY, 0);
+
+ +

+(this serves as an example only, to open a file in Go please use Go's +os.Open function instead). + +

+The name of Go functions accessed from C is subject to change. At present +the name of a Go function that does not have a receiver is +prefix.package.Functionname. The prefix is set by +the -fgo-prefix option used when the package is compiled; +if the option is not used, the default is simply go. +To call the function from C you must set the name using +a gcc extension similar to the gccgo +extension. + +

+extern int go_function(int) __asm__ ("myprefix.mypackage.Function");
+
+ +

+Automatic generation of Go declarations from C source code

+ +

+The Go version of gcc supports automatically generating +Go declarations from C code. The facility is rather awkward at present, +and a better mechanism is under development. + +

+Compile your C code as usual, but replace -c with +-S -ggo. The result will be an assembler file +with a .s extension. This assembler file will contain +comments beginning with #GO. Those comments are declarations in the Go +language for the C types, variables and functions declared in the C code. +C types which can not be represented in Go will contain the string INVALID. +Unsupported macro definitions will be recorded as unknowndefine, +and uses of #undef will be recorded as undef. +So it is very approximately possible to get Go code by running + +

+gcc -S -ggo foo.c
+grep '#GO' foo.s | grep -v INVALID | grep -v unknowndefine | grep -v undef > foo.go
+
+ +

+This procedure is full of unstated caveats and restrictions and we make no +guarantee that it will not change in the future. It is more useful as a +starting point for real Go code than as a regular procedure. + +

RTEMS Port

+

+The gccgo compiler has been ported to +RTEMS. RTEMS is a real-time executive +that provides a high performance environment for embedded applications +on a range of processors and embedded hardware. The current gccgo +port is for x86. The goal is to extend the port to most of the + +architectures supported by RTEMS. For more information on the port, +as well as instructions on how to install it, please see this +RTEMS Wiki page. diff --git a/doc/go-logo-black.png b/doc/go-logo-black.png new file mode 100644 index 000000000..29be31943 Binary files /dev/null and b/doc/go-logo-black.png differ diff --git a/doc/go-logo-blue.png b/doc/go-logo-blue.png new file mode 100644 index 000000000..b9eac2727 Binary files /dev/null and b/doc/go-logo-blue.png differ diff --git a/doc/go-logo-white.png b/doc/go-logo-white.png new file mode 100644 index 000000000..4011069eb Binary files /dev/null and b/doc/go-logo-white.png differ diff --git a/doc/go_faq.html b/doc/go_faq.html new file mode 100644 index 000000000..aeed53795 --- /dev/null +++ b/doc/go_faq.html @@ -0,0 +1,1496 @@ + + +

Origins

+ +

+What is the purpose of the project?

+ +

+No major systems language has emerged in over a decade, but over that time +the computing landscape has changed tremendously. There are several trends: +

+ +
    +
  • +Computers are enormously quicker but software development is not faster. +
  • +Dependency management is a big part of software development today but the +“header files” of languages in the C tradition are antithetical to clean +dependency analysis—and fast compilation. +
  • +There is a growing rebellion against cumbersome type systems like those of +Java and C++, pushing people towards dynamically typed languages such as +Python and JavaScript. +
  • +Some fundamental concepts such as garbage collection and parallel computation +are not well supported by popular systems languages. +
  • +The emergence of multicore computers has generated worry and confusion. +
+ +

+We believe it's worth trying again with a new language, a concurrent, +garbage-collected language with fast compilation. Regarding the points above: +

+ +
    +
  • +It is possible to compile a large Go program in a few seconds on a single computer. +
  • +Go provides a model for software construction that makes dependency +analysis easy and avoids much of the overhead of C-style include files and +libraries. +
  • +Go's type system has no hierarchy, so no time is spent defining the +relationships between types. Also, although Go has static types the language +attempts to make types feel lighter weight than in typical OO languages. +
  • +Go is fully garbage-collected and provides fundamental support for +concurrent execution and communication. +
  • +By its design, Go proposes an approach for the construction of system +software on multicore machines. +
+ +

+What is the origin of the name?

+ +

+“Ogle” would be a good name for a Go debugger. +

+ +

+What's the origin of the mascot?

+ +

+The mascot and logo were designed by +Renée French, who also designed +Glenda, +the Plan 9 bunny. +The gopher is derived from one she used for an WFMU +T-shirt design some years ago. +The logo and mascot are covered by the +Creative Commons Attribution 3.0 +license. +

+ +

+What kind of a name is 6g?

+ +

+The 6g (and 8g and 5g) compiler is named in the +tradition of the Plan 9 C compilers, described in + +http://plan9.bell-labs.com/sys/doc/compiler.html +(see the table in section 2). + +6 is the architecture letter for amd64 (or x86-64, if you prefer), while +g stands for Go. +

+ +

+What is the history of the project?

+

+Robert Griesemer, Rob Pike and Ken Thompson started sketching the +goals for a new language on the white board on September 21, 2007. +Within a few days the goals had settled into a plan to do something +and a fair idea of what it would be. Design continued part-time in +parallel with unrelated work. By January 2008, Ken had started work +on a compiler with which to explore ideas; it generated C code as its +output. By mid-year the language had become a full-time project and +had settled enough to attempt a production compiler. In May 2008, +Ian Taylor independently started on a GCC front end for Go using the +draft specification. Russ Cox joined in late 2008 and helped move the language +and libraries from prototype to reality. +

+ +

+Go became a public open source project on November 10, 2009. +Many people from the community have contributed ideas, discussions, and code. +

+ +

+Why are you creating a new language?

+

+Go was born out of frustration with existing languages and +environments for systems programming. Programming had become too +difficult and the choice of languages was partly to blame. One had to +choose either efficient compilation, efficient execution, or ease of +programming; all three were not available in the same mainstream +language. Programmers who could were choosing ease over +safety and efficiency by moving to dynamically typed languages such as +Python and JavaScript rather than C++ or, to a lesser extent, Java. +

+ +

+Go is an attempt to combine the ease of programming of an interpreted, +dynamically typed +language with the efficiency and safety of a statically typed, compiled language. +It also aims to be modern, with support for networked and multicore +computing. Finally, it is intended to be fast: it should take +at most a few seconds to build a large executable on a single computer. +To meet these goals required addressing a number of +linguistic issues: an expressive but lightweight type system; +concurrency and garbage collection; rigid dependency specification; +and so on. These cannot be addressed well by libraries or tools; a new +language was called for. +

+ +

+What are Go's ancestors?

+

+Go is mostly in the C family (basic syntax), +with significant input from the Pascal/Modula/Oberon +family (declarations, packages), +plus some ideas from languages +inspired by Tony Hoare's CSP, +such as Newsqueak and Limbo (concurrency). +However, it is a new language across the board. +In every respect the language was designed by thinking +about what programmers do and how to make programming, at least the +kind of programming we do, more effective, which means more fun. +

+ +

+What are the guiding principles in the design?

+

+Programming today involves too much bookkeeping, repetition, and +clerical work. As Dick Gabriel says, “Old programs read +like quiet conversations between a well-spoken research worker and a +well-studied mechanical colleague, not as a debate with a compiler. +Who'd have guessed sophistication bought such noise?” +The sophistication is worthwhile—no one wants to go back to +the old languages—but can it be more quietly achieved? +

+

+Go attempts to reduce the amount of typing in both senses of the word. +Throughout its design, we have tried to reduce clutter and +complexity. There are no forward declarations and no header files; +everything is declared exactly once. Initialization is expressive, +automatic, and easy to use. Syntax is clean and light on keywords. +Stuttering (foo.Foo* myFoo = new(foo.Foo)) is reduced by +simple type derivation using the := +declare-and-initialize construct. And perhaps most radically, there +is no type hierarchy: types just are, they don't have to +announce their relationships. These simplifications allow Go to be +expressive yet comprehensible without sacrificing, well, sophistication. +

+

+Another important principle is to keep the concepts orthogonal. +Methods can be implemented for any type; structures represent data while +interfaces represent abstraction; and so on. Orthogonality makes it +easier to understand what happens when things combine. +

+ +

Usage

+ +

Is Google using Go internally?

+ +

+Yes. There are now several Go programs deployed in +production inside Google. For instance, the server behind +http://golang.org is a Go program; +in fact it's just the godoc +document server running in a production configuration. +

+ + + +

+There are two Go compiler implementations, 6g and friends, +generically called gc, and gccgo. +Gc uses a different calling convention and linker and can +therefore only be linked with C programs using the same convention. +There is such a C compiler but no C++ compiler. +Gccgo is a GCC front-end that can, with care, be linked with +GCC-compiled C or C++ programs. +

+ +

+The cgo program provides the mechanism for a +“foreign function interface” to allow safe calling of +C libraries from Go code. SWIG extends this capability to C++ libraries. +

+ + +

+Does Go support Google's protocol buffers?

+ +

+A separate open source project provides the necessary compiler plugin and library. +It is available at +http://code.google.com/p/goprotobuf/ +

+ + +

+Can I translate the Go home page into another language?

+ +

+Absolutely. We encourage developers to make Go Language sites in their own languages. +However, if you choose to add the Google logo or branding to your site +(it does not appear on golang.org), +you will need to abide by the guidelines at +http://www.google.com/permissions/guidelines.html +

+ +

Design

+ +

+What's up with Unicode identifiers?

+ +

+It was important to us to extend the space of identifiers from the +confines of ASCII. Go's rule—identifier characters must be +letters or digits as defined by Unicode—is simple to understand +and to implement but has restrictions. Combining characters are +excluded by design, for instance. +Until there +is an agreed external definition of what an identifier might be, +plus a definition of canonicalization of identifiers that guarantees +no ambiguity, it seemed better to keep combining characters out of +the mix. Thus we have a simple rule that can be expanded later +without breaking programs, one that avoids bugs that would surely arise +from a rule that admits ambiguous identifiers. +

+ +

+On a related note, since an exported identifier must begin with an +upper-case letter, identifiers created from “letters” +in some languages can, by definition, not be exported. For now the +only solution is to use something like X日本語, which +is clearly unsatisfactory; we are considering other options. The +case-for-visibility rule is unlikely to change however; it's one +of our favorite features of Go. +

+ +

Why does Go not have feature X?

+ +

+Every language contains novel features and omits someone's favorite +feature. Go was designed with an eye on felicity of programming, speed of +compilation, orthogonality of concepts, and the need to support features +such as concurrency and garbage collection. Your favorite feature may be +missing because it doesn't fit, because it affects compilation speed or +clarity of design, or because it would make the fundamental system model +too difficult. +

+ +

+If it bothers you that Go is missing feature X, +please forgive us and investigate the features that Go does have. You might find that +they compensate in interesting ways for the lack of X. +

+ +

+Why does Go not have generic types?

+

+Generics may well be added at some point. We don't feel an urgency for +them, although we understand some programmers do. +

+ +

+Generics are convenient but they come at a cost in +complexity in the type system and run-time. We haven't yet found a +design that gives value proportionate to the complexity, although we +continue to think about it. Meanwhile, Go's built-in maps and slices, +plus the ability to use the empty interface to construct containers +(with explicit unboxing) mean in many cases it is possible to write +code that does what generics would enable, if less smoothly. +

+ +

+This remains an open issue. +

+ +

+Why does Go not have exceptions?

+

+We believe that coupling exceptions to a control +structure, as in the try-catch-finally idiom, results in +convoluted code. It also tends to encourage programmers to label +too many ordinary errors, such as failing to open a file, as +exceptional. +

+ +

+Go takes a different approach. For plain error handling, Go's multi-value +returns make it easy to report an error without overloading the return value. +A +canonical error type, coupled +with Go's other features, makes error +handling pleasant but quite different from that in other languages. +

+ +

+Go also has a couple +of built-in functions to signal and recover from truly exceptional +conditions. The recovery mechanism is executed only as part of a +function's state being torn down after an error, which is sufficient +to handle catastrophe but requires no extra control structures and, +when used well, can result in clean error-handling code. +

+ +

+See the Defer, Panic, and Recover article for details. +

+ +

+Why does Go not have assertions?

+ +

+Go doesn't provide assertions. They are undeniably convenient, but our +experience has been that programmers use them as a crutch to avoid thinking +about proper error handling and reporting. Proper error handling means that +servers continue operation after non-fatal errors instead of crashing. +Proper error reporting means that errors are direct and to the point, +saving the programmer from interpreting a large crash trace. Precise +errors are particularly important when the programmer seeing the errors is +not familiar with the code. +

+ +

+The same arguments apply to the use of assert() in test programs. Proper +error handling means letting other tests run after one has failed, so +that the person debugging the failure gets a complete picture of what is +wrong. It is more useful for a test to report that +isPrime gives the wrong answer for 2, 3, 5, and 7 (or for +2, 4, 8, and 16) than to report that isPrime gives the wrong +answer for 2 and therefore no more tests were run. The programmer who +triggers the test failure may not be familiar with the code that fails. +Time invested writing a good error message now pays off later when the +test breaks. +

+ +

+In testing, if the amount of extra code required to write +good errors seems repetitive and overwhelming, it might work better as a +table-driven test instead. +Go has excellent support for data structure literals. +

+ +

+We understand that this is a point of contention. There are many things in +the Go language and libraries that differ from modern practices, simply +because we feel it's sometimes worth trying a different approach. +

+ +

+Why build concurrency on the ideas of CSP?

+

+Concurrency and multi-threaded programming have a reputation +for difficulty. We believe the problem is due partly to complex +designs such as pthreads and partly to overemphasis on low-level details +such as mutexes, condition variables, and memory barriers. +Higher-level interfaces enable much simpler code, even if there are still +mutexes and such under the covers. +

+ +

+One of the most successful models for providing high-level linguistic support +for concurrency comes from Hoare's Communicating Sequential Processes, or CSP. +Occam and Erlang are two well known languages that stem from CSP. +Go's concurrency primitives derive from a different part of the family tree +whose main contribution is the powerful notion of channels as first class objects. +

+ +

+Why goroutines instead of threads?

+

+Goroutines are part of making concurrency easy to use. The idea, which has +been around for a while, is to multiplex independently executing +functions—coroutines—onto a set of threads. +When a coroutine blocks, such as by calling a blocking system call, +the run-time automatically moves other coroutines on the same operating +system thread to a different, runnable thread so they won't be blocked. +The programmer sees none of this, which is the point. +The result, which we call goroutines, can be very cheap: unless they spend a lot of time +in long-running system calls, they cost little more than the memory +for the stack, which is just a few kilobytes. +

+ +

+To make the stacks small, Go's run-time uses segmented stacks. A newly +minted goroutine is given a few kilobytes, which is almost always enough. +When it isn't, the run-time allocates (and frees) extension segments automatically. +The overhead averages about three cheap instructions per function call. +It is practical to create hundreds of thousands of goroutines in the same +address space. If goroutines were just threads, system resources would +run out at a much smaller number. +

+ +

+Why are map operations not defined to be atomic?

+ +

+After long discussion it was decided that the typical use of maps did not require +safe access from multiple threads, and in those cases where it did, the map was +probably part of some larger data structure or computation that was already +synchronized. Therefore requiring that all map operations grab a mutex would slow +down most programs and add safety to few. This was not an easy decision, +however, since it means uncontrolled map access can crash the program. +

+ +

+The language does not preclude atomic map updates. When required, such +as when hosting an untrusted program, the implementation could interlock +map access. +

+ +

Types

+ +

+Is Go an object-oriented language?

+ +

+Yes and no. Although Go has types and methods and allows an +object-oriented style of programming, there is no type hierarchy. +The concept of “interface” in Go provides a different approach that +we believe is easy to use and in some ways more general. There are +also ways to embed types in other types to provide something +analogous—but not identical—to subclassing. +Moreover, methods in Go are more general than in C++ or Java: +they can be defined for any sort of data, not just structs. +

+ +

+Also, the lack of type hierarchy makes “objects” in Go feel much more +lightweight than in languages such as C++ or Java. +

+ +

+How do I get dynamic dispatch of methods?

+ +

+The only way to have dynamically dispatched methods is through an +interface. Methods on structs or other types are always resolved statically. +

+ +

+Why is there no type inheritance?

+

+Object-oriented programming, at least in the best-known languages, +involves too much discussion of the relationships between types, +relationships that often could be derived automatically. Go takes a +different approach. +

+ +

+Rather than requiring the programmer to declare ahead of time that two +types are related, in Go a type automatically satisfies any interface +that specifies a subset of its methods. Besides reducing the +bookkeeping, this approach has real advantages. Types can satisfy +many interfaces at once, without the complexities of traditional +multiple inheritance. +Interfaces can be very lightweight—an interface with +one or even zero methods can express a useful concept. +Interfaces can be added after the fact if a new idea comes along +or for testing—without annotating the original types. +Because there are no explicit relationships between types +and interfaces, there is no type hierarchy to manage or discuss. +

+ +

+It's possible to use these ideas to construct something analogous to +type-safe Unix pipes. For instance, see how fmt.Fprintf +enables formatted printing to any output, not just a file, or how the +bufio package can be completely separate from file I/O, +or how the crypto packages stitch together block and +stream ciphers. All these ideas stem from a single interface +(io.Writer) representing a single method +(Write). And that's only scratching the surface. +

+ +

+It takes some getting used to but this implicit style of type +dependency is one of the most productive things about Go. +

+ +

+Why is len a function and not a method?

+

+We debated this issue but decided +implementing len and friends as functions was fine in practice and +didn't complicate questions about the interface (in the Go type sense) +of basic types. +

+ +

+Why does Go not support overloading of methods and operators?

+

+Method dispatch is simplified if it doesn't need to do type matching as well. +Experience with other languages told us that having a variety of +methods with the same name but different signatures was occasionally useful +but that it could also be confusing and fragile in practice. Matching only by name +and requiring consistency in the types was a major simplifying decision +in Go's type system. +

+ +

+Regarding operator overloading, it seems more a convenience than an absolute +requirement. Again, things are simpler without it. +

+ +

+Why doesn't Go have "implements" declarations?

+ +

+A Go type satisfies an interface by implementing the methods of that interface, +nothing more. This property allows interfaces to be defined and used without +having to modify existing code. It enables a kind of "duck typing" that +promotes separation of concerns and improves code re-use, and makes it easier +to build on patterns that emerge as the code develops. +The semantics of interfaces is one of the main reasons for Go's nimble, +lightweight feel. +

+ +

+See the question on type inheritance for more detail. +

+ +

+How can I guarantee my type satisfies an interface?

+ +

+You can ask the compiler to check that the type T implements the +interface I by attempting an assignment: +

+ +
+type T struct{}
+var _ I = T{}
+
+ +

+If T doesn't implement I, the mistake will be caught +at compile time. +

+ +

+If you wish the users of an interface to explicitly declare that they implement +it, you can add a method with a descriptive name to the interface's method set. +For example: +

+ +
+type Fooer interface {
+	Foo()
+	ImplementsFooer()
+}
+
+ +

+A type must then implement the ImplementsFooer method to be a +Fooer, clearly documenting the fact and announcing it in +godoc's output. +

+ +
+type Bar struct{}
+func (b Bar) ImplementsFooer() {}
+func (b Bar) Foo() {}
+
+ +

+Most code doesn't make use of such constraints, since they limit the utility of +the interface idea. Sometimes, though, they're necessary to resolve ambiguities +among similar interfaces. +

+ +

+Why doesn't type T satisfy the Equal interface?

+ +

+Consider this simple interface to represent an object that can compare +itself with another value: +

+ +
+type Equaler interface {
+	Equal(Equaler) bool
+}
+
+ +

+and this type, T: +

+ +
+type T int
+func (t T) Equal(u T) bool { return t == u } // does not satisfy Equaler
+
+ +

+Unlike the analogous situation in some polymorphic type systems, +T does not implement Equaler. +The argument type of T.Equal is T, +not literally the required type Equaler. +

+ +

+In Go, the type system does not promote the argument of +Equal; that is the programmer's responsibility, as +illustrated by the type T2, which does implement +Equaler: +

+ +
+type T2 int
+func (t T2) Equal(u Equaler) bool { return t == u.(T2) }  // satisfies Equaler
+
+ +

+Even this isn't like other type systems, though, because in Go any +type that satisfies Equaler could be passed as the +argument to T2.Equal, and at run time we must +check that the argument is of type T2. +Some languages arrange to make that guarantee at compile time. +

+ +

+A related example goes the other way: +

+ +
+type Opener interface {
+   Open(name) Reader
+}
+
+func (t T3) Open() *os.File
+
+ +

+In Go, T3 does not satisfy Opener, +although it might in another language. +

+ +

+While it is true that Go's type system does less for the programmer +in such cases, the lack of subtyping makes the rules about +interface satisfaction very easy to state: are the function's names +and signatures exactly those of the interface? +Go's rule is also easy to implement efficiently. +We feel these benefits offset the lack of +automatic type promotion. Should Go one day adopt some form of generic +typing, we expect there would be a way to express the idea of these +examples and also have them be statically checked. +

+ +

+Can I convert a []T to an []interface{}?

+ +

+Not directly because they do not have the same representation in memory. +It is necessary to copy the elements individually to the destination +slice. This example converts a slice of int to a slice of +interface{}: +

+ +
+t := []int{1, 2, 3, 4}
+s := make([]interface{}, len(t))
+for i, v := range t {
+	s[i] = v
+}
+
+ +

+Why are there no untagged unions, as in C?

+ +

+Untagged unions would violate Go's memory safety +guarantees. +

+ +

+Why does Go not have variant types?

+ +

+Variant types, also known as algebraic types, provide a way to specify +that a value might take one of a set of other types, but only those +types. A common example in systems programming would specify that an +error is, say, a network error, a security error or an application +error and allow the caller to discriminate the source of the problem +by examining the type of the error. Another example is a syntax tree +in which each node can be a different type: declaration, statement, +assignment and so on. +

+ +

+We considered adding variant types to Go, but after discussion +decided to leave them out because they overlap in confusing ways +with interfaces. What would happen if the elements of a variant type +were themselves interfaces? +

+ +

+Also, some of what variant types address is already covered by the +language. The error example is easy to express using an interface +value to hold the error and a type switch to discriminate cases. The +syntax tree example is also doable, although not as elegantly. +

+ +

Values

+ +

+Why does Go not provide implicit numeric conversions?

+

+The convenience of automatic conversion between numeric types in C is +outweighed by the confusion it causes. When is an expression unsigned? +How big is the value? Does it overflow? Is the result portable, independent +of the machine on which it executes? +It also complicates the compiler; “the usual arithmetic conversions” +are not easy to implement and inconsistent across architectures. +For reasons of portability, we decided to make things clear and straightforward +at the cost of some explicit conversions in the code. +The definition of constants in Go—arbitrary precision values free +of signedness and size annotations—ameliorates matters considerably, +though. +

+ +

+A related detail is that, unlike in C, int and int64 +are distinct types even if int is a 64-bit type. The int +type is generic; if you care about how many bits an integer holds, Go +encourages you to be explicit. +

+ +

+Why are maps built in?

+

+The same reason strings are: they are such a powerful and important data +structure that providing one excellent implementation with syntactic support +makes programming more pleasant. We believe that Go's implementation of maps +is strong enough that it will serve for the vast majority of uses. +If a specific application can benefit from a custom implementation, it's possible +to write one but it will not be as convenient syntactically; this seems a reasonable tradeoff. +

+ +

+Why don't maps allow structs and arrays as keys?

+

+Map lookup requires an equality operator, which structs and arrays do not implement. +They don't implement equality because equality is not well defined on such types; +there are multiple considerations involving shallow vs. deep comparison, pointer vs. +value comparison, how to deal with recursive structures, and so on. +We may revisit this issue—and implementing equality for structs and arrays +will not invalidate any existing programs—but without a clear idea of what +equality of structs and arrays should mean, it was simpler to leave it out for now. +

+ +

+Why are maps, slices, and channels references while arrays are values?

+

+There's a lot of history on that topic. Early on, maps and channels +were syntactically pointers and it was impossible to declare or use a +non-pointer instance. Also, we struggled with how arrays should work. +Eventually we decided that the strict separation of pointers and +values made the language harder to use. Introducing reference types, +including slices to handle the reference form of arrays, resolved +these issues. Reference types add some regrettable complexity to the +language but they have a large effect on usability: Go became a more +productive, comfortable language when they were introduced. +

+ +

Writing Code

+ +

+How are libraries documented?

+ +

+There is a program, godoc, written in Go, that extracts +package documentation from the source code. It can be used on the +command line or on the web. An instance is running at +http://golang.org/pkg/. +In fact, godoc implements the full site at +http://golang.org/. +

+ +

+Is there a Go programming style guide?

+ +

+Eventually, there may be a small number of rules to guide things +like naming, layout, and file organization. +The document Effective Go +contains some style advice. +More directly, the program gofmt is a pretty-printer +whose purpose is to enforce layout rules; it replaces the usual +compendium of do's and don'ts that allows interpretation. +All the Go code in the repository has been run through gofmt. +

+ +

+How do I submit patches to the Go libraries?

+ +

+The library sources are in go/src/pkg. +If you want to make a significant change, please discuss on the mailing list before embarking. +

+ +

+See the document +Contributing to the Go project +for more information about how to proceed. +

+ +

Pointers and Allocation

+ +

+When are function parameters passed by value?

+ +

+Everything in Go is passed by value. A function always gets a copy of the +thing being passed, as if there were an assignment statement assigning the +value to the parameter. For instance, copying a pointer value makes a copy of +the pointer, not the data it points to. +

+ +

+Map and slice values behave like pointers; they are descriptors that +contain pointers to the underlying map or slice data. Copying a map or +slice value doesn't copy the data it points to. Copying an interface value +makes a copy of the thing stored in the interface value. If the interface +value holds a struct, copying the interface value makes a copy of the +struct. If the interface value holds a pointer, copying the interface value +makes a copy of the pointer, but again not the data it points to. +

+ +

+Should I define methods on values or pointers?

+ +
+func (s *MyStruct) pointerMethod() { } // method on pointer
+func (s MyStruct)  valueMethod()   { } // method on value
+
+ +

+For programmers unaccustomed to pointers, the distinction between these +two examples can be confusing, but the situation is actually very simple. +When defining a method on a type, the receiver (s in the above +example) behaves exactly as if it were an argument to the method. +Whether to define the receiver as a value or as a pointer is the same +question, then, as whether a function argument should be a value or +a pointer. +There are several considerations. +

+ +

+First, and most important, does the method need to modify the +receiver? +If it does, the receiver must be a pointer. +(Slices and maps are reference types, so their story is a little +more subtle, but for instance to change the length of a slice +in a method the receiver must still be a pointer.) +In the examples above, if pointerMethod modifies +the fields of s, +the caller will see those changes, but valueMethod +is called with a copy of the caller's argument (that's the definition +of passing a value), so changes it makes will be invisible to the caller. +

+ +

+By the way, pointer receivers are identical to the situation in Java, +although in Java the pointers are hidden under the covers; it's Go's +value receivers that are unusual. +

+ +

+Second is the consideration of efficiency. If the receiver is large, +a big struct for instance, it will be much cheaper to +use a pointer receiver. +

+ +

+Next is consistency. If some of the methods of the type must have +pointer receivers, the rest should too, so the method set is +consistent regardless of how the type is used. +See the section on method sets +for details. +

+ +

+For types such as basic types, slices, and small structs, +a value receiver is very cheap so unless the semantics of the method +requires a pointer, a value receiver is efficient and clear. +

+ + +

+What's the difference between new and make?

+ +

+In short: new allocates memory, make initializes +the slice, map, and channel types. +

+ +

+See the relevant section +of Effective Go for more details. +

+ +

+Why is int 32 bits on 64 bit machines?

+ +

+The sizes of int and uint are implementation-specific +but the same as each other on a given platform. +The 64 bit Go compilers (both 6g and gccgo) use a 32 bit representation for +int. Code that relies on a particular +size of value should use an explicitly sized type, like int64. +On the other hand, floating-point scalars and complex +numbers are always sized: float32, complex64, +etc., because programmers should be aware of precision when using +floating-point numbers. +The default size of a floating-point constant is float64. +

+ +

+How do I know whether a variable is allocated on the heap or the stack?

+ +

+From a correctness standpoint, you don't need to know. +Each variable in Go exists as long as there are references to it. +The storage location chosen by the implementation is irrelevant to the +semantics of the language. +

+ +

+The storage location does have an effect on writing efficient programs. +When possible, the Go compilers will allocate variables that are +local to a function in that function's stack frame. However, if the +compiler cannot prove that the variable is not referenced after the +function returns, then the compiler must allocate the variable on the +garbage-collected heap to avoid dangling pointer errors. +

+ +

+In the current compilers, the analysis is crude: if a variable has its address +taken, that variable is allocated on the heap. We are working to improve this +analysis so that more data is kept on the stack. +

+ +

Concurrency

+ +

+What operations are atomic? What about mutexes?

+ +

+We haven't fully defined it all yet, but some details about atomicity are +available in the Go Memory Model specification. +

+ +

+Regarding mutexes, the sync +package implements them, but we hope Go programming style will +encourage people to try higher-level techniques. In particular, consider +structuring your program so that only one goroutine at a time is ever +responsible for a particular piece of data. +

+ +

+Do not communicate by sharing memory. Instead, share memory by communicating. +

+ +

+See the Share Memory By Communicating code walk and its associated article for a detailed discussion of this concept. +

+ +

+Why doesn't my multi-goroutine program use multiple CPUs?

+ +

+Under the gc compilers you must set GOMAXPROCS to allow the +run-time support to utilise more than one OS thread. Under gccgo an OS +thread will be created for each goroutine, and GOMAXPROCS is +effectively equal to the number of running goroutines. +

+ +

+Programs that perform concurrent computation should benefit from an increase in +GOMAXPROCS. (See the runtime package's +documentation.) +

+ +

+Why does using GOMAXPROCS > 1 sometimes make my program +slower?

+ +

+(This is specific to the gc compilers. See above.) +

+ +

+It depends on the nature of your program. +Programs that contain several goroutines that spend a lot of time +communicating on channels will experience performance degradation when using +multiple OS threads. This is because of the significant context-switching +penalty involved in sending data between threads. +

+ +

+Go's goroutine scheduler is not as good as it needs to be. In future, it +should recognize such cases and optimize its use of OS threads. For now, +GOMAXPROCS should be set on a per-application basis. +

+ +

Functions and Methods

+ +

+Why do T and *T have different method sets?

+ +

+From the Go Spec: +

+ +
+The method set of any other named type T consists of all methods +with receiver type T. The method set of the corresponding pointer +type *T is the set of all methods with receiver *T or +T (that is, it also contains the method set of T). +
+ +

+If an interface value contains a pointer *T, +a method call can obtain a value by dereferencing the pointer, +but if an interface value contains a value T, +there is no useful way for a method call to obtain a pointer. +

+ +

+If not for this restriction, this code: +

+ +
+var buf bytes.Buffer
+io.Copy(buf, os.Stdin)
+
+ +

+would copy standard input into a copy of buf, +not into buf itself. +This is almost never the desired behavior. +

+ +

+Why am I confused by the way my closures behave as goroutines?

+ +

+Some confusion may arise when using closures with concurrency. +Consider the following program: +

+ +
+func main() {
+	done := make(chan bool)
+
+	values := []string{ "a", "b", "c" }
+	for _, v := range values {
+		go func() {
+			fmt.Println(v)
+			done <- true
+		}()
+	}
+
+	// wait for all goroutines to complete before exiting
+	for _ = range values {
+		<-done 
+	}
+}
+
+ +

+One might mistakenly expect to see a, b, c as the output. +What you'll probably see instead is c, c, c. This is because +each closure shares the same variable v. Each closure prints the +value of v at the time fmt.Println is executed, +rather than the value of v when the goroutine was launched. +

+ +

+To bind the value of v to each closure as they are launched, one +could modify the inner loop to read: +

+ +
+	for _, v := range values {
+		go func(u string) {
+			fmt.Println(u)
+			done <- true
+		}(v)
+	}
+
+ +

+In this example, the value of v is passed as an argument to the +anonymous function. That value is then accessible inside the function as +the variable u. +

+ +

Control flow

+ +

+Does Go have the ?: operator?

+ +

+There is no ternary form in Go. You may use the following to achieve the same +result: +

+ +
+if expr {
+	n = trueVal
+} else {
+	n = falseVal
+}
+
+ +

Packages and Testing

+ +

+How do I create a multifile package?

+ +

+Put all the source files for the package in a directory by themselves. +Source files can refer to items from different files at will; there is +no need for forward declarations or a header file. +

+ +

+Other than being split into multiple files, the package will compile and test +just like a single-file package. +

+ +

+How do I write a unit test?

+ +

+Create a new file ending in _test.go in the same directory +as your package sources. Inside that file, import "testing" +and write functions of the form +

+ +
+func TestFoo(t *testing.T) {
+    ...
+}
+
+ +

+Run gotest in that directory. +That script finds the Test functions, +builds a test binary, and runs it. +

+ +

See the How to Write Go Code document for more details.

+ + +

Implementation

+ +

+What compiler technology is used to build the compilers?

+ +

+Gccgo has a C++ front-end with a recursive descent parser coupled to the +standard GCC back end. Gc is written in C using +yacc/bison for the parser. +Although it's a new program, it fits in the Plan 9 C compiler suite +(http://plan9.bell-labs.com/sys/doc/compiler.html) +and uses a variant of the Plan 9 loader to generate ELF binaries. +

+ +

+We considered writing 6g, the original Go compiler, in Go itself but +elected not to do so because of the difficulties of bootstrapping and +especially of open source distribution—you'd need a Go compiler to +set up a Go environment. Gccgo, which came later, makes it possible to +consider writing a compiler in Go, which might well happen. (Go would be a +fine language in which to implement a compiler; a native lexer and +parser are already available in /pkg/go.) +

+ +

+We also considered using LLVM for 6g but we felt it was too large and +slow to meet our performance goals. +

+ +

+How is the run-time support implemented?

+ +

+Again due to bootstrapping issues, the run-time code is mostly in C (with a +tiny bit of assembler) although Go is capable of implementing most of +it now. Gccgo's run-time support uses glibc. +Gc uses a custom library to keep the footprint under +control; it is +compiled with a version of the Plan 9 C compiler that supports +segmented stacks for goroutines. +Work is underway to provide the same stack management in +gccgo. +

+ +

+Why is my trivial program such a large binary?

+ +

+The gc tool chain (5l, 6l, and 8l) only +generate statically linked binaries. All Go binaries therefore include the Go +run-time, along with the run-time type information necessary to support dynamic +type checks, reflection, and even panic-time stack traces. +

+ +

+A trivial C "hello, world" program compiled and linked statically using gcc +on Linux is around 750 kB. An equivalent Go program is around 1.1 MB, but +that includes more powerful run-time support. We believe that with some effort +the size of Go binaries can be reduced. +

+ +

+Can I stop these complaints about my unused variable/import?

+ +

+The presence of an unused variable may indicate a bug, while +unused imports just slow down compilation. +Accumulate enough unused imports in your code tree and +things can get very slow. +For these reasons, Go allows neither. +

+ +

+When developing code, it's common to create these situations +temporarily and it can be annoying to have to edit them out before the +program will compile. +

+ +

+Some have asked for a compiler option to turn those checks off +or at least reduce them to warnings. +Such an option has not been added, though, +because compiler options should not affect the semantics of the +language and because the Go compiler does not report warnings, only +errors that prevent compilation. +

+ +

+There are two reasons for having no warnings. First, if it's worth +complaining about, it's worth fixing in the code. (And if it's not +worth fixing, it's not worth mentioning.) Second, having the compiler +generate warnings encourages the implementation to warn about weak +cases that can make compilation noisy, masking real errors that +should be fixed. +

+ +

+It's easy to address the situation, though. Use the blank identifier +to let unused things persist while you're developing. +

+ +
+import "unused"
+
+// This declaration marks the import as used by referencing an
+// item from the package.
+var _ = unused.Item  // TODO: Delete before committing!
+
+func main() {
+	debugData := debug.Profile()
+	_ = debugData // Used only during debugging.
+	....
+}
+
+ +

Performance

+ +

+Why does Go perform badly on benchmark X?

+ +

+One of Go's design goals is to approach the performance of C for comparable +programs, yet on some benchmarks it does quite poorly, including several +in test/bench. The slowest depend on libraries +for which versions of comparable performance are not available in Go. +For instance, pidigits depends on a multi-precision math package, and the C +versions, unlike Go's, use GMP (which is +written in optimized assembler). +Benchmarks that depend on regular expressions (regex-dna, for instance) are +essentially comparing Go's stopgap regexp package to +mature, highly optimized regular expression libraries like PCRE. +

+ +

+Benchmark games are won by extensive tuning and the Go versions of most +of the benchmarks need attention. If you measure comparable C +and Go programs (reverse-complement is one example), you'll see the two +languages are much closer in raw performance than this suite would +indicate. +

+ +

+Still, there is room for improvement. The compilers are good but could be +better, many libraries need major performance work, and the garbage collector +isn't fast enough yet (even if it were, taking care not to generate unnecessary +garbage can have a huge effect). +

+ +

+In any case, Go can often be very competitive. See the blog post about +profiling +Go programs for an informative example. + +

Changes from C

+ +

+Why is the syntax so different from C?

+

+Other than declaration syntax, the differences are not major and stem +from two desires. First, the syntax should feel light, without too +many mandatory keywords, repetition, or arcana. Second, the language +has been designed to be easy to analyze +and can be parsed without a symbol table. This makes it much easier +to build tools such as debuggers, dependency analyzers, automated +documentation extractors, IDE plug-ins, and so on. C and its +descendants are notoriously difficult in this regard. +

+ +

+Why are declarations backwards?

+

+They're only backwards if you're used to C. In C, the notion is that a +variable is declared like an expression denoting its type, which is a +nice idea, but the type and expression grammars don't mix very well and +the results can be confusing; consider function pointers. Go mostly +separates expression and type syntax and that simplifies things (using +prefix * for pointers is an exception that proves the rule). In C, +the declaration +

+
+	int* a, b;
+
+

+declares a to be a pointer but not b; in Go +

+
+	var a, b *int;
+
+

+declares both to be pointers. This is clearer and more regular. +Also, the := short declaration form argues that a full variable +declaration should present the same order as := so +

+
+	var a uint64 = 1;
+
+has the same effect as +
+	a := uint64(1);
+
+

+Parsing is also simplified by having a distinct grammar for types that +is not just the expression grammar; keywords such as func +and chan keep things clear. +

+ +

+See the article about +Go's Declaration Syntax +for more details. +

+ +

+Why is there no pointer arithmetic?

+

+Safety. Without pointer arithmetic it's possible to create a +language that can never derive an illegal address that succeeds +incorrectly. Compiler and hardware technology have advanced to the +point where a loop using array indices can be as efficient as a loop +using pointer arithmetic. Also, the lack of pointer arithmetic can +simplify the implementation of the garbage collector. +

+ +

+Why are ++ and -- statements and not expressions? And why postfix, not prefix?

+

+Without pointer arithmetic, the convenience value of pre- and postfix +increment operators drops. By removing them from the expression +hierarchy altogether, expression syntax is simplified and the messy +issues around order of evaluation of ++ and -- +(consider f(i++) and p[i] = q[++i]) +are eliminated as well. The simplification is +significant. As for postfix vs. prefix, either would work fine but +the postfix version is more traditional; insistence on prefix arose +with the STL, a library for a language whose name contains, ironically, a +postfix increment. +

+ +

+Why are there braces but no semicolons? And why can't I put the opening +brace on the next line?

+

+Go uses brace brackets for statement grouping, a syntax familiar to +programmers who have worked with any language in the C family. +Semicolons, however, are for parsers, not for people, and we wanted to +eliminate them as much as possible. To achieve this goal, Go borrows +a trick from BCPL: the semicolons that separate statements are in the +formal grammar but are injected automatically, without lookahead, by +the lexer at the end of any line that could be the end of a statement. +This works very well in practice but has the effect that it forces a +brace style. For instance, the opening brace of a function cannot +appear on a line by itself. +

+ +

+Some have argued that the lexer should do lookahead to permit the +brace to live on the next line. We disagree. Since Go code is meant +to be formatted automatically by +gofmt, +some style must be chosen. That style may differ from what +you've used in C or Java, but Go is a new language and +gofmt's style is as good as any other. More +important—much more important—the advantages of a single, +programmatically mandated format for all Go programs greatly outweigh +any perceived disadvantages of the particular style. +Note too that Go's style means that an interactive implementation of +Go can use the standard syntax one line at a time without special rules. +

+ +

+Why do garbage collection? Won't it be too expensive?

+

+One of the biggest sources of bookkeeping in systems programs is +memory management. We feel it's critical to eliminate that +programmer overhead, and advances in garbage collection +technology in the last few years give us confidence that we can +implement it with low enough overhead and no significant +latency. (The current implementation is a plain mark-and-sweep +collector but a replacement is in the works.) +

+ +

+Another point is that a large part of the difficulty of concurrent +and multi-threaded programming is memory management; +as objects get passed among threads it becomes cumbersome +to guarantee they become freed safely. +Automatic garbage collection makes concurrent code far easier to write. +Of course, implementing garbage collection in a concurrent environment is +itself a challenge, but meeting it once rather than in every +program helps everyone. +

+ +

+Finally, concurrency aside, garbage collection makes interfaces +simpler because they don't need to specify how memory is managed across them. +

+ +

+On the topic of performance, keep in mind that Go gives the programmer +considerable control over memory layout and allocation, much more than +is typical in garbage-collected languages. A careful programmer can reduce +the garbage collection overhead dramatically by using the language well; +see the article about +profiling +Go programs for a worked example, including a demonstration of Go's +profiling tools. +

diff --git a/doc/go_for_cpp_programmers.html b/doc/go_for_cpp_programmers.html new file mode 100644 index 000000000..7168f1d05 --- /dev/null +++ b/doc/go_for_cpp_programmers.html @@ -0,0 +1,707 @@ + + +

+Go is a systems programming language intended to be a general-purpose +systems language, like C++. +These are some notes on Go for experienced C++ programmers. This +document discusses the differences between Go and C++, and says little +to nothing about the similarities. + +

+For a more general introduction to Go, see the +Go tutorial and +Effective Go. + +

+For a detailed description of the Go language, see the +Go spec. + +

Conceptual Differences

+ +
    +
  • Go does not have classes with constructors or destructors. + Instead of class methods, a class inheritance hierarchy, + and virtual functions, Go provides interfaces, which are + discussed in more detail below. + Interfaces are also used where C++ uses templates. + +
  • Go uses garbage collection. It is not necessary (or possible) + to release memory explicitly. The garbage collection is (intended to be) + incremental and highly efficient on modern processors. + +
  • Go has pointers but not pointer arithmetic. You cannot + use a pointer variable to walk through the bytes of a string. + +
  • Arrays in Go are first class values. When an array is used as a + function parameter, the function receives a copy of the array, not + a pointer to it. However, in practice functions often use slices + for parameters; slices hold pointers to underlying arrays. Slices + are discussed further below. + +
  • Strings are provided by the language. They may not be changed once they + have been created. + +
  • Hash tables are provided by the language. They are called maps. + +
  • Separate threads of execution, and communication channels between + them, are provided by the language. This + is discussed further below. + +
  • Certain types (maps and channels, described further below) + are passed by reference, not by value. That is, passing a map to a + function does not copy the map, and if the function changes the map + the change will be seen by the caller. In C++ terms, one can + think of these as being reference types. + +
  • Go does not use header files. Instead, each source file is part of a + defined package. When a package defines an object + (type, constant, variable, function) with a name starting with an + upper case letter, that object is visible to any other file which + imports that package. + +
  • Go does not support implicit type conversion. Operations that mix + different types require casts (called conversions in Go). + +
  • Go does not support function overloading and does not support user + defined operators. + +
  • Go does not support const or volatile qualifiers. + +
  • Go uses nil for invalid pointers, where C++ uses + NULL or simply 0. +
+ +

Syntax

+ +

+The declaration syntax is reversed compared to C++. You write the name +followed by the type. Unlike in C++, the syntax for a type does not match +the way in which the variable is used. Type declarations may be read +easily from left to right. + +

+Go                           C++
+var v1 int                // int v1;
+var v2 string             // const std::string v2;  (approximately)
+var v3 [10]int            // int v3[10];
+var v4 []int              // int* v4;  (approximately)
+var v5 struct { f int }   // struct { int f; } v5;
+var v6 *int               // int* v6;  (but no pointer arithmetic)
+var v7 map[string]int     // unordered_map<string, int>* v7;  (approximately)
+var v8 func(a int) int    // int (*v8)(int a);
+
+ +

+Declarations generally take the form of a keyword followed by the name +of the object being declared. The keyword is one of var, +func, +const, or type. Method declarations are a minor +exception in that +the receiver appears before the name of the object being declared; see +the discussion of interfaces. + +

+You can also use a keyword followed by a series of declarations in +parentheses. + +

+var (
+    i int
+    m float64
+)
+
+ +

+When declaring a function, you must either provide a name for each parameter +or not provide a name for any parameter; you can't omit some names +and provide others. You may group several names with the same type: + +

+func f(i, j, k int, s, t string)
+
+ +

+A variable may be initialized when it is declared. When this is done, +specifying the type is permitted but not required. When the type is +not specified, the type of the variable is the type of the +initialization expression. + +

+var v = *p
+
+ +

+See also the discussion of constants, below. +If a variable is not initialized explicitly, the type must be specified. +In that case it will be +implicitly initialized to the type's zero value (0, nil, etc.). There are no +uninitialized variables in Go. + +

+Within a function, a short declaration syntax is available with +:= . + +

+v1 := v2
+
+ +

+This is equivalent to + +

+var v1 = v2
+
+ +

+Go permits multiple assignments, which are done in parallel. + +

+i, j = j, i    // Swap i and j.
+
+ +

+Functions may have multiple return values, indicated by a list in +parentheses. The returned values can be stored by assignment +to a list of variables. + +

+func f() (i int, j int) { ... }
+v1, v2 = f()
+
+ +

+Go code uses very few semicolons in practice. Technically, all Go +statements are terminated by a semicolon. However, Go treats the end +of a non-blank line as a semicolon unless the line is clearly +incomplete (the exact rules are +in the language specification). +A consequence of this is that in some cases Go does not permit you to +use a line break. For example, you may not write +

+func g()
+{                  // INVALID
+}
+
+A semicolon will be inserted after g(), causing it to be +a function declaration rather than a function definition. Similarly, +you may not write +
+if x {
+}
+else {             // INVALID
+}
+
+A semicolon will be inserted after the } preceding +the else, causing a syntax error. + +

+Since semicolons do end statements, you may continue using them as in +C++. However, that is not the recommended style. Idiomatic Go code +omits unnecessary semicolons, which in practice is all of them other +than the initial for loop clause and cases where you want several +short statements on a single line. + +

+While we're on the topic, we recommend that rather than worry about +semicolons and brace placement, you format your code with +the gofmt program. That will produce a single standard +Go style, and let you worry about your code rather than your +formatting. While the style may initially seem odd, it is as good as +any other style, and familiarity will lead to comfort. + +

+When using a pointer to a struct, you use . instead +of ->. +Thus syntactically speaking a structure and a pointer to a structure +are used in the same way. + +

+type myStruct struct { i int }
+var v9 myStruct              // v9 has structure type
+var p9 *myStruct             // p9 is a pointer to a structure
+f(v9.i, p9.i)
+
+ +

+Go does not require parentheses around the condition of a if +statement, or the expressions of a for statement, or the value of a +switch statement. On the other hand, it does require curly braces +around the body of an if or for statement. + +

+if a < b { f() }             // Valid
+if (a < b) { f() }           // Valid (condition is a parenthesized expression)
+if (a < b) f()               // INVALID
+for i = 0; i < 10; i++ {}    // Valid
+for (i = 0; i < 10; i++) {}  // INVALID
+
+ +

+Go does not have a while statement nor does it have a +do/while +statement. The for statement may be used with a single condition, +which makes it equivalent to a while statement. Omitting the +condition entirely is an endless loop. + +

+Go permits break and continue to specify a label. +The label must +refer to a for, switch, or select +statement. + +

+In a switch statement, case labels do not fall +through. You can +make them fall through using the fallthrough keyword. This applies +even to adjacent cases. + +

+switch i {
+case 0:  // empty case body
+case 1:
+    f()  // f is not called when i == 0!
+}
+
+ +

+But a case can have multiple values. + +

+switch i {
+case 0, 1:
+    f()  // f is called if i == 0 || i == 1.
+}
+
+ +

+The values in a case need not be constants—or even integers; +any type +that supports the equality comparison operator, such as strings or +pointers, can be used—and if the switch +value is omitted it defaults to true. + +

+switch {
+case i < 0:
+    f1()
+case i == 0:
+    f2()
+case i > 0:
+    f3()
+}
+
+ +

+The ++ and -- operators may only be used in +statements, not in expressions. +You cannot write c = *p++. *p++ is parsed as +(*p)++. + +

+The defer statement may be used to call a function after +the function containing the defer statement returns. + +

+fd := open("filename")
+defer close(fd)         // fd will be closed when this function returns.
+
+ +

Constants

+ +

+In Go constants may be untyped. This applies even to constants +named with a const declaration, if no +type is given in the declaration and the initializer expression uses only +untyped constants. +A value derived from an untyped constant becomes typed when it +is used within a context that +requires a typed value. This permits constants to be used relatively +freely without requiring general implicit type conversion. + +

+var a uint
+f(a + 1)  // untyped numeric constant "1" becomes typed as uint
+
+ +

+The language does not impose any limits on the size of an untyped +numeric constant or constant expression. A limit is only applied when +a constant is used where a type is required. + +

+const huge = 1 << 100
+f(huge >> 98)
+
+ +

+Go does not support enums. Instead, you can use the special name +iota in a single const declaration to get a +series of increasing +value. When an initialization expression is omitted for a const, +it reuses the preceding expression. + +

+const (
+    red = iota   // red == 0
+    blue         // blue == 1
+    green        // green == 2
+)
+
+ +

Slices

+ +

+A slice is conceptually a struct with three fields: a +pointer to an array, a length, and a capacity. +Slices support +the [] operator to access elements of the underlying array. +The builtin +len function returns the +length of the slice. The builtin cap function returns the +capacity. + +

+Given an array, or another slice, a new slice is created via +a[I:J]. This +creates a new slice which refers to a, starts at +index I, and ends before index +J. It has length J - I. +The new slice refers to the same array +to which a +refers. That is, changes made using the new slice may be seen using +a. The +capacity of the new slice is simply the capacity of a minus +I. The capacity +of an array is the length of the array. You may also assign an array pointer +to a variable of slice type; given var s []int; var a[10] int, +the assignment s = &a is equivalent to +s = a[0:len(a)]. + +

+What this means is that Go uses slices for some cases where C++ uses pointers. +If you create a value of type [100]byte (an array of 100 bytes, +perhaps a +buffer) and you want to pass it to a function without copying it, you should +declare the function parameter to have type []byte, and pass the +address +of the array. Unlike in C++, it is not +necessary to pass the length of the buffer; it is efficiently accessible via +len. + +

+The slice syntax may also be used with a string. It returns a new string, +whose value is a substring of the original string. +Because strings are immutable, string slices can be implemented +without allocating new storage for the slices's contents. + +

Making values

+ +

+Go has a builtin function new which takes a type and +allocates space +on the heap. The allocated space will be zero-initialized for the type. +For example, new(int) allocates a new int on the heap, +initializes it with the value 0, +and returns its address, which has type *int. +Unlike in C++, new is a function, not an operator; +new int is a syntax error. + +

+Map and channel values must be allocated using the builtin function +make. +A variable declared with map or channel type without an initializer will be +automatically initialized to nil. +Calling make(map[int]int) returns a newly allocated value of +type map[int]int. +Note that make returns a value, not a pointer. This is +consistent with +the fact that map and channel values are passed by reference. Calling +make with +a map type takes an optional argument which is the expected capacity of the +map. Calling make with a channel type takes an optional +argument which sets the +buffering capacity of the channel; the default is 0 (unbuffered). + +

+The make function may also be used to allocate a slice. +In this case it +allocates memory for the underlying array and returns a slice referring to it. +There is one required argument, which is the number of elements in the slice. +A second, optional, argument is the capacity of the slice. For example, +make([]int, 10, 20). This is identical to +new([20]int)[0:10]. Since +Go uses garbage collection, the newly allocated array will be discarded +sometime after there are no references to the returned slice. + +

Interfaces

+ +

+Where C++ provides classes, subclasses and templates, +Go provides interfaces. A +Go interface is similar to a C++ pure abstract class: a class with no +data members, with methods which are all pure virtual. However, in +Go, any type which provides the methods named in the interface may be +treated as an implementation of the interface. No explicitly declared +inheritance is required. The implementation of the interface is +entirely separate from the interface itself. + +

+A method looks like an ordinary function definition, except that it +has a receiver. The receiver is similar to +the this pointer in a C++ class method. + +

+type myType struct { i int }
+func (p *myType) get() int { return p.i }
+
+ +

+This declares a method get associated with myType. +The receiver is named p in the body of the function. + +

+Methods are defined on named types. If you convert the value +to a different type, the new value will have the methods of the new type, +not the old type. + +

+You may define methods on a builtin type by declaring a new named type +derived from it. The new type is distinct from the builtin type. + +

+type myInteger int
+func (p myInteger) get() int { return int(p) } // Conversion required.
+func f(i int) { }
+var v myInteger
+// f(v) is invalid.
+// f(int(v)) is valid; int(v) has no defined methods.
+
+ +

+Given this interface: + +

+type myInterface interface {
+	get() int
+	set(i int)
+}
+
+ +

+we can make myType satisfy the interface by adding + +

+func (p *myType) set(i int) { p.i = i }
+
+ +

+Now any function which takes myInterface as a parameter +will accept a +variable of type *myType. + +

+func getAndSet(x myInterface) {}
+func f1() {
+	var p myType
+	getAndSet(&p)
+}
+
+ +

+In other words, if we view myInterface as a C++ pure abstract +base +class, defining set and get for +*myType made *myType automatically +inherit from myInterface. A type may satisfy multiple interfaces. + +

+An anonymous field may be used to implement something much like a C++ child +class. + +

+type myChildType struct { myType; j int }
+func (p *myChildType) get() int { p.j++; return p.myType.get() }
+
+ +

+This effectively implements myChildType as a child of +myType. + +

+func f2() {
+	var p myChildType
+	getAndSet(&p)
+}
+
+ +

+The set method is effectively inherited from +myChildType, because +methods associated with the anonymous field are promoted to become methods +of the enclosing type. In this case, because myChildType has an +anonymous field of type myType, the methods of +myType also become methods of myChildType. +In this example, the get method was +overridden, and the set method was inherited. + +

+This is not precisely the same as a child class in C++. +When a method of an anonymous field is called, +its receiver is the field, not the surrounding struct. +In other words, methods on anonymous fields are not virtual functions. +When you want the equivalent of a virtual function, use an interface. + +

+A variable which has an interface type may be converted to have a +different interface type using a special construct called a type assertion. +This is implemented dynamically +at run time, like C++ dynamic_cast. Unlike +dynamic_cast, there does +not need to be any declared relationship between the two interfaces. + +

+type myPrintInterface interface {
+  print()
+}
+func f3(x myInterface) {
+	x.(myPrintInterface).print()  // type assertion to myPrintInterface
+}
+
+ +

+The conversion to myPrintInterface is entirely dynamic. +It will +work as long as the underlying type of x (the dynamic type) defines +a print method. + +

+Because the conversion is dynamic, it may be used to implement generic +programming similar to templates in C++. This is done by +manipulating values of the minimal interface. + +

+type Any interface { }
+
+ +

+Containers may be written in terms of Any, but the caller +must unbox using a type assertion to recover +values of the contained type. As the typing is dynamic rather +than static, there is no equivalent of the way that a C++ template may +inline the relevant operations. The operations are fully type-checked +at run time, but all operations will involve a function call. + +

+type iterator interface {
+	get() Any
+	set(v Any)
+	increment()
+	equal(arg *iterator) bool
+}
+
+ +

Goroutines

+ +

+Go permits starting a new thread of execution (a goroutine) +using the go +statement. The go statement runs a function in a +different, newly created, goroutine. +All goroutines in a single program share the same address space. + +

+Internally, goroutines act like coroutines that are multiplexed among +multiple operating system threads. You do not have to worry +about these details. + +

+func server(i int) {
+    for {
+        print(i)
+        sys.sleep(10)
+    }
+}
+go server(1)
+go server(2)
+
+ +

+(Note that the for statement in the server +function is equivalent to a C++ while (true) loop.) + +

+Goroutines are (intended to be) cheap. + +

+Function literals (which Go implements as closures) +can be useful with the go statement. + +

+var g int
+go func(i int) {
+	s := 0
+	for j := 0; j < i; j++ { s += j }
+	g = s
+}(1000)  // Passes argument 1000 to the function literal.
+
+ +

Channels

+ +

+Channels are used to communicate between goroutines. Any value may be +sent over a channel. Channels are (intended to be) efficient and +cheap. To send a value on a channel, use <- as a binary +operator. To +receive a value on a channel, use <- as a unary operator. +When calling +functions, channels are passed by reference. + +

+The Go library provides mutexes, but you can also use +a single goroutine with a shared channel. +Here is an example of using a manager function to control access to a +single value. + +

+type cmd struct { get bool; val int }
+func manager(ch chan cmd) {
+	var val int = 0
+	for {
+		c := <- ch
+		if c.get { c.val = val; ch <- c }
+		else { val = c.val }
+	}
+}
+
+ +

+In that example the same channel is used for input and output. +This is incorrect if there are multiple goroutines communicating +with the manager at once: a goroutine waiting for a response +from the manager might receive a request from another goroutine +instead. +A solution is to pass in a channel. + +

+type cmd2 struct { get bool; val int; ch <- chan int }
+func manager2(ch chan cmd2) {
+	var val int = 0
+	for {
+		c := <- ch
+		if c.get { c.ch <- val }
+		else { val = c.val }
+	}
+}
+
+ +

+To use manager2, given a channel to it: + +

+func f4(ch <- chan cmd2) int {
+	myCh := make(chan int)
+	c := cmd2{ true, 0, myCh }   // Composite literal syntax.
+	ch <- c
+	return <-myCh
+}
+
diff --git a/doc/go_mem.html b/doc/go_mem.html new file mode 100644 index 000000000..a38828358 --- /dev/null +++ b/doc/go_mem.html @@ -0,0 +1,510 @@ + + + + + +

Introduction

+ +

+The Go memory model specifies the conditions under which +reads of a variable in one goroutine can be guaranteed to +observe values produced by writes to the same variable in a different goroutine. +

+ +

Happens Before

+ +

+Within a single goroutine, reads and writes must behave +as if they executed in the order specified by the program. +That is, compilers and processors may reorder the reads and writes +executed within a single goroutine only when the reordering +does not change the behavior within that goroutine +as defined by the language specification. +Because of this reordering, the execution order observed +by one goroutine may differ from the order perceived +by another. For example, if one goroutine +executes a = 1; b = 2;, another might observe +the updated value of b before the updated value of a. +

+ +

+To specify the requirements of reads and writes, we define +happens before, a partial order on the execution +of memory operations in a Go program. If event e1 happens +before event e2, then we say that e2 happens after e1. +Also, if e1 does not happen before e2 and does not happen +after e2, then we say that e1 and e2 happen concurrently. +

+ +

+Within a single goroutine, the happens-before order is the +order expressed by the program. +

+ +

+A read r of a variable v is allowed to observe a write w to v +if both of the following hold: +

+ +
    +
  1. w happens before r.
  2. +
  3. There is no other write w' to v that happens + after w but before r.
  4. +
+ +

+To guarantee that a read r of a variable v observes a +particular write w to v, ensure that w is the only +write r is allowed to observe. +That is, r is guaranteed to observe w if both of the following hold: +

+ +
    +
  1. w happens before r.
  2. +
  3. Any other write to the shared variable v +either happens before w or after r.
  4. +
+ +

+This pair of conditions is stronger than the first pair; +it requires that there are no other writes happening +concurrently with w or r. +

+ +

+Within a single goroutine, +there is no concurrency, so the two definitions are equivalent: +a read r observes the value written by the most recent write w to v. +When multiple goroutines access a shared variable v, +they must use synchronization events to establish +happens-before conditions that ensure reads observe the +desired writes. +

+ +

+The initialization of variable v with the zero value +for v's type behaves as a write in the memory model. +

+ +

+Reads and writes of values larger than a single machine word +behave as multiple machine-word-sized operations in an +unspecified order. +

+ +

Synchronization

+ +

Initialization

+ +

+Program initialization runs in a single goroutine and +new goroutines created during initialization do not +start running until initialization ends. +

+ +

+If a package p imports package q, the completion of +q's init functions happens before the start of any of p's. +

+ +

+The start of the function main.main happens after +all init functions have finished. +

+ +

+The execution of any goroutines created during init +functions happens after all init functions have finished. +

+ +

Goroutine creation

+ +

+The go statement that starts a new goroutine +happens before the goroutine's execution begins. +

+ +

+For example, in this program: +

+ +
+var a string
+
+func f() {
+	print(a)
+}
+
+func hello() {
+	a = "hello, world"
+	go f()
+}
+
+ +

+calling hello will print "hello, world" +at some point in the future (perhaps after hello has returned). +

+ +

Goroutine destruction

+ +

+The exit of a goroutine is not guaranteed to happen before +any event in the program. For example, in this program: +

+ +
+var a string
+
+func hello() {
+	go func() { a = "hello" }()
+	print(a)
+}
+
+ +

+the assignment to a is not followed by +any synchronization event, so it is not guaranteed to be +observed by any other goroutine. +In fact, an aggressive compiler might delete the entire go statement. +

+ +

+If the effects of a goroutine must be observed by another goroutine, +use a synchronization mechanism such as a lock or channel +communication to establish a relative ordering. +

+ +

Channel communication

+ +

+Channel communication is the main method of synchronization +between goroutines. Each send on a particular channel +is matched to a corresponding receive from that channel, +usually in a different goroutine. +

+ +

+A send on a channel happens before the corresponding +receive from that channel completes. +

+ +

+This program: +

+ +
+var c = make(chan int, 10)
+var a string
+
+func f() {
+	a = "hello, world"
+	c <- 0
+}
+
+func main() {
+	go f()
+	<-c
+	print(a)
+}
+
+ +

+is guaranteed to print "hello, world". The write to a +happens before the send on c, which happens before +the corresponding receive on c completes, which happens before +the print. +

+ +

+The closing of a channel happens before a receive that returns a zero value +because the channel is closed. +

+ +

+In the previous example, replacing +c <- 0 with close(c) +yields a program with the same guaranteed behavior. +

+ +

+A receive from an unbuffered channel happens before +the send on that channel completes. +

+ +

+This program (as above, but with the send and receive statements swapped and +using an unbuffered channel): +

+ +
+var c = make(chan int)
+var a string
+
+func f() {
+	a = "hello, world"
+	<-c
+}
+
+ +
+func main() {
+	go f()
+	c <- 0
+	print(a)
+}
+
+ +

+is also guaranteed to print "hello, world". The write to a +happens before the receive on c, which happens before +the corresponding send on c completes, which happens +before the print. +

+ +

+If the channel were buffered (e.g., c = make(chan int, 1)) +then the program would not be guaranteed to print +"hello, world". (It might print the empty string; +it cannot print "goodbye, universe", nor can it crash.) +

+ +

Locks

+ +

+The sync package implements two lock data types, +sync.Mutex and sync.RWMutex. +

+ +

+For any sync.Mutex or sync.RWMutex variable l and n < m, +the n'th call to l.Unlock() happens before the m'th call to l.Lock() returns. +

+ +

+This program: +

+ +
+var l sync.Mutex
+var a string
+
+func f() {
+	a = "hello, world"
+	l.Unlock()
+}
+
+func main() {
+	l.Lock()
+	go f()
+	l.Lock()
+	print(a)
+}
+
+ +

+is guaranteed to print "hello, world". +The first call to l.Unlock() (in f) happens +before the second call to l.Lock() (in main) returns, +which happens before the print. +

+ +

+For any call to l.RLock on a sync.RWMutex variable l, +there is an n such that the l.RLock happens (returns) after the n'th call to +l.Unlock and the matching l.RUnlock happens +before the n+1'th call to l.Lock. +

+ +

Once

+ +

+The sync package provides a safe mechanism for +initialization in the presence of multiple goroutines +through the use of the Once type. +Multiple threads can execute once.Do(f) for a particular f, +but only one will run f(), and the other calls block +until f() has returned. +

+ +

+A single call of f() from once.Do(f) happens (returns) before any call of once.Do(f) returns. +

+ +

+In this program: +

+ +
+var a string
+var once sync.Once
+
+func setup() {
+	a = "hello, world"
+}
+
+func doprint() {
+	once.Do(setup)
+	print(a)
+}
+
+func twoprint() {
+	go doprint()
+	go doprint()
+}
+
+ +

+calling twoprint causes "hello, world" to be printed twice. +The first call to twoprint runs setup once. +

+ +

Incorrect synchronization

+ +

+Note that a read r may observe the value written by a write w +that happens concurrently with r. +Even if this occurs, it does not imply that reads happening after r +will observe writes that happened before w. +

+ +

+In this program: +

+ +
+var a, b int
+
+func f() {
+	a = 1
+	b = 2
+}
+
+func g() {
+	print(b)
+	print(a)
+}
+
+func main() {
+	go f()
+	g()
+}
+
+ +

+it can happen that g prints 2 and then 0. +

+ +

+This fact invalidates a few common idioms. +

+ +

+Double-checked locking is an attempt to avoid the overhead of synchronization. +For example, the twoprint program might be +incorrectly written as: +

+ +
+var a string
+var done bool
+
+func setup() {
+	a = "hello, world"
+	done = true
+}
+
+func doprint() {
+	if !done {
+		once.Do(setup)
+	}
+	print(a)
+}
+
+func twoprint() {
+	go doprint()
+	go doprint()
+}
+
+ +

+but there is no guarantee that, in doprint, observing the write to done +implies observing the write to a. This +version can (incorrectly) print an empty string +instead of "hello, world". +

+ +

+Another incorrect idiom is busy waiting for a value, as in: +

+ +
+var a string
+var done bool
+
+func setup() {
+	a = "hello, world"
+	done = true
+}
+
+func main() {
+	go setup()
+	for !done {
+	}
+	print(a)
+}
+
+ +

+As before, there is no guarantee that, in main, +observing the write to done +implies observing the write to a, so this program could +print an empty string too. +Worse, there is no guarantee that the write to done will ever +be observed by main, since there are no synchronization +events between the two threads. The loop in main is not +guaranteed to finish. +

+ +

+There are subtler variants on this theme, such as this program. +

+ +
+type T struct {
+	msg string
+}
+
+var g *T
+
+func setup() {
+	t := new(T)
+	t.msg = "hello, world"
+	g = t
+}
+
+func main() {
+	go setup()
+	for g == nil {
+	}
+	print(g.msg)
+}
+
+ +

+Even if main observes g != nil and exits its loop, +there is no guarantee that it will observe the initialized +value for g.msg. +

+ +

+In all these examples, the solution is the same: +use explicit synchronization. +

diff --git a/doc/go_spec.html b/doc/go_spec.html new file mode 100644 index 000000000..82c7ed419 --- /dev/null +++ b/doc/go_spec.html @@ -0,0 +1,5272 @@ + + + + + + +

Introduction

+ +

+This is a reference manual for the Go programming language. For +more information and other documents, see http://golang.org. +

+ +

+Go is a general-purpose language designed with systems programming +in mind. It is strongly typed and garbage-collected and has explicit +support for concurrent programming. Programs are constructed from +packages, whose properties allow efficient management of +dependencies. The existing implementations use a traditional +compile/link model to generate executable binaries. +

+ +

+The grammar is compact and regular, allowing for easy analysis by +automatic tools such as integrated development environments. +

+ +

Notation

+

+The syntax is specified using Extended Backus-Naur Form (EBNF): +

+ +
+Production  = production_name "=" [ Expression ] "." .
+Expression  = Alternative { "|" Alternative } .
+Alternative = Term { Term } .
+Term        = production_name | token [ "…" token ] | Group | Option | Repetition .
+Group       = "(" Expression ")" .
+Option      = "[" Expression "]" .
+Repetition  = "{" Expression "}" .
+
+ +

+Productions are expressions constructed from terms and the following +operators, in increasing precedence: +

+
+|   alternation
+()  grouping
+[]  option (0 or 1 times)
+{}  repetition (0 to n times)
+
+ +

+Lower-case production names are used to identify lexical tokens. +Non-terminals are in CamelCase. Lexical symbols are enclosed in +double quotes "" or back quotes ``. +

+ +

+The form a … b represents the set of characters from +a through b as alternatives. The horizontal +ellipis … is also used elsewhere in the spec to informally denote various +enumerations or code snippets that are not further specified. The character … +(as opposed to the three characters ...) is not a token of the Go +language. +

+ +

Source code representation

+ +

+Source code is Unicode text encoded in +UTF-8. The text is not +canonicalized, so a single accented code point is distinct from the +same character constructed from combining an accent and a letter; +those are treated as two code points. For simplicity, this document +will use the term character to refer to a Unicode code point. +

+

+Each code point is distinct; for instance, upper and lower case letters +are different characters. +

+

+Implementation restriction: For compatibility with other tools, a +compiler may disallow the NUL character (U+0000) in the source text. +

+ +

Characters

+ +

+The following terms are used to denote specific Unicode character classes: +

+
+newline        = /* the Unicode code point U+000A */ .
+unicode_char   = /* an arbitrary Unicode code point except newline */ .
+unicode_letter = /* a Unicode code point classified as "Letter" */ .
+unicode_digit  = /* a Unicode code point classified as "Decimal Digit" */ .
+
+ +

+In The Unicode Standard 6.0, +Section 4.5 "General Category" +defines a set of character categories. Go treats +those characters in category Lu, Ll, Lt, Lm, or Lo as Unicode letters, +and those in category Nd as Unicode digits. +

+ +

Letters and digits

+ +

+The underscore character _ (U+005F) is considered a letter. +

+
+letter        = unicode_letter | "_" .
+decimal_digit = "0" … "9" .
+octal_digit   = "0" … "7" .
+hex_digit     = "0" … "9" | "A" … "F" | "a" … "f" .
+
+ +

Lexical elements

+ +

Comments

+ +

+There are two forms of comments: +

+ +
    +
  1. +Line comments start with the character sequence // +and stop at the end of the line. A line comment acts like a newline. +
  2. +
  3. +General comments start with the character sequence /* +and continue through the character sequence */. A general +comment that spans multiple lines acts like a newline, otherwise it acts +like a space. +
  4. +
+ +

+Comments do not nest. +

+ + +

Tokens

+ +

+Tokens form the vocabulary of the Go language. +There are four classes: identifiers, keywords, operators +and delimiters, and literals. White space, formed from +spaces (U+0020), horizontal tabs (U+0009), +carriage returns (U+000D), and newlines (U+000A), +is ignored except as it separates tokens +that would otherwise combine into a single token. Also, a newline or end of file +may trigger the insertion of a semicolon. +While breaking the input into tokens, +the next token is the longest sequence of characters that form a +valid token. +

+ +

Semicolons

+ +

+The formal grammar uses semicolons ";" as terminators in +a number of productions. Go programs may omit most of these semicolons +using the following two rules: +

+ +
    +
  1. +

    +When the input is broken into tokens, a semicolon is automatically inserted +into the token stream at the end of a non-blank line if the line's final +token is +

    + +
  2. + +
  3. +To allow complex statements to occupy a single line, a semicolon +may be omitted before a closing ")" or "}". +
  4. +
+ +

+To reflect idiomatic use, code examples in this document elide semicolons +using these rules. +

+ + +

Identifiers

+ +

+Identifiers name program entities such as variables and types. +An identifier is a sequence of one or more letters and digits. +The first character in an identifier must be a letter. +

+
+identifier = letter { letter | unicode_digit } .
+
+
+a
+_x9
+ThisVariableIsExported
+αβ
+
+ +

+Some identifiers are predeclared. +

+ + +

Keywords

+ +

+The following keywords are reserved and may not be used as identifiers. +

+
+break        default      func         interface    select
+case         defer        go           map          struct
+chan         else         goto         package      switch
+const        fallthrough  if           range        type
+continue     for          import       return       var
+
+ +

Operators and Delimiters

+ +

+The following character sequences represent operators, delimiters, and other special tokens: +

+
++    &     +=    &=     &&    ==    !=    (    )
+-    |     -=    |=     ||    <     <=    [    ]
+*    ^     *=    ^=     <-    >     >=    {    }
+/    <<    /=    <<=    ++    =     :=    ,    ;
+%    >>    %=    >>=    --    !     ...   .    :
+     &^          &^=
+
+ +

Integer literals

+ +

+An integer literal is a sequence of digits representing an +integer constant. +An optional prefix sets a non-decimal base: 0 for octal, 0x or +0X for hexadecimal. In hexadecimal literals, letters +a-f and A-F represent values 10 through 15. +

+
+int_lit     = decimal_lit | octal_lit | hex_lit .
+decimal_lit = ( "1" … "9" ) { decimal_digit } .
+octal_lit   = "0" { octal_digit } .
+hex_lit     = "0" ( "x" | "X" ) hex_digit { hex_digit } .
+
+ +
+42
+0600
+0xBadFace
+170141183460469231731687303715884105727
+
+ +

Floating-point literals

+

+A floating-point literal is a decimal representation of a +floating-point constant. +It has an integer part, a decimal point, a fractional part, +and an exponent part. The integer and fractional part comprise +decimal digits; the exponent part is an e or E +followed by an optionally signed decimal exponent. One of the +integer part or the fractional part may be elided; one of the decimal +point or the exponent may be elided. +

+
+float_lit = decimals "." [ decimals ] [ exponent ] |
+            decimals exponent |
+            "." decimals [ exponent ] .
+decimals  = decimal_digit { decimal_digit } .
+exponent  = ( "e" | "E" ) [ "+" | "-" ] decimals .
+
+ +
+0.
+72.40
+072.40  // == 72.40
+2.71828
+1.e+0
+6.67428e-11
+1E6
+.25
+.12345E+5
+
+ +

Imaginary literals

+

+An imaginary literal is a decimal representation of the imaginary part of a +complex constant. +It consists of a +floating-point literal +or decimal integer followed +by the lower-case letter i. +

+
+imaginary_lit = (decimals | float_lit) "i" .
+
+ +
+0i
+011i  // == 11i
+0.i
+2.71828i
+1.e+0i
+6.67428e-11i
+1E6i
+.25i
+.12345E+5i
+
+ + +

Character literals

+ +

+A character literal represents an integer constant, +typically a Unicode code point, as one or more characters enclosed in single +quotes. Within the quotes, any character may appear except single +quote and newline. A single quoted character represents itself, +while multi-character sequences beginning with a backslash encode +values in various formats. +

+

+The simplest form represents the single character within the quotes; +since Go source text is Unicode characters encoded in UTF-8, multiple +UTF-8-encoded bytes may represent a single integer value. For +instance, the literal 'a' holds a single byte representing +a literal a, Unicode U+0061, value 0x61, while +'ä' holds two bytes (0xc3 0xa4) representing +a literal a-dieresis, U+00E4, value 0xe4. +

+

+Several backslash escapes allow arbitrary values to be represented +as ASCII text. There are four ways to represent the integer value +as a numeric constant: \x followed by exactly two hexadecimal +digits; \u followed by exactly four hexadecimal digits; +\U followed by exactly eight hexadecimal digits, and a +plain backslash \ followed by exactly three octal digits. +In each case the value of the literal is the value represented by +the digits in the corresponding base. +

+

+Although these representations all result in an integer, they have +different valid ranges. Octal escapes must represent a value between +0 and 255 inclusive. Hexadecimal escapes satisfy this condition +by construction. The escapes \u and \U +represent Unicode code points so within them some values are illegal, +in particular those above 0x10FFFF and surrogate halves. +

+

+After a backslash, certain single-character escapes represent special values: +

+
+\a   U+0007 alert or bell
+\b   U+0008 backspace
+\f   U+000C form feed
+\n   U+000A line feed or newline
+\r   U+000D carriage return
+\t   U+0009 horizontal tab
+\v   U+000b vertical tab
+\\   U+005c backslash
+\'   U+0027 single quote  (valid escape only within character literals)
+\"   U+0022 double quote  (valid escape only within string literals)
+
+

+All other sequences starting with a backslash are illegal inside character literals. +

+
+char_lit         = "'" ( unicode_value | byte_value ) "'" .
+unicode_value    = unicode_char | little_u_value | big_u_value | escaped_char .
+byte_value       = octal_byte_value | hex_byte_value .
+octal_byte_value = `\` octal_digit octal_digit octal_digit .
+hex_byte_value   = `\` "x" hex_digit hex_digit .
+little_u_value   = `\` "u" hex_digit hex_digit hex_digit hex_digit .
+big_u_value      = `\` "U" hex_digit hex_digit hex_digit hex_digit
+                           hex_digit hex_digit hex_digit hex_digit .
+escaped_char     = `\` ( "a" | "b" | "f" | "n" | "r" | "t" | "v" | `\` | "'" | `"` ) .
+
+ +
+'a'
+'ä'
+'本'
+'\t'
+'\000'
+'\007'
+'\377'
+'\x07'
+'\xff'
+'\u12e4'
+'\U00101234'
+
+ + +

String literals

+ +

+A string literal represents a string constant +obtained from concatenating a sequence of characters. There are two forms: +raw string literals and interpreted string literals. +

+

+Raw string literals are character sequences between back quotes +``. Within the quotes, any character is legal except +back quote. The value of a raw string literal is the +string composed of the uninterpreted characters between the quotes; +in particular, backslashes have no special meaning and the string may +span multiple lines. +

+

+Interpreted string literals are character sequences between double +quotes "". The text between the quotes, +which may not span multiple lines, forms the +value of the literal, with backslash escapes interpreted as they +are in character literals (except that \' is illegal and +\" is legal). The three-digit octal (\nnn) +and two-digit hexadecimal (\xnn) escapes represent individual +bytes of the resulting string; all other escapes represent +the (possibly multi-byte) UTF-8 encoding of individual characters. +Thus inside a string literal \377 and \xFF represent +a single byte of value 0xFF=255, while ÿ, +\u00FF, \U000000FF and \xc3\xbf represent +the two bytes 0xc3 0xbf of the UTF-8 encoding of character +U+00FF. +

+ +
+string_lit             = raw_string_lit | interpreted_string_lit .
+raw_string_lit         = "`" { unicode_char | newline } "`" .
+interpreted_string_lit = `"` { unicode_value | byte_value } `"` .
+
+ +
+`abc`  // same as "abc"
+`\n
+\n`    // same as "\\n\n\\n"
+"\n"
+""
+"Hello, world!\n"
+"日本語"
+"\u65e5本\U00008a9e"
+"\xff\u00FF"
+
+ +

+These examples all represent the same string: +

+ +
+"日本語"                                 // UTF-8 input text
+`日本語`                                 // UTF-8 input text as a raw literal
+"\u65e5\u672c\u8a9e"                    // The explicit Unicode code points
+"\U000065e5\U0000672c\U00008a9e"        // The explicit Unicode code points
+"\xe6\x97\xa5\xe6\x9c\xac\xe8\xaa\x9e"  // The explicit UTF-8 bytes
+
+ +

+If the source code represents a character as two code points, such as +a combining form involving an accent and a letter, the result will be +an error if placed in a character literal (it is not a single code +point), and will appear as two code points if placed in a string +literal. +

+ + +

Constants

+ +

There are boolean constants, integer constants, +floating-point constants, complex constants, +and string constants. Integer, floating-point, +and complex constants are +collectively called numeric constants. +

+ +

+A constant value is represented by an +integer, +floating-point, +imaginary, +character, or +string literal, +an identifier denoting a constant, +a constant expression, +a conversion with a result that is a constant, or +the result value of some built-in functions such as +unsafe.Sizeof applied to any value, +cap or len applied to +some expressions, +real and imag applied to a complex constant +and complex applied to numeric constants. +The boolean truth values are represented by the predeclared constants +true and false. The predeclared identifier +iota denotes an integer constant. +

+ +

+In general, complex constants are a form of +constant expression +and are discussed in that section. +

+ +

+Numeric constants represent values of arbitrary precision and do not overflow. +

+ +

+Constants may be typed or untyped. +Literal constants, true, false, iota, +and certain constant expressions +containing only untyped constant operands are untyped. +

+ +

+A constant may be given a type explicitly by a constant declaration +or conversion, or implicitly when used in a +variable declaration or an +assignment or as an +operand in an expression. +It is an error if the constant value +cannot be represented as a value of the respective type. +For instance, 3.0 can be given any integer or any +floating-point type, while 2147483648.0 (equal to 1<<31) +can be given the types float32, float64, or uint32 but +not int32 or string. +

+ +

+There are no constants denoting the IEEE-754 infinity and not-a-number values, +but the math package's +Inf, +NaN, +IsInf, and +IsNaN +functions return and test for those values at run time. +

+ +

+Implementation restriction: A compiler may implement numeric constants by choosing +an internal representation with at least twice as many bits as any machine type; +for floating-point values, both the mantissa and exponent must be twice as large. +

+ + +

Types

+ +

+A type determines the set of values and operations specific to values of that +type. A type may be specified by a (possibly qualified) type name +(§Qualified identifier, §Type declarations) or a type literal, +which composes a new type from previously declared types. +

+ +
+Type      = TypeName | TypeLit | "(" Type ")" .
+TypeName  = QualifiedIdent .
+TypeLit   = ArrayType | StructType | PointerType | FunctionType | InterfaceType |
+	    SliceType | MapType | ChannelType .
+
+ +

+Named instances of the boolean, numeric, and string types are +predeclared. +Composite types—array, struct, pointer, function, +interface, slice, map, and channel types—may be constructed using +type literals. +

+ +

+The static type (or just type) of a variable is the +type defined by its declaration. Variables of interface type +also have a distinct dynamic type, which +is the actual type of the value stored in the variable at run-time. +The dynamic type may vary during execution but is always +assignable +to the static type of the interface variable. For non-interface +types, the dynamic type is always the static type. +

+ +

+Each type T has an underlying type: If T +is a predeclared type or a type literal, the corresponding underlying +type is T itself. Otherwise, T's underlying type +is the underlying type of the type to which T refers in its +type declaration. +

+ +
+   type T1 string
+   type T2 T1
+   type T3 []T1
+   type T4 T3
+
+ +

+The underlying type of string, T1, and T2 +is string. The underlying type of []T1, T3, +and T4 is []T1. +

+ +

Method sets

+

+A type may have a method set associated with it +(§Interface types, §Method declarations). +The method set of an interface type is its interface. +The method set of any other named type T +consists of all methods with receiver type T. +The method set of the corresponding pointer type *T +is the set of all methods with receiver *T or T +(that is, it also contains the method set of T). +Any other type has an empty method set. +In a method set, each method must have a unique name. +

+ + +

Boolean types

+ +A boolean type represents the set of Boolean truth values +denoted by the predeclared constants true +and false. The predeclared boolean type is bool. + + +

Numeric types

+ +

+A numeric type represents sets of integer or floating-point values. +The predeclared architecture-independent numeric types are: +

+ +
+uint8       the set of all unsigned  8-bit integers (0 to 255)
+uint16      the set of all unsigned 16-bit integers (0 to 65535)
+uint32      the set of all unsigned 32-bit integers (0 to 4294967295)
+uint64      the set of all unsigned 64-bit integers (0 to 18446744073709551615)
+
+int8        the set of all signed  8-bit integers (-128 to 127)
+int16       the set of all signed 16-bit integers (-32768 to 32767)
+int32       the set of all signed 32-bit integers (-2147483648 to 2147483647)
+int64       the set of all signed 64-bit integers (-9223372036854775808 to 9223372036854775807)
+
+float32     the set of all IEEE-754 32-bit floating-point numbers
+float64     the set of all IEEE-754 64-bit floating-point numbers
+
+complex64   the set of all complex numbers with float32 real and imaginary parts
+complex128  the set of all complex numbers with float64 real and imaginary parts
+
+byte        familiar alias for uint8
+
+ +

+The value of an n-bit integer is n bits wide and represented using +two's complement arithmetic. +

+ +

+There is also a set of predeclared numeric types with implementation-specific sizes: +

+ +
+uint     either 32 or 64 bits
+int      same size as uint
+uintptr  an unsigned integer large enough to store the uninterpreted bits of a pointer value
+
+ +

+To avoid portability issues all numeric types are distinct except +byte, which is an alias for uint8. +Conversions +are required when different numeric types are mixed in an expression +or assignment. For instance, int32 and int +are not the same type even though they may have the same size on a +particular architecture. + + +

String types

+ +

+A string type represents the set of string values. +Strings behave like arrays of bytes but are immutable: once created, +it is impossible to change the contents of a string. +The predeclared string type is string. + +

+The elements of strings have type byte and may be +accessed using the usual indexing operations. It is +illegal to take the address of such an element; if +s[i] is the ith byte of a +string, &s[i] is invalid. The length of string +s can be discovered using the built-in function +len. The length is a compile-time constant if s +is a string literal. +

+ + +

Array types

+ +

+An array is a numbered sequence of elements of a single +type, called the element type. +The number of elements is called the length and is never +negative. +

+ +
+ArrayType   = "[" ArrayLength "]" ElementType .
+ArrayLength = Expression .
+ElementType = Type .
+
+ +

+The length is part of the array's type and must be a +constant expression that evaluates to a non-negative +integer value. The length of array a can be discovered +using the built-in function len(a). +The elements can be indexed by integer +indices 0 through the len(a)-1Indexes). +Array types are always one-dimensional but may be composed to form +multi-dimensional types. +

+ +
+[32]byte
+[2*N] struct { x, y int32 }
+[1000]*float64
+[3][5]int
+[2][2][2]float64  // same as [2]([2]([2]float64))
+
+ +

Slice types

+ +

+A slice is a reference to a contiguous segment of an array and +contains a numbered sequence of elements from that array. A slice +type denotes the set of all slices of arrays of its element type. +The value of an uninitialized slice is nil. +

+ +
+SliceType = "[" "]" ElementType .
+
+ +

+Like arrays, slices are indexable and have a length. The length of a +slice s can be discovered by the built-in function +len(s); unlike with arrays it may change during +execution. The elements can be addressed by integer indices 0 +through len(s)-1Indexes). The slice index of a +given element may be less than the index of the same element in the +underlying array. +

+

+A slice, once initialized, is always associated with an underlying +array that holds its elements. A slice therefore shares storage +with its array and with other slices of the same array; by contrast, +distinct arrays always represent distinct storage. +

+

+The array underlying a slice may extend past the end of the slice. +The capacity is a measure of that extent: it is the sum of +the length of the slice and the length of the array beyond the slice; +a slice of length up to that capacity can be created by `slicing' a new +one from the original slice (§Slices). +The capacity of a slice a can be discovered using the +built-in function cap(a). +

+ +

+A new, initialized slice value for a given element type T is +made using the built-in function +make, +which takes a slice type +and parameters specifying the length and optionally the capacity: +

+ +
+make([]T, length)
+make([]T, length, capacity)
+
+ +

+A call to make allocates a new, hidden array to which the returned +slice value refers. That is, executing +

+ +
+make([]T, length, capacity)
+
+ +

+produces the same slice as allocating an array and slicing it, so these two examples +result in the same slice: +

+ +
+make([]int, 50, 100)
+new([100]int)[0:50]
+
+ +

+Like arrays, slices are always one-dimensional but may be composed to construct +higher-dimensional objects. +With arrays of arrays, the inner arrays are, by construction, always the same length; +however with slices of slices (or arrays of slices), the lengths may vary dynamically. +Moreover, the inner slices must be allocated individually (with make). +

+ +

Struct types

+ +

+A struct is a sequence of named elements, called fields, each of which has a +name and a type. Field names may be specified explicitly (IdentifierList) or +implicitly (AnonymousField). +Within a struct, non-blank field names must +be unique. +

+ +
+StructType     = "struct" "{" { FieldDecl ";" } "}" .
+FieldDecl      = (IdentifierList Type | AnonymousField) [ Tag ] .
+AnonymousField = [ "*" ] TypeName .
+Tag            = string_lit .
+
+ +
+// An empty struct.
+struct {}
+
+// A struct with 6 fields.
+struct {
+	x, y int
+	u float32
+	_ float32  // padding
+	A *[]int
+	F func()
+}
+
+ +

+A field declared with a type but no explicit field name is an anonymous field +(colloquially called an embedded field). +Such a field type must be specified as +a type name T or as a pointer to a non-interface type name *T, +and T itself may not be +a pointer type. The unqualified type name acts as the field name. +

+ +
+// A struct with four anonymous fields of type T1, *T2, P.T3 and *P.T4
+struct {
+	T1        // field name is T1
+	*T2       // field name is T2
+	P.T3      // field name is T3
+	*P.T4     // field name is T4
+	x, y int  // field names are x and y
+}
+
+ +

+The following declaration is illegal because field names must be unique +in a struct type: +

+ +
+struct {
+	T         // conflicts with anonymous field *T and *P.T
+	*T        // conflicts with anonymous field T and *P.T
+	*P.T      // conflicts with anonymous field T and *T
+}
+
+ +

+Fields and methods (§Method declarations) of an anonymous field are +promoted to be ordinary fields and methods of the struct (§Selectors). +The following rules apply for a struct type named S and +a type named T: +

+
    +
  • If S contains an anonymous field T, the + method set of S includes the + method set of T. +
  • + +
  • If S contains an anonymous field *T, the + method set of S includes the method set of *T + (which itself includes the method set of T). +
  • + +
  • If S contains an anonymous field T or + *T, the method set of *S includes the + method set of *T (which itself includes the method + set of T). +
  • +
+

+A field declaration may be followed by an optional string literal tag, +which becomes an attribute for all the fields in the corresponding +field declaration. The tags are made +visible through a reflection interface +but are otherwise ignored. +

+ +
+// A struct corresponding to the TimeStamp protocol buffer.
+// The tag strings define the protocol buffer field numbers.
+struct {
+	microsec  uint64 "field 1"
+	serverIP6 uint64 "field 2"
+	process   string "field 3"
+}
+
+ +

Pointer types

+ +

+A pointer type denotes the set of all pointers to variables of a given +type, called the base type of the pointer. +The value of an uninitialized pointer is nil. +

+ +
+PointerType = "*" BaseType .
+BaseType = Type .
+
+ +
+*int
+*map[string] *chan int
+
+ +

Function types

+ +

+A function type denotes the set of all functions with the same parameter +and result types. The value of an uninitialized variable of function type +is nil. +

+ +
+FunctionType   = "func" Signature .
+Signature      = Parameters [ Result ] .
+Result         = Parameters | Type .
+Parameters     = "(" [ ParameterList [ "," ] ] ")" .
+ParameterList  = ParameterDecl { "," ParameterDecl } .
+ParameterDecl  = [ IdentifierList ] [ "..." ] Type .
+
+ +

+Within a list of parameters or results, the names (IdentifierList) +must either all be present or all be absent. If present, each name +stands for one item (parameter or result) of the specified type; if absent, each +type stands for one item of that type. Parameter and result +lists are always parenthesized except that if there is exactly +one unnamed result it may be written as an unparenthesized type. +

+ +

+The final parameter in a function signature may have +a type prefixed with .... +A function with such a parameter is called variadic and +may be invoked with zero or more arguments for that parameter. +

+ +
+func()
+func(x int)
+func() int
+func(prefix string, values ...int)
+func(a, b int, z float32) bool
+func(a, b int, z float32) (bool)
+func(a, b int, z float64, opt ...interface{}) (success bool)
+func(int, int, float64) (float64, *[]int)
+func(n int) func(p *T)
+
+ + +

Interface types

+ +

+An interface type specifies a method set called its interface. +A variable of interface type can store a value of any type with a method set +that is any superset of the interface. Such a type is said to +implement the interface. +The value of an uninitialized variable of interface type is nil. +

+ +
+InterfaceType      = "interface" "{" { MethodSpec ";" } "}" .
+MethodSpec         = MethodName Signature | InterfaceTypeName .
+MethodName         = identifier .
+InterfaceTypeName  = TypeName .
+
+ +

+As with all method sets, in an interface type, each method must have a unique name. +

+ +
+// A simple File interface
+interface {
+	Read(b Buffer) bool
+	Write(b Buffer) bool
+	Close()
+}
+
+ +

+More than one type may implement an interface. +For instance, if two types S1 and S2 +have the method set +

+ +
+func (p T) Read(b Buffer) bool { return … }
+func (p T) Write(b Buffer) bool { return … }
+func (p T) Close() { … }
+
+ +

+(where T stands for either S1 or S2) +then the File interface is implemented by both S1 and +S2, regardless of what other methods +S1 and S2 may have or share. +

+ +

+A type implements any interface comprising any subset of its methods +and may therefore implement several distinct interfaces. For +instance, all types implement the empty interface: +

+ +
+interface{}
+
+ +

+Similarly, consider this interface specification, +which appears within a type declaration +to define an interface called Lock: +

+ +
+type Lock interface {
+	Lock()
+	Unlock()
+}
+
+ +

+If S1 and S2 also implement +

+ +
+func (p T) Lock() { … }
+func (p T) Unlock() { … }
+
+ +

+they implement the Lock interface as well +as the File interface. +

+

+An interface may contain an interface type name T +in place of a method specification. +The effect is equivalent to enumerating the methods of T explicitly +in the interface. +

+ +
+type ReadWrite interface {
+	Read(b Buffer) bool
+	Write(b Buffer) bool
+}
+
+type File interface {
+	ReadWrite  // same as enumerating the methods in ReadWrite
+	Lock       // same as enumerating the methods in Lock
+	Close()
+}
+
+ +

Map types

+ +

+A map is an unordered group of elements of one type, called the +element type, indexed by a set of unique keys of another type, +called the key type. +The value of an uninitialized map is nil. +

+ +
+MapType     = "map" "[" KeyType "]" ElementType .
+KeyType     = Type .
+
+ +

+The comparison operators == and != +(§Comparison operators) must be fully defined +for operands of the key type; thus the key type must not be a struct, array or slice. +If the key type is an interface type, these +comparison operators must be defined for the dynamic key values; +failure will cause a run-time panic. + +

+ +
+map [string] int
+map [*T] struct { x, y float64 }
+map [string] interface {}
+
+ +

+The number of map elements is called its length. +For a map m, it can be discovered using the +built-in function len(m) +and may change during execution. Elements may be added and removed +during execution using special forms of assignment; +and they may be accessed with index expressions. +

+

+A new, empty map value is made using the built-in +function make, +which takes the map type and an optional capacity hint as arguments: +

+ +
+make(map[string] int)
+make(map[string] int, 100)
+
+ +

+The initial capacity does not bound its size: +maps grow to accommodate the number of items +stored in them, with the exception of nil maps. +A nil map is equivalent to an empty map except that no elements +may be added. + +

Channel types

+ +

+A channel provides a mechanism for two concurrently executing functions +to synchronize execution and communicate by passing a value of a +specified element type. +The value of an uninitialized channel is nil. +

+ +
+ChannelType = ( "chan" [ "<-" ] | "<-" "chan" ) ElementType .
+
+ +

+The <- operator specifies the channel direction, +send or receive. If no direction is given, the channel is +bi-directional. +A channel may be constrained only to send or only to receive by +conversion or assignment. +

+ +
+chan T         // can be used to send and receive values of type T
+chan<- float64 // can only be used to send float64s
+<-chan int     // can only be used to receive ints
+
+ +

+The <- operator associates with the leftmost chan +possible: +

+ +
+chan<- chan int     // same as chan<- (chan int)
+chan<- <-chan int   // same as chan<- (<-chan int)
+<-chan <-chan int   // same as <-chan (<-chan int)
+chan (<-chan int)
+
+ +

+A new, initialized channel +value can be made using the built-in function +make, +which takes the channel type and an optional capacity as arguments: +

+ +
+make(chan int, 100)
+
+ +

+The capacity, in number of elements, sets the size of the buffer in the channel. If the +capacity is greater than zero, the channel is asynchronous: communication operations +succeed without blocking if the buffer is not full (sends) or not empty (receives), +and elements are received in the order they are sent. +If the capacity is zero or absent, the communication succeeds only when both a sender and +receiver are ready. +A nil channel is never ready for communication. +

+ +

+A channel may be closed with the built-in function +close; the +multi-valued assignment form of the +receive operator +tests whether a channel has been closed. +

+ +

Properties of types and values

+ +

Type identity

+ +

+Two types are either identical or different. +

+ +

+Two named types are identical if their type names originate in the same +type declaration. +A named and an unnamed type are always different. Two unnamed types are identical +if the corresponding type literals are identical, that is, if they have the same +literal structure and corresponding components have identical types. In detail: +

+ +
    +
  • Two array types are identical if they have identical element types and + the same array length.
  • + +
  • Two slice types are identical if they have identical element types.
  • + +
  • Two struct types are identical if they have the same sequence of fields, + and if corresponding fields have the same names, and identical types, + and identical tags. + Two anonymous fields are considered to have the same name. Lower-case field + names from different packages are always different.
  • + +
  • Two pointer types are identical if they have identical base types.
  • + +
  • Two function types are identical if they have the same number of parameters + and result values, corresponding parameter and result types are + identical, and either both functions are variadic or neither is. + Parameter and result names are not required to match.
  • + +
  • Two interface types are identical if they have the same set of methods + with the same names and identical function types. Lower-case method names from + different packages are always different. The order of the methods is irrelevant.
  • + +
  • Two map types are identical if they have identical key and value types.
  • + +
  • Two channel types are identical if they have identical value types and + the same direction.
  • +
+ +

+Given the declarations +

+ +
+type (
+	T0 []string
+	T1 []string
+	T2 struct { a, b int }
+	T3 struct { a, c int }
+	T4 func(int, float64) *T0
+	T5 func(x int, y float64) *[]string
+)
+
+ +

+these types are identical: +

+ +
+T0 and T0
+[]int and []int
+struct { a, b *T5 } and struct { a, b *T5 }
+func(x int, y float64) *[]string and func(int, float64) (result *[]string)
+
+ +

+T0 and T1 are different because they are named types +with distinct declarations; func(int, float64) *T0 and +func(x int, y float64) *[]string are different because T0 +is different from []string. +

+ + +

Assignability

+ +

+A value x is assignable to a variable of type T +("x is assignable to T") in any of these cases: +

+ +
    +
  • +x's type is identical to T. +
  • +
  • +x's type V and T have identical +underlying types and at least one of V +or T is not a named type. +
  • +
  • +T is an interface type and +x implements T. +
  • +
  • +x is a bidirectional channel value, T is a channel type, +x's type V and T have identical element types, +and at least one of V or T is not a named type. +
  • +
  • +x is the predeclared identifier nil and T +is a pointer, function, slice, map, channel, or interface type. +
  • +
  • +x is an untyped constant representable +by a value of type T. +
  • +
+ +

+If T is a struct type with non-exported +fields, the assignment must be in the same package in which T is declared, +or x must be the receiver of a method call. +In other words, a struct value can be assigned to a struct variable only if +every field of the struct may be legally assigned individually by the program, +or if the assignment is initializing the receiver of a method of the struct type. +

+ +

+Any value may be assigned to the blank identifier. +

+ + +

Blocks

+ +

+A block is a sequence of declarations and statements within matching +brace brackets. +

+ +
+Block = "{" { Statement ";" } "}" .
+
+ +

+In addition to explicit blocks in the source code, there are implicit blocks: +

+ +
    +
  1. The universe block encompasses all Go source text.
  2. + +
  3. Each package has a package block containing all + Go source text for that package.
  4. + +
  5. Each file has a file block containing all Go source text + in that file.
  6. + +
  7. Each if, for, and switch + statement is considered to be in its own implicit block.
  8. + +
  9. Each clause in a switch or select statement + acts as an implicit block.
  10. +
+ +

+Blocks nest and influence scoping. +

+ + +

Declarations and scope

+ +

+A declaration binds a non-blank +identifier to a constant, type, variable, function, or package. +Every identifier in a program must be declared. +No identifier may be declared twice in the same block, and +no identifier may be declared in both the file and package block. +

+ +
+Declaration   = ConstDecl | TypeDecl | VarDecl .
+TopLevelDecl  = Declaration | FunctionDecl | MethodDecl .
+
+ +

+The scope of a declared identifier is the extent of source text in which +the identifier denotes the specified constant, type, variable, function, or package. +

+ +

+Go is lexically scoped using blocks: +

+ +
    +
  1. The scope of a predeclared identifier is the universe block.
  2. + +
  3. The scope of an identifier denoting a constant, type, variable, + or function (but not method) declared at top level (outside any + function) is the package block.
  4. + +
  5. The scope of an imported package identifier is the file block + of the file containing the import declaration.
  6. + +
  7. The scope of an identifier denoting a function parameter or + result variable is the function body.
  8. + +
  9. The scope of a constant or variable identifier declared + inside a function begins at the end of the ConstSpec or VarSpec + (ShortVarDecl for short variable declarations) + and ends at the end of the innermost containing block.
  10. + +
  11. The scope of a type identifier declared inside a function + begins at the identifier in the TypeSpec + and ends at the end of the innermost containing block.
  12. +
+ +

+An identifier declared in a block may be redeclared in an inner block. +While the identifier of the inner declaration is in scope, it denotes +the entity declared by the inner declaration. +

+ +

+The package clause is not a declaration; the package name +does not appear in any scope. Its purpose is to identify the files belonging +to the same package and to specify the default package name for import +declarations. +

+ + +

Label scopes

+ +

+Labels are declared by labeled statements and are +used in the break, continue, and goto +statements (§Break statements, §Continue statements, §Goto statements). +It is illegal to define a label that is never used. +In contrast to other identifiers, labels are not block scoped and do +not conflict with identifiers that are not labels. The scope of a label +is the body of the function in which it is declared and excludes +the body of any nested function. +

+ + +

Predeclared identifiers

+ +

+The following identifiers are implicitly declared in the universe block: +

+
+Basic types:
+	bool byte complex64 complex128 float32 float64
+	int8 int16 int32 int64 string uint8 uint16 uint32 uint64
+
+Architecture-specific convenience types:
+	int uint uintptr
+
+Constants:
+	true false iota
+
+Zero value:
+	nil
+
+Functions:
+	append cap close complex copy imag len
+	make new panic print println real recover
+
+ + +

Exported identifiers

+ +

+An identifier may be exported to permit access to it from another package +using a qualified identifier. An identifier +is exported if both: +

+
    +
  1. the first character of the identifier's name is a Unicode upper case letter (Unicode class "Lu"); and
  2. +
  3. the identifier is declared in the package block or denotes a field or method of a type + declared in that block.
  4. +
+

+All other identifiers are not exported. +

+ + +

Blank identifier

+ +

+The blank identifier, represented by the underscore character _, may be used in a declaration like +any other identifier but the declaration does not introduce a new binding. +

+ + +

Constant declarations

+ +

+A constant declaration binds a list of identifiers (the names of +the constants) to the values of a list of constant expressions. +The number of identifiers must be equal +to the number of expressions, and the nth identifier on +the left is bound to the value of the nth expression on the +right. +

+ +
+ConstDecl      = "const" ( ConstSpec | "(" { ConstSpec ";" } ")" ) .
+ConstSpec      = IdentifierList [ [ Type ] "=" ExpressionList ] .
+
+IdentifierList = identifier { "," identifier } .
+ExpressionList = Expression { "," Expression } .
+
+ +

+If the type is present, all constants take the type specified, and +the expressions must be assignable to that type. +If the type is omitted, the constants take the +individual types of the corresponding expressions. +If the expression values are untyped constants, +the declared constants remain untyped and the constant identifiers +denote the constant values. For instance, if the expression is a +floating-point literal, the constant identifier denotes a floating-point +constant, even if the literal's fractional part is zero. +

+ +
+const Pi float64 = 3.14159265358979323846
+const zero = 0.0             // untyped floating-point constant
+const (
+	size int64 = 1024
+	eof = -1             // untyped integer constant
+)
+const a, b, c = 3, 4, "foo"  // a = 3, b = 4, c = "foo", untyped integer and string constants
+const u, v float32 = 0, 3    // u = 0.0, v = 3.0
+
+ +

+Within a parenthesized const declaration list the +expression list may be omitted from any but the first declaration. +Such an empty list is equivalent to the textual substitution of the +first preceding non-empty expression list and its type if any. +Omitting the list of expressions is therefore equivalent to +repeating the previous list. The number of identifiers must be equal +to the number of expressions in the previous list. +Together with the iota constant generator +this mechanism permits light-weight declaration of sequential values: +

+ +
+const (
+	Sunday = iota
+	Monday
+	Tuesday
+	Wednesday
+	Thursday
+	Friday
+	Partyday
+	numberOfDays  // this constant is not exported
+)
+
+ + +

Iota

+ +

+Within a constant declaration, the predeclared identifier +iota represents successive untyped integer +constants. It is reset to 0 whenever the reserved word const +appears in the source and increments after each ConstSpec. +It can be used to construct a set of related constants: +

+ +
+const (  // iota is reset to 0
+	c0 = iota  // c0 == 0
+	c1 = iota  // c1 == 1
+	c2 = iota  // c2 == 2
+)
+
+const (
+	a = 1 << iota  // a == 1 (iota has been reset)
+	b = 1 << iota  // b == 2
+	c = 1 << iota  // c == 4
+)
+
+const (
+	u         = iota * 42  // u == 0     (untyped integer constant)
+	v float64 = iota * 42  // v == 42.0  (float64 constant)
+	w         = iota * 42  // w == 84    (untyped integer constant)
+)
+
+const x = iota  // x == 0 (iota has been reset)
+const y = iota  // y == 0 (iota has been reset)
+
+ +

+Within an ExpressionList, the value of each iota is the same because +it is only incremented after each ConstSpec: +

+ +
+const (
+	bit0, mask0 = 1 << iota, 1 << iota - 1  // bit0 == 1, mask0 == 0
+	bit1, mask1                             // bit1 == 2, mask1 == 1
+	_, _                                    // skips iota == 2
+	bit3, mask3                             // bit3 == 8, mask3 == 7
+)
+
+ +

+This last example exploits the implicit repetition of the +last non-empty expression list. +

+ + +

Type declarations

+ +

+A type declaration binds an identifier, the type name, to a new type +that has the same underlying type as +an existing type. The new type is different from +the existing type. +

+ +
+TypeDecl     = "type" ( TypeSpec | "(" { TypeSpec ";" } ")" ) .
+TypeSpec     = identifier Type .
+
+ +
+type IntArray [16]int
+
+type (
+	Point struct { x, y float64 }
+	Polar Point
+)
+
+type TreeNode struct {
+	left, right *TreeNode
+	value *Comparable
+}
+
+type Cipher interface {
+	BlockSize() int
+	Encrypt(src, dst []byte)
+	Decrypt(src, dst []byte)
+}
+
+ +

+The declared type does not inherit any methods +bound to the existing type, but the method set +of an interface type or of elements of a composite type remains unchanged: +

+ +
+// A Mutex is a data type with two methods, Lock and Unlock.
+type Mutex struct         { /* Mutex fields */ }
+func (m *Mutex) Lock()    { /* Lock implementation */ }
+func (m *Mutex) Unlock()  { /* Unlock implementation */ }
+
+// NewMutex has the same composition as Mutex but its method set is empty.
+type NewMutex Mutex
+
+// The method set of the base type of PtrMutex remains unchanged,
+// but the method set of PtrMutex is empty.
+type PtrMutex *Mutex
+
+// The method set of *PrintableMutex contains the methods
+// Lock and Unlock bound to its anonymous field Mutex.
+type PrintableMutex struct {
+	Mutex
+}
+
+// MyCipher is an interface type that has the same method set as Cipher.
+type MyCipher Cipher
+
+ +

+A type declaration may be used to define a different boolean, numeric, or string +type and attach methods to it: +

+ +
+type TimeZone int
+
+const (
+	EST TimeZone = -(5 + iota)
+	CST
+	MST
+	PST
+)
+
+func (tz TimeZone) String() string {
+	return fmt.Sprintf("GMT+%dh", tz)
+}
+
+ + +

Variable declarations

+ +

+A variable declaration creates a variable, binds an identifier to it and +gives it a type and optionally an initial value. +

+
+VarDecl     = "var" ( VarSpec | "(" { VarSpec ";" } ")" ) .
+VarSpec     = IdentifierList ( Type [ "=" ExpressionList ] | "=" ExpressionList ) .
+
+ +
+var i int
+var U, V, W float64
+var k = 0
+var x, y float32 = -1, -2
+var (
+	i int
+	u, v, s = 2.0, 3.0, "bar"
+)
+var re, im = complexSqrt(-1)
+var _, found = entries[name]  // map lookup; only interested in "found"
+
+ +

+If a list of expressions is given, the variables are initialized +by assigning the expressions to the variables (§Assignments) +in order; all expressions must be consumed and all variables initialized from them. +Otherwise, each variable is initialized to its zero value. +

+ +

+If the type is present, each variable is given that type. +Otherwise, the types are deduced from the assignment +of the expression list. +

+ +

+If the type is absent and the corresponding expression evaluates to an +untyped constant, the type of the declared variable +is bool, int, float64, or string +respectively, depending on whether the value is a boolean, integer, +floating-point, or string constant: +

+ +
+var b = true    // t has type bool
+var i = 0       // i has type int
+var f = 3.0     // f has type float64
+var s = "OMDB"  // s has type string
+
+ +

Short variable declarations

+ +

+A short variable declaration uses the syntax: +

+ +
+ShortVarDecl = IdentifierList ":=" ExpressionList .
+
+ +

+It is a shorthand for a regular variable declaration with +initializer expressions but no types: +

+ +
+"var" IdentifierList = ExpressionList .
+
+ +
+i, j := 0, 10
+f := func() int { return 7 }
+ch := make(chan int)
+r, w := os.Pipe(fd)  // os.Pipe() returns two values
+_, y, _ := coord(p)  // coord() returns three values; only interested in y coordinate
+
+ +

+Unlike regular variable declarations, a short variable declaration may redeclare variables provided they +were originally declared in the same block with the same type, and at +least one of the non-blank variables is new. As a consequence, redeclaration +can only appear in a multi-variable short declaration. +Redeclaration does not introduce a new +variable; it just assigns a new value to the original. +

+ +
+field1, offset := nextField(str, 0)
+field2, offset := nextField(str, offset)  // redeclares offset
+
+ +

+Short variable declarations may appear only inside functions. +In some contexts such as the initializers for if, +for, or switch statements, +they can be used to declare local temporary variables (§Statements). +

+ +

Function declarations

+ +

+A function declaration binds an identifier to a function (§Function types). +

+ +
+FunctionDecl = "func" identifier Signature [ Body ] .
+Body         = Block .
+
+ +

+A function declaration may omit the body. Such a declaration provides the +signature for a function implemented outside Go, such as an assembly routine. +

+ +
+func min(x int, y int) int {
+	if x < y {
+		return x
+	}
+	return y
+}
+
+func flushICache(begin, end uintptr)  // implemented externally
+
+ +

Method declarations

+ +

+A method is a function with a receiver. +A method declaration binds an identifier to a method. +

+
+MethodDecl   = "func" Receiver MethodName Signature [ Body ] .
+Receiver     = "(" [ identifier ] [ "*" ] BaseTypeName ")" .
+BaseTypeName = identifier .
+
+ +

+The receiver type must be of the form T or *T where +T is a type name. T is called the +receiver base type or just base type. +The base type must not be a pointer or interface type and must be +declared in the same package as the method. +The method is said to be bound to the base type +and is visible only within selectors for that type +(§Type declarations, §Selectors). +

+ +

+Given type Point, the declarations +

+ +
+func (p *Point) Length() float64 {
+	return math.Sqrt(p.x * p.x + p.y * p.y)
+}
+
+func (p *Point) Scale(factor float64) {
+	p.x *= factor
+	p.y *= factor
+}
+
+ +

+bind the methods Length and Scale, +with receiver type *Point, +to the base type Point. +

+ +

+If the receiver's value is not referenced inside the body of the method, +its identifier may be omitted in the declaration. The same applies in +general to parameters of functions and methods. +

+ +

+The type of a method is the type of a function with the receiver as first +argument. For instance, the method Scale has type +

+ +
+func(p *Point, factor float64)
+
+ +

+However, a function declared this way is not a method. +

+ + +

Expressions

+ +

+An expression specifies the computation of a value by applying +operators and functions to operands. +

+ +

Operands

+ +

+Operands denote the elementary values in an expression. +

+ +
+Operand    = Literal | QualifiedIdent | MethodExpr | "(" Expression ")" .
+Literal    = BasicLit | CompositeLit | FunctionLit .
+BasicLit   = int_lit | float_lit | imaginary_lit | char_lit | string_lit .
+
+ + +

Qualified identifiers

+ +

+A qualified identifier is a non-blank identifier qualified by a package name prefix. +

+ +
+QualifiedIdent = [ PackageName "." ] identifier .
+
+ +

+A qualified identifier accesses an identifier in a separate package. +The identifier must be exported by that +package, which means that it must begin with a Unicode upper case letter. +

+ +
+math.Sin
+
+ + + +

Composite literals

+ +

+Composite literals construct values for structs, arrays, slices, and maps +and create a new value each time they are evaluated. +They consist of the type of the value +followed by a brace-bound list of composite elements. An element may be +a single expression or a key-value pair. +

+ +
+CompositeLit  = LiteralType LiteralValue .
+LiteralType   = StructType | ArrayType | "[" "..." "]" ElementType |
+                SliceType | MapType | TypeName .
+LiteralValue  = "{" [ ElementList [ "," ] ] "}" .
+ElementList   = Element { "," Element } .
+Element       = [ Key ":" ] Value .
+Key           = FieldName | ElementIndex .
+FieldName     = identifier .
+ElementIndex  = Expression .
+Value         = Expression | LiteralValue .
+
+ +

+The LiteralType must be a struct, array, slice, or map type +(the grammar enforces this constraint except when the type is given +as a TypeName). +The types of the expressions must be assignable +to the respective field, element, and key types of the LiteralType; +there is no additional conversion. +The key is interpreted as a field name for struct literals, +an index expression for array and slice literals, and a key for map literals. +For map literals, all elements must have a key. It is an error +to specify multiple elements with the same field name or +constant key value. +

+ +

+For struct literals the following rules apply: +

+
    +
  • A key must be a field name declared in the LiteralType. +
  • +
  • A literal that does not contain any keys must + list an element for each struct field in the + order in which the fields are declared. +
  • +
  • If any element has a key, every element must have a key. +
  • +
  • A literal that contains keys does not need to + have an element for each struct field. Omitted fields + get the zero value for that field. +
  • +
  • A literal may omit the element list; such a literal evaluates + to the zero value for its type. +
  • +
  • It is an error to specify an element for a non-exported + field of a struct belonging to a different package. +
  • +
+ +

+Given the declarations +

+
+type Point3D struct { x, y, z float64 }
+type Line struct { p, q Point3D }
+
+ +

+one may write +

+ +
+origin := Point3D{}                            // zero value for Point3D
+line := Line{origin, Point3D{y: -4, z: 12.3}}  // zero value for line.q.x
+
+ +

+For array and slice literals the following rules apply: +

+
    +
  • Each element has an associated integer index marking + its position in the array. +
  • +
  • An element with a key uses the key as its index; the + key must be a constant integer expression. +
  • +
  • An element without a key uses the previous element's index plus one. + If the first element has no key, its index is zero. +
  • +
+ +

+Taking the address of a composite literal (§Address operators) +generates a pointer to a unique instance of the literal's value. +

+
+var pointer *Point3D = &Point3D{y: 1000}
+
+ +

+The length of an array literal is the length specified in the LiteralType. +If fewer elements than the length are provided in the literal, the missing +elements are set to the zero value for the array element type. +It is an error to provide elements with index values outside the index range +of the array. The notation ... specifies an array length equal +to the maximum element index plus one. +

+ +
+buffer := [10]string{}               // len(buffer) == 10
+intSet := [6]int{1, 2, 3, 5}         // len(intSet) == 6
+days := [...]string{"Sat", "Sun"}    // len(days) == 2
+
+ +

+A slice literal describes the entire underlying array literal. +Thus, the length and capacity of a slice literal are the maximum +element index plus one. A slice literal has the form +

+ +
+[]T{x1, x2, … xn}
+
+ +

+and is a shortcut for a slice operation applied to an array literal: +

+ +
+[n]T{x1, x2, … xn}[0 : n]
+
+ +

+Within a composite literal of array, slice, or map type T, +elements that are themselves composite literals may elide the respective +literal type if it is identical to the element type of T. +

+ +
+[...]Point{{1.5, -3.5}, {0, 0}}  // same as [...]Point{Point{1.5, -3.5}, Point{0, 0}}
+[][]int{{1, 2, 3}, {4, 5}}       // same as [][]int{[]int{1, 2, 3}, []int{4, 5}}
+
+ +

+A parsing ambiguity arises when a composite literal using the +TypeName form of the LiteralType appears between the +keyword and the opening brace of the block of an +"if", "for", or "switch" statement, because the braces surrounding +the expressions in the literal are confused with those introducing +the block of statements. To resolve the ambiguity in this rare case, +the composite literal must appear within +parentheses. +

+ +
+if x == (T{a,b,c}[i]) { … }
+if (x == T{a,b,c}[i]) { … }
+
+ +

+Examples of valid array, slice, and map literals: +

+ +
+// list of prime numbers
+primes := []int{2, 3, 5, 7, 9, 11, 13, 17, 19, 991}
+
+// vowels[ch] is true if ch is a vowel
+vowels := [128]bool{'a': true, 'e': true, 'i': true, 'o': true, 'u': true, 'y': true}
+
+// the array [10]float32{-1, 0, 0, 0, -0.1, -0.1, 0, 0, 0, -1}
+filter := [10]float32{-1, 4: -0.1, -0.1, 9: -1}
+
+// frequencies in Hz for equal-tempered scale (A4 = 440Hz)
+noteFrequency := map[string]float32{
+	"C0": 16.35, "D0": 18.35, "E0": 20.60, "F0": 21.83,
+	"G0": 24.50, "A0": 27.50, "B0": 30.87,
+}
+
+ + +

Function literals

+ +

+A function literal represents an anonymous function. +It consists of a specification of the function type and a function body. +

+ +
+FunctionLit = FunctionType Body .
+
+ +
+func(a, b int, z float64) bool { return a*b < int(z) }
+
+ +

+A function literal can be assigned to a variable or invoked directly. +

+ +
+f := func(x, y int) int { return x + y }
+func(ch chan int) { ch <- ACK } (reply_chan)
+
+ +

+Function literals are closures: they may refer to variables +defined in a surrounding function. Those variables are then shared between +the surrounding function and the function literal, and they survive as long +as they are accessible. +

+ + +

Primary expressions

+ +

+Primary expressions are the operands for unary and binary expressions. +

+ +
+PrimaryExpr =
+	Operand |
+	Conversion |
+	BuiltinCall |
+	PrimaryExpr Selector |
+	PrimaryExpr Index |
+	PrimaryExpr Slice |
+	PrimaryExpr TypeAssertion |
+	PrimaryExpr Call .
+
+Selector       = "." identifier .
+Index          = "[" Expression "]" .
+Slice          = "[" [ Expression ] ":" [ Expression ] "]" .
+TypeAssertion  = "." "(" Type ")" .
+Call           = "(" [ ArgumentList [ "," ] ] ")" .
+ArgumentList   = ExpressionList [ "..." ] .
+
+ + +
+x
+2
+(s + ".txt")
+f(3.1415, true)
+Point{1, 2}
+m["foo"]
+s[i : j + 1]
+obj.color
+math.Sin
+f.p[i].x()
+
+ + +

Selectors

+ +

+A primary expression of the form +

+ +
+x.f
+
+ +

+denotes the field or method f of the value denoted by x +(or sometimes *x; see below). The identifier f +is called the (field or method) +selector; it must not be the blank identifier. +The type of the expression is the type of f. +

+

+A selector f may denote a field or method f of +a type T, or it may refer +to a field or method f of a nested anonymous field of +T. +The number of anonymous fields traversed +to reach f is called its depth in T. +The depth of a field or method f +declared in T is zero. +The depth of a field or method f declared in +an anonymous field A in T is the +depth of f in A plus one. +

+

+The following rules apply to selectors: +

+
    +
  1. +For a value x of type T or *T +where T is not an interface type, +x.f denotes the field or method at the shallowest depth +in T where there +is such an f. +If there is not exactly one f with shallowest depth, the selector +expression is illegal. +
  2. +
  3. +For a variable x of type I +where I is an interface type, +x.f denotes the actual method with name f of the value assigned +to x if there is such a method. +If no value or nil was assigned to x, x.f is illegal. +
  4. +
  5. +In all other cases, x.f is illegal. +
  6. +
+

+Selectors automatically dereference pointers to structs. +If x is a pointer to a struct, x.y +is shorthand for (*x).y; if the field y +is also a pointer to a struct, x.y.z is shorthand +for (*(*x).y).z, and so on. +If x contains an anonymous field of type *A, +where A is also a struct type, +x.f is a shortcut for (*x.A).f. +

+

+For example, given the declarations: +

+ +
+type T0 struct {
+	x int
+}
+
+func (recv *T0) M0()
+
+type T1 struct {
+	y int
+}
+
+func (recv T1) M1()
+
+type T2 struct {
+	z int
+	T1
+	*T0
+}
+
+func (recv *T2) M2()
+
+var p *T2  // with p != nil and p.T1 != nil
+
+ +

+one may write: +

+ +
+p.z         // (*p).z
+p.y         // ((*p).T1).y
+p.x         // (*(*p).T0).x
+
+p.M2        // (*p).M2
+p.M1        // ((*p).T1).M1
+p.M0        // ((*p).T0).M0
+
+ + + + + +

Indexes

+ +

+A primary expression of the form +

+ +
+a[x]
+
+ +

+denotes the element of the array, slice, string or map a indexed by x. +The value x is called the +index or map key, respectively. The following +rules apply: +

+ +

+For a of type A or *A +where A is an array type, +or for a of type S where S is a slice type: +

+
    +
  • x must be an integer value and 0 <= x < len(a)
  • +
  • a[x] is the array element at index x and the type of + a[x] is the element type of A
  • +
  • if a is nil or if the index x is out of range, + a run-time panic occurs
  • +
+ +

+For a of type T +where T is a string type: +

+
    +
  • x must be an integer value and 0 <= x < len(a)
  • +
  • a[x] is the byte at index x and the type of + a[x] is byte
  • +
  • a[x] may not be assigned to
  • +
  • if the index x is out of range, + a run-time panic occurs
  • +
+ +

+For a of type M +where M is a map type: +

+
    +
  • x's type must be + assignable + to the key type of M
  • +
  • if the map contains an entry with key x, + a[x] is the map value with key x + and the type of a[x] is the value type of M
  • +
  • if the map is nil or does not contain such an entry, + a[x] is the zero value + for the value type of M
  • +
+ +

+Otherwise a[x] is illegal. +

+ +

+An index expression on a map a of type map[K]V +may be used in an assignment or initialization of the special form +

+ +
+v, ok = a[x]
+v, ok := a[x]
+var v, ok = a[x]
+
+ +

+where the result of the index expression is a pair of values with types +(V, bool). In this form, the value of ok is +true if the key x is present in the map, and +false otherwise. The value of v is the value +a[x] as in the single-result form. +

+ +

+Similarly, if an assignment to a map element has the special form +

+ +
+a[x] = v, ok
+
+ +

+and boolean ok has the value false, +the entry for key x is deleted from the map; if +ok is true, the construct acts like +a regular assignment to an element of the map. +

+ +

+Assigning to an element of a nil map causes a +run-time panic. +

+ + +

Slices

+ +

+For a string, array, or slice a, the primary expression +

+ +
+a[low : high]
+
+ +

+constructs a substring or slice. The index expressions low and +high select which elements appear in the result. The result has +indexes starting at 0 and length equal to +high - low. +After slicing the array a +

+ +
+a := [5]int{1, 2, 3, 4, 5}
+s := a[1:4]
+
+ +

+the slice s has type []int, length 3, capacity 4, and elements +

+ +
+s[0] == 2
+s[1] == 3
+s[2] == 4
+
+ +

+For convenience, any of the index expressions may be omitted. A missing low +index defaults to zero; a missing high index defaults to the length of the +sliced operand: +

+ +
+a[2:]	// same a[2 : len(a)]
+a[:3]   // same as a[0 : 3]
+a[:]    // same as a[0 : len(a)]
+
+ +

+For arrays or strings, the indexes low and high must +satisfy 0 <= low <= high <= length; for +slices, the upper bound is the capacity rather than the length. +

+ +

+If the sliced operand is a string or slice, the result of the slice operation +is a string or slice of the same type. +If the sliced operand is an array, it must be addressable +and the result of the slice operation is a slice with the same element type as the array. +

+ + +

Type assertions

+ +

+For an expression x of interface type +and a type T, the primary expression +

+ +
+x.(T)
+
+ +

+asserts that x is not nil +and that the value stored in x is of type T. +The notation x.(T) is called a type assertion. +

+

+More precisely, if T is not an interface type, x.(T) asserts +that the dynamic type of x is identical +to the type T. +If T is an interface type, x.(T) asserts that the dynamic type +of x implements the interface TInterface types). +

+

+If the type assertion holds, the value of the expression is the value +stored in x and its type is T. If the type assertion is false, +a run-time panic occurs. +In other words, even though the dynamic type of x +is known only at run-time, the type of x.(T) is +known to be T in a correct program. +

+

+If a type assertion is used in an assignment or initialization of the form +

+ +
+v, ok = x.(T)
+v, ok := x.(T)
+var v, ok = x.(T)
+
+ +

+the result of the assertion is a pair of values with types (T, bool). +If the assertion holds, the expression returns the pair (x.(T), true); +otherwise, the expression returns (Z, false) where Z +is the zero value for type T. +No run-time panic occurs in this case. +The type assertion in this construct thus acts like a function call +returning a value and a boolean indicating success. (§Assignments) +

+ + +

Calls

+ +

+Given an expression f of function type +F, +

+ +
+f(a1, a2, … an)
+
+ +

+calls f with arguments a1, a2, … an. +Except for one special case, arguments must be single-valued expressions +assignable to the parameter types of +F and are evaluated before the function is called. +The type of the expression is the result type +of F. +A method invocation is similar but the method itself +is specified as a selector upon a value of the receiver type for +the method. +

+ +
+math.Atan2(x, y)    // function call
+var pt *Point
+pt.Scale(3.5)  // method call with receiver pt
+
+ +

+As a special case, if the return parameters of a function or method +g are equal in number and individually +assignable to the parameters of another function or method +f, then the call f(g(parameters_of_g)) +will invoke f after binding the return values of +g to the parameters of f in order. The call +of f must contain no parameters other than the call of g. +If f has a final ... parameter, it is +assigned the return values of g that remain after +assignment of regular parameters. +

+ +
+func Split(s string, pos int) (string, string) {
+	return s[0:pos], s[pos:]
+}
+
+func Join(s, t string) string {
+	return s + t
+}
+
+if Join(Split(value, len(value)/2)) != value {
+	log.Panic("test fails")
+}
+
+ +

+A method call x.m() is valid if the method set +of (the type of) x contains m and the +argument list can be assigned to the parameter list of m. +If x is addressable and &x's method +set contains m, x.m() is shorthand +for (&x).m(): +

+ +
+var p Point
+p.Scale(3.5)
+
+ +

+There is no distinct method type and there are no method literals. +

+ +

Passing arguments to ... parameters

+ +

+If f is variadic with final parameter type ...T, +then within the function the argument is equivalent to a parameter of type +[]T. At each call of f, the argument +passed to the final parameter is +a new slice of type []T whose successive elements are +the actual arguments, which all must be assignable +to the type T. The length of the slice is therefore the number of +arguments bound to the final parameter and may differ for each call site. +

+ +

+Given the function and call +

+
+func Greeting(prefix string, who ...string)
+Greeting("hello:", "Joe", "Anna", "Eileen")
+
+ +

+within Greeting, who will have the value +[]string{"Joe", "Anna", "Eileen"} +

+ +

+If the final argument is assignable to a slice type []T, it may be +passed unchanged as the value for a ...T parameter if the argument +is followed by .... In this case no new slice is created. +

+ +

+Given the slice s and call +

+ +
+s := []string{"James", "Jasmine"}
+Greeting("goodbye:", s...)
+
+ +

+within Greeting, who will have the same value as s +with the same underlying array. +

+ + +

Operators

+ +

+Operators combine operands into expressions. +

+ +
+Expression = UnaryExpr | Expression binary_op UnaryExpr .
+UnaryExpr  = PrimaryExpr | unary_op UnaryExpr .
+
+binary_op  = "||" | "&&" | rel_op | add_op | mul_op .
+rel_op     = "==" | "!=" | "<" | "<=" | ">" | ">=" .
+add_op     = "+" | "-" | "|" | "^" .
+mul_op     = "*" | "/" | "%" | "<<" | ">>" | "&" | "&^" .
+
+unary_op   = "+" | "-" | "!" | "^" | "*" | "&" | "<-" .
+
+ +

+Comparisons are discussed elsewhere. +For other binary operators, the operand types must be identical +unless the operation involves shifts or untyped constants. +For operations involving constants only, see the section on +constant expressions. +

+ +

+Except for shift operations, if one operand is an untyped constant +and the other operand is not, the constant is converted +to the type of the other operand. +

+ +

+The right operand in a shift expression must have unsigned integer type +or be an untyped constant that can be converted to unsigned integer type. +If the left operand of a non-constant shift expression is an untyped constant, +the type of the constant is what it would be if the shift expression were +replaced by its left operand alone; the type is int if it cannot +be determined from the context (for instance, if the shift expression is an +operand in a comparison against an untyped constant). +

+ +
+var s uint = 33
+var i = 1<<s           // 1 has type int
+var j int32 = 1<<s     // 1 has type int32; j == 0
+var k = uint64(1<<s)   // 1 has type uint64; k == 1<<33
+var m int = 1.0<<s     // legal: 1.0 has type int
+var n = 1.0<<s != 0    // legal: 1.0 has type int; n == false if ints are 32bits in size
+var o = 1<<s == 2<<s   // legal: 1 and 2 have type int; o == true if ints are 32bits in size
+var p = 1<<s == 1<<33  // illegal if ints are 32bits in size: 1 has type int, but 1<<33 overflows int
+var u = 1.0<<s         // illegal: 1.0 has type float64, cannot shift
+var v float32 = 1<<s   // illegal: 1 has type float32, cannot shift
+var w int64 = 1.0<<33  // legal: 1.0<<33 is a constant shift expression
+
+ +

Operator precedence

+

+Unary operators have the highest precedence. +As the ++ and -- operators form +statements, not expressions, they fall +outside the operator hierarchy. +As a consequence, statement *p++ is the same as (*p)++. +

+There are five precedence levels for binary operators. +Multiplication operators bind strongest, followed by addition +operators, comparison operators, && (logical and), +and finally || (logical or): +

+ +
+Precedence    Operator
+    5             *  /  %  <<  >>  &  &^
+    4             +  -  |  ^
+    3             ==  !=  <  <=  >  >=
+    2             &&
+    1             ||
+
+ +

+Binary operators of the same precedence associate from left to right. +For instance, x / y * z is the same as (x / y) * z. +

+ +
++x
+23 + 3*x[i]
+x <= f()
+^a >> b
+f() || g()
+x == y+1 && <-chan_ptr > 0
+
+ + +

Arithmetic operators

+

+Arithmetic operators apply to numeric values and yield a result of the same +type as the first operand. The four standard arithmetic operators (+, +-, *, /) apply to integer, +floating-point, and complex types; + also applies +to strings. All other arithmetic operators apply to integers only. +

+ +
++    sum                    integers, floats, complex values, strings
+-    difference             integers, floats, complex values
+*    product                integers, floats, complex values
+/    quotient               integers, floats, complex values
+%    remainder              integers
+
+&    bitwise and            integers
+|    bitwise or             integers
+^    bitwise xor            integers
+&^   bit clear (and not)    integers
+
+<<   left shift             integer << unsigned integer
+>>   right shift            integer >> unsigned integer
+
+ +

+Strings can be concatenated using the + operator +or the += assignment operator: +

+ +
+s := "hi" + string(c)
+s += " and good bye"
+
+ +

+String addition creates a new string by concatenating the operands. +

+

+For two integer values x and y, the integer quotient +q = x / y and remainder r = x % y satisfy the following +relationships: +

+ +
+x = q*y + r  and  |r| < |y|
+
+ +

+with x / y truncated towards zero +("truncated division"). +

+ +
+ x     y     x / y     x % y
+ 5     3       1         2
+-5     3      -1        -2
+ 5    -3      -1         2
+-5    -3       1        -2
+
+ +

+As an exception to this rule, if the dividend x is the most +negative value for the int type of x, the quotient +q = x / -1 is equal to x (and r = 0). +

+ +
+			 x, q
+int8                     -128
+int16                  -32768
+int32             -2147483648
+int64    -9223372036854775808
+
+ +

+If the divisor is zero, a run-time panic occurs. +If the dividend is positive and the divisor is a constant power of 2, +the division may be replaced by a right shift, and computing the remainder may +be replaced by a bitwise "and" operation: +

+ +
+ x     x / 4     x % 4     x >> 2     x & 3
+ 11      2         3         2          3
+-11     -2        -3        -3          1
+
+ +

+The shift operators shift the left operand by the shift count specified by the +right operand. They implement arithmetic shifts if the left operand is a signed +integer and logical shifts if it is an unsigned integer. +There is no upper limit on the shift count. Shifts behave +as if the left operand is shifted n times by 1 for a shift +count of n. +As a result, x << 1 is the same as x*2 +and x >> 1 is the same as +x/2 but truncated towards negative infinity. +

+ +

+For integer operands, the unary operators ++, -, and ^ are defined as +follows: +

+ +
++x                          is 0 + x
+-x    negation              is 0 - x
+^x    bitwise complement    is m ^ x  with m = "all bits set to 1" for unsigned x
+                                      and  m = -1 for signed x
+
+ +

+For floating-point numbers, ++x is the same as x, +while -x is the negation of x. +The result of a floating-point division by zero is not specified beyond the +IEEE-754 standard; whether a run-time panic +occurs is implementation-specific. +

+ +

Integer overflow

+ +

+For unsigned integer values, the operations +, +-, *, and << are +computed modulo 2n, where n is the bit width of +the unsigned integer's type +(§Numeric types). Loosely speaking, these unsigned integer operations +discard high bits upon overflow, and programs may rely on ``wrap around''. +

+

+For signed integers, the operations +, +-, *, and << may legally +overflow and the resulting value exists and is deterministically defined +by the signed integer representation, the operation, and its operands. +No exception is raised as a result of overflow. A +compiler may not optimize code under the assumption that overflow does +not occur. For instance, it may not assume that x < x + 1 is always true. +

+ + +

Comparison operators

+ +

+Comparison operators compare two operands and yield a value of type bool. +

+ +
+==    equal
+!=    not equal
+<     less
+<=    less or equal
+>     greater
+>=    greater or equal
+
+ +

+The operands must be comparable; that is, the first operand +must be assignable +to the type of the second operand, or vice versa. +

+

+The operators == and != apply +to operands of all types except arrays and structs. +All other comparison operators apply only to integer, floating-point +and string values. The result of a comparison is defined as follows: +

+ +
    +
  • + Integer values are compared in the usual way. +
  • +
  • + Floating point values are compared as defined by the IEEE-754 + standard. +
  • +
  • + Two complex values u, v are + equal if both real(u) == real(v) and + imag(u) == imag(v). +
  • +
  • + String values are compared byte-wise (lexically). +
  • +
  • + Boolean values are equal if they are either both + true or both false. +
  • +
  • + Pointer values are equal if they point to the same location + or if both are nil. +
  • +
  • + Function values are equal if they refer to the same function + or if both are nil. +
  • +
  • + A slice value may only be compared to nil. +
  • +
  • + Channel and map values are equal if they were created by the same call to make + (§Making slices, maps, and channels) + or if both are nil. +
  • +
  • + Interface values are equal if they have identical dynamic types and + equal dynamic values or if both are nil. +
  • +
  • + An interface value x is equal to a non-interface value + y if the dynamic type of x is identical to + the static type of y and the dynamic value of x + is equal to y. +
  • +
  • + A pointer, function, slice, channel, map, or interface value is equal + to nil if it has been assigned the explicit value + nil, if it is uninitialized, or if it has been assigned + another value equal to nil. +
  • +
+ + +

Logical operators

+ +

+Logical operators apply to boolean values +and yield a result of the same type as the operands. +The right operand is evaluated conditionally. +

+ +
+&&    conditional and    p && q  is  "if p then q else false"
+||    conditional or     p || q  is  "if p then true else q"
+!     not                !p      is  "not p"
+
+ + +

Address operators

+ +

+For an operand x of type T, the address operation +&x generates a pointer of type *T to x. +The operand must be addressable, +that is, either a variable, pointer indirection, or slice indexing +operation; or a field selector of an addressable struct operand; +or an array indexing operation of an addressable array. +As an exception to the addressability requirement, x may also be a +composite literal. +

+

+For an operand x of pointer type *T, the pointer +indirection *x denotes the value of type T pointed +to by x. +

+ +
+&x
+&a[f(2)]
+*p
+*pf(x)
+
+ + +

Receive operator

+ +

+For an operand ch of channel type, +the value of the receive operation <-ch is the value received +from the channel ch. The type of the value is the element type of +the channel. The expression blocks until a value is available. +Receiving from a nil channel blocks forever. +

+ +
+v1 := <-ch
+v2 = <-ch
+f(<-ch)
+<-strobe  // wait until clock pulse and discard received value
+
+ +

+A receive expression used in an assignment or initialization of the form +

+ +
+x, ok = <-ch
+x, ok := <-ch
+var x, ok = <-ch
+
+ +

+yields an additional result. +The boolean variable ok indicates whether +the received value was sent on the channel (true) +or is a zero value returned +because the channel is closed and empty (false). +

+ + + + +

Method expressions

+ +

+If M is in the method set of type T, +T.M is a function that is callable as a regular function +with the same arguments as M prefixed by an additional +argument that is the receiver of the method. +

+ +
+MethodExpr    = ReceiverType "." MethodName .
+ReceiverType  = TypeName | "(" "*" TypeName ")" .
+
+ +

+Consider a struct type T with two methods, +Mv, whose receiver is of type T, and +Mp, whose receiver is of type *T. +

+ +
+type T struct {
+	a int
+}
+func (tv  T) Mv(a int)     int     { return 0 }  // value receiver
+func (tp *T) Mp(f float32) float32 { return 1 }  // pointer receiver
+var t T
+
+ +

+The expression +

+ +
+T.Mv
+
+ +

+yields a function equivalent to Mv but +with an explicit receiver as its first argument; it has signature +

+ +
+func(tv T, a int) int
+
+ +

+That function may be called normally with an explicit receiver, so +these three invocations are equivalent: +

+ +
+t.Mv(7)
+T.Mv(t, 7)
+f := T.Mv; f(t, 7)
+
+ +

+Similarly, the expression +

+ +
+(*T).Mp
+
+ +

+yields a function value representing Mp with signature +

+ +
+func(tp *T, f float32) float32
+
+ +

+For a method with a value receiver, one can derive a function +with an explicit pointer receiver, so +

+ +
+(*T).Mv
+
+ +

+yields a function value representing Mv with signature +

+ +
+func(tv *T, a int) int
+
+ +

+Such a function indirects through the receiver to create a value +to pass as the receiver to the underlying method; +the method does not overwrite the value whose address is passed in +the function call. +

+ +

+The final case, a value-receiver function for a pointer-receiver method, +is illegal because pointer-receiver methods are not in the method set +of the value type. +

+ +

+Function values derived from methods are called with function call syntax; +the receiver is provided as the first argument to the call. +That is, given f := T.Mv, f is invoked +as f(t, 7) not t.f(7). +To construct a function that binds the receiver, use a +closure. +

+ +

+It is legal to derive a function value from a method of an interface type. +The resulting function takes an explicit receiver of that interface type. +

+ +

Conversions

+ +

+Conversions are expressions of the form T(x) +where T is a type and x is an expression +that can be converted to type T. +

+ +
+Conversion = Type "(" Expression ")" .
+
+ +

+If the type starts with an operator it must be parenthesized: +

+ +
+*Point(p)        // same as *(Point(p))
+(*Point)(p)      // p is converted to (*Point)
+<-chan int(c)    // same as <-(chan int(c))
+(<-chan int)(c)  // c is converted to (<-chan int)
+
+ +

+A constant value x can be converted to +type T in any of these cases: +

+ + + +

+Converting a constant yields a typed constant as result. +

+ +
+uint(iota)               // iota value of type uint
+float32(2.718281828)     // 2.718281828 of type float32
+complex128(1)            // 1.0 + 0.0i of type complex128
+string('x')              // "x" of type string
+string(0x266c)           // "♬" of type string
+MyString("foo" + "bar")  // "foobar" of type MyString
+string([]byte{'a'})      // not a constant: []byte{'a'} is not a constant
+(*int)(nil)              // not a constant: nil is not a constant, *int is not a boolean, numeric, or string type
+int(1.2)                 // illegal: 1.2 cannot be represented as an int
+string(65.0)             // illegal: 65.0 is not an integer constant
+
+ +

+A non-constant value x can be converted to type T +in any of these cases: +

+ +
    +
  • + x is assignable + to T. +
  • +
  • + x's type and T have identical + underlying types. +
  • +
  • + x's type and T are unnamed pointer types + and their pointer base types have identical underlying types. +
  • +
  • + x's type and T are both integer or floating + point types. +
  • +
  • + x's type and T are both complex types. +
  • +
  • + x is an integer or has type []byte or + []int and T is a string type. +
  • +
  • + x is a string and T is []byte or + []int. +
  • +
+ +

+Specific rules apply to (non-constant) conversions between numeric types or +to and from a string type. +These conversions may change the representation of x +and incur a run-time cost. +All other conversions only change the type but not the representation +of x. +

+ +

+There is no linguistic mechanism to convert between pointers and integers. +The package unsafe +implements this functionality under +restricted circumstances. +

+ +

Conversions between numeric types

+ +

+For the conversion of non-constant numeric values, the following rules apply: +

+ +
    +
  1. +When converting between integer types, if the value is a signed integer, it is +sign extended to implicit infinite precision; otherwise it is zero extended. +It is then truncated to fit in the result type's size. +For example, if v := uint16(0x10F0), then uint32(int8(v)) == 0xFFFFFFF0. +The conversion always yields a valid value; there is no indication of overflow. +
  2. +
  3. +When converting a floating-point number to an integer, the fraction is discarded +(truncation towards zero). +
  4. +
  5. +When converting an integer or floating-point number to a floating-point type, +or a complex number to another complex type, the result value is rounded +to the precision specified by the destination type. +For instance, the value of a variable x of type float32 +may be stored using additional precision beyond that of an IEEE-754 32-bit number, +but float32(x) represents the result of rounding x's value to +32-bit precision. Similarly, x + 0.1 may use more than 32 bits +of precision, but float32(x + 0.1) does not. +
  6. +
+ +

+In all non-constant conversions involving floating-point or complex values, +if the result type cannot represent the value the conversion +succeeds but the result value is implementation-dependent. +

+ +

Conversions to and from a string type

+ +
    +
  1. +Converting a signed or unsigned integer value to a string type yields a +string containing the UTF-8 representation of the integer. Values outside +the range of valid Unicode code points are converted to "\uFFFD". + +
    +string('a')           // "a"
    +string(-1)            // "\ufffd" == "\xef\xbf\xbd "
    +string(0xf8)          // "\u00f8" == "ø" == "\xc3\xb8"
    +type MyString string
    +MyString(0x65e5)      // "\u65e5" == "日" == "\xe6\x97\xa5"
    +
    +
  2. + +
  3. +Converting a value of type []byte (or +the equivalent []uint8) to a string type yields a +string whose successive bytes are the elements of the slice. If +the slice value is nil, the result is the empty string. + +
    +string([]byte{'h', 'e', 'l', 'l', '\xc3', '\xb8'})  // "hellø"
    +
    +
  4. + +
  5. +Converting a value of type []int to a string type yields +a string that is the concatenation of the individual integers +converted to strings. If the slice value is nil, the +result is the empty string. +
    +string([]int{0x767d, 0x9d6c, 0x7fd4})  // "\u767d\u9d6c\u7fd4" == "白鵬翔"
    +
    +
  6. + +
  7. +Converting a value of a string type to []byte (or []uint8) +yields a slice whose successive elements are the bytes of the string. +If the string is empty, the result is []byte(nil). + +
    +[]byte("hellø")  // []byte{'h', 'e', 'l', 'l', '\xc3', '\xb8'}
    +
    +
  8. + +
  9. +Converting a value of a string type to []int yields a +slice containing the individual Unicode code points of the string. +If the string is empty, the result is []int(nil). +
    +[]int(MyString("白鵬翔"))  // []int{0x767d, 0x9d6c, 0x7fd4}
    +
    +
  10. +
+ + +

Constant expressions

+ +

+Constant expressions may contain only constant +operands and are evaluated at compile-time. +

+ +

+Untyped boolean, numeric, and string constants may be used as operands +wherever it is legal to use an operand of boolean, numeric, or string type, +respectively. Except for shift operations, if the operands of a binary operation +are an untyped integer constant and an untyped floating-point constant, +the integer constant is converted to an untyped floating-point constant +(relevant for / and %). +Similarly, untyped integer or floating-point constants may be used as operands +wherever it is legal to use an operand of complex type; +the integer or floating point constant is converted to a +complex constant with a zero imaginary part. +

+ +

+A constant comparison always yields +a constant of type bool. If the left operand of a constant +shift expression is an untyped constant, the +result is an integer constant; otherwise it is a constant of the same +type as the left operand, which must be of integer type +(§Arithmetic operators). +Applying all other operators to untyped constants results in an untyped +constant of the same kind (that is, a boolean, integer, floating-point, +complex, or string constant). +

+ +
+const a = 2 + 3.0          // a == 5.0   (floating-point constant)
+const b = 15 / 4           // b == 3     (integer constant)
+const c = 15 / 4.0         // c == 3.75  (floating-point constant)
+const d = 1 << 3.0         // d == 8     (integer constant)
+const e = 1.0 << 3         // e == 8     (integer constant)
+const f = int32(1) << 33   // f == 0     (type int32)
+const g = float64(2) >> 1  // illegal    (float64(2) is a typed floating-point constant)
+const h = "foo" > "bar"    // h == true  (type bool)
+
+ +

+Imaginary literals are untyped complex constants (with zero real part) +and may be combined in binary +operations with untyped integer and floating-point constants; the +result is an untyped complex constant. +Complex constants are always constructed from +constant expressions involving imaginary +literals or constants derived from them, or calls of the built-in function +complex. +

+ +
+const Σ = 1 - 0.707i
+const Δ = Σ + 2.0e-4 - 1/1i
+const Φ = iota * 1i
+const iΓ = complex(0, Γ)
+
+ +

+Constant expressions are always evaluated exactly; intermediate values and the +constants themselves may require precision significantly larger than supported +by any predeclared type in the language. The following are legal declarations: +

+ +
+const Huge = 1 << 100
+const Four int8 = Huge >> 98
+
+ +

+The values of typed constants must always be accurately representable as values +of the constant type. The following constant expressions are illegal: +

+ +
+uint(-1)       // -1 cannot be represented as a uint
+int(3.14)      // 3.14 cannot be represented as an int
+int64(Huge)    // 1<<100 cannot be represented as an int64
+Four * 300     // 300 cannot be represented as an int8
+Four * 100     // 400 cannot be represented as an int8
+
+ +

+The mask used by the unary bitwise complement operator ^ matches +the rule for non-constants: the mask is all 1s for unsigned constants +and -1 for signed and untyped constants. +

+ +
+^1          // untyped integer constant, equal to -2
+uint8(^1)   // error, same as uint8(-2), out of range
+^uint8(1)   // typed uint8 constant, same as 0xFF ^ uint8(1) = uint8(0xFE)
+int8(^1)    // same as int8(-2)
+^int8(1)    // same as -1 ^ int8(1) = -2
+
+ + + +

Order of evaluation

+ +

+When evaluating the elements of an assignment or expression, +all function calls, method calls and +communication operations are evaluated in lexical left-to-right +order. +

+ +

+For example, in the assignment +

+
+y[f()], ok = g(h(), i() + x[j()], <-c), k()
+
+

+the function calls and communication happen in the order +f(), h(), i(), j(), +<-c, g(), and k(). +However, the order of those events compared to the evaluation +and indexing of x and the evaluation +of y is not specified. +

+ +

+Floating-point operations within a single expression are evaluated according to +the associativity of the operators. Explicit parentheses affect the evaluation +by overriding the default associativity. +In the expression x + (y + z) the addition y + z +is performed before adding x. +

+ +

Statements

+ +

+Statements control execution. +

+ +
+Statement =
+	Declaration | LabeledStmt | SimpleStmt |
+	GoStmt | ReturnStmt | BreakStmt | ContinueStmt | GotoStmt |
+	FallthroughStmt | Block | IfStmt | SwitchStmt | SelectStmt | ForStmt |
+	DeferStmt .
+
+SimpleStmt = EmptyStmt | ExpressionStmt | SendStmt | IncDecStmt | Assignment | ShortVarDecl .
+
+ + +

Empty statements

+ +

+The empty statement does nothing. +

+ +
+EmptyStmt = .
+
+ + +

Labeled statements

+ +

+A labeled statement may be the target of a goto, +break or continue statement. +

+ +
+LabeledStmt = Label ":" Statement .
+Label       = identifier .
+
+ +
+Error: log.Panic("error encountered")
+
+ + +

Expression statements

+ +

+Function calls, method calls, and receive operations +can appear in statement context. Such statements may be parenthesized. +

+ +
+ExpressionStmt = Expression .
+
+ +
+h(x+y)
+f.Close()
+<-ch
+(<-ch)
+
+ + +

Send statements

+ +

+A send statement sends a value on a channel. +The channel expression must be of channel type +and the type of the value must be assignable +to the channel's element type. +

+ +
+SendStmt = Channel "<-" Expression .
+Channel  = Expression .
+
+ +

+Both the channel and the value expression are evaluated before communication +begins. Communication blocks until the send can proceed, at which point the +value is transmitted on the channel. +A send on an unbuffered channel can proceed if a receiver is ready. +A send on a buffered channel can proceed if there is room in the buffer. +A send on a nil channel blocks forever. +

+ +
+ch <- 3
+
+ + +

IncDec statements

+ +

+The "++" and "--" statements increment or decrement their operands +by the untyped constant 1. +As with an assignment, the operand must be addressable +or a map index expression. +

+ +
+IncDecStmt = Expression ( "++" | "--" ) .
+
+ +

+The following assignment statements are semantically +equivalent: +

+ +
+IncDec statement    Assignment
+x++                 x += 1
+x--                 x -= 1
+
+ + +

Assignments

+ +
+Assignment = ExpressionList assign_op ExpressionList .
+
+assign_op = [ add_op | mul_op ] "=" .
+
+ +

+Each left-hand side operand must be addressable, +a map index expression, or the blank identifier. +Operands may be parenthesized. +

+ +
+x = 1
+*p = f()
+a[i] = 23
+(k) = <-ch  // same as: k = <-ch
+
+ +

+An assignment operation x op= +y where op is a binary arithmetic operation is equivalent +to x = x op +y but evaluates x +only once. The op= construct is a single token. +In assignment operations, both the left- and right-hand expression lists +must contain exactly one single-valued expression. +

+ +
+a[i] <<= 2
+i &^= 1<<n
+
+ +

+A tuple assignment assigns the individual elements of a multi-valued +operation to a list of variables. There are two forms. In the +first, the right hand operand is a single multi-valued expression +such as a function evaluation or channel or +map operation or a type assertion. +The number of operands on the left +hand side must match the number of values. For instance, if +f is a function returning two values, +

+ +
+x, y = f()
+
+ +

+assigns the first value to x and the second to y. +The blank identifier provides a +way to ignore values returned by a multi-valued expression: +

+ +
+x, _ = f()  // ignore second value returned by f()
+
+ +

+In the second form, the number of operands on the left must equal the number +of expressions on the right, each of which must be single-valued, and the +nth expression on the right is assigned to the nth +operand on the left. +The expressions on the right are evaluated before assigning to +any of the operands on the left, but otherwise the evaluation +order is unspecified beyond the usual rules. +

+ +
+a, b = b, a  // exchange a and b
+
+ +

+In assignments, each value must be +assignable to the type of the +operand to which it is assigned. If an untyped constant +is assigned to a variable of interface type, the constant is converted +to type bool, int, float64, +complex128 or string +respectively, depending on whether the value is a boolean, integer, floating-point, +complex, or string constant. +

+ + +

If statements

+ +

+"If" statements specify the conditional execution of two branches +according to the value of a boolean expression. If the expression +evaluates to true, the "if" branch is executed, otherwise, if +present, the "else" branch is executed. +

+ +
+IfStmt = "if" [ SimpleStmt ";" ] Expression Block [ "else" ( IfStmt | Block ) ] .
+
+ +
+if x > max {
+	x = max
+}
+
+ +

+The expression may be preceded by a simple statement, which +executes before the expression is evaluated. +

+ +
+if x := f(); x < y {
+	return x
+} else if x > z {
+	return z
+} else {
+	return y
+}
+
+ + +

Switch statements

+ +

+"Switch" statements provide multi-way execution. +An expression or type specifier is compared to the "cases" +inside the "switch" to determine which branch +to execute. +

+ +
+SwitchStmt = ExprSwitchStmt | TypeSwitchStmt .
+
+ +

+There are two forms: expression switches and type switches. +In an expression switch, the cases contain expressions that are compared +against the value of the switch expression. +In a type switch, the cases contain types that are compared against the +type of a specially annotated switch expression. +

+ +

Expression switches

+ +

+In an expression switch, +the switch expression is evaluated and +the case expressions, which need not be constants, +are evaluated left-to-right and top-to-bottom; the first one that equals the +switch expression +triggers execution of the statements of the associated case; +the other cases are skipped. +If no case matches and there is a "default" case, +its statements are executed. +There can be at most one default case and it may appear anywhere in the +"switch" statement. +A missing switch expression is equivalent to +the expression true. +

+ +
+ExprSwitchStmt = "switch" [ SimpleStmt ";" ] [ Expression ] "{" { ExprCaseClause } "}" .
+ExprCaseClause = ExprSwitchCase ":" { Statement ";" } .
+ExprSwitchCase = "case" ExpressionList | "default" .
+
+ +

+In a case or default clause, +the last statement only may be a "fallthrough" statement +(§Fallthrough statement) to +indicate that control should flow from the end of this clause to +the first statement of the next clause. +Otherwise control flows to the end of the "switch" statement. +

+ +

+The expression may be preceded by a simple statement, which +executes before the expression is evaluated. +

+ +
+switch tag {
+default: s3()
+case 0, 1, 2, 3: s1()
+case 4, 5, 6, 7: s2()
+}
+
+switch x := f(); {  // missing switch expression means "true"
+case x < 0: return -x
+default: return x
+}
+
+switch {
+case x < y: f1()
+case x < z: f2()
+case x == 4: f3()
+}
+
+ +

Type switches

+ +

+A type switch compares types rather than values. It is otherwise similar +to an expression switch. It is marked by a special switch expression that +has the form of a type assertion +using the reserved word type rather than an actual type. +Cases then match literal types against the dynamic type of the expression +in the type assertion. +

+ +
+TypeSwitchStmt  = "switch" [ SimpleStmt ";" ] TypeSwitchGuard "{" { TypeCaseClause } "}" .
+TypeSwitchGuard = [ identifier ":=" ] PrimaryExpr "." "(" "type" ")" .
+TypeCaseClause  = TypeSwitchCase ":" { Statement ";" } .
+TypeSwitchCase  = "case" TypeList | "default" .
+TypeList        = Type { "," Type } .
+
+ +

+The TypeSwitchGuard may include a +short variable declaration. +When that form is used, the variable is declared in each clause. +In clauses with a case listing exactly one type, the variable +has that type; otherwise, the variable has the type of the expression +in the TypeSwitchGuard. +

+ +

+The type in a case may be nil +(§Predeclared identifiers); +that case is used when the expression in the TypeSwitchGuard +is a nil interface value. +

+ +

+Given an expression x of type interface{}, +the following type switch: +

+ +
+switch i := x.(type) {
+case nil:
+	printString("x is nil")
+case int:
+	printInt(i)  // i is an int
+case float64:
+	printFloat64(i)  // i is a float64
+case func(int) float64:
+	printFunction(i)  // i is a function
+case bool, string:
+	printString("type is bool or string")  // i is an interface{}
+default:
+	printString("don't know the type")
+}
+
+ +

+could be rewritten: +

+ +
+v := x  // x is evaluated exactly once
+if v == nil {
+	printString("x is nil")
+} else if i, is_int := v.(int); is_int {
+	printInt(i)  // i is an int
+} else if i, is_float64 := v.(float64); is_float64 {
+	printFloat64(i)  // i is a float64
+} else if i, is_func := v.(func(int) float64); is_func {
+	printFunction(i)  // i is a function
+} else {
+	i1, is_bool := v.(bool)
+	i2, is_string := v.(string)
+	if is_bool || is_string {
+		i := v
+		printString("type is bool or string")  // i is an interface{}
+	} else {
+		i := v
+		printString("don't know the type")  // i is an interface{}
+	}
+}
+
+ +

+The type switch guard may be preceded by a simple statement, which +executes before the guard is evaluated. +

+ +

+The "fallthrough" statement is not permitted in a type switch. +

+ +

For statements

+ +

+A "for" statement specifies repeated execution of a block. The iteration is +controlled by a condition, a "for" clause, or a "range" clause. +

+ +
+ForStmt = "for" [ Condition | ForClause | RangeClause ] Block .
+Condition = Expression .
+
+ +

+In its simplest form, a "for" statement specifies the repeated execution of +a block as long as a boolean condition evaluates to true. +The condition is evaluated before each iteration. +If the condition is absent, it is equivalent to true. +

+ +
+for a < b {
+	a *= 2
+}
+
+ +

+A "for" statement with a ForClause is also controlled by its condition, but +additionally it may specify an init +and a post statement, such as an assignment, +an increment or decrement statement. The init statement may be a +short variable declaration, but the post statement must not. +

+ +
+ForClause = [ InitStmt ] ";" [ Condition ] ";" [ PostStmt ] .
+InitStmt = SimpleStmt .
+PostStmt = SimpleStmt .
+
+ +
+for i := 0; i < 10; i++ {
+	f(i)
+}
+
+ +

+If non-empty, the init statement is executed once before evaluating the +condition for the first iteration; +the post statement is executed after each execution of the block (and +only if the block was executed). +Any element of the ForClause may be empty but the +semicolons are +required unless there is only a condition. +If the condition is absent, it is equivalent to true. +

+ +
+for cond { S() }    is the same as    for ; cond ; { S() }
+for      { S() }    is the same as    for true     { S() }
+
+ +

+A "for" statement with a "range" clause +iterates through all entries of an array, slice, string or map, +or values received on a channel. For each entry it assigns iteration values +to corresponding iteration variables and then executes the block. +

+ +
+RangeClause = Expression [ "," Expression ] ( "=" | ":=" ) "range" Expression .
+
+ +

+The expression on the right in the "range" clause is called the range expression, +which may be an array, pointer to an array, slice, string, map, or channel. +As with an assignment, the operands on the left must be +addressable or map index expressions; they +denote the iteration variables. If the range expression is a channel, only +one iteration variable is permitted, otherwise there may be one or two. +If the second iteration variable is the blank identifier, +the range clause is equivalent to the same clause with only the first variable present. +

+ +

+The range expression is evaluated once before beginning the loop +except if the expression is an array, in which case, depending on +the expression, it might not be evaluated (see below). +Function calls on the left are evaluated once per iteration. +For each iteration, iteration values are produced as follows: +

+ +
+Range expression                          1st value          2nd value (if 2nd variable is present)
+
+array or slice  a  [n]E, *[n]E, or []E    index    i  int    a[i]       E
+string          s  string type            index    i  int    see below  int
+map             m  map[K]V                key      k  K      m[k]       V
+channel         c  chan E                 element  e  E
+
+ +
    +
  1. +For an array, pointer to array, or slice value a, the index iteration +values are produced in increasing order, starting at element index 0. As a special +case, if only the first iteration variable is present, the range loop produces +iteration values from 0 up to len(a) and does not index into the array +or slice itself. For a nil slice, the number of iterations is 0. +
  2. + +
  3. +For a string value, the "range" clause iterates over the Unicode code points +in the string starting at byte index 0. On successive iterations, the index value will be the +index of the first byte of successive UTF-8-encoded code points in the string, +and the second value, of type int, will be the value of +the corresponding code point. If the iteration encounters an invalid +UTF-8 sequence, the second value will be 0xFFFD, +the Unicode replacement character, and the next iteration will advance +a single byte in the string. +
  4. + +
  5. +The iteration order over maps is not specified. +If map entries that have not yet been reached are deleted during iteration, +the corresponding iteration values will not be produced. If map entries are +inserted during iteration, the behavior is implementation-dependent, but the +iteration values for each entry will be produced at most once. If the map +is nil, the number of iterations is 0. +
  6. + +
  7. +For channels, the iteration values produced are the successive values sent on +the channel until the channel is closed. If the channel +is nil, the range expression blocks forever. +
  8. +
+ +

+The iteration values are assigned to the respective +iteration variables as in an assignment statement. +

+ +

+The iteration variables may be declared by the "range" clause (:=). +In this case their types are set to the types of the respective iteration values +and their scope ends at the end of the "for" +statement; they are re-used in each iteration. +If the iteration variables are declared outside the "for" statement, +after execution their values will be those of the last iteration. +

+ +
+var testdata *struct {
+	a *[7]int
+}
+for i, _ := range testdata.a {
+	// testdata.a is never evaluated; len(testdata.a) is constant
+	// i ranges from 0 to 6
+	f(i)
+}
+
+var a [10]string
+m := map[string]int{"mon":0, "tue":1, "wed":2, "thu":3, "fri":4, "sat":5, "sun":6}
+for i, s := range a {
+	// type of i is int
+	// type of s is string
+	// s == a[i]
+	g(i, s)
+}
+
+var key string
+var val interface {}  // value type of m is assignable to val
+for key, val = range m {
+	h(key, val)
+}
+// key == last map key encountered in iteration
+// val == map[key]
+
+var ch chan Work = producer()
+for w := range ch {
+	doWork(w)
+}
+
+ + +

Go statements

+ +

+A "go" statement starts the execution of a function or method call +as an independent concurrent thread of control, or goroutine, +within the same address space. +

+ +
+GoStmt = "go" Expression .
+
+ +

+The expression must be a call, and +unlike with a regular call, program execution does not wait +for the invoked function to complete. +

+ +
+go Server()
+go func(ch chan<- bool) { for { sleep(10); ch <- true; }} (c)
+
+ + +

Select statements

+ +

+A "select" statement chooses which of a set of possible communications +will proceed. It looks similar to a "switch" statement but with the +cases all referring to communication operations. +

+ +
+SelectStmt = "select" "{" { CommClause } "}" .
+CommClause = CommCase ":" { Statement ";" } .
+CommCase   = "case" ( SendStmt | RecvStmt ) | "default" .
+RecvStmt   = [ Expression [ "," Expression ] ( "=" | ":=" ) ] RecvExpr .
+RecvExpr   = Expression .
+
+ +

+RecvExpr must be a receive operation. +For all the cases in the "select" +statement, the channel expressions are evaluated in top-to-bottom order, along with +any expressions that appear on the right hand side of send statements. +A channel may be nil, +which is equivalent to that case not +being present in the select statement +except, if a send, its expression is still evaluated. +If any of the resulting operations can proceed, one of those is +chosen and the corresponding communication and statements are +evaluated. Otherwise, if there is a default case, that executes; +if there is no default case, the statement blocks until one of the communications can +complete. +If there are no cases with non-nil channels, +the statement blocks forever. +Even if the statement blocks, +the channel and send expressions are evaluated only once, +upon entering the select statement. +

+

+Since all the channels and send expressions are evaluated, any side +effects in that evaluation will occur for all the communications +in the "select" statement. +

+

+If multiple cases can proceed, a pseudo-random fair choice is made to decide +which single communication will execute. +

+The receive case may declare one or two new variables using a +short variable declaration. +

+ +
+var c, c1, c2, c3 chan int
+var i1, i2 int
+select {
+case i1 = <-c1:
+	print("received ", i1, " from c1\n")
+case c2 <- i2:
+	print("sent ", i2, " to c2\n")
+case i3, ok := (<-c3):  // same as: i3, ok := <-c3
+	if ok {
+		print("received ", i3, " from c3\n")
+	} else {
+		print("c3 is closed\n")
+	}
+default:
+	print("no communication\n")
+}
+
+for {  // send random sequence of bits to c
+	select {
+	case c <- 0:  // note: no statement, no fallthrough, no folding of cases
+	case c <- 1:
+	}
+}
+
+select { }  // block forever
+
+ + +

Return statements

+ +

+A "return" statement terminates execution of the containing function +and optionally provides a result value or values to the caller. +

+ +
+ReturnStmt = "return" [ ExpressionList ] .
+
+ +

+In a function without a result type, a "return" statement must not +specify any result values. +

+
+func no_result() {
+	return
+}
+
+ +

+There are three ways to return values from a function with a result +type: +

+ +
    +
  1. The return value or values may be explicitly listed + in the "return" statement. Each expression must be single-valued + and assignable + to the corresponding element of the function's result type. +
    +func simple_f() int {
    +	return 2
    +}
    +
    +func complex_f1() (re float64, im float64) {
    +	return -7.0, -4.0
    +}
    +
    +
  2. +
  3. The expression list in the "return" statement may be a single + call to a multi-valued function. The effect is as if each value + returned from that function were assigned to a temporary + variable with the type of the respective value, followed by a + "return" statement listing these variables, at which point the + rules of the previous case apply. +
    +func complex_f2() (re float64, im float64) {
    +	return complex_f1()
    +}
    +
    +
  4. +
  5. The expression list may be empty if the function's result + type specifies names for its result parameters (§Function Types). + The result parameters act as ordinary local variables + and the function may assign values to them as necessary. + The "return" statement returns the values of these variables. +
    +func complex_f3() (re float64, im float64) {
    +	re = 7.0
    +	im = 4.0
    +	return
    +}
    +
    +func (devnull) Write(p []byte) (n int, _ os.Error) {
    +	n = len(p)
    +	return
    +} 
    +
    +
  6. +
+ +

+Regardless of how they are declared, all the result values are initialized to the zero values for their type (§The zero value) upon entry to the function. +

+ + + +

Break statements

+ +

+A "break" statement terminates execution of the innermost +"for", "switch" or "select" statement. +

+ +
+BreakStmt = "break" [ Label ] .
+
+ +

+If there is a label, it must be that of an enclosing +"for", "switch" or "select" statement, and that is the one whose execution +terminates +(§For statements, §Switch statements, §Select statements). +

+ +
+L:
+	for i < n {
+		switch i {
+		case 5:
+			break L
+		}
+	}
+
+ +

Continue statements

+ +

+A "continue" statement begins the next iteration of the +innermost "for" loop at its post statement (§For statements). +

+ +
+ContinueStmt = "continue" [ Label ] .
+
+ +

+If there is a label, it must be that of an enclosing +"for" statement, and that is the one whose execution +advances +(§For statements). +

+ +

Goto statements

+ +

+A "goto" statement transfers control to the statement with the corresponding label. +

+ +
+GotoStmt = "goto" Label .
+
+ +
+goto Error
+
+ +

+Executing the "goto" statement must not cause any variables to come into +scope that were not already in scope at the point of the goto. +For instance, this example: +

+ +
+	goto L  // BAD
+	v := 3
+L:
+
+ +

+is erroneous because the jump to label L skips +the creation of v. +

+ +

+A "goto" statement outside a block cannot jump to a label inside that block. +For instance, this example: +

+ +
+if n%2 == 1 {
+	goto L1
+}
+for n > 0 {
+	f()
+	n--
+L1:
+	f()
+	n--
+}
+
+ +

+is erroneous because the label L1 is inside +the "for" statement's block but the goto is not. +

+ +

Fallthrough statements

+ +

+A "fallthrough" statement transfers control to the first statement of the +next case clause in a expression "switch" statement (§Expression switches). It may +be used only as the final non-empty statement in a case or default clause in an +expression "switch" statement. +

+ +
+FallthroughStmt = "fallthrough" .
+
+ + +

Defer statements

+ +

+A "defer" statement invokes a function whose execution is deferred to the moment +the surrounding function returns. +

+ +
+DeferStmt = "defer" Expression .
+
+ +

+The expression must be a function or method call. +Each time the "defer" statement +executes, the parameters to the function call are evaluated and saved anew but the +function is not invoked. +Deferred function calls are executed in LIFO order +immediately before the surrounding function returns, +after the return values, if any, have been evaluated, but before they +are returned to the caller. For instance, if the deferred function is +a function literal and the surrounding +function has named result parameters that +are in scope within the literal, the deferred function may access and modify +the result parameters before they are returned. +

+ +
+lock(l)
+defer unlock(l)  // unlocking happens before surrounding function returns
+
+// prints 3 2 1 0 before surrounding function returns
+for i := 0; i <= 3; i++ {
+	defer fmt.Print(i)
+}
+
+// f returns 1
+func f() (result int) {
+	defer func() {
+		result++
+	}()
+	return 0
+}
+
+ +

Built-in functions

+ +

+Built-in functions are +predeclared. +They are called like any other function but some of them +accept a type instead of an expression as the first argument. +

+ +

+The built-in functions do not have standard Go types, +so they can only appear in call expressions; +they cannot be used as function values. +

+ +
+BuiltinCall = identifier "(" [ BuiltinArgs [ "," ] ] ")" .
+BuiltinArgs = Type [ "," ExpressionList ] | ExpressionList .
+
+ +

Close

+ +

+For a channel c, the built-in function close(c) +marks the channel as unable to accept more values through a send operation; +sending to or closing a closed channel causes a run-time panic. +After calling close, and after any previously +sent values have been received, receive operations will return +the zero value for the channel's type without blocking. + +The multi-valued receive operation +returns a received value along with an indication of whether the channel is closed. +

+ + +

Length and capacity

+ +

+The built-in functions len and cap take arguments +of various types and return a result of type int. +The implementation guarantees that the result always fits into an int. +

+ +
+Call      Argument type    Result
+
+len(s)    string type      string length in bytes
+          [n]T, *[n]T      array length (== n)
+          []T              slice length
+          map[K]T          map length (number of defined keys)
+          chan T           number of elements queued in channel buffer
+
+cap(s)    [n]T, *[n]T      array length (== n)
+          []T              slice capacity
+          chan T           channel buffer capacity
+
+ +

+The capacity of a slice is the number of elements for which there is +space allocated in the underlying array. +At any time the following relationship holds: +

+ +
+0 <= len(s) <= cap(s)
+
+ +

+The length and capacity of a nil slice, map, or channel are 0. +

+ +

+The expression len(s) is constant if +s is a string constant. The expressions len(s) and +cap(s) are constants if the type of s is an array +or pointer to an array and the expression s does not contain +channel receives or +function calls; in this case s is not evaluated. +Otherwise, invocations of len and cap are not +constant and s is evaluated. +

+ + +

Allocation

+ +

+The built-in function new takes a type T and +returns a value of type *T. +The memory is initialized as described in the section on initial values +(§The zero value). +

+ +
+new(T)
+
+ +

+For instance +

+ +
+type S struct { a int; b float64 }
+new(S)
+
+ +

+dynamically allocates memory for a variable of type S, +initializes it (a=0, b=0.0), +and returns a value of type *S containing the address +of the memory. +

+ +

Making slices, maps and channels

+ +

+Slices, maps and channels are reference types that do not require the +extra indirection of an allocation with new. +The built-in function make takes a type T, +which must be a slice, map or channel type, +optionally followed by a type-specific list of expressions. +It returns a value of type T (not *T). +The memory is initialized as described in the section on initial values +(§The zero value). +

+ +
+Call             Type T     Result
+
+make(T, n)       slice      slice of type T with length n and capacity n
+make(T, n, m)    slice      slice of type T with length n and capacity m
+
+make(T)          map        map of type T
+make(T, n)       map        map of type T with initial space for n elements
+
+make(T)          channel    synchronous channel of type T
+make(T, n)       channel    asynchronous channel of type T, buffer size n
+
+ + +

+The arguments n and m must be of integer type. +A run-time panic occurs if n +is negative or larger than m, or if n or +m cannot be represented by an int. +

+ +
+s := make([]int, 10, 100)        // slice with len(s) == 10, cap(s) == 100
+s := make([]int, 10)             // slice with len(s) == cap(s) == 10
+c := make(chan int, 10)          // channel with a buffer size of 10
+m := make(map[string] int, 100)  // map with initial space for 100 elements
+
+ + +

Appending to and copying slices

+ +

+Two built-in functions assist in common slice operations. +

+ +

+The variadic function append +appends zero or more values x +to s of type S, which must be a slice type, and +returns the resulting slice, also of type S. +The values x are passed to a parameter of type ...T +where T is the element type of +S and the respective +parameter passing rules apply. +

+ +
+append(s S, x ...T) S  // T is the element type of S
+
+ +

+If the capacity of s is not large enough to fit the additional +values, append allocates a new, sufficiently large slice that fits +both the existing slice elements and the additional values. Thus, the returned +slice may refer to a different underlying array. +

+ +
+s0 := []int{0, 0}
+s1 := append(s0, 2)        // append a single element     s1 == []int{0, 0, 2}
+s2 := append(s1, 3, 5, 7)  // append multiple elements    s2 == []int{0, 0, 2, 3, 5, 7}
+s3 := append(s2, s0...)    // append a slice              s3 == []int{0, 0, 2, 3, 5, 7, 0, 0}
+
+var t []interface{}
+t = append(t, 42, 3.1415, "foo")                          t == []interface{}{42, 3.1415, "foo"}
+
+ +

+The function copy copies slice elements from +a source src to a destination dst and returns the +number of elements copied. Source and destination may overlap. +Both arguments must have identical element type T and must be +assignable to a slice of type []T. +The number of elements copied is the minimum of +len(src) and len(dst). +As a special case, copy also accepts a destination argument assignable +to type []byte with a source argument of a string type. +This form copies the bytes from the string into the byte slice. +

+ +
+copy(dst, src []T) int
+copy(dst []byte, src string) int
+
+ +

+Examples: +

+ +
+var a = [...]int{0, 1, 2, 3, 4, 5, 6, 7}
+var s = make([]int, 6)
+var b = make([]byte, 5)
+n1 := copy(s, a[0:])            // n1 == 6, s == []int{0, 1, 2, 3, 4, 5}
+n2 := copy(s, s[2:])            // n2 == 4, s == []int{2, 3, 4, 5, 4, 5}
+n3 := copy(b, "Hello, World!")  // n3 == 5, b == []byte("Hello")
+
+ +

Assembling and disassembling complex numbers

+ +

+Three functions assemble and disassemble complex numbers. +The built-in function complex constructs a complex +value from a floating-point real and imaginary part, while +real and imag +extract the real and imaginary parts of a complex value. +

+ +
+complex(realPart, imaginaryPart floatT) complexT
+real(complexT) floatT
+imag(complexT) floatT
+
+ +

+The type of the arguments and return value correspond. +For complex, the two arguments must be of the same +floating-point type and the return type is the complex type +with the corresponding floating-point constituents: +complex64 for float32, +complex128 for float64. +The real and imag functions +together form the inverse, so for a complex value z, +z == complex(real(z), imag(z)). +

+ +

+If the operands of these functions are all constants, the return +value is a constant. +

+ +
+var a = complex(2, -2)             // complex128
+var b = complex(1.0, -1.4)         // complex128
+x := float32(math.Cos(math.Pi/2))  // float32
+var c64 = complex(5, -x)           // complex64
+var im = imag(b)                   // float64
+var rl = real(c64)                 // float32
+
+ +

Handling panics

+ +

Two built-in functions, panic and recover, +assist in reporting and handling run-time panics +and program-defined error conditions. +

+ +
+func panic(interface{})
+func recover() interface{}
+
+ +

+When a function F calls panic, normal +execution of F stops immediately. Any functions whose +execution was deferred by the +invocation of F are run in the usual way, and then +F returns to its caller. To the caller, F +then behaves like a call to panic, terminating its own +execution and running deferred functions. This continues until all +functions in the goroutine have ceased execution, in reverse order. +At that point, the program is +terminated and the error condition is reported, including the value of +the argument to panic. This termination sequence is +called panicking. +

+ +
+panic(42)
+panic("unreachable")
+panic(Error("cannot parse"))
+
+ +

+The recover function allows a program to manage behavior +of a panicking goroutine. Executing a recover call +inside a deferred function (but not any function called by it) stops +the panicking sequence by restoring normal execution, and retrieves +the error value passed to the call of panic. If +recover is called outside the deferred function it will +not stop a panicking sequence. In this case, or when the goroutine +is not panicking, or if the argument supplied to panic +was nil, recover returns nil. +

+ +

+The protect function in the example below invokes +the function argument g and protects callers from +run-time panics raised by g. +

+ +
+func protect(g func()) {
+	defer func() {
+		log.Println("done")  // Println executes normally even in there is a panic
+		if x := recover(); x != nil {
+			log.Printf("run time panic: %v", x)
+		}
+	}()
+	log.Println("start")
+	g()
+}
+
+ + +

Bootstrapping

+ +

+Current implementations provide several built-in functions useful during +bootstrapping. These functions are documented for completeness but are not +guaranteed to stay in the language. They do not return a result. +

+ +
+Function   Behavior
+
+print      prints all arguments; formatting of arguments is implementation-specific
+println    like print but prints spaces between arguments and a newline at the end
+
+ + +

Packages

+ +

+Go programs are constructed by linking together packages. +A package in turn is constructed from one or more source files +that together declare constants, types, variables and functions +belonging to the package and which are accessible in all files +of the same package. Those elements may be +exported and used in another package. +

+ +

Source file organization

+ +

+Each source file consists of a package clause defining the package +to which it belongs, followed by a possibly empty set of import +declarations that declare packages whose contents it wishes to use, +followed by a possibly empty set of declarations of functions, +types, variables, and constants. +

+ +
+SourceFile       = PackageClause ";" { ImportDecl ";" } { TopLevelDecl ";" } .
+
+ +

Package clause

+ +

+A package clause begins each source file and defines the package +to which the file belongs. +

+ +
+PackageClause  = "package" PackageName .
+PackageName    = identifier .
+
+ +

+The PackageName must not be the blank identifier. +

+ +
+package math
+
+ +

+A set of files sharing the same PackageName form the implementation of a package. +An implementation may require that all source files for a package inhabit the same directory. +

+ +

Import declarations

+ +

+An import declaration states that the source file containing the +declaration uses identifiers +exported by the imported +package and enables access to them. The import names an +identifier (PackageName) to be used for access and an ImportPath +that specifies the package to be imported. +

+ +
+ImportDecl       = "import" ( ImportSpec | "(" { ImportSpec ";" } ")" ) .
+ImportSpec       = [ "." | PackageName ] ImportPath .
+ImportPath       = string_lit .
+
+ +

+The PackageName is used in qualified identifiers +to access the exported identifiers of the package within the importing source file. +It is declared in the file block. +If the PackageName is omitted, it defaults to the identifier specified in the +package clause of the imported package. +If an explicit period (.) appears instead of a name, all the +package's exported identifiers will be declared in the current file's +file block and can be accessed without a qualifier. +

+ +

+The interpretation of the ImportPath is implementation-dependent but +it is typically a substring of the full file name of the compiled +package and may be relative to a repository of installed packages. +

+ +

+Assume we have compiled a package containing the package clause +package math, which exports function Sin, and +installed the compiled package in the file identified by +"lib/math". +This table illustrates how Sin may be accessed in files +that import the package after the +various types of import declaration. +

+ +
+Import declaration          Local name of Sin
+
+import   "lib/math"         math.Sin
+import M "lib/math"         M.Sin
+import . "lib/math"         Sin
+
+ +

+An import declaration declares a dependency relation between +the importing and imported package. +It is illegal for a package to import itself or to import a package without +referring to any of its exported identifiers. To import a package solely for +its side-effects (initialization), use the blank +identifier as explicit package name: +

+ +
+import _ "lib/math"
+
+ + +

An example package

+ +

+Here is a complete Go package that implements a concurrent prime sieve. +

+ +
+package main
+
+import "fmt"
+
+// Send the sequence 2, 3, 4, … to channel 'ch'.
+func generate(ch chan<- int) {
+	for i := 2; ; i++ {
+		ch <- i  // Send 'i' to channel 'ch'.
+	}
+}
+
+// Copy the values from channel 'src' to channel 'dst',
+// removing those divisible by 'prime'.
+func filter(src <-chan int, dst chan<- int, prime int) {
+	for i := range src {	// Loop over values received from 'src'.
+		if i%prime != 0 {
+			dst <- i  // Send 'i' to channel 'dst'.
+		}
+	}
+}
+
+// The prime sieve: Daisy-chain filter processes together.
+func sieve() {
+	ch := make(chan int)  // Create a new channel.
+	go generate(ch)       // Start generate() as a subprocess.
+	for {
+		prime := <-ch
+		fmt.Print(prime, "\n")
+		ch1 := make(chan int)
+		go filter(ch, ch1, prime)
+		ch = ch1
+	}
+}
+
+func main() {
+	sieve()
+}
+
+ +

Program initialization and execution

+ +

The zero value

+

+When memory is allocated to store a value, either through a declaration +or a call of make or new, +and no explicit initialization is provided, the memory is +given a default initialization. Each element of such a value is +set to the zero value for its type: false for booleans, +0 for integers, 0.0 for floats, "" +for strings, and nil for pointers, functions, interfaces, slices, channels, and maps. +This initialization is done recursively, so for instance each element of an +array of structs will have its fields zeroed if no value is specified. +

+

+These two simple declarations are equivalent: +

+ +
+var i int
+var i int = 0
+
+ +

+After +

+ +
+type T struct { i int; f float64; next *T }
+t := new(T)
+
+ +

+the following holds: +

+ +
+t.i == 0
+t.f == 0.0
+t.next == nil
+
+ +

+The same would also be true after +

+ +
+var t T
+
+ +

Program execution

+

+A package with no imports is initialized by assigning initial values to +all its package-level variables +and then calling any +package-level function with the name and signature of +

+
+func init()
+
+

+defined in its source. +A package may contain multiple +init functions, even +within a single source file; they execute +in unspecified order. +

+

+Within a package, package-level variables are initialized, +and constant values are determined, in +data-dependent order: if the initializer of A +depends on the value of B, A +will be set after B. +It is an error if such dependencies form a cycle. +Dependency analysis is done lexically: A +depends on B if the value of A +contains a mention of B, contains a value +whose initializer +mentions B, or mentions a function that +mentions B, recursively. +If two items are not interdependent, they will be initialized +in the order they appear in the source. +Since the dependency analysis is done per package, it can produce +unspecified results if A's initializer calls a function defined +in another package that refers to B. +

+

+Initialization code may contain "go" statements, but the functions +they invoke do not begin execution until initialization of the entire +program is complete. Therefore, all initialization code is run in a single +goroutine. +

+

+An init function cannot be referred to from anywhere +in a program. In particular, init cannot be called explicitly, +nor can a pointer to init be assigned to a function variable. +

+

+If a package has imports, the imported packages are initialized +before initializing the package itself. If multiple packages import +a package P, P will be initialized only once. +

+

+The importing of packages, by construction, guarantees that there can +be no cyclic dependencies in initialization. +

+

+A complete program is created by linking a single, unimported package +called the main package with all the packages it imports, transitively. +The main package must +have package name main and +declare a function main that takes no +arguments and returns no value. +

+ +
+func main() { … }
+
+ +

+Program execution begins by initializing the main package and then +invoking the function main. +

+

+When the function main returns, the program exits. +It does not wait for other (non-main) goroutines to complete. +

+ +

Run-time panics

+ +

+Execution errors such as attempting to index an array out +of bounds trigger a run-time panic equivalent to a call of +the built-in function panic +with a value of the implementation-defined interface type runtime.Error. +That type defines at least the method +String() string. The exact error values that +represent distinct run-time error conditions are unspecified, +at least for now. +

+ +
+package runtime
+
+type Error interface {
+	String() string
+	// and perhaps others
+}
+
+ +

System considerations

+ +

Package unsafe

+ +

+The built-in package unsafe, known to the compiler, +provides facilities for low-level programming including operations +that violate the type system. A package using unsafe +must be vetted manually for type safety. The package provides the +following interface: +

+ +
+package unsafe
+
+type ArbitraryType int  // shorthand for an arbitrary Go type; it is not a real type
+type Pointer *ArbitraryType
+
+func Alignof(variable ArbitraryType) uintptr
+func Offsetof(selector ArbitraryType) uinptr
+func Sizeof(variable ArbitraryType) uintptr
+
+func Reflect(val interface{}) (typ runtime.Type, addr uintptr)
+func Typeof(val interface{}) (typ interface{})
+func Unreflect(typ runtime.Type, addr uintptr) interface{}
+
+ +

+Any pointer or value of type uintptr can be converted into +a Pointer and vice versa. +

+

+The function Sizeof takes an expression denoting a +variable of any type and returns the size of the variable in bytes. +

+

+The function Offsetof takes a selector (§Selectors) denoting a struct +field of any type and returns the field offset in bytes relative to the +struct's address. +For a struct s with field f: +

+ +
+uintptr(unsafe.Pointer(&s)) + unsafe.Offsetof(s.f) == uintptr(unsafe.Pointer(&s.f))
+
+ +

+Computer architectures may require memory addresses to be aligned; +that is, for addresses of a variable to be a multiple of a factor, +the variable's type's alignment. The function Alignof +takes an expression denoting a variable of any type and returns the +alignment of the (type of the) variable in bytes. For a variable +x: +

+ +
+uintptr(unsafe.Pointer(&x)) % unsafe.Alignof(x) == 0
+
+ +

+Calls to Alignof, Offsetof, and +Sizeof are compile-time constant expressions of type uintptr. +

+

+The functions unsafe.Typeof, +unsafe.Reflect, +and unsafe.Unreflect allow access at run time to the dynamic +types and values stored in interfaces. +Typeof returns a representation of +val's +dynamic type as a runtime.Type. +Reflect allocates a copy of +val's dynamic +value and returns both the type and the address of the copy. +Unreflect inverts Reflect, +creating an +interface value from a type and address. +The reflect package built on these primitives +provides a safe, more convenient way to inspect interface values. +

+ + +

Size and alignment guarantees

+ +

+For the numeric types (§Numeric types), the following sizes are guaranteed: +

+ +
+type                                 size in bytes
+
+byte, uint8, int8                     1
+uint16, int16                         2
+uint32, int32, float32                4
+uint64, int64, float64, complex64     8
+complex128                           16
+
+ +

+The following minimal alignment properties are guaranteed: +

+
    +
  1. For a variable x of any type: unsafe.Alignof(x) is at least 1. +
  2. + +
  3. For a variable x of struct type: unsafe.Alignof(x) is the largest of + all the values unsafe.Alignof(x.f) for each field f of x, but at least 1. +
  4. + +
  5. For a variable x of array type: unsafe.Alignof(x) is the same as + unsafe.Alignof(x[0]), but at least 1. +
  6. +
+ + +

Implementation differences - TODO

+
    +
  • len(a) is only a constant if a is a (qualified) identifier denoting an array or pointer to an array.
  • +
  • nil maps are not treated like empty maps.
  • +
  • Trying to send/receive from a nil channel causes a run-time panic.
  • +
  • unsafe.Alignof, unsafe.Offsetof, and unsafe.Sizeof return an int.
  • +
+
diff --git a/doc/go_tutorial.html b/doc/go_tutorial.html new file mode 100644 index 000000000..0b366bb2b --- /dev/null +++ b/doc/go_tutorial.html @@ -0,0 +1,1455 @@ + +

Introduction

+

+This document is a tutorial introduction to the basics of the Go programming +language, intended for programmers familiar with C or C++. It is not a comprehensive +guide to the language; at the moment the document closest to that is the +language specification. +After you've read this tutorial, you should look at +Effective Go, +which digs deeper into how the language is used and +talks about the style and idioms of programming in Go. +Also, slides from a 3-day course about Go are available. +They provide some background and a lot of examples: +Day 1, +Day 2, +Day 3. +

+The presentation here proceeds through a series of modest programs to illustrate +key features of the language. All the programs work (at time of writing) and are +checked into the repository in the directory /doc/progs/. +

+

Hello, World

+

+Let's start in the usual way: +

+

package main
+
+import fmt "fmt" // Package implementing formatted I/O.
+
+func main() {
+    fmt.Printf("Hello, world; or Καλημέρα κόσμε; or こんにちは 世界\n")
+}
+
+

+Every Go source file declares, using a package statement, which package it's part of. +It may also import other packages to use their facilities. +This program imports the package fmt to gain access to +our old, now capitalized and package-qualified, friend, fmt.Printf. +

+Functions are introduced with the func keyword. +The main package's main function is where the program starts running (after +any initialization). +

+String constants can contain Unicode characters, encoded in UTF-8. +(In fact, Go source files are defined to be encoded in UTF-8.) +

+The comment convention is the same as in C++: +

+

+/* ... */
+// ...
+
+

+Later we'll have much more to say about printing. +

+

Semicolons

+

+You might have noticed that our program has no semicolons. In Go +code, the only place you typically see semicolons is separating the +clauses of for loops and the like; they are not necessary after +every statement. +

+In fact, what happens is that the formal language uses semicolons, +much as in C or Java, but they are inserted automatically +at the end of every line that looks like the end of a statement. You +don't need to type them yourself. +

+For details about how this is done you can see the language +specification, but in practice all you need to know is that you +never need to put a semicolon at the end of a line. (You can put +them in if you want to write multiple statements per line.) As an +extra help, you can also leave out a semicolon immediately before +a closing brace. +

+This approach makes for clean-looking, semicolon-free code. The +one surprise is that it's important to put the opening +brace of a construct such as an if statement on the same line as +the if; if you don't, there are situations that may not compile +or may give the wrong result. The language forces the brace style +to some extent. +

+

Compiling

+

+Go is a compiled language. At the moment there are two compilers. +Gccgo is a Go compiler that uses the GCC back end. There is also a +suite of compilers with different (and odd) names for each architecture: +6g for the 64-bit x86, 8g for the 32-bit x86, and more. These +compilers run significantly faster but generate less efficient code +than gccgo. At the time of writing (late 2009), they also have +a more robust run-time system although gccgo is catching up. +

+Here's how to compile and run our program. With 6g, say, +

+

+$ 6g helloworld.go  # compile; object goes into helloworld.6
+$ 6l helloworld.6   # link; output goes into 6.out
+$ 6.out
+Hello, world; or Καλημέρα κόσμε; or こんにちは 世界
+$
+
+

+With gccgo it looks a little more traditional. +

+

+$ gccgo helloworld.go
+$ a.out
+Hello, world; or Καλημέρα κόσμε; or こんにちは 世界
+$
+
+

+

Echo

+

+Next up, here's a version of the Unix utility echo(1): +

+

package main
+
+import (
+    "os"
+    "flag" // command line option parser
+)
+
+var omitNewline = flag.Bool("n", false, "don't print final newline")
+
+const (
+    Space   = " "
+    Newline = "\n"
+)
+
+func main() {
+    flag.Parse() // Scans the arg list and sets up flags
+    var s string = ""
+    for i := 0; i < flag.NArg(); i++ {
+        if i > 0 {
+            s += Space
+        }
+        s += flag.Arg(i)
+    }
+    if !*omitNewline {
+        s += Newline
+    }
+    os.Stdout.WriteString(s)
+}
+
+

+This program is small but it's doing a number of new things. In the last example, +we saw func introduce a function. The keywords var, const, and type +(not used yet) also introduce declarations, as does import. +Notice that we can group declarations of the same sort into +parenthesized lists, one item per line, as in the import and const clauses here. +But it's not necessary to do so; we could have said +

+

+const Space = " "
+const Newline = "\n"
+
+

+This program imports the "os" package to access its Stdout variable, of type +*os.File. The import statement is actually a declaration: in its general form, +as used in our ``hello world'' program, +it names the identifier (fmt) +that will be used to access members of the package imported from the file ("fmt"), +found in the current directory or in a standard location. +In this program, though, we've dropped the explicit name from the imports; by default, +packages are imported using the name defined by the imported package, +which by convention is of course the file name itself. Our ``hello world'' program +could have said just import "fmt". +

+You can specify your +own import names if you want but it's only necessary if you need to resolve +a naming conflict. +

+Given os.Stdout we can use its WriteString method to print the string. +

+After importing the flag package, we use a var declaration +to create and initialize a global variable, called omitNewline, +to hold the value of echo's -n flag. +The variable has type *bool, pointer to bool. +

+In main.main, we parse the arguments (the call to flag.Parse) and then create a local +string variable with which to build the output. +

+The declaration statement has the form +

+

+var s string = ""
+
+

+This is the var keyword, followed by the name of the variable, followed by +its type, followed by an equals sign and an initial value for the variable. +

+Go tries to be terse, and this declaration could be shortened. Since the +string constant is of type string, we don't have to tell the compiler that. +We could write +

+

+var s = ""
+
+

+or we could go even shorter and write the idiom +

+

+s := ""
+
+

+The := operator is used a lot in Go to represent an initializing declaration. +There's one in the for clause on the next line: +

+

    for i := 0; i < flag.NArg(); i++ {
+
+

+The flag package has parsed the arguments and left the non-flag arguments +in a list that can be iterated over in the obvious way. +

+The Go for statement differs from that of C in a number of ways. First, +it's the only looping construct; there is no while or do. Second, +there are no parentheses on the clause, but the braces on the body +are mandatory. The same applies to the if and switch statements. +Later examples will show some other ways for can be written. +

+The body of the loop builds up the string s by appending (using +=) +the arguments and separating spaces. After the loop, if the -n flag is not +set, the program appends a newline. Finally, it writes the result. +

+Notice that main.main is a niladic function with no return type. +It's defined that way. Falling off the end of main.main means +''success''; if you want to signal an erroneous return, call +

+

+os.Exit(1)
+
+

+The os package contains other essentials for getting +started; for instance, os.Args is a slice used by the +flag package to access the command-line arguments. +

+

An Interlude about Types

+

+Go has some familiar types such as int and uint (unsigned int), which represent +values of the ''appropriate'' size for the machine. It also defines +explicitly-sized types such as int8, float64, and so on, plus +unsigned integer types such as uint, uint32, etc. +These are distinct types; even if int and int32 are both 32 bits in size, +they are not the same type. There is also a byte synonym for +uint8, which is the element type for strings. +

+Floating-point types are always sized: float32 and float64, +plus complex64 (two float32s) and complex128 +(two float64s). Complex numbers are outside the +scope of this tutorial. +

+Speaking of string, that's a built-in type as well. Strings are +immutable values—they are not just arrays of byte values. +Once you've built a string value, you can't change it, although +of course you can change a string variable simply by +reassigning it. This snippet from strings.go is legal code: +

+

    s := "hello"
+    if s[1] != 'e' {
+        os.Exit(1)
+    }
+    s = "good bye"
+    var p *string = &s
+    *p = "ciao"
+
+

+However the following statements are illegal because they would modify +a string value: +

+

+s[0] = 'x'
+(*p)[1] = 'y'
+
+

+In C++ terms, Go strings are a bit like const strings, while pointers +to strings are analogous to const string references. +

+Yes, there are pointers. However, Go simplifies their use a little; +read on. +

+Arrays are declared like this: +

+

+var arrayOfInt [10]int
+
+

+Arrays, like strings, are values, but they are mutable. This differs +from C, in which arrayOfInt would be usable as a pointer to int. +In Go, since arrays are values, it's meaningful (and useful) to talk +about pointers to arrays. +

+The size of the array is part of its type; however, one can declare +a slice variable to hold a reference to any array, of any size, +with the same element type. +A slice +expression has the form a[low : high], representing +the internal array indexed from low through high-1; the resulting +slice is indexed from 0 through high-low-1. +In short, slices look a lot like arrays but with +no explicit size ([] vs. [10]) and they reference a segment of +an underlying, usually anonymous, regular array. Multiple slices +can share data if they represent pieces of the same array; +multiple arrays can never share data. +

+Slices are much more common in Go programs than +regular arrays; they're more flexible, have reference semantics, +and are efficient. What they lack is the precise control of storage +layout of a regular array; if you want to have a hundred elements +of an array stored within your structure, you should use a regular +array. To create one, use a compound value constructor—an +expression formed +from a type followed by a brace-bounded expression like this: +

+

+[3]int{1,2,3}
+
+

+In this case the constructor builds an array of 3 ints. +

+When passing an array to a function, you almost always want +to declare the formal parameter to be a slice. When you call +the function, slice the array to create +(efficiently) a slice reference and pass that. +By default, the lower and upper bounds of a slice match the +ends of the existing object, so the concise notation [:] +will slice the whole array. +

+Using slices one can write this function (from sum.go): +

+

func sum(a []int) int { // returns an int
+    s := 0
+    for i := 0; i < len(a); i++ {
+        s += a[i]
+    }
+    return s
+}
+
+

+Note how the return type (int) is defined for sum by stating it +after the parameter list. +

+To call the function, we slice the array. This intricate call (we'll show +a simpler way in a moment) constructs +an array and slices it: +

+

+s := sum([3]int{1,2,3}[:])
+
+

+If you are creating a regular array but want the compiler to count the +elements for you, use ... as the array size: +

+

+s := sum([...]int{1,2,3}[:])
+
+

+That's fussier than necessary, though. +In practice, unless you're meticulous about storage layout within a +data structure, a slice itself—using empty brackets with no size—is all you need: +

+

+s := sum([]int{1,2,3})
+
+

+There are also maps, which you can initialize like this: +

+

+m := map[string]int{"one":1 , "two":2}
+
+

+The built-in function len, which returns number of elements, +makes its first appearance in sum. It works on strings, arrays, +slices, maps, and channels. +

+By the way, another thing that works on strings, arrays, slices, maps +and channels is the range clause on for loops. Instead of writing +

+

+for i := 0; i < len(a); i++ { ... }
+
+

+to loop over the elements of a slice (or map or ...) , we could write +

+

+for i, v := range a { ... }
+
+

+This assigns i to the index and v to the value of the successive +elements of the target of the range. See +Effective Go +for more examples of its use. +

+

+

An Interlude about Allocation

+

+Most types in Go are values. If you have an int or a struct +or an array, assignment +copies the contents of the object. +To allocate a new variable, use the built-in function new, which +returns a pointer to the allocated storage. +

+

+type T struct { a, b int }
+var t *T = new(T)
+
+

+or the more idiomatic +

+

+t := new(T)
+
+

+Some types—maps, slices, and channels (see below)—have reference semantics. +If you're holding a slice or a map and you modify its contents, other variables +referencing the same underlying data will see the modification. For these three +types you want to use the built-in function make: +

+

+m := make(map[string]int)
+
+

+This statement initializes a new map ready to store entries. +If you just declare the map, as in +

+

+var m map[string]int
+
+

+it creates a nil reference that cannot hold anything. To use the map, +you must first initialize the reference using make or by assignment from an +existing map. +

+Note that new(T) returns type *T while make(T) returns type +T. If you (mistakenly) allocate a reference object with new rather than make, +you receive a pointer to a nil reference, equivalent to +declaring an uninitialized variable and taking its address. +

+

An Interlude about Constants

+

+Although integers come in lots of sizes in Go, integer constants do not. +There are no constants like 0LL or 0x0UL. Instead, integer +constants are evaluated as large-precision values that +can overflow only when they are assigned to an integer variable with +too little precision to represent the value. +

+

+const hardEight = (1 << 100) >> 97  // legal
+
+

+There are nuances that deserve redirection to the legalese of the +language specification but here are some illustrative examples: +

+

+var a uint64 = 0  // a has type uint64, value 0
+a := uint64(0)    // equivalent; uses a "conversion"
+i := 0x1234       // i gets default type: int
+var j int = 1e6   // legal - 1000000 is representable in an int
+x := 1.5          // a float64, the default type for floating constants
+i3div2 := 3/2     // integer division - result is 1
+f3div2 := 3./2.   // floating-point division - result is 1.5
+
+

+Conversions only work for simple cases such as converting ints of one +sign or size to another and between integers and floating-point numbers, +plus a couple of other instances outside the scope of a tutorial. +There are no automatic numeric conversions of any kind in Go, +other than that of making constants have concrete size and type when +assigned to a variable. +

+

An I/O Package

+

+Next we'll look at a simple package for doing file I/O with an +open/close/read/write interface. Here's the start of file.go: +

+

package file
+
+import (
+    "os"
+    "syscall"
+)
+
+type File struct {
+    fd   int    // file descriptor number
+    name string // file name at Open time
+}
+
+

+The first few lines declare the name of the +package—file—and then import two packages. The os +package hides the differences +between various operating systems to give a consistent view of files and +so on; here we're going to use its error handling utilities +and reproduce the rudiments of its file I/O. +

+The other item is the low-level, external syscall package, which provides +a primitive interface to the underlying operating system's calls. +

+Next is a type definition: the type keyword introduces a type declaration, +in this case a data structure called File. +To make things a little more interesting, our File includes the name of the file +that the file descriptor refers to. +

+Because File starts with a capital letter, the type is available outside the package, +that is, by users of the package. In Go the rule about visibility of information is +simple: if a name (of a top-level type, function, method, constant or variable, or of +a structure field or method) is capitalized, users of the package may see it. Otherwise, the +name and hence the thing being named is visible only inside the package in which +it is declared. This is more than a convention; the rule is enforced by the compiler. +In Go, the term for publicly visible names is ''exported''. +

+In the case of File, all its fields are lower case and so invisible to users, but we +will soon give it some exported, upper-case methods. +

+First, though, here is a factory to create a File: +

+

func newFile(fd int, name string) *File {
+    if fd < 0 {
+        return nil
+    }
+    return &File{fd, name}
+}
+
+

+This returns a pointer to a new File structure with the file descriptor and name +filled in. This code uses Go's notion of a ''composite literal'', analogous to +the ones used to build maps and arrays, to construct a new heap-allocated +object. We could write +

+

+n := new(File)
+n.fd = fd
+n.name = name
+return n
+
+

+but for simple structures like File it's easier to return the address of a +composite literal, as is done here in the return statement from newFile. +

+We can use the factory to construct some familiar, exported variables of type *File: +

+

var (
+    Stdin  = newFile(syscall.Stdin, "/dev/stdin")
+    Stdout = newFile(syscall.Stdout, "/dev/stdout")
+    Stderr = newFile(syscall.Stderr, "/dev/stderr")
+)
+
+
+

+The newFile function was not exported because it's internal. The proper, +exported factory to use is OpenFile (we'll explain that name in a moment): +

+

func OpenFile(name string, mode int, perm uint32) (file *File, err os.Error) {
+    r, e := syscall.Open(name, mode, perm)
+    if e != 0 {
+        err = os.Errno(e)
+    }
+    return newFile(r, name), err
+}
+
+

+There are a number of new things in these few lines. First, OpenFile returns +multiple values, a File and an error (more about errors in a moment). +We declare the +multi-value return as a parenthesized list of declarations; syntactically +they look just like a second parameter list. The function +syscall.Open +also has a multi-value return, which we can grab with the multi-variable +declaration on the first line; it declares r and e to hold the two values, +both of type int (although you'd have to look at the syscall package +to see that). Finally, OpenFile returns two values: a pointer to the new File +and the error. If syscall.Open fails, the file descriptor r will +be negative and newFile will return nil. +

+About those errors: The os library includes a general notion of an error. +It's a good idea to use its facility in your own interfaces, as we do here, for +consistent error handling throughout Go code. In Open we use a +conversion to translate Unix's integer errno value into the integer type +os.Errno, which implements os.Error. +

+Why OpenFile and not Open? To mimic Go's os package, which +our exercise is emulating. The os package takes the opportunity +to make the two commonest cases - open for read and create for +write - the simplest, just Open and Create. OpenFile is the +general case, analogous to the Unix system call Open. Here is +the implementation of our Open and Create; they're trivial +wrappers that eliminate common errors by capturing +the tricky standard arguments to open and, especially, to create a file: +

+

const (
+    O_RDONLY = syscall.O_RDONLY
+    O_RDWR   = syscall.O_RDWR
+    O_CREATE = syscall.O_CREAT
+    O_TRUNC  = syscall.O_TRUNC
+)
+
+func Open(name string) (file *File, err os.Error) {
+    return OpenFile(name, O_RDONLY, 0)
+}
+
+

+

func Create(name string) (file *File, err os.Error) {
+    return OpenFile(name, O_RDWR|O_CREATE|O_TRUNC, 0666)
+}
+
+

+Back to our main story. +Now that we can build Files, we can write methods for them. To declare +a method of a type, we define a function to have an explicit receiver +of that type, placed +in parentheses before the function name. Here are some methods for *File, +each of which declares a receiver variable file. +

+

func (file *File) Close() os.Error {
+    if file == nil {
+        return os.EINVAL
+    }
+    e := syscall.Close(file.fd)
+    file.fd = -1 // so it can't be closed again
+    if e != 0 {
+        return os.Errno(e)
+    }
+    return nil
+}
+
+func (file *File) Read(b []byte) (ret int, err os.Error) {
+    if file == nil {
+        return -1, os.EINVAL
+    }
+    r, e := syscall.Read(file.fd, b)
+    if e != 0 {
+        err = os.Errno(e)
+    }
+    return int(r), err
+}
+
+func (file *File) Write(b []byte) (ret int, err os.Error) {
+    if file == nil {
+        return -1, os.EINVAL
+    }
+    r, e := syscall.Write(file.fd, b)
+    if e != 0 {
+        err = os.Errno(e)
+    }
+    return int(r), err
+}
+
+func (file *File) String() string {
+    return file.name
+}
+
+

+There is no implicit this and the receiver variable must be used to access +members of the structure. Methods are not declared within +the struct declaration itself. The struct declaration defines only data members. +In fact, methods can be created for almost any type you name, such as an integer or +array, not just for structs. We'll see an example with arrays later. +

+The String method is so called because of a printing convention we'll +describe later. +

+The methods use the public variable os.EINVAL to return the (os.Error +version of the) Unix error code EINVAL. The os library defines a standard +set of such error values. +

+We can now use our new package: +

+

package main
+
+import (
+    "./file"
+    "fmt"
+    "os"
+)
+
+func main() {
+    hello := []byte("hello, world\n")
+    file.Stdout.Write(hello)
+    f, err := file.Open("/does/not/exist")
+    if f == nil {
+        fmt.Printf("can't open file; err=%s\n", err.String())
+        os.Exit(1)
+    }
+}
+
+

+The ''./'' in the import of ''./file'' tells the compiler +to use our own package rather than +something from the directory of installed packages. +(Also, ''file.go'' must be compiled before we can import the +package.) +

+Now we can compile and run the program. On Unix, this would be the result: +

+

+$ 6g file.go                       # compile file package
+$ 6g helloworld3.go                # compile main package
+$ 6l -o helloworld3 helloworld3.6  # link - no need to mention "file"
+$ helloworld3
+hello, world
+can't open file; err=No such file or directory
+$
+
+

+

Rotting cats

+

+Building on the file package, here's a simple version of the Unix utility cat(1), +progs/cat.go: +

+

package main
+
+import (
+    "./file"
+    "flag"
+    "fmt"
+    "os"
+)
+
+func cat(f *file.File) {
+    const NBUF = 512
+    var buf [NBUF]byte
+    for {
+        switch nr, er := f.Read(buf[:]); true {
+        case nr < 0:
+            fmt.Fprintf(os.Stderr, "cat: error reading from %s: %s\n", f.String(), er.String())
+            os.Exit(1)
+        case nr == 0: // EOF
+            return
+        case nr > 0:
+            if nw, ew := file.Stdout.Write(buf[0:nr]); nw != nr {
+                fmt.Fprintf(os.Stderr, "cat: error writing from %s: %s\n", f.String(), ew.String())
+                os.Exit(1)
+            }
+        }
+    }
+}
+
+func main() {
+    flag.Parse() // Scans the arg list and sets up flags
+    if flag.NArg() == 0 {
+        cat(file.Stdin)
+    }
+    for i := 0; i < flag.NArg(); i++ {
+        f, err := file.Open(flag.Arg(i))
+        if f == nil {
+            fmt.Fprintf(os.Stderr, "cat: can't open %s: error %s\n", flag.Arg(i), err)
+            os.Exit(1)
+        }
+        cat(f)
+        f.Close()
+    }
+}
+
+

+By now this should be easy to follow, but the switch statement introduces some +new features. Like a for loop, an if or switch can include an +initialization statement. The switch statement in cat uses one to create variables +nr and er to hold the return values from the call to f.Read. (The if a few lines later +has the same idea.) The switch statement is general: it evaluates the cases +from top to bottom looking for the first case that matches the value; the +case expressions don't need to be constants or even integers, as long as +they all have the same type. +

+Since the switch value is just true, we could leave it off—as is also +the situation +in a for statement, a missing value means true. In fact, such a switch +is a form of if-else chain. While we're here, it should be mentioned that in +switch statements each case has an implicit break. +

+The argument to file.Stdout.Write is created by slicing the array buf. +Slices provide the standard Go way to handle I/O buffers. +

+Now let's make a variant of cat that optionally does rot13 on its input. +It's easy to do by just processing the bytes, but instead we will exploit +Go's notion of an interface. +

+The cat subroutine uses only two methods of f: Read and String, +so let's start by defining an interface that has exactly those two methods. +Here is code from progs/cat_rot13.go: +

+

type reader interface {
+    Read(b []byte) (ret int, err os.Error)
+    String() string
+}
+
+

+Any type that has the two methods of reader—regardless of whatever +other methods the type may also have—is said to implement the +interface. Since file.File implements these methods, it implements the +reader interface. We could tweak the cat subroutine to accept a reader +instead of a *file.File and it would work just fine, but let's embellish a little +first by writing a second type that implements reader, one that wraps an +existing reader and does rot13 on the data. To do this, we just define +the type and implement the methods and with no other bookkeeping, +we have a second implementation of the reader interface. +

+

type rotate13 struct {
+    source reader
+}
+
+func newRotate13(source reader) *rotate13 {
+    return &rotate13{source}
+}
+
+func (r13 *rotate13) Read(b []byte) (ret int, err os.Error) {
+    r, e := r13.source.Read(b)
+    for i := 0; i < r; i++ {
+        b[i] = rot13(b[i])
+    }
+    return r, e
+}
+
+func (r13 *rotate13) String() string {
+    return r13.source.String()
+}
+// end of rotate13 implementation
+
+

+(The rot13 function called in Read is trivial and not worth reproducing here.) +

+To use the new feature, we define a flag: +

+

var rot13Flag = flag.Bool("rot13", false, "rot13 the input")
+
+

+and use it from within a mostly unchanged cat function: +

+

func cat(r reader) {
+    const NBUF = 512
+    var buf [NBUF]byte
+
+    if *rot13Flag {
+        r = newRotate13(r)
+    }
+    for {
+        switch nr, er := r.Read(buf[:]); {
+        case nr < 0:
+            fmt.Fprintf(os.Stderr, "cat: error reading from %s: %s\n", r.String(), er.String())
+            os.Exit(1)
+        case nr == 0: // EOF
+            return
+        case nr > 0:
+            nw, ew := file.Stdout.Write(buf[0:nr])
+            if nw != nr {
+                fmt.Fprintf(os.Stderr, "cat: error writing from %s: %s\n", r.String(), ew.String())
+                os.Exit(1)
+            }
+        }
+    }
+}
+
+

+(We could also do the wrapping in main and leave cat mostly alone, except +for changing the type of the argument; consider that an exercise.) +The if at the top of cat sets it all up: If the rot13 flag is true, wrap the reader +we received into a rotate13 and proceed. Note that the interface variables +are values, not pointers: the argument is of type reader, not *reader, +even though under the covers it holds a pointer to a struct. +

+Here it is in action: +

+

+$ echo abcdefghijklmnopqrstuvwxyz | ./cat
+abcdefghijklmnopqrstuvwxyz
+$ echo abcdefghijklmnopqrstuvwxyz | ./cat --rot13
+nopqrstuvwxyzabcdefghijklm
+$
+
+

+Fans of dependency injection may take cheer from how easily interfaces +allow us to substitute the implementation of a file descriptor. +

+Interfaces are a distinctive feature of Go. An interface is implemented by a +type if the type implements all the methods declared in the interface. +This means +that a type may implement an arbitrary number of different interfaces. +There is no type hierarchy; things can be much more ad hoc, +as we saw with rot13. The type file.File implements reader; it could also +implement a writer, or any other interface built from its methods that +fits the current situation. Consider the empty interface +

+

+type Empty interface {}
+
+

+Every type implements the empty interface, which makes it +useful for things like containers. +

+

Sorting

+

+Interfaces provide a simple form of polymorphism. They completely +separate the definition of what an object does from how it does it, allowing +distinct implementations to be represented at different times by the +same interface variable. +

+As an example, consider this simple sort algorithm taken from progs/sort.go: +

+

func Sort(data Interface) {
+    for i := 1; i < data.Len(); i++ {
+        for j := i; j > 0 && data.Less(j, j-1); j-- {
+            data.Swap(j, j-1)
+        }
+    }
+}
+
+

+The code needs only three methods, which we wrap into sort's Interface: +

+

type Interface interface {
+    Len() int
+    Less(i, j int) bool
+    Swap(i, j int)
+}
+
+

+We can apply Sort to any type that implements Len, Less, and Swap. +The sort package includes the necessary methods to allow sorting of +arrays of integers, strings, etc.; here's the code for arrays of int +

+

type IntSlice []int
+
+func (p IntSlice) Len() int           { return len(p) }
+func (p IntSlice) Less(i, j int) bool { return p[i] < p[j] }
+func (p IntSlice) Swap(i, j int)      { p[i], p[j] = p[j], p[i] }
+
+

+Here we see methods defined for non-struct types. You can define methods +for any type you define and name in your package. +

+And now a routine to test it out, from progs/sortmain.go. This +uses a function in the sort package, omitted here for brevity, +to test that the result is sorted. +

+

func ints() {
+    data := []int{74, 59, 238, -784, 9845, 959, 905, 0, 0, 42, 7586, -5467984, 7586}
+    a := sort.IntSlice(data)
+    sort.Sort(a)
+    if !sort.IsSorted(a) {
+        panic("fail")
+    }
+}
+
+

+If we have a new type we want to be able to sort, all we need to do is +to implement the three methods for that type, like this: +

+

type day struct {
+    num       int
+    shortName string
+    longName  string
+}
+
+type dayArray struct {
+    data []*day
+}
+
+func (p *dayArray) Len() int           { return len(p.data) }
+func (p *dayArray) Less(i, j int) bool { return p.data[i].num < p.data[j].num }
+func (p *dayArray) Swap(i, j int)      { p.data[i], p.data[j] = p.data[j], p.data[i] }
+
+

+

+

Printing

+

+The examples of formatted printing so far have been modest. In this section +we'll talk about how formatted I/O can be done well in Go. +

+We've seen simple uses of the package fmt, which +implements Printf, Fprintf, and so on. +Within the fmt package, Printf is declared with this signature: +

+

+Printf(format string, v ...interface{}) (n int, errno os.Error)
+
+

+The token ... introduces a variable-length argument list that in C would +be handled using the stdarg.h macros. +In Go, variadic functions are passed a slice of the arguments of the +specified type. In Printf's case, the declaration says ...interface{} +so the actual type is a slice of empty interface values, []interface{}. +Printf can examine the arguments by iterating over the slice +and, for each element, using a type switch or the reflection library +to interpret the value. +It's off topic here but such run-time type analysis +helps explain some of the nice properties of Go's Printf, +due to the ability of Printf to discover the type of its arguments +dynamically. +

+For example, in C each format must correspond to the type of its +argument. It's easier in many cases in Go. Instead of %llud you +can just say %d; Printf knows the size and signedness of the +integer and can do the right thing for you. The snippet +

+

    var u64 uint64 = 1<<64 - 1
+    fmt.Printf("%d %d\n", u64, int64(u64))
+
+

+prints +

+

+18446744073709551615 -1
+
+

+In fact, if you're lazy the format %v will print, in a simple +appropriate style, any value, even an array or structure. The output of +

+

    type T struct {
+        a int
+        b string
+    }
+    t := T{77, "Sunset Strip"}
+    a := []int{1, 2, 3, 4}
+    fmt.Printf("%v %v %v\n", u64, t, a)
+
+

+is +

+

+18446744073709551615 {77 Sunset Strip} [1 2 3 4]
+
+

+You can drop the formatting altogether if you use Print or Println +instead of Printf. Those routines do fully automatic formatting. +The Print function just prints its elements out using the equivalent +of %v while Println inserts spaces between arguments +and adds a newline. The output of each of these two lines is identical +to that of the Printf call above. +

+

    fmt.Print(u64, " ", t, " ", a, "\n")
+    fmt.Println(u64, t, a)
+
+

+If you have your own type you'd like Printf or Print to format, +just give it a String method that returns a string. The print +routines will examine the value to inquire whether it implements +the method and if so, use it rather than some other formatting. +Here's a simple example. +

+

type testType struct {
+    a int
+    b string
+}
+
+func (t *testType) String() string {
+    return fmt.Sprint(t.a) + " " + t.b
+}
+
+func main() {
+    t := &testType{77, "Sunset Strip"}
+    fmt.Println(t)
+}
+
+

+Since *testType has a String method, the +default formatter for that type will use it and produce the output +

+

+77 Sunset Strip
+
+

+Observe that the String method calls Sprint (the obvious Go +variant that returns a string) to do its formatting; special formatters +can use the fmt library recursively. +

+Another feature of Printf is that the format %T will print a string +representation of the type of a value, which can be handy when debugging +polymorphic code. +

+It's possible to write full custom print formats with flags and precisions +and such, but that's getting a little off the main thread so we'll leave it +as an exploration exercise. +

+You might ask, though, how Printf can tell whether a type implements +the String method. Actually what it does is ask if the value can +be converted to an interface variable that implements the method. +Schematically, given a value v, it does this: +

+

+

+type Stringer interface {
+    String() string
+}
+
+

+

+s, ok := v.(Stringer)  // Test whether v implements "String()"
+if ok {
+    result = s.String()
+} else {
+    result = defaultOutput(v)
+}
+
+

+The code uses a ``type assertion'' (v.(Stringer)) to test if the value stored in +v satisfies the Stringer interface; if it does, s +will become an interface variable implementing the method and ok will +be true. We then use the interface variable to call the method. +(The ''comma, ok'' pattern is a Go idiom used to test the success of +operations such as type conversion, map update, communications, and so on, +although this is the only appearance in this tutorial.) +If the value does not satisfy the interface, ok will be false. +

+In this snippet the name Stringer follows the convention that we add ''[e]r'' +to interfaces describing simple method sets like this. +

+One last wrinkle. To complete the suite, besides Printf etc. and Sprintf +etc., there are also Fprintf etc. Unlike in C, Fprintf's first argument is +not a file. Instead, it is a variable of type io.Writer, which is an +interface type defined in the io library: +

+

+type Writer interface {
+    Write(p []byte) (n int, err os.Error)
+}
+
+

+(This interface is another conventional name, this time for Write; there are also +io.Reader, io.ReadWriter, and so on.) +Thus you can call Fprintf on any type that implements a standard Write +method, not just files but also network channels, buffers, whatever +you want. +

+

Prime numbers

+

+Now we come to processes and communication—concurrent programming. +It's a big subject so to be brief we assume some familiarity with the topic. +

+A classic program in the style is a prime sieve. +(The sieve of Eratosthenes is computationally more efficient than +the algorithm presented here, but we are more interested in concurrency than +algorithmics at the moment.) +It works by taking a stream of all the natural numbers and introducing +a sequence of filters, one for each prime, to winnow the multiples of +that prime. At each step we have a sequence of filters of the primes +so far, and the next number to pop out is the next prime, which triggers +the creation of the next filter in the chain. +

+Here's a flow diagram; each box represents a filter element whose +creation is triggered by the first number that flowed from the +elements before it. +

+
+

+      +

+
+

+To create a stream of integers, we use a Go channel, which, +borrowing from CSP's descendants, represents a communications +channel that can connect two concurrent computations. +In Go, channel variables are references to a run-time object that +coordinates the communication; as with maps and slices, use +make to create a new channel. +

+Here is the first function in progs/sieve.go: +

+

// Send the sequence 2, 3, 4, ... to channel 'ch'.
+func generate(ch chan int) {
+    for i := 2; ; i++ {
+        ch <- i // Send 'i' to channel 'ch'.
+    }
+}
+
+

+The generate function sends the sequence 2, 3, 4, 5, ... to its +argument channel, ch, using the binary communications operator <-. +Channel operations block, so if there's no recipient for the value on ch, +the send operation will wait until one becomes available. +

+The filter function has three arguments: an input channel, an output +channel, and a prime number. It copies values from the input to the +output, discarding anything divisible by the prime. The unary communications +operator <- (receive) retrieves the next value on the channel. +

+

// Copy the values from channel 'in' to channel 'out',
+// removing those divisible by 'prime'.
+func filter(in, out chan int, prime int) {
+    for {
+        i := <-in // Receive value of new variable 'i' from 'in'.
+        if i%prime != 0 {
+            out <- i // Send 'i' to channel 'out'.
+        }
+    }
+}
+
+

+The generator and filters execute concurrently. Go has +its own model of process/threads/light-weight processes/coroutines, +so to avoid notational confusion we call concurrently executing +computations in Go goroutines. To start a goroutine, +invoke the function, prefixing the call with the keyword go; +this starts the function running in parallel with the current +computation but in the same address space: +

+

+go sum(hugeArray) // calculate sum in the background
+
+

+If you want to know when the calculation is done, pass a channel +on which it can report back: +

+

+ch := make(chan int)
+go sum(hugeArray, ch)
+// ... do something else for a while
+result := <-ch  // wait for, and retrieve, result
+
+

+Back to our prime sieve. Here's how the sieve pipeline is stitched +together: +

+

func main() {
+    ch := make(chan int)       // Create a new channel.
+    go generate(ch)            // Start generate() as a goroutine.
+    for i := 0; i < 100; i++ { // Print the first hundred primes.
+        prime := <-ch
+        fmt.Println(prime)
+        ch1 := make(chan int)
+        go filter(ch, ch1, prime)
+        ch = ch1
+    }
+}
+
+

+The first line of main creates the initial channel to pass to generate, which it +then starts up. As each prime pops out of the channel, a new filter +is added to the pipeline and its output becomes the new value +of ch. +

+The sieve program can be tweaked to use a pattern common +in this style of programming. Here is a variant version +of generate, from progs/sieve1.go: +

+

func generate() chan int {
+    ch := make(chan int)
+    go func() {
+        for i := 2; ; i++ {
+            ch <- i
+        }
+    }()
+    return ch
+}
+
+

+This version does all the setup internally. It creates the output +channel, launches a goroutine running a function literal, and +returns the channel to the caller. It is a factory for concurrent +execution, starting the goroutine and returning its connection. +

+The function literal notation used in the go statement allows us to construct an +anonymous function and invoke it on the spot. Notice that the local +variable ch is available to the function literal and lives on even +after generate returns. +

+The same change can be made to filter: +

+

func filter(in chan int, prime int) chan int {
+    out := make(chan int)
+    go func() {
+        for {
+            if i := <-in; i%prime != 0 {
+                out <- i
+            }
+        }
+    }()
+    return out
+}
+
+

+The sieve function's main loop becomes simpler and clearer as a +result, and while we're at it let's turn it into a factory too: +

+

func sieve() chan int {
+    out := make(chan int)
+    go func() {
+        ch := generate()
+        for {
+            prime := <-ch
+            out <- prime
+            ch = filter(ch, prime)
+        }
+    }()
+    return out
+}
+
+

+Now main's interface to the prime sieve is a channel of primes: +

+

func main() {
+    primes := sieve()
+    for i := 0; i < 100; i++ { // Print the first hundred primes.
+        fmt.Println(<-primes)
+    }
+}
+
+

+

Multiplexing

+

+With channels, it's possible to serve multiple independent client goroutines without +writing an explicit multiplexer. The trick is to send the server a channel in the message, +which it will then use to reply to the original sender. +A realistic client-server program is a lot of code, so here is a very simple substitute +to illustrate the idea. It starts by defining a request type, which embeds a channel +that will be used for the reply. +

+

type request struct {
+    a, b   int
+    replyc chan int
+}
+
+

+The server will be trivial: it will do simple binary operations on integers. Here's the +code that invokes the operation and responds to the request: +

+

type binOp func(a, b int) int
+
+func run(op binOp, req *request) {
+    reply := op(req.a, req.b)
+    req.replyc <- reply
+}
+
+

+The type declaration makes binOp represent a function taking two integers and +returning a third. +

+The server routine loops forever, receiving requests and, to avoid blocking due to +a long-running operation, starting a goroutine to do the actual work. +

+

func server(op binOp, service chan *request) {
+    for {
+        req := <-service
+        go run(op, req) // don't wait for it
+    }
+}
+
+

+We construct a server in a familiar way, starting it and returning a channel +connected to it: +

+

func startServer(op binOp) chan *request {
+    req := make(chan *request)
+    go server(op, req)
+    return req
+}
+
+

+Here's a simple test. It starts a server with an addition operator and sends out +N requests without waiting for the replies. Only after all the requests are sent +does it check the results. +

+

func main() {
+    adder := startServer(func(a, b int) int { return a + b })
+    const N = 100
+    var reqs [N]request
+    for i := 0; i < N; i++ {
+        req := &reqs[i]
+        req.a = i
+        req.b = i + N
+        req.replyc = make(chan int)
+        adder <- req
+    }
+    for i := N - 1; i >= 0; i-- { // doesn't matter what order
+        if <-reqs[i].replyc != N+2*i {
+            fmt.Println("fail at", i)
+        }
+    }
+    fmt.Println("done")
+}
+
+

+One annoyance with this program is that it doesn't shut down the server cleanly; when main returns +there are a number of lingering goroutines blocked on communication. To solve this, +we can provide a second, quit channel to the server: +

+

func startServer(op binOp) (service chan *request, quit chan bool) {
+    service = make(chan *request)
+    quit = make(chan bool)
+    go server(op, service, quit)
+    return service, quit
+}
+
+

+It passes the quit channel to the server function, which uses it like this: +

+

func server(op binOp, service chan *request, quit chan bool) {
+    for {
+        select {
+        case req := <-service:
+            go run(op, req) // don't wait for it
+        case <-quit:
+            return
+        }
+    }
+}
+
+

+Inside server, the select statement chooses which of the multiple communications +listed by its cases can proceed. If all are blocked, it waits until one can proceed; if +multiple can proceed, it chooses one at random. In this instance, the select allows +the server to honor requests until it receives a quit message, at which point it +returns, terminating its execution. +

+

+All that's left is to strobe the quit channel +at the end of main: +

+

    adder, quit := startServer(func(a, b int) int { return a + b })
+
+... +
    quit <- true
+
+

+There's a lot more to Go programming and concurrent programming in general but this +quick tour should give you some of the basics. diff --git a/doc/go_tutorial.tmpl b/doc/go_tutorial.tmpl new file mode 100644 index 000000000..c170c25aa --- /dev/null +++ b/doc/go_tutorial.tmpl @@ -0,0 +1,990 @@ + +

Introduction

+

+This document is a tutorial introduction to the basics of the Go programming +language, intended for programmers familiar with C or C++. It is not a comprehensive +guide to the language; at the moment the document closest to that is the +language specification. +After you've read this tutorial, you should look at +Effective Go, +which digs deeper into how the language is used and +talks about the style and idioms of programming in Go. +Also, slides from a 3-day course about Go are available. +They provide some background and a lot of examples: +Day 1, +Day 2, +Day 3. +

+The presentation here proceeds through a series of modest programs to illustrate +key features of the language. All the programs work (at time of writing) and are +checked into the repository in the directory /doc/progs/. +

+

Hello, World

+

+Let's start in the usual way: +

+{{code "progs/helloworld.go" `/package/` "$"}} +

+Every Go source file declares, using a package statement, which package it's part of. +It may also import other packages to use their facilities. +This program imports the package fmt to gain access to +our old, now capitalized and package-qualified, friend, fmt.Printf. +

+Functions are introduced with the func keyword. +The main package's main function is where the program starts running (after +any initialization). +

+String constants can contain Unicode characters, encoded in UTF-8. +(In fact, Go source files are defined to be encoded in UTF-8.) +

+The comment convention is the same as in C++: +

+

+/* ... */
+// ...
+
+

+Later we'll have much more to say about printing. +

+

Semicolons

+

+You might have noticed that our program has no semicolons. In Go +code, the only place you typically see semicolons is separating the +clauses of for loops and the like; they are not necessary after +every statement. +

+In fact, what happens is that the formal language uses semicolons, +much as in C or Java, but they are inserted automatically +at the end of every line that looks like the end of a statement. You +don't need to type them yourself. +

+For details about how this is done you can see the language +specification, but in practice all you need to know is that you +never need to put a semicolon at the end of a line. (You can put +them in if you want to write multiple statements per line.) As an +extra help, you can also leave out a semicolon immediately before +a closing brace. +

+This approach makes for clean-looking, semicolon-free code. The +one surprise is that it's important to put the opening +brace of a construct such as an if statement on the same line as +the if; if you don't, there are situations that may not compile +or may give the wrong result. The language forces the brace style +to some extent. +

+

Compiling

+

+Go is a compiled language. At the moment there are two compilers. +Gccgo is a Go compiler that uses the GCC back end. There is also a +suite of compilers with different (and odd) names for each architecture: +6g for the 64-bit x86, 8g for the 32-bit x86, and more. These +compilers run significantly faster but generate less efficient code +than gccgo. At the time of writing (late 2009), they also have +a more robust run-time system although gccgo is catching up. +

+Here's how to compile and run our program. With 6g, say, +

+

+$ 6g helloworld.go  # compile; object goes into helloworld.6
+$ 6l helloworld.6   # link; output goes into 6.out
+$ 6.out
+Hello, world; or Καλημέρα κόσμε; or こんにちは 世界
+$
+
+

+With gccgo it looks a little more traditional. +

+

+$ gccgo helloworld.go
+$ a.out
+Hello, world; or Καλημέρα κόσμε; or こんにちは 世界
+$
+
+

+

Echo

+

+Next up, here's a version of the Unix utility echo(1): +

+{{code "progs/echo.go" `/package/` "$"}} +

+This program is small but it's doing a number of new things. In the last example, +we saw func introduce a function. The keywords var, const, and type +(not used yet) also introduce declarations, as does import. +Notice that we can group declarations of the same sort into +parenthesized lists, one item per line, as in the import and const clauses here. +But it's not necessary to do so; we could have said +

+

+const Space = " "
+const Newline = "\n"
+
+

+This program imports the "os" package to access its Stdout variable, of type +*os.File. The import statement is actually a declaration: in its general form, +as used in our ``hello world'' program, +it names the identifier (fmt) +that will be used to access members of the package imported from the file ("fmt"), +found in the current directory or in a standard location. +In this program, though, we've dropped the explicit name from the imports; by default, +packages are imported using the name defined by the imported package, +which by convention is of course the file name itself. Our ``hello world'' program +could have said just import "fmt". +

+You can specify your +own import names if you want but it's only necessary if you need to resolve +a naming conflict. +

+Given os.Stdout we can use its WriteString method to print the string. +

+After importing the flag package, we use a var declaration +to create and initialize a global variable, called omitNewline, +to hold the value of echo's -n flag. +The variable has type *bool, pointer to bool. +

+In main.main, we parse the arguments (the call to flag.Parse) and then create a local +string variable with which to build the output. +

+The declaration statement has the form +

+

+var s string = ""
+
+

+This is the var keyword, followed by the name of the variable, followed by +its type, followed by an equals sign and an initial value for the variable. +

+Go tries to be terse, and this declaration could be shortened. Since the +string constant is of type string, we don't have to tell the compiler that. +We could write +

+

+var s = ""
+
+

+or we could go even shorter and write the idiom +

+

+s := ""
+
+

+The := operator is used a lot in Go to represent an initializing declaration. +There's one in the for clause on the next line: +

+{{code "progs/echo.go" `/for/`}} +

+The flag package has parsed the arguments and left the non-flag arguments +in a list that can be iterated over in the obvious way. +

+The Go for statement differs from that of C in a number of ways. First, +it's the only looping construct; there is no while or do. Second, +there are no parentheses on the clause, but the braces on the body +are mandatory. The same applies to the if and switch statements. +Later examples will show some other ways for can be written. +

+The body of the loop builds up the string s by appending (using +=) +the arguments and separating spaces. After the loop, if the -n flag is not +set, the program appends a newline. Finally, it writes the result. +

+Notice that main.main is a niladic function with no return type. +It's defined that way. Falling off the end of main.main means +''success''; if you want to signal an erroneous return, call +

+

+os.Exit(1)
+
+

+The os package contains other essentials for getting +started; for instance, os.Args is a slice used by the +flag package to access the command-line arguments. +

+

An Interlude about Types

+

+Go has some familiar types such as int and uint (unsigned int), which represent +values of the ''appropriate'' size for the machine. It also defines +explicitly-sized types such as int8, float64, and so on, plus +unsigned integer types such as uint, uint32, etc. +These are distinct types; even if int and int32 are both 32 bits in size, +they are not the same type. There is also a byte synonym for +uint8, which is the element type for strings. +

+Floating-point types are always sized: float32 and float64, +plus complex64 (two float32s) and complex128 +(two float64s). Complex numbers are outside the +scope of this tutorial. +

+Speaking of string, that's a built-in type as well. Strings are +immutable values—they are not just arrays of byte values. +Once you've built a string value, you can't change it, although +of course you can change a string variable simply by +reassigning it. This snippet from strings.go is legal code: +

+{{code "progs/strings.go" `/hello/` `/ciao/`}} +

+However the following statements are illegal because they would modify +a string value: +

+

+s[0] = 'x'
+(*p)[1] = 'y'
+
+

+In C++ terms, Go strings are a bit like const strings, while pointers +to strings are analogous to const string references. +

+Yes, there are pointers. However, Go simplifies their use a little; +read on. +

+Arrays are declared like this: +

+

+var arrayOfInt [10]int
+
+

+Arrays, like strings, are values, but they are mutable. This differs +from C, in which arrayOfInt would be usable as a pointer to int. +In Go, since arrays are values, it's meaningful (and useful) to talk +about pointers to arrays. +

+The size of the array is part of its type; however, one can declare +a slice variable to hold a reference to any array, of any size, +with the same element type. +A slice +expression has the form a[low : high], representing +the internal array indexed from low through high-1; the resulting +slice is indexed from 0 through high-low-1. +In short, slices look a lot like arrays but with +no explicit size ([] vs. [10]) and they reference a segment of +an underlying, usually anonymous, regular array. Multiple slices +can share data if they represent pieces of the same array; +multiple arrays can never share data. +

+Slices are much more common in Go programs than +regular arrays; they're more flexible, have reference semantics, +and are efficient. What they lack is the precise control of storage +layout of a regular array; if you want to have a hundred elements +of an array stored within your structure, you should use a regular +array. To create one, use a compound value constructor—an +expression formed +from a type followed by a brace-bounded expression like this: +

+

+[3]int{1,2,3}
+
+

+In this case the constructor builds an array of 3 ints. +

+When passing an array to a function, you almost always want +to declare the formal parameter to be a slice. When you call +the function, slice the array to create +(efficiently) a slice reference and pass that. +By default, the lower and upper bounds of a slice match the +ends of the existing object, so the concise notation [:] +will slice the whole array. +

+Using slices one can write this function (from sum.go): +

+{{code "progs/sum.go" `/sum/` `/^}/`}} +

+Note how the return type (int) is defined for sum by stating it +after the parameter list. +

+To call the function, we slice the array. This intricate call (we'll show +a simpler way in a moment) constructs +an array and slices it: +

+

+s := sum([3]int{1,2,3}[:])
+
+

+If you are creating a regular array but want the compiler to count the +elements for you, use ... as the array size: +

+

+s := sum([...]int{1,2,3}[:])
+
+

+That's fussier than necessary, though. +In practice, unless you're meticulous about storage layout within a +data structure, a slice itself—using empty brackets with no size—is all you need: +

+

+s := sum([]int{1,2,3})
+
+

+There are also maps, which you can initialize like this: +

+

+m := map[string]int{"one":1 , "two":2}
+
+

+The built-in function len, which returns number of elements, +makes its first appearance in sum. It works on strings, arrays, +slices, maps, and channels. +

+By the way, another thing that works on strings, arrays, slices, maps +and channels is the range clause on for loops. Instead of writing +

+

+for i := 0; i < len(a); i++ { ... }
+
+

+to loop over the elements of a slice (or map or ...) , we could write +

+

+for i, v := range a { ... }
+
+

+This assigns i to the index and v to the value of the successive +elements of the target of the range. See +Effective Go +for more examples of its use. +

+

+

An Interlude about Allocation

+

+Most types in Go are values. If you have an int or a struct +or an array, assignment +copies the contents of the object. +To allocate a new variable, use the built-in function new, which +returns a pointer to the allocated storage. +

+

+type T struct { a, b int }
+var t *T = new(T)
+
+

+or the more idiomatic +

+

+t := new(T)
+
+

+Some types—maps, slices, and channels (see below)—have reference semantics. +If you're holding a slice or a map and you modify its contents, other variables +referencing the same underlying data will see the modification. For these three +types you want to use the built-in function make: +

+

+m := make(map[string]int)
+
+

+This statement initializes a new map ready to store entries. +If you just declare the map, as in +

+

+var m map[string]int
+
+

+it creates a nil reference that cannot hold anything. To use the map, +you must first initialize the reference using make or by assignment from an +existing map. +

+Note that new(T) returns type *T while make(T) returns type +T. If you (mistakenly) allocate a reference object with new rather than make, +you receive a pointer to a nil reference, equivalent to +declaring an uninitialized variable and taking its address. +

+

An Interlude about Constants

+

+Although integers come in lots of sizes in Go, integer constants do not. +There are no constants like 0LL or 0x0UL. Instead, integer +constants are evaluated as large-precision values that +can overflow only when they are assigned to an integer variable with +too little precision to represent the value. +

+

+const hardEight = (1 << 100) >> 97  // legal
+
+

+There are nuances that deserve redirection to the legalese of the +language specification but here are some illustrative examples: +

+

+var a uint64 = 0  // a has type uint64, value 0
+a := uint64(0)    // equivalent; uses a "conversion"
+i := 0x1234       // i gets default type: int
+var j int = 1e6   // legal - 1000000 is representable in an int
+x := 1.5          // a float64, the default type for floating constants
+i3div2 := 3/2     // integer division - result is 1
+f3div2 := 3./2.   // floating-point division - result is 1.5
+
+

+Conversions only work for simple cases such as converting ints of one +sign or size to another and between integers and floating-point numbers, +plus a couple of other instances outside the scope of a tutorial. +There are no automatic numeric conversions of any kind in Go, +other than that of making constants have concrete size and type when +assigned to a variable. +

+

An I/O Package

+

+Next we'll look at a simple package for doing file I/O with an +open/close/read/write interface. Here's the start of file.go: +

+{{code "progs/file.go" `/package/` `/^}/`}} +

+The first few lines declare the name of the +package—file—and then import two packages. The os +package hides the differences +between various operating systems to give a consistent view of files and +so on; here we're going to use its error handling utilities +and reproduce the rudiments of its file I/O. +

+The other item is the low-level, external syscall package, which provides +a primitive interface to the underlying operating system's calls. +

+Next is a type definition: the type keyword introduces a type declaration, +in this case a data structure called File. +To make things a little more interesting, our File includes the name of the file +that the file descriptor refers to. +

+Because File starts with a capital letter, the type is available outside the package, +that is, by users of the package. In Go the rule about visibility of information is +simple: if a name (of a top-level type, function, method, constant or variable, or of +a structure field or method) is capitalized, users of the package may see it. Otherwise, the +name and hence the thing being named is visible only inside the package in which +it is declared. This is more than a convention; the rule is enforced by the compiler. +In Go, the term for publicly visible names is ''exported''. +

+In the case of File, all its fields are lower case and so invisible to users, but we +will soon give it some exported, upper-case methods. +

+First, though, here is a factory to create a File: +

+{{code "progs/file.go" `/newFile/` `/^}/`}} +

+This returns a pointer to a new File structure with the file descriptor and name +filled in. This code uses Go's notion of a ''composite literal'', analogous to +the ones used to build maps and arrays, to construct a new heap-allocated +object. We could write +

+

+n := new(File)
+n.fd = fd
+n.name = name
+return n
+
+

+but for simple structures like File it's easier to return the address of a +composite literal, as is done here in the return statement from newFile. +

+We can use the factory to construct some familiar, exported variables of type *File: +

+{{code "progs/file.go" `/var/` `/^.$/`}} +

+The newFile function was not exported because it's internal. The proper, +exported factory to use is OpenFile (we'll explain that name in a moment): +

+{{code "progs/file.go" `/func.OpenFile/` `/^}/`}} +

+There are a number of new things in these few lines. First, OpenFile returns +multiple values, a File and an error (more about errors in a moment). +We declare the +multi-value return as a parenthesized list of declarations; syntactically +they look just like a second parameter list. The function +syscall.Open +also has a multi-value return, which we can grab with the multi-variable +declaration on the first line; it declares r and e to hold the two values, +both of type int (although you'd have to look at the syscall package +to see that). Finally, OpenFile returns two values: a pointer to the new File +and the error. If syscall.Open fails, the file descriptor r will +be negative and newFile will return nil. +

+About those errors: The os library includes a general notion of an error. +It's a good idea to use its facility in your own interfaces, as we do here, for +consistent error handling throughout Go code. In Open we use a +conversion to translate Unix's integer errno value into the integer type +os.Errno, which implements os.Error. +

+Why OpenFile and not Open? To mimic Go's os package, which +our exercise is emulating. The os package takes the opportunity +to make the two commonest cases - open for read and create for +write - the simplest, just Open and Create. OpenFile is the +general case, analogous to the Unix system call Open. Here is +the implementation of our Open and Create; they're trivial +wrappers that eliminate common errors by capturing +the tricky standard arguments to open and, especially, to create a file: +

+{{code "progs/file.go" `/^const/` `/^}/`}} +

+{{code "progs/file.go" `/func.Create/` `/^}/`}} +

+Back to our main story. +Now that we can build Files, we can write methods for them. To declare +a method of a type, we define a function to have an explicit receiver +of that type, placed +in parentheses before the function name. Here are some methods for *File, +each of which declares a receiver variable file. +

+{{code "progs/file.go" `/Close/` "$"}} +

+There is no implicit this and the receiver variable must be used to access +members of the structure. Methods are not declared within +the struct declaration itself. The struct declaration defines only data members. +In fact, methods can be created for almost any type you name, such as an integer or +array, not just for structs. We'll see an example with arrays later. +

+The String method is so called because of a printing convention we'll +describe later. +

+The methods use the public variable os.EINVAL to return the (os.Error +version of the) Unix error code EINVAL. The os library defines a standard +set of such error values. +

+We can now use our new package: +

+{{code "progs/helloworld3.go" `/package/` "$"}} +

+The ''./'' in the import of ''./file'' tells the compiler +to use our own package rather than +something from the directory of installed packages. +(Also, ''file.go'' must be compiled before we can import the +package.) +

+Now we can compile and run the program. On Unix, this would be the result: +

+

+$ 6g file.go                       # compile file package
+$ 6g helloworld3.go                # compile main package
+$ 6l -o helloworld3 helloworld3.6  # link - no need to mention "file"
+$ helloworld3
+hello, world
+can't open file; err=No such file or directory
+$
+
+

+

Rotting cats

+

+Building on the file package, here's a simple version of the Unix utility cat(1), +progs/cat.go: +

+{{code "progs/cat.go" `/package/` "$"}} +

+By now this should be easy to follow, but the switch statement introduces some +new features. Like a for loop, an if or switch can include an +initialization statement. The switch statement in cat uses one to create variables +nr and er to hold the return values from the call to f.Read. (The if a few lines later +has the same idea.) The switch statement is general: it evaluates the cases +from top to bottom looking for the first case that matches the value; the +case expressions don't need to be constants or even integers, as long as +they all have the same type. +

+Since the switch value is just true, we could leave it off—as is also +the situation +in a for statement, a missing value means true. In fact, such a switch +is a form of if-else chain. While we're here, it should be mentioned that in +switch statements each case has an implicit break. +

+The argument to file.Stdout.Write is created by slicing the array buf. +Slices provide the standard Go way to handle I/O buffers. +

+Now let's make a variant of cat that optionally does rot13 on its input. +It's easy to do by just processing the bytes, but instead we will exploit +Go's notion of an interface. +

+The cat subroutine uses only two methods of f: Read and String, +so let's start by defining an interface that has exactly those two methods. +Here is code from progs/cat_rot13.go: +

+{{code "progs/cat_rot13.go" `/type.reader/` `/^}/`}} +

+Any type that has the two methods of reader—regardless of whatever +other methods the type may also have—is said to implement the +interface. Since file.File implements these methods, it implements the +reader interface. We could tweak the cat subroutine to accept a reader +instead of a *file.File and it would work just fine, but let's embellish a little +first by writing a second type that implements reader, one that wraps an +existing reader and does rot13 on the data. To do this, we just define +the type and implement the methods and with no other bookkeeping, +we have a second implementation of the reader interface. +

+{{code "progs/cat_rot13.go" `/type.rotate13/` `/end.of.rotate13/`}} +

+(The rot13 function called in Read is trivial and not worth reproducing here.) +

+To use the new feature, we define a flag: +

+{{code "progs/cat_rot13.go" `/rot13Flag/`}} +

+and use it from within a mostly unchanged cat function: +

+{{code "progs/cat_rot13.go" `/func.cat/` `/^}/`}} +

+(We could also do the wrapping in main and leave cat mostly alone, except +for changing the type of the argument; consider that an exercise.) +The if at the top of cat sets it all up: If the rot13 flag is true, wrap the reader +we received into a rotate13 and proceed. Note that the interface variables +are values, not pointers: the argument is of type reader, not *reader, +even though under the covers it holds a pointer to a struct. +

+Here it is in action: +

+

+$ echo abcdefghijklmnopqrstuvwxyz | ./cat
+abcdefghijklmnopqrstuvwxyz
+$ echo abcdefghijklmnopqrstuvwxyz | ./cat --rot13
+nopqrstuvwxyzabcdefghijklm
+$
+
+

+Fans of dependency injection may take cheer from how easily interfaces +allow us to substitute the implementation of a file descriptor. +

+Interfaces are a distinctive feature of Go. An interface is implemented by a +type if the type implements all the methods declared in the interface. +This means +that a type may implement an arbitrary number of different interfaces. +There is no type hierarchy; things can be much more ad hoc, +as we saw with rot13. The type file.File implements reader; it could also +implement a writer, or any other interface built from its methods that +fits the current situation. Consider the empty interface +

+

+type Empty interface {}
+
+

+Every type implements the empty interface, which makes it +useful for things like containers. +

+

Sorting

+

+Interfaces provide a simple form of polymorphism. They completely +separate the definition of what an object does from how it does it, allowing +distinct implementations to be represented at different times by the +same interface variable. +

+As an example, consider this simple sort algorithm taken from progs/sort.go: +

+{{code "progs/sort.go" `/func.Sort/` `/^}/`}} +

+The code needs only three methods, which we wrap into sort's Interface: +

+{{code "progs/sort.go" `/interface/` `/^}/`}} +

+We can apply Sort to any type that implements Len, Less, and Swap. +The sort package includes the necessary methods to allow sorting of +arrays of integers, strings, etc.; here's the code for arrays of int +

+{{code "progs/sort.go" `/type.*IntSlice/` `/Swap/`}} +

+Here we see methods defined for non-struct types. You can define methods +for any type you define and name in your package. +

+And now a routine to test it out, from progs/sortmain.go. This +uses a function in the sort package, omitted here for brevity, +to test that the result is sorted. +

+{{code "progs/sortmain.go" `/func.ints/` `/^}/`}} +

+If we have a new type we want to be able to sort, all we need to do is +to implement the three methods for that type, like this: +

+{{code "progs/sortmain.go" `/type.day/` `/Swap/`}} +

+

+

Printing

+

+The examples of formatted printing so far have been modest. In this section +we'll talk about how formatted I/O can be done well in Go. +

+We've seen simple uses of the package fmt, which +implements Printf, Fprintf, and so on. +Within the fmt package, Printf is declared with this signature: +

+

+Printf(format string, v ...interface{}) (n int, errno os.Error)
+
+

+The token ... introduces a variable-length argument list that in C would +be handled using the stdarg.h macros. +In Go, variadic functions are passed a slice of the arguments of the +specified type. In Printf's case, the declaration says ...interface{} +so the actual type is a slice of empty interface values, []interface{}. +Printf can examine the arguments by iterating over the slice +and, for each element, using a type switch or the reflection library +to interpret the value. +It's off topic here but such run-time type analysis +helps explain some of the nice properties of Go's Printf, +due to the ability of Printf to discover the type of its arguments +dynamically. +

+For example, in C each format must correspond to the type of its +argument. It's easier in many cases in Go. Instead of %llud you +can just say %d; Printf knows the size and signedness of the +integer and can do the right thing for you. The snippet +

+{{code "progs/print.go" 10 11}} +

+prints +

+

+18446744073709551615 -1
+
+

+In fact, if you're lazy the format %v will print, in a simple +appropriate style, any value, even an array or structure. The output of +

+{{code "progs/print.go" 14 20}} +

+is +

+

+18446744073709551615 {77 Sunset Strip} [1 2 3 4]
+
+

+You can drop the formatting altogether if you use Print or Println +instead of Printf. Those routines do fully automatic formatting. +The Print function just prints its elements out using the equivalent +of %v while Println inserts spaces between arguments +and adds a newline. The output of each of these two lines is identical +to that of the Printf call above. +

+{{code "progs/print.go" 21 22}} +

+If you have your own type you'd like Printf or Print to format, +just give it a String method that returns a string. The print +routines will examine the value to inquire whether it implements +the method and if so, use it rather than some other formatting. +Here's a simple example. +

+{{code "progs/print_string.go" 9 "$"}} +

+Since *testType has a String method, the +default formatter for that type will use it and produce the output +

+

+77 Sunset Strip
+
+

+Observe that the String method calls Sprint (the obvious Go +variant that returns a string) to do its formatting; special formatters +can use the fmt library recursively. +

+Another feature of Printf is that the format %T will print a string +representation of the type of a value, which can be handy when debugging +polymorphic code. +

+It's possible to write full custom print formats with flags and precisions +and such, but that's getting a little off the main thread so we'll leave it +as an exploration exercise. +

+You might ask, though, how Printf can tell whether a type implements +the String method. Actually what it does is ask if the value can +be converted to an interface variable that implements the method. +Schematically, given a value v, it does this: +

+

+

+type Stringer interface {
+    String() string
+}
+
+

+

+s, ok := v.(Stringer)  // Test whether v implements "String()"
+if ok {
+    result = s.String()
+} else {
+    result = defaultOutput(v)
+}
+
+

+The code uses a ``type assertion'' (v.(Stringer)) to test if the value stored in +v satisfies the Stringer interface; if it does, s +will become an interface variable implementing the method and ok will +be true. We then use the interface variable to call the method. +(The ''comma, ok'' pattern is a Go idiom used to test the success of +operations such as type conversion, map update, communications, and so on, +although this is the only appearance in this tutorial.) +If the value does not satisfy the interface, ok will be false. +

+In this snippet the name Stringer follows the convention that we add ''[e]r'' +to interfaces describing simple method sets like this. +

+One last wrinkle. To complete the suite, besides Printf etc. and Sprintf +etc., there are also Fprintf etc. Unlike in C, Fprintf's first argument is +not a file. Instead, it is a variable of type io.Writer, which is an +interface type defined in the io library: +

+

+type Writer interface {
+    Write(p []byte) (n int, err os.Error)
+}
+
+

+(This interface is another conventional name, this time for Write; there are also +io.Reader, io.ReadWriter, and so on.) +Thus you can call Fprintf on any type that implements a standard Write +method, not just files but also network channels, buffers, whatever +you want. +

+

Prime numbers

+

+Now we come to processes and communication—concurrent programming. +It's a big subject so to be brief we assume some familiarity with the topic. +

+A classic program in the style is a prime sieve. +(The sieve of Eratosthenes is computationally more efficient than +the algorithm presented here, but we are more interested in concurrency than +algorithmics at the moment.) +It works by taking a stream of all the natural numbers and introducing +a sequence of filters, one for each prime, to winnow the multiples of +that prime. At each step we have a sequence of filters of the primes +so far, and the next number to pop out is the next prime, which triggers +the creation of the next filter in the chain. +

+Here's a flow diagram; each box represents a filter element whose +creation is triggered by the first number that flowed from the +elements before it. +

+
+

+      +

+
+

+To create a stream of integers, we use a Go channel, which, +borrowing from CSP's descendants, represents a communications +channel that can connect two concurrent computations. +In Go, channel variables are references to a run-time object that +coordinates the communication; as with maps and slices, use +make to create a new channel. +

+Here is the first function in progs/sieve.go: +

+{{code "progs/sieve.go" `/Send/` `/^}/`}} +

+The generate function sends the sequence 2, 3, 4, 5, ... to its +argument channel, ch, using the binary communications operator <-. +Channel operations block, so if there's no recipient for the value on ch, +the send operation will wait until one becomes available. +

+The filter function has three arguments: an input channel, an output +channel, and a prime number. It copies values from the input to the +output, discarding anything divisible by the prime. The unary communications +operator <- (receive) retrieves the next value on the channel. +

+{{code "progs/sieve.go" `/Copy.the/` `/^}/`}} +

+The generator and filters execute concurrently. Go has +its own model of process/threads/light-weight processes/coroutines, +so to avoid notational confusion we call concurrently executing +computations in Go goroutines. To start a goroutine, +invoke the function, prefixing the call with the keyword go; +this starts the function running in parallel with the current +computation but in the same address space: +

+

+go sum(hugeArray) // calculate sum in the background
+
+

+If you want to know when the calculation is done, pass a channel +on which it can report back: +

+

+ch := make(chan int)
+go sum(hugeArray, ch)
+// ... do something else for a while
+result := <-ch  // wait for, and retrieve, result
+
+

+Back to our prime sieve. Here's how the sieve pipeline is stitched +together: +

+{{code "progs/sieve.go" `/func.main/` `/^}/`}} +

+The first line of main creates the initial channel to pass to generate, which it +then starts up. As each prime pops out of the channel, a new filter +is added to the pipeline and its output becomes the new value +of ch. +

+The sieve program can be tweaked to use a pattern common +in this style of programming. Here is a variant version +of generate, from progs/sieve1.go: +

+{{code "progs/sieve1.go" `/func.generate/` `/^}/`}} +

+This version does all the setup internally. It creates the output +channel, launches a goroutine running a function literal, and +returns the channel to the caller. It is a factory for concurrent +execution, starting the goroutine and returning its connection. +

+The function literal notation used in the go statement allows us to construct an +anonymous function and invoke it on the spot. Notice that the local +variable ch is available to the function literal and lives on even +after generate returns. +

+The same change can be made to filter: +

+{{code "progs/sieve1.go" `/func.filter/` `/^}/`}} +

+The sieve function's main loop becomes simpler and clearer as a +result, and while we're at it let's turn it into a factory too: +

+{{code "progs/sieve1.go" `/func.sieve/` `/^}/`}} +

+Now main's interface to the prime sieve is a channel of primes: +

+{{code "progs/sieve1.go" `/func.main/` `/^}/`}} +

+

Multiplexing

+

+With channels, it's possible to serve multiple independent client goroutines without +writing an explicit multiplexer. The trick is to send the server a channel in the message, +which it will then use to reply to the original sender. +A realistic client-server program is a lot of code, so here is a very simple substitute +to illustrate the idea. It starts by defining a request type, which embeds a channel +that will be used for the reply. +

+{{code "progs/server.go" `/type.request/` `/^}/`}} +

+The server will be trivial: it will do simple binary operations on integers. Here's the +code that invokes the operation and responds to the request: +

+{{code "progs/server.go" `/type.binOp/` `/^}/`}} +

+The type declaration makes binOp represent a function taking two integers and +returning a third. +

+The server routine loops forever, receiving requests and, to avoid blocking due to +a long-running operation, starting a goroutine to do the actual work. +

+{{code "progs/server.go" `/func.server/` `/^}/`}} +

+We construct a server in a familiar way, starting it and returning a channel +connected to it: +

+{{code "progs/server.go" `/func.startServer/` `/^}/`}} +

+Here's a simple test. It starts a server with an addition operator and sends out +N requests without waiting for the replies. Only after all the requests are sent +does it check the results. +

+{{code "progs/server.go" `/func.main/` `/^}/`}} +

+One annoyance with this program is that it doesn't shut down the server cleanly; when main returns +there are a number of lingering goroutines blocked on communication. To solve this, +we can provide a second, quit channel to the server: +

+{{code "progs/server1.go" `/func.startServer/` `/^}/`}} +

+It passes the quit channel to the server function, which uses it like this: +

+{{code "progs/server1.go" `/func.server/` `/^}/`}} +

+Inside server, the select statement chooses which of the multiple communications +listed by its cases can proceed. If all are blocked, it waits until one can proceed; if +multiple can proceed, it chooses one at random. In this instance, the select allows +the server to honor requests until it receives a quit message, at which point it +returns, terminating its execution. +

+

+All that's left is to strobe the quit channel +at the end of main: +

+{{code "progs/server1.go" `/adder,.quit/`}} +... +{{code "progs/server1.go" `/quit....true/`}} +

+There's a lot more to Go programming and concurrent programming in general but this +quick tour should give you some of the basics. diff --git a/doc/godocs.js b/doc/godocs.js new file mode 100644 index 000000000..946c4c39f --- /dev/null +++ b/doc/godocs.js @@ -0,0 +1,190 @@ +// Except as noted, this content is licensed under Creative Commons +// Attribution 3.0 + +/* A little code to ease navigation of these documents. + * + * On window load we: + * + Generate a table of contents (godocs_generateTOC) + * + Add links up to the top of the doc from each section (godocs_addTopLinks) + */ + +/* We want to do some stuff on page load (after the HTML is rendered). + So listen for that: + */ +function bindEvent(el, e, fn) { + if (el.addEventListener){ + el.addEventListener(e, fn, false); + } else if (el.attachEvent){ + el.attachEvent('on'+e, fn); + } +} +bindEvent(window, 'load', godocs_onload); + +function godocs_onload() { + godocs_bindSearchEvents(); + godocs_generateTOC(); + godocs_addTopLinks(); +} + +function godocs_bindSearchEvents() { + var search = document.getElementById('search'); + if (!search) { + // no search box (index disabled) + return; + } + function clearInactive() { + if (search.className == "inactive") { + search.value = ""; + search.className = ""; + } + } + function restoreInactive() { + if (search.value != "") { + return; + } + if (search.type != "search") { + search.value = search.getAttribute("placeholder"); + } + search.className = "inactive"; + } + restoreInactive(); + bindEvent(search, 'focus', clearInactive); + bindEvent(search, 'blur', restoreInactive); +} + +/* Generates a table of contents: looks for h2 and h3 elements and generates + * links. "Decorates" the element with id=="nav" with this table of contents. + */ +function godocs_generateTOC() { + var navbar = document.getElementById('nav'); + if (!navbar) { return; } + + var toc_items = []; + + var i; + for (i = 0; i < navbar.parentNode.childNodes.length; i++) { + var node = navbar.parentNode.childNodes[i]; + if ((node.tagName == 'h2') || (node.tagName == 'H2')) { + if (!node.id) { + node.id = 'tmp_' + i; + } + var text = godocs_nodeToText(node); + if (!text) { continue; } + + var textNode = document.createTextNode(text); + + var link = document.createElement('a'); + link.href = '#' + node.id; + link.appendChild(textNode); + + // Then create the item itself + var item = document.createElement('dt'); + + item.appendChild(link); + toc_items.push(item); + } + if ((node.tagName == 'h3') || (node.tagName == 'H3')) { + if (!node.id) { + node.id = 'tmp_' + i; + } + var text = godocs_nodeToText(node); + if (!text) { continue; } + + var textNode = document.createTextNode(text); + + var link = document.createElement('a'); + link.href = '#' + node.id; + link.appendChild(textNode); + + // Then create the item itself + var item = document.createElement('dd'); + + item.appendChild(link); + toc_items.push(item); + } + } + + if (toc_items.length <= 1) { return; } + + var dl1 = document.createElement('dl'); + var dl2 = document.createElement('dl'); + + var split_index = (toc_items.length / 2) + 1; + if (split_index < 8) { + split_index = toc_items.length; + } + + for (i = 0; i < split_index; i++) { + dl1.appendChild(toc_items[i]); + } + for (/* keep using i */; i < toc_items.length; i++) { + dl2.appendChild(toc_items[i]); + } + + var tocTable = document.createElement('table'); + navbar.appendChild(tocTable); + tocTable.className = 'unruled'; + var tocBody = document.createElement('tbody'); + tocTable.appendChild(tocBody); + + var tocRow = document.createElement('tr'); + tocBody.appendChild(tocRow); + + // 1st column + var tocCell = document.createElement('td'); + tocCell.className = 'first'; + tocRow.appendChild(tocCell); + tocCell.appendChild(dl1); + + // 2nd column + tocCell = document.createElement('td'); + tocRow.appendChild(tocCell); + tocCell.appendChild(dl2); +} + +/* Returns the "This sweet header" from

This sweet header

. + * Takes a node, returns a string. + */ +function godocs_nodeToText(node) { + var TEXT_NODE = 3; // Defined in Mozilla but not MSIE :( + + var text = ''; + for (var j = 0; j != node.childNodes.length; j++) { + var child = node.childNodes[j]; + if (child.nodeType == TEXT_NODE) { + if (child.nodeValue != '[Top]') { //ok, that's a hack, but it works. + text = text + child.nodeValue; + } + } else { + text = text + godocs_nodeToText(child); + } + } + return text; +} + +/* For each H2 heading, add a link up to the #top of the document. + * (As part of this: ensure existence of 'top' named anchor link + * (theoretically at doc's top).) + */ +function godocs_addTopLinks() { + /* Make sure there's a "top" to link to. */ + var top = document.getElementById('top'); + if (!top) { + document.body.id = 'top'; + } + + if (!document.getElementsByTagName) return; // no browser support + + var headers = document.getElementsByTagName('h2'); + + for (var i = 0; i < headers.length; i++) { + var span = document.createElement('span'); + span.className = 'navtop'; + var link = document.createElement('a'); + span.appendChild(link); + link.href = '#top'; + var textNode = document.createTextNode('[Top]'); + link.appendChild(textNode); + headers[i].appendChild(span); + } +} diff --git a/doc/gopher/appenginegopher.jpg b/doc/gopher/appenginegopher.jpg new file mode 100644 index 000000000..0a6430666 Binary files /dev/null and b/doc/gopher/appenginegopher.jpg differ diff --git a/doc/gopher/appenginegophercolor.jpg b/doc/gopher/appenginegophercolor.jpg new file mode 100644 index 000000000..68795a99b Binary files /dev/null and b/doc/gopher/appenginegophercolor.jpg differ diff --git a/doc/gopher/appenginelogo.gif b/doc/gopher/appenginelogo.gif new file mode 100644 index 000000000..46b3c1eeb Binary files /dev/null and b/doc/gopher/appenginelogo.gif differ diff --git a/doc/gopher/bumper.png b/doc/gopher/bumper.png new file mode 100644 index 000000000..6b41c1fd0 Binary files /dev/null and b/doc/gopher/bumper.png differ diff --git a/doc/gopher/bumper192x108.png b/doc/gopher/bumper192x108.png new file mode 100644 index 000000000..470a74df5 Binary files /dev/null and b/doc/gopher/bumper192x108.png differ diff --git a/doc/gopher/bumper320x180.png b/doc/gopher/bumper320x180.png new file mode 100644 index 000000000..5b31b5d31 Binary files /dev/null and b/doc/gopher/bumper320x180.png differ diff --git a/doc/gopher/bumper480x270.png b/doc/gopher/bumper480x270.png new file mode 100644 index 000000000..621f51b65 Binary files /dev/null and b/doc/gopher/bumper480x270.png differ diff --git a/doc/gopher/bumper640x360.png b/doc/gopher/bumper640x360.png new file mode 100644 index 000000000..9c898d0c7 Binary files /dev/null and b/doc/gopher/bumper640x360.png differ diff --git a/doc/gopher/gopherbw.png b/doc/gopher/gopherbw.png new file mode 100644 index 000000000..48a08cc61 Binary files /dev/null and b/doc/gopher/gopherbw.png differ diff --git a/doc/gopher/gophercolor.png b/doc/gopher/gophercolor.png new file mode 100644 index 000000000..b48ffba37 Binary files /dev/null and b/doc/gopher/gophercolor.png differ diff --git a/doc/gopher/gophercolor16x16.png b/doc/gopher/gophercolor16x16.png new file mode 100644 index 000000000..48854ff3b Binary files /dev/null and b/doc/gopher/gophercolor16x16.png differ diff --git a/doc/ie.css b/doc/ie.css new file mode 100644 index 000000000..bb89d54be --- /dev/null +++ b/doc/ie.css @@ -0,0 +1 @@ +#nav-main li { display: inline; } diff --git a/doc/install.html b/doc/install.html new file mode 100644 index 000000000..a1bc89982 --- /dev/null +++ b/doc/install.html @@ -0,0 +1,474 @@ + + +

Introduction

+ +

Go is an open source project, distributed under a +BSD-style license. +This document explains how to check out the sources, +build them on your own machine, and run them. +

+ +
+ +

+There are two distinct ways to experiment with Go. +This document focuses on the gc Go +compiler and tools (6g, 8g etc.). +For information on how to use gccgo, a more traditional +compiler using the GCC back end, see +Setting up and using gccgo. +

+ +

+The Go compilers support three instruction sets. +There are important differences in the quality of the compilers for the different +architectures. +

+ +
+
+ amd64 (a.k.a. x86-64); 6g,6l,6c,6a +
+
+ The most mature implementation. The compiler has an effective optimizer + (registerizer) and generates good code (although gccgo + can do noticeably better sometimes). +
+
+ 386 (a.k.a. x86 or x86-32); 8g,8l,8c,8a +
+
+ Comparable to the amd64 port. +
+
+ arm (a.k.a. ARM); 5g,5l,5c,5a +
+
+ Incomplete. + It only supports Linux binaries, the optimizer is incomplete, + and floating point uses the VFP unit. + However, all tests pass. + Work on the optimizer is continuing. + Tested against a Nexus One. +
+
+ +

+Except for things like low-level operating system interface code, the run-time +support is the same in all ports and includes a mark-and-sweep garbage collector +(a fancier one is in the works), efficient array and string slicing, +support for segmented stacks, and a strong goroutine implementation. +

+ +

+The compilers can target the FreeBSD, Linux, +and OS X (a.k.a. Darwin) operating systems. +(A port to Microsoft Windows is in progress but incomplete. See the +Windows Port +page for details.) +The full set of supported combinations is listed in the discussion of +environment variables below. +

+ +
+ +

Install C tools, if needed

+ +

The Go tool chain is written in C. +To build it, you need these programs installed: +

    +
  • GCC, +
  • the standard C libraries, +
  • the parser generator Bison, +
  • GNU make (version 3.81 or later), +and +
  • awk. +
+

+ +

On OS X, they can be +installed as part of +Xcode. +

+ +

On Ubuntu/Debian, use sudo apt-get install bison gawk gcc libc6-dev +make. If you want to build 32-bit binaries on a 64-bit system you'll +also need the libc6-dev-i386 package. +

+ +

Install Mercurial, if needed

+ +

+To perform the next step you must have Mercurial installed. (Check that you have an hg command.) This suffices to install Mercurial on most systems: +

+
+sudo easy_install mercurial
+
+(On Ubuntu/Debian, you might try apt-get install python-setuptools +python-dev build-essential first. The Mercurial in your distribution's +package repository will most likely be old and broken.) +

+

+If that fails, try installing manually from the Mercurial Download page.

+

+ +

+Mercurial versions 1.7.x and up require the configuration of +Certification Authorities +(CAs). Error messages of the form: +

+
+warning: go.googlecode.com certificate with fingerprint b1:af: ... bc not verified (check hostfingerprints or web.cacerts config setting)
+
+

+when using Mercurial indicate that the CAs are missing. +Check your Mercurial version (hg --version) and +configure the CAs +if necessary. +

+ +

Fetch the repository

+ +

+

Go will install to a directory named go. +Change to the directory that will be its parent +and make sure the go directory does not exist. +Then check out the repository:

+ +
+$ hg clone -u release https://go.googlecode.com/hg/ go
+
+ +

Install Go

+ +

+To build the Go distribution, run +

+ +
+$ cd go/src
+$ ./all.bash
+
+ +

+If all goes well, it will finish by printing output like: +

+ +
+ALL TESTS PASSED
+
+---
+Installed Go for linux/amd64 in /home/you/go.
+Installed commands in /home/you/go/bin.
+*** You need to add /home/you/go/bin to your $PATH. ***
+The compiler is 6g.
+
+ +

+where the details on the last few lines reflect the operating system, +architecture, and root directory used during the install. +

+ +
+ +

For more information about ways to control the build, +see the discussion of environment variables below.

+
+ +

Writing programs

+ +

+Given a file file.go, compile it using +

+ +
+$ 6g file.go
+
+ +

+6g is the Go compiler for amd64; it will write the output +in file.6. The ‘6’ identifies +files for the amd64 architecture. +The identifier letters for 386 and arm +are ‘8’ and ‘5’. +That is, if you were compiling for 386, you would use +8g and the output would be named file.8. +

+ +

+To link the file, use +

+ +
+$ 6l file.6
+
+ +

+and to run it +

+ +
+$ ./6.out
+
+ +

A complete example: +

+ +
+$ cat >hello.go <<EOF
+package main
+
+import "fmt"
+
+func main() {
+	fmt.Printf("hello, world\n")
+}
+EOF
+$ 6g hello.go
+$ 6l hello.6
+$ ./6.out
+hello, world
+$
+
+ +

+There is no need to list hello.6's package dependencies +(in this case, package fmt) on the 6l +command line. +The linker learns about them by reading hello.6. +

+ +
+

+To build more complicated programs, you will probably +want to use a +Makefile. +There are examples in places like +go/src/cmd/godoc/Makefile +and go/src/pkg/*/Makefile. +The +document +about contributing to the Go project +gives more detail about +the process of building and testing Go programs. +

+
+ +

What's next

+ +

+Start by reading the Go Tutorial. +

+ +

+Build a web application by following the Wiki +Codelab. +

+ +

+Read Effective Go to learn about writing +idiomatic Go code. +

+ +

+For the full story, consult Go's extensive +documentation. +

+ +

Keeping up with releases

+ +

+The Go project maintains two stable tags in its Mercurial repository: +release and weekly. +The weekly tag is updated about once a week, and should be used by +those who want to track the project's development. +The release tag is given, less often, to those weekly releases +that have proven themselves to be robust. +

+ +

+Most Go users will want to keep their Go installation at the latest +release tag. +New releases are announced on the +golang-announce +mailing list. +

+ +

+To update an existing tree to the latest release, you can run: +

+ +
+$ cd go/src
+$ hg pull
+$ hg update release
+$ ./all.bash
+
+ +

+To use the weekly tag run hg update weekly instead. +

+ +

Community resources

+ +

+For real-time help, there may be users or developers on +#go-nuts on the Freenode IRC server. +

+ +

+The official mailing list for discussion of the Go language is +Go Nuts. +

+ +

+Bugs can be reported using the Go issue tracker. +

+ +

+For those who wish to keep up with development, +there is another mailing list, golang-checkins, +that receives a message summarizing each checkin to the Go repository. +

+ +

Environment variables

+ +

+The Go compilation environment can be customized by environment variables. +None are required by the build, but you may wish to set them +to override the defaults. +

+ +
+
+ $GOROOT +
+
+ The root of the Go tree, often $HOME/go. + This defaults to the parent of the directory where all.bash is run. + If you choose not to set $GOROOT, you must + run gomake instead of make or gmake + when developing Go programs using the conventional makefiles. +
+ +
+ $GOROOT_FINAL +
+
+ The value assumed by installed binaries and scripts when + $GOROOT is not set. + It defaults to the value used for $GOROOT. + If you want to build the Go tree in one location + but move it elsewhere after the build, set + $GOROOT_FINAL to the eventual location. +
+ +
+$GOOS and $GOARCH +
+
+ The name of the target operating system and compilation architecture. + These default to the values of $GOHOSTOS and + $GOHOSTARCH respectively (described below). + +

+ Choices for $GOOS are linux, + freebsd, + darwin (Mac OS X 10.5 or 10.6), + and windows (Windows, an incomplete port). + Choices for $GOARCH are amd64 (64-bit x86, the most mature port), + 386 (32-bit x86), and + arm (32-bit ARM, an incomplete port). + The valid combinations of $GOOS and $GOARCH are: + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
$GOOS $GOARCH
darwin 386
darwin amd64
freebsd 386
freebsd amd64
linux 386
linux amd64
linux arm incomplete
windows 386 incomplete
+

+ +
+$GOHOSTOS and $GOHOSTARCH +
+
+ The name of the host operating system and compilation architecture. + These default to the local system's operating system and + architecture. + +

+ Valid choices are the same as for $GOOS and + $GOARCH, listed above. + The specified values must be compatible with the local system. + For example, you should not set $GOHOSTARCH to + arm on an x86 system. +

+ +
+$GOBIN +
+
+ The location where binaries will be installed. + The default is $GOROOT/bin. + After installing, you will want to arrange to add this + directory to your $PATH, so you can use the tools. +
+ +
+$GOARM (arm, default=6) +
+
+ The ARM architecture version the run-time libraries should target. + ARMv6 cores have more efficient synchronization primitives. Setting + $GOARM to 5 will compile the run-time libraries using + just SWP instructions that work on older architectures as well. + Running v6 code on an older core will cause an illegal instruction trap. +
+
+ +

+Note that $GOARCH and $GOOS identify the +target environment, not the environment you are running on. +In effect, you are always cross-compiling. +By architecture, we mean the kind of binaries +that the target environment can run: +an x86-64 system running a 32-bit-only operating system +must set GOARCH to 386, +not amd64. +

+ +

+If you choose to override the defaults, +set these variables in your shell profile ($HOME/.bashrc, +$HOME/.profile, or equivalent). The settings might look +something like this: +

+ +
+export GOROOT=$HOME/go
+export GOARCH=386
+export GOOS=linux
+
diff --git a/doc/logo-153x55.png b/doc/logo-153x55.png new file mode 100644 index 000000000..4a2446ce7 Binary files /dev/null and b/doc/logo-153x55.png differ diff --git a/doc/logo.png b/doc/logo.png new file mode 100644 index 000000000..076ce398e Binary files /dev/null and b/doc/logo.png differ diff --git a/doc/makehtml b/doc/makehtml new file mode 100755 index 000000000..2418c68fa --- /dev/null +++ b/doc/makehtml @@ -0,0 +1,17 @@ +#!/bin/sh +# Copyright 2009 The Go Authors. All rights reserved. +# Use of this source code is governed by a BSD-style +# license that can be found in the LICENSE file. + +set -e + +TMPL=${1:-go_tutorial.tmpl} # input file +HTML=$(basename $TMPL .tmpl).html # output file (basename) + +if ! test -w $HTML +then + echo 1>&2 makehtml: cannot open $HTML for write + exit 1 +fi + +make && ./tmpltohtml $TMPL > $HTML diff --git a/doc/play_overlay.png b/doc/play_overlay.png new file mode 100644 index 000000000..20ef7f399 Binary files /dev/null and b/doc/play_overlay.png differ diff --git a/doc/playground.html b/doc/playground.html new file mode 100644 index 000000000..01d3adc9c --- /dev/null +++ b/doc/playground.html @@ -0,0 +1,27 @@ + + +
+

+The Go Playground is a web service that runs on +golang.org's servers. +The service receives a Go program, compiles, links, and runs the program inside +a sandbox, then returns the output. +

+ +

+There are limitations to the programs that can be run in the Playground. +They must be single-threaded (but they may use many goroutines). +There are also limits on execution time, and CPU and memory usage. +The Playground can access only a subset of the standard library +(notably absent are network and file system access). +Therefore, the only communication a Playground program has to the outside world +is via standard output. +

+ +
+ +
+ +
+ +
diff --git a/doc/popups.js b/doc/popups.js new file mode 100644 index 000000000..23ccc8c75 --- /dev/null +++ b/doc/popups.js @@ -0,0 +1,24 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +function godocs_bindPopups(data) { + + $('#content span').bind('mouseenter', function() { + var id = $(this).attr('id'); + //var txt = $(this).text(); + if (typeof data[id] == 'undefined') + return; + var content = data[id]; + + var $el = $('.popup', this); + if (!$el.length) { // create it + $el = $(''); + $el.prependTo(this).css($(this).offset()).text(content); + } + }); + $('#content span').bind('mouseleave', function() { + $('.popup', this).remove(); + }); + +} diff --git a/doc/progs/cat.go b/doc/progs/cat.go new file mode 100644 index 000000000..9f0b8d4a3 --- /dev/null +++ b/doc/progs/cat.go @@ -0,0 +1,47 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package main + +import ( + "./file" + "flag" + "fmt" + "os" +) + +func cat(f *file.File) { + const NBUF = 512 + var buf [NBUF]byte + for { + switch nr, er := f.Read(buf[:]); true { + case nr < 0: + fmt.Fprintf(os.Stderr, "cat: error reading from %s: %s\n", f.String(), er.String()) + os.Exit(1) + case nr == 0: // EOF + return + case nr > 0: + if nw, ew := file.Stdout.Write(buf[0:nr]); nw != nr { + fmt.Fprintf(os.Stderr, "cat: error writing from %s: %s\n", f.String(), ew.String()) + os.Exit(1) + } + } + } +} + +func main() { + flag.Parse() // Scans the arg list and sets up flags + if flag.NArg() == 0 { + cat(file.Stdin) + } + for i := 0; i < flag.NArg(); i++ { + f, err := file.Open(flag.Arg(i)) + if f == nil { + fmt.Fprintf(os.Stderr, "cat: can't open %s: error %s\n", flag.Arg(i), err) + os.Exit(1) + } + cat(f) + f.Close() + } +} diff --git a/doc/progs/cat_rot13.go b/doc/progs/cat_rot13.go new file mode 100644 index 000000000..0eefe7cfc --- /dev/null +++ b/doc/progs/cat_rot13.go @@ -0,0 +1,90 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package main + +import ( + "./file" + "flag" + "fmt" + "os" +) + +var rot13Flag = flag.Bool("rot13", false, "rot13 the input") + +func rot13(b byte) byte { + if 'a' <= b && b <= 'z' { + b = 'a' + ((b-'a')+13)%26 + } + if 'A' <= b && b <= 'Z' { + b = 'A' + ((b-'A')+13)%26 + } + return b +} + +type reader interface { + Read(b []byte) (ret int, err os.Error) + String() string +} + +type rotate13 struct { + source reader +} + +func newRotate13(source reader) *rotate13 { + return &rotate13{source} +} + +func (r13 *rotate13) Read(b []byte) (ret int, err os.Error) { + r, e := r13.source.Read(b) + for i := 0; i < r; i++ { + b[i] = rot13(b[i]) + } + return r, e +} + +func (r13 *rotate13) String() string { + return r13.source.String() +} +// end of rotate13 implementation + +func cat(r reader) { + const NBUF = 512 + var buf [NBUF]byte + + if *rot13Flag { + r = newRotate13(r) + } + for { + switch nr, er := r.Read(buf[:]); { + case nr < 0: + fmt.Fprintf(os.Stderr, "cat: error reading from %s: %s\n", r.String(), er.String()) + os.Exit(1) + case nr == 0: // EOF + return + case nr > 0: + nw, ew := file.Stdout.Write(buf[0:nr]) + if nw != nr { + fmt.Fprintf(os.Stderr, "cat: error writing from %s: %s\n", r.String(), ew.String()) + os.Exit(1) + } + } + } +} + +func main() { + flag.Parse() // Scans the arg list and sets up flags + if flag.NArg() == 0 { + cat(file.Stdin) + } + for i := 0; i < flag.NArg(); i++ { + f, err := file.Open(flag.Arg(i)) + if f == nil { + fmt.Fprintf(os.Stderr, "cat: can't open %s: error %s\n", flag.Arg(i), err) + os.Exit(1) + } + cat(f) + f.Close() + } +} diff --git a/doc/progs/echo.go b/doc/progs/echo.go new file mode 100644 index 000000000..3260edd74 --- /dev/null +++ b/doc/progs/echo.go @@ -0,0 +1,32 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package main + +import ( + "os" + "flag" // command line option parser +) + +var omitNewline = flag.Bool("n", false, "don't print final newline") + +const ( + Space = " " + Newline = "\n" +) + +func main() { + flag.Parse() // Scans the arg list and sets up flags + var s string = "" + for i := 0; i < flag.NArg(); i++ { + if i > 0 { + s += Space + } + s += flag.Arg(i) + } + if !*omitNewline { + s += Newline + } + os.Stdout.WriteString(s) +} diff --git a/doc/progs/file.go b/doc/progs/file.go new file mode 100644 index 000000000..2875ce73a --- /dev/null +++ b/doc/progs/file.go @@ -0,0 +1,89 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package file + +import ( + "os" + "syscall" +) + +type File struct { + fd int // file descriptor number + name string // file name at Open time +} + +func newFile(fd int, name string) *File { + if fd < 0 { + return nil + } + return &File{fd, name} +} + +var ( + Stdin = newFile(syscall.Stdin, "/dev/stdin") + Stdout = newFile(syscall.Stdout, "/dev/stdout") + Stderr = newFile(syscall.Stderr, "/dev/stderr") +) + +func OpenFile(name string, mode int, perm uint32) (file *File, err os.Error) { + r, e := syscall.Open(name, mode, perm) + if e != 0 { + err = os.Errno(e) + } + return newFile(r, name), err +} + +const ( + O_RDONLY = syscall.O_RDONLY + O_RDWR = syscall.O_RDWR + O_CREATE = syscall.O_CREAT + O_TRUNC = syscall.O_TRUNC +) + +func Open(name string) (file *File, err os.Error) { + return OpenFile(name, O_RDONLY, 0) +} + +func Create(name string) (file *File, err os.Error) { + return OpenFile(name, O_RDWR|O_CREATE|O_TRUNC, 0666) +} + +func (file *File) Close() os.Error { + if file == nil { + return os.EINVAL + } + e := syscall.Close(file.fd) + file.fd = -1 // so it can't be closed again + if e != 0 { + return os.Errno(e) + } + return nil +} + +func (file *File) Read(b []byte) (ret int, err os.Error) { + if file == nil { + return -1, os.EINVAL + } + r, e := syscall.Read(file.fd, b) + if e != 0 { + err = os.Errno(e) + } + return int(r), err +} + +func (file *File) Write(b []byte) (ret int, err os.Error) { + if file == nil { + return -1, os.EINVAL + } + r, e := syscall.Write(file.fd, b) + if e != 0 { + err = os.Errno(e) + } + return int(r), err +} + +func (file *File) String() string { + return file.name +} diff --git a/doc/progs/file_windows.go b/doc/progs/file_windows.go new file mode 100644 index 000000000..03003a3f7 --- /dev/null +++ b/doc/progs/file_windows.go @@ -0,0 +1,89 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package file + +import ( + "os" + "syscall" +) + +type File struct { + fd syscall.Handle // file descriptor number + name string // file name at Open time +} + +func newFile(fd syscall.Handle, name string) *File { + if fd < 0 { + return nil + } + return &File{fd, name} +} + +var ( + Stdin = newFile(syscall.Stdin, "/dev/stdin") + Stdout = newFile(syscall.Stdout, "/dev/stdout") + Stderr = newFile(syscall.Stderr, "/dev/stderr") +) + +func OpenFile(name string, mode int, perm uint32) (file *File, err os.Error) { + r, e := syscall.Open(name, mode, perm) + if e != 0 { + err = os.Errno(e) + } + return newFile(r, name), err +} + +const ( + O_RDONLY = syscall.O_RDONLY + O_RDWR = syscall.O_RDWR + O_CREATE = syscall.O_CREAT + O_TRUNC = syscall.O_TRUNC +) + +func Open(name string) (file *File, err os.Error) { + return OpenFile(name, O_RDONLY, 0) +} + +func Create(name string) (file *File, err os.Error) { + return OpenFile(name, O_RDWR|O_CREATE|O_TRUNC, 0666) +} + +func (file *File) Close() os.Error { + if file == nil { + return os.EINVAL + } + e := syscall.Close(file.fd) + file.fd = syscall.InvalidHandle // so it can't be closed again + if e != 0 { + return os.Errno(e) + } + return nil +} + +func (file *File) Read(b []byte) (ret int, err os.Error) { + if file == nil { + return -1, os.EINVAL + } + r, e := syscall.Read(file.fd, b) + if e != 0 { + err = os.Errno(e) + } + return int(r), err +} + +func (file *File) Write(b []byte) (ret int, err os.Error) { + if file == nil { + return -1, os.EINVAL + } + r, e := syscall.Write(file.fd, b) + if e != 0 { + err = os.Errno(e) + } + return int(r), err +} + +func (file *File) String() string { + return file.name +} diff --git a/doc/progs/helloworld.go b/doc/progs/helloworld.go new file mode 100644 index 000000000..8185038d9 --- /dev/null +++ b/doc/progs/helloworld.go @@ -0,0 +1,11 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package main + +import fmt "fmt" // Package implementing formatted I/O. + +func main() { + fmt.Printf("Hello, world; or Καλημέρα κόσμε; or こんにちは 世界\n") +} diff --git a/doc/progs/helloworld3.go b/doc/progs/helloworld3.go new file mode 100644 index 000000000..2011513b7 --- /dev/null +++ b/doc/progs/helloworld3.go @@ -0,0 +1,21 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package main + +import ( + "./file" + "fmt" + "os" +) + +func main() { + hello := []byte("hello, world\n") + file.Stdout.Write(hello) + f, err := file.Open("/does/not/exist") + if f == nil { + fmt.Printf("can't open file; err=%s\n", err.String()) + os.Exit(1) + } +} diff --git a/doc/progs/print.go b/doc/progs/print.go new file mode 100644 index 000000000..8f44ba8c6 --- /dev/null +++ b/doc/progs/print.go @@ -0,0 +1,23 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package main + +import "fmt" + +func main() { + var u64 uint64 = 1<<64 - 1 + fmt.Printf("%d %d\n", u64, int64(u64)) + + // harder stuff + type T struct { + a int + b string + } + t := T{77, "Sunset Strip"} + a := []int{1, 2, 3, 4} + fmt.Printf("%v %v %v\n", u64, t, a) + fmt.Print(u64, " ", t, " ", a, "\n") + fmt.Println(u64, t, a) +} diff --git a/doc/progs/print_string.go b/doc/progs/print_string.go new file mode 100644 index 000000000..46ab1d91a --- /dev/null +++ b/doc/progs/print_string.go @@ -0,0 +1,21 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package main + +import "fmt" + +type testType struct { + a int + b string +} + +func (t *testType) String() string { + return fmt.Sprint(t.a) + " " + t.b +} + +func main() { + t := &testType{77, "Sunset Strip"} + fmt.Println(t) +} diff --git a/doc/progs/run b/doc/progs/run new file mode 100755 index 000000000..81781c9d2 --- /dev/null +++ b/doc/progs/run @@ -0,0 +1,85 @@ +#!/usr/bin/env bash +# Copyright 2009 The Go Authors. All rights reserved. +# Use of this source code is governed by a BSD-style +# license that can be found in the LICENSE file. + +set -e + +eval $(gomake --no-print-directory -f ../../src/Make.inc go-env) + +if [ -z "$O" ]; then + echo 'missing $O - maybe no Make.$GOARCH?' 1>&2 + exit 1 +fi + +rm -f *.$O + +if [ "$GOOS" = "windows" ];then + $GC -o file.8 file_windows.go +else + $GC file.go +fi + +for i in \ + helloworld.go \ + helloworld3.go \ + echo.go \ + cat.go \ + cat_rot13.go \ + sum.go \ + sort.go \ + sortmain.go \ + print.go \ + print_string.go \ + sieve.go \ + sieve1.go \ + server1.go \ + strings.go \ +; do + $GC $i +done + +function testit { + $LD $1.$O + x=$(echo $(./$O.out $2 2>&1)) # extra echo canonicalizes + if [ "$x" != "$3" ] + then + echo $1 failed: '"'$x'"' is not '"'$3'"' + fi +} + +function testitpipe { + $LD $1.$O + x=$(echo $(./$O.out | $2 2>&1)) # extra echo canonicalizes + if [ "$x" != "$3" ] + then + echo $1 failed: '"'$x'"' is not '"'$3'"' + fi +} + + +testit helloworld "" "Hello, world; or Καλημέρα κόσμε; or こんにちは 世界" +testit helloworld3 "" "hello, world can't open file; err=no such file or directory" +testit echo "hello, world" "hello, world" +testit sum "" "6" +testit strings "" "" + +alphabet=abcdefghijklmnopqrstuvwxyz +rot13=nopqrstuvwxyzabcdefghijklm +echo $alphabet | testit cat "" $alphabet +echo $alphabet | testit cat_rot13 "--rot13" $rot13 +echo $rot13 | testit cat_rot13 "--rot13" $alphabet + +testit sortmain "" "Sunday Monday Tuesday Wednesday Thursday Friday Saturday" + +testit print "" "18446744073709551615 -1 18446744073709551615 {77 Sunset Strip} [1 2 3 4] 18446744073709551615 {77 Sunset Strip} [1 2 3 4] 18446744073709551615 {77 Sunset Strip} [1 2 3 4]" +testit print_string "" "77 Sunset Strip" + +testitpipe sieve "sed 10q" "2 3 5 7 11 13 17 19 23 29" +testitpipe sieve "sed 10q" "2 3 5 7 11 13 17 19 23 29" + +# server hangs; don't run it, just compile it +$GC server.go +testit server1 "" "" + +rm -f $O.out *.$O diff --git a/doc/progs/server.go b/doc/progs/server.go new file mode 100644 index 000000000..b498b53a6 --- /dev/null +++ b/doc/progs/server.go @@ -0,0 +1,51 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package main + +import "fmt" + +type request struct { + a, b int + replyc chan int +} + +type binOp func(a, b int) int + +func run(op binOp, req *request) { + reply := op(req.a, req.b) + req.replyc <- reply +} + +func server(op binOp, service chan *request) { + for { + req := <-service + go run(op, req) // don't wait for it + } +} + +func startServer(op binOp) chan *request { + req := make(chan *request) + go server(op, req) + return req +} + +func main() { + adder := startServer(func(a, b int) int { return a + b }) + const N = 100 + var reqs [N]request + for i := 0; i < N; i++ { + req := &reqs[i] + req.a = i + req.b = i + N + req.replyc = make(chan int) + adder <- req + } + for i := N - 1; i >= 0; i-- { // doesn't matter what order + if <-reqs[i].replyc != N+2*i { + fmt.Println("fail at", i) + } + } + fmt.Println("done") +} diff --git a/doc/progs/server1.go b/doc/progs/server1.go new file mode 100644 index 000000000..a4093924b --- /dev/null +++ b/doc/progs/server1.go @@ -0,0 +1,56 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package main + +import "fmt" + +type request struct { + a, b int + replyc chan int +} + +type binOp func(a, b int) int + +func run(op binOp, req *request) { + reply := op(req.a, req.b) + req.replyc <- reply +} + +func server(op binOp, service chan *request, quit chan bool) { + for { + select { + case req := <-service: + go run(op, req) // don't wait for it + case <-quit: + return + } + } +} + +func startServer(op binOp) (service chan *request, quit chan bool) { + service = make(chan *request) + quit = make(chan bool) + go server(op, service, quit) + return service, quit +} + +func main() { + adder, quit := startServer(func(a, b int) int { return a + b }) + const N = 100 + var reqs [N]request + for i := 0; i < N; i++ { + req := &reqs[i] + req.a = i + req.b = i + N + req.replyc = make(chan int) + adder <- req + } + for i := N - 1; i >= 0; i-- { // doesn't matter what order + if <-reqs[i].replyc != N+2*i { + fmt.Println("fail at", i) + } + } + quit <- true +} diff --git a/doc/progs/sieve.go b/doc/progs/sieve.go new file mode 100644 index 000000000..b31530981 --- /dev/null +++ b/doc/progs/sieve.go @@ -0,0 +1,38 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package main + +import "fmt" + +// Send the sequence 2, 3, 4, ... to channel 'ch'. +func generate(ch chan int) { + for i := 2; ; i++ { + ch <- i // Send 'i' to channel 'ch'. + } +} + +// Copy the values from channel 'in' to channel 'out', +// removing those divisible by 'prime'. +func filter(in, out chan int, prime int) { + for { + i := <-in // Receive value of new variable 'i' from 'in'. + if i%prime != 0 { + out <- i // Send 'i' to channel 'out'. + } + } +} + +// The prime sieve: Daisy-chain filter processes together. +func main() { + ch := make(chan int) // Create a new channel. + go generate(ch) // Start generate() as a goroutine. + for i := 0; i < 100; i++ { // Print the first hundred primes. + prime := <-ch + fmt.Println(prime) + ch1 := make(chan int) + go filter(ch, ch1, prime) + ch = ch1 + } +} diff --git a/doc/progs/sieve1.go b/doc/progs/sieve1.go new file mode 100644 index 000000000..e1411a334 --- /dev/null +++ b/doc/progs/sieve1.go @@ -0,0 +1,51 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package main + +import "fmt" + +// Send the sequence 2, 3, 4, ... to returned channel +func generate() chan int { + ch := make(chan int) + go func() { + for i := 2; ; i++ { + ch <- i + } + }() + return ch +} + +// Filter out input values divisible by 'prime', send rest to returned channel +func filter(in chan int, prime int) chan int { + out := make(chan int) + go func() { + for { + if i := <-in; i%prime != 0 { + out <- i + } + } + }() + return out +} + +func sieve() chan int { + out := make(chan int) + go func() { + ch := generate() + for { + prime := <-ch + out <- prime + ch = filter(ch, prime) + } + }() + return out +} + +func main() { + primes := sieve() + for i := 0; i < 100; i++ { // Print the first hundred primes. + fmt.Println(<-primes) + } +} diff --git a/doc/progs/sort.go b/doc/progs/sort.go new file mode 100644 index 000000000..894693f0d --- /dev/null +++ b/doc/progs/sort.go @@ -0,0 +1,59 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package sort + +type Interface interface { + Len() int + Less(i, j int) bool + Swap(i, j int) +} + +func Sort(data Interface) { + for i := 1; i < data.Len(); i++ { + for j := i; j > 0 && data.Less(j, j-1); j-- { + data.Swap(j, j-1) + } + } +} + +func IsSorted(data Interface) bool { + n := data.Len() + for i := n - 1; i > 0; i-- { + if data.Less(i, i-1) { + return false + } + } + return true +} + +// Convenience types for common cases + +type IntSlice []int + +func (p IntSlice) Len() int { return len(p) } +func (p IntSlice) Less(i, j int) bool { return p[i] < p[j] } +func (p IntSlice) Swap(i, j int) { p[i], p[j] = p[j], p[i] } + +type Float64Slice []float64 + +func (p Float64Slice) Len() int { return len(p) } +func (p Float64Slice) Less(i, j int) bool { return p[i] < p[j] } +func (p Float64Slice) Swap(i, j int) { p[i], p[j] = p[j], p[i] } + +type StringSlice []string + +func (p StringSlice) Len() int { return len(p) } +func (p StringSlice) Less(i, j int) bool { return p[i] < p[j] } +func (p StringSlice) Swap(i, j int) { p[i], p[j] = p[j], p[i] } + +// Convenience wrappers for common cases + +func SortInts(a []int) { Sort(IntSlice(a)) } +func SortFloat64s(a []float64) { Sort(Float64Slice(a)) } +func SortStrings(a []string) { Sort(StringSlice(a)) } + +func IntsAreSorted(a []int) bool { return IsSorted(IntSlice(a)) } +func Float64sAreSorted(a []float64) bool { return IsSorted(Float64Slice(a)) } +func StringsAreSorted(a []string) bool { return IsSorted(StringSlice(a)) } diff --git a/doc/progs/sortmain.go b/doc/progs/sortmain.go new file mode 100644 index 000000000..c1babb01f --- /dev/null +++ b/doc/progs/sortmain.go @@ -0,0 +1,69 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package main + +import ( + "fmt" + "./sort" +) + +func ints() { + data := []int{74, 59, 238, -784, 9845, 959, 905, 0, 0, 42, 7586, -5467984, 7586} + a := sort.IntSlice(data) + sort.Sort(a) + if !sort.IsSorted(a) { + panic("fail") + } +} + +func strings() { + data := []string{"monday", "tuesday", "wednesday", "thursday", "friday", "saturday", "sunday"} + a := sort.StringSlice(data) + sort.Sort(a) + if !sort.IsSorted(a) { + panic("fail") + } +} + +type day struct { + num int + shortName string + longName string +} + +type dayArray struct { + data []*day +} + +func (p *dayArray) Len() int { return len(p.data) } +func (p *dayArray) Less(i, j int) bool { return p.data[i].num < p.data[j].num } +func (p *dayArray) Swap(i, j int) { p.data[i], p.data[j] = p.data[j], p.data[i] } + +func days() { + Sunday := day{0, "SUN", "Sunday"} + Monday := day{1, "MON", "Monday"} + Tuesday := day{2, "TUE", "Tuesday"} + Wednesday := day{3, "WED", "Wednesday"} + Thursday := day{4, "THU", "Thursday"} + Friday := day{5, "FRI", "Friday"} + Saturday := day{6, "SAT", "Saturday"} + data := []*day{&Tuesday, &Thursday, &Wednesday, &Sunday, &Monday, &Friday, &Saturday} + a := dayArray{data} + sort.Sort(&a) + if !sort.IsSorted(&a) { + panic("fail") + } + for _, d := range data { + fmt.Printf("%s ", d.longName) + } + fmt.Printf("\n") +} + + +func main() { + ints() + strings() + days() +} diff --git a/doc/progs/strings.go b/doc/progs/strings.go new file mode 100644 index 000000000..e6739b385 --- /dev/null +++ b/doc/progs/strings.go @@ -0,0 +1,17 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package main + +import "os" + +func main() { + s := "hello" + if s[1] != 'e' { + os.Exit(1) + } + s = "good bye" + var p *string = &s + *p = "ciao" +} diff --git a/doc/progs/sum.go b/doc/progs/sum.go new file mode 100644 index 000000000..e022195ed --- /dev/null +++ b/doc/progs/sum.go @@ -0,0 +1,20 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package main + +import "fmt" + +func sum(a []int) int { // returns an int + s := 0 + for i := 0; i < len(a); i++ { + s += a[i] + } + return s +} + +func main() { + s := sum([3]int{1, 2, 3}[:]) // a slice of the array is passed to sum + fmt.Print(s, "\n") +} diff --git a/doc/root.html b/doc/root.html new file mode 100644 index 000000000..8d76928c8 --- /dev/null +++ b/doc/root.html @@ -0,0 +1,98 @@ + + + + + +
+
+

+ The Go programming language is an open source project to make + programmers more productive. Go is expressive, concise, clean, + and efficient. Its concurrency mechanisms make it easy to write + programs that get the most out of multicore and networked machines, + while its novel type system enables flexible and modular program + construction. Go compiles quickly to machine code yet has the + convenience of garbage collection and the power of run-time reflection. + It's a fast, statically typed, compiled language that feels like a + dynamically typed, interpreted language. +

+

Check it out!

+

+

+ Install Go now, or try it right here in your browser:

+
+ +
+
+
+ +

Go Blog | More...

+
+
    +
+
+
+

Quick Links

+ +
+
+
diff --git a/doc/sieve.gif b/doc/sieve.gif new file mode 100644 index 000000000..8018ae225 Binary files /dev/null and b/doc/sieve.gif differ diff --git a/doc/talks/go_talk-20100112.html b/doc/talks/go_talk-20100112.html new file mode 100644 index 000000000..2e3643512 --- /dev/null +++ b/doc/talks/go_talk-20100112.html @@ -0,0 +1,411 @@ + + + +Go (January 12, 2010) + + + + + + + + +
+ +
+ +
+ + +
+ +
+
+ +
+ +
+ +
+
+

The Go Programming Language

+
+

Russ Cox

+ +
+

Stanford University

January 12, 2010

+
+
+ +
+

Go

+ +

New

+

Experimental

+

Concurrent

+

Garbage-collected

+

Systems

+

Language

+
+ +
+

Hello, world

+
+package main
+
+import "fmt"
+
+func main() {
+	fmt.Printf("Hello, 世界\n")
+}
+
+
+ +
+

History

+ +

Design started in late 2007.

+

Implementation starting to work mid-2008.

+

Released as an open source project in November 2009.

+

Work continues.

+

Robert Griesemer, Ken Thompson, Rob Pike, Ian Lance Taylor, Russ Cox, many others

+
+ +
+

Why?

+ +

Go fast!

+

Make programming fun again.

+
+ +
+

Why isn't programming fun?

+ +
+

Compiled, statically-typed languages (C, C++, Java) require too much typing and too much typing:

+ +
    +
  • verbose, lots of repetition
  • +
  • too much focus on type hierarchy
  • +
  • types get in the way as much as they help
  • +
  • compiles take far too long
  • +
+
+ +
+

Dynamic languages (Python, JavaScript) fix these problems (no more types, no more compiler) but introduce others:

+ +
    +
  • errors at run time that should be caught statically
  • +
  • no compilation means slow code
  • +
+
+ +

Can we combine the best of both?

+
+ +
+

Go

+ +

Make the language fast.

+

Make the tools fast.

+
+ +
+

Go Approach: Static Types

+ +

Static types, but declarations can infer type from expression:

+ +
+var one, hi = 1, "hello"
+
+var double = func(x int) int { return x*2 }
+
+ +

Not full Hindley-Milner type inference.

+
+ + +
+

Go Approach: Methods

+ +

Methods can be defined on any type.

+ +
+type Point struct {
+	X, Y float64
+}
+
+func (p Point) Abs() float64 {
+	return math.Sqrt(p.X*p.X + p.Y*p.Y)
+}
+
+
+ +
+

Go Approach: Methods

+ +

Methods can be defined on any type.

+ +
+type MyFloat float64
+
+func (f MyFloat) Abs() float64 {
+	v := float64(f)
+	if v < 0 {
+		v = -v
+	}
+	return v
+}
+
+
+ +
+

Go Approach: Abstract Types

+ +

An interface type lists a set of methods. Any value with those methods satisfies the interface.

+ +
+type Abser interface {
+	Abs() float64
+}
+
+func AbsPrinter(a Abser)
+
+ +

Can use Point or MyFloat (or ...):

+ +
+p := Point{3, 4}
+AbsPrinter(p)
+
+f := MyFloat(-10)
+AbsPrinter(f)
+
+ +

Notice that Point never declared that it implements Abser. It just does. Same with MyFloat.

+
+ +
+

Go Approach: Packages

+ +

A Go program comprises one or more packages.

+

Each package is one or more source files compiled and imported as a unit.

+
+package draw
+
+type Point struct {
+	X, Y int
+}
+
+ +
+package main
+
+import "draw"
+
+var p draw.Point
+
+
+ +
+

Go Approach: Visibility

+ +

Inside a package, all locally defined names are visible in all source files.

+ +

When imported, only the upper case names are visible.

+ +
+package draw
+
+type Point struct {
+	X, Y int
+	dist float64
+}
+
+type cache map[Point] float64
+
+ +

Clients that import "draw" can use the black names only.

+ +

“Shift is the new public.”

+
+ +
+

Go Approach: Concurrency

+ +

Cheap to create a new flow of control (goroutine):

+ +
+func main() {
+	go expensiveComputation(x, y, z)
+	anotherExpensiveComputation(a, b, c)
+}
+
+ +

Two expensive computations in parallel.

+
+ +
+

Go Approach: Synchronization

+ +

Use explicit messages to communicate and synchronize.

+ +
+func computeAndSend(ch chan int, x, y, z int) {
+	ch <- expensiveComputation(x, y, z)
+}
+
+func main() {
+	ch := make(chan int)
+	go computeAndSend(ch, x, y, z)
+	v2 := anotherExpensiveComputation(a, b, c)
+	v1 := <-ch
+	fmt.Println(v1, v2)
+}
+
+

Notice communication of result in addition to synchronization.

+
+ +
+

Go Fast: Language

+ +

Static types: enough to compile well, but inferred much of the time.

+ +

Methods: on any type, orthogonal to type system.

+ +

Abstract types: interface values, relations inferred statically.

+ +

Visibility: inferred from case of name.

+ +

Concurrency: lightweight way to start new thread of control.

+ +

Synchronization: explicit, easy message passing.

+ +
+ +

Lightweight feel of a scripting language but compiled.

+
+ +
+

Compile fast

+ +
+

Observation: much of the compile time for a source file is spent processing + other, often unrelated files.

+ +

In C: a.c includes b.h, which includes c.h, which includes d.h. +

+ +

Except that it's more often a tree instead of a chain.

+ +

On my Mac (OS X 10.5.8, gcc 4.0.1):

+
    +
  • C: #include <stdio.h> reads 360 lines from 9 files. +
  • C++: #include <iostream> reads 25,326 lines from 131 files. +
  • Objective C: #include <Carbon/Carbon.h> reads 124,730 lines from 689 files. +
+ +

And we haven't done any real work yet!

+ +

Same story in Java, Python, but reading binaries instead of source files.

+
+
+ +
+

Implementation: Summarize Dependencies

+ +
+package gui
+
+import "draw"
+
+type Mouse struct {
+	Loc draw.Point
+	Buttons uint
+}
+
+

Compiled form of gui summarizes the necessary part of draw (just Point).

+ +
+ +
+

Implementation: Summarize Dependencies

+ +

Compiled form of gui summarizes the necessary part of draw (just Point). Pseudo-object:

+ +
+package gui
+type draw.Point struct {
+	X, Y int
+}
+type gui.Mouse struct {
+	Loc draw.Point
+	Buttons uint
+}
+
+ +

A file that imports gui compiles without consulting draw or its dependencies.

+ +

In Go: import "fmt" reads one file: 184 lines summarizing types from 7 packages.

+ +

Tiny effect in this program but can be exponential in large programs.

+
+ +
+

Compilation Demo

+ +

Build all standard Go packages: ~120,000 lines of code.

+
+ +
+

Go Status

+ +
+
+

Open source:

+
    +
  • released on November 10, 2009 +
  • regular releases (~ weekly) +
  • all development done in public Mercurial repository +
  • outside contributions welcome +
+
+ +
+

Portable:

+
    +
  • FreeBSD, Linux, OS X (x86, x86-64) +
  • (in progress) Linux arm, Native Client x86, Windows x86. +
+
+ +
+

Still in progress, experimental. Yet to come:

+
    +
  • mature garbage collector +
  • generics? +
  • exceptions? +
  • unions or sum types? +
+
+
+ +
+ +
+

Questions?

+

+
+ +
+

+
+ +
+
+ + diff --git a/doc/talks/go_talk-20100121.html b/doc/talks/go_talk-20100121.html new file mode 100644 index 000000000..d5e4bc66f --- /dev/null +++ b/doc/talks/go_talk-20100121.html @@ -0,0 +1,453 @@ + + + +Go, Networked (January 21, 2010) + + + + + + + + +
+ +
+ +
+ + +
+ +
+
+ +
+ +
+ +
+
+

The Go Programming Language

+
+

Russ Cox

+ +
+

CNS Winter Research Review

January 21, 2010

+
+
+ +
+
+ +
+

Go

+ +

New

+

Experimental

+

Concurrent

+

Garbage-collected

+

Systems

+

Language

+
+ +
+

Hello, world

+
+package main
+
+import "fmt"
+
+func main() {
+	fmt.Printf("Hello, 世界\n")
+}
+
+
+ +
+

History

+ +

Design started in late 2007.

+

Implementation starting to work mid-2008.

+

Released as an open source project in November 2009.

+

Work continues.

+

Robert Griesemer, Ken Thompson, Rob Pike, Ian Lance Taylor, Russ Cox, many others

+
+ +
+

Goals and Motivation

+ +

Go fast!

+

Make programming fun again.

+

Targeted at systems software, broadly.

+
+ +
+

Why isn't programming fun?

+ +
+

Compiled, statically-typed languages (C, C++, Java) require too much typing and too much typing:

+ +
    +
  • verbose, lots of repetition
  • +
  • too much focus on type hierarchy
  • +
  • types get in the way as much as they help
  • +
  • compiles take far too long
  • +
+
+ +
+

Dynamic languages (Python, JavaScript) fix these problems (no more types, no more compiler) but introduce others:

+ +
    +
  • errors at run time that should be caught statically
  • +
  • no compilation means slow code
  • +
+
+ +

Can we combine the best of both?

+
+ +
+

Why a new language?

+ +
+

No new systems language in 10+ years.

+

Current languages designed before ...

+

... rise of large-scale, networked and multicore computing

+

... rise of Internet-scale distributed development (many libraries)

+
+
+ +
+

Go

+ +

Make the language fast.

+

Make the tools fast.

+
+ +
+

Compilation Demo

+ +

Build all standard Go packages: ~120,000 lines of code.

+
+ +
+

Go in one slide

+ +

Lightweight syntax.

+ +

Static types: enough to compile well, but inferred much of the time.

+ +

Methods: on any type, orthogonal to type system.

+ +

Abstract types: interface values, relations inferred statically.

+ +

Visibility: inferred from case of name.

+ +

First-class functions.

+ +

Garbage collection.

+ +
+ +

Lightweight feel of a scripting language but compiled.

+
+ +
+

Go, concurrently

+ +

Cheap to create a new flow of control (goroutine):

+ +
+func main() {
+	go expensiveComputation(x, y, z)
+	anotherExpensiveComputation(a, b, c)
+}
+
+ +

Two expensive computations in parallel.

+
+ +
+

Go, concurrently

+ +

Cheap to create a new flow of control (goroutine):

+ +
+	for {
+		rw := l.Accept()
+		conn := newConn(rw, handler)
+		go conn.serve()
+	}
+
+ +

Concurrent web server.

+

Network connections multiplexed onto epoll.

+
    +
  • many blocked Read calls != many blocked OS threads
  • +
+ +
+ +
+

Go, synchronized

+ +

Use explicit messages to communicate and synchronize.

+ +
+func computeAndSend(ch chan int, x, y, z int) {
+	ch <- expensiveComputation(x, y, z)
+}
+
+func main() {
+	ch := make(chan int)
+	go computeAndSend(ch, x, y, z)
+	v2 := anotherExpensiveComputation(a, b, c)
+	v1 := <-ch
+	fmt.Println(v1, v2)
+}
+
+

Notice communication of result in addition to synchronization.

+
+ +
+

Go, synchronized

+ +

RPC client

+ +
+func (client *Client) Call(method string, args, reply interface{}) os.Error {
+    // Send RPC message.
+    call := client.Go(method, args, reply, nil)
+	
+    // Read reply from Done channel.
+    <-call.Done
+
+    return call.Error
+}
+
+
+ +
+

Go, synchronized

+ +

RPC client demux

+ +
+func (client *Client) input() {
+	for {
+		resp := client.readResponse()
+		client.mutex.Lock()
+		c := client.pending[resp.Seq]
+		client.pending[resp.Seq] = c, false
+		client.mutex.Unlock()
+		if resp.Error != "" {
+			c.Error = os.ErrorString(resp.error)
+		}
+		resp.Decode(c.Reply)
+		c.Done <- c
+	}
+}
+
+
+ +
+

Go, synchronized

+ +

RPC client demux

+ +
+func (client *Client) input() {
+	for {
+		resp := client.readResponse()
+		client.mutex.Lock()
+		c := client.pending[resp.Seq]
+		client.pending[resp.Seq] = c, false
+		client.mutex.Unlock()
+		if resp.Error != "" {
+			c.Error = os.ErrorString(resp.error)
+		}
+		resp.Decode(c.Reply)
+		c.Done <- c
+	}
+}
+
+

Read response from network.

+ +
+

Go, synchronized

+ +

RPC client demux

+ +
+func (client *Client) input() {
+	for {
+		resp := client.readResponse()
+		client.mutex.Lock()
+		c := client.pending[resp.Seq]
+		client.pending[resp.Seq] = c, false
+		client.mutex.Unlock()
+		if resp.Error != "" {
+			c.Error = os.ErrorString(resp.error)
+		}
+		resp.Decode(c.Reply)
+		c.Done <- c
+	}
+}
+
+

Look up request by sequence number.

+ +
+

Go, synchronized

+ +

RPC client demux

+ +
+func (client *Client) input() {
+	for {
+		resp := client.readResponse()
+		client.mutex.Lock()
+		c := client.pending[resp.Seq]
+		client.pending[resp.Seq] = c, false
+		client.mutex.Unlock()
+		if resp.Error != "" {
+			c.Error = os.ErrorString(resp.error)
+		}
+		resp.Decode(c.Reply)
+		c.Done <- c
+	}
+}
+
+

Decode response fields from payload.

+ +
+

Go, synchronized

+ +

RPC client demux

+ +
+func (client *Client) input() {
+	for {
+		resp := client.readResponse()
+		client.mutex.Lock()
+		c := client.pending[resp.Seq]
+		client.pending[resp.Seq] = c, false
+		client.mutex.Unlock()
+		if resp.Error != "" {
+			c.Error = os.ErrorString(resp.error)
+		}
+		resp.Decode(c.Reply)
+		c.Done <- c
+	}
+}
+
+

Tell client that it finished.

+ +
+

Go, synchronized

+ +

RPC client demux

+ +
+func (client *Client) input() {
+	for {
+		resp := client.readResponse()
+		client.mutex.Lock()
+		c := client.pending[resp.Seq]
+		client.pending[resp.Seq] = c, false
+		client.mutex.Unlock()
+		if resp.Error != "" {
+			c.Error = os.ErrorString(resp.error)
+		}
+		resp.Decode(c.Reply)
+		c.Done <- c
+	}
+}
+
+ +

Can create multiple Calls with same Done channel +and distinguish which finished by inspecting value sent on channel. +

+ +
+ +
+

Goroutine demo

+ +

Chain together 100,000 goroutines connected by 100,001 channels.

+ +

Send a value to one end of the chain.

+ +

Each passes it along, increments.

+ +

Receive value out the other end of the chain.

+
+ + +
+

Go Status

+
+ +
+

Go Status

+ +

Open source:

+
    +
  • released on November 10, 2009 +
  • regular releases (~ weekly) +
  • all development done in public Mercurial repository +
  • outside contributions welcome +
  • two independent compiler implementations +
  • XML, JSON, HTTP, TLS/SSL, native RPC, (network channels,) ... +
+
+ +
+

Go Status

+ +

Open source

+ +

Portable:

+
    +
  • FreeBSD, Linux, OS X (x86, x86-64) +
  • (in progress) Linux arm, Native Client x86, Windows x86. +
+
+ +
+

Go Status

+ +

Open source

+

Portable

+ +

Still in progress, experimental. Yet to come:

+
    +
  • production garbage collector +
  • generics? +
  • exceptions? +
  • unions or sum types? +
+
+ +
+

Questions?

+

+
+ +
+

+
+ +
+
+ + diff --git a/doc/talks/go_talk-20100323.html b/doc/talks/go_talk-20100323.html new file mode 100644 index 000000000..3143b079a --- /dev/null +++ b/doc/talks/go_talk-20100323.html @@ -0,0 +1,395 @@ + + + +Go Tech Talk + + + + + + + + +
+ +
+ +
+ + +
+ +
+
+
+ +
+

The Go Programming Language

+
+
+

Sydney University

March 23, 2010

+
+
+ +
+

Go

+ +

New

+

Experimental

+

Concurrent

+

Garbage Collected

+

Systems Language

+
+ +
+

Hello, world

+
+package main
+
+import "fmt"
+
+func main() {
+	fmt.Printf("Hello, 世界\n")
+}
+
+
+ +
+

Hello, world 2.0

+ +

Serving http://localhost:8080/world

+
+package main
+
+import (
+	"fmt"
+	"http"
+)
+
+func handler(c *http.Conn, r *http.Request) { 
+	fmt.Fprintf(c, "Hello, %s.", r.URL.Path[1:]) 
+}
+
+func main() {
+	http.ListenAndServe(":8080",
+			http.HandlerFunc(handler))
+}
+
+
+ +
+

New

+ +

It's about two years old:

+
    +
  • Design started in late 2007
  • +
  • Implementation starting to work mid-2008
  • +
  • Released as an open source project in November 2009
  • +
  • Development continues with an active community
  • +
+ +

Why invent a new language? Older languages weren't designed for concurrency, but modern software needs it:

+
    +
  • Large scale, networked computing, such as Google web search
  • +
  • Multi-core hardware
  • +
+
+ +
+

New

+ +

Older languages are also frustrating on a day-to-day basis

+

Statically-typed languages (C, C++, Java) have issues:

+
    +
  • Edit-Compile-Run cycle takes far too long
  • +
  • Type hierarchy can hurt as much as it helps
  • +
+
+
+
+ +

Dynamic languages (Python, JavaScript) fix some issues but introduce others:

+
    +
  • No compilation means slow code
  • +
  • Runtime errors that should be caught statically
  • +
+ +

Go has the lighter feel of a scripting language but is compiled

+
+ +
+

New

+ +

Large C++ programs (e.g. Firefox, OpenOffice, Chromium) have enormous build times:

+ + +

On a Mac (OS X 10.5.8, gcc 4.0.1):

+
    +
  • C: #include <stdio.h> reads 360 lines from 9 files
  • +
  • C++: #include <iostream> reads 25,326 lines from 131 files
  • +
  • Objective-C: #include <Carbon/Carbon.h> reads 124,730 lines from 689 files
  • +
  • We haven't done any real work yet!
  • +
+ +

In Go: import "fmt" reads one file: 184 lines summarizing 7 packages

+
+ +
+

New

+ +

Compilation demo

+
+ +
+

Experimental

+ +

Go is still unproven

+

Language is still evolving

+

Package library is incomplete

+

Concurrent garbage collection is an active research problem

+

Reviving forgotten concepts:

+
    +
  • Go's concurrency is strongly influenced by Communicating Sequential Processes (Hoare, 1978)
  • +
  • Go has types and interfaces, but no inheritance. It is arguably more object-oriented than previously mentioned languages, being closer to the original Smalltalk meaning (1970s)
  • +
+
+ +
+

Concurrent

+ +

Unix philosophy: write programs that do one thing and do it well

+

Connect them with pipes:

+
    +
  • How many lines of test code are there in the Go standard library?
  • +
  • find ~/go/src/pkg | grep _test.go$ | xargs wc -l
  • +
+ +

Unlike other languages, Go makes it easy to:

+
    +
  • Launch goroutines
  • +
  • Connect them with channels
  • +
+
+ +
+

Concurrent

+ +

Start a new flow of control with the go keyword

+

Parallel computation is easy:

+
+func main() {
+	go expensiveComputation(x, y, z)
+	anotherExpensiveComputation(a, b, c)
+}
+
+ +

Roughly speaking, a goroutine is like a thread, but lighter weight:

+
    +
  • Goroutines have segmented stacks, and typically smaller stacks
  • +
  • This requires compiler support. Goroutines can't just be a C++ library on top of a thread library
  • +
+
+ +
+

Concurrent

+ +

Consider web servers ("the C10k problem"):

+
    +
  • "Thread per connection" approach is conceptually neat, but doesn't scale well in practice
  • +
  • What does scale well (event-driven callbacks, asynchronous APIs) are harder to understand, maintain, and debug
  • +
  • We think "goroutine per connection" can scale well, and is conceptually neat
  • +
+
+	for {
+		rw := socket.Accept()
+		conn := newConn(rw, handler)
+		go conn.serve()
+	}
+
+
+ +
+

Concurrent

+ +

Let's look again at our simple parallel computation:

+
+func main() {
+	go expensiveComputation(x, y, z)
+	anotherExpensiveComputation(a, b, c)
+}
+
+ +

This story is incomplete:

+
    +
  • How do we know when the two computations are done?
  • +
  • What are their values?
  • +
+
+ +
+

Concurrent

+ +

Goroutines communicate with other goroutines via channels

+
+func computeAndSend(ch chan int, x, y, z int) {
+	ch <- expensiveComputation(x, y, z)
+}
+
+func main() {
+	ch := make(chan int)
+	go computeAndSend(ch, x, y, z)
+	v2 := anotherExpensiveComputation(a, b, c)
+	v1 := <-ch
+	fmt.Println(v1, v2)
+}
+
+ +
+ +
+

Concurrent

+ +

In traditional concurrent programs, you communicate by sharing memory. In Go, you share memory by communicating:

+
    +
  • Communication (the <- operator) is sharing and synchronization
  • +
+ +

Threads and locks are concurrency primitives; CSP is a concurrency model:

+
    +
  • Analogy: "Go To Statement Considered Harmful" (Dijsktra, 1968)
  • +
  • goto is a control flow primitive; structured programming (if statements, for loops, function calls) is a control flow model
  • +
+ +

Learning CSP changes the way you think about concurrent programming:

+
    +
  • Every language has its grain. If your Go program uses mutexes, you're probably working against the grain
  • +
+
+ +
+

Garbage Collected

+ +

Automatic memory management makes writing (and maintaining) programs easier

+

Especially in a concurrent world:

+
    +
  • Who "owns" a shared piece of memory, and is responsible for destroying it?
  • +
+ +

Large C++ programs usually end up with semi-automatic memory management anyway, via "smart pointers"

+

Mixing the two models can be problematic:

+
    +
  • Browsers can leak memory easily; DOM elements are C++ objects, but JavaScript is garbage collected
  • +
+
+ +
+

Garbage Collected

+ +

Go is also a safer language:

+
    +
  • Pointers but no pointer arithmetic
  • +
  • No dangling pointers
  • +
  • Variables are zero-initialized
  • +
  • Array access is bounds-checked
  • +
+ +

No buffer overflow exploits

+
+ +
+

Systems Language

+ +

This just means you could write decently large programs in Go:

+
    +
  • Web servers
  • +
  • Web browsers
  • +
  • Web crawlers
  • +
  • Search indexers
  • +
  • Databases
  • +
  • Word processors
  • +
  • Integrated Development Environments (IDEs)
  • +
  • Operating systems
  • +
  • ...
  • +
+
+ +
+

Systems Language

+ +

Garbage collection has a reputation for being "slower"

+

We're expecting Go to be slightly slower than optimized C, but faster than Java, depending on the task. Nonetheless:

+
    +
  • Fast and buggy is worse than almost-as-fast and correct
  • +
  • It is easier to optimize a correct program than to correct an optimized program
  • +
  • Fundamentally, it's simply a trade-off we're willing to make
  • +
+ +

Memory layout can drastically affect performance. These two designs are equivalent in Go, but significantly different in Java:

+
+type Point struct { X, Y int }
+type Rect struct { P0, P1 Point }
+
+// or ...
+
+type Rect struct { X0, Y0, X1, Y1 int }
+
+
+ +
+

Systems Language

+ +

Quote from http://loadcode.blogspot.com/2009/12/go-vs-java.html

+ +

+"[Git] is known to be very fast. It is written in C. A Java version +JGit was made. It was considerably slower. Handling of memory and lack +of unsigned types was some of the important reasons. +

+ +

Shawn O. Pearce wrote on the git mailinglist:

+
  • "JGit struggles with not +having an efficient way to represent a SHA-1. C can just say "unsigned +char[20]" and have it inline into the container's memory allocation. A +byte[20] in Java will cost an *additional* 16 bytes of memory, and be +slower to access because the bytes themselves are in a different area +of memory from the container object. We try to work around it by +converting from a byte[20] to 5 ints, but that costs us machine +instructions" +
+ +

+Like C, Go does allow unsigned types and defining data structures +containing other data structures as continuous blocks of memory." +

+
+ +
+

Go

+ +

New

+

Experimental

+

Concurrent

+

Garbage Collected

+

Systems Language

+ +

And more:

+
    +
  • I haven't talked about the type system, interfaces, slices, closures, selects, ...
  • +
  • Tutorial, documentation, mailing list, source code all online
  • +
+
+ +
+

Questions?

+

+
+ +
+
+ + diff --git a/doc/talks/gofrontend-gcc-summit-2010.pdf b/doc/talks/gofrontend-gcc-summit-2010.pdf new file mode 100644 index 000000000..157fd7676 Binary files /dev/null and b/doc/talks/gofrontend-gcc-summit-2010.pdf differ diff --git a/doc/talks/io2010/balance.go b/doc/talks/io2010/balance.go new file mode 100644 index 000000000..b01f7468c --- /dev/null +++ b/doc/talks/io2010/balance.go @@ -0,0 +1,168 @@ +// Copyright 2010 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package main + +import ( + "container/heap" + "flag" + "fmt" + "rand" + "time" +) + +const nRequester = 100 +const nWorker = 10 + +var roundRobin = flag.Bool("r", false, "use round-robin scheduling") + +// Simulation of some work: just sleep for a while and report how long. +func op() int { + n := rand.Int63n(1e9) + time.Sleep(nWorker * n) + return int(n) +} + +type Request struct { + fn func() int + c chan int +} + +func requester(work chan Request) { + c := make(chan int) + for { + time.Sleep(rand.Int63n(nWorker * 2e9)) + work <- Request{op, c} + <-c + } +} + +type Worker struct { + i int + requests chan Request + pending int +} + +func (w *Worker) work(done chan *Worker) { + for { + req := <-w.requests + req.c <- req.fn() + done <- w + } +} + +type Pool []*Worker + +func (p Pool) Len() int { return len(p) } + +func (p Pool) Less(i, j int) bool { + return p[i].pending < p[j].pending +} + +func (p *Pool) Swap(i, j int) { + a := *p + a[i], a[j] = a[j], a[i] + a[i].i = i + a[j].i = j +} + +func (p *Pool) Push(x interface{}) { + a := *p + n := len(a) + a = a[0 : n+1] + w := x.(*Worker) + a[n] = w + w.i = n + *p = a +} + +func (p *Pool) Pop() interface{} { + a := *p + *p = a[0 : len(a)-1] + w := a[len(a)-1] + w.i = -1 // for safety + return w +} + +type Balancer struct { + pool Pool + done chan *Worker + i int +} + +func NewBalancer() *Balancer { + done := make(chan *Worker, nWorker) + b := &Balancer{make(Pool, 0, nWorker), done, 0} + for i := 0; i < nWorker; i++ { + w := &Worker{requests: make(chan Request, nRequester)} + heap.Push(&b.pool, w) + go w.work(b.done) + } + return b +} + +func (b *Balancer) balance(work chan Request) { + for { + select { + case req := <-work: + b.dispatch(req) + case w := <-b.done: + b.completed(w) + } + b.print() + } +} + +func (b *Balancer) print() { + sum := 0 + sumsq := 0 + for _, w := range b.pool { + fmt.Printf("%d ", w.pending) + sum += w.pending + sumsq += w.pending * w.pending + } + avg := float64(sum) / float64(len(b.pool)) + variance := float64(sumsq)/float64(len(b.pool)) - avg*avg + fmt.Printf(" %.2f %.2f\n", avg, variance) +} + +func (b *Balancer) dispatch(req Request) { + if *roundRobin { + w := b.pool[b.i] + w.requests <- req + w.pending++ + b.i++ + if b.i >= len(b.pool) { + b.i = 0 + } + return + } + + w := heap.Pop(&b.pool).(*Worker) + w.requests <- req + w.pending++ + // fmt.Printf("started %p; now %d\n", w, w.pending) + heap.Push(&b.pool, w) +} + +func (b *Balancer) completed(w *Worker) { + if *roundRobin { + w.pending-- + return + } + + w.pending-- + // fmt.Printf("finished %p; now %d\n", w, w.pending) + heap.Remove(&b.pool, w.i) + heap.Push(&b.pool, w) +} + +func main() { + flag.Parse() + work := make(chan Request) + for i := 0; i < nRequester; i++ { + go requester(work) + } + NewBalancer().balance(work) +} diff --git a/doc/talks/io2010/decrypt.go b/doc/talks/io2010/decrypt.go new file mode 100644 index 000000000..0a6c006e2 --- /dev/null +++ b/doc/talks/io2010/decrypt.go @@ -0,0 +1,85 @@ +// Copyright 2010 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// This code differs from the slides in that it handles errors. + +package main + +import ( + "crypto/aes" + "crypto/cipher" + "compress/gzip" + "io" + "log" + "os" +) + +func EncryptAndGzip(dstfile, srcfile string, key, iv []byte) os.Error { + r, err := os.Open(srcfile) + if err != nil { + return err + } + var w io.Writer + w, err = os.Create(dstfile) + if err != nil { + return err + } + c, err := aes.NewCipher(key) + if err != nil { + return err + } + w = cipher.StreamWriter{S: cipher.NewOFB(c, iv), W: w} + w2, err := gzip.NewWriter(w) + if err != nil { + return err + } + defer w2.Close() + _, err = io.Copy(w2, r) + return err +} + +func DecryptAndGunzip(dstfile, srcfile string, key, iv []byte) os.Error { + f, err := os.Open(srcfile) + if err != nil { + return err + } + defer f.Close() + c, err := aes.NewCipher(key) + if err != nil { + return err + } + r := cipher.StreamReader{S: cipher.NewOFB(c, iv), R: f} + r2, err := gzip.NewReader(r) + if err != nil { + return err + } + w, err := os.Create(dstfile) + if err != nil { + return err + } + defer w.Close() + _, err = io.Copy(w, r2) + return err +} + +func main() { + err := EncryptAndGzip( + "/tmp/passwd.gz", + "/etc/passwd", + make([]byte, 16), + make([]byte, 16), + ) + if err != nil { + log.Fatal(err) + } + err = DecryptAndGunzip( + "/dev/stdout", + "/tmp/passwd.gz", + make([]byte, 16), + make([]byte, 16), + ) + if err != nil { + log.Fatal(err) + } +} diff --git a/doc/talks/io2010/encrypt.go b/doc/talks/io2010/encrypt.go new file mode 100644 index 000000000..c6508bba1 --- /dev/null +++ b/doc/talks/io2010/encrypt.go @@ -0,0 +1,52 @@ +// Copyright 2010 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// This code differs from the slides in that it handles errors. + +package main + +import ( + "crypto/aes" + "crypto/cipher" + "compress/gzip" + "io" + "log" + "os" +) + +func EncryptAndGzip(dstfile, srcfile string, key, iv []byte) os.Error { + r, err := os.Open(srcfile) + if err != nil { + return err + } + var w io.WriteCloser + w, err = os.Create(dstfile) + if err != nil { + return err + } + defer w.Close() + w, err = gzip.NewWriter(w) + if err != nil { + return err + } + defer w.Close() + c, err := aes.NewCipher(key) + if err != nil { + return err + } + _, err = io.Copy(cipher.StreamWriter{S: cipher.NewOFB(c, iv), W: w}, r) + return err +} + +func main() { + err := EncryptAndGzip( + "/tmp/passwd.gz", + "/etc/passwd", + make([]byte, 16), + make([]byte, 16), + ) + if err != nil { + log.Fatal(err) + } +} diff --git a/doc/talks/io2010/eval1.go b/doc/talks/io2010/eval1.go new file mode 100644 index 000000000..2d7fc3be6 --- /dev/null +++ b/doc/talks/io2010/eval1.go @@ -0,0 +1,229 @@ +// Copyright 2010 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package main + +import ( + "bufio" + "fmt" + "os" + "strconv" + "strings" +) + +// Generic expression parser/evaluator + +type Value interface { + String() string + BinaryOp(op string, y Value) Value +} + +type Parser struct { + precTab map[string]int + newVal func(string) Value + src string + pos int + tok string +} + +const alphanum = "_abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789" + +func (p *Parser) stop(c uint8) bool { + switch { + case p.pos >= len(p.src): + return true + case c == '"': + if p.src[p.pos] == '"' { + p.pos++ + return true + } + return false + case strings.IndexRune(alphanum, int(c)) >= 0: + return strings.IndexRune(alphanum, int(p.src[p.pos])) < 0 + } + return true +} + +func (p *Parser) next() { + // skip blanks + for ; p.pos < len(p.src) && p.src[p.pos] <= ' '; p.pos++ { + } + if p.pos >= len(p.src) { + p.tok = "" + return + } + start := p.pos + c := p.src[p.pos] + for p.pos < len(p.src) { + p.pos++ + if p.stop(c) { + break + } + } + p.tok = p.src[start:p.pos] +} + +func (p *Parser) binaryExpr(prec1 int) Value { + x := p.newVal(p.tok) + p.next() + for prec := p.precTab[p.tok]; prec >= prec1; prec-- { + for p.precTab[p.tok] == prec { + op := p.tok + p.next() + y := p.binaryExpr(prec + 1) + x = x.BinaryOp(op, y) + } + } + return x +} + +func Eval(precTab map[string]int, newVal func(string) Value, src string) Value { + var p Parser + p.precTab = precTab + p.newVal = newVal + p.src = src + p.next() + return p.binaryExpr(1) +} + +// Command-line expression evaluator + +func main() { + r := bufio.NewReader(os.Stdin) + for { + fmt.Printf("> ") + line, err := r.ReadString('\n') + if err != nil { + break + } + fmt.Printf("%s\n", Eval(precTab, trace(newVal), line)) + } +} + + +// Custom grammar and values + +var precTab = map[string]int{ + "&&": 1, + "||": 2, + "==": 3, + "!=": 3, + "<": 3, + "<=": 3, + ">": 3, + ">=": 3, + "+": 4, + "-": 4, + "*": 5, + "/": 5, + "%": 5, +} + +func newVal(lit string) Value { + x, err := strconv.Atoi(lit) + if err == nil { + return Int(x) + } + b, err := strconv.Atob(lit) + if err == nil { + return Bool(b) + } + return Error(fmt.Sprintf("illegal literal '%s'", lit)) +} + +type Error string + +func (e Error) String() string { return string(e) } +func (e Error) BinaryOp(op string, y Value) Value { return e } + +type Int int + +func (x Int) String() string { return strconv.Itoa(int(x)) } +func (x Int) BinaryOp(op string, y Value) Value { + switch y := y.(type) { + case Error: + return y + case Int: + switch op { + case "+": + return x + y + case "-": + return x - y + case "*": + return x * y + case "/": + return x / y + case "%": + return x % y + case "==": + return Bool(x == y) + case "!=": + return Bool(x != y) + case "<": + return Bool(x < y) + case "<=": + return Bool(x <= y) + case ">": + return Bool(x > y) + case ">=": + return Bool(x >= y) + } + } + return Error(fmt.Sprintf("illegal operation: '%v %s %v'", x, op, y)) +} + +type Bool bool + +func (x Bool) String() string { return strconv.Btoa(bool(x)) } +func (x Bool) BinaryOp(op string, y Value) Value { + switch y := y.(type) { + case Error: + return y + case Bool: + switch op { + case "&&": + return Bool(x && y) + case "||": + return Bool(x || y) + case "==": + return Bool(x == y) + case "!=": + return Bool(x != y) + } + } + return Error(fmt.Sprintf("illegal operation: '%v %s %v'", x, op, y)) +} + + +func trace(newVal func(string) Value) func(string) Value { + return func(s string) Value { + v := newVal(s) + fmt.Printf("\tnewVal(%q) = %s\n", s, fmtv(v)) + return &traceValue{v} + } +} + +type traceValue struct { + Value +} + +func (x *traceValue) BinaryOp(op string, y Value) Value { + z := x.Value.BinaryOp(op, y.(*traceValue).Value) + fmt.Printf("\t%s.BinaryOp(%q, %s) = %s\n", fmtv(x.Value), op, fmtv(y.(*traceValue).Value), fmtv(z)) + return &traceValue{z} +} + +func (x *traceValue) String() string { + s := x.Value.String() + fmt.Printf("\t%s.String() = %#v\n", fmtv(x.Value), s) + return s +} + +func fmtv(v Value) string { + t := fmt.Sprintf("%T", v) + if i := strings.LastIndex(t, "."); i >= 0 { // strip package + t = t[i+1:] + } + return fmt.Sprintf("%s(%#v)", t, v) +} diff --git a/doc/talks/io2010/eval2.go b/doc/talks/io2010/eval2.go new file mode 100644 index 000000000..5524c8b3a --- /dev/null +++ b/doc/talks/io2010/eval2.go @@ -0,0 +1,261 @@ +// Copyright 2010 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package main + +import ( + "bufio" + "fmt" + "os" + "strconv" + "strings" +) + +// Generic expression parser/evaluator + +type Value interface { + String() string + BinaryOp(op string, y Value) Value +} + +type Parser struct { + precTab map[string]int + newVal func(string) Value + src string + pos int + tok string +} + +const alphanum = "_abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789" + +func (p *Parser) stop(c uint8) bool { + switch { + case p.pos >= len(p.src): + return true + case c == '"': + if p.src[p.pos] == '"' { + p.pos++ + return true + } + return false + case strings.IndexRune(alphanum, int(c)) >= 0: + return strings.IndexRune(alphanum, int(p.src[p.pos])) < 0 + } + return true +} + +func (p *Parser) next() { + // skip blanks + for ; p.pos < len(p.src) && p.src[p.pos] <= ' '; p.pos++ { + } + if p.pos >= len(p.src) { + p.tok = "" + return + } + start := p.pos + c := p.src[p.pos] + for p.pos < len(p.src) { + p.pos++ + if p.stop(c) { + break + } + } + p.tok = p.src[start:p.pos] +} + +func (p *Parser) binaryExpr(prec1 int) Value { + x := p.newVal(p.tok) + p.next() + for prec := p.precTab[p.tok]; prec >= prec1; prec-- { + for p.precTab[p.tok] == prec { + op := p.tok + p.next() + y := p.binaryExpr(prec + 1) + x = x.BinaryOp(op, y) + } + } + return x +} + +func Eval(precTab map[string]int, newVal func(string) Value, src string) Value { + var p Parser + p.precTab = precTab + p.newVal = newVal + p.src = src + p.next() + return p.binaryExpr(1) +} + +// Command-line expression evaluator + +func main() { + r := bufio.NewReader(os.Stdin) + for { + fmt.Printf("> ") + line, err := r.ReadString('\n') + if err != nil { + break + } + fmt.Printf("%s\n", Eval(precTab, trace(newVal), line)) + } +} + + +// Custom grammar and values + +var precTab = map[string]int{ + "&&": 1, + "||": 2, + "==": 3, + "!=": 3, + "<": 3, + "<=": 3, + ">": 3, + ">=": 3, + "+": 4, + "-": 4, + "*": 5, + "/": 5, + "%": 5, +} + +func newVal(lit string) Value { + x, err := strconv.Atoi(lit) + if err == nil { + return Int(x) + } + b, err := strconv.Atob(lit) + if err == nil { + return Bool(b) + } + s, err := strconv.Unquote(lit) + if err == nil { + return String(s) + } + return Error(fmt.Sprintf("illegal literal '%s'", lit)) +} + +type Error string + +func (e Error) String() string { return string(e) } +func (e Error) BinaryOp(op string, y Value) Value { return e } + +type Int int + +func (x Int) String() string { return strconv.Itoa(int(x)) } +func (x Int) BinaryOp(op string, y Value) Value { + switch y := y.(type) { + case Error: + return y + case String: + switch op { + case "*": + return String(strings.Repeat(string(y), int(x))) + } + case Int: + switch op { + case "+": + return x + y + case "-": + return x - y + case "*": + return x * y + case "/": + return x / y + case "%": + return x % y + case "==": + return Bool(x == y) + case "!=": + return Bool(x != y) + case "<": + return Bool(x < y) + case "<=": + return Bool(x <= y) + case ">": + return Bool(x > y) + case ">=": + return Bool(x >= y) + } + } + return Error(fmt.Sprintf("illegal operation: '%v %s %v'", x, op, y)) +} + +type Bool bool + +func (x Bool) String() string { return strconv.Btoa(bool(x)) } +func (x Bool) BinaryOp(op string, y Value) Value { + switch y := y.(type) { + case Error: + return y + case Bool: + switch op { + case "&&": + return Bool(x && y) + case "||": + return Bool(x || y) + case "==": + return Bool(x == y) + case "!=": + return Bool(x != y) + } + } + return Error(fmt.Sprintf("illegal operation: '%v %s %v'", x, op, y)) +} + +type String string + +func (x String) String() string { return strconv.Quote(string(x)) } +func (x String) BinaryOp(op string, y Value) Value { + switch y := y.(type) { + case Error: + return y + case Int: + switch op { + case "*": + return String(strings.Repeat(string(x), int(y))) + } + case String: + switch op { + case "+": + return x + y + case "<": + return Bool(x < y) + } + } + return Error(fmt.Sprintf("illegal operation: '%v %s %v'", x, op, y)) +} + + +func trace(newVal func(string) Value) func(string) Value { + return func(s string) Value { + v := newVal(s) + fmt.Printf("\tnewVal(%q) = %s\n", s, fmtv(v)) + return &traceValue{v} + } +} + +type traceValue struct { + Value +} + +func (x *traceValue) BinaryOp(op string, y Value) Value { + z := x.Value.BinaryOp(op, y.(*traceValue).Value) + fmt.Printf("\t%s.BinaryOp(%q, %s) = %s\n", fmtv(x.Value), op, fmtv(y.(*traceValue).Value), fmtv(z)) + return &traceValue{z} +} + +func (x *traceValue) String() string { + s := x.Value.String() + fmt.Printf("\t%s.String() = %#v\n", fmtv(x.Value), s) + return s +} + +func fmtv(v Value) string { + t := fmt.Sprintf("%T", v) + if i := strings.LastIndex(t, "."); i >= 0 { // strip package + t = t[i+1:] + } + return fmt.Sprintf("%s(%#v)", t, v) +} diff --git a/doc/talks/io2010/talk.pdf b/doc/talks/io2010/talk.pdf new file mode 100644 index 000000000..aff42c21d Binary files /dev/null and b/doc/talks/io2010/talk.pdf differ diff --git a/doc/talks/io2011/Real_World_Go.pdf b/doc/talks/io2011/Real_World_Go.pdf new file mode 100644 index 000000000..2a187116b Binary files /dev/null and b/doc/talks/io2011/Real_World_Go.pdf differ diff --git a/doc/talks/io2011/Writing_Web_Apps_in_Go.pdf b/doc/talks/io2011/Writing_Web_Apps_in_Go.pdf new file mode 100644 index 000000000..ca4702ee9 Binary files /dev/null and b/doc/talks/io2011/Writing_Web_Apps_in_Go.pdf differ diff --git a/doc/talks/java-typing.png b/doc/talks/java-typing.png new file mode 100644 index 000000000..54abf0186 Binary files /dev/null and b/doc/talks/java-typing.png differ diff --git a/doc/talks/slidy.css b/doc/talks/slidy.css new file mode 100644 index 000000000..e9ff53218 --- /dev/null +++ b/doc/talks/slidy.css @@ -0,0 +1,277 @@ +/* http://www.w3.org/Talks/Tools/Slidy/slidy.css + + Copyright (c) 2005 W3C (MIT, ERCIM, Keio), All Rights Reserved. + W3C liability, trademark, document use and software licensing + rules apply, see: + + http://www.w3.org/Consortium/Legal/copyright-documents + http://www.w3.org/Consortium/Legal/copyright-software +*/ +body +{ + margin: 0 0 0 0; + padding: 0 0 0 0; + width: 100%; + height: 100%; + color: black; + background-color: white; + font-family: "Lucida Sans", "Lucida Grande", Lucida, sans-serif; + font-size: 14pt; +} + +.hidden { display: none; visibility: hidden } + +div.toolbar { + position: fixed; z-index: 200; + top: auto; bottom: 0; left: 0; right: 0; + height: 1.2em; text-align: right; + padding-left: 1em; + padding-right: 1em; + font-size: 60%; + color: red; background: rgb(240,240,240); +} + +div.background { + display: none; +} + +div.handout { + margin-left: 20px; + margin-right: 20px; +} + +div.slide.titlepage { + color: white; + background: black; + text-align: center; +} + +div.slide { + z-index: 20; + margin: 0 0 0 0; + padding-top: 0; + padding-bottom: 0; + padding-left: 20px; + padding-right: 20px; + border-width: 0; + top: 0; + bottom: 0; + left: 0; + right: 0; + line-height: 120%; + background-color: transparent; +} + +/* this rule is hidden from IE 6 and below which don't support + selector */ +div.slide + div[class].slide { page-break-before: always;} + +div.slide h1 { + padding-left: 20px; + padding-right: 20px; + padding-top: 10px; + padding-bottom: 10px; + margin-top: 0; + margin-left: 0; + margin-right: 0; + margin-bottom: 0.5em; + border-bottom: 4px solid #36c; + display: block; + font-size: 160%; + line-height: 1.2em; +} + +div.slide h2 { + font-size:120%; + line-height: 1.2em; +} + +div.toc { + position: absolute; + top: auto; + bottom: 4em; + left: 4em; + right: auto; + width: 60%; + max-width: 30em; + height: 30em; + border: solid thin black; + padding: 1em; + background: rgb(240,240,240); + color: black; + z-index: 300; + overflow: auto; + display: block; + visibility: visible; +} + +div.toc-heading { + width: 100%; + border-bottom: solid 1px rgb(180,180,180); + margin-bottom: 1em; + text-align: center; +} + +pre { + font-size: 120%; + font-weight: bold; + line-height: 140%; + padding-top: 0.2em; + padding-bottom: 0.2em; + padding-left: 1em; + padding-right: 1em; +/* + border-style: solid; + border-left-width: 1em; + border-top-width: thin; + border-right-width: thin; + border-bottom-width: thin; + border-color: #95ABD0; +*/ + color: #0F398D; + background-color: #fff8f8; +} + +@media print { + div.slide { + display: block; + visibility: visible; + position: relative; + border-top-style: solid; + border-top-width: thin; + border-top-color: black; + } + div.slide pre { font-size: 60%; padding-left: 0.5em; } + div.handout { display: block; visibility: visible; } +} + +blockquote { font-style: italic } + +img { background-color: transparent } + +p.copyright { font-size: smaller } + +.center { text-align: center } +.footnote { font-size: smaller; margin-left: 2em; } + +a img { border-width: 0; border-style: none } + +a:visited { color: navy } +a:link { color: navy } +a:hover { color: red; text-decoration: underline } +a:active { color: red; text-decoration: underline } + +a {text-decoration: none} +.navbar a:link {color: white} +.navbar a:visited {color: yellow} +.navbar a:active {color: red} +.navbar a:hover {color: red} + +ul { list-style-type: square; } +ul ul { list-style-type: disc; } +ul ul ul { list-style-type: circle; } +ul ul ul ul { list-style-type: disc; } +li { margin-left: 2em; margin-top: 0.5em; } +li li { font-size: 85%; font-style: italic } +li li li { font-size: 85%; font-style: normal } + +div dt +{ + margin-left: 0; + margin-top: 1em; + margin-bottom: 0.5em; + font-weight: bold; +} +div dd +{ + margin-left: 2em; + margin-bottom: 0.5em; +} + + +p,pre,ul,ol,blockquote,h2,h3,h4,h5,h6,dl,table { + margin-left: 1em; + margin-right: 1em; +} + +p.subhead { font-weight: bold; margin-top: 2em; } + +p.smaller { font-size: smaller } + +td,th { padding: 0.2em } + +ul { + margin: 0.5em 1.5em 0.5em 1.5em; + padding: 0; +} + +ol { + margin: 0.5em 1.5em 0.5em 1.5em; + padding: 0; +} + +ul { list-style-type: square; } +ul ul { list-style-type: disc; } +ul ul ul { list-style-type: circle; } +ul ul ul ul { list-style-type: disc; } + +ul li { + list-style: square; + //margin: 0.1em 0em 0.6em 0; + padding: 0 0 0 0; + line-height: 140%; +} + +ol li { + margin: 0.1em 0em 0.6em 1.5em; + padding: 0 0 0 0px; + line-height: 140%; + list-style-type: decimal; +} + +li ul li { + font-size: 85%; + font-style: italic; + list-style-type: disc; + background: transparent; + padding: 0 0 0 0; +} +li li ul li { + font-size: 85%; + font-style: normal; + list-style-type: circle; + background: transparent; + padding: 0 0 0 0; +} +li li li ul li { + list-style-type: disc; + background: transparent; + padding: 0 0 0 0; +} + +li ol li { + list-style-type: decimal; +} + + +li li ol li { + list-style-type: decimal; +} + +/* + setting class="outline on ol or ul makes it behave as an + ouline list where blocklevel content in li elements is + hidden by default and can be expanded or collapsed with + mouse click. Set class="expand" on li to override default +*/ + +ol.outline li:hover { cursor: pointer } +ol.outline li.nofold:hover { cursor: default } + +ul.outline li:hover { cursor: pointer } +ul.outline li.nofold:hover { cursor: default } + +ol.outline { list-style:decimal; } +ol.outline ol { list-style-type:lower-alpha } + +/* for slides with class "title" in table of contents */ +a.titleslide { font-weight: bold; font-style: italic } diff --git a/doc/talks/slidy.js b/doc/talks/slidy.js new file mode 100644 index 000000000..6a5561a6c --- /dev/null +++ b/doc/talks/slidy.js @@ -0,0 +1,2772 @@ +/* http://www.w3.org/Talks/Tools/Slidy/slidy.js + + Copyright (c) 2005 W3C (MIT, ERCIM, Keio), All Rights Reserved. + W3C liability, trademark, document use and software licensing + rules apply, see: + + http://www.w3.org/Consortium/Legal/copyright-documents + http://www.w3.org/Consortium/Legal/copyright-software +*/ + +var ns_pos = (typeof window.pageYOffset!='undefined'); +var khtml = ((navigator.userAgent).indexOf("KHTML") >= 0 ? true : false); +var opera = ((navigator.userAgent).indexOf("Opera") >= 0 ? true : false); +var ie7 = (!ns_pos && navigator.userAgent.indexOf("MSIE 7") != -1); + +window.onload = startup; // equivalent to onload on body element + +// IE only event handlers to ensure all slides are printed +// I don't yet know how to emulate these for other browsers +window.onbeforeprint = beforePrint; +window.onafterprint = afterPrint; + +// hack to hide slides while loading +setTimeout(hideAll, 50); + +function hideAll() +{ + if (document.body) + document.body.style.visibility = "hidden"; + else + setTimeout(hideAll, 50); +} + +var slidenum = 0; // integer slide count: 0, 1, 2, ... +var slides; // set to array of slide div's +var slideNumElement; // element containing slide number +var notes; // set to array of handout div's +var backgrounds; // set to array of background div's +var toolbar; // element containing toolbar +var title; // document title +var lastShown = null; // last incrementally shown item +var eos = null; // span element for end of slide indicator +var toc = null; // table of contents +var outline = null; // outline element with the focus +var selectedTextLen; // length of drag selection on document + +var viewAll = 0; // 1 to view all slides + handouts +var wantToolbar = 1; // 0 if toolbar isn't wanted +var mouseClickEnabled = true; // enables left click for next slide +var scrollhack = 0; // IE work around for position: fixed + +var helpAnchor; // used for keyboard focus hack in showToolbar() +var helpPage = "http://www.w3.org/Talks/Tools/Slidy/help.html"; +var helpText = "Navigate with mouse click, space bar, Cursor Left/Right, " + + "or Pg Up and Pg Dn. Use S and B to change font size."; + +var sizeIndex = 0; +var sizeAdjustment = 0; +var sizes = new Array("10pt", "12pt", "14pt", "16pt", "18pt", "20pt", + "22pt", "24pt", "26pt", "28pt", "30pt", "32pt"); + +var okayForIncremental = incrementalElementList(); + +// needed for efficient resizing +var lastWidth = 0; +var lastHeight = 0; + +// Needed for cross browser support for relative width/height on +// object elements. The work around is to save width/height attributes +// and then to recompute absolute width/height dimensions on resizing +var objects; + +// updated to language specified by html file +var lang = "en"; + +//var localize = {}; + +// for each language there is an associative array +var strings_es = { + "slide":"pág.", + "help?":"Ayuda", + "contents?":"Índice", + "table of contents":"tabla de contenidos", + "Table of Contents":"Tabla de Contenidos", + "restart presentation":"Reiniciar presentación", + "restart?":"Inicio" + }; + +strings_es[helpText] = + "Utilice el ratón, barra espaciadora, teclas Izda/Dhca, " + + "o Re pág y Av pág. Use S y B para cambiar el tamaño de fuente."; + +var strings_nl = { + "slide":"pagina", + "help?":"Help?", + "contents?":"Inhoud?", + "table of contents":"inhoudsopgave", + "Table of Contents":"Inhoudsopgave", + "restart presentation":"herstart presentatie", + "restart?":"Herstart?" + }; + +strings_nl[helpText] = + "Navigeer d.m.v. het muis, spatiebar, Links/Rechts toetsen, " + + "of PgUp en PgDn. Gebruik S en B om de karaktergrootte te veranderen."; + +var strings_de = { + "slide":"Seite", + "help?":"Hilfe", + "contents?":"Übersicht", + "table of contents":"Inhaltsverzeichnis", + "Table of Contents":"Inhaltsverzeichnis", + "restart presentation":"Präsentation neu starten", + "restart?":"Neustart" + }; + +strings_de[helpText] = + "Benutzen Sie die Maus, Leerschlag, die Cursortasten links/rechts" + + "oder Page up/Page Down zum Wechseln der Seiten und S und B für die Schriftgrösse."; + +var strings_pl = { + "slide":"slajd", + "help?":"pomoc?", + "contents?":"spis treści?", + "table of contents":"spis treści", + "Table of Contents":"Spis Treści", + "restart presentation":"Restartuj prezentację", + "restart?":"restart?" + }; + +strings_pl[helpText] = + "Zmieniaj slajdy klikając myszą, naciskając spację, strzałki lewo/prawo" + + "lub PgUp / PgDn. Użyj klawiszy S i B, aby zmienić rozmiar czczionki."; + +var strings_fr = { + "slide":"page", + "help?":"Aide", + "contents?":"Index", + "table of contents":"table des matières", + "Table of Contents":"Table des matières", + "restart presentation":"Recommencer l'exposé", + "restart?":"Début" + }; + +strings_fr[helpText] = +    "Naviguez avec la souris, la barre d'espace, les flèches" + + "gauche/droite ou les touches Pg Up, Pg Dn. Utilisez " + +    "les touches S et B pour modifier la taille de la police."; + +var strings_hu = { + "slide":"oldal", + "help?":"segítség", + "contents?":"tartalom", + "table of contents":"tartalomjegyzék", + "Table of Contents":"Tartalomjegyzék", + "restart presentation":"bemutató újraindítása", + "restart?":"újraindítás" + }; + +strings_hu[helpText] = + "Az oldalak közti lépkedéshez kattintson az egérrel, vagy használja a szóköz, a bal, vagy a jobb nyíl, " + + "illetve a Page Down, Page Up billentyűket. Az S és a B billentyűkkel változtathatja a szöveg méretét."; + +var strings_it = { + "slide":"pag.", + "help?":"Aiuto", + "contents?":"Indice", + "table of contents":"indice", + "Table of Contents":"Indice", + "restart presentation":"Ricominciare la presentazione", + "restart?":"Inizio" + }; + +strings_it[helpText] = + "Navigare con mouse, barra spazio, frecce sinistra/destra o " + + "PgUp e PgDn. Usare S e B per cambiare la dimensione dei caratteri."; + +var strings_el = { + "slide":"σελίδα", + "help?":"βοήθεια;", + "contents?":"περιεχόμενα;", + "table of contents":"πίνακας περιεχομένων", + "Table of Contents":"Πίνακας Περιεχομένων", + "restart presentation":"επανεκκίνηση παρουσίασης", + "restart?":"επανεκκίνηση;" + }; + +strings_el[helpText] = + "Πλοηγηθείτε με το κλίκ του ποντικιού, το space, τα βέλη αριστερά/δεξιά, " + + "ή Page Up και Page Down. Χρησιμοποιήστε τα πλήκτρα S και B για να αλλάξετε " + + "το μέγεθος της γραμματοσειράς."; + +var strings_ja = { + "slide":"スライド", + "help?":"ヘルプ", + "contents?":"目次", + "table of contents":"目次を表示", + "Table of Contents":"目次", + "restart presentation":"最初から再生", + "restart?":"最初から" +}; + +strings_ja[helpText] = + "マウス左クリック ・ スペース ・ 左右キー " + + "または Page Up ・ Page Downで操作, S ・ Bでフォントサイズ変更"; + + +// each such language array is declared in the localize array +// used indirectly as in help.innerHTML = "help".localize(); +var localize = { + "es":strings_es, + "nl":strings_nl, + "de":strings_de, + "pl":strings_pl, + "fr":strings_fr, + "hu":strings_hu, + "it":strings_it, + "el":strings_el, + "jp":strings_ja + }; + +/* general initialization */ +function startup() +{ + // find human language from html element + // for use in localizing strings + lang = document.body.parentNode.getAttribute("lang"); + + if (!lang) + lang = document.body.parentNode.getAttribute("xml:lang"); + + if (!lang) + lang = "en"; + + document.body.style.visibility = "visible"; + title = document.title; + toolbar = addToolbar(); + wrapImplicitSlides(); + slides = collectSlides(); + notes = collectNotes(); + objects = document.body.getElementsByTagName("object"); + backgrounds = collectBackgrounds(); + patchAnchors(); + + slidenum = findSlideNumber(location.href); + window.offscreenbuffering = true; + sizeAdjustment = findSizeAdjust(); + hideImageToolbar(); // suppress IE image toolbar popup + initOutliner(); // activate fold/unfold support + + if (slides.length > 0) + { + var slide = slides[slidenum]; + slide.style.position = "absolute"; + + if (slidenum > 0) + { + setVisibilityAllIncremental("visible"); + lastShown = previousIncrementalItem(null); + setEosStatus(true); + } + else + { + lastShown = null; + setVisibilityAllIncremental("hidden"); + setEosStatus(!nextIncrementalItem(lastShown)); + } + + setLocation(); + } + + toc = tableOfContents(); + hideTableOfContents(); + + // bind event handlers + document.onclick = mouseButtonClick; + document.onmouseup = mouseButtonUp; + document.onkeydown = keyDown; + window.onresize = resized; + window.onscroll = scrolled; + singleSlideView(); + + setLocation(); + resized(); + + if (ie7) + setTimeout("ieHack()", 100); + + showToolbar(); +} + +// add localize method to all strings for use +// as in help.innerHTML = "help".localize(); +String.prototype.localize = function() +{ + if (this == "") + return this; + + // try full language code, e.g. en-US + var s, lookup = localize[lang]; + + if (lookup) + { + s = lookup[this]; + + if (s) + return s; + } + + // try en if undefined for en-US + var lg = lang.split("-"); + + if (lg.length > 1) + { + lookup = localize[lg[0]]; + + if (lookup) + { + s = lookup[this]; + + if (s) + return s; + } + } + + // otherwise string as is + return this; +} + +// suppress IE's image toolbar pop up +function hideImageToolbar() +{ + if (!ns_pos) + { + var images = document.getElementsByTagName("IMG"); + + for (var i = 0; i < images.length; ++i) + images[i].setAttribute("galleryimg", "no"); + } +} + +// hack to persuade IE to compute correct document height +// as needed for simulating fixed positioning of toolbar +function ieHack() +{ + window.resizeBy(0,-1); + window.resizeBy(0, 1); +} + +// Firefox reload SVG bug work around +function reload(e) +{ + if (!e) + var e = window.event; + + hideBackgrounds(); + setTimeout("document.reload();", 100); + + stopPropagation(e); + e.cancel = true; + e.returnValue = false; + + return false; +} + +// Safari and Konqueror don't yet support getComputedStyle() +// and they always reload page when location.href is updated +function isKHTML() +{ + var agent = navigator.userAgent; + return (agent.indexOf("KHTML") >= 0 ? true : false); +} + +function resized() +{ + var width = 0; + + if ( typeof( window.innerWidth ) == 'number' ) + width = window.innerWidth; // Non IE browser + else if (document.documentElement && document.documentElement.clientWidth) + width = document.documentElement.clientWidth; // IE6 + else if (document.body && document.body.clientWidth) + width = document.body.clientWidth; // IE4 + + var height = 0; + + if ( typeof( window.innerHeight ) == 'number' ) + height = window.innerHeight; // Non IE browser + else if (document.documentElement && document.documentElement.clientHeight) + height = document.documentElement.clientHeight; // IE6 + else if (document.body && document.body.clientHeight) + height = document.body.clientHeight; // IE4 + + if (height && (width/height > 1.05*1024/768)) + { + width = height * 1024.0/768; + } + + // IE fires onresize even when only font size is changed! + // so we do a check to avoid blocking < and > actions + if (width != lastWidth || height != lastHeight) + { + if (width >= 1100) + sizeIndex = 5; // 4 + else if (width >= 1000) + sizeIndex = 4; // 3 + else if (width >= 800) + sizeIndex = 3; // 2 + else if (width >= 600) + sizeIndex = 2; // 1 + else if (width) + sizeIndex = 0; + + // add in font size adjustment from meta element e.g. + // + // useful when slides have too much content ;-) + + if (0 <= sizeIndex + sizeAdjustment && + sizeIndex + sizeAdjustment < sizes.length) + sizeIndex = sizeIndex + sizeAdjustment; + + // enables cross browser use of relative width/height + // on object elements for use with SVG and Flash media + adjustObjectDimensions(width, height); + + document.body.style.fontSize = sizes[sizeIndex]; + + lastWidth = width; + lastHeight = height; + + // force reflow to work around Mozilla bug + //if (ns_pos) + { + var slide = slides[slidenum]; + hideSlide(slide); + showSlide(slide); + } + + // force correct positioning of toolbar + refreshToolbar(200); + } +} + +function scrolled() +{ + if (toolbar && !ns_pos && !ie7) + { + hackoffset = scrollXOffset(); + // hide toolbar + toolbar.style.display = "none"; + + // make it reappear later + if (scrollhack == 0 && !viewAll) + { + setTimeout(showToolbar, 1000); + scrollhack = 1; + } + } +} + +// used to ensure IE refreshes toolbar in correct position +function refreshToolbar(interval) +{ + if (!ns_pos && !ie7) + { + hideToolbar(); + setTimeout(showToolbar, interval); + } +} + +// restores toolbar after short delay +function showToolbar() +{ + if (wantToolbar) + { + if (!ns_pos) + { + // adjust position to allow for scrolling + var xoffset = scrollXOffset(); + toolbar.style.left = xoffset; + toolbar.style.right = xoffset; + + // determine vertical scroll offset + //var yoffset = scrollYOffset(); + + // bottom is doc height - window height - scroll offset + //var bottom = documentHeight() - lastHeight - yoffset + + //if (yoffset > 0 || documentHeight() > lastHeight) + // bottom += 16; // allow for height of scrollbar + + toolbar.style.bottom = 0; //bottom; + } + + toolbar.style.display = "block"; + toolbar.style.visibility = "visible"; + } + + scrollhack = 0; + + + // set the keyboard focus to the help link on the + // toolbar to ensure that document has the focus + // IE doesn't always work with window.focus() + // and this hack has benefit of Enter for help + + try + { + if (!opera) + helpAnchor.focus(); + } + catch (e) + { + } +} + +function test() +{ + var s = "docH: " + documentHeight() + + " winH: " + lastHeight + + " yoffset: " + scrollYOffset() + + " toolbot: " + (documentHeight() - lastHeight - scrollYOffset()); + + //alert(s); + + var slide = slides[slidenum]; + // IE getAttribute requires "class" to be "className" + var name = ns_pos ? "class" : "className"; + var style = (slide.currentStyle ? slide.currentStyle["backgroundColor"] : + document.defaultView.getComputedStyle(slide, '').getPropertyValue("background-color")); + alert("class='" + slide.getAttribute(name) + "' backgroundColor: " + style); +} + +function hideToolbar() +{ + toolbar.style.display = "none"; + toolbar.style.visibility = "hidden"; + window.focus(); +} + +// invoked via F key +function toggleToolbar() +{ + if (!viewAll) + { + if (toolbar.style.display == "none") + { + toolbar.style.display = "block"; + toolbar.style.visibility = "visible"; + wantToolbar = 1; + } + else + { + toolbar.style.display = "none"; + toolbar.style.visibility = "hidden"; + wantToolbar = 0; + } + } +} + +function scrollXOffset() +{ + if (window.pageXOffset) + return self.pageXOffset; + + if (document.documentElement && + document.documentElement.scrollLeft) + return document.documentElement.scrollLeft; + + if (document.body) + return document.body.scrollLeft; + + return 0; +} + + +function scrollYOffset() +{ + if (window.pageYOffset) + return self.pageYOffset; + + if (document.documentElement && + document.documentElement.scrollTop) + return document.documentElement.scrollTop; + + if (document.body) + return document.body.scrollTop; + + return 0; +} + +// looking for a way to determine height of slide content +// the slide itself is set to the height of the window +function optimizeFontSize() +{ + var slide = slides[slidenum]; + + //var dh = documentHeight(); //getDocHeight(document); + var dh = slide.scrollHeight; + var wh = getWindowHeight(); + var u = 100 * dh / wh; + + alert("window utilization = " + u + "% (doc " + + dh + " win " + wh + ")"); +} + +function getDocHeight(doc) // from document object +{ + if (!doc) + doc = document; + + if (doc && doc.body && doc.body.offsetHeight) + return doc.body.offsetHeight; // ns/gecko syntax + + if (doc && doc.body && doc.body.scrollHeight) + return doc.body.scrollHeight; + + alert("couldn't determine document height"); +} + +function getWindowHeight() +{ + if ( typeof( window.innerHeight ) == 'number' ) + return window.innerHeight; // Non IE browser + + if (document.documentElement && document.documentElement.clientHeight) + return document.documentElement.clientHeight; // IE6 + + if (document.body && document.body.clientHeight) + return document.body.clientHeight; // IE4 +} + + + +function documentHeight() +{ + var sh, oh; + + sh = document.body.scrollHeight; + oh = document.body.offsetHeight; + + if (sh && oh) + { + return (sh > oh ? sh : oh); + } + + // no idea! + return 0; +} + +function smaller() +{ + if (sizeIndex > 0) + { + --sizeIndex; + } + + toolbar.style.display = "none"; + document.body.style.fontSize = sizes[sizeIndex]; + var slide = slides[slidenum]; + hideSlide(slide); + showSlide(slide); + setTimeout(showToolbar, 300); +} + +function bigger() +{ + if (sizeIndex < sizes.length - 1) + { + ++sizeIndex; + } + + toolbar.style.display = "none"; + document.body.style.fontSize = sizes[sizeIndex]; + var slide = slides[slidenum]; + hideSlide(slide); + showSlide(slide); + setTimeout(showToolbar, 300); +} + +// enables cross browser use of relative width/height +// on object elements for use with SVG and Flash media +// with thanks to Ivan Herman for the suggestion +function adjustObjectDimensions(width, height) +{ + for( var i = 0; i < objects.length; i++ ) + { + var obj = objects[i]; + var mimeType = obj.getAttribute("type"); + + if (mimeType == "image/svg+xml" || mimeType == "application/x-shockwave-flash") + { + if ( !obj.initialWidth ) + obj.initialWidth = obj.getAttribute("width"); + + if ( !obj.initialHeight ) + obj.initialHeight = obj.getAttribute("height"); + + if ( obj.initialWidth && obj.initialWidth.charAt(obj.initialWidth.length-1) == "%" ) + { + var w = parseInt(obj.initialWidth.slice(0, obj.initialWidth.length-1)); + var newW = width * (w/100.0); + obj.setAttribute("width",newW); + } + + if ( obj.initialHeight && obj.initialHeight.charAt(obj.initialHeight.length-1) == "%" ) + { + var h = parseInt(obj.initialHeight.slice(0, obj.initialHeight.length-1)); + var newH = height * (h/100.0); + obj.setAttribute("height", newH); + } + } + } +} + +function cancel(event) +{ + if (event) + { + event.cancel = true; + event.returnValue = false; + + if (event.preventDefault) + event.preventDefault(); + } + + return false; +} + +// See e.g. http://www.quirksmode.org/js/events/keys.html for keycodes +function keyDown(event) +{ + var key; + + if (!event) + var event = window.event; + + // kludge around NS/IE differences + if (window.event) + key = window.event.keyCode; + else if (event.which) + key = event.which; + else + return true; // Yikes! unknown browser + + // ignore event if key value is zero + // as for alt on Opera and Konqueror + if (!key) + return true; + + // check for concurrent control/command/alt key + // but are these only present on mouse events? + + if (event.ctrlKey || event.altKey || event.metaKey) + return true; + + // dismiss table of contents if visible + if (isShownToc() && key != 9 && key != 16 && key != 38 && key != 40) + { + hideTableOfContents(); + + if (key == 27 || key == 84 || key == 67) + return cancel(event); + } + + if (key == 34) // Page Down + { + nextSlide(false); + return cancel(event); + } + else if (key == 33) // Page Up + { + previousSlide(false); + return cancel(event); + } + else if (key == 32) // space bar + { + nextSlide(true); + return cancel(event); + } + else if (key == 37 || key == 38) // Left arrow || Up arrow + { + previousSlide(!event.shiftKey); + return cancel(event); + } + else if (key == 36) // Home + { + firstSlide(); + return cancel(event); + } + else if (key == 35) // End + { + lastSlide(); + return cancel(event); + } + else if (key == 39 || key == 40) // Right arrow || Down arrow + { + nextSlide(!event.shiftKey); + return cancel(event); + } + else if (key == 13) // Enter + { + if (outline) + { + if (outline.visible) + fold(outline); + else + unfold(outline); + + return cancel(event); + } + } + else if (key == 188) // < for smaller fonts + { + smaller(); + return cancel(event); + } + else if (key == 190) // > for larger fonts + { + bigger(); + return cancel(event); + } + else if (key == 189 || key == 109) // - for smaller fonts + { + smaller(); + return cancel(event); + } + else if (key == 187 || key == 191 || key == 107) // = + for larger fonts + { + bigger(); + return cancel(event); + } + else if (key == 83) // S for smaller fonts + { + smaller(); + return cancel(event); + } + else if (key == 66) // B for larger fonts + { + bigger(); + return cancel(event); + } + else if (key == 90) // Z for last slide + { + lastSlide(); + return cancel(event); + } + else if (key == 70) // F for toggle toolbar + { + toggleToolbar(); + return cancel(event); + } + else if (key == 65) // A for toggle view single/all slides + { + toggleView(); + return cancel(event); + } + else if (key == 75) // toggle action of left click for next page + { + mouseClickEnabled = !mouseClickEnabled; + alert((mouseClickEnabled ? "enabled" : "disabled") + " mouse click advance"); + return cancel(event); + } + else if (key == 84 || key == 67) // T or C for table of contents + { + if (toc) + showTableOfContents(); + + return cancel(event); + } + else if (key == 72) // H for help + { + window.location = helpPage; + return cancel(event); + } + + //else if (key == 93) // Windows menu key + //alert("lastShown is " + lastShown); + //else alert("key code is "+ key); + + + return true; +} + +// make note of length of selected text +// as this evaluates to zero in click event +function mouseButtonUp(e) +{ + selectedTextLen = getSelectedText().length; +} + +// right mouse button click is reserved for context menus +// it is more reliable to detect rightclick than leftclick +function mouseButtonClick(e) +{ + var rightclick = false; + var leftclick = false; + var middleclick = false; + var target; + + if (!e) + var e = window.event; + + if (e.target) + target = e.target; + else if (e.srcElement) + target = e.srcElement; + + // work around Safari bug + if (target.nodeType == 3) + target = target.parentNode; + + if (e.which) // all browsers except IE + { + leftclick = (e.which == 1); + middleclick = (e.which == 2); + rightclick = (e.which == 3); + } + else if (e.button) + { + // Konqueror gives 1 for left, 4 for middle + // IE6 gives 0 for left and not 1 as I expected + + if (e.button == 4) + middleclick = true; + + // all browsers agree on 2 for right button + rightclick = (e.button == 2); + } + else leftclick = true; + + // dismiss table of contents + hideTableOfContents(); + + if (selectedTextLen > 0) + { + stopPropagation(e); + e.cancel = true; + e.returnValue = false; + return false; + } + + // check if target is something that probably want's clicks + // e.g. embed, object, input, textarea, select, option + + if (mouseClickEnabled && leftclick && + target.nodeName != "EMBED" && + target.nodeName != "OBJECT" && + target.nodeName != "INPUT" && + target.nodeName != "TEXTAREA" && + target.nodeName != "SELECT" && + target.nodeName != "OPTION") + { + nextSlide(true); + stopPropagation(e); + e.cancel = true; + e.returnValue = false; + } +} + +function previousSlide(incremental) +{ + if (!viewAll) + { + var slide; + + if ((incremental || slidenum == 0) && lastShown != null) + { + lastShown = hidePreviousItem(lastShown); + setEosStatus(false); + } + else if (slidenum > 0) + { + slide = slides[slidenum]; + hideSlide(slide); + + slidenum = slidenum - 1; + slide = slides[slidenum]; + setVisibilityAllIncremental("visible"); + lastShown = previousIncrementalItem(null); + setEosStatus(true); + showSlide(slide); + } + + setLocation(); + + if (!ns_pos) + refreshToolbar(200); + } +} + +function nextSlide(incremental) +{ + if (!viewAll) + { + var slide, last = lastShown; + + if (incremental || slidenum == slides.length - 1) + lastShown = revealNextItem(lastShown); + + if ((!incremental || lastShown == null) && slidenum < slides.length - 1) + { + slide = slides[slidenum]; + hideSlide(slide); + + slidenum = slidenum + 1; + slide = slides[slidenum]; + lastShown = null; + setVisibilityAllIncremental("hidden"); + showSlide(slide); + } + else if (!lastShown) + { + if (last && incremental) + lastShown = last; + } + + setLocation(); + + setEosStatus(!nextIncrementalItem(lastShown)); + + if (!ns_pos) + refreshToolbar(200); + } +} + +// to first slide with nothing revealed +// i.e. state at start of presentation +function firstSlide() +{ + if (!viewAll) + { + var slide; + + if (slidenum != 0) + { + slide = slides[slidenum]; + hideSlide(slide); + + slidenum = 0; + slide = slides[slidenum]; + lastShown = null; + setVisibilityAllIncremental("hidden"); + showSlide(slide); + } + + setEosStatus(!nextIncrementalItem(lastShown)); + setLocation(); + } +} + + +// to last slide with everything revealed +// i.e. state at end of presentation +function lastSlide() +{ + if (!viewAll) + { + var slide; + + lastShown = null; //revealNextItem(lastShown); + + if (lastShown == null && slidenum < slides.length - 1) + { + slide = slides[slidenum]; + hideSlide(slide); + slidenum = slides.length - 1; + slide = slides[slidenum]; + setVisibilityAllIncremental("visible"); + lastShown = previousIncrementalItem(null); + + showSlide(slide); + } + else + { + setVisibilityAllIncremental("visible"); + lastShown = previousIncrementalItem(null); + } + + setEosStatus(true); + setLocation(); + } +} + +function setEosStatus(state) +{ + if (eos) + eos.style.color = (state ? "rgb(240,240,240)" : "red"); +} + +function showSlide(slide) +{ + syncBackground(slide); + window.scrollTo(0,0); + slide.style.visibility = "visible"; + slide.style.display = "block"; +} + +function hideSlide(slide) +{ + slide.style.visibility = "hidden"; + slide.style.display = "none"; +} + +function beforePrint() +{ + showAllSlides(); + hideToolbar(); +} + +function afterPrint() +{ + if (!viewAll) + { + singleSlideView(); + showToolbar(); + } +} + +function printSlides() +{ + beforePrint(); + window.print(); + afterPrint(); +} + +function toggleView() +{ + if (viewAll) + { + singleSlideView(); + showToolbar(); + viewAll = 0; + } + else + { + showAllSlides(); + hideToolbar(); + viewAll = 1; + } +} + +// prepare for printing +function showAllSlides() +{ + var slide; + + for (var i = 0; i < slides.length; ++i) + { + slide = slides[i]; + + slide.style.position = "relative"; + slide.style.borderTopStyle = "solid"; + slide.style.borderTopWidth = "thin"; + slide.style.borderTopColor = "black"; + + try { + if (i == 0) + slide.style.pageBreakBefore = "avoid"; + else + slide.style.pageBreakBefore = "always"; + } + catch (e) + { + //do nothing + } + + setVisibilityAllIncremental("visible"); + showSlide(slide); + } + + var note; + + for (var i = 0; i < notes.length; ++i) + { + showSlide(notes[i]); + } + + // no easy way to render background under each slide + // without duplicating the background divs for each slide + // therefore hide backgrounds to avoid messing up slides + hideBackgrounds(); +} + +// restore after printing +function singleSlideView() +{ + var slide; + + for (var i = 0; i < slides.length; ++i) + { + slide = slides[i]; + + slide.style.position = "absolute"; + + if (i == slidenum) + { + slide.style.borderStyle = "none"; + showSlide(slide); + } + else + { + slide.style.borderStyle = "none"; + hideSlide(slide); + } + } + + setVisibilityAllIncremental("visible"); + lastShown = previousIncrementalItem(null); + + var note; + + for (var i = 0; i < notes.length; ++i) + { + hideSlide(notes[i]); + } +} + +// the string str is a whitespace separated list of tokens +// test if str contains a particular token, e.g. "slide" +function hasToken(str, token) +{ + if (str) + { + // define pattern as regular expression + var pattern = /\w+/g; + + // check for matches + // place result in array + var result = str.match(pattern); + + // now check if desired token is present + for (var i = 0; i < result.length; i++) + { + if (result[i] == token) + return true; + } + } + + return false; +} + +function getClassList(element) +{ + if (typeof window.pageYOffset =='undefined') + return element.getAttribute("className"); + + return element.getAttribute("class"); +} + +function hasClass(element, name) +{ + var regexp = new RegExp("(^| )" + name + "\W*"); + + if (regexp.test(getClassList(element))) + return true; + + return false; + +} + +function removeClass(element, name) +{ + // IE getAttribute requires "class" to be "className" + var clsname = ns_pos ? "class" : "className"; + var clsval = element.getAttribute(clsname); + + var regexp = new RegExp("(^| )" + name + "\W*"); + + if (clsval) + { + clsval = clsval.replace(regexp, ""); + element.setAttribute(clsname, clsval); + } +} + +function addClass(element, name) +{ + if (!hasClass(element, name)) + { + // IE getAttribute requires "class" to be "className" + var clsname = ns_pos ? "class" : "className"; + var clsval = element.getAttribute(clsname); + element.setAttribute(clsname, (clsval ? clsval + " " + name : name)); + } +} + +// wysiwyg editors make it hard to use div elements +// e.g. amaya loses the div when you copy and paste +// this function wraps div elements around implicit +// slides which start with an h1 element and continue +// up to the next heading or div element +function wrapImplicitSlides() +{ + var i, heading, node, next, div; + var headings = document.getElementsByTagName("h1"); + + if (!headings) + return; + + for (i = 0; i < headings.length; ++i) + { + heading = headings[i]; + + if (heading.parentNode != document.body) + continue; + + node = heading.nextSibling; + + div = document.createElement("div"); + div.setAttribute((ns_pos ? "class" : "className"), "slide"); + document.body.replaceChild(div, heading); + div.appendChild(heading); + + while (node) + { + if (node.nodeType == 1 && // an element + (node.nodeName == "H1" || + node.nodeName == "h1" || + node.nodeName == "DIV" || + node.nodeName == "div")) + break; + + next = node.nextSibling; + node = document.body.removeChild(node); + div.appendChild(node); + node = next; + } + } +} + +// return new array of all slides +function collectSlides() +{ + var slides = new Array(); + var divs = document.body.getElementsByTagName("div"); + + for (var i = 0; i < divs.length; ++i) + { + div = divs.item(i); + + if (hasClass(div, "slide")) + { + // add slide to collection + slides[slides.length] = div; + + // hide each slide as it is found + div.style.display = "none"; + div.style.visibility = "hidden"; + + // add dummy
at end for scrolling hack + var node1 = document.createElement("br"); + div.appendChild(node1); + var node2 = document.createElement("br"); + div.appendChild(node2); + } + else if (hasClass(div, "background")) + { // work around for Firefox SVG reload bug + // which otherwise replaces 1st SVG graphic with 2nd + div.style.display = "block"; + } + } + + return slides; +} + +// return new array of all
+function collectNotes() +{ + var notes = new Array(); + var divs = document.body.getElementsByTagName("div"); + + for (var i = 0; i < divs.length; ++i) + { + div = divs.item(i); + + if (hasClass(div, "handout")) + { + // add slide to collection + notes[notes.length] = div; + + // hide handout notes as they are found + div.style.display = "none"; + div.style.visibility = "hidden"; + } + } + + return notes; +} + +// return new array of all
+// including named backgrounds e.g. class="background titlepage" +function collectBackgrounds() +{ + var backgrounds = new Array(); + var divs = document.body.getElementsByTagName("div"); + + for (var i = 0; i < divs.length; ++i) + { + div = divs.item(i); + + if (hasClass(div, "background")) + { + // add slide to collection + backgrounds[backgrounds.length] = div; + + // hide named backgrounds as they are found + // e.g. class="background epilog" + if (getClassList(div) != "background") + { + div.style.display = "none"; + div.style.visibility = "hidden"; + } + } + } + + return backgrounds; +} + +// show just the backgrounds pertinent to this slide +function syncBackground(slide) +{ + var background; + var bgColor; + + if (slide.currentStyle) + bgColor = slide.currentStyle["backgroundColor"]; + else if (document.defaultView) + { + var styles = document.defaultView.getComputedStyle(slide,null); + + if (styles) + bgColor = styles.getPropertyValue("background-color"); + else // broken implementation probably due Safari or Konqueror + { + //alert("defective implementation of getComputedStyle()"); + bgColor = "transparent"; + } + } + else + bgColor == "transparent"; + + if (bgColor == "transparent") + { + var slideClass = getClassList(slide); + + for (var i = 0; i < backgrounds.length; i++) + { + background = backgrounds[i]; + + var bgClass = getClassList(background); + + if (matchingBackground(slideClass, bgClass)) + { + background.style.display = "block"; + background.style.visibility = "visible"; + } + else + { + background.style.display = "none"; + background.style.visibility = "hidden"; + } + } + } + else // forcibly hide all backgrounds + hideBackgrounds(); +} + +function hideBackgrounds() +{ + for (var i = 0; i < backgrounds.length; i++) + { + background = backgrounds[i]; + background.style.display = "none"; + background.style.visibility = "hidden"; + } +} + +// compare classes for slide and background +function matchingBackground(slideClass, bgClass) +{ + if (bgClass == "background") + return true; + + // define pattern as regular expression + var pattern = /\w+/g; + + // check for matches and place result in array + var result = slideClass.match(pattern); + + // now check if desired name is present for background + for (var i = 0; i < result.length; i++) + { + if (hasToken(bgClass, result[i])) + return true; + } + + return false; +} + +// left to right traversal of root's content +function nextNode(root, node) +{ + if (node == null) + return root.firstChild; + + if (node.firstChild) + return node.firstChild; + + if (node.nextSibling) + return node.nextSibling; + + for (;;) + { + node = node.parentNode; + + if (!node || node == root) + break; + + if (node && node.nextSibling) + return node.nextSibling; + } + + return null; +} + +// right to left traversal of root's content +function previousNode(root, node) +{ + if (node == null) + { + node = root.lastChild; + + if (node) + { + while (node.lastChild) + node = node.lastChild; + } + + return node; + } + + if (node.previousSibling) + { + node = node.previousSibling; + + while (node.lastChild) + node = node.lastChild; + + return node; + } + + if (node.parentNode != root) + return node.parentNode; + + return null; +} + +// HTML elements that can be used with class="incremental" +// note that you can also put the class on containers like +// up, ol, dl, and div to make their contents appear +// incrementally. Upper case is used since this is what +// browsers report for HTML node names (text/html). +function incrementalElementList() +{ + var inclist = new Array(); + inclist["P"] = true; + inclist["PRE"] = true; + inclist["LI"] = true; + inclist["BLOCKQUOTE"] = true; + inclist["DT"] = true; + inclist["DD"] = true; + inclist["H2"] = true; + inclist["H3"] = true; + inclist["H4"] = true; + inclist["H5"] = true; + inclist["H6"] = true; + inclist["SPAN"] = true; + inclist["ADDRESS"] = true; + inclist["TABLE"] = true; + inclist["TR"] = true; + inclist["TH"] = true; + inclist["TD"] = true; + inclist["IMG"] = true; + inclist["OBJECT"] = true; + return inclist; +} + +function nextIncrementalItem(node) +{ + var slide = slides[slidenum]; + + for (;;) + { + node = nextNode(slide, node); + + if (node == null || node.parentNode == null) + break; + + if (node.nodeType == 1) // ELEMENT + { + if (node.nodeName == "BR") + continue; + + if (hasClass(node, "incremental") + && okayForIncremental[node.nodeName]) + return node; + + if (hasClass(node.parentNode, "incremental") + && !hasClass(node, "non-incremental")) + return node; + } + } + + return node; +} + +function previousIncrementalItem(node) +{ + var slide = slides[slidenum]; + + for (;;) + { + node = previousNode(slide, node); + + if (node == null || node.parentNode == null) + break; + + if (node.nodeType == 1) + { + if (node.nodeName == "BR") + continue; + + if (hasClass(node, "incremental") + && okayForIncremental[node.nodeName]) + return node; + + if (hasClass(node.parentNode, "incremental") + && !hasClass(node, "non-incremental")) + return node; + } + } + + return node; +} + +// set visibility for all elements on current slide with +// a parent element with attribute class="incremental" +function setVisibilityAllIncremental(value) +{ + var node = nextIncrementalItem(null); + + while (node) + { + node.style.visibility = value; + node = nextIncrementalItem(node); + } +} + +// reveal the next hidden item on the slide +// node is null or the node that was last revealed +function revealNextItem(node) +{ + node = nextIncrementalItem(node); + + if (node && node.nodeType == 1) // an element + node.style.visibility = "visible"; + + return node; +} + + +// exact inverse of revealNextItem(node) +function hidePreviousItem(node) +{ + if (node && node.nodeType == 1) // an element + node.style.visibility = "hidden"; + + return previousIncrementalItem(node); +} + + +/* set click handlers on all anchors */ +function patchAnchors() +{ + var anchors = document.body.getElementsByTagName("a"); + + for (var i = 0; i < anchors.length; ++i) + { + anchors[i].onclick = clickedAnchor; + } +} + +function clickedAnchor(e) +{ + if (!e) + var e = window.event; + + // compare this.href with location.href + // for link to another slide in this doc + + if (pageAddress(this.href) == pageAddress(location.href)) + { + // yes, so find new slide number + var newslidenum = findSlideNumber(this.href); + + if (newslidenum != slidenum) + { + slide = slides[slidenum]; + hideSlide(slide); + slidenum = newslidenum; + slide = slides[slidenum]; + showSlide(slide); + setLocation(); + } + } + else if (this.target == null) + location.href = this.href; + + this.blur(); + stopPropagation(e); +} + +function pageAddress(uri) +{ + var i = uri.indexOf("#"); + + // check if anchor is entire page + + if (i < 0) + return uri; // yes + + return uri.substr(0, i); +} + +function showSlideNumber() +{ + slideNumElement.innerHTML = "slide".localize() + " " + + (slidenum + 1) + "/" + slides.length; +} + +function setLocation() +{ + var uri = pageAddress(location.href); + + //if (slidenum > 0) + uri = uri + "#(" + (slidenum+1) + ")"; + + if (uri != location.href && !khtml) + location.href = uri; + + document.title = title + " (" + (slidenum+1) + ")"; + //document.title = (slidenum+1) + ") " + slideName(slidenum); + + showSlideNumber(); +} + +// find current slide based upon location +// first find target anchor and then look +// for associated div element enclosing it +// finally map that to slide number +function findSlideNumber(uri) +{ + // first get anchor from page location + + var i = uri.indexOf("#"); + + // check if anchor is entire page + + if (i < 0) + return 0; // yes + + var anchor = unescape(uri.substr(i+1)); + + // now use anchor as XML ID to find target + var target = document.getElementById(anchor); + + if (!target) + { + // does anchor look like "(2)" for slide 2 ?? + // where first slide is (1) + var re = /\((\d)+\)/; + + if (anchor.match(re)) + { + var num = parseInt(anchor.substring(1, anchor.length-1)); + + if (num > slides.length) + num = 1; + + if (--num < 0) + num = 0; + + return num; + } + + // accept [2] for backwards compatibility + re = /\[(\d)+\]/; + + if (anchor.match(re)) + { + var num = parseInt(anchor.substring(1, anchor.length-1)); + + if (num > slides.length) + num = 1; + + if (--num < 0) + num = 0; + + return num; + } + + // oh dear unknown anchor + return 0; + } + + // search for enclosing slide + + while (true) + { + // browser coerces html elements to uppercase! + if (target.nodeName.toLowerCase() == "div" && + hasClass(target, "slide")) + { + // found the slide element + break; + } + + // otherwise try parent element if any + + target = target.parentNode; + + if (!target) + { + return 0; // no luck! + } + }; + + for (i = 0; i < slides.length; ++i) + { + if (slides[i] == target) + return i; // success + } + + // oh dear still no luck + return 0; +} + +// find slide name from first h1 element +// default to document title + slide number +function slideName(index) +{ + var name = null; + var slide = slides[index]; + + var heading = findHeading(slide); + + if (heading) + name = extractText(heading); + + if (!name) + name = title + "(" + (index + 1) + ")"; + + name.replace(/\&/g, "&"); + name.replace(/\/g, ">"); + + return name; +} + +// find first h1 element in DOM tree +function findHeading(node) +{ + if (!node || node.nodeType != 1) + return null; + + if (node.nodeName == "H1" || node.nodeName == "h1") + return node; + + var child = node.firstChild; + + while (child) + { + node = findHeading(child); + + if (node) + return node; + + child = child.nextSibling; + } + + return null; +} + +// recursively extract text from DOM tree +function extractText(node) +{ + if (!node) + return ""; + + // text nodes + if (node.nodeType == 3) + return node.nodeValue; + + // elements + if (node.nodeType == 1) + { + node = node.firstChild; + var text = ""; + + while (node) + { + text = text + extractText(node); + node = node.nextSibling; + } + + return text; + } + + return ""; +} + + +// find copyright text from meta element +function findCopyright() +{ + var name, content; + var meta = document.getElementsByTagName("meta"); + + for (var i = 0; i < meta.length; ++i) + { + name = meta[i].getAttribute("name"); + content = meta[i].getAttribute("content"); + + if (name == "copyright") + return content; + } + + return null; +} + +function findSizeAdjust() +{ + var name, content, offset; + var meta = document.getElementsByTagName("meta"); + + for (var i = 0; i < meta.length; ++i) + { + name = meta[i].getAttribute("name"); + content = meta[i].getAttribute("content"); + + if (name == "font-size-adjustment") + return 1 * content; + } + + return 0; +} + +function addToolbar() +{ + var slideCounter, page; + + var toolbar = createElement("div"); + toolbar.setAttribute("class", "toolbar"); + + if (ns_pos) // a reasonably behaved browser + { + var right = document.createElement("div"); + right.setAttribute("style", "float: right; text-align: right"); + + slideCounter = document.createElement("div") + slideCounter.innerHTML = "slide".localize() + " n/m"; + right.appendChild(slideCounter); + toolbar.appendChild(right); + + var left = document.createElement("div"); + left.setAttribute("style", "text-align: left"); + + // global end of slide indicator + eos = document.createElement("span"); + eos.innerHTML = "* "; + left.appendChild(eos); + + var help = document.createElement("a"); + help.setAttribute("href", helpPage); + help.setAttribute("title", helpText.localize()); + help.innerHTML = "help?".localize(); + left.appendChild(help); + helpAnchor = help; // save for focus hack + + var gap1 = document.createTextNode(" "); + left.appendChild(gap1); + + var contents = document.createElement("a"); + contents.setAttribute("href", "javascript:toggleTableOfContents()"); + contents.setAttribute("title", "table of contents".localize()); + contents.innerHTML = "contents?".localize(); + left.appendChild(contents); + + var gap2 = document.createTextNode(" "); + left.appendChild(gap2); + + var i = location.href.indexOf("#"); + + // check if anchor is entire page + + if (i > 0) + page = location.href.substr(0, i); + else + page = location.href; + + var start = document.createElement("a"); + start.setAttribute("href", page); + start.setAttribute("title", "restart presentation".localize()); + start.innerHTML = "restart?".localize(); +// start.setAttribute("href", "javascript:printSlides()"); +// start.setAttribute("title", "print all slides".localize()); +// start.innerHTML = "print!".localize(); + left.appendChild(start); + + var copyright = findCopyright(); + + if (copyright) + { + var span = document.createElement("span"); + span.innerHTML = copyright; + span.style.color = "black"; + span.style.marginLeft = "4em"; + left.appendChild(span); + } + + toolbar.appendChild(left); + } + else // IE so need to work around its poor CSS support + { + toolbar.style.position = (ie7 ? "fixed" : "absolute"); + toolbar.style.zIndex = "200"; + toolbar.style.width = "99.9%"; + toolbar.style.height = "1.2em"; + toolbar.style.top = "auto"; + toolbar.style.bottom = "0"; + toolbar.style.left = "0"; + toolbar.style.right = "0"; + toolbar.style.textAlign = "left"; + toolbar.style.fontSize = "60%"; + toolbar.style.color = "red"; + toolbar.borderWidth = 0; + toolbar.style.background = "rgb(240,240,240)"; + + // would like to have help text left aligned + // and page counter right aligned, floating + // div's don't work, so instead use nested + // absolutely positioned div's. + + var sp = document.createElement("span"); + sp.innerHTML = "  * "; + toolbar.appendChild(sp); + eos = sp; // end of slide indicator + + var help = document.createElement("a"); + help.setAttribute("href", helpPage); + help.setAttribute("title", helpText.localize()); + help.innerHTML = "help?".localize(); + toolbar.appendChild(help); + helpAnchor = help; // save for focus hack + + var gap1 = document.createTextNode(" "); + toolbar.appendChild(gap1); + + var contents = document.createElement("a"); + contents.setAttribute("href", "javascript:toggleTableOfContents()"); + contents.setAttribute("title", "table of contents".localize()); + contents.innerHTML = "contents?".localize(); + toolbar.appendChild(contents); + + var gap2 = document.createTextNode(" "); + toolbar.appendChild(gap2); + + var i = location.href.indexOf("#"); + + // check if anchor is entire page + + if (i > 0) + page = location.href.substr(0, i); + else + page = location.href; + + var start = document.createElement("a"); + start.setAttribute("href", page); + start.setAttribute("title", "restart presentation".localize()); + start.innerHTML = "restart?".localize(); +// start.setAttribute("href", "javascript:printSlides()"); +// start.setAttribute("title", "print all slides".localize()); +// start.innerHTML = "print!".localize(); + toolbar.appendChild(start); + + var copyright = findCopyright(); + + if (copyright) + { + var span = document.createElement("span"); + span.innerHTML = copyright; + span.style.color = "black"; + span.style.marginLeft = "2em"; + toolbar.appendChild(span); + } + + slideCounter = document.createElement("div") + slideCounter.style.position = "absolute"; + slideCounter.style.width = "auto"; //"20%"; + slideCounter.style.height = "1.2em"; + slideCounter.style.top = "auto"; + slideCounter.style.bottom = 0; + slideCounter.style.right = "0"; + slideCounter.style.textAlign = "right"; + slideCounter.style.color = "red"; + slideCounter.style.background = "rgb(240,240,240)"; + + slideCounter.innerHTML = "slide".localize() + " n/m"; + toolbar.appendChild(slideCounter); + } + + // ensure that click isn't passed through to the page + toolbar.onclick = stopPropagation; + document.body.appendChild(toolbar); + slideNumElement = slideCounter; + setEosStatus(false); + + return toolbar; +} + +function isShownToc() +{ + if (toc && toc.style.visible == "visible") + return true; + + return false; +} + +function showTableOfContents() +{ + if (toc) + { + if (toc.style.visibility != "visible") + { + toc.style.visibility = "visible"; + toc.style.display = "block"; + toc.focus(); + + if (ie7 && slidenum == 0) + setTimeout("ieHack()", 100); + } + else + hideTableOfContents(); + } +} + +function hideTableOfContents() +{ + if (toc && toc.style.visibility != "hidden") + { + toc.style.visibility = "hidden"; + toc.style.display = "none"; + + try + { + if (!opera) + helpAnchor.focus(); + } + catch (e) + { + } + } +} + +function toggleTableOfContents() +{ + if (toc) + { + if (toc.style.visible != "visible") + showTableOfContents(); + else + hideTableOfContents(); + } +} + +// called on clicking toc entry +function gotoEntry(e) +{ + var target; + + if (!e) + var e = window.event; + + if (e.target) + target = e.target; + else if (e.srcElement) + target = e.srcElement; + + // work around Safari bug + if (target.nodeType == 3) + target = target.parentNode; + + if (target && target.nodeType == 1) + { + var uri = target.getAttribute("href"); + + if (uri) + { + //alert("going to " + uri); + var slide = slides[slidenum]; + hideSlide(slide); + slidenum = findSlideNumber(uri); + slide = slides[slidenum]; + lastShown = null; + setLocation(); + setVisibilityAllIncremental("hidden"); + setEosStatus(!nextIncrementalItem(lastShown)); + showSlide(slide); + //target.focus(); + + try + { + if (!opera) + helpAnchor.focus(); + } + catch (e) + { + } + } + } + + hideTableOfContents(e); + if (ie7) ieHack(); + stopPropagation(e); + return cancel(e); +} + +// called onkeydown for toc entry +function gotoTocEntry(event) +{ + var key; + + if (!event) + var event = window.event; + + // kludge around NS/IE differences + if (window.event) + key = window.event.keyCode; + else if (event.which) + key = event.which; + else + return true; // Yikes! unknown browser + + // ignore event if key value is zero + // as for alt on Opera and Konqueror + if (!key) + return true; + + // check for concurrent control/command/alt key + // but are these only present on mouse events? + + if (event.ctrlKey || event.altKey) + return true; + + if (key == 13) + { + var uri = this.getAttribute("href"); + + if (uri) + { + //alert("going to " + uri); + var slide = slides[slidenum]; + hideSlide(slide); + slidenum = findSlideNumber(uri); + slide = slides[slidenum]; + lastShown = null; + setLocation(); + setVisibilityAllIncremental("hidden"); + setEosStatus(!nextIncrementalItem(lastShown)); + showSlide(slide); + //target.focus(); + + try + { + if (!opera) + helpAnchor.focus(); + } + catch (e) + { + } + } + + hideTableOfContents(); + if (ie7) ieHack(); + return cancel(event); + } + + if (key == 40 && this.next) + { + this.next.focus(); + return cancel(event); + } + + if (key == 38 && this.previous) + { + this.previous.focus(); + return cancel(event); + } + + return true; +} + +function isTitleSlide(slide) +{ + return hasClass(slide, "title"); +} + +// create div element with links to each slide +function tableOfContents() +{ + var toc = document.createElement("div"); + addClass(toc, "toc"); + //toc.setAttribute("tabindex", "0"); + + var heading = document.createElement("div"); + addClass(heading, "toc-heading"); + heading.innerHTML = "Table of Contents".localize(); + + heading.style.textAlign = "center"; + heading.style.width = "100%"; + heading.style.margin = "0"; + heading.style.marginBottom = "1em"; + heading.style.borderBottomStyle = "solid"; + heading.style.borderBottomColor = "rgb(180,180,180)"; + heading.style.borderBottomWidth = "1px"; + + toc.appendChild(heading); + var previous = null; + + for (var i = 0; i < slides.length; ++i) + { + var title = hasClass(slides[i], "title"); + var num = document.createTextNode((i + 1) + ". "); + + toc.appendChild(num); + + var a = document.createElement("a"); + a.setAttribute("href", "#(" + (i+1) + ")"); + + if (title) + addClass(a, "titleslide"); + + var name = document.createTextNode(slideName(i)); + a.appendChild(name); + a.onclick = gotoEntry; + a.onkeydown = gotoTocEntry; + a.previous = previous; + + if (previous) + previous.next = a; + + toc.appendChild(a); + + if (i == 0) + toc.first = a; + + if (i < slides.length - 1) + { + var br = document.createElement("br"); + toc.appendChild(br); + } + + previous = a; + } + + toc.focus = function () { + if (this.first) + this.first.focus(); + } + + toc.onclick = function (e) { + e||(e=window.event); + hideTableOfContents(); + stopPropagation(e); + + if (e.cancel != undefined) + e.cancel = true; + + if (e.returnValue != undefined) + e.returnValue = false; + + return false; + }; + + toc.style.position = "absolute"; + toc.style.zIndex = "300"; + toc.style.width = "60%"; + toc.style.maxWidth = "30em"; + toc.style.height = "30em"; + toc.style.overflow = "auto"; + toc.style.top = "auto"; + toc.style.right = "auto"; + toc.style.left = "4em"; + toc.style.bottom = "4em"; + toc.style.padding = "1em"; + toc.style.background = "rgb(240,240,240)"; + toc.style.borderStyle = "solid"; + toc.style.borderWidth = "2px"; + toc.style.fontSize = "60%"; + + document.body.insertBefore(toc, document.body.firstChild); + return toc; +} + +function replaceByNonBreakingSpace(str) +{ + for (var i = 0; i < str.length; ++i) + str[i] = 160; +} + + +function initOutliner() +{ + var items = document.getElementsByTagName("LI"); + + for (var i = 0; i < items.length; ++i) + { + var target = items[i]; + + if (!hasClass(target.parentNode, "outline")) + continue; + + target.onclick = outlineClick; + + if (!ns_pos) + { + target.onmouseover = hoverOutline; + target.onmouseout = unhoverOutline; + } + + if (foldable(target)) + { + target.foldable = true; + target.onfocus = function () {outline = this;}; + target.onblur = function () {outline = null;}; + + if (!target.getAttribute("tabindex")) + target.setAttribute("tabindex", "0"); + + if (hasClass(target, "expand")) + unfold(target); + else + fold(target); + } + else + { + addClass(target, "nofold"); + target.visible = true; + target.foldable = false; + } + } +} + +function foldable(item) +{ + if (!item || item.nodeType != 1) + return false; + + var node = item.firstChild; + + while (node) + { + if (node.nodeType == 1 && isBlock(node)) + return true; + + node = node.nextSibling; + } + + return false; +} + +function fold(item) +{ + if (item) + { + removeClass(item, "unfolded"); + addClass(item, "folded"); + } + + var node = item ? item.firstChild : null; + + while (node) + { + if (node.nodeType == 1 && isBlock(node)) // element + { + // note that getElementStyle won't work for Safari 1.3 + node.display = getElementStyle(node, "display", "display"); + node.style.display = "none"; + node.style.visibility = "hidden"; + } + + node = node.nextSibling; + } + + item.visible = false; +} + +function unfold(item) +{ + if (item) + { + addClass(item, "unfolded"); + removeClass(item, "folded"); + } + + var node = item ? item.firstChild : null; + + while (node) + { + if (node.nodeType == 1 && isBlock(node)) // element + { + // with fallback for Safari, see above + node.style.display = (node.display ? node.display : "block"); + node.style.visibility = "visible"; + } + + node = node.nextSibling; + } + + item.visible = true; +} + +function outlineClick(e) +{ + var rightclick = false; + var target; + + if (!e) + var e = window.event; + + if (e.target) + target = e.target; + else if (e.srcElement) + target = e.srcElement; + + // work around Safari bug + if (target.nodeType == 3) + target = target.parentNode; + + while (target && target.visible == undefined) + target = target.parentNode; + + if (!target) + return true; + + if (e.which) + rightclick = (e.which == 3); + else if (e.button) + rightclick = (e.button == 2); + + if (!rightclick && target.visible != undefined) + { + if (target.foldable) + { + if (target.visible) + fold(target); + else + unfold(target); + } + + stopPropagation(e); + e.cancel = true; + e.returnValue = false; + } + + return false; +} + +function hoverOutline(e) +{ + var target; + + if (!e) + var e = window.event; + + if (e.target) + target = e.target; + else if (e.srcElement) + target = e.srcElement; + + // work around Safari bug + if (target.nodeType == 3) + target = target.parentNode; + + while (target && target.visible == undefined) + target = target.parentNode; + + if (target && target.foldable) + target.style.cursor = "pointer"; + + return true; +} + +function unhoverOutline(e) +{ + var target; + + if (!e) + var e = window.event; + + if (e.target) + target = e.target; + else if (e.srcElement) + target = e.srcElement; + + // work around Safari bug + if (target.nodeType == 3) + target = target.parentNode; + + while (target && target.visible == undefined) + target = target.parentNode; + + if (target) + target.style.cursor = "default"; + + return true; +} + + +function stopPropagation(e) +{ + if (window.event) + { + window.event.cancelBubble = true; + //window.event.returnValue = false; + } + else if (e) + { + e.cancelBubble = true; + e.stopPropagation(); + //e.preventDefault(); + } +} + +/* can't rely on display since we set that to none to hide things */ +function isBlock(elem) +{ + var tag = elem.nodeName; + + return tag == "OL" || tag == "UL" || tag == "P" || + tag == "LI" || tag == "TABLE" || tag == "PRE" || + tag == "H1" || tag == "H2" || tag == "H3" || + tag == "H4" || tag == "H5" || tag == "H6" || + tag == "BLOCKQUOTE" || tag == "ADDRESS"; +} + +function getElementStyle(elem, IEStyleProp, CSSStyleProp) +{ + if (elem.currentStyle) + { + return elem.currentStyle[IEStyleProp]; + } + else if (window.getComputedStyle) + { + var compStyle = window.getComputedStyle(elem, ""); + return compStyle.getPropertyValue(CSSStyleProp); + } + return ""; +} + +// works with text/html and text/xhtml+xml with thanks to Simon Willison +function createElement(element) +{ + if (typeof document.createElementNS != 'undefined') + { + return document.createElementNS('http://www.w3.org/1999/xhtml', element); + } + + if (typeof document.createElement != 'undefined') + { + return document.createElement(element); + } + + return false; +} + +// designed to work with both text/html and text/xhtml+xml +function getElementsByTagName(name) +{ + if (typeof document.getElementsByTagNameNS != 'undefined') + { + return document.getElementsByTagNameNS('http://www.w3.org/1999/xhtml', name); + } + + if (typeof document.getElementsByTagName != 'undefined') + { + return document.getElementsByTagName(name); + } + + return null; +} + +/* +// clean alternative to innerHTML method, but on IE6 +// it doesn't work with named entities like   +// which need to be replaced by numeric entities +function insertText(element, text) +{ + try + { + element.textContent = text; // DOM3 only + } + catch (e) + { + if (element.firstChild) + { + // remove current children + while (element.firstChild) + element.removeChild(element.firstChild); + } + + element.appendChild(document.createTextNode(text)); + } +} + +// as above, but as method of all element nodes +// doesn't work in IE6 which doesn't allow you to +// add methods to the HTMLElement prototype +if (HTMLElement != undefined) +{ + HTMLElement.prototype.insertText = function(text) { + var element = this; + + try + { + element.textContent = text; // DOM3 only + } + catch (e) + { + if (element.firstChild) + { + // remove current children + while (element.firstChild) + element.removeChild(element.firstChild); + } + + element.appendChild(document.createTextNode(text)); + } + }; +} +*/ + +function getSelectedText() +{ + try + { + if (window.getSelection) + return window.getSelection().toString(); + + if (document.getSelection) + return document.getSelection().toString(); + + if (document.selection) + return document.selection.createRange().text; + } + catch (e) + { + return ""; + } + return ""; +} diff --git a/doc/tmpltohtml.go b/doc/tmpltohtml.go new file mode 100644 index 000000000..f4d2e2c2c --- /dev/null +++ b/doc/tmpltohtml.go @@ -0,0 +1,176 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + + +// The template uses the function "code" to inject program +// source into the output by extracting code from files and +// injecting them as HTML-escaped
 blocks.
+//
+// The syntax is simple: 1, 2, or 3 space-separated arguments:
+//
+// Whole file:
+//	{{code "foo.go"}}
+// One line (here the signature of main):
+//	{{code "foo.go" `/^func.main/`}}
+// Block of text, determined by start and end (here the body of main):
+//	{{code "foo.go" `/^func.main/` `/^}/`
+//
+// Patterns can be `/regular expression/`, a decimal number, or "$"
+// to signify the end of the file.
+package main
+
+import (
+	"exp/template"
+	"flag"
+	"fmt"
+	"io/ioutil"
+	"log"
+	"os"
+	"regexp"
+	"strings"
+)
+
+func Usage() {
+	fmt.Fprintf(os.Stderr, "usage: tmpltohtml file\n")
+	os.Exit(2)
+}
+
+func main() {
+	flag.Usage = Usage
+	flag.Parse()
+	if len(flag.Args()) != 1 {
+		Usage()
+	}
+
+	// Read and parse the input.
+	name := flag.Args()[0]
+	tmpl := template.New(name).Funcs(template.FuncMap{"code": code})
+	if _, err := tmpl.ParseFile(name); err != nil {
+		log.Fatal(err)
+	}
+
+	// Execute the template.
+	if err := tmpl.Execute(os.Stdout, 0); err != nil {
+		log.Fatal(err)
+	}
+}
+
+// contents reads a file by name and returns its contents as a string.
+func contents(name string) string {
+	file, err := ioutil.ReadFile(name)
+	if err != nil {
+		log.Fatal(err)
+	}
+	return string(file)
+}
+
+// format returns a textual representation of the arg, formatted according to its nature.
+func format(arg interface{}) string {
+	switch arg := arg.(type) {
+	case int:
+		return fmt.Sprintf("%d", arg)
+	case string:
+		if len(arg) > 2 && arg[0] == '/' && arg[len(arg)-1] == '/' {
+			return fmt.Sprintf("%#q", arg)
+		}
+		return fmt.Sprintf("%q", arg)
+	default:
+		log.Fatalf("unrecognized argument: %v type %T", arg, arg)
+	}
+	return ""
+}
+
+func code(file string, arg ...interface{}) (string, os.Error) {
+	text := contents(file)
+	var command string
+	switch len(arg) {
+	case 0:
+		// text is already whole file.
+		command = fmt.Sprintf("code %q", file)
+	case 1:
+		command = fmt.Sprintf("code %q %s", file, format(arg[0]))
+		text = oneLine(file, text, arg[0])
+	case 2:
+		command = fmt.Sprintf("code %q %s %s", file, format(arg[0]), format(arg[1]))
+		text = multipleLines(file, text, arg[0], arg[1])
+	default:
+		return "", fmt.Errorf("incorrect code invocation: code %q %q", file, arg)
+	}
+	// Replace tabs by spaces, which work better in HTML.
+	text = strings.Replace(text, "\t", "    ", -1)
+	// Escape the program text for HTML.
+	text = template.HTMLEscapeString(text)
+	// Include the command as a comment.
+	text = fmt.Sprintf("
%s
", command, text) + return text, nil +} + +// parseArg returns the integer or string value of the argument and tells which it is. +func parseArg(arg interface{}, file string, max int) (ival int, sval string, isInt bool) { + switch n := arg.(type) { + case int: + if n <= 0 || n > max { + log.Fatalf("%q:%d is out of range", file, n) + } + return n, "", true + case string: + return 0, n, false + } + log.Fatalf("unrecognized argument %v type %T", arg, arg) + return +} + +// oneLine returns the single line generated by a two-argument code invocation. +func oneLine(file, text string, arg interface{}) string { + lines := strings.SplitAfter(contents(file), "\n") + line, pattern, isInt := parseArg(arg, file, len(lines)) + if isInt { + return lines[line-1] + } + return lines[match(file, 0, lines, pattern)-1] +} + +// multipleLines returns the text generated by a three-argument code invocation. +func multipleLines(file, text string, arg1, arg2 interface{}) string { + lines := strings.SplitAfter(contents(file), "\n") + line1, pattern1, isInt1 := parseArg(arg1, file, len(lines)) + line2, pattern2, isInt2 := parseArg(arg2, file, len(lines)) + if !isInt1 { + line1 = match(file, 0, lines, pattern1) + } + if !isInt2 { + line2 = match(file, line1, lines, pattern2) + } else if line2 < line1 { + log.Fatal("lines out of order for %q: %d %d", line1, line2) + } + return strings.Join(lines[line1-1:line2], "") +} + +// match identifies the input line that matches the pattern in a code invocation. +// If start>0, match lines starting there rather than at the beginning. +// The return value is 1-indexed. +func match(file string, start int, lines []string, pattern string) int { + // $ matches the end of the file. + if pattern == "$" { + if len(lines) == 0 { + log.Fatal("%q: empty file", file) + } + return len(lines) + } + // /regexp/ matches the line that matches the regexp. + if len(pattern) > 2 && pattern[0] == '/' && pattern[len(pattern)-1] == '/' { + re, err := regexp.Compile(pattern[1 : len(pattern)-1]) + if err != nil { + log.Fatal(err) + } + for i := start; i < len(lines); i++ { + if re.MatchString(lines[i]) { + return i + 1 + } + } + log.Fatalf("%s: no match for %#q", file, pattern) + } + log.Fatalf("unrecognized pattern: %q", pattern) + return 0 +} diff --git a/doc/video-001.png b/doc/video-001.png new file mode 100644 index 000000000..d3468bbe8 Binary files /dev/null and b/doc/video-001.png differ diff --git a/doc/video-002.png b/doc/video-002.png new file mode 100644 index 000000000..4f7c5d184 Binary files /dev/null and b/doc/video-002.png differ diff --git a/doc/video-003.png b/doc/video-003.png new file mode 100644 index 000000000..3dff68602 Binary files /dev/null and b/doc/video-003.png differ diff --git a/doc/video-004.png b/doc/video-004.png new file mode 100644 index 000000000..92144549a Binary files /dev/null and b/doc/video-004.png differ diff --git a/doc/video-005.jpg b/doc/video-005.jpg new file mode 100644 index 000000000..32371581f Binary files /dev/null and b/doc/video-005.jpg differ diff --git a/favicon.ico b/favicon.ico new file mode 100644 index 000000000..48854ff3b Binary files /dev/null and b/favicon.ico differ diff --git a/include/ar.h b/include/ar.h new file mode 100644 index 000000000..b565ac90b --- /dev/null +++ b/include/ar.h @@ -0,0 +1,47 @@ +// Inferno utils/include/ar.h +// http://code.google.com/p/inferno-os/source/browse/utils/include/ar.h +// +// Copyright © 1994-1999 Lucent Technologies Inc. All rights reserved. +// Portions Copyright © 1995-1997 C H Forsyth (forsyth@terzarima.net) +// Portions Copyright © 1997-1999 Vita Nuova Limited +// Portions Copyright © 2000-2007 Vita Nuova Holdings Limited (www.vitanuova.com) +// Portions Copyright © 2004,2006 Bruce Ellis +// Portions Copyright © 2005-2007 C H Forsyth (forsyth@terzarima.net) +// Revisions Copyright © 2000-2007 Lucent Technologies Inc. and others +// Portions Copyright © 2009 The Go Authors. All rights reserved. +// +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to deal +// in the Software without restriction, including without limitation the rights +// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +// copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +// THE SOFTWARE. + +#define ARMAG "!\n" +#define SARMAG 8 + +#define ARFMAG "`\n" +#define SARNAME 64 + +struct ar_hdr +{ + char name[SARNAME]; + char date[12]; + char uid[6]; + char gid[6]; + char mode[8]; + char size[10]; + char fmag[2]; +}; +#define SAR_HDR (SARNAME+44) diff --git a/include/bio.h b/include/bio.h new file mode 100644 index 000000000..c754d7a57 --- /dev/null +++ b/include/bio.h @@ -0,0 +1,116 @@ +/* +http://code.google.com/p/inferno-os/source/browse/include/bio.h + + Copyright © 1994-1999 Lucent Technologies Inc. All rights reserved. + Revisions Copyright © 2000-2007 Vita Nuova Holdings Limited (www.vitanuova.com). All rights reserved. + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. +*/ + +#ifndef _BIO_H_ +#define _BIO_H_ 1 +#if defined(__cplusplus) +extern "C" { +#endif + +#ifdef AUTOLIB +AUTOLIB(bio) +#endif + +#include /* for O_RDONLY, O_WRONLY */ + +typedef struct Biobuf Biobuf; + +enum +{ + Bsize = 8*1024, + Bungetsize = 4, /* space for ungetc */ + Bmagic = 0x314159, + Beof = -1, + Bbad = -2, + + Binactive = 0, /* states */ + Bractive, + Bwactive, + Bracteof, + + Bend +}; + +struct Biobuf +{ + int icount; /* neg num of bytes at eob */ + int ocount; /* num of bytes at bob */ + int rdline; /* num of bytes after rdline */ + int runesize; /* num of bytes of last getrune */ + int state; /* r/w/inactive */ + int fid; /* open file */ + int flag; /* magic if malloc'ed */ + vlong offset; /* offset of buffer in file */ + int bsize; /* size of buffer */ + unsigned char* bbuf; /* pointer to beginning of buffer */ + unsigned char* ebuf; /* pointer to end of buffer */ + unsigned char* gbuf; /* pointer to good data in buf */ + unsigned char b[Bungetsize+Bsize]; +}; + +#define BGETC(bp)\ + ((bp)->icount?(bp)->bbuf[(bp)->bsize+(bp)->icount++]:Bgetc((bp))) +#define BPUTC(bp,c)\ + ((bp)->ocount?(bp)->bbuf[(bp)->bsize+(bp)->ocount++]=(c),0:Bputc((bp),(c))) +#define BOFFSET(bp)\ + (((bp)->state==Bractive)?\ + (bp)->offset + (bp)->icount:\ + (((bp)->state==Bwactive)?\ + (bp)->offset + ((bp)->bsize + (bp)->ocount):\ + -1)) +#define BLINELEN(bp)\ + (bp)->rdline +#define BFILDES(bp)\ + (bp)->fid + +int Bbuffered(Biobuf*); +Biobuf* Bfdopen(int, int); +int Bfildes(Biobuf*); +int Bflush(Biobuf*); +int Bgetc(Biobuf*); +int Bgetd(Biobuf*, double*); +long Bgetrune(Biobuf*); +int Binit(Biobuf*, int, int); +int Binits(Biobuf*, int, int, unsigned char*, int); +int Blinelen(Biobuf*); +vlong Boffset(Biobuf*); +Biobuf* Bopen(char*, int); +int Bprint(Biobuf*, char*, ...); +int Bputc(Biobuf*, int); +int Bputrune(Biobuf*, long); +void* Brdline(Biobuf*, int); +char* Brdstr(Biobuf*, int, int); +long Bread(Biobuf*, void*, long); +vlong Bseek(Biobuf*, vlong, int); +int Bterm(Biobuf*); +int Bungetc(Biobuf*); +int Bungetrune(Biobuf*); +long Bwrite(Biobuf*, void*, long); +int Bvprint(Biobuf*, char*, va_list); + +#if defined(__cplusplus) +} +#endif +#endif diff --git a/include/bootexec.h b/include/bootexec.h new file mode 100644 index 000000000..49721ea33 --- /dev/null +++ b/include/bootexec.h @@ -0,0 +1,169 @@ +// Inferno libmach/bootexec.h +// http://code.google.com/p/inferno-os/source/browse/utils/libmach/bootexec.h +// +// Copyright © 1994-1999 Lucent Technologies Inc. +// Power PC support Copyright © 1995-2004 C H Forsyth (forsyth@terzarima.net). +// Portions Copyright © 1997-1999 Vita Nuova Limited. +// Portions Copyright © 2000-2007 Vita Nuova Holdings Limited (www.vitanuova.com). +// Revisions Copyright © 2000-2004 Lucent Technologies Inc. and others. +// Portions Copyright © 2009 The Go Authors. All rights reserved. +// +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to deal +// in the Software without restriction, including without limitation the rights +// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +// copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +// THE SOFTWARE. + +struct coffsect +{ + char name[8]; + uint32 phys; + uint32 virt; + uint32 size; + uint32 fptr; + uint32 fptrreloc; + uint32 fptrlineno; + uint32 nrelocnlineno; + uint32 flags; +}; + +/* + * proprietary exec headers, needed to bootstrap various machines + */ +struct mipsexec +{ + short mmagic; /* (0x160) mips magic number */ + short nscns; /* (unused) number of sections */ + int32 timdat; /* (unused) time & date stamp */ + int32 symptr; /* offset to symbol table */ + int32 nsyms; /* size of symbol table */ + short opthdr; /* (0x38) sizeof(optional hdr) */ + short pcszs; /* flags */ + short amagic; /* see above */ + short vstamp; /* version stamp */ + int32 tsize; /* text size in bytes */ + int32 dsize; /* initialized data */ + int32 bsize; /* uninitialized data */ + int32 mentry; /* entry pt. */ + int32 text_start; /* base of text used for this file */ + int32 data_start; /* base of data used for this file */ + int32 bss_start; /* base of bss used for this file */ + int32 gprmask; /* general purpose register mask */ +union{ + int32 cprmask[4]; /* co-processor register masks */ + int32 pcsize; +}; + int32 gp_value; /* the gp value used for this object */ +}; + +struct mips4kexec +{ + struct mipsexec h; + struct coffsect itexts; + struct coffsect idatas; + struct coffsect ibsss; +}; + +struct sparcexec +{ + short sjunk; /* dynamic bit and version number */ + short smagic; /* 0407 */ + uint32 stext; + uint32 sdata; + uint32 sbss; + uint32 ssyms; + uint32 sentry; + uint32 strsize; + uint32 sdrsize; +}; + +struct nextexec +{ +/* UNUSED + struct nexthdr{ + uint32 nmagic; + uint32 ncputype; + uint32 ncpusubtype; + uint32 nfiletype; + uint32 ncmds; + uint32 nsizeofcmds; + uint32 nflags; + }; + + struct nextcmd{ + uint32 cmd; + uint32 cmdsize; + uchar segname[16]; + uint32 vmaddr; + uint32 vmsize; + uint32 fileoff; + uint32 filesize; + uint32 maxprot; + uint32 initprot; + uint32 nsects; + uint32 flags; + }textc; + struct nextsect{ + char sectname[16]; + char segname[16]; + uint32 addr; + uint32 size; + uint32 offset; + uint32 align; + uint32 reloff; + uint32 nreloc; + uint32 flags; + uint32 reserved1; + uint32 reserved2; + }texts; + struct nextcmd datac; + struct nextsect datas; + struct nextsect bsss; + struct nextsym{ + uint32 cmd; + uint32 cmdsize; + uint32 symoff; + uint32 nsyms; + uint32 spoff; + uint32 pcoff; + }symc; +*/ +}; + +struct i386exec +{ +/* UNUSED + struct i386coff{ + uint32 isectmagic; + uint32 itime; + uint32 isyms; + uint32 insyms; + uint32 iflags; + }; + struct i386hdr{ + uint32 imagic; + uint32 itextsize; + uint32 idatasize; + uint32 ibsssize; + uint32 ientry; + uint32 itextstart; + uint32 idatastart; + }; + struct coffsect itexts; + struct coffsect idatas; + struct coffsect ibsss; + struct coffsect icomments; +*/ +}; diff --git a/include/fmt.h b/include/fmt.h new file mode 100644 index 000000000..2280f2525 --- /dev/null +++ b/include/fmt.h @@ -0,0 +1,116 @@ +#ifndef _FMT_H_ +#define _FMT_H_ 1 +#if defined(__cplusplus) +extern "C" { +#endif +/* + * The authors of this software are Rob Pike and Ken Thompson. + * Copyright (c) 2002 by Lucent Technologies. + * Permission to use, copy, modify, and distribute this software for any + * purpose without fee is hereby granted, provided that this entire notice + * is included in all copies of any software which is or includes a copy + * or modification of this software and in all copies of the supporting + * documentation for such software. + * THIS SOFTWARE IS BEING PROVIDED "AS IS", WITHOUT ANY EXPRESS OR IMPLIED + * WARRANTY. IN PARTICULAR, NEITHER THE AUTHORS NOR LUCENT TECHNOLOGIES MAKE ANY + * REPRESENTATION OR WARRANTY OF ANY KIND CONCERNING THE MERCHANTABILITY + * OF THIS SOFTWARE OR ITS FITNESS FOR ANY PARTICULAR PURPOSE. + */ + +#include +#include + +typedef struct Fmt Fmt; +struct Fmt{ + unsigned char runes; /* output buffer is runes or chars? */ + void *start; /* of buffer */ + void *to; /* current place in the buffer */ + void *stop; /* end of the buffer; overwritten if flush fails */ + int (*flush)(Fmt *); /* called when to == stop */ + void *farg; /* to make flush a closure */ + int nfmt; /* num chars formatted so far */ + va_list args; /* args passed to dofmt */ + Rune r; /* % format Rune */ + int width; + int prec; + unsigned long flags; + char *decimal; /* decimal point; cannot be "" */ + + /* For %'d */ + char *thousands; /* separator for thousands */ + + /* + * Each char is an integer indicating #digits before next separator. Values: + * \xFF: no more grouping (or \x7F; defined to be CHAR_MAX in POSIX) + * \x00: repeat previous indefinitely + * \x**: count that many + */ + char *grouping; /* descriptor of separator placement */ +}; + +enum{ + FmtWidth = 1, + FmtLeft = FmtWidth << 1, + FmtPrec = FmtLeft << 1, + FmtSharp = FmtPrec << 1, + FmtSpace = FmtSharp << 1, + FmtSign = FmtSpace << 1, + FmtApost = FmtSign << 1, + FmtZero = FmtApost << 1, + FmtUnsigned = FmtZero << 1, + FmtShort = FmtUnsigned << 1, + FmtLong = FmtShort << 1, + FmtVLong = FmtLong << 1, + FmtComma = FmtVLong << 1, + FmtByte = FmtComma << 1, + FmtLDouble = FmtByte << 1, + + FmtFlag = FmtLDouble << 1 +}; + +extern int (*fmtdoquote)(int); + +/* Edit .+1,/^$/ | cfn $PLAN9/src/lib9/fmt/?*.c | grep -v static |grep -v __ */ +int dofmt(Fmt *f, char *fmt); +int dorfmt(Fmt *f, const Rune *fmt); +double fmtcharstod(int(*f)(void*), void *vp); +int fmtfdflush(Fmt *f); +int fmtfdinit(Fmt *f, int fd, char *buf, int size); +int fmtinstall(int c, int (*f)(Fmt*)); +int fmtnullinit(Fmt*); +void fmtlocaleinit(Fmt*, char*, char*, char*); +int fmtprint(Fmt *f, char *fmt, ...); +int fmtrune(Fmt *f, int r); +int fmtrunestrcpy(Fmt *f, Rune *s); +int fmtstrcpy(Fmt *f, char *s); +char* fmtstrflush(Fmt *f); +int fmtstrinit(Fmt *f); +double fmtstrtod(const char *as, char **aas); +int fmtvprint(Fmt *f, char *fmt, va_list args); +int fprint(int fd, char *fmt, ...); +int print(char *fmt, ...); +void quotefmtinstall(void); +int quoterunestrfmt(Fmt *f); +int quotestrfmt(Fmt *f); +Rune* runefmtstrflush(Fmt *f); +int runefmtstrinit(Fmt *f); +Rune* runeseprint(Rune *buf, Rune *e, char *fmt, ...); +Rune* runesmprint(char *fmt, ...); +int runesnprint(Rune *buf, int len, char *fmt, ...); +int runesprint(Rune *buf, char *fmt, ...); +Rune* runevseprint(Rune *buf, Rune *e, char *fmt, va_list args); +Rune* runevsmprint(char *fmt, va_list args); +int runevsnprint(Rune *buf, int len, char *fmt, va_list args); +char* seprint(char *buf, char *e, char *fmt, ...); +char* smprint(char *fmt, ...); +int snprint(char *buf, int len, char *fmt, ...); +int sprint(char *buf, char *fmt, ...); +int vfprint(int fd, char *fmt, va_list args); +char* vseprint(char *buf, char *e, char *fmt, va_list args); +char* vsmprint(char *fmt, va_list args); +int vsnprint(char *buf, int len, char *fmt, va_list args); + +#if defined(__cplusplus) +} +#endif +#endif diff --git a/include/libc.h b/include/libc.h new file mode 100644 index 000000000..03e247ff6 --- /dev/null +++ b/include/libc.h @@ -0,0 +1,386 @@ +/* +Derived from Inferno include/kern.h and +Plan 9 from User Space include/libc.h + +http://code.google.com/p/inferno-os/source/browse/include/kern.h +http://code.swtch.com/plan9port/src/tip/include/libc.h + + Copyright © 1994-1999 Lucent Technologies Inc. All rights reserved. + Revisions Copyright © 2000-2007 Vita Nuova Holdings Limited (www.vitanuova.com). All rights reserved. + Portions Copyright © 2001-2007 Russ Cox. All rights reserved. + Portions Copyright © 2009 The Go Authors. All rights reserved. + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. +*/ + +/* + * Lib9 is miscellany from the Plan 9 C library that doesn't + * fit into libutf or into libfmt, but is still missing from traditional + * Unix C libraries. + */ +#ifndef _LIBC_H_ +#define _LIBC_H_ 1 +#if defined(__cplusplus) +extern "C" { +#endif + +#include +#include + +/* + * Begin trimmed down usual libc.h + */ + +#ifndef nil +#define nil ((void*)0) +#endif +#define nelem(x) (sizeof(x)/sizeof((x)[0])) + +#ifndef offsetof +#define offsetof(s, m) (ulong)(&(((s*)0)->m)) +#endif + +extern char* strecpy(char*, char*, char*); +extern int tokenize(char*, char**, int); + +extern double p9cputime(void); +#ifndef NOPLAN9DEFINES +#define cputime p9cputime +#endif +/* + * one-of-a-kind + */ +enum +{ + PNPROC = 1, + PNGROUP = 2 +}; +int isInf(double, int); + +extern int p9atoi(char*); +extern long p9atol(char*); +extern vlong p9atoll(char*); +extern double fmtcharstod(int(*)(void*), void*); +extern char* cleanname(char*); +extern int exitcode(char*); +extern void exits(char*); +extern double frexp(double, int*); +extern char* p9getenv(char*); +extern int p9putenv(char*, char*); +extern int getfields(char*, char**, int, int, char*); +extern int gettokens(char *, char **, int, char *); +extern char* getuser(void); +extern char* p9getwd(char*, int); +extern void p9longjmp(p9jmp_buf, int); +extern char* mktemp(char*); +extern int opentemp(char*); +extern void p9notejmp(void*, p9jmp_buf, int); +extern void perror(const char*); +extern int postnote(int, int, char *); +extern double p9pow10(int); +extern char* searchpath(char*); +#define p9setjmp(b) sigsetjmp((void*)(b), 1) + +extern void sysfatal(char*, ...); + +#ifndef NOPLAN9DEFINES +#define atoi p9atoi +#define atol p9atol +#define atoll p9atoll +#define getenv p9getenv +#define getwd p9getwd +#define longjmp p9longjmp +#undef setjmp +#define setjmp p9setjmp +#define putenv p9putenv +#define notejmp p9notejmp +#define jmp_buf p9jmp_buf +#define pow10 p9pow10 +#define strtod fmtstrtod +#define charstod fmtcharstod +#endif + +/* + * system calls + * + */ +#define STATMAX 65535U /* max length of machine-independent stat structure */ +#define DIRMAX (sizeof(Dir)+STATMAX) /* max length of Dir structure */ +#define ERRMAX 128 /* max length of error string */ + +#define MORDER 0x0003 /* mask for bits defining order of mounting */ +#define MREPL 0x0000 /* mount replaces object */ +#define MBEFORE 0x0001 /* mount goes before others in union directory */ +#define MAFTER 0x0002 /* mount goes after others in union directory */ +#define MCREATE 0x0004 /* permit creation in mounted directory */ +#define MCACHE 0x0010 /* cache some data */ +#define MMASK 0x0017 /* all bits on */ + +#define OREAD 0 /* open for read */ +#define OWRITE 1 /* write */ +#define ORDWR 2 /* read and write */ +#define OEXEC 3 /* execute, == read but check execute permission */ +#define OTRUNC 16 /* or'ed in (except for exec), truncate file first */ +#define ORCLOSE 64 /* or'ed in, remove on close */ +#define ODIRECT 128 /* or'ed in, direct access */ +#define OEXCL 0x1000 /* or'ed in, exclusive use (create only) */ +#define OAPPEND 0x4000 /* or'ed in, append only */ + +#define AEXIST 0 /* accessible: exists */ +#define AEXEC 1 /* execute access */ +#define AWRITE 2 /* write access */ +#define AREAD 4 /* read access */ + +/* Segattch */ +#define SG_RONLY 0040 /* read only */ +#define SG_CEXEC 0100 /* detach on exec */ + +#define NCONT 0 /* continue after note */ +#define NDFLT 1 /* terminate after note */ +#define NSAVE 2 /* clear note but hold state */ +#define NRSTR 3 /* restore saved state */ + +/* bits in Qid.type */ +#define QTDIR 0x80 /* type bit for directories */ +#define QTAPPEND 0x40 /* type bit for append only files */ +#define QTEXCL 0x20 /* type bit for exclusive use files */ +#define QTMOUNT 0x10 /* type bit for mounted channel */ +#define QTAUTH 0x08 /* type bit for authentication file */ +#define QTTMP 0x04 /* type bit for non-backed-up file */ +#define QTSYMLINK 0x02 /* type bit for symbolic link */ +#define QTFILE 0x00 /* type bits for plain file */ + +/* bits in Dir.mode */ +#define DMDIR 0x80000000 /* mode bit for directories */ +#define DMAPPEND 0x40000000 /* mode bit for append only files */ +#define DMEXCL 0x20000000 /* mode bit for exclusive use files */ +#define DMMOUNT 0x10000000 /* mode bit for mounted channel */ +#define DMAUTH 0x08000000 /* mode bit for authentication file */ +#define DMTMP 0x04000000 /* mode bit for non-backed-up file */ +#define DMSYMLINK 0x02000000 /* mode bit for symbolic link (Unix, 9P2000.u) */ +#define DMDEVICE 0x00800000 /* mode bit for device file (Unix, 9P2000.u) */ +#define DMNAMEDPIPE 0x00200000 /* mode bit for named pipe (Unix, 9P2000.u) */ +#define DMSOCKET 0x00100000 /* mode bit for socket (Unix, 9P2000.u) */ +#define DMSETUID 0x00080000 /* mode bit for setuid (Unix, 9P2000.u) */ +#define DMSETGID 0x00040000 /* mode bit for setgid (Unix, 9P2000.u) */ + +#define DMREAD 0x4 /* mode bit for read permission */ +#define DMWRITE 0x2 /* mode bit for write permission */ +#define DMEXEC 0x1 /* mode bit for execute permission */ + +#ifdef RFMEM /* FreeBSD, OpenBSD */ +#undef RFFDG +#undef RFNOTEG +#undef RFPROC +#undef RFMEM +#undef RFNOWAIT +#undef RFCFDG +#undef RFNAMEG +#undef RFENVG +#undef RFCENVG +#undef RFCFDG +#undef RFCNAMEG +#endif + +enum +{ + RFNAMEG = (1<<0), + RFENVG = (1<<1), + RFFDG = (1<<2), + RFNOTEG = (1<<3), + RFPROC = (1<<4), + RFMEM = (1<<5), + RFNOWAIT = (1<<6), + RFCNAMEG = (1<<10), + RFCENVG = (1<<11), + RFCFDG = (1<<12) +/* RFREND = (1<<13), */ +/* RFNOMNT = (1<<14) */ +}; + +typedef +struct Qid +{ + uvlong path; + ulong vers; + uchar type; +} Qid; + +typedef +struct Dir { + /* system-modified data */ + ushort type; /* server type */ + uint dev; /* server subtype */ + /* file data */ + Qid qid; /* unique id from server */ + ulong mode; /* permissions */ + ulong atime; /* last read time */ + ulong mtime; /* last write time */ + vlong length; /* file length */ + char *name; /* last element of path */ + char *uid; /* owner name */ + char *gid; /* group name */ + char *muid; /* last modifier name */ + + /* 9P2000.u extensions */ + uint uidnum; /* numeric uid */ + uint gidnum; /* numeric gid */ + uint muidnum; /* numeric muid */ + char *ext; /* extended info */ +} Dir; + +typedef +struct Waitmsg +{ + int pid; /* of loved one */ + ulong time[3]; /* of loved one & descendants */ + char *msg; +} Waitmsg; + +extern void _exits(char*); + +extern void abort(void); +extern long p9alarm(ulong); +extern int await(char*, int); +extern int awaitfor(int, char*, int); +extern int awaitnohang(char*, int); +extern int p9chdir(char*); +extern int close(int); +extern int p9create(char*, int, ulong); +extern int p9dup(int, int); +extern int errstr(char*, uint); +extern int p9exec(char*, char*[]); +extern int p9execl(char*, ...); +extern int p9rfork(int); +extern int noted(int); +extern int notify(void(*)(void*, char*)); +extern int noteenable(char*); +extern int notedisable(char*); +extern int notifyon(char*); +extern int notifyoff(char*); +extern int p9open(char*, int); +extern int fd2path(int, char*, int); +extern long readn(int, void*, long); +extern int remove(const char*); +extern vlong p9seek(int, vlong, int); +extern int p9sleep(long); +extern Waitmsg* p9wait(void); +extern Waitmsg* p9waitfor(int); +extern Waitmsg* waitnohang(void); +extern int p9waitpid(void); +extern ulong rendezvous(ulong, ulong); + +extern char* getgoos(void); +extern char* getgoarch(void); +extern char* getgoroot(void); +extern char* getgoversion(void); + +#ifdef _WIN32 + +#ifndef _WIN64 +struct timespec { + int tv_sec; + long tv_nsec; +}; +#define execv(prog, argv) execv(prog, (const char* const*)(argv)) +#define execvp(prog, argv) execvp(prog, (const char**)(argv)) +#endif + +extern int nanosleep(const struct timespec *rqtp, struct timespec *rmtp); +extern int fork(void); +extern int pread(int fd, void *buf, int n, int off); +extern int pwrite(int fd, void *buf, int n, int off); +#define lseek(fd, n, base) _lseeki64(fd, n, base) +#define mkdir(path, perm) mkdir(path) +#define pipe(fd) _pipe(fd, 512, O_BINARY) +#else +#define O_BINARY 0 +#endif + +#ifndef NOPLAN9DEFINES +#define alarm p9alarm +#define dup p9dup +#define exec p9exec +#define execl p9execl +#define seek p9seek +#define sleep p9sleep +#define wait p9wait +#define waitpid p9waitpid +#define rfork p9rfork +#define create p9create +#undef open +#define open p9open +#define waitfor p9waitfor +#endif + +extern Dir* dirstat(char*); +extern Dir* dirfstat(int); +extern int dirwstat(char*, Dir*); +extern int dirfwstat(int, Dir*); +extern void nulldir(Dir*); +extern long dirreadall(int, Dir**); +extern void rerrstr(char*, uint); +extern char* sysname(void); +extern void werrstr(char*, ...); +extern char* getns(void); +extern char* get9root(void); +extern char* unsharp(char*); + +/* external names that we don't want to step on */ +#ifndef NOPLAN9DEFINES +#define main p9main +#endif + +/* compiler directives on plan 9 */ +#define SET(x) ((x)=0) +#define USED(x) if(x){}else{} +#ifdef __GNUC__ +# if __GNUC__ >= 3 +# undef USED +# define USED(x) ((void)(x)) +# endif +#endif + +/* command line */ +extern char *argv0; +extern void __fixargv0(void); +#define ARGBEGIN for((argv0?0:(argv0=(__fixargv0(),*argv))),argv++,argc--;\ + argv[0] && argv[0][0]=='-' && argv[0][1];\ + argc--, argv++) {\ + char *_args, *_argt;\ + Rune _argc;\ + _args = &argv[0][1];\ + if(_args[0]=='-' && _args[1]==0){\ + argc--; argv++; break;\ + }\ + _argc = 0;\ + while(*_args && (_args += chartorune(&_argc, _args)))\ + switch(_argc) +#define ARGEND SET(_argt);USED(_argt);USED(_argc);USED(_args);}USED(argv);USED(argc); +#define ARGF() (_argt=_args, _args="",\ + (*_argt? _argt: argv[1]? (argc--, *++argv): 0)) +#define EARGF(x) (_argt=_args, _args="",\ + (*_argt? _argt: argv[1]? (argc--, *++argv): ((x), abort(), (char*)0))) + +#define ARGC() _argc + +#if defined(__cplusplus) +} +#endif +#endif /* _LIB9_H_ */ diff --git a/include/mach.h b/include/mach.h new file mode 100644 index 000000000..5b1ce7b3a --- /dev/null +++ b/include/mach.h @@ -0,0 +1,410 @@ +// Inferno libmach/a.out.h and libmach/mach.h +// http://code.google.com/p/inferno-os/source/browse/utils/libmach/a.out.h +// http://code.google.com/p/inferno-os/source/browse/utils/libmach/mach.h +// +// Copyright © 1994-1999 Lucent Technologies Inc. +// Power PC support Copyright © 1995-2004 C H Forsyth (forsyth@terzarima.net). +// Portions Copyright © 1997-1999 Vita Nuova Limited. +// Portions Copyright © 2000-2007 Vita Nuova Holdings Limited (www.vitanuova.com). +// Revisions Copyright © 2000-2004 Lucent Technologies Inc. and others. +// Portions Copyright © 2009 The Go Authors. All rights reserved. +// +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to deal +// in the Software without restriction, including without limitation the rights +// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +// copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +// THE SOFTWARE. + +/* + * Architecture-dependent application data + */ + +typedef struct Exec Exec; +struct Exec +{ + int32 magic; /* magic number */ + int32 text; /* size of text segment */ + int32 data; /* size of initialized data */ + int32 bss; /* size of uninitialized data */ + int32 syms; /* size of symbol table */ + int32 entry; /* entry point */ + int32 spsz; /* size of pc/sp offset table */ + int32 pcsz; /* size of pc/line number table */ +}; + +#define HDR_MAGIC 0x00008000 /* header expansion */ + +#define _MAGIC(f, b) ((f)|((((4*(b))+0)*(b))+7)) +#define A_MAGIC _MAGIC(0, 8) /* 68020 */ +#define I_MAGIC _MAGIC(0, 11) /* intel 386 */ +#define J_MAGIC _MAGIC(0, 12) /* intel 960 (retired) */ +#define K_MAGIC _MAGIC(0, 13) /* sparc */ +#define V_MAGIC _MAGIC(0, 16) /* mips 3000 BE */ +#define X_MAGIC _MAGIC(0, 17) /* att dsp 3210 (retired) */ +#define M_MAGIC _MAGIC(0, 18) /* mips 4000 BE */ +#define D_MAGIC _MAGIC(0, 19) /* amd 29000 (retired) */ +#define E_MAGIC _MAGIC(0, 20) /* arm */ +#define Q_MAGIC _MAGIC(0, 21) /* powerpc */ +#define N_MAGIC _MAGIC(0, 22) /* mips 4000 LE */ +#define L_MAGIC _MAGIC(0, 23) /* dec alpha */ +#define P_MAGIC _MAGIC(0, 24) /* mips 3000 LE */ +#define U_MAGIC _MAGIC(0, 25) /* sparc64 */ +#define S_MAGIC _MAGIC(HDR_MAGIC, 26) /* amd64 */ +#define T_MAGIC _MAGIC(HDR_MAGIC, 27) /* powerpc64 */ + +#define MIN_MAGIC 8 +#define MAX_MAGIC 27 /* <= 90 */ + +#define DYN_MAGIC 0x80000000 /* dlm */ + +typedef struct Sym Sym; +struct Sym +{ + vlong value; + uint sig; + char type; + char *name; + vlong gotype; + int sequence; // order in file +}; + + +/* + * Supported architectures: + * mips, + * 68020, + * i386, + * amd64, + * sparc, + * sparc64, + * mips2 (R4000) + * arm + * powerpc, + * powerpc64 + * alpha + */ +enum +{ + MMIPS, /* machine types */ + MSPARC, + M68020, + MI386, + MI960, /* retired */ + M3210, /* retired */ + MMIPS2, + NMIPS2, + M29000, /* retired */ + MARM, + MPOWER, + MALPHA, + NMIPS, + MSPARC64, + MAMD64, + MPOWER64, + /* types of executables */ + FNONE = 0, /* unidentified */ + FMIPS, /* v.out */ + FMIPSB, /* mips bootable */ + FSPARC, /* k.out */ + FSPARCB, /* Sparc bootable */ + F68020, /* 2.out */ + F68020B, /* 68020 bootable */ + FNEXTB, /* Next bootable */ + FI386, /* 8.out */ + FI386B, /* I386 bootable */ + FI960, /* retired */ + FI960B, /* retired */ + F3210, /* retired */ + FMIPS2BE, /* 4.out */ + F29000, /* retired */ + FARM, /* 5.out */ + FARMB, /* ARM bootable */ + FPOWER, /* q.out */ + FPOWERB, /* power pc bootable */ + FMIPS2LE, /* 0.out */ + FALPHA, /* 7.out */ + FALPHAB, /* DEC Alpha bootable */ + FMIPSLE, /* 3k little endian */ + FSPARC64, /* u.out */ + FAMD64, /* 6.out */ + FAMD64B, /* 6.out bootable */ + FPOWER64, /* 9.out */ + FPOWER64B, /* 9.out bootable */ + + ANONE = 0, /* dissembler types */ + AMIPS, + AMIPSCO, /* native mips */ + ASPARC, + ASUNSPARC, /* native sun */ + A68020, + AI386, + AI8086, /* oh god */ + AI960, /* retired */ + A29000, /* retired */ + AARM, + APOWER, + AALPHA, + ASPARC64, + AAMD64, + APOWER64, + /* object file types */ + Obj68020 = 0, /* .2 */ + ObjSparc, /* .k */ + ObjMips, /* .v */ + Obj386, /* .8 */ + Obj960, /* retired */ + Obj3210, /* retired */ + ObjMips2, /* .4 */ + Obj29000, /* retired */ + ObjArm, /* .5 */ + ObjPower, /* .q */ + ObjMips2le, /* .0 */ + ObjAlpha, /* .7 */ + ObjSparc64, /* .u */ + ObjAmd64, /* .6 */ + ObjSpim, /* .0 */ + ObjPower64, /* .9 */ + Maxobjtype, + + CNONE = 0, /* symbol table classes */ + CAUTO, + CPARAM, + CSTAB, + CTEXT, + CDATA, + CANY, /* to look for any class */ +}; + +typedef struct Map Map; +typedef struct Symbol Symbol; +typedef struct Reglist Reglist; +typedef struct Mach Mach; +typedef struct Machdata Machdata; +typedef struct Seg Seg; + +typedef int Maprw(Map *m, Seg *s, uvlong addr, void *v, uint n, int isread); + +struct Seg { + char *name; /* the segment name */ + int fd; /* file descriptor */ + int inuse; /* in use - not in use */ + int cache; /* should cache reads? */ + uvlong b; /* base */ + uvlong e; /* end */ + vlong f; /* offset within file */ + Maprw *rw; /* read/write fn for seg */ +}; + +/* + * Structure to map a segment to data + */ +struct Map { + int pid; + int tid; + int nsegs; /* number of segments */ + Seg seg[1]; /* actually n of these */ +}; + +/* + * Internal structure describing a symbol table entry + */ +struct Symbol { + void *handle; /* used internally - owning func */ + struct { + char *name; + vlong value; /* address or stack offset */ + char type; /* as in a.out.h */ + char class; /* as above */ + int index; /* in findlocal, globalsym, textsym */ + }; +}; + +/* + * machine register description + */ +struct Reglist { + char *rname; /* register name */ + short roffs; /* offset in u-block */ + char rflags; /* INTEGER/FLOAT, WRITABLE */ + char rformat; /* print format: 'x', 'X', 'f', '8', '3', 'Y', 'W' */ +}; + +enum { /* bits in rflags field */ + RINT = (0<<0), + RFLT = (1<<0), + RRDONLY = (1<<1), +}; + +/* + * Machine-dependent data is stored in two structures: + * Mach - miscellaneous general parameters + * Machdata - jump vector of service functions used by debuggers + * + * Mach is defined in ?.c and set in executable.c + * + * Machdata is defined in ?db.c + * and set in the debugger startup. + */ +struct Mach{ + char *name; + int mtype; /* machine type code */ + Reglist *reglist; /* register set */ + int32 regsize; /* sizeof registers in bytes */ + int32 fpregsize; /* sizeof fp registers in bytes */ + char *pc; /* pc name */ + char *sp; /* sp name */ + char *link; /* link register name */ + char *sbreg; /* static base register name */ + uvlong sb; /* static base register value */ + int pgsize; /* page size */ + uvlong kbase; /* kernel base address */ + uvlong ktmask; /* ktzero = kbase & ~ktmask */ + uvlong utop; /* user stack top */ + int pcquant; /* quantization of pc */ + int szaddr; /* sizeof(void*) */ + int szreg; /* sizeof(register) */ + int szfloat; /* sizeof(float) */ + int szdouble; /* sizeof(double) */ +}; + +extern Mach *mach; /* Current machine */ + +typedef uvlong (*Rgetter)(Map*, char*); +typedef void (*Tracer)(Map*, uvlong, uvlong, Symbol*); + +struct Machdata { /* Machine-dependent debugger support */ + uchar bpinst[4]; /* break point instr. */ + short bpsize; /* size of break point instr. */ + + ushort (*swab)(ushort); /* ushort to local byte order */ + uint32 (*swal)(uint32); /* uint32 to local byte order */ + uvlong (*swav)(uvlong); /* uvlong to local byte order */ + int (*ctrace)(Map*, uvlong, uvlong, uvlong, Tracer); /* C traceback */ + uvlong (*findframe)(Map*, uvlong, uvlong, uvlong, uvlong);/* frame finder */ + char* (*excep)(Map*, Rgetter); /* last exception */ + uint32 (*bpfix)(uvlong); /* breakpoint fixup */ + int (*sftos)(char*, int, void*); /* single precision float */ + int (*dftos)(char*, int, void*); /* double precision float */ + int (*foll)(Map*, uvlong, Rgetter, uvlong*);/* follow set */ + int (*das)(Map*, uvlong, char, char*, int); /* symbolic disassembly */ + int (*hexinst)(Map*, uvlong, char*, int); /* hex disassembly */ + int (*instsize)(Map*, uvlong); /* instruction size */ +}; + +/* + * Common a.out header describing all architectures + */ +typedef struct Fhdr +{ + char *name; /* identifier of executable */ + uchar type; /* file type - see codes above */ + uchar hdrsz; /* header size */ + uchar _magic; /* _MAGIC() magic */ + uchar spare; + int32 magic; /* magic number */ + uvlong txtaddr; /* text address */ + vlong txtoff; /* start of text in file */ + uvlong dataddr; /* start of data segment */ + vlong datoff; /* offset to data seg in file */ + vlong symoff; /* offset of symbol table in file */ + uvlong entry; /* entry point */ + vlong sppcoff; /* offset of sp-pc table in file */ + vlong lnpcoff; /* offset of line number-pc table in file */ + int32 txtsz; /* text size */ + int32 datsz; /* size of data seg */ + int32 bsssz; /* size of bss */ + int32 symsz; /* size of symbol table */ + int32 sppcsz; /* size of sp-pc table */ + int32 lnpcsz; /* size of line number-pc table */ +} Fhdr; + +extern int asstype; /* dissembler type - machdata.c */ +extern Machdata *machdata; /* jump vector - machdata.c */ + +int beieee80ftos(char*, int, void*); +int beieeesftos(char*, int, void*); +int beieeedftos(char*, int, void*); +ushort beswab(ushort); +uint32 beswal(uint32); +uvlong beswav(uvlong); +uvlong ciscframe(Map*, uvlong, uvlong, uvlong, uvlong); +int cisctrace(Map*, uvlong, uvlong, uvlong, Tracer); +int crackhdr(int fd, Fhdr*); +uvlong file2pc(char*, int32); +int fileelem(Sym**, uchar *, char*, int); +int32 fileline(char*, int, uvlong); +int filesym(int, char*, int); +int findlocal(Symbol*, char*, Symbol*); +int findseg(Map*, char*); +int findsym(uvlong, int, Symbol *); +int fnbound(uvlong, uvlong*); +int fpformat(Map*, Reglist*, char*, int, int); +int get1(Map*, uvlong, uchar*, int); +int get2(Map*, uvlong, ushort*); +int get4(Map*, uvlong, uint32*); +int get8(Map*, uvlong, uvlong*); +int geta(Map*, uvlong, uvlong*); +int getauto(Symbol*, int, int, Symbol*); +Sym* getsym(int); +int globalsym(Symbol *, int); +char* _hexify(char*, uint32, int); +int ieeesftos(char*, int, uint32); +int ieeedftos(char*, int, uint32, uint32); +int isar(Biobuf*); +int leieee80ftos(char*, int, void*); +int leieeesftos(char*, int, void*); +int leieeedftos(char*, int, void*); +ushort leswab(ushort); +uint32 leswal(uint32); +uvlong leswav(uvlong); +uvlong line2addr(int32, uvlong, uvlong); +Map* loadmap(Map*, int, Fhdr*); +int localaddr(Map*, char*, char*, uvlong*, Rgetter); +int localsym(Symbol*, int); +int lookup(char*, char*, Symbol*); +void machbytype(int); +int machbyname(char*); +int nextar(Biobuf*, int, char*); +Map* newmap(Map*, int); +void objtraverse(void(*)(Sym*, void*), void*); +int objtype(Biobuf*, char**); +uvlong pc2sp(uvlong); +int32 pc2line(uvlong); +int put1(Map*, uvlong, uchar*, int); +int put2(Map*, uvlong, ushort); +int put4(Map*, uvlong, uint32); +int put8(Map*, uvlong, uvlong); +int puta(Map*, uvlong, uvlong); +int readar(Biobuf*, int, vlong, int); +int readobj(Biobuf*, int); +uvlong riscframe(Map*, uvlong, uvlong, uvlong, uvlong); +int risctrace(Map*, uvlong, uvlong, uvlong, Tracer); +int setmap(Map*, int, uvlong, uvlong, vlong, char*, Maprw *rw); +Sym* symbase(int32*); +int syminit(int, Fhdr*); +int symoff(char*, int, uvlong, int); +void textseg(uvlong, Fhdr*); +int textsym(Symbol*, int); +void unusemap(Map*, int); + +Map* attachproc(int pid, Fhdr *fp); +int ctlproc(int pid, char *msg); +void detachproc(Map *m); +int procnotes(int pid, char ***pnotes); +char* proctextfile(int pid); +int procthreadpids(int pid, int *tid, int ntid); +char* procstatus(int); + +Maprw fdrw; diff --git a/include/u.h b/include/u.h new file mode 100644 index 000000000..44bfcd63b --- /dev/null +++ b/include/u.h @@ -0,0 +1,230 @@ +/* +Plan 9 from User Space include/u.h +http://code.swtch.com/plan9port/src/tip/include/u.h + +Copyright 2001-2007 Russ Cox. All Rights Reserved. + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. +*/ + +#ifndef _U_H_ +#define _U_H_ 1 +#if defined(__cplusplus) +extern "C" { +#endif + +#define __BSD_VISIBLE 1 /* FreeBSD 5.x */ +#if defined(__sun__) +# define __EXTENSIONS__ 1 /* SunOS */ +# if defined(__SunOS5_6__) || defined(__SunOS5_7__) || defined(__SunOS5_8__) + /* NOT USING #define __MAKECONTEXT_V2_SOURCE 1 / * SunOS */ +# else +# define __MAKECONTEXT_V2_SOURCE 1 +# endif +#endif +#define _BSD_SOURCE 1 +#define _NETBSD_SOURCE 1 /* NetBSD */ +#define _SVID_SOURCE 1 +#if !defined(__APPLE__) && !defined(__OpenBSD__) +# define _XOPEN_SOURCE 1000 +# define _XOPEN_SOURCE_EXTENDED 1 +#endif +#if defined(__FreeBSD__) +# include + /* for strtoll */ +# undef __ISO_C_VISIBLE +# define __ISO_C_VISIBLE 1999 +# undef __LONG_LONG_SUPPORTED +# define __LONG_LONG_SUPPORTED +#endif +#define _LARGEFILE64_SOURCE 1 +#define _FILE_OFFSET_BITS 64 + +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include /* for tolower */ +#include +#include + +/* + * OS-specific crap + */ +#define _NEEDUCHAR 1 +#define _NEEDUSHORT 1 +#define _NEEDUINT 1 +#define _NEEDULONG 1 + +#ifdef _WIN32 +typedef jmp_buf sigjmp_buf; +#endif +typedef long p9jmp_buf[sizeof(sigjmp_buf)/sizeof(long)]; + +#if defined(__linux__) +# include +# if defined(__Linux26__) +# include +# define PLAN9PORT_USING_PTHREADS 1 +# endif +# if defined(__USE_MISC) +# undef _NEEDUSHORT +# undef _NEEDUINT +# undef _NEEDULONG +# endif +#elif defined(__sun__) +# include +# include +# define PLAN9PORT_USING_PTHREADS 1 +# undef _NEEDUSHORT +# undef _NEEDUINT +# undef _NEEDULONG +# define nil 0 /* no cast to void* */ +#elif defined(__FreeBSD__) +# include +# include +# if __FreeBSD_version >= 500000 +# define PLAN9PORT_USING_PTHREADS 1 +# include +# endif +# if !defined(_POSIX_SOURCE) +# undef _NEEDUSHORT +# undef _NEEDUINT +# endif +#elif defined(__APPLE__) +# include +# include +# define PLAN9PORT_USING_PTHREADS 1 +# if __GNUC__ < 4 +# undef _NEEDUSHORT +# undef _NEEDUINT +# endif +# undef _ANSI_SOURCE +# undef _POSIX_C_SOURCE +# undef _XOPEN_SOURCE +# if !defined(NSIG) +# define NSIG 32 +# endif +# define _NEEDLL 1 +#elif defined(__NetBSD__) +# include +# include +# undef _NEEDUSHORT +# undef _NEEDUINT +# undef _NEEDULONG +#elif defined(__OpenBSD__) +# include +# undef _NEEDUSHORT +# undef _NEEDUINT +# undef _NEEDULONG +#elif defined(_WIN32) +#else + /* No idea what system this is -- try some defaults */ +# include +# define PLAN9PORT_USING_PTHREADS 1 +#endif + +#ifndef O_DIRECT +#define O_DIRECT 0 +#endif + +typedef signed char schar; + +#ifdef _NEEDUCHAR + typedef unsigned char uchar; +#endif +#ifdef _NEEDUSHORT + typedef unsigned short ushort; +#endif +#ifdef _NEEDUINT + typedef unsigned int uint; +#endif +#ifdef _NEEDULONG + typedef unsigned long ulong; +#endif +typedef unsigned long long uvlong; +typedef long long vlong; + +typedef uint64_t u64int; +typedef int64_t s64int; +typedef uint8_t u8int; +typedef int8_t s8int; +typedef uint16_t u16int; +typedef int16_t s16int; +typedef uintptr_t uintptr; +typedef intptr_t intptr; +typedef uint32_t u32int; +typedef int32_t s32int; + +typedef s8int int8; +typedef u8int uint8; +typedef s16int int16; +typedef u16int uint16; +typedef s32int int32; +typedef u32int uint32; +typedef s64int int64; +typedef u64int uint64; + + +#undef _NEEDUCHAR +#undef _NEEDUSHORT +#undef _NEEDUINT +#undef _NEEDULONG + +#define getcallerpc(x) __builtin_return_address(0) + +#ifndef SIGBUS +#define SIGBUS SIGSEGV /* close enough */ +#endif + +/* + * Funny-named symbols to tip off 9l to autolink. + */ +#define AUTOLIB(x) static int __p9l_autolib_ ## x = 1; +#define AUTOFRAMEWORK(x) static int __p9l_autoframework_ ## x = 1; + +/* + * Gcc is too smart for its own good. + */ +#if defined(__GNUC__) +# undef strcmp /* causes way too many warnings */ +# if __GNUC__ >= 4 || (__GNUC__==3 && !defined(__APPLE_CC__) && !defined(_WIN32)) +# undef AUTOLIB +# define AUTOLIB(x) int __p9l_autolib_ ## x __attribute__ ((weak)); +# undef AUTOFRAMEWORK +# define AUTOFRAMEWORK(x) int __p9l_autoframework_ ## x __attribute__ ((weak)); +# else +# undef AUTOLIB +# define AUTOLIB(x) static int __p9l_autolib_ ## x __attribute__ ((unused)); +# undef AUTOFRAMEWORK +# define AUTOFRAMEWORK(x) static int __p9l_autoframework_ ## x __attribute__ ((unused)); +# endif +#endif + +#if defined(__cplusplus) +} +#endif +#endif diff --git a/include/ureg_amd64.h b/include/ureg_amd64.h new file mode 100644 index 000000000..2c39f17ce --- /dev/null +++ b/include/ureg_amd64.h @@ -0,0 +1,58 @@ +// Inferno utils/libmach/ureg6.h +// http://code.google.com/p/inferno-os/source/browse/utils/libmach/ureg6.h +// +// Copyright © 1994-1999 Lucent Technologies Inc. +// Power PC support Copyright © 1995-2004 C H Forsyth (forsyth@terzarima.net). +// Portions Copyright © 1997-1999 Vita Nuova Limited. +// Portions Copyright © 2000-2007 Vita Nuova Holdings Limited (www.vitanuova.com). +// Revisions Copyright © 2000-2004 Lucent Technologies Inc. and others. +// Portions Copyright © 2009 The Go Authors. All rights reserved. +// +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to deal +// in the Software without restriction, including without limitation the rights +// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +// copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +// THE SOFTWARE. + +struct Ureg { + u64int ax; + u64int bx; + u64int cx; + u64int dx; + u64int si; + u64int di; + u64int bp; + u64int r8; + u64int r9; + u64int r10; + u64int r11; + u64int r12; + u64int r13; + u64int r14; + u64int r15; + + u16int ds; + u16int es; + u16int fs; + u16int gs; + + u64int type; + u64int error; /* error code (or zero) */ + u64int ip; /* pc */ + u64int cs; /* old context */ + u64int flags; /* old flags */ + u64int sp; /* sp */ + u64int ss; /* old stack segment */ +}; diff --git a/include/ureg_arm.h b/include/ureg_arm.h new file mode 100644 index 000000000..c740b0302 --- /dev/null +++ b/include/ureg_arm.h @@ -0,0 +1,49 @@ +// Inferno utils/libmach/ureg5.h +// http://code.google.com/p/inferno-os/source/browse/utils/libmach/ureg5.h +// +// Copyright © 1994-1999 Lucent Technologies Inc. +// Power PC support Copyright © 1995-2004 C H Forsyth (forsyth@terzarima.net). +// Portions Copyright © 1997-1999 Vita Nuova Limited. +// Portions Copyright © 2000-2007 Vita Nuova Holdings Limited (www.vitanuova.com). +// Revisions Copyright © 2000-2004 Lucent Technologies Inc. and others. +// Portions Copyright © 2009 The Go Authors. All rights reserved. +// +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to deal +// in the Software without restriction, including without limitation the rights +// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +// copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +// THE SOFTWARE. + +struct Ureg { + uint r0; + uint r1; + uint r2; + uint r3; + uint r4; + uint r5; + uint r6; + uint r7; + uint r8; + uint r9; + uint r10; + uint r11; + uint r12; + uint r13; + uint r14; + uint link; + uint type; + uint psr; + uint pc; +}; diff --git a/include/ureg_x86.h b/include/ureg_x86.h new file mode 100644 index 000000000..c20fe4e4c --- /dev/null +++ b/include/ureg_x86.h @@ -0,0 +1,53 @@ +// Inferno utils/libmach/ureg8.h +// http://code.google.com/p/inferno-os/source/browse/utils/libmach/ureg8.h +// +// Copyright © 1994-1999 Lucent Technologies Inc. +// Power PC support Copyright © 1995-2004 C H Forsyth (forsyth@terzarima.net). +// Portions Copyright © 1997-1999 Vita Nuova Limited. +// Portions Copyright © 2000-2007 Vita Nuova Holdings Limited (www.vitanuova.com). +// Revisions Copyright © 2000-2004 Lucent Technologies Inc. and others. +// Portions Copyright © 2009 The Go Authors. All rights reserved. +// +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to deal +// in the Software without restriction, including without limitation the rights +// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +// copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +// THE SOFTWARE. + +struct Ureg +{ + uint32 di; /* general registers */ + uint32 si; /* ... */ + uint32 bp; /* ... */ + uint32 nsp; + uint32 bx; /* ... */ + uint32 dx; /* ... */ + uint32 cx; /* ... */ + uint32 ax; /* ... */ + uint32 gs; /* data segments */ + uint32 fs; /* ... */ + uint32 es; /* ... */ + uint32 ds; /* ... */ + uint32 trap; /* trap type */ + uint32 ecode; /* error code (or zero) */ + uint32 pc; /* pc */ + uint32 cs; /* old context */ + uint32 flags; /* old flags */ + union { + uint32 usp; + uint32 sp; + }; + uint32 ss; /* old stack segment */ +}; diff --git a/include/utf.h b/include/utf.h new file mode 100644 index 000000000..be1c46e7f --- /dev/null +++ b/include/utf.h @@ -0,0 +1 @@ +#include "../src/lib9/utf/utf.h" diff --git a/lib/codereview/codereview.cfg b/lib/codereview/codereview.cfg new file mode 100644 index 000000000..93b55c0a3 --- /dev/null +++ b/lib/codereview/codereview.cfg @@ -0,0 +1 @@ +defaultcc: golang-dev@googlegroups.com diff --git a/lib/codereview/codereview.py b/lib/codereview/codereview.py new file mode 100644 index 000000000..63f67fff9 --- /dev/null +++ b/lib/codereview/codereview.py @@ -0,0 +1,3292 @@ +# coding=utf-8 +# (The line above is necessary so that I can use 世界 in the +# *comment* below without Python getting all bent out of shape.) + +# Copyright 2007-2009 Google Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +'''Mercurial interface to codereview.appspot.com. + +To configure, set the following options in +your repository's .hg/hgrc file. + + [extensions] + codereview = path/to/codereview.py + + [codereview] + server = codereview.appspot.com + +The server should be running Rietveld; see http://code.google.com/p/rietveld/. + +In addition to the new commands, this extension introduces +the file pattern syntax @nnnnnn, where nnnnnn is a change list +number, to mean the files included in that change list, which +must be associated with the current client. + +For example, if change 123456 contains the files x.go and y.go, +"hg diff @123456" is equivalent to"hg diff x.go y.go". +''' + +from mercurial import cmdutil, commands, hg, util, error, match +from mercurial.node import nullrev, hex, nullid, short +import os, re, time +import stat +import subprocess +import threading +from HTMLParser import HTMLParser + +# The standard 'json' package is new in Python 2.6. +# Before that it was an external package named simplejson. +try: + # Standard location in 2.6 and beyond. + import json +except Exception, e: + try: + # Conventional name for earlier package. + import simplejson as json + except: + try: + # Was also bundled with django, which is commonly installed. + from django.utils import simplejson as json + except: + # We give up. + raise e + +try: + hgversion = util.version() +except: + from mercurial.version import version as v + hgversion = v.get_version() + +try: + from mercurial.discovery import findcommonincoming +except: + def findcommonincoming(repo, remote): + return repo.findcommonincoming(remote) + +# in Mercurial 1.9 the cmdutil.match and cmdutil.revpair moved to scmutil +if hgversion >= '1.9': + from mercurial import scmutil +else: + scmutil = cmdutil + +oldMessage = """ +The code review extension requires Mercurial 1.3 or newer. + +To install a new Mercurial, + + sudo easy_install mercurial + +works on most systems. +""" + +linuxMessage = """ +You may need to clear your current Mercurial installation by running: + + sudo apt-get remove mercurial mercurial-common + sudo rm -rf /etc/mercurial +""" + +if hgversion < '1.3': + msg = oldMessage + if os.access("/etc/mercurial", 0): + msg += linuxMessage + raise util.Abort(msg) + +def promptyesno(ui, msg): + # Arguments to ui.prompt changed between 1.3 and 1.3.1. + # Even so, some 1.3.1 distributions seem to have the old prompt!?!? + # What a terrible way to maintain software. + try: + return ui.promptchoice(msg, ["&yes", "&no"], 0) == 0 + except AttributeError: + return ui.prompt(msg, ["&yes", "&no"], "y") != "n" + +# To experiment with Mercurial in the python interpreter: +# >>> repo = hg.repository(ui.ui(), path = ".") + +####################################################################### +# Normally I would split this into multiple files, but it simplifies +# import path headaches to keep it all in one file. Sorry. + +import sys +if __name__ == "__main__": + print >>sys.stderr, "This is a Mercurial extension and should not be invoked directly." + sys.exit(2) + +server = "codereview.appspot.com" +server_url_base = None +defaultcc = None +contributors = {} +missing_codereview = None +real_rollback = None +releaseBranch = None + +####################################################################### +# RE: UNICODE STRING HANDLING +# +# Python distinguishes between the str (string of bytes) +# and unicode (string of code points) types. Most operations +# work on either one just fine, but some (like regexp matching) +# require unicode, and others (like write) require str. +# +# As befits the language, Python hides the distinction between +# unicode and str by converting between them silently, but +# *only* if all the bytes/code points involved are 7-bit ASCII. +# This means that if you're not careful, your program works +# fine on "hello, world" and fails on "hello, 世界". And of course, +# the obvious way to be careful - use static types - is unavailable. +# So the only way is trial and error to find where to put explicit +# conversions. +# +# Because more functions do implicit conversion to str (string of bytes) +# than do implicit conversion to unicode (string of code points), +# the convention in this module is to represent all text as str, +# converting to unicode only when calling a unicode-only function +# and then converting back to str as soon as possible. + +def typecheck(s, t): + if type(s) != t: + raise util.Abort("type check failed: %s has type %s != %s" % (repr(s), type(s), t)) + +# If we have to pass unicode instead of str, ustr does that conversion clearly. +def ustr(s): + typecheck(s, str) + return s.decode("utf-8") + +# Even with those, Mercurial still sometimes turns unicode into str +# and then tries to use it as ascii. Change Mercurial's default. +def set_mercurial_encoding_to_utf8(): + from mercurial import encoding + encoding.encoding = 'utf-8' + +set_mercurial_encoding_to_utf8() + +# Even with those we still run into problems. +# I tried to do things by the book but could not convince +# Mercurial to let me check in a change with UTF-8 in the +# CL description or author field, no matter how many conversions +# between str and unicode I inserted and despite changing the +# default encoding. I'm tired of this game, so set the default +# encoding for all of Python to 'utf-8', not 'ascii'. +def default_to_utf8(): + import sys + reload(sys) # site.py deleted setdefaultencoding; get it back + sys.setdefaultencoding('utf-8') + +default_to_utf8() + +####################################################################### +# Change list parsing. +# +# Change lists are stored in .hg/codereview/cl.nnnnnn +# where nnnnnn is the number assigned by the code review server. +# Most data about a change list is stored on the code review server +# too: the description, reviewer, and cc list are all stored there. +# The only thing in the cl.nnnnnn file is the list of relevant files. +# Also, the existence of the cl.nnnnnn file marks this repository +# as the one where the change list lives. + +emptydiff = """Index: ~rietveld~placeholder~ +=================================================================== +diff --git a/~rietveld~placeholder~ b/~rietveld~placeholder~ +new file mode 100644 +""" + +class CL(object): + def __init__(self, name): + typecheck(name, str) + self.name = name + self.desc = '' + self.files = [] + self.reviewer = [] + self.cc = [] + self.url = '' + self.local = False + self.web = False + self.copied_from = None # None means current user + self.mailed = False + self.private = False + + def DiskText(self): + cl = self + s = "" + if cl.copied_from: + s += "Author: " + cl.copied_from + "\n\n" + if cl.private: + s += "Private: " + str(self.private) + "\n" + s += "Mailed: " + str(self.mailed) + "\n" + s += "Description:\n" + s += Indent(cl.desc, "\t") + s += "Files:\n" + for f in cl.files: + s += "\t" + f + "\n" + typecheck(s, str) + return s + + def EditorText(self): + cl = self + s = _change_prolog + s += "\n" + if cl.copied_from: + s += "Author: " + cl.copied_from + "\n" + if cl.url != '': + s += 'URL: ' + cl.url + ' # cannot edit\n\n' + if cl.private: + s += "Private: True\n" + s += "Reviewer: " + JoinComma(cl.reviewer) + "\n" + s += "CC: " + JoinComma(cl.cc) + "\n" + s += "\n" + s += "Description:\n" + if cl.desc == '': + s += "\t\n" + else: + s += Indent(cl.desc, "\t") + s += "\n" + if cl.local or cl.name == "new": + s += "Files:\n" + for f in cl.files: + s += "\t" + f + "\n" + s += "\n" + typecheck(s, str) + return s + + def PendingText(self): + cl = self + s = cl.name + ":" + "\n" + s += Indent(cl.desc, "\t") + s += "\n" + if cl.copied_from: + s += "\tAuthor: " + cl.copied_from + "\n" + s += "\tReviewer: " + JoinComma(cl.reviewer) + "\n" + s += "\tCC: " + JoinComma(cl.cc) + "\n" + s += "\tFiles:\n" + for f in cl.files: + s += "\t\t" + f + "\n" + typecheck(s, str) + return s + + def Flush(self, ui, repo): + if self.name == "new": + self.Upload(ui, repo, gofmt_just_warn=True, creating=True) + dir = CodeReviewDir(ui, repo) + path = dir + '/cl.' + self.name + f = open(path+'!', "w") + f.write(self.DiskText()) + f.close() + if sys.platform == "win32" and os.path.isfile(path): + os.remove(path) + os.rename(path+'!', path) + if self.web and not self.copied_from: + EditDesc(self.name, desc=self.desc, + reviewers=JoinComma(self.reviewer), cc=JoinComma(self.cc), + private=self.private) + + def Delete(self, ui, repo): + dir = CodeReviewDir(ui, repo) + os.unlink(dir + "/cl." + self.name) + + def Subject(self): + s = line1(self.desc) + if len(s) > 60: + s = s[0:55] + "..." + if self.name != "new": + s = "code review %s: %s" % (self.name, s) + typecheck(s, str) + return s + + def Upload(self, ui, repo, send_mail=False, gofmt=True, gofmt_just_warn=False, creating=False, quiet=False): + if not self.files and not creating: + ui.warn("no files in change list\n") + if ui.configbool("codereview", "force_gofmt", True) and gofmt: + CheckFormat(ui, repo, self.files, just_warn=gofmt_just_warn) + set_status("uploading CL metadata + diffs") + os.chdir(repo.root) + form_fields = [ + ("content_upload", "1"), + ("reviewers", JoinComma(self.reviewer)), + ("cc", JoinComma(self.cc)), + ("description", self.desc), + ("base_hashes", ""), + ] + + if self.name != "new": + form_fields.append(("issue", self.name)) + vcs = None + # We do not include files when creating the issue, + # because we want the patch sets to record the repository + # and base revision they are diffs against. We use the patch + # set message for that purpose, but there is no message with + # the first patch set. Instead the message gets used as the + # new CL's overall subject. So omit the diffs when creating + # and then we'll run an immediate upload. + # This has the effect that every CL begins with an empty "Patch set 1". + if self.files and not creating: + vcs = MercurialVCS(upload_options, ui, repo) + data = vcs.GenerateDiff(self.files) + files = vcs.GetBaseFiles(data) + if len(data) > MAX_UPLOAD_SIZE: + uploaded_diff_file = [] + form_fields.append(("separate_patches", "1")) + else: + uploaded_diff_file = [("data", "data.diff", data)] + else: + uploaded_diff_file = [("data", "data.diff", emptydiff)] + + if vcs and self.name != "new": + form_fields.append(("subject", "diff -r " + vcs.base_rev + " " + getremote(ui, repo, {}).path)) + else: + # First upload sets the subject for the CL itself. + form_fields.append(("subject", self.Subject())) + ctype, body = EncodeMultipartFormData(form_fields, uploaded_diff_file) + response_body = MySend("/upload", body, content_type=ctype) + patchset = None + msg = response_body + lines = msg.splitlines() + if len(lines) >= 2: + msg = lines[0] + patchset = lines[1].strip() + patches = [x.split(" ", 1) for x in lines[2:]] + if response_body.startswith("Issue updated.") and quiet: + pass + else: + ui.status(msg + "\n") + set_status("uploaded CL metadata + diffs") + if not response_body.startswith("Issue created.") and not response_body.startswith("Issue updated."): + raise util.Abort("failed to update issue: " + response_body) + issue = msg[msg.rfind("/")+1:] + self.name = issue + if not self.url: + self.url = server_url_base + self.name + if not uploaded_diff_file: + set_status("uploading patches") + patches = UploadSeparatePatches(issue, rpc, patchset, data, upload_options) + if vcs: + set_status("uploading base files") + vcs.UploadBaseFiles(issue, rpc, patches, patchset, upload_options, files) + if send_mail: + set_status("sending mail") + MySend("/" + issue + "/mail", payload="") + self.web = True + set_status("flushing changes to disk") + self.Flush(ui, repo) + return + + def Mail(self, ui, repo): + pmsg = "Hello " + JoinComma(self.reviewer) + if self.cc: + pmsg += " (cc: %s)" % (', '.join(self.cc),) + pmsg += ",\n" + pmsg += "\n" + repourl = getremote(ui, repo, {}).path + if not self.mailed: + pmsg += "I'd like you to review this change to\n" + repourl + "\n" + else: + pmsg += "Please take another look.\n" + typecheck(pmsg, str) + PostMessage(ui, self.name, pmsg, subject=self.Subject()) + self.mailed = True + self.Flush(ui, repo) + +def GoodCLName(name): + typecheck(name, str) + return re.match("^[0-9]+$", name) + +def ParseCL(text, name): + typecheck(text, str) + typecheck(name, str) + sname = None + lineno = 0 + sections = { + 'Author': '', + 'Description': '', + 'Files': '', + 'URL': '', + 'Reviewer': '', + 'CC': '', + 'Mailed': '', + 'Private': '', + } + for line in text.split('\n'): + lineno += 1 + line = line.rstrip() + if line != '' and line[0] == '#': + continue + if line == '' or line[0] == ' ' or line[0] == '\t': + if sname == None and line != '': + return None, lineno, 'text outside section' + if sname != None: + sections[sname] += line + '\n' + continue + p = line.find(':') + if p >= 0: + s, val = line[:p].strip(), line[p+1:].strip() + if s in sections: + sname = s + if val != '': + sections[sname] += val + '\n' + continue + return None, lineno, 'malformed section header' + + for k in sections: + sections[k] = StripCommon(sections[k]).rstrip() + + cl = CL(name) + if sections['Author']: + cl.copied_from = sections['Author'] + cl.desc = sections['Description'] + for line in sections['Files'].split('\n'): + i = line.find('#') + if i >= 0: + line = line[0:i].rstrip() + line = line.strip() + if line == '': + continue + cl.files.append(line) + cl.reviewer = SplitCommaSpace(sections['Reviewer']) + cl.cc = SplitCommaSpace(sections['CC']) + cl.url = sections['URL'] + if sections['Mailed'] != 'False': + # Odd default, but avoids spurious mailings when + # reading old CLs that do not have a Mailed: line. + # CLs created with this update will always have + # Mailed: False on disk. + cl.mailed = True + if sections['Private'] in ('True', 'true', 'Yes', 'yes'): + cl.private = True + if cl.desc == '': + cl.desc = '' + return cl, 0, '' + +def SplitCommaSpace(s): + typecheck(s, str) + s = s.strip() + if s == "": + return [] + return re.split(", *", s) + +def CutDomain(s): + typecheck(s, str) + i = s.find('@') + if i >= 0: + s = s[0:i] + return s + +def JoinComma(l): + for s in l: + typecheck(s, str) + return ", ".join(l) + +def ExceptionDetail(): + s = str(sys.exc_info()[0]) + if s.startswith(""): + s = s[7:-2] + elif s.startswith(""): + s = s[8:-2] + arg = str(sys.exc_info()[1]) + if len(arg) > 0: + s += ": " + arg + return s + +def IsLocalCL(ui, repo, name): + return GoodCLName(name) and os.access(CodeReviewDir(ui, repo) + "/cl." + name, 0) + +# Load CL from disk and/or the web. +def LoadCL(ui, repo, name, web=True): + typecheck(name, str) + set_status("loading CL " + name) + if not GoodCLName(name): + return None, "invalid CL name" + dir = CodeReviewDir(ui, repo) + path = dir + "cl." + name + if os.access(path, 0): + ff = open(path) + text = ff.read() + ff.close() + cl, lineno, err = ParseCL(text, name) + if err != "": + return None, "malformed CL data: "+err + cl.local = True + else: + cl = CL(name) + if web: + set_status("getting issue metadata from web") + d = JSONGet(ui, "/api/" + name + "?messages=true") + set_status(None) + if d is None: + return None, "cannot load CL %s from server" % (name,) + if 'owner_email' not in d or 'issue' not in d or str(d['issue']) != name: + return None, "malformed response loading CL data from code review server" + cl.dict = d + cl.reviewer = d.get('reviewers', []) + cl.cc = d.get('cc', []) + if cl.local and cl.copied_from and cl.desc: + # local copy of CL written by someone else + # and we saved a description. use that one, + # so that committers can edit the description + # before doing hg submit. + pass + else: + cl.desc = d.get('description', "") + cl.url = server_url_base + name + cl.web = True + cl.private = d.get('private', False) != False + set_status("loaded CL " + name) + return cl, '' + +global_status = None + +def set_status(s): + # print >>sys.stderr, "\t", time.asctime(), s + global global_status + global_status = s + +class StatusThread(threading.Thread): + def __init__(self): + threading.Thread.__init__(self) + def run(self): + # pause a reasonable amount of time before + # starting to display status messages, so that + # most hg commands won't ever see them. + time.sleep(30) + + # now show status every 15 seconds + while True: + time.sleep(15 - time.time() % 15) + s = global_status + if s is None: + continue + if s == "": + s = "(unknown status)" + print >>sys.stderr, time.asctime(), s + +def start_status_thread(): + t = StatusThread() + t.setDaemon(True) # allowed to exit if t is still running + t.start() + +class LoadCLThread(threading.Thread): + def __init__(self, ui, repo, dir, f, web): + threading.Thread.__init__(self) + self.ui = ui + self.repo = repo + self.dir = dir + self.f = f + self.web = web + self.cl = None + def run(self): + cl, err = LoadCL(self.ui, self.repo, self.f[3:], web=self.web) + if err != '': + self.ui.warn("loading "+self.dir+self.f+": " + err + "\n") + return + self.cl = cl + +# Load all the CLs from this repository. +def LoadAllCL(ui, repo, web=True): + dir = CodeReviewDir(ui, repo) + m = {} + files = [f for f in os.listdir(dir) if f.startswith('cl.')] + if not files: + return m + active = [] + first = True + for f in files: + t = LoadCLThread(ui, repo, dir, f, web) + t.start() + if web and first: + # first request: wait in case it needs to authenticate + # otherwise we get lots of user/password prompts + # running in parallel. + t.join() + if t.cl: + m[t.cl.name] = t.cl + first = False + else: + active.append(t) + for t in active: + t.join() + if t.cl: + m[t.cl.name] = t.cl + return m + +# Find repository root. On error, ui.warn and return None +def RepoDir(ui, repo): + url = repo.url(); + if not url.startswith('file:'): + ui.warn("repository %s is not in local file system\n" % (url,)) + return None + url = url[5:] + if url.endswith('/'): + url = url[:-1] + typecheck(url, str) + return url + +# Find (or make) code review directory. On error, ui.warn and return None +def CodeReviewDir(ui, repo): + dir = RepoDir(ui, repo) + if dir == None: + return None + dir += '/.hg/codereview/' + if not os.path.isdir(dir): + try: + os.mkdir(dir, 0700) + except: + ui.warn('cannot mkdir %s: %s\n' % (dir, ExceptionDetail())) + return None + typecheck(dir, str) + return dir + +# Turn leading tabs into spaces, so that the common white space +# prefix doesn't get confused when people's editors write out +# some lines with spaces, some with tabs. Only a heuristic +# (some editors don't use 8 spaces either) but a useful one. +def TabsToSpaces(line): + i = 0 + while i < len(line) and line[i] == '\t': + i += 1 + return ' '*(8*i) + line[i:] + +# Strip maximal common leading white space prefix from text +def StripCommon(text): + typecheck(text, str) + ws = None + for line in text.split('\n'): + line = line.rstrip() + if line == '': + continue + line = TabsToSpaces(line) + white = line[:len(line)-len(line.lstrip())] + if ws == None: + ws = white + else: + common = '' + for i in range(min(len(white), len(ws))+1): + if white[0:i] == ws[0:i]: + common = white[0:i] + ws = common + if ws == '': + break + if ws == None: + return text + t = '' + for line in text.split('\n'): + line = line.rstrip() + line = TabsToSpaces(line) + if line.startswith(ws): + line = line[len(ws):] + if line == '' and t == '': + continue + t += line + '\n' + while len(t) >= 2 and t[-2:] == '\n\n': + t = t[:-1] + typecheck(t, str) + return t + +# Indent text with indent. +def Indent(text, indent): + typecheck(text, str) + typecheck(indent, str) + t = '' + for line in text.split('\n'): + t += indent + line + '\n' + typecheck(t, str) + return t + +# Return the first line of l +def line1(text): + typecheck(text, str) + return text.split('\n')[0] + +_change_prolog = """# Change list. +# Lines beginning with # are ignored. +# Multi-line values should be indented. +""" + +####################################################################### +# Mercurial helper functions + +# Get effective change nodes taking into account applied MQ patches +def effective_revpair(repo): + try: + return scmutil.revpair(repo, ['qparent']) + except: + return scmutil.revpair(repo, None) + +# Return list of changed files in repository that match pats. +# Warn about patterns that did not match. +def matchpats(ui, repo, pats, opts): + matcher = scmutil.match(repo, pats, opts) + node1, node2 = effective_revpair(repo) + modified, added, removed, deleted, unknown, ignored, clean = repo.status(node1, node2, matcher, ignored=True, clean=True, unknown=True) + return (modified, added, removed, deleted, unknown, ignored, clean) + +# Return list of changed files in repository that match pats. +# The patterns came from the command line, so we warn +# if they have no effect or cannot be understood. +def ChangedFiles(ui, repo, pats, opts, taken=None): + taken = taken or {} + # Run each pattern separately so that we can warn about + # patterns that didn't do anything useful. + for p in pats: + modified, added, removed, deleted, unknown, ignored, clean = matchpats(ui, repo, [p], opts) + redo = False + for f in unknown: + promptadd(ui, repo, f) + redo = True + for f in deleted: + promptremove(ui, repo, f) + redo = True + if redo: + modified, added, removed, deleted, unknown, ignored, clean = matchpats(ui, repo, [p], opts) + for f in modified + added + removed: + if f in taken: + ui.warn("warning: %s already in CL %s\n" % (f, taken[f].name)) + if not modified and not added and not removed: + ui.warn("warning: %s did not match any modified files\n" % (p,)) + + # Again, all at once (eliminates duplicates) + modified, added, removed = matchpats(ui, repo, pats, opts)[:3] + l = modified + added + removed + l.sort() + if taken: + l = Sub(l, taken.keys()) + return l + +# Return list of changed files in repository that match pats and still exist. +def ChangedExistingFiles(ui, repo, pats, opts): + modified, added = matchpats(ui, repo, pats, opts)[:2] + l = modified + added + l.sort() + return l + +# Return list of files claimed by existing CLs +def Taken(ui, repo): + all = LoadAllCL(ui, repo, web=False) + taken = {} + for _, cl in all.items(): + for f in cl.files: + taken[f] = cl + return taken + +# Return list of changed files that are not claimed by other CLs +def DefaultFiles(ui, repo, pats, opts): + return ChangedFiles(ui, repo, pats, opts, taken=Taken(ui, repo)) + +def Sub(l1, l2): + return [l for l in l1 if l not in l2] + +def Add(l1, l2): + l = l1 + Sub(l2, l1) + l.sort() + return l + +def Intersect(l1, l2): + return [l for l in l1 if l in l2] + +def getremote(ui, repo, opts): + # save $http_proxy; creating the HTTP repo object will + # delete it in an attempt to "help" + proxy = os.environ.get('http_proxy') + source = hg.parseurl(ui.expandpath("default"), None)[0] + try: + remoteui = hg.remoteui # hg 1.6 + except: + remoteui = cmdutil.remoteui + other = hg.repository(remoteui(repo, opts), source) + if proxy is not None: + os.environ['http_proxy'] = proxy + return other + +def Incoming(ui, repo, opts): + _, incoming, _ = findcommonincoming(repo, getremote(ui, repo, opts)) + return incoming + +desc_re = '^(.+: |(tag )?(release|weekly)\.|fix build|undo CL)' + +desc_msg = '''Your CL description appears not to use the standard form. + +The first line of your change description is conventionally a +one-line summary of the change, prefixed by the primary affected package, +and is used as the subject for code review mail; the rest of the description +elaborates. + +Examples: + + encoding/rot13: new package + + math: add IsInf, IsNaN + + net: fix cname in LookupHost + + unicode: update to Unicode 5.0.2 + +''' + + +def promptremove(ui, repo, f): + if promptyesno(ui, "hg remove %s (y/n)?" % (f,)): + if commands.remove(ui, repo, 'path:'+f) != 0: + ui.warn("error removing %s" % (f,)) + +def promptadd(ui, repo, f): + if promptyesno(ui, "hg add %s (y/n)?" % (f,)): + if commands.add(ui, repo, 'path:'+f) != 0: + ui.warn("error adding %s" % (f,)) + +def EditCL(ui, repo, cl): + set_status(None) # do not show status + s = cl.EditorText() + while True: + s = ui.edit(s, ui.username()) + clx, line, err = ParseCL(s, cl.name) + if err != '': + if not promptyesno(ui, "error parsing change list: line %d: %s\nre-edit (y/n)?" % (line, err)): + return "change list not modified" + continue + + # Check description. + if clx.desc == '': + if promptyesno(ui, "change list should have a description\nre-edit (y/n)?"): + continue + elif re.search('', clx.desc): + if promptyesno(ui, "change list description omits reason for undo\nre-edit (y/n)?"): + continue + elif not re.match(desc_re, clx.desc.split('\n')[0]): + if promptyesno(ui, desc_msg + "re-edit (y/n)?"): + continue + + # Check file list for files that need to be hg added or hg removed + # or simply aren't understood. + pats = ['path:'+f for f in clx.files] + modified, added, removed, deleted, unknown, ignored, clean = matchpats(ui, repo, pats, {}) + files = [] + for f in clx.files: + if f in modified or f in added or f in removed: + files.append(f) + continue + if f in deleted: + promptremove(ui, repo, f) + files.append(f) + continue + if f in unknown: + promptadd(ui, repo, f) + files.append(f) + continue + if f in ignored: + ui.warn("error: %s is excluded by .hgignore; omitting\n" % (f,)) + continue + if f in clean: + ui.warn("warning: %s is listed in the CL but unchanged\n" % (f,)) + files.append(f) + continue + p = repo.root + '/' + f + if os.path.isfile(p): + ui.warn("warning: %s is a file but not known to hg\n" % (f,)) + files.append(f) + continue + if os.path.isdir(p): + ui.warn("error: %s is a directory, not a file; omitting\n" % (f,)) + continue + ui.warn("error: %s does not exist; omitting\n" % (f,)) + clx.files = files + + cl.desc = clx.desc + cl.reviewer = clx.reviewer + cl.cc = clx.cc + cl.files = clx.files + cl.private = clx.private + break + return "" + +# For use by submit, etc. (NOT by change) +# Get change list number or list of files from command line. +# If files are given, make a new change list. +def CommandLineCL(ui, repo, pats, opts, defaultcc=None): + if len(pats) > 0 and GoodCLName(pats[0]): + if len(pats) != 1: + return None, "cannot specify change number and file names" + if opts.get('message'): + return None, "cannot use -m with existing CL" + cl, err = LoadCL(ui, repo, pats[0], web=True) + if err != "": + return None, err + else: + cl = CL("new") + cl.local = True + cl.files = ChangedFiles(ui, repo, pats, opts, taken=Taken(ui, repo)) + if not cl.files: + return None, "no files changed" + if opts.get('reviewer'): + cl.reviewer = Add(cl.reviewer, SplitCommaSpace(opts.get('reviewer'))) + if opts.get('cc'): + cl.cc = Add(cl.cc, SplitCommaSpace(opts.get('cc'))) + if defaultcc: + cl.cc = Add(cl.cc, defaultcc) + if cl.name == "new": + if opts.get('message'): + cl.desc = opts.get('message') + else: + err = EditCL(ui, repo, cl) + if err != '': + return None, err + return cl, "" + +# reposetup replaces cmdutil.match with this wrapper, +# which expands the syntax @clnumber to mean the files +# in that CL. +original_match = None +def ReplacementForCmdutilMatch(repo, pats=None, opts=None, globbed=False, default='relpath'): + taken = [] + files = [] + pats = pats or [] + opts = opts or {} + for p in pats: + if p.startswith('@'): + taken.append(p) + clname = p[1:] + if not GoodCLName(clname): + raise util.Abort("invalid CL name " + clname) + cl, err = LoadCL(repo.ui, repo, clname, web=False) + if err != '': + raise util.Abort("loading CL " + clname + ": " + err) + if not cl.files: + raise util.Abort("no files in CL " + clname) + files = Add(files, cl.files) + pats = Sub(pats, taken) + ['path:'+f for f in files] + + # work-around for http://selenic.com/hg/rev/785bbc8634f8 + if hgversion >= '1.9' and not hasattr(repo, 'match'): + repo = repo[None] + + return original_match(repo, pats=pats, opts=opts, globbed=globbed, default=default) + +def RelativePath(path, cwd): + n = len(cwd) + if path.startswith(cwd) and path[n] == '/': + return path[n+1:] + return path + +def CheckFormat(ui, repo, files, just_warn=False): + set_status("running gofmt") + CheckGofmt(ui, repo, files, just_warn) + CheckTabfmt(ui, repo, files, just_warn) + +# Check that gofmt run on the list of files does not change them +def CheckGofmt(ui, repo, files, just_warn): + files = [f for f in files if (f.startswith('src/') or f.startswith('test/bench/')) and f.endswith('.go')] + if not files: + return + cwd = os.getcwd() + files = [RelativePath(repo.root + '/' + f, cwd) for f in files] + files = [f for f in files if os.access(f, 0)] + if not files: + return + try: + cmd = subprocess.Popen(["gofmt", "-l"] + files, shell=False, stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE, close_fds=sys.platform != "win32") + cmd.stdin.close() + except: + raise util.Abort("gofmt: " + ExceptionDetail()) + data = cmd.stdout.read() + errors = cmd.stderr.read() + cmd.wait() + set_status("done with gofmt") + if len(errors) > 0: + ui.warn("gofmt errors:\n" + errors.rstrip() + "\n") + return + if len(data) > 0: + msg = "gofmt needs to format these files (run hg gofmt):\n" + Indent(data, "\t").rstrip() + if just_warn: + ui.warn("warning: " + msg + "\n") + else: + raise util.Abort(msg) + return + +# Check that *.[chys] files indent using tabs. +def CheckTabfmt(ui, repo, files, just_warn): + files = [f for f in files if f.startswith('src/') and re.search(r"\.[chys]$", f)] + if not files: + return + cwd = os.getcwd() + files = [RelativePath(repo.root + '/' + f, cwd) for f in files] + files = [f for f in files if os.access(f, 0)] + badfiles = [] + for f in files: + try: + for line in open(f, 'r'): + # Four leading spaces is enough to complain about, + # except that some Plan 9 code uses four spaces as the label indent, + # so allow that. + if line.startswith(' ') and not re.match(' [A-Za-z0-9_]+:', line): + badfiles.append(f) + break + except: + # ignore cannot open file, etc. + pass + if len(badfiles) > 0: + msg = "these files use spaces for indentation (use tabs instead):\n\t" + "\n\t".join(badfiles) + if just_warn: + ui.warn("warning: " + msg + "\n") + else: + raise util.Abort(msg) + return + +####################################################################### +# Mercurial commands + +# every command must take a ui and and repo as arguments. +# opts is a dict where you can find other command line flags +# +# Other parameters are taken in order from items on the command line that +# don't start with a dash. If no default value is given in the parameter list, +# they are required. +# + +def change(ui, repo, *pats, **opts): + """create, edit or delete a change list + + Create, edit or delete a change list. + A change list is a group of files to be reviewed and submitted together, + plus a textual description of the change. + Change lists are referred to by simple alphanumeric names. + + Changes must be reviewed before they can be submitted. + + In the absence of options, the change command opens the + change list for editing in the default editor. + + Deleting a change with the -d or -D flag does not affect + the contents of the files listed in that change. To revert + the files listed in a change, use + + hg revert @123456 + + before running hg change -d 123456. + """ + + if missing_codereview: + return missing_codereview + + dirty = {} + if len(pats) > 0 and GoodCLName(pats[0]): + name = pats[0] + if len(pats) != 1: + return "cannot specify CL name and file patterns" + pats = pats[1:] + cl, err = LoadCL(ui, repo, name, web=True) + if err != '': + return err + if not cl.local and (opts["stdin"] or not opts["stdout"]): + return "cannot change non-local CL " + name + else: + if repo[None].branch() != "default": + return "cannot run hg change outside default branch" + name = "new" + cl = CL("new") + dirty[cl] = True + files = ChangedFiles(ui, repo, pats, opts, taken=Taken(ui, repo)) + + if opts["delete"] or opts["deletelocal"]: + if opts["delete"] and opts["deletelocal"]: + return "cannot use -d and -D together" + flag = "-d" + if opts["deletelocal"]: + flag = "-D" + if name == "new": + return "cannot use "+flag+" with file patterns" + if opts["stdin"] or opts["stdout"]: + return "cannot use "+flag+" with -i or -o" + if not cl.local: + return "cannot change non-local CL " + name + if opts["delete"]: + if cl.copied_from: + return "original author must delete CL; hg change -D will remove locally" + PostMessage(ui, cl.name, "*** Abandoned ***", send_mail=cl.mailed) + EditDesc(cl.name, closed=True, private=cl.private) + cl.Delete(ui, repo) + return + + if opts["stdin"]: + s = sys.stdin.read() + clx, line, err = ParseCL(s, name) + if err != '': + return "error parsing change list: line %d: %s" % (line, err) + if clx.desc is not None: + cl.desc = clx.desc; + dirty[cl] = True + if clx.reviewer is not None: + cl.reviewer = clx.reviewer + dirty[cl] = True + if clx.cc is not None: + cl.cc = clx.cc + dirty[cl] = True + if clx.files is not None: + cl.files = clx.files + dirty[cl] = True + if clx.private != cl.private: + cl.private = clx.private + dirty[cl] = True + + if not opts["stdin"] and not opts["stdout"]: + if name == "new": + cl.files = files + err = EditCL(ui, repo, cl) + if err != "": + return err + dirty[cl] = True + + for d, _ in dirty.items(): + name = d.name + d.Flush(ui, repo) + if name == "new": + d.Upload(ui, repo, quiet=True) + + if opts["stdout"]: + ui.write(cl.EditorText()) + elif opts["pending"]: + ui.write(cl.PendingText()) + elif name == "new": + if ui.quiet: + ui.write(cl.name) + else: + ui.write("CL created: " + cl.url + "\n") + return + +def code_login(ui, repo, **opts): + """log in to code review server + + Logs in to the code review server, saving a cookie in + a file in your home directory. + """ + if missing_codereview: + return missing_codereview + + MySend(None) + +def clpatch(ui, repo, clname, **opts): + """import a patch from the code review server + + Imports a patch from the code review server into the local client. + If the local client has already modified any of the files that the + patch modifies, this command will refuse to apply the patch. + + Submitting an imported patch will keep the original author's + name as the Author: line but add your own name to a Committer: line. + """ + if repo[None].branch() != "default": + return "cannot run hg clpatch outside default branch" + return clpatch_or_undo(ui, repo, clname, opts, mode="clpatch") + +def undo(ui, repo, clname, **opts): + """undo the effect of a CL + + Creates a new CL that undoes an earlier CL. + After creating the CL, opens the CL text for editing so that + you can add the reason for the undo to the description. + """ + if repo[None].branch() != "default": + return "cannot run hg undo outside default branch" + return clpatch_or_undo(ui, repo, clname, opts, mode="undo") + +def release_apply(ui, repo, clname, **opts): + """apply a CL to the release branch + + Creates a new CL copying a previously committed change + from the main branch to the release branch. + The current client must either be clean or already be in + the release branch. + + The release branch must be created by starting with a + clean client, disabling the code review plugin, and running: + + hg update weekly.YYYY-MM-DD + hg branch release-branch.rNN + hg commit -m 'create release-branch.rNN' + hg push --new-branch + + Then re-enable the code review plugin. + + People can test the release branch by running + + hg update release-branch.rNN + + in a clean client. To return to the normal tree, + + hg update default + + Move changes since the weekly into the release branch + using hg release-apply followed by the usual code review + process and hg submit. + + When it comes time to tag the release, record the + final long-form tag of the release-branch.rNN + in the *default* branch's .hgtags file. That is, run + + hg update default + + and then edit .hgtags as you would for a weekly. + + """ + c = repo[None] + if not releaseBranch: + return "no active release branches" + if c.branch() != releaseBranch: + if c.modified() or c.added() or c.removed(): + raise util.Abort("uncommitted local changes - cannot switch branches") + err = hg.clean(repo, releaseBranch) + if err: + return err + try: + err = clpatch_or_undo(ui, repo, clname, opts, mode="backport") + if err: + raise util.Abort(err) + except Exception, e: + hg.clean(repo, "default") + raise e + return None + +def rev2clname(rev): + # Extract CL name from revision description. + # The last line in the description that is a codereview URL is the real one. + # Earlier lines might be part of the user-written description. + all = re.findall('(?m)^http://codereview.appspot.com/([0-9]+)$', rev.description()) + if len(all) > 0: + return all[-1] + return "" + +undoHeader = """undo CL %s / %s + + + +««« original CL description +""" + +undoFooter = """ +»»» +""" + +backportHeader = """[%s] %s + +««« CL %s / %s +""" + +backportFooter = """ +»»» +""" + +# Implementation of clpatch/undo. +def clpatch_or_undo(ui, repo, clname, opts, mode): + if missing_codereview: + return missing_codereview + + if mode == "undo" or mode == "backport": + if hgversion < '1.4': + # Don't have cmdutil.match (see implementation of sync command). + return "hg is too old to run hg %s - update to 1.4 or newer" % mode + + # Find revision in Mercurial repository. + # Assume CL number is 7+ decimal digits. + # Otherwise is either change log sequence number (fewer decimal digits), + # hexadecimal hash, or tag name. + # Mercurial will fall over long before the change log + # sequence numbers get to be 7 digits long. + if re.match('^[0-9]{7,}$', clname): + found = False + matchfn = scmutil.match(repo, [], {'rev': None}) + def prep(ctx, fns): + pass + for ctx in cmdutil.walkchangerevs(repo, matchfn, {'rev': None}, prep): + rev = repo[ctx.rev()] + # Last line with a code review URL is the actual review URL. + # Earlier ones might be part of the CL description. + n = rev2clname(rev) + if n == clname: + found = True + break + if not found: + return "cannot find CL %s in local repository" % clname + else: + rev = repo[clname] + if not rev: + return "unknown revision %s" % clname + clname = rev2clname(rev) + if clname == "": + return "cannot find CL name in revision description" + + # Create fresh CL and start with patch that would reverse the change. + vers = short(rev.node()) + cl = CL("new") + desc = str(rev.description()) + if mode == "undo": + cl.desc = (undoHeader % (clname, vers)) + desc + undoFooter + else: + cl.desc = (backportHeader % (releaseBranch, line1(desc), clname, vers)) + desc + undoFooter + v1 = vers + v0 = short(rev.parents()[0].node()) + if mode == "undo": + arg = v1 + ":" + v0 + else: + vers = v0 + arg = v0 + ":" + v1 + patch = RunShell(["hg", "diff", "--git", "-r", arg]) + + else: # clpatch + cl, vers, patch, err = DownloadCL(ui, repo, clname) + if err != "": + return err + if patch == emptydiff: + return "codereview issue %s has no diff" % clname + + # find current hg version (hg identify) + ctx = repo[None] + parents = ctx.parents() + id = '+'.join([short(p.node()) for p in parents]) + + # if version does not match the patch version, + # try to update the patch line numbers. + if vers != "" and id != vers: + # "vers in repo" gives the wrong answer + # on some versions of Mercurial. Instead, do the actual + # lookup and catch the exception. + try: + repo[vers].description() + except: + return "local repository is out of date; sync to get %s" % (vers) + patch1, err = portPatch(repo, patch, vers, id) + if err != "": + if not opts["ignore_hgpatch_failure"]: + return "codereview issue %s is out of date: %s (%s->%s)" % (clname, err, vers, id) + else: + patch = patch1 + argv = ["hgpatch"] + if opts["no_incoming"] or mode == "backport": + argv += ["--checksync=false"] + try: + cmd = subprocess.Popen(argv, shell=False, stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=None, close_fds=sys.platform != "win32") + except: + return "hgpatch: " + ExceptionDetail() + + out, err = cmd.communicate(patch) + if cmd.returncode != 0 and not opts["ignore_hgpatch_failure"]: + return "hgpatch failed" + cl.local = True + cl.files = out.strip().split() + if not cl.files and not opts["ignore_hgpatch_failure"]: + return "codereview issue %s has no changed files" % clname + files = ChangedFiles(ui, repo, [], opts) + extra = Sub(cl.files, files) + if extra: + ui.warn("warning: these files were listed in the patch but not changed:\n\t" + "\n\t".join(extra) + "\n") + cl.Flush(ui, repo) + if mode == "undo": + err = EditCL(ui, repo, cl) + if err != "": + return "CL created, but error editing: " + err + cl.Flush(ui, repo) + else: + ui.write(cl.PendingText() + "\n") + +# portPatch rewrites patch from being a patch against +# oldver to being a patch against newver. +def portPatch(repo, patch, oldver, newver): + lines = patch.splitlines(True) # True = keep \n + delta = None + for i in range(len(lines)): + line = lines[i] + if line.startswith('--- a/'): + file = line[6:-1] + delta = fileDeltas(repo, file, oldver, newver) + if not delta or not line.startswith('@@ '): + continue + # @@ -x,y +z,w @@ means the patch chunk replaces + # the original file's line numbers x up to x+y with the + # line numbers z up to z+w in the new file. + # Find the delta from x in the original to the same + # line in the current version and add that delta to both + # x and z. + m = re.match('@@ -([0-9]+),([0-9]+) \+([0-9]+),([0-9]+) @@', line) + if not m: + return None, "error parsing patch line numbers" + n1, len1, n2, len2 = int(m.group(1)), int(m.group(2)), int(m.group(3)), int(m.group(4)) + d, err = lineDelta(delta, n1, len1) + if err != "": + return "", err + n1 += d + n2 += d + lines[i] = "@@ -%d,%d +%d,%d @@\n" % (n1, len1, n2, len2) + + newpatch = ''.join(lines) + return newpatch, "" + +# fileDelta returns the line number deltas for the given file's +# changes from oldver to newver. +# The deltas are a list of (n, len, newdelta) triples that say +# lines [n, n+len) were modified, and after that range the +# line numbers are +newdelta from what they were before. +def fileDeltas(repo, file, oldver, newver): + cmd = ["hg", "diff", "--git", "-r", oldver + ":" + newver, "path:" + file] + data = RunShell(cmd, silent_ok=True) + deltas = [] + for line in data.splitlines(): + m = re.match('@@ -([0-9]+),([0-9]+) \+([0-9]+),([0-9]+) @@', line) + if not m: + continue + n1, len1, n2, len2 = int(m.group(1)), int(m.group(2)), int(m.group(3)), int(m.group(4)) + deltas.append((n1, len1, n2+len2-(n1+len1))) + return deltas + +# lineDelta finds the appropriate line number delta to apply to the lines [n, n+len). +# It returns an error if those lines were rewritten by the patch. +def lineDelta(deltas, n, len): + d = 0 + for (old, oldlen, newdelta) in deltas: + if old >= n+len: + break + if old+len > n: + return 0, "patch and recent changes conflict" + d = newdelta + return d, "" + +def download(ui, repo, clname, **opts): + """download a change from the code review server + + Download prints a description of the given change list + followed by its diff, downloaded from the code review server. + """ + if missing_codereview: + return missing_codereview + + cl, vers, patch, err = DownloadCL(ui, repo, clname) + if err != "": + return err + ui.write(cl.EditorText() + "\n") + ui.write(patch + "\n") + return + +def file(ui, repo, clname, pat, *pats, **opts): + """assign files to or remove files from a change list + + Assign files to or (with -d) remove files from a change list. + + The -d option only removes files from the change list. + It does not edit them or remove them from the repository. + """ + if missing_codereview: + return missing_codereview + + pats = tuple([pat] + list(pats)) + if not GoodCLName(clname): + return "invalid CL name " + clname + + dirty = {} + cl, err = LoadCL(ui, repo, clname, web=False) + if err != '': + return err + if not cl.local: + return "cannot change non-local CL " + clname + + files = ChangedFiles(ui, repo, pats, opts) + + if opts["delete"]: + oldfiles = Intersect(files, cl.files) + if oldfiles: + if not ui.quiet: + ui.status("# Removing files from CL. To undo:\n") + ui.status("# cd %s\n" % (repo.root)) + for f in oldfiles: + ui.status("# hg file %s %s\n" % (cl.name, f)) + cl.files = Sub(cl.files, oldfiles) + cl.Flush(ui, repo) + else: + ui.status("no such files in CL") + return + + if not files: + return "no such modified files" + + files = Sub(files, cl.files) + taken = Taken(ui, repo) + warned = False + for f in files: + if f in taken: + if not warned and not ui.quiet: + ui.status("# Taking files from other CLs. To undo:\n") + ui.status("# cd %s\n" % (repo.root)) + warned = True + ocl = taken[f] + if not ui.quiet: + ui.status("# hg file %s %s\n" % (ocl.name, f)) + if ocl not in dirty: + ocl.files = Sub(ocl.files, files) + dirty[ocl] = True + cl.files = Add(cl.files, files) + dirty[cl] = True + for d, _ in dirty.items(): + d.Flush(ui, repo) + return + +def gofmt(ui, repo, *pats, **opts): + """apply gofmt to modified files + + Applies gofmt to the modified files in the repository that match + the given patterns. + """ + if missing_codereview: + return missing_codereview + + files = ChangedExistingFiles(ui, repo, pats, opts) + files = [f for f in files if f.endswith(".go")] + if not files: + return "no modified go files" + cwd = os.getcwd() + files = [RelativePath(repo.root + '/' + f, cwd) for f in files] + try: + cmd = ["gofmt", "-l"] + if not opts["list"]: + cmd += ["-w"] + if os.spawnvp(os.P_WAIT, "gofmt", cmd + files) != 0: + raise util.Abort("gofmt did not exit cleanly") + except error.Abort, e: + raise + except: + raise util.Abort("gofmt: " + ExceptionDetail()) + return + +def mail(ui, repo, *pats, **opts): + """mail a change for review + + Uploads a patch to the code review server and then sends mail + to the reviewer and CC list asking for a review. + """ + if missing_codereview: + return missing_codereview + + cl, err = CommandLineCL(ui, repo, pats, opts, defaultcc=defaultcc) + if err != "": + return err + cl.Upload(ui, repo, gofmt_just_warn=True) + if not cl.reviewer: + # If no reviewer is listed, assign the review to defaultcc. + # This makes sure that it appears in the + # codereview.appspot.com/user/defaultcc + # page, so that it doesn't get dropped on the floor. + if not defaultcc: + return "no reviewers listed in CL" + cl.cc = Sub(cl.cc, defaultcc) + cl.reviewer = defaultcc + cl.Flush(ui, repo) + + if cl.files == []: + return "no changed files, not sending mail" + + cl.Mail(ui, repo) + +def pending(ui, repo, *pats, **opts): + """show pending changes + + Lists pending changes followed by a list of unassigned but modified files. + """ + if missing_codereview: + return missing_codereview + + m = LoadAllCL(ui, repo, web=True) + names = m.keys() + names.sort() + for name in names: + cl = m[name] + ui.write(cl.PendingText() + "\n") + + files = DefaultFiles(ui, repo, [], opts) + if len(files) > 0: + s = "Changed files not in any CL:\n" + for f in files: + s += "\t" + f + "\n" + ui.write(s) + +def reposetup(ui, repo): + global original_match + if original_match is None: + start_status_thread() + original_match = scmutil.match + scmutil.match = ReplacementForCmdutilMatch + RietveldSetup(ui, repo) + +def CheckContributor(ui, repo, user=None): + set_status("checking CONTRIBUTORS file") + user, userline = FindContributor(ui, repo, user, warn=False) + if not userline: + raise util.Abort("cannot find %s in CONTRIBUTORS" % (user,)) + return userline + +def FindContributor(ui, repo, user=None, warn=True): + if not user: + user = ui.config("ui", "username") + if not user: + raise util.Abort("[ui] username is not configured in .hgrc") + user = user.lower() + m = re.match(r".*<(.*)>", user) + if m: + user = m.group(1) + + if user not in contributors: + if warn: + ui.warn("warning: cannot find %s in CONTRIBUTORS\n" % (user,)) + return user, None + + user, email = contributors[user] + return email, "%s <%s>" % (user, email) + +def submit(ui, repo, *pats, **opts): + """submit change to remote repository + + Submits change to remote repository. + Bails out if the local repository is not in sync with the remote one. + """ + if missing_codereview: + return missing_codereview + + # We already called this on startup but sometimes Mercurial forgets. + set_mercurial_encoding_to_utf8() + + repo.ui.quiet = True + if not opts["no_incoming"] and Incoming(ui, repo, opts): + return "local repository out of date; must sync before submit" + + cl, err = CommandLineCL(ui, repo, pats, opts, defaultcc=defaultcc) + if err != "": + return err + + user = None + if cl.copied_from: + user = cl.copied_from + userline = CheckContributor(ui, repo, user) + typecheck(userline, str) + + about = "" + if cl.reviewer: + about += "R=" + JoinComma([CutDomain(s) for s in cl.reviewer]) + "\n" + if opts.get('tbr'): + tbr = SplitCommaSpace(opts.get('tbr')) + cl.reviewer = Add(cl.reviewer, tbr) + about += "TBR=" + JoinComma([CutDomain(s) for s in tbr]) + "\n" + if cl.cc: + about += "CC=" + JoinComma([CutDomain(s) for s in cl.cc]) + "\n" + + if not cl.reviewer: + return "no reviewers listed in CL" + + if not cl.local: + return "cannot submit non-local CL" + + # upload, to sync current patch and also get change number if CL is new. + if not cl.copied_from: + cl.Upload(ui, repo, gofmt_just_warn=True) + + # check gofmt for real; allowed upload to warn in order to save CL. + cl.Flush(ui, repo) + CheckFormat(ui, repo, cl.files) + + about += "%s%s\n" % (server_url_base, cl.name) + + if cl.copied_from: + about += "\nCommitter: " + CheckContributor(ui, repo, None) + "\n" + typecheck(about, str) + + if not cl.mailed and not cl.copied_from: # in case this is TBR + cl.Mail(ui, repo) + + # submit changes locally + date = opts.get('date') + if date: + opts['date'] = util.parsedate(date) + typecheck(opts['date'], str) + opts['message'] = cl.desc.rstrip() + "\n\n" + about + typecheck(opts['message'], str) + + if opts['dryrun']: + print "NOT SUBMITTING:" + print "User: ", userline + print "Message:" + print Indent(opts['message'], "\t") + print "Files:" + print Indent('\n'.join(cl.files), "\t") + return "dry run; not submitted" + + set_status("pushing " + cl.name + " to remote server") + m = match.exact(repo.root, repo.getcwd(), cl.files) + node = repo.commit(ustr(opts['message']), ustr(userline), opts.get('date'), m) + if not node: + return "nothing changed" + + # push to remote; if it fails for any reason, roll back + try: + log = repo.changelog + rev = log.rev(node) + parents = log.parentrevs(rev) + if (rev-1 not in parents and + (parents == (nullrev, nullrev) or + len(log.heads(log.node(parents[0]))) > 1 and + (parents[1] == nullrev or len(log.heads(log.node(parents[1]))) > 1))): + # created new head + raise util.Abort("local repository out of date; must sync before submit") + + # push changes to remote. + # if it works, we're committed. + # if not, roll back + other = getremote(ui, repo, opts) + r = repo.push(other, False, None) + if r == 0: + raise util.Abort("local repository out of date; must sync before submit") + except: + real_rollback() + raise + + # we're committed. upload final patch, close review, add commit message + changeURL = short(node) + url = other.url() + m = re.match("^https?://([^@/]+@)?([^.]+)\.googlecode\.com/hg/?", url) + if m: + changeURL = "http://code.google.com/p/%s/source/detail?r=%s" % (m.group(2), changeURL) + else: + print >>sys.stderr, "URL: ", url + pmsg = "*** Submitted as " + changeURL + " ***\n\n" + opts['message'] + + # When posting, move reviewers to CC line, + # so that the issue stops showing up in their "My Issues" page. + PostMessage(ui, cl.name, pmsg, reviewers="", cc=JoinComma(cl.reviewer+cl.cc)) + + if not cl.copied_from: + EditDesc(cl.name, closed=True, private=cl.private) + cl.Delete(ui, repo) + + c = repo[None] + if c.branch() == releaseBranch and not c.modified() and not c.added() and not c.removed(): + ui.write("switching from %s to default branch.\n" % releaseBranch) + err = hg.clean(repo, "default") + if err: + return err + return None + +def sync(ui, repo, **opts): + """synchronize with remote repository + + Incorporates recent changes from the remote repository + into the local repository. + """ + if missing_codereview: + return missing_codereview + + if not opts["local"]: + ui.status = sync_note + ui.note = sync_note + other = getremote(ui, repo, opts) + modheads = repo.pull(other) + err = commands.postincoming(ui, repo, modheads, True, "tip") + if err: + return err + commands.update(ui, repo, rev="default") + sync_changes(ui, repo) + +def sync_note(msg): + # we run sync (pull -u) in verbose mode to get the + # list of files being updated, but that drags along + # a bunch of messages we don't care about. + # omit them. + if msg == 'resolving manifests\n': + return + if msg == 'searching for changes\n': + return + if msg == "couldn't find merge tool hgmerge\n": + return + sys.stdout.write(msg) + +def sync_changes(ui, repo): + # Look through recent change log descriptions to find + # potential references to http://.*/our-CL-number. + # Double-check them by looking at the Rietveld log. + def Rev(rev): + desc = repo[rev].description().strip() + for clname in re.findall('(?m)^http://(?:[^\n]+)/([0-9]+)$', desc): + if IsLocalCL(ui, repo, clname) and IsRietveldSubmitted(ui, clname, repo[rev].hex()): + ui.warn("CL %s submitted as %s; closing\n" % (clname, repo[rev])) + cl, err = LoadCL(ui, repo, clname, web=False) + if err != "": + ui.warn("loading CL %s: %s\n" % (clname, err)) + continue + if not cl.copied_from: + EditDesc(cl.name, closed=True, private=cl.private) + cl.Delete(ui, repo) + + if hgversion < '1.4': + get = util.cachefunc(lambda r: repo[r].changeset()) + changeiter, matchfn = cmdutil.walkchangerevs(ui, repo, [], get, {'rev': None}) + n = 0 + for st, rev, fns in changeiter: + if st != 'iter': + continue + n += 1 + if n > 100: + break + Rev(rev) + else: + matchfn = scmutil.match(repo, [], {'rev': None}) + def prep(ctx, fns): + pass + for ctx in cmdutil.walkchangerevs(repo, matchfn, {'rev': None}, prep): + Rev(ctx.rev()) + + # Remove files that are not modified from the CLs in which they appear. + all = LoadAllCL(ui, repo, web=False) + changed = ChangedFiles(ui, repo, [], {}) + for _, cl in all.items(): + extra = Sub(cl.files, changed) + if extra: + ui.warn("Removing unmodified files from CL %s:\n" % (cl.name,)) + for f in extra: + ui.warn("\t%s\n" % (f,)) + cl.files = Sub(cl.files, extra) + cl.Flush(ui, repo) + if not cl.files: + if not cl.copied_from: + ui.warn("CL %s has no files; delete (abandon) with hg change -d %s\n" % (cl.name, cl.name)) + else: + ui.warn("CL %s has no files; delete locally with hg change -D %s\n" % (cl.name, cl.name)) + return + +def upload(ui, repo, name, **opts): + """upload diffs to the code review server + + Uploads the current modifications for a given change to the server. + """ + if missing_codereview: + return missing_codereview + + repo.ui.quiet = True + cl, err = LoadCL(ui, repo, name, web=True) + if err != "": + return err + if not cl.local: + return "cannot upload non-local change" + cl.Upload(ui, repo) + print "%s%s\n" % (server_url_base, cl.name) + return + +review_opts = [ + ('r', 'reviewer', '', 'add reviewer'), + ('', 'cc', '', 'add cc'), + ('', 'tbr', '', 'add future reviewer'), + ('m', 'message', '', 'change description (for new change)'), +] + +cmdtable = { + # The ^ means to show this command in the help text that + # is printed when running hg with no arguments. + "^change": ( + change, + [ + ('d', 'delete', None, 'delete existing change list'), + ('D', 'deletelocal', None, 'delete locally, but do not change CL on server'), + ('i', 'stdin', None, 'read change list from standard input'), + ('o', 'stdout', None, 'print change list to standard output'), + ('p', 'pending', None, 'print pending summary to standard output'), + ], + "[-d | -D] [-i] [-o] change# or FILE ..." + ), + "^clpatch": ( + clpatch, + [ + ('', 'ignore_hgpatch_failure', None, 'create CL metadata even if hgpatch fails'), + ('', 'no_incoming', None, 'disable check for incoming changes'), + ], + "change#" + ), + # Would prefer to call this codereview-login, but then + # hg help codereview prints the help for this command + # instead of the help for the extension. + "code-login": ( + code_login, + [], + "", + ), + "^download": ( + download, + [], + "change#" + ), + "^file": ( + file, + [ + ('d', 'delete', None, 'delete files from change list (but not repository)'), + ], + "[-d] change# FILE ..." + ), + "^gofmt": ( + gofmt, + [ + ('l', 'list', None, 'list files that would change, but do not edit them'), + ], + "FILE ..." + ), + "^pending|p": ( + pending, + [], + "[FILE ...]" + ), + "^mail": ( + mail, + review_opts + [ + ] + commands.walkopts, + "[-r reviewer] [--cc cc] [change# | file ...]" + ), + "^release-apply": ( + release_apply, + [ + ('', 'ignore_hgpatch_failure', None, 'create CL metadata even if hgpatch fails'), + ('', 'no_incoming', None, 'disable check for incoming changes'), + ], + "change#" + ), + # TODO: release-start, release-tag, weekly-tag + "^submit": ( + submit, + review_opts + [ + ('', 'no_incoming', None, 'disable initial incoming check (for testing)'), + ('n', 'dryrun', None, 'make change only locally (for testing)'), + ] + commands.walkopts + commands.commitopts + commands.commitopts2, + "[-r reviewer] [--cc cc] [change# | file ...]" + ), + "^sync": ( + sync, + [ + ('', 'local', None, 'do not pull changes from remote repository') + ], + "[--local]", + ), + "^undo": ( + undo, + [ + ('', 'ignore_hgpatch_failure', None, 'create CL metadata even if hgpatch fails'), + ('', 'no_incoming', None, 'disable check for incoming changes'), + ], + "change#" + ), + "^upload": ( + upload, + [], + "change#" + ), +} + + +####################################################################### +# Wrappers around upload.py for interacting with Rietveld + +# HTML form parser +class FormParser(HTMLParser): + def __init__(self): + self.map = {} + self.curtag = None + self.curdata = None + HTMLParser.__init__(self) + def handle_starttag(self, tag, attrs): + if tag == "input": + key = None + value = '' + for a in attrs: + if a[0] == 'name': + key = a[1] + if a[0] == 'value': + value = a[1] + if key is not None: + self.map[key] = value + if tag == "textarea": + key = None + for a in attrs: + if a[0] == 'name': + key = a[1] + if key is not None: + self.curtag = key + self.curdata = '' + def handle_endtag(self, tag): + if tag == "textarea" and self.curtag is not None: + self.map[self.curtag] = self.curdata + self.curtag = None + self.curdata = None + def handle_charref(self, name): + self.handle_data(unichr(int(name))) + def handle_entityref(self, name): + import htmlentitydefs + if name in htmlentitydefs.entitydefs: + self.handle_data(htmlentitydefs.entitydefs[name]) + else: + self.handle_data("&" + name + ";") + def handle_data(self, data): + if self.curdata is not None: + self.curdata += data + +def JSONGet(ui, path): + try: + data = MySend(path, force_auth=False) + typecheck(data, str) + d = fix_json(json.loads(data)) + except: + ui.warn("JSONGet %s: %s\n" % (path, ExceptionDetail())) + return None + return d + +# Clean up json parser output to match our expectations: +# * all strings are UTF-8-encoded str, not unicode. +# * missing fields are missing, not None, +# so that d.get("foo", defaultvalue) works. +def fix_json(x): + if type(x) in [str, int, float, bool, type(None)]: + pass + elif type(x) is unicode: + x = x.encode("utf-8") + elif type(x) is list: + for i in range(len(x)): + x[i] = fix_json(x[i]) + elif type(x) is dict: + todel = [] + for k in x: + if x[k] is None: + todel.append(k) + else: + x[k] = fix_json(x[k]) + for k in todel: + del x[k] + else: + raise util.Abort("unknown type " + str(type(x)) + " in fix_json") + if type(x) is str: + x = x.replace('\r\n', '\n') + return x + +def IsRietveldSubmitted(ui, clname, hex): + dict = JSONGet(ui, "/api/" + clname + "?messages=true") + if dict is None: + return False + for msg in dict.get("messages", []): + text = msg.get("text", "") + m = re.match('\*\*\* Submitted as [^*]*?([0-9a-f]+) \*\*\*', text) + if m is not None and len(m.group(1)) >= 8 and hex.startswith(m.group(1)): + return True + return False + +def IsRietveldMailed(cl): + for msg in cl.dict.get("messages", []): + if msg.get("text", "").find("I'd like you to review this change") >= 0: + return True + return False + +def DownloadCL(ui, repo, clname): + set_status("downloading CL " + clname) + cl, err = LoadCL(ui, repo, clname, web=True) + if err != "": + return None, None, None, "error loading CL %s: %s" % (clname, err) + + # Find most recent diff + diffs = cl.dict.get("patchsets", []) + if not diffs: + return None, None, None, "CL has no patch sets" + patchid = diffs[-1] + + patchset = JSONGet(ui, "/api/" + clname + "/" + str(patchid)) + if patchset is None: + return None, None, None, "error loading CL patchset %s/%d" % (clname, patchid) + if patchset.get("patchset", 0) != patchid: + return None, None, None, "malformed patchset information" + + vers = "" + msg = patchset.get("message", "").split() + if len(msg) >= 3 and msg[0] == "diff" and msg[1] == "-r": + vers = msg[2] + diff = "/download/issue" + clname + "_" + str(patchid) + ".diff" + + diffdata = MySend(diff, force_auth=False) + + # Print warning if email is not in CONTRIBUTORS file. + email = cl.dict.get("owner_email", "") + if not email: + return None, None, None, "cannot find owner for %s" % (clname) + him = FindContributor(ui, repo, email) + me = FindContributor(ui, repo, None) + if him == me: + cl.mailed = IsRietveldMailed(cl) + else: + cl.copied_from = email + + return cl, vers, diffdata, "" + +def MySend(request_path, payload=None, + content_type="application/octet-stream", + timeout=None, force_auth=True, + **kwargs): + """Run MySend1 maybe twice, because Rietveld is unreliable.""" + try: + return MySend1(request_path, payload, content_type, timeout, force_auth, **kwargs) + except Exception, e: + if type(e) != urllib2.HTTPError or e.code != 500: # only retry on HTTP 500 error + raise + print >>sys.stderr, "Loading "+request_path+": "+ExceptionDetail()+"; trying again in 2 seconds." + time.sleep(2) + return MySend1(request_path, payload, content_type, timeout, force_auth, **kwargs) + +# Like upload.py Send but only authenticates when the +# redirect is to www.google.com/accounts. This keeps +# unnecessary redirects from happening during testing. +def MySend1(request_path, payload=None, + content_type="application/octet-stream", + timeout=None, force_auth=True, + **kwargs): + """Sends an RPC and returns the response. + + Args: + request_path: The path to send the request to, eg /api/appversion/create. + payload: The body of the request, or None to send an empty request. + content_type: The Content-Type header to use. + timeout: timeout in seconds; default None i.e. no timeout. + (Note: for large requests on OS X, the timeout doesn't work right.) + kwargs: Any keyword arguments are converted into query string parameters. + + Returns: + The response body, as a string. + """ + # TODO: Don't require authentication. Let the server say + # whether it is necessary. + global rpc + if rpc == None: + rpc = GetRpcServer(upload_options) + self = rpc + if not self.authenticated and force_auth: + self._Authenticate() + if request_path is None: + return + + old_timeout = socket.getdefaulttimeout() + socket.setdefaulttimeout(timeout) + try: + tries = 0 + while True: + tries += 1 + args = dict(kwargs) + url = "http://%s%s" % (self.host, request_path) + if args: + url += "?" + urllib.urlencode(args) + req = self._CreateRequest(url=url, data=payload) + req.add_header("Content-Type", content_type) + try: + f = self.opener.open(req) + response = f.read() + f.close() + # Translate \r\n into \n, because Rietveld doesn't. + response = response.replace('\r\n', '\n') + # who knows what urllib will give us + if type(response) == unicode: + response = response.encode("utf-8") + typecheck(response, str) + return response + except urllib2.HTTPError, e: + if tries > 3: + raise + elif e.code == 401: + self._Authenticate() + elif e.code == 302: + loc = e.info()["location"] + if not loc.startswith('https://www.google.com/a') or loc.find('/ServiceLogin') < 0: + return '' + self._Authenticate() + else: + raise + finally: + socket.setdefaulttimeout(old_timeout) + +def GetForm(url): + f = FormParser() + f.feed(ustr(MySend(url))) # f.feed wants unicode + f.close() + # convert back to utf-8 to restore sanity + m = {} + for k,v in f.map.items(): + m[k.encode("utf-8")] = v.replace("\r\n", "\n").encode("utf-8") + return m + +def EditDesc(issue, subject=None, desc=None, reviewers=None, cc=None, closed=False, private=False): + set_status("uploading change to description") + form_fields = GetForm("/" + issue + "/edit") + if subject is not None: + form_fields['subject'] = subject + if desc is not None: + form_fields['description'] = desc + if reviewers is not None: + form_fields['reviewers'] = reviewers + if cc is not None: + form_fields['cc'] = cc + if closed: + form_fields['closed'] = "checked" + if private: + form_fields['private'] = "checked" + ctype, body = EncodeMultipartFormData(form_fields.items(), []) + response = MySend("/" + issue + "/edit", body, content_type=ctype) + if response != "": + print >>sys.stderr, "Error editing description:\n" + "Sent form: \n", form_fields, "\n", response + sys.exit(2) + +def PostMessage(ui, issue, message, reviewers=None, cc=None, send_mail=True, subject=None): + set_status("uploading message") + form_fields = GetForm("/" + issue + "/publish") + if reviewers is not None: + form_fields['reviewers'] = reviewers + if cc is not None: + form_fields['cc'] = cc + if send_mail: + form_fields['send_mail'] = "checked" + else: + del form_fields['send_mail'] + if subject is not None: + form_fields['subject'] = subject + form_fields['message'] = message + + form_fields['message_only'] = '1' # Don't include draft comments + if reviewers is not None or cc is not None: + form_fields['message_only'] = '' # Must set '' in order to override cc/reviewer + ctype = "applications/x-www-form-urlencoded" + body = urllib.urlencode(form_fields) + response = MySend("/" + issue + "/publish", body, content_type=ctype) + if response != "": + print response + sys.exit(2) + +class opt(object): + pass + +def nocommit(*pats, **opts): + """(disabled when using this extension)""" + raise util.Abort("codereview extension enabled; use mail, upload, or submit instead of commit") + +def nobackout(*pats, **opts): + """(disabled when using this extension)""" + raise util.Abort("codereview extension enabled; use undo instead of backout") + +def norollback(*pats, **opts): + """(disabled when using this extension)""" + raise util.Abort("codereview extension enabled; use undo instead of rollback") + +def RietveldSetup(ui, repo): + global defaultcc, upload_options, rpc, server, server_url_base, force_google_account, verbosity, contributors + global missing_codereview + + repo_config_path = '' + # Read repository-specific options from lib/codereview/codereview.cfg + try: + repo_config_path = repo.root + '/lib/codereview/codereview.cfg' + f = open(repo_config_path) + for line in f: + if line.startswith('defaultcc: '): + defaultcc = SplitCommaSpace(line[10:]) + except: + # If there are no options, chances are good this is not + # a code review repository; stop now before we foul + # things up even worse. Might also be that repo doesn't + # even have a root. See issue 959. + if repo_config_path == '': + missing_codereview = 'codereview disabled: repository has no root' + else: + missing_codereview = 'codereview disabled: cannot open ' + repo_config_path + return + + # Should only modify repository with hg submit. + # Disable the built-in Mercurial commands that might + # trip things up. + cmdutil.commit = nocommit + global real_rollback + real_rollback = repo.rollback + repo.rollback = norollback + # would install nobackout if we could; oh well + + try: + f = open(repo.root + '/CONTRIBUTORS', 'r') + except: + raise util.Abort("cannot open %s: %s" % (repo.root+'/CONTRIBUTORS', ExceptionDetail())) + for line in f: + # CONTRIBUTORS is a list of lines like: + # Person + # Person + # The first email address is the one used in commit logs. + if line.startswith('#'): + continue + m = re.match(r"([^<>]+\S)\s+(<[^<>\s]+>)((\s+<[^<>\s]+>)*)\s*$", line) + if m: + name = m.group(1) + email = m.group(2)[1:-1] + contributors[email.lower()] = (name, email) + for extra in m.group(3).split(): + contributors[extra[1:-1].lower()] = (name, email) + + if not ui.verbose: + verbosity = 0 + + # Config options. + x = ui.config("codereview", "server") + if x is not None: + server = x + + # TODO(rsc): Take from ui.username? + email = None + x = ui.config("codereview", "email") + if x is not None: + email = x + + server_url_base = "http://" + server + "/" + + testing = ui.config("codereview", "testing") + force_google_account = ui.configbool("codereview", "force_google_account", False) + + upload_options = opt() + upload_options.email = email + upload_options.host = None + upload_options.verbose = 0 + upload_options.description = None + upload_options.description_file = None + upload_options.reviewers = None + upload_options.cc = None + upload_options.message = None + upload_options.issue = None + upload_options.download_base = False + upload_options.revision = None + upload_options.send_mail = False + upload_options.vcs = None + upload_options.server = server + upload_options.save_cookies = True + + if testing: + upload_options.save_cookies = False + upload_options.email = "test@example.com" + + rpc = None + + global releaseBranch + tags = repo.branchtags().keys() + if 'release-branch.r100' in tags: + # NOTE(rsc): This tags.sort is going to get the wrong + # answer when comparing release-branch.r99 with + # release-branch.r100. If we do ten releases a year + # that gives us 4 years before we have to worry about this. + raise util.Abort('tags.sort needs to be fixed for release-branch.r100') + tags.sort() + for t in tags: + if t.startswith('release-branch.'): + releaseBranch = t + +####################################################################### +# http://codereview.appspot.com/static/upload.py, heavily edited. + +#!/usr/bin/env python +# +# Copyright 2007 Google Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Tool for uploading diffs from a version control system to the codereview app. + +Usage summary: upload.py [options] [-- diff_options] + +Diff options are passed to the diff command of the underlying system. + +Supported version control systems: + Git + Mercurial + Subversion + +It is important for Git/Mercurial users to specify a tree/node/branch to diff +against by using the '--rev' option. +""" +# This code is derived from appcfg.py in the App Engine SDK (open source), +# and from ASPN recipe #146306. + +import cookielib +import getpass +import logging +import mimetypes +import optparse +import os +import re +import socket +import subprocess +import sys +import urllib +import urllib2 +import urlparse + +# The md5 module was deprecated in Python 2.5. +try: + from hashlib import md5 +except ImportError: + from md5 import md5 + +try: + import readline +except ImportError: + pass + +# The logging verbosity: +# 0: Errors only. +# 1: Status messages. +# 2: Info logs. +# 3: Debug logs. +verbosity = 1 + +# Max size of patch or base file. +MAX_UPLOAD_SIZE = 900 * 1024 + +# whitelist for non-binary filetypes which do not start with "text/" +# .mm (Objective-C) shows up as application/x-freemind on my Linux box. +TEXT_MIMETYPES = [ + 'application/javascript', + 'application/x-javascript', + 'application/x-freemind' +] + +def GetEmail(prompt): + """Prompts the user for their email address and returns it. + + The last used email address is saved to a file and offered up as a suggestion + to the user. If the user presses enter without typing in anything the last + used email address is used. If the user enters a new address, it is saved + for next time we prompt. + + """ + last_email_file_name = os.path.expanduser("~/.last_codereview_email_address") + last_email = "" + if os.path.exists(last_email_file_name): + try: + last_email_file = open(last_email_file_name, "r") + last_email = last_email_file.readline().strip("\n") + last_email_file.close() + prompt += " [%s]" % last_email + except IOError, e: + pass + email = raw_input(prompt + ": ").strip() + if email: + try: + last_email_file = open(last_email_file_name, "w") + last_email_file.write(email) + last_email_file.close() + except IOError, e: + pass + else: + email = last_email + return email + + +def StatusUpdate(msg): + """Print a status message to stdout. + + If 'verbosity' is greater than 0, print the message. + + Args: + msg: The string to print. + """ + if verbosity > 0: + print msg + + +def ErrorExit(msg): + """Print an error message to stderr and exit.""" + print >>sys.stderr, msg + sys.exit(1) + + +class ClientLoginError(urllib2.HTTPError): + """Raised to indicate there was an error authenticating with ClientLogin.""" + + def __init__(self, url, code, msg, headers, args): + urllib2.HTTPError.__init__(self, url, code, msg, headers, None) + self.args = args + self.reason = args["Error"] + + +class AbstractRpcServer(object): + """Provides a common interface for a simple RPC server.""" + + def __init__(self, host, auth_function, host_override=None, extra_headers={}, save_cookies=False): + """Creates a new HttpRpcServer. + + Args: + host: The host to send requests to. + auth_function: A function that takes no arguments and returns an + (email, password) tuple when called. Will be called if authentication + is required. + host_override: The host header to send to the server (defaults to host). + extra_headers: A dict of extra headers to append to every request. + save_cookies: If True, save the authentication cookies to local disk. + If False, use an in-memory cookiejar instead. Subclasses must + implement this functionality. Defaults to False. + """ + self.host = host + self.host_override = host_override + self.auth_function = auth_function + self.authenticated = False + self.extra_headers = extra_headers + self.save_cookies = save_cookies + self.opener = self._GetOpener() + if self.host_override: + logging.info("Server: %s; Host: %s", self.host, self.host_override) + else: + logging.info("Server: %s", self.host) + + def _GetOpener(self): + """Returns an OpenerDirector for making HTTP requests. + + Returns: + A urllib2.OpenerDirector object. + """ + raise NotImplementedError() + + def _CreateRequest(self, url, data=None): + """Creates a new urllib request.""" + logging.debug("Creating request for: '%s' with payload:\n%s", url, data) + req = urllib2.Request(url, data=data) + if self.host_override: + req.add_header("Host", self.host_override) + for key, value in self.extra_headers.iteritems(): + req.add_header(key, value) + return req + + def _GetAuthToken(self, email, password): + """Uses ClientLogin to authenticate the user, returning an auth token. + + Args: + email: The user's email address + password: The user's password + + Raises: + ClientLoginError: If there was an error authenticating with ClientLogin. + HTTPError: If there was some other form of HTTP error. + + Returns: + The authentication token returned by ClientLogin. + """ + account_type = "GOOGLE" + if self.host.endswith(".google.com") and not force_google_account: + # Needed for use inside Google. + account_type = "HOSTED" + req = self._CreateRequest( + url="https://www.google.com/accounts/ClientLogin", + data=urllib.urlencode({ + "Email": email, + "Passwd": password, + "service": "ah", + "source": "rietveld-codereview-upload", + "accountType": account_type, + }), + ) + try: + response = self.opener.open(req) + response_body = response.read() + response_dict = dict(x.split("=") for x in response_body.split("\n") if x) + return response_dict["Auth"] + except urllib2.HTTPError, e: + if e.code == 403: + body = e.read() + response_dict = dict(x.split("=", 1) for x in body.split("\n") if x) + raise ClientLoginError(req.get_full_url(), e.code, e.msg, e.headers, response_dict) + else: + raise + + def _GetAuthCookie(self, auth_token): + """Fetches authentication cookies for an authentication token. + + Args: + auth_token: The authentication token returned by ClientLogin. + + Raises: + HTTPError: If there was an error fetching the authentication cookies. + """ + # This is a dummy value to allow us to identify when we're successful. + continue_location = "http://localhost/" + args = {"continue": continue_location, "auth": auth_token} + req = self._CreateRequest("http://%s/_ah/login?%s" % (self.host, urllib.urlencode(args))) + try: + response = self.opener.open(req) + except urllib2.HTTPError, e: + response = e + if (response.code != 302 or + response.info()["location"] != continue_location): + raise urllib2.HTTPError(req.get_full_url(), response.code, response.msg, response.headers, response.fp) + self.authenticated = True + + def _Authenticate(self): + """Authenticates the user. + + The authentication process works as follows: + 1) We get a username and password from the user + 2) We use ClientLogin to obtain an AUTH token for the user + (see http://code.google.com/apis/accounts/AuthForInstalledApps.html). + 3) We pass the auth token to /_ah/login on the server to obtain an + authentication cookie. If login was successful, it tries to redirect + us to the URL we provided. + + If we attempt to access the upload API without first obtaining an + authentication cookie, it returns a 401 response (or a 302) and + directs us to authenticate ourselves with ClientLogin. + """ + for i in range(3): + credentials = self.auth_function() + try: + auth_token = self._GetAuthToken(credentials[0], credentials[1]) + except ClientLoginError, e: + if e.reason == "BadAuthentication": + print >>sys.stderr, "Invalid username or password." + continue + if e.reason == "CaptchaRequired": + print >>sys.stderr, ( + "Please go to\n" + "https://www.google.com/accounts/DisplayUnlockCaptcha\n" + "and verify you are a human. Then try again.") + break + if e.reason == "NotVerified": + print >>sys.stderr, "Account not verified." + break + if e.reason == "TermsNotAgreed": + print >>sys.stderr, "User has not agreed to TOS." + break + if e.reason == "AccountDeleted": + print >>sys.stderr, "The user account has been deleted." + break + if e.reason == "AccountDisabled": + print >>sys.stderr, "The user account has been disabled." + break + if e.reason == "ServiceDisabled": + print >>sys.stderr, "The user's access to the service has been disabled." + break + if e.reason == "ServiceUnavailable": + print >>sys.stderr, "The service is not available; try again later." + break + raise + self._GetAuthCookie(auth_token) + return + + def Send(self, request_path, payload=None, + content_type="application/octet-stream", + timeout=None, + **kwargs): + """Sends an RPC and returns the response. + + Args: + request_path: The path to send the request to, eg /api/appversion/create. + payload: The body of the request, or None to send an empty request. + content_type: The Content-Type header to use. + timeout: timeout in seconds; default None i.e. no timeout. + (Note: for large requests on OS X, the timeout doesn't work right.) + kwargs: Any keyword arguments are converted into query string parameters. + + Returns: + The response body, as a string. + """ + # TODO: Don't require authentication. Let the server say + # whether it is necessary. + if not self.authenticated: + self._Authenticate() + + old_timeout = socket.getdefaulttimeout() + socket.setdefaulttimeout(timeout) + try: + tries = 0 + while True: + tries += 1 + args = dict(kwargs) + url = "http://%s%s" % (self.host, request_path) + if args: + url += "?" + urllib.urlencode(args) + req = self._CreateRequest(url=url, data=payload) + req.add_header("Content-Type", content_type) + try: + f = self.opener.open(req) + response = f.read() + f.close() + return response + except urllib2.HTTPError, e: + if tries > 3: + raise + elif e.code == 401 or e.code == 302: + self._Authenticate() + else: + raise + finally: + socket.setdefaulttimeout(old_timeout) + + +class HttpRpcServer(AbstractRpcServer): + """Provides a simplified RPC-style interface for HTTP requests.""" + + def _Authenticate(self): + """Save the cookie jar after authentication.""" + super(HttpRpcServer, self)._Authenticate() + if self.save_cookies: + StatusUpdate("Saving authentication cookies to %s" % self.cookie_file) + self.cookie_jar.save() + + def _GetOpener(self): + """Returns an OpenerDirector that supports cookies and ignores redirects. + + Returns: + A urllib2.OpenerDirector object. + """ + opener = urllib2.OpenerDirector() + opener.add_handler(urllib2.ProxyHandler()) + opener.add_handler(urllib2.UnknownHandler()) + opener.add_handler(urllib2.HTTPHandler()) + opener.add_handler(urllib2.HTTPDefaultErrorHandler()) + opener.add_handler(urllib2.HTTPSHandler()) + opener.add_handler(urllib2.HTTPErrorProcessor()) + if self.save_cookies: + self.cookie_file = os.path.expanduser("~/.codereview_upload_cookies_" + server) + self.cookie_jar = cookielib.MozillaCookieJar(self.cookie_file) + if os.path.exists(self.cookie_file): + try: + self.cookie_jar.load() + self.authenticated = True + StatusUpdate("Loaded authentication cookies from %s" % self.cookie_file) + except (cookielib.LoadError, IOError): + # Failed to load cookies - just ignore them. + pass + else: + # Create an empty cookie file with mode 600 + fd = os.open(self.cookie_file, os.O_CREAT, 0600) + os.close(fd) + # Always chmod the cookie file + os.chmod(self.cookie_file, 0600) + else: + # Don't save cookies across runs of update.py. + self.cookie_jar = cookielib.CookieJar() + opener.add_handler(urllib2.HTTPCookieProcessor(self.cookie_jar)) + return opener + + +def GetRpcServer(options): + """Returns an instance of an AbstractRpcServer. + + Returns: + A new AbstractRpcServer, on which RPC calls can be made. + """ + + rpc_server_class = HttpRpcServer + + def GetUserCredentials(): + """Prompts the user for a username and password.""" + # Disable status prints so they don't obscure the password prompt. + global global_status + st = global_status + global_status = None + + email = options.email + if email is None: + email = GetEmail("Email (login for uploading to %s)" % options.server) + password = getpass.getpass("Password for %s: " % email) + + # Put status back. + global_status = st + return (email, password) + + # If this is the dev_appserver, use fake authentication. + host = (options.host or options.server).lower() + if host == "localhost" or host.startswith("localhost:"): + email = options.email + if email is None: + email = "test@example.com" + logging.info("Using debug user %s. Override with --email" % email) + server = rpc_server_class( + options.server, + lambda: (email, "password"), + host_override=options.host, + extra_headers={"Cookie": 'dev_appserver_login="%s:False"' % email}, + save_cookies=options.save_cookies) + # Don't try to talk to ClientLogin. + server.authenticated = True + return server + + return rpc_server_class(options.server, GetUserCredentials, + host_override=options.host, save_cookies=options.save_cookies) + + +def EncodeMultipartFormData(fields, files): + """Encode form fields for multipart/form-data. + + Args: + fields: A sequence of (name, value) elements for regular form fields. + files: A sequence of (name, filename, value) elements for data to be + uploaded as files. + Returns: + (content_type, body) ready for httplib.HTTP instance. + + Source: + http://aspn.activestate.com/ASPN/Cookbook/Python/Recipe/146306 + """ + BOUNDARY = '-M-A-G-I-C---B-O-U-N-D-A-R-Y-' + CRLF = '\r\n' + lines = [] + for (key, value) in fields: + typecheck(key, str) + typecheck(value, str) + lines.append('--' + BOUNDARY) + lines.append('Content-Disposition: form-data; name="%s"' % key) + lines.append('') + lines.append(value) + for (key, filename, value) in files: + typecheck(key, str) + typecheck(filename, str) + typecheck(value, str) + lines.append('--' + BOUNDARY) + lines.append('Content-Disposition: form-data; name="%s"; filename="%s"' % (key, filename)) + lines.append('Content-Type: %s' % GetContentType(filename)) + lines.append('') + lines.append(value) + lines.append('--' + BOUNDARY + '--') + lines.append('') + body = CRLF.join(lines) + content_type = 'multipart/form-data; boundary=%s' % BOUNDARY + return content_type, body + + +def GetContentType(filename): + """Helper to guess the content-type from the filename.""" + return mimetypes.guess_type(filename)[0] or 'application/octet-stream' + + +# Use a shell for subcommands on Windows to get a PATH search. +use_shell = sys.platform.startswith("win") + +def RunShellWithReturnCode(command, print_output=False, + universal_newlines=True, env=os.environ): + """Executes a command and returns the output from stdout and the return code. + + Args: + command: Command to execute. + print_output: If True, the output is printed to stdout. + If False, both stdout and stderr are ignored. + universal_newlines: Use universal_newlines flag (default: True). + + Returns: + Tuple (output, return code) + """ + logging.info("Running %s", command) + p = subprocess.Popen(command, stdout=subprocess.PIPE, stderr=subprocess.PIPE, + shell=use_shell, universal_newlines=universal_newlines, env=env) + if print_output: + output_array = [] + while True: + line = p.stdout.readline() + if not line: + break + print line.strip("\n") + output_array.append(line) + output = "".join(output_array) + else: + output = p.stdout.read() + p.wait() + errout = p.stderr.read() + if print_output and errout: + print >>sys.stderr, errout + p.stdout.close() + p.stderr.close() + return output, p.returncode + + +def RunShell(command, silent_ok=False, universal_newlines=True, + print_output=False, env=os.environ): + data, retcode = RunShellWithReturnCode(command, print_output, universal_newlines, env) + if retcode: + ErrorExit("Got error status from %s:\n%s" % (command, data)) + if not silent_ok and not data: + ErrorExit("No output from %s" % command) + return data + + +class VersionControlSystem(object): + """Abstract base class providing an interface to the VCS.""" + + def __init__(self, options): + """Constructor. + + Args: + options: Command line options. + """ + self.options = options + + def GenerateDiff(self, args): + """Return the current diff as a string. + + Args: + args: Extra arguments to pass to the diff command. + """ + raise NotImplementedError( + "abstract method -- subclass %s must override" % self.__class__) + + def GetUnknownFiles(self): + """Return a list of files unknown to the VCS.""" + raise NotImplementedError( + "abstract method -- subclass %s must override" % self.__class__) + + def CheckForUnknownFiles(self): + """Show an "are you sure?" prompt if there are unknown files.""" + unknown_files = self.GetUnknownFiles() + if unknown_files: + print "The following files are not added to version control:" + for line in unknown_files: + print line + prompt = "Are you sure to continue?(y/N) " + answer = raw_input(prompt).strip() + if answer != "y": + ErrorExit("User aborted") + + def GetBaseFile(self, filename): + """Get the content of the upstream version of a file. + + Returns: + A tuple (base_content, new_content, is_binary, status) + base_content: The contents of the base file. + new_content: For text files, this is empty. For binary files, this is + the contents of the new file, since the diff output won't contain + information to reconstruct the current file. + is_binary: True iff the file is binary. + status: The status of the file. + """ + + raise NotImplementedError( + "abstract method -- subclass %s must override" % self.__class__) + + + def GetBaseFiles(self, diff): + """Helper that calls GetBase file for each file in the patch. + + Returns: + A dictionary that maps from filename to GetBaseFile's tuple. Filenames + are retrieved based on lines that start with "Index:" or + "Property changes on:". + """ + files = {} + for line in diff.splitlines(True): + if line.startswith('Index:') or line.startswith('Property changes on:'): + unused, filename = line.split(':', 1) + # On Windows if a file has property changes its filename uses '\' + # instead of '/'. + filename = filename.strip().replace('\\', '/') + files[filename] = self.GetBaseFile(filename) + return files + + + def UploadBaseFiles(self, issue, rpc_server, patch_list, patchset, options, + files): + """Uploads the base files (and if necessary, the current ones as well).""" + + def UploadFile(filename, file_id, content, is_binary, status, is_base): + """Uploads a file to the server.""" + set_status("uploading " + filename) + file_too_large = False + if is_base: + type = "base" + else: + type = "current" + if len(content) > MAX_UPLOAD_SIZE: + print ("Not uploading the %s file for %s because it's too large." % + (type, filename)) + file_too_large = True + content = "" + checksum = md5(content).hexdigest() + if options.verbose > 0 and not file_too_large: + print "Uploading %s file for %s" % (type, filename) + url = "/%d/upload_content/%d/%d" % (int(issue), int(patchset), file_id) + form_fields = [ + ("filename", filename), + ("status", status), + ("checksum", checksum), + ("is_binary", str(is_binary)), + ("is_current", str(not is_base)), + ] + if file_too_large: + form_fields.append(("file_too_large", "1")) + if options.email: + form_fields.append(("user", options.email)) + ctype, body = EncodeMultipartFormData(form_fields, [("data", filename, content)]) + response_body = rpc_server.Send(url, body, content_type=ctype) + if not response_body.startswith("OK"): + StatusUpdate(" --> %s" % response_body) + sys.exit(1) + + # Don't want to spawn too many threads, nor do we want to + # hit Rietveld too hard, or it will start serving 500 errors. + # When 8 works, it's no better than 4, and sometimes 8 is + # too many for Rietveld to handle. + MAX_PARALLEL_UPLOADS = 4 + + sema = threading.BoundedSemaphore(MAX_PARALLEL_UPLOADS) + upload_threads = [] + finished_upload_threads = [] + + class UploadFileThread(threading.Thread): + def __init__(self, args): + threading.Thread.__init__(self) + self.args = args + def run(self): + UploadFile(*self.args) + finished_upload_threads.append(self) + sema.release() + + def StartUploadFile(*args): + sema.acquire() + while len(finished_upload_threads) > 0: + t = finished_upload_threads.pop() + upload_threads.remove(t) + t.join() + t = UploadFileThread(args) + upload_threads.append(t) + t.start() + + def WaitForUploads(): + for t in upload_threads: + t.join() + + patches = dict() + [patches.setdefault(v, k) for k, v in patch_list] + for filename in patches.keys(): + base_content, new_content, is_binary, status = files[filename] + file_id_str = patches.get(filename) + if file_id_str.find("nobase") != -1: + base_content = None + file_id_str = file_id_str[file_id_str.rfind("_") + 1:] + file_id = int(file_id_str) + if base_content != None: + StartUploadFile(filename, file_id, base_content, is_binary, status, True) + if new_content != None: + StartUploadFile(filename, file_id, new_content, is_binary, status, False) + WaitForUploads() + + def IsImage(self, filename): + """Returns true if the filename has an image extension.""" + mimetype = mimetypes.guess_type(filename)[0] + if not mimetype: + return False + return mimetype.startswith("image/") + + def IsBinary(self, filename): + """Returns true if the guessed mimetyped isnt't in text group.""" + mimetype = mimetypes.guess_type(filename)[0] + if not mimetype: + return False # e.g. README, "real" binaries usually have an extension + # special case for text files which don't start with text/ + if mimetype in TEXT_MIMETYPES: + return False + return not mimetype.startswith("text/") + +class FakeMercurialUI(object): + def __init__(self): + self.quiet = True + self.output = '' + + def write(self, *args, **opts): + self.output += ' '.join(args) + +use_hg_shell = False # set to True to shell out to hg always; slower + +class MercurialVCS(VersionControlSystem): + """Implementation of the VersionControlSystem interface for Mercurial.""" + + def __init__(self, options, ui, repo): + super(MercurialVCS, self).__init__(options) + self.ui = ui + self.repo = repo + # Absolute path to repository (we can be in a subdir) + self.repo_dir = os.path.normpath(repo.root) + # Compute the subdir + cwd = os.path.normpath(os.getcwd()) + assert cwd.startswith(self.repo_dir) + self.subdir = cwd[len(self.repo_dir):].lstrip(r"\/") + if self.options.revision: + self.base_rev = self.options.revision + else: + mqparent, err = RunShellWithReturnCode(['hg', 'log', '--rev', 'qparent', '--template={node}']) + if not err and mqparent != "": + self.base_rev = mqparent + else: + self.base_rev = RunShell(["hg", "parents", "-q"]).split(':')[1].strip() + def _GetRelPath(self, filename): + """Get relative path of a file according to the current directory, + given its logical path in the repo.""" + assert filename.startswith(self.subdir), (filename, self.subdir) + return filename[len(self.subdir):].lstrip(r"\/") + + def GenerateDiff(self, extra_args): + # If no file specified, restrict to the current subdir + extra_args = extra_args or ["."] + cmd = ["hg", "diff", "--git", "-r", self.base_rev] + extra_args + data = RunShell(cmd, silent_ok=True) + svndiff = [] + filecount = 0 + for line in data.splitlines(): + m = re.match("diff --git a/(\S+) b/(\S+)", line) + if m: + # Modify line to make it look like as it comes from svn diff. + # With this modification no changes on the server side are required + # to make upload.py work with Mercurial repos. + # NOTE: for proper handling of moved/copied files, we have to use + # the second filename. + filename = m.group(2) + svndiff.append("Index: %s" % filename) + svndiff.append("=" * 67) + filecount += 1 + logging.info(line) + else: + svndiff.append(line) + if not filecount: + ErrorExit("No valid patches found in output from hg diff") + return "\n".join(svndiff) + "\n" + + def GetUnknownFiles(self): + """Return a list of files unknown to the VCS.""" + args = [] + status = RunShell(["hg", "status", "--rev", self.base_rev, "-u", "."], + silent_ok=True) + unknown_files = [] + for line in status.splitlines(): + st, fn = line.split(" ", 1) + if st == "?": + unknown_files.append(fn) + return unknown_files + + def GetBaseFile(self, filename): + set_status("inspecting " + filename) + # "hg status" and "hg cat" both take a path relative to the current subdir + # rather than to the repo root, but "hg diff" has given us the full path + # to the repo root. + base_content = "" + new_content = None + is_binary = False + oldrelpath = relpath = self._GetRelPath(filename) + # "hg status -C" returns two lines for moved/copied files, one otherwise + if use_hg_shell: + out = RunShell(["hg", "status", "-C", "--rev", self.base_rev, relpath]) + else: + fui = FakeMercurialUI() + ret = commands.status(fui, self.repo, *[relpath], **{'rev': [self.base_rev], 'copies': True}) + if ret: + raise util.Abort(ret) + out = fui.output + out = out.splitlines() + # HACK: strip error message about missing file/directory if it isn't in + # the working copy + if out[0].startswith('%s: ' % relpath): + out = out[1:] + status, what = out[0].split(' ', 1) + if len(out) > 1 and status == "A" and what == relpath: + oldrelpath = out[1].strip() + status = "M" + if ":" in self.base_rev: + base_rev = self.base_rev.split(":", 1)[0] + else: + base_rev = self.base_rev + if status != "A": + if use_hg_shell: + base_content = RunShell(["hg", "cat", "-r", base_rev, oldrelpath], silent_ok=True) + else: + base_content = str(self.repo[base_rev][oldrelpath].data()) + is_binary = "\0" in base_content # Mercurial's heuristic + if status != "R": + new_content = open(relpath, "rb").read() + is_binary = is_binary or "\0" in new_content + if is_binary and base_content and use_hg_shell: + # Fetch again without converting newlines + base_content = RunShell(["hg", "cat", "-r", base_rev, oldrelpath], + silent_ok=True, universal_newlines=False) + if not is_binary or not self.IsImage(relpath): + new_content = None + return base_content, new_content, is_binary, status + + +# NOTE: The SplitPatch function is duplicated in engine.py, keep them in sync. +def SplitPatch(data): + """Splits a patch into separate pieces for each file. + + Args: + data: A string containing the output of svn diff. + + Returns: + A list of 2-tuple (filename, text) where text is the svn diff output + pertaining to filename. + """ + patches = [] + filename = None + diff = [] + for line in data.splitlines(True): + new_filename = None + if line.startswith('Index:'): + unused, new_filename = line.split(':', 1) + new_filename = new_filename.strip() + elif line.startswith('Property changes on:'): + unused, temp_filename = line.split(':', 1) + # When a file is modified, paths use '/' between directories, however + # when a property is modified '\' is used on Windows. Make them the same + # otherwise the file shows up twice. + temp_filename = temp_filename.strip().replace('\\', '/') + if temp_filename != filename: + # File has property changes but no modifications, create a new diff. + new_filename = temp_filename + if new_filename: + if filename and diff: + patches.append((filename, ''.join(diff))) + filename = new_filename + diff = [line] + continue + if diff is not None: + diff.append(line) + if filename and diff: + patches.append((filename, ''.join(diff))) + return patches + + +def UploadSeparatePatches(issue, rpc_server, patchset, data, options): + """Uploads a separate patch for each file in the diff output. + + Returns a list of [patch_key, filename] for each file. + """ + patches = SplitPatch(data) + rv = [] + for patch in patches: + set_status("uploading patch for " + patch[0]) + if len(patch[1]) > MAX_UPLOAD_SIZE: + print ("Not uploading the patch for " + patch[0] + + " because the file is too large.") + continue + form_fields = [("filename", patch[0])] + if not options.download_base: + form_fields.append(("content_upload", "1")) + files = [("data", "data.diff", patch[1])] + ctype, body = EncodeMultipartFormData(form_fields, files) + url = "/%d/upload_patch/%d" % (int(issue), int(patchset)) + print "Uploading patch for " + patch[0] + response_body = rpc_server.Send(url, body, content_type=ctype) + lines = response_body.splitlines() + if not lines or lines[0] != "OK": + StatusUpdate(" --> %s" % response_body) + sys.exit(1) + rv.append([lines[1], patch[0]]) + return rv diff --git a/lib/godoc/codewalk.html b/lib/godoc/codewalk.html new file mode 100644 index 000000000..2835c6e82 --- /dev/null +++ b/lib/godoc/codewalk.html @@ -0,0 +1,58 @@ + + + + + + + +
+
+
+
+
+ + Pop Out Code + + +
+
+ +
+
+
+ code on leftright + code width 70% + filepaths shownhidden +
+
+
+
+ {{range .Step}} +
+ +
{{html .Title}}
+
+ {{with .Err}} + ERROR LOADING FILE: {{html .}}

+ {{end}} + {{.XML}} +
+
{{html .}}
+
+ {{end}} +
+
+ previous step + • + next step +
+
+
diff --git a/lib/godoc/codewalkdir.html b/lib/godoc/codewalkdir.html new file mode 100644 index 000000000..b7674c6ce --- /dev/null +++ b/lib/godoc/codewalkdir.html @@ -0,0 +1,16 @@ + + + +{{range .}} + + {{$name_html := html .Name}} + + + + +{{end}} +
{{$name_html}} {{html .Title}}
diff --git a/lib/godoc/dirlist.html b/lib/godoc/dirlist.html new file mode 100644 index 000000000..a3e1a2fa8 --- /dev/null +++ b/lib/godoc/dirlist.html @@ -0,0 +1,31 @@ + + +

+ + + + + + + + + + + +{{range .}} + + {{$name_html := fileInfoName . | html}} + + + + + + +{{end}} + +
File Bytes Modified
..
{{$name_html}}{{html .Size}}{{fileInfoTime . | html}}
+

diff --git a/lib/godoc/error.html b/lib/godoc/error.html new file mode 100644 index 000000000..7573aa236 --- /dev/null +++ b/lib/godoc/error.html @@ -0,0 +1,9 @@ + + +

+{{html .}} +

diff --git a/lib/godoc/godoc.html b/lib/godoc/godoc.html new file mode 100644 index 000000000..671160d5a --- /dev/null +++ b/lib/godoc/godoc.html @@ -0,0 +1,69 @@ + + + + +{{with .Title}} + {{html .}} - The Go Programming Language +{{else}} + The Go Programming Language +{{end}} + + + + + +
+
+

The Go Programming Language

+ + +
+
+ + {{with .Menu}} + + {{end}} + + {{with .Title}} +

{{html .}}

+ {{end}} + {{with .Subtitle}} + {{html .}} + {{end}} + + + + + + {{printf "%s" .Content}} +
+
+

Build version {{html .Version}}. Except as noted, this content is licensed under a Creative Commons Attribution 3.0 License.

+
+
+ + diff --git a/lib/godoc/package.html b/lib/godoc/package.html new file mode 100644 index 000000000..7a89d780c --- /dev/null +++ b/lib/godoc/package.html @@ -0,0 +1,122 @@ + +{{with .PAst}} +
{{node_html . $.FSet}}
+{{end}} +{{with .PDoc}} + + {{if $.IsPkg}} +

import "{{html .ImportPath}}"

+ {{end}} + {{comment_html .Doc}} + {{if $.IsPkg}} + {{with .Filenames}} +

+

Package files

+ + {{range .}} + {{.|filename|html}} + {{end}} + +

+ {{end}} + {{end}} + {{with .Consts}} +

Constants

+ {{range .}} + {{comment_html .Doc}} +
{{node_html .Decl $.FSet}}
+ {{end}} + {{end}} + {{with .Vars}} +

Variables

+ {{range .}} + {{comment_html .Doc}} +
{{node_html .Decl $.FSet}}
+ {{end}} + {{end}} + {{with .Funcs}} + {{range .}} + {{/* Name is a string - no need for FSet */}} + {{$name_html := html .Name}} +

func {{$name_html}}

+

{{node_html .Decl $.FSet}}

+ {{comment_html .Doc}} + {{end}} + {{end}} + {{with .Types}} + {{range .}} + {{$tname_html := node_html .Type.Name $.FSet}} +

type {{$tname_html}}

+ {{comment_html .Doc}} +

{{node_html .Decl $.FSet}}

+ {{range .Consts}} + {{comment_html .Doc}} +
{{node_html .Decl $.FSet}}
+ {{end}} + {{range .Vars}} + {{comment_html .Doc}} +
{{node_html .Decl $.FSet}}
+ {{end}} + {{range .Factories}} + {{$name_html := html .Name}} +

func {{$name_html}}

+

{{node_html .Decl $.FSet}}

+ {{comment_html .Doc}} + {{end}} + {{range .Methods}} + {{$name_html := html .Name}} +

func ({{node_html .Recv $.FSet}}) {{$name_html}}

+

{{node_html .Decl $.FSet}}

+ {{comment_html .Doc}} + {{end}} + {{end}} + {{end}} + {{with .Bugs}} +

Bugs

+ {{range .}} + {{comment_html .}} + {{end}} + {{end}} +{{end}} +{{with .PList}} +

Other packages

+

+ {{/* PList entries are strings - no need for FSet */}} + {{range .}} + {{html .}}
+ {{end}} +

+{{end}} +{{with .Dirs}} +

+ Need more packages? The + Package Dashboard + provides a list of goinstallable packages. +

+ {{/* DirList entries are numbers and strings - no need for FSet */}} +

Subdirectories

+

+ + + + + + + + + + {{range .List}} + + {{repeat `` .Depth}} + + + + + {{end}} +
Name Synopsis
..
{{html .Name}}{{html .Synopsis}}
+

+{{end}} diff --git a/lib/godoc/package.txt b/lib/godoc/package.txt new file mode 100644 index 000000000..179b33493 --- /dev/null +++ b/lib/godoc/package.txt @@ -0,0 +1,82 @@ +{{with .PAst}}{{node . $.FSet}}{{end}}{{/* + +--------------------------------------- + +*/}}{{with .PDoc}}{{if $.IsPkg}}PACKAGE + +package {{.PackageName}} +import "{{.ImportPath}}" + +{{else}}COMMAND DOCUMENTATION + +{{end}}{{.Doc}}{{/* + +--------------------------------------- + +*/}}{{with .Consts}} +CONSTANTS + +{{range .}}{{node .Decl $.FSet}} +{{.Doc}}{{end}} +{{end}}{{/* + +--------------------------------------- + +*/}}{{with .Vars}} +VARIABLES + +{{range .}}{{node .Decl $.FSet}} +{{.Doc}}{{end}} +{{end}}{{/* + +--------------------------------------- + +*/}}{{with .Funcs}} +FUNCTIONS + +{{range .}}{{node .Decl $.FSet}} +{{.Doc}} +{{end}}{{end}}{{/* + +--------------------------------------- + +*/}}{{with .Types}} +TYPES + +{{range .}}{{node .Decl $.FSet}} +{{.Doc}} +{{range .Consts}}{{node .Decl $.FSet}} +{{.Doc}} +{{end}}{{range .Vars}}{{node .Decl $.FSet}} +{{.Doc}} +{{end}}{{range .Factories}}{{node .Decl $.FSet}} +{{.Doc}} +{{end}}{{range .Methods}}{{node .Decl $.FSet}} +{{.Doc}} +{{end}}{{end}}{{end}}{{/* + +--------------------------------------- + +*/}}{{with .Bugs}} +BUGS + +{{range .}}{{.}} +{{end}}{{end}}{{end}}{{/* + +--------------------------------------- + +*/}}{{with .PList}} +OTHER PACKAGES + +{{range .}} +{{.}}{{end}} +{{end}}{{/* + +--------------------------------------- + +*/}}{{with .Dirs}} +SUBDIRECTORIES + +{{range .List}} + {{.Name}}{{end}} +{{end}} diff --git a/lib/godoc/search.html b/lib/godoc/search.html new file mode 100644 index 000000000..36c34f54d --- /dev/null +++ b/lib/godoc/search.html @@ -0,0 +1,98 @@ + +{{$query_url := urlquery .Query}} +{{with .Alert}} +

+ {{html .}} +

+{{end}} +{{with .Alt}} +

+ Did you mean: + {{range .Alts}} + {{html .}} + {{end}} +

+{{end}} +{{with .Hit}} + {{with .Decls}} +

Package-level declarations

+ {{range .}} + {{$pkg_html := pkgLink .Pak.Path | html}} +

package {{html .Pak.Name}}

+ {{range .Files}} + {{$src_html := srcLink .File.Path | html}} + {{range .Groups}} + {{range .Infos}} + {{$src_html}}:{{infoLine .}} + {{infoSnippet_html .}} + {{end}} + {{end}} + {{end}} + {{end}} + {{end}} + {{with .Others}} +

Local declarations and uses

+ {{range .}} + {{$pkg_html := pkgLink .Pak.Path | html}} +

package {{html .Pak.Name}}

+ {{range .Files}} + {{$src_html := srcLink .File.Path | html}} + {{$src_html}} + + {{range .Groups}} + + + + + + + {{end}} +
{{infoKind_html .Kind}} + {{range .Infos}} + {{infoLine .}} + {{end}} +
+ {{end}} + {{end}} + {{end}} +{{end}} +{{with .Textual}} + {{if $.Complete}} +

{{html $.Found}} textual occurrences

+ {{else}} +

More than {{html $.Found}} textual occurrences

+

+ Not all files or lines containing "{{html $.Query}}" are shown. +

+ {{end}} +

+ + {{range .}} + {{$src_html := srcLink .Filename | html}} + + + + + + + + {{end}} + {{if not $.Complete}} + + {{end}} +
+ {{$src_html}}: + {{len .Lines}} + {{range .Lines}} + {{html .}} + {{end}} + {{if not $.Complete}} + ... + {{end}} +
...
+

+{{end}} diff --git a/lib/godoc/search.txt b/lib/godoc/search.txt new file mode 100644 index 000000000..1dd64afdb --- /dev/null +++ b/lib/godoc/search.txt @@ -0,0 +1,40 @@ +QUERY + {{.Query}} +{{with .Alert}} +{{.}} +{{end}}{{/* .Alert */}}{{/* + +--------------------------------------- + +*/}}{{with .Alt}} +DID YOU MEAN +{{range .Alts}} {{.}} +{{end}}{{end}}{{/* .Alts */}}{{/* + +--------------------------------------- + +*/}}{{with .Hit}}{{with .Decls}} +PACKAGE-LEVEL DECLARATIONS + +{{range .}}package {{.Pak.Name}} +{{range $file := .Files}}{{range .Groups}}{{range .Infos}} {{srcLink $file.File.Path}}:{{infoLine .}}{{end}} +{{end}}{{end}}{{/* .Files */}} +{{end}}{{end}}{{/* .Decls */}}{{/* + +--------------------------------------- + +*/}}{{with .Others}} +LOCAL DECLARATIONS AND USES + +{{range .}}package {{.Pak.Name}} +{{range $file := .Files}}{{range .Groups}}{{range .Infos}} {{srcLink $file.File.Path}}:{{infoLine .}} +{{end}}{{end}}{{end}}{{/* .Files */}} +{{end}}{{end}}{{/* .Others */}}{{end}}{{/* .Hit */}}{{/* + +--------------------------------------- + +*/}}{{if .Textual}}{{if .Complete}}{{.Found}} TEXTUAL OCCURRENCES{{else}}MORE THAN {{.Found}} TEXTUAL OCCURRENCES{{end}} + +{{range .Textual}}{{len .Lines}} {{srcLink .Filename}} +{{end}}{{if not .Complete}}... ... +{{end}}{{end}} diff --git a/misc/IntelliJIDEA/Go.xml b/misc/IntelliJIDEA/Go.xml new file mode 100644 index 000000000..09265a2e0 --- /dev/null +++ b/misc/IntelliJIDEA/Go.xml @@ -0,0 +1,98 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/misc/arm/a b/misc/arm/a new file mode 100755 index 000000000..701f4941f --- /dev/null +++ b/misc/arm/a @@ -0,0 +1,58 @@ +#!/usr/bin/env bash + +# Copyright 2010 The Go Authors. All rights reserved. +# Use of this source code is governed by a BSD-style +# license that can be found in the LICENSE file. + +# This is a small script for executing go binaries on the android platform. +# +# example: +# ./a 5.out foo bar baz +# +# The script exports the local values of GOARCH, GOTRACEBACK and GOGC +# to the android environment. +# +# Known issues: +# The script fails unless the last character output by the program is "\n" +# +# TODO(kaib): add gdb bridge support + +exp () +{ + if [ ${!1} ]; then + echo "export $1=\"${!1}\"; " + fi +} + +# adb does not correctly return the exit value of the executed program. use this +# wrapper to manually extract the exit value +rloc=/data/local/tmp/retval +rsize=$(adb shell "ls -l $rloc"|tr -s ' '|cut -d' ' -f4) +rcheck=38 +if [ "$rsize" != "$rcheck" ]; then +# echo "debug: retval size incorrect want $rcheck, got $rsize. uploading" + echo >/tmp/adb.retval '#!/system/bin/sh +"$@" +echo RETVAL: $?' + adb push /tmp/adb.retval $rloc >/dev/null 2>&1 + adb shell chmod 755 $rloc +fi + +# run the main binary +if [ "-g" == "$1" ]; then + adb forward tcp:$2 tcp:$2 + args=$(echo $*| cut -d' ' -f4-) + adb push $3 /data/local/tmp/$3 >/dev/null 2>&1 + adb shell "$(exp GOARCH) $(exp GOTRACEBACK) $(exp GOGC) \ + gdbserver :$2 /data/local/tmp/retval /data/local/tmp/$3 $args" \ + 2>&1|tr -d '\r' |tee /tmp/adb.out|grep -v RETVAL +else + if [ "$*" != "$1" ]; then + args=$(echo $*| cut -d' ' -f2-) + fi + adb push $1 /data/local/tmp/$1 >/dev/null 2>&1 + adb shell "$(exp GOARCH) $(exp GOTRACEBACK) $(exp GOGC) \ + /data/local/tmp/retval /data/local/tmp/$1 $args" \ + 2>&1|tr -d '\r' |tee /tmp/adb.out|grep -v RETVAL +fi +exit $(grep RETVAL /tmp/adb.out|tr -d '\n\r'| cut -d' ' -f2) diff --git a/misc/bash/go b/misc/bash/go new file mode 100644 index 000000000..caced154f --- /dev/null +++ b/misc/bash/go @@ -0,0 +1,6 @@ +# install in /etc/bash_completion.d/ or your personal directory + +complete -f -X '!*.8' 8l +complete -f -X '!*.6' 6l +complete -f -X '!*.5' 5l +complete -f -X '!*.go' 8g 6g 5g gofmt gccgo diff --git a/misc/bbedit/Go.plist b/misc/bbedit/Go.plist new file mode 100755 index 000000000..45535350a --- /dev/null +++ b/misc/bbedit/Go.plist @@ -0,0 +1,101 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. +{ + BBEditDocumentType = "CodelessLanguageModule"; + BBLMColorsSyntax = YES; + BBLMIsCaseSensitive = YES; + BBLMKeywordList = ( + append, + bool, + break, + byte, + cap, + case, + chan, + close, + complex, + complex128, + complex64, + const, + continue, + copy, + default, + defer, + else, + fallthrough, + false, + float32, + float64, + for, + func, + go, + goto, + if, + iota, + imag, + import, + int, + int16, + int32, + int64, + int8, + interface, + len, + make, + map, + new, + nil, + package, + panic, + print, + println, + range, + real, + recover, + return, + select, + string, + struct, + switch, + true, + type, + uint, + uint16, + uint32, + uint64, + uint8, + uintptr, + var, + ); + BBLMLanguageCode = go; + "BBLMLanguageDisplayName" = "Go"; + BBLMScansFunctions = YES; + BBLMSuffixMap = ( + { + BBLMLanguageSuffix = ".go"; + }, + ); + "Language Features" = { + "Close Block Comments" = "*/"; + "Close Parameter Lists" = ")"; + "Close Statement Blocks" = "}"; + "Close Strings 1" = "`"; + "Close Strings 2" = "\""; + "End-of-line Ends Strings 1" = YES; + "End-of-line Ends Strings 2" = YES; + "Escape Char in Strings 1" = "\\"; + "Escape Char in Strings 2" = "\\"; + "Identifier and Keyword Characters" = "0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZ_abcdefghijklmnopqrstuvwxyz"; + "Open Block Comments" = "/*"; + "Open Line Comments" = "//"; + "Open Parameter Lists" = "("; + "Open Statement Blocks" = "{"; + "Open Strings 1" = "`"; + "Open Strings 2" = "\""; + "Prefix for Functions" = "func"; + "Prefix for Procedures" = "func"; + "Terminator for Prototypes 1" = ";"; + "Terminator for Prototypes 2" = ""; + }; +} diff --git a/misc/cgo/gmp/Makefile b/misc/cgo/gmp/Makefile new file mode 100644 index 000000000..fc6209f27 --- /dev/null +++ b/misc/cgo/gmp/Makefile @@ -0,0 +1,38 @@ +# Copyright 2009 The Go Authors. All rights reserved. +# Use of this source code is governed by a BSD-style +# license that can be found in the LICENSE file. + +include ../../../src/Make.inc + +TARG=gmp + +# Can have plain GOFILES too, but this example doesn't. + +CGOFILES=\ + gmp.go + +CGO_LDFLAGS=-lgmp + +# To add flags necessary for locating the library or its include files, +# set CGO_CFLAGS or CGO_LDFLAGS. For example, to use an +# alternate installation of the library: +# CGO_CFLAGS=-I/home/rsc/gmp32/include +# CGO_LDFLAGS+=-L/home/rsc/gmp32/lib +# Note the += on the second line. + +CLEANFILES+=pi fib + +include ../../../src/Make.pkg + +# Simple test programs + +# Computes 1000 digits of pi; single-threaded. +pi: install pi.go + $(GC) pi.go + $(LD) -o $@ pi.$O + +# Computes 200 Fibonacci numbers; multi-threaded. +fib: install fib.go + $(GC) fib.go + $(LD) -o $@ fib.$O + diff --git a/misc/cgo/gmp/fib.go b/misc/cgo/gmp/fib.go new file mode 100644 index 000000000..3eda39e17 --- /dev/null +++ b/misc/cgo/gmp/fib.go @@ -0,0 +1,43 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Compute Fibonacci numbers with two goroutines +// that pass integers back and forth. No actual +// concurrency, just threads and synchronization +// and foreign code on multiple pthreads. + +package main + +import ( + big "gmp" + "runtime" +) + +func fibber(c chan *big.Int, out chan string, n int64) { + // Keep the fibbers in dedicated operating system + // threads, so that this program tests coordination + // between pthreads and not just goroutines. + runtime.LockOSThread() + + i := big.NewInt(n) + if n == 0 { + c <- i + } + for { + j := <-c + out <- j.String() + i.Add(i, j) + c <- i + } +} + +func main() { + c := make(chan *big.Int) + out := make(chan string) + go fibber(c, out, 0) + go fibber(c, out, 1) + for i := 0; i < 200; i++ { + println(<-out) + } +} diff --git a/misc/cgo/gmp/gmp.go b/misc/cgo/gmp/gmp.go new file mode 100644 index 000000000..3dbc022ce --- /dev/null +++ b/misc/cgo/gmp/gmp.go @@ -0,0 +1,368 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +/* +An example of wrapping a C library in Go. This is the GNU +multiprecision library gmp's integer type mpz_t wrapped to look like +the Go package big's integer type Int. + +This is a syntactically valid Go program—it can be parsed with the Go +parser and processed by godoc—but it is not compiled directly by 6g. +Instead, a separate tool, cgo, processes it to produce three output +files. The first two, 6g.go and 6c.c, are a Go source file for 6g and +a C source file for 6c; both compile as part of the named package +(gmp, in this example). The third, gcc.c, is a C source file for gcc; +it compiles into a shared object (.so) that is dynamically linked into +any 6.out that imports the first two files. + +The stanza + + // #include + import "C" + +is a signal to cgo. The doc comment on the import of "C" provides +additional context for the C file. Here it is just a single #include +but it could contain arbitrary C definitions to be imported and used. + +Cgo recognizes any use of a qualified identifier C.xxx and uses gcc to +find the definition of xxx. If xxx is a type, cgo replaces C.xxx with +a Go translation. C arithmetic types translate to precisely-sized Go +arithmetic types. A C struct translates to a Go struct, field by +field; unrepresentable fields are replaced with opaque byte arrays. A +C union translates into a struct containing the first union member and +perhaps additional padding. C arrays become Go arrays. C pointers +become Go pointers. C function pointers become Go's uintptr. +C void pointer's become Go's unsafe.Pointer. + +For example, mpz_t is defined in as: + + typedef unsigned long int mp_limb_t; + + typedef struct + { + int _mp_alloc; + int _mp_size; + mp_limb_t *_mp_d; + } __mpz_struct; + + typedef __mpz_struct mpz_t[1]; + +Cgo generates: + + type _C_int int32 + type _C_mp_limb_t uint64 + type _C___mpz_struct struct { + _mp_alloc _C_int; + _mp_size _C_int; + _mp_d *_C_mp_limb_t; + } + type _C_mpz_t [1]_C___mpz_struct + +and then replaces each occurrence of a type C.xxx with _C_xxx. + +If xxx is data, cgo arranges for C.xxx to refer to the C variable, +with the type translated as described above. To do this, cgo must +introduce a Go variable that points at the C variable (the linker can +be told to initialize this pointer). For example, if the gmp library +provided + + mpz_t zero; + +then cgo would rewrite a reference to C.zero by introducing + + var _C_zero *C.mpz_t + +and then replacing all instances of C.zero with (*_C_zero). + +Cgo's most interesting translation is for functions. If xxx is a C +function, then cgo rewrites C.xxx into a new function _C_xxx that +calls the C xxx in a standard pthread. The new function translates +its arguments, calls xxx, and translates the return value. + +Translation of parameters and the return value follows the type +translation above except that arrays passed as parameters translate +explicitly in Go to pointers to arrays, as they do (implicitly) in C. + +Garbage collection is the big problem. It is fine for the Go world to +have pointers into the C world and to free those pointers when they +are no longer needed. To help, the Go code can define Go objects +holding the C pointers and use runtime.SetFinalizer on those Go objects. + +It is much more difficult for the C world to have pointers into the Go +world, because the Go garbage collector is unaware of the memory +allocated by C. The most important consideration is not to +constrain future implementations, so the rule is that Go code can +hand a Go pointer to C code but must separately arrange for +Go to hang on to a reference to the pointer until C is done with it. +*/ +package gmp + +// #include +// #include +import "C" + +import ( + "os" + "unsafe" +) + +/* + * one of a kind + */ + +// An Int represents a signed multi-precision integer. +// The zero value for an Int represents the value 0. +type Int struct { + i C.mpz_t + init bool +} + +// NewInt returns a new Int initialized to x. +func NewInt(x int64) *Int { return new(Int).SetInt64(x) } + +// Int promises that the zero value is a 0, but in gmp +// the zero value is a crash. To bridge the gap, the +// init bool says whether this is a valid gmp value. +// doinit initializes z.i if it needs it. This is not inherent +// to FFI, just a mismatch between Go's convention of +// making zero values useful and gmp's decision not to. +func (z *Int) doinit() { + if z.init { + return + } + z.init = true + C.mpz_init(&z.i[0]) +} + +// Bytes returns z's representation as a big-endian byte array. +func (z *Int) Bytes() []byte { + b := make([]byte, (z.Len()+7)/8) + n := C.size_t(len(b)) + C.mpz_export(unsafe.Pointer(&b[0]), &n, 1, 1, 1, 0, &z.i[0]) + return b[0:n] +} + +// Len returns the length of z in bits. 0 is considered to have length 1. +func (z *Int) Len() int { + z.doinit() + return int(C.mpz_sizeinbase(&z.i[0], 2)) +} + +// Set sets z = x and returns z. +func (z *Int) Set(x *Int) *Int { + z.doinit() + C.mpz_set(&z.i[0], &x.i[0]) + return z +} + +// SetBytes interprets b as the bytes of a big-endian integer +// and sets z to that value. +func (z *Int) SetBytes(b []byte) *Int { + z.doinit() + if len(b) == 0 { + z.SetInt64(0) + } else { + C.mpz_import(&z.i[0], C.size_t(len(b)), 1, 1, 1, 0, unsafe.Pointer(&b[0])) + } + return z +} + +// SetInt64 sets z = x and returns z. +func (z *Int) SetInt64(x int64) *Int { + z.doinit() + // TODO(rsc): more work on 32-bit platforms + C.mpz_set_si(&z.i[0], C.long(x)) + return z +} + +// SetString interprets s as a number in the given base +// and sets z to that value. The base must be in the range [2,36]. +// SetString returns an error if s cannot be parsed or the base is invalid. +func (z *Int) SetString(s string, base int) os.Error { + z.doinit() + if base < 2 || base > 36 { + return os.EINVAL + } + p := C.CString(s) + defer C.free(unsafe.Pointer(p)) + if C.mpz_set_str(&z.i[0], p, C.int(base)) < 0 { + return os.EINVAL + } + return nil +} + +// String returns the decimal representation of z. +func (z *Int) String() string { + if z == nil { + return "nil" + } + z.doinit() + p := C.mpz_get_str(nil, 10, &z.i[0]) + s := C.GoString(p) + C.free(unsafe.Pointer(p)) + return s +} + +func (z *Int) destroy() { + if z.init { + C.mpz_clear(&z.i[0]) + } + z.init = false +} + +/* + * arithmetic + */ + +// Add sets z = x + y and returns z. +func (z *Int) Add(x, y *Int) *Int { + x.doinit() + y.doinit() + z.doinit() + C.mpz_add(&z.i[0], &x.i[0], &y.i[0]) + return z +} + +// Sub sets z = x - y and returns z. +func (z *Int) Sub(x, y *Int) *Int { + x.doinit() + y.doinit() + z.doinit() + C.mpz_sub(&z.i[0], &x.i[0], &y.i[0]) + return z +} + +// Mul sets z = x * y and returns z. +func (z *Int) Mul(x, y *Int) *Int { + x.doinit() + y.doinit() + z.doinit() + C.mpz_mul(&z.i[0], &x.i[0], &y.i[0]) + return z +} + +// Div sets z = x / y, rounding toward zero, and returns z. +func (z *Int) Div(x, y *Int) *Int { + x.doinit() + y.doinit() + z.doinit() + C.mpz_tdiv_q(&z.i[0], &x.i[0], &y.i[0]) + return z +} + +// Mod sets z = x % y and returns z. +// Like the result of the Go % operator, z has the same sign as x. +func (z *Int) Mod(x, y *Int) *Int { + x.doinit() + y.doinit() + z.doinit() + C.mpz_tdiv_r(&z.i[0], &x.i[0], &y.i[0]) + return z +} + +// Lsh sets z = x << s and returns z. +func (z *Int) Lsh(x *Int, s uint) *Int { + x.doinit() + z.doinit() + C.mpz_mul_2exp(&z.i[0], &x.i[0], C.mp_bitcnt_t(s)) + return z +} + +// Rsh sets z = x >> s and returns z. +func (z *Int) Rsh(x *Int, s uint) *Int { + x.doinit() + z.doinit() + C.mpz_div_2exp(&z.i[0], &x.i[0], C.mp_bitcnt_t(s)) + return z +} + +// Exp sets z = x^y % m and returns z. +// If m == nil, Exp sets z = x^y. +func (z *Int) Exp(x, y, m *Int) *Int { + m.doinit() + x.doinit() + y.doinit() + z.doinit() + if m == nil { + C.mpz_pow_ui(&z.i[0], &x.i[0], C.mpz_get_ui(&y.i[0])) + } else { + C.mpz_powm(&z.i[0], &x.i[0], &y.i[0], &m.i[0]) + } + return z +} + +func (z *Int) Int64() int64 { + if !z.init { + return 0 + } + return int64(C.mpz_get_si(&z.i[0])) +} + +// Neg sets z = -x and returns z. +func (z *Int) Neg(x *Int) *Int { + x.doinit() + z.doinit() + C.mpz_neg(&z.i[0], &x.i[0]) + return z +} + +// Abs sets z to the absolute value of x and returns z. +func (z *Int) Abs(x *Int) *Int { + x.doinit() + z.doinit() + C.mpz_abs(&z.i[0], &x.i[0]) + return z +} + +/* + * functions without a clear receiver + */ + +// CmpInt compares x and y. The result is +// +// -1 if x < y +// 0 if x == y +// +1 if x > y +// +func CmpInt(x, y *Int) int { + x.doinit() + y.doinit() + switch cmp := C.mpz_cmp(&x.i[0], &y.i[0]); { + case cmp < 0: + return -1 + case cmp == 0: + return 0 + } + return +1 +} + +// DivModInt sets q = x / y and r = x % y. +func DivModInt(q, r, x, y *Int) { + q.doinit() + r.doinit() + x.doinit() + y.doinit() + C.mpz_tdiv_qr(&q.i[0], &r.i[0], &x.i[0], &y.i[0]) +} + +// GcdInt sets d to the greatest common divisor of a and b, +// which must be positive numbers. +// If x and y are not nil, GcdInt sets x and y such that d = a*x + b*y. +// If either a or b is not positive, GcdInt sets d = x = y = 0. +func GcdInt(d, x, y, a, b *Int) { + d.doinit() + x.doinit() + y.doinit() + a.doinit() + b.doinit() + C.mpz_gcdext(&d.i[0], &x.i[0], &y.i[0], &a.i[0], &b.i[0]) +} + +// ProbablyPrime performs n Miller-Rabin tests to check whether z is prime. +// If it returns true, z is prime with probability 1 - 1/4^n. +// If it returns false, z is not prime. +func (z *Int) ProbablyPrime(n int) bool { + z.doinit() + return int(C.mpz_probab_prime_p(&z.i[0], C.int(n))) > 0 +} diff --git a/misc/cgo/gmp/pi.go b/misc/cgo/gmp/pi.go new file mode 100644 index 000000000..45f61abbd --- /dev/null +++ b/misc/cgo/gmp/pi.go @@ -0,0 +1,104 @@ +/* +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + + * Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + + * Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. + + * Neither the name of "The Computer Language Benchmarks Game" nor the + name of "The Computer Language Shootout Benchmarks" nor the names of + its contributors may be used to endorse or promote products derived + from this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE +LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +POSSIBILITY OF SUCH DAMAGE. +*/ + +/* The Computer Language Benchmarks Game + * http://shootout.alioth.debian.org/ + * + * contributed by The Go Authors. + * based on pidigits.c (by Paolo Bonzini & Sean Bartlett, + * modified by Michael Mellor) + */ + +package main + +import ( + big "gmp" + "fmt" + "runtime" +) + +var ( + tmp1 = big.NewInt(0) + tmp2 = big.NewInt(0) + numer = big.NewInt(1) + accum = big.NewInt(0) + denom = big.NewInt(1) + ten = big.NewInt(10) +) + +func extractDigit() int64 { + if big.CmpInt(numer, accum) > 0 { + return -1 + } + tmp1.Lsh(numer, 1).Add(tmp1, numer).Add(tmp1, accum) + big.DivModInt(tmp1, tmp2, tmp1, denom) + tmp2.Add(tmp2, numer) + if big.CmpInt(tmp2, denom) >= 0 { + return -1 + } + return tmp1.Int64() +} + +func nextTerm(k int64) { + y2 := k*2 + 1 + accum.Add(accum, tmp1.Lsh(numer, 1)) + accum.Mul(accum, tmp1.SetInt64(y2)) + numer.Mul(numer, tmp1.SetInt64(k)) + denom.Mul(denom, tmp1.SetInt64(y2)) +} + +func eliminateDigit(d int64) { + accum.Sub(accum, tmp1.Mul(denom, tmp1.SetInt64(d))) + accum.Mul(accum, ten) + numer.Mul(numer, ten) +} + +func main() { + i := 0 + k := int64(0) + for { + d := int64(-1) + for d < 0 { + k++ + nextTerm(k) + d = extractDigit() + } + eliminateDigit(d) + fmt.Printf("%c", d+'0') + + if i++; i%50 == 0 { + fmt.Printf("\n") + if i >= 1000 { + break + } + } + } + + fmt.Printf("\n%d calls; bit sizes: %d %d %d\n", runtime.Cgocalls(), numer.Len(), accum.Len(), denom.Len()) +} diff --git a/misc/cgo/life/Makefile b/misc/cgo/life/Makefile new file mode 100644 index 000000000..5a10380ed --- /dev/null +++ b/misc/cgo/life/Makefile @@ -0,0 +1,21 @@ +# Copyright 2010 The Go Authors. All rights reserved. +# Use of this source code is governed by a BSD-style +# license that can be found in the LICENSE file. + +include ../../../src/Make.inc + +TARG=life + +CGOFILES=\ + life.go\ + +CGO_OFILES=\ + c-life.o\ + +CLEANFILES+=life + +include ../../../src/Make.pkg + +life: install main.go + $(GC) main.go + $(LD) -o $@ main.$O diff --git a/misc/cgo/life/c-life.c b/misc/cgo/life/c-life.c new file mode 100644 index 000000000..657245595 --- /dev/null +++ b/misc/cgo/life/c-life.c @@ -0,0 +1,56 @@ +// Copyright 2010 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +#include +#include "life.h" +#include "_cgo_export.h" + +const int MYCONST = 0; + +// Do the actual manipulation of the life board in C. This could be +// done easily in Go, we are just using C for demonstration +// purposes. +void +Step(int x, int y, int *a, int *n) +{ + struct GoStart_return r; + + // Use Go to start 4 goroutines each of which handles 1/4 of the + // board. + r = GoStart(0, x, y, 0, x / 2, 0, y / 2, a, n); + assert(r.r0 == 0 && r.r1 == 100); // test multiple returns + r = GoStart(1, x, y, x / 2, x, 0, y / 2, a, n); + assert(r.r0 == 1 && r.r1 == 101); // test multiple returns + GoStart(2, x, y, 0, x / 2, y / 2, y, a, n); + GoStart(3, x, y, x / 2, x, y / 2, y, a, n); + GoWait(0); + GoWait(1); + GoWait(2); + GoWait(3); +} + +// The actual computation. This is called in parallel. +void +DoStep(int xdim, int ydim, int xstart, int xend, int ystart, int yend, int *a, int *n) +{ + int x, y, c, i, j; + + for(x = xstart; x < xend; x++) { + for(y = ystart; y < yend; y++) { + c = 0; + for(i = -1; i <= 1; i++) { + for(j = -1; j <= 1; j++) { + if(x+i >= 0 && x+i < xdim && + y+j >= 0 && y+j < ydim && + (i != 0 || j != 0)) + c += a[(x+i)*xdim + (y+j)] != 0; + } + } + if(c == 3 || (c == 2 && a[x*xdim + y] != 0)) + n[x*xdim + y] = 1; + else + n[x*xdim + y] = 0; + } + } +} diff --git a/misc/cgo/life/golden.out b/misc/cgo/life/golden.out new file mode 100644 index 000000000..539d2106d --- /dev/null +++ b/misc/cgo/life/golden.out @@ -0,0 +1,17 @@ +* life + + + XXX XXX + + + + + + + + XXX XXX + + + + + diff --git a/misc/cgo/life/life.go b/misc/cgo/life/life.go new file mode 100644 index 000000000..ec000ce3a --- /dev/null +++ b/misc/cgo/life/life.go @@ -0,0 +1,39 @@ +// Copyright 2010 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package life + +// #include "life.h" +import "C" + +import "unsafe" + +func Run(gen, x, y int, a []int) { + n := make([]int, x*y) + for i := 0; i < gen; i++ { + C.Step(C.int(x), C.int(y), (*C.int)(unsafe.Pointer(&a[0])), (*C.int)(unsafe.Pointer(&n[0]))) + copy(a, n) + } +} + +// Keep the channels visible from Go. +var chans [4]chan bool + +//export GoStart +// Double return value is just for testing. +func GoStart(i, xdim, ydim, xstart, xend, ystart, yend C.int, a *C.int, n *C.int) (int, int) { + c := make(chan bool, int(C.MYCONST)) + go func() { + C.DoStep(xdim, ydim, xstart, xend, ystart, yend, a, n) + c <- true + }() + chans[i] = c + return int(i), int(i + 100) +} + +//export GoWait +func GoWait(i C.int) { + <-chans[i] + chans[i] = nil +} diff --git a/misc/cgo/life/life.h b/misc/cgo/life/life.h new file mode 100644 index 000000000..b2011b25f --- /dev/null +++ b/misc/cgo/life/life.h @@ -0,0 +1,7 @@ +// Copyright 2010 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +extern void Step(int, int, int *, int *); +extern void DoStep(int, int, int, int, int, int, int *, int *); +extern const int MYCONST; diff --git a/misc/cgo/life/main.go b/misc/cgo/life/main.go new file mode 100644 index 000000000..9cfed434b --- /dev/null +++ b/misc/cgo/life/main.go @@ -0,0 +1,44 @@ +// Copyright 2010 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Run the game of life in C using Go for parallelization. + +package main + +import ( + "flag" + "fmt" + "life" +) + +const MAXDIM = 100 + +var dim = flag.Int("dim", 16, "board dimensions") +var gen = flag.Int("gen", 10, "generations") + +func main() { + flag.Parse() + + var a [MAXDIM * MAXDIM]int + for i := 2; i < *dim; i += 8 { + for j := 2; j < *dim-3; j += 8 { + for y := 0; y < 3; y++ { + a[i**dim+j+y] = 1 + } + } + } + + life.Run(*gen, *dim, *dim, a[:]) + + for i := 0; i < *dim; i++ { + for j := 0; j < *dim; j++ { + if a[i**dim+j] == 0 { + fmt.Print(" ") + } else { + fmt.Print("X") + } + } + fmt.Print("\n") + } +} diff --git a/misc/cgo/life/test.bash b/misc/cgo/life/test.bash new file mode 100755 index 000000000..5c5fba1a9 --- /dev/null +++ b/misc/cgo/life/test.bash @@ -0,0 +1,11 @@ +#!/bin/sh +# Copyright 2010 The Go Authors. All rights reserved. +# Use of this source code is governed by a BSD-style +# license that can be found in the LICENSE file. + +set -e +gomake life +echo '*' life >run.out +./life >>run.out +diff run.out golden.out +gomake clean diff --git a/misc/cgo/stdio/Makefile b/misc/cgo/stdio/Makefile new file mode 100644 index 000000000..3f7a4c01c --- /dev/null +++ b/misc/cgo/stdio/Makefile @@ -0,0 +1,17 @@ +# Copyright 2009 The Go Authors. All rights reserved. +# Use of this source code is governed by a BSD-style +# license that can be found in the LICENSE file. + +include ../../../src/Make.inc + +TARG=stdio +CGOFILES=\ + file.go\ + +CLEANFILES+=hello fib chain run.out + +include ../../../src/Make.pkg + +%: install %.go + $(GC) $*.go + $(LD) -o $@ $*.$O diff --git a/misc/cgo/stdio/chain.go b/misc/cgo/stdio/chain.go new file mode 100644 index 000000000..c188b2dd9 --- /dev/null +++ b/misc/cgo/stdio/chain.go @@ -0,0 +1,43 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Pass numbers along a chain of threads. + +package main + +import ( + "runtime" + "stdio" + "strconv" +) + +const N = 10 +const R = 5 + +func link(left chan<- int, right <-chan int) { + // Keep the links in dedicated operating system + // threads, so that this program tests coordination + // between pthreads and not just goroutines. + runtime.LockOSThread() + for { + v := <-right + stdio.Stdout.WriteString(strconv.Itoa(v) + "\n") + left <- 1 + v + } +} + +func main() { + leftmost := make(chan int) + var left chan int + right := leftmost + for i := 0; i < N; i++ { + left, right = right, make(chan int) + go link(left, right) + } + for i := 0; i < R; i++ { + right <- 0 + x := <-leftmost + stdio.Stdout.WriteString(strconv.Itoa(x) + "\n") + } +} diff --git a/misc/cgo/stdio/fib.go b/misc/cgo/stdio/fib.go new file mode 100644 index 000000000..c02e31fd8 --- /dev/null +++ b/misc/cgo/stdio/fib.go @@ -0,0 +1,47 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Compute Fibonacci numbers with two goroutines +// that pass integers back and forth. No actual +// concurrency, just threads and synchronization +// and foreign code on multiple pthreads. + +package main + +import ( + "runtime" + "stdio" + "strconv" +) + +func fibber(c, out chan int64, i int64) { + // Keep the fibbers in dedicated operating system + // threads, so that this program tests coordination + // between pthreads and not just goroutines. + runtime.LockOSThread() + + if i == 0 { + c <- i + } + for { + j := <-c + stdio.Stdout.WriteString(strconv.Itoa64(j) + "\n") + out <- j + <-out + i += j + c <- i + } +} + +func main() { + c := make(chan int64) + out := make(chan int64) + go fibber(c, out, 0) + go fibber(c, out, 1) + <-out + for i := 0; i < 90; i++ { + out <- 1 + <-out + } +} diff --git a/misc/cgo/stdio/file.go b/misc/cgo/stdio/file.go new file mode 100644 index 000000000..ab1e88436 --- /dev/null +++ b/misc/cgo/stdio/file.go @@ -0,0 +1,45 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +/* +A trivial example of wrapping a C library in Go. +For a more complex example and explanation, +see ../gmp/gmp.go. +*/ + +package stdio + +/* +#include +#include +#include +#include + +char* greeting = "hello, world"; +*/ +import "C" +import "unsafe" + +type File C.FILE + +var Stdout = (*File)(C.stdout) +var Stderr = (*File)(C.stderr) + +// Test reference to library symbol. +// Stdout and stderr are too special to be a reliable test. +var myerr = C.sys_errlist + +func (f *File) WriteString(s string) { + p := C.CString(s) + C.fputs(p, (*C.FILE)(f)) + C.free(unsafe.Pointer(p)) + f.Flush() +} + +func (f *File) Flush() { + C.fflush((*C.FILE)(f)) +} + +var Greeting = C.GoString(C.greeting) +var Gbytes = C.GoBytes(unsafe.Pointer(C.greeting), C.int(len(Greeting))) diff --git a/misc/cgo/stdio/golden.out b/misc/cgo/stdio/golden.out new file mode 100644 index 000000000..c0e496547 --- /dev/null +++ b/misc/cgo/stdio/golden.out @@ -0,0 +1,150 @@ +* hello +hello, world +* fib +0 +1 +1 +2 +3 +5 +8 +13 +21 +34 +55 +89 +144 +233 +377 +610 +987 +1597 +2584 +4181 +6765 +10946 +17711 +28657 +46368 +75025 +121393 +196418 +317811 +514229 +832040 +1346269 +2178309 +3524578 +5702887 +9227465 +14930352 +24157817 +39088169 +63245986 +102334155 +165580141 +267914296 +433494437 +701408733 +1134903170 +1836311903 +2971215073 +4807526976 +7778742049 +12586269025 +20365011074 +32951280099 +53316291173 +86267571272 +139583862445 +225851433717 +365435296162 +591286729879 +956722026041 +1548008755920 +2504730781961 +4052739537881 +6557470319842 +10610209857723 +17167680177565 +27777890035288 +44945570212853 +72723460248141 +117669030460994 +190392490709135 +308061521170129 +498454011879264 +806515533049393 +1304969544928657 +2111485077978050 +3416454622906707 +5527939700884757 +8944394323791464 +14472334024676221 +23416728348467685 +37889062373143906 +61305790721611591 +99194853094755497 +160500643816367088 +259695496911122585 +420196140727489673 +679891637638612258 +1100087778366101931 +1779979416004714189 +2880067194370816120 +* chain +0 +1 +2 +3 +4 +5 +6 +7 +8 +9 +10 +0 +1 +2 +3 +4 +5 +6 +7 +8 +9 +10 +0 +1 +2 +3 +4 +5 +6 +7 +8 +9 +10 +0 +1 +2 +3 +4 +5 +6 +7 +8 +9 +10 +0 +1 +2 +3 +4 +5 +6 +7 +8 +9 +10 diff --git a/misc/cgo/stdio/hello.go b/misc/cgo/stdio/hello.go new file mode 100644 index 000000000..58fc6d574 --- /dev/null +++ b/misc/cgo/stdio/hello.go @@ -0,0 +1,11 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package main + +import "stdio" + +func main() { + stdio.Stdout.WriteString(stdio.Greeting + "\n") +} diff --git a/misc/cgo/stdio/test.bash b/misc/cgo/stdio/test.bash new file mode 100755 index 000000000..82e3f7b45 --- /dev/null +++ b/misc/cgo/stdio/test.bash @@ -0,0 +1,15 @@ +#!/bin/sh +# Copyright 2009 The Go Authors. All rights reserved. +# Use of this source code is governed by a BSD-style +# license that can be found in the LICENSE file. + +set -e +gomake hello fib chain +echo '*' hello >run.out +./hello >>run.out +echo '*' fib >>run.out +./fib >>run.out +echo '*' chain >>run.out +./chain >>run.out +diff run.out golden.out +gomake clean diff --git a/misc/cgo/test/Makefile b/misc/cgo/test/Makefile new file mode 100644 index 000000000..d4309be3c --- /dev/null +++ b/misc/cgo/test/Makefile @@ -0,0 +1,26 @@ +# Copyright 2011 The Go Authors. All rights reserved. +# Use of this source code is governed by a BSD-style +# license that can be found in the LICENSE file. + +include ../../../src/Make.inc + +TARG=runtime/cgotest + +CGOFILES=\ + align.go\ + basic.go\ + callback.go\ + env.go\ + exports.go\ + issue1222.go\ + issue1328.go\ + issue1560.go\ + duplicate_symbol.go\ + +CGO_OFILES=\ + callback_c.o\ + +OFILES=\ + runtime.$O\ + +include ../../../src/Make.pkg diff --git a/misc/cgo/test/align.go b/misc/cgo/test/align.go new file mode 100644 index 000000000..07ab9ef50 --- /dev/null +++ b/misc/cgo/test/align.go @@ -0,0 +1,72 @@ +package cgotest + +/* +#include + +typedef unsigned char Uint8; +typedef unsigned short Uint16; + +typedef enum { + MOD1 = 0x0000, + MODX = 0x8000 +} SDLMod; + +typedef enum { + A = 1, + B = 322, + SDLK_LAST +} SDLKey; + +typedef struct SDL_keysym { + Uint8 scancode; + SDLKey sym; + SDLMod mod; + Uint16 unicode; +} SDL_keysym; + +typedef struct SDL_KeyboardEvent { + Uint8 typ; + Uint8 which; + Uint8 state; + SDL_keysym keysym; +} SDL_KeyboardEvent; + +void makeEvent(SDL_KeyboardEvent *event) { + unsigned char *p; + int i; + + p = (unsigned char*)event; + for (i=0; ityp == typ && e->which == which && e->state == state && e->keysym.scancode == scan && e->keysym.sym == sym && e->keysym.mod == mod && e->keysym.unicode == uni; +} + +void cTest(SDL_KeyboardEvent *event) { + printf("C: %#x %#x %#x %#x %#x %#x %#x\n", event->typ, event->which, event->state, + event->keysym.scancode, event->keysym.sym, event->keysym.mod, event->keysym.unicode); + fflush(stdout); +} + +*/ +import "C" + +import ( + "testing" +) + +func testAlign(t *testing.T) { + var evt C.SDL_KeyboardEvent + C.makeEvent(&evt) + if C.same(&evt, evt.typ, evt.which, evt.state, evt.keysym.scancode, evt.keysym.sym, evt.keysym.mod, evt.keysym.unicode) == 0 { + t.Error("*** bad alignment") + C.cTest(&evt) + t.Errorf("Go: %#x %#x %#x %#x %#x %#x %#x\n", + evt.typ, evt.which, evt.state, evt.keysym.scancode, + evt.keysym.sym, evt.keysym.mod, evt.keysym.unicode) + t.Error(evt) + } +} diff --git a/misc/cgo/test/basic.go b/misc/cgo/test/basic.go new file mode 100644 index 000000000..b9d0953bd --- /dev/null +++ b/misc/cgo/test/basic.go @@ -0,0 +1,134 @@ +// Copyright 2010 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Basic test cases for cgo. + +package cgotest + +/* +#include +#include +#include +#include + +#define SHIFT(x, y) ((x)<<(y)) +#define KILO SHIFT(1, 10) + +enum E { + Enum1 = 1, + Enum2 = 2, +}; + +typedef unsigned char uuid_t[20]; + +void uuid_generate(uuid_t x) { + x[0] = 0; +} + +struct S { + int x; +}; + +extern enum E myConstFunc(struct S* const ctx, int const id, struct S **const filter); + +enum E myConstFunc(struct S *const ctx, int const id, struct S **const filter) { return 0; } + +// issue 1222 +typedef union { + long align; +} xxpthread_mutex_t; + +struct ibv_async_event { + union { + int x; + } element; +}; + +struct ibv_context { + xxpthread_mutex_t mutex; +}; +*/ +import "C" +import ( + "os" + "testing" + "unsafe" +) + +const EINVAL = C.EINVAL /* test #define */ + +var KILO = C.KILO + +func uuidgen() { + var uuid C.uuid_t + C.uuid_generate(&uuid[0]) +} + +func Size(name string) (int64, os.Error) { + var st C.struct_stat + p := C.CString(name) + _, err := C.stat(p, &st) + C.free(unsafe.Pointer(p)) + if err != nil { + return 0, err + } + return int64(C.ulong(st.st_size)), nil +} + +func Strtol(s string, base int) (int, os.Error) { + p := C.CString(s) + n, err := C.strtol(p, nil, C.int(base)) + C.free(unsafe.Pointer(p)) + return int(n), err +} + +func Atol(s string) int { + p := C.CString(s) + n := C.atol(p) + C.free(unsafe.Pointer(p)) + return int(n) +} + +func testConst(t *testing.T) { + C.myConstFunc(nil, 0, nil) +} + +func testEnum(t *testing.T) { + if C.Enum1 != 1 || C.Enum2 != 2 { + t.Error("bad enum", C.Enum1, C.Enum2) + } +} + +func testAtol(t *testing.T) { + l := Atol("123") + if l != 123 { + t.Error("Atol 123: ", l) + } +} + +func testErrno(t *testing.T) { + n, err := Strtol("asdf", 123) + if n != 0 || err != os.EINVAL { + t.Error("Strtol: ", n, err) + } +} + +func testMultipleAssign(t *testing.T) { + p := C.CString("234") + n, m := C.strtol(p, nil, 345), C.strtol(p, nil, 10) + if n != 0 || m != 234 { + t.Fatal("Strtol x2: ", n, m) + } + C.free(unsafe.Pointer(p)) +} + +var ( + uint = (C.uint)(0) + ulong C.ulong + char C.char +) + +type Context struct { + ctx *C.struct_ibv_context +} diff --git a/misc/cgo/test/callback.go b/misc/cgo/test/callback.go new file mode 100644 index 000000000..3edee9758 --- /dev/null +++ b/misc/cgo/test/callback.go @@ -0,0 +1,136 @@ +package cgotest + +/* +void callback(void *f); +void callGoFoo(void) { + extern void goFoo(void); + goFoo(); +} +*/ +import "C" + +import ( + "runtime" + "testing" + "unsafe" +) + +// nestedCall calls into C, back into Go, and finally to f. +func nestedCall(f func()) { + // NOTE: Depends on representation of f. + // callback(x) calls goCallback(x) + C.callback(*(*unsafe.Pointer)(unsafe.Pointer(&f))) +} + +//export goCallback +func goCallback(p unsafe.Pointer) { + (*(*func())(unsafe.Pointer(&p)))() +} + +func testCallback(t *testing.T) { + var x = false + nestedCall(func() { x = true }) + if !x { + t.Fatal("nestedCall did not call func") + } +} + +func testCallbackGC(t *testing.T) { + nestedCall(runtime.GC) +} + +func lockedOSThread() bool // in runtime.c + +func testCallbackPanic(t *testing.T) { + // Make sure panic during callback unwinds properly. + if lockedOSThread() { + t.Fatal("locked OS thread on entry to TestCallbackPanic") + } + defer func() { + s := recover() + if s == nil { + t.Fatal("did not panic") + } + if s.(string) != "callback panic" { + t.Fatal("wrong panic:", s) + } + if lockedOSThread() { + t.Fatal("locked OS thread on exit from TestCallbackPanic") + } + }() + nestedCall(func() { panic("callback panic") }) + panic("nestedCall returned") +} + +func testCallbackPanicLoop(t *testing.T) { + // Make sure we don't blow out m->g0 stack. + for i := 0; i < 100000; i++ { + TestCallbackPanic(t) + } +} + +func testCallbackPanicLocked(t *testing.T) { + runtime.LockOSThread() + defer runtime.UnlockOSThread() + + if !lockedOSThread() { + t.Fatal("runtime.LockOSThread didn't") + } + defer func() { + s := recover() + if s == nil { + t.Fatal("did not panic") + } + if s.(string) != "callback panic" { + t.Fatal("wrong panic:", s) + } + if !lockedOSThread() { + t.Fatal("lost lock on OS thread after panic") + } + }() + nestedCall(func() { panic("callback panic") }) + panic("nestedCall returned") +} + +// Callback with zero arguments used to make the stack misaligned, +// which broke the garbage collector and other things. +func testZeroArgCallback(t *testing.T) { + defer func() { + s := recover() + if s != nil { + t.Fatal("panic during callback:", s) + } + }() + C.callGoFoo() +} + +//export goFoo +func goFoo() { + x := 1 + for i := 0; i < 10000; i++ { + // variadic call mallocs + writes to + variadic(x, x, x) + if x != 1 { + panic("bad x") + } + } +} + +func variadic(x ...interface{}) {} + +func testBlocking(t *testing.T) { + c := make(chan int) + go func() { + for i := 0; i < 10; i++ { + c <- <-c + } + }() + nestedCall(func() { + for i := 0; i < 10; i++ { + c <- i + if j := <-c; j != i { + t.Errorf("out of sync %d != %d", j, i) + } + } + }) +} diff --git a/misc/cgo/test/callback_c.c b/misc/cgo/test/callback_c.c new file mode 100644 index 000000000..5983a5e11 --- /dev/null +++ b/misc/cgo/test/callback_c.c @@ -0,0 +1,12 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +#include +#include "_cgo_export.h" + +void +callback(void *f) +{ + goCallback(f); +} diff --git a/misc/cgo/test/cgo_test.go b/misc/cgo/test/cgo_test.go new file mode 100644 index 000000000..94fba15db --- /dev/null +++ b/misc/cgo/test/cgo_test.go @@ -0,0 +1,28 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package cgotest + +import "testing" + +// The actual test functions are in non-_test.go files +// so that they can use cgo (import "C"). +// These wrappers are here for gotest to find. + +func TestAlign(t *testing.T) { testAlign(t) } +func TestConst(t *testing.T) { testConst(t) } +func TestEnum(t *testing.T) { testEnum(t) } +func TestAtol(t *testing.T) { testAtol(t) } +func TestErrno(t *testing.T) { testErrno(t) } +func TestMultipleAssign(t *testing.T) { testMultipleAssign(t) } +func TestCallback(t *testing.T) { testCallback(t) } +func TestCallbackGC(t *testing.T) { testCallbackGC(t) } +func TestCallbackPanic(t *testing.T) { testCallbackPanic(t) } +func TestCallbackPanicLoop(t *testing.T) { testCallbackPanicLoop(t) } +func TestCallbackPanicLocked(t *testing.T) { testCallbackPanicLocked(t) } +func TestZeroArgCallback(t *testing.T) { testZeroArgCallback(t) } +func TestBlocking(t *testing.T) { testBlocking(t) } +func Test1328(t *testing.T) { test1328(t) } +func TestParallelSleep(t *testing.T) { testParallelSleep(t) } +func TestSetEnv(t *testing.T) { testSetEnv(t) } diff --git a/misc/cgo/test/duplicate_symbol.go b/misc/cgo/test/duplicate_symbol.go new file mode 100644 index 000000000..69600de9c --- /dev/null +++ b/misc/cgo/test/duplicate_symbol.go @@ -0,0 +1,21 @@ +// Copyright 2010 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// This file contains test cases for cgo. + +package cgotest + +/* +int base_symbol = 0; + +#define alias_one base_symbol +#define alias_two base_symbol +*/ +import "C" + +import "fmt" + +func duplicateSymbols() { + fmt.Printf("%v %v %v\n", C.base_symbol, C.alias_one, C.alias_two) +} diff --git a/misc/cgo/test/env.go b/misc/cgo/test/env.go new file mode 100644 index 000000000..1fb4e684c --- /dev/null +++ b/misc/cgo/test/env.go @@ -0,0 +1,32 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package cgotest + +/* +#include +*/ +import "C" +import ( + "os" + "testing" + "unsafe" +) + +// This is really an os package test but here for convenience. +func testSetEnv(t *testing.T) { + const key = "CGO_OS_TEST_KEY" + const val = "CGO_OS_TEST_VALUE" + os.Setenv(key, val) + keyc := C.CString(key) + defer C.free(unsafe.Pointer(keyc)) + v := C.getenv(keyc) + if v == (*C.char)(unsafe.Pointer(uintptr(0))) { + t.Fatal("getenv returned NULL") + } + vs := C.GoString(v) + if vs != val { + t.Fatalf("getenv() = %q; want %q", vs, val) + } +} diff --git a/misc/cgo/test/exports.go b/misc/cgo/test/exports.go new file mode 100644 index 000000000..f96c60b00 --- /dev/null +++ b/misc/cgo/test/exports.go @@ -0,0 +1,12 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package cgotest + +import "C" + +//export ReturnIntLong +func ReturnIntLong() (int, C.long) { + return 1, 2 +} diff --git a/misc/cgo/test/issue1222.go b/misc/cgo/test/issue1222.go new file mode 100644 index 000000000..c396a0c41 --- /dev/null +++ b/misc/cgo/test/issue1222.go @@ -0,0 +1,29 @@ +// Copyright 2010 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// This file contains test cases for cgo. + +package cgotest + +/* +// issue 1222 +typedef union { + long align; +} xxpthread_mutex_t; + +struct ibv_async_event { + union { + int x; + } element; +}; + +struct ibv_context { + xxpthread_mutex_t mutex; +}; +*/ +import "C" + +type AsyncEvent struct { + event C.struct_ibv_async_event +} diff --git a/misc/cgo/test/issue1328.go b/misc/cgo/test/issue1328.go new file mode 100644 index 000000000..e01207dd9 --- /dev/null +++ b/misc/cgo/test/issue1328.go @@ -0,0 +1,30 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package cgotest + +import "testing" + +// extern void BackIntoGo(void); +// void IntoC() { BackIntoGo(); } +import "C" + +//export BackIntoGo +func BackIntoGo() { + x := 1 + + for i := 0; i < 10000; i++ { + xvariadic(x) + if x != 1 { + panic("x is not 1?") + } + } +} + +func xvariadic(x ...interface{}) { +} + +func test1328(t *testing.T) { + C.IntoC() +} diff --git a/misc/cgo/test/issue1560.go b/misc/cgo/test/issue1560.go new file mode 100644 index 000000000..e534cce47 --- /dev/null +++ b/misc/cgo/test/issue1560.go @@ -0,0 +1,46 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package cgotest + +/* +#include + +extern void BackgroundSleep(int); +void twoSleep(int n) { + BackgroundSleep(n); + sleep(n); +} +*/ +import "C" + +import ( + "testing" + "time" +) + +var sleepDone = make(chan bool) + +func parallelSleep(n int) { + C.twoSleep(C.int(n)) + <-sleepDone +} + +//export BackgroundSleep +func BackgroundSleep(n int) { + go func() { + C.sleep(C.uint(n)) + sleepDone <- true + }() +} + +func testParallelSleep(t *testing.T) { + dt := -time.Nanoseconds() + parallelSleep(1) + dt += time.Nanoseconds() + // bug used to run sleeps in serial, producing a 2-second delay. + if dt >= 1.3e9 { + t.Fatalf("parallel 1-second sleeps slept for %f seconds", float64(dt)/1e9) + } +} diff --git a/misc/cgo/test/runtime.c b/misc/cgo/test/runtime.c new file mode 100644 index 000000000..e087c7622 --- /dev/null +++ b/misc/cgo/test/runtime.c @@ -0,0 +1,21 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Expose some runtime functions for testing. + +typedef char bool; + +bool runtime·lockedOSThread(void); + +static void +FLUSH(void*) +{ +} + +void +·lockedOSThread(bool b) +{ + b = runtime·lockedOSThread(); + FLUSH(&b); +} diff --git a/misc/chrome/gophertool/README.txt b/misc/chrome/gophertool/README.txt new file mode 100644 index 000000000..a7c0b4b26 --- /dev/null +++ b/misc/chrome/gophertool/README.txt @@ -0,0 +1,8 @@ +To install: + +1) chrome://extensions/ +2) click "[+] Developer Mode" in top right +3) "Load unpacked extension..." +4) pick $GOROOT/misc/chrome/gophertool + +Done. It'll now auto-reload from source. diff --git a/misc/chrome/gophertool/background.html b/misc/chrome/gophertool/background.html new file mode 100644 index 000000000..058c18142 --- /dev/null +++ b/misc/chrome/gophertool/background.html @@ -0,0 +1,24 @@ + + + + + + + + diff --git a/misc/chrome/gophertool/gopher.js b/misc/chrome/gophertool/gopher.js new file mode 100644 index 000000000..847c1c70d --- /dev/null +++ b/misc/chrome/gophertool/gopher.js @@ -0,0 +1,34 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +var numericRE = /^\d+$/; +var commitRE = /^(?:\d+:)?([0-9a-f]{6,20})$/; // e.g "8486:ab29d2698a47" or "ab29d2698a47" +var pkgRE = /^[a-z0-9_\/]+$/; + +function urlForInput(t) { + if (!t) { + return null; + } + + if (numericRE.test(t)) { + if (t < 1000000) { + return "http://code.google.com/p/go/issues/detail?id=" + t; + } + return "http://codereview.appspot.com/" + t + "/"; + } + + var match = commitRE.exec(t); + if (match) { + return "http://code.google.com/p/go/source/detail?r=" + match[1]; + } + + if (pkgRE.test(t)) { + // TODO: make this smarter, using a list of packages + substring matches. + // Get the list from godoc itself in JSON format? + // TODO: prefer localhost:6060 to golang.org if localhost:6060 is responding. + return "http://golang.org/pkg/" + t; + } + + return null; +} diff --git a/misc/chrome/gophertool/gopher.png b/misc/chrome/gophertool/gopher.png new file mode 100644 index 000000000..0d1abb741 Binary files /dev/null and b/misc/chrome/gophertool/gopher.png differ diff --git a/misc/chrome/gophertool/manifest.json b/misc/chrome/gophertool/manifest.json new file mode 100644 index 000000000..3a2540a86 --- /dev/null +++ b/misc/chrome/gophertool/manifest.json @@ -0,0 +1,17 @@ +{ + "name": "Hacking Gopher", + "version": "1.0", + "description": "Go Hacking utility", + "background_page": "background.html", + "browser_action": { + "default_icon": "gopher.png", + "popup": "popup.html" + }, + "omnibox": { "keyword": "golang" }, + "icons": { + "16": "gopher.png" + }, + "permissions": [ + "tabs" + ] +} diff --git a/misc/chrome/gophertool/popup.html b/misc/chrome/gophertool/popup.html new file mode 100644 index 000000000..ebbc71f3a --- /dev/null +++ b/misc/chrome/gophertool/popup.html @@ -0,0 +1,54 @@ + + + + + + + +issue, +codereview, +commit, or +pkg id/name: +
+Also: buildbots + + diff --git a/misc/dashboard/README b/misc/dashboard/README new file mode 100644 index 000000000..c00311ef7 --- /dev/null +++ b/misc/dashboard/README @@ -0,0 +1,26 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +The files in this directory constitute the continuous builder: + +godashboard/: an AppEngine server +builder/: gobuilder, a Go continuous build client + +If you wish to run a Go builder, please email golang-dev@googlegroups.com + +To run a builder: + +* Write the key ~gobuild/.gobuildkey + You need to get it from someone who knows the key. + You may also use a filename of the form .gobuildkey-$BUILDER if you + wish to run builders for multiple targets. + +* Append your username and password googlecode.com credentials from + https://code.google.com/hosting/settings + to the buildkey file in the format "Username\nPassword\n". + (This is for uploading tarballs to the project downloads section, + and is an optional step.) + +* Build and run gobuilder (see its documentation for command-line options). + diff --git a/misc/dashboard/builder/Makefile b/misc/dashboard/builder/Makefile new file mode 100644 index 000000000..f1d9c5497 --- /dev/null +++ b/misc/dashboard/builder/Makefile @@ -0,0 +1,14 @@ +# Copyright 2009 The Go Authors. All rights reserved. +# Use of this source code is governed by a BSD-style +# license that can be found in the LICENSE file. + +include ../../../src/Make.inc + +TARG=gobuilder +GOFILES=\ + exec.go\ + http.go\ + main.go\ + package.go\ + +include ../../../src/Make.cmd diff --git a/misc/dashboard/builder/doc.go b/misc/dashboard/builder/doc.go new file mode 100644 index 000000000..30d8fe948 --- /dev/null +++ b/misc/dashboard/builder/doc.go @@ -0,0 +1,58 @@ +// Copyright 2010 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +/* + +Go Builder is a continuous build client for the Go project. +It integrates with the Go Dashboard AppEngine application. + +Go Builder is intended to run continuously as a background process. + +It periodically pulls updates from the Go Mercurial repository. + +When a newer revision is found, Go Builder creates a clone of the repository, +runs all.bash, and reports build success or failure to the Go Dashboard. + +For a release revision (a change description that matches "release.YYYY-MM-DD"), +Go Builder will create a tar.gz archive of the GOROOT and deliver it to the +Go Google Code project's downloads section. + +Usage: + + gobuilder goos-goarch... + + Several goos-goarch combinations can be provided, and the builder will + build them in serial. + +Optional flags: + + -dashboard="godashboard.appspot.com": Go Dashboard Host + The location of the Go Dashboard application to which Go Builder will + report its results. + + -release: Build and deliver binary release archive + + -rev=N: Build revision N and exit + + -cmd="./all.bash": Build command (specify absolute or relative to go/src) + + -v: Verbose logging + + -external: External package builder mode (will not report Go build + state to dashboard or issue releases) + +The key file should be located at $HOME/.gobuildkey or, for a builder-specific +key, $HOME/.gobuildkey-$BUILDER (eg, $HOME/.gobuildkey-linux-amd64). + +The build key file is a text file of the format: + + godashboard-key + googlecode-username + googlecode-password + +If the Google Code credentials are not provided the archival step +will be skipped. + +*/ +package documentation diff --git a/misc/dashboard/builder/exec.go b/misc/dashboard/builder/exec.go new file mode 100644 index 000000000..a042c5699 --- /dev/null +++ b/misc/dashboard/builder/exec.go @@ -0,0 +1,74 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package main + +import ( + "bytes" + "exec" + "io" + "log" + "os" + "strings" +) + +// run is a simple wrapper for exec.Run/Close +func run(envv []string, dir string, argv ...string) os.Error { + if *verbose { + log.Println("run", argv) + } + argv = useBash(argv) + cmd := exec.Command(argv[0], argv[1:]...) + cmd.Dir = dir + cmd.Env = envv + cmd.Stderr = os.Stderr + return cmd.Run() +} + +// runLog runs a process and returns the combined stdout/stderr, +// as well as writing it to logfile (if specified). It returns +// process combined stdout and stderr output, exit status and error. +// The error returned is nil, if process is started successfully, +// even if exit status is not 0. +func runLog(envv []string, logfile, dir string, argv ...string) (string, int, os.Error) { + if *verbose { + log.Println("runLog", argv) + } + argv = useBash(argv) + + b := new(bytes.Buffer) + var w io.Writer = b + if logfile != "" { + f, err := os.OpenFile(logfile, os.O_WRONLY|os.O_CREATE|os.O_APPEND, 0666) + if err != nil { + return "", 0, err + } + defer f.Close() + w = io.MultiWriter(f, b) + } + + cmd := exec.Command(argv[0], argv[1:]...) + cmd.Dir = dir + cmd.Env = envv + cmd.Stdout = w + cmd.Stderr = w + + err := cmd.Run() + if err != nil { + if ws, ok := err.(*os.Waitmsg); ok { + return b.String(), ws.ExitStatus(), nil + } + } + return b.String(), 0, nil +} + +// useBash prefixes a list of args with 'bash' if the first argument +// is a bash script. +func useBash(argv []string) []string { + // TODO(brainman): choose a more reliable heuristic here. + if strings.HasSuffix(argv[0], ".bash") { + argv = append([]string{"bash"}, argv...) + } + return argv +} diff --git a/misc/dashboard/builder/http.go b/misc/dashboard/builder/http.go new file mode 100644 index 000000000..abef8faa4 --- /dev/null +++ b/misc/dashboard/builder/http.go @@ -0,0 +1,148 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package main + +import ( + "bytes" + "fmt" + "http" + "json" + "log" + "os" + "strconv" + "url" +) + +type param map[string]string + +// dash runs the given method and command on the dashboard. +// If args is not nil, it is the query or post parameters. +// If resp is not nil, dash unmarshals the body as JSON into resp. +func dash(meth, cmd string, resp interface{}, args param) os.Error { + var r *http.Response + var err os.Error + if *verbose { + log.Println("dash", cmd, args) + } + cmd = "http://" + *dashboard + "/" + cmd + vals := make(url.Values) + for k, v := range args { + vals.Add(k, v) + } + switch meth { + case "GET": + if q := vals.Encode(); q != "" { + cmd += "?" + q + } + r, err = http.Get(cmd) + case "POST": + r, err = http.PostForm(cmd, vals) + default: + return fmt.Errorf("unknown method %q", meth) + } + if err != nil { + return err + } + defer r.Body.Close() + var buf bytes.Buffer + buf.ReadFrom(r.Body) + if resp != nil { + if err = json.Unmarshal(buf.Bytes(), resp); err != nil { + log.Printf("json unmarshal %#q: %s\n", buf.Bytes(), err) + return err + } + } + return nil +} + +func dashStatus(meth, cmd string, args param) os.Error { + var resp struct { + Status string + Error string + } + err := dash(meth, cmd, &resp, args) + if err != nil { + return err + } + if resp.Status != "OK" { + return os.NewError("/build: " + resp.Error) + } + return nil +} + +// todo returns the next hash to build. +func (b *Builder) todo() (rev string, err os.Error) { + var resp []struct { + Hash string + } + if err = dash("GET", "todo", &resp, param{"builder": b.name}); err != nil { + return + } + if len(resp) > 0 { + rev = resp[0].Hash + } + return +} + +// recordResult sends build results to the dashboard +func (b *Builder) recordResult(buildLog string, hash string) os.Error { + return dash("POST", "build", nil, param{ + "builder": b.name, + "key": b.key, + "node": hash, + "log": buildLog, + }) +} + +// packages fetches a list of package paths from the dashboard +func packages() (pkgs []string, err os.Error) { + var resp struct { + Packages []struct { + Path string + } + } + err = dash("GET", "package", &resp, param{"fmt": "json"}) + if err != nil { + return + } + for _, p := range resp.Packages { + pkgs = append(pkgs, p.Path) + } + return +} + +// updatePackage sends package build results and info dashboard +func (b *Builder) updatePackage(pkg string, ok bool, buildLog, info string) os.Error { + return dash("POST", "package", nil, param{ + "builder": b.name, + "key": b.key, + "path": pkg, + "ok": strconv.Btoa(ok), + "log": buildLog, + "info": info, + }) +} + +// postCommit informs the dashboard of a new commit +func postCommit(key string, l *HgLog) os.Error { + return dashStatus("POST", "commit", param{ + "key": key, + "node": l.Hash, + "date": l.Date, + "user": l.Author, + "parent": l.Parent, + "desc": l.Desc, + }) +} + +// dashboardCommit returns true if the dashboard knows about hash. +func dashboardCommit(hash string) bool { + err := dashStatus("GET", "commit", param{"node": hash}) + if err != nil { + log.Printf("check %s: %s", hash, err) + return false + } + return true +} diff --git a/misc/dashboard/builder/main.go b/misc/dashboard/builder/main.go new file mode 100644 index 000000000..989965bc4 --- /dev/null +++ b/misc/dashboard/builder/main.go @@ -0,0 +1,624 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package main + +import ( + "flag" + "fmt" + "io/ioutil" + "log" + "os" + "path" + "regexp" + "runtime" + "strconv" + "strings" + "time" + "xml" +) + +const ( + codeProject = "go" + codePyScript = "misc/dashboard/googlecode_upload.py" + hgUrl = "https://go.googlecode.com/hg/" + waitInterval = 30e9 // time to wait before checking for new revs + mkdirPerm = 0750 + pkgBuildInterval = 1e9 * 60 * 60 * 24 // rebuild packages every 24 hours +) + +// These variables are copied from the gobuilder's environment +// to the envv of its subprocesses. +var extraEnv = []string{ + "GOHOSTOS", + "GOHOSTARCH", + "PATH", + "DISABLE_NET_TESTS", + "MAKEFLAGS", + "GOARM", +} + +type Builder struct { + name string + goos, goarch string + key string + codeUsername string + codePassword string +} + +var ( + buildroot = flag.String("buildroot", path.Join(os.TempDir(), "gobuilder"), "Directory under which to build") + commitFlag = flag.Bool("commit", false, "upload information about new commits") + dashboard = flag.String("dashboard", "godashboard.appspot.com", "Go Dashboard Host") + buildRelease = flag.Bool("release", false, "Build and upload binary release archives") + buildRevision = flag.String("rev", "", "Build specified revision and exit") + buildCmd = flag.String("cmd", "./all.bash", "Build command (specify absolute or relative to go/src/)") + external = flag.Bool("external", false, "Build external packages") + parallel = flag.Bool("parallel", false, "Build multiple targets in parallel") + verbose = flag.Bool("v", false, "verbose") +) + +var ( + goroot string + binaryTagRe = regexp.MustCompile(`^(release\.r|weekly\.)[0-9\-.]+`) + releaseRe = regexp.MustCompile(`^release\.r[0-9\-.]+`) +) + +func main() { + flag.Usage = func() { + fmt.Fprintf(os.Stderr, "usage: %s goos-goarch...\n", os.Args[0]) + flag.PrintDefaults() + os.Exit(2) + } + flag.Parse() + if len(flag.Args()) == 0 && !*commitFlag { + flag.Usage() + } + goroot = path.Join(*buildroot, "goroot") + builders := make([]*Builder, len(flag.Args())) + for i, builder := range flag.Args() { + b, err := NewBuilder(builder) + if err != nil { + log.Fatal(err) + } + builders[i] = b + } + + // set up work environment + if err := os.RemoveAll(*buildroot); err != nil { + log.Fatalf("Error removing build root (%s): %s", *buildroot, err) + } + if err := os.Mkdir(*buildroot, mkdirPerm); err != nil { + log.Fatalf("Error making build root (%s): %s", *buildroot, err) + } + if err := run(nil, *buildroot, "hg", "clone", hgUrl, goroot); err != nil { + log.Fatal("Error cloning repository:", err) + } + + if *commitFlag { + if len(flag.Args()) == 0 { + commitWatcher() + return + } + go commitWatcher() + } + + // if specified, build revision and return + if *buildRevision != "" { + hash, err := fullHash(*buildRevision) + if err != nil { + log.Fatal("Error finding revision: ", err) + } + for _, b := range builders { + if err := b.buildHash(hash); err != nil { + log.Println(err) + } + } + return + } + + // external package build mode + if *external { + if len(builders) != 1 { + log.Fatal("only one goos-goarch should be specified with -external") + } + builders[0].buildExternal() + } + + // go continuous build mode (default) + // check for new commits and build them + for { + built := false + t := time.Nanoseconds() + if *parallel { + done := make(chan bool) + for _, b := range builders { + go func(b *Builder) { + done <- b.build() + }(b) + } + for _ = range builders { + built = <-done || built + } + } else { + for _, b := range builders { + built = b.build() || built + } + } + // sleep if there was nothing to build + if !built { + time.Sleep(waitInterval) + } + // sleep if we're looping too fast. + t1 := time.Nanoseconds() - t + if t1 < waitInterval { + time.Sleep(waitInterval - t1) + } + } +} + +func NewBuilder(builder string) (*Builder, os.Error) { + b := &Builder{name: builder} + + // get goos/goarch from builder string + s := strings.SplitN(builder, "-", 3) + if len(s) >= 2 { + b.goos, b.goarch = s[0], s[1] + } else { + return nil, fmt.Errorf("unsupported builder form: %s", builder) + } + + // read keys from keyfile + fn := path.Join(os.Getenv("HOME"), ".gobuildkey") + if s := fn + "-" + b.name; isFile(s) { // builder-specific file + fn = s + } + c, err := ioutil.ReadFile(fn) + if err != nil { + return nil, fmt.Errorf("readKeys %s (%s): %s", b.name, fn, err) + } + v := strings.Split(string(c), "\n") + b.key = v[0] + if len(v) >= 3 { + b.codeUsername, b.codePassword = v[1], v[2] + } + + return b, nil +} + +// buildExternal downloads and builds external packages, and +// reports their build status to the dashboard. +// It will re-build all packages after pkgBuildInterval nanoseconds or +// a new release tag is found. +func (b *Builder) buildExternal() { + var prevTag string + var nextBuild int64 + for { + time.Sleep(waitInterval) + err := run(nil, goroot, "hg", "pull", "-u") + if err != nil { + log.Println("hg pull failed:", err) + continue + } + hash, tag, err := firstTag(releaseRe) + if err != nil { + log.Println(err) + continue + } + if *verbose { + log.Println("latest release:", tag) + } + // don't rebuild if there's no new release + // and it's been less than pkgBuildInterval + // nanoseconds since the last build. + if tag == prevTag && time.Nanoseconds() < nextBuild { + continue + } + // build will also build the packages + if err := b.buildHash(hash); err != nil { + log.Println(err) + continue + } + prevTag = tag + nextBuild = time.Nanoseconds() + pkgBuildInterval + } +} + +// build checks for a new commit for this builder +// and builds it if one is found. +// It returns true if a build was attempted. +func (b *Builder) build() bool { + defer func() { + err := recover() + if err != nil { + log.Println(b.name, "build:", err) + } + }() + hash, err := b.todo() + if err != nil { + log.Println(err) + return false + } + if hash == "" { + return false + } + // Look for hash locally before running hg pull. + + if _, err := fullHash(hash[:12]); err != nil { + // Don't have hash, so run hg pull. + if err := run(nil, goroot, "hg", "pull"); err != nil { + log.Println("hg pull failed:", err) + return false + } + } + err = b.buildHash(hash) + if err != nil { + log.Println(err) + } + return true +} + +func (b *Builder) buildHash(hash string) (err os.Error) { + defer func() { + if err != nil { + err = fmt.Errorf("%s build: %s: %s", b.name, hash, err) + } + }() + + log.Println(b.name, "building", hash) + + // create place in which to do work + workpath := path.Join(*buildroot, b.name+"-"+hash[:12]) + err = os.Mkdir(workpath, mkdirPerm) + if err != nil { + return + } + defer os.RemoveAll(workpath) + + // clone repo + err = run(nil, workpath, "hg", "clone", goroot, "go") + if err != nil { + return + } + + // update to specified revision + err = run(nil, path.Join(workpath, "go"), + "hg", "update", hash) + if err != nil { + return + } + + srcDir := path.Join(workpath, "go", "src") + + // build + logfile := path.Join(workpath, "build.log") + buildLog, status, err := runLog(b.envv(), logfile, srcDir, *buildCmd) + if err != nil { + return fmt.Errorf("%s: %s", *buildCmd, err) + } + + // if we're in external mode, build all packages and return + if *external { + if status != 0 { + return os.NewError("go build failed") + } + return b.buildPackages(workpath, hash) + } + + if status != 0 { + // record failure + return b.recordResult(buildLog, hash) + } + + // record success + if err = b.recordResult("", hash); err != nil { + return fmt.Errorf("recordResult: %s", err) + } + + // finish here if codeUsername and codePassword aren't set + if b.codeUsername == "" || b.codePassword == "" || !*buildRelease { + return + } + + // if this is a release, create tgz and upload to google code + releaseHash, release, err := firstTag(binaryTagRe) + if hash == releaseHash { + // clean out build state + err = run(b.envv(), srcDir, "./clean.bash", "--nopkg") + if err != nil { + return fmt.Errorf("clean.bash: %s", err) + } + // upload binary release + fn := fmt.Sprintf("go.%s.%s-%s.tar.gz", release, b.goos, b.goarch) + err = run(nil, workpath, "tar", "czf", fn, "go") + if err != nil { + return fmt.Errorf("tar: %s", err) + } + err = run(nil, workpath, path.Join(goroot, codePyScript), + "-s", release, + "-p", codeProject, + "-u", b.codeUsername, + "-w", b.codePassword, + "-l", fmt.Sprintf("%s,%s", b.goos, b.goarch), + fn) + } + + return +} + +// envv returns an environment for build/bench execution +func (b *Builder) envv() []string { + if runtime.GOOS == "windows" { + return b.envvWindows() + } + e := []string{ + "GOOS=" + b.goos, + "GOARCH=" + b.goarch, + "GOROOT_FINAL=/usr/local/go", + } + for _, k := range extraEnv { + s, err := os.Getenverror(k) + if err == nil { + e = append(e, k+"="+s) + } + } + return e +} + +// windows version of envv +func (b *Builder) envvWindows() []string { + start := map[string]string{ + "GOOS": b.goos, + "GOARCH": b.goarch, + "GOROOT_FINAL": "/c/go", + // TODO(brainman): remove once we find make that does not hang. + "MAKEFLAGS": "-j1", + } + for _, name := range extraEnv { + s, err := os.Getenverror(name) + if err == nil { + start[name] = s + } + } + skip := map[string]bool{ + "GOBIN": true, + "GOROOT": true, + "INCLUDE": true, + "LIB": true, + } + var e []string + for name, v := range start { + e = append(e, name+"="+v) + skip[name] = true + } + for _, kv := range os.Environ() { + s := strings.SplitN(kv, "=", 2) + name := strings.ToUpper(s[0]) + switch { + case name == "": + // variables, like "=C:=C:\", just copy them + e = append(e, kv) + case !skip[name]: + e = append(e, kv) + skip[name] = true + } + } + return e +} + +func isDirectory(name string) bool { + s, err := os.Stat(name) + return err == nil && s.IsDirectory() +} + +func isFile(name string) bool { + s, err := os.Stat(name) + return err == nil && (s.IsRegular() || s.IsSymlink()) +} + +// commitWatcher polls hg for new commits and tells the dashboard about them. +func commitWatcher() { + // Create builder just to get master key. + b, err := NewBuilder("mercurial-commit") + if err != nil { + log.Fatal(err) + } + for { + if *verbose { + log.Printf("poll...") + } + commitPoll(b.key) + if *verbose { + log.Printf("sleep...") + } + time.Sleep(60e9) + } +} + +// HgLog represents a single Mercurial revision. +type HgLog struct { + Hash string + Author string + Date string + Desc string + Parent string + + // Internal metadata + added bool +} + +// logByHash is a cache of all Mercurial revisions we know about, +// indexed by full hash. +var logByHash = map[string]*HgLog{} + +// xmlLogTemplate is a template to pass to Mercurial to make +// hg log print the log in valid XML for parsing with xml.Unmarshal. +const xmlLogTemplate = ` + + {node|escape} + {parent|escape} + {author|escape} + {date} + {desc|escape} + +` + +// commitPoll pulls any new revisions from the hg server +// and tells the server about them. +func commitPoll(key string) { + // Catch unexpected panics. + defer func() { + if err := recover(); err != nil { + log.Printf("commitPoll panic: %s", err) + } + }() + + if err := run(nil, goroot, "hg", "pull"); err != nil { + log.Printf("hg pull: %v", err) + return + } + + const N = 20 // how many revisions to grab + + data, _, err := runLog(nil, "", goroot, "hg", "log", + "--encoding=utf-8", + "--limit="+strconv.Itoa(N), + "--template="+xmlLogTemplate, + ) + if err != nil { + log.Printf("hg log: %v", err) + return + } + + var logStruct struct { + Log []HgLog + } + err = xml.Unmarshal(strings.NewReader(""+data+""), &logStruct) + if err != nil { + log.Printf("unmarshal hg log: %v", err) + return + } + + logs := logStruct.Log + + // Pass 1. Fill in parents and add new log entries to logsByHash. + // Empty parent means take parent from next log entry. + // Non-empty parent has form 1234:hashhashhash; we want full hash. + for i := range logs { + l := &logs[i] + log.Printf("hg log: %s < %s\n", l.Hash, l.Parent) + if l.Parent == "" && i+1 < len(logs) { + l.Parent = logs[i+1].Hash + } else if l.Parent != "" { + l.Parent, _ = fullHash(l.Parent) + } + if l.Parent == "" { + // Can't create node without parent. + continue + } + + if logByHash[l.Hash] == nil { + // Make copy to avoid pinning entire slice when only one entry is new. + t := *l + logByHash[t.Hash] = &t + } + } + + for i := range logs { + l := &logs[i] + if l.Parent == "" { + continue + } + addCommit(l.Hash, key) + } +} + +// addCommit adds the commit with the named hash to the dashboard. +// key is the secret key for authentication to the dashboard. +// It avoids duplicate effort. +func addCommit(hash, key string) bool { + l := logByHash[hash] + if l == nil { + return false + } + if l.added { + return true + } + + // Check for already added, perhaps in an earlier run. + if dashboardCommit(hash) { + log.Printf("%s already on dashboard\n", hash) + // Record that this hash is on the dashboard, + // as must be all its parents. + for l != nil { + l.added = true + l = logByHash[l.Parent] + } + return true + } + + // Create parent first, to maintain some semblance of order. + if !addCommit(l.Parent, key) { + return false + } + + // Create commit. + if err := postCommit(key, l); err != nil { + log.Printf("failed to add %s to dashboard: %v", key, err) + return false + } + return true +} + +// fullHash returns the full hash for the given Mercurial revision. +func fullHash(rev string) (hash string, err os.Error) { + defer func() { + if err != nil { + err = fmt.Errorf("fullHash: %s: %s", rev, err) + } + }() + s, _, err := runLog(nil, "", goroot, + "hg", "log", + "--encoding=utf-8", + "--rev="+rev, + "--limit=1", + "--template={node}", + ) + if err != nil { + return + } + s = strings.TrimSpace(s) + if s == "" { + return "", fmt.Errorf("cannot find revision") + } + if len(s) != 40 { + return "", fmt.Errorf("hg returned invalid hash " + s) + } + return s, nil +} + +var revisionRe = regexp.MustCompile(`^([^ ]+) +[0-9]+:([0-9a-f]+)$`) + +// firstTag returns the hash and tag of the most recent tag matching re. +func firstTag(re *regexp.Regexp) (hash string, tag string, err os.Error) { + o, _, err := runLog(nil, "", goroot, "hg", "tags") + for _, l := range strings.Split(o, "\n") { + if l == "" { + continue + } + s := revisionRe.FindStringSubmatch(l) + if s == nil { + err = os.NewError("couldn't find revision number") + return + } + if !re.MatchString(s[1]) { + continue + } + tag = s[1] + hash, err = fullHash(s[2]) + return + } + err = os.NewError("no matching tag found") + return +} diff --git a/misc/dashboard/builder/package.go b/misc/dashboard/builder/package.go new file mode 100644 index 000000000..b2a83fa13 --- /dev/null +++ b/misc/dashboard/builder/package.go @@ -0,0 +1,94 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package main + +import ( + "go/doc" + "go/parser" + "go/token" + "log" + "os" + "path/filepath" + "strings" +) + +const MaxCommentLength = 500 // App Engine won't store more in a StringProperty. + +func (b *Builder) buildPackages(workpath string, hash string) os.Error { + pkgs, err := packages() + if err != nil { + return err + } + for _, p := range pkgs { + goroot := filepath.Join(workpath, "go") + gobin := filepath.Join(goroot, "bin") + goinstall := filepath.Join(gobin, "goinstall") + envv := append(b.envv(), "GOROOT="+goroot) + + // add GOBIN to path + for i, v := range envv { + if strings.HasPrefix(v, "PATH=") { + p := filepath.SplitList(v[5:]) + p = append([]string{gobin}, p...) + s := strings.Join(p, string(filepath.ListSeparator)) + envv[i] = "PATH=" + s + } + } + + // goinstall + buildLog, code, err := runLog(envv, "", goroot, goinstall, "-dashboard=false", p) + if err != nil { + log.Printf("goinstall %v: %v", p, err) + } + + // get doc comment from package source + info, err := packageComment(p, filepath.Join(goroot, "src", "pkg", p)) + if err != nil { + log.Printf("packageComment %v: %v", p, err) + } + + // update dashboard with build state + info + err = b.updatePackage(p, code == 0, buildLog, info) + if err != nil { + log.Printf("updatePackage %v: %v", p, err) + } + } + return nil +} + +func isGoFile(fi *os.FileInfo) bool { + return fi.IsRegular() && // exclude directories + !strings.HasPrefix(fi.Name, ".") && // ignore .files + filepath.Ext(fi.Name) == ".go" +} + +func packageComment(pkg, pkgpath string) (info string, err os.Error) { + fset := token.NewFileSet() + pkgs, err := parser.ParseDir(fset, pkgpath, isGoFile, parser.PackageClauseOnly|parser.ParseComments) + if err != nil { + return + } + for name := range pkgs { + if name == "main" { + continue + } + if info != "" { + return "", os.NewError("multiple non-main package docs") + } + pdoc := doc.NewPackageDoc(pkgs[name], pkg) + info = pdoc.Doc + } + // grab only first paragraph + if parts := strings.SplitN(info, "\n\n", 2); len(parts) > 1 { + info = parts[0] + } + // replace newlines with spaces + info = strings.Replace(info, "\n", " ", -1) + // truncate + if len(info) > MaxCommentLength { + info = info[:MaxCommentLength] + } + return +} diff --git a/misc/dashboard/godashboard/_multiprocessing.py b/misc/dashboard/godashboard/_multiprocessing.py new file mode 100644 index 000000000..8c66c0659 --- /dev/null +++ b/misc/dashboard/godashboard/_multiprocessing.py @@ -0,0 +1,5 @@ +# Copyright 2009 The Go Authors. All rights reserved. +# Use of this source code is governed by a BSD-style +# license that can be found in the LICENSE file. + +import multiprocessing diff --git a/misc/dashboard/godashboard/app.yaml b/misc/dashboard/godashboard/app.yaml new file mode 100644 index 000000000..7b77a85cc --- /dev/null +++ b/misc/dashboard/godashboard/app.yaml @@ -0,0 +1,25 @@ +application: godashboard +version: 7 +runtime: python +api_version: 1 + +handlers: +- url: /favicon\.ico + static_files: static/favicon.ico + upload: static/favicon\.ico + +- url: /static + static_dir: static + +- url: /package + script: package.py + +- url: /package/daily + script: package.py + login: admin + +- url: /project.* + script: package.py + +- url: /.* + script: gobuild.py diff --git a/misc/dashboard/godashboard/auth.py b/misc/dashboard/godashboard/auth.py new file mode 100644 index 000000000..73a54c0d4 --- /dev/null +++ b/misc/dashboard/godashboard/auth.py @@ -0,0 +1,13 @@ +# Copyright 2011 The Go Authors. All rights reserved. +# Use of this source code is governed by a BSD-style +# license that can be found in the LICENSE file. + +import hmac + +# local imports +import key + +def auth(req): + k = req.get('key') + return k == hmac.new(key.accessKey, req.get('builder')).hexdigest() or k == key.accessKey + diff --git a/misc/dashboard/godashboard/const.py b/misc/dashboard/godashboard/const.py new file mode 100644 index 000000000..b0110c635 --- /dev/null +++ b/misc/dashboard/godashboard/const.py @@ -0,0 +1,13 @@ +# Copyright 2011 The Go Authors. All rights reserved. +# Use of this source code is governed by a BSD-style +# license that can be found in the LICENSE file. + +mail_from = "Go Dashboard " + +mail_submit_to = "adg@golang.org" +mail_submit_subject = "New Project Submitted" + +mail_fail_to = "golang-dev@googlegroups.com" +mail_fail_reply_to = "golang-dev@googlegroups.com" +mail_fail_subject = "%s broken by %s" + diff --git a/misc/dashboard/godashboard/cron.yaml b/misc/dashboard/godashboard/cron.yaml new file mode 100644 index 000000000..953b6a1cd --- /dev/null +++ b/misc/dashboard/godashboard/cron.yaml @@ -0,0 +1,4 @@ +cron: +- description: daily package maintenance + url: /package/daily + schedule: every 24 hours diff --git a/misc/dashboard/godashboard/fail-notify.txt b/misc/dashboard/godashboard/fail-notify.txt new file mode 100644 index 000000000..a699005ea --- /dev/null +++ b/misc/dashboard/godashboard/fail-notify.txt @@ -0,0 +1,6 @@ +Change {{node}} broke the {{builder}} build: +http://godashboard.appspot.com/log/{{loghash}} + +{{desc}} + +http://code.google.com/p/go/source/detail?r={{node}} diff --git a/misc/dashboard/godashboard/gobuild.py b/misc/dashboard/godashboard/gobuild.py new file mode 100644 index 000000000..685dc83a9 --- /dev/null +++ b/misc/dashboard/godashboard/gobuild.py @@ -0,0 +1,558 @@ +# Copyright 2009 The Go Authors. All rights reserved. +# Use of this source code is governed by a BSD-style +# license that can be found in the LICENSE file. + +# This is the server part of the continuous build system for Go. It must be run +# by AppEngine. + +from django.utils import simplejson +from google.appengine.api import mail +from google.appengine.api import memcache +from google.appengine.ext import db +from google.appengine.ext import webapp +from google.appengine.ext.webapp import template +from google.appengine.ext.webapp.util import run_wsgi_app +import datetime +import hashlib +import logging +import os +import re +import bz2 + +# local imports +from auth import auth +import const + +# The majority of our state are commit objects. One of these exists for each of +# the commits known to the build system. Their key names are of the form +# "-" . This means that a sorting by the key +# name is sufficient to order the commits. +# +# The commit numbers are purely local. They need not match up to the commit +# numbers in an hg repo. When inserting a new commit, the parent commit must be +# given and this is used to generate the new commit number. In order to create +# the first Commit object, a special command (/init) is used. +class Commit(db.Model): + num = db.IntegerProperty() # internal, monotonic counter. + node = db.StringProperty() # Hg hash + parentnode = db.StringProperty() # Hg hash + user = db.StringProperty() + date = db.DateTimeProperty() + desc = db.BlobProperty() + + # This is the list of builds. Each element is a string of the form '`' . If the log hash is empty, then the build was + # successful. + builds = db.StringListProperty() + + fail_notification_sent = db.BooleanProperty() + +# A CompressedLog contains the textual build log of a failed build. +# The key name is the hex digest of the SHA256 hash of the contents. +# The contents is bz2 compressed. +class CompressedLog(db.Model): + log = db.BlobProperty() + +N = 30 + +def builderInfo(b): + f = b.split('-', 3) + goos = f[0] + goarch = f[1] + note = "" + if len(f) > 2: + note = f[2] + return {'name': b, 'goos': goos, 'goarch': goarch, 'note': note} + +def builderset(): + q = Commit.all() + q.order('-__key__') + results = q.fetch(N) + builders = set() + for c in results: + builders.update(set(parseBuild(build)['builder'] for build in c.builds)) + return builders + +class MainPage(webapp.RequestHandler): + def get(self): + self.response.headers['Content-Type'] = 'text/html; charset=utf-8' + + try: + page = int(self.request.get('p', 1)) + if not page > 0: + raise + except: + page = 1 + + try: + num = int(self.request.get('n', N)) + if num <= 0 or num > 200: + raise + except: + num = N + + offset = (page-1) * num + + q = Commit.all() + q.order('-__key__') + results = q.fetch(num, offset) + + revs = [toRev(r) for r in results] + builders = {} + + for r in revs: + for b in r['builds']: + builders[b['builder']] = builderInfo(b['builder']) + + for r in revs: + have = set(x['builder'] for x in r['builds']) + need = set(builders.keys()).difference(have) + for n in need: + r['builds'].append({'builder': n, 'log':'', 'ok': False}) + r['builds'].sort(cmp = byBuilder) + + builders = list(builders.items()) + builders.sort() + values = {"revs": revs, "builders": [v for k,v in builders]} + + values['num'] = num + values['prev'] = page - 1 + if len(results) == num: + values['next'] = page + 1 + + path = os.path.join(os.path.dirname(__file__), 'main.html') + self.response.out.write(template.render(path, values)) + +# A DashboardHandler is a webapp.RequestHandler but provides +# authenticated_post - called by post after authenticating +# json - writes object in json format to response output +class DashboardHandler(webapp.RequestHandler): + def post(self): + if not auth(self.request): + self.response.set_status(403) + return + self.authenticated_post() + + def authenticated_post(self): + return + + def json(self, obj): + self.response.set_status(200) + simplejson.dump(obj, self.response.out) + return + +# Todo serves /todo. It tells the builder which commits need to be built. +class Todo(DashboardHandler): + def get(self): + builder = self.request.get('builder') + key = 'todo-%s' % builder + response = memcache.get(key) + if response is None: + # Fell out of memcache. Rebuild from datastore results. + # We walk the commit list looking for nodes that have not + # been built by this builder. + q = Commit.all() + q.order('-__key__') + todo = [] + first = None + for c in q.fetch(N+1): + if first is None: + first = c + if not built(c, builder): + todo.append({'Hash': c.node}) + response = simplejson.dumps(todo) + memcache.set(key, response, 3600) + self.response.set_status(200) + self.response.out.write(response) + +def built(c, builder): + for b in c.builds: + if b.startswith(builder+'`'): + return True + return False + +# Log serves /log/. It retrieves log data by content hash. +class LogHandler(DashboardHandler): + def get(self): + self.response.headers['Content-Type'] = 'text/plain; charset=utf-8' + hash = self.request.path[5:] + l = CompressedLog.get_by_key_name(hash) + if l is None: + self.response.set_status(404) + return + log = bz2.decompress(l.log) + self.response.set_status(200) + self.response.out.write(log) + +# Init creates the commit with id 0. Since this commit doesn't have a parent, +# it cannot be created by Build. +class Init(DashboardHandler): + def authenticated_post(self): + date = parseDate(self.request.get('date')) + node = self.request.get('node') + if not validNode(node) or date is None: + logging.error("Not valid node ('%s') or bad date (%s %s)", node, date, self.request.get('date')) + self.response.set_status(500) + return + + commit = Commit(key_name = '00000000-%s' % node) + commit.num = 0 + commit.node = node + commit.parentnode = '' + commit.user = self.request.get('user').encode('utf8') + commit.date = date + commit.desc = self.request.get('desc').encode('utf8') + + commit.put() + + self.response.set_status(200) + +# The last commit when we switched to using entity groups. +# This is the root of the new commit entity group. +RootCommitKeyName = '00000f26-f32c6f1038207c55d5780231f7484f311020747e' + +# CommitHandler serves /commit. +# A GET of /commit retrieves information about the specified commit. +# A POST of /commit creates a node for the given commit. +# If the commit already exists, the POST silently succeeds (like mkdir -p). +class CommitHandler(DashboardHandler): + def get(self): + node = self.request.get('node') + if not validNode(node): + return self.json({'Status': 'FAIL', 'Error': 'malformed node hash'}) + n = nodeByHash(node) + if n is None: + return self.json({'Status': 'FAIL', 'Error': 'unknown revision'}) + return self.json({'Status': 'OK', 'Node': nodeObj(n)}) + + def authenticated_post(self): + # Require auth with the master key, not a per-builder key. + if self.request.get('builder'): + self.response.set_status(403) + return + + node = self.request.get('node') + date = parseDate(self.request.get('date')) + user = self.request.get('user').encode('utf8') + desc = self.request.get('desc').encode('utf8') + parenthash = self.request.get('parent') + + if not validNode(node) or not validNode(parenthash) or date is None: + return self.json({'Status': 'FAIL', 'Error': 'malformed node, parent, or date'}) + + n = nodeByHash(node) + if n is None: + p = nodeByHash(parenthash) + if p is None: + return self.json({'Status': 'FAIL', 'Error': 'unknown parent'}) + + # Want to create new node in a transaction so that multiple + # requests creating it do not collide and so that multiple requests + # creating different nodes get different sequence numbers. + # All queries within a transaction must include an ancestor, + # but the original datastore objects we used for the dashboard + # have no common ancestor. Instead, we use a well-known + # root node - the last one before we switched to entity groups - + # as the as the common ancestor. + root = Commit.get_by_key_name(RootCommitKeyName) + + def add_commit(): + if nodeByHash(node, ancestor=root) is not None: + return + + # Determine number for this commit. + # Once we have created one new entry it will be lastRooted.num+1, + # but the very first commit created in this scheme will have to use + # last.num's number instead (last is likely not rooted). + q = Commit.all() + q.order('-__key__') + q.ancestor(root) + last = q.fetch(1)[0] + num = last.num+1 + + n = Commit(key_name = '%08x-%s' % (num, node), parent = root) + n.num = num + n.node = node + n.parentnode = parenthash + n.user = user + n.date = date + n.desc = desc + n.put() + db.run_in_transaction(add_commit) + n = nodeByHash(node) + if n is None: + return self.json({'Status': 'FAIL', 'Error': 'failed to create commit node'}) + + return self.json({'Status': 'OK', 'Node': nodeObj(n)}) + +# Build serves /build. +# A POST to /build records a new build result. +class Build(webapp.RequestHandler): + def post(self): + if not auth(self.request): + self.response.set_status(403) + return + + builder = self.request.get('builder') + log = self.request.get('log').encode('utf-8') + + loghash = '' + if len(log) > 0: + loghash = hashlib.sha256(log).hexdigest() + l = CompressedLog(key_name=loghash) + l.log = bz2.compress(log) + l.put() + + node = self.request.get('node') + if not validNode(node): + logging.error('Invalid node %s' % (node)) + self.response.set_status(500) + return + + n = nodeByHash(node) + if n is None: + logging.error('Cannot find node %s' % (node)) + self.response.set_status(404) + return + nn = n + + def add_build(): + n = nodeByHash(node, ancestor=nn) + if n is None: + logging.error('Cannot find hash in add_build: %s %s' % (builder, node)) + return + + s = '%s`%s' % (builder, loghash) + for i, b in enumerate(n.builds): + if b.split('`', 1)[0] == builder: + # logging.error('Found result for %s %s already' % (builder, node)) + n.builds[i] = s + break + else: + # logging.error('Added result for %s %s' % (builder, node)) + n.builds.append(s) + n.put() + + db.run_in_transaction(add_build) + + key = 'todo-%s' % builder + memcache.delete(key) + + c = getBrokenCommit(node, builder) + if c is not None and not c.fail_notification_sent: + notifyBroken(c, builder) + + self.response.set_status(200) + + +def getBrokenCommit(node, builder): + """ + getBrokenCommit returns a Commit that breaks the build. + The Commit will be either the one specified by node or the one after. + """ + + # Squelch mail if already fixed. + head = firstResult(builder) + if broken(head, builder) == False: + return + + # Get current node and node before, after. + cur = nodeByHash(node) + if cur is None: + return + before = nodeBefore(cur) + after = nodeAfter(cur) + + if broken(before, builder) == False and broken(cur, builder): + return cur + if broken(cur, builder) == False and broken(after, builder): + return after + + return + +def firstResult(builder): + q = Commit.all().order('-__key__') + for c in q.fetch(20): + for i, b in enumerate(c.builds): + p = b.split('`', 1) + if p[0] == builder: + return c + return None + +def nodeBefore(c): + return nodeByHash(c.parentnode) + +def nodeAfter(c): + return Commit.all().filter('parenthash', c.node).get() + +def notifyBroken(c, builder): + def send(): + n = Commit.get(c.key()) + if n is None: + logging.error("couldn't retrieve Commit '%s'" % c.key()) + return False + if n.fail_notification_sent: + return False + n.fail_notification_sent = True + return n.put() + if not db.run_in_transaction(send): + return + + subject = const.mail_fail_subject % (builder, c.desc.split('\n')[0]) + path = os.path.join(os.path.dirname(__file__), 'fail-notify.txt') + body = template.render(path, { + "builder": builder, + "node": c.node, + "user": c.user, + "desc": c.desc, + "loghash": logHash(c, builder) + }) + mail.send_mail( + sender=const.mail_from, + to=const.mail_fail_to, + subject=subject, + body=body + ) + +def logHash(c, builder): + for i, b in enumerate(c.builds): + p = b.split('`', 1) + if p[0] == builder: + return p[1] + return "" + +def broken(c, builder): + """ + broken returns True if commit c breaks the build for the specified builder, + False if it is a good build, and None if no results exist for this builder. + """ + if c is None: + return None + for i, b in enumerate(c.builds): + p = b.split('`', 1) + if p[0] == builder: + return len(p[1]) > 0 + return None + +def node(num): + q = Commit.all() + q.filter('num =', num) + n = q.get() + return n + +def nodeByHash(hash, ancestor=None): + q = Commit.all() + q.filter('node =', hash) + if ancestor is not None: + q.ancestor(ancestor) + n = q.get() + return n + +# nodeObj returns a JSON object (ready to be passed to simplejson.dump) describing node. +def nodeObj(n): + return { + 'Hash': n.node, + 'ParentHash': n.parentnode, + 'User': n.user, + 'Date': n.date.strftime('%Y-%m-%d %H:%M %z'), + 'Desc': n.desc, + } + +class FixedOffset(datetime.tzinfo): + """Fixed offset in minutes east from UTC.""" + + def __init__(self, offset): + self.__offset = datetime.timedelta(seconds = offset) + + def utcoffset(self, dt): + return self.__offset + + def tzname(self, dt): + return None + + def dst(self, dt): + return datetime.timedelta(0) + +def validNode(node): + if len(node) != 40: + return False + for x in node: + o = ord(x) + if (o < ord('0') or o > ord('9')) and (o < ord('a') or o > ord('f')): + return False + return True + +def parseDate(date): + if '-' in date: + (a, offset) = date.split('-', 1) + try: + return datetime.datetime.fromtimestamp(float(a), FixedOffset(0-int(offset))) + except ValueError: + return None + if '+' in date: + (a, offset) = date.split('+', 1) + try: + return datetime.datetime.fromtimestamp(float(a), FixedOffset(int(offset))) + except ValueError: + return None + try: + return datetime.datetime.utcfromtimestamp(float(date)) + except ValueError: + return None + +email_re = re.compile('^[^<]+<([^>]*)>$') + +def toUsername(user): + r = email_re.match(user) + if r is None: + return user + email = r.groups()[0] + return email.replace('@golang.org', '') + +def dateToShortStr(d): + return d.strftime('%a %b %d %H:%M') + +def parseBuild(build): + [builder, logblob] = build.split('`') + return {'builder': builder, 'log': logblob, 'ok': len(logblob) == 0} + +def nodeInfo(c): + return { + "node": c.node, + "user": toUsername(c.user), + "date": dateToShortStr(c.date), + "desc": c.desc, + "shortdesc": c.desc.split('\n', 2)[0] + } + +def toRev(c): + b = nodeInfo(c) + b['builds'] = [parseBuild(build) for build in c.builds] + return b + +def byBuilder(x, y): + return cmp(x['builder'], y['builder']) + +# Give old builders work; otherwise they pound on the web site. +class Hwget(DashboardHandler): + def get(self): + self.response.out.write("8000\n") + +# This is the URL map for the server. The first three entries are public, the +# rest are only used by the builders. +application = webapp.WSGIApplication( + [('/', MainPage), + ('/hw-get', Hwget), + ('/log/.*', LogHandler), + ('/commit', CommitHandler), + ('/init', Init), + ('/todo', Todo), + ('/build', Build), + ], debug=True) + +def main(): + run_wsgi_app(application) + +if __name__ == "__main__": + main() + diff --git a/misc/dashboard/godashboard/index.yaml b/misc/dashboard/godashboard/index.yaml new file mode 100644 index 000000000..f39299d5d --- /dev/null +++ b/misc/dashboard/godashboard/index.yaml @@ -0,0 +1,51 @@ +indexes: + +- kind: BenchmarkResult + ancestor: yes + properties: + - name: builder + - name: __key__ + direction: desc + +- kind: BenchmarkResult + ancestor: yes + properties: + - name: __key__ + direction: desc + +- kind: BenchmarkResults + properties: + - name: builder + - name: benchmark + +- kind: Commit + properties: + - name: __key__ + direction: desc + +- kind: Commit + ancestor: yes + properties: + - name: __key__ + direction: desc + +- kind: Project + properties: + - name: approved + - name: category + - name: name + +- kind: Project + properties: + - name: category + - name: name + +# AUTOGENERATED + +# This index.yaml is automatically updated whenever the dev_appserver +# detects that a new type of query is run. If you want to manage the +# index.yaml file manually, remove the above marker line (the line +# saying "# AUTOGENERATED"). If you want to manage some indexes +# manually, move them above the marker line. The index.yaml file is +# automatically uploaded to the admin console when you next deploy +# your application using appcfg.py. diff --git a/misc/dashboard/godashboard/key.py.dummy b/misc/dashboard/godashboard/key.py.dummy new file mode 100644 index 000000000..5b8bab186 --- /dev/null +++ b/misc/dashboard/godashboard/key.py.dummy @@ -0,0 +1,10 @@ +# Copyright 2009 The Go Authors. All rights reserved. +# Use of this source code is governed by a BSD-style +# license that can be found in the LICENSE file. + +# Copy this file to key.py after substituting the real key. + +# accessKey controls private access to the build server (i.e. to record new +# builds). It's tranmitted in the clear but, given the low value of the target, +# this should be sufficient. +accessKey = "this is not the real key" diff --git a/misc/dashboard/godashboard/main.html b/misc/dashboard/godashboard/main.html new file mode 100644 index 000000000..5390afce6 --- /dev/null +++ b/misc/dashboard/godashboard/main.html @@ -0,0 +1,62 @@ + + + + Build Status - Go Dashboard + + + + + + + + +

Go Dashboard

+ +

Build Status

+ + + + {% for b in builders %} + + {% endfor %} + + + + + + {% for r in revs %} + + + + {% for b in r.builds %} + + {% endfor %} + + + + + + {% endfor %} +
{{b.goos}}
{{b.goarch}}
{{b.note}}
{{r.node|slice:":12"}} + {% if b.ok %} + ok + {% else %} + {% if b.log %} + fail + {% else %} +   + {% endif %} + {% endif %} + {{r.user|escape}}{{r.date|escape}}{{r.shortdesc|escape}}
+
+ prev + next + top +
+ + diff --git a/misc/dashboard/godashboard/package.html b/misc/dashboard/godashboard/package.html new file mode 100644 index 000000000..8a9d0a3a0 --- /dev/null +++ b/misc/dashboard/godashboard/package.html @@ -0,0 +1,77 @@ + + + + Packages - Go Dashboard + + + + + + +

Go Dashboard

+ +

+ Packages listed on this page are written by third parties and + may or may not build or be safe to use. +

+ +

+ An "ok" in the build column indicates that the package is + goinstallable + with the latest + release of Go. +

+ +

+ The info column shows the first paragraph from the + package doc comment. +

+ +

Most Installed Packages (this week)

+ + + {% for r in by_week_count %} + + + + + + + + {% endfor %} +
last installcountbuildpathinfo
{{r.last_install|date:"Y-M-d H:i"}}{{r.week_count}}{% if r.ok %}ok{% else %} {% endif %}{{r.path}}{% if r.info %}{{r.info|escape}}{% endif %}
+ +

Recently Installed Packages

+ + + {% for r in by_time %} + + + + + + + + {% endfor %} +
last installcountbuildpathinfo
{{r.last_install|date:"Y-M-d H:i"}}{{r.count}}{% if r.ok %}ok{% else %} {% endif %}{{r.path}}{% if r.info %}{{r.info|escape}}{% endif %}
+ +

Most Installed Packages (all time)

+ + + {% for r in by_count %} + + + + + + + + {% endfor %} +
last installcountbuildpathinfo
{{r.last_install|date:"Y-M-d H:i"}}{{r.count}}{% if r.ok %}ok{% else %} {% endif %}{{r.path}}{% if r.info %}{{r.info|escape}}{% endif %}
+ + diff --git a/misc/dashboard/godashboard/package.py b/misc/dashboard/godashboard/package.py new file mode 100644 index 000000000..5cc2d2404 --- /dev/null +++ b/misc/dashboard/godashboard/package.py @@ -0,0 +1,429 @@ +# Copyright 2010 The Go Authors. All rights reserved. +# Use of this source code is governed by a BSD-style +# license that can be found in the LICENSE file. + +# This is the server part of the package dashboard. +# It must be run by App Engine. + +from google.appengine.api import mail +from google.appengine.api import memcache +from google.appengine.api import taskqueue +from google.appengine.api import urlfetch +from google.appengine.api import users +from google.appengine.ext import db +from google.appengine.ext import webapp +from google.appengine.ext.webapp import template +from google.appengine.ext.webapp.util import run_wsgi_app +import datetime +import logging +import os +import re +import sets +import urllib2 + +# local imports +from auth import auth +import toutf8 +import const + +template.register_template_library('toutf8') + +# Storage model for package info recorded on server. +class Package(db.Model): + path = db.StringProperty() + web_url = db.StringProperty() # derived from path + count = db.IntegerProperty() # grand total + week_count = db.IntegerProperty() # rolling weekly count + day_count = db.TextProperty(default='') # daily count + last_install = db.DateTimeProperty() + + # data contributed by gobuilder + info = db.StringProperty() + ok = db.BooleanProperty() + last_ok = db.DateTimeProperty() + + def get_day_count(self): + counts = {} + if not self.day_count: + return counts + for d in str(self.day_count).split('\n'): + date, count = d.split(' ') + counts[date] = int(count) + return counts + + def set_day_count(self, count): + days = [] + for day, count in count.items(): + days.append('%s %d' % (day, count)) + days.sort(reverse=True) + days = days[:28] + self.day_count = '\n'.join(days) + + def inc(self): + count = self.get_day_count() + today = str(datetime.date.today()) + count[today] = count.get(today, 0) + 1 + self.set_day_count(count) + self.update_week_count(count) + self.count += 1 + + def update_week_count(self, count=None): + if count is None: + count = self.get_day_count() + total = 0 + today = datetime.date.today() + for i in range(7): + day = str(today - datetime.timedelta(days=i)) + if day in count: + total += count[day] + self.week_count = total + + +# PackageDaily kicks off the daily package maintenance cron job +# and serves the associated task queue. +class PackageDaily(webapp.RequestHandler): + + def get(self): + # queue a task to update each package with a week_count > 0 + keys = Package.all(keys_only=True).filter('week_count >', 0) + for key in keys: + taskqueue.add(url='/package/daily', params={'key': key.name()}) + + def post(self): + # update a single package (in a task queue) + def update(key): + p = Package.get_by_key_name(key) + if not p: + return + p.update_week_count() + p.put() + key = self.request.get('key') + if not key: + return + db.run_in_transaction(update, key) + + +class Project(db.Model): + name = db.StringProperty(indexed=True) + descr = db.StringProperty() + web_url = db.StringProperty() + package = db.ReferenceProperty(Package) + category = db.StringProperty(indexed=True) + tags = db.ListProperty(str) + approved = db.BooleanProperty(indexed=True) + + +re_bitbucket = re.compile(r'^(bitbucket\.org/[a-z0-9A-Z_.\-]+/[a-zA-Z0-9_.\-]+)(/[a-z0-9A-Z_.\-/]+)?$') +re_googlecode = re.compile(r'^[a-z0-9\-]+\.googlecode\.com/(svn|hg|git)(/[a-z0-9A-Z_.\-/]+)?$') +re_github = re.compile(r'^github\.com/[a-z0-9A-Z_.\-]+(/[a-z0-9A-Z_.\-]+)+$') +re_launchpad = re.compile(r'^launchpad\.net/([a-z0-9A-Z_.\-]+(/[a-z0-9A-Z_.\-]+)?|~[a-z0-9A-Z_.\-]+/(\+junk|[a-z0-9A-Z_.\-]+)/[a-z0-9A-Z_.\-]+)(/[a-z0-9A-Z_.\-/]+)?$') + +def vc_to_web(path): + if re_bitbucket.match(path): + m = re_bitbucket.match(path) + check_url = 'http://' + m.group(1) + '/?cmd=heads' + web = 'http://' + m.group(1) + '/' + elif re_github.match(path): + m = re_github_web.match(path) + check_url = 'https://raw.github.com/' + m.group(1) + '/' + m.group(2) + '/master/' + web = 'http://github.com/' + m.group(1) + '/' + m.group(2) + '/' + elif re_googlecode.match(path): + m = re_googlecode.match(path) + check_url = 'http://'+path + if not m.group(2): # append / after bare '/hg' or '/git' + check_url += '/' + web = 'http://code.google.com/p/' + path[:path.index('.')] + elif re_launchpad.match(path): + check_url = web = 'https://'+path + else: + return False, False + return web, check_url + +re_bitbucket_web = re.compile(r'bitbucket\.org/([a-z0-9A-Z_.\-]+)/([a-z0-9A-Z_.\-]+)') +re_googlecode_web = re.compile(r'code.google.com/p/([a-z0-9\-]+)') +re_github_web = re.compile(r'github\.com/([a-z0-9A-Z_.\-]+)/([a-z0-9A-Z_.\-]+)') +re_launchpad_web = re.compile(r'launchpad\.net/([a-z0-9A-Z_.\-]+(/[a-z0-9A-Z_.\-]+)?|~[a-z0-9A-Z_.\-]+/(\+junk|[a-z0-9A-Z_.\-]+)/[a-z0-9A-Z_.\-]+)(/[a-z0-9A-Z_.\-/]+)?') +re_striphttp = re.compile(r'https?://(www\.)?') + +def find_googlecode_vcs(path): + # Perform http request to path/hg or path/git to check if they're + # using mercurial or git. Otherwise, assume svn. + for vcs in ['git', 'hg']: + try: + response = urlfetch.fetch('http://'+path+vcs, deadline=1) + if response.status_code == 200: + return vcs + except: pass + return 'svn' + +def web_to_vc(url): + url = re_striphttp.sub('', url) + m = re_bitbucket_web.match(url) + if m: + return 'bitbucket.org/'+m.group(1)+'/'+m.group(2) + m = re_github_web.match(url) + if m: + return 'github.com/'+m.group(1)+'/'+m.group(2) + m = re_googlecode_web.match(url) + if m: + path = m.group(1)+'.googlecode.com/' + vcs = find_googlecode_vcs(path) + return path + vcs + m = re_launchpad_web.match(url) + if m: + return m.group(0) + return False + +MaxPathLength = 100 +CacheTimeout = 3600 + +class PackagePage(webapp.RequestHandler): + def get(self): + if self.request.get('fmt') == 'json': + return self.json() + + html = memcache.get('view-package') + if not html: + tdata = {} + + q = Package.all().filter('week_count >', 0) + q.order('-week_count') + tdata['by_week_count'] = q.fetch(50) + + q = Package.all() + q.order('-last_install') + tdata['by_time'] = q.fetch(20) + + q = Package.all() + q.order('-count') + tdata['by_count'] = q.fetch(100) + + path = os.path.join(os.path.dirname(__file__), 'package.html') + html = template.render(path, tdata) + memcache.set('view-package', html, time=CacheTimeout) + + self.response.headers['Content-Type'] = 'text/html; charset=utf-8' + self.response.out.write(html) + + def json(self): + json = memcache.get('view-package-json') + if not json: + q = Package.all() + s = '{"packages": [' + sep = '' + for r in q.fetch(1000): + s += '%s\n\t{"path": "%s", "last_install": "%s", "count": "%s"}' % (sep, r.path, r.last_install, r.count) + sep = ',' + s += '\n]}\n' + json = s + memcache.set('view-package-json', json, time=CacheTimeout) + self.response.set_status(200) + self.response.headers['Content-Type'] = 'text/plain; charset=utf-8' + self.response.out.write(json) + + def can_get_url(self, url): + try: + urllib2.urlopen(urllib2.Request(url)) + return True + except: + return False + + def is_valid_package_path(self, path): + return (re_bitbucket.match(path) or + re_googlecode.match(path) or + re_github.match(path) or + re_launchpad.match(path)) + + def record_pkg(self, path): + # sanity check string + if not path or len(path) > MaxPathLength or not self.is_valid_package_path(path): + return False + + # look in datastore + key = 'pkg-' + path + p = Package.get_by_key_name(key) + if p is None: + # not in datastore - verify URL before creating + web, check_url = vc_to_web(path) + if not web: + logging.error('unrecognized path: %s', path) + return False + if not self.can_get_url(check_url): + logging.error('cannot get %s', check_url) + return False + p = Package(key_name = key, path = path, count = 0, web_url = web) + + if auth(self.request): + # builder updating package metadata + p.info = self.request.get('info') + p.ok = self.request.get('ok') == "true" + if p.ok: + p.last_ok = datetime.datetime.utcnow() + else: + # goinstall reporting an install + p.inc() + p.last_install = datetime.datetime.utcnow() + + # update package object + p.put() + return True + + def post(self): + path = self.request.get('path') + ok = db.run_in_transaction(self.record_pkg, path) + if ok: + self.response.set_status(200) + self.response.out.write('ok') + else: + logging.error('invalid path in post: %s', path) + self.response.set_status(500) + self.response.out.write('not ok') + +class ProjectPage(webapp.RequestHandler): + + def get(self): + admin = users.is_current_user_admin() + if self.request.path == "/project/login": + self.redirect(users.create_login_url("/project")) + elif self.request.path == "/project/logout": + self.redirect(users.create_logout_url("/project")) + elif self.request.path == "/project/edit" and admin: + self.edit() + elif self.request.path == "/project/assoc" and admin: + self.assoc() + else: + self.list() + + def assoc(self): + projects = Project.all() + for p in projects: + if p.package: + continue + path = web_to_vc(p.web_url) + if not path: + continue + pkg = Package.get_by_key_name("pkg-"+path) + if not pkg: + self.response.out.write('no: %s %s
' % (p.web_url, path)) + continue + p.package = pkg + p.put() + self.response.out.write('yes: %s %s
' % (p.web_url, path)) + + def post(self): + if self.request.path == "/project/edit": + self.edit(True) + else: + data = dict(map(lambda x: (x, self.request.get(x)), ["name","descr","web_url"])) + if reduce(lambda x, y: x or not y, data.values(), False): + data["submitMsg"] = "You must complete all the fields." + self.list(data) + return + p = Project.get_by_key_name("proj-"+data["name"]) + if p is not None: + data["submitMsg"] = "A project by this name already exists." + self.list(data) + return + p = Project(key_name="proj-"+data["name"], **data) + p.put() + + path = os.path.join(os.path.dirname(__file__), 'project-notify.txt') + mail.send_mail( + sender=const.mail_from, + to=const.mail_submit_to, + subject=const.mail_submit_subject, + body=template.render(path, {'project': p})) + + self.list({"submitMsg": "Your project has been submitted."}) + + def list(self, additional_data={}): + cache_key = 'view-project-data' + tag = self.request.get('tag', None) + if tag: + cache_key += '-'+tag + data = memcache.get(cache_key) + admin = users.is_current_user_admin() + if admin or not data: + projects = Project.all().order('category').order('name') + if not admin: + projects = projects.filter('approved =', True) + projects = list(projects) + + tags = sets.Set() + for p in projects: + for t in p.tags: + tags.add(t) + + if tag: + projects = filter(lambda x: tag in x.tags, projects) + + data = {} + data['tag'] = tag + data['tags'] = tags + data['projects'] = projects + data['admin']= admin + if not admin: + memcache.set(cache_key, data, time=CacheTimeout) + + for k, v in additional_data.items(): + data[k] = v + + self.response.headers['Content-Type'] = 'text/html; charset=utf-8' + path = os.path.join(os.path.dirname(__file__), 'project.html') + self.response.out.write(template.render(path, data)) + + def edit(self, save=False): + if save: + name = self.request.get("orig_name") + else: + name = self.request.get("name") + + p = Project.get_by_key_name("proj-"+name) + if not p: + self.response.out.write("Couldn't find that Project.") + return + + if save: + if self.request.get("do") == "Delete": + p.delete() + else: + pkg_name = self.request.get("package", None) + if pkg_name: + pkg = Package.get_by_key_name("pkg-"+pkg_name) + if pkg: + p.package = pkg.key() + for f in ['name', 'descr', 'web_url', 'category']: + setattr(p, f, self.request.get(f, None)) + p.approved = self.request.get("approved") == "1" + p.tags = filter(lambda x: x, self.request.get("tags", "").split(",")) + p.put() + memcache.delete('view-project-data') + self.redirect('/project') + return + + # get all project categories and tags + cats, tags = sets.Set(), sets.Set() + for r in Project.all(): + cats.add(r.category) + for t in r.tags: + tags.add(t) + + self.response.headers['Content-Type'] = 'text/html; charset=utf-8' + path = os.path.join(os.path.dirname(__file__), 'project-edit.html') + self.response.out.write(template.render(path, { + "taglist": tags, "catlist": cats, "p": p, "tags": ",".join(p.tags) })) + + def redirect(self, url): + self.response.set_status(302) + self.response.headers.add_header("Location", url) + +def main(): + app = webapp.WSGIApplication([ + ('/package', PackagePage), + ('/package/daily', PackageDaily), + ('/project.*', ProjectPage), + ], debug=True) + run_wsgi_app(app) + +if __name__ == '__main__': + main() diff --git a/misc/dashboard/godashboard/project-edit.html b/misc/dashboard/godashboard/project-edit.html new file mode 100644 index 000000000..ce18fb3fb --- /dev/null +++ b/misc/dashboard/godashboard/project-edit.html @@ -0,0 +1,47 @@ + + + + + + + +
+Name:
+
+Description:
+
+Category:
+
+Tags: (comma-separated)
+
+Web URL:
+
+Package URL: (to link to a goinstall'd package)
+
+Approved:
+
+ + +
+ + + diff --git a/misc/dashboard/godashboard/project-notify.txt b/misc/dashboard/godashboard/project-notify.txt new file mode 100644 index 000000000..f55bf6421 --- /dev/null +++ b/misc/dashboard/godashboard/project-notify.txt @@ -0,0 +1,9 @@ +A new project has been submitted: + +Name: {{project.name}} +Description: {{project.descr}} +URL: {{project.web_url}} + +To edit/approve/delete: +http://godashboard.appspot.com/project/edit?name={{project.name|toutf8|urlencode}} + diff --git a/misc/dashboard/godashboard/project.html b/misc/dashboard/godashboard/project.html new file mode 100644 index 000000000..4fe1741c6 --- /dev/null +++ b/misc/dashboard/godashboard/project.html @@ -0,0 +1,85 @@ + + + + Projects - Go Dashboard + + + + + + + +

Go Dashboard

+ +

+ These are external projects and not endorsed or supported by the Go project. +

+ +

Projects

+ +
+

Submit a Project

+

+ Using this form you can submit a project to be included in the list. +

+
+ + + {% endif %} +
Name: +
Description: +
URL: +
  + {% if submitMsg %} +
{{ submitMsg }}
+
+
+ +

+ Filter by tag: + {% if tag %} + all + {% else %} + all + {% endif %} + {% for t in tags %} + {% ifequal t tag %} + {{t}} + {% else %} + {{t}} + {% endifequal %} + {% endfor %} +

+ + {% for r in projects %} + {% ifchanged r.category %} + {% if not forloop.first %} + + {% endif %} +

{{r.category}}

+
    + {% endifchanged %} + + {% if admin %}[edit]{% endif %} + {{r.name}} - {{r.descr}} + {% for tag in r.tags %} + {{tag}} + {% endfor %} + + {% if forloop.last %} +
+ {% endif %} + {% endfor %} + + + + + diff --git a/misc/dashboard/godashboard/static/favicon.ico b/misc/dashboard/godashboard/static/favicon.ico new file mode 100644 index 000000000..48854ff3b Binary files /dev/null and b/misc/dashboard/godashboard/static/favicon.ico differ diff --git a/misc/dashboard/godashboard/static/style.css b/misc/dashboard/godashboard/static/style.css new file mode 100644 index 000000000..d6d23b536 --- /dev/null +++ b/misc/dashboard/godashboard/static/style.css @@ -0,0 +1,118 @@ +body { + font-family: sans-serif; + margin: 0; + padding: 0; +} +h1, h2, h3, ul.menu, table, p { + padding: 0 0.5em; +} +h1, h2 { + margin: 0; + background: #eee; +} +h1 { + border-bottom: 1px solid #ccc; + font-size: 1em; + padding: 0.5em; + margin-bottom: 0.5em; + text-align: right; +} +h2 { + border-top: 1px solid #ccc; + padding-left: 0.2em; +} +.submit { + float: right; + border: 1px solid #ccc; + width: 350px; + padding-bottom: 1em; + margin: 0.5em; + background: #eee; +} +.submit table { + width: 100%; +} +.submit input[type=text] { + width: 200px; +} +.submit .msg { + text-align: center; + color: red; +} +table.alternate { + white-space: nowrap; + margin: 0.5em 0; +} +table.alternate td, +table.alternate th { + padding: 0.1em 0.25em; + font-size: small; +} +table.alternate tr td:last-child { + padding-right: 0; +} +table.alternate tr:nth-child(2n) { + background-color: #f0f0f0; +} +td.result { + text-align: center; +} +span.hash { + font-family: monospace; + font-size: small; + color: #aaa; +} +td.date { + color: #aaa; +} +td.ok { + text-align: center; + color: #060; + font-weight: bold; +} +td.ok a { + cursor: help; +} +th { + text-align: left; +} +th.builder { + text-align: center; + font-weight: bold; +} +a.fail { + color: #F00; +} +a.fail:visited { + color: #900; +} +ul.menu { + margin: 0; + padding: 0; + list-style-type: none; +} +ul.menu li { + float: left; + display: block; + font-size: 1em; + padding: 0.5em; + background: #EEF; + margin-left: 0.5em; + border-left: 1px solid #999; + border-right: 1px solid #999; +} +div.paginate { + padding: 0.5em; +} +div.paginate a { + padding: 0.5em; + margin-right: 0.5em; + background: #eee; + color: blue; +} +div.paginate a.inactive { + color: #999; +} +td.time { + font-family: monospace; +} diff --git a/misc/dashboard/godashboard/toutf8.py b/misc/dashboard/godashboard/toutf8.py new file mode 100644 index 000000000..544c681b6 --- /dev/null +++ b/misc/dashboard/godashboard/toutf8.py @@ -0,0 +1,14 @@ +# Copyright 2010 The Go Authors. All rights reserved. +# Use of this source code is governed by a BSD-style +# license that can be found in the LICENSE file. + +# This is a Django custom template filter to work around the +# fact that GAE's urlencode filter doesn't handle unicode strings. + +from google.appengine.ext import webapp + +register = webapp.template.create_template_register() + +@register.filter +def toutf8(value): + return value.encode("utf-8") diff --git a/misc/dashboard/googlecode_upload.py b/misc/dashboard/googlecode_upload.py new file mode 100755 index 000000000..e87db884a --- /dev/null +++ b/misc/dashboard/googlecode_upload.py @@ -0,0 +1,248 @@ +#!/usr/bin/env python2 +# +# Copyright 2006, 2007 Google Inc. All Rights Reserved. +# Author: danderson@google.com (David Anderson) +# +# Script for uploading files to a Google Code project. +# +# This is intended to be both a useful script for people who want to +# streamline project uploads and a reference implementation for +# uploading files to Google Code projects. +# +# To upload a file to Google Code, you need to provide a path to the +# file on your local machine, a small summary of what the file is, a +# project name, and a valid account that is a member or owner of that +# project. You can optionally provide a list of labels that apply to +# the file. The file will be uploaded under the same name that it has +# in your local filesystem (that is, the "basename" or last path +# component). Run the script with '--help' to get the exact syntax +# and available options. +# +# Note that the upload script requests that you enter your +# googlecode.com password. This is NOT your Gmail account password! +# This is the password you use on googlecode.com for committing to +# Subversion and uploading files. You can find your password by going +# to http://code.google.com/hosting/settings when logged in with your +# Gmail account. If you have already committed to your project's +# Subversion repository, the script will automatically retrieve your +# credentials from there (unless disabled, see the output of '--help' +# for details). +# +# If you are looking at this script as a reference for implementing +# your own Google Code file uploader, then you should take a look at +# the upload() function, which is the meat of the uploader. You +# basically need to build a multipart/form-data POST request with the +# right fields and send it to https://PROJECT.googlecode.com/files . +# Authenticate the request using HTTP Basic authentication, as is +# shown below. +# +# Licensed under the terms of the Apache Software License 2.0: +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Questions, comments, feature requests and patches are most welcome. +# Please direct all of these to the Google Code users group: +# http://groups.google.com/group/google-code-hosting + +"""Google Code file uploader script. +""" + +__author__ = 'danderson@google.com (David Anderson)' + +import httplib +import os.path +import optparse +import getpass +import base64 +import sys + + +def upload(file, project_name, user_name, password, summary, labels=None): + """Upload a file to a Google Code project's file server. + + Args: + file: The local path to the file. + project_name: The name of your project on Google Code. + user_name: Your Google account name. + password: The googlecode.com password for your account. + Note that this is NOT your global Google Account password! + summary: A small description for the file. + labels: an optional list of label strings with which to tag the file. + + Returns: a tuple: + http_status: 201 if the upload succeeded, something else if an + error occurred. + http_reason: The human-readable string associated with http_status + file_url: If the upload succeeded, the URL of the file on Google + Code, None otherwise. + """ + # The login is the user part of user@gmail.com. If the login provided + # is in the full user@domain form, strip it down. + if user_name.endswith('@gmail.com'): + user_name = user_name[:user_name.index('@gmail.com')] + + form_fields = [('summary', summary)] + if labels is not None: + form_fields.extend([('label', l.strip()) for l in labels]) + + content_type, body = encode_upload_request(form_fields, file) + + upload_host = '%s.googlecode.com' % project_name + upload_uri = '/files' + auth_token = base64.b64encode('%s:%s'% (user_name, password)) + headers = { + 'Authorization': 'Basic %s' % auth_token, + 'User-Agent': 'Googlecode.com uploader v0.9.4', + 'Content-Type': content_type, + } + + server = httplib.HTTPSConnection(upload_host) + server.request('POST', upload_uri, body, headers) + resp = server.getresponse() + server.close() + + if resp.status == 201: + location = resp.getheader('Location', None) + else: + location = None + return resp.status, resp.reason, location + + +def encode_upload_request(fields, file_path): + """Encode the given fields and file into a multipart form body. + + fields is a sequence of (name, value) pairs. file is the path of + the file to upload. The file will be uploaded to Google Code with + the same file name. + + Returns: (content_type, body) ready for httplib.HTTP instance + """ + BOUNDARY = '----------Googlecode_boundary_reindeer_flotilla' + CRLF = '\r\n' + + body = [] + + # Add the metadata about the upload first + for key, value in fields: + body.extend( + ['--' + BOUNDARY, + 'Content-Disposition: form-data; name="%s"' % key, + '', + value, + ]) + + # Now add the file itself + file_name = os.path.basename(file_path) + f = open(file_path, 'rb') + file_content = f.read() + f.close() + + body.extend( + ['--' + BOUNDARY, + 'Content-Disposition: form-data; name="filename"; filename="%s"' + % file_name, + # The upload server determines the mime-type, no need to set it. + 'Content-Type: application/octet-stream', + '', + file_content, + ]) + + # Finalize the form body + body.extend(['--' + BOUNDARY + '--', '']) + + return 'multipart/form-data; boundary=%s' % BOUNDARY, CRLF.join(body) + + +def upload_find_auth(file_path, project_name, summary, labels=None, + user_name=None, password=None, tries=3): + """Find credentials and upload a file to a Google Code project's file server. + + file_path, project_name, summary, and labels are passed as-is to upload. + + Args: + file_path: The local path to the file. + project_name: The name of your project on Google Code. + summary: A small description for the file. + labels: an optional list of label strings with which to tag the file. + config_dir: Path to Subversion configuration directory, 'none', or None. + user_name: Your Google account name. + tries: How many attempts to make. + """ + + while tries > 0: + if user_name is None: + # Read username if not specified or loaded from svn config, or on + # subsequent tries. + sys.stdout.write('Please enter your googlecode.com username: ') + sys.stdout.flush() + user_name = sys.stdin.readline().rstrip() + if password is None: + # Read password if not loaded from svn config, or on subsequent tries. + print 'Please enter your googlecode.com password.' + print '** Note that this is NOT your Gmail account password! **' + print 'It is the password you use to access Subversion repositories,' + print 'and can be found here: http://code.google.com/hosting/settings' + password = getpass.getpass() + + status, reason, url = upload(file_path, project_name, user_name, password, + summary, labels) + # Returns 403 Forbidden instead of 401 Unauthorized for bad + # credentials as of 2007-07-17. + if status in [httplib.FORBIDDEN, httplib.UNAUTHORIZED]: + # Rest for another try. + user_name = password = None + tries = tries - 1 + else: + # We're done. + break + + return status, reason, url + + +def main(): + parser = optparse.OptionParser(usage='googlecode-upload.py -s SUMMARY ' + '-p PROJECT [options] FILE') + parser.add_option('-s', '--summary', dest='summary', + help='Short description of the file') + parser.add_option('-p', '--project', dest='project', + help='Google Code project name') + parser.add_option('-u', '--user', dest='user', + help='Your Google Code username') + parser.add_option('-w', '--password', dest='password', + help='Your Google Code password') + parser.add_option('-l', '--labels', dest='labels', + help='An optional list of comma-separated labels to attach ' + 'to the file') + + options, args = parser.parse_args() + + if not options.summary: + parser.error('File summary is missing.') + elif not options.project: + parser.error('Project name is missing.') + elif len(args) < 1: + parser.error('File to upload not provided.') + elif len(args) > 1: + parser.error('Only one file may be specified.') + + file_path = args[0] + + if options.labels: + labels = options.labels.split(',') + else: + labels = None + + status, reason, url = upload_find_auth(file_path, options.project, + options.summary, labels, + options.user, options.password) + if url: + print 'The file was uploaded successfully.' + print 'URL: %s' % url + return 0 + else: + print 'An error occurred. Your file was not uploaded.' + print 'Google Code upload server said: %s (%s)' % (reason, status) + return 1 + + +if __name__ == '__main__': + sys.exit(main()) diff --git a/misc/emacs/go-mode-load.el b/misc/emacs/go-mode-load.el new file mode 100644 index 000000000..d453166a4 --- /dev/null +++ b/misc/emacs/go-mode-load.el @@ -0,0 +1,50 @@ +;;; go-mode-load.el --- Major mode for the Go programming language + +;;; Commentary: + +;; To install go-mode, add the following lines to your .emacs file: +;; (add-to-list 'load-path "PATH CONTAINING go-mode-load.el" t) +;; (require 'go-mode-load) +;; After this, go-mode will be used for files ending in '.go'. + +;; To compile go-mode from the command line, run the following +;; emacs -batch -f batch-byte-compile go-mode.el + +;; See go-mode.el for documentation. + +;;; Code: + +;; To update this file, evaluate the following form +;; (let ((generated-autoload-file buffer-file-name)) (update-file-autoloads "go-mode.el")) + + +;;;### (autoloads (gofmt-before-save gofmt go-mode) "go-mode" "go-mode.el" +;;;;;; (19917 17808)) +;;; Generated autoloads from go-mode.el + +(autoload 'go-mode "go-mode" "\ +Major mode for editing Go source text. + +This provides basic syntax highlighting for keywords, built-ins, +functions, and some types. It also provides indentation that is +\(almost) identical to gofmt. + +\(fn)" t nil) + +(add-to-list 'auto-mode-alist (cons "\\.go$" #'go-mode)) + +(autoload 'gofmt "go-mode" "\ +Pipe the current buffer through the external tool `gofmt`. +Replace the current buffer on success; display errors on failure. + +\(fn)" t nil) + +(autoload 'gofmt-before-save "go-mode" "\ +Add this to .emacs to run gofmt on the current buffer when saving: + (add-hook 'before-save-hook #'gofmt-before-save) + +\(fn)" t nil) + +;;;*** + +(provide 'go-mode-load) diff --git a/misc/emacs/go-mode.el b/misc/emacs/go-mode.el new file mode 100644 index 000000000..ba7f72397 --- /dev/null +++ b/misc/emacs/go-mode.el @@ -0,0 +1,544 @@ +;;; go-mode.el --- Major mode for the Go programming language + +;;; Commentary: + +;; For installation instructions, see go-mode-load.el + +;;; To do: + +;; * Indentation is *almost* identical to gofmt +;; ** We think struct literal keys are labels and outdent them +;; ** We disagree on the indentation of function literals in arguments +;; ** There are bugs with the close brace of struct literals +;; * Highlight identifiers according to their syntactic context: type, +;; variable, function call, or tag +;; * Command for adding an import +;; ** Check if it's already there +;; ** Factor/unfactor the import line +;; ** Alphabetize +;; * Remove unused imports +;; ** This is hard, since I have to be aware of shadowing to do it +;; right +;; * Format region using gofmt + +;;; Code: + +(eval-when-compile (require 'cl)) + +(defvar go-mode-syntax-table + (let ((st (make-syntax-table))) + ;; Add _ to :word: character class + (modify-syntax-entry ?_ "w" st) + + ;; Operators (punctuation) + (modify-syntax-entry ?+ "." st) + (modify-syntax-entry ?- "." st) + (modify-syntax-entry ?* "." st) + (modify-syntax-entry ?/ "." st) + (modify-syntax-entry ?% "." st) + (modify-syntax-entry ?& "." st) + (modify-syntax-entry ?| "." st) + (modify-syntax-entry ?^ "." st) + (modify-syntax-entry ?! "." st) + (modify-syntax-entry ?= "." st) + (modify-syntax-entry ?< "." st) + (modify-syntax-entry ?> "." st) + + ;; Strings + (modify-syntax-entry ?\" "\"" st) + (modify-syntax-entry ?\' "\"" st) + (modify-syntax-entry ?` "\"" st) + (modify-syntax-entry ?\\ "\\" st) + + ;; Comments + (modify-syntax-entry ?/ ". 124b" st) + (modify-syntax-entry ?* ". 23" st) + (modify-syntax-entry ?\n "> b" st) + (modify-syntax-entry ?\^m "> b" st) + + st) + "Syntax table for Go mode.") + +(defvar go-mode-keywords + '("break" "default" "func" "interface" "select" + "case" "defer" "go" "map" "struct" + "chan" "else" "goto" "package" "switch" + "const" "fallthrough" "if" "range" "type" + "continue" "for" "import" "return" "var") + "All keywords in the Go language. Used for font locking and +some syntax analysis.") + +(defvar go-mode-font-lock-keywords + (let ((builtins '("append" "cap" "close" "complex" "copy" "imag" "len" + "make" "new" "panic" "print" "println" "real" "recover")) + (constants '("nil" "true" "false" "iota")) + (type-name "\\s *\\(?:[*(]\\s *\\)*\\(?:\\w+\\s *\\.\\s *\\)?\\(\\w+\\)") + ) + `((,(regexp-opt go-mode-keywords 'words) . font-lock-keyword-face) + (,(regexp-opt builtins 'words) . font-lock-builtin-face) + (,(regexp-opt constants 'words) . font-lock-constant-face) + ;; Function names in declarations + ("\\\\s *\\(\\w+\\)" 1 font-lock-function-name-face) + ;; Function names in methods are handled by function call pattern + ;; Function names in calls + ;; XXX Doesn't match if function name is surrounded by parens + ("\\(\\w+\\)\\s *(" 1 font-lock-function-name-face) + ;; Type names + ("\\\\s *\\(\\w+\\)" 1 font-lock-type-face) + (,(concat "\\\\s *\\w+\\s *" type-name) 1 font-lock-type-face) + ;; Arrays/slices/map value type + ;; XXX Wrong. Marks 0 in expression "foo[0] * x" +;; (,(concat "]" type-name) 1 font-lock-type-face) + ;; Map key type + (,(concat "\\\\s *\\(?:<-\\)?" type-name) 1 font-lock-type-face) + ;; new/make type + (,(concat "\\<\\(?:new\\|make\\)\\>\\(?:\\s \\|)\\)*(" type-name) 1 font-lock-type-face) + ;; Type conversion + (,(concat "\\.\\s *(" type-name) 1 font-lock-type-face) + ;; Method receiver type + (,(concat "\\\\s *(\\w+\\s +" type-name) 1 font-lock-type-face) + ;; Labels + ;; XXX Not quite right. Also marks compound literal fields. + ("^\\s *\\(\\w+\\)\\s *:\\(\\S.\\|$\\)" 1 font-lock-constant-face) + ("\\<\\(goto\\|break\\|continue\\)\\>\\s *\\(\\w+\\)" 2 font-lock-constant-face))) + "Basic font lock keywords for Go mode. Highlights keywords, +built-ins, functions, and some types.") + +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +;; Key map +;; + +(defvar go-mode-map + (let ((m (make-sparse-keymap))) + (define-key m "}" #'go-mode-insert-and-indent) + (define-key m ")" #'go-mode-insert-and-indent) + (define-key m ":" #'go-mode-delayed-electric) + ;; In case we get : indentation wrong, correct ourselves + (define-key m "=" #'go-mode-insert-and-indent) + m) + "Keymap used by Go mode to implement electric keys.") + +(defun go-mode-insert-and-indent (key) + "Invoke the global binding of KEY, then reindent the line." + + (interactive (list (this-command-keys))) + (call-interactively (lookup-key (current-global-map) key)) + (indent-according-to-mode)) + +(defvar go-mode-delayed-point nil + "The point following the previous insertion if the insertion +was a delayed electric key. Used to communicate between +`go-mode-delayed-electric' and `go-mode-delayed-electric-hook'.") +(make-variable-buffer-local 'go-mode-delayed-point) + +(defun go-mode-delayed-electric (p) + "Perform electric insertion, but delayed by one event. + +This inserts P into the buffer, as usual, then waits for another key. +If that second key causes a buffer modification starting at the +point after the insertion of P, reindents the line containing P." + + (interactive "p") + (self-insert-command p) + (setq go-mode-delayed-point (point))) + +(defun go-mode-delayed-electric-hook (b e l) + "An after-change-function that implements `go-mode-delayed-electric'." + + (when (and go-mode-delayed-point + (= go-mode-delayed-point b)) + (save-excursion + (save-match-data + (goto-char go-mode-delayed-point) + (indent-according-to-mode)))) + (setq go-mode-delayed-point nil)) + +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +;; Parser +;; + +(defvar go-mode-mark-cs-end 1 + "The point at which the comment/string cache ends. The buffer +will be marked from the beginning up to this point (that is, up +to and including character (1- go-mode-mark-cs-end)).") +(make-variable-buffer-local 'go-mode-mark-cs-end) + +(defvar go-mode-mark-cs-state nil + "The `parse-partial-sexp' state of the comment/string parser as +of the point `go-mode-mark-cs-end'.") +(make-variable-buffer-local 'go-mode-mark-cs-state) + +(defvar go-mode-mark-nesting-end 1 + "The point at which the nesting cache ends. The buffer will be +marked from the beginning up to this point.") +(make-variable-buffer-local 'go-mode-mark-nesting-end) + +(defun go-mode-mark-clear-cache (b e l) + "An after-change-function that clears the comment/string and +nesting caches from the modified point on." + + (save-restriction + (widen) + (when (< b go-mode-mark-cs-end) + (remove-text-properties b (min go-mode-mark-cs-end (point-max)) '(go-mode-cs nil)) + (setq go-mode-mark-cs-end b + go-mode-mark-cs-state nil)) + + (when (< b go-mode-mark-nesting-end) + (remove-text-properties b (min go-mode-mark-nesting-end (point-max)) '(go-mode-nesting nil)) + (setq go-mode-mark-nesting-end b)))) + +(defmacro go-mode-parser (&rest body) + "Evaluate BODY in an environment set up for parsers that use +text properties to mark text. This inhibits changes to the undo +list or the buffer's modification status and inhibits calls to +the modification hooks. It also saves the excursion and +restriction and widens the buffer, since most parsers are +context-sensitive." + + (let ((modified-var (make-symbol "modified"))) + `(let ((buffer-undo-list t) + (,modified-var (buffer-modified-p)) + (inhibit-modification-hooks t) + (inhibit-read-only t)) + (save-excursion + (save-restriction + (widen) + (unwind-protect + (progn ,@body) + (set-buffer-modified-p ,modified-var))))))) + +(defsubst go-mode-cs (&optional pos) + "Return the comment/string state at point POS. If point is +inside a comment or string (including the delimiters), this +returns a pair (START . END) indicating the extents of the +comment or string." + + (unless pos + (setq pos (point))) + (if (= pos 1) + nil + (when (> pos go-mode-mark-cs-end) + (go-mode-mark-cs pos)) + (get-text-property (- pos 1) 'go-mode-cs))) + +(defun go-mode-mark-cs (end) + "Mark comments and strings up to point END. Don't call this +directly; use `go-mode-cs'." + + (setq end (min end (point-max))) + (go-mode-parser + (let* ((pos go-mode-mark-cs-end) + (state (or go-mode-mark-cs-state (syntax-ppss pos)))) + ;; Mark comments and strings + (when (nth 8 state) + ;; Get to the beginning of the comment/string + (setq pos (nth 8 state) + state nil)) + (while (> end pos) + ;; Find beginning of comment/string + (while (and (> end pos) + (progn + (setq state (parse-partial-sexp pos end nil nil state 'syntax-table) + pos (point)) + (not (nth 8 state))))) + ;; Find end of comment/string + (let ((start (nth 8 state))) + (when start + (setq state (parse-partial-sexp pos (point-max) nil nil state 'syntax-table) + pos (point)) + ;; Mark comment + (put-text-property start (- pos 1) 'go-mode-cs (cons start pos)) + (when nil + (put-text-property start (- pos 1) 'face + `((:background "midnight blue"))))))) + ;; Update state + (setq go-mode-mark-cs-end pos + go-mode-mark-cs-state state)))) + +(defsubst go-mode-nesting (&optional pos) + "Return the nesting at point POS. The nesting is a list +of (START . END) pairs for all braces, parens, and brackets +surrounding POS, starting at the inner-most nesting. START is +the location of the open character. END is the location of the +close character or nil if the nesting scanner has not yet +encountered the close character." + + (unless pos + (setq pos (point))) + (if (= pos 1) + '() + (when (> pos go-mode-mark-nesting-end) + (go-mode-mark-nesting pos)) + (get-text-property (- pos 1) 'go-mode-nesting))) + +(defun go-mode-mark-nesting (pos) + "Mark nesting up to point END. Don't call this directly; use +`go-mode-nesting'." + + (go-mode-cs pos) + (go-mode-parser + ;; Mark depth + (goto-char go-mode-mark-nesting-end) + (let ((nesting (go-mode-nesting)) + (last (point))) + (while (< last pos) + ;; Find the next depth-changing character + (skip-chars-forward "^(){}[]" pos) + ;; Mark everything up to this character with the current + ;; nesting + (put-text-property last (point) 'go-mode-nesting nesting) + (when nil + (let ((depth (length nesting))) + (put-text-property last (point) 'face + `((:background + ,(format "gray%d" (* depth 10))))))) + (setq last (point)) + ;; Update nesting + (unless (eobp) + (let ((ch (unless (go-mode-cs) (char-after)))) + (forward-char 1) + (case ch + ((?\( ?\{ ?\[) + (setq nesting (cons (cons (- (point) 1) nil) + nesting))) + ((?\) ?\} ?\]) + (when nesting + (setcdr (car nesting) (- (point) 1)) + (setq nesting (cdr nesting)))))))) + ;; Update state + (setq go-mode-mark-nesting-end last)))) + +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +;; Indentation +;; + +(defvar go-mode-non-terminating-keywords-regexp + (let* ((kws go-mode-keywords) + (kws (remove "break" kws)) + (kws (remove "continue" kws)) + (kws (remove "fallthrough" kws)) + (kws (remove "return" kws))) + (regexp-opt kws 'words)) + "Regular expression matching all Go keywords that *do not* +implicitly terminate a statement.") + +(defun go-mode-semicolon-p () + "True iff point immediately follows either an explicit or +implicit semicolon. Point should immediately follow the last +token on the line." + + ;; #Semicolons + (case (char-before) + ((?\;) t) + ;; String literal + ((?' ?\" ?`) t) + ;; One of the operators and delimiters ++, --, ), ], or } + ((?+) (eq (char-before (1- (point))) ?+)) + ((?-) (eq (char-before (1- (point))) ?-)) + ((?\) ?\] ?\}) t) + ;; An identifier or one of the keywords break, continue, + ;; fallthrough, or return or a numeric literal + (otherwise + (save-excursion + (when (/= (skip-chars-backward "[:word:]_") 0) + (not (looking-at go-mode-non-terminating-keywords-regexp))))))) + +(defun go-mode-indentation () + "Compute the ideal indentation level of the current line. + +To the first order, this is the brace depth of the current line, +plus parens that follow certain keywords. case, default, and +labels are outdented one level, and continuation lines are +indented one level." + + (save-excursion + (back-to-indentation) + (let ((cs (go-mode-cs))) + ;; Treat comments and strings differently only if the beginning + ;; of the line is contained within them + (when (and cs (= (point) (car cs))) + (setq cs nil)) + ;; What type of context am I in? + (cond + ((and cs (save-excursion + (goto-char (car cs)) + (looking-at "\\s\""))) + ;; Inside a multi-line string. Don't mess with indentation. + nil) + (cs + ;; Inside a general comment + (goto-char (car cs)) + (forward-char 1) + (current-column)) + (t + ;; Not in a multi-line string or comment + (let ((indent 0) + (inside-indenting-paren nil)) + ;; Count every enclosing brace, plus parens that follow + ;; import, const, var, or type and indent according to + ;; depth. This simple rule does quite well, but also has a + ;; very large extent. It would be better if we could mimic + ;; some nearby indentation. + (save-excursion + (skip-chars-forward "})") + (let ((first t)) + (dolist (nest (go-mode-nesting)) + (case (char-after (car nest)) + ((?\{) + (incf indent tab-width)) + ((?\() + (goto-char (car nest)) + (forward-comment (- (buffer-size))) + ;; Really just want the token before + (when (looking-back "\\\\|\\\\|\\w+\\s *:\\(\\S.\\|$\\)") + (decf indent tab-width)) + + ;; Continuation lines are indented 1 level + (forward-comment (- (buffer-size))) + (when (case (char-before) + ((nil ?\{ ?:) + ;; At the beginning of a block or the statement + ;; following a label. + nil) + ((?\() + ;; Usually a continuation line in an expression, + ;; unless this paren is part of a factored + ;; declaration. + (not inside-indenting-paren)) + ((?,) + ;; Could be inside a literal. We're a little + ;; conservative here and consider any comma within + ;; curly braces (as opposed to parens) to be a + ;; literal separator. This will fail to recognize + ;; line-breaks in parallel assignments as + ;; continuation lines. + (let ((depth (go-mode-nesting))) + (and depth + (not (eq (char-after (caar depth)) ?\{))))) + (t + ;; We're in the middle of a block. Did the + ;; previous line end with an implicit or explicit + ;; semicolon? + (not (go-mode-semicolon-p)))) + (incf indent tab-width)) + + (max indent 0))))))) + +(defun go-mode-indent-line () + "Indent the current line according to `go-mode-indentation'." + (interactive) + + (let ((col (go-mode-indentation))) + (when col + (let ((offset (- (current-column) (current-indentation)))) + (indent-line-to col) + (when (> offset 0) + (forward-char offset)))))) + +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +;; Go mode +;; + +;;;###autoload +(define-derived-mode go-mode nil "Go" + "Major mode for editing Go source text. + +This provides basic syntax highlighting for keywords, built-ins, +functions, and some types. It also provides indentation that is +\(almost) identical to gofmt." + + ;; Font lock + (set (make-local-variable 'font-lock-defaults) + '(go-mode-font-lock-keywords nil nil nil nil)) + + ;; Remove stale text properties + (save-restriction + (widen) + (remove-text-properties 1 (point-max) + '(go-mode-cs nil go-mode-nesting nil))) + + ;; Reset the syntax mark caches + (setq go-mode-mark-cs-end 1 + go-mode-mark-cs-state nil + go-mode-mark-nesting-end 1) + (add-hook 'after-change-functions #'go-mode-mark-clear-cache nil t) + + ;; Indentation + (set (make-local-variable 'indent-line-function) + #'go-mode-indent-line) + (add-hook 'after-change-functions #'go-mode-delayed-electric-hook nil t) + + ;; Comments + (set (make-local-variable 'comment-start) "// ") + (set (make-local-variable 'comment-end) "") + + ;; Go style + (setq indent-tabs-mode t)) + +;;;###autoload +(add-to-list 'auto-mode-alist (cons "\\.go$" #'go-mode)) + +(defun go-mode-reload () + "Reload go-mode.el and put the current buffer into Go mode. +Useful for development work." + + (interactive) + (unload-feature 'go-mode) + (require 'go-mode) + (go-mode)) + +;;;###autoload +(defun gofmt () + "Pipe the current buffer through the external tool `gofmt`. +Replace the current buffer on success; display errors on failure." + + (interactive) + (let ((srcbuf (current-buffer))) + (with-temp-buffer + (let ((outbuf (current-buffer)) + (errbuf (get-buffer-create "*Gofmt Errors*")) + (coding-system-for-read 'utf-8) ;; use utf-8 with subprocesses + (coding-system-for-write 'utf-8)) + (with-current-buffer errbuf (erase-buffer)) + (with-current-buffer srcbuf + (save-restriction + (let (deactivate-mark) + (widen) + (if (= 0 (shell-command-on-region (point-min) (point-max) "gofmt" + outbuf nil errbuf)) + ;; gofmt succeeded: replace the current buffer with outbuf, + ;; restore the mark and point, and discard errbuf. + (let ((old-mark (mark t)) (old-point (point))) + (erase-buffer) + (insert-buffer-substring outbuf) + (goto-char (min old-point (point-max))) + (if old-mark (push-mark (min old-mark (point-max)) t)) + (kill-buffer errbuf)) + + ;; gofmt failed: display the errors + (display-buffer errbuf))))) + + ;; Collapse any window opened on outbuf if shell-command-on-region + ;; displayed it. + (delete-windows-on outbuf))))) + +;;;###autoload +(defun gofmt-before-save () + "Add this to .emacs to run gofmt on the current buffer when saving: + (add-hook 'before-save-hook #'gofmt-before-save)" + + (interactive) + (when (eq major-mode 'go-mode) (gofmt))) + +(provide 'go-mode) diff --git a/misc/fraise/go.plist b/misc/fraise/go.plist new file mode 100644 index 000000000..17f416221 --- /dev/null +++ b/misc/fraise/go.plist @@ -0,0 +1,92 @@ + + + + + beginCommand + + endCommand + + beginInstruction + + endInstruction + + beginVariable + + endVariable + + firstString + " + secondString + ' + firstSingleLineComment + // + secondSingleLineComment + + beginFirstMultiLineComment + /* + endFirstMultiLineComment + */ + beginSecondMultiLineComment + + endSecondMultiLineComment + + functionDefinition + ^func\s*.*\(.*\)\s?\{ + removeFromFunction + + keywordsCaseSensitive + + recolourKeywordIfAlreadyColoured + + keywords + + break + case + chan + const + continue + default + defer + else + fallthrough + for + func + go + goto + if + import + interface + map + package + range + return + select + struct + switch + type + var + bool + byte + chan + complex64 + complex128 + float32 + float64 + int + int8 + int16 + int32 + int64 + map + string + uint + uintptr + uint8 + uint16 + uint32 + uint64 + + autocompleteWords + + + diff --git a/misc/fraise/readme.txt b/misc/fraise/readme.txt new file mode 100644 index 000000000..fb0f2c8c1 --- /dev/null +++ b/misc/fraise/readme.txt @@ -0,0 +1,16 @@ +##Instructions for enabling Go syntax highlighting in Fraise.app## +1. Move go.plist to /Applications/Fraise.app/Contents/Resources/Syntax\ Definitions/ +2. Open /Applications/Fraise.app/Contents/Resources/SyntaxDefinitions.plist and add + + + name + GoogleGo + file + go + extensions + go + + +before + +3. Restart Fraise and you're good to Go! \ No newline at end of file diff --git a/misc/godoc/README b/misc/godoc/README new file mode 100644 index 000000000..3c8d830e4 --- /dev/null +++ b/misc/godoc/README @@ -0,0 +1,22 @@ +Instructions to get an initial godoc running on a local app engine emulator +--------------------------------------------------------------------------- + +To run godoc under the app engine emulator, create a ("goroot") godoc +directory that contains the app.yaml file, the doc and lib directories +from the Go distribution, as well as a godoc directory with the godoc +sources from src/cmd/godoc. In the godoc source directory, replace +main.go with init.go. The directory structure should look as follows: + +godoc // "goroot" directory + app.yaml // app engine control file + doc // goroot/doc directory + favicon.ico + godoc // contains godoc sources + godoc.go // unchanged godoc file + init.go // this file instead of godoc/main.go + ... // remaining godoc files + lib // goroot/lib directory + +Run app engine emulator locally: dev_appserver.py -a godoc +where godoc is the top-level "goroot" directory. The godoc home page +is then served at: :8080 . diff --git a/misc/godoc/app.yaml b/misc/godoc/app.yaml new file mode 100644 index 000000000..f8b46db31 --- /dev/null +++ b/misc/godoc/app.yaml @@ -0,0 +1,12 @@ +# Copyright 2011 The Go Authors. All rights reserved. +# Use of this source code is governed by a BSD-style +# license that can be found in the LICENSE file. + +application: godoc +version: 1 +runtime: go +api_version: 1 + +handlers: +- url: /.* + script: _go_app diff --git a/misc/godoc/init.go b/misc/godoc/init.go new file mode 100644 index 000000000..0fd0bd542 --- /dev/null +++ b/misc/godoc/init.go @@ -0,0 +1,35 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// This file replaces main.go when running godoc under the app engine emulator. +// See the README file for instructions. + +package main + +import ( + "http" + "log" + "os" + "path/filepath" +) + +func serveError(w http.ResponseWriter, r *http.Request, relpath string, err os.Error) { + contents := applyTemplate(errorHTML, "errorHTML", err) // err may contain an absolute path! + w.WriteHeader(http.StatusNotFound) + servePage(w, "File "+relpath, "", "", contents) +} + +func init() { + // set goroot + cwd, err := os.Getwd() + if err != nil { + log.Fatalf("cwd: %s", err) + } + log.Printf("cwd = %s", cwd) + *goroot = filepath.Clean(cwd) + + initHandlers() + readTemplates() + registerPublicHandlers(http.DefaultServeMux) +} diff --git a/misc/goplay/Makefile b/misc/goplay/Makefile new file mode 100644 index 000000000..28d024511 --- /dev/null +++ b/misc/goplay/Makefile @@ -0,0 +1,13 @@ +# Copyright 2010 The Go Authors. All rights reserved. +# Use of this source code is governed by a BSD-style +# license that can be found in the LICENSE file. + +include ../../src/Make.inc + +TARG=goplay + +GOFILES=\ + goplay.go\ + +include ../../src/Make.cmd + diff --git a/misc/goplay/README b/misc/goplay/README new file mode 100644 index 000000000..e8a1d290f --- /dev/null +++ b/misc/goplay/README @@ -0,0 +1 @@ +See doc.go. diff --git a/misc/goplay/doc.go b/misc/goplay/doc.go new file mode 100644 index 000000000..9685551bd --- /dev/null +++ b/misc/goplay/doc.go @@ -0,0 +1,25 @@ +// Copyright 2010 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Goplay is a web interface for experimenting with Go code. +// It is similar to the Go Playground: http://golang.org/doc/play/ +// +// To use goplay, first build and install it: +// $ cd $GOROOT/misc/goplay +// $ gomake install +// Then, run it: +// $ goplay +// and load http://localhost:3999/ in a web browser. +// +// You should see a Hello World program, which you can compile and run by +// pressing shift-enter. There is also a "compile-on-keypress" feature that can +// be enabled by checking a checkbox. +// +// WARNING! CUIDADO! ACHTUNG! ATTENZIONE! +// A note on security: anyone with access to the goplay web interface can run +// arbitrary code on your computer. Goplay is not a sandbox, and has no other +// security mechanisms. Do not deploy it in untrusted environments. +// By default, goplay listens only on localhost. This can be overridden with +// the -http parameter. Do so at your own risk. +package documentation diff --git a/misc/goplay/goplay.go b/misc/goplay/goplay.go new file mode 100644 index 000000000..bbc388ba4 --- /dev/null +++ b/misc/goplay/goplay.go @@ -0,0 +1,280 @@ +// Copyright 2010 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package main + +import ( + "exec" + "flag" + "http" + "io" + "io/ioutil" + "log" + "os" + "runtime" + "strconv" + "template" +) + +var ( + httpListen = flag.String("http", "127.0.0.1:3999", "host:port to listen on") + htmlOutput = flag.Bool("html", false, "render program output as HTML") +) + +var ( + // a source of numbers, for naming temporary files + uniq = make(chan int) + // the architecture-identifying character of the tool chain, 5, 6, or 8 + archChar string +) + +func main() { + flag.Parse() + + // set archChar + switch runtime.GOARCH { + case "arm": + archChar = "5" + case "amd64": + archChar = "6" + case "386": + archChar = "8" + default: + log.Fatalln("unrecognized GOARCH:", runtime.GOARCH) + } + + // source of unique numbers + go func() { + for i := 0; ; i++ { + uniq <- i + } + }() + + http.HandleFunc("/", FrontPage) + http.HandleFunc("/compile", Compile) + log.Fatal(http.ListenAndServe(*httpListen, nil)) +} + +// FrontPage is an HTTP handler that renders the goplay interface. +// If a filename is supplied in the path component of the URI, +// its contents will be put in the interface's text area. +// Otherwise, the default "hello, world" program is displayed. +func FrontPage(w http.ResponseWriter, req *http.Request) { + data, err := ioutil.ReadFile(req.URL.Path[1:]) + if err != nil { + data = helloWorld + } + frontPage.Execute(w, data) +} + +// Compile is an HTTP handler that reads Go source code from the request, +// compiles and links the code (returning any errors), runs the program, +// and sends the program's output as the HTTP response. +func Compile(w http.ResponseWriter, req *http.Request) { + // x is the base name for .go, .6, executable files + x := os.TempDir() + "/compile" + strconv.Itoa(<-uniq) + src := x + ".go" + obj := x + "." + archChar + bin := x + if runtime.GOOS == "windows" { + bin += ".exe" + } + + // write request Body to x.go + f, err := os.Create(src) + if err != nil { + error(w, nil, err) + return + } + defer os.Remove(src) + defer f.Close() + _, err = io.Copy(f, req.Body) + if err != nil { + error(w, nil, err) + return + } + f.Close() + + // build x.go, creating x.6 + out, err := run(archChar+"g", "-o", obj, src) + defer os.Remove(obj) + if err != nil { + error(w, out, err) + return + } + + // link x.6, creating x (the program binary) + out, err = run(archChar+"l", "-o", bin, obj) + defer os.Remove(bin) + if err != nil { + error(w, out, err) + return + } + + // run x + out, err = run(bin) + if err != nil { + error(w, out, err) + } + + // write the output of x as the http response + if *htmlOutput { + w.Write(out) + } else { + output.Execute(w, out) + } +} + +// error writes compile, link, or runtime errors to the HTTP connection. +// The JavaScript interface uses the 404 status code to identify the error. +func error(w http.ResponseWriter, out []byte, err os.Error) { + w.WriteHeader(404) + if out != nil { + output.Execute(w, out) + } else { + output.Execute(w, err.String()) + } +} + +// run executes the specified command and returns its output and an error. +func run(cmd ...string) ([]byte, os.Error) { + return exec.Command(cmd[0], cmd[1:]...).CombinedOutput() +} + +var frontPage = template.Must(template.New("frontPage").Parse(frontPageText)) // HTML template +var output = template.Must(template.New("output").Parse(outputText)) // HTML template + +var outputText = `
{{html .}}
` + +var frontPageText = ` + + + + + + +
+ +
+(Shift-Enter to compile and run.)     + Compile and run after each keystroke +
+
+ +
+
+
+ + +` + +var helloWorld = []byte(`package main + +import "fmt" + +func main() { + fmt.Println("hello, world") +} +`) diff --git a/misc/kate/go.xml b/misc/kate/go.xml new file mode 100644 index 000000000..14d88b26a --- /dev/null +++ b/misc/kate/go.xml @@ -0,0 +1,147 @@ + + + + + + break + case + chan + const + continue + default + defer + else + fallthrough + for + func + go + goto + if + import + interface + map + package + range + return + select + struct + switch + type + var + + + false + iota + nil + true + + + bool + byte + complex64 + complex128 + float32 + float64 + int + int8 + int16 + int32 + int64 + string + uint + uintptr + uint8 + uint16 + uint32 + uint64 + + + append + cap + close + complex + copy + imag + len + make + new + panic + print + println + real + recover + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/misc/notepadplus/README b/misc/notepadplus/README new file mode 100755 index 000000000..000d31746 --- /dev/null +++ b/misc/notepadplus/README @@ -0,0 +1,8 @@ +Given a Notepad++ installation at : + +1. Add the contents of userDefineLang.xml at \userDefineLang.xml + between ... + +2. Copy go.xml to \plugins\APIs + +3. Restart Notepad++ diff --git a/misc/notepadplus/go.xml b/misc/notepadplus/go.xml new file mode 100755 index 000000000..7c5d8a173 --- /dev/null +++ b/misc/notepadplus/go.xml @@ -0,0 +1,66 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/misc/notepadplus/userDefineLang.xml b/misc/notepadplus/userDefineLang.xml new file mode 100755 index 000000000..d1927a340 --- /dev/null +++ b/misc/notepadplus/userDefineLang.xml @@ -0,0 +1,36 @@ + + + + + + + + + "`0"` + + + ( ) [ ] { } ... . , _ & ^ % > < ! = + - * | : + 1/* 2*/ 0// + append bool break byte cap case chan close complex complex128 complex64 const continue copy default defer else fallthrough false float32 float64 for func go goto if iota imag import int int16 int32 int64 int8 interface len make map new nil package panic print println range real recover return select string struct switch true type uint uint16 uint32 uint64 uint8 uintptr var + + + + + + + + + + + + + + + + + + + + + + diff --git a/misc/swig/callback/Makefile b/misc/swig/callback/Makefile new file mode 100644 index 000000000..fde0d107b --- /dev/null +++ b/misc/swig/callback/Makefile @@ -0,0 +1,17 @@ +# Copyright 2011 The Go Authors. All rights reserved. +# Use of this source code is governed by a BSD-style +# license that can be found in the LICENSE file. + +include ../../../src/Make.inc + +TARG=swig/callback +SWIGFILES=\ + callback.swigcxx + +CLEANFILES+=run + +include ../../../src/Make.pkg + +%: install %.go + $(GC) $*.go + $(LD) $(SWIG_RPATH) -o $@ $*.$O diff --git a/misc/swig/callback/callback.h b/misc/swig/callback/callback.h new file mode 100644 index 000000000..80232a8b3 --- /dev/null +++ b/misc/swig/callback/callback.h @@ -0,0 +1,24 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +class Callback { +public: + virtual ~Callback() { } + virtual std::string run() { return "Callback::run"; } +}; + +class Caller { +private: + Callback *callback_; +public: + Caller(): callback_(0) { } + ~Caller() { delCallback(); } + void delCallback() { delete callback_; callback_ = 0; } + void setCallback(Callback *cb) { delCallback(); callback_ = cb; } + std::string call() { + if (callback_ != 0) + return callback_->run(); + return ""; + } +}; diff --git a/misc/swig/callback/callback.swigcxx b/misc/swig/callback/callback.swigcxx new file mode 100644 index 000000000..0c97ef101 --- /dev/null +++ b/misc/swig/callback/callback.swigcxx @@ -0,0 +1,18 @@ +/* Copyright 2011 The Go Authors. All rights reserved. + Use of this source code is governed by a BSD-style + license that can be found in the LICENSE file. */ + +/* An example of writing a C++ virtual function in Go. */ + +%module(directors="1") callback + +%{ +#include +#include "callback.h" +%} + +%include "std_string.i" + +%feature("director"); + +%include "callback.h" diff --git a/misc/swig/callback/run b/misc/swig/callback/run new file mode 100755 index 000000000..de150ed05 Binary files /dev/null and b/misc/swig/callback/run differ diff --git a/misc/swig/callback/run.go b/misc/swig/callback/run.go new file mode 100644 index 000000000..a76e636cb --- /dev/null +++ b/misc/swig/callback/run.go @@ -0,0 +1,39 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package main + +import ( + "swig/callback" + "fmt" +) + +type GoCallback struct{} + +func (p *GoCallback) Run() string { + return "GoCallback.Run" +} + +func main() { + c := callback.NewCaller() + cb := callback.NewCallback() + + c.SetCallback(cb) + s := c.Call() + fmt.Println(s) + if s != "Callback::run" { + panic(s) + } + c.DelCallback() + + cb = callback.NewDirectorCallback(&GoCallback{}) + c.SetCallback(cb) + s = c.Call() + fmt.Println(s) + if s != "GoCallback.Run" { + panic(s) + } + c.DelCallback() + callback.DeleteDirectorCallback(cb) +} diff --git a/misc/swig/stdio/Makefile b/misc/swig/stdio/Makefile new file mode 100644 index 000000000..e7d330587 --- /dev/null +++ b/misc/swig/stdio/Makefile @@ -0,0 +1,17 @@ +# Copyright 2011 The Go Authors. All rights reserved. +# Use of this source code is governed by a BSD-style +# license that can be found in the LICENSE file. + +include ../../../src/Make.inc + +TARG=swig/file +SWIGFILES=\ + file.swig + +CLEANFILES+=hello + +include ../../../src/Make.pkg + +%: install %.go + $(GC) $*.go + $(LD) $(SWIG_RPATH) -o $@ $*.$O diff --git a/misc/swig/stdio/file.swig b/misc/swig/stdio/file.swig new file mode 100644 index 000000000..57c623f8f --- /dev/null +++ b/misc/swig/stdio/file.swig @@ -0,0 +1,11 @@ +/* Copyright 2011 The Go Authors. All rights reserved. + Use of this source code is governed by a BSD-style + license that can be found in the LICENSE file. */ + +/* A trivial example of wrapping a C library using SWIG. */ + +%{ +#include +%} + +int puts(const char *); diff --git a/misc/swig/stdio/hello b/misc/swig/stdio/hello new file mode 100755 index 000000000..10c55631f Binary files /dev/null and b/misc/swig/stdio/hello differ diff --git a/misc/swig/stdio/hello.go b/misc/swig/stdio/hello.go new file mode 100644 index 000000000..eec294278 --- /dev/null +++ b/misc/swig/stdio/hello.go @@ -0,0 +1,11 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package main + +import "swig/file" + +func main() { + file.Puts("Hello, world") +} diff --git a/misc/vim/autoload/go/complete.vim b/misc/vim/autoload/go/complete.vim new file mode 100644 index 000000000..d4ae3b97f --- /dev/null +++ b/misc/vim/autoload/go/complete.vim @@ -0,0 +1,49 @@ +" Copyright 2011 The Go Authors. All rights reserved. +" Use of this source code is governed by a BSD-style +" license that can be found in the LICENSE file. +" +" This file provides a utility function that performs auto-completion of +" package names, for use by other commands. + +let s:goos = $GOOS +let s:goarch = $GOARCH + +if len(s:goos) == 0 + if exists('g:golang_goos') + let s:goos = g:golang_goos + elseif has('win32') || has('win64') + let s:goos = 'windows' + elseif has('macunix') + let s:goos = 'darwin' + else + let s:goos = '*' + endif +endif + +if len(s:goarch) == 0 + if exists('g:golang_goarch') + let s:goarch = g:golang_goarch + else + let s:goarch = '*' + endif +endif + +function! go#complete#Package(ArgLead, CmdLine, CursorPos) + let goroot = $GOROOT + if len(goroot) == 0 + " should not occur. + return [] + endif + let ret = {} + let root = expand(goroot.'/pkg/'.s:goos.'_'.s:goarch) + for i in split(globpath(root, a:ArgLead.'*'), "\n") + if isdirectory(i) + let i .= '/' + elseif i !~ '\.a$' + continue + endif + let i = substitute(substitute(i[len(root)+1:], '[\\]', '/', 'g'), '\.a$', '', 'g') + let ret[i] = i + endfor + return sort(keys(ret)) +endfunction diff --git a/misc/vim/ftdetect/gofiletype.vim b/misc/vim/ftdetect/gofiletype.vim new file mode 100644 index 000000000..f03a1d8dc --- /dev/null +++ b/misc/vim/ftdetect/gofiletype.vim @@ -0,0 +1 @@ +au BufReadPre,BufNewFile *.go set filetype=go fileencoding=utf-8 fileencodings=utf-8 diff --git a/misc/vim/ftplugin/go/fmt.vim b/misc/vim/ftplugin/go/fmt.vim new file mode 100644 index 000000000..a299dfcee --- /dev/null +++ b/misc/vim/ftplugin/go/fmt.vim @@ -0,0 +1,30 @@ +" Copyright 2011 The Go Authors. All rights reserved. +" Use of this source code is governed by a BSD-style +" license that can be found in the LICENSE file. +" +" fmt.vim: Vim command to format Go files with gofmt. +" +" This filetype plugin add a new commands for go buffers: +" +" :Fmt +" +" Filter the current Go buffer through gofmt. +" It tries to preserve cursor position and avoids +" replacing the buffer with stderr output. +" + +command! -buffer Fmt call s:GoFormat() + +function! s:GoFormat() + let view = winsaveview() + %!gofmt + if v:shell_error + %| " output errors returned by gofmt + " TODO(dchest): perhaps, errors should go to quickfix + undo + echohl Error | echomsg "Gofmt returned error" | echohl None + endif + call winrestview(view) +endfunction + +" vim:ts=4:sw=4:et diff --git a/misc/vim/ftplugin/go/godoc.vim b/misc/vim/ftplugin/go/godoc.vim new file mode 100644 index 000000000..55195a674 --- /dev/null +++ b/misc/vim/ftplugin/go/godoc.vim @@ -0,0 +1,13 @@ +" Copyright 2011 The Go Authors. All rights reserved. +" Use of this source code is governed by a BSD-style +" license that can be found in the LICENSE file. +" +" godoc.vim: Vim command to see godoc. + +if exists("b:did_ftplugin") + finish +endif + +silent! nmap K (godoc-keyword) + +" vim:ts=4:sw=4:et diff --git a/misc/vim/ftplugin/go/import.vim b/misc/vim/ftplugin/go/import.vim new file mode 100644 index 000000000..6705a476b --- /dev/null +++ b/misc/vim/ftplugin/go/import.vim @@ -0,0 +1,201 @@ +" Copyright 2011 The Go Authors. All rights reserved. +" Use of this source code is governed by a BSD-style +" license that can be found in the LICENSE file. +" +" import.vim: Vim commands to import/drop Go packages. +" +" This filetype plugin adds three new commands for go buffers: +" +" :Import {path} +" +" Import ensures that the provided package {path} is imported +" in the current Go buffer, using proper style and ordering. +" If {path} is already being imported, an error will be +" displayed and the buffer will be untouched. +" +" :ImportAs {localname} {path} +" +" Same as Import, but uses a custom local name for the package. +" +" :Drop {path} +" +" Remove the import line for the provided package {path}, if +" present in the current Go buffer. If {path} is not being +" imported, an error will be displayed and the buffer will be +" untouched. +" +" In addition to these commands, there are also two shortcuts mapped: +" +" \f - Runs :Import fmt +" \F - Runs :Drop fmt +" +" The backslash is the default maplocalleader, so it is possible that +" your vim is set to use a different character (:help maplocalleader). +" +if exists("b:did_ftplugin") + finish +endif + +command! -buffer -nargs=? -complete=customlist,go#complete#Package Drop call s:SwitchImport(0, '', ) +command! -buffer -nargs=1 -complete=customlist,go#complete#Package Import call s:SwitchImport(1, '', ) +command! -buffer -nargs=* -complete=customlist,go#complete#Package ImportAs call s:SwitchImport(1, ) +map f :Import fmt +map F :Drop fmt + +function! s:SwitchImport(enabled, localname, path) + let view = winsaveview() + let path = a:path + + " Quotes are not necessary, so remove them if provided. + if path[0] == '"' + let path = strpart(path, 1) + endif + if path[len(path)-1] == '"' + let path = strpart(path, 0, len(path) - 1) + endif + if path == '' + call s:Error('Import path not provided') + return + endif + + let qpath = '"' . path . '"' + if a:localname != '' + let qlocalpath = a:localname . ' ' . qpath + else + let qlocalpath = qpath + endif + let indentstr = 0 + let packageline = -1 " Position of package name statement + let appendline = -1 " Position to introduce new import + let deleteline = -1 " Position of line with existing import + let linesdelta = 0 " Lines added/removed + + " Find proper place to add/remove import. + let line = 0 + while line <= line('$') + let linestr = getline(line) + + if linestr =~# '^package\s' + let packageline = line + let appendline = line + + elseif linestr =~# '^import\s\+(' + let appendstr = qlocalpath + let indentstr = 1 + let appendline = line + while line <= line("$") + let line = line + 1 + let linestr = getline(line) + let m = matchlist(getline(line), '^\()\|\(\s\+\)\(\S*\s*\)"\(.\+\)"\)') + if empty(m) + continue + endif + if m[1] == ')' + break + endif + if a:localname != '' && m[3] != '' + let qlocalpath = printf('%-' . (len(m[3])-1) . 's %s', a:localname, qpath) + endif + let appendstr = m[2] . qlocalpath + let indentstr = 0 + if m[4] == path + let appendline = -1 + let deleteline = line + break + elseif m[4] < path + let appendline = line + endif + endwhile + break + + elseif linestr =~# '^import ' + if appendline == packageline + let appendstr = 'import ' . qlocalpath + let appendline = line - 1 + endif + let m = matchlist(linestr, '^import\(\s\+\)\(\S*\s*\)"\(.\+\)"') + if !empty(m) + if m[3] == path + let appendline = -1 + let deleteline = line + break + endif + if m[3] < path + let appendline = line + endif + if a:localname != '' && m[2] != '' + let qlocalpath = printf("%s %" . len(m[2])-1 . "s", a:localname, qpath) + endif + let appendstr = 'import' . m[1] . qlocalpath + endif + + elseif linestr =~# '^\(var\|const\|type\|func\)\>' + break + + endif + let line = line + 1 + endwhile + + " Append or remove the package import, as requested. + if a:enabled + if deleteline != -1 + call s:Error(qpath . ' already being imported') + elseif appendline == -1 + call s:Error('No package line found') + else + if appendline == packageline + call append(appendline + 0, '') + call append(appendline + 1, 'import (') + call append(appendline + 2, ')') + let appendline += 2 + let linesdelta += 3 + let appendstr = qlocalpath + let indentstr = 1 + endif + call append(appendline, appendstr) + execute appendline + 1 + if indentstr + execute 'normal >>' + endif + let linesdelta += 1 + endif + else + if deleteline == -1 + call s:Error(qpath . ' not being imported') + else + execute deleteline . 'd' + let linesdelta -= 1 + + if getline(deleteline-1) =~# '^import\s\+(' && getline(deleteline) =~# '^)' + " Delete empty import block + let deleteline -= 1 + execute deleteline . "d" + execute deleteline . "d" + let linesdelta -= 2 + endif + + if getline(deleteline) == '' && getline(deleteline - 1) == '' + " Delete spacing for removed line too. + execute deleteline . "d" + let linesdelta -= 1 + endif + endif + endif + + " Adjust view for any changes. + let view.lnum += linesdelta + let view.topline += linesdelta + if view.topline < 0 + let view.topline = 0 + endif + + " Put buffer back where it was. + call winrestview(view) + +endfunction + +function! s:Error(s) + echohl Error | echo a:s | echohl None +endfunction + +" vim:ts=4:sw=4:et diff --git a/misc/vim/indent/go.vim b/misc/vim/indent/go.vim new file mode 100644 index 000000000..faf4d79e2 --- /dev/null +++ b/misc/vim/indent/go.vim @@ -0,0 +1,65 @@ +" Copyright 2011 The Go Authors. All rights reserved. +" Use of this source code is governed by a BSD-style +" license that can be found in the LICENSE file. +" +" indent/go.vim: Vim indent file for Go. +" +" TODO: +" - function invocations split across lines +" - general line splits (line ends in an operator) + +if exists("b:did_indent") + finish +endif +let b:did_indent = 1 + +" C indentation is too far off useful, mainly due to Go's := operator. +" Let's just define our own. +setlocal nolisp +setlocal autoindent +setlocal indentexpr=GoIndent(v:lnum) +setlocal indentkeys+=<:>,0=},0=) + +if exists("*GoIndent") + finish +endif + +function! GoIndent(lnum) + let prevlnum = prevnonblank(a:lnum-1) + if prevlnum == 0 + " top of file + return 0 + endif + + " grab the previous and current line, stripping comments. + let prevl = substitute(getline(prevlnum), '//.*$', '', '') + let thisl = substitute(getline(a:lnum), '//.*$', '', '') + let previ = indent(prevlnum) + + let ind = previ + + if prevl =~ '[({]\s*$' + " previous line opened a block + let ind += &sw + endif + if prevl =~# '^\s*\(case .*\|default\):$' + " previous line is part of a switch statement + let ind += &sw + endif + " TODO: handle if the previous line is a label. + + if thisl =~ '^\s*[)}]' + " this line closed a block + let ind -= &sw + endif + + " Colons are tricky. + " We want to outdent if it's part of a switch ("case foo:" or "default:"). + " We ignore trying to deal with jump labels because (a) they're rare, and + " (b) they're hard to disambiguate from a composite literal key. + if thisl =~# '^\s*\(case .*\|default\):$' + let ind -= &sw + endif + + return ind +endfunction diff --git a/misc/vim/plugin/godoc.vim b/misc/vim/plugin/godoc.vim new file mode 100644 index 000000000..fdb496631 --- /dev/null +++ b/misc/vim/plugin/godoc.vim @@ -0,0 +1,85 @@ +" Copyright 2011 The Go Authors. All rights reserved. +" Use of this source code is governed by a BSD-style +" license that can be found in the LICENSE file. +" +" godoc.vim: Vim command to see godoc. + +if exists("g:loaded_godoc") + finish +endif +let g:loaded_godoc = 1 + +let s:buf_nr = -1 +let s:last_word = '' + +function! s:GodocView() + if !bufexists(s:buf_nr) + leftabove new + file `="[Godoc]"` + let s:buf_nr = bufnr('%') + elseif bufwinnr(s:buf_nr) == -1 + leftabove split + execute s:buf_nr . 'buffer' + delete _ + elseif bufwinnr(s:buf_nr) != bufwinnr('%') + execute bufwinnr(s:buf_nr) . 'wincmd w' + endif + + setlocal filetype=godoc + setlocal bufhidden=delete + setlocal buftype=nofile + setlocal noswapfile + setlocal nobuflisted + setlocal modifiable + setlocal nocursorline + setlocal nocursorcolumn + setlocal iskeyword+=: + setlocal iskeyword-=- + + nnoremap K :Godoc + + au BufHidden call let buf_nr = -1 +endfunction + +function! s:GodocWord(word) + let word = a:word + silent! let content = system('godoc ' . word) + if v:shell_error || !len(content) + if len(s:last_word) + silent! let content = system('godoc ' . s:last_word.'/'.word) + if v:shell_error || !len(content) + echo 'No documentation found for "' . word . '".' + return + endif + let word = s:last_word.'/'.word + else + echo 'No documentation found for "' . word . '".' + return + endif + endif + let s:last_word = word + silent! call s:GodocView() + setlocal modifiable + silent! %d _ + silent! put! =content + silent! normal gg + setlocal nomodifiable + setfiletype godoc +endfunction + +function! s:Godoc(...) + let word = join(a:000, ' ') + if !len(word) + let word = expand('') + endif + let word = substitute(word, '[^a-zA-Z0-9\/]', '', 'g') + if !len(word) + return + endif + call s:GodocWord(word) +endfunction + +command! -nargs=* -range -complete=customlist,go#complete#Package Godoc :call s:Godoc() +nnoremap (godoc-keyword) :call Godoc('') + +" vim:ts=4:sw=4:et diff --git a/misc/vim/readme.txt b/misc/vim/readme.txt new file mode 100644 index 000000000..fe15da993 --- /dev/null +++ b/misc/vim/readme.txt @@ -0,0 +1,76 @@ +Vim plugins for Go (http://golang.org) +====================================== + +To use all the Vim plugins, add these lines to your vimrc. + + set rtp+=$GOROOT/misc/vim + filetype plugin indent on + syntax on + +If you want to select fewer plugins, use the instructions in the rest of +this file. + +Vim syntax highlighting +----------------------- + +To install automatic syntax highlighting for GO programs: + + 1. Copy or link the filetype detection script to the ftdetect directory + underneath your vim runtime directory (normally $HOME/.vim/ftdetect) + 2. Copy or link syntax/go.vim to the syntax directory underneath your vim + runtime directory (normally $HOME/.vim/syntax). Linking this file rather + than just copying it will ensure any changes are automatically reflected + in your syntax highlighting. + 3. Add the following line to your .vimrc file (normally $HOME/.vimrc): + + syntax on + +In a typical unix environment you might accomplish this using the following +commands: + + mkdir -p $HOME/.vim/ftdetect + mkdir -p $HOME/.vim/syntax + mkdir -p $HOME/.vim/autoload/go + ln -s $GOROOT/misc/vim/ftdetect/gofiletype.vim $HOME/.vim/ftdetect/ + ln -s $GOROOT/misc/vim/syntax/go.vim $HOME/.vim/syntax + ln -s $GOROOT/misc/vim/autoload/go/complete.vim $HOME/.vim/autoload/go + echo "syntax on" >> $HOME/.vimrc + + +Vim filetype plugins +-------------------- + +To install one of the available filetype plugins: + + 1. Same as 1 above. + 2. Copy or link one or more plugins from ftplugin/go/*.vim to the + Go-specific ftplugin directory underneath your vim runtime directory + (normally $HOME/.vim/ftplugin/go/*.vim). + 3. Add the following line to your .vimrc file (normally $HOME/.vimrc): + + filetype plugin on + + +Vim indentation plugin +---------------------- + +To install automatic indentation: + + 1. Same as 1 above. + 2. Copy or link indent/go.vim to the indent directory underneath your vim + runtime directory (normally $HOME/.vim/indent). + 3. Add the following line to your .vimrc file (normally $HOME/.vimrc): + + filetype indent on + + +Godoc plugin +------------ + +To install godoc plugin: + + 1. Same as 1 above. + 2. Copy or link plugin/godoc.vim to $HOME/.vim/plugin/godoc, + syntax/godoc.vim to $HOME/.vim/syntax/godoc.vim, + ftplugin/go/godoc.vim to $HOME/.vim/ftplugin/go/godoc.vim. + and autoload/go/complete.vim to $HOME/.vim/autoload/go/complete.vim. diff --git a/misc/vim/syntax/go.vim b/misc/vim/syntax/go.vim new file mode 100644 index 000000000..26d7defe3 --- /dev/null +++ b/misc/vim/syntax/go.vim @@ -0,0 +1,208 @@ +" Copyright 2009 The Go Authors. All rights reserved. +" Use of this source code is governed by a BSD-style +" license that can be found in the LICENSE file. +" +" go.vim: Vim syntax file for Go. +" +" Options: +" There are some options for customizing the highlighting; the recommended +" settings are the default values, but you can write: +" let OPTION_NAME = 0 +" in your ~/.vimrc file to disable particular options. You can also write: +" let OPTION_NAME = 1 +" to enable particular options. At present, all options default to on. +" +" - go_highlight_array_whitespace_error +" Highlights white space after "[]". +" - go_highlight_chan_whitespace_error +" Highlights white space around the communications operator that don't follow +" the standard style. +" - go_highlight_extra_types +" Highlights commonly used library types (os.Error, etc.). +" - go_highlight_space_tab_error +" Highlights instances of tabs following spaces. +" - go_highlight_trailing_whitespace_error +" Highlights trailing white space. + +" Quit when a (custom) syntax file was already loaded +if exists("b:current_syntax") + finish +endif + +if !exists("go_highlight_array_whitespace_error") + let go_highlight_array_whitespace_error = 1 +endif +if !exists("go_highlight_chan_whitespace_error") + let go_highlight_chan_whitespace_error = 1 +endif +if !exists("go_highlight_extra_types") + let go_highlight_extra_types = 1 +endif +if !exists("go_highlight_space_tab_error") + let go_highlight_space_tab_error = 1 +endif +if !exists("go_highlight_trailing_whitespace_error") + let go_highlight_trailing_whitespace_error = 1 +endif + +syn case match + +syn keyword goDirective package import +syn keyword goDeclaration var const type +syn keyword goDeclType struct interface + +hi def link goDirective Statement +hi def link goDeclaration Keyword +hi def link goDeclType Keyword + +" Keywords within functions +syn keyword goStatement defer go goto return break continue fallthrough +syn keyword goConditional if else switch select +syn keyword goLabel case default +syn keyword goRepeat for range + +hi def link goStatement Statement +hi def link goConditional Conditional +hi def link goLabel Label +hi def link goRepeat Repeat + +" Predefined types +syn keyword goType chan map bool string +syn keyword goSignedInts int int8 int16 int32 int64 +syn keyword goUnsignedInts byte uint uint8 uint16 uint32 uint64 uintptr +syn keyword goFloats float32 float64 +syn keyword goComplexes complex64 complex128 + +hi def link goType Type +hi def link goSignedInts Type +hi def link goUnsignedInts Type +hi def link goFloats Type +hi def link goComplexes Type + +" Treat func specially: it's a declaration at the start of a line, but a type +" elsewhere. Order matters here. +syn match goType /\/ +syn match goDeclaration /^func\>/ + +" Predefined functions and values +syn keyword goBuiltins append cap close complex copy imag len +syn keyword goBuiltins make new panic print println real recover +syn keyword goConstants iota true false nil + +hi def link goBuiltins Keyword +hi def link goConstants Keyword + +" Comments; their contents +syn keyword goTodo contained TODO FIXME XXX BUG +syn cluster goCommentGroup contains=goTodo +syn region goComment start="/\*" end="\*/" contains=@goCommentGroup,@Spell +syn region goComment start="//" end="$" contains=@goCommentGroup,@Spell + +hi def link goComment Comment +hi def link goTodo Todo + +" Go escapes +syn match goEscapeOctal display contained "\\[0-7]\{3}" +syn match goEscapeC display contained +\\[abfnrtv\\'"]+ +syn match goEscapeX display contained "\\x\x\{2}" +syn match goEscapeU display contained "\\u\x\{4}" +syn match goEscapeBigU display contained "\\U\x\{8}" +syn match goEscapeError display contained +\\[^0-7xuUabfnrtv\\'"]+ + +hi def link goEscapeOctal goSpecialString +hi def link goEscapeC goSpecialString +hi def link goEscapeX goSpecialString +hi def link goEscapeU goSpecialString +hi def link goEscapeBigU goSpecialString +hi def link goSpecialString Special +hi def link goEscapeError Error + +" Strings and their contents +syn cluster goStringGroup contains=goEscapeOctal,goEscapeC,goEscapeX,goEscapeU,goEscapeBigU,goEscapeError +syn region goString start=+"+ skip=+\\\\\|\\"+ end=+"+ contains=@goStringGroup +syn region goRawString start=+`+ end=+`+ + +hi def link goString String +hi def link goRawString String + +" Characters; their contents +syn cluster goCharacterGroup contains=goEscapeOctal,goEscapeC,goEscapeX,goEscapeU,goEscapeBigU +syn region goCharacter start=+'+ skip=+\\\\\|\\'+ end=+'+ contains=@goCharacterGroup + +hi def link goCharacter Character + +" Regions +syn region goBlock start="{" end="}" transparent fold +syn region goParen start='(' end=')' transparent + +" Integers +syn match goDecimalInt "\<\d\+\([Ee]\d\+\)\?\>" +syn match goHexadecimalInt "\<0x\x\+\>" +syn match goOctalInt "\<0\o\+\>" +syn match goOctalError "\<0\o*[89]\d*\>" + +hi def link goDecimalInt Integer +hi def link goHexadecimalInt Integer +hi def link goOctalInt Integer +hi def link Integer Number + +" Floating point +syn match goFloat "\<\d\+\.\d*\([Ee][-+]\d\+\)\?\>" +syn match goFloat "\<\.\d\+\([Ee][-+]\d\+\)\?\>" +syn match goFloat "\<\d\+[Ee][-+]\d\+\>" + +hi def link goFloat Float + +" Imaginary literals +syn match goImaginary "\<\d\+i\>" +syn match goImaginary "\<\d\+\.\d*\([Ee][-+]\d\+\)\?i\>" +syn match goImaginary "\<\.\d\+\([Ee][-+]\d\+\)\?i\>" +syn match goImaginary "\<\d\+[Ee][-+]\d\+i\>" + +hi def link goImaginary Number + +" Spaces after "[]" +if go_highlight_array_whitespace_error != 0 + syn match goSpaceError display "\(\[\]\)\@<=\s\+" +endif + +" Spacing errors around the 'chan' keyword +if go_highlight_chan_whitespace_error != 0 + " receive-only annotation on chan type + syn match goSpaceError display "\(<-\)\@<=\s\+\(chan\>\)\@=" + " send-only annotation on chan type + syn match goSpaceError display "\(\/ + syn match goExtraType /\/ + syn match goExtraType /\<\(os\.Error\)\>/ + syn match goExtraType /\/ + syn match goExtraType /\/ +endif + +" Space-tab error +if go_highlight_space_tab_error != 0 + syn match goSpaceError display " \+\t"me=e-1 +endif + +" Trailing white space error +if go_highlight_trailing_whitespace_error != 0 + syn match goSpaceError display excludenl "\s\+$" +endif + +hi def link goExtraType Type +hi def link goSpaceError Error + +" Search backwards for a global declaration to start processing the syntax. +"syn sync match goSync grouphere NONE /^\(const\|var\|type\|func\)\>/ + +" There's a bug in the implementation of grouphere. For now, use the +" following as a more expensive/less precise workaround. +syn sync minlines=500 + +let b:current_syntax = "go" diff --git a/misc/vim/syntax/godoc.vim b/misc/vim/syntax/godoc.vim new file mode 100644 index 000000000..82f78aa3c --- /dev/null +++ b/misc/vim/syntax/godoc.vim @@ -0,0 +1,20 @@ +" Copyright 2011 The Go Authors. All rights reserved. +" Use of this source code is governed by a BSD-style +" license that can be found in the LICENSE file. + +if exists("b:current_syntax") + finish +endif + +syn case match +syn match godocTitle "^\([A-Z]*\)$" + +command -nargs=+ HiLink hi def link + +HiLink godocTitle Title + +delcommand HiLink + +let b:current_syntax = "godoc" + +" vim:ts=4 sts=2 sw=2: diff --git a/misc/xcode/go.pbfilespec b/misc/xcode/go.pbfilespec new file mode 100644 index 000000000..1034778f5 --- /dev/null +++ b/misc/xcode/go.pbfilespec @@ -0,0 +1,31 @@ +/* + Copyright 2009 The Go Authors. All rights reserved. + Use of this source code is governed by a BSD-style + license that can be found in the LICENSE file. + + go.pbfilespec + Go source file spec for Xcode 3 + + There is not much documentation available regarding the format + of .pbfilespec files. As a starting point, see for instance the + outdated documentation at: + http://maxao.free.fr/xcode-plugin-interface/specifications.html + and the files in: + /Developer/Library/PrivateFrameworks/XcodeEdit.framework/Versions/A/Resources/ + + Place this file in directory: + ~/Library/Application Support/Developer/Shared/Xcode/Specifications/ +*/ + +( + { + Identifier = sourcecode.go; + BasedOn = sourcecode; + Name = "Go Files"; + Extensions = ("go"); + MIMETypes = ("text/go"); + Language = "xcode.lang.go"; + IsTextFile = YES; + IsSourceFile = YES; + } +) diff --git a/misc/xcode/go.xclangspec b/misc/xcode/go.xclangspec new file mode 100644 index 000000000..e515564da --- /dev/null +++ b/misc/xcode/go.xclangspec @@ -0,0 +1,293 @@ +/* + Copyright 2009 The Go Authors. All rights reserved. + Use of this source code is governed by a BSD-style + license that can be found in the LICENSE file. + + Go.xclangspec + Go language specification for Xcode 3 + + This is a preliminary version that supports basic syntax high-lighting + (such as keywords, literals, and comments) and an attempt to provide + some structure information (incomplete). + + There is not much documentation available regarding the format + of .xclangspec files. As a starting point, see for instance the + outdated documentation at: + http://maxao.free.fr/xcode-plugin-interface/specifications.html + and the files in: + /Developer/Library/PrivateFrameworks/XcodeEdit.framework/Versions/A/Resources/ + + Place this file in directory: + ~/Library/Application Support/Developer/Shared/Xcode/Specifications/ +*/ + +( + +// ---------------------------------------------------------------------------- +// Keywords + +// TODO How do we get general Unicode identifiers? + + { + Identifier = "xcode.lang.go.identifier"; + Syntax = { + StartChars = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ_"; + Chars = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789_"; + Words = ( + "break", + "case", + "chan", + "const", + "continue", + + "default", + "defer", + "else", + "fallthrough", + "for", + + "func", + "go", + "goto", + "if", + "import", + + "interface", + "map", + "package", + "range", + "return", + + "select", + "struct", + "switch", + "type", + "var", + ); + Type = "xcode.syntax.keyword"; + AltType = "xcode.syntax.identifier"; // non-keywords are identifiers + }; + }, + +// TODO decide what should go here, if anything + { + Identifier = "xcode.lang.go.interestingOperators"; + Syntax = { + Words = ( + "...", + ".", + "*", + ",", + ":", + ); + Type = "xcode.syntax.plain"; + }; + }, + + { + Identifier = "xcode.lang.go.rawstring"; + Syntax = { + Start = "`"; + End = "`"; + Type = "xcode.syntax.string"; + }; + }, + +// ---------------------------------------------------------------------------- +// Syntax Coloring + + { + Identifier = "xcode.lang.go"; + Description = "Go Coloring"; + BasedOn = "xcode.lang.simpleColoring"; + IncludeInMenu = YES; + Name = "Go"; + Syntax = { + Tokenizer = "xcode.lang.go.lexer.toplevel"; + IncludeRules = ( + "xcode.lang.go.block", + "xcode.lang.go.bracketexpr", + "xcode.lang.go.parenexpr", + ); + Type = "xcode.syntax.plain"; + }; + }, + + // The following rule returns tokens to the other rules + { + Identifier = "xcode.lang.go.lexer"; + Syntax = { + IncludeRules = ( + "xcode.lang.go.comment", + "xcode.lang.go.comment.singleline", + "xcode.lang.string", + "xcode.lang.character", + "xcode.lang.go.rawstring", + "xcode.lang.go.identifier", + "xcode.lang.number", + "xcode.lang.go.interestingOperators", + ); + }; + }, + + { + Identifier = "xcode.lang.go.lexer.toplevel"; + Syntax = { + IncludeRules = ( + "xcode.lang.go.comment", + "xcode.lang.go.comment.singleline", + "xcode.lang.string", + "xcode.lang.character", + "xcode.lang.go.rawstring", + "xcode.lang.go.type.declaration", + "xcode.lang.go.method.declaration", + "xcode.lang.go.function.declaration", + "xcode.lang.go.identifier", + "xcode.lang.number", + ); + }; + }, + + { + Identifier = "xcode.lang.go.method.declaration"; + Syntax = { + Tokenizer = "xcode.lang.go.lexer"; + Rules = ( + "func", + "xcode.lang.go.parenexpr", + "xcode.lang.go.identifier", + "xcode.lang.go.parenexpr", + ); + Type = "xcode.syntax.declaration.method"; + }; + }, + + { + Identifier = "xcode.lang.go.type.declaration"; + Syntax = { + Tokenizer = "xcode.lang.go.lexer"; + Rules = ( + "type", + "xcode.lang.go.identifier", + ); + Type = "xcode.syntax.typedef"; + }; + }, + + { + Identifier = "xcode.lang.go.function.declaration"; + Syntax = { + Tokenizer = "xcode.lang.go.lexer"; + Rules = ( + "func", + "xcode.lang.go.identifier", + "xcode.lang.go.parenexpr", + ); + Type = "xcode.syntax.declaration.function"; + }; + }, + +// ---------------------------------------------------------------------------- +// Blocks + + { + Identifier = "xcode.lang.go.block"; + Syntax = { + Tokenizer = "xcode.lang.go.lexer"; + Start = "{"; + End = "}"; + Foldable = YES; + Recursive = YES; + IncludeRules = ( + "xcode.lang.go.bracketexpr", + "xcode.lang.go.parenexpr", + ); + }; + }, + + { + Identifier = "xcode.lang.go.parenexpr"; + Syntax = { + Tokenizer = "xcode.lang.go.lexer"; + Start = "("; + End = ")"; + Recursive = YES; + IncludeRules = ( + "xcode.lang.go.bracketexpr", + "xcode.lang.go.block", + ); + }; + }, + + { + Identifier = "xcode.lang.go.bracketexpr"; + Syntax = { + Tokenizer = "xcode.lang.go.lexer"; + Start = "["; + End = "]"; + Recursive = YES; + IncludeRules = ( + "xcode.lang.go.parenexpr", + ); + }; + }, + + { + Identifier = "xcode.lang.go.comment"; + Syntax = { + Start = "/*"; + End = "*/"; + Foldable = YES; + IncludeRules = ( + "xcode.lang.url", + "xcode.lang.url.mail", + "xcode.lang.comment.mark", + ); + Type = "xcode.syntax.comment"; + }; + }, + + { + Identifier = "xcode.lang.go.comment.singleline"; + Syntax = { + Start = "//"; + End = "\n"; + IncludeRules = ( + "xcode.lang.url", + "xcode.lang.url.mail", + "xcode.lang.comment.mark", + ); + Type = "xcode.syntax.comment"; + }; + }, + + // This rule recognizes special comments markers and adds them + // to the list of file markers at the top of the editor window. + // This overrides the markers specified in + // /Developer/Library/PrivateFrameworks/XcodeEdit.framework/Versions/A/Resources/BaseSupport.xclangspec + // and appears to apply them to all languages. Thus, for now + // "inherit" the existing markers here for backward-compatibility. + { + Identifier = "xcode.lang.comment.mark"; + Syntax = { + StartChars = "BMTF!?"; + Match = ( + // Go-specific markers + "^\(BUG.*$\)$", // inlude "BUG" in the markers list + "^\(TODO.*$\)$", // inlude "TODO" in the markers list + // inherited markers + "^MARK:[ \t]+\(.*\)$", + "^\(TODO:[ \t]+.*\)$", // include "TODO: " in the markers list + "^\(FIXME:[ \t]+.*\)$", // include "FIXME: " in the markers list + "^\(!!!:.*\)$", // include "!!!:" in the markers list + "^\(\\?\\?\\?:.*\)$" // include "???:" in the markers list + ); + // This is the order of captures. All of the match strings above need the same order. + CaptureTypes = ( + "xcode.syntax.mark" + ); + Type = "xcode.syntax.comment"; + }; + }, + +) diff --git a/misc/zsh/go b/misc/zsh/go new file mode 100644 index 000000000..f17763d93 --- /dev/null +++ b/misc/zsh/go @@ -0,0 +1,14 @@ +# install in /etc/zsh/zshrc or your personal .zshrc + +# gc +prefixes=(5 6 8) +for p in $prefixes; do + compctl -g "*.${p}" ${p}l + compctl -g "*.go" ${p}g +done + +# standard go tools +compctl -g "*.go" gofmt + +# gccgo +compctl -g "*.go" gccgo diff --git a/robots.txt b/robots.txt new file mode 100644 index 000000000..1f53798bb --- /dev/null +++ b/robots.txt @@ -0,0 +1,2 @@ +User-agent: * +Disallow: / diff --git a/src/Make.ccmd b/src/Make.ccmd new file mode 100644 index 000000000..40cc3a0e8 --- /dev/null +++ b/src/Make.ccmd @@ -0,0 +1,48 @@ +# Copyright 2010 The Go Authors. All rights reserved. +# Use of this source code is governed by a BSD-style +# license that can be found in the LICENSE file. + +# Makefile for commands written in C. + +ifeq (windows,$(findstring windows, $(shell uname | tr A-Z a-z | sed 's/mingw/windows/'))) +TARG:=$(TARG).exe +endif + +$(TARG): $(OFILES) $(LIB) + $(HOST_LD) -o $(TARG) -L"$(GOROOT)"/lib $(OFILES) $(LIB) -lmach -lbio -l9 -lm $(HOST_LDFLAGS) + +$(OFILES): $(HFILES) + +CLEANFILES+=y.tab.[ch] + +clean: + rm -f *.$(HOST_O) $(TARG) $(CLEANFILES) + +nuke: clean + rm -f "$(GOBIN)/$(TARG)" + +ifneq ($(NOINSTALL),1) +install: $(QUOTED_GOBIN)/$(TARG) +endif + +$(QUOTED_GOBIN)/$(TARG): $(TARG) + cp $(TARG) "$(GOBIN)"/$(TARG) + +y.tab.h: $(YFILES) + bison -y $(HOST_YFLAGS) $(YFILES) + +y.tab.c: y.tab.h + test -f y.tab.c && touch y.tab.c + +all: $(TARG) + +# Use $(PWD)/$*.c so that gdb shows full path in stack traces. +%.$(HOST_O): %.c + $(HOST_CC) $(HOST_CFLAGS) -c "$(PWD)/$*.c" + +# These are used by enough different Makefiles to be +# worth writing down in one place, even if they don't +# apply to every command that builds with Make.ccmd +../%l/enam.o: + cd ../$*l; $(MAKE) enam.o + diff --git a/src/Make.clib b/src/Make.clib new file mode 100644 index 000000000..4a7ea02d9 --- /dev/null +++ b/src/Make.clib @@ -0,0 +1,37 @@ +# Copyright 2010 The Go Authors. All rights reserved. +# Use of this source code is governed by a BSD-style +# license that can be found in the LICENSE file. + +# Makefile included for C libraries + +all: $(LIB) + +# Use $(PWD)/$*.c so that gdb shows full path in stack traces. +%.$(HOST_O): %.c + $(HOST_CC) $(HOST_CFLAGS) -c "$(PWD)/$*.c" + +$(OFILES): $(HFILES) + +ifneq ($(NOINSTALL),1) +install: $(QUOTED_GOROOT)/lib/$(LIB) +endif + +$(QUOTED_GOROOT)/lib/$(LIB): $(LIB) + cp $(LIB) "$(GOROOT)/lib/$(LIB)" + +$(LIB): $(OFILES) + $(HOST_AR) rsc $(LIB) $(OFILES) + +CLEANFILES+=y.tab.[ch] y.output a.out $(LIB) + +clean: + rm -f *.$(HOST_O) $(CLEANFILES) + +nuke: clean + rm -f "$(GOROOT)/lib/$(LIB)" + +y.tab.h: $(YFILES) + LANG=C LANGUAGE="en_US.UTF8" bison -v -y $(HOST_YFLAGS) $(YFILES) + +y.tab.c: y.tab.h + test -f y.tab.c && touch y.tab.c diff --git a/src/Make.cmd b/src/Make.cmd new file mode 100644 index 000000000..27c6a2e13 --- /dev/null +++ b/src/Make.cmd @@ -0,0 +1,50 @@ +# Copyright 2009 The Go Authors. All rights reserved. +# Use of this source code is governed by a BSD-style +# license that can be found in the LICENSE file. + +ifeq ($(GOOS),windows) +TARG:=$(TARG).exe +endif + +ifeq ($(TARGDIR),) +TARGDIR:=$(QUOTED_GOBIN) +endif + +all: $(TARG) + +include $(QUOTED_GOROOT)/src/Make.common + +PREREQ+=$(patsubst %,%.make,$(DEPS)) + +$(TARG): _go_.$O + $(LD) $(LDIMPORTS) -o $@ _go_.$O + +_go_.$O: $(GOFILES) $(PREREQ) + $(GC) $(GCIMPORTS) -o $@ $(GOFILES) + +install: $(TARGDIR)/$(TARG) + +$(TARGDIR)/$(TARG): $(TARG) + mkdir -p $(TARGDIR) && cp -f $(TARG) $(TARGDIR) + +CLEANFILES+=$(TARG) _test _testmain.go test.out build.out + +nuke: clean + rm -f $(TARGDIR)/$(TARG) + +# for gotest +testpackage: _test/main.a + +testpackage-clean: + rm -f _test/main.a _gotest_.$O + +_test/main.a: _gotest_.$O + @mkdir -p _test + rm -f $@ + gopack grc $@ _gotest_.$O + +_gotest_.$O: $(GOFILES) $(GOTESTFILES) + $(GC) $(GCIMPORTS) -o $@ $(GOFILES) $(GOTESTFILES) + +importpath: + echo main diff --git a/src/Make.common b/src/Make.common new file mode 100644 index 000000000..0b27d07f9 --- /dev/null +++ b/src/Make.common @@ -0,0 +1,22 @@ +# Copyright 2009 The Go Authors. All rights reserved. +# Use of this source code is governed by a BSD-style +# license that can be found in the LICENSE file. + +clean: + rm -rf *.o *.a *.[$(OS)] [$(OS)].out $(CLEANFILES) + +install.clean: install + rm -rf *.o *.a *.[$(OS)] [$(OS)].out $(CLEANFILES) || true + +test.clean: test + rm -rf *.o *.a *.[$(OS)] [$(OS)].out $(CLEANFILES) || true + +testshort.clean: testshort + rm -rf *.o *.a *.[$(OS)] [$(OS)].out $(CLEANFILES) || true + +%.make: + $(MAKE) -C $* install + +.PHONY: all clean nuke install coverage test bench testpackage-clean\ + importpath dir + diff --git a/src/Make.inc b/src/Make.inc new file mode 100644 index 000000000..8f549f624 --- /dev/null +++ b/src/Make.inc @@ -0,0 +1,169 @@ +# Copyright 2009 The Go Authors. All rights reserved. +# Use of this source code is governed by a BSD-style +# license that can be found in the LICENSE file. + +# Makefile included by all other Go makefiles. + +# Clear variables that must come from Makefiles, +# not the environment. +LIB:= +TARG:= +GOFILES:= +HFILES:= +OFILES:= +YFILES:= + +# GOROOT must be set. +ifeq ($(GOROOT),) +$(error $$GOROOT is not set; use gomake or set $$GOROOT in your environment) +endif + +# Set up GOROOT_FINAL, GOARCH, GOOS if needed. +GOROOT_FINAL?=$(GOROOT) + +ifeq ($(GOHOSTOS),) +GOHOSTOS:=$(shell uname | tr A-Z a-z | sed 's/mingw/windows/; s/.*windows.*/windows/') +endif + +ifeq ($(GOOS),) +GOOS:=$(GOHOSTOS) +endif + +GOOS_LIST=\ + darwin\ + freebsd\ + linux\ + plan9\ + windows\ + +GOARCH_LIST=\ + 386\ + amd64\ + arm\ + +ifeq ($(filter $(GOOS),$(GOOS_LIST)),) +$(error Invalid $$GOOS '$(GOOS)'; must be one of: $(GOOS_LIST)) +endif + +ifeq ($(GOHOSTARCH),) +ifeq ($(GOHOSTOS),darwin) +# Even on 64-bit platform, darwin uname -m prints i386. +# Check for amd64 with sysctl instead. +GOHOSTARCH:=${shell if sysctl machdep.cpu.extfeatures | grep EM64T >/dev/null; then echo amd64; else uname -m | sed 's/i386/386/'; fi} +else +# Ask uname -m for the processor. +GOHOSTARCH:=${shell uname -m | sed 's/^..86$$/386/; s/^.86$$/386/; s/x86_64/amd64/; s/arm.*/arm/'} +endif +endif + +ifeq ($(GOARCH),) +GOARCH:=$(GOHOSTARCH) +endif + +# darwin requires GOHOSTARCH match GOARCH +ifeq ($(GOOS),darwin) +GOHOSTARCH:=$(GOARCH) +endif + +ifeq ($(filter $(GOARCH),$(GOARCH_LIST)),) +$(error Invalid $$GOARCH '$(GOARCH)'; must be one of: $(GOARCH_LIST)) +endif + +ifeq ($(GOARCH),386) +O:=8 +else ifeq ($(GOARCH),amd64) +O:=6 +else ifeq ($(GOARCH),arm) +O:=5 +ifneq ($(GOOS),linux) +$(error Invalid $$GOOS '$(GOOS)' for GOARCH=arm; must be linux) +endif +else +$(error Missing $$O for '$(GOARCH)') +endif + +# Save for recursive make to avoid recomputing. +export GOARCH GOOS GOHOSTARCH GOHOSTOS GOARCH_LIST GOOS_LIST + +# ugly hack to deal with whitespaces in $GOROOT +nullstring := +space := $(nullstring) # a space at the end +QUOTED_GOROOT:=$(subst $(space),\ ,$(GOROOT)) + +# default GOBIN +ifndef GOBIN +GOBIN=$(QUOTED_GOROOT)/bin +endif +QUOTED_GOBIN=$(subst $(space),\ ,$(GOBIN)) + +AS=${O}a +CC=${O}c +GC=${O}g +LD=${O}l +OS=568vq +CFLAGS=-FVw + +HOST_CC=quietgcc +HOST_LD=quietgcc +HOST_O=o +HOST_YFLAGS=-d +HOST_AR?=ar + +# These two variables can be overridden in the environment +# to build with other flags. They are like $CFLAGS and $LDFLAGS +# in a more typical GNU build. We are more explicit about the names +# here because there are different compilers being run during the +# build (both gcc and 6c, for example). +HOST_EXTRA_CFLAGS?=-ggdb -O2 +HOST_EXTRA_LDFLAGS?= + +HOST_CFLAGS=-I"$(GOROOT)/include" $(HOST_EXTRA_CFLAGS) +HOST_LDFLAGS=$(HOST_EXTRA_LDFLAGS) +PWD=$(shell pwd) + +# Decide whether use of cgo is okay. +ifeq ($(CGO_ENABLED),) +# Default on... +CGO_ENABLED:=1 +ifeq ($(GOARCH),arm) # ... but not on ARM +CGO_ENABLED:=0 +endif +ifeq ($(GOOS),plan9) # ... and not on Plan 9 +CGO_ENABLED:=0 +endif +ifeq ($(GOOS),openbsd) # ... and not on OpenBSD +CGO_ENABLED:=0 +endif +endif + +# Make environment more standard. +LANG:= +LC_ALL:=C +LC_CTYPE:=C +GREP_OPTIONS:= +GREP_COLORS:= +export LANG LC_ALL LC_CTYPE GREP_OPTIONS GREP_COLORS + +go-env: + @echo export GOARCH="$(GOARCH)" + @echo export GOOS="$(GOOS)" + @echo export GOHOSTARCH="$(GOHOSTARCH)" + @echo export GOHOSTOS="$(GOHOSTOS)" + @echo export CGO_ENABLED="$(CGO_ENABLED)" + @echo export O="$O" + @echo export AS="$(AS)" + @echo export CC="$(CC)" + @echo export GC="$(GC)" + @echo export LD="$(LD)" + @echo export OS="$(OS)" + @echo export CFLAGS="$(CFLAGS)" + @echo export LANG="$(LANG)" + @echo export LC_ALL="$(LC_ALL)" + @echo export LC_CTYPE="$(LC_CTYPE)" + @echo export GREP_OPTIONS="$(GREP_OPTIONS)" + @echo export GREP_COLORS="$(GREP_COLORS)" + @echo MAKE_GO_ENV_WORKED=1 + +# Don't let the targets in this file be used +# as the default make target. +.DEFAULT_GOAL:= diff --git a/src/Make.pkg b/src/Make.pkg new file mode 100644 index 000000000..fc80cf6e6 --- /dev/null +++ b/src/Make.pkg @@ -0,0 +1,249 @@ +# Copyright 2009 The Go Authors. All rights reserved. +# Use of this source code is governed by a BSD-style +# license that can be found in the LICENSE file. + +all: package +package: _obj/$(TARG).a +testpackage: _test/$(TARG).a + +include $(QUOTED_GOROOT)/src/Make.common + +# The quietgcc wrapper is for our own source code +# while building the libraries, not arbitrary source code +# as encountered by cgo. +ifeq ($(HOST_CC),quietgcc) +HOST_CC:=gcc +endif +ifeq ($(HOST_LD),quietgcc) +HOST_LD:=gcc +endif + +# GNU Make 3.80 has a bug in lastword +# elem=$(lastword $(subst /, ,$(TARG))) +TARG_words=$(subst /, ,$(TARG)) +elem=$(word $(words $(TARG_words)),$(TARG_words)) + +ifeq ($(elem),$(TARG)) +dir= +else +dir=$(patsubst %/$(elem),%,$(TARG)) +endif + +pkgdir=$(QUOTED_GOROOT)/pkg/$(GOOS)_$(GOARCH) + +ifeq ($(TARGDIR),) +TARGDIR:=$(pkgdir) +endif + +INSTALLFILES+=$(TARGDIR)/$(TARG).a + +# The rest of the cgo rules are below, but these variable updates +# must be done here so they apply to the main rules. +ifdef CGOFILES +GOFILES+=$(patsubst %.go,_obj/%.cgo1.go,$(CGOFILES)) _obj/_cgo_gotypes.go +CGO_OFILES+=$(patsubst %.go,%.cgo2.o,$(CGOFILES)) _cgo_export.o +OFILES+=_cgo_defun.$O _cgo_import.$O $(CGO_OFILES) +endif + +ifdef SWIGFILES +GOFILES+=$(patsubst %.swig,_obj/%.go,$(patsubst %.swigcxx,%.swig,$(SWIGFILES))) +OFILES+=$(patsubst %.swig,_obj/%_gc.$O,$(patsubst %.swigcxx,%.swig,$(SWIGFILES))) +SWIG_PREFIX=$(subst /,-,$(TARG)) +SWIG_SOS+=$(patsubst %.swig,_obj/$(SWIG_PREFIX)-%.so,$(patsubst %.swigcxx,%.swig,$(SWIGFILES))) +INSTALLFILES+=$(patsubst %.swig,$(TARGDIR)/swig/$(SWIG_PREFIX)-%.so,$(patsubst %.swigcxx,%.swig,$(SWIGFILES))) +endif + +PREREQ+=$(patsubst %,%.make,$(DEPS)) + +coverage: + gotest + 6cov -g $(shell pwd) $O.out | grep -v '_test\.go:' + +CLEANFILES+=*.so _obj _test _testmain.go *.exe _cgo* test.out build.out + +test: + gotest + +testshort: + gotest -test.short -test.timeout=120 + +bench: + gotest -test.bench=. -test.run="Do not run tests" + +nuke: clean + rm -f $(TARGDIR)/$(TARG).a + +testpackage-clean: + rm -f _test/$(TARG).a + +install: $(INSTALLFILES) + +$(TARGDIR)/$(TARG).a: _obj/$(TARG).a + @mkdir -p $(TARGDIR)/$(dir) + cp _obj/$(TARG).a "$@" + +_go_.$O: $(GOFILES) $(PREREQ) + $(GC) $(GCIMPORTS) -o $@ $(GOFILES) + +_gotest_.$O: $(GOFILES) $(GOTESTFILES) $(PREREQ) + $(GC) $(GCIMPORTS) -o $@ $(GOFILES) $(GOTESTFILES) + +_obj/$(TARG).a: _go_.$O $(OFILES) + @mkdir -p _obj/$(dir) + rm -f _obj/$(TARG).a + gopack grc $@ _go_.$O $(OFILES) + +_test/$(TARG).a: _gotest_.$O $(OFILES) + @mkdir -p _test/$(dir) + rm -f _test/$(TARG).a + gopack grc $@ _gotest_.$O $(OFILES) + +importpath: + @echo $(TARG) + +dir: + @echo $(dir) + +# To use cgo in a Go package, add a line +# +# CGOFILES=x.go y.go +# +# to the main Makefile. This signals that cgo should process x.go +# and y.go when building the package. +# There are three optional variables to set, CGO_CFLAGS, CGO_LDFLAGS, +# and CGO_DEPS, which specify compiler flags, linker flags, and linker +# dependencies to use when compiling (using gcc) the C support for +# x.go and y.go. + +# Cgo translates each x.go file listed in $(CGOFILES) into a basic +# translation of x.go, called _obj/x.cgo1.go. Additionally, three other +# files are created: +# +# _obj/_cgo_gotypes.go - declarations needed for all .go files in the package; imports "unsafe" +# _obj/_cgo_defun.c - C trampoline code to be compiled with 6c and linked into the package +# _obj/x.cgo2.c - C implementations compiled with gcc to create a dynamic library +# + +ifdef CGOFILES +_obj/_cgo_run: $(CGOFILES) + @mkdir -p _obj + CGOPKGPATH=$(dir) cgo -- $(CGO_CFLAGS) $(CGOFILES) + touch _obj/_cgo_run + +# _CGO_CFLAGS and _CGO_LDFLAGS are defined via the evaluation of _cgo_flags. +# The include happens before the commands in the recipe run, +# so it cannot be done in the same recipe that runs cgo. +_obj/_load_cgo_flags: _obj/_cgo_run + $(eval include _obj/_cgo_flags) + +# Include any previous flags in case cgo files are up to date. +-include _obj/_cgo_flags + +# Ugly but necessary - cgo writes these files too. +_obj/_cgo_gotypes.go _obj/_cgo_export.c _obj/_cgo_export.h _obj/_cgo_main.c _obj/_cgo_defun.c: _obj/_load_cgo_flags + @true + +_obj/%.cgo1.go _obj/%.cgo2.c: _obj/_cgo_defun.c + @true +endif + +# Compile rules for gcc source files. +%.o: %.c + $(HOST_CC) $(_CGO_CFLAGS_$(GOARCH)) -g -fPIC -O2 -o $@ -c $(CGO_CFLAGS) $(_CGO_CFLAGS) $*.c + +%.o: _obj/%.c + $(HOST_CC) $(_CGO_CFLAGS_$(GOARCH)) -I . -g -fPIC -O2 -o $@ -c $(CGO_CFLAGS) $(_CGO_CFLAGS) $^ + +# To find out which symbols are needed from external libraries +# and which libraries are needed, we build a simple a.out that +# links all the objects we just created and then use cgo -dynimport +# to inspect it. That is, we make gcc tell us which dynamic symbols +# and libraries are involved, instead of duplicating gcc's logic ourselves. +# After main we have to define all the symbols that will be provided +# by Go code. That's crosscall2 and any exported symbols. + +_cgo1_.o: _cgo_main.o $(CGO_OFILES) + $(HOST_CC) $(_CGO_CFLAGS_$(GOARCH)) -g -fPIC -O2 -o $@ $^ $(CGO_LDFLAGS) $(_CGO_LDFLAGS) + +_obj/_cgo_import.c: _cgo1_.o + @mkdir -p _obj + cgo -dynimport _cgo1_.o >$@_ && mv -f $@_ $@ + +# The rules above added x.cgo1.go and _cgo_gotypes.go to $(GOFILES), +# added _cgo_defun.$O to $OFILES, and added the installed copy of +# package_x.so (built from x.cgo2.c) to $(INSTALLFILES). + +# Have to run gcc with the right size argument on hybrid 32/64 machines. +_CGO_CFLAGS_386=-m32 +_CGO_CFLAGS_amd64=-m64 +_CGO_LDFLAGS_freebsd=-shared -lpthread -lm +_CGO_LDFLAGS_linux=-shared -lpthread -lm +_CGO_LDFLAGS_darwin=-dynamiclib -Wl,-undefined,dynamic_lookup +_CGO_LDFLAGS_windows=-shared -lm -mthreads + +# Have to compile the runtime header. +RUNTIME_CFLAGS=-I$(pkgdir) + +# Compile _cgo_defun.c with 6c; needs access to the runtime headers. +_cgo_defun.$O: _obj/_cgo_defun.c + $(CC) $(CFLAGS) $(RUNTIME_CFLAGS) -I . -o "$@" _obj/_cgo_defun.c + +# To use swig in a Go package, add a line +# +# SWIGFILES=x.swig +# +# to the main Makefile. This signals that SWIG should process the +#.swig file when building the package. +# +# To wrap C code, use an extension of .swig. To wrap C++ code, use an +# extension of .swigcxx. +# +# SWIGFILES=myclib.swig mycxxlib.swigcxx + +ifdef SWIGFILES +_obj/%._swig_run _obj/%.go _obj/%_gc.c _obj/%_wrap.c: %.swig + @mkdir -p _obj + swig -go -module $* -soname $(SWIG_PREFIX)-$*.so -o _obj/$*_wrap.c -outdir _obj $< + +_obj/%._swig_run _obj/%.go _obj/%_gc.c _obj/%_wrap.cxx: %.swigcxx + @mkdir -p _obj + swig -go -c++ -module $* -soname $(SWIG_PREFIX)-$*.so -o _obj/$*_wrap.cxx -outdir _obj $< + +_obj/%_gc.$O: _obj/%_gc.c + $(CC) $(CFLAGS) -I . -I$(pkgdir) -o "$@" _obj/$*_gc.c + +_obj/%_wrap.o: _obj/%_wrap.c + $(HOST_CC) $(_CGO_CFLAGS_$(GOARCH)) -I . -g -fPIC -O2 -o $@ -c $^ $(SWIG_CFLAGS) + +HOST_CXX=g++ + +_obj/%_wrapcxx.o: _obj/%_wrap.cxx + $(HOST_CXX) $(_CGO_CFLAGS_$(GOARCH)) -I . -g -fPIC -O2 -o $@ -c $^ $(SWIG_CXXFLAGS) + +_obj/$(SWIG_PREFIX)-%.so: _obj/%_wrap.o + $(HOST_CC) $(_CGO_CFLAGS_$(GOARCH)) -o $@ $^ $(SWIG_LDFLAGS) $(_CGO_LDFLAGS_$(GOOS)) $(_SWIG_LDFLAGS_$(GOOS)) + +_obj/$(SWIG_PREFIX)-%.so: _obj/%_wrapcxx.o + $(HOST_CXX) $(_CGO_CFLAGS_$(GOARCH)) -o $@ $^ $(SWIG_LDFLAGS) $(_CGO_LDFLAGS_$(GOOS)) $(_SWIG_LDFLAGS_$(GOOS)) + +$(TARGDIR)/swig/$(SWIG_PREFIX)-%.so: _obj/$(SWIG_PREFIX)-%.so + @mkdir -p $(TARGDIR)/swig + cp $< "$@" + +all: $(SWIG_SOS) + +SWIG_RPATH=-r $(TARGDIR)/swig + +endif + +# Generic build rules. +# These come last so that the rules above can override them +# for more specific file names. +%.$O: %.c $(HFILES) + $(CC) $(CFLAGS) -o "$@" $*.c + +%.$O: _obj/%.c $(HFILES) + $(CC) $(CFLAGS) -I . -o "$@" _obj/$*.c + +%.$O: %.s + $(AS) $*.s diff --git a/src/all-qemu.bash b/src/all-qemu.bash new file mode 100755 index 000000000..6a659d6d4 --- /dev/null +++ b/src/all-qemu.bash @@ -0,0 +1,16 @@ +#!/usr/bin/env bash +# Copyright 2011 The Go Authors. All rights reserved. +# Use of this source code is governed by a BSD-style +# license that can be found in the LICENSE file. + +# Run all.bash but exclude tests that depend on functionality +# missing in QEMU's system call emulation. + +export GOARCH=arm +export NOTEST="" +NOTEST="$NOTEST big" # just slow +NOTEST="$NOTEST go/build" # wants to run cgo +NOTEST="$NOTEST http http/cgi net rpc syslog websocket" # no localhost network +NOTEST="$NOTEST os" # 64-bit seek fails + +./all.bash diff --git a/src/all.bash b/src/all.bash new file mode 100755 index 000000000..00110d2da --- /dev/null +++ b/src/all.bash @@ -0,0 +1,14 @@ +#!/usr/bin/env bash +# Copyright 2009 The Go Authors. All rights reserved. +# Use of this source code is governed by a BSD-style +# license that can be found in the LICENSE file. + +set -e +if [ ! -f make.bash ]; then + echo 'all.bash must be run from $GOROOT/src' 1>&2 + exit 1 +fi +. ./make.bash +bash run.bash --no-env --no-rebuild +installed # function defined by make.bash + diff --git a/src/clean.bash b/src/clean.bash new file mode 100755 index 000000000..1955b583b --- /dev/null +++ b/src/clean.bash @@ -0,0 +1,29 @@ +#!/usr/bin/env bash +# Copyright 2009 The Go Authors. All rights reserved. +# Use of this source code is governed by a BSD-style +# license that can be found in the LICENSE file. + +set -e + +if [ ! -f env.bash ]; then + echo 'clean.bash must be run from $GOROOT/src' 1>&2 + exit 1 +fi +. ./env.bash +if [ ! -f Make.inc ] ; then + GOROOT_FINAL=${GOROOT_FINAL:-$GOROOT} + sed 's!@@GOROOT@@!'"$GOROOT_FINAL"'!' Make.inc.in >Make.inc +fi + +if [ "$1" != "--nopkg" ]; then + rm -rf "$GOROOT"/pkg/${GOOS}_$GOARCH +fi +rm -f "$GOROOT"/lib/*.a +for i in lib9 libbio libmach cmd pkg \ + ../misc/cgo/gmp ../misc/cgo/stdio \ + ../misc/cgo/life ../misc/cgo/test \ + ../test/bench ../test/garbage +do + # Do not use gomake here. It may not be available. + $MAKE -C "$GOROOT/src/$i" clean +done diff --git a/src/cmd/5a/Makefile b/src/cmd/5a/Makefile new file mode 100644 index 000000000..f4463c97b --- /dev/null +++ b/src/cmd/5a/Makefile @@ -0,0 +1,25 @@ +# Copyright 2009 The Go Authors. All rights reserved. +# Use of this source code is governed by a BSD-style +# license that can be found in the LICENSE file. + +include ../../Make.inc +O:=$(HOST_O) + +TARG=5a + +HFILES=\ + a.h\ + y.tab.h\ + ../5l/5.out.h\ + +OFILES=\ + y.tab.$O\ + lex.$O\ + ../5l/enam.$O\ + +YFILES=\ + a.y\ + +include ../../Make.ccmd + +lex.$O: ../cc/macbody ../cc/lexbody diff --git a/src/cmd/5a/a.h b/src/cmd/5a/a.h new file mode 100644 index 000000000..a2c87cf48 --- /dev/null +++ b/src/cmd/5a/a.h @@ -0,0 +1,200 @@ +// Inferno utils/5a/a.h +// http://code.google.com/p/inferno-os/source/browse/utils/5a/a.h +// +// Copyright © 1994-1999 Lucent Technologies Inc. All rights reserved. +// Portions Copyright © 1995-1997 C H Forsyth (forsyth@terzarima.net) +// Portions Copyright © 1997-1999 Vita Nuova Limited +// Portions Copyright © 2000-2007 Vita Nuova Holdings Limited (www.vitanuova.com) +// Portions Copyright © 2004,2006 Bruce Ellis +// Portions Copyright © 2005-2007 C H Forsyth (forsyth@terzarima.net) +// Revisions Copyright © 2000-2007 Lucent Technologies Inc. and others +// Portions Copyright © 2009 The Go Authors. All rights reserved. +// +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to deal +// in the Software without restriction, including without limitation the rights +// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +// copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +// THE SOFTWARE. + +#include +#include "../5l/5.out.h" + +#ifndef EXTERN +#define EXTERN extern +#endif + +#undef getc +#undef ungetc +#undef BUFSIZ + +#define getc ccgetc +#define ungetc ccungetc + +typedef struct Sym Sym; +typedef struct Gen Gen; +typedef struct Io Io; +typedef struct Hist Hist; + +#define MAXALIGN 7 +#define FPCHIP 1 +#define NSYMB 8192 +#define BUFSIZ 8192 +#define HISTSZ 20 +#ifndef EOF +#define EOF (-1) +#endif +#define IGN (-2) +#define GETC() ((--fi.c < 0)? filbuf(): *fi.p++ & 0xff) +#define NHASH 503 +#define STRINGSZ 200 +#define NMACRO 10 + +struct Sym +{ + Sym* link; + char* macro; + int32 value; + ushort type; + char *name; + char sym; +}; +#define S ((Sym*)0) + +EXTERN struct +{ + char* p; + int c; +} fi; + +struct Io +{ + Io* link; + char b[BUFSIZ]; + char* p; + short c; + short f; +}; +#define I ((Io*)0) + +EXTERN struct +{ + Sym* sym; + short type; +} h[NSYM]; + +struct Gen +{ + Sym* sym; + int32 offset; + short type; + short reg; + short name; + double dval; + char sval[8]; +}; + +struct Hist +{ + Hist* link; + char* name; + int32 line; + int32 offset; +}; +#define H ((Hist*)0) + +enum +{ + CLAST, + CMACARG, + CMACRO, + CPREPROC, + + Always = 14, +}; + +EXTERN char debug[256]; +EXTERN Sym* hash[NHASH]; +EXTERN char** Dlist; +EXTERN int nDlist; +EXTERN Hist* ehist; +EXTERN int newflag; +EXTERN Hist* hist; +EXTERN char* hunk; +EXTERN char** include; +EXTERN Io* iofree; +EXTERN Io* ionext; +EXTERN Io* iostack; +EXTERN int32 lineno; +EXTERN int nerrors; +EXTERN int32 nhunk; +EXTERN int ninclude; +EXTERN int32 nsymb; +EXTERN Gen nullgen; +EXTERN char* outfile; +EXTERN int pass; +EXTERN char* pathname; +EXTERN int32 pc; +EXTERN int peekc; +EXTERN int32 stmtline; +EXTERN int sym; +EXTERN char* symb; +EXTERN int thechar; +EXTERN char* thestring; +EXTERN int32 thunk; +EXTERN Biobuf obuf; + +void* alloc(int32); +void* allocn(void*, int32, int32); +void ensuresymb(int32); +void errorexit(void); +void pushio(void); +void newio(void); +void newfile(char*, int); +Sym* slookup(char*); +Sym* lookup(void); +void syminit(Sym*); +int32 yylex(void); +int getc(void); +int getnsc(void); +void unget(int); +int escchar(int); +void cinit(void); +void pinit(char*); +void cclean(void); +int isreg(Gen*); +void outcode(int, int, Gen*, int, Gen*); +void zname(char*, int, int); +void zaddr(Gen*, int); +void ieeedtod(Ieee*, double); +int filbuf(void); +Sym* getsym(void); +void domacro(void); +void macund(void); +void macdef(void); +void macexpand(Sym*, char*); +void macinc(void); +void maclin(void); +void macprag(void); +void macif(int); +void macend(void); +void outhist(void); +void dodefine(char*); +void prfile(int32); +void linehist(char*, int); +void gethunk(void); +void yyerror(char*, ...); +int yyparse(void); +void setinclude(char*); +int assemble(char*); diff --git a/src/cmd/5a/a.y b/src/cmd/5a/a.y new file mode 100644 index 000000000..9a0efd5e0 --- /dev/null +++ b/src/cmd/5a/a.y @@ -0,0 +1,710 @@ +// Inferno utils/5a/a.y +// http://code.google.com/p/inferno-os/source/browse/utils/5a/a.y +// +// Copyright © 1994-1999 Lucent Technologies Inc. All rights reserved. +// Portions Copyright © 1995-1997 C H Forsyth (forsyth@terzarima.net) +// Portions Copyright © 1997-1999 Vita Nuova Limited +// Portions Copyright © 2000-2007 Vita Nuova Holdings Limited (www.vitanuova.com) +// Portions Copyright © 2004,2006 Bruce Ellis +// Portions Copyright © 2005-2007 C H Forsyth (forsyth@terzarima.net) +// Revisions Copyright © 2000-2007 Lucent Technologies Inc. and others +// Portions Copyright © 2009 The Go Authors. All rights reserved. +// +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to deal +// in the Software without restriction, including without limitation the rights +// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +// copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +// THE SOFTWARE. + +%{ +#include +#include /* if we don't, bison will, and a.h re-#defines getc */ +#include +#include "a.h" +%} +%union +{ + Sym *sym; + int32 lval; + double dval; + char sval[8]; + Gen gen; +} +%left '|' +%left '^' +%left '&' +%left '<' '>' +%left '+' '-' +%left '*' '/' '%' +%token LTYPE1 LTYPE2 LTYPE3 LTYPE4 LTYPE5 +%token LTYPE6 LTYPE7 LTYPE8 LTYPE9 LTYPEA +%token LTYPEB LTYPEC LTYPED LTYPEE LTYPEF +%token LTYPEG LTYPEH LTYPEI LTYPEJ LTYPEK +%token LTYPEL LTYPEM LTYPEN LTYPEBX +%token LCONST LSP LSB LFP LPC +%token LTYPEX LR LREG LF LFREG LC LCREG LPSR LFCR +%token LCOND LS LAT +%token LFCONST +%token LSCONST +%token LNAME LLAB LVAR +%type con expr oexpr pointer offset sreg spreg creg +%type rcon cond reglist +%type gen rel reg regreg freg shift fcon frcon +%type imm ximm name oreg ireg nireg ioreg imsr +%% +prog: +| prog + { + stmtline = lineno; + } + line + +line: + LLAB ':' + { + if($1->value != pc) + yyerror("redeclaration of %s", $1->name); + $1->value = pc; + } + line +| LNAME ':' + { + $1->type = LLAB; + $1->value = pc; + } + line +| LNAME '=' expr ';' + { + $1->type = LVAR; + $1->value = $3; + } +| LVAR '=' expr ';' + { + if($1->value != $3) + yyerror("redeclaration of %s", $1->name); + $1->value = $3; + } +| ';' +| inst ';' +| error ';' + +inst: +/* + * ADD + */ + LTYPE1 cond imsr ',' spreg ',' reg + { + outcode($1, $2, &$3, $5, &$7); + } +| LTYPE1 cond imsr ',' spreg ',' + { + outcode($1, $2, &$3, $5, &nullgen); + } +| LTYPE1 cond imsr ',' reg + { + outcode($1, $2, &$3, NREG, &$5); + } +/* + * MVN + */ +| LTYPE2 cond imsr ',' reg + { + outcode($1, $2, &$3, NREG, &$5); + } +/* + * MOVW + */ +| LTYPE3 cond gen ',' gen + { + outcode($1, $2, &$3, NREG, &$5); + } +/* + * B/BL + */ +| LTYPE4 cond comma rel + { + outcode($1, $2, &nullgen, NREG, &$4); + } +| LTYPE4 cond comma nireg + { + outcode($1, $2, &nullgen, NREG, &$4); + } +/* + * BX + */ +| LTYPEBX comma ireg + { + outcode($1, Always, &nullgen, NREG, &$3); + } +/* + * BEQ + */ +| LTYPE5 comma rel + { + outcode($1, Always, &nullgen, NREG, &$3); + } +/* + * SWI + */ +| LTYPE6 cond comma gen + { + outcode($1, $2, &nullgen, NREG, &$4); + } +/* + * CMP + */ +| LTYPE7 cond imsr ',' spreg comma + { + outcode($1, $2, &$3, $5, &nullgen); + } +/* + * MOVM + */ +| LTYPE8 cond ioreg ',' '[' reglist ']' + { + Gen g; + + g = nullgen; + g.type = D_CONST; + g.offset = $6; + outcode($1, $2, &$3, NREG, &g); + } +| LTYPE8 cond '[' reglist ']' ',' ioreg + { + Gen g; + + g = nullgen; + g.type = D_CONST; + g.offset = $4; + outcode($1, $2, &g, NREG, &$7); + } +/* + * SWAP + */ +| LTYPE9 cond reg ',' ireg ',' reg + { + outcode($1, $2, &$5, $3.reg, &$7); + } +| LTYPE9 cond reg ',' ireg comma + { + outcode($1, $2, &$5, $3.reg, &$3); + } +| LTYPE9 cond comma ireg ',' reg + { + outcode($1, $2, &$4, $6.reg, &$6); + } +/* + * RET + */ +| LTYPEA cond comma + { + outcode($1, $2, &nullgen, NREG, &nullgen); + } +/* + * TEXT/GLOBL + */ +| LTYPEB name ',' imm + { + outcode($1, Always, &$2, NREG, &$4); + } +| LTYPEB name ',' con ',' imm + { + outcode($1, Always, &$2, $4, &$6); + } +/* + * DATA + */ +| LTYPEC name '/' con ',' ximm + { + outcode($1, Always, &$2, $4, &$6); + } +/* + * CASE + */ +| LTYPED cond reg comma + { + outcode($1, $2, &$3, NREG, &nullgen); + } +/* + * word + */ +| LTYPEH comma ximm + { + outcode($1, Always, &nullgen, NREG, &$3); + } +/* + * floating-point coprocessor + */ +| LTYPEI cond freg ',' freg + { + outcode($1, $2, &$3, NREG, &$5); + } +| LTYPEK cond frcon ',' freg + { + outcode($1, $2, &$3, NREG, &$5); + } +| LTYPEK cond frcon ',' LFREG ',' freg + { + outcode($1, $2, &$3, $5, &$7); + } +| LTYPEL cond freg ',' freg comma + { + outcode($1, $2, &$3, $5.reg, &nullgen); + } +/* + * MCR MRC + */ +| LTYPEJ cond con ',' expr ',' spreg ',' creg ',' creg oexpr + { + Gen g; + + g = nullgen; + g.type = D_CONST; + g.offset = + (0xe << 24) | /* opcode */ + ($1 << 20) | /* MCR/MRC */ + ($2 << 28) | /* scond */ + (($3 & 15) << 8) | /* coprocessor number */ + (($5 & 7) << 21) | /* coprocessor operation */ + (($7 & 15) << 12) | /* arm register */ + (($9 & 15) << 16) | /* Crn */ + (($11 & 15) << 0) | /* Crm */ + (($12 & 7) << 5) | /* coprocessor information */ + (1<<4); /* must be set */ + outcode(AWORD, Always, &nullgen, NREG, &g); + } +/* + * MULL hi,lo,r1,r2 + */ +| LTYPEM cond reg ',' reg ',' regreg + { + outcode($1, $2, &$3, $5.reg, &$7); + } +/* + * MULA hi,lo,r1,r2 + */ +| LTYPEN cond reg ',' reg ',' reg ',' spreg + { + $7.type = D_REGREG; + $7.offset = $9; + outcode($1, $2, &$3, $5.reg, &$7); + } +/* + * END + */ +| LTYPEE comma + { + outcode($1, Always, &nullgen, NREG, &nullgen); + } + +cond: + { + $$ = Always; + } +| cond LCOND + { + $$ = ($1 & ~C_SCOND) | $2; + } +| cond LS + { + $$ = $1 | $2; + } + +comma: +| ',' comma + +rel: + con '(' LPC ')' + { + $$ = nullgen; + $$.type = D_BRANCH; + $$.offset = $1 + pc; + } +| LNAME offset + { + $$ = nullgen; + if(pass == 2) + yyerror("undefined label: %s", $1->name); + $$.type = D_BRANCH; + $$.sym = $1; + $$.offset = $2; + } +| LLAB offset + { + $$ = nullgen; + $$.type = D_BRANCH; + $$.sym = $1; + $$.offset = $1->value + $2; + } + +ximm: '$' con + { + $$ = nullgen; + $$.type = D_CONST; + $$.offset = $2; + } +| '$' oreg + { + $$ = $2; + $$.type = D_CONST; + } +| '$' '*' '$' oreg + { + $$ = $4; + $$.type = D_OCONST; + } +| '$' LSCONST + { + $$ = nullgen; + $$.type = D_SCONST; + memcpy($$.sval, $2, sizeof($$.sval)); + } +| fcon + +fcon: + '$' LFCONST + { + $$ = nullgen; + $$.type = D_FCONST; + $$.dval = $2; + } +| '$' '-' LFCONST + { + $$ = nullgen; + $$.type = D_FCONST; + $$.dval = -$3; + } + +reglist: + spreg + { + $$ = 1 << $1; + } +| spreg '-' spreg + { + int i; + $$=0; + for(i=$1; i<=$3; i++) + $$ |= 1<' '>' rcon + { + $$ = nullgen; + $$.type = D_SHIFT; + $$.offset = $1 | $4 | (1 << 5); + } +| spreg '-' '>' rcon + { + $$ = nullgen; + $$.type = D_SHIFT; + $$.offset = $1 | $4 | (2 << 5); + } +| spreg LAT '>' rcon + { + $$ = nullgen; + $$.type = D_SHIFT; + $$.offset = $1 | $4 | (3 << 5); + } + +rcon: + spreg + { + if($$ < 0 || $$ >= 16) + print("register value out of range\n"); + $$ = (($1&15) << 8) | (1 << 4); + } +| con + { + if($$ < 0 || $$ >= 32) + print("shift value out of range\n"); + $$ = ($1&31) << 7; + } + +sreg: + LREG +| LPC + { + $$ = REGPC; + } +| LR '(' expr ')' + { + if($3 < 0 || $3 >= NREG) + print("register value out of range\n"); + $$ = $3; + } + +spreg: + sreg +| LSP + { + $$ = REGSP; + } + +creg: + LCREG +| LC '(' expr ')' + { + if($3 < 0 || $3 >= NREG) + print("register value out of range\n"); + $$ = $3; + } + +frcon: + freg +| fcon + +freg: + LFREG + { + $$ = nullgen; + $$.type = D_FREG; + $$.reg = $1; + } +| LF '(' con ')' + { + $$ = nullgen; + $$.type = D_FREG; + $$.reg = $3; + } + +name: + con '(' pointer ')' + { + $$ = nullgen; + $$.type = D_OREG; + $$.name = $3; + $$.sym = S; + $$.offset = $1; + } +| LNAME offset '(' pointer ')' + { + $$ = nullgen; + $$.type = D_OREG; + $$.name = $4; + $$.sym = $1; + $$.offset = $2; + } +| LNAME '<' '>' offset '(' LSB ')' + { + $$ = nullgen; + $$.type = D_OREG; + $$.name = D_STATIC; + $$.sym = $1; + $$.offset = $4; + } + +offset: + { + $$ = 0; + } +| '+' con + { + $$ = $2; + } +| '-' con + { + $$ = -$2; + } + +pointer: + LSB +| LSP +| LFP + +con: + LCONST +| LVAR + { + $$ = $1->value; + } +| '-' con + { + $$ = -$2; + } +| '+' con + { + $$ = $2; + } +| '~' con + { + $$ = ~$2; + } +| '(' expr ')' + { + $$ = $2; + } + +oexpr: + { + $$ = 0; + } +| ',' expr + { + $$ = $2; + } + +expr: + con +| expr '+' expr + { + $$ = $1 + $3; + } +| expr '-' expr + { + $$ = $1 - $3; + } +| expr '*' expr + { + $$ = $1 * $3; + } +| expr '/' expr + { + $$ = $1 / $3; + } +| expr '%' expr + { + $$ = $1 % $3; + } +| expr '<' '<' expr + { + $$ = $1 << $4; + } +| expr '>' '>' expr + { + $$ = $1 >> $4; + } +| expr '&' expr + { + $$ = $1 & $3; + } +| expr '^' expr + { + $$ = $1 ^ $3; + } +| expr '|' expr + { + $$ = $1 | $3; + } diff --git a/src/cmd/5a/doc.go b/src/cmd/5a/doc.go new file mode 100644 index 000000000..a0d2c4c64 --- /dev/null +++ b/src/cmd/5a/doc.go @@ -0,0 +1,14 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +/* + +5a is a version of the Plan 9 assembler. The original is documented at + + http://plan9.bell-labs.com/magic/man2html/1/2a + +Its target architecture is the ARM, referred to by these tools as arm. + +*/ +package documentation diff --git a/src/cmd/5a/lex.c b/src/cmd/5a/lex.c new file mode 100644 index 000000000..ad7ed05dd --- /dev/null +++ b/src/cmd/5a/lex.c @@ -0,0 +1,704 @@ +// Inferno utils/5a/lex.c +// http://code.google.com/p/inferno-os/source/browse/utils/5a/lex.c +// +// Copyright © 1994-1999 Lucent Technologies Inc. All rights reserved. +// Portions Copyright © 1995-1997 C H Forsyth (forsyth@terzarima.net) +// Portions Copyright © 1997-1999 Vita Nuova Limited +// Portions Copyright © 2000-2007 Vita Nuova Holdings Limited (www.vitanuova.com) +// Portions Copyright © 2004,2006 Bruce Ellis +// Portions Copyright © 2005-2007 C H Forsyth (forsyth@terzarima.net) +// Revisions Copyright © 2000-2007 Lucent Technologies Inc. and others +// Portions Copyright © 2009 The Go Authors. All rights reserved. +// +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to deal +// in the Software without restriction, including without limitation the rights +// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +// copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +// THE SOFTWARE. + +#define EXTERN +#include +#include +#include "a.h" +#include "y.tab.h" + +enum +{ + Plan9 = 1<<0, + Unix = 1<<1, + Windows = 1<<2, +}; + +int +systemtype(int sys) +{ + return sys&Plan9; +} + +void +main(int argc, char *argv[]) +{ + char *p; + int c; + + thechar = '5'; + thestring = "arm"; + + ensuresymb(NSYMB); + memset(debug, 0, sizeof(debug)); + cinit(); + outfile = 0; + setinclude("."); + ARGBEGIN { + default: + c = ARGC(); + if(c >= 0 || c < sizeof(debug)) + debug[c] = 1; + break; + + case 'o': + outfile = ARGF(); + break; + + case 'D': + p = ARGF(); + if(p) { + if (nDlist%8 == 0) + Dlist = allocn(Dlist, nDlist*sizeof(char *), + 8*sizeof(char *)); + Dlist[nDlist++] = p; + } + break; + + case 'I': + p = ARGF(); + setinclude(p); + break; + case 't': + thechar = 't'; + thestring = "thumb"; + break; + } ARGEND + if(*argv == 0) { + print("usage: %ca [-options] file.s\n", thechar); + errorexit(); + } + if(argc > 1){ + print("can't assemble multiple files\n"); + errorexit(); + } + if(assemble(argv[0])) + errorexit(); + exits(0); +} + +int +assemble(char *file) +{ + char *ofile, *p; + int i, of; + + ofile = alloc(strlen(file)+3); // +3 for .x\0 (x=thechar) + strcpy(ofile, file); + p = utfrrune(ofile, '/'); + if(p) { + include[0] = ofile; + *p++ = 0; + } else + p = ofile; + if(outfile == 0) { + outfile = p; + if(outfile){ + p = utfrrune(outfile, '.'); + if(p) + if(p[1] == 's' && p[2] == 0) + p[0] = 0; + p = utfrune(outfile, 0); + p[0] = '.'; + p[1] = thechar; + p[2] = 0; + } else + outfile = "/dev/null"; + } + + of = create(outfile, OWRITE, 0664); + if(of < 0) { + yyerror("%ca: cannot create %s", thechar, outfile); + errorexit(); + } + Binit(&obuf, of, OWRITE); + + pass = 1; + pinit(file); + + Bprint(&obuf, "go object %s %s %s\n", getgoos(), thestring, getgoversion()); + + for(i=0; itype = itab[i].type; + s->value = itab[i].value; + } + + pathname = allocn(pathname, 0, 100); + if(getwd(pathname, 99) == 0) { + pathname = allocn(pathname, 100, 900); + if(getwd(pathname, 999) == 0) + strcpy(pathname, "/???"); + } +} + +void +syminit(Sym *s) +{ + + s->type = LNAME; + s->value = 0; +} + +int +isreg(Gen *g) +{ + + USED(g); + return 1; +} + +void +cclean(void) +{ + + outcode(AEND, Always, &nullgen, NREG, &nullgen); + Bflush(&obuf); +} + +void +zname(char *n, int t, int s) +{ + + Bputc(&obuf, ANAME); + Bputc(&obuf, t); /* type */ + Bputc(&obuf, s); /* sym */ + while(*n) { + Bputc(&obuf, *n); + n++; + } + Bputc(&obuf, 0); +} + +void +zaddr(Gen *a, int s) +{ + int32 l; + int i; + char *n; + Ieee e; + + Bputc(&obuf, a->type); + Bputc(&obuf, a->reg); + Bputc(&obuf, s); + Bputc(&obuf, a->name); + switch(a->type) { + default: + print("unknown type %d\n", a->type); + exits("arg"); + + case D_NONE: + case D_REG: + case D_FREG: + case D_PSR: + case D_FPCR: + break; + + case D_REGREG: + Bputc(&obuf, a->offset); + break; + + case D_OREG: + case D_CONST: + case D_BRANCH: + case D_SHIFT: + l = a->offset; + Bputc(&obuf, l); + Bputc(&obuf, l>>8); + Bputc(&obuf, l>>16); + Bputc(&obuf, l>>24); + break; + + case D_SCONST: + n = a->sval; + for(i=0; idval); + Bputc(&obuf, e.l); + Bputc(&obuf, e.l>>8); + Bputc(&obuf, e.l>>16); + Bputc(&obuf, e.l>>24); + Bputc(&obuf, e.h); + Bputc(&obuf, e.h>>8); + Bputc(&obuf, e.h>>16); + Bputc(&obuf, e.h>>24); + break; + } +} + +static int bcode[] = +{ + ABEQ, + ABNE, + ABCS, + ABCC, + ABMI, + ABPL, + ABVS, + ABVC, + ABHI, + ABLS, + ABGE, + ABLT, + ABGT, + ABLE, + AB, + ANOP, +}; + +void +outcode(int a, int scond, Gen *g1, int reg, Gen *g2) +{ + int sf, st, t; + Sym *s; + + /* hack to make B.NE etc. work: turn it into the corresponding conditional */ + if(a == AB){ + a = bcode[scond&0xf]; + scond = (scond & ~0xf) | Always; + } + + if(pass == 1) + goto out; +jackpot: + sf = 0; + s = g1->sym; + while(s != S) { + sf = s->sym; + if(sf < 0 || sf >= NSYM) + sf = 0; + t = g1->name; + if(h[sf].type == t) + if(h[sf].sym == s) + break; + zname(s->name, t, sym); + s->sym = sym; + h[sym].sym = s; + h[sym].type = t; + sf = sym; + sym++; + if(sym >= NSYM) + sym = 1; + break; + } + st = 0; + s = g2->sym; + while(s != S) { + st = s->sym; + if(st < 0 || st >= NSYM) + st = 0; + t = g2->name; + if(h[st].type == t) + if(h[st].sym == s) + break; + zname(s->name, t, sym); + s->sym = sym; + h[sym].sym = s; + h[sym].type = t; + st = sym; + sym++; + if(sym >= NSYM) + sym = 1; + if(st == sf) + goto jackpot; + break; + } + Bputc(&obuf, a); + Bputc(&obuf, scond); + Bputc(&obuf, reg); + Bputc(&obuf, stmtline); + Bputc(&obuf, stmtline>>8); + Bputc(&obuf, stmtline>>16); + Bputc(&obuf, stmtline>>24); + zaddr(g1, sf); + zaddr(g2, st); + +out: + if(a != AGLOBL && a != ADATA) + pc++; +} + +void +outhist(void) +{ + Gen g; + Hist *h; + char *p, *q, *op, c; + int n; + + g = nullgen; + c = '/'; + for(h = hist; h != H; h = h->link) { + p = h->name; + op = 0; + /* on windows skip drive specifier in pathname */ + if(systemtype(Windows) && p && p[1] == ':'){ + p += 2; + c = *p; + } + if(p && p[0] != c && h->offset == 0 && pathname){ + /* on windows skip drive specifier in pathname */ + if(systemtype(Windows) && pathname[1] == ':') { + op = p; + p = pathname+2; + c = *p; + } else if(pathname[0] == c){ + op = p; + p = pathname; + } + } + while(p) { + q = strchr(p, c); + if(q) { + n = q-p; + if(n == 0){ + n = 1; /* leading "/" */ + *p = '/'; /* don't emit "\" on windows */ + } + q++; + } else { + n = strlen(p); + q = 0; + } + if(n) { + Bputc(&obuf, ANAME); + Bputc(&obuf, D_FILE); /* type */ + Bputc(&obuf, 1); /* sym */ + Bputc(&obuf, '<'); + Bwrite(&obuf, p, n); + Bputc(&obuf, 0); + } + p = q; + if(p == 0 && op) { + p = op; + op = 0; + } + } + g.offset = h->offset; + + Bputc(&obuf, AHISTORY); + Bputc(&obuf, Always); + Bputc(&obuf, 0); + Bputc(&obuf, h->line); + Bputc(&obuf, h->line>>8); + Bputc(&obuf, h->line>>16); + Bputc(&obuf, h->line>>24); + zaddr(&nullgen, 0); + zaddr(&g, 0); + } +} + +#include "../cc/lexbody" +#include "../cc/macbody" diff --git a/src/cmd/5c/Makefile b/src/cmd/5c/Makefile new file mode 100644 index 000000000..70b614e8a --- /dev/null +++ b/src/cmd/5c/Makefile @@ -0,0 +1,34 @@ +# Copyright 2009 The Go Authors. All rights reserved. +# Use of this source code is governed by a BSD-style +# license that can be found in the LICENSE file. + +include ../../Make.inc +O:=$(HOST_O) + +TARG=5c + +HFILES=\ + gc.h\ + ../5l/5.out.h\ + ../cc/cc.h\ + +OFILES=\ + cgen.$O\ + list.$O\ + sgen.$O\ + swt.$O\ + txt.$O\ + mul.$O\ + reg.$O\ + peep.$O\ + pgen.$O\ + pswt.$O\ + ../5l/enam.$O\ + +LIB=\ + ../cc/cc.a\ + +include ../../Make.ccmd + +%.$O: ../cc/%.c + $(HOST_CC) $(HOST_CFLAGS) -c -I. -o $@ ../cc/$*.c diff --git a/src/cmd/5c/cgen.c b/src/cmd/5c/cgen.c new file mode 100644 index 000000000..9e74f515b --- /dev/null +++ b/src/cmd/5c/cgen.c @@ -0,0 +1,1199 @@ +// Inferno utils/5c/cgen.c +// http://code.google.com/p/inferno-os/source/browse/utils/5c/cgen.c +// +// Copyright © 1994-1999 Lucent Technologies Inc. All rights reserved. +// Portions Copyright © 1995-1997 C H Forsyth (forsyth@terzarima.net) +// Portions Copyright © 1997-1999 Vita Nuova Limited +// Portions Copyright © 2000-2007 Vita Nuova Holdings Limited (www.vitanuova.com) +// Portions Copyright © 2004,2006 Bruce Ellis +// Portions Copyright © 2005-2007 C H Forsyth (forsyth@terzarima.net) +// Revisions Copyright © 2000-2007 Lucent Technologies Inc. and others +// Portions Copyright © 2009 The Go Authors. All rights reserved. +// +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to deal +// in the Software without restriction, including without limitation the rights +// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +// copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +// THE SOFTWARE. + + +#include "gc.h" + +void +_cgen(Node *n, Node *nn, int inrel) +{ + Node *l, *r; + Prog *p1; + Node nod, nod1, nod2, nod3, nod4; + int o, t; + int32 v, curs; + + if(debug['g']) { + prtree(nn, "cgen lhs"); + prtree(n, "cgen"); + } + if(n == Z || n->type == T) + return; + if(typesuv[n->type->etype]) { + sugen(n, nn, n->type->width); + return; + } + l = n->left; + r = n->right; + o = n->op; + if(n->addable >= INDEXED) { + if(nn == Z) { + switch(o) { + default: + nullwarn(Z, Z); + break; + case OINDEX: + nullwarn(l, r); + break; + } + return; + } + gmove(n, nn); + return; + } + curs = cursafe; + + if(n->complex >= FNX) + if(l->complex >= FNX) + if(r != Z && r->complex >= FNX) + switch(o) { + default: + regret(&nod, r); + cgen(r, &nod); + + regsalloc(&nod1, r); + gopcode(OAS, &nod, Z, &nod1); + + regfree(&nod); + nod = *n; + nod.right = &nod1; + cgen(&nod, nn); + return; + + case OFUNC: + case OCOMMA: + case OANDAND: + case OOROR: + case OCOND: + case ODOT: + break; + } + + switch(o) { + default: + diag(n, "unknown op in cgen: %O", o); + break; + + case OAS: + if(l->op == OBIT) + goto bitas; + if(l->addable >= INDEXED && l->complex < FNX) { + if(nn != Z || r->addable < INDEXED) { + if(r->complex >= FNX && nn == Z) + regret(&nod, r); + else + regalloc(&nod, r, nn); + cgen(r, &nod); + gmove(&nod, l); + if(nn != Z) + gmove(&nod, nn); + regfree(&nod); + } else + gmove(r, l); + break; + } + if(l->complex >= r->complex) { + reglcgen(&nod1, l, Z); + if(r->addable >= INDEXED) { + gmove(r, &nod1); + if(nn != Z) + gmove(r, nn); + regfree(&nod1); + break; + } + regalloc(&nod, r, nn); + cgen(r, &nod); + } else { + regalloc(&nod, r, nn); + cgen(r, &nod); + reglcgen(&nod1, l, Z); + } + gmove(&nod, &nod1); + regfree(&nod); + regfree(&nod1); + break; + + bitas: + n = l->left; + regalloc(&nod, r, nn); + if(l->complex >= r->complex) { + reglcgen(&nod1, n, Z); + cgen(r, &nod); + } else { + cgen(r, &nod); + reglcgen(&nod1, n, Z); + } + regalloc(&nod2, n, Z); + gopcode(OAS, &nod1, Z, &nod2); + bitstore(l, &nod, &nod1, &nod2, nn); + break; + + case OBIT: + if(nn == Z) { + nullwarn(l, Z); + break; + } + bitload(n, &nod, Z, Z, nn); + gopcode(OAS, &nod, Z, nn); + regfree(&nod); + break; + + case ODIV: + case OMOD: + if(nn != Z) + if((t = vlog(r)) >= 0) { + /* signed div/mod by constant power of 2 */ + cgen(l, nn); + gopcode(OGE, nodconst(0), nn, Z); + p1 = p; + if(o == ODIV) { + gopcode(OADD, nodconst((1<op == OCONST) + if(!typefd[n->type->etype]) { + cgen(r, nn); + gopcode(o, Z, l, nn); + break; + } + case OADD: + case OAND: + case OOR: + case OXOR: + case OLSHR: + case OASHL: + case OASHR: + /* + * immediate operands + */ + if(nn != Z) + if(r->op == OCONST) + if(!typefd[n->type->etype]) { + cgen(l, nn); + if(r->vconst == 0) + if(o != OAND) + break; + if(nn != Z) + gopcode(o, r, Z, nn); + break; + } + + case OLMUL: + case OLDIV: + case OLMOD: + case OMUL: + muldiv: + if(nn == Z) { + nullwarn(l, r); + break; + } + if(o == OMUL || o == OLMUL) { + if(mulcon(n, nn)) + break; + } + if(l->complex >= r->complex) { + regalloc(&nod, l, nn); + cgen(l, &nod); + regalloc(&nod1, r, Z); + cgen(r, &nod1); + gopcode(o, &nod1, Z, &nod); + } else { + regalloc(&nod, r, nn); + cgen(r, &nod); + regalloc(&nod1, l, Z); + cgen(l, &nod1); + gopcode(o, &nod, &nod1, &nod); + } + gopcode(OAS, &nod, Z, nn); + regfree(&nod); + regfree(&nod1); + break; + + case OASLSHR: + case OASASHL: + case OASASHR: + case OASAND: + case OASADD: + case OASSUB: + case OASXOR: + case OASOR: + if(l->op == OBIT) + goto asbitop; + if(r->op == OCONST) + if(!typefd[r->type->etype]) + if(!typefd[n->type->etype]) { + if(l->addable < INDEXED) + reglcgen(&nod2, l, Z); + else + nod2 = *l; + regalloc(&nod, r, nn); + gopcode(OAS, &nod2, Z, &nod); + gopcode(o, r, Z, &nod); + gopcode(OAS, &nod, Z, &nod2); + + regfree(&nod); + if(l->addable < INDEXED) + regfree(&nod2); + break; + } + + case OASLMUL: + case OASLDIV: + case OASLMOD: + case OASMUL: + case OASDIV: + case OASMOD: + if(l->op == OBIT) + goto asbitop; + if(l->complex >= r->complex) { + if(l->addable < INDEXED) + reglcgen(&nod2, l, Z); + else + nod2 = *l; + regalloc(&nod1, r, Z); + cgen(r, &nod1); + } else { + regalloc(&nod1, r, Z); + cgen(r, &nod1); + if(l->addable < INDEXED) + reglcgen(&nod2, l, Z); + else + nod2 = *l; + } + + regalloc(&nod, n, nn); + gmove(&nod2, &nod); + gopcode(o, &nod1, Z, &nod); + gmove(&nod, &nod2); + if(nn != Z) + gopcode(OAS, &nod, Z, nn); + regfree(&nod); + regfree(&nod1); + if(l->addable < INDEXED) + regfree(&nod2); + break; + + asbitop: + regalloc(&nod4, n, nn); + if(l->complex >= r->complex) { + bitload(l, &nod, &nod1, &nod2, &nod4); + regalloc(&nod3, r, Z); + cgen(r, &nod3); + } else { + regalloc(&nod3, r, Z); + cgen(r, &nod3); + bitload(l, &nod, &nod1, &nod2, &nod4); + } + gmove(&nod, &nod4); + gopcode(o, &nod3, Z, &nod4); + regfree(&nod3); + gmove(&nod4, &nod); + regfree(&nod4); + bitstore(l, &nod, &nod1, &nod2, nn); + break; + + case OADDR: + if(nn == Z) { + nullwarn(l, Z); + break; + } + lcgen(l, nn); + break; + + case OFUNC: + if(l->complex >= FNX) { + if(l->op != OIND) + diag(n, "bad function call"); + + regret(&nod, l->left); + cgen(l->left, &nod); + regsalloc(&nod1, l->left); + gopcode(OAS, &nod, Z, &nod1); + regfree(&nod); + + nod = *n; + nod.left = &nod2; + nod2 = *l; + nod2.left = &nod1; + nod2.complex = 1; + cgen(&nod, nn); + + return; + } + if(REGARG >= 0) + o = reg[REGARG]; + gargs(r, &nod, &nod1); + if(l->addable < INDEXED) { + reglcgen(&nod, l, Z); + gopcode(OFUNC, Z, Z, &nod); + regfree(&nod); + } else + gopcode(OFUNC, Z, Z, l); + if(REGARG >= 0) + if(o != reg[REGARG]) + reg[REGARG]--; + if(nn != Z) { + regret(&nod, n); + gopcode(OAS, &nod, Z, nn); + regfree(&nod); + } + break; + + case OIND: + if(nn == Z) { + nullwarn(l, Z); + break; + } + regialloc(&nod, n, nn); + r = l; + while(r->op == OADD) + r = r->right; + if(sconst(r) && (v = r->vconst+nod.xoffset) > -4096 && v < 4096) { + v = r->vconst; + r->vconst = 0; + cgen(l, &nod); + nod.xoffset += v; + r->vconst = v; + } else + cgen(l, &nod); + regind(&nod, n); + gopcode(OAS, &nod, Z, nn); + regfree(&nod); + break; + + case OEQ: + case ONE: + case OLE: + case OLT: + case OGE: + case OGT: + case OLO: + case OLS: + case OHI: + case OHS: + if(nn == Z) { + nullwarn(l, r); + break; + } + boolgen(n, 1, nn); + break; + + case OANDAND: + case OOROR: + boolgen(n, 1, nn); + if(nn == Z) + patch(p, pc); + break; + + case ONOT: + if(nn == Z) { + nullwarn(l, Z); + break; + } + boolgen(n, 1, nn); + break; + + case OCOMMA: + cgen(l, Z); + cgen(r, nn); + break; + + case OCAST: + if(nn == Z) { + nullwarn(l, Z); + break; + } + /* + * convert from types l->n->nn + */ + if(nocast(l->type, n->type)) { + if(nocast(n->type, nn->type)) { + cgen(l, nn); + break; + } + } + regalloc(&nod, l, nn); + cgen(l, &nod); + regalloc(&nod1, n, &nod); + if(inrel) + gmover(&nod, &nod1); + else + gopcode(OAS, &nod, Z, &nod1); + gopcode(OAS, &nod1, Z, nn); + regfree(&nod1); + regfree(&nod); + break; + + case ODOT: + sugen(l, nodrat, l->type->width); + if(nn != Z) { + warn(n, "non-interruptable temporary"); + nod = *nodrat; + if(!r || r->op != OCONST) { + diag(n, "DOT and no offset"); + break; + } + nod.xoffset += (int32)r->vconst; + nod.type = n->type; + cgen(&nod, nn); + } + break; + + case OCOND: + bcgen(l, 1); + p1 = p; + cgen(r->left, nn); + gbranch(OGOTO); + patch(p1, pc); + p1 = p; + cgen(r->right, nn); + patch(p1, pc); + break; + + case OPOSTINC: + case OPOSTDEC: + v = 1; + if(l->type->etype == TIND) + v = l->type->link->width; + if(o == OPOSTDEC) + v = -v; + if(l->op == OBIT) + goto bitinc; + if(nn == Z) + goto pre; + + if(l->addable < INDEXED) + reglcgen(&nod2, l, Z); + else + nod2 = *l; + + regalloc(&nod, l, nn); + gopcode(OAS, &nod2, Z, &nod); + regalloc(&nod1, l, Z); + if(typefd[l->type->etype]) { + regalloc(&nod3, l, Z); + if(v < 0) { + gopcode(OAS, nodfconst(-v), Z, &nod3); + gopcode(OSUB, &nod3, &nod, &nod1); + } else { + gopcode(OAS, nodfconst(v), Z, &nod3); + gopcode(OADD, &nod3, &nod, &nod1); + } + regfree(&nod3); + } else + gopcode(OADD, nodconst(v), &nod, &nod1); + gopcode(OAS, &nod1, Z, &nod2); + + regfree(&nod); + regfree(&nod1); + if(l->addable < INDEXED) + regfree(&nod2); + break; + + case OPREINC: + case OPREDEC: + v = 1; + if(l->type->etype == TIND) + v = l->type->link->width; + if(o == OPREDEC) + v = -v; + if(l->op == OBIT) + goto bitinc; + + pre: + if(l->addable < INDEXED) + reglcgen(&nod2, l, Z); + else + nod2 = *l; + + regalloc(&nod, l, nn); + gopcode(OAS, &nod2, Z, &nod); + if(typefd[l->type->etype]) { + regalloc(&nod3, l, Z); + if(v < 0) { + gopcode(OAS, nodfconst(-v), Z, &nod3); + gopcode(OSUB, &nod3, Z, &nod); + } else { + gopcode(OAS, nodfconst(v), Z, &nod3); + gopcode(OADD, &nod3, Z, &nod); + } + regfree(&nod3); + } else + gopcode(OADD, nodconst(v), Z, &nod); + gopcode(OAS, &nod, Z, &nod2); + + regfree(&nod); + if(l->addable < INDEXED) + regfree(&nod2); + break; + + bitinc: + if(nn != Z && (o == OPOSTINC || o == OPOSTDEC)) { + bitload(l, &nod, &nod1, &nod2, Z); + gopcode(OAS, &nod, Z, nn); + gopcode(OADD, nodconst(v), Z, &nod); + bitstore(l, &nod, &nod1, &nod2, Z); + break; + } + bitload(l, &nod, &nod1, &nod2, nn); + gopcode(OADD, nodconst(v), Z, &nod); + bitstore(l, &nod, &nod1, &nod2, nn); + break; + } + cursafe = curs; + return; +} + +void +cgen(Node *n, Node *nn) +{ + _cgen(n, nn, 0); +} + +void +cgenrel(Node *n, Node *nn) +{ + _cgen(n, nn, 1); +} + +void +reglcgen(Node *t, Node *n, Node *nn) +{ + Node *r; + int32 v; + + regialloc(t, n, nn); + if(n->op == OIND) { + r = n->left; + while(r->op == OADD) + r = r->right; + if(sconst(r) && (v = r->vconst+t->xoffset) > -4096 && v < 4096) { + v = r->vconst; + r->vconst = 0; + lcgen(n, t); + t->xoffset += v; + r->vconst = v; + regind(t, n); + return; + } + } else if(n->op == OINDREG) { + if((v = n->xoffset) > -4096 && v < 4096) { + n->op = OREGISTER; + cgen(n, t); + t->xoffset += v; + n->op = OINDREG; + regind(t, n); + return; + } + } + lcgen(n, t); + regind(t, n); +} + +void +reglpcgen(Node *n, Node *nn, int f) +{ + Type *t; + + t = nn->type; + nn->type = types[TLONG]; + if(f) + reglcgen(n, nn, Z); + else { + regialloc(n, nn, Z); + lcgen(nn, n); + regind(n, nn); + } + nn->type = t; +} + +void +lcgen(Node *n, Node *nn) +{ + Prog *p1; + Node nod; + + if(debug['g']) { + prtree(nn, "lcgen lhs"); + prtree(n, "lcgen"); + } + if(n == Z || n->type == T) + return; + if(nn == Z) { + nn = &nod; + regalloc(&nod, n, Z); + } + switch(n->op) { + default: + if(n->addable < INDEXED) { + diag(n, "unknown op in lcgen: %O", n->op); + break; + } + nod = *n; + nod.op = OADDR; + nod.left = n; + nod.right = Z; + nod.type = types[TIND]; + gopcode(OAS, &nod, Z, nn); + break; + + case OCOMMA: + cgen(n->left, n->left); + lcgen(n->right, nn); + break; + + case OIND: + cgen(n->left, nn); + break; + + case OCOND: + bcgen(n->left, 1); + p1 = p; + lcgen(n->right->left, nn); + gbranch(OGOTO); + patch(p1, pc); + p1 = p; + lcgen(n->right->right, nn); + patch(p1, pc); + break; + } +} + +void +bcgen(Node *n, int true) +{ + + if(n->type == T) + gbranch(OGOTO); + else + boolgen(n, true, Z); +} + +void +boolgen(Node *n, int true, Node *nn) +{ + int o; + Prog *p1, *p2; + Node *l, *r, nod, nod1; + int32 curs; + + if(debug['g']) { + prtree(nn, "boolgen lhs"); + prtree(n, "boolgen"); + } + curs = cursafe; + l = n->left; + r = n->right; + switch(n->op) { + + default: + regalloc(&nod, n, nn); + cgen(n, &nod); + o = ONE; + if(true) + o = comrel[relindex(o)]; + if(typefd[n->type->etype]) { + gopcode(o, nodfconst(0), &nod, Z); + } else + gopcode(o, nodconst(0), &nod, Z); + regfree(&nod); + goto com; + + case OCONST: + o = vconst(n); + if(!true) + o = !o; + gbranch(OGOTO); + if(o) { + p1 = p; + gbranch(OGOTO); + patch(p1, pc); + } + goto com; + + case OCOMMA: + cgen(l, Z); + boolgen(r, true, nn); + break; + + case ONOT: + boolgen(l, !true, nn); + break; + + case OCOND: + bcgen(l, 1); + p1 = p; + bcgen(r->left, true); + p2 = p; + gbranch(OGOTO); + patch(p1, pc); + p1 = p; + bcgen(r->right, !true); + patch(p2, pc); + p2 = p; + gbranch(OGOTO); + patch(p1, pc); + patch(p2, pc); + goto com; + + case OANDAND: + if(!true) + goto caseor; + + caseand: + bcgen(l, true); + p1 = p; + bcgen(r, !true); + p2 = p; + patch(p1, pc); + gbranch(OGOTO); + patch(p2, pc); + goto com; + + case OOROR: + if(!true) + goto caseand; + + caseor: + bcgen(l, !true); + p1 = p; + bcgen(r, !true); + p2 = p; + gbranch(OGOTO); + patch(p1, pc); + patch(p2, pc); + goto com; + + case OEQ: + case ONE: + case OLE: + case OLT: + case OGE: + case OGT: + case OHI: + case OHS: + case OLO: + case OLS: + o = n->op; + if(true) + o = comrel[relindex(o)]; + if(l->complex >= FNX && r->complex >= FNX) { + regret(&nod, r); + cgenrel(r, &nod); + regsalloc(&nod1, r); + gopcode(OAS, &nod, Z, &nod1); + regfree(&nod); + nod = *n; + nod.right = &nod1; + boolgen(&nod, true, nn); + break; + } + if(sconst(l)) { + regalloc(&nod, r, nn); + cgenrel(r, &nod); + o = invrel[relindex(o)]; + gopcode(o, l, &nod, Z); + regfree(&nod); + goto com; + } + if(sconst(r)) { + regalloc(&nod, l, nn); + cgenrel(l, &nod); + gopcode(o, r, &nod, Z); + regfree(&nod); + goto com; + } + if(l->complex >= r->complex) { + regalloc(&nod1, l, nn); + cgenrel(l, &nod1); + regalloc(&nod, r, Z); + cgenrel(r, &nod); + } else { + regalloc(&nod, r, nn); + cgenrel(r, &nod); + regalloc(&nod1, l, Z); + cgenrel(l, &nod1); + } + gopcode(o, &nod, &nod1, Z); + regfree(&nod); + regfree(&nod1); + + com: + if(nn != Z) { + p1 = p; + gopcode(OAS, nodconst(1), Z, nn); + gbranch(OGOTO); + p2 = p; + patch(p1, pc); + gopcode(OAS, nodconst(0), Z, nn); + patch(p2, pc); + } + break; + } + cursafe = curs; +} + +void +sugen(Node *n, Node *nn, int32 w) +{ + Prog *p1; + Node nod0, nod1, nod2, nod3, nod4, *l, *r; + Type *t; + int32 pc1; + int i, m, c; + + if(n == Z || n->type == T) + return; + if(debug['g']) { + prtree(nn, "sugen lhs"); + prtree(n, "sugen"); + } + if(nn == nodrat) + if(w > nrathole) + nrathole = w; + switch(n->op) { + case OIND: + if(nn == Z) { + nullwarn(n->left, Z); + break; + } + + default: + goto copy; + + case OCONST: + if(n->type && typev[n->type->etype]) { + if(nn == Z) { + nullwarn(n->left, Z); + break; + } + + t = nn->type; + nn->type = types[TLONG]; + reglcgen(&nod1, nn, Z); + nn->type = t; + + if(isbigendian) + gopcode(OAS, nod32const(n->vconst>>32), Z, &nod1); + else + gopcode(OAS, nod32const(n->vconst), Z, &nod1); + nod1.xoffset += SZ_LONG; + if(isbigendian) + gopcode(OAS, nod32const(n->vconst), Z, &nod1); + else + gopcode(OAS, nod32const(n->vconst>>32), Z, &nod1); + + regfree(&nod1); + break; + } + goto copy; + + case ODOT: + l = n->left; + sugen(l, nodrat, l->type->width); + if(nn != Z) { + warn(n, "non-interruptable temporary"); + nod1 = *nodrat; + r = n->right; + if(!r || r->op != OCONST) { + diag(n, "DOT and no offset"); + break; + } + nod1.xoffset += (int32)r->vconst; + nod1.type = n->type; + sugen(&nod1, nn, w); + } + break; + + case OSTRUCT: + /* + * rewrite so lhs has no fn call + */ + if(nn != Z && nn->complex >= FNX) { + nod1 = *n; + nod1.type = typ(TIND, n->type); + regret(&nod2, &nod1); + lcgen(nn, &nod2); + regsalloc(&nod0, &nod1); + gopcode(OAS, &nod2, Z, &nod0); + regfree(&nod2); + + nod1 = *n; + nod1.op = OIND; + nod1.left = &nod0; + nod1.right = Z; + nod1.complex = 1; + + sugen(n, &nod1, w); + return; + } + + r = n->left; + for(t = n->type->link; t != T; t = t->down) { + l = r; + if(r->op == OLIST) { + l = r->left; + r = r->right; + } + if(nn == Z) { + cgen(l, nn); + continue; + } + /* + * hand craft *(&nn + o) = l + */ + nod0 = znode; + nod0.op = OAS; + nod0.type = t; + nod0.left = &nod1; + nod0.right = l; + + nod1 = znode; + nod1.op = OIND; + nod1.type = t; + nod1.left = &nod2; + + nod2 = znode; + nod2.op = OADD; + nod2.type = typ(TIND, t); + nod2.left = &nod3; + nod2.right = &nod4; + + nod3 = znode; + nod3.op = OADDR; + nod3.type = nod2.type; + nod3.left = nn; + + nod4 = znode; + nod4.op = OCONST; + nod4.type = nod2.type; + nod4.vconst = t->offset; + + ccom(&nod0); + acom(&nod0); + xcom(&nod0); + nod0.addable = 0; + + cgen(&nod0, Z); + } + break; + + case OAS: + if(nn == Z) { + if(n->addable < INDEXED) + sugen(n->right, n->left, w); + break; + } + sugen(n->right, nodrat, w); + warn(n, "non-interruptable temporary"); + sugen(nodrat, n->left, w); + sugen(nodrat, nn, w); + break; + + case OFUNC: + if(nn == Z) { + sugen(n, nodrat, w); + break; + } + if(nn->op != OIND) { + nn = new1(OADDR, nn, Z); + nn->type = types[TIND]; + nn->addable = 0; + } else + nn = nn->left; + n = new(OFUNC, n->left, new(OLIST, nn, n->right)); + n->type = types[TVOID]; + n->left->type = types[TVOID]; + cgen(n, Z); + break; + + case OCOND: + bcgen(n->left, 1); + p1 = p; + sugen(n->right->left, nn, w); + gbranch(OGOTO); + patch(p1, pc); + p1 = p; + sugen(n->right->right, nn, w); + patch(p1, pc); + break; + + case OCOMMA: + cgen(n->left, Z); + sugen(n->right, nn, w); + break; + } + return; + +copy: + if(nn == Z) + return; + if(n->complex >= FNX && nn->complex >= FNX) { + t = nn->type; + nn->type = types[TLONG]; + regialloc(&nod1, nn, Z); + lcgen(nn, &nod1); + regsalloc(&nod2, nn); + nn->type = t; + + gopcode(OAS, &nod1, Z, &nod2); + regfree(&nod1); + + nod2.type = typ(TIND, t); + + nod1 = nod2; + nod1.op = OIND; + nod1.left = &nod2; + nod1.right = Z; + nod1.complex = 1; + nod1.type = t; + + sugen(n, &nod1, w); + return; + } + + w /= SZ_LONG; + if(w <= 2) { + if(n->complex > nn->complex) { + reglpcgen(&nod1, n, 1); + reglpcgen(&nod2, nn, 1); + } else { + reglpcgen(&nod2, nn, 1); + reglpcgen(&nod1, n, 1); + } + regalloc(&nod3, ®node, Z); + regalloc(&nod4, ®node, Z); + nod0 = *nodconst((1<complex > nn->complex) { + reglpcgen(&nod1, n, 0); + reglpcgen(&nod2, nn, 0); + } else { + reglpcgen(&nod2, nn, 0); + reglpcgen(&nod1, n, 0); + } + + m = 0; + for(c = 0; c < w && c < 4; c++) { + i = tmpreg(); + if (i == 0) + break; + reg[i]++; + m |= 1<c; w-=c) { + gmovm(&nod1, &nod4, 1); + gmovm(&nod4, &nod2, 1); + } + goto out; + } + + regalloc(&nod3, ®node, Z); + gopcode(OAS, nodconst(w/c), Z, &nod3); + w %= c; + + pc1 = pc; + gmovm(&nod1, &nod4, 1); + gmovm(&nod4, &nod2, 1); + + gopcode(OSUB, nodconst(1), Z, &nod3); + gopcode(OEQ, nodconst(0), &nod3, Z); + p->as = ABGT; + patch(p, pc1); + regfree(&nod3); + +out: + if (w) { + i = 0; + while (c>w) { + while ((m&(1<0); + regfree(&nod1); + regfree(&nod2); +} diff --git a/src/cmd/5c/doc.go b/src/cmd/5c/doc.go new file mode 100644 index 000000000..0874293bf --- /dev/null +++ b/src/cmd/5c/doc.go @@ -0,0 +1,14 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +/* + +5c is a version of the Plan 9 C compiler. The original is documented at + + http://plan9.bell-labs.com/magic/man2html/1/2c + +Its target architecture is the ARM, referred to by these tools as arm. + +*/ +package documentation diff --git a/src/cmd/5c/gc.h b/src/cmd/5c/gc.h new file mode 100644 index 000000000..ff6d51916 --- /dev/null +++ b/src/cmd/5c/gc.h @@ -0,0 +1,384 @@ +// Inferno utils/5c/gc.h +// http://code.google.com/p/inferno-os/source/browse/utils/5c/gc.h +// +// Copyright © 1994-1999 Lucent Technologies Inc. All rights reserved. +// Portions Copyright © 1995-1997 C H Forsyth (forsyth@terzarima.net) +// Portions Copyright © 1997-1999 Vita Nuova Limited +// Portions Copyright © 2000-2007 Vita Nuova Holdings Limited (www.vitanuova.com) +// Portions Copyright © 2004,2006 Bruce Ellis +// Portions Copyright © 2005-2007 C H Forsyth (forsyth@terzarima.net) +// Revisions Copyright © 2000-2007 Lucent Technologies Inc. and others +// Portions Copyright © 2009 The Go Authors. All rights reserved. +// +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to deal +// in the Software without restriction, including without limitation the rights +// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +// copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +// THE SOFTWARE. + +#include +#include "../cc/cc.h" +#include "../5l/5.out.h" + +/* + * 5c/arm + * Arm 7500 + */ +#define SZ_CHAR 1 +#define SZ_SHORT 2 +#define SZ_INT 4 +#define SZ_LONG 4 +#define SZ_IND 4 +#define SZ_FLOAT 4 +#define SZ_VLONG 8 +#define SZ_DOUBLE 8 +#define FNX 100 + +typedef struct Adr Adr; +typedef struct Prog Prog; +typedef struct Case Case; +typedef struct C1 C1; +typedef struct Multab Multab; +typedef struct Hintab Hintab; +typedef struct Var Var; +typedef struct Reg Reg; +typedef struct Rgn Rgn; + + +#define R0ISZERO 0 + +struct Adr +{ + int32 offset; + int32 offset2; + double dval; + char sval[NSNAME]; + Ieee ieee; + + Sym* sym; + char type; + uchar reg; + char name; + char etype; +}; +#define A ((Adr*)0) + +#define INDEXED 9 +struct Prog +{ + Adr from; + Adr to; + Prog* link; + int32 lineno; + char as; + uchar reg; + uchar scond; +}; +#define P ((Prog*)0) + +struct Case +{ + Case* link; + int32 val; + int32 label; + char def; + char isv; +}; +#define C ((Case*)0) + +struct C1 +{ + int32 val; + int32 label; +}; + +struct Multab +{ + int32 val; + char code[20]; +}; + +struct Hintab +{ + ushort val; + char hint[10]; +}; + +struct Var +{ + int32 offset; + Sym* sym; + char name; + char etype; +}; + +struct Reg +{ + int32 pc; + int32 rpo; /* reverse post ordering */ + + Bits set; + Bits use1; + Bits use2; + + Bits refbehind; + Bits refahead; + Bits calbehind; + Bits calahead; + Bits regdiff; + Bits act; + + int32 regu; + int32 loop; /* could be shorter */ + + + Reg* log5; + int32 active; + + Reg* p1; + Reg* p2; + Reg* p2link; + Reg* s1; + Reg* s2; + Reg* link; + Prog* prog; +}; +#define R ((Reg*)0) + +#define NRGN 600 +struct Rgn +{ + Reg* enter; + short cost; + short varno; + short regno; +}; + +EXTERN int32 breakpc; +EXTERN int32 nbreak; +EXTERN Case* cases; +EXTERN Node constnode; +EXTERN Node fconstnode; +EXTERN int32 continpc; +EXTERN int32 curarg; +EXTERN int32 cursafe; +EXTERN Prog* firstp; +EXTERN int32 isbigendian; +EXTERN Prog* lastp; +EXTERN int32 maxargsafe; +EXTERN int mnstring; +EXTERN Multab multab[20]; +EXTERN int retok; +EXTERN int hintabsize; +EXTERN Node* nodrat; +EXTERN Node* nodret; +EXTERN Node* nodsafe; +EXTERN int32 nrathole; +EXTERN int32 nstring; +EXTERN Prog* p; +EXTERN int32 pc; +EXTERN Node regnode; +EXTERN char string[NSNAME]; +EXTERN Sym* symrathole; +EXTERN Node znode; +EXTERN Prog zprog; +EXTERN char reg[NREG+NFREG]; +EXTERN int32 exregoffset; +EXTERN int32 exfregoffset; +EXTERN int suppress; + +#define BLOAD(r) band(bnot(r->refbehind), r->refahead) +#define BSTORE(r) band(bnot(r->calbehind), r->calahead) +#define LOAD(r) (~r->refbehind.b[z] & r->refahead.b[z]) +#define STORE(r) (~r->calbehind.b[z] & r->calahead.b[z]) + +#define bset(a,n) ((a).b[(n)/32]&(1L<<(n)%32)) + +#define CLOAD 4 +#define CREF 5 +#define CINF 1000 +#define LOOP 3 + +EXTERN Rgn region[NRGN]; +EXTERN Rgn* rgp; +EXTERN int nregion; +EXTERN int nvar; + +EXTERN Bits externs; +EXTERN Bits params; +EXTERN Bits consts; +EXTERN Bits addrs; + +EXTERN int32 regbits; +EXTERN int32 exregbits; + +EXTERN int change; + +EXTERN Reg* firstr; +EXTERN Reg* lastr; +EXTERN Reg zreg; +EXTERN Reg* freer; +EXTERN Var var[NVAR]; +EXTERN int32* idom; +EXTERN Reg** rpo2r; +EXTERN int32 maxnr; + +extern char* anames[]; +extern Hintab hintab[]; + +/* + * sgen.c + */ +void codgen(Node*, Node*); +void gen(Node*); +void noretval(int); +void usedset(Node*, int); +void xcom(Node*); +int bcomplex(Node*, Node*); +Prog* gtext(Sym*, int32); +vlong argsize(void); + +/* + * cgen.c + */ +void cgen(Node*, Node*); +void reglcgen(Node*, Node*, Node*); +void lcgen(Node*, Node*); +void bcgen(Node*, int); +void boolgen(Node*, int, Node*); +void sugen(Node*, Node*, int32); +void layout(Node*, Node*, int, int, Node*); +void cgenrel(Node*, Node*); + +/* + * txt.c + */ +void ginit(void); +void gclean(void); +void nextpc(void); +void gargs(Node*, Node*, Node*); +void garg1(Node*, Node*, Node*, int, Node**); +Node* nodconst(int32); +Node* nod32const(vlong); +Node* nodfconst(double); +void nodreg(Node*, Node*, int); +void regret(Node*, Node*); +int tmpreg(void); +void regalloc(Node*, Node*, Node*); +void regfree(Node*); +void regialloc(Node*, Node*, Node*); +void regsalloc(Node*, Node*); +void regaalloc1(Node*, Node*); +void regaalloc(Node*, Node*); +void regind(Node*, Node*); +void gprep(Node*, Node*); +void raddr(Node*, Prog*); +void naddr(Node*, Adr*); +void gmovm(Node*, Node*, int); +void gmove(Node*, Node*); +void gmover(Node*, Node*); +void gins(int a, Node*, Node*); +void gopcode(int, Node*, Node*, Node*); +int samaddr(Node*, Node*); +void gbranch(int); +void patch(Prog*, int32); +int sconst(Node*); +int sval(int32); +void gpseudo(int, Sym*, Node*); + +/* + * swt.c + */ +int swcmp(const void*, const void*); +void doswit(Node*); +void swit1(C1*, int, int32, Node*); +void cas(void); +void bitload(Node*, Node*, Node*, Node*, Node*); +void bitstore(Node*, Node*, Node*, Node*, Node*); +int mulcon(Node*, Node*); +Multab* mulcon0(int32); +void nullwarn(Node*, Node*); +void outcode(void); +void ieeedtod(Ieee*, double); + +/* + * list + */ +void listinit(void); +int Pconv(Fmt*); +int Aconv(Fmt*); +int Dconv(Fmt*); +int Sconv(Fmt*); +int Nconv(Fmt*); +int Bconv(Fmt*); +int Rconv(Fmt*); + +/* + * reg.c + */ +Reg* rega(void); +int rcmp(const void*, const void*); +void regopt(Prog*); +void addmove(Reg*, int, int, int); +Bits mkvar(Adr*, int); +void prop(Reg*, Bits, Bits); +void loopit(Reg*, int32); +void synch(Reg*, Bits); +uint32 allreg(uint32, Rgn*); +void paint1(Reg*, int); +uint32 paint2(Reg*, int); +void paint3(Reg*, int, int32, int); +void addreg(Adr*, int); + +/* + * peep.c + */ +void peep(void); +void excise(Reg*); +Reg* uniqp(Reg*); +Reg* uniqs(Reg*); +int regtyp(Adr*); +int regzer(Adr*); +int anyvar(Adr*); +int subprop(Reg*); +int copyprop(Reg*); +int shiftprop(Reg*); +void constprop(Adr*, Adr*, Reg*); +int copy1(Adr*, Adr*, Reg*, int); +int copyu(Prog*, Adr*, Adr*); + +int copyas(Adr*, Adr*); +int copyau(Adr*, Adr*); +int copyau1(Prog*, Adr*); +int copysub(Adr*, Adr*, Adr*, int); +int copysub1(Prog*, Adr*, Adr*, int); + +int32 RtoB(int); +int32 FtoB(int); +int BtoR(int32); +int BtoF(int32); + +void predicate(void); +int isbranch(Prog *); +int predicable(Prog *p); +int modifiescpsr(Prog *p); + +#pragma varargck type "A" int +#pragma varargck type "B" Bits +#pragma varargck type "D" Adr* +#pragma varargck type "N" Adr* +#pragma varargck type "R" Adr* +#pragma varargck type "P" Prog* +#pragma varargck type "S" char* diff --git a/src/cmd/5c/list.c b/src/cmd/5c/list.c new file mode 100644 index 000000000..ab0fae83c --- /dev/null +++ b/src/cmd/5c/list.c @@ -0,0 +1,340 @@ +// Inferno utils/5c/list.c +// http://code.google.com/p/inferno-os/source/browse/utils/5c/list.c +// +// Copyright © 1994-1999 Lucent Technologies Inc. All rights reserved. +// Portions Copyright © 1995-1997 C H Forsyth (forsyth@terzarima.net) +// Portions Copyright © 1997-1999 Vita Nuova Limited +// Portions Copyright © 2000-2007 Vita Nuova Holdings Limited (www.vitanuova.com) +// Portions Copyright © 2004,2006 Bruce Ellis +// Portions Copyright © 2005-2007 C H Forsyth (forsyth@terzarima.net) +// Revisions Copyright © 2000-2007 Lucent Technologies Inc. and others +// Portions Copyright © 2009 The Go Authors. All rights reserved. +// +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to deal +// in the Software without restriction, including without limitation the rights +// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +// copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +// THE SOFTWARE. + + +#define EXTERN +#include "gc.h" + +void +listinit(void) +{ + + fmtinstall('A', Aconv); + fmtinstall('P', Pconv); + fmtinstall('S', Sconv); + fmtinstall('N', Nconv); + fmtinstall('B', Bconv); + fmtinstall('D', Dconv); + fmtinstall('R', Rconv); +} + +int +Bconv(Fmt *fp) +{ + char str[STRINGSZ], ss[STRINGSZ], *s; + Bits bits; + int i; + + str[0] = 0; + bits = va_arg(fp->args, Bits); + while(bany(&bits)) { + i = bnum(bits); + if(str[0]) + strcat(str, " "); + if(var[i].sym == S) { + sprint(ss, "$%d", var[i].offset); + s = ss; + } else + s = var[i].sym->name; + if(strlen(str) + strlen(s) + 1 >= STRINGSZ) + break; + strcat(str, s); + bits.b[i/32] &= ~(1L << (i%32)); + } + return fmtstrcpy(fp, str); +} + +char *extra [] = { + ".EQ", ".NE", ".CS", ".CC", + ".MI", ".PL", ".VS", ".VC", + ".HI", ".LS", ".GE", ".LT", + ".GT", ".LE", "", ".NV", +}; + +int +Pconv(Fmt *fp) +{ + char str[STRINGSZ], sc[20]; + Prog *p; + int a, s; + + p = va_arg(fp->args, Prog*); + a = p->as; + s = p->scond; + strcpy(sc, extra[s & C_SCOND]); + if(s & C_SBIT) + strcat(sc, ".S"); + if(s & C_PBIT) + strcat(sc, ".P"); + if(s & C_WBIT) + strcat(sc, ".W"); + if(s & C_UBIT) /* ambiguous with FBIT */ + strcat(sc, ".U"); + if(a == AMOVM) { + if(p->from.type == D_CONST) + sprint(str, " %A%s %R,%D", a, sc, &p->from, &p->to); + else + if(p->to.type == D_CONST) + sprint(str, " %A%s %D,%R", a, sc, &p->from, &p->to); + else + sprint(str, " %A%s %D,%D", a, sc, &p->from, &p->to); + } else + if(a == ADATA) + sprint(str, " %A %D/%d,%D", a, &p->from, p->reg, &p->to); + else + if(p->as == ATEXT) + sprint(str, " %A %D,%d,%D", a, &p->from, p->reg, &p->to); + else + if(p->reg == NREG) + sprint(str, " %A%s %D,%D", a, sc, &p->from, &p->to); + else + if(p->from.type != D_FREG) + sprint(str, " %A%s %D,R%d,%D", a, sc, &p->from, p->reg, &p->to); + else + sprint(str, " %A%s %D,F%d,%D", a, sc, &p->from, p->reg, &p->to); + return fmtstrcpy(fp, str); +} + +int +Aconv(Fmt *fp) +{ + char *s; + int a; + + a = va_arg(fp->args, int); + s = "???"; + if(a >= AXXX && a < ALAST) + s = anames[a]; + return fmtstrcpy(fp, s); +} + +int +Dconv(Fmt *fp) +{ + char str[STRINGSZ]; + Adr *a; + char *op; + int v; + + a = va_arg(fp->args, Adr*); + switch(a->type) { + + default: + sprint(str, "GOK-type(%d)", a->type); + break; + + case D_NONE: + str[0] = 0; + if(a->name != D_NONE || a->reg != NREG || a->sym != S) + sprint(str, "%N(R%d)(NONE)", a, a->reg); + break; + + case D_CONST: + if(a->reg != NREG) + sprint(str, "$%N(R%d)", a, a->reg); + else + sprint(str, "$%N", a); + break; + + case D_CONST2: + sprint(str, "$%d-%d", a->offset, a->offset2); + break; + + case D_SHIFT: + v = a->offset; + op = "<<>>->@>" + (((v>>5) & 3) << 1); + if(v & (1<<4)) + sprint(str, "R%d%c%cR%d", v&15, op[0], op[1], (v>>8)&15); + else + sprint(str, "R%d%c%c%d", v&15, op[0], op[1], (v>>7)&31); + if(a->reg != NREG) + sprint(str+strlen(str), "(R%d)", a->reg); + break; + + case D_OREG: + if(a->reg != NREG) + sprint(str, "%N(R%d)", a, a->reg); + else + sprint(str, "%N", a); + break; + + case D_REG: + sprint(str, "R%d", a->reg); + if(a->name != D_NONE || a->sym != S) + sprint(str, "%N(R%d)(REG)", a, a->reg); + break; + + case D_FREG: + sprint(str, "F%d", a->reg); + if(a->name != D_NONE || a->sym != S) + sprint(str, "%N(R%d)(REG)", a, a->reg); + break; + + case D_PSR: + sprint(str, "PSR"); + if(a->name != D_NONE || a->sym != S) + sprint(str, "%N(PSR)(REG)", a); + break; + + case D_BRANCH: + sprint(str, "%d(PC)", a->offset-pc); + break; + + case D_FCONST: + sprint(str, "$%.17e", a->dval); + break; + + case D_SCONST: + sprint(str, "$\"%S\"", a->sval); + break; + } + return fmtstrcpy(fp, str); +} + +int +Rconv(Fmt *fp) +{ + char str[STRINGSZ]; + Adr *a; + int i, v; + + a = va_arg(fp->args, Adr*); + sprint(str, "GOK-reglist"); + switch(a->type) { + case D_CONST: + case D_CONST2: + if(a->reg != NREG) + break; + if(a->sym != S) + break; + v = a->offset; + strcpy(str, ""); + for(i=0; iargs, char*); + p = str; + for(i=0; i= 'a' && c <= 'z' || + c >= 'A' && c <= 'Z' || + c >= '0' && c <= '9' || + c == ' ' || c == '%') { + *p++ = c; + continue; + } + *p++ = '\\'; + switch(c) { + case 0: + *p++ = 'z'; + continue; + case '\\': + case '"': + *p++ = c; + continue; + case '\n': + *p++ = 'n'; + continue; + case '\t': + *p++ = 't'; + continue; + case '\r': + *p++ = 'r'; + continue; + case '\f': + *p++ = 'f'; + continue; + } + *p++ = (c>>6) + '0'; + *p++ = ((c>>3) & 7) + '0'; + *p++ = (c & 7) + '0'; + } + *p = 0; + return fmtstrcpy(fp, str); +} + +int +Nconv(Fmt *fp) +{ + char str[STRINGSZ]; + Adr *a; + Sym *s; + + a = va_arg(fp->args, Adr*); + s = a->sym; + if(s == S) { + sprint(str, "%d", a->offset); + goto out; + } + switch(a->name) { + default: + sprint(str, "GOK-name(%d)", a->name); + break; + + case D_NONE: + sprint(str, "%d", a->offset); + break; + + case D_EXTERN: + sprint(str, "%s+%d(SB)", s->name, a->offset); + break; + + case D_STATIC: + sprint(str, "%s<>+%d(SB)", s->name, a->offset); + break; + + case D_AUTO: + sprint(str, "%s-%d(SP)", s->name, -a->offset); + break; + + case D_PARAM: + sprint(str, "%s+%d(FP)", s->name, a->offset); + break; + } +out: + return fmtstrcpy(fp, str); +} diff --git a/src/cmd/5c/mul.c b/src/cmd/5c/mul.c new file mode 100644 index 000000000..ff50c4845 --- /dev/null +++ b/src/cmd/5c/mul.c @@ -0,0 +1,640 @@ +// Inferno utils/5c/mul.c +// http://code.google.com/p/inferno-os/source/browse/utils/5c/mul.c +// +// Copyright © 1994-1999 Lucent Technologies Inc. All rights reserved. +// Portions Copyright © 1995-1997 C H Forsyth (forsyth@terzarima.net) +// Portions Copyright © 1997-1999 Vita Nuova Limited +// Portions Copyright © 2000-2007 Vita Nuova Holdings Limited (www.vitanuova.com) +// Portions Copyright © 2004,2006 Bruce Ellis +// Portions Copyright © 2005-2007 C H Forsyth (forsyth@terzarima.net) +// Revisions Copyright © 2000-2007 Lucent Technologies Inc. and others +// Portions Copyright © 2009 The Go Authors. All rights reserved. +// +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to deal +// in the Software without restriction, including without limitation the rights +// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +// copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +// THE SOFTWARE. + + +#include "gc.h" + +/* + * code sequences for multiply by constant. + * [a-l][0-3] + * lsl $(A-'a'),r0,r1 + * [+][0-7] + * add r0,r1,r2 + * [-][0-7] + * sub r0,r1,r2 + */ + +static int maxmulops = 3; /* max # of ops to replace mul with */ +static int multabp; +static int32 mulval; +static char* mulcp; +static int32 valmax; +static int shmax; + +static int docode(char *hp, char *cp, int r0, int r1); +static int gen1(int len); +static int gen2(int len, int32 r1); +static int gen3(int len, int32 r0, int32 r1, int flag); +enum +{ + SR1 = 1<<0, /* r1 has been shifted */ + SR0 = 1<<1, /* r0 has been shifted */ + UR1 = 1<<2, /* r1 has not been used */ + UR0 = 1<<3, /* r0 has not been used */ +}; + +Multab* +mulcon0(int32 v) +{ + int a1, a2, g; + Multab *m, *m1; + char hint[10]; + + if(v < 0) + v = -v; + + /* + * look in cache + */ + m = multab; + for(g=0; gval == v) { + if(m->code[0] == 0) + return 0; + return m; + } + m++; + } + + /* + * select a spot in cache to overwrite + */ + multabp++; + if(multabp < 0 || multabp >= nelem(multab)) + multabp = 0; + m = multab+multabp; + m->val = v; + mulval = v; + + /* + * look in execption hint table + */ + a1 = 0; + a2 = hintabsize; + for(;;) { + if(a1 >= a2) + goto no; + g = (a2 + a1)/2; + if(v < hintab[g].val) { + a2 = g; + continue; + } + if(v > hintab[g].val) { + a1 = g+1; + continue; + } + break; + } + + if(docode(hintab[g].hint, m->code, 1, 0)) + return m; + print("multiply table failure %d\n", v); + m->code[0] = 0; + return 0; + +no: + /* + * try to search + */ + hint[0] = 0; + for(g=1; g<=maxmulops; g++) { + if(g >= maxmulops && v >= 65535) + break; + mulcp = hint+g; + *mulcp = 0; + if(gen1(g)) { + if(docode(hint, m->code, 1, 0)) + return m; + print("multiply table failure %d\n", v); + break; + } + } + + /* + * try a recur followed by a shift + */ + g = 0; + while(!(v & 1)) { + g++; + v >>= 1; + } + if(g) { + m1 = mulcon0(v); + if(m1) { + strcpy(m->code, m1->code); + sprint(strchr(m->code, 0), "%c0", g+'a'); + return m; + } + } + m->code[0] = 0; + return 0; +} + +static int +docode(char *hp, char *cp, int r0, int r1) +{ + int c, i; + + c = *hp++; + *cp = c; + cp += 2; + switch(c) { + default: + c -= 'a'; + if(c < 1 || c >= 30) + break; + for(i=0; i<4; i++) { + switch(i) { + case 0: + if(docode(hp, cp, r0<= mulval) + break; + } + if(mulval == 1) + return 1; + + len--; + for(i=1; i<=shmax; i++) + if(gen2(len, 1<= r1 || + r1 > valmax) + return 0; + + len--; + if(len == 0) + goto calcr0; + + if(!(flag & UR1)) { + f1 = UR1|SR1; + for(i=1; i<=shmax; i++) { + x = r0< valmax) + break; + if(gen3(len, r0, x, f1)) { + i += 'a'; + goto out; + } + } + } + + if(!(flag & UR0)) { + f1 = UR1|SR1; + for(i=1; i<=shmax; i++) { + x = r1< valmax) + break; + if(gen3(len, r1, x, f1)) { + i += 'a'; + goto out; + } + } + } + + if(!(flag & SR1)) { + f1 = UR1|SR1|(flag&UR0); + for(i=1; i<=shmax; i++) { + x = r1< valmax) + break; + if(gen3(len, r0, x, f1)) { + i += 'a'; + goto out; + } + } + } + + if(!(flag & SR0)) { + f1 = UR0|SR0|(flag&(SR1|UR1)); + + f2 = UR1|SR1; + if(flag & UR1) + f2 |= UR0; + if(flag & SR1) + f2 |= SR0; + + for(i=1; i<=shmax; i++) { + x = r0< valmax) + break; + if(x > r1) { + if(gen3(len, r1, x, f2)) { + i += 'a'; + goto out; + } + } else + if(gen3(len, x, r1, f1)) { + i += 'a'; + goto out; + } + } + } + + x = r1+r0; + if(gen3(len, r0, x, UR1)) { + i = '+'; + goto out; + } + + if(gen3(len, r1, x, UR1)) { + i = '+'; + goto out; + } + + x = r1-r0; + if(gen3(len, x, r1, UR0)) { + i = '-'; + goto out; + } + + if(x > r0) { + if(gen3(len, r0, x, UR1)) { + i = '-'; + goto out; + } + } else + if(gen3(len, x, r0, UR0)) { + i = '-'; + goto out; + } + + return 0; + +calcr0: + f1 = flag & (UR0|UR1); + if(f1 == UR1) { + for(i=1; i<=shmax; i++) { + x = r1<= mulval) { + if(x == mulval) { + i += 'a'; + goto out; + } + break; + } + } + } + + if(mulval == r1+r0) { + i = '+'; + goto out; + } + if(mulval == r1-r0) { + i = '-'; + goto out; + } + + return 0; + +out: + *--mulcp = i; + return 1; +} + +/* + * hint table has numbers that + * the search algorithm fails on. + * <1000: + * all numbers + * <5000: + * ÷ by 5 + * <10000: + * ÷ by 50 + * <65536: + * ÷ by 250 + */ +Hintab hintab[] = +{ + 683, "b++d+e+", + 687, "b+e++e-", + 691, "b++d+e+", + 731, "b++d+e+", + 811, "b++d+i+", + 821, "b++e+e+", + 843, "b+d++e+", + 851, "b+f-+e-", + 853, "b++e+e+", + 877, "c++++g-", + 933, "b+c++g-", + 981, "c-+e-d+", + 1375, "b+c+b+h-", + 1675, "d+b++h+", + 2425, "c++f-e+", + 2675, "c+d++f-", + 2750, "b+d-b+h-", + 2775, "c-+g-e-", + 3125, "b++e+g+", + 3275, "b+c+g+e+", + 3350, "c++++i+", + 3475, "c-+e-f-", + 3525, "c-+d+g-", + 3625, "c-+e-j+", + 3675, "b+d+d+e+", + 3725, "b+d-+h+", + 3925, "b+d+f-d-", + 4275, "b+g++e+", + 4325, "b+h-+d+", + 4425, "b+b+g-j-", + 4525, "b+d-d+f+", + 4675, "c++d-g+", + 4775, "b+d+b+g-", + 4825, "c+c-+i-", + 4850, "c++++i-", + 4925, "b++e-g-", + 4975, "c+f++e-", + 5500, "b+g-c+d+", + 6700, "d+b++i+", + 9700, "d++++j-", + 11000, "b+f-c-h-", + 11750, "b+d+g+j-", + 12500, "b+c+e-k+", + 13250, "b+d+e-f+", + 13750, "b+h-c-d+", + 14250, "b+g-c+e-", + 14500, "c+f+j-d-", + 14750, "d-g--f+", + 16750, "b+e-d-n+", + 17750, "c+h-b+e+", + 18250, "d+b+h-d+", + 18750, "b+g-++f+", + 19250, "b+e+b+h+", + 19750, "b++h--f-", + 20250, "b+e-l-c+", + 20750, "c++bi+e-", + 21250, "b+i+l+c+", + 22000, "b+e+d-g-", + 22250, "b+d-h+k-", + 22750, "b+d-e-g+", + 23250, "b+c+h+e-", + 23500, "b+g-c-g-", + 23750, "b+g-b+h-", + 24250, "c++g+m-", + 24750, "b+e+e+j-", + 25000, "b++dh+g+", + 25250, "b+e+d-g-", + 25750, "b+e+b+j+", + 26250, "b+h+c+e+", + 26500, "b+h+c+g+", + 26750, "b+d+e+g-", + 27250, "b+e+e+f+", + 27500, "c-i-c-d+", + 27750, "b+bd++j+", + 28250, "d-d-++i-", + 28500, "c+c-h-e-", + 29000, "b+g-d-f+", + 29500, "c+h+++e-", + 29750, "b+g+f-c+", + 30250, "b+f-g-c+", + 33500, "c-f-d-n+", + 33750, "b+d-b+j-", + 34250, "c+e+++i+", + 35250, "e+b+d+k+", + 35500, "c+e+d-g-", + 35750, "c+i-++e+", + 36250, "b+bh-d+e+", + 36500, "c+c-h-e-", + 36750, "d+e--i+", + 37250, "b+g+g+b+", + 37500, "b+h-b+f+", + 37750, "c+be++j-", + 38500, "b+e+b+i+", + 38750, "d+i-b+d+", + 39250, "b+g-l-+d+", + 39500, "b+g-c+g-", + 39750, "b+bh-c+f-", + 40250, "b+bf+d+g-", + 40500, "b+g-c+g+", + 40750, "c+b+i-e+", + 41250, "d++bf+h+", + 41500, "b+j+c+d-", + 41750, "c+f+b+h-", + 42500, "c+h++g+", + 42750, "b+g+d-f-", + 43250, "b+l-e+d-", + 43750, "c+bd+h+f-", + 44000, "b+f+g-d-", + 44250, "b+d-g--f+", + 44500, "c+e+c+h+", + 44750, "b+e+d-h-", + 45250, "b++g+j-g+", + 45500, "c+d+e-g+", + 45750, "b+d-h-e-", + 46250, "c+bd++j+", + 46500, "b+d-c-j-", + 46750, "e-e-b+g-", + 47000, "b+c+d-j-", + 47250, "b+e+e-g-", + 47500, "b+g-c-h-", + 47750, "b+f-c+h-", + 48250, "d--h+n-", + 48500, "b+c-g+m-", + 48750, "b+e+e-g+", + 49500, "c-f+e+j-", + 49750, "c+c+g++f-", + 50000, "b+e+e+k+", + 50250, "b++i++g+", + 50500, "c+g+f-i+", + 50750, "b+e+d+k-", + 51500, "b+i+c-f+", + 51750, "b+bd+g-e-", + 52250, "b+d+g-j+", + 52500, "c+c+f+g+", + 52750, "b+c+e+i+", + 53000, "b+i+c+g+", + 53500, "c+g+g-n+", + 53750, "b+j+d-c+", + 54250, "b+d-g-j-", + 54500, "c-f+e+f+", + 54750, "b+f-+c+g+", + 55000, "b+g-d-g-", + 55250, "b+e+e+g+", + 55500, "b+cd++j+", + 55750, "b+bh-d-f-", + 56250, "c+d-b+j-", + 56500, "c+d+c+i+", + 56750, "b+e+d++h-", + 57000, "b+d+g-f+", + 57250, "b+f-m+d-", + 57750, "b+i+c+e-", + 58000, "b+e+d+h+", + 58250, "c+b+g+g+", + 58750, "d-e-j--e+", + 59000, "d-i-+e+", + 59250, "e--h-m+", + 59500, "c+c-h+f-", + 59750, "b+bh-e+i-", + 60250, "b+bh-e-e-", + 60500, "c+c-g-g-", + 60750, "b+e-l-e-", + 61250, "b+g-g-c+", + 61750, "b+g-c+g+", + 62250, "f--+c-i-", + 62750, "e+f--+g+", + 64750, "b+f+d+p-", +}; +int hintabsize = nelem(hintab); diff --git a/src/cmd/5c/peep.c b/src/cmd/5c/peep.c new file mode 100644 index 000000000..3a905f099 --- /dev/null +++ b/src/cmd/5c/peep.c @@ -0,0 +1,1468 @@ +// Inferno utils/5c/peep.c +// http://code.google.com/p/inferno-os/source/browse/utils/5c/peep.c +// +// Copyright © 1994-1999 Lucent Technologies Inc. All rights reserved. +// Portions Copyright © 1995-1997 C H Forsyth (forsyth@terzarima.net) +// Portions Copyright © 1997-1999 Vita Nuova Limited +// Portions Copyright © 2000-2007 Vita Nuova Holdings Limited (www.vitanuova.com) +// Portions Copyright © 2004,2006 Bruce Ellis +// Portions Copyright © 2005-2007 C H Forsyth (forsyth@terzarima.net) +// Revisions Copyright © 2000-2007 Lucent Technologies Inc. and others +// Portions Copyright © 2009 The Go Authors. All rights reserved. +// +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to deal +// in the Software without restriction, including without limitation the rights +// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +// copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +// THE SOFTWARE. + + +#include "gc.h" + +int xtramodes(Reg*, Adr*); + +void +peep(void) +{ + Reg *r, *r1, *r2; + Prog *p, *p1; + int t; +/* + * complete R structure + */ + t = 0; + for(r=firstr; r!=R; r=r1) { + r1 = r->link; + if(r1 == R) + break; + p = r->prog->link; + while(p != r1->prog) + switch(p->as) { + default: + r2 = rega(); + r->link = r2; + r2->link = r1; + + r2->prog = p; + r2->p1 = r; + r->s1 = r2; + r2->s1 = r1; + r1->p1 = r2; + + r = r2; + t++; + + case ADATA: + case AGLOBL: + case ANAME: + case ASIGNAME: + p = p->link; + } + } + +loop1: + t = 0; + for(r=firstr; r!=R; r=r->link) { + p = r->prog; + if(p->as == ASLL || p->as == ASRL || p->as == ASRA) { + /* + * elide shift into D_SHIFT operand of subsequent instruction + */ + if(shiftprop(r)) { + excise(r); + t++; + } + } + if(p->as == AMOVW || p->as == AMOVF || p->as == AMOVD) + if(regtyp(&p->to)) { + if(p->from.type == D_CONST) + constprop(&p->from, &p->to, r->s1); + else if(regtyp(&p->from)) + if(p->from.type == p->to.type) { + if(copyprop(r)) { + excise(r); + t++; + } else + if(subprop(r) && copyprop(r)) { + excise(r); + t++; + } + } + } + } + if(t) + goto loop1; + /* + * look for MOVB x,R; MOVB R,R + */ + for(r=firstr; r!=R; r=r->link) { + p = r->prog; + switch(p->as) { + default: + continue; + case AEOR: + /* + * EOR -1,x,y => MVN x,y + */ + if(p->from.type == D_CONST && p->from.offset == -1) { + p->as = AMVN; + p->from.type = D_REG; + if(p->reg != NREG) + p->from.reg = p->reg; + else + p->from.reg = p->to.reg; + p->reg = NREG; + } + continue; + case AMOVH: + case AMOVHU: + case AMOVB: + case AMOVBU: + if(p->to.type != D_REG) + continue; + break; + } + r1 = r->link; + if(r1 == R) + continue; + p1 = r1->prog; + if(p1->as != p->as) + continue; + if(p1->from.type != D_REG || p1->from.reg != p->to.reg) + continue; + if(p1->to.type != D_REG || p1->to.reg != p->to.reg) + continue; + excise(r1); + } + + for(r=firstr; r!=R; r=r->link) { + p = r->prog; + switch(p->as) { + case AMOVW: + case AMOVB: + case AMOVBU: + if(p->from.type == D_OREG && p->from.offset == 0) + xtramodes(r, &p->from); + else if(p->to.type == D_OREG && p->to.offset == 0) + xtramodes(r, &p->to); + else + continue; + break; + case ACMP: + /* + * elide CMP $0,x if calculation of x can set condition codes + */ + if(p->from.type != D_CONST || p->from.offset != 0) + continue; + r2 = r->s1; + if(r2 == R) + continue; + t = r2->prog->as; + switch(t) { + default: + continue; + case ABEQ: + case ABNE: + case ABMI: + case ABPL: + break; + case ABGE: + t = ABPL; + break; + case ABLT: + t = ABMI; + break; + case ABHI: + t = ABNE; + break; + case ABLS: + t = ABEQ; + break; + } + r1 = r; + do + r1 = uniqp(r1); + while (r1 != R && r1->prog->as == ANOP); + if(r1 == R) + continue; + p1 = r1->prog; + if(p1->to.type != D_REG) + continue; + if(p1->to.reg != p->reg) + if(!(p1->as == AMOVW && p1->from.type == D_REG && p1->from.reg == p->reg)) + continue; + switch(p1->as) { + default: + continue; + case AMOVW: + if(p1->from.type != D_REG) + continue; + case AAND: + case AEOR: + case AORR: + case ABIC: + case AMVN: + case ASUB: + case ARSB: + case AADD: + case AADC: + case ASBC: + case ARSC: + break; + } + p1->scond |= C_SBIT; + r2->prog->as = t; + excise(r); + continue; + } + } + + predicate(); +} + +void +excise(Reg *r) +{ + Prog *p; + + p = r->prog; + p->as = ANOP; + p->scond = zprog.scond; + p->from = zprog.from; + p->to = zprog.to; + p->reg = zprog.reg; /**/ +} + +Reg* +uniqp(Reg *r) +{ + Reg *r1; + + r1 = r->p1; + if(r1 == R) { + r1 = r->p2; + if(r1 == R || r1->p2link != R) + return R; + } else + if(r->p2 != R) + return R; + return r1; +} + +Reg* +uniqs(Reg *r) +{ + Reg *r1; + + r1 = r->s1; + if(r1 == R) { + r1 = r->s2; + if(r1 == R) + return R; + } else + if(r->s2 != R) + return R; + return r1; +} + +int +regtyp(Adr *a) +{ + + if(a->type == D_REG) + return 1; + if(a->type == D_FREG) + return 1; + return 0; +} + +/* + * the idea is to substitute + * one register for another + * from one MOV to another + * MOV a, R0 + * ADD b, R0 / no use of R1 + * MOV R0, R1 + * would be converted to + * MOV a, R1 + * ADD b, R1 + * MOV R1, R0 + * hopefully, then the former or latter MOV + * will be eliminated by copy propagation. + */ +int +subprop(Reg *r0) +{ + Prog *p; + Adr *v1, *v2; + Reg *r; + int t; + + p = r0->prog; + v1 = &p->from; + if(!regtyp(v1)) + return 0; + v2 = &p->to; + if(!regtyp(v2)) + return 0; + for(r=uniqp(r0); r!=R; r=uniqp(r)) { + if(uniqs(r) == R) + break; + p = r->prog; + switch(p->as) { + case ABL: + return 0; + + case ACMP: + case ACMN: + case AADD: + case ASUB: + case ARSB: + case ASLL: + case ASRL: + case ASRA: + case AORR: + case AAND: + case AEOR: + case AMUL: + case ADIV: + case ADIVU: + + case ACMPF: + case ACMPD: + case AADDD: + case AADDF: + case ASUBD: + case ASUBF: + case AMULD: + case AMULF: + case ADIVD: + case ADIVF: + if(p->to.type == v1->type) + if(p->to.reg == v1->reg) { + if(p->reg == NREG) + p->reg = p->to.reg; + goto gotit; + } + break; + + case AMOVF: + case AMOVD: + case AMOVW: + if(p->to.type == v1->type) + if(p->to.reg == v1->reg) + goto gotit; + break; + + case AMOVM: + t = 1<reg; + if((p->from.type == D_CONST && (p->from.offset&t)) || + (p->to.type == D_CONST && (p->to.offset&t))) + return 0; + break; + } + if(copyau(&p->from, v2) || + copyau1(p, v2) || + copyau(&p->to, v2)) + break; + if(copysub(&p->from, v1, v2, 0) || + copysub1(p, v1, v2, 0) || + copysub(&p->to, v1, v2, 0)) + break; + } + return 0; + +gotit: + copysub(&p->to, v1, v2, 1); + if(debug['P']) { + print("gotit: %D->%D\n%P", v1, v2, r->prog); + if(p->from.type == v2->type) + print(" excise"); + print("\n"); + } + for(r=uniqs(r); r!=r0; r=uniqs(r)) { + p = r->prog; + copysub(&p->from, v1, v2, 1); + copysub1(p, v1, v2, 1); + copysub(&p->to, v1, v2, 1); + if(debug['P']) + print("%P\n", r->prog); + } + t = v1->reg; + v1->reg = v2->reg; + v2->reg = t; + if(debug['P']) + print("%P last\n", r->prog); + return 1; +} + +/* + * The idea is to remove redundant copies. + * v1->v2 F=0 + * (use v2 s/v2/v1/)* + * set v1 F=1 + * use v2 return fail + * ----------------- + * v1->v2 F=0 + * (use v2 s/v2/v1/)* + * set v1 F=1 + * set v2 return success + */ +int +copyprop(Reg *r0) +{ + Prog *p; + Adr *v1, *v2; + Reg *r; + + p = r0->prog; + v1 = &p->from; + v2 = &p->to; + if(copyas(v1, v2)) + return 1; + for(r=firstr; r!=R; r=r->link) + r->active = 0; + return copy1(v1, v2, r0->s1, 0); +} + +int +copy1(Adr *v1, Adr *v2, Reg *r, int f) +{ + int t; + Prog *p; + + if(r->active) { + if(debug['P']) + print("act set; return 1\n"); + return 1; + } + r->active = 1; + if(debug['P']) + print("copy %D->%D f=%d\n", v1, v2, f); + for(; r != R; r = r->s1) { + p = r->prog; + if(debug['P']) + print("%P", p); + if(!f && uniqp(r) == R) { + f = 1; + if(debug['P']) + print("; merge; f=%d", f); + } + t = copyu(p, v2, A); + switch(t) { + case 2: /* rar, cant split */ + if(debug['P']) + print("; %Drar; return 0\n", v2); + return 0; + + case 3: /* set */ + if(debug['P']) + print("; %Dset; return 1\n", v2); + return 1; + + case 1: /* used, substitute */ + case 4: /* use and set */ + if(f) { + if(!debug['P']) + return 0; + if(t == 4) + print("; %Dused+set and f=%d; return 0\n", v2, f); + else + print("; %Dused and f=%d; return 0\n", v2, f); + return 0; + } + if(copyu(p, v2, v1)) { + if(debug['P']) + print("; sub fail; return 0\n"); + return 0; + } + if(debug['P']) + print("; sub%D/%D", v2, v1); + if(t == 4) { + if(debug['P']) + print("; %Dused+set; return 1\n", v2); + return 1; + } + break; + } + if(!f) { + t = copyu(p, v1, A); + if(!f && (t == 2 || t == 3 || t == 4)) { + f = 1; + if(debug['P']) + print("; %Dset and !f; f=%d", v1, f); + } + } + if(debug['P']) + print("\n"); + if(r->s2) + if(!copy1(v1, v2, r->s2, f)) + return 0; + } + return 1; +} + +/* + * The idea is to remove redundant constants. + * $c1->v1 + * ($c1->v2 s/$c1/v1)* + * set v1 return + * The v1->v2 should be eliminated by copy propagation. + */ +void +constprop(Adr *c1, Adr *v1, Reg *r) +{ + Prog *p; + + if(debug['C']) + print("constprop %D->%D\n", c1, v1); + for(; r != R; r = r->s1) { + p = r->prog; + if(debug['C']) + print("%P", p); + if(uniqp(r) == R) { + if(debug['C']) + print("; merge; return\n"); + return; + } + if(p->as == AMOVW && copyas(&p->from, c1)) { + if(debug['C']) + print("; sub%D/%D", &p->from, v1); + p->from = *v1; + } else if(copyu(p, v1, A) > 1) { + if(debug['C']) + print("; %Dset; return\n", v1); + return; + } + if(debug['C']) + print("\n"); + if(r->s2) + constprop(c1, v1, r->s2); + } +} + +/* + * ASLL x,y,w + * .. (not use w, not set x y w) + * AXXX w,a,b (a != w) + * .. (not use w) + * (set w) + * ----------- changed to + * .. + * AXXX (x<prog; + if(p->to.type != D_REG) + FAIL("BOTCH: result not reg"); + n = p->to.reg; + a = zprog.from; + if(p->reg != NREG && p->reg != p->to.reg) { + a.type = D_REG; + a.reg = p->reg; + } + if(debug['H']) + print("shiftprop\n%P", p); + r1 = r; + for(;;) { + /* find first use of shift result; abort if shift operands or result are changed */ + r1 = uniqs(r1); + if(r1 == R) + FAIL("branch"); + if(uniqp(r1) == R) + FAIL("merge"); + p1 = r1->prog; + if(debug['H']) + print("\n%P", p1); + switch(copyu(p1, &p->to, A)) { + case 0: /* not used or set */ + if((p->from.type == D_REG && copyu(p1, &p->from, A) > 1) || + (a.type == D_REG && copyu(p1, &a, A) > 1)) + FAIL("args modified"); + continue; + case 3: /* set, not used */ + FAIL("BOTCH: noref"); + } + break; + } + /* check whether substitution can be done */ + switch(p1->as) { + default: + FAIL("non-dpi"); + case AAND: + case AEOR: + case AADD: + case AADC: + case AORR: + case ASUB: + case ARSB: + case ASBC: + case ARSC: + if(p1->reg == n || (p1->reg == NREG && p1->to.type == D_REG && p1->to.reg == n)) { + if(p1->from.type != D_REG) + FAIL("can't swap"); + p1->reg = p1->from.reg; + p1->from.reg = n; + switch(p1->as) { + case ASUB: + p1->as = ARSB; + break; + case ARSB: + p1->as = ASUB; + break; + case ASBC: + p1->as = ARSC; + break; + case ARSC: + p1->as = ASBC; + break; + } + if(debug['H']) + print("\t=>%P", p1); + } + case ABIC: + case ACMP: + case ACMN: + if(p1->reg == n) + FAIL("can't swap"); + if(p1->reg == NREG && p1->to.reg == n) + FAIL("shift result used twice"); + case AMVN: + if(p1->from.type == D_SHIFT) + FAIL("shift result used in shift"); + if(p1->from.type != D_REG || p1->from.reg != n) + FAIL("BOTCH: where is it used?"); + break; + } + /* check whether shift result is used subsequently */ + p2 = p1; + if(p1->to.reg != n) + for (;;) { + r1 = uniqs(r1); + if(r1 == R) + FAIL("inconclusive"); + p1 = r1->prog; + if(debug['H']) + print("\n%P", p1); + switch(copyu(p1, &p->to, A)) { + case 0: /* not used or set */ + continue; + case 3: /* set, not used */ + break; + default:/* used */ + FAIL("reused"); + } + break; + } + /* make the substitution */ + p2->from.type = D_SHIFT; + p2->from.reg = NREG; + o = p->reg; + if(o == NREG) + o = p->to.reg; + switch(p->from.type){ + case D_CONST: + o |= (p->from.offset&0x1f)<<7; + break; + case D_REG: + o |= (1<<4) | (p->from.reg<<8); + break; + } + switch(p->as){ + case ASLL: + o |= 0<<5; + break; + case ASRL: + o |= 1<<5; + break; + case ASRA: + o |= 2<<5; + break; + } + p2->from.offset = o; + if(debug['H']) + print("\t=>%P\tSUCCEED\n", p2); + return 1; +} + +Reg* +findpre(Reg *r, Adr *v) +{ + Reg *r1; + + for(r1=uniqp(r); r1!=R; r=r1,r1=uniqp(r)) { + if(uniqs(r1) != r) + return R; + switch(copyu(r1->prog, v, A)) { + case 1: /* used */ + case 2: /* read-alter-rewrite */ + return R; + case 3: /* set */ + case 4: /* set and used */ + return r1; + } + } + return R; +} + +Reg* +findinc(Reg *r, Reg *r2, Adr *v) +{ + Reg *r1; + Prog *p; + + + for(r1=uniqs(r); r1!=R && r1!=r2; r=r1,r1=uniqs(r)) { + if(uniqp(r1) != r) + return R; + switch(copyu(r1->prog, v, A)) { + case 0: /* not touched */ + continue; + case 4: /* set and used */ + p = r1->prog; + if(p->as == AADD) + if(p->from.type == D_CONST) + if(p->from.offset > -4096 && p->from.offset < 4096) + return r1; + default: + return R; + } + } + return R; +} + +int +nochange(Reg *r, Reg *r2, Prog *p) +{ + Adr a[3]; + int i, n; + + if(r == r2) + return 1; + n = 0; + if(p->reg != NREG && p->reg != p->to.reg) { + a[n].type = D_REG; + a[n++].reg = p->reg; + } + switch(p->from.type) { + case D_SHIFT: + a[n].type = D_REG; + a[n++].reg = p->from.offset&0xf; + case D_REG: + a[n].type = D_REG; + a[n++].reg = p->from.reg; + } + if(n == 0) + return 1; + for(; r!=R && r!=r2; r=uniqs(r)) { + p = r->prog; + for(i=0; i 1) + return 0; + } + return 1; +} + +int +findu1(Reg *r, Adr *v) +{ + for(; r != R; r = r->s1) { + if(r->active) + return 0; + r->active = 1; + switch(copyu(r->prog, v, A)) { + case 1: /* used */ + case 2: /* read-alter-rewrite */ + case 4: /* set and used */ + return 1; + case 3: /* set */ + return 0; + } + if(r->s2) + if (findu1(r->s2, v)) + return 1; + } + return 0; +} + +int +finduse(Reg *r, Adr *v) +{ + Reg *r1; + + for(r1=firstr; r1!=R; r1=r1->link) + r1->active = 0; + return findu1(r, v); +} + +int +xtramodes(Reg *r, Adr *a) +{ + Reg *r1, *r2, *r3; + Prog *p, *p1; + Adr v; + + p = r->prog; + if(debug['h'] && p->as == AMOVB && p->from.type == D_OREG) /* byte load */ + return 0; + v = *a; + v.type = D_REG; + r1 = findpre(r, &v); + if(r1 != R) { + p1 = r1->prog; + if(p1->to.type == D_REG && p1->to.reg == v.reg) + switch(p1->as) { + case AADD: + if(p1->from.type == D_REG || + (p1->from.type == D_SHIFT && (p1->from.offset&(1<<4)) == 0 && + (p->as != AMOVB || (a == &p->from && (p1->from.offset&~0xf) == 0))) || + (p1->from.type == D_CONST && + p1->from.offset > -4096 && p1->from.offset < 4096)) + if(nochange(uniqs(r1), r, p1)) { + if(a != &p->from || v.reg != p->to.reg) + if (finduse(r->s1, &v)) { + if(p1->reg == NREG || p1->reg == v.reg) + /* pre-indexing */ + p->scond |= C_WBIT; + else return 0; + } + switch (p1->from.type) { + case D_REG: + /* register offset */ + a->type = D_SHIFT; + a->offset = p1->from.reg; + break; + case D_SHIFT: + /* scaled register offset */ + a->type = D_SHIFT; + case D_CONST: + /* immediate offset */ + a->offset = p1->from.offset; + break; + } + if(p1->reg != NREG) + a->reg = p1->reg; + excise(r1); + return 1; + } + break; + case AMOVW: + if(p1->from.type == D_REG) + if((r2 = findinc(r1, r, &p1->from)) != R) { + for(r3=uniqs(r2); r3->prog->as==ANOP; r3=uniqs(r3)) + ; + if(r3 == r) { + /* post-indexing */ + p1 = r2->prog; + a->reg = p1->to.reg; + a->offset = p1->from.offset; + p->scond |= C_PBIT; + if(!finduse(r, &r1->prog->to)) + excise(r1); + excise(r2); + return 1; + } + } + break; + } + } + if(a != &p->from || a->reg != p->to.reg) + if((r1 = findinc(r, R, &v)) != R) { + /* post-indexing */ + p1 = r1->prog; + a->offset = p1->from.offset; + p->scond |= C_PBIT; + excise(r1); + return 1; + } + return 0; +} + +/* + * return + * 1 if v only used (and substitute), + * 2 if read-alter-rewrite + * 3 if set + * 4 if set and used + * 0 otherwise (not touched) + */ +int +copyu(Prog *p, Adr *v, Adr *s) +{ + + switch(p->as) { + + default: + if(debug['P']) + print(" (?)"); + return 2; + + case AMOVM: + if(v->type != D_REG) + return 0; + if(p->from.type == D_CONST) { /* read reglist, read/rar */ + if(s != A) { + if(p->from.offset&(1<reg)) + return 1; + if(copysub(&p->to, v, s, 1)) + return 1; + return 0; + } + if(copyau(&p->to, v)) { + if(p->scond&C_WBIT) + return 2; + return 1; + } + if(p->from.offset&(1<reg)) + return 1; + } else { /* read/rar, write reglist */ + if(s != A) { + if(p->to.offset&(1<reg)) + return 1; + if(copysub(&p->from, v, s, 1)) + return 1; + return 0; + } + if(copyau(&p->from, v)) { + if(p->scond&C_WBIT) + return 2; + if(p->to.offset&(1<reg)) + return 4; + return 1; + } + if(p->to.offset&(1<reg)) + return 3; + } + return 0; + + case ANOP: /* read, write */ + case AMOVW: + case AMOVF: + case AMOVD: + case AMOVH: + case AMOVHU: + case AMOVB: + case AMOVBU: + case AMOVDW: + case AMOVWD: + case AMOVFD: + case AMOVDF: + if(p->scond&(C_WBIT|C_PBIT)) + if(v->type == D_REG) { + if(p->from.type == D_OREG || p->from.type == D_SHIFT) { + if(p->from.reg == v->reg) + return 2; + } else { + if(p->to.reg == v->reg) + return 2; + } + } + if(s != A) { + if(copysub(&p->from, v, s, 1)) + return 1; + if(!copyas(&p->to, v)) + if(copysub(&p->to, v, s, 1)) + return 1; + return 0; + } + if(copyas(&p->to, v)) { + if(copyau(&p->from, v)) + return 4; + return 3; + } + if(copyau(&p->from, v)) + return 1; + if(copyau(&p->to, v)) + return 1; + return 0; + + + case AADD: /* read, read, write */ + case ASUB: + case ARSB: + case ASLL: + case ASRL: + case ASRA: + case AORR: + case AAND: + case AEOR: + case AMUL: + case ADIV: + case ADIVU: + case AADDF: + case AADDD: + case ASUBF: + case ASUBD: + case AMULF: + case AMULD: + case ADIVF: + case ADIVD: + + case ACMPF: + case ACMPD: + case ACMP: + case ACMN: + case ACASE: + if(s != A) { + if(copysub(&p->from, v, s, 1)) + return 1; + if(copysub1(p, v, s, 1)) + return 1; + if(!copyas(&p->to, v)) + if(copysub(&p->to, v, s, 1)) + return 1; + return 0; + } + if(copyas(&p->to, v)) { + if(p->reg == NREG) + p->reg = p->to.reg; + if(copyau(&p->from, v)) + return 4; + if(copyau1(p, v)) + return 4; + return 3; + } + if(copyau(&p->from, v)) + return 1; + if(copyau1(p, v)) + return 1; + if(copyau(&p->to, v)) + return 1; + return 0; + + case ABEQ: /* read, read */ + case ABNE: + case ABCS: + case ABHS: + case ABCC: + case ABLO: + case ABMI: + case ABPL: + case ABVS: + case ABVC: + case ABHI: + case ABLS: + case ABGE: + case ABLT: + case ABGT: + case ABLE: + if(s != A) { + if(copysub(&p->from, v, s, 1)) + return 1; + return copysub1(p, v, s, 1); + } + if(copyau(&p->from, v)) + return 1; + if(copyau1(p, v)) + return 1; + return 0; + + case AB: /* funny */ + if(s != A) { + if(copysub(&p->to, v, s, 1)) + return 1; + return 0; + } + if(copyau(&p->to, v)) + return 1; + return 0; + + case ARET: /* funny */ + if(v->type == D_REG) + if(v->reg == REGRET) + return 2; + if(v->type == D_FREG) + if(v->reg == FREGRET) + return 2; + + case ABL: /* funny */ + if(v->type == D_REG) { + if(v->reg <= REGEXT && v->reg > exregoffset) + return 2; + if(v->reg == (uchar)REGARG) + return 2; + } + if(v->type == D_FREG) + if(v->reg <= FREGEXT && v->reg > exfregoffset) + return 2; + + if(s != A) { + if(copysub(&p->to, v, s, 1)) + return 1; + return 0; + } + if(copyau(&p->to, v)) + return 4; + return 3; + + case ATEXT: /* funny */ + if(v->type == D_REG) + if(v->reg == (uchar)REGARG) + return 3; + return 0; + } +} + +int +a2type(Prog *p) +{ + + switch(p->as) { + + case ACMP: + case ACMN: + + case AADD: + case ASUB: + case ARSB: + case ASLL: + case ASRL: + case ASRA: + case AORR: + case AAND: + case AEOR: + case AMUL: + case ADIV: + case ADIVU: + return D_REG; + + case ACMPF: + case ACMPD: + + case AADDF: + case AADDD: + case ASUBF: + case ASUBD: + case AMULF: + case AMULD: + case ADIVF: + case ADIVD: + return D_FREG; + } + return D_NONE; +} + +/* + * direct reference, + * could be set/use depending on + * semantics + */ +int +copyas(Adr *a, Adr *v) +{ + + if(regtyp(v)) { + if(a->type == v->type) + if(a->reg == v->reg) + return 1; + } else if(v->type == D_CONST) { /* for constprop */ + if(a->type == v->type) + if(a->name == v->name) + if(a->sym == v->sym) + if(a->reg == v->reg) + if(a->offset == v->offset) + return 1; + } + return 0; +} + +/* + * either direct or indirect + */ +int +copyau(Adr *a, Adr *v) +{ + + if(copyas(a, v)) + return 1; + if(v->type == D_REG) { + if(a->type == D_OREG) { + if(v->reg == a->reg) + return 1; + } else if(a->type == D_SHIFT) { + if((a->offset&0xf) == v->reg) + return 1; + if((a->offset&(1<<4)) && (a->offset>>8) == v->reg) + return 1; + } + } + return 0; +} + +int +copyau1(Prog *p, Adr *v) +{ + + if(regtyp(v)) { + if(a2type(p) == v->type) + if(p->reg == v->reg) { + if(a2type(p) != v->type) + print("botch a2type %P\n", p); + return 1; + } + } + return 0; +} + +/* + * substitute s for v in a + * return failure to substitute + */ +int +copysub(Adr *a, Adr *v, Adr *s, int f) +{ + + if(f) + if(copyau(a, v)) { + if(a->type == D_SHIFT) { + if((a->offset&0xf) == v->reg) + a->offset = (a->offset&~0xf)|s->reg; + if((a->offset&(1<<4)) && (a->offset>>8) == v->reg) + a->offset = (a->offset&~(0xf<<8))|(s->reg<<8); + } else + a->reg = s->reg; + } + return 0; +} + +int +copysub1(Prog *p1, Adr *v, Adr *s, int f) +{ + + if(f) + if(copyau1(p1, v)) + p1->reg = s->reg; + return 0; +} + +struct { + int opcode; + int notopcode; + int scond; + int notscond; +} predinfo[] = { + { ABEQ, ABNE, 0x0, 0x1, }, + { ABNE, ABEQ, 0x1, 0x0, }, + { ABCS, ABCC, 0x2, 0x3, }, + { ABHS, ABLO, 0x2, 0x3, }, + { ABCC, ABCS, 0x3, 0x2, }, + { ABLO, ABHS, 0x3, 0x2, }, + { ABMI, ABPL, 0x4, 0x5, }, + { ABPL, ABMI, 0x5, 0x4, }, + { ABVS, ABVC, 0x6, 0x7, }, + { ABVC, ABVS, 0x7, 0x6, }, + { ABHI, ABLS, 0x8, 0x9, }, + { ABLS, ABHI, 0x9, 0x8, }, + { ABGE, ABLT, 0xA, 0xB, }, + { ABLT, ABGE, 0xB, 0xA, }, + { ABGT, ABLE, 0xC, 0xD, }, + { ABLE, ABGT, 0xD, 0xC, }, +}; + +typedef struct { + Reg *start; + Reg *last; + Reg *end; + int len; +} Joininfo; + +enum { + Join, + Split, + End, + Branch, + Setcond, + Toolong +}; + +enum { + Falsecond, + Truecond, + Delbranch, + Keepbranch +}; + +int +isbranch(Prog *p) +{ + return (ABEQ <= p->as) && (p->as <= ABLE); +} + +int +predicable(Prog *p) +{ + if (isbranch(p) + || p->as == ANOP + || p->as == AXXX + || p->as == ADATA + || p->as == AGLOBL + || p->as == AGOK + || p->as == AHISTORY + || p->as == ANAME + || p->as == ASIGNAME + || p->as == ATEXT + || p->as == AWORD + || p->as == ABCASE + || p->as == ACASE) + return 0; + return 1; +} + +/* + * Depends on an analysis of the encodings performed by 5l. + * These seem to be all of the opcodes that lead to the "S" bit + * being set in the instruction encodings. + * + * C_SBIT may also have been set explicitly in p->scond. + */ +int +modifiescpsr(Prog *p) +{ + return (p->scond&C_SBIT) + || p->as == ATST + || p->as == ATEQ + || p->as == ACMN + || p->as == ACMP + || p->as == AMULU + || p->as == ADIVU + || p->as == AMUL + || p->as == ADIV + || p->as == AMOD + || p->as == AMODU + || p->as == ABL; +} + +/* + * Find the maximal chain of instructions starting with r which could + * be executed conditionally + */ +int +joinsplit(Reg *r, Joininfo *j) +{ + j->start = r; + j->last = r; + j->len = 0; + do { + if (r->p2 && (r->p1 || r->p2->p2link)) { + j->end = r; + return Join; + } + if (r->s1 && r->s2) { + j->end = r; + return Split; + } + j->last = r; + if (r->prog->as != ANOP) + j->len++; + if (!r->s1 && !r->s2) { + j->end = r->link; + return End; + } + if (r->s2) { + j->end = r->s2; + return Branch; + } + if (modifiescpsr(r->prog)) { + j->end = r->s1; + return Setcond; + } + r = r->s1; + } while (j->len < 4); + j->end = r; + return Toolong; +} + +Reg * +successor(Reg *r) +{ + if (r->s1) + return r->s1; + else + return r->s2; +} + +void +applypred(Reg *rstart, Joininfo *j, int cond, int branch) +{ + int pred; + Reg *r; + + if(j->len == 0) + return; + if (cond == Truecond) + pred = predinfo[rstart->prog->as - ABEQ].scond; + else + pred = predinfo[rstart->prog->as - ABEQ].notscond; + + for (r = j->start; ; r = successor(r)) { + if (r->prog->as == AB) { + if (r != j->last || branch == Delbranch) + excise(r); + else { + if (cond == Truecond) + r->prog->as = predinfo[rstart->prog->as - ABEQ].opcode; + else + r->prog->as = predinfo[rstart->prog->as - ABEQ].notopcode; + } + } + else if (predicable(r->prog)) + r->prog->scond = (r->prog->scond&~C_SCOND)|pred; + if (r->s1 != r->link) { + r->s1 = r->link; + r->link->p1 = r; + } + if (r == j->last) + break; + } +} + +void +predicate(void) +{ + Reg *r; + int t1, t2; + Joininfo j1, j2; + + for(r=firstr; r!=R; r=r->link) { + if (isbranch(r->prog)) { + t1 = joinsplit(r->s1, &j1); + t2 = joinsplit(r->s2, &j2); + if(j1.last->link != j2.start) + continue; + if(j1.end == j2.end) + if((t1 == Branch && (t2 == Join || t2 == Setcond)) || + (t2 == Join && (t1 == Join || t1 == Setcond))) { + applypred(r, &j1, Falsecond, Delbranch); + applypred(r, &j2, Truecond, Delbranch); + excise(r); + continue; + } + if(t1 == End || t1 == Branch) { + applypred(r, &j1, Falsecond, Keepbranch); + excise(r); + continue; + } + } + } +} diff --git a/src/cmd/5c/reg.c b/src/cmd/5c/reg.c new file mode 100644 index 000000000..1ccf74a35 --- /dev/null +++ b/src/cmd/5c/reg.c @@ -0,0 +1,1195 @@ +// Inferno utils/5c/reg.c +// http://code.google.com/p/inferno-os/source/browse/utils/5c/reg.c +// +// Copyright © 1994-1999 Lucent Technologies Inc. All rights reserved. +// Portions Copyright © 1995-1997 C H Forsyth (forsyth@terzarima.net) +// Portions Copyright © 1997-1999 Vita Nuova Limited +// Portions Copyright © 2000-2007 Vita Nuova Holdings Limited (www.vitanuova.com) +// Portions Copyright © 2004,2006 Bruce Ellis +// Portions Copyright © 2005-2007 C H Forsyth (forsyth@terzarima.net) +// Revisions Copyright © 2000-2007 Lucent Technologies Inc. and others +// Portions Copyright © 2009 The Go Authors. All rights reserved. +// +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to deal +// in the Software without restriction, including without limitation the rights +// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +// copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +// THE SOFTWARE. + + +#include "gc.h" + +void addsplits(void); + +Reg* +rega(void) +{ + Reg *r; + + r = freer; + if(r == R) { + r = alloc(sizeof(*r)); + } else + freer = r->link; + + *r = zreg; + return r; +} + +int +rcmp(const void *a1, const void *a2) +{ + Rgn *p1, *p2; + int c1, c2; + + p1 = (Rgn*)a1; + p2 = (Rgn*)a2; + c1 = p2->cost; + c2 = p1->cost; + if(c1 -= c2) + return c1; + return p2->varno - p1->varno; +} + +void +regopt(Prog *p) +{ + USED(p); + // TODO(kaib): optimizer disabled because it smashes R8 when running out of registers + // the disable is unconventionally here because the call is in common code shared by 5c/6c/8c + return; + +#ifdef NOTDEF + Reg *r, *r1, *r2; + Prog *p1; + int i, z; + int32 initpc, val, npc; + uint32 vreg; + Bits bit; + struct + { + int32 m; + int32 c; + Reg* p; + } log5[6], *lp; + + firstr = R; + lastr = R; + nvar = 0; + regbits = 0; + for(z=0; zm = val; + lp->c = 0; + lp->p = R; + val /= 5L; + lp++; + } + val = 0; + for(; p != P; p = p->link) { + switch(p->as) { + case ADATA: + case AGLOBL: + case ANAME: + case ASIGNAME: + continue; + } + r = rega(); + if(firstr == R) { + firstr = r; + lastr = r; + } else { + lastr->link = r; + r->p1 = lastr; + lastr->s1 = r; + lastr = r; + } + r->prog = p; + r->pc = val; + val++; + + lp = log5; + for(i=0; i<5; i++) { + lp->c--; + if(lp->c <= 0) { + lp->c = lp->m; + if(lp->p != R) + lp->p->log5 = r; + lp->p = r; + (lp+1)->c = 0; + break; + } + lp++; + } + + r1 = r->p1; + if(r1 != R) + switch(r1->prog->as) { + case ARET: + case AB: + case ARFE: + r->p1 = R; + r1->s1 = R; + } + + /* + * left side always read + */ + bit = mkvar(&p->from, p->as==AMOVW); + for(z=0; zuse1.b[z] |= bit.b[z]; + + /* + * right side depends on opcode + */ + bit = mkvar(&p->to, 0); + if(bany(&bit)) + switch(p->as) { + default: + diag(Z, "reg: unknown asop: %A", p->as); + break; + + /* + * right side write + */ + case ANOP: + case AMOVB: + case AMOVBU: + case AMOVH: + case AMOVHU: + case AMOVW: + case AMOVF: + case AMOVD: + for(z=0; zset.b[z] |= bit.b[z]; + break; + + /* + * funny + */ + case ABL: + for(z=0; zas == AMOVM) { + if(p->from.type == D_CONST) + z = p->from.offset; + else + z = p->to.offset; + for(i=0; z; i++) { + if(z&1) + regbits |= RtoB(i); + z >>= 1; + } + } + } + if(firstr == R) + return; + initpc = pc - val; + npc = val; + + /* + * pass 2 + * turn branch references to pointers + * build back pointers + */ + for(r = firstr; r != R; r = r->link) { + p = r->prog; + if(p->to.type == D_BRANCH) { + val = p->to.offset - initpc; + r1 = firstr; + while(r1 != R) { + r2 = r1->log5; + if(r2 != R && val >= r2->pc) { + r1 = r2; + continue; + } + if(r1->pc == val) + break; + r1 = r1->link; + } + if(r1 == R) { + nearln = p->lineno; + diag(Z, "ref not found\n%P", p); + continue; + } + if(r1 == r) { + nearln = p->lineno; + diag(Z, "ref to self\n%P", p); + continue; + } + r->s2 = r1; + r->p2link = r1->p2; + r1->p2 = r; + } + } + if(debug['R']) { + p = firstr->prog; + print("\n%L %D\n", p->lineno, &p->from); + } + + /* + * pass 2.5 + * find looping structure + */ + for(r = firstr; r != R; r = r->link) + r->active = 0; + change = 0; + loopit(firstr, npc); + + /* + * pass 3 + * iterate propagating usage + * back until flow graph is complete + */ +loop1: + change = 0; + for(r = firstr; r != R; r = r->link) + r->active = 0; + for(r = firstr; r != R; r = r->link) + if(r->prog->as == ARET) + prop(r, zbits, zbits); +loop11: + /* pick up unreachable code */ + i = 0; + for(r = firstr; r != R; r = r1) { + r1 = r->link; + if(r1 && r1->active && !r->active) { + prop(r, zbits, zbits); + i = 1; + } + } + if(i) + goto loop11; + if(change) + goto loop1; + + + /* + * pass 4 + * iterate propagating register/variable synchrony + * forward until graph is complete + */ +loop2: + change = 0; + for(r = firstr; r != R; r = r->link) + r->active = 0; + synch(firstr, zbits); + if(change) + goto loop2; + + addsplits(); + + if(debug['R'] && debug['v']) { + print("\nprop structure:\n"); + for(r = firstr; r != R; r = r->link) { + print("%d:%P", r->loop, r->prog); + for(z=0; zset.b[z] | + r->refahead.b[z] | r->calahead.b[z] | + r->refbehind.b[z] | r->calbehind.b[z] | + r->use1.b[z] | r->use2.b[z]; + if(bany(&bit)) { + print("\t"); + if(bany(&r->use1)) + print(" u1=%B", r->use1); + if(bany(&r->use2)) + print(" u2=%B", r->use2); + if(bany(&r->set)) + print(" st=%B", r->set); + if(bany(&r->refahead)) + print(" ra=%B", r->refahead); + if(bany(&r->calahead)) + print(" ca=%B", r->calahead); + if(bany(&r->refbehind)) + print(" rb=%B", r->refbehind); + if(bany(&r->calbehind)) + print(" cb=%B", r->calbehind); + } + print("\n"); + } + } + + /* + * pass 5 + * isolate regions + * calculate costs (paint1) + */ + r = firstr; + if(r) { + for(z=0; zrefahead.b[z] | r->calahead.b[z]) & + ~(externs.b[z] | params.b[z] | addrs.b[z] | consts.b[z]); + if(bany(&bit)) { + nearln = r->prog->lineno; + warn(Z, "used and not set: %B", bit); + if(debug['R'] && !debug['w']) + print("used and not set: %B\n", bit); + } + } + + for(r = firstr; r != R; r = r->link) + r->act = zbits; + rgp = region; + nregion = 0; + for(r = firstr; r != R; r = r->link) { + for(z=0; zset.b[z] & + ~(r->refahead.b[z] | r->calahead.b[z] | addrs.b[z]); + if(bany(&bit)) { + nearln = r->prog->lineno; + warn(Z, "set and not used: %B", bit); + if(debug['R']) + print("set and not used: %B\n", bit); + excise(r); + } + for(z=0; zact.b[z] | addrs.b[z]); + while(bany(&bit)) { + i = bnum(bit); + rgp->enter = r; + rgp->varno = i; + change = 0; + if(debug['R'] && debug['v']) + print("\n"); + paint1(r, i); + bit.b[i/32] &= ~(1L<<(i%32)); + if(change <= 0) { + if(debug['R']) + print("%L $%d: %B\n", + r->prog->lineno, change, blsh(i)); + continue; + } + rgp->cost = change; + nregion++; + if(nregion >= NRGN) { + warn(Z, "too many regions"); + goto brk; + } + rgp++; + } + } +brk: + qsort(region, nregion, sizeof(region[0]), rcmp); + + /* + * pass 6 + * determine used registers (paint2) + * replace code (paint3) + */ + rgp = region; + for(i=0; ivarno); + vreg = paint2(rgp->enter, rgp->varno); + vreg = allreg(vreg, rgp); + if(debug['R']) { + if(rgp->regno >= NREG) + print("%L $%d F%d: %B\n", + rgp->enter->prog->lineno, + rgp->cost, + rgp->regno-NREG, + bit); + else + print("%L $%d R%d: %B\n", + rgp->enter->prog->lineno, + rgp->cost, + rgp->regno, + bit); + } + if(rgp->regno != 0) + paint3(rgp->enter, rgp->varno, vreg, rgp->regno); + rgp++; + } + /* + * pass 7 + * peep-hole on basic block + */ + if(!debug['R'] || debug['P']) + peep(); + + /* + * pass 8 + * recalculate pc + */ + val = initpc; + for(r = firstr; r != R; r = r1) { + r->pc = val; + p = r->prog; + p1 = P; + r1 = r->link; + if(r1 != R) + p1 = r1->prog; + for(; p != p1; p = p->link) { + switch(p->as) { + default: + val++; + break; + + case ANOP: + case ADATA: + case AGLOBL: + case ANAME: + case ASIGNAME: + break; + } + } + } + pc = val; + + /* + * fix up branches + */ + if(debug['R']) + if(bany(&addrs)) + print("addrs: %B\n", addrs); + + r1 = 0; /* set */ + for(r = firstr; r != R; r = r->link) { + p = r->prog; + if(p->to.type == D_BRANCH) + p->to.offset = r->s2->pc; + r1 = r; + } + + /* + * last pass + * eliminate nops + * free aux structures + */ + for(p = firstr->prog; p != P; p = p->link){ + while(p->link && p->link->as == ANOP) + p->link = p->link->link; + } + if(r1 != R) { + r1->link = freer; + freer = firstr; + } +#endif +} + +void +addsplits(void) +{ + Reg *r, *r1; + int z, i; + Bits bit; + + for(r = firstr; r != R; r = r->link) { + if(r->loop > 1) + continue; + if(r->prog->as == ABL) + continue; + for(r1 = r->p2; r1 != R; r1 = r1->p2link) { + if(r1->loop <= 1) + continue; + for(z=0; zcalbehind.b[z] & + (r->refahead.b[z] | r->use1.b[z] | r->use2.b[z]) & + ~(r->calahead.b[z] & addrs.b[z]); + while(bany(&bit)) { + i = bnum(bit); + bit.b[i/32] &= ~(1L << (i%32)); + } + } + } +} + +/* + * add mov b,rn + * just after r + */ +void +addmove(Reg *r, int bn, int rn, int f) +{ + Prog *p, *p1; + Adr *a; + Var *v; + + p1 = alloc(sizeof(*p1)); + *p1 = zprog; + p = r->prog; + + p1->link = p->link; + p->link = p1; + p1->lineno = p->lineno; + + v = var + bn; + + a = &p1->to; + a->sym = v->sym; + a->name = v->name; + a->offset = v->offset; + a->etype = v->etype; + a->type = D_OREG; + if(a->etype == TARRAY || a->sym == S) + a->type = D_CONST; + + p1->as = AMOVW; + if(v->etype == TCHAR || v->etype == TUCHAR) + p1->as = AMOVB; + if(v->etype == TSHORT || v->etype == TUSHORT) + p1->as = AMOVH; + if(v->etype == TFLOAT) + p1->as = AMOVF; + if(v->etype == TDOUBLE) + p1->as = AMOVD; + + p1->from.type = D_REG; + p1->from.reg = rn; + if(rn >= NREG) { + p1->from.type = D_FREG; + p1->from.reg = rn-NREG; + } + if(!f) { + p1->from = *a; + *a = zprog.from; + a->type = D_REG; + a->reg = rn; + if(rn >= NREG) { + a->type = D_FREG; + a->reg = rn-NREG; + } + if(v->etype == TUCHAR) + p1->as = AMOVBU; + if(v->etype == TUSHORT) + p1->as = AMOVHU; + } + if(debug['R']) + print("%P\t.a%P\n", p, p1); +} + +Bits +mkvar(Adr *a, int docon) +{ + Var *v; + int i, t, n, et, z; + int32 o; + Bits bit; + Sym *s; + + t = a->type; + if(t == D_REG && a->reg != NREG) + regbits |= RtoB(a->reg); + if(t == D_FREG && a->reg != NREG) + regbits |= FtoB(a->reg); + s = a->sym; + o = a->offset; + et = a->etype; + if(s == S) { + if(t != D_CONST || !docon || a->reg != NREG) + goto none; + et = TLONG; + } + if(t == D_CONST) { + if(s == S && sval(o)) + goto none; + } + + n = a->name; + v = var; + for(i=0; isym) + if(n == v->name) + if(o == v->offset) + goto out; + v++; + } + if(s) + if(s->name[0] == '.') + goto none; + if(nvar >= NVAR) { + if(debug['w'] > 1 && s) + warn(Z, "variable not optimized: %s", s->name); + goto none; + } + i = nvar; + nvar++; + v = &var[i]; + v->sym = s; + v->offset = o; + v->etype = et; + v->name = n; + if(debug['R']) + print("bit=%2d et=%2d %D\n", i, et, a); +out: + bit = blsh(i); + if(n == D_EXTERN || n == D_STATIC) + for(z=0; zetype != et || !typechlpfd[et]) /* funny punning */ + for(z=0; zp1) { + for(z=0; zrefahead.b[z]; + if(ref.b[z] != r1->refahead.b[z]) { + r1->refahead.b[z] = ref.b[z]; + change++; + } + cal.b[z] |= r1->calahead.b[z]; + if(cal.b[z] != r1->calahead.b[z]) { + r1->calahead.b[z] = cal.b[z]; + change++; + } + } + switch(r1->prog->as) { + case ABL: + for(z=0; zset.b[z]) | + r1->use1.b[z] | r1->use2.b[z]; + cal.b[z] &= ~(r1->set.b[z] | r1->use1.b[z] | r1->use2.b[z]); + r1->refbehind.b[z] = ref.b[z]; + r1->calbehind.b[z] = cal.b[z]; + } + if(r1->active) + break; + r1->active = 1; + } + for(; r != r1; r = r->p1) + for(r2 = r->p2; r2 != R; r2 = r2->p2link) + prop(r2, r->refbehind, r->calbehind); +} + +/* + * find looping structure + * + * 1) find reverse postordering + * 2) find approximate dominators, + * the actual dominators if the flow graph is reducible + * otherwise, dominators plus some other non-dominators. + * See Matthew S. Hecht and Jeffrey D. Ullman, + * "Analysis of a Simple Algorithm for Global Data Flow Problems", + * Conf. Record of ACM Symp. on Principles of Prog. Langs, Boston, Massachusetts, + * Oct. 1-3, 1973, pp. 207-217. + * 3) find all nodes with a predecessor dominated by the current node. + * such a node is a loop head. + * recursively, all preds with a greater rpo number are in the loop + */ +int32 +postorder(Reg *r, Reg **rpo2r, int32 n) +{ + Reg *r1; + + r->rpo = 1; + r1 = r->s1; + if(r1 && !r1->rpo) + n = postorder(r1, rpo2r, n); + r1 = r->s2; + if(r1 && !r1->rpo) + n = postorder(r1, rpo2r, n); + rpo2r[n] = r; + n++; + return n; +} + +int32 +rpolca(int32 *idom, int32 rpo1, int32 rpo2) +{ + int32 t; + + if(rpo1 == -1) + return rpo2; + while(rpo1 != rpo2){ + if(rpo1 > rpo2){ + t = rpo2; + rpo2 = rpo1; + rpo1 = t; + } + while(rpo1 < rpo2){ + t = idom[rpo2]; + if(t >= rpo2) + fatal(Z, "bad idom"); + rpo2 = t; + } + } + return rpo1; +} + +int +doms(int32 *idom, int32 r, int32 s) +{ + while(s > r) + s = idom[s]; + return s == r; +} + +int +loophead(int32 *idom, Reg *r) +{ + int32 src; + + src = r->rpo; + if(r->p1 != R && doms(idom, src, r->p1->rpo)) + return 1; + for(r = r->p2; r != R; r = r->p2link) + if(doms(idom, src, r->rpo)) + return 1; + return 0; +} + +void +loopmark(Reg **rpo2r, int32 head, Reg *r) +{ + if(r->rpo < head || r->active == head) + return; + r->active = head; + r->loop += LOOP; + if(r->p1 != R) + loopmark(rpo2r, head, r->p1); + for(r = r->p2; r != R; r = r->p2link) + loopmark(rpo2r, head, r); +} + +void +loopit(Reg *r, int32 nr) +{ + Reg *r1; + int32 i, d, me; + + if(nr > maxnr) { + rpo2r = alloc(nr * sizeof(Reg*)); + idom = alloc(nr * sizeof(int32)); + maxnr = nr; + } + d = postorder(r, rpo2r, 0); + if(d > nr) + fatal(Z, "too many reg nodes"); + nr = d; + for(i = 0; i < nr / 2; i++){ + r1 = rpo2r[i]; + rpo2r[i] = rpo2r[nr - 1 - i]; + rpo2r[nr - 1 - i] = r1; + } + for(i = 0; i < nr; i++) + rpo2r[i]->rpo = i; + + idom[0] = 0; + for(i = 0; i < nr; i++){ + r1 = rpo2r[i]; + me = r1->rpo; + d = -1; + if(r1->p1 != R && r1->p1->rpo < me) + d = r1->p1->rpo; + for(r1 = r1->p2; r1 != nil; r1 = r1->p2link) + if(r1->rpo < me) + d = rpolca(idom, d, r1->rpo); + idom[i] = d; + } + + for(i = 0; i < nr; i++){ + r1 = rpo2r[i]; + r1->loop++; + if(r1->p2 != R && loophead(idom, r1)) + loopmark(rpo2r, i, r1); + } +} + +void +synch(Reg *r, Bits dif) +{ + Reg *r1; + int z; + + for(r1 = r; r1 != R; r1 = r1->s1) { + for(z=0; zrefbehind.b[z] & r1->refahead.b[z])) | + r1->set.b[z] | r1->regdiff.b[z]; + if(dif.b[z] != r1->regdiff.b[z]) { + r1->regdiff.b[z] = dif.b[z]; + change++; + } + } + if(r1->active) + break; + r1->active = 1; + for(z=0; zcalbehind.b[z] & r1->calahead.b[z]); + if(r1->s2 != R) + synch(r1->s2, dif); + } +} + +uint32 +allreg(uint32 b, Rgn *r) +{ + Var *v; + int i; + + v = var + r->varno; + r->regno = 0; + switch(v->etype) { + + default: + diag(Z, "unknown etype %d/%d", bitno(b), v->etype); + break; + + case TCHAR: + case TUCHAR: + case TSHORT: + case TUSHORT: + case TINT: + case TUINT: + case TLONG: + case TULONG: + case TIND: + case TARRAY: + i = BtoR(~b); + if(i && r->cost >= 0) { + r->regno = i; + return RtoB(i); + } + break; + + case TVLONG: + case TDOUBLE: + case TFLOAT: + i = BtoF(~b); + if(i && r->cost >= 0) { + r->regno = i+NREG; + return FtoB(i); + } + break; + } + return 0; +} + +void +paint1(Reg *r, int bn) +{ + Reg *r1; + Prog *p; + int z; + uint32 bb; + + z = bn/32; + bb = 1L<<(bn%32); + if(r->act.b[z] & bb) + return; + for(;;) { + if(!(r->refbehind.b[z] & bb)) + break; + r1 = r->p1; + if(r1 == R) + break; + if(!(r1->refahead.b[z] & bb)) + break; + if(r1->act.b[z] & bb) + break; + r = r1; + } + + if(LOAD(r) & ~(r->set.b[z] & ~(r->use1.b[z]|r->use2.b[z])) & bb) { + change -= CLOAD * r->loop; + if(debug['R'] && debug['v']) + print("%d%P\td %B $%d\n", r->loop, + r->prog, blsh(bn), change); + } + for(;;) { + r->act.b[z] |= bb; + p = r->prog; + + if(r->use1.b[z] & bb) { + change += CREF * r->loop; + if(debug['R'] && debug['v']) + print("%d%P\tu1 %B $%d\n", r->loop, + p, blsh(bn), change); + } + + if((r->use2.b[z]|r->set.b[z]) & bb) { + change += CREF * r->loop; + if(debug['R'] && debug['v']) + print("%d%P\tu2 %B $%d\n", r->loop, + p, blsh(bn), change); + } + + if(STORE(r) & r->regdiff.b[z] & bb) { + change -= CLOAD * r->loop; + if(debug['R'] && debug['v']) + print("%d%P\tst %B $%d\n", r->loop, + p, blsh(bn), change); + } + + if(r->refbehind.b[z] & bb) + for(r1 = r->p2; r1 != R; r1 = r1->p2link) + if(r1->refahead.b[z] & bb) + paint1(r1, bn); + + if(!(r->refahead.b[z] & bb)) + break; + r1 = r->s2; + if(r1 != R) + if(r1->refbehind.b[z] & bb) + paint1(r1, bn); + r = r->s1; + if(r == R) + break; + if(r->act.b[z] & bb) + break; + if(!(r->refbehind.b[z] & bb)) + break; + } +} + +uint32 +paint2(Reg *r, int bn) +{ + Reg *r1; + int z; + uint32 bb, vreg; + + z = bn/32; + bb = 1L << (bn%32); + vreg = regbits; + if(!(r->act.b[z] & bb)) + return vreg; + for(;;) { + if(!(r->refbehind.b[z] & bb)) + break; + r1 = r->p1; + if(r1 == R) + break; + if(!(r1->refahead.b[z] & bb)) + break; + if(!(r1->act.b[z] & bb)) + break; + r = r1; + } + for(;;) { + r->act.b[z] &= ~bb; + + vreg |= r->regu; + + if(r->refbehind.b[z] & bb) + for(r1 = r->p2; r1 != R; r1 = r1->p2link) + if(r1->refahead.b[z] & bb) + vreg |= paint2(r1, bn); + + if(!(r->refahead.b[z] & bb)) + break; + r1 = r->s2; + if(r1 != R) + if(r1->refbehind.b[z] & bb) + vreg |= paint2(r1, bn); + r = r->s1; + if(r == R) + break; + if(!(r->act.b[z] & bb)) + break; + if(!(r->refbehind.b[z] & bb)) + break; + } + return vreg; +} + +void +paint3(Reg *r, int bn, int32 rb, int rn) +{ + Reg *r1; + Prog *p; + int z; + uint32 bb; + + z = bn/32; + bb = 1L << (bn%32); + if(r->act.b[z] & bb) + return; + for(;;) { + if(!(r->refbehind.b[z] & bb)) + break; + r1 = r->p1; + if(r1 == R) + break; + if(!(r1->refahead.b[z] & bb)) + break; + if(r1->act.b[z] & bb) + break; + r = r1; + } + + if(LOAD(r) & ~(r->set.b[z] & ~(r->use1.b[z]|r->use2.b[z])) & bb) + addmove(r, bn, rn, 0); + for(;;) { + r->act.b[z] |= bb; + p = r->prog; + + if(r->use1.b[z] & bb) { + if(debug['R']) + print("%P", p); + addreg(&p->from, rn); + if(debug['R']) + print("\t.c%P\n", p); + } + if((r->use2.b[z]|r->set.b[z]) & bb) { + if(debug['R']) + print("%P", p); + addreg(&p->to, rn); + if(debug['R']) + print("\t.c%P\n", p); + } + + if(STORE(r) & r->regdiff.b[z] & bb) + addmove(r, bn, rn, 1); + r->regu |= rb; + + if(r->refbehind.b[z] & bb) + for(r1 = r->p2; r1 != R; r1 = r1->p2link) + if(r1->refahead.b[z] & bb) + paint3(r1, bn, rb, rn); + + if(!(r->refahead.b[z] & bb)) + break; + r1 = r->s2; + if(r1 != R) + if(r1->refbehind.b[z] & bb) + paint3(r1, bn, rb, rn); + r = r->s1; + if(r == R) + break; + if(r->act.b[z] & bb) + break; + if(!(r->refbehind.b[z] & bb)) + break; + } +} + +void +addreg(Adr *a, int rn) +{ + + a->sym = 0; + a->name = D_NONE; + a->type = D_REG; + a->reg = rn; + if(rn >= NREG) { + a->type = D_FREG; + a->reg = rn-NREG; + } +} + +/* + * bit reg + * 0 R0 + * 1 R1 + * ... ... + * 10 R10 + */ +int32 +RtoB(int r) +{ + + if(r < 2 || r >= REGTMP-2) // excluded R9 and R10 for m and g + return 0; + return 1L << r; +} + +int +BtoR(int32 b) +{ + b &= 0x01fcL; // excluded R9 and R10 for m and g + if(b == 0) + return 0; + return bitno(b); +} + +/* + * bit reg + * 18 F2 + * 19 F3 + * ... ... + * 23 F7 + */ +int32 +FtoB(int f) +{ + + if(f < 2 || f > NFREG-1) + return 0; + return 1L << (f + 16); +} + +int +BtoF(int32 b) +{ + + b &= 0xfc0000L; + if(b == 0) + return 0; + return bitno(b) - 16; +} diff --git a/src/cmd/5c/sgen.c b/src/cmd/5c/sgen.c new file mode 100644 index 000000000..92a0f64f8 --- /dev/null +++ b/src/cmd/5c/sgen.c @@ -0,0 +1,267 @@ +// Inferno utils/5c/sgen.c +// http://code.google.com/p/inferno-os/source/browse/utils/5c/sgen.c +// +// Copyright © 1994-1999 Lucent Technologies Inc. All rights reserved. +// Portions Copyright © 1995-1997 C H Forsyth (forsyth@terzarima.net) +// Portions Copyright © 1997-1999 Vita Nuova Limited +// Portions Copyright © 2000-2007 Vita Nuova Holdings Limited (www.vitanuova.com) +// Portions Copyright © 2004,2006 Bruce Ellis +// Portions Copyright © 2005-2007 C H Forsyth (forsyth@terzarima.net) +// Revisions Copyright © 2000-2007 Lucent Technologies Inc. and others +// Portions Copyright © 2009 The Go Authors. All rights reserved. +// +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to deal +// in the Software without restriction, including without limitation the rights +// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +// copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +// THE SOFTWARE. + + +#include "gc.h" + +Prog* +gtext(Sym *s, int32 stkoff) +{ + int32 a; + + a = 0; + if(!(textflag & NOSPLIT)) + a = argsize(); + else if(stkoff >= 128) + yyerror("stack frame too large for NOSPLIT function"); + + gpseudo(ATEXT, s, nodconst(stkoff)); + p->to.type = D_CONST2; + p->to.offset2 = a; + return p; +} + +void +noretval(int n) +{ + + if(n & 1) { + gins(ANOP, Z, Z); + p->to.type = D_REG; + p->to.reg = REGRET; + } + if(n & 2) { + gins(ANOP, Z, Z); + p->to.type = D_FREG; + p->to.reg = FREGRET; + } +} + +/* + * calculate addressability as follows + * CONST ==> 20 $value + * NAME ==> 10 name + * REGISTER ==> 11 register + * INDREG ==> 12 *[(reg)+offset] + * &10 ==> 2 $name + * ADD(2, 20) ==> 2 $name+offset + * ADD(3, 20) ==> 3 $(reg)+offset + * &12 ==> 3 $(reg)+offset + * *11 ==> 11 ?? + * *2 ==> 10 name + * *3 ==> 12 *(reg)+offset + * calculate complexity (number of registers) + */ +void +xcom(Node *n) +{ + Node *l, *r; + int t; + + if(n == Z) + return; + l = n->left; + r = n->right; + n->addable = 0; + n->complex = 0; + switch(n->op) { + case OCONST: + n->addable = 20; + return; + + case OREGISTER: + n->addable = 11; + return; + + case OINDREG: + n->addable = 12; + return; + + case ONAME: + n->addable = 10; + return; + + case OADDR: + xcom(l); + if(l->addable == 10) + n->addable = 2; + if(l->addable == 12) + n->addable = 3; + break; + + case OIND: + xcom(l); + if(l->addable == 11) + n->addable = 12; + if(l->addable == 3) + n->addable = 12; + if(l->addable == 2) + n->addable = 10; + break; + + case OADD: + xcom(l); + xcom(r); + if(l->addable == 20) { + if(r->addable == 2) + n->addable = 2; + if(r->addable == 3) + n->addable = 3; + } + if(r->addable == 20) { + if(l->addable == 2) + n->addable = 2; + if(l->addable == 3) + n->addable = 3; + } + break; + + case OASLMUL: + case OASMUL: + xcom(l); + xcom(r); + t = vlog(r); + if(t >= 0) { + n->op = OASASHL; + r->vconst = t; + r->type = types[TINT]; + } + break; + + case OMUL: + case OLMUL: + xcom(l); + xcom(r); + t = vlog(r); + if(t >= 0) { + n->op = OASHL; + r->vconst = t; + r->type = types[TINT]; + } + t = vlog(l); + if(t >= 0) { + n->op = OASHL; + n->left = r; + n->right = l; + r = l; + l = n->left; + r->vconst = t; + r->type = types[TINT]; + } + break; + + case OASLDIV: + xcom(l); + xcom(r); + t = vlog(r); + if(t >= 0) { + n->op = OASLSHR; + r->vconst = t; + r->type = types[TINT]; + } + break; + + case OLDIV: + xcom(l); + xcom(r); + t = vlog(r); + if(t >= 0) { + n->op = OLSHR; + r->vconst = t; + r->type = types[TINT]; + } + break; + + case OASLMOD: + xcom(l); + xcom(r); + t = vlog(r); + if(t >= 0) { + n->op = OASAND; + r->vconst--; + } + break; + + case OLMOD: + xcom(l); + xcom(r); + t = vlog(r); + if(t >= 0) { + n->op = OAND; + r->vconst--; + } + break; + + default: + if(l != Z) + xcom(l); + if(r != Z) + xcom(r); + break; + } + if(n->addable >= 10) + return; + + if(l != Z) + n->complex = l->complex; + if(r != Z) { + if(r->complex == n->complex) + n->complex = r->complex+1; + else + if(r->complex > n->complex) + n->complex = r->complex; + } + if(n->complex == 0) + n->complex++; + + if(com64(n)) + return; + + switch(n->op) { + case OFUNC: + n->complex = FNX; + break; + + case OADD: + case OXOR: + case OAND: + case OOR: + case OEQ: + case ONE: + /* + * immediate operators, make const on right + */ + if(l->op == OCONST) { + n->left = r; + n->right = l; + } + break; + } +} diff --git a/src/cmd/5c/swt.c b/src/cmd/5c/swt.c new file mode 100644 index 000000000..7cbaadba9 --- /dev/null +++ b/src/cmd/5c/swt.c @@ -0,0 +1,694 @@ +// Inferno utils/5c/swt.c +// http://code.google.com/p/inferno-os/source/browse/utils/5c/swt.c +// +// Copyright © 1994-1999 Lucent Technologies Inc. All rights reserved. +// Portions Copyright © 1995-1997 C H Forsyth (forsyth@terzarima.net) +// Portions Copyright © 1997-1999 Vita Nuova Limited +// Portions Copyright © 2000-2007 Vita Nuova Holdings Limited (www.vitanuova.com) +// Portions Copyright © 2004,2006 Bruce Ellis +// Portions Copyright © 2005-2007 C H Forsyth (forsyth@terzarima.net) +// Revisions Copyright © 2000-2007 Lucent Technologies Inc. and others +// Portions Copyright © 2009 The Go Authors. All rights reserved. +// +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to deal +// in the Software without restriction, including without limitation the rights +// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +// copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +// THE SOFTWARE. + + +#include "gc.h" + +void +swit1(C1 *q, int nc, int32 def, Node *n) +{ + C1 *r; + int i; + int32 v; + Prog *sp; + + if(nc >= 3) { + i = (q+nc-1)->val - (q+0)->val; + if(i > 0 && i < nc*2) + goto direct; + } + if(nc < 5) { + for(i=0; ival); + gopcode(OEQ, nodconst(q->val), n, Z); + patch(p, q->label); + q++; + } + gbranch(OGOTO); + patch(p, def); + return; + } + + i = nc / 2; + r = q+i; + if(debug['W']) + print("case > %.8ux\n", r->val); + gopcode(OGT, nodconst(r->val), n, Z); + sp = p; + gopcode(OEQ, nodconst(r->val), n, Z); /* just gen the B.EQ */ + patch(p, r->label); + swit1(q, i, def, n); + + if(debug['W']) + print("case < %.8ux\n", r->val); + patch(sp, pc); + swit1(r+1, nc-i-1, def, n); + return; + +direct: + v = q->val; + if(v != 0) + gopcode(OSUB, nodconst(v), Z, n); + gopcode(OCASE, nodconst((q+nc-1)->val - v), n, Z); + patch(p, def); + for(i=0; ival); + while(q->val != v) { + nextpc(); + p->as = ABCASE; + patch(p, def); + v++; + } + nextpc(); + p->as = ABCASE; + patch(p, q->label); + q++; + v++; + } + gbranch(OGOTO); /* so that regopt() won't be confused */ + patch(p, def); +} + +void +bitload(Node *b, Node *n1, Node *n2, Node *n3, Node *nn) +{ + int sh; + int32 v; + Node *l; + + /* + * n1 gets adjusted/masked value + * n2 gets address of cell + * n3 gets contents of cell + */ + l = b->left; + if(n2 != Z) { + regalloc(n1, l, nn); + reglcgen(n2, l, Z); + regalloc(n3, l, Z); + gopcode(OAS, n2, Z, n3); + gopcode(OAS, n3, Z, n1); + } else { + regalloc(n1, l, nn); + cgen(l, n1); + } + if(b->type->shift == 0 && typeu[b->type->etype]) { + v = ~0 + (1L << b->type->nbits); + gopcode(OAND, nodconst(v), Z, n1); + } else { + sh = 32 - b->type->shift - b->type->nbits; + if(sh > 0) + gopcode(OASHL, nodconst(sh), Z, n1); + sh += b->type->shift; + if(sh > 0) + if(typeu[b->type->etype]) + gopcode(OLSHR, nodconst(sh), Z, n1); + else + gopcode(OASHR, nodconst(sh), Z, n1); + } +} + +void +bitstore(Node *b, Node *n1, Node *n2, Node *n3, Node *nn) +{ + int32 v; + Node nod, *l; + int sh; + + /* + * n1 has adjusted/masked value + * n2 has address of cell + * n3 has contents of cell + */ + l = b->left; + regalloc(&nod, l, Z); + v = ~0 + (1L << b->type->nbits); + gopcode(OAND, nodconst(v), Z, n1); + gopcode(OAS, n1, Z, &nod); + if(nn != Z) + gopcode(OAS, n1, Z, nn); + sh = b->type->shift; + if(sh > 0) + gopcode(OASHL, nodconst(sh), Z, &nod); + v <<= sh; + gopcode(OAND, nodconst(~v), Z, n3); + gopcode(OOR, n3, Z, &nod); + gopcode(OAS, &nod, Z, n2); + + regfree(&nod); + regfree(n1); + regfree(n2); + regfree(n3); +} + +int32 +outstring(char *s, int32 n) +{ + int32 r; + + if(suppress) + return nstring; + r = nstring; + while(n) { + string[mnstring] = *s++; + mnstring++; + nstring++; + if(mnstring >= NSNAME) { + gpseudo(ADATA, symstring, nodconst(0L)); + p->from.offset += nstring - NSNAME; + p->reg = NSNAME; + p->to.type = D_SCONST; + memmove(p->to.sval, string, NSNAME); + mnstring = 0; + } + n--; + } + return r; +} + +int +mulcon(Node *n, Node *nn) +{ + Node *l, *r, nod1, nod2; + Multab *m; + int32 v, vs; + int o; + char code[sizeof(m->code)+2], *p; + + if(typefd[n->type->etype]) + return 0; + l = n->left; + r = n->right; + if(l->op == OCONST) { + l = r; + r = n->left; + } + if(r->op != OCONST) + return 0; + v = convvtox(r->vconst, n->type->etype); + if(v != r->vconst) { + if(debug['M']) + print("%L multiply conv: %lld\n", n->lineno, r->vconst); + return 0; + } + m = mulcon0(v); + if(!m) { + if(debug['M']) + print("%L multiply table: %lld\n", n->lineno, r->vconst); + return 0; + } + if(debug['M'] && debug['v']) + print("%L multiply: %d\n", n->lineno, v); + + memmove(code, m->code, sizeof(m->code)); + code[sizeof(m->code)] = 0; + + p = code; + if(p[1] == 'i') + p += 2; + regalloc(&nod1, n, nn); + cgen(l, &nod1); + vs = v; + regalloc(&nod2, n, Z); + +loop: + switch(*p) { + case 0: + regfree(&nod2); + if(vs < 0) { + gopcode(OAS, &nod1, Z, &nod1); + gopcode(OSUB, &nod1, nodconst(0), nn); + } else + gopcode(OAS, &nod1, Z, nn); + regfree(&nod1); + return 1; + case '+': + o = OADD; + goto addsub; + case '-': + o = OSUB; + addsub: /* number is r,n,l */ + v = p[1] - '0'; + r = &nod1; + if(v&4) + r = &nod2; + n = &nod1; + if(v&2) + n = &nod2; + l = &nod1; + if(v&1) + l = &nod2; + gopcode(o, l, n, r); + break; + default: /* op is shiftcount, number is r,l */ + v = p[1] - '0'; + r = &nod1; + if(v&2) + r = &nod2; + l = &nod1; + if(v&1) + l = &nod2; + v = *p - 'a'; + if(v < 0 || v >= 32) { + diag(n, "mulcon unknown op: %c%c", p[0], p[1]); + break; + } + gopcode(OASHL, nodconst(v), l, r); + break; + } + p += 2; + goto loop; +} + +void +sextern(Sym *s, Node *a, int32 o, int32 w) +{ + int32 e, lw; + + for(e=0; efrom.offset += o+e; + p->reg = lw; + p->to.type = D_SCONST; + memmove(p->to.sval, a->cstring+e, lw); + } +} + +void +gextern(Sym *s, Node *a, int32 o, int32 w) +{ + + if(a->op == OCONST && typev[a->type->etype]) { + if(isbigendian) + gpseudo(ADATA, s, nod32const(a->vconst>>32)); + else + gpseudo(ADATA, s, nod32const(a->vconst)); + p->from.offset += o; + p->reg = 4; + if(isbigendian) + gpseudo(ADATA, s, nod32const(a->vconst)); + else + gpseudo(ADATA, s, nod32const(a->vconst>>32)); + p->from.offset += o + 4; + p->reg = 4; + return; + } + gpseudo(ADATA, s, a); + p->from.offset += o; + p->reg = w; + if(p->to.type == D_OREG) + p->to.type = D_CONST; +} + +void zname(Biobuf*, Sym*, int); +char* zaddr(char*, Adr*, int); +void zwrite(Biobuf*, Prog*, int, int); +void outhist(Biobuf*); + +void +zwrite(Biobuf *b, Prog *p, int sf, int st) +{ + char bf[100], *bp; + + bf[0] = p->as; + bf[1] = p->scond; + bf[2] = p->reg; + bf[3] = p->lineno; + bf[4] = p->lineno>>8; + bf[5] = p->lineno>>16; + bf[6] = p->lineno>>24; + bp = zaddr(bf+7, &p->from, sf); + bp = zaddr(bp, &p->to, st); + Bwrite(b, bf, bp-bf); +} + +void +outcode(void) +{ + struct { Sym *sym; short type; } h[NSYM]; + Prog *p; + Sym *s; + int sf, st, t, sym; + + if(debug['S']) { + for(p = firstp; p != P; p = p->link) + if(p->as != ADATA && p->as != AGLOBL) + pc--; + for(p = firstp; p != P; p = p->link) { + print("%P\n", p); + if(p->as != ADATA && p->as != AGLOBL) + pc++; + } + } + + Bprint(&outbuf, "go object %s %s %s\n", getgoos(), thestring, getgoversion()); + if(ndynimp > 0 || ndynexp > 0) { + int i; + + Bprint(&outbuf, "\n"); + Bprint(&outbuf, "$$ // exports\n\n"); + Bprint(&outbuf, "$$ // local types\n\n"); + Bprint(&outbuf, "$$ // dynimport\n"); + for(i=0; ilink) { + jackpot: + sf = 0; + s = p->from.sym; + while(s != S) { + sf = s->sym; + if(sf < 0 || sf >= NSYM) + sf = 0; + t = p->from.name; + if(h[sf].type == t) + if(h[sf].sym == s) + break; + s->sym = sym; + zname(&outbuf, s, t); + h[sym].sym = s; + h[sym].type = t; + sf = sym; + sym++; + if(sym >= NSYM) + sym = 1; + break; + } + st = 0; + s = p->to.sym; + while(s != S) { + st = s->sym; + if(st < 0 || st >= NSYM) + st = 0; + t = p->to.name; + if(h[st].type == t) + if(h[st].sym == s) + break; + s->sym = sym; + zname(&outbuf, s, t); + h[sym].sym = s; + h[sym].type = t; + st = sym; + sym++; + if(sym >= NSYM) + sym = 1; + if(st == sf) + goto jackpot; + break; + } + zwrite(&outbuf, p, sf, st); + } + firstp = P; + lastp = P; +} + +void +outhist(Biobuf *b) +{ + Hist *h; + char *p, *q, *op, c; + Prog pg; + int n; + + pg = zprog; + pg.as = AHISTORY; + c = pathchar(); + for(h = hist; h != H; h = h->link) { + p = h->name; + op = 0; + /* on windows skip drive specifier in pathname */ + if(systemtype(Windows) && p && p[1] == ':'){ + p += 2; + c = *p; + } + if(p && p[0] != c && h->offset == 0 && pathname){ + /* on windows skip drive specifier in pathname */ + if(systemtype(Windows) && pathname[1] == ':') { + op = p; + p = pathname+2; + c = *p; + } else if(pathname[0] == c){ + op = p; + p = pathname; + } + } + while(p) { + q = utfrune(p, c); + if(q) { + n = q-p; + if(n == 0){ + n = 1; /* leading "/" */ + *p = '/'; /* don't emit "\" on windows */ + } + q++; + } else { + n = strlen(p); + q = 0; + } + if(n) { + Bputc(b, ANAME); + Bputc(b, D_FILE); + Bputc(b, 1); + Bputc(b, '<'); + Bwrite(b, p, n); + Bputc(b, 0); + } + p = q; + if(p == 0 && op) { + p = op; + op = 0; + } + } + pg.lineno = h->line; + pg.to.type = zprog.to.type; + pg.to.offset = h->offset; + if(h->offset) + pg.to.type = D_CONST; + + zwrite(b, &pg, 0, 0); + } +} + +void +zname(Biobuf *b, Sym *s, int t) +{ + char *n, bf[7]; + uint32 sig; + + n = s->name; + if(debug['T'] && t == D_EXTERN && s->sig != SIGDONE && s->type != types[TENUM] && s != symrathole){ + sig = sign(s); + bf[0] = ASIGNAME; + bf[1] = sig; + bf[2] = sig>>8; + bf[3] = sig>>16; + bf[4] = sig>>24; + bf[5] = t; + bf[6] = s->sym; + Bwrite(b, bf, 7); + s->sig = SIGDONE; + } + else{ + bf[0] = ANAME; + bf[1] = t; /* type */ + bf[2] = s->sym; /* sym */ + Bwrite(b, bf, 3); + } + Bwrite(b, n, strlen(n)+1); +} + +char* +zaddr(char *bp, Adr *a, int s) +{ + int32 l; + Ieee e; + + bp[0] = a->type; + bp[1] = a->reg; + bp[2] = s; + bp[3] = a->name; + bp += 4; + switch(a->type) { + default: + diag(Z, "unknown type %d in zaddr", a->type); + + case D_NONE: + case D_REG: + case D_FREG: + case D_PSR: + break; + + case D_CONST2: + l = a->offset2; + bp[0] = l; + bp[1] = l>>8; + bp[2] = l>>16; + bp[3] = l>>24; + bp += 4; // fall through + case D_OREG: + case D_CONST: + case D_BRANCH: + case D_SHIFT: + l = a->offset; + bp[0] = l; + bp[1] = l>>8; + bp[2] = l>>16; + bp[3] = l>>24; + bp += 4; + break; + + case D_SCONST: + memmove(bp, a->sval, NSNAME); + bp += NSNAME; + break; + + case D_FCONST: + ieeedtod(&e, a->dval); + l = e.l; + bp[0] = l; + bp[1] = l>>8; + bp[2] = l>>16; + bp[3] = l>>24; + bp += 4; + l = e.h; + bp[0] = l; + bp[1] = l>>8; + bp[2] = l>>16; + bp[3] = l>>24; + bp += 4; + break; + } + return bp; +} + +int32 +align(int32 i, Type *t, int op, int32 *maxalign) +{ + int32 o; + Type *v; + int w; + + o = i; + w = 1; + switch(op) { + default: + diag(Z, "unknown align opcode %d", op); + break; + + case Asu2: /* padding at end of a struct */ + w = *maxalign; + if(w < 1) + w = 1; + if(packflg) + w = packflg; + break; + + case Ael1: /* initial align of struct element */ + for(v=t; v->etype==TARRAY; v=v->link) + ; + if(v->etype == TSTRUCT || v->etype == TUNION) + w = v->align; + else { + w = ewidth[v->etype]; + if(w == 8) + w = 4; + } + if(w < 1 || w > SZ_LONG) + fatal(Z, "align"); + if(packflg) + w = packflg; + break; + + case Ael2: /* width of a struct element */ + o += t->width; + break; + + case Aarg0: /* initial passbyptr argument in arg list */ + if(typesuv[t->etype]) { + o = align(o, types[TIND], Aarg1, nil); + o = align(o, types[TIND], Aarg2, nil); + } + break; + + case Aarg1: /* initial align of parameter */ + w = ewidth[t->etype]; + if(w <= 0 || w >= SZ_LONG) { + w = SZ_LONG; + break; + } + w = 1; /* little endian no adjustment */ + break; + + case Aarg2: /* width of a parameter */ + o += t->width; + w = t->width; + if(w > SZ_LONG) + w = SZ_LONG; + break; + + case Aaut3: /* total align of automatic */ + o = align(o, t, Ael2, nil); + o = align(o, t, Ael1, nil); + w = SZ_LONG; /* because of a pun in cc/dcl.c:contig() */ + break; + } + o = xround(o, w); + if(maxalign != nil && *maxalign < w) + *maxalign = w; + if(debug['A']) + print("align %s %d %T = %d\n", bnames[op], i, t, o); + return o; +} + +int32 +maxround(int32 max, int32 v) +{ + v = xround(v, SZ_LONG); + if(v > max) + return v; + return max; +} diff --git a/src/cmd/5c/txt.c b/src/cmd/5c/txt.c new file mode 100644 index 000000000..a32387bc1 --- /dev/null +++ b/src/cmd/5c/txt.c @@ -0,0 +1,1298 @@ +// Inferno utils/5c/txt.c +// http://code.google.com/p/inferno-os/source/browse/utils/5c/txt.c +// +// Copyright © 1994-1999 Lucent Technologies Inc. All rights reserved. +// Portions Copyright © 1995-1997 C H Forsyth (forsyth@terzarima.net) +// Portions Copyright © 1997-1999 Vita Nuova Limited +// Portions Copyright © 2000-2007 Vita Nuova Holdings Limited (www.vitanuova.com) +// Portions Copyright © 2004,2006 Bruce Ellis +// Portions Copyright © 2005-2007 C H Forsyth (forsyth@terzarima.net) +// Revisions Copyright © 2000-2007 Lucent Technologies Inc. and others +// Portions Copyright © 2009 The Go Authors. All rights reserved. +// +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to deal +// in the Software without restriction, including without limitation the rights +// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +// copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +// THE SOFTWARE. + + +#include "gc.h" + +void +ginit(void) +{ + Type *t; + + thechar = '5'; + thestring = "arm"; + exregoffset = REGEXT; + exfregoffset = FREGEXT; + listinit(); + nstring = 0; + mnstring = 0; + nrathole = 0; + pc = 0; + breakpc = -1; + continpc = -1; + cases = C; + firstp = P; + lastp = P; + tfield = types[TLONG]; + + zprog.link = P; + zprog.as = AGOK; + zprog.reg = NREG; + zprog.from.type = D_NONE; + zprog.from.name = D_NONE; + zprog.from.reg = NREG; + zprog.to = zprog.from; + zprog.scond = 0xE; + + regnode.op = OREGISTER; + regnode.class = CEXREG; + regnode.reg = REGTMP; + regnode.complex = 0; + regnode.addable = 11; + regnode.type = types[TLONG]; + + constnode.op = OCONST; + constnode.class = CXXX; + constnode.complex = 0; + constnode.addable = 20; + constnode.type = types[TLONG]; + + fconstnode.op = OCONST; + fconstnode.class = CXXX; + fconstnode.complex = 0; + fconstnode.addable = 20; + fconstnode.type = types[TDOUBLE]; + + nodsafe = new(ONAME, Z, Z); + nodsafe->sym = slookup(".safe"); + nodsafe->type = types[TINT]; + nodsafe->etype = types[TINT]->etype; + nodsafe->class = CAUTO; + complex(nodsafe); + + t = typ(TARRAY, types[TCHAR]); + symrathole = slookup(".rathole"); + symrathole->class = CGLOBL; + symrathole->type = t; + + nodrat = new(ONAME, Z, Z); + nodrat->sym = symrathole; + nodrat->type = types[TIND]; + nodrat->etype = TVOID; + nodrat->class = CGLOBL; + complex(nodrat); + nodrat->type = t; + + nodret = new(ONAME, Z, Z); + nodret->sym = slookup(".ret"); + nodret->type = types[TIND]; + nodret->etype = TIND; + nodret->class = CPARAM; + nodret = new(OIND, nodret, Z); + complex(nodret); + + com64init(); + + memset(reg, 0, sizeof(reg)); +} + +void +gclean(void) +{ + int i; + Sym *s; + + for(i=0; itype->width = nstring; + symrathole->type->width = nrathole; + for(i=0; ilink) { + if(s->type == T) + continue; + if(s->type->width == 0) + continue; + if(s->class != CGLOBL && s->class != CSTATIC) + continue; + if(s->type == types[TENUM]) + continue; + gpseudo(AGLOBL, s, nodconst(s->type->width)); + } + nextpc(); + p->as = AEND; + outcode(); +} + +void +nextpc(void) +{ + + p = alloc(sizeof(*p)); + *p = zprog; + p->lineno = nearln; + pc++; + if(firstp == P) { + firstp = p; + lastp = p; + return; + } + lastp->link = p; + lastp = p; +} + +void +gargs(Node *n, Node *tn1, Node *tn2) +{ + int32 regs; + Node fnxargs[20], *fnxp; + + regs = cursafe; + + fnxp = fnxargs; + garg1(n, tn1, tn2, 0, &fnxp); /* compile fns to temps */ + + curarg = 0; + fnxp = fnxargs; + garg1(n, tn1, tn2, 1, &fnxp); /* compile normal args and temps */ + + cursafe = regs; +} + +void +garg1(Node *n, Node *tn1, Node *tn2, int f, Node **fnxp) +{ + Node nod; + + if(n == Z) + return; + if(n->op == OLIST) { + garg1(n->left, tn1, tn2, f, fnxp); + garg1(n->right, tn1, tn2, f, fnxp); + return; + } + if(f == 0) { + if(n->complex >= FNX) { + regsalloc(*fnxp, n); + nod = znode; + nod.op = OAS; + nod.left = *fnxp; + nod.right = n; + nod.type = n->type; + cgen(&nod, Z); + (*fnxp)++; + } + return; + } + if(typesuv[n->type->etype]) { + regaalloc(tn2, n); + if(n->complex >= FNX) { + sugen(*fnxp, tn2, n->type->width); + (*fnxp)++; + } else + sugen(n, tn2, n->type->width); + return; + } + if(REGARG >= 0 && curarg == 0 && typechlp[n->type->etype]) { + regaalloc1(tn1, n); + if(n->complex >= FNX) { + cgen(*fnxp, tn1); + (*fnxp)++; + } else + cgen(n, tn1); + return; + } + regalloc(tn1, n, Z); + if(n->complex >= FNX) { + cgen(*fnxp, tn1); + (*fnxp)++; + } else + cgen(n, tn1); + regaalloc(tn2, n); + gopcode(OAS, tn1, Z, tn2); + regfree(tn1); +} + +Node* +nodconst(int32 v) +{ + constnode.vconst = v; + return &constnode; +} + +Node* +nod32const(vlong v) +{ + constnode.vconst = v & MASK(32); + return &constnode; +} + +Node* +nodfconst(double d) +{ + fconstnode.fconst = d; + return &fconstnode; +} + +void +nodreg(Node *n, Node *nn, int reg) +{ + *n = regnode; + n->reg = reg; + n->type = nn->type; + n->lineno = nn->lineno; +} + +void +regret(Node *n, Node *nn) +{ + int r; + + r = REGRET; + if(typefd[nn->type->etype]) + r = FREGRET+NREG; + nodreg(n, nn, r); + reg[r]++; +} + +int +tmpreg(void) +{ + int i; + + for(i=REGRET+1; itype->etype) { + case TCHAR: + case TUCHAR: + case TSHORT: + case TUSHORT: + case TINT: + case TUINT: + case TLONG: + case TULONG: + case TIND: + if(o != Z && o->op == OREGISTER) { + i = o->reg; + if(i >= 0 && i < NREG) + goto out; + } + for(i=REGRET+1; i<=REGEXT-2; i++) + if(reg[i] == 0) + goto out; + diag(tn, "out of fixed registers"); + goto err; + + case TFLOAT: + case TDOUBLE: + case TVLONG: + if(o != Z && o->op == OREGISTER) { + i = o->reg; + if(i >= NREG && i < NREG+NFREG) + goto out; + } + for(i=NREG; itype); +err: + nodreg(n, tn, 0); + return; +out: + reg[i]++; + nodreg(n, tn, i); +} + +void +regialloc(Node *n, Node *tn, Node *o) +{ + Node nod; + + nod = *tn; + nod.type = types[TIND]; + regalloc(n, &nod, o); +} + +void +regfree(Node *n) +{ + int i; + + i = 0; + if(n->op != OREGISTER && n->op != OINDREG) + goto err; + i = n->reg; + if(i < 0 || i >= sizeof(reg)) + goto err; + if(reg[i] <= 0) + goto err; + reg[i]--; + return; +err: + diag(n, "error in regfree: %d", i); +} + +void +regsalloc(Node *n, Node *nn) +{ + cursafe = align(cursafe, nn->type, Aaut3, nil); + maxargsafe = maxround(maxargsafe, cursafe+curarg); + *n = *nodsafe; + n->xoffset = -(stkoff + cursafe); + n->type = nn->type; + n->etype = nn->type->etype; + n->lineno = nn->lineno; +} + +void +regaalloc1(Node *n, Node *nn) +{ + if(REGARG < 0) { + fatal(n, "regaalloc1 and REGARG<0"); + return; + } + nodreg(n, nn, REGARG); + reg[REGARG]++; + curarg = align(curarg, nn->type, Aarg1, nil); + curarg = align(curarg, nn->type, Aarg2, nil); + maxargsafe = maxround(maxargsafe, cursafe+curarg); +} + +void +regaalloc(Node *n, Node *nn) +{ + curarg = align(curarg, nn->type, Aarg1, nil); + *n = *nn; + n->op = OINDREG; + n->reg = REGSP; + n->xoffset = curarg + SZ_LONG; + n->complex = 0; + n->addable = 20; + curarg = align(curarg, nn->type, Aarg2, nil); + maxargsafe = maxround(maxargsafe, cursafe+curarg); +} + +void +regind(Node *n, Node *nn) +{ + + if(n->op != OREGISTER) { + diag(n, "regind not OREGISTER"); + return; + } + n->op = OINDREG; + n->type = nn->type; +} + +void +raddr(Node *n, Prog *p) +{ + Adr a; + + naddr(n, &a); + if(R0ISZERO && a.type == D_CONST && a.offset == 0) { + a.type = D_REG; + a.reg = 0; + } + if(a.type != D_REG && a.type != D_FREG) { + if(n) + diag(n, "bad in raddr: %O", n->op); + else + diag(n, "bad in raddr: "); + p->reg = NREG; + } else + p->reg = a.reg; +} + +void +naddr(Node *n, Adr *a) +{ + int32 v; + + a->type = D_NONE; + if(n == Z) + return; + switch(n->op) { + default: + bad: + diag(n, "bad in naddr: %O", n->op); + break; + + case OREGISTER: + a->type = D_REG; + a->sym = S; + a->reg = n->reg; + if(a->reg >= NREG) { + a->type = D_FREG; + a->reg -= NREG; + } + break; + + case OIND: + naddr(n->left, a); + if(a->type == D_REG) { + a->type = D_OREG; + break; + } + if(a->type == D_CONST) { + a->type = D_OREG; + break; + } + goto bad; + + case OINDREG: + a->type = D_OREG; + a->sym = S; + a->offset = n->xoffset; + a->reg = n->reg; + break; + + case ONAME: + a->etype = n->etype; + a->type = D_OREG; + a->name = D_STATIC; + a->sym = n->sym; + a->offset = n->xoffset; + if(n->class == CSTATIC) + break; + if(n->class == CEXTERN || n->class == CGLOBL) { + a->name = D_EXTERN; + break; + } + if(n->class == CAUTO) { + a->name = D_AUTO; + break; + } + if(n->class == CPARAM) { + a->name = D_PARAM; + break; + } + goto bad; + + case OCONST: + a->sym = S; + a->reg = NREG; + if(typefd[n->type->etype]) { + a->type = D_FCONST; + a->dval = n->fconst; + } else { + a->type = D_CONST; + a->offset = n->vconst; + } + break; + + case OADDR: + naddr(n->left, a); + if(a->type == D_OREG) { + a->type = D_CONST; + break; + } + goto bad; + + case OADD: + if(n->left->op == OCONST) { + naddr(n->left, a); + v = a->offset; + naddr(n->right, a); + } else { + naddr(n->right, a); + v = a->offset; + naddr(n->left, a); + } + a->offset += v; + break; + + } +} + +void +fop(int as, int f1, int f2, Node *t) +{ + Node nod1, nod2, nod3; + + nodreg(&nod1, t, NREG+f1); + nodreg(&nod2, t, NREG+f2); + regalloc(&nod3, t, t); + gopcode(as, &nod1, &nod2, &nod3); + gmove(&nod3, t); + regfree(&nod3); +} + +void +gmovm(Node *f, Node *t, int w) +{ + gins(AMOVM, f, t); + p->scond |= C_UBIT; + if(w) + p->scond |= C_WBIT; +} + +void +gmove(Node *f, Node *t) +{ + int ft, tt, a; + Node nod, nod1; + Prog *p1; + + ft = f->type->etype; + tt = t->type->etype; + + if(ft == TDOUBLE && f->op == OCONST) { + } + if(ft == TFLOAT && f->op == OCONST) { + } + + /* + * a load -- + * put it into a register then + * worry what to do with it. + */ + if(f->op == ONAME || f->op == OINDREG || f->op == OIND) { + switch(ft) { + default: + a = AMOVW; + break; + case TFLOAT: + a = AMOVF; + break; + case TDOUBLE: + a = AMOVD; + break; + case TCHAR: + a = AMOVB; + break; + case TUCHAR: + a = AMOVBU; + break; + case TSHORT: + a = AMOVH; + break; + case TUSHORT: + a = AMOVHU; + break; + } + if(typechlp[ft] && typeilp[tt]) + regalloc(&nod, t, t); + else + regalloc(&nod, f, t); + gins(a, f, &nod); + gmove(&nod, t); + regfree(&nod); + return; + } + + /* + * a store -- + * put it into a register then + * store it. + */ + if(t->op == ONAME || t->op == OINDREG || t->op == OIND) { + switch(tt) { + default: + a = AMOVW; + break; + case TUCHAR: + a = AMOVBU; + break; + case TCHAR: + a = AMOVB; + break; + case TUSHORT: + a = AMOVHU; + break; + case TSHORT: + a = AMOVH; + break; + case TFLOAT: + a = AMOVF; + break; + case TVLONG: + case TDOUBLE: + a = AMOVD; + break; + } + if(ft == tt) + regalloc(&nod, t, f); + else + regalloc(&nod, t, Z); + gmove(f, &nod); + gins(a, &nod, t); + regfree(&nod); + return; + } + + /* + * type x type cross table + */ + a = AGOK; + switch(ft) { + case TDOUBLE: + case TVLONG: + case TFLOAT: + switch(tt) { + case TDOUBLE: + case TVLONG: + a = AMOVD; + if(ft == TFLOAT) + a = AMOVFD; + break; + case TFLOAT: + a = AMOVDF; + if(ft == TFLOAT) + a = AMOVF; + break; + case TINT: + case TUINT: + case TLONG: + case TULONG: + case TIND: + a = AMOVDW; + if(ft == TFLOAT) + a = AMOVFW; + break; + case TSHORT: + case TUSHORT: + case TCHAR: + case TUCHAR: + a = AMOVDW; + if(ft == TFLOAT) + a = AMOVFW; + break; + } + break; + case TUINT: + case TULONG: + if(tt == TFLOAT || tt == TDOUBLE) { + // ugly and probably longer than necessary, + // but vfp has a single instruction for this, + // so hopefully it won't last long. + // + // tmp = f + // tmp1 = tmp & 0x80000000 + // tmp ^= tmp1 + // t = float(int32(tmp)) + // if(tmp1) + // t += 2147483648. + // + regalloc(&nod, f, Z); + regalloc(&nod1, f, Z); + gins(AMOVW, f, &nod); + gins(AMOVW, &nod, &nod1); + gins(AAND, nodconst(0x80000000), &nod1); + gins(AEOR, &nod1, &nod); + if(tt == TFLOAT) + gins(AMOVWF, &nod, t); + else + gins(AMOVWD, &nod, t); + gins(ACMP, nodconst(0), Z); + raddr(&nod1, p); + gins(ABEQ, Z, Z); + regfree(&nod); + regfree(&nod1); + p1 = p; + regalloc(&nod, t, Z); + gins(AMOVF, nodfconst(2147483648.), &nod); + gins(AADDF, &nod, t); + regfree(&nod); + patch(p1, pc); + return; + } + // fall through + + case TINT: + case TLONG: + case TIND: + switch(tt) { + case TDOUBLE: + gins(AMOVWD, f, t); + return; + case TFLOAT: + gins(AMOVWF, f, t); + return; + case TINT: + case TUINT: + case TLONG: + case TULONG: + case TIND: + case TSHORT: + case TUSHORT: + case TCHAR: + case TUCHAR: + a = AMOVW; + break; + } + break; + case TSHORT: + switch(tt) { + case TDOUBLE: + regalloc(&nod, f, Z); + gins(AMOVH, f, &nod); + gins(AMOVWD, &nod, t); + regfree(&nod); + return; + case TFLOAT: + regalloc(&nod, f, Z); + gins(AMOVH, f, &nod); + gins(AMOVWF, &nod, t); + regfree(&nod); + return; + case TUINT: + case TINT: + case TULONG: + case TLONG: + case TIND: + a = AMOVH; + break; + case TSHORT: + case TUSHORT: + case TCHAR: + case TUCHAR: + a = AMOVW; + break; + } + break; + case TUSHORT: + switch(tt) { + case TDOUBLE: + regalloc(&nod, f, Z); + gins(AMOVHU, f, &nod); + gins(AMOVWD, &nod, t); + regfree(&nod); + return; + case TFLOAT: + regalloc(&nod, f, Z); + gins(AMOVHU, f, &nod); + gins(AMOVWF, &nod, t); + regfree(&nod); + return; + case TINT: + case TUINT: + case TLONG: + case TULONG: + case TIND: + a = AMOVHU; + break; + case TSHORT: + case TUSHORT: + case TCHAR: + case TUCHAR: + a = AMOVW; + break; + } + break; + case TCHAR: + switch(tt) { + case TDOUBLE: + regalloc(&nod, f, Z); + gins(AMOVB, f, &nod); + gins(AMOVWD, &nod, t); + regfree(&nod); + return; + case TFLOAT: + regalloc(&nod, f, Z); + gins(AMOVB, f, &nod); + gins(AMOVWF, &nod, t); + regfree(&nod); + return; + case TINT: + case TUINT: + case TLONG: + case TULONG: + case TIND: + case TSHORT: + case TUSHORT: + a = AMOVB; + break; + case TCHAR: + case TUCHAR: + a = AMOVW; + break; + } + break; + case TUCHAR: + switch(tt) { + case TDOUBLE: + regalloc(&nod, f, Z); + gins(AMOVBU, f, &nod); + gins(AMOVWD, &nod, t); + regfree(&nod); + return; + case TFLOAT: + regalloc(&nod, f, Z); + gins(AMOVBU, f, &nod); + gins(AMOVWF, &nod, t); + regfree(&nod); + return; + case TINT: + case TUINT: + case TLONG: + case TULONG: + case TIND: + case TSHORT: + case TUSHORT: + a = AMOVBU; + break; + case TCHAR: + case TUCHAR: + a = AMOVW; + break; + } + break; + } + if(a == AGOK) + diag(Z, "bad opcode in gmove %T -> %T", f->type, t->type); + if(a == AMOVW || a == AMOVF || a == AMOVD) + if(samaddr(f, t)) + return; + gins(a, f, t); +} + +void +gmover(Node *f, Node *t) +{ + int ft, tt, a; + + ft = f->type->etype; + tt = t->type->etype; + a = AGOK; + if(typechlp[ft] && typechlp[tt] && ewidth[ft] >= ewidth[tt]){ + switch(tt){ + case TSHORT: + a = AMOVH; + break; + case TUSHORT: + a = AMOVHU; + break; + case TCHAR: + a = AMOVB; + break; + case TUCHAR: + a = AMOVBU; + break; + } + } + if(a == AGOK) + gmove(f, t); + else + gins(a, f, t); +} + +void +gins(int a, Node *f, Node *t) +{ + + nextpc(); + p->as = a; + if(f != Z) + naddr(f, &p->from); + if(t != Z) + naddr(t, &p->to); + if(debug['g']) + print("%P\n", p); +} + +void +gopcode(int o, Node *f1, Node *f2, Node *t) +{ + int a, et; + Adr ta; + + et = TLONG; + if(f1 != Z && f1->type != T) + et = f1->type->etype; + a = AGOK; + switch(o) { + case OAS: + gmove(f1, t); + return; + + case OASADD: + case OADD: + a = AADD; + if(et == TFLOAT) + a = AADDF; + else + if(et == TDOUBLE || et == TVLONG) + a = AADDD; + break; + + case OASSUB: + case OSUB: + if(f2 && f2->op == OCONST) { + Node *t = f1; + f1 = f2; + f2 = t; + a = ARSB; + } else + a = ASUB; + if(et == TFLOAT) + a = ASUBF; + else + if(et == TDOUBLE || et == TVLONG) + a = ASUBD; + break; + + case OASOR: + case OOR: + a = AORR; + break; + + case OASAND: + case OAND: + a = AAND; + break; + + case OASXOR: + case OXOR: + a = AEOR; + break; + + case OASLSHR: + case OLSHR: + a = ASRL; + break; + + case OASASHR: + case OASHR: + a = ASRA; + break; + + case OASASHL: + case OASHL: + a = ASLL; + break; + + case OFUNC: + a = ABL; + break; + + case OASMUL: + case OMUL: + a = AMUL; + if(et == TFLOAT) + a = AMULF; + else + if(et == TDOUBLE || et == TVLONG) + a = AMULD; + break; + + case OASDIV: + case ODIV: + a = ADIV; + if(et == TFLOAT) + a = ADIVF; + else + if(et == TDOUBLE || et == TVLONG) + a = ADIVD; + break; + + case OASMOD: + case OMOD: + a = AMOD; + break; + + case OASLMUL: + case OLMUL: + a = AMULU; + break; + + case OASLMOD: + case OLMOD: + a = AMODU; + break; + + case OASLDIV: + case OLDIV: + a = ADIVU; + break; + + case OCASE: + case OEQ: + case ONE: + case OLT: + case OLE: + case OGE: + case OGT: + case OLO: + case OLS: + case OHS: + case OHI: + a = ACMP; + if(et == TFLOAT) + a = ACMPF; + else + if(et == TDOUBLE || et == TVLONG) + a = ACMPD; + nextpc(); + p->as = a; + naddr(f1, &p->from); + if(a == ACMP && f1->op == OCONST && p->from.offset < 0) { + p->as = ACMN; + p->from.offset = -p->from.offset; + } + raddr(f2, p); + switch(o) { + case OEQ: + a = ABEQ; + break; + case ONE: + a = ABNE; + break; + case OLT: + a = ABLT; + break; + case OLE: + a = ABLE; + break; + case OGE: + a = ABGE; + break; + case OGT: + a = ABGT; + break; + case OLO: + a = ABLO; + break; + case OLS: + a = ABLS; + break; + case OHS: + a = ABHS; + break; + case OHI: + a = ABHI; + break; + case OCASE: + nextpc(); + p->as = ACASE; + p->scond = 0x9; + naddr(f2, &p->from); + a = ABHI; + break; + } + f1 = Z; + f2 = Z; + break; + } + if(a == AGOK) + diag(Z, "bad in gopcode %O", o); + nextpc(); + p->as = a; + if(f1 != Z) + naddr(f1, &p->from); + if(f2 != Z) { + naddr(f2, &ta); + p->reg = ta.reg; + } + if(t != Z) + naddr(t, &p->to); + if(debug['g']) + print("%P\n", p); +} + +int +samaddr(Node *f, Node *t) +{ + + if(f->op != t->op) + return 0; + switch(f->op) { + + case OREGISTER: + if(f->reg != t->reg) + break; + return 1; + } + return 0; +} + +void +gbranch(int o) +{ + int a; + + a = AGOK; + switch(o) { + case ORETURN: + a = ARET; + break; + case OGOTO: + a = AB; + break; + } + nextpc(); + if(a == AGOK) { + diag(Z, "bad in gbranch %O", o); + nextpc(); + } + p->as = a; +} + +void +patch(Prog *op, int32 pc) +{ + + op->to.offset = pc; + op->to.type = D_BRANCH; +} + +void +gpseudo(int a, Sym *s, Node *n) +{ + + nextpc(); + p->as = a; + p->from.type = D_OREG; + p->from.sym = s; + p->from.name = D_EXTERN; + if(a == ATEXT) { + p->reg = textflag; + textflag = 0; + } + if(s->class == CSTATIC) + p->from.name = D_STATIC; + naddr(n, &p->to); + if(a == ADATA || a == AGLOBL) + pc--; +} + +int +sconst(Node *n) +{ + vlong vv; + + if(n->op == OCONST) { + if(!typefd[n->type->etype]) { + vv = n->vconst; + if(vv >= (vlong)(-32766) && vv < (vlong)32766) + return 1; + /* + * should be specialised for constant values which will + * fit in different instructionsl; for now, let 5l + * sort it out + */ + return 1; + } + } + return 0; +} + +int +sval(int32 v) +{ + int i; + + for(i=0; i<16; i++) { + if((v & ~0xff) == 0) + return 1; + if((~v & ~0xff) == 0) + return 1; + v = (v<<2) | ((uint32)v>>30); + } + return 0; +} + +int32 +exreg(Type *t) +{ + int32 o; + + if(typechlp[t->etype]) { + if(exregoffset <= REGEXT-4) + return 0; + o = exregoffset; + exregoffset--; + return o; + } + if(typefd[t->etype]) { + if(exfregoffset <= NFREG-1) + return 0; + o = exfregoffset + NREG; + exfregoffset--; + return o; + } + return 0; +} + +schar ewidth[NTYPE] = +{ + -1, /* [TXXX] */ + SZ_CHAR, /* [TCHAR] */ + SZ_CHAR, /* [TUCHAR] */ + SZ_SHORT, /* [TSHORT] */ + SZ_SHORT, /* [TUSHORT] */ + SZ_INT, /* [TINT] */ + SZ_INT, /* [TUINT] */ + SZ_LONG, /* [TLONG] */ + SZ_LONG, /* [TULONG] */ + SZ_VLONG, /* [TVLONG] */ + SZ_VLONG, /* [TUVLONG] */ + SZ_FLOAT, /* [TFLOAT] */ + SZ_DOUBLE, /* [TDOUBLE] */ + SZ_IND, /* [TIND] */ + 0, /* [TFUNC] */ + -1, /* [TARRAY] */ + 0, /* [TVOID] */ + -1, /* [TSTRUCT] */ + -1, /* [TUNION] */ + SZ_INT, /* [TENUM] */ +}; + +int32 ncast[NTYPE] = +{ + 0, /* [TXXX] */ + BCHAR|BUCHAR, /* [TCHAR] */ + BCHAR|BUCHAR, /* [TUCHAR] */ + BSHORT|BUSHORT, /* [TSHORT] */ + BSHORT|BUSHORT, /* [TUSHORT] */ + BINT|BUINT|BLONG|BULONG|BIND, /* [TINT] */ + BINT|BUINT|BLONG|BULONG|BIND, /* [TUINT] */ + BINT|BUINT|BLONG|BULONG|BIND, /* [TLONG] */ + BINT|BUINT|BLONG|BULONG|BIND, /* [TULONG] */ + BVLONG|BUVLONG, /* [TVLONG] */ + BVLONG|BUVLONG, /* [TUVLONG] */ + BFLOAT, /* [TFLOAT] */ + BDOUBLE, /* [TDOUBLE] */ + BLONG|BULONG|BIND, /* [TIND] */ + 0, /* [TFUNC] */ + 0, /* [TARRAY] */ + 0, /* [TVOID] */ + BSTRUCT, /* [TSTRUCT] */ + BUNION, /* [TUNION] */ + 0, /* [TENUM] */ +}; diff --git a/src/cmd/5g/Makefile b/src/cmd/5g/Makefile new file mode 100644 index 000000000..b47014a4e --- /dev/null +++ b/src/cmd/5g/Makefile @@ -0,0 +1,36 @@ +# Copyright 2009 The Go Authors. All rights reserved. +# Use of this source code is governed by a BSD-style +# license that can be found in the LICENSE file. + +include ../../Make.inc +O:=$(HOST_O) + +TARG=5g + +HFILES=\ + ../gc/go.h\ + ../5l/5.out.h\ + gg.h\ + opt.h\ + +OFILES=\ + ../5l/enam.$O\ + cgen.$O\ + cgen64.$O\ + cplx.$O\ + galign.$O\ + ggen.$O\ + gobj.$O\ + gsubr.$O\ + list.$O\ + peep.$O\ + pgen.$O\ + reg.$O\ + +LIB=\ + ../gc/gc.a\ + +include ../../Make.ccmd + +%.$O: ../gc/%.c + $(HOST_CC) $(HOST_CFLAGS) -c -I. -o $@ ../gc/$*.c diff --git a/src/cmd/5g/cgen.c b/src/cmd/5g/cgen.c new file mode 100644 index 000000000..6e2fbe20f --- /dev/null +++ b/src/cmd/5g/cgen.c @@ -0,0 +1,1328 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +#include "gg.h" + +/* + * generate: + * res = n; + * simplifies and calls gmove. + */ +void +cgen(Node *n, Node *res) +{ + Node *nl, *nr, *r; + Node n1, n2, n3, f0, f1; + int a, w; + Prog *p1, *p2, *p3; + Addr addr; + + if(debug['g']) { + dump("\ncgen-n", n); + dump("cgen-res", res); + } + if(n == N || n->type == T) + goto ret; + + if(res == N || res->type == T) + fatal("cgen: res nil"); + + while(n->op == OCONVNOP) + n = n->left; + + if(n->ullman >= UINF) { + if(n->op == OINDREG) + fatal("cgen: this is going to misscompile"); + if(res->ullman >= UINF) { + tempname(&n1, n->type); + cgen(n, &n1); + cgen(&n1, res); + goto ret; + } + } + + if(isfat(n->type)) { + if(n->type->width < 0) + fatal("forgot to compute width for %T", n->type); + sgen(n, res, n->type->width); + goto ret; + } + + + // update addressability for string, slice + // can't do in walk because n->left->addable + // changes if n->left is an escaping local variable. + switch(n->op) { + case OLEN: + if(isslice(n->left->type) || istype(n->left->type, TSTRING)) + n->addable = n->left->addable; + break; + case OCAP: + if(isslice(n->left->type)) + n->addable = n->left->addable; + break; + } + + // if both are addressable, move + if(n->addable && res->addable) { + if(is64(n->type) || is64(res->type) || + n->op == OREGISTER || res->op == OREGISTER || + iscomplex[n->type->etype] || iscomplex[res->type->etype]) { + gmove(n, res); + } else { + regalloc(&n1, n->type, N); + gmove(n, &n1); + cgen(&n1, res); + regfree(&n1); + } + goto ret; + } + + // if both are not addressable, use a temporary. + if(!n->addable && !res->addable) { + // could use regalloc here sometimes, + // but have to check for ullman >= UINF. + tempname(&n1, n->type); + cgen(n, &n1); + cgen(&n1, res); + return; + } + + // if result is not addressable directly but n is, + // compute its address and then store via the address. + if(!res->addable) { + igen(res, &n1, N); + cgen(n, &n1); + regfree(&n1); + return; + } + + if(complexop(n, res)) { + complexgen(n, res); + return; + } + + // if n is sudoaddable generate addr and move + if (!is64(n->type) && !is64(res->type) && !iscomplex[n->type->etype] && !iscomplex[res->type->etype]) { + a = optoas(OAS, n->type); + if(sudoaddable(a, n, &addr, &w)) { + if (res->op != OREGISTER) { + regalloc(&n2, res->type, N); + p1 = gins(a, N, &n2); + p1->from = addr; + if(debug['g']) + print("%P [ignore previous line]\n", p1); + gmove(&n2, res); + regfree(&n2); + } else { + p1 = gins(a, N, res); + p1->from = addr; + if(debug['g']) + print("%P [ignore previous line]\n", p1); + } + sudoclean(); + goto ret; + } + } + + // otherwise, the result is addressable but n is not. + // let's do some computation. + + nl = n->left; + nr = n->right; + + if(nl != N && nl->ullman >= UINF) + if(nr != N && nr->ullman >= UINF) { + tempname(&n1, nl->type); + cgen(nl, &n1); + n2 = *n; + n2.left = &n1; + cgen(&n2, res); + goto ret; + } + + // 64-bit ops are hard on 32-bit machine. + if(is64(n->type) || is64(res->type) || n->left != N && is64(n->left->type)) { + switch(n->op) { + // math goes to cgen64. + case OMINUS: + case OCOM: + case OADD: + case OSUB: + case OMUL: + case OLSH: + case ORSH: + case OAND: + case OOR: + case OXOR: + cgen64(n, res); + return; + } + } + + if(nl != N && isfloat[n->type->etype] && isfloat[nl->type->etype]) + goto flt; + switch(n->op) { + default: + dump("cgen", n); + fatal("cgen: unknown op %N", n); + break; + + case OREAL: + case OIMAG: + case OCOMPLEX: + fatal("unexpected complex"); + break; + + // these call bgen to get a bool value + case OOROR: + case OANDAND: + case OEQ: + case ONE: + case OLT: + case OLE: + case OGE: + case OGT: + case ONOT: + p1 = gbranch(AB, T); + p2 = pc; + gmove(nodbool(1), res); + p3 = gbranch(AB, T); + patch(p1, pc); + bgen(n, 1, p2); + gmove(nodbool(0), res); + patch(p3, pc); + goto ret; + + case OPLUS: + cgen(nl, res); + goto ret; + + // unary + case OCOM: + a = optoas(OXOR, nl->type); + regalloc(&n1, nl->type, N); + cgen(nl, &n1); + nodconst(&n2, nl->type, -1); + gins(a, &n2, &n1); + gmove(&n1, res); + regfree(&n1); + goto ret; + + case OMINUS: + nodconst(&n3, nl->type, 0); + regalloc(&n2, nl->type, res); + regalloc(&n1, nl->type, N); + gmove(&n3, &n2); + cgen(nl, &n1); + gins(optoas(OSUB, nl->type), &n1, &n2); + gmove(&n2, res); + regfree(&n1); + regfree(&n2); + goto ret; + + // symmetric binary + case OAND: + case OOR: + case OXOR: + case OADD: + case OMUL: + a = optoas(n->op, nl->type); + goto sbop; + + // asymmetric binary + case OSUB: + a = optoas(n->op, nl->type); + goto abop; + + case OLSH: + case ORSH: + cgen_shift(n->op, nl, nr, res); + break; + + case OCONV: + if(eqtype(n->type, nl->type) || noconv(n->type, nl->type)) { + cgen(nl, res); + break; + } + if(nl->addable && !is64(nl->type)) { + regalloc(&n1, nl->type, res); + gmove(nl, &n1); + } else { + if(n->type->width > widthptr || is64(nl->type) || isfloat[nl->type->etype]) + tempname(&n1, nl->type); + else + regalloc(&n1, nl->type, res); + cgen(nl, &n1); + } + if(n->type->width > widthptr || is64(n->type) || isfloat[n->type->etype]) + tempname(&n2, n->type); + else + regalloc(&n2, n->type, N); + gmove(&n1, &n2); + gmove(&n2, res); + if(n1.op == OREGISTER) + regfree(&n1); + if(n2.op == OREGISTER) + regfree(&n2); + break; + + case ODOT: + case ODOTPTR: + case OINDEX: + case OIND: + case ONAME: // PHEAP or PPARAMREF var + igen(n, &n1, res); + gmove(&n1, res); + regfree(&n1); + break; + + case OLEN: + if(istype(nl->type, TMAP) || istype(nl->type, TCHAN)) { + // map has len in the first 32-bit word. + // a zero pointer means zero length + regalloc(&n1, types[tptr], res); + cgen(nl, &n1); + + nodconst(&n2, types[tptr], 0); + regalloc(&n3, n2.type, N); + gmove(&n2, &n3); + gcmp(optoas(OCMP, types[tptr]), &n1, &n3); + regfree(&n3); + p1 = gbranch(optoas(OEQ, types[tptr]), T); + + n2 = n1; + n2.op = OINDREG; + n2.type = types[TINT32]; + gmove(&n2, &n1); + + patch(p1, pc); + + gmove(&n1, res); + regfree(&n1); + break; + } + if(istype(nl->type, TSTRING) || isslice(nl->type)) { + // both slice and string have len one pointer into the struct. + igen(nl, &n1, res); + n1.op = OREGISTER; // was OINDREG + regalloc(&n2, types[TUINT32], &n1); + n1.op = OINDREG; + n1.type = types[TUINT32]; + n1.xoffset = Array_nel; + gmove(&n1, &n2); + gmove(&n2, res); + regfree(&n1); + regfree(&n2); + break; + } + fatal("cgen: OLEN: unknown type %lT", nl->type); + break; + + case OCAP: + if(istype(nl->type, TCHAN)) { + // chan has cap in the second 32-bit word. + // a zero pointer means zero length + regalloc(&n1, types[tptr], res); + cgen(nl, &n1); + + nodconst(&n2, types[tptr], 0); + regalloc(&n3, n2.type, N); + gmove(&n2, &n3); + gcmp(optoas(OCMP, types[tptr]), &n1, &n3); + regfree(&n3); + p1 = gbranch(optoas(OEQ, types[tptr]), T); + + n2 = n1; + n2.op = OINDREG; + n2.xoffset = 4; + n2.type = types[TINT32]; + gmove(&n2, &n1); + + patch(p1, pc); + + gmove(&n1, res); + regfree(&n1); + break; + } + if(isslice(nl->type)) { + regalloc(&n1, types[tptr], res); + agen(nl, &n1); + n1.op = OINDREG; + n1.type = types[TUINT32]; + n1.xoffset = Array_cap; + gmove(&n1, res); + regfree(&n1); + break; + } + fatal("cgen: OCAP: unknown type %lT", nl->type); + break; + + case OADDR: + agen(nl, res); + break; + + case OCALLMETH: + cgen_callmeth(n, 0); + cgen_callret(n, res); + break; + + case OCALLINTER: + cgen_callinter(n, res, 0); + cgen_callret(n, res); + break; + + case OCALLFUNC: + cgen_call(n, 0); + cgen_callret(n, res); + break; + + case OMOD: + case ODIV: + a = optoas(n->op, nl->type); + goto abop; + } + goto ret; + +sbop: // symmetric binary + if(nl->ullman < nr->ullman) { + r = nl; + nl = nr; + nr = r; + } + +abop: // asymmetric binary + // TODO(kaib): use fewer registers here. + if(nl->ullman >= nr->ullman) { + regalloc(&n1, nl->type, res); + cgen(nl, &n1); + regalloc(&n2, nr->type, N); + cgen(nr, &n2); + } else { + regalloc(&n2, nr->type, N); + cgen(nr, &n2); + regalloc(&n1, nl->type, res); + cgen(nl, &n1); + } + gins(a, &n2, &n1); + gmove(&n1, res); + regfree(&n1); + regfree(&n2); + goto ret; + +flt: // floating-point. + regalloc(&f0, nl->type, res); + if(nr != N) + goto flt2; + + if(n->op == OMINUS) { + nr = nodintconst(-1); + convlit(&nr, n->type); + n->op = OMUL; + goto flt2; + } + + // unary + cgen(nl, &f0); + if(n->op != OCONV && n->op != OPLUS) + gins(optoas(n->op, n->type), &f0, &f0); + gmove(&f0, res); + regfree(&f0); + goto ret; + +flt2: // binary + if(nl->ullman >= nr->ullman) { + cgen(nl, &f0); + regalloc(&f1, n->type, N); + gmove(&f0, &f1); + cgen(nr, &f0); + gins(optoas(n->op, n->type), &f0, &f1); + } else { + cgen(nr, &f0); + regalloc(&f1, n->type, N); + cgen(nl, &f1); + gins(optoas(n->op, n->type), &f0, &f1); + } + gmove(&f1, res); + regfree(&f0); + regfree(&f1); + goto ret; + +ret: + ; +} + +/* + * generate array index into res. + * n might be any size; res is 32-bit. + * returns Prog* to patch to panic call. + */ +Prog* +cgenindex(Node *n, Node *res) +{ + Node tmp, lo, hi, zero, n1, n2; + + if(!is64(n->type)) { + cgen(n, res); + return nil; + } + + tempname(&tmp, types[TINT64]); + cgen(n, &tmp); + split64(&tmp, &lo, &hi); + gmove(&lo, res); + if(debug['B']) { + splitclean(); + return nil; + } + regalloc(&n1, types[TINT32], N); + regalloc(&n2, types[TINT32], N); + nodconst(&zero, types[TINT32], 0); + gmove(&hi, &n1); + gmove(&zero, &n2); + gcmp(ACMP, &n1, &n2); + regfree(&n2); + regfree(&n1); + splitclean(); + return gbranch(ABNE, T); +} + +/* + * generate: + * res = &n; + */ +void +agen(Node *n, Node *res) +{ + Node *nl, *nr; + Node n1, n2, n3, n4, n5, tmp; + Prog *p1, *p2; + uint32 w; + uint64 v; + int r; + + if(debug['g']) { + dump("\nagen-res", res); + dump("agen-r", n); + } + if(n == N || n->type == T || res == N || res->type == T) + fatal("agen"); + + while(n->op == OCONVNOP) + n = n->left; + + if(n->addable) { + memset(&n1, 0, sizeof n1); + n1.op = OADDR; + n1.left = n; + regalloc(&n2, types[tptr], res); + gins(AMOVW, &n1, &n2); + gmove(&n2, res); + regfree(&n2); + goto ret; + } + + nl = n->left; + nr = n->right; + + switch(n->op) { + default: + fatal("agen: unknown op %N", n); + break; + + case OCALLMETH: + case OCALLFUNC: + // Release res so that it is available for cgen_call. + // Pick it up again after the call. + r = -1; + if(n->ullman >= UINF) { + if(res->op == OREGISTER || res->op == OINDREG) { + r = res->val.u.reg; + reg[r]--; + } + } + if(n->op == OCALLMETH) + cgen_callmeth(n, 0); + else + cgen_call(n, 0); + if(r >= 0) + reg[r]++; + cgen_aret(n, res); + break; + + case OCALLINTER: + cgen_callinter(n, res, 0); + cgen_aret(n, res); + break; + + case OINDEX: + p2 = nil; // to be patched to panicindex. + w = n->type->width; + if(nr->addable) { + if(!isconst(nr, CTINT)) + tempname(&tmp, types[TINT32]); + if(!isconst(nl, CTSTR)) + agenr(nl, &n3, res); + if(!isconst(nr, CTINT)) { + p2 = cgenindex(nr, &tmp); + regalloc(&n1, tmp.type, N); + gmove(&tmp, &n1); + } + } else + if(nl->addable) { + if(!isconst(nr, CTINT)) { + tempname(&tmp, types[TINT32]); + p2 = cgenindex(nr, &tmp); + regalloc(&n1, tmp.type, N); + gmove(&tmp, &n1); + } + if(!isconst(nl, CTSTR)) { + regalloc(&n3, types[tptr], res); + agen(nl, &n3); + } + } else { + tempname(&tmp, types[TINT32]); + p2 = cgenindex(nr, &tmp); + nr = &tmp; + if(!isconst(nl, CTSTR)) + agenr(nl, &n3, res); + regalloc(&n1, tmp.type, N); + gins(optoas(OAS, tmp.type), &tmp, &n1); + } + + // &a is in &n3 (allocated in res) + // i is in &n1 (if not constant) + // w is width + + // constant index + if(isconst(nr, CTINT)) { + if(isconst(nl, CTSTR)) + fatal("constant string constant index"); + v = mpgetfix(nr->val.u.xval); + if(isslice(nl->type) || nl->type->etype == TSTRING) { + if(!debug['B'] && !n->etype) { + n1 = n3; + n1.op = OINDREG; + n1.type = types[tptr]; + n1.xoffset = Array_nel; + regalloc(&n4, n1.type, N); + cgen(&n1, &n4); + nodconst(&n2, types[TUINT32], v); + regalloc(&n5, n2.type, N); + gmove(&n2, &n5); + gcmp(optoas(OCMP, types[TUINT32]), &n4, &n5); + regfree(&n4); + regfree(&n5); + p1 = gbranch(optoas(OGT, types[TUINT32]), T); + ginscall(panicindex, 0); + patch(p1, pc); + } + + n1 = n3; + n1.op = OINDREG; + n1.type = types[tptr]; + n1.xoffset = Array_array; + gmove(&n1, &n3); + } + + nodconst(&n2, types[tptr], v*w); + regalloc(&n4, n2.type, N); + gmove(&n2, &n4); + gins(optoas(OADD, types[tptr]), &n4, &n3); + regfree(&n4); + + gmove(&n3, res); + regfree(&n3); + break; + } + + regalloc(&n2, types[TINT32], &n1); // i + gmove(&n1, &n2); + regfree(&n1); + + if(!debug['B'] && !n->etype) { + // check bounds + regalloc(&n4, types[TUINT32], N); + if(isconst(nl, CTSTR)) { + nodconst(&n1, types[TUINT32], nl->val.u.sval->len); + gmove(&n1, &n4); + } else if(isslice(nl->type) || nl->type->etype == TSTRING) { + n1 = n3; + n1.op = OINDREG; + n1.type = types[tptr]; + n1.xoffset = Array_nel; + cgen(&n1, &n4); + } else { + nodconst(&n1, types[TUINT32], nl->type->bound); + gmove(&n1, &n4); + } + gcmp(optoas(OCMP, types[TUINT32]), &n2, &n4); + regfree(&n4); + p1 = gbranch(optoas(OLT, types[TUINT32]), T); + if(p2) + patch(p2, pc); + ginscall(panicindex, 0); + patch(p1, pc); + } + + if(isconst(nl, CTSTR)) { + regalloc(&n3, types[tptr], res); + p1 = gins(AMOVW, N, &n3); + datastring(nl->val.u.sval->s, nl->val.u.sval->len, &p1->from); + p1->from.type = D_CONST; + } else + if(isslice(nl->type) || nl->type->etype == TSTRING) { + n1 = n3; + n1.op = OINDREG; + n1.type = types[tptr]; + n1.xoffset = Array_array; + gmove(&n1, &n3); + } + + if(w == 0) { + // nothing to do + } else if(w == 1 || w == 2 || w == 4 || w == 8) { + memset(&n4, 0, sizeof n4); + n4.op = OADDR; + n4.left = &n2; + cgen(&n4, &n3); + if (w == 1) + gins(AADD, &n2, &n3); + else if(w == 2) + gshift(AADD, &n2, SHIFT_LL, 1, &n3); + else if(w == 4) + gshift(AADD, &n2, SHIFT_LL, 2, &n3); + else if(w == 8) + gshift(AADD, &n2, SHIFT_LL, 3, &n3); + } else { + regalloc(&n4, types[TUINT32], N); + nodconst(&n1, types[TUINT32], w); + gmove(&n1, &n4); + gins(optoas(OMUL, types[TUINT32]), &n4, &n2); + gins(optoas(OADD, types[tptr]), &n2, &n3); + regfree(&n4); + gmove(&n3, res); + } + + gmove(&n3, res); + regfree(&n2); + regfree(&n3); + break; + + case ONAME: + // should only get here with names in this func. + if(n->funcdepth > 0 && n->funcdepth != funcdepth) { + dump("bad agen", n); + fatal("agen: bad ONAME funcdepth %d != %d", + n->funcdepth, funcdepth); + } + + // should only get here for heap vars or paramref + if(!(n->class & PHEAP) && n->class != PPARAMREF) { + dump("bad agen", n); + fatal("agen: bad ONAME class %#x", n->class); + } + cgen(n->heapaddr, res); + if(n->xoffset != 0) { + nodconst(&n1, types[TINT32], n->xoffset); + regalloc(&n2, n1.type, N); + regalloc(&n3, types[TINT32], N); + gmove(&n1, &n2); + gmove(res, &n3); + gins(optoas(OADD, types[tptr]), &n2, &n3); + gmove(&n3, res); + regfree(&n2); + regfree(&n3); + } + break; + + case OIND: + cgen(nl, res); + break; + + case ODOT: + agen(nl, res); + if(n->xoffset != 0) { + nodconst(&n1, types[TINT32], n->xoffset); + regalloc(&n2, n1.type, N); + regalloc(&n3, types[TINT32], N); + gmove(&n1, &n2); + gmove(res, &n3); + gins(optoas(OADD, types[tptr]), &n2, &n3); + gmove(&n3, res); + regfree(&n2); + regfree(&n3); + } + break; + + case ODOTPTR: + cgen(nl, res); + if(n->xoffset != 0) { + // explicit check for nil if struct is large enough + // that we might derive too big a pointer. + if(nl->type->type->width >= unmappedzero) { + regalloc(&n1, types[tptr], N); + gmove(res, &n1); + p1 = gins(AMOVW, &n1, &n1); + p1->from.type = D_OREG; + p1->from.offset = 0; + regfree(&n1); + } + nodconst(&n1, types[TINT32], n->xoffset); + regalloc(&n2, n1.type, N); + regalloc(&n3, types[tptr], N); + gmove(&n1, &n2); + gmove(res, &n3); + gins(optoas(OADD, types[tptr]), &n2, &n3); + gmove(&n3, res); + regfree(&n2); + regfree(&n3); + } + break; + } + +ret: + ; +} + +/* + * generate: + * newreg = &n; + * res = newreg + * + * on exit, a has been changed to be *newreg. + * caller must regfree(a). + */ +void +igen(Node *n, Node *a, Node *res) +{ + regalloc(a, types[tptr], res); + agen(n, a); + a->op = OINDREG; + a->type = n->type; +} + +/* + * generate: + * newreg = &n; + * + * caller must regfree(a). + */ +void +agenr(Node *n, Node *a, Node *res) +{ + regalloc(a, types[tptr], res); + agen(n, a); +} + +void +gencmp0(Node *n, Type *t, int o, Prog *to) +{ + Node n1, n2, n3; + int a; + + regalloc(&n1, t, N); + cgen(n, &n1); + a = optoas(OCMP, t); + if(a != ACMP) { + nodconst(&n2, t, 0); + regalloc(&n3, t, N); + gmove(&n2, &n3); + gcmp(a, &n1, &n3); + regfree(&n3); + } else + gins(ATST, &n1, N); + a = optoas(o, t); + patch(gbranch(a, t), to); + regfree(&n1); +} + +/* + * generate: + * if(n == true) goto to; + */ +void +bgen(Node *n, int true, Prog *to) +{ + int et, a; + Node *nl, *nr, *r; + Node n1, n2, n3, n4, tmp; + Prog *p1, *p2; + + if(debug['g']) { + dump("\nbgen", n); + } + + if(n == N) + n = nodbool(1); + + if(n->ninit != nil) + genlist(n->ninit); + + nl = n->left; + nr = n->right; + + if(n->type == T) { + convlit(&n, types[TBOOL]); + if(n->type == T) + goto ret; + } + + et = n->type->etype; + if(et != TBOOL) { + yyerror("cgen: bad type %T for %O", n->type, n->op); + patch(gins(AEND, N, N), to); + goto ret; + } + nl = N; + nr = N; + + switch(n->op) { + default: + a = ONE; + if(!true) + a = OEQ; + gencmp0(n, n->type, a, to); + goto ret; + + case OLITERAL: + // need to ask if it is bool? + if(!true == !n->val.u.bval) + patch(gbranch(AB, T), to); + goto ret; + + case OANDAND: + if(!true) + goto caseor; + + caseand: + p1 = gbranch(AB, T); + p2 = gbranch(AB, T); + patch(p1, pc); + bgen(n->left, !true, p2); + bgen(n->right, !true, p2); + p1 = gbranch(AB, T); + patch(p1, to); + patch(p2, pc); + goto ret; + + case OOROR: + if(!true) + goto caseand; + + caseor: + bgen(n->left, true, to); + bgen(n->right, true, to); + goto ret; + + case OEQ: + case ONE: + case OLT: + case OGT: + case OLE: + case OGE: + nr = n->right; + if(nr == N || nr->type == T) + goto ret; + + case ONOT: // unary + nl = n->left; + if(nl == N || nl->type == T) + goto ret; + } + + switch(n->op) { + + case ONOT: + bgen(nl, !true, to); + goto ret; + + case OEQ: + case ONE: + case OLT: + case OGT: + case OLE: + case OGE: + a = n->op; + if(!true) { + if(isfloat[nl->type->etype]) { + // brcom is not valid on floats when NaN is involved. + p1 = gbranch(AB, T); + p2 = gbranch(AB, T); + patch(p1, pc); + bgen(n, 1, p2); + patch(gbranch(AB, T), to); + patch(p2, pc); + goto ret; + } + a = brcom(a); + true = !true; + } + + // make simplest on right + if(nl->op == OLITERAL || (nl->ullman < UINF && nl->ullman < nr->ullman)) { + a = brrev(a); + r = nl; + nl = nr; + nr = r; + } + + if(isslice(nl->type)) { + // only valid to cmp darray to literal nil + if((a != OEQ && a != ONE) || nr->op != OLITERAL) { + yyerror("illegal array comparison"); + break; + } + + regalloc(&n1, types[tptr], N); + agen(nl, &n1); + n2 = n1; + n2.op = OINDREG; + n2.xoffset = Array_array; + gencmp0(&n2, types[tptr], a, to); + regfree(&n1); + break; + + a = optoas(a, types[tptr]); + regalloc(&n1, types[tptr], N); + regalloc(&n3, types[tptr], N); + regalloc(&n4, types[tptr], N); + agen(nl, &n1); + n2 = n1; + n2.op = OINDREG; + n2.xoffset = Array_array; + gmove(&n2, &n4); + nodconst(&tmp, types[tptr], 0); + gmove(&tmp, &n3); + gcmp(optoas(OCMP, types[tptr]), &n4, &n3); + patch(gbranch(a, types[tptr]), to); + regfree(&n4); + regfree(&n3); + regfree(&n1); + break; + } + + if(isinter(nl->type)) { + // front end shold only leave cmp to literal nil + if((a != OEQ && a != ONE) || nr->op != OLITERAL) { + yyerror("illegal interface comparison"); + break; + } + + regalloc(&n1, types[tptr], N); + agen(nl, &n1); + n2 = n1; + n2.op = OINDREG; + n2.xoffset = 0; + gencmp0(&n2, types[tptr], a, to); + regfree(&n1); + break; + + a = optoas(a, types[tptr]); + regalloc(&n1, types[tptr], N); + regalloc(&n3, types[tptr], N); + regalloc(&n4, types[tptr], N); + agen(nl, &n1); + n2 = n1; + n2.op = OINDREG; + n2.xoffset = 0; + gmove(&n2, &n4); + nodconst(&tmp, types[tptr], 0); + gmove(&tmp, &n3); + gcmp(optoas(OCMP, types[tptr]), &n4, &n3); + patch(gbranch(a, types[tptr]), to); + regfree(&n1); + regfree(&n3); + regfree(&n4); + break; + } + + if(iscomplex[nl->type->etype]) { + complexbool(a, nl, nr, true, to); + break; + } + + if(is64(nr->type)) { + if(!nl->addable) { + tempname(&n1, nl->type); + cgen(nl, &n1); + nl = &n1; + } + if(!nr->addable) { + tempname(&n2, nr->type); + cgen(nr, &n2); + nr = &n2; + } + cmp64(nl, nr, a, to); + break; + } + + if(nr->op == OLITERAL) { + if(nr->val.ctype == CTINT && mpgetfix(nr->val.u.xval) == 0) { + gencmp0(nl, nl->type, a, to); + break; + } + if(nr->val.ctype == CTNIL) { + gencmp0(nl, nl->type, a, to); + break; + } + } + + a = optoas(a, nr->type); + + if(nr->ullman >= UINF) { + regalloc(&n1, nl->type, N); + cgen(nl, &n1); + + tempname(&tmp, nl->type); + gmove(&n1, &tmp); + regfree(&n1); + + regalloc(&n2, nr->type, N); + cgen(nr, &n2); + + regalloc(&n1, nl->type, N); + cgen(&tmp, &n1); + + gcmp(optoas(OCMP, nr->type), &n1, &n2); + patch(gbranch(a, nr->type), to); + + regfree(&n1); + regfree(&n2); + break; + } + + tempname(&n3, nl->type); + cgen(nl, &n3); + + tempname(&tmp, nr->type); + cgen(nr, &tmp); + + regalloc(&n1, nl->type, N); + gmove(&n3, &n1); + + regalloc(&n2, nr->type, N); + gmove(&tmp, &n2); + + gcmp(optoas(OCMP, nr->type), &n1, &n2); + if(isfloat[nl->type->etype]) { + p1 = gbranch(ABVS, nr->type); + patch(gbranch(a, nr->type), to); + if(n->op == ONE) + patch(p1, to); + else + patch(p1, pc); + } else { + patch(gbranch(a, nr->type), to); + } + regfree(&n1); + regfree(&n2); + break; + } + goto ret; + +ret: + ; +} + +/* + * n is on stack, either local variable + * or return value from function call. + * return n's offset from SP. + */ +int32 +stkof(Node *n) +{ + Type *t; + Iter flist; + int32 off; + + switch(n->op) { + case OINDREG: + return n->xoffset; + + case ODOT: + t = n->left->type; + if(isptr[t->etype]) + break; + off = stkof(n->left); + if(off == -1000 || off == 1000) + return off; + return off + n->xoffset; + + case OINDEX: + t = n->left->type; + if(!isfixedarray(t)) + break; + off = stkof(n->left); + if(off == -1000 || off == 1000) + return off; + if(isconst(n->right, CTINT)) + return off + t->type->width * mpgetfix(n->right->val.u.xval); + return 1000; + + case OCALLMETH: + case OCALLINTER: + case OCALLFUNC: + t = n->left->type; + if(isptr[t->etype]) + t = t->type; + + t = structfirst(&flist, getoutarg(t)); + if(t != T) + return t->width + 4; // correct for LR + break; + } + + // botch - probably failing to recognize address + // arithmetic on the above. eg INDEX and DOT + return -1000; +} + +/* + * block copy: + * memmove(&res, &n, w); + * NB: character copy assumed little endian architecture + */ +void +sgen(Node *n, Node *res, int32 w) +{ + Node dst, src, tmp, nend; + int32 c, odst, osrc; + int dir, align, op; + Prog *p, *ploop; + + if(debug['g']) { + print("\nsgen w=%d\n", w); + dump("r", n); + dump("res", res); + } + if(w == 0) + return; + if(w < 0) + fatal("sgen copy %d", w); + if(n->ullman >= UINF && res->ullman >= UINF) + fatal("sgen UINF"); + if(n->type == T) + fatal("sgen: missing type"); + + // determine alignment. + // want to avoid unaligned access, so have to use + // smaller operations for less aligned types. + // for example moving [4]byte must use 4 MOVB not 1 MOVW. + align = n->type->align; + op = 0; + switch(align) { + default: + fatal("sgen: invalid alignment %d for %T", align, n->type); + case 1: + op = AMOVB; + break; + case 2: + op = AMOVH; + break; + case 4: + op = AMOVW; + break; + } + if(w%align) + fatal("sgen: unaligned size %d (align=%d) for %T", w, align, n->type); + c = w / align; + + // offset on the stack + osrc = stkof(n); + odst = stkof(res); + if(osrc != -1000 && odst != -1000 && (osrc == 1000 || odst == 1000)) { + // osrc and odst both on stack, and at least one is in + // an unknown position. Could generate code to test + // for forward/backward copy, but instead just copy + // to a temporary location first. + tempname(&tmp, n->type); + sgen(n, &tmp, w); + sgen(&tmp, res, w); + return; + } + if(osrc%align != 0 || odst%align != 0) + fatal("sgen: unaligned offset src %d or dst %d (align %d)", osrc, odst, align); + // if we are copying forward on the stack and + // the src and dst overlap, then reverse direction + dir = align; + if(osrc < odst && odst < osrc+w) + dir = -dir; + + regalloc(&dst, types[tptr], res); + if(n->ullman >= res->ullman) { + agen(n, &dst); // temporarily use dst + regalloc(&src, types[tptr], N); + gins(AMOVW, &dst, &src); + agen(res, &dst); + } else { + agen(res, &dst); + regalloc(&src, types[tptr], N); + agen(n, &src); + } + + regalloc(&tmp, types[TUINT32], N); + + // set up end marker + memset(&nend, 0, sizeof nend); + if(c >= 4) { + regalloc(&nend, types[TUINT32], N); + + p = gins(AMOVW, &src, &nend); + p->from.type = D_CONST; + if(dir < 0) + p->from.offset = dir; + else + p->from.offset = w; + } + + // move src and dest to the end of block if necessary + if(dir < 0) { + p = gins(AMOVW, &src, &src); + p->from.type = D_CONST; + p->from.offset = w + dir; + + p = gins(AMOVW, &dst, &dst); + p->from.type = D_CONST; + p->from.offset = w + dir; + } + + // move + if(c >= 4) { + p = gins(op, &src, &tmp); + p->from.type = D_OREG; + p->from.offset = dir; + p->scond |= C_PBIT; + ploop = p; + + p = gins(op, &tmp, &dst); + p->to.type = D_OREG; + p->to.offset = dir; + p->scond |= C_PBIT; + + p = gins(ACMP, &src, N); + raddr(&nend, p); + + patch(gbranch(ABNE, T), ploop); + regfree(&nend); + } else { + while(c-- > 0) { + p = gins(op, &src, &tmp); + p->from.type = D_OREG; + p->from.offset = dir; + p->scond |= C_PBIT; + ploop = p; + + p = gins(op, &tmp, &dst); + p->to.type = D_OREG; + p->to.offset = dir; + p->scond |= C_PBIT; + } + } + + regfree(&dst); + regfree(&src); + regfree(&tmp); +} diff --git a/src/cmd/5g/cgen64.c b/src/cmd/5g/cgen64.c new file mode 100644 index 000000000..b56df765b --- /dev/null +++ b/src/cmd/5g/cgen64.c @@ -0,0 +1,716 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +#include "gg.h" + +/* + * attempt to generate 64-bit + * res = n + * return 1 on success, 0 if op not handled. + */ +void +cgen64(Node *n, Node *res) +{ + Node t1, t2, *l, *r; + Node lo1, lo2, hi1, hi2; + Node al, ah, bl, bh, cl, ch, s, n1, creg; + Prog *p1, *p2, *p3, *p4, *p5, *p6; + + uint64 v; + + if(res->op != OINDREG && res->op != ONAME) { + dump("n", n); + dump("res", res); + fatal("cgen64 %O of %O", n->op, res->op); + } + + l = n->left; + if(!l->addable) { + tempname(&t1, l->type); + cgen(l, &t1); + l = &t1; + } + + split64(l, &lo1, &hi1); + switch(n->op) { + default: + fatal("cgen64 %O", n->op); + + case OMINUS: + split64(res, &lo2, &hi2); + + regalloc(&t1, lo1.type, N); + regalloc(&al, lo1.type, N); + regalloc(&ah, hi1.type, N); + + gins(AMOVW, &lo1, &al); + gins(AMOVW, &hi1, &ah); + + gmove(ncon(0), &t1); + p1 = gins(ASUB, &al, &t1); + p1->scond |= C_SBIT; + gins(AMOVW, &t1, &lo2); + + gmove(ncon(0), &t1); + gins(ASBC, &ah, &t1); + gins(AMOVW, &t1, &hi2); + + regfree(&t1); + regfree(&al); + regfree(&ah); + splitclean(); + splitclean(); + return; + + case OCOM: + regalloc(&t1, lo1.type, N); + gmove(ncon(-1), &t1); + + split64(res, &lo2, &hi2); + regalloc(&n1, lo1.type, N); + + gins(AMOVW, &lo1, &n1); + gins(AEOR, &t1, &n1); + gins(AMOVW, &n1, &lo2); + + gins(AMOVW, &hi1, &n1); + gins(AEOR, &t1, &n1); + gins(AMOVW, &n1, &hi2); + + regfree(&t1); + regfree(&n1); + splitclean(); + splitclean(); + return; + + case OADD: + case OSUB: + case OMUL: + case OLSH: + case ORSH: + case OAND: + case OOR: + case OXOR: + // binary operators. + // common setup below. + break; + } + + // setup for binary operators + r = n->right; + if(r != N && !r->addable) { + tempname(&t2, r->type); + cgen(r, &t2); + r = &t2; + } + if(is64(r->type)) + split64(r, &lo2, &hi2); + + regalloc(&al, lo1.type, N); + regalloc(&ah, hi1.type, N); + + // Do op. Leave result in ah:al. + switch(n->op) { + default: + fatal("cgen64: not implemented: %N\n", n); + + case OADD: + // TODO: Constants + regalloc(&bl, types[TPTR32], N); + regalloc(&bh, types[TPTR32], N); + gins(AMOVW, &hi1, &ah); + gins(AMOVW, &lo1, &al); + gins(AMOVW, &hi2, &bh); + gins(AMOVW, &lo2, &bl); + p1 = gins(AADD, &bl, &al); + p1->scond |= C_SBIT; + gins(AADC, &bh, &ah); + regfree(&bl); + regfree(&bh); + break; + + case OSUB: + // TODO: Constants. + regalloc(&bl, types[TPTR32], N); + regalloc(&bh, types[TPTR32], N); + gins(AMOVW, &lo1, &al); + gins(AMOVW, &hi1, &ah); + gins(AMOVW, &lo2, &bl); + gins(AMOVW, &hi2, &bh); + p1 = gins(ASUB, &bl, &al); + p1->scond |= C_SBIT; + gins(ASBC, &bh, &ah); + regfree(&bl); + regfree(&bh); + break; + + case OMUL: + // TODO(kaib): this can be done with 4 regs and does not need 6 + regalloc(&bl, types[TPTR32], N); + regalloc(&bh, types[TPTR32], N); + regalloc(&cl, types[TPTR32], N); + regalloc(&ch, types[TPTR32], N); + + // load args into bh:bl and bh:bl. + gins(AMOVW, &hi1, &bh); + gins(AMOVW, &lo1, &bl); + gins(AMOVW, &hi2, &ch); + gins(AMOVW, &lo2, &cl); + + // bl * cl -> ah al + p1 = gins(AMULLU, N, N); + p1->from.type = D_REG; + p1->from.reg = bl.val.u.reg; + p1->reg = cl.val.u.reg; + p1->to.type = D_REGREG; + p1->to.reg = ah.val.u.reg; + p1->to.offset = al.val.u.reg; +//print("%P\n", p1); + + // bl * ch + ah -> ah + p1 = gins(AMULA, N, N); + p1->from.type = D_REG; + p1->from.reg = bl.val.u.reg; + p1->reg = ch.val.u.reg; + p1->to.type = D_REGREG; + p1->to.reg = ah.val.u.reg; + p1->to.offset = ah.val.u.reg; +//print("%P\n", p1); + + // bh * cl + ah -> ah + p1 = gins(AMULA, N, N); + p1->from.type = D_REG; + p1->from.reg = bh.val.u.reg; + p1->reg = cl.val.u.reg; + p1->to.type = D_REGREG; + p1->to.reg = ah.val.u.reg; + p1->to.offset = ah.val.u.reg; +//print("%P\n", p1); + + regfree(&bh); + regfree(&bl); + regfree(&ch); + regfree(&cl); + + break; + + case OLSH: + regalloc(&bl, lo1.type, N); + regalloc(&bh, hi1.type, N); + gins(AMOVW, &hi1, &bh); + gins(AMOVW, &lo1, &bl); + + if(r->op == OLITERAL) { + v = mpgetfix(r->val.u.xval); + if(v >= 64) { + // TODO(kaib): replace with gins(AMOVW, nodintconst(0), &al) + // here and below (verify it optimizes to EOR) + gins(AEOR, &al, &al); + gins(AEOR, &ah, &ah); + } else + if(v > 32) { + gins(AEOR, &al, &al); + // MOVW bl<<(v-32), ah + gshift(AMOVW, &bl, SHIFT_LL, (v-32), &ah); + } else + if(v == 32) { + gins(AEOR, &al, &al); + gins(AMOVW, &bl, &ah); + } else + if(v > 0) { + // MOVW bl<>(32-v), ah + gshift(AORR, &bl, SHIFT_LR, 32-v, &ah); + } else { + gins(AMOVW, &bl, &al); + gins(AMOVW, &bh, &ah); + } + goto olsh_break; + } + + regalloc(&s, types[TUINT32], N); + regalloc(&creg, types[TUINT32], N); + if (is64(r->type)) { + // shift is >= 1<<32 + split64(r, &cl, &ch); + gmove(&ch, &s); + p1 = gins(ATST, &s, N); + p6 = gbranch(ABNE, T); + gmove(&cl, &s); + splitclean(); + } else { + gmove(r, &s); + p6 = P; + } + p1 = gins(ATST, &s, N); + + // shift == 0 + p1 = gins(AMOVW, &bl, &al); + p1->scond = C_SCOND_EQ; + p1 = gins(AMOVW, &bh, &ah); + p1->scond = C_SCOND_EQ; + p2 = gbranch(ABEQ, T); + + // shift is < 32 + nodconst(&n1, types[TUINT32], 32); + gmove(&n1, &creg); + gcmp(ACMP, &s, &creg); + + // MOVW.LO bl<scond = C_SCOND_LO; + + // MOVW.LO bh<scond = C_SCOND_LO; + + // SUB.LO s, creg + p1 = gins(ASUB, &s, &creg); + p1->scond = C_SCOND_LO; + + // OR.LO bl>>creg, ah + p1 = gregshift(AORR, &bl, SHIFT_LR, &creg, &ah); + p1->scond = C_SCOND_LO; + + // BLO end + p3 = gbranch(ABLO, T); + + // shift == 32 + p1 = gins(AEOR, &al, &al); + p1->scond = C_SCOND_EQ; + p1 = gins(AMOVW, &bl, &ah); + p1->scond = C_SCOND_EQ; + p4 = gbranch(ABEQ, T); + + // shift is < 64 + nodconst(&n1, types[TUINT32], 64); + gmove(&n1, &creg); + gcmp(ACMP, &s, &creg); + + // EOR.LO al, al + p1 = gins(AEOR, &al, &al); + p1->scond = C_SCOND_LO; + + // MOVW.LO creg>>1, creg + p1 = gshift(AMOVW, &creg, SHIFT_LR, 1, &creg); + p1->scond = C_SCOND_LO; + + // SUB.LO creg, s + p1 = gins(ASUB, &creg, &s); + p1->scond = C_SCOND_LO; + + // MOVW bl<scond = C_SCOND_LO; + + p5 = gbranch(ABLO, T); + + // shift >= 64 + if (p6 != P) patch(p6, pc); + gins(AEOR, &al, &al); + gins(AEOR, &ah, &ah); + + patch(p2, pc); + patch(p3, pc); + patch(p4, pc); + patch(p5, pc); + regfree(&s); + regfree(&creg); + +olsh_break: + regfree(&bl); + regfree(&bh); + break; + + + case ORSH: + regalloc(&bl, lo1.type, N); + regalloc(&bh, hi1.type, N); + gins(AMOVW, &hi1, &bh); + gins(AMOVW, &lo1, &bl); + + if(r->op == OLITERAL) { + v = mpgetfix(r->val.u.xval); + if(v >= 64) { + if(bh.type->etype == TINT32) { + // MOVW bh->31, al + gshift(AMOVW, &bh, SHIFT_AR, 31, &al); + + // MOVW bh->31, ah + gshift(AMOVW, &bh, SHIFT_AR, 31, &ah); + } else { + gins(AEOR, &al, &al); + gins(AEOR, &ah, &ah); + } + } else + if(v > 32) { + if(bh.type->etype == TINT32) { + // MOVW bh->(v-32), al + gshift(AMOVW, &bh, SHIFT_AR, v-32, &al); + + // MOVW bh->31, ah + gshift(AMOVW, &bh, SHIFT_AR, 31, &ah); + } else { + // MOVW bh>>(v-32), al + gshift(AMOVW, &bh, SHIFT_LR, v-32, &al); + gins(AEOR, &ah, &ah); + } + } else + if(v == 32) { + gins(AMOVW, &bh, &al); + if(bh.type->etype == TINT32) { + // MOVW bh->31, ah + gshift(AMOVW, &bh, SHIFT_AR, 31, &ah); + } else { + gins(AEOR, &ah, &ah); + } + } else + if( v > 0) { + // MOVW bl>>v, al + gshift(AMOVW, &bl, SHIFT_LR, v, &al); + + // OR bh<<(32-v), al + gshift(AORR, &bh, SHIFT_LL, 32-v, &al); + + if(bh.type->etype == TINT32) { + // MOVW bh->v, ah + gshift(AMOVW, &bh, SHIFT_AR, v, &ah); + } else { + // MOVW bh>>v, ah + gshift(AMOVW, &bh, SHIFT_LR, v, &ah); + } + } else { + gins(AMOVW, &bl, &al); + gins(AMOVW, &bh, &ah); + } + goto orsh_break; + } + + regalloc(&s, types[TUINT32], N); + regalloc(&creg, types[TUINT32], N); + if(is64(r->type)) { + // shift is >= 1<<32 + split64(r, &cl, &ch); + gmove(&ch, &s); + gins(ATST, &s, N); + if(bh.type->etype == TINT32) + p1 = gshift(AMOVW, &bh, SHIFT_AR, 31, &ah); + else + p1 = gins(AEOR, &ah, &ah); + p1->scond = C_SCOND_NE; + p6 = gbranch(ABNE, T); + gmove(&cl, &s); + splitclean(); + } else { + gmove(r, &s); + p6 = P; + } + p1 = gins(ATST, &s, N); + + // shift == 0 + p1 = gins(AMOVW, &bl, &al); + p1->scond = C_SCOND_EQ; + p1 = gins(AMOVW, &bh, &ah); + p1->scond = C_SCOND_EQ; + p2 = gbranch(ABEQ, T); + + // check if shift is < 32 + nodconst(&n1, types[TUINT32], 32); + gmove(&n1, &creg); + gcmp(ACMP, &s, &creg); + + // MOVW.LO bl>>s, al + p1 = gregshift(AMOVW, &bl, SHIFT_LR, &s, &al); + p1->scond = C_SCOND_LO; + + // SUB.LO s,creg + p1 = gins(ASUB, &s, &creg); + p1->scond = C_SCOND_LO; + + // OR.LO bh<<(32-s), al + p1 = gregshift(AORR, &bh, SHIFT_LL, &creg, &al); + p1->scond = C_SCOND_LO; + + if(bh.type->etype == TINT32) { + // MOVW bh->s, ah + p1 = gregshift(AMOVW, &bh, SHIFT_AR, &s, &ah); + } else { + // MOVW bh>>s, ah + p1 = gregshift(AMOVW, &bh, SHIFT_LR, &s, &ah); + } + p1->scond = C_SCOND_LO; + + // BLO end + p3 = gbranch(ABLO, T); + + // shift == 32 + p1 = gins(AMOVW, &bh, &al); + p1->scond = C_SCOND_EQ; + if(bh.type->etype == TINT32) + p1 = gshift(AMOVW, &bh, SHIFT_AR, 31, &ah); + else + p1 = gins(AEOR, &ah, &ah); + p4 = gbranch(ABEQ, T); + + // check if shift is < 64 + nodconst(&n1, types[TUINT32], 64); + gmove(&n1, &creg); + gcmp(ACMP, &s, &creg); + + // MOVW.LO creg>>1, creg + p1 = gshift(AMOVW, &creg, SHIFT_LR, 1, &creg); + p1->scond = C_SCOND_LO; + + // SUB.LO creg, s + p1 = gins(ASUB, &creg, &s); + p1->scond = C_SCOND_LO; + + if(bh.type->etype == TINT32) { + // MOVW bh->(s-32), al + p1 = gregshift(AMOVW, &bh, SHIFT_AR, &s, &al); + p1->scond = C_SCOND_LO; + } else { + // MOVW bh>>(v-32), al + p1 = gregshift(AMOVW, &bh, SHIFT_LR, &s, &al); + p1->scond = C_SCOND_LO; + } + + // BLO end + p5 = gbranch(ABLO, T); + + // s >= 64 + if(p6 != P) + patch(p6, pc); + if(bh.type->etype == TINT32) { + // MOVW bh->31, al + gshift(AMOVW, &bh, SHIFT_AR, 31, &al); + } else { + gins(AEOR, &al, &al); + } + + patch(p2, pc); + patch(p3, pc); + patch(p4, pc); + patch(p5, pc); + regfree(&s); + regfree(&creg); + + +orsh_break: + regfree(&bl); + regfree(&bh); + break; + + case OXOR: + case OAND: + case OOR: + // TODO(kaib): literal optimizations + // make constant the right side (it usually is anyway). +// if(lo1.op == OLITERAL) { +// nswap(&lo1, &lo2); +// nswap(&hi1, &hi2); +// } +// if(lo2.op == OLITERAL) { +// // special cases for constants. +// lv = mpgetfix(lo2.val.u.xval); +// hv = mpgetfix(hi2.val.u.xval); +// splitclean(); // right side +// split64(res, &lo2, &hi2); +// switch(n->op) { +// case OXOR: +// gmove(&lo1, &lo2); +// gmove(&hi1, &hi2); +// switch(lv) { +// case 0: +// break; +// case 0xffffffffu: +// gins(ANOTL, N, &lo2); +// break; +// default: +// gins(AXORL, ncon(lv), &lo2); +// break; +// } +// switch(hv) { +// case 0: +// break; +// case 0xffffffffu: +// gins(ANOTL, N, &hi2); +// break; +// default: +// gins(AXORL, ncon(hv), &hi2); +// break; +// } +// break; + +// case OAND: +// switch(lv) { +// case 0: +// gins(AMOVL, ncon(0), &lo2); +// break; +// default: +// gmove(&lo1, &lo2); +// if(lv != 0xffffffffu) +// gins(AANDL, ncon(lv), &lo2); +// break; +// } +// switch(hv) { +// case 0: +// gins(AMOVL, ncon(0), &hi2); +// break; +// default: +// gmove(&hi1, &hi2); +// if(hv != 0xffffffffu) +// gins(AANDL, ncon(hv), &hi2); +// break; +// } +// break; + +// case OOR: +// switch(lv) { +// case 0: +// gmove(&lo1, &lo2); +// break; +// case 0xffffffffu: +// gins(AMOVL, ncon(0xffffffffu), &lo2); +// break; +// default: +// gmove(&lo1, &lo2); +// gins(AORL, ncon(lv), &lo2); +// break; +// } +// switch(hv) { +// case 0: +// gmove(&hi1, &hi2); +// break; +// case 0xffffffffu: +// gins(AMOVL, ncon(0xffffffffu), &hi2); +// break; +// default: +// gmove(&hi1, &hi2); +// gins(AORL, ncon(hv), &hi2); +// break; +// } +// break; +// } +// splitclean(); +// splitclean(); +// goto out; +// } + regalloc(&n1, lo1.type, N); + gins(AMOVW, &lo1, &al); + gins(AMOVW, &hi1, &ah); + gins(AMOVW, &lo2, &n1); + gins(optoas(n->op, lo1.type), &n1, &al); + gins(AMOVW, &hi2, &n1); + gins(optoas(n->op, lo1.type), &n1, &ah); + regfree(&n1); + break; + } + if(is64(r->type)) + splitclean(); + splitclean(); + + split64(res, &lo1, &hi1); + gins(AMOVW, &al, &lo1); + gins(AMOVW, &ah, &hi1); + splitclean(); + +//out: + regfree(&al); + regfree(&ah); +} + +/* + * generate comparison of nl, nr, both 64-bit. + * nl is memory; nr is constant or memory. + */ +void +cmp64(Node *nl, Node *nr, int op, Prog *to) +{ + Node lo1, hi1, lo2, hi2, r1, r2; + Prog *br; + Type *t; + + split64(nl, &lo1, &hi1); + split64(nr, &lo2, &hi2); + + // compare most significant word; + // if they differ, we're done. + t = hi1.type; + regalloc(&r1, types[TINT32], N); + regalloc(&r2, types[TINT32], N); + gins(AMOVW, &hi1, &r1); + gins(AMOVW, &hi2, &r2); + gcmp(ACMP, &r1, &r2); + regfree(&r1); + regfree(&r2); + + br = P; + switch(op) { + default: + fatal("cmp64 %O %T", op, t); + case OEQ: + // cmp hi + // bne L + // cmp lo + // beq to + // L: + br = gbranch(ABNE, T); + break; + case ONE: + // cmp hi + // bne to + // cmp lo + // bne to + patch(gbranch(ABNE, T), to); + break; + case OGE: + case OGT: + // cmp hi + // bgt to + // blt L + // cmp lo + // bge to (or bgt to) + // L: + patch(gbranch(optoas(OGT, t), T), to); + br = gbranch(optoas(OLT, t), T); + break; + case OLE: + case OLT: + // cmp hi + // blt to + // bgt L + // cmp lo + // ble to (or jlt to) + // L: + patch(gbranch(optoas(OLT, t), T), to); + br = gbranch(optoas(OGT, t), T); + break; + } + + // compare least significant word + t = lo1.type; + regalloc(&r1, types[TINT32], N); + regalloc(&r2, types[TINT32], N); + gins(AMOVW, &lo1, &r1); + gins(AMOVW, &lo2, &r2); + gcmp(ACMP, &r1, &r2); + regfree(&r1); + regfree(&r2); + + // jump again + patch(gbranch(optoas(op, t), T), to); + + // point first branch down here if appropriate + if(br != P) + patch(br, pc); + + splitclean(); + splitclean(); +} diff --git a/src/cmd/5g/doc.go b/src/cmd/5g/doc.go new file mode 100644 index 000000000..e86013bdd --- /dev/null +++ b/src/cmd/5g/doc.go @@ -0,0 +1,15 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +/* + +5g is the version of the gc compiler for the ARM. +The $GOARCH for these tools is arm. + +It reads .go files and outputs .5 files. The flags are documented in ../gc/doc.go. + +There is no instruction optimizer, so the -N flag is a no-op. + +*/ +package documentation diff --git a/src/cmd/5g/galign.c b/src/cmd/5g/galign.c new file mode 100644 index 000000000..12766102f --- /dev/null +++ b/src/cmd/5g/galign.c @@ -0,0 +1,39 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +#include "gg.h" + +int thechar = '5'; +char* thestring = "arm"; + +vlong MAXWIDTH = (1LL<<32) - 1; + +/* + * go declares several platform-specific type aliases: + * int, uint, float, and uintptr + */ +Typedef typedefs[] = +{ + "int", TINT, TINT32, + "uint", TUINT, TUINT32, + "uintptr", TUINTPTR, TUINT32, + 0 +}; + +void +betypeinit(void) +{ + widthptr = 4; + + zprog.link = P; + zprog.as = AGOK; + zprog.scond = C_SCOND_NONE; + zprog.reg = NREG; + zprog.from.type = D_NONE; + zprog.from.name = D_NONE; + zprog.from.reg = NREG; + zprog.to = zprog.from; + + listinit(); +} diff --git a/src/cmd/5g/gg.h b/src/cmd/5g/gg.h new file mode 100644 index 000000000..ce4558e21 --- /dev/null +++ b/src/cmd/5g/gg.h @@ -0,0 +1,171 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +#include +#include + +#include "../gc/go.h" +#include "../5l/5.out.h" + +#ifndef EXTERN +#define EXTERN extern +#endif + +typedef struct Addr Addr; + +struct Addr +{ + int32 offset; + int32 offset2; + double dval; + Prog* branch; + char sval[NSNAME]; + + Sym* sym; + Node* node; + int width; + uchar type; + char name; + uchar reg; + char pun; + uchar etype; +}; +#define A ((Addr*)0) + +struct Prog +{ + short as; // opcode + uint32 loc; // pc offset in this func + uint32 lineno; // source line that generated this + Addr from; // src address + Addr to; // dst address + Prog* link; // next instruction in this func + void* regp; // points to enclosing Reg struct + uchar reg; // doubles as width in DATA op + uchar scond; +}; + +#define REGALLOC_R0 0 +#define REGALLOC_RMAX REGEXT +#define REGALLOC_F0 (REGALLOC_RMAX+1) +#define REGALLOC_FMAX (REGALLOC_F0 + FREGEXT) + +EXTERN Biobuf* bout; +EXTERN int32 dynloc; +EXTERN uchar reg[REGALLOC_FMAX+1]; +EXTERN int32 pcloc; // instruction counter +EXTERN Strlit emptystring; +extern char* anames[]; +EXTERN Hist* hist; +EXTERN Prog zprog; +EXTERN Node* curfn; +EXTERN Node* newproc; +EXTERN Node* deferproc; +EXTERN Node* deferreturn; +EXTERN Node* panicindex; +EXTERN Node* panicslice; +EXTERN Node* throwreturn; +EXTERN long unmappedzero; +EXTERN int maxstksize; + +/* + * gen.c + */ +void compile(Node*); +void proglist(void); +void gen(Node*); +Node* lookdot(Node*, Node*, int); +void cgen_as(Node*, Node*); +void cgen_callmeth(Node*, int); +void cgen_callinter(Node*, Node*, int); +void cgen_proc(Node*, int); +void cgen_callret(Node*, Node*); +void cgen_dcl(Node*); +int cgen_inline(Node*, Node*); +int needconvert(Type*, Type*); +void genconv(Type*, Type*); +void allocparams(void); +void checklabels(); +void ginscall(Node*, int); + +/* + * cgen + */ +void agen(Node*, Node*); +Prog* cgenindex(Node *, Node *); +void igen(Node*, Node*, Node*); +void agenr(Node *n, Node *a, Node *res); +vlong fieldoffset(Type*, Node*); +void bgen(Node*, int, Prog*); +void sgen(Node*, Node*, int32); +void gmove(Node*, Node*); +Prog* gins(int, Node*, Node*); +int samaddr(Node*, Node*); +void raddr(Node *n, Prog *p); +Prog* gcmp(int, Node*, Node*); +Prog* gshift(int as, Node *lhs, int32 stype, int32 sval, Node *rhs); +Prog * gregshift(int as, Node *lhs, int32 stype, Node *reg, Node *rhs); +void naddr(Node*, Addr*, int); +void cgen_aret(Node*, Node*); +void cgen_shift(int, Node*, Node*, Node*); + +/* + * cgen64.c + */ +void cmp64(Node*, Node*, int, Prog*); +void cgen64(Node*, Node*); + +/* + * gsubr.c + */ +void clearp(Prog*); +void proglist(void); +Prog* gbranch(int, Type*); +Prog* prog(int); +void gaddoffset(Node*); +void gconv(int, int); +int conv2pt(Type*); +vlong convvtox(vlong, int); +void fnparam(Type*, int, int); +Prog* gop(int, Node*, Node*, Node*); +void setconst(Addr*, vlong); +void setaddr(Addr*, Node*); +int optoas(int, Type*); +void ginit(void); +void gclean(void); +void regalloc(Node*, Type*, Node*); +void regfree(Node*); +Node* nodarg(Type*, int); +void nodreg(Node*, Type*, int); +void nodindreg(Node*, Type*, int); +void buildtxt(void); +Plist* newplist(void); +int isfat(Type*); +int dotaddable(Node*, Node*); +void sudoclean(void); +int sudoaddable(int, Node*, Addr*, int*); +void afunclit(Addr*); +void datagostring(Strlit*, Addr*); +void split64(Node*, Node*, Node*); +void splitclean(void); +Node* ncon(uint32 i); + +/* + * obj.c + */ +void datastring(char*, int, Addr*); + +/* + * list.c + */ +int Aconv(Fmt*); +int Cconv(Fmt*); +int Dconv(Fmt*); +int Mconv(Fmt*); +int Pconv(Fmt*); +int Rconv(Fmt*); +int Yconv(Fmt*); +void listinit(void); + +void zaddr(Biobuf*, Addr*, int); diff --git a/src/cmd/5g/ggen.c b/src/cmd/5g/ggen.c new file mode 100644 index 000000000..3f5f47e7b --- /dev/null +++ b/src/cmd/5g/ggen.c @@ -0,0 +1,1030 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +#undef EXTERN +#define EXTERN +#include "gg.h" +#include "opt.h" + +void +defframe(Prog *ptxt) +{ + // fill in argument size + ptxt->to.type = D_CONST2; + ptxt->reg = 0; // flags + ptxt->to.offset2 = rnd(curfn->type->argwid, widthptr); + + // fill in final stack size + if(stksize > maxstksize) + maxstksize = stksize; + ptxt->to.offset = rnd(maxstksize+maxarg, widthptr); + maxstksize = 0; +} + +// Sweep the prog list to mark any used nodes. +void +markautoused(Prog* p) +{ + for (; p; p = p->link) { + if (p->from.name == D_AUTO && p->from.node) + p->from.node->used++; + + if (p->to.name == D_AUTO && p->to.node) + p->to.node->used++; + } +} + +// Fixup instructions after compactframe has moved all autos around. +void +fixautoused(Prog* p) +{ + for (; p; p = p->link) { + if (p->from.name == D_AUTO && p->from.node) + p->from.offset += p->from.node->stkdelta; + + if (p->to.name == D_AUTO && p->to.node) + p->to.offset += p->to.node->stkdelta; + } +} + +/* + * generate: + * call f + * proc=0 normal call + * proc=1 goroutine run in new proc + * proc=2 defer call save away stack + */ +void +ginscall(Node *f, int proc) +{ + Prog *p; + Node n1, r, con; + + switch(proc) { + default: + fatal("ginscall: bad proc %d", proc); + break; + + case 0: // normal call + p = gins(ABL, N, f); + afunclit(&p->to); + break; + + case 1: // call in new proc (go) + case 2: // deferred call (defer) + regalloc(&r, types[tptr], N); + p = gins(AMOVW, N, &r); + p->from.type = D_OREG; + p->from.reg = REGSP; + + p = gins(AMOVW, &r, N); + p->to.type = D_OREG; + p->to.reg = REGSP; + p->to.offset = -12; + p->scond |= C_WBIT; + + memset(&n1, 0, sizeof n1); + n1.op = OADDR; + n1.left = f; + gins(AMOVW, &n1, &r); + + p = gins(AMOVW, &r, N); + p->to.type = D_OREG; + p->to.reg = REGSP; + p->to.offset = 8; + + nodconst(&con, types[TINT32], argsize(f->type)); + gins(AMOVW, &con, &r); + p = gins(AMOVW, &r, N); + p->to.type = D_OREG; + p->to.reg = REGSP; + p->to.offset = 4; + regfree(&r); + + if(proc == 1) + ginscall(newproc, 0); + else + ginscall(deferproc, 0); + + nodreg(&r, types[tptr], 1); + p = gins(AMOVW, N, N); + p->from.type = D_CONST; + p->from.reg = REGSP; + p->from.offset = 12; + p->to.reg = REGSP; + p->to.type = D_REG; + + if(proc == 2) { + nodconst(&con, types[TINT32], 0); + p = gins(ACMP, &con, N); + p->reg = 0; + patch(gbranch(ABNE, T), retpc); + } + break; + } +} + +/* + * n is call to interface method. + * generate res = n. + */ +void +cgen_callinter(Node *n, Node *res, int proc) +{ + int r; + Node *i, *f; + Node tmpi, nodo, nodr, nodsp; + + i = n->left; + if(i->op != ODOTINTER) + fatal("cgen_callinter: not ODOTINTER %O", i->op); + + f = i->right; // field + if(f->op != ONAME) + fatal("cgen_callinter: not ONAME %O", f->op); + + i = i->left; // interface + + // Release res register during genlist and cgen, + // which might have their own function calls. + r = -1; + if(res != N && (res->op == OREGISTER || res->op == OINDREG)) { + r = res->val.u.reg; + reg[r]--; + } + + if(!i->addable) { + tempname(&tmpi, i->type); + cgen(i, &tmpi); + i = &tmpi; + } + + genlist(n->list); // args + if(r >= 0) + reg[r]++; + + regalloc(&nodr, types[tptr], res); + regalloc(&nodo, types[tptr], &nodr); + nodo.op = OINDREG; + + agen(i, &nodr); // REG = &inter + + nodindreg(&nodsp, types[tptr], REGSP); + nodsp.xoffset = 4; + nodo.xoffset += widthptr; + cgen(&nodo, &nodsp); // 4(SP) = 4(REG) -- i.data + + nodo.xoffset -= widthptr; + cgen(&nodo, &nodr); // REG = 0(REG) -- i.tab + + nodo.xoffset = n->left->xoffset + 3*widthptr + 8; + cgen(&nodo, &nodr); // REG = 20+offset(REG) -- i.tab->fun[f] + + // BOTCH nodr.type = fntype; + nodr.type = n->left->type; + ginscall(&nodr, proc); + + regfree(&nodr); + regfree(&nodo); + + setmaxarg(n->left->type); +} + +/* + * generate function call; + * proc=0 normal call + * proc=1 goroutine run in new proc + * proc=2 defer call save away stack + */ +void +cgen_call(Node *n, int proc) +{ + Type *t; + Node nod, afun; + + if(n == N) + return; + + if(n->left->ullman >= UINF) { + // if name involves a fn call + // precompute the address of the fn + tempname(&afun, types[tptr]); + cgen(n->left, &afun); + } + + genlist(n->list); // assign the args + t = n->left->type; + + setmaxarg(t); + + // call tempname pointer + if(n->left->ullman >= UINF) { + regalloc(&nod, types[tptr], N); + cgen_as(&nod, &afun); + nod.type = t; + ginscall(&nod, proc); + regfree(&nod); + goto ret; + } + + // call pointer + if(n->left->op != ONAME || n->left->class != PFUNC) { + regalloc(&nod, types[tptr], N); + cgen_as(&nod, n->left); + nod.type = t; + ginscall(&nod, proc); + regfree(&nod); + goto ret; + } + + // call direct + n->left->method = 1; + ginscall(n->left, proc); + + +ret: + ; +} + +/* + * call to n has already been generated. + * generate: + * res = return value from call. + */ +void +cgen_callret(Node *n, Node *res) +{ + Node nod; + Type *fp, *t; + Iter flist; + + t = n->left->type; + if(t->etype == TPTR32 || t->etype == TPTR64) + t = t->type; + + fp = structfirst(&flist, getoutarg(t)); + if(fp == T) + fatal("cgen_callret: nil"); + + memset(&nod, 0, sizeof(nod)); + nod.op = OINDREG; + nod.val.u.reg = REGSP; + nod.addable = 1; + + nod.xoffset = fp->width + 4; // +4: saved lr at 0(SP) + nod.type = fp->type; + cgen_as(res, &nod); +} + +/* + * call to n has already been generated. + * generate: + * res = &return value from call. + */ +void +cgen_aret(Node *n, Node *res) +{ + Node nod1, nod2; + Type *fp, *t; + Iter flist; + + t = n->left->type; + if(isptr[t->etype]) + t = t->type; + + fp = structfirst(&flist, getoutarg(t)); + if(fp == T) + fatal("cgen_aret: nil"); + + memset(&nod1, 0, sizeof(nod1)); + nod1.op = OINDREG; + nod1.val.u.reg = REGSP; + nod1.addable = 1; + + nod1.xoffset = fp->width + 4; // +4: saved lr at 0(SP) + nod1.type = fp->type; + + if(res->op != OREGISTER) { + regalloc(&nod2, types[tptr], res); + agen(&nod1, &nod2); + gins(AMOVW, &nod2, res); + regfree(&nod2); + } else + agen(&nod1, res); +} + +/* + * generate return. + * n->left is assignments to return values. + */ +void +cgen_ret(Node *n) +{ + genlist(n->list); // copy out args + if(hasdefer || curfn->exit) + gjmp(retpc); + else + gins(ARET, N, N); +} + +/* + * generate += *= etc. + */ +void +cgen_asop(Node *n) +{ + Node n1, n2, n3, n4; + Node *nl, *nr; + Prog *p1; + Addr addr; + int a, w; + + nl = n->left; + nr = n->right; + + if(nr->ullman >= UINF && nl->ullman >= UINF) { + tempname(&n1, nr->type); + cgen(nr, &n1); + n2 = *n; + n2.right = &n1; + cgen_asop(&n2); + goto ret; + } + + if(!isint[nl->type->etype]) + goto hard; + if(!isint[nr->type->etype]) + goto hard; + if(is64(nl->type) || is64(nr->type)) + goto hard64; + + switch(n->etype) { + case OADD: + case OSUB: + case OXOR: + case OAND: + case OOR: + a = optoas(n->etype, nl->type); + if(nl->addable) { + regalloc(&n3, nr->type, N); + cgen(nr, &n3); + regalloc(&n2, nl->type, N); + cgen(nl, &n2); + gins(a, &n3, &n2); + cgen(&n2, nl); + regfree(&n2); + regfree(&n3); + goto ret; + } + if(nr->ullman < UINF) + if(sudoaddable(a, nl, &addr, &w)) { + w = optoas(OAS, nl->type); + regalloc(&n2, nl->type, N); + p1 = gins(w, N, &n2); + p1->from = addr; + regalloc(&n3, nr->type, N); + cgen(nr, &n3); + gins(a, &n3, &n2); + p1 = gins(w, &n2, N); + p1->to = addr; + regfree(&n2); + regfree(&n3); + sudoclean(); + goto ret; + } + } + +hard: + n2.op = 0; + n1.op = 0; + if(nr->ullman >= nl->ullman || nl->addable) { + regalloc(&n2, nr->type, N); + cgen(nr, &n2); + nr = &n2; + } else { + tempname(&n2, nr->type); + cgen(nr, &n2); + nr = &n2; + } + if(!nl->addable) { + igen(nl, &n1, N); + nl = &n1; + } + + n3 = *n; + n3.left = nl; + n3.right = nr; + n3.op = n->etype; + + regalloc(&n4, nl->type, N); + cgen(&n3, &n4); + gmove(&n4, nl); + + if(n1.op) + regfree(&n1); + if(n2.op == OREGISTER) + regfree(&n2); + regfree(&n4); + goto ret; + +hard64: + if(nr->ullman > nl->ullman) { + tempname(&n2, nr->type); + cgen(nr, &n2); + igen(nl, &n1, N); + } else { + igen(nl, &n1, N); + tempname(&n2, nr->type); + cgen(nr, &n2); + } + + n3 = *n; + n3.left = &n1; + n3.right = &n2; + n3.op = n->etype; + + cgen(&n3, &n1); + +ret: + ; +} + +int +samereg(Node *a, Node *b) +{ + if(a->op != OREGISTER) + return 0; + if(b->op != OREGISTER) + return 0; + if(a->val.u.reg != b->val.u.reg) + return 0; + return 1; +} + +/* + * generate shift according to op, one of: + * res = nl << nr + * res = nl >> nr + */ +void +cgen_shift(int op, Node *nl, Node *nr, Node *res) +{ + Node n1, n2, n3, nt, t, lo, hi; + int w; + Prog *p1, *p2, *p3; + Type *tr; + uvlong sc; + + if(nl->type->width > 4) + fatal("cgen_shift %T", nl->type); + + w = nl->type->width * 8; + + if(nr->op == OLITERAL) { + regalloc(&n1, nl->type, res); + cgen(nl, &n1); + sc = mpgetfix(nr->val.u.xval); + if(sc == 0) { + // nothing to do + } else if(sc >= nl->type->width*8) { + if(op == ORSH && issigned[nl->type->etype]) + gshift(AMOVW, &n1, SHIFT_AR, w, &n1); + else + gins(AEOR, &n1, &n1); + } else { + if(op == ORSH && issigned[nl->type->etype]) + gshift(AMOVW, &n1, SHIFT_AR, sc, &n1); + else if(op == ORSH) + gshift(AMOVW, &n1, SHIFT_LR, sc, &n1); + else // OLSH + gshift(AMOVW, &n1, SHIFT_LL, sc, &n1); + } + gmove(&n1, res); + regfree(&n1); + return; + } + + tr = nr->type; + if(tr->width > 4) { + tempname(&nt, nr->type); + if(nl->ullman >= nr->ullman) { + regalloc(&n2, nl->type, res); + cgen(nl, &n2); + cgen(nr, &nt); + n1 = nt; + } else { + cgen(nr, &nt); + regalloc(&n2, nl->type, res); + cgen(nl, &n2); + } + split64(&nt, &lo, &hi); + regalloc(&n1, types[TUINT32], N); + regalloc(&n3, types[TUINT32], N); + gmove(&lo, &n1); + gmove(&hi, &n3); + gins(ATST, &n3, N); + nodconst(&t, types[TUINT32], w); + p1 = gins(AMOVW, &t, &n1); + p1->scond = C_SCOND_NE; + tr = types[TUINT32]; + regfree(&n3); + } else { + if(nl->ullman >= nr->ullman) { + regalloc(&n2, nl->type, res); + cgen(nl, &n2); + regalloc(&n1, nr->type, N); + cgen(nr, &n1); + } else { + regalloc(&n1, nr->type, N); + cgen(nr, &n1); + regalloc(&n2, nl->type, res); + cgen(nl, &n2); + } + } + + // test for shift being 0 + p1 = gins(ATST, &n1, N); + p3 = gbranch(ABEQ, T); + + // test and fix up large shifts + regalloc(&n3, tr, N); + nodconst(&t, types[TUINT32], w); + gmove(&t, &n3); + gcmp(ACMP, &n1, &n3); + if(op == ORSH) { + if(issigned[nl->type->etype]) { + p1 = gshift(AMOVW, &n2, SHIFT_AR, w-1, &n2); + p2 = gregshift(AMOVW, &n2, SHIFT_AR, &n1, &n2); + } else { + p1 = gins(AEOR, &n2, &n2); + p2 = gregshift(AMOVW, &n2, SHIFT_LR, &n1, &n2); + } + p1->scond = C_SCOND_HS; + p2->scond = C_SCOND_LO; + } else { + p1 = gins(AEOR, &n2, &n2); + p2 = gregshift(AMOVW, &n2, SHIFT_LL, &n1, &n2); + p1->scond = C_SCOND_HS; + p2->scond = C_SCOND_LO; + } + regfree(&n3); + + patch(p3, pc); + gmove(&n2, res); + + regfree(&n1); + regfree(&n2); +} + +void +clearfat(Node *nl) +{ + uint32 w, c, q; + Node dst, nc, nz, end; + Prog *p, *pl; + + /* clear a fat object */ + if(debug['g']) + dump("\nclearfat", nl); + + w = nl->type->width; + c = w % 4; // bytes + q = w / 4; // quads + + regalloc(&dst, types[tptr], N); + agen(nl, &dst); + nodconst(&nc, types[TUINT32], 0); + regalloc(&nz, types[TUINT32], 0); + cgen(&nc, &nz); + + if(q >= 4) { + regalloc(&end, types[tptr], N); + p = gins(AMOVW, &dst, &end); + p->from.type = D_CONST; + p->from.offset = q*4; + + p = gins(AMOVW, &nz, &dst); + p->to.type = D_OREG; + p->to.offset = 4; + p->scond |= C_PBIT; + pl = p; + + p = gins(ACMP, &dst, N); + raddr(&end, p); + patch(gbranch(ABNE, T), pl); + + regfree(&end); + } else + while(q > 0) { + p = gins(AMOVW, &nz, &dst); + p->to.type = D_OREG; + p->to.offset = 4; + p->scond |= C_PBIT; +//print("1. %P\n", p); + q--; + } + + while(c > 0) { + p = gins(AMOVBU, &nz, &dst); + p->to.type = D_OREG; + p->to.offset = 1; + p->scond |= C_PBIT; +//print("2. %P\n", p); + c--; + } + regfree(&dst); + regfree(&nz); +} + +static int +regcmp(const void *va, const void *vb) +{ + Node *ra, *rb; + + ra = (Node*)va; + rb = (Node*)vb; + return ra->local - rb->local; +} + +static Prog* throwpc; + +// We're only going to bother inlining if we can +// convert all the arguments to 32 bits safely. Can we? +static int +fix64(NodeList *nn, int n) +{ + NodeList *l; + Node *r; + int i; + + l = nn; + for(i=0; in->right; + if(is64(r->type) && !smallintconst(r)) { + if(r->op == OCONV) + r = r->left; + if(is64(r->type)) + return 0; + } + l = l->next; + } + return 1; +} + +void +getargs(NodeList *nn, Node *reg, int n) +{ + NodeList *l; + int i; + + throwpc = nil; + + l = nn; + for(i=0; in->right) && !isslice(l->n->right->type)) { + regalloc(reg+i, l->n->right->type, N); + cgen(l->n->right, reg+i); + } else + reg[i] = *l->n->right; + if(reg[i].local != 0) + yyerror("local used"); + reg[i].local = l->n->left->xoffset; + l = l->next; + } + qsort((void*)reg, n, sizeof(*reg), regcmp); + for(i=0; ival.u.xval); + if(cl == 0) + return; + if(smallintconst(nr)) + return; + + // put the constant on the right + op = brrev(op); + c = nl; + nl = nr; + nr = c; + } + + n1.op = OXXX; + if(nr->op != OREGISTER) { + regalloc(&n1, types[TUINT32], N); + gmove(nr, &n1); + nr = &n1; + } + n2.op = OXXX; + if(nl->op != OREGISTER) { + regalloc(&n2, types[TUINT32], N); + gmove(nl, &n2); + nl = &n2; + } + gcmp(optoas(OCMP, types[TUINT32]), nl, nr); + if(nr == &n1) + regfree(&n1); + if(nl == &n2) + regfree(&n2); + if(throwpc == nil) { + p1 = gbranch(optoas(op, types[TUINT32]), T); + throwpc = pc; + ginscall(panicslice, 0); + patch(p1, pc); + } else { + op = brcom(op); + p1 = gbranch(optoas(op, types[TUINT32]), T); + patch(p1, throwpc); + } +} + +int +sleasy(Node *n) +{ + if(n->op != ONAME) + return 0; + if(!n->addable) + return 0; + return 1; +} + +// generate inline code for +// slicearray +// sliceslice +// arraytoslice +int +cgen_inline(Node *n, Node *res) +{ + Node nodes[5]; + Node n1, n2, n3, nres, ntemp; + vlong v; + int i, narg; + + if(n->op != OCALLFUNC) + goto no; + if(!n->left->addable) + goto no; + if(n->left->sym == S) + goto no; + if(n->left->sym->pkg != runtimepkg) + goto no; + if(strcmp(n->left->sym->name, "slicearray") == 0) + goto slicearray; + if(strcmp(n->left->sym->name, "sliceslice") == 0) { + narg = 4; + goto sliceslice; + } + if(strcmp(n->left->sym->name, "sliceslice1") == 0) { + narg = 3; + goto sliceslice; + } + goto no; + +slicearray: + if(!sleasy(res)) + goto no; + if(!fix64(n->list, 5)) + goto no; + getargs(n->list, nodes, 5); + + // if(hb[3] > nel[1]) goto throw + cmpandthrow(&nodes[3], &nodes[1]); + + // if(lb[2] > hb[3]) goto throw + cmpandthrow(&nodes[2], &nodes[3]); + + // len = hb[3] - lb[2] (destroys hb) + n2 = *res; + n2.type = types[TUINT32]; + n2.xoffset += Array_nel; + + if(smallintconst(&nodes[3]) && smallintconst(&nodes[2])) { + v = mpgetfix(nodes[3].val.u.xval) - + mpgetfix(nodes[2].val.u.xval); + nodconst(&n1, types[TUINT32], v); + gmove(&n1, &n2); + } else { + regalloc(&n1, types[TUINT32], &nodes[3]); + gmove(&nodes[3], &n1); + if(!smallintconst(&nodes[2]) || mpgetfix(nodes[2].val.u.xval) != 0) + gins(optoas(OSUB, types[TUINT32]), &nodes[2], &n1); + gmove(&n1, &n2); + regfree(&n1); + } + + // cap = nel[1] - lb[2] (destroys nel) + n2 = *res; + n2.type = types[TUINT32]; + n2.xoffset += Array_cap; + + if(smallintconst(&nodes[1]) && smallintconst(&nodes[2])) { + v = mpgetfix(nodes[1].val.u.xval) - + mpgetfix(nodes[2].val.u.xval); + nodconst(&n1, types[TUINT32], v); + gmove(&n1, &n2); + } else { + regalloc(&n1, types[TUINT32], &nodes[1]); + gmove(&nodes[1], &n1); + if(!smallintconst(&nodes[2]) || mpgetfix(nodes[2].val.u.xval) != 0) + gins(optoas(OSUB, types[TUINT32]), &nodes[2], &n1); + gmove(&n1, &n2); + regfree(&n1); + } + + // if slice could be too big, dereference to + // catch nil array pointer. + if(nodes[0].op == OREGISTER && nodes[0].type->type->width >= unmappedzero) { + n2 = nodes[0]; + n2.xoffset = 0; + n2.op = OINDREG; + n2.type = types[TUINT8]; + regalloc(&n1, types[TUINT32], N); + gins(AMOVB, &n2, &n1); + regfree(&n1); + } + + // ary = old[0] + (lb[2] * width[4]) (destroys old) + n2 = *res; + n2.type = types[tptr]; + n2.xoffset += Array_array; + + if(smallintconst(&nodes[2]) && smallintconst(&nodes[4])) { + v = mpgetfix(nodes[2].val.u.xval) * + mpgetfix(nodes[4].val.u.xval); + if(v != 0) { + nodconst(&n1, types[tptr], v); + gins(optoas(OADD, types[tptr]), &n1, &nodes[0]); + } + } else { + regalloc(&n1, types[tptr], &nodes[2]); + gmove(&nodes[2], &n1); + if(!smallintconst(&nodes[4]) || mpgetfix(nodes[4].val.u.xval) != 1) { + regalloc(&n3, types[tptr], N); + gmove(&nodes[4], &n3); + gins(optoas(OMUL, types[tptr]), &n3, &n1); + regfree(&n3); + } + gins(optoas(OADD, types[tptr]), &n1, &nodes[0]); + regfree(&n1); + } + gmove(&nodes[0], &n2); + + for(i=0; i<5; i++) { + if(nodes[i].op == OREGISTER) + regfree(&nodes[i]); + } + return 1; + +sliceslice: + if(!fix64(n->list, narg)) + goto no; + ntemp.op = OXXX; + if(!sleasy(n->list->n->right)) { + Node *n0; + + n0 = n->list->n->right; + tempname(&ntemp, res->type); + cgen(n0, &ntemp); + n->list->n->right = &ntemp; + getargs(n->list, nodes, narg); + n->list->n->right = n0; + } else + getargs(n->list, nodes, narg); + + nres = *res; // result + if(!sleasy(res)) { + if(ntemp.op == OXXX) + tempname(&ntemp, res->type); + nres = ntemp; + } + + if(narg == 3) { // old[lb:] + // move width to where it would be for old[lb:hb] + nodes[3] = nodes[2]; + nodes[2].op = OXXX; + + // if(lb[1] > old.nel[0]) goto throw; + n2 = nodes[0]; + n2.xoffset += Array_nel; + n2.type = types[TUINT32]; + cmpandthrow(&nodes[1], &n2); + + // ret.nel = old.nel[0]-lb[1]; + n2 = nodes[0]; + n2.type = types[TUINT32]; + n2.xoffset += Array_nel; + + regalloc(&n1, types[TUINT32], N); + gmove(&n2, &n1); + if(!smallintconst(&nodes[1]) || mpgetfix(nodes[1].val.u.xval) != 0) + gins(optoas(OSUB, types[TUINT32]), &nodes[1], &n1); + + n2 = nres; + n2.type = types[TUINT32]; + n2.xoffset += Array_nel; + gmove(&n1, &n2); + regfree(&n1); + } else { // old[lb:hb] + // if(hb[2] > old.cap[0]) goto throw; + n2 = nodes[0]; + n2.xoffset += Array_cap; + n2.type = types[TUINT32]; + cmpandthrow(&nodes[2], &n2); + + // if(lb[1] > hb[2]) goto throw; + cmpandthrow(&nodes[1], &nodes[2]); + + // ret.len = hb[2]-lb[1]; (destroys hb[2]) + n2 = nres; + n2.type = types[TUINT32]; + n2.xoffset += Array_nel; + + if(smallintconst(&nodes[2]) && smallintconst(&nodes[1])) { + v = mpgetfix(nodes[2].val.u.xval) - + mpgetfix(nodes[1].val.u.xval); + nodconst(&n1, types[TUINT32], v); + gmove(&n1, &n2); + } else { + regalloc(&n1, types[TUINT32], &nodes[2]); + gmove(&nodes[2], &n1); + if(!smallintconst(&nodes[1]) || mpgetfix(nodes[1].val.u.xval) != 0) + gins(optoas(OSUB, types[TUINT32]), &nodes[1], &n1); + gmove(&n1, &n2); + regfree(&n1); + } + } + + // ret.cap = old.cap[0]-lb[1]; (uses hb[2]) + n2 = nodes[0]; + n2.type = types[TUINT32]; + n2.xoffset += Array_cap; + + regalloc(&n1, types[TUINT32], &nodes[2]); + gmove(&n2, &n1); + if(!smallintconst(&nodes[1]) || mpgetfix(nodes[1].val.u.xval) != 0) + gins(optoas(OSUB, types[TUINT32]), &nodes[1], &n1); + + n2 = nres; + n2.type = types[TUINT32]; + n2.xoffset += Array_cap; + gmove(&n1, &n2); + regfree(&n1); + + // ret.array = old.array[0]+lb[1]*width[3]; (uses lb[1]) + n2 = nodes[0]; + n2.type = types[tptr]; + n2.xoffset += Array_array; + regalloc(&n3, types[tptr], N); + gmove(&n2, &n3); + + regalloc(&n1, types[tptr], &nodes[1]); + if(smallintconst(&nodes[1]) && smallintconst(&nodes[3])) { + gmove(&n2, &n1); + v = mpgetfix(nodes[1].val.u.xval) * + mpgetfix(nodes[3].val.u.xval); + if(v != 0) { + nodconst(&n2, types[tptr], v); + gins(optoas(OADD, types[tptr]), &n3, &n1); + } + } else { + gmove(&nodes[1], &n1); + if(!smallintconst(&nodes[3]) || mpgetfix(nodes[3].val.u.xval) != 1) { + regalloc(&n2, types[tptr], N); + gmove(&nodes[3], &n2); + gins(optoas(OMUL, types[tptr]), &n2, &n1); + regfree(&n2); + } + gins(optoas(OADD, types[tptr]), &n3, &n1); + } + regfree(&n3); + + n2 = nres; + n2.type = types[tptr]; + n2.xoffset += Array_array; + gmove(&n1, &n2); + regfree(&n1); + + for(i=0; i<4; i++) { + if(nodes[i].op == OREGISTER) + regfree(&nodes[i]); + } + + if(!sleasy(res)) { + cgen(&nres, res); + } + return 1; + +no: + return 0; +} diff --git a/src/cmd/5g/gobj.c b/src/cmd/5g/gobj.c new file mode 100644 index 000000000..27c8be67d --- /dev/null +++ b/src/cmd/5g/gobj.c @@ -0,0 +1,623 @@ +// Derived from Inferno utils/5c/swt.c +// http://code.google.com/p/inferno-os/source/browse/utils/5c/swt.c +// +// Copyright © 1994-1999 Lucent Technologies Inc. All rights reserved. +// Portions Copyright © 1995-1997 C H Forsyth (forsyth@terzarima.net) +// Portions Copyright © 1997-1999 Vita Nuova Limited +// Portions Copyright © 2000-2007 Vita Nuova Holdings Limited (www.vitanuova.com) +// Portions Copyright © 2004,2006 Bruce Ellis +// Portions Copyright © 2005-2007 C H Forsyth (forsyth@terzarima.net) +// Revisions Copyright © 2000-2007 Lucent Technologies Inc. and others +// Portions Copyright © 2009 The Go Authors. All rights reserved. +// +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to deal +// in the Software without restriction, including without limitation the rights +// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +// copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +// THE SOFTWARE. + +#include "gg.h" + +void +zname(Biobuf *b, Sym *s, int t) +{ + Bputc(b, ANAME); /* as */ + Bputc(b, t); /* type */ + Bputc(b, s->sym); /* sym */ + + Bputname(b, s); +} + +void +zfile(Biobuf *b, char *p, int n) +{ + Bputc(b, ANAME); + Bputc(b, D_FILE); + Bputc(b, 1); + Bputc(b, '<'); + Bwrite(b, p, n); + Bputc(b, 0); +} + +void +zhist(Biobuf *b, int line, vlong offset) +{ + Addr a; + + Bputc(b, AHISTORY); + Bputc(b, C_SCOND_NONE); + Bputc(b, NREG); + Bputc(b, line); + Bputc(b, line>>8); + Bputc(b, line>>16); + Bputc(b, line>>24); + zaddr(b, &zprog.from, 0); + a = zprog.to; + if(offset != 0) { + a.offset = offset; + a.type = D_CONST; + } + zaddr(b, &a, 0); +} + +void +zaddr(Biobuf *b, Addr *a, int s) +{ + int32 l; + uint64 e; + int i; + char *n; + + switch(a->type) { + case D_STATIC: + case D_AUTO: + case D_EXTERN: + case D_PARAM: + // TODO(kaib): remove once everything seems to work + fatal("We should no longer generate these as types"); + + default: + Bputc(b, a->type); + Bputc(b, a->reg); + Bputc(b, s); + Bputc(b, a->name); + } + + switch(a->type) { + default: + print("unknown type %d in zaddr\n", a->type); + + case D_NONE: + case D_REG: + case D_FREG: + case D_PSR: + break; + + case D_CONST2: + l = a->offset2; + Bputc(b, l); + Bputc(b, l>>8); + Bputc(b, l>>16); + Bputc(b, l>>24); // fall through + case D_OREG: + case D_CONST: + case D_SHIFT: + case D_STATIC: + case D_AUTO: + case D_EXTERN: + case D_PARAM: + l = a->offset; + Bputc(b, l); + Bputc(b, l>>8); + Bputc(b, l>>16); + Bputc(b, l>>24); + break; + + case D_BRANCH: + if(a->branch == nil) + fatal("unpatched branch"); + a->offset = a->branch->loc; + l = a->offset; + Bputc(b, l); + Bputc(b, l>>8); + Bputc(b, l>>16); + Bputc(b, l>>24); + break; + + case D_SCONST: + n = a->sval; + for(i=0; ioffset); + break; + + case D_FCONST: + ieeedtod(&e, a->dval); + l = e; + Bputc(b, l); + Bputc(b, l>>8); + Bputc(b, l>>16); + Bputc(b, l>>24); + l = e >> 32; + Bputc(b, l); + Bputc(b, l>>8); + Bputc(b, l>>16); + Bputc(b, l>>24); + break; + } +} + +void +dumpfuncs(void) +{ + Plist *pl; + int sf, st, t, sym; + struct { Sym *sym; short type; } h[NSYM]; + Sym *s; + Prog *p; + + for(sym=0; symlink) { + if(isblank(pl->name)) + continue; + for(p=pl->firstpc; p!=P; p=p->link) { + p->loc = pcloc; + if(p->as != ADATA && p->as != AGLOBL) + pcloc++; + } + } + + // put out functions + for(pl=plist; pl!=nil; pl=pl->link) { + if(isblank(pl->name)) + continue; + + if(debug['S']) { + s = S; + if(pl->name != N) + s = pl->name->sym; + print("\n--- prog list \"%S\" ---\n", s); + for(p=pl->firstpc; p!=P; p=p->link) + print("%P\n", p); + } + + for(p=pl->firstpc; p!=P; p=p->link) { + jackpot: + sf = 0; + s = p->from.sym; + while(s != S) { + sf = s->sym; + if(sf < 0 || sf >= NSYM) + sf = 0; + t = p->from.name; + if(t == D_ADDR) + t = p->from.name; + if(h[sf].type == t) + if(h[sf].sym == s) + break; + s->sym = sym; + zname(bout, s, t); + h[sym].sym = s; + h[sym].type = t; + sf = sym; + sym++; + if(sym >= NSYM) + sym = 1; + break; + } + st = 0; + s = p->to.sym; + while(s != S) { + st = s->sym; + if(st < 0 || st >= NSYM) + st = 0; + t = p->to.name; + if(t == D_ADDR) + t = p->to.name; + if(h[st].type == t) + if(h[st].sym == s) + break; + s->sym = sym; + zname(bout, s, t); + h[sym].sym = s; + h[sym].type = t; + st = sym; + sym++; + if(sym >= NSYM) + sym = 1; + if(st == sf) + goto jackpot; + break; + } + Bputc(bout, p->as); + Bputc(bout, p->scond); + Bputc(bout, p->reg); + Bputc(bout, p->lineno); + Bputc(bout, p->lineno>>8); + Bputc(bout, p->lineno>>16); + Bputc(bout, p->lineno>>24); + zaddr(bout, &p->from, sf); + zaddr(bout, &p->to, st); + } + } +} + +/* deferred DATA output */ +static Prog *strdat; +static Prog *estrdat; +static int gflag; +static Prog *savepc; + +void +data(void) +{ + gflag = debug['g']; + debug['g'] = 0; + + if(estrdat == nil) { + strdat = mal(sizeof(*pc)); + clearp(strdat); + estrdat = strdat; + } + if(savepc) + fatal("data phase error"); + savepc = pc; + pc = estrdat; +} + +void +text(void) +{ + if(!savepc) + fatal("text phase error"); + debug['g'] = gflag; + estrdat = pc; + pc = savepc; + savepc = nil; +} + +void +dumpdata(void) +{ + Prog *p; + + if(estrdat == nil) + return; + *pc = *strdat; + if(gflag) + for(p=pc; p!=estrdat; p=p->link) + print("%P\n", p); + pc = estrdat; +} + +int +dsname(Sym *sym, int off, char *t, int n) +{ + Prog *p; + + p = gins(ADATA, N, N); + p->from.type = D_OREG; + p->from.name = D_EXTERN; + p->from.etype = TINT32; + p->from.offset = off; + p->from.reg = NREG; + p->from.sym = sym; + + p->reg = n; + + p->to.type = D_SCONST; + p->to.name = D_NONE; + p->to.reg = NREG; + p->to.offset = 0; + memmove(p->to.sval, t, n); + return off + n; +} + +/* + * make a refer to the data s, s+len + * emitting DATA if needed. + */ +void +datastring(char *s, int len, Addr *a) +{ + Sym *sym; + + sym = stringsym(s, len); + a->type = D_OREG; + a->name = D_EXTERN; + a->etype = TINT32; + a->offset = widthptr+4; // skip header + a->reg = NREG; + a->sym = sym; +} + +/* + * make a refer to the string sval, + * emitting DATA if needed. + */ +void +datagostring(Strlit *sval, Addr *a) +{ + Sym *sym; + + sym = stringsym(sval->s, sval->len); + a->type = D_OREG; + a->name = D_EXTERN; + a->etype = TINT32; + a->offset = 0; // header + a->reg = NREG; + a->sym = sym; +} + +void +gdata(Node *nam, Node *nr, int wid) +{ + Prog *p; + vlong v; + + if(wid == 8 && is64(nr->type)) { + v = mpgetfix(nr->val.u.xval); + p = gins(ADATA, nam, nodintconst(v)); + p->reg = 4; + p = gins(ADATA, nam, nodintconst(v>>32)); + p->reg = 4; + p->from.offset += 4; + return; + } + p = gins(ADATA, nam, nr); + p->reg = wid; +} + +void +gdatacomplex(Node *nam, Mpcplx *cval) +{ + Prog *p; + int w; + + w = cplxsubtype(nam->type->etype); + w = types[w]->width; + + p = gins(ADATA, nam, N); + p->reg = w; + p->to.type = D_FCONST; + p->to.dval = mpgetflt(&cval->real); + + p = gins(ADATA, nam, N); + p->reg = w; + p->from.offset += w; + p->to.type = D_FCONST; + p->to.dval = mpgetflt(&cval->imag); +} + +void +gdatastring(Node *nam, Strlit *sval) +{ + Prog *p; + Node nod1; + + p = gins(ADATA, nam, N); + datastring(sval->s, sval->len, &p->to); + p->reg = types[tptr]->width; + p->to.type = D_CONST; + p->to.etype = TINT32; +//print("%P\n", p); + + nodconst(&nod1, types[TINT32], sval->len); + p = gins(ADATA, nam, &nod1); + p->reg = types[TINT32]->width; + p->from.offset += types[tptr]->width; +} + +int +dstringptr(Sym *s, int off, char *str) +{ + Prog *p; + + off = rnd(off, widthptr); + p = gins(ADATA, N, N); + p->from.type = D_OREG; + p->from.name = D_EXTERN; + p->from.sym = s; + p->from.offset = off; + p->reg = widthptr; + + datastring(str, strlen(str)+1, &p->to); + p->to.type = D_CONST; + p->to.etype = TINT32; + off += widthptr; + + return off; +} + +int +dgostrlitptr(Sym *s, int off, Strlit *lit) +{ + Prog *p; + + if(lit == nil) + return duintptr(s, off, 0); + + off = rnd(off, widthptr); + p = gins(ADATA, N, N); + p->from.type = D_OREG; + p->from.name = D_EXTERN; + p->from.sym = s; + p->from.offset = off; + p->reg = widthptr; + datagostring(lit, &p->to); + p->to.type = D_CONST; + p->to.etype = TINT32; + off += widthptr; + + return off; +} + +int +dgostringptr(Sym *s, int off, char *str) +{ + int n; + Strlit *lit; + + if(str == nil) + return duintptr(s, off, 0); + + n = strlen(str); + lit = mal(sizeof *lit + n); + strcpy(lit->s, str); + lit->len = n; + return dgostrlitptr(s, off, lit); +} + +int +duintxx(Sym *s, int off, uint64 v, int wid) +{ + Prog *p; + + off = rnd(off, wid); + + p = gins(ADATA, N, N); + p->from.type = D_OREG; + p->from.name = D_EXTERN; + p->from.sym = s; + p->from.offset = off; + p->reg = wid; + p->to.type = D_CONST; + p->to.name = D_NONE; + p->to.offset = v; + off += wid; + + return off; +} + +int +dsymptr(Sym *s, int off, Sym *x, int xoff) +{ + Prog *p; + + off = rnd(off, widthptr); + + p = gins(ADATA, N, N); + p->from.type = D_OREG; + p->from.name = D_EXTERN; + p->from.sym = s; + p->from.offset = off; + p->reg = widthptr; + p->to.type = D_CONST; + p->to.name = D_EXTERN; + p->to.sym = x; + p->to.offset = xoff; + off += widthptr; + + return off; +} + + +void +genembedtramp(Type *rcvr, Type *method, Sym *newnam, int iface) +{ + // TODO(kaib): re-implement genembedtramp + genwrapper(rcvr, method, newnam, iface); +/* + Sym *e; + int c, d, o; + Prog *p; + Type *f; + + e = method->sym; + for(d=0; dsym); + +out: + newplist()->name = newname(newnam); + + //TEXT main·S_test2(SB),7,$0 + p = pc; + gins(ATEXT, N, N); + p->from.type = D_OREG; + p->from.name = D_EXTERN; + p->from.sym = newnam; + p->to.type = D_CONST2; + p->reg = 7; + p->to.offset2 = 0; + p->to.reg = NREG; +//print("1. %P\n", p); + + o = 0; + for(c=d-1; c>=0; c--) { + f = dotlist[c].field; + o += f->width; + if(!isptr[f->type->etype]) + continue; + + //MOVW o(R0), R0 + p = pc; + gins(AMOVW, N, N); + p->from.type = D_OREG; + p->from.reg = REGARG; + p->from.offset = o; + p->to.type = D_REG; + p->to.reg = REGARG; +//print("2. %P\n", p); + o = 0; + } + if(o != 0) { + //MOVW $XX(R0), R0 + p = pc; + gins(AMOVW, N, N); + p->from.type = D_CONST; + p->from.reg = REGARG; + p->from.offset = o; + p->to.type = D_REG; + p->to.reg = REGARG; +//print("3. %P\n", p); + } + + f = dotlist[0].field; + //B main·*Sub_test2(SB) + if(isptr[f->type->etype]) + f = f->type; + p = pc; + gins(AB, N, N); + p->to.type = D_OREG; + p->to.reg = NREG; + p->to.name = D_EXTERN; + p->to.sym = methodsym(method->sym, ptrto(f->type), 0); +//print("4. %P\n", p); + + pc->as = ARET; // overwrite AEND +*/ +} + +void +nopout(Prog *p) +{ + p->as = ANOP; +} diff --git a/src/cmd/5g/gsubr.c b/src/cmd/5g/gsubr.c new file mode 100644 index 000000000..ddaf52a88 --- /dev/null +++ b/src/cmd/5g/gsubr.c @@ -0,0 +1,2010 @@ +// Derived from Inferno utils/5c/txt.c +// http://code.google.com/p/inferno-os/source/browse/utils/5c/txt.c +// +// Copyright © 1994-1999 Lucent Technologies Inc. All rights reserved. +// Portions Copyright © 1995-1997 C H Forsyth (forsyth@terzarima.net) +// Portions Copyright © 1997-1999 Vita Nuova Limited +// Portions Copyright © 2000-2007 Vita Nuova Holdings Limited (www.vitanuova.com) +// Portions Copyright © 2004,2006 Bruce Ellis +// Portions Copyright © 2005-2007 C H Forsyth (forsyth@terzarima.net) +// Revisions Copyright © 2000-2007 Lucent Technologies Inc. and others +// Portions Copyright © 2009 The Go Authors. All rights reserved. +// +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to deal +// in the Software without restriction, including without limitation the rights +// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +// copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +// THE SOFTWARE. + +#include "gg.h" + +// TODO(kaib): Can make this bigger if we move +// the text segment up higher in 5l for all GOOS. +long unmappedzero = 4096; + +void +clearp(Prog *p) +{ + p->as = AEND; + p->reg = NREG; + p->scond = C_SCOND_NONE; + p->from.type = D_NONE; + p->from.name = D_NONE; + p->from.reg = NREG; + p->to.type = D_NONE; + p->to.name = D_NONE; + p->to.reg = NREG; + p->loc = pcloc; + pcloc++; +} + +/* + * generate and return proc with p->as = as, + * linked into program. pc is next instruction. + */ +Prog* +prog(int as) +{ + Prog *p; + + p = pc; + pc = mal(sizeof(*pc)); + + clearp(pc); + + if(lineno == 0) { + if(debug['K']) + warn("prog: line 0"); + } + + p->as = as; + p->lineno = lineno; + p->link = pc; + return p; +} + +/* + * generate a branch. + * t is ignored. + */ +Prog* +gbranch(int as, Type *t) +{ + Prog *p; + + p = prog(as); + p->to.type = D_BRANCH; + p->to.branch = P; + return p; +} + +/* + * patch previous branch to jump to to. + */ +void +patch(Prog *p, Prog *to) +{ + if(p->to.type != D_BRANCH) + fatal("patch: not a branch"); + p->to.branch = to; + p->to.offset = to->loc; +} + +Prog* +unpatch(Prog *p) +{ + Prog *q; + + if(p->to.type != D_BRANCH) + fatal("unpatch: not a branch"); + q = p->to.branch; + p->to.branch = P; + p->to.offset = 0; + return q; +} + +/* + * start a new Prog list. + */ +Plist* +newplist(void) +{ + Plist *pl; + + pl = mal(sizeof(*pl)); + if(plist == nil) + plist = pl; + else + plast->link = pl; + plast = pl; + + pc = mal(sizeof(*pc)); + clearp(pc); + pl->firstpc = pc; + + return pl; +} + +void +clearstk(void) +{ + Plist *pl; + Prog *p, *p1, *p2, *p3; + Node dst, end, zero, con; + + if(plast->firstpc->to.offset <= 0) + return; + + // reestablish context for inserting code + // at beginning of function. + pl = plast; + p1 = pl->firstpc; + p2 = p1->link; + pc = mal(sizeof(*pc)); + clearp(pc); + p1->link = pc; + + // zero stack frame + + // MOVW $4(SP), R1 + nodreg(&dst, types[tptr], 1); + p = gins(AMOVW, N, &dst); + p->from.type = D_CONST; + p->from.reg = REGSP; + p->from.offset = 4; + + // MOVW $n(R1), R2 + nodreg(&end, types[tptr], 2); + p = gins(AMOVW, N, &end); + p->from.type = D_CONST; + p->from.reg = 1; + p->from.offset = p1->to.offset; + + // MOVW $0, R3 + nodreg(&zero, types[TUINT32], 3); + nodconst(&con, types[TUINT32], 0); + gmove(&con, &zero); + + // L: + // MOVW.P R3, 0(R1) +4 + // CMP R1, R2 + // BNE L + p = gins(AMOVW, &zero, &dst); + p->to.type = D_OREG; + p->to.offset = 4; + p->scond |= C_PBIT; + p3 = p; + p = gins(ACMP, &dst, N); + raddr(&end, p); + patch(gbranch(ABNE, T), p3); + + // continue with original code. + gins(ANOP, N, N)->link = p2; + pc = P; +} + +void +gused(Node *n) +{ + gins(ANOP, n, N); // used +} + +Prog* +gjmp(Prog *to) +{ + Prog *p; + + p = gbranch(AB, T); + if(to != P) + patch(p, to); + return p; +} + +void +ggloblnod(Node *nam, int32 width) +{ + Prog *p; + + p = gins(AGLOBL, nam, N); + p->lineno = nam->lineno; + p->to.sym = S; + p->to.type = D_CONST; + p->to.offset = width; +} + +void +ggloblsym(Sym *s, int32 width, int dupok) +{ + Prog *p; + + p = gins(AGLOBL, N, N); + p->from.type = D_OREG; + p->from.name = D_EXTERN; + p->from.sym = s; + p->to.type = D_CONST; + p->to.name = D_NONE; + p->to.offset = width; + if(dupok) + p->reg = DUPOK; +} + +int +isfat(Type *t) +{ + if(t != T) + switch(t->etype) { + case TSTRUCT: + case TARRAY: + case TSTRING: + case TINTER: // maybe remove later + return 1; + } + return 0; +} + +/* + * naddr of func generates code for address of func. + * if using opcode that can take address implicitly, + * call afunclit to fix up the argument. + * also fix up direct register references to be D_OREG. + */ +void +afunclit(Addr *a) +{ + if(a->type == D_CONST && a->name == D_EXTERN || a->type == D_REG) { + a->type = D_OREG; + } +} + +static int resvd[] = +{ + 9, // reserved for m + 10, // reserved for g +}; + +void +ginit(void) +{ + int i; + + for(i=0; ietype]; + if(is64(t)) + fatal("regalloc: 64 bit type %T"); + + switch(et) { + case TINT8: + case TUINT8: + case TINT16: + case TUINT16: + case TINT32: + case TUINT32: + case TPTR32: + case TBOOL: + if(o != N && o->op == OREGISTER) { + i = o->val.u.reg; + if(i >= REGALLOC_R0 && i <= REGALLOC_RMAX) + goto out; + } + for(i=REGALLOC_R0; i<=REGALLOC_RMAX; i++) + if(reg[i] == 0) + goto out; + + yyerror("out of fixed registers"); + goto err; + + case TFLOAT32: + case TFLOAT64: + if(o != N && o->op == OREGISTER) { + i = o->val.u.reg; + if(i >= REGALLOC_F0 && i <= REGALLOC_FMAX) + goto out; + } + for(i=REGALLOC_F0; i<=REGALLOC_FMAX; i++) + if(reg[i] == 0) + goto out; + yyerror("out of floating point registers"); + goto err; + + case TCOMPLEX64: + case TCOMPLEX128: + tempname(n, t); + return; + } + yyerror("regalloc: unknown type %T", t); + +err: + nodreg(n, t, 0); + return; + +out: + reg[i]++; + nodreg(n, t, i); +} + +void +regfree(Node *n) +{ + int i, fixfree, floatfree; + + if(debug['r']) { + fixfree = 0; + for(i=REGALLOC_R0; i<=REGALLOC_RMAX; i++) + if(reg[i] == 0) + fixfree++; + floatfree = 0; + for(i=REGALLOC_F0; i<=REGALLOC_FMAX; i++) + if(reg[i] == 0) + floatfree++; + print("regalloc fix %d float %d\n", fixfree, floatfree); + } + + if(n->op == ONAME && iscomplex[n->type->etype]) + return; + if(n->op != OREGISTER && n->op != OINDREG) + fatal("regfree: not a register"); + i = n->val.u.reg; + if(i < 0 || i >= sizeof(reg)) + fatal("regfree: reg out of range"); + if(reg[i] <= 0) + fatal("regfree: reg not allocated"); + reg[i]--; +} + +/* + * initialize n to be register r of type t. + */ +void +nodreg(Node *n, Type *t, int r) +{ + if(t == T) + fatal("nodreg: t nil"); + + memset(n, 0, sizeof(*n)); + n->op = OREGISTER; + n->addable = 1; + ullmancalc(n); + n->val.u.reg = r; + n->type = t; +} + +/* + * initialize n to be indirect of register r; n is type t. + */ +void +nodindreg(Node *n, Type *t, int r) +{ + nodreg(n, t, r); + n->op = OINDREG; +} + +Node* +nodarg(Type *t, int fp) +{ + Node *n; + Type *first; + Iter savet; + + // entire argument struct, not just one arg + if(t->etype == TSTRUCT && t->funarg) { + n = nod(ONAME, N, N); + n->sym = lookup(".args"); + n->type = t; + first = structfirst(&savet, &t); + if(first == nil) + fatal("nodarg: bad struct"); + if(first->width == BADWIDTH) + fatal("nodarg: offset not computed for %T", t); + n->xoffset = first->width; + n->addable = 1; + goto fp; + } + + if(t->etype != TFIELD) + fatal("nodarg: not field %T", t); + + n = nod(ONAME, N, N); + n->type = t->type; + n->sym = t->sym; + if(t->width == BADWIDTH) + fatal("nodarg: offset not computed for %T", t); + n->xoffset = t->width; + n->addable = 1; + +fp: + switch(fp) { + default: + fatal("nodarg %T %d", t, fp); + + case 0: // output arg for calling another function + n->op = OINDREG; + n->val.u.reg = REGSP; + n->xoffset += 4; + break; + + case 1: // input arg to current function + n->class = PPARAM; + break; + } + n->typecheck = 1; + return n; +} + +/* + * return constant i node. + * overwritten by next call, but useful in calls to gins. + */ +Node* +ncon(uint32 i) +{ + static Node n; + + if(n.type == T) + nodconst(&n, types[TUINT32], 0); + mpmovecfix(n.val.u.xval, i); + return &n; +} + +/* + * Is this node a memory operand? + */ +int +ismem(Node *n) +{ + switch(n->op) { + case OINDREG: + case ONAME: + case OPARAM: + return 1; + } + return 0; +} + +Node sclean[10]; +int nsclean; + +/* + * n is a 64-bit value. fill in lo and hi to refer to its 32-bit halves. + */ +void +split64(Node *n, Node *lo, Node *hi) +{ + Node n1; + int64 i; + + if(!is64(n->type)) + fatal("split64 %T", n->type); + + sclean[nsclean].op = OEMPTY; + if(nsclean >= nelem(sclean)) + fatal("split64 clean"); + nsclean++; + switch(n->op) { + default: + if(!dotaddable(n, &n1)) { + igen(n, &n1, N); + sclean[nsclean-1] = n1; + } + n = &n1; + goto common; + case ONAME: + if(n->class == PPARAMREF) { + cgen(n->heapaddr, &n1); + sclean[nsclean-1] = n1; + // fall through. + n = &n1; + } + goto common; + case OINDREG: + common: + *lo = *n; + *hi = *n; + lo->type = types[TUINT32]; + if(n->type->etype == TINT64) + hi->type = types[TINT32]; + else + hi->type = types[TUINT32]; + hi->xoffset += 4; + break; + + case OLITERAL: + convconst(&n1, n->type, &n->val); + i = mpgetfix(n1.val.u.xval); + nodconst(lo, types[TUINT32], (uint32)i); + i >>= 32; + if(n->type->etype == TINT64) + nodconst(hi, types[TINT32], (int32)i); + else + nodconst(hi, types[TUINT32], (uint32)i); + break; + } +} + +void +splitclean(void) +{ + if(nsclean <= 0) + fatal("splitclean"); + nsclean--; + if(sclean[nsclean].op != OEMPTY) + regfree(&sclean[nsclean]); +} + +#define CASE(a,b) (((a)<<16)|((b)<<0)) + +void +gmove(Node *f, Node *t) +{ + int a, ft, tt, fa, ta; + Type *cvt; + Node r1, r2, flo, fhi, tlo, thi, con; + Prog *p1; + + if(debug['M']) + print("gmove %N -> %N\n", f, t); + + ft = simsimtype(f->type); + tt = simsimtype(t->type); + cvt = t->type; + + if(iscomplex[ft] || iscomplex[tt]) { + complexmove(f, t); + return; + } + + // cannot have two memory operands; + // except 64-bit, which always copies via registers anyway. + if(!is64(f->type) && !is64(t->type) && ismem(f) && ismem(t)) + goto hard; + + // convert constant to desired type + if(f->op == OLITERAL) { + switch(tt) { + default: + convconst(&con, t->type, &f->val); + break; + + case TINT16: + case TINT8: + convconst(&con, types[TINT32], &f->val); + regalloc(&r1, con.type, t); + gins(AMOVW, &con, &r1); + gmove(&r1, t); + regfree(&r1); + return; + + case TUINT16: + case TUINT8: + convconst(&con, types[TUINT32], &f->val); + regalloc(&r1, con.type, t); + gins(AMOVW, &con, &r1); + gmove(&r1, t); + regfree(&r1); + return; + } + + f = &con; + ft = simsimtype(con.type); + + // constants can't move directly to memory + if(ismem(t) && !is64(t->type)) goto hard; + } + + // value -> value copy, only one memory operand. + // figure out the instruction to use. + // break out of switch for one-instruction gins. + // goto rdst for "destination must be register". + // goto hard for "convert to cvt type first". + // otherwise handle and return. + + switch(CASE(ft, tt)) { + default: + goto fatal; + + /* + * integer copy and truncate + */ + case CASE(TINT8, TINT8): // same size + case CASE(TUINT8, TINT8): + case CASE(TINT16, TINT8): // truncate + case CASE(TUINT16, TINT8): + case CASE(TINT32, TINT8): + case CASE(TUINT32, TINT8): + a = AMOVB; + break; + + case CASE(TINT8, TUINT8): + case CASE(TUINT8, TUINT8): + case CASE(TINT16, TUINT8): + case CASE(TUINT16, TUINT8): + case CASE(TINT32, TUINT8): + case CASE(TUINT32, TUINT8): + a = AMOVBU; + break; + + case CASE(TINT64, TINT8): // truncate low word + case CASE(TUINT64, TINT8): + a = AMOVB; + goto trunc64; + + case CASE(TINT64, TUINT8): + case CASE(TUINT64, TUINT8): + a = AMOVBU; + goto trunc64; + + case CASE(TINT16, TINT16): // same size + case CASE(TUINT16, TINT16): + case CASE(TINT32, TINT16): // truncate + case CASE(TUINT32, TINT16): + a = AMOVH; + break; + + case CASE(TINT16, TUINT16): + case CASE(TUINT16, TUINT16): + case CASE(TINT32, TUINT16): + case CASE(TUINT32, TUINT16): + a = AMOVHU; + break; + + case CASE(TINT64, TINT16): // truncate low word + case CASE(TUINT64, TINT16): + a = AMOVH; + goto trunc64; + + case CASE(TINT64, TUINT16): + case CASE(TUINT64, TUINT16): + a = AMOVHU; + goto trunc64; + + case CASE(TINT32, TINT32): // same size + case CASE(TINT32, TUINT32): + case CASE(TUINT32, TINT32): + case CASE(TUINT32, TUINT32): + a = AMOVW; + break; + + case CASE(TINT64, TINT32): // truncate + case CASE(TUINT64, TINT32): + case CASE(TINT64, TUINT32): + case CASE(TUINT64, TUINT32): + split64(f, &flo, &fhi); + regalloc(&r1, t->type, N); + gins(AMOVW, &flo, &r1); + gins(AMOVW, &r1, t); + regfree(&r1); + splitclean(); + return; + + case CASE(TINT64, TINT64): // same size + case CASE(TINT64, TUINT64): + case CASE(TUINT64, TINT64): + case CASE(TUINT64, TUINT64): + split64(f, &flo, &fhi); + split64(t, &tlo, &thi); + regalloc(&r1, flo.type, N); + regalloc(&r2, fhi.type, N); + gins(AMOVW, &flo, &r1); + gins(AMOVW, &fhi, &r2); + gins(AMOVW, &r1, &tlo); + gins(AMOVW, &r2, &thi); + regfree(&r1); + regfree(&r2); + splitclean(); + splitclean(); + return; + + /* + * integer up-conversions + */ + case CASE(TINT8, TINT16): // sign extend int8 + case CASE(TINT8, TUINT16): + case CASE(TINT8, TINT32): + case CASE(TINT8, TUINT32): + a = AMOVB; + goto rdst; + case CASE(TINT8, TINT64): // convert via int32 + case CASE(TINT8, TUINT64): + cvt = types[TINT32]; + goto hard; + + case CASE(TUINT8, TINT16): // zero extend uint8 + case CASE(TUINT8, TUINT16): + case CASE(TUINT8, TINT32): + case CASE(TUINT8, TUINT32): + a = AMOVBU; + goto rdst; + case CASE(TUINT8, TINT64): // convert via uint32 + case CASE(TUINT8, TUINT64): + cvt = types[TUINT32]; + goto hard; + + case CASE(TINT16, TINT32): // sign extend int16 + case CASE(TINT16, TUINT32): + a = AMOVH; + goto rdst; + case CASE(TINT16, TINT64): // convert via int32 + case CASE(TINT16, TUINT64): + cvt = types[TINT32]; + goto hard; + + case CASE(TUINT16, TINT32): // zero extend uint16 + case CASE(TUINT16, TUINT32): + a = AMOVHU; + goto rdst; + case CASE(TUINT16, TINT64): // convert via uint32 + case CASE(TUINT16, TUINT64): + cvt = types[TUINT32]; + goto hard; + + case CASE(TINT32, TINT64): // sign extend int32 + case CASE(TINT32, TUINT64): + split64(t, &tlo, &thi); + regalloc(&r1, tlo.type, N); + regalloc(&r2, thi.type, N); + gmove(f, &r1); + p1 = gins(AMOVW, &r1, &r2); + p1->from.type = D_SHIFT; + p1->from.offset = 2 << 5 | 31 << 7 | r1.val.u.reg; // r1->31 + p1->from.reg = NREG; +//print("gmove: %P\n", p1); + gins(AMOVW, &r1, &tlo); + gins(AMOVW, &r2, &thi); + regfree(&r1); + regfree(&r2); + splitclean(); + return; + + case CASE(TUINT32, TINT64): // zero extend uint32 + case CASE(TUINT32, TUINT64): + split64(t, &tlo, &thi); + gmove(f, &tlo); + regalloc(&r1, thi.type, N); + gins(AMOVW, ncon(0), &r1); + gins(AMOVW, &r1, &thi); + regfree(&r1); + splitclean(); + return; + + /* + * float to integer + */ + case CASE(TFLOAT32, TINT8): + case CASE(TFLOAT32, TUINT8): + case CASE(TFLOAT32, TINT16): + case CASE(TFLOAT32, TUINT16): + case CASE(TFLOAT32, TINT32): + case CASE(TFLOAT32, TUINT32): +// case CASE(TFLOAT32, TUINT64): + + case CASE(TFLOAT64, TINT8): + case CASE(TFLOAT64, TUINT8): + case CASE(TFLOAT64, TINT16): + case CASE(TFLOAT64, TUINT16): + case CASE(TFLOAT64, TINT32): + case CASE(TFLOAT64, TUINT32): +// case CASE(TFLOAT64, TUINT64): + fa = AMOVF; + a = AMOVFW; + if(ft == TFLOAT64) { + fa = AMOVD; + a = AMOVDW; + } + ta = AMOVW; + switch(tt) { + case TINT8: + ta = AMOVB; + break; + case TUINT8: + ta = AMOVBU; + break; + case TINT16: + ta = AMOVH; + break; + case TUINT16: + ta = AMOVHU; + break; + } + + regalloc(&r1, types[ft], f); + regalloc(&r2, types[tt], t); + gins(fa, f, &r1); // load to fpu + p1 = gins(a, &r1, &r1); // convert to w + switch(tt) { + case TUINT8: + case TUINT16: + case TUINT32: + p1->scond |= C_UBIT; + } + gins(AMOVW, &r1, &r2); // copy to cpu + gins(ta, &r2, t); // store + regfree(&r1); + regfree(&r2); + return; + + /* + * integer to float + */ + case CASE(TINT8, TFLOAT32): + case CASE(TUINT8, TFLOAT32): + case CASE(TINT16, TFLOAT32): + case CASE(TUINT16, TFLOAT32): + case CASE(TINT32, TFLOAT32): + case CASE(TUINT32, TFLOAT32): + case CASE(TINT8, TFLOAT64): + case CASE(TUINT8, TFLOAT64): + case CASE(TINT16, TFLOAT64): + case CASE(TUINT16, TFLOAT64): + case CASE(TINT32, TFLOAT64): + case CASE(TUINT32, TFLOAT64): + fa = AMOVW; + switch(ft) { + case TINT8: + fa = AMOVB; + break; + case TUINT8: + fa = AMOVBU; + break; + case TINT16: + fa = AMOVH; + break; + case TUINT16: + fa = AMOVHU; + break; + } + a = AMOVWF; + ta = AMOVF; + if(tt == TFLOAT64) { + a = AMOVWD; + ta = AMOVD; + } + regalloc(&r1, types[ft], f); + regalloc(&r2, types[tt], t); + gins(fa, f, &r1); // load to cpu + gins(AMOVW, &r1, &r2); // copy to fpu + p1 = gins(a, &r2, &r2); // convert + switch(ft) { + case TUINT8: + case TUINT16: + case TUINT32: + p1->scond |= C_UBIT; + } + gins(ta, &r2, t); // store + regfree(&r1); + regfree(&r2); + return; + + case CASE(TUINT64, TFLOAT32): + case CASE(TUINT64, TFLOAT64): + fatal("gmove UINT64, TFLOAT not implemented"); + return; + + + /* + * float to float + */ + case CASE(TFLOAT32, TFLOAT32): + a = AMOVF; + break; + + case CASE(TFLOAT64, TFLOAT64): + a = AMOVD; + break; + + case CASE(TFLOAT32, TFLOAT64): + regalloc(&r1, types[TFLOAT64], t); + gins(AMOVF, f, &r1); + gins(AMOVFD, &r1, &r1); + gins(AMOVD, &r1, t); + regfree(&r1); + return; + + case CASE(TFLOAT64, TFLOAT32): + regalloc(&r1, types[TFLOAT64], t); + gins(AMOVD, f, &r1); + gins(AMOVDF, &r1, &r1); + gins(AMOVF, &r1, t); + regfree(&r1); + return; + } + + gins(a, f, t); + return; + +rdst: + // TODO(kaib): we almost always require a register dest anyway, this can probably be + // removed. + // requires register destination + regalloc(&r1, t->type, t); + gins(a, f, &r1); + gmove(&r1, t); + regfree(&r1); + return; + +hard: + // requires register intermediate + regalloc(&r1, cvt, t); + gmove(f, &r1); + gmove(&r1, t); + regfree(&r1); + return; + +trunc64: + // truncate 64 bit integer + split64(f, &flo, &fhi); + regalloc(&r1, t->type, N); + gins(a, &flo, &r1); + gins(a, &r1, t); + regfree(&r1); + splitclean(); + return; + +fatal: + // should not happen + fatal("gmove %N -> %N", f, t); +} + +int +samaddr(Node *f, Node *t) +{ + + if(f->op != t->op) + return 0; + + switch(f->op) { + case OREGISTER: + if(f->val.u.reg != t->val.u.reg) + break; + return 1; + } + return 0; +} + +/* + * generate one instruction: + * as f, t + */ +Prog* +gins(int as, Node *f, Node *t) +{ +// Node nod; +// int32 v; + Prog *p; + Addr af, at; + + if(f != N && f->op == OINDEX) { + fatal("gins OINDEX not implemented"); +// regalloc(&nod, ®node, Z); +// v = constnode.vconst; +// cgen(f->right, &nod); +// constnode.vconst = v; +// idx.reg = nod.reg; +// regfree(&nod); + } + if(t != N && t->op == OINDEX) { + fatal("gins OINDEX not implemented"); +// regalloc(&nod, ®node, Z); +// v = constnode.vconst; +// cgen(t->right, &nod); +// constnode.vconst = v; +// idx.reg = nod.reg; +// regfree(&nod); + } + + memset(&af, 0, sizeof af); + memset(&at, 0, sizeof at); + if(f != N) + naddr(f, &af, 1); + if(t != N) + naddr(t, &at, 1); p = prog(as); + if(f != N) + p->from = af; + if(t != N) + p->to = at; + if(debug['g']) + print("%P\n", p); + return p; +} + +/* + * insert n into reg slot of p + */ +void +raddr(Node *n, Prog *p) +{ + Addr a; + + naddr(n, &a, 1); + if(a.type != D_REG && a.type != D_FREG) { + if(n) + fatal("bad in raddr: %O", n->op); + else + fatal("bad in raddr: "); + p->reg = NREG; + } else + p->reg = a.reg; +} + +/* generate a comparison +TODO(kaib): one of the args can actually be a small constant. relax the constraint and fix call sites. + */ +Prog* +gcmp(int as, Node *lhs, Node *rhs) +{ + Prog *p; + + if(lhs->op != OREGISTER || rhs->op != OREGISTER) + fatal("bad operands to gcmp: %O %O", lhs->op, rhs->op); + + p = gins(as, rhs, N); + raddr(lhs, p); + return p; +} + +/* generate a constant shift + * arm encodes a shift by 32 as 0, thus asking for 0 shift is illegal. +*/ +Prog* +gshift(int as, Node *lhs, int32 stype, int32 sval, Node *rhs) +{ + Prog *p; + + if(sval <= 0 || sval > 32) + fatal("bad shift value: %d", sval); + + sval = sval&0x1f; + + p = gins(as, N, rhs); + p->from.type = D_SHIFT; + p->from.offset = stype | sval<<7 | lhs->val.u.reg; + return p; +} + +/* generate a register shift +*/ +Prog * +gregshift(int as, Node *lhs, int32 stype, Node *reg, Node *rhs) +{ + Prog *p; + p = gins(as, N, rhs); + p->from.type = D_SHIFT; + p->from.offset = stype | reg->val.u.reg << 8 | 1<<4 | lhs->val.u.reg; + return p; +} + +static void +checkoffset(Addr *a, int canemitcode) +{ + Prog *p; + Node n1; + + if(a->offset < unmappedzero) + return; + if(!canemitcode) + fatal("checkoffset %#x, cannot emit code", a->offset); + + // cannot rely on unmapped nil page at 0 to catch + // reference with large offset. instead, emit explicit + // test of 0(reg). + regalloc(&n1, types[TUINTPTR], N); + p = gins(AMOVW, N, &n1); + p->from = *a; + p->from.offset = 0; + regfree(&n1); +} + +/* + * generate code to compute n; + * make a refer to result. + */ +void +naddr(Node *n, Addr *a, int canemitcode) +{ + a->type = D_NONE; + a->name = D_NONE; + a->reg = NREG; + a->node = N; + a->etype = 0; + if(n == N) + return; + + switch(n->op) { + default: + fatal("naddr: bad %O %D", n->op, a); + break; + + case OREGISTER: + if(n->val.u.reg <= REGALLOC_RMAX) { + a->type = D_REG; + a->reg = n->val.u.reg; + } else { + a->type = D_FREG; + a->reg = n->val.u.reg - REGALLOC_F0; + } + a->sym = S; + break; + + case OINDEX: + case OIND: + fatal("naddr: OINDEX"); +// naddr(n->left, a); +// if(a->type >= D_AX && a->type <= D_DI) +// a->type += D_INDIR; +// else +// if(a->type == D_CONST) +// a->type = D_NONE+D_INDIR; +// else +// if(a->type == D_ADDR) { +// a->type = a->index; +// a->index = D_NONE; +// } else +// goto bad; +// if(n->op == OINDEX) { +// a->index = idx.reg; +// a->scale = n->scale; +// } +// break; + + case OINDREG: + a->type = D_OREG; + a->reg = n->val.u.reg; + a->sym = n->sym; + a->offset = n->xoffset; + checkoffset(a, canemitcode); + break; + + case OPARAM: + // n->left is PHEAP ONAME for stack parameter. + // compute address of actual parameter on stack. + a->etype = simtype[n->left->type->etype]; + a->width = n->left->type->width; + a->offset = n->xoffset; + a->sym = n->left->sym; + a->type = D_OREG; + a->name = D_PARAM; + break; + + case ONAME: + a->etype = 0; + a->width = 0; + a->reg = NREG; + if(n->type != T) { + a->etype = simtype[n->type->etype]; + a->width = n->type->width; + } + a->pun = n->pun; + a->offset = n->xoffset; + a->sym = n->sym; + if(a->sym == S) + a->sym = lookup(".noname"); + if(n->method) { + if(n->type != T) + if(n->type->sym != S) + if(n->type->sym->pkg != nil) + a->sym = pkglookup(a->sym->name, n->type->sym->pkg); + } + + a->type = D_OREG; + switch(n->class) { + default: + fatal("naddr: ONAME class %S %d\n", n->sym, n->class); + case PEXTERN: + a->name = D_EXTERN; + break; + case PAUTO: + a->name = D_AUTO; + if (n->sym) + a->node = n->orig; + break; + case PPARAM: + case PPARAMOUT: + a->name = D_PARAM; + break; + case PFUNC: + a->name = D_EXTERN; + a->type = D_CONST; + break; + } + break; + + case OLITERAL: + switch(n->val.ctype) { + default: + fatal("naddr: const %lT", n->type); + break; + case CTFLT: + a->type = D_FCONST; + a->dval = mpgetflt(n->val.u.fval); + break; + case CTINT: + a->sym = S; + a->type = D_CONST; + a->offset = mpgetfix(n->val.u.xval); + break; + case CTSTR: + datagostring(n->val.u.sval, a); + break; + case CTBOOL: + a->sym = S; + a->type = D_CONST; + a->offset = n->val.u.bval; + break; + case CTNIL: + a->sym = S; + a->type = D_CONST; + a->offset = 0; + break; + } + break; + + case OLEN: + // len of string or slice + naddr(n->left, a, canemitcode); + a->etype = TINT32; + if(a->type == D_CONST && a->offset == 0) + break; // len(nil) + a->offset += Array_nel; + if(a->offset >= unmappedzero && a->offset-Array_nel < unmappedzero) + checkoffset(a, canemitcode); + break; + + case OCAP: + // cap of string or slice + naddr(n->left, a, canemitcode); + a->etype = TINT32; + if(a->type == D_CONST && a->offset == 0) + break; // cap(nil) + a->offset += Array_cap; + if(a->offset >= unmappedzero && a->offset-Array_cap < unmappedzero) + checkoffset(a, canemitcode); + break; + + case OADDR: + naddr(n->left, a, canemitcode); + a->etype = tptr; + switch(a->type) { + case D_OREG: + a->type = D_CONST; + break; + + case D_REG: + case D_CONST: + break; + + default: + fatal("naddr: OADDR %d\n", a->type); + } + } +} + +/* + * return Axxx for Oxxx on type t. + */ +int +optoas(int op, Type *t) +{ + int a; + + if(t == T) + fatal("optoas: t is nil"); + + a = AGOK; + switch(CASE(op, simtype[t->etype])) { + default: + fatal("optoas: no entry %O-%T etype %T simtype %T", op, t, types[t->etype], types[simtype[t->etype]]); + break; + +/* case CASE(OADDR, TPTR32): + a = ALEAL; + break; + + case CASE(OADDR, TPTR64): + a = ALEAQ; + break; +*/ + // TODO(kaib): make sure the conditional branches work on all edge cases + case CASE(OEQ, TBOOL): + case CASE(OEQ, TINT8): + case CASE(OEQ, TUINT8): + case CASE(OEQ, TINT16): + case CASE(OEQ, TUINT16): + case CASE(OEQ, TINT32): + case CASE(OEQ, TUINT32): + case CASE(OEQ, TINT64): + case CASE(OEQ, TUINT64): + case CASE(OEQ, TPTR32): + case CASE(OEQ, TPTR64): + case CASE(OEQ, TFLOAT32): + case CASE(OEQ, TFLOAT64): + a = ABEQ; + break; + + case CASE(ONE, TBOOL): + case CASE(ONE, TINT8): + case CASE(ONE, TUINT8): + case CASE(ONE, TINT16): + case CASE(ONE, TUINT16): + case CASE(ONE, TINT32): + case CASE(ONE, TUINT32): + case CASE(ONE, TINT64): + case CASE(ONE, TUINT64): + case CASE(ONE, TPTR32): + case CASE(ONE, TPTR64): + case CASE(ONE, TFLOAT32): + case CASE(ONE, TFLOAT64): + a = ABNE; + break; + + case CASE(OLT, TINT8): + case CASE(OLT, TINT16): + case CASE(OLT, TINT32): + case CASE(OLT, TINT64): + case CASE(OLT, TFLOAT32): + case CASE(OLT, TFLOAT64): + a = ABLT; + break; + + case CASE(OLT, TUINT8): + case CASE(OLT, TUINT16): + case CASE(OLT, TUINT32): + case CASE(OLT, TUINT64): + a = ABLO; + break; + + case CASE(OLE, TINT8): + case CASE(OLE, TINT16): + case CASE(OLE, TINT32): + case CASE(OLE, TINT64): + case CASE(OLE, TFLOAT32): + case CASE(OLE, TFLOAT64): + a = ABLE; + break; + + case CASE(OLE, TUINT8): + case CASE(OLE, TUINT16): + case CASE(OLE, TUINT32): + case CASE(OLE, TUINT64): + a = ABLS; + break; + + case CASE(OGT, TINT8): + case CASE(OGT, TINT16): + case CASE(OGT, TINT32): + case CASE(OGT, TINT64): + case CASE(OGT, TFLOAT32): + case CASE(OGT, TFLOAT64): + a = ABGT; + break; + + case CASE(OGT, TUINT8): + case CASE(OGT, TUINT16): + case CASE(OGT, TUINT32): + case CASE(OGT, TUINT64): + a = ABHI; + break; + + case CASE(OGE, TINT8): + case CASE(OGE, TINT16): + case CASE(OGE, TINT32): + case CASE(OGE, TINT64): + case CASE(OGE, TFLOAT32): + case CASE(OGE, TFLOAT64): + a = ABGE; + break; + + case CASE(OGE, TUINT8): + case CASE(OGE, TUINT16): + case CASE(OGE, TUINT32): + case CASE(OGE, TUINT64): + a = ABHS; + break; + + case CASE(OCMP, TBOOL): + case CASE(OCMP, TINT8): + case CASE(OCMP, TUINT8): + case CASE(OCMP, TINT16): + case CASE(OCMP, TUINT16): + case CASE(OCMP, TINT32): + case CASE(OCMP, TUINT32): + case CASE(OCMP, TPTR32): + a = ACMP; + break; + + case CASE(OCMP, TFLOAT32): + a = ACMPF; + break; + + case CASE(OCMP, TFLOAT64): + a = ACMPD; + break; + + case CASE(OAS, TBOOL): + case CASE(OAS, TINT8): + a = AMOVB; + break; + + case CASE(OAS, TUINT8): + a = AMOVBU; + break; + + case CASE(OAS, TINT16): + a = AMOVH; + break; + + case CASE(OAS, TUINT16): + a = AMOVHU; + break; + + case CASE(OAS, TINT32): + case CASE(OAS, TUINT32): + case CASE(OAS, TPTR32): + a = AMOVW; + break; + + case CASE(OAS, TFLOAT32): + a = AMOVF; + break; + + case CASE(OAS, TFLOAT64): + a = AMOVD; + break; + + case CASE(OADD, TINT8): + case CASE(OADD, TUINT8): + case CASE(OADD, TINT16): + case CASE(OADD, TUINT16): + case CASE(OADD, TINT32): + case CASE(OADD, TUINT32): + case CASE(OADD, TPTR32): + a = AADD; + break; + + case CASE(OADD, TFLOAT32): + a = AADDF; + break; + + case CASE(OADD, TFLOAT64): + a = AADDD; + break; + + case CASE(OSUB, TINT8): + case CASE(OSUB, TUINT8): + case CASE(OSUB, TINT16): + case CASE(OSUB, TUINT16): + case CASE(OSUB, TINT32): + case CASE(OSUB, TUINT32): + case CASE(OSUB, TPTR32): + a = ASUB; + break; + + case CASE(OSUB, TFLOAT32): + a = ASUBF; + break; + + case CASE(OSUB, TFLOAT64): + a = ASUBD; + break; + + case CASE(OAND, TINT8): + case CASE(OAND, TUINT8): + case CASE(OAND, TINT16): + case CASE(OAND, TUINT16): + case CASE(OAND, TINT32): + case CASE(OAND, TUINT32): + case CASE(OAND, TPTR32): + a = AAND; + break; + + case CASE(OOR, TINT8): + case CASE(OOR, TUINT8): + case CASE(OOR, TINT16): + case CASE(OOR, TUINT16): + case CASE(OOR, TINT32): + case CASE(OOR, TUINT32): + case CASE(OOR, TPTR32): + a = AORR; + break; + + case CASE(OXOR, TINT8): + case CASE(OXOR, TUINT8): + case CASE(OXOR, TINT16): + case CASE(OXOR, TUINT16): + case CASE(OXOR, TINT32): + case CASE(OXOR, TUINT32): + case CASE(OXOR, TPTR32): + a = AEOR; + break; + + case CASE(OLSH, TINT8): + case CASE(OLSH, TUINT8): + case CASE(OLSH, TINT16): + case CASE(OLSH, TUINT16): + case CASE(OLSH, TINT32): + case CASE(OLSH, TUINT32): + case CASE(OLSH, TPTR32): + a = ASLL; + break; + + case CASE(ORSH, TUINT8): + case CASE(ORSH, TUINT16): + case CASE(ORSH, TUINT32): + case CASE(ORSH, TPTR32): + a = ASRL; + break; + + case CASE(ORSH, TINT8): + case CASE(ORSH, TINT16): + case CASE(ORSH, TINT32): + a = ASRA; + break; + + case CASE(OMUL, TUINT8): + case CASE(OMUL, TUINT16): + case CASE(OMUL, TUINT32): + case CASE(OMUL, TPTR32): + a = AMULU; + break; + + case CASE(OMUL, TINT8): + case CASE(OMUL, TINT16): + case CASE(OMUL, TINT32): + a = AMUL; + break; + + case CASE(OMUL, TFLOAT32): + a = AMULF; + break; + + case CASE(OMUL, TFLOAT64): + a = AMULD; + break; + + case CASE(ODIV, TUINT8): + case CASE(ODIV, TUINT16): + case CASE(ODIV, TUINT32): + case CASE(ODIV, TPTR32): + a = ADIVU; + break; + + case CASE(ODIV, TINT8): + case CASE(ODIV, TINT16): + case CASE(ODIV, TINT32): + a = ADIV; + break; + + case CASE(OMOD, TUINT8): + case CASE(OMOD, TUINT16): + case CASE(OMOD, TUINT32): + case CASE(OMOD, TPTR32): + a = AMODU; + break; + + case CASE(OMOD, TINT8): + case CASE(OMOD, TINT16): + case CASE(OMOD, TINT32): + a = AMOD; + break; + +// case CASE(OEXTEND, TINT16): +// a = ACWD; +// break; + +// case CASE(OEXTEND, TINT32): +// a = ACDQ; +// break; + +// case CASE(OEXTEND, TINT64): +// a = ACQO; +// break; + + case CASE(ODIV, TFLOAT32): + a = ADIVF; + break; + + case CASE(ODIV, TFLOAT64): + a = ADIVD; + break; + + } + return a; +} + +enum +{ + ODynam = 1<<0, + OPtrto = 1<<1, +}; + +static Node clean[20]; +static int cleani = 0; + +void +sudoclean(void) +{ + if(clean[cleani-1].op != OEMPTY) + regfree(&clean[cleani-1]); + if(clean[cleani-2].op != OEMPTY) + regfree(&clean[cleani-2]); + cleani -= 2; +} + +int +dotaddable(Node *n, Node *n1) +{ + int o, oary[10]; + Node *nn; + + if(n->op != ODOT) + return 0; + + o = dotoffset(n, oary, &nn); + if(nn != N && nn->addable && o == 1 && oary[0] >= 0) { + *n1 = *nn; + n1->type = n->type; + n1->xoffset += oary[0]; + return 1; + } + return 0; +} + +/* + * generate code to compute address of n, + * a reference to a (perhaps nested) field inside + * an array or struct. + * return 0 on failure, 1 on success. + * on success, leaves usable address in a. + * + * caller is responsible for calling sudoclean + * after successful sudoaddable, + * to release the register used for a. + */ +int +sudoaddable(int as, Node *n, Addr *a, int *w) +{ + int o, i; + int oary[10]; + int64 v; + Node n1, n2, n3, n4, *nn, *l, *r; + Node *reg, *reg1; + Prog *p1, *p2; + Type *t; + + if(n->type == T) + return 0; + + switch(n->op) { + case OLITERAL: + if(n->val.ctype != CTINT) + break; + v = mpgetfix(n->val.u.xval); + if(v >= 32000 || v <= -32000) + break; + goto lit; + + case ODOT: + case ODOTPTR: + cleani += 2; + reg = &clean[cleani-1]; + reg1 = &clean[cleani-2]; + reg->op = OEMPTY; + reg1->op = OEMPTY; + goto odot; + + case OINDEX: + if(n->left->type->etype == TSTRING) + return 0; + cleani += 2; + reg = &clean[cleani-1]; + reg1 = &clean[cleani-2]; + reg->op = OEMPTY; + reg1->op = OEMPTY; + goto oindex; + } + return 0; + +lit: + switch(as) { + default: + return 0; + case AADD: case ASUB: case AAND: case AORR: case AEOR: + case AMOVB: case AMOVBU: case AMOVH: case AMOVHU: + case AMOVW: + break; + } + + cleani += 2; + reg = &clean[cleani-1]; + reg1 = &clean[cleani-2]; + reg->op = OEMPTY; + reg1->op = OEMPTY; + naddr(n, a, 1); + goto yes; + +odot: + o = dotoffset(n, oary, &nn); + if(nn == N) + goto no; + + if(nn->addable && o == 1 && oary[0] >= 0) { + // directly addressable set of DOTs + n1 = *nn; + n1.type = n->type; + n1.xoffset += oary[0]; + naddr(&n1, a, 1); + goto yes; + } + + regalloc(reg, types[tptr], N); + n1 = *reg; + n1.op = OINDREG; + if(oary[0] >= 0) { + agen(nn, reg); + n1.xoffset = oary[0]; + } else { + cgen(nn, reg); + n1.xoffset = -(oary[0]+1); + } + + for(i=1; i= 0) + fatal("cant happen"); + gins(AMOVW, &n1, reg); + n1.xoffset = -(oary[i]+1); + } + + a->type = D_NONE; + a->name = D_NONE; + n1.type = n->type; + naddr(&n1, a, 1); + goto yes; + +oindex: + l = n->left; + r = n->right; + if(l->ullman >= UINF && r->ullman >= UINF) + goto no; + + // set o to type of array + o = 0; + if(isptr[l->type->etype]) { + o += OPtrto; + if(l->type->type->etype != TARRAY) + fatal("not ptr ary"); + if(l->type->type->bound < 0) + o += ODynam; + } else { + if(l->type->etype != TARRAY) + fatal("not ary"); + if(l->type->bound < 0) + o += ODynam; + } + + *w = n->type->width; + if(isconst(r, CTINT)) + goto oindex_const; + + switch(*w) { + default: + goto no; + case 1: + case 2: + case 4: + case 8: + break; + } + + // load the array (reg) + if(l->ullman > r->ullman) { + regalloc(reg, types[tptr], N); + if(o & OPtrto) + cgen(l, reg); + else + agen(l, reg); + } + + // load the index (reg1) + t = types[TUINT32]; + if(issigned[r->type->etype]) + t = types[TINT32]; + regalloc(reg1, t, N); + regalloc(&n3, types[TINT32], reg1); + p2 = cgenindex(r, &n3); + gmove(&n3, reg1); + regfree(&n3); + + // load the array (reg) + if(l->ullman <= r->ullman) { + regalloc(reg, types[tptr], N); + if(o & OPtrto) + cgen(l, reg); + else + agen(l, reg); + } + + // check bounds + if(!debug['B']) { + if(o & ODynam) { + n2 = *reg; + n2.op = OINDREG; + n2.type = types[tptr]; + n2.xoffset = Array_nel; + } else { + if(l->type->width >= unmappedzero && l->op == OIND) { + // cannot rely on page protections to + // catch array ptr == 0, so dereference. + n2 = *reg; + n2.op = OINDREG; + n2.type = types[TUINTPTR]; + n2.xoffset = 0; + regalloc(&n3, n2.type, N); + gins(AMOVW, &n2, &n3); + regfree(&n3); + } + nodconst(&n2, types[TUINT32], l->type->bound); + if(o & OPtrto) + nodconst(&n2, types[TUINT32], l->type->type->bound); + } + regalloc(&n3, n2.type, N); + cgen(&n2, &n3); + gcmp(optoas(OCMP, types[TUINT32]), reg1, &n3); + regfree(&n3); + p1 = gbranch(optoas(OLT, types[TUINT32]), T); + if(p2) + patch(p2, pc); + ginscall(panicindex, 0); + patch(p1, pc); + } + + if(o & ODynam) { + n2 = *reg; + n2.op = OINDREG; + n2.type = types[tptr]; + n2.xoffset = Array_array; + gmove(&n2, reg); + } + + switch(*w) { + case 1: + gins(AADD, reg1, reg); + break; + case 2: + gshift(AADD, reg1, SHIFT_LL, 1, reg); + break; + case 4: + gshift(AADD, reg1, SHIFT_LL, 2, reg); + break; + case 8: + gshift(AADD, reg1, SHIFT_LL, 3, reg); + break; + } + + naddr(reg1, a, 1); + a->type = D_OREG; + a->reg = reg->val.u.reg; + a->offset = 0; + goto yes; + +oindex_const: + // index is constant + // can check statically and + // can multiply by width statically + + regalloc(reg, types[tptr], N); + if(o & OPtrto) + cgen(l, reg); + else + agen(l, reg); + + v = mpgetfix(r->val.u.xval); + if(o & ODynam) { + + if(!debug['B'] && !n->etype) { + n1 = *reg; + n1.op = OINDREG; + n1.type = types[tptr]; + n1.xoffset = Array_nel; + nodconst(&n2, types[TUINT32], v); + regalloc(&n3, types[TUINT32], N); + cgen(&n2, &n3); + regalloc(&n4, n1.type, N); + cgen(&n1, &n4); + gcmp(optoas(OCMP, types[TUINT32]), &n4, &n3); + regfree(&n4); + regfree(&n3); + p1 = gbranch(optoas(OGT, types[TUINT32]), T); + ginscall(panicindex, 0); + patch(p1, pc); + } + + n1 = *reg; + n1.op = OINDREG; + n1.type = types[tptr]; + n1.xoffset = Array_array; + gmove(&n1, reg); + } + + n2 = *reg; + n2.op = OINDREG; + n2.xoffset = v * (*w); + a->type = D_NONE; + a->name = D_NONE; + naddr(&n2, a, 1); + goto yes; + +yes: + return 1; + +no: + sudoclean(); + return 0; +} diff --git a/src/cmd/5g/list.c b/src/cmd/5g/list.c new file mode 100644 index 000000000..0c6dbbf71 --- /dev/null +++ b/src/cmd/5g/list.c @@ -0,0 +1,331 @@ +// Derived from Inferno utils/5c/list.c +// http://code.google.com/p/inferno-os/source/browse/utils/5c/list.c +// +// Copyright © 1994-1999 Lucent Technologies Inc. All rights reserved. +// Portions Copyright © 1995-1997 C H Forsyth (forsyth@terzarima.net) +// Portions Copyright © 1997-1999 Vita Nuova Limited +// Portions Copyright © 2000-2007 Vita Nuova Holdings Limited (www.vitanuova.com) +// Portions Copyright © 2004,2006 Bruce Ellis +// Portions Copyright © 2005-2007 C H Forsyth (forsyth@terzarima.net) +// Revisions Copyright © 2000-2007 Lucent Technologies Inc. and others +// Portions Copyright © 2009 The Go Authors. All rights reserved. +// +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to deal +// in the Software without restriction, including without limitation the rights +// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +// copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +// THE SOFTWARE. + +#include "gg.h" + +// TODO(kaib): make 5g/list.c congruent with 5l/list.c + +static int sconsize; +void +listinit(void) +{ + + fmtinstall('A', Aconv); // as + fmtinstall('C', Cconv); // conditional execution bit + fmtinstall('P', Pconv); // Prog* + fmtinstall('D', Dconv); // Addr* + fmtinstall('Y', Yconv); // sconst + fmtinstall('R', Rconv); // register + fmtinstall('M', Mconv); // names +} + +int +Pconv(Fmt *fp) +{ + char str[STRINGSZ], str1[STRINGSZ]; + Prog *p; + + p = va_arg(fp->args, Prog*); + sconsize = 8; + switch(p->as) { + default: + snprint(str1, sizeof(str1), "%A%C", p->as, p->scond); + if(p->reg == NREG) + snprint(str, sizeof(str), "%.4d (%L) %-7s %D,%D", + p->loc, p->lineno, str1, &p->from, &p->to); + else + if (p->from.type != D_FREG) { + snprint(str, sizeof(str), "%.4d (%L) %-7s %D,R%d,%D", + p->loc, p->lineno, str1, &p->from, p->reg, &p->to); + } else + snprint(str, sizeof(str), "%.4d (%L) %-7A%C %D,F%d,%D", + p->loc, p->lineno, p->as, p->scond, &p->from, p->reg, &p->to); + break; + + case ADATA: + snprint(str, sizeof(str), "%.4d (%L) %-7A %D/%d,%D", + p->loc, p->lineno, p->as, &p->from, p->reg, &p->to); + break; + } + return fmtstrcpy(fp, str); +} + +int +Dconv(Fmt *fp) +{ + char str[STRINGSZ]; + char *op; + Addr *a; + int i; + int32 v; + + a = va_arg(fp->args, Addr*); + if(a == A) { + sprint(str, ""); + goto conv; + } + i = a->type; + switch(i) { + + default: + sprint(str, "GOK-type(%d)", a->type); + break; + + case D_NONE: + str[0] = 0; + if(a->name != D_NONE || a->reg != NREG || a->sym != S) + sprint(str, "%M(R%d)(NONE)", a, a->reg); + break; + + case D_CONST: + if(a->reg != NREG) + sprint(str, "$%M(R%d)", a, a->reg); + else + sprint(str, "$%M", a); + break; + + case D_CONST2: + sprint(str, "$%d-%d", a->offset, a->offset2); + break; + + case D_SHIFT: + v = a->offset; + op = "<<>>->@>" + (((v>>5) & 3) << 1); + if(v & (1<<4)) + sprint(str, "R%d%c%cR%d", v&15, op[0], op[1], (v>>8)&15); + else + sprint(str, "R%d%c%c%d", v&15, op[0], op[1], (v>>7)&31); + if(a->reg != NREG) + sprint(str+strlen(str), "(R%d)", a->reg); + break; + + case D_OCONST: + sprint(str, "$*$%M", a); + if(a->reg != NREG) + sprint(str, "%M(R%d)(CONST)", a, a->reg); + break; + + case D_OREG: + if(a->reg != NREG) + sprint(str, "%M(R%d)", a, a->reg); + else + sprint(str, "%M", a); + break; + + case D_REG: + sprint(str, "R%d", a->reg); + if(a->name != D_NONE || a->sym != S) + sprint(str, "%M(R%d)(REG)", a, a->reg); + break; + + case D_REGREG: + sprint(str, "(R%d,R%d)", a->reg, (int)a->offset); + if(a->name != D_NONE || a->sym != S) + sprint(str, "%M(R%d)(REG)", a, a->reg); + break; + + case D_FREG: + sprint(str, "F%d", a->reg); + if(a->name != D_NONE || a->sym != S) + sprint(str, "%M(R%d)(REG)", a, a->reg); + break; + + case D_BRANCH: + if(a->branch == P || a->branch->loc == 0) { + if(a->sym != S) + sprint(str, "%s+%d(APC)", a->sym->name, a->offset); + else + sprint(str, "%d(APC)", a->offset); + } else + if(a->sym != S) + sprint(str, "%s+%d(APC)", a->sym->name, a->branch->loc); + else + sprint(str, "%d(APC)", a->branch->loc); + break; + + case D_FCONST: + snprint(str, sizeof(str), "$(%.17e)", a->dval); + break; + + case D_SCONST: + snprint(str, sizeof(str), "$\"%Y\"", a->sval); + break; + + // TODO(kaib): Add back +// case D_ADDR: +// a->type = a->index; +// a->index = D_NONE; +// snprint(str, sizeof(str), "$%D", a); +// a->index = a->type; +// a->type = D_ADDR; +// goto conv; + } +conv: + return fmtstrcpy(fp, str); +} + +int +Aconv(Fmt *fp) +{ + int i; + + i = va_arg(fp->args, int); + return fmtstrcpy(fp, anames[i]); +} + +char* strcond[16] = +{ + ".EQ", + ".NE", + ".HS", + ".LO", + ".MI", + ".PL", + ".VS", + ".VC", + ".HI", + ".LS", + ".GE", + ".LT", + ".GT", + ".LE", + "", + ".NV" +}; + +int +Cconv(Fmt *fp) +{ + char s[STRINGSZ]; + int c; + + c = va_arg(fp->args, int); + strcpy(s, strcond[c & C_SCOND]); + if(c & C_SBIT) + strcat(s, ".S"); + if(c & C_PBIT) + strcat(s, ".P"); + if(c & C_WBIT) + strcat(s, ".W"); + if(c & C_UBIT) /* ambiguous with FBIT */ + strcat(s, ".U"); + return fmtstrcpy(fp, s); +} + +int +Yconv(Fmt *fp) +{ + int i, c; + char str[STRINGSZ], *p, *a; + + a = va_arg(fp->args, char*); + p = str; + for(i=0; i= 'a' && c <= 'z') || + (c >= 'A' && c <= 'Z') || + (c >= '0' && c <= '9')) { + *p++ = c; + continue; + } + *p++ = '\\'; + switch(c) { + default: + if(c < 040 || c >= 0177) + break; /* not portable */ + p[-1] = c; + continue; + case 0: + *p++ = 'z'; + continue; + case '\\': + case '"': + *p++ = c; + continue; + case '\n': + *p++ = 'n'; + continue; + case '\t': + *p++ = 't'; + continue; + } + *p++ = (c>>6) + '0'; + *p++ = ((c>>3) & 7) + '0'; + *p++ = (c & 7) + '0'; + } + *p = 0; + return fmtstrcpy(fp, str); +} + +int +Rconv(Fmt *fp) +{ + int r; + char str[STRINGSZ]; + + r = va_arg(fp->args, int); + snprint(str, sizeof(str), "R%d", r); + return fmtstrcpy(fp, str); +} + +int +Mconv(Fmt *fp) +{ + char str[STRINGSZ]; + Addr *a; + + a = va_arg(fp->args, Addr*); + switch(a->name) { + default: + snprint(str, sizeof(str), "GOK-name(%d)", a->name); + break; + + case D_NONE: + snprint(str, sizeof(str), "%d", a->offset); + break; + + case D_EXTERN: + snprint(str, sizeof(str), "%S+%d(SB)", a->sym, a->offset); + break; + + case D_STATIC: + snprint(str, sizeof(str), "%S<>+%d(SB)", a->sym, a->offset); + break; + + case D_AUTO: + snprint(str, sizeof(str), "%S+%d(SP)", a->sym, a->offset); + break; + + case D_PARAM: + snprint(str, sizeof(str), "%S+%d(FP)", a->sym, a->offset); + break; + } + return fmtstrcpy(fp, str); +} diff --git a/src/cmd/5g/opt.h b/src/cmd/5g/opt.h new file mode 100644 index 000000000..7a0070fc9 --- /dev/null +++ b/src/cmd/5g/opt.h @@ -0,0 +1,165 @@ +// Inferno utils/5c/gc.h +// http://code.google.com/p/inferno-os/source/browse/utils/5c/gc.h +// +// Copyright © 1994-1999 Lucent Technologies Inc. All rights reserved. +// Portions Copyright © 1995-1997 C H Forsyth (forsyth@terzarima.net) +// Portions Copyright © 1997-1999 Vita Nuova Limited +// Portions Copyright © 2000-2007 Vita Nuova Holdings Limited (www.vitanuova.com) +// Portions Copyright © 2004,2006 Bruce Ellis +// Portions Copyright © 2005-2007 C H Forsyth (forsyth@terzarima.net) +// Revisions Copyright © 2000-2007 Lucent Technologies Inc. and others +// Portions Copyright © 2009 The Go Authors. All rights reserved. +// +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to deal +// in the Software without restriction, including without limitation the rights +// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +// copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +// THE SOFTWARE. + +#define Z N +#define Adr Addr + +#define D_HI D_NONE +#define D_LO D_NONE + +#define isregtype(t) ((t)>= D_AX && (t)<=D_R15) + +#define BLOAD(r) band(bnot(r->refbehind), r->refahead) +#define BSTORE(r) band(bnot(r->calbehind), r->calahead) +#define LOAD(r) (~r->refbehind.b[z] & r->refahead.b[z]) +#define STORE(r) (~r->calbehind.b[z] & r->calahead.b[z]) + +#define CLOAD 5 +#define CREF 5 +#define CINF 1000 +#define LOOP 3 + +typedef struct Reg Reg; +typedef struct Rgn Rgn; + +struct Reg +{ + + Bits set; + Bits use1; + Bits use2; + + Bits refbehind; + Bits refahead; + Bits calbehind; + Bits calahead; + Bits regdiff; + Bits act; + + int32 regu; // register used bitmap + int32 rpo; // reverse post ordering + int32 active; + + uint16 loop; // x5 for every loop + uchar refset; // diagnostic generated + + Reg* p1; + Reg* p2; + Reg* p2link; + Reg* s1; + Reg* s2; + Reg* link; + Prog* prog; +}; +#define R ((Reg*)0) + +#define NRGN 600 +struct Rgn +{ + Reg* enter; + short cost; + short varno; + short regno; +}; + +EXTERN int32 exregoffset; // not set +EXTERN int32 exfregoffset; // not set +EXTERN Reg* firstr; +EXTERN Reg* lastr; +EXTERN Reg zreg; +EXTERN Reg* freer; +EXTERN Reg** rpo2r; +EXTERN Rgn region[NRGN]; +EXTERN Rgn* rgp; +EXTERN int nregion; +EXTERN int nvar; +EXTERN int32 regbits; +EXTERN int32 exregbits; +EXTERN Bits externs; +EXTERN Bits params; +EXTERN Bits consts; +EXTERN Bits addrs; +EXTERN Bits ovar; +EXTERN int change; +EXTERN int32 maxnr; +EXTERN int32* idom; + +EXTERN struct +{ + int32 ncvtreg; + int32 nspill; + int32 nreload; + int32 ndelmov; + int32 nvar; + int32 naddr; +} ostats; + +/* + * reg.c + */ +Reg* rega(void); +int rcmp(const void*, const void*); +void regopt(Prog*); +void addmove(Reg*, int, int, int); +Bits mkvar(Reg *r, Adr *a); +void prop(Reg*, Bits, Bits); +void loopit(Reg*, int32); +void synch(Reg*, Bits); +uint32 allreg(uint32, Rgn*); +void paint1(Reg*, int); +uint32 paint2(Reg*, int); +void paint3(Reg*, int, int32, int); +void addreg(Adr*, int); +void dumpit(char *str, Reg *r0); +int noreturn(Prog *p); + +/* + * peep.c + */ +void peep(void); +void excise(Reg*); +Reg* uniqp(Reg*); +Reg* uniqs(Reg*); +int regtyp(Adr*); +int anyvar(Adr*); +int subprop(Reg*); +int copyprop(Reg*); +int copy1(Adr*, Adr*, Reg*, int); +int copyu(Prog*, Adr*, Adr*); + +int copyas(Adr*, Adr*); +int copyau(Adr*, Adr*); +int copysub(Adr*, Adr*, Adr*, int); +int copysub1(Prog*, Adr*, Adr*, int); + +int32 RtoB(int); +int32 FtoB(int); +int BtoR(int32); +int BtoF(int32); diff --git a/src/cmd/5g/peep.c b/src/cmd/5g/peep.c new file mode 100644 index 000000000..6cc93db12 --- /dev/null +++ b/src/cmd/5g/peep.c @@ -0,0 +1,1521 @@ +// Inferno utils/5c/peep.c +// http://code.google.com/p/inferno-os/source/browse/utils/5g/peep.c +// +// Copyright © 1994-1999 Lucent Technologies Inc. All rights reserved. +// Portions Copyright © 1995-1997 C H Forsyth (forsyth@terzarima.net) +// Portions Copyright © 1997-1999 Vita Nuova Limited +// Portions Copyright © 2000-2007 Vita Nuova Holdings Limited (www.vitanuova.com) +// Portions Copyright © 2004,2006 Bruce Ellis +// Portions Copyright © 2005-2007 C H Forsyth (forsyth@terzarima.net) +// Revisions Copyright © 2000-2007 Lucent Technologies Inc. and others +// Portions Copyright © 2009 The Go Authors. All rights reserved. +// +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to deal +// in the Software without restriction, including without limitation the rights +// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +// copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +// THE SOFTWARE. + + +#include "gg.h" +#include "opt.h" + +int xtramodes(Reg*, Adr*); +int shiftprop(Reg *r); +void constprop(Adr *c1, Adr *v1, Reg *r); +void predicate(void); +int copyau1(Prog *p, Adr *v); +int isdconst(Addr *a); + +void +peep(void) +{ + Reg *r, *r1, *r2; + Prog *p, *p1; + int t; +/* + * complete R structure + */ + for(r=firstr; r!=R; r=r1) { + r1 = r->link; + if(r1 == R) + break; + p = r->prog->link; + while(p != r1->prog) + switch(p->as) { + default: + r2 = rega(); + r->link = r2; + r2->link = r1; + + r2->prog = p; + r2->p1 = r; + r->s1 = r2; + r2->s1 = r1; + r1->p1 = r2; + + r = r2; + + case ADATA: + case AGLOBL: + case ANAME: + case ASIGNAME: + p = p->link; + } + } +//dumpit("begin", firstr); + +loop1: + + t = 0; + for(r=firstr; r!=R; r=r->link) { + p = r->prog; + switch(p->as) { + case ASLL: + case ASRL: + case ASRA: + /* + * elide shift into D_SHIFT operand of subsequent instruction + */ +// if(shiftprop(r)) { +// excise(r); +// t++; +// break; +// } + break; + + case AMOVW: + case AMOVF: + case AMOVD: + if(regtyp(&p->from)) + if(p->from.type == p->to.type) + if(p->scond == C_SCOND_NONE) { + if(copyprop(r)) { + excise(r); + t++; + break; + } + if(subprop(r) && copyprop(r)) { + excise(r); + t++; + break; + } + } + break; + + if(p->scond == C_SCOND_NONE) + if(regtyp(&p->to)) + if(isdconst(&p->from)) { + constprop(&p->from, &p->to, r->s1); + } + break; + } + } + if(t) + goto loop1; + +return; + + for(r=firstr; r!=R; r=r->link) { + p = r->prog; + switch(p->as) { +// case AEOR: +// /* +// * EOR -1,x,y => MVN x,y +// */ +// if(isdconst(&p->from) && p->from.offset == -1) { +// p->as = AMVN; +// p->from.type = D_REG; +// if(p->reg != NREG) +// p->from.reg = p->reg; +// else +// p->from.reg = p->to.reg; +// p->reg = NREG; +// } +// break; + + case AMOVH: + case AMOVHU: + case AMOVB: + case AMOVBU: + /* + * look for MOVB x,R; MOVB R,R + */ + if(p->to.type != D_REG) + break; + if(r1 == R) + break; + p1 = r1->prog; + if(p1->as != p->as) + break; + if(p1->from.type != D_REG || p1->from.reg != p->to.reg) + break; + if(p1->to.type != D_REG || p1->to.reg != p->to.reg) + break; + excise(r1); + break; + } + r1 = r->link; + } + +// for(r=firstr; r!=R; r=r->link) { +// p = r->prog; +// switch(p->as) { +// case AMOVW: +// case AMOVB: +// case AMOVBU: +// if(p->from.type == D_OREG && p->from.offset == 0) +// xtramodes(r, &p->from); +// else +// if(p->to.type == D_OREG && p->to.offset == 0) +// xtramodes(r, &p->to); +// else +// continue; +// break; +// case ACMP: +// /* +// * elide CMP $0,x if calculation of x can set condition codes +// */ +// if(isdconst(&p->from) || p->from.offset != 0) +// continue; +// r2 = r->s1; +// if(r2 == R) +// continue; +// t = r2->prog->as; +// switch(t) { +// default: +// continue; +// case ABEQ: +// case ABNE: +// case ABMI: +// case ABPL: +// break; +// case ABGE: +// t = ABPL; +// break; +// case ABLT: +// t = ABMI; +// break; +// case ABHI: +// t = ABNE; +// break; +// case ABLS: +// t = ABEQ; +// break; +// } +// r1 = r; +// do +// r1 = uniqp(r1); +// while (r1 != R && r1->prog->as == ANOP); +// if(r1 == R) +// continue; +// p1 = r1->prog; +// if(p1->to.type != D_REG) +// continue; +// if(p1->to.reg != p->reg) +// if(!(p1->as == AMOVW && p1->from.type == D_REG && p1->from.reg == p->reg)) +// continue; +// +// switch(p1->as) { +// default: +// continue; +// case AMOVW: +// if(p1->from.type != D_REG) +// continue; +// case AAND: +// case AEOR: +// case AORR: +// case ABIC: +// case AMVN: +// case ASUB: +// case ARSB: +// case AADD: +// case AADC: +// case ASBC: +// case ARSC: +// break; +// } +// p1->scond |= C_SBIT; +// r2->prog->as = t; +// excise(r); +// continue; +// } +// } + + predicate(); +} + +Reg* +uniqp(Reg *r) +{ + Reg *r1; + + r1 = r->p1; + if(r1 == R) { + r1 = r->p2; + if(r1 == R || r1->p2link != R) + return R; + } else + if(r->p2 != R) + return R; + return r1; +} + +Reg* +uniqs(Reg *r) +{ + Reg *r1; + + r1 = r->s1; + if(r1 == R) { + r1 = r->s2; + if(r1 == R) + return R; + } else + if(r->s2 != R) + return R; + return r1; +} + +int +regtyp(Adr *a) +{ + + if(a->type == D_REG) + return 1; + if(a->type == D_FREG) + return 1; + return 0; +} + +/* + * the idea is to substitute + * one register for another + * from one MOV to another + * MOV a, R0 + * ADD b, R0 / no use of R1 + * MOV R0, R1 + * would be converted to + * MOV a, R1 + * ADD b, R1 + * MOV R1, R0 + * hopefully, then the former or latter MOV + * will be eliminated by copy propagation. + */ +int +subprop(Reg *r0) +{ + Prog *p; + Adr *v1, *v2; + Reg *r; + int t; + + p = r0->prog; + v1 = &p->from; + if(!regtyp(v1)) + return 0; + v2 = &p->to; + if(!regtyp(v2)) + return 0; + for(r=uniqp(r0); r!=R; r=uniqp(r)) { + if(uniqs(r) == R) + break; + p = r->prog; + switch(p->as) { + case ABL: + return 0; + + case AMULLU: + case AMULA: + case AMVN: + return 0; + + case ACMN: + case AADD: + case ASUB: + case ASBC: + case ARSB: + case ASLL: + case ASRL: + case ASRA: + case AORR: + case AAND: + case AEOR: + case AMUL: + case AMULU: + case ADIV: + case ADIVU: + case AMOD: + case AMODU: + + case AADDD: + case AADDF: + case ASUBD: + case ASUBF: + case AMULD: + case AMULF: + case ADIVD: + case ADIVF: + if(p->to.type == v1->type) + if(p->to.reg == v1->reg) + if(p->scond == C_SCOND_NONE) { + if(p->reg == NREG) + p->reg = p->to.reg; + goto gotit; + } + break; + + case AMOVF: + case AMOVD: + case AMOVW: + if(p->to.type == v1->type) + if(p->to.reg == v1->reg) + if(p->scond == C_SCOND_NONE) + goto gotit; + break; + + case AMOVM: + t = 1<reg; + if((p->from.type == D_CONST && (p->from.offset&t)) || + (p->to.type == D_CONST && (p->to.offset&t))) + return 0; + break; + } + if(copyau(&p->from, v2) || + copyau1(p, v2) || + copyau(&p->to, v2)) + break; + if(copysub(&p->from, v1, v2, 0) || + copysub1(p, v1, v2, 0) || + copysub(&p->to, v1, v2, 0)) + break; + } + return 0; + +gotit: + copysub(&p->to, v1, v2, 1); + if(debug['P']) { + print("gotit: %D->%D\n%P", v1, v2, r->prog); + if(p->from.type == v2->type) + print(" excise"); + print("\n"); + } + for(r=uniqs(r); r!=r0; r=uniqs(r)) { + p = r->prog; + copysub(&p->from, v1, v2, 1); + copysub1(p, v1, v2, 1); + copysub(&p->to, v1, v2, 1); + if(debug['P']) + print("%P\n", r->prog); + } + t = v1->reg; + v1->reg = v2->reg; + v2->reg = t; + if(debug['P']) + print("%P last\n", r->prog); + return 1; +} + +/* + * The idea is to remove redundant copies. + * v1->v2 F=0 + * (use v2 s/v2/v1/)* + * set v1 F=1 + * use v2 return fail + * ----------------- + * v1->v2 F=0 + * (use v2 s/v2/v1/)* + * set v1 F=1 + * set v2 return success + */ +int +copyprop(Reg *r0) +{ + Prog *p; + Adr *v1, *v2; + Reg *r; + + p = r0->prog; + v1 = &p->from; + v2 = &p->to; + if(copyas(v1, v2)) + return 1; + for(r=firstr; r!=R; r=r->link) + r->active = 0; + return copy1(v1, v2, r0->s1, 0); +} + +int +copy1(Adr *v1, Adr *v2, Reg *r, int f) +{ + int t; + Prog *p; + + if(r->active) { + if(debug['P']) + print("act set; return 1\n"); + return 1; + } + r->active = 1; + if(debug['P']) + print("copy %D->%D f=%d\n", v1, v2, f); + for(; r != R; r = r->s1) { + p = r->prog; + if(debug['P']) + print("%P", p); + if(!f && uniqp(r) == R) { + f = 1; + if(debug['P']) + print("; merge; f=%d", f); + } + t = copyu(p, v2, A); + switch(t) { + case 2: /* rar, cant split */ + if(debug['P']) + print("; %Drar; return 0\n", v2); + return 0; + + case 3: /* set */ + if(debug['P']) + print("; %Dset; return 1\n", v2); + return 1; + + case 1: /* used, substitute */ + case 4: /* use and set */ + if(f) { + if(!debug['P']) + return 0; + if(t == 4) + print("; %Dused+set and f=%d; return 0\n", v2, f); + else + print("; %Dused and f=%d; return 0\n", v2, f); + return 0; + } + if(copyu(p, v2, v1)) { + if(debug['P']) + print("; sub fail; return 0\n"); + return 0; + } + if(debug['P']) + print("; sub%D/%D", v2, v1); + if(t == 4) { + if(debug['P']) + print("; %Dused+set; return 1\n", v2); + return 1; + } + break; + } + if(!f) { + t = copyu(p, v1, A); + if(!f && (t == 2 || t == 3 || t == 4)) { + f = 1; + if(debug['P']) + print("; %Dset and !f; f=%d", v1, f); + } + } + if(debug['P']) + print("\n"); + if(r->s2) + if(!copy1(v1, v2, r->s2, f)) + return 0; + } + return 1; +} + +/* + * The idea is to remove redundant constants. + * $c1->v1 + * ($c1->v2 s/$c1/v1)* + * set v1 return + * The v1->v2 should be eliminated by copy propagation. + */ +void +constprop(Adr *c1, Adr *v1, Reg *r) +{ + Prog *p; + + if(debug['P']) + print("constprop %D->%D\n", c1, v1); + for(; r != R; r = r->s1) { + p = r->prog; + if(debug['P']) + print("%P", p); + if(uniqp(r) == R) { + if(debug['P']) + print("; merge; return\n"); + return; + } + if(p->as == AMOVW && copyas(&p->from, c1)) { + if(debug['P']) + print("; sub%D/%D", &p->from, v1); + p->from = *v1; + } else if(copyu(p, v1, A) > 1) { + if(debug['P']) + print("; %Dset; return\n", v1); + return; + } + if(debug['P']) + print("\n"); + if(r->s2) + constprop(c1, v1, r->s2); + } +} + +/* + * ASLL x,y,w + * .. (not use w, not set x y w) + * AXXX w,a,b (a != w) + * .. (not use w) + * (set w) + * ----------- changed to + * .. + * AXXX (x<prog; + if(p->to.type != D_REG) + FAIL("BOTCH: result not reg"); + n = p->to.reg; + a = zprog.from; + if(p->reg != NREG && p->reg != p->to.reg) { + a.type = D_REG; + a.reg = p->reg; + } + if(debug['P']) + print("shiftprop\n%P", p); + r1 = r; + for(;;) { + /* find first use of shift result; abort if shift operands or result are changed */ + r1 = uniqs(r1); + if(r1 == R) + FAIL("branch"); + if(uniqp(r1) == R) + FAIL("merge"); + p1 = r1->prog; + if(debug['P']) + print("\n%P", p1); + switch(copyu(p1, &p->to, A)) { + case 0: /* not used or set */ + if((p->from.type == D_REG && copyu(p1, &p->from, A) > 1) || + (a.type == D_REG && copyu(p1, &a, A) > 1)) + FAIL("args modified"); + continue; + case 3: /* set, not used */ + FAIL("BOTCH: noref"); + } + break; + } + /* check whether substitution can be done */ + switch(p1->as) { + default: + FAIL("non-dpi"); + case AAND: + case AEOR: + case AADD: + case AADC: + case AORR: + case ASUB: + case ASBC: + case ARSB: + case ARSC: + if(p1->reg == n || (p1->reg == NREG && p1->to.type == D_REG && p1->to.reg == n)) { + if(p1->from.type != D_REG) + FAIL("can't swap"); + p1->reg = p1->from.reg; + p1->from.reg = n; + switch(p1->as) { + case ASUB: + p1->as = ARSB; + break; + case ARSB: + p1->as = ASUB; + break; + case ASBC: + p1->as = ARSC; + break; + case ARSC: + p1->as = ASBC; + break; + } + if(debug['P']) + print("\t=>%P", p1); + } + case ABIC: + case ATST: + case ACMP: + case ACMN: + if(p1->reg == n) + FAIL("can't swap"); + if(p1->reg == NREG && p1->to.reg == n) + FAIL("shift result used twice"); +// case AMVN: + if(p1->from.type == D_SHIFT) + FAIL("shift result used in shift"); + if(p1->from.type != D_REG || p1->from.reg != n) + FAIL("BOTCH: where is it used?"); + break; + } + /* check whether shift result is used subsequently */ + p2 = p1; + if(p1->to.reg != n) + for (;;) { + r1 = uniqs(r1); + if(r1 == R) + FAIL("inconclusive"); + p1 = r1->prog; + if(debug['P']) + print("\n%P", p1); + switch(copyu(p1, &p->to, A)) { + case 0: /* not used or set */ + continue; + case 3: /* set, not used */ + break; + default:/* used */ + FAIL("reused"); + } + break; + } + + /* make the substitution */ + p2->from.type = D_SHIFT; + p2->from.reg = NREG; + o = p->reg; + if(o == NREG) + o = p->to.reg; + + switch(p->from.type){ + case D_CONST: + o |= (p->from.offset&0x1f)<<7; + break; + case D_REG: + o |= (1<<4) | (p->from.reg<<8); + break; + } + switch(p->as){ + case ASLL: + o |= 0<<5; + break; + case ASRL: + o |= 1<<5; + break; + case ASRA: + o |= 2<<5; + break; + } + p2->from.offset = o; + if(debug['P']) + print("\t=>%P\tSUCCEED\n", p2); + return 1; +} + +Reg* +findpre(Reg *r, Adr *v) +{ + Reg *r1; + + for(r1=uniqp(r); r1!=R; r=r1,r1=uniqp(r)) { + if(uniqs(r1) != r) + return R; + switch(copyu(r1->prog, v, A)) { + case 1: /* used */ + case 2: /* read-alter-rewrite */ + return R; + case 3: /* set */ + case 4: /* set and used */ + return r1; + } + } + return R; +} + +Reg* +findinc(Reg *r, Reg *r2, Adr *v) +{ + Reg *r1; + Prog *p; + + + for(r1=uniqs(r); r1!=R && r1!=r2; r=r1,r1=uniqs(r)) { + if(uniqp(r1) != r) + return R; + switch(copyu(r1->prog, v, A)) { + case 0: /* not touched */ + continue; + case 4: /* set and used */ + p = r1->prog; + if(p->as == AADD) + if(isdconst(&p->from)) + if(p->from.offset > -4096 && p->from.offset < 4096) + return r1; + default: + return R; + } + } + return R; +} + +int +nochange(Reg *r, Reg *r2, Prog *p) +{ + Adr a[3]; + int i, n; + + if(r == r2) + return 1; + n = 0; + if(p->reg != NREG && p->reg != p->to.reg) { + a[n].type = D_REG; + a[n++].reg = p->reg; + } + switch(p->from.type) { + case D_SHIFT: + a[n].type = D_REG; + a[n++].reg = p->from.offset&0xf; + case D_REG: + a[n].type = D_REG; + a[n++].reg = p->from.reg; + } + if(n == 0) + return 1; + for(; r!=R && r!=r2; r=uniqs(r)) { + p = r->prog; + for(i=0; i 1) + return 0; + } + return 1; +} + +int +findu1(Reg *r, Adr *v) +{ + for(; r != R; r = r->s1) { + if(r->active) + return 0; + r->active = 1; + switch(copyu(r->prog, v, A)) { + case 1: /* used */ + case 2: /* read-alter-rewrite */ + case 4: /* set and used */ + return 1; + case 3: /* set */ + return 0; + } + if(r->s2) + if (findu1(r->s2, v)) + return 1; + } + return 0; +} + +int +finduse(Reg *r, Adr *v) +{ + Reg *r1; + + for(r1=firstr; r1!=R; r1=r1->link) + r1->active = 0; + return findu1(r, v); +} + +int +xtramodes(Reg *r, Adr *a) +{ + Reg *r1, *r2, *r3; + Prog *p, *p1; + Adr v; + + p = r->prog; + if(debug['h'] && p->as == AMOVB && p->from.type == D_OREG) /* byte load */ + return 0; + v = *a; + v.type = D_REG; + r1 = findpre(r, &v); + if(r1 != R) { + p1 = r1->prog; + if(p1->to.type == D_REG && p1->to.reg == v.reg) + switch(p1->as) { + case AADD: + if(p1->from.type == D_REG || + (p1->from.type == D_SHIFT && (p1->from.offset&(1<<4)) == 0 && + (p->as != AMOVB || (a == &p->from && (p1->from.offset&~0xf) == 0))) || + (p1->from.type == D_CONST && + p1->from.offset > -4096 && p1->from.offset < 4096)) + if(nochange(uniqs(r1), r, p1)) { + if(a != &p->from || v.reg != p->to.reg) + if (finduse(r->s1, &v)) { + if(p1->reg == NREG || p1->reg == v.reg) + /* pre-indexing */ + p->scond |= C_WBIT; + else return 0; + } + switch (p1->from.type) { + case D_REG: + /* register offset */ + a->type = D_SHIFT; + a->offset = p1->from.reg; + break; + case D_SHIFT: + /* scaled register offset */ + a->type = D_SHIFT; + case D_CONST: + /* immediate offset */ + a->offset = p1->from.offset; + break; + } + if(p1->reg != NREG) + a->reg = p1->reg; + excise(r1); + return 1; + } + break; + case AMOVW: + if(p1->from.type == D_REG) + if((r2 = findinc(r1, r, &p1->from)) != R) { + for(r3=uniqs(r2); r3->prog->as==ANOP; r3=uniqs(r3)) + ; + if(r3 == r) { + /* post-indexing */ + p1 = r2->prog; + a->reg = p1->to.reg; + a->offset = p1->from.offset; + p->scond |= C_PBIT; + if(!finduse(r, &r1->prog->to)) + excise(r1); + excise(r2); + return 1; + } + } + break; + } + } + if(a != &p->from || a->reg != p->to.reg) + if((r1 = findinc(r, R, &v)) != R) { + /* post-indexing */ + p1 = r1->prog; + a->offset = p1->from.offset; + p->scond |= C_PBIT; + excise(r1); + return 1; + } + return 0; +} + +/* + * return + * 1 if v only used (and substitute), + * 2 if read-alter-rewrite + * 3 if set + * 4 if set and used + * 0 otherwise (not touched) + */ +int +copyu(Prog *p, Adr *v, Adr *s) +{ + switch(p->as) { + + default: + print("copyu: cant find %A\n", p->as); + return 2; + + case AMOVM: + if(v->type != D_REG) + return 0; + if(p->from.type == D_CONST) { /* read reglist, read/rar */ + if(s != A) { + if(p->from.offset&(1<reg)) + return 1; + if(copysub(&p->to, v, s, 1)) + return 1; + return 0; + } + if(copyau(&p->to, v)) { + if(p->scond&C_WBIT) + return 2; + return 1; + } + if(p->from.offset&(1<reg)) + return 1; + } else { /* read/rar, write reglist */ + if(s != A) { + if(p->to.offset&(1<reg)) + return 1; + if(copysub(&p->from, v, s, 1)) + return 1; + return 0; + } + if(copyau(&p->from, v)) { + if(p->scond&C_WBIT) + return 2; + if(p->to.offset&(1<reg)) + return 4; + return 1; + } + if(p->to.offset&(1<reg)) + return 3; + } + return 0; + + case ANOP: /* read,, write */ + case AMOVW: + case AMOVF: + case AMOVD: + case AMOVH: + case AMOVHU: + case AMOVB: + case AMOVBU: + case AMOVFW: + case AMOVWF: + case AMOVDW: + case AMOVWD: + case AMOVFD: + case AMOVDF: + if(p->scond&(C_WBIT|C_PBIT)) + if(v->type == D_REG) { + if(p->from.type == D_OREG || p->from.type == D_SHIFT) { + if(p->from.reg == v->reg) + return 2; + } else { + if(p->to.reg == v->reg) + return 2; + } + } + if(s != A) { + if(copysub(&p->from, v, s, 1)) + return 1; + if(!copyas(&p->to, v)) + if(copysub(&p->to, v, s, 1)) + return 1; + return 0; + } + if(copyas(&p->to, v)) { + if(p->scond != C_SCOND_NONE) + return 2; + if(copyau(&p->from, v)) + return 4; + return 3; + } + if(copyau(&p->from, v)) + return 1; + if(copyau(&p->to, v)) + return 1; + return 0; + + case AMULLU: /* read, read, write, write */ + case AMULA: + case AMVN: + return 2; + + case AADD: /* read, read, write */ + case AADC: + case ASUB: + case ASBC: + case ARSB: + case ASLL: + case ASRL: + case ASRA: + case AORR: + case AAND: + case AEOR: + case AMUL: + case AMULU: + case ADIV: + case ADIVU: + case AMOD: + case AMODU: + case AADDF: + case AADDD: + case ASUBF: + case ASUBD: + case AMULF: + case AMULD: + case ADIVF: + case ADIVD: + + case ACMPF: /* read, read, */ + case ACMPD: + case ACMP: + case ACMN: + case ACASE: + case ATST: /* read,, */ + if(s != A) { + if(copysub(&p->from, v, s, 1)) + return 1; + if(copysub1(p, v, s, 1)) + return 1; + if(!copyas(&p->to, v)) + if(copysub(&p->to, v, s, 1)) + return 1; + return 0; + } + if(copyas(&p->to, v)) { + if(p->scond != C_SCOND_NONE) + return 2; + if(p->reg == NREG) + p->reg = p->to.reg; + if(copyau(&p->from, v)) + return 4; + if(copyau1(p, v)) + return 4; + return 3; + } + if(copyau(&p->from, v)) + return 1; + if(copyau1(p, v)) + return 1; + if(copyau(&p->to, v)) + return 1; + return 0; + + case ABEQ: /* read, read */ + case ABNE: + case ABCS: + case ABHS: + case ABCC: + case ABLO: + case ABMI: + case ABPL: + case ABVS: + case ABVC: + case ABHI: + case ABLS: + case ABGE: + case ABLT: + case ABGT: + case ABLE: + if(s != A) { + if(copysub(&p->from, v, s, 1)) + return 1; + return copysub1(p, v, s, 1); + } + if(copyau(&p->from, v)) + return 1; + if(copyau1(p, v)) + return 1; + return 0; + + case AB: /* funny */ + if(s != A) { + if(copysub(&p->to, v, s, 1)) + return 1; + return 0; + } + if(copyau(&p->to, v)) + return 1; + return 0; + + case ARET: /* funny */ + if(v->type == D_REG) + if(v->reg == REGRET) + return 2; + if(v->type == D_FREG) + if(v->reg == FREGRET) + return 2; + + case ABL: /* funny */ + if(v->type == D_REG) { + if(v->reg <= REGEXT && v->reg > exregoffset) + return 2; + if(v->reg == (uchar)REGARG) + return 2; + } + if(v->type == D_FREG) + if(v->reg <= FREGEXT && v->reg > exfregoffset) + return 2; + + if(s != A) { + if(copysub(&p->to, v, s, 1)) + return 1; + return 0; + } + if(copyau(&p->to, v)) + return 4; + return 3; + + case ATEXT: /* funny */ + if(v->type == D_REG) + if(v->reg == (uchar)REGARG) + return 3; + return 0; + } + return 0; +} + +/* + * direct reference, + * could be set/use depending on + * semantics + */ +int +copyas(Adr *a, Adr *v) +{ + + if(regtyp(v)) { + if(a->type == v->type) + if(a->reg == v->reg) + return 1; + } else + if(v->type == D_CONST) { /* for constprop */ + if(a->type == v->type) + if(a->name == v->name) + if(a->sym == v->sym) + if(a->reg == v->reg) + if(a->offset == v->offset) + return 1; + } + return 0; +} + +/* + * either direct or indirect + */ +int +copyau(Adr *a, Adr *v) +{ + + if(copyas(a, v)) + return 1; + if(v->type == D_REG) { + if(a->type == D_CONST && a->reg != NREG) { + if(a->reg == v->reg) + return 1; + } else + if(a->type == D_OREG) { + if(a->reg == v->reg) + return 1; + } else + if(a->type == D_REGREG) { + if(a->reg == v->reg) + return 1; + if(a->offset == v->reg) + return 1; + } else + if(a->type == D_SHIFT) { + if((a->offset&0xf) == v->reg) + return 1; + if((a->offset&(1<<4)) && (a->offset>>8) == v->reg) + return 1; + } + } + return 0; +} + +/* + * compare v to the center + * register in p (p->reg) + * the trick is that this + * register might be D_REG + * D_FREG. there are basically + * two cases, + * ADD r,r,r + * CMP r,r, + */ +int +copyau1(Prog *p, Adr *v) +{ + + if(regtyp(v)) + if(p->reg == v->reg) { + if(p->to.type != D_NONE) { + if(v->type == p->to.type) + return 1; + return 0; + } + if(p->from.type != D_NONE) { + if(v->type == p->from.type) + return 1; + return 0; + } + print("copyau1: cant tell %P\n", p); + } + return 0; +} + +/* + * substitute s for v in a + * return failure to substitute + */ +int +copysub(Adr *a, Adr *v, Adr *s, int f) +{ + + if(f) + if(copyau(a, v)) { + if(a->type == D_SHIFT) { + if((a->offset&0xf) == v->reg) + a->offset = (a->offset&~0xf)|s->reg; + if((a->offset&(1<<4)) && (a->offset>>8) == v->reg) + a->offset = (a->offset&~(0xf<<8))|(s->reg<<8); + } else + if(a->type == D_REGREG) { + if(a->offset == v->reg) + a->offset = s->reg; + if(a->reg == v->reg) + a->reg = s->reg; + } else + a->reg = s->reg; + } + return 0; +} + +int +copysub1(Prog *p1, Adr *v, Adr *s, int f) +{ + + if(f) + if(copyau1(p1, v)) + p1->reg = s->reg; + return 0; +} + +struct { + int opcode; + int notopcode; + int scond; + int notscond; +} predinfo[] = { + { ABEQ, ABNE, 0x0, 0x1, }, + { ABNE, ABEQ, 0x1, 0x0, }, + { ABCS, ABCC, 0x2, 0x3, }, + { ABHS, ABLO, 0x2, 0x3, }, + { ABCC, ABCS, 0x3, 0x2, }, + { ABLO, ABHS, 0x3, 0x2, }, + { ABMI, ABPL, 0x4, 0x5, }, + { ABPL, ABMI, 0x5, 0x4, }, + { ABVS, ABVC, 0x6, 0x7, }, + { ABVC, ABVS, 0x7, 0x6, }, + { ABHI, ABLS, 0x8, 0x9, }, + { ABLS, ABHI, 0x9, 0x8, }, + { ABGE, ABLT, 0xA, 0xB, }, + { ABLT, ABGE, 0xB, 0xA, }, + { ABGT, ABLE, 0xC, 0xD, }, + { ABLE, ABGT, 0xD, 0xC, }, +}; + +typedef struct { + Reg *start; + Reg *last; + Reg *end; + int len; +} Joininfo; + +enum { + Join, + Split, + End, + Branch, + Setcond, + Toolong +}; + +enum { + Falsecond, + Truecond, + Delbranch, + Keepbranch +}; + +int +isbranch(Prog *p) +{ + return (ABEQ <= p->as) && (p->as <= ABLE); +} + +int +predicable(Prog *p) +{ + switch(p->as) { + case ANOP: + case AXXX: + case ADATA: + case AGLOBL: + case AGOK: + case AHISTORY: + case ANAME: + case ASIGNAME: + case ATEXT: + case AWORD: + case ABCASE: + case ACASE: + return 0; + } + if(isbranch(p)) + return 0; + return 1; +} + +/* + * Depends on an analysis of the encodings performed by 5l. + * These seem to be all of the opcodes that lead to the "S" bit + * being set in the instruction encodings. + * + * C_SBIT may also have been set explicitly in p->scond. + */ +int +modifiescpsr(Prog *p) +{ + switch(p->as) { + case AMULLU: + case AMULA: + case AMULU: + case ADIVU: + + case ATEQ: + case ACMN: + case ATST: + case ACMP: + case AMUL: + case ADIV: + case AMOD: + case AMODU: + case ABL: + return 1; + } + if(p->scond & C_SBIT) + return 1; + return 0; +} + +/* + * Find the maximal chain of instructions starting with r which could + * be executed conditionally + */ +int +joinsplit(Reg *r, Joininfo *j) +{ + j->start = r; + j->last = r; + j->len = 0; + do { + if (r->p2 && (r->p1 || r->p2->p2link)) { + j->end = r; + return Join; + } + if (r->s1 && r->s2) { + j->end = r; + return Split; + } + j->last = r; + if (r->prog->as != ANOP) + j->len++; + if (!r->s1 && !r->s2) { + j->end = r->link; + return End; + } + if (r->s2) { + j->end = r->s2; + return Branch; + } + if (modifiescpsr(r->prog)) { + j->end = r->s1; + return Setcond; + } + r = r->s1; + } while (j->len < 4); + j->end = r; + return Toolong; +} + +Reg* +successor(Reg *r) +{ + if(r->s1) + return r->s1; + else + return r->s2; +} + +void +applypred(Reg *rstart, Joininfo *j, int cond, int branch) +{ + int pred; + Reg *r; + + if(j->len == 0) + return; + if(cond == Truecond) + pred = predinfo[rstart->prog->as - ABEQ].scond; + else + pred = predinfo[rstart->prog->as - ABEQ].notscond; + + for(r = j->start;; r = successor(r)) { + if(r->prog->as == AB) { + if(r != j->last || branch == Delbranch) + excise(r); + else { + if(cond == Truecond) + r->prog->as = predinfo[rstart->prog->as - ABEQ].opcode; + else + r->prog->as = predinfo[rstart->prog->as - ABEQ].notopcode; + } + } + else + if(predicable(r->prog)) + r->prog->scond = (r->prog->scond&~C_SCOND)|pred; + if(r->s1 != r->link) { + r->s1 = r->link; + r->link->p1 = r; + } + if(r == j->last) + break; + } +} + +void +predicate(void) +{ + Reg *r; + int t1, t2; + Joininfo j1, j2; + + for(r=firstr; r!=R; r=r->link) { + if (isbranch(r->prog)) { + t1 = joinsplit(r->s1, &j1); + t2 = joinsplit(r->s2, &j2); + if(j1.last->link != j2.start) + continue; + if(j1.end == j2.end) + if((t1 == Branch && (t2 == Join || t2 == Setcond)) || + (t2 == Join && (t1 == Join || t1 == Setcond))) { + applypred(r, &j1, Falsecond, Delbranch); + applypred(r, &j2, Truecond, Delbranch); + excise(r); + continue; + } + if(t1 == End || t1 == Branch) { + applypred(r, &j1, Falsecond, Keepbranch); + excise(r); + continue; + } + } + } +} + +int +isdconst(Addr *a) +{ + if(a->type == D_CONST && a->reg == NREG) + return 1; + return 0; +} diff --git a/src/cmd/5g/reg.c b/src/cmd/5g/reg.c new file mode 100644 index 000000000..2d2a6d01a --- /dev/null +++ b/src/cmd/5g/reg.c @@ -0,0 +1,1607 @@ +// Inferno utils/5c/reg.c +// http://code.google.com/p/inferno-os/source/browse/utils/5g/reg.c +// +// Copyright © 1994-1999 Lucent Technologies Inc. All rights reserved. +// Portions Copyright © 1995-1997 C H Forsyth (forsyth@terzarima.net) +// Portions Copyright © 1997-1999 Vita Nuova Limited +// Portions Copyright © 2000-2007 Vita Nuova Holdings Limited (www.vitanuova.com) +// Portions Copyright © 2004,2006 Bruce Ellis +// Portions Copyright © 2005-2007 C H Forsyth (forsyth@terzarima.net) +// Revisions Copyright © 2000-2007 Lucent Technologies Inc. and others +// Portions Copyright © 2009 The Go Authors. All rights reserved. +// +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to deal +// in the Software without restriction, including without limitation the rights +// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +// copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +// THE SOFTWARE. + + +#include "gg.h" +#include "opt.h" + +#define NREGVAR 24 +#define REGBITS ((uint32)0xffffff) +#define P2R(p) (Reg*)(p->reg) + + void addsplits(void); + int noreturn(Prog *p); +static int first = 0; + +Reg* +rega(void) +{ + Reg *r; + + r = freer; + if(r == R) { + r = mal(sizeof(*r)); + } else + freer = r->link; + + *r = zreg; + return r; +} + +int +rcmp(const void *a1, const void *a2) +{ + Rgn *p1, *p2; + int c1, c2; + + p1 = (Rgn*)a1; + p2 = (Rgn*)a2; + c1 = p2->cost; + c2 = p1->cost; + if(c1 -= c2) + return c1; + return p2->varno - p1->varno; +} + +static void +setoutvar(void) +{ + Type *t; + Node *n; + Addr a; + Iter save; + Bits bit; + int z; + + t = structfirst(&save, getoutarg(curfn->type)); + while(t != T) { + n = nodarg(t, 1); + a = zprog.from; + naddr(n, &a, 0); + bit = mkvar(R, &a); + for(z=0; zprog; + p->as = ANOP; + p->scond = zprog.scond; + p->from = zprog.from; + p->to = zprog.to; + p->reg = zprog.reg; +} + +static void +setaddrs(Bits bit) +{ + int i, n; + Var *v; + Sym *s; + + while(bany(&bit)) { + // convert each bit to a variable + i = bnum(bit); + s = var[i].sym; + n = var[i].name; + bit.b[i/32] &= ~(1L<<(i%32)); + + // disable all pieces of that variable + for(i=0; isym == s && v->name == n) + v->addr = 2; + } + } +} + +static char* regname[] = { + ".R0", + ".R1", + ".R2", + ".R3", + ".R4", + ".R5", + ".R6", + ".R7", + ".R8", + ".R9", + ".R10", + ".R11", + ".R12", + ".R13", + ".R14", + ".R15", + ".F0", + ".F1", + ".F2", + ".F3", + ".F4", + ".F5", + ".F6", + ".F7", +}; + +void +regopt(Prog *firstp) +{ + Reg *r, *r1; + Prog *p; + int i, z, nr; + uint32 vreg; + Bits bit; + + if(first == 0) { + fmtinstall('Q', Qconv); + } + + first++; + if(debug['K']) { + if(first != 13) + return; +// debug['R'] = 2; +// debug['P'] = 2; + print("optimizing %S\n", curfn->nname->sym); + } + + // count instructions + nr = 0; + for(p=firstp; p!=P; p=p->link) + nr++; + + // if too big dont bother + if(nr >= 10000) { +// print("********** %S is too big (%d)\n", curfn->nname->sym, nr); + return; + } + + r1 = R; + firstr = R; + lastr = R; + + /* + * control flow is more complicated in generated go code + * than in generated c code. define pseudo-variables for + * registers, so we have complete register usage information. + */ + nvar = NREGVAR; + memset(var, 0, NREGVAR*sizeof var[0]); + for(i=0; ilink) { + switch(p->as) { + case ADATA: + case AGLOBL: + case ANAME: + case ASIGNAME: + continue; + } + r = rega(); + nr++; + if(firstr == R) { + firstr = r; + lastr = r; + } else { + lastr->link = r; + r->p1 = lastr; + lastr->s1 = r; + lastr = r; + } + r->prog = p; + p->regp = r; + + r1 = r->p1; + if(r1 != R) { + switch(r1->prog->as) { + case ARET: + case AB: + case ARFE: + r->p1 = R; + r1->s1 = R; + } + } + + /* + * left side always read + */ + bit = mkvar(r, &p->from); + for(z=0; zuse1.b[z] |= bit.b[z]; + + /* + * middle always read when present + */ + if(p->reg != NREG) { + if(p->from.type != D_FREG) + r->use1.b[0] |= RtoB(p->reg); + else + r->use1.b[0] |= FtoB(p->reg); + } + + /* + * right side depends on opcode + */ + bit = mkvar(r, &p->to); + if(bany(&bit)) + switch(p->as) { + default: + yyerror("reg: unknown op: %A", p->as); + break; + + /* + * right side read + */ + case ATST: + case ATEQ: + case ACMP: + case ACMN: + case ACMPD: + case ACMPF: + rightread: + for(z=0; zuse2.b[z] |= bit.b[z]; + break; + + /* + * right side read or read+write, depending on middle + * ADD x, z => z += x + * ADD x, y, z => z = x + y + */ + case AADD: + case AAND: + case AEOR: + case ASUB: + case ARSB: + case AADC: + case ASBC: + case ARSC: + case AORR: + case ABIC: + case ASLL: + case ASRL: + case ASRA: + case AMUL: + case AMULU: + case ADIV: + case AMOD: + case AMODU: + case ADIVU: + if(p->reg != NREG) + goto rightread; + // fall through + + /* + * right side read+write + */ + case AADDF: + case AADDD: + case ASUBF: + case ASUBD: + case AMULF: + case AMULD: + case ADIVF: + case ADIVD: + case AMULA: + case AMULAL: + case AMULALU: + for(z=0; zuse2.b[z] |= bit.b[z]; + r->set.b[z] |= bit.b[z]; + } + break; + + /* + * right side write + */ + case ANOP: + case AMOVB: + case AMOVBU: + case AMOVD: + case AMOVDF: + case AMOVDW: + case AMOVF: + case AMOVFW: + case AMOVH: + case AMOVHU: + case AMOVW: + case AMOVWD: + case AMOVWF: + case AMVN: + case AMULL: + case AMULLU: + if((p->scond & C_SCOND) != C_SCOND_NONE) + for(z=0; zuse2.b[z] |= bit.b[z]; + for(z=0; zset.b[z] |= bit.b[z]; + break; + + /* + * funny + */ + case ABL: + setaddrs(bit); + break; + } + + if(p->as == AMOVM) { + z = p->to.offset; + if(p->from.type == D_CONST) + z = p->from.offset; + for(i=0; z; i++) { + if(z&1) + regbits |= RtoB(i); + z >>= 1; + } + } + } + if(firstr == R) + return; + + for(i=0; iaddr) { + bit = blsh(i); + for(z=0; zaddr, v->etype, v->width, v->sym, v->offset); + } + + /* + * pass 2 + * turn branch references to pointers + * build back pointers + */ + for(r=firstr; r!=R; r=r->link) { + p = r->prog; + if(p->to.type == D_BRANCH) { + if(p->to.branch == P) + fatal("pnil %P", p); + r1 = p->to.branch->regp; + if(r1 == R) + fatal("rnil %P", p); + if(r1 == r) { + //fatal("ref to self %P", p); + continue; + } + r->s2 = r1; + r->p2link = r1->p2; + r1->p2 = r; + } + } + if(debug['R']) { + p = firstr->prog; + print("\n%L %D\n", p->lineno, &p->from); + print(" addr = %Q\n", addrs); + } + + /* + * pass 2.5 + * find looping structure + */ + for(r = firstr; r != R; r = r->link) + r->active = 0; + change = 0; + loopit(firstr, nr); + + /* + * pass 3 + * iterate propagating usage + * back until flow graph is complete + */ +loop1: + change = 0; + for(r = firstr; r != R; r = r->link) + r->active = 0; + for(r = firstr; r != R; r = r->link) + if(r->prog->as == ARET) + prop(r, zbits, zbits); +loop11: + /* pick up unreachable code */ + i = 0; + for(r = firstr; r != R; r = r1) { + r1 = r->link; + if(r1 && r1->active && !r->active) { + prop(r, zbits, zbits); + i = 1; + } + } + if(i) + goto loop11; + if(change) + goto loop1; + + + /* + * pass 4 + * iterate propagating register/variable synchrony + * forward until graph is complete + */ +loop2: + change = 0; + for(r = firstr; r != R; r = r->link) + r->active = 0; + synch(firstr, zbits); + if(change) + goto loop2; + + addsplits(); + + if(debug['R'] > 1) { + print("\nprop structure:\n"); + for(r = firstr; r != R; r = r->link) { + print("%d:%P", r->loop, r->prog); + for(z=0; zset.b[z] | + r->refahead.b[z] | r->calahead.b[z] | + r->refbehind.b[z] | r->calbehind.b[z] | + r->use1.b[z] | r->use2.b[z]; + bit.b[z] &= ~addrs.b[z]; + } + + if(bany(&bit)) { + print("\t"); + if(bany(&r->use1)) + print(" u1=%Q", r->use1); + if(bany(&r->use2)) + print(" u2=%Q", r->use2); + if(bany(&r->set)) + print(" st=%Q", r->set); + if(bany(&r->refahead)) + print(" ra=%Q", r->refahead); + if(bany(&r->calahead)) + print(" ca=%Q", r->calahead); + if(bany(&r->refbehind)) + print(" rb=%Q", r->refbehind); + if(bany(&r->calbehind)) + print(" cb=%Q", r->calbehind); + } + print("\n"); + } + } + + /* + * pass 4.5 + * move register pseudo-variables into regu. + */ + for(r = firstr; r != R; r = r->link) { + r->regu = (r->refbehind.b[0] | r->set.b[0]) & REGBITS; + + r->set.b[0] &= ~REGBITS; + r->use1.b[0] &= ~REGBITS; + r->use2.b[0] &= ~REGBITS; + r->refbehind.b[0] &= ~REGBITS; + r->refahead.b[0] &= ~REGBITS; + r->calbehind.b[0] &= ~REGBITS; + r->calahead.b[0] &= ~REGBITS; + r->regdiff.b[0] &= ~REGBITS; + r->act.b[0] &= ~REGBITS; + } + + /* + * pass 5 + * isolate regions + * calculate costs (paint1) + */ + r = firstr; + if(r) { + for(z=0; zrefahead.b[z] | r->calahead.b[z]) & + ~(externs.b[z] | params.b[z] | addrs.b[z] | consts.b[z]); + if(bany(&bit) & !r->refset) { + // should never happen - all variables are preset + if(debug['w']) + print("%L: used and not set: %Q\n", r->prog->lineno, bit); + r->refset = 1; + } + } + + for(r = firstr; r != R; r = r->link) + r->act = zbits; + rgp = region; + nregion = 0; + for(r = firstr; r != R; r = r->link) { + for(z=0; zset.b[z] & + ~(r->refahead.b[z] | r->calahead.b[z] | addrs.b[z]); + if(bany(&bit) && !r->refset) { + if(debug['w']) + print("%L: set and not used: %Q\n", r->prog->lineno, bit); + r->refset = 1; + excise(r); + } + for(z=0; zact.b[z] | addrs.b[z]); + while(bany(&bit)) { + i = bnum(bit); + rgp->enter = r; + rgp->varno = i; + change = 0; + if(debug['R'] > 1) + print("\n"); + paint1(r, i); + bit.b[i/32] &= ~(1L<<(i%32)); + if(change <= 0) { + if(debug['R']) + print("%L $%d: %Q\n", + r->prog->lineno, change, blsh(i)); + continue; + } + rgp->cost = change; + nregion++; + if(nregion >= NRGN) { + if(debug['R'] > 1) + print("too many regions\n"); + goto brk; + } + rgp++; + } + } +brk: + qsort(region, nregion, sizeof(region[0]), rcmp); + + /* + * pass 6 + * determine used registers (paint2) + * replace code (paint3) + */ + rgp = region; + for(i=0; ivarno); + vreg = paint2(rgp->enter, rgp->varno); + vreg = allreg(vreg, rgp); + if(debug['R']) { + if(rgp->regno >= NREG) + print("%L $%d F%d: %Q\n", + rgp->enter->prog->lineno, + rgp->cost, + rgp->regno-NREG, + bit); + else + print("%L $%d R%d: %Q\n", + rgp->enter->prog->lineno, + rgp->cost, + rgp->regno, + bit); + } + if(rgp->regno != 0) + paint3(rgp->enter, rgp->varno, vreg, rgp->regno); + rgp++; + } + /* + * pass 7 + * peep-hole on basic block + */ + if(!debug['R'] || debug['P']) { + peep(); + } + + /* + * last pass + * eliminate nops + * free aux structures + * adjust the stack pointer + * MOVW.W R1,-12(R13) <<- start + * MOVW R0,R1 + * MOVW R1,8(R13) + * MOVW $0,R1 + * MOVW R1,4(R13) + * BL ,runtime.newproc+0(SB) + * MOVW &ft+-32(SP),R7 <<- adjust + * MOVW &j+-40(SP),R6 <<- adjust + * MOVW autotmp_0003+-24(SP),R5 <<- adjust + * MOVW $12(R13),R13 <<- finish + */ + vreg = 0; + for(p = firstp; p != P; p = p->link) { + while(p->link != P && p->link->as == ANOP) + p->link = p->link->link; + if(p->to.type == D_BRANCH) + while(p->to.branch != P && p->to.branch->as == ANOP) + p->to.branch = p->to.branch->link; + if(p->as == AMOVW && p->to.reg == 13) { + if(p->scond & C_WBIT) { + vreg = -p->to.offset; // in adjust region +// print("%P adjusting %d\n", p, vreg); + continue; + } + if(p->from.type == D_CONST && p->to.type == D_REG) { + if(p->from.offset != vreg) + print("in and out different\n"); +// print("%P finish %d\n", p, vreg); + vreg = 0; // done adjust region + continue; + } + +// print("%P %d %d from type\n", p, p->from.type, D_CONST); +// print("%P %d %d to type\n\n", p, p->to.type, D_REG); + } + + if(p->as == AMOVW && vreg != 0) { + if(p->from.sym != S) + if(p->from.name == D_AUTO || p->from.name == D_PARAM) { + p->from.offset += vreg; +// print("%P adjusting from %d %d\n", p, vreg, p->from.type); + } + if(p->to.sym != S) + if(p->to.name == D_AUTO || p->to.name == D_PARAM) { + p->to.offset += vreg; +// print("%P adjusting to %d %d\n", p, vreg, p->from.type); + } + } + } + if(r1 != R) { + r1->link = freer; + freer = firstr; + } + +} + +void +addsplits(void) +{ + Reg *r, *r1; + int z, i; + Bits bit; + + for(r = firstr; r != R; r = r->link) { + if(r->loop > 1) + continue; + if(r->prog->as == ABL) + continue; + for(r1 = r->p2; r1 != R; r1 = r1->p2link) { + if(r1->loop <= 1) + continue; + for(z=0; zcalbehind.b[z] & + (r->refahead.b[z] | r->use1.b[z] | r->use2.b[z]) & + ~(r->calahead.b[z] & addrs.b[z]); + while(bany(&bit)) { + i = bnum(bit); + bit.b[i/32] &= ~(1L << (i%32)); + } + } + } +} + +/* + * add mov b,rn + * just after r + */ +void +addmove(Reg *r, int bn, int rn, int f) +{ + Prog *p, *p1, *p2; + Adr *a; + Var *v; + + p1 = mal(sizeof(*p1)); + *p1 = zprog; + p = r->prog; + + // If there's a stack fixup coming (after BL newproc or BL deferproc), + // delay the load until after the fixup. + p2 = p->link; + if(p2 && p2->as == AMOVW && p2->from.type == D_CONST && p2->from.reg == REGSP && p2->to.reg == REGSP && p2->to.type == D_REG) + p = p2; + + p1->link = p->link; + p->link = p1; + p1->lineno = p->lineno; + + v = var + bn; + + a = &p1->to; + a->sym = v->sym; + a->name = v->name; + a->node = v->node; + a->offset = v->offset; + a->etype = v->etype; + a->type = D_OREG; + if(a->etype == TARRAY || a->sym == S) + a->type = D_CONST; + + if(v->addr) + fatal("addmove: shouldnt be doing this %A\n", a); + + switch(v->etype) { + default: + print("What is this %E\n", v->etype); + + case TINT8: + p1->as = AMOVB; + break; + case TBOOL: + case TUINT8: +//print("movbu %E %d %S\n", v->etype, bn, v->sym); + p1->as = AMOVBU; + break; + case TINT16: + p1->as = AMOVH; + break; + case TUINT16: + p1->as = AMOVHU; + break; + case TINT32: + case TUINT32: + case TPTR32: + p1->as = AMOVW; + break; + case TFLOAT32: + p1->as = AMOVF; + break; + case TFLOAT64: + p1->as = AMOVD; + break; + } + + p1->from.type = D_REG; + p1->from.reg = rn; + if(rn >= NREG) { + p1->from.type = D_FREG; + p1->from.reg = rn-NREG; + } + if(!f) { + p1->from = *a; + *a = zprog.from; + a->type = D_REG; + a->reg = rn; + if(rn >= NREG) { + a->type = D_FREG; + a->reg = rn-NREG; + } + if(v->etype == TUINT8 || v->etype == TBOOL) + p1->as = AMOVBU; + if(v->etype == TUINT16) + p1->as = AMOVHU; + } + if(debug['R']) + print("%P\t.a%P\n", p, p1); +} + +static int +overlap(int32 o1, int w1, int32 o2, int w2) +{ + int32 t1, t2; + + t1 = o1+w1; + t2 = o2+w2; + + if(!(t1 > o2 && t2 > o1)) + return 0; + + return 1; +} + +Bits +mkvar(Reg *r, Adr *a) +{ + Var *v; + int i, t, n, et, z, w, flag; + int32 o; + Bits bit; + Sym *s; + + // mark registers used + t = a->type; + n = D_NONE; + + flag = 0; + switch(t) { + default: + print("type %d %d %D\n", t, a->name, a); + goto none; + + case D_NONE: + case D_FCONST: + case D_BRANCH: + break; + + case D_CONST: + flag = 1; + goto onereg; + + case D_REGREG: + bit = zbits; + if(a->offset != NREG) + bit.b[0] |= RtoB(a->offset); + if(a->reg != NREG) + bit.b[0] |= RtoB(a->reg); + return bit; + + case D_REG: + case D_SHIFT: + onereg: + if(a->reg != NREG) { + bit = zbits; + bit.b[0] = RtoB(a->reg); + return bit; + } + break; + + case D_OREG: + if(a->reg != NREG) { + if(a == &r->prog->from) + r->use1.b[0] |= RtoB(a->reg); + else + r->use2.b[0] |= RtoB(a->reg); + if(r->prog->scond & (C_PBIT|C_WBIT)) + r->set.b[0] |= RtoB(a->reg); + } + break; + + case D_FREG: + if(a->reg != NREG) { + bit = zbits; + bit.b[0] = FtoB(a->reg); + return bit; + } + break; + } + + switch(a->name) { + default: + goto none; + + case D_EXTERN: + case D_STATIC: + case D_AUTO: + case D_PARAM: + n = a->name; + break; + } + + s = a->sym; + if(s == S) + goto none; + if(s->name[0] == '.') + goto none; + et = a->etype; + o = a->offset; + w = a->width; + + for(i=0; isym == s && v->name == n) { + if(v->offset == o) + if(v->etype == et) + if(v->width == w) + if(!flag) + return blsh(i); + + // if they overlap, disable both + if(overlap(v->offset, v->width, o, w)) { + v->addr = 1; + flag = 1; + } + } + } + + switch(et) { + case 0: + case TFUNC: + case TARRAY: + case TSTRING: + goto none; + } + + if(nvar >= NVAR) { + if(debug['w'] > 1 && s) + fatal("variable not optimized: %D", a); + goto none; + } + + i = nvar; + nvar++; +//print("var %d %E %D %S\n", i, et, a, s); + v = var+i; + v->sym = s; + v->offset = o; + v->name = n; +// v->gotype = a->gotype; + v->etype = et; + v->width = w; + v->addr = flag; // funny punning + v->node = a->node; + + if(debug['R']) + print("bit=%2d et=%2d w=%d+%d %S %D flag=%d\n", i, et, o, w, s, a, v->addr); + + bit = blsh(i); + if(n == D_EXTERN || n == D_STATIC) + for(z=0; zp1) { + for(z=0; zrefahead.b[z]; + if(ref.b[z] != r1->refahead.b[z]) { + r1->refahead.b[z] = ref.b[z]; + change++; + } + cal.b[z] |= r1->calahead.b[z]; + if(cal.b[z] != r1->calahead.b[z]) { + r1->calahead.b[z] = cal.b[z]; + change++; + } + } + switch(r1->prog->as) { + case ABL: + if(noreturn(r1->prog)) + break; + for(z=0; zset.b[z]) | + r1->use1.b[z] | r1->use2.b[z]; + cal.b[z] &= ~(r1->set.b[z] | r1->use1.b[z] | r1->use2.b[z]); + r1->refbehind.b[z] = ref.b[z]; + r1->calbehind.b[z] = cal.b[z]; + } + if(r1->active) + break; + r1->active = 1; + } + for(; r != r1; r = r->p1) + for(r2 = r->p2; r2 != R; r2 = r2->p2link) + prop(r2, r->refbehind, r->calbehind); +} + +/* + * find looping structure + * + * 1) find reverse postordering + * 2) find approximate dominators, + * the actual dominators if the flow graph is reducible + * otherwise, dominators plus some other non-dominators. + * See Matthew S. Hecht and Jeffrey D. Ullman, + * "Analysis of a Simple Algorithm for Global Data Flow Problems", + * Conf. Record of ACM Symp. on Principles of Prog. Langs, Boston, Massachusetts, + * Oct. 1-3, 1973, pp. 207-217. + * 3) find all nodes with a predecessor dominated by the current node. + * such a node is a loop head. + * recursively, all preds with a greater rpo number are in the loop + */ +int32 +postorder(Reg *r, Reg **rpo2r, int32 n) +{ + Reg *r1; + + r->rpo = 1; + r1 = r->s1; + if(r1 && !r1->rpo) + n = postorder(r1, rpo2r, n); + r1 = r->s2; + if(r1 && !r1->rpo) + n = postorder(r1, rpo2r, n); + rpo2r[n] = r; + n++; + return n; +} + +int32 +rpolca(int32 *idom, int32 rpo1, int32 rpo2) +{ + int32 t; + + if(rpo1 == -1) + return rpo2; + while(rpo1 != rpo2){ + if(rpo1 > rpo2){ + t = rpo2; + rpo2 = rpo1; + rpo1 = t; + } + while(rpo1 < rpo2){ + t = idom[rpo2]; + if(t >= rpo2) + fatal("bad idom"); + rpo2 = t; + } + } + return rpo1; +} + +int +doms(int32 *idom, int32 r, int32 s) +{ + while(s > r) + s = idom[s]; + return s == r; +} + +int +loophead(int32 *idom, Reg *r) +{ + int32 src; + + src = r->rpo; + if(r->p1 != R && doms(idom, src, r->p1->rpo)) + return 1; + for(r = r->p2; r != R; r = r->p2link) + if(doms(idom, src, r->rpo)) + return 1; + return 0; +} + +void +loopmark(Reg **rpo2r, int32 head, Reg *r) +{ + if(r->rpo < head || r->active == head) + return; + r->active = head; + r->loop += LOOP; + if(r->p1 != R) + loopmark(rpo2r, head, r->p1); + for(r = r->p2; r != R; r = r->p2link) + loopmark(rpo2r, head, r); +} + +void +loopit(Reg *r, int32 nr) +{ + Reg *r1; + int32 i, d, me; + + if(nr > maxnr) { + rpo2r = mal(nr * sizeof(Reg*)); + idom = mal(nr * sizeof(int32)); + maxnr = nr; + } + d = postorder(r, rpo2r, 0); + if(d > nr) + fatal("too many reg nodes"); + nr = d; + for(i = 0; i < nr / 2; i++){ + r1 = rpo2r[i]; + rpo2r[i] = rpo2r[nr - 1 - i]; + rpo2r[nr - 1 - i] = r1; + } + for(i = 0; i < nr; i++) + rpo2r[i]->rpo = i; + + idom[0] = 0; + for(i = 0; i < nr; i++){ + r1 = rpo2r[i]; + me = r1->rpo; + d = -1; + if(r1->p1 != R && r1->p1->rpo < me) + d = r1->p1->rpo; + for(r1 = r1->p2; r1 != nil; r1 = r1->p2link) + if(r1->rpo < me) + d = rpolca(idom, d, r1->rpo); + idom[i] = d; + } + + for(i = 0; i < nr; i++){ + r1 = rpo2r[i]; + r1->loop++; + if(r1->p2 != R && loophead(idom, r1)) + loopmark(rpo2r, i, r1); + } +} + +void +synch(Reg *r, Bits dif) +{ + Reg *r1; + int z; + + for(r1 = r; r1 != R; r1 = r1->s1) { + for(z=0; zrefbehind.b[z] & r1->refahead.b[z])) | + r1->set.b[z] | r1->regdiff.b[z]; + if(dif.b[z] != r1->regdiff.b[z]) { + r1->regdiff.b[z] = dif.b[z]; + change++; + } + } + if(r1->active) + break; + r1->active = 1; + for(z=0; zcalbehind.b[z] & r1->calahead.b[z]); + if(r1->s2 != R) + synch(r1->s2, dif); + } +} + +uint32 +allreg(uint32 b, Rgn *r) +{ + Var *v; + int i; + + v = var + r->varno; + r->regno = 0; + switch(v->etype) { + + default: + fatal("unknown etype %d/%E", bitno(b), v->etype); + break; + + case TINT8: + case TUINT8: + case TINT16: + case TUINT16: + case TINT32: + case TUINT32: + case TINT: + case TUINT: + case TUINTPTR: + case TBOOL: + case TPTR32: + i = BtoR(~b); + if(i && r->cost >= 0) { + r->regno = i; + return RtoB(i); + } + break; + + case TFLOAT32: + case TFLOAT64: + i = BtoF(~b); + if(i && r->cost >= 0) { + r->regno = i+NREG; + return FtoB(i); + } + break; + + case TINT64: + case TUINT64: + case TPTR64: + case TINTER: + case TSTRUCT: + case TARRAY: + break; + } + return 0; +} + +void +paint1(Reg *r, int bn) +{ + Reg *r1; + Prog *p; + int z; + uint32 bb; + + z = bn/32; + bb = 1L<<(bn%32); + if(r->act.b[z] & bb) + return; + for(;;) { + if(!(r->refbehind.b[z] & bb)) + break; + r1 = r->p1; + if(r1 == R) + break; + if(!(r1->refahead.b[z] & bb)) + break; + if(r1->act.b[z] & bb) + break; + r = r1; + } + + if(LOAD(r) & ~(r->set.b[z] & ~(r->use1.b[z]|r->use2.b[z])) & bb) { + change -= CLOAD * r->loop; + if(debug['R'] > 1) + print("%d%P\td %Q $%d\n", r->loop, + r->prog, blsh(bn), change); + } + for(;;) { + r->act.b[z] |= bb; + p = r->prog; + + if(r->use1.b[z] & bb) { + change += CREF * r->loop; + if(debug['R'] > 1) + print("%d%P\tu1 %Q $%d\n", r->loop, + p, blsh(bn), change); + } + + if((r->use2.b[z]|r->set.b[z]) & bb) { + change += CREF * r->loop; + if(debug['R'] > 1) + print("%d%P\tu2 %Q $%d\n", r->loop, + p, blsh(bn), change); + } + + if(STORE(r) & r->regdiff.b[z] & bb) { + change -= CLOAD * r->loop; + if(debug['R'] > 1) + print("%d%P\tst %Q $%d\n", r->loop, + p, blsh(bn), change); + } + + if(r->refbehind.b[z] & bb) + for(r1 = r->p2; r1 != R; r1 = r1->p2link) + if(r1->refahead.b[z] & bb) + paint1(r1, bn); + + if(!(r->refahead.b[z] & bb)) + break; + r1 = r->s2; + if(r1 != R) + if(r1->refbehind.b[z] & bb) + paint1(r1, bn); + r = r->s1; + if(r == R) + break; + if(r->act.b[z] & bb) + break; + if(!(r->refbehind.b[z] & bb)) + break; + } +} + +uint32 +paint2(Reg *r, int bn) +{ + Reg *r1; + int z; + uint32 bb, vreg; + + z = bn/32; + bb = 1L << (bn%32); + vreg = regbits; + if(!(r->act.b[z] & bb)) + return vreg; + for(;;) { + if(!(r->refbehind.b[z] & bb)) + break; + r1 = r->p1; + if(r1 == R) + break; + if(!(r1->refahead.b[z] & bb)) + break; + if(!(r1->act.b[z] & bb)) + break; + r = r1; + } + for(;;) { + r->act.b[z] &= ~bb; + + vreg |= r->regu; + + if(r->refbehind.b[z] & bb) + for(r1 = r->p2; r1 != R; r1 = r1->p2link) + if(r1->refahead.b[z] & bb) + vreg |= paint2(r1, bn); + + if(!(r->refahead.b[z] & bb)) + break; + r1 = r->s2; + if(r1 != R) + if(r1->refbehind.b[z] & bb) + vreg |= paint2(r1, bn); + r = r->s1; + if(r == R) + break; + if(!(r->act.b[z] & bb)) + break; + if(!(r->refbehind.b[z] & bb)) + break; + } + return vreg; +} + +void +paint3(Reg *r, int bn, int32 rb, int rn) +{ + Reg *r1; + Prog *p; + int z; + uint32 bb; + + z = bn/32; + bb = 1L << (bn%32); + if(r->act.b[z] & bb) + return; + for(;;) { + if(!(r->refbehind.b[z] & bb)) + break; + r1 = r->p1; + if(r1 == R) + break; + if(!(r1->refahead.b[z] & bb)) + break; + if(r1->act.b[z] & bb) + break; + r = r1; + } + + if(LOAD(r) & ~(r->set.b[z] & ~(r->use1.b[z]|r->use2.b[z])) & bb) + addmove(r, bn, rn, 0); + + for(;;) { + r->act.b[z] |= bb; + p = r->prog; + + if(r->use1.b[z] & bb) { + if(debug['R']) + print("%P", p); + addreg(&p->from, rn); + if(debug['R']) + print("\t.c%P\n", p); + } + if((r->use2.b[z]|r->set.b[z]) & bb) { + if(debug['R']) + print("%P", p); + addreg(&p->to, rn); + if(debug['R']) + print("\t.c%P\n", p); + } + + if(STORE(r) & r->regdiff.b[z] & bb) + addmove(r, bn, rn, 1); + r->regu |= rb; + + if(r->refbehind.b[z] & bb) + for(r1 = r->p2; r1 != R; r1 = r1->p2link) + if(r1->refahead.b[z] & bb) + paint3(r1, bn, rb, rn); + + if(!(r->refahead.b[z] & bb)) + break; + r1 = r->s2; + if(r1 != R) + if(r1->refbehind.b[z] & bb) + paint3(r1, bn, rb, rn); + r = r->s1; + if(r == R) + break; + if(r->act.b[z] & bb) + break; + if(!(r->refbehind.b[z] & bb)) + break; + } +} + +void +addreg(Adr *a, int rn) +{ + a->sym = 0; + a->name = D_NONE; + a->type = D_REG; + a->reg = rn; + if(rn >= NREG) { + a->type = D_FREG; + a->reg = rn-NREG; + } +} + +/* + * bit reg + * 0 R0 + * 1 R1 + * ... ... + * 10 R10 + */ +int32 +RtoB(int r) +{ + if(r >= REGTMP-2) // excluded R9 and R10 for m and g + return 0; + return 1L << r; +} + +int +BtoR(int32 b) +{ + b &= 0x01fcL; // excluded R9 and R10 for m and g + if(b == 0) + return 0; + return bitno(b); +} + +/* + * bit reg + * 18 F2 + * 19 F3 + * ... ... + * 23 F7 + */ +int32 +FtoB(int f) +{ + + if(f < 2 || f > NFREG-1) + return 0; + return 1L << (f + 16); +} + +int +BtoF(int32 b) +{ + + b &= 0xfc0000L; + if(b == 0) + return 0; + return bitno(b) - 16; +} + +static Sym* symlist[10]; + +int +noreturn(Prog *p) +{ + Sym *s; + int i; + + if(symlist[0] == S) { + symlist[0] = pkglookup("panicindex", runtimepkg); + symlist[1] = pkglookup("panicslice", runtimepkg); + symlist[2] = pkglookup("throwinit", runtimepkg); + symlist[3] = pkglookup("panic", runtimepkg); + symlist[4] = pkglookup("panicwrap", runtimepkg); + } + + s = p->to.sym; + if(s == S) + return 0; + for(i=0; symlist[i]!=S; i++) + if(s == symlist[i]) + return 1; + return 0; +} + +void +dumpone(Reg *r) +{ + int z; + Bits bit; + + print("%d:%P", r->loop, r->prog); + for(z=0; zset.b[z] | + r->use1.b[z] | + r->use2.b[z] | + r->refbehind.b[z] | + r->refahead.b[z] | + r->calbehind.b[z] | + r->calahead.b[z] | + r->regdiff.b[z] | + r->act.b[z] | + 0; + if(bany(&bit)) { + print("\t"); + if(bany(&r->set)) + print(" s:%Q", r->set); + if(bany(&r->use1)) + print(" u1:%Q", r->use1); + if(bany(&r->use2)) + print(" u2:%Q", r->use2); + if(bany(&r->refbehind)) + print(" rb:%Q ", r->refbehind); + if(bany(&r->refahead)) + print(" ra:%Q ", r->refahead); + if(bany(&r->calbehind)) + print("cb:%Q ", r->calbehind); + if(bany(&r->calahead)) + print(" ca:%Q ", r->calahead); + if(bany(&r->regdiff)) + print(" d:%Q ", r->regdiff); + if(bany(&r->act)) + print(" a:%Q ", r->act); + } + print("\n"); +} + +void +dumpit(char *str, Reg *r0) +{ + Reg *r, *r1; + + print("\n%s\n", str); + for(r = r0; r != R; r = r->link) { + dumpone(r); + r1 = r->p2; + if(r1 != R) { + print(" pred:"); + for(; r1 != R; r1 = r1->p2link) + print(" %.4ud", r1->prog->loc); + print("\n"); + } +// r1 = r->s1; +// if(r1 != R) { +// print(" succ:"); +// for(; r1 != R; r1 = r1->s1) +// print(" %.4ud", r1->prog->loc); +// print("\n"); +// } + } +} diff --git a/src/cmd/5l/5.out.h b/src/cmd/5l/5.out.h new file mode 100644 index 000000000..cf86ae48b --- /dev/null +++ b/src/cmd/5l/5.out.h @@ -0,0 +1,270 @@ +// Inferno utils/5c/5.out.h +// http://code.google.com/p/inferno-os/source/browse/utils/5c/5.out.h +// +// Copyright © 1994-1999 Lucent Technologies Inc. All rights reserved. +// Portions Copyright © 1995-1997 C H Forsyth (forsyth@terzarima.net) +// Portions Copyright © 1997-1999 Vita Nuova Limited +// Portions Copyright © 2000-2007 Vita Nuova Holdings Limited (www.vitanuova.com) +// Portions Copyright © 2004,2006 Bruce Ellis +// Portions Copyright © 2005-2007 C H Forsyth (forsyth@terzarima.net) +// Revisions Copyright © 2000-2007 Lucent Technologies Inc. and others +// Portions Copyright © 2009 The Go Authors. All rights reserved. +// +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to deal +// in the Software without restriction, including without limitation the rights +// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +// copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +// THE SOFTWARE. + +#define NSNAME 8 +#define NSYM 50 +#define NREG 16 + +#define NOPROF (1<<0) +#define DUPOK (1<<1) +#define NOSPLIT (1<<2) +#define ALLTHUMBS (1<<3) + +#define REGRET 0 +/* -1 disables use of REGARG */ +#define REGARG -1 +/* compiler allocates R1 up as temps */ +/* compiler allocates register variables R3 up */ +#define REGEXT 10 +/* these two registers are declared in runtime.h */ +#define REGG (REGEXT-0) +#define REGM (REGEXT-1) +/* compiler allocates external registers R10 down */ +#define REGTMP 11 +#define REGSB 12 +#define REGSP 13 +#define REGLINK 14 +#define REGPC 15 + +#define NFREG 8 +#define FREGRET 0 +#define FREGEXT 7 +#define FREGTMP 15 +/* compiler allocates register variables F0 up */ +/* compiler allocates external registers F7 down */ + +enum as +{ + AXXX, + + AAND, + AEOR, + ASUB, + ARSB, + AADD, + AADC, + ASBC, + ARSC, + ATST, + ATEQ, + ACMP, + ACMN, + AORR, + ABIC, + + AMVN, + + AB, + ABL, + +/* + * Do not reorder or fragment the conditional branch + * opcodes, or the predication code will break + */ + ABEQ, + ABNE, + ABCS, + ABHS, + ABCC, + ABLO, + ABMI, + ABPL, + ABVS, + ABVC, + ABHI, + ABLS, + ABGE, + ABLT, + ABGT, + ABLE, + + AMOVWD, + AMOVWF, + AMOVDW, + AMOVFW, + AMOVFD, + AMOVDF, + AMOVF, + AMOVD, + + ACMPF, + ACMPD, + AADDF, + AADDD, + ASUBF, + ASUBD, + AMULF, + AMULD, + ADIVF, + ADIVD, + ASQRTF, + ASQRTD, + + ASRL, + ASRA, + ASLL, + AMULU, + ADIVU, + AMUL, + ADIV, + AMOD, + AMODU, + + AMOVB, + AMOVBU, + AMOVH, + AMOVHU, + AMOVW, + AMOVM, + ASWPBU, + ASWPW, + + ANOP, + ARFE, + ASWI, + AMULA, + + ADATA, + AGLOBL, + AGOK, + AHISTORY, + ANAME, + ARET, + ATEXT, + AWORD, + ADYNT_, + AINIT_, + ABCASE, + ACASE, + + AEND, + + AMULL, + AMULAL, + AMULLU, + AMULALU, + + ABX, + ABXRET, + ADWORD, + + ASIGNAME, + + ALDREX, + ASTREX, + + ALDREXD, + ASTREXD, + + ALAST, +}; + +/* scond byte */ +#define C_SCOND ((1<<4)-1) +#define C_SBIT (1<<4) +#define C_PBIT (1<<5) +#define C_WBIT (1<<6) +#define C_FBIT (1<<7) /* psr flags-only */ +#define C_UBIT (1<<7) /* up bit, unsigned bit */ + +#define C_SCOND_EQ 0 +#define C_SCOND_NE 1 +#define C_SCOND_HS 2 +#define C_SCOND_LO 3 +#define C_SCOND_MI 4 +#define C_SCOND_PL 5 +#define C_SCOND_VS 6 +#define C_SCOND_VC 7 +#define C_SCOND_HI 8 +#define C_SCOND_LS 9 +#define C_SCOND_GE 10 +#define C_SCOND_LT 11 +#define C_SCOND_GT 12 +#define C_SCOND_LE 13 +#define C_SCOND_NONE 14 +#define C_SCOND_NV 15 + +/* D_SHIFT type */ +#define SHIFT_LL 0<<5 +#define SHIFT_LR 1<<5 +#define SHIFT_AR 2<<5 +#define SHIFT_RR 3<<5 + +/* type/name */ +#define D_GOK 0 +#define D_NONE 1 + +/* type */ +#define D_BRANCH (D_NONE+1) +#define D_OREG (D_NONE+2) +#define D_CONST (D_NONE+7) +#define D_FCONST (D_NONE+8) +#define D_SCONST (D_NONE+9) +#define D_PSR (D_NONE+10) +#define D_REG (D_NONE+12) +#define D_FREG (D_NONE+13) +#define D_FILE (D_NONE+16) +#define D_OCONST (D_NONE+17) +#define D_FILE1 (D_NONE+18) + +#define D_SHIFT (D_NONE+19) +#define D_FPCR (D_NONE+20) +#define D_REGREG (D_NONE+21) +#define D_ADDR (D_NONE+22) + +#define D_SBIG (D_NONE+23) +#define D_CONST2 (D_NONE+24) + +/* name */ +#define D_EXTERN (D_NONE+3) +#define D_STATIC (D_NONE+4) +#define D_AUTO (D_NONE+5) +#define D_PARAM (D_NONE+6) + +/* internal only */ +#define D_SIZE (D_NONE+40) +#define D_PCREL (D_NONE+41) + +/* + * this is the ranlib header + */ +#define SYMDEF "__.SYMDEF" + +/* + * this is the simulated IEEE floating point + */ +typedef struct ieee Ieee; +struct ieee +{ + int32 l; /* contains ls-man 0xffffffff */ + int32 h; /* contains sign 0x80000000 + exp 0x7ff00000 + ms-man 0x000fffff */ +}; diff --git a/src/cmd/5l/Makefile b/src/cmd/5l/Makefile new file mode 100644 index 000000000..8489abc64 --- /dev/null +++ b/src/cmd/5l/Makefile @@ -0,0 +1,43 @@ +# Copyright 2009 The Go Authors. All rights reserved. +# Use of this source code is governed by a BSD-style +# license that can be found in the LICENSE file. + +include ../../Make.inc +O:=$(HOST_O) + +TARG=5l + +OFILES=\ + asm.$O\ + data.$O\ + elf.$O\ + enam.$O\ + ldelf.$O\ + ldmacho.$O\ + ldpe.$O\ + lib.$O\ + list.$O\ + noop.$O\ + obj.$O\ + optab.$O\ + pass.$O\ + prof.$O\ + softfloat.$O\ + span.$O\ + symtab.$O\ + go.$O\ + +HFILES=\ + l.h\ + 5.out.h\ + ../ld/elf.h\ + +include ../../Make.ccmd + +enam.c: 5.out.h + sh mkenam + +CLEANFILES+=enam.c + +%.$O: ../ld/%.c + $(HOST_CC) $(HOST_CFLAGS) -c -I. ../ld/$*.c diff --git a/src/cmd/5l/asm.c b/src/cmd/5l/asm.c new file mode 100644 index 000000000..5b7f6f111 --- /dev/null +++ b/src/cmd/5l/asm.c @@ -0,0 +1,1878 @@ +// Inferno utils/5l/asm.c +// http://code.google.com/p/inferno-os/source/browse/utils/5l/asm.c +// +// Copyright © 1994-1999 Lucent Technologies Inc. All rights reserved. +// Portions Copyright © 1995-1997 C H Forsyth (forsyth@terzarima.net) +// Portions Copyright © 1997-1999 Vita Nuova Limited +// Portions Copyright © 2000-2007 Vita Nuova Holdings Limited (www.vitanuova.com) +// Portions Copyright © 2004,2006 Bruce Ellis +// Portions Copyright © 2005-2007 C H Forsyth (forsyth@terzarima.net) +// Revisions Copyright © 2000-2007 Lucent Technologies Inc. and others +// Portions Copyright © 2009 The Go Authors. All rights reserved. +// +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to deal +// in the Software without restriction, including without limitation the rights +// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +// copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +// THE SOFTWARE. + +// Writing object files. + +#include "l.h" +#include "../ld/lib.h" +#include "../ld/elf.h" + +static Prog *PP; + +char linuxdynld[] = "/lib/ld-linux.so.2"; + +int32 +entryvalue(void) +{ + char *a; + Sym *s; + + a = INITENTRY; + if(*a >= '0' && *a <= '9') + return atolwhex(a); + s = lookup(a, 0); + if(s->type == 0) + return INITTEXT; + if(s->type != STEXT) + diag("entry not text: %s", s->name); + return s->value; +} + +enum { + ElfStrEmpty, + ElfStrInterp, + ElfStrHash, + ElfStrGot, + ElfStrGotPlt, + ElfStrDynamic, + ElfStrDynsym, + ElfStrDynstr, + ElfStrRel, + ElfStrText, + ElfStrData, + ElfStrBss, + ElfStrSymtab, + ElfStrStrtab, + ElfStrShstrtab, + ElfStrRelPlt, + ElfStrPlt, + NElfStr +}; + +vlong elfstr[NElfStr]; + +static int +needlib(char *name) +{ + char *p; + Sym *s; + + if(*name == '\0') + return 0; + + /* reuse hash code in symbol table */ + p = smprint(".dynlib.%s", name); + s = lookup(p, 0); + if(s->type == 0) { + s->type = 100; // avoid SDATA, etc. + return 1; + } + return 0; +} + +int nelfsym = 1; + +void +adddynrel(Sym *s, Reloc *r) +{ + USED(s); + USED(r); + diag("adddynrel: unsupported binary format"); +} + +void +adddynsym(Sym *s) +{ + USED(s); + diag("adddynsym: not implemented"); +} + +static void +elfsetupplt(void) +{ + // TODO +} + +int +archreloc(Reloc *r, Sym *s, vlong *val) +{ + USED(r); + USED(s); + USED(val); + return -1; +} + +void +adddynlib(char *lib) +{ + Sym *s; + + if(!needlib(lib)) + return; + + if(iself) { + s = lookup(".dynstr", 0); + if(s->size == 0) + addstring(s, ""); + elfwritedynent(lookup(".dynamic", 0), DT_NEEDED, addstring(s, lib)); + } else { + diag("adddynlib: unsupported binary format"); + } +} + +void +doelf(void) +{ + Sym *s, *shstrtab, *dynstr; + + if(!iself) + return; + + /* predefine strings we need for section headers */ + shstrtab = lookup(".shstrtab", 0); + shstrtab->type = SELFROSECT; + shstrtab->reachable = 1; + + elfstr[ElfStrEmpty] = addstring(shstrtab, ""); + elfstr[ElfStrText] = addstring(shstrtab, ".text"); + elfstr[ElfStrData] = addstring(shstrtab, ".data"); + elfstr[ElfStrBss] = addstring(shstrtab, ".bss"); + addstring(shstrtab, ".rodata"); + addstring(shstrtab, ".gosymtab"); + addstring(shstrtab, ".gopclntab"); + if(!debug['s']) { + elfstr[ElfStrSymtab] = addstring(shstrtab, ".symtab"); + elfstr[ElfStrStrtab] = addstring(shstrtab, ".strtab"); + } + elfstr[ElfStrShstrtab] = addstring(shstrtab, ".shstrtab"); + + if(!debug['d']) { /* -d suppresses dynamic loader format */ + elfstr[ElfStrInterp] = addstring(shstrtab, ".interp"); + elfstr[ElfStrHash] = addstring(shstrtab, ".hash"); + elfstr[ElfStrGot] = addstring(shstrtab, ".got"); + elfstr[ElfStrGotPlt] = addstring(shstrtab, ".got.plt"); + elfstr[ElfStrDynamic] = addstring(shstrtab, ".dynamic"); + elfstr[ElfStrDynsym] = addstring(shstrtab, ".dynsym"); + elfstr[ElfStrDynstr] = addstring(shstrtab, ".dynstr"); + elfstr[ElfStrRel] = addstring(shstrtab, ".rel"); + elfstr[ElfStrRelPlt] = addstring(shstrtab, ".rel.plt"); + elfstr[ElfStrPlt] = addstring(shstrtab, ".plt"); + + /* dynamic symbol table - first entry all zeros */ + s = lookup(".dynsym", 0); + s->type = SELFROSECT; + s->reachable = 1; + s->value += ELF32SYMSIZE; + + /* dynamic string table */ + s = lookup(".dynstr", 0); + s->type = SELFROSECT; + s->reachable = 1; + if(s->size == 0) + addstring(s, ""); + dynstr = s; + + /* relocation table */ + s = lookup(".rel", 0); + s->reachable = 1; + s->type = SELFROSECT; + + /* global offset table */ + s = lookup(".got", 0); + s->reachable = 1; + s->type = SELFSECT; // writable + + /* hash */ + s = lookup(".hash", 0); + s->reachable = 1; + s->type = SELFROSECT; + + /* got.plt */ + s = lookup(".got.plt", 0); + s->reachable = 1; + s->type = SELFSECT; // writable + + s = lookup(".plt", 0); + s->reachable = 1; + s->type = SELFROSECT; + + s = lookup(".rel.plt", 0); + s->reachable = 1; + s->type = SELFROSECT; + + elfsetupplt(); + + /* define dynamic elf table */ + s = lookup(".dynamic", 0); + s->reachable = 1; + s->type = SELFROSECT; + + /* + * .dynamic table + */ + elfwritedynentsym(s, DT_HASH, lookup(".hash", 0)); + elfwritedynentsym(s, DT_SYMTAB, lookup(".dynsym", 0)); + elfwritedynent(s, DT_SYMENT, ELF32SYMSIZE); + elfwritedynentsym(s, DT_STRTAB, lookup(".dynstr", 0)); + elfwritedynentsymsize(s, DT_STRSZ, lookup(".dynstr", 0)); + elfwritedynentsym(s, DT_REL, lookup(".rel", 0)); + elfwritedynentsymsize(s, DT_RELSZ, lookup(".rel", 0)); + elfwritedynent(s, DT_RELENT, ELF32RELSIZE); + if(rpath) + elfwritedynent(s, DT_RUNPATH, addstring(dynstr, rpath)); + elfwritedynentsym(s, DT_PLTGOT, lookup(".got.plt", 0)); + elfwritedynent(s, DT_PLTREL, DT_REL); + elfwritedynentsymsize(s, DT_PLTRELSZ, lookup(".rel.plt", 0)); + elfwritedynentsym(s, DT_JMPREL, lookup(".rel.plt", 0)); + elfwritedynent(s, DT_NULL, 0); + } +} + +vlong +datoff(vlong addr) +{ + if(addr >= segdata.vaddr) + return addr - segdata.vaddr + segdata.fileoff; + if(addr >= segtext.vaddr) + return addr - segtext.vaddr + segtext.fileoff; + diag("datoff %#x", addr); + return 0; +} + +void +shsym(Elf64_Shdr *sh, Sym *s) +{ + vlong addr; + addr = symaddr(s); + if(sh->flags&SHF_ALLOC) + sh->addr = addr; + sh->off = datoff(addr); + sh->size = s->size; +} + +void +phsh(Elf64_Phdr *ph, Elf64_Shdr *sh) +{ + ph->vaddr = sh->addr; + ph->paddr = ph->vaddr; + ph->off = sh->off; + ph->filesz = sh->size; + ph->memsz = sh->size; + ph->align = sh->addralign; +} + +void +asmb(void) +{ + int32 t; + int a, dynsym; + uint32 fo, symo, startva; + ElfEhdr *eh; + ElfPhdr *ph, *pph; + ElfShdr *sh; + Section *sect; + int o; + + if(debug['v']) + Bprint(&bso, "%5.2f asmb\n", cputime()); + Bflush(&bso); + + sect = segtext.sect; + cseek(sect->vaddr - segtext.vaddr + segtext.fileoff); + codeblk(sect->vaddr, sect->len); + + /* output read-only data in text segment (rodata, gosymtab and pclntab) */ + for(sect = sect->next; sect != nil; sect = sect->next) { + cseek(sect->vaddr - segtext.vaddr + segtext.fileoff); + datblk(sect->vaddr, sect->len); + } + + if(debug['v']) + Bprint(&bso, "%5.2f datblk\n", cputime()); + Bflush(&bso); + + cseek(segdata.fileoff); + datblk(segdata.vaddr, segdata.filelen); + + /* output read-only data in text segment */ + sect = segtext.sect->next; + cseek(sect->vaddr - segtext.vaddr + segtext.fileoff); + datblk(sect->vaddr, sect->len); + + if(iself) { + /* index of elf text section; needed by asmelfsym, double-checked below */ + /* !debug['d'] causes extra sections before the .text section */ + elftextsh = 2; + if(!debug['d']) { + elftextsh += 10; + if(elfverneed) + elftextsh += 2; + } + } + + /* output symbol table */ + symsize = 0; + lcsize = 0; + symo = 0; + if(!debug['s']) { + // TODO: rationalize + if(debug['v']) + Bprint(&bso, "%5.2f sym\n", cputime()); + Bflush(&bso); + switch(HEADTYPE) { + default: + if(iself) + goto ElfSym; + case Hnoheader: + case Hrisc: + case Hixp1200: + case Hipaq: + debug['s'] = 1; + break; + case Hplan9x32: + symo = HEADR+segtext.len+segdata.filelen; + break; + case Hnetbsd: + symo = rnd(segdata.filelen, 4096); + break; + ElfSym: + symo = rnd(HEADR+segtext.filelen, INITRND)+segdata.filelen; + symo = rnd(symo, INITRND); + break; + } + cseek(symo); + if(iself) { + if(debug['v']) + Bprint(&bso, "%5.2f elfsym\n", cputime()); + asmelfsym(); + cflush(); + cwrite(elfstrdat, elfstrsize); + + // if(debug['v']) + // Bprint(&bso, "%5.2f dwarf\n", cputime()); + // dwarfemitdebugsections(); + } + cflush(); + + } + + cursym = nil; + if(debug['v']) + Bprint(&bso, "%5.2f header\n", cputime()); + Bflush(&bso); + cseek(0L); + switch(HEADTYPE) { + case Hnoheader: /* no header */ + break; + case Hrisc: /* aif for risc os */ + lputl(0xe1a00000); /* NOP - decompress code */ + lputl(0xe1a00000); /* NOP - relocation code */ + lputl(0xeb000000 + 12); /* BL - zero init code */ + lputl(0xeb000000 + + (entryvalue() + - INITTEXT + + HEADR + - 12 + - 8) / 4); /* BL - entry code */ + + lputl(0xef000011); /* SWI - exit code */ + lputl(textsize+HEADR); /* text size */ + lputl(segdata.filelen); /* data size */ + lputl(0); /* sym size */ + + lputl(segdata.len - segdata.filelen); /* bss size */ + lputl(0); /* sym type */ + lputl(INITTEXT-HEADR); /* text addr */ + lputl(0); /* workspace - ignored */ + + lputl(32); /* addr mode / data addr flag */ + lputl(0); /* data addr */ + for(t=0; t<2; t++) + lputl(0); /* reserved */ + + for(t=0; t<15; t++) + lputl(0xe1a00000); /* NOP - zero init code */ + lputl(0xe1a0f00e); /* B (R14) - zero init return */ + break; + case Hplan9x32: /* plan 9 */ + lput(0x647); /* magic */ + lput(textsize); /* sizes */ + lput(segdata.filelen); + lput(segdata.len - segdata.filelen); + lput(symsize); /* nsyms */ + lput(entryvalue()); /* va of entry */ + lput(0L); + lput(lcsize); + break; + case Hnetbsd: /* boot for NetBSD */ + lput((143<<16)|0413); /* magic */ + lputl(rnd(HEADR+textsize, 4096)); + lputl(rnd(segdata.filelen, 4096)); + lputl(segdata.len - segdata.filelen); + lputl(symsize); /* nsyms */ + lputl(entryvalue()); /* va of entry */ + lputl(0L); + lputl(0L); + break; + case Hixp1200: /* boot for IXP1200 */ + break; + case Hipaq: /* boot for ipaq */ + lputl(0xe3300000); /* nop */ + lputl(0xe3300000); /* nop */ + lputl(0xe3300000); /* nop */ + lputl(0xe3300000); /* nop */ + break; + case Hlinux: + /* elf arm */ + eh = getElfEhdr(); + fo = HEADR; + startva = INITTEXT - fo; /* va of byte 0 of file */ + + /* This null SHdr must appear before all others */ + newElfShdr(elfstr[ElfStrEmpty]); + + /* program header info */ + pph = newElfPhdr(); + pph->type = PT_PHDR; + pph->flags = PF_R + PF_X; + pph->off = eh->ehsize; + pph->vaddr = INITTEXT - HEADR + pph->off; + pph->paddr = INITTEXT - HEADR + pph->off; + pph->align = INITRND; + + /* + * PHDR must be in a loaded segment. Adjust the text + * segment boundaries downwards to include it. + */ + o = segtext.vaddr - pph->vaddr; + segtext.vaddr -= o; + segtext.len += o; + o = segtext.fileoff - pph->off; + segtext.fileoff -= o; + segtext.filelen += o; + + if(!debug['d']) { + /* interpreter for dynamic linking */ + sh = newElfShdr(elfstr[ElfStrInterp]); + sh->type = SHT_PROGBITS; + sh->flags = SHF_ALLOC; + sh->addralign = 1; + if(interpreter == nil) + interpreter = linuxdynld; + elfinterp(sh, startva, interpreter); + + ph = newElfPhdr(); + ph->type = PT_INTERP; + ph->flags = PF_R; + phsh(ph, sh); + } + + elfphload(&segtext); + elfphload(&segdata); + + /* Dynamic linking sections */ + if (!debug['d']) { /* -d suppresses dynamic loader format */ + /* S headers for dynamic linking */ + sh = newElfShdr(elfstr[ElfStrGot]); + sh->type = SHT_PROGBITS; + sh->flags = SHF_ALLOC+SHF_WRITE; + sh->entsize = 4; + sh->addralign = 4; + shsym(sh, lookup(".got", 0)); + + sh = newElfShdr(elfstr[ElfStrGotPlt]); + sh->type = SHT_PROGBITS; + sh->flags = SHF_ALLOC+SHF_WRITE; + sh->entsize = 4; + sh->addralign = 4; + shsym(sh, lookup(".got.plt", 0)); + + dynsym = eh->shnum; + sh = newElfShdr(elfstr[ElfStrDynsym]); + sh->type = SHT_DYNSYM; + sh->flags = SHF_ALLOC; + sh->entsize = ELF32SYMSIZE; + sh->addralign = 4; + sh->link = dynsym+1; // dynstr + // sh->info = index of first non-local symbol (number of local symbols) + shsym(sh, lookup(".dynsym", 0)); + + sh = newElfShdr(elfstr[ElfStrDynstr]); + sh->type = SHT_STRTAB; + sh->flags = SHF_ALLOC; + sh->addralign = 1; + shsym(sh, lookup(".dynstr", 0)); + + sh = newElfShdr(elfstr[ElfStrHash]); + sh->type = SHT_HASH; + sh->flags = SHF_ALLOC; + sh->entsize = 4; + sh->addralign = 4; + sh->link = dynsym; + shsym(sh, lookup(".hash", 0)); + + sh = newElfShdr(elfstr[ElfStrRel]); + sh->type = SHT_REL; + sh->flags = SHF_ALLOC; + sh->entsize = ELF32RELSIZE; + sh->addralign = 4; + sh->link = dynsym; + shsym(sh, lookup(".rel", 0)); + + /* sh and PT_DYNAMIC for .dynamic section */ + sh = newElfShdr(elfstr[ElfStrDynamic]); + sh->type = SHT_DYNAMIC; + sh->flags = SHF_ALLOC+SHF_WRITE; + sh->entsize = 8; + sh->addralign = 4; + sh->link = dynsym+1; // dynstr + shsym(sh, lookup(".dynamic", 0)); + + ph = newElfPhdr(); + ph->type = PT_DYNAMIC; + ph->flags = PF_R + PF_W; + phsh(ph, sh); + + /* + * Thread-local storage segment (really just size). + if(tlsoffset != 0) { + ph = newElfPhdr(); + ph->type = PT_TLS; + ph->flags = PF_R; + ph->memsz = -tlsoffset; + ph->align = 4; + } + */ + } + + ph = newElfPhdr(); + ph->type = PT_GNU_STACK; + ph->flags = PF_W+PF_R; + ph->align = 4; + + sh = newElfShstrtab(elfstr[ElfStrShstrtab]); + sh->type = SHT_STRTAB; + sh->addralign = 1; + shsym(sh, lookup(".shstrtab", 0)); + + if(elftextsh != eh->shnum) + diag("elftextsh = %d, want %d", elftextsh, eh->shnum); + for(sect=segtext.sect; sect!=nil; sect=sect->next) + elfshbits(sect); + for(sect=segdata.sect; sect!=nil; sect=sect->next) + elfshbits(sect); + + if (!debug['s']) { + sh = newElfShdr(elfstr[ElfStrSymtab]); + sh->type = SHT_SYMTAB; + sh->off = symo; + sh->size = symsize; + sh->addralign = 4; + sh->entsize = 16; + sh->link = eh->shnum; // link to strtab + + sh = newElfShdr(elfstr[ElfStrStrtab]); + sh->type = SHT_STRTAB; + sh->off = symo+symsize; + sh->size = elfstrsize; + sh->addralign = 1; + + // dwarfaddelfheaders(); + } + + /* Main header */ + eh->ident[EI_MAG0] = '\177'; + eh->ident[EI_MAG1] = 'E'; + eh->ident[EI_MAG2] = 'L'; + eh->ident[EI_MAG3] = 'F'; + eh->ident[EI_CLASS] = ELFCLASS32; + eh->ident[EI_DATA] = ELFDATA2LSB; + eh->ident[EI_VERSION] = EV_CURRENT; + + eh->type = ET_EXEC; + eh->machine = EM_ARM; + eh->version = EV_CURRENT; + eh->entry = entryvalue(); + + if(pph != nil) { + pph->filesz = eh->phnum * eh->phentsize; + pph->memsz = pph->filesz; + } + + cseek(0); + a = 0; + a += elfwritehdr(); + a += elfwritephdrs(); + a += elfwriteshdrs(); + cflush(); + if(a+elfwriteinterp() > ELFRESERVE) + diag("ELFRESERVE too small: %d > %d", a, ELFRESERVE); + break; + } + cflush(); + if(debug['c']){ + print("textsize=%d\n", textsize); + print("datsize=%ulld\n", segdata.filelen); + print("bsssize=%ulld\n", segdata.len - segdata.filelen); + print("symsize=%d\n", symsize); + print("lcsize=%d\n", lcsize); + print("total=%lld\n", textsize+segdata.len+symsize+lcsize); + } +} + +/* +void +cput(int32 c) +{ + *cbp++ = c; + if(--cbc <= 0) + cflush(); +} +*/ + +void +wput(int32 l) +{ + + cbp[0] = l>>8; + cbp[1] = l; + cbp += 2; + cbc -= 2; + if(cbc <= 0) + cflush(); +} + + +void +hput(int32 l) +{ + + cbp[0] = l>>8; + cbp[1] = l; + cbp += 2; + cbc -= 2; + if(cbc <= 0) + cflush(); +} + +void +lput(int32 l) +{ + + cbp[0] = l>>24; + cbp[1] = l>>16; + cbp[2] = l>>8; + cbp[3] = l; + cbp += 4; + cbc -= 4; + if(cbc <= 0) + cflush(); +} + +void +nopstat(char *f, Count *c) +{ + if(c->outof) + Bprint(&bso, "%s delay %d/%d (%.2f)\n", f, + c->outof - c->count, c->outof, + (double)(c->outof - c->count)/c->outof); +} + +void +asmout(Prog *p, Optab *o, int32 *out) +{ + int32 o1, o2, o3, o4, o5, o6, v; + int r, rf, rt, rt2; + Reloc *rel; + +PP = p; + o1 = 0; + o2 = 0; + o3 = 0; + o4 = 0; + o5 = 0; + o6 = 0; + armsize += o->size; +if(debug['P']) print("%ux: %P type %d\n", (uint32)(p->pc), p, o->type); + switch(o->type) { + default: + diag("unknown asm %d", o->type); + prasm(p); + break; + + case 0: /* pseudo ops */ +if(debug['G']) print("%ux: %s: arm %d\n", (uint32)(p->pc), p->from.sym->name, p->from.sym->fnptr); + break; + + case 1: /* op R,[R],R */ + o1 = oprrr(p->as, p->scond); + rf = p->from.reg; + rt = p->to.reg; + r = p->reg; + if(p->to.type == D_NONE) + rt = 0; + if(p->as == AMOVW || p->as == AMVN) + r = 0; + else + if(r == NREG) + r = rt; + o1 |= rf | (r<<16) | (rt<<12); + break; + + case 2: /* movbu $I,[R],R */ + aclass(&p->from); + o1 = oprrr(p->as, p->scond); + o1 |= immrot(instoffset); + rt = p->to.reg; + r = p->reg; + if(p->to.type == D_NONE) + rt = 0; + if(p->as == AMOVW || p->as == AMVN) + r = 0; + else if(r == NREG) + r = rt; + o1 |= (r<<16) | (rt<<12); + break; + + case 3: /* add R<<[IR],[R],R */ + mov: + aclass(&p->from); + o1 = oprrr(p->as, p->scond); + o1 |= p->from.offset; + rt = p->to.reg; + r = p->reg; + if(p->to.type == D_NONE) + rt = 0; + if(p->as == AMOVW || p->as == AMVN) + r = 0; + else if(r == NREG) + r = rt; + o1 |= (r<<16) | (rt<<12); + break; + + case 4: /* add $I,[R],R */ + aclass(&p->from); + o1 = oprrr(AADD, p->scond); + o1 |= immrot(instoffset); + r = p->from.reg; + if(r == NREG) + r = o->param; + o1 |= r << 16; + o1 |= p->to.reg << 12; + break; + + case 5: /* bra s */ + v = -8; + if(p->cond != P) + v = (p->cond->pc - pc) - 8; + o1 = opbra(p->as, p->scond); + o1 |= (v >> 2) & 0xffffff; + break; + + case 6: /* b ,O(R) -> add $O,R,PC */ + aclass(&p->to); + o1 = oprrr(AADD, p->scond); + o1 |= immrot(instoffset); + o1 |= p->to.reg << 16; + o1 |= REGPC << 12; + break; + + case 7: /* bl ,O(R) -> mov PC,link; add $O,R,PC */ + aclass(&p->to); + o1 = oprrr(AADD, p->scond); + o1 |= immrot(0); + o1 |= REGPC << 16; + o1 |= REGLINK << 12; + + o2 = oprrr(AADD, p->scond); + o2 |= immrot(instoffset); + o2 |= p->to.reg << 16; + o2 |= REGPC << 12; + break; + + case 8: /* sll $c,[R],R -> mov (R<<$c),R */ + aclass(&p->from); + o1 = oprrr(p->as, p->scond); + r = p->reg; + if(r == NREG) + r = p->to.reg; + o1 |= r; + o1 |= (instoffset&31) << 7; + o1 |= p->to.reg << 12; + break; + + case 9: /* sll R,[R],R -> mov (R<as, p->scond); + r = p->reg; + if(r == NREG) + r = p->to.reg; + o1 |= r; + o1 |= (p->from.reg << 8) | (1<<4); + o1 |= p->to.reg << 12; + break; + + case 10: /* swi [$con] */ + o1 = oprrr(p->as, p->scond); + if(p->to.type != D_NONE) { + aclass(&p->to); + o1 |= instoffset & 0xffffff; + } + break; + + case 11: /* word */ + aclass(&p->to); + o1 = instoffset; + if(p->to.sym != S) { + rel = addrel(cursym); + rel->off = pc - cursym->value; + rel->siz = 4; + rel->type = D_ADDR; + rel->sym = p->to.sym; + rel->add = p->to.offset; + o1 = 0; + } + break; + + case 12: /* movw $lcon, reg */ + o1 = omvl(p, &p->from, p->to.reg); + break; + + case 13: /* op $lcon, [R], R */ + o1 = omvl(p, &p->from, REGTMP); + if(!o1) + break; + o2 = oprrr(p->as, p->scond); + o2 |= REGTMP; + r = p->reg; + if(p->as == AMOVW || p->as == AMVN) + r = 0; + else if(r == NREG) + r = p->to.reg; + o2 |= r << 16; + if(p->to.type != D_NONE) + o2 |= p->to.reg << 12; + break; + + case 14: /* movb/movbu/movh/movhu R,R */ + o1 = oprrr(ASLL, p->scond); + + if(p->as == AMOVBU || p->as == AMOVHU) + o2 = oprrr(ASRL, p->scond); + else + o2 = oprrr(ASRA, p->scond); + + r = p->to.reg; + o1 |= (p->from.reg)|(r<<12); + o2 |= (r)|(r<<12); + if(p->as == AMOVB || p->as == AMOVBU) { + o1 |= (24<<7); + o2 |= (24<<7); + } else { + o1 |= (16<<7); + o2 |= (16<<7); + } + break; + + case 15: /* mul r,[r,]r */ + o1 = oprrr(p->as, p->scond); + rf = p->from.reg; + rt = p->to.reg; + r = p->reg; + if(r == NREG) + r = rt; + if(rt == r) { + r = rf; + rf = rt; + } + if(0) + if(rt == r || rf == REGPC || r == REGPC || rt == REGPC) { + diag("bad registers in MUL"); + prasm(p); + } + o1 |= (rf<<8) | r | (rt<<16); + break; + + + case 16: /* div r,[r,]r */ + o1 = 0xf << 28; + o2 = 0; + break; + + case 17: + o1 = oprrr(p->as, p->scond); + rf = p->from.reg; + rt = p->to.reg; + rt2 = p->to.offset; + r = p->reg; + o1 |= (rf<<8) | r | (rt<<16) | (rt2<<12); + break; + + case 20: /* mov/movb/movbu R,O(R) */ + aclass(&p->to); + r = p->to.reg; + if(r == NREG) + r = o->param; + o1 = osr(p->as, p->from.reg, instoffset, r, p->scond); + break; + + case 21: /* mov/movbu O(R),R -> lr */ + aclass(&p->from); + r = p->from.reg; + if(r == NREG) + r = o->param; + o1 = olr(instoffset, r, p->to.reg, p->scond); + if(p->as != AMOVW) + o1 |= 1<<22; + break; + + case 30: /* mov/movb/movbu R,L(R) */ + o1 = omvl(p, &p->to, REGTMP); + if(!o1) + break; + r = p->to.reg; + if(r == NREG) + r = o->param; + o2 = osrr(p->from.reg, REGTMP,r, p->scond); + if(p->as != AMOVW) + o2 |= 1<<22; + break; + + case 31: /* mov/movbu L(R),R -> lr[b] */ + o1 = omvl(p, &p->from, REGTMP); + if(!o1) + break; + r = p->from.reg; + if(r == NREG) + r = o->param; + o2 = olrr(REGTMP,r, p->to.reg, p->scond); + if(p->as == AMOVBU || p->as == AMOVB) + o2 |= 1<<22; + break; + + case 34: /* mov $lacon,R */ + o1 = omvl(p, &p->from, REGTMP); + if(!o1) + break; + + o2 = oprrr(AADD, p->scond); + o2 |= REGTMP; + r = p->from.reg; + if(r == NREG) + r = o->param; + o2 |= r << 16; + if(p->to.type != D_NONE) + o2 |= p->to.reg << 12; + break; + + case 35: /* mov PSR,R */ + o1 = (2<<23) | (0xf<<16) | (0<<0); + o1 |= (p->scond & C_SCOND) << 28; + o1 |= (p->from.reg & 1) << 22; + o1 |= p->to.reg << 12; + break; + + case 36: /* mov R,PSR */ + o1 = (2<<23) | (0x29f<<12) | (0<<4); + if(p->scond & C_FBIT) + o1 ^= 0x010 << 12; + o1 |= (p->scond & C_SCOND) << 28; + o1 |= (p->to.reg & 1) << 22; + o1 |= p->from.reg << 0; + break; + + case 37: /* mov $con,PSR */ + aclass(&p->from); + o1 = (2<<23) | (0x29f<<12) | (0<<4); + if(p->scond & C_FBIT) + o1 ^= 0x010 << 12; + o1 |= (p->scond & C_SCOND) << 28; + o1 |= immrot(instoffset); + o1 |= (p->to.reg & 1) << 22; + o1 |= p->from.reg << 0; + break; + + case 38: /* movm $con,oreg -> stm */ + o1 = (0x4 << 25); + o1 |= p->from.offset & 0xffff; + o1 |= p->to.reg << 16; + aclass(&p->to); + goto movm; + + case 39: /* movm oreg,$con -> ldm */ + o1 = (0x4 << 25) | (1 << 20); + o1 |= p->to.offset & 0xffff; + o1 |= p->from.reg << 16; + aclass(&p->from); + movm: + if(instoffset != 0) + diag("offset must be zero in MOVM"); + o1 |= (p->scond & C_SCOND) << 28; + if(p->scond & C_PBIT) + o1 |= 1 << 24; + if(p->scond & C_UBIT) + o1 |= 1 << 23; + if(p->scond & C_SBIT) + o1 |= 1 << 22; + if(p->scond & C_WBIT) + o1 |= 1 << 21; + break; + + case 40: /* swp oreg,reg,reg */ + aclass(&p->from); + if(instoffset != 0) + diag("offset must be zero in SWP"); + o1 = (0x2<<23) | (0x9<<4); + if(p->as != ASWPW) + o1 |= 1 << 22; + o1 |= p->from.reg << 16; + o1 |= p->reg << 0; + o1 |= p->to.reg << 12; + o1 |= (p->scond & C_SCOND) << 28; + break; + + case 41: /* rfe -> movm.s.w.u 0(r13),[r15] */ + o1 = 0xe8fd8000; + break; + + case 50: /* floating point store */ + v = regoff(&p->to); + r = p->to.reg; + if(r == NREG) + r = o->param; + o1 = ofsr(p->as, p->from.reg, v, r, p->scond, p); + break; + + case 51: /* floating point load */ + v = regoff(&p->from); + r = p->from.reg; + if(r == NREG) + r = o->param; + o1 = ofsr(p->as, p->to.reg, v, r, p->scond, p) | (1<<20); + break; + + case 52: /* floating point store, int32 offset UGLY */ + o1 = omvl(p, &p->to, REGTMP); + if(!o1) + break; + r = p->to.reg; + if(r == NREG) + r = o->param; + o2 = oprrr(AADD, p->scond) | (REGTMP << 12) | (REGTMP << 16) | r; + o3 = ofsr(p->as, p->from.reg, 0, REGTMP, p->scond, p); + break; + + case 53: /* floating point load, int32 offset UGLY */ + o1 = omvl(p, &p->from, REGTMP); + if(!o1) + break; + r = p->from.reg; + if(r == NREG) + r = o->param; + o2 = oprrr(AADD, p->scond) | (REGTMP << 12) | (REGTMP << 16) | r; + o3 = ofsr(p->as, p->to.reg, 0, REGTMP, p->scond, p) | (1<<20); + break; + + case 54: /* floating point arith */ + o1 = oprrr(p->as, p->scond); + rf = p->from.reg; + rt = p->to.reg; + r = p->reg; + if(r == NREG) { + r = rt; + if(p->as == AMOVF || p->as == AMOVD || p->as == ASQRTF || p->as == ASQRTD) + r = 0; + } + o1 |= rf | (r<<16) | (rt<<12); + break; + + case 56: /* move to FP[CS]R */ + o1 = ((p->scond & C_SCOND) << 28) | (0xe << 24) | (1<<8) | (1<<4); + o1 |= ((p->to.reg+1)<<21) | (p->from.reg << 12); + break; + + case 57: /* move from FP[CS]R */ + o1 = ((p->scond & C_SCOND) << 28) | (0xe << 24) | (1<<8) | (1<<4); + o1 |= ((p->from.reg+1)<<21) | (p->to.reg<<12) | (1<<20); + break; + case 58: /* movbu R,R */ + o1 = oprrr(AAND, p->scond); + o1 |= immrot(0xff); + rt = p->to.reg; + r = p->from.reg; + if(p->to.type == D_NONE) + rt = 0; + if(r == NREG) + r = rt; + o1 |= (r<<16) | (rt<<12); + break; + + case 59: /* movw/bu R< ldr indexed */ + if(p->from.reg == NREG) { + if(p->as != AMOVW) + diag("byte MOV from shifter operand"); + goto mov; + } + if(p->from.offset&(1<<4)) + diag("bad shift in LDR"); + o1 = olrr(p->from.offset, p->from.reg, p->to.reg, p->scond); + if(p->as == AMOVBU) + o1 |= 1<<22; + break; + + case 60: /* movb R(R),R -> ldrsb indexed */ + if(p->from.reg == NREG) { + diag("byte MOV from shifter operand"); + goto mov; + } + if(p->from.offset&(~0xf)) + diag("bad shift in LDRSB"); + o1 = olhrr(p->from.offset, p->from.reg, p->to.reg, p->scond); + o1 ^= (1<<5)|(1<<6); + break; + + case 61: /* movw/b/bu R,R<<[IR](R) -> str indexed */ + if(p->to.reg == NREG) + diag("MOV to shifter operand"); + o1 = osrr(p->from.reg, p->to.offset, p->to.reg, p->scond); + if(p->as == AMOVB || p->as == AMOVBU) + o1 |= 1<<22; + break; + + case 62: /* case R -> movw R<<2(PC),PC */ + o1 = olrr(p->from.reg, REGPC, REGPC, p->scond); + o1 |= 2<<7; + break; + + case 63: /* bcase */ + if(p->cond != P) + o1 = p->cond->pc; + break; + + /* reloc ops */ + case 64: /* mov/movb/movbu R,addr */ + o1 = omvl(p, &p->to, REGTMP); + if(!o1) + break; + o2 = osr(p->as, p->from.reg, 0, REGTMP, p->scond); + break; + + case 65: /* mov/movbu addr,R */ + o1 = omvl(p, &p->from, REGTMP); + if(!o1) + break; + o2 = olr(0, REGTMP, p->to.reg, p->scond); + if(p->as == AMOVBU || p->as == AMOVB) + o2 |= 1<<22; + break; + + case 68: /* floating point store -> ADDR */ + o1 = omvl(p, &p->to, REGTMP); + if(!o1) + break; + o2 = ofsr(p->as, p->from.reg, 0, REGTMP, p->scond, p); + break; + + case 69: /* floating point load <- ADDR */ + o1 = omvl(p, &p->from, REGTMP); + if(!o1) + break; + o2 = ofsr(p->as, p->to.reg, 0, REGTMP, p->scond, p) | (1<<20); + break; + + /* ArmV4 ops: */ + case 70: /* movh/movhu R,O(R) -> strh */ + aclass(&p->to); + r = p->to.reg; + if(r == NREG) + r = o->param; + o1 = oshr(p->from.reg, instoffset, r, p->scond); + break; + case 71: /* movb/movh/movhu O(R),R -> ldrsb/ldrsh/ldrh */ + aclass(&p->from); + r = p->from.reg; + if(r == NREG) + r = o->param; + o1 = olhr(instoffset, r, p->to.reg, p->scond); + if(p->as == AMOVB) + o1 ^= (1<<5)|(1<<6); + else if(p->as == AMOVH) + o1 ^= (1<<6); + break; + case 72: /* movh/movhu R,L(R) -> strh */ + o1 = omvl(p, &p->to, REGTMP); + if(!o1) + break; + r = p->to.reg; + if(r == NREG) + r = o->param; + o2 = oshrr(p->from.reg, REGTMP,r, p->scond); + break; + case 73: /* movb/movh/movhu L(R),R -> ldrsb/ldrsh/ldrh */ + o1 = omvl(p, &p->from, REGTMP); + if(!o1) + break; + r = p->from.reg; + if(r == NREG) + r = o->param; + o2 = olhrr(REGTMP, r, p->to.reg, p->scond); + if(p->as == AMOVB) + o2 ^= (1<<5)|(1<<6); + else if(p->as == AMOVH) + o2 ^= (1<<6); + break; + case 74: /* bx $I */ + diag("ABX $I"); + break; + case 75: /* bx O(R) */ + aclass(&p->to); + if(instoffset != 0) + diag("non-zero offset in ABX"); +/* + o1 = oprrr(AADD, p->scond) | immrot(0) | (REGPC<<16) | (REGLINK<<12); // mov PC, LR + o2 = ((p->scond&C_SCOND)<<28) | (0x12fff<<8) | (1<<4) | p->to.reg; // BX R +*/ + // p->to.reg may be REGLINK + o1 = oprrr(AADD, p->scond); + o1 |= immrot(instoffset); + o1 |= p->to.reg << 16; + o1 |= REGTMP << 12; + o2 = oprrr(AADD, p->scond) | immrot(0) | (REGPC<<16) | (REGLINK<<12); // mov PC, LR + o3 = ((p->scond&C_SCOND)<<28) | (0x12fff<<8) | (1<<4) | REGTMP; // BX Rtmp + break; + case 76: /* bx O(R) when returning from fn*/ + diag("ABXRET"); + break; + case 77: /* ldrex oreg,reg */ + aclass(&p->from); + if(instoffset != 0) + diag("offset must be zero in LDREX"); + o1 = (0x19<<20) | (0xf9f); + o1 |= p->from.reg << 16; + o1 |= p->to.reg << 12; + o1 |= (p->scond & C_SCOND) << 28; + break; + case 78: /* strex reg,oreg,reg */ + aclass(&p->from); + if(instoffset != 0) + diag("offset must be zero in STREX"); + o1 = (0x18<<20) | (0xf90); + o1 |= p->from.reg << 16; + o1 |= p->reg << 0; + o1 |= p->to.reg << 12; + o1 |= (p->scond & C_SCOND) << 28; + break; + case 80: /* fmov zfcon,freg */ + if(p->as == AMOVD) { + o1 = 0xeeb00b00; // VMOV imm 64 + o2 = oprrr(ASUBD, p->scond); + } else { + o1 = 0x0eb00a00; // VMOV imm 32 + o2 = oprrr(ASUBF, p->scond); + } + v = 0x70; // 1.0 + r = p->to.reg; + + // movf $1.0, r + o1 |= (p->scond & C_SCOND) << 28; + o1 |= r << 12; + o1 |= (v&0xf) << 0; + o1 |= (v&0xf0) << 12; + + // subf r,r,r + o2 |= r | (r<<16) | (r<<12); + break; + case 81: /* fmov sfcon,freg */ + o1 = 0x0eb00a00; // VMOV imm 32 + if(p->as == AMOVD) + o1 = 0xeeb00b00; // VMOV imm 64 + o1 |= (p->scond & C_SCOND) << 28; + o1 |= p->to.reg << 12; + v = chipfloat(&p->from.ieee); + o1 |= (v&0xf) << 0; + o1 |= (v&0xf0) << 12; + break; + case 82: /* fcmp freg,freg, */ + o1 = oprrr(p->as, p->scond); + o1 |= (p->reg<<12) | (p->from.reg<<0); + o2 = 0x0ef1fa10; // VMRS R15 + o2 |= (p->scond & C_SCOND) << 28; + break; + case 83: /* fcmp freg,, */ + o1 = oprrr(p->as, p->scond); + o1 |= (p->from.reg<<12) | (1<<16); + o2 = 0x0ef1fa10; // VMRS R15 + o2 |= (p->scond & C_SCOND) << 28; + break; + case 84: /* movfw freg,freg - truncate float-to-fix */ + o1 = oprrr(p->as, p->scond); + o1 |= (p->from.reg<<0); + o1 |= (p->to.reg<<12); + break; + case 85: /* movwf freg,freg - fix-to-float */ + o1 = oprrr(p->as, p->scond); + o1 |= (p->from.reg<<0); + o1 |= (p->to.reg<<12); + break; + case 86: /* movfw freg,reg - truncate float-to-fix */ + // macro for movfw freg,FTMP; movw FTMP,reg + o1 = oprrr(p->as, p->scond); + o1 |= (p->from.reg<<0); + o1 |= (FREGTMP<<12); + o2 = oprrr(AMOVFW+AEND, p->scond); + o2 |= (FREGTMP<<16); + o2 |= (p->to.reg<<12); + break; + case 87: /* movwf reg,freg - fix-to-float */ + // macro for movw reg,FTMP; movwf FTMP,freg + o1 = oprrr(AMOVWF+AEND, p->scond); + o1 |= (p->from.reg<<12); + o1 |= (FREGTMP<<16); + o2 = oprrr(p->as, p->scond); + o2 |= (FREGTMP<<0); + o2 |= (p->to.reg<<12); + break; + case 88: /* movw reg,freg */ + o1 = oprrr(AMOVWF+AEND, p->scond); + o1 |= (p->from.reg<<12); + o1 |= (p->to.reg<<16); + break; + case 89: /* movw freg,reg */ + o1 = oprrr(AMOVFW+AEND, p->scond); + o1 |= (p->from.reg<<16); + o1 |= (p->to.reg<<12); + break; + case 90: /* tst reg */ + o1 = oprrr(ACMP+AEND, p->scond); + o1 |= p->from.reg<<16; + break; + case 91: /* ldrexd oreg,reg */ + aclass(&p->from); + if(instoffset != 0) + diag("offset must be zero in LDREX"); + o1 = (0x1b<<20) | (0xf9f); + o1 |= p->from.reg << 16; + o1 |= p->to.reg << 12; + o1 |= (p->scond & C_SCOND) << 28; + break; + case 92: /* strexd reg,oreg,reg */ + aclass(&p->from); + if(instoffset != 0) + diag("offset must be zero in STREX"); + o1 = (0x1a<<20) | (0xf90); + o1 |= p->from.reg << 16; + o1 |= p->reg << 0; + o1 |= p->to.reg << 12; + o1 |= (p->scond & C_SCOND) << 28; + break; + case 93: /* movb/movh/movhu addr,R -> ldrsb/ldrsh/ldrh */ + o1 = omvl(p, &p->from, REGTMP); + if(!o1) + break; + o2 = olhr(0, REGTMP, p->to.reg, p->scond); + if(p->as == AMOVB) + o2 ^= (1<<5)|(1<<6); + else if(p->as == AMOVH) + o2 ^= (1<<6); + break; + case 94: /* movh/movhu R,addr -> strh */ + o1 = omvl(p, &p->to, REGTMP); + if(!o1) + break; + o2 = oshr(p->from.reg, 0, REGTMP, p->scond); + break; + } + + out[0] = o1; + out[1] = o2; + out[2] = o3; + out[3] = o4; + out[4] = o5; + out[5] = o6; + return; + +#ifdef NOTDEF + v = p->pc; + switch(o->size) { + default: + if(debug['a']) + Bprint(&bso, " %.8ux:\t\t%P\n", v, p); + break; + case 4: + if(debug['a']) + Bprint(&bso, " %.8ux: %.8ux\t%P\n", v, o1, p); + lputl(o1); + break; + case 8: + if(debug['a']) + Bprint(&bso, " %.8ux: %.8ux %.8ux%P\n", v, o1, o2, p); + lputl(o1); + lputl(o2); + break; + case 12: + if(debug['a']) + Bprint(&bso, " %.8ux: %.8ux %.8ux %.8ux%P\n", v, o1, o2, o3, p); + lputl(o1); + lputl(o2); + lputl(o3); + break; + case 16: + if(debug['a']) + Bprint(&bso, " %.8ux: %.8ux %.8ux %.8ux %.8ux%P\n", + v, o1, o2, o3, o4, p); + lputl(o1); + lputl(o2); + lputl(o3); + lputl(o4); + break; + case 20: + if(debug['a']) + Bprint(&bso, " %.8ux: %.8ux %.8ux %.8ux %.8ux %.8ux%P\n", + v, o1, o2, o3, o4, o5, p); + lputl(o1); + lputl(o2); + lputl(o3); + lputl(o4); + lputl(o5); + break; + case 24: + if(debug['a']) + Bprint(&bso, " %.8ux: %.8ux %.8ux %.8ux %.8ux %.8ux %.8ux%P\n", + v, o1, o2, o3, o4, o5, o6, p); + lputl(o1); + lputl(o2); + lputl(o3); + lputl(o4); + lputl(o5); + lputl(o6); + break; + } +#endif +} + +int32 +oprrr(int a, int sc) +{ + int32 o; + + o = (sc & C_SCOND) << 28; + if(sc & C_SBIT) + o |= 1 << 20; + if(sc & (C_PBIT|C_WBIT)) + diag(".P/.W on dp instruction"); + switch(a) { + case AMULU: + case AMUL: return o | (0x0<<21) | (0x9<<4); + case AMULA: return o | (0x1<<21) | (0x9<<4); + case AMULLU: return o | (0x4<<21) | (0x9<<4); + case AMULL: return o | (0x6<<21) | (0x9<<4); + case AMULALU: return o | (0x5<<21) | (0x9<<4); + case AMULAL: return o | (0x7<<21) | (0x9<<4); + case AAND: return o | (0x0<<21); + case AEOR: return o | (0x1<<21); + case ASUB: return o | (0x2<<21); + case ARSB: return o | (0x3<<21); + case AADD: return o | (0x4<<21); + case AADC: return o | (0x5<<21); + case ASBC: return o | (0x6<<21); + case ARSC: return o | (0x7<<21); + case ATST: return o | (0x8<<21) | (1<<20); + case ATEQ: return o | (0x9<<21) | (1<<20); + case ACMP: return o | (0xa<<21) | (1<<20); + case ACMN: return o | (0xb<<21) | (1<<20); + case AORR: return o | (0xc<<21); + case AMOVW: return o | (0xd<<21); + case ABIC: return o | (0xe<<21); + case AMVN: return o | (0xf<<21); + case ASLL: return o | (0xd<<21) | (0<<5); + case ASRL: return o | (0xd<<21) | (1<<5); + case ASRA: return o | (0xd<<21) | (2<<5); + case ASWI: return o | (0xf<<24); + + case AADDD: return o | (0xe<<24) | (0x3<<20) | (0xb<<8) | (0<<4); + case AADDF: return o | (0xe<<24) | (0x3<<20) | (0xa<<8) | (0<<4); + case ASUBD: return o | (0xe<<24) | (0x3<<20) | (0xb<<8) | (4<<4); + case ASUBF: return o | (0xe<<24) | (0x3<<20) | (0xa<<8) | (4<<4); + case AMULD: return o | (0xe<<24) | (0x2<<20) | (0xb<<8) | (0<<4); + case AMULF: return o | (0xe<<24) | (0x2<<20) | (0xa<<8) | (0<<4); + case ADIVD: return o | (0xe<<24) | (0x8<<20) | (0xb<<8) | (0<<4); + case ADIVF: return o | (0xe<<24) | (0x8<<20) | (0xa<<8) | (0<<4); + case ASQRTD: return o | (0xe<<24) | (0xb<<20) | (1<<16) | (0xb<<8) | (0xc<<4); + case ASQRTF: return o | (0xe<<24) | (0xb<<20) | (1<<16) | (0xa<<8) | (0xc<<4); + case ACMPD: return o | (0xe<<24) | (0xb<<20) | (4<<16) | (0xb<<8) | (0xc<<4); + case ACMPF: return o | (0xe<<24) | (0xb<<20) | (4<<16) | (0xa<<8) | (0xc<<4); + + case AMOVF: return o | (0xe<<24) | (0xb<<20) | (0<<16) | (0xa<<8) | (4<<4); + case AMOVD: return o | (0xe<<24) | (0xb<<20) | (0<<16) | (0xb<<8) | (4<<4); + + case AMOVDF: return o | (0xe<<24) | (0xb<<20) | (7<<16) | (0xa<<8) | (0xc<<4) | + (1<<8); // dtof + case AMOVFD: return o | (0xe<<24) | (0xb<<20) | (7<<16) | (0xa<<8) | (0xc<<4) | + (0<<8); // dtof + + case AMOVWF: + if((sc & C_UBIT) == 0) + o |= 1<<7; /* signed */ + return o | (0xe<<24) | (0xb<<20) | (8<<16) | (0xa<<8) | (4<<4) | + (0<<18) | (0<<8); // toint, double + case AMOVWD: + if((sc & C_UBIT) == 0) + o |= 1<<7; /* signed */ + return o | (0xe<<24) | (0xb<<20) | (8<<16) | (0xa<<8) | (4<<4) | + (0<<18) | (1<<8); // toint, double + + case AMOVFW: + if((sc & C_UBIT) == 0) + o |= 1<<16; /* signed */ + return o | (0xe<<24) | (0xb<<20) | (8<<16) | (0xa<<8) | (4<<4) | + (1<<18) | (0<<8) | (1<<7); // toint, double, trunc + case AMOVDW: + if((sc & C_UBIT) == 0) + o |= 1<<16; /* signed */ + return o | (0xe<<24) | (0xb<<20) | (8<<16) | (0xa<<8) | (4<<4) | + (1<<18) | (1<<8) | (1<<7); // toint, double, trunc + + case AMOVWF+AEND: // copy WtoF + return o | (0xe<<24) | (0x0<<20) | (0xb<<8) | (1<<4); + case AMOVFW+AEND: // copy FtoW + return o | (0xe<<24) | (0x1<<20) | (0xb<<8) | (1<<4); + case ACMP+AEND: // cmp imm + return o | (0x3<<24) | (0x5<<20); + } + diag("bad rrr %d", a); + prasm(curp); + return 0; +} + +int32 +opbra(int a, int sc) +{ + + if(sc & (C_SBIT|C_PBIT|C_WBIT)) + diag(".S/.P/.W on bra instruction"); + sc &= C_SCOND; + if(a == ABL) + return (sc<<28)|(0x5<<25)|(0x1<<24); + if(sc != 0xe) + diag(".COND on bcond instruction"); + switch(a) { + case ABEQ: return (0x0<<28)|(0x5<<25); + case ABNE: return (0x1<<28)|(0x5<<25); + case ABCS: return (0x2<<28)|(0x5<<25); + case ABHS: return (0x2<<28)|(0x5<<25); + case ABCC: return (0x3<<28)|(0x5<<25); + case ABLO: return (0x3<<28)|(0x5<<25); + case ABMI: return (0x4<<28)|(0x5<<25); + case ABPL: return (0x5<<28)|(0x5<<25); + case ABVS: return (0x6<<28)|(0x5<<25); + case ABVC: return (0x7<<28)|(0x5<<25); + case ABHI: return (0x8<<28)|(0x5<<25); + case ABLS: return (0x9<<28)|(0x5<<25); + case ABGE: return (0xa<<28)|(0x5<<25); + case ABLT: return (0xb<<28)|(0x5<<25); + case ABGT: return (0xc<<28)|(0x5<<25); + case ABLE: return (0xd<<28)|(0x5<<25); + case AB: return (0xe<<28)|(0x5<<25); + } + diag("bad bra %A", a); + prasm(curp); + return 0; +} + +int32 +olr(int32 v, int b, int r, int sc) +{ + int32 o; + + if(sc & C_SBIT) + diag(".S on LDR/STR instruction"); + o = (sc & C_SCOND) << 28; + if(!(sc & C_PBIT)) + o |= 1 << 24; + if(!(sc & C_UBIT)) + o |= 1 << 23; + if(sc & C_WBIT) + o |= 1 << 21; + o |= (1<<26) | (1<<20); + if(v < 0) { + if(sc & C_UBIT) diag(".U on neg offset"); + v = -v; + o ^= 1 << 23; + } + if(v >= (1<<12) || v < 0) + diag("literal span too large: %d (R%d)\n%P", v, b, PP); + o |= v; + o |= b << 16; + o |= r << 12; + return o; +} + +int32 +olhr(int32 v, int b, int r, int sc) +{ + int32 o; + + if(sc & C_SBIT) + diag(".S on LDRH/STRH instruction"); + o = (sc & C_SCOND) << 28; + if(!(sc & C_PBIT)) + o |= 1 << 24; + if(sc & C_WBIT) + o |= 1 << 21; + o |= (1<<23) | (1<<20)|(0xb<<4); + if(v < 0) { + v = -v; + o ^= 1 << 23; + } + if(v >= (1<<8) || v < 0) + diag("literal span too large: %d (R%d)\n%P", v, b, PP); + o |= (v&0xf)|((v>>4)<<8)|(1<<22); + o |= b << 16; + o |= r << 12; + return o; +} + +int32 +osr(int a, int r, int32 v, int b, int sc) +{ + int32 o; + + o = olr(v, b, r, sc) ^ (1<<20); + if(a != AMOVW) + o |= 1<<22; + return o; +} + +int32 +oshr(int r, int32 v, int b, int sc) +{ + int32 o; + + o = olhr(v, b, r, sc) ^ (1<<20); + return o; +} + + +int32 +osrr(int r, int i, int b, int sc) +{ + + return olr(i, b, r, sc) ^ ((1<<25) | (1<<20)); +} + +int32 +oshrr(int r, int i, int b, int sc) +{ + return olhr(i, b, r, sc) ^ ((1<<22) | (1<<20)); +} + +int32 +olrr(int i, int b, int r, int sc) +{ + + return olr(i, b, r, sc) ^ (1<<25); +} + +int32 +olhrr(int i, int b, int r, int sc) +{ + return olhr(i, b, r, sc) ^ (1<<22); +} + +int32 +ofsr(int a, int r, int32 v, int b, int sc, Prog *p) +{ + int32 o; + + if(sc & C_SBIT) + diag(".S on FLDR/FSTR instruction"); + o = (sc & C_SCOND) << 28; + if(!(sc & C_PBIT)) + o |= 1 << 24; + if(sc & C_WBIT) + o |= 1 << 21; + o |= (6<<25) | (1<<24) | (1<<23) | (10<<8); + if(v < 0) { + v = -v; + o ^= 1 << 23; + } + if(v & 3) + diag("odd offset for floating point op: %d\n%P", v, p); + else + if(v >= (1<<10) || v < 0) + diag("literal span too large: %d\n%P", v, p); + o |= (v>>2) & 0xFF; + o |= b << 16; + o |= r << 12; + + switch(a) { + default: + diag("bad fst %A", a); + case AMOVD: + o |= 1 << 8; + case AMOVF: + break; + } + return o; +} + +int32 +omvl(Prog *p, Adr *a, int dr) +{ + int32 v, o1; + if(!p->cond) { + aclass(a); + v = immrot(~instoffset); + if(v == 0) { + diag("missing literal"); + prasm(p); + return 0; + } + o1 = oprrr(AMVN, p->scond&C_SCOND); + o1 |= v; + o1 |= dr << 12; + } else { + v = p->cond->pc - p->pc - 8; + o1 = olr(v, REGPC, dr, p->scond&C_SCOND); + } + return o1; +} + +int +chipzero(Ieee *e) +{ + if(e->l != 0 || e->h != 0) + return -1; + return 0; +} + +int +chipfloat(Ieee *e) +{ + int n; + ulong h; + + if(e->l != 0 || (e->h&0xffff) != 0) + goto no; + h = e->h & 0x7fc00000; + if(h != 0x40000000 && h != 0x3fc00000) + goto no; + n = 0; + + // sign bit (a) + if(e->h & 0x80000000) + n |= 1<<7; + + // exp sign bit (b) + if(h == 0x3fc00000) + n |= 1<<6; + + // rest of exp and mantissa (cd-efgh) + n |= (e->h >> 16) & 0x3f; + +//print("match %.8lux %.8lux %d\n", e->l, e->h, n); + return n; + +no: + return -1; +} + + +void +genasmsym(void (*put)(Sym*, char*, int, vlong, vlong, int, Sym*)) +{ + Auto *a; + Sym *s; + int h; + + s = lookup("etext", 0); + if(s->type == STEXT) + put(s, s->name, 'T', s->value, s->size, s->version, 0); + + for(h=0; hhash) { + if(s->hide) + continue; + switch(s->type) { + case SCONST: + case SRODATA: + case SDATA: + case SELFROSECT: + case STYPE: + case SSTRING: + case SGOSTRING: + if(!s->reachable) + continue; + put(s, s->name, 'D', s->value, s->size, s->version, s->gotype); + continue; + + case SBSS: + if(!s->reachable) + continue; + put(s, s->name, 'B', s->value, s->size, s->version, s->gotype); + continue; + + case SFILE: + put(nil, s->name, 'f', s->value, 0, s->version, 0); + continue; + } + } + } + + for(s = textp; s != nil; s = s->next) { + /* filenames first */ + for(a=s->autom; a; a=a->link) + if(a->type == D_FILE) + put(nil, a->asym->name, 'z', a->aoffset, 0, 0, 0); + else + if(a->type == D_FILE1) + put(nil, a->asym->name, 'Z', a->aoffset, 0, 0, 0); + + put(s, s->name, 'T', s->value, s->size, s->version, s->gotype); + + /* frame, auto and param after */ + put(nil, ".frame", 'm', s->text->to.offset+4, 0, 0, 0); + + for(a=s->autom; a; a=a->link) + if(a->type == D_AUTO) + put(nil, a->asym->name, 'a', -a->aoffset, 0, 0, a->gotype); + else + if(a->type == D_PARAM) + put(nil, a->asym->name, 'p', a->aoffset, 0, 0, a->gotype); + } + if(debug['v'] || debug['n']) + Bprint(&bso, "symsize = %ud\n", symsize); + Bflush(&bso); +} + +void +setpersrc(Sym *s) +{ + USED(s); +} diff --git a/src/cmd/5l/doc.go b/src/cmd/5l/doc.go new file mode 100644 index 000000000..aa7ccebfc --- /dev/null +++ b/src/cmd/5l/doc.go @@ -0,0 +1,39 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +/* + +5l is a modified version of the Plan 9 linker. The original is documented at + + http://plan9.bell-labs.com/magic/man2html/1/2l + +Its target architecture is the ARM, referred to by these tools as arm. +It reads files in .5 format generated by 5g, 5c, and 5a and emits +a binary called 5.out by default. + +Major changes include: + - support for segmented stacks (this feature is implemented here, not in the compilers). + + +Original options are listed in the link above. + +Options new in this version: + +-F + Force use of software floating point. + Also implied by setting GOARM=5 in the environment. +-Hlinux + Write Linux ELF binaries (default when $GOOS is linux) +-I interpreter + Set the ELF dynamic linker to use. +-L dir1 -L dir2 + Search for libraries (package files) in dir1, dir2, etc. + The default is the single location $GOROOT/pkg/$GOOS_arm. +-r dir1:dir2:... + Set the dynamic linker search path when using ELF. +-V + Print the linker version. + +*/ +package documentation diff --git a/src/cmd/5l/l.h b/src/cmd/5l/l.h new file mode 100644 index 000000000..dabe93d37 --- /dev/null +++ b/src/cmd/5l/l.h @@ -0,0 +1,426 @@ +// Inferno utils/5l/l.h +// http://code.google.com/p/inferno-os/source/browse/utils/5l/l.h +// +// Copyright © 1994-1999 Lucent Technologies Inc. All rights reserved. +// Portions Copyright © 1995-1997 C H Forsyth (forsyth@terzarima.net) +// Portions Copyright © 1997-1999 Vita Nuova Limited +// Portions Copyright © 2000-2007 Vita Nuova Holdings Limited (www.vitanuova.com) +// Portions Copyright © 2004,2006 Bruce Ellis +// Portions Copyright © 2005-2007 C H Forsyth (forsyth@terzarima.net) +// Revisions Copyright © 2000-2007 Lucent Technologies Inc. and others +// Portions Copyright © 2009 The Go Authors. All rights reserved. +// +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to deal +// in the Software without restriction, including without limitation the rights +// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +// copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +// THE SOFTWARE. + +#include +#include +#include +#include "5.out.h" + +enum +{ + thechar = '5', + PtrSize = 4 +}; + +#ifndef EXTERN +#define EXTERN extern +#endif + +/* do not undefine this - code will be removed eventually */ +#define CALLEEBX + +#define dynptrsize 0 + +typedef struct Adr Adr; +typedef struct Sym Sym; +typedef struct Autom Auto; +typedef struct Prog Prog; +typedef struct Reloc Reloc; +typedef struct Optab Optab; +typedef struct Oprang Oprang; +typedef uchar Opcross[32][2][32]; +typedef struct Count Count; + +#define P ((Prog*)0) +#define S ((Sym*)0) +#define TNAME (cursym?cursym->name:noname) + +struct Adr +{ + union + { + int32 u0offset; + char* u0sval; + Ieee u0ieee; + char* u0sbig; + } u0; + Sym* sym; + char type; + uchar index; // not used on arm, required by ld/go.c + char reg; + char name; + int32 offset2; // argsize + char class; + Sym* gotype; +}; + +#define offset u0.u0offset +#define sval u0.u0sval +#define scon sval +#define ieee u0.u0ieee +#define sbig u0.u0sbig + +struct Reloc +{ + int32 off; + uchar siz; + int16 type; + int32 add; + Sym* sym; +}; + +struct Prog +{ + Adr from; + Adr to; + union + { + int32 u0regused; + Prog* u0forwd; + } u0; + Prog* cond; + Prog* link; + Prog* dlink; + int32 pc; + int32 line; + int32 spadj; + uchar mark; + uchar optab; + uchar as; + uchar scond; + uchar reg; + uchar align; +}; + +#define regused u0.u0regused +#define forwd u0.u0forwd +#define datasize reg +#define textflag reg + +#define iscall(p) ((p)->as == ABL) + +struct Sym +{ + char* name; + short type; + short version; + uchar dupok; + uchar reachable; + uchar dynexport; + uchar leaf; + uchar stkcheck; + uchar hide; + int32 dynid; + int32 plt; + int32 got; + int32 value; + int32 sig; + int32 size; + uchar special; + uchar fnptr; // used as fn ptr + Sym* hash; // in hash table + Sym* allsym; // in all symbol list + Sym* next; // in text or data list + Sym* sub; // in SSUB list + Sym* outer; // container of sub + Sym* gotype; + char* file; + char* dynimpname; + char* dynimplib; + char* dynimpvers; + + // STEXT + Auto* autom; + Prog* text; + + // SDATA, SBSS + uchar* p; + int32 np; + int32 maxp; + Reloc* r; + int32 nr; + int32 maxr; +}; + +#define SIGNINTERN (1729*325*1729) + +struct Autom +{ + Sym* asym; + Auto* link; + int32 aoffset; + short type; + Sym* gotype; +}; +struct Optab +{ + char as; + uchar a1; + char a2; + uchar a3; + uchar type; + char size; + char param; + char flag; +}; +struct Oprang +{ + Optab* start; + Optab* stop; +}; +struct Count +{ + int32 count; + int32 outof; +}; + +enum +{ + LFROM = 1<<0, + LTO = 1<<1, + LPOOL = 1<<2, + + C_NONE = 0, + C_REG, + C_REGREG, + C_SHIFT, + C_FREG, + C_PSR, + C_FCR, + + C_RCON, /* 0xff rotated */ + C_NCON, /* ~RCON */ + C_SCON, /* 0xffff */ + C_LCON, + C_ZFCON, + C_SFCON, + C_LFCON, + + C_RACON, + C_LACON, + + C_SBRA, + C_LBRA, + + C_HAUTO, /* halfword insn offset (-0xff to 0xff) */ + C_FAUTO, /* float insn offset (0 to 0x3fc, word aligned) */ + C_HFAUTO, /* both H and F */ + C_SAUTO, /* -0xfff to 0xfff */ + C_LAUTO, + + C_HOREG, + C_FOREG, + C_HFOREG, + C_SOREG, + C_ROREG, + C_SROREG, /* both S and R */ + C_LOREG, + + C_PC, + C_SP, + C_HREG, + + C_ADDR, /* reference to relocatable address */ + + C_GOK, + +/* mark flags */ + FOLL = 1<<0, + LABEL = 1<<1, + LEAF = 1<<2, + + STRINGSZ = 200, + MINSIZ = 64, + NENT = 100, + MAXIO = 8192, + MAXHIST = 20, /* limit of path elements for history symbols */ + MINLC = 4, +}; + +#ifndef COFFCVT + +EXTERN int32 HEADR; /* length of header */ +EXTERN int HEADTYPE; /* type of header */ +EXTERN int32 INITDAT; /* data location */ +EXTERN int32 INITRND; /* data round above text location */ +EXTERN int32 INITTEXT; /* text location */ +EXTERN char* INITENTRY; /* entry point */ +EXTERN int32 autosize; +EXTERN Auto* curauto; +EXTERN Auto* curhist; +EXTERN Prog* curp; +EXTERN Sym* cursym; +EXTERN Sym* datap; +EXTERN int32 elfdatsize; +EXTERN char debug[128]; +EXTERN Sym* etextp; +EXTERN char* noname; +EXTERN Prog* lastp; +EXTERN int32 lcsize; +EXTERN char literal[32]; +EXTERN int nerrors; +EXTERN int32 instoffset; +EXTERN Opcross opcross[8]; +EXTERN Oprang oprange[ALAST]; +EXTERN char* outfile; +EXTERN int32 pc; +EXTERN uchar repop[ALAST]; +EXTERN char* interpreter; +EXTERN char* rpath; +EXTERN uint32 stroffset; +EXTERN int32 symsize; +EXTERN Sym* textp; +EXTERN int32 textsize; +EXTERN int version; +EXTERN char xcmp[C_GOK+1][C_GOK+1]; +EXTERN Prog zprg; +EXTERN int dtype; +EXTERN int armsize; + +extern char* anames[]; +extern Optab optab[]; + +void addpool(Prog*, Adr*); +EXTERN Prog* blitrl; +EXTERN Prog* elitrl; + +void initdiv(void); +EXTERN Prog* prog_div; +EXTERN Prog* prog_divu; +EXTERN Prog* prog_mod; +EXTERN Prog* prog_modu; + +#pragma varargck type "A" int +#pragma varargck type "C" int +#pragma varargck type "D" Adr* +#pragma varargck type "I" uchar* +#pragma varargck type "N" Adr* +#pragma varargck type "P" Prog* +#pragma varargck type "S" char* +#pragma varargck type "Z" char* +#pragma varargck type "i" char* + +int Aconv(Fmt*); +int Cconv(Fmt*); +int Dconv(Fmt*); +int Iconv(Fmt*); +int Nconv(Fmt*); +int Oconv(Fmt*); +int Pconv(Fmt*); +int Sconv(Fmt*); +int aclass(Adr*); +void addhist(int32, int); +Prog* appendp(Prog*); +void asmb(void); +void asmout(Prog*, Optab*, int32*); +int32 atolwhex(char*); +Prog* brloop(Prog*); +void buildop(void); +void buildrep(int, int); +void cflush(void); +int chipzero(Ieee*); +int chipfloat(Ieee*); +int cmp(int, int); +int compound(Prog*); +double cputime(void); +void diag(char*, ...); +void divsig(void); +void dodata(void); +void doprof1(void); +void doprof2(void); +int32 entryvalue(void); +void exchange(Prog*); +void follow(void); +void hputl(int); +int isnop(Prog*); +void listinit(void); +Sym* lookup(char*, int); +void cput(int); +void hput(int32); +void lput(int32); +void lputb(int32); +void lputl(int32); +void* mysbrk(uint32); +void names(void); +void nocache(Prog*); +int ocmp(const void*, const void*); +int32 opirr(int); +Optab* oplook(Prog*); +int32 oprrr(int, int); +int32 olr(int32, int, int, int); +int32 olhr(int32, int, int, int); +int32 olrr(int, int, int, int); +int32 olhrr(int, int, int, int); +int32 osr(int, int, int32, int, int); +int32 oshr(int, int32, int, int); +int32 ofsr(int, int, int32, int, int, Prog*); +int32 osrr(int, int, int, int); +int32 oshrr(int, int, int, int); +int32 omvl(Prog*, Adr*, int); +void patch(void); +void prasm(Prog*); +void prepend(Prog*, Prog*); +Prog* prg(void); +int pseudo(Prog*); +int32 regoff(Adr*); +int relinv(int); +int32 rnd(int32, int32); +void softfloat(void); +void span(void); +void strnput(char*, int); +int32 symaddr(Sym*); +void undef(void); +void wput(int32); +void wputl(ushort w); +void xdefine(char*, int, int32); +void noops(void); +int32 immrot(uint32); +int32 immaddr(int32); +int32 opbra(int, int); +int brextra(Prog*); +int isbranch(Prog*); +void fnptrs(void); +void doelf(void); + +vlong addaddr(Sym *s, Sym *t); +vlong addsize(Sym *s, Sym *t); +vlong addstring(Sym *s, char *str); +vlong adduint16(Sym *s, uint16 v); +vlong adduint32(Sym *s, uint32 v); +vlong adduint64(Sym *s, uint64 v); +vlong adduint8(Sym *s, uint8 v); +vlong adduintxx(Sym *s, uint64 v, int wid); + +/* Native is little-endian */ +#define LPUT(a) lputl(a) +#define WPUT(a) wputl(a) +#define VPUT(a) abort() + +#endif diff --git a/src/cmd/5l/list.c b/src/cmd/5l/list.c new file mode 100644 index 000000000..fa838215b --- /dev/null +++ b/src/cmd/5l/list.c @@ -0,0 +1,487 @@ +// Inferno utils/5l/list.h +// http://code.google.com/p/inferno-os/source/browse/utils/5l/list.c +// +// Copyright © 1994-1999 Lucent Technologies Inc. All rights reserved. +// Portions Copyright © 1995-1997 C H Forsyth (forsyth@terzarima.net) +// Portions Copyright © 1997-1999 Vita Nuova Limited +// Portions Copyright © 2000-2007 Vita Nuova Holdings Limited (www.vitanuova.com) +// Portions Copyright © 2004,2006 Bruce Ellis +// Portions Copyright © 2005-2007 C H Forsyth (forsyth@terzarima.net) +// Revisions Copyright © 2000-2007 Lucent Technologies Inc. and others +// Portions Copyright © 2009 The Go Authors. All rights reserved. +// +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to deal +// in the Software without restriction, including without limitation the rights +// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +// copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +// THE SOFTWARE. + +// Printing. + +#include "l.h" +#include "../ld/lib.h" + +void +listinit(void) +{ + + fmtinstall('A', Aconv); + fmtinstall('C', Cconv); + fmtinstall('D', Dconv); + fmtinstall('P', Pconv); + fmtinstall('S', Sconv); + fmtinstall('N', Nconv); + fmtinstall('O', Oconv); // C_type constants + fmtinstall('I', Iconv); +} + +void +prasm(Prog *p) +{ + print("%P\n", p); +} + +int +Pconv(Fmt *fp) +{ + Prog *p; + int a; + + p = va_arg(fp->args, Prog*); + curp = p; + a = p->as; + switch(a) { + default: + fmtprint(fp, "(%d)", p->line); + if(p->reg == NREG) + fmtprint(fp, " %A%C %D,%D", + a, p->scond, &p->from, &p->to); + else + if(p->from.type != D_FREG) + fmtprint(fp, " %A%C %D,R%d,%D", + a, p->scond, &p->from, p->reg, &p->to); + else + fmtprint(fp, " %A%C %D,F%d,%D", + a, p->scond, &p->from, p->reg, &p->to); + break; + + case ASWPW: + case ASWPBU: + fmtprint(fp, "(%d) %A%C R%d,%D,%D", + p->line, a, p->scond, p->reg, &p->from, &p->to); + break; + + case ADATA: + case AINIT_: + case ADYNT_: + fmtprint(fp, "(%d) %A%C %D/%d,%D", + p->line, a, p->scond, &p->from, p->reg, &p->to); + break; + + case AWORD: + fmtprint(fp, "(%d) WORD %D", p->line, &p->to); + break; + + case ADWORD: + fmtprint(fp, "(%d) DWORD %D %D", p->line, &p->from, &p->to); + break; + } + + if(p->spadj) + fmtprint(fp, " (spadj%+d)", p->spadj); + + return 0; +} + +int +Aconv(Fmt *fp) +{ + char *s; + int a; + + a = va_arg(fp->args, int); + s = "???"; + if(a >= AXXX && a < ALAST) + s = anames[a]; + return fmtstrcpy(fp, s); +} + +char* strcond[16] = +{ + ".EQ", + ".NE", + ".HS", + ".LO", + ".MI", + ".PL", + ".VS", + ".VC", + ".HI", + ".LS", + ".GE", + ".LT", + ".GT", + ".LE", + "", + ".NV" +}; + +int +Cconv(Fmt *fp) +{ + char s[20]; + int c; + + c = va_arg(fp->args, int); + strcpy(s, strcond[c & C_SCOND]); + if(c & C_SBIT) + strcat(s, ".S"); + if(c & C_PBIT) + strcat(s, ".P"); + if(c & C_WBIT) + strcat(s, ".W"); + if(c & C_UBIT) /* ambiguous with FBIT */ + strcat(s, ".U"); + return fmtstrcpy(fp, s); +} + +int +Dconv(Fmt *fp) +{ + char str[STRINGSZ]; + char *op; + Adr *a; + int32 v; + + a = va_arg(fp->args, Adr*); + switch(a->type) { + + default: + snprint(str, sizeof str, "GOK-type(%d)", a->type); + break; + + case D_NONE: + str[0] = 0; + if(a->name != D_NONE || a->reg != NREG || a->sym != S) + snprint(str, sizeof str, "%N(R%d)(NONE)", a, a->reg); + break; + + case D_CONST: + if(a->reg == NREG) + snprint(str, sizeof str, "$%N", a); + else + snprint(str, sizeof str, "$%N(R%d)", a, a->reg); + break; + + case D_CONST2: + snprint(str, sizeof str, "$%d-%d", a->offset, a->offset2); + break; + + case D_SHIFT: + v = a->offset; + op = "<<>>->@>" + (((v>>5) & 3) << 1); + if(v & (1<<4)) + snprint(str, sizeof str, "R%d%c%cR%d", v&15, op[0], op[1], (v>>8)&15); + else + snprint(str, sizeof str, "R%d%c%c%d", v&15, op[0], op[1], (v>>7)&31); + if(a->reg != NREG) + seprint(str+strlen(str), str+sizeof str, "(R%d)", a->reg); + break; + + case D_OCONST: + snprint(str, sizeof str, "$*$%N", a); + if(a->reg != NREG) + snprint(str, sizeof str, "%N(R%d)(CONST)", a, a->reg); + break; + + case D_OREG: + if(a->reg != NREG) + snprint(str, sizeof str, "%N(R%d)", a, a->reg); + else + snprint(str, sizeof str, "%N", a); + break; + + case D_REG: + snprint(str, sizeof str, "R%d", a->reg); + if(a->name != D_NONE || a->sym != S) + snprint(str, sizeof str, "%N(R%d)(REG)", a, a->reg); + break; + + case D_REGREG: + snprint(str, sizeof str, "(R%d,R%d)", a->reg, (int)a->offset); + if(a->name != D_NONE || a->sym != S) + snprint(str, sizeof str, "%N(R%d)(REG)", a, a->reg); + break; + + case D_FREG: + snprint(str, sizeof str, "F%d", a->reg); + if(a->name != D_NONE || a->sym != S) + snprint(str, sizeof str, "%N(R%d)(REG)", a, a->reg); + break; + + case D_PSR: + switch(a->reg) { + case 0: + snprint(str, sizeof str, "CPSR"); + break; + case 1: + snprint(str, sizeof str, "SPSR"); + break; + default: + snprint(str, sizeof str, "PSR%d", a->reg); + break; + } + if(a->name != D_NONE || a->sym != S) + snprint(str, sizeof str, "%N(PSR%d)(REG)", a, a->reg); + break; + + case D_FPCR: + switch(a->reg){ + case 0: + snprint(str, sizeof str, "FPSR"); + break; + case 1: + snprint(str, sizeof str, "FPCR"); + break; + default: + snprint(str, sizeof str, "FCR%d", a->reg); + break; + } + if(a->name != D_NONE || a->sym != S) + snprint(str, sizeof str, "%N(FCR%d)(REG)", a, a->reg); + + break; + + case D_BRANCH: /* botch */ + if(curp->cond != P) { + v = curp->cond->pc; + if(a->sym != S) + snprint(str, sizeof str, "%s+%.5ux(BRANCH)", a->sym->name, v); + else + snprint(str, sizeof str, "%.5ux(BRANCH)", v); + } else + if(a->sym != S) + snprint(str, sizeof str, "%s+%d(APC)", a->sym->name, a->offset); + else + snprint(str, sizeof str, "%d(APC)", a->offset); + break; + + case D_FCONST: + snprint(str, sizeof str, "$%e", ieeedtod(&a->ieee)); + break; + + case D_SCONST: + snprint(str, sizeof str, "$\"%S\"", a->sval); + break; + } + return fmtstrcpy(fp, str); +} + +int +Nconv(Fmt *fp) +{ + char str[STRINGSZ]; + Adr *a; + Sym *s; + + a = va_arg(fp->args, Adr*); + s = a->sym; + switch(a->name) { + default: + sprint(str, "GOK-name(%d)", a->name); + break; + + case D_NONE: + sprint(str, "%d", a->offset); + break; + + case D_EXTERN: + if(s == S) + sprint(str, "%d(SB)", a->offset); + else + sprint(str, "%s+%d(SB)", s->name, a->offset); + break; + + case D_STATIC: + if(s == S) + sprint(str, "<>+%d(SB)", a->offset); + else + sprint(str, "%s<>+%d(SB)", s->name, a->offset); + break; + + case D_AUTO: + if(s == S) + sprint(str, "%d(SP)", a->offset); + else + sprint(str, "%s-%d(SP)", s->name, -a->offset); + break; + + case D_PARAM: + if(s == S) + sprint(str, "%d(FP)", a->offset); + else + sprint(str, "%s+%d(FP)", s->name, a->offset); + break; + } + return fmtstrcpy(fp, str); +} + +int +Sconv(Fmt *fp) +{ + int i, c; + char str[STRINGSZ], *p, *a; + + a = va_arg(fp->args, char*); + p = str; + for(i=0; i= 'a' && c <= 'z' || + c >= 'A' && c <= 'Z' || + c >= '0' && c <= '9' || + c == ' ' || c == '%') { + *p++ = c; + continue; + } + *p++ = '\\'; + switch(c) { + case 0: + *p++ = 'z'; + continue; + case '\\': + case '"': + *p++ = c; + continue; + case '\n': + *p++ = 'n'; + continue; + case '\t': + *p++ = 't'; + continue; + } + *p++ = (c>>6) + '0'; + *p++ = ((c>>3) & 7) + '0'; + *p++ = (c & 7) + '0'; + } + *p = 0; + return fmtstrcpy(fp, str); +} + +int +Iconv(Fmt *fp) +{ + int i, n; + uint32 *p; + char *s; + Fmt fmt; + + n = fp->prec; + fp->prec = 0; + if(!(fp->flags&FmtPrec) || n < 0) + return fmtstrcpy(fp, "%I"); + fp->flags &= ~FmtPrec; + p = va_arg(fp->args, uint32*); + + // format into temporary buffer and + // call fmtstrcpy to handle padding. + fmtstrinit(&fmt); + for(i=0; i 0) + fmtprint(&fmt, " "); + fmtprint(&fmt, "%.8ux", *p++); + } + s = fmtstrflush(&fmt); + fmtstrcpy(fp, s); + free(s); + return 0; +} + +static char* +cnames[] = +{ + [C_ADDR] = "C_ADDR", + [C_FAUTO] = "C_FAUTO", + [C_ZFCON] = "C_SFCON", + [C_SFCON] = "C_SFCON", + [C_LFCON] = "C_LFCON", + [C_FCR] = "C_FCR", + [C_FOREG] = "C_FOREG", + [C_FREG] = "C_FREG", + [C_GOK] = "C_GOK", + [C_HAUTO] = "C_HAUTO", + [C_HFAUTO] = "C_HFAUTO", + [C_HFOREG] = "C_HFOREG", + [C_HOREG] = "C_HOREG", + [C_HREG] = "C_HREG", + [C_LACON] = "C_LACON", + [C_LAUTO] = "C_LAUTO", + [C_LBRA] = "C_LBRA", + [C_LCON] = "C_LCON", + [C_LOREG] = "C_LOREG", + [C_NCON] = "C_NCON", + [C_NONE] = "C_NONE", + [C_PC] = "C_PC", + [C_PSR] = "C_PSR", + [C_RACON] = "C_RACON", + [C_RCON] = "C_RCON", + [C_REG] = "C_REG", + [C_REGREG] = "C_REGREG", + [C_ROREG] = "C_ROREG", + [C_SAUTO] = "C_SAUTO", + [C_SBRA] = "C_SBRA", + [C_SCON] = "C_SCON", + [C_SHIFT] = "C_SHIFT", + [C_SOREG] = "C_SOREG", + [C_SP] = "C_SP", + [C_SROREG] = "C_SROREG" +}; + +int +Oconv(Fmt *fp) +{ + char buf[500]; + int o; + + o = va_arg(fp->args, int); + if(o < 0 || o >= nelem(cnames) || cnames[o] == nil) { + snprint(buf, sizeof(buf), "C_%d", o); + return fmtstrcpy(fp, buf); + } + return fmtstrcpy(fp, cnames[o]); +} + +void +diag(char *fmt, ...) +{ + char buf[STRINGSZ], *tn, *sep; + va_list arg; + + tn = ""; + sep = ""; + if(cursym != S) { + tn = cursym->name; + sep = ": "; + } + va_start(arg, fmt); + vseprint(buf, buf+sizeof(buf), fmt, arg); + va_end(arg); + print("%s%s%s\n", tn, sep, buf); + + nerrors++; + if(nerrors > 20) { + print("too many errors\n"); + errorexit(); + } +} diff --git a/src/cmd/5l/mkenam b/src/cmd/5l/mkenam new file mode 100644 index 000000000..6cccb0263 --- /dev/null +++ b/src/cmd/5l/mkenam @@ -0,0 +1,45 @@ +# Inferno utils/5c/mkenam +# http://code.google.com/p/inferno-os/source/browse/utils/5c/mkenam +# +# Copyright © 1994-1999 Lucent Technologies Inc. All rights reserved. +# Portions Copyright © 1995-1997 C H Forsyth (forsyth@terzarima.net) +# Portions Copyright © 1997-1999 Vita Nuova Limited +# Portions Copyright © 2000-2007 Vita Nuova Holdings Limited (www.vitanuova.com) +# Portions Copyright © 2004,2006 Bruce Ellis +# Portions Copyright © 2005-2007 C H Forsyth (forsyth@terzarima.net) +# Revisions Copyright © 2000-2007 Lucent Technologies Inc. and others +# Portions Copyright © 2009 The Go Authors. All rights reserved. +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in +# all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +# THE SOFTWARE. + +awk ' +BEGIN { + print "char* anames[] =" + print "{" +} + +/^ A/ { + name=$1 + sub(/,/, "", name) + sub(/^A/, "", name) + print "\t\"" name "\"," +} + +END { print "};" } +' ../5l/5.out.h >enam.c diff --git a/src/cmd/5l/noop.c b/src/cmd/5l/noop.c new file mode 100644 index 000000000..eb44344f4 --- /dev/null +++ b/src/cmd/5l/noop.c @@ -0,0 +1,539 @@ +// Inferno utils/5l/noop.c +// http://code.google.com/p/inferno-os/source/browse/utils/5l/noop.c +// +// Copyright © 1994-1999 Lucent Technologies Inc. All rights reserved. +// Portions Copyright © 1995-1997 C H Forsyth (forsyth@terzarima.net) +// Portions Copyright © 1997-1999 Vita Nuova Limited +// Portions Copyright © 2000-2007 Vita Nuova Holdings Limited (www.vitanuova.com) +// Portions Copyright © 2004,2006 Bruce Ellis +// Portions Copyright © 2005-2007 C H Forsyth (forsyth@terzarima.net) +// Revisions Copyright © 2000-2007 Lucent Technologies Inc. and others +// Portions Copyright © 2009 The Go Authors. All rights reserved. +// +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to deal +// in the Software without restriction, including without limitation the rights +// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +// copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +// THE SOFTWARE. + +// Code transformations. + +#include "l.h" +#include "../ld/lib.h" + +// see ../../runtime/proc.c:/StackGuard +enum +{ + StackBig = 4096, + StackSmall = 128, +}; + +static Sym* sym_div; +static Sym* sym_divu; +static Sym* sym_mod; +static Sym* sym_modu; + +void +noops(void) +{ + Prog *p, *q, *q1; + int o; + Prog *pmorestack; + Sym *symmorestack; + + /* + * find leaf subroutines + * strip NOPs + * expand RET + * expand BECOME pseudo + */ + + if(debug['v']) + Bprint(&bso, "%5.2f noops\n", cputime()); + Bflush(&bso); + + symmorestack = lookup("runtime.morestack", 0); + if(symmorestack->type != STEXT) { + diag("runtime·morestack not defined"); + errorexit(); + } + pmorestack = symmorestack->text; + pmorestack->reg |= NOSPLIT; + + q = P; + for(cursym = textp; cursym != nil; cursym = cursym->next) { + for(p = cursym->text; p != P; p = p->link) { + switch(p->as) { + case ATEXT: + p->mark |= LEAF; + break; + + case ARET: + break; + + case ADIV: + case ADIVU: + case AMOD: + case AMODU: + q = p; + if(prog_div == P) + initdiv(); + cursym->text->mark &= ~LEAF; + continue; + + case ANOP: + q1 = p->link; + q->link = q1; /* q is non-nop */ + if(q1 != P) + q1->mark |= p->mark; + continue; + + case ABL: + case ABX: + cursym->text->mark &= ~LEAF; + + case ABCASE: + case AB: + + case ABEQ: + case ABNE: + case ABCS: + case ABHS: + case ABCC: + case ABLO: + case ABMI: + case ABPL: + case ABVS: + case ABVC: + case ABHI: + case ABLS: + case ABGE: + case ABLT: + case ABGT: + case ABLE: + q1 = p->cond; + if(q1 != P) { + while(q1->as == ANOP) { + q1 = q1->link; + p->cond = q1; + } + } + break; + } + q = p; + } + } + + for(cursym = textp; cursym != nil; cursym = cursym->next) { + for(p = cursym->text; p != P; p = p->link) { + o = p->as; + switch(o) { + case ATEXT: + autosize = p->to.offset + 4; + if(autosize <= 4) + if(cursym->text->mark & LEAF) { + p->to.offset = -4; + autosize = 0; + } + + if(!autosize && !(cursym->text->mark & LEAF)) { + if(debug['v']) + Bprint(&bso, "save suppressed in: %s\n", + cursym->name); + Bflush(&bso); + cursym->text->mark |= LEAF; + } + if(cursym->text->mark & LEAF) { + cursym->leaf = 1; + if(!autosize) + break; + } + + if(p->reg & NOSPLIT) { + q1 = prg(); + q1->as = AMOVW; + q1->scond |= C_WBIT; + q1->line = p->line; + q1->from.type = D_REG; + q1->from.reg = REGLINK; + q1->to.type = D_OREG; + q1->to.offset = -autosize; + q1->to.reg = REGSP; + q1->spadj = autosize; + q1->link = p->link; + p->link = q1; + } else if (autosize < StackBig) { + // split stack check for small functions + // MOVW g_stackguard(g), R1 + // CMP R1, $-autosize(SP) + // MOVW.LO $autosize, R1 + // MOVW.LO $args, R2 + // MOVW.LO R14, R3 + // BL.LO runtime.morestack(SB) // modifies LR + // MOVW.W R14,$-autosize(SP) + + // TODO(kaib): add more trampolines + // TODO(kaib): put stackguard in register + // TODO(kaib): add support for -K and underflow detection + + // MOVW g_stackguard(g), R1 + p = appendp(p); + p->as = AMOVW; + p->from.type = D_OREG; + p->from.reg = REGG; + p->to.type = D_REG; + p->to.reg = 1; + + if(autosize < StackSmall) { + // CMP R1, SP + p = appendp(p); + p->as = ACMP; + p->from.type = D_REG; + p->from.reg = 1; + p->reg = REGSP; + } else { + // MOVW $-autosize(SP), R2 + // CMP R1, R2 + p = appendp(p); + p->as = AMOVW; + p->from.type = D_CONST; + p->from.reg = REGSP; + p->from.offset = -autosize; + p->to.type = D_REG; + p->to.reg = 2; + + p = appendp(p); + p->as = ACMP; + p->from.type = D_REG; + p->from.reg = 1; + p->reg = 2; + } + + // MOVW.LO $autosize, R1 + p = appendp(p); + p->as = AMOVW; + p->scond = C_SCOND_LO; + p->from.type = D_CONST; + /* 160 comes from 3 calls (3*8) 4 safes (4*8) and 104 guard */ + p->from.offset = autosize+160; + p->to.type = D_REG; + p->to.reg = 1; + + // MOVW.LO $args, R2 + p = appendp(p); + p->as = AMOVW; + p->scond = C_SCOND_LO; + p->from.type = D_CONST; + p->from.offset = (cursym->text->to.offset2 + 3) & ~3; + p->to.type = D_REG; + p->to.reg = 2; + + // MOVW.LO R14, R3 + p = appendp(p); + p->as = AMOVW; + p->scond = C_SCOND_LO; + p->from.type = D_REG; + p->from.reg = REGLINK; + p->to.type = D_REG; + p->to.reg = 3; + + // BL.LO runtime.morestack(SB) // modifies LR + p = appendp(p); + p->as = ABL; + p->scond = C_SCOND_LO; + p->to.type = D_BRANCH; + p->to.sym = symmorestack; + p->cond = pmorestack; + + // MOVW.W R14,$-autosize(SP) + p = appendp(p); + p->as = AMOVW; + p->scond |= C_WBIT; + p->from.type = D_REG; + p->from.reg = REGLINK; + p->to.type = D_OREG; + p->to.offset = -autosize; + p->to.reg = REGSP; + p->spadj = autosize; + } else { // > StackBig + // MOVW $autosize, R1 + // MOVW $args, R2 + // MOVW R14, R3 + // BL runtime.morestack(SB) // modifies LR + // MOVW.W R14,$-autosize(SP) + + // MOVW $autosize, R1 + p = appendp(p); + p->as = AMOVW; + p->from.type = D_CONST; + p->from.offset = autosize; + p->to.type = D_REG; + p->to.reg = 1; + + // MOVW $args, R2 + // also need to store the extra 4 bytes. + p = appendp(p); + p->as = AMOVW; + p->from.type = D_CONST; + p->from.offset = (cursym->text->to.offset2 + 3) & ~3; + p->to.type = D_REG; + p->to.reg = 2; + + // MOVW R14, R3 + p = appendp(p); + p->as = AMOVW; + p->from.type = D_REG; + p->from.reg = REGLINK; + p->to.type = D_REG; + p->to.reg = 3; + + // BL runtime.morestack(SB) // modifies LR + p = appendp(p); + p->as = ABL; + p->to.type = D_BRANCH; + p->to.sym = symmorestack; + p->cond = pmorestack; + + // MOVW.W R14,$-autosize(SP) + p = appendp(p); + p->as = AMOVW; + p->scond |= C_WBIT; + p->from.type = D_REG; + p->from.reg = REGLINK; + p->to.type = D_OREG; + p->to.offset = -autosize; + p->to.reg = REGSP; + p->spadj = autosize; + } + break; + + case ARET: + nocache(p); + if(cursym->text->mark & LEAF) { + if(!autosize) { + p->as = AB; + p->from = zprg.from; + p->to.type = D_OREG; + p->to.offset = 0; + p->to.reg = REGLINK; + break; + } + } + p->as = AMOVW; + p->scond |= C_PBIT; + p->from.type = D_OREG; + p->from.offset = autosize; + p->from.reg = REGSP; + p->to.type = D_REG; + p->to.reg = REGPC; + // If there are instructions following + // this ARET, they come from a branch + // with the same stackframe, so no spadj. + break; + + case AADD: + if(p->from.type == D_CONST && p->from.reg == NREG && p->to.type == D_REG && p->to.reg == REGSP) + p->spadj = -p->from.offset; + break; + + case ASUB: + if(p->from.type == D_CONST && p->from.reg == NREG && p->to.type == D_REG && p->to.reg == REGSP) + p->spadj = p->from.offset; + break; + + case ADIV: + case ADIVU: + case AMOD: + case AMODU: + if(debug['M']) + break; + if(p->from.type != D_REG) + break; + if(p->to.type != D_REG) + break; + q1 = p; + + /* MOV a,4(SP) */ + q = prg(); + q->link = p->link; + p->link = q; + p = q; + + p->as = AMOVW; + p->line = q1->line; + p->from.type = D_REG; + p->from.reg = q1->from.reg; + p->to.type = D_OREG; + p->to.reg = REGSP; + p->to.offset = 4; + + /* MOV b,REGTMP */ + q = prg(); + q->link = p->link; + p->link = q; + p = q; + + p->as = AMOVW; + p->line = q1->line; + p->from.type = D_REG; + p->from.reg = q1->reg; + if(q1->reg == NREG) + p->from.reg = q1->to.reg; + p->to.type = D_REG; + p->to.reg = REGTMP; + p->to.offset = 0; + + /* CALL appropriate */ + q = prg(); + q->link = p->link; + p->link = q; + p = q; + + p->as = ABL; + p->line = q1->line; + p->to.type = D_BRANCH; + p->cond = p; + switch(o) { + case ADIV: + p->cond = prog_div; + p->to.sym = sym_div; + break; + case ADIVU: + p->cond = prog_divu; + p->to.sym = sym_divu; + break; + case AMOD: + p->cond = prog_mod; + p->to.sym = sym_mod; + break; + case AMODU: + p->cond = prog_modu; + p->to.sym = sym_modu; + break; + } + + /* MOV REGTMP, b */ + q = prg(); + q->link = p->link; + p->link = q; + p = q; + + p->as = AMOVW; + p->line = q1->line; + p->from.type = D_REG; + p->from.reg = REGTMP; + p->from.offset = 0; + p->to.type = D_REG; + p->to.reg = q1->to.reg; + + /* ADD $8,SP */ + q = prg(); + q->link = p->link; + p->link = q; + p = q; + + p->as = AADD; + p->from.type = D_CONST; + p->from.reg = NREG; + p->from.offset = 8; + p->reg = NREG; + p->to.type = D_REG; + p->to.reg = REGSP; + p->spadj = -8; + + /* SUB $8,SP */ + q1->as = ASUB; + q1->from.type = D_CONST; + q1->from.offset = 8; + q1->from.reg = NREG; + q1->reg = NREG; + q1->to.type = D_REG; + q1->to.reg = REGSP; + q1->spadj = 8; + + break; + case AMOVW: + if((p->scond & C_WBIT) && p->to.type == D_OREG && p->to.reg == REGSP) + p->spadj = -p->to.offset; + if((p->scond & C_PBIT) && p->from.type == D_OREG && p->from.reg == REGSP && p->to.reg != REGPC) + p->spadj = -p->from.offset; + if(p->from.type == D_CONST && p->from.reg == REGSP && p->to.type == D_REG && p->to.reg == REGSP) + p->spadj = -p->from.offset; + break; + } + } + } +} + +static void +sigdiv(char *n) +{ + Sym *s; + + s = lookup(n, 0); + if(s->type == STEXT) + if(s->sig == 0) + s->sig = SIGNINTERN; +} + +void +divsig(void) +{ + sigdiv("_div"); + sigdiv("_divu"); + sigdiv("_mod"); + sigdiv("_modu"); +} + +void +initdiv(void) +{ + Sym *s2, *s3, *s4, *s5; + + if(prog_div != P) + return; + sym_div = s2 = lookup("_div", 0); + sym_divu = s3 = lookup("_divu", 0); + sym_mod = s4 = lookup("_mod", 0); + sym_modu = s5 = lookup("_modu", 0); + prog_div = s2->text; + prog_divu = s3->text; + prog_mod = s4->text; + prog_modu = s5->text; + if(prog_div == P) { + diag("undefined: %s", s2->name); + prog_div = cursym->text; + } + if(prog_divu == P) { + diag("undefined: %s", s3->name); + prog_divu = cursym->text; + } + if(prog_mod == P) { + diag("undefined: %s", s4->name); + prog_mod = cursym->text; + } + if(prog_modu == P) { + diag("undefined: %s", s5->name); + prog_modu = cursym->text; + } +} + +void +nocache(Prog *p) +{ + p->optab = 0; + p->from.class = 0; + p->to.class = 0; +} diff --git a/src/cmd/5l/obj.c b/src/cmd/5l/obj.c new file mode 100644 index 000000000..fc5806aac --- /dev/null +++ b/src/cmd/5l/obj.c @@ -0,0 +1,744 @@ +// Inferno utils/5l/obj.c +// http://code.google.com/p/inferno-os/source/browse/utils/5l/obj.c +// +// Copyright © 1994-1999 Lucent Technologies Inc. All rights reserved. +// Portions Copyright © 1995-1997 C H Forsyth (forsyth@terzarima.net) +// Portions Copyright © 1997-1999 Vita Nuova Limited +// Portions Copyright © 2000-2007 Vita Nuova Holdings Limited (www.vitanuova.com) +// Portions Copyright © 2004,2006 Bruce Ellis +// Portions Copyright © 2005-2007 C H Forsyth (forsyth@terzarima.net) +// Revisions Copyright © 2000-2007 Lucent Technologies Inc. and others +// Portions Copyright © 2009 The Go Authors. All rights reserved. +// +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to deal +// in the Software without restriction, including without limitation the rights +// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +// copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +// THE SOFTWARE. + +// Reading object files. + +#define EXTERN +#include "l.h" +#include "../ld/lib.h" +#include "../ld/elf.h" +#include + +#ifndef DEFAULT +#define DEFAULT '9' +#endif + +char *noname = ""; +char *thestring = "arm"; + +Header headers[] = { + "noheader", Hnoheader, + "risc", Hrisc, + "plan9", Hplan9x32, + "netbsd", Hnetbsd, + "ixp1200", Hixp1200, + "ipaq", Hipaq, + "linux", Hlinux, + 0, 0 +}; + +/* + * -Hrisc -T0x10005000 -R4 is aif for risc os + * -Hplan9 -T4128 -R4096 is plan9 format + * -Hnetbsd -T0xF0000020 -R4 is NetBSD format + * -Hixp1200 is IXP1200 (raw) + * -Hipaq -T0xC0008010 -R1024 is ipaq + * -Hlinux -Tx -Rx is linux elf + */ + +static char* +linkername[] = +{ + "runtime.softfloat", + "math.sqrtGoC", +}; + +void +usage(void) +{ + fprint(2, "usage: 5l [-E entry] [-H head] [-I interpreter] [-L dir] [-T text] [-D data] [-R rnd] [-r path] [-o out] main.5\n"); + errorexit(); +} + +void +main(int argc, char *argv[]) +{ + int c, i; + char *p; + + Binit(&bso, 1, OWRITE); + listinit(); + nerrors = 0; + outfile = "5.out"; + HEADTYPE = -1; + INITTEXT = -1; + INITDAT = -1; + INITRND = -1; + INITENTRY = 0; + + p = getenv("GOARM"); + if(p != nil && strcmp(p, "5") == 0) + debug['F'] = 1; + + ARGBEGIN { + default: + c = ARGC(); + if(c == 'l') + usage(); + if(c >= 0 && c < sizeof(debug)) + debug[c]++; + break; + case 'o': + outfile = EARGF(usage()); + break; + case 'E': + INITENTRY = EARGF(usage()); + break; + case 'I': + interpreter = EARGF(usage()); + break; + case 'L': + Lflag(EARGF(usage())); + break; + case 'T': + INITTEXT = atolwhex(EARGF(usage())); + break; + case 'D': + INITDAT = atolwhex(EARGF(usage())); + break; + case 'R': + INITRND = atolwhex(EARGF(usage())); + break; + case 'r': + rpath = EARGF(usage()); + break; + case 'H': + HEADTYPE = headtype(EARGF(usage())); + /* do something about setting INITTEXT */ + break; + case 'V': + print("%cl version %s\n", thechar, getgoversion()); + errorexit(); + } ARGEND + + USED(argc); + + if(argc != 1) + usage(); + + libinit(); + + if(HEADTYPE == -1) + HEADTYPE = Hlinux; + switch(HEADTYPE) { + default: + diag("unknown -H option"); + errorexit(); + case Hnoheader: /* no header */ + HEADR = 0L; + if(INITTEXT == -1) + INITTEXT = 0; + if(INITDAT == -1) + INITDAT = 0; + if(INITRND == -1) + INITRND = 4; + break; + case Hrisc: /* aif for risc os */ + HEADR = 128L; + if(INITTEXT == -1) + INITTEXT = 0x10005000 + HEADR; + if(INITDAT == -1) + INITDAT = 0; + if(INITRND == -1) + INITRND = 4; + break; + case Hplan9x32: /* plan 9 */ + HEADR = 32L; + if(INITTEXT == -1) + INITTEXT = 4128; + if(INITDAT == -1) + INITDAT = 0; + if(INITRND == -1) + INITRND = 4096; + break; + case Hnetbsd: /* boot for NetBSD */ + HEADR = 32L; + if(INITTEXT == -1) + INITTEXT = 0xF0000020L; + if(INITDAT == -1) + INITDAT = 0; + if(INITRND == -1) + INITRND = 4096; + break; + case Hixp1200: /* boot for IXP1200 */ + HEADR = 0L; + if(INITTEXT == -1) + INITTEXT = 0x0; + if(INITDAT == -1) + INITDAT = 0; + if(INITRND == -1) + INITRND = 4; + break; + case Hipaq: /* boot for ipaq */ + HEADR = 16L; + if(INITTEXT == -1) + INITTEXT = 0xC0008010; + if(INITDAT == -1) + INITDAT = 0; + if(INITRND == -1) + INITRND = 1024; + break; + case Hlinux: /* arm elf */ + debug['d'] = 1; // no dynamic linking + elfinit(); + HEADR = ELFRESERVE; + if(INITTEXT == -1) + INITTEXT = 0x10000 + HEADR; + if(INITDAT == -1) + INITDAT = 0; + if(INITRND == -1) + INITRND = 4096; + break; + } + if(INITDAT != 0 && INITRND != 0) + print("warning: -D0x%ux is ignored because of -R0x%ux\n", + INITDAT, INITRND); + if(debug['v']) + Bprint(&bso, "HEADER = -H0x%d -T0x%ux -D0x%ux -R0x%ux\n", + HEADTYPE, INITTEXT, INITDAT, INITRND); + Bflush(&bso); + zprg.as = AGOK; + zprg.scond = 14; + zprg.reg = NREG; + zprg.from.name = D_NONE; + zprg.from.type = D_NONE; + zprg.from.reg = NREG; + zprg.to = zprg.from; + buildop(); + histgen = 0; + pc = 0; + dtype = 4; + nuxiinit(); + + version = 0; + cbp = buf.cbuf; + cbc = sizeof(buf.cbuf); + + addlibpath("command line", "command line", argv[0], "main"); + loadlib(); + + // mark some functions that are only referenced after linker code editing + // TODO(kaib): this doesn't work, the prog can't be found in runtime + for(i=0; itype = Bgetc(f); + a->reg = Bgetc(f); + c = Bgetc(f); + if(c < 0 || c > NSYM){ + print("sym out of range: %d\n", c); + Bputc(f, ALAST+1); + return; + } + a->sym = h[c]; + a->name = Bgetc(f); + + if((schar)a->reg < 0 || a->reg > NREG) { + print("register out of range %d\n", a->reg); + Bputc(f, ALAST+1); + return; /* force real diagnostic */ + } + + if(a->type == D_CONST || a->type == D_OCONST) { + if(a->name == D_EXTERN || a->name == D_STATIC) { + s = a->sym; + if(s != S && (s->type == STEXT || s->type == SCONST || s->type == SXREF)) { + if(0 && !s->fnptr && s->name[0] != '.') + print("%s used as function pointer\n", s->name); + s->fnptr = 1; // over the top cos of SXREF + } + } + } + + switch(a->type) { + default: + print("unknown type %d\n", a->type); + Bputc(f, ALAST+1); + return; /* force real diagnostic */ + + case D_NONE: + case D_REG: + case D_FREG: + case D_PSR: + case D_FPCR: + break; + + case D_REGREG: + a->offset = Bgetc(f); + break; + + case D_CONST2: + a->offset2 = Bget4(f); // fall through + case D_BRANCH: + case D_OREG: + case D_CONST: + case D_OCONST: + case D_SHIFT: + a->offset = Bget4(f); + break; + + case D_SCONST: + a->sval = mal(NSNAME); + Bread(f, a->sval, NSNAME); + break; + + case D_FCONST: + a->ieee.l = Bget4(f); + a->ieee.h = Bget4(f); + break; + } + s = a->sym; + if(s == S) + return; + i = a->name; + if(i != D_AUTO && i != D_PARAM) + return; + + l = a->offset; + for(u=curauto; u; u=u->link) + if(u->asym == s) + if(u->type == i) { + if(u->aoffset > l) + u->aoffset = l; + return; + } + + u = mal(sizeof(Auto)); + u->link = curauto; + curauto = u; + u->asym = s; + u->aoffset = l; + u->type = i; +} + +void +nopout(Prog *p) +{ + p->as = ANOP; + p->from.type = D_NONE; + p->to.type = D_NONE; +} + +void +ldobj1(Biobuf *f, char *pkg, int64 len, char *pn) +{ + int32 ipc; + Prog *p; + Sym *h[NSYM], *s; + int v, o, r, skip; + uint32 sig; + char *name; + int ntext; + int32 eof; + char src[1024], *x; + Prog *lastp; + + lastp = nil; + ntext = 0; + eof = Boffset(f) + len; + src[0] = 0; + +newloop: + memset(h, 0, sizeof(h)); + version++; + histfrogp = 0; + ipc = pc; + skip = 0; + +loop: + if(f->state == Bracteof || Boffset(f) >= eof) + goto eof; + o = Bgetc(f); + if(o == Beof) + goto eof; + + if(o <= AXXX || o >= ALAST) { + diag("%s:#%lld: opcode out of range: %#ux", pn, Boffset(f), o); + print(" probably not a .5 file\n"); + errorexit(); + } + if(o == ANAME || o == ASIGNAME) { + sig = 0; + if(o == ASIGNAME) + sig = Bget4(f); + v = Bgetc(f); /* type */ + o = Bgetc(f); /* sym */ + r = 0; + if(v == D_STATIC) + r = version; + name = Brdline(f, '\0'); + if(name == nil) { + if(Blinelen(f) > 0) { + fprint(2, "%s: name too long\n", pn); + errorexit(); + } + goto eof; + } + x = expandpkg(name, pkg); + s = lookup(x, r); + if(x != name) + free(x); + + if(sig != 0){ + if(s->sig != 0 && s->sig != sig) + diag("incompatible type signatures %ux(%s) and %ux(%s) for %s", s->sig, s->file, sig, pn, s->name); + s->sig = sig; + s->file = pn; + } + + if(debug['W']) + print(" ANAME %s\n", s->name); + if(o < 0 || o >= nelem(h)) { + fprint(2, "%s: mangled input file\n", pn); + errorexit(); + } + h[o] = s; + if((v == D_EXTERN || v == D_STATIC) && s->type == 0) + s->type = SXREF; + if(v == D_FILE) { + if(s->type != SFILE) { + histgen++; + s->type = SFILE; + s->value = histgen; + } + if(histfrogp < MAXHIST) { + histfrog[histfrogp] = s; + histfrogp++; + } else + collapsefrog(s); + } + goto loop; + } + + p = mal(sizeof(Prog)); + p->as = o; + p->scond = Bgetc(f); + p->reg = Bgetc(f); + p->line = Bget4(f); + + zaddr(f, &p->from, h); + zaddr(f, &p->to, h); + + if(p->as != ATEXT && p->as != AGLOBL && p->reg > NREG) + diag("register out of range %A %d", p->as, p->reg); + + p->link = P; + p->cond = P; + + if(debug['W']) + print("%P\n", p); + + switch(o) { + case AHISTORY: + if(p->to.offset == -1) { + addlib(src, pn); + histfrogp = 0; + goto loop; + } + if(src[0] == '\0') + copyhistfrog(src, sizeof src); + addhist(p->line, D_FILE); /* 'z' */ + if(p->to.offset) + addhist(p->to.offset, D_FILE1); /* 'Z' */ + histfrogp = 0; + goto loop; + + case AEND: + histtoauto(); + if(cursym != nil && cursym->text) + cursym->autom = curauto; + curauto = 0; + cursym = nil; + if(Boffset(f) == eof) + return; + goto newloop; + + case AGLOBL: + s = p->from.sym; + if(s == S) { + diag("GLOBL must have a name\n%P", p); + errorexit(); + } + if(s->type == 0 || s->type == SXREF) { + s->type = SBSS; + s->value = 0; + } + if(s->type != SBSS) { + diag("redefinition: %s\n%P", s->name, p); + s->type = SBSS; + s->value = 0; + } + if(p->to.offset > s->size) + s->size = p->to.offset; + if(p->reg & DUPOK) + s->dupok = 1; + break; + + case ADATA: + // Assume that AGLOBL comes after ADATA. + // If we've seen an AGLOBL that said this sym was DUPOK, + // ignore any more ADATA we see, which must be + // redefinitions. + s = p->from.sym; + if(s->dupok) { + if(debug['v']) + Bprint(&bso, "skipping %s in %s: dupok\n", s->name, pn); + goto loop; + } + if(s->file == nil) + s->file = pn; + else if(s->file != pn) { + diag("multiple initialization for %s: in both %s and %s", s->name, s->file, pn); + errorexit(); + } + savedata(s, p, pn); + unmal(p, sizeof *p); + break; + + case AGOK: + diag("unknown opcode\n%P", p); + p->pc = pc; + pc++; + break; + + case ATEXT: + if(cursym != nil && cursym->text) { + histtoauto(); + cursym->autom = curauto; + curauto = 0; + } + s = p->from.sym; + if(s == S) { + diag("TEXT must have a name\n%P", p); + errorexit(); + } + cursym = s; + if(ntext++ == 0 && s->type != 0 && s->type != SXREF) { + /* redefinition, so file has probably been seen before */ + if(debug['v']) + Bprint(&bso, "skipping: %s: redefinition: %s", pn, s->name); + return; + } + skip = 0; + if(s->type != 0 && s->type != SXREF) { + if(p->reg & DUPOK) { + skip = 1; + goto casedef; + } + diag("redefinition: %s\n%P", s->name, p); + } + if(etextp) + etextp->next = s; + else + textp = s; + etextp = s; + p->align = 4; + autosize = (p->to.offset+3L) & ~3L; + p->to.offset = autosize; + autosize += 4; + s->type = STEXT; + s->text = p; + s->value = pc; + lastp = p; + p->pc = pc; + pc++; + break; + + case ASUB: + if(p->from.type == D_CONST) + if(p->from.name == D_NONE) + if(p->from.offset < 0) { + p->from.offset = -p->from.offset; + p->as = AADD; + } + goto casedef; + + case AADD: + if(p->from.type == D_CONST) + if(p->from.name == D_NONE) + if(p->from.offset < 0) { + p->from.offset = -p->from.offset; + p->as = ASUB; + } + goto casedef; + + case AMOVWD: + case AMOVWF: + case AMOVDW: + case AMOVFW: + case AMOVFD: + case AMOVDF: + // case AMOVF: + // case AMOVD: + case ACMPF: + case ACMPD: + case AADDF: + case AADDD: + case ASUBF: + case ASUBD: + case AMULF: + case AMULD: + case ADIVF: + case ADIVD: + goto casedef; + + case AMOVF: + if(skip) + goto casedef; + + if(p->from.type == D_FCONST && chipfloat(&p->from.ieee) < 0 && + (chipzero(&p->from.ieee) < 0 || (p->scond & C_SCOND) != C_SCOND_NONE)) { + /* size sb 9 max */ + sprint(literal, "$%ux", ieeedtof(&p->from.ieee)); + s = lookup(literal, 0); + if(s->type == 0) { + s->type = SBSS; + adduint32(s, ieeedtof(&p->from.ieee)); + s->reachable = 0; + } + p->from.type = D_OREG; + p->from.sym = s; + p->from.name = D_EXTERN; + p->from.offset = 0; + } + goto casedef; + + case AMOVD: + if(skip) + goto casedef; + + if(p->from.type == D_FCONST && chipfloat(&p->from.ieee) < 0 && + (chipzero(&p->from.ieee) < 0 || (p->scond & C_SCOND) != C_SCOND_NONE)) { + /* size sb 18 max */ + sprint(literal, "$%ux.%ux", + p->from.ieee.l, p->from.ieee.h); + s = lookup(literal, 0); + if(s->type == 0) { + s->type = SBSS; + adduint32(s, p->from.ieee.l); + adduint32(s, p->from.ieee.h); + s->reachable = 0; + } + p->from.type = D_OREG; + p->from.sym = s; + p->from.name = D_EXTERN; + p->from.offset = 0; + } + goto casedef; + + default: + casedef: + if(skip) + nopout(p); + p->pc = pc; + pc++; + if(p->to.type == D_BRANCH) + p->to.offset += ipc; + if(lastp == nil) { + if(p->as != ANOP) + diag("unexpected instruction: %P", p); + break; + } + lastp->link = p; + lastp = p; + break; + } + goto loop; + +eof: + diag("truncated object file: %s", pn); +} + +Prog* +prg(void) +{ + Prog *p; + + p = mal(sizeof(Prog)); + *p = zprg; + return p; +} + +Prog* +appendp(Prog *q) +{ + Prog *p; + + p = prg(); + p->link = q->link; + q->link = p; + p->line = q->line; + return p; +} diff --git a/src/cmd/5l/optab.c b/src/cmd/5l/optab.c new file mode 100644 index 000000000..514786f85 --- /dev/null +++ b/src/cmd/5l/optab.c @@ -0,0 +1,236 @@ +// Inferno utils/5l/optab.c +// http://code.google.com/p/inferno-os/source/browse/utils/5l/optab.c +// +// Copyright © 1994-1999 Lucent Technologies Inc. All rights reserved. +// Portions Copyright © 1995-1997 C H Forsyth (forsyth@terzarima.net) +// Portions Copyright © 1997-1999 Vita Nuova Limited +// Portions Copyright © 2000-2007 Vita Nuova Holdings Limited (www.vitanuova.com) +// Portions Copyright © 2004,2006 Bruce Ellis +// Portions Copyright © 2005-2007 C H Forsyth (forsyth@terzarima.net) +// Revisions Copyright © 2000-2007 Lucent Technologies Inc. and others +// Portions Copyright © 2009 The Go Authors. All rights reserved. +// +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to deal +// in the Software without restriction, including without limitation the rights +// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +// copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +// THE SOFTWARE. + +#include "l.h" + +Optab optab[] = +{ + /* struct Optab: + OPCODE, from, prog->reg, to, type,size,param,flag */ + { ATEXT, C_ADDR, C_NONE, C_LCON, 0, 0, 0 }, + { ATEXT, C_ADDR, C_REG, C_LCON, 0, 0, 0 }, + + { AADD, C_REG, C_REG, C_REG, 1, 4, 0 }, + { AADD, C_REG, C_NONE, C_REG, 1, 4, 0 }, + { AMOVW, C_REG, C_NONE, C_REG, 1, 4, 0 }, + { AMVN, C_REG, C_NONE, C_REG, 1, 4, 0 }, + { ACMP, C_REG, C_REG, C_NONE, 1, 4, 0 }, + + { AADD, C_RCON, C_REG, C_REG, 2, 4, 0 }, + { AADD, C_RCON, C_NONE, C_REG, 2, 4, 0 }, + { AMOVW, C_RCON, C_NONE, C_REG, 2, 4, 0 }, + { AMVN, C_RCON, C_NONE, C_REG, 2, 4, 0 }, + { ACMP, C_RCON, C_REG, C_NONE, 2, 4, 0 }, + + { AADD, C_SHIFT,C_REG, C_REG, 3, 4, 0 }, + { AADD, C_SHIFT,C_NONE, C_REG, 3, 4, 0 }, + { AMVN, C_SHIFT,C_NONE, C_REG, 3, 4, 0 }, + { ACMP, C_SHIFT,C_REG, C_NONE, 3, 4, 0 }, + + { AMOVW, C_RACON,C_NONE, C_REG, 4, 4, REGSP }, + + { AB, C_NONE, C_NONE, C_SBRA, 5, 4, 0, LPOOL }, + { ABL, C_NONE, C_NONE, C_SBRA, 5, 4, 0 }, + { ABX, C_NONE, C_NONE, C_SBRA, 74, 20, 0 }, + { ABEQ, C_NONE, C_NONE, C_SBRA, 5, 4, 0 }, + + { AB, C_NONE, C_NONE, C_ROREG, 6, 4, 0, LPOOL }, + { ABL, C_NONE, C_NONE, C_ROREG, 7, 8, 0 }, + { ABX, C_NONE, C_NONE, C_ROREG, 75, 12, 0 }, + { ABXRET, C_NONE, C_NONE, C_ROREG, 76, 4, 0 }, + + { ASLL, C_RCON, C_REG, C_REG, 8, 4, 0 }, + { ASLL, C_RCON, C_NONE, C_REG, 8, 4, 0 }, + + { ASLL, C_REG, C_NONE, C_REG, 9, 4, 0 }, + { ASLL, C_REG, C_REG, C_REG, 9, 4, 0 }, + + { ASWI, C_NONE, C_NONE, C_NONE, 10, 4, 0 }, + { ASWI, C_NONE, C_NONE, C_LOREG, 10, 4, 0 }, + { ASWI, C_NONE, C_NONE, C_LCON, 10, 4, 0 }, + + { AWORD, C_NONE, C_NONE, C_LCON, 11, 4, 0 }, + { AWORD, C_NONE, C_NONE, C_ADDR, 11, 4, 0 }, + + { AMOVW, C_NCON, C_NONE, C_REG, 12, 4, 0 }, + { AMOVW, C_LCON, C_NONE, C_REG, 12, 4, 0, LFROM }, + + { AADD, C_NCON, C_REG, C_REG, 13, 8, 0 }, + { AADD, C_NCON, C_NONE, C_REG, 13, 8, 0 }, + { AMVN, C_NCON, C_NONE, C_REG, 13, 8, 0 }, + { ACMP, C_NCON, C_REG, C_NONE, 13, 8, 0 }, + { AADD, C_LCON, C_REG, C_REG, 13, 8, 0, LFROM }, + { AADD, C_LCON, C_NONE, C_REG, 13, 8, 0, LFROM }, + { AMVN, C_LCON, C_NONE, C_REG, 13, 8, 0, LFROM }, + { ACMP, C_LCON, C_REG, C_NONE, 13, 8, 0, LFROM }, + + { AMOVB, C_REG, C_NONE, C_REG, 14, 8, 0 }, + { AMOVBU, C_REG, C_NONE, C_REG, 58, 4, 0 }, + { AMOVH, C_REG, C_NONE, C_REG, 14, 8, 0 }, + { AMOVHU, C_REG, C_NONE, C_REG, 14, 8, 0 }, + + { AMUL, C_REG, C_REG, C_REG, 15, 4, 0 }, + { AMUL, C_REG, C_NONE, C_REG, 15, 4, 0 }, + + { ADIV, C_REG, C_REG, C_REG, 16, 4, 0 }, + { ADIV, C_REG, C_NONE, C_REG, 16, 4, 0 }, + + { AMULL, C_REG, C_REG, C_REGREG, 17, 4, 0 }, + + { AMOVW, C_REG, C_NONE, C_SAUTO, 20, 4, REGSP }, + { AMOVW, C_REG, C_NONE, C_SOREG, 20, 4, 0 }, + { AMOVB, C_REG, C_NONE, C_SAUTO, 20, 4, REGSP }, + { AMOVB, C_REG, C_NONE, C_SOREG, 20, 4, 0 }, + { AMOVBU, C_REG, C_NONE, C_SAUTO, 20, 4, REGSP }, + { AMOVBU, C_REG, C_NONE, C_SOREG, 20, 4, 0 }, + + { AMOVW, C_SAUTO,C_NONE, C_REG, 21, 4, REGSP }, + { AMOVW, C_SOREG,C_NONE, C_REG, 21, 4, 0 }, + { AMOVBU, C_SAUTO,C_NONE, C_REG, 21, 4, REGSP }, + { AMOVBU, C_SOREG,C_NONE, C_REG, 21, 4, 0 }, + + { AMOVW, C_REG, C_NONE, C_LAUTO, 30, 8, REGSP, LTO }, + { AMOVW, C_REG, C_NONE, C_LOREG, 30, 8, 0, LTO }, + { AMOVW, C_REG, C_NONE, C_ADDR, 64, 8, 0, LTO }, + { AMOVB, C_REG, C_NONE, C_LAUTO, 30, 8, REGSP, LTO }, + { AMOVB, C_REG, C_NONE, C_LOREG, 30, 8, 0, LTO }, + { AMOVB, C_REG, C_NONE, C_ADDR, 64, 8, 0, LTO }, + { AMOVBU, C_REG, C_NONE, C_LAUTO, 30, 8, REGSP, LTO }, + { AMOVBU, C_REG, C_NONE, C_LOREG, 30, 8, 0, LTO }, + { AMOVBU, C_REG, C_NONE, C_ADDR, 64, 8, 0, LTO }, + + { AMOVW, C_LAUTO,C_NONE, C_REG, 31, 8, REGSP, LFROM }, + { AMOVW, C_LOREG,C_NONE, C_REG, 31, 8, 0, LFROM }, + { AMOVW, C_ADDR, C_NONE, C_REG, 65, 8, 0, LFROM }, + { AMOVBU, C_LAUTO,C_NONE, C_REG, 31, 8, REGSP, LFROM }, + { AMOVBU, C_LOREG,C_NONE, C_REG, 31, 8, 0, LFROM }, + { AMOVBU, C_ADDR, C_NONE, C_REG, 65, 8, 0, LFROM }, + + { AMOVW, C_LACON,C_NONE, C_REG, 34, 8, REGSP, LFROM }, + + { AMOVW, C_PSR, C_NONE, C_REG, 35, 4, 0 }, + { AMOVW, C_REG, C_NONE, C_PSR, 36, 4, 0 }, + { AMOVW, C_RCON, C_NONE, C_PSR, 37, 4, 0 }, + + { AMOVM, C_LCON, C_NONE, C_SOREG, 38, 4, 0 }, + { AMOVM, C_SOREG,C_NONE, C_LCON, 39, 4, 0 }, + + { ASWPW, C_SOREG,C_REG, C_REG, 40, 4, 0 }, + + { ARFE, C_NONE, C_NONE, C_NONE, 41, 4, 0 }, + + { AMOVF, C_FREG, C_NONE, C_FAUTO, 50, 4, REGSP }, + { AMOVF, C_FREG, C_NONE, C_FOREG, 50, 4, 0 }, + + { AMOVF, C_FAUTO,C_NONE, C_FREG, 51, 4, REGSP }, + { AMOVF, C_FOREG,C_NONE, C_FREG, 51, 4, 0 }, + + { AMOVF, C_FREG, C_NONE, C_LAUTO, 52, 12, REGSP, LTO }, + { AMOVF, C_FREG, C_NONE, C_LOREG, 52, 12, 0, LTO }, + + { AMOVF, C_LAUTO,C_NONE, C_FREG, 53, 12, REGSP, LFROM }, + { AMOVF, C_LOREG,C_NONE, C_FREG, 53, 12, 0, LFROM }, + + { AMOVF, C_FREG, C_NONE, C_ADDR, 68, 8, 0, LTO }, + { AMOVF, C_ADDR, C_NONE, C_FREG, 69, 8, 0, LFROM }, + + { AADDF, C_FREG, C_NONE, C_FREG, 54, 4, 0 }, + { AADDF, C_FREG, C_REG, C_FREG, 54, 4, 0 }, + { AMOVF, C_FREG, C_NONE, C_FREG, 54, 4, 0 }, + + { AMOVW, C_REG, C_NONE, C_FCR, 56, 4, 0 }, + { AMOVW, C_FCR, C_NONE, C_REG, 57, 4, 0 }, + + { AMOVW, C_SHIFT,C_NONE, C_REG, 59, 4, 0 }, + { AMOVBU, C_SHIFT,C_NONE, C_REG, 59, 4, 0 }, + + { AMOVB, C_SHIFT,C_NONE, C_REG, 60, 4, 0 }, + + { AMOVW, C_REG, C_NONE, C_SHIFT, 61, 4, 0 }, + { AMOVB, C_REG, C_NONE, C_SHIFT, 61, 4, 0 }, + { AMOVBU, C_REG, C_NONE, C_SHIFT, 61, 4, 0 }, + + { ACASE, C_REG, C_NONE, C_NONE, 62, 4, 0 }, + { ABCASE, C_NONE, C_NONE, C_SBRA, 63, 4, 0 }, + + { AMOVH, C_REG, C_NONE, C_HAUTO, 70, 4, REGSP, 0 }, + { AMOVH, C_REG, C_NONE, C_HOREG, 70, 4, 0, 0 }, + { AMOVHU, C_REG, C_NONE, C_HAUTO, 70, 4, REGSP, 0 }, + { AMOVHU, C_REG, C_NONE, C_HOREG, 70, 4, 0, 0 }, + + { AMOVB, C_HAUTO,C_NONE, C_REG, 71, 4, REGSP, 0 }, + { AMOVB, C_HOREG,C_NONE, C_REG, 71, 4, 0, 0 }, + { AMOVH, C_HAUTO,C_NONE, C_REG, 71, 4, REGSP, 0 }, + { AMOVH, C_HOREG,C_NONE, C_REG, 71, 4, 0, 0 }, + { AMOVHU, C_HAUTO,C_NONE, C_REG, 71, 4, REGSP, 0 }, + { AMOVHU, C_HOREG,C_NONE, C_REG, 71, 4, 0, 0 }, + + { AMOVH, C_REG, C_NONE, C_LAUTO, 72, 8, REGSP, LTO }, + { AMOVH, C_REG, C_NONE, C_LOREG, 72, 8, 0, LTO }, + { AMOVH, C_REG, C_NONE, C_ADDR, 94, 8, 0, LTO }, + { AMOVHU, C_REG, C_NONE, C_LAUTO, 72, 8, REGSP, LTO }, + { AMOVHU, C_REG, C_NONE, C_LOREG, 72, 8, 0, LTO }, + { AMOVHU, C_REG, C_NONE, C_ADDR, 94, 8, 0, LTO }, + + { AMOVB, C_LAUTO,C_NONE, C_REG, 73, 8, REGSP, LFROM }, + { AMOVB, C_LOREG,C_NONE, C_REG, 73, 8, 0, LFROM }, + { AMOVB, C_ADDR, C_NONE, C_REG, 93, 8, 0, LFROM }, + { AMOVH, C_LAUTO,C_NONE, C_REG, 73, 8, REGSP, LFROM }, + { AMOVH, C_LOREG,C_NONE, C_REG, 73, 8, 0, LFROM }, + { AMOVH, C_ADDR, C_NONE, C_REG, 93, 8, 0, LFROM }, + { AMOVHU, C_LAUTO,C_NONE, C_REG, 73, 8, REGSP, LFROM }, + { AMOVHU, C_LOREG,C_NONE, C_REG, 73, 8, 0, LFROM }, + { AMOVHU, C_ADDR, C_NONE, C_REG, 93, 8, 0, LFROM }, + + { ALDREX, C_SOREG,C_NONE, C_REG, 77, 4, 0 }, + { ASTREX, C_SOREG,C_REG, C_REG, 78, 4, 0 }, + + { AMOVF, C_ZFCON,C_NONE, C_FREG, 80, 8, 0 }, + { AMOVF, C_SFCON,C_NONE, C_FREG, 81, 4, 0 }, + + { ACMPF, C_FREG, C_REG, C_NONE, 82, 8, 0 }, + { ACMPF, C_FREG, C_NONE, C_NONE, 83, 8, 0 }, + + { AMOVFW, C_FREG, C_NONE, C_FREG, 84, 4, 0 }, + { AMOVWF, C_FREG, C_NONE, C_FREG, 85, 4, 0 }, + + { AMOVFW, C_FREG, C_NONE, C_REG, 86, 8, 0 }, + { AMOVWF, C_REG, C_NONE, C_FREG, 87, 8, 0 }, + + { AMOVW, C_REG, C_NONE, C_FREG, 88, 4, 0 }, + { AMOVW, C_FREG, C_NONE, C_REG, 89, 4, 0 }, + + { ATST, C_REG, C_NONE, C_NONE, 90, 4, 0 }, + + { ALDREXD, C_SOREG,C_NONE, C_REG, 91, 4, 0 }, + { ASTREXD, C_SOREG,C_REG, C_REG, 92, 4, 0 }, + + { AXXX, C_NONE, C_NONE, C_NONE, 0, 4, 0 }, +}; diff --git a/src/cmd/5l/pass.c b/src/cmd/5l/pass.c new file mode 100644 index 000000000..c43049459 --- /dev/null +++ b/src/cmd/5l/pass.c @@ -0,0 +1,332 @@ +// Inferno utils/5l/pass.c +// http://code.google.com/p/inferno-os/source/browse/utils/5l/pass.c +// +// Copyright © 1994-1999 Lucent Technologies Inc. All rights reserved. +// Portions Copyright © 1995-1997 C H Forsyth (forsyth@terzarima.net) +// Portions Copyright © 1997-1999 Vita Nuova Limited +// Portions Copyright © 2000-2007 Vita Nuova Holdings Limited (www.vitanuova.com) +// Portions Copyright © 2004,2006 Bruce Ellis +// Portions Copyright © 2005-2007 C H Forsyth (forsyth@terzarima.net) +// Revisions Copyright © 2000-2007 Lucent Technologies Inc. and others +// Portions Copyright © 2009 The Go Authors. All rights reserved. +// +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to deal +// in the Software without restriction, including without limitation the rights +// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +// copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +// THE SOFTWARE. + +// Code and data passes. + +#include "l.h" +#include "../ld/lib.h" + +static void xfol(Prog*, Prog**); + +Prog* +brchain(Prog *p) +{ + int i; + + for(i=0; i<20; i++) { + if(p == P || p->as != AB) + return p; + p = p->cond; + } + return P; +} + +int +relinv(int a) +{ + switch(a) { + case ABEQ: return ABNE; + case ABNE: return ABEQ; + case ABCS: return ABCC; + case ABHS: return ABLO; + case ABCC: return ABCS; + case ABLO: return ABHS; + case ABMI: return ABPL; + case ABPL: return ABMI; + case ABVS: return ABVC; + case ABVC: return ABVS; + case ABHI: return ABLS; + case ABLS: return ABHI; + case ABGE: return ABLT; + case ABLT: return ABGE; + case ABGT: return ABLE; + case ABLE: return ABGT; + } + diag("unknown relation: %s", anames[a]); + return a; +} + +void +follow(void) +{ + Prog *firstp, *lastp; + + if(debug['v']) + Bprint(&bso, "%5.2f follow\n", cputime()); + Bflush(&bso); + + for(cursym = textp; cursym != nil; cursym = cursym->next) { + firstp = prg(); + lastp = firstp; + xfol(cursym->text, &lastp); + lastp->link = nil; + cursym->text = firstp->link; + } +} + +static void +xfol(Prog *p, Prog **last) +{ + Prog *q, *r; + int a, i; + +loop: + if(p == P) + return; + a = p->as; + if(a == AB) { + q = p->cond; + if(q != P && q->as != ATEXT) { + p->mark |= FOLL; + p = q; + if(!(p->mark & FOLL)) + goto loop; + } + } + if(p->mark & FOLL) { + for(i=0,q=p; i<4; i++,q=q->link) { + if(q == *last || q == nil) + break; + a = q->as; + if(a == ANOP) { + i--; + continue; + } + if(a == AB || (a == ARET && q->scond == 14) || a == ARFE) + goto copy; + if(q->cond == P || (q->cond->mark&FOLL)) + continue; + if(a != ABEQ && a != ABNE) + continue; + copy: + for(;;) { + r = prg(); + *r = *p; + if(!(r->mark&FOLL)) + print("cant happen 1\n"); + r->mark |= FOLL; + if(p != q) { + p = p->link; + (*last)->link = r; + *last = r; + continue; + } + (*last)->link = r; + *last = r; + if(a == AB || (a == ARET && q->scond == 14) || a == ARFE) + return; + r->as = ABNE; + if(a == ABNE) + r->as = ABEQ; + r->cond = p->link; + r->link = p->cond; + if(!(r->link->mark&FOLL)) + xfol(r->link, last); + if(!(r->cond->mark&FOLL)) + print("cant happen 2\n"); + return; + } + } + a = AB; + q = prg(); + q->as = a; + q->line = p->line; + q->to.type = D_BRANCH; + q->to.offset = p->pc; + q->cond = p; + p = q; + } + p->mark |= FOLL; + (*last)->link = p; + *last = p; + if(a == AB || (a == ARET && p->scond == 14) || a == ARFE){ + return; + } + if(p->cond != P) + if(a != ABL && a != ABX && p->link != P) { + q = brchain(p->link); + if(a != ATEXT && a != ABCASE) + if(q != P && (q->mark&FOLL)) { + p->as = relinv(a); + p->link = p->cond; + p->cond = q; + } + xfol(p->link, last); + q = brchain(p->cond); + if(q == P) + q = p->cond; + if(q->mark&FOLL) { + p->cond = q; + return; + } + p = q; + goto loop; + } + p = p->link; + goto loop; +} + +void +patch(void) +{ + int32 c, vexit; + Prog *p, *q; + Sym *s; + int a; + + if(debug['v']) + Bprint(&bso, "%5.2f patch\n", cputime()); + Bflush(&bso); + mkfwd(); + s = lookup("exit", 0); + vexit = s->value; + for(cursym = textp; cursym != nil; cursym = cursym->next) { + for(p = cursym->text; p != P; p = p->link) { + a = p->as; + if((a == ABL || a == ABX || a == AB || a == ARET) && + p->to.type != D_BRANCH && p->to.sym != S) { + s = p->to.sym; + switch(s->type) { + default: + diag("undefined: %s", s->name); + s->type = STEXT; + s->value = vexit; + continue; // avoid more error messages + case STEXT: + p->to.offset = s->value; + p->to.type = D_BRANCH; + break; + } + } + if(p->to.type != D_BRANCH) + continue; + c = p->to.offset; + for(q = textp->text; q != P;) { + if(c == q->pc) + break; + if(q->forwd != P && c >= q->forwd->pc) + q = q->forwd; + else + q = q->link; + } + if(q == P) { + diag("branch out of range %d\n%P", c, p); + p->to.type = D_NONE; + } + p->cond = q; + } + } + + for(cursym = textp; cursym != nil; cursym = cursym->next) { + for(p = cursym->text; p != P; p = p->link) { + if(p->cond != P) { + p->cond = brloop(p->cond); + if(p->cond != P) + if(p->to.type == D_BRANCH) + p->to.offset = p->cond->pc; + } + } + } +} + +Prog* +brloop(Prog *p) +{ + Prog *q; + int c; + + for(c=0; p!=P;) { + if(p->as != AB) + return p; + q = p->cond; + if(q <= p) { + c++; + if(q == p || c > 5000) + break; + } + p = q; + } + return P; +} + +int32 +atolwhex(char *s) +{ + int32 n; + int f; + + n = 0; + f = 0; + while(*s == ' ' || *s == '\t') + s++; + if(*s == '-' || *s == '+') { + if(*s++ == '-') + f = 1; + while(*s == ' ' || *s == '\t') + s++; + } + if(s[0]=='0' && s[1]){ + if(s[1]=='x' || s[1]=='X'){ + s += 2; + for(;;){ + if(*s >= '0' && *s <= '9') + n = n*16 + *s++ - '0'; + else if(*s >= 'a' && *s <= 'f') + n = n*16 + *s++ - 'a' + 10; + else if(*s >= 'A' && *s <= 'F') + n = n*16 + *s++ - 'A' + 10; + else + break; + } + } else + while(*s >= '0' && *s <= '7') + n = n*8 + *s++ - '0'; + } else + while(*s >= '0' && *s <= '9') + n = n*10 + *s++ - '0'; + if(f) + n = -n; + return n; +} + +int32 +rnd(int32 v, int32 r) +{ + int32 c; + + if(r <= 0) + return v; + v += r - 1; + c = v % r; + if(c < 0) + c += r; + v -= c; + return v; +} diff --git a/src/cmd/5l/prof.c b/src/cmd/5l/prof.c new file mode 100644 index 000000000..225a52435 --- /dev/null +++ b/src/cmd/5l/prof.c @@ -0,0 +1,211 @@ +// Inferno utils/5l/obj.c +// http://code.google.com/p/inferno-os/source/browse/utils/5l/obj.c +// +// Copyright © 1994-1999 Lucent Technologies Inc. All rights reserved. +// Portions Copyright © 1995-1997 C H Forsyth (forsyth@terzarima.net) +// Portions Copyright © 1997-1999 Vita Nuova Limited +// Portions Copyright © 2000-2007 Vita Nuova Holdings Limited (www.vitanuova.com) +// Portions Copyright © 2004,2006 Bruce Ellis +// Portions Copyright © 2005-2007 C H Forsyth (forsyth@terzarima.net) +// Revisions Copyright © 2000-2007 Lucent Technologies Inc. and others +// Portions Copyright © 2009 The Go Authors. All rights reserved. +// +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to deal +// in the Software without restriction, including without limitation the rights +// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +// copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +// THE SOFTWARE. + +// Profiling. + +#include "l.h" +#include "../ld/lib.h" + +void +doprof1(void) +{ +#ifdef NOTDEF // TODO(rsc) + Sym *s; + int32 n; + Prog *p, *q; + + if(debug['v']) + Bprint(&bso, "%5.2f profile 1\n", cputime()); + Bflush(&bso); + s = lookup("__mcount", 0); + n = 1; + for(p = firstp->link; p != P; p = p->link) { + if(p->as == ATEXT) { + q = prg(); + q->line = p->line; + q->link = datap; + datap = q; + q->as = ADATA; + q->from.type = D_OREG; + q->from.name = D_EXTERN; + q->from.offset = n*4; + q->from.sym = s; + q->reg = 4; + q->to = p->from; + q->to.type = D_CONST; + + q = prg(); + q->line = p->line; + q->pc = p->pc; + q->link = p->link; + p->link = q; + p = q; + p->as = AMOVW; + p->from.type = D_OREG; + p->from.name = D_EXTERN; + p->from.sym = s; + p->from.offset = n*4 + 4; + p->to.type = D_REG; + p->to.reg = REGTMP; + + q = prg(); + q->line = p->line; + q->pc = p->pc; + q->link = p->link; + p->link = q; + p = q; + p->as = AADD; + p->from.type = D_CONST; + p->from.offset = 1; + p->to.type = D_REG; + p->to.reg = REGTMP; + + q = prg(); + q->line = p->line; + q->pc = p->pc; + q->link = p->link; + p->link = q; + p = q; + p->as = AMOVW; + p->from.type = D_REG; + p->from.reg = REGTMP; + p->to.type = D_OREG; + p->to.name = D_EXTERN; + p->to.sym = s; + p->to.offset = n*4 + 4; + + n += 2; + continue; + } + } + q = prg(); + q->line = 0; + q->link = datap; + datap = q; + + q->as = ADATA; + q->from.type = D_OREG; + q->from.name = D_EXTERN; + q->from.sym = s; + q->reg = 4; + q->to.type = D_CONST; + q->to.offset = n; + + s->type = SBSS; + s->value = n*4; +#endif +} + +void +doprof2(void) +{ + Sym *s2, *s4; + Prog *p, *q, *ps2, *ps4; + + if(debug['v']) + Bprint(&bso, "%5.2f profile 2\n", cputime()); + Bflush(&bso); + s2 = lookup("_profin", 0); + s4 = lookup("_profout", 0); + if(s2->type != STEXT || s4->type != STEXT) { + diag("_profin/_profout not defined"); + return; + } + ps2 = P; + ps4 = P; + for(cursym = textp; cursym != nil; cursym = cursym->next) { + p = cursym->text; + if(cursym == s2) { + ps2 = p; + p->reg = 1; + } + if(cursym == s4) { + ps4 = p; + p->reg = 1; + } + } + for(cursym = textp; cursym != nil; cursym = cursym->next) + for(p = cursym->text; p != P; p = p->link) { + if(p->as == ATEXT) { + if(p->reg & NOPROF) { + for(;;) { + q = p->link; + if(q == P) + break; + if(q->as == ATEXT) + break; + p = q; + } + continue; + } + + /* + * BL profin, R2 + */ + q = prg(); + q->line = p->line; + q->pc = p->pc; + q->link = p->link; + p->link = q; + p = q; + p->as = ABL; + p->to.type = D_BRANCH; + p->cond = ps2; + p->to.sym = s2; + + continue; + } + if(p->as == ARET) { + /* + * RET + */ + q = prg(); + q->as = ARET; + q->from = p->from; + q->to = p->to; + q->link = p->link; + p->link = q; + + /* + * BL profout + */ + p->as = ABL; + p->from = zprg.from; + p->to = zprg.to; + p->to.type = D_BRANCH; + p->cond = ps4; + p->to.sym = s4; + + p = q; + + continue; + } + } +} diff --git a/src/cmd/5l/softfloat.c b/src/cmd/5l/softfloat.c new file mode 100644 index 000000000..4f799d17e --- /dev/null +++ b/src/cmd/5l/softfloat.c @@ -0,0 +1,89 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +#define EXTERN +#include "l.h" +#include "../ld/lib.h" + +// Software floating point. + +void +softfloat(void) +{ + Prog *p, *next, *psfloat; + Sym *symsfloat; + int wasfloat; + + if(!debug['F']) + return; + + symsfloat = lookup("_sfloat", 0); + psfloat = P; + if(symsfloat->type == STEXT) + psfloat = symsfloat->text; + + for(cursym = textp; cursym != nil; cursym = cursym->next) { + wasfloat = 0; + for(p = cursym->text; p != P; p = p->link) + if(p->cond != P) + p->cond->mark |= LABEL; + for(p = cursym->text; p != P; p = p->link) { + switch(p->as) { + case AMOVW: + if(p->to.type == D_FREG || p->from.type == D_FREG) + goto soft; + goto notsoft; + + case AMOVWD: + case AMOVWF: + case AMOVDW: + case AMOVFW: + case AMOVFD: + case AMOVDF: + case AMOVF: + case AMOVD: + + case ACMPF: + case ACMPD: + case AADDF: + case AADDD: + case ASUBF: + case ASUBD: + case AMULF: + case AMULD: + case ADIVF: + case ADIVD: + case ASQRTF: + case ASQRTD: + goto soft; + + default: + goto notsoft; + + soft: + if (psfloat == P) + diag("floats used with _sfloat not defined"); + if (!wasfloat || (p->mark&LABEL)) { + next = prg(); + *next = *p; + + // BL _sfloat(SB) + *p = zprg; + p->link = next; + p->as = ABL; + p->to.type = D_BRANCH; + p->to.sym = symsfloat; + p->cond = psfloat; + + p = next; + wasfloat = 1; + } + break; + + notsoft: + wasfloat = 0; + } + } + } +} diff --git a/src/cmd/5l/span.c b/src/cmd/5l/span.c new file mode 100644 index 000000000..2e1232a1a --- /dev/null +++ b/src/cmd/5l/span.c @@ -0,0 +1,885 @@ +// Inferno utils/5l/span.c +// http://code.google.com/p/inferno-os/source/browse/utils/5l/span.c +// +// Copyright © 1994-1999 Lucent Technologies Inc. All rights reserved. +// Portions Copyright © 1995-1997 C H Forsyth (forsyth@terzarima.net) +// Portions Copyright © 1997-1999 Vita Nuova Limited +// Portions Copyright © 2000-2007 Vita Nuova Holdings Limited (www.vitanuova.com) +// Portions Copyright © 2004,2006 Bruce Ellis +// Portions Copyright © 2005-2007 C H Forsyth (forsyth@terzarima.net) +// Revisions Copyright © 2000-2007 Lucent Technologies Inc. and others +// Portions Copyright © 2009 The Go Authors. All rights reserved. +// +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to deal +// in the Software without restriction, including without limitation the rights +// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +// copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +// THE SOFTWARE. + +// Instruction layout. + +#include "l.h" +#include "../ld/lib.h" + +static struct { + uint32 start; + uint32 size; + uint32 extra; +} pool; + +int checkpool(Prog*, int); +int flushpool(Prog*, int, int); + +int +isbranch(Prog *p) +{ + int as = p->as; + return (as >= ABEQ && as <= ABLE) || as == AB || as == ABL || as == ABX; +} + +static int +scan(Prog *op, Prog *p, int c) +{ + Prog *q; + + for(q = op->link; q != p && q != P; q = q->link){ + q->pc = c; + c += oplook(q)->size; + nocache(q); + } + return c; +} + +/* size of a case statement including jump table */ +static int32 +casesz(Prog *p) +{ + int jt = 0; + int32 n = 0; + Optab *o; + + for( ; p != P; p = p->link){ + if(p->as == ABCASE) + jt = 1; + else if(jt) + break; + o = oplook(p); + n += o->size; + } + return n; +} + +void +span(void) +{ + Prog *p, *op; + Optab *o; + int m, bflag, i, v; + int32 c, otxt, out[6]; + Section *sect; + uchar *bp; + + if(debug['v']) + Bprint(&bso, "%5.2f span\n", cputime()); + Bflush(&bso); + + bflag = 0; + c = INITTEXT; + otxt = c; + for(cursym = textp; cursym != nil; cursym = cursym->next) { + p = cursym->text; + p->pc = c; + cursym->value = c; + + autosize = p->to.offset + 4; + if(p->from.sym != S) + p->from.sym->value = c; + /* need passes to resolve branches */ + if(c-otxt >= 1L<<17) + bflag = 1; + otxt = c; + + for(op = p, p = p->link; p != P; op = p, p = p->link) { + curp = p; + p->pc = c; + o = oplook(p); + m = o->size; + // must check literal pool here in case p generates many instructions + if(blitrl){ + if(checkpool(op, p->as == ACASE ? casesz(p) : m)) + c = p->pc = scan(op, p, c); + } + if(m == 0) { + diag("zero-width instruction\n%P", p); + continue; + } + switch(o->flag & (LFROM|LTO|LPOOL)) { + case LFROM: + addpool(p, &p->from); + break; + case LTO: + addpool(p, &p->to); + break; + case LPOOL: + if ((p->scond&C_SCOND) == 14) + flushpool(p, 0, 0); + break; + } + if(p->as==AMOVW && p->to.type==D_REG && p->to.reg==REGPC && (p->scond&C_SCOND) == 14) + flushpool(p, 0, 0); + c += m; + } + if(blitrl){ + if(checkpool(op, 0)) + c = scan(op, P, c); + } + cursym->size = c - cursym->value; + } + + /* + * if any procedure is large enough to + * generate a large SBRA branch, then + * generate extra passes putting branches + * around jmps to fix. this is rare. + */ + while(bflag) { + if(debug['v']) + Bprint(&bso, "%5.2f span1\n", cputime()); + bflag = 0; + c = INITTEXT; + for(cursym = textp; cursym != nil; cursym = cursym->next) { + cursym->value = c; + for(p = cursym->text; p != P; p = p->link) { + curp = p; + p->pc = c; + o = oplook(p); +/* very large branches + if(o->type == 6 && p->cond) { + otxt = p->cond->pc - c; + if(otxt < 0) + otxt = -otxt; + if(otxt >= (1L<<17) - 10) { + q = prg(); + q->link = p->link; + p->link = q; + q->as = AB; + q->to.type = D_BRANCH; + q->cond = p->cond; + p->cond = q; + q = prg(); + q->link = p->link; + p->link = q; + q->as = AB; + q->to.type = D_BRANCH; + q->cond = q->link->link; + bflag = 1; + } + } + */ + m = o->size; + if(m == 0) { + if(p->as == ATEXT) { + autosize = p->to.offset + 4; + if(p->from.sym != S) + p->from.sym->value = c; + continue; + } + diag("zero-width instruction\n%P", p); + continue; + } + c += m; + } + cursym->size = c - cursym->value; + } + } + + c = rnd(c, 8); + + /* + * lay out the code. all the pc-relative code references, + * even cross-function, are resolved now; + * only data references need to be relocated. + * with more work we could leave cross-function + * code references to be relocated too, and then + * perhaps we'd be able to parallelize the span loop above. + */ + for(cursym = textp; cursym != nil; cursym = cursym->next) { + p = cursym->text; + autosize = p->to.offset + 4; + symgrow(cursym, cursym->size); + + bp = cursym->p; + for(p = p->link; p != P; p = p->link) { + pc = p->pc; + curp = p; + o = oplook(p); + asmout(p, o, out); + for(i=0; isize/4; i++) { + v = out[i]; + *bp++ = v; + *bp++ = v>>8; + *bp++ = v>>16; + *bp++ = v>>24; + } + } + } + sect = addsection(&segtext, ".text", 05); + sect->vaddr = INITTEXT; + sect->len = c - INITTEXT; +} + +/* + * when the first reference to the literal pool threatens + * to go out of range of a 12-bit PC-relative offset, + * drop the pool now, and branch round it. + * this happens only in extended basic blocks that exceed 4k. + */ +int +checkpool(Prog *p, int sz) +{ + if(pool.size >= 0xffc || immaddr((p->pc+sz+4)+4+pool.size - pool.start+8) == 0) + return flushpool(p, 1, 0); + else if(p->link == P) + return flushpool(p, 2, 0); + return 0; +} + +int +flushpool(Prog *p, int skip, int force) +{ + Prog *q; + + if(blitrl) { + if(skip){ + if(0 && skip==1)print("note: flush literal pool at %ux: len=%ud ref=%ux\n", p->pc+4, pool.size, pool.start); + q = prg(); + q->as = AB; + q->to.type = D_BRANCH; + q->cond = p->link; + q->link = blitrl; + blitrl = q; + } + else if(!force && (p->pc+pool.size-pool.start < 2048)) + return 0; + elitrl->link = p->link; + p->link = blitrl; + blitrl = 0; /* BUG: should refer back to values until out-of-range */ + elitrl = 0; + pool.size = 0; + pool.start = 0; + pool.extra = 0; + return 1; + } + return 0; +} + +void +addpool(Prog *p, Adr *a) +{ + Prog *q, t; + int c; + + c = aclass(a); + + t = zprg; + t.as = AWORD; + + switch(c) { + default: + t.to = *a; + break; + + case C_SROREG: + case C_LOREG: + case C_ROREG: + case C_FOREG: + case C_SOREG: + case C_HOREG: + case C_FAUTO: + case C_SAUTO: + case C_LAUTO: + case C_LACON: + t.to.type = D_CONST; + t.to.offset = instoffset; + break; + } + + for(q = blitrl; q != P; q = q->link) /* could hash on t.t0.offset */ + if(memcmp(&q->to, &t.to, sizeof(t.to)) == 0) { + p->cond = q; + return; + } + + q = prg(); + *q = t; + q->pc = pool.size; + + if(blitrl == P) { + blitrl = q; + pool.start = p->pc; + q->align = 4; + } else + elitrl->link = q; + elitrl = q; + pool.size += 4; + + p->cond = q; +} + +void +xdefine(char *p, int t, int32 v) +{ + Sym *s; + + s = lookup(p, 0); + s->type = t; + s->value = v; + s->reachable = 1; + s->special = 1; +} + +int32 +regoff(Adr *a) +{ + + instoffset = 0; + aclass(a); + return instoffset; +} + +int32 +immrot(uint32 v) +{ + int i; + + for(i=0; i<16; i++) { + if((v & ~0xff) == 0) + return (i<<8) | v | (1<<25); + v = (v<<2) | (v>>30); + } + return 0; +} + +int32 +immaddr(int32 v) +{ + if(v >= 0 && v <= 0xfff) + return (v & 0xfff) | + (1<<24) | /* pre indexing */ + (1<<23); /* pre indexing, up */ + if(v >= -0xfff && v < 0) + return (-v & 0xfff) | + (1<<24); /* pre indexing */ + return 0; +} + +int +immfloat(int32 v) +{ + return (v & 0xC03) == 0; /* offset will fit in floating-point load/store */ +} + +int +immhalf(int32 v) +{ + if(v >= 0 && v <= 0xff) + return v| + (1<<24)| /* pre indexing */ + (1<<23); /* pre indexing, up */ + if(v >= -0xff && v < 0) + return (-v & 0xff)| + (1<<24); /* pre indexing */ + return 0; +} + +int32 +symaddr(Sym *s) +{ + int32 v; + + v = s->value; + switch(s->type) { + default: + diag("unexpected type %d in symaddr(%s)", s->type, s->name); + return 0; + + case STEXT: + case SELFROSECT: + case SRODATA: + case SDATA: + case SBSS: + case SCONST: + break; + } + return v; +} + +int +aclass(Adr *a) +{ + Sym *s; + int t; + + switch(a->type) { + case D_NONE: + return C_NONE; + + case D_REG: + return C_REG; + + case D_REGREG: + return C_REGREG; + + case D_SHIFT: + return C_SHIFT; + + case D_FREG: + return C_FREG; + + case D_FPCR: + return C_FCR; + + case D_OREG: + switch(a->name) { + case D_EXTERN: + case D_STATIC: + if(a->sym == 0 || a->sym->name == 0) { + print("null sym external\n"); + print("%D\n", a); + return C_GOK; + } + instoffset = 0; // s.b. unused but just in case + return C_ADDR; + + case D_AUTO: + instoffset = autosize + a->offset; + t = immaddr(instoffset); + if(t){ + if(immhalf(instoffset)) + return immfloat(t) ? C_HFAUTO : C_HAUTO; + if(immfloat(t)) + return C_FAUTO; + return C_SAUTO; + } + return C_LAUTO; + + case D_PARAM: + instoffset = autosize + a->offset + 4L; + t = immaddr(instoffset); + if(t){ + if(immhalf(instoffset)) + return immfloat(t) ? C_HFAUTO : C_HAUTO; + if(immfloat(t)) + return C_FAUTO; + return C_SAUTO; + } + return C_LAUTO; + case D_NONE: + instoffset = a->offset; + t = immaddr(instoffset); + if(t) { + if(immhalf(instoffset)) /* n.b. that it will also satisfy immrot */ + return immfloat(t) ? C_HFOREG : C_HOREG; + if(immfloat(t)) + return C_FOREG; /* n.b. that it will also satisfy immrot */ + t = immrot(instoffset); + if(t) + return C_SROREG; + if(immhalf(instoffset)) + return C_HOREG; + return C_SOREG; + } + t = immrot(instoffset); + if(t) + return C_ROREG; + return C_LOREG; + } + return C_GOK; + + case D_PSR: + return C_PSR; + + case D_OCONST: + switch(a->name) { + case D_EXTERN: + case D_STATIC: + instoffset = 0; // s.b. unused but just in case + return C_ADDR; + } + return C_GOK; + + case D_FCONST: + if(chipzero(&a->ieee) >= 0) + return C_ZFCON; + if(chipfloat(&a->ieee) >= 0) + return C_SFCON; + return C_LFCON; + + case D_CONST: + case D_CONST2: + switch(a->name) { + + case D_NONE: + instoffset = a->offset; + if(a->reg != NREG) + goto aconsize; + + t = immrot(instoffset); + if(t) + return C_RCON; + t = immrot(~instoffset); + if(t) + return C_NCON; + return C_LCON; + + case D_EXTERN: + case D_STATIC: + s = a->sym; + if(s == S) + break; + instoffset = 0; // s.b. unused but just in case + return C_LCON; + + case D_AUTO: + instoffset = autosize + a->offset; + goto aconsize; + + case D_PARAM: + instoffset = autosize + a->offset + 4L; + aconsize: + t = immrot(instoffset); + if(t) + return C_RACON; + return C_LACON; + } + return C_GOK; + + case D_BRANCH: + return C_SBRA; + } + return C_GOK; +} + +Optab* +oplook(Prog *p) +{ + int a1, a2, a3, r; + char *c1, *c3; + Optab *o, *e; + + a1 = p->optab; + if(a1) + return optab+(a1-1); + a1 = p->from.class; + if(a1 == 0) { + a1 = aclass(&p->from) + 1; + p->from.class = a1; + } + a1--; + a3 = p->to.class; + if(a3 == 0) { + a3 = aclass(&p->to) + 1; + p->to.class = a3; + } + a3--; + a2 = C_NONE; + if(p->reg != NREG) + a2 = C_REG; + r = p->as; + o = oprange[r].start; + if(o == 0) { + a1 = opcross[repop[r]][a1][a2][a3]; + if(a1) { + p->optab = a1+1; + return optab+a1; + } + o = oprange[r].stop; /* just generate an error */ + } + if(debug['O']) { + print("oplook %A %O %O %O\n", + (int)p->as, a1, a2, a3); + print(" %d %d\n", p->from.type, p->to.type); + } + e = oprange[r].stop; + c1 = xcmp[a1]; + c3 = xcmp[a3]; + for(; oa2 == a2) + if(c1[o->a1]) + if(c3[o->a3]) { + p->optab = (o-optab)+1; + return o; + } + diag("illegal combination %A %O %O %O, %d %d", + p->as, a1, a2, a3, p->from.type, p->to.type); + prasm(p); + if(o == 0) + o = optab; + return o; +} + +int +cmp(int a, int b) +{ + + if(a == b) + return 1; + switch(a) { + case C_LCON: + if(b == C_RCON || b == C_NCON) + return 1; + break; + case C_LACON: + if(b == C_RACON) + return 1; + break; + case C_LFCON: + if(b == C_ZFCON || b == C_SFCON) + return 1; + break; + + case C_HFAUTO: + return b == C_HAUTO || b == C_FAUTO; + case C_FAUTO: + case C_HAUTO: + return b == C_HFAUTO; + case C_SAUTO: + return cmp(C_HFAUTO, b); + case C_LAUTO: + return cmp(C_SAUTO, b); + + case C_HFOREG: + return b == C_HOREG || b == C_FOREG; + case C_FOREG: + case C_HOREG: + return b == C_HFOREG; + case C_SROREG: + return cmp(C_SOREG, b) || cmp(C_ROREG, b); + case C_SOREG: + case C_ROREG: + return b == C_SROREG || cmp(C_HFOREG, b); + case C_LOREG: + return cmp(C_SROREG, b); + + case C_LBRA: + if(b == C_SBRA) + return 1; + break; + + case C_HREG: + return cmp(C_SP, b) || cmp(C_PC, b); + + } + return 0; +} + +int +ocmp(const void *a1, const void *a2) +{ + Optab *p1, *p2; + int n; + + p1 = (Optab*)a1; + p2 = (Optab*)a2; + n = p1->as - p2->as; + if(n) + return n; + n = p1->a1 - p2->a1; + if(n) + return n; + n = p1->a2 - p2->a2; + if(n) + return n; + n = p1->a3 - p2->a3; + if(n) + return n; + return 0; +} + +void +buildop(void) +{ + int i, n, r; + + for(i=0; i= 32 || x >= nelem(opcross)) { + diag("assumptions fail in buildrep"); + errorexit(); + } + repop[as] = x; + p = (opcross + x); + s = oprange[as].start; + e = oprange[as].stop; + for(o=e-1; o>=s; o--) { + n = o-optab; + for(a2=0; a2<2; a2++) { + if(a2) { + if(o->a2 == C_NONE) + continue; + } else + if(o->a2 != C_NONE) + continue; + for(a1=0; a1<32; a1++) { + if(!xcmp[a1][o->a1]) + continue; + for(a3=0; a3<32; a3++) + if(xcmp[a3][o->a3]) + (*p)[a1][a2][a3] = n; + } + } + } + oprange[as].start = 0; +} +*/ diff --git a/src/cmd/6a/Makefile b/src/cmd/6a/Makefile new file mode 100644 index 000000000..30180bd24 --- /dev/null +++ b/src/cmd/6a/Makefile @@ -0,0 +1,25 @@ +# Copyright 2009 The Go Authors. All rights reserved. +# Use of this source code is governed by a BSD-style +# license that can be found in the LICENSE file. + +include ../../Make.inc +O:=$(HOST_O) + +TARG=6a + +HFILES=\ + a.h\ + y.tab.h\ + ../6l/6.out.h\ + +OFILES=\ + y.tab.$O\ + lex.$O\ + ../6l/enam.$O\ + +YFILES=\ + a.y\ + +include ../../Make.ccmd + +lex.$O: ../cc/macbody ../cc/lexbody diff --git a/src/cmd/6a/a.h b/src/cmd/6a/a.h new file mode 100644 index 000000000..5c7868070 --- /dev/null +++ b/src/cmd/6a/a.h @@ -0,0 +1,214 @@ +// Inferno utils/6a/a.h +// http://code.google.com/p/inferno-os/source/browse/utils/6a/a.h +// +// Copyright © 1994-1999 Lucent Technologies Inc. All rights reserved. +// Portions Copyright © 1995-1997 C H Forsyth (forsyth@terzarima.net) +// Portions Copyright © 1997-1999 Vita Nuova Limited +// Portions Copyright © 2000-2007 Vita Nuova Holdings Limited (www.vitanuova.com) +// Portions Copyright © 2004,2006 Bruce Ellis +// Portions Copyright © 2005-2007 C H Forsyth (forsyth@terzarima.net) +// Revisions Copyright © 2000-2007 Lucent Technologies Inc. and others +// Portions Copyright © 2009 The Go Authors. All rights reserved. +// +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to deal +// in the Software without restriction, including without limitation the rights +// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +// copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +// THE SOFTWARE. + +#include +#include "../6l/6.out.h" + + +#ifndef EXTERN +#define EXTERN extern +#endif + +#undef getc +#undef ungetc +#undef BUFSIZ + +#define getc ccgetc +#define ungetc ccungetc + +typedef struct Sym Sym; +typedef struct Ref Ref; +typedef struct Gen Gen; +typedef struct Io Io; +typedef struct Hist Hist; +typedef struct Gen2 Gen2; + +#define MAXALIGN 7 +#define FPCHIP 1 +#define NSYMB 500 +#define BUFSIZ 8192 +#define HISTSZ 20 +#ifndef EOF +#define EOF (-1) +#endif +#define IGN (-2) +#define GETC() ((--fi.c < 0)? filbuf(): *fi.p++ & 0xff) +#define NHASH 503 +#define STRINGSZ 200 +#define NMACRO 10 + +struct Sym +{ + Sym* link; + Ref* ref; + char* macro; + vlong value; + ushort type; + char *name; + char sym; +}; +#define S ((Sym*)0) + +struct Ref +{ + int class; +}; + +EXTERN struct +{ + char* p; + int c; +} fi; + +struct Io +{ + Io* link; + char b[BUFSIZ]; + char* p; + short c; + short f; +}; +#define I ((Io*)0) + +EXTERN struct +{ + Sym* sym; + short type; +} h[NSYM]; + +struct Gen +{ + double dval; + char sval[8]; + vlong offset; + Sym* sym; + short type; + short index; + short scale; +}; +struct Gen2 +{ + Gen from; + Gen to; +}; + +struct Hist +{ + Hist* link; + char* name; + int32 line; + vlong offset; +}; +#define H ((Hist*)0) + +enum +{ + CLAST, + CMACARG, + CMACRO, + CPREPROC, +}; + + +EXTERN char debug[256]; +EXTERN Sym* hash[NHASH]; +EXTERN char** Dlist; +EXTERN int nDlist; +EXTERN Hist* ehist; +EXTERN int newflag; +EXTERN Hist* hist; +EXTERN char* hunk; +EXTERN char** include; +EXTERN Io* iofree; +EXTERN Io* ionext; +EXTERN Io* iostack; +EXTERN int32 lineno; +EXTERN int nerrors; +EXTERN int32 nhunk; +EXTERN int ninclude; +EXTERN int32 nsymb; +EXTERN Gen nullgen; +EXTERN char* outfile; +EXTERN int pass; +EXTERN char* pathname; +EXTERN int32 pc; +EXTERN int peekc; +EXTERN int32 stmtline; +EXTERN int sym; +EXTERN char* symb; +EXTERN int thechar; +EXTERN char* thestring; +EXTERN int32 thunk; +EXTERN Biobuf obuf; + +void* alloc(int32); +void* allocn(void*, int32, int32); +void ensuresymb(int32); +void errorexit(void); +void pushio(void); +void newio(void); +void newfile(char*, int); +Sym* slookup(char*); +Sym* lookup(void); +void syminit(Sym*); +int32 yylex(void); +int getc(void); +int getnsc(void); +void unget(int); +int escchar(int); +void cinit(void); +void checkscale(int); +void pinit(char*); +void cclean(void); +int isreg(Gen*); +void outcode(int, Gen2*); +void outhist(void); +void zaddr(Gen*, int); +void zname(char*, int, int); +void ieeedtod(Ieee*, double); +int filbuf(void); +Sym* getsym(void); +void domacro(void); +void macund(void); +void macdef(void); +void macexpand(Sym*, char*); +void macinc(void); +void macprag(void); +void maclin(void); +void macif(int); +void macend(void); +void dodefine(char*); +void prfile(int32); +void linehist(char*, int); +void gethunk(void); +void yyerror(char*, ...); +int yyparse(void); +void setinclude(char*); +int assemble(char*); diff --git a/src/cmd/6a/a.y b/src/cmd/6a/a.y new file mode 100644 index 000000000..c0fa4106e --- /dev/null +++ b/src/cmd/6a/a.y @@ -0,0 +1,647 @@ +// Inferno utils/6a/a.y +// http://code.google.com/p/inferno-os/source/browse/utils/6a/a.y +// +// Copyright © 1994-1999 Lucent Technologies Inc. All rights reserved. +// Portions Copyright © 1995-1997 C H Forsyth (forsyth@terzarima.net) +// Portions Copyright © 1997-1999 Vita Nuova Limited +// Portions Copyright © 2000-2007 Vita Nuova Holdings Limited (www.vitanuova.com) +// Portions Copyright © 2004,2006 Bruce Ellis +// Portions Copyright © 2005-2007 C H Forsyth (forsyth@terzarima.net) +// Revisions Copyright © 2000-2007 Lucent Technologies Inc. and others +// Portions Copyright © 2009 The Go Authors. All rights reserved. +// +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to deal +// in the Software without restriction, including without limitation the rights +// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +// copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +// THE SOFTWARE. + +%{ +#include +#include /* if we don't, bison will, and a.h re-#defines getc */ +#include +#include "a.h" +%} +%union { + Sym *sym; + vlong lval; + double dval; + char sval[8]; + Gen gen; + Gen2 gen2; +} +%left '|' +%left '^' +%left '&' +%left '<' '>' +%left '+' '-' +%left '*' '/' '%' +%token LTYPE0 LTYPE1 LTYPE2 LTYPE3 LTYPE4 +%token LTYPEC LTYPED LTYPEN LTYPER LTYPET LTYPEG +%token LTYPES LTYPEM LTYPEI LTYPEXC LTYPEX LTYPERT +%token LCONST LFP LPC LSB +%token LBREG LLREG LSREG LFREG LMREG LXREG +%token LFCONST +%token LSCONST LSP +%token LNAME LLAB LVAR +%type con con2 expr pointer offset +%type mem imm imm2 reg nam rel rem rim rom omem nmem +%type nonnon nonrel nonrem rimnon rimrem remrim spec10 spec11 +%type spec1 spec2 spec3 spec4 spec5 spec6 spec7 spec8 spec9 +%% +prog: +| prog + { + stmtline = lineno; + } + line + +line: + LLAB ':' + { + if($1->value != pc) + yyerror("redeclaration of %s", $1->name); + $1->value = pc; + } + line +| LNAME ':' + { + $1->type = LLAB; + $1->value = pc; + } + line +| ';' +| inst ';' +| error ';' + +inst: + LNAME '=' expr + { + $1->type = LVAR; + $1->value = $3; + } +| LVAR '=' expr + { + if($1->value != $3) + yyerror("redeclaration of %s", $1->name); + $1->value = $3; + } +| LTYPE0 nonnon { outcode($1, &$2); } +| LTYPE1 nonrem { outcode($1, &$2); } +| LTYPE2 rimnon { outcode($1, &$2); } +| LTYPE3 rimrem { outcode($1, &$2); } +| LTYPE4 remrim { outcode($1, &$2); } +| LTYPER nonrel { outcode($1, &$2); } +| LTYPED spec1 { outcode($1, &$2); } +| LTYPET spec2 { outcode($1, &$2); } +| LTYPEC spec3 { outcode($1, &$2); } +| LTYPEN spec4 { outcode($1, &$2); } +| LTYPES spec5 { outcode($1, &$2); } +| LTYPEM spec6 { outcode($1, &$2); } +| LTYPEI spec7 { outcode($1, &$2); } +| LTYPEXC spec8 { outcode($1, &$2); } +| LTYPEX spec9 { outcode($1, &$2); } +| LTYPERT spec10 { outcode($1, &$2); } +| LTYPEG spec11 { outcode($1, &$2); } + +nonnon: + { + $$.from = nullgen; + $$.to = nullgen; + } +| ',' + { + $$.from = nullgen; + $$.to = nullgen; + } + +rimrem: + rim ',' rem + { + $$.from = $1; + $$.to = $3; + } + +remrim: + rem ',' rim + { + $$.from = $1; + $$.to = $3; + } + +rimnon: + rim ',' + { + $$.from = $1; + $$.to = nullgen; + } +| rim + { + $$.from = $1; + $$.to = nullgen; + } + +nonrem: + ',' rem + { + $$.from = nullgen; + $$.to = $2; + } +| rem + { + $$.from = nullgen; + $$.to = $1; + } + +nonrel: + ',' rel + { + $$.from = nullgen; + $$.to = $2; + } +| rel + { + $$.from = nullgen; + $$.to = $1; + } + +spec1: /* DATA */ + nam '/' con ',' imm + { + $$.from = $1; + $$.from.scale = $3; + $$.to = $5; + } + +spec2: /* TEXT */ + mem ',' imm2 + { + $$.from = $1; + $$.to = $3; + } +| mem ',' con ',' imm2 + { + $$.from = $1; + $$.from.scale = $3; + $$.to = $5; + } + +spec3: /* JMP/CALL */ + ',' rom + { + $$.from = nullgen; + $$.to = $2; + } +| rom + { + $$.from = nullgen; + $$.to = $1; + } + +spec4: /* NOP */ + nonnon +| nonrem + +spec5: /* SHL/SHR */ + rim ',' rem + { + $$.from = $1; + $$.to = $3; + } +| rim ',' rem ':' LLREG + { + $$.from = $1; + $$.to = $3; + if($$.from.index != D_NONE) + yyerror("dp shift with lhs index"); + $$.from.index = $5; + } + +spec6: /* MOVW/MOVL */ + rim ',' rem + { + $$.from = $1; + $$.to = $3; + } +| rim ',' rem ':' LSREG + { + $$.from = $1; + $$.to = $3; + if($$.to.index != D_NONE) + yyerror("dp move with lhs index"); + $$.to.index = $5; + } + +spec7: + rim ',' + { + $$.from = $1; + $$.to = nullgen; + } +| rim + { + $$.from = $1; + $$.to = nullgen; + } +| rim ',' rem + { + $$.from = $1; + $$.to = $3; + } + +spec8: /* CMPPS/CMPPD */ + reg ',' rem ',' con + { + $$.from = $1; + $$.to = $3; + $$.to.offset = $5; + } + +spec9: /* shufl */ + imm ',' rem ',' reg + { + $$.from = $3; + $$.to = $5; + if($1.type != D_CONST) + yyerror("illegal constant"); + $$.to.offset = $1.offset; + } + +spec10: /* RET/RETF */ + { + $$.from = nullgen; + $$.to = nullgen; + } +| imm + { + $$.from = $1; + $$.to = nullgen; + } + +spec11: /* GLOBL */ + mem ',' imm + { + $$.from = $1; + $$.to = $3; + } +| mem ',' con ',' imm + { + $$.from = $1; + $$.from.scale = $3; + $$.to = $5; + } + +rem: + reg +| mem + +rom: + rel +| nmem +| '*' reg + { + $$ = $2; + } +| '*' omem + { + $$ = $2; + } +| reg +| omem + +rim: + rem +| imm + +rel: + con '(' LPC ')' + { + $$ = nullgen; + $$.type = D_BRANCH; + $$.offset = $1 + pc; + } +| LNAME offset + { + $$ = nullgen; + if(pass == 2) + yyerror("undefined label: %s", $1->name); + $$.type = D_BRANCH; + $$.sym = $1; + $$.offset = $2; + } +| LLAB offset + { + $$ = nullgen; + $$.type = D_BRANCH; + $$.sym = $1; + $$.offset = $1->value + $2; + } + +reg: + LBREG + { + $$ = nullgen; + $$.type = $1; + } +| LFREG + { + $$ = nullgen; + $$.type = $1; + } +| LLREG + { + $$ = nullgen; + $$.type = $1; + } +| LMREG + { + $$ = nullgen; + $$.type = $1; + } +| LSP + { + $$ = nullgen; + $$.type = D_SP; + } +| LSREG + { + $$ = nullgen; + $$.type = $1; + } +| LXREG + { + $$ = nullgen; + $$.type = $1; + } +imm2: + '$' con2 + { + $$ = nullgen; + $$.type = D_CONST; + $$.offset = $2; + } + +imm: + '$' con + { + $$ = nullgen; + $$.type = D_CONST; + $$.offset = $2; + } +| '$' nam + { + $$ = $2; + $$.index = $2.type; + $$.type = D_ADDR; + /* + if($2.type == D_AUTO || $2.type == D_PARAM) + yyerror("constant cannot be automatic: %s", + $2.sym->name); + */ + } +| '$' LSCONST + { + $$ = nullgen; + $$.type = D_SCONST; + memcpy($$.sval, $2, sizeof($$.sval)); + } +| '$' LFCONST + { + $$ = nullgen; + $$.type = D_FCONST; + $$.dval = $2; + } +| '$' '(' LFCONST ')' + { + $$ = nullgen; + $$.type = D_FCONST; + $$.dval = $3; + } +| '$' '-' LFCONST + { + $$ = nullgen; + $$.type = D_FCONST; + $$.dval = -$3; + } + +mem: + omem +| nmem + +omem: + con + { + $$ = nullgen; + $$.type = D_INDIR+D_NONE; + $$.offset = $1; + } +| con '(' LLREG ')' + { + $$ = nullgen; + $$.type = D_INDIR+$3; + $$.offset = $1; + } +| con '(' LSP ')' + { + $$ = nullgen; + $$.type = D_INDIR+D_SP; + $$.offset = $1; + } +| con '(' LSREG ')' + { + $$ = nullgen; + $$.type = D_INDIR+$3; + $$.offset = $1; + } +| con '(' LLREG '*' con ')' + { + $$ = nullgen; + $$.type = D_INDIR+D_NONE; + $$.offset = $1; + $$.index = $3; + $$.scale = $5; + checkscale($$.scale); + } +| con '(' LLREG ')' '(' LLREG '*' con ')' + { + $$ = nullgen; + $$.type = D_INDIR+$3; + $$.offset = $1; + $$.index = $6; + $$.scale = $8; + checkscale($$.scale); + } +| '(' LLREG ')' + { + $$ = nullgen; + $$.type = D_INDIR+$2; + } +| '(' LSP ')' + { + $$ = nullgen; + $$.type = D_INDIR+D_SP; + } +| '(' LLREG '*' con ')' + { + $$ = nullgen; + $$.type = D_INDIR+D_NONE; + $$.index = $2; + $$.scale = $4; + checkscale($$.scale); + } +| '(' LLREG ')' '(' LLREG '*' con ')' + { + $$ = nullgen; + $$.type = D_INDIR+$2; + $$.index = $5; + $$.scale = $7; + checkscale($$.scale); + } + +nmem: + nam + { + $$ = $1; + } +| nam '(' LLREG '*' con ')' + { + $$ = $1; + $$.index = $3; + $$.scale = $5; + checkscale($$.scale); + } + +nam: + LNAME offset '(' pointer ')' + { + $$ = nullgen; + $$.type = $4; + $$.sym = $1; + $$.offset = $2; + } +| LNAME '<' '>' offset '(' LSB ')' + { + $$ = nullgen; + $$.type = D_STATIC; + $$.sym = $1; + $$.offset = $4; + } + +offset: + { + $$ = 0; + } +| '+' con + { + $$ = $2; + } +| '-' con + { + $$ = -$2; + } + +pointer: + LSB +| LSP + { + $$ = D_AUTO; + } +| LFP + +con: + LCONST +| LVAR + { + $$ = $1->value; + } +| '-' con + { + $$ = -$2; + } +| '+' con + { + $$ = $2; + } +| '~' con + { + $$ = ~$2; + } +| '(' expr ')' + { + $$ = $2; + } + +con2: + LCONST + { + $$ = $1 & 0xffffffffLL; + } +| '-' LCONST + { + $$ = -$2 & 0xffffffffLL; + } +| LCONST '-' LCONST + { + $$ = ($1 & 0xffffffffLL) + + (($3 & 0xffffLL) << 32); + } +| '-' LCONST '-' LCONST + { + $$ = (-$2 & 0xffffffffLL) + + (($4 & 0xffffLL) << 32); + } + +expr: + con +| expr '+' expr + { + $$ = $1 + $3; + } +| expr '-' expr + { + $$ = $1 - $3; + } +| expr '*' expr + { + $$ = $1 * $3; + } +| expr '/' expr + { + $$ = $1 / $3; + } +| expr '%' expr + { + $$ = $1 % $3; + } +| expr '<' '<' expr + { + $$ = $1 << $4; + } +| expr '>' '>' expr + { + $$ = $1 >> $4; + } +| expr '&' expr + { + $$ = $1 & $3; + } +| expr '^' expr + { + $$ = $1 ^ $3; + } +| expr '|' expr + { + $$ = $1 | $3; + } diff --git a/src/cmd/6a/doc.go b/src/cmd/6a/doc.go new file mode 100644 index 000000000..92fb74de6 --- /dev/null +++ b/src/cmd/6a/doc.go @@ -0,0 +1,14 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +/* + +6a is a version of the Plan 9 assembler. The original is documented at + + http://plan9.bell-labs.com/magic/man2html/1/2a + +Its target architecture is the x86-64, referred to by these tools as amd64. + +*/ +package documentation diff --git a/src/cmd/6a/lex.c b/src/cmd/6a/lex.c new file mode 100644 index 000000000..42f4b1d11 --- /dev/null +++ b/src/cmd/6a/lex.c @@ -0,0 +1,1326 @@ +// Inferno utils/6a/lex.c +// http://code.google.com/p/inferno-os/source/browse/utils/6a/lex.c +// +// Copyright © 1994-1999 Lucent Technologies Inc. All rights reserved. +// Portions Copyright © 1995-1997 C H Forsyth (forsyth@terzarima.net) +// Portions Copyright © 1997-1999 Vita Nuova Limited +// Portions Copyright © 2000-2007 Vita Nuova Holdings Limited (www.vitanuova.com) +// Portions Copyright © 2004,2006 Bruce Ellis +// Portions Copyright © 2005-2007 C H Forsyth (forsyth@terzarima.net) +// Revisions Copyright © 2000-2007 Lucent Technologies Inc. and others +// Portions Copyright © 2009 The Go Authors. All rights reserved. +// +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to deal +// in the Software without restriction, including without limitation the rights +// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +// copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +// THE SOFTWARE. + +#define EXTERN +#include +#include +#include "a.h" +#include "y.tab.h" + +enum +{ + Plan9 = 1<<0, + Unix = 1<<1, + Windows = 1<<2, +}; + +int +systemtype(int sys) +{ + return sys&Plan9; +} + +int +pathchar(void) +{ + return '/'; +} + +void +main(int argc, char *argv[]) +{ + char *p; + int c; + + thechar = '6'; + thestring = "amd64"; + + ensuresymb(NSYMB); + memset(debug, 0, sizeof(debug)); + cinit(); + outfile = 0; + setinclude("."); + ARGBEGIN { + default: + c = ARGC(); + if(c >= 0 || c < sizeof(debug)) + debug[c] = 1; + break; + + case 'o': + outfile = ARGF(); + break; + + case 'D': + p = ARGF(); + if(p) { + if (nDlist%8 == 0) + Dlist = allocn(Dlist, nDlist*sizeof(char *), + 8*sizeof(char *)); + Dlist[nDlist++] = p; + } + break; + + case 'I': + p = ARGF(); + setinclude(p); + break; + } ARGEND + if(*argv == 0) { + print("usage: %ca [-options] file.s\n", thechar); + errorexit(); + } + if(argc > 1){ + print("can't assemble multiple files\n"); + errorexit(); + } + if(assemble(argv[0])) + errorexit(); + exits(0); +} + +int +assemble(char *file) +{ + char *ofile, *p; + int i, of; + + ofile = alloc(strlen(file)+3); // +3 for .x\0 (x=thechar) + strcpy(ofile, file); + p = utfrrune(ofile, pathchar()); + if(p) { + include[0] = ofile; + *p++ = 0; + } else + p = ofile; + if(outfile == 0) { + outfile = p; + if(outfile){ + p = utfrrune(outfile, '.'); + if(p) + if(p[1] == 's' && p[2] == 0) + p[0] = 0; + p = utfrune(outfile, 0); + p[0] = '.'; + p[1] = thechar; + p[2] = 0; + } else + outfile = "/dev/null"; + } + + of = create(outfile, OWRITE, 0664); + if(of < 0) { + yyerror("%ca: cannot create %s", thechar, outfile); + errorexit(); + } + Binit(&obuf, of, OWRITE); + + pass = 1; + pinit(file); + + Bprint(&obuf, "go object %s %s %s\n", getgoos(), thestring, getgoversion()); + + for(i=0; itype != LNAME) + yyerror("double initialization %s", itab[i].name); + s->type = itab[i].type; + s->value = itab[i].value; + } + + pathname = allocn(pathname, 0, 100); + if(getwd(pathname, 99) == 0) { + pathname = allocn(pathname, 100, 900); + if(getwd(pathname, 999) == 0) + strcpy(pathname, "/???"); + } +} + +void +checkscale(int scale) +{ + + switch(scale) { + case 1: + case 2: + case 4: + case 8: + return; + } + yyerror("scale must be 1248: %d", scale); +} + +void +syminit(Sym *s) +{ + + s->type = LNAME; + s->value = 0; +} + +void +cclean(void) +{ + Gen2 g2; + + g2.from = nullgen; + g2.to = nullgen; + outcode(AEND, &g2); + Bflush(&obuf); +} + +void +zname(char *n, int t, int s) +{ + + Bputc(&obuf, ANAME); /* as(2) */ + Bputc(&obuf, ANAME>>8); + Bputc(&obuf, t); /* type */ + Bputc(&obuf, s); /* sym */ + while(*n) { + Bputc(&obuf, *n); + n++; + } + Bputc(&obuf, 0); +} + +void +zaddr(Gen *a, int s) +{ + int32 l; + int i, t; + char *n; + Ieee e; + + t = 0; + if(a->index != D_NONE || a->scale != 0) + t |= T_INDEX; + if(a->offset != 0) { + t |= T_OFFSET; + l = a->offset; + if((vlong)l != a->offset) + t |= T_64; + } + if(s != 0) + t |= T_SYM; + + switch(a->type) { + default: + t |= T_TYPE; + break; + case D_FCONST: + t |= T_FCONST; + break; + case D_SCONST: + t |= T_SCONST; + break; + case D_NONE: + break; + } + Bputc(&obuf, t); + + if(t & T_INDEX) { /* implies index, scale */ + Bputc(&obuf, a->index); + Bputc(&obuf, a->scale); + } + if(t & T_OFFSET) { /* implies offset */ + l = a->offset; + Bputc(&obuf, l); + Bputc(&obuf, l>>8); + Bputc(&obuf, l>>16); + Bputc(&obuf, l>>24); + if(t & T_64) { + l = a->offset>>32; + Bputc(&obuf, l); + Bputc(&obuf, l>>8); + Bputc(&obuf, l>>16); + Bputc(&obuf, l>>24); + } + } + if(t & T_SYM) /* implies sym */ + Bputc(&obuf, s); + if(t & T_FCONST) { + ieeedtod(&e, a->dval); + l = e.l; + Bputc(&obuf, l); + Bputc(&obuf, l>>8); + Bputc(&obuf, l>>16); + Bputc(&obuf, l>>24); + l = e.h; + Bputc(&obuf, l); + Bputc(&obuf, l>>8); + Bputc(&obuf, l>>16); + Bputc(&obuf, l>>24); + return; + } + if(t & T_SCONST) { + n = a->sval; + for(i=0; itype); +} + +void +outcode(int a, Gen2 *g2) +{ + int sf, st, t; + Sym *s; + + if(pass == 1) + goto out; + +jackpot: + sf = 0; + s = g2->from.sym; + while(s != S) { + sf = s->sym; + if(sf < 0 || sf >= NSYM) + sf = 0; + t = g2->from.type; + if(t == D_ADDR) + t = g2->from.index; + if(h[sf].type == t) + if(h[sf].sym == s) + break; + zname(s->name, t, sym); + s->sym = sym; + h[sym].sym = s; + h[sym].type = t; + sf = sym; + sym++; + if(sym >= NSYM) + sym = 1; + break; + } + st = 0; + s = g2->to.sym; + while(s != S) { + st = s->sym; + if(st < 0 || st >= NSYM) + st = 0; + t = g2->to.type; + if(t == D_ADDR) + t = g2->to.index; + if(h[st].type == t) + if(h[st].sym == s) + break; + zname(s->name, t, sym); + s->sym = sym; + h[sym].sym = s; + h[sym].type = t; + st = sym; + sym++; + if(sym >= NSYM) + sym = 1; + if(st == sf) + goto jackpot; + break; + } + Bputc(&obuf, a); + Bputc(&obuf, a>>8); + Bputc(&obuf, stmtline); + Bputc(&obuf, stmtline>>8); + Bputc(&obuf, stmtline>>16); + Bputc(&obuf, stmtline>>24); + zaddr(&g2->from, sf); + zaddr(&g2->to, st); + +out: + if(a != AGLOBL && a != ADATA) + pc++; +} + +void +outhist(void) +{ + Gen g; + Hist *h; + char *p, *q, *op, c; + int n; + + g = nullgen; + c = pathchar(); + for(h = hist; h != H; h = h->link) { + p = h->name; + op = 0; + /* on windows skip drive specifier in pathname */ + if(systemtype(Windows) && p && p[1] == ':'){ + p += 2; + c = *p; + } + if(p && p[0] != c && h->offset == 0 && pathname){ + /* on windows skip drive specifier in pathname */ + if(systemtype(Windows) && pathname[1] == ':') { + op = p; + p = pathname+2; + c = *p; + } else if(pathname[0] == c){ + op = p; + p = pathname; + } + } + while(p) { + q = strchr(p, c); + if(q) { + n = q-p; + if(n == 0){ + n = 1; /* leading "/" */ + *p = '/'; /* don't emit "\" on windows */ + } + q++; + } else { + n = strlen(p); + q = 0; + } + if(n) { + Bputc(&obuf, ANAME); + Bputc(&obuf, ANAME>>8); + Bputc(&obuf, D_FILE); /* type */ + Bputc(&obuf, 1); /* sym */ + Bputc(&obuf, '<'); + Bwrite(&obuf, p, n); + Bputc(&obuf, 0); + } + p = q; + if(p == 0 && op) { + p = op; + op = 0; + } + } + g.offset = h->offset; + + Bputc(&obuf, AHISTORY); + Bputc(&obuf, AHISTORY>>8); + Bputc(&obuf, h->line); + Bputc(&obuf, h->line>>8); + Bputc(&obuf, h->line>>16); + Bputc(&obuf, h->line>>24); + zaddr(&nullgen, 0); + zaddr(&g, 0); + } +} + +void +pragbldicks(void) +{ + while(getnsc() != '\n') + ; +} + +void +praghjdicks(void) +{ + while(getnsc() != '\n') + ; +} + +#include "../cc/lexbody" +#include "../cc/macbody" diff --git a/src/cmd/6c/Makefile b/src/cmd/6c/Makefile new file mode 100644 index 000000000..484e16def --- /dev/null +++ b/src/cmd/6c/Makefile @@ -0,0 +1,36 @@ +# Copyright 2009 The Go Authors. All rights reserved. +# Use of this source code is governed by a BSD-style +# license that can be found in the LICENSE file. + +include ../../Make.inc +O:=$(HOST_O) + +TARG=6c + +HFILES=\ + gc.h\ + ../6l/6.out.h\ + ../cc/cc.h\ + +OFILES=\ + cgen.$O\ + list.$O\ + sgen.$O\ + swt.$O\ + txt.$O\ + pgen.$O\ + pswt.$O\ + div.$O\ + mul.$O\ + reg.$O\ + peep.$O\ + machcap.$O\ + ../6l/enam.$O\ + +LIB=\ + ../cc/cc.a\ + +include ../../Make.ccmd + +%.$O: ../cc/%.c + $(HOST_CC) $(HOST_CFLAGS) -c -I. -o $@ ../cc/$*.c diff --git a/src/cmd/6c/cgen.c b/src/cmd/6c/cgen.c new file mode 100644 index 000000000..7f717dcbb --- /dev/null +++ b/src/cmd/6c/cgen.c @@ -0,0 +1,1976 @@ +// Inferno utils/6c/cgen.c +// http://code.google.com/p/inferno-os/source/browse/utils/6c/cgen.c +// +// Copyright © 1994-1999 Lucent Technologies Inc. All rights reserved. +// Portions Copyright © 1995-1997 C H Forsyth (forsyth@terzarima.net) +// Portions Copyright © 1997-1999 Vita Nuova Limited +// Portions Copyright © 2000-2007 Vita Nuova Holdings Limited (www.vitanuova.com) +// Portions Copyright © 2004,2006 Bruce Ellis +// Portions Copyright © 2005-2007 C H Forsyth (forsyth@terzarima.net) +// Revisions Copyright © 2000-2007 Lucent Technologies Inc. and others +// Portions Copyright © 2009 The Go Authors. All rights reserved. +// +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to deal +// in the Software without restriction, including without limitation the rights +// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +// copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +// THE SOFTWARE. + +#include "gc.h" + +/* ,x/^(print|prtree)\(/i/\/\/ */ +int castup(Type*, Type*); +int vaddr(Node *n, int a); + +void +cgen(Node *n, Node *nn) +{ + Node *l, *r, *t; + Prog *p1; + Node nod, nod1, nod2, nod3, nod4; + int o, hardleft; + int32 v, curs; + vlong c; + + if(debug['g']) { + prtree(nn, "cgen lhs"); + prtree(n, "cgen"); + } + if(n == Z || n->type == T) + return; + if(typesu[n->type->etype]) { + sugen(n, nn, n->type->width); + return; + } + l = n->left; + r = n->right; + o = n->op; + + if(n->op == OEXREG || (nn != Z && nn->op == OEXREG)) { + gmove(n, nn); + return; + } + + if(n->addable >= INDEXED) { + if(nn == Z) { + switch(o) { + default: + nullwarn(Z, Z); + break; + case OINDEX: + nullwarn(l, r); + break; + } + return; + } + gmove(n, nn); + return; + } + curs = cursafe; + + if(l->complex >= FNX) + if(r != Z && r->complex >= FNX) + switch(o) { + default: + if(cond(o) && typesu[l->type->etype]) + break; + + regret(&nod, r); + cgen(r, &nod); + + regsalloc(&nod1, r); + gmove(&nod, &nod1); + + regfree(&nod); + nod = *n; + nod.right = &nod1; + + cgen(&nod, nn); + return; + + case OFUNC: + case OCOMMA: + case OANDAND: + case OOROR: + case OCOND: + case ODOT: + break; + } + + hardleft = l->addable < INDEXED || l->complex >= FNX; + switch(o) { + default: + diag(n, "unknown op in cgen: %O", o); + break; + + case ONEG: + case OCOM: + if(nn == Z) { + nullwarn(l, Z); + break; + } + regalloc(&nod, l, nn); + cgen(l, &nod); + gopcode(o, n->type, Z, &nod); + gmove(&nod, nn); + regfree(&nod); + break; + + case OAS: + if(l->op == OBIT) + goto bitas; + if(!hardleft) { + if(nn != Z || r->addable < INDEXED || hardconst(r)) { + if(r->complex >= FNX && nn == Z) + regret(&nod, r); + else + regalloc(&nod, r, nn); + cgen(r, &nod); + gmove(&nod, l); + if(nn != Z) + gmove(&nod, nn); + regfree(&nod); + } else + gmove(r, l); + break; + } + if(l->complex >= r->complex) { + if(l->op == OINDEX && immconst(r)) { + gmove(r, l); + break; + } + reglcgen(&nod1, l, Z); + if(r->addable >= INDEXED && !hardconst(r)) { + gmove(r, &nod1); + if(nn != Z) + gmove(r, nn); + regfree(&nod1); + break; + } + regalloc(&nod, r, nn); + cgen(r, &nod); + } else { + regalloc(&nod, r, nn); + cgen(r, &nod); + reglcgen(&nod1, l, Z); + } + gmove(&nod, &nod1); + regfree(&nod); + regfree(&nod1); + break; + + bitas: + n = l->left; + regalloc(&nod, r, nn); + if(l->complex >= r->complex) { + reglcgen(&nod1, n, Z); + cgen(r, &nod); + } else { + cgen(r, &nod); + reglcgen(&nod1, n, Z); + } + regalloc(&nod2, n, Z); + gmove(&nod1, &nod2); + bitstore(l, &nod, &nod1, &nod2, nn); + break; + + case OBIT: + if(nn == Z) { + nullwarn(l, Z); + break; + } + bitload(n, &nod, Z, Z, nn); + gmove(&nod, nn); + regfree(&nod); + break; + + case OLSHR: + case OASHL: + case OASHR: + if(nn == Z) { + nullwarn(l, r); + break; + } + if(r->op == OCONST) { + if(r->vconst == 0) { + cgen(l, nn); + break; + } + regalloc(&nod, l, nn); + cgen(l, &nod); + if(o == OASHL && r->vconst == 1) + gopcode(OADD, n->type, &nod, &nod); + else + gopcode(o, n->type, r, &nod); + gmove(&nod, nn); + regfree(&nod); + break; + } + + /* + * get nod to be D_CX + */ + if(nodreg(&nod, nn, D_CX)) { + regsalloc(&nod1, n); + gmove(&nod, &nod1); + cgen(n, &nod); /* probably a bug */ + gmove(&nod, nn); + gmove(&nod1, &nod); + break; + } + reg[D_CX]++; + if(nn->op == OREGISTER && nn->reg == D_CX) + regalloc(&nod1, l, Z); + else + regalloc(&nod1, l, nn); + if(r->complex >= l->complex) { + cgen(r, &nod); + cgen(l, &nod1); + } else { + cgen(l, &nod1); + cgen(r, &nod); + } + gopcode(o, n->type, &nod, &nod1); + gmove(&nod1, nn); + regfree(&nod); + regfree(&nod1); + break; + + case OADD: + case OSUB: + case OOR: + case OXOR: + case OAND: + if(nn == Z) { + nullwarn(l, r); + break; + } + if(typefd[n->type->etype]) + goto fop; + if(r->op == OCONST) { + if(r->vconst == 0 && o != OAND) { + cgen(l, nn); + break; + } + } + if(n->op == OADD && l->op == OASHL && l->right->op == OCONST + && (r->op != OCONST || r->vconst < -128 || r->vconst > 127)) { + c = l->right->vconst; + if(c > 0 && c <= 3) { + if(l->left->complex >= r->complex) { + regalloc(&nod, l->left, nn); + cgen(l->left, &nod); + if(r->addable < INDEXED) { + regalloc(&nod1, r, Z); + cgen(r, &nod1); + genmuladd(&nod, &nod, 1 << c, &nod1); + regfree(&nod1); + } + else + genmuladd(&nod, &nod, 1 << c, r); + } + else { + regalloc(&nod, r, nn); + cgen(r, &nod); + regalloc(&nod1, l->left, Z); + cgen(l->left, &nod1); + genmuladd(&nod, &nod1, 1 << c, &nod); + regfree(&nod1); + } + gmove(&nod, nn); + regfree(&nod); + break; + } + } + if(r->addable >= INDEXED && !hardconst(r)) { + regalloc(&nod, l, nn); + cgen(l, &nod); + gopcode(o, n->type, r, &nod); + gmove(&nod, nn); + regfree(&nod); + break; + } + if(l->complex >= r->complex) { + regalloc(&nod, l, nn); + cgen(l, &nod); + regalloc(&nod1, r, Z); + cgen(r, &nod1); + gopcode(o, n->type, &nod1, &nod); + } else { + regalloc(&nod1, r, nn); + cgen(r, &nod1); + regalloc(&nod, l, Z); + cgen(l, &nod); + gopcode(o, n->type, &nod1, &nod); + } + gmove(&nod, nn); + regfree(&nod); + regfree(&nod1); + break; + + case OLMOD: + case OMOD: + case OLMUL: + case OLDIV: + case OMUL: + case ODIV: + if(nn == Z) { + nullwarn(l, r); + break; + } + if(typefd[n->type->etype]) + goto fop; + if(r->op == OCONST && typechl[n->type->etype]) { /* TO DO */ + SET(v); + switch(o) { + case ODIV: + case OMOD: + c = r->vconst; + if(c < 0) + c = -c; + v = xlog2(c); + if(v < 0) + break; + /* fall thru */ + case OMUL: + case OLMUL: + regalloc(&nod, l, nn); + cgen(l, &nod); + switch(o) { + case OMUL: + case OLMUL: + mulgen(n->type, r, &nod); + break; + case ODIV: + sdiv2(r->vconst, v, l, &nod); + break; + case OMOD: + smod2(r->vconst, v, l, &nod); + break; + } + gmove(&nod, nn); + regfree(&nod); + goto done; + case OLDIV: + c = r->vconst; + if((c & 0x80000000) == 0) + break; + regalloc(&nod1, l, Z); + cgen(l, &nod1); + regalloc(&nod, l, nn); + zeroregm(&nod); + gins(ACMPL, &nod1, nodconst(c)); + gins(ASBBL, nodconst(-1), &nod); + regfree(&nod1); + gmove(&nod, nn); + regfree(&nod); + goto done; + } + } + + if(o == OMUL) { + if(l->addable >= INDEXED) { + t = l; + l = r; + r = t; + } + /* should favour AX */ + regalloc(&nod, l, nn); + cgen(l, &nod); + if(r->addable < INDEXED || hardconst(r)) { + regalloc(&nod1, r, Z); + cgen(r, &nod1); + gopcode(OMUL, n->type, &nod1, &nod); + regfree(&nod1); + }else + gopcode(OMUL, n->type, r, &nod); /* addressible */ + gmove(&nod, nn); + regfree(&nod); + break; + } + + /* + * get nod to be D_AX + * get nod1 to be D_DX + */ + if(nodreg(&nod, nn, D_AX)) { + regsalloc(&nod2, n); + gmove(&nod, &nod2); + v = reg[D_AX]; + reg[D_AX] = 0; + + if(isreg(l, D_AX)) { + nod3 = *n; + nod3.left = &nod2; + cgen(&nod3, nn); + } else + if(isreg(r, D_AX)) { + nod3 = *n; + nod3.right = &nod2; + cgen(&nod3, nn); + } else + cgen(n, nn); + + gmove(&nod2, &nod); + reg[D_AX] = v; + break; + } + if(nodreg(&nod1, nn, D_DX)) { + regsalloc(&nod2, n); + gmove(&nod1, &nod2); + v = reg[D_DX]; + reg[D_DX] = 0; + + if(isreg(l, D_DX)) { + nod3 = *n; + nod3.left = &nod2; + cgen(&nod3, nn); + } else + if(isreg(r, D_DX)) { + nod3 = *n; + nod3.right = &nod2; + cgen(&nod3, nn); + } else + cgen(n, nn); + + gmove(&nod2, &nod1); + reg[D_DX] = v; + break; + } + reg[D_AX]++; + + if(r->op == OCONST && (o == ODIV || o == OLDIV) && immconst(r) && typechl[r->type->etype]) { + reg[D_DX]++; + if(l->addable < INDEXED) { + regalloc(&nod2, l, Z); + cgen(l, &nod2); + l = &nod2; + } + if(o == ODIV) + sdivgen(l, r, &nod, &nod1); + else + udivgen(l, r, &nod, &nod1); + gmove(&nod1, nn); + if(l == &nod2) + regfree(l); + goto freeaxdx; + } + + if(l->complex >= r->complex) { + cgen(l, &nod); + reg[D_DX]++; + if(o == ODIV || o == OMOD) + gins(typechl[l->type->etype]? ACDQ: ACQO, Z, Z); + if(o == OLDIV || o == OLMOD) + zeroregm(&nod1); + if(r->addable < INDEXED || r->op == OCONST) { + regsalloc(&nod3, r); + cgen(r, &nod3); + gopcode(o, n->type, &nod3, Z); + } else + gopcode(o, n->type, r, Z); + } else { + regsalloc(&nod3, r); + cgen(r, &nod3); + cgen(l, &nod); + reg[D_DX]++; + if(o == ODIV || o == OMOD) + gins(typechl[l->type->etype]? ACDQ: ACQO, Z, Z); + if(o == OLDIV || o == OLMOD) + zeroregm(&nod1); + gopcode(o, n->type, &nod3, Z); + } + if(o == OMOD || o == OLMOD) + gmove(&nod1, nn); + else + gmove(&nod, nn); + freeaxdx: + regfree(&nod); + regfree(&nod1); + break; + + case OASLSHR: + case OASASHL: + case OASASHR: + if(r->op == OCONST) + goto asand; + if(l->op == OBIT) + goto asbitop; + if(typefd[n->type->etype]) + goto asand; /* can this happen? */ + + /* + * get nod to be D_CX + */ + if(nodreg(&nod, nn, D_CX)) { + regsalloc(&nod1, n); + gmove(&nod, &nod1); + cgen(n, &nod); + if(nn != Z) + gmove(&nod, nn); + gmove(&nod1, &nod); + break; + } + reg[D_CX]++; + + if(r->complex >= l->complex) { + cgen(r, &nod); + if(hardleft) + reglcgen(&nod1, l, Z); + else + nod1 = *l; + } else { + if(hardleft) + reglcgen(&nod1, l, Z); + else + nod1 = *l; + cgen(r, &nod); + } + + gopcode(o, l->type, &nod, &nod1); + regfree(&nod); + if(nn != Z) + gmove(&nod1, nn); + if(hardleft) + regfree(&nod1); + break; + + case OASAND: + case OASADD: + case OASSUB: + case OASXOR: + case OASOR: + asand: + if(l->op == OBIT) + goto asbitop; + if(typefd[l->type->etype] || typefd[r->type->etype]) + goto asfop; + if(l->complex >= r->complex) { + if(hardleft) + reglcgen(&nod, l, Z); + else + nod = *l; + if(!immconst(r)) { + regalloc(&nod1, r, nn); + cgen(r, &nod1); + gopcode(o, l->type, &nod1, &nod); + regfree(&nod1); + } else + gopcode(o, l->type, r, &nod); + } else { + regalloc(&nod1, r, nn); + cgen(r, &nod1); + if(hardleft) + reglcgen(&nod, l, Z); + else + nod = *l; + gopcode(o, l->type, &nod1, &nod); + regfree(&nod1); + } + if(nn != Z) + gmove(&nod, nn); + if(hardleft) + regfree(&nod); + break; + + asfop: + if(l->complex >= r->complex) { + if(hardleft) + reglcgen(&nod, l, Z); + else + nod = *l; + if(r->addable < INDEXED){ + regalloc(&nod1, r, nn); + cgen(r, &nod1); + }else + nod1 = *r; + regalloc(&nod2, r, Z); + gmove(&nod, &nod2); + gopcode(o, r->type, &nod1, &nod2); + gmove(&nod2, &nod); + regfree(&nod2); + if(r->addable < INDEXED) + regfree(&nod1); + } else { + regalloc(&nod1, r, nn); + cgen(r, &nod1); + if(hardleft) + reglcgen(&nod, l, Z); + else + nod = *l; + if(o != OASMUL && o != OASADD) { + regalloc(&nod2, r, Z); + gmove(&nod, &nod2); + gopcode(o, r->type, &nod1, &nod2); + regfree(&nod1); + gmove(&nod2, &nod); + regfree(&nod2); + } else { + gopcode(o, r->type, &nod, &nod1); + gmove(&nod1, &nod); + regfree(&nod1); + } + } + if(nn != Z) + gmove(&nod, nn); + if(hardleft) + regfree(&nod); + break; + + case OASLMUL: + case OASLDIV: + case OASLMOD: + case OASMUL: + case OASDIV: + case OASMOD: + if(l->op == OBIT) + goto asbitop; + if(typefd[n->type->etype] || typefd[r->type->etype]) + goto asfop; + if(r->op == OCONST && typechl[n->type->etype]) { + SET(v); + switch(o) { + case OASDIV: + case OASMOD: + c = r->vconst; + if(c < 0) + c = -c; + v = xlog2(c); + if(v < 0) + break; + /* fall thru */ + case OASMUL: + case OASLMUL: + if(hardleft) + reglcgen(&nod2, l, Z); + else + nod2 = *l; + regalloc(&nod, l, nn); + cgen(&nod2, &nod); + switch(o) { + case OASMUL: + case OASLMUL: + mulgen(n->type, r, &nod); + break; + case OASDIV: + sdiv2(r->vconst, v, l, &nod); + break; + case OASMOD: + smod2(r->vconst, v, l, &nod); + break; + } + havev: + gmove(&nod, &nod2); + if(nn != Z) + gmove(&nod, nn); + if(hardleft) + regfree(&nod2); + regfree(&nod); + goto done; + case OASLDIV: + c = r->vconst; + if((c & 0x80000000) == 0) + break; + if(hardleft) + reglcgen(&nod2, l, Z); + else + nod2 = *l; + regalloc(&nod1, l, nn); + cgen(&nod2, &nod1); + regalloc(&nod, l, nn); + zeroregm(&nod); + gins(ACMPL, &nod1, nodconst(c)); + gins(ASBBL, nodconst(-1), &nod); + regfree(&nod1); + goto havev; + } + } + + if(o == OASMUL) { + /* should favour AX */ + regalloc(&nod, l, nn); + if(r->complex >= FNX) { + regalloc(&nod1, r, Z); + cgen(r, &nod1); + r = &nod1; + } + if(hardleft) + reglcgen(&nod2, l, Z); + else + nod2 = *l; + cgen(&nod2, &nod); + if(r->addable < INDEXED || hardconst(r)) { + if(r->complex < FNX) { + regalloc(&nod1, r, Z); + cgen(r, &nod1); + } + gopcode(OASMUL, n->type, &nod1, &nod); + regfree(&nod1); + } + else + gopcode(OASMUL, n->type, r, &nod); + if(r == &nod1) + regfree(r); + gmove(&nod, &nod2); + if(nn != Z) + gmove(&nod, nn); + regfree(&nod); + if(hardleft) + regfree(&nod2); + break; + } + + /* + * get nod to be D_AX + * get nod1 to be D_DX + */ + if(nodreg(&nod, nn, D_AX)) { + regsalloc(&nod2, n); + gmove(&nod, &nod2); + v = reg[D_AX]; + reg[D_AX] = 0; + + if(isreg(l, D_AX)) { + nod3 = *n; + nod3.left = &nod2; + cgen(&nod3, nn); + } else + if(isreg(r, D_AX)) { + nod3 = *n; + nod3.right = &nod2; + cgen(&nod3, nn); + } else + cgen(n, nn); + + gmove(&nod2, &nod); + reg[D_AX] = v; + break; + } + if(nodreg(&nod1, nn, D_DX)) { + regsalloc(&nod2, n); + gmove(&nod1, &nod2); + v = reg[D_DX]; + reg[D_DX] = 0; + + if(isreg(l, D_DX)) { + nod3 = *n; + nod3.left = &nod2; + cgen(&nod3, nn); + } else + if(isreg(r, D_DX)) { + nod3 = *n; + nod3.right = &nod2; + cgen(&nod3, nn); + } else + cgen(n, nn); + + gmove(&nod2, &nod1); + reg[D_DX] = v; + break; + } + reg[D_AX]++; + reg[D_DX]++; + + if(l->complex >= r->complex) { + if(hardleft) + reglcgen(&nod2, l, Z); + else + nod2 = *l; + cgen(&nod2, &nod); + if(r->op == OCONST && typechl[r->type->etype]) { + switch(o) { + case OASDIV: + sdivgen(&nod2, r, &nod, &nod1); + goto divdone; + case OASLDIV: + udivgen(&nod2, r, &nod, &nod1); + divdone: + gmove(&nod1, &nod2); + if(nn != Z) + gmove(&nod1, nn); + goto freelxaxdx; + } + } + if(o == OASDIV || o == OASMOD) + gins(typechl[l->type->etype]? ACDQ: ACQO, Z, Z); + if(o == OASLDIV || o == OASLMOD) + zeroregm(&nod1); + if(r->addable < INDEXED || r->op == OCONST || + !typeil[r->type->etype]) { + regalloc(&nod3, r, Z); + cgen(r, &nod3); + gopcode(o, l->type, &nod3, Z); + regfree(&nod3); + } else + gopcode(o, n->type, r, Z); + } else { + regalloc(&nod3, r, Z); + cgen(r, &nod3); + if(hardleft) + reglcgen(&nod2, l, Z); + else + nod2 = *l; + cgen(&nod2, &nod); + if(o == OASDIV || o == OASMOD) + gins(typechl[l->type->etype]? ACDQ: ACQO, Z, Z); + if(o == OASLDIV || o == OASLMOD) + zeroregm(&nod1); + gopcode(o, l->type, &nod3, Z); + regfree(&nod3); + } + if(o == OASMOD || o == OASLMOD) { + gmove(&nod1, &nod2); + if(nn != Z) + gmove(&nod1, nn); + } else { + gmove(&nod, &nod2); + if(nn != Z) + gmove(&nod, nn); + } + freelxaxdx: + if(hardleft) + regfree(&nod2); + regfree(&nod); + regfree(&nod1); + break; + + fop: + if(l->complex >= r->complex) { + regalloc(&nod, l, nn); + cgen(l, &nod); + if(r->addable < INDEXED) { + regalloc(&nod1, r, Z); + cgen(r, &nod1); + gopcode(o, n->type, &nod1, &nod); + regfree(&nod1); + } else + gopcode(o, n->type, r, &nod); + } else { + /* TO DO: could do better with r->addable >= INDEXED */ + regalloc(&nod1, r, Z); + cgen(r, &nod1); + regalloc(&nod, l, nn); + cgen(l, &nod); + gopcode(o, n->type, &nod1, &nod); + regfree(&nod1); + } + gmove(&nod, nn); + regfree(&nod); + break; + + asbitop: + regalloc(&nod4, n, nn); + if(l->complex >= r->complex) { + bitload(l, &nod, &nod1, &nod2, &nod4); + regalloc(&nod3, r, Z); + cgen(r, &nod3); + } else { + regalloc(&nod3, r, Z); + cgen(r, &nod3); + bitload(l, &nod, &nod1, &nod2, &nod4); + } + gmove(&nod, &nod4); + + { /* TO DO: check floating point source */ + Node onod; + + /* incredible grot ... */ + onod = nod3; + onod.op = o; + onod.complex = 2; + onod.addable = 0; + onod.type = tfield; + onod.left = &nod4; + onod.right = &nod3; + cgen(&onod, Z); + } + regfree(&nod3); + gmove(&nod4, &nod); + regfree(&nod4); + bitstore(l, &nod, &nod1, &nod2, nn); + break; + + case OADDR: + if(nn == Z) { + nullwarn(l, Z); + break; + } + lcgen(l, nn); + break; + + case OFUNC: + if(l->complex >= FNX) { + if(l->op != OIND) + diag(n, "bad function call"); + + regret(&nod, l->left); + cgen(l->left, &nod); + regsalloc(&nod1, l->left); + gmove(&nod, &nod1); + regfree(&nod); + + nod = *n; + nod.left = &nod2; + nod2 = *l; + nod2.left = &nod1; + nod2.complex = 1; + cgen(&nod, nn); + + return; + } + gargs(r, &nod, &nod1); + if(l->addable < INDEXED) { + reglcgen(&nod, l, nn); + nod.op = OREGISTER; + gopcode(OFUNC, n->type, Z, &nod); + regfree(&nod); + } else + gopcode(OFUNC, n->type, Z, l); + if(REGARG >= 0 && reg[REGARG]) + reg[REGARG]--; + if(nn != Z) { + regret(&nod, n); + gmove(&nod, nn); + regfree(&nod); + } + break; + + case OIND: + if(nn == Z) { + nullwarn(l, Z); + break; + } + regialloc(&nod, n, nn); + r = l; + while(r->op == OADD) + r = r->right; + if(sconst(r)) { + v = r->vconst; + r->vconst = 0; + cgen(l, &nod); + nod.xoffset += v; + r->vconst = v; + } else + cgen(l, &nod); + regind(&nod, n); + gmove(&nod, nn); + regfree(&nod); + break; + + case OEQ: + case ONE: + case OLE: + case OLT: + case OGE: + case OGT: + case OLO: + case OLS: + case OHI: + case OHS: + if(nn == Z) { + nullwarn(l, r); + break; + } + boolgen(n, 1, nn); + break; + + case OANDAND: + case OOROR: + boolgen(n, 1, nn); + if(nn == Z) + patch(p, pc); + break; + + case ONOT: + if(nn == Z) { + nullwarn(l, Z); + break; + } + boolgen(n, 1, nn); + break; + + case OCOMMA: + cgen(l, Z); + cgen(r, nn); + break; + + case OCAST: + if(nn == Z) { + nullwarn(l, Z); + break; + } + /* + * convert from types l->n->nn + */ + if(nocast(l->type, n->type) && nocast(n->type, nn->type)) { + /* both null, gen l->nn */ + cgen(l, nn); + break; + } + if(ewidth[n->type->etype] < ewidth[l->type->etype]){ + if(l->type->etype == TIND && typechlp[n->type->etype]) + warn(n, "conversion of pointer to shorter integer"); + }else if(0){ + if(nocast(n->type, nn->type) || castup(n->type, nn->type)){ + if(typefd[l->type->etype] != typefd[nn->type->etype]) + regalloc(&nod, l, nn); + else + regalloc(&nod, nn, nn); + cgen(l, &nod); + gmove(&nod, nn); + regfree(&nod); + break; + } + } + regalloc(&nod, l, nn); + cgen(l, &nod); + regalloc(&nod1, n, &nod); + gmove(&nod, &nod1); + gmove(&nod1, nn); + regfree(&nod1); + regfree(&nod); + break; + + case ODOT: + sugen(l, nodrat, l->type->width); + if(nn == Z) + break; + warn(n, "non-interruptable temporary"); + nod = *nodrat; + if(!r || r->op != OCONST) { + diag(n, "DOT and no offset"); + break; + } + nod.xoffset += (int32)r->vconst; + nod.type = n->type; + cgen(&nod, nn); + break; + + case OCOND: + bcgen(l, 1); + p1 = p; + cgen(r->left, nn); + gbranch(OGOTO); + patch(p1, pc); + p1 = p; + cgen(r->right, nn); + patch(p1, pc); + break; + + case OPOSTINC: + case OPOSTDEC: + v = 1; + if(l->type->etype == TIND) + v = l->type->link->width; + if(o == OPOSTDEC) + v = -v; + if(l->op == OBIT) + goto bitinc; + if(nn == Z) + goto pre; + + if(hardleft) + reglcgen(&nod, l, Z); + else + nod = *l; + + gmove(&nod, nn); + if(typefd[n->type->etype]) { + regalloc(&nod1, l, Z); + gmove(&nod, &nod1); + if(v < 0) + gopcode(OSUB, n->type, nodfconst(-v), &nod1); + else + gopcode(OADD, n->type, nodfconst(v), &nod1); + gmove(&nod1, &nod); + regfree(&nod1); + } else + gopcode(OADD, n->type, nodconst(v), &nod); + if(hardleft) + regfree(&nod); + break; + + case OPREINC: + case OPREDEC: + v = 1; + if(l->type->etype == TIND) + v = l->type->link->width; + if(o == OPREDEC) + v = -v; + if(l->op == OBIT) + goto bitinc; + + pre: + if(hardleft) + reglcgen(&nod, l, Z); + else + nod = *l; + if(typefd[n->type->etype]) { + regalloc(&nod1, l, Z); + gmove(&nod, &nod1); + if(v < 0) + gopcode(OSUB, n->type, nodfconst(-v), &nod1); + else + gopcode(OADD, n->type, nodfconst(v), &nod1); + gmove(&nod1, &nod); + regfree(&nod1); + } else + gopcode(OADD, n->type, nodconst(v), &nod); + if(nn != Z) + gmove(&nod, nn); + if(hardleft) + regfree(&nod); + break; + + bitinc: + if(nn != Z && (o == OPOSTINC || o == OPOSTDEC)) { + bitload(l, &nod, &nod1, &nod2, Z); + gmove(&nod, nn); + gopcode(OADD, tfield, nodconst(v), &nod); + bitstore(l, &nod, &nod1, &nod2, Z); + break; + } + bitload(l, &nod, &nod1, &nod2, nn); + gopcode(OADD, tfield, nodconst(v), &nod); + bitstore(l, &nod, &nod1, &nod2, nn); + break; + } +done: + cursafe = curs; +} + +void +reglcgen(Node *t, Node *n, Node *nn) +{ + Node *r; + int32 v; + + regialloc(t, n, nn); + if(n->op == OIND) { + r = n->left; + while(r->op == OADD) + r = r->right; + if(sconst(r)) { + v = r->vconst; + r->vconst = 0; + lcgen(n, t); + t->xoffset += v; + r->vconst = v; + regind(t, n); + return; + } + } + lcgen(n, t); + regind(t, n); +} + +void +lcgen(Node *n, Node *nn) +{ + Prog *p1; + Node nod; + + if(debug['g']) { + prtree(nn, "lcgen lhs"); + prtree(n, "lcgen"); + } + if(n == Z || n->type == T) + return; + if(nn == Z) { + nn = &nod; + regalloc(&nod, n, Z); + } + switch(n->op) { + default: + if(n->addable < INDEXED) { + diag(n, "unknown op in lcgen: %O", n->op); + break; + } + gopcode(OADDR, n->type, n, nn); + break; + + case OCOMMA: + cgen(n->left, n->left); + lcgen(n->right, nn); + break; + + case OIND: + cgen(n->left, nn); + break; + + case OCOND: + bcgen(n->left, 1); + p1 = p; + lcgen(n->right->left, nn); + gbranch(OGOTO); + patch(p1, pc); + p1 = p; + lcgen(n->right->right, nn); + patch(p1, pc); + break; + } +} + +void +bcgen(Node *n, int true) +{ + + if(n->type == T) + gbranch(OGOTO); + else + boolgen(n, true, Z); +} + +void +boolgen(Node *n, int true, Node *nn) +{ + int o; + Prog *p1, *p2; + Node *l, *r, nod, nod1; + int32 curs; + + if(debug['g']) { + prtree(nn, "boolgen lhs"); + prtree(n, "boolgen"); + } + curs = cursafe; + l = n->left; + r = n->right; + switch(n->op) { + + default: + o = ONE; + if(true) + o = OEQ; + /* bad, 13 is address of external that becomes constant */ + if(n->addable >= INDEXED && n->addable != 13) { + if(typefd[n->type->etype]) { + regalloc(&nod1, n, Z); + gmove(nodfconst(0.0), &nod1); /* TO DO: FREGZERO */ + gopcode(o, n->type, n, &nod1); + regfree(&nod1); + } else + gopcode(o, n->type, n, nodconst(0)); + goto com; + } + regalloc(&nod, n, nn); + cgen(n, &nod); + if(typefd[n->type->etype]) { + regalloc(&nod1, n, Z); + gmove(nodfconst(0.0), &nod1); /* TO DO: FREGZERO */ + gopcode(o, n->type, &nod, &nod1); + regfree(&nod1); + } else + gopcode(o, n->type, &nod, nodconst(0)); + regfree(&nod); + goto com; + + case OCONST: + o = vconst(n); + if(!true) + o = !o; + gbranch(OGOTO); + if(o) { + p1 = p; + gbranch(OGOTO); + patch(p1, pc); + } + goto com; + + case OCOMMA: + cgen(l, Z); + boolgen(r, true, nn); + break; + + case ONOT: + boolgen(l, !true, nn); + break; + + case OCOND: + bcgen(l, 1); + p1 = p; + bcgen(r->left, true); + p2 = p; + gbranch(OGOTO); + patch(p1, pc); + p1 = p; + bcgen(r->right, !true); + patch(p2, pc); + p2 = p; + gbranch(OGOTO); + patch(p1, pc); + patch(p2, pc); + goto com; + + case OANDAND: + if(!true) + goto caseor; + + caseand: + bcgen(l, true); + p1 = p; + bcgen(r, !true); + p2 = p; + patch(p1, pc); + gbranch(OGOTO); + patch(p2, pc); + goto com; + + case OOROR: + if(!true) + goto caseand; + + caseor: + bcgen(l, !true); + p1 = p; + bcgen(r, !true); + p2 = p; + gbranch(OGOTO); + patch(p1, pc); + patch(p2, pc); + goto com; + + case OEQ: + case ONE: + case OLE: + case OLT: + case OGE: + case OGT: + case OHI: + case OHS: + case OLO: + case OLS: + o = n->op; + if(true) + o = comrel[relindex(o)]; + if(l->complex >= FNX && r->complex >= FNX) { + regret(&nod, r); + cgen(r, &nod); + regsalloc(&nod1, r); + gmove(&nod, &nod1); + regfree(&nod); + nod = *n; + nod.right = &nod1; + boolgen(&nod, true, nn); + break; + } + if(immconst(l)) { + o = invrel[relindex(o)]; + /* bad, 13 is address of external that becomes constant */ + if(r->addable < INDEXED || r->addable == 13) { + regalloc(&nod, r, nn); + cgen(r, &nod); + gopcode(o, l->type, &nod, l); + regfree(&nod); + } else + gopcode(o, l->type, r, l); + goto com; + } + if(typefd[l->type->etype]) + o = invrel[relindex(logrel[relindex(o)])]; + if(l->complex >= r->complex) { + regalloc(&nod, l, nn); + cgen(l, &nod); + if(r->addable < INDEXED || hardconst(r) || typefd[l->type->etype]) { + regalloc(&nod1, r, Z); + cgen(r, &nod1); + gopcode(o, l->type, &nod, &nod1); + regfree(&nod1); + } else + gopcode(o, l->type, &nod, r); + regfree(&nod); + goto com; + } + regalloc(&nod, r, nn); + cgen(r, &nod); + if(l->addable < INDEXED || l->addable == 13 || hardconst(l)) { + regalloc(&nod1, l, Z); + cgen(l, &nod1); + if(typechl[l->type->etype] && ewidth[l->type->etype] <= ewidth[TINT]) + gopcode(o, types[TINT], &nod1, &nod); + else + gopcode(o, l->type, &nod1, &nod); + regfree(&nod1); + } else + gopcode(o, l->type, l, &nod); + regfree(&nod); + + com: + if(nn != Z) { + p1 = p; + gmove(nodconst(1L), nn); + gbranch(OGOTO); + p2 = p; + patch(p1, pc); + gmove(nodconst(0L), nn); + patch(p2, pc); + } + break; + } + cursafe = curs; +} + +void +sugen(Node *n, Node *nn, int32 w) +{ + Prog *p1; + Node nod0, nod1, nod2, nod3, nod4, *l, *r; + Type *t; + int c, mt, mo; + vlong o0, o1; + + if(n == Z || n->type == T) + return; + if(debug['g']) { + prtree(nn, "sugen lhs"); + prtree(n, "sugen"); + } + if(nn == nodrat) + if(w > nrathole) + nrathole = w; + switch(n->op) { + case OIND: + if(nn == Z) { + nullwarn(n->left, Z); + break; + } + + default: + goto copy; + + case OCONST: + goto copy; + + case ODOT: + l = n->left; + sugen(l, nodrat, l->type->width); + if(nn == Z) + break; + warn(n, "non-interruptable temporary"); + nod1 = *nodrat; + r = n->right; + if(!r || r->op != OCONST) { + diag(n, "DOT and no offset"); + break; + } + nod1.xoffset += (int32)r->vconst; + nod1.type = n->type; + sugen(&nod1, nn, w); + break; + + case OSTRUCT: + /* + * rewrite so lhs has no fn call + */ + if(nn != Z && side(nn)) { + nod1 = *n; + nod1.type = typ(TIND, n->type); + regret(&nod2, &nod1); + lcgen(nn, &nod2); + regsalloc(&nod0, &nod1); + cgen(&nod2, &nod0); + regfree(&nod2); + + nod1 = *n; + nod1.op = OIND; + nod1.left = &nod0; + nod1.right = Z; + nod1.complex = 1; + + sugen(n, &nod1, w); + return; + } + + r = n->left; + for(t = n->type->link; t != T; t = t->down) { + l = r; + if(r->op == OLIST) { + l = r->left; + r = r->right; + } + if(nn == Z) { + cgen(l, nn); + continue; + } + /* + * hand craft *(&nn + o) = l + */ + nod0 = znode; + nod0.op = OAS; + nod0.type = t; + nod0.left = &nod1; + nod0.right = nil; + + nod1 = znode; + nod1.op = OIND; + nod1.type = t; + nod1.left = &nod2; + + nod2 = znode; + nod2.op = OADD; + nod2.type = typ(TIND, t); + nod2.left = &nod3; + nod2.right = &nod4; + + nod3 = znode; + nod3.op = OADDR; + nod3.type = nod2.type; + nod3.left = nn; + + nod4 = znode; + nod4.op = OCONST; + nod4.type = nod2.type; + nod4.vconst = t->offset; + + ccom(&nod0); + acom(&nod0); + xcom(&nod0); + nod0.addable = 0; + nod0.right = l; + + // prtree(&nod0, "hand craft"); + cgen(&nod0, Z); + } + break; + + case OAS: + if(nn == Z) { + if(n->addable < INDEXED) + sugen(n->right, n->left, w); + break; + } + + sugen(n->right, nodrat, w); + warn(n, "non-interruptable temporary"); + sugen(nodrat, n->left, w); + sugen(nodrat, nn, w); + break; + + case OFUNC: + if(nn == Z) { + sugen(n, nodrat, w); + break; + } + if(nn->op != OIND) { + nn = new1(OADDR, nn, Z); + nn->type = types[TIND]; + nn->addable = 0; + } else + nn = nn->left; + n = new(OFUNC, n->left, new(OLIST, nn, n->right)); + n->type = types[TVOID]; + n->left->type = types[TVOID]; + cgen(n, Z); + break; + + case OCOND: + bcgen(n->left, 1); + p1 = p; + sugen(n->right->left, nn, w); + gbranch(OGOTO); + patch(p1, pc); + p1 = p; + sugen(n->right->right, nn, w); + patch(p1, pc); + break; + + case OCOMMA: + cgen(n->left, Z); + sugen(n->right, nn, w); + break; + } + return; + +copy: + if(nn == Z) { + switch(n->op) { + case OASADD: + case OASSUB: + case OASAND: + case OASOR: + case OASXOR: + + case OASMUL: + case OASLMUL: + + + case OASASHL: + case OASASHR: + case OASLSHR: + break; + + case OPOSTINC: + case OPOSTDEC: + case OPREINC: + case OPREDEC: + break; + + default: + return; + } + } + + if(n->complex >= FNX && nn != nil && nn->complex >= FNX) { + t = nn->type; + nn->type = types[TLONG]; + regialloc(&nod1, nn, Z); + lcgen(nn, &nod1); + regsalloc(&nod2, nn); + nn->type = t; + + gins(AMOVL, &nod1, &nod2); + regfree(&nod1); + + nod2.type = typ(TIND, t); + + nod1 = nod2; + nod1.op = OIND; + nod1.left = &nod2; + nod1.right = Z; + nod1.complex = 1; + nod1.type = t; + + sugen(n, &nod1, w); + return; + } + + if(w <= 32) { + c = cursafe; + if(n->left != Z && n->left->complex >= FNX + && n->right != Z && n->right->complex >= FNX) { + regsalloc(&nod1, n->right); + cgen(n->right, &nod1); + nod2 = *n; + nod2.right = &nod1; + cgen(&nod2, nn); + cursafe = c; + return; + } + if(w & 7) { + mt = TLONG; + mo = AMOVL; + } else { + mt = TVLONG; + mo = AMOVQ; + } + if(n->complex > nn->complex) { + t = n->type; + n->type = types[mt]; + regalloc(&nod0, n, Z); + if(!vaddr(n, 0)) { + reglcgen(&nod1, n, Z); + n->type = t; + n = &nod1; + } + else + n->type = t; + + t = nn->type; + nn->type = types[mt]; + if(!vaddr(nn, 0)) { + reglcgen(&nod2, nn, Z); + nn->type = t; + nn = &nod2; + } + else + nn->type = t; + } else { + t = nn->type; + nn->type = types[mt]; + regalloc(&nod0, nn, Z); + if(!vaddr(nn, 0)) { + reglcgen(&nod2, nn, Z); + nn->type = t; + nn = &nod2; + } + else + nn->type = t; + + t = n->type; + n->type = types[mt]; + if(!vaddr(n, 0)) { + reglcgen(&nod1, n, Z); + n->type = t; + n = &nod1; + } + else + n->type = t; + } + o0 = n->xoffset; + o1 = nn->xoffset; + w /= ewidth[mt]; + while(--w >= 0) { + gins(mo, n, &nod0); + gins(mo, &nod0, nn); + n->xoffset += ewidth[mt]; + nn->xoffset += ewidth[mt]; + } + n->xoffset = o0; + nn->xoffset = o1; + if(nn == &nod2) + regfree(&nod2); + if(n == &nod1) + regfree(&nod1); + regfree(&nod0); + return; + } + + /* botch, need to save in .safe */ + c = 0; + if(n->complex > nn->complex) { + t = n->type; + n->type = types[TLONG]; + nodreg(&nod1, n, D_SI); + if(reg[D_SI]) { + gins(APUSHQ, &nod1, Z); + c |= 1; + reg[D_SI]++; + } + lcgen(n, &nod1); + n->type = t; + + t = nn->type; + nn->type = types[TLONG]; + nodreg(&nod2, nn, D_DI); + if(reg[D_DI]) { +warn(Z, "DI botch"); + gins(APUSHQ, &nod2, Z); + c |= 2; + reg[D_DI]++; + } + lcgen(nn, &nod2); + nn->type = t; + } else { + t = nn->type; + nn->type = types[TLONG]; + nodreg(&nod2, nn, D_DI); + if(reg[D_DI]) { +warn(Z, "DI botch"); + gins(APUSHQ, &nod2, Z); + c |= 2; + reg[D_DI]++; + } + lcgen(nn, &nod2); + nn->type = t; + + t = n->type; + n->type = types[TLONG]; + nodreg(&nod1, n, D_SI); + if(reg[D_SI]) { + gins(APUSHQ, &nod1, Z); + c |= 1; + reg[D_SI]++; + } + lcgen(n, &nod1); + n->type = t; + } + nodreg(&nod3, n, D_CX); + if(reg[D_CX]) { + gins(APUSHQ, &nod3, Z); + c |= 4; + reg[D_CX]++; + } + gins(AMOVL, nodconst(w/SZ_INT), &nod3); + gins(ACLD, Z, Z); + gins(AREP, Z, Z); + gins(AMOVSL, Z, Z); + if(c & 4) { + gins(APOPQ, Z, &nod3); + reg[D_CX]--; + } + if(c & 2) { + gins(APOPQ, Z, &nod2); + reg[nod2.reg]--; + } + if(c & 1) { + gins(APOPQ, Z, &nod1); + reg[nod1.reg]--; + } +} + +/* + * TO DO + */ +void +layout(Node *f, Node *t, int c, int cv, Node *cn) +{ + Node t1, t2; + + while(c > 3) { + layout(f, t, 2, 0, Z); + c -= 2; + } + + regalloc(&t1, &lregnode, Z); + regalloc(&t2, &lregnode, Z); + if(c > 0) { + gmove(f, &t1); + f->xoffset += SZ_INT; + } + if(cn != Z) + gmove(nodconst(cv), cn); + if(c > 1) { + gmove(f, &t2); + f->xoffset += SZ_INT; + } + if(c > 0) { + gmove(&t1, t); + t->xoffset += SZ_INT; + } + if(c > 2) { + gmove(f, &t1); + f->xoffset += SZ_INT; + } + if(c > 1) { + gmove(&t2, t); + t->xoffset += SZ_INT; + } + if(c > 2) { + gmove(&t1, t); + t->xoffset += SZ_INT; + } + regfree(&t1); + regfree(&t2); +} + +/* + * constant is not vlong or fits as 32-bit signed immediate + */ +int +immconst(Node *n) +{ + int32 v; + + if(n->op != OCONST || !typechlpv[n->type->etype]) + return 0; + if(typechl[n->type->etype]) + return 1; + v = n->vconst; + return n->vconst == (vlong)v; +} + +/* + * if a constant and vlong, doesn't fit as 32-bit signed immediate + */ +int +hardconst(Node *n) +{ + return n->op == OCONST && !immconst(n); +} + +/* + * casting up to t2 covers an intermediate cast to t1 + */ +int +castup(Type *t1, Type *t2) +{ + int ft; + + if(!nilcast(t1, t2)) + return 0; + /* known to be small to large */ + ft = t1->etype; + switch(t2->etype){ + case TINT: + case TLONG: + return ft == TLONG || ft == TINT || ft == TSHORT || ft == TCHAR; + case TUINT: + case TULONG: + return ft == TULONG || ft == TUINT || ft == TUSHORT || ft == TUCHAR; + case TVLONG: + return ft == TLONG || ft == TINT || ft == TSHORT; + case TUVLONG: + return ft == TULONG || ft == TUINT || ft == TUSHORT; + } + return 0; +} + +void +zeroregm(Node *n) +{ + gins(AMOVL, nodconst(0), n); +} + +/* do we need to load the address of a vlong? */ +int +vaddr(Node *n, int a) +{ + switch(n->op) { + case ONAME: + if(a) + return 1; + return !(n->class == CEXTERN || n->class == CGLOBL || n->class == CSTATIC); + + case OCONST: + case OREGISTER: + case OINDREG: + return 1; + } + return 0; +} + +int32 +hi64v(Node *n) +{ + if(align(0, types[TCHAR], Aarg1, nil)) /* isbigendian */ + return (int32)(n->vconst) & ~0L; + else + return (int32)((uvlong)n->vconst>>32) & ~0L; +} + +int32 +lo64v(Node *n) +{ + if(align(0, types[TCHAR], Aarg1, nil)) /* isbigendian */ + return (int32)((uvlong)n->vconst>>32) & ~0L; + else + return (int32)(n->vconst) & ~0L; +} + +Node * +hi64(Node *n) +{ + return nodconst(hi64v(n)); +} + +Node * +lo64(Node *n) +{ + return nodconst(lo64v(n)); +} + +int +cond(int op) +{ + switch(op) { + case OANDAND: + case OOROR: + case ONOT: + return 1; + + case OEQ: + case ONE: + case OLE: + case OLT: + case OGE: + case OGT: + case OHI: + case OHS: + case OLO: + case OLS: + return 1; + } + return 0; +} diff --git a/src/cmd/6c/div.c b/src/cmd/6c/div.c new file mode 100644 index 000000000..bad6c5e27 --- /dev/null +++ b/src/cmd/6c/div.c @@ -0,0 +1,236 @@ +// Inferno utils/6c/div.c +// http://code.google.com/p/inferno-os/source/browse/utils/6c/div.c +// +// Copyright © 1994-1999 Lucent Technologies Inc. All rights reserved. +// Portions Copyright © 1995-1997 C H Forsyth (forsyth@terzarima.net) +// Portions Copyright © 1997-1999 Vita Nuova Limited +// Portions Copyright © 2000-2007 Vita Nuova Holdings Limited (www.vitanuova.com) +// Portions Copyright © 2004,2006 Bruce Ellis +// Portions Copyright © 2005-2007 C H Forsyth (forsyth@terzarima.net) +// Revisions Copyright © 2000-2007 Lucent Technologies Inc. and others +// Portions Copyright © 2009 The Go Authors. All rights reserved. +// +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to deal +// in the Software without restriction, including without limitation the rights +// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +// copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +// THE SOFTWARE. + +#include "gc.h" + +/* + * Based on: Granlund, T.; Montgomery, P.L. + * "Division by Invariant Integers using Multiplication". + * SIGPLAN Notices, Vol. 29, June 1994, page 61. + */ + +#define TN(n) ((uvlong)1 << (n)) +#define T31 TN(31) +#define T32 TN(32) + +int +multiplier(uint32 d, int p, uvlong *mp) +{ + int l; + uvlong mlo, mhi, tlo, thi; + + l = topbit(d - 1) + 1; + mlo = (((TN(l) - d) << 32) / d) + T32; + if(l + p == 64) + mhi = (((TN(l) + 1 - d) << 32) / d) + T32; + else + mhi = (TN(32 + l) + TN(32 + l - p)) / d; + /*assert(mlo < mhi);*/ + while(l > 0) { + tlo = mlo >> 1; + thi = mhi >> 1; + if(tlo == thi) + break; + mlo = tlo; + mhi = thi; + l--; + } + *mp = mhi; + return l; +} + +int +sdiv(uint32 d, uint32 *mp, int *sp) +{ + int s; + uvlong m; + + s = multiplier(d, 32 - 1, &m); + *mp = m; + *sp = s; + if(m >= T31) + return 1; + else + return 0; +} + +int +udiv(uint32 d, uint32 *mp, int *sp, int *pp) +{ + int p, s; + uvlong m; + + s = multiplier(d, 32, &m); + p = 0; + if(m >= T32) { + while((d & 1) == 0) { + d >>= 1; + p++; + } + s = multiplier(d, 32 - p, &m); + } + *mp = m; + *pp = p; + if(m >= T32) { + /*assert(p == 0);*/ + *sp = s - 1; + return 1; + } + else { + *sp = s; + return 0; + } +} + +void +sdivgen(Node *l, Node *r, Node *ax, Node *dx) +{ + int a, s; + uint32 m; + vlong c; + + c = r->vconst; + if(c < 0) + c = -c; + a = sdiv(c, &m, &s); +//print("a=%d i=%d s=%d m=%ux\n", a, (long)r->vconst, s, m); + gins(AMOVL, nodconst(m), ax); + gins(AIMULL, l, Z); + gins(AMOVL, l, ax); + if(a) + gins(AADDL, ax, dx); + gins(ASHRL, nodconst(31), ax); + gins(ASARL, nodconst(s), dx); + gins(AADDL, ax, dx); + if(r->vconst < 0) + gins(ANEGL, Z, dx); +} + +void +udivgen(Node *l, Node *r, Node *ax, Node *dx) +{ + int a, s, t; + uint32 m; + Node nod; + + a = udiv(r->vconst, &m, &s, &t); +//print("a=%ud i=%d p=%d s=%d m=%ux\n", a, (long)r->vconst, t, s, m); + if(t != 0) { + gins(AMOVL, l, ax); + gins(ASHRL, nodconst(t), ax); + gins(AMOVL, nodconst(m), dx); + gins(AMULL, dx, Z); + } + else if(a) { + if(l->op != OREGISTER) { + regalloc(&nod, l, Z); + gins(AMOVL, l, &nod); + l = &nod; + } + gins(AMOVL, nodconst(m), ax); + gins(AMULL, l, Z); + gins(AADDL, l, dx); + gins(ARCRL, nodconst(1), dx); + if(l == &nod) + regfree(l); + } + else { + gins(AMOVL, nodconst(m), ax); + gins(AMULL, l, Z); + } + if(s != 0) + gins(ASHRL, nodconst(s), dx); +} + +void +sext(Node *d, Node *s, Node *l) +{ + if(s->reg == D_AX && !nodreg(d, Z, D_DX)) { + reg[D_DX]++; + gins(ACDQ, Z, Z); + } + else { + regalloc(d, l, Z); + gins(AMOVL, s, d); + gins(ASARL, nodconst(31), d); + } +} + +void +sdiv2(int32 c, int v, Node *l, Node *n) +{ + Node nod; + + if(v > 0) { + if(v > 1) { + sext(&nod, n, l); + gins(AANDL, nodconst((1 << v) - 1), &nod); + gins(AADDL, &nod, n); + regfree(&nod); + } + else { + gins(ACMPL, n, nodconst(0x80000000)); + gins(ASBBL, nodconst(-1), n); + } + gins(ASARL, nodconst(v), n); + } + if(c < 0) + gins(ANEGL, Z, n); +} + +void +smod2(int32 c, int v, Node *l, Node *n) +{ + Node nod; + + if(c == 1) { + zeroregm(n); + return; + } + + sext(&nod, n, l); + if(v == 0) { + zeroregm(n); + gins(AXORL, &nod, n); + gins(ASUBL, &nod, n); + } + else if(v > 1) { + gins(AANDL, nodconst((1 << v) - 1), &nod); + gins(AADDL, &nod, n); + gins(AANDL, nodconst((1 << v) - 1), n); + gins(ASUBL, &nod, n); + } + else { + gins(AANDL, nodconst(1), n); + gins(AXORL, &nod, n); + gins(ASUBL, &nod, n); + } + regfree(&nod); +} diff --git a/src/cmd/6c/doc.go b/src/cmd/6c/doc.go new file mode 100644 index 000000000..249a8ed80 --- /dev/null +++ b/src/cmd/6c/doc.go @@ -0,0 +1,14 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +/* + +6c is a version of the Plan 9 C compiler. The original is documented at + + http://plan9.bell-labs.com/magic/man2html/1/2c + +Its target architecture is the x86-64, referred to by these tools as amd64. + +*/ +package documentation diff --git a/src/cmd/6c/gc.h b/src/cmd/6c/gc.h new file mode 100644 index 000000000..0c23b115c --- /dev/null +++ b/src/cmd/6c/gc.h @@ -0,0 +1,408 @@ +// Inferno utils/6c/gc.h +// http://code.google.com/p/inferno-os/source/browse/utils/6c/gc.h +// +// Copyright © 1994-1999 Lucent Technologies Inc. All rights reserved. +// Portions Copyright © 1995-1997 C H Forsyth (forsyth@terzarima.net) +// Portions Copyright © 1997-1999 Vita Nuova Limited +// Portions Copyright © 2000-2007 Vita Nuova Holdings Limited (www.vitanuova.com) +// Portions Copyright © 2004,2006 Bruce Ellis +// Portions Copyright © 2005-2007 C H Forsyth (forsyth@terzarima.net) +// Revisions Copyright © 2000-2007 Lucent Technologies Inc. and others +// Portions Copyright © 2009 The Go Authors. All rights reserved. +// +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to deal +// in the Software without restriction, including without limitation the rights +// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +// copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +// THE SOFTWARE. + +#include +#include "../cc/cc.h" +#include "../6l/6.out.h" + +/* + * 6c/amd64 + * Intel 386 with AMD64 extensions + */ +#define SZ_CHAR 1 +#define SZ_SHORT 2 +#define SZ_INT 4 +#define SZ_LONG 4 +#define SZ_IND 8 +#define SZ_FLOAT 4 +#define SZ_VLONG 8 +#define SZ_DOUBLE 8 +#define FNX 100 + +typedef struct Adr Adr; +typedef struct Prog Prog; +typedef struct Case Case; +typedef struct C1 C1; +typedef struct Var Var; +typedef struct Reg Reg; +typedef struct Rgn Rgn; +typedef struct Renv Renv; + +EXTERN struct +{ + Node* regtree; + Node* basetree; + short scale; + short reg; + short ptr; +} idx; + +struct Adr +{ + vlong offset; + double dval; + char sval[NSNAME]; + + Sym* sym; + uchar type; + uchar index; + uchar etype; + uchar scale; /* doubles as width in DATA op */ +}; +#define A ((Adr*)0) + +#define INDEXED 9 +struct Prog +{ + Adr from; + Adr to; + Prog* link; + int32 lineno; + short as; +}; +#define P ((Prog*)0) + +struct Case +{ + Case* link; + vlong val; + int32 label; + char def; + char isv; +}; +#define C ((Case*)0) + +struct C1 +{ + vlong val; + int32 label; +}; + +struct Var +{ + vlong offset; + Sym* sym; + char name; + char etype; +}; + +struct Reg +{ + int32 pc; + int32 rpo; /* reverse post ordering */ + + Bits set; + Bits use1; + Bits use2; + + Bits refbehind; + Bits refahead; + Bits calbehind; + Bits calahead; + Bits regdiff; + Bits act; + + int32 regu; + int32 loop; /* could be shorter */ + + Reg* log5; + int32 active; + + Reg* p1; + Reg* p2; + Reg* p2link; + Reg* s1; + Reg* s2; + Reg* link; + Prog* prog; +}; +#define R ((Reg*)0) + +struct Renv +{ + int safe; + Node base; + Node* saved; + Node* scope; +}; + +#define NRGN 600 +struct Rgn +{ + Reg* enter; + short cost; + short varno; + short regno; +}; + +EXTERN int32 breakpc; +EXTERN int32 nbreak; +EXTERN Case* cases; +EXTERN Node constnode; +EXTERN Node fconstnode; +EXTERN Node vconstnode; +EXTERN int32 continpc; +EXTERN int32 curarg; +EXTERN int32 cursafe; +EXTERN Prog* firstp; +EXTERN Prog* lastp; +EXTERN int32 maxargsafe; +EXTERN int mnstring; +EXTERN int retok; +EXTERN Node* nodrat; +EXTERN Node* nodret; +EXTERN Node* nodsafe; +EXTERN int32 nrathole; +EXTERN int32 nstring; +EXTERN Prog* p; +EXTERN int32 pc; +EXTERN Node lregnode; +EXTERN Node qregnode; +EXTERN char string[NSNAME]; +EXTERN Sym* symrathole; +EXTERN Node znode; +EXTERN Prog zprog; +EXTERN int reg[D_NONE]; +EXTERN int32 exregoffset; +EXTERN int32 exfregoffset; +EXTERN uchar typechlpv[NTYPE]; + +#define BLOAD(r) band(bnot(r->refbehind), r->refahead) +#define BSTORE(r) band(bnot(r->calbehind), r->calahead) +#define LOAD(r) (~r->refbehind.b[z] & r->refahead.b[z]) +#define STORE(r) (~r->calbehind.b[z] & r->calahead.b[z]) + +#define bset(a,n) ((a).b[(n)/32]&(1L<<(n)%32)) + +#define CLOAD 5 +#define CREF 5 +#define CINF 1000 +#define LOOP 3 + +EXTERN Rgn region[NRGN]; +EXTERN Rgn* rgp; +EXTERN int nregion; +EXTERN int nvar; + +EXTERN Bits externs; +EXTERN Bits params; +EXTERN Bits consts; +EXTERN Bits addrs; + +EXTERN int32 regbits; +EXTERN int32 exregbits; + +EXTERN int change; +EXTERN int suppress; + +EXTERN Reg* firstr; +EXTERN Reg* lastr; +EXTERN Reg zreg; +EXTERN Reg* freer; +EXTERN Var var[NVAR]; +EXTERN int32* idom; +EXTERN Reg** rpo2r; +EXTERN int32 maxnr; + +extern char* anames[]; + +/* + * sgen.c + */ +void codgen(Node*, Node*); +void gen(Node*); +void noretval(int); +void usedset(Node*, int); +void xcom(Node*); +void indx(Node*); +int bcomplex(Node*, Node*); +Prog* gtext(Sym*, int32); +vlong argsize(void); + +/* + * cgen.c + */ +void zeroregm(Node*); +void cgen(Node*, Node*); +void reglcgen(Node*, Node*, Node*); +void lcgen(Node*, Node*); +void bcgen(Node*, int); +void boolgen(Node*, int, Node*); +void sugen(Node*, Node*, int32); +int needreg(Node*, int); +int hardconst(Node*); +int immconst(Node*); + +/* + * txt.c + */ +void ginit(void); +void gclean(void); +void nextpc(void); +void gargs(Node*, Node*, Node*); +void garg1(Node*, Node*, Node*, int, Node**); +Node* nodconst(int32); +Node* nodfconst(double); +Node* nodgconst(vlong, Type*); +int nodreg(Node*, Node*, int); +int isreg(Node*, int); +void regret(Node*, Node*); +void regalloc(Node*, Node*, Node*); +void regfree(Node*); +void regialloc(Node*, Node*, Node*); +void regsalloc(Node*, Node*); +void regaalloc1(Node*, Node*); +void regaalloc(Node*, Node*); +void regind(Node*, Node*); +void gprep(Node*, Node*); +void naddr(Node*, Adr*); +void gcmp(int, Node*, vlong); +void gmove(Node*, Node*); +void gins(int a, Node*, Node*); +void gopcode(int, Type*, Node*, Node*); +int samaddr(Node*, Node*); +void gbranch(int); +void patch(Prog*, int32); +int sconst(Node*); +void gpseudo(int, Sym*, Node*); + +/* + * swt.c + */ +int swcmp(const void*, const void*); +void doswit(Node*); +void swit1(C1*, int, int32, Node*); +void cas(void); +void bitload(Node*, Node*, Node*, Node*, Node*); +void bitstore(Node*, Node*, Node*, Node*, Node*); +int32 outstring(char*, int32); +void nullwarn(Node*, Node*); +void sextern(Sym*, Node*, int32, int32); +void gextern(Sym*, Node*, int32, int32); +void outcode(void); +void ieeedtod(Ieee*, double); + +/* + * list + */ +void listinit(void); +int Pconv(Fmt*); +int Aconv(Fmt*); +int Dconv(Fmt*); +int Sconv(Fmt*); +int Rconv(Fmt*); +int Xconv(Fmt*); +int Bconv(Fmt*); + +/* + * reg.c + */ +Reg* rega(void); +int rcmp(const void*, const void*); +void regopt(Prog*); +void addmove(Reg*, int, int, int); +Bits mkvar(Reg*, Adr*); +void prop(Reg*, Bits, Bits); +void loopit(Reg*, int32); +void synch(Reg*, Bits); +uint32 allreg(uint32, Rgn*); +void paint1(Reg*, int); +uint32 paint2(Reg*, int); +void paint3(Reg*, int, int32, int); +void addreg(Adr*, int); + +/* + * peep.c + */ +void peep(void); +void excise(Reg*); +Reg* uniqp(Reg*); +Reg* uniqs(Reg*); +int regtyp(Adr*); +int anyvar(Adr*); +int subprop(Reg*); +int copyprop(Reg*); +int copy1(Adr*, Adr*, Reg*, int); +int copyu(Prog*, Adr*, Adr*); + +int copyas(Adr*, Adr*); +int copyau(Adr*, Adr*); +int copysub(Adr*, Adr*, Adr*, int); +int copysub1(Prog*, Adr*, Adr*, int); + +int32 RtoB(int); +int32 FtoB(int); +int BtoR(int32); +int BtoF(int32); + +#define D_HI D_NONE +#define D_LO D_NONE + +#define isregtype(t) ((t)>= D_AX && (t)<=D_R15) + +/* + * bound + */ +void comtarg(void); + +/* + * com64 + */ +int cond(int); +int com64(Node*); +void com64init(void); +void bool64(Node*); +int32 lo64v(Node*); +int32 hi64v(Node*); +Node* lo64(Node*); +Node* hi64(Node*); + +/* + * div/mul + */ +void sdivgen(Node*, Node*, Node*, Node*); +void udivgen(Node*, Node*, Node*, Node*); +void sdiv2(int32, int, Node*, Node*); +void smod2(int32, int, Node*, Node*); +void mulgen(Type*, Node*, Node*); +void genmuladd(Node*, Node*, int, Node*); +void shiftit(Type*, Node*, Node*); + +#pragma varargck type "A" int +#pragma varargck type "B" Bits +#pragma varargck type "D" Adr* +#pragma varargck type "lD" Adr* +#pragma varargck type "P" Prog* +#pragma varargck type "R" int +#pragma varargck type "S" char* + +#define D_X7 (D_X0+7) + +void fgopcode(int, Node*, Node*, int, int); diff --git a/src/cmd/6c/list.c b/src/cmd/6c/list.c new file mode 100644 index 000000000..4293203c0 --- /dev/null +++ b/src/cmd/6c/list.c @@ -0,0 +1,396 @@ +// Inferno utils/6c/list.c +// http://code.google.com/p/inferno-os/source/browse/utils/6c/list.c +// +// Copyright © 1994-1999 Lucent Technologies Inc. All rights reserved. +// Portions Copyright © 1995-1997 C H Forsyth (forsyth@terzarima.net) +// Portions Copyright © 1997-1999 Vita Nuova Limited +// Portions Copyright © 2000-2007 Vita Nuova Holdings Limited (www.vitanuova.com) +// Portions Copyright © 2004,2006 Bruce Ellis +// Portions Copyright © 2005-2007 C H Forsyth (forsyth@terzarima.net) +// Revisions Copyright © 2000-2007 Lucent Technologies Inc. and others +// Portions Copyright © 2009 The Go Authors. All rights reserved. +// +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to deal +// in the Software without restriction, including without limitation the rights +// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +// copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +// THE SOFTWARE. + +#define EXTERN +#include "gc.h" + +void +listinit(void) +{ + + fmtinstall('A', Aconv); + fmtinstall('B', Bconv); + fmtinstall('P', Pconv); + fmtinstall('S', Sconv); + fmtinstall('D', Dconv); + fmtinstall('R', Rconv); +} + +int +Bconv(Fmt *fp) +{ + char str[STRINGSZ], ss[STRINGSZ], *s; + Bits bits; + int i; + + str[0] = 0; + bits = va_arg(fp->args, Bits); + while(bany(&bits)) { + i = bnum(bits); + if(str[0]) + strcat(str, " "); + if(var[i].sym == S) { + sprint(ss, "$%lld", var[i].offset); + s = ss; + } else + s = var[i].sym->name; + if(strlen(str) + strlen(s) + 1 >= STRINGSZ) + break; + strcat(str, s); + bits.b[i/32] &= ~(1L << (i%32)); + } + return fmtstrcpy(fp, str); +} + +int +Pconv(Fmt *fp) +{ + char str[STRINGSZ]; + Prog *p; + + p = va_arg(fp->args, Prog*); + switch(p->as) { + case ADATA: + sprint(str, "(%L) %A %D/%d,%D", + p->lineno, p->as, &p->from, p->from.scale, &p->to); + break; + + case ATEXT: + if(p->from.scale) { + sprint(str, "(%L) %A %D,%d,%lD", + p->lineno, p->as, &p->from, p->from.scale, &p->to); + break; + } + sprint(str, "(%L) %A %D,%lD", + p->lineno, p->as, &p->from, &p->to); + break; + + default: + sprint(str, "(%L) %A %D,%lD", + p->lineno, p->as, &p->from, &p->to); + break; + } + return fmtstrcpy(fp, str); +} + +int +Aconv(Fmt *fp) +{ + int i; + + i = va_arg(fp->args, int); + return fmtstrcpy(fp, anames[i]); +} + +int +Dconv(Fmt *fp) +{ + char str[STRINGSZ], s[STRINGSZ]; + Adr *a; + int i; + + a = va_arg(fp->args, Adr*); + i = a->type; + + if(fp->flags & FmtLong) { + if(i != D_CONST) { + // ATEXT dst is not constant + sprint(str, "!!%D", a); + goto brk; + } + sprint(str, "$%lld-%lld", a->offset&0xffffffffLL, + (a->offset>>32)&0xffffffffLL); + goto brk; + } + + if(i >= D_INDIR) { + if(a->offset) + sprint(str, "%lld(%R)", a->offset, i-D_INDIR); + else + sprint(str, "(%R)", i-D_INDIR); + goto brk; + } + switch(i) { + + default: + if(a->offset) + sprint(str, "$%lld,%R", a->offset, i); + else + sprint(str, "%R", i); + break; + + case D_NONE: + str[0] = 0; + break; + + case D_BRANCH: + sprint(str, "%lld(PC)", a->offset-pc); + break; + + case D_EXTERN: + sprint(str, "%s+%lld(SB)", a->sym->name, a->offset); + break; + + case D_STATIC: + sprint(str, "%s<>+%lld(SB)", a->sym->name, + a->offset); + break; + + case D_AUTO: + if(a->sym) { + sprint(str, "%s+%lld(SP)", a->sym->name, a->offset); + break; + } + sprint(str, "%lld(SP)", a->offset); + break; + + case D_PARAM: + if(a->sym) { + sprint(str, "%s+%lld(FP)", a->sym->name, a->offset); + break; + } + sprint(str, "%lld(FP)", a->offset); + break; + + case D_CONST: + sprint(str, "$%lld", a->offset); + break; + + case D_FCONST: + sprint(str, "$(%.17e)", a->dval); + break; + + case D_SCONST: + sprint(str, "$\"%S\"", a->sval); + break; + + case D_ADDR: + a->type = a->index; + a->index = D_NONE; + sprint(str, "$%D", a); + a->index = a->type; + a->type = D_ADDR; + goto conv; + } +brk: + if(a->index != D_NONE) { + sprint(s, "(%R*%d)", (int)a->index, (int)a->scale); + strcat(str, s); + } +conv: + return fmtstrcpy(fp, str); +} + +char* regstr[] = +{ + "AL", /* [D_AL] */ + "CL", + "DL", + "BL", + "SPB", + "BPB", + "SIB", + "DIB", + "R8B", + "R9B", + "R10B", + "R11B", + "R12B", + "R13B", + "R14B", + "R15B", + + "AX", /* [D_AX] */ + "CX", + "DX", + "BX", + "SP", + "BP", + "SI", + "DI", + "R8", + "R9", + "R10", + "R11", + "R12", + "R13", + "R14", + "R15", + + "AH", + "CH", + "DH", + "BH", + + "F0", /* [D_F0] */ + "F1", + "F2", + "F3", + "F4", + "F5", + "F6", + "F7", + + "M0", + "M1", + "M2", + "M3", + "M4", + "M5", + "M6", + "M7", + + "X0", + "X1", + "X2", + "X3", + "X4", + "X5", + "X6", + "X7", + "X8", + "X9", + "X10", + "X11", + "X12", + "X13", + "X14", + "X15", + + "CS", /* [D_CS] */ + "SS", + "DS", + "ES", + "FS", + "GS", + + "GDTR", /* [D_GDTR] */ + "IDTR", /* [D_IDTR] */ + "LDTR", /* [D_LDTR] */ + "MSW", /* [D_MSW] */ + "TASK", /* [D_TASK] */ + + "CR0", /* [D_CR] */ + "CR1", + "CR2", + "CR3", + "CR4", + "CR5", + "CR6", + "CR7", + "CR8", + "CR9", + "CR10", + "CR11", + "CR12", + "CR13", + "CR14", + "CR15", + + "DR0", /* [D_DR] */ + "DR1", + "DR2", + "DR3", + "DR4", + "DR5", + "DR6", + "DR7", + + "TR0", /* [D_TR] */ + "TR1", + "TR2", + "TR3", + "TR4", + "TR5", + "TR6", + "TR7", + + "NONE", /* [D_NONE] */ +}; + +int +Rconv(Fmt *fp) +{ + char str[STRINGSZ]; + int r; + + r = va_arg(fp->args, int); + if(r >= D_AL && r <= D_NONE) + sprint(str, "%s", regstr[r-D_AL]); + else + sprint(str, "gok(%d)", r); + + return fmtstrcpy(fp, str); +} + +int +Sconv(Fmt *fp) +{ + int i, c; + char str[STRINGSZ], *p, *a; + + a = va_arg(fp->args, char*); + p = str; + for(i=0; i= 'a' && c <= 'z' || + c >= 'A' && c <= 'Z' || + c >= '0' && c <= '9') { + *p++ = c; + continue; + } + *p++ = '\\'; + switch(c) { + default: + if(c < 040 || c >= 0177) + break; /* not portable */ + p[-1] = c; + continue; + case 0: + *p++ = 'z'; + continue; + case '\\': + case '"': + *p++ = c; + continue; + case '\n': + *p++ = 'n'; + continue; + case '\t': + *p++ = 't'; + continue; + } + *p++ = (c>>6) + '0'; + *p++ = ((c>>3) & 7) + '0'; + *p++ = (c & 7) + '0'; + } + *p = 0; + return fmtstrcpy(fp, str); +} diff --git a/src/cmd/6c/machcap.c b/src/cmd/6c/machcap.c new file mode 100644 index 000000000..820d9a0aa --- /dev/null +++ b/src/cmd/6c/machcap.c @@ -0,0 +1,107 @@ +// Inferno utils/6c/machcap.c +// http://code.google.com/p/inferno-os/source/browse/utils/6c/machcap.c +// +// Copyright © 1994-1999 Lucent Technologies Inc. All rights reserved. +// Portions Copyright © 1995-1997 C H Forsyth (forsyth@terzarima.net) +// Portions Copyright © 1997-1999 Vita Nuova Limited +// Portions Copyright © 2000-2007 Vita Nuova Holdings Limited (www.vitanuova.com) +// Portions Copyright © 2004,2006 Bruce Ellis +// Portions Copyright © 2005-2007 C H Forsyth (forsyth@terzarima.net) +// Revisions Copyright © 2000-2007 Lucent Technologies Inc. and others +// Portions Copyright © 2009 The Go Authors. All rights reserved. +// +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to deal +// in the Software without restriction, including without limitation the rights +// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +// copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +// THE SOFTWARE. + +#include "gc.h" + +int +machcap(Node *n) +{ + + if(n == Z) + return 1; /* test */ + + switch(n->op) { + case OMUL: + case OLMUL: + case OASMUL: + case OASLMUL: + if(typechl[n->type->etype]) + return 1; + if(typev[n->type->etype]) + return 1; + break; + + case OCOM: + case ONEG: + case OADD: + case OAND: + case OOR: + case OSUB: + case OXOR: + case OASHL: + case OLSHR: + case OASHR: + if(typechlv[n->left->type->etype]) + return 1; + break; + + case OCAST: + return 1; + + case OCOND: + case OCOMMA: + case OLIST: + case OANDAND: + case OOROR: + case ONOT: + return 1; + + case OASADD: + case OASSUB: + case OASAND: + case OASOR: + case OASXOR: + return 1; + + case OASASHL: + case OASASHR: + case OASLSHR: + return 1; + + case OPOSTINC: + case OPOSTDEC: + case OPREINC: + case OPREDEC: + return 1; + + case OEQ: + case ONE: + case OLE: + case OGT: + case OLT: + case OGE: + case OHI: + case OHS: + case OLO: + case OLS: + return 1; + } + return 0; +} diff --git a/src/cmd/6c/mul.c b/src/cmd/6c/mul.c new file mode 100644 index 000000000..ab6883e7a --- /dev/null +++ b/src/cmd/6c/mul.c @@ -0,0 +1,458 @@ +// Inferno utils/6c/mul.c +// http://code.google.com/p/inferno-os/source/browse/utils/6c/mul.c +// +// Copyright © 1994-1999 Lucent Technologies Inc. All rights reserved. +// Portions Copyright © 1995-1997 C H Forsyth (forsyth@terzarima.net) +// Portions Copyright © 1997-1999 Vita Nuova Limited +// Portions Copyright © 2000-2007 Vita Nuova Holdings Limited (www.vitanuova.com) +// Portions Copyright © 2004,2006 Bruce Ellis +// Portions Copyright © 2005-2007 C H Forsyth (forsyth@terzarima.net) +// Revisions Copyright © 2000-2007 Lucent Technologies Inc. and others +// Portions Copyright © 2009 The Go Authors. All rights reserved. +// +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to deal +// in the Software without restriction, including without limitation the rights +// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +// copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +// THE SOFTWARE. + +#include "gc.h" + +typedef struct Malg Malg; +typedef struct Mparam Mparam; + +struct Malg +{ + char vals[10]; +}; + +struct Mparam +{ + uint32 value; + char alg; + char neg; + char shift; + char arg; + char off; +}; + +static Mparam multab[32]; +static int mulptr; + +static Malg malgs[] = +{ + {0, 100}, + {-1, 1, 100}, + {-9, -5, -3, 3, 5, 9, 100}, + {6, 10, 12, 18, 20, 24, 36, 40, 72, 100}, + {-8, -4, -2, 2, 4, 8, 100}, +}; + +/* + * return position of lowest 1 + */ +int +lowbit(uint32 v) +{ + int s, i; + uint32 m; + + s = 0; + m = 0xFFFFFFFFUL; + for(i = 16; i > 0; i >>= 1) { + m >>= i; + if((v & m) == 0) { + v >>= i; + s += i; + } + } + return s; +} + +void +genmuladd(Node *d, Node *s, int m, Node *a) +{ + Node nod; + + nod.op = OINDEX; + nod.left = a; + nod.right = s; + nod.scale = m; + nod.type = types[TIND]; + nod.xoffset = 0; + xcom(&nod); + gopcode(OADDR, d->type, &nod, d); +} + +void +mulparam(uint32 m, Mparam *mp) +{ + int c, i, j, n, o, q, s; + int bc, bi, bn, bo, bq, bs, bt; + char *p; + int32 u; + uint32 t; + + bc = bq = 10; + bi = bn = bo = bs = bt = 0; + for(i = 0; i < nelem(malgs); i++) { + for(p = malgs[i].vals, j = 0; (o = p[j]) < 100; j++) + for(s = 0; s < 2; s++) { + c = 10; + q = 10; + u = m - o; + if(u == 0) + continue; + if(s) { + o = -o; + if(o > 0) + continue; + u = -u; + } + n = lowbit(u); + t = (uint32)u >> n; + switch(i) { + case 0: + if(t == 1) { + c = s + 1; + q = 0; + break; + } + switch(t) { + case 3: + case 5: + case 9: + c = s + 1; + if(n) + c++; + q = 0; + break; + } + if(s) + break; + switch(t) { + case 15: + case 25: + case 27: + case 45: + case 81: + c = 2; + if(n) + c++; + q = 1; + break; + } + break; + case 1: + if(t == 1) { + c = 3; + q = 3; + break; + } + switch(t) { + case 3: + case 5: + case 9: + c = 3; + q = 2; + break; + } + break; + case 2: + if(t == 1) { + c = 3; + q = 2; + break; + } + break; + case 3: + if(s) + break; + if(t == 1) { + c = 3; + q = 1; + break; + } + break; + case 4: + if(t == 1) { + c = 3; + q = 0; + break; + } + break; + } + if(c < bc || (c == bc && q > bq)) { + bc = c; + bi = i; + bn = n; + bo = o; + bq = q; + bs = s; + bt = t; + } + } + } + mp->value = m; + if(bc <= 3) { + mp->alg = bi; + mp->shift = bn; + mp->off = bo; + mp->neg = bs; + mp->arg = bt; + } + else + mp->alg = -1; +} + +int +m0(int a) +{ + switch(a) { + case -2: + case 2: + return 2; + case -3: + case 3: + return 2; + case -4: + case 4: + return 4; + case -5: + case 5: + return 4; + case 6: + return 2; + case -8: + case 8: + return 8; + case -9: + case 9: + return 8; + case 10: + return 4; + case 12: + return 2; + case 15: + return 2; + case 18: + return 8; + case 20: + return 4; + case 24: + return 2; + case 25: + return 4; + case 27: + return 2; + case 36: + return 8; + case 40: + return 4; + case 45: + return 4; + case 72: + return 8; + case 81: + return 8; + } + diag(Z, "bad m0"); + return 0; +} + +int +m1(int a) +{ + switch(a) { + case 15: + return 4; + case 25: + return 4; + case 27: + return 8; + case 45: + return 8; + case 81: + return 8; + } + diag(Z, "bad m1"); + return 0; +} + +int +m2(int a) +{ + switch(a) { + case 6: + return 2; + case 10: + return 2; + case 12: + return 4; + case 18: + return 2; + case 20: + return 4; + case 24: + return 8; + case 36: + return 4; + case 40: + return 8; + case 72: + return 8; + } + diag(Z, "bad m2"); + return 0; +} + +void +shiftit(Type *t, Node *s, Node *d) +{ + int32 c; + + c = (int32)s->vconst & 31; + switch(c) { + case 0: + break; + case 1: + gopcode(OADD, t, d, d); + break; + default: + gopcode(OASHL, t, s, d); + } +} + +static int +mulgen1(uint32 v, Node *n) +{ + int i, o; + Mparam *p; + Node nod, nods; + + for(i = 0; i < nelem(multab); i++) { + p = &multab[i]; + if(p->value == v) + goto found; + } + + p = &multab[mulptr]; + if(++mulptr == nelem(multab)) + mulptr = 0; + + mulparam(v, p); + +found: +// print("v=%.x a=%d n=%d s=%d g=%d o=%d \n", p->value, p->alg, p->neg, p->shift, p->arg, p->off); + if(p->alg < 0) + return 0; + + nods = *nodconst(p->shift); + + o = OADD; + if(p->alg > 0) { + regalloc(&nod, n, Z); + if(p->off < 0) + o = OSUB; + } + + switch(p->alg) { + case 0: + switch(p->arg) { + case 1: + shiftit(n->type, &nods, n); + break; + case 15: + case 25: + case 27: + case 45: + case 81: + genmuladd(n, n, m1(p->arg), n); + /* fall thru */ + case 3: + case 5: + case 9: + genmuladd(n, n, m0(p->arg), n); + shiftit(n->type, &nods, n); + break; + default: + goto bad; + } + if(p->neg == 1) + gins(ANEGL, Z, n); + break; + case 1: + switch(p->arg) { + case 1: + gmove(n, &nod); + shiftit(n->type, &nods, &nod); + break; + case 3: + case 5: + case 9: + genmuladd(&nod, n, m0(p->arg), n); + shiftit(n->type, &nods, &nod); + break; + default: + goto bad; + } + if(p->neg) + gopcode(o, n->type, &nod, n); + else { + gopcode(o, n->type, n, &nod); + gmove(&nod, n); + } + break; + case 2: + genmuladd(&nod, n, m0(p->off), n); + shiftit(n->type, &nods, n); + goto comop; + case 3: + genmuladd(&nod, n, m0(p->off), n); + shiftit(n->type, &nods, n); + genmuladd(n, &nod, m2(p->off), n); + break; + case 4: + genmuladd(&nod, n, m0(p->off), nodconst(0)); + shiftit(n->type, &nods, n); + goto comop; + default: + diag(Z, "bad mul alg"); + break; + comop: + if(p->neg) { + gopcode(o, n->type, n, &nod); + gmove(&nod, n); + } + else + gopcode(o, n->type, &nod, n); + } + + if(p->alg > 0) + regfree(&nod); + + return 1; + +bad: + diag(Z, "mulgen botch"); + return 1; +} + +void +mulgen(Type *t, Node *r, Node *n) +{ + if(!mulgen1(r->vconst, n)) + gopcode(OMUL, t, r, n); +} diff --git a/src/cmd/6c/peep.c b/src/cmd/6c/peep.c new file mode 100644 index 000000000..8b82adbf5 --- /dev/null +++ b/src/cmd/6c/peep.c @@ -0,0 +1,890 @@ +// Inferno utils/6c/peep.c +// http://code.google.com/p/inferno-os/source/browse/utils/6c/peep.c +// +// Copyright © 1994-1999 Lucent Technologies Inc. All rights reserved. +// Portions Copyright © 1995-1997 C H Forsyth (forsyth@terzarima.net) +// Portions Copyright © 1997-1999 Vita Nuova Limited +// Portions Copyright © 2000-2007 Vita Nuova Holdings Limited (www.vitanuova.com) +// Portions Copyright © 2004,2006 Bruce Ellis +// Portions Copyright © 2005-2007 C H Forsyth (forsyth@terzarima.net) +// Revisions Copyright © 2000-2007 Lucent Technologies Inc. and others +// Portions Copyright © 2009 The Go Authors. All rights reserved. +// +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to deal +// in the Software without restriction, including without limitation the rights +// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +// copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +// THE SOFTWARE. + +#include "gc.h" + +static int +needc(Prog *p) +{ + while(p != P) { + switch(p->as) { + case AADCL: + case AADCQ: + case ASBBL: + case ASBBQ: + case ARCRL: + case ARCRQ: + return 1; + case AADDL: + case AADDQ: + case ASUBL: + case ASUBQ: + case AJMP: + case ARET: + case ACALL: + return 0; + default: + if(p->to.type == D_BRANCH) + return 0; + } + p = p->link; + } + return 0; +} + +static Reg* +rnops(Reg *r) +{ + Prog *p; + Reg *r1; + + if(r != R) + for(;;){ + p = r->prog; + if(p->as != ANOP || p->from.type != D_NONE || p->to.type != D_NONE) + break; + r1 = uniqs(r); + if(r1 == R) + break; + r = r1; + } + return r; +} + +void +peep(void) +{ + Reg *r, *r1, *r2; + Prog *p, *p1; + int t; + + /* + * complete R structure + */ + t = 0; + for(r=firstr; r!=R; r=r1) { + r1 = r->link; + if(r1 == R) + break; + p = r->prog->link; + while(p != r1->prog) + switch(p->as) { + default: + r2 = rega(); + r->link = r2; + r2->link = r1; + + r2->prog = p; + r2->p1 = r; + r->s1 = r2; + r2->s1 = r1; + r1->p1 = r2; + + r = r2; + t++; + + case ADATA: + case AGLOBL: + case ANAME: + case ASIGNAME: + p = p->link; + } + } + + pc = 0; /* speculating it won't kill */ + +loop1: + + t = 0; + for(r=firstr; r!=R; r=r->link) { + p = r->prog; + switch(p->as) { + case AMOVL: + case AMOVQ: + case AMOVSS: + case AMOVSD: + if(regtyp(&p->to)) + if(regtyp(&p->from)) { + if(copyprop(r)) { + excise(r); + t++; + } else + if(subprop(r) && copyprop(r)) { + excise(r); + t++; + } + } + break; + + case AMOVBLZX: + case AMOVWLZX: + case AMOVBLSX: + case AMOVWLSX: + if(regtyp(&p->to)) { + r1 = rnops(uniqs(r)); + if(r1 != R) { + p1 = r1->prog; + if(p->as == p1->as && p->to.type == p1->from.type){ + p1->as = AMOVL; + t++; + } + } + } + break; + + case AMOVBQSX: + case AMOVBQZX: + case AMOVWQSX: + case AMOVWQZX: + case AMOVLQSX: + case AMOVLQZX: + if(regtyp(&p->to)) { + r1 = rnops(uniqs(r)); + if(r1 != R) { + p1 = r1->prog; + if(p->as == p1->as && p->to.type == p1->from.type){ + p1->as = AMOVQ; + t++; + } + } + } + break; + + case AADDL: + case AADDQ: + case AADDW: + if(p->from.type != D_CONST || needc(p->link)) + break; + if(p->from.offset == -1){ + if(p->as == AADDQ) + p->as = ADECQ; + else if(p->as == AADDL) + p->as = ADECL; + else + p->as = ADECW; + p->from = zprog.from; + } + else if(p->from.offset == 1){ + if(p->as == AADDQ) + p->as = AINCQ; + else if(p->as == AADDL) + p->as = AINCL; + else + p->as = AINCW; + p->from = zprog.from; + } + break; + + case ASUBL: + case ASUBQ: + case ASUBW: + if(p->from.type != D_CONST || needc(p->link)) + break; + if(p->from.offset == -1) { + if(p->as == ASUBQ) + p->as = AINCQ; + else if(p->as == ASUBL) + p->as = AINCL; + else + p->as = AINCW; + p->from = zprog.from; + } + else if(p->from.offset == 1){ + if(p->as == ASUBQ) + p->as = ADECQ; + else if(p->as == ASUBL) + p->as = ADECL; + else + p->as = ADECW; + p->from = zprog.from; + } + break; + } + } + if(t) + goto loop1; +} + +void +excise(Reg *r) +{ + Prog *p; + + p = r->prog; + p->as = ANOP; + p->from = zprog.from; + p->to = zprog.to; +} + +Reg* +uniqp(Reg *r) +{ + Reg *r1; + + r1 = r->p1; + if(r1 == R) { + r1 = r->p2; + if(r1 == R || r1->p2link != R) + return R; + } else + if(r->p2 != R) + return R; + return r1; +} + +Reg* +uniqs(Reg *r) +{ + Reg *r1; + + r1 = r->s1; + if(r1 == R) { + r1 = r->s2; + if(r1 == R) + return R; + } else + if(r->s2 != R) + return R; + return r1; +} + +int +regtyp(Adr *a) +{ + int t; + + t = a->type; + if(t >= D_AX && t <= D_R15) + return 1; + if(t >= D_X0 && t <= D_X0+15) + return 1; + return 0; +} + +/* + * the idea is to substitute + * one register for another + * from one MOV to another + * MOV a, R0 + * ADD b, R0 / no use of R1 + * MOV R0, R1 + * would be converted to + * MOV a, R1 + * ADD b, R1 + * MOV R1, R0 + * hopefully, then the former or latter MOV + * will be eliminated by copy propagation. + */ +int +subprop(Reg *r0) +{ + Prog *p; + Adr *v1, *v2; + Reg *r; + int t; + + p = r0->prog; + v1 = &p->from; + if(!regtyp(v1)) + return 0; + v2 = &p->to; + if(!regtyp(v2)) + return 0; + for(r=uniqp(r0); r!=R; r=uniqp(r)) { + if(uniqs(r) == R) + break; + p = r->prog; + switch(p->as) { + case ACALL: + return 0; + + case AIMULL: + case AIMULQ: + case AIMULW: + if(p->to.type != D_NONE) + break; + + case ADIVB: + case ADIVL: + case ADIVQ: + case ADIVW: + case AIDIVB: + case AIDIVL: + case AIDIVQ: + case AIDIVW: + case AIMULB: + case AMULB: + case AMULL: + case AMULQ: + case AMULW: + + case AROLB: + case AROLL: + case AROLQ: + case AROLW: + case ARORB: + case ARORL: + case ARORQ: + case ARORW: + case ASALB: + case ASALL: + case ASALQ: + case ASALW: + case ASARB: + case ASARL: + case ASARQ: + case ASARW: + case ASHLB: + case ASHLL: + case ASHLQ: + case ASHLW: + case ASHRB: + case ASHRL: + case ASHRQ: + case ASHRW: + + case AREP: + case AREPN: + + case ACWD: + case ACDQ: + case ACQO: + + case ASTOSB: + case ASTOSL: + case ASTOSQ: + case AMOVSB: + case AMOVSL: + case AMOVSQ: + return 0; + + case AMOVL: + case AMOVQ: + if(p->to.type == v1->type) + goto gotit; + break; + } + if(copyau(&p->from, v2) || + copyau(&p->to, v2)) + break; + if(copysub(&p->from, v1, v2, 0) || + copysub(&p->to, v1, v2, 0)) + break; + } + return 0; + +gotit: + copysub(&p->to, v1, v2, 1); + if(debug['P']) { + print("gotit: %D->%D\n%P", v1, v2, r->prog); + if(p->from.type == v2->type) + print(" excise"); + print("\n"); + } + for(r=uniqs(r); r!=r0; r=uniqs(r)) { + p = r->prog; + copysub(&p->from, v1, v2, 1); + copysub(&p->to, v1, v2, 1); + if(debug['P']) + print("%P\n", r->prog); + } + t = v1->type; + v1->type = v2->type; + v2->type = t; + if(debug['P']) + print("%P last\n", r->prog); + return 1; +} + +/* + * The idea is to remove redundant copies. + * v1->v2 F=0 + * (use v2 s/v2/v1/)* + * set v1 F=1 + * use v2 return fail + * ----------------- + * v1->v2 F=0 + * (use v2 s/v2/v1/)* + * set v1 F=1 + * set v2 return success + */ +int +copyprop(Reg *r0) +{ + Prog *p; + Adr *v1, *v2; + Reg *r; + + p = r0->prog; + v1 = &p->from; + v2 = &p->to; + if(copyas(v1, v2)) + return 1; + for(r=firstr; r!=R; r=r->link) + r->active = 0; + return copy1(v1, v2, r0->s1, 0); +} + +int +copy1(Adr *v1, Adr *v2, Reg *r, int f) +{ + int t; + Prog *p; + + if(r->active) { + if(debug['P']) + print("act set; return 1\n"); + return 1; + } + r->active = 1; + if(debug['P']) + print("copy %D->%D f=%d\n", v1, v2, f); + for(; r != R; r = r->s1) { + p = r->prog; + if(debug['P']) + print("%P", p); + if(!f && uniqp(r) == R) { + f = 1; + if(debug['P']) + print("; merge; f=%d", f); + } + t = copyu(p, v2, A); + switch(t) { + case 2: /* rar, cant split */ + if(debug['P']) + print("; %D rar; return 0\n", v2); + return 0; + + case 3: /* set */ + if(debug['P']) + print("; %D set; return 1\n", v2); + return 1; + + case 1: /* used, substitute */ + case 4: /* use and set */ + if(f) { + if(!debug['P']) + return 0; + if(t == 4) + print("; %D used+set and f=%d; return 0\n", v2, f); + else + print("; %D used and f=%d; return 0\n", v2, f); + return 0; + } + if(copyu(p, v2, v1)) { + if(debug['P']) + print("; sub fail; return 0\n"); + return 0; + } + if(debug['P']) + print("; sub %D/%D", v2, v1); + if(t == 4) { + if(debug['P']) + print("; %D used+set; return 1\n", v2); + return 1; + } + break; + } + if(!f) { + t = copyu(p, v1, A); + if(!f && (t == 2 || t == 3 || t == 4)) { + f = 1; + if(debug['P']) + print("; %D set and !f; f=%d", v1, f); + } + } + if(debug['P']) + print("\n"); + if(r->s2) + if(!copy1(v1, v2, r->s2, f)) + return 0; + } + return 1; +} + +/* + * return + * 1 if v only used (and substitute), + * 2 if read-alter-rewrite + * 3 if set + * 4 if set and used + * 0 otherwise (not touched) + */ +int +copyu(Prog *p, Adr *v, Adr *s) +{ + + switch(p->as) { + + default: + if(debug['P']) + print("unknown op %A\n", p->as); + /* SBBL; ADCL; FLD1; SAHF */ + return 2; + + + case ANEGB: + case ANEGW: + case ANEGL: + case ANEGQ: + case ANOTB: + case ANOTW: + case ANOTL: + case ANOTQ: + if(copyas(&p->to, v)) + return 2; + break; + + case ALEAL: /* lhs addr, rhs store */ + case ALEAQ: + if(copyas(&p->from, v)) + return 2; + + + case ANOP: /* rhs store */ + case AMOVL: + case AMOVQ: + case AMOVBLSX: + case AMOVBLZX: + case AMOVBQSX: + case AMOVBQZX: + case AMOVLQSX: + case AMOVLQZX: + case AMOVWLSX: + case AMOVWLZX: + case AMOVWQSX: + case AMOVWQZX: + + case AMOVSS: + case AMOVSD: + case ACVTSD2SL: + case ACVTSD2SQ: + case ACVTSD2SS: + case ACVTSL2SD: + case ACVTSL2SS: + case ACVTSQ2SD: + case ACVTSQ2SS: + case ACVTSS2SD: + case ACVTSS2SL: + case ACVTSS2SQ: + case ACVTTSD2SL: + case ACVTTSD2SQ: + case ACVTTSS2SL: + case ACVTTSS2SQ: + if(copyas(&p->to, v)) { + if(s != A) + return copysub(&p->from, v, s, 1); + if(copyau(&p->from, v)) + return 4; + return 3; + } + goto caseread; + + case AROLB: + case AROLL: + case AROLQ: + case AROLW: + case ARORB: + case ARORL: + case ARORQ: + case ARORW: + case ASALB: + case ASALL: + case ASALQ: + case ASALW: + case ASARB: + case ASARL: + case ASARQ: + case ASARW: + case ASHLB: + case ASHLL: + case ASHLQ: + case ASHLW: + case ASHRB: + case ASHRL: + case ASHRQ: + case ASHRW: + if(copyas(&p->to, v)) + return 2; + if(copyas(&p->from, v)) + if(p->from.type == D_CX) + return 2; + goto caseread; + + case AADDB: /* rhs rar */ + case AADDL: + case AADDQ: + case AADDW: + case AANDB: + case AANDL: + case AANDQ: + case AANDW: + case ADECL: + case ADECQ: + case ADECW: + case AINCL: + case AINCQ: + case AINCW: + case ASUBB: + case ASUBL: + case ASUBQ: + case ASUBW: + case AORB: + case AORL: + case AORQ: + case AORW: + case AXORB: + case AXORL: + case AXORQ: + case AXORW: + case AMOVB: + case AMOVW: + + case AADDSD: + case AADDSS: + case ACMPSD: + case ACMPSS: + case ADIVSD: + case ADIVSS: + case AMAXSD: + case AMAXSS: + case AMINSD: + case AMINSS: + case AMULSD: + case AMULSS: + case ARCPSS: + case ARSQRTSS: + case ASQRTSD: + case ASQRTSS: + case ASUBSD: + case ASUBSS: + case AXORPD: + if(copyas(&p->to, v)) + return 2; + goto caseread; + + case ACMPL: /* read only */ + case ACMPW: + case ACMPB: + case ACMPQ: + + case ACOMISD: + case ACOMISS: + case AUCOMISD: + case AUCOMISS: + caseread: + if(s != A) { + if(copysub(&p->from, v, s, 1)) + return 1; + return copysub(&p->to, v, s, 1); + } + if(copyau(&p->from, v)) + return 1; + if(copyau(&p->to, v)) + return 1; + break; + + case AJGE: /* no reference */ + case AJNE: + case AJLE: + case AJEQ: + case AJHI: + case AJLS: + case AJMI: + case AJPL: + case AJGT: + case AJLT: + case AJCC: + case AJCS: + + case AADJSP: + case AWAIT: + case ACLD: + break; + + case AIMULL: + case AIMULQ: + case AIMULW: + if(p->to.type != D_NONE) { + if(copyas(&p->to, v)) + return 2; + goto caseread; + } + + case ADIVB: + case ADIVL: + case ADIVQ: + case ADIVW: + case AIDIVB: + case AIDIVL: + case AIDIVQ: + case AIDIVW: + case AIMULB: + case AMULB: + case AMULL: + case AMULQ: + case AMULW: + + case ACWD: + case ACDQ: + case ACQO: + if(v->type == D_AX || v->type == D_DX) + return 2; + goto caseread; + + case AREP: + case AREPN: + if(v->type == D_CX) + return 2; + goto caseread; + + case AMOVSB: + case AMOVSL: + case AMOVSQ: + if(v->type == D_DI || v->type == D_SI) + return 2; + goto caseread; + + case ASTOSB: + case ASTOSL: + case ASTOSQ: + if(v->type == D_AX || v->type == D_DI) + return 2; + goto caseread; + + case AJMP: /* funny */ + if(s != A) { + if(copysub(&p->to, v, s, 1)) + return 1; + return 0; + } + if(copyau(&p->to, v)) + return 1; + return 0; + + case ARET: /* funny */ + if(v->type == REGRET || v->type == FREGRET) + return 2; + if(s != A) + return 1; + return 3; + + case ACALL: /* funny */ + if(REGARG >= 0 && v->type == (uchar)REGARG) + return 2; + + if(s != A) { + if(copysub(&p->to, v, s, 1)) + return 1; + return 0; + } + if(copyau(&p->to, v)) + return 4; + return 3; + + case ATEXT: /* funny */ + if(REGARG >= 0 && v->type == (uchar)REGARG) + return 3; + return 0; + } + return 0; +} + +/* + * direct reference, + * could be set/use depending on + * semantics + */ +int +copyas(Adr *a, Adr *v) +{ + if(a->type != v->type) + return 0; + if(regtyp(v)) + return 1; + if(v->type == D_AUTO || v->type == D_PARAM) + if(v->offset == a->offset) + return 1; + return 0; +} + +/* + * either direct or indirect + */ +int +copyau(Adr *a, Adr *v) +{ + + if(copyas(a, v)) + return 1; + if(regtyp(v)) { + if(a->type-D_INDIR == v->type) + return 1; + if(a->index == v->type) + return 1; + } + return 0; +} + +/* + * substitute s for v in a + * return failure to substitute + */ +int +copysub(Adr *a, Adr *v, Adr *s, int f) +{ + int t; + + if(copyas(a, v)) { + t = s->type; + if(t >= D_AX && t <= D_R15 || t >= D_X0 && t <= D_X0+15) { + if(f) + a->type = t; + } + return 0; + } + if(regtyp(v)) { + t = v->type; + if(a->type == t+D_INDIR) { + if((s->type == D_BP || s->type == D_R13) && a->index != D_NONE) + return 1; /* can't use BP-base with index */ + if(f) + a->type = s->type+D_INDIR; +// return 0; + } + if(a->index == t) { + if(f) + a->index = s->type; + return 0; + } + return 0; + } + return 0; +} diff --git a/src/cmd/6c/reg.c b/src/cmd/6c/reg.c new file mode 100644 index 000000000..996128f75 --- /dev/null +++ b/src/cmd/6c/reg.c @@ -0,0 +1,1386 @@ +// Inferno utils/6c/reg.c +// http://code.google.com/p/inferno-os/source/browse/utils/6c/reg.c +// +// Copyright © 1994-1999 Lucent Technologies Inc. All rights reserved. +// Portions Copyright © 1995-1997 C H Forsyth (forsyth@terzarima.net) +// Portions Copyright © 1997-1999 Vita Nuova Limited +// Portions Copyright © 2000-2007 Vita Nuova Holdings Limited (www.vitanuova.com) +// Portions Copyright © 2004,2006 Bruce Ellis +// Portions Copyright © 2005-2007 C H Forsyth (forsyth@terzarima.net) +// Revisions Copyright © 2000-2007 Lucent Technologies Inc. and others +// Portions Copyright © 2009 The Go Authors. All rights reserved. +// +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to deal +// in the Software without restriction, including without limitation the rights +// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +// copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +// THE SOFTWARE. + +#include "gc.h" + +Reg* +rega(void) +{ + Reg *r; + + r = freer; + if(r == R) { + r = alloc(sizeof(*r)); + } else + freer = r->link; + + *r = zreg; + return r; +} + +int +rcmp(const void *a1, const void *a2) +{ + Rgn *p1, *p2; + int c1, c2; + + p1 = (Rgn*)a1; + p2 = (Rgn*)a2; + c1 = p2->cost; + c2 = p1->cost; + if(c1 -= c2) + return c1; + return p2->varno - p1->varno; +} + +void +regopt(Prog *p) +{ + Reg *r, *r1, *r2; + Prog *p1; + int i, z; + int32 initpc, val, npc; + uint32 vreg; + Bits bit; + struct + { + int32 m; + int32 c; + Reg* p; + } log5[6], *lp; + + firstr = R; + lastr = R; + nvar = 0; + regbits = RtoB(D_SP) | RtoB(D_AX) | RtoB(D_X0); + for(z=0; zm = val; + lp->c = 0; + lp->p = R; + val /= 5L; + lp++; + } + val = 0; + for(; p != P; p = p->link) { + switch(p->as) { + case ADATA: + case AGLOBL: + case ANAME: + case ASIGNAME: + continue; + } + r = rega(); + if(firstr == R) { + firstr = r; + lastr = r; + } else { + lastr->link = r; + r->p1 = lastr; + lastr->s1 = r; + lastr = r; + } + r->prog = p; + r->pc = val; + val++; + + lp = log5; + for(i=0; i<5; i++) { + lp->c--; + if(lp->c <= 0) { + lp->c = lp->m; + if(lp->p != R) + lp->p->log5 = r; + lp->p = r; + (lp+1)->c = 0; + break; + } + lp++; + } + + r1 = r->p1; + if(r1 != R) + switch(r1->prog->as) { + case ARET: + case AJMP: + case AIRETL: + case AIRETQ: + r->p1 = R; + r1->s1 = R; + } + + bit = mkvar(r, &p->from); + if(bany(&bit)) + switch(p->as) { + /* + * funny + */ + case ALEAL: + case ALEAQ: + for(z=0; zuse1.b[z] |= bit.b[z]; + break; + } + + bit = mkvar(r, &p->to); + if(bany(&bit)) + switch(p->as) { + default: + diag(Z, "reg: unknown op: %A", p->as); + break; + + /* + * right side read + */ + case ACMPB: + case ACMPL: + case ACMPQ: + case ACMPW: + case ACOMISS: + case ACOMISD: + case AUCOMISS: + case AUCOMISD: + for(z=0; zuse2.b[z] |= bit.b[z]; + break; + + /* + * right side write + */ + case ANOP: + case AMOVL: + case AMOVQ: + case AMOVB: + case AMOVW: + case AMOVBLSX: + case AMOVBLZX: + case AMOVBQSX: + case AMOVBQZX: + case AMOVLQSX: + case AMOVLQZX: + case AMOVWLSX: + case AMOVWLZX: + case AMOVWQSX: + case AMOVWQZX: + + case AMOVSS: + case AMOVSD: + case ACVTSD2SL: + case ACVTSD2SQ: + case ACVTSD2SS: + case ACVTSL2SD: + case ACVTSL2SS: + case ACVTSQ2SD: + case ACVTSQ2SS: + case ACVTSS2SD: + case ACVTSS2SL: + case ACVTSS2SQ: + case ACVTTSD2SL: + case ACVTTSD2SQ: + case ACVTTSS2SL: + case ACVTTSS2SQ: + for(z=0; zset.b[z] |= bit.b[z]; + break; + + /* + * right side read+write + */ + case AADDB: + case AADDL: + case AADDQ: + case AADDW: + case AANDB: + case AANDL: + case AANDQ: + case AANDW: + case ASUBB: + case ASUBL: + case ASUBQ: + case ASUBW: + case AORB: + case AORL: + case AORQ: + case AORW: + case AXORB: + case AXORL: + case AXORQ: + case AXORW: + case ASALB: + case ASALL: + case ASALQ: + case ASALW: + case ASARB: + case ASARL: + case ASARQ: + case ASARW: + case AROLB: + case AROLL: + case AROLQ: + case AROLW: + case ARORB: + case ARORL: + case ARORQ: + case ARORW: + case ASHLB: + case ASHLL: + case ASHLQ: + case ASHLW: + case ASHRB: + case ASHRL: + case ASHRQ: + case ASHRW: + case AIMULL: + case AIMULQ: + case AIMULW: + case ANEGL: + case ANEGQ: + case ANOTL: + case ANOTQ: + case AADCL: + case AADCQ: + case ASBBL: + case ASBBQ: + + case AADDSD: + case AADDSS: + case ACMPSD: + case ACMPSS: + case ADIVSD: + case ADIVSS: + case AMAXSD: + case AMAXSS: + case AMINSD: + case AMINSS: + case AMULSD: + case AMULSS: + case ARCPSS: + case ARSQRTSS: + case ASQRTSD: + case ASQRTSS: + case ASUBSD: + case ASUBSS: + case AXORPD: + for(z=0; zset.b[z] |= bit.b[z]; + r->use2.b[z] |= bit.b[z]; + } + break; + + /* + * funny + */ + case ACALL: + for(z=0; zas) { + case AIMULL: + case AIMULQ: + case AIMULW: + if(p->to.type != D_NONE) + break; + + case AIDIVB: + case AIDIVL: + case AIDIVQ: + case AIDIVW: + case AIMULB: + case ADIVB: + case ADIVL: + case ADIVQ: + case ADIVW: + case AMULB: + case AMULL: + case AMULQ: + case AMULW: + + case ACWD: + case ACDQ: + case ACQO: + r->regu |= RtoB(D_AX) | RtoB(D_DX); + break; + + case AREP: + case AREPN: + case ALOOP: + case ALOOPEQ: + case ALOOPNE: + r->regu |= RtoB(D_CX); + break; + + case AMOVSB: + case AMOVSL: + case AMOVSQ: + case AMOVSW: + case ACMPSB: + case ACMPSL: + case ACMPSQ: + case ACMPSW: + r->regu |= RtoB(D_SI) | RtoB(D_DI); + break; + + case ASTOSB: + case ASTOSL: + case ASTOSQ: + case ASTOSW: + case ASCASB: + case ASCASL: + case ASCASQ: + case ASCASW: + r->regu |= RtoB(D_AX) | RtoB(D_DI); + break; + + case AINSB: + case AINSL: + case AINSW: + case AOUTSB: + case AOUTSL: + case AOUTSW: + r->regu |= RtoB(D_DI) | RtoB(D_DX); + break; + } + } + if(firstr == R) + return; + initpc = pc - val; + npc = val; + + /* + * pass 2 + * turn branch references to pointers + * build back pointers + */ + for(r = firstr; r != R; r = r->link) { + p = r->prog; + if(p->to.type == D_BRANCH) { + val = p->to.offset - initpc; + r1 = firstr; + while(r1 != R) { + r2 = r1->log5; + if(r2 != R && val >= r2->pc) { + r1 = r2; + continue; + } + if(r1->pc == val) + break; + r1 = r1->link; + } + if(r1 == R) { + nearln = p->lineno; + diag(Z, "ref not found\n%P", p); + continue; + } + if(r1 == r) { + nearln = p->lineno; + diag(Z, "ref to self\n%P", p); + continue; + } + r->s2 = r1; + r->p2link = r1->p2; + r1->p2 = r; + } + } + if(debug['R']) { + p = firstr->prog; + print("\n%L %D\n", p->lineno, &p->from); + } + + /* + * pass 2.5 + * find looping structure + */ + for(r = firstr; r != R; r = r->link) + r->active = 0; + change = 0; + loopit(firstr, npc); + if(debug['R'] && debug['v']) { + print("\nlooping structure:\n"); + for(r = firstr; r != R; r = r->link) { + print("%d:%P", r->loop, r->prog); + for(z=0; zuse1.b[z] | + r->use2.b[z] | + r->set.b[z]; + if(bany(&bit)) { + print("\t"); + if(bany(&r->use1)) + print(" u1=%B", r->use1); + if(bany(&r->use2)) + print(" u2=%B", r->use2); + if(bany(&r->set)) + print(" st=%B", r->set); + } + print("\n"); + } + } + + /* + * pass 3 + * iterate propagating usage + * back until flow graph is complete + */ +loop1: + change = 0; + for(r = firstr; r != R; r = r->link) + r->active = 0; + for(r = firstr; r != R; r = r->link) + if(r->prog->as == ARET) + prop(r, zbits, zbits); +loop11: + /* pick up unreachable code */ + i = 0; + for(r = firstr; r != R; r = r1) { + r1 = r->link; + if(r1 && r1->active && !r->active) { + prop(r, zbits, zbits); + i = 1; + } + } + if(i) + goto loop11; + if(change) + goto loop1; + + + /* + * pass 4 + * iterate propagating register/variable synchrony + * forward until graph is complete + */ +loop2: + change = 0; + for(r = firstr; r != R; r = r->link) + r->active = 0; + synch(firstr, zbits); + if(change) + goto loop2; + + + /* + * pass 5 + * isolate regions + * calculate costs (paint1) + */ + r = firstr; + if(r) { + for(z=0; zrefahead.b[z] | r->calahead.b[z]) & + ~(externs.b[z] | params.b[z] | addrs.b[z] | consts.b[z]); + if(bany(&bit)) { + nearln = r->prog->lineno; + warn(Z, "used and not set: %B", bit); + if(debug['R'] && !debug['w']) + print("used and not set: %B\n", bit); + } + } + if(debug['R'] && debug['v']) + print("\nprop structure:\n"); + for(r = firstr; r != R; r = r->link) + r->act = zbits; + rgp = region; + nregion = 0; + for(r = firstr; r != R; r = r->link) { + if(debug['R'] && debug['v']) { + print("%P\t", r->prog); + if(bany(&r->set)) + print("s:%B ", r->set); + if(bany(&r->refahead)) + print("ra:%B ", r->refahead); + if(bany(&r->calahead)) + print("ca:%B ", r->calahead); + print("\n"); + } + for(z=0; zset.b[z] & + ~(r->refahead.b[z] | r->calahead.b[z] | addrs.b[z]); + if(bany(&bit)) { + nearln = r->prog->lineno; + warn(Z, "set and not used: %B", bit); + if(debug['R']) + print("set and not used: %B\n", bit); + excise(r); + } + for(z=0; zact.b[z] | addrs.b[z]); + while(bany(&bit)) { + i = bnum(bit); + rgp->enter = r; + rgp->varno = i; + change = 0; + if(debug['R'] && debug['v']) + print("\n"); + paint1(r, i); + bit.b[i/32] &= ~(1L<<(i%32)); + if(change <= 0) { + if(debug['R']) + print("%L$%d: %B\n", + r->prog->lineno, change, blsh(i)); + continue; + } + rgp->cost = change; + nregion++; + if(nregion >= NRGN) { + warn(Z, "too many regions"); + goto brk; + } + rgp++; + } + } +brk: + qsort(region, nregion, sizeof(region[0]), rcmp); + + /* + * pass 6 + * determine used registers (paint2) + * replace code (paint3) + */ + rgp = region; + for(i=0; ivarno); + vreg = paint2(rgp->enter, rgp->varno); + vreg = allreg(vreg, rgp); + if(debug['R']) { + print("%L$%d %R: %B\n", + rgp->enter->prog->lineno, + rgp->cost, + rgp->regno, + bit); + } + if(rgp->regno != 0) + paint3(rgp->enter, rgp->varno, vreg, rgp->regno); + rgp++; + } + /* + * pass 7 + * peep-hole on basic block + */ + if(!debug['R'] || debug['P']) + peep(); + + /* + * pass 8 + * recalculate pc + */ + val = initpc; + for(r = firstr; r != R; r = r1) { + r->pc = val; + p = r->prog; + p1 = P; + r1 = r->link; + if(r1 != R) + p1 = r1->prog; + for(; p != p1; p = p->link) { + switch(p->as) { + default: + val++; + break; + + case ANOP: + case ADATA: + case AGLOBL: + case ANAME: + case ASIGNAME: + break; + } + } + } + pc = val; + + /* + * fix up branches + */ + if(debug['R']) + if(bany(&addrs)) + print("addrs: %B\n", addrs); + + r1 = 0; /* set */ + for(r = firstr; r != R; r = r->link) { + p = r->prog; + if(p->to.type == D_BRANCH) + p->to.offset = r->s2->pc; + r1 = r; + } + + /* + * last pass + * eliminate nops + * free aux structures + */ + for(p = firstr->prog; p != P; p = p->link){ + while(p->link && p->link->as == ANOP) + p->link = p->link->link; + } + if(r1 != R) { + r1->link = freer; + freer = firstr; + } +} + +/* + * add mov b,rn + * just after r + */ +void +addmove(Reg *r, int bn, int rn, int f) +{ + Prog *p, *p1; + Adr *a; + Var *v; + + p1 = alloc(sizeof(*p1)); + *p1 = zprog; + p = r->prog; + + p1->link = p->link; + p->link = p1; + p1->lineno = p->lineno; + + v = var + bn; + + a = &p1->to; + a->sym = v->sym; + a->offset = v->offset; + a->etype = v->etype; + a->type = v->name; + + p1->as = AMOVL; + if(v->etype == TCHAR || v->etype == TUCHAR) + p1->as = AMOVB; + if(v->etype == TSHORT || v->etype == TUSHORT) + p1->as = AMOVW; + if(v->etype == TVLONG || v->etype == TUVLONG || v->etype == TIND) + p1->as = AMOVQ; + if(v->etype == TFLOAT) + p1->as = AMOVSS; + if(v->etype == TDOUBLE) + p1->as = AMOVSD; + + p1->from.type = rn; + if(!f) { + p1->from = *a; + *a = zprog.from; + a->type = rn; + if(v->etype == TUCHAR) + p1->as = AMOVB; + if(v->etype == TUSHORT) + p1->as = AMOVW; + } + if(debug['R']) + print("%P\t.a%P\n", p, p1); +} + +uint32 +doregbits(int r) +{ + uint32 b; + + b = 0; + if(r >= D_INDIR) + r -= D_INDIR; + if(r >= D_AX && r <= D_R15) + b |= RtoB(r); + else + if(r >= D_AL && r <= D_R15B) + b |= RtoB(r-D_AL+D_AX); + else + if(r >= D_AH && r <= D_BH) + b |= RtoB(r-D_AH+D_AX); + else + if(r >= D_X0 && r <= D_X0+15) + b |= FtoB(r); + return b; +} + +Bits +mkvar(Reg *r, Adr *a) +{ + Var *v; + int i, t, n, et, z; + int32 o; + Bits bit; + Sym *s; + + /* + * mark registers used + */ + t = a->type; + r->regu |= doregbits(t); + r->regu |= doregbits(a->index); + + switch(t) { + default: + goto none; + case D_ADDR: + a->type = a->index; + bit = mkvar(r, a); + for(z=0; ztype = t; + goto none; + case D_EXTERN: + case D_STATIC: + case D_PARAM: + case D_AUTO: + n = t; + break; + } + s = a->sym; + if(s == S) + goto none; + if(s->name[0] == '.') + goto none; + et = a->etype; + o = a->offset; + v = var; + for(i=0; isym) + if(n == v->name) + if(o == v->offset) + goto out; + v++; + } + if(nvar >= NVAR) { + if(debug['w'] > 1 && s) + warn(Z, "variable not optimized: %s", s->name); + goto none; + } + i = nvar; + nvar++; + v = &var[i]; + v->sym = s; + v->offset = o; + v->name = n; + v->etype = et; + if(debug['R']) + print("bit=%2d et=%2d %D\n", i, et, a); + +out: + bit = blsh(i); + if(n == D_EXTERN || n == D_STATIC) + for(z=0; zetype != et || !(typechlpfd[et] || typev[et])) /* funny punning */ + for(z=0; zp1) { + for(z=0; zrefahead.b[z]; + if(ref.b[z] != r1->refahead.b[z]) { + r1->refahead.b[z] = ref.b[z]; + change++; + } + cal.b[z] |= r1->calahead.b[z]; + if(cal.b[z] != r1->calahead.b[z]) { + r1->calahead.b[z] = cal.b[z]; + change++; + } + } + switch(r1->prog->as) { + case ACALL: + for(z=0; zset.b[z]) | + r1->use1.b[z] | r1->use2.b[z]; + cal.b[z] &= ~(r1->set.b[z] | r1->use1.b[z] | r1->use2.b[z]); + r1->refbehind.b[z] = ref.b[z]; + r1->calbehind.b[z] = cal.b[z]; + } + if(r1->active) + break; + r1->active = 1; + } + for(; r != r1; r = r->p1) + for(r2 = r->p2; r2 != R; r2 = r2->p2link) + prop(r2, r->refbehind, r->calbehind); +} + +/* + * find looping structure + * + * 1) find reverse postordering + * 2) find approximate dominators, + * the actual dominators if the flow graph is reducible + * otherwise, dominators plus some other non-dominators. + * See Matthew S. Hecht and Jeffrey D. Ullman, + * "Analysis of a Simple Algorithm for Global Data Flow Problems", + * Conf. Record of ACM Symp. on Principles of Prog. Langs, Boston, Massachusetts, + * Oct. 1-3, 1973, pp. 207-217. + * 3) find all nodes with a predecessor dominated by the current node. + * such a node is a loop head. + * recursively, all preds with a greater rpo number are in the loop + */ +int32 +postorder(Reg *r, Reg **rpo2r, int32 n) +{ + Reg *r1; + + r->rpo = 1; + r1 = r->s1; + if(r1 && !r1->rpo) + n = postorder(r1, rpo2r, n); + r1 = r->s2; + if(r1 && !r1->rpo) + n = postorder(r1, rpo2r, n); + rpo2r[n] = r; + n++; + return n; +} + +int32 +rpolca(int32 *idom, int32 rpo1, int32 rpo2) +{ + int32 t; + + if(rpo1 == -1) + return rpo2; + while(rpo1 != rpo2){ + if(rpo1 > rpo2){ + t = rpo2; + rpo2 = rpo1; + rpo1 = t; + } + while(rpo1 < rpo2){ + t = idom[rpo2]; + if(t >= rpo2) + fatal(Z, "bad idom"); + rpo2 = t; + } + } + return rpo1; +} + +int +doms(int32 *idom, int32 r, int32 s) +{ + while(s > r) + s = idom[s]; + return s == r; +} + +int +loophead(int32 *idom, Reg *r) +{ + int32 src; + + src = r->rpo; + if(r->p1 != R && doms(idom, src, r->p1->rpo)) + return 1; + for(r = r->p2; r != R; r = r->p2link) + if(doms(idom, src, r->rpo)) + return 1; + return 0; +} + +void +loopmark(Reg **rpo2r, int32 head, Reg *r) +{ + if(r->rpo < head || r->active == head) + return; + r->active = head; + r->loop += LOOP; + if(r->p1 != R) + loopmark(rpo2r, head, r->p1); + for(r = r->p2; r != R; r = r->p2link) + loopmark(rpo2r, head, r); +} + +void +loopit(Reg *r, int32 nr) +{ + Reg *r1; + int32 i, d, me; + + if(nr > maxnr) { + rpo2r = alloc(nr * sizeof(Reg*)); + idom = alloc(nr * sizeof(int32)); + maxnr = nr; + } + + d = postorder(r, rpo2r, 0); + if(d > nr) + fatal(Z, "too many reg nodes"); + nr = d; + for(i = 0; i < nr / 2; i++){ + r1 = rpo2r[i]; + rpo2r[i] = rpo2r[nr - 1 - i]; + rpo2r[nr - 1 - i] = r1; + } + for(i = 0; i < nr; i++) + rpo2r[i]->rpo = i; + + idom[0] = 0; + for(i = 0; i < nr; i++){ + r1 = rpo2r[i]; + me = r1->rpo; + d = -1; + if(r1->p1 != R && r1->p1->rpo < me) + d = r1->p1->rpo; + for(r1 = r1->p2; r1 != nil; r1 = r1->p2link) + if(r1->rpo < me) + d = rpolca(idom, d, r1->rpo); + idom[i] = d; + } + + for(i = 0; i < nr; i++){ + r1 = rpo2r[i]; + r1->loop++; + if(r1->p2 != R && loophead(idom, r1)) + loopmark(rpo2r, i, r1); + } +} + +void +synch(Reg *r, Bits dif) +{ + Reg *r1; + int z; + + for(r1 = r; r1 != R; r1 = r1->s1) { + for(z=0; zrefbehind.b[z] & r1->refahead.b[z])) | + r1->set.b[z] | r1->regdiff.b[z]; + if(dif.b[z] != r1->regdiff.b[z]) { + r1->regdiff.b[z] = dif.b[z]; + change++; + } + } + if(r1->active) + break; + r1->active = 1; + for(z=0; zcalbehind.b[z] & r1->calahead.b[z]); + if(r1->s2 != R) + synch(r1->s2, dif); + } +} + +uint32 +allreg(uint32 b, Rgn *r) +{ + Var *v; + int i; + + v = var + r->varno; + r->regno = 0; + switch(v->etype) { + + default: + diag(Z, "unknown etype %d/%d", bitno(b), v->etype); + break; + + case TCHAR: + case TUCHAR: + case TSHORT: + case TUSHORT: + case TINT: + case TUINT: + case TLONG: + case TULONG: + case TVLONG: + case TUVLONG: + case TIND: + case TARRAY: + i = BtoR(~b); + if(i && r->cost > 0) { + r->regno = i; + return RtoB(i); + } + break; + + case TDOUBLE: + case TFLOAT: + i = BtoF(~b); + if(i && r->cost > 0) { + r->regno = i; + return FtoB(i); + } + break; + } + return 0; +} + +void +paint1(Reg *r, int bn) +{ + Reg *r1; + Prog *p; + int z; + uint32 bb; + + z = bn/32; + bb = 1L<<(bn%32); + if(r->act.b[z] & bb) + return; + for(;;) { + if(!(r->refbehind.b[z] & bb)) + break; + r1 = r->p1; + if(r1 == R) + break; + if(!(r1->refahead.b[z] & bb)) + break; + if(r1->act.b[z] & bb) + break; + r = r1; + } + + if(LOAD(r) & ~(r->set.b[z]&~(r->use1.b[z]|r->use2.b[z])) & bb) { + change -= CLOAD * r->loop; + if(debug['R'] && debug['v']) + print("%d%P\td %B $%d\n", r->loop, + r->prog, blsh(bn), change); + } + for(;;) { + r->act.b[z] |= bb; + p = r->prog; + + if(r->use1.b[z] & bb) { + change += CREF * r->loop; + if(debug['R'] && debug['v']) + print("%d%P\tu1 %B $%d\n", r->loop, + p, blsh(bn), change); + } + + if((r->use2.b[z]|r->set.b[z]) & bb) { + change += CREF * r->loop; + if(debug['R'] && debug['v']) + print("%d%P\tu2 %B $%d\n", r->loop, + p, blsh(bn), change); + } + + if(STORE(r) & r->regdiff.b[z] & bb) { + change -= CLOAD * r->loop; + if(debug['R'] && debug['v']) + print("%d%P\tst %B $%d\n", r->loop, + p, blsh(bn), change); + } + + if(r->refbehind.b[z] & bb) + for(r1 = r->p2; r1 != R; r1 = r1->p2link) + if(r1->refahead.b[z] & bb) + paint1(r1, bn); + + if(!(r->refahead.b[z] & bb)) + break; + r1 = r->s2; + if(r1 != R) + if(r1->refbehind.b[z] & bb) + paint1(r1, bn); + r = r->s1; + if(r == R) + break; + if(r->act.b[z] & bb) + break; + if(!(r->refbehind.b[z] & bb)) + break; + } +} + +uint32 +regset(Reg *r, uint32 bb) +{ + uint32 b, set; + Adr v; + int c; + + set = 0; + v = zprog.from; + while(b = bb & ~(bb-1)) { + v.type = b & 0xFFFF? BtoR(b): BtoF(b); + if(v.type == 0) + diag(Z, "zero v.type for %#ux", b); + c = copyu(r->prog, &v, A); + if(c == 3) + set |= b; + bb &= ~b; + } + return set; +} + +uint32 +reguse(Reg *r, uint32 bb) +{ + uint32 b, set; + Adr v; + int c; + + set = 0; + v = zprog.from; + while(b = bb & ~(bb-1)) { + v.type = b & 0xFFFF? BtoR(b): BtoF(b); + c = copyu(r->prog, &v, A); + if(c == 1 || c == 2 || c == 4) + set |= b; + bb &= ~b; + } + return set; +} + +uint32 +paint2(Reg *r, int bn) +{ + Reg *r1; + int z; + uint32 bb, vreg, x; + + z = bn/32; + bb = 1L << (bn%32); + vreg = regbits; + if(!(r->act.b[z] & bb)) + return vreg; + for(;;) { + if(!(r->refbehind.b[z] & bb)) + break; + r1 = r->p1; + if(r1 == R) + break; + if(!(r1->refahead.b[z] & bb)) + break; + if(!(r1->act.b[z] & bb)) + break; + r = r1; + } + for(;;) { + r->act.b[z] &= ~bb; + + vreg |= r->regu; + + if(r->refbehind.b[z] & bb) + for(r1 = r->p2; r1 != R; r1 = r1->p2link) + if(r1->refahead.b[z] & bb) + vreg |= paint2(r1, bn); + + if(!(r->refahead.b[z] & bb)) + break; + r1 = r->s2; + if(r1 != R) + if(r1->refbehind.b[z] & bb) + vreg |= paint2(r1, bn); + r = r->s1; + if(r == R) + break; + if(!(r->act.b[z] & bb)) + break; + if(!(r->refbehind.b[z] & bb)) + break; + } + + bb = vreg; + for(; r; r=r->s1) { + x = r->regu & ~bb; + if(x) { + vreg |= reguse(r, x); + bb |= regset(r, x); + } + } + return vreg; +} + +void +paint3(Reg *r, int bn, int32 rb, int rn) +{ + Reg *r1; + Prog *p; + int z; + uint32 bb; + + z = bn/32; + bb = 1L << (bn%32); + if(r->act.b[z] & bb) + return; + for(;;) { + if(!(r->refbehind.b[z] & bb)) + break; + r1 = r->p1; + if(r1 == R) + break; + if(!(r1->refahead.b[z] & bb)) + break; + if(r1->act.b[z] & bb) + break; + r = r1; + } + + if(LOAD(r) & ~(r->set.b[z] & ~(r->use1.b[z]|r->use2.b[z])) & bb) + addmove(r, bn, rn, 0); + for(;;) { + r->act.b[z] |= bb; + p = r->prog; + + if(r->use1.b[z] & bb) { + if(debug['R']) + print("%P", p); + addreg(&p->from, rn); + if(debug['R']) + print("\t.c%P\n", p); + } + if((r->use2.b[z]|r->set.b[z]) & bb) { + if(debug['R']) + print("%P", p); + addreg(&p->to, rn); + if(debug['R']) + print("\t.c%P\n", p); + } + + if(STORE(r) & r->regdiff.b[z] & bb) + addmove(r, bn, rn, 1); + r->regu |= rb; + + if(r->refbehind.b[z] & bb) + for(r1 = r->p2; r1 != R; r1 = r1->p2link) + if(r1->refahead.b[z] & bb) + paint3(r1, bn, rb, rn); + + if(!(r->refahead.b[z] & bb)) + break; + r1 = r->s2; + if(r1 != R) + if(r1->refbehind.b[z] & bb) + paint3(r1, bn, rb, rn); + r = r->s1; + if(r == R) + break; + if(r->act.b[z] & bb) + break; + if(!(r->refbehind.b[z] & bb)) + break; + } +} + +void +addreg(Adr *a, int rn) +{ + + a->sym = 0; + a->offset = 0; + a->type = rn; +} + +int32 +RtoB(int r) +{ + + if(r < D_AX || r > D_R15) + return 0; + return 1L << (r-D_AX); +} + +int +BtoR(int32 b) +{ + + b &= 0xffffL; + if(b == 0) + return 0; + return bitno(b) + D_AX; +} + +/* + * bit reg + * 16 X5 + * 17 X6 + * 18 X7 + */ +int32 +FtoB(int f) +{ + if(f < FREGMIN || f > FREGEXT) + return 0; + return 1L << (f - FREGMIN + 16); +} + +int +BtoF(int32 b) +{ + + b &= 0x70000L; + if(b == 0) + return 0; + return bitno(b) - 16 + FREGMIN; +} diff --git a/src/cmd/6c/sgen.c b/src/cmd/6c/sgen.c new file mode 100644 index 000000000..42045f8fa --- /dev/null +++ b/src/cmd/6c/sgen.c @@ -0,0 +1,485 @@ +// Inferno utils/6c/sgen.c +// http://code.google.com/p/inferno-os/source/browse/utils/6c/sgen.c +// +// Copyright © 1994-1999 Lucent Technologies Inc. All rights reserved. +// Portions Copyright © 1995-1997 C H Forsyth (forsyth@terzarima.net) +// Portions Copyright © 1997-1999 Vita Nuova Limited +// Portions Copyright © 2000-2007 Vita Nuova Holdings Limited (www.vitanuova.com) +// Portions Copyright © 2004,2006 Bruce Ellis +// Portions Copyright © 2005-2007 C H Forsyth (forsyth@terzarima.net) +// Revisions Copyright © 2000-2007 Lucent Technologies Inc. and others +// Portions Copyright © 2009 The Go Authors. All rights reserved. +// +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to deal +// in the Software without restriction, including without limitation the rights +// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +// copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +// THE SOFTWARE. + +#include "gc.h" + +Prog* +gtext(Sym *s, int32 stkoff) +{ + vlong v; + + v = 0; + if(!(textflag & NOSPLIT)) + v |= argsize() << 32; + v |= stkoff & 0xffffffff; + if((textflag & NOSPLIT) && stkoff >= 128) + yyerror("stack frame too large for NOSPLIT function"); + + gpseudo(ATEXT, s, nodgconst(v, types[TVLONG])); + return p; +} + +void +noretval(int n) +{ + + if(n & 1) { + gins(ANOP, Z, Z); + p->to.type = REGRET; + } + if(n & 2) { + gins(ANOP, Z, Z); + p->to.type = FREGRET; + } +} + +/* welcome to commute */ +static void +commute(Node *n) +{ + Node *l, *r; + + l = n->left; + r = n->right; + if(r->complex > l->complex) { + n->left = r; + n->right = l; + } +} + +void +indexshift(Node *n) +{ + int g; + + if(!typechlpv[n->type->etype]) + return; + simplifyshift(n); + if(n->op == OASHL && n->right->op == OCONST){ + g = vconst(n->right); + if(g >= 0 && g <= 3) + n->addable = 7; + } +} + +/* + * calculate addressability as follows + * NAME ==> 10/11 name+value(SB/SP) + * REGISTER ==> 12 register + * CONST ==> 20 $value + * *(20) ==> 21 value + * &(10) ==> 13 $name+value(SB) + * &(11) ==> 1 $name+value(SP) + * (13) + (20) ==> 13 fold constants + * (1) + (20) ==> 1 fold constants + * *(13) ==> 10 back to name + * *(1) ==> 11 back to name + * + * (20) * (X) ==> 7 multiplier in indexing + * (X,7) + (13,1) ==> 8 adder in indexing (addresses) + * (8) ==> &9(OINDEX) index, almost addressable + * + * calculate complexity (number of registers) + */ +void +xcom(Node *n) +{ + Node *l, *r; + int g; + + if(n == Z) + return; + l = n->left; + r = n->right; + n->complex = 0; + n->addable = 0; + switch(n->op) { + case OCONST: + n->addable = 20; + break; + + case ONAME: + n->addable = 10; + if(n->class == CPARAM || n->class == CAUTO) + n->addable = 11; + break; + + case OEXREG: + n->addable = 0; + break; + + case OREGISTER: + n->addable = 12; + break; + + case OINDREG: + n->addable = 12; + break; + + case OADDR: + xcom(l); + if(l->addable == 10) + n->addable = 13; + else + if(l->addable == 11) + n->addable = 1; + break; + + case OADD: + xcom(l); + xcom(r); + if(n->type->etype != TIND) + break; + + switch(r->addable) { + case 20: + switch(l->addable) { + case 1: + case 13: + commadd: + l->type = n->type; + *n = *l; + l = new(0, Z, Z); + *l = *(n->left); + l->xoffset += r->vconst; + n->left = l; + r = n->right; + goto brk; + } + break; + + case 1: + case 13: + case 10: + case 11: + /* l is the base, r is the index */ + if(l->addable != 20) + n->addable = 8; + break; + } + switch(l->addable) { + case 20: + switch(r->addable) { + case 13: + case 1: + r = n->left; + l = n->right; + n->left = l; + n->right = r; + goto commadd; + } + break; + + case 13: + case 1: + case 10: + case 11: + /* r is the base, l is the index */ + if(r->addable != 20) + n->addable = 8; + break; + } + if(n->addable == 8 && !side(n)) { + indx(n); + l = new1(OINDEX, idx.basetree, idx.regtree); + l->scale = idx.scale; + l->addable = 9; + l->complex = l->right->complex; + l->type = l->left->type; + n->op = OADDR; + n->left = l; + n->right = Z; + n->addable = 8; + break; + } + break; + + case OINDEX: + xcom(l); + xcom(r); + n->addable = 9; + break; + + case OIND: + xcom(l); + if(l->op == OADDR) { + l = l->left; + l->type = n->type; + *n = *l; + return; + } + switch(l->addable) { + case 20: + n->addable = 21; + break; + case 1: + n->addable = 11; + break; + case 13: + n->addable = 10; + break; + } + break; + + case OASHL: + xcom(l); + xcom(r); + indexshift(n); + break; + + case OMUL: + case OLMUL: + xcom(l); + xcom(r); + g = vlog(l); + if(g >= 0) { + n->left = r; + n->right = l; + l = r; + r = n->right; + } + g = vlog(r); + if(g >= 0) { + n->op = OASHL; + r->vconst = g; + r->type = types[TINT]; + indexshift(n); + break; + } + commute(n); + break; + + case OASLDIV: + xcom(l); + xcom(r); + g = vlog(r); + if(g >= 0) { + n->op = OASLSHR; + r->vconst = g; + r->type = types[TINT]; + } + break; + + case OLDIV: + xcom(l); + xcom(r); + g = vlog(r); + if(g >= 0) { + n->op = OLSHR; + r->vconst = g; + r->type = types[TINT]; + indexshift(n); + break; + } + break; + + case OASLMOD: + xcom(l); + xcom(r); + g = vlog(r); + if(g >= 0) { + n->op = OASAND; + r->vconst--; + } + break; + + case OLMOD: + xcom(l); + xcom(r); + g = vlog(r); + if(g >= 0) { + n->op = OAND; + r->vconst--; + } + break; + + case OASMUL: + case OASLMUL: + xcom(l); + xcom(r); + g = vlog(r); + if(g >= 0) { + n->op = OASASHL; + r->vconst = g; + } + break; + + case OLSHR: + case OASHR: + xcom(l); + xcom(r); + indexshift(n); + break; + + default: + if(l != Z) + xcom(l); + if(r != Z) + xcom(r); + break; + } +brk: + if(n->addable >= 10) + return; + if(l != Z) + n->complex = l->complex; + if(r != Z) { + if(r->complex == n->complex) + n->complex = r->complex+1; + else + if(r->complex > n->complex) + n->complex = r->complex; + } + if(n->complex == 0) + n->complex++; + + switch(n->op) { + + case OFUNC: + n->complex = FNX; + break; + + case OCAST: + if(l->type->etype == TUVLONG && typefd[n->type->etype]) + n->complex += 2; + break; + + case OLMOD: + case OMOD: + case OLMUL: + case OLDIV: + case OMUL: + case ODIV: + case OASLMUL: + case OASLDIV: + case OASLMOD: + case OASMUL: + case OASDIV: + case OASMOD: + if(r->complex >= l->complex) { + n->complex = l->complex + 3; + if(r->complex > n->complex) + n->complex = r->complex; + } else { + n->complex = r->complex + 3; + if(l->complex > n->complex) + n->complex = l->complex; + } + break; + + case OLSHR: + case OASHL: + case OASHR: + case OASLSHR: + case OASASHL: + case OASASHR: + if(r->complex >= l->complex) { + n->complex = l->complex + 2; + if(r->complex > n->complex) + n->complex = r->complex; + } else { + n->complex = r->complex + 2; + if(l->complex > n->complex) + n->complex = l->complex; + } + break; + + case OADD: + case OXOR: + case OAND: + case OOR: + /* + * immediate operators, make const on right + */ + if(l->op == OCONST) { + n->left = r; + n->right = l; + } + break; + + case OEQ: + case ONE: + case OLE: + case OLT: + case OGE: + case OGT: + case OHI: + case OHS: + case OLO: + case OLS: + /* + * compare operators, make const on left + */ + if(r->op == OCONST) { + n->left = r; + n->right = l; + n->op = invrel[relindex(n->op)]; + } + break; + } +} + +void +indx(Node *n) +{ + Node *l, *r; + + if(debug['x']) + prtree(n, "indx"); + + l = n->left; + r = n->right; + if(l->addable == 1 || l->addable == 13 || r->complex > l->complex) { + n->right = l; + n->left = r; + l = r; + r = n->right; + } + if(l->addable != 7) { + idx.regtree = l; + idx.scale = 1; + } else + if(l->right->addable == 20) { + idx.regtree = l->left; + idx.scale = 1 << l->right->vconst; + } else + if(l->left->addable == 20) { + idx.regtree = l->right; + idx.scale = 1 << l->left->vconst; + } else + diag(n, "bad index"); + + idx.basetree = r; + if(debug['x']) { + print("scale = %d\n", idx.scale); + prtree(idx.regtree, "index"); + prtree(idx.basetree, "base"); + } +} diff --git a/src/cmd/6c/swt.c b/src/cmd/6c/swt.c new file mode 100644 index 000000000..d7a917043 --- /dev/null +++ b/src/cmd/6c/swt.c @@ -0,0 +1,587 @@ +// Inferno utils/6c/swt.c +// http://code.google.com/p/inferno-os/source/browse/utils/6c/swt.c +// +// Copyright © 1994-1999 Lucent Technologies Inc. All rights reserved. +// Portions Copyright © 1995-1997 C H Forsyth (forsyth@terzarima.net) +// Portions Copyright © 1997-1999 Vita Nuova Limited +// Portions Copyright © 2000-2007 Vita Nuova Holdings Limited (www.vitanuova.com) +// Portions Copyright © 2004,2006 Bruce Ellis +// Portions Copyright © 2005-2007 C H Forsyth (forsyth@terzarima.net) +// Revisions Copyright © 2000-2007 Lucent Technologies Inc. and others +// Portions Copyright © 2009 The Go Authors. All rights reserved. +// +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to deal +// in the Software without restriction, including without limitation the rights +// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +// copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +// THE SOFTWARE. + +#include "gc.h" + +void +swit1(C1 *q, int nc, int32 def, Node *n) +{ + C1 *r; + int i; + Prog *sp; + + if(nc < 5) { + for(i=0; ival); + gcmp(OEQ, n, q->val); + patch(p, q->label); + q++; + } + gbranch(OGOTO); + patch(p, def); + return; + } + i = nc / 2; + r = q+i; + if(debug['W']) + print("case > %.8llux\n", r->val); + gcmp(OGT, n, r->val); + sp = p; + gbranch(OGOTO); + p->as = AJEQ; + patch(p, r->label); + swit1(q, i, def, n); + + if(debug['W']) + print("case < %.8llux\n", r->val); + patch(sp, pc); + swit1(r+1, nc-i-1, def, n); +} + +void +bitload(Node *b, Node *n1, Node *n2, Node *n3, Node *nn) +{ + int sh; + int32 v; + Node *l; + + /* + * n1 gets adjusted/masked value + * n2 gets address of cell + * n3 gets contents of cell + */ + l = b->left; + if(n2 != Z) { + regalloc(n1, l, nn); + reglcgen(n2, l, Z); + regalloc(n3, l, Z); + gmove(n2, n3); + gmove(n3, n1); + } else { + regalloc(n1, l, nn); + cgen(l, n1); + } + if(b->type->shift == 0 && typeu[b->type->etype]) { + v = ~0 + (1L << b->type->nbits); + gopcode(OAND, tfield, nodconst(v), n1); + } else { + sh = 32 - b->type->shift - b->type->nbits; + if(sh > 0) + gopcode(OASHL, tfield, nodconst(sh), n1); + sh += b->type->shift; + if(sh > 0) + if(typeu[b->type->etype]) + gopcode(OLSHR, tfield, nodconst(sh), n1); + else + gopcode(OASHR, tfield, nodconst(sh), n1); + } +} + +void +bitstore(Node *b, Node *n1, Node *n2, Node *n3, Node *nn) +{ + int32 v; + Node nod; + int sh; + + regalloc(&nod, b->left, Z); + v = ~0 + (1L << b->type->nbits); + gopcode(OAND, types[TLONG], nodconst(v), n1); + gmove(n1, &nod); + if(nn != Z) + gmove(n1, nn); + sh = b->type->shift; + if(sh > 0) + gopcode(OASHL, types[TLONG], nodconst(sh), &nod); + v <<= sh; + gopcode(OAND, types[TLONG], nodconst(~v), n3); + gopcode(OOR, types[TLONG], n3, &nod); + gmove(&nod, n2); + + regfree(&nod); + regfree(n1); + regfree(n2); + regfree(n3); +} + +int32 +outstring(char *s, int32 n) +{ + int32 r; + + if(suppress) + return nstring; + r = nstring; + while(n) { + string[mnstring] = *s++; + mnstring++; + nstring++; + if(mnstring >= NSNAME) { + gpseudo(ADATA, symstring, nodconst(0L)); + p->from.offset += nstring - NSNAME; + p->from.scale = NSNAME; + p->to.type = D_SCONST; + memmove(p->to.sval, string, NSNAME); + mnstring = 0; + } + n--; + } + return r; +} + +void +sextern(Sym *s, Node *a, int32 o, int32 w) +{ + int32 e, lw; + + for(e=0; efrom.offset += o+e; + p->from.scale = lw; + p->to.type = D_SCONST; + memmove(p->to.sval, a->cstring+e, lw); + } +} + +void +gextern(Sym *s, Node *a, int32 o, int32 w) +{ + if(0 && a->op == OCONST && typev[a->type->etype]) { + gpseudo(ADATA, s, lo64(a)); + p->from.offset += o; + p->from.scale = 4; + gpseudo(ADATA, s, hi64(a)); + p->from.offset += o + 4; + p->from.scale = 4; + return; + } + gpseudo(ADATA, s, a); + p->from.offset += o; + p->from.scale = w; + switch(p->to.type) { + default: + p->to.index = p->to.type; + p->to.type = D_ADDR; + case D_CONST: + case D_FCONST: + case D_ADDR: + break; + } +} + +void zname(Biobuf*, Sym*, int); +void zaddr(Biobuf*, Adr*, int); +void outhist(Biobuf*); + +void +outcode(void) +{ + struct { Sym *sym; short type; } h[NSYM]; + Prog *p; + Sym *s; + int f, sf, st, t, sym; + Biobuf b; + + if(debug['S']) { + for(p = firstp; p != P; p = p->link) + if(p->as != ADATA && p->as != AGLOBL) + pc--; + for(p = firstp; p != P; p = p->link) { + print("%P\n", p); + if(p->as != ADATA && p->as != AGLOBL) + pc++; + } + } + + f = open(outfile, OWRITE); + if(f < 0) { + diag(Z, "cannot open %s", outfile); + return; + } + Binit(&b, f, OWRITE); + + Bprint(&b, "go object %s %s %s\n", getgoos(), thestring, getgoversion()); + if(ndynimp > 0 || ndynexp > 0) { + int i; + + Bprint(&b, "\n"); + Bprint(&b, "$$ // exports\n\n"); + Bprint(&b, "$$ // local types\n\n"); + Bprint(&b, "$$ // dynimport\n"); + for(i=0; ilink) { + jackpot: + sf = 0; + s = p->from.sym; + while(s != S) { + sf = s->sym; + if(sf < 0 || sf >= NSYM) + sf = 0; + t = p->from.type; + if(t == D_ADDR) + t = p->from.index; + if(h[sf].type == t) + if(h[sf].sym == s) + break; + s->sym = sym; + zname(&b, s, t); + h[sym].sym = s; + h[sym].type = t; + sf = sym; + sym++; + if(sym >= NSYM) + sym = 1; + break; + } + st = 0; + s = p->to.sym; + while(s != S) { + st = s->sym; + if(st < 0 || st >= NSYM) + st = 0; + t = p->to.type; + if(t == D_ADDR) + t = p->to.index; + if(h[st].type == t) + if(h[st].sym == s) + break; + s->sym = sym; + zname(&b, s, t); + h[sym].sym = s; + h[sym].type = t; + st = sym; + sym++; + if(sym >= NSYM) + sym = 1; + if(st == sf) + goto jackpot; + break; + } + Bputc(&b, p->as); + Bputc(&b, p->as>>8); + Bputc(&b, p->lineno); + Bputc(&b, p->lineno>>8); + Bputc(&b, p->lineno>>16); + Bputc(&b, p->lineno>>24); + zaddr(&b, &p->from, sf); + zaddr(&b, &p->to, st); + } + Bflush(&b); + close(f); + firstp = P; + lastp = P; +} + +void +outhist(Biobuf *b) +{ + Hist *h; + char *p, *q, *op, c; + Prog pg; + int n; + + pg = zprog; + pg.as = AHISTORY; + c = pathchar(); + for(h = hist; h != H; h = h->link) { + p = h->name; + op = 0; + /* on windows skip drive specifier in pathname */ + if(systemtype(Windows) && p && p[1] == ':'){ + p += 2; + c = *p; + } + if(p && p[0] != c && h->offset == 0 && pathname){ + /* on windows skip drive specifier in pathname */ + if(systemtype(Windows) && pathname[1] == ':') { + op = p; + p = pathname+2; + c = *p; + } else if(pathname[0] == c){ + op = p; + p = pathname; + } + } + while(p) { + q = utfrune(p, c); + if(q) { + n = q-p; + if(n == 0){ + n = 1; /* leading "/" */ + *p = '/'; /* don't emit "\" on windows */ + } + q++; + } else { + n = strlen(p); + q = 0; + } + if(n) { + Bputc(b, ANAME); + Bputc(b, ANAME>>8); + Bputc(b, D_FILE); + Bputc(b, 1); + Bputc(b, '<'); + Bwrite(b, p, n); + Bputc(b, 0); + } + p = q; + if(p == 0 && op) { + p = op; + op = 0; + } + } + pg.lineno = h->line; + pg.to.type = zprog.to.type; + pg.to.offset = h->offset; + if(h->offset) + pg.to.type = D_CONST; + + Bputc(b, pg.as); + Bputc(b, pg.as>>8); + Bputc(b, pg.lineno); + Bputc(b, pg.lineno>>8); + Bputc(b, pg.lineno>>16); + Bputc(b, pg.lineno>>24); + zaddr(b, &pg.from, 0); + zaddr(b, &pg.to, 0); + } +} + +void +zname(Biobuf *b, Sym *s, int t) +{ + char *n; + uint32 sig; + + if(debug['T'] && t == D_EXTERN && s->sig != SIGDONE && s->type != types[TENUM] && s != symrathole){ + sig = sign(s); + Bputc(b, ASIGNAME); + Bputc(b, ASIGNAME>>8); + Bputc(b, sig); + Bputc(b, sig>>8); + Bputc(b, sig>>16); + Bputc(b, sig>>24); + s->sig = SIGDONE; + } + else{ + Bputc(b, ANAME); /* as */ + Bputc(b, ANAME>>8); /* as */ + } + Bputc(b, t); /* type */ + Bputc(b, s->sym); /* sym */ + n = s->name; + while(*n) { + Bputc(b, *n); + n++; + } + Bputc(b, 0); +} + +void +zaddr(Biobuf *b, Adr *a, int s) +{ + int32 l; + int i, t; + char *n; + Ieee e; + + t = 0; + if(a->index != D_NONE || a->scale != 0) + t |= T_INDEX; + if(s != 0) + t |= T_SYM; + + switch(a->type) { + default: + t |= T_TYPE; + case D_NONE: + if(a->offset != 0) { + t |= T_OFFSET; + l = a->offset; + if((vlong)l != a->offset) + t |= T_64; + } + break; + case D_FCONST: + t |= T_FCONST; + break; + case D_SCONST: + t |= T_SCONST; + break; + } + Bputc(b, t); + + if(t & T_INDEX) { /* implies index, scale */ + Bputc(b, a->index); + Bputc(b, a->scale); + } + if(t & T_OFFSET) { /* implies offset */ + l = a->offset; + Bputc(b, l); + Bputc(b, l>>8); + Bputc(b, l>>16); + Bputc(b, l>>24); + if(t & T_64) { + l = a->offset>>32; + Bputc(b, l); + Bputc(b, l>>8); + Bputc(b, l>>16); + Bputc(b, l>>24); + } + } + if(t & T_SYM) /* implies sym */ + Bputc(b, s); + if(t & T_FCONST) { + ieeedtod(&e, a->dval); + l = e.l; + Bputc(b, l); + Bputc(b, l>>8); + Bputc(b, l>>16); + Bputc(b, l>>24); + l = e.h; + Bputc(b, l); + Bputc(b, l>>8); + Bputc(b, l>>16); + Bputc(b, l>>24); + return; + } + if(t & T_SCONST) { + n = a->sval; + for(i=0; itype); +} + +int32 +align(int32 i, Type *t, int op, int32 *maxalign) +{ + int32 o; + Type *v; + int w; + + o = i; + w = 1; + switch(op) { + default: + diag(Z, "unknown align opcode %d", op); + break; + + case Asu2: /* padding at end of a struct */ + w = *maxalign; + if(w < 1) + w = 1; + if(packflg) + w = packflg; + break; + + case Ael1: /* initial align of struct element */ + for(v=t; v->etype==TARRAY; v=v->link) + ; + if(v->etype == TSTRUCT || v->etype == TUNION) + w = v->align; + else + w = ewidth[v->etype]; + if(w < 1 || w > SZ_VLONG) + fatal(Z, "align"); + if(packflg) + w = packflg; + break; + + case Ael2: /* width of a struct element */ + o += t->width; + break; + + case Aarg0: /* initial passbyptr argument in arg list */ + if(typesu[t->etype]) { + o = align(o, types[TIND], Aarg1, nil); + o = align(o, types[TIND], Aarg2, nil); + } + break; + + case Aarg1: /* initial align of parameter */ + w = ewidth[t->etype]; + if(w <= 0 || w >= SZ_VLONG) { + w = SZ_VLONG; + break; + } + w = 1; /* little endian no adjustment */ + break; + + case Aarg2: /* width of a parameter */ + o += t->width; + w = t->width; + if(w > SZ_VLONG) + w = SZ_VLONG; + break; + + case Aaut3: /* total align of automatic */ + o = align(o, t, Ael1, nil); + o = align(o, t, Ael2, nil); + break; + } + o = xround(o, w); + if(maxalign && *maxalign < w) + *maxalign = w; + if(debug['A']) + print("align %s %d %T = %d\n", bnames[op], i, t, o); + return o; +} + +int32 +maxround(int32 max, int32 v) +{ + v += SZ_VLONG-1; + if(v > max) + max = xround(v, SZ_VLONG); + return max; +} diff --git a/src/cmd/6c/txt.c b/src/cmd/6c/txt.c new file mode 100644 index 000000000..12fc5b498 --- /dev/null +++ b/src/cmd/6c/txt.c @@ -0,0 +1,1564 @@ +// Inferno utils/6c/txt.c +// http://code.google.com/p/inferno-os/source/browse/utils/6c/txt.c +// +// Copyright © 1994-1999 Lucent Technologies Inc. All rights reserved. +// Portions Copyright © 1995-1997 C H Forsyth (forsyth@terzarima.net) +// Portions Copyright © 1997-1999 Vita Nuova Limited +// Portions Copyright © 2000-2007 Vita Nuova Holdings Limited (www.vitanuova.com) +// Portions Copyright © 2004,2006 Bruce Ellis +// Portions Copyright © 2005-2007 C H Forsyth (forsyth@terzarima.net) +// Revisions Copyright © 2000-2007 Lucent Technologies Inc. and others +// Portions Copyright © 2009 The Go Authors. All rights reserved. +// +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to deal +// in the Software without restriction, including without limitation the rights +// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +// copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +// THE SOFTWARE. + +#include "gc.h" + +void +ginit(void) +{ + int i; + Type *t; + + thechar = '6'; + thestring = "amd64"; + dodefine("_64BIT"); + listinit(); + nstring = 0; + mnstring = 0; + nrathole = 0; + pc = 0; + breakpc = -1; + continpc = -1; + cases = C; + firstp = P; + lastp = P; + tfield = types[TINT]; + + typeword = typechlvp; + typecmplx = typesu; + + /* TO DO */ + memmove(typechlpv, typechlp, sizeof(typechlpv)); + typechlpv[TVLONG] = 1; + typechlpv[TUVLONG] = 1; + + zprog.link = P; + zprog.as = AGOK; + zprog.from.type = D_NONE; + zprog.from.index = D_NONE; + zprog.from.scale = 0; + zprog.to = zprog.from; + + lregnode.op = OREGISTER; + lregnode.class = CEXREG; + lregnode.reg = REGTMP; + lregnode.complex = 0; + lregnode.addable = 11; + lregnode.type = types[TLONG]; + + qregnode = lregnode; + qregnode.type = types[TVLONG]; + + constnode.op = OCONST; + constnode.class = CXXX; + constnode.complex = 0; + constnode.addable = 20; + constnode.type = types[TLONG]; + + vconstnode = constnode; + vconstnode.type = types[TVLONG]; + + fconstnode.op = OCONST; + fconstnode.class = CXXX; + fconstnode.complex = 0; + fconstnode.addable = 20; + fconstnode.type = types[TDOUBLE]; + + nodsafe = new(ONAME, Z, Z); + nodsafe->sym = slookup(".safe"); + nodsafe->type = types[TINT]; + nodsafe->etype = types[TINT]->etype; + nodsafe->class = CAUTO; + complex(nodsafe); + + t = typ(TARRAY, types[TCHAR]); + symrathole = slookup(".rathole"); + symrathole->class = CGLOBL; + symrathole->type = t; + + nodrat = new(ONAME, Z, Z); + nodrat->sym = symrathole; + nodrat->type = types[TIND]; + nodrat->etype = TVOID; + nodrat->class = CGLOBL; + complex(nodrat); + nodrat->type = t; + + nodret = new(ONAME, Z, Z); + nodret->sym = slookup(".ret"); + nodret->type = types[TIND]; + nodret->etype = TIND; + nodret->class = CPARAM; + nodret = new(OIND, nodret, Z); + complex(nodret); + + if(0) + com64init(); + + for(i=0; i= D_AX && i <= D_R15 && i != D_SP) + reg[i] = 0; + if(i >= D_X0 && i <= D_X7) + reg[i] = 0; + } +} + +void +gclean(void) +{ + int i; + Sym *s; + + reg[D_SP]--; + for(i=D_AX; i<=D_R15; i++) + if(reg[i]) + diag(Z, "reg %R left allocated", i); + for(i=D_X0; i<=D_X7; i++) + if(reg[i]) + diag(Z, "reg %R left allocated", i); + while(mnstring) + outstring("", 1L); + symstring->type->width = nstring; + symrathole->type->width = nrathole; + for(i=0; ilink) { + if(s->type == T) + continue; + if(s->type->width == 0) + continue; + if(s->class != CGLOBL && s->class != CSTATIC) + continue; + if(s->type == types[TENUM]) + continue; + gpseudo(AGLOBL, s, nodconst(s->type->width)); + } + nextpc(); + p->as = AEND; + outcode(); +} + +void +nextpc(void) +{ + + p = alloc(sizeof(*p)); + *p = zprog; + p->lineno = nearln; + pc++; + if(firstp == P) { + firstp = p; + lastp = p; + return; + } + lastp->link = p; + lastp = p; +} + +void +gargs(Node *n, Node *tn1, Node *tn2) +{ + int32 regs; + Node fnxargs[20], *fnxp; + + regs = cursafe; + + fnxp = fnxargs; + garg1(n, tn1, tn2, 0, &fnxp); /* compile fns to temps */ + + curarg = 0; + fnxp = fnxargs; + garg1(n, tn1, tn2, 1, &fnxp); /* compile normal args and temps */ + + cursafe = regs; +} + +int +nareg(void) +{ + int i, n; + + n = 0; + for(i=D_AX; i<=D_R15; i++) + if(reg[i] == 0) + n++; + return n; +} + +void +garg1(Node *n, Node *tn1, Node *tn2, int f, Node **fnxp) +{ + Node nod; + + if(n == Z) + return; + if(n->op == OLIST) { + garg1(n->left, tn1, tn2, f, fnxp); + garg1(n->right, tn1, tn2, f, fnxp); + return; + } + if(f == 0) { + if(n->complex >= FNX) { + regsalloc(*fnxp, n); + nod = znode; + nod.op = OAS; + nod.left = *fnxp; + nod.right = n; + nod.type = n->type; + cgen(&nod, Z); + (*fnxp)++; + } + return; + } + if(typesu[n->type->etype]) { + regaalloc(tn2, n); + if(n->complex >= FNX) { + sugen(*fnxp, tn2, n->type->width); + (*fnxp)++; + } else + sugen(n, tn2, n->type->width); + return; + } + if(REGARG >= 0 && curarg == 0 && typechlpv[n->type->etype]) { + regaalloc1(tn1, n); + if(n->complex >= FNX) { + cgen(*fnxp, tn1); + (*fnxp)++; + } else + cgen(n, tn1); + return; + } + if(vconst(n) == 0) { + regaalloc(tn2, n); + gmove(n, tn2); + return; + } + regalloc(tn1, n, Z); + if(n->complex >= FNX) { + cgen(*fnxp, tn1); + (*fnxp)++; + } else + cgen(n, tn1); + regaalloc(tn2, n); + gmove(tn1, tn2); + regfree(tn1); +} + +Node* +nodgconst(vlong v, Type *t) +{ + if(!typev[t->etype]) + return nodconst((int32)v); + vconstnode.vconst = v; + return &vconstnode; +} + +Node* +nodconst(int32 v) +{ + constnode.vconst = v; + return &constnode; +} + +Node* +nodfconst(double d) +{ + fconstnode.fconst = d; + return &fconstnode; +} + +int +isreg(Node *n, int r) +{ + + if(n->op == OREGISTER) + if(n->reg == r) + return 1; + return 0; +} + +int +nodreg(Node *n, Node *nn, int r) +{ + int et; + + *n = qregnode; + n->reg = r; + if(nn != Z){ + et = nn->type->etype; + if(!typefd[et] && nn->type->width <= SZ_LONG && 0) + n->type = typeu[et]? types[TUINT]: types[TINT]; + else + n->type = nn->type; +//print("nodreg %s [%s]\n", tnames[et], tnames[n->type->etype]); + n->lineno = nn->lineno; + } + if(reg[r] == 0) + return 0; + if(nn != Z) { + if(nn->op == OREGISTER) + if(nn->reg == r) + return 0; + } + return 1; +} + +void +regret(Node *n, Node *nn) +{ + int r; + + r = REGRET; + if(typefd[nn->type->etype]) + r = FREGRET; + nodreg(n, nn, r); + reg[r]++; +} + +void +regalloc(Node *n, Node *tn, Node *o) +{ + int i; + + switch(tn->type->etype) { + case TCHAR: + case TUCHAR: + case TSHORT: + case TUSHORT: + case TINT: + case TUINT: + case TLONG: + case TULONG: + case TVLONG: + case TUVLONG: + case TIND: + if(o != Z && o->op == OREGISTER) { + i = o->reg; + if(i >= D_AX && i <= D_R15) + goto out; + } + for(i=D_AX; i<=D_R15; i++) + if(reg[i] == 0) + goto out; + diag(tn, "out of fixed registers"); + goto err; + + case TFLOAT: + case TDOUBLE: + if(o != Z && o->op == OREGISTER) { + i = o->reg; + if(i >= D_X0 && i <= D_X7) + goto out; + } + for(i=D_X0; i<=D_X7; i++) + if(reg[i] == 0) + goto out; + diag(tn, "out of float registers"); + goto out; + } + diag(tn, "unknown type in regalloc: %T", tn->type); +err: + i = 0; +out: + if(i) + reg[i]++; + nodreg(n, tn, i); +} + +void +regialloc(Node *n, Node *tn, Node *o) +{ + Node nod; + + nod = *tn; + nod.type = types[TIND]; + regalloc(n, &nod, o); +} + +void +regfree(Node *n) +{ + int i; + + i = 0; + if(n->op != OREGISTER && n->op != OINDREG) + goto err; + i = n->reg; + if(i < 0 || i >= sizeof(reg)) + goto err; + if(reg[i] <= 0) + goto err; + reg[i]--; + return; +err: + diag(n, "error in regfree: %R", i); +} + +void +regsalloc(Node *n, Node *nn) +{ + cursafe = align(cursafe, nn->type, Aaut3, nil); + maxargsafe = maxround(maxargsafe, cursafe+curarg); + *n = *nodsafe; + n->xoffset = -(stkoff + cursafe); + n->type = nn->type; + n->etype = nn->type->etype; + n->lineno = nn->lineno; +} + +void +regaalloc1(Node *n, Node *nn) +{ + if(REGARG < 0) { + fatal(n, "regaalloc1 and REGARG<0"); + return; + } + nodreg(n, nn, REGARG); + reg[REGARG]++; + curarg = align(curarg, nn->type, Aarg1, nil); + curarg = align(curarg, nn->type, Aarg2, nil); + maxargsafe = maxround(maxargsafe, cursafe+curarg); +} + +void +regaalloc(Node *n, Node *nn) +{ + curarg = align(curarg, nn->type, Aarg1, nil); + *n = *nn; + n->op = OINDREG; + n->reg = REGSP; + n->xoffset = curarg; + n->complex = 0; + n->addable = 20; + curarg = align(curarg, nn->type, Aarg2, nil); + maxargsafe = maxround(maxargsafe, cursafe+curarg); +} + +void +regind(Node *n, Node *nn) +{ + + if(n->op != OREGISTER) { + diag(n, "regind not OREGISTER"); + return; + } + n->op = OINDREG; + n->type = nn->type; +} + +void +naddr(Node *n, Adr *a) +{ + int32 v; + + a->type = D_NONE; + if(n == Z) + return; + switch(n->op) { + default: + bad: + diag(n, "bad in naddr: %O %D", n->op, a); + break; + + case OREGISTER: + a->type = n->reg; + a->sym = S; + break; + + case OEXREG: + a->type = D_INDIR + D_GS; + a->offset = n->reg - 1; + break; + + case OIND: + naddr(n->left, a); + if(a->type >= D_AX && a->type <= D_R15) + a->type += D_INDIR; + else + if(a->type == D_CONST) + a->type = D_NONE+D_INDIR; + else + if(a->type == D_ADDR) { + a->type = a->index; + a->index = D_NONE; + } else + goto bad; + break; + + case OINDEX: + a->type = idx.ptr; + if(n->left->op == OADDR || n->left->op == OCONST) + naddr(n->left, a); + if(a->type >= D_AX && a->type <= D_R15) + a->type += D_INDIR; + else + if(a->type == D_CONST) + a->type = D_NONE+D_INDIR; + else + if(a->type == D_ADDR) { + a->type = a->index; + a->index = D_NONE; + } else + goto bad; + a->index = idx.reg; + a->scale = n->scale; + a->offset += n->xoffset; + break; + + case OINDREG: + a->type = n->reg+D_INDIR; + a->sym = S; + a->offset = n->xoffset; + break; + + case ONAME: + a->etype = n->etype; + a->type = D_STATIC; + a->sym = n->sym; + a->offset = n->xoffset; + if(n->class == CSTATIC) + break; + if(n->class == CEXTERN || n->class == CGLOBL) { + a->type = D_EXTERN; + break; + } + if(n->class == CAUTO) { + a->type = D_AUTO; + break; + } + if(n->class == CPARAM) { + a->type = D_PARAM; + break; + } + goto bad; + + case OCONST: + if(typefd[n->type->etype]) { + a->type = D_FCONST; + a->dval = n->fconst; + break; + } + a->sym = S; + a->type = D_CONST; + if(typev[n->type->etype] || n->type->etype == TIND) + a->offset = n->vconst; + else + a->offset = convvtox(n->vconst, typeu[n->type->etype]? TULONG: TLONG); + break; + + case OADDR: + naddr(n->left, a); + if(a->type >= D_INDIR) { + a->type -= D_INDIR; + break; + } + if(a->type == D_EXTERN || a->type == D_STATIC || + a->type == D_AUTO || a->type == D_PARAM) + if(a->index == D_NONE) { + a->index = a->type; + a->type = D_ADDR; + break; + } + goto bad; + + case OADD: + if(n->right->op == OCONST) { + v = n->right->vconst; + naddr(n->left, a); + } else + if(n->left->op == OCONST) { + v = n->left->vconst; + naddr(n->right, a); + } else + goto bad; + a->offset += v; + break; + + } +} + +void +gcmp(int op, Node *n, vlong val) +{ + Node *cn, nod; + + cn = nodgconst(val, n->type); + if(!immconst(cn)){ + regalloc(&nod, n, Z); + gmove(cn, &nod); + gopcode(op, n->type, n, &nod); + regfree(&nod); + }else + gopcode(op, n->type, n, cn); +} + +#define CASE(a,b) ((a<<8)|(b<<0)) + +void +gmove(Node *f, Node *t) +{ + int ft, tt, t64, a; + Node nod, nod1, nod2, nod3; + Prog *p1, *p2; + + ft = f->type->etype; + tt = t->type->etype; + t64 = tt == TVLONG || tt == TUVLONG || tt == TIND; + if(debug['M']) + print("gop: %O %O[%s],%O[%s]\n", OAS, + f->op, tnames[ft], t->op, tnames[tt]); + if(typefd[ft] && f->op == OCONST) { + /* TO DO: pick up special constants, possibly preloaded */ + if(f->fconst == 0.0){ + regalloc(&nod, t, t); + gins(AXORPD, &nod, &nod); + gmove(&nod, t); + regfree(&nod); + return; + } + } +/* + * load + */ + if(ft == TVLONG || ft == TUVLONG) + if(f->op == OCONST) + if(f->vconst > 0x7fffffffLL || f->vconst < -0x7fffffffLL) + if(t->op != OREGISTER) { + regalloc(&nod, f, Z); + gmove(f, &nod); + gmove(&nod, t); + regfree(&nod); + return; + } + + if(f->op == ONAME || f->op == OINDREG || + f->op == OIND || f->op == OINDEX) + switch(ft) { + case TCHAR: + a = AMOVBLSX; + if(t64) + a = AMOVBQSX; + goto ld; + case TUCHAR: + a = AMOVBLZX; + if(t64) + a = AMOVBQZX; + goto ld; + case TSHORT: + a = AMOVWLSX; + if(t64) + a = AMOVWQSX; + goto ld; + case TUSHORT: + a = AMOVWLZX; + if(t64) + a = AMOVWQZX; + goto ld; + case TINT: + case TLONG: + if(typefd[tt]) { + regalloc(&nod, t, t); + if(tt == TDOUBLE) + a = ACVTSL2SD; + else + a = ACVTSL2SS; + gins(a, f, &nod); + gmove(&nod, t); + regfree(&nod); + return; + } + a = AMOVL; + if(t64) + a = AMOVLQSX; + goto ld; + case TUINT: + case TULONG: + a = AMOVL; + if(t64) + a = AMOVLQZX; /* could probably use plain MOVL */ + goto ld; + case TVLONG: + if(typefd[tt]) { + regalloc(&nod, t, t); + if(tt == TDOUBLE) + a = ACVTSQ2SD; + else + a = ACVTSQ2SS; + gins(a, f, &nod); + gmove(&nod, t); + regfree(&nod); + return; + } + case TUVLONG: + a = AMOVQ; + goto ld; + case TIND: + a = AMOVQ; + + ld: + regalloc(&nod, f, t); + nod.type = t64? types[TVLONG]: types[TINT]; + gins(a, f, &nod); + gmove(&nod, t); + regfree(&nod); + return; + + case TFLOAT: + a = AMOVSS; + goto fld; + case TDOUBLE: + a = AMOVSD; + fld: + regalloc(&nod, f, t); + if(tt != TDOUBLE && tt != TFLOAT){ /* TO DO: why is this here */ + prtree(f, "odd tree"); + nod.type = t64? types[TVLONG]: types[TINT]; + } + gins(a, f, &nod); + gmove(&nod, t); + regfree(&nod); + return; + } + +/* + * store + */ + if(t->op == ONAME || t->op == OINDREG || + t->op == OIND || t->op == OINDEX) + switch(tt) { + case TCHAR: + case TUCHAR: + a = AMOVB; goto st; + case TSHORT: + case TUSHORT: + a = AMOVW; goto st; + case TINT: + case TUINT: + case TLONG: + case TULONG: + a = AMOVL; goto st; + case TVLONG: + case TUVLONG: + case TIND: + a = AMOVQ; goto st; + + st: + if(f->op == OCONST) { + gins(a, f, t); + return; + } + fst: + regalloc(&nod, t, f); + gmove(f, &nod); + gins(a, &nod, t); + regfree(&nod); + return; + + case TFLOAT: + a = AMOVSS; + goto fst; + case TDOUBLE: + a = AMOVSD; + goto fst; + } + +/* + * convert + */ + switch(CASE(ft,tt)) { + default: +/* + * integer to integer + ******** + a = AGOK; break; + + case CASE( TCHAR, TCHAR): + case CASE( TUCHAR, TCHAR): + case CASE( TSHORT, TCHAR): + case CASE( TUSHORT,TCHAR): + case CASE( TINT, TCHAR): + case CASE( TUINT, TCHAR): + case CASE( TLONG, TCHAR): + case CASE( TULONG, TCHAR): + case CASE( TIND, TCHAR): + + case CASE( TCHAR, TUCHAR): + case CASE( TUCHAR, TUCHAR): + case CASE( TSHORT, TUCHAR): + case CASE( TUSHORT,TUCHAR): + case CASE( TINT, TUCHAR): + case CASE( TUINT, TUCHAR): + case CASE( TLONG, TUCHAR): + case CASE( TULONG, TUCHAR): + case CASE( TIND, TUCHAR): + + case CASE( TSHORT, TSHORT): + case CASE( TUSHORT,TSHORT): + case CASE( TINT, TSHORT): + case CASE( TUINT, TSHORT): + case CASE( TLONG, TSHORT): + case CASE( TULONG, TSHORT): + case CASE( TIND, TSHORT): + + case CASE( TSHORT, TUSHORT): + case CASE( TUSHORT,TUSHORT): + case CASE( TINT, TUSHORT): + case CASE( TUINT, TUSHORT): + case CASE( TLONG, TUSHORT): + case CASE( TULONG, TUSHORT): + case CASE( TIND, TUSHORT): + + case CASE( TINT, TINT): + case CASE( TUINT, TINT): + case CASE( TLONG, TINT): + case CASE( TULONG, TINT): + case CASE( TIND, TINT): + + case CASE( TINT, TUINT): + case CASE( TUINT, TUINT): + case CASE( TLONG, TUINT): + case CASE( TULONG, TUINT): + case CASE( TIND, TUINT): + + case CASE( TUINT, TIND): + case CASE( TVLONG, TUINT): + case CASE( TVLONG, TULONG): + case CASE( TUVLONG, TUINT): + case CASE( TUVLONG, TULONG): + *****/ + a = AMOVL; + break; + + case CASE( TVLONG, TCHAR): + case CASE( TVLONG, TSHORT): + case CASE( TVLONG, TINT): + case CASE( TVLONG, TLONG): + case CASE( TUVLONG, TCHAR): + case CASE( TUVLONG, TSHORT): + case CASE( TUVLONG, TINT): + case CASE( TUVLONG, TLONG): + case CASE( TINT, TVLONG): + case CASE( TINT, TUVLONG): + case CASE( TLONG, TVLONG): + case CASE( TINT, TIND): + case CASE( TLONG, TIND): + a = AMOVLQSX; + if(f->op == OCONST) { + f->vconst &= (uvlong)0xffffffffU; + if(f->vconst & 0x80000000) + f->vconst |= (vlong)0xffffffff << 32; + a = AMOVQ; + } + break; + + case CASE( TUINT, TIND): + case CASE( TUINT, TVLONG): + case CASE( TUINT, TUVLONG): + case CASE( TULONG, TVLONG): + case CASE( TULONG, TUVLONG): + case CASE( TULONG, TIND): + a = AMOVL; /* same effect as AMOVLQZX */ + if(f->op == OCONST) { + f->vconst &= (uvlong)0xffffffffU; + a = AMOVQ; + } + break; + + case CASE( TIND, TVLONG): + case CASE( TVLONG, TVLONG): + case CASE( TUVLONG, TVLONG): + case CASE( TVLONG, TUVLONG): + case CASE( TUVLONG, TUVLONG): + case CASE( TIND, TUVLONG): + case CASE( TVLONG, TIND): + case CASE( TUVLONG, TIND): + case CASE( TIND, TIND): + a = AMOVQ; + break; + + case CASE( TSHORT, TINT): + case CASE( TSHORT, TUINT): + case CASE( TSHORT, TLONG): + case CASE( TSHORT, TULONG): + a = AMOVWLSX; + if(f->op == OCONST) { + f->vconst &= 0xffff; + if(f->vconst & 0x8000) + f->vconst |= 0xffff0000; + a = AMOVL; + } + break; + + case CASE( TSHORT, TVLONG): + case CASE( TSHORT, TUVLONG): + case CASE( TSHORT, TIND): + a = AMOVWQSX; + if(f->op == OCONST) { + f->vconst &= 0xffff; + if(f->vconst & 0x8000){ + f->vconst |= 0xffff0000; + f->vconst |= (vlong)~0 << 32; + } + a = AMOVL; + } + break; + + case CASE( TUSHORT,TINT): + case CASE( TUSHORT,TUINT): + case CASE( TUSHORT,TLONG): + case CASE( TUSHORT,TULONG): + a = AMOVWLZX; + if(f->op == OCONST) { + f->vconst &= 0xffff; + a = AMOVL; + } + break; + + case CASE( TUSHORT,TVLONG): + case CASE( TUSHORT,TUVLONG): + case CASE( TUSHORT,TIND): + a = AMOVWQZX; + if(f->op == OCONST) { + f->vconst &= 0xffff; + a = AMOVL; /* MOVL also zero-extends to 64 bits */ + } + break; + + case CASE( TCHAR, TSHORT): + case CASE( TCHAR, TUSHORT): + case CASE( TCHAR, TINT): + case CASE( TCHAR, TUINT): + case CASE( TCHAR, TLONG): + case CASE( TCHAR, TULONG): + a = AMOVBLSX; + if(f->op == OCONST) { + f->vconst &= 0xff; + if(f->vconst & 0x80) + f->vconst |= 0xffffff00; + a = AMOVL; + } + break; + + case CASE( TCHAR, TVLONG): + case CASE( TCHAR, TUVLONG): + case CASE( TCHAR, TIND): + a = AMOVBQSX; + if(f->op == OCONST) { + f->vconst &= 0xff; + if(f->vconst & 0x80){ + f->vconst |= 0xffffff00; + f->vconst |= (vlong)~0 << 32; + } + a = AMOVQ; + } + break; + + case CASE( TUCHAR, TSHORT): + case CASE( TUCHAR, TUSHORT): + case CASE( TUCHAR, TINT): + case CASE( TUCHAR, TUINT): + case CASE( TUCHAR, TLONG): + case CASE( TUCHAR, TULONG): + a = AMOVBLZX; + if(f->op == OCONST) { + f->vconst &= 0xff; + a = AMOVL; + } + break; + + case CASE( TUCHAR, TVLONG): + case CASE( TUCHAR, TUVLONG): + case CASE( TUCHAR, TIND): + a = AMOVBQZX; + if(f->op == OCONST) { + f->vconst &= 0xff; + a = AMOVL; /* zero-extends to 64-bits */ + } + break; + +/* + * float to fix + */ + case CASE( TFLOAT, TCHAR): + case CASE( TFLOAT, TUCHAR): + case CASE( TFLOAT, TSHORT): + case CASE( TFLOAT, TUSHORT): + case CASE( TFLOAT, TINT): + case CASE( TFLOAT, TUINT): + case CASE( TFLOAT, TLONG): + case CASE( TFLOAT, TULONG): + case CASE( TFLOAT, TVLONG): + case CASE( TFLOAT, TUVLONG): + case CASE( TFLOAT, TIND): + + case CASE( TDOUBLE,TCHAR): + case CASE( TDOUBLE,TUCHAR): + case CASE( TDOUBLE,TSHORT): + case CASE( TDOUBLE,TUSHORT): + case CASE( TDOUBLE,TINT): + case CASE( TDOUBLE,TUINT): + case CASE( TDOUBLE,TLONG): + case CASE( TDOUBLE,TULONG): + case CASE( TDOUBLE,TVLONG): + case CASE( TDOUBLE,TUVLONG): + case CASE( TDOUBLE,TIND): + regalloc(&nod, t, Z); + if(ewidth[tt] == SZ_VLONG || typeu[tt] && ewidth[tt] == SZ_INT){ + if(ft == TFLOAT) + a = ACVTTSS2SQ; + else + a = ACVTTSD2SQ; + }else{ + if(ft == TFLOAT) + a = ACVTTSS2SL; + else + a = ACVTTSD2SL; + } + gins(a, f, &nod); + gmove(&nod, t); + regfree(&nod); + return; + +/* + * uvlong to float + */ + case CASE( TUVLONG, TDOUBLE): + case CASE( TUVLONG, TFLOAT): + a = ACVTSQ2SS; + if(tt == TDOUBLE) + a = ACVTSQ2SD; + regalloc(&nod, f, f); + gmove(f, &nod); + regalloc(&nod1, t, t); + gins(ACMPQ, &nod, nodconst(0)); + gins(AJLT, Z, Z); + p1 = p; + gins(a, &nod, &nod1); + gins(AJMP, Z, Z); + p2 = p; + patch(p1, pc); + regalloc(&nod2, f, Z); + regalloc(&nod3, f, Z); + gmove(&nod, &nod2); + gins(ASHRQ, nodconst(1), &nod2); + gmove(&nod, &nod3); + gins(AANDL, nodconst(1), &nod3); + gins(AORQ, &nod3, &nod2); + gins(a, &nod2, &nod1); + gins(tt == TDOUBLE? AADDSD: AADDSS, &nod1, &nod1); + regfree(&nod2); + regfree(&nod3); + patch(p2, pc); + regfree(&nod); + regfree(&nod1); + return; + + case CASE( TULONG, TDOUBLE): + case CASE( TUINT, TDOUBLE): + case CASE( TULONG, TFLOAT): + case CASE( TUINT, TFLOAT): + a = ACVTSQ2SS; + if(tt == TDOUBLE) + a = ACVTSQ2SD; + regalloc(&nod, f, f); + gins(AMOVLQZX, f, &nod); + regalloc(&nod1, t, t); + gins(a, &nod, &nod1); + gmove(&nod1, t); + regfree(&nod); + regfree(&nod1); + return; + +/* + * fix to float + */ + case CASE( TCHAR, TFLOAT): + case CASE( TUCHAR, TFLOAT): + case CASE( TSHORT, TFLOAT): + case CASE( TUSHORT,TFLOAT): + case CASE( TINT, TFLOAT): + case CASE( TLONG, TFLOAT): + case CASE( TVLONG, TFLOAT): + case CASE( TIND, TFLOAT): + + case CASE( TCHAR, TDOUBLE): + case CASE( TUCHAR, TDOUBLE): + case CASE( TSHORT, TDOUBLE): + case CASE( TUSHORT,TDOUBLE): + case CASE( TINT, TDOUBLE): + case CASE( TLONG, TDOUBLE): + case CASE( TVLONG, TDOUBLE): + case CASE( TIND, TDOUBLE): + regalloc(&nod, t, t); + if(ewidth[ft] == SZ_VLONG){ + if(tt == TFLOAT) + a = ACVTSQ2SS; + else + a = ACVTSQ2SD; + }else{ + if(tt == TFLOAT) + a = ACVTSL2SS; + else + a = ACVTSL2SD; + } + gins(a, f, &nod); + gmove(&nod, t); + regfree(&nod); + return; + +/* + * float to float + */ + case CASE( TFLOAT, TFLOAT): + a = AMOVSS; + break; + case CASE( TDOUBLE,TFLOAT): + a = ACVTSD2SS; + break; + case CASE( TFLOAT, TDOUBLE): + a = ACVTSS2SD; + break; + case CASE( TDOUBLE,TDOUBLE): + a = AMOVSD; + break; + } + if(a == AMOVQ || a == AMOVSD || a == AMOVSS || a == AMOVL && ewidth[ft] == ewidth[tt]) /* TO DO: check AMOVL */ + if(samaddr(f, t)) + return; + gins(a, f, t); +} + +void +doindex(Node *n) +{ + Node nod, nod1; + int32 v; + +if(debug['Y']) +prtree(n, "index"); + +if(n->left->complex >= FNX) +print("botch in doindex\n"); + + regalloc(&nod, &qregnode, Z); + v = constnode.vconst; + cgen(n->right, &nod); + idx.ptr = D_NONE; + if(n->left->op == OCONST) + idx.ptr = D_CONST; + else if(n->left->op == OREGISTER) + idx.ptr = n->left->reg; + else if(n->left->op != OADDR) { + reg[D_BP]++; // cant be used as a base + regalloc(&nod1, &qregnode, Z); + cgen(n->left, &nod1); + idx.ptr = nod1.reg; + regfree(&nod1); + reg[D_BP]--; + } + idx.reg = nod.reg; + regfree(&nod); + constnode.vconst = v; +} + +void +gins(int a, Node *f, Node *t) +{ + + if(f != Z && f->op == OINDEX) + doindex(f); + if(t != Z && t->op == OINDEX) + doindex(t); + nextpc(); + p->as = a; + if(f != Z) + naddr(f, &p->from); + if(t != Z) + naddr(t, &p->to); + if(debug['g']) + print("%P\n", p); +} + +void +gopcode(int o, Type *ty, Node *f, Node *t) +{ + int a, et; + + et = TLONG; + if(ty != T) + et = ty->etype; + if(debug['M']) { + if(f != Z && f->type != T) + print("gop: %O %O[%s],", o, f->op, tnames[et]); + else + print("gop: %O Z,", o); + if(t != Z && t->type != T) + print("%O[%s]\n", t->op, tnames[t->type->etype]); + else + print("Z\n"); + } + a = AGOK; + switch(o) { + case OCOM: + a = ANOTL; + if(et == TCHAR || et == TUCHAR) + a = ANOTB; + if(et == TSHORT || et == TUSHORT) + a = ANOTW; + if(et == TVLONG || et == TUVLONG || et == TIND) + a = ANOTQ; + break; + + case ONEG: + a = ANEGL; + if(et == TCHAR || et == TUCHAR) + a = ANEGB; + if(et == TSHORT || et == TUSHORT) + a = ANEGW; + if(et == TVLONG || et == TUVLONG || et == TIND) + a = ANEGQ; + break; + + case OADDR: + a = ALEAQ; + break; + + case OASADD: + case OADD: + a = AADDL; + if(et == TCHAR || et == TUCHAR) + a = AADDB; + if(et == TSHORT || et == TUSHORT) + a = AADDW; + if(et == TVLONG || et == TUVLONG || et == TIND) + a = AADDQ; + if(et == TFLOAT) + a = AADDSS; + if(et == TDOUBLE) + a = AADDSD; + break; + + case OASSUB: + case OSUB: + a = ASUBL; + if(et == TCHAR || et == TUCHAR) + a = ASUBB; + if(et == TSHORT || et == TUSHORT) + a = ASUBW; + if(et == TVLONG || et == TUVLONG || et == TIND) + a = ASUBQ; + if(et == TFLOAT) + a = ASUBSS; + if(et == TDOUBLE) + a = ASUBSD; + break; + + case OASOR: + case OOR: + a = AORL; + if(et == TCHAR || et == TUCHAR) + a = AORB; + if(et == TSHORT || et == TUSHORT) + a = AORW; + if(et == TVLONG || et == TUVLONG || et == TIND) + a = AORQ; + break; + + case OASAND: + case OAND: + a = AANDL; + if(et == TCHAR || et == TUCHAR) + a = AANDB; + if(et == TSHORT || et == TUSHORT) + a = AANDW; + if(et == TVLONG || et == TUVLONG || et == TIND) + a = AANDQ; + break; + + case OASXOR: + case OXOR: + a = AXORL; + if(et == TCHAR || et == TUCHAR) + a = AXORB; + if(et == TSHORT || et == TUSHORT) + a = AXORW; + if(et == TVLONG || et == TUVLONG || et == TIND) + a = AXORQ; + break; + + case OASLSHR: + case OLSHR: + a = ASHRL; + if(et == TCHAR || et == TUCHAR) + a = ASHRB; + if(et == TSHORT || et == TUSHORT) + a = ASHRW; + if(et == TVLONG || et == TUVLONG || et == TIND) + a = ASHRQ; + break; + + case OASASHR: + case OASHR: + a = ASARL; + if(et == TCHAR || et == TUCHAR) + a = ASARB; + if(et == TSHORT || et == TUSHORT) + a = ASARW; + if(et == TVLONG || et == TUVLONG || et == TIND) + a = ASARQ; + break; + + case OASASHL: + case OASHL: + a = ASALL; + if(et == TCHAR || et == TUCHAR) + a = ASALB; + if(et == TSHORT || et == TUSHORT) + a = ASALW; + if(et == TVLONG || et == TUVLONG || et == TIND) + a = ASALQ; + break; + + case OFUNC: + a = ACALL; + break; + + case OASMUL: + case OMUL: + if(f->op == OREGISTER && t != Z && isreg(t, D_AX) && reg[D_DX] == 0) + t = Z; + a = AIMULL; + if(et == TVLONG || et == TUVLONG || et == TIND) + a = AIMULQ; + if(et == TFLOAT) + a = AMULSS; + if(et == TDOUBLE) + a = AMULSD; + break; + + case OASMOD: + case OMOD: + case OASDIV: + case ODIV: + a = AIDIVL; + if(et == TVLONG || et == TUVLONG || et == TIND) + a = AIDIVQ; + if(et == TFLOAT) + a = ADIVSS; + if(et == TDOUBLE) + a = ADIVSD; + break; + + case OASLMUL: + case OLMUL: + a = AMULL; + if(et == TVLONG || et == TUVLONG || et == TIND) + a = AMULQ; + break; + + case OASLMOD: + case OLMOD: + case OASLDIV: + case OLDIV: + a = ADIVL; + if(et == TVLONG || et == TUVLONG || et == TIND) + a = ADIVQ; + break; + + case OEQ: + case ONE: + case OLT: + case OLE: + case OGE: + case OGT: + case OLO: + case OLS: + case OHS: + case OHI: + a = ACMPL; + if(et == TCHAR || et == TUCHAR) + a = ACMPB; + if(et == TSHORT || et == TUSHORT) + a = ACMPW; + if(et == TVLONG || et == TUVLONG || et == TIND) + a = ACMPQ; + if(et == TFLOAT) + a = AUCOMISS; + if(et == TDOUBLE) + a = AUCOMISD; + gins(a, f, t); + switch(o) { + case OEQ: a = AJEQ; break; + case ONE: a = AJNE; break; + case OLT: a = AJLT; break; + case OLE: a = AJLE; break; + case OGE: a = AJGE; break; + case OGT: a = AJGT; break; + case OLO: a = AJCS; break; + case OLS: a = AJLS; break; + case OHS: a = AJCC; break; + case OHI: a = AJHI; break; + } + gins(a, Z, Z); + return; + } + if(a == AGOK) + diag(Z, "bad in gopcode %O", o); + gins(a, f, t); +} + +int +samaddr(Node *f, Node *t) +{ + return f->op == OREGISTER && t->op == OREGISTER && f->reg == t->reg; +} + +void +gbranch(int o) +{ + int a; + + a = AGOK; + switch(o) { + case ORETURN: + a = ARET; + break; + case OGOTO: + a = AJMP; + break; + } + nextpc(); + if(a == AGOK) { + diag(Z, "bad in gbranch %O", o); + nextpc(); + } + p->as = a; +} + +void +patch(Prog *op, int32 pc) +{ + + op->to.offset = pc; + op->to.type = D_BRANCH; +} + +void +gpseudo(int a, Sym *s, Node *n) +{ + + nextpc(); + p->as = a; + p->from.type = D_EXTERN; + p->from.sym = s; + p->from.scale = textflag; + textflag = 0; + + if(s->class == CSTATIC) + p->from.type = D_STATIC; + naddr(n, &p->to); + if(a == ADATA || a == AGLOBL) + pc--; +} + +int +sconst(Node *n) +{ + int32 v; + + if(n->op == OCONST && !typefd[n->type->etype]) { + v = n->vconst; + if(v >= -32766L && v < 32766L) + return 1; + } + return 0; +} + +int32 +exreg(Type *t) +{ + int32 o; + + if(typechlpv[t->etype]) { + if(exregoffset >= 64) + return 0; + o = exregoffset; + exregoffset += 8; + return o+1; // +1 to avoid 0 == failure; naddr's case OEXREG will subtract 1. + } + return 0; +} + +schar ewidth[NTYPE] = +{ + -1, /*[TXXX]*/ + SZ_CHAR, /*[TCHAR]*/ + SZ_CHAR, /*[TUCHAR]*/ + SZ_SHORT, /*[TSHORT]*/ + SZ_SHORT, /*[TUSHORT]*/ + SZ_INT, /*[TINT]*/ + SZ_INT, /*[TUINT]*/ + SZ_LONG, /*[TLONG]*/ + SZ_LONG, /*[TULONG]*/ + SZ_VLONG, /*[TVLONG]*/ + SZ_VLONG, /*[TUVLONG]*/ + SZ_FLOAT, /*[TFLOAT]*/ + SZ_DOUBLE, /*[TDOUBLE]*/ + SZ_IND, /*[TIND]*/ + 0, /*[TFUNC]*/ + -1, /*[TARRAY]*/ + 0, /*[TVOID]*/ + -1, /*[TSTRUCT]*/ + -1, /*[TUNION]*/ + SZ_INT, /*[TENUM]*/ +}; +int32 ncast[NTYPE] = +{ + 0, /*[TXXX]*/ + BCHAR|BUCHAR, /*[TCHAR]*/ + BCHAR|BUCHAR, /*[TUCHAR]*/ + BSHORT|BUSHORT, /*[TSHORT]*/ + BSHORT|BUSHORT, /*[TUSHORT]*/ + BINT|BUINT|BLONG|BULONG, /*[TINT]*/ + BINT|BUINT|BLONG|BULONG, /*[TUINT]*/ + BINT|BUINT|BLONG|BULONG, /*[TLONG]*/ + BINT|BUINT|BLONG|BULONG, /*[TULONG]*/ + BVLONG|BUVLONG|BIND, /*[TVLONG]*/ + BVLONG|BUVLONG|BIND, /*[TUVLONG]*/ + BFLOAT, /*[TFLOAT]*/ + BDOUBLE, /*[TDOUBLE]*/ + BVLONG|BUVLONG|BIND, /*[TIND]*/ + 0, /*[TFUNC]*/ + 0, /*[TARRAY]*/ + 0, /*[TVOID]*/ + BSTRUCT, /*[TSTRUCT]*/ + BUNION, /*[TUNION]*/ + 0, /*[TENUM]*/ +}; diff --git a/src/cmd/6g/Makefile b/src/cmd/6g/Makefile new file mode 100644 index 000000000..64fa15399 --- /dev/null +++ b/src/cmd/6g/Makefile @@ -0,0 +1,35 @@ +# Copyright 2009 The Go Authors. All rights reserved. +# Use of this source code is governed by a BSD-style +# license that can be found in the LICENSE file. + +include ../../Make.inc +O:=$(HOST_O) + +TARG=6g + +HFILES=\ + ../gc/go.h\ + ../6l/6.out.h\ + gg.h\ + opt.h\ + +OFILES=\ + ../6l/enam.$O\ + cgen.$O\ + cplx.$O\ + galign.$O\ + ggen.$O\ + gobj.$O\ + gsubr.$O\ + list.$O\ + peep.$O\ + pgen.$O\ + reg.$O\ + +LIB=\ + ../gc/gc.a\ + +include ../../Make.ccmd + +%.$O: ../gc/%.c + $(HOST_CC) $(HOST_CFLAGS) -c -I. -o $@ ../gc/$*.c diff --git a/src/cmd/6g/cgen.c b/src/cmd/6g/cgen.c new file mode 100644 index 000000000..24f88a416 --- /dev/null +++ b/src/cmd/6g/cgen.c @@ -0,0 +1,1299 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +#include "gg.h" + +/* + * generate: + * res = n; + * simplifies and calls gmove. + */ +void +cgen(Node *n, Node *res) +{ + Node *nl, *nr, *r; + Node n1, n2; + int a, f; + Prog *p1, *p2, *p3; + Addr addr; + + if(debug['g']) { + dump("\ncgen-n", n); + dump("cgen-res", res); + } + if(n == N || n->type == T) + goto ret; + + if(res == N || res->type == T) + fatal("cgen: res nil"); + + while(n->op == OCONVNOP) + n = n->left; + + // inline slices + if(cgen_inline(n, res)) + goto ret; + + if(n->ullman >= UINF) { + if(n->op == OINDREG) + fatal("cgen: this is going to misscompile"); + if(res->ullman >= UINF) { + tempname(&n1, n->type); + cgen(n, &n1); + cgen(&n1, res); + goto ret; + } + } + + if(isfat(n->type)) { + if(n->type->width < 0) + fatal("forgot to compute width for %T", n->type); + sgen(n, res, n->type->width); + goto ret; + } + + if(!res->addable) { + if(n->ullman > res->ullman) { + regalloc(&n1, n->type, res); + cgen(n, &n1); + if(n1.ullman > res->ullman) { + dump("n1", &n1); + dump("res", res); + fatal("loop in cgen"); + } + cgen(&n1, res); + regfree(&n1); + goto ret; + } + + if(res->ullman >= UINF) + goto gen; + + if(complexop(n, res)) { + complexgen(n, res); + goto ret; + } + + f = 1; // gen thru register + switch(n->op) { + case OLITERAL: + if(smallintconst(n)) + f = 0; + break; + case OREGISTER: + f = 0; + break; + } + + if(!iscomplex[n->type->etype]) { + a = optoas(OAS, res->type); + if(sudoaddable(a, res, &addr)) { + if(f) { + regalloc(&n2, res->type, N); + cgen(n, &n2); + p1 = gins(a, &n2, N); + regfree(&n2); + } else + p1 = gins(a, n, N); + p1->to = addr; + if(debug['g']) + print("%P [ignore previous line]\n", p1); + sudoclean(); + goto ret; + } + } + + gen: + igen(res, &n1, N); + cgen(n, &n1); + regfree(&n1); + goto ret; + } + + // update addressability for string, slice + // can't do in walk because n->left->addable + // changes if n->left is an escaping local variable. + switch(n->op) { + case OLEN: + if(isslice(n->left->type) || istype(n->left->type, TSTRING)) + n->addable = n->left->addable; + break; + case OCAP: + if(isslice(n->left->type)) + n->addable = n->left->addable; + break; + } + + if(complexop(n, res)) { + complexgen(n, res); + goto ret; + } + + if(n->addable) { + gmove(n, res); + goto ret; + } + + nl = n->left; + nr = n->right; + + if(nl != N && nl->ullman >= UINF) + if(nr != N && nr->ullman >= UINF) { + tempname(&n1, nl->type); + cgen(nl, &n1); + n2 = *n; + n2.left = &n1; + cgen(&n2, res); + goto ret; + } + + if(!iscomplex[n->type->etype]) { + a = optoas(OAS, n->type); + if(sudoaddable(a, n, &addr)) { + if(res->op == OREGISTER) { + p1 = gins(a, N, res); + p1->from = addr; + } else { + regalloc(&n2, n->type, N); + p1 = gins(a, N, &n2); + p1->from = addr; + gins(a, &n2, res); + regfree(&n2); + } + sudoclean(); + goto ret; + } + } + + switch(n->op) { + default: + dump("cgen", n); + fatal("cgen: unknown op %N", n); + break; + + // these call bgen to get a bool value + case OOROR: + case OANDAND: + case OEQ: + case ONE: + case OLT: + case OLE: + case OGE: + case OGT: + case ONOT: + p1 = gbranch(AJMP, T); + p2 = pc; + gmove(nodbool(1), res); + p3 = gbranch(AJMP, T); + patch(p1, pc); + bgen(n, 1, p2); + gmove(nodbool(0), res); + patch(p3, pc); + goto ret; + + case OPLUS: + cgen(nl, res); + goto ret; + + // unary + case OCOM: + a = optoas(OXOR, nl->type); + regalloc(&n1, nl->type, N); + cgen(nl, &n1); + nodconst(&n2, nl->type, -1); + gins(a, &n2, &n1); + gmove(&n1, res); + regfree(&n1); + goto ret; + + case OMINUS: + if(isfloat[nl->type->etype]) { + nr = nodintconst(-1); + convlit(&nr, n->type); + a = optoas(OMUL, nl->type); + goto sbop; + } + a = optoas(n->op, nl->type); + goto uop; + + // symmetric binary + case OAND: + case OOR: + case OXOR: + case OADD: + case OMUL: + a = optoas(n->op, nl->type); + if(a != AIMULB) + goto sbop; + cgen_bmul(n->op, nl, nr, res); + break; + + // asymmetric binary + case OSUB: + a = optoas(n->op, nl->type); + goto abop; + + case OCONV: + regalloc(&n1, nl->type, res); + regalloc(&n2, n->type, &n1); + cgen(nl, &n1); + + // if we do the conversion n1 -> n2 here + // reusing the register, then gmove won't + // have to allocate its own register. + gmove(&n1, &n2); + gmove(&n2, res); + regfree(&n2); + regfree(&n1); + break; + + case ODOT: + case ODOTPTR: + case OINDEX: + case OIND: + case ONAME: // PHEAP or PPARAMREF var + igen(n, &n1, res); + gmove(&n1, res); + regfree(&n1); + break; + + case OLEN: + if(istype(nl->type, TMAP) || istype(nl->type, TCHAN)) { + // map and chan have len in the first 32-bit word. + // a zero pointer means zero length + regalloc(&n1, types[tptr], res); + cgen(nl, &n1); + + nodconst(&n2, types[tptr], 0); + gins(optoas(OCMP, types[tptr]), &n1, &n2); + p1 = gbranch(optoas(OEQ, types[tptr]), T); + + n2 = n1; + n2.op = OINDREG; + n2.type = types[TINT32]; + gmove(&n2, &n1); + + patch(p1, pc); + + gmove(&n1, res); + regfree(&n1); + break; + } + if(istype(nl->type, TSTRING) || isslice(nl->type)) { + // both slice and string have len one pointer into the struct. + // a zero pointer means zero length + igen(nl, &n1, res); + n1.type = types[TUINT32]; + n1.xoffset += Array_nel; + gmove(&n1, res); + regfree(&n1); + break; + } + fatal("cgen: OLEN: unknown type %lT", nl->type); + break; + + case OCAP: + if(istype(nl->type, TCHAN)) { + // chan has cap in the second 32-bit word. + // a zero pointer means zero length + regalloc(&n1, types[tptr], res); + cgen(nl, &n1); + + nodconst(&n2, types[tptr], 0); + gins(optoas(OCMP, types[tptr]), &n1, &n2); + p1 = gbranch(optoas(OEQ, types[tptr]), T); + + n2 = n1; + n2.op = OINDREG; + n2.xoffset = 4; + n2.type = types[TINT32]; + gmove(&n2, &n1); + + patch(p1, pc); + + gmove(&n1, res); + regfree(&n1); + break; + } + if(isslice(nl->type)) { + igen(nl, &n1, res); + n1.type = types[TUINT32]; + n1.xoffset += Array_cap; + gmove(&n1, res); + regfree(&n1); + break; + } + fatal("cgen: OCAP: unknown type %lT", nl->type); + break; + + case OADDR: + agen(nl, res); + break; + + case OCALLMETH: + cgen_callmeth(n, 0); + cgen_callret(n, res); + break; + + case OCALLINTER: + cgen_callinter(n, res, 0); + cgen_callret(n, res); + break; + + case OCALLFUNC: + cgen_call(n, 0); + cgen_callret(n, res); + break; + + case OMOD: + case ODIV: + if(isfloat[n->type->etype]) { + a = optoas(n->op, nl->type); + goto abop; + } + cgen_div(n->op, nl, nr, res); + break; + + case OLSH: + case ORSH: + cgen_shift(n->op, nl, nr, res); + break; + } + goto ret; + +sbop: // symmetric binary + if(nl->ullman < nr->ullman) { + r = nl; + nl = nr; + nr = r; + } + +abop: // asymmetric binary + if(nl->ullman >= nr->ullman) { + regalloc(&n1, nl->type, res); + cgen(nl, &n1); + + if(sudoaddable(a, nr, &addr)) { + p1 = gins(a, N, &n1); + p1->from = addr; + gmove(&n1, res); + sudoclean(); + regfree(&n1); + goto ret; + } + regalloc(&n2, nr->type, N); + cgen(nr, &n2); + } else { + regalloc(&n2, nr->type, N); + cgen(nr, &n2); + regalloc(&n1, nl->type, res); + cgen(nl, &n1); + } + gins(a, &n2, &n1); + gmove(&n1, res); + regfree(&n1); + regfree(&n2); + goto ret; + +uop: // unary + regalloc(&n1, nl->type, res); + cgen(nl, &n1); + gins(a, N, &n1); + gmove(&n1, res); + regfree(&n1); + goto ret; + +ret: + ; +} + +/* + * generate: + * res = &n; + */ +void +agen(Node *n, Node *res) +{ + Node *nl, *nr; + Node n1, n2, n3, tmp, n4, n5; + Prog *p1; + uint32 w; + uint64 v; + Type *t; + + if(debug['g']) { + dump("\nagen-res", res); + dump("agen-r", n); + } + if(n == N || n->type == T) + return; + + while(n->op == OCONVNOP) + n = n->left; + + if(n->addable) { + regalloc(&n1, types[tptr], res); + gins(ALEAQ, n, &n1); + gmove(&n1, res); + regfree(&n1); + goto ret; + } + + nl = n->left; + nr = n->right; + + switch(n->op) { + default: + fatal("agen: unknown op %N", n); + break; + + case OCALLMETH: + cgen_callmeth(n, 0); + cgen_aret(n, res); + break; + + case OCALLINTER: + cgen_callinter(n, res, 0); + cgen_aret(n, res); + break; + + case OCALLFUNC: + cgen_call(n, 0); + cgen_aret(n, res); + break; + + case OINDEX: + w = n->type->width; + if(nr->addable) + goto irad; + if(nl->addable) { + if(!isconst(nr, CTINT)) { + regalloc(&n1, nr->type, N); + cgen(nr, &n1); + } + if(!isconst(nl, CTSTR)) { + regalloc(&n3, types[tptr], res); + agen(nl, &n3); + } + goto index; + } + tempname(&tmp, nr->type); + cgen(nr, &tmp); + nr = &tmp; + + irad: + if(!isconst(nl, CTSTR)) { + regalloc(&n3, types[tptr], res); + agen(nl, &n3); + } + if(!isconst(nr, CTINT)) { + regalloc(&n1, nr->type, N); + cgen(nr, &n1); + } + goto index; + + index: + // &a is in &n3 (allocated in res) + // i is in &n1 (if not constant) + // w is width + + // explicit check for nil if array is large enough + // that we might derive too big a pointer. + if(isfixedarray(nl->type) && nl->type->width >= unmappedzero) { + regalloc(&n4, types[tptr], &n3); + gmove(&n3, &n4); + n4.op = OINDREG; + n4.type = types[TUINT8]; + n4.xoffset = 0; + gins(ATESTB, nodintconst(0), &n4); + regfree(&n4); + } + + // constant index + if(isconst(nr, CTINT)) { + if(isconst(nl, CTSTR)) + fatal("constant string constant index"); // front end should handle + v = mpgetfix(nr->val.u.xval); + if(isslice(nl->type) || nl->type->etype == TSTRING) { + if(!debug['B'] && !n->etype) { + n1 = n3; + n1.op = OINDREG; + n1.type = types[tptr]; + n1.xoffset = Array_nel; + nodconst(&n2, types[TUINT32], v); + gins(optoas(OCMP, types[TUINT32]), &n1, &n2); + p1 = gbranch(optoas(OGT, types[TUINT32]), T); + ginscall(panicindex, 0); + patch(p1, pc); + } + + n1 = n3; + n1.op = OINDREG; + n1.type = types[tptr]; + n1.xoffset = Array_array; + gmove(&n1, &n3); + } + + if (v*w != 0) + ginscon(optoas(OADD, types[tptr]), v*w, &n3); + gmove(&n3, res); + regfree(&n3); + break; + } + + // type of the index + t = types[TUINT64]; + if(issigned[n1.type->etype]) + t = types[TINT64]; + + regalloc(&n2, t, &n1); // i + gmove(&n1, &n2); + regfree(&n1); + + if(!debug['B'] && !n->etype) { + // check bounds + n5.op = OXXX; + t = types[TUINT32]; + if(is64(nr->type)) + t = types[TUINT64]; + if(isconst(nl, CTSTR)) { + nodconst(&n1, t, nl->val.u.sval->len); + } else if(isslice(nl->type) || nl->type->etype == TSTRING) { + n1 = n3; + n1.op = OINDREG; + n1.type = types[TUINT32]; + n1.xoffset = Array_nel; + if(is64(nr->type)) { + regalloc(&n5, t, N); + gmove(&n1, &n5); + n1 = n5; + } + } else { + nodconst(&n1, t, nl->type->bound); + } + gins(optoas(OCMP, t), &n2, &n1); + p1 = gbranch(optoas(OLT, t), T); + if(n5.op != OXXX) + regfree(&n5); + ginscall(panicindex, 0); + patch(p1, pc); + } + + if(isconst(nl, CTSTR)) { + regalloc(&n3, types[tptr], res); + p1 = gins(ALEAQ, N, &n3); + datastring(nl->val.u.sval->s, nl->val.u.sval->len, &p1->from); + p1->from.scale = 1; + p1->from.index = n2.val.u.reg; + goto indexdone; + } + + if(isslice(nl->type) || nl->type->etype == TSTRING) { + n1 = n3; + n1.op = OINDREG; + n1.type = types[tptr]; + n1.xoffset = Array_array; + gmove(&n1, &n3); + } + + if(w == 0) { + // nothing to do + } else if(w == 1 || w == 2 || w == 4 || w == 8) { + p1 = gins(ALEAQ, &n2, &n3); + p1->from.scale = w; + p1->from.index = p1->from.type; + p1->from.type = p1->to.type + D_INDIR; + } else { + ginscon(optoas(OMUL, t), w, &n2); + gins(optoas(OADD, types[tptr]), &n2, &n3); + } + + indexdone: + gmove(&n3, res); + regfree(&n2); + regfree(&n3); + break; + + case ONAME: + // should only get here with names in this func. + if(n->funcdepth > 0 && n->funcdepth != funcdepth) { + dump("bad agen", n); + fatal("agen: bad ONAME funcdepth %d != %d", + n->funcdepth, funcdepth); + } + + // should only get here for heap vars or paramref + if(!(n->class & PHEAP) && n->class != PPARAMREF) { + dump("bad agen", n); + fatal("agen: bad ONAME class %#x", n->class); + } + cgen(n->heapaddr, res); + if(n->xoffset != 0) + ginscon(optoas(OADD, types[tptr]), n->xoffset, res); + break; + + case OIND: + cgen(nl, res); + break; + + case ODOT: + agen(nl, res); + if(n->xoffset != 0) + ginscon(optoas(OADD, types[tptr]), n->xoffset, res); + break; + + case ODOTPTR: + cgen(nl, res); + if(n->xoffset != 0) { + // explicit check for nil if struct is large enough + // that we might derive too big a pointer. + if(nl->type->type->width >= unmappedzero) { + regalloc(&n1, types[tptr], res); + gmove(res, &n1); + n1.op = OINDREG; + n1.type = types[TUINT8]; + n1.xoffset = 0; + gins(ATESTB, nodintconst(0), &n1); + regfree(&n1); + } + ginscon(optoas(OADD, types[tptr]), n->xoffset, res); + } + break; + } + +ret: + ; +} + +/* + * generate: + * newreg = &n; + * res = newreg + * + * on exit, a has been changed to be *newreg. + * caller must regfree(a). + */ +void +igen(Node *n, Node *a, Node *res) +{ + Type *fp; + Iter flist; + + switch(n->op) { + case ONAME: + if((n->class&PHEAP) || n->class == PPARAMREF) + break; + *a = *n; + return; + + case OCALLFUNC: + fp = structfirst(&flist, getoutarg(n->left->type)); + cgen_call(n, 0); + memset(a, 0, sizeof *a); + a->op = OINDREG; + a->val.u.reg = D_SP; + a->addable = 1; + a->xoffset = fp->width; + a->type = n->type; + return; + } + + regalloc(a, types[tptr], res); + agen(n, a); + a->op = OINDREG; + a->type = n->type; +} + +/* + * generate: + * if(n == true) goto to; + */ +void +bgen(Node *n, int true, Prog *to) +{ + int et, a; + Node *nl, *nr, *l, *r; + Node n1, n2, tmp; + Prog *p1, *p2; + + if(debug['g']) { + dump("\nbgen", n); + } + + if(n == N) + n = nodbool(1); + + if(n->ninit != nil) + genlist(n->ninit); + + nl = n->left; + nr = n->right; + + if(n->type == T) { + convlit(&n, types[TBOOL]); + if(n->type == T) + goto ret; + } + + et = n->type->etype; + if(et != TBOOL) { + yyerror("cgen: bad type %T for %O", n->type, n->op); + patch(gins(AEND, N, N), to); + goto ret; + } + nl = N; + nr = N; + + switch(n->op) { + default: + def: + regalloc(&n1, n->type, N); + cgen(n, &n1); + nodconst(&n2, n->type, 0); + gins(optoas(OCMP, n->type), &n1, &n2); + a = AJNE; + if(!true) + a = AJEQ; + patch(gbranch(a, n->type), to); + regfree(&n1); + goto ret; + + case OLITERAL: + // need to ask if it is bool? + if(!true == !n->val.u.bval) + patch(gbranch(AJMP, T), to); + goto ret; + + case ONAME: + if(n->addable == 0) + goto def; + nodconst(&n1, n->type, 0); + gins(optoas(OCMP, n->type), n, &n1); + a = AJNE; + if(!true) + a = AJEQ; + patch(gbranch(a, n->type), to); + goto ret; + + case OANDAND: + if(!true) + goto caseor; + + caseand: + p1 = gbranch(AJMP, T); + p2 = gbranch(AJMP, T); + patch(p1, pc); + bgen(n->left, !true, p2); + bgen(n->right, !true, p2); + p1 = gbranch(AJMP, T); + patch(p1, to); + patch(p2, pc); + goto ret; + + case OOROR: + if(!true) + goto caseand; + + caseor: + bgen(n->left, true, to); + bgen(n->right, true, to); + goto ret; + + case OEQ: + case ONE: + case OLT: + case OGT: + case OLE: + case OGE: + nr = n->right; + if(nr == N || nr->type == T) + goto ret; + + case ONOT: // unary + nl = n->left; + if(nl == N || nl->type == T) + goto ret; + break; + } + + switch(n->op) { + + case ONOT: + bgen(nl, !true, to); + goto ret; + + case OEQ: + case ONE: + case OLT: + case OGT: + case OLE: + case OGE: + a = n->op; + if(!true) { + if(isfloat[nr->type->etype]) { + // brcom is not valid on floats when NaN is involved. + p1 = gbranch(AJMP, T); + p2 = gbranch(AJMP, T); + patch(p1, pc); + bgen(n, 1, p2); + patch(gbranch(AJMP, T), to); + patch(p2, pc); + goto ret; + } + a = brcom(a); + true = !true; + } + + // make simplest on right + if(nl->op == OLITERAL || (nl->ullman < nr->ullman && nl->ullman < UINF)) { + a = brrev(a); + r = nl; + nl = nr; + nr = r; + } + + if(isslice(nl->type)) { + // only valid to cmp darray to literal nil + if((a != OEQ && a != ONE) || nr->op != OLITERAL) { + yyerror("illegal array comparison"); + break; + } + a = optoas(a, types[tptr]); + regalloc(&n1, types[tptr], N); + agen(nl, &n1); + n2 = n1; + n2.op = OINDREG; + n2.xoffset = Array_array; + n2.type = types[tptr]; + nodconst(&tmp, types[tptr], 0); + gins(optoas(OCMP, types[tptr]), &n2, &tmp); + patch(gbranch(a, types[tptr]), to); + regfree(&n1); + break; + } + + if(isinter(nl->type)) { + // front end shold only leave cmp to literal nil + if((a != OEQ && a != ONE) || nr->op != OLITERAL) { + yyerror("illegal interface comparison"); + break; + } + a = optoas(a, types[tptr]); + regalloc(&n1, types[tptr], N); + agen(nl, &n1); + n2 = n1; + n2.op = OINDREG; + n2.xoffset = 0; + nodconst(&tmp, types[tptr], 0); + gins(optoas(OCMP, types[tptr]), &n2, &tmp); + patch(gbranch(a, types[tptr]), to); + regfree(&n1); + break; + } + if(iscomplex[nl->type->etype]) { + complexbool(a, nl, nr, true, to); + break; + } + + if(nr->ullman >= UINF) { + regalloc(&n1, nl->type, N); + cgen(nl, &n1); + + tempname(&tmp, nl->type); + gmove(&n1, &tmp); + regfree(&n1); + + regalloc(&n2, nr->type, N); + cgen(nr, &n2); + + regalloc(&n1, nl->type, N); + cgen(&tmp, &n1); + + goto cmp; + } + + regalloc(&n1, nl->type, N); + cgen(nl, &n1); + + if(smallintconst(nr)) { + gins(optoas(OCMP, nr->type), &n1, nr); + patch(gbranch(optoas(a, nr->type), nr->type), to); + regfree(&n1); + break; + } + + regalloc(&n2, nr->type, N); + cgen(nr, &n2); + cmp: + // only < and <= work right with NaN; reverse if needed + l = &n1; + r = &n2; + if(isfloat[nl->type->etype] && (a == OGT || a == OGE)) { + l = &n2; + r = &n1; + a = brrev(a); + } + + gins(optoas(OCMP, nr->type), l, r); + + if(isfloat[nr->type->etype] && (n->op == OEQ || n->op == ONE)) { + if(n->op == OEQ) { + // neither NE nor P + p1 = gbranch(AJNE, T); + p2 = gbranch(AJPS, T); + patch(gbranch(AJMP, T), to); + patch(p1, pc); + patch(p2, pc); + } else { + // either NE or P + patch(gbranch(AJNE, T), to); + patch(gbranch(AJPS, T), to); + } + } else + patch(gbranch(optoas(a, nr->type), nr->type), to); + regfree(&n1); + regfree(&n2); + break; + } + goto ret; + +ret: + ; +} + +/* + * n is on stack, either local variable + * or return value from function call. + * return n's offset from SP. + */ +int32 +stkof(Node *n) +{ + Type *t; + Iter flist; + int32 off; + + switch(n->op) { + case OINDREG: + return n->xoffset; + + case ODOT: + t = n->left->type; + if(isptr[t->etype]) + break; + off = stkof(n->left); + if(off == -1000 || off == 1000) + return off; + return off + n->xoffset; + + case OINDEX: + t = n->left->type; + if(!isfixedarray(t)) + break; + off = stkof(n->left); + if(off == -1000 || off == 1000) + return off; + if(isconst(n->right, CTINT)) + return off + t->type->width * mpgetfix(n->right->val.u.xval); + return 1000; + + case OCALLMETH: + case OCALLINTER: + case OCALLFUNC: + t = n->left->type; + if(isptr[t->etype]) + t = t->type; + + t = structfirst(&flist, getoutarg(t)); + if(t != T) + return t->width; + break; + } + + // botch - probably failing to recognize address + // arithmetic on the above. eg INDEX and DOT + return -1000; +} + +/* + * block copy: + * memmove(&ns, &n, w); + */ +void +sgen(Node *n, Node *ns, int32 w) +{ + Node nodl, nodr, oldl, oldr, cx, oldcx, tmp; + int32 c, q, odst, osrc; + + if(debug['g']) { + print("\nsgen w=%d\n", w); + dump("r", n); + dump("res", ns); + } + if(w == 0) + return; + if(n->ullman >= UINF && ns->ullman >= UINF) { + fatal("sgen UINF"); + } + + if(w < 0) + fatal("sgen copy %d", w); + + if(w == 16) + if(componentgen(n, ns)) + return; + + // offset on the stack + osrc = stkof(n); + odst = stkof(ns); + + if(osrc != -1000 && odst != -1000 && (osrc == 1000 || odst == 1000)) { + // osrc and odst both on stack, and at least one is in + // an unknown position. Could generate code to test + // for forward/backward copy, but instead just copy + // to a temporary location first. + tempname(&tmp, n->type); + sgen(n, &tmp, w); + sgen(&tmp, ns, w); + return; + } + + if(n->ullman >= ns->ullman) { + savex(D_SI, &nodr, &oldr, N, types[tptr]); + agen(n, &nodr); + + regalloc(&nodr, types[tptr], &nodr); // mark nodr as live + savex(D_DI, &nodl, &oldl, N, types[tptr]); + agen(ns, &nodl); + regfree(&nodr); + } else { + savex(D_DI, &nodl, &oldl, N, types[tptr]); + agen(ns, &nodl); + + regalloc(&nodl, types[tptr], &nodl); // mark nodl as live + savex(D_SI, &nodr, &oldr, N, types[tptr]); + agen(n, &nodr); + regfree(&nodl); + } + + c = w % 8; // bytes + q = w / 8; // quads + + savex(D_CX, &cx, &oldcx, N, types[TINT64]); + + // if we are copying forward on the stack and + // the src and dst overlap, then reverse direction + if(osrc < odst && odst < osrc+w) { + // reverse direction + gins(ASTD, N, N); // set direction flag + if(c > 0) { + gconreg(AADDQ, w-1, D_SI); + gconreg(AADDQ, w-1, D_DI); + + gconreg(AMOVQ, c, D_CX); + gins(AREP, N, N); // repeat + gins(AMOVSB, N, N); // MOVB *(SI)-,*(DI)- + } + + if(q > 0) { + if(c > 0) { + gconreg(AADDQ, -7, D_SI); + gconreg(AADDQ, -7, D_DI); + } else { + gconreg(AADDQ, w-8, D_SI); + gconreg(AADDQ, w-8, D_DI); + } + gconreg(AMOVQ, q, D_CX); + gins(AREP, N, N); // repeat + gins(AMOVSQ, N, N); // MOVQ *(SI)-,*(DI)- + } + // we leave with the flag clear + gins(ACLD, N, N); + } else { + // normal direction + if(q >= 4) { + gconreg(AMOVQ, q, D_CX); + gins(AREP, N, N); // repeat + gins(AMOVSQ, N, N); // MOVQ *(SI)+,*(DI)+ + } else + while(q > 0) { + gins(AMOVSQ, N, N); // MOVQ *(SI)+,*(DI)+ + q--; + } + + if(c >= 4) { + gins(AMOVSL, N, N); // MOVL *(SI)+,*(DI)+ + c -= 4; + } + while(c > 0) { + gins(AMOVSB, N, N); // MOVB *(SI)+,*(DI)+ + c--; + } + } + + + restx(&nodl, &oldl); + restx(&nodr, &oldr); + restx(&cx, &oldcx); +} + +static int +cadable(Node *n) +{ + if(!n->addable) { + // dont know how it happens, + // but it does + return 0; + } + + switch(n->op) { + case ONAME: + return 1; + } + return 0; +} + +/* + * copy a structure component by component + * return 1 if can do, 0 if cant. + * nr is N for copy zero + */ +int +componentgen(Node *nr, Node *nl) +{ + Node nodl, nodr; + int freel, freer; + + freel = 0; + freer = 0; + + switch(nl->type->etype) { + default: + goto no; + + case TARRAY: + if(!isslice(nl->type)) + goto no; + case TSTRING: + case TINTER: + break; + } + + nodl = *nl; + if(!cadable(nl)) { + if(nr == N || !cadable(nr)) + goto no; + igen(nl, &nodl, N); + freel = 1; + } + + if(nr != N) { + nodr = *nr; + if(!cadable(nr)) { + igen(nr, &nodr, N); + freer = 1; + } + } + + switch(nl->type->etype) { + case TARRAY: + if(!isslice(nl->type)) + goto no; + + nodl.xoffset += Array_array; + nodl.type = ptrto(nl->type->type); + + if(nr != N) { + nodr.xoffset += Array_array; + nodr.type = nodl.type; + } else + nodconst(&nodr, nodl.type, 0); + gmove(&nodr, &nodl); + + nodl.xoffset += Array_nel-Array_array; + nodl.type = types[TUINT32]; + + if(nr != N) { + nodr.xoffset += Array_nel-Array_array; + nodr.type = nodl.type; + } else + nodconst(&nodr, nodl.type, 0); + gmove(&nodr, &nodl); + + nodl.xoffset += Array_cap-Array_nel; + nodl.type = types[TUINT32]; + + if(nr != N) { + nodr.xoffset += Array_cap-Array_nel; + nodr.type = nodl.type; + } else + nodconst(&nodr, nodl.type, 0); + gmove(&nodr, &nodl); + + goto yes; + + case TSTRING: + nodl.xoffset += Array_array; + nodl.type = ptrto(types[TUINT8]); + + if(nr != N) { + nodr.xoffset += Array_array; + nodr.type = nodl.type; + } else + nodconst(&nodr, nodl.type, 0); + gmove(&nodr, &nodl); + + nodl.xoffset += Array_nel-Array_array; + nodl.type = types[TUINT32]; + + if(nr != N) { + nodr.xoffset += Array_nel-Array_array; + nodr.type = nodl.type; + } else + nodconst(&nodr, nodl.type, 0); + gmove(&nodr, &nodl); + + goto yes; + + case TINTER: + nodl.xoffset += Array_array; + nodl.type = ptrto(types[TUINT8]); + + if(nr != N) { + nodr.xoffset += Array_array; + nodr.type = nodl.type; + } else + nodconst(&nodr, nodl.type, 0); + gmove(&nodr, &nodl); + + nodl.xoffset += Array_nel-Array_array; + nodl.type = ptrto(types[TUINT8]); + + if(nr != N) { + nodr.xoffset += Array_nel-Array_array; + nodr.type = nodl.type; + } else + nodconst(&nodr, nodl.type, 0); + gmove(&nodr, &nodl); + + goto yes; + + case TSTRUCT: + goto no; + } + +no: + if(freer) + regfree(&nodr); + if(freel) + regfree(&nodl); + return 0; + +yes: + if(freer) + regfree(&nodr); + if(freel) + regfree(&nodl); + return 1; +} diff --git a/src/cmd/6g/doc.go b/src/cmd/6g/doc.go new file mode 100644 index 000000000..64f1d2ba9 --- /dev/null +++ b/src/cmd/6g/doc.go @@ -0,0 +1,13 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +/* + +6g is the version of the gc compiler for the x86-64. +The $GOARCH for these tools is amd64. + +It reads .go files and outputs .6 files. The flags are documented in ../gc/doc.go. + +*/ +package documentation diff --git a/src/cmd/6g/galign.c b/src/cmd/6g/galign.c new file mode 100644 index 000000000..e366362b3 --- /dev/null +++ b/src/cmd/6g/galign.c @@ -0,0 +1,37 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +#include "gg.h" + +int thechar = '6'; +char* thestring = "amd64"; + +vlong MAXWIDTH = 1LL<<50; + +/* + * go declares several platform-specific type aliases: + * int, uint, float, and uintptr + */ +Typedef typedefs[] = +{ + "int", TINT, TINT32, + "uint", TUINT, TUINT32, + "uintptr", TUINTPTR, TUINT64, + 0 +}; + +void +betypeinit(void) +{ + widthptr = 8; + + zprog.link = P; + zprog.as = AGOK; + zprog.from.type = D_NONE; + zprog.from.index = D_NONE; + zprog.from.scale = 0; + zprog.to = zprog.from; + + listinit(); +} diff --git a/src/cmd/6g/gg.h b/src/cmd/6g/gg.h new file mode 100644 index 000000000..2493771a0 --- /dev/null +++ b/src/cmd/6g/gg.h @@ -0,0 +1,161 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +#include +#include + +#include "../gc/go.h" +#include "../6l/6.out.h" + +#ifndef EXTERN +#define EXTERN extern +#endif + +typedef struct Addr Addr; + +struct Addr +{ + vlong offset; + double dval; + Prog* branch; + char sval[NSNAME]; + + Sym* gotype; + Sym* sym; + Node* node; + int width; + uchar type; + uchar index; + uchar etype; + uchar scale; /* doubles as width in DATA op */ + uchar pun; /* dont register variable */ +}; +#define A ((Addr*)0) + +struct Prog +{ + short as; // opcode + uint32 loc; // pc offset in this func + uint32 lineno; // source line that generated this + Addr from; // src address + Addr to; // dst address + Prog* link; // next instruction in this func + void* reg; // pointer to containing Reg struct +}; + +EXTERN Biobuf* bout; +EXTERN int32 dynloc; +EXTERN uchar reg[D_NONE]; +EXTERN int32 pcloc; // instruction counter +EXTERN Strlit emptystring; +extern char* anames[]; +EXTERN Hist* hist; +EXTERN Prog zprog; +EXTERN Node* curfn; +EXTERN Node* newproc; +EXTERN Node* deferproc; +EXTERN Node* deferreturn; +EXTERN Node* panicindex; +EXTERN Node* panicslice; +EXTERN Node* throwreturn; +EXTERN vlong unmappedzero; + +/* + * gen.c + */ +void compile(Node*); +void proglist(void); +void gen(Node*); +Node* lookdot(Node*, Node*, int); +void cgen_as(Node*, Node*); +void cgen_callmeth(Node*, int); +void cgen_callinter(Node*, Node*, int); +void cgen_proc(Node*, int); +void cgen_callret(Node*, Node*); +void cgen_div(int, Node*, Node*, Node*); +void cgen_bmul(int, Node*, Node*, Node*); +void cgen_shift(int, Node*, Node*, Node*); +void cgen_dcl(Node*); +int needconvert(Type*, Type*); +void genconv(Type*, Type*); +void allocparams(void); +void checklabels(); +void ginscall(Node*, int); +int gen_as_init(Node*); + +/* + * cgen + */ +void agen(Node*, Node*); +void igen(Node*, Node*, Node*); +vlong fieldoffset(Type*, Node*); +void bgen(Node*, int, Prog*); +void sgen(Node*, Node*, int32); +void gmove(Node*, Node*); +Prog* gins(int, Node*, Node*); +int samaddr(Node*, Node*); +void naddr(Node*, Addr*, int); +void cgen_aret(Node*, Node*); +int cgen_inline(Node*, Node*); +void restx(Node*, Node*); +void savex(int, Node*, Node*, Node*, Type*); +int componentgen(Node*, Node*); + +/* + * gsubr.c + */ +void clearp(Prog*); +void proglist(void); +Prog* gbranch(int, Type*); +Prog* prog(int); +void gaddoffset(Node*); +void gconv(int, int); +int conv2pt(Type*); +vlong convvtox(vlong, int); +void fnparam(Type*, int, int); +Prog* gop(int, Node*, Node*, Node*); +int optoas(int, Type*); +void ginit(void); +void gclean(void); +void regalloc(Node*, Type*, Node*); +void regfree(Node*); +Node* nodarg(Type*, int); +void nodreg(Node*, Type*, int); +void nodindreg(Node*, Type*, int); +void gconreg(int, vlong, int); +void ginscon(int, vlong, Node*); +void buildtxt(void); +Plist* newplist(void); +int isfat(Type*); +void sudoclean(void); +int sudoaddable(int, Node*, Addr*); +void afunclit(Addr*); +void datagostring(Strlit*, Addr*); +void nodfconst(Node*, Type*, Mpflt*); + +/* + * cplx.c + */ +int complexop(Node*, Node*); +void complexmove(Node*, Node*); +void complexgen(Node*, Node*); +void complexbool(int, Node*, Node*, int, Prog*); + +/* + * gobj.c + */ +void datastring(char*, int, Addr*); + +/* + * list.c + */ +int Aconv(Fmt*); +int Dconv(Fmt*); +int Pconv(Fmt*); +int Rconv(Fmt*); +int Yconv(Fmt*); +void listinit(void); + +void zaddr(Biobuf*, Addr*, int, int); + diff --git a/src/cmd/6g/ggen.c b/src/cmd/6g/ggen.c new file mode 100644 index 000000000..a5f278384 --- /dev/null +++ b/src/cmd/6g/ggen.c @@ -0,0 +1,1412 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +#undef EXTERN +#define EXTERN +#include "gg.h" +#include "opt.h" + +void +defframe(Prog *ptxt) +{ + // fill in argument size + ptxt->to.offset = rnd(curfn->type->argwid, widthptr); + + // fill in final stack size + ptxt->to.offset <<= 32; + ptxt->to.offset |= rnd(stksize+maxarg, widthptr); +} + +// Sweep the prog list to mark any used nodes. +void +markautoused(Prog* p) +{ + for (; p; p = p->link) { + if (p->from.type == D_AUTO && p->from.node) + p->from.node->used++; + + if (p->to.type == D_AUTO && p->to.node) + p->to.node->used++; + } +} + +// Fixup instructions after compactframe has moved all autos around. +void +fixautoused(Prog* p) +{ + for (; p; p = p->link) { + if (p->from.type == D_AUTO && p->from.node) + p->from.offset += p->from.node->stkdelta; + + if (p->to.type == D_AUTO && p->to.node) + p->to.offset += p->to.node->stkdelta; + } +} + + +/* + * generate: + * call f + * proc=0 normal call + * proc=1 goroutine run in new proc + * proc=2 defer call save away stack + */ +void +ginscall(Node *f, int proc) +{ + Prog *p; + Node reg, con; + + switch(proc) { + default: + fatal("ginscall: bad proc %d", proc); + break; + + case 0: // normal call + p = gins(ACALL, N, f); + afunclit(&p->to); + break; + + case 1: // call in new proc (go) + case 2: // deferred call (defer) + nodreg(®, types[TINT64], D_CX); + gins(APUSHQ, f, N); + nodconst(&con, types[TINT32], argsize(f->type)); + gins(APUSHQ, &con, N); + if(proc == 1) + ginscall(newproc, 0); + else { + if(!hasdefer) + fatal("hasdefer=0 but has defer"); + ginscall(deferproc, 0); + } + gins(APOPQ, N, ®); + gins(APOPQ, N, ®); + if(proc == 2) { + nodreg(®, types[TINT64], D_AX); + gins(ATESTQ, ®, ®); + patch(gbranch(AJNE, T), retpc); + } + break; + } +} + +/* + * n is call to interface method. + * generate res = n. + */ +void +cgen_callinter(Node *n, Node *res, int proc) +{ + Node *i, *f; + Node tmpi, nodo, nodr, nodsp; + + i = n->left; + if(i->op != ODOTINTER) + fatal("cgen_callinter: not ODOTINTER %O", i->op); + + f = i->right; // field + if(f->op != ONAME) + fatal("cgen_callinter: not ONAME %O", f->op); + + i = i->left; // interface + + if(!i->addable) { + tempname(&tmpi, i->type); + cgen(i, &tmpi); + i = &tmpi; + } + + genlist(n->list); // assign the args + + regalloc(&nodr, types[tptr], res); + regalloc(&nodo, types[tptr], &nodr); + nodo.op = OINDREG; + + agen(i, &nodr); // REG = &inter + + nodindreg(&nodsp, types[tptr], D_SP); + nodo.xoffset += widthptr; + cgen(&nodo, &nodsp); // 0(SP) = 8(REG) -- i.data + + nodo.xoffset -= widthptr; + cgen(&nodo, &nodr); // REG = 0(REG) -- i.tab + + nodo.xoffset = n->left->xoffset + 3*widthptr + 8; + cgen(&nodo, &nodr); // REG = 32+offset(REG) -- i.tab->fun[f] + + // BOTCH nodr.type = fntype; + nodr.type = n->left->type; + ginscall(&nodr, proc); + + regfree(&nodr); + regfree(&nodo); + + setmaxarg(n->left->type); +} + +/* + * generate function call; + * proc=0 normal call + * proc=1 goroutine run in new proc + * proc=2 defer call save away stack + */ +void +cgen_call(Node *n, int proc) +{ + Type *t; + Node nod, afun; + + if(n == N) + return; + + if(n->left->ullman >= UINF) { + // if name involves a fn call + // precompute the address of the fn + tempname(&afun, types[tptr]); + cgen(n->left, &afun); + } + + genlist(n->list); // assign the args + t = n->left->type; + + setmaxarg(t); + + // call tempname pointer + if(n->left->ullman >= UINF) { + regalloc(&nod, types[tptr], N); + cgen_as(&nod, &afun); + nod.type = t; + ginscall(&nod, proc); + regfree(&nod); + goto ret; + } + + // call pointer + if(n->left->op != ONAME || n->left->class != PFUNC) { + regalloc(&nod, types[tptr], N); + cgen_as(&nod, n->left); + nod.type = t; + ginscall(&nod, proc); + regfree(&nod); + goto ret; + } + + // call direct + n->left->method = 1; + ginscall(n->left, proc); + + +ret: + ; +} + +/* + * call to n has already been generated. + * generate: + * res = return value from call. + */ +void +cgen_callret(Node *n, Node *res) +{ + Node nod; + Type *fp, *t; + Iter flist; + + t = n->left->type; + if(t->etype == TPTR32 || t->etype == TPTR64) + t = t->type; + + fp = structfirst(&flist, getoutarg(t)); + if(fp == T) + fatal("cgen_callret: nil"); + + memset(&nod, 0, sizeof(nod)); + nod.op = OINDREG; + nod.val.u.reg = D_SP; + nod.addable = 1; + + nod.xoffset = fp->width; + nod.type = fp->type; + cgen_as(res, &nod); +} + +/* + * call to n has already been generated. + * generate: + * res = &return value from call. + */ +void +cgen_aret(Node *n, Node *res) +{ + Node nod1, nod2; + Type *fp, *t; + Iter flist; + + t = n->left->type; + if(isptr[t->etype]) + t = t->type; + + fp = structfirst(&flist, getoutarg(t)); + if(fp == T) + fatal("cgen_aret: nil"); + + memset(&nod1, 0, sizeof(nod1)); + nod1.op = OINDREG; + nod1.val.u.reg = D_SP; + nod1.addable = 1; + + nod1.xoffset = fp->width; + nod1.type = fp->type; + + if(res->op != OREGISTER) { + regalloc(&nod2, types[tptr], res); + gins(ALEAQ, &nod1, &nod2); + gins(AMOVQ, &nod2, res); + regfree(&nod2); + } else + gins(ALEAQ, &nod1, res); +} + +/* + * generate return. + * n->left is assignments to return values. + */ +void +cgen_ret(Node *n) +{ + genlist(n->list); // copy out args + if(hasdefer || curfn->exit) + gjmp(retpc); + else + gins(ARET, N, N); +} + +/* + * generate += *= etc. + */ +void +cgen_asop(Node *n) +{ + Node n1, n2, n3, n4; + Node *nl, *nr; + Prog *p1; + Addr addr; + int a; + + nl = n->left; + nr = n->right; + + if(nr->ullman >= UINF && nl->ullman >= UINF) { + tempname(&n1, nr->type); + cgen(nr, &n1); + n2 = *n; + n2.right = &n1; + cgen_asop(&n2); + goto ret; + } + + if(!isint[nl->type->etype]) + goto hard; + if(!isint[nr->type->etype]) + goto hard; + + switch(n->etype) { + case OADD: + if(smallintconst(nr)) + if(mpgetfix(nr->val.u.xval) == 1) { + a = optoas(OINC, nl->type); + if(nl->addable) { + gins(a, N, nl); + goto ret; + } + if(sudoaddable(a, nl, &addr)) { + p1 = gins(a, N, N); + p1->to = addr; + sudoclean(); + goto ret; + } + } + break; + + case OSUB: + if(smallintconst(nr)) + if(mpgetfix(nr->val.u.xval) == 1) { + a = optoas(ODEC, nl->type); + if(nl->addable) { + gins(a, N, nl); + goto ret; + } + if(sudoaddable(a, nl, &addr)) { + p1 = gins(a, N, N); + p1->to = addr; + sudoclean(); + goto ret; + } + } + break; + } + + switch(n->etype) { + case OADD: + case OSUB: + case OXOR: + case OAND: + case OOR: + a = optoas(n->etype, nl->type); + if(nl->addable) { + if(smallintconst(nr)) { + gins(a, nr, nl); + goto ret; + } + regalloc(&n2, nr->type, N); + cgen(nr, &n2); + gins(a, &n2, nl); + regfree(&n2); + goto ret; + } + if(nr->ullman < UINF) + if(sudoaddable(a, nl, &addr)) { + if(smallintconst(nr)) { + p1 = gins(a, nr, N); + p1->to = addr; + sudoclean(); + goto ret; + } + regalloc(&n2, nr->type, N); + cgen(nr, &n2); + p1 = gins(a, &n2, N); + p1->to = addr; + regfree(&n2); + sudoclean(); + goto ret; + } + } + +hard: + n2.op = 0; + n1.op = 0; + if(nr->ullman >= nl->ullman || nl->addable) { + regalloc(&n2, nr->type, N); + cgen(nr, &n2); + nr = &n2; + } else { + tempname(&n2, nr->type); + cgen(nr, &n2); + nr = &n2; + } + if(!nl->addable) { + igen(nl, &n1, N); + nl = &n1; + } + + n3 = *n; + n3.left = nl; + n3.right = nr; + n3.op = n->etype; + + regalloc(&n4, nl->type, N); + cgen(&n3, &n4); + gmove(&n4, nl); + + if(n1.op) + regfree(&n1); + if(n2.op == OREGISTER) + regfree(&n2); + regfree(&n4); + +ret: + ; +} + +int +samereg(Node *a, Node *b) +{ + if(a == N || b == N) + return 0; + if(a->op != OREGISTER) + return 0; + if(b->op != OREGISTER) + return 0; + if(a->val.u.reg != b->val.u.reg) + return 0; + return 1; +} + +/* + * generate division. + * generates one of: + * res = nl / nr + * res = nl % nr + * according to op. + */ +void +dodiv(int op, Node *nl, Node *nr, Node *res) +{ + int a, check; + Node n3, n4, n5; + Type *t; + Node ax, dx, oldax, olddx; + Prog *p1, *p2, *p3; + + // Have to be careful about handling + // most negative int divided by -1 correctly. + // The hardware will trap. + // Also the byte divide instruction needs AH, + // which we otherwise don't have to deal with. + // Easiest way to avoid for int8, int16: use int32. + // For int32 and int64, use explicit test. + // Could use int64 hw for int32. + t = nl->type; + check = 0; + if(issigned[t->etype]) { + check = 1; + if(isconst(nl, CTINT) && mpgetfix(nl->val.u.xval) != -1LL<<(t->width*8-1)) + check = 0; + else if(isconst(nr, CTINT) && mpgetfix(nr->val.u.xval) != -1) + check = 0; + } + if(t->width < 4) { + if(issigned[t->etype]) + t = types[TINT32]; + else + t = types[TUINT32]; + check = 0; + } + a = optoas(op, t); + + regalloc(&n3, t, N); + if(nl->ullman >= nr->ullman) { + savex(D_AX, &ax, &oldax, res, t); + cgen(nl, &ax); + regalloc(&ax, t, &ax); // mark ax live during cgen + cgen(nr, &n3); + regfree(&ax); + } else { + cgen(nr, &n3); + savex(D_AX, &ax, &oldax, res, t); + cgen(nl, &ax); + } + p3 = P; + if(check) { + nodconst(&n4, t, -1); + gins(optoas(OCMP, t), &n3, &n4); + p1 = gbranch(optoas(ONE, t), T); + nodconst(&n4, t, -1LL<<(t->width*8-1)); + if(t->width == 8) { + n5 = n4; + regalloc(&n4, t, N); + gins(AMOVQ, &n5, &n4); + } + gins(optoas(OCMP, t), &ax, &n4); + p2 = gbranch(optoas(ONE, t), T); + if(op == ODIV) + gmove(&n4, res); + if(t->width == 8) + regfree(&n4); + if(op == OMOD) { + nodconst(&n4, t, 0); + gmove(&n4, res); + } + p3 = gbranch(AJMP, T); + patch(p1, pc); + patch(p2, pc); + } + savex(D_DX, &dx, &olddx, res, t); + if(!issigned[t->etype]) { + nodconst(&n4, t, 0); + gmove(&n4, &dx); + } else + gins(optoas(OEXTEND, t), N, N); + gins(a, &n3, N); + regfree(&n3); + if(op == ODIV) + gmove(&ax, res); + else + gmove(&dx, res); + restx(&dx, &olddx); + if(check) + patch(p3, pc); + restx(&ax, &oldax); +} + +/* + * register dr is one of the special ones (AX, CX, DI, SI, etc.). + * we need to use it. if it is already allocated as a temporary + * (r > 1; can only happen if a routine like sgen passed a + * special as cgen's res and then cgen used regalloc to reuse + * it as its own temporary), then move it for now to another + * register. caller must call restx to move it back. + * the move is not necessary if dr == res, because res is + * known to be dead. + */ +void +savex(int dr, Node *x, Node *oldx, Node *res, Type *t) +{ + int r; + + r = reg[dr]; + + // save current ax and dx if they are live + // and not the destination + memset(oldx, 0, sizeof *oldx); + nodreg(x, t, dr); + if(r > 1 && !samereg(x, res)) { + regalloc(oldx, types[TINT64], N); + x->type = types[TINT64]; + gmove(x, oldx); + x->type = t; + oldx->ostk = r; // squirrel away old r value + reg[dr] = 1; + } +} + +void +restx(Node *x, Node *oldx) +{ + if(oldx->op != 0) { + x->type = types[TINT64]; + reg[x->val.u.reg] = oldx->ostk; + gmove(oldx, x); + regfree(oldx); + } +} + +/* + * generate division according to op, one of: + * res = nl / nr + * res = nl % nr + */ +void +cgen_div(int op, Node *nl, Node *nr, Node *res) +{ + Node n1, n2, n3, savl, savr; + Node ax, dx, oldax, olddx; + int n, w, s, a; + Magic m; + + if(nl->ullman >= UINF) { + tempname(&savl, nl->type); + cgen(nl, &savl); + nl = &savl; + } + if(nr->ullman >= UINF) { + tempname(&savr, nr->type); + cgen(nr, &savr); + nr = &savr; + } + + if(nr->op != OLITERAL) + goto longdiv; + + // special cases of mod/div + // by a constant + w = nl->type->width*8; + s = 0; + n = powtwo(nr); + if(n >= 1000) { + // negative power of 2 + s = 1; + n -= 1000; + } + + if(n+1 >= w) { + // just sign bit + goto longdiv; + } + + if(n < 0) + goto divbymul; + switch(n) { + case 0: + // divide by 1 + regalloc(&n1, nl->type, res); + cgen(nl, &n1); + if(op == OMOD) { + gins(optoas(OXOR, nl->type), &n1, &n1); + } else + if(s) + gins(optoas(OMINUS, nl->type), N, &n1); + gmove(&n1, res); + regfree(&n1); + return; + case 1: + // divide by 2 + if(op == OMOD) { + if(issigned[nl->type->etype]) + goto longmod; + regalloc(&n1, nl->type, res); + cgen(nl, &n1); + nodconst(&n2, nl->type, 1); + gins(optoas(OAND, nl->type), &n2, &n1); + gmove(&n1, res); + regfree(&n1); + return; + } + regalloc(&n1, nl->type, res); + cgen(nl, &n1); + if(!issigned[nl->type->etype]) + break; + + // develop -1 iff nl is negative + regalloc(&n2, nl->type, N); + gmove(&n1, &n2); + nodconst(&n3, nl->type, w-1); + gins(optoas(ORSH, nl->type), &n3, &n2); + gins(optoas(OSUB, nl->type), &n2, &n1); + regfree(&n2); + break; + default: + if(op == OMOD) { + if(issigned[nl->type->etype]) + goto longmod; + regalloc(&n1, nl->type, res); + cgen(nl, &n1); + nodconst(&n2, nl->type, mpgetfix(nr->val.u.xval)-1); + if(!smallintconst(&n2)) { + regalloc(&n3, nl->type, N); + gmove(&n2, &n3); + gins(optoas(OAND, nl->type), &n3, &n1); + regfree(&n3); + } else + gins(optoas(OAND, nl->type), &n2, &n1); + gmove(&n1, res); + regfree(&n1); + return; + } + regalloc(&n1, nl->type, res); + cgen(nl, &n1); + if(!issigned[nl->type->etype]) + break; + + // develop (2^k)-1 iff nl is negative + regalloc(&n2, nl->type, N); + gmove(&n1, &n2); + nodconst(&n3, nl->type, w-1); + gins(optoas(ORSH, nl->type), &n3, &n2); + nodconst(&n3, nl->type, w-n); + gins(optoas(ORSH, tounsigned(nl->type)), &n3, &n2); + gins(optoas(OADD, nl->type), &n2, &n1); + regfree(&n2); + break; + } + nodconst(&n2, nl->type, n); + gins(optoas(ORSH, nl->type), &n2, &n1); + if(s) + gins(optoas(OMINUS, nl->type), N, &n1); + gmove(&n1, res); + regfree(&n1); + return; + +divbymul: + // try to do division by multiply by (2^w)/d + // see hacker's delight chapter 10 + switch(simtype[nl->type->etype]) { + default: + goto longdiv; + + case TUINT8: + case TUINT16: + case TUINT32: + case TUINT64: + m.w = w; + m.ud = mpgetfix(nr->val.u.xval); + umagic(&m); + if(m.bad) + break; + if(op == OMOD) + goto longmod; + + regalloc(&n1, nl->type, N); + cgen(nl, &n1); // num -> reg(n1) + + savex(D_AX, &ax, &oldax, res, nl->type); + savex(D_DX, &dx, &olddx, res, nl->type); + + nodconst(&n2, nl->type, m.um); + gmove(&n2, &ax); // const->ax + + gins(optoas(OHMUL, nl->type), &n1, N); // imul reg + if(w == 8) { + // fix up 8-bit multiply + Node ah, dl; + nodreg(&ah, types[TUINT8], D_AH); + nodreg(&dl, types[TUINT8], D_DL); + gins(AMOVB, &ah, &dl); + } + + if(m.ua) { + // need to add numerator accounting for overflow + gins(optoas(OADD, nl->type), &n1, &dx); + nodconst(&n2, nl->type, 1); + gins(optoas(ORRC, nl->type), &n2, &dx); + nodconst(&n2, nl->type, m.s-1); + gins(optoas(ORSH, nl->type), &n2, &dx); + } else { + nodconst(&n2, nl->type, m.s); + gins(optoas(ORSH, nl->type), &n2, &dx); // shift dx + } + + + regfree(&n1); + gmove(&dx, res); + + restx(&ax, &oldax); + restx(&dx, &olddx); + return; + + case TINT8: + case TINT16: + case TINT32: + case TINT64: + m.w = w; + m.sd = mpgetfix(nr->val.u.xval); + smagic(&m); + if(m.bad) + break; + if(op == OMOD) + goto longmod; + + regalloc(&n1, nl->type, N); + cgen(nl, &n1); // num -> reg(n1) + + savex(D_AX, &ax, &oldax, res, nl->type); + savex(D_DX, &dx, &olddx, res, nl->type); + + nodconst(&n2, nl->type, m.sm); + gmove(&n2, &ax); // const->ax + + gins(optoas(OHMUL, nl->type), &n1, N); // imul reg + if(w == 8) { + // fix up 8-bit multiply + Node ah, dl; + nodreg(&ah, types[TUINT8], D_AH); + nodreg(&dl, types[TUINT8], D_DL); + gins(AMOVB, &ah, &dl); + } + + if(m.sm < 0) { + // need to add numerator + gins(optoas(OADD, nl->type), &n1, &dx); + } + + nodconst(&n2, nl->type, m.s); + gins(optoas(ORSH, nl->type), &n2, &dx); // shift dx + + nodconst(&n2, nl->type, w-1); + gins(optoas(ORSH, nl->type), &n2, &n1); // -1 iff num is neg + gins(optoas(OSUB, nl->type), &n1, &dx); // added + + if(m.sd < 0) { + // this could probably be removed + // by factoring it into the multiplier + gins(optoas(OMINUS, nl->type), N, &dx); + } + + regfree(&n1); + gmove(&dx, res); + + restx(&ax, &oldax); + restx(&dx, &olddx); + return; + } + goto longdiv; + +longdiv: + // division and mod using (slow) hardware instruction + dodiv(op, nl, nr, res); + return; + +longmod: + // mod using formula A%B = A-(A/B*B) but + // we know that there is a fast algorithm for A/B + regalloc(&n1, nl->type, res); + cgen(nl, &n1); + regalloc(&n2, nl->type, N); + cgen_div(ODIV, &n1, nr, &n2); + a = optoas(OMUL, nl->type); + if(w == 8) { + // use 2-operand 16-bit multiply + // because there is no 2-operand 8-bit multiply + a = AIMULW; + } + if(!smallintconst(nr)) { + regalloc(&n3, nl->type, N); + cgen(nr, &n3); + gins(a, &n3, &n2); + regfree(&n3); + } else + gins(a, nr, &n2); + gins(optoas(OSUB, nl->type), &n2, &n1); + gmove(&n1, res); + regfree(&n1); + regfree(&n2); +} + +/* + * generate shift according to op, one of: + * res = nl << nr + * res = nl >> nr + */ +void +cgen_shift(int op, Node *nl, Node *nr, Node *res) +{ + Node n1, n2, n3, n4, n5, cx, oldcx; + int a, rcx; + Prog *p1; + uvlong sc; + Type *tcount; + + a = optoas(op, nl->type); + + if(nr->op == OLITERAL) { + regalloc(&n1, nl->type, res); + cgen(nl, &n1); + sc = mpgetfix(nr->val.u.xval); + if(sc >= nl->type->width*8) { + // large shift gets 2 shifts by width + nodconst(&n3, types[TUINT32], nl->type->width*8-1); + gins(a, &n3, &n1); + gins(a, &n3, &n1); + } else + gins(a, nr, &n1); + gmove(&n1, res); + regfree(&n1); + goto ret; + } + + if(nl->ullman >= UINF) { + tempname(&n4, nl->type); + cgen(nl, &n4); + nl = &n4; + } + if(nr->ullman >= UINF) { + tempname(&n5, nr->type); + cgen(nr, &n5); + nr = &n5; + } + + rcx = reg[D_CX]; + nodreg(&n1, types[TUINT32], D_CX); + + // Allow either uint32 or uint64 as shift type, + // to avoid unnecessary conversion from uint32 to uint64 + // just to do the comparison. + tcount = types[simtype[nr->type->etype]]; + if(tcount->etype < TUINT32) + tcount = types[TUINT32]; + + regalloc(&n1, nr->type, &n1); // to hold the shift type in CX + regalloc(&n3, tcount, &n1); // to clear high bits of CX + + nodreg(&cx, types[TUINT64], D_CX); + memset(&oldcx, 0, sizeof oldcx); + if(rcx > 0 && !samereg(&cx, res)) { + regalloc(&oldcx, types[TUINT64], N); + gmove(&cx, &oldcx); + } + cx.type = tcount; + + if(samereg(&cx, res)) + regalloc(&n2, nl->type, N); + else + regalloc(&n2, nl->type, res); + if(nl->ullman >= nr->ullman) { + cgen(nl, &n2); + cgen(nr, &n1); + gmove(&n1, &n3); + } else { + cgen(nr, &n1); + gmove(&n1, &n3); + cgen(nl, &n2); + } + regfree(&n3); + + // test and fix up large shifts + nodconst(&n3, tcount, nl->type->width*8); + gins(optoas(OCMP, tcount), &n1, &n3); + p1 = gbranch(optoas(OLT, tcount), T); + if(op == ORSH && issigned[nl->type->etype]) { + nodconst(&n3, types[TUINT32], nl->type->width*8-1); + gins(a, &n3, &n2); + } else { + nodconst(&n3, nl->type, 0); + gmove(&n3, &n2); + } + patch(p1, pc); + gins(a, &n1, &n2); + + if(oldcx.op != 0) { + cx.type = types[TUINT64]; + gmove(&oldcx, &cx); + regfree(&oldcx); + } + + gmove(&n2, res); + + regfree(&n1); + regfree(&n2); + +ret: + ; +} + +/* + * generate byte multiply: + * res = nl * nr + * no 2-operand byte multiply instruction so have to do + * 16-bit multiply and take bottom half. + */ +void +cgen_bmul(int op, Node *nl, Node *nr, Node *res) +{ + Node n1b, n2b, n1w, n2w; + Type *t; + int a; + + if(nl->ullman >= nr->ullman) { + regalloc(&n1b, nl->type, res); + cgen(nl, &n1b); + regalloc(&n2b, nr->type, N); + cgen(nr, &n2b); + } else { + regalloc(&n2b, nr->type, N); + cgen(nr, &n2b); + regalloc(&n1b, nl->type, res); + cgen(nl, &n1b); + } + + // copy from byte to short registers + t = types[TUINT16]; + if(issigned[nl->type->etype]) + t = types[TINT16]; + + regalloc(&n2w, t, &n2b); + cgen(&n2b, &n2w); + + regalloc(&n1w, t, &n1b); + cgen(&n1b, &n1w); + + a = optoas(op, t); + gins(a, &n2w, &n1w); + cgen(&n1w, &n1b); + cgen(&n1b, res); + + regfree(&n1w); + regfree(&n2w); + regfree(&n1b); + regfree(&n2b); +} + +void +clearfat(Node *nl) +{ + uint32 w, c, q; + Node n1, oldn1, ax, oldax; + + /* clear a fat object */ + if(debug['g']) + dump("\nclearfat", nl); + + + w = nl->type->width; + if(w == 16) + if(componentgen(N, nl)) + return; + + c = w % 8; // bytes + q = w / 8; // quads + + savex(D_DI, &n1, &oldn1, N, types[tptr]); + agen(nl, &n1); + + savex(D_AX, &ax, &oldax, N, types[tptr]); + gconreg(AMOVQ, 0, D_AX); + + if(q >= 4) { + gconreg(AMOVQ, q, D_CX); + gins(AREP, N, N); // repeat + gins(ASTOSQ, N, N); // STOQ AL,*(DI)+ + } else + while(q > 0) { + gins(ASTOSQ, N, N); // STOQ AL,*(DI)+ + q--; + } + + if(c >= 4) { + gconreg(AMOVQ, c, D_CX); + gins(AREP, N, N); // repeat + gins(ASTOSB, N, N); // STOB AL,*(DI)+ + } else + while(c > 0) { + gins(ASTOSB, N, N); // STOB AL,*(DI)+ + c--; + } + + restx(&n1, &oldn1); + restx(&ax, &oldax); +} + +static int +regcmp(const void *va, const void *vb) +{ + Node *ra, *rb; + + ra = (Node*)va; + rb = (Node*)vb; + return ra->local - rb->local; +} + +static Prog* throwpc; + +void +getargs(NodeList *nn, Node *reg, int n) +{ + NodeList *l; + int i; + + throwpc = nil; + + l = nn; + for(i=0; in->right) && !isslice(l->n->right->type)) { + regalloc(reg+i, l->n->right->type, N); + cgen(l->n->right, reg+i); + } else + reg[i] = *l->n->right; + if(reg[i].local != 0) + yyerror("local used"); + reg[i].local = l->n->left->xoffset; + l = l->next; + } + qsort((void*)reg, n, sizeof(*reg), regcmp); + for(i=0; iop == OCONV && is64(nl->type)) + nl = nl->left; + if(nr->op == OCONV && is64(nr->type)) + nr = nr->left; + + op = OLE; + if(smallintconst(nl)) { + cl = mpgetfix(nl->val.u.xval); + if(cl == 0) + return; + if(smallintconst(nr)) + return; + // put the constant on the right + op = brrev(op); + c = nl; + nl = nr; + nr = c; + } + if(is64(nr->type) && smallintconst(nr)) + nr->type = types[TUINT32]; + + n1.op = OXXX; + t = types[TUINT32]; + if(nl->type->width != t->width || nr->type->width != t->width) { + if((is64(nl->type) && nl->op != OLITERAL) || (is64(nr->type) && nr->op != OLITERAL)) + t = types[TUINT64]; + + // Check if we need to use a temporary. + // At least one of the arguments is 32 bits + // (the len or cap) so one temporary suffices. + if(nl->type->width != t->width && nl->op != OLITERAL) { + regalloc(&n1, t, nl); + gmove(nl, &n1); + nl = &n1; + } else if(nr->type->width != t->width && nr->op != OLITERAL) { + regalloc(&n1, t, nr); + gmove(nr, &n1); + nr = &n1; + } + } + gins(optoas(OCMP, t), nl, nr); + if(n1.op != OXXX) + regfree(&n1); + if(throwpc == nil) { + p1 = gbranch(optoas(op, t), T); + throwpc = pc; + ginscall(panicslice, 0); + patch(p1, pc); + } else { + op = brcom(op); + p1 = gbranch(optoas(op, t), T); + patch(p1, throwpc); + } +} + +int +sleasy(Node *n) +{ + if(n->op != ONAME) + return 0; + if(!n->addable) + return 0; + return 1; +} + +// generate inline code for +// slicearray +// sliceslice +// arraytoslice +int +cgen_inline(Node *n, Node *res) +{ + Node nodes[5]; + Node n1, n2, nres, ntemp; + vlong v; + int i, narg, nochk; + + if(n->op != OCALLFUNC) + goto no; + if(!n->left->addable) + goto no; + if(n->left->sym == S) + goto no; + if(n->left->sym->pkg != runtimepkg) + goto no; + if(strcmp(n->left->sym->name, "slicearray") == 0) + goto slicearray; + if(strcmp(n->left->sym->name, "sliceslice") == 0) { + narg = 4; + goto sliceslice; + } + if(strcmp(n->left->sym->name, "sliceslice1") == 0) { + narg = 3; + goto sliceslice; + } + goto no; + +slicearray: + if(!sleasy(res)) + goto no; + getargs(n->list, nodes, 5); + + // if(hb[3] > nel[1]) goto throw + cmpandthrow(&nodes[3], &nodes[1]); + + // if(lb[2] > hb[3]) goto throw + cmpandthrow(&nodes[2], &nodes[3]); + + // len = hb[3] - lb[2] (destroys hb) + n2 = *res; + n2.xoffset += Array_nel; + n2.type = types[TUINT32]; + + if(smallintconst(&nodes[3]) && smallintconst(&nodes[2])) { + v = mpgetfix(nodes[3].val.u.xval) - + mpgetfix(nodes[2].val.u.xval); + nodconst(&n1, types[TUINT32], v); + gins(optoas(OAS, types[TUINT32]), &n1, &n2); + } else { + regalloc(&n1, types[TUINT32], &nodes[3]); + gmove(&nodes[3], &n1); + if(!smallintconst(&nodes[2]) || mpgetfix(nodes[2].val.u.xval) != 0) + gins(optoas(OSUB, types[TUINT32]), &nodes[2], &n1); + gins(optoas(OAS, types[TUINT32]), &n1, &n2); + regfree(&n1); + } + + // cap = nel[1] - lb[2] (destroys nel) + n2 = *res; + n2.xoffset += Array_cap; + n2.type = types[TUINT32]; + + if(smallintconst(&nodes[1]) && smallintconst(&nodes[2])) { + v = mpgetfix(nodes[1].val.u.xval) - + mpgetfix(nodes[2].val.u.xval); + nodconst(&n1, types[TUINT32], v); + gins(optoas(OAS, types[TUINT32]), &n1, &n2); + } else { + regalloc(&n1, types[TUINT32], &nodes[1]); + gmove(&nodes[1], &n1); + if(!smallintconst(&nodes[2]) || mpgetfix(nodes[2].val.u.xval) != 0) + gins(optoas(OSUB, types[TUINT32]), &nodes[2], &n1); + gins(optoas(OAS, types[TUINT32]), &n1, &n2); + regfree(&n1); + } + + // if slice could be too big, dereference to + // catch nil array pointer. + if(nodes[0].op == OREGISTER && nodes[0].type->type->width >= unmappedzero) { + n2 = nodes[0]; + n2.xoffset = 0; + n2.op = OINDREG; + n2.type = types[TUINT8]; + gins(ATESTB, nodintconst(0), &n2); + } + + // ary = old[0] + (lb[2] * width[4]) (destroys old) + n2 = *res; + n2.xoffset += Array_array; + n2.type = types[tptr]; + + if(smallintconst(&nodes[2]) && smallintconst(&nodes[4])) { + v = mpgetfix(nodes[2].val.u.xval) * + mpgetfix(nodes[4].val.u.xval); + if(v != 0) + ginscon(optoas(OADD, types[tptr]), v, &nodes[0]); + } else { + regalloc(&n1, types[tptr], &nodes[2]); + gmove(&nodes[2], &n1); + if(!smallintconst(&nodes[4]) || mpgetfix(nodes[4].val.u.xval) != 1) + gins(optoas(OMUL, types[tptr]), &nodes[4], &n1); + gins(optoas(OADD, types[tptr]), &n1, &nodes[0]); + regfree(&n1); + } + gins(optoas(OAS, types[tptr]), &nodes[0], &n2); + + for(i=0; i<5; i++) { + if(nodes[i].op == OREGISTER) + regfree(&nodes[i]); + } + return 1; + +sliceslice: + nochk = n->etype; // skip bounds checking + ntemp.op = OXXX; + if(!sleasy(n->list->n->right)) { + Node *n0; + + n0 = n->list->n->right; + tempname(&ntemp, res->type); + cgen(n0, &ntemp); + n->list->n->right = &ntemp; + getargs(n->list, nodes, narg); + n->list->n->right = n0; + } else + getargs(n->list, nodes, narg); + + nres = *res; // result + if(!sleasy(res)) { + if(ntemp.op == OXXX) + tempname(&ntemp, res->type); + nres = ntemp; + } + + if(narg == 3) { // old[lb:] + // move width to where it would be for old[lb:hb] + nodes[3] = nodes[2]; + nodes[2].op = OXXX; + + // if(lb[1] > old.nel[0]) goto throw; + n2 = nodes[0]; + n2.xoffset += Array_nel; + n2.type = types[TUINT32]; + if(!nochk) + cmpandthrow(&nodes[1], &n2); + + // ret.nel = old.nel[0]-lb[1]; + n2 = nodes[0]; + n2.xoffset += Array_nel; + n2.type = types[TUINT32]; + + regalloc(&n1, types[TUINT32], N); + gins(optoas(OAS, types[TUINT32]), &n2, &n1); + if(!smallintconst(&nodes[1]) || mpgetfix(nodes[1].val.u.xval) != 0) + gins(optoas(OSUB, types[TUINT32]), &nodes[1], &n1); + + n2 = nres; + n2.xoffset += Array_nel; + n2.type = types[TUINT32]; + gins(optoas(OAS, types[TUINT32]), &n1, &n2); + regfree(&n1); + } else { // old[lb:hb] + n2 = nodes[0]; + n2.xoffset += Array_cap; + n2.type = types[TUINT32]; + if(!nochk) { + // if(hb[2] > old.cap[0]) goto throw; + cmpandthrow(&nodes[2], &n2); + // if(lb[1] > hb[2]) goto throw; + cmpandthrow(&nodes[1], &nodes[2]); + } + // ret.len = hb[2]-lb[1]; (destroys hb[2]) + n2 = nres; + n2.xoffset += Array_nel; + n2.type = types[TUINT32]; + + if(smallintconst(&nodes[2]) && smallintconst(&nodes[1])) { + v = mpgetfix(nodes[2].val.u.xval) - + mpgetfix(nodes[1].val.u.xval); + nodconst(&n1, types[TUINT32], v); + gins(optoas(OAS, types[TUINT32]), &n1, &n2); + } else { + regalloc(&n1, types[TUINT32], &nodes[2]); + gmove(&nodes[2], &n1); + if(!smallintconst(&nodes[1]) || mpgetfix(nodes[1].val.u.xval) != 0) + gins(optoas(OSUB, types[TUINT32]), &nodes[1], &n1); + gins(optoas(OAS, types[TUINT32]), &n1, &n2); + regfree(&n1); + } + } + + // ret.cap = old.cap[0]-lb[1]; (uses hb[2]) + n2 = nodes[0]; + n2.xoffset += Array_cap; + n2.type = types[TUINT32]; + + regalloc(&n1, types[TUINT32], &nodes[2]); + gins(optoas(OAS, types[TUINT32]), &n2, &n1); + if(!smallintconst(&nodes[1]) || mpgetfix(nodes[1].val.u.xval) != 0) + gins(optoas(OSUB, types[TUINT32]), &nodes[1], &n1); + + n2 = nres; + n2.xoffset += Array_cap; + n2.type = types[TUINT32]; + + gins(optoas(OAS, types[TUINT32]), &n1, &n2); + regfree(&n1); + + // ret.array = old.array[0]+lb[1]*width[3]; (uses lb[1]) + n2 = nodes[0]; + n2.xoffset += Array_array; + n2.type = types[tptr]; + regalloc(&n1, types[tptr], &nodes[1]); + if(smallintconst(&nodes[1]) && smallintconst(&nodes[3])) { + gins(optoas(OAS, types[tptr]), &n2, &n1); + v = mpgetfix(nodes[1].val.u.xval) * + mpgetfix(nodes[3].val.u.xval); + if(v != 0) + ginscon(optoas(OADD, types[tptr]), v, &n1); + } else { + gmove(&nodes[1], &n1); + if(!smallintconst(&nodes[3]) || mpgetfix(nodes[3].val.u.xval) != 1) + gins(optoas(OMUL, types[tptr]), &nodes[3], &n1); + gins(optoas(OADD, types[tptr]), &n2, &n1); + } + + n2 = nres; + n2.xoffset += Array_array; + n2.type = types[tptr]; + gins(optoas(OAS, types[tptr]), &n1, &n2); + regfree(&n1); + + for(i=0; i<4; i++) { + if(nodes[i].op == OREGISTER) + regfree(&nodes[i]); + } + + if(!sleasy(res)) { + cgen(&nres, res); + } + return 1; + +no: + return 0; +} diff --git a/src/cmd/6g/gobj.c b/src/cmd/6g/gobj.c new file mode 100644 index 000000000..ba8a4870e --- /dev/null +++ b/src/cmd/6g/gobj.c @@ -0,0 +1,644 @@ +// Derived from Inferno utils/6c/swt.c +// http://code.google.com/p/inferno-os/source/browse/utils/6c/swt.c +// +// Copyright © 1994-1999 Lucent Technologies Inc. All rights reserved. +// Portions Copyright © 1995-1997 C H Forsyth (forsyth@terzarima.net) +// Portions Copyright © 1997-1999 Vita Nuova Limited +// Portions Copyright © 2000-2007 Vita Nuova Holdings Limited (www.vitanuova.com) +// Portions Copyright © 2004,2006 Bruce Ellis +// Portions Copyright © 2005-2007 C H Forsyth (forsyth@terzarima.net) +// Revisions Copyright © 2000-2007 Lucent Technologies Inc. and others +// Portions Copyright © 2009 The Go Authors. All rights reserved. +// +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to deal +// in the Software without restriction, including without limitation the rights +// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +// copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +// THE SOFTWARE. + +#include "gg.h" + +void +zname(Biobuf *b, Sym *s, int t) +{ + Bputc(b, ANAME); /* as */ + Bputc(b, ANAME>>8); /* as */ + Bputc(b, t); /* type */ + Bputc(b, s->sym); /* sym */ + + Bputname(b, s); +} + +void +zfile(Biobuf *b, char *p, int n) +{ + Bputc(b, ANAME); + Bputc(b, ANAME>>8); + Bputc(b, D_FILE); + Bputc(b, 1); + Bputc(b, '<'); + Bwrite(b, p, n); + Bputc(b, 0); +} + +void +zhist(Biobuf *b, int line, vlong offset) +{ + Addr a; + + Bputc(b, AHISTORY); + Bputc(b, AHISTORY>>8); + Bputc(b, line); + Bputc(b, line>>8); + Bputc(b, line>>16); + Bputc(b, line>>24); + zaddr(b, &zprog.from, 0, 0); + a = zprog.to; + if(offset != 0) { + a.offset = offset; + a.type = D_CONST; + } + zaddr(b, &a, 0, 0); +} + +void +zaddr(Biobuf *b, Addr *a, int s, int gotype) +{ + int32 l; + uint64 e; + int i, t; + char *n; + + t = 0; + if(a->index != D_NONE || a->scale != 0) + t |= T_INDEX; + if(s != 0) + t |= T_SYM; + if(gotype != 0) + t |= T_GOTYPE; + + switch(a->type) { + + case D_BRANCH: + if(a->branch == nil) + fatal("unpatched branch"); + a->offset = a->branch->loc; + + default: + t |= T_TYPE; + + case D_NONE: + if(a->offset != 0) { + t |= T_OFFSET; + l = a->offset; + if((vlong)l != a->offset) + t |= T_64; + } + break; + case D_FCONST: + t |= T_FCONST; + break; + case D_SCONST: + t |= T_SCONST; + break; + } + Bputc(b, t); + + if(t & T_INDEX) { /* implies index, scale */ + Bputc(b, a->index); + Bputc(b, a->scale); + } + if(t & T_OFFSET) { /* implies offset */ + l = a->offset; + Bputc(b, l); + Bputc(b, l>>8); + Bputc(b, l>>16); + Bputc(b, l>>24); + if(t & T_64) { + l = a->offset>>32; + Bputc(b, l); + Bputc(b, l>>8); + Bputc(b, l>>16); + Bputc(b, l>>24); + } + } + if(t & T_SYM) /* implies sym */ + Bputc(b, s); + if(t & T_FCONST) { + ieeedtod(&e, a->dval); + l = e; + Bputc(b, l); + Bputc(b, l>>8); + Bputc(b, l>>16); + Bputc(b, l>>24); + l = e >> 32; + Bputc(b, l); + Bputc(b, l>>8); + Bputc(b, l>>16); + Bputc(b, l>>24); + return; + } + if(t & T_SCONST) { + n = a->sval; + for(i=0; itype); + if(t & T_GOTYPE) + Bputc(b, gotype); +} + +static struct { + struct { Sym *sym; short type; } h[NSYM]; + int sym; +} z; + +static void +zsymreset(void) +{ + for(z.sym=0; z.symsym; + if(i < 0 || i >= NSYM) + i = 0; + if(z.h[i].type == t && z.h[i].sym == s) + return i; + i = z.sym; + s->sym = i; + zname(bout, s, t); + z.h[i].sym = s; + z.h[i].type = t; + if(++z.sym >= NSYM) + z.sym = 1; + *new = 1; + return i; +} + +static int +zsymaddr(Addr *a, int *new) +{ + int t; + + t = a->type; + if(t == D_ADDR) + t = a->index; + return zsym(a->sym, t, new); +} + +void +dumpfuncs(void) +{ + Plist *pl; + int sf, st, gf, gt, new; + Sym *s; + Prog *p; + + zsymreset(); + + // fix up pc + pcloc = 0; + for(pl=plist; pl!=nil; pl=pl->link) { + if(isblank(pl->name)) + continue; + for(p=pl->firstpc; p!=P; p=p->link) { + p->loc = pcloc; + if(p->as != ADATA && p->as != AGLOBL) + pcloc++; + } + } + + // put out functions + for(pl=plist; pl!=nil; pl=pl->link) { + if(isblank(pl->name)) + continue; + + if(debug['S']) { + s = S; + if(pl->name != N) + s = pl->name->sym; + print("\n--- prog list \"%S\" ---\n", s); + for(p=pl->firstpc; p!=P; p=p->link) + print("%P\n", p); + } + + for(p=pl->firstpc; p!=P; p=p->link) { + for(;;) { + sf = zsymaddr(&p->from, &new); + gf = zsym(p->from.gotype, D_EXTERN, &new); + if(new && sf == gf) + continue; + st = zsymaddr(&p->to, &new); + if(new && (st == sf || st == gf)) + continue; + gt = zsym(p->to.gotype, D_EXTERN, &new); + if(new && (gt == sf || gt == gf || gt == st)) + continue; + break; + } + + Bputc(bout, p->as); + Bputc(bout, p->as>>8); + Bputc(bout, p->lineno); + Bputc(bout, p->lineno>>8); + Bputc(bout, p->lineno>>16); + Bputc(bout, p->lineno>>24); + zaddr(bout, &p->from, sf, gf); + zaddr(bout, &p->to, st, gt); + } + } +} + +/* deferred DATA output */ +static Prog *strdat; +static Prog *estrdat; +static int gflag; +static Prog *savepc; + +void +data(void) +{ + gflag = debug['g']; + debug['g'] = 0; + + if(estrdat == nil) { + strdat = mal(sizeof(*pc)); + clearp(strdat); + estrdat = strdat; + } + if(savepc) + fatal("data phase error"); + savepc = pc; + pc = estrdat; +} + +void +text(void) +{ + if(!savepc) + fatal("text phase error"); + debug['g'] = gflag; + estrdat = pc; + pc = savepc; + savepc = nil; +} + +void +dumpdata(void) +{ + Prog *p; + + if(estrdat == nil) + return; + *pc = *strdat; + if(gflag) + for(p=pc; p!=estrdat; p=p->link) + print("%P\n", p); + pc = estrdat; +} + +int +dsname(Sym *s, int off, char *t, int n) +{ + Prog *p; + + p = gins(ADATA, N, N); + p->from.type = D_EXTERN; + p->from.index = D_NONE; + p->from.offset = off; + p->from.scale = n; + p->from.sym = s; + + p->to.type = D_SCONST; + p->to.index = D_NONE; + memmove(p->to.sval, t, n); + return off + n; +} + +/* + * make a refer to the data s, s+len + * emitting DATA if needed. + */ +void +datastring(char *s, int len, Addr *a) +{ + Sym *sym; + + sym = stringsym(s, len); + a->type = D_EXTERN; + a->sym = sym; + a->offset = widthptr+4; // skip header + a->etype = TINT32; +} + +/* + * make a refer to the string sval, + * emitting DATA if needed. + */ +void +datagostring(Strlit *sval, Addr *a) +{ + Sym *sym; + + sym = stringsym(sval->s, sval->len); + a->type = D_EXTERN; + a->sym = sym; + a->offset = 0; // header + a->etype = TINT32; +} + +void +gdata(Node *nam, Node *nr, int wid) +{ + Prog *p; + + p = gins(ADATA, nam, nr); + p->from.scale = wid; +} + +void +gdatacomplex(Node *nam, Mpcplx *cval) +{ + Prog *p; + int w; + + w = cplxsubtype(nam->type->etype); + w = types[w]->width; + + p = gins(ADATA, nam, N); + p->from.scale = w; + p->to.type = D_FCONST; + p->to.dval = mpgetflt(&cval->real); + + p = gins(ADATA, nam, N); + p->from.scale = w; + p->from.offset += w; + p->to.type = D_FCONST; + p->to.dval = mpgetflt(&cval->imag); +} + +void +gdatastring(Node *nam, Strlit *sval) +{ + Prog *p; + Node nod1; + + p = gins(ADATA, nam, N); + datastring(sval->s, sval->len, &p->to); + p->from.scale = types[tptr]->width; + p->to.index = p->to.type; + p->to.type = D_ADDR; +//print("%P\n", p); + + nodconst(&nod1, types[TINT32], sval->len); + p = gins(ADATA, nam, &nod1); + p->from.scale = types[TINT32]->width; + p->from.offset += types[tptr]->width; +} + +int +dstringptr(Sym *s, int off, char *str) +{ + Prog *p; + + off = rnd(off, widthptr); + p = gins(ADATA, N, N); + p->from.type = D_EXTERN; + p->from.index = D_NONE; + p->from.sym = s; + p->from.offset = off; + p->from.scale = widthptr; + + datastring(str, strlen(str)+1, &p->to); + p->to.index = p->to.type; + p->to.type = D_ADDR; + p->to.etype = TINT32; + off += widthptr; + + return off; +} + +int +dgostrlitptr(Sym *s, int off, Strlit *lit) +{ + Prog *p; + + if(lit == nil) + return duintptr(s, off, 0); + + off = rnd(off, widthptr); + p = gins(ADATA, N, N); + p->from.type = D_EXTERN; + p->from.index = D_NONE; + p->from.sym = s; + p->from.offset = off; + p->from.scale = widthptr; + datagostring(lit, &p->to); + p->to.index = p->to.type; + p->to.type = D_ADDR; + p->to.etype = TINT32; + off += widthptr; + + return off; +} + +int +dgostringptr(Sym *s, int off, char *str) +{ + int n; + Strlit *lit; + + if(str == nil) + return duintptr(s, off, 0); + + n = strlen(str); + lit = mal(sizeof *lit + n); + strcpy(lit->s, str); + lit->len = n; + return dgostrlitptr(s, off, lit); +} + +int +duintxx(Sym *s, int off, uint64 v, int wid) +{ + Prog *p; + + off = rnd(off, wid); + + p = gins(ADATA, N, N); + p->from.type = D_EXTERN; + p->from.index = D_NONE; + p->from.sym = s; + p->from.offset = off; + p->from.scale = wid; + p->to.type = D_CONST; + p->to.index = D_NONE; + p->to.offset = v; + off += wid; + + return off; +} + +int +dsymptr(Sym *s, int off, Sym *x, int xoff) +{ + Prog *p; + + off = rnd(off, widthptr); + + p = gins(ADATA, N, N); + p->from.type = D_EXTERN; + p->from.index = D_NONE; + p->from.sym = s; + p->from.offset = off; + p->from.scale = widthptr; + p->to.type = D_ADDR; + p->to.index = D_EXTERN; + p->to.sym = x; + p->to.offset = xoff; + off += widthptr; + + return off; +} + +void +genembedtramp(Type *rcvr, Type *method, Sym *newnam, int iface) +{ + Sym *e; + int c, d, o, mov, add, loaded; + Prog *p; + Type *f; + + if(debug['r']) + print("genembedtramp %T %T %S\n", rcvr, method, newnam); + + e = method->sym; + for(d=0; dsym); + +out: + newplist()->name = newname(newnam); + + //TEXT main·S_test2(SB),7,$0 + p = pc; + gins(ATEXT, N, N); + p->from.type = D_EXTERN; + p->from.sym = newnam; + p->to.type = D_CONST; + p->to.offset = 0; + p->from.scale = 7; +//print("1. %P\n", p); + + mov = AMOVQ; + add = AADDQ; + loaded = 0; + o = 0; + for(c=d-1; c>=0; c--) { + f = dotlist[c].field; + o += f->width; + if(!isptr[f->type->etype]) + continue; + if(!loaded) { + loaded = 1; + //MOVQ 8(SP), AX + p = pc; + gins(mov, N, N); + p->from.type = D_INDIR+D_SP; + p->from.offset = widthptr; + p->to.type = D_AX; +//print("2. %P\n", p); + } + + //MOVQ o(AX), AX + p = pc; + gins(mov, N, N); + p->from.type = D_INDIR+D_AX; + p->from.offset = o; + p->to.type = D_AX; +//print("3. %P\n", p); + o = 0; + } + if(o != 0) { + //ADDQ $XX, AX + p = pc; + gins(add, N, N); + p->from.type = D_CONST; + p->from.offset = o; + if(loaded) + p->to.type = D_AX; + else { + p->to.type = D_INDIR+D_SP; + p->to.offset = widthptr; + } +//print("4. %P\n", p); + } + + //MOVQ AX, 8(SP) + if(loaded) { + p = pc; + gins(mov, N, N); + p->from.type = D_AX; + p->to.type = D_INDIR+D_SP; + p->to.offset = widthptr; +//print("5. %P\n", p); + } else { + // TODO(rsc): obviously this is unnecessary, + // but 6l has a bug, and it can't handle + // JMP instructions too close to the top of + // a new function. + p = pc; + gins(ANOP, N, N); + } + + f = dotlist[0].field; + //JMP main·*Sub_test2(SB) + if(isptr[f->type->etype]) + f = f->type; + p = pc; + gins(AJMP, N, N); + p->to.type = D_EXTERN; + p->to.sym = methodsym(method->sym, ptrto(f->type), 0); +//print("6. %P\n", p); + + pc->as = ARET; // overwrite AEND +} + +void +nopout(Prog *p) +{ + p->as = ANOP; +} + diff --git a/src/cmd/6g/gsubr.c b/src/cmd/6g/gsubr.c new file mode 100644 index 000000000..d0d6d0c96 --- /dev/null +++ b/src/cmd/6g/gsubr.c @@ -0,0 +1,2159 @@ +// Derived from Inferno utils/6c/txt.c +// http://code.google.com/p/inferno-os/source/browse/utils/6c/txt.c +// +// Copyright © 1994-1999 Lucent Technologies Inc. All rights reserved. +// Portions Copyright © 1995-1997 C H Forsyth (forsyth@terzarima.net) +// Portions Copyright © 1997-1999 Vita Nuova Limited +// Portions Copyright © 2000-2007 Vita Nuova Holdings Limited (www.vitanuova.com) +// Portions Copyright © 2004,2006 Bruce Ellis +// Portions Copyright © 2005-2007 C H Forsyth (forsyth@terzarima.net) +// Revisions Copyright © 2000-2007 Lucent Technologies Inc. and others +// Portions Copyright © 2009 The Go Authors. All rights reserved. +// +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to deal +// in the Software without restriction, including without limitation the rights +// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +// copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +// THE SOFTWARE. + +#include "gg.h" + +// TODO(rsc): Can make this bigger if we move +// the text segment up higher in 6l for all GOOS. +vlong unmappedzero = 4096; + +void +clearp(Prog *p) +{ + p->as = AEND; + p->from.type = D_NONE; + p->from.index = D_NONE; + p->to.type = D_NONE; + p->to.index = D_NONE; + p->loc = pcloc; + pcloc++; +} + +/* + * generate and return proc with p->as = as, + * linked into program. pc is next instruction. + */ +Prog* +prog(int as) +{ + Prog *p; + + p = pc; + pc = mal(sizeof(*pc)); + + clearp(pc); + + if(lineno == 0) { + if(debug['K']) + warn("prog: line 0"); + } + + p->as = as; + p->lineno = lineno; + p->link = pc; + return p; +} + +/* + * generate a branch. + * t is ignored. + */ +Prog* +gbranch(int as, Type *t) +{ + Prog *p; + + p = prog(as); + p->to.type = D_BRANCH; + p->to.branch = P; + return p; +} + +/* + * patch previous branch to jump to to. + */ +void +patch(Prog *p, Prog *to) +{ + if(p->to.type != D_BRANCH) + fatal("patch: not a branch"); + p->to.branch = to; + p->to.offset = to->loc; +} + +Prog* +unpatch(Prog *p) +{ + Prog *q; + + if(p->to.type != D_BRANCH) + fatal("unpatch: not a branch"); + q = p->to.branch; + p->to.branch = P; + p->to.offset = 0; + return q; +} + +/* + * start a new Prog list. + */ +Plist* +newplist(void) +{ + Plist *pl; + + pl = mal(sizeof(*pl)); + if(plist == nil) + plist = pl; + else + plast->link = pl; + plast = pl; + + pc = mal(sizeof(*pc)); + clearp(pc); + pl->firstpc = pc; + + return pl; +} + +void +clearstk(void) +{ + Plist *pl; + Prog *p1, *p2; + Node sp, di, cx, con, ax; + + if((uint32)plast->firstpc->to.offset <= 0) + return; + + // reestablish context for inserting code + // at beginning of function. + pl = plast; + p1 = pl->firstpc; + p2 = p1->link; + pc = mal(sizeof(*pc)); + clearp(pc); + p1->link = pc; + + // zero stack frame + nodreg(&sp, types[tptr], D_SP); + nodreg(&di, types[tptr], D_DI); + nodreg(&cx, types[TUINT64], D_CX); + nodconst(&con, types[TUINT64], (uint32)p1->to.offset / widthptr); + gins(ACLD, N, N); + gins(AMOVQ, &sp, &di); + gins(AMOVQ, &con, &cx); + nodconst(&con, types[TUINT64], 0); + nodreg(&ax, types[TUINT64], D_AX); + gins(AMOVQ, &con, &ax); + gins(AREP, N, N); + gins(ASTOSQ, N, N); + + // continue with original code. + gins(ANOP, N, N)->link = p2; + pc = P; +} + +void +gused(Node *n) +{ + gins(ANOP, n, N); // used +} + +Prog* +gjmp(Prog *to) +{ + Prog *p; + + p = gbranch(AJMP, T); + if(to != P) + patch(p, to); + return p; +} + +void +ggloblnod(Node *nam, int32 width) +{ + Prog *p; + + p = gins(AGLOBL, nam, N); + p->lineno = nam->lineno; + p->to.sym = S; + p->to.type = D_CONST; + p->to.offset = width; + if(nam->readonly) + p->from.scale = RODATA; +} + +void +ggloblsym(Sym *s, int32 width, int dupok) +{ + Prog *p; + + p = gins(AGLOBL, N, N); + p->from.type = D_EXTERN; + p->from.index = D_NONE; + p->from.sym = s; + p->to.type = D_CONST; + p->to.index = D_NONE; + p->to.offset = width; + if(dupok) + p->from.scale = DUPOK; + p->from.scale |= RODATA; +} + +int +isfat(Type *t) +{ + if(t != T) + switch(t->etype) { + case TSTRUCT: + case TARRAY: + case TSTRING: + case TINTER: // maybe remove later + return 1; + } + return 0; +} + +/* + * naddr of func generates code for address of func. + * if using opcode that can take address implicitly, + * call afunclit to fix up the argument. + */ +void +afunclit(Addr *a) +{ + if(a->type == D_ADDR && a->index == D_EXTERN) { + a->type = D_EXTERN; + a->index = D_NONE; + } +} + +static int resvd[] = +{ + D_DI, // for movstring + D_SI, // for movstring + + D_AX, // for divide + D_CX, // for shift + D_DX, // for divide + D_SP, // for stack + D_R14, // reserved for m + D_R15, // reserved for u +}; + +void +ginit(void) +{ + int i; + + for(i=0; ietype]; + + switch(et) { + case TINT8: + case TUINT8: + case TINT16: + case TUINT16: + case TINT32: + case TUINT32: + case TINT64: + case TUINT64: + case TPTR32: + case TPTR64: + case TBOOL: + if(o != N && o->op == OREGISTER) { + i = o->val.u.reg; + if(i >= D_AX && i <= D_R15) + goto out; + } + for(i=D_AX; i<=D_R15; i++) + if(reg[i] == 0) + goto out; + + yyerror("out of fixed registers"); + goto err; + + case TFLOAT32: + case TFLOAT64: + if(o != N && o->op == OREGISTER) { + i = o->val.u.reg; + if(i >= D_X0 && i <= D_X7) + goto out; + } + for(i=D_X0; i<=D_X7; i++) + if(reg[i] == 0) + goto out; + yyerror("out of floating registers"); + goto err; + + case TCOMPLEX64: + case TCOMPLEX128: + tempname(n, t); + return; + } + yyerror("regalloc: unknown type %T", t); + +err: + nodreg(n, t, 0); + return; + +out: + reg[i]++; + nodreg(n, t, i); +} + +void +regfree(Node *n) +{ + int i; + + if(n->op == ONAME) + return; + if(n->op != OREGISTER && n->op != OINDREG) + fatal("regfree: not a register"); + i = n->val.u.reg; + if(i == D_SP) + return; + if(i < 0 || i >= sizeof(reg)) + fatal("regfree: reg out of range"); + if(reg[i] <= 0) + fatal("regfree: reg not allocated"); + reg[i]--; +} + +/* + * initialize n to be register r of type t. + */ +void +nodreg(Node *n, Type *t, int r) +{ + if(t == T) + fatal("nodreg: t nil"); + + memset(n, 0, sizeof(*n)); + n->op = OREGISTER; + n->addable = 1; + ullmancalc(n); + n->val.u.reg = r; + n->type = t; +} + +/* + * initialize n to be indirect of register r; n is type t. + */ +void +nodindreg(Node *n, Type *t, int r) +{ + nodreg(n, t, r); + n->op = OINDREG; +} + +Node* +nodarg(Type *t, int fp) +{ + Node *n; + Type *first; + Iter savet; + + // entire argument struct, not just one arg + if(t->etype == TSTRUCT && t->funarg) { + n = nod(ONAME, N, N); + n->sym = lookup(".args"); + n->type = t; + first = structfirst(&savet, &t); + if(first == nil) + fatal("nodarg: bad struct"); + if(first->width == BADWIDTH) + fatal("nodarg: offset not computed for %T", t); + n->xoffset = first->width; + n->addable = 1; + goto fp; + } + + if(t->etype != TFIELD) + fatal("nodarg: not field %T", t); + + n = nod(ONAME, N, N); + n->type = t->type; + n->sym = t->sym; + if(t->width == BADWIDTH) + fatal("nodarg: offset not computed for %T", t); + n->xoffset = t->width; + n->addable = 1; + +fp: + switch(fp) { + case 0: // output arg + n->op = OINDREG; + n->val.u.reg = D_SP; + break; + + case 1: // input arg + n->class = PPARAM; + break; + + case 2: // offset output arg +fatal("shouldnt be used"); + n->op = OINDREG; + n->val.u.reg = D_SP; + n->xoffset += types[tptr]->width; + break; + } + n->typecheck = 1; + return n; +} + +/* + * generate + * as $c, reg + */ +void +gconreg(int as, vlong c, int reg) +{ + Node nr; + + nodreg(&nr, types[TINT64], reg); + ginscon(as, c, &nr); +} + +/* + * generate + * as $c, n + */ +void +ginscon(int as, vlong c, Node *n2) +{ + Node n1, ntmp; + + nodconst(&n1, types[TINT64], c); + + if(as != AMOVQ && (c < -1LL<<31 || c >= 1LL<<31)) { + // cannot have 64-bit immediokate in ADD, etc. + // instead, MOV into register first. + regalloc(&ntmp, types[TINT64], N); + gins(AMOVQ, &n1, &ntmp); + gins(as, &ntmp, n2); + regfree(&ntmp); + return; + } + gins(as, &n1, n2); +} + +#define CASE(a,b) (((a)<<16)|((b)<<0)) + +/* + * Is this node a memory operand? + */ +int +ismem(Node *n) +{ + switch(n->op) { + case OLEN: + case OCAP: + case OINDREG: + case ONAME: + case OPARAM: + return 1; + } + return 0; +} + +/* + * set up nodes representing 2^63 + */ +Node bigi; +Node bigf; + +void +bignodes(void) +{ + static int did; + + if(did) + return; + did = 1; + + nodconst(&bigi, types[TUINT64], 1); + mpshiftfix(bigi.val.u.xval, 63); + + bigf = bigi; + bigf.type = types[TFLOAT64]; + bigf.val.ctype = CTFLT; + bigf.val.u.fval = mal(sizeof *bigf.val.u.fval); + mpmovefixflt(bigf.val.u.fval, bigi.val.u.xval); +} + +/* + * generate move: + * t = f + * hard part is conversions. + */ +// TODO: lost special constants for floating point. XORPD for 0.0? +void +gmove(Node *f, Node *t) +{ + int a, ft, tt; + Type *cvt; + Node r1, r2, r3, r4, zero, one, con; + Prog *p1, *p2; + + if(debug['M']) + print("gmove %N -> %N\n", f, t); + + ft = simsimtype(f->type); + tt = simsimtype(t->type); + cvt = t->type; + + if(iscomplex[ft] || iscomplex[tt]) { + complexmove(f, t); + return; + } + + // cannot have two memory operands + if(ismem(f) && ismem(t)) + goto hard; + + // convert constant to desired type + if(f->op == OLITERAL) { + convconst(&con, t->type, &f->val); + f = &con; + ft = tt; // so big switch will choose a simple mov + + // some constants can't move directly to memory. + if(ismem(t)) { + // float constants come from memory. + if(isfloat[tt]) + goto hard; + + // 64-bit immediates are really 32-bit sign-extended + // unless moving into a register. + if(isint[tt]) { + if(mpcmpfixfix(con.val.u.xval, minintval[TINT32]) < 0) + goto hard; + if(mpcmpfixfix(con.val.u.xval, maxintval[TINT32]) > 0) + goto hard; + } + } + } + + // value -> value copy, only one memory operand. + // figure out the instruction to use. + // break out of switch for one-instruction gins. + // goto rdst for "destination must be register". + // goto hard for "convert to cvt type first". + // otherwise handle and return. + + switch(CASE(ft, tt)) { + default: + fatal("gmove %lT -> %lT", f->type, t->type); + + /* + * integer copy and truncate + */ + case CASE(TINT8, TINT8): // same size + case CASE(TINT8, TUINT8): + case CASE(TUINT8, TINT8): + case CASE(TUINT8, TUINT8): + case CASE(TINT16, TINT8): // truncate + case CASE(TUINT16, TINT8): + case CASE(TINT32, TINT8): + case CASE(TUINT32, TINT8): + case CASE(TINT64, TINT8): + case CASE(TUINT64, TINT8): + case CASE(TINT16, TUINT8): + case CASE(TUINT16, TUINT8): + case CASE(TINT32, TUINT8): + case CASE(TUINT32, TUINT8): + case CASE(TINT64, TUINT8): + case CASE(TUINT64, TUINT8): + a = AMOVB; + break; + + case CASE(TINT16, TINT16): // same size + case CASE(TINT16, TUINT16): + case CASE(TUINT16, TINT16): + case CASE(TUINT16, TUINT16): + case CASE(TINT32, TINT16): // truncate + case CASE(TUINT32, TINT16): + case CASE(TINT64, TINT16): + case CASE(TUINT64, TINT16): + case CASE(TINT32, TUINT16): + case CASE(TUINT32, TUINT16): + case CASE(TINT64, TUINT16): + case CASE(TUINT64, TUINT16): + a = AMOVW; + break; + + case CASE(TINT32, TINT32): // same size + case CASE(TINT32, TUINT32): + case CASE(TUINT32, TINT32): + case CASE(TUINT32, TUINT32): + case CASE(TINT64, TINT32): // truncate + case CASE(TUINT64, TINT32): + case CASE(TINT64, TUINT32): + case CASE(TUINT64, TUINT32): + a = AMOVL; + break; + + case CASE(TINT64, TINT64): // same size + case CASE(TINT64, TUINT64): + case CASE(TUINT64, TINT64): + case CASE(TUINT64, TUINT64): + a = AMOVQ; + break; + + /* + * integer up-conversions + */ + case CASE(TINT8, TINT16): // sign extend int8 + case CASE(TINT8, TUINT16): + a = AMOVBWSX; + goto rdst; + case CASE(TINT8, TINT32): + case CASE(TINT8, TUINT32): + a = AMOVBLSX; + goto rdst; + case CASE(TINT8, TINT64): + case CASE(TINT8, TUINT64): + a = AMOVBQSX; + goto rdst; + + case CASE(TUINT8, TINT16): // zero extend uint8 + case CASE(TUINT8, TUINT16): + a = AMOVBWZX; + goto rdst; + case CASE(TUINT8, TINT32): + case CASE(TUINT8, TUINT32): + a = AMOVBLZX; + goto rdst; + case CASE(TUINT8, TINT64): + case CASE(TUINT8, TUINT64): + a = AMOVBQZX; + goto rdst; + + case CASE(TINT16, TINT32): // sign extend int16 + case CASE(TINT16, TUINT32): + a = AMOVWLSX; + goto rdst; + case CASE(TINT16, TINT64): + case CASE(TINT16, TUINT64): + a = AMOVWQSX; + goto rdst; + + case CASE(TUINT16, TINT32): // zero extend uint16 + case CASE(TUINT16, TUINT32): + a = AMOVWLZX; + goto rdst; + case CASE(TUINT16, TINT64): + case CASE(TUINT16, TUINT64): + a = AMOVWQZX; + goto rdst; + + case CASE(TINT32, TINT64): // sign extend int32 + case CASE(TINT32, TUINT64): + a = AMOVLQSX; + goto rdst; + + case CASE(TUINT32, TINT64): // zero extend uint32 + case CASE(TUINT32, TUINT64): + // AMOVL into a register zeros the top of the register, + // so this is not always necessary, but if we rely on AMOVL + // the optimizer is almost certain to screw with us. + a = AMOVLQZX; + goto rdst; + + /* + * float to integer + */ + case CASE(TFLOAT32, TINT32): + a = ACVTTSS2SL; + goto rdst; + + case CASE(TFLOAT64, TINT32): + a = ACVTTSD2SL; + goto rdst; + + case CASE(TFLOAT32, TINT64): + a = ACVTTSS2SQ; + goto rdst; + + case CASE(TFLOAT64, TINT64): + a = ACVTTSD2SQ; + goto rdst; + + case CASE(TFLOAT32, TINT16): + case CASE(TFLOAT32, TINT8): + case CASE(TFLOAT32, TUINT16): + case CASE(TFLOAT32, TUINT8): + case CASE(TFLOAT64, TINT16): + case CASE(TFLOAT64, TINT8): + case CASE(TFLOAT64, TUINT16): + case CASE(TFLOAT64, TUINT8): + // convert via int32. + cvt = types[TINT32]; + goto hard; + + case CASE(TFLOAT32, TUINT32): + case CASE(TFLOAT64, TUINT32): + // convert via int64. + cvt = types[TINT64]; + goto hard; + + case CASE(TFLOAT32, TUINT64): + case CASE(TFLOAT64, TUINT64): + // algorithm is: + // if small enough, use native float64 -> int64 conversion. + // otherwise, subtract 2^63, convert, and add it back. + a = ACVTSS2SQ; + if(ft == TFLOAT64) + a = ACVTSD2SQ; + bignodes(); + regalloc(&r1, types[ft], N); + regalloc(&r2, types[tt], t); + regalloc(&r3, types[ft], N); + regalloc(&r4, types[tt], N); + gins(optoas(OAS, f->type), f, &r1); + gins(optoas(OCMP, f->type), &bigf, &r1); + p1 = gbranch(optoas(OLE, f->type), T); + gins(a, &r1, &r2); + p2 = gbranch(AJMP, T); + patch(p1, pc); + gins(optoas(OAS, f->type), &bigf, &r3); + gins(optoas(OSUB, f->type), &r3, &r1); + gins(a, &r1, &r2); + gins(AMOVQ, &bigi, &r4); + gins(AXORQ, &r4, &r2); + patch(p2, pc); + gmove(&r2, t); + regfree(&r4); + regfree(&r3); + regfree(&r2); + regfree(&r1); + return; + + /* + * integer to float + */ + case CASE(TINT32, TFLOAT32): + a = ACVTSL2SS; + goto rdst; + + + case CASE(TINT32, TFLOAT64): + a = ACVTSL2SD; + goto rdst; + + case CASE(TINT64, TFLOAT32): + a = ACVTSQ2SS; + goto rdst; + + case CASE(TINT64, TFLOAT64): + a = ACVTSQ2SD; + goto rdst; + + case CASE(TINT16, TFLOAT32): + case CASE(TINT16, TFLOAT64): + case CASE(TINT8, TFLOAT32): + case CASE(TINT8, TFLOAT64): + case CASE(TUINT16, TFLOAT32): + case CASE(TUINT16, TFLOAT64): + case CASE(TUINT8, TFLOAT32): + case CASE(TUINT8, TFLOAT64): + // convert via int32 + cvt = types[TINT32]; + goto hard; + + case CASE(TUINT32, TFLOAT32): + case CASE(TUINT32, TFLOAT64): + // convert via int64. + cvt = types[TINT64]; + goto hard; + + case CASE(TUINT64, TFLOAT32): + case CASE(TUINT64, TFLOAT64): + // algorithm is: + // if small enough, use native int64 -> uint64 conversion. + // otherwise, halve (rounding to odd?), convert, and double. + a = ACVTSQ2SS; + if(tt == TFLOAT64) + a = ACVTSQ2SD; + nodconst(&zero, types[TUINT64], 0); + nodconst(&one, types[TUINT64], 1); + regalloc(&r1, f->type, f); + regalloc(&r2, t->type, t); + regalloc(&r3, f->type, N); + regalloc(&r4, f->type, N); + gmove(f, &r1); + gins(ACMPQ, &r1, &zero); + p1 = gbranch(AJLT, T); + gins(a, &r1, &r2); + p2 = gbranch(AJMP, T); + patch(p1, pc); + gmove(&r1, &r3); + gins(ASHRQ, &one, &r3); + gmove(&r1, &r4); + gins(AANDL, &one, &r4); + gins(AORQ, &r4, &r3); + gins(a, &r3, &r2); + gins(optoas(OADD, t->type), &r2, &r2); + patch(p2, pc); + gmove(&r2, t); + regfree(&r4); + regfree(&r3); + regfree(&r2); + regfree(&r1); + return; + + /* + * float to float + */ + case CASE(TFLOAT32, TFLOAT32): + a = AMOVSS; + break; + + case CASE(TFLOAT64, TFLOAT64): + a = AMOVSD; + break; + + case CASE(TFLOAT32, TFLOAT64): + a = ACVTSS2SD; + goto rdst; + + case CASE(TFLOAT64, TFLOAT32): + a = ACVTSD2SS; + goto rdst; + } + + gins(a, f, t); + return; + +rdst: + // requires register destination + regalloc(&r1, t->type, t); + gins(a, f, &r1); + gmove(&r1, t); + regfree(&r1); + return; + +hard: + // requires register intermediate + regalloc(&r1, cvt, t); + gmove(f, &r1); + gmove(&r1, t); + regfree(&r1); + return; +} + +int +samaddr(Node *f, Node *t) +{ + + if(f->op != t->op) + return 0; + + switch(f->op) { + case OREGISTER: + if(f->val.u.reg != t->val.u.reg) + break; + return 1; + } + return 0; +} + +/* + * generate one instruction: + * as f, t + */ +Prog* +gins(int as, Node *f, Node *t) +{ +// Node nod; + int32 w; + Prog *p; + Addr af, at; + +// if(f != N && f->op == OINDEX) { +// regalloc(&nod, ®node, Z); +// v = constnode.vconst; +// cgen(f->right, &nod); +// constnode.vconst = v; +// idx.reg = nod.reg; +// regfree(&nod); +// } +// if(t != N && t->op == OINDEX) { +// regalloc(&nod, ®node, Z); +// v = constnode.vconst; +// cgen(t->right, &nod); +// constnode.vconst = v; +// idx.reg = nod.reg; +// regfree(&nod); +// } + + switch(as) { + case AMOVB: + case AMOVW: + case AMOVL: + case AMOVQ: + case AMOVSS: + case AMOVSD: + if(f != N && t != N && samaddr(f, t)) + return nil; + } + + memset(&af, 0, sizeof af); + memset(&at, 0, sizeof at); + if(f != N) + naddr(f, &af, 1); + if(t != N) + naddr(t, &at, 1); + p = prog(as); + if(f != N) + p->from = af; + if(t != N) + p->to = at; + if(debug['g']) + print("%P\n", p); + + w = 0; + switch(as) { + case AMOVB: + w = 1; + break; + case AMOVW: + w = 2; + break; + case AMOVL: + w = 4; + break; + case AMOVQ: + w = 8; + break; + } + if(w != 0 && f != N && (af.width > w || at.width > w)) { + fatal("bad width: %P (%d, %d)\n", p, af.width, at.width); + } + + return p; +} + +static void +checkoffset(Addr *a, int canemitcode) +{ + Prog *p; + + if(a->offset < unmappedzero) + return; + if(!canemitcode) + fatal("checkoffset %#llx, cannot emit code", a->offset); + + // cannot rely on unmapped nil page at 0 to catch + // reference with large offset. instead, emit explicit + // test of 0(reg). + p = gins(ATESTB, nodintconst(0), N); + p->to = *a; + p->to.offset = 0; +} + +/* + * generate code to compute n; + * make a refer to result. + */ +void +naddr(Node *n, Addr *a, int canemitcode) +{ + a->scale = 0; + a->index = D_NONE; + a->type = D_NONE; + a->gotype = S; + a->node = N; + if(n == N) + return; + + switch(n->op) { + default: + fatal("naddr: bad %O %D", n->op, a); + break; + + case OREGISTER: + a->type = n->val.u.reg; + a->sym = S; + break; + +// case OINDEX: +// case OIND: +// naddr(n->left, a); +// if(a->type >= D_AX && a->type <= D_DI) +// a->type += D_INDIR; +// else +// if(a->type == D_CONST) +// a->type = D_NONE+D_INDIR; +// else +// if(a->type == D_ADDR) { +// a->type = a->index; +// a->index = D_NONE; +// } else +// goto bad; +// if(n->op == OINDEX) { +// a->index = idx.reg; +// a->scale = n->scale; +// } +// break; + + case OINDREG: + a->type = n->val.u.reg+D_INDIR; + a->sym = n->sym; + a->offset = n->xoffset; + checkoffset(a, canemitcode); + break; + + case OPARAM: + // n->left is PHEAP ONAME for stack parameter. + // compute address of actual parameter on stack. + a->etype = simtype[n->left->type->etype]; + a->width = n->left->type->width; + a->offset = n->xoffset; + a->sym = n->left->sym; + a->type = D_PARAM; + break; + + case ONAME: + a->etype = 0; + a->width = 0; + if(n->type != T) { + a->etype = simtype[n->type->etype]; + a->width = n->type->width; + a->gotype = ngotype(n); + } + a->pun = n->pun; + a->offset = n->xoffset; + a->sym = n->sym; + if(a->sym == S) + a->sym = lookup(".noname"); + if(n->method) { + if(n->type != T) + if(n->type->sym != S) + if(n->type->sym->pkg != nil) + a->sym = pkglookup(a->sym->name, n->type->sym->pkg); + } + + switch(n->class) { + default: + fatal("naddr: ONAME class %S %d\n", n->sym, n->class); + case PEXTERN: + a->type = D_EXTERN; + break; + case PAUTO: + a->type = D_AUTO; + if (n->sym) + a->node = n->orig; + break; + case PPARAM: + case PPARAMOUT: + a->type = D_PARAM; + break; + case PFUNC: + a->index = D_EXTERN; + a->type = D_ADDR; + break; + } + break; + + case OLITERAL: + switch(n->val.ctype) { + default: + fatal("naddr: const %lT", n->type); + break; + case CTFLT: + a->type = D_FCONST; + a->dval = mpgetflt(n->val.u.fval); + break; + case CTINT: + a->sym = S; + a->type = D_CONST; + a->offset = mpgetfix(n->val.u.xval); + break; + case CTSTR: + datagostring(n->val.u.sval, a); + break; + case CTBOOL: + a->sym = S; + a->type = D_CONST; + a->offset = n->val.u.bval; + break; + case CTNIL: + a->sym = S; + a->type = D_CONST; + a->offset = 0; + break; + } + break; + + case OADDR: + naddr(n->left, a, canemitcode); + if(a->type >= D_INDIR) { + a->type -= D_INDIR; + break; + } + if(a->type == D_EXTERN || a->type == D_STATIC || + a->type == D_AUTO || a->type == D_PARAM) + if(a->index == D_NONE) { + a->index = a->type; + a->type = D_ADDR; + break; + } + fatal("naddr: OADDR\n"); + + case OLEN: + // len of string or slice + naddr(n->left, a, canemitcode); + if(a->type == D_CONST && a->offset == 0) + break; // len(nil) + a->etype = TUINT32; + a->offset += Array_nel; + a->width = 4; + if(a->offset >= unmappedzero && a->offset-Array_nel < unmappedzero) + checkoffset(a, canemitcode); + break; + + case OCAP: + // cap of string or slice + naddr(n->left, a, canemitcode); + if(a->type == D_CONST && a->offset == 0) + break; // cap(nil) + a->etype = TUINT32; + a->offset += Array_cap; + a->width = 4; + if(a->offset >= unmappedzero && a->offset-Array_cap < unmappedzero) + checkoffset(a, canemitcode); + break; + +// case OADD: +// if(n->right->op == OLITERAL) { +// v = n->right->vconst; +// naddr(n->left, a, canemitcode); +// } else +// if(n->left->op == OLITERAL) { +// v = n->left->vconst; +// naddr(n->right, a, canemitcode); +// } else +// goto bad; +// a->offset += v; +// break; + + } +} + +/* + * return Axxx for Oxxx on type t. + */ +int +optoas(int op, Type *t) +{ + int a; + + if(t == T) + fatal("optoas: t is nil"); + + a = AGOK; + switch(CASE(op, simtype[t->etype])) { + default: + fatal("optoas: no entry %O-%T", op, t); + break; + + case CASE(OADDR, TPTR32): + a = ALEAL; + break; + + case CASE(OADDR, TPTR64): + a = ALEAQ; + break; + + case CASE(OEQ, TBOOL): + case CASE(OEQ, TINT8): + case CASE(OEQ, TUINT8): + case CASE(OEQ, TINT16): + case CASE(OEQ, TUINT16): + case CASE(OEQ, TINT32): + case CASE(OEQ, TUINT32): + case CASE(OEQ, TINT64): + case CASE(OEQ, TUINT64): + case CASE(OEQ, TPTR32): + case CASE(OEQ, TPTR64): + case CASE(OEQ, TFLOAT32): + case CASE(OEQ, TFLOAT64): + a = AJEQ; + break; + + case CASE(ONE, TBOOL): + case CASE(ONE, TINT8): + case CASE(ONE, TUINT8): + case CASE(ONE, TINT16): + case CASE(ONE, TUINT16): + case CASE(ONE, TINT32): + case CASE(ONE, TUINT32): + case CASE(ONE, TINT64): + case CASE(ONE, TUINT64): + case CASE(ONE, TPTR32): + case CASE(ONE, TPTR64): + case CASE(ONE, TFLOAT32): + case CASE(ONE, TFLOAT64): + a = AJNE; + break; + + case CASE(OLT, TINT8): + case CASE(OLT, TINT16): + case CASE(OLT, TINT32): + case CASE(OLT, TINT64): + a = AJLT; + break; + + case CASE(OLT, TUINT8): + case CASE(OLT, TUINT16): + case CASE(OLT, TUINT32): + case CASE(OLT, TUINT64): + a = AJCS; + break; + + case CASE(OLE, TINT8): + case CASE(OLE, TINT16): + case CASE(OLE, TINT32): + case CASE(OLE, TINT64): + a = AJLE; + break; + + case CASE(OLE, TUINT8): + case CASE(OLE, TUINT16): + case CASE(OLE, TUINT32): + case CASE(OLE, TUINT64): + a = AJLS; + break; + + case CASE(OGT, TINT8): + case CASE(OGT, TINT16): + case CASE(OGT, TINT32): + case CASE(OGT, TINT64): + a = AJGT; + break; + + case CASE(OGT, TUINT8): + case CASE(OGT, TUINT16): + case CASE(OGT, TUINT32): + case CASE(OGT, TUINT64): + case CASE(OLT, TFLOAT32): + case CASE(OLT, TFLOAT64): + a = AJHI; + break; + + case CASE(OGE, TINT8): + case CASE(OGE, TINT16): + case CASE(OGE, TINT32): + case CASE(OGE, TINT64): + a = AJGE; + break; + + case CASE(OGE, TUINT8): + case CASE(OGE, TUINT16): + case CASE(OGE, TUINT32): + case CASE(OGE, TUINT64): + case CASE(OLE, TFLOAT32): + case CASE(OLE, TFLOAT64): + a = AJCC; + break; + + case CASE(OCMP, TBOOL): + case CASE(OCMP, TINT8): + case CASE(OCMP, TUINT8): + a = ACMPB; + break; + + case CASE(OCMP, TINT16): + case CASE(OCMP, TUINT16): + a = ACMPW; + break; + + case CASE(OCMP, TINT32): + case CASE(OCMP, TUINT32): + case CASE(OCMP, TPTR32): + a = ACMPL; + break; + + case CASE(OCMP, TINT64): + case CASE(OCMP, TUINT64): + case CASE(OCMP, TPTR64): + a = ACMPQ; + break; + + case CASE(OCMP, TFLOAT32): + a = AUCOMISS; + break; + + case CASE(OCMP, TFLOAT64): + a = AUCOMISD; + break; + + case CASE(OAS, TBOOL): + case CASE(OAS, TINT8): + case CASE(OAS, TUINT8): + a = AMOVB; + break; + + case CASE(OAS, TINT16): + case CASE(OAS, TUINT16): + a = AMOVW; + break; + + case CASE(OAS, TINT32): + case CASE(OAS, TUINT32): + case CASE(OAS, TPTR32): + a = AMOVL; + break; + + case CASE(OAS, TINT64): + case CASE(OAS, TUINT64): + case CASE(OAS, TPTR64): + a = AMOVQ; + break; + + case CASE(OAS, TFLOAT32): + a = AMOVSS; + break; + + case CASE(OAS, TFLOAT64): + a = AMOVSD; + break; + + case CASE(OADD, TINT8): + case CASE(OADD, TUINT8): + a = AADDB; + break; + + case CASE(OADD, TINT16): + case CASE(OADD, TUINT16): + a = AADDW; + break; + + case CASE(OADD, TINT32): + case CASE(OADD, TUINT32): + case CASE(OADD, TPTR32): + a = AADDL; + break; + + case CASE(OADD, TINT64): + case CASE(OADD, TUINT64): + case CASE(OADD, TPTR64): + a = AADDQ; + break; + + case CASE(OADD, TFLOAT32): + a = AADDSS; + break; + + case CASE(OADD, TFLOAT64): + a = AADDSD; + break; + + case CASE(OSUB, TINT8): + case CASE(OSUB, TUINT8): + a = ASUBB; + break; + + case CASE(OSUB, TINT16): + case CASE(OSUB, TUINT16): + a = ASUBW; + break; + + case CASE(OSUB, TINT32): + case CASE(OSUB, TUINT32): + case CASE(OSUB, TPTR32): + a = ASUBL; + break; + + case CASE(OSUB, TINT64): + case CASE(OSUB, TUINT64): + case CASE(OSUB, TPTR64): + a = ASUBQ; + break; + + case CASE(OSUB, TFLOAT32): + a = ASUBSS; + break; + + case CASE(OSUB, TFLOAT64): + a = ASUBSD; + break; + + case CASE(OINC, TINT8): + case CASE(OINC, TUINT8): + a = AINCB; + break; + + case CASE(OINC, TINT16): + case CASE(OINC, TUINT16): + a = AINCW; + break; + + case CASE(OINC, TINT32): + case CASE(OINC, TUINT32): + case CASE(OINC, TPTR32): + a = AINCL; + break; + + case CASE(OINC, TINT64): + case CASE(OINC, TUINT64): + case CASE(OINC, TPTR64): + a = AINCQ; + break; + + case CASE(ODEC, TINT8): + case CASE(ODEC, TUINT8): + a = ADECB; + break; + + case CASE(ODEC, TINT16): + case CASE(ODEC, TUINT16): + a = ADECW; + break; + + case CASE(ODEC, TINT32): + case CASE(ODEC, TUINT32): + case CASE(ODEC, TPTR32): + a = ADECL; + break; + + case CASE(ODEC, TINT64): + case CASE(ODEC, TUINT64): + case CASE(ODEC, TPTR64): + a = ADECQ; + break; + + case CASE(OMINUS, TINT8): + case CASE(OMINUS, TUINT8): + a = ANEGB; + break; + + case CASE(OMINUS, TINT16): + case CASE(OMINUS, TUINT16): + a = ANEGW; + break; + + case CASE(OMINUS, TINT32): + case CASE(OMINUS, TUINT32): + case CASE(OMINUS, TPTR32): + a = ANEGL; + break; + + case CASE(OMINUS, TINT64): + case CASE(OMINUS, TUINT64): + case CASE(OMINUS, TPTR64): + a = ANEGQ; + break; + + case CASE(OAND, TINT8): + case CASE(OAND, TUINT8): + a = AANDB; + break; + + case CASE(OAND, TINT16): + case CASE(OAND, TUINT16): + a = AANDW; + break; + + case CASE(OAND, TINT32): + case CASE(OAND, TUINT32): + case CASE(OAND, TPTR32): + a = AANDL; + break; + + case CASE(OAND, TINT64): + case CASE(OAND, TUINT64): + case CASE(OAND, TPTR64): + a = AANDQ; + break; + + case CASE(OOR, TINT8): + case CASE(OOR, TUINT8): + a = AORB; + break; + + case CASE(OOR, TINT16): + case CASE(OOR, TUINT16): + a = AORW; + break; + + case CASE(OOR, TINT32): + case CASE(OOR, TUINT32): + case CASE(OOR, TPTR32): + a = AORL; + break; + + case CASE(OOR, TINT64): + case CASE(OOR, TUINT64): + case CASE(OOR, TPTR64): + a = AORQ; + break; + + case CASE(OXOR, TINT8): + case CASE(OXOR, TUINT8): + a = AXORB; + break; + + case CASE(OXOR, TINT16): + case CASE(OXOR, TUINT16): + a = AXORW; + break; + + case CASE(OXOR, TINT32): + case CASE(OXOR, TUINT32): + case CASE(OXOR, TPTR32): + a = AXORL; + break; + + case CASE(OXOR, TINT64): + case CASE(OXOR, TUINT64): + case CASE(OXOR, TPTR64): + a = AXORQ; + break; + + case CASE(OLSH, TINT8): + case CASE(OLSH, TUINT8): + a = ASHLB; + break; + + case CASE(OLSH, TINT16): + case CASE(OLSH, TUINT16): + a = ASHLW; + break; + + case CASE(OLSH, TINT32): + case CASE(OLSH, TUINT32): + case CASE(OLSH, TPTR32): + a = ASHLL; + break; + + case CASE(OLSH, TINT64): + case CASE(OLSH, TUINT64): + case CASE(OLSH, TPTR64): + a = ASHLQ; + break; + + case CASE(ORSH, TUINT8): + a = ASHRB; + break; + + case CASE(ORSH, TUINT16): + a = ASHRW; + break; + + case CASE(ORSH, TUINT32): + case CASE(ORSH, TPTR32): + a = ASHRL; + break; + + case CASE(ORSH, TUINT64): + case CASE(ORSH, TPTR64): + a = ASHRQ; + break; + + case CASE(ORSH, TINT8): + a = ASARB; + break; + + case CASE(ORSH, TINT16): + a = ASARW; + break; + + case CASE(ORSH, TINT32): + a = ASARL; + break; + + case CASE(ORSH, TINT64): + a = ASARQ; + break; + + case CASE(ORRC, TINT8): + case CASE(ORRC, TUINT8): + a = ARCRB; + break; + + case CASE(ORRC, TINT16): + case CASE(ORRC, TUINT16): + a = ARCRW; + break; + + case CASE(ORRC, TINT32): + case CASE(ORRC, TUINT32): + a = ARCRL; + break; + + case CASE(ORRC, TINT64): + case CASE(ORRC, TUINT64): + a = ARCRQ; + break; + + case CASE(OHMUL, TINT8): + case CASE(OMUL, TINT8): + case CASE(OMUL, TUINT8): + a = AIMULB; + break; + + case CASE(OHMUL, TINT16): + case CASE(OMUL, TINT16): + case CASE(OMUL, TUINT16): + a = AIMULW; + break; + + case CASE(OHMUL, TINT32): + case CASE(OMUL, TINT32): + case CASE(OMUL, TUINT32): + case CASE(OMUL, TPTR32): + a = AIMULL; + break; + + case CASE(OHMUL, TINT64): + case CASE(OMUL, TINT64): + case CASE(OMUL, TUINT64): + case CASE(OMUL, TPTR64): + a = AIMULQ; + break; + + case CASE(OHMUL, TUINT8): + a = AMULB; + break; + + case CASE(OHMUL, TUINT16): + a = AMULW; + break; + + case CASE(OHMUL, TUINT32): + case CASE(OHMUL, TPTR32): + a = AMULL; + break; + + case CASE(OHMUL, TUINT64): + case CASE(OHMUL, TPTR64): + a = AMULQ; + break; + + case CASE(OMUL, TFLOAT32): + a = AMULSS; + break; + + case CASE(OMUL, TFLOAT64): + a = AMULSD; + break; + + case CASE(ODIV, TINT8): + case CASE(OMOD, TINT8): + a = AIDIVB; + break; + + case CASE(ODIV, TUINT8): + case CASE(OMOD, TUINT8): + a = ADIVB; + break; + + case CASE(ODIV, TINT16): + case CASE(OMOD, TINT16): + a = AIDIVW; + break; + + case CASE(ODIV, TUINT16): + case CASE(OMOD, TUINT16): + a = ADIVW; + break; + + case CASE(ODIV, TINT32): + case CASE(OMOD, TINT32): + a = AIDIVL; + break; + + case CASE(ODIV, TUINT32): + case CASE(ODIV, TPTR32): + case CASE(OMOD, TUINT32): + case CASE(OMOD, TPTR32): + a = ADIVL; + break; + + case CASE(ODIV, TINT64): + case CASE(OMOD, TINT64): + a = AIDIVQ; + break; + + case CASE(ODIV, TUINT64): + case CASE(ODIV, TPTR64): + case CASE(OMOD, TUINT64): + case CASE(OMOD, TPTR64): + a = ADIVQ; + break; + + case CASE(OEXTEND, TINT16): + a = ACWD; + break; + + case CASE(OEXTEND, TINT32): + a = ACDQ; + break; + + case CASE(OEXTEND, TINT64): + a = ACQO; + break; + + case CASE(ODIV, TFLOAT32): + a = ADIVSS; + break; + + case CASE(ODIV, TFLOAT64): + a = ADIVSD; + break; + + } + return a; +} + +enum +{ + ODynam = 1<<0, + OAddable = 1<<1, +}; + +static Node clean[20]; +static int cleani = 0; + +int +xgen(Node *n, Node *a, int o) +{ + regalloc(a, types[tptr], N); + + if(o & ODynam) + if(n->addable) + if(n->op != OINDREG) + if(n->op != OREGISTER) + return 1; + + agen(n, a); + return 0; +} + +void +sudoclean(void) +{ + if(clean[cleani-1].op != OEMPTY) + regfree(&clean[cleani-1]); + if(clean[cleani-2].op != OEMPTY) + regfree(&clean[cleani-2]); + cleani -= 2; +} + +/* + * generate code to compute address of n, + * a reference to a (perhaps nested) field inside + * an array or struct. + * return 0 on failure, 1 on success. + * on success, leaves usable address in a. + * + * caller is responsible for calling sudoclean + * after successful sudoaddable, + * to release the register used for a. + */ +int +sudoaddable(int as, Node *n, Addr *a) +{ + int o, i, w; + int oary[10]; + int64 v; + Node n1, n2, n3, n4, *nn, *l, *r; + Node *reg, *reg1; + Prog *p1; + Type *t; + + if(n->type == T) + return 0; + + switch(n->op) { + case OLITERAL: + if(n->val.ctype != CTINT) + break; + v = mpgetfix(n->val.u.xval); + if(v >= 32000 || v <= -32000) + break; + goto lit; + + case ODOT: + case ODOTPTR: + cleani += 2; + reg = &clean[cleani-1]; + reg1 = &clean[cleani-2]; + reg->op = OEMPTY; + reg1->op = OEMPTY; + goto odot; + + case OINDEX: + if(n->left->type->etype == TSTRING) + return 0; + goto oindex; + } + return 0; + +lit: + switch(as) { + default: + return 0; + case AADDB: case AADDW: case AADDL: case AADDQ: + case ASUBB: case ASUBW: case ASUBL: case ASUBQ: + case AANDB: case AANDW: case AANDL: case AANDQ: + case AORB: case AORW: case AORL: case AORQ: + case AXORB: case AXORW: case AXORL: case AXORQ: + case AINCB: case AINCW: case AINCL: case AINCQ: + case ADECB: case ADECW: case ADECL: case ADECQ: + case AMOVB: case AMOVW: case AMOVL: case AMOVQ: + break; + } + + cleani += 2; + reg = &clean[cleani-1]; + reg1 = &clean[cleani-2]; + reg->op = OEMPTY; + reg1->op = OEMPTY; + naddr(n, a, 1); + goto yes; + +odot: + o = dotoffset(n, oary, &nn); + if(nn == N) + goto no; + + if(nn->addable && o == 1 && oary[0] >= 0) { + // directly addressable set of DOTs + n1 = *nn; + n1.type = n->type; + n1.xoffset += oary[0]; + naddr(&n1, a, 1); + goto yes; + } + + regalloc(reg, types[tptr], N); + n1 = *reg; + n1.op = OINDREG; + if(oary[0] >= 0) { + agen(nn, reg); + n1.xoffset = oary[0]; + } else { + cgen(nn, reg); + n1.xoffset = -(oary[0]+1); + } + + for(i=1; i= 0) + fatal("cant happen"); + gins(AMOVQ, &n1, reg); + n1.xoffset = -(oary[i]+1); + } + + a->type = D_NONE; + a->index = D_NONE; + naddr(&n1, a, 1); + goto yes; + +oindex: + l = n->left; + r = n->right; + if(l->ullman >= UINF && r->ullman >= UINF) + return 0; + + // set o to type of array + o = 0; + if(isptr[l->type->etype]) + fatal("ptr ary"); + if(l->type->etype != TARRAY) + fatal("not ary"); + if(l->type->bound < 0) + o |= ODynam; + + w = n->type->width; + if(isconst(r, CTINT)) + goto oindex_const; + + switch(w) { + default: + return 0; + case 1: + case 2: + case 4: + case 8: + break; + } + + cleani += 2; + reg = &clean[cleani-1]; + reg1 = &clean[cleani-2]; + reg->op = OEMPTY; + reg1->op = OEMPTY; + + // load the array (reg) + if(l->ullman > r->ullman) { + if(xgen(l, reg, o)) + o |= OAddable; + } + + // load the index (reg1) + t = types[TUINT64]; + if(issigned[r->type->etype]) + t = types[TINT64]; + regalloc(reg1, t, N); + regalloc(&n3, r->type, reg1); + cgen(r, &n3); + gmove(&n3, reg1); + regfree(&n3); + + // load the array (reg) + if(l->ullman <= r->ullman) { + if(xgen(l, reg, o)) + o |= OAddable; + } + + if(!(o & ODynam) && l->type->width >= unmappedzero && l->op == OIND) { + // cannot rely on page protections to + // catch array ptr == 0, so dereference. + n2 = *reg; + n2.xoffset = 0; + n2.op = OINDREG; + n2.type = types[TUINT8]; + gins(ATESTB, nodintconst(0), &n2); + } + + // check bounds + if(!debug['B'] && !n->etype) { + // check bounds + n4.op = OXXX; + t = types[TUINT32]; + if(o & ODynam) { + if(o & OAddable) { + n2 = *l; + n2.xoffset += Array_nel; + n2.type = types[TUINT32]; + if(is64(r->type)) { + t = types[TUINT64]; + regalloc(&n4, t, N); + gmove(&n2, &n4); + n2 = n4; + } + } else { + n2 = *reg; + n2.xoffset = Array_nel; + n2.op = OINDREG; + n2.type = types[TUINT32]; + if(is64(r->type)) { + t = types[TUINT64]; + regalloc(&n4, t, N); + gmove(&n2, &n4); + n2 = n4; + } + } + } else { + if(is64(r->type)) + t = types[TUINT64]; + nodconst(&n2, types[TUINT64], l->type->bound); + } + gins(optoas(OCMP, t), reg1, &n2); + p1 = gbranch(optoas(OLT, t), T); + if(n4.op != OXXX) + regfree(&n4); + ginscall(panicindex, 0); + patch(p1, pc); + } + + if(o & ODynam) { + if(o & OAddable) { + n2 = *l; + n2.xoffset += Array_array; + n2.type = types[tptr]; + gmove(&n2, reg); + } else { + n2 = *reg; + n2.op = OINDREG; + n2.xoffset = Array_array; + n2.type = types[tptr]; + gmove(&n2, reg); + } + } + + if(o & OAddable) { + naddr(reg1, a, 1); + a->offset = 0; + a->scale = w; + a->index = a->type; + a->type = reg->val.u.reg + D_INDIR; + } else { + naddr(reg1, a, 1); + a->offset = 0; + a->scale = w; + a->index = a->type; + a->type = reg->val.u.reg + D_INDIR; + } + + goto yes; + +oindex_const: + // index is constant + // can check statically and + // can multiply by width statically + + v = mpgetfix(r->val.u.xval); + + if(sudoaddable(as, l, a)) + goto oindex_const_sudo; + + cleani += 2; + reg = &clean[cleani-1]; + reg1 = &clean[cleani-2]; + reg->op = OEMPTY; + reg1->op = OEMPTY; + + regalloc(reg, types[tptr], N); + agen(l, reg); + + if(o & ODynam) { + if(!debug['B'] && !n->etype) { + n1 = *reg; + n1.op = OINDREG; + n1.type = types[tptr]; + n1.xoffset = Array_nel; + nodconst(&n2, types[TUINT64], v); + gins(optoas(OCMP, types[TUINT32]), &n1, &n2); + p1 = gbranch(optoas(OGT, types[TUINT32]), T); + ginscall(panicindex, 0); + patch(p1, pc); + } + + n1 = *reg; + n1.op = OINDREG; + n1.type = types[tptr]; + n1.xoffset = Array_array; + gmove(&n1, reg); + + } + + n2 = *reg; + n2.op = OINDREG; + n2.xoffset = v*w; + a->type = D_NONE; + a->index = D_NONE; + naddr(&n2, a, 1); + goto yes; + +oindex_const_sudo: + if((o & ODynam) == 0) { + // array indexed by a constant + a->offset += v*w; + goto yes; + } + + // slice indexed by a constant + if(!debug['B'] && !n->etype) { + a->offset += Array_nel; + nodconst(&n2, types[TUINT64], v); + p1 = gins(optoas(OCMP, types[TUINT32]), N, &n2); + p1->from = *a; + p1 = gbranch(optoas(OGT, types[TUINT32]), T); + ginscall(panicindex, 0); + patch(p1, pc); + a->offset -= Array_nel; + } + + a->offset += Array_array; + reg = &clean[cleani-1]; + if(reg->op == OEMPTY) + regalloc(reg, types[tptr], N); + + p1 = gins(AMOVQ, N, reg); + p1->from = *a; + + n2 = *reg; + n2.op = OINDREG; + n2.xoffset = v*w; + a->type = D_NONE; + a->index = D_NONE; + naddr(&n2, a, 1); + goto yes; + +yes: + return 1; + +no: + sudoclean(); + return 0; +} diff --git a/src/cmd/6g/list.c b/src/cmd/6g/list.c new file mode 100644 index 000000000..c8077c97a --- /dev/null +++ b/src/cmd/6g/list.c @@ -0,0 +1,359 @@ +// Derived from Inferno utils/6c/list.c +// http://code.google.com/p/inferno-os/source/browse/utils/6c/list.c +// +// Copyright © 1994-1999 Lucent Technologies Inc. All rights reserved. +// Portions Copyright © 1995-1997 C H Forsyth (forsyth@terzarima.net) +// Portions Copyright © 1997-1999 Vita Nuova Limited +// Portions Copyright © 2000-2007 Vita Nuova Holdings Limited (www.vitanuova.com) +// Portions Copyright © 2004,2006 Bruce Ellis +// Portions Copyright © 2005-2007 C H Forsyth (forsyth@terzarima.net) +// Revisions Copyright © 2000-2007 Lucent Technologies Inc. and others +// Portions Copyright © 2009 The Go Authors. All rights reserved. +// +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to deal +// in the Software without restriction, including without limitation the rights +// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +// copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +// THE SOFTWARE. + +#include "gg.h" + +static int sconsize; +void +listinit(void) +{ + + fmtinstall('A', Aconv); // as + fmtinstall('P', Pconv); // Prog* + fmtinstall('D', Dconv); // Addr* + fmtinstall('R', Rconv); // reg + fmtinstall('Y', Yconv); // sconst +} + +int +Pconv(Fmt *fp) +{ + char str[STRINGSZ]; + Prog *p; + char scale[40]; + + p = va_arg(fp->args, Prog*); + sconsize = 8; + scale[0] = '\0'; + if(p->from.scale != 0 && (p->as == AGLOBL || p->as == ATEXT)) + snprint(scale, sizeof scale, "%d,", p->from.scale); + switch(p->as) { + default: + snprint(str, sizeof(str), "%.4d (%L) %-7A %D,%s%D", + p->loc, p->lineno, p->as, &p->from, scale, &p->to); + break; + + case ADATA: + sconsize = p->from.scale; + snprint(str, sizeof(str), "%.4d (%L) %-7A %D/%d,%D", + p->loc, p->lineno, p->as, &p->from, sconsize, &p->to); + break; + + case ATEXT: + snprint(str, sizeof(str), "%.4d (%L) %-7A %D,%s%lD", + p->loc, p->lineno, p->as, &p->from, scale, &p->to); + break; + } + return fmtstrcpy(fp, str); +} + +int +Dconv(Fmt *fp) +{ + char str[STRINGSZ], s[STRINGSZ]; + Addr *a; + int i; + uint32 d1, d2; + + a = va_arg(fp->args, Addr*); + i = a->type; + if(i >= D_INDIR) { + if(a->offset) + snprint(str, sizeof(str), "%lld(%R)", a->offset, i-D_INDIR); + else + snprint(str, sizeof(str), "(%R)", i-D_INDIR); + goto brk; + } + switch(i) { + + default: + if(a->offset) + snprint(str, sizeof(str), "$%lld,%R", a->offset, i); + else + snprint(str, sizeof(str), "%R", i); + break; + + case D_NONE: + str[0] = 0; + break; + + case D_BRANCH: + if(a->branch == nil) + snprint(str, sizeof(str), ""); + else + snprint(str, sizeof(str), "%d", a->branch->loc); + break; + + case D_EXTERN: + snprint(str, sizeof(str), "%S+%lld(SB)", a->sym, a->offset); + break; + + case D_STATIC: + snprint(str, sizeof(str), "%S<>+%lld(SB)", a->sym, a->offset); + break; + + case D_AUTO: + snprint(str, sizeof(str), "%S+%lld(SP)", a->sym, a->offset); + break; + + case D_PARAM: + snprint(str, sizeof(str), "%S+%lld(FP)", a->sym, a->offset); + break; + + case D_CONST: + if(fp->flags & FmtLong) { + d1 = a->offset & 0xffffffffLL; + d2 = (a->offset>>32) & 0xffffffffLL; + snprint(str, sizeof(str), "$%ud-%ud", (ulong)d1, (ulong)d2); + break; + } + snprint(str, sizeof(str), "$%lld", a->offset); + break; + + case D_FCONST: + snprint(str, sizeof(str), "$(%.17e)", a->dval); + break; + + case D_SCONST: + snprint(str, sizeof(str), "$\"%Y\"", a->sval); + break; + + case D_ADDR: + a->type = a->index; + a->index = D_NONE; + snprint(str, sizeof(str), "$%D", a); + a->index = a->type; + a->type = D_ADDR; + goto conv; + } +brk: + if(a->index != D_NONE) { + snprint(s, sizeof(s), "(%R*%d)", (int)a->index, (int)a->scale); + strcat(str, s); + } +conv: + return fmtstrcpy(fp, str); +} + +static char* regstr[] = +{ + "AL", /* [D_AL] */ + "CL", + "DL", + "BL", + "SPB", + "BPB", + "SIB", + "DIB", + "R8B", + "R9B", + "R10B", + "R11B", + "R12B", + "R13B", + "R14B", + "R15B", + + "AX", /* [D_AX] */ + "CX", + "DX", + "BX", + "SP", + "BP", + "SI", + "DI", + "R8", + "R9", + "R10", + "R11", + "R12", + "R13", + "R14", + "R15", + + "AH", + "CH", + "DH", + "BH", + + "F0", /* [D_F0] */ + "F1", + "F2", + "F3", + "F4", + "F5", + "F6", + "F7", + + "M0", + "M1", + "M2", + "M3", + "M4", + "M5", + "M6", + "M7", + + "X0", + "X1", + "X2", + "X3", + "X4", + "X5", + "X6", + "X7", + "X8", + "X9", + "X10", + "X11", + "X12", + "X13", + "X14", + "X15", + + "CS", /* [D_CS] */ + "SS", + "DS", + "ES", + "FS", + "GS", + + "GDTR", /* [D_GDTR] */ + "IDTR", /* [D_IDTR] */ + "LDTR", /* [D_LDTR] */ + "MSW", /* [D_MSW] */ + "TASK", /* [D_TASK] */ + + "CR0", /* [D_CR] */ + "CR1", + "CR2", + "CR3", + "CR4", + "CR5", + "CR6", + "CR7", + "CR8", + "CR9", + "CR10", + "CR11", + "CR12", + "CR13", + "CR14", + "CR15", + + "DR0", /* [D_DR] */ + "DR1", + "DR2", + "DR3", + "DR4", + "DR5", + "DR6", + "DR7", + + "TR0", /* [D_TR] */ + "TR1", + "TR2", + "TR3", + "TR4", + "TR5", + "TR6", + "TR7", + + "NONE", /* [D_NONE] */ +}; + +int +Rconv(Fmt *fp) +{ + char str[STRINGSZ]; + int r; + + r = va_arg(fp->args, int); + if(r < 0 || r >= nelem(regstr) || regstr[r] == nil) { + snprint(str, sizeof(str), "BAD_R(%d)", r); + return fmtstrcpy(fp, str); + } + return fmtstrcpy(fp, regstr[r]); +} + +int +Aconv(Fmt *fp) +{ + int i; + + i = va_arg(fp->args, int); + return fmtstrcpy(fp, anames[i]); +} + + +int +Yconv(Fmt *fp) +{ + int i, c; + char str[STRINGSZ], *p, *a; + + a = va_arg(fp->args, char*); + p = str; + for(i=0; i= 'a' && c <= 'z') || + (c >= 'A' && c <= 'Z') || + (c >= '0' && c <= '9')) { + *p++ = c; + continue; + } + *p++ = '\\'; + switch(c) { + default: + if(c < 040 || c >= 0177) + break; /* not portable */ + p[-1] = c; + continue; + case 0: + *p++ = 'z'; + continue; + case '\\': + case '"': + *p++ = c; + continue; + case '\n': + *p++ = 'n'; + continue; + case '\t': + *p++ = 't'; + continue; + } + *p++ = (c>>6) + '0'; + *p++ = ((c>>3) & 7) + '0'; + *p++ = (c & 7) + '0'; + } + *p = 0; + return fmtstrcpy(fp, str); +} diff --git a/src/cmd/6g/opt.h b/src/cmd/6g/opt.h new file mode 100644 index 000000000..9a8866b8d --- /dev/null +++ b/src/cmd/6g/opt.h @@ -0,0 +1,166 @@ +// Derived from Inferno utils/6c/gc.h +// http://code.google.com/p/inferno-os/source/browse/utils/6c/gc.h +// +// Copyright © 1994-1999 Lucent Technologies Inc. All rights reserved. +// Portions Copyright © 1995-1997 C H Forsyth (forsyth@terzarima.net) +// Portions Copyright © 1997-1999 Vita Nuova Limited +// Portions Copyright © 2000-2007 Vita Nuova Holdings Limited (www.vitanuova.com) +// Portions Copyright © 2004,2006 Bruce Ellis +// Portions Copyright © 2005-2007 C H Forsyth (forsyth@terzarima.net) +// Revisions Copyright © 2000-2007 Lucent Technologies Inc. and others +// Portions Copyright © 2009 The Go Authors. All rights reserved. +// +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to deal +// in the Software without restriction, including without limitation the rights +// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +// copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +// THE SOFTWARE. + +#define Z N +#define Adr Addr + +#define D_HI D_NONE +#define D_LO D_NONE + +#define isregtype(t) ((t)>= D_AX && (t)<=D_R15) + +#define BLOAD(r) band(bnot(r->refbehind), r->refahead) +#define BSTORE(r) band(bnot(r->calbehind), r->calahead) +#define LOAD(r) (~r->refbehind.b[z] & r->refahead.b[z]) +#define STORE(r) (~r->calbehind.b[z] & r->calahead.b[z]) + +#define CLOAD 5 +#define CREF 5 +#define CINF 1000 +#define LOOP 3 + +typedef struct Reg Reg; +typedef struct Rgn Rgn; + +struct Reg +{ + + Bits set; + Bits use1; + Bits use2; + + Bits refbehind; + Bits refahead; + Bits calbehind; + Bits calahead; + Bits regdiff; + Bits act; + + int32 regu; // register used bitmap + int32 rpo; // reverse post ordering + int32 active; + + uint16 loop; // x5 for every loop + uchar refset; // diagnostic generated + + Reg* p1; + Reg* p2; + Reg* p2link; + Reg* s1; + Reg* s2; + Reg* link; + Prog* prog; +}; +#define R ((Reg*)0) + +#define NRGN 600 +struct Rgn +{ + Reg* enter; + short cost; + short varno; + short regno; +}; + +EXTERN int32 exregoffset; // not set +EXTERN int32 exfregoffset; // not set +EXTERN Reg* firstr; +EXTERN Reg* lastr; +EXTERN Reg zreg; +EXTERN Reg* freer; +EXTERN Reg** rpo2r; +EXTERN Rgn region[NRGN]; +EXTERN Rgn* rgp; +EXTERN int nregion; +EXTERN int nvar; +EXTERN int32 regbits; +EXTERN int32 exregbits; +EXTERN Bits externs; +EXTERN Bits params; +EXTERN Bits consts; +EXTERN Bits addrs; +EXTERN Bits ovar; +EXTERN int change; +EXTERN int32 maxnr; +EXTERN int32* idom; + +EXTERN struct +{ + int32 ncvtreg; + int32 nspill; + int32 nreload; + int32 ndelmov; + int32 nvar; + int32 naddr; +} ostats; + +/* + * reg.c + */ +Reg* rega(void); +int rcmp(const void*, const void*); +void regopt(Prog*); +void addmove(Reg*, int, int, int); +Bits mkvar(Reg*, Adr*); +void prop(Reg*, Bits, Bits); +void loopit(Reg*, int32); +void synch(Reg*, Bits); +uint32 allreg(uint32, Rgn*); +void paint1(Reg*, int); +uint32 paint2(Reg*, int); +void paint3(Reg*, int, int32, int); +void addreg(Adr*, int); +void dumpone(Reg*); +void dumpit(char*, Reg*); +int noreturn(Prog *p); + +/* + * peep.c + */ +void peep(void); +void excise(Reg*); +Reg* uniqp(Reg*); +Reg* uniqs(Reg*); +int regtyp(Adr*); +int anyvar(Adr*); +int subprop(Reg*); +int copyprop(Reg*); +int copy1(Adr*, Adr*, Reg*, int); +int copyu(Prog*, Adr*, Adr*); + +int copyas(Adr*, Adr*); +int copyau(Adr*, Adr*); +int copysub(Adr*, Adr*, Adr*, int); +int copysub1(Prog*, Adr*, Adr*, int); + +int32 RtoB(int); +int32 FtoB(int); +int BtoR(int32); +int BtoF(int32); diff --git a/src/cmd/6g/peep.c b/src/cmd/6g/peep.c new file mode 100644 index 000000000..4432203f2 --- /dev/null +++ b/src/cmd/6g/peep.c @@ -0,0 +1,999 @@ +// Derived from Inferno utils/6c/peep.c +// http://code.google.com/p/inferno-os/source/browse/utils/6c/peep.c +// +// Copyright © 1994-1999 Lucent Technologies Inc. All rights reserved. +// Portions Copyright © 1995-1997 C H Forsyth (forsyth@terzarima.net) +// Portions Copyright © 1997-1999 Vita Nuova Limited +// Portions Copyright © 2000-2007 Vita Nuova Holdings Limited (www.vitanuova.com) +// Portions Copyright © 2004,2006 Bruce Ellis +// Portions Copyright © 2005-2007 C H Forsyth (forsyth@terzarima.net) +// Revisions Copyright © 2000-2007 Lucent Technologies Inc. and others +// Portions Copyright © 2009 The Go Authors. All rights reserved. +// +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to deal +// in the Software without restriction, including without limitation the rights +// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +// copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +// THE SOFTWARE. + +#include "gg.h" +#include "opt.h" + +static void conprop(Reg *r); + +// do we need the carry bit +static int +needc(Prog *p) +{ + while(p != P) { + switch(p->as) { + case AADCL: + case AADCQ: + case ASBBL: + case ASBBQ: + case ARCRL: + case ARCRQ: + return 1; + case AADDL: + case AADDQ: + case ASUBL: + case ASUBQ: + case AJMP: + case ARET: + case ACALL: + return 0; + default: + if(p->to.type == D_BRANCH) + return 0; + } + p = p->link; + } + return 0; +} + +static Reg* +rnops(Reg *r) +{ + Prog *p; + Reg *r1; + + if(r != R) + for(;;) { + p = r->prog; + if(p->as != ANOP || p->from.type != D_NONE || p->to.type != D_NONE) + break; + r1 = uniqs(r); + if(r1 == R) + break; + r = r1; + } + return r; +} + +void +peep(void) +{ + Reg *r, *r1, *r2; + Prog *p, *p1; + int t; + + /* + * complete R structure + */ + t = 0; + for(r=firstr; r!=R; r=r1) { + r1 = r->link; + if(r1 == R) + break; + p = r->prog->link; + while(p != r1->prog) + switch(p->as) { + default: + r2 = rega(); + r->link = r2; + r2->link = r1; + + r2->prog = p; + p->reg = r2; + + r2->p1 = r; + r->s1 = r2; + r2->s1 = r1; + r1->p1 = r2; + + r = r2; + t++; + + case ADATA: + case AGLOBL: + case ANAME: + case ASIGNAME: + p = p->link; + } + } + + // constant propagation + // find MOV $con,R followed by + // another MOV $con,R without + // setting R in the interim + for(r=firstr; r!=R; r=r->link) { + p = r->prog; + switch(p->as) { + case ALEAL: + case ALEAQ: + if(regtyp(&p->to)) + if(p->from.sym != S) + conprop(r); + break; + + case AMOVB: + case AMOVW: + case AMOVL: + case AMOVQ: + case AMOVSS: + case AMOVSD: + if(regtyp(&p->to)) + if(p->from.type == D_CONST) + conprop(r); + break; + } + } + +loop1: + if(debug['P'] && debug['v']) + dumpit("loop1", firstr); + + t = 0; + for(r=firstr; r!=R; r=r->link) { + p = r->prog; + switch(p->as) { + case AMOVL: + case AMOVQ: + case AMOVSS: + case AMOVSD: + if(regtyp(&p->to)) + if(regtyp(&p->from)) { + if(copyprop(r)) { + excise(r); + t++; + } else + if(subprop(r) && copyprop(r)) { + excise(r); + t++; + } + } + break; + + case AMOVBLZX: + case AMOVWLZX: + case AMOVBLSX: + case AMOVWLSX: + if(regtyp(&p->to)) { + r1 = rnops(uniqs(r)); + if(r1 != R) { + p1 = r1->prog; + if(p->as == p1->as && p->to.type == p1->from.type){ + p1->as = AMOVL; + t++; + } + } + } + break; + + case AMOVBQSX: + case AMOVBQZX: + case AMOVWQSX: + case AMOVWQZX: + case AMOVLQSX: + case AMOVLQZX: + if(regtyp(&p->to)) { + r1 = rnops(uniqs(r)); + if(r1 != R) { + p1 = r1->prog; + if(p->as == p1->as && p->to.type == p1->from.type){ + p1->as = AMOVQ; + t++; + } + } + } + break; + + case AADDL: + case AADDQ: + case AADDW: + if(p->from.type != D_CONST || needc(p->link)) + break; + if(p->from.offset == -1){ + if(p->as == AADDQ) + p->as = ADECQ; + else + if(p->as == AADDL) + p->as = ADECL; + else + p->as = ADECW; + p->from = zprog.from; + break; + } + if(p->from.offset == 1){ + if(p->as == AADDQ) + p->as = AINCQ; + else if(p->as == AADDL) + p->as = AINCL; + else + p->as = AINCW; + p->from = zprog.from; + break; + } + break; + + case ASUBL: + case ASUBQ: + case ASUBW: + if(p->from.type != D_CONST || needc(p->link)) + break; + if(p->from.offset == -1) { + if(p->as == ASUBQ) + p->as = AINCQ; + else + if(p->as == ASUBL) + p->as = AINCL; + else + p->as = AINCW; + p->from = zprog.from; + break; + } + if(p->from.offset == 1){ + if(p->as == ASUBQ) + p->as = ADECQ; + else + if(p->as == ASUBL) + p->as = ADECL; + else + p->as = ADECW; + p->from = zprog.from; + break; + } + break; + } + } + if(t) + goto loop1; +} + +void +excise(Reg *r) +{ + Prog *p; + + p = r->prog; + if(debug['P'] && debug['v']) + print("%P ===delete===\n", p); + + p->as = ANOP; + p->from = zprog.from; + p->to = zprog.to; + + ostats.ndelmov++; +} + +Reg* +uniqp(Reg *r) +{ + Reg *r1; + + r1 = r->p1; + if(r1 == R) { + r1 = r->p2; + if(r1 == R || r1->p2link != R) + return R; + } else + if(r->p2 != R) + return R; + return r1; +} + +Reg* +uniqs(Reg *r) +{ + Reg *r1; + + r1 = r->s1; + if(r1 == R) { + r1 = r->s2; + if(r1 == R) + return R; + } else + if(r->s2 != R) + return R; + return r1; +} + +int +regtyp(Adr *a) +{ + int t; + + t = a->type; + if(t >= D_AX && t <= D_R15) + return 1; + if(t >= D_X0 && t <= D_X0+15) + return 1; + return 0; +} + +/* + * the idea is to substitute + * one register for another + * from one MOV to another + * MOV a, R0 + * ADD b, R0 / no use of R1 + * MOV R0, R1 + * would be converted to + * MOV a, R1 + * ADD b, R1 + * MOV R1, R0 + * hopefully, then the former or latter MOV + * will be eliminated by copy propagation. + */ +int +subprop(Reg *r0) +{ + Prog *p; + Adr *v1, *v2; + Reg *r; + int t; + + p = r0->prog; + v1 = &p->from; + if(!regtyp(v1)) + return 0; + v2 = &p->to; + if(!regtyp(v2)) + return 0; + for(r=uniqp(r0); r!=R; r=uniqp(r)) { + if(uniqs(r) == R) + break; + p = r->prog; + switch(p->as) { + case ACALL: + return 0; + + case AIMULL: + case AIMULQ: + case AIMULW: + if(p->to.type != D_NONE) + break; + + case ADIVB: + case ADIVL: + case ADIVQ: + case ADIVW: + case AIDIVB: + case AIDIVL: + case AIDIVQ: + case AIDIVW: + case AIMULB: + case AMULB: + case AMULL: + case AMULQ: + case AMULW: + + case ARCLB: + case ARCLL: + case ARCLQ: + case ARCLW: + case ARCRB: + case ARCRL: + case ARCRQ: + case ARCRW: + case AROLB: + case AROLL: + case AROLQ: + case AROLW: + case ARORB: + case ARORL: + case ARORQ: + case ARORW: + case ASALB: + case ASALL: + case ASALQ: + case ASALW: + case ASARB: + case ASARL: + case ASARQ: + case ASARW: + case ASHLB: + case ASHLL: + case ASHLQ: + case ASHLW: + case ASHRB: + case ASHRL: + case ASHRQ: + case ASHRW: + + case AREP: + case AREPN: + + case ACWD: + case ACDQ: + case ACQO: + + case ASTOSB: + case ASTOSL: + case ASTOSQ: + case AMOVSB: + case AMOVSL: + case AMOVSQ: + return 0; + + case AMOVL: + case AMOVQ: + if(p->to.type == v1->type) + goto gotit; + break; + } + if(copyau(&p->from, v2) || + copyau(&p->to, v2)) + break; + if(copysub(&p->from, v1, v2, 0) || + copysub(&p->to, v1, v2, 0)) + break; + } + return 0; + +gotit: + copysub(&p->to, v1, v2, 1); + if(debug['P']) { + print("gotit: %D->%D\n%P", v1, v2, r->prog); + if(p->from.type == v2->type) + print(" excise"); + print("\n"); + } + for(r=uniqs(r); r!=r0; r=uniqs(r)) { + p = r->prog; + copysub(&p->from, v1, v2, 1); + copysub(&p->to, v1, v2, 1); + if(debug['P']) + print("%P\n", r->prog); + } + t = v1->type; + v1->type = v2->type; + v2->type = t; + if(debug['P']) + print("%P last\n", r->prog); + return 1; +} + +/* + * The idea is to remove redundant copies. + * v1->v2 F=0 + * (use v2 s/v2/v1/)* + * set v1 F=1 + * use v2 return fail + * ----------------- + * v1->v2 F=0 + * (use v2 s/v2/v1/)* + * set v1 F=1 + * set v2 return success + */ +int +copyprop(Reg *r0) +{ + Prog *p; + Adr *v1, *v2; + Reg *r; + + p = r0->prog; + v1 = &p->from; + v2 = &p->to; + if(copyas(v1, v2)) + return 1; + for(r=firstr; r!=R; r=r->link) + r->active = 0; + return copy1(v1, v2, r0->s1, 0); +} + +int +copy1(Adr *v1, Adr *v2, Reg *r, int f) +{ + int t; + Prog *p; + + if(r->active) { + if(debug['P']) + print("act set; return 1\n"); + return 1; + } + r->active = 1; + if(debug['P']) + print("copy %D->%D f=%d\n", v1, v2, f); + for(; r != R; r = r->s1) { + p = r->prog; + if(debug['P']) + print("%P", p); + if(!f && uniqp(r) == R) { + f = 1; + if(debug['P']) + print("; merge; f=%d", f); + } + t = copyu(p, v2, A); + switch(t) { + case 2: /* rar, cant split */ + if(debug['P']) + print("; %D rar; return 0\n", v2); + return 0; + + case 3: /* set */ + if(debug['P']) + print("; %D set; return 1\n", v2); + return 1; + + case 1: /* used, substitute */ + case 4: /* use and set */ + if(f) { + if(!debug['P']) + return 0; + if(t == 4) + print("; %D used+set and f=%d; return 0\n", v2, f); + else + print("; %D used and f=%d; return 0\n", v2, f); + return 0; + } + if(copyu(p, v2, v1)) { + if(debug['P']) + print("; sub fail; return 0\n"); + return 0; + } + if(debug['P']) + print("; sub %D/%D", v2, v1); + if(t == 4) { + if(debug['P']) + print("; %D used+set; return 1\n", v2); + return 1; + } + break; + } + if(!f) { + t = copyu(p, v1, A); + if(!f && (t == 2 || t == 3 || t == 4)) { + f = 1; + if(debug['P']) + print("; %D set and !f; f=%d", v1, f); + } + } + if(debug['P']) + print("\n"); + if(r->s2) + if(!copy1(v1, v2, r->s2, f)) + return 0; + } + return 1; +} + +/* + * return + * 1 if v only used (and substitute), + * 2 if read-alter-rewrite + * 3 if set + * 4 if set and used + * 0 otherwise (not touched) + */ +int +copyu(Prog *p, Adr *v, Adr *s) +{ + + switch(p->as) { + + default: + if(debug['P']) + print("unknown op %A\n", p->as); + /* SBBL; ADCL; FLD1; SAHF */ + return 2; + + + case ANEGB: + case ANEGW: + case ANEGL: + case ANEGQ: + case ANOTB: + case ANOTW: + case ANOTL: + case ANOTQ: + if(copyas(&p->to, v)) + return 2; + break; + + case ALEAL: /* lhs addr, rhs store */ + case ALEAQ: + if(copyas(&p->from, v)) + return 2; + + + case ANOP: /* rhs store */ + case AMOVL: + case AMOVQ: + case AMOVBLSX: + case AMOVBLZX: + case AMOVBQSX: + case AMOVBQZX: + case AMOVLQSX: + case AMOVLQZX: + case AMOVWLSX: + case AMOVWLZX: + case AMOVWQSX: + case AMOVWQZX: + + case AMOVSS: + case AMOVSD: + case ACVTSD2SL: + case ACVTSD2SQ: + case ACVTSD2SS: + case ACVTSL2SD: + case ACVTSL2SS: + case ACVTSQ2SD: + case ACVTSQ2SS: + case ACVTSS2SD: + case ACVTSS2SL: + case ACVTSS2SQ: + case ACVTTSD2SL: + case ACVTTSD2SQ: + case ACVTTSS2SL: + case ACVTTSS2SQ: + if(copyas(&p->to, v)) { + if(s != A) + return copysub(&p->from, v, s, 1); + if(copyau(&p->from, v)) + return 4; + return 3; + } + goto caseread; + + case ARCLB: + case ARCLL: + case ARCLQ: + case ARCLW: + case ARCRB: + case ARCRL: + case ARCRQ: + case ARCRW: + case AROLB: + case AROLL: + case AROLQ: + case AROLW: + case ARORB: + case ARORL: + case ARORQ: + case ARORW: + case ASALB: + case ASALL: + case ASALQ: + case ASALW: + case ASARB: + case ASARL: + case ASARQ: + case ASARW: + case ASHLB: + case ASHLL: + case ASHLQ: + case ASHLW: + case ASHRB: + case ASHRL: + case ASHRQ: + case ASHRW: + if(copyas(&p->to, v)) + return 2; + if(copyas(&p->from, v)) + if(p->from.type == D_CX) + return 2; + goto caseread; + + case AADDB: /* rhs rar */ + case AADDL: + case AADDQ: + case AADDW: + case AANDB: + case AANDL: + case AANDQ: + case AANDW: + case ADECL: + case ADECQ: + case ADECW: + case AINCL: + case AINCQ: + case AINCW: + case ASUBB: + case ASUBL: + case ASUBQ: + case ASUBW: + case AORB: + case AORL: + case AORQ: + case AORW: + case AXORB: + case AXORL: + case AXORQ: + case AXORW: + case AMOVB: + case AMOVW: + + case AADDSD: + case AADDSS: + case ACMPSD: + case ACMPSS: + case ADIVSD: + case ADIVSS: + case AMAXSD: + case AMAXSS: + case AMINSD: + case AMINSS: + case AMULSD: + case AMULSS: + case ARCPSS: + case ARSQRTSS: + case ASQRTSD: + case ASQRTSS: + case ASUBSD: + case ASUBSS: + case AXORPD: + if(copyas(&p->to, v)) + return 2; + goto caseread; + + case ACMPL: /* read only */ + case ACMPW: + case ACMPB: + case ACMPQ: + + case ACOMISD: + case ACOMISS: + case AUCOMISD: + case AUCOMISS: + caseread: + if(s != A) { + if(copysub(&p->from, v, s, 1)) + return 1; + return copysub(&p->to, v, s, 1); + } + if(copyau(&p->from, v)) + return 1; + if(copyau(&p->to, v)) + return 1; + break; + + case AJGE: /* no reference */ + case AJNE: + case AJLE: + case AJEQ: + case AJHI: + case AJLS: + case AJMI: + case AJPL: + case AJGT: + case AJLT: + case AJCC: + case AJCS: + + case AADJSP: + case AWAIT: + case ACLD: + break; + + case AIMULL: + case AIMULQ: + case AIMULW: + if(p->to.type != D_NONE) { + if(copyas(&p->to, v)) + return 2; + goto caseread; + } + + case ADIVB: + case ADIVL: + case ADIVQ: + case ADIVW: + case AIDIVB: + case AIDIVL: + case AIDIVQ: + case AIDIVW: + case AIMULB: + case AMULB: + case AMULL: + case AMULQ: + case AMULW: + + case ACWD: + case ACDQ: + case ACQO: + if(v->type == D_AX || v->type == D_DX) + return 2; + goto caseread; + + case AREP: + case AREPN: + if(v->type == D_CX) + return 2; + goto caseread; + + case AMOVSB: + case AMOVSL: + case AMOVSQ: + if(v->type == D_DI || v->type == D_SI) + return 2; + goto caseread; + + case ASTOSB: + case ASTOSL: + case ASTOSQ: + if(v->type == D_AX || v->type == D_DI) + return 2; + goto caseread; + + case AJMP: /* funny */ + if(s != A) { + if(copysub(&p->to, v, s, 1)) + return 1; + return 0; + } + if(copyau(&p->to, v)) + return 1; + return 0; + + case ARET: /* funny */ + if(v->type == REGRET || v->type == FREGRET) + return 2; + if(s != A) + return 1; + return 3; + + case ACALL: /* funny */ + if(REGEXT && v->type <= REGEXT && v->type > exregoffset) + return 2; + if(REGARG >= 0 && v->type == (uchar)REGARG) + return 2; + + if(s != A) { + if(copysub(&p->to, v, s, 1)) + return 1; + return 0; + } + if(copyau(&p->to, v)) + return 4; + return 3; + + case ATEXT: /* funny */ + if(REGARG >= 0 && v->type == (uchar)REGARG) + return 3; + return 0; + } + return 0; +} + +/* + * direct reference, + * could be set/use depending on + * semantics + */ +int +copyas(Adr *a, Adr *v) +{ + if(a->type != v->type) + return 0; + if(regtyp(v)) + return 1; + if(v->type == D_AUTO || v->type == D_PARAM) + if(v->offset == a->offset) + return 1; + return 0; +} + +/* + * either direct or indirect + */ +int +copyau(Adr *a, Adr *v) +{ + + if(copyas(a, v)) + return 1; + if(regtyp(v)) { + if(a->type-D_INDIR == v->type) + return 1; + if(a->index == v->type) + return 1; + } + return 0; +} + +/* + * substitute s for v in a + * return failure to substitute + */ +int +copysub(Adr *a, Adr *v, Adr *s, int f) +{ + int t; + + if(copyas(a, v)) { + t = s->type; + if(t >= D_AX && t <= D_R15 || t >= D_X0 && t <= D_X0+15) { + if(f) + a->type = t; + } + return 0; + } + if(regtyp(v)) { + t = v->type; + if(a->type == t+D_INDIR) { + if((s->type == D_BP || s->type == D_R13) && a->index != D_NONE) + return 1; /* can't use BP-base with index */ + if(f) + a->type = s->type+D_INDIR; +// return 0; + } + if(a->index == t) { + if(f) + a->index = s->type; + return 0; + } + return 0; + } + return 0; +} + +static void +conprop(Reg *r0) +{ + Reg *r; + Prog *p, *p0; + int t; + Adr *v0; + + p0 = r0->prog; + v0 = &p0->to; + r = r0; + +loop: + r = uniqs(r); + if(r == R || r == r0) + return; + if(uniqp(r) == R) + return; + + p = r->prog; + t = copyu(p, v0, A); + switch(t) { + case 0: // miss + case 1: // use + goto loop; + + case 2: // rar + case 4: // use and set + break; + + case 3: // set + if(p->as == p0->as) + if(p->from.type == p0->from.type) + if(p->from.sym == p0->from.sym) + if(p->from.offset == p0->from.offset) + if(p->from.scale == p0->from.scale) + if(p->from.dval == p0->from.dval) + if(p->from.index == p0->from.index) { + excise(r); + t++; + goto loop; + } + break; + } +} diff --git a/src/cmd/6g/reg.c b/src/cmd/6g/reg.c new file mode 100644 index 000000000..4d4263047 --- /dev/null +++ b/src/cmd/6g/reg.c @@ -0,0 +1,1690 @@ +// Derived from Inferno utils/6c/reg.c +// http://code.google.com/p/inferno-os/source/browse/utils/6c/reg.c +// +// Copyright © 1994-1999 Lucent Technologies Inc. All rights reserved. +// Portions Copyright © 1995-1997 C H Forsyth (forsyth@terzarima.net) +// Portions Copyright © 1997-1999 Vita Nuova Limited +// Portions Copyright © 2000-2007 Vita Nuova Holdings Limited (www.vitanuova.com) +// Portions Copyright © 2004,2006 Bruce Ellis +// Portions Copyright © 2005-2007 C H Forsyth (forsyth@terzarima.net) +// Revisions Copyright © 2000-2007 Lucent Technologies Inc. and others +// Portions Copyright © 2009 The Go Authors. All rights reserved. +// +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to deal +// in the Software without restriction, including without limitation the rights +// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +// copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +// THE SOFTWARE. + +#include "gg.h" +#undef EXTERN +#define EXTERN +#include "opt.h" + +#define NREGVAR 32 /* 16 general + 16 floating */ +#define REGBITS ((uint32)0xffffffff) +#define P2R(p) (Reg*)(p->reg) + +static int first = 1; + +Reg* +rega(void) +{ + Reg *r; + + r = freer; + if(r == R) { + r = mal(sizeof(*r)); + } else + freer = r->link; + + *r = zreg; + return r; +} + +int +rcmp(const void *a1, const void *a2) +{ + Rgn *p1, *p2; + int c1, c2; + + p1 = (Rgn*)a1; + p2 = (Rgn*)a2; + c1 = p2->cost; + c2 = p1->cost; + if(c1 -= c2) + return c1; + return p2->varno - p1->varno; +} + +static void +setoutvar(void) +{ + Type *t; + Node *n; + Addr a; + Iter save; + Bits bit; + int z; + + t = structfirst(&save, getoutarg(curfn->type)); + while(t != T) { + n = nodarg(t, 1); + a = zprog.from; + naddr(n, &a, 0); + bit = mkvar(R, &a); + for(z=0; zsym == s && v->name == n) + v->addr = 2; + } + } +} + +static char* regname[] = { + ".AX", + ".CX", + ".DX", + ".BX", + ".SP", + ".BP", + ".SI", + ".DI", + ".R8", + ".R9", + ".R10", + ".R11", + ".R12", + ".R13", + ".R14", + ".R15", + ".X0", + ".X1", + ".X2", + ".X3", + ".X4", + ".X5", + ".X6", + ".X7", + ".X8", + ".X9", + ".X10", + ".X11", + ".X12", + ".X13", + ".X14", + ".X15", +}; + +void +regopt(Prog *firstp) +{ + Reg *r, *r1; + Prog *p; + int i, z, nr; + uint32 vreg; + Bits bit; + + if(first) { + fmtinstall('Q', Qconv); + exregoffset = D_R13; // R14,R15 are external + first = 0; + } + + // count instructions + nr = 0; + for(p=firstp; p!=P; p=p->link) + nr++; + // if too big dont bother + if(nr >= 10000) { +// print("********** %S is too big (%d)\n", curfn->nname->sym, nr); + return; + } + + r1 = R; + firstr = R; + lastr = R; + nvar = 0; + + /* + * control flow is more complicated in generated go code + * than in generated c code. define pseudo-variables for + * registers, so we have complete register usage information. + */ + nvar = NREGVAR; + memset(var, 0, NREGVAR*sizeof var[0]); + for(i=0; ilink) { + switch(p->as) { + case ADATA: + case AGLOBL: + case ANAME: + case ASIGNAME: + continue; + } + r = rega(); + nr++; + if(firstr == R) { + firstr = r; + lastr = r; + } else { + lastr->link = r; + r->p1 = lastr; + lastr->s1 = r; + lastr = r; + } + r->prog = p; + p->reg = r; + + r1 = r->p1; + if(r1 != R) { + switch(r1->prog->as) { + case ARET: + case AJMP: + case AIRETL: + case AIRETQ: + r->p1 = R; + r1->s1 = R; + } + } + + bit = mkvar(r, &p->from); + if(bany(&bit)) + switch(p->as) { + /* + * funny + */ + case ALEAL: + case ALEAQ: + setaddrs(bit); + break; + + /* + * left side read + */ + default: + for(z=0; zuse1.b[z] |= bit.b[z]; + break; + + /* + * left side read+write + */ + case AXCHGB: + case AXCHGW: + case AXCHGL: + case AXCHGQ: + for(z=0; zuse1.b[z] |= bit.b[z]; + r->set.b[z] |= bit.b[z]; + } + break; + } + + bit = mkvar(r, &p->to); + if(bany(&bit)) + switch(p->as) { + default: + yyerror("reg: unknown op: %A", p->as); + break; + + /* + * right side read + */ + case ACMPB: + case ACMPL: + case ACMPQ: + case ACMPW: + case ACOMISS: + case ACOMISD: + case AUCOMISS: + case AUCOMISD: + case ATESTB: + case ATESTL: + case ATESTQ: + for(z=0; zuse2.b[z] |= bit.b[z]; + break; + + /* + * right side write + */ + case ALEAQ: + case ANOP: + case AMOVL: + case AMOVQ: + case AMOVB: + case AMOVW: + case AMOVBLSX: + case AMOVBLZX: + case AMOVBWSX: + case AMOVBWZX: + case AMOVBQSX: + case AMOVBQZX: + case AMOVLQSX: + case AMOVLQZX: + case AMOVWLSX: + case AMOVWLZX: + case AMOVWQSX: + case AMOVWQZX: + case APOPQ: + + case AMOVSS: + case AMOVSD: + case ACVTSD2SL: + case ACVTSD2SQ: + case ACVTSD2SS: + case ACVTSL2SD: + case ACVTSL2SS: + case ACVTSQ2SD: + case ACVTSQ2SS: + case ACVTSS2SD: + case ACVTSS2SL: + case ACVTSS2SQ: + case ACVTTSD2SL: + case ACVTTSD2SQ: + case ACVTTSS2SL: + case ACVTTSS2SQ: + for(z=0; zset.b[z] |= bit.b[z]; + break; + + /* + * right side read+write + */ + case AINCB: + case AINCL: + case AINCQ: + case AINCW: + case ADECB: + case ADECL: + case ADECQ: + case ADECW: + + case AADDB: + case AADDL: + case AADDQ: + case AADDW: + case AANDB: + case AANDL: + case AANDQ: + case AANDW: + case ASUBB: + case ASUBL: + case ASUBQ: + case ASUBW: + case AORB: + case AORL: + case AORQ: + case AORW: + case AXORB: + case AXORL: + case AXORQ: + case AXORW: + case ASALB: + case ASALL: + case ASALQ: + case ASALW: + case ASARB: + case ASARL: + case ASARQ: + case ASARW: + case ARCLB: + case ARCLL: + case ARCLQ: + case ARCLW: + case ARCRB: + case ARCRL: + case ARCRQ: + case ARCRW: + case AROLB: + case AROLL: + case AROLQ: + case AROLW: + case ARORB: + case ARORL: + case ARORQ: + case ARORW: + case ASHLB: + case ASHLL: + case ASHLQ: + case ASHLW: + case ASHRB: + case ASHRL: + case ASHRQ: + case ASHRW: + case AIMULL: + case AIMULQ: + case AIMULW: + case ANEGB: + case ANEGW: + case ANEGL: + case ANEGQ: + case ANOTL: + case ANOTQ: + case AADCL: + case AADCQ: + case ASBBL: + case ASBBQ: + + case ASETCC: + case ASETCS: + case ASETEQ: + case ASETGE: + case ASETGT: + case ASETHI: + case ASETLE: + case ASETLS: + case ASETLT: + case ASETMI: + case ASETNE: + case ASETOC: + case ASETOS: + case ASETPC: + case ASETPL: + case ASETPS: + + case AXCHGB: + case AXCHGW: + case AXCHGL: + case AXCHGQ: + + case AADDSD: + case AADDSS: + case ACMPSD: + case ACMPSS: + case ADIVSD: + case ADIVSS: + case AMAXSD: + case AMAXSS: + case AMINSD: + case AMINSS: + case AMULSD: + case AMULSS: + case ARCPSS: + case ARSQRTSS: + case ASQRTSD: + case ASQRTSS: + case ASUBSD: + case ASUBSS: + case AXORPD: + for(z=0; zset.b[z] |= bit.b[z]; + r->use2.b[z] |= bit.b[z]; + } + break; + + /* + * funny + */ + case ACALL: + setaddrs(bit); + break; + } + + switch(p->as) { + case AIMULL: + case AIMULQ: + case AIMULW: + if(p->to.type != D_NONE) + break; + + case AIDIVL: + case AIDIVW: + case AIDIVQ: + case ADIVL: + case ADIVW: + case ADIVQ: + case AMULL: + case AMULW: + case AMULQ: + r->set.b[0] |= RtoB(D_AX) | RtoB(D_DX); + r->use1.b[0] |= RtoB(D_AX) | RtoB(D_DX); + break; + + case AIDIVB: + case AIMULB: + case ADIVB: + case AMULB: + r->set.b[0] |= RtoB(D_AX); + r->use1.b[0] |= RtoB(D_AX); + break; + + case ACWD: + r->set.b[0] |= RtoB(D_AX) | RtoB(D_DX); + r->use1.b[0] |= RtoB(D_AX); + break; + + case ACDQ: + r->set.b[0] |= RtoB(D_DX); + r->use1.b[0] |= RtoB(D_AX); + break; + + case AREP: + case AREPN: + case ALOOP: + case ALOOPEQ: + case ALOOPNE: + r->set.b[0] |= RtoB(D_CX); + r->use1.b[0] |= RtoB(D_CX); + break; + + case AMOVSB: + case AMOVSL: + case AMOVSQ: + case AMOVSW: + case ACMPSB: + case ACMPSL: + case ACMPSQ: + case ACMPSW: + r->set.b[0] |= RtoB(D_SI) | RtoB(D_DI); + r->use1.b[0] |= RtoB(D_SI) | RtoB(D_DI); + break; + + case ASTOSB: + case ASTOSL: + case ASTOSQ: + case ASTOSW: + case ASCASB: + case ASCASL: + case ASCASQ: + case ASCASW: + r->set.b[0] |= RtoB(D_DI); + r->use1.b[0] |= RtoB(D_AX) | RtoB(D_DI); + break; + + case AINSB: + case AINSL: + case AINSW: + r->set.b[0] |= RtoB(D_DX) | RtoB(D_DI); + r->use1.b[0] |= RtoB(D_DI); + break; + + case AOUTSB: + case AOUTSL: + case AOUTSW: + r->set.b[0] |= RtoB(D_DI); + r->use1.b[0] |= RtoB(D_DX) | RtoB(D_DI); + break; + } + } + if(firstr == R) + return; + + for(i=0; iaddr) { + bit = blsh(i); + for(z=0; zaddr, v->etype, v->width, v->sym, v->offset); + } + + if(debug['R'] && debug['v']) + dumpit("pass1", firstr); + + /* + * pass 2 + * turn branch references to pointers + * build back pointers + */ + for(r=firstr; r!=R; r=r->link) { + p = r->prog; + if(p->to.type == D_BRANCH) { + if(p->to.branch == P) + fatal("pnil %P", p); + r1 = p->to.branch->reg; + if(r1 == R) + fatal("rnil %P", p); + if(r1 == r) { + //fatal("ref to self %P", p); + continue; + } + r->s2 = r1; + r->p2link = r1->p2; + r1->p2 = r; + } + } + + if(debug['R'] && debug['v']) + dumpit("pass2", firstr); + + /* + * pass 2.5 + * find looping structure + */ + for(r = firstr; r != R; r = r->link) + r->active = 0; + change = 0; + loopit(firstr, nr); + + if(debug['R'] && debug['v']) + dumpit("pass2.5", firstr); + + /* + * pass 3 + * iterate propagating usage + * back until flow graph is complete + */ +loop1: + change = 0; + for(r = firstr; r != R; r = r->link) + r->active = 0; + for(r = firstr; r != R; r = r->link) + if(r->prog->as == ARET) + prop(r, zbits, zbits); +loop11: + /* pick up unreachable code */ + i = 0; + for(r = firstr; r != R; r = r1) { + r1 = r->link; + if(r1 && r1->active && !r->active) { + prop(r, zbits, zbits); + i = 1; + } + } + if(i) + goto loop11; + if(change) + goto loop1; + + if(debug['R'] && debug['v']) + dumpit("pass3", firstr); + + /* + * pass 4 + * iterate propagating register/variable synchrony + * forward until graph is complete + */ +loop2: + change = 0; + for(r = firstr; r != R; r = r->link) + r->active = 0; + synch(firstr, zbits); + if(change) + goto loop2; + + if(debug['R'] && debug['v']) + dumpit("pass4", firstr); + + /* + * pass 4.5 + * move register pseudo-variables into regu. + */ + for(r = firstr; r != R; r = r->link) { + r->regu = (r->refbehind.b[0] | r->set.b[0]) & REGBITS; + + r->set.b[0] &= ~REGBITS; + r->use1.b[0] &= ~REGBITS; + r->use2.b[0] &= ~REGBITS; + r->refbehind.b[0] &= ~REGBITS; + r->refahead.b[0] &= ~REGBITS; + r->calbehind.b[0] &= ~REGBITS; + r->calahead.b[0] &= ~REGBITS; + r->regdiff.b[0] &= ~REGBITS; + r->act.b[0] &= ~REGBITS; + } + + /* + * pass 5 + * isolate regions + * calculate costs (paint1) + */ + r = firstr; + if(r) { + for(z=0; zrefahead.b[z] | r->calahead.b[z]) & + ~(externs.b[z] | params.b[z] | addrs.b[z] | consts.b[z]); + if(bany(&bit) && !r->refset) { + // should never happen - all variables are preset + if(debug['w']) + print("%L: used and not set: %Q\n", r->prog->lineno, bit); + r->refset = 1; + } + } + for(r = firstr; r != R; r = r->link) + r->act = zbits; + rgp = region; + nregion = 0; + for(r = firstr; r != R; r = r->link) { + for(z=0; zset.b[z] & + ~(r->refahead.b[z] | r->calahead.b[z] | addrs.b[z]); + if(bany(&bit) && !r->refset) { + if(debug['w']) + print("%L: set and not used: %Q\n", r->prog->lineno, bit); + r->refset = 1; + excise(r); + } + for(z=0; zact.b[z] | addrs.b[z]); + while(bany(&bit)) { + i = bnum(bit); + rgp->enter = r; + rgp->varno = i; + change = 0; + paint1(r, i); + bit.b[i/32] &= ~(1L<<(i%32)); + if(change <= 0) + continue; + rgp->cost = change; + nregion++; + if(nregion >= NRGN) { + if(debug['R'] && debug['v']) + print("too many regions\n"); + goto brk; + } + rgp++; + } + } +brk: + qsort(region, nregion, sizeof(region[0]), rcmp); + + /* + * pass 6 + * determine used registers (paint2) + * replace code (paint3) + */ + rgp = region; + for(i=0; ivarno); + vreg = paint2(rgp->enter, rgp->varno); + vreg = allreg(vreg, rgp); + if(rgp->regno != 0) + paint3(rgp->enter, rgp->varno, vreg, rgp->regno); + rgp++; + } + + if(debug['R'] && debug['v']) + dumpit("pass6", firstr); + + /* + * pass 7 + * peep-hole on basic block + */ + if(!debug['R'] || debug['P']) { + peep(); + } + + /* + * eliminate nops + * free aux structures + */ + for(p=firstp; p!=P; p=p->link) { + while(p->link != P && p->link->as == ANOP) + p->link = p->link->link; + if(p->to.type == D_BRANCH) + while(p->to.branch != P && p->to.branch->as == ANOP) + p->to.branch = p->to.branch->link; + } + + if(r1 != R) { + r1->link = freer; + freer = firstr; + } + + if(debug['R']) { + if(ostats.ncvtreg || + ostats.nspill || + ostats.nreload || + ostats.ndelmov || + ostats.nvar || + ostats.naddr || + 0) + print("\nstats\n"); + + if(ostats.ncvtreg) + print(" %4d cvtreg\n", ostats.ncvtreg); + if(ostats.nspill) + print(" %4d spill\n", ostats.nspill); + if(ostats.nreload) + print(" %4d reload\n", ostats.nreload); + if(ostats.ndelmov) + print(" %4d delmov\n", ostats.ndelmov); + if(ostats.nvar) + print(" %4d delmov\n", ostats.nvar); + if(ostats.naddr) + print(" %4d delmov\n", ostats.naddr); + + memset(&ostats, 0, sizeof(ostats)); + } +} + +/* + * add mov b,rn + * just after r + */ +void +addmove(Reg *r, int bn, int rn, int f) +{ + Prog *p, *p1; + Adr *a; + Var *v; + + p1 = mal(sizeof(*p1)); + clearp(p1); + p1->loc = 9999; + + p = r->prog; + p1->link = p->link; + p->link = p1; + p1->lineno = p->lineno; + + v = var + bn; + + a = &p1->to; + a->sym = v->sym; + a->offset = v->offset; + a->etype = v->etype; + a->type = v->name; + a->gotype = v->gotype; + a->node = v->node; + + // need to clean this up with wptr and + // some of the defaults + p1->as = AMOVL; + switch(v->etype) { + default: + fatal("unknown type\n"); + case TINT8: + case TUINT8: + case TBOOL: + p1->as = AMOVB; + break; + case TINT16: + case TUINT16: + p1->as = AMOVW; + break; + case TINT64: + case TUINT64: + case TUINTPTR: + case TPTR64: + p1->as = AMOVQ; + break; + case TFLOAT32: + p1->as = AMOVSS; + break; + case TFLOAT64: + p1->as = AMOVSD; + break; + case TINT: + case TUINT: + case TINT32: + case TUINT32: + case TPTR32: + break; + } + + p1->from.type = rn; + if(!f) { + p1->from = *a; + *a = zprog.from; + a->type = rn; + if(v->etype == TUINT8) + p1->as = AMOVB; + if(v->etype == TUINT16) + p1->as = AMOVW; + } + if(debug['R'] && debug['v']) + print("%P ===add=== %P\n", p, p1); + ostats.nspill++; +} + +uint32 +doregbits(int r) +{ + uint32 b; + + b = 0; + if(r >= D_INDIR) + r -= D_INDIR; + if(r >= D_AX && r <= D_R15) + b |= RtoB(r); + else + if(r >= D_AL && r <= D_R15B) + b |= RtoB(r-D_AL+D_AX); + else + if(r >= D_AH && r <= D_BH) + b |= RtoB(r-D_AH+D_AX); + else + if(r >= D_X0 && r <= D_X0+15) + b |= FtoB(r); + return b; +} + +static int +overlap(int32 o1, int w1, int32 o2, int w2) +{ + int32 t1, t2; + + t1 = o1+w1; + t2 = o2+w2; + + if(!(t1 > o2 && t2 > o1)) + return 0; + + return 1; +} + +Bits +mkvar(Reg *r, Adr *a) +{ + Var *v; + int i, t, n, et, z, w, flag; + uint32 regu; + int32 o; + Bits bit; + Sym *s; + + /* + * mark registers used + */ + t = a->type; + if(t == D_NONE) + goto none; + + if(r != R) + r->use1.b[0] |= doregbits(a->index); + + switch(t) { + default: + regu = doregbits(t); + if(regu == 0) + goto none; + bit = zbits; + bit.b[0] = regu; + return bit; + + case D_ADDR: + a->type = a->index; + bit = mkvar(r, a); + setaddrs(bit); + a->type = t; + ostats.naddr++; + goto none; + + case D_EXTERN: + case D_STATIC: + case D_PARAM: + case D_AUTO: + n = t; + break; + } + s = a->sym; + if(s == S) + goto none; + if(s->name[0] == '.') + goto none; + et = a->etype; + o = a->offset; + w = a->width; + + flag = 0; + for(i=0; isym == s && v->name == n) { + if(v->offset == o) + if(v->etype == et) + if(v->width == w) + return blsh(i); + + // if they overlaps, disable both + if(overlap(v->offset, v->width, o, w)) { +// print("disable overlap %s %d %d %d %d, %E != %E\n", s->name, v->offset, v->width, o, w, v->etype, et); + v->addr = 1; + flag = 1; + } + } + } + if(a->pun) { +// print("disable pun %s\n", s->name); + flag = 1; + + } + switch(et) { + case 0: + case TFUNC: + goto none; + } + + if(nvar >= NVAR) { + if(debug['w'] > 1 && s) + fatal("variable not optimized: %D", a); + goto none; + } + + i = nvar; + nvar++; + v = var+i; + v->sym = s; + v->offset = o; + v->name = n; + v->gotype = a->gotype; + v->etype = et; + v->width = w; + v->addr = flag; // funny punning + v->node = a->node; + + if(debug['R']) + print("bit=%2d et=%2d w=%d %S %D\n", i, et, w, s, a); + ostats.nvar++; + + bit = blsh(i); + if(n == D_EXTERN || n == D_STATIC) + for(z=0; zp1) { + for(z=0; zrefahead.b[z]; + if(ref.b[z] != r1->refahead.b[z]) { + r1->refahead.b[z] = ref.b[z]; + change++; + } + cal.b[z] |= r1->calahead.b[z]; + if(cal.b[z] != r1->calahead.b[z]) { + r1->calahead.b[z] = cal.b[z]; + change++; + } + } + switch(r1->prog->as) { + case ACALL: + if(noreturn(r1->prog)) + break; + for(z=0; zset.b[z]) | + r1->use1.b[z] | r1->use2.b[z]; + cal.b[z] &= ~(r1->set.b[z] | r1->use1.b[z] | r1->use2.b[z]); + r1->refbehind.b[z] = ref.b[z]; + r1->calbehind.b[z] = cal.b[z]; + } + if(r1->active) + break; + r1->active = 1; + } + for(; r != r1; r = r->p1) + for(r2 = r->p2; r2 != R; r2 = r2->p2link) + prop(r2, r->refbehind, r->calbehind); +} + +/* + * find looping structure + * + * 1) find reverse postordering + * 2) find approximate dominators, + * the actual dominators if the flow graph is reducible + * otherwise, dominators plus some other non-dominators. + * See Matthew S. Hecht and Jeffrey D. Ullman, + * "Analysis of a Simple Algorithm for Global Data Flow Problems", + * Conf. Record of ACM Symp. on Principles of Prog. Langs, Boston, Massachusetts, + * Oct. 1-3, 1973, pp. 207-217. + * 3) find all nodes with a predecessor dominated by the current node. + * such a node is a loop head. + * recursively, all preds with a greater rpo number are in the loop + */ +int32 +postorder(Reg *r, Reg **rpo2r, int32 n) +{ + Reg *r1; + + r->rpo = 1; + r1 = r->s1; + if(r1 && !r1->rpo) + n = postorder(r1, rpo2r, n); + r1 = r->s2; + if(r1 && !r1->rpo) + n = postorder(r1, rpo2r, n); + rpo2r[n] = r; + n++; + return n; +} + +int32 +rpolca(int32 *idom, int32 rpo1, int32 rpo2) +{ + int32 t; + + if(rpo1 == -1) + return rpo2; + while(rpo1 != rpo2){ + if(rpo1 > rpo2){ + t = rpo2; + rpo2 = rpo1; + rpo1 = t; + } + while(rpo1 < rpo2){ + t = idom[rpo2]; + if(t >= rpo2) + fatal("bad idom"); + rpo2 = t; + } + } + return rpo1; +} + +int +doms(int32 *idom, int32 r, int32 s) +{ + while(s > r) + s = idom[s]; + return s == r; +} + +int +loophead(int32 *idom, Reg *r) +{ + int32 src; + + src = r->rpo; + if(r->p1 != R && doms(idom, src, r->p1->rpo)) + return 1; + for(r = r->p2; r != R; r = r->p2link) + if(doms(idom, src, r->rpo)) + return 1; + return 0; +} + +void +loopmark(Reg **rpo2r, int32 head, Reg *r) +{ + if(r->rpo < head || r->active == head) + return; + r->active = head; + r->loop += LOOP; + if(r->p1 != R) + loopmark(rpo2r, head, r->p1); + for(r = r->p2; r != R; r = r->p2link) + loopmark(rpo2r, head, r); +} + +void +loopit(Reg *r, int32 nr) +{ + Reg *r1; + int32 i, d, me; + + if(nr > maxnr) { + rpo2r = mal(nr * sizeof(Reg*)); + idom = mal(nr * sizeof(int32)); + maxnr = nr; + } + + d = postorder(r, rpo2r, 0); + if(d > nr) + fatal("too many reg nodes %d %d", d, nr); + nr = d; + for(i = 0; i < nr / 2; i++) { + r1 = rpo2r[i]; + rpo2r[i] = rpo2r[nr - 1 - i]; + rpo2r[nr - 1 - i] = r1; + } + for(i = 0; i < nr; i++) + rpo2r[i]->rpo = i; + + idom[0] = 0; + for(i = 0; i < nr; i++) { + r1 = rpo2r[i]; + me = r1->rpo; + d = -1; + if(r1->p1 != R && r1->p1->rpo < me) + d = r1->p1->rpo; + for(r1 = r1->p2; r1 != nil; r1 = r1->p2link) + if(r1->rpo < me) + d = rpolca(idom, d, r1->rpo); + idom[i] = d; + } + + for(i = 0; i < nr; i++) { + r1 = rpo2r[i]; + r1->loop++; + if(r1->p2 != R && loophead(idom, r1)) + loopmark(rpo2r, i, r1); + } +} + +void +synch(Reg *r, Bits dif) +{ + Reg *r1; + int z; + + for(r1 = r; r1 != R; r1 = r1->s1) { + for(z=0; zrefbehind.b[z] & r1->refahead.b[z])) | + r1->set.b[z] | r1->regdiff.b[z]; + if(dif.b[z] != r1->regdiff.b[z]) { + r1->regdiff.b[z] = dif.b[z]; + change++; + } + } + if(r1->active) + break; + r1->active = 1; + for(z=0; zcalbehind.b[z] & r1->calahead.b[z]); + if(r1->s2 != R) + synch(r1->s2, dif); + } +} + +uint32 +allreg(uint32 b, Rgn *r) +{ + Var *v; + int i; + + v = var + r->varno; + r->regno = 0; + switch(v->etype) { + + default: + fatal("unknown etype %d/%E", bitno(b), v->etype); + break; + + case TINT8: + case TUINT8: + case TINT16: + case TUINT16: + case TINT32: + case TUINT32: + case TINT64: + case TUINT64: + case TINT: + case TUINT: + case TUINTPTR: + case TBOOL: + case TPTR32: + case TPTR64: + i = BtoR(~b); + if(i && r->cost > 0) { + r->regno = i; + return RtoB(i); + } + break; + + case TFLOAT32: + case TFLOAT64: + i = BtoF(~b); + if(i && r->cost > 0) { + r->regno = i; + return FtoB(i); + } + break; + } + return 0; +} + +void +paint1(Reg *r, int bn) +{ + Reg *r1; + int z; + uint32 bb; + + z = bn/32; + bb = 1L<<(bn%32); + if(r->act.b[z] & bb) + return; + for(;;) { + if(!(r->refbehind.b[z] & bb)) + break; + r1 = r->p1; + if(r1 == R) + break; + if(!(r1->refahead.b[z] & bb)) + break; + if(r1->act.b[z] & bb) + break; + r = r1; + } + + if(LOAD(r) & ~(r->set.b[z]&~(r->use1.b[z]|r->use2.b[z])) & bb) { + change -= CLOAD * r->loop; + } + for(;;) { + r->act.b[z] |= bb; + + if(r->use1.b[z] & bb) { + change += CREF * r->loop; + } + + if((r->use2.b[z]|r->set.b[z]) & bb) { + change += CREF * r->loop; + } + + if(STORE(r) & r->regdiff.b[z] & bb) { + change -= CLOAD * r->loop; + } + + if(r->refbehind.b[z] & bb) + for(r1 = r->p2; r1 != R; r1 = r1->p2link) + if(r1->refahead.b[z] & bb) + paint1(r1, bn); + + if(!(r->refahead.b[z] & bb)) + break; + r1 = r->s2; + if(r1 != R) + if(r1->refbehind.b[z] & bb) + paint1(r1, bn); + r = r->s1; + if(r == R) + break; + if(r->act.b[z] & bb) + break; + if(!(r->refbehind.b[z] & bb)) + break; + } +} + +uint32 +regset(Reg *r, uint32 bb) +{ + uint32 b, set; + Adr v; + int c; + + set = 0; + v = zprog.from; + while(b = bb & ~(bb-1)) { + v.type = b & 0xFFFF? BtoR(b): BtoF(b); + if(v.type == 0) + fatal("zero v.type for %#ux", b); + c = copyu(r->prog, &v, A); + if(c == 3) + set |= b; + bb &= ~b; + } + return set; +} + +uint32 +reguse(Reg *r, uint32 bb) +{ + uint32 b, set; + Adr v; + int c; + + set = 0; + v = zprog.from; + while(b = bb & ~(bb-1)) { + v.type = b & 0xFFFF? BtoR(b): BtoF(b); + c = copyu(r->prog, &v, A); + if(c == 1 || c == 2 || c == 4) + set |= b; + bb &= ~b; + } + return set; +} + +uint32 +paint2(Reg *r, int bn) +{ + Reg *r1; + int z; + uint32 bb, vreg, x; + + z = bn/32; + bb = 1L << (bn%32); + vreg = regbits; + if(!(r->act.b[z] & bb)) + return vreg; + for(;;) { + if(!(r->refbehind.b[z] & bb)) + break; + r1 = r->p1; + if(r1 == R) + break; + if(!(r1->refahead.b[z] & bb)) + break; + if(!(r1->act.b[z] & bb)) + break; + r = r1; + } + for(;;) { + r->act.b[z] &= ~bb; + + vreg |= r->regu; + + if(r->refbehind.b[z] & bb) + for(r1 = r->p2; r1 != R; r1 = r1->p2link) + if(r1->refahead.b[z] & bb) + vreg |= paint2(r1, bn); + + if(!(r->refahead.b[z] & bb)) + break; + r1 = r->s2; + if(r1 != R) + if(r1->refbehind.b[z] & bb) + vreg |= paint2(r1, bn); + r = r->s1; + if(r == R) + break; + if(!(r->act.b[z] & bb)) + break; + if(!(r->refbehind.b[z] & bb)) + break; + } + + bb = vreg; + for(; r; r=r->s1) { + x = r->regu & ~bb; + if(x) { + vreg |= reguse(r, x); + bb |= regset(r, x); + } + } + return vreg; +} + +void +paint3(Reg *r, int bn, int32 rb, int rn) +{ + Reg *r1; + Prog *p; + int z; + uint32 bb; + + z = bn/32; + bb = 1L << (bn%32); + if(r->act.b[z] & bb) + return; + for(;;) { + if(!(r->refbehind.b[z] & bb)) + break; + r1 = r->p1; + if(r1 == R) + break; + if(!(r1->refahead.b[z] & bb)) + break; + if(r1->act.b[z] & bb) + break; + r = r1; + } + + if(LOAD(r) & ~(r->set.b[z] & ~(r->use1.b[z]|r->use2.b[z])) & bb) + addmove(r, bn, rn, 0); + for(;;) { + r->act.b[z] |= bb; + p = r->prog; + + if(r->use1.b[z] & bb) { + if(debug['R'] && debug['v']) + print("%P", p); + addreg(&p->from, rn); + if(debug['R'] && debug['v']) + print(" ===change== %P\n", p); + } + if((r->use2.b[z]|r->set.b[z]) & bb) { + if(debug['R'] && debug['v']) + print("%P", p); + addreg(&p->to, rn); + if(debug['R'] && debug['v']) + print(" ===change== %P\n", p); + } + + if(STORE(r) & r->regdiff.b[z] & bb) + addmove(r, bn, rn, 1); + r->regu |= rb; + + if(r->refbehind.b[z] & bb) + for(r1 = r->p2; r1 != R; r1 = r1->p2link) + if(r1->refahead.b[z] & bb) + paint3(r1, bn, rb, rn); + + if(!(r->refahead.b[z] & bb)) + break; + r1 = r->s2; + if(r1 != R) + if(r1->refbehind.b[z] & bb) + paint3(r1, bn, rb, rn); + r = r->s1; + if(r == R) + break; + if(r->act.b[z] & bb) + break; + if(!(r->refbehind.b[z] & bb)) + break; + } +} + +void +addreg(Adr *a, int rn) +{ + + a->sym = 0; + a->offset = 0; + a->type = rn; + + ostats.ncvtreg++; +} + +int32 +RtoB(int r) +{ + + if(r < D_AX || r > D_R15) + return 0; + return 1L << (r-D_AX); +} + +int +BtoR(int32 b) +{ + b &= 0x3fffL; // no R14 or R15 + if(b == 0) + return 0; + return bitno(b) + D_AX; +} + +/* + * bit reg + * 16 X5 (FREGMIN) + * ... + * 26 X15 (FREGEXT) + */ +int32 +FtoB(int f) +{ + if(f < FREGMIN || f > FREGEXT) + return 0; + return 1L << (f - FREGMIN + 16); +} + +int +BtoF(int32 b) +{ + + b &= 0xFF0000L; + if(b == 0) + return 0; + return bitno(b) - 16 + FREGMIN; +} + +void +dumpone(Reg *r) +{ + int z; + Bits bit; + + print("%d:%P", r->loop, r->prog); + for(z=0; zset.b[z] | + r->use1.b[z] | + r->use2.b[z] | + r->refbehind.b[z] | + r->refahead.b[z] | + r->calbehind.b[z] | + r->calahead.b[z] | + r->regdiff.b[z] | + r->act.b[z] | + 0; + if(bany(&bit)) { + print("\t"); + if(bany(&r->set)) + print(" s:%Q", r->set); + if(bany(&r->use1)) + print(" u1:%Q", r->use1); + if(bany(&r->use2)) + print(" u2:%Q", r->use2); + if(bany(&r->refbehind)) + print(" rb:%Q ", r->refbehind); + if(bany(&r->refahead)) + print(" ra:%Q ", r->refahead); + if(bany(&r->calbehind)) + print("cb:%Q ", r->calbehind); + if(bany(&r->calahead)) + print(" ca:%Q ", r->calahead); + if(bany(&r->regdiff)) + print(" d:%Q ", r->regdiff); + if(bany(&r->act)) + print(" a:%Q ", r->act); + } + print("\n"); +} + +void +dumpit(char *str, Reg *r0) +{ + Reg *r, *r1; + + print("\n%s\n", str); + for(r = r0; r != R; r = r->link) { + dumpone(r); + r1 = r->p2; + if(r1 != R) { + print(" pred:"); + for(; r1 != R; r1 = r1->p2link) + print(" %.4ud", r1->prog->loc); + print("\n"); + } +// r1 = r->s1; +// if(r1 != R) { +// print(" succ:"); +// for(; r1 != R; r1 = r1->s1) +// print(" %.4ud", r1->prog->loc); +// print("\n"); +// } + } +} + +static Sym* symlist[10]; + +int +noreturn(Prog *p) +{ + Sym *s; + int i; + + if(symlist[0] == S) { + symlist[0] = pkglookup("panicindex", runtimepkg); + symlist[1] = pkglookup("panicslice", runtimepkg); + symlist[2] = pkglookup("throwinit", runtimepkg); + symlist[3] = pkglookup("panic", runtimepkg); + symlist[4] = pkglookup("panicwrap", runtimepkg); + } + + s = p->to.sym; + if(s == S) + return 0; + for(i=0; symlist[i]!=S; i++) + if(s == symlist[i]) + return 1; + return 0; +} diff --git a/src/cmd/6l/6.out.h b/src/cmd/6l/6.out.h new file mode 100644 index 000000000..262da02ab --- /dev/null +++ b/src/cmd/6l/6.out.h @@ -0,0 +1,866 @@ +// Inferno utils/6c/6.out.h +// http://code.google.com/p/inferno-os/source/browse/utils/6c/6.out.h +// +// Copyright © 1994-1999 Lucent Technologies Inc. All rights reserved. +// Portions Copyright © 1995-1997 C H Forsyth (forsyth@terzarima.net) +// Portions Copyright © 1997-1999 Vita Nuova Limited +// Portions Copyright © 2000-2007 Vita Nuova Holdings Limited (www.vitanuova.com) +// Portions Copyright © 2004,2006 Bruce Ellis +// Portions Copyright © 2005-2007 C H Forsyth (forsyth@terzarima.net) +// Revisions Copyright © 2000-2007 Lucent Technologies Inc. and others +// Portions Copyright © 2009 The Go Authors. All rights reserved. +// +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to deal +// in the Software without restriction, including without limitation the rights +// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +// copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +// THE SOFTWARE. + +#define NSYM 50 +#define NSNAME 8 +#define NOPROF (1<<0) +#define DUPOK (1<<1) +#define NOSPLIT (1<<2) +#define RODATA (1<<3) + +/* + * amd64 + */ + +enum as +{ + AXXX, + AAAA, + AAAD, + AAAM, + AAAS, + AADCB, + AADCL, + AADCW, + AADDB, + AADDL, + AADDW, + AADJSP, + AANDB, + AANDL, + AANDW, + AARPL, + ABOUNDL, + ABOUNDW, + ABSFL, + ABSFW, + ABSRL, + ABSRW, + ABTL, + ABTW, + ABTCL, + ABTCW, + ABTRL, + ABTRW, + ABTSL, + ABTSW, + ABYTE, + ACALL, + ACLC, + ACLD, + ACLI, + ACLTS, + ACMC, + ACMPB, + ACMPL, + ACMPW, + ACMPSB, + ACMPSL, + ACMPSW, + ADAA, + ADAS, + ADATA, + ADECB, + ADECL, + ADECQ, + ADECW, + ADIVB, + ADIVL, + ADIVW, + AENTER, + AGLOBL, + AGOK, + AHISTORY, + AHLT, + AIDIVB, + AIDIVL, + AIDIVW, + AIMULB, + AIMULL, + AIMULW, + AINB, + AINL, + AINW, + AINCB, + AINCL, + AINCQ, + AINCW, + AINSB, + AINSL, + AINSW, + AINT, + AINTO, + AIRETL, + AIRETW, + AJCC, + AJCS, + AJCXZ, + AJEQ, + AJGE, + AJGT, + AJHI, + AJLE, + AJLS, + AJLT, + AJMI, + AJMP, + AJNE, + AJOC, + AJOS, + AJPC, + AJPL, + AJPS, + ALAHF, + ALARL, + ALARW, + ALEAL, + ALEAW, + ALEAVEL, + ALEAVEW, + ALOCK, + ALODSB, + ALODSL, + ALODSW, + ALONG, + ALOOP, + ALOOPEQ, + ALOOPNE, + ALSLL, + ALSLW, + AMOVB, + AMOVL, + AMOVW, + AMOVBLSX, + AMOVBLZX, + AMOVBQSX, + AMOVBQZX, + AMOVBWSX, + AMOVBWZX, + AMOVWLSX, + AMOVWLZX, + AMOVWQSX, + AMOVWQZX, + AMOVSB, + AMOVSL, + AMOVSW, + AMULB, + AMULL, + AMULW, + ANAME, + ANEGB, + ANEGL, + ANEGW, + ANOP, + ANOTB, + ANOTL, + ANOTW, + AORB, + AORL, + AORW, + AOUTB, + AOUTL, + AOUTW, + AOUTSB, + AOUTSL, + AOUTSW, + APAUSE, + APOPAL, + APOPAW, + APOPFL, + APOPFW, + APOPL, + APOPW, + APUSHAL, + APUSHAW, + APUSHFL, + APUSHFW, + APUSHL, + APUSHW, + ARCLB, + ARCLL, + ARCLW, + ARCRB, + ARCRL, + ARCRW, + AREP, + AREPN, + ARET, + AROLB, + AROLL, + AROLW, + ARORB, + ARORL, + ARORW, + ASAHF, + ASALB, + ASALL, + ASALW, + ASARB, + ASARL, + ASARW, + ASBBB, + ASBBL, + ASBBW, + ASCASB, + ASCASL, + ASCASW, + ASETCC, + ASETCS, + ASETEQ, + ASETGE, + ASETGT, + ASETHI, + ASETLE, + ASETLS, + ASETLT, + ASETMI, + ASETNE, + ASETOC, + ASETOS, + ASETPC, + ASETPL, + ASETPS, + ACDQ, + ACWD, + ASHLB, + ASHLL, + ASHLW, + ASHRB, + ASHRL, + ASHRW, + ASTC, + ASTD, + ASTI, + ASTOSB, + ASTOSL, + ASTOSW, + ASUBB, + ASUBL, + ASUBW, + ASYSCALL, + ATESTB, + ATESTL, + ATESTW, + ATEXT, + AVERR, + AVERW, + AWAIT, + AWORD, + AXCHGB, + AXCHGL, + AXCHGW, + AXLAT, + AXORB, + AXORL, + AXORW, + + AFMOVB, + AFMOVBP, + AFMOVD, + AFMOVDP, + AFMOVF, + AFMOVFP, + AFMOVL, + AFMOVLP, + AFMOVV, + AFMOVVP, + AFMOVW, + AFMOVWP, + AFMOVX, + AFMOVXP, + + AFCOMB, + AFCOMBP, + AFCOMD, + AFCOMDP, + AFCOMDPP, + AFCOMF, + AFCOMFP, + AFCOML, + AFCOMLP, + AFCOMW, + AFCOMWP, + AFUCOM, + AFUCOMP, + AFUCOMPP, + + AFADDDP, + AFADDW, + AFADDL, + AFADDF, + AFADDD, + + AFMULDP, + AFMULW, + AFMULL, + AFMULF, + AFMULD, + + AFSUBDP, + AFSUBW, + AFSUBL, + AFSUBF, + AFSUBD, + + AFSUBRDP, + AFSUBRW, + AFSUBRL, + AFSUBRF, + AFSUBRD, + + AFDIVDP, + AFDIVW, + AFDIVL, + AFDIVF, + AFDIVD, + + AFDIVRDP, + AFDIVRW, + AFDIVRL, + AFDIVRF, + AFDIVRD, + + AFXCHD, + AFFREE, + + AFLDCW, + AFLDENV, + AFRSTOR, + AFSAVE, + AFSTCW, + AFSTENV, + AFSTSW, + + AF2XM1, + AFABS, + AFCHS, + AFCLEX, + AFCOS, + AFDECSTP, + AFINCSTP, + AFINIT, + AFLD1, + AFLDL2E, + AFLDL2T, + AFLDLG2, + AFLDLN2, + AFLDPI, + AFLDZ, + AFNOP, + AFPATAN, + AFPREM, + AFPREM1, + AFPTAN, + AFRNDINT, + AFSCALE, + AFSIN, + AFSINCOS, + AFSQRT, + AFTST, + AFXAM, + AFXTRACT, + AFYL2X, + AFYL2XP1, + + AEND, + + ADYNT_, + AINIT_, + + ASIGNAME, + + /* extra 32-bit operations */ + ACMPXCHGB, + ACMPXCHGL, + ACMPXCHGW, + ACMPXCHG8B, + ACPUID, + AINVD, + AINVLPG, + ALFENCE, + AMFENCE, + AMOVNTIL, + ARDMSR, + ARDPMC, + ARDTSC, + ARSM, + ASFENCE, + ASYSRET, + AWBINVD, + AWRMSR, + AXADDB, + AXADDL, + AXADDW, + + /* conditional move */ + ACMOVLCC, + ACMOVLCS, + ACMOVLEQ, + ACMOVLGE, + ACMOVLGT, + ACMOVLHI, + ACMOVLLE, + ACMOVLLS, + ACMOVLLT, + ACMOVLMI, + ACMOVLNE, + ACMOVLOC, + ACMOVLOS, + ACMOVLPC, + ACMOVLPL, + ACMOVLPS, + ACMOVQCC, + ACMOVQCS, + ACMOVQEQ, + ACMOVQGE, + ACMOVQGT, + ACMOVQHI, + ACMOVQLE, + ACMOVQLS, + ACMOVQLT, + ACMOVQMI, + ACMOVQNE, + ACMOVQOC, + ACMOVQOS, + ACMOVQPC, + ACMOVQPL, + ACMOVQPS, + ACMOVWCC, + ACMOVWCS, + ACMOVWEQ, + ACMOVWGE, + ACMOVWGT, + ACMOVWHI, + ACMOVWLE, + ACMOVWLS, + ACMOVWLT, + ACMOVWMI, + ACMOVWNE, + ACMOVWOC, + ACMOVWOS, + ACMOVWPC, + ACMOVWPL, + ACMOVWPS, + + /* 64-bit */ + AADCQ, + AADDQ, + AANDQ, + ABSFQ, + ABSRQ, + ABTCQ, + ABTQ, + ABTRQ, + ABTSQ, + ACMPQ, + ACMPSQ, + ACMPXCHGQ, + ACQO, + ADIVQ, + AIDIVQ, + AIMULQ, + AIRETQ, + ALEAQ, + ALEAVEQ, + ALODSQ, + AMOVQ, + AMOVLQSX, + AMOVLQZX, + AMOVNTIQ, + AMOVSQ, + AMULQ, + ANEGQ, + ANOTQ, + AORQ, + APOPFQ, + APOPQ, + APUSHFQ, + APUSHQ, + ARCLQ, + ARCRQ, + AROLQ, + ARORQ, + AQUAD, + ASALQ, + ASARQ, + ASBBQ, + ASCASQ, + ASHLQ, + ASHRQ, + ASTOSQ, + ASUBQ, + ATESTQ, + AXADDQ, + AXCHGQ, + AXORQ, + + /* media */ + AADDPD, + AADDPS, + AADDSD, + AADDSS, + AANDNPD, + AANDNPS, + AANDPD, + AANDPS, + ACMPPD, + ACMPPS, + ACMPSD, + ACMPSS, + ACOMISD, + ACOMISS, + ACVTPD2PL, + ACVTPD2PS, + ACVTPL2PD, + ACVTPL2PS, + ACVTPS2PD, + ACVTPS2PL, + ACVTSD2SL, + ACVTSD2SQ, + ACVTSD2SS, + ACVTSL2SD, + ACVTSL2SS, + ACVTSQ2SD, + ACVTSQ2SS, + ACVTSS2SD, + ACVTSS2SL, + ACVTSS2SQ, + ACVTTPD2PL, + ACVTTPS2PL, + ACVTTSD2SL, + ACVTTSD2SQ, + ACVTTSS2SL, + ACVTTSS2SQ, + ADIVPD, + ADIVPS, + ADIVSD, + ADIVSS, + AEMMS, + AFXRSTOR, + AFXRSTOR64, + AFXSAVE, + AFXSAVE64, + ALDMXCSR, + AMASKMOVOU, + AMASKMOVQ, + AMAXPD, + AMAXPS, + AMAXSD, + AMAXSS, + AMINPD, + AMINPS, + AMINSD, + AMINSS, + AMOVAPD, + AMOVAPS, + AMOVOU, + AMOVHLPS, + AMOVHPD, + AMOVHPS, + AMOVLHPS, + AMOVLPD, + AMOVLPS, + AMOVMSKPD, + AMOVMSKPS, + AMOVNTO, + AMOVNTPD, + AMOVNTPS, + AMOVNTQ, + AMOVO, + AMOVQOZX, + AMOVSD, + AMOVSS, + AMOVUPD, + AMOVUPS, + AMULPD, + AMULPS, + AMULSD, + AMULSS, + AORPD, + AORPS, + APACKSSLW, + APACKSSWB, + APACKUSWB, + APADDB, + APADDL, + APADDQ, + APADDSB, + APADDSW, + APADDUSB, + APADDUSW, + APADDW, + APANDB, + APANDL, + APANDSB, + APANDSW, + APANDUSB, + APANDUSW, + APANDW, + APAND, + APANDN, + APAVGB, + APAVGW, + APCMPEQB, + APCMPEQL, + APCMPEQW, + APCMPGTB, + APCMPGTL, + APCMPGTW, + APEXTRW, + APFACC, + APFADD, + APFCMPEQ, + APFCMPGE, + APFCMPGT, + APFMAX, + APFMIN, + APFMUL, + APFNACC, + APFPNACC, + APFRCP, + APFRCPIT1, + APFRCPI2T, + APFRSQIT1, + APFRSQRT, + APFSUB, + APFSUBR, + APINSRW, + APMADDWL, + APMAXSW, + APMAXUB, + APMINSW, + APMINUB, + APMOVMSKB, + APMULHRW, + APMULHUW, + APMULHW, + APMULLW, + APMULULQ, + APOR, + APSADBW, + APSHUFHW, + APSHUFL, + APSHUFLW, + APSHUFW, + APSLLO, + APSLLL, + APSLLQ, + APSLLW, + APSRAL, + APSRAW, + APSRLO, + APSRLL, + APSRLQ, + APSRLW, + APSUBB, + APSUBL, + APSUBQ, + APSUBSB, + APSUBSW, + APSUBUSB, + APSUBUSW, + APSUBW, + APSWAPL, + APUNPCKHBW, + APUNPCKHLQ, + APUNPCKHQDQ, + APUNPCKHWL, + APUNPCKLBW, + APUNPCKLLQ, + APUNPCKLQDQ, + APUNPCKLWL, + APXOR, + ARCPPS, + ARCPSS, + ARSQRTPS, + ARSQRTSS, + ASHUFPD, + ASHUFPS, + ASQRTPD, + ASQRTPS, + ASQRTSD, + ASQRTSS, + ASTMXCSR, + ASUBPD, + ASUBPS, + ASUBSD, + ASUBSS, + AUCOMISD, + AUCOMISS, + AUNPCKHPD, + AUNPCKHPS, + AUNPCKLPD, + AUNPCKLPS, + AXORPD, + AXORPS, + + APF2IW, + APF2IL, + API2FW, + API2FL, + ARETFW, + ARETFL, + ARETFQ, + ASWAPGS, + + AMODE, + ACRC32B, + ACRC32Q, + + ALAST +}; + +enum +{ + + D_AL = 0, + D_CL, + D_DL, + D_BL, + D_SPB, + D_BPB, + D_SIB, + D_DIB, + D_R8B, + D_R9B, + D_R10B, + D_R11B, + D_R12B, + D_R13B, + D_R14B, + D_R15B, + + D_AX = 16, + D_CX, + D_DX, + D_BX, + D_SP, + D_BP, + D_SI, + D_DI, + D_R8, + D_R9, + D_R10, + D_R11, + D_R12, + D_R13, + D_R14, + D_R15, + + D_AH = 32, + D_CH, + D_DH, + D_BH, + + D_F0 = 36, + + D_M0 = 44, + + D_X0 = 52, + D_X1, + D_X2, + D_X3, + D_X4, + D_X5, + D_X6, + D_X7, + + D_CS = 68, + D_SS, + D_DS, + D_ES, + D_FS, + D_GS, + + D_GDTR, /* global descriptor table register */ + D_IDTR, /* interrupt descriptor table register */ + D_LDTR, /* local descriptor table register */ + D_MSW, /* machine status word */ + D_TASK, /* task register */ + + D_CR = 79, + D_DR = 95, + D_TR = 103, + + D_NONE = 111, + + D_BRANCH = 112, + D_EXTERN = 113, + D_STATIC = 114, + D_AUTO = 115, + D_PARAM = 116, + D_CONST = 117, + D_FCONST = 118, + D_SCONST = 119, + D_ADDR = 120, + + D_FILE, + D_FILE1, + + D_INDIR, /* additive */ + + D_SIZE = D_INDIR + D_INDIR, /* 6l internal */ + D_PCREL, + + T_TYPE = 1<<0, + T_INDEX = 1<<1, + T_OFFSET = 1<<2, + T_FCONST = 1<<3, + T_SYM = 1<<4, + T_SCONST = 1<<5, + T_64 = 1<<6, + T_GOTYPE = 1<<7, + + REGARG = -1, + REGRET = D_AX, + FREGRET = D_X0, + REGSP = D_SP, + REGTMP = D_DI, + REGEXT = D_R15, /* compiler allocates external registers R15 down */ + FREGMIN = D_X0+5, /* first register variable */ + FREGEXT = D_X0+15 /* first external register */ +}; + +/* + * this is the ranlib header + */ +#define SYMDEF "__.SYMDEF" + +/* + * this is the simulated IEEE floating point + */ +typedef struct ieee Ieee; +struct ieee +{ + int32 l; /* contains ls-man 0xffffffff */ + int32 h; /* contains sign 0x80000000 + exp 0x7ff00000 + ms-man 0x000fffff */ +}; diff --git a/src/cmd/6l/Makefile b/src/cmd/6l/Makefile new file mode 100644 index 000000000..8ed3e1411 --- /dev/null +++ b/src/cmd/6l/Makefile @@ -0,0 +1,48 @@ +# Copyright 2009 The Go Authors. All rights reserved. +# Use of this source code is governed by a BSD-style +# license that can be found in the LICENSE file. + +include ../../Make.inc +O:=$(HOST_O) + +TARG=6l + +OFILES=\ + asm.$O\ + data.$O\ + dwarf.$O\ + elf.$O\ + enam.$O\ + go.$O\ + ldelf.$O\ + ldmacho.$O\ + ldpe.$O\ + lib.$O\ + list.$O\ + macho.$O\ + obj.$O\ + optab.$O\ + pass.$O\ + pe.$O\ + prof.$O\ + span.$O\ + symtab.$O\ + +HFILES=\ + l.h\ + 6.out.h\ + ../ld/lib.h\ + ../ld/elf.h\ + ../ld/macho.h\ + ../ld/dwarf.h\ + ../ld/pe.h\ + +include ../../Make.ccmd + +enam.c: 6.out.h + sh mkenam + +CLEANFILES+=enam.c + +%.$O: ../ld/%.c + $(HOST_CC) $(HOST_CFLAGS) -c -I. ../ld/$*.c diff --git a/src/cmd/6l/asm.c b/src/cmd/6l/asm.c new file mode 100644 index 000000000..3a8223e65 --- /dev/null +++ b/src/cmd/6l/asm.c @@ -0,0 +1,1179 @@ +// Inferno utils/6l/asm.c +// http://code.google.com/p/inferno-os/source/browse/utils/6l/asm.c +// +// Copyright © 1994-1999 Lucent Technologies Inc. All rights reserved. +// Portions Copyright © 1995-1997 C H Forsyth (forsyth@terzarima.net) +// Portions Copyright © 1997-1999 Vita Nuova Limited +// Portions Copyright © 2000-2007 Vita Nuova Holdings Limited (www.vitanuova.com) +// Portions Copyright © 2004,2006 Bruce Ellis +// Portions Copyright © 2005-2007 C H Forsyth (forsyth@terzarima.net) +// Revisions Copyright © 2000-2007 Lucent Technologies Inc. and others +// Portions Copyright © 2009 The Go Authors. All rights reserved. +// +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to deal +// in the Software without restriction, including without limitation the rights +// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +// copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +// THE SOFTWARE. + +// Writing object files. + +#include "l.h" +#include "../ld/lib.h" +#include "../ld/elf.h" +#include "../ld/dwarf.h" +#include "../ld/macho.h" +#include "../ld/pe.h" + +#define Dbufslop 100 + +#define PADDR(a) ((uint32)(a) & ~0x80000000) + +char linuxdynld[] = "/lib64/ld-linux-x86-64.so.2"; +char freebsddynld[] = "/libexec/ld-elf.so.1"; +char openbsddynld[] = "/usr/libexec/ld.so"; + +char zeroes[32]; + +vlong +entryvalue(void) +{ + char *a; + Sym *s; + + a = INITENTRY; + if(*a >= '0' && *a <= '9') + return atolwhex(a); + s = lookup(a, 0); + if(s->type == 0) + return INITTEXT; + if(s->type != STEXT) + diag("entry not text: %s", s->name); + return s->value; +} + +vlong +datoff(vlong addr) +{ + if(addr >= segdata.vaddr) + return addr - segdata.vaddr + segdata.fileoff; + if(addr >= segtext.vaddr) + return addr - segtext.vaddr + segtext.fileoff; + diag("datoff %#llx", addr); + return 0; +} + +enum { + ElfStrEmpty, + ElfStrInterp, + ElfStrHash, + ElfStrGot, + ElfStrGotPlt, + ElfStrDynamic, + ElfStrDynsym, + ElfStrDynstr, + ElfStrRela, + ElfStrText, + ElfStrData, + ElfStrBss, + ElfStrShstrtab, + ElfStrSymtab, + ElfStrStrtab, + ElfStrRelaPlt, + ElfStrPlt, + ElfStrGnuVersion, + ElfStrGnuVersionR, + NElfStr +}; + +vlong elfstr[NElfStr]; + +static int +needlib(char *name) +{ + char *p; + Sym *s; + + if(*name == '\0') + return 0; + + /* reuse hash code in symbol table */ + p = smprint(".elfload.%s", name); + s = lookup(p, 0); + if(s->type == 0) { + s->type = 100; // avoid SDATA, etc. + return 1; + } + return 0; +} + +int nelfsym = 1; + +static void addpltsym(Sym*); +static void addgotsym(Sym*); + +void +adddynrel(Sym *s, Reloc *r) +{ + Sym *targ, *rela, *got; + + targ = r->sym; + cursym = s; + + switch(r->type) { + default: + if(r->type >= 256) { + diag("unexpected relocation type %d", r->type); + return; + } + break; + + // Handle relocations found in ELF object files. + case 256 + R_X86_64_PC32: + if(targ->dynimpname != nil && !targ->dynexport) + diag("unexpected R_X86_64_PC32 relocation for dynamic symbol %s", targ->name); + if(targ->type == 0 || targ->type == SXREF) + diag("unknown symbol %s in pcrel", targ->name); + r->type = D_PCREL; + r->add += 4; + return; + + case 256 + R_X86_64_PLT32: + r->type = D_PCREL; + r->add += 4; + if(targ->dynimpname != nil && !targ->dynexport) { + addpltsym(targ); + r->sym = lookup(".plt", 0); + r->add += targ->plt; + } + return; + + case 256 + R_X86_64_GOTPCREL: + if(targ->dynimpname == nil || targ->dynexport) { + // have symbol + if(r->off >= 2 && s->p[r->off-2] == 0x8b) { + // turn MOVQ of GOT entry into LEAQ of symbol itself + s->p[r->off-2] = 0x8d; + r->type = D_PCREL; + r->add += 4; + return; + } + // fall back to using GOT and hope for the best (CMOV*) + // TODO: just needs relocation, no need to put in .dynsym + targ->dynimpname = targ->name; + } + addgotsym(targ); + r->type = D_PCREL; + r->sym = lookup(".got", 0); + r->add += 4; + r->add += targ->got; + return; + + case 256 + R_X86_64_64: + if(targ->dynimpname != nil && !targ->dynexport) + diag("unexpected R_X86_64_64 relocation for dynamic symbol %s", targ->name); + r->type = D_ADDR; + return; + + // Handle relocations found in Mach-O object files. + case 512 + MACHO_X86_64_RELOC_UNSIGNED*2 + 0: + case 512 + MACHO_X86_64_RELOC_SIGNED*2 + 0: + case 512 + MACHO_X86_64_RELOC_BRANCH*2 + 0: + // TODO: What is the difference between all these? + r->type = D_ADDR; + if(targ->dynimpname != nil && !targ->dynexport) + diag("unexpected reloc for dynamic symbol %s", targ->name); + return; + + case 512 + MACHO_X86_64_RELOC_BRANCH*2 + 1: + if(targ->dynimpname != nil && !targ->dynexport) { + addpltsym(targ); + r->sym = lookup(".plt", 0); + r->add = targ->plt; + r->type = D_PCREL; + return; + } + // fall through + case 512 + MACHO_X86_64_RELOC_UNSIGNED*2 + 1: + case 512 + MACHO_X86_64_RELOC_SIGNED*2 + 1: + case 512 + MACHO_X86_64_RELOC_SIGNED_1*2 + 1: + case 512 + MACHO_X86_64_RELOC_SIGNED_2*2 + 1: + case 512 + MACHO_X86_64_RELOC_SIGNED_4*2 + 1: + r->type = D_PCREL; + if(targ->dynimpname != nil && !targ->dynexport) + diag("unexpected pc-relative reloc for dynamic symbol %s", targ->name); + return; + + case 512 + MACHO_X86_64_RELOC_GOT_LOAD*2 + 1: + if(targ->dynimpname == nil || targ->dynexport) { + // have symbol + // turn MOVQ of GOT entry into LEAQ of symbol itself + if(r->off < 2 || s->p[r->off-2] != 0x8b) { + diag("unexpected GOT_LOAD reloc for non-dynamic symbol %s", targ->name); + return; + } + s->p[r->off-2] = 0x8d; + r->type = D_PCREL; + return; + } + // fall through + case 512 + MACHO_X86_64_RELOC_GOT*2 + 1: + if(targ->dynimpname == nil || targ->dynexport) + diag("unexpected GOT reloc for non-dynamic symbol %s", targ->name); + addgotsym(targ); + r->type = D_PCREL; + r->sym = lookup(".got", 0); + r->add += targ->got; + return; + } + + // Handle references to ELF symbols from our own object files. + if(targ->dynimpname == nil || targ->dynexport) + return; + + switch(r->type) { + case D_PCREL: + addpltsym(targ); + r->sym = lookup(".plt", 0); + r->add = targ->plt; + return; + + case D_ADDR: + if(s->type != SDATA) + break; + if(iself) { + adddynsym(targ); + rela = lookup(".rela", 0); + addaddrplus(rela, s, r->off); + if(r->siz == 8) + adduint64(rela, ELF64_R_INFO(targ->dynid, R_X86_64_64)); + else + adduint64(rela, ELF64_R_INFO(targ->dynid, R_X86_64_32)); + adduint64(rela, r->add); + r->type = 256; // ignore during relocsym + return; + } + if(HEADTYPE == Hdarwin && s->size == PtrSize && r->off == 0) { + // Mach-O relocations are a royal pain to lay out. + // They use a compact stateful bytecode representation + // that is too much bother to deal with. + // Instead, interpret the C declaration + // void *_Cvar_stderr = &stderr; + // as making _Cvar_stderr the name of a GOT entry + // for stderr. This is separate from the usual GOT entry, + // just in case the C code assigns to the variable, + // and of course it only works for single pointers, + // but we only need to support cgo and that's all it needs. + adddynsym(targ); + got = lookup(".got", 0); + s->type = got->type | SSUB; + s->outer = got; + s->sub = got->sub; + got->sub = s; + s->value = got->size; + adduint64(got, 0); + adduint32(lookup(".linkedit.got", 0), targ->dynid); + r->type = 256; // ignore during relocsym + return; + } + break; + } + + cursym = s; + diag("unsupported relocation for dynamic symbol %s (type=%d stype=%d)", targ->name, r->type, targ->type); +} + +int +archreloc(Reloc *r, Sym *s, vlong *val) +{ + USED(r); + USED(s); + USED(val); + return -1; +} + +static void +elfsetupplt(void) +{ + Sym *plt, *got; + + plt = lookup(".plt", 0); + got = lookup(".got.plt", 0); + if(plt->size == 0) { + // pushq got+8(IP) + adduint8(plt, 0xff); + adduint8(plt, 0x35); + addpcrelplus(plt, got, 8); + + // jmpq got+16(IP) + adduint8(plt, 0xff); + adduint8(plt, 0x25); + addpcrelplus(plt, got, 16); + + // nopl 0(AX) + adduint32(plt, 0x00401f0f); + + // assume got->size == 0 too + addaddrplus(got, lookup(".dynamic", 0), 0); + adduint64(got, 0); + adduint64(got, 0); + } +} + +static void +addpltsym(Sym *s) +{ + if(s->plt >= 0) + return; + + adddynsym(s); + + if(iself) { + Sym *plt, *got, *rela; + + plt = lookup(".plt", 0); + got = lookup(".got.plt", 0); + rela = lookup(".rela.plt", 0); + if(plt->size == 0) + elfsetupplt(); + + // jmpq *got+size(IP) + adduint8(plt, 0xff); + adduint8(plt, 0x25); + addpcrelplus(plt, got, got->size); + + // add to got: pointer to current pos in plt + addaddrplus(got, plt, plt->size); + + // pushq $x + adduint8(plt, 0x68); + adduint32(plt, (got->size-24-8)/8); + + // jmpq .plt + adduint8(plt, 0xe9); + adduint32(plt, -(plt->size+4)); + + // rela + addaddrplus(rela, got, got->size-8); + adduint64(rela, ELF64_R_INFO(s->dynid, R_X86_64_JMP_SLOT)); + adduint64(rela, 0); + + s->plt = plt->size - 16; + } else if(HEADTYPE == Hdarwin) { + // To do lazy symbol lookup right, we're supposed + // to tell the dynamic loader which library each + // symbol comes from and format the link info + // section just so. I'm too lazy (ha!) to do that + // so for now we'll just use non-lazy pointers, + // which don't need to be told which library to use. + // + // http://networkpx.blogspot.com/2009/09/about-lcdyldinfoonly-command.html + // has details about what we're avoiding. + + Sym *plt; + + addgotsym(s); + plt = lookup(".plt", 0); + + adduint32(lookup(".linkedit.plt", 0), s->dynid); + + // jmpq *got+size(IP) + s->plt = plt->size; + + adduint8(plt, 0xff); + adduint8(plt, 0x25); + addpcrelplus(plt, lookup(".got", 0), s->got); + } else { + diag("addpltsym: unsupported binary format"); + } +} + +static void +addgotsym(Sym *s) +{ + Sym *got, *rela; + + if(s->got >= 0) + return; + + adddynsym(s); + got = lookup(".got", 0); + s->got = got->size; + adduint64(got, 0); + + if(iself) { + rela = lookup(".rela", 0); + addaddrplus(rela, got, s->got); + adduint64(rela, ELF64_R_INFO(s->dynid, R_X86_64_GLOB_DAT)); + adduint64(rela, 0); + } else if(HEADTYPE == Hdarwin) { + adduint32(lookup(".linkedit.got", 0), s->dynid); + } else { + diag("addgotsym: unsupported binary format"); + } +} + +void +adddynsym(Sym *s) +{ + Sym *d, *str; + int t; + char *name; + + if(s->dynid >= 0) + return; + + if(s->dynimpname == nil) + diag("adddynsym: no dynamic name for %s", s->name); + + if(iself) { + s->dynid = nelfsym++; + + d = lookup(".dynsym", 0); + + name = s->dynimpname; + if(name == nil) + name = s->name; + adduint32(d, addstring(lookup(".dynstr", 0), name)); + /* type */ + t = STB_GLOBAL << 4; + if(s->dynexport && s->type == STEXT) + t |= STT_FUNC; + else + t |= STT_OBJECT; + adduint8(d, t); + + /* reserved */ + adduint8(d, 0); + + /* section where symbol is defined */ + if(!s->dynexport && s->dynimpname != nil) + adduint16(d, SHN_UNDEF); + else { + switch(s->type) { + default: + case STEXT: + t = 11; + break; + case SRODATA: + t = 12; + break; + case SDATA: + t = 13; + break; + case SBSS: + t = 14; + break; + } + adduint16(d, t); + } + + /* value */ + if(s->type == SDYNIMPORT) + adduint64(d, 0); + else + addaddr(d, s); + + /* size of object */ + adduint64(d, 0); + + if(!s->dynexport && s->dynimplib && needlib(s->dynimplib)) { + elfwritedynent(lookup(".dynamic", 0), DT_NEEDED, + addstring(lookup(".dynstr", 0), s->dynimplib)); + } + } else if(HEADTYPE == Hdarwin) { + // Mach-o symbol nlist64 + d = lookup(".dynsym", 0); + name = s->dynimpname; + if(name == nil) + name = s->name; + s->dynid = d->size/16; + // darwin still puts _ prefixes on all C symbols + str = lookup(".dynstr", 0); + adduint32(d, str->size); + adduint8(str, '_'); + addstring(str, name); + if(s->type == SDYNIMPORT) { + adduint8(d, 0x01); // type - N_EXT - external symbol + adduint8(d, 0); // section + } else { + adduint8(d, 0x0f); + switch(s->type) { + default: + case STEXT: + adduint8(d, 1); + break; + case SDATA: + adduint8(d, 2); + break; + case SBSS: + adduint8(d, 4); + break; + } + } + adduint16(d, 0); // desc + if(s->type == SDYNIMPORT) + adduint64(d, 0); // value + else + addaddr(d, s); + } else if(HEADTYPE != Hwindows) { + diag("adddynsym: unsupported binary format"); + } +} + +void +adddynlib(char *lib) +{ + Sym *s; + + if(!needlib(lib)) + return; + + if(iself) { + s = lookup(".dynstr", 0); + if(s->size == 0) + addstring(s, ""); + elfwritedynent(lookup(".dynamic", 0), DT_NEEDED, addstring(s, lib)); + } else if(HEADTYPE == Hdarwin) { + machoadddynlib(lib); + } else { + diag("adddynlib: unsupported binary format"); + } +} + +void +doelf(void) +{ + Sym *s, *shstrtab, *dynstr; + + if(HEADTYPE != Hlinux && HEADTYPE != Hfreebsd && HEADTYPE != Hopenbsd) + return; + + /* predefine strings we need for section headers */ + shstrtab = lookup(".shstrtab", 0); + shstrtab->type = SELFROSECT; + shstrtab->reachable = 1; + + elfstr[ElfStrEmpty] = addstring(shstrtab, ""); + elfstr[ElfStrText] = addstring(shstrtab, ".text"); + elfstr[ElfStrData] = addstring(shstrtab, ".data"); + elfstr[ElfStrBss] = addstring(shstrtab, ".bss"); + addstring(shstrtab, ".elfdata"); + addstring(shstrtab, ".rodata"); + addstring(shstrtab, ".gosymtab"); + addstring(shstrtab, ".gopclntab"); + if(!debug['s']) { + elfstr[ElfStrSymtab] = addstring(shstrtab, ".symtab"); + elfstr[ElfStrStrtab] = addstring(shstrtab, ".strtab"); + dwarfaddshstrings(shstrtab); + } + elfstr[ElfStrShstrtab] = addstring(shstrtab, ".shstrtab"); + + if(!debug['d']) { /* -d suppresses dynamic loader format */ + elfstr[ElfStrInterp] = addstring(shstrtab, ".interp"); + elfstr[ElfStrHash] = addstring(shstrtab, ".hash"); + elfstr[ElfStrGot] = addstring(shstrtab, ".got"); + elfstr[ElfStrGotPlt] = addstring(shstrtab, ".got.plt"); + elfstr[ElfStrDynamic] = addstring(shstrtab, ".dynamic"); + elfstr[ElfStrDynsym] = addstring(shstrtab, ".dynsym"); + elfstr[ElfStrDynstr] = addstring(shstrtab, ".dynstr"); + elfstr[ElfStrRela] = addstring(shstrtab, ".rela"); + elfstr[ElfStrRelaPlt] = addstring(shstrtab, ".rela.plt"); + elfstr[ElfStrPlt] = addstring(shstrtab, ".plt"); + elfstr[ElfStrGnuVersion] = addstring(shstrtab, ".gnu.version"); + elfstr[ElfStrGnuVersionR] = addstring(shstrtab, ".gnu.version_r"); + + /* dynamic symbol table - first entry all zeros */ + s = lookup(".dynsym", 0); + s->type = SELFROSECT; + s->reachable = 1; + s->size += ELF64SYMSIZE; + + /* dynamic string table */ + s = lookup(".dynstr", 0); + s->type = SELFROSECT; + s->reachable = 1; + if(s->size == 0) + addstring(s, ""); + dynstr = s; + + /* relocation table */ + s = lookup(".rela", 0); + s->reachable = 1; + s->type = SELFROSECT; + + /* global offset table */ + s = lookup(".got", 0); + s->reachable = 1; + s->type = SELFSECT; // writable + + /* hash */ + s = lookup(".hash", 0); + s->reachable = 1; + s->type = SELFROSECT; + + s = lookup(".got.plt", 0); + s->reachable = 1; + s->type = SELFSECT; // writable + + s = lookup(".plt", 0); + s->reachable = 1; + s->type = SELFROSECT; + + elfsetupplt(); + + s = lookup(".rela.plt", 0); + s->reachable = 1; + s->type = SELFROSECT; + + s = lookup(".gnu.version", 0); + s->reachable = 1; + s->type = SELFROSECT; + + s = lookup(".gnu.version_r", 0); + s->reachable = 1; + s->type = SELFROSECT; + + /* define dynamic elf table */ + s = lookup(".dynamic", 0); + s->reachable = 1; + s->type = SELFROSECT; + + /* + * .dynamic table + */ + elfwritedynentsym(s, DT_HASH, lookup(".hash", 0)); + elfwritedynentsym(s, DT_SYMTAB, lookup(".dynsym", 0)); + elfwritedynent(s, DT_SYMENT, ELF64SYMSIZE); + elfwritedynentsym(s, DT_STRTAB, lookup(".dynstr", 0)); + elfwritedynentsymsize(s, DT_STRSZ, lookup(".dynstr", 0)); + elfwritedynentsym(s, DT_RELA, lookup(".rela", 0)); + elfwritedynentsymsize(s, DT_RELASZ, lookup(".rela", 0)); + elfwritedynent(s, DT_RELAENT, ELF64RELASIZE); + if(rpath) + elfwritedynent(s, DT_RUNPATH, addstring(dynstr, rpath)); + + elfwritedynentsym(s, DT_PLTGOT, lookup(".got.plt", 0)); + elfwritedynent(s, DT_PLTREL, DT_RELA); + elfwritedynentsymsize(s, DT_PLTRELSZ, lookup(".rela.plt", 0)); + elfwritedynentsym(s, DT_JMPREL, lookup(".rela.plt", 0)); + + // Do not write DT_NULL. elfdynhash will finish it. + } +} + +void +shsym(ElfShdr *sh, Sym *s) +{ + vlong addr; + addr = symaddr(s); + if(sh->flags&SHF_ALLOC) + sh->addr = addr; + sh->off = datoff(addr); + sh->size = s->size; +} + +void +phsh(ElfPhdr *ph, ElfShdr *sh) +{ + ph->vaddr = sh->addr; + ph->paddr = ph->vaddr; + ph->off = sh->off; + ph->filesz = sh->size; + ph->memsz = sh->size; + ph->align = sh->addralign; +} + +void +asmb(void) +{ + int32 magic; + int a, dynsym; + vlong vl, startva, symo, machlink; + ElfEhdr *eh; + ElfPhdr *ph, *pph; + ElfShdr *sh; + Section *sect; + int o; + + if(debug['v']) + Bprint(&bso, "%5.2f asmb\n", cputime()); + Bflush(&bso); + + elftextsh = 0; + + if(debug['v']) + Bprint(&bso, "%5.2f codeblk\n", cputime()); + Bflush(&bso); + + sect = segtext.sect; + cseek(sect->vaddr - segtext.vaddr + segtext.fileoff); + codeblk(sect->vaddr, sect->len); + + /* output read-only data in text segment (rodata, gosymtab and pclntab) */ + for(sect = sect->next; sect != nil; sect = sect->next) { + cseek(sect->vaddr - segtext.vaddr + segtext.fileoff); + datblk(sect->vaddr, sect->len); + } + + if(debug['v']) + Bprint(&bso, "%5.2f datblk\n", cputime()); + Bflush(&bso); + + cseek(segdata.fileoff); + datblk(segdata.vaddr, segdata.filelen); + + machlink = 0; + if(HEADTYPE == Hdarwin) + machlink = domacholink(); + + switch(HEADTYPE) { + default: + diag("unknown header type %d", HEADTYPE); + case Hplan9x32: + case Helf: + break; + case Hdarwin: + debug['8'] = 1; /* 64-bit addresses */ + break; + case Hlinux: + case Hfreebsd: + case Hopenbsd: + debug['8'] = 1; /* 64-bit addresses */ + /* index of elf text section; needed by asmelfsym, double-checked below */ + /* !debug['d'] causes extra sections before the .text section */ + elftextsh = 2; + if(!debug['d']) { + elftextsh += 10; + if(elfverneed) + elftextsh += 2; + } + break; + case Hwindows: + break; + } + + symsize = 0; + spsize = 0; + lcsize = 0; + symo = 0; + if(!debug['s']) { + if(debug['v']) + Bprint(&bso, "%5.2f sym\n", cputime()); + Bflush(&bso); + switch(HEADTYPE) { + default: + case Hplan9x32: + case Helf: + debug['s'] = 1; + symo = HEADR+segtext.len+segdata.filelen; + break; + case Hdarwin: + symo = rnd(HEADR+segtext.len, INITRND)+rnd(segdata.filelen, INITRND)+machlink; + break; + case Hlinux: + case Hfreebsd: + case Hopenbsd: + symo = rnd(HEADR+segtext.len, INITRND)+segdata.filelen; + symo = rnd(symo, INITRND); + break; + case Hwindows: + symo = rnd(HEADR+segtext.filelen, PEFILEALIGN)+segdata.filelen; + symo = rnd(symo, PEFILEALIGN); + break; + } + cseek(symo); + switch(HEADTYPE) { + default: + if(iself) { + cseek(symo); + asmelfsym(); + cflush(); + cwrite(elfstrdat, elfstrsize); + + if(debug['v']) + Bprint(&bso, "%5.2f dwarf\n", cputime()); + + dwarfemitdebugsections(); + } + break; + case Hdarwin: + case Hwindows: + if(debug['v']) + Bprint(&bso, "%5.2f dwarf\n", cputime()); + + dwarfemitdebugsections(); + break; + } + } + + if(debug['v']) + Bprint(&bso, "%5.2f headr\n", cputime()); + Bflush(&bso); + cseek(0L); + switch(HEADTYPE) { + default: + case Hplan9x32: /* plan9 */ + magic = 4*26*26+7; + magic |= 0x00008000; /* fat header */ + lputb(magic); /* magic */ + lputb(segtext.filelen); /* sizes */ + lputb(segdata.filelen); + lputb(segdata.len - segdata.filelen); + lputb(symsize); /* nsyms */ + vl = entryvalue(); + lputb(PADDR(vl)); /* va of entry */ + lputb(spsize); /* sp offsets */ + lputb(lcsize); /* line offsets */ + vputb(vl); /* va of entry */ + break; + case Hplan9x64: /* plan9 */ + magic = 4*26*26+7; + lputb(magic); /* magic */ + lputb(segtext.filelen); /* sizes */ + lputb(segdata.filelen); + lputb(segdata.len - segdata.filelen); + lputb(symsize); /* nsyms */ + lputb(entryvalue()); /* va of entry */ + lputb(spsize); /* sp offsets */ + lputb(lcsize); /* line offsets */ + break; + case Hdarwin: + asmbmacho(); + break; + case Hlinux: + case Hfreebsd: + case Hopenbsd: + /* elf amd-64 */ + + eh = getElfEhdr(); + startva = INITTEXT - HEADR; + + /* This null SHdr must appear before all others */ + newElfShdr(elfstr[ElfStrEmpty]); + + /* program header info */ + pph = newElfPhdr(); + pph->type = PT_PHDR; + pph->flags = PF_R + PF_X; + pph->off = eh->ehsize; + pph->vaddr = INITTEXT - HEADR + pph->off; + pph->paddr = INITTEXT - HEADR + pph->off; + pph->align = INITRND; + + /* + * PHDR must be in a loaded segment. Adjust the text + * segment boundaries downwards to include it. + */ + o = segtext.vaddr - pph->vaddr; + segtext.vaddr -= o; + segtext.len += o; + o = segtext.fileoff - pph->off; + segtext.fileoff -= o; + segtext.filelen += o; + + if(!debug['d']) { + /* interpreter */ + sh = newElfShdr(elfstr[ElfStrInterp]); + sh->type = SHT_PROGBITS; + sh->flags = SHF_ALLOC; + sh->addralign = 1; + if(interpreter == nil) { + switch(HEADTYPE) { + case Hlinux: + interpreter = linuxdynld; + break; + case Hfreebsd: + interpreter = freebsddynld; + break; + case Hopenbsd: + interpreter = openbsddynld; + break; + } + } + elfinterp(sh, startva, interpreter); + + ph = newElfPhdr(); + ph->type = PT_INTERP; + ph->flags = PF_R; + phsh(ph, sh); + } + + elfphload(&segtext); + elfphload(&segdata); + + /* Dynamic linking sections */ + if (!debug['d']) { /* -d suppresses dynamic loader format */ + /* S headers for dynamic linking */ + sh = newElfShdr(elfstr[ElfStrGot]); + sh->type = SHT_PROGBITS; + sh->flags = SHF_ALLOC+SHF_WRITE; + sh->entsize = 8; + sh->addralign = 8; + shsym(sh, lookup(".got", 0)); + + sh = newElfShdr(elfstr[ElfStrGotPlt]); + sh->type = SHT_PROGBITS; + sh->flags = SHF_ALLOC+SHF_WRITE; + sh->entsize = 8; + sh->addralign = 8; + shsym(sh, lookup(".got.plt", 0)); + + dynsym = eh->shnum; + sh = newElfShdr(elfstr[ElfStrDynsym]); + sh->type = SHT_DYNSYM; + sh->flags = SHF_ALLOC; + sh->entsize = ELF64SYMSIZE; + sh->addralign = 8; + sh->link = dynsym+1; // dynstr + // sh->info = index of first non-local symbol (number of local symbols) + shsym(sh, lookup(".dynsym", 0)); + + sh = newElfShdr(elfstr[ElfStrDynstr]); + sh->type = SHT_STRTAB; + sh->flags = SHF_ALLOC; + sh->addralign = 1; + shsym(sh, lookup(".dynstr", 0)); + + if(elfverneed) { + sh = newElfShdr(elfstr[ElfStrGnuVersion]); + sh->type = SHT_GNU_VERSYM; + sh->flags = SHF_ALLOC; + sh->addralign = 2; + sh->link = dynsym; + sh->entsize = 2; + shsym(sh, lookup(".gnu.version", 0)); + + sh = newElfShdr(elfstr[ElfStrGnuVersionR]); + sh->type = SHT_GNU_VERNEED; + sh->flags = SHF_ALLOC; + sh->addralign = 8; + sh->info = elfverneed; + sh->link = dynsym+1; // dynstr + shsym(sh, lookup(".gnu.version_r", 0)); + } + + sh = newElfShdr(elfstr[ElfStrRelaPlt]); + sh->type = SHT_RELA; + sh->flags = SHF_ALLOC; + sh->entsize = ELF64RELASIZE; + sh->addralign = 8; + sh->link = dynsym; + sh->info = eh->shnum; // .plt + shsym(sh, lookup(".rela.plt", 0)); + + sh = newElfShdr(elfstr[ElfStrPlt]); + sh->type = SHT_PROGBITS; + sh->flags = SHF_ALLOC+SHF_EXECINSTR; + sh->entsize = 16; + sh->addralign = 4; + shsym(sh, lookup(".plt", 0)); + + sh = newElfShdr(elfstr[ElfStrHash]); + sh->type = SHT_HASH; + sh->flags = SHF_ALLOC; + sh->entsize = 4; + sh->addralign = 8; + sh->link = dynsym; + shsym(sh, lookup(".hash", 0)); + + sh = newElfShdr(elfstr[ElfStrRela]); + sh->type = SHT_RELA; + sh->flags = SHF_ALLOC; + sh->entsize = ELF64RELASIZE; + sh->addralign = 8; + sh->link = dynsym; + shsym(sh, lookup(".rela", 0)); + + /* sh and PT_DYNAMIC for .dynamic section */ + sh = newElfShdr(elfstr[ElfStrDynamic]); + sh->type = SHT_DYNAMIC; + sh->flags = SHF_ALLOC+SHF_WRITE; + sh->entsize = 16; + sh->addralign = 8; + sh->link = dynsym+1; // dynstr + shsym(sh, lookup(".dynamic", 0)); + ph = newElfPhdr(); + ph->type = PT_DYNAMIC; + ph->flags = PF_R + PF_W; + phsh(ph, sh); + + /* + * Thread-local storage segment (really just size). + */ + if(tlsoffset != 0) { + ph = newElfPhdr(); + ph->type = PT_TLS; + ph->flags = PF_R; + ph->memsz = -tlsoffset; + ph->align = 8; + } + } + + ph = newElfPhdr(); + ph->type = PT_GNU_STACK; + ph->flags = PF_W+PF_R; + ph->align = 8; + + sh = newElfShstrtab(elfstr[ElfStrShstrtab]); + sh->type = SHT_STRTAB; + sh->addralign = 1; + shsym(sh, lookup(".shstrtab", 0)); + + if(elftextsh != eh->shnum) + diag("elftextsh = %d, want %d", elftextsh, eh->shnum); + for(sect=segtext.sect; sect!=nil; sect=sect->next) + elfshbits(sect); + for(sect=segdata.sect; sect!=nil; sect=sect->next) + elfshbits(sect); + + if (!debug['s']) { + sh = newElfShdr(elfstr[ElfStrSymtab]); + sh->type = SHT_SYMTAB; + sh->off = symo; + sh->size = symsize; + sh->addralign = 8; + sh->entsize = 24; + sh->link = eh->shnum; // link to strtab + + sh = newElfShdr(elfstr[ElfStrStrtab]); + sh->type = SHT_STRTAB; + sh->off = symo+symsize; + sh->size = elfstrsize; + sh->addralign = 1; + + dwarfaddelfheaders(); + } + + /* Main header */ + eh->ident[EI_MAG0] = '\177'; + eh->ident[EI_MAG1] = 'E'; + eh->ident[EI_MAG2] = 'L'; + eh->ident[EI_MAG3] = 'F'; + if(HEADTYPE == Hfreebsd) + eh->ident[EI_OSABI] = ELFOSABI_FREEBSD; + else if(HEADTYPE == Hopenbsd) + eh->ident[EI_OSABI] = ELFOSABI_OPENBSD; + eh->ident[EI_CLASS] = ELFCLASS64; + eh->ident[EI_DATA] = ELFDATA2LSB; + eh->ident[EI_VERSION] = EV_CURRENT; + + eh->type = ET_EXEC; + eh->machine = EM_X86_64; + eh->version = EV_CURRENT; + eh->entry = entryvalue(); + + pph->filesz = eh->phnum * eh->phentsize; + pph->memsz = pph->filesz; + + cseek(0); + a = 0; + a += elfwritehdr(); + a += elfwritephdrs(); + a += elfwriteshdrs(); + cflush(); + if(a+elfwriteinterp() > ELFRESERVE) + diag("ELFRESERVE too small: %d > %d", a, ELFRESERVE); + break; + case Hwindows: + asmbpe(); + break; + } + cflush(); +} + +vlong +rnd(vlong v, vlong r) +{ + vlong c; + + if(r <= 0) + return v; + v += r - 1; + c = v % r; + if(c < 0) + c += r; + v -= c; + return v; +} + +void +genasmsym(void (*put)(Sym*, char*, int, vlong, vlong, int, Sym*)) +{ + Auto *a; + Sym *s; + + s = lookup("etext", 0); + if(s->type == STEXT) + put(s, s->name, 'T', s->value, s->size, s->version, 0); + + for(s=allsym; s!=S; s=s->allsym) { + if(s->hide) + continue; + switch(s->type&~SSUB) { + case SCONST: + case SRODATA: + case SDATA: + case SELFROSECT: + case SMACHOGOT: + case STYPE: + case SSTRING: + case SGOSTRING: + case SWINDOWS: + if(!s->reachable) + continue; + put(s, s->name, 'D', symaddr(s), s->size, s->version, s->gotype); + continue; + + case SBSS: + if(!s->reachable) + continue; + put(s, s->name, 'B', symaddr(s), s->size, s->version, s->gotype); + continue; + + case SFILE: + put(nil, s->name, 'f', s->value, 0, s->version, 0); + continue; + } + } + + for(s = textp; s != nil; s = s->next) { + if(s->text == nil) + continue; + + /* filenames first */ + for(a=s->autom; a; a=a->link) + if(a->type == D_FILE) + put(nil, a->asym->name, 'z', a->aoffset, 0, 0, 0); + else + if(a->type == D_FILE1) + put(nil, a->asym->name, 'Z', a->aoffset, 0, 0, 0); + + put(s, s->name, 'T', s->value, s->size, s->version, s->gotype); + + /* frame, auto and param after */ + put(nil, ".frame", 'm', s->text->to.offset+8, 0, 0, 0); + + for(a=s->autom; a; a=a->link) + if(a->type == D_AUTO) + put(nil, a->asym->name, 'a', -a->aoffset, 0, 0, a->gotype); + else + if(a->type == D_PARAM) + put(nil, a->asym->name, 'p', a->aoffset, 0, 0, a->gotype); + } + if(debug['v'] || debug['n']) + Bprint(&bso, "symsize = %ud\n", symsize); + Bflush(&bso); +} diff --git a/src/cmd/6l/doc.go b/src/cmd/6l/doc.go new file mode 100644 index 000000000..b8a6013d6 --- /dev/null +++ b/src/cmd/6l/doc.go @@ -0,0 +1,53 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +/* + +6l is a modified version of the Plan 9 linker. The original is documented at + + http://plan9.bell-labs.com/magic/man2html/1/2l + +Its target architecture is the x86-64, referred to by these tools as amd64. +It reads files in .6 format generated by 6g, 6c, and 6a and emits +a binary called 6.out by default. + +Major changes include: + - support for ELF and Mach-O binary files + - support for segmented stacks (this feature is implemented here, not in the compilers). + + +Original options are listed in the link above. + +Options new in this version: + +-d + Elide the dynamic linking header. With this option, the binary + is statically linked and does not refer to dynld. Without this option + (the default), the binary's contents are identical but it is loaded with dynld. +-e + Emit an extra ELF-compatible symbol table useful with tools such as + nm, gdb, and oprofile. This option makes the binary file considerably larger. +-Hdarwin + Write Apple Mach-O binaries (default when $GOOS is darwin) +-Hlinux + Write Linux ELF binaries (default when $GOOS is linux) +-Hfreebsd + Write FreeBSD ELF binaries (default when $GOOS is freebsd) +-Hopenbsd + Write OpenBSD ELF binaries (default when $GOOS is openbsd) +-Hwindows + Write Windows PE32+ binaries (default when $GOOS is windows) +-I interpreter + Set the ELF dynamic linker to use. +-L dir1 -L dir2 + Search for libraries (package files) in dir1, dir2, etc. + The default is the single location $GOROOT/pkg/$GOOS_amd64. +-r dir1:dir2:... + Set the dynamic linker search path when using ELF. +-V + Print the linker version. + + +*/ +package documentation diff --git a/src/cmd/6l/l.h b/src/cmd/6l/l.h new file mode 100644 index 000000000..b291d5f3d --- /dev/null +++ b/src/cmd/6l/l.h @@ -0,0 +1,421 @@ +// Inferno utils/6l/l.h +// http://code.google.com/p/inferno-os/source/browse/utils/6l/l.h +// +// Copyright © 1994-1999 Lucent Technologies Inc. All rights reserved. +// Portions Copyright © 1995-1997 C H Forsyth (forsyth@terzarima.net) +// Portions Copyright © 1997-1999 Vita Nuova Limited +// Portions Copyright © 2000-2007 Vita Nuova Holdings Limited (www.vitanuova.com) +// Portions Copyright © 2004,2006 Bruce Ellis +// Portions Copyright © 2005-2007 C H Forsyth (forsyth@terzarima.net) +// Revisions Copyright © 2000-2007 Lucent Technologies Inc. and others +// Portions Copyright © 2009 The Go Authors. All rights reserved. +// +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to deal +// in the Software without restriction, including without limitation the rights +// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +// copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +// THE SOFTWARE. + +#include +#include +#include +#include "6.out.h" + +#ifndef EXTERN +#define EXTERN extern +#endif + +enum +{ + thechar = '6', + PtrSize = 8 +}; + +#define P ((Prog*)0) +#define S ((Sym*)0) +#define TNAME (cursym?cursym->name:noname) + +typedef struct Adr Adr; +typedef struct Prog Prog; +typedef struct Sym Sym; +typedef struct Auto Auto; +typedef struct Optab Optab; +typedef struct Movtab Movtab; +typedef struct Reloc Reloc; + +struct Adr +{ + union + { + vlong u0offset; + char u0scon[8]; + Prog *u0cond; /* not used, but should be D_BRANCH */ + Ieee u0ieee; + char *u0sbig; + } u0; + Sym* sym; + short type; + char index; + char scale; +}; + +#define offset u0.u0offset +#define scon u0.u0scon +#define cond u0.u0cond +#define ieee u0.u0ieee +#define sbig u0.u0sbig + +struct Reloc +{ + int32 off; + uchar siz; + int32 type; + int64 add; + Sym* sym; +}; + +struct Prog +{ + Adr from; + Adr to; + Prog* forwd; + Prog* comefrom; + Prog* link; + Prog* pcond; /* work on this */ + vlong pc; + int32 spadj; + int32 line; + short as; + char ft; /* oclass cache */ + char tt; + uchar mark; /* work on these */ + uchar back; + + char width; /* fake for DATA */ + char mode; /* 16, 32, or 64 */ +}; +#define datasize from.scale +#define textflag from.scale +#define iscall(p) ((p)->as == ACALL) + +struct Auto +{ + Sym* asym; + Auto* link; + int32 aoffset; + short type; + Sym* gotype; +}; +struct Sym +{ + char* name; + short type; + short version; + uchar dupok; + uchar reachable; + uchar dynexport; + uchar special; + uchar stkcheck; + uchar hide; + int32 dynid; + int32 sig; + int32 plt; + int32 got; + Sym* hash; // in hash table + Sym* allsym; // in all symbol list + Sym* next; // in text or data list + Sym* sub; // in SSUB list + Sym* outer; // container of sub + vlong value; + vlong size; + Sym* gotype; + char* file; + char* dynimpname; + char* dynimplib; + char* dynimpvers; + + // STEXT + Auto* autom; + Prog* text; + + // SDATA, SBSS + uchar* p; + int32 np; + int32 maxp; + Reloc* r; + int32 nr; + int32 maxr; +}; +struct Optab +{ + short as; + uchar* ytab; + uchar prefix; + uchar op[20]; +}; +struct Movtab +{ + short as; + uchar ft; + uchar tt; + uchar code; + uchar op[4]; +}; + +enum +{ + MINSIZ = 8, + STRINGSZ = 200, + MINLC = 1, + MAXIO = 8192, + MAXHIST = 20, /* limit of path elements for history symbols */ + + Yxxx = 0, + Ynone, + Yi0, + Yi1, + Yi8, + Ys32, + Yi32, + Yi64, + Yiauto, + Yal, + Ycl, + Yax, + Ycx, + Yrb, + Yrl, + Yrf, + Yf0, + Yrx, + Ymb, + Yml, + Ym, + Ybr, + Ycol, + + Ycs, Yss, Yds, Yes, Yfs, Ygs, + Ygdtr, Yidtr, Yldtr, Ymsw, Ytask, + Ycr0, Ycr1, Ycr2, Ycr3, Ycr4, Ycr5, Ycr6, Ycr7, Ycr8, + Ydr0, Ydr1, Ydr2, Ydr3, Ydr4, Ydr5, Ydr6, Ydr7, + Ytr0, Ytr1, Ytr2, Ytr3, Ytr4, Ytr5, Ytr6, Ytr7, Yrl32, Yrl64, + Ymr, Ymm, + Yxr, Yxm, + Ymax, + + Zxxx = 0, + + Zlit, + Zlitm_r, + Z_rp, + Zbr, + Zcall, + Zib_, + Zib_rp, + Zibo_m, + Zibo_m_xm, + Zil_, + Zil_rp, + Ziq_rp, + Zilo_m, + Ziqo_m, + Zjmp, + Zloop, + Zo_iw, + Zm_o, + Zm_r, + Zm_r_xm, + Zm_r_i_xm, + Zm_r_3d, + Zm_r_xm_nr, + Zr_m_xm_nr, + Zibm_r, /* mmx1,mmx2/mem64,imm8 */ + Zmb_r, + Zaut_r, + Zo_m, + Zo_m64, + Zpseudo, + Zr_m, + Zr_m_xm, + Zr_m_i_xm, + Zrp_, + Z_ib, + Z_il, + Zm_ibo, + Zm_ilo, + Zib_rr, + Zil_rr, + Zclr, + Zbyte, + Zmax, + + Px = 0, + P32 = 0x32, /* 32-bit only */ + Pe = 0x66, /* operand escape */ + Pm = 0x0f, /* 2byte opcode escape */ + Pq = 0xff, /* both escape */ + Pb = 0xfe, /* byte operands */ + Pf2 = 0xf2, /* xmm escape 1 */ + Pf3 = 0xf3, /* xmm escape 2 */ + Pw = 0x48, /* Rex.w */ + Py = 0x80, /* defaults to 64-bit mode */ + + Rxf = 1<<9, /* internal flag for Rxr on from */ + Rxt = 1<<8, /* internal flag for Rxr on to */ + Rxw = 1<<3, /* =1, 64-bit operand size */ + Rxr = 1<<2, /* extend modrm reg */ + Rxx = 1<<1, /* extend sib index */ + Rxb = 1<<0, /* extend modrm r/m, sib base, or opcode reg */ + + Maxand = 10, /* in -a output width of the byte codes */ +}; + +#pragma varargck type "A" uint +#pragma varargck type "D" Adr* +#pragma varargck type "I" uchar* +#pragma varargck type "P" Prog* +#pragma varargck type "R" int +#pragma varargck type "S" char* +#pragma varargck type "i" char* + +EXTERN int32 HEADR; +EXTERN int32 HEADTYPE; +EXTERN int32 INITRND; +EXTERN vlong INITTEXT; +EXTERN vlong INITDAT; +EXTERN char* INITENTRY; /* entry point */ +EXTERN char* pcstr; +EXTERN Auto* curauto; +EXTERN Auto* curhist; +EXTERN Prog* curp; +EXTERN Sym* cursym; +EXTERN Sym* datap; +EXTERN vlong elfdatsize; +EXTERN char debug[128]; +EXTERN char literal[32]; +EXTERN Sym* textp; +EXTERN Sym* etextp; +EXTERN char ycover[Ymax*Ymax]; +EXTERN uchar* andptr; +EXTERN uchar* rexptr; +EXTERN uchar and[30]; +EXTERN int reg[D_NONE]; +EXTERN int regrex[D_NONE+1]; +EXTERN int32 lcsize; +EXTERN int nerrors; +EXTERN char* noname; +EXTERN char* outfile; +EXTERN vlong pc; +EXTERN char* interpreter; +EXTERN char* rpath; +EXTERN int32 spsize; +EXTERN Sym* symlist; +EXTERN int32 symsize; +EXTERN int tlsoffset; +EXTERN int version; +EXTERN Prog zprg; +EXTERN int dtype; +EXTERN char* paramspace; +EXTERN Sym* adrgotype; // type symbol on last Adr read +EXTERN Sym* fromgotype; // type symbol on last p->from read + +EXTERN vlong textstksiz; +EXTERN vlong textarg; + +extern Optab optab[]; +extern Optab* opindex[]; +extern char* anames[]; + +int Aconv(Fmt*); +int Dconv(Fmt*); +int Iconv(Fmt*); +int Pconv(Fmt*); +int Rconv(Fmt*); +int Sconv(Fmt*); +void addhist(int32, int); +void addstackmark(void); +Prog* appendp(Prog*); +void asmb(void); +void asmdyn(void); +void asmins(Prog*); +void asmsym(void); +void asmelfsym(void); +vlong atolwhex(char*); +Prog* brchain(Prog*); +Prog* brloop(Prog*); +void buildop(void); +Prog* copyp(Prog*); +double cputime(void); +void datblk(int32, int32); +void deadcode(void); +void diag(char*, ...); +void dodata(void); +void doelf(void); +void domacho(void); +void doprof1(void); +void doprof2(void); +void dostkoff(void); +vlong entryvalue(void); +void follow(void); +void gethunk(void); +void gotypestrings(void); +void listinit(void); +Sym* lookup(char*, int); +void lputb(int32); +void lputl(int32); +void instinit(void); +void main(int, char*[]); +void* mysbrk(uint32); +Prog* newtext(Prog*, Sym*); +void nopout(Prog*); +int opsize(Prog*); +void patch(void); +Prog* prg(void); +void parsetextconst(vlong); +int relinv(int); +vlong rnd(vlong, vlong); +void span(void); +void undef(void); +vlong symaddr(Sym*); +void vputb(uint64); +void vputl(uint64); +void wputb(uint16); +void wputl(uint16); +void xdefine(char*, int, vlong); + +void machseg(char*, vlong, vlong, vlong, vlong, uint32, uint32, uint32, uint32); +void machsymseg(uint32, uint32); +void machsect(char*, char*, vlong, vlong, uint32, uint32, uint32, uint32, uint32); +void machstack(vlong); +void machdylink(void); +uint32 machheadr(void); + +/* Native is little-endian */ +#define LPUT(a) lputl(a) +#define WPUT(a) wputl(a) +#define VPUT(a) vputl(a) + +#pragma varargck type "D" Adr* +#pragma varargck type "P" Prog* +#pragma varargck type "R" int +#pragma varargck type "Z" char* +#pragma varargck type "A" int +#pragma varargck argpos diag 1 + +/* Used by ../ld/dwarf.c */ +enum +{ + DWARFREGSP = 7 +}; diff --git a/src/cmd/6l/list.c b/src/cmd/6l/list.c new file mode 100644 index 000000000..f39efa2e8 --- /dev/null +++ b/src/cmd/6l/list.c @@ -0,0 +1,458 @@ +// Inferno utils/6l/list.c +// http://code.google.com/p/inferno-os/source/browse/utils/6l/list.c +// +// Copyright © 1994-1999 Lucent Technologies Inc. All rights reserved. +// Portions Copyright © 1995-1997 C H Forsyth (forsyth@terzarima.net) +// Portions Copyright © 1997-1999 Vita Nuova Limited +// Portions Copyright © 2000-2007 Vita Nuova Holdings Limited (www.vitanuova.com) +// Portions Copyright © 2004,2006 Bruce Ellis +// Portions Copyright © 2005-2007 C H Forsyth (forsyth@terzarima.net) +// Revisions Copyright © 2000-2007 Lucent Technologies Inc. and others +// Portions Copyright © 2009 The Go Authors. All rights reserved. +// +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to deal +// in the Software without restriction, including without limitation the rights +// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +// copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +// THE SOFTWARE. + +// Printing. + +#include "l.h" +#include "../ld/lib.h" + +static Prog* bigP; + +void +listinit(void) +{ + + fmtinstall('R', Rconv); + fmtinstall('A', Aconv); + fmtinstall('D', Dconv); + fmtinstall('S', Sconv); + fmtinstall('P', Pconv); + fmtinstall('I', Iconv); +} + +int +Pconv(Fmt *fp) +{ + Prog *p; + + p = va_arg(fp->args, Prog*); + bigP = p; + switch(p->as) { + case ATEXT: + if(p->from.scale) { + fmtprint(fp, "(%d) %A %D,%d,%D", + p->line, p->as, &p->from, p->from.scale, &p->to); + break; + } + default: + fmtprint(fp, "(%d) %A %D,%D", + p->line, p->as, &p->from, &p->to); + break; + case ADATA: + case AINIT_: + case ADYNT_: + fmtprint(fp, "(%d) %A %D/%d,%D", + p->line, p->as, &p->from, p->from.scale, &p->to); + break; + } + bigP = P; + return 0; +} + +int +Aconv(Fmt *fp) +{ + int i; + + i = va_arg(fp->args, int); + return fmtstrcpy(fp, anames[i]); +} + +int +Dconv(Fmt *fp) +{ + char str[STRINGSZ], s[STRINGSZ]; + Adr *a; + int i; + + a = va_arg(fp->args, Adr*); + i = a->type; + + if(fp->flags & FmtLong) { + if(i != D_CONST) { + // ATEXT dst is not constant + snprint(str, sizeof(str), "!!%D", a); + goto brk; + } + parsetextconst(a->offset); + if(textarg == 0) { + snprint(str, sizeof(str), "$%lld", textstksiz); + goto brk; + } + snprint(str, sizeof(str), "$%lld-%lld", textstksiz, textarg); + goto brk; + } + + if(i >= D_INDIR) { + if(a->offset) + snprint(str, sizeof(str), "%lld(%R)", a->offset, i-D_INDIR); + else + snprint(str, sizeof(str), "(%R)", i-D_INDIR); + goto brk; + } + switch(i) { + + default: + if(a->offset) + snprint(str, sizeof(str), "$%lld,%R", a->offset, i); + else + snprint(str, sizeof(str), "%R", i); + break; + + case D_NONE: + str[0] = 0; + break; + + case D_BRANCH: + if(bigP != P && bigP->pcond != P) + if(a->sym != S) + snprint(str, sizeof(str), "%llux+%s", bigP->pcond->pc, + a->sym->name); + else + snprint(str, sizeof(str), "%llux", bigP->pcond->pc); + else + snprint(str, sizeof(str), "%lld(PC)", a->offset); + break; + + case D_EXTERN: + if(a->sym) { + snprint(str, sizeof(str), "%s+%lld(SB)", a->sym->name, a->offset); + break; + } + snprint(str, sizeof(str), "!!noname!!+%lld(SB)", a->offset); + break; + + case D_STATIC: + if(a->sym) { + snprint(str, sizeof(str), "%s<%d>+%lld(SB)", a->sym->name, + a->sym->version, a->offset); + break; + } + snprint(str, sizeof(str), "!!noname!!<999>+%lld(SB)", a->offset); + break; + + case D_AUTO: + if(a->sym) { + snprint(str, sizeof(str), "%s+%lld(SP)", a->sym->name, a->offset); + break; + } + snprint(str, sizeof(str), "!!noname!!+%lld(SP)", a->offset); + break; + + case D_PARAM: + if(a->sym) { + snprint(str, sizeof(str), "%s+%lld(%s)", a->sym->name, a->offset, paramspace); + break; + } + snprint(str, sizeof(str), "!!noname!!+%lld(%s)", a->offset, paramspace); + break; + + case D_CONST: + snprint(str, sizeof(str), "$%lld", a->offset); + break; + + case D_FCONST: + snprint(str, sizeof(str), "$(%.8ux,%.8ux)", a->ieee.h, a->ieee.l); + break; + + case D_SCONST: + snprint(str, sizeof(str), "$\"%S\"", a->scon); + break; + + case D_ADDR: + a->type = a->index; + a->index = D_NONE; + snprint(str, sizeof(str), "$%D", a); + a->index = a->type; + a->type = D_ADDR; + goto conv; + } +brk: + if(a->index != D_NONE) { + snprint(s, sizeof(s), "(%R*%d)", a->index, a->scale); + strcat(str, s); + } +conv: + fmtstrcpy(fp, str); +// if(a->gotype) +// fmtprint(fp, "«%s»", a->gotype->name); + return 0; + +} + +char* regstr[] = +{ + "AL", /* [D_AL] */ + "CL", + "DL", + "BL", + "SPB", + "BPB", + "SIB", + "DIB", + "R8B", + "R9B", + "R10B", + "R11B", + "R12B", + "R13B", + "R14B", + "R15B", + + "AX", /* [D_AX] */ + "CX", + "DX", + "BX", + "SP", + "BP", + "SI", + "DI", + "R8", + "R9", + "R10", + "R11", + "R12", + "R13", + "R14", + "R15", + + "AH", + "CH", + "DH", + "BH", + + "F0", /* [D_F0] */ + "F1", + "F2", + "F3", + "F4", + "F5", + "F6", + "F7", + + "M0", + "M1", + "M2", + "M3", + "M4", + "M5", + "M6", + "M7", + + "X0", + "X1", + "X2", + "X3", + "X4", + "X5", + "X6", + "X7", + "X8", + "X9", + "X10", + "X11", + "X12", + "X13", + "X14", + "X15", + + "CS", /* [D_CS] */ + "SS", + "DS", + "ES", + "FS", + "GS", + + "GDTR", /* [D_GDTR] */ + "IDTR", /* [D_IDTR] */ + "LDTR", /* [D_LDTR] */ + "MSW", /* [D_MSW] */ + "TASK", /* [D_TASK] */ + + "CR0", /* [D_CR] */ + "CR1", + "CR2", + "CR3", + "CR4", + "CR5", + "CR6", + "CR7", + "CR8", + "CR9", + "CR10", + "CR11", + "CR12", + "CR13", + "CR14", + "CR15", + + "DR0", /* [D_DR] */ + "DR1", + "DR2", + "DR3", + "DR4", + "DR5", + "DR6", + "DR7", + + "TR0", /* [D_TR] */ + "TR1", + "TR2", + "TR3", + "TR4", + "TR5", + "TR6", + "TR7", + + "NONE", /* [D_NONE] */ +}; + +int +Rconv(Fmt *fp) +{ + char str[STRINGSZ]; + int r; + + r = va_arg(fp->args, int); + if(r >= D_AL && r <= D_NONE) + snprint(str, sizeof(str), "%s", regstr[r-D_AL]); + else + snprint(str, sizeof(str), "gok(%d)", r); + + return fmtstrcpy(fp, str); +} + +int +Sconv(Fmt *fp) +{ + int i, c; + char str[STRINGSZ], *p, *a; + + a = va_arg(fp->args, char*); + p = str; + for(i=0; i= 'a' && c <= 'z' || + c >= 'A' && c <= 'Z' || + c >= '0' && c <= '9') { + *p++ = c; + continue; + } + *p++ = '\\'; + switch(c) { + default: + if(c < 040 || c >= 0177) + break; /* not portable */ + p[-1] = c; + continue; + case 0: + *p++ = 'z'; + continue; + case '\\': + case '"': + *p++ = c; + continue; + case '\n': + *p++ = 'n'; + continue; + case '\t': + *p++ = 't'; + continue; + } + *p++ = (c>>6) + '0'; + *p++ = ((c>>3) & 7) + '0'; + *p++ = (c & 7) + '0'; + } + *p = 0; + return fmtstrcpy(fp, str); +} + +int +Iconv(Fmt *fp) +{ + int i, n; + uchar *p; + char *s; + Fmt fmt; + + n = fp->prec; + fp->prec = 0; + if(!(fp->flags&FmtPrec) || n < 0) + return fmtstrcpy(fp, "%I"); + fp->flags &= ~FmtPrec; + p = va_arg(fp->args, uchar*); + + // format into temporary buffer and + // call fmtstrcpy to handle padding. + fmtstrinit(&fmt); + for(i=0; iname; + sep = ": "; + } + va_start(arg, fmt); + vseprint(buf, buf+sizeof(buf), fmt, arg); + va_end(arg); + print("%s%s%s\n", tn, sep, buf); + + nerrors++; + if(nerrors > 20) { + print("too many errors\n"); + errorexit(); + } +} + +void +parsetextconst(vlong arg) +{ + textstksiz = arg & 0xffffffffLL; + if(textstksiz & 0x80000000LL) + textstksiz = -(-textstksiz & 0xffffffffLL); + + textarg = (arg >> 32) & 0xffffffffLL; + if(textarg & 0x80000000LL) + textarg = 0; + textarg = (textarg+7) & ~7LL; +} diff --git a/src/cmd/6l/mkenam b/src/cmd/6l/mkenam new file mode 100644 index 000000000..3001dbe93 --- /dev/null +++ b/src/cmd/6l/mkenam @@ -0,0 +1,45 @@ +# Inferno utils/6c/mkenam +# http://code.google.com/p/inferno-os/source/browse/utils/6c/mkenam +# +# Copyright © 1994-1999 Lucent Technologies Inc. All rights reserved. +# Portions Copyright © 1995-1997 C H Forsyth (forsyth@terzarima.net) +# Portions Copyright © 1997-1999 Vita Nuova Limited +# Portions Copyright © 2000-2007 Vita Nuova Holdings Limited (www.vitanuova.com) +# Portions Copyright © 2004,2006 Bruce Ellis +# Portions Copyright © 2005-2007 C H Forsyth (forsyth@terzarima.net) +# Revisions Copyright © 2000-2007 Lucent Technologies Inc. and others +# Portions Copyright © 2009 The Go Authors. All rights reserved. +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in +# all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +# THE SOFTWARE. + +awk ' +BEGIN { + print "char* anames[] =" + print "{" +} + +/^ A/ { + name=$1 + sub(/,/, "", name) + sub(/^A/, "", name) + print "\t\"" name "\"," +} + +END { print "};" } +' ../6l/6.out.h >enam.c diff --git a/src/cmd/6l/obj.c b/src/cmd/6l/obj.c new file mode 100644 index 000000000..a7ef58db4 --- /dev/null +++ b/src/cmd/6l/obj.c @@ -0,0 +1,765 @@ +// Inferno utils/6l/obj.c +// http://code.google.com/p/inferno-os/source/browse/utils/6l/obj.c +// +// Copyright © 1994-1999 Lucent Technologies Inc. All rights reserved. +// Portions Copyright © 1995-1997 C H Forsyth (forsyth@terzarima.net) +// Portions Copyright © 1997-1999 Vita Nuova Limited +// Portions Copyright © 2000-2007 Vita Nuova Holdings Limited (www.vitanuova.com) +// Portions Copyright © 2004,2006 Bruce Ellis +// Portions Copyright © 2005-2007 C H Forsyth (forsyth@terzarima.net) +// Revisions Copyright © 2000-2007 Lucent Technologies Inc. and others +// Portions Copyright © 2009 The Go Authors. All rights reserved. +// +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to deal +// in the Software without restriction, including without limitation the rights +// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +// copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +// THE SOFTWARE. + +// Reading object files. + +#define EXTERN +#include "l.h" +#include "../ld/lib.h" +#include "../ld/elf.h" +#include "../ld/macho.h" +#include "../ld/dwarf.h" +#include "../ld/pe.h" +#include + +char *noname = ""; +char* thestring = "amd64"; +char* paramspace = "FP"; + +Header headers[] = { + "plan9x32", Hplan9x32, + "plan9", Hplan9x64, + "elf", Helf, + "darwin", Hdarwin, + "linux", Hlinux, + "freebsd", Hfreebsd, + "openbsd", Hopenbsd, + "windows", Hwindows, + "windowsgui", Hwindows, + 0, 0 +}; + +/* + * -Hplan9x32 -T4136 -R4096 is plan9 64-bit format + * -Hplan9 -T4128 -R4096 is plan9 32-bit format + * -Helf -T0x80110000 -R4096 is ELF32 + * -Hdarwin -Tx -Rx is apple MH-exec + * -Hlinux -Tx -Rx is linux elf-exec + * -Hfreebsd -Tx -Rx is FreeBSD elf-exec + * -Hopenbsd -Tx -Rx is OpenBSD elf-exec + * -Hwindows -Tx -Rx is MS Windows PE32+ + * + * options used: 189BLQSWabcjlnpsvz + */ + +void +usage(void) +{ + fprint(2, "usage: 6l [-options] [-E entry] [-H head] [-I interpreter] [-L dir] [-T text] [-R rnd] [-r path] [-o out] main.6\n"); + exits("usage"); +} + +void +main(int argc, char *argv[]) +{ + int c; + + Binit(&bso, 1, OWRITE); + listinit(); + memset(debug, 0, sizeof(debug)); + nerrors = 0; + outfile = nil; + HEADTYPE = -1; + INITTEXT = -1; + INITDAT = -1; + INITRND = -1; + INITENTRY = 0; + + ARGBEGIN { + default: + c = ARGC(); + if(c == 'l') + usage(); + if(c >= 0 && c < sizeof(debug)) + debug[c]++; + break; + case 'o': /* output to (next arg) */ + outfile = EARGF(usage()); + break; + case 'E': + INITENTRY = EARGF(usage()); + break; + case 'H': + HEADTYPE = headtype(EARGF(usage())); + break; + case 'I': + interpreter = EARGF(usage()); + break; + case 'L': + Lflag(EARGF(usage())); + break; + case 'T': + INITTEXT = atolwhex(EARGF(usage())); + break; + case 'D': + INITDAT = atolwhex(EARGF(usage())); + break; + case 'R': + INITRND = atolwhex(EARGF(usage())); + break; + case 'r': + rpath = EARGF(usage()); + break; + case 'V': + print("%cl version %s\n", thechar, getgoversion()); + errorexit(); + } ARGEND + + if(argc != 1) + usage(); + + mywhatsys(); // get goos + + if(HEADTYPE == -1) + HEADTYPE = headtype(goos); + + if(outfile == nil) { + if(HEADTYPE == Hwindows) + outfile = "6.out.exe"; + else + outfile = "6.out"; + } + + libinit(); + + switch(HEADTYPE) { + default: + diag("unknown -H option"); + errorexit(); + case Hplan9x32: /* plan 9 */ + HEADR = 32L+8L; + if(INITTEXT == -1) + INITTEXT = 4096+HEADR; + if(INITDAT == -1) + INITDAT = 0; + if(INITRND == -1) + INITRND = 4096; + break; + case Hplan9x64: /* plan 9 */ + HEADR = 32L; + if(INITTEXT == -1) + INITTEXT = 4096+32; + if(INITDAT == -1) + INITDAT = 0; + if(INITRND == -1) + INITRND = 4096; + break; + case Helf: /* elf32 executable */ + HEADR = rnd(52L+3*32L, 16); + if(INITTEXT == -1) + INITTEXT = 0x80110000L; + if(INITDAT == -1) + INITDAT = 0; + if(INITRND == -1) + INITRND = 4096; + break; + case Hdarwin: /* apple MACH */ + /* + * OS X system constant - offset from 0(GS) to our TLS. + * Explained in ../../libcgo/darwin_amd64.c. + */ + tlsoffset = 0x8a0; + machoinit(); + HEADR = INITIAL_MACHO_HEADR; + if(INITRND == -1) + INITRND = 4096; + if(INITTEXT == -1) + INITTEXT = 4096+HEADR; + if(INITDAT == -1) + INITDAT = 0; + break; + case Hlinux: /* elf64 executable */ + case Hfreebsd: /* freebsd */ + case Hopenbsd: /* openbsd */ + /* + * ELF uses TLS offset negative from FS. + * Translate 0(FS) and 8(FS) into -16(FS) and -8(FS). + * Also known to ../../pkg/runtime/linux/amd64/sys.s + * and ../../libcgo/linux_amd64.s. + */ + tlsoffset = -16; + elfinit(); + HEADR = ELFRESERVE; + if(INITTEXT == -1) + INITTEXT = (1<<22)+HEADR; + if(INITDAT == -1) + INITDAT = 0; + if(INITRND == -1) + INITRND = 4096; + break; + case Hwindows: /* PE executable */ + peinit(); + HEADR = PEFILEHEADR; + if(INITTEXT == -1) + INITTEXT = PEBASE+PESECTHEADR; + if(INITDAT == -1) + INITDAT = 0; + if(INITRND == -1) + INITRND = PESECTALIGN; + break; + } + if(INITDAT != 0 && INITRND != 0) + print("warning: -D0x%llux is ignored because of -R0x%ux\n", + INITDAT, INITRND); + if(debug['v']) + Bprint(&bso, "HEADER = -H%d -T0x%llux -D0x%llux -R0x%ux\n", + HEADTYPE, INITTEXT, INITDAT, INITRND); + Bflush(&bso); + instinit(); + + zprg.link = P; + zprg.pcond = P; + zprg.back = 2; + zprg.as = AGOK; + zprg.from.type = D_NONE; + zprg.from.index = D_NONE; + zprg.from.scale = 1; + zprg.to = zprg.from; + zprg.mode = 64; + + pcstr = "%.6llux "; + nuxiinit(); + histgen = 0; + pc = 0; + dtype = 4; + version = 0; + cbp = buf.cbuf; + cbc = sizeof(buf.cbuf); + + addlibpath("command line", "command line", argv[0], "main"); + loadlib(); + deadcode(); + patch(); + follow(); + doelf(); + if(HEADTYPE == Hdarwin) + domacho(); + dostkoff(); + dostkcheck(); + paramspace = "SP"; /* (FP) now (SP) on output */ + if(debug['p']) + if(debug['1']) + doprof1(); + else + doprof2(); + span(); + if(HEADTYPE == Hwindows) + dope(); + addexport(); + textaddress(); + pclntab(); + symtab(); + dodata(); + address(); + doweak(); + reloc(); + asmb(); + undef(); + if(debug['v']) { + Bprint(&bso, "%5.2f cpu time\n", cputime()); + Bprint(&bso, "%d symbols\n", nsymbol); + Bprint(&bso, "%d sizeof adr\n", sizeof(Adr)); + Bprint(&bso, "%d sizeof prog\n", sizeof(Prog)); + } + Bflush(&bso); + + errorexit(); +} + +static Sym* +zsym(char *pn, Biobuf *f, Sym *h[]) +{ + int o; + + o = BGETC(f); + if(o < 0 || o >= NSYM || h[o] == nil) + mangle(pn); + return h[o]; +} + +static void +zaddr(char *pn, Biobuf *f, Adr *a, Sym *h[]) +{ + int t; + int32 l; + Sym *s; + Auto *u; + + t = BGETC(f); + a->index = D_NONE; + a->scale = 0; + if(t & T_INDEX) { + a->index = BGETC(f); + a->scale = BGETC(f); + } + a->offset = 0; + if(t & T_OFFSET) { + a->offset = Bget4(f); + if(t & T_64) { + a->offset &= 0xFFFFFFFFULL; + a->offset |= (vlong)Bget4(f) << 32; + } + } + a->sym = S; + if(t & T_SYM) + a->sym = zsym(pn, f, h); + a->type = D_NONE; + if(t & T_FCONST) { + a->ieee.l = Bget4(f); + a->ieee.h = Bget4(f); + a->type = D_FCONST; + } else + if(t & T_SCONST) { + Bread(f, a->scon, NSNAME); + a->type = D_SCONST; + } + if(t & T_TYPE) + a->type = BGETC(f); + if(a->type < 0 || a->type >= D_SIZE) + mangle(pn); + adrgotype = S; + if(t & T_GOTYPE) + adrgotype = zsym(pn, f, h); + s = a->sym; + t = a->type; + if(t == D_INDIR+D_GS) + a->offset += tlsoffset; + if(t != D_AUTO && t != D_PARAM) { + if(s && adrgotype) + s->gotype = adrgotype; + return; + } + l = a->offset; + for(u=curauto; u; u=u->link) { + if(u->asym == s) + if(u->type == t) { + if(u->aoffset > l) + u->aoffset = l; + if(adrgotype) + u->gotype = adrgotype; + return; + } + } + + switch(t) { + case D_FILE: + case D_FILE1: + case D_AUTO: + case D_PARAM: + if(s == S) + mangle(pn); + } + + u = mal(sizeof(*u)); + u->link = curauto; + curauto = u; + u->asym = s; + u->aoffset = l; + u->type = t; + u->gotype = adrgotype; +} + +void +nopout(Prog *p) +{ + p->as = ANOP; + p->from.type = D_NONE; + p->to.type = D_NONE; +} + +void +ldobj1(Biobuf *f, char *pkg, int64 len, char *pn) +{ + vlong ipc; + Prog *p; + int v, o, r, skip, mode; + Sym *h[NSYM], *s; + uint32 sig; + char *name, *x; + int ntext; + vlong eof; + char src[1024]; + Prog *lastp; + + lastp = nil; + ntext = 0; + eof = Boffset(f) + len; + src[0] = 0; + +newloop: + memset(h, 0, sizeof(h)); + version++; + histfrogp = 0; + ipc = pc; + skip = 0; + mode = 64; + +loop: + if(f->state == Bracteof || Boffset(f) >= eof) + goto eof; + o = BGETC(f); + if(o == Beof) + goto eof; + o |= BGETC(f) << 8; + if(o <= AXXX || o >= ALAST) { + if(o < 0) + goto eof; + diag("%s:#%lld: opcode out of range: %#ux", pn, Boffset(f), o); + print(" probably not a .6 file\n"); + errorexit(); + } + + if(o == ANAME || o == ASIGNAME) { + sig = 0; + if(o == ASIGNAME) + sig = Bget4(f); + v = BGETC(f); /* type */ + o = BGETC(f); /* sym */ + r = 0; + if(v == D_STATIC) + r = version; + name = Brdline(f, '\0'); + if(name == nil) { + if(Blinelen(f) > 0) { + fprint(2, "%s: name too long\n", pn); + errorexit(); + } + goto eof; + } + x = expandpkg(name, pkg); + s = lookup(x, r); + if(x != name) + free(x); + + if(debug['S'] && r == 0) + sig = 1729; + if(sig != 0){ + if(s->sig != 0 && s->sig != sig) + diag("incompatible type signatures" + "%ux(%s) and %ux(%s) for %s", + s->sig, s->file, sig, pn, s->name); + s->sig = sig; + s->file = pn; + } + + if(debug['W']) + print(" ANAME %s\n", s->name); + if(o < 0 || o >= nelem(h)) + mangle(pn); + h[o] = s; + if((v == D_EXTERN || v == D_STATIC) && s->type == 0) + s->type = SXREF; + if(v == D_FILE) { + if(s->type != SFILE) { + histgen++; + s->type = SFILE; + s->value = histgen; + } + if(histfrogp < MAXHIST) { + histfrog[histfrogp] = s; + histfrogp++; + } else + collapsefrog(s); + dwarfaddfrag(s->value, s->name); + } + goto loop; + } + + p = mal(sizeof(*p)); + p->as = o; + p->line = Bget4(f); + p->back = 2; + p->mode = mode; + p->ft = 0; + p->tt = 0; + zaddr(pn, f, &p->from, h); + fromgotype = adrgotype; + zaddr(pn, f, &p->to, h); + + switch(p->as) { + case ATEXT: + case ADATA: + case AGLOBL: + if(p->from.sym == S) + mangle(pn); + break; + } + + if(debug['W']) + print("%P\n", p); + + switch(p->as) { + case AHISTORY: + if(p->to.offset == -1) { + addlib(src, pn); + histfrogp = 0; + goto loop; + } + if(src[0] == '\0') + copyhistfrog(src, sizeof src); + addhist(p->line, D_FILE); /* 'z' */ + if(p->to.offset) + addhist(p->to.offset, D_FILE1); /* 'Z' */ + histfrogp = 0; + goto loop; + + case AEND: + histtoauto(); + if(cursym != nil && cursym->text) + cursym->autom = curauto; + curauto = 0; + cursym = nil; + if(Boffset(f) == eof) + return; + goto newloop; + + case AGLOBL: + s = p->from.sym; + if(s->type == 0 || s->type == SXREF) { + s->type = SBSS; + s->size = 0; + } + if(s->type != SBSS && !s->dupok) { + diag("%s: redefinition: %s in %s", + pn, s->name, TNAME); + s->type = SBSS; + s->size = 0; + } + if(p->to.offset > s->size) + s->size = p->to.offset; + if(p->from.scale & DUPOK) + s->dupok = 1; + if(p->from.scale & RODATA) + s->type = SRODATA; + goto loop; + + case ADATA: + // Assume that AGLOBL comes after ADATA. + // If we've seen an AGLOBL that said this sym was DUPOK, + // ignore any more ADATA we see, which must be + // redefinitions. + s = p->from.sym; + if(s->dupok) { +// if(debug['v']) +// Bprint(&bso, "skipping %s in %s: dupok\n", s->name, pn); + goto loop; + } + if(s->file == nil) + s->file = pn; + else if(s->file != pn) { + diag("multiple initialization for %s: in both %s and %s", s->name, s->file, pn); + errorexit(); + } + savedata(s, p, pn); + unmal(p, sizeof *p); + goto loop; + + case AGOK: + diag("%s: GOK opcode in %s", pn, TNAME); + pc++; + goto loop; + + case ATEXT: + s = p->from.sym; + if(s->text != nil) { + diag("%s: %s: redefinition", pn, s->name); + return; + } + if(ntext++ == 0 && s->type != 0 && s->type != SXREF) { + /* redefinition, so file has probably been seen before */ + if(debug['v']) + Bprint(&bso, "skipping: %s: redefinition: %s", pn, s->name); + return; + } + if(cursym != nil && cursym->text) { + histtoauto(); + cursym->autom = curauto; + curauto = 0; + } + skip = 0; + if(etextp) + etextp->next = s; + else + textp = s; + etextp = s; + s->text = p; + cursym = s; + if(s->type != 0 && s->type != SXREF) { + if(p->from.scale & DUPOK) { + skip = 1; + goto casdef; + } + diag("%s: redefinition: %s\n%P", pn, s->name, p); + } + if(fromgotype) { + if(s->gotype && s->gotype != fromgotype) + diag("%s: type mismatch for %s", pn, s->name); + s->gotype = fromgotype; + } + s->type = STEXT; + s->value = pc; + lastp = p; + p->pc = pc++; + goto loop; + + case AMODE: + if(p->from.type == D_CONST || p->from.type == D_INDIR+D_NONE){ + switch((int)p->from.offset){ + case 16: case 32: case 64: + mode = p->from.offset; + break; + } + } + goto loop; + + case AFMOVF: + case AFADDF: + case AFSUBF: + case AFSUBRF: + case AFMULF: + case AFDIVF: + case AFDIVRF: + case AFCOMF: + case AFCOMFP: + case AMOVSS: + case AADDSS: + case ASUBSS: + case AMULSS: + case ADIVSS: + case ACOMISS: + case AUCOMISS: + if(skip) + goto casdef; + if(p->from.type == D_FCONST) { + /* size sb 9 max */ + sprint(literal, "$%ux", ieeedtof(&p->from.ieee)); + s = lookup(literal, 0); + if(s->type == 0) { + s->type = SDATA; + adduint32(s, ieeedtof(&p->from.ieee)); + s->reachable = 0; + } + p->from.type = D_EXTERN; + p->from.sym = s; + p->from.offset = 0; + } + goto casdef; + + case AFMOVD: + case AFADDD: + case AFSUBD: + case AFSUBRD: + case AFMULD: + case AFDIVD: + case AFDIVRD: + case AFCOMD: + case AFCOMDP: + case AMOVSD: + case AADDSD: + case ASUBSD: + case AMULSD: + case ADIVSD: + case ACOMISD: + case AUCOMISD: + if(skip) + goto casdef; + if(p->from.type == D_FCONST) { + /* size sb 18 max */ + sprint(literal, "$%ux.%ux", + p->from.ieee.l, p->from.ieee.h); + s = lookup(literal, 0); + if(s->type == 0) { + s->type = SDATA; + adduint32(s, p->from.ieee.l); + adduint32(s, p->from.ieee.h); + s->reachable = 0; + } + p->from.type = D_EXTERN; + p->from.sym = s; + p->from.offset = 0; + } + goto casdef; + + casdef: + default: + if(skip) + nopout(p); + p->pc = pc; + pc++; + + if(p->to.type == D_BRANCH) + p->to.offset += ipc; + if(lastp == nil) { + if(p->as != ANOP) + diag("unexpected instruction: %P", p); + goto loop; + } + lastp->link = p; + lastp = p; + goto loop; + } + +eof: + diag("truncated object file: %s", pn); +} + +Prog* +prg(void) +{ + Prog *p; + + p = mal(sizeof(*p)); + + *p = zprg; + return p; +} + +Prog* +copyp(Prog *q) +{ + Prog *p; + + p = prg(); + *p = *q; + return p; +} + +Prog* +appendp(Prog *q) +{ + Prog *p; + + p = prg(); + p->link = q->link; + q->link = p; + p->line = q->line; + p->mode = q->mode; + return p; +} diff --git a/src/cmd/6l/optab.c b/src/cmd/6l/optab.c new file mode 100644 index 000000000..36806ec4b --- /dev/null +++ b/src/cmd/6l/optab.c @@ -0,0 +1,1279 @@ +// Inferno utils/6l/optab.c +// http://code.google.com/p/inferno-os/source/browse/utils/6l/optab.c +// +// Copyright © 1994-1999 Lucent Technologies Inc. All rights reserved. +// Portions Copyright © 1995-1997 C H Forsyth (forsyth@terzarima.net) +// Portions Copyright © 1997-1999 Vita Nuova Limited +// Portions Copyright © 2000-2007 Vita Nuova Holdings Limited (www.vitanuova.com) +// Portions Copyright © 2004,2006 Bruce Ellis +// Portions Copyright © 2005-2007 C H Forsyth (forsyth@terzarima.net) +// Revisions Copyright © 2000-2007 Lucent Technologies Inc. and others +// Portions Copyright © 2009 The Go Authors. All rights reserved. +// +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to deal +// in the Software without restriction, including without limitation the rights +// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +// copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +// THE SOFTWARE. + +#include "l.h" + +uchar ynone[] = +{ + Ynone, Ynone, Zlit, 1, + 0 +}; +uchar ytext[] = +{ + Ymb, Yi64, Zpseudo,1, + 0 +}; +uchar ynop[] = +{ + Ynone, Ynone, Zpseudo,1, + Ynone, Yml, Zpseudo,1, + Ynone, Yrf, Zpseudo,1, + Yml, Ynone, Zpseudo,1, + Yrf, Ynone, Zpseudo,1, + 0 +}; +uchar yxorb[] = +{ + Yi32, Yal, Zib_, 1, + Yi32, Ymb, Zibo_m, 2, + Yrb, Ymb, Zr_m, 1, + Ymb, Yrb, Zm_r, 1, + 0 +}; +uchar yxorl[] = +{ + Yi8, Yml, Zibo_m, 2, + Yi32, Yax, Zil_, 1, + Yi32, Yml, Zilo_m, 2, + Yrl, Yml, Zr_m, 1, + Yml, Yrl, Zm_r, 1, + 0 +}; +uchar yaddl[] = +{ + Yi8, Yml, Zibo_m, 2, + Yi32, Yax, Zil_, 1, + Yi32, Yml, Zilo_m, 2, + Yrl, Yml, Zr_m, 1, + Yml, Yrl, Zm_r, 1, + 0 +}; +uchar yincb[] = +{ + Ynone, Ymb, Zo_m, 2, + 0 +}; +uchar yincw[] = +{ + Ynone, Yml, Zo_m, 2, + 0 +}; +uchar yincl[] = +{ + Ynone, Yml, Zo_m, 2, + 0 +}; +uchar ycmpb[] = +{ + Yal, Yi32, Z_ib, 1, + Ymb, Yi32, Zm_ibo, 2, + Ymb, Yrb, Zm_r, 1, + Yrb, Ymb, Zr_m, 1, + 0 +}; +uchar ycmpl[] = +{ + Yml, Yi8, Zm_ibo, 2, + Yax, Yi32, Z_il, 1, + Yml, Yi32, Zm_ilo, 2, + Yml, Yrl, Zm_r, 1, + Yrl, Yml, Zr_m, 1, + 0 +}; +uchar yshb[] = +{ + Yi1, Ymb, Zo_m, 2, + Yi32, Ymb, Zibo_m, 2, + Ycx, Ymb, Zo_m, 2, + 0 +}; +uchar yshl[] = +{ + Yi1, Yml, Zo_m, 2, + Yi32, Yml, Zibo_m, 2, + Ycl, Yml, Zo_m, 2, + Ycx, Yml, Zo_m, 2, + 0 +}; +uchar ytestb[] = +{ + Yi32, Yal, Zib_, 1, + Yi32, Ymb, Zibo_m, 2, + Yrb, Ymb, Zr_m, 1, + Ymb, Yrb, Zm_r, 1, + 0 +}; +uchar ytestl[] = +{ + Yi32, Yax, Zil_, 1, + Yi32, Yml, Zilo_m, 2, + Yrl, Yml, Zr_m, 1, + Yml, Yrl, Zm_r, 1, + 0 +}; +uchar ymovb[] = +{ + Yrb, Ymb, Zr_m, 1, + Ymb, Yrb, Zm_r, 1, + Yi32, Yrb, Zib_rp, 1, + Yi32, Ymb, Zibo_m, 2, + 0 +}; +uchar ymbs[] = +{ + Ymb, Ynone, Zm_o, 2, + 0 +}; +uchar ybtl[] = +{ + Yi8, Yml, Zibo_m, 2, + Yrl, Yml, Zr_m, 1, + 0 +}; +uchar ymovw[] = +{ + Yrl, Yml, Zr_m, 1, + Yml, Yrl, Zm_r, 1, + Yi0, Yrl, Zclr, 1, + Yi32, Yrl, Zil_rp, 1, + Yi32, Yml, Zilo_m, 2, + Yiauto, Yrl, Zaut_r, 2, + 0 +}; +uchar ymovl[] = +{ + Yrl, Yml, Zr_m, 1, + Yml, Yrl, Zm_r, 1, + Yi0, Yrl, Zclr, 1, + Yi32, Yrl, Zil_rp, 1, + Yi32, Yml, Zilo_m, 2, + Yml, Ymr, Zm_r_xm, 1, // MMX MOVD + Ymr, Yml, Zr_m_xm, 1, // MMX MOVD + Yml, Yxr, Zm_r_xm, 2, // XMM MOVD (32 bit) + Yxr, Yml, Zr_m_xm, 2, // XMM MOVD (32 bit) + Yiauto, Yrl, Zaut_r, 2, + 0 +}; +uchar yret[] = +{ + Ynone, Ynone, Zo_iw, 1, + Yi32, Ynone, Zo_iw, 1, + 0 +}; +uchar ymovq[] = +{ + Yrl, Yml, Zr_m, 1, // 0x89 + Yml, Yrl, Zm_r, 1, // 0x8b + Yi0, Yrl, Zclr, 1, // 0x31 + Ys32, Yrl, Zilo_m, 2, // 32 bit signed 0xc7,(0) + Yi64, Yrl, Ziq_rp, 1, // 0xb8 -- 32/64 bit immediate + Yi32, Yml, Zilo_m, 2, // 0xc7,(0) + Ym, Ymr, Zm_r_xm_nr, 1, // MMX MOVQ (shorter encoding) + Ymr, Ym, Zr_m_xm_nr, 1, // MMX MOVQ + Ymm, Ymr, Zm_r_xm, 1, // MMX MOVD + Ymr, Ymm, Zr_m_xm, 1, // MMX MOVD + Yxr, Ymr, Zm_r_xm_nr, 2, // MOVDQ2Q + Yxr, Ym, Zr_m_xm_nr, 2, // MOVQ xmm store + Yml, Yxr, Zm_r_xm, 2, // MOVD xmm load + Yxr, Yml, Zr_m_xm, 2, // MOVD xmm store + Yiauto, Yrl, Zaut_r, 2, // built-in LEAQ + 0 +}; +uchar ym_rl[] = +{ + Ym, Yrl, Zm_r, 1, + 0 +}; +uchar yrl_m[] = +{ + Yrl, Ym, Zr_m, 1, + 0 +}; +uchar ymb_rl[] = +{ + Ymb, Yrl, Zmb_r, 1, + 0 +}; +uchar yml_rl[] = +{ + Yml, Yrl, Zm_r, 1, + 0 +}; +uchar yrl_ml[] = +{ + Yrl, Yml, Zr_m, 1, + 0 +}; +uchar yml_mb[] = +{ + Yrb, Ymb, Zr_m, 1, + Ymb, Yrb, Zm_r, 1, + 0 +}; +uchar yrb_mb[] = +{ + Yrb, Ymb, Zr_m, 1, + 0 +}; +uchar yml_ml[] = +{ + Yrl, Yml, Zr_m, 1, + Yml, Yrl, Zm_r, 1, + 0 +}; +uchar ydivl[] = +{ + Yml, Ynone, Zm_o, 2, + 0 +}; +uchar ydivb[] = +{ + Ymb, Ynone, Zm_o, 2, + 0 +}; +uchar yimul[] = +{ + Yml, Ynone, Zm_o, 2, + Yi8, Yrl, Zib_rr, 1, + Yi32, Yrl, Zil_rr, 1, + Yml, Yrl, Zm_r, 2, + 0 +}; +uchar ybyte[] = +{ + Yi64, Ynone, Zbyte, 1, + 0 +}; +uchar yin[] = +{ + Yi32, Ynone, Zib_, 1, + Ynone, Ynone, Zlit, 1, + 0 +}; +uchar yint[] = +{ + Yi32, Ynone, Zib_, 1, + 0 +}; +uchar ypushl[] = +{ + Yrl, Ynone, Zrp_, 1, + Ym, Ynone, Zm_o, 2, + Yi8, Ynone, Zib_, 1, + Yi32, Ynone, Zil_, 1, + 0 +}; +uchar ypopl[] = +{ + Ynone, Yrl, Z_rp, 1, + Ynone, Ym, Zo_m, 2, + 0 +}; +uchar yscond[] = +{ + Ynone, Ymb, Zo_m, 2, + 0 +}; +uchar yjcond[] = +{ + Ynone, Ybr, Zbr, 1, + 0 +}; +uchar yloop[] = +{ + Ynone, Ybr, Zloop, 1, + 0 +}; +uchar ycall[] = +{ + Ynone, Yml, Zo_m64, 2, + Ynone, Ybr, Zcall, 1, + 0 +}; +uchar yjmp[] = +{ + Ynone, Yml, Zo_m64, 2, + Ynone, Ybr, Zjmp, 1, + 0 +}; + +uchar yfmvd[] = +{ + Ym, Yf0, Zm_o, 2, + Yf0, Ym, Zo_m, 2, + Yrf, Yf0, Zm_o, 2, + Yf0, Yrf, Zo_m, 2, + 0 +}; +uchar yfmvdp[] = +{ + Yf0, Ym, Zo_m, 2, + Yf0, Yrf, Zo_m, 2, + 0 +}; +uchar yfmvf[] = +{ + Ym, Yf0, Zm_o, 2, + Yf0, Ym, Zo_m, 2, + 0 +}; +uchar yfmvx[] = +{ + Ym, Yf0, Zm_o, 2, + 0 +}; +uchar yfmvp[] = +{ + Yf0, Ym, Zo_m, 2, + 0 +}; +uchar yfadd[] = +{ + Ym, Yf0, Zm_o, 2, + Yrf, Yf0, Zm_o, 2, + Yf0, Yrf, Zo_m, 2, + 0 +}; +uchar yfaddp[] = +{ + Yf0, Yrf, Zo_m, 2, + 0 +}; +uchar yfxch[] = +{ + Yf0, Yrf, Zo_m, 2, + Yrf, Yf0, Zm_o, 2, + 0 +}; +uchar ycompp[] = +{ + Yf0, Yrf, Zo_m, 2, /* botch is really f0,f1 */ + 0 +}; +uchar ystsw[] = +{ + Ynone, Ym, Zo_m, 2, + Ynone, Yax, Zlit, 1, + 0 +}; +uchar ystcw[] = +{ + Ynone, Ym, Zo_m, 2, + Ym, Ynone, Zm_o, 2, + 0 +}; +uchar ysvrs[] = +{ + Ynone, Ym, Zo_m, 2, + Ym, Ynone, Zm_o, 2, + 0 +}; +uchar ymm[] = +{ + Ymm, Ymr, Zm_r_xm, 1, + Yxm, Yxr, Zm_r_xm, 2, + 0 +}; +uchar yxm[] = +{ + Yxm, Yxr, Zm_r_xm, 1, + 0 +}; +uchar yxcvm1[] = +{ + Yxm, Yxr, Zm_r_xm, 2, + Yxm, Ymr, Zm_r_xm, 2, + 0 +}; +uchar yxcvm2[] = +{ + Yxm, Yxr, Zm_r_xm, 2, + Ymm, Yxr, Zm_r_xm, 2, + 0 +}; +uchar yxmq[] = +{ + Yxm, Yxr, Zm_r_xm, 2, + 0 +}; +uchar yxr[] = +{ + Yxr, Yxr, Zm_r_xm, 1, + 0 +}; +uchar yxr_ml[] = +{ + Yxr, Yml, Zr_m_xm, 1, + 0 +}; +uchar ymr[] = +{ + Ymr, Ymr, Zm_r, 1, + 0 +}; +uchar ymr_ml[] = +{ + Ymr, Yml, Zr_m_xm, 1, + 0 +}; +uchar yxcmp[] = +{ + Yxm, Yxr, Zm_r_xm, 1, + 0 +}; +uchar yxcmpi[] = +{ + Yxm, Yxr, Zm_r_i_xm, 2, + 0 +}; +uchar yxmov[] = +{ + Yxm, Yxr, Zm_r_xm, 1, + Yxr, Yxm, Zr_m_xm, 1, + 0 +}; +uchar yxcvfl[] = +{ + Yxm, Yrl, Zm_r_xm, 1, + 0 +}; +uchar yxcvlf[] = +{ + Yml, Yxr, Zm_r_xm, 1, + 0 +}; +uchar yxcvfq[] = +{ + Yxm, Yrl, Zm_r_xm, 2, + 0 +}; +uchar yxcvqf[] = +{ + Yml, Yxr, Zm_r_xm, 2, + 0 +}; +uchar yps[] = +{ + Ymm, Ymr, Zm_r_xm, 1, + Yi8, Ymr, Zibo_m_xm, 2, + Yxm, Yxr, Zm_r_xm, 2, + Yi8, Yxr, Zibo_m_xm, 3, + 0 +}; +uchar yxrrl[] = +{ + Yxr, Yrl, Zm_r, 1, + 0 +}; +uchar ymfp[] = +{ + Ymm, Ymr, Zm_r_3d, 1, + 0, +}; +uchar ymrxr[] = +{ + Ymr, Yxr, Zm_r, 1, + Yxm, Yxr, Zm_r_xm, 1, + 0 +}; +uchar ymshuf[] = +{ + Ymm, Ymr, Zibm_r, 1, + 0 +}; +uchar yxshuf[] = +{ + Yxm, Yxr, Zibm_r, 1, + 0 +}; +uchar yextrw[] = +{ + Yxr, Yrl, Zibm_r, 1, + 0 +}; +uchar ypsdq[] = +{ + Yi8, Yxr, Zibo_m, 2, + 0 +}; +uchar ymskb[] = +{ + Yxr, Yrl, Zm_r_xm, 2, + Ymr, Yrl, Zm_r_xm, 1, + 0 +}; +uchar ycrc32l[] = +{ + Yml, Yrl, Zlitm_r, 0, +}; + +/* + * You are doasm, holding in your hand a Prog* with p->as set to, say, ACRC32, + * and p->from and p->to as operands (Adr*). The linker scans optab to find + * the entry with the given p->as and then looks through the ytable for that + * instruction (the second field in the optab struct) for a line whose first + * two values match the Ytypes of the p->from and p->to operands. The function + * oclass in span.c computes the specific Ytype of an operand and then the set + * of more general Ytypes that it satisfies is implied by the ycover table, set + * up in instinit. For example, oclass distinguishes the constants 0 and 1 + * from the more general 8-bit constants, but instinit says + * + * ycover[Yi0*Ymax + Ys32] = 1; + * ycover[Yi1*Ymax + Ys32] = 1; + * ycover[Yi8*Ymax + Ys32] = 1; + * + * which means that Yi0, Yi1, and Yi8 all count as Ys32 (signed 32) + * if that's what an instruction can handle. + * + * In parallel with the scan through the ytable for the appropriate line, there + * is a z pointer that starts out pointing at the strange magic byte list in + * the Optab struct. With each step past a non-matching ytable line, z + * advances by the 4th entry in the line. When a matching line is found, that + * z pointer has the extra data to use in laying down the instruction bytes. + * The actual bytes laid down are a function of the 3rd entry in the line (that + * is, the Ztype) and the z bytes. + * + * For example, let's look at AADDL. The optab line says: + * { AADDL, yaddl, Px, 0x83,(00),0x05,0x81,(00),0x01,0x03 }, + * + * and yaddl says + * uchar yaddl[] = + * { + * Yi8, Yml, Zibo_m, 2, + * Yi32, Yax, Zil_, 1, + * Yi32, Yml, Zilo_m, 2, + * Yrl, Yml, Zr_m, 1, + * Yml, Yrl, Zm_r, 1, + * 0 + * }; + * + * so there are 5 possible types of ADDL instruction that can be laid down, and + * possible states used to lay them down (Ztype and z pointer, assuming z + * points at {0x83,(00),0x05,0x81,(00),0x01,0x03}) are: + * + * Yi8, Yml -> Zibo_m, z (0x83, 00) + * Yi32, Yax -> Zil_, z+2 (0x05) + * Yi32, Yml -> Zilo_m, z+2+1 (0x81, 0x00) + * Yrl, Yml -> Zr_m, z+2+1+2 (0x01) + * Yml, Yrl -> Zm_r, z+2+1+2+1 (0x03) + * + * The Pconstant in the optab line controls the prefix bytes to emit. That's + * relatively straightforward as this program goes. + * + * The switch on t[2] in doasm implements the various Z cases. Zibo_m, for + * example, is an opcode byte (z[0]) then an asmando (which is some kind of + * encoded addressing mode for the Yml arg), and then a single immediate byte. + * Zilo_m is the same but a long (32-bit) immediate. + */ +Optab optab[] = +/* as, ytab, andproto, opcode */ +{ + { AXXX }, + { AAAA, ynone, P32, 0x37 }, + { AAAD, ynone, P32, 0xd5,0x0a }, + { AAAM, ynone, P32, 0xd4,0x0a }, + { AAAS, ynone, P32, 0x3f }, + { AADCB, yxorb, Pb, 0x14,0x80,(02),0x10,0x10 }, + { AADCL, yxorl, Px, 0x83,(02),0x15,0x81,(02),0x11,0x13 }, + { AADCQ, yxorl, Pw, 0x83,(02),0x15,0x81,(02),0x11,0x13 }, + { AADCW, yxorl, Pe, 0x83,(02),0x15,0x81,(02),0x11,0x13 }, + { AADDB, yxorb, Pb, 0x04,0x80,(00),0x00,0x02 }, + { AADDL, yaddl, Px, 0x83,(00),0x05,0x81,(00),0x01,0x03 }, + { AADDPD, yxm, Pq, 0x58 }, + { AADDPS, yxm, Pm, 0x58 }, + { AADDQ, yaddl, Pw, 0x83,(00),0x05,0x81,(00),0x01,0x03 }, + { AADDSD, yxm, Pf2, 0x58 }, + { AADDSS, yxm, Pf3, 0x58 }, + { AADDW, yaddl, Pe, 0x83,(00),0x05,0x81,(00),0x01,0x03 }, + { AADJSP }, + { AANDB, yxorb, Pb, 0x24,0x80,(04),0x20,0x22 }, + { AANDL, yxorl, Px, 0x83,(04),0x25,0x81,(04),0x21,0x23 }, + { AANDNPD, yxm, Pq, 0x55 }, + { AANDNPS, yxm, Pm, 0x55 }, + { AANDPD, yxm, Pq, 0x54 }, + { AANDPS, yxm, Pq, 0x54 }, + { AANDQ, yxorl, Pw, 0x83,(04),0x25,0x81,(04),0x21,0x23 }, + { AANDW, yxorl, Pe, 0x83,(04),0x25,0x81,(04),0x21,0x23 }, + { AARPL, yrl_ml, P32, 0x63 }, + { ABOUNDL, yrl_m, P32, 0x62 }, + { ABOUNDW, yrl_m, Pe, 0x62 }, + { ABSFL, yml_rl, Pm, 0xbc }, + { ABSFQ, yml_rl, Pw, 0x0f,0xbc }, + { ABSFW, yml_rl, Pq, 0xbc }, + { ABSRL, yml_rl, Pm, 0xbd }, + { ABSRQ, yml_rl, Pw, 0x0f,0xbd }, + { ABSRW, yml_rl, Pq, 0xbd }, + { ABTCL, ybtl, Pm, 0xba,(07),0xbb }, + { ABTCQ, ybtl, Pw, 0x0f,0xba,(07),0x0f,0xbb }, + { ABTCW, ybtl, Pq, 0xba,(07),0xbb }, + { ABTL, ybtl, Pm, 0xba,(04),0xa3 }, + { ABTQ, ybtl, Pw, 0x0f,0xba,(04),0x0f,0xa3}, + { ABTRL, ybtl, Pm, 0xba,(06),0xb3 }, + { ABTRQ, ybtl, Pw, 0x0f,0xba,(06),0x0f,0xb3 }, + { ABTRW, ybtl, Pq, 0xba,(06),0xb3 }, + { ABTSL, ybtl, Pm, 0xba,(05),0xab }, + { ABTSQ, ybtl, Pw, 0x0f,0xba,(05),0x0f,0xab }, + { ABTSW, ybtl, Pq, 0xba,(05),0xab }, + { ABTW, ybtl, Pq, 0xba,(04),0xa3 }, + { ABYTE, ybyte, Px, 1 }, + { ACALL, ycall, Px, 0xff,(02),0xe8 }, + { ACDQ, ynone, Px, 0x99 }, + { ACLC, ynone, Px, 0xf8 }, + { ACLD, ynone, Px, 0xfc }, + { ACLI, ynone, Px, 0xfa }, + { ACLTS, ynone, Pm, 0x06 }, + { ACMC, ynone, Px, 0xf5 }, + { ACMOVLCC, yml_rl, Pm, 0x43 }, + { ACMOVLCS, yml_rl, Pm, 0x42 }, + { ACMOVLEQ, yml_rl, Pm, 0x44 }, + { ACMOVLGE, yml_rl, Pm, 0x4d }, + { ACMOVLGT, yml_rl, Pm, 0x4f }, + { ACMOVLHI, yml_rl, Pm, 0x47 }, + { ACMOVLLE, yml_rl, Pm, 0x4e }, + { ACMOVLLS, yml_rl, Pm, 0x46 }, + { ACMOVLLT, yml_rl, Pm, 0x4c }, + { ACMOVLMI, yml_rl, Pm, 0x48 }, + { ACMOVLNE, yml_rl, Pm, 0x45 }, + { ACMOVLOC, yml_rl, Pm, 0x41 }, + { ACMOVLOS, yml_rl, Pm, 0x40 }, + { ACMOVLPC, yml_rl, Pm, 0x4b }, + { ACMOVLPL, yml_rl, Pm, 0x49 }, + { ACMOVLPS, yml_rl, Pm, 0x4a }, + { ACMOVQCC, yml_rl, Pw, 0x0f,0x43 }, + { ACMOVQCS, yml_rl, Pw, 0x0f,0x42 }, + { ACMOVQEQ, yml_rl, Pw, 0x0f,0x44 }, + { ACMOVQGE, yml_rl, Pw, 0x0f,0x4d }, + { ACMOVQGT, yml_rl, Pw, 0x0f,0x4f }, + { ACMOVQHI, yml_rl, Pw, 0x0f,0x47 }, + { ACMOVQLE, yml_rl, Pw, 0x0f,0x4e }, + { ACMOVQLS, yml_rl, Pw, 0x0f,0x46 }, + { ACMOVQLT, yml_rl, Pw, 0x0f,0x4c }, + { ACMOVQMI, yml_rl, Pw, 0x0f,0x48 }, + { ACMOVQNE, yml_rl, Pw, 0x0f,0x45 }, + { ACMOVQOC, yml_rl, Pw, 0x0f,0x41 }, + { ACMOVQOS, yml_rl, Pw, 0x0f,0x40 }, + { ACMOVQPC, yml_rl, Pw, 0x0f,0x4b }, + { ACMOVQPL, yml_rl, Pw, 0x0f,0x49 }, + { ACMOVQPS, yml_rl, Pw, 0x0f,0x4a }, + { ACMOVWCC, yml_rl, Pq, 0x43 }, + { ACMOVWCS, yml_rl, Pq, 0x42 }, + { ACMOVWEQ, yml_rl, Pq, 0x44 }, + { ACMOVWGE, yml_rl, Pq, 0x4d }, + { ACMOVWGT, yml_rl, Pq, 0x4f }, + { ACMOVWHI, yml_rl, Pq, 0x47 }, + { ACMOVWLE, yml_rl, Pq, 0x4e }, + { ACMOVWLS, yml_rl, Pq, 0x46 }, + { ACMOVWLT, yml_rl, Pq, 0x4c }, + { ACMOVWMI, yml_rl, Pq, 0x48 }, + { ACMOVWNE, yml_rl, Pq, 0x45 }, + { ACMOVWOC, yml_rl, Pq, 0x41 }, + { ACMOVWOS, yml_rl, Pq, 0x40 }, + { ACMOVWPC, yml_rl, Pq, 0x4b }, + { ACMOVWPL, yml_rl, Pq, 0x49 }, + { ACMOVWPS, yml_rl, Pq, 0x4a }, + { ACMPB, ycmpb, Pb, 0x3c,0x80,(07),0x38,0x3a }, + { ACMPL, ycmpl, Px, 0x83,(07),0x3d,0x81,(07),0x39,0x3b }, + { ACMPPD, yxcmpi, Px, Pe,0xc2 }, + { ACMPPS, yxcmpi, Pm, 0xc2,0 }, + { ACMPQ, ycmpl, Pw, 0x83,(07),0x3d,0x81,(07),0x39,0x3b }, + { ACMPSB, ynone, Pb, 0xa6 }, + { ACMPSD, yxcmpi, Px, Pf2,0xc2 }, + { ACMPSL, ynone, Px, 0xa7 }, + { ACMPSQ, ynone, Pw, 0xa7 }, + { ACMPSS, yxcmpi, Px, Pf3,0xc2 }, + { ACMPSW, ynone, Pe, 0xa7 }, + { ACMPW, ycmpl, Pe, 0x83,(07),0x3d,0x81,(07),0x39,0x3b }, + { ACOMISD, yxcmp, Pe, 0x2f }, + { ACOMISS, yxcmp, Pm, 0x2f }, + { ACPUID, ynone, Pm, 0xa2 }, + { ACVTPL2PD, yxcvm2, Px, Pf3,0xe6,Pe,0x2a }, + { ACVTPL2PS, yxcvm2, Pm, 0x5b,0,0x2a,0, }, + { ACVTPD2PL, yxcvm1, Px, Pf2,0xe6,Pe,0x2d }, + { ACVTPD2PS, yxm, Pe, 0x5a }, + { ACVTPS2PL, yxcvm1, Px, Pe,0x5b,Pm,0x2d }, + { ACVTPS2PD, yxm, Pm, 0x5a }, + { API2FW, ymfp, Px, 0x0c }, + { ACVTSD2SL, yxcvfl, Pf2, 0x2d }, + { ACVTSD2SQ, yxcvfq, Pw, Pf2,0x2d }, + { ACVTSD2SS, yxm, Pf2, 0x5a }, + { ACVTSL2SD, yxcvlf, Pf2, 0x2a }, + { ACVTSQ2SD, yxcvqf, Pw, Pf2,0x2a }, + { ACVTSL2SS, yxcvlf, Pf3, 0x2a }, + { ACVTSQ2SS, yxcvqf, Pw, Pf3,0x2a }, + { ACVTSS2SD, yxm, Pf3, 0x5a }, + { ACVTSS2SL, yxcvfl, Pf3, 0x2d }, + { ACVTSS2SQ, yxcvfq, Pw, Pf3,0x2d }, + { ACVTTPD2PL, yxcvm1, Px, Pe,0xe6,Pe,0x2c }, + { ACVTTPS2PL, yxcvm1, Px, Pf3,0x5b,Pm,0x2c }, + { ACVTTSD2SL, yxcvfl, Pf2, 0x2c }, + { ACVTTSD2SQ, yxcvfq, Pw, Pf2,0x2c }, + { ACVTTSS2SL, yxcvfl, Pf3, 0x2c }, + { ACVTTSS2SQ, yxcvfq, Pw, Pf3,0x2c }, + { ACWD, ynone, Pe, 0x99 }, + { ACQO, ynone, Pw, 0x99 }, + { ADAA, ynone, P32, 0x27 }, + { ADAS, ynone, P32, 0x2f }, + { ADATA }, + { ADECB, yincb, Pb, 0xfe,(01) }, + { ADECL, yincl, Px, 0xff,(01) }, + { ADECQ, yincl, Pw, 0xff,(01) }, + { ADECW, yincw, Pe, 0xff,(01) }, + { ADIVB, ydivb, Pb, 0xf6,(06) }, + { ADIVL, ydivl, Px, 0xf7,(06) }, + { ADIVPD, yxm, Pe, 0x5e }, + { ADIVPS, yxm, Pm, 0x5e }, + { ADIVQ, ydivl, Pw, 0xf7,(06) }, + { ADIVSD, yxm, Pf2, 0x5e }, + { ADIVSS, yxm, Pf3, 0x5e }, + { ADIVW, ydivl, Pe, 0xf7,(06) }, + { AEMMS, ynone, Pm, 0x77 }, + { AENTER }, /* botch */ + { AFXRSTOR, ysvrs, Pm, 0xae,(01),0xae,(01) }, + { AFXSAVE, ysvrs, Pm, 0xae,(00),0xae,(00) }, + { AFXRSTOR64, ysvrs, Pw, 0x0f,0xae,(01),0x0f,0xae,(01) }, + { AFXSAVE64, ysvrs, Pw, 0x0f,0xae,(00),0x0f,0xae,(00) }, + { AGLOBL }, + { AGOK }, + { AHISTORY }, + { AHLT, ynone, Px, 0xf4 }, + { AIDIVB, ydivb, Pb, 0xf6,(07) }, + { AIDIVL, ydivl, Px, 0xf7,(07) }, + { AIDIVQ, ydivl, Pw, 0xf7,(07) }, + { AIDIVW, ydivl, Pe, 0xf7,(07) }, + { AIMULB, ydivb, Pb, 0xf6,(05) }, + { AIMULL, yimul, Px, 0xf7,(05),0x6b,0x69,Pm,0xaf }, + { AIMULQ, yimul, Pw, 0xf7,(05),0x6b,0x69,Pm,0xaf }, + { AIMULW, yimul, Pe, 0xf7,(05),0x6b,0x69,Pm,0xaf }, + { AINB, yin, Pb, 0xe4,0xec }, + { AINCB, yincb, Pb, 0xfe,(00) }, + { AINCL, yincl, Px, 0xff,(00) }, + { AINCQ, yincl, Pw, 0xff,(00) }, + { AINCW, yincw, Pe, 0xff,(00) }, + { AINL, yin, Px, 0xe5,0xed }, + { AINSB, ynone, Pb, 0x6c }, + { AINSL, ynone, Px, 0x6d }, + { AINSW, ynone, Pe, 0x6d }, + { AINT, yint, Px, 0xcd }, + { AINTO, ynone, P32, 0xce }, + { AINW, yin, Pe, 0xe5,0xed }, + { AIRETL, ynone, Px, 0xcf }, + { AIRETQ, ynone, Pw, 0xcf }, + { AIRETW, ynone, Pe, 0xcf }, + { AJCC, yjcond, Px, 0x73,0x83,(00) }, + { AJCS, yjcond, Px, 0x72,0x82 }, + { AJCXZ, yloop, Px, 0xe3 }, + { AJEQ, yjcond, Px, 0x74,0x84 }, + { AJGE, yjcond, Px, 0x7d,0x8d }, + { AJGT, yjcond, Px, 0x7f,0x8f }, + { AJHI, yjcond, Px, 0x77,0x87 }, + { AJLE, yjcond, Px, 0x7e,0x8e }, + { AJLS, yjcond, Px, 0x76,0x86 }, + { AJLT, yjcond, Px, 0x7c,0x8c }, + { AJMI, yjcond, Px, 0x78,0x88 }, + { AJMP, yjmp, Px, 0xff,(04),0xeb,0xe9 }, + { AJNE, yjcond, Px, 0x75,0x85 }, + { AJOC, yjcond, Px, 0x71,0x81,(00) }, + { AJOS, yjcond, Px, 0x70,0x80,(00) }, + { AJPC, yjcond, Px, 0x7b,0x8b }, + { AJPL, yjcond, Px, 0x79,0x89 }, + { AJPS, yjcond, Px, 0x7a,0x8a }, + { ALAHF, ynone, Px, 0x9f }, + { ALARL, yml_rl, Pm, 0x02 }, + { ALARW, yml_rl, Pq, 0x02 }, + { ALDMXCSR, ysvrs, Pm, 0xae,(02),0xae,(02) }, + { ALEAL, ym_rl, Px, 0x8d }, + { ALEAQ, ym_rl, Pw, 0x8d }, + { ALEAVEL, ynone, P32, 0xc9 }, + { ALEAVEQ, ynone, Py, 0xc9 }, + { ALEAVEW, ynone, Pe, 0xc9 }, + { ALEAW, ym_rl, Pe, 0x8d }, + { ALOCK, ynone, Px, 0xf0 }, + { ALODSB, ynone, Pb, 0xac }, + { ALODSL, ynone, Px, 0xad }, + { ALODSQ, ynone, Pw, 0xad }, + { ALODSW, ynone, Pe, 0xad }, + { ALONG, ybyte, Px, 4 }, + { ALOOP, yloop, Px, 0xe2 }, + { ALOOPEQ, yloop, Px, 0xe1 }, + { ALOOPNE, yloop, Px, 0xe0 }, + { ALSLL, yml_rl, Pm, 0x03 }, + { ALSLW, yml_rl, Pq, 0x03 }, + { AMASKMOVOU, yxr, Pe, 0xf7 }, + { AMASKMOVQ, ymr, Pm, 0xf7 }, + { AMAXPD, yxm, Pe, 0x5f }, + { AMAXPS, yxm, Pm, 0x5f }, + { AMAXSD, yxm, Pf2, 0x5f }, + { AMAXSS, yxm, Pf3, 0x5f }, + { AMINPD, yxm, Pe, 0x5d }, + { AMINPS, yxm, Pm, 0x5d }, + { AMINSD, yxm, Pf2, 0x5d }, + { AMINSS, yxm, Pf3, 0x5d }, + { AMOVAPD, yxmov, Pe, 0x28,0x29 }, + { AMOVAPS, yxmov, Pm, 0x28,0x29 }, + { AMOVB, ymovb, Pb, 0x88,0x8a,0xb0,0xc6,(00) }, + { AMOVBLSX, ymb_rl, Pm, 0xbe }, + { AMOVBLZX, ymb_rl, Pm, 0xb6 }, + { AMOVBQSX, ymb_rl, Pw, 0x0f,0xbe }, + { AMOVBQZX, ymb_rl, Pw, 0x0f,0xb6 }, + { AMOVBWSX, ymb_rl, Pq, 0xbe }, + { AMOVBWZX, ymb_rl, Pq, 0xb6 }, + { AMOVO, yxmov, Pe, 0x6f,0x7f }, + { AMOVOU, yxmov, Pf3, 0x6f,0x7f }, + { AMOVHLPS, yxr, Pm, 0x12 }, + { AMOVHPD, yxmov, Pe, 0x16,0x17 }, + { AMOVHPS, yxmov, Pm, 0x16,0x17 }, + { AMOVL, ymovl, Px, 0x89,0x8b,0x31,0xb8,0xc7,(00),0x6e,0x7e,Pe,0x6e,Pe,0x7e }, + { AMOVLHPS, yxr, Pm, 0x16 }, + { AMOVLPD, yxmov, Pe, 0x12,0x13 }, + { AMOVLPS, yxmov, Pm, 0x12,0x13 }, + { AMOVLQSX, yml_rl, Pw, 0x63 }, + { AMOVLQZX, yml_rl, Px, 0x8b }, + { AMOVMSKPD, yxrrl, Pq, 0x50 }, + { AMOVMSKPS, yxrrl, Pm, 0x50 }, + { AMOVNTO, yxr_ml, Pe, 0xe7 }, + { AMOVNTPD, yxr_ml, Pe, 0x2b }, + { AMOVNTPS, yxr_ml, Pm, 0x2b }, + { AMOVNTQ, ymr_ml, Pm, 0xe7 }, + { AMOVQ, ymovq, Pw, 0x89,0x8b,0x31,0xc7,(00),0xb8,0xc7,(00),0x6f,0x7f,0x6e,0x7e,Pf2,0xd6,Pe,0xd6,Pe,0x6e,Pe,0x7e }, + { AMOVQOZX, ymrxr, Pf3, 0xd6,0x7e }, + { AMOVSB, ynone, Pb, 0xa4 }, + { AMOVSD, yxmov, Pf2, 0x10,0x11 }, + { AMOVSL, ynone, Px, 0xa5 }, + { AMOVSQ, ynone, Pw, 0xa5 }, + { AMOVSS, yxmov, Pf3, 0x10,0x11 }, + { AMOVSW, ynone, Pe, 0xa5 }, + { AMOVUPD, yxmov, Pe, 0x10,0x11 }, + { AMOVUPS, yxmov, Pm, 0x10,0x11 }, + { AMOVW, ymovw, Pe, 0x89,0x8b,0x31,0xb8,0xc7,(00) }, + { AMOVWLSX, yml_rl, Pm, 0xbf }, + { AMOVWLZX, yml_rl, Pm, 0xb7 }, + { AMOVWQSX, yml_rl, Pw, 0x0f,0xbf }, + { AMOVWQZX, yml_rl, Pw, 0x0f,0xb7 }, + { AMULB, ydivb, Pb, 0xf6,(04) }, + { AMULL, ydivl, Px, 0xf7,(04) }, + { AMULPD, yxm, Pe, 0x59 }, + { AMULPS, yxm, Ym, 0x59 }, + { AMULQ, ydivl, Pw, 0xf7,(04) }, + { AMULSD, yxm, Pf2, 0x59 }, + { AMULSS, yxm, Pf3, 0x59 }, + { AMULW, ydivl, Pe, 0xf7,(04) }, + { ANAME }, + { ANEGB, yscond, Pb, 0xf6,(03) }, + { ANEGL, yscond, Px, 0xf7,(03) }, + { ANEGQ, yscond, Pw, 0xf7,(03) }, + { ANEGW, yscond, Pe, 0xf7,(03) }, + { ANOP, ynop, Px, 0,0 }, + { ANOTB, yscond, Pb, 0xf6,(02) }, + { ANOTL, yscond, Px, 0xf7,(02) }, + { ANOTQ, yscond, Pw, 0xf7,(02) }, + { ANOTW, yscond, Pe, 0xf7,(02) }, + { AORB, yxorb, Pb, 0x0c,0x80,(01),0x08,0x0a }, + { AORL, yxorl, Px, 0x83,(01),0x0d,0x81,(01),0x09,0x0b }, + { AORPD, yxm, Pq, 0x56 }, + { AORPS, yxm, Pm, 0x56 }, + { AORQ, yxorl, Pw, 0x83,(01),0x0d,0x81,(01),0x09,0x0b }, + { AORW, yxorl, Pe, 0x83,(01),0x0d,0x81,(01),0x09,0x0b }, + { AOUTB, yin, Pb, 0xe6,0xee }, + { AOUTL, yin, Px, 0xe7,0xef }, + { AOUTSB, ynone, Pb, 0x6e }, + { AOUTSL, ynone, Px, 0x6f }, + { AOUTSW, ynone, Pe, 0x6f }, + { AOUTW, yin, Pe, 0xe7,0xef }, + { APACKSSLW, ymm, Py, 0x6b,Pe,0x6b }, + { APACKSSWB, ymm, Py, 0x63,Pe,0x63 }, + { APACKUSWB, ymm, Py, 0x67,Pe,0x67 }, + { APADDB, ymm, Py, 0xfc,Pe,0xfc }, + { APADDL, ymm, Py, 0xfe,Pe,0xfe }, + { APADDQ, yxm, Pe, 0xd4 }, + { APADDSB, ymm, Py, 0xec,Pe,0xec }, + { APADDSW, ymm, Py, 0xed,Pe,0xed }, + { APADDUSB, ymm, Py, 0xdc,Pe,0xdc }, + { APADDUSW, ymm, Py, 0xdd,Pe,0xdd }, + { APADDW, ymm, Py, 0xfd,Pe,0xfd }, + { APAND, ymm, Py, 0xdb,Pe,0xdb }, + { APANDN, ymm, Py, 0xdf,Pe,0xdf }, + { APAUSE, ynone, Px, 0xf3,0x90 }, + { APAVGB, ymm, Py, 0xe0,Pe,0xe0 }, + { APAVGW, ymm, Py, 0xe3,Pe,0xe3 }, + { APCMPEQB, ymm, Py, 0x74,Pe,0x74 }, + { APCMPEQL, ymm, Py, 0x76,Pe,0x76 }, + { APCMPEQW, ymm, Py, 0x75,Pe,0x75 }, + { APCMPGTB, ymm, Py, 0x64,Pe,0x64 }, + { APCMPGTL, ymm, Py, 0x66,Pe,0x66 }, + { APCMPGTW, ymm, Py, 0x65,Pe,0x65 }, + { APEXTRW, yextrw, Pq, 0xc5 }, + { APF2IL, ymfp, Px, 0x1d }, + { APF2IW, ymfp, Px, 0x1c }, + { API2FL, ymfp, Px, 0x0d }, + { APFACC, ymfp, Px, 0xae }, + { APFADD, ymfp, Px, 0x9e }, + { APFCMPEQ, ymfp, Px, 0xb0 }, + { APFCMPGE, ymfp, Px, 0x90 }, + { APFCMPGT, ymfp, Px, 0xa0 }, + { APFMAX, ymfp, Px, 0xa4 }, + { APFMIN, ymfp, Px, 0x94 }, + { APFMUL, ymfp, Px, 0xb4 }, + { APFNACC, ymfp, Px, 0x8a }, + { APFPNACC, ymfp, Px, 0x8e }, + { APFRCP, ymfp, Px, 0x96 }, + { APFRCPIT1, ymfp, Px, 0xa6 }, + { APFRCPI2T, ymfp, Px, 0xb6 }, + { APFRSQIT1, ymfp, Px, 0xa7 }, + { APFRSQRT, ymfp, Px, 0x97 }, + { APFSUB, ymfp, Px, 0x9a }, + { APFSUBR, ymfp, Px, 0xaa }, + { APINSRW, yextrw, Pq, 0xc4 }, + { APMADDWL, ymm, Py, 0xf5,Pe,0xf5 }, + { APMAXSW, yxm, Pe, 0xee }, + { APMAXUB, yxm, Pe, 0xde }, + { APMINSW, yxm, Pe, 0xea }, + { APMINUB, yxm, Pe, 0xda }, + { APMOVMSKB, ymskb, Px, Pe,0xd7,0xd7 }, + { APMULHRW, ymfp, Px, 0xb7 }, + { APMULHUW, ymm, Py, 0xe4,Pe,0xe4 }, + { APMULHW, ymm, Py, 0xe5,Pe,0xe5 }, + { APMULLW, ymm, Py, 0xd5,Pe,0xd5 }, + { APMULULQ, ymm, Py, 0xf4,Pe,0xf4 }, + { APOPAL, ynone, P32, 0x61 }, + { APOPAW, ynone, Pe, 0x61 }, + { APOPFL, ynone, P32, 0x9d }, + { APOPFQ, ynone, Py, 0x9d }, + { APOPFW, ynone, Pe, 0x9d }, + { APOPL, ypopl, P32, 0x58,0x8f,(00) }, + { APOPQ, ypopl, Py, 0x58,0x8f,(00) }, + { APOPW, ypopl, Pe, 0x58,0x8f,(00) }, + { APOR, ymm, Py, 0xeb,Pe,0xeb }, + { APSADBW, yxm, Pq, 0xf6 }, + { APSHUFHW, yxshuf, Pf3, 0x70 }, + { APSHUFL, yxshuf, Pq, 0x70 }, + { APSHUFLW, yxshuf, Pf2, 0x70 }, + { APSHUFW, ymshuf, Pm, 0x70 }, + { APSLLO, ypsdq, Pq, 0x73,(07) }, + { APSLLL, yps, Py, 0xf2, 0x72,(06), Pe,0xf2, Pe,0x72,(06) }, + { APSLLQ, yps, Py, 0xf3, 0x73,(06), Pe,0xf3, Pe,0x7e,(06) }, + { APSLLW, yps, Py, 0xf1, 0x71,(06), Pe,0xf1, Pe,0x71,(06) }, + { APSRAL, yps, Py, 0xe2, 0x72,(04), Pe,0xe2, Pe,0x72,(04) }, + { APSRAW, yps, Py, 0xe1, 0x71,(04), Pe,0xe1, Pe,0x71,(04) }, + { APSRLO, ypsdq, Pq, 0x73,(03) }, + { APSRLL, yps, Py, 0xd2, 0x72,(02), Pe,0xd2, Pe,0x72,(02) }, + { APSRLQ, yps, Py, 0xd3, 0x73,(02), Pe,0xd3, Pe,0x73,(02) }, + { APSRLW, yps, Py, 0xd1, 0x71,(02), Pe,0xe1, Pe,0x71,(02) }, + { APSUBB, yxm, Pe, 0xf8 }, + { APSUBL, yxm, Pe, 0xfa }, + { APSUBQ, yxm, Pe, 0xfb }, + { APSUBSB, yxm, Pe, 0xe8 }, + { APSUBSW, yxm, Pe, 0xe9 }, + { APSUBUSB, yxm, Pe, 0xd8 }, + { APSUBUSW, yxm, Pe, 0xd9 }, + { APSUBW, yxm, Pe, 0xf9 }, + { APSWAPL, ymfp, Px, 0xbb }, + { APUNPCKHBW, ymm, Py, 0x68,Pe,0x68 }, + { APUNPCKHLQ, ymm, Py, 0x6a,Pe,0x6a }, + { APUNPCKHQDQ, yxm, Pe, 0x6d }, + { APUNPCKHWL, ymm, Py, 0x69,Pe,0x69 }, + { APUNPCKLBW, ymm, Py, 0x60,Pe,0x60 }, + { APUNPCKLLQ, ymm, Py, 0x62,Pe,0x62 }, + { APUNPCKLQDQ, yxm, Pe, 0x6c }, + { APUNPCKLWL, ymm, Py, 0x61,Pe,0x61 }, + { APUSHAL, ynone, P32, 0x60 }, + { APUSHAW, ynone, Pe, 0x60 }, + { APUSHFL, ynone, P32, 0x9c }, + { APUSHFQ, ynone, Py, 0x9c }, + { APUSHFW, ynone, Pe, 0x9c }, + { APUSHL, ypushl, P32, 0x50,0xff,(06),0x6a,0x68 }, + { APUSHQ, ypushl, Py, 0x50,0xff,(06),0x6a,0x68 }, + { APUSHW, ypushl, Pe, 0x50,0xff,(06),0x6a,0x68 }, + { APXOR, ymm, Py, 0xef,Pe,0xef }, + { AQUAD, ybyte, Px, 8 }, + { ARCLB, yshb, Pb, 0xd0,(02),0xc0,(02),0xd2,(02) }, + { ARCLL, yshl, Px, 0xd1,(02),0xc1,(02),0xd3,(02),0xd3,(02) }, + { ARCLQ, yshl, Pw, 0xd1,(02),0xc1,(02),0xd3,(02),0xd3,(02) }, + { ARCLW, yshl, Pe, 0xd1,(02),0xc1,(02),0xd3,(02),0xd3,(02) }, + { ARCPPS, yxm, Pm, 0x53 }, + { ARCPSS, yxm, Pf3, 0x53 }, + { ARCRB, yshb, Pb, 0xd0,(03),0xc0,(03),0xd2,(03) }, + { ARCRL, yshl, Px, 0xd1,(03),0xc1,(03),0xd3,(03),0xd3,(03) }, + { ARCRQ, yshl, Pw, 0xd1,(03),0xc1,(03),0xd3,(03),0xd3,(03) }, + { ARCRW, yshl, Pe, 0xd1,(03),0xc1,(03),0xd3,(03),0xd3,(03) }, + { AREP, ynone, Px, 0xf3 }, + { AREPN, ynone, Px, 0xf2 }, + { ARET, ynone, Px, 0xc3 }, + { ARETFW, yret, Pe, 0xcb,0xca }, + { ARETFL, yret, Px, 0xcb,0xca }, + { ARETFQ, yret, Pw, 0xcb,0xca }, + { AROLB, yshb, Pb, 0xd0,(00),0xc0,(00),0xd2,(00) }, + { AROLL, yshl, Px, 0xd1,(00),0xc1,(00),0xd3,(00),0xd3,(00) }, + { AROLQ, yshl, Pw, 0xd1,(00),0xc1,(00),0xd3,(00),0xd3,(00) }, + { AROLW, yshl, Pe, 0xd1,(00),0xc1,(00),0xd3,(00),0xd3,(00) }, + { ARORB, yshb, Pb, 0xd0,(01),0xc0,(01),0xd2,(01) }, + { ARORL, yshl, Px, 0xd1,(01),0xc1,(01),0xd3,(01),0xd3,(01) }, + { ARORQ, yshl, Pw, 0xd1,(01),0xc1,(01),0xd3,(01),0xd3,(01) }, + { ARORW, yshl, Pe, 0xd1,(01),0xc1,(01),0xd3,(01),0xd3,(01) }, + { ARSQRTPS, yxm, Pm, 0x52 }, + { ARSQRTSS, yxm, Pf3, 0x52 }, + { ASAHF, ynone, Px, 0x86,0xe0,0x50,0x9d }, /* XCHGB AH,AL; PUSH AX; POPFL */ + { ASALB, yshb, Pb, 0xd0,(04),0xc0,(04),0xd2,(04) }, + { ASALL, yshl, Px, 0xd1,(04),0xc1,(04),0xd3,(04),0xd3,(04) }, + { ASALQ, yshl, Pw, 0xd1,(04),0xc1,(04),0xd3,(04),0xd3,(04) }, + { ASALW, yshl, Pe, 0xd1,(04),0xc1,(04),0xd3,(04),0xd3,(04) }, + { ASARB, yshb, Pb, 0xd0,(07),0xc0,(07),0xd2,(07) }, + { ASARL, yshl, Px, 0xd1,(07),0xc1,(07),0xd3,(07),0xd3,(07) }, + { ASARQ, yshl, Pw, 0xd1,(07),0xc1,(07),0xd3,(07),0xd3,(07) }, + { ASARW, yshl, Pe, 0xd1,(07),0xc1,(07),0xd3,(07),0xd3,(07) }, + { ASBBB, yxorb, Pb, 0x1c,0x80,(03),0x18,0x1a }, + { ASBBL, yxorl, Px, 0x83,(03),0x1d,0x81,(03),0x19,0x1b }, + { ASBBQ, yxorl, Pw, 0x83,(03),0x1d,0x81,(03),0x19,0x1b }, + { ASBBW, yxorl, Pe, 0x83,(03),0x1d,0x81,(03),0x19,0x1b }, + { ASCASB, ynone, Pb, 0xae }, + { ASCASL, ynone, Px, 0xaf }, + { ASCASQ, ynone, Pw, 0xaf }, + { ASCASW, ynone, Pe, 0xaf }, + { ASETCC, yscond, Pm, 0x93,(00) }, + { ASETCS, yscond, Pm, 0x92,(00) }, + { ASETEQ, yscond, Pm, 0x94,(00) }, + { ASETGE, yscond, Pm, 0x9d,(00) }, + { ASETGT, yscond, Pm, 0x9f,(00) }, + { ASETHI, yscond, Pm, 0x97,(00) }, + { ASETLE, yscond, Pm, 0x9e,(00) }, + { ASETLS, yscond, Pm, 0x96,(00) }, + { ASETLT, yscond, Pm, 0x9c,(00) }, + { ASETMI, yscond, Pm, 0x98,(00) }, + { ASETNE, yscond, Pm, 0x95,(00) }, + { ASETOC, yscond, Pm, 0x91,(00) }, + { ASETOS, yscond, Pm, 0x90,(00) }, + { ASETPC, yscond, Pm, 0x96,(00) }, + { ASETPL, yscond, Pm, 0x99,(00) }, + { ASETPS, yscond, Pm, 0x9a,(00) }, + { ASHLB, yshb, Pb, 0xd0,(04),0xc0,(04),0xd2,(04) }, + { ASHLL, yshl, Px, 0xd1,(04),0xc1,(04),0xd3,(04),0xd3,(04) }, + { ASHLQ, yshl, Pw, 0xd1,(04),0xc1,(04),0xd3,(04),0xd3,(04) }, + { ASHLW, yshl, Pe, 0xd1,(04),0xc1,(04),0xd3,(04),0xd3,(04) }, + { ASHRB, yshb, Pb, 0xd0,(05),0xc0,(05),0xd2,(05) }, + { ASHRL, yshl, Px, 0xd1,(05),0xc1,(05),0xd3,(05),0xd3,(05) }, + { ASHRQ, yshl, Pw, 0xd1,(05),0xc1,(05),0xd3,(05),0xd3,(05) }, + { ASHRW, yshl, Pe, 0xd1,(05),0xc1,(05),0xd3,(05),0xd3,(05) }, + { ASHUFPD, yxshuf, Pq, 0xc6 }, + { ASHUFPS, yxshuf, Pm, 0xc6 }, + { ASQRTPD, yxm, Pe, 0x51 }, + { ASQRTPS, yxm, Pm, 0x51 }, + { ASQRTSD, yxm, Pf2, 0x51 }, + { ASQRTSS, yxm, Pf3, 0x51 }, + { ASTC, ynone, Px, 0xf9 }, + { ASTD, ynone, Px, 0xfd }, + { ASTI, ynone, Px, 0xfb }, + { ASTMXCSR, ysvrs, Pm, 0xae,(03),0xae,(03) }, + { ASTOSB, ynone, Pb, 0xaa }, + { ASTOSL, ynone, Px, 0xab }, + { ASTOSQ, ynone, Pw, 0xab }, + { ASTOSW, ynone, Pe, 0xab }, + { ASUBB, yxorb, Pb, 0x2c,0x80,(05),0x28,0x2a }, + { ASUBL, yaddl, Px, 0x83,(05),0x2d,0x81,(05),0x29,0x2b }, + { ASUBPD, yxm, Pe, 0x5c }, + { ASUBPS, yxm, Pm, 0x5c }, + { ASUBQ, yaddl, Pw, 0x83,(05),0x2d,0x81,(05),0x29,0x2b }, + { ASUBSD, yxm, Pf2, 0x5c }, + { ASUBSS, yxm, Pf3, 0x5c }, + { ASUBW, yaddl, Pe, 0x83,(05),0x2d,0x81,(05),0x29,0x2b }, + { ASWAPGS, ynone, Pm, 0x01,0xf8 }, + { ASYSCALL, ynone, Px, 0x0f,0x05 }, /* fast syscall */ + { ATESTB, ytestb, Pb, 0xa8,0xf6,(00),0x84,0x84 }, + { ATESTL, ytestl, Px, 0xa9,0xf7,(00),0x85,0x85 }, + { ATESTQ, ytestl, Pw, 0xa9,0xf7,(00),0x85,0x85 }, + { ATESTW, ytestl, Pe, 0xa9,0xf7,(00),0x85,0x85 }, + { ATEXT, ytext, Px }, + { AUCOMISD, yxcmp, Pe, 0x2e }, + { AUCOMISS, yxcmp, Pm, 0x2e }, + { AUNPCKHPD, yxm, Pe, 0x15 }, + { AUNPCKHPS, yxm, Pm, 0x15 }, + { AUNPCKLPD, yxm, Pe, 0x14 }, + { AUNPCKLPS, yxm, Pm, 0x14 }, + { AVERR, ydivl, Pm, 0x00,(04) }, + { AVERW, ydivl, Pm, 0x00,(05) }, + { AWAIT, ynone, Px, 0x9b }, + { AWORD, ybyte, Px, 2 }, + { AXCHGB, yml_mb, Pb, 0x86,0x86 }, + { AXCHGL, yml_ml, Px, 0x87,0x87 }, + { AXCHGQ, yml_ml, Pw, 0x87,0x87 }, + { AXCHGW, yml_ml, Pe, 0x87,0x87 }, + { AXLAT, ynone, Px, 0xd7 }, + { AXORB, yxorb, Pb, 0x34,0x80,(06),0x30,0x32 }, + { AXORL, yxorl, Px, 0x83,(06),0x35,0x81,(06),0x31,0x33 }, + { AXORPD, yxm, Pe, 0x57 }, + { AXORPS, yxm, Pm, 0x57 }, + { AXORQ, yxorl, Pw, 0x83,(06),0x35,0x81,(06),0x31,0x33 }, + { AXORW, yxorl, Pe, 0x83,(06),0x35,0x81,(06),0x31,0x33 }, + + { AFMOVB, yfmvx, Px, 0xdf,(04) }, + { AFMOVBP, yfmvp, Px, 0xdf,(06) }, + { AFMOVD, yfmvd, Px, 0xdd,(00),0xdd,(02),0xd9,(00),0xdd,(02) }, + { AFMOVDP, yfmvdp, Px, 0xdd,(03),0xdd,(03) }, + { AFMOVF, yfmvf, Px, 0xd9,(00),0xd9,(02) }, + { AFMOVFP, yfmvp, Px, 0xd9,(03) }, + { AFMOVL, yfmvf, Px, 0xdb,(00),0xdb,(02) }, + { AFMOVLP, yfmvp, Px, 0xdb,(03) }, + { AFMOVV, yfmvx, Px, 0xdf,(05) }, + { AFMOVVP, yfmvp, Px, 0xdf,(07) }, + { AFMOVW, yfmvf, Px, 0xdf,(00),0xdf,(02) }, + { AFMOVWP, yfmvp, Px, 0xdf,(03) }, + { AFMOVX, yfmvx, Px, 0xdb,(05) }, + { AFMOVXP, yfmvp, Px, 0xdb,(07) }, + + { AFCOMB }, + { AFCOMBP }, + { AFCOMD, yfadd, Px, 0xdc,(02),0xd8,(02),0xdc,(02) }, /* botch */ + { AFCOMDP, yfadd, Px, 0xdc,(03),0xd8,(03),0xdc,(03) }, /* botch */ + { AFCOMDPP, ycompp, Px, 0xde,(03) }, + { AFCOMF, yfmvx, Px, 0xd8,(02) }, + { AFCOMFP, yfmvx, Px, 0xd8,(03) }, + { AFCOML, yfmvx, Px, 0xda,(02) }, + { AFCOMLP, yfmvx, Px, 0xda,(03) }, + { AFCOMW, yfmvx, Px, 0xde,(02) }, + { AFCOMWP, yfmvx, Px, 0xde,(03) }, + + { AFUCOM, ycompp, Px, 0xdd,(04) }, + { AFUCOMP, ycompp, Px, 0xdd,(05) }, + { AFUCOMPP, ycompp, Px, 0xda,(13) }, + + { AFADDDP, yfaddp, Px, 0xde,(00) }, + { AFADDW, yfmvx, Px, 0xde,(00) }, + { AFADDL, yfmvx, Px, 0xda,(00) }, + { AFADDF, yfmvx, Px, 0xd8,(00) }, + { AFADDD, yfadd, Px, 0xdc,(00),0xd8,(00),0xdc,(00) }, + + { AFMULDP, yfaddp, Px, 0xde,(01) }, + { AFMULW, yfmvx, Px, 0xde,(01) }, + { AFMULL, yfmvx, Px, 0xda,(01) }, + { AFMULF, yfmvx, Px, 0xd8,(01) }, + { AFMULD, yfadd, Px, 0xdc,(01),0xd8,(01),0xdc,(01) }, + + { AFSUBDP, yfaddp, Px, 0xde,(05) }, + { AFSUBW, yfmvx, Px, 0xde,(04) }, + { AFSUBL, yfmvx, Px, 0xda,(04) }, + { AFSUBF, yfmvx, Px, 0xd8,(04) }, + { AFSUBD, yfadd, Px, 0xdc,(04),0xd8,(04),0xdc,(05) }, + + { AFSUBRDP, yfaddp, Px, 0xde,(04) }, + { AFSUBRW, yfmvx, Px, 0xde,(05) }, + { AFSUBRL, yfmvx, Px, 0xda,(05) }, + { AFSUBRF, yfmvx, Px, 0xd8,(05) }, + { AFSUBRD, yfadd, Px, 0xdc,(05),0xd8,(05),0xdc,(04) }, + + { AFDIVDP, yfaddp, Px, 0xde,(07) }, + { AFDIVW, yfmvx, Px, 0xde,(06) }, + { AFDIVL, yfmvx, Px, 0xda,(06) }, + { AFDIVF, yfmvx, Px, 0xd8,(06) }, + { AFDIVD, yfadd, Px, 0xdc,(06),0xd8,(06),0xdc,(07) }, + + { AFDIVRDP, yfaddp, Px, 0xde,(06) }, + { AFDIVRW, yfmvx, Px, 0xde,(07) }, + { AFDIVRL, yfmvx, Px, 0xda,(07) }, + { AFDIVRF, yfmvx, Px, 0xd8,(07) }, + { AFDIVRD, yfadd, Px, 0xdc,(07),0xd8,(07),0xdc,(06) }, + + { AFXCHD, yfxch, Px, 0xd9,(01),0xd9,(01) }, + { AFFREE }, + { AFLDCW, ystcw, Px, 0xd9,(05),0xd9,(05) }, + { AFLDENV, ystcw, Px, 0xd9,(04),0xd9,(04) }, + { AFRSTOR, ysvrs, Px, 0xdd,(04),0xdd,(04) }, + { AFSAVE, ysvrs, Px, 0xdd,(06),0xdd,(06) }, + { AFSTCW, ystcw, Px, 0xd9,(07),0xd9,(07) }, + { AFSTENV, ystcw, Px, 0xd9,(06),0xd9,(06) }, + { AFSTSW, ystsw, Px, 0xdd,(07),0xdf,0xe0 }, + { AF2XM1, ynone, Px, 0xd9, 0xf0 }, + { AFABS, ynone, Px, 0xd9, 0xe1 }, + { AFCHS, ynone, Px, 0xd9, 0xe0 }, + { AFCLEX, ynone, Px, 0xdb, 0xe2 }, + { AFCOS, ynone, Px, 0xd9, 0xff }, + { AFDECSTP, ynone, Px, 0xd9, 0xf6 }, + { AFINCSTP, ynone, Px, 0xd9, 0xf7 }, + { AFINIT, ynone, Px, 0xdb, 0xe3 }, + { AFLD1, ynone, Px, 0xd9, 0xe8 }, + { AFLDL2E, ynone, Px, 0xd9, 0xea }, + { AFLDL2T, ynone, Px, 0xd9, 0xe9 }, + { AFLDLG2, ynone, Px, 0xd9, 0xec }, + { AFLDLN2, ynone, Px, 0xd9, 0xed }, + { AFLDPI, ynone, Px, 0xd9, 0xeb }, + { AFLDZ, ynone, Px, 0xd9, 0xee }, + { AFNOP, ynone, Px, 0xd9, 0xd0 }, + { AFPATAN, ynone, Px, 0xd9, 0xf3 }, + { AFPREM, ynone, Px, 0xd9, 0xf8 }, + { AFPREM1, ynone, Px, 0xd9, 0xf5 }, + { AFPTAN, ynone, Px, 0xd9, 0xf2 }, + { AFRNDINT, ynone, Px, 0xd9, 0xfc }, + { AFSCALE, ynone, Px, 0xd9, 0xfd }, + { AFSIN, ynone, Px, 0xd9, 0xfe }, + { AFSINCOS, ynone, Px, 0xd9, 0xfb }, + { AFSQRT, ynone, Px, 0xd9, 0xfa }, + { AFTST, ynone, Px, 0xd9, 0xe4 }, + { AFXAM, ynone, Px, 0xd9, 0xe5 }, + { AFXTRACT, ynone, Px, 0xd9, 0xf4 }, + { AFYL2X, ynone, Px, 0xd9, 0xf1 }, + { AFYL2XP1, ynone, Px, 0xd9, 0xf9 }, + + { ACMPXCHGB, yrb_mb, Pb, 0x0f,0xb0 }, + { ACMPXCHGL, yrl_ml, Px, 0x0f,0xb1 }, + { ACMPXCHGW, yrl_ml, Pe, 0x0f,0xb1 }, + { ACMPXCHGQ, yrl_ml, Pw, 0x0f,0xb1 }, + { ACMPXCHG8B, yscond, Pm, 0xc7,(01) }, + { AINVD, ynone, Pm, 0x08 }, + { AINVLPG, ymbs, Pm, 0x01,(07) }, + { ALFENCE, ynone, Pm, 0xae,0xe8 }, + { AMFENCE, ynone, Pm, 0xae,0xf0 }, + { AMOVNTIL, yrl_ml, Pm, 0xc3 }, + { AMOVNTIQ, yrl_ml, Pw, 0x0f,0xc3 }, + { ARDMSR, ynone, Pm, 0x32 }, + { ARDPMC, ynone, Pm, 0x33 }, + { ARDTSC, ynone, Pm, 0x31 }, + { ARSM, ynone, Pm, 0xaa }, + { ASFENCE, ynone, Pm, 0xae,0xf8 }, + { ASYSRET, ynone, Pm, 0x07 }, + { AWBINVD, ynone, Pm, 0x09 }, + { AWRMSR, ynone, Pm, 0x30 }, + + { AXADDB, yrb_mb, Pb, 0x0f,0xc0 }, + { AXADDL, yrl_ml, Px, 0x0f,0xc1 }, + { AXADDQ, yrl_ml, Pw, 0x0f,0xc1 }, + { AXADDW, yrl_ml, Pe, 0x0f,0xc1 }, + + { ACRC32B, ycrc32l,Px, 0xf2,0x0f,0x38,0xf0,0}, + { ACRC32Q, ycrc32l,Pw, 0xf2,0x0f,0x38,0xf1,0}, + + { AEND }, + 0 +}; + +Optab* opindex[ALAST+1]; + +/* +AMOVD 0f 6e/r mmx,reg/mem32[mem64-rex?] +AMOVD 0f 7e/r reg/mem32[64],mmx STORE +AMOVQ 0f 6f/r mmx1,mmx2/mem64 +AMOVQ 0f 7f/r mmx1/mem64,mmx2 +*/ diff --git a/src/cmd/6l/pass.c b/src/cmd/6l/pass.c new file mode 100644 index 000000000..d9e0b2fc1 --- /dev/null +++ b/src/cmd/6l/pass.c @@ -0,0 +1,722 @@ +// Inferno utils/6l/pass.c +// http://code.google.com/p/inferno-os/source/browse/utils/6l/pass.c +// +// Copyright © 1994-1999 Lucent Technologies Inc. All rights reserved. +// Portions Copyright © 1995-1997 C H Forsyth (forsyth@terzarima.net) +// Portions Copyright © 1997-1999 Vita Nuova Limited +// Portions Copyright © 2000-2007 Vita Nuova Holdings Limited (www.vitanuova.com) +// Portions Copyright © 2004,2006 Bruce Ellis +// Portions Copyright © 2005-2007 C H Forsyth (forsyth@terzarima.net) +// Revisions Copyright © 2000-2007 Lucent Technologies Inc. and others +// Portions Copyright © 2009 The Go Authors. All rights reserved. +// +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to deal +// in the Software without restriction, including without limitation the rights +// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +// copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +// THE SOFTWARE. + +// Code and data passes. + +#include "l.h" +#include "../ld/lib.h" +#include "../../pkg/runtime/stack.h" + +static void xfol(Prog*, Prog**); + +Prog* +brchain(Prog *p) +{ + int i; + + for(i=0; i<20; i++) { + if(p == P || p->as != AJMP) + return p; + p = p->pcond; + } + return P; +} + +void +follow(void) +{ + Prog *firstp, *lastp; + + if(debug['v']) + Bprint(&bso, "%5.2f follow\n", cputime()); + Bflush(&bso); + + for(cursym = textp; cursym != nil; cursym = cursym->next) { + firstp = prg(); + lastp = firstp; + xfol(cursym->text, &lastp); + lastp->link = nil; + cursym->text = firstp->link; + } +} + +static int +nofollow(int a) +{ + switch(a) { + case AJMP: + case ARET: + case AIRETL: + case AIRETQ: + case AIRETW: + case ARETFL: + case ARETFQ: + case ARETFW: + return 1; + } + return 0; +} + +static int +pushpop(int a) +{ + switch(a) { + case APUSHL: + case APUSHFL: + case APUSHQ: + case APUSHFQ: + case APUSHW: + case APUSHFW: + case APOPL: + case APOPFL: + case APOPQ: + case APOPFQ: + case APOPW: + case APOPFW: + return 1; + } + return 0; +} + +static void +xfol(Prog *p, Prog **last) +{ + Prog *q; + int i; + enum as a; + +loop: + if(p == P) + return; + if(p->as == AJMP) + if((q = p->pcond) != P && q->as != ATEXT) { + /* mark instruction as done and continue layout at target of jump */ + p->mark = 1; + p = q; + if(p->mark == 0) + goto loop; + } + if(p->mark) { + /* + * p goes here, but already used it elsewhere. + * copy up to 4 instructions or else branch to other copy. + */ + for(i=0,q=p; i<4; i++,q=q->link) { + if(q == P) + break; + if(q == *last) + break; + a = q->as; + if(a == ANOP) { + i--; + continue; + } + if(nofollow(a) || pushpop(a)) + break; // NOTE(rsc): arm does goto copy + if(q->pcond == P || q->pcond->mark) + continue; + if(a == ACALL || a == ALOOP) + continue; + for(;;) { + if(p->as == ANOP) { + p = p->link; + continue; + } + q = copyp(p); + p = p->link; + q->mark = 1; + (*last)->link = q; + *last = q; + if(q->as != a || q->pcond == P || q->pcond->mark) + continue; + + q->as = relinv(q->as); + p = q->pcond; + q->pcond = q->link; + q->link = p; + xfol(q->link, last); + p = q->link; + if(p->mark) + return; + goto loop; + } + } /* */ + q = prg(); + q->as = AJMP; + q->line = p->line; + q->to.type = D_BRANCH; + q->to.offset = p->pc; + q->pcond = p; + p = q; + } + + /* emit p */ + p->mark = 1; + (*last)->link = p; + *last = p; + a = p->as; + + /* continue loop with what comes after p */ + if(nofollow(a)) + return; + if(p->pcond != P && a != ACALL) { + /* + * some kind of conditional branch. + * recurse to follow one path. + * continue loop on the other. + */ + q = brchain(p->link); + if(q != P && q->mark) + if(a != ALOOP) { + p->as = relinv(a); + p->link = p->pcond; + p->pcond = q; + } + xfol(p->link, last); + q = brchain(p->pcond); + if(q->mark) { + p->pcond = q; + return; + } + p = q; + goto loop; + } + p = p->link; + goto loop; +} + +Prog* +byteq(int v) +{ + Prog *p; + + p = prg(); + p->as = ABYTE; + p->from.type = D_CONST; + p->from.offset = v&0xff; + return p; +} + +int +relinv(int a) +{ + + switch(a) { + case AJEQ: return AJNE; + case AJNE: return AJEQ; + case AJLE: return AJGT; + case AJLS: return AJHI; + case AJLT: return AJGE; + case AJMI: return AJPL; + case AJGE: return AJLT; + case AJPL: return AJMI; + case AJGT: return AJLE; + case AJHI: return AJLS; + case AJCS: return AJCC; + case AJCC: return AJCS; + case AJPS: return AJPC; + case AJPC: return AJPS; + case AJOS: return AJOC; + case AJOC: return AJOS; + } + diag("unknown relation: %s in %s", anames[a], TNAME); + errorexit(); + return a; +} + +void +patch(void) +{ + int32 c; + Prog *p, *q; + Sym *s; + int32 vexit; + + if(debug['v']) + Bprint(&bso, "%5.2f mkfwd\n", cputime()); + Bflush(&bso); + mkfwd(); + if(debug['v']) + Bprint(&bso, "%5.2f patch\n", cputime()); + Bflush(&bso); + + s = lookup("exit", 0); + vexit = s->value; + for(cursym = textp; cursym != nil; cursym = cursym->next) + for(p = cursym->text; p != P; p = p->link) { + if(HEADTYPE == Hwindows) { + // Windows + // Convert + // op n(GS), reg + // to + // MOVL 0x58(GS), reg + // op n(reg), reg + // The purpose of this patch is to fix some accesses + // to extern register variables (TLS) on Windows, as + // a different method is used to access them. + if(p->from.type == D_INDIR+D_GS + && p->to.type >= D_AX && p->to.type <= D_DI + && p->from.offset <= 8) { + q = appendp(p); + q->from = p->from; + q->from.type = D_INDIR + p->to.type; + q->to = p->to; + q->as = p->as; + p->as = AMOVQ; + p->from.type = D_INDIR+D_GS; + p->from.offset = 0x58; + } + } + if(HEADTYPE == Hlinux || HEADTYPE == Hfreebsd + || HEADTYPE == Hopenbsd) { + // ELF uses FS instead of GS. + if(p->from.type == D_INDIR+D_GS) + p->from.type = D_INDIR+D_FS; + if(p->to.type == D_INDIR+D_GS) + p->to.type = D_INDIR+D_FS; + } + if(p->as == ACALL || (p->as == AJMP && p->to.type != D_BRANCH)) { + s = p->to.sym; + if(s) { + if(debug['c']) + Bprint(&bso, "%s calls %s\n", TNAME, s->name); + if((s->type&~SSUB) != STEXT) { + /* diag prints TNAME first */ + diag("undefined: %s", s->name); + s->type = STEXT; + s->value = vexit; + continue; // avoid more error messages + } + if(s->text == nil) + continue; + p->to.type = D_BRANCH; + p->to.offset = s->text->pc; + p->pcond = s->text; + continue; + } + } + if(p->to.type != D_BRANCH) + continue; + c = p->to.offset; + for(q = cursym->text; q != P;) { + if(c == q->pc) + break; + if(q->forwd != P && c >= q->forwd->pc) + q = q->forwd; + else + q = q->link; + } + if(q == P) { + diag("branch out of range in %s (%#ux)\n%P [%s]", + TNAME, c, p, p->to.sym ? p->to.sym->name : ""); + p->to.type = D_NONE; + } + p->pcond = q; + } + + for(cursym = textp; cursym != nil; cursym = cursym->next) + for(p = cursym->text; p != P; p = p->link) { + p->mark = 0; /* initialization for follow */ + if(p->pcond != P) { + p->pcond = brloop(p->pcond); + if(p->pcond != P) + if(p->to.type == D_BRANCH) + p->to.offset = p->pcond->pc; + } + } +} + +Prog* +brloop(Prog *p) +{ + int c; + Prog *q; + + c = 0; + for(q = p; q != P; q = q->pcond) { + if(q->as != AJMP) + break; + c++; + if(c >= 5000) + return P; + } + return q; +} + +static char* +morename[] = +{ + "runtime.morestack00", + "runtime.morestack10", + "runtime.morestack01", + "runtime.morestack11", + + "runtime.morestack8", + "runtime.morestack16", + "runtime.morestack24", + "runtime.morestack32", + "runtime.morestack40", + "runtime.morestack48", +}; +Prog* pmorestack[nelem(morename)]; +Sym* symmorestack[nelem(morename)]; + +void +dostkoff(void) +{ + Prog *p, *q, *q1; + int32 autoffset, deltasp; + int a, pcsize; + uint32 moreconst1, moreconst2, i; + + for(i=0; itype != STEXT) + diag("morestack trampoline not defined - %s", morename[i]); + pmorestack[i] = symmorestack[i]->text; + } + + for(cursym = textp; cursym != nil; cursym = cursym->next) { + if(cursym->text == nil || cursym->text->link == nil) + continue; + + p = cursym->text; + parsetextconst(p->to.offset); + autoffset = textstksiz; + if(autoffset < 0) + autoffset = 0; + + q = P; + if((p->from.scale & NOSPLIT) && autoffset >= StackSmall) + diag("nosplit func likely to overflow stack"); + + if(!(p->from.scale & NOSPLIT)) { + p = appendp(p); // load g into CX + p->as = AMOVQ; + if(HEADTYPE == Hlinux || HEADTYPE == Hfreebsd + || HEADTYPE == Hopenbsd) // ELF uses FS + p->from.type = D_INDIR+D_FS; + else + p->from.type = D_INDIR+D_GS; + p->from.offset = tlsoffset+0; + p->to.type = D_CX; + if(HEADTYPE == Hwindows) { + // movq %gs:0x58, %rcx + // movq (%rcx), %rcx + p->as = AMOVQ; + p->from.type = D_INDIR+D_GS; + p->from.offset = 0x58; + p->to.type = D_CX; + + + p = appendp(p); + p->as = AMOVQ; + p->from.type = D_INDIR+D_CX; + p->from.offset = 0; + p->to.type = D_CX; + } + + if(debug['K']) { + // 6l -K means check not only for stack + // overflow but stack underflow. + // On underflow, INT 3 (breakpoint). + // Underflow itself is rare but this also + // catches out-of-sync stack guard info + + p = appendp(p); + p->as = ACMPQ; + p->from.type = D_INDIR+D_CX; + p->from.offset = 8; + p->to.type = D_SP; + + p = appendp(p); + p->as = AJHI; + p->to.type = D_BRANCH; + p->to.offset = 4; + q1 = p; + + p = appendp(p); + p->as = AINT; + p->from.type = D_CONST; + p->from.offset = 3; + + p = appendp(p); + p->as = ANOP; + q1->pcond = p; + } + + if(autoffset < StackBig) { // do we need to call morestack? + if(autoffset <= StackSmall) { + // small stack + p = appendp(p); + p->as = ACMPQ; + p->from.type = D_SP; + p->to.type = D_INDIR+D_CX; + } else { + // large stack + p = appendp(p); + p->as = ALEAQ; + p->from.type = D_INDIR+D_SP; + p->from.offset = -(autoffset-StackSmall); + p->to.type = D_AX; + + p = appendp(p); + p->as = ACMPQ; + p->from.type = D_AX; + p->to.type = D_INDIR+D_CX; + } + + // common + p = appendp(p); + p->as = AJHI; + p->to.type = D_BRANCH; + p->to.offset = 4; + q = p; + } + + /* 160 comes from 3 calls (3*8) 4 safes (4*8) and 104 guard */ + moreconst1 = 0; + if(autoffset+160+textarg > 4096) + moreconst1 = (autoffset+160) & ~7LL; + moreconst2 = textarg; + + // 4 varieties varieties (const1==0 cross const2==0) + // and 6 subvarieties of (const1==0 and const2!=0) + p = appendp(p); + if(moreconst1 == 0 && moreconst2 == 0) { + p->as = ACALL; + p->to.type = D_BRANCH; + p->pcond = pmorestack[0]; + p->to.sym = symmorestack[0]; + } else + if(moreconst1 != 0 && moreconst2 == 0) { + p->as = AMOVL; + p->from.type = D_CONST; + p->from.offset = moreconst1; + p->to.type = D_AX; + + p = appendp(p); + p->as = ACALL; + p->to.type = D_BRANCH; + p->pcond = pmorestack[1]; + p->to.sym = symmorestack[1]; + } else + if(moreconst1 == 0 && moreconst2 <= 48 && moreconst2%8 == 0) { + i = moreconst2/8 + 3; + p->as = ACALL; + p->to.type = D_BRANCH; + p->pcond = pmorestack[i]; + p->to.sym = symmorestack[i]; + } else + if(moreconst1 == 0 && moreconst2 != 0) { + p->as = AMOVL; + p->from.type = D_CONST; + p->from.offset = moreconst2; + p->to.type = D_AX; + + p = appendp(p); + p->as = ACALL; + p->to.type = D_BRANCH; + p->pcond = pmorestack[2]; + p->to.sym = symmorestack[2]; + } else { + p->as = AMOVQ; + p->from.type = D_CONST; + p->from.offset = (uint64)moreconst2 << 32; + p->from.offset |= moreconst1; + p->to.type = D_AX; + + p = appendp(p); + p->as = ACALL; + p->to.type = D_BRANCH; + p->pcond = pmorestack[3]; + p->to.sym = symmorestack[3]; + } + } + + if(q != P) + q->pcond = p->link; + + if(autoffset) { + p = appendp(p); + p->as = AADJSP; + p->from.type = D_CONST; + p->from.offset = autoffset; + p->spadj = autoffset; + if(q != P) + q->pcond = p; + } + deltasp = autoffset; + + if(debug['K'] > 1 && autoffset) { + // 6l -KK means double-check for stack overflow + // even after calling morestack and even if the + // function is marked as nosplit. + p = appendp(p); + p->as = AMOVQ; + p->from.type = D_INDIR+D_CX; + p->from.offset = 0; + p->to.type = D_BX; + + p = appendp(p); + p->as = ASUBQ; + p->from.type = D_CONST; + p->from.offset = StackSmall+32; + p->to.type = D_BX; + + p = appendp(p); + p->as = ACMPQ; + p->from.type = D_SP; + p->to.type = D_BX; + + p = appendp(p); + p->as = AJHI; + p->to.type = D_BRANCH; + q1 = p; + + p = appendp(p); + p->as = AINT; + p->from.type = D_CONST; + p->from.offset = 3; + + p = appendp(p); + p->as = ANOP; + q1->pcond = p; + } + + for(; p != P; p = p->link) { + pcsize = p->mode/8; + a = p->from.type; + if(a == D_AUTO) + p->from.offset += deltasp; + if(a == D_PARAM) + p->from.offset += deltasp + pcsize; + a = p->to.type; + if(a == D_AUTO) + p->to.offset += deltasp; + if(a == D_PARAM) + p->to.offset += deltasp + pcsize; + + switch(p->as) { + default: + continue; + case APUSHL: + case APUSHFL: + deltasp += 4; + p->spadj = 4; + continue; + case APUSHQ: + case APUSHFQ: + deltasp += 8; + p->spadj = 8; + continue; + case APUSHW: + case APUSHFW: + deltasp += 2; + p->spadj = 2; + continue; + case APOPL: + case APOPFL: + deltasp -= 4; + p->spadj = -4; + continue; + case APOPQ: + case APOPFQ: + deltasp -= 8; + p->spadj = -8; + continue; + case APOPW: + case APOPFW: + deltasp -= 2; + p->spadj = -2; + continue; + case ARET: + break; + } + + if(autoffset != deltasp) + diag("unbalanced PUSH/POP"); + + if(autoffset) { + p->as = AADJSP; + p->from.type = D_CONST; + p->from.offset = -autoffset; + p->spadj = -autoffset; + p = appendp(p); + p->as = ARET; + // If there are instructions following + // this ARET, they come from a branch + // with the same stackframe, so undo + // the cleanup. + p->spadj = +autoffset; + } + } + } +} + +vlong +atolwhex(char *s) +{ + vlong n; + int f; + + n = 0; + f = 0; + while(*s == ' ' || *s == '\t') + s++; + if(*s == '-' || *s == '+') { + if(*s++ == '-') + f = 1; + while(*s == ' ' || *s == '\t') + s++; + } + if(s[0]=='0' && s[1]){ + if(s[1]=='x' || s[1]=='X'){ + s += 2; + for(;;){ + if(*s >= '0' && *s <= '9') + n = n*16 + *s++ - '0'; + else if(*s >= 'a' && *s <= 'f') + n = n*16 + *s++ - 'a' + 10; + else if(*s >= 'A' && *s <= 'F') + n = n*16 + *s++ - 'A' + 10; + else + break; + } + } else + while(*s >= '0' && *s <= '7') + n = n*8 + *s++ - '0'; + } else + while(*s >= '0' && *s <= '9') + n = n*10 + *s++ - '0'; + if(f) + n = -n; + return n; +} diff --git a/src/cmd/6l/prof.c b/src/cmd/6l/prof.c new file mode 100644 index 000000000..862ce080c --- /dev/null +++ b/src/cmd/6l/prof.c @@ -0,0 +1,171 @@ +// Inferno utils/6l/obj.c +// http://code.google.com/p/inferno-os/source/browse/utils/6l/obj.c +// +// Copyright © 1994-1999 Lucent Technologies Inc. All rights reserved. +// Portions Copyright © 1995-1997 C H Forsyth (forsyth@terzarima.net) +// Portions Copyright © 1997-1999 Vita Nuova Limited +// Portions Copyright © 2000-2007 Vita Nuova Holdings Limited (www.vitanuova.com) +// Portions Copyright © 2004,2006 Bruce Ellis +// Portions Copyright © 2005-2007 C H Forsyth (forsyth@terzarima.net) +// Revisions Copyright © 2000-2007 Lucent Technologies Inc. and others +// Portions Copyright © 2009 The Go Authors. All rights reserved. +// +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to deal +// in the Software without restriction, including without limitation the rights +// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +// copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +// THE SOFTWARE. + +// Profiling. + +#include "l.h" +#include "../ld/lib.h" + +void +doprof1(void) +{ +#ifdef NOTDEF + Sym *s; + int32 n; + Prog *p, *q; + + if(debug['v']) + Bprint(&bso, "%5.2f profile 1\n", cputime()); + Bflush(&bso); + s = lookup("__mcount", 0); + n = 1; + for(cursym = textp; cursym != nil; cursym = cursym->next) { + p = cursym->text; + q = prg(); + q->line = p->line; + q->link = datap; + datap = q; + q->as = ADATA; + q->from.type = D_EXTERN; + q->from.offset = n*4; + q->from.sym = s; + q->from.scale = 4; + q->to = p->from; + q->to.type = D_CONST; + + q = prg(); + q->line = p->line; + q->pc = p->pc; + q->link = p->link; + p->link = q; + p = q; + p->as = AADDL; + p->from.type = D_CONST; + p->from.offset = 1; + p->to.type = D_EXTERN; + p->to.sym = s; + p->to.offset = n*4 + 4; + + n += 2; + } + q = prg(); + q->line = 0; + q->link = datap; + datap = q; + + q->as = ADATA; + q->from.type = D_EXTERN; + q->from.sym = s; + q->from.scale = 4; + q->to.type = D_CONST; + q->to.offset = n; + + s->type = SBSS; + s->size = n*4; +#endif +} + +void +doprof2(void) +{ + Sym *s2, *s4; + Prog *p, *q, *ps2, *ps4; + + if(debug['v']) + Bprint(&bso, "%5.2f profile 2\n", cputime()); + Bflush(&bso); + + s2 = lookup("_profin", 0); + s4 = lookup("_profout", 0); + if(s2->type != STEXT || s4->type != STEXT) { + diag("_profin/_profout not defined"); + return; + } + + ps2 = P; + ps4 = P; + for(cursym = textp; cursym != nil; cursym = cursym->next) { + p = cursym->text; + if(p->from.sym == s2) { + p->from.scale = 1; + ps2 = p; + } + if(p->from.sym == s4) { + p->from.scale = 1; + ps4 = p; + } + } + for(cursym = textp; cursym != nil; cursym = cursym->next) { + p = cursym->text; + + if(p->from.scale & NOPROF) /* dont profile */ + continue; + + /* + * JMPL profin + */ + q = prg(); + q->line = p->line; + q->pc = p->pc; + q->link = p->link; + p->link = q; + p = q; + p->as = ACALL; + p->to.type = D_BRANCH; + p->pcond = ps2; + p->to.sym = s2; + + for(; p; p=p->link) { + if(p->as == ARET) { + /* + * RET + */ + q = prg(); + q->as = ARET; + q->from = p->from; + q->to = p->to; + q->link = p->link; + p->link = q; + + /* + * JAL profout + */ + p->as = ACALL; + p->from = zprg.from; + p->to = zprg.to; + p->to.type = D_BRANCH; + p->pcond = ps4; + p->to.sym = s4; + + p = q; + } + } + } +} diff --git a/src/cmd/6l/span.c b/src/cmd/6l/span.c new file mode 100644 index 000000000..5d13ad44b --- /dev/null +++ b/src/cmd/6l/span.c @@ -0,0 +1,1747 @@ +// Inferno utils/6l/span.c +// http://code.google.com/p/inferno-os/source/browse/utils/6l/span.c +// +// Copyright © 1994-1999 Lucent Technologies Inc. All rights reserved. +// Portions Copyright © 1995-1997 C H Forsyth (forsyth@terzarima.net) +// Portions Copyright © 1997-1999 Vita Nuova Limited +// Portions Copyright © 2000-2007 Vita Nuova Holdings Limited (www.vitanuova.com) +// Portions Copyright © 2004,2006 Bruce Ellis +// Portions Copyright © 2005-2007 C H Forsyth (forsyth@terzarima.net) +// Revisions Copyright © 2000-2007 Lucent Technologies Inc. and others +// Portions Copyright © 2009 The Go Authors. All rights reserved. +// +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to deal +// in the Software without restriction, including without limitation the rights +// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +// copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +// THE SOFTWARE. + +// Instruction layout. + +#include "l.h" +#include "../ld/lib.h" + +static int rexflag; +static int asmode; +static vlong vaddr(Adr*, Reloc*); + +void +span1(Sym *s) +{ + Prog *p, *q; + int32 c, v, loop; + uchar *bp; + int n, m, i; + + cursym = s; + + if(s->p != nil) + return; + + for(p = s->text; p != P; p = p->link) { + p->back = 2; // use short branches first time through + if((q = p->pcond) != P && (q->back & 2)) + p->back |= 1; // backward jump + + if(p->as == AADJSP) { + p->to.type = D_SP; + v = -p->from.offset; + p->from.offset = v; + p->as = p->mode != 64? AADDL: AADDQ; + if(v < 0) { + p->as = p->mode != 64? ASUBL: ASUBQ; + v = -v; + p->from.offset = v; + } + if(v == 0) + p->as = ANOP; + } + } + + n = 0; + do { + loop = 0; + memset(s->r, 0, s->nr*sizeof s->r[0]); + s->nr = 0; + s->np = 0; + c = 0; + for(p = s->text; p != P; p = p->link) { + p->pc = c; + + // process forward jumps to p + for(q = p->comefrom; q != P; q = q->forwd) { + v = p->pc - (q->pc + q->mark); + if(q->back & 2) { // short + if(v > 127) { + loop++; + q->back ^= 2; + } + s->p[q->pc+1] = v; + } else { + bp = s->p + q->pc + q->mark - 4; + *bp++ = v; + *bp++ = v>>8; + *bp++ = v>>16; + *bp = v>>24; + } + } + p->comefrom = P; + + asmins(p); + p->pc = c; + m = andptr-and; + symgrow(s, p->pc+m); + memmove(s->p+p->pc, and, m); + p->mark = m; + c += m; + } + if(++n > 20) { + diag("span must be looping"); + errorexit(); + } + } while(loop); + s->size = c; + + if(debug['a'] > 1) { + print("span1 %s %lld (%d tries)\n %.6ux", s->name, s->size, n, 0); + for(i=0; inp; i++) { + print(" %.2ux", s->p[i]); + if(i%16 == 15) + print("\n %.6ux", i+1); + } + if(i%16) + print("\n"); + + for(i=0; inr; i++) { + Reloc *r; + + r = &s->r[i]; + print(" rel %#.4ux/%d %s%+lld\n", r->off, r->siz, r->sym->name, r->add); + } + } +} + +void +span(void) +{ + Prog *p, *q; + int32 v; + int n; + + if(debug['v']) + Bprint(&bso, "%5.2f span\n", cputime()); + + // NOTE(rsc): If we get rid of the globals we should + // be able to parallelize these iterations. + for(cursym = textp; cursym != nil; cursym = cursym->next) { + if(cursym->p != nil) + continue; + // TODO: move into span1 + for(p = cursym->text; p != P; p = p->link) { + n = 0; + if(p->to.type == D_BRANCH) + if(p->pcond == P) + p->pcond = p; + if((q = p->pcond) != P) + if(q->back != 2) + n = 1; + p->back = n; + if(p->as == AADJSP) { + p->to.type = D_SP; + v = -p->from.offset; + p->from.offset = v; + p->as = p->mode != 64? AADDL: AADDQ; + if(v < 0) { + p->as = p->mode != 64? ASUBL: ASUBQ; + v = -v; + p->from.offset = v; + } + if(v == 0) + p->as = ANOP; + } + } + span1(cursym); + } +} + +void +xdefine(char *p, int t, vlong v) +{ + Sym *s; + + s = lookup(p, 0); + s->type = t; + s->value = v; + s->reachable = 1; + s->special = 1; +} + +void +instinit(void) +{ + int c, i; + + for(i=1; optab[i].as; i++) { + c = optab[i].as; + if(opindex[c] != nil) { + diag("phase error in optab: %d (%A)", i, c); + errorexit(); + } + opindex[c] = &optab[i]; + } + + for(i=0; i= D_AL && i <= D_R15B) { + reg[i] = (i-D_AL) & 7; + if(i >= D_SPB && i <= D_DIB) + regrex[i] = 0x40; + if(i >= D_R8B && i <= D_R15B) + regrex[i] = Rxr | Rxx | Rxb; + } + if(i >= D_AH && i<= D_BH) + reg[i] = 4 + ((i-D_AH) & 7); + if(i >= D_AX && i <= D_R15) { + reg[i] = (i-D_AX) & 7; + if(i >= D_R8) + regrex[i] = Rxr | Rxx | Rxb; + } + if(i >= D_F0 && i <= D_F0+7) + reg[i] = (i-D_F0) & 7; + if(i >= D_M0 && i <= D_M0+7) + reg[i] = (i-D_M0) & 7; + if(i >= D_X0 && i <= D_X0+15) { + reg[i] = (i-D_X0) & 7; + if(i >= D_X0+8) + regrex[i] = Rxr | Rxx | Rxb; + } + if(i >= D_CR+8 && i <= D_CR+15) + regrex[i] = Rxr; + } +} + +int +prefixof(Adr *a) +{ + switch(a->type) { + case D_INDIR+D_CS: + return 0x2e; + case D_INDIR+D_DS: + return 0x3e; + case D_INDIR+D_ES: + return 0x26; + case D_INDIR+D_FS: + return 0x64; + case D_INDIR+D_GS: + return 0x65; + } + return 0; +} + +int +oclass(Adr *a) +{ + vlong v; + int32 l; + + if(a->type >= D_INDIR || a->index != D_NONE) { + if(a->index != D_NONE && a->scale == 0) { + if(a->type == D_ADDR) { + switch(a->index) { + case D_EXTERN: + case D_STATIC: + return Yi32; /* TO DO: Yi64 */ + case D_AUTO: + case D_PARAM: + return Yiauto; + } + return Yxxx; + } + return Ycol; + } + return Ym; + } + switch(a->type) + { + case D_AL: + return Yal; + + case D_AX: + return Yax; + +/* + case D_SPB: +*/ + case D_BPB: + case D_SIB: + case D_DIB: + case D_R8B: + case D_R9B: + case D_R10B: + case D_R11B: + case D_R12B: + case D_R13B: + case D_R14B: + case D_R15B: + if(asmode != 64) + return Yxxx; + case D_DL: + case D_BL: + case D_AH: + case D_CH: + case D_DH: + case D_BH: + return Yrb; + + case D_CL: + return Ycl; + + case D_CX: + return Ycx; + + case D_DX: + case D_BX: + return Yrx; + + case D_R8: /* not really Yrl */ + case D_R9: + case D_R10: + case D_R11: + case D_R12: + case D_R13: + case D_R14: + case D_R15: + if(asmode != 64) + return Yxxx; + case D_SP: + case D_BP: + case D_SI: + case D_DI: + return Yrl; + + case D_F0+0: + return Yf0; + + case D_F0+1: + case D_F0+2: + case D_F0+3: + case D_F0+4: + case D_F0+5: + case D_F0+6: + case D_F0+7: + return Yrf; + + case D_M0+0: + case D_M0+1: + case D_M0+2: + case D_M0+3: + case D_M0+4: + case D_M0+5: + case D_M0+6: + case D_M0+7: + return Ymr; + + case D_X0+0: + case D_X0+1: + case D_X0+2: + case D_X0+3: + case D_X0+4: + case D_X0+5: + case D_X0+6: + case D_X0+7: + case D_X0+8: + case D_X0+9: + case D_X0+10: + case D_X0+11: + case D_X0+12: + case D_X0+13: + case D_X0+14: + case D_X0+15: + return Yxr; + + case D_NONE: + return Ynone; + + case D_CS: return Ycs; + case D_SS: return Yss; + case D_DS: return Yds; + case D_ES: return Yes; + case D_FS: return Yfs; + case D_GS: return Ygs; + + case D_GDTR: return Ygdtr; + case D_IDTR: return Yidtr; + case D_LDTR: return Yldtr; + case D_MSW: return Ymsw; + case D_TASK: return Ytask; + + case D_CR+0: return Ycr0; + case D_CR+1: return Ycr1; + case D_CR+2: return Ycr2; + case D_CR+3: return Ycr3; + case D_CR+4: return Ycr4; + case D_CR+5: return Ycr5; + case D_CR+6: return Ycr6; + case D_CR+7: return Ycr7; + case D_CR+8: return Ycr8; + + case D_DR+0: return Ydr0; + case D_DR+1: return Ydr1; + case D_DR+2: return Ydr2; + case D_DR+3: return Ydr3; + case D_DR+4: return Ydr4; + case D_DR+5: return Ydr5; + case D_DR+6: return Ydr6; + case D_DR+7: return Ydr7; + + case D_TR+0: return Ytr0; + case D_TR+1: return Ytr1; + case D_TR+2: return Ytr2; + case D_TR+3: return Ytr3; + case D_TR+4: return Ytr4; + case D_TR+5: return Ytr5; + case D_TR+6: return Ytr6; + case D_TR+7: return Ytr7; + + case D_EXTERN: + case D_STATIC: + case D_AUTO: + case D_PARAM: + return Ym; + + case D_CONST: + case D_ADDR: + if(a->sym == S) { + v = a->offset; + if(v == 0) + return Yi0; + if(v == 1) + return Yi1; + if(v >= -128 && v <= 127) + return Yi8; + l = v; + if((vlong)l == v) + return Ys32; /* can sign extend */ + if((v>>32) == 0) + return Yi32; /* unsigned */ + return Yi64; + } + return Yi32; /* TO DO: D_ADDR as Yi64 */ + + case D_BRANCH: + return Ybr; + } + return Yxxx; +} + +void +asmidx(int scale, int index, int base) +{ + int i; + + switch(index) { + default: + goto bad; + + case D_NONE: + i = 4 << 3; + goto bas; + + case D_R8: + case D_R9: + case D_R10: + case D_R11: + case D_R12: + case D_R13: + case D_R14: + case D_R15: + if(asmode != 64) + goto bad; + case D_AX: + case D_CX: + case D_DX: + case D_BX: + case D_BP: + case D_SI: + case D_DI: + i = reg[index] << 3; + break; + } + switch(scale) { + default: + goto bad; + case 1: + break; + case 2: + i |= (1<<6); + break; + case 4: + i |= (2<<6); + break; + case 8: + i |= (3<<6); + break; + } +bas: + switch(base) { + default: + goto bad; + case D_NONE: /* must be mod=00 */ + i |= 5; + break; + case D_R8: + case D_R9: + case D_R10: + case D_R11: + case D_R12: + case D_R13: + case D_R14: + case D_R15: + if(asmode != 64) + goto bad; + case D_AX: + case D_CX: + case D_DX: + case D_BX: + case D_SP: + case D_BP: + case D_SI: + case D_DI: + i |= reg[base]; + break; + } + *andptr++ = i; + return; +bad: + diag("asmidx: bad address %d/%d/%d", scale, index, base); + *andptr++ = 0; + return; +} + +static void +put4(int32 v) +{ + andptr[0] = v; + andptr[1] = v>>8; + andptr[2] = v>>16; + andptr[3] = v>>24; + andptr += 4; +} + +static void +relput4(Prog *p, Adr *a) +{ + vlong v; + Reloc rel, *r; + + v = vaddr(a, &rel); + if(rel.siz != 0) { + if(rel.siz != 4) + diag("bad reloc"); + r = addrel(cursym); + *r = rel; + r->off = p->pc + andptr - and; + } + put4(v); +} + +static void +put8(vlong v) +{ + andptr[0] = v; + andptr[1] = v>>8; + andptr[2] = v>>16; + andptr[3] = v>>24; + andptr[4] = v>>32; + andptr[5] = v>>40; + andptr[6] = v>>48; + andptr[7] = v>>56; + andptr += 8; +} + +/* +static void +relput8(Prog *p, Adr *a) +{ + vlong v; + Reloc rel, *r; + + v = vaddr(a, &rel); + if(rel.siz != 0) { + r = addrel(cursym); + *r = rel; + r->siz = 8; + r->off = p->pc + andptr - and; + } + put8(v); +} +*/ + +vlong +symaddr(Sym *s) +{ + if(!s->reachable) + diag("unreachable symbol in symaddr - %s", s->name); + return s->value; +} + +static vlong +vaddr(Adr *a, Reloc *r) +{ + int t; + vlong v; + Sym *s; + + if(r != nil) + memset(r, 0, sizeof *r); + + t = a->type; + v = a->offset; + if(t == D_ADDR) + t = a->index; + switch(t) { + case D_STATIC: + case D_EXTERN: + s = a->sym; + if(!s->reachable) + diag("unreachable symbol in vaddr - %s", s->name); + if(r == nil) { + diag("need reloc for %D", a); + errorexit(); + } + r->type = D_ADDR; + r->siz = 4; // TODO: 8 for external symbols + r->off = -1; // caller must fill in + r->sym = s; + r->add = v; + v = 0; + } + return v; +} + +static void +asmandsz(Adr *a, int r, int rex, int m64) +{ + int32 v; + int t, scale; + Reloc rel; + + USED(m64); + rex &= (0x40 | Rxr); + v = a->offset; + t = a->type; + rel.siz = 0; + if(a->index != D_NONE) { + if(t < D_INDIR) { + switch(t) { + default: + goto bad; + case D_STATIC: + case D_EXTERN: + t = D_NONE; + v = vaddr(a, &rel); + break; + case D_AUTO: + case D_PARAM: + t = D_SP; + break; + } + } else + t -= D_INDIR; + rexflag |= (regrex[(int)a->index] & Rxx) | (regrex[t] & Rxb) | rex; + if(t == D_NONE) { + *andptr++ = (0 << 6) | (4 << 0) | (r << 3); + asmidx(a->scale, a->index, t); + goto putrelv; + } + if(v == 0 && rel.siz == 0 && t != D_BP && t != D_R13) { + *andptr++ = (0 << 6) | (4 << 0) | (r << 3); + asmidx(a->scale, a->index, t); + return; + } + if(v >= -128 && v < 128 && rel.siz == 0) { + *andptr++ = (1 << 6) | (4 << 0) | (r << 3); + asmidx(a->scale, a->index, t); + *andptr++ = v; + return; + } + *andptr++ = (2 << 6) | (4 << 0) | (r << 3); + asmidx(a->scale, a->index, t); + goto putrelv; + } + if(t >= D_AL && t <= D_X0+15) { + if(v) + goto bad; + *andptr++ = (3 << 6) | (reg[t] << 0) | (r << 3); + rexflag |= (regrex[t] & (0x40 | Rxb)) | rex; + return; + } + + scale = a->scale; + if(t < D_INDIR) { + switch(a->type) { + default: + goto bad; + case D_STATIC: + case D_EXTERN: + t = D_NONE; + v = vaddr(a, &rel); + break; + case D_AUTO: + case D_PARAM: + t = D_SP; + break; + } + scale = 1; + } else + t -= D_INDIR; + + rexflag |= (regrex[t] & Rxb) | rex; + if(t == D_NONE || (D_CS <= t && t <= D_GS)) { + if(asmode != 64){ + *andptr++ = (0 << 6) | (5 << 0) | (r << 3); + goto putrelv; + } + /* temporary */ + *andptr++ = (0 << 6) | (4 << 0) | (r << 3); /* sib present */ + *andptr++ = (0 << 6) | (4 << 3) | (5 << 0); /* DS:d32 */ + goto putrelv; + } + if(t == D_SP || t == D_R12) { + if(v == 0) { + *andptr++ = (0 << 6) | (reg[t] << 0) | (r << 3); + asmidx(scale, D_NONE, t); + return; + } + if(v >= -128 && v < 128) { + *andptr++ = (1 << 6) | (reg[t] << 0) | (r << 3); + asmidx(scale, D_NONE, t); + *andptr++ = v; + return; + } + *andptr++ = (2 << 6) | (reg[t] << 0) | (r << 3); + asmidx(scale, D_NONE, t); + goto putrelv; + } + if(t >= D_AX && t <= D_R15) { + if(v == 0 && t != D_BP && t != D_R13) { + *andptr++ = (0 << 6) | (reg[t] << 0) | (r << 3); + return; + } + if(v >= -128 && v < 128) { + andptr[0] = (1 << 6) | (reg[t] << 0) | (r << 3); + andptr[1] = v; + andptr += 2; + return; + } + *andptr++ = (2 << 6) | (reg[t] << 0) | (r << 3); + goto putrelv; + } + goto bad; + +putrelv: + if(rel.siz != 0) { + Reloc *r; + + if(rel.siz != 4) { + diag("bad rel"); + goto bad; + } + r = addrel(cursym); + *r = rel; + r->off = curp->pc + andptr - and; + } + put4(v); + return; + +bad: + diag("asmand: bad address %D", a); + return; +} + +void +asmand(Adr *a, Adr *ra) +{ + asmandsz(a, reg[ra->type], regrex[ra->type], 0); +} + +void +asmando(Adr *a, int o) +{ + asmandsz(a, o, 0, 0); +} + +static void +bytereg(Adr *a, char *t) +{ + if(a->index == D_NONE && (a->type >= D_AX && a->type <= D_R15)) { + a->type = D_AL + (a->type-D_AX); + *t = 0; + } +} + +#define E 0xff +Movtab ymovtab[] = +{ +/* push */ + {APUSHL, Ycs, Ynone, 0, 0x0e,E,0,0}, + {APUSHL, Yss, Ynone, 0, 0x16,E,0,0}, + {APUSHL, Yds, Ynone, 0, 0x1e,E,0,0}, + {APUSHL, Yes, Ynone, 0, 0x06,E,0,0}, + {APUSHL, Yfs, Ynone, 0, 0x0f,0xa0,E,0}, + {APUSHL, Ygs, Ynone, 0, 0x0f,0xa8,E,0}, + {APUSHQ, Yfs, Ynone, 0, 0x0f,0xa0,E,0}, + {APUSHQ, Ygs, Ynone, 0, 0x0f,0xa8,E,0}, + + {APUSHW, Ycs, Ynone, 0, Pe,0x0e,E,0}, + {APUSHW, Yss, Ynone, 0, Pe,0x16,E,0}, + {APUSHW, Yds, Ynone, 0, Pe,0x1e,E,0}, + {APUSHW, Yes, Ynone, 0, Pe,0x06,E,0}, + {APUSHW, Yfs, Ynone, 0, Pe,0x0f,0xa0,E}, + {APUSHW, Ygs, Ynone, 0, Pe,0x0f,0xa8,E}, + +/* pop */ + {APOPL, Ynone, Yds, 0, 0x1f,E,0,0}, + {APOPL, Ynone, Yes, 0, 0x07,E,0,0}, + {APOPL, Ynone, Yss, 0, 0x17,E,0,0}, + {APOPL, Ynone, Yfs, 0, 0x0f,0xa1,E,0}, + {APOPL, Ynone, Ygs, 0, 0x0f,0xa9,E,0}, + {APOPQ, Ynone, Yfs, 0, 0x0f,0xa1,E,0}, + {APOPQ, Ynone, Ygs, 0, 0x0f,0xa9,E,0}, + + {APOPW, Ynone, Yds, 0, Pe,0x1f,E,0}, + {APOPW, Ynone, Yes, 0, Pe,0x07,E,0}, + {APOPW, Ynone, Yss, 0, Pe,0x17,E,0}, + {APOPW, Ynone, Yfs, 0, Pe,0x0f,0xa1,E}, + {APOPW, Ynone, Ygs, 0, Pe,0x0f,0xa9,E}, + +/* mov seg */ + {AMOVW, Yes, Yml, 1, 0x8c,0,0,0}, + {AMOVW, Ycs, Yml, 1, 0x8c,1,0,0}, + {AMOVW, Yss, Yml, 1, 0x8c,2,0,0}, + {AMOVW, Yds, Yml, 1, 0x8c,3,0,0}, + {AMOVW, Yfs, Yml, 1, 0x8c,4,0,0}, + {AMOVW, Ygs, Yml, 1, 0x8c,5,0,0}, + + {AMOVW, Yml, Yes, 2, 0x8e,0,0,0}, + {AMOVW, Yml, Ycs, 2, 0x8e,1,0,0}, + {AMOVW, Yml, Yss, 2, 0x8e,2,0,0}, + {AMOVW, Yml, Yds, 2, 0x8e,3,0,0}, + {AMOVW, Yml, Yfs, 2, 0x8e,4,0,0}, + {AMOVW, Yml, Ygs, 2, 0x8e,5,0,0}, + +/* mov cr */ + {AMOVL, Ycr0, Yml, 3, 0x0f,0x20,0,0}, + {AMOVL, Ycr2, Yml, 3, 0x0f,0x20,2,0}, + {AMOVL, Ycr3, Yml, 3, 0x0f,0x20,3,0}, + {AMOVL, Ycr4, Yml, 3, 0x0f,0x20,4,0}, + {AMOVL, Ycr8, Yml, 3, 0x0f,0x20,8,0}, + {AMOVQ, Ycr0, Yml, 3, 0x0f,0x20,0,0}, + {AMOVQ, Ycr2, Yml, 3, 0x0f,0x20,2,0}, + {AMOVQ, Ycr3, Yml, 3, 0x0f,0x20,3,0}, + {AMOVQ, Ycr4, Yml, 3, 0x0f,0x20,4,0}, + {AMOVQ, Ycr8, Yml, 3, 0x0f,0x20,8,0}, + + {AMOVL, Yml, Ycr0, 4, 0x0f,0x22,0,0}, + {AMOVL, Yml, Ycr2, 4, 0x0f,0x22,2,0}, + {AMOVL, Yml, Ycr3, 4, 0x0f,0x22,3,0}, + {AMOVL, Yml, Ycr4, 4, 0x0f,0x22,4,0}, + {AMOVL, Yml, Ycr8, 4, 0x0f,0x22,8,0}, + {AMOVQ, Yml, Ycr0, 4, 0x0f,0x22,0,0}, + {AMOVQ, Yml, Ycr2, 4, 0x0f,0x22,2,0}, + {AMOVQ, Yml, Ycr3, 4, 0x0f,0x22,3,0}, + {AMOVQ, Yml, Ycr4, 4, 0x0f,0x22,4,0}, + {AMOVQ, Yml, Ycr8, 4, 0x0f,0x22,8,0}, + +/* mov dr */ + {AMOVL, Ydr0, Yml, 3, 0x0f,0x21,0,0}, + {AMOVL, Ydr6, Yml, 3, 0x0f,0x21,6,0}, + {AMOVL, Ydr7, Yml, 3, 0x0f,0x21,7,0}, + {AMOVQ, Ydr0, Yml, 3, 0x0f,0x21,0,0}, + {AMOVQ, Ydr6, Yml, 3, 0x0f,0x21,6,0}, + {AMOVQ, Ydr7, Yml, 3, 0x0f,0x21,7,0}, + + {AMOVL, Yml, Ydr0, 4, 0x0f,0x23,0,0}, + {AMOVL, Yml, Ydr6, 4, 0x0f,0x23,6,0}, + {AMOVL, Yml, Ydr7, 4, 0x0f,0x23,7,0}, + {AMOVQ, Yml, Ydr0, 4, 0x0f,0x23,0,0}, + {AMOVQ, Yml, Ydr6, 4, 0x0f,0x23,6,0}, + {AMOVQ, Yml, Ydr7, 4, 0x0f,0x23,7,0}, + +/* mov tr */ + {AMOVL, Ytr6, Yml, 3, 0x0f,0x24,6,0}, + {AMOVL, Ytr7, Yml, 3, 0x0f,0x24,7,0}, + + {AMOVL, Yml, Ytr6, 4, 0x0f,0x26,6,E}, + {AMOVL, Yml, Ytr7, 4, 0x0f,0x26,7,E}, + +/* lgdt, sgdt, lidt, sidt */ + {AMOVL, Ym, Ygdtr, 4, 0x0f,0x01,2,0}, + {AMOVL, Ygdtr, Ym, 3, 0x0f,0x01,0,0}, + {AMOVL, Ym, Yidtr, 4, 0x0f,0x01,3,0}, + {AMOVL, Yidtr, Ym, 3, 0x0f,0x01,1,0}, + {AMOVQ, Ym, Ygdtr, 4, 0x0f,0x01,2,0}, + {AMOVQ, Ygdtr, Ym, 3, 0x0f,0x01,0,0}, + {AMOVQ, Ym, Yidtr, 4, 0x0f,0x01,3,0}, + {AMOVQ, Yidtr, Ym, 3, 0x0f,0x01,1,0}, + +/* lldt, sldt */ + {AMOVW, Yml, Yldtr, 4, 0x0f,0x00,2,0}, + {AMOVW, Yldtr, Yml, 3, 0x0f,0x00,0,0}, + +/* lmsw, smsw */ + {AMOVW, Yml, Ymsw, 4, 0x0f,0x01,6,0}, + {AMOVW, Ymsw, Yml, 3, 0x0f,0x01,4,0}, + +/* ltr, str */ + {AMOVW, Yml, Ytask, 4, 0x0f,0x00,3,0}, + {AMOVW, Ytask, Yml, 3, 0x0f,0x00,1,0}, + +/* load full pointer */ + {AMOVL, Yml, Ycol, 5, 0,0,0,0}, + {AMOVW, Yml, Ycol, 5, Pe,0,0,0}, + +/* double shift */ + {ASHLL, Ycol, Yml, 6, 0xa4,0xa5,0,0}, + {ASHRL, Ycol, Yml, 6, 0xac,0xad,0,0}, + {ASHLQ, Ycol, Yml, 6, Pw,0xa4,0xa5,0}, + {ASHRQ, Ycol, Yml, 6, Pw,0xac,0xad,0}, + {ASHLW, Ycol, Yml, 6, Pe,0xa4,0xa5,0}, + {ASHRW, Ycol, Yml, 6, Pe,0xac,0xad,0}, + 0 +}; + +int +isax(Adr *a) +{ + + switch(a->type) { + case D_AX: + case D_AL: + case D_AH: + case D_INDIR+D_AX: + return 1; + } + if(a->index == D_AX) + return 1; + return 0; +} + +void +subreg(Prog *p, int from, int to) +{ + + if(debug['Q']) + print("\n%P s/%R/%R/\n", p, from, to); + + if(p->from.type == from) + p->from.type = to; + if(p->to.type == from) + p->to.type = to; + + if(p->from.index == from) + p->from.index = to; + if(p->to.index == from) + p->to.index = to; + + from += D_INDIR; + if(p->from.type == from) + p->from.type = to+D_INDIR; + if(p->to.type == from) + p->to.type = to+D_INDIR; + + if(debug['Q']) + print("%P\n", p); +} + +static int +mediaop(Optab *o, int op, int osize, int z) +{ + switch(op){ + case Pm: + case Pe: + case Pf2: + case Pf3: + if(osize != 1){ + if(op != Pm) + *andptr++ = op; + *andptr++ = Pm; + op = o->op[++z]; + break; + } + default: + if(andptr == and || andptr[-1] != Pm) + *andptr++ = Pm; + break; + } + *andptr++ = op; + return z; +} + +void +doasm(Prog *p) +{ + Optab *o; + Prog *q, pp; + uchar *t; + Movtab *mo; + int z, op, ft, tt, xo, l, pre; + vlong v; + Reloc rel, *r; + Adr *a; + + curp = p; // TODO + + o = opindex[p->as]; + if(o == nil) { + diag("asmins: missing op %P", p); + return; + } + + pre = prefixof(&p->from); + if(pre) + *andptr++ = pre; + pre = prefixof(&p->to); + if(pre) + *andptr++ = pre; + + if(p->ft == 0) + p->ft = oclass(&p->from); + if(p->tt == 0) + p->tt = oclass(&p->to); + + ft = p->ft * Ymax; + tt = p->tt * Ymax; + + t = o->ytab; + if(t == 0) { + diag("asmins: noproto %P", p); + return; + } + xo = o->op[0] == 0x0f; + for(z=0; *t; z+=t[3]+xo,t+=4) + if(ycover[ft+t[0]]) + if(ycover[tt+t[1]]) + goto found; + goto domov; + +found: + switch(o->prefix) { + case Pq: /* 16 bit escape and opcode escape */ + *andptr++ = Pe; + *andptr++ = Pm; + break; + + case Pf2: /* xmm opcode escape */ + case Pf3: + *andptr++ = o->prefix; + *andptr++ = Pm; + break; + + case Pm: /* opcode escape */ + *andptr++ = Pm; + break; + + case Pe: /* 16 bit escape */ + *andptr++ = Pe; + break; + + case Pw: /* 64-bit escape */ + if(p->mode != 64) + diag("asmins: illegal 64: %P", p); + rexflag |= Pw; + break; + + case Pb: /* botch */ + bytereg(&p->from, &p->ft); + bytereg(&p->to, &p->tt); + break; + + case P32: /* 32 bit but illegal if 64-bit mode */ + if(p->mode == 64) + diag("asmins: illegal in 64-bit mode: %P", p); + break; + + case Py: /* 64-bit only, no prefix */ + if(p->mode != 64) + diag("asmins: illegal in %d-bit mode: %P", p->mode, p); + break; + } + + op = o->op[z]; + if(op == 0x0f) { + *andptr++ = op; + op = o->op[++z]; + } + switch(t[2]) { + default: + diag("asmins: unknown z %d %P", t[2], p); + return; + + case Zpseudo: + break; + + case Zlit: + for(; op = o->op[z]; z++) + *andptr++ = op; + break; + + case Zlitm_r: + for(; op = o->op[z]; z++) + *andptr++ = op; + asmand(&p->from, &p->to); + break; + + case Zmb_r: + bytereg(&p->from, &p->ft); + /* fall through */ + case Zm_r: + *andptr++ = op; + asmand(&p->from, &p->to); + break; + + case Zm_r_xm: + mediaop(o, op, t[3], z); + asmand(&p->from, &p->to); + break; + + case Zm_r_xm_nr: + rexflag = 0; + mediaop(o, op, t[3], z); + asmand(&p->from, &p->to); + break; + + case Zm_r_i_xm: + mediaop(o, op, t[3], z); + asmand(&p->from, &p->to); + *andptr++ = p->to.offset; + break; + + case Zm_r_3d: + *andptr++ = 0x0f; + *andptr++ = 0x0f; + asmand(&p->from, &p->to); + *andptr++ = op; + break; + + case Zibm_r: + *andptr++ = op; + asmand(&p->from, &p->to); + *andptr++ = p->to.offset; + break; + + case Zaut_r: + *andptr++ = 0x8d; /* leal */ + if(p->from.type != D_ADDR) + diag("asmins: Zaut sb type ADDR"); + p->from.type = p->from.index; + p->from.index = D_NONE; + asmand(&p->from, &p->to); + p->from.index = p->from.type; + p->from.type = D_ADDR; + break; + + case Zm_o: + *andptr++ = op; + asmando(&p->from, o->op[z+1]); + break; + + case Zr_m: + *andptr++ = op; + asmand(&p->to, &p->from); + break; + + case Zr_m_xm: + mediaop(o, op, t[3], z); + asmand(&p->to, &p->from); + break; + + case Zr_m_xm_nr: + rexflag = 0; + mediaop(o, op, t[3], z); + asmand(&p->to, &p->from); + break; + + case Zr_m_i_xm: + mediaop(o, op, t[3], z); + asmand(&p->to, &p->from); + *andptr++ = p->from.offset; + break; + + case Zo_m: + *andptr++ = op; + asmando(&p->to, o->op[z+1]); + break; + + case Zo_m64: + *andptr++ = op; + asmandsz(&p->to, o->op[z+1], 0, 1); + break; + + case Zm_ibo: + *andptr++ = op; + asmando(&p->from, o->op[z+1]); + *andptr++ = vaddr(&p->to, nil); + break; + + case Zibo_m: + *andptr++ = op; + asmando(&p->to, o->op[z+1]); + *andptr++ = vaddr(&p->from, nil); + break; + + case Zibo_m_xm: + z = mediaop(o, op, t[3], z); + asmando(&p->to, o->op[z+1]); + *andptr++ = vaddr(&p->from, nil); + break; + + case Z_ib: + case Zib_: + if(t[2] == Zib_) + a = &p->from; + else + a = &p->to; + *andptr++ = op; + *andptr++ = vaddr(a, nil); + break; + + case Zib_rp: + rexflag |= regrex[p->to.type] & (Rxb|0x40); + *andptr++ = op + reg[p->to.type]; + *andptr++ = vaddr(&p->from, nil); + break; + + case Zil_rp: + rexflag |= regrex[p->to.type] & Rxb; + *andptr++ = op + reg[p->to.type]; + if(o->prefix == Pe) { + v = vaddr(&p->from, nil); + *andptr++ = v; + *andptr++ = v>>8; + } + else + relput4(p, &p->from); + break; + + case Zo_iw: + *andptr++ = op; + if(p->from.type != D_NONE){ + v = vaddr(&p->from, nil); + *andptr++ = v; + *andptr++ = v>>8; + } + break; + + case Ziq_rp: + v = vaddr(&p->from, &rel); + l = v>>32; + if(l == 0 && rel.siz != 8){ + //p->mark |= 0100; + //print("zero: %llux %P\n", v, p); + rexflag &= ~(0x40|Rxw); + rexflag |= regrex[p->to.type] & Rxb; + *andptr++ = 0xb8 + reg[p->to.type]; + if(rel.type != 0) { + r = addrel(cursym); + *r = rel; + r->off = p->pc + andptr - and; + } + put4(v); + }else if(l == -1 && (v&((uvlong)1<<31))!=0){ /* sign extend */ + //p->mark |= 0100; + //print("sign: %llux %P\n", v, p); + *andptr ++ = 0xc7; + asmando(&p->to, 0); + put4(v); + }else{ /* need all 8 */ + //print("all: %llux %P\n", v, p); + rexflag |= regrex[p->to.type] & Rxb; + *andptr++ = op + reg[p->to.type]; + if(rel.type != 0) { + r = addrel(cursym); + *r = rel; + r->off = p->pc + andptr - and; + } + put8(v); + } + break; + + case Zib_rr: + *andptr++ = op; + asmand(&p->to, &p->to); + *andptr++ = vaddr(&p->from, nil); + break; + + case Z_il: + case Zil_: + if(t[2] == Zil_) + a = &p->from; + else + a = &p->to; + *andptr++ = op; + if(o->prefix == Pe) { + v = vaddr(a, nil); + *andptr++ = v; + *andptr++ = v>>8; + } + else + relput4(p, a); + break; + + case Zm_ilo: + case Zilo_m: + *andptr++ = op; + if(t[2] == Zilo_m) { + a = &p->from; + asmando(&p->to, o->op[z+1]); + } else { + a = &p->to; + asmando(&p->from, o->op[z+1]); + } + if(o->prefix == Pe) { + v = vaddr(a, nil); + *andptr++ = v; + *andptr++ = v>>8; + } + else + relput4(p, a); + break; + + case Zil_rr: + *andptr++ = op; + asmand(&p->to, &p->to); + if(o->prefix == Pe) { + v = vaddr(&p->from, nil); + *andptr++ = v; + *andptr++ = v>>8; + } + else + relput4(p, &p->from); + break; + + case Z_rp: + rexflag |= regrex[p->to.type] & (Rxb|0x40); + *andptr++ = op + reg[p->to.type]; + break; + + case Zrp_: + rexflag |= regrex[p->from.type] & (Rxb|0x40); + *andptr++ = op + reg[p->from.type]; + break; + + case Zclr: + *andptr++ = op; + asmand(&p->to, &p->to); + break; + + case Zcall: + q = p->pcond; + if(q == nil) { + diag("call without target"); + errorexit(); + } + if(q->as != ATEXT) { + // Could handle this case by making D_PCREL + // record the Prog* instead of the Sym*, but let's + // wait until the need arises. + diag("call of non-TEXT %P", q); + errorexit(); + } + *andptr++ = op; + r = addrel(cursym); + r->off = p->pc + andptr - and; + r->sym = q->from.sym; + r->type = D_PCREL; + r->siz = 4; + put4(0); + break; + + case Zbr: + case Zjmp: + // TODO: jump across functions needs reloc + q = p->pcond; + if(q == nil) { + diag("jmp/branch without target"); + errorexit(); + } + if(q->as == ATEXT) { + if(t[2] == Zbr) { + diag("branch to ATEXT"); + errorexit(); + } + *andptr++ = o->op[z+1]; + r = addrel(cursym); + r->off = p->pc + andptr - and; + r->sym = q->from.sym; + r->type = D_PCREL; + r->siz = 4; + put4(0); + break; + } + // Assumes q is in this function. + // TODO: Check in input, preserve in brchain. + + // Fill in backward jump now. + if(p->back & 1) { + v = q->pc - (p->pc + 2); + if(v >= -128) { + *andptr++ = op; + *andptr++ = v; + } else { + v -= 5-2; + if(t[2] == Zbr) { + *andptr++ = 0x0f; + v--; + } + *andptr++ = o->op[z+1]; + *andptr++ = v; + *andptr++ = v>>8; + *andptr++ = v>>16; + *andptr++ = v>>24; + } + break; + } + + // Annotate target; will fill in later. + p->forwd = q->comefrom; + q->comefrom = p; + if(p->back & 2) { // short + *andptr++ = op; + *andptr++ = 0; + } else { + if(t[2] == Zbr) + *andptr++ = 0x0f; + *andptr++ = o->op[z+1]; + *andptr++ = 0; + *andptr++ = 0; + *andptr++ = 0; + *andptr++ = 0; + } + break; + +/* + v = q->pc - p->pc - 2; + if((v >= -128 && v <= 127) || p->pc == -1 || q->pc == -1) { + *andptr++ = op; + *andptr++ = v; + } else { + v -= 5-2; + if(t[2] == Zbr) { + *andptr++ = 0x0f; + v--; + } + *andptr++ = o->op[z+1]; + *andptr++ = v; + *andptr++ = v>>8; + *andptr++ = v>>16; + *andptr++ = v>>24; + } +*/ + break; + + case Zloop: + q = p->pcond; + if(q == nil) { + diag("loop without target"); + errorexit(); + } + v = q->pc - p->pc - 2; + if(v < -128 && v > 127) + diag("loop too far: %P", p); + *andptr++ = op; + *andptr++ = v; + break; + + case Zbyte: + v = vaddr(&p->from, &rel); + if(rel.siz != 0) { + rel.siz = op; + r = addrel(cursym); + *r = rel; + r->off = p->pc + andptr - and; + } + *andptr++ = v; + if(op > 1) { + *andptr++ = v>>8; + if(op > 2) { + *andptr++ = v>>16; + *andptr++ = v>>24; + if(op > 4) { + *andptr++ = v>>32; + *andptr++ = v>>40; + *andptr++ = v>>48; + *andptr++ = v>>56; + } + } + } + break; + } + return; + +domov: + for(mo=ymovtab; mo->as; mo++) + if(p->as == mo->as) + if(ycover[ft+mo->ft]) + if(ycover[tt+mo->tt]){ + t = mo->op; + goto mfound; + } +bad: + if(p->mode != 64){ + /* + * here, the assembly has failed. + * if its a byte instruction that has + * unaddressable registers, try to + * exchange registers and reissue the + * instruction with the operands renamed. + */ + pp = *p; + z = p->from.type; + if(z >= D_BP && z <= D_DI) { + if(isax(&p->to)) { + *andptr++ = 0x87; /* xchg lhs,bx */ + asmando(&p->from, reg[D_BX]); + subreg(&pp, z, D_BX); + doasm(&pp); + *andptr++ = 0x87; /* xchg lhs,bx */ + asmando(&p->from, reg[D_BX]); + } else { + *andptr++ = 0x90 + reg[z]; /* xchg lsh,ax */ + subreg(&pp, z, D_AX); + doasm(&pp); + *andptr++ = 0x90 + reg[z]; /* xchg lsh,ax */ + } + return; + } + z = p->to.type; + if(z >= D_BP && z <= D_DI) { + if(isax(&p->from)) { + *andptr++ = 0x87; /* xchg rhs,bx */ + asmando(&p->to, reg[D_BX]); + subreg(&pp, z, D_BX); + doasm(&pp); + *andptr++ = 0x87; /* xchg rhs,bx */ + asmando(&p->to, reg[D_BX]); + } else { + *andptr++ = 0x90 + reg[z]; /* xchg rsh,ax */ + subreg(&pp, z, D_AX); + doasm(&pp); + *andptr++ = 0x90 + reg[z]; /* xchg rsh,ax */ + } + return; + } + } + diag("doasm: notfound from=%ux to=%ux %P", p->from.type, p->to.type, p); + return; + +mfound: + switch(mo->code) { + default: + diag("asmins: unknown mov %d %P", mo->code, p); + break; + + case 0: /* lit */ + for(z=0; t[z]!=E; z++) + *andptr++ = t[z]; + break; + + case 1: /* r,m */ + *andptr++ = t[0]; + asmando(&p->to, t[1]); + break; + + case 2: /* m,r */ + *andptr++ = t[0]; + asmando(&p->from, t[1]); + break; + + case 3: /* r,m - 2op */ + *andptr++ = t[0]; + *andptr++ = t[1]; + asmando(&p->to, t[2]); + rexflag |= regrex[p->from.type] & (Rxr|0x40); + break; + + case 4: /* m,r - 2op */ + *andptr++ = t[0]; + *andptr++ = t[1]; + asmando(&p->from, t[2]); + rexflag |= regrex[p->to.type] & (Rxr|0x40); + break; + + case 5: /* load full pointer, trash heap */ + if(t[0]) + *andptr++ = t[0]; + switch(p->to.index) { + default: + goto bad; + case D_DS: + *andptr++ = 0xc5; + break; + case D_SS: + *andptr++ = 0x0f; + *andptr++ = 0xb2; + break; + case D_ES: + *andptr++ = 0xc4; + break; + case D_FS: + *andptr++ = 0x0f; + *andptr++ = 0xb4; + break; + case D_GS: + *andptr++ = 0x0f; + *andptr++ = 0xb5; + break; + } + asmand(&p->from, &p->to); + break; + + case 6: /* double shift */ + if(t[0] == Pw){ + if(p->mode != 64) + diag("asmins: illegal 64: %P", p); + rexflag |= Pw; + t++; + }else if(t[0] == Pe){ + *andptr++ = Pe; + t++; + } + z = p->from.type; + switch(z) { + default: + goto bad; + case D_CONST: + *andptr++ = 0x0f; + *andptr++ = t[0]; + asmandsz(&p->to, reg[(int)p->from.index], regrex[(int)p->from.index], 0); + *andptr++ = p->from.offset; + break; + case D_CL: + case D_CX: + *andptr++ = 0x0f; + *andptr++ = t[1]; + asmandsz(&p->to, reg[(int)p->from.index], regrex[(int)p->from.index], 0); + break; + } + break; + } +} + +void +asmins(Prog *p) +{ + int n, np, c; + Reloc *r; + + rexflag = 0; + andptr = and; + asmode = p->mode; + doasm(p); + if(rexflag){ + /* + * as befits the whole approach of the architecture, + * the rex prefix must appear before the first opcode byte + * (and thus after any 66/67/f2/f3/26/2e/3e prefix bytes, but + * before the 0f opcode escape!), or it might be ignored. + * note that the handbook often misleadingly shows 66/f2/f3 in `opcode'. + */ + if(p->mode != 64) + diag("asmins: illegal in mode %d: %P", p->mode, p); + n = andptr - and; + for(np = 0; np < n; np++) { + c = and[np]; + if(c != 0xf2 && c != 0xf3 && (c < 0x64 || c > 0x67) && c != 0x2e && c != 0x3e && c != 0x26) + break; + } + for(r=cursym->r+cursym->nr; r-- > cursym->r; ) { + if(r->off < p->pc) + break; + r->off++; + } + memmove(and+np+1, and+np, n-np); + and[np] = 0x40 | rexflag; + andptr++; + } +} diff --git a/src/cmd/8a/Makefile b/src/cmd/8a/Makefile new file mode 100644 index 000000000..78d361dbd --- /dev/null +++ b/src/cmd/8a/Makefile @@ -0,0 +1,25 @@ +# Copyright 2009 The Go Authors. All rights reserved. +# Use of this source code is governed by a BSD-style +# license that can be found in the LICENSE file. + +include ../../Make.inc +O:=$(HOST_O) + +TARG=8a + +HFILES=\ + a.h\ + y.tab.h\ + ../8l/8.out.h\ + +OFILES=\ + y.tab.$O\ + lex.$O\ + ../8l/enam.$O\ + +YFILES=\ + a.y\ + +include ../../Make.ccmd + +lex.$O: ../cc/macbody ../cc/lexbody diff --git a/src/cmd/8a/a.h b/src/cmd/8a/a.h new file mode 100644 index 000000000..c5c22d7ba --- /dev/null +++ b/src/cmd/8a/a.h @@ -0,0 +1,215 @@ +// Inferno utils/8a/a.h +// http://code.google.com/p/inferno-os/source/browse/utils/8a/a.h +// +// Copyright © 1994-1999 Lucent Technologies Inc. All rights reserved. +// Portions Copyright © 1995-1997 C H Forsyth (forsyth@terzarima.net) +// Portions Copyright © 1997-1999 Vita Nuova Limited +// Portions Copyright © 2000-2007 Vita Nuova Holdings Limited (www.vitanuova.com) +// Portions Copyright © 2004,2006 Bruce Ellis +// Portions Copyright © 2005-2007 C H Forsyth (forsyth@terzarima.net) +// Revisions Copyright © 2000-2007 Lucent Technologies Inc. and others +// Portions Copyright © 2009 The Go Authors. All rights reserved. +// +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to deal +// in the Software without restriction, including without limitation the rights +// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +// copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +// THE SOFTWARE. + +#include +#include "../8l/8.out.h" + + +#ifndef EXTERN +#define EXTERN extern +#endif + +#undef getc +#undef ungetc +#undef BUFSIZ + +#define getc ccgetc +#define ungetc ccungetc + +typedef struct Sym Sym; +typedef struct Ref Ref; +typedef struct Gen Gen; +typedef struct Io Io; +typedef struct Hist Hist; +typedef struct Gen2 Gen2; + +#define MAXALIGN 7 +#define FPCHIP 1 +#define NSYMB 500 +#define BUFSIZ 8192 +#define HISTSZ 20 +#ifndef EOF +#define EOF (-1) +#endif +#define IGN (-2) +#define GETC() ((--fi.c < 0)? filbuf(): *fi.p++ & 0xff) +#define NHASH 503 +#define STRINGSZ 200 +#define NMACRO 10 + +struct Sym +{ + Sym* link; + Ref* ref; + char* macro; + int32 value; + ushort type; + char *name; + char sym; +}; +#define S ((Sym*)0) + +struct Ref +{ + int class; +}; + +EXTERN struct +{ + char* p; + int c; +} fi; + +struct Io +{ + Io* link; + char b[BUFSIZ]; + char* p; + short c; + short f; +}; +#define I ((Io*)0) + +EXTERN struct +{ + Sym* sym; + short type; +} h[NSYM]; + +struct Gen +{ + double dval; + char sval[8]; + int32 offset; + int32 offset2; + Sym* sym; + short type; + short index; + short scale; +}; +struct Gen2 +{ + Gen from; + Gen to; +}; + +struct Hist +{ + Hist* link; + char* name; + int32 line; + int32 offset; +}; +#define H ((Hist*)0) + +enum +{ + CLAST, + CMACARG, + CMACRO, + CPREPROC, +}; + + +EXTERN char debug[256]; +EXTERN Sym* hash[NHASH]; +EXTERN char** Dlist; +EXTERN int nDlist; +EXTERN Hist* ehist; +EXTERN int newflag; +EXTERN Hist* hist; +EXTERN char* hunk; +EXTERN char** include; +EXTERN Io* iofree; +EXTERN Io* ionext; +EXTERN Io* iostack; +EXTERN int32 lineno; +EXTERN int nerrors; +EXTERN int32 nhunk; +EXTERN int ninclude; +EXTERN int32 nsymb; +EXTERN Gen nullgen; +EXTERN char* outfile; +EXTERN int pass; +EXTERN char* pathname; +EXTERN int32 pc; +EXTERN int peekc; +EXTERN int32 stmtline; +EXTERN int sym; +EXTERN char* symb; +EXTERN int thechar; +EXTERN char* thestring; +EXTERN int32 thunk; +EXTERN Biobuf obuf; + +void* alloc(int32); +void* allocn(void*, int32, int32); +void ensuresymb(int32); +void errorexit(void); +void pushio(void); +void newio(void); +void newfile(char*, int); +Sym* slookup(char*); +Sym* lookup(void); +void syminit(Sym*); +int32 yylex(void); +int getc(void); +int getnsc(void); +void unget(int); +int escchar(int); +void cinit(void); +void checkscale(int); +void pinit(char*); +void cclean(void); +int isreg(Gen*); +void outcode(int, Gen2*); +void outhist(void); +void zaddr(Gen*, int); +void zname(char*, int, int); +void ieeedtod(Ieee*, double); +int filbuf(void); +Sym* getsym(void); +void domacro(void); +void macund(void); +void macdef(void); +void macexpand(Sym*, char*); +void macinc(void); +void macprag(void); +void maclin(void); +void macif(int); +void macend(void); +void dodefine(char*); +void prfile(int32); +void linehist(char*, int); +void gethunk(void); +void yyerror(char*, ...); +int yyparse(void); +void setinclude(char*); +int assemble(char*); diff --git a/src/cmd/8a/a.y b/src/cmd/8a/a.y new file mode 100644 index 000000000..a8ac773da --- /dev/null +++ b/src/cmd/8a/a.y @@ -0,0 +1,614 @@ +// Inferno utils/8a/a.y +// http://code.google.com/p/inferno-os/source/browse/utils/8a/a.y +// +// Copyright © 1994-1999 Lucent Technologies Inc. All rights reserved. +// Portions Copyright © 1995-1997 C H Forsyth (forsyth@terzarima.net) +// Portions Copyright © 1997-1999 Vita Nuova Limited +// Portions Copyright © 2000-2007 Vita Nuova Holdings Limited (www.vitanuova.com) +// Portions Copyright © 2004,2006 Bruce Ellis +// Portions Copyright © 2005-2007 C H Forsyth (forsyth@terzarima.net) +// Revisions Copyright © 2000-2007 Lucent Technologies Inc. and others +// Portions Copyright © 2009 The Go Authors. All rights reserved. +// +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to deal +// in the Software without restriction, including without limitation the rights +// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +// copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +// THE SOFTWARE. + +%{ +#include +#include /* if we don't, bison will, and a.h re-#defines getc */ +#include +#include "a.h" +%} +%union { + Sym *sym; + int32 lval; + struct { + int32 v1; + int32 v2; + } con2; + double dval; + char sval[8]; + Gen gen; + Gen2 gen2; +} +%left '|' +%left '^' +%left '&' +%left '<' '>' +%left '+' '-' +%left '*' '/' '%' +%token LTYPE0 LTYPE1 LTYPE2 LTYPE3 LTYPE4 +%token LTYPEC LTYPED LTYPEN LTYPER LTYPET LTYPES LTYPEM LTYPEI LTYPEG +%token LCONST LFP LPC LSB +%token LBREG LLREG LSREG LFREG +%token LFCONST +%token LSCONST LSP +%token LNAME LLAB LVAR +%type con expr pointer offset +%type con2 +%type mem imm imm2 reg nam rel rem rim rom omem nmem +%type nonnon nonrel nonrem rimnon rimrem remrim +%type spec1 spec2 spec3 spec4 spec5 spec6 spec7 spec8 +%% +prog: +| prog + { + stmtline = lineno; + } + line + +line: + LLAB ':' + { + if($1->value != pc) + yyerror("redeclaration of %s", $1->name); + $1->value = pc; + } + line +| LNAME ':' + { + $1->type = LLAB; + $1->value = pc; + } + line +| ';' +| inst ';' +| error ';' + +inst: + LNAME '=' expr + { + $1->type = LVAR; + $1->value = $3; + } +| LVAR '=' expr + { + if($1->value != $3) + yyerror("redeclaration of %s", $1->name); + $1->value = $3; + } +| LTYPE0 nonnon { outcode($1, &$2); } +| LTYPE1 nonrem { outcode($1, &$2); } +| LTYPE2 rimnon { outcode($1, &$2); } +| LTYPE3 rimrem { outcode($1, &$2); } +| LTYPE4 remrim { outcode($1, &$2); } +| LTYPER nonrel { outcode($1, &$2); } +| LTYPED spec1 { outcode($1, &$2); } +| LTYPET spec2 { outcode($1, &$2); } +| LTYPEC spec3 { outcode($1, &$2); } +| LTYPEN spec4 { outcode($1, &$2); } +| LTYPES spec5 { outcode($1, &$2); } +| LTYPEM spec6 { outcode($1, &$2); } +| LTYPEI spec7 { outcode($1, &$2); } +| LTYPEG spec8 { outcode($1, &$2); } + +nonnon: + { + $$.from = nullgen; + $$.to = nullgen; + } +| ',' + { + $$.from = nullgen; + $$.to = nullgen; + } + +rimrem: + rim ',' rem + { + $$.from = $1; + $$.to = $3; + } + +remrim: + rem ',' rim + { + $$.from = $1; + $$.to = $3; + } + +rimnon: + rim ',' + { + $$.from = $1; + $$.to = nullgen; + } +| rim + { + $$.from = $1; + $$.to = nullgen; + } + +nonrem: + ',' rem + { + $$.from = nullgen; + $$.to = $2; + } +| rem + { + $$.from = nullgen; + $$.to = $1; + } + +nonrel: + ',' rel + { + $$.from = nullgen; + $$.to = $2; + } +| rel + { + $$.from = nullgen; + $$.to = $1; + } + +spec1: /* DATA */ + nam '/' con ',' imm + { + $$.from = $1; + $$.from.scale = $3; + $$.to = $5; + } + +spec2: /* TEXT */ + mem ',' imm2 + { + $$.from = $1; + $$.to = $3; + } +| mem ',' con ',' imm2 + { + $$.from = $1; + $$.from.scale = $3; + $$.to = $5; + } + +spec3: /* JMP/CALL */ + ',' rom + { + $$.from = nullgen; + $$.to = $2; + } +| rom + { + $$.from = nullgen; + $$.to = $1; + } + +spec4: /* NOP */ + nonnon +| nonrem + +spec5: /* SHL/SHR */ + rim ',' rem + { + $$.from = $1; + $$.to = $3; + } +| rim ',' rem ':' LLREG + { + $$.from = $1; + $$.to = $3; + if($$.from.index != D_NONE) + yyerror("dp shift with lhs index"); + $$.from.index = $5; + } + +spec6: /* MOVW/MOVL */ + rim ',' rem + { + $$.from = $1; + $$.to = $3; + } +| rim ',' rem ':' LSREG + { + $$.from = $1; + $$.to = $3; + if($$.to.index != D_NONE) + yyerror("dp move with lhs index"); + $$.to.index = $5; + } + +spec7: + rim ',' + { + $$.from = $1; + $$.to = nullgen; + } +| rim + { + $$.from = $1; + $$.to = nullgen; + } +| rim ',' rem + { + $$.from = $1; + $$.to = $3; + } + +spec8: /* GLOBL */ + mem ',' imm + { + $$.from = $1; + $$.to = $3; + } +| mem ',' con ',' imm + { + $$.from = $1; + $$.from.scale = $3; + $$.to = $5; + } + +rem: + reg +| mem + +rom: + rel +| nmem +| '*' reg + { + $$ = $2; + } +| '*' omem + { + $$ = $2; + } +| reg +| omem +| imm + +rim: + rem +| imm + +rel: + con '(' LPC ')' + { + $$ = nullgen; + $$.type = D_BRANCH; + $$.offset = $1 + pc; + } +| LNAME offset + { + $$ = nullgen; + if(pass == 2) + yyerror("undefined label: %s", $1->name); + $$.type = D_BRANCH; + $$.sym = $1; + $$.offset = $2; + } +| LLAB offset + { + $$ = nullgen; + $$.type = D_BRANCH; + $$.sym = $1; + $$.offset = $1->value + $2; + } + +reg: + LBREG + { + $$ = nullgen; + $$.type = $1; + } +| LFREG + { + $$ = nullgen; + $$.type = $1; + } +| LLREG + { + $$ = nullgen; + $$.type = $1; + } +| LSP + { + $$ = nullgen; + $$.type = D_SP; + } +| LSREG + { + $$ = nullgen; + $$.type = $1; + } + +imm: + '$' con + { + $$ = nullgen; + $$.type = D_CONST; + $$.offset = $2; + } +| '$' nam + { + $$ = $2; + $$.index = $2.type; + $$.type = D_ADDR; + /* + if($2.type == D_AUTO || $2.type == D_PARAM) + yyerror("constant cannot be automatic: %s", + $2.sym->name); + */ + } +| '$' LSCONST + { + $$ = nullgen; + $$.type = D_SCONST; + memcpy($$.sval, $2, sizeof($$.sval)); + } +| '$' LFCONST + { + $$ = nullgen; + $$.type = D_FCONST; + $$.dval = $2; + } +| '$' '(' LFCONST ')' + { + $$ = nullgen; + $$.type = D_FCONST; + $$.dval = $3; + } +| '$' '-' LFCONST + { + $$ = nullgen; + $$.type = D_FCONST; + $$.dval = -$3; + } + +imm2: + '$' con2 + { + $$ = nullgen; + $$.type = D_CONST2; + $$.offset = $2.v1; + $$.offset2 = $2.v2; + } + +con2: + LCONST + { + $$.v1 = $1; + $$.v2 = 0; + } +| '-' LCONST + { + $$.v1 = -$2; + $$.v2 = 0; + } +| LCONST '-' LCONST + { + $$.v1 = $1; + $$.v2 = $3; + } +| '-' LCONST '-' LCONST + { + $$.v1 = -$2; + $$.v2 = $4; + } + +mem: + omem +| nmem + +omem: + con + { + $$ = nullgen; + $$.type = D_INDIR+D_NONE; + $$.offset = $1; + } +| con '(' LLREG ')' + { + $$ = nullgen; + $$.type = D_INDIR+$3; + $$.offset = $1; + } +| con '(' LSP ')' + { + $$ = nullgen; + $$.type = D_INDIR+D_SP; + $$.offset = $1; + } +| con '(' LLREG '*' con ')' + { + $$ = nullgen; + $$.type = D_INDIR+D_NONE; + $$.offset = $1; + $$.index = $3; + $$.scale = $5; + checkscale($$.scale); + } +| con '(' LLREG ')' '(' LLREG '*' con ')' + { + $$ = nullgen; + $$.type = D_INDIR+$3; + $$.offset = $1; + $$.index = $6; + $$.scale = $8; + checkscale($$.scale); + } +| '(' LLREG ')' + { + $$ = nullgen; + $$.type = D_INDIR+$2; + } +| '(' LSP ')' + { + $$ = nullgen; + $$.type = D_INDIR+D_SP; + } +| con '(' LSREG ')' + { + $$ = nullgen; + $$.type = D_INDIR+$3; + $$.offset = $1; + } +| '(' LLREG '*' con ')' + { + $$ = nullgen; + $$.type = D_INDIR+D_NONE; + $$.index = $2; + $$.scale = $4; + checkscale($$.scale); + } +| '(' LLREG ')' '(' LLREG '*' con ')' + { + $$ = nullgen; + $$.type = D_INDIR+$2; + $$.index = $5; + $$.scale = $7; + checkscale($$.scale); + } + +nmem: + nam + { + $$ = $1; + } +| nam '(' LLREG '*' con ')' + { + $$ = $1; + $$.index = $3; + $$.scale = $5; + checkscale($$.scale); + } + +nam: + LNAME offset '(' pointer ')' + { + $$ = nullgen; + $$.type = $4; + $$.sym = $1; + $$.offset = $2; + } +| LNAME '<' '>' offset '(' LSB ')' + { + $$ = nullgen; + $$.type = D_STATIC; + $$.sym = $1; + $$.offset = $4; + } + +offset: + { + $$ = 0; + } +| '+' con + { + $$ = $2; + } +| '-' con + { + $$ = -$2; + } + +pointer: + LSB +| LSP + { + $$ = D_AUTO; + } +| LFP + +con: + LCONST +| LVAR + { + $$ = $1->value; + } +| '-' con + { + $$ = -$2; + } +| '+' con + { + $$ = $2; + } +| '~' con + { + $$ = ~$2; + } +| '(' expr ')' + { + $$ = $2; + } + +expr: + con +| expr '+' expr + { + $$ = $1 + $3; + } +| expr '-' expr + { + $$ = $1 - $3; + } +| expr '*' expr + { + $$ = $1 * $3; + } +| expr '/' expr + { + $$ = $1 / $3; + } +| expr '%' expr + { + $$ = $1 % $3; + } +| expr '<' '<' expr + { + $$ = $1 << $4; + } +| expr '>' '>' expr + { + $$ = $1 >> $4; + } +| expr '&' expr + { + $$ = $1 & $3; + } +| expr '^' expr + { + $$ = $1 ^ $3; + } +| expr '|' expr + { + $$ = $1 | $3; + } diff --git a/src/cmd/8a/doc.go b/src/cmd/8a/doc.go new file mode 100644 index 000000000..a43b4461f --- /dev/null +++ b/src/cmd/8a/doc.go @@ -0,0 +1,14 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +/* + +8a is a version of the Plan 9 assembler. The original is documented at + + http://plan9.bell-labs.com/magic/man2html/1/2a + +Its target architecture is the x86, referred to by these tools for historical reasons as 386. + +*/ +package documentation diff --git a/src/cmd/8a/lex.c b/src/cmd/8a/lex.c new file mode 100644 index 000000000..e56460e4b --- /dev/null +++ b/src/cmd/8a/lex.c @@ -0,0 +1,972 @@ +// Inferno utils/8a/lex.c +// http://code.google.com/p/inferno-os/source/browse/utils/8a/lex.c +// +// Copyright © 1994-1999 Lucent Technologies Inc. All rights reserved. +// Portions Copyright © 1995-1997 C H Forsyth (forsyth@terzarima.net) +// Portions Copyright © 1997-1999 Vita Nuova Limited +// Portions Copyright © 2000-2007 Vita Nuova Holdings Limited (www.vitanuova.com) +// Portions Copyright © 2004,2006 Bruce Ellis +// Portions Copyright © 2005-2007 C H Forsyth (forsyth@terzarima.net) +// Revisions Copyright © 2000-2007 Lucent Technologies Inc. and others +// Portions Copyright © 2009 The Go Authors. All rights reserved. +// +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to deal +// in the Software without restriction, including without limitation the rights +// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +// copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +// THE SOFTWARE. + +#define EXTERN +#include +#include +#include "a.h" +#include "y.tab.h" + +enum +{ + Plan9 = 1<<0, + Unix = 1<<1, + Windows = 1<<2, +}; + +int +systemtype(int sys) +{ + return sys&Plan9; +} + +int +pathchar(void) +{ + return '/'; +} + +void +main(int argc, char *argv[]) +{ + char *p; + int c; + + thechar = '8'; + thestring = "386"; + + ensuresymb(NSYMB); + memset(debug, 0, sizeof(debug)); + cinit(); + outfile = 0; + setinclude("."); + ARGBEGIN { + default: + c = ARGC(); + if(c >= 0 || c < sizeof(debug)) + debug[c] = 1; + break; + + case 'o': + outfile = ARGF(); + break; + + case 'D': + p = ARGF(); + if(p) { + if (nDlist%8 == 0) + Dlist = allocn(Dlist, nDlist*sizeof(char *), + 8*sizeof(char *)); + Dlist[nDlist++] = p; + } + break; + + case 'I': + p = ARGF(); + setinclude(p); + break; + } ARGEND + if(*argv == 0) { + print("usage: %ca [-options] file.s\n", thechar); + errorexit(); + } + if(argc > 1){ + print("can't assemble multiple files\n"); + errorexit(); + } + if(assemble(argv[0])) + errorexit(); + exits(0); +} + +int +assemble(char *file) +{ + char *ofile, *p; + int i, of; + + ofile = alloc(strlen(file)+3); // +3 for .x\0 (x=thechar) + strcpy(ofile, file); + p = utfrrune(ofile, pathchar()); + if(p) { + include[0] = ofile; + *p++ = 0; + } else + p = ofile; + if(outfile == 0) { + outfile = p; + if(outfile){ + p = utfrrune(outfile, '.'); + if(p) + if(p[1] == 's' && p[2] == 0) + p[0] = 0; + p = utfrune(outfile, 0); + p[0] = '.'; + p[1] = thechar; + p[2] = 0; + } else + outfile = "/dev/null"; + } + + of = create(outfile, OWRITE, 0664); + if(of < 0) { + yyerror("%ca: cannot create %s", thechar, outfile); + errorexit(); + } + Binit(&obuf, of, OWRITE); + + pass = 1; + pinit(file); + + Bprint(&obuf, "go object %s %s %s\n", getgoos(), thestring, getgoversion()); + + for(i=0; itype != LNAME) + yyerror("double initialization %s", itab[i].name); + s->type = itab[i].type; + s->value = itab[i].value; + } + + pathname = allocn(pathname, 0, 100); + if(getwd(pathname, 99) == 0) { + pathname = allocn(pathname, 100, 900); + if(getwd(pathname, 999) == 0) + strcpy(pathname, "/???"); + } +} + +void +checkscale(int scale) +{ + + switch(scale) { + case 1: + case 2: + case 4: + case 8: + return; + } + yyerror("scale must be 1248: %d", scale); +} + +void +syminit(Sym *s) +{ + + s->type = LNAME; + s->value = 0; +} + +void +cclean(void) +{ + Gen2 g2; + + g2.from = nullgen; + g2.to = nullgen; + outcode(AEND, &g2); + Bflush(&obuf); +} + +void +zname(char *n, int t, int s) +{ + + Bputc(&obuf, ANAME); /* as(2) */ + Bputc(&obuf, ANAME>>8); + Bputc(&obuf, t); /* type */ + Bputc(&obuf, s); /* sym */ + while(*n) { + Bputc(&obuf, *n); + n++; + } + Bputc(&obuf, 0); +} + +void +zaddr(Gen *a, int s) +{ + int32 l; + int i, t; + char *n; + Ieee e; + + t = 0; + if(a->index != D_NONE || a->scale != 0) + t |= T_INDEX; + if(a->offset != 0) + t |= T_OFFSET; + if(s != 0) + t |= T_SYM; + + switch(a->type) { + default: + t |= T_TYPE; + break; + case D_FCONST: + t |= T_FCONST; + break; + case D_CONST2: + t |= T_OFFSET|T_OFFSET2; + break; + case D_SCONST: + t |= T_SCONST; + break; + case D_NONE: + break; + } + Bputc(&obuf, t); + + if(t & T_INDEX) { /* implies index, scale */ + Bputc(&obuf, a->index); + Bputc(&obuf, a->scale); + } + if(t & T_OFFSET) { /* implies offset */ + l = a->offset; + Bputc(&obuf, l); + Bputc(&obuf, l>>8); + Bputc(&obuf, l>>16); + Bputc(&obuf, l>>24); + } + if(t & T_OFFSET2) { + l = a->offset2; + Bputc(&obuf, l); + Bputc(&obuf, l>>8); + Bputc(&obuf, l>>16); + Bputc(&obuf, l>>24); + } + if(t & T_SYM) /* implies sym */ + Bputc(&obuf, s); + if(t & T_FCONST) { + ieeedtod(&e, a->dval); + l = e.l; + Bputc(&obuf, l); + Bputc(&obuf, l>>8); + Bputc(&obuf, l>>16); + Bputc(&obuf, l>>24); + l = e.h; + Bputc(&obuf, l); + Bputc(&obuf, l>>8); + Bputc(&obuf, l>>16); + Bputc(&obuf, l>>24); + return; + } + if(t & T_SCONST) { + n = a->sval; + for(i=0; itype); +} + +void +outcode(int a, Gen2 *g2) +{ + int sf, st, t; + Sym *s; + + if(pass == 1) + goto out; + +jackpot: + sf = 0; + s = g2->from.sym; + while(s != S) { + sf = s->sym; + if(sf < 0 || sf >= NSYM) + sf = 0; + t = g2->from.type; + if(t == D_ADDR) + t = g2->from.index; + if(h[sf].type == t) + if(h[sf].sym == s) + break; + zname(s->name, t, sym); + s->sym = sym; + h[sym].sym = s; + h[sym].type = t; + sf = sym; + sym++; + if(sym >= NSYM) + sym = 1; + break; + } + st = 0; + s = g2->to.sym; + while(s != S) { + st = s->sym; + if(st < 0 || st >= NSYM) + st = 0; + t = g2->to.type; + if(t == D_ADDR) + t = g2->to.index; + if(h[st].type == t) + if(h[st].sym == s) + break; + zname(s->name, t, sym); + s->sym = sym; + h[sym].sym = s; + h[sym].type = t; + st = sym; + sym++; + if(sym >= NSYM) + sym = 1; + if(st == sf) + goto jackpot; + break; + } + Bputc(&obuf, a); + Bputc(&obuf, a>>8); + Bputc(&obuf, stmtline); + Bputc(&obuf, stmtline>>8); + Bputc(&obuf, stmtline>>16); + Bputc(&obuf, stmtline>>24); + zaddr(&g2->from, sf); + zaddr(&g2->to, st); + +out: + if(a != AGLOBL && a != ADATA) + pc++; +} + +void +outhist(void) +{ + Gen g; + Hist *h; + char *p, *q, *op, c; + int n; + + g = nullgen; + c = pathchar(); + for(h = hist; h != H; h = h->link) { + p = h->name; + op = 0; + /* on windows skip drive specifier in pathname */ + if(systemtype(Windows) && p && p[1] == ':'){ + p += 2; + c = *p; + } + if(p && p[0] != c && h->offset == 0 && pathname){ + /* on windows skip drive specifier in pathname */ + if(systemtype(Windows) && pathname[1] == ':') { + op = p; + p = pathname+2; + c = *p; + } else if(pathname[0] == c){ + op = p; + p = pathname; + } + } + while(p) { + q = strchr(p, c); + if(q) { + n = q-p; + if(n == 0){ + n = 1; /* leading "/" */ + *p = '/'; /* don't emit "\" on windows */ + } + q++; + } else { + n = strlen(p); + q = 0; + } + if(n) { + Bputc(&obuf, ANAME); + Bputc(&obuf, ANAME>>8); + Bputc(&obuf, D_FILE); /* type */ + Bputc(&obuf, 1); /* sym */ + Bputc(&obuf, '<'); + Bwrite(&obuf, p, n); + Bputc(&obuf, 0); + } + p = q; + if(p == 0 && op) { + p = op; + op = 0; + } + } + g.offset = h->offset; + + Bputc(&obuf, AHISTORY); + Bputc(&obuf, AHISTORY>>8); + Bputc(&obuf, h->line); + Bputc(&obuf, h->line>>8); + Bputc(&obuf, h->line>>16); + Bputc(&obuf, h->line>>24); + zaddr(&nullgen, 0); + zaddr(&g, 0); + } +} + +#include "../cc/lexbody" +#include "../cc/macbody" diff --git a/src/cmd/8c/Makefile b/src/cmd/8c/Makefile new file mode 100644 index 000000000..60f46d3c9 --- /dev/null +++ b/src/cmd/8c/Makefile @@ -0,0 +1,37 @@ +# Copyright 2009 The Go Authors. All rights reserved. +# Use of this source code is governed by a BSD-style +# license that can be found in the LICENSE file. + +include ../../Make.inc +O:=$(HOST_O) + +TARG=8c + +HFILES=\ + gc.h\ + ../8l/8.out.h\ + ../cc/cc.h\ + +OFILES=\ + cgen.$O\ + cgen64.$O\ + div.$O\ + list.$O\ + machcap.$O\ + mul.$O\ + pgen.$O\ + pswt.$O\ + peep.$O\ + reg.$O\ + sgen.$O\ + swt.$O\ + txt.$O\ + ../8l/enam.$O\ + +LIB=\ + ../cc/cc.a\ + +include ../../Make.ccmd + +%.$O: ../cc/%.c + $(HOST_CC) $(HOST_CFLAGS) -c -I. -o $@ ../cc/$*.c diff --git a/src/cmd/8c/cgen.c b/src/cmd/8c/cgen.c new file mode 100644 index 000000000..7f02bd96e --- /dev/null +++ b/src/cmd/8c/cgen.c @@ -0,0 +1,1857 @@ +// Inferno utils/8c/cgen.c +// http://code.google.com/p/inferno-os/source/browse/utils/8c/cgen.c +// +// Copyright © 1994-1999 Lucent Technologies Inc. All rights reserved. +// Portions Copyright © 1995-1997 C H Forsyth (forsyth@terzarima.net) +// Portions Copyright © 1997-1999 Vita Nuova Limited +// Portions Copyright © 2000-2007 Vita Nuova Holdings Limited (www.vitanuova.com) +// Portions Copyright © 2004,2006 Bruce Ellis +// Portions Copyright © 2005-2007 C H Forsyth (forsyth@terzarima.net) +// Revisions Copyright © 2000-2007 Lucent Technologies Inc. and others +// Portions Copyright © 2009 The Go Authors. All rights reserved. +// +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to deal +// in the Software without restriction, including without limitation the rights +// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +// copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +// THE SOFTWARE. + +#include "gc.h" + +/* ,x/^(print|prtree)\(/i/\/\/ */ + +void +cgen(Node *n, Node *nn) +{ + Node *l, *r, *t; + Prog *p1; + Node nod, nod1, nod2, nod3, nod4; + int o, hardleft; + int32 v, curs; + vlong c; + + if(debug['g']) { + prtree(nn, "cgen lhs"); + prtree(n, "cgen"); + } + if(n == Z || n->type == T) + return; + if(typesuv[n->type->etype]) { + sugen(n, nn, n->type->width); + return; + } + l = n->left; + r = n->right; + o = n->op; + + if(n->op == OEXREG || (nn != Z && nn->op == OEXREG)) { + gmove(n, nn); + return; + } + + if(n->addable >= INDEXED) { + if(nn == Z) { + switch(o) { + default: + nullwarn(Z, Z); + break; + case OINDEX: + nullwarn(l, r); + break; + } + return; + } + gmove(n, nn); + return; + } + curs = cursafe; + + if(l->complex >= FNX) + if(r != Z && r->complex >= FNX) + switch(o) { + default: + if(cond(o) && typesuv[l->type->etype]) + break; + + regret(&nod, r); + cgen(r, &nod); + + regsalloc(&nod1, r); + gmove(&nod, &nod1); + + regfree(&nod); + nod = *n; + nod.right = &nod1; + + cgen(&nod, nn); + return; + + case OFUNC: + case OCOMMA: + case OANDAND: + case OOROR: + case OCOND: + case ODOT: + break; + } + + hardleft = l->addable < INDEXED || l->complex >= FNX; + switch(o) { + default: + diag(n, "unknown op in cgen: %O", o); + break; + + case ONEG: + case OCOM: + if(nn == Z) { + nullwarn(l, Z); + break; + } + regalloc(&nod, l, nn); + cgen(l, &nod); + gopcode(o, n->type, Z, &nod); + gmove(&nod, nn); + regfree(&nod); + break; + + case OAS: + if(typefd[n->type->etype]) { + cgen(r, &fregnode0); + if(nn != Z) + gins(AFMOVD, &fregnode0, &fregnode0); + if(l->addable < INDEXED) { + reglcgen(&nod, l, Z); + gmove(&fregnode0, &nod); + regfree(&nod); + } else + gmove(&fregnode0, l); + if(nn != Z) + gmove(&fregnode0, nn); + return; + } + if(l->op == OBIT) + goto bitas; + if(!hardleft) { + if(nn != Z || r->addable < INDEXED) { + if(r->complex >= FNX && nn == Z) + regret(&nod, r); + else + regalloc(&nod, r, nn); + cgen(r, &nod); + gmove(&nod, l); + if(nn != Z) + gmove(&nod, nn); + regfree(&nod); + } else + gmove(r, l); + break; + } + if(l->complex >= r->complex) { + if(l->op == OINDEX && r->op == OCONST) { + gmove(r, l); + break; + } + reglcgen(&nod1, l, Z); + if(r->addable >= INDEXED) { + gmove(r, &nod1); + if(nn != Z) + gmove(r, nn); + regfree(&nod1); + break; + } + regalloc(&nod, r, nn); + cgen(r, &nod); + } else { + regalloc(&nod, r, nn); + cgen(r, &nod); + reglcgen(&nod1, l, Z); + } + gmove(&nod, &nod1); + regfree(&nod); + regfree(&nod1); + break; + + bitas: + n = l->left; + regalloc(&nod, r, nn); + if(l->complex >= r->complex) { + reglcgen(&nod1, n, Z); + cgen(r, &nod); + } else { + cgen(r, &nod); + reglcgen(&nod1, n, Z); + } + regalloc(&nod2, n, Z); + gmove(&nod1, &nod2); + bitstore(l, &nod, &nod1, &nod2, nn); + break; + + case OBIT: + if(nn == Z) { + nullwarn(l, Z); + break; + } + bitload(n, &nod, Z, Z, nn); + gmove(&nod, nn); + regfree(&nod); + break; + + case OLSHR: + case OASHL: + case OASHR: + if(nn == Z) { + nullwarn(l, r); + break; + } + if(r->op == OCONST) { + if(r->vconst == 0) { + cgen(l, nn); + break; + } + regalloc(&nod, l, nn); + cgen(l, &nod); + if(o == OASHL && r->vconst == 1) + gopcode(OADD, n->type, &nod, &nod); + else + gopcode(o, n->type, r, &nod); + gmove(&nod, nn); + regfree(&nod); + break; + } + + /* + * get nod to be D_CX + */ + if(nodreg(&nod, nn, D_CX)) { + regsalloc(&nod1, n); + gmove(&nod, &nod1); + cgen(n, &nod); /* probably a bug */ + gmove(&nod, nn); + gmove(&nod1, &nod); + break; + } + reg[D_CX]++; + if(nn->op == OREGISTER && nn->reg == D_CX) + regalloc(&nod1, l, Z); + else + regalloc(&nod1, l, nn); + if(r->complex >= l->complex) { + cgen(r, &nod); + cgen(l, &nod1); + } else { + cgen(l, &nod1); + cgen(r, &nod); + } + gopcode(o, n->type, &nod, &nod1); + gmove(&nod1, nn); + regfree(&nod); + regfree(&nod1); + break; + + case OADD: + case OSUB: + case OOR: + case OXOR: + case OAND: + if(nn == Z) { + nullwarn(l, r); + break; + } + if(typefd[n->type->etype]) + goto fop; + if(r->op == OCONST) { + if(r->vconst == 0 && o != OAND) { + cgen(l, nn); + break; + } + } + if(n->op == OADD && l->op == OASHL && l->right->op == OCONST + && (r->op != OCONST || r->vconst < -128 || r->vconst > 127)) { + c = l->right->vconst; + if(c > 0 && c <= 3) { + if(l->left->complex >= r->complex) { + regalloc(&nod, l->left, nn); + cgen(l->left, &nod); + if(r->addable < INDEXED) { + regalloc(&nod1, r, Z); + cgen(r, &nod1); + genmuladd(&nod, &nod, 1 << c, &nod1); + regfree(&nod1); + } + else + genmuladd(&nod, &nod, 1 << c, r); + } + else { + regalloc(&nod, r, nn); + cgen(r, &nod); + regalloc(&nod1, l->left, Z); + cgen(l->left, &nod1); + genmuladd(&nod, &nod1, 1 << c, &nod); + regfree(&nod1); + } + gmove(&nod, nn); + regfree(&nod); + break; + } + } + if(r->addable >= INDEXED) { + regalloc(&nod, l, nn); + cgen(l, &nod); + gopcode(o, n->type, r, &nod); + gmove(&nod, nn); + regfree(&nod); + break; + } + if(l->complex >= r->complex) { + regalloc(&nod, l, nn); + cgen(l, &nod); + regalloc(&nod1, r, Z); + cgen(r, &nod1); + gopcode(o, n->type, &nod1, &nod); + } else { + regalloc(&nod1, r, nn); + cgen(r, &nod1); + regalloc(&nod, l, Z); + cgen(l, &nod); + gopcode(o, n->type, &nod1, &nod); + } + gmove(&nod, nn); + regfree(&nod); + regfree(&nod1); + break; + + case OLMOD: + case OMOD: + case OLMUL: + case OLDIV: + case OMUL: + case ODIV: + if(nn == Z) { + nullwarn(l, r); + break; + } + if(typefd[n->type->etype]) + goto fop; + if(r->op == OCONST) { + SET(v); + switch(o) { + case ODIV: + case OMOD: + c = r->vconst; + if(c < 0) + c = -c; + v = xlog2(c); + if(v < 0) + break; + /* fall thru */ + case OMUL: + case OLMUL: + regalloc(&nod, l, nn); + cgen(l, &nod); + switch(o) { + case OMUL: + case OLMUL: + mulgen(n->type, r, &nod); + break; + case ODIV: + sdiv2(r->vconst, v, l, &nod); + break; + case OMOD: + smod2(r->vconst, v, l, &nod); + break; + } + gmove(&nod, nn); + regfree(&nod); + goto done; + case OLDIV: + c = r->vconst; + if((c & 0x80000000) == 0) + break; + regalloc(&nod1, l, Z); + cgen(l, &nod1); + regalloc(&nod, l, nn); + zeroregm(&nod); + gins(ACMPL, &nod1, nodconst(c)); + gins(ASBBL, nodconst(-1), &nod); + regfree(&nod1); + gmove(&nod, nn); + regfree(&nod); + goto done; + } + } + + if(o == OMUL) { + if(l->addable >= INDEXED) { + t = l; + l = r; + r = t; + } + /* should favour AX */ + regalloc(&nod, l, nn); + cgen(l, &nod); + if(r->addable < INDEXED) { + regalloc(&nod1, r, Z); + cgen(r, &nod1); + gopcode(OMUL, n->type, &nod1, &nod); + regfree(&nod1); + }else + gopcode(OMUL, n->type, r, &nod); /* addressible */ + gmove(&nod, nn); + regfree(&nod); + break; + } + + /* + * get nod to be D_AX + * get nod1 to be D_DX + */ + if(nodreg(&nod, nn, D_AX)) { + regsalloc(&nod2, n); + gmove(&nod, &nod2); + v = reg[D_AX]; + reg[D_AX] = 0; + + if(isreg(l, D_AX)) { + nod3 = *n; + nod3.left = &nod2; + cgen(&nod3, nn); + } else + if(isreg(r, D_AX)) { + nod3 = *n; + nod3.right = &nod2; + cgen(&nod3, nn); + } else + cgen(n, nn); + + gmove(&nod2, &nod); + reg[D_AX] = v; + break; + } + if(nodreg(&nod1, nn, D_DX)) { + regsalloc(&nod2, n); + gmove(&nod1, &nod2); + v = reg[D_DX]; + reg[D_DX] = 0; + + if(isreg(l, D_DX)) { + nod3 = *n; + nod3.left = &nod2; + cgen(&nod3, nn); + } else + if(isreg(r, D_DX)) { + nod3 = *n; + nod3.right = &nod2; + cgen(&nod3, nn); + } else + cgen(n, nn); + + gmove(&nod2, &nod1); + reg[D_DX] = v; + break; + } + reg[D_AX]++; + + if(r->op == OCONST && (o == ODIV || o == OLDIV)) { + reg[D_DX]++; + if(l->addable < INDEXED) { + regalloc(&nod2, l, Z); + cgen(l, &nod2); + l = &nod2; + } + if(o == ODIV) + sdivgen(l, r, &nod, &nod1); + else + udivgen(l, r, &nod, &nod1); + gmove(&nod1, nn); + if(l == &nod2) + regfree(l); + goto freeaxdx; + } + + if(l->complex >= r->complex) { + cgen(l, &nod); + reg[D_DX]++; + if(o == ODIV || o == OMOD) + gins(ACDQ, Z, Z); + if(o == OLDIV || o == OLMOD) + zeroregm(&nod1); + if(r->addable < INDEXED || r->op == OCONST) { + regsalloc(&nod3, r); + cgen(r, &nod3); + gopcode(o, n->type, &nod3, Z); + } else + gopcode(o, n->type, r, Z); + } else { + regsalloc(&nod3, r); + cgen(r, &nod3); + cgen(l, &nod); + reg[D_DX]++; + if(o == ODIV || o == OMOD) + gins(ACDQ, Z, Z); + if(o == OLDIV || o == OLMOD) + zeroregm(&nod1); + gopcode(o, n->type, &nod3, Z); + } + if(o == OMOD || o == OLMOD) + gmove(&nod1, nn); + else + gmove(&nod, nn); + freeaxdx: + regfree(&nod); + regfree(&nod1); + break; + + case OASLSHR: + case OASASHL: + case OASASHR: + if(r->op == OCONST) + goto asand; + if(l->op == OBIT) + goto asbitop; + if(typefd[n->type->etype]) + goto asfop; + + /* + * get nod to be D_CX + */ + if(nodreg(&nod, nn, D_CX)) { + regsalloc(&nod1, n); + gmove(&nod, &nod1); + cgen(n, &nod); + if(nn != Z) + gmove(&nod, nn); + gmove(&nod1, &nod); + break; + } + reg[D_CX]++; + + if(r->complex >= l->complex) { + cgen(r, &nod); + if(hardleft) + reglcgen(&nod1, l, Z); + else + nod1 = *l; + } else { + if(hardleft) + reglcgen(&nod1, l, Z); + else + nod1 = *l; + cgen(r, &nod); + } + + gopcode(o, l->type, &nod, &nod1); + regfree(&nod); + if(nn != Z) + gmove(&nod1, nn); + if(hardleft) + regfree(&nod1); + break; + + case OASAND: + case OASADD: + case OASSUB: + case OASXOR: + case OASOR: + asand: + if(l->op == OBIT) + goto asbitop; + if(typefd[n->type->etype]||typefd[r->type->etype]) + goto asfop; + if(l->complex >= r->complex) { + if(hardleft) + reglcgen(&nod, l, Z); + else + nod = *l; + if(r->op != OCONST) { + regalloc(&nod1, r, nn); + cgen(r, &nod1); + gopcode(o, l->type, &nod1, &nod); + regfree(&nod1); + } else + gopcode(o, l->type, r, &nod); + } else { + regalloc(&nod1, r, nn); + cgen(r, &nod1); + if(hardleft) + reglcgen(&nod, l, Z); + else + nod = *l; + gopcode(o, l->type, &nod1, &nod); + regfree(&nod1); + } + if(nn != Z) + gmove(&nod, nn); + if(hardleft) + regfree(&nod); + break; + + case OASLMUL: + case OASLDIV: + case OASLMOD: + case OASMUL: + case OASDIV: + case OASMOD: + if(l->op == OBIT) + goto asbitop; + if(typefd[n->type->etype]||typefd[r->type->etype]) + goto asfop; + if(r->op == OCONST) { + SET(v); + switch(o) { + case OASDIV: + case OASMOD: + c = r->vconst; + if(c < 0) + c = -c; + v = xlog2(c); + if(v < 0) + break; + /* fall thru */ + case OASMUL: + case OASLMUL: + if(hardleft) + reglcgen(&nod2, l, Z); + else + nod2 = *l; + regalloc(&nod, l, nn); + cgen(&nod2, &nod); + switch(o) { + case OASMUL: + case OASLMUL: + mulgen(n->type, r, &nod); + break; + case OASDIV: + sdiv2(r->vconst, v, l, &nod); + break; + case OASMOD: + smod2(r->vconst, v, l, &nod); + break; + } + havev: + gmove(&nod, &nod2); + if(nn != Z) + gmove(&nod, nn); + if(hardleft) + regfree(&nod2); + regfree(&nod); + goto done; + case OASLDIV: + c = r->vconst; + if((c & 0x80000000) == 0) + break; + if(hardleft) + reglcgen(&nod2, l, Z); + else + nod2 = *l; + regalloc(&nod1, l, nn); + cgen(&nod2, &nod1); + regalloc(&nod, l, nn); + zeroregm(&nod); + gins(ACMPL, &nod1, nodconst(c)); + gins(ASBBL, nodconst(-1), &nod); + regfree(&nod1); + goto havev; + } + } + + if(o == OASMUL) { + /* should favour AX */ + regalloc(&nod, l, nn); + if(r->complex >= FNX) { + regalloc(&nod1, r, Z); + cgen(r, &nod1); + r = &nod1; + } + if(hardleft) + reglcgen(&nod2, l, Z); + else + nod2 = *l; + cgen(&nod2, &nod); + if(r->addable < INDEXED) { + if(r->complex < FNX) { + regalloc(&nod1, r, Z); + cgen(r, &nod1); + } + gopcode(OASMUL, n->type, &nod1, &nod); + regfree(&nod1); + } + else + gopcode(OASMUL, n->type, r, &nod); + if(r == &nod1) + regfree(r); + gmove(&nod, &nod2); + if(nn != Z) + gmove(&nod, nn); + regfree(&nod); + if(hardleft) + regfree(&nod2); + break; + } + + /* + * get nod to be D_AX + * get nod1 to be D_DX + */ + if(nodreg(&nod, nn, D_AX)) { + regsalloc(&nod2, n); + gmove(&nod, &nod2); + v = reg[D_AX]; + reg[D_AX] = 0; + + if(isreg(l, D_AX)) { + nod3 = *n; + nod3.left = &nod2; + cgen(&nod3, nn); + } else + if(isreg(r, D_AX)) { + nod3 = *n; + nod3.right = &nod2; + cgen(&nod3, nn); + } else + cgen(n, nn); + + gmove(&nod2, &nod); + reg[D_AX] = v; + break; + } + if(nodreg(&nod1, nn, D_DX)) { + regsalloc(&nod2, n); + gmove(&nod1, &nod2); + v = reg[D_DX]; + reg[D_DX] = 0; + + if(isreg(l, D_DX)) { + nod3 = *n; + nod3.left = &nod2; + cgen(&nod3, nn); + } else + if(isreg(r, D_DX)) { + nod3 = *n; + nod3.right = &nod2; + cgen(&nod3, nn); + } else + cgen(n, nn); + + gmove(&nod2, &nod1); + reg[D_DX] = v; + break; + } + reg[D_AX]++; + reg[D_DX]++; + + if(l->complex >= r->complex) { + if(hardleft) + reglcgen(&nod2, l, Z); + else + nod2 = *l; + cgen(&nod2, &nod); + if(r->op == OCONST) { + switch(o) { + case OASDIV: + sdivgen(&nod2, r, &nod, &nod1); + goto divdone; + case OASLDIV: + udivgen(&nod2, r, &nod, &nod1); + divdone: + gmove(&nod1, &nod2); + if(nn != Z) + gmove(&nod1, nn); + goto freelxaxdx; + } + } + if(o == OASDIV || o == OASMOD) + gins(ACDQ, Z, Z); + if(o == OASLDIV || o == OASLMOD) + zeroregm(&nod1); + if(r->addable < INDEXED || r->op == OCONST || + !typeil[r->type->etype]) { + regalloc(&nod3, r, Z); + cgen(r, &nod3); + gopcode(o, l->type, &nod3, Z); + regfree(&nod3); + } else + gopcode(o, n->type, r, Z); + } else { + regalloc(&nod3, r, Z); + cgen(r, &nod3); + if(hardleft) + reglcgen(&nod2, l, Z); + else + nod2 = *l; + cgen(&nod2, &nod); + if(o == OASDIV || o == OASMOD) + gins(ACDQ, Z, Z); + if(o == OASLDIV || o == OASLMOD) + zeroregm(&nod1); + gopcode(o, l->type, &nod3, Z); + regfree(&nod3); + } + if(o == OASMOD || o == OASLMOD) { + gmove(&nod1, &nod2); + if(nn != Z) + gmove(&nod1, nn); + } else { + gmove(&nod, &nod2); + if(nn != Z) + gmove(&nod, nn); + } + freelxaxdx: + if(hardleft) + regfree(&nod2); + regfree(&nod); + regfree(&nod1); + break; + + fop: + if(l->complex >= r->complex) { + cgen(l, &fregnode0); + if(r->addable < INDEXED) { + cgen(r, &fregnode0); + fgopcode(o, &fregnode0, &fregnode1, 1, 0); + } else + fgopcode(o, r, &fregnode0, 0, 0); + } else { + cgen(r, &fregnode0); + if(l->addable < INDEXED) { + cgen(l, &fregnode0); + fgopcode(o, &fregnode0, &fregnode1, 1, 1); + } else + fgopcode(o, l, &fregnode0, 0, 1); + } + gmove(&fregnode0, nn); + break; + + asfop: + if(l->complex >= r->complex) { + if(hardleft) + reglcgen(&nod, l, Z); + else + nod = *l; + cgen(r, &fregnode0); + } else { + cgen(r, &fregnode0); + if(hardleft) + reglcgen(&nod, l, Z); + else + nod = *l; + } + if(!typefd[l->type->etype]) { + gmove(&nod, &fregnode0); + fgopcode(o, &fregnode0, &fregnode1, 1, 1); + } else + fgopcode(o, &nod, &fregnode0, 0, 1); + if(nn != Z) + gins(AFMOVD, &fregnode0, &fregnode0); + gmove(&fregnode0, &nod); + if(nn != Z) + gmove(&fregnode0, nn); + if(hardleft) + regfree(&nod); + break; + + asbitop: + regalloc(&nod4, n, nn); + if(l->complex >= r->complex) { + bitload(l, &nod, &nod1, &nod2, &nod4); + regalloc(&nod3, r, Z); + cgen(r, &nod3); + } else { + regalloc(&nod3, r, Z); + cgen(r, &nod3); + bitload(l, &nod, &nod1, &nod2, &nod4); + } + gmove(&nod, &nod4); + + if(typefd[nod3.type->etype]) + fgopcode(o, &fregnode0, &fregnode1, 1, 1); + else { + Node onod; + + /* incredible grot ... */ + onod = nod3; + onod.op = o; + onod.complex = 2; + onod.addable = 0; + onod.type = tfield; + onod.left = &nod4; + onod.right = &nod3; + cgen(&onod, Z); + } + regfree(&nod3); + gmove(&nod4, &nod); + regfree(&nod4); + bitstore(l, &nod, &nod1, &nod2, nn); + break; + + case OADDR: + if(nn == Z) { + nullwarn(l, Z); + break; + } + lcgen(l, nn); + break; + + case OFUNC: + if(l->complex >= FNX) { + if(l->op != OIND) + diag(n, "bad function call"); + + regret(&nod, l->left); + cgen(l->left, &nod); + regsalloc(&nod1, l->left); + gmove(&nod, &nod1); + regfree(&nod); + + nod = *n; + nod.left = &nod2; + nod2 = *l; + nod2.left = &nod1; + nod2.complex = 1; + cgen(&nod, nn); + + return; + } + gargs(r, &nod, &nod1); + if(l->addable < INDEXED) { + reglcgen(&nod, l, nn); + nod.op = OREGISTER; + gopcode(OFUNC, n->type, Z, &nod); + regfree(&nod); + } else + gopcode(OFUNC, n->type, Z, l); + if(REGARG >= 0 && reg[REGARG]) + reg[REGARG]--; + if(nn != Z) { + regret(&nod, n); + gmove(&nod, nn); + regfree(&nod); + } else + if(typefd[n->type->etype]) + gins(AFMOVDP, &fregnode0, &fregnode0); + break; + + case OIND: + if(nn == Z) { + nullwarn(l, Z); + break; + } + regialloc(&nod, n, nn); + r = l; + while(r->op == OADD) + r = r->right; + if(sconst(r)) { + v = r->vconst; + r->vconst = 0; + cgen(l, &nod); + nod.xoffset += v; + r->vconst = v; + } else + cgen(l, &nod); + regind(&nod, n); + gmove(&nod, nn); + regfree(&nod); + break; + + case OEQ: + case ONE: + case OLE: + case OLT: + case OGE: + case OGT: + case OLO: + case OLS: + case OHI: + case OHS: + if(nn == Z) { + nullwarn(l, r); + break; + } + boolgen(n, 1, nn); + break; + + case OANDAND: + case OOROR: + boolgen(n, 1, nn); + if(nn == Z) + patch(p, pc); + break; + + case ONOT: + if(nn == Z) { + nullwarn(l, Z); + break; + } + boolgen(n, 1, nn); + break; + + case OCOMMA: + cgen(l, Z); + cgen(r, nn); + break; + + case OCAST: + if(nn == Z) { + nullwarn(l, Z); + break; + } + /* + * convert from types l->n->nn + */ + if(nocast(l->type, n->type) && nocast(n->type, nn->type)) { + /* both null, gen l->nn */ + cgen(l, nn); + break; + } + if(typev[l->type->etype]) { + cgen64(n, nn); + break; + } + regalloc(&nod, l, nn); + cgen(l, &nod); + regalloc(&nod1, n, &nod); + gmove(&nod, &nod1); + gmove(&nod1, nn); + regfree(&nod1); + regfree(&nod); + break; + + case ODOT: + sugen(l, nodrat, l->type->width); + if(nn == Z) + break; + warn(n, "non-interruptable temporary"); + nod = *nodrat; + if(!r || r->op != OCONST) { + diag(n, "DOT and no offset"); + break; + } + nod.xoffset += (int32)r->vconst; + nod.type = n->type; + cgen(&nod, nn); + break; + + case OCOND: + bcgen(l, 1); + p1 = p; + cgen(r->left, nn); + gbranch(OGOTO); + patch(p1, pc); + p1 = p; + cgen(r->right, nn); + patch(p1, pc); + break; + + case OPOSTINC: + case OPOSTDEC: + v = 1; + if(l->type->etype == TIND) + v = l->type->link->width; + if(o == OPOSTDEC) + v = -v; + if(l->op == OBIT) + goto bitinc; + if(nn == Z) + goto pre; + + if(hardleft) + reglcgen(&nod, l, Z); + else + nod = *l; + + if(typefd[n->type->etype]) + goto fltinc; + gmove(&nod, nn); + gopcode(OADD, n->type, nodconst(v), &nod); + if(hardleft) + regfree(&nod); + break; + + case OPREINC: + case OPREDEC: + v = 1; + if(l->type->etype == TIND) + v = l->type->link->width; + if(o == OPREDEC) + v = -v; + if(l->op == OBIT) + goto bitinc; + + pre: + if(hardleft) + reglcgen(&nod, l, Z); + else + nod = *l; + if(typefd[n->type->etype]) + goto fltinc; + gopcode(OADD, n->type, nodconst(v), &nod); + if(nn != Z) + gmove(&nod, nn); + if(hardleft) + regfree(&nod); + break; + + fltinc: + gmove(&nod, &fregnode0); + if(nn != Z && (o == OPOSTINC || o == OPOSTDEC)) + gins(AFMOVD, &fregnode0, &fregnode0); + gins(AFLD1, Z, Z); + if(v < 0) + fgopcode(OSUB, &fregnode0, &fregnode1, 1, 0); + else + fgopcode(OADD, &fregnode0, &fregnode1, 1, 0); + if(nn != Z && (o == OPREINC || o == OPREDEC)) + gins(AFMOVD, &fregnode0, &fregnode0); + gmove(&fregnode0, &nod); + if(hardleft) + regfree(&nod); + break; + + bitinc: + if(nn != Z && (o == OPOSTINC || o == OPOSTDEC)) { + bitload(l, &nod, &nod1, &nod2, Z); + gmove(&nod, nn); + gopcode(OADD, tfield, nodconst(v), &nod); + bitstore(l, &nod, &nod1, &nod2, Z); + break; + } + bitload(l, &nod, &nod1, &nod2, nn); + gopcode(OADD, tfield, nodconst(v), &nod); + bitstore(l, &nod, &nod1, &nod2, nn); + break; + } +done: + cursafe = curs; +} + +void +reglcgen(Node *t, Node *n, Node *nn) +{ + Node *r; + int32 v; + + regialloc(t, n, nn); + if(n->op == OIND) { + r = n->left; + while(r->op == OADD) + r = r->right; + if(sconst(r)) { + v = r->vconst; + r->vconst = 0; + lcgen(n, t); + t->xoffset += v; + r->vconst = v; + regind(t, n); + return; + } + } + lcgen(n, t); + regind(t, n); +} + +void +lcgen(Node *n, Node *nn) +{ + Prog *p1; + Node nod; + + if(debug['g']) { + prtree(nn, "lcgen lhs"); + prtree(n, "lcgen"); + } + if(n == Z || n->type == T) + return; + if(nn == Z) { + nn = &nod; + regalloc(&nod, n, Z); + } + switch(n->op) { + default: + if(n->addable < INDEXED) { + diag(n, "unknown op in lcgen: %O", n->op); + break; + } + gopcode(OADDR, n->type, n, nn); + break; + + case OCOMMA: + cgen(n->left, n->left); + lcgen(n->right, nn); + break; + + case OIND: + cgen(n->left, nn); + break; + + case OCOND: + bcgen(n->left, 1); + p1 = p; + lcgen(n->right->left, nn); + gbranch(OGOTO); + patch(p1, pc); + p1 = p; + lcgen(n->right->right, nn); + patch(p1, pc); + break; + } +} + +void +bcgen(Node *n, int true) +{ + + if(n->type == T) + gbranch(OGOTO); + else + boolgen(n, true, Z); +} + +void +boolgen(Node *n, int true, Node *nn) +{ + int o; + Prog *p1, *p2; + Node *l, *r, nod, nod1; + int32 curs; + + if(debug['g']) { + prtree(nn, "boolgen lhs"); + prtree(n, "boolgen"); + } + curs = cursafe; + l = n->left; + r = n->right; + switch(n->op) { + + default: + if(typev[n->type->etype]) { + testv(n, true); + goto com; + } + o = ONE; + if(true) + o = OEQ; + if(typefd[n->type->etype]) { + if(n->addable < INDEXED) { + cgen(n, &fregnode0); + gins(AFLDZ, Z, Z); + fgopcode(o, &fregnode0, &fregnode1, 1, 1); + } else { + gins(AFLDZ, Z, Z); + fgopcode(o, n, &fregnode0, 0, 1); + } + goto com; + } + /* bad, 13 is address of external that becomes constant */ + if(n->addable >= INDEXED && n->addable != 13) { + gopcode(o, n->type, n, nodconst(0)); + goto com; + } + regalloc(&nod, n, nn); + cgen(n, &nod); + gopcode(o, n->type, &nod, nodconst(0)); + regfree(&nod); + goto com; + + case OCONST: + o = vconst(n); + if(!true) + o = !o; + gbranch(OGOTO); + if(o) { + p1 = p; + gbranch(OGOTO); + patch(p1, pc); + } + goto com; + + case OCOMMA: + cgen(l, Z); + boolgen(r, true, nn); + break; + + case ONOT: + boolgen(l, !true, nn); + break; + + case OCOND: + bcgen(l, 1); + p1 = p; + bcgen(r->left, true); + p2 = p; + gbranch(OGOTO); + patch(p1, pc); + p1 = p; + bcgen(r->right, !true); + patch(p2, pc); + p2 = p; + gbranch(OGOTO); + patch(p1, pc); + patch(p2, pc); + goto com; + + case OANDAND: + if(!true) + goto caseor; + + caseand: + bcgen(l, true); + p1 = p; + bcgen(r, !true); + p2 = p; + patch(p1, pc); + gbranch(OGOTO); + patch(p2, pc); + goto com; + + case OOROR: + if(!true) + goto caseand; + + caseor: + bcgen(l, !true); + p1 = p; + bcgen(r, !true); + p2 = p; + gbranch(OGOTO); + patch(p1, pc); + patch(p2, pc); + goto com; + + case OEQ: + case ONE: + case OLE: + case OLT: + case OGE: + case OGT: + case OHI: + case OHS: + case OLO: + case OLS: + o = n->op; + if(typev[l->type->etype]) { + if(!true) + n->op = comrel[relindex(o)]; + cgen64(n, Z); + goto com; + } + if(true) + o = comrel[relindex(o)]; + if(l->complex >= FNX && r->complex >= FNX) { + regret(&nod, r); + cgen(r, &nod); + regsalloc(&nod1, r); + gmove(&nod, &nod1); + regfree(&nod); + nod = *n; + nod.right = &nod1; + boolgen(&nod, true, nn); + break; + } + if(typefd[l->type->etype]) { + if(l->complex >= r->complex) { + cgen(l, &fregnode0); + if(r->addable < INDEXED) { + cgen(r, &fregnode0); + o = invrel[relindex(o)]; + fgopcode(o, &fregnode0, &fregnode1, 1, 1); + } else + fgopcode(o, r, &fregnode0, 0, 1); + } else { + o = invrel[relindex(o)]; + cgen(r, &fregnode0); + if(l->addable < INDEXED) { + cgen(l, &fregnode0); + o = invrel[relindex(o)]; + fgopcode(o, &fregnode0, &fregnode1, 1, 1); + } else + fgopcode(o, l, &fregnode0, 0, 1); + } + goto com; + } + if(l->op == OCONST) { + o = invrel[relindex(o)]; + /* bad, 13 is address of external that becomes constant */ + if(r->addable < INDEXED || r->addable == 13) { + regalloc(&nod, r, nn); + cgen(r, &nod); + gopcode(o, l->type, &nod, l); + regfree(&nod); + } else + gopcode(o, l->type, r, l); + goto com; + } + if(l->complex >= r->complex) { + regalloc(&nod, l, nn); + cgen(l, &nod); + if(r->addable < INDEXED) { + regalloc(&nod1, r, Z); + cgen(r, &nod1); + gopcode(o, l->type, &nod, &nod1); + regfree(&nod1); + } else + gopcode(o, l->type, &nod, r); + regfree(&nod); + goto com; + } + regalloc(&nod, r, nn); + cgen(r, &nod); + if(l->addable < INDEXED || l->addable == 13) { + regalloc(&nod1, l, Z); + cgen(l, &nod1); + if(typechlp[l->type->etype]) + gopcode(o, types[TINT], &nod1, &nod); + else + gopcode(o, l->type, &nod1, &nod); + regfree(&nod1); + } else + gopcode(o, l->type, l, &nod); + regfree(&nod); + + com: + if(nn != Z) { + p1 = p; + gmove(nodconst(1L), nn); + gbranch(OGOTO); + p2 = p; + patch(p1, pc); + gmove(nodconst(0L), nn); + patch(p2, pc); + } + break; + } + cursafe = curs; +} + +void +sugen(Node *n, Node *nn, int32 w) +{ + Prog *p1; + Node nod0, nod1, nod2, nod3, nod4, *h, *l, *r; + Type *t; + int c, v, x; + + if(n == Z || n->type == T) + return; + if(debug['g']) { + prtree(nn, "sugen lhs"); + prtree(n, "sugen"); + } + if(nn == nodrat) + if(w > nrathole) + nrathole = w; + switch(n->op) { + case OIND: + if(nn == Z) { + nullwarn(n->left, Z); + break; + } + + default: + goto copy; + + case OCONST: + if(n->type && typev[n->type->etype]) { + if(nn == Z) { + nullwarn(n->left, Z); + break; + } + + if(nn->op == OREGPAIR) { + loadpair(n, nn); + break; + } + else if(!vaddr(nn, 0)) { + t = nn->type; + nn->type = types[TLONG]; + reglcgen(&nod1, nn, Z); + nn->type = t; + + gmove(lo64(n), &nod1); + nod1.xoffset += SZ_LONG; + gmove(hi64(n), &nod1); + regfree(&nod1); + } + else { + gins(AMOVL, lo64(n), nn); + nn->xoffset += SZ_LONG; + gins(AMOVL, hi64(n), nn); + nn->xoffset -= SZ_LONG; + break; + } + break; + } + goto copy; + + case ODOT: + l = n->left; + sugen(l, nodrat, l->type->width); + if(nn == Z) + break; + warn(n, "non-interruptable temporary"); + nod1 = *nodrat; + r = n->right; + if(!r || r->op != OCONST) { + diag(n, "DOT and no offset"); + break; + } + nod1.xoffset += (int32)r->vconst; + nod1.type = n->type; + sugen(&nod1, nn, w); + break; + + case OSTRUCT: + /* + * rewrite so lhs has no fn call + */ + if(nn != Z && side(nn)) { + nod1 = *n; + nod1.type = typ(TIND, n->type); + regret(&nod2, &nod1); + lcgen(nn, &nod2); + regsalloc(&nod0, &nod1); + cgen(&nod2, &nod0); + regfree(&nod2); + + nod1 = *n; + nod1.op = OIND; + nod1.left = &nod0; + nod1.right = Z; + nod1.complex = 1; + + sugen(n, &nod1, w); + return; + } + + r = n->left; + for(t = n->type->link; t != T; t = t->down) { + l = r; + if(r->op == OLIST) { + l = r->left; + r = r->right; + } + if(nn == Z) { + cgen(l, nn); + continue; + } + /* + * hand craft *(&nn + o) = l + */ + nod0 = znode; + nod0.op = OAS; + nod0.type = t; + nod0.left = &nod1; + nod0.right = nil; + + nod1 = znode; + nod1.op = OIND; + nod1.type = t; + nod1.left = &nod2; + + nod2 = znode; + nod2.op = OADD; + nod2.type = typ(TIND, t); + nod2.left = &nod3; + nod2.right = &nod4; + + nod3 = znode; + nod3.op = OADDR; + nod3.type = nod2.type; + nod3.left = nn; + + nod4 = znode; + nod4.op = OCONST; + nod4.type = nod2.type; + nod4.vconst = t->offset; + + ccom(&nod0); + acom(&nod0); + xcom(&nod0); + nod0.addable = 0; + nod0.right = l; + + // prtree(&nod0, "hand craft"); + cgen(&nod0, Z); + } + break; + + case OAS: + if(nn == Z) { + if(n->addable < INDEXED) + sugen(n->right, n->left, w); + break; + } + + sugen(n->right, nodrat, w); + warn(n, "non-interruptable temporary"); + sugen(nodrat, n->left, w); + sugen(nodrat, nn, w); + break; + + case OFUNC: + if(nn == Z) { + sugen(n, nodrat, w); + break; + } + h = nn; + if(nn->op == OREGPAIR) { + regsalloc(&nod1, nn); + nn = &nod1; + } + if(nn->op != OIND) { + nn = new1(OADDR, nn, Z); + nn->type = types[TIND]; + nn->addable = 0; + } else + nn = nn->left; + n = new(OFUNC, n->left, new(OLIST, nn, n->right)); + n->type = types[TVOID]; + n->left->type = types[TVOID]; + cgen(n, Z); + if(h->op == OREGPAIR) + loadpair(nn->left, h); + break; + + case OCOND: + bcgen(n->left, 1); + p1 = p; + sugen(n->right->left, nn, w); + gbranch(OGOTO); + patch(p1, pc); + p1 = p; + sugen(n->right->right, nn, w); + patch(p1, pc); + break; + + case OCOMMA: + cgen(n->left, Z); + sugen(n->right, nn, w); + break; + } + return; + +copy: + if(nn == Z) { + switch(n->op) { + case OASADD: + case OASSUB: + case OASAND: + case OASOR: + case OASXOR: + + case OASMUL: + case OASLMUL: + + + case OASASHL: + case OASASHR: + case OASLSHR: + break; + + case OPOSTINC: + case OPOSTDEC: + case OPREINC: + case OPREDEC: + break; + + default: + return; + } + } + + if(n->complex >= FNX && nn != nil && nn->complex >= FNX) { + t = nn->type; + nn->type = types[TLONG]; + regialloc(&nod1, nn, Z); + lcgen(nn, &nod1); + regsalloc(&nod2, nn); + nn->type = t; + + gins(AMOVL, &nod1, &nod2); + regfree(&nod1); + + nod2.type = typ(TIND, t); + + nod1 = nod2; + nod1.op = OIND; + nod1.left = &nod2; + nod1.right = Z; + nod1.complex = 1; + nod1.type = t; + + sugen(n, &nod1, w); + return; + } + + x = 0; + v = w == 8; + if(v) { + c = cursafe; + if(n->left != Z && n->left->complex >= FNX + && n->right != Z && n->right->complex >= FNX) { +// warn(n, "toughie"); + regsalloc(&nod1, n->right); + cgen(n->right, &nod1); + nod2 = *n; + nod2.right = &nod1; + cgen(&nod2, nn); + cursafe = c; + return; + } + if(cgen64(n, nn)) { + cursafe = c; + return; + } + if(n->op == OCOM) { + n = n->left; + x = 1; + } + } + + /* botch, need to save in .safe */ + c = 0; + if(n->complex > nn->complex) { + t = n->type; + n->type = types[TLONG]; + if(v) { + regalloc(&nod0, n, Z); + if(!vaddr(n, 0)) { + reglcgen(&nod1, n, Z); + n->type = t; + n = &nod1; + } + else + n->type = t; + } + else { + nodreg(&nod1, n, D_SI); + if(reg[D_SI]) { + gins(APUSHL, &nod1, Z); + c |= 1; + reg[D_SI]++; + } + lcgen(n, &nod1); + n->type = t; + } + + t = nn->type; + nn->type = types[TLONG]; + if(v) { + if(!vaddr(nn, 0)) { + reglcgen(&nod2, nn, Z); + nn->type = t; + nn = &nod2; + } + else + nn->type = t; + } + else { + nodreg(&nod2, nn, D_DI); + if(reg[D_DI]) { + gins(APUSHL, &nod2, Z); + c |= 2; + reg[D_DI]++; + } + lcgen(nn, &nod2); + nn->type = t; + } + } else { + t = nn->type; + nn->type = types[TLONG]; + if(v) { + regalloc(&nod0, nn, Z); + if(!vaddr(nn, 0)) { + reglcgen(&nod2, nn, Z); + nn->type = t; + nn = &nod2; + } + else + nn->type = t; + } + else { + nodreg(&nod2, nn, D_DI); + if(reg[D_DI]) { + gins(APUSHL, &nod2, Z); + c |= 2; + reg[D_DI]++; + } + lcgen(nn, &nod2); + nn->type = t; + } + + t = n->type; + n->type = types[TLONG]; + if(v) { + if(!vaddr(n, 0)) { + reglcgen(&nod1, n, Z); + n->type = t; + n = &nod1; + } + else + n->type = t; + } + else { + nodreg(&nod1, n, D_SI); + if(reg[D_SI]) { + gins(APUSHL, &nod1, Z); + c |= 1; + reg[D_SI]++; + } + lcgen(n, &nod1); + n->type = t; + } + } + if(v) { + gins(AMOVL, n, &nod0); + if(x) + gins(ANOTL, Z, &nod0); + gins(AMOVL, &nod0, nn); + n->xoffset += SZ_LONG; + nn->xoffset += SZ_LONG; + gins(AMOVL, n, &nod0); + if(x) + gins(ANOTL, Z, &nod0); + gins(AMOVL, &nod0, nn); + n->xoffset -= SZ_LONG; + nn->xoffset -= SZ_LONG; + if(nn == &nod2) + regfree(&nod2); + if(n == &nod1) + regfree(&nod1); + regfree(&nod0); + return; + } + nodreg(&nod3, n, D_CX); + if(reg[D_CX]) { + gins(APUSHL, &nod3, Z); + c |= 4; + reg[D_CX]++; + } + gins(AMOVL, nodconst(w/SZ_LONG), &nod3); + gins(ACLD, Z, Z); + gins(AREP, Z, Z); + gins(AMOVSL, Z, Z); + if(c & 4) { + gins(APOPL, Z, &nod3); + reg[D_CX]--; + } + if(c & 2) { + gins(APOPL, Z, &nod2); + reg[nod2.reg]--; + } + if(c & 1) { + gins(APOPL, Z, &nod1); + reg[nod1.reg]--; + } +} diff --git a/src/cmd/8c/cgen64.c b/src/cmd/8c/cgen64.c new file mode 100644 index 000000000..3424f762c --- /dev/null +++ b/src/cmd/8c/cgen64.c @@ -0,0 +1,2657 @@ +// Inferno utils/8c/cgen64.c +// http://code.google.com/p/inferno-os/source/browse/utils/8c/cgen64.c +// +// Copyright © 1994-1999 Lucent Technologies Inc. All rights reserved. +// Portions Copyright © 1995-1997 C H Forsyth (forsyth@terzarima.net) +// Portions Copyright © 1997-1999 Vita Nuova Limited +// Portions Copyright © 2000-2007 Vita Nuova Holdings Limited (www.vitanuova.com) +// Portions Copyright © 2004,2006 Bruce Ellis +// Portions Copyright © 2005-2007 C H Forsyth (forsyth@terzarima.net) +// Revisions Copyright © 2000-2007 Lucent Technologies Inc. and others +// Portions Copyright © 2009 The Go Authors. All rights reserved. +// +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to deal +// in the Software without restriction, including without limitation the rights +// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +// copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +// THE SOFTWARE. + +#include "gc.h" + +void +zeroregm(Node *n) +{ + gins(AMOVL, nodconst(0), n); +} + +/* do we need to load the address of a vlong? */ +int +vaddr(Node *n, int a) +{ + switch(n->op) { + case ONAME: + if(a) + return 1; + return !(n->class == CEXTERN || n->class == CGLOBL || n->class == CSTATIC); + + case OCONST: + case OREGISTER: + case OINDREG: + return 1; + } + return 0; +} + +int32 +hi64v(Node *n) +{ + if(align(0, types[TCHAR], Aarg1, nil)) /* isbigendian */ + return (int32)(n->vconst) & ~0L; + else + return (int32)((uvlong)n->vconst>>32) & ~0L; +} + +int32 +lo64v(Node *n) +{ + if(align(0, types[TCHAR], Aarg1, nil)) /* isbigendian */ + return (int32)((uvlong)n->vconst>>32) & ~0L; + else + return (int32)(n->vconst) & ~0L; +} + +Node * +hi64(Node *n) +{ + return nodconst(hi64v(n)); +} + +Node * +lo64(Node *n) +{ + return nodconst(lo64v(n)); +} + +static Node * +anonreg(void) +{ + Node *n; + + n = new(OREGISTER, Z, Z); + n->reg = D_NONE; + n->type = types[TLONG]; + return n; +} + +static Node * +regpair(Node *n, Node *t) +{ + Node *r; + + if(n != Z && n->op == OREGPAIR) + return n; + r = new(OREGPAIR, anonreg(), anonreg()); + if(n != Z) + r->type = n->type; + else + r->type = t->type; + return r; +} + +static void +evacaxdx(Node *r) +{ + Node nod1, nod2; + + if(r->reg == D_AX || r->reg == D_DX) { + reg[D_AX]++; + reg[D_DX]++; + /* + * this is just an optim that should + * check for spill + */ + r->type = types[TULONG]; + regalloc(&nod1, r, Z); + nodreg(&nod2, Z, r->reg); + gins(AMOVL, &nod2, &nod1); + regfree(r); + r->reg = nod1.reg; + reg[D_AX]--; + reg[D_DX]--; + } +} + +/* lazy instantiation of register pair */ +static int +instpair(Node *n, Node *l) +{ + int r; + + r = 0; + if(n->left->reg == D_NONE) { + if(l != Z) { + n->left->reg = l->reg; + r = 1; + } + else + regalloc(n->left, n->left, Z); + } + if(n->right->reg == D_NONE) + regalloc(n->right, n->right, Z); + return r; +} + +static void +zapreg(Node *n) +{ + if(n->reg != D_NONE) { + regfree(n); + n->reg = D_NONE; + } +} + +static void +freepair(Node *n) +{ + regfree(n->left); + regfree(n->right); +} + +/* n is not OREGPAIR, nn is */ +void +loadpair(Node *n, Node *nn) +{ + Node nod; + + instpair(nn, Z); + if(n->op == OCONST) { + gins(AMOVL, lo64(n), nn->left); + n->xoffset += SZ_LONG; + gins(AMOVL, hi64(n), nn->right); + n->xoffset -= SZ_LONG; + return; + } + if(!vaddr(n, 0)) { + /* steal the right register for the laddr */ + nod = regnode; + nod.reg = nn->right->reg; + lcgen(n, &nod); + n = &nod; + regind(n, n); + n->xoffset = 0; + } + gins(AMOVL, n, nn->left); + n->xoffset += SZ_LONG; + gins(AMOVL, n, nn->right); + n->xoffset -= SZ_LONG; +} + +/* n is OREGPAIR, nn is not */ +static void +storepair(Node *n, Node *nn, int f) +{ + Node nod; + + if(!vaddr(nn, 0)) { + reglcgen(&nod, nn, Z); + nn = &nod; + } + gins(AMOVL, n->left, nn); + nn->xoffset += SZ_LONG; + gins(AMOVL, n->right, nn); + nn->xoffset -= SZ_LONG; + if(nn == &nod) + regfree(&nod); + if(f) + freepair(n); +} + +enum +{ +/* 4 only, see WW */ + WNONE = 0, + WCONST, + WADDR, + WHARD, +}; + +static int +whatof(Node *n, int a) +{ + if(n->op == OCONST) + return WCONST; + return !vaddr(n, a) ? WHARD : WADDR; +} + +/* can upgrade an extern to addr for AND */ +static int +reduxv(Node *n) +{ + return lo64v(n) == 0 || hi64v(n) == 0; +} + +int +cond(int op) +{ + switch(op) { + case OANDAND: + case OOROR: + case ONOT: + return 1; + + case OEQ: + case ONE: + case OLE: + case OLT: + case OGE: + case OGT: + case OHI: + case OHS: + case OLO: + case OLS: + return 1; + } + return 0; +} + +/* + * for a func operand call it and then return + * the safe node + */ +static Node * +vfunc(Node *n, Node *nn) +{ + Node *t; + + if(n->op != OFUNC) + return n; + t = new(0, Z, Z); + if(nn == Z || nn == nodret) + nn = n; + regsalloc(t, nn); + sugen(n, t, 8); + return t; +} + +/* try to steal a reg */ +static int +getreg(Node **np, Node *t, int r) +{ + Node *n, *p; + + n = *np; + if(n->reg == r) { + p = new(0, Z, Z); + regalloc(p, n, Z); + gins(AMOVL, n, p); + *t = *n; + *np = p; + return 1; + } + return 0; +} + +static Node * +snarfreg(Node *n, Node *t, int r, Node *d, Node *c) +{ + if(n == Z || n->op != OREGPAIR || (!getreg(&n->left, t, r) && !getreg(&n->right, t, r))) { + if(nodreg(t, Z, r)) { + regalloc(c, d, Z); + gins(AMOVL, t, c); + reg[r]++; + return c; + } + reg[r]++; + } + return Z; +} + +enum +{ + Vstart = OEND, + + Vgo, + Vamv, + Vmv, + Vzero, + Vop, + Vopx, + Vins, + Vins0, + Vinsl, + Vinsr, + Vinsla, + Vinsra, + Vinsx, + Vmul, + Vshll, + VT, + VF, + V_l_lo_f, + V_l_hi_f, + V_l_lo_t, + V_l_hi_t, + V_l_lo_u, + V_l_hi_u, + V_r_lo_f, + V_r_hi_f, + V_r_lo_t, + V_r_hi_t, + V_r_lo_u, + V_r_hi_u, + Vspazz, + Vend, + + V_T0, + V_T1, + V_F0, + V_F1, + + V_a0, + V_a1, + V_f0, + V_f1, + + V_p0, + V_p1, + V_p2, + V_p3, + V_p4, + + V_s0, + V_s1, + V_s2, + V_s3, + V_s4, + + C00, + C01, + C31, + C32, + + O_l_lo, + O_l_hi, + O_r_lo, + O_r_hi, + O_t_lo, + O_t_hi, + O_l, + O_r, + O_l_rp, + O_r_rp, + O_t_rp, + O_r0, + O_r1, + O_Zop, + + O_a0, + O_a1, + + V_C0, + V_C1, + + V_S0, + V_S1, + + VOPS = 5, + VLEN = 5, + VARGS = 2, + + S00 = 0, + Sc0, + Sc1, + Sc2, + Sac3, + Sac4, + S10, + + SAgen = 0, + SAclo, + SAc32, + SAchi, + SAdgen, + SAdclo, + SAdc32, + SAdchi, + + B0c = 0, + Bca, + Bac, + + T0i = 0, + Tii, + + Bop0 = 0, + Bop1, +}; + +/* + * _testv: + * CMPL lo,$0 + * JNE true + * CMPL hi,$0 + * JNE true + * GOTO false + * false: + * GOTO code + * true: + * GOTO patchme + * code: + */ + +static uchar testi[][VLEN] = +{ + {Vop, ONE, O_l_lo, C00}, + {V_s0, Vop, ONE, O_l_hi, C00}, + {V_s1, Vgo, V_s2, Vgo, V_s3}, + {VF, V_p0, V_p1, VT, V_p2}, + {Vgo, V_p3}, + {VT, V_p0, V_p1, VF, V_p2}, + {Vend}, +}; + +/* shift left general case */ +static uchar shll00[][VLEN] = +{ + {Vop, OGE, O_r, C32}, + {V_s0, Vinsl, ASHLL, O_r, O_l_rp}, + {Vins, ASHLL, O_r, O_l_lo, Vgo}, + {V_p0, V_s0}, + {Vins, ASHLL, O_r, O_l_lo}, + {Vins, AMOVL, O_l_lo, O_l_hi}, + {Vzero, O_l_lo, V_p0, Vend}, +}; + +/* shift left rp, const < 32 */ +static uchar shllc0[][VLEN] = +{ + {Vinsl, ASHLL, O_r, O_l_rp}, + {Vshll, O_r, O_l_lo, Vend}, +}; + +/* shift left rp, const == 32 */ +static uchar shllc1[][VLEN] = +{ + {Vins, AMOVL, O_l_lo, O_l_hi}, + {Vzero, O_l_lo, Vend}, +}; + +/* shift left rp, const > 32 */ +static uchar shllc2[][VLEN] = +{ + {Vshll, O_r, O_l_lo}, + {Vins, AMOVL, O_l_lo, O_l_hi}, + {Vzero, O_l_lo, Vend}, +}; + +/* shift left addr, const == 32 */ +static uchar shllac3[][VLEN] = +{ + {Vins, AMOVL, O_l_lo, O_t_hi}, + {Vzero, O_t_lo, Vend}, +}; + +/* shift left addr, const > 32 */ +static uchar shllac4[][VLEN] = +{ + {Vins, AMOVL, O_l_lo, O_t_hi}, + {Vshll, O_r, O_t_hi}, + {Vzero, O_t_lo, Vend}, +}; + +/* shift left of constant */ +static uchar shll10[][VLEN] = +{ + {Vop, OGE, O_r, C32}, + {V_s0, Vins, AMOVL, O_l_lo, O_t_lo}, + {Vins, AMOVL, O_l_hi, O_t_hi}, + {Vinsl, ASHLL, O_r, O_t_rp}, + {Vins, ASHLL, O_r, O_t_lo, Vgo}, + {V_p0, V_s0}, + {Vins, AMOVL, O_l_lo, O_t_hi}, + {V_l_lo_t, Vins, ASHLL, O_r, O_t_hi}, + {Vzero, O_t_lo, V_p0, Vend}, +}; + +static uchar (*shlltab[])[VLEN] = +{ + shll00, + shllc0, + shllc1, + shllc2, + shllac3, + shllac4, + shll10, +}; + +/* shift right general case */ +static uchar shrl00[][VLEN] = +{ + {Vop, OGE, O_r, C32}, + {V_s0, Vinsr, ASHRL, O_r, O_l_rp}, + {Vins, O_a0, O_r, O_l_hi, Vgo}, + {V_p0, V_s0}, + {Vins, O_a0, O_r, O_l_hi}, + {Vins, AMOVL, O_l_hi, O_l_lo}, + {V_T1, Vzero, O_l_hi}, + {V_F1, Vins, ASARL, C31, O_l_hi}, + {V_p0, Vend}, +}; + +/* shift right rp, const < 32 */ +static uchar shrlc0[][VLEN] = +{ + {Vinsr, ASHRL, O_r, O_l_rp}, + {Vins, O_a0, O_r, O_l_hi, Vend}, +}; + +/* shift right rp, const == 32 */ +static uchar shrlc1[][VLEN] = +{ + {Vins, AMOVL, O_l_hi, O_l_lo}, + {V_T1, Vzero, O_l_hi}, + {V_F1, Vins, ASARL, C31, O_l_hi}, + {Vend}, +}; + +/* shift right rp, const > 32 */ +static uchar shrlc2[][VLEN] = +{ + {Vins, O_a0, O_r, O_l_hi}, + {Vins, AMOVL, O_l_hi, O_l_lo}, + {V_T1, Vzero, O_l_hi}, + {V_F1, Vins, ASARL, C31, O_l_hi}, + {Vend}, +}; + +/* shift right addr, const == 32 */ +static uchar shrlac3[][VLEN] = +{ + {Vins, AMOVL, O_l_hi, O_t_lo}, + {V_T1, Vzero, O_t_hi}, + {V_F1, Vins, AMOVL, O_t_lo, O_t_hi}, + {V_F1, Vins, ASARL, C31, O_t_hi}, + {Vend}, +}; + +/* shift right addr, const > 32 */ +static uchar shrlac4[][VLEN] = +{ + {Vins, AMOVL, O_l_hi, O_t_lo}, + {Vins, O_a0, O_r, O_t_lo}, + {V_T1, Vzero, O_t_hi}, + {V_F1, Vins, AMOVL, O_t_lo, O_t_hi}, + {V_F1, Vins, ASARL, C31, O_t_hi}, + {Vend}, +}; + +/* shift right of constant */ +static uchar shrl10[][VLEN] = +{ + {Vop, OGE, O_r, C32}, + {V_s0, Vins, AMOVL, O_l_lo, O_t_lo}, + {Vins, AMOVL, O_l_hi, O_t_hi}, + {Vinsr, ASHRL, O_r, O_t_rp}, + {Vins, O_a0, O_r, O_t_hi, Vgo}, + {V_p0, V_s0}, + {Vins, AMOVL, O_l_hi, O_t_lo}, + {V_l_hi_t, Vins, O_a0, O_r, O_t_lo}, + {V_l_hi_u, V_S1}, + {V_T1, Vzero, O_t_hi, V_p0}, + {V_F1, Vins, AMOVL, O_t_lo, O_t_hi}, + {V_F1, Vins, ASARL, C31, O_t_hi}, + {Vend}, +}; + +static uchar (*shrltab[])[VLEN] = +{ + shrl00, + shrlc0, + shrlc1, + shrlc2, + shrlac3, + shrlac4, + shrl10, +}; + +/* shift asop left general case */ +static uchar asshllgen[][VLEN] = +{ + {V_a0, V_a1}, + {Vop, OGE, O_r, C32}, + {V_s0, Vins, AMOVL, O_l_lo, O_r0}, + {Vins, AMOVL, O_l_hi, O_r1}, + {Vinsla, ASHLL, O_r, O_r0}, + {Vins, ASHLL, O_r, O_r0}, + {Vins, AMOVL, O_r1, O_l_hi}, + {Vins, AMOVL, O_r0, O_l_lo, Vgo}, + {V_p0, V_s0}, + {Vins, AMOVL, O_l_lo, O_r0}, + {Vzero, O_l_lo}, + {Vins, ASHLL, O_r, O_r0}, + {Vins, AMOVL, O_r0, O_l_hi, V_p0}, + {V_f0, V_f1, Vend}, +}; + +/* shift asop left, const < 32 */ +static uchar asshllclo[][VLEN] = +{ + {V_a0, V_a1}, + {Vins, AMOVL, O_l_lo, O_r0}, + {Vins, AMOVL, O_l_hi, O_r1}, + {Vinsla, ASHLL, O_r, O_r0}, + {Vshll, O_r, O_r0}, + {Vins, AMOVL, O_r1, O_l_hi}, + {Vins, AMOVL, O_r0, O_l_lo}, + {V_f0, V_f1, Vend}, +}; + +/* shift asop left, const == 32 */ +static uchar asshllc32[][VLEN] = +{ + {V_a0}, + {Vins, AMOVL, O_l_lo, O_r0}, + {Vzero, O_l_lo}, + {Vins, AMOVL, O_r0, O_l_hi}, + {V_f0, Vend}, +}; + +/* shift asop left, const > 32 */ +static uchar asshllchi[][VLEN] = +{ + {V_a0}, + {Vins, AMOVL, O_l_lo, O_r0}, + {Vzero, O_l_lo}, + {Vshll, O_r, O_r0}, + {Vins, AMOVL, O_r0, O_l_hi}, + {V_f0, Vend}, +}; + +/* shift asop dest left general case */ +static uchar asdshllgen[][VLEN] = +{ + {Vop, OGE, O_r, C32}, + {V_s0, Vins, AMOVL, O_l_lo, O_t_lo}, + {Vins, AMOVL, O_l_hi, O_t_hi}, + {Vinsl, ASHLL, O_r, O_t_rp}, + {Vins, ASHLL, O_r, O_t_lo}, + {Vins, AMOVL, O_t_hi, O_l_hi}, + {Vins, AMOVL, O_t_lo, O_l_lo, Vgo}, + {V_p0, V_s0}, + {Vins, AMOVL, O_l_lo, O_t_hi}, + {Vzero, O_l_lo}, + {Vins, ASHLL, O_r, O_t_hi}, + {Vzero, O_t_lo}, + {Vins, AMOVL, O_t_hi, O_l_hi, V_p0}, + {Vend}, +}; + +/* shift asop dest left, const < 32 */ +static uchar asdshllclo[][VLEN] = +{ + {Vins, AMOVL, O_l_lo, O_t_lo}, + {Vins, AMOVL, O_l_hi, O_t_hi}, + {Vinsl, ASHLL, O_r, O_t_rp}, + {Vshll, O_r, O_t_lo}, + {Vins, AMOVL, O_t_hi, O_l_hi}, + {Vins, AMOVL, O_t_lo, O_l_lo}, + {Vend}, +}; + +/* shift asop dest left, const == 32 */ +static uchar asdshllc32[][VLEN] = +{ + {Vins, AMOVL, O_l_lo, O_t_hi}, + {Vzero, O_t_lo}, + {Vins, AMOVL, O_t_hi, O_l_hi}, + {Vins, AMOVL, O_t_lo, O_l_lo}, + {Vend}, +}; + +/* shift asop dest, const > 32 */ +static uchar asdshllchi[][VLEN] = +{ + {Vins, AMOVL, O_l_lo, O_t_hi}, + {Vzero, O_t_lo}, + {Vshll, O_r, O_t_hi}, + {Vins, AMOVL, O_t_lo, O_l_lo}, + {Vins, AMOVL, O_t_hi, O_l_hi}, + {Vend}, +}; + +static uchar (*asshlltab[])[VLEN] = +{ + asshllgen, + asshllclo, + asshllc32, + asshllchi, + asdshllgen, + asdshllclo, + asdshllc32, + asdshllchi, +}; + +/* shift asop right general case */ +static uchar asshrlgen[][VLEN] = +{ + {V_a0, V_a1}, + {Vop, OGE, O_r, C32}, + {V_s0, Vins, AMOVL, O_l_lo, O_r0}, + {Vins, AMOVL, O_l_hi, O_r1}, + {Vinsra, ASHRL, O_r, O_r0}, + {Vinsx, Bop0, O_r, O_r1}, + {Vins, AMOVL, O_r0, O_l_lo}, + {Vins, AMOVL, O_r1, O_l_hi, Vgo}, + {V_p0, V_s0}, + {Vins, AMOVL, O_l_hi, O_r0}, + {Vinsx, Bop0, O_r, O_r0}, + {V_T1, Vzero, O_l_hi}, + {Vins, AMOVL, O_r0, O_l_lo}, + {V_F1, Vins, ASARL, C31, O_r0}, + {V_F1, Vins, AMOVL, O_r0, O_l_hi}, + {V_p0, V_f0, V_f1, Vend}, +}; + +/* shift asop right, const < 32 */ +static uchar asshrlclo[][VLEN] = +{ + {V_a0, V_a1}, + {Vins, AMOVL, O_l_lo, O_r0}, + {Vins, AMOVL, O_l_hi, O_r1}, + {Vinsra, ASHRL, O_r, O_r0}, + {Vinsx, Bop0, O_r, O_r1}, + {Vins, AMOVL, O_r0, O_l_lo}, + {Vins, AMOVL, O_r1, O_l_hi}, + {V_f0, V_f1, Vend}, +}; + +/* shift asop right, const == 32 */ +static uchar asshrlc32[][VLEN] = +{ + {V_a0}, + {Vins, AMOVL, O_l_hi, O_r0}, + {V_T1, Vzero, O_l_hi}, + {Vins, AMOVL, O_r0, O_l_lo}, + {V_F1, Vins, ASARL, C31, O_r0}, + {V_F1, Vins, AMOVL, O_r0, O_l_hi}, + {V_f0, Vend}, +}; + +/* shift asop right, const > 32 */ +static uchar asshrlchi[][VLEN] = +{ + {V_a0}, + {Vins, AMOVL, O_l_hi, O_r0}, + {V_T1, Vzero, O_l_hi}, + {Vinsx, Bop0, O_r, O_r0}, + {Vins, AMOVL, O_r0, O_l_lo}, + {V_F1, Vins, ASARL, C31, O_r0}, + {V_F1, Vins, AMOVL, O_r0, O_l_hi}, + {V_f0, Vend}, +}; + +/* shift asop dest right general case */ +static uchar asdshrlgen[][VLEN] = +{ + {Vop, OGE, O_r, C32}, + {V_s0, Vins, AMOVL, O_l_lo, O_t_lo}, + {Vins, AMOVL, O_l_hi, O_t_hi}, + {Vinsr, ASHRL, O_r, O_t_rp}, + {Vinsx, Bop0, O_r, O_t_hi}, + {Vins, AMOVL, O_t_lo, O_l_lo}, + {Vins, AMOVL, O_t_hi, O_l_hi, Vgo}, + {V_p0, V_s0}, + {Vins, AMOVL, O_l_hi, O_t_lo}, + {V_T1, Vzero, O_t_hi}, + {Vinsx, Bop0, O_r, O_t_lo}, + {V_F1, Vins, AMOVL, O_t_lo, O_t_hi}, + {V_F1, Vins, ASARL, C31, O_t_hi}, + {Vins, AMOVL, O_t_hi, O_l_hi, V_p0}, + {Vend}, +}; + +/* shift asop dest right, const < 32 */ +static uchar asdshrlclo[][VLEN] = +{ + {Vins, AMOVL, O_l_lo, O_t_lo}, + {Vins, AMOVL, O_l_hi, O_t_hi}, + {Vinsr, ASHRL, O_r, O_t_rp}, + {Vinsx, Bop0, O_r, O_t_hi}, + {Vins, AMOVL, O_t_lo, O_l_lo}, + {Vins, AMOVL, O_t_hi, O_l_hi}, + {Vend}, +}; + +/* shift asop dest right, const == 32 */ +static uchar asdshrlc32[][VLEN] = +{ + {Vins, AMOVL, O_l_hi, O_t_lo}, + {V_T1, Vzero, O_t_hi}, + {V_F1, Vins, AMOVL, O_t_lo, O_t_hi}, + {V_F1, Vins, ASARL, C31, O_t_hi}, + {Vins, AMOVL, O_t_lo, O_l_lo}, + {Vins, AMOVL, O_t_hi, O_l_hi}, + {Vend}, +}; + +/* shift asop dest, const > 32 */ +static uchar asdshrlchi[][VLEN] = +{ + {Vins, AMOVL, O_l_hi, O_t_lo}, + {V_T1, Vzero, O_t_hi}, + {Vinsx, Bop0, O_r, O_t_lo}, + {V_T1, Vins, AMOVL, O_t_hi, O_l_hi}, + {V_T1, Vins, AMOVL, O_t_lo, O_l_lo}, + {V_F1, Vins, AMOVL, O_t_lo, O_t_hi}, + {V_F1, Vins, ASARL, C31, O_t_hi}, + {V_F1, Vins, AMOVL, O_t_lo, O_l_lo}, + {V_F1, Vins, AMOVL, O_t_hi, O_l_hi}, + {Vend}, +}; + +static uchar (*asshrltab[])[VLEN] = +{ + asshrlgen, + asshrlclo, + asshrlc32, + asshrlchi, + asdshrlgen, + asdshrlclo, + asdshrlc32, + asdshrlchi, +}; + +static uchar shrlargs[] = { ASHRL, 1 }; +static uchar sarlargs[] = { ASARL, 0 }; + +/* ++ -- */ +static uchar incdec[][VLEN] = +{ + {Vinsx, Bop0, C01, O_l_lo}, + {Vinsx, Bop1, C00, O_l_hi, Vend}, +}; + +/* ++ -- *p */ +static uchar incdecpre[][VLEN] = +{ + {Vins, AMOVL, O_l_lo, O_t_lo}, + {Vins, AMOVL, O_l_hi, O_t_hi}, + {Vinsx, Bop0, C01, O_t_lo}, + {Vinsx, Bop1, C00, O_t_hi}, + {Vins, AMOVL, O_t_lo, O_l_lo}, + {Vins, AMOVL, O_t_hi, O_l_hi, Vend}, +}; + +/* *p ++ -- */ +static uchar incdecpost[][VLEN] = +{ + {Vins, AMOVL, O_l_lo, O_t_lo}, + {Vins, AMOVL, O_l_hi, O_t_hi}, + {Vinsx, Bop0, C01, O_l_lo}, + {Vinsx, Bop1, C00, O_l_hi, Vend}, +}; + +/* binop rp, rp */ +static uchar binop00[][VLEN] = +{ + {Vinsx, Bop0, O_r_lo, O_l_lo}, + {Vinsx, Bop1, O_r_hi, O_l_hi, Vend}, + {Vend}, +}; + +/* binop rp, addr */ +static uchar binoptmp[][VLEN] = +{ + {V_a0, Vins, AMOVL, O_r_lo, O_r0}, + {Vinsx, Bop0, O_r0, O_l_lo}, + {Vins, AMOVL, O_r_hi, O_r0}, + {Vinsx, Bop1, O_r0, O_l_hi}, + {V_f0, Vend}, +}; + +/* binop t = *a op *b */ +static uchar binop11[][VLEN] = +{ + {Vins, AMOVL, O_l_lo, O_t_lo}, + {Vinsx, Bop0, O_r_lo, O_t_lo}, + {Vins, AMOVL, O_l_hi, O_t_hi}, + {Vinsx, Bop1, O_r_hi, O_t_hi, Vend}, +}; + +/* binop t = rp +- c */ +static uchar add0c[][VLEN] = +{ + {V_r_lo_t, Vinsx, Bop0, O_r_lo, O_l_lo}, + {V_r_lo_f, Vamv, Bop0, Bop1}, + {Vinsx, Bop1, O_r_hi, O_l_hi}, + {Vend}, +}; + +/* binop t = rp & c */ +static uchar and0c[][VLEN] = +{ + {V_r_lo_t, Vinsx, Bop0, O_r_lo, O_l_lo}, + {V_r_lo_f, Vins, AMOVL, C00, O_l_lo}, + {V_r_hi_t, Vinsx, Bop1, O_r_hi, O_l_hi}, + {V_r_hi_f, Vins, AMOVL, C00, O_l_hi}, + {Vend}, +}; + +/* binop t = rp | c */ +static uchar or0c[][VLEN] = +{ + {V_r_lo_t, Vinsx, Bop0, O_r_lo, O_l_lo}, + {V_r_hi_t, Vinsx, Bop1, O_r_hi, O_l_hi}, + {Vend}, +}; + +/* binop t = c - rp */ +static uchar sub10[][VLEN] = +{ + {V_a0, Vins, AMOVL, O_l_lo, O_r0}, + {Vinsx, Bop0, O_r_lo, O_r0}, + {Vins, AMOVL, O_l_hi, O_r_lo}, + {Vinsx, Bop1, O_r_hi, O_r_lo}, + {Vspazz, V_f0, Vend}, +}; + +/* binop t = c + *b */ +static uchar addca[][VLEN] = +{ + {Vins, AMOVL, O_r_lo, O_t_lo}, + {V_l_lo_t, Vinsx, Bop0, O_l_lo, O_t_lo}, + {V_l_lo_f, Vamv, Bop0, Bop1}, + {Vins, AMOVL, O_r_hi, O_t_hi}, + {Vinsx, Bop1, O_l_hi, O_t_hi}, + {Vend}, +}; + +/* binop t = c & *b */ +static uchar andca[][VLEN] = +{ + {V_l_lo_t, Vins, AMOVL, O_r_lo, O_t_lo}, + {V_l_lo_t, Vinsx, Bop0, O_l_lo, O_t_lo}, + {V_l_lo_f, Vzero, O_t_lo}, + {V_l_hi_t, Vins, AMOVL, O_r_hi, O_t_hi}, + {V_l_hi_t, Vinsx, Bop1, O_l_hi, O_t_hi}, + {V_l_hi_f, Vzero, O_t_hi}, + {Vend}, +}; + +/* binop t = c | *b */ +static uchar orca[][VLEN] = +{ + {Vins, AMOVL, O_r_lo, O_t_lo}, + {V_l_lo_t, Vinsx, Bop0, O_l_lo, O_t_lo}, + {Vins, AMOVL, O_r_hi, O_t_hi}, + {V_l_hi_t, Vinsx, Bop1, O_l_hi, O_t_hi}, + {Vend}, +}; + +/* binop t = c - *b */ +static uchar subca[][VLEN] = +{ + {Vins, AMOVL, O_l_lo, O_t_lo}, + {Vins, AMOVL, O_l_hi, O_t_hi}, + {Vinsx, Bop0, O_r_lo, O_t_lo}, + {Vinsx, Bop1, O_r_hi, O_t_hi}, + {Vend}, +}; + +/* binop t = *a +- c */ +static uchar addac[][VLEN] = +{ + {Vins, AMOVL, O_l_lo, O_t_lo}, + {V_r_lo_t, Vinsx, Bop0, O_r_lo, O_t_lo}, + {V_r_lo_f, Vamv, Bop0, Bop1}, + {Vins, AMOVL, O_l_hi, O_t_hi}, + {Vinsx, Bop1, O_r_hi, O_t_hi}, + {Vend}, +}; + +/* binop t = *a | c */ +static uchar orac[][VLEN] = +{ + {Vins, AMOVL, O_l_lo, O_t_lo}, + {V_r_lo_t, Vinsx, Bop0, O_r_lo, O_t_lo}, + {Vins, AMOVL, O_l_hi, O_t_hi}, + {V_r_hi_t, Vinsx, Bop1, O_r_hi, O_t_hi}, + {Vend}, +}; + +/* binop t = *a & c */ +static uchar andac[][VLEN] = +{ + {V_r_lo_t, Vins, AMOVL, O_l_lo, O_t_lo}, + {V_r_lo_t, Vinsx, Bop0, O_r_lo, O_t_lo}, + {V_r_lo_f, Vzero, O_t_lo}, + {V_r_hi_t, Vins, AMOVL, O_l_hi, O_t_hi}, + {V_r_hi_t, Vinsx, Bop0, O_r_hi, O_t_hi}, + {V_r_hi_f, Vzero, O_t_hi}, + {Vend}, +}; + +static uchar ADDargs[] = { AADDL, AADCL }; +static uchar ANDargs[] = { AANDL, AANDL }; +static uchar ORargs[] = { AORL, AORL }; +static uchar SUBargs[] = { ASUBL, ASBBL }; +static uchar XORargs[] = { AXORL, AXORL }; + +static uchar (*ADDtab[])[VLEN] = +{ + add0c, addca, addac, +}; + +static uchar (*ANDtab[])[VLEN] = +{ + and0c, andca, andac, +}; + +static uchar (*ORtab[])[VLEN] = +{ + or0c, orca, orac, +}; + +static uchar (*SUBtab[])[VLEN] = +{ + add0c, subca, addac, +}; + +/* mul of const32 */ +static uchar mulc32[][VLEN] = +{ + {V_a0, Vop, ONE, O_l_hi, C00}, + {V_s0, Vins, AMOVL, O_r_lo, O_r0}, + {Vins, AMULL, O_r0, O_Zop}, + {Vgo, V_p0, V_s0}, + {Vins, AMOVL, O_l_hi, O_r0}, + {Vmul, O_r_lo, O_r0}, + {Vins, AMOVL, O_r_lo, O_l_hi}, + {Vins, AMULL, O_l_hi, O_Zop}, + {Vins, AADDL, O_r0, O_l_hi}, + {V_f0, V_p0, Vend}, +}; + +/* mul of const64 */ +static uchar mulc64[][VLEN] = +{ + {V_a0, Vins, AMOVL, O_r_hi, O_r0}, + {Vop, OOR, O_l_hi, O_r0}, + {Vop, ONE, O_r0, C00}, + {V_s0, Vins, AMOVL, O_r_lo, O_r0}, + {Vins, AMULL, O_r0, O_Zop}, + {Vgo, V_p0, V_s0}, + {Vmul, O_r_lo, O_l_hi}, + {Vins, AMOVL, O_l_lo, O_r0}, + {Vmul, O_r_hi, O_r0}, + {Vins, AADDL, O_l_hi, O_r0}, + {Vins, AMOVL, O_r_lo, O_l_hi}, + {Vins, AMULL, O_l_hi, O_Zop}, + {Vins, AADDL, O_r0, O_l_hi}, + {V_f0, V_p0, Vend}, +}; + +/* mul general */ +static uchar mull[][VLEN] = +{ + {V_a0, Vins, AMOVL, O_r_hi, O_r0}, + {Vop, OOR, O_l_hi, O_r0}, + {Vop, ONE, O_r0, C00}, + {V_s0, Vins, AMOVL, O_r_lo, O_r0}, + {Vins, AMULL, O_r0, O_Zop}, + {Vgo, V_p0, V_s0}, + {Vins, AIMULL, O_r_lo, O_l_hi}, + {Vins, AMOVL, O_l_lo, O_r0}, + {Vins, AIMULL, O_r_hi, O_r0}, + {Vins, AADDL, O_l_hi, O_r0}, + {Vins, AMOVL, O_r_lo, O_l_hi}, + {Vins, AMULL, O_l_hi, O_Zop}, + {Vins, AADDL, O_r0, O_l_hi}, + {V_f0, V_p0, Vend}, +}; + +/* cast rp l to rp t */ +static uchar castrp[][VLEN] = +{ + {Vmv, O_l, O_t_lo}, + {VT, Vins, AMOVL, O_t_lo, O_t_hi}, + {VT, Vins, ASARL, C31, O_t_hi}, + {VF, Vzero, O_t_hi}, + {Vend}, +}; + +/* cast rp l to addr t */ +static uchar castrpa[][VLEN] = +{ + {VT, V_a0, Vmv, O_l, O_r0}, + {VT, Vins, AMOVL, O_r0, O_t_lo}, + {VT, Vins, ASARL, C31, O_r0}, + {VT, Vins, AMOVL, O_r0, O_t_hi}, + {VT, V_f0}, + {VF, Vmv, O_l, O_t_lo}, + {VF, Vzero, O_t_hi}, + {Vend}, +}; + +static uchar netab0i[][VLEN] = +{ + {Vop, ONE, O_l_lo, O_r_lo}, + {V_s0, Vop, ONE, O_l_hi, O_r_hi}, + {V_s1, Vgo, V_s2, Vgo, V_s3}, + {VF, V_p0, V_p1, VT, V_p2}, + {Vgo, V_p3}, + {VT, V_p0, V_p1, VF, V_p2}, + {Vend}, +}; + +static uchar netabii[][VLEN] = +{ + {V_a0, Vins, AMOVL, O_l_lo, O_r0}, + {Vop, ONE, O_r0, O_r_lo}, + {V_s0, Vins, AMOVL, O_l_hi, O_r0}, + {Vop, ONE, O_r0, O_r_hi}, + {V_s1, Vgo, V_s2, Vgo, V_s3}, + {VF, V_p0, V_p1, VT, V_p2}, + {Vgo, V_p3}, + {VT, V_p0, V_p1, VF, V_p2}, + {V_f0, Vend}, +}; + +static uchar cmptab0i[][VLEN] = +{ + {Vopx, Bop0, O_l_hi, O_r_hi}, + {V_s0, Vins0, AJNE}, + {V_s1, Vopx, Bop1, O_l_lo, O_r_lo}, + {V_s2, Vgo, V_s3, Vgo, V_s4}, + {VT, V_p1, V_p3}, + {VF, V_p0, V_p2}, + {Vgo, V_p4}, + {VT, V_p0, V_p2}, + {VF, V_p1, V_p3}, + {Vend}, +}; + +static uchar cmptabii[][VLEN] = +{ + {V_a0, Vins, AMOVL, O_l_hi, O_r0}, + {Vopx, Bop0, O_r0, O_r_hi}, + {V_s0, Vins0, AJNE}, + {V_s1, Vins, AMOVL, O_l_lo, O_r0}, + {Vopx, Bop1, O_r0, O_r_lo}, + {V_s2, Vgo, V_s3, Vgo, V_s4}, + {VT, V_p1, V_p3}, + {VF, V_p0, V_p2}, + {Vgo, V_p4}, + {VT, V_p0, V_p2}, + {VF, V_p1, V_p3}, + {V_f0, Vend}, +}; + +static uchar (*NEtab[])[VLEN] = +{ + netab0i, netabii, +}; + +static uchar (*cmptab[])[VLEN] = +{ + cmptab0i, cmptabii, +}; + +static uchar GEargs[] = { OGT, OHS }; +static uchar GTargs[] = { OGT, OHI }; +static uchar HIargs[] = { OHI, OHI }; +static uchar HSargs[] = { OHI, OHS }; + +/* Big Generator */ +static void +biggen(Node *l, Node *r, Node *t, int true, uchar code[][VLEN], uchar *a) +{ + int i, j, g, oc, op, lo, ro, to, xo, *xp; + Type *lt; + Prog *pr[VOPS]; + Node *ot, *tl, *tr, tmps[2]; + uchar *c, (*cp)[VLEN], args[VARGS]; + + if(a != nil) + memmove(args, a, VARGS); +//print("biggen %d %d %d\n", args[0], args[1], args[2]); +//if(l) prtree(l, "l"); +//if(r) prtree(r, "r"); +//if(t) prtree(t, "t"); + lo = ro = to = 0; + cp = code; + + for (;;) { + c = *cp++; + g = 1; + i = 0; +//print("code %d %d %d %d %d\n", c[0], c[1], c[2], c[3], c[4]); + for(;;) { + switch(op = c[i]) { + case Vgo: + if(g) + gbranch(OGOTO); + i++; + break; + + case Vamv: + i += 3; + if(i > VLEN) { + diag(l, "bad Vop"); + return; + } + if(g) + args[c[i - 1]] = args[c[i - 2]]; + break; + + case Vzero: + i += 2; + if(i > VLEN) { + diag(l, "bad Vop"); + return; + } + j = i - 1; + goto op; + + case Vspazz: // nasty hack to save a reg in SUB +//print("spazz\n"); + if(g) { +//print("hi %R lo %R t %R\n", r->right->reg, r->left->reg, tmps[0].reg); + ot = r->right; + r->right = r->left; + tl = new(0, Z, Z); + *tl = tmps[0]; + r->left = tl; + tmps[0] = *ot; +//print("hi %R lo %R t %R\n", r->right->reg, r->left->reg, tmps[0].reg); + } + i++; + break; + + case Vmv: + case Vmul: + case Vshll: + i += 3; + if(i > VLEN) { + diag(l, "bad Vop"); + return; + } + j = i - 2; + goto op; + + case Vins0: + i += 2; + if(i > VLEN) { + diag(l, "bad Vop"); + return; + } + gins(c[i - 1], Z, Z); + break; + + case Vop: + case Vopx: + case Vins: + case Vinsl: + case Vinsr: + case Vinsla: + case Vinsra: + case Vinsx: + i += 4; + if(i > VLEN) { + diag(l, "bad Vop"); + return; + } + j = i - 2; + goto op; + + op: + if(!g) + break; + tl = Z; + tr = Z; + for(; j < i; j++) { + switch(c[j]) { + case C00: + ot = nodconst(0); + break; + case C01: + ot = nodconst(1); + break; + case C31: + ot = nodconst(31); + break; + case C32: + ot = nodconst(32); + break; + + case O_l: + case O_l_lo: + ot = l; xp = &lo; xo = 0; + goto op0; + case O_l_hi: + ot = l; xp = &lo; xo = SZ_LONG; + goto op0; + case O_r: + case O_r_lo: + ot = r; xp = &ro; xo = 0; + goto op0; + case O_r_hi: + ot = r; xp = &ro; xo = SZ_LONG; + goto op0; + case O_t_lo: + ot = t; xp = &to; xo = 0; + goto op0; + case O_t_hi: + ot = t; xp = &to; xo = SZ_LONG; + goto op0; + case O_l_rp: + ot = l; + break; + case O_r_rp: + ot = r; + break; + case O_t_rp: + ot = t; + break; + case O_r0: + case O_r1: + ot = &tmps[c[j] - O_r0]; + break; + case O_Zop: + ot = Z; + break; + + op0: + switch(ot->op) { + case OCONST: + if(xo) + ot = hi64(ot); + else + ot = lo64(ot); + break; + case OREGPAIR: + if(xo) + ot = ot->right; + else + ot = ot->left; + break; + case OREGISTER: + break; + default: + if(xo != *xp) { + ot->xoffset += xo - *xp; + *xp = xo; + } + } + break; + + default: + diag(l, "bad V_lop"); + return; + } + if(tl == nil) + tl = ot; + else + tr = ot; + } + if(op == Vzero) { + zeroregm(tl); + break; + } + oc = c[i - 3]; + if(op == Vinsx || op == Vopx) { +//print("%d -> %d\n", oc, args[oc]); + oc = args[oc]; + } + else { + switch(oc) { + case O_a0: + case O_a1: + oc = args[oc - O_a0]; + break; + } + } + switch(op) { + case Vmul: + mulgen(tr->type, tl, tr); + break; + case Vmv: + gmove(tl, tr); + break; + case Vshll: + shiftit(tr->type, tl, tr); + break; + case Vop: + case Vopx: + gopcode(oc, types[TULONG], tl, tr); + break; + case Vins: + case Vinsx: + gins(oc, tl, tr); + break; + case Vinsl: + gins(oc, tl, tr->right); + p->from.index = tr->left->reg; + break; + case Vinsr: + gins(oc, tl, tr->left); + p->from.index = tr->right->reg; + break; + case Vinsla: + gins(oc, tl, tr + 1); + p->from.index = tr->reg; + break; + case Vinsra: + gins(oc, tl, tr); + p->from.index = (tr + 1)->reg; + break; + } + break; + + case VT: + g = true; + i++; + break; + case VF: + g = !true; + i++; + break; + + case V_T0: case V_T1: + g = args[op - V_T0]; + i++; + break; + + case V_F0: case V_F1: + g = !args[op - V_F0]; + i++; + break; + + case V_C0: case V_C1: + if(g) + args[op - V_C0] = 0; + i++; + break; + + case V_S0: case V_S1: + if(g) + args[op - V_S0] = 1; + i++; + break; + + case V_l_lo_f: + g = lo64v(l) == 0; + i++; + break; + case V_l_hi_f: + g = hi64v(l) == 0; + i++; + break; + case V_l_lo_t: + g = lo64v(l) != 0; + i++; + break; + case V_l_hi_t: + g = hi64v(l) != 0; + i++; + break; + case V_l_lo_u: + g = lo64v(l) >= 0; + i++; + break; + case V_l_hi_u: + g = hi64v(l) >= 0; + i++; + break; + case V_r_lo_f: + g = lo64v(r) == 0; + i++; + break; + case V_r_hi_f: + g = hi64v(r) == 0; + i++; + break; + case V_r_lo_t: + g = lo64v(r) != 0; + i++; + break; + case V_r_hi_t: + g = hi64v(r) != 0; + i++; + break; + case V_r_lo_u: + g = lo64v(r) >= 0; + i++; + break; + case V_r_hi_u: + g = hi64v(r) >= 0; + i++; + break; + + case Vend: + goto out; + + case V_a0: case V_a1: + if(g) { + lt = l->type; + l->type = types[TULONG]; + regalloc(&tmps[op - V_a0], l, Z); + l->type = lt; + } + i++; + break; + + case V_f0: case V_f1: + if(g) + regfree(&tmps[op - V_f0]); + i++; + break; + + case V_p0: case V_p1: case V_p2: case V_p3: case V_p4: + if(g) + patch(pr[op - V_p0], pc); + i++; + break; + + case V_s0: case V_s1: case V_s2: case V_s3: case V_s4: + if(g) + pr[op - V_s0] = p; + i++; + break; + + default: + diag(l, "bad biggen: %d", op); + return; + } + if(i == VLEN || c[i] == 0) + break; + } + } +out: + if(lo) + l->xoffset -= lo; + if(ro) + r->xoffset -= ro; + if(to) + t->xoffset -= to; +} + +int +cgen64(Node *n, Node *nn) +{ + Type *dt; + uchar *args, (*cp)[VLEN], (**optab)[VLEN]; + int li, ri, lri, dr, si, m, op, sh, cmp, true; + Node *c, *d, *l, *r, *t, *s, nod1, nod2, nod3, nod4, nod5; + + if(debug['g']) { + prtree(nn, "cgen64 lhs"); + prtree(n, "cgen64"); + print("AX = %d\n", reg[D_AX]); + } + cmp = 0; + sh = 0; + + switch(n->op) { + case ONEG: + d = regpair(nn, n); + sugen(n->left, d, 8); + gins(ANOTL, Z, d->right); + gins(ANEGL, Z, d->left); + gins(ASBBL, nodconst(-1), d->right); + break; + + case OCOM: + if(!vaddr(n->left, 0) || !vaddr(nn, 0)) + d = regpair(nn, n); + else + return 0; + sugen(n->left, d, 8); + gins(ANOTL, Z, d->left); + gins(ANOTL, Z, d->right); + break; + + case OADD: + optab = ADDtab; + args = ADDargs; + goto twoop; + case OAND: + optab = ANDtab; + args = ANDargs; + goto twoop; + case OOR: + optab = ORtab; + args = ORargs; + goto twoop; + case OSUB: + optab = SUBtab; + args = SUBargs; + goto twoop; + case OXOR: + optab = ORtab; + args = XORargs; + goto twoop; + case OASHL: + sh = 1; + args = nil; + optab = shlltab; + goto twoop; + case OLSHR: + sh = 1; + args = shrlargs; + optab = shrltab; + goto twoop; + case OASHR: + sh = 1; + args = sarlargs; + optab = shrltab; + goto twoop; + case OEQ: + cmp = 1; + args = nil; + optab = nil; + goto twoop; + case ONE: + cmp = 1; + args = nil; + optab = nil; + goto twoop; + case OLE: + cmp = 1; + args = nil; + optab = nil; + goto twoop; + case OLT: + cmp = 1; + args = nil; + optab = nil; + goto twoop; + case OGE: + cmp = 1; + args = nil; + optab = nil; + goto twoop; + case OGT: + cmp = 1; + args = nil; + optab = nil; + goto twoop; + case OHI: + cmp = 1; + args = nil; + optab = nil; + goto twoop; + case OHS: + cmp = 1; + args = nil; + optab = nil; + goto twoop; + case OLO: + cmp = 1; + args = nil; + optab = nil; + goto twoop; + case OLS: + cmp = 1; + args = nil; + optab = nil; + goto twoop; + +twoop: + dr = nn != Z && nn->op == OREGPAIR; + l = vfunc(n->left, nn); + if(sh) + r = n->right; + else + r = vfunc(n->right, nn); + + li = l->op == ONAME || l->op == OINDREG || l->op == OCONST; + ri = r->op == ONAME || r->op == OINDREG || r->op == OCONST; + +#define IMM(l, r) ((l) | ((r) << 1)) + + lri = IMM(li, ri); + + /* find out what is so easy about some operands */ + if(li) + li = whatof(l, sh | cmp); + if(ri) + ri = whatof(r, cmp); + + if(sh) + goto shift; + + if(cmp) + goto cmp; + + /* evaluate hard subexps, stealing nn if possible. */ + switch(lri) { + case IMM(0, 0): + bin00: + if(l->complex > r->complex) { + if(dr) + t = nn; + else + t = regpair(Z, n); + sugen(l, t, 8); + l = t; + t = regpair(Z, n); + sugen(r, t, 8); + r = t; + } + else { + t = regpair(Z, n); + sugen(r, t, 8); + r = t; + if(dr) + t = nn; + else + t = regpair(Z, n); + sugen(l, t, 8); + l = t; + } + break; + case IMM(0, 1): + if(dr) + t = nn; + else + t = regpair(Z, n); + sugen(l, t, 8); + l = t; + break; + case IMM(1, 0): + if(n->op == OSUB && l->op == OCONST && hi64v(l) == 0) { + lri = IMM(0, 0); + goto bin00; + } + if(dr) + t = nn; + else + t = regpair(Z, n); + sugen(r, t, 8); + r = t; + break; + case IMM(1, 1): + break; + } + +#define WW(l, r) ((l) | ((r) << 2)) + d = Z; + dt = nn->type; + nn->type = types[TLONG]; + + switch(lri) { + case IMM(0, 0): + biggen(l, r, Z, 0, binop00, args); + break; + case IMM(0, 1): + switch(ri) { + case WNONE: + diag(r, "bad whatof\n"); + break; + case WCONST: + biggen(l, r, Z, 0, optab[B0c], args); + break; + case WHARD: + reglcgen(&nod2, r, Z); + r = &nod2; + /* fall thru */ + case WADDR: + biggen(l, r, Z, 0, binoptmp, args); + if(ri == WHARD) + regfree(r); + break; + } + break; + case IMM(1, 0): + if(n->op == OSUB) { + switch(li) { + case WNONE: + diag(l, "bad whatof\n"); + break; + case WHARD: + reglcgen(&nod2, l, Z); + l = &nod2; + /* fall thru */ + case WADDR: + case WCONST: + biggen(l, r, Z, 0, sub10, args); + break; + } + if(li == WHARD) + regfree(l); + } + else { + switch(li) { + case WNONE: + diag(l, "bad whatof\n"); + break; + case WCONST: + biggen(r, l, Z, 0, optab[B0c], args); + break; + case WHARD: + reglcgen(&nod2, l, Z); + l = &nod2; + /* fall thru */ + case WADDR: + biggen(r, l, Z, 0, binoptmp, args); + if(li == WHARD) + regfree(l); + break; + } + } + break; + case IMM(1, 1): + switch(WW(li, ri)) { + case WW(WCONST, WHARD): + if(r->op == ONAME && n->op == OAND && reduxv(l)) + ri = WADDR; + break; + case WW(WHARD, WCONST): + if(l->op == ONAME && n->op == OAND && reduxv(r)) + li = WADDR; + break; + } + if(li == WHARD) { + reglcgen(&nod3, l, Z); + l = &nod3; + } + if(ri == WHARD) { + reglcgen(&nod2, r, Z); + r = &nod2; + } + d = regpair(nn, n); + instpair(d, Z); + switch(WW(li, ri)) { + case WW(WCONST, WADDR): + case WW(WCONST, WHARD): + biggen(l, r, d, 0, optab[Bca], args); + break; + + case WW(WADDR, WCONST): + case WW(WHARD, WCONST): + biggen(l, r, d, 0, optab[Bac], args); + break; + + case WW(WADDR, WADDR): + case WW(WADDR, WHARD): + case WW(WHARD, WADDR): + case WW(WHARD, WHARD): + biggen(l, r, d, 0, binop11, args); + break; + + default: + diag(r, "bad whatof pair %d %d\n", li, ri); + break; + } + if(li == WHARD) + regfree(l); + if(ri == WHARD) + regfree(r); + break; + } + + nn->type = dt; + + if(d != Z) + goto finished; + + switch(lri) { + case IMM(0, 0): + freepair(r); + /* fall thru */; + case IMM(0, 1): + if(!dr) + storepair(l, nn, 1); + break; + case IMM(1, 0): + if(!dr) + storepair(r, nn, 1); + break; + case IMM(1, 1): + break; + } + return 1; + + shift: + c = Z; + + /* evaluate hard subexps, stealing nn if possible. */ + /* must also secure CX. not as many optims as binop. */ + switch(lri) { + case IMM(0, 0): + imm00: + if(l->complex + 1 > r->complex) { + if(dr) + t = nn; + else + t = regpair(Z, l); + sugen(l, t, 8); + l = t; + t = &nod1; + c = snarfreg(l, t, D_CX, r, &nod2); + cgen(r, t); + r = t; + } + else { + t = &nod1; + c = snarfreg(nn, t, D_CX, r, &nod2); + cgen(r, t); + r = t; + if(dr) + t = nn; + else + t = regpair(Z, l); + sugen(l, t, 8); + l = t; + } + break; + case IMM(0, 1): + imm01: + if(ri != WCONST) { + lri = IMM(0, 0); + goto imm00; + } + if(dr) + t = nn; + else + t = regpair(Z, n); + sugen(l, t, 8); + l = t; + break; + case IMM(1, 0): + imm10: + if(li != WCONST) { + lri = IMM(0, 0); + goto imm00; + } + t = &nod1; + c = snarfreg(nn, t, D_CX, r, &nod2); + cgen(r, t); + r = t; + break; + case IMM(1, 1): + if(ri != WCONST) { + lri = IMM(1, 0); + goto imm10; + } + if(li == WHARD) { + lri = IMM(0, 1); + goto imm01; + } + break; + } + + d = Z; + + switch(lri) { + case IMM(0, 0): + biggen(l, r, Z, 0, optab[S00], args); + break; + case IMM(0, 1): + switch(ri) { + case WNONE: + case WADDR: + case WHARD: + diag(r, "bad whatof\n"); + break; + case WCONST: + m = r->vconst & 63; + s = nodconst(m); + if(m < 32) + cp = optab[Sc0]; + else if(m == 32) + cp = optab[Sc1]; + else + cp = optab[Sc2]; + biggen(l, s, Z, 0, cp, args); + break; + } + break; + case IMM(1, 0): + /* left is const */ + d = regpair(nn, n); + instpair(d, Z); + biggen(l, r, d, 0, optab[S10], args); + regfree(r); + break; + case IMM(1, 1): + d = regpair(nn, n); + instpair(d, Z); + switch(WW(li, ri)) { + case WW(WADDR, WCONST): + m = r->vconst & 63; + s = nodconst(m); + if(m < 32) { + loadpair(l, d); + l = d; + cp = optab[Sc0]; + } + else if(m == 32) + cp = optab[Sac3]; + else + cp = optab[Sac4]; + biggen(l, s, d, 0, cp, args); + break; + + default: + diag(r, "bad whatof pair %d %d\n", li, ri); + break; + } + break; + } + + if(c != Z) { + gins(AMOVL, c, r); + regfree(c); + } + + if(d != Z) + goto finished; + + switch(lri) { + case IMM(0, 0): + regfree(r); + /* fall thru */ + case IMM(0, 1): + if(!dr) + storepair(l, nn, 1); + break; + case IMM(1, 0): + regfree(r); + break; + case IMM(1, 1): + break; + } + return 1; + + cmp: + op = n->op; + /* evaluate hard subexps */ + switch(lri) { + case IMM(0, 0): + if(l->complex > r->complex) { + t = regpair(Z, l); + sugen(l, t, 8); + l = t; + t = regpair(Z, r); + sugen(r, t, 8); + r = t; + } + else { + t = regpair(Z, r); + sugen(r, t, 8); + r = t; + t = regpair(Z, l); + sugen(l, t, 8); + l = t; + } + break; + case IMM(1, 0): + t = r; + r = l; + l = t; + ri = li; + op = invrel[relindex(op)]; + /* fall thru */ + case IMM(0, 1): + t = regpair(Z, l); + sugen(l, t, 8); + l = t; + break; + case IMM(1, 1): + break; + } + + true = 1; + optab = cmptab; + switch(op) { + case OEQ: + optab = NEtab; + true = 0; + break; + case ONE: + optab = NEtab; + break; + case OLE: + args = GTargs; + true = 0; + break; + case OGT: + args = GTargs; + break; + case OLS: + args = HIargs; + true = 0; + break; + case OHI: + args = HIargs; + break; + case OLT: + args = GEargs; + true = 0; + break; + case OGE: + args = GEargs; + break; + case OLO: + args = HSargs; + true = 0; + break; + case OHS: + args = HSargs; + break; + default: + diag(n, "bad cmp\n"); + SET(optab); + } + + switch(lri) { + case IMM(0, 0): + biggen(l, r, Z, true, optab[T0i], args); + break; + case IMM(0, 1): + case IMM(1, 0): + switch(ri) { + case WNONE: + diag(l, "bad whatof\n"); + break; + case WCONST: + biggen(l, r, Z, true, optab[T0i], args); + break; + case WHARD: + reglcgen(&nod2, r, Z); + r = &nod2; + /* fall thru */ + case WADDR: + biggen(l, r, Z, true, optab[T0i], args); + if(ri == WHARD) + regfree(r); + break; + } + break; + case IMM(1, 1): + if(li == WHARD) { + reglcgen(&nod3, l, Z); + l = &nod3; + } + if(ri == WHARD) { + reglcgen(&nod2, r, Z); + r = &nod2; + } + biggen(l, r, Z, true, optab[Tii], args); + if(li == WHARD) + regfree(l); + if(ri == WHARD) + regfree(r); + break; + } + + switch(lri) { + case IMM(0, 0): + freepair(r); + /* fall thru */; + case IMM(0, 1): + case IMM(1, 0): + freepair(l); + break; + case IMM(1, 1): + break; + } + return 1; + + case OASMUL: + case OASLMUL: + m = 0; + goto mulop; + + case OMUL: + case OLMUL: + m = 1; + goto mulop; + + mulop: + dr = nn != Z && nn->op == OREGPAIR; + l = vfunc(n->left, nn); + r = vfunc(n->right, nn); + if(r->op != OCONST) { + if(l->complex > r->complex) { + if(m) { + t = l; + l = r; + r = t; + } + else if(!vaddr(l, 1)) { + reglcgen(&nod5, l, Z); + l = &nod5; + evacaxdx(l); + } + } + t = regpair(Z, n); + sugen(r, t, 8); + r = t; + evacaxdx(r->left); + evacaxdx(r->right); + if(l->complex <= r->complex && !m && !vaddr(l, 1)) { + reglcgen(&nod5, l, Z); + l = &nod5; + evacaxdx(l); + } + } + if(dr) + t = nn; + else + t = regpair(Z, n); + c = Z; + d = Z; + if(!nodreg(&nod1, t->left, D_AX)) { + if(t->left->reg != D_AX){ + t->left->reg = D_AX; + reg[D_AX]++; + }else if(reg[D_AX] == 0) + fatal(Z, "vlong mul AX botch"); + } + if(!nodreg(&nod2, t->right, D_DX)) { + if(t->right->reg != D_DX){ + t->right->reg = D_DX; + reg[D_DX]++; + }else if(reg[D_DX] == 0) + fatal(Z, "vlong mul DX botch"); + } + if(m) + sugen(l, t, 8); + else + loadpair(l, t); + if(t->left->reg != D_AX) { + c = &nod3; + regsalloc(c, t->left); + gmove(&nod1, c); + gmove(t->left, &nod1); + zapreg(t->left); + } + if(t->right->reg != D_DX) { + d = &nod4; + regsalloc(d, t->right); + gmove(&nod2, d); + gmove(t->right, &nod2); + zapreg(t->right); + } + if(c != Z || d != Z) { + s = regpair(Z, n); + s->left = &nod1; + s->right = &nod2; + } + else + s = t; + if(r->op == OCONST) { + if(hi64v(r) == 0) + biggen(s, r, Z, 0, mulc32, nil); + else + biggen(s, r, Z, 0, mulc64, nil); + } + else + biggen(s, r, Z, 0, mull, nil); + instpair(t, Z); + if(c != Z) { + gmove(&nod1, t->left); + gmove(&nod3, &nod1); + } + if(d != Z) { + gmove(&nod2, t->right); + gmove(&nod4, &nod2); + } + if(r->op == OREGPAIR) + freepair(r); + if(!m) + storepair(t, l, 0); + if(l == &nod5) + regfree(l); + if(!dr) { + if(nn != Z) + storepair(t, nn, 1); + else + freepair(t); + } + return 1; + + case OASADD: + args = ADDargs; + goto vasop; + case OASAND: + args = ANDargs; + goto vasop; + case OASOR: + args = ORargs; + goto vasop; + case OASSUB: + args = SUBargs; + goto vasop; + case OASXOR: + args = XORargs; + goto vasop; + + vasop: + l = n->left; + r = n->right; + dr = nn != Z && nn->op == OREGPAIR; + m = 0; + if(l->complex > r->complex) { + if(!vaddr(l, 1)) { + reglcgen(&nod1, l, Z); + l = &nod1; + } + if(!vaddr(r, 1) || nn != Z || r->op == OCONST) { + if(dr) + t = nn; + else + t = regpair(Z, r); + sugen(r, t, 8); + r = t; + m = 1; + } + } + else { + if(!vaddr(r, 1) || nn != Z || r->op == OCONST) { + if(dr) + t = nn; + else + t = regpair(Z, r); + sugen(r, t, 8); + r = t; + m = 1; + } + if(!vaddr(l, 1)) { + reglcgen(&nod1, l, Z); + l = &nod1; + } + } + if(nn != Z) { + if(n->op == OASSUB) + biggen(l, r, Z, 0, sub10, args); + else + biggen(r, l, Z, 0, binoptmp, args); + storepair(r, l, 0); + } + else { + if(m) + biggen(l, r, Z, 0, binop00, args); + else + biggen(l, r, Z, 0, binoptmp, args); + } + if(l == &nod1) + regfree(&nod1); + if(m) { + if(nn == Z) + freepair(r); + else if(!dr) + storepair(r, nn, 1); + } + return 1; + + case OASASHL: + args = nil; + optab = asshlltab; + goto assh; + case OASLSHR: + args = shrlargs; + optab = asshrltab; + goto assh; + case OASASHR: + args = sarlargs; + optab = asshrltab; + goto assh; + + assh: + c = Z; + l = n->left; + r = n->right; + if(r->op == OCONST) { + m = r->vconst & 63; + if(m < 32) + m = SAclo; + else if(m == 32) + m = SAc32; + else + m = SAchi; + } + else + m = SAgen; + if(l->complex > r->complex) { + if(!vaddr(l, 0)) { + reglcgen(&nod1, l, Z); + l = &nod1; + } + if(m == SAgen) { + t = &nod2; + if(l->reg == D_CX) { + regalloc(t, r, Z); + gmove(l, t); + l->reg = t->reg; + t->reg = D_CX; + } + else + c = snarfreg(nn, t, D_CX, r, &nod3); + cgen(r, t); + r = t; + } + } + else { + if(m == SAgen) { + t = &nod2; + c = snarfreg(nn, t, D_CX, r, &nod3); + cgen(r, t); + r = t; + } + if(!vaddr(l, 0)) { + reglcgen(&nod1, l, Z); + l = &nod1; + } + } + + if(nn != Z) { + m += SAdgen - SAgen; + d = regpair(nn, n); + instpair(d, Z); + biggen(l, r, d, 0, optab[m], args); + if(l == &nod1) { + regfree(&nod1); + l = Z; + } + if(r == &nod2 && c == Z) { + regfree(&nod2); + r = Z; + } + if(d != nn) + storepair(d, nn, 1); + } + else + biggen(l, r, Z, 0, optab[m], args); + + if(c != Z) { + gins(AMOVL, c, r); + regfree(c); + } + if(l == &nod1) + regfree(&nod1); + if(r == &nod2) + regfree(&nod2); + return 1; + + case OPOSTINC: + args = ADDargs; + cp = incdecpost; + goto vinc; + case OPOSTDEC: + args = SUBargs; + cp = incdecpost; + goto vinc; + case OPREINC: + args = ADDargs; + cp = incdecpre; + goto vinc; + case OPREDEC: + args = SUBargs; + cp = incdecpre; + goto vinc; + + vinc: + l = n->left; + if(!vaddr(l, 1)) { + reglcgen(&nod1, l, Z); + l = &nod1; + } + + if(nn != Z) { + d = regpair(nn, n); + instpair(d, Z); + biggen(l, Z, d, 0, cp, args); + if(l == &nod1) { + regfree(&nod1); + l = Z; + } + if(d != nn) + storepair(d, nn, 1); + } + else + biggen(l, Z, Z, 0, incdec, args); + + if(l == &nod1) + regfree(&nod1); + return 1; + + case OCAST: + l = n->left; + if(typev[l->type->etype]) { + if(!vaddr(l, 1)) { + if(l->complex + 1 > nn->complex) { + d = regpair(Z, l); + sugen(l, d, 8); + if(!vaddr(nn, 1)) { + reglcgen(&nod1, nn, Z); + r = &nod1; + } + else + r = nn; + } + else { + if(!vaddr(nn, 1)) { + reglcgen(&nod1, nn, Z); + r = &nod1; + } + else + r = nn; + d = regpair(Z, l); + sugen(l, d, 8); + } +// d->left->type = r->type; + d->left->type = types[TLONG]; + gmove(d->left, r); + freepair(d); + } + else { + if(nn->op != OREGISTER && !vaddr(nn, 1)) { + reglcgen(&nod1, nn, Z); + r = &nod1; + } + else + r = nn; +// l->type = r->type; + l->type = types[TLONG]; + gmove(l, r); + } + if(r != nn) + regfree(r); + } + else { + if(typeu[l->type->etype] || cond(l->op)) + si = TUNSIGNED; + else + si = TSIGNED; + regalloc(&nod1, l, Z); + cgen(l, &nod1); + if(nn->op == OREGPAIR) { + m = instpair(nn, &nod1); + biggen(&nod1, Z, nn, si == TSIGNED, castrp, nil); + } + else { + m = 0; + if(!vaddr(nn, si != TSIGNED)) { + dt = nn->type; + nn->type = types[TLONG]; + reglcgen(&nod2, nn, Z); + nn->type = dt; + nn = &nod2; + } + dt = nn->type; + nn->type = types[TLONG]; + biggen(&nod1, Z, nn, si == TSIGNED, castrpa, nil); + nn->type = dt; + if(nn == &nod2) + regfree(&nod2); + } + if(!m) + regfree(&nod1); + } + return 1; + + default: + if(n->op == OREGPAIR) { + storepair(n, nn, 1); + return 1; + } + if(nn->op == OREGPAIR) { + loadpair(n, nn); + return 1; + } + return 0; + } +finished: + if(d != nn) + storepair(d, nn, 1); + return 1; +} + +void +testv(Node *n, int true) +{ + Type *t; + Node *nn, nod; + + switch(n->op) { + case OINDREG: + case ONAME: + biggen(n, Z, Z, true, testi, nil); + break; + + default: + n = vfunc(n, n); + if(n->addable >= INDEXED) { + t = n->type; + n->type = types[TLONG]; + reglcgen(&nod, n, Z); + n->type = t; + n = &nod; + biggen(n, Z, Z, true, testi, nil); + if(n == &nod) + regfree(n); + } + else { + nn = regpair(Z, n); + sugen(n, nn, 8); + biggen(nn, Z, Z, true, testi, nil); + freepair(nn); + } + } +} diff --git a/src/cmd/8c/div.c b/src/cmd/8c/div.c new file mode 100644 index 000000000..14945052e --- /dev/null +++ b/src/cmd/8c/div.c @@ -0,0 +1,236 @@ +// Inferno utils/8c/div.c +// http://code.google.com/p/inferno-os/source/browse/utils/8c/div.c +// +// Copyright © 1994-1999 Lucent Technologies Inc. All rights reserved. +// Portions Copyright © 1995-1997 C H Forsyth (forsyth@terzarima.net) +// Portions Copyright © 1997-1999 Vita Nuova Limited +// Portions Copyright © 2000-2007 Vita Nuova Holdings Limited (www.vitanuova.com) +// Portions Copyright © 2004,2006 Bruce Ellis +// Portions Copyright © 2005-2007 C H Forsyth (forsyth@terzarima.net) +// Revisions Copyright © 2000-2007 Lucent Technologies Inc. and others +// Portions Copyright © 2009 The Go Authors. All rights reserved. +// +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to deal +// in the Software without restriction, including without limitation the rights +// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +// copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +// THE SOFTWARE. + +#include "gc.h" + +/* + * Based on: Granlund, T.; Montgomery, P.L. + * "Division by Invariant Integers using Multiplication". + * SIGPLAN Notices, Vol. 29, June 1994, page 61. + */ + +#define TN(n) ((uvlong)1 << (n)) +#define T31 TN(31) +#define T32 TN(32) + +int +multiplier(uint32 d, int p, uvlong *mp) +{ + int l; + uvlong mlo, mhi, tlo, thi; + + l = topbit(d - 1) + 1; + mlo = (((TN(l) - d) << 32) / d) + T32; + if(l + p == 64) + mhi = (((TN(l) + 1 - d) << 32) / d) + T32; + else + mhi = (TN(32 + l) + TN(32 + l - p)) / d; + /*assert(mlo < mhi);*/ + while(l > 0) { + tlo = mlo >> 1; + thi = mhi >> 1; + if(tlo == thi) + break; + mlo = tlo; + mhi = thi; + l--; + } + *mp = mhi; + return l; +} + +int +sdiv(uint32 d, uint32 *mp, int *sp) +{ + int s; + uvlong m; + + s = multiplier(d, 32 - 1, &m); + *mp = m; + *sp = s; + if(m >= T31) + return 1; + else + return 0; +} + +int +udiv(uint32 d, uint32 *mp, int *sp, int *pp) +{ + int p, s; + uvlong m; + + s = multiplier(d, 32, &m); + p = 0; + if(m >= T32) { + while((d & 1) == 0) { + d >>= 1; + p++; + } + s = multiplier(d, 32 - p, &m); + } + *mp = m; + *pp = p; + if(m >= T32) { + /*assert(p == 0);*/ + *sp = s - 1; + return 1; + } + else { + *sp = s; + return 0; + } +} + +void +sdivgen(Node *l, Node *r, Node *ax, Node *dx) +{ + int a, s; + uint32 m; + vlong c; + + c = r->vconst; + if(c < 0) + c = -c; + a = sdiv(c, &m, &s); +//print("a=%d i=%d s=%d m=%ux\n", a, (int32)r->vconst, s, m); + gins(AMOVL, nodconst(m), ax); + gins(AIMULL, l, Z); + gins(AMOVL, l, ax); + if(a) + gins(AADDL, ax, dx); + gins(ASHRL, nodconst(31), ax); + gins(ASARL, nodconst(s), dx); + gins(AADDL, ax, dx); + if(r->vconst < 0) + gins(ANEGL, Z, dx); +} + +void +udivgen(Node *l, Node *r, Node *ax, Node *dx) +{ + int a, s, t; + uint32 m; + Node nod; + + a = udiv(r->vconst, &m, &s, &t); +//print("a=%ud i=%d p=%d s=%d m=%ux\n", a, (int32)r->vconst, t, s, m); + if(t != 0) { + gins(AMOVL, l, ax); + gins(ASHRL, nodconst(t), ax); + gins(AMOVL, nodconst(m), dx); + gins(AMULL, dx, Z); + } + else if(a) { + if(l->op != OREGISTER) { + regalloc(&nod, l, Z); + gins(AMOVL, l, &nod); + l = &nod; + } + gins(AMOVL, nodconst(m), ax); + gins(AMULL, l, Z); + gins(AADDL, l, dx); + gins(ARCRL, nodconst(1), dx); + if(l == &nod) + regfree(l); + } + else { + gins(AMOVL, nodconst(m), ax); + gins(AMULL, l, Z); + } + if(s != 0) + gins(ASHRL, nodconst(s), dx); +} + +void +sext(Node *d, Node *s, Node *l) +{ + if(s->reg == D_AX && !nodreg(d, Z, D_DX)) { + reg[D_DX]++; + gins(ACDQ, Z, Z); + } + else { + regalloc(d, l, Z); + gins(AMOVL, s, d); + gins(ASARL, nodconst(31), d); + } +} + +void +sdiv2(int32 c, int v, Node *l, Node *n) +{ + Node nod; + + if(v > 0) { + if(v > 1) { + sext(&nod, n, l); + gins(AANDL, nodconst((1 << v) - 1), &nod); + gins(AADDL, &nod, n); + regfree(&nod); + } + else { + gins(ACMPL, n, nodconst(0x80000000)); + gins(ASBBL, nodconst(-1), n); + } + gins(ASARL, nodconst(v), n); + } + if(c < 0) + gins(ANEGL, Z, n); +} + +void +smod2(int32 c, int v, Node *l, Node *n) +{ + Node nod; + + if(c == 1) { + zeroregm(n); + return; + } + + sext(&nod, n, l); + if(v == 0) { + zeroregm(n); + gins(AXORL, &nod, n); + gins(ASUBL, &nod, n); + } + else if(v > 1) { + gins(AANDL, nodconst((1 << v) - 1), &nod); + gins(AADDL, &nod, n); + gins(AANDL, nodconst((1 << v) - 1), n); + gins(ASUBL, &nod, n); + } + else { + gins(AANDL, nodconst(1), n); + gins(AXORL, &nod, n); + gins(ASUBL, &nod, n); + } + regfree(&nod); +} diff --git a/src/cmd/8c/doc.go b/src/cmd/8c/doc.go new file mode 100644 index 000000000..e3aae857f --- /dev/null +++ b/src/cmd/8c/doc.go @@ -0,0 +1,14 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +/* + +8c is a version of the Plan 9 C compiler. The original is documented at + + http://plan9.bell-labs.com/magic/man2html/1/2c + +Its target architecture is the x86, referred to by these tools for historical reasons as 386. + +*/ +package documentation diff --git a/src/cmd/8c/gc.h b/src/cmd/8c/gc.h new file mode 100644 index 000000000..32b80e995 --- /dev/null +++ b/src/cmd/8c/gc.h @@ -0,0 +1,411 @@ +// Inferno utils/8c/gc.h +// http://code.google.com/p/inferno-os/source/browse/utils/8c/gc.h +// +// Copyright © 1994-1999 Lucent Technologies Inc. All rights reserved. +// Portions Copyright © 1995-1997 C H Forsyth (forsyth@terzarima.net) +// Portions Copyright © 1997-1999 Vita Nuova Limited +// Portions Copyright © 2000-2007 Vita Nuova Holdings Limited (www.vitanuova.com) +// Portions Copyright © 2004,2006 Bruce Ellis +// Portions Copyright © 2005-2007 C H Forsyth (forsyth@terzarima.net) +// Revisions Copyright © 2000-2007 Lucent Technologies Inc. and others +// Portions Copyright © 2009 The Go Authors. All rights reserved. +// +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to deal +// in the Software without restriction, including without limitation the rights +// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +// copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +// THE SOFTWARE. + +#include +#include "../cc/cc.h" +#include "../8l/8.out.h" + +/* + * 8c/386 + * Intel 386 + */ +#define SZ_CHAR 1 +#define SZ_SHORT 2 +#define SZ_INT 4 +#define SZ_LONG 4 +#define SZ_IND 4 +#define SZ_FLOAT 4 +#define SZ_VLONG 8 +#define SZ_DOUBLE 8 +#define FNX 100 + +typedef struct Adr Adr; +typedef struct Prog Prog; +typedef struct Case Case; +typedef struct C1 C1; +typedef struct Var Var; +typedef struct Reg Reg; +typedef struct Rgn Rgn; +typedef struct Renv Renv; + +EXTERN struct +{ + Node* regtree; + Node* basetree; + short scale; + short reg; + short ptr; +} idx; + +struct Adr +{ + int32 offset; + int32 offset2; + double dval; + char sval[NSNAME]; + + Sym* sym; + uchar type; + uchar index; + uchar etype; + uchar scale; /* doubles as width in DATA op */ +}; +#define A ((Adr*)0) + +#define INDEXED 9 +struct Prog +{ + Adr from; + Adr to; + Prog* link; + int32 lineno; + short as; +}; +#define P ((Prog*)0) + +struct Case +{ + Case* link; + int32 val; + int32 label; + char def; + char isv; +}; +#define C ((Case*)0) + +struct C1 +{ + int32 val; + int32 label; +}; + +struct Var +{ + int32 offset; + Sym* sym; + char name; + char etype; +}; + +struct Reg +{ + int32 pc; + int32 rpo; /* reverse post ordering */ + + Bits set; + Bits use1; + Bits use2; + + Bits refbehind; + Bits refahead; + Bits calbehind; + Bits calahead; + Bits regdiff; + Bits act; + + int32 regu; + int32 loop; /* could be shorter */ + + Reg* log5; + int32 active; + + Reg* p1; + Reg* p2; + Reg* p2link; + Reg* s1; + Reg* s2; + Reg* link; + Prog* prog; +}; +#define R ((Reg*)0) + +struct Renv +{ + int safe; + Node base; + Node* saved; + Node* scope; +}; + +#define NRGN 600 +struct Rgn +{ + Reg* enter; + short cost; + short varno; + short regno; +}; + +EXTERN int32 breakpc; +EXTERN int32 nbreak; +EXTERN Case* cases; +EXTERN Node constnode; +EXTERN Node fconstnode; +EXTERN int32 continpc; +EXTERN int32 curarg; +EXTERN int32 cursafe; +EXTERN Prog* firstp; +EXTERN Prog* lastp; +EXTERN int32 maxargsafe; +EXTERN int mnstring; +EXTERN int retok; +EXTERN Node* nodrat; +EXTERN Node* nodret; +EXTERN Node* nodsafe; +EXTERN int32 nrathole; +EXTERN int32 nstring; +EXTERN Prog* p; +EXTERN int32 pc; +EXTERN Node regnode; +EXTERN Node fregnode0; +EXTERN Node fregnode1; +EXTERN char string[NSNAME]; +EXTERN Sym* symrathole; +EXTERN Node znode; +EXTERN Prog zprog; +EXTERN int reg[D_NONE]; +EXTERN int32 exregoffset; +EXTERN int32 exfregoffset; + +#define BLOAD(r) band(bnot(r->refbehind), r->refahead) +#define BSTORE(r) band(bnot(r->calbehind), r->calahead) +#define LOAD(r) (~r->refbehind.b[z] & r->refahead.b[z]) +#define STORE(r) (~r->calbehind.b[z] & r->calahead.b[z]) + +#define bset(a,n) ((a).b[(n)/32]&(1L<<(n)%32)) + +#define CLOAD 5 +#define CREF 5 +#define CINF 1000 +#define LOOP 3 + +EXTERN Rgn region[NRGN]; +EXTERN Rgn* rgp; +EXTERN int nregion; +EXTERN int nvar; + +EXTERN Bits externs; +EXTERN Bits params; +EXTERN Bits consts; +EXTERN Bits addrs; + +EXTERN int32 regbits; +EXTERN int32 exregbits; + +EXTERN int change; +EXTERN int suppress; + +EXTERN Reg* firstr; +EXTERN Reg* lastr; +EXTERN Reg zreg; +EXTERN Reg* freer; +EXTERN Var var[NVAR]; +EXTERN int32* idom; +EXTERN Reg** rpo2r; +EXTERN int32 maxnr; + +extern char* anames[]; + +/* + * sgen.c + */ +void codgen(Node*, Node*); +void gen(Node*); +void noretval(int); +void usedset(Node*, int); +void xcom(Node*); +void indx(Node*); +int bcomplex(Node*, Node*); +Prog* gtext(Sym*, int32); +vlong argsize(void); + +/* + * cgen.c + */ +void zeroregm(Node*); +void cgen(Node*, Node*); +void reglcgen(Node*, Node*, Node*); +void lcgen(Node*, Node*); +void bcgen(Node*, int); +void boolgen(Node*, int, Node*); +void sugen(Node*, Node*, int32); +int needreg(Node*, int); + +/* + * cgen64.c + */ +int vaddr(Node*, int); +void loadpair(Node*, Node*); +int cgen64(Node*, Node*); +void testv(Node*, int); + +/* + * txt.c + */ +void ginit(void); +void gclean(void); +void nextpc(void); +void gargs(Node*, Node*, Node*); +void garg1(Node*, Node*, Node*, int, Node**); +Node* nodconst(int32); +Node* nodfconst(double); +int nodreg(Node*, Node*, int); +int isreg(Node*, int); +void regret(Node*, Node*); +void regalloc(Node*, Node*, Node*); +void regfree(Node*); +void regialloc(Node*, Node*, Node*); +void regsalloc(Node*, Node*); +void regaalloc1(Node*, Node*); +void regaalloc(Node*, Node*); +void regind(Node*, Node*); +void gprep(Node*, Node*); +void naddr(Node*, Adr*); +void gmove(Node*, Node*); +void gins(int a, Node*, Node*); +void fgopcode(int, Node*, Node*, int, int); +void gopcode(int, Type*, Node*, Node*); +int samaddr(Node*, Node*); +void gbranch(int); +void patch(Prog*, int32); +int sconst(Node*); +void gpseudo(int, Sym*, Node*); + +/* + * swt.c + */ +int swcmp(const void*, const void*); +void doswit(Node*); +void swit1(C1*, int, int32, Node*); +void cas(void); +void bitload(Node*, Node*, Node*, Node*, Node*); +void bitstore(Node*, Node*, Node*, Node*, Node*); +int32 outstring(char*, int32); +void nullwarn(Node*, Node*); +void sextern(Sym*, Node*, int32, int32); +void gextern(Sym*, Node*, int32, int32); +void outcode(void); +void ieeedtod(Ieee*, double); + +/* + * list + */ +void listinit(void); +int Pconv(Fmt*); +int Aconv(Fmt*); +int Dconv(Fmt*); +int Sconv(Fmt*); +int Rconv(Fmt*); +int Xconv(Fmt*); +int Bconv(Fmt*); + +/* + * reg.c + */ +Reg* rega(void); +int rcmp(const void*, const void*); +void regopt(Prog*); +void addmove(Reg*, int, int, int); +Bits mkvar(Reg*, Adr*); +void prop(Reg*, Bits, Bits); +void loopit(Reg*, int32); +void synch(Reg*, Bits); +uint32 allreg(uint32, Rgn*); +void paint1(Reg*, int); +uint32 paint2(Reg*, int); +void paint3(Reg*, int, int32, int); +void addreg(Adr*, int); + +/* + * peep.c + */ +void peep(void); +void excise(Reg*); +Reg* uniqp(Reg*); +Reg* uniqs(Reg*); +int regtyp(Adr*); +int anyvar(Adr*); +int subprop(Reg*); +int copyprop(Reg*); +int copy1(Adr*, Adr*, Reg*, int); +int copyu(Prog*, Adr*, Adr*); + +int copyas(Adr*, Adr*); +int copyau(Adr*, Adr*); +int copysub(Adr*, Adr*, Adr*, int); +int copysub1(Prog*, Adr*, Adr*, int); + +int32 RtoB(int); +int32 FtoB(int); +int BtoR(int32); +int BtoF(int32); + +#define D_HI D_NONE +#define D_LO D_NONE + +/* + * bound + */ +void comtarg(void); + +/* + * com64 + */ +int cond(int); +int com64(Node*); +void com64init(void); +void bool64(Node*); +int32 lo64v(Node*); +int32 hi64v(Node*); +Node* lo64(Node*); +Node* hi64(Node*); + +/* + * div/mul + */ +void sdivgen(Node*, Node*, Node*, Node*); +void udivgen(Node*, Node*, Node*, Node*); +void sdiv2(int32, int, Node*, Node*); +void smod2(int32, int, Node*, Node*); +void mulgen(Type*, Node*, Node*); +void genmuladd(Node*, Node*, int, Node*); +void shiftit(Type*, Node*, Node*); + +#pragma varargck type "A" int +#pragma varargck type "B" Bits +#pragma varargck type "D" Adr* +#pragma varargck type "lD" Adr* +#pragma varargck type "P" Prog* +#pragma varargck type "R" int +#pragma varargck type "S" char* + +/* wrecklessly steal a field */ + +#define rplink label diff --git a/src/cmd/8c/list.c b/src/cmd/8c/list.c new file mode 100644 index 000000000..c422905cd --- /dev/null +++ b/src/cmd/8c/list.c @@ -0,0 +1,328 @@ +// Inferno utils/8c/list.c +// http://code.google.com/p/inferno-os/source/browse/utils/8c/list.c +// +// Copyright © 1994-1999 Lucent Technologies Inc. All rights reserved. +// Portions Copyright © 1995-1997 C H Forsyth (forsyth@terzarima.net) +// Portions Copyright © 1997-1999 Vita Nuova Limited +// Portions Copyright © 2000-2007 Vita Nuova Holdings Limited (www.vitanuova.com) +// Portions Copyright © 2004,2006 Bruce Ellis +// Portions Copyright © 2005-2007 C H Forsyth (forsyth@terzarima.net) +// Revisions Copyright © 2000-2007 Lucent Technologies Inc. and others +// Portions Copyright © 2009 The Go Authors. All rights reserved. +// +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to deal +// in the Software without restriction, including without limitation the rights +// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +// copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +// THE SOFTWARE. + +#define EXTERN +#include "gc.h" + +void +listinit(void) +{ + + fmtinstall('A', Aconv); + fmtinstall('B', Bconv); + fmtinstall('P', Pconv); + fmtinstall('S', Sconv); + fmtinstall('D', Dconv); + fmtinstall('R', Rconv); +} + +int +Bconv(Fmt *fp) +{ + char str[STRINGSZ], ss[STRINGSZ], *s; + Bits bits; + int i; + + str[0] = 0; + bits = va_arg(fp->args, Bits); + while(bany(&bits)) { + i = bnum(bits); + if(str[0]) + strcat(str, " "); + if(var[i].sym == S) { + sprint(ss, "$%d", var[i].offset); + s = ss; + } else + s = var[i].sym->name; + if(strlen(str) + strlen(s) + 1 >= STRINGSZ) + break; + strcat(str, s); + bits.b[i/32] &= ~(1L << (i%32)); + } + return fmtstrcpy(fp, str); +} + +int +Pconv(Fmt *fp) +{ + char str[STRINGSZ]; + Prog *p; + + p = va_arg(fp->args, Prog*); + switch(p->as) { + case ADATA: + sprint(str, "(%L) %A %D/%d,%D", + p->lineno, p->as, &p->from, p->from.scale, &p->to); + break; + + case ATEXT: + if(p->from.scale) { + sprint(str, "(%L) %A %D,%d,%lD", + p->lineno, p->as, &p->from, p->from.scale, &p->to); + break; + } + sprint(str, "(%L) %A %D,%lD", + p->lineno, p->as, &p->from, &p->to); + break; + + default: + sprint(str, "(%L) %A %D,%lD", + p->lineno, p->as, &p->from, &p->to); + break; + } + return fmtstrcpy(fp, str); +} + +int +Aconv(Fmt *fp) +{ + int i; + + i = va_arg(fp->args, int); + return fmtstrcpy(fp, anames[i]); +} + +int +Dconv(Fmt *fp) +{ + char str[STRINGSZ], s[STRINGSZ]; + Adr *a; + int i; + + a = va_arg(fp->args, Adr*); + i = a->type; + if(i >= D_INDIR) { + if(a->offset) + sprint(str, "%d(%R)", a->offset, i-D_INDIR); + else + sprint(str, "(%R)", i-D_INDIR); + goto brk; + } + switch(i) { + + default: + if(a->offset) + sprint(str, "$%d,%R", a->offset, i); + else + sprint(str, "%R", i); + break; + + case D_NONE: + str[0] = 0; + break; + + case D_BRANCH: + sprint(str, "%d(PC)", a->offset-pc); + break; + + case D_EXTERN: + sprint(str, "%s+%d(SB)", a->sym->name, a->offset); + break; + + case D_STATIC: + sprint(str, "%s<>+%d(SB)", a->sym->name, + a->offset); + break; + + case D_AUTO: + sprint(str, "%s+%d(SP)", a->sym->name, a->offset); + break; + + case D_PARAM: + if(a->sym) + sprint(str, "%s+%d(FP)", a->sym->name, a->offset); + else + sprint(str, "%d(FP)", a->offset); + break; + + case D_CONST: + sprint(str, "$%d", a->offset); + break; + + case D_CONST2: + sprint(str, "$%d-%d", a->offset, a->offset2); + break; + + case D_FCONST: + sprint(str, "$(%.17e)", a->dval); + break; + + case D_SCONST: + sprint(str, "$\"%S\"", a->sval); + break; + + case D_ADDR: + a->type = a->index; + a->index = D_NONE; + sprint(str, "$%D", a); + a->index = a->type; + a->type = D_ADDR; + goto conv; + } +brk: + if(a->index != D_NONE) { + sprint(s, "(%R*%d)", (int)a->index, (int)a->scale); + strcat(str, s); + } +conv: + return fmtstrcpy(fp, str); +} + +char* regstr[] = +{ + "AL", /*[D_AL]*/ + "CL", + "DL", + "BL", + "AH", + "CH", + "DH", + "BH", + + "AX", /*[D_AX]*/ + "CX", + "DX", + "BX", + "SP", + "BP", + "SI", + "DI", + + "F0", /*[D_F0]*/ + "F1", + "F2", + "F3", + "F4", + "F5", + "F6", + "F7", + + "CS", /*[D_CS]*/ + "SS", + "DS", + "ES", + "FS", + "GS", + + "GDTR", /*[D_GDTR]*/ + "IDTR", /*[D_IDTR]*/ + "LDTR", /*[D_LDTR]*/ + "MSW", /*[D_MSW] */ + "TASK", /*[D_TASK]*/ + + "CR0", /*[D_CR]*/ + "CR1", + "CR2", + "CR3", + "CR4", + "CR5", + "CR6", + "CR7", + + "DR0", /*[D_DR]*/ + "DR1", + "DR2", + "DR3", + "DR4", + "DR5", + "DR6", + "DR7", + + "TR0", /*[D_TR]*/ + "TR1", + "TR2", + "TR3", + "TR4", + "TR5", + "TR6", + "TR7", + + "NONE", /*[D_NONE]*/ +}; + +int +Rconv(Fmt *fp) +{ + char str[STRINGSZ]; + int r; + + r = va_arg(fp->args, int); + if(r >= D_AL && r <= D_NONE) + sprint(str, "%s", regstr[r-D_AL]); + else + sprint(str, "gok(%d)", r); + + return fmtstrcpy(fp, str); +} + +int +Sconv(Fmt *fp) +{ + int i, c; + char str[STRINGSZ], *p, *a; + + a = va_arg(fp->args, char*); + p = str; + for(i=0; i= 'a' && c <= 'z' || + c >= 'A' && c <= 'Z' || + c >= '0' && c <= '9') { + *p++ = c; + continue; + } + *p++ = '\\'; + switch(c) { + default: + if(c < 040 || c >= 0177) + break; /* not portable */ + p[-1] = c; + continue; + case 0: + *p++ = 'z'; + continue; + case '\\': + case '"': + *p++ = c; + continue; + case '\n': + *p++ = 'n'; + continue; + case '\t': + *p++ = 't'; + continue; + } + *p++ = (c>>6) + '0'; + *p++ = ((c>>3) & 7) + '0'; + *p++ = (c & 7) + '0'; + } + *p = 0; + return fmtstrcpy(fp, str); +} diff --git a/src/cmd/8c/machcap.c b/src/cmd/8c/machcap.c new file mode 100644 index 000000000..61e5aad16 --- /dev/null +++ b/src/cmd/8c/machcap.c @@ -0,0 +1,116 @@ +// Inferno utils/8c/machcap.c +// http://code.google.com/p/inferno-os/source/browse/utils/8c/machcap.c +// +// Copyright © 1994-1999 Lucent Technologies Inc. All rights reserved. +// Portions Copyright © 1995-1997 C H Forsyth (forsyth@terzarima.net) +// Portions Copyright © 1997-1999 Vita Nuova Limited +// Portions Copyright © 2000-2007 Vita Nuova Holdings Limited (www.vitanuova.com) +// Portions Copyright © 2004,2006 Bruce Ellis +// Portions Copyright © 2005-2007 C H Forsyth (forsyth@terzarima.net) +// Revisions Copyright © 2000-2007 Lucent Technologies Inc. and others +// Portions Copyright © 2009 The Go Authors. All rights reserved. +// +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to deal +// in the Software without restriction, including without limitation the rights +// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +// copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +// THE SOFTWARE. + +#include "gc.h" + +int +machcap(Node *n) +{ + + if(n == Z) + return 1; /* test */ + + switch(n->op) { + case OMUL: + case OLMUL: + case OASMUL: + case OASLMUL: + if(typechl[n->type->etype]) + return 1; + if(typev[n->type->etype]) { + return 1; + } + break; + + case OCOM: + case ONEG: + case OADD: + case OAND: + case OOR: + case OSUB: + case OXOR: + case OASHL: + case OLSHR: + case OASHR: + if(typechlv[n->left->type->etype]) + return 1; + break; + + case OCAST: + if(typev[n->type->etype]) { + if(typechlp[n->left->type->etype]) + return 1; + } + else if(!typefd[n->type->etype]) { + if(typev[n->left->type->etype]) + return 1; + } + break; + + case OCOND: + case OCOMMA: + case OLIST: + case OANDAND: + case OOROR: + case ONOT: + return 1; + + case OASADD: + case OASSUB: + case OASAND: + case OASOR: + case OASXOR: + return 1; + + case OASASHL: + case OASASHR: + case OASLSHR: + return 1; + + case OPOSTINC: + case OPOSTDEC: + case OPREINC: + case OPREDEC: + return 1; + + case OEQ: + case ONE: + case OLE: + case OGT: + case OLT: + case OGE: + case OHI: + case OHS: + case OLO: + case OLS: + return 1; + } + return 0; +} diff --git a/src/cmd/8c/mul.c b/src/cmd/8c/mul.c new file mode 100644 index 000000000..a0742807e --- /dev/null +++ b/src/cmd/8c/mul.c @@ -0,0 +1,458 @@ +// Inferno utils/8c/mul.c +// http://code.google.com/p/inferno-os/source/browse/utils/8c/mul.c +// +// Copyright © 1994-1999 Lucent Technologies Inc. All rights reserved. +// Portions Copyright © 1995-1997 C H Forsyth (forsyth@terzarima.net) +// Portions Copyright © 1997-1999 Vita Nuova Limited +// Portions Copyright © 2000-2007 Vita Nuova Holdings Limited (www.vitanuova.com) +// Portions Copyright © 2004,2006 Bruce Ellis +// Portions Copyright © 2005-2007 C H Forsyth (forsyth@terzarima.net) +// Revisions Copyright © 2000-2007 Lucent Technologies Inc. and others +// Portions Copyright © 2009 The Go Authors. All rights reserved. +// +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to deal +// in the Software without restriction, including without limitation the rights +// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +// copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +// THE SOFTWARE. + +#include "gc.h" + +typedef struct Malg Malg; +typedef struct Mparam Mparam; + +struct Malg +{ + char vals[10]; +}; + +struct Mparam +{ + uint32 value; + char alg; + char neg; + char shift; + char arg; + char off; +}; + +static Mparam multab[32]; +static int mulptr; + +static Malg malgs[] = +{ + {0, 100}, + {-1, 1, 100}, + {-9, -5, -3, 3, 5, 9, 100}, + {6, 10, 12, 18, 20, 24, 36, 40, 72, 100}, + {-8, -4, -2, 2, 4, 8, 100}, +}; + +/* + * return position of lowest 1 + */ +int +lowbit(uint32 v) +{ + int s, i; + uint32 m; + + s = 0; + m = 0xFFFFFFFFUL; + for(i = 16; i > 0; i >>= 1) { + m >>= i; + if((v & m) == 0) { + v >>= i; + s += i; + } + } + return s; +} + +void +genmuladd(Node *d, Node *s, int m, Node *a) +{ + Node nod; + + nod.op = OINDEX; + nod.left = a; + nod.right = s; + nod.scale = m; + nod.type = types[TIND]; + nod.xoffset = 0; + xcom(&nod); + gopcode(OADDR, d->type, &nod, d); +} + +void +mulparam(uint32 m, Mparam *mp) +{ + int c, i, j, n, o, q, s; + int bc, bi, bn, bo, bq, bs, bt; + char *p; + int32 u; + uint32 t; + + bc = bq = 10; + bi = bn = bo = bs = bt = 0; + for(i = 0; i < nelem(malgs); i++) { + for(p = malgs[i].vals, j = 0; (o = p[j]) < 100; j++) + for(s = 0; s < 2; s++) { + c = 10; + q = 10; + u = m - o; + if(u == 0) + continue; + if(s) { + o = -o; + if(o > 0) + continue; + u = -u; + } + n = lowbit(u); + t = (uint32)u >> n; + switch(i) { + case 0: + if(t == 1) { + c = s + 1; + q = 0; + break; + } + switch(t) { + case 3: + case 5: + case 9: + c = s + 1; + if(n) + c++; + q = 0; + break; + } + if(s) + break; + switch(t) { + case 15: + case 25: + case 27: + case 45: + case 81: + c = 2; + if(n) + c++; + q = 1; + break; + } + break; + case 1: + if(t == 1) { + c = 3; + q = 3; + break; + } + switch(t) { + case 3: + case 5: + case 9: + c = 3; + q = 2; + break; + } + break; + case 2: + if(t == 1) { + c = 3; + q = 2; + break; + } + break; + case 3: + if(s) + break; + if(t == 1) { + c = 3; + q = 1; + break; + } + break; + case 4: + if(t == 1) { + c = 3; + q = 0; + break; + } + break; + } + if(c < bc || (c == bc && q > bq)) { + bc = c; + bi = i; + bn = n; + bo = o; + bq = q; + bs = s; + bt = t; + } + } + } + mp->value = m; + if(bc <= 3) { + mp->alg = bi; + mp->shift = bn; + mp->off = bo; + mp->neg = bs; + mp->arg = bt; + } + else + mp->alg = -1; +} + +int +m0(int a) +{ + switch(a) { + case -2: + case 2: + return 2; + case -3: + case 3: + return 2; + case -4: + case 4: + return 4; + case -5: + case 5: + return 4; + case 6: + return 2; + case -8: + case 8: + return 8; + case -9: + case 9: + return 8; + case 10: + return 4; + case 12: + return 2; + case 15: + return 2; + case 18: + return 8; + case 20: + return 4; + case 24: + return 2; + case 25: + return 4; + case 27: + return 2; + case 36: + return 8; + case 40: + return 4; + case 45: + return 4; + case 72: + return 8; + case 81: + return 8; + } + diag(Z, "bad m0"); + return 0; +} + +int +m1(int a) +{ + switch(a) { + case 15: + return 4; + case 25: + return 4; + case 27: + return 8; + case 45: + return 8; + case 81: + return 8; + } + diag(Z, "bad m1"); + return 0; +} + +int +m2(int a) +{ + switch(a) { + case 6: + return 2; + case 10: + return 2; + case 12: + return 4; + case 18: + return 2; + case 20: + return 4; + case 24: + return 8; + case 36: + return 4; + case 40: + return 8; + case 72: + return 8; + } + diag(Z, "bad m2"); + return 0; +} + +void +shiftit(Type *t, Node *s, Node *d) +{ + int32 c; + + c = (int32)s->vconst & 31; + switch(c) { + case 0: + break; + case 1: + gopcode(OADD, t, d, d); + break; + default: + gopcode(OASHL, t, s, d); + } +} + +static int +mulgen1(uint32 v, Node *n) +{ + int i, o; + Mparam *p; + Node nod, nods; + + for(i = 0; i < nelem(multab); i++) { + p = &multab[i]; + if(p->value == v) + goto found; + } + + p = &multab[mulptr]; + if(++mulptr == nelem(multab)) + mulptr = 0; + + mulparam(v, p); + +found: +// print("v=%.x a=%d n=%d s=%d g=%d o=%d \n", p->value, p->alg, p->neg, p->shift, p->arg, p->off); + if(p->alg < 0) + return 0; + + nods = *nodconst(p->shift); + + o = OADD; + if(p->alg > 0) { + regalloc(&nod, n, Z); + if(p->off < 0) + o = OSUB; + } + + switch(p->alg) { + case 0: + switch(p->arg) { + case 1: + shiftit(n->type, &nods, n); + break; + case 15: + case 25: + case 27: + case 45: + case 81: + genmuladd(n, n, m1(p->arg), n); + /* fall thru */ + case 3: + case 5: + case 9: + genmuladd(n, n, m0(p->arg), n); + shiftit(n->type, &nods, n); + break; + default: + goto bad; + } + if(p->neg == 1) + gins(ANEGL, Z, n); + break; + case 1: + switch(p->arg) { + case 1: + gmove(n, &nod); + shiftit(n->type, &nods, &nod); + break; + case 3: + case 5: + case 9: + genmuladd(&nod, n, m0(p->arg), n); + shiftit(n->type, &nods, &nod); + break; + default: + goto bad; + } + if(p->neg) + gopcode(o, n->type, &nod, n); + else { + gopcode(o, n->type, n, &nod); + gmove(&nod, n); + } + break; + case 2: + genmuladd(&nod, n, m0(p->off), n); + shiftit(n->type, &nods, n); + goto comop; + case 3: + genmuladd(&nod, n, m0(p->off), n); + shiftit(n->type, &nods, n); + genmuladd(n, &nod, m2(p->off), n); + break; + case 4: + genmuladd(&nod, n, m0(p->off), nodconst(0)); + shiftit(n->type, &nods, n); + goto comop; + default: + diag(Z, "bad mul alg"); + break; + comop: + if(p->neg) { + gopcode(o, n->type, n, &nod); + gmove(&nod, n); + } + else + gopcode(o, n->type, &nod, n); + } + + if(p->alg > 0) + regfree(&nod); + + return 1; + +bad: + diag(Z, "mulgen botch"); + return 1; +} + +void +mulgen(Type *t, Node *r, Node *n) +{ + if(!mulgen1(r->vconst, n)) + gopcode(OMUL, t, r, n); +} diff --git a/src/cmd/8c/peep.c b/src/cmd/8c/peep.c new file mode 100644 index 000000000..9511a5579 --- /dev/null +++ b/src/cmd/8c/peep.c @@ -0,0 +1,801 @@ +// Inferno utils/8c/peep.c +// http://code.google.com/p/inferno-os/source/browse/utils/8c/peep.c +// +// Copyright © 1994-1999 Lucent Technologies Inc. All rights reserved. +// Portions Copyright © 1995-1997 C H Forsyth (forsyth@terzarima.net) +// Portions Copyright © 1997-1999 Vita Nuova Limited +// Portions Copyright © 2000-2007 Vita Nuova Holdings Limited (www.vitanuova.com) +// Portions Copyright © 2004,2006 Bruce Ellis +// Portions Copyright © 2005-2007 C H Forsyth (forsyth@terzarima.net) +// Revisions Copyright © 2000-2007 Lucent Technologies Inc. and others +// Portions Copyright © 2009 The Go Authors. All rights reserved. +// +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to deal +// in the Software without restriction, including without limitation the rights +// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +// copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +// THE SOFTWARE. + +#include "gc.h" + +static int +needc(Prog *p) +{ + while(p != P) { + switch(p->as) { + case AADCL: + case ASBBL: + case ARCRL: + return 1; + case AADDL: + case ASUBL: + case AJMP: + case ARET: + case ACALL: + return 0; + default: + if(p->to.type == D_BRANCH) + return 0; + } + p = p->link; + } + return 0; +} + +void +peep(void) +{ + Reg *r, *r1, *r2; + Prog *p, *p1; + int t; + + /* + * complete R structure + */ + t = 0; + for(r=firstr; r!=R; r=r1) { + r1 = r->link; + if(r1 == R) + break; + p = r->prog->link; + while(p != r1->prog) + switch(p->as) { + default: + r2 = rega(); + r->link = r2; + r2->link = r1; + + r2->prog = p; + r2->p1 = r; + r->s1 = r2; + r2->s1 = r1; + r1->p1 = r2; + + r = r2; + t++; + + case ADATA: + case AGLOBL: + case ANAME: + case ASIGNAME: + p = p->link; + } + } + + pc = 0; /* speculating it won't kill */ + +loop1: + + t = 0; + for(r=firstr; r!=R; r=r->link) { + p = r->prog; + switch(p->as) { + case AMOVL: + if(regtyp(&p->to)) + if(regtyp(&p->from)) { + if(copyprop(r)) { + excise(r); + t++; + } + if(subprop(r) && copyprop(r)) { + excise(r); + t++; + } + } + break; + + case AMOVBLSX: + case AMOVBLZX: + case AMOVWLSX: + case AMOVWLZX: + if(regtyp(&p->to)) { + r1 = uniqs(r); + if(r1 != R) { + p1 = r1->prog; + if(p->as == p1->as && p->to.type == p1->from.type) + p1->as = AMOVL; + } + } + break; + case AADDL: + case AADDW: + if(p->from.type != D_CONST || needc(p->link)) + break; + if(p->from.offset == -1){ + if(p->as == AADDL) + p->as = ADECL; + else + p->as = ADECW; + p->from = zprog.from; + } + else if(p->from.offset == 1){ + if(p->as == AADDL) + p->as = AINCL; + else + p->as = AINCW; + p->from = zprog.from; + } + break; + case ASUBL: + case ASUBW: + if(p->from.type != D_CONST || needc(p->link)) + break; + if(p->from.offset == -1) { + if(p->as == ASUBL) + p->as = AINCL; + else + p->as = AINCW; + p->from = zprog.from; + } + else if(p->from.offset == 1){ + if(p->as == ASUBL) + p->as = ADECL; + else + p->as = ADECW; + p->from = zprog.from; + } + break; + } + } + if(t) + goto loop1; +} + +void +excise(Reg *r) +{ + Prog *p; + + p = r->prog; + p->as = ANOP; + p->from = zprog.from; + p->to = zprog.to; +} + +Reg* +uniqp(Reg *r) +{ + Reg *r1; + + r1 = r->p1; + if(r1 == R) { + r1 = r->p2; + if(r1 == R || r1->p2link != R) + return R; + } else + if(r->p2 != R) + return R; + return r1; +} + +Reg* +uniqs(Reg *r) +{ + Reg *r1; + + r1 = r->s1; + if(r1 == R) { + r1 = r->s2; + if(r1 == R) + return R; + } else + if(r->s2 != R) + return R; + return r1; +} + +int +regtyp(Adr *a) +{ + int t; + + t = a->type; + if(t >= D_AX && t <= D_DI) + return 1; + return 0; +} + +/* + * the idea is to substitute + * one register for another + * from one MOV to another + * MOV a, R0 + * ADD b, R0 / no use of R1 + * MOV R0, R1 + * would be converted to + * MOV a, R1 + * ADD b, R1 + * MOV R1, R0 + * hopefully, then the former or latter MOV + * will be eliminated by copy propagation. + */ +int +subprop(Reg *r0) +{ + Prog *p; + Adr *v1, *v2; + Reg *r; + int t; + + p = r0->prog; + v1 = &p->from; + if(!regtyp(v1)) + return 0; + v2 = &p->to; + if(!regtyp(v2)) + return 0; + for(r=uniqp(r0); r!=R; r=uniqp(r)) { + if(uniqs(r) == R) + break; + p = r->prog; + switch(p->as) { + case ACALL: + return 0; + + case AIMULL: + case AIMULW: + if(p->to.type != D_NONE) + break; + + case ADIVB: + case ADIVL: + case ADIVW: + case AIDIVB: + case AIDIVL: + case AIDIVW: + case AIMULB: + case AMULB: + case AMULL: + case AMULW: + + case AROLB: + case AROLL: + case AROLW: + case ARORB: + case ARORL: + case ARORW: + case ASALB: + case ASALL: + case ASALW: + case ASARB: + case ASARL: + case ASARW: + case ASHLB: + case ASHLL: + case ASHLW: + case ASHRB: + case ASHRL: + case ASHRW: + + case AREP: + case AREPN: + + case ACWD: + case ACDQ: + + case ASTOSB: + case ASTOSL: + case AMOVSB: + case AMOVSL: + case AFSTSW: + return 0; + + case AMOVL: + if(p->to.type == v1->type) + goto gotit; + break; + } + if(copyau(&p->from, v2) || + copyau(&p->to, v2)) + break; + if(copysub(&p->from, v1, v2, 0) || + copysub(&p->to, v1, v2, 0)) + break; + } + return 0; + +gotit: + copysub(&p->to, v1, v2, 1); + if(debug['P']) { + print("gotit: %D->%D\n%P", v1, v2, r->prog); + if(p->from.type == v2->type) + print(" excise"); + print("\n"); + } + for(r=uniqs(r); r!=r0; r=uniqs(r)) { + p = r->prog; + copysub(&p->from, v1, v2, 1); + copysub(&p->to, v1, v2, 1); + if(debug['P']) + print("%P\n", r->prog); + } + t = v1->type; + v1->type = v2->type; + v2->type = t; + if(debug['P']) + print("%P last\n", r->prog); + return 1; +} + +/* + * The idea is to remove redundant copies. + * v1->v2 F=0 + * (use v2 s/v2/v1/)* + * set v1 F=1 + * use v2 return fail + * ----------------- + * v1->v2 F=0 + * (use v2 s/v2/v1/)* + * set v1 F=1 + * set v2 return success + */ +int +copyprop(Reg *r0) +{ + Prog *p; + Adr *v1, *v2; + Reg *r; + + p = r0->prog; + v1 = &p->from; + v2 = &p->to; + if(copyas(v1, v2)) + return 1; + for(r=firstr; r!=R; r=r->link) + r->active = 0; + return copy1(v1, v2, r0->s1, 0); +} + +int +copy1(Adr *v1, Adr *v2, Reg *r, int f) +{ + int t; + Prog *p; + + if(r->active) { + if(debug['P']) + print("act set; return 1\n"); + return 1; + } + r->active = 1; + if(debug['P']) + print("copy %D->%D f=%d\n", v1, v2, f); + for(; r != R; r = r->s1) { + p = r->prog; + if(debug['P']) + print("%P", p); + if(!f && uniqp(r) == R) { + f = 1; + if(debug['P']) + print("; merge; f=%d", f); + } + t = copyu(p, v2, A); + switch(t) { + case 2: /* rar, cant split */ + if(debug['P']) + print("; %D rar; return 0\n", v2); + return 0; + + case 3: /* set */ + if(debug['P']) + print("; %D set; return 1\n", v2); + return 1; + + case 1: /* used, substitute */ + case 4: /* use and set */ + if(f) { + if(!debug['P']) + return 0; + if(t == 4) + print("; %D used+set and f=%d; return 0\n", v2, f); + else + print("; %D used and f=%d; return 0\n", v2, f); + return 0; + } + if(copyu(p, v2, v1)) { + if(debug['P']) + print("; sub fail; return 0\n"); + return 0; + } + if(debug['P']) + print("; sub %D/%D", v2, v1); + if(t == 4) { + if(debug['P']) + print("; %D used+set; return 1\n", v2); + return 1; + } + break; + } + if(!f) { + t = copyu(p, v1, A); + if(!f && (t == 2 || t == 3 || t == 4)) { + f = 1; + if(debug['P']) + print("; %D set and !f; f=%d", v1, f); + } + } + if(debug['P']) + print("\n"); + if(r->s2) + if(!copy1(v1, v2, r->s2, f)) + return 0; + } + return 1; +} + +/* + * return + * 1 if v only used (and substitute), + * 2 if read-alter-rewrite + * 3 if set + * 4 if set and used + * 0 otherwise (not touched) + */ +int +copyu(Prog *p, Adr *v, Adr *s) +{ + + switch(p->as) { + + default: + if(debug['P']) + print("unknown op %A\n", p->as); + return 2; + + case ANEGB: + case ANEGW: + case ANEGL: + case ANOTB: + case ANOTW: + case ANOTL: + if(copyas(&p->to, v)) + return 2; + break; + + case ALEAL: /* lhs addr, rhs store */ + if(copyas(&p->from, v)) + return 2; + + + case ANOP: /* rhs store */ + case AMOVL: + case AMOVBLSX: + case AMOVBLZX: + case AMOVWLSX: + case AMOVWLZX: + if(copyas(&p->to, v)) { + if(s != A) + return copysub(&p->from, v, s, 1); + if(copyau(&p->from, v)) + return 4; + return 3; + } + goto caseread; + + case AROLB: + case AROLL: + case AROLW: + case ARORB: + case ARORL: + case ARORW: + case ASALB: + case ASALL: + case ASALW: + case ASARB: + case ASARL: + case ASARW: + case ASHLB: + case ASHLL: + case ASHLW: + case ASHRB: + case ASHRL: + case ASHRW: + if(copyas(&p->to, v)) + return 2; + if(copyas(&p->from, v)) + if(p->from.type == D_CX) + return 2; + goto caseread; + + case AADDB: /* rhs rar */ + case AADDL: + case AADDW: + case AANDB: + case AANDL: + case AANDW: + case ADECL: + case ADECW: + case AINCL: + case AINCW: + case ASUBB: + case ASUBL: + case ASUBW: + case AORB: + case AORL: + case AORW: + case AXORB: + case AXORL: + case AXORW: + case AMOVB: + case AMOVW: + + case AFMOVB: + case AFMOVBP: + case AFMOVD: + case AFMOVDP: + case AFMOVF: + case AFMOVFP: + case AFMOVL: + case AFMOVLP: + case AFMOVV: + case AFMOVVP: + case AFMOVW: + case AFMOVWP: + case AFMOVX: + case AFMOVXP: + case AFADDDP: + case AFADDW: + case AFADDL: + case AFADDF: + case AFADDD: + case AFMULDP: + case AFMULW: + case AFMULL: + case AFMULF: + case AFMULD: + case AFSUBDP: + case AFSUBW: + case AFSUBL: + case AFSUBF: + case AFSUBD: + case AFSUBRDP: + case AFSUBRW: + case AFSUBRL: + case AFSUBRF: + case AFSUBRD: + case AFDIVDP: + case AFDIVW: + case AFDIVL: + case AFDIVF: + case AFDIVD: + case AFDIVRDP: + case AFDIVRW: + case AFDIVRL: + case AFDIVRF: + case AFDIVRD: + if(copyas(&p->to, v)) + return 2; + goto caseread; + + case ACMPL: /* read only */ + case ACMPW: + case ACMPB: + + case AFCOMB: + case AFCOMBP: + case AFCOMD: + case AFCOMDP: + case AFCOMDPP: + case AFCOMF: + case AFCOMFP: + case AFCOML: + case AFCOMLP: + case AFCOMW: + case AFCOMWP: + case AFUCOM: + case AFUCOMP: + case AFUCOMPP: + caseread: + if(s != A) { + if(copysub(&p->from, v, s, 1)) + return 1; + return copysub(&p->to, v, s, 1); + } + if(copyau(&p->from, v)) + return 1; + if(copyau(&p->to, v)) + return 1; + break; + + case AJGE: /* no reference */ + case AJNE: + case AJLE: + case AJEQ: + case AJHI: + case AJLS: + case AJMI: + case AJPL: + case AJGT: + case AJLT: + case AJCC: + case AJCS: + + case AADJSP: + case AFLDZ: + case AWAIT: + break; + + case AIMULL: + case AIMULW: + if(p->to.type != D_NONE) { + if(copyas(&p->to, v)) + return 2; + goto caseread; + } + + case ADIVB: + case ADIVL: + case ADIVW: + case AIDIVB: + case AIDIVL: + case AIDIVW: + case AIMULB: + case AMULB: + case AMULL: + case AMULW: + + case ACWD: + case ACDQ: + if(v->type == D_AX || v->type == D_DX) + return 2; + goto caseread; + + case AREP: + case AREPN: + if(v->type == D_CX) + return 2; + goto caseread; + + case AMOVSB: + case AMOVSL: + if(v->type == D_DI || v->type == D_SI) + return 2; + goto caseread; + + case ASTOSB: + case ASTOSL: + if(v->type == D_AX || v->type == D_DI) + return 2; + goto caseread; + + case AFSTSW: + if(v->type == D_AX) + return 2; + goto caseread; + + case AJMP: /* funny */ + if(s != A) { + if(copysub(&p->to, v, s, 1)) + return 1; + return 0; + } + if(copyau(&p->to, v)) + return 1; + return 0; + + case ARET: /* funny */ + if(v->type == REGRET) + return 2; + if(s != A) + return 1; + return 3; + + case ACALL: /* funny */ + if(REGARG >= 0 && v->type == (uchar)REGARG) + return 2; + + if(s != A) { + if(copysub(&p->to, v, s, 1)) + return 1; + return 0; + } + if(copyau(&p->to, v)) + return 4; + return 3; + } + return 0; +} + +/* + * direct reference, + * could be set/use depending on + * semantics + */ +int +copyas(Adr *a, Adr *v) +{ + if(a->type != v->type) + return 0; + if(regtyp(v)) + return 1; + if(v->type == D_AUTO || v->type == D_PARAM) + if(v->offset == a->offset) + return 1; + return 0; +} + +/* + * either direct or indirect + */ +int +copyau(Adr *a, Adr *v) +{ + + if(copyas(a, v)) + return 1; + if(regtyp(v)) { + if(a->type-D_INDIR == v->type) + return 1; + if(a->index == v->type) + return 1; + } + return 0; +} + +/* + * substitute s for v in a + * return failure to substitute + */ +int +copysub(Adr *a, Adr *v, Adr *s, int f) +{ + int t; + + if(copyas(a, v)) { + t = s->type; + if(t >= D_AX && t <= D_DI) { + if(f) + a->type = t; + } + return 0; + } + if(regtyp(v)) { + t = v->type; + if(a->type == t+D_INDIR) { + if(s->type == D_BP && a->index != D_NONE) + return 1; /* can't use BP-base with index */ + if(f) + a->type = s->type+D_INDIR; +// return 0; + } + if(a->index == t) { + if(f) + a->index = s->type; + return 0; + } + return 0; + } + return 0; +} diff --git a/src/cmd/8c/reg.c b/src/cmd/8c/reg.c new file mode 100644 index 000000000..6ba07bed2 --- /dev/null +++ b/src/cmd/8c/reg.c @@ -0,0 +1,1287 @@ +// Inferno utils/8c/reg.c +// http://code.google.com/p/inferno-os/source/browse/utils/8c/reg.c +// +// Copyright © 1994-1999 Lucent Technologies Inc. All rights reserved. +// Portions Copyright © 1995-1997 C H Forsyth (forsyth@terzarima.net) +// Portions Copyright © 1997-1999 Vita Nuova Limited +// Portions Copyright © 2000-2007 Vita Nuova Holdings Limited (www.vitanuova.com) +// Portions Copyright © 2004,2006 Bruce Ellis +// Portions Copyright © 2005-2007 C H Forsyth (forsyth@terzarima.net) +// Revisions Copyright © 2000-2007 Lucent Technologies Inc. and others +// Portions Copyright © 2009 The Go Authors. All rights reserved. +// +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to deal +// in the Software without restriction, including without limitation the rights +// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +// copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +// THE SOFTWARE. + +#include "gc.h" + +Reg* +rega(void) +{ + Reg *r; + + r = freer; + if(r == R) { + r = alloc(sizeof(*r)); + } else + freer = r->link; + + *r = zreg; + return r; +} + +int +rcmp(const void *a1, const void *a2) +{ + Rgn *p1, *p2; + int c1, c2; + + p1 = (Rgn*)a1; + p2 = (Rgn*)a2; + c1 = p2->cost; + c2 = p1->cost; + if(c1 -= c2) + return c1; + return p2->varno - p1->varno; +} + +void +regopt(Prog *p) +{ + Reg *r, *r1, *r2; + Prog *p1; + int i, z; + int32 initpc, val, npc; + uint32 vreg; + Bits bit; + struct + { + int32 m; + int32 c; + Reg* p; + } log5[6], *lp; + + firstr = R; + lastr = R; + nvar = 0; + regbits = RtoB(D_SP) | RtoB(D_AX); + for(z=0; zm = val; + lp->c = 0; + lp->p = R; + val /= 5L; + lp++; + } + val = 0; + for(; p != P; p = p->link) { + switch(p->as) { + case ADATA: + case AGLOBL: + case ANAME: + case ASIGNAME: + continue; + } + r = rega(); + if(firstr == R) { + firstr = r; + lastr = r; + } else { + lastr->link = r; + r->p1 = lastr; + lastr->s1 = r; + lastr = r; + } + r->prog = p; + r->pc = val; + val++; + + lp = log5; + for(i=0; i<5; i++) { + lp->c--; + if(lp->c <= 0) { + lp->c = lp->m; + if(lp->p != R) + lp->p->log5 = r; + lp->p = r; + (lp+1)->c = 0; + break; + } + lp++; + } + + r1 = r->p1; + if(r1 != R) + switch(r1->prog->as) { + case ARET: + case AJMP: + case AIRETL: + r->p1 = R; + r1->s1 = R; + } + + bit = mkvar(r, &p->from); + if(bany(&bit)) + switch(p->as) { + /* + * funny + */ + case ALEAL: + for(z=0; zuse1.b[z] |= bit.b[z]; + break; + } + + bit = mkvar(r, &p->to); + if(bany(&bit)) + switch(p->as) { + default: + diag(Z, "reg: unknown op: %A", p->as); + break; + + /* + * right side read + */ + case ACMPB: + case ACMPL: + case ACMPW: + for(z=0; zuse2.b[z] |= bit.b[z]; + break; + + /* + * right side write + */ + case ANOP: + case AMOVL: + case AMOVB: + case AMOVW: + case AMOVBLSX: + case AMOVBLZX: + case AMOVWLSX: + case AMOVWLZX: + for(z=0; zset.b[z] |= bit.b[z]; + break; + + /* + * right side read+write + */ + case AADDB: + case AADDL: + case AADDW: + case AANDB: + case AANDL: + case AANDW: + case ASUBB: + case ASUBL: + case ASUBW: + case AORB: + case AORL: + case AORW: + case AXORB: + case AXORL: + case AXORW: + case ASALB: + case ASALL: + case ASALW: + case ASARB: + case ASARL: + case ASARW: + case AROLB: + case AROLL: + case AROLW: + case ARORB: + case ARORL: + case ARORW: + case ASHLB: + case ASHLL: + case ASHLW: + case ASHRB: + case ASHRL: + case ASHRW: + case AIMULL: + case AIMULW: + case ANEGL: + case ANOTL: + case AADCL: + case ASBBL: + for(z=0; zset.b[z] |= bit.b[z]; + r->use2.b[z] |= bit.b[z]; + } + break; + + /* + * funny + */ + case AFMOVDP: + case AFMOVFP: + case AFMOVLP: + case AFMOVVP: + case AFMOVWP: + case ACALL: + for(z=0; zas) { + case AIMULL: + case AIMULW: + if(p->to.type != D_NONE) + break; + + case AIDIVB: + case AIDIVL: + case AIDIVW: + case AIMULB: + case ADIVB: + case ADIVL: + case ADIVW: + case AMULB: + case AMULL: + case AMULW: + + case ACWD: + case ACDQ: + r->regu |= RtoB(D_AX) | RtoB(D_DX); + break; + + case AREP: + case AREPN: + case ALOOP: + case ALOOPEQ: + case ALOOPNE: + r->regu |= RtoB(D_CX); + break; + + case AMOVSB: + case AMOVSL: + case AMOVSW: + case ACMPSB: + case ACMPSL: + case ACMPSW: + r->regu |= RtoB(D_SI) | RtoB(D_DI); + break; + + case ASTOSB: + case ASTOSL: + case ASTOSW: + case ASCASB: + case ASCASL: + case ASCASW: + r->regu |= RtoB(D_AX) | RtoB(D_DI); + break; + + case AINSB: + case AINSL: + case AINSW: + case AOUTSB: + case AOUTSL: + case AOUTSW: + r->regu |= RtoB(D_DI) | RtoB(D_DX); + break; + + case AFSTSW: + case ASAHF: + r->regu |= RtoB(D_AX); + break; + } + } + if(firstr == R) + return; + initpc = pc - val; + npc = val; + + /* + * pass 2 + * turn branch references to pointers + * build back pointers + */ + for(r = firstr; r != R; r = r->link) { + p = r->prog; + if(p->to.type == D_BRANCH) { + val = p->to.offset - initpc; + r1 = firstr; + while(r1 != R) { + r2 = r1->log5; + if(r2 != R && val >= r2->pc) { + r1 = r2; + continue; + } + if(r1->pc == val) + break; + r1 = r1->link; + } + if(r1 == R) { + nearln = p->lineno; + diag(Z, "ref not found\n%P", p); + continue; + } + if(r1 == r) { + nearln = p->lineno; + diag(Z, "ref to self\n%P", p); + continue; + } + r->s2 = r1; + r->p2link = r1->p2; + r1->p2 = r; + } + } + if(debug['R']) { + p = firstr->prog; + print("\n%L %D\n", p->lineno, &p->from); + } + + /* + * pass 2.5 + * find looping structure + */ + for(r = firstr; r != R; r = r->link) + r->active = 0; + change = 0; + loopit(firstr, npc); + if(debug['R'] && debug['v']) { + print("\nlooping structure:\n"); + for(r = firstr; r != R; r = r->link) { + print("%d:%P", r->loop, r->prog); + for(z=0; zuse1.b[z] | + r->use2.b[z] | + r->set.b[z]; + if(bany(&bit)) { + print("\t"); + if(bany(&r->use1)) + print(" u1=%B", r->use1); + if(bany(&r->use2)) + print(" u2=%B", r->use2); + if(bany(&r->set)) + print(" st=%B", r->set); + } + print("\n"); + } + } + + /* + * pass 3 + * iterate propagating usage + * back until flow graph is complete + */ +loop1: + change = 0; + for(r = firstr; r != R; r = r->link) + r->active = 0; + for(r = firstr; r != R; r = r->link) + if(r->prog->as == ARET) + prop(r, zbits, zbits); +loop11: + /* pick up unreachable code */ + i = 0; + for(r = firstr; r != R; r = r1) { + r1 = r->link; + if(r1 && r1->active && !r->active) { + prop(r, zbits, zbits); + i = 1; + } + } + if(i) + goto loop11; + if(change) + goto loop1; + + + /* + * pass 4 + * iterate propagating register/variable synchrony + * forward until graph is complete + */ +loop2: + change = 0; + for(r = firstr; r != R; r = r->link) + r->active = 0; + synch(firstr, zbits); + if(change) + goto loop2; + + + /* + * pass 5 + * isolate regions + * calculate costs (paint1) + */ + r = firstr; + if(r) { + for(z=0; zrefahead.b[z] | r->calahead.b[z]) & + ~(externs.b[z] | params.b[z] | addrs.b[z] | consts.b[z]); + if(bany(&bit)) { + nearln = r->prog->lineno; + warn(Z, "used and not set: %B", bit); + if(debug['R'] && !debug['w']) + print("used and not set: %B\n", bit); + } + } + if(debug['R'] && debug['v']) + print("\nprop structure:\n"); + for(r = firstr; r != R; r = r->link) + r->act = zbits; + rgp = region; + nregion = 0; + for(r = firstr; r != R; r = r->link) { + if(debug['R'] && debug['v']) { + print("%P\t", r->prog); + if(bany(&r->set)) + print("s:%B ", r->set); + if(bany(&r->refahead)) + print("ra:%B ", r->refahead); + if(bany(&r->calahead)) + print("ca:%B ", r->calahead); + print("\n"); + } + for(z=0; zset.b[z] & + ~(r->refahead.b[z] | r->calahead.b[z] | addrs.b[z]); + if(bany(&bit)) { + nearln = r->prog->lineno; + warn(Z, "set and not used: %B", bit); + if(debug['R']) + print("set and not used: %B\n", bit); + excise(r); + } + for(z=0; zact.b[z] | addrs.b[z]); + while(bany(&bit)) { + i = bnum(bit); + rgp->enter = r; + rgp->varno = i; + change = 0; + if(debug['R'] && debug['v']) + print("\n"); + paint1(r, i); + bit.b[i/32] &= ~(1L<<(i%32)); + if(change <= 0) { + if(debug['R']) + print("%L$%d: %B\n", + r->prog->lineno, change, blsh(i)); + continue; + } + rgp->cost = change; + nregion++; + if(nregion >= NRGN) { + warn(Z, "too many regions"); + goto brk; + } + rgp++; + } + } +brk: + qsort(region, nregion, sizeof(region[0]), rcmp); + + /* + * pass 6 + * determine used registers (paint2) + * replace code (paint3) + */ + rgp = region; + for(i=0; ivarno); + vreg = paint2(rgp->enter, rgp->varno); + vreg = allreg(vreg, rgp); + if(debug['R']) { + print("%L$%d %R: %B\n", + rgp->enter->prog->lineno, + rgp->cost, + rgp->regno, + bit); + } + if(rgp->regno != 0) + paint3(rgp->enter, rgp->varno, vreg, rgp->regno); + rgp++; + } + /* + * pass 7 + * peep-hole on basic block + */ + if(!debug['R'] || debug['P']) + peep(); + + /* + * pass 8 + * recalculate pc + */ + val = initpc; + for(r = firstr; r != R; r = r1) { + r->pc = val; + p = r->prog; + p1 = P; + r1 = r->link; + if(r1 != R) + p1 = r1->prog; + for(; p != p1; p = p->link) { + switch(p->as) { + default: + val++; + break; + + case ANOP: + case ADATA: + case AGLOBL: + case ANAME: + case ASIGNAME: + break; + } + } + } + pc = val; + + /* + * fix up branches + */ + if(debug['R']) + if(bany(&addrs)) + print("addrs: %B\n", addrs); + + r1 = 0; /* set */ + for(r = firstr; r != R; r = r->link) { + p = r->prog; + if(p->to.type == D_BRANCH) + p->to.offset = r->s2->pc; + r1 = r; + } + + /* + * last pass + * eliminate nops + * free aux structures + */ + for(p = firstr->prog; p != P; p = p->link){ + while(p->link && p->link->as == ANOP) + p->link = p->link->link; + } + if(r1 != R) { + r1->link = freer; + freer = firstr; + } +} + +/* + * add mov b,rn + * just after r + */ +void +addmove(Reg *r, int bn, int rn, int f) +{ + Prog *p, *p1; + Adr *a; + Var *v; + + p1 = alloc(sizeof(*p1)); + *p1 = zprog; + p = r->prog; + + p1->link = p->link; + p->link = p1; + p1->lineno = p->lineno; + + v = var + bn; + + a = &p1->to; + a->sym = v->sym; + a->offset = v->offset; + a->etype = v->etype; + a->type = v->name; + + p1->as = AMOVL; + if(v->etype == TCHAR || v->etype == TUCHAR) + p1->as = AMOVB; + if(v->etype == TSHORT || v->etype == TUSHORT) + p1->as = AMOVW; + + p1->from.type = rn; + if(!f) { + p1->from = *a; + *a = zprog.from; + a->type = rn; + if(v->etype == TUCHAR) + p1->as = AMOVB; + if(v->etype == TUSHORT) + p1->as = AMOVW; + } + if(debug['R']) + print("%P\t.a%P\n", p, p1); +} + +uint32 +doregbits(int r) +{ + uint32 b; + + b = 0; + if(r >= D_INDIR) + r -= D_INDIR; + if(r >= D_AX && r <= D_DI) + b |= RtoB(r); + else + if(r >= D_AL && r <= D_BL) + b |= RtoB(r-D_AL+D_AX); + else + if(r >= D_AH && r <= D_BH) + b |= RtoB(r-D_AH+D_AX); + return b; +} + +Bits +mkvar(Reg *r, Adr *a) +{ + Var *v; + int i, t, n, et, z; + int32 o; + Bits bit; + Sym *s; + + /* + * mark registers used + */ + t = a->type; + r->regu |= doregbits(t); + r->regu |= doregbits(a->index); + + switch(t) { + default: + goto none; + case D_ADDR: + a->type = a->index; + bit = mkvar(r, a); + for(z=0; ztype = t; + goto none; + case D_EXTERN: + case D_STATIC: + case D_PARAM: + case D_AUTO: + n = t; + break; + } + s = a->sym; + if(s == S) + goto none; + if(s->name[0] == '.') + goto none; + et = a->etype; + o = a->offset; + v = var; + for(i=0; isym) + if(n == v->name) + if(o == v->offset) + goto out; + v++; + } + if(nvar >= NVAR) { + if(debug['w'] > 1 && s) + warn(Z, "variable not optimized: %s", s->name); + goto none; + } + i = nvar; + nvar++; + v = &var[i]; + v->sym = s; + v->offset = o; + v->name = n; + v->etype = et; + if(debug['R']) + print("bit=%2d et=%2d %D\n", i, et, a); + +out: + bit = blsh(i); + if(n == D_EXTERN || n == D_STATIC) + for(z=0; zetype != et || !typechlpfd[et]) /* funny punning */ + for(z=0; zp1) { + for(z=0; zrefahead.b[z]; + if(ref.b[z] != r1->refahead.b[z]) { + r1->refahead.b[z] = ref.b[z]; + change++; + } + cal.b[z] |= r1->calahead.b[z]; + if(cal.b[z] != r1->calahead.b[z]) { + r1->calahead.b[z] = cal.b[z]; + change++; + } + } + switch(r1->prog->as) { + case ACALL: + for(z=0; zset.b[z]) | + r1->use1.b[z] | r1->use2.b[z]; + cal.b[z] &= ~(r1->set.b[z] | r1->use1.b[z] | r1->use2.b[z]); + r1->refbehind.b[z] = ref.b[z]; + r1->calbehind.b[z] = cal.b[z]; + } + if(r1->active) + break; + r1->active = 1; + } + for(; r != r1; r = r->p1) + for(r2 = r->p2; r2 != R; r2 = r2->p2link) + prop(r2, r->refbehind, r->calbehind); +} + +/* + * find looping structure + * + * 1) find reverse postordering + * 2) find approximate dominators, + * the actual dominators if the flow graph is reducible + * otherwise, dominators plus some other non-dominators. + * See Matthew S. Hecht and Jeffrey D. Ullman, + * "Analysis of a Simple Algorithm for Global Data Flow Problems", + * Conf. Record of ACM Symp. on Principles of Prog. Langs, Boston, Massachusetts, + * Oct. 1-3, 1973, pp. 207-217. + * 3) find all nodes with a predecessor dominated by the current node. + * such a node is a loop head. + * recursively, all preds with a greater rpo number are in the loop + */ +int32 +postorder(Reg *r, Reg **rpo2r, int32 n) +{ + Reg *r1; + + r->rpo = 1; + r1 = r->s1; + if(r1 && !r1->rpo) + n = postorder(r1, rpo2r, n); + r1 = r->s2; + if(r1 && !r1->rpo) + n = postorder(r1, rpo2r, n); + rpo2r[n] = r; + n++; + return n; +} + +int32 +rpolca(int32 *idom, int32 rpo1, int32 rpo2) +{ + int32 t; + + if(rpo1 == -1) + return rpo2; + while(rpo1 != rpo2){ + if(rpo1 > rpo2){ + t = rpo2; + rpo2 = rpo1; + rpo1 = t; + } + while(rpo1 < rpo2){ + t = idom[rpo2]; + if(t >= rpo2) + fatal(Z, "bad idom"); + rpo2 = t; + } + } + return rpo1; +} + +int +doms(int32 *idom, int32 r, int32 s) +{ + while(s > r) + s = idom[s]; + return s == r; +} + +int +loophead(int32 *idom, Reg *r) +{ + int32 src; + + src = r->rpo; + if(r->p1 != R && doms(idom, src, r->p1->rpo)) + return 1; + for(r = r->p2; r != R; r = r->p2link) + if(doms(idom, src, r->rpo)) + return 1; + return 0; +} + +void +loopmark(Reg **rpo2r, int32 head, Reg *r) +{ + if(r->rpo < head || r->active == head) + return; + r->active = head; + r->loop += LOOP; + if(r->p1 != R) + loopmark(rpo2r, head, r->p1); + for(r = r->p2; r != R; r = r->p2link) + loopmark(rpo2r, head, r); +} + +void +loopit(Reg *r, int32 nr) +{ + Reg *r1; + int32 i, d, me; + + if(nr > maxnr) { + rpo2r = alloc(nr * sizeof(Reg*)); + idom = alloc(nr * sizeof(int32)); + maxnr = nr; + } + + d = postorder(r, rpo2r, 0); + if(d > nr) + fatal(Z, "too many reg nodes"); + nr = d; + for(i = 0; i < nr / 2; i++){ + r1 = rpo2r[i]; + rpo2r[i] = rpo2r[nr - 1 - i]; + rpo2r[nr - 1 - i] = r1; + } + for(i = 0; i < nr; i++) + rpo2r[i]->rpo = i; + + idom[0] = 0; + for(i = 0; i < nr; i++){ + r1 = rpo2r[i]; + me = r1->rpo; + d = -1; + if(r1->p1 != R && r1->p1->rpo < me) + d = r1->p1->rpo; + for(r1 = r1->p2; r1 != nil; r1 = r1->p2link) + if(r1->rpo < me) + d = rpolca(idom, d, r1->rpo); + idom[i] = d; + } + + for(i = 0; i < nr; i++){ + r1 = rpo2r[i]; + r1->loop++; + if(r1->p2 != R && loophead(idom, r1)) + loopmark(rpo2r, i, r1); + } +} + +void +synch(Reg *r, Bits dif) +{ + Reg *r1; + int z; + + for(r1 = r; r1 != R; r1 = r1->s1) { + for(z=0; zrefbehind.b[z] & r1->refahead.b[z])) | + r1->set.b[z] | r1->regdiff.b[z]; + if(dif.b[z] != r1->regdiff.b[z]) { + r1->regdiff.b[z] = dif.b[z]; + change++; + } + } + if(r1->active) + break; + r1->active = 1; + for(z=0; zcalbehind.b[z] & r1->calahead.b[z]); + if(r1->s2 != R) + synch(r1->s2, dif); + } +} + +uint32 +allreg(uint32 b, Rgn *r) +{ + Var *v; + int i; + + v = var + r->varno; + r->regno = 0; + switch(v->etype) { + + default: + diag(Z, "unknown etype %d/%d", bitno(b), v->etype); + break; + + case TCHAR: + case TUCHAR: + case TSHORT: + case TUSHORT: + case TINT: + case TUINT: + case TLONG: + case TULONG: + case TIND: + case TARRAY: + i = BtoR(~b); + if(i && r->cost > 0) { + r->regno = i; + return RtoB(i); + } + break; + + case TDOUBLE: + case TFLOAT: + break; + } + return 0; +} + +void +paint1(Reg *r, int bn) +{ + Reg *r1; + Prog *p; + int z; + uint32 bb; + + z = bn/32; + bb = 1L<<(bn%32); + if(r->act.b[z] & bb) + return; + for(;;) { + if(!(r->refbehind.b[z] & bb)) + break; + r1 = r->p1; + if(r1 == R) + break; + if(!(r1->refahead.b[z] & bb)) + break; + if(r1->act.b[z] & bb) + break; + r = r1; + } + + if(LOAD(r) & ~(r->set.b[z]&~(r->use1.b[z]|r->use2.b[z])) & bb) { + change -= CLOAD * r->loop; + if(debug['R'] && debug['v']) + print("%d%P\td %B $%d\n", r->loop, + r->prog, blsh(bn), change); + } + for(;;) { + r->act.b[z] |= bb; + p = r->prog; + + if(r->use1.b[z] & bb) { + change += CREF * r->loop; + if(p->as == AFMOVL) + if(BtoR(bb) != D_F0) + change = -CINF; + if(debug['R'] && debug['v']) + print("%d%P\tu1 %B $%d\n", r->loop, + p, blsh(bn), change); + } + + if((r->use2.b[z]|r->set.b[z]) & bb) { + change += CREF * r->loop; + if(p->as == AFMOVL) + if(BtoR(bb) != D_F0) + change = -CINF; + if(debug['R'] && debug['v']) + print("%d%P\tu2 %B $%d\n", r->loop, + p, blsh(bn), change); + } + + if(STORE(r) & r->regdiff.b[z] & bb) { + change -= CLOAD * r->loop; + if(p->as == AFMOVL) + if(BtoR(bb) != D_F0) + change = -CINF; + if(debug['R'] && debug['v']) + print("%d%P\tst %B $%d\n", r->loop, + p, blsh(bn), change); + } + + if(r->refbehind.b[z] & bb) + for(r1 = r->p2; r1 != R; r1 = r1->p2link) + if(r1->refahead.b[z] & bb) + paint1(r1, bn); + + if(!(r->refahead.b[z] & bb)) + break; + r1 = r->s2; + if(r1 != R) + if(r1->refbehind.b[z] & bb) + paint1(r1, bn); + r = r->s1; + if(r == R) + break; + if(r->act.b[z] & bb) + break; + if(!(r->refbehind.b[z] & bb)) + break; + } +} + +uint32 +regset(Reg *r, uint32 bb) +{ + uint32 b, set; + Adr v; + int c; + + set = 0; + v = zprog.from; + while(b = bb & ~(bb-1)) { + v.type = BtoR(b); + c = copyu(r->prog, &v, A); + if(c == 3) + set |= b; + bb &= ~b; + } + return set; +} + +uint32 +reguse(Reg *r, uint32 bb) +{ + uint32 b, set; + Adr v; + int c; + + set = 0; + v = zprog.from; + while(b = bb & ~(bb-1)) { + v.type = BtoR(b); + c = copyu(r->prog, &v, A); + if(c == 1 || c == 2 || c == 4) + set |= b; + bb &= ~b; + } + return set; +} + +uint32 +paint2(Reg *r, int bn) +{ + Reg *r1; + int z; + uint32 bb, vreg, x; + + z = bn/32; + bb = 1L << (bn%32); + vreg = regbits; + if(!(r->act.b[z] & bb)) + return vreg; + for(;;) { + if(!(r->refbehind.b[z] & bb)) + break; + r1 = r->p1; + if(r1 == R) + break; + if(!(r1->refahead.b[z] & bb)) + break; + if(!(r1->act.b[z] & bb)) + break; + r = r1; + } + for(;;) { + r->act.b[z] &= ~bb; + + vreg |= r->regu; + + if(r->refbehind.b[z] & bb) + for(r1 = r->p2; r1 != R; r1 = r1->p2link) + if(r1->refahead.b[z] & bb) + vreg |= paint2(r1, bn); + + if(!(r->refahead.b[z] & bb)) + break; + r1 = r->s2; + if(r1 != R) + if(r1->refbehind.b[z] & bb) + vreg |= paint2(r1, bn); + r = r->s1; + if(r == R) + break; + if(!(r->act.b[z] & bb)) + break; + if(!(r->refbehind.b[z] & bb)) + break; + } + + bb = vreg; + for(; r; r=r->s1) { + x = r->regu & ~bb; + if(x) { + vreg |= reguse(r, x); + bb |= regset(r, x); + } + } + return vreg; +} + +void +paint3(Reg *r, int bn, int32 rb, int rn) +{ + Reg *r1; + Prog *p; + int z; + uint32 bb; + + z = bn/32; + bb = 1L << (bn%32); + if(r->act.b[z] & bb) + return; + for(;;) { + if(!(r->refbehind.b[z] & bb)) + break; + r1 = r->p1; + if(r1 == R) + break; + if(!(r1->refahead.b[z] & bb)) + break; + if(r1->act.b[z] & bb) + break; + r = r1; + } + + if(LOAD(r) & ~(r->set.b[z] & ~(r->use1.b[z]|r->use2.b[z])) & bb) + addmove(r, bn, rn, 0); + for(;;) { + r->act.b[z] |= bb; + p = r->prog; + + if(r->use1.b[z] & bb) { + if(debug['R']) + print("%P", p); + addreg(&p->from, rn); + if(debug['R']) + print("\t.c%P\n", p); + } + if((r->use2.b[z]|r->set.b[z]) & bb) { + if(debug['R']) + print("%P", p); + addreg(&p->to, rn); + if(debug['R']) + print("\t.c%P\n", p); + } + + if(STORE(r) & r->regdiff.b[z] & bb) + addmove(r, bn, rn, 1); + r->regu |= rb; + + if(r->refbehind.b[z] & bb) + for(r1 = r->p2; r1 != R; r1 = r1->p2link) + if(r1->refahead.b[z] & bb) + paint3(r1, bn, rb, rn); + + if(!(r->refahead.b[z] & bb)) + break; + r1 = r->s2; + if(r1 != R) + if(r1->refbehind.b[z] & bb) + paint3(r1, bn, rb, rn); + r = r->s1; + if(r == R) + break; + if(r->act.b[z] & bb) + break; + if(!(r->refbehind.b[z] & bb)) + break; + } +} + +void +addreg(Adr *a, int rn) +{ + + a->sym = 0; + a->offset = 0; + a->type = rn; +} + +int32 +RtoB(int r) +{ + + if(r < D_AX || r > D_DI) + return 0; + return 1L << (r-D_AX); +} + +int +BtoR(int32 b) +{ + + b &= 0xffL; + if(b == 0) + return 0; + return bitno(b) + D_AX; +} diff --git a/src/cmd/8c/sgen.c b/src/cmd/8c/sgen.c new file mode 100644 index 000000000..b0f2bc544 --- /dev/null +++ b/src/cmd/8c/sgen.c @@ -0,0 +1,485 @@ +// Inferno utils/8c/sgen.c +// http://code.google.com/p/inferno-os/source/browse/utils/8c/sgen.c +// +// Copyright © 1994-1999 Lucent Technologies Inc. All rights reserved. +// Portions Copyright © 1995-1997 C H Forsyth (forsyth@terzarima.net) +// Portions Copyright © 1997-1999 Vita Nuova Limited +// Portions Copyright © 2000-2007 Vita Nuova Holdings Limited (www.vitanuova.com) +// Portions Copyright © 2004,2006 Bruce Ellis +// Portions Copyright © 2005-2007 C H Forsyth (forsyth@terzarima.net) +// Revisions Copyright © 2000-2007 Lucent Technologies Inc. and others +// Portions Copyright © 2009 The Go Authors. All rights reserved. +// +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to deal +// in the Software without restriction, including without limitation the rights +// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +// copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +// THE SOFTWARE. + +#include "gc.h" + +Prog* +gtext(Sym *s, int32 stkoff) +{ + int32 a; + + a = 0; + if(!(textflag & NOSPLIT)) + a = argsize(); + else if(stkoff >= 128) + yyerror("stack frame too large for NOSPLIT function"); + + gpseudo(ATEXT, s, nodconst(stkoff)); + p->to.type = D_CONST2; + p->to.offset2 = a; + return p; +} + +void +noretval(int n) +{ + + if(n & 1) { + gins(ANOP, Z, Z); + p->to.type = REGRET; + } + if(n & 2) { + gins(ANOP, Z, Z); + p->to.type = FREGRET; + } +} + +/* welcome to commute */ +static void +commute(Node *n) +{ + Node *l, *r; + + l = n->left; + r = n->right; + if(r->complex > l->complex) { + n->left = r; + n->right = l; + } +} + +void +indexshift(Node *n) +{ + int g; + + if(!typechlp[n->type->etype]) + return; + simplifyshift(n); + if(n->op == OASHL && n->right->op == OCONST){ + g = vconst(n->right); + if(g >= 0 && g < 4) + n->addable = 7; + } +} + +/* + * calculate addressability as follows + * NAME ==> 10/11 name+value(SB/SP) + * REGISTER ==> 12 register + * CONST ==> 20 $value + * *(20) ==> 21 value + * &(10) ==> 13 $name+value(SB) + * &(11) ==> 1 $name+value(SP) + * (13) + (20) ==> 13 fold constants + * (1) + (20) ==> 1 fold constants + * *(13) ==> 10 back to name + * *(1) ==> 11 back to name + * + * (20) * (X) ==> 7 multiplier in indexing + * (X,7) + (13,1) ==> 8 adder in indexing (addresses) + * (8) ==> &9(OINDEX) index, almost addressable + * 100 extern register + * + * calculate complexity (number of registers) + */ +void +xcom(Node *n) +{ + Node *l, *r; + int g; + + if(n == Z) + return; + l = n->left; + r = n->right; + n->complex = 0; + n->addable = 0; + switch(n->op) { + case OCONST: + n->addable = 20; + break; + + case ONAME: + n->addable = 10; + if(n->class == CPARAM || n->class == CAUTO) + n->addable = 11; + break; + + case OEXREG: + n->addable = 0; + break; + + case OREGISTER: + n->addable = 12; + break; + + case OINDREG: + n->addable = 12; + break; + + case OADDR: + xcom(l); + if(l->addable == 10) + n->addable = 13; + else + if(l->addable == 11) + n->addable = 1; + break; + + case OADD: + xcom(l); + xcom(r); + if(n->type->etype != TIND) + break; + + switch(r->addable) { + case 20: + switch(l->addable) { + case 1: + case 13: + commadd: + l->type = n->type; + *n = *l; + l = new(0, Z, Z); + *l = *(n->left); + l->xoffset += r->vconst; + n->left = l; + r = n->right; + goto brk; + } + break; + + case 1: + case 13: + case 10: + case 11: + /* l is the base, r is the index */ + if(l->addable != 20) + n->addable = 8; + break; + } + switch(l->addable) { + case 20: + switch(r->addable) { + case 13: + case 1: + r = n->left; + l = n->right; + n->left = l; + n->right = r; + goto commadd; + } + break; + + case 13: + case 1: + case 10: + case 11: + /* r is the base, l is the index */ + if(r->addable != 20) + n->addable = 8; + break; + } + if(n->addable == 8 && !side(n)) { + indx(n); + l = new1(OINDEX, idx.basetree, idx.regtree); + l->scale = idx.scale; + l->addable = 9; + l->complex = l->right->complex; + l->type = l->left->type; + n->op = OADDR; + n->left = l; + n->right = Z; + n->addable = 8; + break; + } + break; + + case OINDEX: + xcom(l); + xcom(r); + n->addable = 9; + break; + + case OIND: + xcom(l); + if(l->op == OADDR) { + l = l->left; + l->type = n->type; + *n = *l; + return; + } + switch(l->addable) { + case 20: + n->addable = 21; + break; + case 1: + n->addable = 11; + break; + case 13: + n->addable = 10; + break; + } + break; + + case OASHL: + xcom(l); + xcom(r); + indexshift(n); + break; + + case OMUL: + case OLMUL: + xcom(l); + xcom(r); + g = vlog(l); + if(g >= 0) { + n->left = r; + n->right = l; + l = r; + r = n->right; + } + g = vlog(r); + if(g >= 0) { + n->op = OASHL; + r->vconst = g; + r->type = types[TINT]; + indexshift(n); + break; + } +commute(n); + break; + + case OASLDIV: + xcom(l); + xcom(r); + g = vlog(r); + if(g >= 0) { + n->op = OASLSHR; + r->vconst = g; + r->type = types[TINT]; + } + break; + + case OLDIV: + xcom(l); + xcom(r); + g = vlog(r); + if(g >= 0) { + n->op = OLSHR; + r->vconst = g; + r->type = types[TINT]; + indexshift(n); + break; + } + break; + + case OASLMOD: + xcom(l); + xcom(r); + g = vlog(r); + if(g >= 0) { + n->op = OASAND; + r->vconst--; + } + break; + + case OLMOD: + xcom(l); + xcom(r); + g = vlog(r); + if(g >= 0) { + n->op = OAND; + r->vconst--; + } + break; + + case OASMUL: + case OASLMUL: + xcom(l); + xcom(r); + g = vlog(r); + if(g >= 0) { + n->op = OASASHL; + r->vconst = g; + } + break; + + case OLSHR: + case OASHR: + xcom(l); + xcom(r); + indexshift(n); + break; + + default: + if(l != Z) + xcom(l); + if(r != Z) + xcom(r); + break; + } +brk: + if(n->addable >= 10) + return; + if(l != Z) + n->complex = l->complex; + if(r != Z) { + if(r->complex == n->complex) + n->complex = r->complex+1; + else + if(r->complex > n->complex) + n->complex = r->complex; + } + if(n->complex == 0) + n->complex++; + + if(com64(n)) + return; + + switch(n->op) { + + case OFUNC: + n->complex = FNX; + break; + + case OLMOD: + case OMOD: + case OLMUL: + case OLDIV: + case OMUL: + case ODIV: + case OASLMUL: + case OASLDIV: + case OASLMOD: + case OASMUL: + case OASDIV: + case OASMOD: + if(r->complex >= l->complex) { + n->complex = l->complex + 3; + if(r->complex > n->complex) + n->complex = r->complex; + } else { + n->complex = r->complex + 3; + if(l->complex > n->complex) + n->complex = l->complex; + } + break; + + case OLSHR: + case OASHL: + case OASHR: + case OASLSHR: + case OASASHL: + case OASASHR: + if(r->complex >= l->complex) { + n->complex = l->complex + 2; + if(r->complex > n->complex) + n->complex = r->complex; + } else { + n->complex = r->complex + 2; + if(l->complex > n->complex) + n->complex = l->complex; + } + break; + + case OADD: + case OXOR: + case OAND: + case OOR: + /* + * immediate operators, make const on right + */ + if(l->op == OCONST) { + n->left = r; + n->right = l; + } + break; + + case OEQ: + case ONE: + case OLE: + case OLT: + case OGE: + case OGT: + case OHI: + case OHS: + case OLO: + case OLS: + /* + * compare operators, make const on left + */ + if(r->op == OCONST) { + n->left = r; + n->right = l; + n->op = invrel[relindex(n->op)]; + } + break; + } +} + +void +indx(Node *n) +{ + Node *l, *r; + + if(debug['x']) + prtree(n, "indx"); + + l = n->left; + r = n->right; + if(l->addable == 1 || l->addable == 13 || r->complex > l->complex) { + n->right = l; + n->left = r; + l = r; + r = n->right; + } + if(l->addable != 7) { + idx.regtree = l; + idx.scale = 1; + } else + if(l->right->addable == 20) { + idx.regtree = l->left; + idx.scale = 1 << l->right->vconst; + } else + if(l->left->addable == 20) { + idx.regtree = l->right; + idx.scale = 1 << l->left->vconst; + } else + diag(n, "bad index"); + + idx.basetree = r; + if(debug['x']) { + print("scale = %d\n", idx.scale); + prtree(idx.regtree, "index"); + prtree(idx.basetree, "base"); + } +} diff --git a/src/cmd/8c/swt.c b/src/cmd/8c/swt.c new file mode 100644 index 000000000..769ef2c66 --- /dev/null +++ b/src/cmd/8c/swt.c @@ -0,0 +1,588 @@ +// Inferno utils/8c/swt.c +// http://code.google.com/p/inferno-os/source/browse/utils/8c/swt.c +// +// Copyright © 1994-1999 Lucent Technologies Inc. All rights reserved. +// Portions Copyright © 1995-1997 C H Forsyth (forsyth@terzarima.net) +// Portions Copyright © 1997-1999 Vita Nuova Limited +// Portions Copyright © 2000-2007 Vita Nuova Holdings Limited (www.vitanuova.com) +// Portions Copyright © 2004,2006 Bruce Ellis +// Portions Copyright © 2005-2007 C H Forsyth (forsyth@terzarima.net) +// Revisions Copyright © 2000-2007 Lucent Technologies Inc. and others +// Portions Copyright © 2009 The Go Authors. All rights reserved. +// +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to deal +// in the Software without restriction, including without limitation the rights +// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +// copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +// THE SOFTWARE. + +#include "gc.h" + +void +swit1(C1 *q, int nc, int32 def, Node *n) +{ + C1 *r; + int i; + Prog *sp; + + if(nc < 5) { + for(i=0; ival); + gopcode(OEQ, n->type, n, nodconst(q->val)); + patch(p, q->label); + q++; + } + gbranch(OGOTO); + patch(p, def); + return; + } + i = nc / 2; + r = q+i; + if(debug['W']) + print("case > %.8ux\n", r->val); + gopcode(OGT, n->type, n, nodconst(r->val)); + sp = p; + gbranch(OGOTO); + p->as = AJEQ; + patch(p, r->label); + swit1(q, i, def, n); + + if(debug['W']) + print("case < %.8ux\n", r->val); + patch(sp, pc); + swit1(r+1, nc-i-1, def, n); +} + +void +bitload(Node *b, Node *n1, Node *n2, Node *n3, Node *nn) +{ + int sh; + int32 v; + Node *l; + + /* + * n1 gets adjusted/masked value + * n2 gets address of cell + * n3 gets contents of cell + */ + l = b->left; + if(n2 != Z) { + regalloc(n1, l, nn); + reglcgen(n2, l, Z); + regalloc(n3, l, Z); + gmove(n2, n3); + gmove(n3, n1); + } else { + regalloc(n1, l, nn); + cgen(l, n1); + } + if(b->type->shift == 0 && typeu[b->type->etype]) { + v = ~0 + (1L << b->type->nbits); + gopcode(OAND, types[TLONG], nodconst(v), n1); + } else { + sh = 32 - b->type->shift - b->type->nbits; + if(sh > 0) + gopcode(OASHL, types[TLONG], nodconst(sh), n1); + sh += b->type->shift; + if(sh > 0) + if(typeu[b->type->etype]) + gopcode(OLSHR, types[TLONG], nodconst(sh), n1); + else + gopcode(OASHR, types[TLONG], nodconst(sh), n1); + } +} + +void +bitstore(Node *b, Node *n1, Node *n2, Node *n3, Node *nn) +{ + int32 v; + Node nod; + int sh; + + regalloc(&nod, b->left, Z); + v = ~0 + (1L << b->type->nbits); + gopcode(OAND, types[TLONG], nodconst(v), n1); + gmove(n1, &nod); + if(nn != Z) + gmove(n1, nn); + sh = b->type->shift; + if(sh > 0) + gopcode(OASHL, types[TLONG], nodconst(sh), &nod); + v <<= sh; + gopcode(OAND, types[TLONG], nodconst(~v), n3); + gopcode(OOR, types[TLONG], n3, &nod); + gmove(&nod, n2); + + regfree(&nod); + regfree(n1); + regfree(n2); + regfree(n3); +} + +int32 +outstring(char *s, int32 n) +{ + int32 r; + + if(suppress) + return nstring; + r = nstring; + while(n) { + string[mnstring] = *s++; + mnstring++; + nstring++; + if(mnstring >= NSNAME) { + gpseudo(ADATA, symstring, nodconst(0L)); + p->from.offset += nstring - NSNAME; + p->from.scale = NSNAME; + p->to.type = D_SCONST; + memmove(p->to.sval, string, NSNAME); + mnstring = 0; + } + n--; + } + return r; +} + +void +sextern(Sym *s, Node *a, int32 o, int32 w) +{ + int32 e, lw; + + for(e=0; efrom.offset += o+e; + p->from.scale = lw; + p->to.type = D_SCONST; + memmove(p->to.sval, a->cstring+e, lw); + } +} + +void +gextern(Sym *s, Node *a, int32 o, int32 w) +{ + if(a->op == OCONST && typev[a->type->etype]) { + gpseudo(ADATA, s, lo64(a)); + p->from.offset += o; + p->from.scale = 4; + gpseudo(ADATA, s, hi64(a)); + p->from.offset += o + 4; + p->from.scale = 4; + return; + } + gpseudo(ADATA, s, a); + p->from.offset += o; + p->from.scale = w; + switch(p->to.type) { + default: + p->to.index = p->to.type; + p->to.type = D_ADDR; + case D_CONST: + case D_FCONST: + case D_ADDR: + break; + } +} + +void zname(Biobuf*, Sym*, int); +void zaddr(Biobuf*, Adr*, int); +void outhist(Biobuf*); + +void +outcode(void) +{ + struct { Sym *sym; short type; } h[NSYM]; + Prog *p; + Sym *s; + int f, sf, st, t, sym; + Biobuf b; + + if(debug['S']) { + for(p = firstp; p != P; p = p->link) + if(p->as != ADATA && p->as != AGLOBL) + pc--; + for(p = firstp; p != P; p = p->link) { + print("%P\n", p); + if(p->as != ADATA && p->as != AGLOBL) + pc++; + } + } + f = open(outfile, OWRITE); + if(f < 0) { + diag(Z, "cannot open %s", outfile); + return; + } + Binit(&b, f, OWRITE); + + Bprint(&b, "go object %s %s %s\n", getgoos(), thestring, getgoversion()); + if(ndynimp > 0 || ndynexp > 0) { + int i; + + Bprint(&b, "\n"); + Bprint(&b, "$$ // exports\n\n"); + Bprint(&b, "$$ // local types\n\n"); + Bprint(&b, "$$ // dynimport\n"); + for(i=0; ilink) { + jackpot: + sf = 0; + s = p->from.sym; + while(s != S) { + sf = s->sym; + if(sf < 0 || sf >= NSYM) + sf = 0; + t = p->from.type; + if(t == D_ADDR) + t = p->from.index; + if(h[sf].type == t) + if(h[sf].sym == s) + break; + s->sym = sym; + zname(&b, s, t); + h[sym].sym = s; + h[sym].type = t; + sf = sym; + sym++; + if(sym >= NSYM) + sym = 1; + break; + } + st = 0; + s = p->to.sym; + while(s != S) { + st = s->sym; + if(st < 0 || st >= NSYM) + st = 0; + t = p->to.type; + if(t == D_ADDR) + t = p->to.index; + if(h[st].type == t) + if(h[st].sym == s) + break; + s->sym = sym; + zname(&b, s, t); + h[sym].sym = s; + h[sym].type = t; + st = sym; + sym++; + if(sym >= NSYM) + sym = 1; + if(st == sf) + goto jackpot; + break; + } + Bputc(&b, p->as); + Bputc(&b, p->as>>8); + Bputc(&b, p->lineno); + Bputc(&b, p->lineno>>8); + Bputc(&b, p->lineno>>16); + Bputc(&b, p->lineno>>24); + zaddr(&b, &p->from, sf); + zaddr(&b, &p->to, st); + } + Bflush(&b); + close(f); + firstp = P; + lastp = P; +} + +void +outhist(Biobuf *b) +{ + Hist *h; + char *p, *q, *op, c; + Prog pg; + int n; + + pg = zprog; + pg.as = AHISTORY; + c = pathchar(); + for(h = hist; h != H; h = h->link) { + p = h->name; + op = 0; + /* on windows skip drive specifier in pathname */ + if(systemtype(Windows) && p && p[1] == ':'){ + p += 2; + c = *p; + } + if(p && p[0] != c && h->offset == 0 && pathname){ + /* on windows skip drive specifier in pathname */ + if(systemtype(Windows) && pathname[1] == ':') { + op = p; + p = pathname+2; + c = *p; + } else if(pathname[0] == c){ + op = p; + p = pathname; + } + } + while(p) { + q = utfrune(p, c); + if(q) { + n = q-p; + if(n == 0){ + n = 1; /* leading "/" */ + *p = '/'; /* don't emit "\" on windows */ + } + q++; + } else { + n = strlen(p); + q = 0; + } + if(n) { + Bputc(b, ANAME); + Bputc(b, ANAME>>8); + Bputc(b, D_FILE); + Bputc(b, 1); + Bputc(b, '<'); + Bwrite(b, p, n); + Bputc(b, 0); + } + p = q; + if(p == 0 && op) { + p = op; + op = 0; + } + } + pg.lineno = h->line; + pg.to.type = zprog.to.type; + pg.to.offset = h->offset; + if(h->offset) + pg.to.type = D_CONST; + + Bputc(b, pg.as); + Bputc(b, pg.as>>8); + Bputc(b, pg.lineno); + Bputc(b, pg.lineno>>8); + Bputc(b, pg.lineno>>16); + Bputc(b, pg.lineno>>24); + zaddr(b, &pg.from, 0); + zaddr(b, &pg.to, 0); + } +} + +void +zname(Biobuf *b, Sym *s, int t) +{ + char *n; + uint32 sig; + + if(debug['T'] && t == D_EXTERN && s->sig != SIGDONE && s->type != types[TENUM] && s != symrathole){ + sig = sign(s); + Bputc(b, ASIGNAME); + Bputc(b, ASIGNAME>>8); + Bputc(b, sig); + Bputc(b, sig>>8); + Bputc(b, sig>>16); + Bputc(b, sig>>24); + s->sig = SIGDONE; + } + else{ + Bputc(b, ANAME); /* as */ + Bputc(b, ANAME>>8); /* as */ + } + Bputc(b, t); /* type */ + Bputc(b, s->sym); /* sym */ + n = s->name; + while(*n) { + Bputc(b, *n); + n++; + } + Bputc(b, 0); +} + +void +zaddr(Biobuf *b, Adr *a, int s) +{ + int32 l; + int i, t; + char *n; + Ieee e; + + t = 0; + if(a->index != D_NONE || a->scale != 0) + t |= T_INDEX; + if(s != 0) + t |= T_SYM; + + switch(a->type) { + default: + t |= T_TYPE; + case D_NONE: + if(a->offset != 0) + t |= T_OFFSET; + break; + case D_FCONST: + t |= T_FCONST; + break; + case D_SCONST: + t |= T_SCONST; + break; + case D_CONST2: + t |= T_OFFSET|T_OFFSET2; + break; + } + Bputc(b, t); + + if(t & T_INDEX) { /* implies index, scale */ + Bputc(b, a->index); + Bputc(b, a->scale); + } + if(t & T_OFFSET) { /* implies offset */ + l = a->offset; + Bputc(b, l); + Bputc(b, l>>8); + Bputc(b, l>>16); + Bputc(b, l>>24); + } + if(t & T_OFFSET2) { /* implies offset2 */ + l = a->offset2; + Bputc(b, l); + Bputc(b, l>>8); + Bputc(b, l>>16); + Bputc(b, l>>24); + } + if(t & T_SYM) /* implies sym */ + Bputc(b, s); + if(t & T_FCONST) { + ieeedtod(&e, a->dval); + l = e.l; + Bputc(b, l); + Bputc(b, l>>8); + Bputc(b, l>>16); + Bputc(b, l>>24); + l = e.h; + Bputc(b, l); + Bputc(b, l>>8); + Bputc(b, l>>16); + Bputc(b, l>>24); + return; + } + if(t & T_SCONST) { + n = a->sval; + for(i=0; itype); +} + +int32 +align(int32 i, Type *t, int op, int32 *maxalign) +{ + int32 o; + Type *v; + int w; + + o = i; + w = 1; + switch(op) { + default: + diag(Z, "unknown align opcode %d", op); + break; + + case Asu2: /* padding at end of a struct */ + w = *maxalign; + if(w < 1) + w = 1; + if(packflg) + w = packflg; + break; + + case Ael1: /* initial align of struct element */ + for(v=t; v->etype==TARRAY; v=v->link) + ; + if(v->etype == TSTRUCT || v->etype == TUNION) + w = v->align; + else { + w = ewidth[v->etype]; + if(w == 8) + w = 4; + } + if(w < 1 || w > SZ_LONG) + fatal(Z, "align"); + if(packflg) + w = packflg; + break; + + case Ael2: /* width of a struct element */ + o += t->width; + break; + + case Aarg0: /* initial passbyptr argument in arg list */ + if(typesuv[t->etype]) { + o = align(o, types[TIND], Aarg1, nil); + o = align(o, types[TIND], Aarg2, nil); + } + break; + + case Aarg1: /* initial align of parameter */ + w = ewidth[t->etype]; + if(w <= 0 || w >= SZ_LONG) { + w = SZ_LONG; + break; + } + w = 1; /* little endian no adjustment */ + break; + + case Aarg2: /* width of a parameter */ + o += t->width; + w = t->width; + if(w > SZ_LONG) + w = SZ_LONG; + break; + + case Aaut3: /* total align of automatic */ + o = align(o, t, Ael1, nil); + o = align(o, t, Ael2, nil); + break; + } + o = xround(o, w); + if(maxalign && *maxalign < w) + *maxalign = w; + if(debug['A']) + print("align %s %d %T = %d\n", bnames[op], i, t, o); + return o; +} + +int32 +maxround(int32 max, int32 v) +{ + v += SZ_LONG-1; + if(v > max) + max = xround(v, SZ_LONG); + return max; +} diff --git a/src/cmd/8c/txt.c b/src/cmd/8c/txt.c new file mode 100644 index 000000000..b2e0148a0 --- /dev/null +++ b/src/cmd/8c/txt.c @@ -0,0 +1,1458 @@ +// Inferno utils/8c/txt.c +// http://code.google.com/p/inferno-os/source/browse/utils/8c/txt.c +// +// Copyright © 1994-1999 Lucent Technologies Inc. All rights reserved. +// Portions Copyright © 1995-1997 C H Forsyth (forsyth@terzarima.net) +// Portions Copyright © 1997-1999 Vita Nuova Limited +// Portions Copyright © 2000-2007 Vita Nuova Holdings Limited (www.vitanuova.com) +// Portions Copyright © 2004,2006 Bruce Ellis +// Portions Copyright © 2005-2007 C H Forsyth (forsyth@terzarima.net) +// Revisions Copyright © 2000-2007 Lucent Technologies Inc. and others +// Portions Copyright © 2009 The Go Authors. All rights reserved. +// +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to deal +// in the Software without restriction, including without limitation the rights +// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +// copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +// THE SOFTWARE. + +#include "gc.h" + +void +ginit(void) +{ + int i; + Type *t; + + thechar = '8'; + thestring = "386"; + exregoffset = 0; + exfregoffset = 0; + listinit(); + nstring = 0; + mnstring = 0; + nrathole = 0; + pc = 0; + breakpc = -1; + continpc = -1; + cases = C; + firstp = P; + lastp = P; + tfield = types[TLONG]; + + zprog.link = P; + zprog.as = AGOK; + zprog.from.type = D_NONE; + zprog.from.index = D_NONE; + zprog.from.scale = 0; + zprog.to = zprog.from; + + regnode.op = OREGISTER; + regnode.class = CEXREG; + regnode.reg = REGTMP; + regnode.complex = 0; + regnode.addable = 11; + regnode.type = types[TLONG]; + + fregnode0 = regnode; + fregnode0.reg = D_F0; + fregnode0.type = types[TDOUBLE]; + + fregnode1 = fregnode0; + fregnode1.reg = D_F0+1; + + constnode.op = OCONST; + constnode.class = CXXX; + constnode.complex = 0; + constnode.addable = 20; + constnode.type = types[TLONG]; + + fconstnode.op = OCONST; + fconstnode.class = CXXX; + fconstnode.complex = 0; + fconstnode.addable = 20; + fconstnode.type = types[TDOUBLE]; + + nodsafe = new(ONAME, Z, Z); + nodsafe->sym = slookup(".safe"); + nodsafe->type = types[TINT]; + nodsafe->etype = types[TINT]->etype; + nodsafe->class = CAUTO; + complex(nodsafe); + + t = typ(TARRAY, types[TCHAR]); + symrathole = slookup(".rathole"); + symrathole->class = CGLOBL; + symrathole->type = t; + + nodrat = new(ONAME, Z, Z); + nodrat->sym = symrathole; + nodrat->type = types[TIND]; + nodrat->etype = TVOID; + nodrat->class = CGLOBL; + complex(nodrat); + nodrat->type = t; + + nodret = new(ONAME, Z, Z); + nodret->sym = slookup(".ret"); + nodret->type = types[TIND]; + nodret->etype = TIND; + nodret->class = CPARAM; + nodret = new(OIND, nodret, Z); + complex(nodret); + + com64init(); + + for(i=0; i= D_AX && i <= D_DI && i != D_SP) + reg[i] = 0; + } +} + +void +gclean(void) +{ + int i; + Sym *s; + + reg[D_SP]--; + for(i=D_AX; i<=D_DI; i++) + if(reg[i]) + diag(Z, "reg %R left allocated", i); + while(mnstring) + outstring("", 1L); + symstring->type->width = nstring; + symrathole->type->width = nrathole; + for(i=0; ilink) { + if(s->type == T) + continue; + if(s->type->width == 0) + continue; + if(s->class != CGLOBL && s->class != CSTATIC) + continue; + if(s->type == types[TENUM]) + continue; + gpseudo(AGLOBL, s, nodconst(s->type->width)); + } + nextpc(); + p->as = AEND; + outcode(); +} + +void +nextpc(void) +{ + + p = alloc(sizeof(*p)); + *p = zprog; + p->lineno = nearln; + pc++; + if(firstp == P) { + firstp = p; + lastp = p; + return; + } + lastp->link = p; + lastp = p; +} + +void +gargs(Node *n, Node *tn1, Node *tn2) +{ + int32 regs; + Node fnxargs[20], *fnxp; + + regs = cursafe; + + fnxp = fnxargs; + garg1(n, tn1, tn2, 0, &fnxp); /* compile fns to temps */ + + curarg = 0; + fnxp = fnxargs; + garg1(n, tn1, tn2, 1, &fnxp); /* compile normal args and temps */ + + cursafe = regs; +} + +int nareg(void) +{ + int i, n; + + n = 0; + for(i=D_AX; i<=D_DI; i++) + if(reg[i] == 0) + n++; + return n; +} + +void +garg1(Node *n, Node *tn1, Node *tn2, int f, Node **fnxp) +{ + Node nod; + + if(n == Z) + return; + if(n->op == OLIST) { + garg1(n->left, tn1, tn2, f, fnxp); + garg1(n->right, tn1, tn2, f, fnxp); + return; + } + if(f == 0) { + if(n->complex >= FNX) { + regsalloc(*fnxp, n); + nod = znode; + nod.op = OAS; + nod.left = *fnxp; + nod.right = n; + nod.type = n->type; + cgen(&nod, Z); + (*fnxp)++; + } + return; + } + if(typesu[n->type->etype] || typev[n->type->etype]) { + regaalloc(tn2, n); + if(n->complex >= FNX) { + sugen(*fnxp, tn2, n->type->width); + (*fnxp)++; + } else + sugen(n, tn2, n->type->width); + return; + } + if(REGARG >= 0 && curarg == 0 && typeilp[n->type->etype]) { + regaalloc1(tn1, n); + if(n->complex >= FNX) { + cgen(*fnxp, tn1); + (*fnxp)++; + } else + cgen(n, tn1); + return; + } + if(vconst(n) == 0) { + regaalloc(tn2, n); + gmove(n, tn2); + return; + } + regalloc(tn1, n, Z); + if(n->complex >= FNX) { + cgen(*fnxp, tn1); + (*fnxp)++; + } else + cgen(n, tn1); + regaalloc(tn2, n); + gmove(tn1, tn2); + regfree(tn1); +} + +Node* +nodconst(int32 v) +{ + constnode.vconst = v; + return &constnode; +} + +Node* +nodfconst(double d) +{ + fconstnode.fconst = d; + return &fconstnode; +} + +int +isreg(Node *n, int r) +{ + + if(n->op == OREGISTER) + if(n->reg == r) + return 1; + return 0; +} + +int +nodreg(Node *n, Node *nn, int r) +{ + + *n = regnode; + n->reg = r; + if(reg[r] == 0) + return 0; + if(nn != Z) { + n->type = nn->type; + n->lineno = nn->lineno; + if(nn->op == OREGISTER) + if(nn->reg == r) + return 0; + } + return 1; +} + +void +regret(Node *n, Node *nn) +{ + int r; + + r = REGRET; + if(typefd[nn->type->etype]) + r = FREGRET; + nodreg(n, nn, r); + reg[r]++; +} + +void +regalloc(Node *n, Node *tn, Node *o) +{ + int i; + + switch(tn->type->etype) { + case TCHAR: + case TUCHAR: + case TSHORT: + case TUSHORT: + case TINT: + case TUINT: + case TLONG: + case TULONG: + case TIND: + if(o != Z && o->op == OREGISTER) { + i = o->reg; + if(i >= D_AX && i <= D_DI) + goto out; + } + for(i=D_AX; i<=D_DI; i++) + if(reg[i] == 0) + goto out; + diag(tn, "out of fixed registers"); + goto err; + + case TFLOAT: + case TDOUBLE: + case TVLONG: + i = D_F0; + goto out; + } + diag(tn, "unknown type in regalloc: %T", tn->type); +err: + i = 0; +out: + if(i) + reg[i]++; + nodreg(n, tn, i); +} + +void +regialloc(Node *n, Node *tn, Node *o) +{ + Node nod; + + nod = *tn; + nod.type = types[TIND]; + regalloc(n, &nod, o); +} + +void +regfree(Node *n) +{ + int i; + + i = 0; + if(n->op != OREGISTER && n->op != OINDREG) + goto err; + i = n->reg; + if(i < 0 || i >= sizeof(reg)) + goto err; + if(reg[i] <= 0) + goto err; + reg[i]--; + return; +err: + diag(n, "error in regfree: %R", i); +} + +void +regsalloc(Node *n, Node *nn) +{ + cursafe = align(cursafe, nn->type, Aaut3, nil); + maxargsafe = maxround(maxargsafe, cursafe+curarg); + *n = *nodsafe; + n->xoffset = -(stkoff + cursafe); + n->type = nn->type; + n->etype = nn->type->etype; + n->lineno = nn->lineno; +} + +void +regaalloc1(Node *n, Node *nn) +{ + if(REGARG < 0) { + fatal(n, "regaalloc1 and REGARG<0"); + return; + } + nodreg(n, nn, REGARG); + reg[REGARG]++; + curarg = align(curarg, nn->type, Aarg1, nil); + curarg = align(curarg, nn->type, Aarg2, nil); + maxargsafe = maxround(maxargsafe, cursafe+curarg); +} + +void +regaalloc(Node *n, Node *nn) +{ + curarg = align(curarg, nn->type, Aarg1, nil); + *n = *nn; + n->op = OINDREG; + n->reg = REGSP; + n->xoffset = curarg; + n->complex = 0; + n->addable = 20; + curarg = align(curarg, nn->type, Aarg2, nil); + maxargsafe = maxround(maxargsafe, cursafe+curarg); +} + +void +regind(Node *n, Node *nn) +{ + + if(n->op != OREGISTER) { + diag(n, "regind not OREGISTER"); + return; + } + n->op = OINDREG; + n->type = nn->type; +} + +void +naddr(Node *n, Adr *a) +{ + int32 v; + + a->type = D_NONE; + if(n == Z) + return; + switch(n->op) { + default: + bad: + diag(n, "bad in naddr: %O %D", n->op, a); + break; + + case OREGISTER: + a->type = n->reg; + a->sym = S; + break; + + case OEXREG: + a->type = D_INDIR + D_GS; + a->offset = n->reg - 1; + break; + + case OIND: + naddr(n->left, a); + if(a->type >= D_AX && a->type <= D_DI) + a->type += D_INDIR; + else + if(a->type == D_CONST) + a->type = D_NONE+D_INDIR; + else + if(a->type == D_ADDR) { + a->type = a->index; + a->index = D_NONE; + } else + goto bad; + break; + + case OINDEX: + a->type = idx.ptr; + if(n->left->op == OADDR || n->left->op == OCONST) + naddr(n->left, a); + if(a->type >= D_AX && a->type <= D_DI) + a->type += D_INDIR; + else + if(a->type == D_CONST) + a->type = D_NONE+D_INDIR; + else + if(a->type == D_ADDR) { + a->type = a->index; + a->index = D_NONE; + } else + goto bad; + a->index = idx.reg; + a->scale = n->scale; + a->offset += n->xoffset; + break; + + case OINDREG: + a->type = n->reg+D_INDIR; + a->sym = S; + a->offset = n->xoffset; + break; + + case ONAME: + a->etype = n->etype; + a->type = D_STATIC; + a->sym = n->sym; + a->offset = n->xoffset; + if(n->class == CSTATIC) + break; + if(n->class == CEXTERN || n->class == CGLOBL) { + a->type = D_EXTERN; + break; + } + if(n->class == CAUTO) { + a->type = D_AUTO; + break; + } + if(n->class == CPARAM) { + a->type = D_PARAM; + break; + } + goto bad; + + case OCONST: + if(typefd[n->type->etype]) { + a->type = D_FCONST; + a->dval = n->fconst; + break; + } + a->sym = S; + a->type = D_CONST; + a->offset = n->vconst; + break; + + case OADDR: + naddr(n->left, a); + if(a->type >= D_INDIR) { + a->type -= D_INDIR; + break; + } + if(a->type == D_EXTERN || a->type == D_STATIC || + a->type == D_AUTO || a->type == D_PARAM) + if(a->index == D_NONE) { + a->index = a->type; + a->type = D_ADDR; + break; + } + goto bad; + + case OADD: + if(n->right->op == OCONST) { + v = n->right->vconst; + naddr(n->left, a); + } else + if(n->left->op == OCONST) { + v = n->left->vconst; + naddr(n->right, a); + } else + goto bad; + a->offset += v; + break; + + } +} + +#define CASE(a,b) ((a<<8)|(b<<0)) + +void +gmove(Node *f, Node *t) +{ + int ft, tt, a; + Node nod, nod1; + Prog *p1; + + ft = f->type->etype; + tt = t->type->etype; + if(debug['M']) + print("gop: %O %O[%s],%O[%s]\n", OAS, + f->op, tnames[ft], t->op, tnames[tt]); + if(typefd[ft] && f->op == OCONST) { + if(f->fconst == 0) + gins(AFLDZ, Z, Z); + else + if(f->fconst == 1) + gins(AFLD1, Z, Z); + else + gins(AFMOVD, f, &fregnode0); + gmove(&fregnode0, t); + return; + } +/* + * load + */ + if(f->op == ONAME || f->op == OINDREG || + f->op == OIND || f->op == OINDEX) + switch(ft) { + case TCHAR: + a = AMOVBLSX; + goto ld; + case TUCHAR: + a = AMOVBLZX; + goto ld; + case TSHORT: + if(typefd[tt]) { + gins(AFMOVW, f, &fregnode0); + gmove(&fregnode0, t); + return; + } + a = AMOVWLSX; + goto ld; + case TUSHORT: + a = AMOVWLZX; + goto ld; + case TINT: + case TUINT: + case TLONG: + case TULONG: + case TIND: + if(typefd[tt]) { + gins(AFMOVL, f, &fregnode0); + gmove(&fregnode0, t); + return; + } + a = AMOVL; + + ld: + regalloc(&nod, f, t); + nod.type = types[TLONG]; + gins(a, f, &nod); + gmove(&nod, t); + regfree(&nod); + return; + + case TFLOAT: + gins(AFMOVF, f, t); + return; + case TDOUBLE: + gins(AFMOVD, f, t); + return; + case TVLONG: + gins(AFMOVV, f, t); + return; + } + +/* + * store + */ + if(t->op == ONAME || t->op == OINDREG || + t->op == OIND || t->op == OINDEX) + switch(tt) { + case TCHAR: + case TUCHAR: + a = AMOVB; goto st; + case TSHORT: + case TUSHORT: + a = AMOVW; goto st; + case TINT: + case TUINT: + case TLONG: + case TULONG: + case TIND: + a = AMOVL; goto st; + + st: + if(f->op == OCONST) { + gins(a, f, t); + return; + } + regalloc(&nod, t, f); + gmove(f, &nod); + gins(a, &nod, t); + regfree(&nod); + return; + + case TFLOAT: + gins(AFMOVFP, f, t); + return; + case TDOUBLE: + gins(AFMOVDP, f, t); + return; + case TVLONG: + gins(AFMOVVP, f, t); + return; + } + +/* + * convert + */ + switch(CASE(ft,tt)) { + default: +/* + * integer to integer + ******** + a = AGOK; break; + + case CASE( TCHAR, TCHAR): + case CASE( TUCHAR, TCHAR): + case CASE( TSHORT, TCHAR): + case CASE( TUSHORT,TCHAR): + case CASE( TINT, TCHAR): + case CASE( TUINT, TCHAR): + case CASE( TLONG, TCHAR): + case CASE( TULONG, TCHAR): + case CASE( TIND, TCHAR): + + case CASE( TCHAR, TUCHAR): + case CASE( TUCHAR, TUCHAR): + case CASE( TSHORT, TUCHAR): + case CASE( TUSHORT,TUCHAR): + case CASE( TINT, TUCHAR): + case CASE( TUINT, TUCHAR): + case CASE( TLONG, TUCHAR): + case CASE( TULONG, TUCHAR): + case CASE( TIND, TUCHAR): + + case CASE( TSHORT, TSHORT): + case CASE( TUSHORT,TSHORT): + case CASE( TINT, TSHORT): + case CASE( TUINT, TSHORT): + case CASE( TLONG, TSHORT): + case CASE( TULONG, TSHORT): + case CASE( TIND, TSHORT): + + case CASE( TSHORT, TUSHORT): + case CASE( TUSHORT,TUSHORT): + case CASE( TINT, TUSHORT): + case CASE( TUINT, TUSHORT): + case CASE( TLONG, TUSHORT): + case CASE( TULONG, TUSHORT): + case CASE( TIND, TUSHORT): + + case CASE( TINT, TINT): + case CASE( TUINT, TINT): + case CASE( TLONG, TINT): + case CASE( TULONG, TINT): + case CASE( TIND, TINT): + + case CASE( TINT, TUINT): + case CASE( TUINT, TUINT): + case CASE( TLONG, TUINT): + case CASE( TULONG, TUINT): + case CASE( TIND, TUINT): + + case CASE( TINT, TLONG): + case CASE( TUINT, TLONG): + case CASE( TLONG, TLONG): + case CASE( TULONG, TLONG): + case CASE( TIND, TLONG): + + case CASE( TINT, TULONG): + case CASE( TUINT, TULONG): + case CASE( TLONG, TULONG): + case CASE( TULONG, TULONG): + case CASE( TIND, TULONG): + + case CASE( TINT, TIND): + case CASE( TUINT, TIND): + case CASE( TLONG, TIND): + case CASE( TULONG, TIND): + case CASE( TIND, TIND): + *****/ + a = AMOVL; + break; + + case CASE( TSHORT, TINT): + case CASE( TSHORT, TUINT): + case CASE( TSHORT, TLONG): + case CASE( TSHORT, TULONG): + case CASE( TSHORT, TIND): + a = AMOVWLSX; + if(f->op == OCONST) { + f->vconst &= 0xffff; + if(f->vconst & 0x8000) + f->vconst |= 0xffff0000; + a = AMOVL; + } + break; + + case CASE( TUSHORT,TINT): + case CASE( TUSHORT,TUINT): + case CASE( TUSHORT,TLONG): + case CASE( TUSHORT,TULONG): + case CASE( TUSHORT,TIND): + a = AMOVWLZX; + if(f->op == OCONST) { + f->vconst &= 0xffff; + a = AMOVL; + } + break; + + case CASE( TCHAR, TSHORT): + case CASE( TCHAR, TUSHORT): + case CASE( TCHAR, TINT): + case CASE( TCHAR, TUINT): + case CASE( TCHAR, TLONG): + case CASE( TCHAR, TULONG): + case CASE( TCHAR, TIND): + a = AMOVBLSX; + if(f->op == OCONST) { + f->vconst &= 0xff; + if(f->vconst & 0x80) + f->vconst |= 0xffffff00; + a = AMOVL; + } + break; + + case CASE( TUCHAR, TSHORT): + case CASE( TUCHAR, TUSHORT): + case CASE( TUCHAR, TINT): + case CASE( TUCHAR, TUINT): + case CASE( TUCHAR, TLONG): + case CASE( TUCHAR, TULONG): + case CASE( TUCHAR, TIND): + a = AMOVBLZX; + if(f->op == OCONST) { + f->vconst &= 0xff; + a = AMOVL; + } + break; + +/* + * float to fix + */ + case CASE( TFLOAT, TCHAR): + case CASE( TFLOAT, TUCHAR): + case CASE( TFLOAT, TSHORT): + case CASE( TFLOAT, TUSHORT): + case CASE( TFLOAT, TINT): + case CASE( TFLOAT, TUINT): + case CASE( TFLOAT, TLONG): + case CASE( TFLOAT, TULONG): + case CASE( TFLOAT, TIND): + + case CASE( TDOUBLE,TCHAR): + case CASE( TDOUBLE,TUCHAR): + case CASE( TDOUBLE,TSHORT): + case CASE( TDOUBLE,TUSHORT): + case CASE( TDOUBLE,TINT): + case CASE( TDOUBLE,TUINT): + case CASE( TDOUBLE,TLONG): + case CASE( TDOUBLE,TULONG): + case CASE( TDOUBLE,TIND): + + case CASE( TVLONG, TCHAR): + case CASE( TVLONG, TUCHAR): + case CASE( TVLONG, TSHORT): + case CASE( TVLONG, TUSHORT): + case CASE( TVLONG, TINT): + case CASE( TVLONG, TUINT): + case CASE( TVLONG, TLONG): + case CASE( TVLONG, TULONG): + case CASE( TVLONG, TIND): + if(fproundflg) { + regsalloc(&nod, ®node); + gins(AFMOVLP, f, &nod); + gmove(&nod, t); + return; + } + regsalloc(&nod, ®node); + regsalloc(&nod1, ®node); + gins(AFSTCW, Z, &nod1); + nod1.xoffset += 2; + gins(AMOVW, nodconst(0xf7f), &nod1); + gins(AFLDCW, &nod1, Z); + gins(AFMOVLP, f, &nod); + nod1.xoffset -= 2; + gins(AFLDCW, &nod1, Z); + gmove(&nod, t); + return; + +/* + * ulong to float + */ + case CASE( TULONG, TDOUBLE): + case CASE( TULONG, TVLONG): + case CASE( TULONG, TFLOAT): + case CASE( TUINT, TDOUBLE): + case CASE( TUINT, TVLONG): + case CASE( TUINT, TFLOAT): + regalloc(&nod, f, f); + gmove(f, &nod); + regsalloc(&nod1, ®node); + gmove(&nod, &nod1); + gins(AFMOVL, &nod1, &fregnode0); + gins(ACMPL, &nod, nodconst(0)); + gins(AJGE, Z, Z); + p1 = p; + gins(AFADDD, nodfconst(4294967296.), &fregnode0); + patch(p1, pc); + regfree(&nod); + return; + +/* + * fix to float + */ + case CASE( TCHAR, TFLOAT): + case CASE( TUCHAR, TFLOAT): + case CASE( TSHORT, TFLOAT): + case CASE( TUSHORT,TFLOAT): + case CASE( TINT, TFLOAT): + case CASE( TLONG, TFLOAT): + case CASE( TIND, TFLOAT): + + case CASE( TCHAR, TDOUBLE): + case CASE( TUCHAR, TDOUBLE): + case CASE( TSHORT, TDOUBLE): + case CASE( TUSHORT,TDOUBLE): + case CASE( TINT, TDOUBLE): + case CASE( TLONG, TDOUBLE): + case CASE( TIND, TDOUBLE): + + case CASE( TCHAR, TVLONG): + case CASE( TUCHAR, TVLONG): + case CASE( TSHORT, TVLONG): + case CASE( TUSHORT,TVLONG): + case CASE( TINT, TVLONG): + case CASE( TLONG, TVLONG): + case CASE( TIND, TVLONG): + regsalloc(&nod, ®node); + gmove(f, &nod); + gins(AFMOVL, &nod, &fregnode0); + return; + +/* + * float to float + */ + case CASE( TFLOAT, TFLOAT): + case CASE( TDOUBLE,TFLOAT): + case CASE( TVLONG, TFLOAT): + + case CASE( TFLOAT, TDOUBLE): + case CASE( TDOUBLE,TDOUBLE): + case CASE( TVLONG, TDOUBLE): + + case CASE( TFLOAT, TVLONG): + case CASE( TDOUBLE,TVLONG): + case CASE( TVLONG, TVLONG): + a = AFMOVD; break; + } + if(a == AMOVL || a == AFMOVD) + if(samaddr(f, t)) + return; + gins(a, f, t); +} + +void +doindex(Node *n) +{ + Node nod, nod1; + int32 v; + +if(debug['Y']) +prtree(n, "index"); + +if(n->left->complex >= FNX) +print("botch in doindex\n"); + + regalloc(&nod, ®node, Z); + v = constnode.vconst; + cgen(n->right, &nod); + idx.ptr = D_NONE; + if(n->left->op == OCONST) + idx.ptr = D_CONST; + else if(n->left->op == OREGISTER) + idx.ptr = n->left->reg; + else if(n->left->op != OADDR) { + reg[D_BP]++; // cant be used as a base + regalloc(&nod1, ®node, Z); + cgen(n->left, &nod1); + idx.ptr = nod1.reg; + regfree(&nod1); + reg[D_BP]--; + } + idx.reg = nod.reg; + regfree(&nod); + constnode.vconst = v; +} + +void +gins(int a, Node *f, Node *t) +{ + + if(f != Z && f->op == OINDEX) + doindex(f); + if(t != Z && t->op == OINDEX) + doindex(t); + nextpc(); + p->as = a; + if(f != Z) + naddr(f, &p->from); + if(t != Z) + naddr(t, &p->to); + if(debug['g']) + print("%P\n", p); +} + +void +fgopcode(int o, Node *f, Node *t, int pop, int rev) +{ + int a, et; + Node nod; + + et = TLONG; + if(f != Z && f->type != T) + et = f->type->etype; + if(!typefd[et]) { + diag(f, "fop: integer %O", o); + return; + } + if(debug['M']) { + if(t != Z && t->type != T) + print("gop: %O %O-%s Z\n", o, f->op, tnames[et]); + else + print("gop: %O %O-%s %O-%s\n", o, + f->op, tnames[et], t->op, tnames[t->type->etype]); + } + a = AGOK; + switch(o) { + + case OASADD: + case OADD: + if(et == TFLOAT) + a = AFADDF; + else + if(et == TDOUBLE || et == TVLONG) { + a = AFADDD; + if(pop) + a = AFADDDP; + } + break; + + case OASSUB: + case OSUB: + if(et == TFLOAT) { + a = AFSUBF; + if(rev) + a = AFSUBRF; + } else + if(et == TDOUBLE || et == TVLONG) { + a = AFSUBD; + if(pop) + a = AFSUBDP; + if(rev) { + a = AFSUBRD; + if(pop) + a = AFSUBRDP; + } + } + break; + + case OASMUL: + case OMUL: + if(et == TFLOAT) + a = AFMULF; + else + if(et == TDOUBLE || et == TVLONG) { + a = AFMULD; + if(pop) + a = AFMULDP; + } + break; + + case OASMOD: + case OMOD: + case OASDIV: + case ODIV: + if(et == TFLOAT) { + a = AFDIVF; + if(rev) + a = AFDIVRF; + } else + if(et == TDOUBLE || et == TVLONG) { + a = AFDIVD; + if(pop) + a = AFDIVDP; + if(rev) { + a = AFDIVRD; + if(pop) + a = AFDIVRDP; + } + } + break; + + case OEQ: + case ONE: + case OLT: + case OLE: + case OGE: + case OGT: + pop += rev; + if(et == TFLOAT) { + a = AFCOMF; + if(pop) { + a = AFCOMFP; + if(pop > 1) + a = AGOK; + } + } else + if(et == TDOUBLE || et == TVLONG) { + a = AFCOMF; + if(pop) { + a = AFCOMDP; + if(pop > 1) + a = AFCOMDPP; + } + } + gins(a, f, t); + regalloc(&nod, ®node, Z); + if(nod.reg != D_AX) { + regfree(&nod); + nod.reg = D_AX; + gins(APUSHL, &nod, Z); + gins(AWAIT, Z, Z); + gins(AFSTSW, Z, &nod); + gins(ASAHF, Z, Z); + gins(APOPL, Z, &nod); + } else { + gins(AWAIT, Z, Z); + gins(AFSTSW, Z, &nod); + gins(ASAHF, Z, Z); + regfree(&nod); + } + switch(o) { + case OEQ: a = AJEQ; break; + case ONE: a = AJNE; break; + case OLT: a = AJCS; break; + case OLE: a = AJLS; break; + case OGE: a = AJCC; break; + case OGT: a = AJHI; break; + } + gins(a, Z, Z); + return; + } + if(a == AGOK) + diag(Z, "bad in gopcode %O", o); + gins(a, f, t); +} + +void +gopcode(int o, Type *ty, Node *f, Node *t) +{ + int a, et; + + et = TLONG; + if(ty != T) + et = ty->etype; + if(typefd[et] && o != OADDR && o != OFUNC) { + diag(f, "gop: float %O", o); + return; + } + if(debug['M']) { + if(f != Z && f->type != T) + print("gop: %O %O[%s],", o, f->op, tnames[et]); + else + print("gop: %O Z,", o); + if(t != Z && t->type != T) + print("%O[%s]\n", t->op, tnames[t->type->etype]); + else + print("Z\n"); + } + a = AGOK; + switch(o) { + case OCOM: + a = ANOTL; + if(et == TCHAR || et == TUCHAR) + a = ANOTB; + if(et == TSHORT || et == TUSHORT) + a = ANOTW; + break; + + case ONEG: + a = ANEGL; + if(et == TCHAR || et == TUCHAR) + a = ANEGB; + if(et == TSHORT || et == TUSHORT) + a = ANEGW; + break; + + case OADDR: + a = ALEAL; + break; + + case OASADD: + case OADD: + a = AADDL; + if(et == TCHAR || et == TUCHAR) + a = AADDB; + if(et == TSHORT || et == TUSHORT) + a = AADDW; + break; + + case OASSUB: + case OSUB: + a = ASUBL; + if(et == TCHAR || et == TUCHAR) + a = ASUBB; + if(et == TSHORT || et == TUSHORT) + a = ASUBW; + break; + + case OASOR: + case OOR: + a = AORL; + if(et == TCHAR || et == TUCHAR) + a = AORB; + if(et == TSHORT || et == TUSHORT) + a = AORW; + break; + + case OASAND: + case OAND: + a = AANDL; + if(et == TCHAR || et == TUCHAR) + a = AANDB; + if(et == TSHORT || et == TUSHORT) + a = AANDW; + break; + + case OASXOR: + case OXOR: + a = AXORL; + if(et == TCHAR || et == TUCHAR) + a = AXORB; + if(et == TSHORT || et == TUSHORT) + a = AXORW; + break; + + case OASLSHR: + case OLSHR: + a = ASHRL; + if(et == TCHAR || et == TUCHAR) + a = ASHRB; + if(et == TSHORT || et == TUSHORT) + a = ASHRW; + break; + + case OASASHR: + case OASHR: + a = ASARL; + if(et == TCHAR || et == TUCHAR) + a = ASARB; + if(et == TSHORT || et == TUSHORT) + a = ASARW; + break; + + case OASASHL: + case OASHL: + a = ASALL; + if(et == TCHAR || et == TUCHAR) + a = ASALB; + if(et == TSHORT || et == TUSHORT) + a = ASALW; + break; + + case OFUNC: + a = ACALL; + break; + + case OASMUL: + case OMUL: + if(f->op == OREGISTER && t != Z && isreg(t, D_AX) && reg[D_DX] == 0) + t = Z; + a = AIMULL; + break; + + case OASMOD: + case OMOD: + case OASDIV: + case ODIV: + a = AIDIVL; + break; + + case OASLMUL: + case OLMUL: + a = AMULL; + break; + + case OASLMOD: + case OLMOD: + case OASLDIV: + case OLDIV: + a = ADIVL; + break; + + case OEQ: + case ONE: + case OLT: + case OLE: + case OGE: + case OGT: + case OLO: + case OLS: + case OHS: + case OHI: + a = ACMPL; + if(et == TCHAR || et == TUCHAR) + a = ACMPB; + if(et == TSHORT || et == TUSHORT) + a = ACMPW; + gins(a, f, t); + switch(o) { + case OEQ: a = AJEQ; break; + case ONE: a = AJNE; break; + case OLT: a = AJLT; break; + case OLE: a = AJLE; break; + case OGE: a = AJGE; break; + case OGT: a = AJGT; break; + case OLO: a = AJCS; break; + case OLS: a = AJLS; break; + case OHS: a = AJCC; break; + case OHI: a = AJHI; break; + } + gins(a, Z, Z); + return; + } + if(a == AGOK) + diag(Z, "bad in gopcode %O", o); + gins(a, f, t); +} + +int +samaddr(Node *f, Node *t) +{ + + if(f->op != t->op) + return 0; + switch(f->op) { + + case OREGISTER: + if(f->reg != t->reg) + break; + return 1; + } + return 0; +} + +void +gbranch(int o) +{ + int a; + + a = AGOK; + switch(o) { + case ORETURN: + a = ARET; + break; + case OGOTO: + a = AJMP; + break; + } + nextpc(); + if(a == AGOK) { + diag(Z, "bad in gbranch %O", o); + nextpc(); + } + p->as = a; +} + +void +patch(Prog *op, int32 pc) +{ + + op->to.offset = pc; + op->to.type = D_BRANCH; +} + +void +gpseudo(int a, Sym *s, Node *n) +{ + + nextpc(); + p->as = a; + p->from.type = D_EXTERN; + p->from.sym = s; + p->from.scale = textflag; + textflag = 0; + + if(s->class == CSTATIC) + p->from.type = D_STATIC; + naddr(n, &p->to); + if(a == ADATA || a == AGLOBL) + pc--; +} + +int +sconst(Node *n) +{ + int32 v; + + if(n->op == OCONST && !typefd[n->type->etype]) { + v = n->vconst; + if(v >= -32766L && v < 32766L) + return 1; + } + return 0; +} + +int32 +exreg(Type *t) +{ + int32 o; + + if(typechlp[t->etype]){ + if(exregoffset >= 32) + return 0; + o = exregoffset; + exregoffset += 4; + return o+1; // +1 to avoid 0 == failure; naddr case OEXREG will -1. + } + + return 0; +} + +schar ewidth[NTYPE] = +{ + -1, /*[TXXX]*/ + SZ_CHAR, /*[TCHAR]*/ + SZ_CHAR, /*[TUCHAR]*/ + SZ_SHORT, /*[TSHORT]*/ + SZ_SHORT, /*[TUSHORT]*/ + SZ_INT, /*[TINT]*/ + SZ_INT, /*[TUINT]*/ + SZ_LONG, /*[TLONG]*/ + SZ_LONG, /*[TULONG]*/ + SZ_VLONG, /*[TVLONG]*/ + SZ_VLONG, /*[TUVLONG]*/ + SZ_FLOAT, /*[TFLOAT]*/ + SZ_DOUBLE, /*[TDOUBLE]*/ + SZ_IND, /*[TIND]*/ + 0, /*[TFUNC]*/ + -1, /*[TARRAY]*/ + 0, /*[TVOID]*/ + -1, /*[TSTRUCT]*/ + -1, /*[TUNION]*/ + SZ_INT, /*[TENUM]*/ +}; +int32 ncast[NTYPE] = +{ + 0, /*[TXXX]*/ + BCHAR|BUCHAR, /*[TCHAR]*/ + BCHAR|BUCHAR, /*[TUCHAR]*/ + BSHORT|BUSHORT, /*[TSHORT]*/ + BSHORT|BUSHORT, /*[TUSHORT]*/ + BINT|BUINT|BLONG|BULONG|BIND, /*[TINT]*/ + BINT|BUINT|BLONG|BULONG|BIND, /*[TUINT]*/ + BINT|BUINT|BLONG|BULONG|BIND, /*[TLONG]*/ + BINT|BUINT|BLONG|BULONG|BIND, /*[TULONG]*/ + BVLONG|BUVLONG, /*[TVLONG]*/ + BVLONG|BUVLONG, /*[TUVLONG]*/ + BFLOAT, /*[TFLOAT]*/ + BDOUBLE, /*[TDOUBLE]*/ + BLONG|BULONG|BIND, /*[TIND]*/ + 0, /*[TFUNC]*/ + 0, /*[TARRAY]*/ + 0, /*[TVOID]*/ + BSTRUCT, /*[TSTRUCT]*/ + BUNION, /*[TUNION]*/ + 0, /*[TENUM]*/ +}; diff --git a/src/cmd/8g/Makefile b/src/cmd/8g/Makefile new file mode 100644 index 000000000..b459782a3 --- /dev/null +++ b/src/cmd/8g/Makefile @@ -0,0 +1,36 @@ +# Copyright 2009 The Go Authors. All rights reserved. +# Use of this source code is governed by a BSD-style +# license that can be found in the LICENSE file. + +include ../../Make.inc +O:=$(HOST_O) + +TARG=8g + +HFILES=\ + ../gc/go.h\ + ../8l/8.out.h\ + gg.h\ + opt.h\ + +OFILES=\ + ../8l/enam.$O\ + cgen.$O\ + cgen64.$O\ + cplx.$O\ + galign.$O\ + ggen.$O\ + gobj.$O\ + gsubr.$O\ + list.$O\ + peep.$O\ + pgen.$O\ + reg.$O\ + +LIB=\ + ../gc/gc.a\ + +include ../../Make.ccmd + +%.$O: ../gc/%.c + $(HOST_CC) $(HOST_CFLAGS) -c -I. -o $@ ../gc/$*.c diff --git a/src/cmd/8g/cgen.c b/src/cmd/8g/cgen.c new file mode 100644 index 000000000..b316e6e34 --- /dev/null +++ b/src/cmd/8g/cgen.c @@ -0,0 +1,1233 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// TODO(rsc): +// assume CLD? + +#include "gg.h" + +void +mgen(Node *n, Node *n1, Node *rg) +{ + Node n2; + + n1->op = OEMPTY; + + if(n->addable) { + *n1 = *n; + if(n1->op == OREGISTER || n1->op == OINDREG) + reg[n->val.u.reg]++; + return; + } + tempname(n1, n->type); + cgen(n, n1); + if(n->type->width <= widthptr || isfloat[n->type->etype]) { + n2 = *n1; + regalloc(n1, n->type, rg); + gmove(&n2, n1); + } +} + +void +mfree(Node *n) +{ + if(n->op == OREGISTER) + regfree(n); +} + +/* + * generate: + * res = n; + * simplifies and calls gmove. + * + * TODO: + * sudoaddable + */ +void +cgen(Node *n, Node *res) +{ + Node *nl, *nr, *r, n1, n2, nt, f0, f1; + Prog *p1, *p2, *p3; + int a; + + if(debug['g']) { + dump("\ncgen-n", n); + dump("cgen-res", res); + } + + if(n == N || n->type == T) + fatal("cgen: n nil"); + if(res == N || res->type == T) + fatal("cgen: res nil"); + + // inline slices + if(cgen_inline(n, res)) + return; + + while(n->op == OCONVNOP) + n = n->left; + + // function calls on both sides? introduce temporary + if(n->ullman >= UINF && res->ullman >= UINF) { + tempname(&n1, n->type); + cgen(n, &n1); + cgen(&n1, res); + return; + } + + // structs etc get handled specially + if(isfat(n->type)) { + if(n->type->width < 0) + fatal("forgot to compute width for %T", n->type); + sgen(n, res, n->type->width); + return; + } + + // update addressability for string, slice + // can't do in walk because n->left->addable + // changes if n->left is an escaping local variable. + switch(n->op) { + case OLEN: + if(isslice(n->left->type) || istype(n->left->type, TSTRING)) + n->addable = n->left->addable; + break; + case OCAP: + if(isslice(n->left->type)) + n->addable = n->left->addable; + break; + } + + // if both are addressable, move + if(n->addable && res->addable) { + gmove(n, res); + return; + } + + // if both are not addressable, use a temporary. + if(!n->addable && !res->addable) { + // could use regalloc here sometimes, + // but have to check for ullman >= UINF. + tempname(&n1, n->type); + cgen(n, &n1); + cgen(&n1, res); + return; + } + + // if result is not addressable directly but n is, + // compute its address and then store via the address. + if(!res->addable) { + igen(res, &n1, N); + cgen(n, &n1); + regfree(&n1); + return; + } + + // complex types + if(complexop(n, res)) { + complexgen(n, res); + return; + } + + // otherwise, the result is addressable but n is not. + // let's do some computation. + + // use ullman to pick operand to eval first. + nl = n->left; + nr = n->right; + if(nl != N && nl->ullman >= UINF) + if(nr != N && nr->ullman >= UINF) { + // both are hard + tempname(&n1, nl->type); + cgen(nl, &n1); + n2 = *n; + n2.left = &n1; + cgen(&n2, res); + return; + } + + // 64-bit ops are hard on 32-bit machine. + if(is64(n->type) || is64(res->type) || n->left != N && is64(n->left->type)) { + switch(n->op) { + // math goes to cgen64. + case OMINUS: + case OCOM: + case OADD: + case OSUB: + case OMUL: + case OLSH: + case ORSH: + case OAND: + case OOR: + case OXOR: + cgen64(n, res); + return; + } + } + + if(nl != N && isfloat[n->type->etype] && isfloat[nl->type->etype]) + goto flt; + + switch(n->op) { + default: + dump("cgen", n); + fatal("cgen %O", n->op); + break; + + case OREAL: + case OIMAG: + case OCOMPLEX: + fatal("unexpected complex"); + return; + + // these call bgen to get a bool value + case OOROR: + case OANDAND: + case OEQ: + case ONE: + case OLT: + case OLE: + case OGE: + case OGT: + case ONOT: + p1 = gbranch(AJMP, T); + p2 = pc; + gmove(nodbool(1), res); + p3 = gbranch(AJMP, T); + patch(p1, pc); + bgen(n, 1, p2); + gmove(nodbool(0), res); + patch(p3, pc); + return; + + case OPLUS: + cgen(nl, res); + return; + + case OMINUS: + case OCOM: + a = optoas(n->op, nl->type); + goto uop; + + // symmetric binary + case OAND: + case OOR: + case OXOR: + case OADD: + case OMUL: + a = optoas(n->op, nl->type); + if(a == AIMULB) { + cgen_bmul(n->op, nl, nr, res); + break; + } + goto sbop; + + // asymmetric binary + case OSUB: + a = optoas(n->op, nl->type); + goto abop; + + case OCONV: + if(eqtype(n->type, nl->type) || noconv(n->type, nl->type)) { + cgen(nl, res); + break; + } + + tempname(&n2, n->type); + mgen(nl, &n1, res); + gmove(&n1, &n2); + gmove(&n2, res); + mfree(&n1); + break; + + case ODOT: + case ODOTPTR: + case OINDEX: + case OIND: + case ONAME: // PHEAP or PPARAMREF var + igen(n, &n1, res); + gmove(&n1, res); + regfree(&n1); + break; + + case OLEN: + if(istype(nl->type, TMAP) || istype(nl->type, TCHAN)) { + // map has len in the first 32-bit word. + // a zero pointer means zero length + tempname(&n1, types[tptr]); + cgen(nl, &n1); + regalloc(&n2, types[tptr], N); + gmove(&n1, &n2); + n1 = n2; + + nodconst(&n2, types[tptr], 0); + gins(optoas(OCMP, types[tptr]), &n1, &n2); + p1 = gbranch(optoas(OEQ, types[tptr]), T); + + n2 = n1; + n2.op = OINDREG; + n2.type = types[TINT32]; + gmove(&n2, &n1); + + patch(p1, pc); + + gmove(&n1, res); + regfree(&n1); + break; + } + if(istype(nl->type, TSTRING) || isslice(nl->type)) { + // both slice and string have len one pointer into the struct. + igen(nl, &n1, res); + n1.type = types[TUINT32]; + n1.xoffset += Array_nel; + gmove(&n1, res); + regfree(&n1); + break; + } + fatal("cgen: OLEN: unknown type %lT", nl->type); + break; + + case OCAP: + if(istype(nl->type, TCHAN)) { + // chan has cap in the second 32-bit word. + // a zero pointer means zero length + regalloc(&n1, types[tptr], res); + cgen(nl, &n1); + + nodconst(&n2, types[tptr], 0); + gins(optoas(OCMP, types[tptr]), &n1, &n2); + p1 = gbranch(optoas(OEQ, types[tptr]), T); + + n2 = n1; + n2.op = OINDREG; + n2.xoffset = 4; + n2.type = types[TINT32]; + gmove(&n2, &n1); + + patch(p1, pc); + + gmove(&n1, res); + regfree(&n1); + break; + } + if(isslice(nl->type)) { + igen(nl, &n1, res); + n1.op = OINDREG; + n1.type = types[TUINT32]; + n1.xoffset = Array_cap; + gmove(&n1, res); + regfree(&n1); + break; + } + fatal("cgen: OCAP: unknown type %lT", nl->type); + break; + + case OADDR: + agen(nl, res); + break; + + case OCALLMETH: + cgen_callmeth(n, 0); + cgen_callret(n, res); + break; + + case OCALLINTER: + cgen_callinter(n, res, 0); + cgen_callret(n, res); + break; + + case OCALLFUNC: + cgen_call(n, 0); + cgen_callret(n, res); + break; + + case OMOD: + case ODIV: + cgen_div(n->op, nl, nr, res); + break; + + case OLSH: + case ORSH: + cgen_shift(n->op, nl, nr, res); + break; + } + return; + +sbop: // symmetric binary + if(nl->ullman < nr->ullman) { + r = nl; + nl = nr; + nr = r; + } + +abop: // asymmetric binary + if(nl->ullman >= nr->ullman) { + tempname(&nt, nl->type); + cgen(nl, &nt); + mgen(nr, &n2, N); + regalloc(&n1, nl->type, res); + gmove(&nt, &n1); + gins(a, &n2, &n1); + gmove(&n1, res); + regfree(&n1); + mfree(&n2); + } else { + regalloc(&n2, nr->type, res); + cgen(nr, &n2); + regalloc(&n1, nl->type, N); + cgen(nl, &n1); + gins(a, &n2, &n1); + regfree(&n2); + gmove(&n1, res); + regfree(&n1); + } + return; + +uop: // unary + tempname(&n1, nl->type); + cgen(nl, &n1); + gins(a, N, &n1); + gmove(&n1, res); + return; + +flt: // floating-point. 387 (not SSE2) to interoperate with 8c + nodreg(&f0, nl->type, D_F0); + nodreg(&f1, n->type, D_F0+1); + if(nr != N) + goto flt2; + + // unary + cgen(nl, &f0); + if(n->op != OCONV && n->op != OPLUS) + gins(foptoas(n->op, n->type, 0), N, N); + gmove(&f0, res); + return; + +flt2: // binary + if(nl->ullman >= nr->ullman) { + cgen(nl, &f0); + if(nr->addable) + gins(foptoas(n->op, n->type, 0), nr, &f0); + else { + cgen(nr, &f0); + gins(foptoas(n->op, n->type, Fpop), &f0, &f1); + } + } else { + cgen(nr, &f0); + if(nl->addable) + gins(foptoas(n->op, n->type, Frev), nl, &f0); + else { + cgen(nl, &f0); + gins(foptoas(n->op, n->type, Frev|Fpop), &f0, &f1); + } + } + gmove(&f0, res); + return; +} + +/* + * generate array index into res. + * n might be any size; res is 32-bit. + * returns Prog* to patch to panic call. + */ +Prog* +cgenindex(Node *n, Node *res) +{ + Node tmp, lo, hi, zero; + + if(!is64(n->type)) { + cgen(n, res); + return nil; + } + + tempname(&tmp, types[TINT64]); + cgen(n, &tmp); + split64(&tmp, &lo, &hi); + gmove(&lo, res); + if(debug['B']) { + splitclean(); + return nil; + } + nodconst(&zero, types[TINT32], 0); + gins(ACMPL, &hi, &zero); + splitclean(); + return gbranch(AJNE, T); +} + +/* + * address gen + * res = &n; + */ +void +agen(Node *n, Node *res) +{ + Node *nl, *nr; + Node n1, n2, n3, n4, tmp; + Type *t; + uint32 w; + uint64 v; + Prog *p1, *p2; + + if(debug['g']) { + dump("\nagen-res", res); + dump("agen-r", n); + } + if(n == N || n->type == T || res == N || res->type == T) + fatal("agen"); + + while(n->op == OCONVNOP) + n = n->left; + + // addressable var is easy + if(n->addable) { + if(n->op == OREGISTER) + fatal("agen OREGISTER"); + regalloc(&n1, types[tptr], res); + gins(ALEAL, n, &n1); + gmove(&n1, res); + regfree(&n1); + return; + } + + // let's compute + nl = n->left; + nr = n->right; + + switch(n->op) { + default: + fatal("agen %O", n->op); + + case OCALLMETH: + cgen_callmeth(n, 0); + cgen_aret(n, res); + break; + + case OCALLINTER: + cgen_callinter(n, res, 0); + cgen_aret(n, res); + break; + + case OCALLFUNC: + cgen_call(n, 0); + cgen_aret(n, res); + break; + + case OINDEX: + p2 = nil; // to be patched to panicindex. + w = n->type->width; + if(nr->addable) { + if(!isconst(nr, CTINT)) + tempname(&tmp, types[TINT32]); + if(!isconst(nl, CTSTR)) + agenr(nl, &n3, res); + if(!isconst(nr, CTINT)) { + p2 = cgenindex(nr, &tmp); + regalloc(&n1, tmp.type, N); + gmove(&tmp, &n1); + } + } else if(nl->addable) { + if(!isconst(nr, CTINT)) { + tempname(&tmp, types[TINT32]); + p2 = cgenindex(nr, &tmp); + regalloc(&n1, tmp.type, N); + gmove(&tmp, &n1); + } + if(!isconst(nl, CTSTR)) { + regalloc(&n3, types[tptr], res); + agen(nl, &n3); + } + } else { + tempname(&tmp, types[TINT32]); + p2 = cgenindex(nr, &tmp); + nr = &tmp; + if(!isconst(nl, CTSTR)) + agenr(nl, &n3, res); + regalloc(&n1, tmp.type, N); + gins(optoas(OAS, tmp.type), &tmp, &n1); + } + + // &a is in &n3 (allocated in res) + // i is in &n1 (if not constant) + // w is width + + // explicit check for nil if array is large enough + // that we might derive too big a pointer. + if(isfixedarray(nl->type) && nl->type->width >= unmappedzero) { + regalloc(&n4, types[tptr], &n3); + gmove(&n3, &n4); + n4.op = OINDREG; + n4.type = types[TUINT8]; + n4.xoffset = 0; + gins(ATESTB, nodintconst(0), &n4); + regfree(&n4); + } + + // constant index + if(isconst(nr, CTINT)) { + if(isconst(nl, CTSTR)) + fatal("constant string constant index"); + v = mpgetfix(nr->val.u.xval); + if(isslice(nl->type) || nl->type->etype == TSTRING) { + if(!debug['B'] && !n->etype) { + n1 = n3; + n1.op = OINDREG; + n1.type = types[tptr]; + n1.xoffset = Array_nel; + nodconst(&n2, types[TUINT32], v); + gins(optoas(OCMP, types[TUINT32]), &n1, &n2); + p1 = gbranch(optoas(OGT, types[TUINT32]), T); + ginscall(panicindex, 0); + patch(p1, pc); + } + + n1 = n3; + n1.op = OINDREG; + n1.type = types[tptr]; + n1.xoffset = Array_array; + gmove(&n1, &n3); + } + + if (v*w != 0) { + nodconst(&n2, types[tptr], v*w); + gins(optoas(OADD, types[tptr]), &n2, &n3); + } + gmove(&n3, res); + regfree(&n3); + break; + } + + regalloc(&n2, types[TINT32], &n1); // i + gmove(&n1, &n2); + regfree(&n1); + + if(!debug['B'] && !n->etype) { + // check bounds + if(isconst(nl, CTSTR)) + nodconst(&n1, types[TUINT32], nl->val.u.sval->len); + else if(isslice(nl->type) || nl->type->etype == TSTRING) { + n1 = n3; + n1.op = OINDREG; + n1.type = types[tptr]; + n1.xoffset = Array_nel; + } else + nodconst(&n1, types[TUINT32], nl->type->bound); + gins(optoas(OCMP, types[TUINT32]), &n2, &n1); + p1 = gbranch(optoas(OLT, types[TUINT32]), T); + if(p2) + patch(p2, pc); + ginscall(panicindex, 0); + patch(p1, pc); + } + + if(isconst(nl, CTSTR)) { + regalloc(&n3, types[tptr], res); + p1 = gins(ALEAL, N, &n3); + datastring(nl->val.u.sval->s, nl->val.u.sval->len, &p1->from); + p1->from.scale = 1; + p1->from.index = n2.val.u.reg; + goto indexdone; + } + + if(isslice(nl->type) || nl->type->etype == TSTRING) { + n1 = n3; + n1.op = OINDREG; + n1.type = types[tptr]; + n1.xoffset = Array_array; + gmove(&n1, &n3); + } + + if(w == 0) { + // nothing to do + } else if(w == 1 || w == 2 || w == 4 || w == 8) { + p1 = gins(ALEAL, &n2, &n3); + p1->from.scale = w; + p1->from.index = p1->from.type; + p1->from.type = p1->to.type + D_INDIR; + } else { + nodconst(&n1, types[TUINT32], w); + gins(optoas(OMUL, types[TUINT32]), &n1, &n2); + gins(optoas(OADD, types[tptr]), &n2, &n3); + } + + indexdone: + gmove(&n3, res); + regfree(&n2); + regfree(&n3); + break; + + case ONAME: + // should only get here with names in this func. + if(n->funcdepth > 0 && n->funcdepth != funcdepth) { + dump("bad agen", n); + fatal("agen: bad ONAME funcdepth %d != %d", + n->funcdepth, funcdepth); + } + + // should only get here for heap vars or paramref + if(!(n->class & PHEAP) && n->class != PPARAMREF) { + dump("bad agen", n); + fatal("agen: bad ONAME class %#x", n->class); + } + cgen(n->heapaddr, res); + if(n->xoffset != 0) { + nodconst(&n1, types[tptr], n->xoffset); + gins(optoas(OADD, types[tptr]), &n1, res); + } + break; + + case OIND: + cgen(nl, res); + break; + + case ODOT: + t = nl->type; + agen(nl, res); + if(n->xoffset != 0) { + nodconst(&n1, types[tptr], n->xoffset); + gins(optoas(OADD, types[tptr]), &n1, res); + } + break; + + case ODOTPTR: + t = nl->type; + if(!isptr[t->etype]) + fatal("agen: not ptr %N", n); + cgen(nl, res); + if(n->xoffset != 0) { + // explicit check for nil if struct is large enough + // that we might derive too big a pointer. + if(nl->type->type->width >= unmappedzero) { + regalloc(&n1, types[tptr], res); + gmove(res, &n1); + n1.op = OINDREG; + n1.type = types[TUINT8]; + n1.xoffset = 0; + gins(ATESTB, nodintconst(0), &n1); + regfree(&n1); + } + nodconst(&n1, types[tptr], n->xoffset); + gins(optoas(OADD, types[tptr]), &n1, res); + } + break; + } +} + +/* + * generate: + * newreg = &n; + * res = newreg + * + * on exit, a has been changed to be *newreg. + * caller must regfree(a). + */ +void +igen(Node *n, Node *a, Node *res) +{ + Node n1; + Type *fp; + Iter flist; + + switch(n->op) { + case ONAME: + if((n->class&PHEAP) || n->class == PPARAMREF) + break; + *a = *n; + return; + + case OCALLFUNC: + fp = structfirst(&flist, getoutarg(n->left->type)); + cgen_call(n, 0); + memset(a, 0, sizeof *a); + a->op = OINDREG; + a->val.u.reg = D_SP; + a->addable = 1; + a->xoffset = fp->width; + a->type = n->type; + return; + } + // release register for now, to avoid + // confusing tempname. + if(res != N && res->op == OREGISTER) + reg[res->val.u.reg]--; + tempname(&n1, types[tptr]); + agen(n, &n1); + if(res != N && res->op == OREGISTER) + reg[res->val.u.reg]++; + regalloc(a, types[tptr], res); + gmove(&n1, a); + a->op = OINDREG; + a->type = n->type; +} + +/* + * generate: + * newreg = &n; + * + * caller must regfree(a). + */ +void +agenr(Node *n, Node *a, Node *res) +{ + Node n1; + + tempname(&n1, types[tptr]); + agen(n, &n1); + regalloc(a, types[tptr], res); + gmove(&n1, a); +} + +/* + * branch gen + * if(n == true) goto to; + */ +void +bgen(Node *n, int true, Prog *to) +{ + int et, a; + Node *nl, *nr, *r; + Node n1, n2, tmp, t1, t2, ax; + Prog *p1, *p2; + + if(debug['g']) { + dump("\nbgen", n); + } + + if(n == N) + n = nodbool(1); + + if(n->ninit != nil) + genlist(n->ninit); + + nl = n->left; + nr = n->right; + + if(n->type == T) { + convlit(&n, types[TBOOL]); + if(n->type == T) + return; + } + + et = n->type->etype; + if(et != TBOOL) { + yyerror("cgen: bad type %T for %O", n->type, n->op); + patch(gins(AEND, N, N), to); + return; + } + nl = N; + nr = N; + + switch(n->op) { + default: + def: + regalloc(&n1, n->type, N); + cgen(n, &n1); + nodconst(&n2, n->type, 0); + gins(optoas(OCMP, n->type), &n1, &n2); + a = AJNE; + if(!true) + a = AJEQ; + patch(gbranch(a, n->type), to); + regfree(&n1); + return; + + case OLITERAL: + // need to ask if it is bool? + if(!true == !n->val.u.bval) + patch(gbranch(AJMP, T), to); + return; + + case ONAME: + if(!n->addable) + goto def; + nodconst(&n1, n->type, 0); + gins(optoas(OCMP, n->type), n, &n1); + a = AJNE; + if(!true) + a = AJEQ; + patch(gbranch(a, n->type), to); + return; + + case OANDAND: + if(!true) + goto caseor; + + caseand: + p1 = gbranch(AJMP, T); + p2 = gbranch(AJMP, T); + patch(p1, pc); + bgen(n->left, !true, p2); + bgen(n->right, !true, p2); + p1 = gbranch(AJMP, T); + patch(p1, to); + patch(p2, pc); + return; + + case OOROR: + if(!true) + goto caseand; + + caseor: + bgen(n->left, true, to); + bgen(n->right, true, to); + return; + + case OEQ: + case ONE: + case OLT: + case OGT: + case OLE: + case OGE: + nr = n->right; + if(nr == N || nr->type == T) + return; + + case ONOT: // unary + nl = n->left; + if(nl == N || nl->type == T) + return; + } + + switch(n->op) { + case ONOT: + bgen(nl, !true, to); + break; + + case OEQ: + case ONE: + case OLT: + case OGT: + case OLE: + case OGE: + a = n->op; + if(!true) { + if(isfloat[nl->type->etype]) { + // brcom is not valid on floats when NaN is involved. + p1 = gbranch(AJMP, T); + p2 = gbranch(AJMP, T); + patch(p1, pc); + bgen(n, 1, p2); + patch(gbranch(AJMP, T), to); + patch(p2, pc); + break; + } + a = brcom(a); + true = !true; + } + + // make simplest on right + if(nl->op == OLITERAL || (nl->ullman < nr->ullman && nl->ullman < UINF)) { + a = brrev(a); + r = nl; + nl = nr; + nr = r; + } + + if(isslice(nl->type)) { + // front end should only leave cmp to literal nil + if((a != OEQ && a != ONE) || nr->op != OLITERAL) { + yyerror("illegal array comparison"); + break; + } + a = optoas(a, types[tptr]); + regalloc(&n1, types[tptr], N); + agen(nl, &n1); + n2 = n1; + n2.op = OINDREG; + n2.xoffset = Array_array; + n2.type = types[tptr]; + nodconst(&tmp, types[tptr], 0); + gins(optoas(OCMP, types[tptr]), &n2, &tmp); + patch(gbranch(a, types[tptr]), to); + regfree(&n1); + break; + } + + if(isinter(nl->type)) { + // front end should only leave cmp to literal nil + if((a != OEQ && a != ONE) || nr->op != OLITERAL) { + yyerror("illegal interface comparison"); + break; + } + a = optoas(a, types[tptr]); + regalloc(&n1, types[tptr], N); + agen(nl, &n1); + n2 = n1; + n2.op = OINDREG; + n2.xoffset = 0; + nodconst(&tmp, types[tptr], 0); + gins(optoas(OCMP, types[tptr]), &n2, &tmp); + patch(gbranch(a, types[tptr]), to); + regfree(&n1); + break; + } + + if(isfloat[nr->type->etype]) { + a = brrev(a); // because the args are stacked + if(a == OGE || a == OGT) { + // only < and <= work right with NaN; reverse if needed + r = nr; + nr = nl; + nl = r; + a = brrev(a); + } + nodreg(&tmp, nr->type, D_F0); + nodreg(&n2, nr->type, D_F0 + 1); + nodreg(&ax, types[TUINT16], D_AX); + et = simsimtype(nr->type); + if(et == TFLOAT64) { + if(nl->ullman > nr->ullman) { + cgen(nl, &tmp); + cgen(nr, &tmp); + gins(AFXCHD, &tmp, &n2); + } else { + cgen(nr, &tmp); + cgen(nl, &tmp); + } + gins(AFUCOMIP, &tmp, &n2); + gins(AFMOVDP, &tmp, &tmp); // annoying pop but still better than STSW+SAHF + } else { + // TODO(rsc): The moves back and forth to memory + // here are for truncating the value to 32 bits. + // This handles 32-bit comparison but presumably + // all the other ops have the same problem. + // We need to figure out what the right general + // solution is, besides telling people to use float64. + tempname(&t1, types[TFLOAT32]); + tempname(&t2, types[TFLOAT32]); + cgen(nr, &t1); + cgen(nl, &t2); + gmove(&t2, &tmp); + gins(AFCOMFP, &t1, &tmp); + gins(AFSTSW, N, &ax); + gins(ASAHF, N, N); + } + if(a == OEQ) { + // neither NE nor P + p1 = gbranch(AJNE, T); + p2 = gbranch(AJPS, T); + patch(gbranch(AJMP, T), to); + patch(p1, pc); + patch(p2, pc); + } else if(a == ONE) { + // either NE or P + patch(gbranch(AJNE, T), to); + patch(gbranch(AJPS, T), to); + } else + patch(gbranch(optoas(a, nr->type), T), to); + break; + } + if(iscomplex[nl->type->etype]) { + complexbool(a, nl, nr, true, to); + break; + } + + if(is64(nr->type)) { + if(!nl->addable) { + tempname(&n1, nl->type); + cgen(nl, &n1); + nl = &n1; + } + if(!nr->addable) { + tempname(&n2, nr->type); + cgen(nr, &n2); + nr = &n2; + } + cmp64(nl, nr, a, to); + break; + } + + a = optoas(a, nr->type); + + if(nr->ullman >= UINF) { + tempname(&n1, nl->type); + tempname(&tmp, nr->type); + cgen(nl, &n1); + cgen(nr, &tmp); + regalloc(&n2, nr->type, N); + cgen(&tmp, &n2); + goto cmp; + } + + tempname(&n1, nl->type); + cgen(nl, &n1); + + if(smallintconst(nr)) { + gins(optoas(OCMP, nr->type), &n1, nr); + patch(gbranch(a, nr->type), to); + break; + } + + tempname(&tmp, nr->type); + cgen(nr, &tmp); + regalloc(&n2, nr->type, N); + gmove(&tmp, &n2); + +cmp: + gins(optoas(OCMP, nr->type), &n1, &n2); + patch(gbranch(a, nr->type), to); + regfree(&n2); + break; + } +} + +/* + * n is on stack, either local variable + * or return value from function call. + * return n's offset from SP. + */ +int32 +stkof(Node *n) +{ + Type *t; + Iter flist; + int32 off; + + switch(n->op) { + case OINDREG: + return n->xoffset; + + case ODOT: + t = n->left->type; + if(isptr[t->etype]) + break; + off = stkof(n->left); + if(off == -1000 || off == 1000) + return off; + return off + n->xoffset; + + case OINDEX: + t = n->left->type; + if(!isfixedarray(t)) + break; + off = stkof(n->left); + if(off == -1000 || off == 1000) + return off; + if(isconst(n->right, CTINT)) + return off + t->type->width * mpgetfix(n->right->val.u.xval); + return 1000; + + case OCALLMETH: + case OCALLINTER: + case OCALLFUNC: + t = n->left->type; + if(isptr[t->etype]) + t = t->type; + + t = structfirst(&flist, getoutarg(t)); + if(t != T) + return t->width; + break; + } + + // botch - probably failing to recognize address + // arithmetic on the above. eg INDEX and DOT + return -1000; +} + +/* + * struct gen + * memmove(&res, &n, w); + */ +void +sgen(Node *n, Node *res, int32 w) +{ + Node dst, src, tdst, tsrc; + int32 c, q, odst, osrc; + + if(debug['g']) { + print("\nsgen w=%d\n", w); + dump("r", n); + dump("res", res); + } + if(w == 0) + return; + if(n->ullman >= UINF && res->ullman >= UINF) { + fatal("sgen UINF"); + } + + if(w < 0) + fatal("sgen copy %d", w); + + // offset on the stack + osrc = stkof(n); + odst = stkof(res); + + if(osrc != -1000 && odst != -1000 && (osrc == 1000 || odst == 1000)) { + // osrc and odst both on stack, and at least one is in + // an unknown position. Could generate code to test + // for forward/backward copy, but instead just copy + // to a temporary location first. + tempname(&tsrc, n->type); + sgen(n, &tsrc, w); + sgen(&tsrc, res, w); + return; + } + + nodreg(&dst, types[tptr], D_DI); + nodreg(&src, types[tptr], D_SI); + + tempname(&tsrc, types[tptr]); + tempname(&tdst, types[tptr]); + if(!n->addable) + agen(n, &tsrc); + if(!res->addable) + agen(res, &tdst); + if(n->addable) + agen(n, &src); + else + gmove(&tsrc, &src); + if(res->addable) + agen(res, &dst); + else + gmove(&tdst, &dst); + + c = w % 4; // bytes + q = w / 4; // doublewords + + // if we are copying forward on the stack and + // the src and dst overlap, then reverse direction + if(osrc < odst && odst < osrc+w) { + // reverse direction + gins(ASTD, N, N); // set direction flag + if(c > 0) { + gconreg(AADDL, w-1, D_SI); + gconreg(AADDL, w-1, D_DI); + + gconreg(AMOVL, c, D_CX); + gins(AREP, N, N); // repeat + gins(AMOVSB, N, N); // MOVB *(SI)-,*(DI)- + } + + if(q > 0) { + if(c > 0) { + gconreg(AADDL, -3, D_SI); + gconreg(AADDL, -3, D_DI); + } else { + gconreg(AADDL, w-4, D_SI); + gconreg(AADDL, w-4, D_DI); + } + gconreg(AMOVL, q, D_CX); + gins(AREP, N, N); // repeat + gins(AMOVSL, N, N); // MOVL *(SI)-,*(DI)- + } + // we leave with the flag clear + gins(ACLD, N, N); + } else { + gins(ACLD, N, N); // paranoia. TODO(rsc): remove? + // normal direction + if(q >= 4) { + gconreg(AMOVL, q, D_CX); + gins(AREP, N, N); // repeat + gins(AMOVSL, N, N); // MOVL *(SI)+,*(DI)+ + } else + while(q > 0) { + gins(AMOVSL, N, N); // MOVL *(SI)+,*(DI)+ + q--; + } + while(c > 0) { + gins(AMOVSB, N, N); // MOVB *(SI)+,*(DI)+ + c--; + } + } +} + diff --git a/src/cmd/8g/cgen64.c b/src/cmd/8g/cgen64.c new file mode 100644 index 000000000..ba99cec74 --- /dev/null +++ b/src/cmd/8g/cgen64.c @@ -0,0 +1,512 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +#include "gg.h" + +/* + * attempt to generate 64-bit + * res = n + * return 1 on success, 0 if op not handled. + */ +void +cgen64(Node *n, Node *res) +{ + Node t1, t2, ax, dx, cx, ex, fx, *l, *r; + Node lo1, lo2, hi1, hi2; + Prog *p1, *p2; + uint64 v; + uint32 lv, hv; + + if(res->op != OINDREG && res->op != ONAME) { + dump("n", n); + dump("res", res); + fatal("cgen64 %O of %O", n->op, res->op); + } + switch(n->op) { + default: + fatal("cgen64 %O", n->op); + + case OMINUS: + cgen(n->left, res); + split64(res, &lo1, &hi1); + gins(ANEGL, N, &lo1); + gins(AADCL, ncon(0), &hi1); + gins(ANEGL, N, &hi1); + splitclean(); + return; + + case OCOM: + cgen(n->left, res); + split64(res, &lo1, &hi1); + gins(ANOTL, N, &lo1); + gins(ANOTL, N, &hi1); + splitclean(); + return; + + case OADD: + case OSUB: + case OMUL: + case OLSH: + case ORSH: + case OAND: + case OOR: + case OXOR: + // binary operators. + // common setup below. + break; + } + + l = n->left; + r = n->right; + if(!l->addable) { + tempname(&t1, l->type); + cgen(l, &t1); + l = &t1; + } + if(r != N && !r->addable) { + tempname(&t2, r->type); + cgen(r, &t2); + r = &t2; + } + + nodreg(&ax, types[TINT32], D_AX); + nodreg(&cx, types[TINT32], D_CX); + nodreg(&dx, types[TINT32], D_DX); + + // Setup for binary operation. + split64(l, &lo1, &hi1); + if(is64(r->type)) + split64(r, &lo2, &hi2); + + // Do op. Leave result in DX:AX. + switch(n->op) { + case OADD: + // TODO: Constants + gins(AMOVL, &lo1, &ax); + gins(AMOVL, &hi1, &dx); + gins(AADDL, &lo2, &ax); + gins(AADCL, &hi2, &dx); + break; + + case OSUB: + // TODO: Constants. + gins(AMOVL, &lo1, &ax); + gins(AMOVL, &hi1, &dx); + gins(ASUBL, &lo2, &ax); + gins(ASBBL, &hi2, &dx); + break; + + case OMUL: + // let's call the next two EX and FX. + regalloc(&ex, types[TPTR32], N); + regalloc(&fx, types[TPTR32], N); + + // load args into DX:AX and EX:CX. + gins(AMOVL, &lo1, &ax); + gins(AMOVL, &hi1, &dx); + gins(AMOVL, &lo2, &cx); + gins(AMOVL, &hi2, &ex); + + // if DX and EX are zero, use 32 x 32 -> 64 unsigned multiply. + gins(AMOVL, &dx, &fx); + gins(AORL, &ex, &fx); + p1 = gbranch(AJNE, T); + gins(AMULL, &cx, N); // implicit &ax + p2 = gbranch(AJMP, T); + patch(p1, pc); + + // full 64x64 -> 64, from 32x32 -> 64. + gins(AIMULL, &cx, &dx); + gins(AMOVL, &ax, &fx); + gins(AIMULL, &ex, &fx); + gins(AADDL, &dx, &fx); + gins(AMOVL, &cx, &dx); + gins(AMULL, &dx, N); // implicit &ax + gins(AADDL, &fx, &dx); + patch(p2, pc); + + regfree(&ex); + regfree(&fx); + break; + + case OLSH: + if(r->op == OLITERAL) { + v = mpgetfix(r->val.u.xval); + if(v >= 64) { + if(is64(r->type)) + splitclean(); + splitclean(); + split64(res, &lo2, &hi2); + gins(AMOVL, ncon(0), &lo2); + gins(AMOVL, ncon(0), &hi2); + splitclean(); + goto out; + } + if(v >= 32) { + if(is64(r->type)) + splitclean(); + split64(res, &lo2, &hi2); + gmove(&lo1, &hi2); + if(v > 32) { + gins(ASHLL, ncon(v - 32), &hi2); + } + gins(AMOVL, ncon(0), &lo2); + splitclean(); + splitclean(); + goto out; + } + + // general shift + gins(AMOVL, &lo1, &ax); + gins(AMOVL, &hi1, &dx); + p1 = gins(ASHLL, ncon(v), &dx); + p1->from.index = D_AX; // double-width shift + p1->from.scale = 0; + gins(ASHLL, ncon(v), &ax); + break; + } + + // load value into DX:AX. + gins(AMOVL, &lo1, &ax); + gins(AMOVL, &hi1, &dx); + + // load shift value into register. + // if high bits are set, zero value. + p1 = P; + if(is64(r->type)) { + gins(ACMPL, &hi2, ncon(0)); + p1 = gbranch(AJNE, T); + gins(AMOVL, &lo2, &cx); + } else { + cx.type = types[TUINT32]; + gmove(r, &cx); + } + + // if shift count is >=64, zero value + gins(ACMPL, &cx, ncon(64)); + p2 = gbranch(optoas(OLT, types[TUINT32]), T); + if(p1 != P) + patch(p1, pc); + gins(AXORL, &dx, &dx); + gins(AXORL, &ax, &ax); + patch(p2, pc); + + // if shift count is >= 32, zero low. + gins(ACMPL, &cx, ncon(32)); + p1 = gbranch(optoas(OLT, types[TUINT32]), T); + gins(AMOVL, &ax, &dx); + gins(ASHLL, &cx, &dx); // SHLL only uses bottom 5 bits of count + gins(AXORL, &ax, &ax); + p2 = gbranch(AJMP, T); + patch(p1, pc); + + // general shift + p1 = gins(ASHLL, &cx, &dx); + p1->from.index = D_AX; // double-width shift + p1->from.scale = 0; + gins(ASHLL, &cx, &ax); + patch(p2, pc); + break; + + case ORSH: + if(r->op == OLITERAL) { + v = mpgetfix(r->val.u.xval); + if(v >= 64) { + if(is64(r->type)) + splitclean(); + splitclean(); + split64(res, &lo2, &hi2); + if(hi1.type->etype == TINT32) { + gmove(&hi1, &lo2); + gins(ASARL, ncon(31), &lo2); + gmove(&hi1, &hi2); + gins(ASARL, ncon(31), &hi2); + } else { + gins(AMOVL, ncon(0), &lo2); + gins(AMOVL, ncon(0), &hi2); + } + splitclean(); + goto out; + } + if(v >= 32) { + if(is64(r->type)) + splitclean(); + split64(res, &lo2, &hi2); + gmove(&hi1, &lo2); + if(v > 32) + gins(optoas(ORSH, hi1.type), ncon(v-32), &lo2); + if(hi1.type->etype == TINT32) { + gmove(&hi1, &hi2); + gins(ASARL, ncon(31), &hi2); + } else + gins(AMOVL, ncon(0), &hi2); + splitclean(); + splitclean(); + goto out; + } + + // general shift + gins(AMOVL, &lo1, &ax); + gins(AMOVL, &hi1, &dx); + p1 = gins(ASHRL, ncon(v), &ax); + p1->from.index = D_DX; // double-width shift + p1->from.scale = 0; + gins(optoas(ORSH, hi1.type), ncon(v), &dx); + break; + } + + // load value into DX:AX. + gins(AMOVL, &lo1, &ax); + gins(AMOVL, &hi1, &dx); + + // load shift value into register. + // if high bits are set, zero value. + p1 = P; + if(is64(r->type)) { + gins(ACMPL, &hi2, ncon(0)); + p1 = gbranch(AJNE, T); + gins(AMOVL, &lo2, &cx); + } else { + cx.type = types[TUINT32]; + gmove(r, &cx); + } + + // if shift count is >=64, zero or sign-extend value + gins(ACMPL, &cx, ncon(64)); + p2 = gbranch(optoas(OLT, types[TUINT32]), T); + if(p1 != P) + patch(p1, pc); + if(hi1.type->etype == TINT32) { + gins(ASARL, ncon(31), &dx); + gins(AMOVL, &dx, &ax); + } else { + gins(AXORL, &dx, &dx); + gins(AXORL, &ax, &ax); + } + patch(p2, pc); + + // if shift count is >= 32, sign-extend hi. + gins(ACMPL, &cx, ncon(32)); + p1 = gbranch(optoas(OLT, types[TUINT32]), T); + gins(AMOVL, &dx, &ax); + if(hi1.type->etype == TINT32) { + gins(ASARL, &cx, &ax); // SARL only uses bottom 5 bits of count + gins(ASARL, ncon(31), &dx); + } else { + gins(ASHRL, &cx, &ax); + gins(AXORL, &dx, &dx); + } + p2 = gbranch(AJMP, T); + patch(p1, pc); + + // general shift + p1 = gins(ASHRL, &cx, &ax); + p1->from.index = D_DX; // double-width shift + p1->from.scale = 0; + gins(optoas(ORSH, hi1.type), &cx, &dx); + patch(p2, pc); + break; + + case OXOR: + case OAND: + case OOR: + // make constant the right side (it usually is anyway). + if(lo1.op == OLITERAL) { + nswap(&lo1, &lo2); + nswap(&hi1, &hi2); + } + if(lo2.op == OLITERAL) { + // special cases for constants. + lv = mpgetfix(lo2.val.u.xval); + hv = mpgetfix(hi2.val.u.xval); + splitclean(); // right side + split64(res, &lo2, &hi2); + switch(n->op) { + case OXOR: + gmove(&lo1, &lo2); + gmove(&hi1, &hi2); + switch(lv) { + case 0: + break; + case 0xffffffffu: + gins(ANOTL, N, &lo2); + break; + default: + gins(AXORL, ncon(lv), &lo2); + break; + } + switch(hv) { + case 0: + break; + case 0xffffffffu: + gins(ANOTL, N, &hi2); + break; + default: + gins(AXORL, ncon(hv), &hi2); + break; + } + break; + + case OAND: + switch(lv) { + case 0: + gins(AMOVL, ncon(0), &lo2); + break; + default: + gmove(&lo1, &lo2); + if(lv != 0xffffffffu) + gins(AANDL, ncon(lv), &lo2); + break; + } + switch(hv) { + case 0: + gins(AMOVL, ncon(0), &hi2); + break; + default: + gmove(&hi1, &hi2); + if(hv != 0xffffffffu) + gins(AANDL, ncon(hv), &hi2); + break; + } + break; + + case OOR: + switch(lv) { + case 0: + gmove(&lo1, &lo2); + break; + case 0xffffffffu: + gins(AMOVL, ncon(0xffffffffu), &lo2); + break; + default: + gmove(&lo1, &lo2); + gins(AORL, ncon(lv), &lo2); + break; + } + switch(hv) { + case 0: + gmove(&hi1, &hi2); + break; + case 0xffffffffu: + gins(AMOVL, ncon(0xffffffffu), &hi2); + break; + default: + gmove(&hi1, &hi2); + gins(AORL, ncon(hv), &hi2); + break; + } + break; + } + splitclean(); + splitclean(); + goto out; + } + gins(AMOVL, &lo1, &ax); + gins(AMOVL, &hi1, &dx); + gins(optoas(n->op, lo1.type), &lo2, &ax); + gins(optoas(n->op, lo1.type), &hi2, &dx); + break; + } + if(is64(r->type)) + splitclean(); + splitclean(); + + split64(res, &lo1, &hi1); + gins(AMOVL, &ax, &lo1); + gins(AMOVL, &dx, &hi1); + splitclean(); + +out:; +} + +/* + * generate comparison of nl, nr, both 64-bit. + * nl is memory; nr is constant or memory. + */ +void +cmp64(Node *nl, Node *nr, int op, Prog *to) +{ + Node lo1, hi1, lo2, hi2, rr; + Prog *br; + Type *t; + + split64(nl, &lo1, &hi1); + split64(nr, &lo2, &hi2); + + // compare most significant word; + // if they differ, we're done. + t = hi1.type; + if(nl->op == OLITERAL || nr->op == OLITERAL) + gins(ACMPL, &hi1, &hi2); + else { + regalloc(&rr, types[TINT32], N); + gins(AMOVL, &hi1, &rr); + gins(ACMPL, &rr, &hi2); + regfree(&rr); + } + br = P; + switch(op) { + default: + fatal("cmp64 %O %T", op, t); + case OEQ: + // cmp hi + // jne L + // cmp lo + // jeq to + // L: + br = gbranch(AJNE, T); + break; + case ONE: + // cmp hi + // jne to + // cmp lo + // jne to + patch(gbranch(AJNE, T), to); + break; + case OGE: + case OGT: + // cmp hi + // jgt to + // jlt L + // cmp lo + // jge to (or jgt to) + // L: + patch(gbranch(optoas(OGT, t), T), to); + br = gbranch(optoas(OLT, t), T); + break; + case OLE: + case OLT: + // cmp hi + // jlt to + // jgt L + // cmp lo + // jle to (or jlt to) + // L: + patch(gbranch(optoas(OLT, t), T), to); + br = gbranch(optoas(OGT, t), T); + break; + } + + // compare least significant word + t = lo1.type; + if(nl->op == OLITERAL || nr->op == OLITERAL) + gins(ACMPL, &lo1, &lo2); + else { + regalloc(&rr, types[TINT32], N); + gins(AMOVL, &lo1, &rr); + gins(ACMPL, &rr, &lo2); + regfree(&rr); + } + + // jump again + patch(gbranch(optoas(op, t), T), to); + + // point first branch down here if appropriate + if(br != P) + patch(br, pc); + + splitclean(); + splitclean(); +} + diff --git a/src/cmd/8g/doc.go b/src/cmd/8g/doc.go new file mode 100644 index 000000000..2d9ff9a42 --- /dev/null +++ b/src/cmd/8g/doc.go @@ -0,0 +1,15 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +/* + +8g is the version of the gc compiler for the x86. +The $GOARCH for these tools is 386. + +It reads .go files and outputs .8 files. The flags are documented in ../gc/doc.go. + +There is no instruction optimizer, so the -N flag is a no-op. + +*/ +package documentation diff --git a/src/cmd/8g/galign.c b/src/cmd/8g/galign.c new file mode 100644 index 000000000..7734603c4 --- /dev/null +++ b/src/cmd/8g/galign.c @@ -0,0 +1,37 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +#include "gg.h" + +int thechar = '8'; +char* thestring = "386"; + +vlong MAXWIDTH = (1LL<<32) - 1; + +/* + * go declares several platform-specific type aliases: + * int, uint, float, and uintptr + */ +Typedef typedefs[] = +{ + "int", TINT, TINT32, + "uint", TUINT, TUINT32, + "uintptr", TUINTPTR, TUINT32, + 0 +}; + +void +betypeinit(void) +{ + widthptr = 4; + + zprog.link = P; + zprog.as = AGOK; + zprog.from.type = D_NONE; + zprog.from.index = D_NONE; + zprog.from.scale = 0; + zprog.to = zprog.from; + + listinit(); +} diff --git a/src/cmd/8g/gg.h b/src/cmd/8g/gg.h new file mode 100644 index 000000000..9f7a66a29 --- /dev/null +++ b/src/cmd/8g/gg.h @@ -0,0 +1,187 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +#include +#include + +#include "../gc/go.h" +#include "../8l/8.out.h" + +#ifndef EXTERN +#define EXTERN extern +#endif + +typedef struct Addr Addr; + +struct Addr +{ + int32 offset; + int32 offset2; + + double dval; + Prog* branch; + char sval[NSNAME]; + + Sym* gotype; + Sym* sym; + Node* node; + int width; + uchar type; + uchar index; + uchar etype; + uchar scale; /* doubles as width in DATA op */ + uchar pun; /* dont register variable */ +}; +#define A ((Addr*)0) + +struct Prog +{ + short as; // opcode + uint32 loc; // pc offset in this func + uint32 lineno; // source line that generated this + Addr from; // src address + Addr to; // dst address + Prog* link; // next instruction in this func + void* reg; // pointer to containing Reg struct +}; + +// foptoas flags +enum +{ + Frev = 1<<0, + Fpop = 1<<1, + Fpop2 = 1<<2, +}; + +EXTERN Biobuf* bout; +EXTERN int32 dynloc; +EXTERN uchar reg[D_NONE]; +EXTERN int32 pcloc; // instruction counter +EXTERN Strlit emptystring; +extern char* anames[]; +EXTERN Hist* hist; +EXTERN Prog zprog; +EXTERN Node* curfn; +EXTERN Node* newproc; +EXTERN Node* deferproc; +EXTERN Node* deferreturn; +EXTERN Node* panicindex; +EXTERN Node* panicslice; +EXTERN Node* throwreturn; +EXTERN int maxstksize; +extern uint32 unmappedzero; + + +/* + * ggen.c + */ +void compile(Node*); +void proglist(void); +void gen(Node*); +Node* lookdot(Node*, Node*, int); +void cgen_as(Node*, Node*); +void cgen_callmeth(Node*, int); +void cgen_callinter(Node*, Node*, int); +void cgen_proc(Node*, int); +void cgen_callret(Node*, Node*); +void cgen_div(int, Node*, Node*, Node*); +void cgen_bmul(int, Node*, Node*, Node*); +void cgen_shift(int, Node*, Node*, Node*); +void cgen_dcl(Node*); +int needconvert(Type*, Type*); +void genconv(Type*, Type*); +void allocparams(void); +void checklabels(); +void ginscall(Node*, int); + +/* + * cgen.c + */ +void agen(Node*, Node*); +void agenr(Node *n, Node *a, Node *res); +void igen(Node*, Node*, Node*); +vlong fieldoffset(Type*, Node*); +void bgen(Node*, int, Prog*); +void sgen(Node*, Node*, int32); +void gmove(Node*, Node*); +Prog* gins(int, Node*, Node*); +int samaddr(Node*, Node*); +void naddr(Node*, Addr*, int); +void cgen_aret(Node*, Node*); +int cgen_inline(Node*, Node*); +Node* ncon(uint32); +void mgen(Node*, Node*, Node*); +void mfree(Node*); + +/* + * cgen64.c + */ +void cmp64(Node*, Node*, int, Prog*); +void cgen64(Node*, Node*); + +/* + * gsubr.c + */ +void clearp(Prog*); +void proglist(void); +Prog* gbranch(int, Type*); +Prog* prog(int); +void gaddoffset(Node*); +void gconv(int, int); +int conv2pt(Type*); +vlong convvtox(vlong, int); +void fnparam(Type*, int, int); +Prog* gop(int, Node*, Node*, Node*); +void setconst(Addr*, vlong); +void setaddr(Addr*, Node*); +int optoas(int, Type*); +int foptoas(int, Type*, int); +void ginit(void); +void gclean(void); +void regalloc(Node*, Type*, Node*); +void regfree(Node*); +Node* nodarg(Type*, int); +void nodreg(Node*, Type*, int); +void nodindreg(Node*, Type*, int); +void nodconst(Node*, Type*, int64); +void gconreg(int, vlong, int); +void datagostring(Strlit*, Addr*); +void datastring(char*, int, Addr*); +void buildtxt(void); +Plist* newplist(void); +int isfat(Type*); +void sudoclean(void); +int sudoaddable(int, Node*, Addr*); +int dotaddable(Node*, Node*); +void afunclit(Addr*); +void split64(Node*, Node*, Node*); +void splitclean(void); +void nswap(Node*, Node*); + +/* + * cplx.c + */ +int complexop(Node*, Node*); +void complexmove(Node*, Node*); +void complexgen(Node*, Node*); +void complexbool(int, Node*, Node*, int, Prog*); + +/* + * gobj.c + */ +void data(void); +void text(void); + +/* + * list.c + */ +int Aconv(Fmt*); +int Dconv(Fmt*); +int Pconv(Fmt*); +int Rconv(Fmt*); +int Yconv(Fmt*); +void listinit(void); + +void zaddr(Biobuf*, Addr*, int, int); + diff --git a/src/cmd/8g/ggen.c b/src/cmd/8g/ggen.c new file mode 100644 index 000000000..108c493aa --- /dev/null +++ b/src/cmd/8g/ggen.c @@ -0,0 +1,1155 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +#undef EXTERN +#define EXTERN +#include "gg.h" +#include "opt.h" + +void +defframe(Prog *ptxt) +{ + // fill in argument size + ptxt->to.offset2 = rnd(curfn->type->argwid, widthptr); + + // fill in final stack size + if(stksize > maxstksize) + maxstksize = stksize; + ptxt->to.offset = rnd(maxstksize+maxarg, widthptr); + maxstksize = 0; +} + +// Sweep the prog list to mark any used nodes. +void +markautoused(Prog* p) +{ + for (; p; p = p->link) { + if (p->from.type == D_AUTO && p->from.node) + p->from.node->used++; + + if (p->to.type == D_AUTO && p->to.node) + p->to.node->used++; + } +} + +// Fixup instructions after compactframe has moved all autos around. +void +fixautoused(Prog* p) +{ + for (; p; p = p->link) { + if (p->from.type == D_AUTO && p->from.node) + p->from.offset += p->from.node->stkdelta; + + if (p->to.type == D_AUTO && p->to.node) + p->to.offset += p->to.node->stkdelta; + } +} + +void +clearfat(Node *nl) +{ + uint32 w, c, q; + Node n1; + + /* clear a fat object */ + if(debug['g']) + dump("\nclearfat", nl); + + w = nl->type->width; + c = w % 4; // bytes + q = w / 4; // quads + + gconreg(AMOVL, 0, D_AX); + nodreg(&n1, types[tptr], D_DI); + agen(nl, &n1); + + if(q >= 4) { + gconreg(AMOVL, q, D_CX); + gins(AREP, N, N); // repeat + gins(ASTOSL, N, N); // STOL AL,*(DI)+ + } else + while(q > 0) { + gins(ASTOSL, N, N); // STOL AL,*(DI)+ + q--; + } + + if(c >= 4) { + gconreg(AMOVL, c, D_CX); + gins(AREP, N, N); // repeat + gins(ASTOSB, N, N); // STOB AL,*(DI)+ + } else + while(c > 0) { + gins(ASTOSB, N, N); // STOB AL,*(DI)+ + c--; + } +} + +/* + * generate: + * call f + * proc=0 normal call + * proc=1 goroutine run in new proc + * proc=2 defer call save away stack + */ +void +ginscall(Node *f, int proc) +{ + Prog *p; + Node reg, con; + + switch(proc) { + default: + fatal("ginscall: bad proc %d", proc); + break; + + case 0: // normal call + p = gins(ACALL, N, f); + afunclit(&p->to); + break; + + case 1: // call in new proc (go) + case 2: // deferred call (defer) + nodreg(®, types[TINT32], D_CX); + gins(APUSHL, f, N); + nodconst(&con, types[TINT32], argsize(f->type)); + gins(APUSHL, &con, N); + if(proc == 1) + ginscall(newproc, 0); + else + ginscall(deferproc, 0); + gins(APOPL, N, ®); + gins(APOPL, N, ®); + if(proc == 2) { + nodreg(®, types[TINT64], D_AX); + gins(ATESTL, ®, ®); + patch(gbranch(AJNE, T), retpc); + } + break; + } +} + +/* + * n is call to interface method. + * generate res = n. + */ +void +cgen_callinter(Node *n, Node *res, int proc) +{ + Node *i, *f; + Node tmpi, nodo, nodr, nodsp; + + i = n->left; + if(i->op != ODOTINTER) + fatal("cgen_callinter: not ODOTINTER %O", i->op); + + f = i->right; // field + if(f->op != ONAME) + fatal("cgen_callinter: not ONAME %O", f->op); + + i = i->left; // interface + + if(!i->addable) { + tempname(&tmpi, i->type); + cgen(i, &tmpi); + i = &tmpi; + } + + genlist(n->list); // assign the args + + // Can regalloc now; i is known to be addable, + // so the agen will be easy. + regalloc(&nodr, types[tptr], res); + regalloc(&nodo, types[tptr], &nodr); + nodo.op = OINDREG; + + agen(i, &nodr); // REG = &inter + + nodindreg(&nodsp, types[tptr], D_SP); + nodo.xoffset += widthptr; + cgen(&nodo, &nodsp); // 0(SP) = 4(REG) -- i.data + + nodo.xoffset -= widthptr; + cgen(&nodo, &nodr); // REG = 0(REG) -- i.tab + + if(n->left->xoffset == BADWIDTH) + fatal("cgen_callinter: badwidth"); + nodo.xoffset = n->left->xoffset + 3*widthptr + 8; + cgen(&nodo, &nodr); // REG = 20+offset(REG) -- i.tab->fun[f] + + // BOTCH nodr.type = fntype; + nodr.type = n->left->type; + ginscall(&nodr, proc); + + regfree(&nodr); + regfree(&nodo); + + setmaxarg(n->left->type); +} + +/* + * generate function call; + * proc=0 normal call + * proc=1 goroutine run in new proc + * proc=2 defer call save away stack + */ +void +cgen_call(Node *n, int proc) +{ + Type *t; + Node nod, afun; + + if(n == N) + return; + + if(n->left->ullman >= UINF) { + // if name involves a fn call + // precompute the address of the fn + tempname(&afun, types[tptr]); + cgen(n->left, &afun); + } + + genlist(n->list); // assign the args + t = n->left->type; + + setmaxarg(t); + + // call tempname pointer + if(n->left->ullman >= UINF) { + regalloc(&nod, types[tptr], N); + cgen_as(&nod, &afun); + nod.type = t; + ginscall(&nod, proc); + regfree(&nod); + return; + } + + // call pointer + if(n->left->op != ONAME || n->left->class != PFUNC) { + regalloc(&nod, types[tptr], N); + cgen_as(&nod, n->left); + nod.type = t; + ginscall(&nod, proc); + regfree(&nod); + return; + } + + // call direct + n->left->method = 1; + ginscall(n->left, proc); +} + +/* + * call to n has already been generated. + * generate: + * res = return value from call. + */ +void +cgen_callret(Node *n, Node *res) +{ + Node nod; + Type *fp, *t; + Iter flist; + + t = n->left->type; + if(t->etype == TPTR32 || t->etype == TPTR64) + t = t->type; + + fp = structfirst(&flist, getoutarg(t)); + if(fp == T) + fatal("cgen_callret: nil"); + + memset(&nod, 0, sizeof(nod)); + nod.op = OINDREG; + nod.val.u.reg = D_SP; + nod.addable = 1; + + nod.xoffset = fp->width; + nod.type = fp->type; + cgen_as(res, &nod); +} + +/* + * call to n has already been generated. + * generate: + * res = &return value from call. + */ +void +cgen_aret(Node *n, Node *res) +{ + Node nod1, nod2; + Type *fp, *t; + Iter flist; + + t = n->left->type; + if(isptr[t->etype]) + t = t->type; + + fp = structfirst(&flist, getoutarg(t)); + if(fp == T) + fatal("cgen_aret: nil"); + + memset(&nod1, 0, sizeof(nod1)); + nod1.op = OINDREG; + nod1.val.u.reg = D_SP; + nod1.addable = 1; + + nod1.xoffset = fp->width; + nod1.type = fp->type; + + if(res->op != OREGISTER) { + regalloc(&nod2, types[tptr], res); + gins(ALEAL, &nod1, &nod2); + gins(AMOVL, &nod2, res); + regfree(&nod2); + } else + gins(ALEAL, &nod1, res); +} + +/* + * generate return. + * n->left is assignments to return values. + */ +void +cgen_ret(Node *n) +{ + genlist(n->list); // copy out args + if(retpc) + gjmp(retpc); + else + gins(ARET, N, N); +} + +/* + * generate += *= etc. + */ +void +cgen_asop(Node *n) +{ + Node n1, n2, n3, n4; + Node *nl, *nr; + Prog *p1; + Addr addr; + int a; + + nl = n->left; + nr = n->right; + + if(nr->ullman >= UINF && nl->ullman >= UINF) { + tempname(&n1, nr->type); + cgen(nr, &n1); + n2 = *n; + n2.right = &n1; + cgen_asop(&n2); + goto ret; + } + + if(!isint[nl->type->etype]) + goto hard; + if(!isint[nr->type->etype]) + goto hard; + if(is64(nl->type) || is64(nr->type)) + goto hard; + + switch(n->etype) { + case OADD: + if(smallintconst(nr)) + if(mpgetfix(nr->val.u.xval) == 1) { + a = optoas(OINC, nl->type); + if(nl->addable) { + gins(a, N, nl); + goto ret; + } + if(sudoaddable(a, nl, &addr)) { + p1 = gins(a, N, N); + p1->to = addr; + sudoclean(); + goto ret; + } + } + break; + + case OSUB: + if(smallintconst(nr)) + if(mpgetfix(nr->val.u.xval) == 1) { + a = optoas(ODEC, nl->type); + if(nl->addable) { + gins(a, N, nl); + goto ret; + } + if(sudoaddable(a, nl, &addr)) { + p1 = gins(a, N, N); + p1->to = addr; + sudoclean(); + goto ret; + } + } + break; + } + + switch(n->etype) { + case OADD: + case OSUB: + case OXOR: + case OAND: + case OOR: + a = optoas(n->etype, nl->type); + if(nl->addable) { + if(smallintconst(nr)) { + gins(a, nr, nl); + goto ret; + } + regalloc(&n2, nr->type, N); + cgen(nr, &n2); + gins(a, &n2, nl); + regfree(&n2); + goto ret; + } + if(nr->ullman < UINF) + if(sudoaddable(a, nl, &addr)) { + if(smallintconst(nr)) { + p1 = gins(a, nr, N); + p1->to = addr; + sudoclean(); + goto ret; + } + regalloc(&n2, nr->type, N); + cgen(nr, &n2); + p1 = gins(a, &n2, N); + p1->to = addr; + regfree(&n2); + sudoclean(); + goto ret; + } + } + +hard: + n2.op = 0; + n1.op = 0; + if(nr->ullman >= nl->ullman || nl->addable) { + mgen(nr, &n2, N); + nr = &n2; + nr = &n2; + } else { + tempname(&n2, nr->type); + cgen(nr, &n2); + nr = &n2; + } + if(!nl->addable) { + igen(nl, &n1, N); + nl = &n1; + } + + n3 = *n; + n3.left = nl; + n3.right = nr; + n3.op = n->etype; + + mgen(&n3, &n4, N); + gmove(&n4, nl); + + if(n1.op) + regfree(&n1); + mfree(&n2); + mfree(&n4); + +ret: + ; +} + +int +samereg(Node *a, Node *b) +{ + if(a->op != OREGISTER) + return 0; + if(b->op != OREGISTER) + return 0; + if(a->val.u.reg != b->val.u.reg) + return 0; + return 1; +} + +/* + * generate division. + * caller must set: + * ax = allocated AX register + * dx = allocated DX register + * generates one of: + * res = nl / nr + * res = nl % nr + * according to op. + */ +void +dodiv(int op, Node *nl, Node *nr, Node *res, Node *ax, Node *dx) +{ + int check; + Node n1, t1, t2, n4, nz; + Type *t; + Prog *p1, *p2, *p3; + + // Have to be careful about handling + // most negative int divided by -1 correctly. + // The hardware will trap. + // Also the byte divide instruction needs AH, + // which we otherwise don't have to deal with. + // Easiest way to avoid for int8, int16: use int32. + // For int32 and int64, use explicit test. + // Could use int64 hw for int32. + t = nl->type; + check = 0; + if(issigned[t->etype]) { + check = 1; + if(isconst(nl, CTINT) && mpgetfix(nl->val.u.xval) != -1LL<<(t->width*8-1)) + check = 0; + else if(isconst(nr, CTINT) && mpgetfix(nr->val.u.xval) != -1) + check = 0; + } + if(t->width < 4) { + if(issigned[t->etype]) + t = types[TINT32]; + else + t = types[TUINT32]; + check = 0; + } + + tempname(&t1, t); + tempname(&t2, t); + cgen(nl, &t1); + cgen(nr, &t2); + + if(!samereg(ax, res) && !samereg(dx, res)) + regalloc(&n1, t, res); + else + regalloc(&n1, t, N); + gmove(&t2, &n1); + gmove(&t1, ax); + p3 = P; + if(check) { + nodconst(&n4, t, -1); + gins(optoas(OCMP, t), &n1, &n4); + p1 = gbranch(optoas(ONE, t), T); + nodconst(&n4, t, -1LL<<(t->width*8-1)); + gins(optoas(OCMP, t), ax, &n4); + p2 = gbranch(optoas(ONE, t), T); + if(op == ODIV) + gmove(&n4, res); + if(op == OMOD) { + nodconst(&n4, t, 0); + gmove(&n4, res); + } + p3 = gbranch(AJMP, T); + patch(p1, pc); + patch(p2, pc); + } + if(!issigned[t->etype]) { + nodconst(&nz, t, 0); + gmove(&nz, dx); + } else + gins(optoas(OEXTEND, t), N, N); + gins(optoas(op, t), &n1, N); + regfree(&n1); + + if(op == ODIV) + gmove(ax, res); + else + gmove(dx, res); + if(check) + patch(p3, pc); +} + +static void +savex(int dr, Node *x, Node *oldx, Node *res, Type *t) +{ + int r; + + r = reg[dr]; + nodreg(x, types[TINT32], dr); + + // save current ax and dx if they are live + // and not the destination + memset(oldx, 0, sizeof *oldx); + if(r > 0 && !samereg(x, res)) { + tempname(oldx, types[TINT32]); + gmove(x, oldx); + } + + regalloc(x, t, x); +} + +static void +restx(Node *x, Node *oldx) +{ + regfree(x); + + if(oldx->op != 0) { + x->type = types[TINT32]; + gmove(oldx, x); + } +} + +/* + * generate division according to op, one of: + * res = nl / nr + * res = nl % nr + */ +void +cgen_div(int op, Node *nl, Node *nr, Node *res) +{ + Node ax, dx, oldax, olddx; + Type *t; + + if(is64(nl->type)) + fatal("cgen_div %T", nl->type); + + if(issigned[nl->type->etype]) + t = types[TINT32]; + else + t = types[TUINT32]; + savex(D_AX, &ax, &oldax, res, t); + savex(D_DX, &dx, &olddx, res, t); + dodiv(op, nl, nr, res, &ax, &dx); + restx(&dx, &olddx); + restx(&ax, &oldax); +} + +/* + * generate shift according to op, one of: + * res = nl << nr + * res = nl >> nr + */ +void +cgen_shift(int op, Node *nl, Node *nr, Node *res) +{ + Node n1, n2, nt, cx, oldcx, hi, lo; + int a, w; + Prog *p1, *p2; + uvlong sc; + + if(nl->type->width > 4) + fatal("cgen_shift %T", nl->type); + + w = nl->type->width * 8; + + a = optoas(op, nl->type); + + if(nr->op == OLITERAL) { + tempname(&n2, nl->type); + cgen(nl, &n2); + regalloc(&n1, nl->type, res); + gmove(&n2, &n1); + sc = mpgetfix(nr->val.u.xval); + if(sc >= nl->type->width*8) { + // large shift gets 2 shifts by width + gins(a, ncon(w-1), &n1); + gins(a, ncon(w-1), &n1); + } else + gins(a, nr, &n1); + gmove(&n1, res); + regfree(&n1); + return; + } + + memset(&oldcx, 0, sizeof oldcx); + nodreg(&cx, types[TUINT32], D_CX); + if(reg[D_CX] > 1 && !samereg(&cx, res)) { + tempname(&oldcx, types[TUINT32]); + gmove(&cx, &oldcx); + } + + if(nr->type->width > 4) { + tempname(&nt, nr->type); + n1 = nt; + } else { + nodreg(&n1, types[TUINT32], D_CX); + regalloc(&n1, nr->type, &n1); // to hold the shift type in CX + } + + if(samereg(&cx, res)) + regalloc(&n2, nl->type, N); + else + regalloc(&n2, nl->type, res); + if(nl->ullman >= nr->ullman) { + cgen(nl, &n2); + cgen(nr, &n1); + } else { + cgen(nr, &n1); + cgen(nl, &n2); + } + + // test and fix up large shifts + if(nr->type->width > 4) { + // delayed reg alloc + nodreg(&n1, types[TUINT32], D_CX); + regalloc(&n1, types[TUINT32], &n1); // to hold the shift type in CX + split64(&nt, &lo, &hi); + gmove(&lo, &n1); + gins(optoas(OCMP, types[TUINT32]), &hi, ncon(0)); + p2 = gbranch(optoas(ONE, types[TUINT32]), T); + gins(optoas(OCMP, types[TUINT32]), &n1, ncon(w)); + p1 = gbranch(optoas(OLT, types[TUINT32]), T); + patch(p2, pc); + } else { + gins(optoas(OCMP, nr->type), &n1, ncon(w)); + p1 = gbranch(optoas(OLT, types[TUINT32]), T); + } + if(op == ORSH && issigned[nl->type->etype]) { + gins(a, ncon(w-1), &n2); + } else { + gmove(ncon(0), &n2); + } + patch(p1, pc); + gins(a, &n1, &n2); + + if(oldcx.op != 0) + gmove(&oldcx, &cx); + + gmove(&n2, res); + + regfree(&n1); + regfree(&n2); +} + +/* + * generate byte multiply: + * res = nl * nr + * no byte multiply instruction so have to do + * 16-bit multiply and take bottom half. + */ +void +cgen_bmul(int op, Node *nl, Node *nr, Node *res) +{ + Node n1b, n2b, n1w, n2w; + Type *t; + int a; + + if(nl->ullman >= nr->ullman) { + regalloc(&n1b, nl->type, res); + cgen(nl, &n1b); + regalloc(&n2b, nr->type, N); + cgen(nr, &n2b); + } else { + regalloc(&n2b, nr->type, N); + cgen(nr, &n2b); + regalloc(&n1b, nl->type, res); + cgen(nl, &n1b); + } + + // copy from byte to short registers + t = types[TUINT16]; + if(issigned[nl->type->etype]) + t = types[TINT16]; + + regalloc(&n2w, t, &n2b); + cgen(&n2b, &n2w); + + regalloc(&n1w, t, &n1b); + cgen(&n1b, &n1w); + + a = optoas(op, t); + gins(a, &n2w, &n1w); + cgen(&n1w, &n1b); + cgen(&n1b, res); + + regfree(&n1w); + regfree(&n2w); + regfree(&n1b); + regfree(&n2b); +} + +static int +regcmp(const void *va, const void *vb) +{ + Node *ra, *rb; + + ra = (Node*)va; + rb = (Node*)vb; + return ra->local - rb->local; +} + +static Prog* throwpc; + +// We're only going to bother inlining if we can +// convert all the arguments to 32 bits safely. Can we? +static int +fix64(NodeList *nn, int n) +{ + NodeList *l; + Node *r; + int i; + + l = nn; + for(i=0; in->right; + if(is64(r->type) && !smallintconst(r)) { + if(r->op == OCONV) + r = r->left; + if(is64(r->type)) + return 0; + } + l = l->next; + } + return 1; +} + +void +getargs(NodeList *nn, Node *reg, int n) +{ + NodeList *l; + Node *r; + int i; + + throwpc = nil; + + l = nn; + for(i=0; in->right; + if(is64(r->type)) { + if(r->op == OCONV) + r = r->left; + else if(smallintconst(r)) + r->type = types[TUINT32]; + if(is64(r->type)) + fatal("getargs"); + } + if(!smallintconst(r) && !isslice(r->type)) { + if(i < 3) // AX CX DX + nodreg(reg+i, r->type, D_AX+i); + else + reg[i].op = OXXX; + regalloc(reg+i, r->type, reg+i); + cgen(r, reg+i); + } else + reg[i] = *r; + if(reg[i].local != 0) + yyerror("local used"); + reg[i].local = l->n->left->xoffset; + l = l->next; + } + qsort((void*)reg, n, sizeof(*reg), regcmp); + for(i=0; ival.u.xval); + if(cl == 0) + return; + if(smallintconst(nr)) + return; + // put the constant on the right + op = brrev(op); + c = nl; + nl = nr; + nr = c; + } + + // Arguments are known not to be 64-bit, + // but they might be smaller than 32 bits. + // Check if we need to use a temporary. + // At least one of the arguments is 32 bits + // (the len or cap) so one temporary suffices. + n1.op = OXXX; + t = types[TUINT32]; + if(nl->type->width != t->width) { + regalloc(&n1, t, nl); + gmove(nl, &n1); + nl = &n1; + } else if(nr->type->width != t->width) { + regalloc(&n1, t, nr); + gmove(nr, &n1); + nr = &n1; + } + gins(optoas(OCMP, t), nl, nr); + if(n1.op != OXXX) + regfree(&n1); + if(throwpc == nil) { + p1 = gbranch(optoas(op, t), T); + throwpc = pc; + ginscall(panicslice, 0); + patch(p1, pc); + } else { + op = brcom(op); + p1 = gbranch(optoas(op, t), T); + patch(p1, throwpc); + } +} + +int +sleasy(Node *n) +{ + if(n->op != ONAME) + return 0; + if(!n->addable) + return 0; + return 1; +} + +// generate inline code for +// slicearray +// sliceslice +// arraytoslice +int +cgen_inline(Node *n, Node *res) +{ + Node nodes[5]; + Node n1, n2, nres, ntemp; + vlong v; + int i, narg, nochk; + + if(n->op != OCALLFUNC) + goto no; + if(!n->left->addable) + goto no; + if(n->left->sym == S) + goto no; + if(n->left->sym->pkg != runtimepkg) + goto no; + if(strcmp(n->left->sym->name, "slicearray") == 0) + goto slicearray; + if(strcmp(n->left->sym->name, "sliceslice") == 0) { + narg = 4; + goto sliceslice; + } + if(strcmp(n->left->sym->name, "sliceslice1") == 0) { + narg = 3; + goto sliceslice; + } + goto no; + +slicearray: + if(!sleasy(res)) + goto no; + if(!fix64(n->list, 5)) + goto no; + getargs(n->list, nodes, 5); + + // if(hb[3] > nel[1]) goto throw + cmpandthrow(&nodes[3], &nodes[1]); + + // if(lb[2] > hb[3]) goto throw + cmpandthrow(&nodes[2], &nodes[3]); + + // len = hb[3] - lb[2] (destroys hb) + n2 = *res; + n2.xoffset += Array_nel; + n2.type = types[TUINT32]; + + if(smallintconst(&nodes[3]) && smallintconst(&nodes[2])) { + v = mpgetfix(nodes[3].val.u.xval) - + mpgetfix(nodes[2].val.u.xval); + nodconst(&n1, types[TUINT32], v); + gins(optoas(OAS, types[TUINT32]), &n1, &n2); + } else { + regalloc(&n1, types[TUINT32], &nodes[3]); + gmove(&nodes[3], &n1); + if(!smallintconst(&nodes[2]) || mpgetfix(nodes[2].val.u.xval) != 0) + gins(optoas(OSUB, types[TUINT32]), &nodes[2], &n1); + gins(optoas(OAS, types[TUINT32]), &n1, &n2); + regfree(&n1); + } + + // cap = nel[1] - lb[2] (destroys nel) + n2 = *res; + n2.xoffset += Array_cap; + n2.type = types[TUINT32]; + + if(smallintconst(&nodes[1]) && smallintconst(&nodes[2])) { + v = mpgetfix(nodes[1].val.u.xval) - + mpgetfix(nodes[2].val.u.xval); + nodconst(&n1, types[TUINT32], v); + gins(optoas(OAS, types[TUINT32]), &n1, &n2); + } else { + regalloc(&n1, types[TUINT32], &nodes[1]); + gmove(&nodes[1], &n1); + if(!smallintconst(&nodes[2]) || mpgetfix(nodes[2].val.u.xval) != 0) + gins(optoas(OSUB, types[TUINT32]), &nodes[2], &n1); + gins(optoas(OAS, types[TUINT32]), &n1, &n2); + regfree(&n1); + } + + // if slice could be too big, dereference to + // catch nil array pointer. + if(nodes[0].op == OREGISTER && nodes[0].type->type->width >= unmappedzero) { + n2 = nodes[0]; + n2.xoffset = 0; + n2.op = OINDREG; + n2.type = types[TUINT8]; + gins(ATESTB, nodintconst(0), &n2); + } + + // ary = old[0] + (lb[2] * width[4]) (destroys old) + n2 = *res; + n2.xoffset += Array_array; + n2.type = types[tptr]; + + if(smallintconst(&nodes[2]) && smallintconst(&nodes[4])) { + v = mpgetfix(nodes[2].val.u.xval) * + mpgetfix(nodes[4].val.u.xval); + if(v != 0) { + nodconst(&n1, types[tptr], v); + gins(optoas(OADD, types[tptr]), &n1, &nodes[0]); + } + } else { + regalloc(&n1, types[tptr], &nodes[2]); + gmove(&nodes[2], &n1); + if(!smallintconst(&nodes[4]) || mpgetfix(nodes[4].val.u.xval) != 1) + gins(optoas(OMUL, types[tptr]), &nodes[4], &n1); + gins(optoas(OADD, types[tptr]), &n1, &nodes[0]); + regfree(&n1); + } + gins(optoas(OAS, types[tptr]), &nodes[0], &n2); + + for(i=0; i<5; i++) { + if(nodes[i].op == OREGISTER) + regfree(&nodes[i]); + } + return 1; + +sliceslice: + if(!fix64(n->list, narg)) + goto no; + nochk = n->etype; // skip bounds checking + ntemp.op = OXXX; + if(!sleasy(n->list->n->right)) { + Node *n0; + + n0 = n->list->n->right; + tempname(&ntemp, res->type); + cgen(n0, &ntemp); + n->list->n->right = &ntemp; + getargs(n->list, nodes, narg); + n->list->n->right = n0; + } else + getargs(n->list, nodes, narg); + + nres = *res; // result + if(!sleasy(res)) { + if(ntemp.op == OXXX) + tempname(&ntemp, res->type); + nres = ntemp; + } + + if(narg == 3) { // old[lb:] + // move width to where it would be for old[lb:hb] + nodes[3] = nodes[2]; + nodes[2].op = OXXX; + + // if(lb[1] > old.nel[0]) goto throw; + n2 = nodes[0]; + n2.xoffset += Array_nel; + n2.type = types[TUINT32]; + if(!nochk) + cmpandthrow(&nodes[1], &n2); + + // ret.nel = old.nel[0]-lb[1]; + n2 = nodes[0]; + n2.xoffset += Array_nel; + n2.type = types[TUINT32]; + + regalloc(&n1, types[TUINT32], N); + gins(optoas(OAS, types[TUINT32]), &n2, &n1); + if(!smallintconst(&nodes[1]) || mpgetfix(nodes[1].val.u.xval) != 0) + gins(optoas(OSUB, types[TUINT32]), &nodes[1], &n1); + + n2 = nres; + n2.xoffset += Array_nel; + n2.type = types[TUINT32]; + gins(optoas(OAS, types[TUINT32]), &n1, &n2); + regfree(&n1); + } else { // old[lb:hb] + n2 = nodes[0]; + n2.xoffset += Array_cap; + n2.type = types[TUINT32]; + if (!nochk) { + // if(hb[2] > old.cap[0]) goto throw; + cmpandthrow(&nodes[2], &n2); + // if(lb[1] > hb[2]) goto throw; + cmpandthrow(&nodes[1], &nodes[2]); + } + + // ret.len = hb[2]-lb[1]; (destroys hb[2]) + n2 = nres; + n2.xoffset += Array_nel; + n2.type = types[TUINT32]; + + if(smallintconst(&nodes[2]) && smallintconst(&nodes[1])) { + v = mpgetfix(nodes[2].val.u.xval) - + mpgetfix(nodes[1].val.u.xval); + nodconst(&n1, types[TUINT32], v); + gins(optoas(OAS, types[TUINT32]), &n1, &n2); + } else { + regalloc(&n1, types[TUINT32], &nodes[2]); + gmove(&nodes[2], &n1); + if(!smallintconst(&nodes[1]) || mpgetfix(nodes[1].val.u.xval) != 0) + gins(optoas(OSUB, types[TUINT32]), &nodes[1], &n1); + gins(optoas(OAS, types[TUINT32]), &n1, &n2); + regfree(&n1); + } + } + + // ret.cap = old.cap[0]-lb[1]; (uses hb[2]) + n2 = nodes[0]; + n2.xoffset += Array_cap; + n2.type = types[TUINT32]; + + regalloc(&n1, types[TUINT32], &nodes[2]); + gins(optoas(OAS, types[TUINT32]), &n2, &n1); + if(!smallintconst(&nodes[1]) || mpgetfix(nodes[1].val.u.xval) != 0) + gins(optoas(OSUB, types[TUINT32]), &nodes[1], &n1); + + n2 = nres; + n2.xoffset += Array_cap; + n2.type = types[TUINT32]; + gins(optoas(OAS, types[TUINT32]), &n1, &n2); + regfree(&n1); + + // ret.array = old.array[0]+lb[1]*width[3]; (uses lb[1]) + n2 = nodes[0]; + n2.xoffset += Array_array; + n2.type = types[tptr]; + + regalloc(&n1, types[tptr], &nodes[1]); + if(smallintconst(&nodes[1]) && smallintconst(&nodes[3])) { + gins(optoas(OAS, types[tptr]), &n2, &n1); + v = mpgetfix(nodes[1].val.u.xval) * + mpgetfix(nodes[3].val.u.xval); + if(v != 0) { + nodconst(&n2, types[tptr], v); + gins(optoas(OADD, types[tptr]), &n2, &n1); + } + } else { + gmove(&nodes[1], &n1); + if(!smallintconst(&nodes[3]) || mpgetfix(nodes[3].val.u.xval) != 1) + gins(optoas(OMUL, types[tptr]), &nodes[3], &n1); + gins(optoas(OADD, types[tptr]), &n2, &n1); + } + + n2 = nres; + n2.xoffset += Array_array; + n2.type = types[tptr]; + gins(optoas(OAS, types[tptr]), &n1, &n2); + regfree(&n1); + + for(i=0; i<4; i++) { + if(nodes[i].op == OREGISTER) + regfree(&nodes[i]); + } + + if(!sleasy(res)) { + cgen(&nres, res); + } + return 1; + +no: + return 0; +} diff --git a/src/cmd/8g/gobj.c b/src/cmd/8g/gobj.c new file mode 100644 index 000000000..31c42a3f2 --- /dev/null +++ b/src/cmd/8g/gobj.c @@ -0,0 +1,651 @@ +// Derived from Inferno utils/8c/swt.c +// http://code.google.com/p/inferno-os/source/browse/utils/8c/swt.c +// +// Copyright © 1994-1999 Lucent Technologies Inc. All rights reserved. +// Portions Copyright © 1995-1997 C H Forsyth (forsyth@terzarima.net) +// Portions Copyright © 1997-1999 Vita Nuova Limited +// Portions Copyright © 2000-2007 Vita Nuova Holdings Limited (www.vitanuova.com) +// Portions Copyright © 2004,2006 Bruce Ellis +// Portions Copyright © 2005-2007 C H Forsyth (forsyth@terzarima.net) +// Revisions Copyright © 2000-2007 Lucent Technologies Inc. and others +// Portions Copyright © 2009 The Go Authors. All rights reserved. +// +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to deal +// in the Software without restriction, including without limitation the rights +// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +// copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +// THE SOFTWARE. + +#include "gg.h" + +void +zname(Biobuf *b, Sym *s, int t) +{ + Bputc(b, ANAME); /* as */ + Bputc(b, ANAME>>8); /* as */ + Bputc(b, t); /* type */ + Bputc(b, s->sym); /* sym */ + + Bputname(b, s); +} + +void +zfile(Biobuf *b, char *p, int n) +{ + Bputc(b, ANAME); + Bputc(b, ANAME>>8); + Bputc(b, D_FILE); + Bputc(b, 1); + Bputc(b, '<'); + Bwrite(b, p, n); + Bputc(b, 0); +} + +void +zhist(Biobuf *b, int line, vlong offset) +{ + Addr a; + + Bputc(b, AHISTORY); + Bputc(b, AHISTORY>>8); + Bputc(b, line); + Bputc(b, line>>8); + Bputc(b, line>>16); + Bputc(b, line>>24); + zaddr(b, &zprog.from, 0, 0); + a = zprog.to; + if(offset != 0) { + a.offset = offset; + a.type = D_CONST; + } + zaddr(b, &a, 0, 0); +} + +void +zaddr(Biobuf *b, Addr *a, int s, int gotype) +{ + int32 l; + uint64 e; + int i, t; + char *n; + + t = 0; + if(a->index != D_NONE || a->scale != 0) + t |= T_INDEX; + if(s != 0) + t |= T_SYM; + if(gotype != 0) + t |= T_GOTYPE; + + switch(a->type) { + + case D_BRANCH: + if(a->branch == nil) + fatal("unpatched branch"); + a->offset = a->branch->loc; + + default: + t |= T_TYPE; + + case D_NONE: + if(a->offset != 0) + t |= T_OFFSET; + if(a->offset2 != 0) + t |= T_OFFSET2; + break; + case D_FCONST: + t |= T_FCONST; + break; + case D_SCONST: + t |= T_SCONST; + break; + } + Bputc(b, t); + + if(t & T_INDEX) { /* implies index, scale */ + Bputc(b, a->index); + Bputc(b, a->scale); + } + if(t & T_OFFSET) { /* implies offset */ + l = a->offset; + Bputc(b, l); + Bputc(b, l>>8); + Bputc(b, l>>16); + Bputc(b, l>>24); + } + if(t & T_OFFSET2) { /* implies offset */ + l = a->offset2; + Bputc(b, l); + Bputc(b, l>>8); + Bputc(b, l>>16); + Bputc(b, l>>24); + } + if(t & T_SYM) /* implies sym */ + Bputc(b, s); + if(t & T_FCONST) { + ieeedtod(&e, a->dval); + l = e; + Bputc(b, l); + Bputc(b, l>>8); + Bputc(b, l>>16); + Bputc(b, l>>24); + l = e >> 32; + Bputc(b, l); + Bputc(b, l>>8); + Bputc(b, l>>16); + Bputc(b, l>>24); + return; + } + if(t & T_SCONST) { + n = a->sval; + for(i=0; itype); + if(t & T_GOTYPE) + Bputc(b, gotype); +} + +static struct { + struct { Sym *sym; short type; } h[NSYM]; + int sym; +} z; + +static void +zsymreset(void) +{ + for(z.sym=0; z.symsym; + if(i < 0 || i >= NSYM) + i = 0; + if(z.h[i].type == t && z.h[i].sym == s) + return i; + i = z.sym; + s->sym = i; + zname(bout, s, t); + z.h[i].sym = s; + z.h[i].type = t; + if(++z.sym >= NSYM) + z.sym = 1; + *new = 1; + return i; +} + +static int +zsymaddr(Addr *a, int *new) +{ + int t; + + t = a->type; + if(t == D_ADDR) + t = a->index; + return zsym(a->sym, t, new); +} + +void +dumpfuncs(void) +{ + Plist *pl; + int sf, st, gf, gt, new; + Sym *s; + Prog *p; + + zsymreset(); + + // fix up pc + pcloc = 0; + for(pl=plist; pl!=nil; pl=pl->link) { + if(isblank(pl->name)) + continue; + for(p=pl->firstpc; p!=P; p=p->link) { + p->loc = pcloc; + if(p->as != ADATA && p->as != AGLOBL) + pcloc++; + } + } + + // put out functions + for(pl=plist; pl!=nil; pl=pl->link) { + if(isblank(pl->name)) + continue; + + if(debug['S']) { + s = S; + if(pl->name != N) + s = pl->name->sym; + print("\n--- prog list \"%S\" ---\n", s); + for(p=pl->firstpc; p!=P; p=p->link) + print("%P\n", p); + } + + for(p=pl->firstpc; p!=P; p=p->link) { + for(;;) { + sf = zsymaddr(&p->from, &new); + gf = zsym(p->from.gotype, D_EXTERN, &new); + if(new && sf == gf) + continue; + st = zsymaddr(&p->to, &new); + if(new && (st == sf || st == gf)) + continue; + gt = zsym(p->to.gotype, D_EXTERN, &new); + if(new && (gt == sf || gt == gf || gt == st)) + continue; + break; + } + + Bputc(bout, p->as); + Bputc(bout, p->as>>8); + Bputc(bout, p->lineno); + Bputc(bout, p->lineno>>8); + Bputc(bout, p->lineno>>16); + Bputc(bout, p->lineno>>24); + zaddr(bout, &p->from, sf, gf); + zaddr(bout, &p->to, st, gt); + } + } +} + +/* deferred DATA output */ +static Prog *strdat; +static Prog *estrdat; +static int gflag; +static Prog *savepc; + +void +data(void) +{ + gflag = debug['g']; + debug['g'] = 0; + + if(estrdat == nil) { + strdat = mal(sizeof(*pc)); + clearp(strdat); + estrdat = strdat; + } + if(savepc) + fatal("data phase error"); + savepc = pc; + pc = estrdat; +} + +void +text(void) +{ + if(!savepc) + fatal("text phase error"); + debug['g'] = gflag; + estrdat = pc; + pc = savepc; + savepc = nil; +} + +void +dumpdata(void) +{ + Prog *p; + + if(estrdat == nil) + return; + *pc = *strdat; + if(gflag) + for(p=pc; p!=estrdat; p=p->link) + print("%P\n", p); + pc = estrdat; +} + +int +dsname(Sym *s, int off, char *t, int n) +{ + Prog *p; + + p = gins(ADATA, N, N); + p->from.type = D_EXTERN; + p->from.index = D_NONE; + p->from.offset = off; + p->from.scale = n; + p->from.sym = s; + + p->to.type = D_SCONST; + p->to.index = D_NONE; + memmove(p->to.sval, t, n); + return off + n; +} + +/* + * make a refer to the data s, s+len + * emitting DATA if needed. + */ +void +datastring(char *s, int len, Addr *a) +{ + Sym *sym; + + sym = stringsym(s, len); + a->type = D_EXTERN; + a->sym = sym; + a->offset = widthptr+4; // skip header + a->etype = TINT32; +} + +/* + * make a refer to the string sval, + * emitting DATA if needed. + */ +void +datagostring(Strlit *sval, Addr *a) +{ + Sym *sym; + + sym = stringsym(sval->s, sval->len); + a->type = D_EXTERN; + a->sym = sym; + a->offset = 0; // header + a->etype = TINT32; +} + +void +gdata(Node *nam, Node *nr, int wid) +{ + Prog *p; + vlong v; + + if(wid == 8 && is64(nr->type)) { + v = mpgetfix(nr->val.u.xval); + p = gins(ADATA, nam, nodintconst(v)); + p->from.scale = 4; + p = gins(ADATA, nam, nodintconst(v>>32)); + p->from.scale = 4; + p->from.offset += 4; + return; + } + p = gins(ADATA, nam, nr); + p->from.scale = wid; +} + +void +gdatacomplex(Node *nam, Mpcplx *cval) +{ + Prog *p; + int w; + + w = cplxsubtype(nam->type->etype); + w = types[w]->width; + + p = gins(ADATA, nam, N); + p->from.scale = w; + p->to.type = D_FCONST; + p->to.dval = mpgetflt(&cval->real); + + p = gins(ADATA, nam, N); + p->from.scale = w; + p->from.offset += w; + p->to.type = D_FCONST; + p->to.dval = mpgetflt(&cval->imag); +} + +void +gdatastring(Node *nam, Strlit *sval) +{ + Prog *p; + Node nod1; + + p = gins(ADATA, nam, N); + datastring(sval->s, sval->len, &p->to); + p->from.scale = types[tptr]->width; + p->to.index = p->to.type; + p->to.type = D_ADDR; +//print("%P\n", p); + + nodconst(&nod1, types[TINT32], sval->len); + p = gins(ADATA, nam, &nod1); + p->from.scale = types[TINT32]->width; + p->from.offset += types[tptr]->width; +} + +int +dstringptr(Sym *s, int off, char *str) +{ + Prog *p; + + off = rnd(off, widthptr); + p = gins(ADATA, N, N); + p->from.type = D_EXTERN; + p->from.index = D_NONE; + p->from.sym = s; + p->from.offset = off; + p->from.scale = widthptr; + + datastring(str, strlen(str)+1, &p->to); + p->to.index = p->to.type; + p->to.type = D_ADDR; + p->to.etype = TINT32; + off += widthptr; + + return off; +} + +int +dgostrlitptr(Sym *s, int off, Strlit *lit) +{ + Prog *p; + + if(lit == nil) + return duintptr(s, off, 0); + + off = rnd(off, widthptr); + p = gins(ADATA, N, N); + p->from.type = D_EXTERN; + p->from.index = D_NONE; + p->from.sym = s; + p->from.offset = off; + p->from.scale = widthptr; + datagostring(lit, &p->to); + p->to.index = p->to.type; + p->to.type = D_ADDR; + p->to.etype = TINT32; + off += widthptr; + + return off; +} + +int +dgostringptr(Sym *s, int off, char *str) +{ + int n; + Strlit *lit; + + if(str == nil) + return duintptr(s, off, 0); + + n = strlen(str); + lit = mal(sizeof *lit + n); + strcpy(lit->s, str); + lit->len = n; + return dgostrlitptr(s, off, lit); +} + + +int +duintxx(Sym *s, int off, uint64 v, int wid) +{ + Prog *p; + + off = rnd(off, wid); + + p = gins(ADATA, N, N); + p->from.type = D_EXTERN; + p->from.index = D_NONE; + p->from.sym = s; + p->from.offset = off; + p->from.scale = wid; + p->to.type = D_CONST; + p->to.index = D_NONE; + p->to.offset = v; + off += wid; + + return off; +} + +int +dsymptr(Sym *s, int off, Sym *x, int xoff) +{ + Prog *p; + + off = rnd(off, widthptr); + + p = gins(ADATA, N, N); + p->from.type = D_EXTERN; + p->from.index = D_NONE; + p->from.sym = s; + p->from.offset = off; + p->from.scale = widthptr; + p->to.type = D_ADDR; + p->to.index = D_EXTERN; + p->to.sym = x; + p->to.offset = xoff; + off += widthptr; + + return off; +} + +void +genembedtramp(Type *rcvr, Type *method, Sym *newnam, int iface) +{ + Sym *e; + int c, d, o, mov, add, loaded; + Prog *p; + Type *f; + + e = method->sym; + for(d=0; dsym); + +out: + newplist()->name = newname(newnam); + + //TEXT main·S_test2(SB),7,$0 + p = pc; + gins(ATEXT, N, N); + p->from.type = D_EXTERN; + p->from.sym = newnam; + p->to.type = D_CONST; + p->to.offset = 0; + p->from.scale = 7; +//print("1. %P\n", p); + + mov = AMOVL; + add = AADDL; + + loaded = 0; + o = 0; + for(c=d-1; c>=0; c--) { + f = dotlist[c].field; + o += f->width; + if(!isptr[f->type->etype]) + continue; + if(!loaded) { + loaded = 1; + //MOVL 4(SP), AX + p = pc; + gins(mov, N, N); + p->from.type = D_INDIR+D_SP; + p->from.offset = widthptr; + p->to.type = D_AX; +//print("2. %P\n", p); + } + + //MOVL o(AX), AX + p = pc; + gins(mov, N, N); + p->from.type = D_INDIR+D_AX; + p->from.offset = o; + p->to.type = D_AX; +//print("3. %P\n", p); + o = 0; + } + if(o != 0) { + //ADDL $XX, AX + p = pc; + gins(add, N, N); + p->from.type = D_CONST; + p->from.offset = o; + if(loaded) + p->to.type = D_AX; + else { + p->to.type = D_INDIR+D_SP; + p->to.offset = widthptr; + } +//print("4. %P\n", p); + } + + //MOVL AX, 4(SP) + if(loaded) { + p = pc; + gins(mov, N, N); + p->from.type = D_AX; + p->to.type = D_INDIR+D_SP; + p->to.offset = widthptr; +//print("5. %P\n", p); + } else { + // TODO(rsc): obviously this is unnecessary, + // but 6l has a bug, and it can't handle + // JMP instructions too close to the top of + // a new function. + p = pc; + gins(ANOP, N, N); + } + + f = dotlist[0].field; + //JMP main·*Sub_test2(SB) + if(isptr[f->type->etype]) + f = f->type; + p = pc; + gins(AJMP, N, N); + p->to.type = D_EXTERN; + p->to.sym = methodsym(method->sym, ptrto(f->type), 0); +//print("6. %P\n", p); + + pc->as = ARET; // overwrite AEND +} + +void +nopout(Prog *p) +{ + p->as = ANOP; +} + diff --git a/src/cmd/8g/gsubr.c b/src/cmd/8g/gsubr.c new file mode 100644 index 000000000..a35c81eb1 --- /dev/null +++ b/src/cmd/8g/gsubr.c @@ -0,0 +1,1959 @@ +// Derived from Inferno utils/8c/txt.c +// http://code.google.com/p/inferno-os/source/browse/utils/8c/txt.c +// +// Copyright © 1994-1999 Lucent Technologies Inc. All rights reserved. +// Portions Copyright © 1995-1997 C H Forsyth (forsyth@terzarima.net) +// Portions Copyright © 1997-1999 Vita Nuova Limited +// Portions Copyright © 2000-2007 Vita Nuova Holdings Limited (www.vitanuova.com) +// Portions Copyright © 2004,2006 Bruce Ellis +// Portions Copyright © 2005-2007 C H Forsyth (forsyth@terzarima.net) +// Revisions Copyright © 2000-2007 Lucent Technologies Inc. and others +// Portions Copyright © 2009 The Go Authors. All rights reserved. +// +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to deal +// in the Software without restriction, including without limitation the rights +// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +// copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +// THE SOFTWARE. + +#include "gg.h" + +// TODO(rsc): Can make this bigger if we move +// the text segment up higher in 8l for all GOOS. +uint32 unmappedzero = 4096; + +#define CASE(a,b) (((a)<<16)|((b)<<0)) + +void +clearp(Prog *p) +{ + p->as = AEND; + p->from.type = D_NONE; + p->from.index = D_NONE; + p->to.type = D_NONE; + p->to.index = D_NONE; + p->loc = pcloc; + pcloc++; +} + +/* + * generate and return proc with p->as = as, + * linked into program. pc is next instruction. + */ +Prog* +prog(int as) +{ + Prog *p; + + p = pc; + pc = mal(sizeof(*pc)); + + clearp(pc); + + if(lineno == 0) { + if(debug['K']) + warn("prog: line 0"); + } + + p->as = as; + p->lineno = lineno; + p->link = pc; + return p; +} + +/* + * generate a branch. + * t is ignored. + */ +Prog* +gbranch(int as, Type *t) +{ + Prog *p; + + p = prog(as); + p->to.type = D_BRANCH; + p->to.branch = P; + return p; +} + +/* + * patch previous branch to jump to to. + */ +void +patch(Prog *p, Prog *to) +{ + if(p->to.type != D_BRANCH) + fatal("patch: not a branch"); + p->to.branch = to; + p->to.offset = to->loc; +} + +Prog* +unpatch(Prog *p) +{ + Prog *q; + + if(p->to.type != D_BRANCH) + fatal("unpatch: not a branch"); + q = p->to.branch; + p->to.branch = P; + p->to.offset = 0; + return q; +} + +/* + * start a new Prog list. + */ +Plist* +newplist(void) +{ + Plist *pl; + + pl = mal(sizeof(*pl)); + if(plist == nil) + plist = pl; + else + plast->link = pl; + plast = pl; + + pc = mal(sizeof(*pc)); + clearp(pc); + pl->firstpc = pc; + + return pl; +} + +void +clearstk(void) +{ + Plist *pl; + Prog *p1, *p2; + Node sp, di, cx, con, ax; + + if(plast->firstpc->to.offset <= 0) + return; + + // reestablish context for inserting code + // at beginning of function. + pl = plast; + p1 = pl->firstpc; + p2 = p1->link; + pc = mal(sizeof(*pc)); + clearp(pc); + p1->link = pc; + + // zero stack frame + nodreg(&sp, types[tptr], D_SP); + nodreg(&di, types[tptr], D_DI); + nodreg(&cx, types[TUINT32], D_CX); + nodconst(&con, types[TUINT32], p1->to.offset / widthptr); + gins(ACLD, N, N); + gins(AMOVL, &sp, &di); + gins(AMOVL, &con, &cx); + nodconst(&con, types[TUINT32], 0); + nodreg(&ax, types[TUINT32], D_AX); + gins(AMOVL, &con, &ax); + gins(AREP, N, N); + gins(ASTOSL, N, N); + + // continue with original code. + gins(ANOP, N, N)->link = p2; + pc = P; +} + +void +gused(Node *n) +{ + gins(ANOP, n, N); // used +} + +Prog* +gjmp(Prog *to) +{ + Prog *p; + + p = gbranch(AJMP, T); + if(to != P) + patch(p, to); + return p; +} + +void +ggloblnod(Node *nam, int32 width) +{ + Prog *p; + + p = gins(AGLOBL, nam, N); + p->lineno = nam->lineno; + p->to.sym = S; + p->to.type = D_CONST; + p->to.offset = width; + if(nam->readonly) + p->from.scale = RODATA; +} + +void +ggloblsym(Sym *s, int32 width, int dupok) +{ + Prog *p; + + p = gins(AGLOBL, N, N); + p->from.type = D_EXTERN; + p->from.index = D_NONE; + p->from.sym = s; + p->to.type = D_CONST; + p->to.index = D_NONE; + p->to.offset = width; + if(dupok) + p->from.scale = DUPOK; + p->from.scale |= RODATA; +} + +int +isfat(Type *t) +{ + if(t != T) + switch(t->etype) { + case TSTRUCT: + case TARRAY: + case TSTRING: + case TINTER: // maybe remove later + return 1; + } + return 0; +} + +/* + * naddr of func generates code for address of func. + * if using opcode that can take address implicitly, + * call afunclit to fix up the argument. + */ +void +afunclit(Addr *a) +{ + if(a->type == D_ADDR && a->index == D_EXTERN) { + a->type = D_EXTERN; + a->index = D_NONE; + } +} + +/* + * return Axxx for Oxxx on type t. + */ +int +optoas(int op, Type *t) +{ + int a; + + if(t == T) + fatal("optoas: t is nil"); + + a = AGOK; + switch(CASE(op, simtype[t->etype])) { + default: + fatal("optoas: no entry %O-%T", op, t); + break; + + case CASE(OADDR, TPTR32): + a = ALEAL; + break; + + case CASE(OEQ, TBOOL): + case CASE(OEQ, TINT8): + case CASE(OEQ, TUINT8): + case CASE(OEQ, TINT16): + case CASE(OEQ, TUINT16): + case CASE(OEQ, TINT32): + case CASE(OEQ, TUINT32): + case CASE(OEQ, TINT64): + case CASE(OEQ, TUINT64): + case CASE(OEQ, TPTR32): + case CASE(OEQ, TPTR64): + case CASE(OEQ, TFLOAT32): + case CASE(OEQ, TFLOAT64): + a = AJEQ; + break; + + case CASE(ONE, TBOOL): + case CASE(ONE, TINT8): + case CASE(ONE, TUINT8): + case CASE(ONE, TINT16): + case CASE(ONE, TUINT16): + case CASE(ONE, TINT32): + case CASE(ONE, TUINT32): + case CASE(ONE, TINT64): + case CASE(ONE, TUINT64): + case CASE(ONE, TPTR32): + case CASE(ONE, TPTR64): + case CASE(ONE, TFLOAT32): + case CASE(ONE, TFLOAT64): + a = AJNE; + break; + + case CASE(OLT, TINT8): + case CASE(OLT, TINT16): + case CASE(OLT, TINT32): + case CASE(OLT, TINT64): + a = AJLT; + break; + + case CASE(OLT, TUINT8): + case CASE(OLT, TUINT16): + case CASE(OLT, TUINT32): + case CASE(OLT, TUINT64): + a = AJCS; + break; + + case CASE(OLE, TINT8): + case CASE(OLE, TINT16): + case CASE(OLE, TINT32): + case CASE(OLE, TINT64): + a = AJLE; + break; + + case CASE(OLE, TUINT8): + case CASE(OLE, TUINT16): + case CASE(OLE, TUINT32): + case CASE(OLE, TUINT64): + a = AJLS; + break; + + case CASE(OGT, TINT8): + case CASE(OGT, TINT16): + case CASE(OGT, TINT32): + case CASE(OGT, TINT64): + a = AJGT; + break; + + case CASE(OGT, TUINT8): + case CASE(OGT, TUINT16): + case CASE(OGT, TUINT32): + case CASE(OGT, TUINT64): + case CASE(OLT, TFLOAT32): + case CASE(OLT, TFLOAT64): + a = AJHI; + break; + + case CASE(OGE, TINT8): + case CASE(OGE, TINT16): + case CASE(OGE, TINT32): + case CASE(OGE, TINT64): + a = AJGE; + break; + + case CASE(OGE, TUINT8): + case CASE(OGE, TUINT16): + case CASE(OGE, TUINT32): + case CASE(OGE, TUINT64): + case CASE(OLE, TFLOAT32): + case CASE(OLE, TFLOAT64): + a = AJCC; + break; + + case CASE(OCMP, TBOOL): + case CASE(OCMP, TINT8): + case CASE(OCMP, TUINT8): + a = ACMPB; + break; + + case CASE(OCMP, TINT16): + case CASE(OCMP, TUINT16): + a = ACMPW; + break; + + case CASE(OCMP, TINT32): + case CASE(OCMP, TUINT32): + case CASE(OCMP, TPTR32): + a = ACMPL; + break; + + case CASE(OAS, TBOOL): + case CASE(OAS, TINT8): + case CASE(OAS, TUINT8): + a = AMOVB; + break; + + case CASE(OAS, TINT16): + case CASE(OAS, TUINT16): + a = AMOVW; + break; + + case CASE(OAS, TINT32): + case CASE(OAS, TUINT32): + case CASE(OAS, TPTR32): + a = AMOVL; + break; + + case CASE(OADD, TINT8): + case CASE(OADD, TUINT8): + a = AADDB; + break; + + case CASE(OADD, TINT16): + case CASE(OADD, TUINT16): + a = AADDW; + break; + + case CASE(OADD, TINT32): + case CASE(OADD, TUINT32): + case CASE(OADD, TPTR32): + a = AADDL; + break; + + case CASE(OSUB, TINT8): + case CASE(OSUB, TUINT8): + a = ASUBB; + break; + + case CASE(OSUB, TINT16): + case CASE(OSUB, TUINT16): + a = ASUBW; + break; + + case CASE(OSUB, TINT32): + case CASE(OSUB, TUINT32): + case CASE(OSUB, TPTR32): + a = ASUBL; + break; + + case CASE(OINC, TINT8): + case CASE(OINC, TUINT8): + a = AINCB; + break; + + case CASE(OINC, TINT16): + case CASE(OINC, TUINT16): + a = AINCW; + break; + + case CASE(OINC, TINT32): + case CASE(OINC, TUINT32): + case CASE(OINC, TPTR32): + a = AINCL; + break; + + case CASE(ODEC, TINT8): + case CASE(ODEC, TUINT8): + a = ADECB; + break; + + case CASE(ODEC, TINT16): + case CASE(ODEC, TUINT16): + a = ADECW; + break; + + case CASE(ODEC, TINT32): + case CASE(ODEC, TUINT32): + case CASE(ODEC, TPTR32): + a = ADECL; + break; + + case CASE(OCOM, TINT8): + case CASE(OCOM, TUINT8): + a = ANOTB; + break; + + case CASE(OCOM, TINT16): + case CASE(OCOM, TUINT16): + a = ANOTW; + break; + + case CASE(OCOM, TINT32): + case CASE(OCOM, TUINT32): + case CASE(OCOM, TPTR32): + a = ANOTL; + break; + + case CASE(OMINUS, TINT8): + case CASE(OMINUS, TUINT8): + a = ANEGB; + break; + + case CASE(OMINUS, TINT16): + case CASE(OMINUS, TUINT16): + a = ANEGW; + break; + + case CASE(OMINUS, TINT32): + case CASE(OMINUS, TUINT32): + case CASE(OMINUS, TPTR32): + a = ANEGL; + break; + + case CASE(OAND, TINT8): + case CASE(OAND, TUINT8): + a = AANDB; + break; + + case CASE(OAND, TINT16): + case CASE(OAND, TUINT16): + a = AANDW; + break; + + case CASE(OAND, TINT32): + case CASE(OAND, TUINT32): + case CASE(OAND, TPTR32): + a = AANDL; + break; + + case CASE(OOR, TINT8): + case CASE(OOR, TUINT8): + a = AORB; + break; + + case CASE(OOR, TINT16): + case CASE(OOR, TUINT16): + a = AORW; + break; + + case CASE(OOR, TINT32): + case CASE(OOR, TUINT32): + case CASE(OOR, TPTR32): + a = AORL; + break; + + case CASE(OXOR, TINT8): + case CASE(OXOR, TUINT8): + a = AXORB; + break; + + case CASE(OXOR, TINT16): + case CASE(OXOR, TUINT16): + a = AXORW; + break; + + case CASE(OXOR, TINT32): + case CASE(OXOR, TUINT32): + case CASE(OXOR, TPTR32): + a = AXORL; + break; + + case CASE(OLSH, TINT8): + case CASE(OLSH, TUINT8): + a = ASHLB; + break; + + case CASE(OLSH, TINT16): + case CASE(OLSH, TUINT16): + a = ASHLW; + break; + + case CASE(OLSH, TINT32): + case CASE(OLSH, TUINT32): + case CASE(OLSH, TPTR32): + a = ASHLL; + break; + + case CASE(ORSH, TUINT8): + a = ASHRB; + break; + + case CASE(ORSH, TUINT16): + a = ASHRW; + break; + + case CASE(ORSH, TUINT32): + case CASE(ORSH, TPTR32): + a = ASHRL; + break; + + case CASE(ORSH, TINT8): + a = ASARB; + break; + + case CASE(ORSH, TINT16): + a = ASARW; + break; + + case CASE(ORSH, TINT32): + a = ASARL; + break; + + case CASE(OMUL, TINT8): + case CASE(OMUL, TUINT8): + a = AIMULB; + break; + + case CASE(OMUL, TINT16): + case CASE(OMUL, TUINT16): + a = AIMULW; + break; + + case CASE(OMUL, TINT32): + case CASE(OMUL, TUINT32): + case CASE(OMUL, TPTR32): + a = AIMULL; + break; + + case CASE(ODIV, TINT8): + case CASE(OMOD, TINT8): + a = AIDIVB; + break; + + case CASE(ODIV, TUINT8): + case CASE(OMOD, TUINT8): + a = ADIVB; + break; + + case CASE(ODIV, TINT16): + case CASE(OMOD, TINT16): + a = AIDIVW; + break; + + case CASE(ODIV, TUINT16): + case CASE(OMOD, TUINT16): + a = ADIVW; + break; + + case CASE(ODIV, TINT32): + case CASE(OMOD, TINT32): + a = AIDIVL; + break; + + case CASE(ODIV, TUINT32): + case CASE(ODIV, TPTR32): + case CASE(OMOD, TUINT32): + case CASE(OMOD, TPTR32): + a = ADIVL; + break; + + case CASE(OEXTEND, TINT16): + a = ACWD; + break; + + case CASE(OEXTEND, TINT32): + a = ACDQ; + break; + } + return a; +} + +#define FCASE(a, b, c) (((a)<<16)|((b)<<8)|(c)) +int +foptoas(int op, Type *t, int flg) +{ + int et; + + et = simtype[t->etype]; + + // If we need Fpop, it means we're working on + // two different floating-point registers, not memory. + // There the instruction only has a float64 form. + if(flg & Fpop) + et = TFLOAT64; + + // clear Frev if unneeded + switch(op) { + case OADD: + case OMUL: + flg &= ~Frev; + break; + } + + switch(FCASE(op, et, flg)) { + case FCASE(OADD, TFLOAT32, 0): + return AFADDF; + case FCASE(OADD, TFLOAT64, 0): + return AFADDD; + case FCASE(OADD, TFLOAT64, Fpop): + return AFADDDP; + + case FCASE(OSUB, TFLOAT32, 0): + return AFSUBF; + case FCASE(OSUB, TFLOAT32, Frev): + return AFSUBRF; + + case FCASE(OSUB, TFLOAT64, 0): + return AFSUBD; + case FCASE(OSUB, TFLOAT64, Frev): + return AFSUBRD; + case FCASE(OSUB, TFLOAT64, Fpop): + return AFSUBDP; + case FCASE(OSUB, TFLOAT64, Fpop|Frev): + return AFSUBRDP; + + case FCASE(OMUL, TFLOAT32, 0): + return AFMULF; + case FCASE(OMUL, TFLOAT64, 0): + return AFMULD; + case FCASE(OMUL, TFLOAT64, Fpop): + return AFMULDP; + + case FCASE(ODIV, TFLOAT32, 0): + return AFDIVF; + case FCASE(ODIV, TFLOAT32, Frev): + return AFDIVRF; + + case FCASE(ODIV, TFLOAT64, 0): + return AFDIVD; + case FCASE(ODIV, TFLOAT64, Frev): + return AFDIVRD; + case FCASE(ODIV, TFLOAT64, Fpop): + return AFDIVDP; + case FCASE(ODIV, TFLOAT64, Fpop|Frev): + return AFDIVRDP; + + case FCASE(OCMP, TFLOAT32, 0): + return AFCOMF; + case FCASE(OCMP, TFLOAT32, Fpop): + return AFCOMFP; + case FCASE(OCMP, TFLOAT64, 0): + return AFCOMD; + case FCASE(OCMP, TFLOAT64, Fpop): + return AFCOMDP; + case FCASE(OCMP, TFLOAT64, Fpop2): + return AFCOMDPP; + + case FCASE(OMINUS, TFLOAT32, 0): + return AFCHS; + case FCASE(OMINUS, TFLOAT64, 0): + return AFCHS; + } + + fatal("foptoas %O %T %#x", op, t, flg); + return 0; +} + +static int resvd[] = +{ +// D_DI, // for movstring +// D_SI, // for movstring + + D_AX, // for divide + D_CX, // for shift + D_DX, // for divide + D_SP, // for stack + + D_BL, // because D_BX can be allocated + D_BH, +}; + +void +ginit(void) +{ + int i; + + for(i=0; ietype]; + + switch(et) { + case TINT8: + case TUINT8: + case TINT16: + case TUINT16: + case TINT32: + case TUINT32: + case TINT64: + case TUINT64: + case TPTR32: + case TPTR64: + case TBOOL: + if(o != N && o->op == OREGISTER) { + i = o->val.u.reg; + if(i >= D_AX && i <= D_DI) + goto out; + } + for(i=D_AX; i<=D_DI; i++) + if(reg[i] == 0) + goto out; + + fprint(2, "registers allocated at\n"); + for(i=D_AX; i<=D_DI; i++) + fprint(2, "\t%R\t%#ux\n", i, regpc[i]); + yyerror("out of fixed registers"); + goto err; + + case TFLOAT32: + case TFLOAT64: + i = D_F0; + goto out; + } + yyerror("regalloc: unknown type %T", t); + i = 0; + +err: + nodreg(n, t, 0); + return; + +out: + if (i == D_SP) + print("alloc SP\n"); + if(reg[i] == 0) { + regpc[i] = (ulong)__builtin_return_address(0); + if(i == D_AX || i == D_CX || i == D_DX || i == D_SP) { + dump("regalloc-o", o); + fatal("regalloc %R", i); + } + } + reg[i]++; + nodreg(n, t, i); +} + +void +regfree(Node *n) +{ + int i; + + if(n->op == ONAME) + return; + if(n->op != OREGISTER && n->op != OINDREG) + fatal("regfree: not a register"); + i = n->val.u.reg; + if(i == D_SP) + return; + if(i < 0 || i >= sizeof(reg)) + fatal("regfree: reg out of range"); + if(reg[i] <= 0) + fatal("regfree: reg not allocated"); + reg[i]--; + if(reg[i] == 0 && (i == D_AX || i == D_CX || i == D_DX || i == D_SP)) + fatal("regfree %R", i); +} + +/* + * initialize n to be register r of type t. + */ +void +nodreg(Node *n, Type *t, int r) +{ + if(t == T) + fatal("nodreg: t nil"); + + memset(n, 0, sizeof(*n)); + n->op = OREGISTER; + n->addable = 1; + ullmancalc(n); + n->val.u.reg = r; + n->type = t; +} + +/* + * initialize n to be indirect of register r; n is type t. + */ +void +nodindreg(Node *n, Type *t, int r) +{ + nodreg(n, t, r); + n->op = OINDREG; +} + +Node* +nodarg(Type *t, int fp) +{ + Node *n; + Type *first; + Iter savet; + + // entire argument struct, not just one arg + switch(t->etype) { + default: + fatal("nodarg %T", t); + + case TSTRUCT: + if(!t->funarg) + fatal("nodarg: TSTRUCT but not funarg"); + n = nod(ONAME, N, N); + n->sym = lookup(".args"); + n->type = t; + first = structfirst(&savet, &t); + if(first == nil) + fatal("nodarg: bad struct"); + if(first->width == BADWIDTH) + fatal("nodarg: offset not computed for %T", t); + n->xoffset = first->width; + n->addable = 1; + break; + + case TFIELD: + n = nod(ONAME, N, N); + n->type = t->type; + n->sym = t->sym; + if(t->width == BADWIDTH) + fatal("nodarg: offset not computed for %T", t); + n->xoffset = t->width; + n->addable = 1; + break; + } + + switch(fp) { + default: + fatal("nodarg %T %d", t, fp); + + case 0: // output arg + n->op = OINDREG; + n->val.u.reg = D_SP; + break; + + case 1: // input arg + n->class = PPARAM; + break; + } + + n->typecheck = 1; + return n; +} + +/* + * generate + * as $c, reg + */ +void +gconreg(int as, vlong c, int reg) +{ + Node n1, n2; + + nodconst(&n1, types[TINT64], c); + nodreg(&n2, types[TINT64], reg); + gins(as, &n1, &n2); +} + +/* + * swap node contents + */ +void +nswap(Node *a, Node *b) +{ + Node t; + + t = *a; + *a = *b; + *b = t; +} + +/* + * return constant i node. + * overwritten by next call, but useful in calls to gins. + */ +Node* +ncon(uint32 i) +{ + static Node n; + + if(n.type == T) + nodconst(&n, types[TUINT32], 0); + mpmovecfix(n.val.u.xval, i); + return &n; +} + +/* + * Is this node a memory operand? + */ +int +ismem(Node *n) +{ + switch(n->op) { + case OLEN: + case OCAP: + case OINDREG: + case ONAME: + case OPARAM: + return 1; + } + return 0; +} + +Node sclean[10]; +int nsclean; + +/* + * n is a 64-bit value. fill in lo and hi to refer to its 32-bit halves. + */ +void +split64(Node *n, Node *lo, Node *hi) +{ + Node n1; + int64 i; + + if(!is64(n->type)) + fatal("split64 %T", n->type); + + sclean[nsclean].op = OEMPTY; + if(nsclean >= nelem(sclean)) + fatal("split64 clean"); + nsclean++; + switch(n->op) { + default: + if(!dotaddable(n, &n1)) { + igen(n, &n1, N); + sclean[nsclean-1] = n1; + } + n = &n1; + goto common; + case ONAME: + if(n->class == PPARAMREF) { + cgen(n->heapaddr, &n1); + sclean[nsclean-1] = n1; + // fall through. + n = &n1; + } + goto common; + case OINDREG: + common: + *lo = *n; + *hi = *n; + lo->type = types[TUINT32]; + if(n->type->etype == TINT64) + hi->type = types[TINT32]; + else + hi->type = types[TUINT32]; + hi->xoffset += 4; + break; + + case OLITERAL: + convconst(&n1, n->type, &n->val); + i = mpgetfix(n1.val.u.xval); + nodconst(lo, types[TUINT32], (uint32)i); + i >>= 32; + if(n->type->etype == TINT64) + nodconst(hi, types[TINT32], (int32)i); + else + nodconst(hi, types[TUINT32], (uint32)i); + break; + } +} + +void +splitclean(void) +{ + if(nsclean <= 0) + fatal("splitclean"); + nsclean--; + if(sclean[nsclean].op != OEMPTY) + regfree(&sclean[nsclean]); +} + +/* + * set up nodes representing fp constants + */ +Node zerof; +Node two64f; +Node two63f; + +void +bignodes(void) +{ + static int did; + + if(did) + return; + did = 1; + + two64f = *ncon(0); + two64f.type = types[TFLOAT64]; + two64f.val.ctype = CTFLT; + two64f.val.u.fval = mal(sizeof *two64f.val.u.fval); + mpmovecflt(two64f.val.u.fval, 18446744073709551616.); + + two63f = two64f; + two63f.val.u.fval = mal(sizeof *two63f.val.u.fval); + mpmovecflt(two63f.val.u.fval, 9223372036854775808.); + + zerof = two64f; + zerof.val.u.fval = mal(sizeof *zerof.val.u.fval); + mpmovecflt(zerof.val.u.fval, 0); +} + +void +memname(Node *n, Type *t) +{ + tempname(n, t); + strcpy(namebuf, n->sym->name); + namebuf[0] = '.'; // keep optimizer from registerizing + n->sym = lookup(namebuf); +} + +void +gmove(Node *f, Node *t) +{ + int a, ft, tt; + Type *cvt; + Node r1, r2, t1, t2, flo, fhi, tlo, thi, con, f0, f1, ax, dx, cx; + Prog *p1, *p2, *p3; + + if(debug['M']) + print("gmove %N -> %N\n", f, t); + + ft = simsimtype(f->type); + tt = simsimtype(t->type); + cvt = t->type; + + if(iscomplex[ft] || iscomplex[tt]) { + complexmove(f, t); + return; + } + + // cannot have two integer memory operands; + // except 64-bit, which always copies via registers anyway. + if(isint[ft] && isint[tt] && !is64(f->type) && !is64(t->type) && ismem(f) && ismem(t)) + goto hard; + + // convert constant to desired type + if(f->op == OLITERAL) { + if(tt == TFLOAT32) + convconst(&con, types[TFLOAT64], &f->val); + else + convconst(&con, t->type, &f->val); + f = &con; + ft = simsimtype(con.type); + + // some constants can't move directly to memory. + if(ismem(t)) { + // float constants come from memory. + if(isfloat[tt]) + goto hard; + } + } + + // value -> value copy, only one memory operand. + // figure out the instruction to use. + // break out of switch for one-instruction gins. + // goto rdst for "destination must be register". + // goto hard for "convert to cvt type first". + // otherwise handle and return. + + switch(CASE(ft, tt)) { + default: + goto fatal; + + /* + * integer copy and truncate + */ + case CASE(TINT8, TINT8): // same size + case CASE(TINT8, TUINT8): + case CASE(TUINT8, TINT8): + case CASE(TUINT8, TUINT8): + a = AMOVB; + break; + + case CASE(TINT16, TINT8): // truncate + case CASE(TUINT16, TINT8): + case CASE(TINT32, TINT8): + case CASE(TUINT32, TINT8): + case CASE(TINT16, TUINT8): + case CASE(TUINT16, TUINT8): + case CASE(TINT32, TUINT8): + case CASE(TUINT32, TUINT8): + a = AMOVB; + goto rsrc; + + case CASE(TINT64, TINT8): // truncate low word + case CASE(TUINT64, TINT8): + case CASE(TINT64, TUINT8): + case CASE(TUINT64, TUINT8): + split64(f, &flo, &fhi); + nodreg(&r1, t->type, D_AX); + gmove(&flo, &r1); + gins(AMOVB, &r1, t); + splitclean(); + return; + + case CASE(TINT16, TINT16): // same size + case CASE(TINT16, TUINT16): + case CASE(TUINT16, TINT16): + case CASE(TUINT16, TUINT16): + a = AMOVW; + break; + + case CASE(TINT32, TINT16): // truncate + case CASE(TUINT32, TINT16): + case CASE(TINT32, TUINT16): + case CASE(TUINT32, TUINT16): + a = AMOVW; + goto rsrc; + + case CASE(TINT64, TINT16): // truncate low word + case CASE(TUINT64, TINT16): + case CASE(TINT64, TUINT16): + case CASE(TUINT64, TUINT16): + split64(f, &flo, &fhi); + nodreg(&r1, t->type, D_AX); + gmove(&flo, &r1); + gins(AMOVW, &r1, t); + splitclean(); + return; + + case CASE(TINT32, TINT32): // same size + case CASE(TINT32, TUINT32): + case CASE(TUINT32, TINT32): + case CASE(TUINT32, TUINT32): + a = AMOVL; + break; + + case CASE(TINT64, TINT32): // truncate + case CASE(TUINT64, TINT32): + case CASE(TINT64, TUINT32): + case CASE(TUINT64, TUINT32): + split64(f, &flo, &fhi); + nodreg(&r1, t->type, D_AX); + gmove(&flo, &r1); + gins(AMOVL, &r1, t); + splitclean(); + return; + + case CASE(TINT64, TINT64): // same size + case CASE(TINT64, TUINT64): + case CASE(TUINT64, TINT64): + case CASE(TUINT64, TUINT64): + split64(f, &flo, &fhi); + split64(t, &tlo, &thi); + if(f->op == OLITERAL) { + gins(AMOVL, &flo, &tlo); + gins(AMOVL, &fhi, &thi); + } else { + nodreg(&r1, t->type, D_AX); + nodreg(&r2, t->type, D_DX); + gins(AMOVL, &flo, &r1); + gins(AMOVL, &fhi, &r2); + gins(AMOVL, &r1, &tlo); + gins(AMOVL, &r2, &thi); + } + splitclean(); + splitclean(); + return; + + /* + * integer up-conversions + */ + case CASE(TINT8, TINT16): // sign extend int8 + case CASE(TINT8, TUINT16): + a = AMOVBWSX; + goto rdst; + case CASE(TINT8, TINT32): + case CASE(TINT8, TUINT32): + a = AMOVBLSX; + goto rdst; + case CASE(TINT8, TINT64): // convert via int32 + case CASE(TINT8, TUINT64): + cvt = types[TINT32]; + goto hard; + + case CASE(TUINT8, TINT16): // zero extend uint8 + case CASE(TUINT8, TUINT16): + a = AMOVBWZX; + goto rdst; + case CASE(TUINT8, TINT32): + case CASE(TUINT8, TUINT32): + a = AMOVBLZX; + goto rdst; + case CASE(TUINT8, TINT64): // convert via uint32 + case CASE(TUINT8, TUINT64): + cvt = types[TUINT32]; + goto hard; + + case CASE(TINT16, TINT32): // sign extend int16 + case CASE(TINT16, TUINT32): + a = AMOVWLSX; + goto rdst; + case CASE(TINT16, TINT64): // convert via int32 + case CASE(TINT16, TUINT64): + cvt = types[TINT32]; + goto hard; + + case CASE(TUINT16, TINT32): // zero extend uint16 + case CASE(TUINT16, TUINT32): + a = AMOVWLZX; + goto rdst; + case CASE(TUINT16, TINT64): // convert via uint32 + case CASE(TUINT16, TUINT64): + cvt = types[TUINT32]; + goto hard; + + case CASE(TINT32, TINT64): // sign extend int32 + case CASE(TINT32, TUINT64): + split64(t, &tlo, &thi); + nodreg(&flo, tlo.type, D_AX); + nodreg(&fhi, thi.type, D_DX); + gmove(f, &flo); + gins(ACDQ, N, N); + gins(AMOVL, &flo, &tlo); + gins(AMOVL, &fhi, &thi); + splitclean(); + return; + + case CASE(TUINT32, TINT64): // zero extend uint32 + case CASE(TUINT32, TUINT64): + split64(t, &tlo, &thi); + gmove(f, &tlo); + gins(AMOVL, ncon(0), &thi); + splitclean(); + return; + + /* + * float to integer + */ + case CASE(TFLOAT32, TINT16): + case CASE(TFLOAT32, TINT32): + case CASE(TFLOAT32, TINT64): + case CASE(TFLOAT64, TINT16): + case CASE(TFLOAT64, TINT32): + case CASE(TFLOAT64, TINT64): + if(t->op == OREGISTER) + goto hardmem; + nodreg(&r1, types[ft], D_F0); + if(f->op != OREGISTER) { + if(ft == TFLOAT32) + gins(AFMOVF, f, &r1); + else + gins(AFMOVD, f, &r1); + } + + // set round to zero mode during conversion + memname(&t1, types[TUINT16]); + memname(&t2, types[TUINT16]); + gins(AFSTCW, N, &t1); + gins(AMOVW, ncon(0xf7f), &t2); + gins(AFLDCW, &t2, N); + if(tt == TINT16) + gins(AFMOVWP, &r1, t); + else if(tt == TINT32) + gins(AFMOVLP, &r1, t); + else + gins(AFMOVVP, &r1, t); + gins(AFLDCW, &t1, N); + return; + + case CASE(TFLOAT32, TINT8): + case CASE(TFLOAT32, TUINT16): + case CASE(TFLOAT32, TUINT8): + case CASE(TFLOAT64, TINT8): + case CASE(TFLOAT64, TUINT16): + case CASE(TFLOAT64, TUINT8): + // convert via int32. + tempname(&t1, types[TINT32]); + gmove(f, &t1); + switch(tt) { + default: + fatal("gmove %T", t); + case TINT8: + gins(ACMPL, &t1, ncon(-0x80)); + p1 = gbranch(optoas(OLT, types[TINT32]), T); + gins(ACMPL, &t1, ncon(0x7f)); + p2 = gbranch(optoas(OGT, types[TINT32]), T); + p3 = gbranch(AJMP, T); + patch(p1, pc); + patch(p2, pc); + gmove(ncon(-0x80), &t1); + patch(p3, pc); + gmove(&t1, t); + break; + case TUINT8: + gins(ATESTL, ncon(0xffffff00), &t1); + p1 = gbranch(AJEQ, T); + gins(AMOVL, ncon(0), &t1); + patch(p1, pc); + gmove(&t1, t); + break; + case TUINT16: + gins(ATESTL, ncon(0xffff0000), &t1); + p1 = gbranch(AJEQ, T); + gins(AMOVL, ncon(0), &t1); + patch(p1, pc); + gmove(&t1, t); + break; + } + return; + + case CASE(TFLOAT32, TUINT32): + case CASE(TFLOAT64, TUINT32): + // convert via int64. + tempname(&t1, types[TINT64]); + gmove(f, &t1); + split64(&t1, &tlo, &thi); + gins(ACMPL, &thi, ncon(0)); + p1 = gbranch(AJEQ, T); + gins(AMOVL, ncon(0), &tlo); + patch(p1, pc); + gmove(&tlo, t); + splitclean(); + return; + + case CASE(TFLOAT32, TUINT64): + case CASE(TFLOAT64, TUINT64): + bignodes(); + nodreg(&f0, types[ft], D_F0); + nodreg(&f1, types[ft], D_F0 + 1); + nodreg(&ax, types[TUINT16], D_AX); + + gmove(f, &f0); + + // if 0 > v { answer = 0 } + gmove(&zerof, &f0); + gins(AFUCOMIP, &f0, &f1); + p1 = gbranch(optoas(OGT, types[tt]), T); + // if 1<<64 <= v { answer = 0 too } + gmove(&two64f, &f0); + gins(AFUCOMIP, &f0, &f1); + p2 = gbranch(optoas(OGT, types[tt]), T); + patch(p1, pc); + gins(AFMOVVP, &f0, t); // don't care about t, but will pop the stack + split64(t, &tlo, &thi); + gins(AMOVL, ncon(0), &tlo); + gins(AMOVL, ncon(0), &thi); + splitclean(); + p1 = gbranch(AJMP, T); + patch(p2, pc); + + // in range; algorithm is: + // if small enough, use native float64 -> int64 conversion. + // otherwise, subtract 2^63, convert, and add it back. + + // set round to zero mode during conversion + memname(&t1, types[TUINT16]); + memname(&t2, types[TUINT16]); + gins(AFSTCW, N, &t1); + gins(AMOVW, ncon(0xf7f), &t2); + gins(AFLDCW, &t2, N); + + // actual work + gmove(&two63f, &f0); + gins(AFUCOMIP, &f0, &f1); + p2 = gbranch(optoas(OLE, types[tt]), T); + gins(AFMOVVP, &f0, t); + p3 = gbranch(AJMP, T); + patch(p2, pc); + gmove(&two63f, &f0); + gins(AFSUBDP, &f0, &f1); + gins(AFMOVVP, &f0, t); + split64(t, &tlo, &thi); + gins(AXORL, ncon(0x80000000), &thi); // + 2^63 + patch(p3, pc); + splitclean(); + // restore rounding mode + gins(AFLDCW, &t1, N); + + patch(p1, pc); + return; + + /* + * integer to float + */ + case CASE(TINT16, TFLOAT32): + case CASE(TINT16, TFLOAT64): + case CASE(TINT32, TFLOAT32): + case CASE(TINT32, TFLOAT64): + case CASE(TINT64, TFLOAT32): + case CASE(TINT64, TFLOAT64): + if(t->op != OREGISTER) + goto hard; + if(f->op == OREGISTER) { + cvt = f->type; + goto hardmem; + } + switch(ft) { + case TINT16: + a = AFMOVW; + break; + case TINT32: + a = AFMOVL; + break; + default: + a = AFMOVV; + break; + } + break; + + case CASE(TINT8, TFLOAT32): + case CASE(TINT8, TFLOAT64): + case CASE(TUINT16, TFLOAT32): + case CASE(TUINT16, TFLOAT64): + case CASE(TUINT8, TFLOAT32): + case CASE(TUINT8, TFLOAT64): + // convert via int32 memory + cvt = types[TINT32]; + goto hardmem; + + case CASE(TUINT32, TFLOAT32): + case CASE(TUINT32, TFLOAT64): + // convert via int64 memory + cvt = types[TINT64]; + goto hardmem; + + case CASE(TUINT64, TFLOAT32): + case CASE(TUINT64, TFLOAT64): + // algorithm is: + // if small enough, use native int64 -> uint64 conversion. + // otherwise, halve (rounding to odd?), convert, and double. + nodreg(&ax, types[TUINT32], D_AX); + nodreg(&dx, types[TUINT32], D_DX); + nodreg(&cx, types[TUINT32], D_CX); + tempname(&t1, f->type); + split64(&t1, &tlo, &thi); + gmove(f, &t1); + gins(ACMPL, &thi, ncon(0)); + p1 = gbranch(AJLT, T); + // native + t1.type = types[TINT64]; + gmove(&t1, t); + p2 = gbranch(AJMP, T); + // simulated + patch(p1, pc); + gmove(&tlo, &ax); + gmove(&thi, &dx); + p1 = gins(ASHRL, ncon(1), &ax); + p1->from.index = D_DX; // double-width shift DX -> AX + p1->from.scale = 0; + gins(ASETCC, N, &cx); + gins(AORB, &cx, &ax); + gins(ASHRL, ncon(1), &dx); + gmove(&dx, &thi); + gmove(&ax, &tlo); + nodreg(&r1, types[tt], D_F0); + nodreg(&r2, types[tt], D_F0 + 1); + gmove(&t1, &r1); // t1.type is TINT64 now, set above + gins(AFMOVD, &r1, &r1); + gins(AFADDDP, &r1, &r2); + gmove(&r1, t); + patch(p2, pc); + splitclean(); + return; + + /* + * float to float + */ + case CASE(TFLOAT32, TFLOAT32): + case CASE(TFLOAT64, TFLOAT64): + // The way the code generator uses floating-point + // registers, a move from F0 to F0 is intended as a no-op. + // On the x86, it's not: it pushes a second copy of F0 + // on the floating point stack. So toss it away here. + // Also, F0 is the *only* register we ever evaluate + // into, so we should only see register/register as F0/F0. + if(ismem(f) && ismem(t)) + goto hard; + if(f->op == OREGISTER && t->op == OREGISTER) { + if(f->val.u.reg != D_F0 || t->val.u.reg != D_F0) + goto fatal; + return; + } + a = AFMOVF; + if(ft == TFLOAT64) + a = AFMOVD; + if(ismem(t)) { + if(f->op != OREGISTER || f->val.u.reg != D_F0) + fatal("gmove %N", f); + a = AFMOVFP; + if(ft == TFLOAT64) + a = AFMOVDP; + } + break; + + case CASE(TFLOAT32, TFLOAT64): + if(ismem(f) && ismem(t)) + goto hard; + if(f->op == OREGISTER && t->op == OREGISTER) { + if(f->val.u.reg != D_F0 || t->val.u.reg != D_F0) + goto fatal; + return; + } + if(f->op == OREGISTER) + gins(AFMOVDP, f, t); + else + gins(AFMOVF, f, t); + return; + + case CASE(TFLOAT64, TFLOAT32): + if(ismem(f) && ismem(t)) + goto hard; + if(f->op == OREGISTER && t->op == OREGISTER) { + tempname(&r1, types[TFLOAT32]); + gins(AFMOVFP, f, &r1); + gins(AFMOVF, &r1, t); + return; + } + if(f->op == OREGISTER) + gins(AFMOVFP, f, t); + else + gins(AFMOVD, f, t); + return; + } + + gins(a, f, t); + return; + +rsrc: + // requires register source + regalloc(&r1, f->type, t); + gmove(f, &r1); + gins(a, &r1, t); + regfree(&r1); + return; + +rdst: + // requires register destination + regalloc(&r1, t->type, t); + gins(a, f, &r1); + gmove(&r1, t); + regfree(&r1); + return; + +hard: + // requires register intermediate + regalloc(&r1, cvt, t); + gmove(f, &r1); + gmove(&r1, t); + regfree(&r1); + return; + +hardmem: + // requires memory intermediate + tempname(&r1, cvt); + gmove(f, &r1); + gmove(&r1, t); + return; + +fatal: + // should not happen + fatal("gmove %N -> %N", f, t); +} + +int +samaddr(Node *f, Node *t) +{ + + if(f->op != t->op) + return 0; + + switch(f->op) { + case OREGISTER: + if(f->val.u.reg != t->val.u.reg) + break; + return 1; + } + return 0; +} +/* + * generate one instruction: + * as f, t + */ +Prog* +gins(int as, Node *f, Node *t) +{ + Prog *p; + Addr af, at; + int w; + + if(as == AFMOVF && f && f->op == OREGISTER && t && t->op == OREGISTER) + fatal("gins MOVF reg, reg"); + + switch(as) { + case AMOVB: + case AMOVW: + case AMOVL: + if(f != N && t != N && samaddr(f, t)) + return nil; + } + + memset(&af, 0, sizeof af); + memset(&at, 0, sizeof at); + if(f != N) + naddr(f, &af, 1); + if(t != N) + naddr(t, &at, 1); + p = prog(as); + if(f != N) + p->from = af; + if(t != N) + p->to = at; + if(debug['g']) + print("%P\n", p); + + w = 0; + switch(as) { + case AMOVB: + w = 1; + break; + case AMOVW: + w = 2; + break; + case AMOVL: + w = 4; + break; + } + + if(1 && w != 0 && f != N && (af.width > w || at.width > w)) { + dump("bad width from:", f); + dump("bad width to:", t); + fatal("bad width: %P (%d, %d)\n", p, af.width, at.width); + } + + return p; +} + +static void +checkoffset(Addr *a, int canemitcode) +{ + Prog *p; + + if(a->offset < unmappedzero) + return; + if(!canemitcode) + fatal("checkoffset %#x, cannot emit code", a->offset); + + // cannot rely on unmapped nil page at 0 to catch + // reference with large offset. instead, emit explicit + // test of 0(reg). + p = gins(ATESTB, nodintconst(0), N); + p->to = *a; + p->to.offset = 0; +} + +/* + * generate code to compute n; + * make a refer to result. + */ +void +naddr(Node *n, Addr *a, int canemitcode) +{ + a->scale = 0; + a->index = D_NONE; + a->type = D_NONE; + a->gotype = S; + a->node = N; + if(n == N) + return; + + switch(n->op) { + default: + fatal("naddr: bad %O %D", n->op, a); + break; + + case OREGISTER: + a->type = n->val.u.reg; + a->sym = S; + break; + + case OINDREG: + a->type = n->val.u.reg+D_INDIR; + a->sym = n->sym; + a->offset = n->xoffset; + break; + + case OPARAM: + // n->left is PHEAP ONAME for stack parameter. + // compute address of actual parameter on stack. + a->etype = n->left->type->etype; + a->width = n->left->type->width; + a->offset = n->xoffset; + a->sym = n->left->sym; + a->type = D_PARAM; + break; + + case ONAME: + a->etype = 0; + a->width = 0; + if(n->type != T) { + a->etype = simtype[n->type->etype]; + a->width = n->type->width; + a->gotype = ngotype(n); + } + a->pun = n->pun; + a->offset = n->xoffset; + a->sym = n->sym; + if(a->sym == S) + a->sym = lookup(".noname"); + if(n->method) { + if(n->type != T) + if(n->type->sym != S) + if(n->type->sym->pkg != nil) + a->sym = pkglookup(a->sym->name, n->type->sym->pkg); + } + + switch(n->class) { + default: + fatal("naddr: ONAME class %S %d\n", n->sym, n->class); + case PEXTERN: + a->type = D_EXTERN; + break; + case PAUTO: + a->type = D_AUTO; + if (n->sym) + a->node = n->orig; + break; + case PPARAM: + case PPARAMOUT: + a->type = D_PARAM; + break; + case PFUNC: + a->index = D_EXTERN; + a->type = D_ADDR; + break; + } + break; + + case OLITERAL: + switch(n->val.ctype) { + default: + fatal("naddr: const %lT", n->type); + break; + case CTFLT: + a->type = D_FCONST; + a->dval = mpgetflt(n->val.u.fval); + break; + case CTINT: + a->sym = S; + a->type = D_CONST; + a->offset = mpgetfix(n->val.u.xval); + break; + case CTSTR: + datagostring(n->val.u.sval, a); + break; + case CTBOOL: + a->sym = S; + a->type = D_CONST; + a->offset = n->val.u.bval; + break; + case CTNIL: + a->sym = S; + a->type = D_CONST; + a->offset = 0; + break; + } + break; + + case OADDR: + naddr(n->left, a, canemitcode); + if(a->type >= D_INDIR) { + a->type -= D_INDIR; + break; + } + if(a->type == D_EXTERN || a->type == D_STATIC || + a->type == D_AUTO || a->type == D_PARAM) + if(a->index == D_NONE) { + a->index = a->type; + a->type = D_ADDR; + break; + } + fatal("naddr: OADDR\n"); + + case OLEN: + // len of string or slice + naddr(n->left, a, canemitcode); + if(a->type == D_CONST && a->offset == 0) + break; // len(nil) + a->etype = TUINT32; + a->offset += Array_nel; + a->width = 4; + if(a->offset >= unmappedzero && a->offset-Array_nel < unmappedzero) + checkoffset(a, canemitcode); + break; + + case OCAP: + // cap of string or slice + naddr(n->left, a, canemitcode); + if(a->type == D_CONST && a->offset == 0) + break; // cap(nil) + a->etype = TUINT32; + a->offset += Array_cap; + a->width = 4; + if(a->offset >= unmappedzero && a->offset-Array_nel < unmappedzero) + checkoffset(a, canemitcode); + break; + +// case OADD: +// if(n->right->op == OLITERAL) { +// v = n->right->vconst; +// naddr(n->left, a, canemitcode); +// } else +// if(n->left->op == OLITERAL) { +// v = n->left->vconst; +// naddr(n->right, a, canemitcode); +// } else +// goto bad; +// a->offset += v; +// break; + + } +} + +int +dotaddable(Node *n, Node *n1) +{ + int o, oary[10]; + Node *nn; + + if(n->op != ODOT) + return 0; + + o = dotoffset(n, oary, &nn); + if(nn != N && nn->addable && o == 1 && oary[0] >= 0) { + *n1 = *nn; + n1->type = n->type; + n1->xoffset += oary[0]; + return 1; + } + return 0; +} + +void +sudoclean(void) +{ +} + +int +sudoaddable(int as, Node *n, Addr *a) +{ + return 0; +} diff --git a/src/cmd/8g/list.c b/src/cmd/8g/list.c new file mode 100644 index 000000000..edb1ece84 --- /dev/null +++ b/src/cmd/8g/list.c @@ -0,0 +1,302 @@ +// Derived from Inferno utils/8c/list.c +// http://code.google.com/p/inferno-os/source/browse/utils/8c/list.c +// +// Copyright © 1994-1999 Lucent Technologies Inc. All rights reserved. +// Portions Copyright © 1995-1997 C H Forsyth (forsyth@terzarima.net) +// Portions Copyright © 1997-1999 Vita Nuova Limited +// Portions Copyright © 2000-2007 Vita Nuova Holdings Limited (www.vitanuova.com) +// Portions Copyright © 2004,2006 Bruce Ellis +// Portions Copyright © 2005-2007 C H Forsyth (forsyth@terzarima.net) +// Revisions Copyright © 2000-2007 Lucent Technologies Inc. and others +// Portions Copyright © 2009 The Go Authors. All rights reserved. +// +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to deal +// in the Software without restriction, including without limitation the rights +// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +// copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +// THE SOFTWARE. + +#include "gg.h" + +static int sconsize; +void +listinit(void) +{ + + fmtinstall('A', Aconv); // as + fmtinstall('P', Pconv); // Prog* + fmtinstall('D', Dconv); // Addr* + fmtinstall('R', Rconv); // reg + fmtinstall('Y', Yconv); // sconst +} + +int +Pconv(Fmt *fp) +{ + char str[STRINGSZ]; + Prog *p; + char scale[40]; + + p = va_arg(fp->args, Prog*); + sconsize = 8; + scale[0] = '\0'; + if(p->from.scale != 0 && (p->as == AGLOBL || p->as == ATEXT)) + snprint(scale, sizeof scale, "%d,", p->from.scale); + switch(p->as) { + default: + snprint(str, sizeof(str), "%.4d (%L) %-7A %D,%s%D", + p->loc, p->lineno, p->as, &p->from, scale, &p->to); + break; + + case ADATA: + sconsize = p->from.scale; + snprint(str, sizeof(str), "%.4d (%L) %-7A %D/%d,%D", + p->loc, p->lineno, p->as, &p->from, sconsize, &p->to); + break; + + case ATEXT: + snprint(str, sizeof(str), "%.4d (%L) %-7A %D,%s%lD", + p->loc, p->lineno, p->as, &p->from, scale, &p->to); + break; + } + return fmtstrcpy(fp, str); +} + +int +Dconv(Fmt *fp) +{ + char str[STRINGSZ], s[STRINGSZ]; + Addr *a; + int i; + uint32 d1, d2; + + a = va_arg(fp->args, Addr*); + i = a->type; + if(i >= D_INDIR) { + if(a->offset) + snprint(str, sizeof(str), "%d(%R)", a->offset, i-D_INDIR); + else + snprint(str, sizeof(str), "(%R)", i-D_INDIR); + goto brk; + } + switch(i) { + + default: + if(a->offset) + snprint(str, sizeof(str), "$%d,%R", a->offset, i); + else + snprint(str, sizeof(str), "%R", i); + break; + + case D_NONE: + str[0] = 0; + break; + + case D_BRANCH: + snprint(str, sizeof(str), "%d", a->branch->loc); + break; + + case D_EXTERN: + snprint(str, sizeof(str), "%S+%d(SB)", a->sym, a->offset); + break; + + case D_STATIC: + snprint(str, sizeof(str), "%S<>+%d(SB)", a->sym, a->offset); + break; + + case D_AUTO: + snprint(str, sizeof(str), "%S+%d(SP)", a->sym, a->offset); + break; + + case D_PARAM: + snprint(str, sizeof(str), "%S+%d(FP)", a->sym, a->offset); + break; + + case D_CONST: + if(fp->flags & FmtLong) { + d1 = a->offset; + d2 = a->offset2; + snprint(str, sizeof(str), "$%ud-%ud", (ulong)d1, (ulong)d2); + break; + } + snprint(str, sizeof(str), "$%d", a->offset); + break; + + case D_FCONST: + snprint(str, sizeof(str), "$(%.17e)", a->dval); + break; + + case D_SCONST: + snprint(str, sizeof(str), "$\"%Y\"", a->sval); + break; + + case D_ADDR: + a->type = a->index; + a->index = D_NONE; + snprint(str, sizeof(str), "$%D", a); + a->index = a->type; + a->type = D_ADDR; + goto conv; + } +brk: + if(a->index != D_NONE) { + snprint(s, sizeof(s), "(%R*%d)", (int)a->index, (int)a->scale); + strcat(str, s); + } +conv: + return fmtstrcpy(fp, str); +} + +static char* regstr[] = +{ + "AL", /* [D_AL] */ + "CL", + "DL", + "BL", + + "AH", /* [D_AH] */ + "CH", + "DH", + "BH", + + "AX", /* [D_AX] */ + "CX", + "DX", + "BX", + "SP", + "BP", + "SI", + "DI", + + "F0", /* [D_F0] */ + "F1", + "F2", + "F3", + "F4", + "F5", + "F6", + "F7", + + "CS", /* [D_CS] */ + "SS", + "DS", + "ES", + "FS", + "GS", + + "GDTR", /* [D_GDTR] */ + "IDTR", /* [D_IDTR] */ + "LDTR", /* [D_LDTR] */ + "MSW", /* [D_MSW] */ + "TASK", /* [D_TASK] */ + + "CR0", /* [D_CR] */ + "CR1", + "CR2", + "CR3", + "CR4", + "CR5", + "CR6", + "CR7", + + "DR0", /* [D_DR] */ + "DR1", + "DR2", + "DR3", + "DR4", + "DR5", + "DR6", + "DR7", + + "TR0", /* [D_TR] */ + "TR1", + "TR2", + "TR3", + "TR4", + "TR5", + "TR6", + "TR7", + + "NONE", /* [D_NONE] */ +}; + +int +Rconv(Fmt *fp) +{ + char str[STRINGSZ]; + int r; + + r = va_arg(fp->args, int); + if(r < 0 || r >= nelem(regstr) || regstr[r] == nil) { + snprint(str, sizeof(str), "BAD_R(%d)", r); + return fmtstrcpy(fp, str); + } + return fmtstrcpy(fp, regstr[r]); +} + +int +Aconv(Fmt *fp) +{ + int i; + + i = va_arg(fp->args, int); + return fmtstrcpy(fp, anames[i]); +} + + +int +Yconv(Fmt *fp) +{ + int i, c; + char str[STRINGSZ], *p, *a; + + a = va_arg(fp->args, char*); + p = str; + for(i=0; i= 'a' && c <= 'z') || + (c >= 'A' && c <= 'Z') || + (c >= '0' && c <= '9')) { + *p++ = c; + continue; + } + *p++ = '\\'; + switch(c) { + default: + if(c < 040 || c >= 0177) + break; /* not portable */ + p[-1] = c; + continue; + case 0: + *p++ = 'z'; + continue; + case '\\': + case '"': + *p++ = c; + continue; + case '\n': + *p++ = 'n'; + continue; + case '\t': + *p++ = 't'; + continue; + } + *p++ = (c>>6) + '0'; + *p++ = ((c>>3) & 7) + '0'; + *p++ = (c & 7) + '0'; + } + *p = 0; + return fmtstrcpy(fp, str); +} diff --git a/src/cmd/8g/opt.h b/src/cmd/8g/opt.h new file mode 100644 index 000000000..8f31dec3b --- /dev/null +++ b/src/cmd/8g/opt.h @@ -0,0 +1,164 @@ +// Derived from Inferno utils/6c/gc.h +// http://code.google.com/p/inferno-os/source/browse/utils/6c/gc.h +// +// Copyright © 1994-1999 Lucent Technologies Inc. All rights reserved. +// Portions Copyright © 1995-1997 C H Forsyth (forsyth@terzarima.net) +// Portions Copyright © 1997-1999 Vita Nuova Limited +// Portions Copyright © 2000-2007 Vita Nuova Holdings Limited (www.vitanuova.com) +// Portions Copyright © 2004,2006 Bruce Ellis +// Portions Copyright © 2005-2007 C H Forsyth (forsyth@terzarima.net) +// Revisions Copyright © 2000-2007 Lucent Technologies Inc. and others +// Portions Copyright © 2009 The Go Authors. All rights reserved. +// +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to deal +// in the Software without restriction, including without limitation the rights +// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +// copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +// THE SOFTWARE. + +#define Z N +#define Adr Addr + +#define D_HI D_NONE +#define D_LO D_NONE + +#define BLOAD(r) band(bnot(r->refbehind), r->refahead) +#define BSTORE(r) band(bnot(r->calbehind), r->calahead) +#define LOAD(r) (~r->refbehind.b[z] & r->refahead.b[z]) +#define STORE(r) (~r->calbehind.b[z] & r->calahead.b[z]) + +#define CLOAD 5 +#define CREF 5 +#define CINF 1000 +#define LOOP 3 + +typedef struct Reg Reg; +typedef struct Rgn Rgn; + +struct Reg +{ + + Bits set; + Bits use1; + Bits use2; + + Bits refbehind; + Bits refahead; + Bits calbehind; + Bits calahead; + Bits regdiff; + Bits act; + + int32 regu; // register used bitmap + int32 rpo; // reverse post ordering + int32 active; + + uint16 loop; // x5 for every loop + uchar refset; // diagnostic generated + + Reg* p1; + Reg* p2; + Reg* p2link; + Reg* s1; + Reg* s2; + Reg* link; + Prog* prog; +}; +#define R ((Reg*)0) + +#define NRGN 600 +struct Rgn +{ + Reg* enter; + short cost; + short varno; + short regno; +}; + +EXTERN int32 exregoffset; // not set +EXTERN int32 exfregoffset; // not set +EXTERN Reg* firstr; +EXTERN Reg* lastr; +EXTERN Reg zreg; +EXTERN Reg* freer; +EXTERN Reg** rpo2r; +EXTERN Rgn region[NRGN]; +EXTERN Rgn* rgp; +EXTERN int nregion; +EXTERN int nvar; +EXTERN int32 regbits; +EXTERN int32 exregbits; +EXTERN Bits externs; +EXTERN Bits params; +EXTERN Bits consts; +EXTERN Bits addrs; +EXTERN Bits ovar; +EXTERN int change; +EXTERN int32 maxnr; +EXTERN int32* idom; + +EXTERN struct +{ + int32 ncvtreg; + int32 nspill; + int32 nreload; + int32 ndelmov; + int32 nvar; + int32 naddr; +} ostats; + +/* + * reg.c + */ +Reg* rega(void); +int rcmp(const void*, const void*); +void regopt(Prog*); +void addmove(Reg*, int, int, int); +Bits mkvar(Reg*, Adr*); +void prop(Reg*, Bits, Bits); +void loopit(Reg*, int32); +void synch(Reg*, Bits); +uint32 allreg(uint32, Rgn*); +void paint1(Reg*, int); +uint32 paint2(Reg*, int); +void paint3(Reg*, int, int32, int); +void addreg(Adr*, int); +void dumpone(Reg*); +void dumpit(char*, Reg*); +int noreturn(Prog *p); + +/* + * peep.c + */ +void peep(void); +void excise(Reg*); +Reg* uniqp(Reg*); +Reg* uniqs(Reg*); +int regtyp(Adr*); +int anyvar(Adr*); +int subprop(Reg*); +int copyprop(Reg*); +int copy1(Adr*, Adr*, Reg*, int); +int copyu(Prog*, Adr*, Adr*); + +int copyas(Adr*, Adr*); +int copyau(Adr*, Adr*); +int copysub(Adr*, Adr*, Adr*, int); +int copysub1(Prog*, Adr*, Adr*, int); + +int32 RtoB(int); +int32 FtoB(int); +int BtoR(int32); +int BtoF(int32); diff --git a/src/cmd/8g/peep.c b/src/cmd/8g/peep.c new file mode 100644 index 000000000..5ad29e1b2 --- /dev/null +++ b/src/cmd/8g/peep.c @@ -0,0 +1,890 @@ +// Derived from Inferno utils/6c/peep.c +// http://code.google.com/p/inferno-os/source/browse/utils/6c/peep.c +// +// Copyright © 1994-1999 Lucent Technologies Inc. All rights reserved. +// Portions Copyright © 1995-1997 C H Forsyth (forsyth@terzarima.net) +// Portions Copyright © 1997-1999 Vita Nuova Limited +// Portions Copyright © 2000-2007 Vita Nuova Holdings Limited (www.vitanuova.com) +// Portions Copyright © 2004,2006 Bruce Ellis +// Portions Copyright © 2005-2007 C H Forsyth (forsyth@terzarima.net) +// Revisions Copyright © 2000-2007 Lucent Technologies Inc. and others +// Portions Copyright © 2009 The Go Authors. All rights reserved. +// +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to deal +// in the Software without restriction, including without limitation the rights +// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +// copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +// THE SOFTWARE. + +#include "gg.h" +#include "opt.h" + +#define REGEXT 0 + +static void conprop(Reg *r); + +// do we need the carry bit +static int +needc(Prog *p) +{ + while(p != P) { + switch(p->as) { + case AADCL: + case ASBBL: + case ARCRL: + return 1; + case AADDL: + case ASUBL: + case AJMP: + case ARET: + case ACALL: + return 0; + default: + if(p->to.type == D_BRANCH) + return 0; + } + p = p->link; + } + return 0; +} + +static Reg* +rnops(Reg *r) +{ + Prog *p; + Reg *r1; + + if(r != R) + for(;;) { + p = r->prog; + if(p->as != ANOP || p->from.type != D_NONE || p->to.type != D_NONE) + break; + r1 = uniqs(r); + if(r1 == R) + break; + r = r1; + } + return r; +} + +void +peep(void) +{ + Reg *r, *r1, *r2; + Prog *p, *p1; + int t; + + /* + * complete R structure + */ + t = 0; + for(r=firstr; r!=R; r=r1) { + r1 = r->link; + if(r1 == R) + break; + p = r->prog->link; + while(p != r1->prog) + switch(p->as) { + default: + r2 = rega(); + r->link = r2; + r2->link = r1; + + r2->prog = p; + p->reg = r2; + + r2->p1 = r; + r->s1 = r2; + r2->s1 = r1; + r1->p1 = r2; + + r = r2; + t++; + + case ADATA: + case AGLOBL: + case ANAME: + case ASIGNAME: + p = p->link; + } + } + + // movb elimination. + // movb is simulated by the linker + // when a register other than ax, bx, cx, dx + // is used, so rewrite to other instructions + // when possible. a movb into a register + // can smash the entire 32-bit register without + // causing any trouble. + for(r=firstr; r!=R; r=r->link) { + p = r->prog; + if(p->as == AMOVB && regtyp(&p->to)) { + // movb into register. + // from another register or constant can be movl. + if(regtyp(&p->from) || p->from.type == D_CONST) + p->as = AMOVL; + else + p->as = AMOVBLZX; + } + } + + // constant propagation + // find MOV $con,R followed by + // another MOV $con,R without + // setting R in the interim + for(r=firstr; r!=R; r=r->link) { + p = r->prog; + switch(p->as) { + case ALEAL: + if(regtyp(&p->to)) + if(p->from.sym != S) + conprop(r); + break; + + case AMOVB: + case AMOVW: + case AMOVL: + if(regtyp(&p->to)) + if(p->from.type == D_CONST) + conprop(r); + break; + } + } + +loop1: + if(debug['P'] && debug['v']) + dumpit("loop1", firstr); + + t = 0; + for(r=firstr; r!=R; r=r->link) { + p = r->prog; + switch(p->as) { + case AMOVB: + case AMOVW: + case AMOVL: + if(regtyp(&p->to)) + if(regtyp(&p->from)) { + if(copyprop(r)) { + excise(r); + t++; + } else + if(subprop(r) && copyprop(r)) { + excise(r); + t++; + } + } + break; + + case AMOVBLZX: + case AMOVWLZX: + case AMOVBLSX: + case AMOVWLSX: + if(regtyp(&p->to)) { + r1 = rnops(uniqs(r)); + if(r1 != R) { + p1 = r1->prog; + if(p->as == p1->as && p->to.type == p1->from.type){ + p1->as = AMOVL; + t++; + } + } + } + break; + + case AADDB: + case AADDL: + case AADDW: + if(p->from.type != D_CONST || needc(p->link)) + break; + if(p->from.offset == -1){ + if(p->as == AADDL) + p->as = ADECL; + else + p->as = ADECW; + p->from = zprog.from; + break; + } + if(p->from.offset == 1){ + if(p->as == AADDL) + p->as = AINCL; + else + p->as = AINCW; + p->from = zprog.from; + break; + } + break; + + case ASUBB: + case ASUBL: + case ASUBW: + if(p->from.type != D_CONST || needc(p->link)) + break; + if(p->from.offset == -1) { + if(p->as == ASUBL) + p->as = AINCL; + else + p->as = AINCW; + p->from = zprog.from; + break; + } + if(p->from.offset == 1){ + if(p->as == ASUBL) + p->as = ADECL; + else + p->as = ADECW; + p->from = zprog.from; + break; + } + break; + } + } + if(t) + goto loop1; +} + +void +excise(Reg *r) +{ + Prog *p; + + p = r->prog; + if(debug['P'] && debug['v']) + print("%P ===delete===\n", p); + + p->as = ANOP; + p->from = zprog.from; + p->to = zprog.to; + + ostats.ndelmov++; +} + +Reg* +uniqp(Reg *r) +{ + Reg *r1; + + r1 = r->p1; + if(r1 == R) { + r1 = r->p2; + if(r1 == R || r1->p2link != R) + return R; + } else + if(r->p2 != R) + return R; + return r1; +} + +Reg* +uniqs(Reg *r) +{ + Reg *r1; + + r1 = r->s1; + if(r1 == R) { + r1 = r->s2; + if(r1 == R) + return R; + } else + if(r->s2 != R) + return R; + return r1; +} + +int +regtyp(Adr *a) +{ + int t; + + t = a->type; + if(t >= D_AX && t <= D_DI) + return 1; + return 0; +} + +/* + * the idea is to substitute + * one register for another + * from one MOV to another + * MOV a, R0 + * ADD b, R0 / no use of R1 + * MOV R0, R1 + * would be converted to + * MOV a, R1 + * ADD b, R1 + * MOV R1, R0 + * hopefully, then the former or latter MOV + * will be eliminated by copy propagation. + */ +int +subprop(Reg *r0) +{ + Prog *p; + Adr *v1, *v2; + Reg *r; + int t; + + p = r0->prog; + v1 = &p->from; + if(!regtyp(v1)) + return 0; + v2 = &p->to; + if(!regtyp(v2)) + return 0; + for(r=uniqp(r0); r!=R; r=uniqp(r)) { + if(uniqs(r) == R) + break; + p = r->prog; + switch(p->as) { + case ACALL: + return 0; + + case AIMULL: + case AIMULW: + if(p->to.type != D_NONE) + break; + + case ADIVB: + case ADIVL: + case ADIVW: + case AIDIVB: + case AIDIVL: + case AIDIVW: + case AIMULB: + case AMULB: + case AMULL: + case AMULW: + + case ARCLB: + case ARCLL: + case ARCLW: + case ARCRB: + case ARCRL: + case ARCRW: + case AROLB: + case AROLL: + case AROLW: + case ARORB: + case ARORL: + case ARORW: + case ASALB: + case ASALL: + case ASALW: + case ASARB: + case ASARL: + case ASARW: + case ASHLB: + case ASHLL: + case ASHLW: + case ASHRB: + case ASHRL: + case ASHRW: + + case AREP: + case AREPN: + + case ACWD: + case ACDQ: + + case ASTOSB: + case ASTOSL: + case AMOVSB: + case AMOVSL: + return 0; + + case AMOVB: + case AMOVW: + case AMOVL: + if(p->to.type == v1->type) + goto gotit; + break; + } + if(copyau(&p->from, v2) || + copyau(&p->to, v2)) + break; + if(copysub(&p->from, v1, v2, 0) || + copysub(&p->to, v1, v2, 0)) + break; + } + return 0; + +gotit: + copysub(&p->to, v1, v2, 1); + if(debug['P']) { + print("gotit: %D->%D\n%P", v1, v2, r->prog); + if(p->from.type == v2->type) + print(" excise"); + print("\n"); + } + for(r=uniqs(r); r!=r0; r=uniqs(r)) { + p = r->prog; + copysub(&p->from, v1, v2, 1); + copysub(&p->to, v1, v2, 1); + if(debug['P']) + print("%P\n", r->prog); + } + t = v1->type; + v1->type = v2->type; + v2->type = t; + if(debug['P']) + print("%P last\n", r->prog); + return 1; +} + +/* + * The idea is to remove redundant copies. + * v1->v2 F=0 + * (use v2 s/v2/v1/)* + * set v1 F=1 + * use v2 return fail + * ----------------- + * v1->v2 F=0 + * (use v2 s/v2/v1/)* + * set v1 F=1 + * set v2 return success + */ +int +copyprop(Reg *r0) +{ + Prog *p; + Adr *v1, *v2; + Reg *r; + + p = r0->prog; + v1 = &p->from; + v2 = &p->to; + if(copyas(v1, v2)) + return 1; + for(r=firstr; r!=R; r=r->link) + r->active = 0; + return copy1(v1, v2, r0->s1, 0); +} + +int +copy1(Adr *v1, Adr *v2, Reg *r, int f) +{ + int t; + Prog *p; + + if(r->active) { + if(debug['P']) + print("act set; return 1\n"); + return 1; + } + r->active = 1; + if(debug['P']) + print("copy %D->%D f=%d\n", v1, v2, f); + for(; r != R; r = r->s1) { + p = r->prog; + if(debug['P']) + print("%P", p); + if(!f && uniqp(r) == R) { + f = 1; + if(debug['P']) + print("; merge; f=%d", f); + } + t = copyu(p, v2, A); + switch(t) { + case 2: /* rar, cant split */ + if(debug['P']) + print("; %D rar; return 0\n", v2); + return 0; + + case 3: /* set */ + if(debug['P']) + print("; %D set; return 1\n", v2); + return 1; + + case 1: /* used, substitute */ + case 4: /* use and set */ + if(f) { + if(!debug['P']) + return 0; + if(t == 4) + print("; %D used+set and f=%d; return 0\n", v2, f); + else + print("; %D used and f=%d; return 0\n", v2, f); + return 0; + } + if(copyu(p, v2, v1)) { + if(debug['P']) + print("; sub fail; return 0\n"); + return 0; + } + if(debug['P']) + print("; sub %D/%D", v2, v1); + if(t == 4) { + if(debug['P']) + print("; %D used+set; return 1\n", v2); + return 1; + } + break; + } + if(!f) { + t = copyu(p, v1, A); + if(!f && (t == 2 || t == 3 || t == 4)) { + f = 1; + if(debug['P']) + print("; %D set and !f; f=%d", v1, f); + } + } + if(debug['P']) + print("\n"); + if(r->s2) + if(!copy1(v1, v2, r->s2, f)) + return 0; + } + return 1; +} + +/* + * return + * 1 if v only used (and substitute), + * 2 if read-alter-rewrite + * 3 if set + * 4 if set and used + * 0 otherwise (not touched) + */ +int +copyu(Prog *p, Adr *v, Adr *s) +{ + + switch(p->as) { + + default: + if(debug['P']) + print("unknown op %A\n", p->as); + /* SBBL; ADCL; FLD1; SAHF */ + return 2; + + + case ANEGB: + case ANEGW: + case ANEGL: + case ANOTB: + case ANOTW: + case ANOTL: + if(copyas(&p->to, v)) + return 2; + break; + + case ALEAL: /* lhs addr, rhs store */ + if(copyas(&p->from, v)) + return 2; + + + case ANOP: /* rhs store */ + case AMOVB: + case AMOVW: + case AMOVL: + case AMOVBLSX: + case AMOVBLZX: + case AMOVWLSX: + case AMOVWLZX: + if(copyas(&p->to, v)) { + if(s != A) + return copysub(&p->from, v, s, 1); + if(copyau(&p->from, v)) + return 4; + return 3; + } + goto caseread; + + case ARCLB: + case ARCLL: + case ARCLW: + case ARCRB: + case ARCRL: + case ARCRW: + case AROLB: + case AROLL: + case AROLW: + case ARORB: + case ARORL: + case ARORW: + case ASALB: + case ASALL: + case ASALW: + case ASARB: + case ASARL: + case ASARW: + case ASHLB: + case ASHLL: + case ASHLW: + case ASHRB: + case ASHRL: + case ASHRW: + if(copyas(&p->to, v)) + return 2; + if(copyas(&p->from, v)) + if(p->from.type == D_CX) + return 2; + goto caseread; + + case AADDB: /* rhs rar */ + case AADDL: + case AADDW: + case AANDB: + case AANDL: + case AANDW: + case ADECL: + case ADECW: + case AINCL: + case AINCW: + case ASUBB: + case ASUBL: + case ASUBW: + case AORB: + case AORL: + case AORW: + case AXORB: + case AXORL: + case AXORW: + if(copyas(&p->to, v)) + return 2; + goto caseread; + + case ACMPL: /* read only */ + case ACMPW: + case ACMPB: + caseread: + if(s != A) { + if(copysub(&p->from, v, s, 1)) + return 1; + return copysub(&p->to, v, s, 1); + } + if(copyau(&p->from, v)) + return 1; + if(copyau(&p->to, v)) + return 1; + break; + + case AJGE: /* no reference */ + case AJNE: + case AJLE: + case AJEQ: + case AJHI: + case AJLS: + case AJMI: + case AJPL: + case AJGT: + case AJLT: + case AJCC: + case AJCS: + + case AADJSP: + case AWAIT: + case ACLD: + break; + + case AIMULL: + case AIMULW: + if(p->to.type != D_NONE) { + if(copyas(&p->to, v)) + return 2; + goto caseread; + } + + case ADIVB: + case ADIVL: + case ADIVW: + case AIDIVB: + case AIDIVL: + case AIDIVW: + case AIMULB: + case AMULB: + case AMULL: + case AMULW: + + case ACWD: + case ACDQ: + if(v->type == D_AX || v->type == D_DX) + return 2; + goto caseread; + + case AREP: + case AREPN: + if(v->type == D_CX) + return 2; + goto caseread; + + case AMOVSB: + case AMOVSL: + if(v->type == D_DI || v->type == D_SI) + return 2; + goto caseread; + + case ASTOSB: + case ASTOSL: + if(v->type == D_AX || v->type == D_DI) + return 2; + goto caseread; + + case AJMP: /* funny */ + if(s != A) { + if(copysub(&p->to, v, s, 1)) + return 1; + return 0; + } + if(copyau(&p->to, v)) + return 1; + return 0; + + case ARET: /* funny */ + if(v->type == REGRET || v->type == FREGRET) + return 2; + if(s != A) + return 1; + return 3; + + case ACALL: /* funny */ + if(REGEXT && v->type <= REGEXT && v->type > exregoffset) + return 2; + if(REGARG >= 0 && v->type == (uchar)REGARG) + return 2; + + if(s != A) { + if(copysub(&p->to, v, s, 1)) + return 1; + return 0; + } + if(copyau(&p->to, v)) + return 4; + return 3; + + case ATEXT: /* funny */ + if(REGARG >= 0 && v->type == (uchar)REGARG) + return 3; + return 0; + } + return 0; +} + +/* + * direct reference, + * could be set/use depending on + * semantics + */ +int +copyas(Adr *a, Adr *v) +{ + if(a->type != v->type) + return 0; + if(regtyp(v)) + return 1; + if(v->type == D_AUTO || v->type == D_PARAM) + if(v->offset == a->offset) + return 1; + return 0; +} + +/* + * either direct or indirect + */ +int +copyau(Adr *a, Adr *v) +{ + + if(copyas(a, v)) + return 1; + if(regtyp(v)) { + if(a->type-D_INDIR == v->type) + return 1; + if(a->index == v->type) + return 1; + } + return 0; +} + +/* + * substitute s for v in a + * return failure to substitute + */ +int +copysub(Adr *a, Adr *v, Adr *s, int f) +{ + int t; + + if(copyas(a, v)) { + t = s->type; + if(t >= D_AX && t <= D_DI) { + if(f) + a->type = t; + } + return 0; + } + if(regtyp(v)) { + t = v->type; + if(a->type == t+D_INDIR) { + if((s->type == D_BP) && a->index != D_NONE) + return 1; /* can't use BP-base with index */ + if(f) + a->type = s->type+D_INDIR; +// return 0; + } + if(a->index == t) { + if(f) + a->index = s->type; + return 0; + } + return 0; + } + return 0; +} + +static void +conprop(Reg *r0) +{ + Reg *r; + Prog *p, *p0; + int t; + Adr *v0; + + p0 = r0->prog; + v0 = &p0->to; + r = r0; + +loop: + r = uniqs(r); + if(r == R || r == r0) + return; + if(uniqp(r) == R) + return; + + p = r->prog; + t = copyu(p, v0, A); + switch(t) { + case 0: // miss + case 1: // use + goto loop; + + case 2: // rar + case 4: // use and set + break; + + case 3: // set + if(p->as == p0->as) + if(p->from.type == p0->from.type) + if(p->from.sym == p0->from.sym) + if(p->from.offset == p0->from.offset) + if(p->from.scale == p0->from.scale) + if(p->from.dval == p0->from.dval) + if(p->from.index == p0->from.index) { + excise(r); + t++; + goto loop; + } + break; + } +} diff --git a/src/cmd/8g/reg.c b/src/cmd/8g/reg.c new file mode 100644 index 000000000..2b878f62a --- /dev/null +++ b/src/cmd/8g/reg.c @@ -0,0 +1,1544 @@ +// Derived from Inferno utils/6c/reg.c +// http://code.google.com/p/inferno-os/source/browse/utils/6c/reg.c +// +// Copyright © 1994-1999 Lucent Technologies Inc. All rights reserved. +// Portions Copyright © 1995-1997 C H Forsyth (forsyth@terzarima.net) +// Portions Copyright © 1997-1999 Vita Nuova Limited +// Portions Copyright © 2000-2007 Vita Nuova Holdings Limited (www.vitanuova.com) +// Portions Copyright © 2004,2006 Bruce Ellis +// Portions Copyright © 2005-2007 C H Forsyth (forsyth@terzarima.net) +// Revisions Copyright © 2000-2007 Lucent Technologies Inc. and others +// Portions Copyright © 2009 The Go Authors. All rights reserved. +// +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to deal +// in the Software without restriction, including without limitation the rights +// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +// copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +// THE SOFTWARE. + +#include "gg.h" +#undef EXTERN +#define EXTERN +#include "opt.h" + +#define NREGVAR 8 +#define REGBITS ((uint32)0xff) +#define P2R(p) (Reg*)(p->reg) + +static int first = 1; + +Reg* +rega(void) +{ + Reg *r; + + r = freer; + if(r == R) { + r = mal(sizeof(*r)); + } else + freer = r->link; + + *r = zreg; + return r; +} + +int +rcmp(const void *a1, const void *a2) +{ + Rgn *p1, *p2; + int c1, c2; + + p1 = (Rgn*)a1; + p2 = (Rgn*)a2; + c1 = p2->cost; + c2 = p1->cost; + if(c1 -= c2) + return c1; + return p2->varno - p1->varno; +} + +static void +setoutvar(void) +{ + Type *t; + Node *n; + Addr a; + Iter save; + Bits bit; + int z; + + t = structfirst(&save, getoutarg(curfn->type)); + while(t != T) { + n = nodarg(t, 1); + a = zprog.from; + naddr(n, &a, 0); + bit = mkvar(R, &a); + for(z=0; zsym == s && v->name == n) + v->addr = 2; + } + } +} + +static char* regname[] = { ".ax", ".cx", ".dx", ".bx", ".sp", ".bp", ".si", ".di" }; + +void +regopt(Prog *firstp) +{ + Reg *r, *r1; + Prog *p; + int i, z, nr; + uint32 vreg; + Bits bit; + + if(first) { + fmtinstall('Q', Qconv); + exregoffset = D_DI; // no externals + first = 0; + } + + // count instructions + nr = 0; + for(p=firstp; p!=P; p=p->link) + nr++; + // if too big dont bother + if(nr >= 10000) { +// print("********** %S is too big (%d)\n", curfn->nname->sym, nr); + return; + } + + r1 = R; + firstr = R; + lastr = R; + + /* + * control flow is more complicated in generated go code + * than in generated c code. define pseudo-variables for + * registers, so we have complete register usage information. + */ + nvar = NREGVAR; + memset(var, 0, NREGVAR*sizeof var[0]); + for(i=0; ilink) { + switch(p->as) { + case ADATA: + case AGLOBL: + case ANAME: + case ASIGNAME: + continue; + } + r = rega(); + nr++; + if(firstr == R) { + firstr = r; + lastr = r; + } else { + lastr->link = r; + r->p1 = lastr; + lastr->s1 = r; + lastr = r; + } + r->prog = p; + p->reg = r; + + r1 = r->p1; + if(r1 != R) { + switch(r1->prog->as) { + case ARET: + case AJMP: + case AIRETL: + r->p1 = R; + r1->s1 = R; + } + } + + bit = mkvar(r, &p->from); + if(bany(&bit)) + switch(p->as) { + /* + * funny + */ + case ALEAL: + setaddrs(bit); + break; + + /* + * left side read + */ + default: + for(z=0; zuse1.b[z] |= bit.b[z]; + break; + + /* + * left side read+write + */ + case AXCHGB: + case AXCHGW: + case AXCHGL: + for(z=0; zuse1.b[z] |= bit.b[z]; + r->set.b[z] |= bit.b[z]; + } + break; + } + + bit = mkvar(r, &p->to); + if(bany(&bit)) + switch(p->as) { + default: + yyerror("reg: unknown op: %A", p->as); + break; + + /* + * right side read + */ + case ACMPB: + case ACMPL: + case ACMPW: + case ATESTB: + case ATESTL: + case ATESTW: + for(z=0; zuse2.b[z] |= bit.b[z]; + break; + + /* + * right side write + */ + case AFSTSW: + case ALEAL: + case ANOP: + case AMOVL: + case AMOVB: + case AMOVW: + case AMOVBLSX: + case AMOVBLZX: + case AMOVBWSX: + case AMOVBWZX: + case AMOVWLSX: + case AMOVWLZX: + case APOPL: + for(z=0; zset.b[z] |= bit.b[z]; + break; + + /* + * right side read+write + */ + case AINCB: + case AINCL: + case AINCW: + case ADECB: + case ADECL: + case ADECW: + + case AADDB: + case AADDL: + case AADDW: + case AANDB: + case AANDL: + case AANDW: + case ASUBB: + case ASUBL: + case ASUBW: + case AORB: + case AORL: + case AORW: + case AXORB: + case AXORL: + case AXORW: + case ASALB: + case ASALL: + case ASALW: + case ASARB: + case ASARL: + case ASARW: + case ARCLB: + case ARCLL: + case ARCLW: + case ARCRB: + case ARCRL: + case ARCRW: + case AROLB: + case AROLL: + case AROLW: + case ARORB: + case ARORL: + case ARORW: + case ASHLB: + case ASHLL: + case ASHLW: + case ASHRB: + case ASHRL: + case ASHRW: + case AIMULL: + case AIMULW: + case ANEGB: + case ANEGL: + case ANEGW: + case ANOTB: + case ANOTL: + case ANOTW: + case AADCL: + case ASBBL: + + case ASETCC: + case ASETCS: + case ASETEQ: + case ASETGE: + case ASETGT: + case ASETHI: + case ASETLE: + case ASETLS: + case ASETLT: + case ASETMI: + case ASETNE: + case ASETOC: + case ASETOS: + case ASETPC: + case ASETPL: + case ASETPS: + + case AXCHGB: + case AXCHGW: + case AXCHGL: + for(z=0; zset.b[z] |= bit.b[z]; + r->use2.b[z] |= bit.b[z]; + } + break; + + /* + * funny + */ + case AFMOVDP: + case AFMOVFP: + case AFMOVLP: + case AFMOVVP: + case AFMOVWP: + case ACALL: + setaddrs(bit); + break; + } + + switch(p->as) { + case AIMULL: + case AIMULW: + if(p->to.type != D_NONE) + break; + + case AIDIVL: + case AIDIVW: + case ADIVL: + case ADIVW: + case AMULL: + case AMULW: + r->set.b[0] |= RtoB(D_AX) | RtoB(D_DX); + r->use1.b[0] |= RtoB(D_AX) | RtoB(D_DX); + break; + + case AIDIVB: + case AIMULB: + case ADIVB: + case AMULB: + r->set.b[0] |= RtoB(D_AX); + r->use1.b[0] |= RtoB(D_AX); + break; + + case ACWD: + r->set.b[0] |= RtoB(D_AX) | RtoB(D_DX); + r->use1.b[0] |= RtoB(D_AX); + break; + + case ACDQ: + r->set.b[0] |= RtoB(D_DX); + r->use1.b[0] |= RtoB(D_AX); + break; + + case AREP: + case AREPN: + case ALOOP: + case ALOOPEQ: + case ALOOPNE: + r->set.b[0] |= RtoB(D_CX); + r->use1.b[0] |= RtoB(D_CX); + break; + + case AMOVSB: + case AMOVSL: + case AMOVSW: + case ACMPSB: + case ACMPSL: + case ACMPSW: + r->set.b[0] |= RtoB(D_SI) | RtoB(D_DI); + r->use1.b[0] |= RtoB(D_SI) | RtoB(D_DI); + break; + + case ASTOSB: + case ASTOSL: + case ASTOSW: + case ASCASB: + case ASCASL: + case ASCASW: + r->set.b[0] |= RtoB(D_DI); + r->use1.b[0] |= RtoB(D_AX) | RtoB(D_DI); + break; + + case AINSB: + case AINSL: + case AINSW: + r->set.b[0] |= RtoB(D_DX) | RtoB(D_DI); + r->use1.b[0] |= RtoB(D_DI); + break; + + case AOUTSB: + case AOUTSL: + case AOUTSW: + r->set.b[0] |= RtoB(D_DI); + r->use1.b[0] |= RtoB(D_DX) | RtoB(D_DI); + break; + } + } + if(firstr == R) + return; + + for(i=0; iaddr) { + bit = blsh(i); + for(z=0; zaddr, v->etype, v->width, v->sym, v->offset); + } + + if(debug['R'] && debug['v']) + dumpit("pass1", firstr); + + /* + * pass 2 + * turn branch references to pointers + * build back pointers + */ + for(r=firstr; r!=R; r=r->link) { + p = r->prog; + if(p->to.type == D_BRANCH) { + if(p->to.branch == P) + fatal("pnil %P", p); + r1 = p->to.branch->reg; + if(r1 == R) + fatal("rnil %P", p); + if(r1 == r) { + //fatal("ref to self %P", p); + continue; + } + r->s2 = r1; + r->p2link = r1->p2; + r1->p2 = r; + } + } + + if(debug['R'] && debug['v']) + dumpit("pass2", firstr); + + /* + * pass 2.5 + * find looping structure + */ + for(r = firstr; r != R; r = r->link) + r->active = 0; + change = 0; + loopit(firstr, nr); + + if(debug['R'] && debug['v']) + dumpit("pass2.5", firstr); + + /* + * pass 3 + * iterate propagating usage + * back until flow graph is complete + */ +loop1: + change = 0; + for(r = firstr; r != R; r = r->link) + r->active = 0; + for(r = firstr; r != R; r = r->link) + if(r->prog->as == ARET) + prop(r, zbits, zbits); +loop11: + /* pick up unreachable code */ + i = 0; + for(r = firstr; r != R; r = r1) { + r1 = r->link; + if(r1 && r1->active && !r->active) { + prop(r, zbits, zbits); + i = 1; + } + } + if(i) + goto loop11; + if(change) + goto loop1; + + if(debug['R'] && debug['v']) + dumpit("pass3", firstr); + + /* + * pass 4 + * iterate propagating register/variable synchrony + * forward until graph is complete + */ +loop2: + change = 0; + for(r = firstr; r != R; r = r->link) + r->active = 0; + synch(firstr, zbits); + if(change) + goto loop2; + + if(debug['R'] && debug['v']) + dumpit("pass4", firstr); + + /* + * pass 4.5 + * move register pseudo-variables into regu. + */ + for(r = firstr; r != R; r = r->link) { + r->regu = (r->refbehind.b[0] | r->set.b[0]) & REGBITS; + + r->set.b[0] &= ~REGBITS; + r->use1.b[0] &= ~REGBITS; + r->use2.b[0] &= ~REGBITS; + r->refbehind.b[0] &= ~REGBITS; + r->refahead.b[0] &= ~REGBITS; + r->calbehind.b[0] &= ~REGBITS; + r->calahead.b[0] &= ~REGBITS; + r->regdiff.b[0] &= ~REGBITS; + r->act.b[0] &= ~REGBITS; + } + + /* + * pass 5 + * isolate regions + * calculate costs (paint1) + */ + r = firstr; + if(r) { + for(z=0; zrefahead.b[z] | r->calahead.b[z]) & + ~(externs.b[z] | params.b[z] | addrs.b[z] | consts.b[z]); + if(bany(&bit) && !r->refset) { + // should never happen - all variables are preset + if(debug['w']) + print("%L: used and not set: %Q\n", r->prog->lineno, bit); + r->refset = 1; + } + } + for(r = firstr; r != R; r = r->link) + r->act = zbits; + rgp = region; + nregion = 0; + for(r = firstr; r != R; r = r->link) { + for(z=0; zset.b[z] & + ~(r->refahead.b[z] | r->calahead.b[z] | addrs.b[z]); + if(bany(&bit) && !r->refset) { + if(debug['w']) + print("%L: set and not used: %Q\n", r->prog->lineno, bit); + r->refset = 1; + excise(r); + } + for(z=0; zact.b[z] | addrs.b[z]); + while(bany(&bit)) { + i = bnum(bit); + rgp->enter = r; + rgp->varno = i; + change = 0; + paint1(r, i); + bit.b[i/32] &= ~(1L<<(i%32)); + if(change <= 0) + continue; + rgp->cost = change; + nregion++; + if(nregion >= NRGN) { + if(debug['R'] && debug['v']) + print("too many regions\n"); + goto brk; + } + rgp++; + } + } +brk: + qsort(region, nregion, sizeof(region[0]), rcmp); + + /* + * pass 6 + * determine used registers (paint2) + * replace code (paint3) + */ + rgp = region; + for(i=0; ivarno); + vreg = paint2(rgp->enter, rgp->varno); + vreg = allreg(vreg, rgp); + if(rgp->regno != 0) + paint3(rgp->enter, rgp->varno, vreg, rgp->regno); + rgp++; + } + + if(debug['R'] && debug['v']) + dumpit("pass6", firstr); + + /* + * pass 7 + * peep-hole on basic block + */ + if(!debug['R'] || debug['P']) { + peep(); + } + + /* + * eliminate nops + * free aux structures + */ + for(p=firstp; p!=P; p=p->link) { + while(p->link != P && p->link->as == ANOP) + p->link = p->link->link; + if(p->to.type == D_BRANCH) + while(p->to.branch != P && p->to.branch->as == ANOP) + p->to.branch = p->to.branch->link; + } + + if(r1 != R) { + r1->link = freer; + freer = firstr; + } + + if(debug['R']) { + if(ostats.ncvtreg || + ostats.nspill || + ostats.nreload || + ostats.ndelmov || + ostats.nvar || + ostats.naddr || + 0) + print("\nstats\n"); + + if(ostats.ncvtreg) + print(" %4d cvtreg\n", ostats.ncvtreg); + if(ostats.nspill) + print(" %4d spill\n", ostats.nspill); + if(ostats.nreload) + print(" %4d reload\n", ostats.nreload); + if(ostats.ndelmov) + print(" %4d delmov\n", ostats.ndelmov); + if(ostats.nvar) + print(" %4d delmov\n", ostats.nvar); + if(ostats.naddr) + print(" %4d delmov\n", ostats.naddr); + + memset(&ostats, 0, sizeof(ostats)); + } +} + +/* + * add mov b,rn + * just after r + */ +void +addmove(Reg *r, int bn, int rn, int f) +{ + Prog *p, *p1; + Adr *a; + Var *v; + + p1 = mal(sizeof(*p1)); + clearp(p1); + p1->loc = 9999; + + p = r->prog; + p1->link = p->link; + p->link = p1; + p1->lineno = p->lineno; + + v = var + bn; + + a = &p1->to; + a->sym = v->sym; + a->offset = v->offset; + a->etype = v->etype; + a->type = v->name; + a->gotype = v->gotype; + a->node = v->node; + + // need to clean this up with wptr and + // some of the defaults + p1->as = AMOVL; + switch(v->etype) { + default: + fatal("unknown type\n"); + case TINT8: + case TUINT8: + case TBOOL: + p1->as = AMOVB; + break; + case TINT16: + case TUINT16: + p1->as = AMOVW; + break; + case TINT: + case TUINT: + case TINT32: + case TUINT32: + case TPTR32: + break; + } + + p1->from.type = rn; + if(!f) { + p1->from = *a; + *a = zprog.from; + a->type = rn; + if(v->etype == TUINT8) + p1->as = AMOVB; + if(v->etype == TUINT16) + p1->as = AMOVW; + } + if(debug['R'] && debug['v']) + print("%P ===add=== %P\n", p, p1); + ostats.nspill++; +} + +uint32 +doregbits(int r) +{ + uint32 b; + + b = 0; + if(r >= D_INDIR) + r -= D_INDIR; + if(r >= D_AX && r <= D_DI) + b |= RtoB(r); + else + if(r >= D_AL && r <= D_BL) + b |= RtoB(r-D_AL+D_AX); + else + if(r >= D_AH && r <= D_BH) + b |= RtoB(r-D_AH+D_AX); + return b; +} + +static int +overlap(int32 o1, int w1, int32 o2, int w2) +{ + int32 t1, t2; + + t1 = o1+w1; + t2 = o2+w2; + + if(!(t1 > o2 && t2 > o1)) + return 0; + + return 1; +} + +Bits +mkvar(Reg *r, Adr *a) +{ + Var *v; + int i, t, n, et, z, w, flag, regu; + int32 o; + Bits bit; + Sym *s; + + /* + * mark registers used + */ + t = a->type; + if(t == D_NONE) + goto none; + + if(r != R) + r->use1.b[0] |= doregbits(a->index); + + switch(t) { + default: + regu = doregbits(t); + if(regu == 0) + goto none; + bit = zbits; + bit.b[0] = regu; + return bit; + + case D_ADDR: + a->type = a->index; + bit = mkvar(r, a); + setaddrs(bit); + a->type = t; + ostats.naddr++; + goto none; + + case D_EXTERN: + case D_STATIC: + case D_PARAM: + case D_AUTO: + n = t; + break; + } + + s = a->sym; + if(s == S) + goto none; + if(s->name[0] == '.') + goto none; + et = a->etype; + o = a->offset; + w = a->width; + + flag = 0; + for(i=0; isym == s && v->name == n) { + if(v->offset == o) + if(v->etype == et) + if(v->width == w) + return blsh(i); + + // if they overlap, disable both + if(overlap(v->offset, v->width, o, w)) { + if(debug['R']) + print("disable %s\n", v->sym->name); + v->addr = 1; + flag = 1; + } + } + } + + switch(et) { + case 0: + case TFUNC: + goto none; + } + + if(nvar >= NVAR) { + if(debug['w'] > 1 && s) + fatal("variable not optimized: %D", a); + goto none; + } + + i = nvar; + nvar++; + v = var+i; + v->sym = s; + v->offset = o; + v->name = n; + v->gotype = a->gotype; + v->etype = et; + v->width = w; + v->addr = flag; // funny punning + v->node = a->node; + + if(debug['R']) + print("bit=%2d et=%2d w=%d+%d %S %D flag=%d\n", i, et, o, w, s, a, v->addr); + ostats.nvar++; + + bit = blsh(i); + if(n == D_EXTERN || n == D_STATIC) + for(z=0; zp1) { + for(z=0; zrefahead.b[z]; + if(ref.b[z] != r1->refahead.b[z]) { + r1->refahead.b[z] = ref.b[z]; + change++; + } + cal.b[z] |= r1->calahead.b[z]; + if(cal.b[z] != r1->calahead.b[z]) { + r1->calahead.b[z] = cal.b[z]; + change++; + } + } + switch(r1->prog->as) { + case ACALL: + if(noreturn(r1->prog)) + break; + for(z=0; zset.b[z]) | + r1->use1.b[z] | r1->use2.b[z]; + cal.b[z] &= ~(r1->set.b[z] | r1->use1.b[z] | r1->use2.b[z]); + r1->refbehind.b[z] = ref.b[z]; + r1->calbehind.b[z] = cal.b[z]; + } + if(r1->active) + break; + r1->active = 1; + } + for(; r != r1; r = r->p1) + for(r2 = r->p2; r2 != R; r2 = r2->p2link) + prop(r2, r->refbehind, r->calbehind); +} + +/* + * find looping structure + * + * 1) find reverse postordering + * 2) find approximate dominators, + * the actual dominators if the flow graph is reducible + * otherwise, dominators plus some other non-dominators. + * See Matthew S. Hecht and Jeffrey D. Ullman, + * "Analysis of a Simple Algorithm for Global Data Flow Problems", + * Conf. Record of ACM Symp. on Principles of Prog. Langs, Boston, Massachusetts, + * Oct. 1-3, 1973, pp. 207-217. + * 3) find all nodes with a predecessor dominated by the current node. + * such a node is a loop head. + * recursively, all preds with a greater rpo number are in the loop + */ +int32 +postorder(Reg *r, Reg **rpo2r, int32 n) +{ + Reg *r1; + + r->rpo = 1; + r1 = r->s1; + if(r1 && !r1->rpo) + n = postorder(r1, rpo2r, n); + r1 = r->s2; + if(r1 && !r1->rpo) + n = postorder(r1, rpo2r, n); + rpo2r[n] = r; + n++; + return n; +} + +int32 +rpolca(int32 *idom, int32 rpo1, int32 rpo2) +{ + int32 t; + + if(rpo1 == -1) + return rpo2; + while(rpo1 != rpo2){ + if(rpo1 > rpo2){ + t = rpo2; + rpo2 = rpo1; + rpo1 = t; + } + while(rpo1 < rpo2){ + t = idom[rpo2]; + if(t >= rpo2) + fatal("bad idom"); + rpo2 = t; + } + } + return rpo1; +} + +int +doms(int32 *idom, int32 r, int32 s) +{ + while(s > r) + s = idom[s]; + return s == r; +} + +int +loophead(int32 *idom, Reg *r) +{ + int32 src; + + src = r->rpo; + if(r->p1 != R && doms(idom, src, r->p1->rpo)) + return 1; + for(r = r->p2; r != R; r = r->p2link) + if(doms(idom, src, r->rpo)) + return 1; + return 0; +} + +void +loopmark(Reg **rpo2r, int32 head, Reg *r) +{ + if(r->rpo < head || r->active == head) + return; + r->active = head; + r->loop += LOOP; + if(r->p1 != R) + loopmark(rpo2r, head, r->p1); + for(r = r->p2; r != R; r = r->p2link) + loopmark(rpo2r, head, r); +} + +void +loopit(Reg *r, int32 nr) +{ + Reg *r1; + int32 i, d, me; + + if(nr > maxnr) { + rpo2r = mal(nr * sizeof(Reg*)); + idom = mal(nr * sizeof(int32)); + maxnr = nr; + } + + d = postorder(r, rpo2r, 0); + if(d > nr) + fatal("too many reg nodes %d %d", d, nr); + nr = d; + for(i = 0; i < nr / 2; i++) { + r1 = rpo2r[i]; + rpo2r[i] = rpo2r[nr - 1 - i]; + rpo2r[nr - 1 - i] = r1; + } + for(i = 0; i < nr; i++) + rpo2r[i]->rpo = i; + + idom[0] = 0; + for(i = 0; i < nr; i++) { + r1 = rpo2r[i]; + me = r1->rpo; + d = -1; + if(r1->p1 != R && r1->p1->rpo < me) + d = r1->p1->rpo; + for(r1 = r1->p2; r1 != nil; r1 = r1->p2link) + if(r1->rpo < me) + d = rpolca(idom, d, r1->rpo); + idom[i] = d; + } + + for(i = 0; i < nr; i++) { + r1 = rpo2r[i]; + r1->loop++; + if(r1->p2 != R && loophead(idom, r1)) + loopmark(rpo2r, i, r1); + } +} + +void +synch(Reg *r, Bits dif) +{ + Reg *r1; + int z; + + for(r1 = r; r1 != R; r1 = r1->s1) { + for(z=0; zrefbehind.b[z] & r1->refahead.b[z])) | + r1->set.b[z] | r1->regdiff.b[z]; + if(dif.b[z] != r1->regdiff.b[z]) { + r1->regdiff.b[z] = dif.b[z]; + change++; + } + } + if(r1->active) + break; + r1->active = 1; + for(z=0; zcalbehind.b[z] & r1->calahead.b[z]); + if(r1->s2 != R) + synch(r1->s2, dif); + } +} + +uint32 +allreg(uint32 b, Rgn *r) +{ + Var *v; + int i; + + v = var + r->varno; + r->regno = 0; + switch(v->etype) { + + default: + fatal("unknown etype %d/%E", bitno(b), v->etype); + break; + + case TINT8: + case TUINT8: + case TINT16: + case TUINT16: + case TINT32: + case TUINT32: + case TINT64: + case TINT: + case TUINT: + case TUINTPTR: + case TBOOL: + case TPTR32: + i = BtoR(~b); + if(i && r->cost > 0) { + r->regno = i; + return RtoB(i); + } + break; + + case TFLOAT32: + case TFLOAT64: + break; + } + return 0; +} + +void +paint1(Reg *r, int bn) +{ + Reg *r1; + Prog *p; + int z; + uint32 bb; + + z = bn/32; + bb = 1L<<(bn%32); + if(r->act.b[z] & bb) + return; + for(;;) { + if(!(r->refbehind.b[z] & bb)) + break; + r1 = r->p1; + if(r1 == R) + break; + if(!(r1->refahead.b[z] & bb)) + break; + if(r1->act.b[z] & bb) + break; + r = r1; + } + + if(LOAD(r) & ~(r->set.b[z]&~(r->use1.b[z]|r->use2.b[z])) & bb) { + change -= CLOAD * r->loop; + } + for(;;) { + r->act.b[z] |= bb; + p = r->prog; + + if(r->use1.b[z] & bb) { + change += CREF * r->loop; + if(p->as == AFMOVL || p->as == AFMOVW) + if(BtoR(bb) != D_F0) + change = -CINF; + } + + if((r->use2.b[z]|r->set.b[z]) & bb) { + change += CREF * r->loop; + if(p->as == AFMOVL || p->as == AFMOVW) + if(BtoR(bb) != D_F0) + change = -CINF; + } + + if(STORE(r) & r->regdiff.b[z] & bb) { + change -= CLOAD * r->loop; + if(p->as == AFMOVL || p->as == AFMOVW) + if(BtoR(bb) != D_F0) + change = -CINF; + } + + if(r->refbehind.b[z] & bb) + for(r1 = r->p2; r1 != R; r1 = r1->p2link) + if(r1->refahead.b[z] & bb) + paint1(r1, bn); + + if(!(r->refahead.b[z] & bb)) + break; + r1 = r->s2; + if(r1 != R) + if(r1->refbehind.b[z] & bb) + paint1(r1, bn); + r = r->s1; + if(r == R) + break; + if(r->act.b[z] & bb) + break; + if(!(r->refbehind.b[z] & bb)) + break; + } +} + +uint32 +regset(Reg *r, uint32 bb) +{ + uint32 b, set; + Adr v; + int c; + + set = 0; + v = zprog.from; + while(b = bb & ~(bb-1)) { + v.type = BtoR(b); + c = copyu(r->prog, &v, A); + if(c == 3) + set |= b; + bb &= ~b; + } + return set; +} + +uint32 +reguse(Reg *r, uint32 bb) +{ + uint32 b, set; + Adr v; + int c; + + set = 0; + v = zprog.from; + while(b = bb & ~(bb-1)) { + v.type = BtoR(b); + c = copyu(r->prog, &v, A); + if(c == 1 || c == 2 || c == 4) + set |= b; + bb &= ~b; + } + return set; +} + +uint32 +paint2(Reg *r, int bn) +{ + Reg *r1; + int z; + uint32 bb, vreg, x; + + z = bn/32; + bb = 1L << (bn%32); + vreg = regbits; + if(!(r->act.b[z] & bb)) + return vreg; + for(;;) { + if(!(r->refbehind.b[z] & bb)) + break; + r1 = r->p1; + if(r1 == R) + break; + if(!(r1->refahead.b[z] & bb)) + break; + if(!(r1->act.b[z] & bb)) + break; + r = r1; + } + for(;;) { + r->act.b[z] &= ~bb; + + vreg |= r->regu; + + if(r->refbehind.b[z] & bb) + for(r1 = r->p2; r1 != R; r1 = r1->p2link) + if(r1->refahead.b[z] & bb) + vreg |= paint2(r1, bn); + + if(!(r->refahead.b[z] & bb)) + break; + r1 = r->s2; + if(r1 != R) + if(r1->refbehind.b[z] & bb) + vreg |= paint2(r1, bn); + r = r->s1; + if(r == R) + break; + if(!(r->act.b[z] & bb)) + break; + if(!(r->refbehind.b[z] & bb)) + break; + } + + bb = vreg; + for(; r; r=r->s1) { + x = r->regu & ~bb; + if(x) { + vreg |= reguse(r, x); + bb |= regset(r, x); + } + } + return vreg; +} + +void +paint3(Reg *r, int bn, int32 rb, int rn) +{ + Reg *r1; + Prog *p; + int z; + uint32 bb; + + z = bn/32; + bb = 1L << (bn%32); + if(r->act.b[z] & bb) + return; + for(;;) { + if(!(r->refbehind.b[z] & bb)) + break; + r1 = r->p1; + if(r1 == R) + break; + if(!(r1->refahead.b[z] & bb)) + break; + if(r1->act.b[z] & bb) + break; + r = r1; + } + + if(LOAD(r) & ~(r->set.b[z] & ~(r->use1.b[z]|r->use2.b[z])) & bb) + addmove(r, bn, rn, 0); + for(;;) { + r->act.b[z] |= bb; + p = r->prog; + + if(r->use1.b[z] & bb) { + if(debug['R'] && debug['v']) + print("%P", p); + addreg(&p->from, rn); + if(debug['R'] && debug['v']) + print(" ===change== %P\n", p); + } + if((r->use2.b[z]|r->set.b[z]) & bb) { + if(debug['R'] && debug['v']) + print("%P", p); + addreg(&p->to, rn); + if(debug['R'] && debug['v']) + print(" ===change== %P\n", p); + } + + if(STORE(r) & r->regdiff.b[z] & bb) + addmove(r, bn, rn, 1); + r->regu |= rb; + + if(r->refbehind.b[z] & bb) + for(r1 = r->p2; r1 != R; r1 = r1->p2link) + if(r1->refahead.b[z] & bb) + paint3(r1, bn, rb, rn); + + if(!(r->refahead.b[z] & bb)) + break; + r1 = r->s2; + if(r1 != R) + if(r1->refbehind.b[z] & bb) + paint3(r1, bn, rb, rn); + r = r->s1; + if(r == R) + break; + if(r->act.b[z] & bb) + break; + if(!(r->refbehind.b[z] & bb)) + break; + } +} + +void +addreg(Adr *a, int rn) +{ + + a->sym = 0; + a->offset = 0; + a->type = rn; + + ostats.ncvtreg++; +} + +int32 +RtoB(int r) +{ + + if(r < D_AX || r > D_DI) + return 0; + return 1L << (r-D_AX); +} + +int +BtoR(int32 b) +{ + + b &= 0xffL; + if(b == 0) + return 0; + return bitno(b) + D_AX; +} + +void +dumpone(Reg *r) +{ + int z; + Bits bit; + + print("%d:%P", r->loop, r->prog); + for(z=0; zset.b[z] | + r->use1.b[z] | + r->use2.b[z] | + r->refbehind.b[z] | + r->refahead.b[z] | + r->calbehind.b[z] | + r->calahead.b[z] | + r->regdiff.b[z] | + r->act.b[z] | + 0; + if(bany(&bit)) { + print("\t"); + if(bany(&r->set)) + print(" s:%Q", r->set); + if(bany(&r->use1)) + print(" u1:%Q", r->use1); + if(bany(&r->use2)) + print(" u2:%Q", r->use2); + if(bany(&r->refbehind)) + print(" rb:%Q ", r->refbehind); + if(bany(&r->refahead)) + print(" ra:%Q ", r->refahead); + if(bany(&r->calbehind)) + print("cb:%Q ", r->calbehind); + if(bany(&r->calahead)) + print(" ca:%Q ", r->calahead); + if(bany(&r->regdiff)) + print(" d:%Q ", r->regdiff); + if(bany(&r->act)) + print(" a:%Q ", r->act); + } + print("\n"); +} + +void +dumpit(char *str, Reg *r0) +{ + Reg *r, *r1; + + print("\n%s\n", str); + for(r = r0; r != R; r = r->link) { + dumpone(r); + r1 = r->p2; + if(r1 != R) { + print(" pred:"); + for(; r1 != R; r1 = r1->p2link) + print(" %.4ud", r1->prog->loc); + print("\n"); + } +// r1 = r->s1; +// if(r1 != R) { +// print(" succ:"); +// for(; r1 != R; r1 = r1->s1) +// print(" %.4ud", r1->prog->loc); +// print("\n"); +// } + } +} + +static Sym* symlist[10]; + +int +noreturn(Prog *p) +{ + Sym *s; + int i; + + if(symlist[0] == S) { + symlist[0] = pkglookup("panicindex", runtimepkg); + symlist[1] = pkglookup("panicslice", runtimepkg); + symlist[2] = pkglookup("throwinit", runtimepkg); + symlist[3] = pkglookup("panic", runtimepkg); + symlist[4] = pkglookup("panicwrap", runtimepkg); + } + + s = p->to.sym; + if(s == S) + return 0; + for(i=0; symlist[i]!=S; i++) + if(s == symlist[i]) + return 1; + return 0; +} diff --git a/src/cmd/8l/8.out.h b/src/cmd/8l/8.out.h new file mode 100644 index 000000000..9a8483aaf --- /dev/null +++ b/src/cmd/8l/8.out.h @@ -0,0 +1,543 @@ +// Inferno utils/8c/8.out.h +// http://code.google.com/p/inferno-os/source/browse/utils/8c/8.out.h +// +// Copyright © 1994-1999 Lucent Technologies Inc. All rights reserved. +// Portions Copyright © 1995-1997 C H Forsyth (forsyth@terzarima.net) +// Portions Copyright © 1997-1999 Vita Nuova Limited +// Portions Copyright © 2000-2007 Vita Nuova Holdings Limited (www.vitanuova.com) +// Portions Copyright © 2004,2006 Bruce Ellis +// Portions Copyright © 2005-2007 C H Forsyth (forsyth@terzarima.net) +// Revisions Copyright © 2000-2007 Lucent Technologies Inc. and others +// Portions Copyright © 2009 The Go Authors. All rights reserved. +// +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to deal +// in the Software without restriction, including without limitation the rights +// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +// copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +// THE SOFTWARE. + +#define NSYM 50 +#define NSNAME 8 +#define NOPROF (1<<0) +#define DUPOK (1<<1) +#define NOSPLIT (1<<2) +#define RODATA (1<<3) + +enum as +{ + AXXX, + AAAA, + AAAD, + AAAM, + AAAS, + AADCB, + AADCL, + AADCW, + AADDB, + AADDL, + AADDW, + AADJSP, + AANDB, + AANDL, + AANDW, + AARPL, + ABOUNDL, + ABOUNDW, + ABSFL, + ABSFW, + ABSRL, + ABSRW, + ABTL, + ABTW, + ABTCL, + ABTCW, + ABTRL, + ABTRW, + ABTSL, + ABTSW, + ABYTE, + ACALL, + ACLC, + ACLD, + ACLI, + ACLTS, + ACMC, + ACMPB, + ACMPL, + ACMPW, + ACMPSB, + ACMPSL, + ACMPSW, + ADAA, + ADAS, + ADATA, + ADECB, + ADECL, + ADECW, + ADIVB, + ADIVL, + ADIVW, + AENTER, + AGLOBL, + AGOK, + AHISTORY, + AHLT, + AIDIVB, + AIDIVL, + AIDIVW, + AIMULB, + AIMULL, + AIMULW, + AINB, + AINL, + AINW, + AINCB, + AINCL, + AINCW, + AINSB, + AINSL, + AINSW, + AINT, + AINTO, + AIRETL, + AIRETW, + AJCC, + AJCS, + AJCXZ, + AJEQ, + AJGE, + AJGT, + AJHI, + AJLE, + AJLS, + AJLT, + AJMI, + AJMP, + AJNE, + AJOC, + AJOS, + AJPC, + AJPL, + AJPS, + ALAHF, + ALARL, + ALARW, + ALEAL, + ALEAW, + ALEAVEL, + ALEAVEW, + ALOCK, + ALODSB, + ALODSL, + ALODSW, + ALONG, + ALOOP, + ALOOPEQ, + ALOOPNE, + ALSLL, + ALSLW, + AMOVB, + AMOVL, + AMOVW, + AMOVBLSX, + AMOVBLZX, + AMOVBWSX, + AMOVBWZX, + AMOVWLSX, + AMOVWLZX, + AMOVSB, + AMOVSL, + AMOVSW, + AMULB, + AMULL, + AMULW, + ANAME, + ANEGB, + ANEGL, + ANEGW, + ANOP, + ANOTB, + ANOTL, + ANOTW, + AORB, + AORL, + AORW, + AOUTB, + AOUTL, + AOUTW, + AOUTSB, + AOUTSL, + AOUTSW, + APAUSE, + APOPAL, + APOPAW, + APOPFL, + APOPFW, + APOPL, + APOPW, + APUSHAL, + APUSHAW, + APUSHFL, + APUSHFW, + APUSHL, + APUSHW, + ARCLB, + ARCLL, + ARCLW, + ARCRB, + ARCRL, + ARCRW, + AREP, + AREPN, + ARET, + AROLB, + AROLL, + AROLW, + ARORB, + ARORL, + ARORW, + ASAHF, + ASALB, + ASALL, + ASALW, + ASARB, + ASARL, + ASARW, + ASBBB, + ASBBL, + ASBBW, + ASCASB, + ASCASL, + ASCASW, + ASETCC, + ASETCS, + ASETEQ, + ASETGE, + ASETGT, + ASETHI, + ASETLE, + ASETLS, + ASETLT, + ASETMI, + ASETNE, + ASETOC, + ASETOS, + ASETPC, + ASETPL, + ASETPS, + ACDQ, + ACWD, + ASHLB, + ASHLL, + ASHLW, + ASHRB, + ASHRL, + ASHRW, + ASTC, + ASTD, + ASTI, + ASTOSB, + ASTOSL, + ASTOSW, + ASUBB, + ASUBL, + ASUBW, + ASYSCALL, + ATESTB, + ATESTL, + ATESTW, + ATEXT, + AVERR, + AVERW, + AWAIT, + AWORD, + AXCHGB, + AXCHGL, + AXCHGW, + AXLAT, + AXORB, + AXORL, + AXORW, + + AFMOVB, + AFMOVBP, + AFMOVD, + AFMOVDP, + AFMOVF, + AFMOVFP, + AFMOVL, + AFMOVLP, + AFMOVV, + AFMOVVP, + AFMOVW, + AFMOVWP, + AFMOVX, + AFMOVXP, + + AFCOMB, + AFCOMBP, + AFCOMD, + AFCOMDP, + AFCOMDPP, + AFCOMF, + AFCOMFP, + AFCOMI, + AFCOMIP, + AFCOML, + AFCOMLP, + AFCOMW, + AFCOMWP, + AFUCOM, + AFUCOMI, + AFUCOMIP, + AFUCOMP, + AFUCOMPP, + + AFADDDP, + AFADDW, + AFADDL, + AFADDF, + AFADDD, + + AFMULDP, + AFMULW, + AFMULL, + AFMULF, + AFMULD, + + AFSUBDP, + AFSUBW, + AFSUBL, + AFSUBF, + AFSUBD, + + AFSUBRDP, + AFSUBRW, + AFSUBRL, + AFSUBRF, + AFSUBRD, + + AFDIVDP, + AFDIVW, + AFDIVL, + AFDIVF, + AFDIVD, + + AFDIVRDP, + AFDIVRW, + AFDIVRL, + AFDIVRF, + AFDIVRD, + + AFXCHD, + AFFREE, + + AFLDCW, + AFLDENV, + AFRSTOR, + AFSAVE, + AFSTCW, + AFSTENV, + AFSTSW, + + AF2XM1, + AFABS, + AFCHS, + AFCLEX, + AFCOS, + AFDECSTP, + AFINCSTP, + AFINIT, + AFLD1, + AFLDL2E, + AFLDL2T, + AFLDLG2, + AFLDLN2, + AFLDPI, + AFLDZ, + AFNOP, + AFPATAN, + AFPREM, + AFPREM1, + AFPTAN, + AFRNDINT, + AFSCALE, + AFSIN, + AFSINCOS, + AFSQRT, + AFTST, + AFXAM, + AFXTRACT, + AFYL2X, + AFYL2XP1, + + AEND, + + ADYNT_, + AINIT_, + + ASIGNAME, + + ACMPXCHGB, + ACMPXCHGL, + ACMPXCHGW, + ACMPXCHG8B, + + AXADDB, + AXADDL, + AXADDW, + + /* conditional move */ + ACMOVLCC, + ACMOVLCS, + ACMOVLEQ, + ACMOVLGE, + ACMOVLGT, + ACMOVLHI, + ACMOVLLE, + ACMOVLLS, + ACMOVLLT, + ACMOVLMI, + ACMOVLNE, + ACMOVLOC, + ACMOVLOS, + ACMOVLPC, + ACMOVLPL, + ACMOVLPS, + ACMOVWCC, + ACMOVWCS, + ACMOVWEQ, + ACMOVWGE, + ACMOVWGT, + ACMOVWHI, + ACMOVWLE, + ACMOVWLS, + ACMOVWLT, + ACMOVWMI, + ACMOVWNE, + ACMOVWOC, + ACMOVWOS, + ACMOVWPC, + ACMOVWPL, + ACMOVWPS, + + AFCMOVCC, + AFCMOVCS, + AFCMOVEQ, + AFCMOVHI, + AFCMOVLS, + AFCMOVNE, + AFCMOVNU, + AFCMOVUN, + + ALAST +}; + +enum +{ + D_AL = 0, + D_CL, + D_DL, + D_BL, + + D_AH = 4, + D_CH, + D_DH, + D_BH, + + D_AX = 8, + D_CX, + D_DX, + D_BX, + D_SP, + D_BP, + D_SI, + D_DI, + + D_F0 = 16, + D_F7 = D_F0 + 7, + + D_CS = 24, + D_SS, + D_DS, + D_ES, + D_FS, + D_GS, + + D_GDTR, /* global descriptor table register */ + D_IDTR, /* interrupt descriptor table register */ + D_LDTR, /* local descriptor table register */ + D_MSW, /* machine status word */ + D_TASK, /* task register */ + + D_CR = 35, + D_DR = 43, + D_TR = 51, + + D_NONE = 59, + + D_BRANCH = 60, + D_EXTERN = 61, + D_STATIC = 62, + D_AUTO = 63, + D_PARAM = 64, + D_CONST = 65, + D_FCONST = 66, + D_SCONST = 67, + D_ADDR = 68, + + D_FILE, + D_FILE1, + + D_INDIR, /* additive */ + + D_CONST2 = D_INDIR+D_INDIR, + D_SIZE, /* 8l internal */ + D_PCREL, + D_GOTOFF, + D_GOTREL, + + T_TYPE = 1<<0, + T_INDEX = 1<<1, + T_OFFSET = 1<<2, + T_FCONST = 1<<3, + T_SYM = 1<<4, + T_SCONST = 1<<5, + T_OFFSET2 = 1<<6, + T_GOTYPE = 1<<7, + + REGARG = -1, + REGRET = D_AX, + FREGRET = D_F0, + REGSP = D_SP, + REGTMP = D_DI, +}; + +/* + * this is the ranlib header + */ +#define SYMDEF "__.SYMDEF" + +/* + * this is the simulated IEEE floating point + */ +typedef struct ieee Ieee; +struct ieee +{ + int32 l; /* contains ls-man 0xffffffff */ + int32 h; /* contains sign 0x80000000 + exp 0x7ff00000 + ms-man 0x000fffff */ +}; diff --git a/src/cmd/8l/Makefile b/src/cmd/8l/Makefile new file mode 100644 index 000000000..7d34e1704 --- /dev/null +++ b/src/cmd/8l/Makefile @@ -0,0 +1,49 @@ +# Copyright 2009 The Go Authors. All rights reserved. +# Use of this source code is governed by a BSD-style +# license that can be found in the LICENSE file. + +include ../../Make.inc +O:=$(HOST_O) + +TARG=8l + +OFILES=\ + asm.$O\ + data.$O\ + dwarf.$O\ + elf.$O\ + enam.$O\ + go.$O\ + ldelf.$O\ + ldmacho.$O\ + ldpe.$O\ + lib.$O\ + list.$O\ + macho.$O\ + obj.$O\ + optab.$O\ + pass.$O\ + pe.$O\ + prof.$O\ + span.$O\ + symtab.$O\ + + +HFILES=\ + l.h\ + 8.out.h\ + ../ld/dwarf.h\ + ../ld/elf.h\ + ../ld/macho.h\ + ../ld/pe.h\ + +include ../../Make.ccmd + +enam.c: 8.out.h + sh mkenam + +CLEANFILES+=enam.c + + +%.$O: ../ld/%.c + $(HOST_CC) $(HOST_CFLAGS) -c -I. ../ld/$*.c diff --git a/src/cmd/8l/asm.c b/src/cmd/8l/asm.c new file mode 100644 index 000000000..22abd8049 --- /dev/null +++ b/src/cmd/8l/asm.c @@ -0,0 +1,1253 @@ +// Inferno utils/8l/asm.c +// http://code.google.com/p/inferno-os/source/browse/utils/8l/asm.c +// +// Copyright © 1994-1999 Lucent Technologies Inc. All rights reserved. +// Portions Copyright © 1995-1997 C H Forsyth (forsyth@terzarima.net) +// Portions Copyright © 1997-1999 Vita Nuova Limited +// Portions Copyright © 2000-2007 Vita Nuova Holdings Limited (www.vitanuova.com) +// Portions Copyright © 2004,2006 Bruce Ellis +// Portions Copyright © 2005-2007 C H Forsyth (forsyth@terzarima.net) +// Revisions Copyright © 2000-2007 Lucent Technologies Inc. and others +// Portions Copyright © 2009 The Go Authors. All rights reserved. +// +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to deal +// in the Software without restriction, including without limitation the rights +// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +// copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +// THE SOFTWARE. + +// Writing object files. + +#include "l.h" +#include "../ld/lib.h" +#include "../ld/elf.h" +#include "../ld/dwarf.h" +#include "../ld/macho.h" +#include "../ld/pe.h" + +#define Dbufslop 100 + +char linuxdynld[] = "/lib/ld-linux.so.2"; +char freebsddynld[] = "/usr/libexec/ld-elf.so.1"; + +int32 +entryvalue(void) +{ + char *a; + Sym *s; + + a = INITENTRY; + if(*a >= '0' && *a <= '9') + return atolwhex(a); + s = lookup(a, 0); + if(s->type == 0) + return INITTEXT; + if(s->type != STEXT) + diag("entry not text: %s", s->name); + return s->value; +} + +vlong +datoff(vlong addr) +{ + if(addr >= segdata.vaddr) + return addr - segdata.vaddr + segdata.fileoff; + if(addr >= segtext.vaddr) + return addr - segtext.vaddr + segtext.fileoff; + diag("datoff %#llx", addr); + return 0; +} + +enum { + ElfStrEmpty, + ElfStrInterp, + ElfStrHash, + ElfStrGot, + ElfStrGotPlt, + ElfStrDynamic, + ElfStrDynsym, + ElfStrDynstr, + ElfStrRel, + ElfStrText, + ElfStrData, + ElfStrBss, + ElfStrShstrtab, + ElfStrSymtab, + ElfStrStrtab, + ElfStrRelPlt, + ElfStrPlt, + ElfStrGnuVersion, + ElfStrGnuVersionR, + NElfStr +}; + +vlong elfstr[NElfStr]; + +static int +needlib(char *name) +{ + char *p; + Sym *s; + + if(*name == '\0') + return 0; + + /* reuse hash code in symbol table */ + p = smprint(".dynlib.%s", name); + s = lookup(p, 0); + if(s->type == 0) { + s->type = 100; // avoid SDATA, etc. + return 1; + } + return 0; +} + +int nelfsym = 1; + +static void addpltsym(Sym*); +static void addgotsym(Sym*); + +void +adddynrel(Sym *s, Reloc *r) +{ + Sym *targ, *rel, *got; + + targ = r->sym; + cursym = s; + + switch(r->type) { + default: + if(r->type >= 256) { + diag("unexpected relocation type %d", r->type); + return; + } + break; + + // Handle relocations found in ELF object files. + case 256 + R_386_PC32: + if(targ->dynimpname != nil && !targ->dynexport) + diag("unexpected R_386_PC32 relocation for dynamic symbol %s", targ->name); + if(targ->type == 0 || targ->type == SXREF) + diag("unknown symbol %s in pcrel", targ->name); + r->type = D_PCREL; + r->add += 4; + return; + + case 256 + R_386_PLT32: + r->type = D_PCREL; + r->add += 4; + if(targ->dynimpname != nil && !targ->dynexport) { + addpltsym(targ); + r->sym = lookup(".plt", 0); + r->add += targ->plt; + } + return; + + case 256 + R_386_GOT32: + if(targ->dynimpname == nil || targ->dynexport) { + // have symbol + // turn MOVL of GOT entry into LEAL of symbol itself + if(r->off < 2 || s->p[r->off-2] != 0x8b) { + diag("unexpected GOT reloc for non-dynamic symbol %s", targ->name); + return; + } + s->p[r->off-2] = 0x8d; + r->type = D_GOTOFF; + return; + } + addgotsym(targ); + r->type = D_CONST; // write r->add during relocsym + r->sym = S; + r->add += targ->got; + return; + + case 256 + R_386_GOTOFF: + r->type = D_GOTOFF; + return; + + case 256 + R_386_GOTPC: + r->type = D_PCREL; + r->sym = lookup(".got", 0); + r->add += 4; + return; + + case 256 + R_386_32: + if(targ->dynimpname != nil && !targ->dynexport) + diag("unexpected R_386_32 relocation for dynamic symbol %s", targ->name); + r->type = D_ADDR; + return; + + case 512 + MACHO_GENERIC_RELOC_VANILLA*2 + 0: + r->type = D_ADDR; + if(targ->dynimpname != nil && !targ->dynexport) + diag("unexpected reloc for dynamic symbol %s", targ->name); + return; + + case 512 + MACHO_GENERIC_RELOC_VANILLA*2 + 1: + if(targ->dynimpname != nil && !targ->dynexport) { + addpltsym(targ); + r->sym = lookup(".plt", 0); + r->add = targ->plt; + r->type = D_PCREL; + return; + } + r->type = D_PCREL; + return; + + case 512 + MACHO_FAKE_GOTPCREL: + if(targ->dynimpname == nil || targ->dynexport) { + // have symbol + // turn MOVL of GOT entry into LEAL of symbol itself + if(r->off < 2 || s->p[r->off-2] != 0x8b) { + diag("unexpected GOT reloc for non-dynamic symbol %s", targ->name); + return; + } + s->p[r->off-2] = 0x8d; + r->type = D_PCREL; + return; + } + addgotsym(targ); + r->sym = lookup(".got", 0); + r->add += targ->got; + r->type = D_PCREL; + return; + } + + // Handle references to ELF symbols from our own object files. + if(targ->dynimpname == nil || targ->dynexport) + return; + + switch(r->type) { + case D_PCREL: + addpltsym(targ); + r->sym = lookup(".plt", 0); + r->add = targ->plt; + return; + + case D_ADDR: + if(s->type != SDATA) + break; + if(iself) { + adddynsym(targ); + rel = lookup(".rel", 0); + addaddrplus(rel, s, r->off); + adduint32(rel, ELF32_R_INFO(targ->dynid, R_386_32)); + r->type = D_CONST; // write r->add during relocsym + r->sym = S; + return; + } + if(HEADTYPE == Hdarwin && s->size == PtrSize && r->off == 0) { + // Mach-O relocations are a royal pain to lay out. + // They use a compact stateful bytecode representation + // that is too much bother to deal with. + // Instead, interpret the C declaration + // void *_Cvar_stderr = &stderr; + // as making _Cvar_stderr the name of a GOT entry + // for stderr. This is separate from the usual GOT entry, + // just in case the C code assigns to the variable, + // and of course it only works for single pointers, + // but we only need to support cgo and that's all it needs. + adddynsym(targ); + got = lookup(".got", 0); + s->type = got->type | SSUB; + s->outer = got; + s->sub = got->sub; + got->sub = s; + s->value = got->size; + adduint32(got, 0); + adduint32(lookup(".linkedit.got", 0), targ->dynid); + r->type = 256; // ignore during relocsym + return; + } + break; + } + + cursym = s; + diag("unsupported relocation for dynamic symbol %s (type=%d stype=%d)", targ->name, r->type, targ->type); +} + +static void +elfsetupplt(void) +{ + Sym *plt, *got; + + plt = lookup(".plt", 0); + got = lookup(".got.plt", 0); + if(plt->size == 0) { + // pushl got+4 + adduint8(plt, 0xff); + adduint8(plt, 0x35); + addaddrplus(plt, got, 4); + + // jmp *got+8 + adduint8(plt, 0xff); + adduint8(plt, 0x25); + addaddrplus(plt, got, 8); + + // zero pad + adduint32(plt, 0); + + // assume got->size == 0 too + addaddrplus(got, lookup(".dynamic", 0), 0); + adduint32(got, 0); + adduint32(got, 0); + } +} + +int +archreloc(Reloc *r, Sym *s, vlong *val) +{ + USED(s); + switch(r->type) { + case D_CONST: + *val = r->add; + return 0; + case D_GOTOFF: + *val = symaddr(r->sym) + r->add - symaddr(lookup(".got", 0)); + return 0; + } + return -1; +} + +static void +addpltsym(Sym *s) +{ + Sym *plt, *got, *rel; + + if(s->plt >= 0) + return; + + adddynsym(s); + + if(iself) { + plt = lookup(".plt", 0); + got = lookup(".got.plt", 0); + rel = lookup(".rel.plt", 0); + if(plt->size == 0) + elfsetupplt(); + + // jmpq *got+size + adduint8(plt, 0xff); + adduint8(plt, 0x25); + addaddrplus(plt, got, got->size); + + // add to got: pointer to current pos in plt + addaddrplus(got, plt, plt->size); + + // pushl $x + adduint8(plt, 0x68); + adduint32(plt, rel->size); + + // jmp .plt + adduint8(plt, 0xe9); + adduint32(plt, -(plt->size+4)); + + // rel + addaddrplus(rel, got, got->size-4); + adduint32(rel, ELF32_R_INFO(s->dynid, R_386_JMP_SLOT)); + + s->plt = plt->size - 16; + } else if(HEADTYPE == Hdarwin) { + // Same laziness as in 6l. + + Sym *plt; + + plt = lookup(".plt", 0); + + addgotsym(s); + + adduint32(lookup(".linkedit.plt", 0), s->dynid); + + // jmpq *got+size(IP) + s->plt = plt->size; + + adduint8(plt, 0xff); + adduint8(plt, 0x25); + addaddrplus(plt, lookup(".got", 0), s->got); + } else { + diag("addpltsym: unsupported binary format"); + } +} + +static void +addgotsym(Sym *s) +{ + Sym *got, *rel; + + if(s->got >= 0) + return; + + adddynsym(s); + got = lookup(".got", 0); + s->got = got->size; + adduint32(got, 0); + + if(iself) { + rel = lookup(".rel", 0); + addaddrplus(rel, got, s->got); + adduint32(rel, ELF32_R_INFO(s->dynid, R_386_GLOB_DAT)); + } else if(HEADTYPE == Hdarwin) { + adduint32(lookup(".linkedit.got", 0), s->dynid); + } else { + diag("addgotsym: unsupported binary format"); + } +} + +void +adddynsym(Sym *s) +{ + Sym *d, *str; + int t; + char *name; + + if(s->dynid >= 0) + return; + + if(s->dynimpname == nil) + diag("adddynsym: no dynamic name for %s", s->name, *(int32*)0); + + if(iself) { + s->dynid = nelfsym++; + + d = lookup(".dynsym", 0); + + /* name */ + name = s->dynimpname; + if(name == nil) + name = s->name; + adduint32(d, addstring(lookup(".dynstr", 0), name)); + + /* value */ + if(s->type == SDYNIMPORT) + adduint32(d, 0); + else + addaddr(d, s); + + /* size */ + adduint32(d, 0); + + /* type */ + t = STB_GLOBAL << 4; + if(s->dynexport && s->type == STEXT) + t |= STT_FUNC; + else + t |= STT_OBJECT; + adduint8(d, t); + adduint8(d, 0); + + /* shndx */ + if(!s->dynexport && s->dynimpname != nil) + adduint16(d, SHN_UNDEF); + else { + switch(s->type) { + default: + case STEXT: + t = 11; + break; + case SRODATA: + t = 12; + break; + case SDATA: + t = 13; + break; + case SBSS: + t = 14; + break; + } + adduint16(d, t); + } + } else if(HEADTYPE == Hdarwin) { + // Mach-O symbol nlist32 + d = lookup(".dynsym", 0); + name = s->dynimpname; + if(name == nil) + name = s->name; + s->dynid = d->size/12; + // darwin still puts _ prefixes on all C symbols + str = lookup(".dynstr", 0); + adduint32(d, str->size); + adduint8(str, '_'); + addstring(str, name); + adduint8(d, 0x01); // type - N_EXT - external symbol + adduint8(d, 0); // section + adduint16(d, 0); // desc + adduint32(d, 0); // value + } else if(HEADTYPE != Hwindows) { + diag("adddynsym: unsupported binary format"); + } +} + +void +adddynlib(char *lib) +{ + Sym *s; + + if(!needlib(lib)) + return; + + if(iself) { + s = lookup(".dynstr", 0); + if(s->size == 0) + addstring(s, ""); + elfwritedynent(lookup(".dynamic", 0), DT_NEEDED, addstring(s, lib)); + } else if(HEADTYPE == Hdarwin) { + machoadddynlib(lib); + } else if(HEADTYPE != Hwindows) { + diag("adddynlib: unsupported binary format"); + } +} + +void +doelf(void) +{ + Sym *s, *shstrtab, *dynstr; + + if(!iself) + return; + + /* predefine strings we need for section headers */ + shstrtab = lookup(".shstrtab", 0); + shstrtab->type = SELFROSECT; + shstrtab->reachable = 1; + + elfstr[ElfStrEmpty] = addstring(shstrtab, ""); + elfstr[ElfStrText] = addstring(shstrtab, ".text"); + elfstr[ElfStrData] = addstring(shstrtab, ".data"); + elfstr[ElfStrBss] = addstring(shstrtab, ".bss"); + addstring(shstrtab, ".elfdata"); + addstring(shstrtab, ".rodata"); + addstring(shstrtab, ".gosymtab"); + addstring(shstrtab, ".gopclntab"); + if(!debug['s']) { + elfstr[ElfStrSymtab] = addstring(shstrtab, ".symtab"); + elfstr[ElfStrStrtab] = addstring(shstrtab, ".strtab"); + dwarfaddshstrings(shstrtab); + } + elfstr[ElfStrShstrtab] = addstring(shstrtab, ".shstrtab"); + + if(!debug['d']) { /* -d suppresses dynamic loader format */ + elfstr[ElfStrInterp] = addstring(shstrtab, ".interp"); + elfstr[ElfStrHash] = addstring(shstrtab, ".hash"); + elfstr[ElfStrGot] = addstring(shstrtab, ".got"); + elfstr[ElfStrGotPlt] = addstring(shstrtab, ".got.plt"); + elfstr[ElfStrDynamic] = addstring(shstrtab, ".dynamic"); + elfstr[ElfStrDynsym] = addstring(shstrtab, ".dynsym"); + elfstr[ElfStrDynstr] = addstring(shstrtab, ".dynstr"); + elfstr[ElfStrRel] = addstring(shstrtab, ".rel"); + elfstr[ElfStrRelPlt] = addstring(shstrtab, ".rel.plt"); + elfstr[ElfStrPlt] = addstring(shstrtab, ".plt"); + elfstr[ElfStrGnuVersion] = addstring(shstrtab, ".gnu.version"); + elfstr[ElfStrGnuVersionR] = addstring(shstrtab, ".gnu.version_r"); + + /* dynamic symbol table - first entry all zeros */ + s = lookup(".dynsym", 0); + s->type = SELFROSECT; + s->reachable = 1; + s->size += ELF32SYMSIZE; + + /* dynamic string table */ + s = lookup(".dynstr", 0); + s->reachable = 1; + s->type = SELFROSECT; + if(s->size == 0) + addstring(s, ""); + dynstr = s; + + /* relocation table */ + s = lookup(".rel", 0); + s->reachable = 1; + s->type = SELFROSECT; + + /* global offset table */ + s = lookup(".got", 0); + s->reachable = 1; + s->type = SELFSECT; // writable + + /* hash */ + s = lookup(".hash", 0); + s->reachable = 1; + s->type = SELFROSECT; + + /* got.plt */ + s = lookup(".got.plt", 0); + s->reachable = 1; + s->type = SELFSECT; // writable + + s = lookup(".plt", 0); + s->reachable = 1; + s->type = SELFROSECT; + + s = lookup(".rel.plt", 0); + s->reachable = 1; + s->type = SELFROSECT; + + s = lookup(".gnu.version", 0); + s->reachable = 1; + s->type = SELFROSECT; + + s = lookup(".gnu.version_r", 0); + s->reachable = 1; + s->type = SELFROSECT; + + elfsetupplt(); + + /* define dynamic elf table */ + s = lookup(".dynamic", 0); + s->reachable = 1; + s->type = SELFROSECT; + + /* + * .dynamic table + */ + elfwritedynentsym(s, DT_HASH, lookup(".hash", 0)); + elfwritedynentsym(s, DT_SYMTAB, lookup(".dynsym", 0)); + elfwritedynent(s, DT_SYMENT, ELF32SYMSIZE); + elfwritedynentsym(s, DT_STRTAB, lookup(".dynstr", 0)); + elfwritedynentsymsize(s, DT_STRSZ, lookup(".dynstr", 0)); + elfwritedynentsym(s, DT_REL, lookup(".rel", 0)); + elfwritedynentsymsize(s, DT_RELSZ, lookup(".rel", 0)); + elfwritedynent(s, DT_RELENT, ELF32RELSIZE); + if(rpath) + elfwritedynent(s, DT_RUNPATH, addstring(dynstr, rpath)); + elfwritedynentsym(s, DT_PLTGOT, lookup(".got.plt", 0)); + elfwritedynent(s, DT_PLTREL, DT_REL); + elfwritedynentsymsize(s, DT_PLTRELSZ, lookup(".rel.plt", 0)); + elfwritedynentsym(s, DT_JMPREL, lookup(".rel.plt", 0)); + + // Do not write DT_NULL. elfdynhash will finish it. + } +} + +void +shsym(Elf64_Shdr *sh, Sym *s) +{ + vlong addr; + addr = symaddr(s); + if(sh->flags&SHF_ALLOC) + sh->addr = addr; + sh->off = datoff(addr); + sh->size = s->size; +} + +void +phsh(Elf64_Phdr *ph, Elf64_Shdr *sh) +{ + ph->vaddr = sh->addr; + ph->paddr = ph->vaddr; + ph->off = sh->off; + ph->filesz = sh->size; + ph->memsz = sh->size; + ph->align = sh->addralign; +} + +void +asmb(void) +{ + int32 v, magic; + int a, dynsym; + uint32 symo, startva, machlink; + ElfEhdr *eh; + ElfPhdr *ph, *pph; + ElfShdr *sh; + Section *sect; + Sym *sym; + int o; + int i; + + if(debug['v']) + Bprint(&bso, "%5.2f asmb\n", cputime()); + Bflush(&bso); + + sect = segtext.sect; + cseek(sect->vaddr - segtext.vaddr + segtext.fileoff); + codeblk(sect->vaddr, sect->len); + + /* output read-only data in text segment (rodata, gosymtab and pclntab) */ + for(sect = sect->next; sect != nil; sect = sect->next) { + cseek(sect->vaddr - segtext.vaddr + segtext.fileoff); + datblk(sect->vaddr, sect->len); + } + + if(debug['v']) + Bprint(&bso, "%5.2f datblk\n", cputime()); + Bflush(&bso); + + cseek(segdata.fileoff); + datblk(segdata.vaddr, segdata.filelen); + + machlink = 0; + if(HEADTYPE == Hdarwin) + machlink = domacholink(); + + if(iself) { + /* index of elf text section; needed by asmelfsym, double-checked below */ + /* !debug['d'] causes extra sections before the .text section */ + elftextsh = 2; + if(!debug['d']) { + elftextsh += 10; + if(elfverneed) + elftextsh += 2; + } + } + + symsize = 0; + spsize = 0; + lcsize = 0; + symo = 0; + if(!debug['s']) { + // TODO: rationalize + if(debug['v']) + Bprint(&bso, "%5.2f sym\n", cputime()); + Bflush(&bso); + switch(HEADTYPE) { + default: + if(iself) + goto Elfsym; + case Hgarbunix: + symo = rnd(HEADR+segtext.filelen, 8192)+segdata.filelen; + break; + case Hunixcoff: + symo = rnd(HEADR+segtext.filelen, INITRND)+segdata.filelen; + break; + case Hplan9x32: + symo = HEADR+segtext.filelen+segdata.filelen; + break; + case Hmsdoscom: + case Hmsdosexe: + debug['s'] = 1; + symo = HEADR+segtext.filelen+segdata.filelen; + break; + case Hdarwin: + symo = rnd(HEADR+segtext.filelen, INITRND)+rnd(segdata.filelen, INITRND)+machlink; + break; + Elfsym: + symo = rnd(HEADR+segtext.filelen, INITRND)+segdata.filelen; + symo = rnd(symo, INITRND); + break; + case Hwindows: + symo = rnd(HEADR+segtext.filelen, PEFILEALIGN)+segdata.filelen; + symo = rnd(symo, PEFILEALIGN); + break; + } + cseek(symo); + switch(HEADTYPE) { + default: + if(iself) { + if(debug['v']) + Bprint(&bso, "%5.2f elfsym\n", cputime()); + asmelfsym(); + cflush(); + cwrite(elfstrdat, elfstrsize); + + if(debug['v']) + Bprint(&bso, "%5.2f dwarf\n", cputime()); + dwarfemitdebugsections(); + } + break; + case Hplan9x32: + asmplan9sym(); + cflush(); + + sym = lookup("pclntab", 0); + if(sym != nil) { + lcsize = sym->np; + for(i=0; i < lcsize; i++) + cput(sym->p[i]); + + cflush(); + } + break; + case Hdarwin: + case Hwindows: + if(debug['v']) + Bprint(&bso, "%5.2f dwarf\n", cputime()); + dwarfemitdebugsections(); + break; + } + } + if(debug['v']) + Bprint(&bso, "%5.2f headr\n", cputime()); + Bflush(&bso); + cseek(0L); + switch(HEADTYPE) { + default: + if(iself) + goto Elfput; + case Hgarbunix: /* garbage */ + lputb(0x160L<<16); /* magic and sections */ + lputb(0L); /* time and date */ + lputb(rnd(HEADR+segtext.filelen, 4096)+segdata.filelen); + lputb(symsize); /* nsyms */ + lputb((0x38L<<16)|7L); /* size of optional hdr and flags */ + lputb((0413<<16)|0437L); /* magic and version */ + lputb(rnd(HEADR+segtext.filelen, 4096)); /* sizes */ + lputb(segdata.filelen); + lputb(segdata.len - segdata.filelen); + lputb(entryvalue()); /* va of entry */ + lputb(INITTEXT-HEADR); /* va of base of text */ + lputb(segdata.vaddr); /* va of base of data */ + lputb(segdata.vaddr+segdata.filelen); /* va of base of bss */ + lputb(~0L); /* gp reg mask */ + lputb(0L); + lputb(0L); + lputb(0L); + lputb(0L); + lputb(~0L); /* gp value ?? */ + break; + case Hunixcoff: /* unix coff */ + /* + * file header + */ + lputl(0x0004014c); /* 4 sections, magic */ + lputl(0); /* unix time stamp */ + lputl(0); /* symbol table */ + lputl(0); /* nsyms */ + lputl(0x0003001c); /* flags, sizeof a.out header */ + /* + * a.out header + */ + lputl(0x10b); /* magic, version stamp */ + lputl(rnd(segtext.filelen, INITRND)); /* text sizes */ + lputl(segdata.filelen); /* data sizes */ + lputl(segdata.len - segdata.filelen); /* bss sizes */ + lputb(entryvalue()); /* va of entry */ + lputl(INITTEXT); /* text start */ + lputl(segdata.vaddr); /* data start */ + /* + * text section header + */ + s8put(".text"); + lputl(HEADR); /* pa */ + lputl(HEADR); /* va */ + lputl(segtext.filelen); /* text size */ + lputl(HEADR); /* file offset */ + lputl(0); /* relocation */ + lputl(0); /* line numbers */ + lputl(0); /* relocation, line numbers */ + lputl(0x20); /* flags text only */ + /* + * data section header + */ + s8put(".data"); + lputl(segdata.vaddr); /* pa */ + lputl(segdata.vaddr); /* va */ + lputl(segdata.filelen); /* data size */ + lputl(HEADR+segtext.filelen); /* file offset */ + lputl(0); /* relocation */ + lputl(0); /* line numbers */ + lputl(0); /* relocation, line numbers */ + lputl(0x40); /* flags data only */ + /* + * bss section header + */ + s8put(".bss"); + lputl(segdata.vaddr+segdata.filelen); /* pa */ + lputl(segdata.vaddr+segdata.filelen); /* va */ + lputl(segdata.len - segdata.filelen); /* bss size */ + lputl(0); /* file offset */ + lputl(0); /* relocation */ + lputl(0); /* line numbers */ + lputl(0); /* relocation, line numbers */ + lputl(0x80); /* flags bss only */ + /* + * comment section header + */ + s8put(".comment"); + lputl(0); /* pa */ + lputl(0); /* va */ + lputl(symsize+lcsize); /* comment size */ + lputl(HEADR+segtext.filelen+segdata.filelen); /* file offset */ + lputl(HEADR+segtext.filelen+segdata.filelen); /* offset of syms */ + lputl(HEADR+segtext.filelen+segdata.filelen+symsize);/* offset of line numbers */ + lputl(0); /* relocation, line numbers */ + lputl(0x200); /* flags comment only */ + break; + case Hplan9x32: /* plan9 */ + magic = 4*11*11+7; + lputb(magic); /* magic */ + lputb(segtext.filelen); /* sizes */ + lputb(segdata.filelen); + lputb(segdata.len - segdata.filelen); + lputb(symsize); /* nsyms */ + lputb(entryvalue()); /* va of entry */ + lputb(spsize); /* sp offsets */ + lputb(lcsize); /* line offsets */ + break; + case Hmsdoscom: + /* MS-DOS .COM */ + break; + case Hmsdosexe: + /* fake MS-DOS .EXE */ + v = rnd(HEADR+segtext.filelen, INITRND)+segdata.filelen; + wputl(0x5A4D); /* 'MZ' */ + wputl(v % 512); /* bytes in last page */ + wputl(rnd(v, 512)/512); /* total number of pages */ + wputl(0x0000); /* number of reloc items */ + v = rnd(HEADR-(INITTEXT & 0xFFFF), 16); + wputl(v/16); /* size of header */ + wputl(0x0000); /* minimum allocation */ + wputl(0xFFFF); /* maximum allocation */ + wputl(0x0000); /* initial ss value */ + wputl(0x0100); /* initial sp value */ + wputl(0x0000); /* complemented checksum */ + v = entryvalue(); + wputl(v); /* initial ip value (!) */ + wputl(0x0000); /* initial cs value */ + wputl(0x0000); + wputl(0x0000); + wputl(0x003E); /* reloc table offset */ + wputl(0x0000); /* overlay number */ + break; + + case Hdarwin: + asmbmacho(); + break; + + Elfput: + eh = getElfEhdr(); + startva = INITTEXT - HEADR; + + /* This null SHdr must appear before all others */ + newElfShdr(elfstr[ElfStrEmpty]); + + /* program header info */ + pph = newElfPhdr(); + pph->type = PT_PHDR; + pph->flags = PF_R + PF_X; + pph->off = eh->ehsize; + pph->vaddr = INITTEXT - HEADR + pph->off; + pph->paddr = INITTEXT - HEADR + pph->off; + pph->align = INITRND; + + /* + * PHDR must be in a loaded segment. Adjust the text + * segment boundaries downwards to include it. + */ + o = segtext.vaddr - pph->vaddr; + segtext.vaddr -= o; + segtext.len += o; + o = segtext.fileoff - pph->off; + segtext.fileoff -= o; + segtext.filelen += o; + + if(!debug['d']) { + /* interpreter */ + sh = newElfShdr(elfstr[ElfStrInterp]); + sh->type = SHT_PROGBITS; + sh->flags = SHF_ALLOC; + sh->addralign = 1; + if(interpreter == nil) { + switch(HEADTYPE) { + case Hlinux: + interpreter = linuxdynld; + break; + case Hfreebsd: + interpreter = freebsddynld; + break; + } + } + elfinterp(sh, startva, interpreter); + + ph = newElfPhdr(); + ph->type = PT_INTERP; + ph->flags = PF_R; + phsh(ph, sh); + } + + elfphload(&segtext); + elfphload(&segdata); + + /* Dynamic linking sections */ + if (!debug['d']) { /* -d suppresses dynamic loader format */ + /* S headers for dynamic linking */ + sh = newElfShdr(elfstr[ElfStrGot]); + sh->type = SHT_PROGBITS; + sh->flags = SHF_ALLOC+SHF_WRITE; + sh->entsize = 4; + sh->addralign = 4; + shsym(sh, lookup(".got", 0)); + + sh = newElfShdr(elfstr[ElfStrGotPlt]); + sh->type = SHT_PROGBITS; + sh->flags = SHF_ALLOC+SHF_WRITE; + sh->entsize = 4; + sh->addralign = 4; + shsym(sh, lookup(".got.plt", 0)); + + dynsym = eh->shnum; + sh = newElfShdr(elfstr[ElfStrDynsym]); + sh->type = SHT_DYNSYM; + sh->flags = SHF_ALLOC; + sh->entsize = ELF32SYMSIZE; + sh->addralign = 4; + sh->link = dynsym+1; // dynstr + // sh->info = index of first non-local symbol (number of local symbols) + shsym(sh, lookup(".dynsym", 0)); + + sh = newElfShdr(elfstr[ElfStrDynstr]); + sh->type = SHT_STRTAB; + sh->flags = SHF_ALLOC; + sh->addralign = 1; + shsym(sh, lookup(".dynstr", 0)); + + if(elfverneed) { + sh = newElfShdr(elfstr[ElfStrGnuVersion]); + sh->type = SHT_GNU_VERSYM; + sh->flags = SHF_ALLOC; + sh->addralign = 2; + sh->link = dynsym; + sh->entsize = 2; + shsym(sh, lookup(".gnu.version", 0)); + + sh = newElfShdr(elfstr[ElfStrGnuVersionR]); + sh->type = SHT_GNU_VERNEED; + sh->flags = SHF_ALLOC; + sh->addralign = 4; + sh->info = elfverneed; + sh->link = dynsym+1; // dynstr + shsym(sh, lookup(".gnu.version_r", 0)); + } + + sh = newElfShdr(elfstr[ElfStrRelPlt]); + sh->type = SHT_REL; + sh->flags = SHF_ALLOC; + sh->entsize = ELF32RELSIZE; + sh->addralign = 4; + sh->link = dynsym; + sh->info = eh->shnum; // .plt + shsym(sh, lookup(".rel.plt", 0)); + + sh = newElfShdr(elfstr[ElfStrPlt]); + sh->type = SHT_PROGBITS; + sh->flags = SHF_ALLOC+SHF_EXECINSTR; + sh->entsize = 4; + sh->addralign = 4; + shsym(sh, lookup(".plt", 0)); + + sh = newElfShdr(elfstr[ElfStrHash]); + sh->type = SHT_HASH; + sh->flags = SHF_ALLOC; + sh->entsize = 4; + sh->addralign = 4; + sh->link = dynsym; + shsym(sh, lookup(".hash", 0)); + + sh = newElfShdr(elfstr[ElfStrRel]); + sh->type = SHT_REL; + sh->flags = SHF_ALLOC; + sh->entsize = ELF32RELSIZE; + sh->addralign = 4; + sh->link = dynsym; + shsym(sh, lookup(".rel", 0)); + + /* sh and PT_DYNAMIC for .dynamic section */ + sh = newElfShdr(elfstr[ElfStrDynamic]); + sh->type = SHT_DYNAMIC; + sh->flags = SHF_ALLOC+SHF_WRITE; + sh->entsize = 8; + sh->addralign = 4; + sh->link = dynsym+1; // dynstr + shsym(sh, lookup(".dynamic", 0)); + ph = newElfPhdr(); + ph->type = PT_DYNAMIC; + ph->flags = PF_R + PF_W; + phsh(ph, sh); + + /* + * Thread-local storage segment (really just size). + */ + if(tlsoffset != 0) { + ph = newElfPhdr(); + ph->type = PT_TLS; + ph->flags = PF_R; + ph->memsz = -tlsoffset; + ph->align = 4; + } + } + + ph = newElfPhdr(); + ph->type = PT_GNU_STACK; + ph->flags = PF_W+PF_R; + ph->align = 4; + + sh = newElfShstrtab(elfstr[ElfStrShstrtab]); + sh->type = SHT_STRTAB; + sh->addralign = 1; + shsym(sh, lookup(".shstrtab", 0)); + + if(elftextsh != eh->shnum) + diag("elftextsh = %d, want %d", elftextsh, eh->shnum); + for(sect=segtext.sect; sect!=nil; sect=sect->next) + elfshbits(sect); + for(sect=segdata.sect; sect!=nil; sect=sect->next) + elfshbits(sect); + + if (!debug['s']) { + sh = newElfShdr(elfstr[ElfStrSymtab]); + sh->type = SHT_SYMTAB; + sh->off = symo; + sh->size = symsize; + sh->addralign = 4; + sh->entsize = 16; + sh->link = eh->shnum; // link to strtab + + sh = newElfShdr(elfstr[ElfStrStrtab]); + sh->type = SHT_STRTAB; + sh->off = symo+symsize; + sh->size = elfstrsize; + sh->addralign = 1; + + dwarfaddelfheaders(); + } + + /* Main header */ + eh->ident[EI_MAG0] = '\177'; + eh->ident[EI_MAG1] = 'E'; + eh->ident[EI_MAG2] = 'L'; + eh->ident[EI_MAG3] = 'F'; + eh->ident[EI_CLASS] = ELFCLASS32; + eh->ident[EI_DATA] = ELFDATA2LSB; + eh->ident[EI_VERSION] = EV_CURRENT; + switch(HEADTYPE) { + case Hfreebsd: + eh->ident[EI_OSABI] = 9; + break; + } + + eh->type = ET_EXEC; + eh->machine = EM_386; + eh->version = EV_CURRENT; + eh->entry = entryvalue(); + + if(pph != nil) { + pph->filesz = eh->phnum * eh->phentsize; + pph->memsz = pph->filesz; + } + + cseek(0); + a = 0; + a += elfwritehdr(); + a += elfwritephdrs(); + a += elfwriteshdrs(); + cflush(); + if(a+elfwriteinterp() > ELFRESERVE) + diag("ELFRESERVE too small: %d > %d", a, ELFRESERVE); + break; + + case Hwindows: + asmbpe(); + break; + } + cflush(); +} + +void +s8put(char *n) +{ + char name[8]; + int i; + + strncpy(name, n, sizeof(name)); + for(i=0; itype == STEXT) + put(s, s->name, 'T', s->value, s->size, s->version, 0); + + for(h=0; hhash) { + if(s->hide) + continue; + switch(s->type&~SSUB) { + case SCONST: + case SRODATA: + case SDATA: + case SELFROSECT: + case SMACHO: + case SMACHOGOT: + case STYPE: + case SSTRING: + case SGOSTRING: + case SWINDOWS: + if(!s->reachable) + continue; + put(s, s->name, 'D', symaddr(s), s->size, s->version, s->gotype); + continue; + + case SBSS: + if(!s->reachable) + continue; + put(s, s->name, 'B', symaddr(s), s->size, s->version, s->gotype); + continue; + + case SFILE: + put(nil, s->name, 'f', s->value, 0, s->version, 0); + continue; + } + } + } + + for(s = textp; s != nil; s = s->next) { + if(s->text == nil) + continue; + + /* filenames first */ + for(a=s->autom; a; a=a->link) + if(a->type == D_FILE) + put(nil, a->asym->name, 'z', a->aoffset, 0, 0, 0); + else + if(a->type == D_FILE1) + put(nil, a->asym->name, 'Z', a->aoffset, 0, 0, 0); + + put(s, s->name, 'T', s->value, s->size, s->version, s->gotype); + + /* frame, auto and param after */ + put(nil, ".frame", 'm', s->text->to.offset+4, 0, 0, 0); + + for(a=s->autom; a; a=a->link) + if(a->type == D_AUTO) + put(nil, a->asym->name, 'a', -a->aoffset, 0, 0, a->gotype); + else + if(a->type == D_PARAM) + put(nil, a->asym->name, 'p', a->aoffset, 0, 0, a->gotype); + } + if(debug['v'] || debug['n']) + Bprint(&bso, "symsize = %d\n", symsize); + Bflush(&bso); +} diff --git a/src/cmd/8l/doc.go b/src/cmd/8l/doc.go new file mode 100644 index 000000000..b70888907 --- /dev/null +++ b/src/cmd/8l/doc.go @@ -0,0 +1,50 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +/* + +8l is a modified version of the Plan 9 linker. The original is documented at + + http://plan9.bell-labs.com/magic/man2html/1/2l + +Its target architecture is the x86, referred to by these tools for historical reasons as 386. +It reads files in .8 format generated by 8g, 8c, and 8a and emits +a binary called 8.out by default. + +Major changes include: + - support for ELF and Mach-O binary files + - support for segmented stacks (this feature is implemented here, not in the compilers). + + +Original options are listed in the link above. + +Options new in this version: + +-d + Elide the dynamic linking header. With this option, the binary + is statically linked and does not refer to dynld. Without this option + (the default), the binary's contents are identical but it is loaded with dynld. +-Hplan9 + Write Plan 9 32-bit format binaries (default when $GOOS is plan9) +-Hdarwin + Write Apple Mach-O binaries (default when $GOOS is darwin) +-Hlinux + Write Linux ELF binaries (default when $GOOS is linux) +-Hfreebsd + Write FreeBSD ELF binaries (default when $GOOS is freebsd) +-Hwindows + Write Windows PE32 binaries (default when $GOOS is windows) +-I interpreter + Set the ELF dynamic linker to use. +-L dir1 -L dir2 + Search for libraries (package files) in dir1, dir2, etc. + The default is the single location $GOROOT/pkg/$GOOS_386. +-r dir1:dir2:... + Set the dynamic linker search path when using ELF. +-V + Print the linker version. + + +*/ +package documentation diff --git a/src/cmd/8l/l.h b/src/cmd/8l/l.h new file mode 100644 index 000000000..4ee0db967 --- /dev/null +++ b/src/cmd/8l/l.h @@ -0,0 +1,374 @@ +// Inferno utils/8l/l.h +// http://code.google.com/p/inferno-os/source/browse/utils/8l/l.h +// +// Copyright © 1994-1999 Lucent Technologies Inc. All rights reserved. +// Portions Copyright © 1995-1997 C H Forsyth (forsyth@terzarima.net) +// Portions Copyright © 1997-1999 Vita Nuova Limited +// Portions Copyright © 2000-2007 Vita Nuova Holdings Limited (www.vitanuova.com) +// Portions Copyright © 2004,2006 Bruce Ellis +// Portions Copyright © 2005-2007 C H Forsyth (forsyth@terzarima.net) +// Revisions Copyright © 2000-2007 Lucent Technologies Inc. and others +// Portions Copyright © 2009 The Go Authors. All rights reserved. +// +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to deal +// in the Software without restriction, including without limitation the rights +// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +// copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +// THE SOFTWARE. + +#include +#include +#include +#include "8.out.h" + +#ifndef EXTERN +#define EXTERN extern +#endif + +enum +{ + thechar = '8', + PtrSize = 4 +}; + +#define P ((Prog*)0) +#define S ((Sym*)0) +#define TNAME (cursym?cursym->name:noname) + +typedef struct Adr Adr; +typedef struct Prog Prog; +typedef struct Sym Sym; +typedef struct Auto Auto; +typedef struct Optab Optab; +typedef struct Reloc Reloc; + +struct Adr +{ + union + { + int32 u0offset; + char u0scon[8]; + Prog *u0cond; /* not used, but should be D_BRANCH */ + Ieee u0ieee; + char *u0sbig; + } u0; + Sym* sym; + short type; + uchar index; + char scale; + int32 offset2; +}; + +#define offset u0.u0offset +#define scon u0.u0scon +#define cond u0.u0cond +#define ieee u0.u0ieee +#define sbig u0.u0sbig + +struct Reloc +{ + int32 off; + uchar siz; + int32 type; + int32 add; + Sym* sym; +}; + +struct Prog +{ + Adr from; + Adr to; + Prog* forwd; + Prog* comefrom; + Prog* link; + Prog* pcond; /* work on this */ + int32 pc; + int32 spadj; + int32 line; + short as; + char width; /* fake for DATA */ + char ft; /* oclass cache */ + char tt; + uchar mark; /* work on these */ + uchar back; + uchar bigjmp; +}; +#define datasize from.scale +#define textflag from.scale +#define iscall(p) ((p)->as == ACALL) + +struct Auto +{ + Sym* asym; + Auto* link; + int32 aoffset; + short type; + Sym* gotype; +}; +struct Sym +{ + char* name; + short type; + short version; + uchar dupok; + uchar reachable; + uchar dynexport; + uchar special; + uchar stkcheck; + uchar hide; + int32 value; + int32 size; + int32 sig; + int32 dynid; + int32 plt; + int32 got; + Sym* hash; // in hash table + Sym* allsym; // in all symbol list + Sym* next; // in text or data list + Sym* sub; // in sub list + Sym* outer; // container of sub + Sym* gotype; + char* file; + char* dynimpname; + char* dynimplib; + char* dynimpvers; + + // STEXT + Auto* autom; + Prog* text; + + // SDATA, SBSS + uchar* p; + int32 np; + int32 maxp; + Reloc* r; + int32 nr; + int32 maxr; +}; +struct Optab +{ + short as; + uchar* ytab; + uchar prefix; + uchar op[10]; +}; + +enum +{ + MINSIZ = 4, + STRINGSZ = 200, + MINLC = 1, + MAXIO = 8192, + MAXHIST = 20, /* limit of path elements for history symbols */ + + Yxxx = 0, + Ynone, + Yi0, + Yi1, + Yi8, + Yi32, + Yiauto, + Yal, + Ycl, + Yax, + Ycx, + Yrb, + Yrl, + Yrf, + Yf0, + Yrx, + Ymb, + Yml, + Ym, + Ybr, + Ycol, + + Ycs, Yss, Yds, Yes, Yfs, Ygs, + Ygdtr, Yidtr, Yldtr, Ymsw, Ytask, + Ycr0, Ycr1, Ycr2, Ycr3, Ycr4, Ycr5, Ycr6, Ycr7, + Ydr0, Ydr1, Ydr2, Ydr3, Ydr4, Ydr5, Ydr6, Ydr7, + Ytr0, Ytr1, Ytr2, Ytr3, Ytr4, Ytr5, Ytr6, Ytr7, + Ymax, + + Zxxx = 0, + + Zlit, + Z_rp, + Zbr, + Zcall, + Zcallcon, + Zib_, + Zib_rp, + Zibo_m, + Zil_, + Zil_rp, + Zilo_m, + Zjmp, + Zjmpcon, + Zloop, + Zm_o, + Zm_r, + Zaut_r, + Zo_m, + Zpseudo, + Zr_m, + Zrp_, + Z_ib, + Z_il, + Zm_ibo, + Zm_ilo, + Zib_rr, + Zil_rr, + Zclr, + Zbyte, + Zmov, + Zmax, + + Px = 0, + Pe = 0x66, /* operand escape */ + Pm = 0x0f, /* 2byte opcode escape */ + Pq = 0xff, /* both escape */ + Pb = 0xfe, /* byte operands */ +}; + +#pragma varargck type "A" int +#pragma varargck type "D" Adr* +#pragma varargck type "I" uchar* +#pragma varargck type "P" Prog* +#pragma varargck type "R" int +#pragma varargck type "S" char* +#pragma varargck type "Y" Sym* +#pragma varargck type "Z" char* +#pragma varargck type "i" char* + +EXTERN int32 HEADR; +EXTERN int32 HEADTYPE; +EXTERN int32 INITRND; +EXTERN int32 INITTEXT; +EXTERN int32 INITDAT; +EXTERN char* INITENTRY; /* entry point */ +EXTERN int32 casepc; +EXTERN char* pcstr; +EXTERN Auto* curauto; +EXTERN Auto* curhist; +EXTERN Prog* curp; +EXTERN Sym* cursym; +EXTERN Sym* datap; +EXTERN int32 elfdatsize; +EXTERN char debug[128]; +EXTERN char literal[32]; +EXTERN Sym* etextp; +EXTERN Prog* firstp; +EXTERN uchar ycover[Ymax*Ymax]; +EXTERN uchar* andptr; +EXTERN uchar and[100]; +EXTERN char reg[D_NONE]; +EXTERN int32 lcsize; +EXTERN int maxop; +EXTERN int nerrors; +EXTERN char* noname; +EXTERN int32 pc; +EXTERN char* interpreter; +EXTERN char* rpath; +EXTERN int32 spsize; +EXTERN Sym* symlist; +EXTERN int32 symsize; +EXTERN Sym* textp; +EXTERN int32 textsize; +EXTERN int version; +EXTERN Prog zprg; +EXTERN int dtype; +EXTERN int tlsoffset; +EXTERN Sym* adrgotype; // type symbol on last Adr read +EXTERN Sym* fromgotype; // type symbol on last p->from read +EXTERN int elftextsh; + +extern Optab optab[]; +extern char* anames[]; + +int Aconv(Fmt*); +int Dconv(Fmt*); +int Iconv(Fmt*); +int Pconv(Fmt*); +int Rconv(Fmt*); +int Sconv(Fmt*); +void addhist(int32, int); +Prog* appendp(Prog*); +void asmb(void); +void asmdyn(void); +void asmins(Prog*); +void asmsym(void); +int32 atolwhex(char*); +Prog* brchain(Prog*); +Prog* brloop(Prog*); +void cflush(void); +Prog* copyp(Prog*); +vlong cpos(void); +double cputime(void); +void diag(char*, ...); +void dodata(void); +void doelf(void); +void doprof1(void); +void doprof2(void); +void dostkoff(void); +int32 entryvalue(void); +void follow(void); +void instinit(void); +void listinit(void); +Sym* lookup(char*, int); +void lputb(int32); +void lputl(int32); +void vputl(uint64); +void strnput(char*, int); +void main(int, char*[]); +void* mal(uint32); +int opsize(Prog*); +void patch(void); +Prog* prg(void); +int relinv(int); +int32 rnd(int32, int32); +void s8put(char*); +void span(void); +void undef(void); +int32 symaddr(Sym*); +void wput(ushort); +void wputl(ushort); +void xdefine(char*, int, int32); + +uint32 machheadr(void); +vlong addaddr(Sym *s, Sym *t); +vlong addsize(Sym *s, Sym *t); +vlong addstring(Sym *s, char *str); +vlong adduint16(Sym *s, uint16 v); +vlong adduint32(Sym *s, uint32 v); +vlong adduint64(Sym *s, uint64 v); +vlong adduint8(Sym *s, uint8 v); +vlong adduintxx(Sym *s, uint64 v, int wid); + +/* + * go.c + */ +void deadcode(void); + +/* Native is little-endian */ +#define LPUT(a) lputl(a) +#define WPUT(a) wputl(a) +#define VPUT(a) vputl(a) + +/* Used by ../ld/dwarf.c */ +enum +{ + DWARFREGSP = 4 +}; diff --git a/src/cmd/8l/list.c b/src/cmd/8l/list.c new file mode 100644 index 000000000..31ae02346 --- /dev/null +++ b/src/cmd/8l/list.c @@ -0,0 +1,369 @@ +// Inferno utils/8l/list.c +// http://code.google.com/p/inferno-os/source/browse/utils/8l/list.c +// +// Copyright © 1994-1999 Lucent Technologies Inc. All rights reserved. +// Portions Copyright © 1995-1997 C H Forsyth (forsyth@terzarima.net) +// Portions Copyright © 1997-1999 Vita Nuova Limited +// Portions Copyright © 2000-2007 Vita Nuova Holdings Limited (www.vitanuova.com) +// Portions Copyright © 2004,2006 Bruce Ellis +// Portions Copyright © 2005-2007 C H Forsyth (forsyth@terzarima.net) +// Revisions Copyright © 2000-2007 Lucent Technologies Inc. and others +// Portions Copyright © 2009 The Go Authors. All rights reserved. +// +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to deal +// in the Software without restriction, including without limitation the rights +// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +// copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +// THE SOFTWARE. + +// Printing. + +#include "l.h" +#include "../ld/lib.h" + +void +listinit(void) +{ + + fmtinstall('R', Rconv); + fmtinstall('A', Aconv); + fmtinstall('D', Dconv); + fmtinstall('S', Sconv); + fmtinstall('P', Pconv); + fmtinstall('I', Iconv); +} + +static Prog *bigP; + +int +Pconv(Fmt *fp) +{ + Prog *p; + + p = va_arg(fp->args, Prog*); + bigP = p; + switch(p->as) { + case ATEXT: + if(p->from.scale) { + fmtprint(fp, "(%d) %A %D,%d,%D", + p->line, p->as, &p->from, p->from.scale, &p->to); + break; + } + default: + fmtprint(fp, "(%d) %A %D,%D", + p->line, p->as, &p->from, &p->to); + break; + case ADATA: + case AINIT_: + case ADYNT_: + fmtprint(fp, "(%d) %A %D/%d,%D", + p->line, p->as, &p->from, p->from.scale, &p->to); + break; + } + bigP = P; + return 0; +} + +int +Aconv(Fmt *fp) +{ + int i; + + i = va_arg(fp->args, int); + return fmtstrcpy(fp, anames[i]); +} + +char* +xsymname(Sym *s) +{ + if(s == nil) + return "!!noname!!"; + return s->name; +} + +int +Dconv(Fmt *fp) +{ + char str[STRINGSZ], s[STRINGSZ]; + Adr *a; + int i; + + a = va_arg(fp->args, Adr*); + i = a->type; + if(i >= D_INDIR && i < 2*D_INDIR) { + if(a->offset) + snprint(str, sizeof str, "%d(%R)", a->offset, i-D_INDIR); + else + snprint(str, sizeof str, "(%R)", i-D_INDIR); + goto brk; + } + switch(i) { + + default: + snprint(str, sizeof str, "%R", i); + break; + + case D_NONE: + str[0] = 0; + break; + + case D_BRANCH: + if(bigP != P && bigP->pcond != P) + if(a->sym != S) + snprint(str, sizeof str, "%ux+%s", bigP->pcond->pc, + a->sym->name); + else + snprint(str, sizeof str, "%ux", bigP->pcond->pc); + else + snprint(str, sizeof str, "%d(PC)", a->offset); + break; + + case D_EXTERN: + snprint(str, sizeof str, "%s+%d(SB)", xsymname(a->sym), a->offset); + break; + + case D_STATIC: + snprint(str, sizeof str, "%s<%d>+%d(SB)", xsymname(a->sym), + a->sym->version, a->offset); + break; + + case D_AUTO: + snprint(str, sizeof str, "%s+%d(SP)", xsymname(a->sym), a->offset); + break; + + case D_PARAM: + if(a->sym) + snprint(str, sizeof str, "%s+%d(FP)", a->sym->name, a->offset); + else + snprint(str, sizeof str, "%d(FP)", a->offset); + break; + + case D_CONST: + snprint(str, sizeof str, "$%d", a->offset); + break; + + case D_CONST2: + snprint(str, sizeof str, "$%d-%d", a->offset, a->offset2); + break; + + case D_FCONST: + snprint(str, sizeof str, "$(%.8ux,%.8ux)", a->ieee.h, a->ieee.l); + break; + + case D_SCONST: + snprint(str, sizeof str, "$\"%S\"", a->scon); + break; + + case D_ADDR: + a->type = a->index; + a->index = D_NONE; + snprint(str, sizeof str, "$%D", a); + a->index = a->type; + a->type = D_ADDR; + goto conv; + } +brk: + if(a->index != D_NONE) { + sprint(s, "(%R*%d)", (int)a->index, a->scale); + strcat(str, s); + } +conv: + fmtstrcpy(fp, str); +// if(a->gotype) +// fmtprint(fp, "«%s»", a->gotype->name); + return 0; +} + +char* regstr[] = +{ + "AL", /* [D_AL] */ + "CL", + "DL", + "BL", + "AH", + "CH", + "DH", + "BH", + + "AX", /* [D_AX] */ + "CX", + "DX", + "BX", + "SP", + "BP", + "SI", + "DI", + + "F0", /* [D_F0] */ + "F1", + "F2", + "F3", + "F4", + "F5", + "F6", + "F7", + + "CS", /* [D_CS] */ + "SS", + "DS", + "ES", + "FS", + "GS", + + "GDTR", /* [D_GDTR] */ + "IDTR", /* [D_IDTR] */ + "LDTR", /* [D_LDTR] */ + "MSW", /* [D_MSW] */ + "TASK", /* [D_TASK] */ + + "CR0", /* [D_CR] */ + "CR1", + "CR2", + "CR3", + "CR4", + "CR5", + "CR6", + "CR7", + + "DR0", /* [D_DR] */ + "DR1", + "DR2", + "DR3", + "DR4", + "DR5", + "DR6", + "DR7", + + "TR0", /* [D_TR] */ + "TR1", + "TR2", + "TR3", + "TR4", + "TR5", + "TR6", + "TR7", + + "NONE", /* [D_NONE] */ +}; + +int +Rconv(Fmt *fp) +{ + char str[STRINGSZ]; + int r; + + r = va_arg(fp->args, int); + if(r >= D_AL && r <= D_NONE) + sprint(str, "%s", regstr[r-D_AL]); + else + sprint(str, "gok(%d)", r); + + return fmtstrcpy(fp, str); +} + +int +Sconv(Fmt *fp) +{ + int i, c; + char str[STRINGSZ], *p, *a; + + a = va_arg(fp->args, char*); + p = str; + for(i=0; i= 'a' && c <= 'z' || + c >= 'A' && c <= 'Z' || + c >= '0' && c <= '9') { + *p++ = c; + continue; + } + *p++ = '\\'; + switch(c) { + default: + if(c < 040 || c >= 0177) + break; /* not portable */ + p[-1] = c; + continue; + case 0: + *p++ = 'z'; + continue; + case '\\': + case '"': + *p++ = c; + continue; + case '\n': + *p++ = 'n'; + continue; + case '\t': + *p++ = 't'; + continue; + } + *p++ = (c>>6) + '0'; + *p++ = ((c>>3) & 7) + '0'; + *p++ = (c & 7) + '0'; + } + *p = 0; + return fmtstrcpy(fp, str); +} + +int +Iconv(Fmt *fp) +{ + int i, n; + uchar *p; + char *s; + Fmt fmt; + + n = fp->prec; + fp->prec = 0; + if(!(fp->flags&FmtPrec) || n < 0) + return fmtstrcpy(fp, "%I"); + fp->flags &= ~FmtPrec; + p = va_arg(fp->args, uchar*); + + // format into temporary buffer and + // call fmtstrcpy to handle padding. + fmtstrinit(&fmt); + for(i=0; iname; + sep = ": "; + } + va_start(arg, fmt); + vseprint(buf, buf+sizeof(buf), fmt, arg); + va_end(arg); + print("%s%s%s\n", tn, sep, buf); + + nerrors++; + if(nerrors > 20) { + print("too many errors\n"); + errorexit(); + } +} diff --git a/src/cmd/8l/mkenam b/src/cmd/8l/mkenam new file mode 100644 index 000000000..992aa3160 --- /dev/null +++ b/src/cmd/8l/mkenam @@ -0,0 +1,45 @@ +# Inferno utils/8c/mkenam +# http://code.google.com/p/inferno-os/source/browse/utils/8c/mkenam +# +# Copyright © 1994-1999 Lucent Technologies Inc. All rights reserved. +# Portions Copyright © 1995-1997 C H Forsyth (forsyth@terzarima.net) +# Portions Copyright © 1997-1999 Vita Nuova Limited +# Portions Copyright © 2000-2007 Vita Nuova Holdings Limited (www.vitanuova.com) +# Portions Copyright © 2004,2006 Bruce Ellis +# Portions Copyright © 2005-2007 C H Forsyth (forsyth@terzarima.net) +# Revisions Copyright © 2000-2007 Lucent Technologies Inc. and others +# Portions Copyright © 2009 The Go Authors. All rights reserved. +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in +# all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +# THE SOFTWARE. + +awk ' +BEGIN { + print "char* anames[] =" + print "{" +} + +/^ A/ { + name=$1 + sub(/,/, "", name) + sub(/^A/, "", name) + print "\t\"" name "\"," +} + +END { print "};" } +' ../8l/8.out.h >enam.c diff --git a/src/cmd/8l/obj.c b/src/cmd/8l/obj.c new file mode 100644 index 000000000..a8e1c34a5 --- /dev/null +++ b/src/cmd/8l/obj.c @@ -0,0 +1,739 @@ +// Inferno utils/8l/obj.c +// http://code.google.com/p/inferno-os/source/browse/utils/8l/obj.c +// +// Copyright © 1994-1999 Lucent Technologies Inc. All rights reserved. +// Portions Copyright © 1995-1997 C H Forsyth (forsyth@terzarima.net) +// Portions Copyright © 1997-1999 Vita Nuova Limited +// Portions Copyright © 2000-2007 Vita Nuova Holdings Limited (www.vitanuova.com) +// Portions Copyright © 2004,2006 Bruce Ellis +// Portions Copyright © 2005-2007 C H Forsyth (forsyth@terzarima.net) +// Revisions Copyright © 2000-2007 Lucent Technologies Inc. and others +// Portions Copyright © 2009 The Go Authors. All rights reserved. +// +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to deal +// in the Software without restriction, including without limitation the rights +// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +// copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +// THE SOFTWARE. + +// Reading object files. + +#define EXTERN +#include "l.h" +#include "../ld/lib.h" +#include "../ld/elf.h" +#include "../ld/macho.h" +#include "../ld/dwarf.h" +#include "../ld/pe.h" +#include + +#ifndef DEFAULT +#define DEFAULT '9' +#endif + +char *noname = ""; +char *thestring = "386"; + +Header headers[] = { + "garbunix", Hgarbunix, + "unixcoff", Hunixcoff, + "plan9", Hplan9x32, + "msdoscom", Hmsdoscom, + "msdosexe", Hmsdosexe, + "darwin", Hdarwin, + "linux", Hlinux, + "freebsd", Hfreebsd, + "windows", Hwindows, + "windowsgui", Hwindows, + 0, 0 +}; + +/* + * -Hgarbunix -T0x40004C -D0x10000000 is garbage unix + * -Hunixcoff -T0xd0 -R4 is unix coff + * -Hplan9 -T4128 -R4096 is plan9 format + * -Hmsdoscom -Tx -Rx is MS-DOS .COM + * -Hmsdosexe -Tx -Rx is fake MS-DOS .EXE + * -Hdarwin -Tx -Rx is Apple Mach-O + * -Hlinux -Tx -Rx is Linux ELF32 + * -Hfreebsd -Tx -Rx is FreeBSD ELF32 + * -Hwindows -Tx -Rx is MS Windows PE32 + */ + +void +usage(void) +{ + fprint(2, "usage: 8l [-options] [-E entry] [-H head] [-I interpreter] [-L dir] [-T text] [-R rnd] [-r path] [-o out] main.8\n"); + exits("usage"); +} + +void +main(int argc, char *argv[]) +{ + int c; + + Binit(&bso, 1, OWRITE); + listinit(); + memset(debug, 0, sizeof(debug)); + nerrors = 0; + outfile = nil; + HEADTYPE = -1; + INITTEXT = -1; + INITDAT = -1; + INITRND = -1; + INITENTRY = 0; + + ARGBEGIN { + default: + c = ARGC(); + if(c == 'l') + usage(); + if(c >= 0 && c < sizeof(debug)) + debug[c]++; + break; + case 'o': /* output to (next arg) */ + outfile = EARGF(usage()); + break; + case 'E': + INITENTRY = EARGF(usage()); + break; + case 'H': + HEADTYPE = headtype(EARGF(usage())); + break; + case 'I': + interpreter = EARGF(usage()); + break; + case 'L': + Lflag(EARGF(usage())); + break; + case 'T': + INITTEXT = atolwhex(EARGF(usage())); + break; + case 'D': + INITDAT = atolwhex(EARGF(usage())); + break; + case 'R': + INITRND = atolwhex(EARGF(usage())); + break; + case 'r': + rpath = EARGF(usage()); + break; + case 'V': + print("%cl version %s\n", thechar, getgoversion()); + errorexit(); + } ARGEND + + if(argc != 1) + usage(); + + mywhatsys(); // get goos + + if(HEADTYPE == -1) + HEADTYPE = headtype(goos); + + if(outfile == nil) { + if(HEADTYPE == Hwindows) + outfile = "8.out.exe"; + else + outfile = "8.out"; + } + + libinit(); + + switch(HEADTYPE) { + default: + diag("unknown -H option"); + errorexit(); + + case Hgarbunix: /* this is garbage */ + HEADR = 20L+56L; + if(INITTEXT == -1) + INITTEXT = 0x40004CL; + if(INITDAT == -1) + INITDAT = 0x10000000L; + if(INITRND == -1) + INITRND = 0; + break; + case Hunixcoff: /* is unix coff */ + HEADR = 0xd0L; + if(INITTEXT == -1) + INITTEXT = 0xd0; + if(INITDAT == -1) + INITDAT = 0x400000; + if(INITRND == -1) + INITRND = 0; + break; + case Hplan9x32: /* plan 9 */ + tlsoffset = -8; + HEADR = 32L; + if(INITTEXT == -1) + INITTEXT = 4096+32; + if(INITDAT == -1) + INITDAT = 0; + if(INITRND == -1) + INITRND = 4096; + break; + case Hmsdoscom: /* MS-DOS .COM */ + HEADR = 0; + if(INITTEXT == -1) + INITTEXT = 0x0100; + if(INITDAT == -1) + INITDAT = 0; + if(INITRND == -1) + INITRND = 4; + break; + case Hmsdosexe: /* fake MS-DOS .EXE */ + HEADR = 0x200; + if(INITTEXT == -1) + INITTEXT = 0x0100; + if(INITDAT == -1) + INITDAT = 0; + if(INITRND == -1) + INITRND = 4; + HEADR += (INITTEXT & 0xFFFF); + if(debug['v']) + Bprint(&bso, "HEADR = 0x%d\n", HEADR); + break; + case Hdarwin: /* apple MACH */ + /* + * OS X system constant - offset from %gs to our TLS. + * Explained in ../../libcgo/darwin_386.c. + */ + tlsoffset = 0x468; + machoinit(); + HEADR = INITIAL_MACHO_HEADR; + if(INITTEXT == -1) + INITTEXT = 4096+HEADR; + if(INITDAT == -1) + INITDAT = 0; + if(INITRND == -1) + INITRND = 4096; + break; + case Hlinux: /* elf32 executable */ + case Hfreebsd: + /* + * ELF uses TLS offsets negative from %gs. + * Translate 0(GS) and 4(GS) into -8(GS) and -4(GS). + * Also known to ../../pkg/runtime/linux/386/sys.s + * and ../../libcgo/linux_386.c. + */ + tlsoffset = -8; + elfinit(); + HEADR = ELFRESERVE; + if(INITTEXT == -1) + INITTEXT = 0x08048000+HEADR; + if(INITDAT == -1) + INITDAT = 0; + if(INITRND == -1) + INITRND = 4096; + break; + case Hwindows: /* PE executable */ + peinit(); + HEADR = PEFILEHEADR; + if(INITTEXT == -1) + INITTEXT = PEBASE+PESECTHEADR; + if(INITDAT == -1) + INITDAT = 0; + if(INITRND == -1) + INITRND = PESECTALIGN; + break; + } + if(INITDAT != 0 && INITRND != 0) + print("warning: -D0x%ux is ignored because of -R0x%ux\n", + INITDAT, INITRND); + if(debug['v']) + Bprint(&bso, "HEADER = -H0x%d -T0x%ux -D0x%ux -R0x%ux\n", + HEADTYPE, INITTEXT, INITDAT, INITRND); + Bflush(&bso); + + instinit(); + zprg.link = P; + zprg.pcond = P; + zprg.back = 2; + zprg.as = AGOK; + zprg.from.type = D_NONE; + zprg.from.index = D_NONE; + zprg.from.scale = 1; + zprg.to = zprg.from; + + pcstr = "%.6ux "; + nuxiinit(); + histgen = 0; + pc = 0; + dtype = 4; + version = 0; + cbp = buf.cbuf; + cbc = sizeof(buf.cbuf); + + addlibpath("command line", "command line", argv[0], "main"); + loadlib(); + deadcode(); + patch(); + follow(); + doelf(); + if(HEADTYPE == Hdarwin) + domacho(); + if(HEADTYPE == Hwindows) + dope(); + dostkoff(); + if(debug['p']) + if(debug['1']) + doprof1(); + else + doprof2(); + span(); + addexport(); + textaddress(); + pclntab(); + symtab(); + dodata(); + address(); + doweak(); + reloc(); + asmb(); + undef(); + if(debug['v']) { + Bprint(&bso, "%5.2f cpu time\n", cputime()); + Bprint(&bso, "%d symbols\n", nsymbol); + Bprint(&bso, "%d sizeof adr\n", sizeof(Adr)); + Bprint(&bso, "%d sizeof prog\n", sizeof(Prog)); + } + Bflush(&bso); + + errorexit(); +} + +static Sym* +zsym(char *pn, Biobuf *f, Sym *h[]) +{ + int o; + + o = Bgetc(f); + if(o < 0 || o >= NSYM || h[o] == nil) + mangle(pn); + return h[o]; +} + +static void +zaddr(char *pn, Biobuf *f, Adr *a, Sym *h[]) +{ + int t; + int32 l; + Sym *s; + Auto *u; + + t = Bgetc(f); + a->index = D_NONE; + a->scale = 0; + if(t & T_INDEX) { + a->index = Bgetc(f); + a->scale = Bgetc(f); + } + a->type = D_NONE; + a->offset = 0; + if(t & T_OFFSET) + a->offset = Bget4(f); + a->offset2 = 0; + if(t & T_OFFSET2) { + a->offset2 = Bget4(f); + a->type = D_CONST2; + } + a->sym = S; + if(t & T_SYM) + a->sym = zsym(pn, f, h); + if(t & T_FCONST) { + a->ieee.l = Bget4(f); + a->ieee.h = Bget4(f); + a->type = D_FCONST; + } else + if(t & T_SCONST) { + Bread(f, a->scon, NSNAME); + a->type = D_SCONST; + } + if(t & T_TYPE) + a->type = Bgetc(f); + adrgotype = S; + if(t & T_GOTYPE) + adrgotype = zsym(pn, f, h); + + t = a->type; + if(t == D_INDIR+D_GS) + a->offset += tlsoffset; + + s = a->sym; + if(s == S) + return; + if(t != D_AUTO && t != D_PARAM) { + if(adrgotype) + s->gotype = adrgotype; + return; + } + l = a->offset; + for(u=curauto; u; u=u->link) { + if(u->asym == s) + if(u->type == t) { + if(u->aoffset > l) + u->aoffset = l; + if(adrgotype) + u->gotype = adrgotype; + return; + } + } + + u = mal(sizeof(*u)); + u->link = curauto; + curauto = u; + u->asym = s; + u->aoffset = l; + u->type = t; + u->gotype = adrgotype; +} + +void +nopout(Prog *p) +{ + p->as = ANOP; + p->from.type = D_NONE; + p->to.type = D_NONE; +} + +void +ldobj1(Biobuf *f, char *pkg, int64 len, char *pn) +{ + int32 ipc; + Prog *p; + int v, o, r, skip; + Sym *h[NSYM], *s; + uint32 sig; + int ntext; + int32 eof; + char *name, *x; + char src[1024]; + Prog *lastp; + + lastp = nil; + ntext = 0; + eof = Boffset(f) + len; + src[0] = 0; + + +newloop: + memset(h, 0, sizeof(h)); + version++; + histfrogp = 0; + ipc = pc; + skip = 0; + +loop: + if(f->state == Bracteof || Boffset(f) >= eof) + goto eof; + o = Bgetc(f); + if(o == Beof) + goto eof; + o |= Bgetc(f) << 8; + if(o <= AXXX || o >= ALAST) { + if(o < 0) + goto eof; + diag("%s:#%lld: opcode out of range: %#ux", pn, Boffset(f), o); + print(" probably not a .%c file\n", thechar); + errorexit(); + } + + if(o == ANAME || o == ASIGNAME) { + sig = 0; + if(o == ASIGNAME) + sig = Bget4(f); + v = Bgetc(f); /* type */ + o = Bgetc(f); /* sym */ + r = 0; + if(v == D_STATIC) + r = version; + name = Brdline(f, '\0'); + if(name == nil) { + if(Blinelen(f) > 0) { + fprint(2, "%s: name too long\n", pn); + errorexit(); + } + goto eof; + } + x = expandpkg(name, pkg); + s = lookup(x, r); + if(x != name) + free(x); + + if(debug['S'] && r == 0) + sig = 1729; + if(sig != 0){ + if(s->sig != 0 && s->sig != sig) + diag("incompatible type signatures" + "%ux(%s) and %ux(%s) for %s", + s->sig, s->file, sig, pn, s->name); + s->sig = sig; + s->file = pn; + } + + if(debug['W']) + print(" ANAME %s\n", s->name); + if(o < 0 || o >= nelem(h)) + mangle(pn); + h[o] = s; + if((v == D_EXTERN || v == D_STATIC) && s->type == 0) + s->type = SXREF; + if(v == D_FILE) { + if(s->type != SFILE) { + histgen++; + s->type = SFILE; + s->value = histgen; + } + if(histfrogp < MAXHIST) { + histfrog[histfrogp] = s; + histfrogp++; + } else + collapsefrog(s); + dwarfaddfrag(s->value, s->name); + } + goto loop; + } + + p = mal(sizeof(*p)); + p->as = o; + p->line = Bget4(f); + p->back = 2; + p->ft = 0; + p->tt = 0; + zaddr(pn, f, &p->from, h); + fromgotype = adrgotype; + zaddr(pn, f, &p->to, h); + + if(debug['W']) + print("%P\n", p); + + switch(p->as) { + case AHISTORY: + if(p->to.offset == -1) { + addlib(src, pn); + histfrogp = 0; + goto loop; + } + if(src[0] == '\0') + copyhistfrog(src, sizeof src); + addhist(p->line, D_FILE); /* 'z' */ + if(p->to.offset) + addhist(p->to.offset, D_FILE1); /* 'Z' */ + histfrogp = 0; + goto loop; + + case AEND: + histtoauto(); + if(cursym != nil && cursym->text) + cursym->autom = curauto; + curauto = 0; + cursym = nil; + if(Boffset(f) == eof) + return; + goto newloop; + + case AGLOBL: + s = p->from.sym; + if(s->type == 0 || s->type == SXREF) { + s->type = SBSS; + s->size = 0; + } + if(s->type != SBSS && !s->dupok) { + diag("%s: redefinition: %s in %s", + pn, s->name, TNAME); + s->type = SBSS; + s->size = 0; + } + if(p->to.offset > s->size) + s->size = p->to.offset; + if(p->from.scale & DUPOK) + s->dupok = 1; + if(p->from.scale & RODATA) + s->type = SRODATA; + goto loop; + + case ADATA: + // Assume that AGLOBL comes after ADATA. + // If we've seen an AGLOBL that said this sym was DUPOK, + // ignore any more ADATA we see, which must be + // redefinitions. + s = p->from.sym; + if(s->dupok) { +// if(debug['v']) +// Bprint(&bso, "skipping %s in %s: dupok\n", s->name, pn); + goto loop; + } + if(s->file == nil) + s->file = pn; + else if(s->file != pn) { + diag("multiple initialization for %s: in both %s and %s", s->name, s->file, pn); + errorexit(); + } + savedata(s, p, pn); + unmal(p, sizeof *p); + goto loop; + + case AGOK: + diag("%s: GOK opcode in %s", pn, TNAME); + pc++; + goto loop; + + case ATEXT: + s = p->from.sym; + if(s->text != nil) { + diag("%s: %s: redefinition", pn, s->name); + return; + } + if(ntext++ == 0 && s->type != 0 && s->type != SXREF) { + /* redefinition, so file has probably been seen before */ + if(debug['v']) + diag("skipping: %s: redefinition: %s", pn, s->name); + return; + } + if(cursym != nil && cursym->text) { + histtoauto(); + cursym->autom = curauto; + curauto = 0; + } + skip = 0; + if(etextp) + etextp->next = s; + else + textp = s; + etextp = s; + s->text = p; + cursym = s; + if(s->type != 0 && s->type != SXREF) { + if(p->from.scale & DUPOK) { + skip = 1; + goto casdef; + } + diag("%s: redefinition: %s\n%P", pn, s->name, p); + } + s->type = STEXT; + s->value = pc; + lastp = p; + p->pc = pc++; + goto loop; + + case AFMOVF: + case AFADDF: + case AFSUBF: + case AFSUBRF: + case AFMULF: + case AFDIVF: + case AFDIVRF: + case AFCOMF: + case AFCOMFP: + if(skip) + goto casdef; + if(p->from.type == D_FCONST) { + /* size sb 9 max */ + sprint(literal, "$%ux", ieeedtof(&p->from.ieee)); + s = lookup(literal, 0); + if(s->type == 0) { + s->type = SDATA; + adduint32(s, ieeedtof(&p->from.ieee)); + s->reachable = 0; + } + p->from.type = D_EXTERN; + p->from.sym = s; + p->from.offset = 0; + } + goto casdef; + + case AFMOVD: + case AFADDD: + case AFSUBD: + case AFSUBRD: + case AFMULD: + case AFDIVD: + case AFDIVRD: + case AFCOMD: + case AFCOMDP: + if(skip) + goto casdef; + if(p->from.type == D_FCONST) { + /* size sb 18 max */ + sprint(literal, "$%ux.%ux", + p->from.ieee.l, p->from.ieee.h); + s = lookup(literal, 0); + if(s->type == 0) { + s->type = SDATA; + adduint32(s, p->from.ieee.l); + adduint32(s, p->from.ieee.h); + s->reachable = 0; + } + p->from.type = D_EXTERN; + p->from.sym = s; + p->from.offset = 0; + } + goto casdef; + + casdef: + default: + if(skip) + nopout(p); + p->pc = pc; + pc++; + + if(p->to.type == D_BRANCH) + p->to.offset += ipc; + if(lastp == nil) { + if(p->as != ANOP) + diag("unexpected instruction: %P", p); + goto loop; + } + lastp->link = p; + lastp = p; + goto loop; + } + +eof: + diag("truncated object file: %s", pn); +} + +Prog* +prg(void) +{ + Prog *p; + + p = mal(sizeof(Prog)); + *p = zprg; + return p; +} + +Prog* +copyp(Prog *q) +{ + Prog *p; + + p = prg(); + *p = *q; + return p; +} + +Prog* +appendp(Prog *q) +{ + Prog *p; + + p = prg(); + p->link = q->link; + q->link = p; + p->line = q->line; + return p; +} diff --git a/src/cmd/8l/optab.c b/src/cmd/8l/optab.c new file mode 100644 index 000000000..f5c195d75 --- /dev/null +++ b/src/cmd/8l/optab.c @@ -0,0 +1,755 @@ +// Inferno utils/8l/optab.c +// http://code.google.com/p/inferno-os/source/browse/utils/8l/optab.c +// +// Copyright © 1994-1999 Lucent Technologies Inc. All rights reserved. +// Portions Copyright © 1995-1997 C H Forsyth (forsyth@terzarima.net) +// Portions Copyright © 1997-1999 Vita Nuova Limited +// Portions Copyright © 2000-2007 Vita Nuova Holdings Limited (www.vitanuova.com) +// Portions Copyright © 2004,2006 Bruce Ellis +// Portions Copyright © 2005-2007 C H Forsyth (forsyth@terzarima.net) +// Revisions Copyright © 2000-2007 Lucent Technologies Inc. and others +// Portions Copyright © 2009 The Go Authors. All rights reserved. +// +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to deal +// in the Software without restriction, including without limitation the rights +// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +// copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +// THE SOFTWARE. + +#include "l.h" + +uchar ynone[] = +{ + Ynone, Ynone, Zlit, 1, + 0 +}; +uchar ytext[] = +{ + Ymb, Yi32, Zpseudo,1, + 0 +}; +uchar ynop[] = +{ + Ynone, Ynone, Zpseudo,1, + Ynone, Yml, Zpseudo,1, + Ynone, Yrf, Zpseudo,1, + Yml, Ynone, Zpseudo,1, + Yrf, Ynone, Zpseudo,1, + 0 +}; +uchar yxorb[] = +{ + Yi32, Yal, Zib_, 1, + Yi32, Ymb, Zibo_m, 2, + Yrb, Ymb, Zr_m, 1, + Ymb, Yrb, Zm_r, 1, + 0 +}; +uchar yxorl[] = +{ + Yi8, Yml, Zibo_m, 2, + Yi32, Yax, Zil_, 1, + Yi32, Yml, Zilo_m, 2, + Yrl, Yml, Zr_m, 1, + Yml, Yrl, Zm_r, 1, + 0 +}; +uchar yaddl[] = +{ + Yi8, Yml, Zibo_m, 2, + Yi32, Yax, Zil_, 1, + Yi32, Yml, Zilo_m, 2, + Yrl, Yml, Zr_m, 1, + Yml, Yrl, Zm_r, 1, + 0 +}; +uchar yincb[] = +{ + Ynone, Ymb, Zo_m, 2, + 0 +}; +uchar yincl[] = +{ + Ynone, Yrl, Z_rp, 1, + Ynone, Yml, Zo_m, 2, + 0 +}; +uchar ycmpb[] = +{ + Yal, Yi32, Z_ib, 1, + Ymb, Yi32, Zm_ibo, 2, + Ymb, Yrb, Zm_r, 1, + Yrb, Ymb, Zr_m, 1, + 0 +}; +uchar ycmpl[] = +{ + Yml, Yi8, Zm_ibo, 2, + Yax, Yi32, Z_il, 1, + Yml, Yi32, Zm_ilo, 2, + Yml, Yrl, Zm_r, 1, + Yrl, Yml, Zr_m, 1, + 0 +}; +uchar yshb[] = +{ + Yi1, Ymb, Zo_m, 2, + Yi32, Ymb, Zibo_m, 2, + Ycx, Ymb, Zo_m, 2, + 0 +}; +uchar yshl[] = +{ + Yi1, Yml, Zo_m, 2, + Yi32, Yml, Zibo_m, 2, + Ycl, Yml, Zo_m, 2, + Ycx, Yml, Zo_m, 2, + 0 +}; +uchar ytestb[] = +{ + Yi32, Yal, Zib_, 1, + Yi32, Ymb, Zibo_m, 2, + Yrb, Ymb, Zr_m, 1, + Ymb, Yrb, Zm_r, 1, + 0 +}; +uchar ytestl[] = +{ + Yi32, Yax, Zil_, 1, + Yi32, Yml, Zilo_m, 2, + Yrl, Yml, Zr_m, 1, + Yml, Yrl, Zm_r, 1, + 0 +}; +uchar ymovb[] = +{ + Yrb, Ymb, Zr_m, 1, + Ymb, Yrb, Zm_r, 1, + Yi32, Yrb, Zib_rp, 1, + Yi32, Ymb, Zibo_m, 2, + 0 +}; +uchar ymovl[] = +{ + Yrl, Yml, Zr_m, 1, + Yml, Yrl, Zm_r, 1, + Yi0, Yrl, Zclr, 1+2, +// Yi0, Yml, Zibo_m, 2, // shorter but slower AND $0,dst + Yi32, Yrl, Zil_rp, 1, + Yi32, Yml, Zilo_m, 2, + Yiauto, Yrl, Zaut_r, 2, + 0 +}; +uchar ym_rl[] = +{ + Ym, Yrl, Zm_r, 1, + 0 +}; +uchar yrl_m[] = +{ + Yrl, Ym, Zr_m, 1, + 0 +}; +uchar ymb_rl[] = +{ + Ymb, Yrl, Zm_r, 1, + 0 +}; +uchar yml_rl[] = +{ + Yml, Yrl, Zm_r, 1, + 0 +}; +uchar yrb_mb[] = +{ + Yrb, Ymb, Zr_m, 1, + 0 +}; +uchar yrl_ml[] = +{ + Yrl, Yml, Zr_m, 1, + 0 +}; +uchar yml_mb[] = +{ + Yrb, Ymb, Zr_m, 1, + Ymb, Yrb, Zm_r, 1, + 0 +}; +uchar yml_ml[] = +{ + Yrl, Yml, Zr_m, 1, + Yml, Yrl, Zm_r, 1, + 0 +}; +uchar ydivl[] = +{ + Yml, Ynone, Zm_o, 2, + 0 +}; +uchar ydivb[] = +{ + Ymb, Ynone, Zm_o, 2, + 0 +}; +uchar yimul[] = +{ + Yml, Ynone, Zm_o, 2, + Yi8, Yrl, Zib_rr, 1, + Yi32, Yrl, Zil_rr, 1, + 0 +}; +uchar ybyte[] = +{ + Yi32, Ynone, Zbyte, 1, + 0 +}; +uchar yin[] = +{ + Yi32, Ynone, Zib_, 1, + Ynone, Ynone, Zlit, 1, + 0 +}; +uchar yint[] = +{ + Yi32, Ynone, Zib_, 1, + 0 +}; +uchar ypushl[] = +{ + Yrl, Ynone, Zrp_, 1, + Ym, Ynone, Zm_o, 2, + Yi8, Ynone, Zib_, 1, + Yi32, Ynone, Zil_, 1, + 0 +}; +uchar ypopl[] = +{ + Ynone, Yrl, Z_rp, 1, + Ynone, Ym, Zo_m, 2, + 0 +}; +uchar yscond[] = +{ + Ynone, Ymb, Zo_m, 2, + 0 +}; +uchar yjcond[] = +{ + Ynone, Ybr, Zbr, 1, + 0 +}; +uchar yloop[] = +{ + Ynone, Ybr, Zloop, 1, + 0 +}; +uchar ycall[] = +{ + Ynone, Yml, Zo_m, 2, + Ynone, Ybr, Zcall, 0, + Ynone, Yi32, Zcallcon, 1, + 0 +}; +uchar yjmp[] = +{ + Ynone, Yml, Zo_m, 2, + Ynone, Ybr, Zjmp, 0, + Ynone, Yi32, Zjmpcon, 1, + 0 +}; + +uchar yfmvd[] = +{ + Ym, Yf0, Zm_o, 2, + Yf0, Ym, Zo_m, 2, + Yrf, Yf0, Zm_o, 2, + Yf0, Yrf, Zo_m, 2, + 0 +}; +uchar yfmvdp[] = +{ + Yf0, Ym, Zo_m, 2, + Yf0, Yrf, Zo_m, 2, + 0 +}; +uchar yfmvf[] = +{ + Ym, Yf0, Zm_o, 2, + Yf0, Ym, Zo_m, 2, + 0 +}; +uchar yfmvx[] = +{ + Ym, Yf0, Zm_o, 2, + 0 +}; +uchar yfmvp[] = +{ + Yf0, Ym, Zo_m, 2, + 0 +}; +uchar yfcmv[] = +{ + Yrf, Yf0, Zm_o, 2, + 0 +}; +uchar yfadd[] = +{ + Ym, Yf0, Zm_o, 2, + Yrf, Yf0, Zm_o, 2, + Yf0, Yrf, Zo_m, 2, + 0 +}; +uchar yfaddp[] = +{ + Yf0, Yrf, Zo_m, 2, + 0 +}; +uchar yfxch[] = +{ + Yf0, Yrf, Zo_m, 2, + Yrf, Yf0, Zm_o, 2, + 0 +}; +uchar ycompp[] = +{ + Yf0, Yrf, Zo_m, 2, /* botch is really f0,f1 */ + 0 +}; +uchar ystsw[] = +{ + Ynone, Ym, Zo_m, 2, + Ynone, Yax, Zlit, 1, + 0 +}; +uchar ystcw[] = +{ + Ynone, Ym, Zo_m, 2, + Ym, Ynone, Zm_o, 2, + 0 +}; +uchar ysvrs[] = +{ + Ynone, Ym, Zo_m, 2, + Ym, Ynone, Zm_o, 2, + 0 +}; + +Optab optab[] = +/* as, ytab, andproto, opcode */ +{ + { AXXX }, + { AAAA, ynone, Px, 0x37 }, + { AAAD, ynone, Px, 0xd5,0x0a }, + { AAAM, ynone, Px, 0xd4,0x0a }, + { AAAS, ynone, Px, 0x3f }, + { AADCB, yxorb, Pb, 0x14,0x80,(02),0x10,0x10 }, + { AADCL, yxorl, Px, 0x83,(02),0x15,0x81,(02),0x11,0x13 }, + { AADCW, yxorl, Pe, 0x83,(02),0x15,0x81,(02),0x11,0x13 }, + { AADDB, yxorb, Px, 0x04,0x80,(00),0x00,0x02 }, + { AADDL, yaddl, Px, 0x83,(00),0x05,0x81,(00),0x01,0x03 }, + { AADDW, yaddl, Pe, 0x83,(00),0x05,0x81,(00),0x01,0x03 }, + { AADJSP }, + { AANDB, yxorb, Pb, 0x24,0x80,(04),0x20,0x22 }, + { AANDL, yxorl, Px, 0x83,(04),0x25,0x81,(04),0x21,0x23 }, + { AANDW, yxorl, Pe, 0x83,(04),0x25,0x81,(04),0x21,0x23 }, + { AARPL, yrl_ml, Px, 0x63 }, + { ABOUNDL, yrl_m, Px, 0x62 }, + { ABOUNDW, yrl_m, Pe, 0x62 }, + { ABSFL, yml_rl, Pm, 0xbc }, + { ABSFW, yml_rl, Pq, 0xbc }, + { ABSRL, yml_rl, Pm, 0xbd }, + { ABSRW, yml_rl, Pq, 0xbd }, + { ABTL, yml_rl, Pm, 0xa3 }, + { ABTW, yml_rl, Pq, 0xa3 }, + { ABTCL, yml_rl, Pm, 0xbb }, + { ABTCW, yml_rl, Pq, 0xbb }, + { ABTRL, yml_rl, Pm, 0xb3 }, + { ABTRW, yml_rl, Pq, 0xb3 }, + { ABTSL, yml_rl, Pm, 0xab }, + { ABTSW, yml_rl, Pq, 0xab }, + { ABYTE, ybyte, Px, 1 }, + { ACALL, ycall, Px, 0xff,(02),0xe8 }, + { ACLC, ynone, Px, 0xf8 }, + { ACLD, ynone, Px, 0xfc }, + { ACLI, ynone, Px, 0xfa }, + { ACLTS, ynone, Pm, 0x06 }, + { ACMC, ynone, Px, 0xf5 }, + { ACMPB, ycmpb, Pb, 0x3c,0x80,(07),0x38,0x3a }, + { ACMPL, ycmpl, Px, 0x83,(07),0x3d,0x81,(07),0x39,0x3b }, + { ACMPW, ycmpl, Pe, 0x83,(07),0x3d,0x81,(07),0x39,0x3b }, + { ACMPSB, ynone, Pb, 0xa6 }, + { ACMPSL, ynone, Px, 0xa7 }, + { ACMPSW, ynone, Pe, 0xa7 }, + { ADAA, ynone, Px, 0x27 }, + { ADAS, ynone, Px, 0x2f }, + { ADATA }, + { ADECB, yincb, Pb, 0xfe,(01) }, + { ADECL, yincl, Px, 0x48,0xff,(01) }, + { ADECW, yincl, Pe, 0x48,0xff,(01) }, + { ADIVB, ydivb, Pb, 0xf6,(06) }, + { ADIVL, ydivl, Px, 0xf7,(06) }, + { ADIVW, ydivl, Pe, 0xf7,(06) }, + { AENTER }, /* botch */ + { AGLOBL }, + { AGOK }, + { AHISTORY }, + { AHLT, ynone, Px, 0xf4 }, + { AIDIVB, ydivb, Pb, 0xf6,(07) }, + { AIDIVL, ydivl, Px, 0xf7,(07) }, + { AIDIVW, ydivl, Pe, 0xf7,(07) }, + { AIMULB, ydivb, Pb, 0xf6,(05) }, + { AIMULL, yimul, Px, 0xf7,(05),0x6b,0x69 }, + { AIMULW, yimul, Pe, 0xf7,(05),0x6b,0x69 }, + { AINB, yin, Pb, 0xe4,0xec }, + { AINL, yin, Px, 0xe5,0xed }, + { AINW, yin, Pe, 0xe5,0xed }, + { AINCB, yincb, Pb, 0xfe,(00) }, + { AINCL, yincl, Px, 0x40,0xff,(00) }, + { AINCW, yincl, Pe, 0x40,0xff,(00) }, + { AINSB, ynone, Pb, 0x6c }, + { AINSL, ynone, Px, 0x6d }, + { AINSW, ynone, Pe, 0x6d }, + { AINT, yint, Px, 0xcd }, + { AINTO, ynone, Px, 0xce }, + { AIRETL, ynone, Px, 0xcf }, + { AIRETW, ynone, Pe, 0xcf }, + { AJCC, yjcond, Px, 0x73,0x83,(00) }, + { AJCS, yjcond, Px, 0x72,0x82 }, + { AJCXZ, yloop, Px, 0xe3 }, + { AJEQ, yjcond, Px, 0x74,0x84 }, + { AJGE, yjcond, Px, 0x7d,0x8d }, + { AJGT, yjcond, Px, 0x7f,0x8f }, + { AJHI, yjcond, Px, 0x77,0x87 }, + { AJLE, yjcond, Px, 0x7e,0x8e }, + { AJLS, yjcond, Px, 0x76,0x86 }, + { AJLT, yjcond, Px, 0x7c,0x8c }, + { AJMI, yjcond, Px, 0x78,0x88 }, + { AJMP, yjmp, Px, 0xff,(04),0xeb,0xe9 }, + { AJNE, yjcond, Px, 0x75,0x85 }, + { AJOC, yjcond, Px, 0x71,0x81,(00) }, + { AJOS, yjcond, Px, 0x70,0x80,(00) }, + { AJPC, yjcond, Px, 0x7b,0x8b }, + { AJPL, yjcond, Px, 0x79,0x89 }, + { AJPS, yjcond, Px, 0x7a,0x8a }, + { ALAHF, ynone, Px, 0x9f }, + { ALARL, yml_rl, Pm, 0x02 }, + { ALARW, yml_rl, Pq, 0x02 }, + { ALEAL, ym_rl, Px, 0x8d }, + { ALEAW, ym_rl, Pe, 0x8d }, + { ALEAVEL, ynone, Px, 0xc9 }, + { ALEAVEW, ynone, Pe, 0xc9 }, + { ALOCK, ynone, Px, 0xf0 }, + { ALODSB, ynone, Pb, 0xac }, + { ALODSL, ynone, Px, 0xad }, + { ALODSW, ynone, Pe, 0xad }, + { ALONG, ybyte, Px, 4 }, + { ALOOP, yloop, Px, 0xe2 }, + { ALOOPEQ, yloop, Px, 0xe1 }, + { ALOOPNE, yloop, Px, 0xe0 }, + { ALSLL, yml_rl, Pm, 0x03 }, + { ALSLW, yml_rl, Pq, 0x03 }, + { AMOVB, ymovb, Pb, 0x88,0x8a,0xb0,0xc6,(00) }, + { AMOVL, ymovl, Px, 0x89,0x8b,0x31,0x83,(04),0xb8,0xc7,(00) }, + { AMOVW, ymovl, Pe, 0x89,0x8b,0x31,0x83,(04),0xb8,0xc7,(00) }, + { AMOVBLSX, ymb_rl, Pm, 0xbe }, + { AMOVBLZX, ymb_rl, Pm, 0xb6 }, + { AMOVBWSX, ymb_rl, Pq, 0xbe }, + { AMOVBWZX, ymb_rl, Pq, 0xb6 }, + { AMOVWLSX, yml_rl, Pm, 0xbf }, + { AMOVWLZX, yml_rl, Pm, 0xb7 }, + { AMOVSB, ynone, Pb, 0xa4 }, + { AMOVSL, ynone, Px, 0xa5 }, + { AMOVSW, ynone, Pe, 0xa5 }, + { AMULB, ydivb, Pb, 0xf6,(04) }, + { AMULL, ydivl, Px, 0xf7,(04) }, + { AMULW, ydivl, Pe, 0xf7,(04) }, + { ANAME }, + { ANEGB, yscond, Px, 0xf6,(03) }, + { ANEGL, yscond, Px, 0xf7,(03) }, + { ANEGW, yscond, Pe, 0xf7,(03) }, + { ANOP, ynop, Px,0,0 }, + { ANOTB, yscond, Px, 0xf6,(02) }, + { ANOTL, yscond, Px, 0xf7,(02) }, + { ANOTW, yscond, Pe, 0xf7,(02) }, + { AORB, yxorb, Pb, 0x0c,0x80,(01),0x08,0x0a }, + { AORL, yxorl, Px, 0x83,(01),0x0d,0x81,(01),0x09,0x0b }, + { AORW, yxorl, Pe, 0x83,(01),0x0d,0x81,(01),0x09,0x0b }, + { AOUTB, yin, Pb, 0xe6,0xee }, + { AOUTL, yin, Px, 0xe7,0xef }, + { AOUTW, yin, Pe, 0xe7,0xef }, + { AOUTSB, ynone, Pb, 0x6e }, + { AOUTSL, ynone, Px, 0x6f }, + { AOUTSW, ynone, Pe, 0x6f }, + { APAUSE, ynone, Px, 0xf3,0x90 }, + { APOPAL, ynone, Px, 0x61 }, + { APOPAW, ynone, Pe, 0x61 }, + { APOPFL, ynone, Px, 0x9d }, + { APOPFW, ynone, Pe, 0x9d }, + { APOPL, ypopl, Px, 0x58,0x8f,(00) }, + { APOPW, ypopl, Pe, 0x58,0x8f,(00) }, + { APUSHAL, ynone, Px, 0x60 }, + { APUSHAW, ynone, Pe, 0x60 }, + { APUSHFL, ynone, Px, 0x9c }, + { APUSHFW, ynone, Pe, 0x9c }, + { APUSHL, ypushl, Px, 0x50,0xff,(06),0x6a,0x68 }, + { APUSHW, ypushl, Pe, 0x50,0xff,(06),0x6a,0x68 }, + { ARCLB, yshb, Pb, 0xd0,(02),0xc0,(02),0xd2,(02) }, + { ARCLL, yshl, Px, 0xd1,(02),0xc1,(02),0xd3,(02),0xd3,(02) }, + { ARCLW, yshl, Pe, 0xd1,(02),0xc1,(02),0xd3,(02),0xd3,(02) }, + { ARCRB, yshb, Pb, 0xd0,(03),0xc0,(03),0xd2,(03) }, + { ARCRL, yshl, Px, 0xd1,(03),0xc1,(03),0xd3,(03),0xd3,(03) }, + { ARCRW, yshl, Pe, 0xd1,(03),0xc1,(03),0xd3,(03),0xd3,(03) }, + { AREP, ynone, Px, 0xf3 }, + { AREPN, ynone, Px, 0xf2 }, + { ARET, ynone, Px, 0xc3 }, + { AROLB, yshb, Pb, 0xd0,(00),0xc0,(00),0xd2,(00) }, + { AROLL, yshl, Px, 0xd1,(00),0xc1,(00),0xd3,(00),0xd3,(00) }, + { AROLW, yshl, Pe, 0xd1,(00),0xc1,(00),0xd3,(00),0xd3,(00) }, + { ARORB, yshb, Pb, 0xd0,(01),0xc0,(01),0xd2,(01) }, + { ARORL, yshl, Px, 0xd1,(01),0xc1,(01),0xd3,(01),0xd3,(01) }, + { ARORW, yshl, Pe, 0xd1,(01),0xc1,(01),0xd3,(01),0xd3,(01) }, + { ASAHF, ynone, Px, 0x9e }, + { ASALB, yshb, Pb, 0xd0,(04),0xc0,(04),0xd2,(04) }, + { ASALL, yshl, Px, 0xd1,(04),0xc1,(04),0xd3,(04),0xd3,(04) }, + { ASALW, yshl, Pe, 0xd1,(04),0xc1,(04),0xd3,(04),0xd3,(04) }, + { ASARB, yshb, Pb, 0xd0,(07),0xc0,(07),0xd2,(07) }, + { ASARL, yshl, Px, 0xd1,(07),0xc1,(07),0xd3,(07),0xd3,(07) }, + { ASARW, yshl, Pe, 0xd1,(07),0xc1,(07),0xd3,(07),0xd3,(07) }, + { ASBBB, yxorb, Pb, 0x1c,0x80,(03),0x18,0x1a }, + { ASBBL, yxorl, Px, 0x83,(03),0x1d,0x81,(03),0x19,0x1b }, + { ASBBW, yxorl, Pe, 0x83,(03),0x1d,0x81,(03),0x19,0x1b }, + { ASCASB, ynone, Pb, 0xae }, + { ASCASL, ynone, Px, 0xaf }, + { ASCASW, ynone, Pe, 0xaf }, + { ASETCC, yscond, Pm, 0x93,(00) }, + { ASETCS, yscond, Pm, 0x92,(00) }, + { ASETEQ, yscond, Pm, 0x94,(00) }, + { ASETGE, yscond, Pm, 0x9d,(00) }, + { ASETGT, yscond, Pm, 0x9f,(00) }, + { ASETHI, yscond, Pm, 0x97,(00) }, + { ASETLE, yscond, Pm, 0x9e,(00) }, + { ASETLS, yscond, Pm, 0x96,(00) }, + { ASETLT, yscond, Pm, 0x9c,(00) }, + { ASETMI, yscond, Pm, 0x98,(00) }, + { ASETNE, yscond, Pm, 0x95,(00) }, + { ASETOC, yscond, Pm, 0x91,(00) }, + { ASETOS, yscond, Pm, 0x90,(00) }, + { ASETPC, yscond, Pm, 0x96,(00) }, + { ASETPL, yscond, Pm, 0x99,(00) }, + { ASETPS, yscond, Pm, 0x9a,(00) }, + { ACDQ, ynone, Px, 0x99 }, + { ACWD, ynone, Pe, 0x99 }, + { ASHLB, yshb, Pb, 0xd0,(04),0xc0,(04),0xd2,(04) }, + { ASHLL, yshl, Px, 0xd1,(04),0xc1,(04),0xd3,(04),0xd3,(04) }, + { ASHLW, yshl, Pe, 0xd1,(04),0xc1,(04),0xd3,(04),0xd3,(04) }, + { ASHRB, yshb, Pb, 0xd0,(05),0xc0,(05),0xd2,(05) }, + { ASHRL, yshl, Px, 0xd1,(05),0xc1,(05),0xd3,(05),0xd3,(05) }, + { ASHRW, yshl, Pe, 0xd1,(05),0xc1,(05),0xd3,(05),0xd3,(05) }, + { ASTC, ynone, Px, 0xf9 }, + { ASTD, ynone, Px, 0xfd }, + { ASTI, ynone, Px, 0xfb }, + { ASTOSB, ynone, Pb, 0xaa }, + { ASTOSL, ynone, Px, 0xab }, + { ASTOSW, ynone, Pe, 0xab }, + { ASUBB, yxorb, Pb, 0x2c,0x80,(05),0x28,0x2a }, + { ASUBL, yaddl, Px, 0x83,(05),0x2d,0x81,(05),0x29,0x2b }, + { ASUBW, yaddl, Pe, 0x83,(05),0x2d,0x81,(05),0x29,0x2b }, + { ASYSCALL, ynone, Px, 0xcd,100 }, + { ATESTB, ytestb, Pb, 0xa8,0xf6,(00),0x84,0x84 }, + { ATESTL, ytestl, Px, 0xa9,0xf7,(00),0x85,0x85 }, + { ATESTW, ytestl, Pe, 0xa9,0xf7,(00),0x85,0x85 }, + { ATEXT, ytext, Px }, + { AVERR, ydivl, Pm, 0x00,(04) }, + { AVERW, ydivl, Pm, 0x00,(05) }, + { AWAIT, ynone, Px, 0x9b }, + { AWORD, ybyte, Px, 2 }, + { AXCHGB, yml_mb, Pb, 0x86,0x86 }, + { AXCHGL, yml_ml, Px, 0x87,0x87 }, + { AXCHGW, yml_ml, Pe, 0x87,0x87 }, + { AXLAT, ynone, Px, 0xd7 }, + { AXORB, yxorb, Pb, 0x34,0x80,(06),0x30,0x32 }, + { AXORL, yxorl, Px, 0x83,(06),0x35,0x81,(06),0x31,0x33 }, + { AXORW, yxorl, Pe, 0x83,(06),0x35,0x81,(06),0x31,0x33 }, + + { AFMOVB, yfmvx, Px, 0xdf,(04) }, + { AFMOVBP, yfmvp, Px, 0xdf,(06) }, + { AFMOVD, yfmvd, Px, 0xdd,(00),0xdd,(02),0xd9,(00),0xdd,(02) }, + { AFMOVDP, yfmvdp, Px, 0xdd,(03),0xdd,(03) }, + { AFMOVF, yfmvf, Px, 0xd9,(00),0xd9,(02) }, + { AFMOVFP, yfmvp, Px, 0xd9,(03) }, + { AFMOVL, yfmvf, Px, 0xdb,(00),0xdb,(02) }, + { AFMOVLP, yfmvp, Px, 0xdb,(03) }, + { AFMOVV, yfmvx, Px, 0xdf,(05) }, + { AFMOVVP, yfmvp, Px, 0xdf,(07) }, + { AFMOVW, yfmvf, Px, 0xdf,(00),0xdf,(02) }, + { AFMOVWP, yfmvp, Px, 0xdf,(03) }, + { AFMOVX, yfmvx, Px, 0xdb,(05) }, + { AFMOVXP, yfmvp, Px, 0xdb,(07) }, + + { AFCOMB }, + { AFCOMBP }, + { AFCOMD, yfadd, Px, 0xdc,(02),0xd8,(02),0xdc,(02) }, /* botch */ + { AFCOMDP, yfadd, Px, 0xdc,(03),0xd8,(03),0xdc,(03) }, /* botch */ + { AFCOMDPP, ycompp, Px, 0xde,(03) }, + { AFCOMF, yfmvx, Px, 0xd8,(02) }, + { AFCOMFP, yfmvx, Px, 0xd8,(03) }, + { AFCOMI, yfmvx, Px, 0xdb,(06) }, + { AFCOMIP, yfmvx, Px, 0xdf,(06) }, + { AFCOML, yfmvx, Px, 0xda,(02) }, + { AFCOMLP, yfmvx, Px, 0xda,(03) }, + { AFCOMW, yfmvx, Px, 0xde,(02) }, + { AFCOMWP, yfmvx, Px, 0xde,(03) }, + + { AFUCOM, ycompp, Px, 0xdd,(04) }, + { AFUCOMI, ycompp, Px, 0xdb,(05) }, + { AFUCOMIP, ycompp, Px, 0xdf,(05) }, + { AFUCOMP, ycompp, Px, 0xdd,(05) }, + { AFUCOMPP, ycompp, Px, 0xda,(13) }, + + { AFADDDP, yfaddp, Px, 0xde,(00) }, + { AFADDW, yfmvx, Px, 0xde,(00) }, + { AFADDL, yfmvx, Px, 0xda,(00) }, + { AFADDF, yfmvx, Px, 0xd8,(00) }, + { AFADDD, yfadd, Px, 0xdc,(00),0xd8,(00),0xdc,(00) }, + + { AFMULDP, yfaddp, Px, 0xde,(01) }, + { AFMULW, yfmvx, Px, 0xde,(01) }, + { AFMULL, yfmvx, Px, 0xda,(01) }, + { AFMULF, yfmvx, Px, 0xd8,(01) }, + { AFMULD, yfadd, Px, 0xdc,(01),0xd8,(01),0xdc,(01) }, + + { AFSUBDP, yfaddp, Px, 0xde,(05) }, + { AFSUBW, yfmvx, Px, 0xde,(04) }, + { AFSUBL, yfmvx, Px, 0xda,(04) }, + { AFSUBF, yfmvx, Px, 0xd8,(04) }, + { AFSUBD, yfadd, Px, 0xdc,(04),0xd8,(04),0xdc,(05) }, + + { AFSUBRDP, yfaddp, Px, 0xde,(04) }, + { AFSUBRW, yfmvx, Px, 0xde,(05) }, + { AFSUBRL, yfmvx, Px, 0xda,(05) }, + { AFSUBRF, yfmvx, Px, 0xd8,(05) }, + { AFSUBRD, yfadd, Px, 0xdc,(05),0xd8,(05),0xdc,(04) }, + + { AFDIVDP, yfaddp, Px, 0xde,(07) }, + { AFDIVW, yfmvx, Px, 0xde,(06) }, + { AFDIVL, yfmvx, Px, 0xda,(06) }, + { AFDIVF, yfmvx, Px, 0xd8,(06) }, + { AFDIVD, yfadd, Px, 0xdc,(06),0xd8,(06),0xdc,(07) }, + + { AFDIVRDP, yfaddp, Px, 0xde,(06) }, + { AFDIVRW, yfmvx, Px, 0xde,(07) }, + { AFDIVRL, yfmvx, Px, 0xda,(07) }, + { AFDIVRF, yfmvx, Px, 0xd8,(07) }, + { AFDIVRD, yfadd, Px, 0xdc,(07),0xd8,(07),0xdc,(06) }, + + { AFXCHD, yfxch, Px, 0xd9,(01),0xd9,(01) }, + { AFFREE }, + { AFLDCW, ystcw, Px, 0xd9,(05),0xd9,(05) }, + { AFLDENV, ystcw, Px, 0xd9,(04),0xd9,(04) }, + { AFRSTOR, ysvrs, Px, 0xdd,(04),0xdd,(04) }, + { AFSAVE, ysvrs, Px, 0xdd,(06),0xdd,(06) }, + { AFSTCW, ystcw, Px, 0xd9,(07),0xd9,(07) }, + { AFSTENV, ystcw, Px, 0xd9,(06),0xd9,(06) }, + { AFSTSW, ystsw, Px, 0xdd,(07),0xdf,0xe0 }, + { AF2XM1, ynone, Px, 0xd9, 0xf0 }, + { AFABS, ynone, Px, 0xd9, 0xe1 }, + { AFCHS, ynone, Px, 0xd9, 0xe0 }, + { AFCLEX, ynone, Px, 0xdb, 0xe2 }, + { AFCOS, ynone, Px, 0xd9, 0xff }, + { AFDECSTP, ynone, Px, 0xd9, 0xf6 }, + { AFINCSTP, ynone, Px, 0xd9, 0xf7 }, + { AFINIT, ynone, Px, 0xdb, 0xe3 }, + { AFLD1, ynone, Px, 0xd9, 0xe8 }, + { AFLDL2E, ynone, Px, 0xd9, 0xea }, + { AFLDL2T, ynone, Px, 0xd9, 0xe9 }, + { AFLDLG2, ynone, Px, 0xd9, 0xec }, + { AFLDLN2, ynone, Px, 0xd9, 0xed }, + { AFLDPI, ynone, Px, 0xd9, 0xeb }, + { AFLDZ, ynone, Px, 0xd9, 0xee }, + { AFNOP, ynone, Px, 0xd9, 0xd0 }, + { AFPATAN, ynone, Px, 0xd9, 0xf3 }, + { AFPREM, ynone, Px, 0xd9, 0xf8 }, + { AFPREM1, ynone, Px, 0xd9, 0xf5 }, + { AFPTAN, ynone, Px, 0xd9, 0xf2 }, + { AFRNDINT, ynone, Px, 0xd9, 0xfc }, + { AFSCALE, ynone, Px, 0xd9, 0xfd }, + { AFSIN, ynone, Px, 0xd9, 0xfe }, + { AFSINCOS, ynone, Px, 0xd9, 0xfb }, + { AFSQRT, ynone, Px, 0xd9, 0xfa }, + { AFTST, ynone, Px, 0xd9, 0xe4 }, + { AFXAM, ynone, Px, 0xd9, 0xe5 }, + { AFXTRACT, ynone, Px, 0xd9, 0xf4 }, + { AFYL2X, ynone, Px, 0xd9, 0xf1 }, + { AFYL2XP1, ynone, Px, 0xd9, 0xf9 }, + { AEND }, + { ADYNT_ }, + { AINIT_ }, + { ASIGNAME }, + { ACMPXCHGB, yrb_mb, Pm, 0xb0 }, + { ACMPXCHGL, yrl_ml, Pm, 0xb1 }, + { ACMPXCHGW, yrl_ml, Pm, 0xb1 }, + { ACMPXCHG8B, yscond, Pm, 0xc7,(01) }, + + { AXADDB, yrb_mb, Pb, 0x0f,0xc0 }, + { AXADDL, yrl_ml, Pm, 0xc1 }, + { AXADDW, yrl_ml, Pe, 0x0f,0xc1 }, + + { ACMOVLCC, yml_rl, Pm, 0x43 }, + { ACMOVLCS, yml_rl, Pm, 0x42 }, + { ACMOVLEQ, yml_rl, Pm, 0x44 }, + { ACMOVLGE, yml_rl, Pm, 0x4d }, + { ACMOVLGT, yml_rl, Pm, 0x4f }, + { ACMOVLHI, yml_rl, Pm, 0x47 }, + { ACMOVLLE, yml_rl, Pm, 0x4e }, + { ACMOVLLS, yml_rl, Pm, 0x46 }, + { ACMOVLLT, yml_rl, Pm, 0x4c }, + { ACMOVLMI, yml_rl, Pm, 0x48 }, + { ACMOVLNE, yml_rl, Pm, 0x45 }, + { ACMOVLOC, yml_rl, Pm, 0x41 }, + { ACMOVLOS, yml_rl, Pm, 0x40 }, + { ACMOVLPC, yml_rl, Pm, 0x4b }, + { ACMOVLPL, yml_rl, Pm, 0x49 }, + { ACMOVLPS, yml_rl, Pm, 0x4a }, + { ACMOVWCC, yml_rl, Pq, 0x43 }, + { ACMOVWCS, yml_rl, Pq, 0x42 }, + { ACMOVWEQ, yml_rl, Pq, 0x44 }, + { ACMOVWGE, yml_rl, Pq, 0x4d }, + { ACMOVWGT, yml_rl, Pq, 0x4f }, + { ACMOVWHI, yml_rl, Pq, 0x47 }, + { ACMOVWLE, yml_rl, Pq, 0x4e }, + { ACMOVWLS, yml_rl, Pq, 0x46 }, + { ACMOVWLT, yml_rl, Pq, 0x4c }, + { ACMOVWMI, yml_rl, Pq, 0x48 }, + { ACMOVWNE, yml_rl, Pq, 0x45 }, + { ACMOVWOC, yml_rl, Pq, 0x41 }, + { ACMOVWOS, yml_rl, Pq, 0x40 }, + { ACMOVWPC, yml_rl, Pq, 0x4b }, + { ACMOVWPL, yml_rl, Pq, 0x49 }, + { ACMOVWPS, yml_rl, Pq, 0x4a }, + + { AFCMOVCC, yfcmv, Px, 0xdb,(00) }, + { AFCMOVCS, yfcmv, Px, 0xda,(00) }, + { AFCMOVEQ, yfcmv, Px, 0xda,(01) }, + { AFCMOVHI, yfcmv, Px, 0xdb,(02) }, + { AFCMOVLS, yfcmv, Px, 0xda,(02) }, + { AFCMOVNE, yfcmv, Px, 0xdb,(01) }, + { AFCMOVNU, yfcmv, Px, 0xdb,(03) }, + { AFCMOVUN, yfcmv, Px, 0xda,(03) }, + + 0 +}; diff --git a/src/cmd/8l/pass.c b/src/cmd/8l/pass.c new file mode 100644 index 000000000..2e0990c5a --- /dev/null +++ b/src/cmd/8l/pass.c @@ -0,0 +1,657 @@ +// Inferno utils/8l/pass.c +// http://code.google.com/p/inferno-os/source/browse/utils/8l/pass.c +// +// Copyright © 1994-1999 Lucent Technologies Inc. All rights reserved. +// Portions Copyright © 1995-1997 C H Forsyth (forsyth@terzarima.net) +// Portions Copyright © 1997-1999 Vita Nuova Limited +// Portions Copyright © 2000-2007 Vita Nuova Holdings Limited (www.vitanuova.com) +// Portions Copyright © 2004,2006 Bruce Ellis +// Portions Copyright © 2005-2007 C H Forsyth (forsyth@terzarima.net) +// Revisions Copyright © 2000-2007 Lucent Technologies Inc. and others +// Portions Copyright © 2009 The Go Authors. All rights reserved. +// +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to deal +// in the Software without restriction, including without limitation the rights +// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +// copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +// THE SOFTWARE. + +// Code and data passes. + +#include "l.h" +#include "../ld/lib.h" +#include "../../pkg/runtime/stack.h" + +static void xfol(Prog*, Prog**); + +Prog* +brchain(Prog *p) +{ + int i; + + for(i=0; i<20; i++) { + if(p == P || p->as != AJMP) + return p; + p = p->pcond; + } + return P; +} + +void +follow(void) +{ + Prog *firstp, *lastp; + + if(debug['v']) + Bprint(&bso, "%5.2f follow\n", cputime()); + Bflush(&bso); + + for(cursym = textp; cursym != nil; cursym = cursym->next) { + firstp = prg(); + lastp = firstp; + xfol(cursym->text, &lastp); + lastp->link = nil; + cursym->text = firstp->link; + } +} + +static int +nofollow(int a) +{ + switch(a) { + case AJMP: + case ARET: + case AIRETL: + case AIRETW: + return 1; + } + return 0; +} + +static int +pushpop(int a) +{ + switch(a) { + case APUSHL: + case APUSHFL: + case APUSHW: + case APUSHFW: + case APOPL: + case APOPFL: + case APOPW: + case APOPFW: + return 1; + } + return 0; +} + +static void +xfol(Prog *p, Prog **last) +{ + Prog *q; + int i; + enum as a; + +loop: + if(p == P) + return; + if(p->as == AJMP) + if((q = p->pcond) != P && q->as != ATEXT) { + /* mark instruction as done and continue layout at target of jump */ + p->mark = 1; + p = q; + if(p->mark == 0) + goto loop; + } + if(p->mark) { + /* + * p goes here, but already used it elsewhere. + * copy up to 4 instructions or else branch to other copy. + */ + for(i=0,q=p; i<4; i++,q=q->link) { + if(q == P) + break; + if(q == *last) + break; + a = q->as; + if(a == ANOP) { + i--; + continue; + } + if(nofollow(a) || pushpop(a)) + break; // NOTE(rsc): arm does goto copy + if(q->pcond == P || q->pcond->mark) + continue; + if(a == ACALL || a == ALOOP) + continue; + for(;;) { + if(p->as == ANOP) { + p = p->link; + continue; + } + q = copyp(p); + p = p->link; + q->mark = 1; + (*last)->link = q; + *last = q; + if(q->as != a || q->pcond == P || q->pcond->mark) + continue; + + q->as = relinv(q->as); + p = q->pcond; + q->pcond = q->link; + q->link = p; + xfol(q->link, last); + p = q->link; + if(p->mark) + return; + goto loop; + } + } /* */ + q = prg(); + q->as = AJMP; + q->line = p->line; + q->to.type = D_BRANCH; + q->to.offset = p->pc; + q->pcond = p; + p = q; + } + + /* emit p */ + p->mark = 1; + (*last)->link = p; + *last = p; + a = p->as; + + /* continue loop with what comes after p */ + if(nofollow(a)) + return; + if(p->pcond != P && a != ACALL) { + /* + * some kind of conditional branch. + * recurse to follow one path. + * continue loop on the other. + */ + q = brchain(p->link); + if(q != P && q->mark) + if(a != ALOOP) { + p->as = relinv(a); + p->link = p->pcond; + p->pcond = q; + } + xfol(p->link, last); + q = brchain(p->pcond); + if(q->mark) { + p->pcond = q; + return; + } + p = q; + goto loop; + } + p = p->link; + goto loop; +} + +int +relinv(int a) +{ + + switch(a) { + case AJEQ: return AJNE; + case AJNE: return AJEQ; + case AJLE: return AJGT; + case AJLS: return AJHI; + case AJLT: return AJGE; + case AJMI: return AJPL; + case AJGE: return AJLT; + case AJPL: return AJMI; + case AJGT: return AJLE; + case AJHI: return AJLS; + case AJCS: return AJCC; + case AJCC: return AJCS; + case AJPS: return AJPC; + case AJPC: return AJPS; + case AJOS: return AJOC; + case AJOC: return AJOS; + } + diag("unknown relation: %s in %s", anames[a], TNAME); + return a; +} + +void +patch(void) +{ + int32 c; + Prog *p, *q; + Sym *s; + int32 vexit; + Sym *plan9_tos; + + if(debug['v']) + Bprint(&bso, "%5.2f mkfwd\n", cputime()); + Bflush(&bso); + mkfwd(); + if(debug['v']) + Bprint(&bso, "%5.2f patch\n", cputime()); + Bflush(&bso); + s = lookup("exit", 0); + vexit = s->value; + + plan9_tos = S; + if(HEADTYPE == Hplan9x32) + plan9_tos = lookup("_tos", 0); + + for(cursym = textp; cursym != nil; cursym = cursym->next) { + for(p = cursym->text; p != P; p = p->link) { + if(HEADTYPE == Hwindows) { + // Convert + // op n(GS), reg + // to + // MOVL 0x2C(FS), reg + // op n(reg), reg + // The purpose of this patch is to fix some accesses + // to extern register variables (TLS) on Windows, as + // a different method is used to access them. + if(p->from.type == D_INDIR+D_GS + && p->to.type >= D_AX && p->to.type <= D_DI) { + q = appendp(p); + q->from = p->from; + q->from.type = D_INDIR + p->to.type; + q->to = p->to; + q->as = p->as; + p->as = AMOVL; + p->from.type = D_INDIR+D_FS; + p->from.offset = 0x2C; + } + } + if(HEADTYPE == Hlinux) { + // Running binaries under Xen requires using + // MOVL 0(GS), reg + // and then off(reg) instead of saying off(GS) directly + // when the offset is negative. + if(p->from.type == D_INDIR+D_GS && p->from.offset < 0 + && p->to.type >= D_AX && p->to.type <= D_DI) { + q = appendp(p); + q->from = p->from; + q->from.type = D_INDIR + p->to.type; + q->to = p->to; + q->as = p->as; + p->as = AMOVL; + p->from.type = D_INDIR+D_GS; + p->from.offset = 0; + } + } + if(HEADTYPE == Hplan9x32) { + if(p->from.type == D_INDIR+D_GS + && p->to.type >= D_AX && p->to.type <= D_DI) { + q = appendp(p); + q->from = p->from; + q->from.type = D_INDIR + p->to.type; + q->to = p->to; + q->as = p->as; + p->as = AMOVL; + p->from.type = D_EXTERN; + p->from.sym = plan9_tos; + p->from.offset = 0; + } + } + if(p->as == ACALL || (p->as == AJMP && p->to.type != D_BRANCH)) { + s = p->to.sym; + if(s) { + if(debug['c']) + Bprint(&bso, "%s calls %s\n", TNAME, s->name); + if((s->type&~SSUB) != STEXT) { + /* diag prints TNAME first */ + diag("undefined: %s", s->name); + s->type = STEXT; + s->value = vexit; + continue; // avoid more error messages + } + if(s->text == nil) + continue; + p->to.type = D_BRANCH; + p->to.offset = s->text->pc; + p->pcond = s->text; + continue; + } + } + if(p->to.type != D_BRANCH) + continue; + c = p->to.offset; + for(q = cursym->text; q != P;) { + if(c == q->pc) + break; + if(q->forwd != P && c >= q->forwd->pc) + q = q->forwd; + else + q = q->link; + } + if(q == P) { + diag("branch out of range in %s (%#ux)\n%P [%s]", + TNAME, c, p, p->to.sym ? p->to.sym->name : ""); + p->to.type = D_NONE; + } + p->pcond = q; + } + } + + for(cursym = textp; cursym != nil; cursym = cursym->next) { + if(cursym->text == nil || cursym->p != nil) + continue; + + for(p = cursym->text; p != P; p = p->link) { + p->mark = 0; /* initialization for follow */ + if(p->pcond != P) { + p->pcond = brloop(p->pcond); + if(p->pcond != P) + if(p->to.type == D_BRANCH) + p->to.offset = p->pcond->pc; + } + } + } +} + +Prog* +brloop(Prog *p) +{ + int c; + Prog *q; + + c = 0; + for(q = p; q != P; q = q->pcond) { + if(q->as != AJMP) + break; + c++; + if(c >= 5000) + return P; + } + return q; +} + +void +dostkoff(void) +{ + Prog *p, *q, *q1; + int32 autoffset, deltasp; + int a; + Prog *pmorestack; + Sym *symmorestack; + Sym *plan9_tos; + + pmorestack = P; + symmorestack = lookup("runtime.morestack", 0); + + if(symmorestack->type != STEXT) + diag("runtime.morestack not defined"); + else { + pmorestack = symmorestack->text; + symmorestack->text->from.scale |= NOSPLIT; + } + + plan9_tos = S; + if(HEADTYPE == Hplan9x32) + plan9_tos = lookup("_tos", 0); + + for(cursym = textp; cursym != nil; cursym = cursym->next) { + if(cursym->text == nil || cursym->text->link == nil) + continue; + + p = cursym->text; + autoffset = p->to.offset; + if(autoffset < 0) + autoffset = 0; + + q = P; + if(pmorestack != P) + if(!(p->from.scale & NOSPLIT)) { + p = appendp(p); // load g into CX + switch(HEADTYPE) { + case Hwindows: + p->as = AMOVL; + p->from.type = D_INDIR+D_FS; + p->from.offset = 0x2c; + p->to.type = D_CX; + + p = appendp(p); + p->as = AMOVL; + p->from.type = D_INDIR+D_CX; + p->from.offset = 0; + p->to.type = D_CX; + break; + + case Hlinux: + p->as = AMOVL; + p->from.type = D_INDIR+D_GS; + p->from.offset = 0; + p->to.type = D_CX; + + p = appendp(p); + p->as = AMOVL; + p->from.type = D_INDIR+D_CX; + p->from.offset = tlsoffset + 0; + p->to.type = D_CX; + break; + + case Hplan9x32: + p->as = AMOVL; + p->from.type = D_EXTERN; + p->from.sym = plan9_tos; + p->to.type = D_CX; + + p = appendp(p); + p->as = AMOVL; + p->from.type = D_INDIR+D_CX; + p->from.offset = tlsoffset + 0; + p->to.type = D_CX; + break; + + default: + p->as = AMOVL; + p->from.type = D_INDIR+D_GS; + p->from.offset = tlsoffset + 0; + p->to.type = D_CX; + } + + if(debug['K']) { + // 8l -K means check not only for stack + // overflow but stack underflow. + // On underflow, INT 3 (breakpoint). + // Underflow itself is rare but this also + // catches out-of-sync stack guard info. + p = appendp(p); + p->as = ACMPL; + p->from.type = D_INDIR+D_CX; + p->from.offset = 4; + p->to.type = D_SP; + + p = appendp(p); + p->as = AJCC; + p->to.type = D_BRANCH; + p->to.offset = 4; + q1 = p; + + p = appendp(p); + p->as = AINT; + p->from.type = D_CONST; + p->from.offset = 3; + + p = appendp(p); + p->as = ANOP; + q1->pcond = p; + } + + if(autoffset < StackBig) { // do we need to call morestack + if(autoffset <= StackSmall) { + // small stack + p = appendp(p); + p->as = ACMPL; + p->from.type = D_SP; + p->to.type = D_INDIR+D_CX; + } else { + // large stack + p = appendp(p); + p->as = ALEAL; + p->from.type = D_INDIR+D_SP; + p->from.offset = -(autoffset-StackSmall); + p->to.type = D_AX; + + p = appendp(p); + p->as = ACMPL; + p->from.type = D_AX; + p->to.type = D_INDIR+D_CX; + } + + // common + p = appendp(p); + p->as = AJHI; + p->to.type = D_BRANCH; + p->to.offset = 4; + q = p; + } + + p = appendp(p); // save frame size in DX + p->as = AMOVL; + p->to.type = D_DX; + /* 160 comes from 3 calls (3*8) 4 safes (4*8) and 104 guard */ + p->from.type = D_CONST; + if(autoffset+160+cursym->text->to.offset2 > 4096) + p->from.offset = (autoffset+160) & ~7LL; + + p = appendp(p); // save arg size in AX + p->as = AMOVL; + p->to.type = D_AX; + p->from.type = D_CONST; + p->from.offset = cursym->text->to.offset2; + + p = appendp(p); + p->as = ACALL; + p->to.type = D_BRANCH; + p->pcond = pmorestack; + p->to.sym = symmorestack; + + } + + if(q != P) + q->pcond = p->link; + + if(autoffset) { + p = appendp(p); + p->as = AADJSP; + p->from.type = D_CONST; + p->from.offset = autoffset; + p->spadj = autoffset; + if(q != P) + q->pcond = p; + } + deltasp = autoffset; + + for(; p != P; p = p->link) { + a = p->from.type; + if(a == D_AUTO) + p->from.offset += deltasp; + if(a == D_PARAM) + p->from.offset += deltasp + 4; + a = p->to.type; + if(a == D_AUTO) + p->to.offset += deltasp; + if(a == D_PARAM) + p->to.offset += deltasp + 4; + + switch(p->as) { + default: + continue; + case APUSHL: + case APUSHFL: + deltasp += 4; + p->spadj = 4; + continue; + case APUSHW: + case APUSHFW: + deltasp += 2; + p->spadj = 2; + continue; + case APOPL: + case APOPFL: + deltasp -= 4; + p->spadj = -4; + continue; + case APOPW: + case APOPFW: + deltasp -= 2; + p->spadj = -2; + continue; + case ARET: + break; + } + + if(autoffset != deltasp) + diag("unbalanced PUSH/POP"); + + if(autoffset) { + p->as = AADJSP; + p->from.type = D_CONST; + p->from.offset = -autoffset; + p->spadj = -autoffset; + p = appendp(p); + p->as = ARET; + // If there are instructions following + // this ARET, they come from a branch + // with the same stackframe, so undo + // the cleanup. + p->spadj = +autoffset; + } + } + } +} + +int32 +atolwhex(char *s) +{ + int32 n; + int f; + + n = 0; + f = 0; + while(*s == ' ' || *s == '\t') + s++; + if(*s == '-' || *s == '+') { + if(*s++ == '-') + f = 1; + while(*s == ' ' || *s == '\t') + s++; + } + if(s[0]=='0' && s[1]){ + if(s[1]=='x' || s[1]=='X'){ + s += 2; + for(;;){ + if(*s >= '0' && *s <= '9') + n = n*16 + *s++ - '0'; + else if(*s >= 'a' && *s <= 'f') + n = n*16 + *s++ - 'a' + 10; + else if(*s >= 'A' && *s <= 'F') + n = n*16 + *s++ - 'A' + 10; + else + break; + } + } else + while(*s >= '0' && *s <= '7') + n = n*8 + *s++ - '0'; + } else + while(*s >= '0' && *s <= '9') + n = n*10 + *s++ - '0'; + if(f) + n = -n; + return n; +} diff --git a/src/cmd/8l/prof.c b/src/cmd/8l/prof.c new file mode 100644 index 000000000..d99c5e408 --- /dev/null +++ b/src/cmd/8l/prof.c @@ -0,0 +1,173 @@ +// Inferno utils/8l/obj.c +// http://code.google.com/p/inferno-os/source/browse/utils/8l/obj.c +// +// Copyright © 1994-1999 Lucent Technologies Inc. All rights reserved. +// Portions Copyright © 1995-1997 C H Forsyth (forsyth@terzarima.net) +// Portions Copyright © 1997-1999 Vita Nuova Limited +// Portions Copyright © 2000-2007 Vita Nuova Holdings Limited (www.vitanuova.com) +// Portions Copyright © 2004,2006 Bruce Ellis +// Portions Copyright © 2005-2007 C H Forsyth (forsyth@terzarima.net) +// Revisions Copyright © 2000-2007 Lucent Technologies Inc. and others +// Portions Copyright © 2009 The Go Authors. All rights reserved. +// +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to deal +// in the Software without restriction, including without limitation the rights +// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +// copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +// THE SOFTWARE. + +// Profiling. + +#include "l.h" +#include "../ld/lib.h" + +void +doprof1(void) +{ +#ifdef NOTDEF // TODO(rsc) + Sym *s; + int32 n; + Prog *p, *q; + + if(debug['v']) + Bprint(&bso, "%5.2f profile 1\n", cputime()); + Bflush(&bso); + s = lookup("__mcount", 0); + n = 1; + for(p = firstp->link; p != P; p = p->link) { + if(p->as == ATEXT) { + q = prg(); + q->line = p->line; + q->link = datap; + datap = q; + q->as = ADATA; + q->from.type = D_EXTERN; + q->from.offset = n*4; + q->from.sym = s; + q->from.scale = 4; + q->to = p->from; + q->to.type = D_CONST; + + q = prg(); + q->line = p->line; + q->pc = p->pc; + q->link = p->link; + p->link = q; + p = q; + p->as = AADDL; + p->from.type = D_CONST; + p->from.offset = 1; + p->to.type = D_EXTERN; + p->to.sym = s; + p->to.offset = n*4 + 4; + + n += 2; + continue; + } + } + q = prg(); + q->line = 0; + q->link = datap; + datap = q; + + q->as = ADATA; + q->from.type = D_EXTERN; + q->from.sym = s; + q->from.scale = 4; + q->to.type = D_CONST; + q->to.offset = n; + + s->type = SBSS; + s->size = n*4; +#endif +} + +void +doprof2(void) +{ + Sym *s2, *s4; + Prog *p, *q, *ps2, *ps4; + + if(debug['v']) + Bprint(&bso, "%5.2f profile 2\n", cputime()); + Bflush(&bso); + + s2 = lookup("_profin", 0); + s4 = lookup("_profout", 0); + if(s2->type != STEXT || s4->type != STEXT) { + diag("_profin/_profout not defined"); + return; + } + + ps2 = P; + ps4 = P; + for(cursym = textp; cursym != nil; cursym = cursym->next) { + p = cursym->text; + if(p->from.sym == s2) { + p->from.scale = 1; + ps2 = p; + } + if(p->from.sym == s4) { + p->from.scale = 1; + ps4 = p; + } + } + for(cursym = textp; cursym != nil; cursym = cursym->next) { + p = cursym->text; + + if(p->from.scale & NOPROF) /* dont profile */ + continue; + + /* + * JMPL profin + */ + q = prg(); + q->line = p->line; + q->pc = p->pc; + q->link = p->link; + p->link = q; + p = q; + p->as = ACALL; + p->to.type = D_BRANCH; + p->pcond = ps2; + p->to.sym = s2; + + for(; p; p=p->link) { + if(p->as == ARET) { + /* + * RET + */ + q = prg(); + q->as = ARET; + q->from = p->from; + q->to = p->to; + q->link = p->link; + p->link = q; + + /* + * JAL profout + */ + p->as = ACALL; + p->from = zprg.from; + p->to = zprg.to; + p->to.type = D_BRANCH; + p->pcond = ps4; + p->to.sym = s4; + + p = q; + } + } + } +} diff --git a/src/cmd/8l/span.c b/src/cmd/8l/span.c new file mode 100644 index 000000000..a4cba1257 --- /dev/null +++ b/src/cmd/8l/span.c @@ -0,0 +1,1325 @@ +// Inferno utils/8l/span.c +// http://code.google.com/p/inferno-os/source/browse/utils/8l/span.c +// +// Copyright © 1994-1999 Lucent Technologies Inc. All rights reserved. +// Portions Copyright © 1995-1997 C H Forsyth (forsyth@terzarima.net) +// Portions Copyright © 1997-1999 Vita Nuova Limited +// Portions Copyright © 2000-2007 Vita Nuova Holdings Limited (www.vitanuova.com) +// Portions Copyright © 2004,2006 Bruce Ellis +// Portions Copyright © 2005-2007 C H Forsyth (forsyth@terzarima.net) +// Revisions Copyright © 2000-2007 Lucent Technologies Inc. and others +// Portions Copyright © 2009 The Go Authors. All rights reserved. +// +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to deal +// in the Software without restriction, including without limitation the rights +// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +// copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +// THE SOFTWARE. + +// Instruction layout. + +#include "l.h" +#include "../ld/lib.h" + +static int32 vaddr(Adr*, Reloc*); + +void +span1(Sym *s) +{ + Prog *p, *q; + int32 c, v, loop; + uchar *bp; + int n, m, i; + + cursym = s; + + for(p = s->text; p != P; p = p->link) { + p->back = 2; // use short branches first time through + if((q = p->pcond) != P && (q->back & 2)) + p->back |= 1; // backward jump + + if(p->as == AADJSP) { + p->to.type = D_SP; + v = -p->from.offset; + p->from.offset = v; + p->as = AADDL; + if(v < 0) { + p->as = ASUBL; + v = -v; + p->from.offset = v; + } + if(v == 0) + p->as = ANOP; + } + } + + n = 0; + do { + loop = 0; + memset(s->r, 0, s->nr*sizeof s->r[0]); + s->nr = 0; + s->np = 0; + c = 0; + for(p = s->text; p != P; p = p->link) { + p->pc = c; + + // process forward jumps to p + for(q = p->comefrom; q != P; q = q->forwd) { + v = p->pc - (q->pc + q->mark); + if(q->back & 2) { // short + if(v > 127) { + loop++; + q->back ^= 2; + } + s->p[q->pc+1] = v; + } else { + bp = s->p + q->pc + q->mark - 4; + *bp++ = v; + *bp++ = v>>8; + *bp++ = v>>16; + *bp = v>>24; + } + } + p->comefrom = P; + + asmins(p); + p->pc = c; + m = andptr-and; + symgrow(s, p->pc+m); + memmove(s->p+p->pc, and, m); + p->mark = m; + c += m; + } + if(++n > 20) { + diag("span must be looping"); + errorexit(); + } + } while(loop); + s->size = c; + + if(debug['a'] > 1) { + print("span1 %s %d (%d tries)\n %.6ux", s->name, s->size, n, 0); + for(i=0; inp; i++) { + print(" %.2ux", s->p[i]); + if(i%16 == 15) + print("\n %.6ux", i+1); + } + if(i%16) + print("\n"); + + for(i=0; inr; i++) { + Reloc *r; + + r = &s->r[i]; + print(" rel %#.4ux/%d %s%+d\n", r->off, r->siz, r->sym->name, r->add); + } + } +} + +void +span(void) +{ + Prog *p, *q; + int32 v; + int n; + + if(debug['v']) + Bprint(&bso, "%5.2f span\n", cputime()); + + // NOTE(rsc): If we get rid of the globals we should + // be able to parallelize these iterations. + for(cursym = textp; cursym != nil; cursym = cursym->next) { + if(cursym->text == nil || cursym->text->link == nil) + continue; + + // TODO: move into span1 + for(p = cursym->text; p != P; p = p->link) { + n = 0; + if(p->to.type == D_BRANCH) + if(p->pcond == P) + p->pcond = p; + if((q = p->pcond) != P) + if(q->back != 2) + n = 1; + p->back = n; + if(p->as == AADJSP) { + p->to.type = D_SP; + v = -p->from.offset; + p->from.offset = v; + p->as = AADDL; + if(v < 0) { + p->as = ASUBL; + v = -v; + p->from.offset = v; + } + if(v == 0) + p->as = ANOP; + } + } + span1(cursym); + } +} + +void +xdefine(char *p, int t, int32 v) +{ + Sym *s; + + s = lookup(p, 0); + s->type = t; + s->value = v; + s->reachable = 1; + s->special = 1; +} + +void +instinit(void) +{ + int i; + + for(i=1; optab[i].as; i++) + if(i != optab[i].as) { + diag("phase error in optab: %d", i); + errorexit(); + } + maxop = i; + + for(i=0; i= D_AL && i <= D_BH) + reg[i] = (i-D_AL) & 7; + if(i >= D_AX && i <= D_DI) + reg[i] = (i-D_AX) & 7; + if(i >= D_F0 && i <= D_F0+7) + reg[i] = (i-D_F0) & 7; + } +} + +int +prefixof(Adr *a) +{ + switch(a->type) { + case D_INDIR+D_CS: + return 0x2e; + case D_INDIR+D_DS: + return 0x3e; + case D_INDIR+D_ES: + return 0x26; + case D_INDIR+D_FS: + return 0x64; + case D_INDIR+D_GS: + return 0x65; + } + return 0; +} + +int +oclass(Adr *a) +{ + int32 v; + + if((a->type >= D_INDIR && a->type < 2*D_INDIR) || a->index != D_NONE) { + if(a->index != D_NONE && a->scale == 0) { + if(a->type == D_ADDR) { + switch(a->index) { + case D_EXTERN: + case D_STATIC: + return Yi32; + case D_AUTO: + case D_PARAM: + return Yiauto; + } + return Yxxx; + } + return Ycol; + } + return Ym; + } + switch(a->type) + { + case D_AL: + return Yal; + + case D_AX: + return Yax; + + case D_CL: + case D_DL: + case D_BL: + case D_AH: + case D_CH: + case D_DH: + case D_BH: + return Yrb; + + case D_CX: + return Ycx; + + case D_DX: + case D_BX: + return Yrx; + + case D_SP: + case D_BP: + case D_SI: + case D_DI: + return Yrl; + + case D_F0+0: + return Yf0; + + case D_F0+1: + case D_F0+2: + case D_F0+3: + case D_F0+4: + case D_F0+5: + case D_F0+6: + case D_F0+7: + return Yrf; + + case D_NONE: + return Ynone; + + case D_CS: return Ycs; + case D_SS: return Yss; + case D_DS: return Yds; + case D_ES: return Yes; + case D_FS: return Yfs; + case D_GS: return Ygs; + + case D_GDTR: return Ygdtr; + case D_IDTR: return Yidtr; + case D_LDTR: return Yldtr; + case D_MSW: return Ymsw; + case D_TASK: return Ytask; + + case D_CR+0: return Ycr0; + case D_CR+1: return Ycr1; + case D_CR+2: return Ycr2; + case D_CR+3: return Ycr3; + case D_CR+4: return Ycr4; + case D_CR+5: return Ycr5; + case D_CR+6: return Ycr6; + case D_CR+7: return Ycr7; + + case D_DR+0: return Ydr0; + case D_DR+1: return Ydr1; + case D_DR+2: return Ydr2; + case D_DR+3: return Ydr3; + case D_DR+4: return Ydr4; + case D_DR+5: return Ydr5; + case D_DR+6: return Ydr6; + case D_DR+7: return Ydr7; + + case D_TR+0: return Ytr0; + case D_TR+1: return Ytr1; + case D_TR+2: return Ytr2; + case D_TR+3: return Ytr3; + case D_TR+4: return Ytr4; + case D_TR+5: return Ytr5; + case D_TR+6: return Ytr6; + case D_TR+7: return Ytr7; + + case D_EXTERN: + case D_STATIC: + case D_AUTO: + case D_PARAM: + return Ym; + + case D_CONST: + case D_CONST2: + case D_ADDR: + if(a->sym == S) { + v = a->offset; + if(v == 0) + return Yi0; + if(v == 1) + return Yi1; + if(v >= -128 && v <= 127) + return Yi8; + } + return Yi32; + + case D_BRANCH: + return Ybr; + } + return Yxxx; +} + +void +asmidx(int scale, int index, int base) +{ + int i; + + switch(index) { + default: + goto bad; + + case D_NONE: + i = 4 << 3; + goto bas; + + case D_AX: + case D_CX: + case D_DX: + case D_BX: + case D_BP: + case D_SI: + case D_DI: + i = reg[index] << 3; + break; + } + switch(scale) { + default: + goto bad; + case 1: + break; + case 2: + i |= (1<<6); + break; + case 4: + i |= (2<<6); + break; + case 8: + i |= (3<<6); + break; + } +bas: + switch(base) { + default: + goto bad; + case D_NONE: /* must be mod=00 */ + i |= 5; + break; + case D_AX: + case D_CX: + case D_DX: + case D_BX: + case D_SP: + case D_BP: + case D_SI: + case D_DI: + i |= reg[base]; + break; + } + *andptr++ = i; + return; +bad: + diag("asmidx: bad address %d,%d,%d", scale, index, base); + *andptr++ = 0; + return; +} + +static void +put4(int32 v) +{ + andptr[0] = v; + andptr[1] = v>>8; + andptr[2] = v>>16; + andptr[3] = v>>24; + andptr += 4; +} + +static void +relput4(Prog *p, Adr *a) +{ + vlong v; + Reloc rel, *r; + + v = vaddr(a, &rel); + if(rel.siz != 0) { + if(rel.siz != 4) + diag("bad reloc"); + r = addrel(cursym); + *r = rel; + r->off = p->pc + andptr - and; + } + put4(v); +} + +int32 +symaddr(Sym *s) +{ + if(!s->reachable) + diag("unreachable symbol in symaddr - %s", s->name); + return s->value; +} + +static int32 +vaddr(Adr *a, Reloc *r) +{ + int t; + int32 v; + Sym *s; + + if(r != nil) + memset(r, 0, sizeof *r); + + t = a->type; + v = a->offset; + if(t == D_ADDR) + t = a->index; + switch(t) { + case D_STATIC: + case D_EXTERN: + s = a->sym; + if(s != nil) { + if(!s->reachable) + sysfatal("unreachable symbol in vaddr - %s", s->name); + if(r == nil) { + diag("need reloc for %D", a); + errorexit(); + } + r->type = D_ADDR; + r->siz = 4; + r->off = -1; + r->sym = s; + r->add = v; + v = 0; + } + } + return v; +} + +void +asmand(Adr *a, int r) +{ + int32 v; + int t, scale; + Reloc rel; + + v = a->offset; + t = a->type; + rel.siz = 0; + if(a->index != D_NONE) { + if(t < D_INDIR || t >= 2*D_INDIR) { + switch(t) { + default: + goto bad; + case D_STATIC: + case D_EXTERN: + t = D_NONE; + v = vaddr(a, &rel); + break; + case D_AUTO: + case D_PARAM: + t = D_SP; + break; + } + } else + t -= D_INDIR; + + if(t == D_NONE) { + *andptr++ = (0 << 6) | (4 << 0) | (r << 3); + asmidx(a->scale, a->index, t); + goto putrelv; + } + if(v == 0 && rel.siz == 0 && t != D_BP) { + *andptr++ = (0 << 6) | (4 << 0) | (r << 3); + asmidx(a->scale, a->index, t); + return; + } + if(v >= -128 && v < 128 && rel.siz == 0) { + *andptr++ = (1 << 6) | (4 << 0) | (r << 3); + asmidx(a->scale, a->index, t); + *andptr++ = v; + return; + } + *andptr++ = (2 << 6) | (4 << 0) | (r << 3); + asmidx(a->scale, a->index, t); + goto putrelv; + } + if(t >= D_AL && t <= D_F0+7) { + if(v) + goto bad; + *andptr++ = (3 << 6) | (reg[t] << 0) | (r << 3); + return; + } + + scale = a->scale; + if(t < D_INDIR || t >= 2*D_INDIR) { + switch(a->type) { + default: + goto bad; + case D_STATIC: + case D_EXTERN: + t = D_NONE; + v = vaddr(a, &rel); + break; + case D_AUTO: + case D_PARAM: + t = D_SP; + break; + } + scale = 1; + } else + t -= D_INDIR; + + if(t == D_NONE || (D_CS <= t && t <= D_GS)) { + *andptr++ = (0 << 6) | (5 << 0) | (r << 3); + goto putrelv; + } + if(t == D_SP) { + if(v == 0 && rel.siz == 0) { + *andptr++ = (0 << 6) | (4 << 0) | (r << 3); + asmidx(scale, D_NONE, t); + return; + } + if(v >= -128 && v < 128 && rel.siz == 0) { + *andptr++ = (1 << 6) | (4 << 0) | (r << 3); + asmidx(scale, D_NONE, t); + *andptr++ = v; + return; + } + *andptr++ = (2 << 6) | (4 << 0) | (r << 3); + asmidx(scale, D_NONE, t); + goto putrelv; + } + if(t >= D_AX && t <= D_DI) { + if(v == 0 && rel.siz == 0 && t != D_BP) { + *andptr++ = (0 << 6) | (reg[t] << 0) | (r << 3); + return; + } + if(v >= -128 && v < 128 && rel.siz == 0) { + andptr[0] = (1 << 6) | (reg[t] << 0) | (r << 3); + andptr[1] = v; + andptr += 2; + return; + } + *andptr++ = (2 << 6) | (reg[t] << 0) | (r << 3); + goto putrelv; + } + goto bad; + +putrelv: + if(rel.siz != 0) { + Reloc *r; + + if(rel.siz != 4) { + diag("bad rel"); + goto bad; + } + r = addrel(cursym); + *r = rel; + r->off = curp->pc + andptr - and; + } + put4(v); + return; + +bad: + diag("asmand: bad address %D", a); + return; +} + +#define E 0xff +uchar ymovtab[] = +{ +/* push */ + APUSHL, Ycs, Ynone, 0, 0x0e,E,0,0, + APUSHL, Yss, Ynone, 0, 0x16,E,0,0, + APUSHL, Yds, Ynone, 0, 0x1e,E,0,0, + APUSHL, Yes, Ynone, 0, 0x06,E,0,0, + APUSHL, Yfs, Ynone, 0, 0x0f,0xa0,E,0, + APUSHL, Ygs, Ynone, 0, 0x0f,0xa8,E,0, + + APUSHW, Ycs, Ynone, 0, Pe,0x0e,E,0, + APUSHW, Yss, Ynone, 0, Pe,0x16,E,0, + APUSHW, Yds, Ynone, 0, Pe,0x1e,E,0, + APUSHW, Yes, Ynone, 0, Pe,0x06,E,0, + APUSHW, Yfs, Ynone, 0, Pe,0x0f,0xa0,E, + APUSHW, Ygs, Ynone, 0, Pe,0x0f,0xa8,E, + +/* pop */ + APOPL, Ynone, Yds, 0, 0x1f,E,0,0, + APOPL, Ynone, Yes, 0, 0x07,E,0,0, + APOPL, Ynone, Yss, 0, 0x17,E,0,0, + APOPL, Ynone, Yfs, 0, 0x0f,0xa1,E,0, + APOPL, Ynone, Ygs, 0, 0x0f,0xa9,E,0, + + APOPW, Ynone, Yds, 0, Pe,0x1f,E,0, + APOPW, Ynone, Yes, 0, Pe,0x07,E,0, + APOPW, Ynone, Yss, 0, Pe,0x17,E,0, + APOPW, Ynone, Yfs, 0, Pe,0x0f,0xa1,E, + APOPW, Ynone, Ygs, 0, Pe,0x0f,0xa9,E, + +/* mov seg */ + AMOVW, Yes, Yml, 1, 0x8c,0,0,0, + AMOVW, Ycs, Yml, 1, 0x8c,1,0,0, + AMOVW, Yss, Yml, 1, 0x8c,2,0,0, + AMOVW, Yds, Yml, 1, 0x8c,3,0,0, + AMOVW, Yfs, Yml, 1, 0x8c,4,0,0, + AMOVW, Ygs, Yml, 1, 0x8c,5,0,0, + + AMOVW, Yml, Yes, 2, 0x8e,0,0,0, + AMOVW, Yml, Ycs, 2, 0x8e,1,0,0, + AMOVW, Yml, Yss, 2, 0x8e,2,0,0, + AMOVW, Yml, Yds, 2, 0x8e,3,0,0, + AMOVW, Yml, Yfs, 2, 0x8e,4,0,0, + AMOVW, Yml, Ygs, 2, 0x8e,5,0,0, + +/* mov cr */ + AMOVL, Ycr0, Yml, 3, 0x0f,0x20,0,0, + AMOVL, Ycr2, Yml, 3, 0x0f,0x20,2,0, + AMOVL, Ycr3, Yml, 3, 0x0f,0x20,3,0, + AMOVL, Ycr4, Yml, 3, 0x0f,0x20,4,0, + + AMOVL, Yml, Ycr0, 4, 0x0f,0x22,0,0, + AMOVL, Yml, Ycr2, 4, 0x0f,0x22,2,0, + AMOVL, Yml, Ycr3, 4, 0x0f,0x22,3,0, + AMOVL, Yml, Ycr4, 4, 0x0f,0x22,4,0, + +/* mov dr */ + AMOVL, Ydr0, Yml, 3, 0x0f,0x21,0,0, + AMOVL, Ydr6, Yml, 3, 0x0f,0x21,6,0, + AMOVL, Ydr7, Yml, 3, 0x0f,0x21,7,0, + + AMOVL, Yml, Ydr0, 4, 0x0f,0x23,0,0, + AMOVL, Yml, Ydr6, 4, 0x0f,0x23,6,0, + AMOVL, Yml, Ydr7, 4, 0x0f,0x23,7,0, + +/* mov tr */ + AMOVL, Ytr6, Yml, 3, 0x0f,0x24,6,0, + AMOVL, Ytr7, Yml, 3, 0x0f,0x24,7,0, + + AMOVL, Yml, Ytr6, 4, 0x0f,0x26,6,E, + AMOVL, Yml, Ytr7, 4, 0x0f,0x26,7,E, + +/* lgdt, sgdt, lidt, sidt */ + AMOVL, Ym, Ygdtr, 4, 0x0f,0x01,2,0, + AMOVL, Ygdtr, Ym, 3, 0x0f,0x01,0,0, + AMOVL, Ym, Yidtr, 4, 0x0f,0x01,3,0, + AMOVL, Yidtr, Ym, 3, 0x0f,0x01,1,0, + +/* lldt, sldt */ + AMOVW, Yml, Yldtr, 4, 0x0f,0x00,2,0, + AMOVW, Yldtr, Yml, 3, 0x0f,0x00,0,0, + +/* lmsw, smsw */ + AMOVW, Yml, Ymsw, 4, 0x0f,0x01,6,0, + AMOVW, Ymsw, Yml, 3, 0x0f,0x01,4,0, + +/* ltr, str */ + AMOVW, Yml, Ytask, 4, 0x0f,0x00,3,0, + AMOVW, Ytask, Yml, 3, 0x0f,0x00,1,0, + +/* load full pointer */ + AMOVL, Yml, Ycol, 5, 0,0,0,0, + AMOVW, Yml, Ycol, 5, Pe,0,0,0, + +/* double shift */ + ASHLL, Ycol, Yml, 6, 0xa4,0xa5,0,0, + ASHRL, Ycol, Yml, 6, 0xac,0xad,0,0, + +/* extra imul */ + AIMULW, Yml, Yrl, 7, Pq,0xaf,0,0, + AIMULL, Yml, Yrl, 7, Pm,0xaf,0,0, + 0 +}; + +int +isax(Adr *a) +{ + + switch(a->type) { + case D_AX: + case D_AL: + case D_AH: + case D_INDIR+D_AX: + return 1; + } + if(a->index == D_AX) + return 1; + return 0; +} + +void +subreg(Prog *p, int from, int to) +{ + + if(debug['Q']) + print("\n%P s/%R/%R/\n", p, from, to); + + if(p->from.type == from) { + p->from.type = to; + p->ft = 0; + } + if(p->to.type == from) { + p->to.type = to; + p->tt = 0; + } + + if(p->from.index == from) { + p->from.index = to; + p->ft = 0; + } + if(p->to.index == from) { + p->to.index = to; + p->tt = 0; + } + + from += D_INDIR; + if(p->from.type == from) { + p->from.type = to+D_INDIR; + p->ft = 0; + } + if(p->to.type == from) { + p->to.type = to+D_INDIR; + p->tt = 0; + } + + if(debug['Q']) + print("%P\n", p); +} + +void +doasm(Prog *p) +{ + Optab *o; + Prog *q, pp; + uchar *t; + int z, op, ft, tt; + int32 v, pre; + Reloc rel, *r; + Adr *a; + + curp = p; // TODO + + pre = prefixof(&p->from); + if(pre) + *andptr++ = pre; + pre = prefixof(&p->to); + if(pre) + *andptr++ = pre; + + if(p->ft == 0) + p->ft = oclass(&p->from); + if(p->tt == 0) + p->tt = oclass(&p->to); + + ft = p->ft * Ymax; + tt = p->tt * Ymax; + o = &optab[p->as]; + t = o->ytab; + if(t == 0) { + diag("asmins: noproto %P", p); + return; + } + for(z=0; *t; z+=t[3],t+=4) + if(ycover[ft+t[0]]) + if(ycover[tt+t[1]]) + goto found; + goto domov; + +found: + switch(o->prefix) { + case Pq: /* 16 bit escape and opcode escape */ + *andptr++ = Pe; + *andptr++ = Pm; + break; + + case Pm: /* opcode escape */ + *andptr++ = Pm; + break; + + case Pe: /* 16 bit escape */ + *andptr++ = Pe; + break; + + case Pb: /* botch */ + break; + } + + op = o->op[z]; + switch(t[2]) { + default: + diag("asmins: unknown z %d %P", t[2], p); + return; + + case Zpseudo: + break; + + case Zlit: + for(; op = o->op[z]; z++) + *andptr++ = op; + break; + + case Zm_r: + *andptr++ = op; + asmand(&p->from, reg[p->to.type]); + break; + + case Zaut_r: + *andptr++ = 0x8d; /* leal */ + if(p->from.type != D_ADDR) + diag("asmins: Zaut sb type ADDR"); + p->from.type = p->from.index; + p->from.index = D_NONE; + p->ft = 0; + asmand(&p->from, reg[p->to.type]); + p->from.index = p->from.type; + p->from.type = D_ADDR; + p->ft = 0; + break; + + case Zm_o: + *andptr++ = op; + asmand(&p->from, o->op[z+1]); + break; + + case Zr_m: + *andptr++ = op; + asmand(&p->to, reg[p->from.type]); + break; + + case Zo_m: + *andptr++ = op; + asmand(&p->to, o->op[z+1]); + break; + + case Zm_ibo: + *andptr++ = op; + asmand(&p->from, o->op[z+1]); + *andptr++ = vaddr(&p->to, nil); + break; + + case Zibo_m: + *andptr++ = op; + asmand(&p->to, o->op[z+1]); + *andptr++ = vaddr(&p->from, nil); + break; + + case Z_ib: + case Zib_: + if(t[2] == Zib_) + a = &p->from; + else + a = &p->to; + v = vaddr(a, nil); + *andptr++ = op; + *andptr++ = v; + break; + + case Zib_rp: + *andptr++ = op + reg[p->to.type]; + *andptr++ = vaddr(&p->from, nil); + break; + + case Zil_rp: + *andptr++ = op + reg[p->to.type]; + if(o->prefix == Pe) { + v = vaddr(&p->from, nil); + *andptr++ = v; + *andptr++ = v>>8; + } + else + relput4(p, &p->from); + break; + + case Zib_rr: + *andptr++ = op; + asmand(&p->to, reg[p->to.type]); + *andptr++ = vaddr(&p->from, nil); + break; + + case Z_il: + case Zil_: + if(t[2] == Zil_) + a = &p->from; + else + a = &p->to; + *andptr++ = op; + if(o->prefix == Pe) { + v = vaddr(a, nil); + *andptr++ = v; + *andptr++ = v>>8; + } + else + relput4(p, a); + break; + + case Zm_ilo: + case Zilo_m: + *andptr++ = op; + if(t[2] == Zilo_m) { + a = &p->from; + asmand(&p->to, o->op[z+1]); + } else { + a = &p->to; + asmand(&p->from, o->op[z+1]); + } + if(o->prefix == Pe) { + v = vaddr(a, nil); + *andptr++ = v; + *andptr++ = v>>8; + } + else + relput4(p, a); + break; + + case Zil_rr: + *andptr++ = op; + asmand(&p->to, reg[p->to.type]); + if(o->prefix == Pe) { + v = vaddr(&p->from, nil); + *andptr++ = v; + *andptr++ = v>>8; + } + else + relput4(p, &p->from); + break; + + case Z_rp: + *andptr++ = op + reg[p->to.type]; + break; + + case Zrp_: + *andptr++ = op + reg[p->from.type]; + break; + + case Zclr: + *andptr++ = op; + asmand(&p->to, reg[p->to.type]); + break; + + case Zcall: + q = p->pcond; + if(q == nil) { + diag("call without target"); + errorexit(); + } + if(q->as != ATEXT) { + // Could handle this case by making D_PCREL + // record the Prog* instead of the Sym*, but let's + // wait until the need arises. + diag("call of non-TEXT %P", q); + errorexit(); + } + *andptr++ = op; + r = addrel(cursym); + r->off = p->pc + andptr - and; + r->type = D_PCREL; + r->siz = 4; + r->sym = q->from.sym; + put4(0); + break; + + case Zbr: + case Zjmp: + q = p->pcond; + if(q == nil) { + diag("jmp/branch without target"); + errorexit(); + } + if(q->as == ATEXT) { + // jump out of function + if(t[2] == Zbr) { + diag("branch to ATEXT"); + errorexit(); + } + *andptr++ = o->op[z+1]; + r = addrel(cursym); + r->off = p->pc + andptr - and; + r->sym = q->from.sym; + r->type = D_PCREL; + r->siz = 4; + put4(0); + break; + } + + // Assumes q is in this function. + // TODO: Check in input, preserve in brchain. + + // Fill in backward jump now. + if(p->back & 1) { + v = q->pc - (p->pc + 2); + if(v >= -128) { + *andptr++ = op; + *andptr++ = v; + } else { + v -= 5-2; + if(t[2] == Zbr) { + *andptr++ = 0x0f; + v--; + } + *andptr++ = o->op[z+1]; + *andptr++ = v; + *andptr++ = v>>8; + *andptr++ = v>>16; + *andptr++ = v>>24; + } + break; + } + + // Annotate target; will fill in later. + p->forwd = q->comefrom; + q->comefrom = p; + if(p->back & 2) { // short + *andptr++ = op; + *andptr++ = 0; + } else { + if(t[2] == Zbr) + *andptr++ = 0x0f; + *andptr++ = o->op[z+1]; + *andptr++ = 0; + *andptr++ = 0; + *andptr++ = 0; + *andptr++ = 0; + } + break; + + case Zcallcon: + case Zjmpcon: + if(t[2] == Zcallcon) + *andptr++ = op; + else + *andptr++ = o->op[z+1]; + r = addrel(cursym); + r->off = p->pc + andptr - and; + r->type = D_PCREL; + r->siz = 4; + r->add = p->to.offset; + put4(0); + break; + + case Zloop: + q = p->pcond; + if(q == nil) { + diag("loop without target"); + errorexit(); + } + v = q->pc - p->pc - 2; + if(v < -128 && v > 127) + diag("loop too far: %P", p); + *andptr++ = op; + *andptr++ = v; + break; + + case Zbyte: + v = vaddr(&p->from, &rel); + if(rel.siz != 0) { + rel.siz = op; + r = addrel(cursym); + *r = rel; + r->off = p->pc + andptr - and; + } + *andptr++ = v; + if(op > 1) { + *andptr++ = v>>8; + if(op > 2) { + *andptr++ = v>>16; + *andptr++ = v>>24; + } + } + break; + + case Zmov: + goto domov; + } + return; + +domov: + for(t=ymovtab; *t; t+=8) + if(p->as == t[0]) + if(ycover[ft+t[1]]) + if(ycover[tt+t[2]]) + goto mfound; +bad: + /* + * here, the assembly has failed. + * if its a byte instruction that has + * unaddressable registers, try to + * exchange registers and reissue the + * instruction with the operands renamed. + */ + pp = *p; + z = p->from.type; + if(z >= D_BP && z <= D_DI) { + if(isax(&p->to)) { + *andptr++ = 0x87; /* xchg lhs,bx */ + asmand(&p->from, reg[D_BX]); + subreg(&pp, z, D_BX); + doasm(&pp); + *andptr++ = 0x87; /* xchg lhs,bx */ + asmand(&p->from, reg[D_BX]); + } else { + *andptr++ = 0x90 + reg[z]; /* xchg lsh,ax */ + subreg(&pp, z, D_AX); + doasm(&pp); + *andptr++ = 0x90 + reg[z]; /* xchg lsh,ax */ + } + return; + } + z = p->to.type; + if(z >= D_BP && z <= D_DI) { + if(isax(&p->from)) { + *andptr++ = 0x87; /* xchg rhs,bx */ + asmand(&p->to, reg[D_BX]); + subreg(&pp, z, D_BX); + doasm(&pp); + *andptr++ = 0x87; /* xchg rhs,bx */ + asmand(&p->to, reg[D_BX]); + } else { + *andptr++ = 0x90 + reg[z]; /* xchg rsh,ax */ + subreg(&pp, z, D_AX); + doasm(&pp); + *andptr++ = 0x90 + reg[z]; /* xchg rsh,ax */ + } + return; + } + diag("doasm: notfound t2=%ux from=%ux to=%ux %P", t[2], p->from.type, p->to.type, p); + return; + +mfound: + switch(t[3]) { + default: + diag("asmins: unknown mov %d %P", t[3], p); + break; + + case 0: /* lit */ + for(z=4; t[z]!=E; z++) + *andptr++ = t[z]; + break; + + case 1: /* r,m */ + *andptr++ = t[4]; + asmand(&p->to, t[5]); + break; + + case 2: /* m,r */ + *andptr++ = t[4]; + asmand(&p->from, t[5]); + break; + + case 3: /* r,m - 2op */ + *andptr++ = t[4]; + *andptr++ = t[5]; + asmand(&p->to, t[6]); + break; + + case 4: /* m,r - 2op */ + *andptr++ = t[4]; + *andptr++ = t[5]; + asmand(&p->from, t[6]); + break; + + case 5: /* load full pointer, trash heap */ + if(t[4]) + *andptr++ = t[4]; + switch(p->to.index) { + default: + goto bad; + case D_DS: + *andptr++ = 0xc5; + break; + case D_SS: + *andptr++ = 0x0f; + *andptr++ = 0xb2; + break; + case D_ES: + *andptr++ = 0xc4; + break; + case D_FS: + *andptr++ = 0x0f; + *andptr++ = 0xb4; + break; + case D_GS: + *andptr++ = 0x0f; + *andptr++ = 0xb5; + break; + } + asmand(&p->from, reg[p->to.type]); + break; + + case 6: /* double shift */ + z = p->from.type; + switch(z) { + default: + goto bad; + case D_CONST: + *andptr++ = 0x0f; + *andptr++ = t[4]; + asmand(&p->to, reg[p->from.index]); + *andptr++ = p->from.offset; + break; + case D_CL: + case D_CX: + *andptr++ = 0x0f; + *andptr++ = t[5]; + asmand(&p->to, reg[p->from.index]); + break; + } + break; + + case 7: /* imul rm,r */ + if(t[4] == Pq) { + *andptr++ = Pe; + *andptr++ = Pm; + } else + *andptr++ = t[4]; + *andptr++ = t[5]; + asmand(&p->from, reg[p->to.type]); + break; + } +} + +void +asmins(Prog *p) +{ + andptr = and; + doasm(p); + if(andptr > and+sizeof and) { + print("and[] is too short - %ld byte instruction\n", andptr - and); + errorexit(); + } +} diff --git a/src/cmd/Makefile b/src/cmd/Makefile new file mode 100644 index 000000000..5a37733de --- /dev/null +++ b/src/cmd/Makefile @@ -0,0 +1,69 @@ +# Copyright 2011 The Go Authors. All rights reserved. +# Use of this source code is governed by a BSD-style +# license that can be found in the LICENSE file. + +include ../Make.inc + +all: install + +# Only build tools for current architecture, and only tools written in C. +# The tools written in Go are managed by ../pkg/Makefile. +DIRS=\ + $(O)a\ + $(O)c\ + $(O)g\ + $(O)l\ + cc\ + cov\ + gc\ + godefs\ + gopack\ + gotry\ + nm\ + prof\ + +# Clean applies to all directories, even for other architectures or +# written in Go. +CLEANDIRS=\ + $(DIRS)\ + 5a\ + 5c\ + 5g\ + 5l\ + 6a\ + 6c\ + 6g\ + 6l\ + 8a\ + 8c\ + 8g\ + 8l\ + cgo\ + ebnflint\ + godoc\ + gofix\ + gofmt\ + goinstall\ + gotest\ + gotype\ + goyacc\ + hgpatch\ + +install: $(patsubst %,%.install,$(DIRS)) +clean: $(patsubst %,%.clean,$(CLEANDIRS)) + +%.install: + @echo + @echo %%%% making $* %%%% + @echo + $(MAKE) -C $* install + +gc.install $(O)c.install: cc.install +$(O)g.install: gc.install +$(O)a.install $(O)c.install $(O)g.install: $(O)l.install + +%.clean: + $(MAKE) -C $* clean + +echo-dirs: + @echo $(DIRS) diff --git a/src/cmd/cc/Makefile b/src/cmd/cc/Makefile new file mode 100644 index 000000000..8327d9516 --- /dev/null +++ b/src/cmd/cc/Makefile @@ -0,0 +1,36 @@ +# Copyright 2009 The Go Authors. All rights reserved. +# Use of this source code is governed by a BSD-style +# license that can be found in the LICENSE file. + +include ../../Make.inc +O:=$(HOST_O) + +LIB=cc.a + +HFILES=\ + cc.h\ + y.tab.h\ + +YFILES=\ + cc.y\ + +OFILES=\ + y.tab.$O\ + lex.$O\ + mac.$O\ + dcl.$O\ + acid.$O\ + godefs.$O\ + bits.$O\ + com.$O\ + scon.$O\ + funct.$O\ + sub.$O\ + com64.$O\ + dpchk.$O\ + omachcap.$O\ + +NOINSTALL=1 +include ../../Make.clib + +install: $(LIB) diff --git a/src/cmd/cc/acid.c b/src/cmd/cc/acid.c new file mode 100644 index 000000000..23147e519 --- /dev/null +++ b/src/cmd/cc/acid.c @@ -0,0 +1,344 @@ +// Inferno utils/cc/acid.c +// http://code.google.com/p/inferno-os/source/browse/utils/cc/acid.c +// +// Copyright © 1994-1999 Lucent Technologies Inc. All rights reserved. +// Portions Copyright © 1995-1997 C H Forsyth (forsyth@terzarima.net) +// Portions Copyright © 1997-1999 Vita Nuova Limited +// Portions Copyright © 2000-2007 Vita Nuova Holdings Limited (www.vitanuova.com) +// Portions Copyright © 2004,2006 Bruce Ellis +// Portions Copyright © 2005-2007 C H Forsyth (forsyth@terzarima.net) +// Revisions Copyright © 2000-2007 Lucent Technologies Inc. and others +// Portions Copyright © 2009 The Go Authors. All rights reserved. +// +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to deal +// in the Software without restriction, including without limitation the rights +// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +// copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +// THE SOFTWARE. + +#include +#include "cc.h" + +static char *kwd[] = +{ + "$adt", "$aggr", "$append", "$complex", "$defn", + "$delete", "$do", "$else", "$eval", "$head", "$if", + "$local", "$loop", "$return", "$tail", "$then", + "$union", "$whatis", "$while", +}; + +char* +amap(char *s) +{ + int i, bot, top, new; + + bot = 0; + top = bot + nelem(kwd) - 1; + while(bot <= top){ + new = bot + (top - bot)/2; + i = strcmp(kwd[new]+1, s); + if(i == 0) + return kwd[new]; + + if(i < 0) + bot = new + 1; + else + top = new - 1; + } + return s; +} + +Sym* +acidsue(Type *t) +{ + int h; + Sym *s; + + if(t != T) + for(h=0; hlink) + if(s->suetag && s->suetag->link == t) + return s; + return 0; +} + +Sym* +acidfun(Type *t) +{ + int h; + Sym *s; + + for(h=0; hlink) + if(s->type == t) + return s; + return 0; +} + +char acidchar[NTYPE]; +Init acidcinit[] = +{ + TCHAR, 'C', 0, + TUCHAR, 'b', 0, + TSHORT, 'd', 0, + TUSHORT, 'u', 0, + TLONG, 'D', 0, + TULONG, 'U', 0, + TVLONG, 'V', 0, + TUVLONG, 'W', 0, + TFLOAT, 'f', 0, + TDOUBLE, 'F', 0, + TARRAY, 'a', 0, + TIND, 'X', 0, + -1, 0, 0, +}; + +static void +acidinit(void) +{ + Init *p; + + for(p=acidcinit; p->code >= 0; p++) + acidchar[p->code] = p->value; + + acidchar[TINT] = acidchar[TLONG]; + acidchar[TUINT] = acidchar[TULONG]; + if(types[TINT]->width != types[TLONG]->width) { + acidchar[TINT] = acidchar[TSHORT]; + acidchar[TUINT] = acidchar[TUSHORT]; + if(types[TINT]->width != types[TSHORT]->width) + warn(Z, "acidmember int not long or short"); + } + if(types[TIND]->width == types[TUVLONG]->width) + acidchar[TIND] = 'Y'; + +} + +void +acidmember(Type *t, int32 off, int flag) +{ + Sym *s, *s1; + Type *l; + static int acidcharinit = 0; + + if(acidcharinit == 0) { + acidinit(); + acidcharinit = 1; + } + s = t->sym; + switch(t->etype) { + default: + Bprint(&outbuf, " T%d\n", t->etype); + break; + + case TIND: + if(s == S) + break; + l = t->link; + if(flag) { + if(typesu[l->etype]) { + s1 = acidsue(l->link); + if(s1 != S) { + Bprint(&outbuf, " 'A' %s %d %s;\n", + amap(s1->name), + t->offset+off, amap(s->name)); + break; + } + } + } else { + l = t->link; + s1 = S; + if(typesu[l->etype]) + s1 = acidsue(l->link); + if(s1 != S) { + Bprint(&outbuf, + "\tprint(indent, \"%s\t(%s)\", addr.%s\\X, \"\\n\");\n", + amap(s->name), amap(s1->name), amap(s->name)); + } else { + Bprint(&outbuf, + "\tprint(indent, \"%s\t\", addr.%s\\X, \"\\n\");\n", + amap(s->name), amap(s->name)); + } + break; + } + + case TINT: + case TUINT: + case TCHAR: + case TUCHAR: + case TSHORT: + case TUSHORT: + case TLONG: + case TULONG: + case TVLONG: + case TUVLONG: + case TFLOAT: + case TDOUBLE: + case TARRAY: + if(s == S) + break; + if(flag) { + Bprint(&outbuf, " '%c' %d %s;\n", + acidchar[t->etype], t->offset+off, amap(s->name)); + } else { + Bprint(&outbuf, "\tprint(indent, \"%s\t\", addr.%s, \"\\n\");\n", + amap(s->name), amap(s->name)); + } + break; + + case TSTRUCT: + case TUNION: + s1 = acidsue(t->link); + if(s1 == S) + break; + if(flag) { + if(s == S) { + Bprint(&outbuf, " {\n"); + for(l = t->link; l != T; l = l->down) + acidmember(l, t->offset+off, flag); + Bprint(&outbuf, " };\n"); + } else { + Bprint(&outbuf, " %s %d %s;\n", + amap(s1->name), + t->offset+off, amap(s->name)); + } + } else { + if(s != S) { + Bprint(&outbuf, "\tprint(indent, \"%s %s {\\n\");\n", + amap(s1->name), amap(s->name)); + Bprint(&outbuf, "\tindent_%s(addr.%s, indent+\"\\t\");\n", + amap(s1->name), amap(s->name)); + Bprint(&outbuf, "\tprint(indent, \"}\\n\");\n"); + } else { + Bprint(&outbuf, "\tprint(indent, \"%s {\\n\");\n", + amap(s1->name)); + Bprint(&outbuf, "\tindent_%s(addr+%d, indent+\"\\t\");\n", + amap(s1->name), t->offset+off); + Bprint(&outbuf, "\tprint(indent, \"}\\n\");\n"); + } + } + break; + } +} + +void +acidtype(Type *t) +{ + Sym *s; + Type *l; + Io *i; + int n; + char *an; + + if(!debug['a']) + return; + if(debug['a'] > 1) { + n = 0; + for(i=iostack; i; i=i->link) + n++; + if(n > 1) + return; + } + s = acidsue(t->link); + if(s == S) + return; + switch(t->etype) { + default: + Bprint(&outbuf, "T%d\n", t->etype); + return; + + case TUNION: + case TSTRUCT: + if(debug['s']) + goto asmstr; + an = amap(s->name); + Bprint(&outbuf, "sizeof%s = %d;\n", an, t->width); + Bprint(&outbuf, "aggr %s\n{\n", an); + for(l = t->link; l != T; l = l->down) + acidmember(l, 0, 1); + Bprint(&outbuf, "};\n\n"); + + Bprint(&outbuf, "defn\n%s(addr) {\n\tindent_%s(addr, \"\\t\");\n}\n", an, an); + Bprint(&outbuf, "defn\nindent_%s(addr, indent) {\n\tcomplex %s addr;\n", an, an); + for(l = t->link; l != T; l = l->down) + acidmember(l, 0, 0); + Bprint(&outbuf, "};\n\n"); + break; + asmstr: + if(s == S) + break; + for(l = t->link; l != T; l = l->down) + if(l->sym != S) + Bprint(&outbuf, "#define\t%s.%s\t%d\n", + s->name, + l->sym->name, + l->offset); + break; + } +} + +void +acidvar(Sym *s) +{ + int n; + Io *i; + Type *t; + Sym *s1, *s2; + + if(!debug['a'] || debug['s']) + return; + if(debug['a'] > 1) { + n = 0; + for(i=iostack; i; i=i->link) + n++; + if(n > 1) + return; + } + t = s->type; + while(t && t->etype == TIND) + t = t->link; + if(t == T) + return; + if(t->etype == TENUM) { + Bprint(&outbuf, "%s = ", amap(s->name)); + if(!typefd[t->etype]) + Bprint(&outbuf, "%lld;\n", s->vconst); + else + Bprint(&outbuf, "%f\n;", s->fconst); + return; + } + if(!typesu[t->etype]) + return; + s1 = acidsue(t->link); + if(s1 == S) + return; + switch(s->class) { + case CAUTO: + case CPARAM: + s2 = acidfun(thisfn); + if(s2) + Bprint(&outbuf, "complex %s %s:%s;\n", + amap(s1->name), amap(s2->name), amap(s->name)); + break; + + case CSTATIC: + case CEXTERN: + case CGLOBL: + case CLOCAL: + Bprint(&outbuf, "complex %s %s;\n", + amap(s1->name), amap(s->name)); + break; + } +} diff --git a/src/cmd/cc/bits.c b/src/cmd/cc/bits.c new file mode 100644 index 000000000..4496d65e7 --- /dev/null +++ b/src/cmd/cc/bits.c @@ -0,0 +1,120 @@ +// Inferno utils/cc/bits.c +// http://code.google.com/p/inferno-os/source/browse/utils/cc/bits.c +// +// Copyright © 1994-1999 Lucent Technologies Inc. All rights reserved. +// Portions Copyright © 1995-1997 C H Forsyth (forsyth@terzarima.net) +// Portions Copyright © 1997-1999 Vita Nuova Limited +// Portions Copyright © 2000-2007 Vita Nuova Holdings Limited (www.vitanuova.com) +// Portions Copyright © 2004,2006 Bruce Ellis +// Portions Copyright © 2005-2007 C H Forsyth (forsyth@terzarima.net) +// Revisions Copyright © 2000-2007 Lucent Technologies Inc. and others +// Portions Copyright © 2009 The Go Authors. All rights reserved. +// +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to deal +// in the Software without restriction, including without limitation the rights +// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +// copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +// THE SOFTWARE. + +#include +#include "cc.h" + +Bits +bor(Bits a, Bits b) +{ + Bits c; + int i; + + for(i=0; ib[i]) + return 1; + return 0; +} + +int +beq(Bits a, Bits b) +{ + int i; + + for(i=0; i +#include + +#pragma lib "../cc/cc.a$O" + +#ifndef EXTERN +#define EXTERN extern +#endif + +#undef getc +#undef ungetc +#undef BUFSIZ + +#define getc ccgetc +#define ungetc ccungetc + +typedef struct Node Node; +typedef struct Sym Sym; +typedef struct Type Type; +typedef struct Funct Funct; +typedef struct Decl Decl; +typedef struct Io Io; +typedef struct Hist Hist; +typedef struct Term Term; +typedef struct Init Init; +typedef struct Bits Bits; +typedef struct Dynimp Dynimp; +typedef struct Dynexp Dynexp; + +#define BUFSIZ 8192 +#define NSYMB 500 +#define NHASH 1024 +#define STRINGSZ 200 +#define HISTSZ 20 +#define YYMAXDEPTH 500 +#define NTERM 10 +#define MAXALIGN 7 + +#define SIGN(n) ((uvlong)1<<(n-1)) +#define MASK(n) (SIGN(n)|(SIGN(n)-1)) + +#define BITS 5 +#define NVAR (BITS*sizeof(uint32)*8) +struct Bits +{ + uint32 b[BITS]; +}; + +struct Node +{ + Node* left; + Node* right; + void* label; + int32 pc; + int reg; + int32 xoffset; + double fconst; /* fp constant */ + vlong vconst; /* non fp const */ + char* cstring; /* character string */ + ushort* rstring; /* rune string */ + + Sym* sym; + Type* type; + int32 lineno; + uchar op; + uchar oldop; + uchar xcast; + uchar class; + uchar etype; + uchar complex; + uchar addable; + uchar scale; + uchar garb; +}; +#define Z ((Node*)0) + +struct Sym +{ + Sym* link; + Type* type; + Type* suetag; + Type* tenum; + char* macro; + int32 varlineno; + int32 offset; + vlong vconst; + double fconst; + Node* label; + ushort lexical; + char *name; + ushort block; + ushort sueblock; + uchar class; + uchar sym; + uchar aused; + uchar sig; +}; +#define S ((Sym*)0) + +enum{ + SIGNONE = 0, + SIGDONE = 1, + SIGINTERN = 2, + + SIGNINTERN = 1729*325*1729, +}; + +struct Decl +{ + Decl* link; + Sym* sym; + Type* type; + int32 varlineno; + int32 offset; + short val; + ushort block; + uchar class; + uchar aused; +}; +#define D ((Decl*)0) + +struct Type +{ + Sym* sym; + Sym* tag; + Funct* funct; + Type* link; + Type* down; + int32 width; + int32 offset; + int32 lineno; + uchar shift; + uchar nbits; + uchar etype; + uchar garb; + uchar align; +}; + +#define T ((Type*)0) +#define NODECL ((void(*)(int, Type*, Sym*))0) + +struct Init /* general purpose initialization */ +{ + int code; + uint32 value; + char* s; +}; + +EXTERN struct +{ + char* p; + int c; +} fi; + +struct Io +{ + Io* link; + char* p; + char b[BUFSIZ]; + short c; + short f; +}; +#define I ((Io*)0) + +struct Hist +{ + Hist* link; + char* name; + int32 line; + int32 offset; +}; +#define H ((Hist*)0) +EXTERN Hist* hist; + +struct Term +{ + vlong mult; + Node *node; +}; + +enum +{ + Axxx, + Ael1, + Ael2, + Asu2, + Aarg0, + Aarg1, + Aarg2, + Aaut3, + NALIGN, +}; + +enum +{ + DMARK, + DAUTO, + DSUE, + DLABEL, +}; +enum +{ + OXXX, + OADD, + OADDR, + OAND, + OANDAND, + OARRAY, + OAS, + OASI, + OASADD, + OASAND, + OASASHL, + OASASHR, + OASDIV, + OASHL, + OASHR, + OASLDIV, + OASLMOD, + OASLMUL, + OASLSHR, + OASMOD, + OASMUL, + OASOR, + OASSUB, + OASXOR, + OBIT, + OBREAK, + OCASE, + OCAST, + OCOMMA, + OCOND, + OCONST, + OCONTINUE, + ODIV, + ODOT, + ODOTDOT, + ODWHILE, + OENUM, + OEQ, + OEXREG, + OFOR, + OFUNC, + OGE, + OGOTO, + OGT, + OHI, + OHS, + OIF, + OIND, + OINDREG, + OINIT, + OLABEL, + OLDIV, + OLE, + OLIST, + OLMOD, + OLMUL, + OLO, + OLS, + OLSHR, + OLT, + OMOD, + OMUL, + ONAME, + ONE, + ONOT, + OOR, + OOROR, + OPOSTDEC, + OPOSTINC, + OPREDEC, + OPREINC, + OPROTO, + OREGISTER, + ORETURN, + OSET, + OSIGN, + OSIZE, + OSTRING, + OLSTRING, + OSTRUCT, + OSUB, + OSWITCH, + OUNION, + OUSED, + OWHILE, + OXOR, + ONEG, + OCOM, + OPOS, + OELEM, + + OTST, /* used in some compilers */ + OINDEX, + OFAS, + OREGPAIR, + + OEND +}; +enum +{ + TXXX, + TCHAR, + TUCHAR, + TSHORT, + TUSHORT, + TINT, + TUINT, + TLONG, + TULONG, + TVLONG, + TUVLONG, + TFLOAT, + TDOUBLE, + TIND, + TFUNC, + TARRAY, + TVOID, + TSTRUCT, + TUNION, + TENUM, + NTYPE, + + TAUTO = NTYPE, + TEXTERN, + TSTATIC, + TTYPEDEF, + TTYPESTR, + TREGISTER, + TCONSTNT, + TVOLATILE, + TUNSIGNED, + TSIGNED, + TDOT, + TFILE, + TOLD, + NALLTYPES, +}; +enum +{ + CXXX, + CAUTO, + CEXTERN, + CGLOBL, + CSTATIC, + CLOCAL, + CTYPEDEF, + CTYPESTR, + CPARAM, + CSELEM, + CLABEL, + CEXREG, + NCTYPES, +}; +enum +{ + GXXX = 0, + GCONSTNT = 1<<0, + GVOLATILE = 1<<1, + NGTYPES = 1<<2, + + GINCOMPLETE = 1<<2, +}; +enum +{ + BCHAR = 1L< +#include /* if we don't, bison will, and cc.h re-#defines getc */ +#include "cc.h" +%} +%union { + Node* node; + Sym* sym; + Type* type; + struct + { + Type* t; + uchar c; + } tycl; + struct + { + Type* t1; + Type* t2; + Type* t3; + uchar c; + } tyty; + struct + { + char* s; + int32 l; + } sval; + int32 lval; + double dval; + vlong vval; +} +%type ltag +%type gctname gcname cname gname tname +%type gctnlist gcnlist zgnlist +%type tlist sbody complex +%type types +%type zarglist arglist zcexpr +%type name block stmnt cexpr expr xuexpr pexpr +%type zelist elist adecl slist uexpr string lstring +%type xdecor xdecor2 labels label ulstmnt +%type adlist edecor tag qual qlist +%type abdecor abdecor1 abdecor2 abdecor3 +%type zexpr lexpr init ilist forexpr + +%left ';' +%left ',' +%right '=' LPE LME LMLE LDVE LMDE LRSHE LLSHE LANDE LXORE LORE +%right '?' ':' +%left LOROR +%left LANDAND +%left '|' +%left '^' +%left '&' +%left LEQ LNE +%left '<' '>' LLE LGE +%left LLSH LRSH +%left '+' '-' +%left '*' '/' '%' +%right LMM LPP LMG '.' '[' '(' + +%token LNAME LTYPE +%token LFCONST LDCONST +%token LCONST LLCONST LUCONST LULCONST LVLCONST LUVLCONST +%token LSTRING LLSTRING +%token LAUTO LBREAK LCASE LCHAR LCONTINUE LDEFAULT LDO +%token LDOUBLE LELSE LEXTERN LFLOAT LFOR LGOTO +%token LIF LINT LLONG LREGISTER LRETURN LSHORT LSIZEOF LUSED +%token LSTATIC LSTRUCT LSWITCH LTYPEDEF LTYPESTR LUNION LUNSIGNED +%token LWHILE LVOID LENUM LSIGNED LCONSTNT LVOLATILE LSET LSIGNOF +%token LRESTRICT LINLINE +%% +prog: +| prog xdecl + +/* + * external declarator + */ +xdecl: + zctlist ';' + { + dodecl(xdecl, lastclass, lasttype, Z); + } +| zctlist xdlist ';' +| zctlist xdecor + { + lastdcl = T; + firstarg = S; + dodecl(xdecl, lastclass, lasttype, $2); + if(lastdcl == T || lastdcl->etype != TFUNC) { + diag($2, "not a function"); + lastdcl = types[TFUNC]; + } + thisfn = lastdcl; + markdcl(); + firstdcl = dclstack; + argmark($2, 0); + } + pdecl + { + argmark($2, 1); + } + block + { + Node *n; + + n = revertdcl(); + if(n) + $6 = new(OLIST, n, $6); + if(!debug['a'] && !debug['Z']) + codgen($6, $2); + } + +xdlist: + xdecor + { + dodecl(xdecl, lastclass, lasttype, $1); + } +| xdecor + { + $1 = dodecl(xdecl, lastclass, lasttype, $1); + } + '=' init + { + doinit($1->sym, $1->type, 0L, $4); + } +| xdlist ',' xdlist + +xdecor: + xdecor2 +| '*' zgnlist xdecor + { + $$ = new(OIND, $3, Z); + $$->garb = simpleg($2); + } + +xdecor2: + tag +| '(' xdecor ')' + { + $$ = $2; + } +| xdecor2 '(' zarglist ')' + { + $$ = new(OFUNC, $1, $3); + } +| xdecor2 '[' zexpr ']' + { + $$ = new(OARRAY, $1, $3); + } + +/* + * automatic declarator + */ +adecl: + ctlist ';' + { + $$ = dodecl(adecl, lastclass, lasttype, Z); + } +| ctlist adlist ';' + { + $$ = $2; + } + +adlist: + xdecor + { + dodecl(adecl, lastclass, lasttype, $1); + $$ = Z; + } +| xdecor + { + $1 = dodecl(adecl, lastclass, lasttype, $1); + } + '=' init + { + int32 w; + + w = $1->sym->type->width; + $$ = doinit($1->sym, $1->type, 0L, $4); + $$ = contig($1->sym, $$, w); + } +| adlist ',' adlist + { + $$ = $1; + if($3 != Z) { + $$ = $3; + if($1 != Z) + $$ = new(OLIST, $1, $3); + } + } + +/* + * parameter declarator + */ +pdecl: +| pdecl ctlist pdlist ';' + +pdlist: + xdecor + { + dodecl(pdecl, lastclass, lasttype, $1); + } +| pdlist ',' pdlist + +/* + * structure element declarator + */ +edecl: + tlist + { + lasttype = $1; + } + zedlist ';' +| edecl tlist + { + lasttype = $2; + } + zedlist ';' + +zedlist: /* extension */ + { + lastfield = 0; + edecl(CXXX, lasttype, S); + } +| edlist + +edlist: + edecor + { + dodecl(edecl, CXXX, lasttype, $1); + } +| edlist ',' edlist + +edecor: + xdecor + { + lastbit = 0; + firstbit = 1; + } +| tag ':' lexpr + { + $$ = new(OBIT, $1, $3); + } +| ':' lexpr + { + $$ = new(OBIT, Z, $2); + } + +/* + * abstract declarator + */ +abdecor: + { + $$ = (Z); + } +| abdecor1 + +abdecor1: + '*' zgnlist + { + $$ = new(OIND, (Z), Z); + $$->garb = simpleg($2); + } +| '*' zgnlist abdecor1 + { + $$ = new(OIND, $3, Z); + $$->garb = simpleg($2); + } +| abdecor2 + +abdecor2: + abdecor3 +| abdecor2 '(' zarglist ')' + { + $$ = new(OFUNC, $1, $3); + } +| abdecor2 '[' zexpr ']' + { + $$ = new(OARRAY, $1, $3); + } + +abdecor3: + '(' ')' + { + $$ = new(OFUNC, (Z), Z); + } +| '[' zexpr ']' + { + $$ = new(OARRAY, (Z), $2); + } +| '(' abdecor1 ')' + { + $$ = $2; + } + +init: + expr +| '{' ilist '}' + { + $$ = new(OINIT, invert($2), Z); + } + +qual: + '[' lexpr ']' + { + $$ = new(OARRAY, $2, Z); + } +| '.' ltag + { + $$ = new(OELEM, Z, Z); + $$->sym = $2; + } +| qual '=' + +qlist: + init ',' +| qlist init ',' + { + $$ = new(OLIST, $1, $2); + } +| qual +| qlist qual + { + $$ = new(OLIST, $1, $2); + } + +ilist: + qlist +| init +| qlist init + { + $$ = new(OLIST, $1, $2); + } + +zarglist: + { + $$ = Z; + } +| arglist + { + $$ = invert($1); + } + + +arglist: + name +| tlist abdecor + { + $$ = new(OPROTO, $2, Z); + $$->type = $1; + } +| tlist xdecor + { + $$ = new(OPROTO, $2, Z); + $$->type = $1; + } +| '.' '.' '.' + { + $$ = new(ODOTDOT, Z, Z); + } +| arglist ',' arglist + { + $$ = new(OLIST, $1, $3); + } + +block: + '{' slist '}' + { + $$ = invert($2); + // if($2 != Z) + // $$ = new(OLIST, $2, $$); + if($$ == Z) + $$ = new(OLIST, Z, Z); + } + +slist: + { + $$ = Z; + } +| slist adecl + { + $$ = new(OLIST, $1, $2); + } +| slist stmnt + { + $$ = new(OLIST, $1, $2); + } + +labels: + label +| labels label + { + $$ = new(OLIST, $1, $2); + } + +label: + LCASE expr ':' + { + $$ = new(OCASE, $2, Z); + } +| LDEFAULT ':' + { + $$ = new(OCASE, Z, Z); + } +| LNAME ':' + { + $$ = new(OLABEL, dcllabel($1, 1), Z); + } + +stmnt: + error ';' + { + $$ = Z; + } +| ulstmnt +| labels ulstmnt + { + $$ = new(OLIST, $1, $2); + } + +forexpr: + zcexpr +| ctlist adlist + { + $$ = $2; + } + +ulstmnt: + zcexpr ';' +| { + markdcl(); + } + block + { + $$ = revertdcl(); + if($$) + $$ = new(OLIST, $$, $2); + else + $$ = $2; + } +| LIF '(' cexpr ')' stmnt + { + $$ = new(OIF, $3, new(OLIST, $5, Z)); + if($5 == Z) + warn($3, "empty if body"); + } +| LIF '(' cexpr ')' stmnt LELSE stmnt + { + $$ = new(OIF, $3, new(OLIST, $5, $7)); + if($5 == Z) + warn($3, "empty if body"); + if($7 == Z) + warn($3, "empty else body"); + } +| { markdcl(); } LFOR '(' forexpr ';' zcexpr ';' zcexpr ')' stmnt + { + $$ = revertdcl(); + if($$){ + if($4) + $4 = new(OLIST, $$, $4); + else + $4 = $$; + } + $$ = new(OFOR, new(OLIST, $6, new(OLIST, $4, $8)), $10); + } +| LWHILE '(' cexpr ')' stmnt + { + $$ = new(OWHILE, $3, $5); + } +| LDO stmnt LWHILE '(' cexpr ')' ';' + { + $$ = new(ODWHILE, $5, $2); + } +| LRETURN zcexpr ';' + { + $$ = new(ORETURN, $2, Z); + $$->type = thisfn->link; + } +| LSWITCH '(' cexpr ')' stmnt + { + $$ = new(OCONST, Z, Z); + $$->vconst = 0; + $$->type = types[TINT]; + $3 = new(OSUB, $$, $3); + + $$ = new(OCONST, Z, Z); + $$->vconst = 0; + $$->type = types[TINT]; + $3 = new(OSUB, $$, $3); + + $$ = new(OSWITCH, $3, $5); + } +| LBREAK ';' + { + $$ = new(OBREAK, Z, Z); + } +| LCONTINUE ';' + { + $$ = new(OCONTINUE, Z, Z); + } +| LGOTO ltag ';' + { + $$ = new(OGOTO, dcllabel($2, 0), Z); + } +| LUSED '(' zelist ')' ';' + { + $$ = new(OUSED, $3, Z); + } +| LSET '(' zelist ')' ';' + { + $$ = new(OSET, $3, Z); + } + +zcexpr: + { + $$ = Z; + } +| cexpr + +zexpr: + { + $$ = Z; + } +| lexpr + +lexpr: + expr + { + $$ = new(OCAST, $1, Z); + $$->type = types[TLONG]; + } + +cexpr: + expr +| cexpr ',' cexpr + { + $$ = new(OCOMMA, $1, $3); + } + +expr: + xuexpr +| expr '*' expr + { + $$ = new(OMUL, $1, $3); + } +| expr '/' expr + { + $$ = new(ODIV, $1, $3); + } +| expr '%' expr + { + $$ = new(OMOD, $1, $3); + } +| expr '+' expr + { + $$ = new(OADD, $1, $3); + } +| expr '-' expr + { + $$ = new(OSUB, $1, $3); + } +| expr LRSH expr + { + $$ = new(OASHR, $1, $3); + } +| expr LLSH expr + { + $$ = new(OASHL, $1, $3); + } +| expr '<' expr + { + $$ = new(OLT, $1, $3); + } +| expr '>' expr + { + $$ = new(OGT, $1, $3); + } +| expr LLE expr + { + $$ = new(OLE, $1, $3); + } +| expr LGE expr + { + $$ = new(OGE, $1, $3); + } +| expr LEQ expr + { + $$ = new(OEQ, $1, $3); + } +| expr LNE expr + { + $$ = new(ONE, $1, $3); + } +| expr '&' expr + { + $$ = new(OAND, $1, $3); + } +| expr '^' expr + { + $$ = new(OXOR, $1, $3); + } +| expr '|' expr + { + $$ = new(OOR, $1, $3); + } +| expr LANDAND expr + { + $$ = new(OANDAND, $1, $3); + } +| expr LOROR expr + { + $$ = new(OOROR, $1, $3); + } +| expr '?' cexpr ':' expr + { + $$ = new(OCOND, $1, new(OLIST, $3, $5)); + } +| expr '=' expr + { + $$ = new(OAS, $1, $3); + } +| expr LPE expr + { + $$ = new(OASADD, $1, $3); + } +| expr LME expr + { + $$ = new(OASSUB, $1, $3); + } +| expr LMLE expr + { + $$ = new(OASMUL, $1, $3); + } +| expr LDVE expr + { + $$ = new(OASDIV, $1, $3); + } +| expr LMDE expr + { + $$ = new(OASMOD, $1, $3); + } +| expr LLSHE expr + { + $$ = new(OASASHL, $1, $3); + } +| expr LRSHE expr + { + $$ = new(OASASHR, $1, $3); + } +| expr LANDE expr + { + $$ = new(OASAND, $1, $3); + } +| expr LXORE expr + { + $$ = new(OASXOR, $1, $3); + } +| expr LORE expr + { + $$ = new(OASOR, $1, $3); + } + +xuexpr: + uexpr +| '(' tlist abdecor ')' xuexpr + { + $$ = new(OCAST, $5, Z); + dodecl(NODECL, CXXX, $2, $3); + $$->type = lastdcl; + $$->xcast = 1; + } +| '(' tlist abdecor ')' '{' ilist '}' /* extension */ + { + $$ = new(OSTRUCT, $6, Z); + dodecl(NODECL, CXXX, $2, $3); + $$->type = lastdcl; + } + +uexpr: + pexpr +| '*' xuexpr + { + $$ = new(OIND, $2, Z); + } +| '&' xuexpr + { + $$ = new(OADDR, $2, Z); + } +| '+' xuexpr + { + $$ = new(OPOS, $2, Z); + } +| '-' xuexpr + { + $$ = new(ONEG, $2, Z); + } +| '!' xuexpr + { + $$ = new(ONOT, $2, Z); + } +| '~' xuexpr + { + $$ = new(OCOM, $2, Z); + } +| LPP xuexpr + { + $$ = new(OPREINC, $2, Z); + } +| LMM xuexpr + { + $$ = new(OPREDEC, $2, Z); + } +| LSIZEOF uexpr + { + $$ = new(OSIZE, $2, Z); + } +| LSIGNOF uexpr + { + $$ = new(OSIGN, $2, Z); + } + +pexpr: + '(' cexpr ')' + { + $$ = $2; + } +| LSIZEOF '(' tlist abdecor ')' + { + $$ = new(OSIZE, Z, Z); + dodecl(NODECL, CXXX, $3, $4); + $$->type = lastdcl; + } +| LSIGNOF '(' tlist abdecor ')' + { + $$ = new(OSIGN, Z, Z); + dodecl(NODECL, CXXX, $3, $4); + $$->type = lastdcl; + } +| pexpr '(' zelist ')' + { + $$ = new(OFUNC, $1, Z); + if($1->op == ONAME) + if($1->type == T) + dodecl(xdecl, CXXX, types[TINT], $$); + $$->right = invert($3); + } +| pexpr '[' cexpr ']' + { + $$ = new(OIND, new(OADD, $1, $3), Z); + } +| pexpr LMG ltag + { + $$ = new(ODOT, new(OIND, $1, Z), Z); + $$->sym = $3; + } +| pexpr '.' ltag + { + $$ = new(ODOT, $1, Z); + $$->sym = $3; + } +| pexpr LPP + { + $$ = new(OPOSTINC, $1, Z); + } +| pexpr LMM + { + $$ = new(OPOSTDEC, $1, Z); + } +| name +| LCONST + { + $$ = new(OCONST, Z, Z); + $$->type = types[TINT]; + $$->vconst = $1; + $$->cstring = strdup(symb); + } +| LLCONST + { + $$ = new(OCONST, Z, Z); + $$->type = types[TLONG]; + $$->vconst = $1; + $$->cstring = strdup(symb); + } +| LUCONST + { + $$ = new(OCONST, Z, Z); + $$->type = types[TUINT]; + $$->vconst = $1; + $$->cstring = strdup(symb); + } +| LULCONST + { + $$ = new(OCONST, Z, Z); + $$->type = types[TULONG]; + $$->vconst = $1; + $$->cstring = strdup(symb); + } +| LDCONST + { + $$ = new(OCONST, Z, Z); + $$->type = types[TDOUBLE]; + $$->fconst = $1; + $$->cstring = strdup(symb); + } +| LFCONST + { + $$ = new(OCONST, Z, Z); + $$->type = types[TFLOAT]; + $$->fconst = $1; + $$->cstring = strdup(symb); + } +| LVLCONST + { + $$ = new(OCONST, Z, Z); + $$->type = types[TVLONG]; + $$->vconst = $1; + $$->cstring = strdup(symb); + } +| LUVLCONST + { + $$ = new(OCONST, Z, Z); + $$->type = types[TUVLONG]; + $$->vconst = $1; + $$->cstring = strdup(symb); + } +| string +| lstring + +string: + LSTRING + { + $$ = new(OSTRING, Z, Z); + $$->type = typ(TARRAY, types[TCHAR]); + $$->type->width = $1.l + 1; + $$->cstring = $1.s; + $$->sym = symstring; + $$->etype = TARRAY; + $$->class = CSTATIC; + } +| string LSTRING + { + char *s; + int n; + + n = $1->type->width - 1; + s = alloc(n+$2.l+MAXALIGN); + + memcpy(s, $1->cstring, n); + memcpy(s+n, $2.s, $2.l); + s[n+$2.l] = 0; + + $$ = $1; + $$->type->width += $2.l; + $$->cstring = s; + } + +lstring: + LLSTRING + { + $$ = new(OLSTRING, Z, Z); + $$->type = typ(TARRAY, types[TUSHORT]); + $$->type->width = $1.l + sizeof(ushort); + $$->rstring = (ushort*)$1.s; + $$->sym = symstring; + $$->etype = TARRAY; + $$->class = CSTATIC; + } +| lstring LLSTRING + { + char *s; + int n; + + n = $1->type->width - sizeof(ushort); + s = alloc(n+$2.l+MAXALIGN); + + memcpy(s, $1->rstring, n); + memcpy(s+n, $2.s, $2.l); + *(ushort*)(s+n+$2.l) = 0; + + $$ = $1; + $$->type->width += $2.l; + $$->rstring = (ushort*)s; + } + +zelist: + { + $$ = Z; + } +| elist + +elist: + expr +| elist ',' elist + { + $$ = new(OLIST, $1, $3); + } + +sbody: + '{' + { + $$.t1 = strf; + $$.t2 = strl; + $$.t3 = lasttype; + $$.c = lastclass; + strf = T; + strl = T; + lastbit = 0; + firstbit = 1; + lastclass = CXXX; + lasttype = T; + } + edecl '}' + { + $$ = strf; + strf = $2.t1; + strl = $2.t2; + lasttype = $2.t3; + lastclass = $2.c; + } + +zctlist: + { + lastclass = CXXX; + lasttype = types[TINT]; + } +| ctlist + +types: + complex + { + $$.t = $1; + $$.c = CXXX; + } +| tname + { + $$.t = simplet($1); + $$.c = CXXX; + } +| gcnlist + { + $$.t = simplet($1); + $$.c = simplec($1); + $$.t = garbt($$.t, $1); + } +| complex gctnlist + { + $$.t = $1; + $$.c = simplec($2); + $$.t = garbt($$.t, $2); + if($2 & ~BCLASS & ~BGARB) + diag(Z, "duplicate types given: %T and %Q", $1, $2); + } +| tname gctnlist + { + $$.t = simplet(typebitor($1, $2)); + $$.c = simplec($2); + $$.t = garbt($$.t, $2); + } +| gcnlist complex zgnlist + { + $$.t = $2; + $$.c = simplec($1); + $$.t = garbt($$.t, $1|$3); + } +| gcnlist tname + { + $$.t = simplet($2); + $$.c = simplec($1); + $$.t = garbt($$.t, $1); + } +| gcnlist tname gctnlist + { + $$.t = simplet(typebitor($2, $3)); + $$.c = simplec($1|$3); + $$.t = garbt($$.t, $1|$3); + } + +tlist: + types + { + $$ = $1.t; + if($1.c != CXXX) + diag(Z, "illegal combination of class 4: %s", cnames[$1.c]); + } + +ctlist: + types + { + lasttype = $1.t; + lastclass = $1.c; + } + +complex: + LSTRUCT ltag + { + dotag($2, TSTRUCT, 0); + $$ = $2->suetag; + } +| LSTRUCT ltag + { + dotag($2, TSTRUCT, autobn); + } + sbody + { + $$ = $2->suetag; + if($$->link != T) + diag(Z, "redeclare tag: %s", $2->name); + $$->link = $4; + sualign($$); + } +| LSTRUCT sbody + { + taggen++; + sprint(symb, "_%d_", taggen); + $$ = dotag(lookup(), TSTRUCT, autobn); + $$->link = $2; + sualign($$); + } +| LUNION ltag + { + dotag($2, TUNION, 0); + $$ = $2->suetag; + } +| LUNION ltag + { + dotag($2, TUNION, autobn); + } + sbody + { + $$ = $2->suetag; + if($$->link != T) + diag(Z, "redeclare tag: %s", $2->name); + $$->link = $4; + sualign($$); + } +| LUNION sbody + { + taggen++; + sprint(symb, "_%d_", taggen); + $$ = dotag(lookup(), TUNION, autobn); + $$->link = $2; + sualign($$); + } +| LENUM ltag + { + dotag($2, TENUM, 0); + $$ = $2->suetag; + if($$->link == T) + $$->link = types[TINT]; + $$ = $$->link; + } +| LENUM ltag + { + dotag($2, TENUM, autobn); + } + '{' + { + en.tenum = T; + en.cenum = T; + } + enum '}' + { + $$ = $2->suetag; + if($$->link != T) + diag(Z, "redeclare tag: %s", $2->name); + if(en.tenum == T) { + diag(Z, "enum type ambiguous: %s", $2->name); + en.tenum = types[TINT]; + } + $$->link = en.tenum; + $$ = en.tenum; + } +| LENUM '{' + { + en.tenum = T; + en.cenum = T; + } + enum '}' + { + $$ = en.tenum; + } +| LTYPE + { + $$ = tcopy($1->type); + } + +gctnlist: + gctname +| gctnlist gctname + { + $$ = typebitor($1, $2); + } + +zgnlist: + { + $$ = 0; + } +| zgnlist gname + { + $$ = typebitor($1, $2); + } + +gctname: + tname +| gname +| cname + +gcnlist: + gcname +| gcnlist gcname + { + $$ = typebitor($1, $2); + } + +gcname: + gname +| cname + +enum: + LNAME + { + doenum($1, Z); + } +| LNAME '=' expr + { + doenum($1, $3); + } +| enum ',' +| enum ',' enum + +tname: /* type words */ + LCHAR { $$ = BCHAR; } +| LSHORT { $$ = BSHORT; } +| LINT { $$ = BINT; } +| LLONG { $$ = BLONG; } +| LSIGNED { $$ = BSIGNED; } +| LUNSIGNED { $$ = BUNSIGNED; } +| LFLOAT { $$ = BFLOAT; } +| LDOUBLE { $$ = BDOUBLE; } +| LVOID { $$ = BVOID; } + +cname: /* class words */ + LAUTO { $$ = BAUTO; } +| LSTATIC { $$ = BSTATIC; } +| LEXTERN { $$ = BEXTERN; } +| LTYPEDEF { $$ = BTYPEDEF; } +| LTYPESTR { $$ = BTYPESTR; } +| LREGISTER { $$ = BREGISTER; } +| LINLINE { $$ = 0; } + +gname: /* garbage words */ + LCONSTNT { $$ = BCONSTNT; } +| LVOLATILE { $$ = BVOLATILE; } +| LRESTRICT { $$ = 0; } + +name: + LNAME + { + $$ = new(ONAME, Z, Z); + if($1->class == CLOCAL) + $1 = mkstatic($1); + $$->sym = $1; + $$->type = $1->type; + $$->etype = TVOID; + if($$->type != T) + $$->etype = $$->type->etype; + $$->xoffset = $1->offset; + $$->class = $1->class; + $1->aused = 1; + } +tag: + ltag + { + $$ = new(ONAME, Z, Z); + $$->sym = $1; + $$->type = $1->type; + $$->etype = TVOID; + if($$->type != T) + $$->etype = $$->type->etype; + $$->xoffset = $1->offset; + $$->class = $1->class; + } +ltag: + LNAME +| LTYPE +%% diff --git a/src/cmd/cc/com.c b/src/cmd/cc/com.c new file mode 100644 index 000000000..6e470ee64 --- /dev/null +++ b/src/cmd/cc/com.c @@ -0,0 +1,1385 @@ +// Inferno utils/cc/com.c +// http://code.google.com/p/inferno-os/source/browse/utils/cc/com.c +// +// Copyright © 1994-1999 Lucent Technologies Inc. All rights reserved. +// Portions Copyright © 1995-1997 C H Forsyth (forsyth@terzarima.net) +// Portions Copyright © 1997-1999 Vita Nuova Limited +// Portions Copyright © 2000-2007 Vita Nuova Holdings Limited (www.vitanuova.com) +// Portions Copyright © 2004,2006 Bruce Ellis +// Portions Copyright © 2005-2007 C H Forsyth (forsyth@terzarima.net) +// Revisions Copyright © 2000-2007 Lucent Technologies Inc. and others +// Portions Copyright © 2009 The Go Authors. All rights reserved. +// +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to deal +// in the Software without restriction, including without limitation the rights +// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +// copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +// THE SOFTWARE. + +#include +#include "cc.h" + +int compar(Node*, int); + +void +complex(Node *n) +{ + + if(n == Z) + return; + + nearln = n->lineno; + if(debug['t']) + if(n->op != OCONST) + prtree(n, "pre complex"); + if(tcom(n)) + return; + if(debug['t']) + if(n->op != OCONST) + prtree(n, "t complex"); + ccom(n); + if(debug['t']) + if(n->op != OCONST) + prtree(n, "c complex"); + acom(n); + if(debug['t']) + if(n->op != OCONST) + prtree(n, "a complex"); + xcom(n); + if(debug['t']) + if(n->op != OCONST) + prtree(n, "x complex"); +} + +/* + * evaluate types + * evaluate lvalues (addable == 1) + */ +enum +{ + ADDROF = 1<<0, + CASTOF = 1<<1, + ADDROP = 1<<2, +}; + +int +tcom(Node *n) +{ + + return tcomo(n, ADDROF); +} + +int +tcomo(Node *n, int f) +{ + Node *l, *r; + Type *t; + int o; + + if(n == Z) { + diag(Z, "Z in tcom"); + errorexit(); + } + n->addable = 0; + l = n->left; + r = n->right; + + switch(n->op) { + default: + diag(n, "unknown op in type complex: %O", n->op); + goto bad; + + case ODOTDOT: + /* + * tcom has already been called on this subtree + */ + *n = *n->left; + if(n->type == T) + goto bad; + break; + + case OCAST: + if(n->type == T) + break; + if(n->type->width == types[TLONG]->width) { + if(tcomo(l, ADDROF|CASTOF)) + goto bad; + } else + if(tcom(l)) + goto bad; + if(isfunct(n)) + break; + if(tcompat(n, l->type, n->type, tcast)) + goto bad; + break; + + case ORETURN: + if(l == Z) { + if(n->type->etype != TVOID) + diag(n, "null return of a typed function"); + break; + } + if(tcom(l)) + goto bad; + typeext(n->type, l); + if(tcompat(n, n->type, l->type, tasign)) + break; + constas(n, n->type, l->type); + if(!sametype(n->type, l->type)) { + l = new1(OCAST, l, Z); + l->type = n->type; + n->left = l; + } + break; + + case OASI: /* same as as, but no test for const */ + n->op = OAS; + o = tcom(l); + if(o | tcom(r)) + goto bad; + + typeext(l->type, r); + if(tlvalue(l) || tcompat(n, l->type, r->type, tasign)) + goto bad; + if(!sametype(l->type, r->type)) { + r = new1(OCAST, r, Z); + r->type = l->type; + n->right = r; + } + n->type = l->type; + break; + + case OAS: + o = tcom(l); + if(o | tcom(r)) + goto bad; + if(tlvalue(l)) + goto bad; + if(isfunct(n)) + break; + typeext(l->type, r); + if(tcompat(n, l->type, r->type, tasign)) + goto bad; + constas(n, l->type, r->type); + if(!sametype(l->type, r->type)) { + r = new1(OCAST, r, Z); + r->type = l->type; + n->right = r; + } + n->type = l->type; + break; + + case OASADD: + case OASSUB: + o = tcom(l); + if(o | tcom(r)) + goto bad; + if(tlvalue(l)) + goto bad; + if(isfunct(n)) + break; + typeext1(l->type, r); + if(tcompat(n, l->type, r->type, tasadd)) + goto bad; + constas(n, l->type, r->type); + t = l->type; + arith(n, 0); + while(n->left->op == OCAST) + n->left = n->left->left; + if(!sametype(t, n->type) && !mixedasop(t, n->type)) { + r = new1(OCAST, n->right, Z); + r->type = t; + n->right = r; + n->type = t; + } + break; + + case OASMUL: + case OASLMUL: + case OASDIV: + case OASLDIV: + o = tcom(l); + if(o | tcom(r)) + goto bad; + if(tlvalue(l)) + goto bad; + if(isfunct(n)) + break; + typeext1(l->type, r); + if(tcompat(n, l->type, r->type, tmul)) + goto bad; + constas(n, l->type, r->type); + t = l->type; + arith(n, 0); + while(n->left->op == OCAST) + n->left = n->left->left; + if(!sametype(t, n->type) && !mixedasop(t, n->type)) { + r = new1(OCAST, n->right, Z); + r->type = t; + n->right = r; + n->type = t; + } + if(typeu[n->type->etype]) { + if(n->op == OASDIV) + n->op = OASLDIV; + if(n->op == OASMUL) + n->op = OASLMUL; + } + break; + + case OASLSHR: + case OASASHR: + case OASASHL: + o = tcom(l); + if(o | tcom(r)) + goto bad; + if(tlvalue(l)) + goto bad; + if(isfunct(n)) + break; + if(tcompat(n, l->type, r->type, tand)) + goto bad; + n->type = l->type; + if(typeu[n->type->etype]) { + if(n->op == OASASHR) + n->op = OASLSHR; + } + break; + + case OASMOD: + case OASLMOD: + case OASOR: + case OASAND: + case OASXOR: + o = tcom(l); + if(o | tcom(r)) + goto bad; + if(tlvalue(l)) + goto bad; + if(isfunct(n)) + break; + if(tcompat(n, l->type, r->type, tand)) + goto bad; + t = l->type; + arith(n, 0); + while(n->left->op == OCAST) + n->left = n->left->left; + if(!sametype(t, n->type) && !mixedasop(t, n->type)) { + r = new1(OCAST, n->right, Z); + r->type = t; + n->right = r; + n->type = t; + } + if(typeu[n->type->etype]) { + if(n->op == OASMOD) + n->op = OASLMOD; + } + break; + + case OPREINC: + case OPREDEC: + case OPOSTINC: + case OPOSTDEC: + if(tcom(l)) + goto bad; + if(tlvalue(l)) + goto bad; + if(isfunct(n)) + break; + if(tcompat(n, l->type, types[TINT], tadd)) + goto bad; + n->type = l->type; + if(n->type->etype == TIND) + if(n->type->link->width < 1) + diag(n, "inc/dec of a void pointer"); + break; + + case OEQ: + case ONE: + o = tcom(l); + if(o | tcom(r)) + goto bad; + if(isfunct(n)) + break; + typeext(l->type, r); + typeext(r->type, l); + if(tcompat(n, l->type, r->type, trel)) + goto bad; + arith(n, 0); + n->type = types[TINT]; + break; + + case OLT: + case OGE: + case OGT: + case OLE: + o = tcom(l); + if(o | tcom(r)) + goto bad; + if(isfunct(n)) + break; + typeext1(l->type, r); + typeext1(r->type, l); + if(tcompat(n, l->type, r->type, trel)) + goto bad; + arith(n, 0); + if(typeu[n->type->etype]) + n->op = logrel[relindex(n->op)]; + n->type = types[TINT]; + break; + + case OCOND: + o = tcom(l); + o |= tcom(r->left); + if(o | tcom(r->right)) + goto bad; + if(r->right->type->etype == TIND && vconst(r->left) == 0) { + r->left->type = r->right->type; + r->left->vconst = 0; + } + if(r->left->type->etype == TIND && vconst(r->right) == 0) { + r->right->type = r->left->type; + r->right->vconst = 0; + } + if(sametype(r->right->type, r->left->type)) { + r->type = r->right->type; + n->type = r->type; + break; + } + if(tcompat(r, r->left->type, r->right->type, trel)) + goto bad; + arith(r, 0); + n->type = r->type; + break; + + case OADD: + o = tcom(l); + if(o | tcom(r)) + goto bad; + if(isfunct(n)) + break; + if(tcompat(n, l->type, r->type, tadd)) + goto bad; + arith(n, 1); + break; + + case OSUB: + o = tcom(l); + if(o | tcom(r)) + goto bad; + if(isfunct(n)) + break; + if(tcompat(n, l->type, r->type, tsub)) + goto bad; + arith(n, 1); + break; + + case OMUL: + case OLMUL: + case ODIV: + case OLDIV: + o = tcom(l); + if(o | tcom(r)) + goto bad; + if(isfunct(n)) + break; + if(tcompat(n, l->type, r->type, tmul)) + goto bad; + arith(n, 1); + if(typeu[n->type->etype]) { + if(n->op == ODIV) + n->op = OLDIV; + if(n->op == OMUL) + n->op = OLMUL; + } + break; + + case OLSHR: + case OASHL: + case OASHR: + o = tcom(l); + if(o | tcom(r)) + goto bad; + if(isfunct(n)) + break; + if(tcompat(n, l->type, r->type, tand)) + goto bad; + n->right = Z; + arith(n, 1); + n->right = new1(OCAST, r, Z); + n->right->type = types[TINT]; + if(typeu[n->type->etype]) + if(n->op == OASHR) + n->op = OLSHR; + break; + + case OAND: + case OOR: + case OXOR: + o = tcom(l); + if(o | tcom(r)) + goto bad; + if(isfunct(n)) + break; + if(tcompat(n, l->type, r->type, tand)) + goto bad; + arith(n, 1); + break; + + case OMOD: + case OLMOD: + o = tcom(l); + if(o | tcom(r)) + goto bad; + if(isfunct(n)) + break; + if(tcompat(n, l->type, r->type, tand)) + goto bad; + arith(n, 1); + if(typeu[n->type->etype]) + n->op = OLMOD; + break; + + case OPOS: + if(tcom(l)) + goto bad; + if(isfunct(n)) + break; + + r = l; + l = new(OCONST, Z, Z); + l->vconst = 0; + l->type = types[TINT]; + n->op = OADD; + n->right = r; + n->left = l; + + if(tcom(l)) + goto bad; + if(tcompat(n, l->type, r->type, tsub)) + goto bad; + arith(n, 1); + break; + + case ONEG: + if(tcom(l)) + goto bad; + if(isfunct(n)) + break; + + if(!machcap(n)) { + r = l; + l = new(OCONST, Z, Z); + l->vconst = 0; + l->type = types[TINT]; + n->op = OSUB; + n->right = r; + n->left = l; + + if(tcom(l)) + goto bad; + if(tcompat(n, l->type, r->type, tsub)) + goto bad; + } + arith(n, 1); + break; + + case OCOM: + if(tcom(l)) + goto bad; + if(isfunct(n)) + break; + + if(!machcap(n)) { + r = l; + l = new(OCONST, Z, Z); + l->vconst = -1; + l->type = types[TINT]; + n->op = OXOR; + n->right = r; + n->left = l; + + if(tcom(l)) + goto bad; + if(tcompat(n, l->type, r->type, tand)) + goto bad; + } + arith(n, 1); + break; + + case ONOT: + if(tcom(l)) + goto bad; + if(isfunct(n)) + break; + if(tcompat(n, T, l->type, tnot)) + goto bad; + n->type = types[TINT]; + break; + + case OANDAND: + case OOROR: + o = tcom(l); + if(o | tcom(r)) + goto bad; + if(tcompat(n, T, l->type, tnot) | + tcompat(n, T, r->type, tnot)) + goto bad; + n->type = types[TINT]; + break; + + case OCOMMA: + o = tcom(l); + if(o | tcom(r)) + goto bad; + n->type = r->type; + break; + + + case OSIGN: /* extension signof(type) returns a hash */ + if(l != Z) { + if(l->op != OSTRING && l->op != OLSTRING) + if(tcomo(l, 0)) + goto bad; + if(l->op == OBIT) { + diag(n, "signof bitfield"); + goto bad; + } + n->type = l->type; + } + if(n->type == T) + goto bad; + if(n->type->width < 0) { + diag(n, "signof undefined type"); + goto bad; + } + n->op = OCONST; + n->left = Z; + n->right = Z; + n->vconst = convvtox(signature(n->type), TULONG); + n->type = types[TULONG]; + break; + + case OSIZE: + if(l != Z) { + if(l->op != OSTRING && l->op != OLSTRING) + if(tcomo(l, 0)) + goto bad; + if(l->op == OBIT) { + diag(n, "sizeof bitfield"); + goto bad; + } + n->type = l->type; + } + if(n->type == T) + goto bad; + if(n->type->width <= 0) { + diag(n, "sizeof undefined type"); + goto bad; + } + if(n->type->etype == TFUNC) { + diag(n, "sizeof function"); + goto bad; + } + n->op = OCONST; + n->left = Z; + n->right = Z; + n->vconst = convvtox(n->type->width, TINT); + n->type = types[TINT]; + break; + + case OFUNC: + o = tcomo(l, 0); + if(o) + goto bad; + if(l->type->etype == TIND && l->type->link->etype == TFUNC) { + l = new1(OIND, l, Z); + l->type = l->left->type->link; + n->left = l; + } + if(tcompat(n, T, l->type, tfunct)) + goto bad; + if(o | tcoma(l, r, l->type->down, 1)) + goto bad; + n->type = l->type->link; + if(!debug['B']) + if(l->type->down == T || l->type->down->etype == TOLD) { + nerrors--; + diag(n, "function args not checked: %F", l); + } + dpcheck(n); + break; + + case ONAME: + if(n->type == T) { + diag(n, "name not declared: %F", n); + goto bad; + } + if(n->type->etype == TENUM) { + n->op = OCONST; + n->type = n->sym->tenum; + if(!typefd[n->type->etype]) + n->vconst = n->sym->vconst; + else + n->fconst = n->sym->fconst; + break; + } + n->addable = 1; + if(n->class == CEXREG) { + n->op = OREGISTER; + // on 386 or amd64, "extern register" generates + // memory references relative to the + // gs or fs segment. + if(thechar == '8' || thechar == '6') // [sic] + n->op = OEXREG; + n->reg = n->sym->offset; + n->xoffset = 0; + break; + } + break; + + case OLSTRING: + if(n->type->link != types[TUSHORT]) { + o = outstring(0, 0); + while(o & 3) { + ushort a[1]; + a[0] = 0; + outlstring(a, sizeof(ushort)); + o = outlstring(0, 0); + } + } + n->op = ONAME; + n->xoffset = outlstring(n->rstring, n->type->width); + n->addable = 1; + break; + + case OSTRING: + if(n->type->link != types[TCHAR]) { + o = outstring(0, 0); + while(o & 3) { + outstring("", 1); + o = outstring(0, 0); + } + } + n->op = ONAME; + n->xoffset = outstring(n->cstring, n->type->width); + n->addable = 1; + break; + + case OCONST: + break; + + case ODOT: + if(tcom(l)) + goto bad; + if(tcompat(n, T, l->type, tdot)) + goto bad; + if(tcomd(n)) + goto bad; + break; + + case OADDR: + if(tcomo(l, ADDROP)) + goto bad; + if(tlvalue(l)) + goto bad; + if(l->type->nbits) { + diag(n, "address of a bit field"); + goto bad; + } + if(l->op == OREGISTER) { + diag(n, "address of a register"); + goto bad; + } + n->type = typ(TIND, l->type); + n->type->width = types[TIND]->width; + break; + + case OIND: + if(tcom(l)) + goto bad; + if(tcompat(n, T, l->type, tindir)) + goto bad; + n->type = l->type->link; + n->addable = 1; + break; + + case OSTRUCT: + if(tcomx(n)) + goto bad; + break; + } + t = n->type; + if(t == T) + goto bad; + if(t->width < 0) { + snap(t); + if(t->width < 0) { + if(typesu[t->etype] && t->tag) + diag(n, "structure not fully declared %s", t->tag->name); + else + diag(n, "structure not fully declared"); + goto bad; + } + } + if(typeaf[t->etype]) { + if(f & ADDROF) + goto addaddr; + if(f & ADDROP) + warn(n, "address of array/func ignored"); + } + return 0; + +addaddr: + if(tlvalue(n)) + goto bad; + l = new1(OXXX, Z, Z); + *l = *n; + n->op = OADDR; + if(l->type->etype == TARRAY) + l->type = l->type->link; + n->left = l; + n->right = Z; + n->addable = 0; + n->type = typ(TIND, l->type); + n->type->width = types[TIND]->width; + return 0; + +bad: + n->type = T; + return 1; +} + +int +tcoma(Node *l, Node *n, Type *t, int f) +{ + Node *n1; + int o; + + if(t != T) + if(t->etype == TOLD || t->etype == TDOT) /* .../old in prototype */ + t = T; + if(n == Z) { + if(t != T && !sametype(t, types[TVOID])) { + diag(n, "not enough function arguments: %F", l); + return 1; + } + return 0; + } + if(n->op == OLIST) { + o = tcoma(l, n->left, t, 0); + if(t != T) { + t = t->down; + if(t == T) + t = types[TVOID]; + } + return o | tcoma(l, n->right, t, 1); + } + if(f && t != T) + tcoma(l, Z, t->down, 0); + if(tcom(n) || tcompat(n, T, n->type, targ)) + return 1; + if(sametype(t, types[TVOID])) { + diag(n, "too many function arguments: %F", l); + return 1; + } + if(t != T) { + typeext(t, n); + if(stcompat(nodproto, t, n->type, tasign)) { + diag(l, "argument prototype mismatch \"%T\" for \"%T\": %F", + n->type, t, l); + return 1; + } +// switch(t->etype) { +// case TCHAR: +// case TSHORT: +// t = types[TINT]; +// break; +// +// case TUCHAR: +// case TUSHORT: +// t = types[TUINT]; +// break; +// } + } else { + switch(n->type->etype) { + case TCHAR: + case TSHORT: + t = types[TINT]; + break; + + case TUCHAR: + case TUSHORT: + t = types[TUINT]; + break; + + case TFLOAT: + t = types[TDOUBLE]; + } + } + + if(t != T && !sametype(t, n->type)) { + n1 = new1(OXXX, Z, Z); + *n1 = *n; + n->op = OCAST; + n->left = n1; + n->right = Z; + n->type = t; + n->addable = 0; + } + return 0; +} + +int +tcomd(Node *n) +{ + Type *t; + int32 o; + + o = 0; + t = dotsearch(n->sym, n->left->type->link, n, &o); + if(t == T) { + diag(n, "not a member of struct/union: %F", n); + return 1; + } + makedot(n, t, o); + return 0; +} + +int +tcomx(Node *n) +{ + Type *t; + Node *l, *r, **ar, **al; + int e; + + e = 0; + if(n->type->etype != TSTRUCT) { + diag(n, "constructor must be a structure"); + return 1; + } + l = invert(n->left); + n->left = l; + al = &n->left; + for(t = n->type->link; t != T; t = t->down) { + if(l == Z) { + diag(n, "constructor list too short"); + return 1; + } + if(l->op == OLIST) { + r = l->left; + ar = &l->left; + al = &l->right; + l = l->right; + } else { + r = l; + ar = al; + l = Z; + } + if(tcom(r)) + e++; + typeext(t, r); + if(tcompat(n, t, r->type, tasign)) + e++; + constas(n, t, r->type); + if(!e && !sametype(t, r->type)) { + r = new1(OCAST, r, Z); + r->type = t; + *ar = r; + } + } + if(l != Z) { + diag(n, "constructor list too long"); + return 1; + } + return e; +} + +int +tlvalue(Node *n) +{ + + if(!n->addable) { + diag(n, "not an l-value"); + return 1; + } + return 0; +} + +/* + * general rewrite + * (IND(ADDR x)) ==> x + * (ADDR(IND x)) ==> x + * remove some zero operands + * remove no op casts + * evaluate constants + */ +void +ccom(Node *n) +{ + Node *l, *r; + int t; + +loop: + if(n == Z) + return; + l = n->left; + r = n->right; + switch(n->op) { + + case OAS: + case OASXOR: + case OASAND: + case OASOR: + case OASMOD: + case OASLMOD: + case OASLSHR: + case OASASHR: + case OASASHL: + case OASDIV: + case OASLDIV: + case OASMUL: + case OASLMUL: + case OASSUB: + case OASADD: + ccom(l); + ccom(r); + if(n->op == OASLSHR || n->op == OASASHR || n->op == OASASHL) + if(r->op == OCONST) { + t = n->type->width * 8; /* bits per byte */ + if(r->vconst >= t || r->vconst < 0) + warn(n, "stupid shift: %lld", r->vconst); + } + break; + + case OCAST: + ccom(l); + if(l->op == OCONST) { + evconst(n); + if(n->op == OCONST) + break; + } + if(nocast(l->type, n->type)) { + l->type = n->type; + *n = *l; + } + break; + + case OCOND: + ccom(l); + ccom(r); + if(l->op == OCONST) + if(vconst(l) == 0) + *n = *r->right; + else + *n = *r->left; + break; + + case OREGISTER: + case OINDREG: + case OCONST: + case ONAME: + break; + + case OADDR: + ccom(l); + l->etype = TVOID; + if(l->op == OIND) { + l->left->type = n->type; + *n = *l->left; + break; + } + goto common; + + case OIND: + ccom(l); + if(l->op == OADDR) { + l->left->type = n->type; + *n = *l->left; + break; + } + goto common; + + case OEQ: + case ONE: + + case OLE: + case OGE: + case OLT: + case OGT: + + case OLS: + case OHS: + case OLO: + case OHI: + ccom(l); + ccom(r); + if(compar(n, 0) || compar(n, 1)) + break; + relcon(l, r); + relcon(r, l); + goto common; + + case OASHR: + case OASHL: + case OLSHR: + ccom(l); + if(vconst(l) == 0 && !side(r)) { + *n = *l; + break; + } + ccom(r); + if(vconst(r) == 0) { + *n = *l; + break; + } + if(r->op == OCONST) { + t = n->type->width * 8; /* bits per byte */ + if(r->vconst >= t || r->vconst <= -t) + warn(n, "stupid shift: %lld", r->vconst); + } + goto common; + + case OMUL: + case OLMUL: + ccom(l); + t = vconst(l); + if(t == 0 && !side(r)) { + *n = *l; + break; + } + if(t == 1) { + *n = *r; + goto loop; + } + ccom(r); + t = vconst(r); + if(t == 0 && !side(l)) { + *n = *r; + break; + } + if(t == 1) { + *n = *l; + break; + } + goto common; + + case ODIV: + case OLDIV: + ccom(l); + if(vconst(l) == 0 && !side(r)) { + *n = *l; + break; + } + ccom(r); + t = vconst(r); + if(t == 0) { + diag(n, "divide check"); + *n = *r; + break; + } + if(t == 1) { + *n = *l; + break; + } + goto common; + + case OSUB: + ccom(r); + if(r->op == OCONST) { + if(typefd[r->type->etype]) { + n->op = OADD; + r->fconst = -r->fconst; + goto loop; + } else { + n->op = OADD; + r->vconst = -r->vconst; + goto loop; + } + } + ccom(l); + goto common; + + case OXOR: + case OOR: + case OADD: + ccom(l); + if(vconst(l) == 0) { + *n = *r; + goto loop; + } + ccom(r); + if(vconst(r) == 0) { + *n = *l; + break; + } + goto commute; + + case OAND: + ccom(l); + ccom(r); + if(vconst(l) == 0 && !side(r)) { + *n = *l; + break; + } + if(vconst(r) == 0 && !side(l)) { + *n = *r; + break; + } + + commute: + /* look for commutative constant */ + if(r->op == OCONST) { + if(l->op == n->op) { + if(l->left->op == OCONST) { + n->right = l->right; + l->right = r; + goto loop; + } + if(l->right->op == OCONST) { + n->right = l->left; + l->left = r; + goto loop; + } + } + } + if(l->op == OCONST) { + if(r->op == n->op) { + if(r->left->op == OCONST) { + n->left = r->right; + r->right = l; + goto loop; + } + if(r->right->op == OCONST) { + n->left = r->left; + r->left = l; + goto loop; + } + } + } + goto common; + + case OANDAND: + ccom(l); + if(vconst(l) == 0) { + *n = *l; + break; + } + ccom(r); + goto common; + + case OOROR: + ccom(l); + if(l->op == OCONST && l->vconst != 0) { + *n = *l; + n->vconst = 1; + break; + } + ccom(r); + goto common; + + default: + if(l != Z) + ccom(l); + if(r != Z) + ccom(r); + common: + if(l != Z) + if(l->op != OCONST) + break; + if(r != Z) + if(r->op != OCONST) + break; + evconst(n); + } +} + +/* OEQ, ONE, OLE, OLS, OLT, OLO, OGE, OHS, OGT, OHI */ +static char *cmps[12] = +{ + "==", "!=", "<=", "<=", "<", "<", ">=", ">=", ">", ">", +}; + +/* 128-bit numbers */ +typedef struct Big Big; +struct Big +{ + vlong a; + uvlong b; +}; +static int +cmp(Big x, Big y) +{ + if(x.a != y.a){ + if(x.a < y.a) + return -1; + return 1; + } + if(x.b != y.b){ + if(x.b < y.b) + return -1; + return 1; + } + return 0; +} +static Big +add(Big x, int y) +{ + uvlong ob; + + ob = x.b; + x.b += y; + if(y > 0 && x.b < ob) + x.a++; + if(y < 0 && x.b > ob) + x.a--; + return x; +} + +Big +big(vlong a, uvlong b) +{ + Big x; + + x.a = a; + x.b = b; + return x; +} + +int +compar(Node *n, int reverse) +{ + Big lo, hi, x; + int op; + char xbuf[40], cmpbuf[50]; + Node *l, *r; + Type *lt, *rt; + + /* + * The point of this function is to diagnose comparisons + * that can never be true or that look misleading because + * of the `usual arithmetic conversions'. As an example + * of the latter, if x is a ulong, then if(x <= -1) really means + * if(x <= 0xFFFFFFFF), while if(x <= -1LL) really means + * what it says (but 8c compiles it wrong anyway). + */ + + if(reverse){ + r = n->left; + l = n->right; + op = comrel[relindex(n->op)]; + }else{ + l = n->left; + r = n->right; + op = n->op; + } + + /* + * Skip over left casts to find out the original expression range. + */ + while(l->op == OCAST) + l = l->left; + if(l->op == OCONST) + return 0; + lt = l->type; + if(l->op == ONAME && l->sym->type){ + lt = l->sym->type; + if(lt->etype == TARRAY) + lt = lt->link; + } + if(lt == T) + return 0; + if(lt->etype == TXXX || lt->etype > TUVLONG) + return 0; + + /* + * Skip over the right casts to find the on-screen value. + */ + if(r->op != OCONST) + return 0; + while(r->oldop == OCAST && !r->xcast) + r = r->left; + rt = r->type; + if(rt == T) + return 0; + + x.b = r->vconst; + x.a = 0; + if((rt->etype&1) && r->vconst < 0) /* signed negative */ + x.a = ~0ULL; + + if((lt->etype&1)==0){ + /* unsigned */ + lo = big(0, 0); + if(lt->width == 8) + hi = big(0, ~0ULL); + else + hi = big(0, (1LL<<(l->type->width*8))-1); + }else{ + lo = big(~0ULL, -(1LL<<(l->type->width*8-1))); + hi = big(0, (1LL<<(l->type->width*8-1))-1); + } + + switch(op){ + case OLT: + case OLO: + case OGE: + case OHS: + if(cmp(x, lo) <= 0) + goto useless; + if(cmp(x, add(hi, 1)) >= 0) + goto useless; + break; + case OLE: + case OLS: + case OGT: + case OHI: + if(cmp(x, add(lo, -1)) <= 0) + goto useless; + if(cmp(x, hi) >= 0) + goto useless; + break; + case OEQ: + case ONE: + /* + * Don't warn about comparisons if the expression + * is as wide as the value: the compiler-supplied casts + * will make both outcomes possible. + */ + if(lt->width >= rt->width && debug['w'] < 2) + return 0; + if(cmp(x, lo) < 0 || cmp(x, hi) > 0) + goto useless; + break; + } + return 0; + +useless: + if((x.a==0 && x.b<=9) || (x.a==~0LL && x.b >= -9ULL)) + snprint(xbuf, sizeof xbuf, "%lld", x.b); + else if(x.a == 0) + snprint(xbuf, sizeof xbuf, "%#llux", x.b); + else + snprint(xbuf, sizeof xbuf, "%#llx", x.b); + if(reverse) + snprint(cmpbuf, sizeof cmpbuf, "%s %s %T", + xbuf, cmps[relindex(n->op)], lt); + else + snprint(cmpbuf, sizeof cmpbuf, "%T %s %s", + lt, cmps[relindex(n->op)], xbuf); + warn(n, "useless or misleading comparison: %s", cmpbuf); + return 0; +} + diff --git a/src/cmd/cc/com64.c b/src/cmd/cc/com64.c new file mode 100644 index 000000000..fb7a3f750 --- /dev/null +++ b/src/cmd/cc/com64.c @@ -0,0 +1,644 @@ +// Inferno utils/cc/com64.c +// http://code.google.com/p/inferno-os/source/browse/utils/cc/com64.c +// +// Copyright © 1994-1999 Lucent Technologies Inc. All rights reserved. +// Portions Copyright © 1995-1997 C H Forsyth (forsyth@terzarima.net) +// Portions Copyright © 1997-1999 Vita Nuova Limited +// Portions Copyright © 2000-2007 Vita Nuova Holdings Limited (www.vitanuova.com) +// Portions Copyright © 2004,2006 Bruce Ellis +// Portions Copyright © 2005-2007 C H Forsyth (forsyth@terzarima.net) +// Revisions Copyright © 2000-2007 Lucent Technologies Inc. and others +// Portions Copyright © 2009 The Go Authors. All rights reserved. +// +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to deal +// in the Software without restriction, including without limitation the rights +// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +// copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +// THE SOFTWARE. + +#include +#include "cc.h" + +/* + * this is machine depend, but it is totally + * common on all of the 64-bit symulating machines. + */ + +#define FNX 100 /* botch -- redefinition */ + +Node* nodaddv; +Node* nodsubv; +Node* nodmulv; +Node* noddivv; +Node* noddivvu; +Node* nodmodv; +Node* nodmodvu; +Node* nodlshv; +Node* nodrshav; +Node* nodrshlv; +Node* nodandv; +Node* nodorv; +Node* nodxorv; +Node* nodnegv; +Node* nodcomv; + +Node* nodtestv; +Node* nodeqv; +Node* nodnev; +Node* nodlev; +Node* nodltv; +Node* nodgev; +Node* nodgtv; +Node* nodhiv; +Node* nodhsv; +Node* nodlov; +Node* nodlsv; + +Node* nodf2v; +Node* nodd2v; +Node* nodp2v; +Node* nodsi2v; +Node* nodui2v; +Node* nodsl2v; +Node* nodul2v; +Node* nodsh2v; +Node* noduh2v; +Node* nodsc2v; +Node* noduc2v; + +Node* nodv2f; +Node* nodv2d; +Node* nodv2ui; +Node* nodv2si; +Node* nodv2ul; +Node* nodv2sl; +Node* nodv2uh; +Node* nodv2sh; +Node* nodv2uc; +Node* nodv2sc; + +Node* nodvpp; +Node* nodppv; +Node* nodvmm; +Node* nodmmv; + +Node* nodvasop; + +char etconv[NTYPE]; /* for _vasop */ +Init initetconv[] = +{ + TCHAR, 1, 0, + TUCHAR, 2, 0, + TSHORT, 3, 0, + TUSHORT, 4, 0, + TLONG, 5, 0, + TULONG, 6, 0, + TVLONG, 7, 0, + TUVLONG, 8, 0, + TINT, 9, 0, + TUINT, 10, 0, + -1, 0, 0, +}; + +Node* +fvn(char *name, int type) +{ + Node *n; + + n = new(ONAME, Z, Z); + n->sym = slookup(name); + n->sym->sig = SIGINTERN; + if(fntypes[type] == 0) + fntypes[type] = typ(TFUNC, types[type]); + n->type = fntypes[type]; + n->etype = type; + n->class = CGLOBL; + n->addable = 10; + n->complex = 0; + return n; +} + +void +com64init(void) +{ + Init *p; + + nodaddv = fvn("_addv", TVLONG); + nodsubv = fvn("_subv", TVLONG); + nodmulv = fvn("_mulv", TVLONG); + noddivv = fvn("_divv", TVLONG); + noddivvu = fvn("_divvu", TVLONG); + nodmodv = fvn("_modv", TVLONG); + nodmodvu = fvn("_modvu", TVLONG); + nodlshv = fvn("_lshv", TVLONG); + nodrshav = fvn("_rshav", TVLONG); + nodrshlv = fvn("_rshlv", TVLONG); + nodandv = fvn("_andv", TVLONG); + nodorv = fvn("_orv", TVLONG); + nodxorv = fvn("_xorv", TVLONG); + nodnegv = fvn("_negv", TVLONG); + nodcomv = fvn("_comv", TVLONG); + + nodtestv = fvn("_testv", TLONG); + nodeqv = fvn("_eqv", TLONG); + nodnev = fvn("_nev", TLONG); + nodlev = fvn("_lev", TLONG); + nodltv = fvn("_ltv", TLONG); + nodgev = fvn("_gev", TLONG); + nodgtv = fvn("_gtv", TLONG); + nodhiv = fvn("_hiv", TLONG); + nodhsv = fvn("_hsv", TLONG); + nodlov = fvn("_lov", TLONG); + nodlsv = fvn("_lsv", TLONG); + + nodf2v = fvn("_f2v", TVLONG); + nodd2v = fvn("_d2v", TVLONG); + nodp2v = fvn("_p2v", TVLONG); + nodsi2v = fvn("_si2v", TVLONG); + nodui2v = fvn("_ui2v", TVLONG); + nodsl2v = fvn("_sl2v", TVLONG); + nodul2v = fvn("_ul2v", TVLONG); + nodsh2v = fvn("_sh2v", TVLONG); + noduh2v = fvn("_uh2v", TVLONG); + nodsc2v = fvn("_sc2v", TVLONG); + noduc2v = fvn("_uc2v", TVLONG); + + nodv2f = fvn("_v2f", TFLOAT); + nodv2d = fvn("_v2d", TDOUBLE); + nodv2sl = fvn("_v2sl", TLONG); + nodv2ul = fvn("_v2ul", TULONG); + nodv2si = fvn("_v2si", TINT); + nodv2ui = fvn("_v2ui", TUINT); + nodv2sh = fvn("_v2sh", TSHORT); + nodv2uh = fvn("_v2ul", TUSHORT); + nodv2sc = fvn("_v2sc", TCHAR); + nodv2uc = fvn("_v2uc", TUCHAR); + + nodvpp = fvn("_vpp", TVLONG); + nodppv = fvn("_ppv", TVLONG); + nodvmm = fvn("_vmm", TVLONG); + nodmmv = fvn("_mmv", TVLONG); + + nodvasop = fvn("_vasop", TVLONG); + + for(p = initetconv; p->code >= 0; p++) + etconv[p->code] = p->value; +} + +int +com64(Node *n) +{ + Node *l, *r, *a, *t; + int lv, rv; + + if(n->type == 0) + return 0; + + l = n->left; + r = n->right; + + lv = 0; + if(l && l->type && typev[l->type->etype]) + lv = 1; + rv = 0; + if(r && r->type && typev[r->type->etype]) + rv = 1; + + if(lv) { + switch(n->op) { + case OEQ: + a = nodeqv; + goto setbool; + case ONE: + a = nodnev; + goto setbool; + case OLE: + a = nodlev; + goto setbool; + case OLT: + a = nodltv; + goto setbool; + case OGE: + a = nodgev; + goto setbool; + case OGT: + a = nodgtv; + goto setbool; + case OHI: + a = nodhiv; + goto setbool; + case OHS: + a = nodhsv; + goto setbool; + case OLO: + a = nodlov; + goto setbool; + case OLS: + a = nodlsv; + goto setbool; + + case OANDAND: + case OOROR: + if(machcap(n)) + return 1; + + if(rv) { + r = new(OFUNC, nodtestv, r); + n->right = r; + r->complex = FNX; + r->op = OFUNC; + r->type = types[TLONG]; + } + + case OCOND: + case ONOT: + if(machcap(n)) + return 1; + + l = new(OFUNC, nodtestv, l); + n->left = l; + l->complex = FNX; + l->op = OFUNC; + l->type = types[TLONG]; + n->complex = FNX; + return 1; + } + } + + if(rv) { + if(machcap(n)) + return 1; + switch(n->op) { + case OANDAND: + case OOROR: + r = new(OFUNC, nodtestv, r); + n->right = r; + r->complex = FNX; + r->op = OFUNC; + r->type = types[TLONG]; + return 1; + } + } + + if(typev[n->type->etype]) { + if(machcap(n)) + return 1; + switch(n->op) { + default: + diag(n, "unknown vlong %O", n->op); + case OFUNC: + n->complex = FNX; + case ORETURN: + case OAS: + case OIND: + return 1; + case OADD: + a = nodaddv; + goto setbop; + case OSUB: + a = nodsubv; + goto setbop; + case OMUL: + case OLMUL: + a = nodmulv; + goto setbop; + case ODIV: + a = noddivv; + goto setbop; + case OLDIV: + a = noddivvu; + goto setbop; + case OMOD: + a = nodmodv; + goto setbop; + case OLMOD: + a = nodmodvu; + goto setbop; + case OASHL: + a = nodlshv; + goto setbop; + case OASHR: + a = nodrshav; + goto setbop; + case OLSHR: + a = nodrshlv; + goto setbop; + case OAND: + a = nodandv; + goto setbop; + case OOR: + a = nodorv; + goto setbop; + case OXOR: + a = nodxorv; + goto setbop; + case OPOSTINC: + a = nodvpp; + goto setvinc; + case OPOSTDEC: + a = nodvmm; + goto setvinc; + case OPREINC: + a = nodppv; + goto setvinc; + case OPREDEC: + a = nodmmv; + goto setvinc; + case ONEG: + a = nodnegv; + goto setfnx; + case OCOM: + a = nodcomv; + goto setfnx; + case OCAST: + switch(l->type->etype) { + case TCHAR: + a = nodsc2v; + goto setfnxl; + case TUCHAR: + a = noduc2v; + goto setfnxl; + case TSHORT: + a = nodsh2v; + goto setfnxl; + case TUSHORT: + a = noduh2v; + goto setfnxl; + case TINT: + a = nodsi2v; + goto setfnx; + case TUINT: + a = nodui2v; + goto setfnx; + case TLONG: + a = nodsl2v; + goto setfnx; + case TULONG: + a = nodul2v; + goto setfnx; + case TFLOAT: + a = nodf2v; + goto setfnx; + case TDOUBLE: + a = nodd2v; + goto setfnx; + case TIND: + a = nodp2v; + goto setfnx; + } + diag(n, "unknown %T->vlong cast", l->type); + return 1; + case OASADD: + a = nodaddv; + goto setasop; + case OASSUB: + a = nodsubv; + goto setasop; + case OASMUL: + case OASLMUL: + a = nodmulv; + goto setasop; + case OASDIV: + a = noddivv; + goto setasop; + case OASLDIV: + a = noddivvu; + goto setasop; + case OASMOD: + a = nodmodv; + goto setasop; + case OASLMOD: + a = nodmodvu; + goto setasop; + case OASASHL: + a = nodlshv; + goto setasop; + case OASASHR: + a = nodrshav; + goto setasop; + case OASLSHR: + a = nodrshlv; + goto setasop; + case OASAND: + a = nodandv; + goto setasop; + case OASOR: + a = nodorv; + goto setasop; + case OASXOR: + a = nodxorv; + goto setasop; + } + } + + if(typefd[n->type->etype] && l && l->op == OFUNC) { + switch(n->op) { + case OASADD: + case OASSUB: + case OASMUL: + case OASLMUL: + case OASDIV: + case OASLDIV: + case OASMOD: + case OASLMOD: + case OASASHL: + case OASASHR: + case OASLSHR: + case OASAND: + case OASOR: + case OASXOR: + if(l->right && typev[l->right->etype]) { + diag(n, "sorry float vlong not implemented\n"); + } + } + } + + if(n->op == OCAST) { + if(l->type && typev[l->type->etype]) { + if(machcap(n)) + return 1; + switch(n->type->etype) { + case TDOUBLE: + a = nodv2d; + goto setfnx; + case TFLOAT: + a = nodv2f; + goto setfnx; + case TLONG: + a = nodv2sl; + goto setfnx; + case TULONG: + a = nodv2ul; + goto setfnx; + case TINT: + a = nodv2si; + goto setfnx; + case TUINT: + a = nodv2ui; + goto setfnx; + case TSHORT: + a = nodv2sh; + goto setfnx; + case TUSHORT: + a = nodv2uh; + goto setfnx; + case TCHAR: + a = nodv2sc; + goto setfnx; + case TUCHAR: + a = nodv2uc; + goto setfnx; + case TIND: // small pun here + a = nodv2ul; + goto setfnx; + } + diag(n, "unknown vlong->%T cast", n->type); + return 1; + } + } + + return 0; + +setbop: + n->left = a; + n->right = new(OLIST, l, r); + n->complex = FNX; + n->op = OFUNC; + return 1; + +setfnxl: + l = new(OCAST, l, 0); + l->type = types[TLONG]; + l->complex = l->left->complex; + +setfnx: + n->left = a; + n->right = l; + n->complex = FNX; + n->op = OFUNC; + return 1; + +setvinc: + n->left = a; + l = new(OADDR, l, Z); + l->type = typ(TIND, l->left->type); + n->right = new(OLIST, l, r); + n->complex = FNX; + n->op = OFUNC; + return 1; + +setbool: + if(machcap(n)) + return 1; + n->left = a; + n->right = new(OLIST, l, r); + n->complex = FNX; + n->op = OFUNC; + n->type = types[TLONG]; + return 1; + +setasop: + if(l->op == OFUNC) { + l = l->right; + goto setasop; + } + + t = new(OCONST, 0, 0); + t->vconst = etconv[l->type->etype]; + t->type = types[TLONG]; + t->addable = 20; + r = new(OLIST, t, r); + + t = new(OADDR, a, 0); + t->type = typ(TIND, a->type); + r = new(OLIST, t, r); + + t = new(OADDR, l, 0); + t->type = typ(TIND, l->type); + r = new(OLIST, t, r); + + n->left = nodvasop; + n->right = r; + n->complex = FNX; + n->op = OFUNC; + + return 1; +} + +void +bool64(Node *n) +{ + Node *n1; + + if(machcap(Z)) + return; + if(typev[n->type->etype]) { + n1 = new(OXXX, 0, 0); + *n1 = *n; + + n->right = n1; + n->left = nodtestv; + n->complex = FNX; + n->addable = 0; + n->op = OFUNC; + n->type = types[TLONG]; + } +} + +/* + * more machine depend stuff. + * this is common for 8,16,32,64 bit machines. + * this is common for ieee machines. + */ +double +convvtof(vlong v) +{ + double d; + + d = v; /* BOTCH */ + return d; +} + +vlong +convftov(double d) +{ + vlong v; + + + v = d; /* BOTCH */ + return v; +} + +double +convftox(double d, int et) +{ + + if(!typefd[et]) + diag(Z, "bad type in castftox %s", tnames[et]); + return d; +} + +vlong +convvtox(vlong c, int et) +{ + int n; + + n = 8 * ewidth[et]; + c &= MASK(n); + if(!typeu[et]) + if(c & SIGN(n)) + c |= ~MASK(n); + return c; +} diff --git a/src/cmd/cc/dcl.c b/src/cmd/cc/dcl.c new file mode 100644 index 000000000..d624bf247 --- /dev/null +++ b/src/cmd/cc/dcl.c @@ -0,0 +1,1669 @@ +// Inferno utils/cc/dcl.c +// http://code.google.com/p/inferno-os/source/browse/utils/cc/dcl.c +// +// Copyright © 1994-1999 Lucent Technologies Inc. All rights reserved. +// Portions Copyright © 1995-1997 C H Forsyth (forsyth@terzarima.net) +// Portions Copyright © 1997-1999 Vita Nuova Limited +// Portions Copyright © 2000-2007 Vita Nuova Holdings Limited (www.vitanuova.com) +// Portions Copyright © 2004,2006 Bruce Ellis +// Portions Copyright © 2005-2007 C H Forsyth (forsyth@terzarima.net) +// Revisions Copyright © 2000-2007 Lucent Technologies Inc. and others +// Portions Copyright © 2009 The Go Authors. All rights reserved. +// +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to deal +// in the Software without restriction, including without limitation the rights +// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +// copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +// THE SOFTWARE. + +#include +#include "cc.h" + +Node* +dodecl(void (*f)(int,Type*,Sym*), int c, Type *t, Node *n) +{ + Sym *s; + Node *n1; + int32 v; + + nearln = lineno; + lastfield = 0; + +loop: + if(n != Z) + switch(n->op) { + default: + diag(n, "unknown declarator: %O", n->op); + break; + + case OARRAY: + t = typ(TARRAY, t); + t->width = 0; + n1 = n->right; + n = n->left; + if(n1 != Z) { + complex(n1); + v = -1; + if(n1->op == OCONST) + v = n1->vconst; + if(v <= 0) { + diag(n, "array size must be a positive constant"); + v = 1; + } + t->width = v * t->link->width; + } + goto loop; + + case OIND: + t = typ(TIND, t); + t->garb = n->garb; + n = n->left; + goto loop; + + case OFUNC: + t = typ(TFUNC, t); + t->down = fnproto(n); + n = n->left; + goto loop; + + case OBIT: + n1 = n->right; + complex(n1); + lastfield = -1; + if(n1->op == OCONST) + lastfield = n1->vconst; + if(lastfield < 0) { + diag(n, "field width must be non-negative constant"); + lastfield = 1; + } + if(lastfield == 0) { + lastbit = 0; + firstbit = 1; + if(n->left != Z) { + diag(n, "zero width named field"); + lastfield = 1; + } + } + if(!typei[t->etype]) { + diag(n, "field type must be int-like"); + t = types[TINT]; + lastfield = 1; + } + if(lastfield > tfield->width*8) { + diag(n, "field width larger than field unit"); + lastfield = 1; + } + lastbit += lastfield; + if(lastbit > tfield->width*8) { + lastbit = lastfield; + firstbit = 1; + } + n = n->left; + goto loop; + + case ONAME: + if(f == NODECL) + break; + s = n->sym; + (*f)(c, t, s); + if(s->class == CLOCAL) + s = mkstatic(s); + firstbit = 0; + n->sym = s; + n->type = s->type; + n->xoffset = s->offset; + n->class = s->class; + n->etype = TVOID; + if(n->type != T) + n->etype = n->type->etype; + if(debug['d']) + dbgdecl(s); + acidvar(s); + godefvar(s); + s->varlineno = lineno; + break; + } + lastdcl = t; + return n; +} + +Sym* +mkstatic(Sym *s) +{ + Sym *s1; + + if(s->class != CLOCAL) + return s; + snprint(symb, NSYMB, "%s$%d", s->name, s->block); + s1 = lookup(); + if(s1->class != CSTATIC) { + s1->type = s->type; + s1->offset = s->offset; + s1->block = s->block; + s1->class = CSTATIC; + } + return s1; +} + +/* + * make a copy of a typedef + * the problem is to split out incomplete + * arrays so that it is in the variable + * rather than the typedef. + */ +Type* +tcopy(Type *t) +{ + Type *tl, *tx; + int et; + + if(t == T) + return t; + et = t->etype; + if(typesu[et]) + return t; + tl = tcopy(t->link); + if(tl != t->link || + (et == TARRAY && t->width == 0)) { + tx = copytyp(t); + tx->link = tl; + return tx; + } + return t; +} + +Node* +doinit(Sym *s, Type *t, int32 o, Node *a) +{ + Node *n; + + if(t == T) + return Z; + if(s->class == CEXTERN) { + s->class = CGLOBL; + if(debug['d']) + dbgdecl(s); + } + if(debug['i']) { + print("t = %T; o = %d; n = %s\n", t, o, s->name); + prtree(a, "doinit value"); + } + + + n = initlist; + if(a->op == OINIT) + a = a->left; + initlist = a; + + a = init1(s, t, o, 0); + if(initlist != Z) + diag(initlist, "more initializers than structure: %s", + s->name); + initlist = n; + + return a; +} + +/* + * get next major operator, + * dont advance initlist. + */ +Node* +peekinit(void) +{ + Node *a; + + a = initlist; + +loop: + if(a == Z) + return a; + if(a->op == OLIST) { + a = a->left; + goto loop; + } + return a; +} + +/* + * consume and return next element on + * initlist. expand strings. + */ +Node* +nextinit(void) +{ + Node *a, *b, *n; + + a = initlist; + n = Z; + + if(a == Z) + return a; + if(a->op == OLIST) { + n = a->right; + a = a->left; + } + if(a->op == OUSED) { + a = a->left; + b = new(OCONST, Z, Z); + b->type = a->type->link; + if(a->op == OSTRING) { + b->vconst = convvtox(*a->cstring, TCHAR); + a->cstring++; + } + if(a->op == OLSTRING) { + b->vconst = convvtox(*a->rstring, TUSHORT); + a->rstring++; + } + a->type->width -= b->type->width; + if(a->type->width <= 0) + initlist = n; + return b; + } + initlist = n; + return a; +} + +int +isstruct(Node *a, Type *t) +{ + Node *n; + + switch(a->op) { + case ODOTDOT: + n = a->left; + if(n && n->type && sametype(n->type, t)) + return 1; + case OSTRING: + case OLSTRING: + case OCONST: + case OINIT: + case OELEM: + return 0; + } + + n = new(ODOTDOT, Z, Z); + *n = *a; + + /* + * ODOTDOT is a flag for tcom + * a second tcom will not be performed + */ + a->op = ODOTDOT; + a->left = n; + a->right = Z; + + if(tcom(n)) + return 0; + + if(sametype(n->type, t)) + return 1; + return 0; +} + +Node* +init1(Sym *s, Type *t, int32 o, int exflag) +{ + Node *a, *l, *r, nod; + Type *t1; + int32 e, w, so, mw; + + a = peekinit(); + if(a == Z) + return Z; + + if(debug['i']) { + print("t = %T; o = %d; n = %s\n", t, o, s->name); + prtree(a, "init1 value"); + } + + if(exflag && a->op == OINIT) + return doinit(s, t, o, nextinit()); + + switch(t->etype) { + default: + diag(Z, "unknown type in initialization: %T to: %s", t, s->name); + return Z; + + case TCHAR: + case TUCHAR: + case TINT: + case TUINT: + case TSHORT: + case TUSHORT: + case TLONG: + case TULONG: + case TVLONG: + case TUVLONG: + case TFLOAT: + case TDOUBLE: + case TIND: + single: + if(a->op == OARRAY || a->op == OELEM) + return Z; + + a = nextinit(); + if(a == Z) + return Z; + + if(t->nbits) + diag(Z, "cannot initialize bitfields"); + if(s->class == CAUTO) { + l = new(ONAME, Z, Z); + l->sym = s; + l->type = t; + l->etype = TVOID; + if(s->type) + l->etype = s->type->etype; + l->xoffset = s->offset + o; + l->class = s->class; + + l = new(OASI, l, a); + return l; + } + + complex(a); + if(a->type == T) + return Z; + + if(a->op == OCONST) { + if(vconst(a) && t->etype == TIND && a->type && a->type->etype != TIND){ + diag(a, "initialize pointer to an integer: %s", s->name); + return Z; + } + if(!sametype(a->type, t)) { + /* hoop jumping to save malloc */ + if(nodcast == Z) + nodcast = new(OCAST, Z, Z); + nod = *nodcast; + nod.left = a; + nod.type = t; + nod.lineno = a->lineno; + complex(&nod); + if(nod.type) + *a = nod; + } + if(a->op != OCONST) { + diag(a, "initializer is not a constant: %s", + s->name); + return Z; + } + if(vconst(a) == 0) + return Z; + goto gext; + } + if(t->etype == TIND) { + while(a->op == OCAST) { + warn(a, "CAST in initialization ignored"); + a = a->left; + } + if(!sametype(t, a->type)) { + diag(a, "initialization of incompatible pointers: %s\n%T and %T", + s->name, t, a->type); + } + if(a->op == OADDR) + a = a->left; + goto gext; + } + + while(a->op == OCAST) + a = a->left; + if(a->op == OADDR) { + warn(a, "initialize pointer to an integer: %s", s->name); + a = a->left; + goto gext; + } + diag(a, "initializer is not a constant: %s", s->name); + return Z; + + gext: + gextern(s, a, o, t->width); + + return Z; + + case TARRAY: + w = t->link->width; + if(a->op == OSTRING || a->op == OLSTRING) + if(typei[t->link->etype]) { + /* + * get rid of null if sizes match exactly + */ + a = nextinit(); + mw = t->width/w; + so = a->type->width/a->type->link->width; + if(mw && so > mw) { + if(so != mw+1) + diag(a, "string initialization larger than array"); + a->type->width -= a->type->link->width; + } + + /* + * arrange strings to be expanded + * inside OINIT braces. + */ + a = new(OUSED, a, Z); + return doinit(s, t, o, a); + } + + mw = -w; + l = Z; + for(e=0;;) { + /* + * peek ahead for element initializer + */ + a = peekinit(); + if(a == Z) + break; + if(a->op == OELEM && t->link->etype != TSTRUCT) + break; + if(a->op == OARRAY) { + if(e && exflag) + break; + a = nextinit(); + r = a->left; + complex(r); + if(r->op != OCONST) { + diag(r, "initializer subscript must be constant"); + return Z; + } + e = r->vconst; + if(t->width != 0) + if(e < 0 || e*w >= t->width) { + diag(a, "initialization index out of range: %d", e); + continue; + } + } + + so = e*w; + if(so > mw) + mw = so; + if(t->width != 0) + if(mw >= t->width) + break; + r = init1(s, t->link, o+so, 1); + l = newlist(l, r); + e++; + } + if(t->width == 0) + t->width = mw+w; + return l; + + case TUNION: + case TSTRUCT: + /* + * peek ahead to find type of rhs. + * if its a structure, then treat + * this element as a variable + * rather than an aggregate. + */ + if(isstruct(a, t)) + goto single; + + if(t->width <= 0) { + diag(Z, "incomplete structure: %s", s->name); + return Z; + } + l = Z; + + again: + for(t1 = t->link; t1 != T; t1 = t1->down) { + if(a->op == OARRAY && t1->etype != TARRAY) + break; + if(a->op == OELEM) { + if(t1->sym != a->sym) + continue; + nextinit(); + } + r = init1(s, t1, o+t1->offset, 1); + l = newlist(l, r); + a = peekinit(); + if(a == Z) + break; + if(a->op == OELEM) + goto again; + } + if(a && a->op == OELEM) + diag(a, "structure element not found %F", a); + return l; + } +} + +Node* +newlist(Node *l, Node *r) +{ + if(r == Z) + return l; + if(l == Z) + return r; + return new(OLIST, l, r); +} + +void +sualign(Type *t) +{ + Type *l; + int32 o, w, maxal; + + o = 0; + maxal = 0; + switch(t->etype) { + + case TSTRUCT: + t->offset = 0; + w = 0; + for(l = t->link; l != T; l = l->down) { + if(l->nbits) { + if(l->shift <= 0) { + l->shift = -l->shift; + w = xround(w, tfield->width); + o = w; + w += tfield->width; + } + l->offset = o; + } else { + if(l->width <= 0) + if(l->down != T) + if(l->sym) + diag(Z, "incomplete structure element: %s", + l->sym->name); + else + diag(Z, "incomplete structure element"); + w = align(w, l, Ael1, &maxal); + l->offset = w; + w = align(w, l, Ael2, &maxal); + } + } + w = align(w, t, Asu2, &maxal); + t->width = w; + t->align = maxal; + acidtype(t); + godeftype(t); + return; + + case TUNION: + t->offset = 0; + w = 0; + for(l = t->link; l != T; l = l->down) { + if(l->width <= 0) + if(l->sym) + diag(Z, "incomplete union element: %s", + l->sym->name); + else + diag(Z, "incomplete union element"); + l->offset = 0; + l->shift = 0; + o = align(align(0, l, Ael1, &maxal), l, Ael2, &maxal); + if(o > w) + w = o; + } + w = align(w, t, Asu2, &maxal); + t->width = w; + t->align = maxal; + acidtype(t); + godeftype(t); + return; + + default: + diag(Z, "unknown type in sualign: %T", t); + break; + } +} + +int32 +xround(int32 v, int w) +{ + int r; + + if(w <= 0 || w > 8) { + diag(Z, "rounding by %d", w); + w = 1; + } + r = v%w; + if(r) + v += w-r; + return v; +} + +Type* +ofnproto(Node *n) +{ + Type *tl, *tr, *t; + + if(n == Z) + return T; + switch(n->op) { + case OLIST: + tl = ofnproto(n->left); + tr = ofnproto(n->right); + if(tl == T) + return tr; + tl->down = tr; + return tl; + + case ONAME: + t = copytyp(n->sym->type); + t->down = T; + return t; + } + return T; +} + +#define ANSIPROTO 1 +#define OLDPROTO 2 + +void +argmark(Node *n, int pass) +{ + Type *t; + + autoffset = align(0, thisfn->link, Aarg0, nil); + stkoff = 0; + for(; n->left != Z; n = n->left) { + if(n->op != OFUNC || n->left->op != ONAME) + continue; + walkparam(n->right, pass); + if(pass != 0 && anyproto(n->right) == OLDPROTO) { + t = typ(TFUNC, n->left->sym->type->link); + t->down = typ(TOLD, T); + t->down->down = ofnproto(n->right); + tmerge(t, n->left->sym); + n->left->sym->type = t; + } + break; + } + autoffset = 0; + stkoff = 0; +} + +void +walkparam(Node *n, int pass) +{ + Sym *s; + Node *n1; + + if(n != Z && n->op == OPROTO && n->left == Z && n->type == types[TVOID]) + return; + +loop: + if(n == Z) + return; + switch(n->op) { + default: + diag(n, "argument not a name/prototype: %O", n->op); + break; + + case OLIST: + walkparam(n->left, pass); + n = n->right; + goto loop; + + case OPROTO: + for(n1 = n; n1 != Z; n1=n1->left) + if(n1->op == ONAME) { + if(pass == 0) { + s = n1->sym; + push1(s); + s->offset = -1; + break; + } + dodecl(pdecl, CPARAM, n->type, n->left); + break; + } + if(n1) + break; + if(pass == 0) { + /* + * extension: + * allow no name in argument declaration + diag(Z, "no name in argument declaration"); + */ + break; + } + dodecl(NODECL, CPARAM, n->type, n->left); + pdecl(CPARAM, lastdcl, S); + break; + + case ODOTDOT: + break; + + case ONAME: + s = n->sym; + if(pass == 0) { + push1(s); + s->offset = -1; + break; + } + if(s->offset != -1) { + if(autoffset == 0) { + firstarg = s; + firstargtype = s->type; + } + autoffset = align(autoffset, s->type, Aarg1, nil); + s->offset = autoffset; + autoffset = align(autoffset, s->type, Aarg2, nil); + } else + dodecl(pdecl, CXXX, types[TINT], n); + break; + } +} + +void +markdcl(void) +{ + Decl *d; + + blockno++; + d = push(); + d->val = DMARK; + d->offset = autoffset; + d->block = autobn; + autobn = blockno; +} + +Node* +revertdcl(void) +{ + Decl *d; + Sym *s; + Node *n, *n1; + + n = Z; + for(;;) { + d = dclstack; + if(d == D) { + diag(Z, "pop off dcl stack"); + break; + } + dclstack = d->link; + s = d->sym; + switch(d->val) { + case DMARK: + autoffset = d->offset; + autobn = d->block; + return n; + + case DAUTO: + if(debug['d']) + print("revert1 \"%s\"\n", s->name); + if(s->aused == 0) { + nearln = s->varlineno; + if(s->class == CAUTO) + warn(Z, "auto declared and not used: %s", s->name); + if(s->class == CPARAM) + warn(Z, "param declared and not used: %s", s->name); + } + if(s->type && (s->type->garb & GVOLATILE)) { + n1 = new(ONAME, Z, Z); + n1->sym = s; + n1->type = s->type; + n1->etype = TVOID; + if(n1->type != T) + n1->etype = n1->type->etype; + n1->xoffset = s->offset; + n1->class = s->class; + + n1 = new(OADDR, n1, Z); + n1 = new(OUSED, n1, Z); + if(n == Z) + n = n1; + else + n = new(OLIST, n1, n); + } + s->type = d->type; + s->class = d->class; + s->offset = d->offset; + s->block = d->block; + s->varlineno = d->varlineno; + s->aused = d->aused; + break; + + case DSUE: + if(debug['d']) + print("revert2 \"%s\"\n", s->name); + s->suetag = d->type; + s->sueblock = d->block; + break; + + case DLABEL: + if(debug['d']) + print("revert3 \"%s\"\n", s->name); + if(s->label && s->label->addable == 0) + warn(s->label, "label declared and not used \"%s\"", s->name); + s->label = Z; + break; + } + } + return n; +} + +Type* +fnproto(Node *n) +{ + int r; + + r = anyproto(n->right); + if(r == 0 || (r & OLDPROTO)) { + if(r & ANSIPROTO) + diag(n, "mixed ansi/old function declaration: %F", n->left); + return T; + } + return fnproto1(n->right); +} + +int +anyproto(Node *n) +{ + int r; + + r = 0; + +loop: + if(n == Z) + return r; + switch(n->op) { + case OLIST: + r |= anyproto(n->left); + n = n->right; + goto loop; + + case ODOTDOT: + case OPROTO: + return r | ANSIPROTO; + } + return r | OLDPROTO; +} + +Type* +fnproto1(Node *n) +{ + Type *t; + + if(n == Z) + return T; + switch(n->op) { + case OLIST: + t = fnproto1(n->left); + if(t != T) + t->down = fnproto1(n->right); + return t; + + case OPROTO: + lastdcl = T; + dodecl(NODECL, CXXX, n->type, n->left); + t = typ(TXXX, T); + if(lastdcl != T) + *t = *paramconv(lastdcl, 1); + return t; + + case ONAME: + diag(n, "incomplete argument prototype"); + return typ(TINT, T); + + case ODOTDOT: + return typ(TDOT, T); + } + diag(n, "unknown op in fnproto"); + return T; +} + +void +dbgdecl(Sym *s) +{ + print("decl \"%s\": C=%s [B=%d:O=%d] T=%T\n", + s->name, cnames[s->class], s->block, s->offset, s->type); +} + +Decl* +push(void) +{ + Decl *d; + + d = alloc(sizeof(*d)); + d->link = dclstack; + dclstack = d; + return d; +} + +Decl* +push1(Sym *s) +{ + Decl *d; + + d = push(); + d->sym = s; + d->val = DAUTO; + d->type = s->type; + d->class = s->class; + d->offset = s->offset; + d->block = s->block; + d->varlineno = s->varlineno; + d->aused = s->aused; + return d; +} + +int +sametype(Type *t1, Type *t2) +{ + + if(t1 == t2) + return 1; + return rsametype(t1, t2, 5, 1); +} + +int +rsametype(Type *t1, Type *t2, int n, int f) +{ + int et; + + n--; + for(;;) { + if(t1 == t2) + return 1; + if(t1 == T || t2 == T) + return 0; + if(n <= 0) + return 1; + et = t1->etype; + if(et != t2->etype) + return 0; + if(et == TFUNC) { + if(!rsametype(t1->link, t2->link, n, 0)) + return 0; + t1 = t1->down; + t2 = t2->down; + while(t1 != T && t2 != T) { + if(t1->etype == TOLD) { + t1 = t1->down; + continue; + } + if(t2->etype == TOLD) { + t2 = t2->down; + continue; + } + while(t1 != T || t2 != T) { + if(!rsametype(t1, t2, n, 0)) + return 0; + t1 = t1->down; + t2 = t2->down; + } + break; + } + return 1; + } + if(et == TARRAY) + if(t1->width != t2->width && t1->width != 0 && t2->width != 0) + return 0; + if(typesu[et]) { + if(t1->link == T) + snap(t1); + if(t2->link == T) + snap(t2); + t1 = t1->link; + t2 = t2->link; + for(;;) { + if(t1 == t2) + return 1; + if(!rsametype(t1, t2, n, 0)) + return 0; + t1 = t1->down; + t2 = t2->down; + } + } + t1 = t1->link; + t2 = t2->link; + if((f || !debug['V']) && et == TIND) { + if(t1 != T && t1->etype == TVOID) + return 1; + if(t2 != T && t2->etype == TVOID) + return 1; + } + } +} + +typedef struct Typetab Typetab; + +struct Typetab{ + int n; + Type **a; +}; + +static int +sigind(Type *t, Typetab *tt) +{ + int n; + Type **a, **na, **p, **e; + + n = tt->n; + a = tt->a; + e = a+n; + /* linear search seems ok */ + for(p = a ; p < e; p++) + if(sametype(*p, t)) + return p-a; + if((n&15) == 0){ + na = malloc((n+16)*sizeof(Type*)); + memmove(na, a, n*sizeof(Type*)); + free(a); + a = tt->a = na; + } + a[tt->n++] = t; + return -1; +} + +static uint32 +signat(Type *t, Typetab *tt) +{ + int i; + Type *t1; + int32 s; + + s = 0; + for(; t; t=t->link) { + s = s*thash1 + thash[t->etype]; + if(t->garb&GINCOMPLETE) + return s; + switch(t->etype) { + default: + return s; + case TARRAY: + s = s*thash2 + 0; /* was t->width */ + break; + case TFUNC: + for(t1=t->down; t1; t1=t1->down) + s = s*thash3 + signat(t1, tt); + break; + case TSTRUCT: + case TUNION: + if((i = sigind(t, tt)) >= 0){ + s = s*thash2 + i; + return s; + } + for(t1=t->link; t1; t1=t1->down) + s = s*thash3 + signat(t1, tt); + return s; + case TIND: + break; + } + } + return s; +} + +uint32 +signature(Type *t) +{ + uint32 s; + Typetab tt; + + tt.n = 0; + tt.a = nil; + s = signat(t, &tt); + free(tt.a); + return s; +} + +uint32 +sign(Sym *s) +{ + uint32 v; + Type *t; + + if(s->sig == SIGINTERN) + return SIGNINTERN; + if((t = s->type) == T) + return 0; + v = signature(t); + if(v == 0) + v = SIGNINTERN; + return v; +} + +void +snap(Type *t) +{ + if(typesu[t->etype]) + if(t->link == T && t->tag && t->tag->suetag) { + t->link = t->tag->suetag->link; + t->width = t->tag->suetag->width; + } +} + +Type* +dotag(Sym *s, int et, int bn) +{ + Decl *d; + + if(bn != 0 && bn != s->sueblock) { + d = push(); + d->sym = s; + d->val = DSUE; + d->type = s->suetag; + d->block = s->sueblock; + s->suetag = T; + } + if(s->suetag == T) { + s->suetag = typ(et, T); + s->sueblock = autobn; + } + if(s->suetag->etype != et) + diag(Z, "tag used for more than one type: %s", + s->name); + if(s->suetag->tag == S) + s->suetag->tag = s; + return s->suetag; +} + +Node* +dcllabel(Sym *s, int f) +{ + Decl *d, d1; + Node *n; + + n = s->label; + if(n != Z) { + if(f) { + if(n->complex) + diag(Z, "label reused: %s", s->name); + n->complex = 1; // declared + } else + n->addable = 1; // used + return n; + } + + d = push(); + d->sym = s; + d->val = DLABEL; + dclstack = d->link; + + d1 = *firstdcl; + *firstdcl = *d; + *d = d1; + + firstdcl->link = d; + firstdcl = d; + + n = new(OXXX, Z, Z); + n->sym = s; + n->complex = f; + n->addable = !f; + s->label = n; + + if(debug['d']) + dbgdecl(s); + return n; +} + +Type* +paramconv(Type *t, int f) +{ + + switch(t->etype) { + case TUNION: + case TSTRUCT: + if(t->width <= 0) + diag(Z, "incomplete structure: %s", t->tag->name); + break; + + case TARRAY: + t = typ(TIND, t->link); + t->width = types[TIND]->width; + break; + + case TFUNC: + t = typ(TIND, t); + t->width = types[TIND]->width; + break; + + case TFLOAT: + if(!f) + t = types[TDOUBLE]; + break; + + case TCHAR: + case TSHORT: + if(!f) + t = types[TINT]; + break; + + case TUCHAR: + case TUSHORT: + if(!f) + t = types[TUINT]; + break; + } + return t; +} + +void +adecl(int c, Type *t, Sym *s) +{ + + if(c == CSTATIC) + c = CLOCAL; + if(t->etype == TFUNC) { + if(c == CXXX) + c = CEXTERN; + if(c == CLOCAL) + c = CSTATIC; + if(c == CAUTO || c == CEXREG) + diag(Z, "function cannot be %s %s", cnames[c], s->name); + } + if(c == CXXX) + c = CAUTO; + if(s) { + if(s->class == CSTATIC) + if(c == CEXTERN || c == CGLOBL) { + warn(Z, "just say static: %s", s->name); + c = CSTATIC; + } + if(s->class == CAUTO || s->class == CPARAM || s->class == CLOCAL) + if(s->block == autobn) + diag(Z, "auto redeclaration of: %s", s->name); + if(c != CPARAM) + push1(s); + s->block = autobn; + s->offset = 0; + s->type = t; + s->class = c; + s->aused = 0; + } + switch(c) { + case CAUTO: + autoffset = align(autoffset, t, Aaut3, nil); + stkoff = maxround(stkoff, autoffset); + s->offset = -autoffset; + break; + + case CPARAM: + if(autoffset == 0) { + firstarg = s; + firstargtype = t; + } + autoffset = align(autoffset, t, Aarg1, nil); + if(s) + s->offset = autoffset; + autoffset = align(autoffset, t, Aarg2, nil); + break; + } +} + +void +pdecl(int c, Type *t, Sym *s) +{ + if(s && s->offset != -1) { + diag(Z, "not a parameter: %s", s->name); + return; + } + t = paramconv(t, c==CPARAM); + if(c == CXXX) + c = CPARAM; + if(c != CPARAM) { + diag(Z, "parameter cannot have class: %s", s->name); + c = CPARAM; + } + adecl(c, t, s); +} + +void +xdecl(int c, Type *t, Sym *s) +{ + int32 o; + + o = 0; + switch(c) { + case CEXREG: + o = exreg(t); + if(o == 0) + c = CEXTERN; + if(s->class == CGLOBL) + c = CGLOBL; + break; + + case CEXTERN: + if(s->class == CGLOBL) + c = CGLOBL; + break; + + case CXXX: + c = CGLOBL; + if(s->class == CEXTERN) + s->class = CGLOBL; + break; + + case CAUTO: + diag(Z, "overspecified class: %s %s %s", s->name, cnames[c], cnames[s->class]); + c = CEXTERN; + break; + + case CTYPESTR: + if(!typesuv[t->etype]) { + diag(Z, "typestr must be struct/union: %s", s->name); + break; + } + dclfunct(t, s); + break; + } + + if(s->class == CSTATIC) + if(c == CEXTERN || c == CGLOBL) { + warn(Z, "overspecified class: %s %s %s", s->name, cnames[c], cnames[s->class]); + c = CSTATIC; + } + if(s->type != T) + if(s->class != c || !sametype(t, s->type) || t->etype == TENUM) { + diag(Z, "external redeclaration of: %s", s->name); + Bprint(&diagbuf, " %s %T %L\n", cnames[c], t, nearln); + Bprint(&diagbuf, " %s %T %L\n", cnames[s->class], s->type, s->varlineno); + } + tmerge(t, s); + s->type = t; + s->class = c; + s->block = 0; + s->offset = o; +} + +void +tmerge(Type *t1, Sym *s) +{ + Type *ta, *tb, *t2; + + t2 = s->type; + for(;;) { + if(t1 == T || t2 == T || t1 == t2) + break; + if(t1->etype != t2->etype) + break; + switch(t1->etype) { + case TFUNC: + ta = t1->down; + tb = t2->down; + if(ta == T) { + t1->down = tb; + break; + } + if(tb == T) + break; + while(ta != T && tb != T) { + if(ta == tb) + break; + /* ignore old-style flag */ + if(ta->etype == TOLD) { + ta = ta->down; + continue; + } + if(tb->etype == TOLD) { + tb = tb->down; + continue; + } + /* checking terminated by ... */ + if(ta->etype == TDOT && tb->etype == TDOT) { + ta = T; + tb = T; + break; + } + if(!sametype(ta, tb)) + break; + ta = ta->down; + tb = tb->down; + } + if(ta != tb) + diag(Z, "function inconsistently declared: %s", s->name); + + /* take new-style over old-style */ + ta = t1->down; + tb = t2->down; + if(ta != T && ta->etype == TOLD) + if(tb != T && tb->etype != TOLD) + t1->down = tb; + break; + + case TARRAY: + /* should we check array size change? */ + if(t2->width > t1->width) + t1->width = t2->width; + break; + + case TUNION: + case TSTRUCT: + return; + } + t1 = t1->link; + t2 = t2->link; + } +} + +void +edecl(int c, Type *t, Sym *s) +{ + Type *t1; + + if(s == S) { + if(!typesu[t->etype]) + diag(Z, "unnamed structure element must be struct/union"); + if(c != CXXX) + diag(Z, "unnamed structure element cannot have class"); + } else + if(c != CXXX) + diag(Z, "structure element cannot have class: %s", s->name); + t1 = t; + t = copytyp(t1); + t->sym = s; + t->down = T; + if(lastfield) { + t->shift = lastbit - lastfield; + t->nbits = lastfield; + if(firstbit) + t->shift = -t->shift; + if(typeu[t->etype]) + t->etype = tufield->etype; + else + t->etype = tfield->etype; + } + if(strf == T) + strf = t; + else + strl->down = t; + strl = t; +} + +/* + * this routine is very suspect. + * ansi requires the enum type to + * be represented as an 'int' + * this means that 0x81234567 + * would be illegal. this routine + * makes signed and unsigned go + * to unsigned. + */ +Type* +maxtype(Type *t1, Type *t2) +{ + + if(t1 == T) + return t2; + if(t2 == T) + return t1; + if(t1->etype > t2->etype) + return t1; + return t2; +} + +void +doenum(Sym *s, Node *n) +{ + + if(n) { + complex(n); + if(n->op != OCONST) { + diag(n, "enum not a constant: %s", s->name); + return; + } + en.cenum = n->type; + en.tenum = maxtype(en.cenum, en.tenum); + + if(!typefd[en.cenum->etype]) + en.lastenum = n->vconst; + else + en.floatenum = n->fconst; + } + if(dclstack) + push1(s); + xdecl(CXXX, types[TENUM], s); + + if(en.cenum == T) { + en.tenum = types[TINT]; + en.cenum = types[TINT]; + en.lastenum = 0; + } + s->tenum = en.cenum; + + if(!typefd[s->tenum->etype]) { + s->vconst = convvtox(en.lastenum, s->tenum->etype); + en.lastenum++; + } else { + s->fconst = en.floatenum; + en.floatenum++; + } + + if(debug['d']) + dbgdecl(s); + acidvar(s); + godefvar(s); +} + +void +symadjust(Sym *s, Node *n, int32 del) +{ + + switch(n->op) { + default: + if(n->left) + symadjust(s, n->left, del); + if(n->right) + symadjust(s, n->right, del); + return; + + case ONAME: + if(n->sym == s) + n->xoffset -= del; + return; + + case OCONST: + case OSTRING: + case OLSTRING: + case OINDREG: + case OREGISTER: + return; + } +} + +Node* +contig(Sym *s, Node *n, int32 v) +{ + Node *p, *r, *q, *m; + int32 w; + Type *zt; + + if(debug['i']) { + print("contig v = %d; s = %s\n", v, s->name); + prtree(n, "doinit value"); + } + + if(n == Z) + goto no; + w = s->type->width; + + /* + * nightmare: an automatic array whose size + * increases when it is initialized + */ + if(v != w) { + if(v != 0) + diag(n, "automatic adjustable array: %s", s->name); + v = s->offset; + autoffset = align(autoffset, s->type, Aaut3, nil); + s->offset = -autoffset; + stkoff = maxround(stkoff, autoffset); + symadjust(s, n, v - s->offset); + } + if(w <= ewidth[TIND]) + goto no; + if(n->op == OAS) + diag(Z, "oops in contig"); +/*ZZZ this appears incorrect +need to check if the list completely covers the data. +if not, bail + */ + if(n->op == OLIST) + goto no; + if(n->op == OASI) + if(n->left->type) + if(n->left->type->width == w) + goto no; + while(w & (ewidth[TIND]-1)) + w++; +/* + * insert the following code, where long becomes vlong if pointers are fat + * + *(long**)&X = (long*)((char*)X + sizeof(X)); + do { + *(long**)&X -= 1; + **(long**)&X = 0; + } while(*(long**)&X); + */ + + for(q=n; q->op != ONAME; q=q->left) + ; + + zt = ewidth[TIND] > ewidth[TLONG]? types[TVLONG]: types[TLONG]; + + p = new(ONAME, Z, Z); + *p = *q; + p->type = typ(TIND, zt); + p->xoffset = s->offset; + + r = new(ONAME, Z, Z); + *r = *p; + r = new(OPOSTDEC, r, Z); + + q = new(ONAME, Z, Z); + *q = *p; + q = new(OIND, q, Z); + + m = new(OCONST, Z, Z); + m->vconst = 0; + m->type = zt; + + q = new(OAS, q, m); + + r = new(OLIST, r, q); + + q = new(ONAME, Z, Z); + *q = *p; + r = new(ODWHILE, q, r); + + q = new(ONAME, Z, Z); + *q = *p; + q->type = q->type->link; + q->xoffset += w; + q = new(OADDR, q, 0); + + q = new(OASI, p, q); + r = new(OLIST, q, r); + + n = new(OLIST, r, n); + +no: + return n; +} diff --git a/src/cmd/cc/doc.go b/src/cmd/cc/doc.go new file mode 100644 index 000000000..51aa8b192 --- /dev/null +++ b/src/cmd/cc/doc.go @@ -0,0 +1,11 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +/* + +This directory contains the portable section of the Plan 9 C compilers. +See ../6c, ../8c, and ../5c for more information. + +*/ +package documentation diff --git a/src/cmd/cc/dpchk.c b/src/cmd/cc/dpchk.c new file mode 100644 index 000000000..084aa0484 --- /dev/null +++ b/src/cmd/cc/dpchk.c @@ -0,0 +1,723 @@ +// Inferno utils/cc/dpchk.c +// http://code.google.com/p/inferno-os/source/browse/utils/cc/dpchk.c +// +// Copyright © 1994-1999 Lucent Technologies Inc. All rights reserved. +// Portions Copyright © 1995-1997 C H Forsyth (forsyth@terzarima.net) +// Portions Copyright © 1997-1999 Vita Nuova Limited +// Portions Copyright © 2000-2007 Vita Nuova Holdings Limited (www.vitanuova.com) +// Portions Copyright © 2004,2006 Bruce Ellis +// Portions Copyright © 2005-2007 C H Forsyth (forsyth@terzarima.net) +// Revisions Copyright © 2000-2007 Lucent Technologies Inc. and others +// Portions Copyright © 2009 The Go Authors. All rights reserved. +// +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to deal +// in the Software without restriction, including without limitation the rights +// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +// copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +// THE SOFTWARE. + +#include +#include "cc.h" +#include "y.tab.h" + +enum +{ + Fnone = 0, + Fl, + Fvl, + Fignor, + Fstar, + Fadj, + + Fverb = 10, +}; + +typedef struct Tprot Tprot; +struct Tprot +{ + Type* type; + Bits flag; + Tprot* link; +}; + +typedef struct Tname Tname; +struct Tname +{ + char* name; + int param; + int count; + Tname* link; + Tprot* prot; +}; + +static Type* indchar; +static uchar flagbits[512]; +static char* lastfmt; +static int lastadj; +static int lastverb; +static int nstar; +static Tprot* tprot; +static Tname* tname; + +void +argflag(int c, int v) +{ + + switch(v) { + case Fignor: + case Fstar: + case Fl: + case Fvl: + flagbits[c] = v; + break; + case Fverb: + flagbits[c] = lastverb; +/*print("flag-v %c %d\n", c, lastadj);*/ + lastverb++; + break; + case Fadj: + flagbits[c] = lastadj; +/*print("flag-l %c %d\n", c, lastadj);*/ + lastadj++; + break; + } +} + +Bits +getflag(char *s) +{ + Bits flag; + int f; + Fmt fmt; + Rune c; + + flag = zbits; + nstar = 0; + fmtstrinit(&fmt); + for(;;) { + s += chartorune(&c, s); + if(c == 0 || c >= nelem(flagbits)) + break; + fmtrune(&fmt, c); + f = flagbits[c]; + switch(f) { + case Fnone: + argflag(c, Fverb); + f = flagbits[c]; + break; + case Fstar: + nstar++; + case Fignor: + continue; + case Fl: + if(bset(flag, Fl)) + flag = bor(flag, blsh(Fvl)); + } + flag = bor(flag, blsh(f)); + if(f >= Fverb) + break; + } + free(lastfmt); + lastfmt = fmtstrflush(&fmt); + return flag; +} + +static void +newprot(Sym *m, Type *t, char *s, Tprot **prot) +{ + Bits flag; + Tprot *l; + + if(t == T) { + warn(Z, "%s: newprot: type not defined", m->name); + return; + } + flag = getflag(s); + for(l=*prot; l; l=l->link) + if(beq(flag, l->flag) && sametype(t, l->type)) + return; + l = alloc(sizeof(*l)); + l->type = t; + l->flag = flag; + l->link = *prot; + *prot = l; +} + +static Tname* +newname(char *s, int p, int count) +{ + Tname *l; + + for(l=tname; l; l=l->link) + if(strcmp(l->name, s) == 0) { + if(p >= 0 && l->param != p) + yyerror("vargck %s already defined\n", s); + return l; + } + if(p < 0) + return nil; + + l = alloc(sizeof(*l)); + l->name = s; + l->param = p; + l->link = tname; + l->count = count; + tname = l; + return l; +} + +void +arginit(void) +{ + int i; + +/* debug['F'] = 1;*/ +/* debug['w'] = 1;*/ + + lastadj = Fadj; + lastverb = Fverb; + indchar = typ(TIND, types[TCHAR]); + + memset(flagbits, Fnone, sizeof(flagbits)); + + for(i='0'; i<='9'; i++) + argflag(i, Fignor); + argflag('.', Fignor); + argflag('#', Fignor); + argflag('u', Fignor); + argflag('h', Fignor); + argflag('+', Fignor); + argflag('-', Fignor); + + argflag('*', Fstar); + argflag('l', Fl); + + argflag('o', Fverb); + flagbits['x'] = flagbits['o']; + flagbits['X'] = flagbits['o']; +} + +static char* +getquoted(void) +{ + int c; + Rune r; + Fmt fmt; + + c = getnsc(); + if(c != '"') + return nil; + fmtstrinit(&fmt); + for(;;) { + r = getr(); + if(r == '\n') { + free(fmtstrflush(&fmt)); + return nil; + } + if(r == '"') + break; + fmtrune(&fmt, r); + } + free(lastfmt); + lastfmt = fmtstrflush(&fmt); + return strdup(lastfmt); +} + +void +pragvararg(void) +{ + Sym *s; + int n, c; + char *t; + Type *ty; + Tname *l; + + if(!debug['F']) + goto out; + s = getsym(); + if(s && strcmp(s->name, "argpos") == 0) + goto ckpos; + if(s && strcmp(s->name, "type") == 0) + goto cktype; + if(s && strcmp(s->name, "flag") == 0) + goto ckflag; + if(s && strcmp(s->name, "countpos") == 0) + goto ckcount; + yyerror("syntax in #pragma varargck"); + goto out; + +ckpos: +/*#pragma varargck argpos warn 2*/ + s = getsym(); + if(s == S) + goto bad; + n = getnsn(); + if(n < 0) + goto bad; + newname(s->name, n, 0); + goto out; + +ckcount: +/*#pragma varargck countpos name 2*/ + s = getsym(); + if(s == S) + goto bad; + n = getnsn(); + if(n < 0) + goto bad; + newname(s->name, 0, n); + goto out; + +ckflag: +/*#pragma varargck flag 'c'*/ + c = getnsc(); + if(c != '\'') + goto bad; + c = getr(); + if(c == '\\') + c = getr(); + else if(c == '\'') + goto bad; + if(c == '\n') + goto bad; + if(getc() != '\'') + goto bad; + argflag(c, Fignor); + goto out; + +cktype: + c = getnsc(); + unget(c); + if(c != '"') { +/*#pragma varargck type name int*/ + s = getsym(); + if(s == S) + goto bad; + l = newname(s->name, -1, -1); + s = getsym(); + if(s == S) + goto bad; + ty = s->type; + while((c = getnsc()) == '*') + ty = typ(TIND, ty); + unget(c); + newprot(s, ty, "a", &l->prot); + goto out; + } + +/*#pragma varargck type O int*/ + t = getquoted(); + if(t == nil) + goto bad; + s = getsym(); + if(s == S) + goto bad; + ty = s->type; + while((c = getnsc()) == '*') + ty = typ(TIND, ty); + unget(c); + newprot(s, ty, t, &tprot); + goto out; + +bad: + yyerror("syntax in #pragma varargck"); + +out: + while(getnsc() != '\n') + ; +} + +Node* +nextarg(Node *n, Node **a) +{ + if(n == Z) { + *a = Z; + return Z; + } + if(n->op == OLIST) { + *a = n->left; + return n->right; + } + *a = n; + return Z; +} + +void +checkargs(Node *nn, char *s, int pos) +{ + Node *a, *n; + Bits flag; + Tprot *l; + + if(!debug['F']) + return; + n = nn; + for(;;) { + s = strchr(s, '%'); + if(s == 0) { + nextarg(n, &a); + if(a != Z) + warn(nn, "more arguments than format %T", + a->type); + return; + } + s++; + flag = getflag(s); + while(nstar > 0) { + n = nextarg(n, &a); + pos++; + nstar--; + if(a == Z) { + warn(nn, "more format than arguments %s", + lastfmt); + return; + } + if(a->type == T) + continue; + if(!sametype(types[TINT], a->type) && + !sametype(types[TUINT], a->type)) + warn(nn, "format mismatch '*' in %s %T, arg %d", + lastfmt, a->type, pos); + } + for(l=tprot; l; l=l->link) + if(sametype(types[TVOID], l->type)) { + if(beq(flag, l->flag)) { + s++; + goto loop; + } + } + + n = nextarg(n, &a); + pos++; + if(a == Z) { + warn(nn, "more format than arguments %s", + lastfmt); + return; + } + if(a->type == 0) + continue; + for(l=tprot; l; l=l->link) + if(sametype(a->type, l->type)) { +/*print("checking %T/%ux %T/%ux\n", a->type, flag.b[0], l->type, l->flag.b[0]);*/ + if(beq(flag, l->flag)) + goto loop; + } + warn(nn, "format mismatch %s %T, arg %d", lastfmt, a->type, pos); + loop:; + } +} + +void +dpcheck(Node *n) +{ + char *s; + Node *a, *b; + Tname *l; + Tprot *tl; + int i, j; + + if(n == Z) + return; + b = n->left; + if(b == Z || b->op != ONAME) + return; + s = b->sym->name; + for(l=tname; l; l=l->link) + if(strcmp(s, l->name) == 0) + break; + if(l == 0) + return; + + if(l->count > 0) { + // fetch count, then check remaining length + i = l->count; + a = nil; + b = n->right; + while(i > 0) { + b = nextarg(b, &a); + i--; + } + if(a == Z) { + diag(n, "can't find count arg"); + return; + } + if(a->op != OCONST || !typechl[a->type->etype]) { + diag(n, "count is invalid constant"); + return; + } + j = a->vconst; + i = 0; + while(b != Z) { + b = nextarg(b, &a); + i++; + } + if(i != j) + diag(n, "found %d argument%s after count %d", i, i == 1 ? "" : "s", j); + } + + if(l->prot != nil) { + // check that all arguments after param or count + // are listed in type list. + i = l->count; + if(i == 0) + i = l->param; + if(i == 0) + return; + a = nil; + b = n->right; + while(i > 0) { + b = nextarg(b, &a); + i--; + } + if(a == Z) { + diag(n, "can't find count/param arg"); + return; + } + while(b != Z) { + b = nextarg(b, &a); + for(tl=l->prot; tl; tl=tl->link) + if(sametype(a->type, tl->type)) + break; + if(tl == nil) + diag(a, "invalid type %T in call to %s", a->type, s); + } + } + + if(l->param <= 0) + return; + i = l->param; + a = nil; + b = n->right; + while(i > 0) { + b = nextarg(b, &a); + i--; + } + if(a == Z) { + diag(n, "can't find format arg"); + return; + } + if(!sametype(indchar, a->type)) { + diag(n, "format arg type %T", a->type); + return; + } + if(a->op != OADDR || a->left->op != ONAME || a->left->sym != symstring) { +/* warn(n, "format arg not constant string");*/ + return; + } + s = a->left->cstring; + checkargs(b, s, l->param); +} + +void +pragpack(void) +{ + Sym *s; + + packflg = 0; + s = getsym(); + if(s) { + packflg = atoi(s->name+1); + if(strcmp(s->name, "on") == 0 || + strcmp(s->name, "yes") == 0) + packflg = 1; + } + while(getnsc() != '\n') + ; + if(debug['f']) + if(packflg) + print("%4d: pack %d\n", lineno, packflg); + else + print("%4d: pack off\n", lineno); +} + +void +pragfpround(void) +{ + Sym *s; + + fproundflg = 0; + s = getsym(); + if(s) { + fproundflg = atoi(s->name+1); + if(strcmp(s->name, "on") == 0 || + strcmp(s->name, "yes") == 0) + fproundflg = 1; + } + while(getnsc() != '\n') + ; + if(debug['f']) + if(fproundflg) + print("%4d: fproundflg %d\n", lineno, fproundflg); + else + print("%4d: fproundflg off\n", lineno); +} + +void +pragtextflag(void) +{ + Sym *s; + + textflag = 0; + s = getsym(); + textflag = 7; + if(s) + textflag = atoi(s->name+1); + while(getnsc() != '\n') + ; + if(debug['f']) + print("%4d: textflag %d\n", lineno, textflag); +} + +void +pragincomplete(void) +{ + Sym *s; + Type *t; + int istag, w, et; + + istag = 0; + s = getsym(); + if(s == nil) + goto out; + et = 0; + w = s->lexical; + if(w == LSTRUCT) + et = TSTRUCT; + else if(w == LUNION) + et = TUNION; + if(et != 0){ + s = getsym(); + if(s == nil){ + yyerror("missing struct/union tag in pragma incomplete"); + goto out; + } + if(s->lexical != LNAME && s->lexical != LTYPE){ + yyerror("invalid struct/union tag: %s", s->name); + goto out; + } + dotag(s, et, 0); + istag = 1; + }else if(strcmp(s->name, "_off_") == 0){ + debug['T'] = 0; + goto out; + }else if(strcmp(s->name, "_on_") == 0){ + debug['T'] = 1; + goto out; + } + t = s->type; + if(istag) + t = s->suetag; + if(t == T) + yyerror("unknown type %s in pragma incomplete", s->name); + else if(!typesu[t->etype]) + yyerror("not struct/union type in pragma incomplete: %s", s->name); + else + t->garb |= GINCOMPLETE; +out: + while(getnsc() != '\n') + ; + if(debug['f']) + print("%s incomplete\n", s->name); +} + +Sym* +getimpsym(void) +{ + int c; + char *cp; + + c = getnsc(); + if(isspace(c) || c == '"') { + unget(c); + return S; + } + for(cp = symb;;) { + if(cp <= symb+NSYMB-4) + *cp++ = c; + c = getc(); + if(c > 0 && !isspace(c) && c != '"') + continue; + unget(c); + break; + } + *cp = 0; + if(cp > symb+NSYMB-4) + yyerror("symbol too large: %s", symb); + return lookup(); +} + +void +pragdynimport(void) +{ + Sym *local, *remote; + char *path; + Dynimp *f; + + local = getimpsym(); + if(local == nil) + goto err; + + remote = getimpsym(); + if(remote == nil) + goto err; + + path = getquoted(); + if(path == nil) + goto err; + + if(ndynimp%32 == 0) + dynimp = realloc(dynimp, (ndynimp+32)*sizeof dynimp[0]); + f = &dynimp[ndynimp++]; + f->local = local->name; + f->remote = remote->name; + f->path = path; + goto out; + +err: + yyerror("usage: #pragma dynimport local remote \"path\""); + +out: + while(getnsc() != '\n') + ; +} + +void +pragdynexport(void) +{ + Sym *local, *remote; + Dynexp *f; + + local = getsym(); + if(local == nil) + goto err; + + remote = getsym(); + if(remote == nil) + goto err; + + if(ndynexp%32 == 0) + dynexp = realloc(dynexp, (ndynexp+32)*sizeof dynexp[0]); + f = &dynexp[ndynexp++]; + f->local = local->name; + f->remote = remote->name; + goto out; + +err: + yyerror("usage: #pragma dynexport local remote"); + +out: + while(getnsc() != '\n') + ; +} diff --git a/src/cmd/cc/funct.c b/src/cmd/cc/funct.c new file mode 100644 index 000000000..99477b2b2 --- /dev/null +++ b/src/cmd/cc/funct.c @@ -0,0 +1,431 @@ +// Inferno utils/cc/funct.c +// http://code.google.com/p/inferno-os/source/browse/utils/cc/funct.c +// +// Copyright © 1994-1999 Lucent Technologies Inc. All rights reserved. +// Portions Copyright © 1995-1997 C H Forsyth (forsyth@terzarima.net) +// Portions Copyright © 1997-1999 Vita Nuova Limited +// Portions Copyright © 2000-2007 Vita Nuova Holdings Limited (www.vitanuova.com) +// Portions Copyright © 2004,2006 Bruce Ellis +// Portions Copyright © 2005-2007 C H Forsyth (forsyth@terzarima.net) +// Revisions Copyright © 2000-2007 Lucent Technologies Inc. and others +// Portions Copyright © 2009 The Go Authors. All rights reserved. +// +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to deal +// in the Software without restriction, including without limitation the rights +// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +// copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +// THE SOFTWARE. + +#include +#include "cc.h" + +typedef struct Ftab Ftab; +struct Ftab +{ + char op; + char* name; + char typ; +}; +typedef struct Gtab Gtab; +struct Gtab +{ + char etype; + char* name; +}; + +Ftab ftabinit[OEND]; +Gtab gtabinit[NTYPE]; + +int +isfunct(Node *n) +{ + Type *t, *t1; + Funct *f; + Node *l; + Sym *s; + int o; + + o = n->op; + if(n->left == Z) + goto no; + t = n->left->type; + if(t == T) + goto no; + f = t->funct; + + switch(o) { + case OAS: // put cast on rhs + case OASI: + case OASADD: + case OASAND: + case OASASHL: + case OASASHR: + case OASDIV: + case OASLDIV: + case OASLMOD: + case OASLMUL: + case OASLSHR: + case OASMOD: + case OASMUL: + case OASOR: + case OASSUB: + case OASXOR: + if(n->right == Z) + goto no; + t1 = n->right->type; + if(t1 == T) + goto no; + if(t1->funct == f) + break; + + l = new(OXXX, Z, Z); + *l = *n->right; + + n->right->left = l; + n->right->right = Z; + n->right->type = t; + n->right->op = OCAST; + + if(!isfunct(n->right)) + prtree(n, "isfunc !"); + break; + + case OCAST: // t f(T) or T f(t) + t1 = n->type; + if(t1 == T) + goto no; + if(f != nil) { + s = f->castfr[t1->etype]; + if(s == S) + goto no; + n->right = n->left; + goto build; + } + f = t1->funct; + if(f != nil) { + s = f->castto[t->etype]; + if(s == S) + goto no; + n->right = n->left; + goto build; + } + goto no; + } + + if(f == nil) + goto no; + s = f->sym[o]; + if(s == S) + goto no; + + /* + * the answer is yes, + * now we rewrite the node + * and give diagnostics + */ + switch(o) { + default: + diag(n, "isfunct op missing %O\n", o); + goto bad; + + case OADD: // T f(T, T) + case OAND: + case OASHL: + case OASHR: + case ODIV: + case OLDIV: + case OLMOD: + case OLMUL: + case OLSHR: + case OMOD: + case OMUL: + case OOR: + case OSUB: + case OXOR: + + case OEQ: // int f(T, T) + case OGE: + case OGT: + case OHI: + case OHS: + case OLE: + case OLO: + case OLS: + case OLT: + case ONE: + if(n->right == Z) + goto bad; + t1 = n->right->type; + if(t1 == T) + goto bad; + if(t1->funct != f) + goto bad; + n->right = new(OLIST, n->left, n->right); + break; + + case OAS: // structure copies done by the compiler + case OASI: + goto no; + + case OASADD: // T f(T*, T) + case OASAND: + case OASASHL: + case OASASHR: + case OASDIV: + case OASLDIV: + case OASLMOD: + case OASLMUL: + case OASLSHR: + case OASMOD: + case OASMUL: + case OASOR: + case OASSUB: + case OASXOR: + if(n->right == Z) + goto bad; + t1 = n->right->type; + if(t1 == T) + goto bad; + if(t1->funct != f) + goto bad; + n->right = new(OLIST, new(OADDR, n->left, Z), n->right); + break; + + case OPOS: // T f(T) + case ONEG: + case ONOT: + case OCOM: + n->right = n->left; + break; + + + } + +build: + l = new(ONAME, Z, Z); + l->sym = s; + l->type = s->type; + l->etype = s->type->etype; + l->xoffset = s->offset; + l->class = s->class; + tcomo(l, 0); + + n->op = OFUNC; + n->left = l; + n->type = l->type->link; + if(tcompat(n, T, l->type, tfunct)) + goto bad; + if(tcoma(n->left, n->right, l->type->down, 1)) + goto bad; + return 1; + +no: + return 0; + +bad: + diag(n, "cant rewrite typestr for op %O\n", o); + prtree(n, "isfunct"); + n->type = T; + return 1; +} + +void +dclfunct(Type *t, Sym *s) +{ + Funct *f; + Node *n; + Type *f1, *f2, *f3, *f4; + int o, i, c; + char str[100]; + + if(t->funct) + return; + + // recognize generated tag of dorm _%d_ + if(t->tag == S) + goto bad; + for(i=0; c = t->tag->name[i]; i++) { + if(c == '_') { + if(i == 0 || t->tag->name[i+1] == 0) + continue; + break; + } + if(c < '0' || c > '9') + break; + } + if(c == 0) + goto bad; + + f = alloc(sizeof(*f)); + for(o=0; osym); o++) + f->sym[o] = S; + + t->funct = f; + + f1 = typ(TFUNC, t); + f1->down = copytyp(t); + f1->down->down = t; + + f2 = typ(TFUNC, types[TINT]); + f2->down = copytyp(t); + f2->down->down = t; + + f3 = typ(TFUNC, t); + f3->down = typ(TIND, t); + f3->down->down = t; + + f4 = typ(TFUNC, t); + f4->down = t; + + for(i=0;; i++) { + o = ftabinit[i].op; + if(o == OXXX) + break; + sprint(str, "%s_%s_", t->tag->name, ftabinit[i].name); + n = new(ONAME, Z, Z); + n->sym = slookup(str); + f->sym[o] = n->sym; + switch(ftabinit[i].typ) { + default: + diag(Z, "dclfunct op missing %d\n", ftabinit[i].typ); + break; + + case 1: // T f(T,T) + + dodecl(xdecl, CEXTERN, f1, n); + break; + + case 2: // int f(T,T) == + dodecl(xdecl, CEXTERN, f2, n); + break; + + case 3: // void f(T*,T) += + dodecl(xdecl, CEXTERN, f3, n); + break; + + case 4: // T f(T) ~ + dodecl(xdecl, CEXTERN, f4, n); + break; + } + } + for(i=0;; i++) { + o = gtabinit[i].etype; + if(o == TXXX) + break; + + /* + * OCAST types T1 _T2_T1_(T2) + */ + sprint(str, "_%s%s_", gtabinit[i].name, t->tag->name); + n = new(ONAME, Z, Z); + n->sym = slookup(str); + f->castto[o] = n->sym; + + f1 = typ(TFUNC, t); + f1->down = types[o]; + dodecl(xdecl, CEXTERN, f1, n); + + sprint(str, "%s_%s_", t->tag->name, gtabinit[i].name); + n = new(ONAME, Z, Z); + n->sym = slookup(str); + f->castfr[o] = n->sym; + + f1 = typ(TFUNC, types[o]); + f1->down = t; + dodecl(xdecl, CEXTERN, f1, n); + } + return; +bad: + diag(Z, "dclfunct bad %T %s\n", t, s->name); +} + +Gtab gtabinit[NTYPE] = +{ + TCHAR, "c", + TUCHAR, "uc", + TSHORT, "h", + TUSHORT, "uh", + TINT, "i", + TUINT, "ui", + TLONG, "l", + TULONG, "ul", + TVLONG, "v", + TUVLONG, "uv", + TFLOAT, "f", + TDOUBLE, "d", + TXXX +}; + +Ftab ftabinit[OEND] = +{ + OADD, "add", 1, + OAND, "and", 1, + OASHL, "ashl", 1, + OASHR, "ashr", 1, + ODIV, "div", 1, + OLDIV, "ldiv", 1, + OLMOD, "lmod", 1, + OLMUL, "lmul", 1, + OLSHR, "lshr", 1, + OMOD, "mod", 1, + OMUL, "mul", 1, + OOR, "or", 1, + OSUB, "sub", 1, + OXOR, "xor", 1, + + OEQ, "eq", 2, + OGE, "ge", 2, + OGT, "gt", 2, + OHI, "hi", 2, + OHS, "hs", 2, + OLE, "le", 2, + OLO, "lo", 2, + OLS, "ls", 2, + OLT, "lt", 2, + ONE, "ne", 2, + + OASADD, "asadd", 3, + OASAND, "asand", 3, + OASASHL, "asashl", 3, + OASASHR, "asashr", 3, + OASDIV, "asdiv", 3, + OASLDIV, "asldiv", 3, + OASLMOD, "aslmod", 3, + OASLMUL, "aslmul", 3, + OASLSHR, "aslshr", 3, + OASMOD, "asmod", 3, + OASMUL, "asmul", 3, + OASOR, "asor", 3, + OASSUB, "assub", 3, + OASXOR, "asxor", 3, + + OPOS, "pos", 4, + ONEG, "neg", 4, + OCOM, "com", 4, + ONOT, "not", 4, + +// OPOSTDEC, +// OPOSTINC, +// OPREDEC, +// OPREINC, + + OXXX, +}; + +// Node* nodtestv; + +// Node* nodvpp; +// Node* nodppv; +// Node* nodvmm; +// Node* nodmmv; diff --git a/src/cmd/cc/godefs.c b/src/cmd/cc/godefs.c new file mode 100644 index 000000000..3ba979c8a --- /dev/null +++ b/src/cmd/cc/godefs.c @@ -0,0 +1,388 @@ +// cmd/cc/godefs.cc +// +// derived from pickle.cc which itself was derived from acid.cc. +// +// Copyright © 1994-1999 Lucent Technologies Inc. All rights reserved. +// Portions Copyright © 1995-1997 C H Forsyth (forsyth@terzarima.net) +// Portions Copyright © 1997-1999 Vita Nuova Limited +// Portions Copyright © 2000-2007 Vita Nuova Holdings Limited (www.vitanuova.com) +// Portions Copyright © 2004,2006 Bruce Ellis +// Portions Copyright © 2005-2007 C H Forsyth (forsyth@terzarima.net) +// Revisions Copyright © 2000-2007 Lucent Technologies Inc. and others +// Portions Copyright © 2009-2011 The Go Authors. All rights reserved. +// +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to deal +// in the Software without restriction, including without limitation the rights +// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +// copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +// THE SOFTWARE. + +#include +#include "cc.h" + +static int upper; + +static char *kwd[] = +{ + "_bool", + "_break", + "_byte", + "_case", + "_chan", + "_complex128", + "_complex64", + "_const", + "_continue", + "_default", + "_defer", + "_else", + "_fallthrough", + "_false", + "_float32", + "_float64", + "_for", + "_func", + "_go", + "_goto", + "_if", + "_import", + "_int", + "_int16", + "_int32", + "_int64", + "_int8", + "_interface", + "_intptr", + "_map", + "_package", + "_panic", + "_range", + "_return", + "_select", + "_string", + "_struct", + "_switch", + "_true", + "_type", + "_uint", + "_uint16", + "_uint32", + "_uint64", + "_uint8", + "_uintptr", + "_var", +}; + +static char* +pmap(char *s) +{ + int i, bot, top, mid; + + bot = -1; + top = nelem(kwd); + while(top - bot > 1){ + mid = (bot + top) / 2; + i = strcmp(kwd[mid]+1, s); + if(i == 0) + return kwd[mid]; + if(i < 0) + bot = mid; + else + top = mid; + } + + return s; +} + + +int +Uconv(Fmt *fp) +{ + char str[STRINGSZ+1]; + char *s, *n; + int i; + + str[0] = 0; + s = va_arg(fp->args, char*); + + // strip package name + n = strrchr(s, '.'); + if(n != nil) + s = n + 1; + + if(s && *s) { + if(upper) + str[0] = toupper(*s); + else + str[0] = tolower(*s); + for(i = 1; i < STRINGSZ && s[i] != 0; i++) + str[i] = tolower(s[i]); + str[i] = 0; + } + + return fmtstrcpy(fp, pmap(str)); +} + + +static Sym* +findsue(Type *t) +{ + int h; + Sym *s; + + if(t != T) + for(h=0; hlink) + if(s->suetag && s->suetag->link == t) + return s; + return 0; +} + +static void +printtypename(Type *t) +{ + Sym *s; + Type *t1; + int w; + char *n; + + for( ; t != nil; t = t->link) { + switch(t->etype) { + case TIND: + // Special handling of *void. + if(t->link != nil && t->link->etype==TVOID) { + Bprint(&outbuf, "unsafe.Pointer"); + return; + } + // *func == func + if(t->link != nil && t->link->etype==TFUNC) + continue; + Bprint(&outbuf, "*"); + continue; + case TARRAY: + w = t->width; + if(t->link && t->link->width) + w /= t->link->width; + Bprint(&outbuf, "[%d]", w); + continue; + } + break; + } + + if(t == nil) { + Bprint(&outbuf, "bad // should not happen"); + return; + } + + switch(t->etype) { + case TINT: + Bprint(&outbuf, "int"); + break; + case TUINT: + Bprint(&outbuf, "uint"); + break; + case TCHAR: + Bprint(&outbuf, "int8"); + break; + case TUCHAR: + Bprint(&outbuf, "uint8"); + break; + case TSHORT: + Bprint(&outbuf, "int16"); + break; + case TUSHORT: + Bprint(&outbuf, "uint16"); + break; + case TLONG: + Bprint(&outbuf, "int32"); + break; + case TULONG: + Bprint(&outbuf, "uint32"); + break; + case TVLONG: + Bprint(&outbuf, "int64"); + break; + case TUVLONG: + Bprint(&outbuf, "uint64"); + break; + case TFLOAT: + Bprint(&outbuf, "float32"); + break; + case TDOUBLE: + Bprint(&outbuf, "float64"); + break; + case TUNION: + case TSTRUCT: + s = findsue(t->link); + n = "bad"; + if(s != S) + n = s->name; + else if(t->tag) + n = t->tag->name; + if(strcmp(n, "String") == 0){ + Bprint(&outbuf, "string"); + } else if(strcmp(n, "Slice") == 0){ + Bprint(&outbuf, "[]byte"); + } else + Bprint(&outbuf, "%U", n); + break; + case TFUNC: + Bprint(&outbuf, "func("); + for(t1 = t->down; t1 != T; t1 = t1->down) { + if(t1->etype == TVOID) + break; + if(t1 != t->down) + Bprint(&outbuf, ", "); + printtypename(t1); + } + Bprint(&outbuf, ")"); + if(t->link && t->link->etype != TVOID) { + Bprint(&outbuf, " "); + printtypename(t->link); + } + break; + case TDOT: + Bprint(&outbuf, "...interface{}"); + break; + default: + Bprint(&outbuf, " weird<%T>", t); + } +} + +static int +dontrun(void) +{ + Io *i; + int n; + + if(!debug['q'] && !debug['Q']) + return 1; + if(debug['q'] + debug['Q'] > 1) { + n = 0; + for(i=iostack; i; i=i->link) + n++; + if(n > 1) + return 1; + } + + upper = debug['Q']; + return 0; +} + +void +godeftype(Type *t) +{ + Sym *s; + Type *l; + int gotone; + + if(dontrun()) + return; + + switch(t->etype) { + case TUNION: + case TSTRUCT: + s = findsue(t->link); + if(s == S) { + Bprint(&outbuf, "/* can't find %T */\n\n", t); + return; + } + + gotone = 0; // for unions, take first member of size equal to union + Bprint(&outbuf, "type %U struct {\n", s->name); + for(l = t->link; l != T; l = l->down) { + Bprint(&outbuf, "\t"); + if(t->etype == TUNION) { + if(!gotone && l->width == t->width) + gotone = 1; + else + Bprint(&outbuf, "// (union)\t"); + } + if(l->sym != nil) // not anonymous field + Bprint(&outbuf, "%U\t", l->sym->name); + printtypename(l); + Bprint(&outbuf, "\n"); + } + Bprint(&outbuf, "}\n\n"); + break; + + default: + Bprint(&outbuf, "/* %T */\n\n", t); + break; + } +} + +void +godefvar(Sym *s) +{ + Type *t, *t1; + char n; + + if(dontrun()) + return; + + t = s->type; + if(t == nil) + return; + + switch(t->etype) { + case TENUM: + if(!typefd[t->etype]) + Bprint(&outbuf, "const %U = %lld\n", s->name, s->vconst); + else + Bprint(&outbuf, "const %U = %f\n;", s->name, s->fconst); + break; + + case TFUNC: + Bprint(&outbuf, "func %U(", s->name); + n = 'a'; + for(t1 = t->down; t1 != T; t1 = t1->down) { + if(t1->etype == TVOID) + break; + if(t1 != t->down) + Bprint(&outbuf, ", "); + Bprint(&outbuf, "%c ", n++); + printtypename(t1); + } + Bprint(&outbuf, ")"); + if(t->link && t->link->etype != TVOID) { + Bprint(&outbuf, " "); + printtypename(t->link); + } + Bprint(&outbuf, "\n"); + break; + + default: + switch(s->class) { + case CTYPEDEF: + if(!typesu[t->etype]) { + Bprint(&outbuf, "// type %U\t", s->name); + printtypename(t); + Bprint(&outbuf, "\n"); + } + break; + case CSTATIC: + case CEXTERN: + case CGLOBL: + if(strchr(s->name, '$') != nil) // TODO(lvd) + break; + Bprint(&outbuf, "var %U\t", s->name); + printtypename(t); + Bprint(&outbuf, "\n"); + break; + } + break; + } +} diff --git a/src/cmd/cc/lex.c b/src/cmd/cc/lex.c new file mode 100644 index 000000000..9fb2f9e4d --- /dev/null +++ b/src/cmd/cc/lex.c @@ -0,0 +1,1561 @@ +// Inferno utils/cc/lex.c +// http://code.google.com/p/inferno-os/source/browse/utils/cc/lex.c +// +// Copyright © 1994-1999 Lucent Technologies Inc. All rights reserved. +// Portions Copyright © 1995-1997 C H Forsyth (forsyth@terzarima.net) +// Portions Copyright © 1997-1999 Vita Nuova Limited +// Portions Copyright © 2000-2007 Vita Nuova Holdings Limited (www.vitanuova.com) +// Portions Copyright © 2004,2006 Bruce Ellis +// Portions Copyright © 2005-2007 C H Forsyth (forsyth@terzarima.net) +// Revisions Copyright © 2000-2007 Lucent Technologies Inc. and others +// Portions Copyright © 2009 The Go Authors. All rights reserved. +// +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to deal +// in the Software without restriction, including without limitation the rights +// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +// copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +// THE SOFTWARE. + +#include +#include "cc.h" +#include "y.tab.h" + +#ifndef CPP +#define CPP "cpp" +#endif + +int +systemtype(int sys) +{ +#ifdef _WIN32 + return sys&Windows; +#else + return sys&Plan9; +#endif +} + +int +pathchar(void) +{ + return '/'; +} + +/* + * known debug flags + * -a acid declaration output + * -A !B + * -B non ANSI + * -d print declarations + * -D name define + * -F format specification check + * -G print pgen stuff + * -g print cgen trees + * -i print initialization + * -I path include + * -l generate little-endian code + * -L print every NAME symbol + * -M constant multiplication + * -m print add/sub/mul trees + * -n print acid or godefs to file (%.c=%.acid) (with -a or -aa) + * -o file output file + * -p use standard cpp ANSI preprocessor (not on windows) + * -p something with peepholes + * -q print equivalent Go code for variables and types (lower-case identifiers) + * -Q print equivalent Go code for variables and types (upper-case identifiers) + * -r print registerization + * -s print structure offsets (with -a or -aa) + * -S print assembly + * -t print type trees + * -V enable void* conversion warnings + * -v verbose printing + * -w print warnings + * -X abort on error + * -. Inhibit search for includes in source directory + */ + +void +main(int argc, char *argv[]) +{ + char **defs, *p; + int c, ndef; + + ensuresymb(NSYMB); + memset(debug, 0, sizeof(debug)); + tinit(); + cinit(); + ginit(); + arginit(); + + tufield = simplet((1L<etype) | BUNSIGNED); + ndef = 0; + defs = nil; + outfile = 0; + setinclude("."); + ARGBEGIN { + default: + c = ARGC(); + if(c >= 0 && c < sizeof(debug)) + debug[c]++; + break; + + case 'l': /* for little-endian mips */ + if(thechar != 'v'){ + print("can only use -l with vc"); + errorexit(); + } + thechar = '0'; + thestring = "spim"; + break; + + case 'o': + outfile = ARGF(); + break; + + case 'D': + p = ARGF(); + if(p) { + if(ndef%8 == 0) + defs = allocn(defs, ndef*sizeof(char *), + 8*sizeof(char *)); + defs[ndef++] = p; + dodefine(p); + } + break; + + case 'I': + p = ARGF(); + setinclude(p); + break; + } ARGEND + if(argc < 1 && outfile == 0) { + print("usage: %cc [-options] files\n", thechar); + errorexit(); + } + if(argc > 1){ + print("can't compile multiple files\n"); + errorexit(); + } + + if(argc == 0) + c = compile("stdin", defs, ndef); + else + c = compile(argv[0], defs, ndef); + + if(c) + errorexit(); + exits(0); +} + +int +compile(char *file, char **defs, int ndef) +{ + char *ofile; + char *p, **av, opt[256]; + int i, c, fd[2]; + static int first = 1; + + ofile = alloc(strlen(file)+10); + strcpy(ofile, file); + p = utfrrune(ofile, pathchar()); + if(p) { + *p++ = 0; + if(!debug['.']) + include[0] = strdup(ofile); + } else + p = ofile; + + if(outfile == 0) { + outfile = p; + if(outfile) { + if(p = utfrrune(outfile, '.')) + if(p[1] == 'c' && p[2] == 0) + p[0] = 0; + p = utfrune(outfile, 0); + if(debug['a'] && debug['n']) + strcat(p, ".acid"); + else if((debug['q'] || debug['Q']) && debug['n']) + strcat(p, ".go"); + else { + p[0] = '.'; + p[1] = thechar; + p[2] = 0; + } + } else + outfile = "/dev/null"; + } + + if (first) + Binit(&diagbuf, 1, OWRITE); + /* + * if we're writing acid to standard output, don't keep scratching + * outbuf. + */ + if((debug['a'] || debug['q'] || debug['Q']) && !debug['n']) { + if (first) { + outfile = 0; + Binit(&outbuf, dup(1, -1), OWRITE); + dup(2, 1); + } + } else { + c = create(outfile, OWRITE, 0664); + if(c < 0) { + diag(Z, "cannot open %s - %r", outfile); + outfile = 0; + errorexit(); + } + Binit(&outbuf, c, OWRITE); + outfile = strdup(outfile); + } + newio(); + first = 0; + + /* Use an ANSI preprocessor */ + if(debug['p']) { + if(systemtype(Windows)) { + diag(Z, "-p option not supported on windows"); + errorexit(); + } + if(access(file, AREAD) < 0) { + diag(Z, "%s does not exist", file); + errorexit(); + } + if(pipe(fd) < 0) { + diag(Z, "pipe failed"); + errorexit(); + } + switch(fork()) { + case -1: + diag(Z, "fork failed"); + errorexit(); + case 0: + close(fd[0]); + dup(fd[1], 1); + close(fd[1]); + av = alloc((ndef+ninclude+5)*sizeof(char *)); + av[0] = CPP; + i = 1; + if(debug['.']){ + sprint(opt, "-."); + av[i++] = strdup(opt); + } + if(debug['+']) { + sprint(opt, "-+"); + av[i++] = strdup(opt); + } + for(c = 0; c < ndef; c++) + av[i++] = smprint("-D%s", defs[c]); + for(c = 0; c < ninclude; c++) + av[i++] = smprint("-I%s", include[c]); + if(strcmp(file, "stdin") != 0) + av[i++] = file; + av[i] = 0; + if(debug['p'] > 1) { + for(c = 0; c < i; c++) + fprint(2, "%s ", av[c]); + fprint(2, "\n"); + } + exec(av[0], av); + fprint(2, "can't exec C preprocessor %s: %r\n", CPP); + errorexit(); + default: + close(fd[1]); + newfile(file, fd[0]); + break; + } + } else { + if(strcmp(file, "stdin") == 0) + newfile(file, 0); + else + newfile(file, -1); + } + yyparse(); + if(!debug['a'] && !debug['q'] && !debug['Q']) + gclean(); + return nerrors; +} + +void +errorexit(void) +{ + if(outfile) + remove(outfile); + exits("error"); +} + +void +pushio(void) +{ + Io *i; + + i = iostack; + if(i == I) { + yyerror("botch in pushio"); + errorexit(); + } + i->p = fi.p; + i->c = fi.c; +} + +void +newio(void) +{ + Io *i; + static int pushdepth = 0; + + i = iofree; + if(i == I) { + pushdepth++; + if(pushdepth > 1000) { + yyerror("macro/io expansion too deep"); + errorexit(); + } + i = alloc(sizeof(*i)); + } else + iofree = i->link; + i->c = 0; + i->f = -1; + ionext = i; +} + +void +newfile(char *s, int f) +{ + Io *i; + + if(debug['e']) + print("%L: %s\n", lineno, s); + + i = ionext; + i->link = iostack; + iostack = i; + i->f = f; + if(f < 0) + i->f = open(s, 0); + if(i->f < 0) { + yyerror("%cc: %r: %s", thechar, s); + errorexit(); + } + fi.c = 0; + linehist(s, 0); +} + +Sym* +slookup(char *s) +{ + ensuresymb(strlen(s)); + strcpy(symb, s); + return lookup(); +} + +Sym* +lookup(void) +{ + Sym *s; + uint32 h; + char *p; + int c, n; + char *r, *w; + + if((uchar)symb[0] == 0xc2 && (uchar)symb[1] == 0xb7) { + // turn leading · into ""· + h = strlen(symb); + ensuresymb(h+2); + memmove(symb+2, symb, h+1); + symb[0] = '"'; + symb[1] = '"'; + } + + // turn · into . + for(r=w=symb; *r; r++) { + if((uchar)*r == 0xc2 && (uchar)*(r+1) == 0xb7) { + *w++ = '.'; + r++; + }else + *w++ = *r; + } + *w = '\0'; + + h = 0; + for(p=symb; *p;) { + h = h * 3; + h += *p++; + } + n = (p - symb) + 1; + h &= 0xffffff; + h %= NHASH; + c = symb[0]; + for(s = hash[h]; s != S; s = s->link) { + if(s->name[0] != c) + continue; + if(strcmp(s->name, symb) == 0) + return s; + } + s = alloc(sizeof(*s)); + s->name = alloc(n); + memmove(s->name, symb, n); + s->link = hash[h]; + hash[h] = s; + syminit(s); + + return s; +} + +void +syminit(Sym *s) +{ + s->lexical = LNAME; + s->block = 0; + s->offset = 0; + s->type = T; + s->suetag = T; + s->class = CXXX; + s->aused = 0; + s->sig = SIGNONE; +} + +#define EOF (-1) +#define IGN (-2) +#define ESC (1<<20) +#define GETC() ((--fi.c < 0)? filbuf(): (*fi.p++ & 0xff)) + +enum +{ + Numdec = 1<<0, + Numlong = 1<<1, + Numuns = 1<<2, + Numvlong = 1<<3, + Numflt = 1<<4, +}; + +int32 +yylex(void) +{ + vlong vv; + int32 c, c1, t; + char *cp; + Rune rune; + Sym *s; + + if(peekc != IGN) { + c = peekc; + peekc = IGN; + goto l1; + } +l0: + c = GETC(); + +l1: + if(c >= Runeself) { + /* + * extension -- + * all multibyte runes are alpha + */ + cp = symb; + goto talph; + } + if(isspace(c)) { + if(c == '\n') + lineno++; + goto l0; + } + if(isalpha(c)) { + cp = symb; + if(c != 'L') + goto talph; + *cp++ = c; + c = GETC(); + if(c == '\'') { + /* L'x' */ + c = escchar('\'', 1, 0); + if(c == EOF) + c = '\''; + c1 = escchar('\'', 1, 0); + if(c1 != EOF) { + yyerror("missing '"); + peekc = c1; + } + yylval.vval = convvtox(c, TUSHORT); + return LUCONST; + } + if(c == '"') { + goto caselq; + } + goto talph; + } + if(isdigit(c)) + goto tnum; + switch(c) + { + + case EOF: + peekc = EOF; + return -1; + + case '_': + cp = symb; + goto talph; + + case '#': + domacro(); + goto l0; + + case '.': + c1 = GETC(); + if(isdigit(c1)) { + cp = symb; + *cp++ = c; + c = c1; + c1 = 0; + goto casedot; + } + break; + + case '"': + strcpy(symb, "\"\""); + cp = alloc(0); + c1 = 0; + + /* "..." */ + for(;;) { + c = escchar('"', 0, 1); + if(c == EOF) + break; + if(c & ESC) { + cp = allocn(cp, c1, 1); + cp[c1++] = c; + } else { + rune = c; + c = runelen(rune); + cp = allocn(cp, c1, c); + runetochar(cp+c1, &rune); + c1 += c; + } + } + yylval.sval.l = c1; + do { + cp = allocn(cp, c1, 1); + cp[c1++] = 0; + } while(c1 & MAXALIGN); + yylval.sval.s = cp; + return LSTRING; + + caselq: + /* L"..." */ + strcpy(symb, "\"L\""); + cp = alloc(0); + c1 = 0; + for(;;) { + c = escchar('"', 1, 0); + if(c == EOF) + break; + cp = allocn(cp, c1, sizeof(ushort)); + *(ushort*)(cp + c1) = c; + c1 += sizeof(ushort); + } + yylval.sval.l = c1; + do { + cp = allocn(cp, c1, sizeof(ushort)); + *(ushort*)(cp + c1) = 0; + c1 += sizeof(ushort); + } while(c1 & MAXALIGN); + yylval.sval.s = cp; + return LLSTRING; + + case '\'': + /* '.' */ + c = escchar('\'', 0, 0); + if(c == EOF) + c = '\''; + c1 = escchar('\'', 0, 0); + if(c1 != EOF) { + yyerror("missing '"); + peekc = c1; + } + vv = c; + yylval.vval = convvtox(vv, TUCHAR); + if(yylval.vval != vv) + yyerror("overflow in character constant: 0x%x", c); + else + if(c & 0x80){ + nearln = lineno; + warn(Z, "sign-extended character constant"); + } + yylval.vval = convvtox(vv, TCHAR); + return LCONST; + + case '/': + c1 = GETC(); + if(c1 == '*') { + for(;;) { + c = getr(); + while(c == '*') { + c = getr(); + if(c == '/') + goto l0; + } + if(c == EOF) { + yyerror("eof in comment"); + errorexit(); + } + } + } + if(c1 == '/') { + for(;;) { + c = getr(); + if(c == '\n') + goto l0; + if(c == EOF) { + yyerror("eof in comment"); + errorexit(); + } + } + } + if(c1 == '=') + return LDVE; + break; + + case '*': + c1 = GETC(); + if(c1 == '=') + return LMLE; + break; + + case '%': + c1 = GETC(); + if(c1 == '=') + return LMDE; + break; + + case '+': + c1 = GETC(); + if(c1 == '+') + return LPP; + if(c1 == '=') + return LPE; + break; + + case '-': + c1 = GETC(); + if(c1 == '-') + return LMM; + if(c1 == '=') + return LME; + if(c1 == '>') + return LMG; + break; + + case '>': + c1 = GETC(); + if(c1 == '>') { + c = LRSH; + c1 = GETC(); + if(c1 == '=') + return LRSHE; + break; + } + if(c1 == '=') + return LGE; + break; + + case '<': + c1 = GETC(); + if(c1 == '<') { + c = LLSH; + c1 = GETC(); + if(c1 == '=') + return LLSHE; + break; + } + if(c1 == '=') + return LLE; + break; + + case '=': + c1 = GETC(); + if(c1 == '=') + return LEQ; + break; + + case '!': + c1 = GETC(); + if(c1 == '=') + return LNE; + break; + + case '&': + c1 = GETC(); + if(c1 == '&') + return LANDAND; + if(c1 == '=') + return LANDE; + break; + + case '|': + c1 = GETC(); + if(c1 == '|') + return LOROR; + if(c1 == '=') + return LORE; + break; + + case '^': + c1 = GETC(); + if(c1 == '=') + return LXORE; + break; + + default: + return c; + } + peekc = c1; + return c; + +talph: + /* + * cp is set to symb and some + * prefix has been stored + */ + for(;;) { + if(c >= Runeself) { + for(c1=0;;) { + cp[c1++] = c; + if(fullrune(cp, c1)) + break; + c = GETC(); + } + cp += c1; + c = GETC(); + continue; + } + if(!isalnum(c) && c != '_') + break; + *cp++ = c; + c = GETC(); + } + *cp = 0; + if(debug['L']) + print("%L: %s\n", lineno, symb); + peekc = c; + s = lookup(); + if(s->macro) { + newio(); + cp = ionext->b; + macexpand(s, cp); + pushio(); + ionext->link = iostack; + iostack = ionext; + fi.p = cp; + fi.c = strlen(cp); + if(peekc != IGN) { + cp[fi.c++] = peekc; + cp[fi.c] = 0; + peekc = IGN; + } + goto l0; + } + yylval.sym = s; + if(s->class == CTYPEDEF || s->class == CTYPESTR) + return LTYPE; + return s->lexical; + +tnum: + c1 = 0; + cp = symb; + if(c != '0') { + c1 |= Numdec; + for(;;) { + *cp++ = c; + c = GETC(); + if(isdigit(c)) + continue; + goto dc; + } + } + *cp++ = c; + c = GETC(); + if(c == 'x' || c == 'X') + for(;;) { + *cp++ = c; + c = GETC(); + if(isdigit(c)) + continue; + if(c >= 'a' && c <= 'f') + continue; + if(c >= 'A' && c <= 'F') + continue; + if(cp == symb+2) + yyerror("malformed hex constant"); + goto ncu; + } + if(c < '0' || c > '7') + goto dc; + for(;;) { + if(c >= '0' && c <= '7') { + *cp++ = c; + c = GETC(); + continue; + } + goto ncu; + } + +dc: + if(c == '.') + goto casedot; + if(c == 'e' || c == 'E') + goto casee; + +ncu: + if((c == 'U' || c == 'u') && !(c1 & Numuns)) { + c = GETC(); + c1 |= Numuns; + goto ncu; + } + if((c == 'L' || c == 'l') && !(c1 & Numvlong)) { + c = GETC(); + if(c1 & Numlong) + c1 |= Numvlong; + c1 |= Numlong; + goto ncu; + } + *cp = 0; + peekc = c; + if(mpatov(symb, &yylval.vval)) + yyerror("overflow in constant"); + + vv = yylval.vval; + if(c1 & Numvlong) { + if((c1 & Numuns) || convvtox(vv, TVLONG) < 0) { + c = LUVLCONST; + t = TUVLONG; + goto nret; + } + c = LVLCONST; + t = TVLONG; + goto nret; + } + if(c1 & Numlong) { + if((c1 & Numuns) || convvtox(vv, TLONG) < 0) { + c = LULCONST; + t = TULONG; + goto nret; + } + c = LLCONST; + t = TLONG; + goto nret; + } + if((c1 & Numuns) || convvtox(vv, TINT) < 0) { + c = LUCONST; + t = TUINT; + goto nret; + } + c = LCONST; + t = TINT; + goto nret; + +nret: + yylval.vval = convvtox(vv, t); + if(yylval.vval != vv){ + nearln = lineno; + warn(Z, "truncated constant: %T %s", types[t], symb); + } + return c; + +casedot: + for(;;) { + *cp++ = c; + c = GETC(); + if(!isdigit(c)) + break; + } + if(c != 'e' && c != 'E') + goto caseout; + +casee: + *cp++ = 'e'; + c = GETC(); + if(c == '+' || c == '-') { + *cp++ = c; + c = GETC(); + } + if(!isdigit(c)) + yyerror("malformed fp constant exponent"); + while(isdigit(c)) { + *cp++ = c; + c = GETC(); + } + +caseout: + if(c == 'L' || c == 'l') { + c = GETC(); + c1 |= Numlong; + } else + if(c == 'F' || c == 'f') { + c = GETC(); + c1 |= Numflt; + } + *cp = 0; + peekc = c; + yylval.dval = strtod(symb, nil); + if(isInf(yylval.dval, 1) || isInf(yylval.dval, -1)) { + yyerror("overflow in float constant"); + yylval.dval = 0; + } + if(c1 & Numflt) + return LFCONST; + return LDCONST; +} + +/* + * convert a string, s, to vlong in *v + * return conversion overflow. + * required syntax is [0[x]]d* + */ +int +mpatov(char *s, vlong *v) +{ + vlong n, nn; + int c; + + n = 0; + c = *s; + if(c == '0') + goto oct; + while(c = *s++) { + if(c >= '0' && c <= '9') + nn = n*10 + c-'0'; + else + goto bad; + if(n < 0 && nn >= 0) + goto bad; + n = nn; + } + goto out; + +oct: + s++; + c = *s; + if(c == 'x' || c == 'X') + goto hex; + while(c = *s++) { + if(c >= '0' || c <= '7') + nn = n*8 + c-'0'; + else + goto bad; + if(n < 0 && nn >= 0) + goto bad; + n = nn; + } + goto out; + +hex: + s++; + while(c = *s++) { + if(c >= '0' && c <= '9') + c += 0-'0'; + else + if(c >= 'a' && c <= 'f') + c += 10-'a'; + else + if(c >= 'A' && c <= 'F') + c += 10-'A'; + else + goto bad; + nn = n*16 + c; + if(n < 0 && nn >= 0) + goto bad; + n = nn; + } +out: + *v = n; + return 0; + +bad: + *v = ~0; + return 1; +} + +int +getc(void) +{ + int c; + + if(peekc != IGN) { + c = peekc; + peekc = IGN; + } else + c = GETC(); + if(c == '\n') + lineno++; + if(c == EOF) { + yyerror("End of file"); + errorexit(); + } + return c; +} + +int32 +getr(void) +{ + int c, i; + char str[UTFmax+1]; + Rune rune; + + + c = getc(); + if(c < Runeself) + return c; + i = 0; + str[i++] = c; + +loop: + c = getc(); + str[i++] = c; + if(!fullrune(str, i)) + goto loop; + c = chartorune(&rune, str); + if(rune == Runeerror && c == 1) { + nearln = lineno; + diag(Z, "illegal rune in string"); + for(c=0; c0; i--) { + c = getc(); + if(c >= '0' && c <= '9') { + l = l*16 + c-'0'; + continue; + } + if(c >= 'a' && c <= 'f') { + l = l*16 + c-'a' + 10; + continue; + } + if(c >= 'A' && c <= 'F') { + l = l*16 + c-'A' + 10; + continue; + } + unget(c); + break; + } + if(escflg) + l |= ESC; + return l; + } + if(c >= '0' && c <= '7') { + /* + * note this is not ansi, + * supposed to only accept 3 oct + */ + i = 2; + if(longflg) + i = 5; + l = c - '0'; + for(; i>0; i--) { + c = getc(); + if(c >= '0' && c <= '7') { + l = l*8 + c-'0'; + continue; + } + unget(c); + } + if(escflg) + l |= ESC; + return l; + } + switch(c) + { + case '\n': goto loop; + case 'n': return '\n'; + case 't': return '\t'; + case 'b': return '\b'; + case 'r': return '\r'; + case 'f': return '\f'; + case 'a': return '\a'; + case 'v': return '\v'; + } + return c; +} + +struct +{ + char *name; + ushort lexical; + ushort type; +} itab[] = +{ + "auto", LAUTO, 0, + "break", LBREAK, 0, + "case", LCASE, 0, + "char", LCHAR, TCHAR, + "const", LCONSTNT, 0, + "continue", LCONTINUE, 0, + "default", LDEFAULT, 0, + "do", LDO, 0, + "double", LDOUBLE, TDOUBLE, + "else", LELSE, 0, + "enum", LENUM, 0, + "extern", LEXTERN, 0, + "float", LFLOAT, TFLOAT, + "for", LFOR, 0, + "goto", LGOTO, 0, + "if", LIF, 0, + "inline", LINLINE, 0, + "int", LINT, TINT, + "long", LLONG, TLONG, + "register", LREGISTER, 0, + "restrict", LRESTRICT, 0, + "return", LRETURN, 0, + "SET", LSET, 0, + "short", LSHORT, TSHORT, + "signed", LSIGNED, 0, + "signof", LSIGNOF, 0, + "sizeof", LSIZEOF, 0, + "static", LSTATIC, 0, + "struct", LSTRUCT, 0, + "switch", LSWITCH, 0, + "typedef", LTYPEDEF, 0, + "typestr", LTYPESTR, 0, + "union", LUNION, 0, + "unsigned", LUNSIGNED, 0, + "USED", LUSED, 0, + "void", LVOID, TVOID, + "volatile", LVOLATILE, 0, + "while", LWHILE, 0, + 0 +}; + +void +cinit(void) +{ + Sym *s; + int i; + Type *t; + + nerrors = 0; + lineno = 1; + iostack = I; + iofree = I; + peekc = IGN; + nhunk = 0; + + types[TXXX] = T; + types[TCHAR] = typ(TCHAR, T); + types[TUCHAR] = typ(TUCHAR, T); + types[TSHORT] = typ(TSHORT, T); + types[TUSHORT] = typ(TUSHORT, T); + types[TINT] = typ(TINT, T); + types[TUINT] = typ(TUINT, T); + types[TLONG] = typ(TLONG, T); + types[TULONG] = typ(TULONG, T); + types[TVLONG] = typ(TVLONG, T); + types[TUVLONG] = typ(TUVLONG, T); + types[TFLOAT] = typ(TFLOAT, T); + types[TDOUBLE] = typ(TDOUBLE, T); + types[TVOID] = typ(TVOID, T); + types[TENUM] = typ(TENUM, T); + types[TFUNC] = typ(TFUNC, types[TINT]); + types[TIND] = typ(TIND, types[TVOID]); + + for(i=0; ilexical = itab[i].lexical; + if(itab[i].type != 0) + s->type = types[itab[i].type]; + } + blockno = 0; + autobn = 0; + autoffset = 0; + + t = typ(TARRAY, types[TCHAR]); + t->width = 0; + symstring = slookup(".string"); + symstring->class = CSTATIC; + symstring->type = t; + + t = typ(TARRAY, types[TCHAR]); + t->width = 0; + + nodproto = new(OPROTO, Z, Z); + dclstack = D; + + pathname = allocn(pathname, 0, 100); + if(getwd(pathname, 99) == 0) { + pathname = allocn(pathname, 100, 900); + if(getwd(pathname, 999) == 0) + strcpy(pathname, "/???"); + } + + fmtinstall('O', Oconv); + fmtinstall('T', Tconv); + fmtinstall('F', FNconv); + fmtinstall('L', Lconv); + fmtinstall('Q', Qconv); + fmtinstall('|', VBconv); + fmtinstall('U', Uconv); +} + +int +filbuf(void) +{ + Io *i; + +loop: + i = iostack; + if(i == I) + return EOF; + if(i->f < 0) + goto pop; + fi.c = read(i->f, i->b, BUFSIZ) - 1; + if(fi.c < 0) { + close(i->f); + linehist(0, 0); + goto pop; + } + fi.p = i->b + 1; + return i->b[0] & 0xff; + +pop: + iostack = i->link; + i->link = iofree; + iofree = i; + i = iostack; + if(i == I) + return EOF; + fi.p = i->p; + fi.c = i->c; + if(--fi.c < 0) + goto loop; + return *fi.p++ & 0xff; +} + +int +Oconv(Fmt *fp) +{ + int a; + + a = va_arg(fp->args, int); + if(a < OXXX || a > OEND) + return fmtprint(fp, "***badO %d***", a); + + return fmtstrcpy(fp, onames[a]); +} + +int +Lconv(Fmt *fp) +{ + char str[STRINGSZ], s[STRINGSZ]; + Hist *h; + struct + { + Hist* incl; /* start of this include file */ + int32 idel; /* delta line number to apply to include */ + Hist* line; /* start of this #line directive */ + int32 ldel; /* delta line number to apply to #line */ + } a[HISTSZ]; + int32 l, d; + int i, n; + + l = va_arg(fp->args, int32); + n = 0; + for(h = hist; h != H; h = h->link) { + if(l < h->line) + break; + if(h->name) { + if(h->offset != 0) { /* #line directive, not #pragma */ + if(n > 0 && n < HISTSZ && h->offset >= 0) { + a[n-1].line = h; + a[n-1].ldel = h->line - h->offset + 1; + } + } else { + if(n < HISTSZ) { /* beginning of file */ + a[n].incl = h; + a[n].idel = h->line; + a[n].line = 0; + } + n++; + } + continue; + } + n--; + if(n > 0 && n < HISTSZ) { + d = h->line - a[n].incl->line; + a[n-1].ldel += d; + a[n-1].idel += d; + } + } + if(n > HISTSZ) + n = HISTSZ; + str[0] = 0; + for(i=n-1; i>=0; i--) { + if(i != n-1) { + if(fp->flags & ~(FmtWidth|FmtPrec)) /* BUG ROB - was f3 */ + break; + strcat(str, " "); + } + if(a[i].line) + snprint(s, STRINGSZ, "%s:%d[%s:%d]", + a[i].line->name, l-a[i].ldel+1, + a[i].incl->name, l-a[i].idel+1); + else + snprint(s, STRINGSZ, "%s:%d", + a[i].incl->name, l-a[i].idel+1); + if(strlen(s)+strlen(str) >= STRINGSZ-10) + break; + strcat(str, s); + l = a[i].incl->line - 1; /* now print out start of this file */ + } + if(n == 0) + strcat(str, ""); + return fmtstrcpy(fp, str); +} + +int +Tconv(Fmt *fp) +{ + char str[STRINGSZ+20], s[STRINGSZ+20]; + Type *t, *t1; + int et; + int32 n; + + str[0] = 0; + for(t = va_arg(fp->args, Type*); t != T; t = t->link) { + et = t->etype; + if(str[0]) + strcat(str, " "); + if(t->garb&~GINCOMPLETE) { + sprint(s, "%s ", gnames[t->garb&~GINCOMPLETE]); + if(strlen(str) + strlen(s) < STRINGSZ) + strcat(str, s); + } + sprint(s, "%s", tnames[et]); + if(strlen(str) + strlen(s) < STRINGSZ) + strcat(str, s); + if(et == TFUNC && (t1 = t->down)) { + sprint(s, "(%T", t1); + if(strlen(str) + strlen(s) < STRINGSZ) + strcat(str, s); + while(t1 = t1->down) { + sprint(s, ", %T", t1); + if(strlen(str) + strlen(s) < STRINGSZ) + strcat(str, s); + } + if(strlen(str) + strlen(s) < STRINGSZ) + strcat(str, ")"); + } + if(et == TARRAY) { + n = t->width; + if(t->link && t->link->width) + n /= t->link->width; + sprint(s, "[%d]", n); + if(strlen(str) + strlen(s) < STRINGSZ) + strcat(str, s); + } + if(t->nbits) { + sprint(s, " %d:%d", t->shift, t->nbits); + if(strlen(str) + strlen(s) < STRINGSZ) + strcat(str, s); + } + if(typesu[et]) { + if(t->tag) { + strcat(str, " "); + if(strlen(str) + strlen(t->tag->name) < STRINGSZ) + strcat(str, t->tag->name); + } else + strcat(str, " {}"); + break; + } + } + return fmtstrcpy(fp, str); +} + +int +FNconv(Fmt *fp) +{ + char *str; + Node *n; + + n = va_arg(fp->args, Node*); + str = ""; + if(n != Z && (n->op == ONAME || n->op == ODOT || n->op == OELEM)) + str = n->sym->name; + return fmtstrcpy(fp, str); +} + +int +Qconv(Fmt *fp) +{ + char str[STRINGSZ+20], *s; + int32 b; + int i; + + str[0] = 0; + for(b = va_arg(fp->args, int32); b;) { + i = bitno(b); + if(str[0]) + strcat(str, " "); + s = qnames[i]; + if(strlen(str) + strlen(s) >= STRINGSZ) + break; + strcat(str, s); + b &= ~(1L << i); + } + return fmtstrcpy(fp, str); +} + +int +VBconv(Fmt *fp) +{ + char str[STRINGSZ]; + int i, n, t, pc; + + n = va_arg(fp->args, int); + pc = 0; /* BUG: was printcol */ + i = 0; + while(pc < n) { + t = (pc+4) & ~3; + if(t <= n) { + str[i++] = '\t'; + pc = t; + continue; + } + str[i++] = ' '; + pc++; + } + str[i] = 0; + + return fmtstrcpy(fp, str); +} + +void +setinclude(char *p) +{ + int i; + + if(*p != 0) { + for(i=1; i < ninclude; i++) + if(strcmp(p, include[i]) == 0) + return; + + if(ninclude%8 == 0) + include = allocn(include, ninclude*sizeof(char *), + 8*sizeof(char *)); + include[ninclude++] = p; + } +} + +void* +alloc(int32 n) +{ + void *p; + + p = malloc(n); + if(p == nil) { + print("alloc out of mem\n"); + exits("alloc: out of mem"); + } + memset(p, 0, n); + return p; +} + +void* +allocn(void *p, int32 n, int32 d) +{ + if(p == nil) + return alloc(n+d); + p = realloc(p, n+d); + if(p == nil) { + print("allocn out of mem\n"); + exits("allocn: out of mem"); + } + if(d > 0) + memset((char*)p+n, 0, d); + return p; +} + +void +ensuresymb(int32 n) +{ + if(symb == nil) { + symb = alloc(NSYMB+1); + nsymb = NSYMB; + } + + if(n > nsymb) { + symb = allocn(symb, nsymb, n+1-nsymb); + nsymb = n; + } +} diff --git a/src/cmd/cc/lexbody b/src/cmd/cc/lexbody new file mode 100644 index 000000000..f4cc19c2e --- /dev/null +++ b/src/cmd/cc/lexbody @@ -0,0 +1,769 @@ +// Inferno utils/cc/lexbody +// http://code.google.com/p/inferno-os/source/browse/utils/cc/lexbody +// +// Copyright © 1994-1999 Lucent Technologies Inc. All rights reserved. +// Portions Copyright © 1995-1997 C H Forsyth (forsyth@terzarima.net) +// Portions Copyright © 1997-1999 Vita Nuova Limited +// Portions Copyright © 2000-2007 Vita Nuova Holdings Limited (www.vitanuova.com) +// Portions Copyright © 2004,2006 Bruce Ellis +// Portions Copyright © 2005-2007 C H Forsyth (forsyth@terzarima.net) +// Revisions Copyright © 2000-2007 Lucent Technologies Inc. and others +// Portions Copyright © 2009 The Go Authors. All rights reserved. +// +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to deal +// in the Software without restriction, including without limitation the rights +// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +// copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +// THE SOFTWARE. + +/* + * common code for all the assemblers + */ + +void +pragpack(void) +{ + while(getnsc() != '\n') + ; +} + +void +pragvararg(void) +{ + while(getnsc() != '\n') + ; +} + +void +pragdynimport(void) +{ + while(getnsc() != '\n') + ; +} + +void +pragdynexport(void) +{ + while(getnsc() != '\n') + ; +} + +void +pragfpround(void) +{ + while(getnsc() != '\n') + ; +} + +void +pragtextflag(void) +{ + while(getnsc() != '\n') + ; +} + +void +pragprofile(void) +{ + while(getnsc() != '\n') + ; +} + +void +pragincomplete(void) +{ + while(getnsc() != '\n') + ; +} + +void* +alloc(int32 n) +{ + void *p; + + p = malloc(n); + if(p == nil) { + print("alloc out of mem\n"); + exits("alloc: out of mem"); + } + memset(p, 0, n); + return p; +} + +void* +allocn(void *p, int32 n, int32 d) +{ + if(p == nil) + return alloc(n+d); + p = realloc(p, n+d); + if(p == nil) { + print("allocn out of mem\n"); + exits("allocn: out of mem"); + } + if(d > 0) + memset((char*)p+n, 0, d); + return p; +} + +void +ensuresymb(int32 n) +{ + if(symb == nil) { + symb = alloc(NSYMB+1); + nsymb = NSYMB; + } + + if(n > nsymb) { + symb = allocn(symb, nsymb, n+1-nsymb); + nsymb = n; + } +} + +void +setinclude(char *p) +{ + int i; + + if(p == 0) + return; + for(i=1; i < ninclude; i++) + if(strcmp(p, include[i]) == 0) + return; + + if(ninclude%8 == 0) + include = allocn(include, ninclude*sizeof(char *), + 8*sizeof(char *)); + include[ninclude++] = p; +} + +void +errorexit(void) +{ + + if(outfile) + remove(outfile); + exits("error"); +} + +void +pushio(void) +{ + Io *i; + + i = iostack; + if(i == I) { + yyerror("botch in pushio"); + errorexit(); + } + i->p = fi.p; + i->c = fi.c; +} + +void +newio(void) +{ + Io *i; + static int pushdepth = 0; + + i = iofree; + if(i == I) { + pushdepth++; + if(pushdepth > 1000) { + yyerror("macro/io expansion too deep"); + errorexit(); + } + i = alloc(sizeof(*i)); + } else + iofree = i->link; + i->c = 0; + i->f = -1; + ionext = i; +} + +void +newfile(char *s, int f) +{ + Io *i; + + i = ionext; + i->link = iostack; + iostack = i; + i->f = f; + if(f < 0) + i->f = open(s, 0); + if(i->f < 0) { + yyerror("%ca: %r: %s", thechar, s); + errorexit(); + } + fi.c = 0; + linehist(s, 0); +} + +Sym* +slookup(char *s) +{ + ensuresymb(strlen(s)); + strcpy(symb, s); + return lookup(); +} + +Sym* +lookup(void) +{ + Sym *s; + int32 h; + char *p; + int c, l; + char *r, *w; + + if((uchar)symb[0] == 0xc2 && (uchar)symb[1] == 0xb7) { + // turn leading · into ""· + h = strlen(symb); + ensuresymb(h+2); + memmove(symb+2, symb, h+1); + symb[0] = '"'; + symb[1] = '"'; + } + + // turn · into . + for(r=w=symb; *r; r++) { + if((uchar)*r == 0xc2 && (uchar)*(r+1) == 0xb7) { + *w++ = '.'; + r++; + }else + *w++ = *r; + } + *w = '\0'; + + h = 0; + for(p=symb; c = *p; p++) + h = h+h+h + c; + l = (p - symb) + 1; + h &= 0xffffff; + h %= NHASH; + c = symb[0]; + for(s = hash[h]; s != S; s = s->link) { + if(s->name[0] != c) + continue; + if(memcmp(s->name, symb, l) == 0) + return s; + } + s = alloc(sizeof(*s)); + s->name = alloc(l); + memmove(s->name, symb, l); + + s->link = hash[h]; + hash[h] = s; + syminit(s); + return s; +} + +int +ISALPHA(int c) +{ + if(isalpha(c)) + return 1; + if(c >= Runeself) + return 1; + return 0; +} + +int32 +yylex(void) +{ + int c, c1; + char *cp; + Sym *s; + + c = peekc; + if(c != IGN) { + peekc = IGN; + goto l1; + } +l0: + c = GETC(); + +l1: + if(c == EOF) { + peekc = EOF; + return -1; + } + if(isspace(c)) { + if(c == '\n') { + lineno++; + return ';'; + } + goto l0; + } + if(ISALPHA(c)) + goto talph; + if(isdigit(c)) + goto tnum; + switch(c) + { + case '\n': + lineno++; + return ';'; + + case '#': + domacro(); + goto l0; + + case '.': + c = GETC(); + if(ISALPHA(c)) { + cp = symb; + *cp++ = '.'; + goto aloop; + } + if(isdigit(c)) { + cp = symb; + *cp++ = '.'; + goto casedot; + } + peekc = c; + return '.'; + + talph: + case '_': + case '@': + cp = symb; + + aloop: + *cp++ = c; + c = GETC(); + if(ISALPHA(c) || isdigit(c) || c == '_' || c == '$') + goto aloop; + *cp = 0; + peekc = c; + s = lookup(); + if(s->macro) { + newio(); + cp = ionext->b; + macexpand(s, cp); + pushio(); + ionext->link = iostack; + iostack = ionext; + fi.p = cp; + fi.c = strlen(cp); + if(peekc != IGN) { + cp[fi.c++] = peekc; + cp[fi.c] = 0; + peekc = IGN; + } + goto l0; + } + if(s->type == 0) + s->type = LNAME; + if(s->type == LNAME || + s->type == LVAR || + s->type == LLAB) { + yylval.sym = s; + return s->type; + } + yylval.lval = s->value; + return s->type; + + tnum: + cp = symb; + if(c != '0') + goto dc; + *cp++ = c; + c = GETC(); + c1 = 3; + if(c == 'x' || c == 'X') { + c1 = 4; + c = GETC(); + } else + if(c < '0' || c > '7') + goto dc; + yylval.lval = 0; + for(;;) { + if(c >= '0' && c <= '9') { + if(c > '7' && c1 == 3) + break; + yylval.lval <<= c1; + yylval.lval += c - '0'; + c = GETC(); + continue; + } + if(c1 == 3) + break; + if(c >= 'A' && c <= 'F') + c += 'a' - 'A'; + if(c >= 'a' && c <= 'f') { + yylval.lval <<= c1; + yylval.lval += c - 'a' + 10; + c = GETC(); + continue; + } + break; + } + goto ncu; + + dc: + for(;;) { + if(!isdigit(c)) + break; + *cp++ = c; + c = GETC(); + } + if(c == '.') + goto casedot; + if(c == 'e' || c == 'E') + goto casee; + *cp = 0; + if(sizeof(yylval.lval) == sizeof(vlong)) + yylval.lval = strtoll(symb, nil, 10); + else + yylval.lval = strtol(symb, nil, 10); + + ncu: + while(c == 'U' || c == 'u' || c == 'l' || c == 'L') + c = GETC(); + peekc = c; + return LCONST; + + casedot: + for(;;) { + *cp++ = c; + c = GETC(); + if(!isdigit(c)) + break; + } + if(c == 'e' || c == 'E') + goto casee; + goto caseout; + + casee: + *cp++ = 'e'; + c = GETC(); + if(c == '+' || c == '-') { + *cp++ = c; + c = GETC(); + } + while(isdigit(c)) { + *cp++ = c; + c = GETC(); + } + + caseout: + *cp = 0; + peekc = c; + if(FPCHIP) { + yylval.dval = atof(symb); + return LFCONST; + } + yyerror("assembler cannot interpret fp constants"); + yylval.lval = 1L; + return LCONST; + + case '"': + memcpy(yylval.sval, nullgen.sval, sizeof(yylval.sval)); + cp = yylval.sval; + c1 = 0; + for(;;) { + c = escchar('"'); + if(c == EOF) + break; + if(c1 < sizeof(yylval.sval)) + *cp++ = c; + c1++; + } + if(c1 > sizeof(yylval.sval)) + yyerror("string constant too long"); + return LSCONST; + + case '\'': + c = escchar('\''); + if(c == EOF) + c = '\''; + if(escchar('\'') != EOF) + yyerror("missing '"); + yylval.lval = c; + return LCONST; + + case '/': + c1 = GETC(); + if(c1 == '/') { + for(;;) { + c = GETC(); + if(c == '\n') + goto l1; + if(c == EOF) { + yyerror("eof in comment"); + errorexit(); + } + } + } + if(c1 == '*') { + for(;;) { + c = GETC(); + while(c == '*') { + c = GETC(); + if(c == '/') + goto l0; + } + if(c == EOF) { + yyerror("eof in comment"); + errorexit(); + } + if(c == '\n') + lineno++; + } + } + break; + + default: + return c; + } + peekc = c1; + return c; +} + +int +getc(void) +{ + int c; + + c = peekc; + if(c != IGN) { + peekc = IGN; + return c; + } + c = GETC(); + if(c == '\n') + lineno++; + if(c == EOF) { + yyerror("End of file"); + errorexit(); + } + return c; +} + +int +getnsc(void) +{ + int c; + + for(;;) { + c = getc(); + if(!isspace(c) || c == '\n') + return c; + } +} + +void +unget(int c) +{ + + peekc = c; + if(c == '\n') + lineno--; +} + +int +escchar(int e) +{ + int c, l; + +loop: + c = getc(); + if(c == '\n') { + yyerror("newline in string"); + return EOF; + } + if(c != '\\') { + if(c == e) + return EOF; + return c; + } + c = getc(); + if(c >= '0' && c <= '7') { + l = c - '0'; + c = getc(); + if(c >= '0' && c <= '7') { + l = l*8 + c-'0'; + c = getc(); + if(c >= '0' && c <= '7') { + l = l*8 + c-'0'; + return l; + } + } + peekc = c; + return l; + } + switch(c) + { + case '\n': goto loop; + case 'n': return '\n'; + case 't': return '\t'; + case 'b': return '\b'; + case 'r': return '\r'; + case 'f': return '\f'; + case 'a': return 0x07; + case 'v': return 0x0b; + case 'z': return 0x00; + } + return c; +} + +void +pinit(char *f) +{ + int i; + Sym *s; + + lineno = 1; + newio(); + newfile(f, -1); + pc = 0; + peekc = IGN; + sym = 1; + for(i=0; ilink) + s->macro = 0; +} + +int +filbuf(void) +{ + Io *i; + +loop: + i = iostack; + if(i == I) + return EOF; + if(i->f < 0) + goto pop; + fi.c = read(i->f, i->b, BUFSIZ) - 1; + if(fi.c < 0) { + close(i->f); + linehist(0, 0); + goto pop; + } + fi.p = i->b + 1; + return i->b[0]; + +pop: + iostack = i->link; + i->link = iofree; + iofree = i; + i = iostack; + if(i == I) + return EOF; + fi.p = i->p; + fi.c = i->c; + if(--fi.c < 0) + goto loop; + return *fi.p++; +} + +void +yyerror(char *a, ...) +{ + char buf[200]; + va_list arg; + + /* + * hack to intercept message from yaccpar + */ + if(strcmp(a, "syntax error") == 0) { + yyerror("syntax error, last name: %s", symb); + return; + } + prfile(lineno); + va_start(arg, a); + vseprint(buf, buf+sizeof(buf), a, arg); + va_end(arg); + print("%s\n", buf); + nerrors++; + if(nerrors > 10) { + print("too many errors\n"); + errorexit(); + } +} + +void +prfile(int32 l) +{ + int i, n; + Hist a[HISTSZ], *h; + int32 d; + + n = 0; + for(h = hist; h != H; h = h->link) { + if(l < h->line) + break; + if(h->name) { + if(h->offset == 0) { + if(n >= 0 && n < HISTSZ) + a[n] = *h; + n++; + continue; + } + if(n > 0 && n < HISTSZ) + if(a[n-1].offset == 0) { + a[n] = *h; + n++; + } else + a[n-1] = *h; + continue; + } + n--; + if(n >= 0 && n < HISTSZ) { + d = h->line - a[n].line; + for(i=0; i HISTSZ) + n = HISTSZ; + for(i=0; ih |= 0x80000000L; + return; + } + if(native == 0) { + ieee->l = 0; + ieee->h = 0; + return; + } + fr = frexp(native, &exp); + f = 2097152L; /* shouldnt use fp constants here */ + fr = modf(fr*f, &ho); + ieee->h = ho; + ieee->h &= 0xfffffL; + ieee->h |= (exp+1022L) << 20; + f = 65536L; + fr = modf(fr*f, &ho); + ieee->l = ho; + ieee->l <<= 16; + ieee->l |= (int32)(fr*f); +} diff --git a/src/cmd/cc/mac.c b/src/cmd/cc/mac.c new file mode 100644 index 000000000..b969662ae --- /dev/null +++ b/src/cmd/cc/mac.c @@ -0,0 +1,34 @@ +// Inferno utils/cc/mac.c +// http://code.google.com/p/inferno-os/source/browse/utils/cc/mac.c +// +// Copyright © 1994-1999 Lucent Technologies Inc. All rights reserved. +// Portions Copyright © 1995-1997 C H Forsyth (forsyth@terzarima.net) +// Portions Copyright © 1997-1999 Vita Nuova Limited +// Portions Copyright © 2000-2007 Vita Nuova Holdings Limited (www.vitanuova.com) +// Portions Copyright © 2004,2006 Bruce Ellis +// Portions Copyright © 2005-2007 C H Forsyth (forsyth@terzarima.net) +// Revisions Copyright © 2000-2007 Lucent Technologies Inc. and others +// Portions Copyright © 2009 The Go Authors. All rights reserved. +// +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to deal +// in the Software without restriction, including without limitation the rights +// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +// copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +// THE SOFTWARE. + +#include +#include "cc.h" + +#include "macbody" diff --git a/src/cmd/cc/macbody b/src/cmd/cc/macbody new file mode 100644 index 000000000..ed66361f1 --- /dev/null +++ b/src/cmd/cc/macbody @@ -0,0 +1,852 @@ +// Inferno utils/cc/macbody +// http://code.google.com/p/inferno-os/source/browse/utils/cc/macbody +// +// Copyright © 1994-1999 Lucent Technologies Inc. All rights reserved. +// Portions Copyright © 1995-1997 C H Forsyth (forsyth@terzarima.net) +// Portions Copyright © 1997-1999 Vita Nuova Limited +// Portions Copyright © 2000-2007 Vita Nuova Holdings Limited (www.vitanuova.com) +// Portions Copyright © 2004,2006 Bruce Ellis +// Portions Copyright © 2005-2007 C H Forsyth (forsyth@terzarima.net) +// Revisions Copyright © 2000-2007 Lucent Technologies Inc. and others +// Portions Copyright © 2009 The Go Authors. All rights reserved. +// +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to deal +// in the Software without restriction, including without limitation the rights +// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +// copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +// THE SOFTWARE. + +#define VARMAC 0x80 + +int32 +getnsn(void) +{ + int32 n; + int c; + + c = getnsc(); + if(c < '0' || c > '9') + return -1; + n = 0; + while(c >= '0' && c <= '9') { + n = n*10 + c-'0'; + c = getc(); + } + unget(c); + return n; +} + +Sym* +getsym(void) +{ + int c; + char *cp; + + c = getnsc(); + if(!isalpha(c) && c != '_' && c < 0x80) { + unget(c); + return S; + } + for(cp = symb;;) { + if(cp <= symb+NSYMB-4) + *cp++ = c; + c = getc(); + if(isalnum(c) || c == '_' || c >= 0x80) + continue; + unget(c); + break; + } + *cp = 0; + if(cp > symb+NSYMB-4) + yyerror("symbol too large: %s", symb); + return lookup(); +} + +Sym* +getsymdots(int *dots) +{ + int c; + Sym *s; + + s = getsym(); + if(s != S) + return s; + + c = getnsc(); + if(c != '.'){ + unget(c); + return S; + } + if(getc() != '.' || getc() != '.') + yyerror("bad dots in macro"); + *dots = 1; + return slookup("__VA_ARGS__"); +} + +int +getcom(void) +{ + int c; + + for(;;) { + c = getnsc(); + if(c != '/') + break; + c = getc(); + if(c == '/') { + while(c != '\n') + c = getc(); + break; + } + if(c != '*') + break; + c = getc(); + for(;;) { + if(c == '*') { + c = getc(); + if(c != '/') + continue; + c = getc(); + break; + } + if(c == '\n') { + yyerror("comment across newline"); + break; + } + c = getc(); + } + if(c == '\n') + break; + } + return c; +} + +void +dodefine(char *cp) +{ + Sym *s; + char *p; + int32 l; + + ensuresymb(strlen(cp)); + strcpy(symb, cp); + p = strchr(symb, '='); + if(p) { + *p++ = 0; + s = lookup(); + l = strlen(p) + 2; /* +1 null, +1 nargs */ + s->macro = alloc(l); + strcpy(s->macro+1, p); + } else { + s = lookup(); + s->macro = "\0001"; /* \000 is nargs */ + } + if(debug['m']) + print("#define (-D) %s %s\n", s->name, s->macro+1); +} + +struct +{ + char *macname; + void (*macf)(void); +} mactab[] = +{ + "ifdef", 0, /* macif(0) */ + "ifndef", 0, /* macif(1) */ + "else", 0, /* macif(2) */ + + "line", maclin, + "define", macdef, + "include", macinc, + "undef", macund, + + "pragma", macprag, + "endif", macend, + 0 +}; + +void +domacro(void) +{ + int i; + Sym *s; + + s = getsym(); + if(s == S) + s = slookup("endif"); + for(i=0; mactab[i].macname; i++) + if(strcmp(s->name, mactab[i].macname) == 0) { + if(mactab[i].macf) + (*mactab[i].macf)(); + else + macif(i); + return; + } + yyerror("unknown #: %s", s->name); + macend(); +} + +void +macund(void) +{ + Sym *s; + + s = getsym(); + macend(); + if(s == S) { + yyerror("syntax in #undef"); + return; + } + s->macro = 0; +} + +#define NARG 25 +void +macdef(void) +{ + Sym *s, *a; + char *args[NARG], *np, *base; + int n, i, c, len, dots; + int ischr; + + s = getsym(); + if(s == S) + goto bad; + if(s->macro) + yyerror("macro redefined: %s", s->name); + c = getc(); + n = -1; + dots = 0; + if(c == '(') { + n++; + c = getnsc(); + if(c != ')') { + unget(c); + for(;;) { + a = getsymdots(&dots); + if(a == S) + goto bad; + if(n >= NARG) { + yyerror("too many arguments in #define: %s", s->name); + goto bad; + } + args[n++] = a->name; + c = getnsc(); + if(c == ')') + break; + if(c != ',' || dots) + goto bad; + } + } + c = getc(); + } + if(isspace(c)) + if(c != '\n') + c = getnsc(); + base = hunk; + len = 1; + ischr = 0; + for(;;) { + if(isalpha(c) || c == '_') { + np = symb; + *np++ = c; + c = getc(); + while(isalnum(c) || c == '_') { + *np++ = c; + c = getc(); + } + *np = 0; + for(i=0; i= n) { + i = strlen(symb); + base = allocn(base, len, i); + memcpy(base+len, symb, i); + len += i; + continue; + } + base = allocn(base, len, 2); + base[len++] = '#'; + base[len++] = 'a' + i; + continue; + } + if(ischr){ + if(c == '\\'){ + base = allocn(base, len, 1); + base[len++] = c; + c = getc(); + }else if(c == ischr) + ischr = 0; + }else{ + if(c == '"' || c == '\''){ + base = allocn(base, len, 1); + base[len++] = c; + ischr = c; + c = getc(); + continue; + } + if(c == '/') { + c = getc(); + if(c == '/'){ + c = getc(); + for(;;) { + if(c == '\n') + break; + c = getc(); + } + continue; + } + if(c == '*'){ + c = getc(); + for(;;) { + if(c == '*') { + c = getc(); + if(c != '/') + continue; + c = getc(); + break; + } + if(c == '\n') { + yyerror("comment and newline in define: %s", s->name); + break; + } + c = getc(); + } + continue; + } + base = allocn(base, len, 1); + base[len++] = '/'; + continue; + } + } + if(c == '\\') { + c = getc(); + if(c == '\n') { + c = getc(); + continue; + } + else if(c == '\r') { + c = getc(); + if(c == '\n') { + c = getc(); + continue; + } + } + base = allocn(base, len, 1); + base[len++] = '\\'; + continue; + } + if(c == '\n') + break; + if(c == '#') + if(n > 0) { + base = allocn(base, len, 1); + base[len++] = c; + } + base = allocn(base, len, 1); + base[len++] = c; + c = ((--fi.c < 0)? filbuf(): (*fi.p++ & 0xff)); + if(c == '\n') + lineno++; + if(c == -1) { + yyerror("eof in a macro: %s", s->name); + break; + } + } + do { + base = allocn(base, len, 1); + base[len++] = 0; + } while(len & 3); + + *base = n+1; + if(dots) + *base |= VARMAC; + s->macro = base; + if(debug['m']) + print("#define %s %s\n", s->name, s->macro+1); + return; + +bad: + if(s == S) + yyerror("syntax in #define"); + else + yyerror("syntax in #define: %s", s->name); + macend(); +} + +void +macexpand(Sym *s, char *b) +{ + char buf[2000]; + int n, l, c, nargs; + char *arg[NARG], *cp, *ob, *ecp, dots; + + ob = b; + if(*s->macro == 0) { + strcpy(b, s->macro+1); + if(debug['m']) + print("#expand %s %s\n", s->name, ob); + return; + } + + nargs = (char)(*s->macro & ~VARMAC) - 1; + dots = *s->macro & VARMAC; + + c = getnsc(); + if(c != '(') + goto bad; + n = 0; + c = getc(); + if(c != ')') { + unget(c); + l = 0; + cp = buf; + ecp = cp + sizeof(buf)-4; + arg[n++] = cp; + for(;;) { + if(cp >= ecp) + goto toobig; + c = getc(); + if(c == '"') + for(;;) { + if(cp >= ecp) + goto toobig; + *cp++ = c; + c = getc(); + if(c == '\\') { + *cp++ = c; + c = getc(); + continue; + } + if(c == '\n') + goto bad; + if(c == '"') + break; + } + if(c == '\'') + for(;;) { + if(cp >= ecp) + goto toobig; + *cp++ = c; + c = getc(); + if(c == '\\') { + *cp++ = c; + c = getc(); + continue; + } + if(c == '\n') + goto bad; + if(c == '\'') + break; + } + if(c == '/') { + c = getc(); + switch(c) { + case '*': + for(;;) { + c = getc(); + if(c == '*') { + c = getc(); + if(c == '/') + break; + } + } + *cp++ = ' '; + continue; + case '/': + while((c = getc()) != '\n') + ; + break; + default: + unget(c); + c = '/'; + } + } + if(l == 0) { + if(c == ',') { + if(n == nargs && dots) { + *cp++ = ','; + continue; + } + *cp++ = 0; + arg[n++] = cp; + if(n > nargs) + break; + continue; + } + if(c == ')') + break; + } + if(c == '\n') + c = ' '; + *cp++ = c; + if(c == '(') + l++; + if(c == ')') + l--; + } + *cp = 0; + } + if(n != nargs) { + yyerror("argument mismatch expanding: %s", s->name); + *b = 0; + return; + } + cp = s->macro+1; + for(;;) { + c = *cp++; + if(c == '\n') + c = ' '; + if(c != '#') { + *b++ = c; + if(c == 0) + break; + continue; + } + c = *cp++; + if(c == 0) + goto bad; + if(c == '#') { + *b++ = c; + continue; + } + c -= 'a'; + if(c < 0 || c >= n) + continue; + strcpy(b, arg[c]); + b += strlen(arg[c]); + } + *b = 0; + if(debug['m']) + print("#expand %s %s\n", s->name, ob); + return; + +bad: + yyerror("syntax in macro expansion: %s", s->name); + *b = 0; + return; + +toobig: + yyerror("too much text in macro expansion: %s", s->name); + *b = 0; +} + +void +macinc(void) +{ + int c0, c, i, f; + char str[STRINGSZ], *hp; + + c0 = getnsc(); + if(c0 != '"') { + c = c0; + if(c0 != '<') + goto bad; + c0 = '>'; + } + for(hp = str;;) { + c = getc(); + if(c == c0) + break; + if(c == '\n') + goto bad; + *hp++ = c; + } + *hp = 0; + + c = getcom(); + if(c != '\n') + goto bad; + + f = -1; + for(i=0; i') + continue; + ensuresymb(strlen(include[i])+strlen(str)+2); + strcpy(symb, include[i]); + strcat(symb, "/"); + if(strcmp(symb, "./") == 0) + symb[0] = 0; + strcat(symb, str); + f = open(symb, OREAD); + if(f >= 0) + break; + } + if(f < 0) + strcpy(symb, str); + c = strlen(symb) + 1; + hp = alloc(c); + memcpy(hp, symb, c); + newio(); + pushio(); + newfile(hp, f); + return; + +bad: + unget(c); + yyerror("syntax in #include"); + macend(); +} + +void +maclin(void) +{ + char *cp; + int c; + int32 n; + + n = getnsn(); + c = getc(); + if(n < 0) + goto bad; + + for(;;) { + if(c == ' ' || c == '\t') { + c = getc(); + continue; + } + if(c == '"') + break; + if(c == '\n') { + strcpy(symb, ""); + goto nn; + } + goto bad; + } + cp = symb; + for(;;) { + c = getc(); + if(c == '"') + break; + *cp++ = c; + } + *cp = 0; + c = getcom(); + if(c != '\n') + goto bad; + +nn: + c = strlen(symb) + 1; + cp = alloc(c); + memcpy(cp, symb, c); + linehist(cp, n); + return; + +bad: + unget(c); + yyerror("syntax in #line"); + macend(); +} + +void +macif(int f) +{ + int c, l, bol; + Sym *s; + + if(f == 2) + goto skip; + s = getsym(); + if(s == S) + goto bad; + if(getcom() != '\n') + goto bad; + if((s->macro != 0) ^ f) + return; + +skip: + bol = 1; + l = 0; + for(;;) { + c = getc(); + if(c != '#') { + if(!isspace(c)) + bol = 0; + if(c == '\n') + bol = 1; + continue; + } + if(!bol) + continue; + s = getsym(); + if(s == S) + continue; + if(strcmp(s->name, "endif") == 0) { + if(l) { + l--; + continue; + } + macend(); + return; + } + if(strcmp(s->name, "ifdef") == 0 || strcmp(s->name, "ifndef") == 0) { + l++; + continue; + } + if(l == 0 && f != 2 && strcmp(s->name, "else") == 0) { + macend(); + return; + } + } + +bad: + yyerror("syntax in #if(n)def"); + macend(); +} + +void +macprag(void) +{ + Sym *s; + int c0, c; + char *hp; + Hist *h; + + s = getsym(); + + if(s && strcmp(s->name, "lib") == 0) + goto praglib; + if(s && strcmp(s->name, "pack") == 0) { + pragpack(); + return; + } + if(s && strcmp(s->name, "fpround") == 0) { + pragfpround(); + return; + } + if(s && strcmp(s->name, "textflag") == 0) { + pragtextflag(); + return; + } + if(s && strcmp(s->name, "varargck") == 0) { + pragvararg(); + return; + } + if(s && strcmp(s->name, "incomplete") == 0) { + pragincomplete(); + return; + } + if(s && strcmp(s->name, "dynimport") == 0) { + pragdynimport(); + return; + } + if(s && strcmp(s->name, "dynexport") == 0) { + pragdynexport(); + return; + } + while(getnsc() != '\n') + ; + return; + +praglib: + c0 = getnsc(); + if(c0 != '"') { + c = c0; + if(c0 != '<') + goto bad; + c0 = '>'; + } + for(hp = symb;;) { + c = getc(); + if(c == c0) + break; + if(c == '\n') + goto bad; + *hp++ = c; + } + *hp = 0; + c = getcom(); + if(c != '\n') + goto bad; + + /* + * put pragma-line in as a funny history + */ + c = strlen(symb) + 1; + hp = alloc(c); + memcpy(hp, symb, c); + + h = alloc(sizeof(Hist)); + h->name = hp; + h->line = lineno; + h->offset = -1; + h->link = H; + if(ehist == H) { + hist = h; + ehist = h; + return; + } + ehist->link = h; + ehist = h; + return; + +bad: + unget(c); + yyerror("syntax in #pragma lib"); + macend(); +} + +void +macend(void) +{ + int c; + + for(;;) { + c = getnsc(); + if(c < 0 || c == '\n') + return; + } +} + +void +linehist(char *f, int offset) +{ + Hist *h; + + /* + * overwrite the last #line directive if + * no alloc has happened since the last one + */ + if(newflag == 0 && ehist != H && offset != 0 && ehist->offset != 0) + if(f && ehist->name && strcmp(f, ehist->name) == 0) { + ehist->line = lineno; + ehist->offset = offset; + return; + } + + if(debug['f']) + if(f) { + if(offset) + print("%4d: %s (#line %d)\n", lineno, f, offset); + else + print("%4d: %s\n", lineno, f); + } else + print("%4d: \n", lineno); + newflag = 0; + + h = alloc(sizeof(Hist)); + h->name = f; + h->line = lineno; + h->offset = offset; + h->link = H; + if(ehist == H) { + hist = h; + ehist = h; + return; + } + ehist->link = h; + ehist = h; +} diff --git a/src/cmd/cc/omachcap.c b/src/cmd/cc/omachcap.c new file mode 100644 index 000000000..f8fc1d88b --- /dev/null +++ b/src/cmd/cc/omachcap.c @@ -0,0 +1,40 @@ +// Inferno utils/cc/machcap.c +// http://code.google.com/p/inferno-os/source/browse/utils/cc/machcap.c +// +// Copyright © 1994-1999 Lucent Technologies Inc. All rights reserved. +// Portions Copyright © 1995-1997 C H Forsyth (forsyth@terzarima.net) +// Portions Copyright © 1997-1999 Vita Nuova Limited +// Portions Copyright © 2000-2007 Vita Nuova Holdings Limited (www.vitanuova.com) +// Portions Copyright © 2004,2006 Bruce Ellis +// Portions Copyright © 2005-2007 C H Forsyth (forsyth@terzarima.net) +// Revisions Copyright © 2000-2007 Lucent Technologies Inc. and others +// Portions Copyright © 2009 The Go Authors. All rights reserved. +// +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to deal +// in the Software without restriction, including without limitation the rights +// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +// copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +// THE SOFTWARE. + +#include +#include "cc.h" + +/* default, like old cc */ +int +machcap(Node *n) +{ + USED(n); + return 0; +} diff --git a/src/cmd/cc/pgen.c b/src/cmd/cc/pgen.c new file mode 100644 index 000000000..0e5e8c059 --- /dev/null +++ b/src/cmd/cc/pgen.c @@ -0,0 +1,594 @@ +// Inferno utils/6c/sgen.c +// http://code.google.com/p/inferno-os/source/browse/utils/6c/sgen.c +// +// Copyright © 1994-1999 Lucent Technologies Inc. All rights reserved. +// Portions Copyright © 1995-1997 C H Forsyth (forsyth@terzarima.net) +// Portions Copyright © 1997-1999 Vita Nuova Limited +// Portions Copyright © 2000-2007 Vita Nuova Holdings Limited (www.vitanuova.com) +// Portions Copyright © 2004,2006 Bruce Ellis +// Portions Copyright © 2005-2007 C H Forsyth (forsyth@terzarima.net) +// Revisions Copyright © 2000-2007 Lucent Technologies Inc. and others +// Portions Copyright © 2009 The Go Authors. All rights reserved. +// +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to deal +// in the Software without restriction, including without limitation the rights +// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +// copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +// THE SOFTWARE. + +#include "gc.h" + +vlong +argsize(void) +{ + Type *t; + int32 s; + +//print("t=%T\n", thisfn); + s = align(0, thisfn->link, Aarg0, nil); + for(t=thisfn->down; t!=T; t=t->down) { + switch(t->etype) { + case TVOID: + break; + case TDOT: + yyerror("function takes ... without textflag NOSPLIT"); + s += 64; + break; + default: + s = align(s, t, Aarg1, nil); + s = align(s, t, Aarg2, nil); + break; + } +//print(" %d %T\n", s, t); + } + if(thechar == '6') + s = (s+7) & ~7; + else + s = (s+3) & ~3; + return s; +} + +void +codgen(Node *n, Node *nn) +{ + Prog *sp; + Node *n1, nod, nod1; + + cursafe = 0; + curarg = 0; + maxargsafe = 0; + + /* + * isolate name + */ + for(n1 = nn;; n1 = n1->left) { + if(n1 == Z) { + diag(nn, "cant find function name"); + return; + } + if(n1->op == ONAME) + break; + } + nearln = nn->lineno; + + p = gtext(n1->sym, stkoff); + sp = p; + + /* + * isolate first argument + */ + if(REGARG >= 0) { + if(typesuv[thisfn->link->etype]) { + nod1 = *nodret->left; + nodreg(&nod, &nod1, REGARG); + gmove(&nod, &nod1); + } else + if(firstarg && typechlp[firstargtype->etype]) { + nod1 = *nodret->left; + nod1.sym = firstarg; + nod1.type = firstargtype; + nod1.xoffset = align(0, firstargtype, Aarg1, nil); + nod1.etype = firstargtype->etype; + nodreg(&nod, &nod1, REGARG); + gmove(&nod, &nod1); + } + } + + retok = 0; + + canreach = 1; + warnreach = 1; + gen(n); + if(canreach && thisfn->link->etype != TVOID) + diag(Z, "no return at end of function: %s", n1->sym->name); + noretval(3); + gbranch(ORETURN); + + if(!debug['N'] || debug['R'] || debug['P']) + regopt(sp); + + if(thechar=='6' || thechar=='7') /* [sic] */ + maxargsafe = xround(maxargsafe, 8); + sp->to.offset += maxargsafe; +} + +void +supgen(Node *n) +{ + int owarn; + long spc; + Prog *sp; + + if(n == Z) + return; + suppress++; + owarn = warnreach; + warnreach = 0; + spc = pc; + sp = lastp; + gen(n); + lastp = sp; + pc = spc; + sp->link = nil; + suppress--; + warnreach = owarn; +} + +void +gen(Node *n) +{ + Node *l, nod; + Prog *sp, *spc, *spb; + Case *cn; + long sbc, scc; + int snbreak, sncontin; + int f, o, oldreach; + +loop: + if(n == Z) + return; + nearln = n->lineno; + o = n->op; + if(debug['G']) + if(o != OLIST) + print("%L %O\n", nearln, o); + + if(!canreach) { + switch(o) { + case OLABEL: + case OCASE: + case OLIST: + case OBREAK: + case OFOR: + case OWHILE: + case ODWHILE: + /* all handled specially - see switch body below */ + break; + default: + if(warnreach) { + warn(n, "unreachable code %O", o); + warnreach = 0; + } + } + } + + switch(o) { + + default: + complex(n); + cgen(n, Z); + break; + + case OLIST: + gen(n->left); + + rloop: + n = n->right; + goto loop; + + case ORETURN: + canreach = 0; + warnreach = !suppress; + complex(n); + if(n->type == T) + break; + l = n->left; + if(l == Z) { + noretval(3); + gbranch(ORETURN); + break; + } + if(typecmplx[n->type->etype]) { + sugen(l, nodret, n->type->width); + noretval(3); + gbranch(ORETURN); + break; + } + regret(&nod, n); + cgen(l, &nod); + regfree(&nod); + if(typefd[n->type->etype]) + noretval(1); + else + noretval(2); + gbranch(ORETURN); + break; + + case OLABEL: + canreach = 1; + l = n->left; + if(l) { + l->pc = pc; + if(l->label) + patch(l->label, pc); + } + gbranch(OGOTO); /* prevent self reference in reg */ + patch(p, pc); + goto rloop; + + case OGOTO: + canreach = 0; + warnreach = !suppress; + n = n->left; + if(n == Z) + return; + if(n->complex == 0) { + diag(Z, "label undefined: %s", n->sym->name); + return; + } + if(suppress) + return; + gbranch(OGOTO); + if(n->pc) { + patch(p, n->pc); + return; + } + if(n->label) + patch(n->label, pc-1); + n->label = p; + return; + + case OCASE: + canreach = 1; + l = n->left; + if(cases == C) + diag(n, "case/default outside a switch"); + if(l == Z) { + cas(); + cases->val = 0; + cases->def = 1; + cases->label = pc; + cases->isv = 0; + goto rloop; + } + complex(l); + if(l->type == T) + goto rloop; + if(l->op == OCONST) + if(typeword[l->type->etype] && l->type->etype != TIND) { + cas(); + cases->val = l->vconst; + cases->def = 0; + cases->label = pc; + cases->isv = typev[l->type->etype]; + goto rloop; + } + diag(n, "case expression must be integer constant"); + goto rloop; + + case OSWITCH: + l = n->left; + complex(l); + if(l->type == T) + break; + if(!typeword[l->type->etype] || l->type->etype == TIND) { + diag(n, "switch expression must be integer"); + break; + } + + gbranch(OGOTO); /* entry */ + sp = p; + + cn = cases; + cases = C; + cas(); + + sbc = breakpc; + breakpc = pc; + snbreak = nbreak; + nbreak = 0; + gbranch(OGOTO); + spb = p; + + gen(n->right); /* body */ + if(canreach){ + gbranch(OGOTO); + patch(p, breakpc); + nbreak++; + } + + patch(sp, pc); + regalloc(&nod, l, Z); + /* always signed */ + if(typev[l->type->etype]) + nod.type = types[TVLONG]; + else + nod.type = types[TLONG]; + cgen(l, &nod); + doswit(&nod); + regfree(&nod); + patch(spb, pc); + + cases = cn; + breakpc = sbc; + canreach = nbreak!=0; + if(canreach == 0) + warnreach = !suppress; + nbreak = snbreak; + break; + + case OWHILE: + case ODWHILE: + l = n->left; + gbranch(OGOTO); /* entry */ + sp = p; + + scc = continpc; + continpc = pc; + gbranch(OGOTO); + spc = p; + + sbc = breakpc; + breakpc = pc; + snbreak = nbreak; + nbreak = 0; + gbranch(OGOTO); + spb = p; + + patch(spc, pc); + if(n->op == OWHILE) + patch(sp, pc); + bcomplex(l, Z); /* test */ + patch(p, breakpc); + if(l->op != OCONST || vconst(l) == 0) + nbreak++; + + if(n->op == ODWHILE) + patch(sp, pc); + gen(n->right); /* body */ + gbranch(OGOTO); + patch(p, continpc); + + patch(spb, pc); + continpc = scc; + breakpc = sbc; + canreach = nbreak!=0; + if(canreach == 0) + warnreach = !suppress; + nbreak = snbreak; + break; + + case OFOR: + l = n->left; + if(!canreach && l->right->left && warnreach) { + warn(n, "unreachable code FOR"); + warnreach = 0; + } + gen(l->right->left); /* init */ + gbranch(OGOTO); /* entry */ + sp = p; + + /* + * if there are no incoming labels in the + * body and the top's not reachable, warn + */ + if(!canreach && warnreach && deadheads(n)) { + warn(n, "unreachable code %O", o); + warnreach = 0; + } + + scc = continpc; + continpc = pc; + gbranch(OGOTO); + spc = p; + + sbc = breakpc; + breakpc = pc; + snbreak = nbreak; + nbreak = 0; + sncontin = ncontin; + ncontin = 0; + gbranch(OGOTO); + spb = p; + + patch(spc, pc); + gen(l->right->right); /* inc */ + patch(sp, pc); + if(l->left != Z) { /* test */ + bcomplex(l->left, Z); + patch(p, breakpc); + if(l->left->op != OCONST || vconst(l->left) == 0) + nbreak++; + } + canreach = 1; + gen(n->right); /* body */ + if(canreach){ + gbranch(OGOTO); + patch(p, continpc); + ncontin++; + } + if(!ncontin && l->right->right && warnreach) { + warn(l->right->right, "unreachable FOR inc"); + warnreach = 0; + } + + patch(spb, pc); + continpc = scc; + breakpc = sbc; + canreach = nbreak!=0; + if(canreach == 0) + warnreach = !suppress; + nbreak = snbreak; + ncontin = sncontin; + break; + + case OCONTINUE: + if(continpc < 0) { + diag(n, "continue not in a loop"); + break; + } + gbranch(OGOTO); + patch(p, continpc); + ncontin++; + canreach = 0; + warnreach = !suppress; + break; + + case OBREAK: + if(breakpc < 0) { + diag(n, "break not in a loop"); + break; + } + /* + * Don't complain about unreachable break statements. + * There are breaks hidden in yacc's output and some people + * write return; break; in their switch statements out of habit. + * However, don't confuse the analysis by inserting an + * unreachable reference to breakpc either. + */ + if(!canreach) + break; + gbranch(OGOTO); + patch(p, breakpc); + nbreak++; + canreach = 0; + warnreach = !suppress; + break; + + case OIF: + l = n->left; + if(bcomplex(l, n->right)) { + if(typefd[l->type->etype]) + f = !l->fconst; + else + f = !l->vconst; + if(debug['c']) + print("%L const if %s\n", nearln, f ? "false" : "true"); + if(f) { + canreach = 1; + supgen(n->right->left); + oldreach = canreach; + canreach = 1; + gen(n->right->right); + /* + * treat constant ifs as regular ifs for + * reachability warnings. + */ + if(!canreach && oldreach && debug['w'] < 2) + warnreach = 0; + } + else { + canreach = 1; + gen(n->right->left); + oldreach = canreach; + canreach = 1; + supgen(n->right->right); + /* + * treat constant ifs as regular ifs for + * reachability warnings. + */ + if(!oldreach && canreach && debug['w'] < 2) + warnreach = 0; + canreach = oldreach; + } + } + else { + sp = p; + canreach = 1; + if(n->right->left != Z) + gen(n->right->left); + oldreach = canreach; + canreach = 1; + if(n->right->right != Z) { + gbranch(OGOTO); + patch(sp, pc); + sp = p; + gen(n->right->right); + } + patch(sp, pc); + canreach = canreach || oldreach; + if(canreach == 0) + warnreach = !suppress; + } + break; + + case OSET: + case OUSED: + usedset(n->left, o); + break; + } +} + +void +usedset(Node *n, int o) +{ + if(n->op == OLIST) { + usedset(n->left, o); + usedset(n->right, o); + return; + } + complex(n); + switch(n->op) { + case OADDR: /* volatile */ + gins(ANOP, n, Z); + break; + case ONAME: + if(o == OSET) + gins(ANOP, Z, n); + else + gins(ANOP, n, Z); + break; + } +} + +int +bcomplex(Node *n, Node *c) +{ + Node *b, nod; + + complex(n); + if(n->type != T) + if(tcompat(n, T, n->type, tnot)) + n->type = T; + if(n->type == T) { + gbranch(OGOTO); + return 0; + } + if(c != Z && n->op == OCONST && deadheads(c)) + return 1; + if(typev[n->type->etype] && machcap(Z)) { + b = &nod; + b->op = ONE; + b->left = n; + b->right = new(0, Z, Z); + *b->right = *nodconst(0); + b->right->type = n->type; + b->type = types[TLONG]; + n = b; + } + bool64(n); + boolgen(n, 1, Z); + return 0; +} diff --git a/src/cmd/cc/pswt.c b/src/cmd/cc/pswt.c new file mode 100644 index 000000000..0e402dea7 --- /dev/null +++ b/src/cmd/cc/pswt.c @@ -0,0 +1,168 @@ +// Inferno utils/6c/swt.c +// http://code.google.com/p/inferno-os/source/browse/utils/6c/swt.c +// +// Copyright © 1994-1999 Lucent Technologies Inc. All rights reserved. +// Portions Copyright © 1995-1997 C H Forsyth (forsyth@terzarima.net) +// Portions Copyright © 1997-1999 Vita Nuova Limited +// Portions Copyright © 2000-2007 Vita Nuova Holdings Limited (www.vitanuova.com) +// Portions Copyright © 2004,2006 Bruce Ellis +// Portions Copyright © 2005-2007 C H Forsyth (forsyth@terzarima.net) +// Revisions Copyright © 2000-2007 Lucent Technologies Inc. and others +// Portions Copyright © 2009 The Go Authors. All rights reserved. +// +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to deal +// in the Software without restriction, including without limitation the rights +// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +// copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +// THE SOFTWARE. + +#include "gc.h" + +int +swcmp(const void *a1, const void *a2) +{ + C1 *p1, *p2; + + p1 = (C1*)a1; + p2 = (C1*)a2; + if(p1->val < p2->val) + return -1; + return p1->val > p2->val; +} + +void +doswit(Node *n) +{ + Case *c; + C1 *q, *iq; + int32 def, nc, i, isv; + + def = 0; + nc = 0; + isv = 0; + for(c = cases; c->link != C; c = c->link) { + if(c->def) { + if(def) + diag(n, "more than one default in switch"); + def = c->label; + continue; + } + isv |= c->isv; + nc++; + } + if(isv && !typev[n->type->etype]) + warn(n, "32-bit switch expression with 64-bit case constant"); + + iq = alloc(nc*sizeof(C1)); + q = iq; + for(c = cases; c->link != C; c = c->link) { + if(c->def) + continue; + q->label = c->label; + if(isv) + q->val = c->val; + else + q->val = (int32)c->val; /* cast ensures correct value for 32-bit switch on 64-bit architecture */ + q++; + } + qsort(iq, nc, sizeof(C1), swcmp); + if(debug['W']) + for(i=0; ilink = cases; + cases = c; +} + +int32 +outlstring(ushort *s, int32 n) +{ + char buf[2]; + int c; + int32 r; + + if(suppress) + return nstring; + while(nstring & 1) + outstring("", 1); + r = nstring; + while(n > 0) { + c = *s++; + if(align(0, types[TCHAR], Aarg1, nil)) { + buf[0] = c>>8; + buf[1] = c; + } else { + buf[0] = c; + buf[1] = c>>8; + } + outstring(buf, 2); + n -= sizeof(ushort); + } + return r; +} + +void +nullwarn(Node *l, Node *r) +{ + warn(Z, "result of operation not used"); + if(l != Z) + cgen(l, Z); + if(r != Z) + cgen(r, Z); +} + +void +ieeedtod(Ieee *ieee, double native) +{ + double fr, ho, f; + int exp; + + if(native < 0) { + ieeedtod(ieee, -native); + ieee->h |= 0x80000000L; + return; + } + if(native == 0) { + ieee->l = 0; + ieee->h = 0; + return; + } + fr = frexp(native, &exp); + f = 2097152L; /* shouldnt use fp constants here */ + fr = modf(fr*f, &ho); + ieee->h = ho; + ieee->h &= 0xfffffL; + ieee->h |= (exp+1022L) << 20; + f = 65536L; + fr = modf(fr*f, &ho); + ieee->l = ho; + ieee->l <<= 16; + ieee->l |= (int32)(fr*f); +} diff --git a/src/cmd/cc/scon.c b/src/cmd/cc/scon.c new file mode 100644 index 000000000..193331f77 --- /dev/null +++ b/src/cmd/cc/scon.c @@ -0,0 +1,637 @@ +// Inferno utils/cc/scon.c +// http://code.google.com/p/inferno-os/source/browse/utils/cc/scon.c +// +// Copyright © 1994-1999 Lucent Technologies Inc. All rights reserved. +// Portions Copyright © 1995-1997 C H Forsyth (forsyth@terzarima.net) +// Portions Copyright © 1997-1999 Vita Nuova Limited +// Portions Copyright © 2000-2007 Vita Nuova Holdings Limited (www.vitanuova.com) +// Portions Copyright © 2004,2006 Bruce Ellis +// Portions Copyright © 2005-2007 C H Forsyth (forsyth@terzarima.net) +// Revisions Copyright © 2000-2007 Lucent Technologies Inc. and others +// Portions Copyright © 2009 The Go Authors. All rights reserved. +// +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to deal +// in the Software without restriction, including without limitation the rights +// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +// copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +// THE SOFTWARE. + +#include +#include "cc.h" + +static Node* +acast(Type *t, Node *n) +{ + if(n->type->etype != t->etype || n->op == OBIT) { + n = new1(OCAST, n, Z); + if(nocast(n->left->type, t)) + *n = *n->left; + n->type = t; + } + return n; +} + + +void +evconst(Node *n) +{ + Node *l, *r; + int et, isf; + vlong v; + double d; + + if(n == Z || n->type == T) + return; + + et = n->type->etype; + isf = typefd[et]; + + l = n->left; + r = n->right; + + d = 0; + v = 0; + + switch(n->op) { + default: + return; + + case ONEG: + if(isf) + d = -l->fconst; + else + v = -l->vconst; + break; + + case OCOM: + v = ~l->vconst; + break; + + case OCAST: + if(et == TVOID) + return; + et = l->type->etype; + if(isf) { + if(typefd[et]) + d = l->fconst; + else + d = l->vconst; + } else { + if(typefd[et]) + v = l->fconst; + else + v = convvtox(l->vconst, n->type->etype); + } + break; + + case OCONST: + break; + + case OADD: + if(isf) + d = l->fconst + r->fconst; + else { + v = l->vconst + r->vconst; + } + break; + + case OSUB: + if(isf) + d = l->fconst - r->fconst; + else + v = l->vconst - r->vconst; + break; + + case OMUL: + if(isf) + d = l->fconst * r->fconst; + else { + v = l->vconst * r->vconst; + } + break; + + case OLMUL: + v = (uvlong)l->vconst * (uvlong)r->vconst; + break; + + + case ODIV: + if(vconst(r) == 0) { + warn(n, "divide by zero"); + return; + } + if(isf) + d = l->fconst / r->fconst; + else + v = l->vconst / r->vconst; + break; + + case OLDIV: + if(vconst(r) == 0) { + warn(n, "divide by zero"); + return; + } + v = (uvlong)l->vconst / (uvlong)r->vconst; + break; + + case OMOD: + if(vconst(r) == 0) { + warn(n, "modulo by zero"); + return; + } + v = l->vconst % r->vconst; + break; + + case OLMOD: + if(vconst(r) == 0) { + warn(n, "modulo by zero"); + return; + } + v = (uvlong)l->vconst % (uvlong)r->vconst; + break; + + case OAND: + v = l->vconst & r->vconst; + break; + + case OOR: + v = l->vconst | r->vconst; + break; + + case OXOR: + v = l->vconst ^ r->vconst; + break; + + case OLSHR: + v = (uvlong)l->vconst >> r->vconst; + break; + + case OASHR: + v = l->vconst >> r->vconst; + break; + + case OASHL: + v = l->vconst << r->vconst; + break; + + case OLO: + v = (uvlong)l->vconst < (uvlong)r->vconst; + break; + + case OLT: + if(typefd[l->type->etype]) + v = l->fconst < r->fconst; + else + v = l->vconst < r->vconst; + break; + + case OHI: + v = (uvlong)l->vconst > (uvlong)r->vconst; + break; + + case OGT: + if(typefd[l->type->etype]) + v = l->fconst > r->fconst; + else + v = l->vconst > r->vconst; + break; + + case OLS: + v = (uvlong)l->vconst <= (uvlong)r->vconst; + break; + + case OLE: + if(typefd[l->type->etype]) + v = l->fconst <= r->fconst; + else + v = l->vconst <= r->vconst; + break; + + case OHS: + v = (uvlong)l->vconst >= (uvlong)r->vconst; + break; + + case OGE: + if(typefd[l->type->etype]) + v = l->fconst >= r->fconst; + else + v = l->vconst >= r->vconst; + break; + + case OEQ: + if(typefd[l->type->etype]) + v = l->fconst == r->fconst; + else + v = l->vconst == r->vconst; + break; + + case ONE: + if(typefd[l->type->etype]) + v = l->fconst != r->fconst; + else + v = l->vconst != r->vconst; + break; + + case ONOT: + if(typefd[l->type->etype]) + v = !l->fconst; + else + v = !l->vconst; + break; + + case OANDAND: + if(typefd[l->type->etype]) + v = l->fconst && r->fconst; + else + v = l->vconst && r->vconst; + break; + + case OOROR: + if(typefd[l->type->etype]) + v = l->fconst || r->fconst; + else + v = l->vconst || r->vconst; + break; + } + if(isf) { + n->fconst = d; + } else { + n->vconst = convvtox(v, n->type->etype); + } + n->oldop = n->op; + n->op = OCONST; +} + +void +acom(Node *n) +{ + Type *t; + Node *l, *r; + int i; + + switch(n->op) + { + + case ONAME: + case OCONST: + case OSTRING: + case OINDREG: + case OREGISTER: + return; + + case ONEG: + l = n->left; + if(addo(n) && addo(l)) + break; + acom(l); + return; + + case OADD: + case OSUB: + case OMUL: + l = n->left; + r = n->right; + if(addo(n)) { + if(addo(r)) + break; + if(addo(l)) + break; + } + acom(l); + acom(r); + return; + + default: + l = n->left; + r = n->right; + if(l != Z) + acom(l); + if(r != Z) + acom(r); + return; + } + + /* bust terms out */ + t = n->type; + term[0].mult = 0; + term[0].node = Z; + nterm = 1; + acom1(1, n); + if(debug['m']) + for(i=0; itype = t; +} + +int +acomcmp1(const void *a1, const void *a2) +{ + vlong c1, c2; + Term *t1, *t2; + + t1 = (Term*)a1; + t2 = (Term*)a2; + c1 = t1->mult; + if(c1 < 0) + c1 = -c1; + c2 = t2->mult; + if(c2 < 0) + c2 = -c2; + if(c1 > c2) + return 1; + if(c1 < c2) + return -1; + c1 = 1; + if(t1->mult < 0) + c1 = 0; + c2 = 1; + if(t2->mult < 0) + c2 = 0; + if(c2 -= c1) + return c2; + if(t2 > t1) + return 1; + return -1; +} + +int +acomcmp2(const void *a1, const void *a2) +{ + vlong c1, c2; + Term *t1, *t2; + + t1 = (Term*)a1; + t2 = (Term*)a2; + c1 = t1->mult; + c2 = t2->mult; + if(c1 > c2) + return 1; + if(c1 < c2) + return -1; + if(t2 > t1) + return 1; + return -1; +} + +void +acom2(Node *n, Type *t) +{ + Node *l, *r; + Term trm[NTERM]; + int et, nt, i, j; + vlong c1, c2; + + /* + * copy into automatic + */ + c2 = 0; + nt = nterm; + for(i=0; ioldop = n->op; + n->op = OCONST; + n->vconst = c1; + return; + } + et = t->etype; + + /* + * prepare constant term, + * combine it with an addressing term + */ + if(c1 != 0) { + l = new1(OCONST, Z, Z); + l->type = t; + l->vconst = c1; + trm[0].mult = 1; + for(i=1; iop != OADDR) + continue; + r->type = t; + l = new1(OADD, r, l); + l->type = t; + trm[i].mult = 0; + break; + } + trm[0].node = l; + } + /* + * look for factorable terms + * c1*i + c1*c2*j -> c1*(i + c2*j) + */ + qsort(trm+1, nt-1, sizeof(trm[0]), acomcmp1); + for(i=nt-1; i>=0; i--) { + c1 = trm[i].mult; + if(c1 < 0) + c1 = -c1; + if(c1 <= 1) + continue; + for(j=i+1; jtype->etype != et) + r = acast(t, r); + c2 = trm[j].mult/trm[i].mult; + if(c2 != 1 && c2 != -1) { + r = new1(OMUL, r, new(OCONST, Z, Z)); + r->type = t; + r->right->type = t; + r->right->vconst = c2; + } + l = trm[i].node; + if(l->type->etype != et) + l = acast(t, l); + r = new1(OADD, l, r); + r->type = t; + if(c2 == -1) + r->op = OSUB; + trm[i].node = r; + trm[j].mult = 0; + } + } + if(debug['m']) { + print("\n"); + for(i=0; i=0; i--) { + c1 = trm[i].mult; + if(c1 == 0) + continue; + r = trm[i].node; + if(r->type->etype != et || r->op == OBIT) + r = acast(t, r); + if(c1 != 1 && c1 != -1) { + r = new1(OMUL, r, new(OCONST, Z, Z)); + r->type = t; + r->right->type = t; + if(c1 < 0) { + r->right->vconst = -c1; + c1 = -1; + } else { + r->right->vconst = c1; + c1 = 1; + } + } + if(l == Z) { + l = r; + c2 = c1; + continue; + } + if(c1 < 0) + if(c2 < 0) + l = new1(OADD, l, r); + else + l = new1(OSUB, l, r); + else + if(c2 < 0) { + l = new1(OSUB, r, l); + c2 = 1; + } else + l = new1(OADD, l, r); + l->type = t; + } + if(c2 < 0) { + r = new1(OCONST, 0, 0); + r->vconst = 0; + r->type = t; + l = new1(OSUB, r, l); + l->type = t; + } + *n = *l; +} + +void +acom1(vlong v, Node *n) +{ + Node *l, *r; + + if(v == 0 || nterm >= NTERM) + return; + if(!addo(n)) { + if(n->op == OCONST) + if(!typefd[n->type->etype]) { + term[0].mult += v*n->vconst; + return; + } + term[nterm].mult = v; + term[nterm].node = n; + nterm++; + return; + } + switch(n->op) { + + case OCAST: + acom1(v, n->left); + break; + + case ONEG: + acom1(-v, n->left); + break; + + case OADD: + acom1(v, n->left); + acom1(v, n->right); + break; + + case OSUB: + acom1(v, n->left); + acom1(-v, n->right); + break; + + case OMUL: + l = n->left; + r = n->right; + if(l->op == OCONST) + if(!typefd[n->type->etype]) { + acom1(v*l->vconst, r); + break; + } + if(r->op == OCONST) + if(!typefd[n->type->etype]) { + acom1(v*r->vconst, l); + break; + } + break; + + default: + diag(n, "not addo"); + } +} + +int +addo(Node *n) +{ + + if(n != Z) + if(!typefd[n->type->etype]) + if(!typev[n->type->etype] || ewidth[TVLONG] == ewidth[TIND]) + switch(n->op) { + + case OCAST: + if(nilcast(n->left->type, n->type)) + return 1; + break; + + case ONEG: + case OADD: + case OSUB: + return 1; + + case OMUL: + if(n->left->op == OCONST) + return 1; + if(n->right->op == OCONST) + return 1; + } + return 0; +} diff --git a/src/cmd/cc/sub.c b/src/cmd/cc/sub.c new file mode 100644 index 000000000..e5992e213 --- /dev/null +++ b/src/cmd/cc/sub.c @@ -0,0 +1,2056 @@ +// Inferno utils/cc/sub.c +// http://code.google.com/p/inferno-os/source/browse/utils/cc/sub.c +// +// Copyright © 1994-1999 Lucent Technologies Inc. All rights reserved. +// Portions Copyright © 1995-1997 C H Forsyth (forsyth@terzarima.net) +// Portions Copyright © 1997-1999 Vita Nuova Limited +// Portions Copyright © 2000-2007 Vita Nuova Holdings Limited (www.vitanuova.com) +// Portions Copyright © 2004,2006 Bruce Ellis +// Portions Copyright © 2005-2007 C H Forsyth (forsyth@terzarima.net) +// Revisions Copyright © 2000-2007 Lucent Technologies Inc. and others +// Portions Copyright © 2009 The Go Authors. All rights reserved. +// +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to deal +// in the Software without restriction, including without limitation the rights +// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +// copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +// THE SOFTWARE. + +#include +#include "cc.h" + +Node* +new(int t, Node *l, Node *r) +{ + Node *n; + + n = alloc(sizeof(*n)); + n->op = t; + n->left = l; + n->right = r; + if(l && t != OGOTO) + n->lineno = l->lineno; + else if(r) + n->lineno = r->lineno; + else + n->lineno = lineno; + newflag = 1; + return n; +} + +Node* +new1(int o, Node *l, Node *r) +{ + Node *n; + + n = new(o, l, r); + n->lineno = nearln; + return n; +} + +void +prtree(Node *n, char *s) +{ + + print(" == %s ==\n", s); + prtree1(n, 0, 0); + print("\n"); +} + +void +prtree1(Node *n, int d, int f) +{ + int i; + + if(f) + for(i=0; iop == OLIST) { + prtree1(n->left, d, 0); + prtree1(n->right, d, 1); + return; + } + d++; + print("%O", n->op); + i = 3; + switch(n->op) + { + case ONAME: + print(" \"%F\"", n); + print(" %d", n->xoffset); + i = 0; + break; + + case OINDREG: + print(" %d(R%d)", n->xoffset, n->reg); + i = 0; + break; + + case OREGISTER: + if(n->xoffset) + print(" %d+R%d", n->xoffset, n->reg); + else + print(" R%d", n->reg); + i = 0; + break; + + case OSTRING: + print(" \"%s\"", n->cstring); + i = 0; + break; + + case OLSTRING: + print(" \"%S\"", n->rstring); + i = 0; + break; + + case ODOT: + case OELEM: + print(" \"%F\"", n); + break; + + case OCONST: + if(typefd[n->type->etype]) + print(" \"%.8e\"", n->fconst); + else + print(" \"%lld\"", n->vconst); + i = 0; + break; + } + if(n->addable != 0) + print(" <%d>", n->addable); + if(n->type != T) + print(" %T", n->type); + if(n->complex != 0) + print(" (%d)", n->complex); + print(" %L\n", n->lineno); + if(i & 2) + prtree1(n->left, d, 1); + if(i & 1) + prtree1(n->right, d, 1); +} + +Type* +typ(int et, Type *d) +{ + Type *t; + + t = alloc(sizeof(*t)); + t->etype = et; + t->link = d; + t->down = T; + t->sym = S; + t->width = ewidth[et]; + t->offset = 0; + t->shift = 0; + t->nbits = 0; + t->garb = 0; + return t; +} + +Type* +copytyp(Type *t) +{ + Type *nt; + + nt = typ(TXXX, T); + *nt = *t; + return nt; +} + +Type* +garbt(Type *t, int32 b) +{ + Type *t1; + + if(b & BGARB) { + t1 = copytyp(t); + t1->garb = simpleg(b); + return t1; + } + return t; +} + +int +simpleg(int32 b) +{ + + b &= BGARB; + switch(b) { + case BCONSTNT: + return GCONSTNT; + case BVOLATILE: + return GVOLATILE; + case BVOLATILE|BCONSTNT: + return GCONSTNT|GVOLATILE; + } + return GXXX; +} + +int +simplec(int32 b) +{ + + b &= BCLASS; + switch(b) { + case 0: + case BREGISTER: + return CXXX; + case BAUTO: + case BAUTO|BREGISTER: + return CAUTO; + case BEXTERN: + return CEXTERN; + case BEXTERN|BREGISTER: + return CEXREG; + case BSTATIC: + return CSTATIC; + case BTYPEDEF: + return CTYPEDEF; + case BTYPESTR: + return CTYPESTR; + } + diag(Z, "illegal combination of classes %Q", b); + return CXXX; +} + +Type* +simplet(int32 b) +{ + + b &= ~BCLASS & ~BGARB; + switch(b) { + case BCHAR: + case BCHAR|BSIGNED: + return types[TCHAR]; + + case BCHAR|BUNSIGNED: + return types[TUCHAR]; + + case BSHORT: + case BSHORT|BINT: + case BSHORT|BSIGNED: + case BSHORT|BINT|BSIGNED: + return types[TSHORT]; + + case BUNSIGNED|BSHORT: + case BUNSIGNED|BSHORT|BINT: + return types[TUSHORT]; + + case 0: + case BINT: + case BINT|BSIGNED: + case BSIGNED: + return types[TINT]; + + case BUNSIGNED: + case BUNSIGNED|BINT: + return types[TUINT]; + + case BLONG: + case BLONG|BINT: + case BLONG|BSIGNED: + case BLONG|BINT|BSIGNED: + return types[TLONG]; + + case BUNSIGNED|BLONG: + case BUNSIGNED|BLONG|BINT: + return types[TULONG]; + + case BVLONG|BLONG: + case BVLONG|BLONG|BINT: + case BVLONG|BLONG|BSIGNED: + case BVLONG|BLONG|BINT|BSIGNED: + return types[TVLONG]; + + case BVLONG|BLONG|BUNSIGNED: + case BVLONG|BLONG|BINT|BUNSIGNED: + return types[TUVLONG]; + + case BFLOAT: + return types[TFLOAT]; + + case BDOUBLE: + case BDOUBLE|BLONG: + case BFLOAT|BLONG: + return types[TDOUBLE]; + + case BVOID: + return types[TVOID]; + } + + diag(Z, "illegal combination of types %Q", b); + return types[TINT]; +} + +int +stcompat(Node *n, Type *t1, Type *t2, int32 ttab[]) +{ + int i; + uint32 b; + + i = 0; + if(t2 != T) + i = t2->etype; + b = 1L << i; + i = 0; + if(t1 != T) + i = t1->etype; + if(b & ttab[i]) { + if(ttab == tasign) + if(b == BSTRUCT || b == BUNION) + if(!sametype(t1, t2)) + return 1; + if(n->op != OCAST) + if(b == BIND && i == TIND) + if(!sametype(t1, t2)) + return 1; + return 0; + } + return 1; +} + +int +tcompat(Node *n, Type *t1, Type *t2, int32 ttab[]) +{ + + if(stcompat(n, t1, t2, ttab)) { + if(t1 == T) + diag(n, "incompatible type: \"%T\" for op \"%O\"", + t2, n->op); + else + diag(n, "incompatible types: \"%T\" and \"%T\" for op \"%O\"", + t1, t2, n->op); + return 1; + } + return 0; +} + +void +makedot(Node *n, Type *t, int32 o) +{ + Node *n1, *n2; + + if(t->nbits) { + n1 = new(OXXX, Z, Z); + *n1 = *n; + n->op = OBIT; + n->left = n1; + n->right = Z; + n->type = t; + n->addable = n1->left->addable; + n = n1; + } + n->addable = n->left->addable; + if(n->addable == 0) { + n1 = new1(OCONST, Z, Z); + n1->vconst = o; + n1->type = types[TLONG]; + n->right = n1; + n->type = t; + return; + } + n->left->type = t; + if(o == 0) { + *n = *n->left; + return; + } + n->type = t; + n1 = new1(OCONST, Z, Z); + n1->vconst = o; + t = typ(TIND, t); + t->width = types[TIND]->width; + n1->type = t; + + n2 = new1(OADDR, n->left, Z); + n2->type = t; + + n1 = new1(OADD, n1, n2); + n1->type = t; + + n->op = OIND; + n->left = n1; + n->right = Z; +} + +Type* +dotsearch(Sym *s, Type *t, Node *n, int32 *off) +{ + Type *t1, *xt, *rt; + + xt = T; + + /* + * look it up by name + */ + for(t1 = t; t1 != T; t1 = t1->down) + if(t1->sym == s) { + if(xt != T) + goto ambig; + xt = t1; + } + + /* + * look it up by type + */ + if(s->class == CTYPEDEF || s->class == CTYPESTR) + for(t1 = t; t1 != T; t1 = t1->down) + if(t1->sym == S && typesu[t1->etype]) + if(sametype(s->type, t1)) { + if(xt != T) + goto ambig; + xt = t1; + } + if(xt != T) { + *off = xt->offset; + return xt; + } + + /* + * look it up in unnamed substructures + */ + for(t1 = t; t1 != T; t1 = t1->down) + if(t1->sym == S && typesu[t1->etype]){ + rt = dotsearch(s, t1->link, n, off); + if(rt != T) { + if(xt != T) + goto ambig; + xt = rt; + *off += t1->offset; + } + } + return xt; + +ambig: + diag(n, "ambiguous structure element: %s", s->name); + return xt; +} + +int32 +dotoffset(Type *st, Type *lt, Node *n) +{ + Type *t; + Sym *g; + int32 o, o1; + + o = -1; + /* + * first try matching at the top level + * for matching tag names + */ + g = st->tag; + if(g != S) + for(t=lt->link; t!=T; t=t->down) + if(t->sym == S) + if(g == t->tag) { + if(o >= 0) + goto ambig; + o = t->offset; + } + if(o >= 0) + return o; + + /* + * second try matching at the top level + * for similar types + */ + for(t=lt->link; t!=T; t=t->down) + if(t->sym == S) + if(sametype(st, t)) { + if(o >= 0) + goto ambig; + o = t->offset; + } + if(o >= 0) + return o; + + /* + * last try matching sub-levels + */ + for(t=lt->link; t!=T; t=t->down) + if(t->sym == S) + if(typesu[t->etype]) { + o1 = dotoffset(st, t, n); + if(o1 >= 0) { + if(o >= 0) + goto ambig; + o = o1 + t->offset; + } + } + return o; + +ambig: + diag(n, "ambiguous unnamed structure element"); + return o; +} + +/* + * look into tree for floating point constant expressions + */ +int +allfloat(Node *n, int flag) +{ + + if(n != Z) { + if(n->type->etype != TDOUBLE) + return 1; + switch(n->op) { + case OCONST: + if(flag) + n->type = types[TFLOAT]; + return 1; + case OADD: /* no need to get more exotic than this */ + case OSUB: + case OMUL: + case ODIV: + if(!allfloat(n->right, flag)) + break; + case OCAST: + if(!allfloat(n->left, flag)) + break; + if(flag) + n->type = types[TFLOAT]; + return 1; + } + } + return 0; +} + +void +constas(Node *n, Type *il, Type *ir) +{ + Type *l, *r; + + l = il; + r = ir; + + if(l == T) + return; + if(l->garb & GCONSTNT) { + warn(n, "assignment to a constant type (%T)", il); + return; + } + if(r == T) + return; + for(;;) { + if(l->etype != TIND || r->etype != TIND) + break; + l = l->link; + r = r->link; + if(l == T || r == T) + break; + if(r->garb & GCONSTNT) + if(!(l->garb & GCONSTNT)) { + warn(n, "assignment of a constant pointer type (%T)", ir); + break; + } + } +} + +void +typeext1(Type *st, Node *l) +{ + if(st->etype == TFLOAT && allfloat(l, 0)) + allfloat(l, 1); +} + +void +typeext(Type *st, Node *l) +{ + Type *lt; + Node *n1, *n2; + int32 o; + + lt = l->type; + if(lt == T) + return; + if(st->etype == TIND && vconst(l) == 0) { + l->type = st; + l->vconst = 0; + return; + } + typeext1(st, l); + + /* + * extension of C + * if assign of struct containing unnamed sub-struct + * to type of sub-struct, insert the DOT. + * if assign of *struct containing unnamed substruct + * to type of *sub-struct, insert the add-offset + */ + if(typesu[st->etype] && typesu[lt->etype]) { + o = dotoffset(st, lt, l); + if(o >= 0) { + n1 = new1(OXXX, Z, Z); + *n1 = *l; + l->op = ODOT; + l->left = n1; + l->right = Z; + makedot(l, st, o); + } + return; + } + if(st->etype == TIND && typesu[st->link->etype]) + if(lt->etype == TIND && typesu[lt->link->etype]) { + o = dotoffset(st->link, lt->link, l); + if(o >= 0) { + l->type = st; + if(o == 0) + return; + n1 = new1(OXXX, Z, Z); + *n1 = *l; + n2 = new1(OCONST, Z, Z); + n2->vconst = o; + n2->type = st; + l->op = OADD; + l->left = n1; + l->right = n2; + } + return; + } +} + +/* + * a cast that generates no code + * (same size move) + */ +int +nocast(Type *t1, Type *t2) +{ + int i, b; + + if(t1->nbits) + return 0; + i = 0; + if(t2 != T) + i = t2->etype; + b = 1<etype; + if(b & ncast[i]) + return 1; + return 0; +} + +/* + * a cast that has a noop semantic + * (small to large, convert) + */ +int +nilcast(Type *t1, Type *t2) +{ + int et1, et2; + + if(t1 == T) + return 0; + if(t1->nbits) + return 0; + if(t2 == T) + return 0; + et1 = t1->etype; + et2 = t2->etype; + if(et1 == et2) + return 1; + if(typefd[et1] && typefd[et2]) { + if(ewidth[et1] < ewidth[et2]) + return 1; + return 0; + } + if(typechlp[et1] && typechlp[et2]) { + if(ewidth[et1] < ewidth[et2]) + return 1; + return 0; + } + return 0; +} + +/* + * "the usual arithmetic conversions are performed" + */ +void +arith(Node *n, int f) +{ + Type *t1, *t2; + int i, j, k; + Node *n1; + int32 w; + + t1 = n->left->type; + if(n->right == Z) + t2 = t1; + else + t2 = n->right->type; + i = TXXX; + if(t1 != T) + i = t1->etype; + j = TXXX; + if(t2 != T) + j = t2->etype; + k = tab[i][j]; + if(k == TIND) { + if(i == TIND) + n->type = t1; + else + if(j == TIND) + n->type = t2; + } else { + /* convert up to at least int */ + if(f == 1) + while(k < TINT) + k += 2; + n->type = types[k]; + } + if(n->op == OSUB) + if(i == TIND && j == TIND) { + w = n->right->type->link->width; + if(w < 1 || n->left->type->link == T || n->left->type->link->width < 1) + goto bad; + n->type = types[ewidth[TIND] <= ewidth[TLONG]? TLONG: TVLONG]; + if(0 && ewidth[TIND] > ewidth[TLONG]){ + n1 = new1(OXXX, Z, Z); + *n1 = *n; + n->op = OCAST; + n->left = n1; + n->right = Z; + n->type = types[TLONG]; + } + if(w > 1) { + n1 = new1(OXXX, Z, Z); + *n1 = *n; + n->op = ODIV; + n->left = n1; + n1 = new1(OCONST, Z, Z); + n1->vconst = w; + n1->type = n->type; + n->right = n1; + w = vlog(n1); + if(w >= 0) { + n->op = OASHR; + n1->vconst = w; + } + } + return; + } + if(!sametype(n->type, n->left->type)) { + n->left = new1(OCAST, n->left, Z); + n->left->type = n->type; + if(n->type->etype == TIND) { + w = n->type->link->width; + if(w < 1) { + snap(n->type->link); + w = n->type->link->width; + if(w < 1) + goto bad; + } + if(w > 1) { + n1 = new1(OCONST, Z, Z); + n1->vconst = w; + n1->type = n->type; + n->left = new1(OMUL, n->left, n1); + n->left->type = n->type; + } + } + } + if(n->right != Z) + if(!sametype(n->type, n->right->type)) { + n->right = new1(OCAST, n->right, Z); + n->right->type = n->type; + if(n->type->etype == TIND) { + w = n->type->link->width; + if(w < 1) { + snap(n->type->link); + w = n->type->link->width; + if(w < 1) + goto bad; + } + if(w != 1) { + n1 = new1(OCONST, Z, Z); + n1->vconst = w; + n1->type = n->type; + n->right = new1(OMUL, n->right, n1); + n->right->type = n->type; + } + } + } + return; +bad: + diag(n, "pointer addition not fully declared: %T", n->type->link); +} + +/* + * try to rewrite shift & mask + */ +void +simplifyshift(Node *n) +{ + uint32 c3; + int o, s1, s2, c1, c2; + + if(!typechlp[n->type->etype]) + return; + switch(n->op) { + default: + return; + case OASHL: + s1 = 0; + break; + case OLSHR: + s1 = 1; + break; + case OASHR: + s1 = 2; + break; + } + if(n->right->op != OCONST) + return; + if(n->left->op != OAND) + return; + if(n->left->right->op != OCONST) + return; + switch(n->left->left->op) { + default: + return; + case OASHL: + s2 = 0; + break; + case OLSHR: + s2 = 1; + break; + case OASHR: + s2 = 2; + break; + } + if(n->left->left->right->op != OCONST) + return; + + c1 = n->right->vconst; + c2 = n->left->left->right->vconst; + c3 = n->left->right->vconst; + +/* + if(debug['h']) + print("%.3o %d %d %d #%.ux\n", + (s1<<3)|s2, c1, c2, topbit(c3), c3); +*/ + + o = n->op; + switch((s1<<3)|s2) { + case 000: /* (((e <>= c2; + c1 += c2; + if(c1 >= 32) + break; + goto rewrite1; + + case 002: /* (((e >>s c2) & c3) <= (32-c2)) + break; + case 001: /* (((e >>u c2) & c3) < c2) { + c3 <<= c2; + c1 -= c2; + o = OASHL; + goto rewrite1; + } + c3 <<= c1; + if(c1 == c2) + goto rewrite0; + c1 = c2-c1; + o = OLSHR; + goto rewrite2; + + case 022: /* (((e >>s c2) & c3) >>s c1) */ + if(c2 <= 0) + break; + case 012: /* (((e >>s c2) & c3) >>u c1) */ + if(topbit(c3) >= (32-c2)) + break; + goto s11; + case 021: /* (((e >>u c2) & c3) >>s c1) */ + if(topbit(c3) >= 31 && c2 <= 0) + break; + goto s11; + case 011: /* (((e >>u c2) & c3) >>u c1) */ + s11: + c3 <<= c2; + c1 += c2; + if(c1 >= 32) + break; + o = OLSHR; + goto rewrite1; + + case 020: /* (((e <>s c1) */ + if(topbit(c3) >= 31) + break; + case 010: /* (((e <>u c1) */ + c3 >>= c1; + if(c1 == c2) + goto rewrite0; + if(c1 > c2) { + c1 -= c2; + goto rewrite2; + } + c1 = c2 - c1; + o = OASHL; + goto rewrite2; + } + return; + +rewrite0: /* get rid of both shifts */ +if(debug['<'])prtree(n, "rewrite0"); + *n = *n->left; + n->left = n->left->left; + n->right->vconst = c3; + return; +rewrite1: /* get rid of lower shift */ +if(debug['<'])prtree(n, "rewrite1"); + n->left->left = n->left->left->left; + n->left->right->vconst = c3; + n->right->vconst = c1; + n->op = o; + return; +rewrite2: /* get rid of upper shift */ +if(debug['<'])prtree(n, "rewrite2"); + *n = *n->left; + n->right->vconst = c3; + n->left->right->vconst = c1; + n->left->op = o; +} + +int +side(Node *n) +{ + +loop: + if(n != Z) + switch(n->op) { + case OCAST: + case ONOT: + case OADDR: + case OIND: + n = n->left; + goto loop; + + case OCOND: + if(side(n->left)) + break; + n = n->right; + + case OEQ: + case ONE: + case OLT: + case OGE: + case OGT: + case OLE: + case OADD: + case OSUB: + case OMUL: + case OLMUL: + case ODIV: + case OLDIV: + case OLSHR: + case OASHL: + case OASHR: + case OAND: + case OOR: + case OXOR: + case OMOD: + case OLMOD: + case OANDAND: + case OOROR: + case OCOMMA: + case ODOT: + if(side(n->left)) + break; + n = n->right; + goto loop; + + case OSIGN: + case OSIZE: + case OCONST: + case OSTRING: + case OLSTRING: + case ONAME: + return 0; + } + return 1; +} + +int +vconst(Node *n) +{ + int i; + + if(n == Z) + goto no; + if(n->op != OCONST) + goto no; + if(n->type == T) + goto no; + switch(n->type->etype) + { + case TFLOAT: + case TDOUBLE: + i = 100; + if(n->fconst > i || n->fconst < -i) + goto no; + i = n->fconst; + if(i != n->fconst) + goto no; + return i; + + case TVLONG: + case TUVLONG: + i = n->vconst; + if(i != n->vconst) + goto no; + return i; + + case TCHAR: + case TUCHAR: + case TSHORT: + case TUSHORT: + case TINT: + case TUINT: + case TLONG: + case TULONG: + case TIND: + i = n->vconst; + if(i != n->vconst) + goto no; + return i; + } +no: + return -159; /* first uninteresting constant */ +} + +/* + * return log(n) if n is a power of 2 constant + */ +int +xlog2(uvlong v) +{ + int s, i; + uvlong m; + + s = 0; + m = MASK(8*sizeof(uvlong)); + for(i=32; i; i>>=1) { + m >>= i; + if(!(v & m)) { + v >>= i; + s += i; + } + } + if(v == 1) + return s; + return -1; +} + +int +vlog(Node *n) +{ + if(n->op != OCONST) + goto bad; + if(typefd[n->type->etype]) + goto bad; + + return xlog2(n->vconst); + +bad: + return -1; +} + +int +topbit(uint32 v) +{ + int i; + + for(i = -1; v; i++) + v >>= 1; + return i; +} + +/* + * try to cast a constant down + * rather than cast a variable up + * example: + * if(c == 'a') + */ +void +relcon(Node *l, Node *r) +{ + vlong v; + + if(l->op != OCONST) + return; + if(r->op != OCAST) + return; + if(!nilcast(r->left->type, r->type)) + return; + switch(r->type->etype) { + default: + return; + case TCHAR: + case TUCHAR: + case TSHORT: + case TUSHORT: + v = convvtox(l->vconst, r->type->etype); + if(v != l->vconst) + return; + break; + } + l->type = r->left->type; + *r = *r->left; +} + +int +relindex(int o) +{ + + switch(o) { + default: + diag(Z, "bad in relindex: %O", o); + case OEQ: return 0; + case ONE: return 1; + case OLE: return 2; + case OLS: return 3; + case OLT: return 4; + case OLO: return 5; + case OGE: return 6; + case OHS: return 7; + case OGT: return 8; + case OHI: return 9; + } +} + +Node* +invert(Node *n) +{ + Node *i; + + if(n == Z || n->op != OLIST) + return n; + i = n; + for(n = n->left; n != Z; n = n->left) { + if(n->op != OLIST) + break; + i->left = n->right; + n->right = i; + i = n; + } + i->left = n; + return i; +} + +int +bitno(int32 b) +{ + int i; + + for(i=0; i<32; i++) + if(b & (1L< vlong */ + else + warn(Z, "once is enough: %Q", a & b); + return c; +} + +void +diag(Node *n, char *fmt, ...) +{ + char buf[STRINGSZ]; + va_list arg; + + va_start(arg, fmt); + vseprint(buf, buf+sizeof(buf), fmt, arg); + va_end(arg); + Bprint(&diagbuf, "%L %s\n", (n==Z)? nearln: n->lineno, buf); + + if(debug['X']){ + Bflush(&diagbuf); + abort(); + } + if(n != Z) + if(debug['v']) + prtree(n, "diagnostic"); + + nerrors++; + if(nerrors > 10) { + Bprint(&diagbuf, "too many errors\n"); + errorexit(); + } +} + +void +warn(Node *n, char *fmt, ...) +{ + char buf[STRINGSZ]; + va_list arg; + + if(debug['w']) { + Bprint(&diagbuf, "warning: "); + va_start(arg, fmt); + vseprint(buf, buf+sizeof(buf), fmt, arg); + va_end(arg); + Bprint(&diagbuf, "%L %s\n", (n==Z)? nearln: n->lineno, buf); + + if(n != Z) + if(debug['v']) + prtree(n, "warning"); + } +} + +void +yyerror(char *fmt, ...) +{ + char buf[STRINGSZ]; + va_list arg; + + /* + * hack to intercept message from yaccpar + */ + if(strcmp(fmt, "syntax error") == 0) { + yyerror("syntax error, last name: %s", symb); + return; + } + va_start(arg, fmt); + vseprint(buf, buf+sizeof(buf), fmt, arg); + va_end(arg); + Bprint(&diagbuf, "%L %s\n", lineno, buf); + nerrors++; + if(nerrors > 10) { + Bprint(&diagbuf, "too many errors\n"); + errorexit(); + } +} + +void +fatal(Node *n, char *fmt, ...) +{ + char buf[STRINGSZ]; + va_list arg; + + va_start(arg, fmt); + vseprint(buf, buf+sizeof(buf), fmt, arg); + va_end(arg); + Bprint(&diagbuf, "%L %s\n", (n==Z)? nearln: n->lineno, buf); + + if(debug['X']){ + Bflush(&diagbuf); + abort(); + } + if(n != Z) + if(debug['v']) + prtree(n, "diagnostic"); + + nerrors++; + errorexit(); +} + +uint32 thash1 = 0x2edab8c9; +uint32 thash2 = 0x1dc74fb8; +uint32 thash3 = 0x1f241331; +uint32 thash[NALLTYPES]; +Init thashinit[] = +{ + TXXX, 0x17527bbd, 0, + TCHAR, 0x5cedd32b, 0, + TUCHAR, 0x552c4454, 0, + TSHORT, 0x63040b4b, 0, + TUSHORT, 0x32a45878, 0, + TINT, 0x4151d5bd, 0, + TUINT, 0x5ae707d6, 0, + TLONG, 0x5ef20f47, 0, + TULONG, 0x36d8eb8f, 0, + TVLONG, 0x6e5e9590, 0, + TUVLONG, 0x75910105, 0, + TFLOAT, 0x25fd7af1, 0, + TDOUBLE, 0x7c40a1b2, 0, + TIND, 0x1b832357, 0, + TFUNC, 0x6babc9cb, 0, + TARRAY, 0x7c50986d, 0, + TVOID, 0x44112eff, 0, + TSTRUCT, 0x7c2da3bf, 0, + TUNION, 0x3eb25e98, 0, + TENUM, 0x44b54f61, 0, + TFILE, 0x19242ac3, 0, + TOLD, 0x22b15988, 0, + TDOT, 0x0204f6b3, 0, + -1, 0, 0, +}; + +char* bnames[NALIGN]; +Init bnamesinit[] = +{ + Axxx, 0, "Axxx", + Ael1, 0, "el1", + Ael2, 0, "el2", + Asu2, 0, "su2", + Aarg0, 0, "arg0", + Aarg1, 0, "arg1", + Aarg2, 0, "arg2", + Aaut3, 0, "aut3", + -1, 0, 0, +}; + +char* tnames[NALLTYPES]; +Init tnamesinit[] = +{ + TXXX, 0, "TXXX", + TCHAR, 0, "CHAR", + TUCHAR, 0, "UCHAR", + TSHORT, 0, "SHORT", + TUSHORT, 0, "USHORT", + TINT, 0, "INT", + TUINT, 0, "UINT", + TLONG, 0, "LONG", + TULONG, 0, "ULONG", + TVLONG, 0, "VLONG", + TUVLONG, 0, "UVLONG", + TFLOAT, 0, "FLOAT", + TDOUBLE, 0, "DOUBLE", + TIND, 0, "IND", + TFUNC, 0, "FUNC", + TARRAY, 0, "ARRAY", + TVOID, 0, "VOID", + TSTRUCT, 0, "STRUCT", + TUNION, 0, "UNION", + TENUM, 0, "ENUM", + TFILE, 0, "FILE", + TOLD, 0, "OLD", + TDOT, 0, "DOT", + -1, 0, 0, +}; + +char* gnames[NGTYPES]; +Init gnamesinit[] = +{ + GXXX, 0, "GXXX", + GCONSTNT, 0, "CONST", + GVOLATILE, 0, "VOLATILE", + GVOLATILE|GCONSTNT, 0, "CONST-VOLATILE", + -1, 0, 0, +}; + +char* qnames[NALLTYPES]; +Init qnamesinit[] = +{ + TXXX, 0, "TXXX", + TCHAR, 0, "CHAR", + TUCHAR, 0, "UCHAR", + TSHORT, 0, "SHORT", + TUSHORT, 0, "USHORT", + TINT, 0, "INT", + TUINT, 0, "UINT", + TLONG, 0, "LONG", + TULONG, 0, "ULONG", + TVLONG, 0, "VLONG", + TUVLONG, 0, "UVLONG", + TFLOAT, 0, "FLOAT", + TDOUBLE, 0, "DOUBLE", + TIND, 0, "IND", + TFUNC, 0, "FUNC", + TARRAY, 0, "ARRAY", + TVOID, 0, "VOID", + TSTRUCT, 0, "STRUCT", + TUNION, 0, "UNION", + TENUM, 0, "ENUM", + + TAUTO, 0, "AUTO", + TEXTERN, 0, "EXTERN", + TSTATIC, 0, "STATIC", + TTYPEDEF, 0, "TYPEDEF", + TTYPESTR, 0, "TYPESTR", + TREGISTER, 0, "REGISTER", + TCONSTNT, 0, "CONSTNT", + TVOLATILE, 0, "VOLATILE", + TUNSIGNED, 0, "UNSIGNED", + TSIGNED, 0, "SIGNED", + TDOT, 0, "DOT", + TFILE, 0, "FILE", + TOLD, 0, "OLD", + -1, 0, 0, +}; +char* cnames[NCTYPES]; +Init cnamesinit[] = +{ + CXXX, 0, "CXXX", + CAUTO, 0, "AUTO", + CEXTERN, 0, "EXTERN", + CGLOBL, 0, "GLOBL", + CSTATIC, 0, "STATIC", + CLOCAL, 0, "LOCAL", + CTYPEDEF, 0, "TYPEDEF", + CTYPESTR, 0, "TYPESTR", + CPARAM, 0, "PARAM", + CSELEM, 0, "SELEM", + CLABEL, 0, "LABEL", + CEXREG, 0, "EXREG", + -1, 0, 0, +}; + +char* onames[OEND+1]; +Init onamesinit[] = +{ + OXXX, 0, "OXXX", + OADD, 0, "ADD", + OADDR, 0, "ADDR", + OAND, 0, "AND", + OANDAND, 0, "ANDAND", + OARRAY, 0, "ARRAY", + OAS, 0, "AS", + OASI, 0, "ASI", + OASADD, 0, "ASADD", + OASAND, 0, "ASAND", + OASASHL, 0, "ASASHL", + OASASHR, 0, "ASASHR", + OASDIV, 0, "ASDIV", + OASHL, 0, "ASHL", + OASHR, 0, "ASHR", + OASLDIV, 0, "ASLDIV", + OASLMOD, 0, "ASLMOD", + OASLMUL, 0, "ASLMUL", + OASLSHR, 0, "ASLSHR", + OASMOD, 0, "ASMOD", + OASMUL, 0, "ASMUL", + OASOR, 0, "ASOR", + OASSUB, 0, "ASSUB", + OASXOR, 0, "ASXOR", + OBIT, 0, "BIT", + OBREAK, 0, "BREAK", + OCASE, 0, "CASE", + OCAST, 0, "CAST", + OCOMMA, 0, "COMMA", + OCOND, 0, "COND", + OCONST, 0, "CONST", + OCONTINUE, 0, "CONTINUE", + ODIV, 0, "DIV", + ODOT, 0, "DOT", + ODOTDOT, 0, "DOTDOT", + ODWHILE, 0, "DWHILE", + OENUM, 0, "ENUM", + OEQ, 0, "EQ", + OEXREG, 0, "EXREG", + OFOR, 0, "FOR", + OFUNC, 0, "FUNC", + OGE, 0, "GE", + OGOTO, 0, "GOTO", + OGT, 0, "GT", + OHI, 0, "HI", + OHS, 0, "HS", + OIF, 0, "IF", + OIND, 0, "IND", + OINDREG, 0, "INDREG", + OINIT, 0, "INIT", + OLABEL, 0, "LABEL", + OLDIV, 0, "LDIV", + OLE, 0, "LE", + OLIST, 0, "LIST", + OLMOD, 0, "LMOD", + OLMUL, 0, "LMUL", + OLO, 0, "LO", + OLS, 0, "LS", + OLSHR, 0, "LSHR", + OLT, 0, "LT", + OMOD, 0, "MOD", + OMUL, 0, "MUL", + ONAME, 0, "NAME", + ONE, 0, "NE", + ONOT, 0, "NOT", + OOR, 0, "OR", + OOROR, 0, "OROR", + OPOSTDEC, 0, "POSTDEC", + OPOSTINC, 0, "POSTINC", + OPREDEC, 0, "PREDEC", + OPREINC, 0, "PREINC", + OPROTO, 0, "PROTO", + OREGISTER, 0, "REGISTER", + ORETURN, 0, "RETURN", + OSET, 0, "SET", + OSIGN, 0, "SIGN", + OSIZE, 0, "SIZE", + OSTRING, 0, "STRING", + OLSTRING, 0, "LSTRING", + OSTRUCT, 0, "STRUCT", + OSUB, 0, "SUB", + OSWITCH, 0, "SWITCH", + OUNION, 0, "UNION", + OUSED, 0, "USED", + OWHILE, 0, "WHILE", + OXOR, 0, "XOR", + OPOS, 0, "POS", + ONEG, 0, "NEG", + OCOM, 0, "COM", + OELEM, 0, "ELEM", + OTST, 0, "TST", + OINDEX, 0, "INDEX", + OFAS, 0, "FAS", + OREGPAIR, 0, "REGPAIR", + OEND, 0, "END", + -1, 0, 0, +}; + +/* OEQ, ONE, OLE, OLS, OLT, OLO, OGE, OHS, OGT, OHI */ +uchar comrel[12] = +{ + ONE, OEQ, OGT, OHI, OGE, OHS, OLT, OLO, OLE, OLS, +}; +uchar invrel[12] = +{ + OEQ, ONE, OGE, OHS, OGT, OHI, OLE, OLS, OLT, OLO, +}; +uchar logrel[12] = +{ + OEQ, ONE, OLS, OLS, OLO, OLO, OHS, OHS, OHI, OHI, +}; + +uchar typei[NTYPE]; +int typeiinit[] = +{ + TCHAR, TUCHAR, TSHORT, TUSHORT, TINT, TUINT, TLONG, TULONG, TVLONG, TUVLONG, -1, +}; +uchar typeu[NTYPE]; +int typeuinit[] = +{ + TUCHAR, TUSHORT, TUINT, TULONG, TUVLONG, TIND, -1, +}; + +uchar typesuv[NTYPE]; +int typesuvinit[] = +{ + TVLONG, TUVLONG, TSTRUCT, TUNION, -1, +}; + +uchar typeilp[NTYPE]; +int typeilpinit[] = +{ + TINT, TUINT, TLONG, TULONG, TIND, -1 +}; + +uchar typechl[NTYPE]; +uchar typechlv[NTYPE]; +uchar typechlvp[NTYPE]; +int typechlinit[] = +{ + TCHAR, TUCHAR, TSHORT, TUSHORT, TINT, TUINT, TLONG, TULONG, -1, +}; + +uchar typechlp[NTYPE]; +int typechlpinit[] = +{ + TCHAR, TUCHAR, TSHORT, TUSHORT, TINT, TUINT, TLONG, TULONG, TIND, -1, +}; + +uchar typechlpfd[NTYPE]; +int typechlpfdinit[] = +{ + TCHAR, TUCHAR, TSHORT, TUSHORT, TINT, TUINT, TLONG, TULONG, TFLOAT, TDOUBLE, TIND, -1, +}; + +uchar typec[NTYPE]; +int typecinit[] = +{ + TCHAR, TUCHAR, -1 +}; + +uchar typeh[NTYPE]; +int typehinit[] = +{ + TSHORT, TUSHORT, -1, +}; + +uchar typeil[NTYPE]; +int typeilinit[] = +{ + TINT, TUINT, TLONG, TULONG, -1, +}; + +uchar typev[NTYPE]; +int typevinit[] = +{ + TVLONG, TUVLONG, -1, +}; + +uchar typefd[NTYPE]; +int typefdinit[] = +{ + TFLOAT, TDOUBLE, -1, +}; + +uchar typeaf[NTYPE]; +int typeafinit[] = +{ + TFUNC, TARRAY, -1, +}; + +uchar typesu[NTYPE]; +int typesuinit[] = +{ + TSTRUCT, TUNION, -1, +}; + +int32 tasign[NTYPE]; +Init tasigninit[] = +{ + TCHAR, BNUMBER, 0, + TUCHAR, BNUMBER, 0, + TSHORT, BNUMBER, 0, + TUSHORT, BNUMBER, 0, + TINT, BNUMBER, 0, + TUINT, BNUMBER, 0, + TLONG, BNUMBER, 0, + TULONG, BNUMBER, 0, + TVLONG, BNUMBER, 0, + TUVLONG, BNUMBER, 0, + TFLOAT, BNUMBER, 0, + TDOUBLE, BNUMBER, 0, + TIND, BIND, 0, + TSTRUCT, BSTRUCT, 0, + TUNION, BUNION, 0, + -1, 0, 0, +}; + +int32 tasadd[NTYPE]; +Init tasaddinit[] = +{ + TCHAR, BNUMBER, 0, + TUCHAR, BNUMBER, 0, + TSHORT, BNUMBER, 0, + TUSHORT, BNUMBER, 0, + TINT, BNUMBER, 0, + TUINT, BNUMBER, 0, + TLONG, BNUMBER, 0, + TULONG, BNUMBER, 0, + TVLONG, BNUMBER, 0, + TUVLONG, BNUMBER, 0, + TFLOAT, BNUMBER, 0, + TDOUBLE, BNUMBER, 0, + TIND, BINTEGER, 0, + -1, 0, 0, +}; + +int32 tcast[NTYPE]; +Init tcastinit[] = +{ + TCHAR, BNUMBER|BIND|BVOID, 0, + TUCHAR, BNUMBER|BIND|BVOID, 0, + TSHORT, BNUMBER|BIND|BVOID, 0, + TUSHORT, BNUMBER|BIND|BVOID, 0, + TINT, BNUMBER|BIND|BVOID, 0, + TUINT, BNUMBER|BIND|BVOID, 0, + TLONG, BNUMBER|BIND|BVOID, 0, + TULONG, BNUMBER|BIND|BVOID, 0, + TVLONG, BNUMBER|BIND|BVOID, 0, + TUVLONG, BNUMBER|BIND|BVOID, 0, + TFLOAT, BNUMBER|BVOID, 0, + TDOUBLE, BNUMBER|BVOID, 0, + TIND, BINTEGER|BIND|BVOID, 0, + TVOID, BVOID, 0, + TSTRUCT, BSTRUCT|BVOID, 0, + TUNION, BUNION|BVOID, 0, + -1, 0, 0, +}; + +int32 tadd[NTYPE]; +Init taddinit[] = +{ + TCHAR, BNUMBER|BIND, 0, + TUCHAR, BNUMBER|BIND, 0, + TSHORT, BNUMBER|BIND, 0, + TUSHORT, BNUMBER|BIND, 0, + TINT, BNUMBER|BIND, 0, + TUINT, BNUMBER|BIND, 0, + TLONG, BNUMBER|BIND, 0, + TULONG, BNUMBER|BIND, 0, + TVLONG, BNUMBER|BIND, 0, + TUVLONG, BNUMBER|BIND, 0, + TFLOAT, BNUMBER, 0, + TDOUBLE, BNUMBER, 0, + TIND, BINTEGER, 0, + -1, 0, 0, +}; + +int32 tsub[NTYPE]; +Init tsubinit[] = +{ + TCHAR, BNUMBER, 0, + TUCHAR, BNUMBER, 0, + TSHORT, BNUMBER, 0, + TUSHORT, BNUMBER, 0, + TINT, BNUMBER, 0, + TUINT, BNUMBER, 0, + TLONG, BNUMBER, 0, + TULONG, BNUMBER, 0, + TVLONG, BNUMBER, 0, + TUVLONG, BNUMBER, 0, + TFLOAT, BNUMBER, 0, + TDOUBLE, BNUMBER, 0, + TIND, BINTEGER|BIND, 0, + -1, 0, 0, +}; + +int32 tmul[NTYPE]; +Init tmulinit[] = +{ + TCHAR, BNUMBER, 0, + TUCHAR, BNUMBER, 0, + TSHORT, BNUMBER, 0, + TUSHORT, BNUMBER, 0, + TINT, BNUMBER, 0, + TUINT, BNUMBER, 0, + TLONG, BNUMBER, 0, + TULONG, BNUMBER, 0, + TVLONG, BNUMBER, 0, + TUVLONG, BNUMBER, 0, + TFLOAT, BNUMBER, 0, + TDOUBLE, BNUMBER, 0, + -1, 0, 0, +}; + +int32 tand[NTYPE]; +Init tandinit[] = +{ + TCHAR, BINTEGER, 0, + TUCHAR, BINTEGER, 0, + TSHORT, BINTEGER, 0, + TUSHORT, BINTEGER, 0, + TINT, BNUMBER, 0, + TUINT, BNUMBER, 0, + TLONG, BINTEGER, 0, + TULONG, BINTEGER, 0, + TVLONG, BINTEGER, 0, + TUVLONG, BINTEGER, 0, + -1, 0, 0, +}; + +int32 trel[NTYPE]; +Init trelinit[] = +{ + TCHAR, BNUMBER, 0, + TUCHAR, BNUMBER, 0, + TSHORT, BNUMBER, 0, + TUSHORT, BNUMBER, 0, + TINT, BNUMBER, 0, + TUINT, BNUMBER, 0, + TLONG, BNUMBER, 0, + TULONG, BNUMBER, 0, + TVLONG, BNUMBER, 0, + TUVLONG, BNUMBER, 0, + TFLOAT, BNUMBER, 0, + TDOUBLE, BNUMBER, 0, + TIND, BIND, 0, + -1, 0, 0, +}; + +int32 tfunct[1] = +{ + BFUNC, +}; + +int32 tindir[1] = +{ + BIND, +}; + +int32 tdot[1] = +{ + BSTRUCT|BUNION, +}; + +int32 tnot[1] = +{ + BNUMBER|BIND, +}; + +int32 targ[1] = +{ + BNUMBER|BIND|BSTRUCT|BUNION, +}; + +uchar tab[NTYPE][NTYPE] = +{ +/*TXXX*/ { 0, + }, + +/*TCHAR*/ { 0, TCHAR, TUCHAR, TSHORT, TUSHORT, TINT, TUINT, TLONG, + TULONG, TVLONG, TUVLONG, TFLOAT, TDOUBLE, TIND, + }, +/*TUCHAR*/ { 0, TUCHAR, TUCHAR, TUSHORT, TUSHORT, TUINT, TUINT, TULONG, + TULONG, TUVLONG, TUVLONG, TFLOAT, TDOUBLE, TIND, + }, +/*TSHORT*/ { 0, TSHORT, TUSHORT, TSHORT, TUSHORT, TINT, TUINT, TLONG, + TULONG, TVLONG, TUVLONG, TFLOAT, TDOUBLE, TIND, + }, +/*TUSHORT*/ { 0, TUSHORT, TUSHORT, TUSHORT, TUSHORT, TUINT, TUINT, TULONG, + TULONG, TUVLONG, TUVLONG, TFLOAT, TDOUBLE, TIND, + }, +/*TINT*/ { 0, TINT, TUINT, TINT, TUINT, TINT, TUINT, TLONG, + TULONG, TVLONG, TUVLONG, TFLOAT, TDOUBLE, TIND, + }, +/*TUINT*/ { 0, TUINT, TUINT, TUINT, TUINT, TUINT, TUINT, TULONG, + TULONG, TUVLONG, TUVLONG, TFLOAT, TDOUBLE, TIND, + }, +/*TLONG*/ { 0, TLONG, TULONG, TLONG, TULONG, TLONG, TULONG, TLONG, + TULONG, TVLONG, TUVLONG, TFLOAT, TDOUBLE, TIND, + }, +/*TULONG*/ { 0, TULONG, TULONG, TULONG, TULONG, TULONG, TULONG, TULONG, + TULONG, TUVLONG, TUVLONG, TFLOAT, TDOUBLE, TIND, + }, +/*TVLONG*/ { 0, TVLONG, TUVLONG, TVLONG, TUVLONG, TVLONG, TUVLONG, TVLONG, + TUVLONG, TVLONG, TUVLONG, TFLOAT, TDOUBLE, TIND, + }, +/*TUVLONG*/ { 0, TUVLONG, TUVLONG, TUVLONG, TUVLONG, TUVLONG, TUVLONG, TUVLONG, + TUVLONG, TUVLONG, TUVLONG, TFLOAT, TDOUBLE, TIND, + }, +/*TFLOAT*/ { 0, TFLOAT, TFLOAT, TFLOAT, TFLOAT, TFLOAT, TFLOAT, TFLOAT, + TFLOAT, TFLOAT, TFLOAT, TFLOAT, TDOUBLE, TIND, + }, +/*TDOUBLE*/ { 0, TDOUBLE, TDOUBLE, TDOUBLE, TDOUBLE, TDOUBLE, TDOUBLE, TDOUBLE, + TDOUBLE, TDOUBLE, TDOUBLE, TFLOAT, TDOUBLE, TIND, + }, +/*TIND*/ { 0, TIND, TIND, TIND, TIND, TIND, TIND, TIND, + TIND, TIND, TIND, TIND, TIND, TIND, + }, +}; + +void +urk(char *name, int max, int i) +{ + if(i >= max) { + fprint(2, "bad tinit: %s %d>=%d\n", name, i, max); + exits("init"); + } +} + +void +tinit(void) +{ + int *ip; + Init *p; + + for(p=thashinit; p->code >= 0; p++) { + urk("thash", nelem(thash), p->code); + thash[p->code] = p->value; + } + for(p=bnamesinit; p->code >= 0; p++) { + urk("bnames", nelem(bnames), p->code); + bnames[p->code] = p->s; + } + for(p=tnamesinit; p->code >= 0; p++) { + urk("tnames", nelem(tnames), p->code); + tnames[p->code] = p->s; + } + for(p=gnamesinit; p->code >= 0; p++) { + urk("gnames", nelem(gnames), p->code); + gnames[p->code] = p->s; + } + for(p=qnamesinit; p->code >= 0; p++) { + urk("qnames", nelem(qnames), p->code); + qnames[p->code] = p->s; + } + for(p=cnamesinit; p->code >= 0; p++) { + urk("cnames", nelem(cnames), p->code); + cnames[p->code] = p->s; + } + for(p=onamesinit; p->code >= 0; p++) { + urk("onames", nelem(onames), p->code); + onames[p->code] = p->s; + } + for(ip=typeiinit; *ip>=0; ip++) { + urk("typei", nelem(typei), *ip); + typei[*ip] = 1; + } + for(ip=typeuinit; *ip>=0; ip++) { + urk("typeu", nelem(typeu), *ip); + typeu[*ip] = 1; + } + for(ip=typesuvinit; *ip>=0; ip++) { + urk("typesuv", nelem(typesuv), *ip); + typesuv[*ip] = 1; + } + for(ip=typeilpinit; *ip>=0; ip++) { + urk("typeilp", nelem(typeilp), *ip); + typeilp[*ip] = 1; + } + for(ip=typechlinit; *ip>=0; ip++) { + urk("typechl", nelem(typechl), *ip); + typechl[*ip] = 1; + typechlv[*ip] = 1; + typechlvp[*ip] = 1; + } + for(ip=typechlpinit; *ip>=0; ip++) { + urk("typechlp", nelem(typechlp), *ip); + typechlp[*ip] = 1; + typechlvp[*ip] = 1; + } + for(ip=typechlpfdinit; *ip>=0; ip++) { + urk("typechlpfd", nelem(typechlpfd), *ip); + typechlpfd[*ip] = 1; + } + for(ip=typecinit; *ip>=0; ip++) { + urk("typec", nelem(typec), *ip); + typec[*ip] = 1; + } + for(ip=typehinit; *ip>=0; ip++) { + urk("typeh", nelem(typeh), *ip); + typeh[*ip] = 1; + } + for(ip=typeilinit; *ip>=0; ip++) { + urk("typeil", nelem(typeil), *ip); + typeil[*ip] = 1; + } + for(ip=typevinit; *ip>=0; ip++) { + urk("typev", nelem(typev), *ip); + typev[*ip] = 1; + typechlv[*ip] = 1; + typechlvp[*ip] = 1; + } + for(ip=typefdinit; *ip>=0; ip++) { + urk("typefd", nelem(typefd), *ip); + typefd[*ip] = 1; + } + for(ip=typeafinit; *ip>=0; ip++) { + urk("typeaf", nelem(typeaf), *ip); + typeaf[*ip] = 1; + } + for(ip=typesuinit; *ip >= 0; ip++) { + urk("typesu", nelem(typesu), *ip); + typesu[*ip] = 1; + } + for(p=tasigninit; p->code >= 0; p++) { + urk("tasign", nelem(tasign), p->code); + tasign[p->code] = p->value; + } + for(p=tasaddinit; p->code >= 0; p++) { + urk("tasadd", nelem(tasadd), p->code); + tasadd[p->code] = p->value; + } + for(p=tcastinit; p->code >= 0; p++) { + urk("tcast", nelem(tcast), p->code); + tcast[p->code] = p->value; + } + for(p=taddinit; p->code >= 0; p++) { + urk("tadd", nelem(tadd), p->code); + tadd[p->code] = p->value; + } + for(p=tsubinit; p->code >= 0; p++) { + urk("tsub", nelem(tsub), p->code); + tsub[p->code] = p->value; + } + for(p=tmulinit; p->code >= 0; p++) { + urk("tmul", nelem(tmul), p->code); + tmul[p->code] = p->value; + } + for(p=tandinit; p->code >= 0; p++) { + urk("tand", nelem(tand), p->code); + tand[p->code] = p->value; + } + for(p=trelinit; p->code >= 0; p++) { + urk("trel", nelem(trel), p->code); + trel[p->code] = p->value; + } + + /* 32-bit defaults */ + typeword = typechlp; + typecmplx = typesuv; +} + +/* + * return 1 if it is impossible to jump into the middle of n. + */ +static int +deadhead(Node *n, int caseok) +{ +loop: + if(n == Z) + return 1; + switch(n->op) { + case OLIST: + if(!deadhead(n->left, caseok)) + return 0; + rloop: + n = n->right; + goto loop; + + case ORETURN: + break; + + case OLABEL: + return 0; + + case OGOTO: + break; + + case OCASE: + if(!caseok) + return 0; + goto rloop; + + case OSWITCH: + return deadhead(n->right, 1); + + case OWHILE: + case ODWHILE: + goto rloop; + + case OFOR: + goto rloop; + + case OCONTINUE: + break; + + case OBREAK: + break; + + case OIF: + return deadhead(n->right->left, caseok) && deadhead(n->right->right, caseok); + + case OSET: + case OUSED: + break; + } + return 1; +} + +int +deadheads(Node *c) +{ + return deadhead(c->left, 0) && deadhead(c->right, 0); +} + +int +mixedasop(Type *l, Type *r) +{ + return !typefd[l->etype] && typefd[r->etype]; +} diff --git a/src/cmd/cgo/Makefile b/src/cmd/cgo/Makefile new file mode 100644 index 000000000..5458c3e4f --- /dev/null +++ b/src/cmd/cgo/Makefile @@ -0,0 +1,15 @@ +# Copyright 2009 The Go Authors. All rights reserved. +# Use of this source code is governed by a BSD-style +# license that can be found in the LICENSE file. + +include ../../Make.inc + +TARG=cgo +GOFILES=\ + ast.go\ + gcc.go\ + main.go\ + out.go\ + util.go\ + +include ../../Make.cmd diff --git a/src/cmd/cgo/ast.go b/src/cmd/cgo/ast.go new file mode 100644 index 000000000..73b7313d6 --- /dev/null +++ b/src/cmd/cgo/ast.go @@ -0,0 +1,414 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Parse input AST and prepare Prog structure. + +package main + +import ( + "fmt" + "go/ast" + "go/doc" + "go/parser" + "go/scanner" + "go/token" + "os" + "strings" +) + +func parse(name string, flags uint) *ast.File { + ast1, err := parser.ParseFile(fset, name, nil, flags) + if err != nil { + if list, ok := err.(scanner.ErrorList); ok { + // If err is a scanner.ErrorList, its String will print just + // the first error and then (+n more errors). + // Instead, turn it into a new Error that will return + // details for all the errors. + for _, e := range list { + fmt.Fprintln(os.Stderr, e) + } + os.Exit(2) + } + fatalf("parsing %s: %s", name, err) + } + return ast1 +} + +func sourceLine(n ast.Node) int { + return fset.Position(n.Pos()).Line +} + +// ReadGo populates f with information learned from reading the +// Go source file with the given file name. It gathers the C preamble +// attached to the import "C" comment, a list of references to C.xxx, +// a list of exported functions, and the actual AST, to be rewritten and +// printed. +func (f *File) ReadGo(name string) { + // Two different parses: once with comments, once without. + // The printer is not good enough at printing comments in the + // right place when we start editing the AST behind its back, + // so we use ast1 to look for the doc comments on import "C" + // and on exported functions, and we use ast2 for translating + // and reprinting. + ast1 := parse(name, parser.ParseComments) + ast2 := parse(name, 0) + + f.Package = ast1.Name.Name + f.Name = make(map[string]*Name) + + // In ast1, find the import "C" line and get any extra C preamble. + sawC := false + for _, decl := range ast1.Decls { + d, ok := decl.(*ast.GenDecl) + if !ok { + continue + } + for _, spec := range d.Specs { + s, ok := spec.(*ast.ImportSpec) + if !ok || string(s.Path.Value) != `"C"` { + continue + } + sawC = true + if s.Name != nil { + error(s.Path.Pos(), `cannot rename import "C"`) + } + cg := s.Doc + if cg == nil && len(d.Specs) == 1 { + cg = d.Doc + } + if cg != nil { + f.Preamble += fmt.Sprintf("#line %d %q\n", sourceLine(cg), name) + f.Preamble += doc.CommentText(cg) + "\n" + } + } + } + if !sawC { + error(token.NoPos, `cannot find import "C"`) + } + + // In ast2, strip the import "C" line. + w := 0 + for _, decl := range ast2.Decls { + d, ok := decl.(*ast.GenDecl) + if !ok { + ast2.Decls[w] = decl + w++ + continue + } + ws := 0 + for _, spec := range d.Specs { + s, ok := spec.(*ast.ImportSpec) + if !ok || string(s.Path.Value) != `"C"` { + d.Specs[ws] = spec + ws++ + } + } + if ws == 0 { + continue + } + d.Specs = d.Specs[0:ws] + ast2.Decls[w] = d + w++ + } + ast2.Decls = ast2.Decls[0:w] + + // Accumulate pointers to uses of C.x. + if f.Ref == nil { + f.Ref = make([]*Ref, 0, 8) + } + f.walk(ast2, "prog", (*File).saveRef) + + // Accumulate exported functions. + // The comments are only on ast1 but we need to + // save the function bodies from ast2. + // The first walk fills in ExpFunc, and the + // second walk changes the entries to + // refer to ast2 instead. + f.walk(ast1, "prog", (*File).saveExport) + f.walk(ast2, "prog", (*File).saveExport2) + + f.AST = ast2 +} + +// Save references to C.xxx for later processing. +func (f *File) saveRef(x interface{}, context string) { + n, ok := x.(*ast.Expr) + if !ok { + return + } + if sel, ok := (*n).(*ast.SelectorExpr); ok { + // For now, assume that the only instance of capital C is + // when used as the imported package identifier. + // The parser should take care of scoping in the future, + // so that we will be able to distinguish a "top-level C" + // from a local C. + if l, ok := sel.X.(*ast.Ident); ok && l.Name == "C" { + if context == "as2" { + context = "expr" + } + goname := sel.Sel.Name + if goname == "errno" { + error(sel.Pos(), "cannot refer to errno directly; see documentation") + return + } + name := f.Name[goname] + if name == nil { + name = &Name{ + Go: goname, + } + f.Name[goname] = name + } + f.Ref = append(f.Ref, &Ref{ + Name: name, + Expr: n, + Context: context, + }) + return + } + } +} + +// If a function should be exported add it to ExpFunc. +func (f *File) saveExport(x interface{}, context string) { + n, ok := x.(*ast.FuncDecl) + if !ok { + return + } + + if n.Doc == nil { + return + } + for _, c := range n.Doc.List { + if !strings.HasPrefix(string(c.Text), "//export ") { + continue + } + + name := strings.TrimSpace(string(c.Text[9:])) + if name == "" { + error(c.Pos(), "export missing name") + } + + if name != n.Name.Name { + error(c.Pos(), "export comment has wrong name %q, want %q", name, n.Name.Name) + } + + f.ExpFunc = append(f.ExpFunc, &ExpFunc{ + Func: n, + ExpName: name, + }) + break + } +} + +// Make f.ExpFunc[i] point at the Func from this AST instead of the other one. +func (f *File) saveExport2(x interface{}, context string) { + n, ok := x.(*ast.FuncDecl) + if !ok { + return + } + + for _, exp := range f.ExpFunc { + if exp.Func.Name.Name == n.Name.Name { + exp.Func = n + break + } + } +} + +// walk walks the AST x, calling visit(f, x, context) for each node. +func (f *File) walk(x interface{}, context string, visit func(*File, interface{}, string)) { + visit(f, x, context) + switch n := x.(type) { + case *ast.Expr: + f.walk(*n, context, visit) + + // everything else just recurs + default: + error(token.NoPos, "unexpected type %T in walk", x, visit) + panic("unexpected type") + + case nil: + + // These are ordered and grouped to match ../../pkg/go/ast/ast.go + case *ast.Field: + f.walk(&n.Type, "type", visit) + case *ast.FieldList: + for _, field := range n.List { + f.walk(field, context, visit) + } + case *ast.BadExpr: + case *ast.Ident: + case *ast.Ellipsis: + case *ast.BasicLit: + case *ast.FuncLit: + f.walk(n.Type, "type", visit) + f.walk(n.Body, "stmt", visit) + case *ast.CompositeLit: + f.walk(&n.Type, "type", visit) + f.walk(n.Elts, "expr", visit) + case *ast.ParenExpr: + f.walk(&n.X, context, visit) + case *ast.SelectorExpr: + f.walk(&n.X, "selector", visit) + case *ast.IndexExpr: + f.walk(&n.X, "expr", visit) + f.walk(&n.Index, "expr", visit) + case *ast.SliceExpr: + f.walk(&n.X, "expr", visit) + if n.Low != nil { + f.walk(&n.Low, "expr", visit) + } + if n.High != nil { + f.walk(&n.High, "expr", visit) + } + case *ast.TypeAssertExpr: + f.walk(&n.X, "expr", visit) + f.walk(&n.Type, "type", visit) + case *ast.CallExpr: + if context == "as2" { + f.walk(&n.Fun, "call2", visit) + } else { + f.walk(&n.Fun, "call", visit) + } + f.walk(n.Args, "expr", visit) + case *ast.StarExpr: + f.walk(&n.X, context, visit) + case *ast.UnaryExpr: + f.walk(&n.X, "expr", visit) + case *ast.BinaryExpr: + f.walk(&n.X, "expr", visit) + f.walk(&n.Y, "expr", visit) + case *ast.KeyValueExpr: + f.walk(&n.Key, "expr", visit) + f.walk(&n.Value, "expr", visit) + + case *ast.ArrayType: + f.walk(&n.Len, "expr", visit) + f.walk(&n.Elt, "type", visit) + case *ast.StructType: + f.walk(n.Fields, "field", visit) + case *ast.FuncType: + f.walk(n.Params, "field", visit) + if n.Results != nil { + f.walk(n.Results, "field", visit) + } + case *ast.InterfaceType: + f.walk(n.Methods, "field", visit) + case *ast.MapType: + f.walk(&n.Key, "type", visit) + f.walk(&n.Value, "type", visit) + case *ast.ChanType: + f.walk(&n.Value, "type", visit) + + case *ast.BadStmt: + case *ast.DeclStmt: + f.walk(n.Decl, "decl", visit) + case *ast.EmptyStmt: + case *ast.LabeledStmt: + f.walk(n.Stmt, "stmt", visit) + case *ast.ExprStmt: + f.walk(&n.X, "expr", visit) + case *ast.SendStmt: + f.walk(&n.Chan, "expr", visit) + f.walk(&n.Value, "expr", visit) + case *ast.IncDecStmt: + f.walk(&n.X, "expr", visit) + case *ast.AssignStmt: + f.walk(n.Lhs, "expr", visit) + if len(n.Lhs) == 2 && len(n.Rhs) == 1 { + f.walk(n.Rhs, "as2", visit) + } else { + f.walk(n.Rhs, "expr", visit) + } + case *ast.GoStmt: + f.walk(n.Call, "expr", visit) + case *ast.DeferStmt: + f.walk(n.Call, "expr", visit) + case *ast.ReturnStmt: + f.walk(n.Results, "expr", visit) + case *ast.BranchStmt: + case *ast.BlockStmt: + f.walk(n.List, context, visit) + case *ast.IfStmt: + f.walk(n.Init, "stmt", visit) + f.walk(&n.Cond, "expr", visit) + f.walk(n.Body, "stmt", visit) + f.walk(n.Else, "stmt", visit) + case *ast.CaseClause: + if context == "typeswitch" { + context = "type" + } else { + context = "expr" + } + f.walk(n.List, context, visit) + f.walk(n.Body, "stmt", visit) + case *ast.SwitchStmt: + f.walk(n.Init, "stmt", visit) + f.walk(&n.Tag, "expr", visit) + f.walk(n.Body, "switch", visit) + case *ast.TypeSwitchStmt: + f.walk(n.Init, "stmt", visit) + f.walk(n.Assign, "stmt", visit) + f.walk(n.Body, "typeswitch", visit) + case *ast.CommClause: + f.walk(n.Comm, "stmt", visit) + f.walk(n.Body, "stmt", visit) + case *ast.SelectStmt: + f.walk(n.Body, "stmt", visit) + case *ast.ForStmt: + f.walk(n.Init, "stmt", visit) + f.walk(&n.Cond, "expr", visit) + f.walk(n.Post, "stmt", visit) + f.walk(n.Body, "stmt", visit) + case *ast.RangeStmt: + f.walk(&n.Key, "expr", visit) + f.walk(&n.Value, "expr", visit) + f.walk(&n.X, "expr", visit) + f.walk(n.Body, "stmt", visit) + + case *ast.ImportSpec: + case *ast.ValueSpec: + f.walk(&n.Type, "type", visit) + f.walk(n.Values, "expr", visit) + case *ast.TypeSpec: + f.walk(&n.Type, "type", visit) + + case *ast.BadDecl: + case *ast.GenDecl: + f.walk(n.Specs, "spec", visit) + case *ast.FuncDecl: + if n.Recv != nil { + f.walk(n.Recv, "field", visit) + } + f.walk(n.Type, "type", visit) + if n.Body != nil { + f.walk(n.Body, "stmt", visit) + } + + case *ast.File: + f.walk(n.Decls, "decl", visit) + + case *ast.Package: + for _, file := range n.Files { + f.walk(file, "file", visit) + } + + case []ast.Decl: + for _, d := range n { + f.walk(d, context, visit) + } + case []ast.Expr: + for i := range n { + f.walk(&n[i], context, visit) + } + case []ast.Stmt: + for _, s := range n { + f.walk(s, context, visit) + } + case []ast.Spec: + for _, s := range n { + f.walk(s, context, visit) + } + } +} diff --git a/src/cmd/cgo/doc.go b/src/cmd/cgo/doc.go new file mode 100644 index 000000000..63413825a --- /dev/null +++ b/src/cmd/cgo/doc.go @@ -0,0 +1,96 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +/* + +Cgo enables the creation of Go packages that call C code. + +Usage: cgo [compiler options] file.go + +The compiler options are passed through uninterpreted when +invoking gcc to compile the C parts of the package. + +The input file.go is a syntactically valid Go source file that imports +the pseudo-package "C" and then refers to types such as C.size_t, +variables such as C.stdout, or functions such as C.putchar. + +If the import of "C" is immediately preceded by a comment, that +comment is used as a header when compiling the C parts of +the package. For example: + + // #include + // #include + import "C" + +CFLAGS and LDFLAGS may be defined with pseudo #cgo directives +within these comments to tweak the behavior of gcc. Values defined +in multiple directives are concatenated together. Options prefixed +by $GOOS, $GOARCH, or $GOOS/$GOARCH are only defined in matching +systems. For example: + + // #cgo CFLAGS: -DPNG_DEBUG=1 + // #cgo linux CFLAGS: -DLINUX=1 + // #cgo LDFLAGS: -lpng + // #include + import "C" + +Alternatively, CFLAGS and LDFLAGS may be obtained via the pkg-config +tool using a '#cgo pkg-config:' directive followed by the package names. +For example: + + // #cgo pkg-config: png cairo + // #include + import "C" + +Within the Go file, C identifiers or field names that are keywords in Go +can be accessed by prefixing them with an underscore: if x points at a C +struct with a field named "type", x._type accesses the field. + +The standard C numeric types are available under the names +C.char, C.schar (signed char), C.uchar (unsigned char), +C.short, C.ushort (unsigned short), C.int, C.uint (unsigned int), +C.long, C.ulong (unsigned long), C.longlong (long long), +C.ulonglong (unsigned long long), C.float, C.double. +The C type void* is represented by Go's unsafe.Pointer. + +To access a struct, union, or enum type directly, prefix it with +struct_, union_, or enum_, as in C.struct_stat. + +Any C function that returns a value may be called in a multiple +assignment context to retrieve both the return value and the +C errno variable as an os.Error. For example: + + n, err := C.atoi("abc") + +In C, a function argument written as a fixed size array +actually requires a pointer to the first element of the array. +C compilers are aware of this calling convention and adjust +the call accordingly, but Go cannot. In Go, you must pass +the pointer to the first element explicitly: C.f(&x[0]). + +A few special functions convert between Go and C types +by making copies of the data. In pseudo-Go definitions: + + // Go string to C string + func C.CString(string) *C.char + + // C string to Go string + func C.GoString(*C.char) string + + // C string, length to Go string + func C.GoStringN(*C.char, C.int) string + + // C pointer, length to Go []byte + func C.GoBytes(unsafe.Pointer, C.int) []byte + +Cgo transforms the input file into four output files: two Go source +files, a C file for 6c (or 8c or 5c), and a C file for gcc. + +The standard package makefile rules in Make.pkg automate the +process of using cgo. See $GOROOT/misc/cgo/stdio and +$GOROOT/misc/cgo/gmp for examples. + +Cgo does not yet work with gccgo. +*/ +package documentation diff --git a/src/cmd/cgo/gcc.go b/src/cmd/cgo/gcc.go new file mode 100644 index 000000000..7ec4d8ccf --- /dev/null +++ b/src/cmd/cgo/gcc.go @@ -0,0 +1,1375 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Annotate Ref in Prog with C types by parsing gcc debug output. +// Conversion of debug output to Go types. + +package main + +import ( + "bytes" + "debug/dwarf" + "debug/elf" + "debug/macho" + "debug/pe" + "encoding/binary" + "flag" + "fmt" + "go/ast" + "go/parser" + "go/token" + "os" + "runtime" + "strconv" + "strings" + "unicode" +) + +var debugDefine = flag.Bool("debug-define", false, "print relevant #defines") +var debugGcc = flag.Bool("debug-gcc", false, "print gcc invocations") + +var nameToC = map[string]string{ + "schar": "signed char", + "uchar": "unsigned char", + "ushort": "unsigned short", + "uint": "unsigned int", + "ulong": "unsigned long", + "longlong": "long long", + "ulonglong": "unsigned long long", + "complexfloat": "float complex", + "complexdouble": "double complex", +} + +// cname returns the C name to use for C.s. +// The expansions are listed in nameToC and also +// struct_foo becomes "struct foo", and similarly for +// union and enum. +func cname(s string) string { + if t, ok := nameToC[s]; ok { + return t + } + + if strings.HasPrefix(s, "struct_") { + return "struct " + s[len("struct_"):] + } + if strings.HasPrefix(s, "union_") { + return "union " + s[len("union_"):] + } + if strings.HasPrefix(s, "enum_") { + return "enum " + s[len("enum_"):] + } + return s +} + +// ParseFlags extracts #cgo CFLAGS and LDFLAGS options from the file +// preamble. Multiple occurrences are concatenated with a separating space, +// even across files. +func (p *Package) ParseFlags(f *File, srcfile string) { + linesIn := strings.Split(f.Preamble, "\n") + linesOut := make([]string, 0, len(linesIn)) + +NextLine: + for _, line := range linesIn { + l := strings.TrimSpace(line) + if len(l) < 5 || l[:4] != "#cgo" || !unicode.IsSpace(int(l[4])) { + linesOut = append(linesOut, line) + continue + } + + l = strings.TrimSpace(l[4:]) + fields := strings.SplitN(l, ":", 2) + if len(fields) != 2 { + fatalf("%s: bad #cgo line: %s", srcfile, line) + } + + var k string + kf := strings.Fields(fields[0]) + switch len(kf) { + case 1: + k = kf[0] + case 2: + k = kf[1] + switch kf[0] { + case runtime.GOOS: + case runtime.GOARCH: + case runtime.GOOS + "/" + runtime.GOARCH: + default: + continue NextLine + } + default: + fatalf("%s: bad #cgo option: %s", srcfile, fields[0]) + } + + args, err := splitQuoted(fields[1]) + if err != nil { + fatalf("%s: bad #cgo option %s: %s", srcfile, k, err) + } + for _, arg := range args { + if !safeName(arg) { + fatalf("%s: #cgo option %s is unsafe: %s", srcfile, k, arg) + } + } + + switch k { + + case "CFLAGS", "LDFLAGS": + p.addToFlag(k, args) + + case "pkg-config": + cflags, ldflags, err := pkgConfig(args) + if err != nil { + fatalf("%s: bad #cgo option %s: %s", srcfile, k, err) + } + p.addToFlag("CFLAGS", cflags) + p.addToFlag("LDFLAGS", ldflags) + + default: + fatalf("%s: unsupported #cgo option %s", srcfile, k) + + } + } + f.Preamble = strings.Join(linesOut, "\n") +} + +// addToFlag appends args to flag. All flags are later written out onto the +// _cgo_flags file for the build system to use. +func (p *Package) addToFlag(flag string, args []string) { + if oldv, ok := p.CgoFlags[flag]; ok { + p.CgoFlags[flag] = oldv + " " + strings.Join(args, " ") + } else { + p.CgoFlags[flag] = strings.Join(args, " ") + } + if flag == "CFLAGS" { + // We'll also need these when preprocessing for dwarf information. + p.GccOptions = append(p.GccOptions, args...) + } +} + +// pkgConfig runs pkg-config and extracts --libs and --cflags information +// for packages. +func pkgConfig(packages []string) (cflags, ldflags []string, err os.Error) { + for _, name := range packages { + if len(name) == 0 || name[0] == '-' { + return nil, nil, os.NewError(fmt.Sprintf("invalid name: %q", name)) + } + } + + args := append([]string{"pkg-config", "--cflags"}, packages...) + stdout, stderr, ok := run(nil, args) + if !ok { + os.Stderr.Write(stderr) + return nil, nil, os.NewError("pkg-config failed") + } + cflags, err = splitQuoted(string(stdout)) + if err != nil { + return + } + + args = append([]string{"pkg-config", "--libs"}, packages...) + stdout, stderr, ok = run(nil, args) + if !ok { + os.Stderr.Write(stderr) + return nil, nil, os.NewError("pkg-config failed") + } + ldflags, err = splitQuoted(string(stdout)) + return +} + +// splitQuoted splits the string s around each instance of one or more consecutive +// white space characters while taking into account quotes and escaping, and +// returns an array of substrings of s or an empty list if s contains only white space. +// Single quotes and double quotes are recognized to prevent splitting within the +// quoted region, and are removed from the resulting substrings. If a quote in s +// isn't closed err will be set and r will have the unclosed argument as the +// last element. The backslash is used for escaping. +// +// For example, the following string: +// +// `a b:"c d" 'e''f' "g\""` +// +// Would be parsed as: +// +// []string{"a", "b:c d", "ef", `g"`} +// +func splitQuoted(s string) (r []string, err os.Error) { + var args []string + arg := make([]int, len(s)) + escaped := false + quoted := false + quote := 0 + i := 0 + for _, rune := range s { + switch { + case escaped: + escaped = false + case rune == '\\': + escaped = true + continue + case quote != 0: + if rune == quote { + quote = 0 + continue + } + case rune == '"' || rune == '\'': + quoted = true + quote = rune + continue + case unicode.IsSpace(rune): + if quoted || i > 0 { + quoted = false + args = append(args, string(arg[:i])) + i = 0 + } + continue + } + arg[i] = rune + i++ + } + if quoted || i > 0 { + args = append(args, string(arg[:i])) + } + if quote != 0 { + err = os.NewError("unclosed quote") + } else if escaped { + err = os.NewError("unfinished escaping") + } + return args, err +} + +var safeBytes = []byte("+-.,/0123456789=ABCDEFGHIJKLMNOPQRSTUVWXYZ_abcdefghijklmnopqrstuvwxyz") + +func safeName(s string) bool { + if s == "" { + return false + } + for i := 0; i < len(s); i++ { + if c := s[i]; c < 0x80 && bytes.IndexByte(safeBytes, c) < 0 { + return false + } + } + return true +} + +// Translate rewrites f.AST, the original Go input, to remove +// references to the imported package C, replacing them with +// references to the equivalent Go types, functions, and variables. +func (p *Package) Translate(f *File) { + for _, cref := range f.Ref { + // Convert C.ulong to C.unsigned long, etc. + cref.Name.C = cname(cref.Name.Go) + } + p.loadDefines(f) + needType := p.guessKinds(f) + if len(needType) > 0 { + p.loadDWARF(f, needType) + } + p.rewriteRef(f) +} + +// loadDefines coerces gcc into spitting out the #defines in use +// in the file f and saves relevant renamings in f.Name[name].Define. +func (p *Package) loadDefines(f *File) { + var b bytes.Buffer + b.WriteString(builtinProlog) + b.WriteString(f.Preamble) + stdout := p.gccDefines(b.Bytes()) + + for _, line := range strings.Split(stdout, "\n") { + if len(line) < 9 || line[0:7] != "#define" { + continue + } + + line = strings.TrimSpace(line[8:]) + + var key, val string + spaceIndex := strings.Index(line, " ") + tabIndex := strings.Index(line, "\t") + + if spaceIndex == -1 && tabIndex == -1 { + continue + } else if tabIndex == -1 || (spaceIndex != -1 && spaceIndex < tabIndex) { + key = line[0:spaceIndex] + val = strings.TrimSpace(line[spaceIndex:]) + } else { + key = line[0:tabIndex] + val = strings.TrimSpace(line[tabIndex:]) + } + + if n := f.Name[key]; n != nil { + if *debugDefine { + fmt.Fprintf(os.Stderr, "#define %s %s\n", key, val) + } + n.Define = val + } + } +} + +// guessKinds tricks gcc into revealing the kind of each +// name xxx for the references C.xxx in the Go input. +// The kind is either a constant, type, or variable. +func (p *Package) guessKinds(f *File) []*Name { + // Coerce gcc into telling us whether each name is + // a type, a value, or undeclared. We compile a function + // containing the line: + // name; + // If name is a type, gcc will print: + // cgo-test:2: warning: useless type name in empty declaration + // If name is a value, gcc will print + // cgo-test:2: warning: statement with no effect + // If name is undeclared, gcc will print + // cgo-test:2: error: 'name' undeclared (first use in this function) + // A line number directive causes the line number to + // correspond to the index in the names array. + // + // The line also has an enum declaration: + // name; enum { _cgo_enum_1 = name }; + // If name is not a constant, gcc will print: + // cgo-test:4: error: enumerator value for '_cgo_enum_4' is not an integer constant + // we assume lines without that error are constants. + + // Make list of names that need sniffing, type lookup. + toSniff := make([]*Name, 0, len(f.Name)) + needType := make([]*Name, 0, len(f.Name)) + + for _, n := range f.Name { + // If we've already found this name as a #define + // and we can translate it as a constant value, do so. + if n.Define != "" { + ok := false + if _, err := strconv.Atoi(n.Define); err == nil { + ok = true + } else if n.Define[0] == '"' || n.Define[0] == '\'' { + _, err := parser.ParseExpr(fset, "", n.Define) + if err == nil { + ok = true + } + } + if ok { + n.Kind = "const" + n.Const = n.Define + continue + } + + if isName(n.Define) { + n.C = n.Define + } + } + + // If this is a struct, union, or enum type name, + // record the kind but also that we need type information. + if strings.HasPrefix(n.C, "struct ") || strings.HasPrefix(n.C, "union ") || strings.HasPrefix(n.C, "enum ") { + n.Kind = "type" + i := len(needType) + needType = needType[0 : i+1] + needType[i] = n + continue + } + + i := len(toSniff) + toSniff = toSniff[0 : i+1] + toSniff[i] = n + } + + if len(toSniff) == 0 { + return needType + } + + var b bytes.Buffer + b.WriteString(builtinProlog) + b.WriteString(f.Preamble) + b.WriteString("void __cgo__f__(void) {\n") + b.WriteString("#line 0 \"cgo-test\"\n") + for i, n := range toSniff { + fmt.Fprintf(&b, "%s; enum { _cgo_enum_%d = %s }; /* cgo-test:%d */\n", n.C, i, n.C, i) + } + b.WriteString("}\n") + stderr := p.gccErrors(b.Bytes()) + if stderr == "" { + fatalf("gcc produced no output\non input:\n%s", b.Bytes()) + } + + names := make([]*Name, len(toSniff)) + copy(names, toSniff) + + isConst := make([]bool, len(toSniff)) + for i := range isConst { + isConst[i] = true // until proven otherwise + } + + for _, line := range strings.Split(stderr, "\n") { + if len(line) < 9 || line[0:9] != "cgo-test:" { + // the user will see any compiler errors when the code is compiled later. + continue + } + line = line[9:] + colon := strings.Index(line, ":") + if colon < 0 { + continue + } + i, err := strconv.Atoi(line[0:colon]) + if err != nil { + continue + } + what := "" + switch { + default: + continue + case strings.Contains(line, ": useless type name in empty declaration"): + what = "type" + isConst[i] = false + case strings.Contains(line, ": statement with no effect"): + what = "not-type" // const or func or var + case strings.Contains(line, "undeclared"): + error(token.NoPos, "%s", strings.TrimSpace(line[colon+1:])) + case strings.Contains(line, "is not an integer constant"): + isConst[i] = false + continue + } + n := toSniff[i] + if n == nil { + continue + } + toSniff[i] = nil + n.Kind = what + + j := len(needType) + needType = needType[0 : j+1] + needType[j] = n + } + for i, b := range isConst { + if b { + names[i].Kind = "const" + } + } + for _, n := range toSniff { + if n == nil { + continue + } + if n.Kind != "" { + continue + } + error(token.NoPos, "could not determine kind of name for C.%s", n.Go) + } + if nerrors > 0 { + fatalf("unresolved names") + } + return needType +} + +// loadDWARF parses the DWARF debug information generated +// by gcc to learn the details of the constants, variables, and types +// being referred to as C.xxx. +func (p *Package) loadDWARF(f *File, names []*Name) { + // Extract the types from the DWARF section of an object + // from a well-formed C program. Gcc only generates DWARF info + // for symbols in the object file, so it is not enough to print the + // preamble and hope the symbols we care about will be there. + // Instead, emit + // typeof(names[i]) *__cgo__i; + // for each entry in names and then dereference the type we + // learn for __cgo__i. + var b bytes.Buffer + b.WriteString(builtinProlog) + b.WriteString(f.Preamble) + for i, n := range names { + fmt.Fprintf(&b, "typeof(%s) *__cgo__%d;\n", n.C, i) + if n.Kind == "const" { + fmt.Fprintf(&b, "enum { __cgo_enum__%d = %s };\n", i, n.C) + } + } + + // Apple's LLVM-based gcc does not include the enumeration + // names and values in its DWARF debug output. In case we're + // using such a gcc, create a data block initialized with the values. + // We can read them out of the object file. + fmt.Fprintf(&b, "long long __cgodebug_data[] = {\n") + for _, n := range names { + if n.Kind == "const" { + fmt.Fprintf(&b, "\t%s,\n", n.C) + } else { + fmt.Fprintf(&b, "\t0,\n") + } + } + fmt.Fprintf(&b, "\t0\n") + fmt.Fprintf(&b, "};\n") + + d, bo, debugData := p.gccDebug(b.Bytes()) + enumVal := make([]int64, len(debugData)/8) + for i := range enumVal { + enumVal[i] = int64(bo.Uint64(debugData[i*8:])) + } + + // Scan DWARF info for top-level TagVariable entries with AttrName __cgo__i. + types := make([]dwarf.Type, len(names)) + enums := make([]dwarf.Offset, len(names)) + nameToIndex := make(map[*Name]int) + for i, n := range names { + nameToIndex[n] = i + } + r := d.Reader() + for { + e, err := r.Next() + if err != nil { + fatalf("reading DWARF entry: %s", err) + } + if e == nil { + break + } + switch e.Tag { + case dwarf.TagEnumerationType: + offset := e.Offset + for { + e, err := r.Next() + if err != nil { + fatalf("reading DWARF entry: %s", err) + } + if e.Tag == 0 { + break + } + if e.Tag == dwarf.TagEnumerator { + entryName := e.Val(dwarf.AttrName).(string) + if strings.HasPrefix(entryName, "__cgo_enum__") { + n, _ := strconv.Atoi(entryName[len("__cgo_enum__"):]) + if 0 <= n && n < len(names) { + enums[n] = offset + } + } + } + } + case dwarf.TagVariable: + name, _ := e.Val(dwarf.AttrName).(string) + typOff, _ := e.Val(dwarf.AttrType).(dwarf.Offset) + if name == "" || typOff == 0 { + fatalf("malformed DWARF TagVariable entry") + } + if !strings.HasPrefix(name, "__cgo__") { + break + } + typ, err := d.Type(typOff) + if err != nil { + fatalf("loading DWARF type: %s", err) + } + t, ok := typ.(*dwarf.PtrType) + if !ok || t == nil { + fatalf("internal error: %s has non-pointer type", name) + } + i, err := strconv.Atoi(name[7:]) + if err != nil { + fatalf("malformed __cgo__ name: %s", name) + } + if enums[i] != 0 { + t, err := d.Type(enums[i]) + if err != nil { + fatalf("loading DWARF type: %s", err) + } + types[i] = t + } else { + types[i] = t.Type + } + } + if e.Tag != dwarf.TagCompileUnit { + r.SkipChildren() + } + } + + // Record types and typedef information. + var conv typeConv + conv.Init(p.PtrSize) + for i, n := range names { + f, fok := types[i].(*dwarf.FuncType) + if n.Kind != "type" && fok { + n.Kind = "func" + n.FuncType = conv.FuncType(f) + } else { + n.Type = conv.Type(types[i]) + if enums[i] != 0 && n.Type.EnumValues != nil { + k := fmt.Sprintf("__cgo_enum__%d", i) + n.Kind = "const" + n.Const = strconv.Itoa64(n.Type.EnumValues[k]) + // Remove injected enum to ensure the value will deep-compare + // equally in future loads of the same constant. + n.Type.EnumValues[k] = 0, false + } else if n.Kind == "const" && i < len(enumVal) { + n.Const = strconv.Itoa64(enumVal[i]) + } + } + } + +} + +// rewriteRef rewrites all the C.xxx references in f.AST to refer to the +// Go equivalents, now that we have figured out the meaning of all +// the xxx. +func (p *Package) rewriteRef(f *File) { + // Assign mangled names. + for _, n := range f.Name { + if n.Kind == "not-type" { + n.Kind = "var" + } + if n.Mangle == "" { + n.Mangle = "_C" + n.Kind + "_" + n.Go + } + } + + // Now that we have all the name types filled in, + // scan through the Refs to identify the ones that + // are trying to do a ,err call. Also check that + // functions are only used in calls. + for _, r := range f.Ref { + if r.Name.Kind == "const" && r.Name.Const == "" { + error(r.Pos(), "unable to find value of constant C.%s", r.Name.Go) + } + var expr ast.Expr = ast.NewIdent(r.Name.Mangle) // default + switch r.Context { + case "call", "call2": + if r.Name.Kind != "func" { + if r.Name.Kind == "type" { + r.Context = "type" + expr = r.Name.Type.Go + break + } + error(r.Pos(), "call of non-function C.%s", r.Name.Go) + break + } + if r.Context == "call2" { + if r.Name.FuncType.Result == nil { + error(r.Pos(), "assignment count mismatch: 2 = 0") + } + // Invent new Name for the two-result function. + n := f.Name["2"+r.Name.Go] + if n == nil { + n = new(Name) + *n = *r.Name + n.AddError = true + n.Mangle = "_C2func_" + n.Go + f.Name["2"+r.Name.Go] = n + } + expr = ast.NewIdent(n.Mangle) + r.Name = n + break + } + case "expr": + if r.Name.Kind == "func" { + error(r.Pos(), "must call C.%s", r.Name.Go) + } + if r.Name.Kind == "type" { + // Okay - might be new(T) + expr = r.Name.Type.Go + } + if r.Name.Kind == "var" { + expr = &ast.StarExpr{X: expr} + } + + case "type": + if r.Name.Kind != "type" { + error(r.Pos(), "expression C.%s used as type", r.Name.Go) + } else { + expr = r.Name.Type.Go + } + default: + if r.Name.Kind == "func" { + error(r.Pos(), "must call C.%s", r.Name.Go) + } + } + *r.Expr = expr + } +} + +// gccName returns the name of the compiler to run. Use $GCC if set in +// the environment, otherwise just "gcc". + +func (p *Package) gccName() (ret string) { + if ret = os.Getenv("GCC"); ret == "" { + ret = "gcc" + } + return +} + +// gccMachine returns the gcc -m flag to use, either "-m32" or "-m64". +func (p *Package) gccMachine() []string { + switch runtime.GOARCH { + case "amd64": + return []string{"-m64"} + case "386": + return []string{"-m32"} + } + return nil +} + +var gccTmp = objDir + "_cgo_.o" + +// gccCmd returns the gcc command line to use for compiling +// the input. +func (p *Package) gccCmd() []string { + c := []string{ + p.gccName(), + "-Wall", // many warnings + "-Werror", // warnings are errors + "-o" + gccTmp, // write object to tmp + "-gdwarf-2", // generate DWARF v2 debugging symbols + "-fno-eliminate-unused-debug-types", // gets rid of e.g. untyped enum otherwise + "-c", // do not link + "-xc", // input language is C + } + c = append(c, p.GccOptions...) + c = append(c, p.gccMachine()...) + c = append(c, "-") //read input from standard input + return c +} + +// gccDebug runs gcc -gdwarf-2 over the C program stdin and +// returns the corresponding DWARF data and, if present, debug data block. +func (p *Package) gccDebug(stdin []byte) (*dwarf.Data, binary.ByteOrder, []byte) { + runGcc(stdin, p.gccCmd()) + + if f, err := macho.Open(gccTmp); err == nil { + d, err := f.DWARF() + if err != nil { + fatalf("cannot load DWARF output from %s: %v", gccTmp, err) + } + var data []byte + if f.Symtab != nil { + for i := range f.Symtab.Syms { + s := &f.Symtab.Syms[i] + // Mach-O still uses a leading _ to denote non-assembly symbols. + if s.Name == "_"+"__cgodebug_data" { + // Found it. Now find data section. + if i := int(s.Sect) - 1; 0 <= i && i < len(f.Sections) { + sect := f.Sections[i] + if sect.Addr <= s.Value && s.Value < sect.Addr+sect.Size { + if sdat, err := sect.Data(); err == nil { + data = sdat[s.Value-sect.Addr:] + } + } + } + } + } + } + return d, f.ByteOrder, data + } + + // Can skip debug data block in ELF and PE for now. + // The DWARF information is complete. + + if f, err := elf.Open(gccTmp); err == nil { + d, err := f.DWARF() + if err != nil { + fatalf("cannot load DWARF output from %s: %v", gccTmp, err) + } + return d, f.ByteOrder, nil + } + + if f, err := pe.Open(gccTmp); err == nil { + d, err := f.DWARF() + if err != nil { + fatalf("cannot load DWARF output from %s: %v", gccTmp, err) + } + return d, binary.LittleEndian, nil + } + + fatalf("cannot parse gcc output %s as ELF, Mach-O, PE object", gccTmp) + panic("not reached") +} + +// gccDefines runs gcc -E -dM -xc - over the C program stdin +// and returns the corresponding standard output, which is the +// #defines that gcc encountered while processing the input +// and its included files. +func (p *Package) gccDefines(stdin []byte) string { + base := []string{p.gccName(), "-E", "-dM", "-xc"} + base = append(base, p.gccMachine()...) + stdout, _ := runGcc(stdin, append(append(base, p.GccOptions...), "-")) + return stdout +} + +// gccErrors runs gcc over the C program stdin and returns +// the errors that gcc prints. That is, this function expects +// gcc to fail. +func (p *Package) gccErrors(stdin []byte) string { + // TODO(rsc): require failure + args := p.gccCmd() + if *debugGcc { + fmt.Fprintf(os.Stderr, "$ %s < 0 { + // Cannot represent bit-sized elements in Go. + t.Go = c.Opaque(t.Size) + break + } + gt := &ast.ArrayType{ + Len: c.intExpr(dt.Count), + } + t.Go = gt // publish before recursive call + sub := c.Type(dt.Type) + t.Align = sub.Align + gt.Elt = sub.Go + t.C.Set("typeof(%s[%d])", sub.C, dt.Count) + + case *dwarf.BoolType: + t.Go = c.bool + t.Align = c.ptrSize + + case *dwarf.CharType: + if t.Size != 1 { + fatalf("unexpected: %d-byte char type - %s", t.Size, dtype) + } + t.Go = c.int8 + t.Align = 1 + + case *dwarf.EnumType: + if t.Align = t.Size; t.Align >= c.ptrSize { + t.Align = c.ptrSize + } + t.C.Set("enum " + dt.EnumName) + signed := 0 + t.EnumValues = make(map[string]int64) + for _, ev := range dt.Val { + t.EnumValues[ev.Name] = ev.Val + if ev.Val < 0 { + signed = signedDelta + } + } + switch t.Size + int64(signed) { + default: + fatalf("unexpected: %d-byte enum type - %s", t.Size, dtype) + case 1: + t.Go = c.uint8 + case 2: + t.Go = c.uint16 + case 4: + t.Go = c.uint32 + case 8: + t.Go = c.uint64 + case 1 + signedDelta: + t.Go = c.int8 + case 2 + signedDelta: + t.Go = c.int16 + case 4 + signedDelta: + t.Go = c.int32 + case 8 + signedDelta: + t.Go = c.int64 + } + + case *dwarf.FloatType: + switch t.Size { + default: + fatalf("unexpected: %d-byte float type - %s", t.Size, dtype) + case 4: + t.Go = c.float32 + case 8: + t.Go = c.float64 + } + if t.Align = t.Size; t.Align >= c.ptrSize { + t.Align = c.ptrSize + } + + case *dwarf.ComplexType: + switch t.Size { + default: + fatalf("unexpected: %d-byte complex type - %s", t.Size, dtype) + case 8: + t.Go = c.complex64 + case 16: + t.Go = c.complex128 + } + if t.Align = t.Size; t.Align >= c.ptrSize { + t.Align = c.ptrSize + } + + case *dwarf.FuncType: + // No attempt at translation: would enable calls + // directly between worlds, but we need to moderate those. + t.Go = c.uintptr + t.Align = c.ptrSize + + case *dwarf.IntType: + if dt.BitSize > 0 { + fatalf("unexpected: %d-bit int type - %s", dt.BitSize, dtype) + } + switch t.Size { + default: + fatalf("unexpected: %d-byte int type - %s", t.Size, dtype) + case 1: + t.Go = c.int8 + case 2: + t.Go = c.int16 + case 4: + t.Go = c.int32 + case 8: + t.Go = c.int64 + } + if t.Align = t.Size; t.Align >= c.ptrSize { + t.Align = c.ptrSize + } + + case *dwarf.PtrType: + t.Align = c.ptrSize + + // Translate void* as unsafe.Pointer + if _, ok := base(dt.Type).(*dwarf.VoidType); ok { + t.Go = c.unsafePointer + t.C.Set("void*") + break + } + + gt := &ast.StarExpr{} + t.Go = gt // publish before recursive call + sub := c.Type(dt.Type) + gt.X = sub.Go + t.C.Set("%s*", sub.C) + + case *dwarf.QualType: + // Ignore qualifier. + t = c.Type(dt.Type) + c.m[dtype] = t + return t + + case *dwarf.StructType: + // Convert to Go struct, being careful about alignment. + // Have to give it a name to simulate C "struct foo" references. + tag := dt.StructName + if tag == "" { + tag = "__" + strconv.Itoa(tagGen) + tagGen++ + } else if t.C.Empty() { + t.C.Set(dt.Kind + " " + tag) + } + name := c.Ident("_Ctype_" + dt.Kind + "_" + tag) + t.Go = name // publish before recursive calls + switch dt.Kind { + case "union", "class": + typedef[name.Name] = c.Opaque(t.Size) + if t.C.Empty() { + t.C.Set("typeof(unsigned char[%d])", t.Size) + } + case "struct": + g, csyntax, align := c.Struct(dt) + if t.C.Empty() { + t.C.Set(csyntax) + } + t.Align = align + typedef[name.Name] = g + } + + case *dwarf.TypedefType: + // Record typedef for printing. + if dt.Name == "_GoString_" { + // Special C name for Go string type. + // Knows string layout used by compilers: pointer plus length, + // which rounds up to 2 pointers after alignment. + t.Go = c.string + t.Size = c.ptrSize * 2 + t.Align = c.ptrSize + break + } + if dt.Name == "_GoBytes_" { + // Special C name for Go []byte type. + // Knows slice layout used by compilers: pointer, length, cap. + t.Go = c.Ident("[]byte") + t.Size = c.ptrSize + 4 + 4 + t.Align = c.ptrSize + break + } + name := c.Ident("_Ctypedef_" + dt.Name) + t.Go = name // publish before recursive call + sub := c.Type(dt.Type) + t.Size = sub.Size + t.Align = sub.Align + if _, ok := typedef[name.Name]; !ok { + typedef[name.Name] = sub.Go + } + + case *dwarf.UcharType: + if t.Size != 1 { + fatalf("unexpected: %d-byte uchar type - %s", t.Size, dtype) + } + t.Go = c.uint8 + t.Align = 1 + + case *dwarf.UintType: + if dt.BitSize > 0 { + fatalf("unexpected: %d-bit uint type - %s", dt.BitSize, dtype) + } + switch t.Size { + default: + fatalf("unexpected: %d-byte uint type - %s", t.Size, dtype) + case 1: + t.Go = c.uint8 + case 2: + t.Go = c.uint16 + case 4: + t.Go = c.uint32 + case 8: + t.Go = c.uint64 + } + if t.Align = t.Size; t.Align >= c.ptrSize { + t.Align = c.ptrSize + } + + case *dwarf.VoidType: + t.Go = c.void + t.C.Set("void") + } + + switch dtype.(type) { + case *dwarf.AddrType, *dwarf.BoolType, *dwarf.CharType, *dwarf.IntType, *dwarf.FloatType, *dwarf.UcharType, *dwarf.UintType: + s := dtype.Common().Name + if s != "" { + if ss, ok := dwarfToName[s]; ok { + s = ss + } + s = strings.Join(strings.Split(s, " "), "") // strip spaces + name := c.Ident("_Ctype_" + s) + typedef[name.Name] = t.Go + t.Go = name + } + } + + if t.C.Empty() { + fatalf("internal error: did not create C name for %s", dtype) + } + + return t +} + +// FuncArg returns a Go type with the same memory layout as +// dtype when used as the type of a C function argument. +func (c *typeConv) FuncArg(dtype dwarf.Type) *Type { + t := c.Type(dtype) + switch dt := dtype.(type) { + case *dwarf.ArrayType: + // Arrays are passed implicitly as pointers in C. + // In Go, we must be explicit. + tr := &TypeRepr{} + tr.Set("%s*", t.C) + return &Type{ + Size: c.ptrSize, + Align: c.ptrSize, + Go: &ast.StarExpr{X: t.Go}, + C: tr, + } + case *dwarf.TypedefType: + // C has much more relaxed rules than Go for + // implicit type conversions. When the parameter + // is type T defined as *X, simulate a little of the + // laxness of C by making the argument *X instead of T. + if ptr, ok := base(dt.Type).(*dwarf.PtrType); ok { + // Unless the typedef happens to point to void* since + // Go has special rules around using unsafe.Pointer. + if _, void := base(ptr.Type).(*dwarf.VoidType); !void { + return c.Type(ptr) + } + } + } + return t +} + +// FuncType returns the Go type analogous to dtype. +// There is no guarantee about matching memory layout. +func (c *typeConv) FuncType(dtype *dwarf.FuncType) *FuncType { + p := make([]*Type, len(dtype.ParamType)) + gp := make([]*ast.Field, len(dtype.ParamType)) + for i, f := range dtype.ParamType { + // gcc's DWARF generator outputs a single DotDotDotType parameter for + // function pointers that specify no parameters (e.g. void + // (*__cgo_0)()). Treat this special case as void. This case is + // invalid according to ISO C anyway (i.e. void (*__cgo_1)(...) is not + // legal). + if _, ok := f.(*dwarf.DotDotDotType); ok && i == 0 { + p, gp = nil, nil + break + } + p[i] = c.FuncArg(f) + gp[i] = &ast.Field{Type: p[i].Go} + } + var r *Type + var gr []*ast.Field + if _, ok := dtype.ReturnType.(*dwarf.VoidType); !ok && dtype.ReturnType != nil { + r = c.Type(dtype.ReturnType) + gr = []*ast.Field{&ast.Field{Type: r.Go}} + } + return &FuncType{ + Params: p, + Result: r, + Go: &ast.FuncType{ + Params: &ast.FieldList{List: gp}, + Results: &ast.FieldList{List: gr}, + }, + } +} + +// Identifier +func (c *typeConv) Ident(s string) *ast.Ident { + return ast.NewIdent(s) +} + +// Opaque type of n bytes. +func (c *typeConv) Opaque(n int64) ast.Expr { + return &ast.ArrayType{ + Len: c.intExpr(n), + Elt: c.byte, + } +} + +// Expr for integer n. +func (c *typeConv) intExpr(n int64) ast.Expr { + return &ast.BasicLit{ + Kind: token.INT, + Value: strconv.Itoa64(n), + } +} + +// Add padding of given size to fld. +func (c *typeConv) pad(fld []*ast.Field, size int64) []*ast.Field { + n := len(fld) + fld = fld[0 : n+1] + fld[n] = &ast.Field{Names: []*ast.Ident{c.Ident("_")}, Type: c.Opaque(size)} + return fld +} + +// Struct conversion: return Go and (6g) C syntax for type. +func (c *typeConv) Struct(dt *dwarf.StructType) (expr *ast.StructType, csyntax string, align int64) { + var buf bytes.Buffer + buf.WriteString("struct {") + fld := make([]*ast.Field, 0, 2*len(dt.Field)+1) // enough for padding around every field + off := int64(0) + + // Rename struct fields that happen to be named Go keywords into + // _{keyword}. Create a map from C ident -> Go ident. The Go ident will + // be mangled. Any existing identifier that already has the same name on + // the C-side will cause the Go-mangled version to be prefixed with _. + // (e.g. in a struct with fields '_type' and 'type', the latter would be + // rendered as '__type' in Go). + ident := make(map[string]string) + used := make(map[string]bool) + for _, f := range dt.Field { + ident[f.Name] = f.Name + used[f.Name] = true + } + for cid, goid := range ident { + if token.Lookup([]byte(goid)).IsKeyword() { + // Avoid keyword + goid = "_" + goid + + // Also avoid existing fields + for _, exist := used[goid]; exist; _, exist = used[goid] { + goid = "_" + goid + } + + used[goid] = true + ident[cid] = goid + } + } + + for _, f := range dt.Field { + if f.BitSize > 0 && f.BitSize != f.ByteSize*8 { + continue + } + if f.ByteOffset > off { + fld = c.pad(fld, f.ByteOffset-off) + off = f.ByteOffset + } + t := c.Type(f.Type) + n := len(fld) + fld = fld[0 : n+1] + + fld[n] = &ast.Field{Names: []*ast.Ident{c.Ident(ident[f.Name])}, Type: t.Go} + off += t.Size + buf.WriteString(t.C.String()) + buf.WriteString(" ") + buf.WriteString(f.Name) + buf.WriteString("; ") + if t.Align > align { + align = t.Align + } + } + if off < dt.ByteSize { + fld = c.pad(fld, dt.ByteSize-off) + off = dt.ByteSize + } + if off != dt.ByteSize { + fatalf("struct size calculation error") + } + buf.WriteString("}") + csyntax = buf.String() + expr = &ast.StructType{Fields: &ast.FieldList{List: fld}} + return +} diff --git a/src/cmd/cgo/main.go b/src/cmd/cgo/main.go new file mode 100644 index 000000000..be9c2bc4f --- /dev/null +++ b/src/cmd/cgo/main.go @@ -0,0 +1,268 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Cgo; see gmp.go for an overview. + +// TODO(rsc): +// Emit correct line number annotations. +// Make 6g understand the annotations. + +package main + +import ( + "crypto/md5" + "flag" + "fmt" + "go/ast" + "go/token" + "io" + "os" + "path/filepath" + "reflect" + "strings" +) + +// A Package collects information about the package we're going to write. +type Package struct { + PackageName string // name of package + PackagePath string + PtrSize int64 + GccOptions []string + CgoFlags map[string]string // #cgo flags (CFLAGS, LDFLAGS) + Written map[string]bool + Name map[string]*Name // accumulated Name from Files + Typedef map[string]ast.Expr // accumulated Typedef from Files + ExpFunc []*ExpFunc // accumulated ExpFunc from Files + Decl []ast.Decl + GoFiles []string // list of Go files + GccFiles []string // list of gcc output files +} + +// A File collects information about a single Go input file. +type File struct { + AST *ast.File // parsed AST + Package string // Package name + Preamble string // C preamble (doc comment on import "C") + Ref []*Ref // all references to C.xxx in AST + ExpFunc []*ExpFunc // exported functions for this file + Name map[string]*Name // map from Go name to Name + Typedef map[string]ast.Expr // translations of all necessary types from C +} + +// A Ref refers to an expression of the form C.xxx in the AST. +type Ref struct { + Name *Name + Expr *ast.Expr + Context string // "type", "expr", "call", or "call2" +} + +func (r *Ref) Pos() token.Pos { + return (*r.Expr).Pos() +} + +// A Name collects information about C.xxx. +type Name struct { + Go string // name used in Go referring to package C + Mangle string // name used in generated Go + C string // name used in C + Define string // #define expansion + Kind string // "const", "type", "var", "func", "not-type" + Type *Type // the type of xxx + FuncType *FuncType + AddError bool + Const string // constant definition +} + +// A ExpFunc is an exported function, callable from C. +// Such functions are identified in the Go input file +// by doc comments containing the line //export ExpName +type ExpFunc struct { + Func *ast.FuncDecl + ExpName string // name to use from C +} + +// A TypeRepr contains the string representation of a type. +type TypeRepr struct { + Repr string + FormatArgs []interface{} +} + +// A Type collects information about a type in both the C and Go worlds. +type Type struct { + Size int64 + Align int64 + C *TypeRepr + Go ast.Expr + EnumValues map[string]int64 +} + +// A FuncType collects information about a function type in both the C and Go worlds. +type FuncType struct { + Params []*Type + Result *Type + Go *ast.FuncType +} + +func usage() { + fmt.Fprint(os.Stderr, "usage: cgo -- [compiler options] file.go ...\n") + flag.PrintDefaults() + os.Exit(2) +} + +var ptrSizeMap = map[string]int64{ + "386": 4, + "amd64": 8, + "arm": 4, +} + +var cPrefix string + +var fset = token.NewFileSet() + +var dynobj = flag.String("dynimport", "", "if non-empty, print dynamic import data for that file") + +func main() { + flag.Usage = usage + flag.Parse() + + if *dynobj != "" { + // cgo -dynimport is essentially a separate helper command + // built into the cgo binary. It scans a gcc-produced executable + // and dumps information about the imported symbols and the + // imported libraries. The Make.pkg rules for cgo prepare an + // appropriate executable and then use its import information + // instead of needing to make the linkers duplicate all the + // specialized knowledge gcc has about where to look for imported + // symbols and which ones to use. + dynimport(*dynobj) + return + } + + args := flag.Args() + if len(args) < 1 { + usage() + } + + // Find first arg that looks like a go file and assume everything before + // that are options to pass to gcc. + var i int + for i = len(args); i > 0; i-- { + if !strings.HasSuffix(args[i-1], ".go") { + break + } + } + if i == len(args) { + usage() + } + + // Copy it to a new slice so it can grow. + gccOptions := make([]string, i) + copy(gccOptions, args[0:i]) + + goFiles := args[i:] + + arch := os.Getenv("GOARCH") + if arch == "" { + fatalf("$GOARCH is not set") + } + ptrSize := ptrSizeMap[arch] + if ptrSize == 0 { + fatalf("unknown $GOARCH %q", arch) + } + + // Clear locale variables so gcc emits English errors [sic]. + os.Setenv("LANG", "en_US.UTF-8") + os.Setenv("LC_ALL", "C") + os.Setenv("LC_CTYPE", "C") + + p := &Package{ + PtrSize: ptrSize, + GccOptions: gccOptions, + CgoFlags: make(map[string]string), + Written: make(map[string]bool), + } + + // Need a unique prefix for the global C symbols that + // we use to coordinate between gcc and ourselves. + // We already put _cgo_ at the beginning, so the main + // concern is other cgo wrappers for the same functions. + // Use the beginning of the md5 of the input to disambiguate. + h := md5.New() + for _, input := range goFiles { + f, err := os.Open(input) + if err != nil { + fatalf("%s", err) + } + io.Copy(h, f) + f.Close() + } + cPrefix = fmt.Sprintf("_%x", h.Sum()[0:6]) + + fs := make([]*File, len(goFiles)) + for i, input := range goFiles { + // Parse flags for all files before translating due to CFLAGS. + f := new(File) + f.ReadGo(input) + p.ParseFlags(f, input) + fs[i] = f + } + + // make sure that _obj directory exists, so that we can write + // all the output files there. + os.Mkdir("_obj", 0777) + + for i, input := range goFiles { + f := fs[i] + p.Translate(f) + for _, cref := range f.Ref { + switch cref.Context { + case "call", "call2": + if cref.Name.Kind != "type" { + break + } + *cref.Expr = cref.Name.Type.Go + } + } + if nerrors > 0 { + os.Exit(2) + } + pkg := f.Package + if dir := os.Getenv("CGOPKGPATH"); dir != "" { + pkg = filepath.Join(dir, pkg) + } + p.PackagePath = pkg + p.writeOutput(f, input) + + p.Record(f) + } + + p.writeDefs() + if nerrors > 0 { + os.Exit(2) + } +} + +// Record what needs to be recorded about f. +func (p *Package) Record(f *File) { + if p.PackageName == "" { + p.PackageName = f.Package + } else if p.PackageName != f.Package { + error(token.NoPos, "inconsistent package names: %s, %s", p.PackageName, f.Package) + } + + if p.Name == nil { + p.Name = f.Name + } else { + for k, v := range f.Name { + if p.Name[k] == nil { + p.Name[k] = v + } else if !reflect.DeepEqual(p.Name[k], v) { + error(token.NoPos, "inconsistent definitions for C.%s", k) + } + } + } + + p.ExpFunc = append(p.ExpFunc, f.ExpFunc...) + p.Decl = append(p.Decl, f.AST.Decls...) +} diff --git a/src/cmd/cgo/out.go b/src/cmd/cgo/out.go new file mode 100644 index 000000000..498ab1566 --- /dev/null +++ b/src/cmd/cgo/out.go @@ -0,0 +1,745 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package main + +import ( + "bytes" + "debug/elf" + "debug/macho" + "debug/pe" + "fmt" + "go/ast" + "go/printer" + "go/token" + "os" + "path/filepath" + "strings" +) + +var objDir = "_obj" + string(filepath.Separator) + +// writeDefs creates output files to be compiled by 6g, 6c, and gcc. +// (The comments here say 6g and 6c but the code applies to the 8 and 5 tools too.) +func (p *Package) writeDefs() { + fgo2 := creat(objDir + "_cgo_gotypes.go") + fc := creat(objDir + "_cgo_defun.c") + fm := creat(objDir + "_cgo_main.c") + + fflg := creat(objDir + "_cgo_flags") + for k, v := range p.CgoFlags { + fmt.Fprintf(fflg, "_CGO_%s=%s\n", k, v) + } + fflg.Close() + + // Write C main file for using gcc to resolve imports. + fmt.Fprintf(fm, "int main() { return 0; }\n") + fmt.Fprintf(fm, "void crosscall2(void(*fn)(void*, int), void *a, int c) { }\n") + fmt.Fprintf(fm, "void _cgo_allocate(void *a, int c) { }\n") + fmt.Fprintf(fm, "void _cgo_panic(void *a, int c) { }\n") + + // Write second Go output: definitions of _C_xxx. + // In a separate file so that the import of "unsafe" does not + // pollute the original file. + fmt.Fprintf(fgo2, "// Created by cgo - DO NOT EDIT\n\n") + fmt.Fprintf(fgo2, "package %s\n\n", p.PackageName) + fmt.Fprintf(fgo2, "import \"unsafe\"\n\n") + fmt.Fprintf(fgo2, "import \"os\"\n\n") + fmt.Fprintf(fgo2, "import _ \"runtime/cgo\"\n\n") + fmt.Fprintf(fgo2, "type _ unsafe.Pointer\n\n") + fmt.Fprintf(fgo2, "func _Cerrno(dst *os.Error, x int) { *dst = os.Errno(x) }\n") + + for name, def := range typedef { + fmt.Fprintf(fgo2, "type %s ", name) + printer.Fprint(fgo2, fset, def) + fmt.Fprintf(fgo2, "\n") + } + fmt.Fprintf(fgo2, "type _Ctype_void [0]byte\n") + + fmt.Fprintf(fc, cProlog) + + cVars := make(map[string]bool) + for _, n := range p.Name { + if n.Kind != "var" { + continue + } + + if !cVars[n.C] { + fmt.Fprintf(fm, "extern char %s[];\n", n.C) + fmt.Fprintf(fm, "void *_cgohack_%s = %s;\n\n", n.C, n.C) + + fmt.Fprintf(fc, "extern byte *%s;\n", n.C) + + cVars[n.C] = true + } + + fmt.Fprintf(fc, "void *·%s = &%s;\n", n.Mangle, n.C) + fmt.Fprintf(fc, "\n") + + fmt.Fprintf(fgo2, "var %s ", n.Mangle) + printer.Fprint(fgo2, fset, &ast.StarExpr{X: n.Type.Go}) + fmt.Fprintf(fgo2, "\n") + } + fmt.Fprintf(fc, "\n") + + for _, n := range p.Name { + if n.Const != "" { + fmt.Fprintf(fgo2, "const _Cconst_%s = %s\n", n.Go, n.Const) + } + } + fmt.Fprintf(fgo2, "\n") + + for _, n := range p.Name { + if n.FuncType != nil { + p.writeDefsFunc(fc, fgo2, n) + } + } + + p.writeExports(fgo2, fc, fm) + + fgo2.Close() + fc.Close() +} + +func dynimport(obj string) { + if f, err := elf.Open(obj); err == nil { + sym, err := f.ImportedSymbols() + if err != nil { + fatalf("cannot load imported symbols from ELF file %s: %v", obj, err) + } + for _, s := range sym { + targ := s.Name + if s.Version != "" { + targ += "@" + s.Version + } + fmt.Printf("#pragma dynimport %s %s %q\n", s.Name, targ, s.Library) + } + lib, err := f.ImportedLibraries() + if err != nil { + fatalf("cannot load imported libraries from ELF file %s: %v", obj, err) + } + for _, l := range lib { + fmt.Printf("#pragma dynimport _ _ %q\n", l) + } + return + } + + if f, err := macho.Open(obj); err == nil { + sym, err := f.ImportedSymbols() + if err != nil { + fatalf("cannot load imported symbols from Mach-O file %s: %v", obj, err) + } + for _, s := range sym { + if len(s) > 0 && s[0] == '_' { + s = s[1:] + } + fmt.Printf("#pragma dynimport %s %s %q\n", s, s, "") + } + lib, err := f.ImportedLibraries() + if err != nil { + fatalf("cannot load imported libraries from Mach-O file %s: %v", obj, err) + } + for _, l := range lib { + fmt.Printf("#pragma dynimport _ _ %q\n", l) + } + return + } + + if f, err := pe.Open(obj); err == nil { + sym, err := f.ImportedSymbols() + if err != nil { + fatalf("cannot load imported symbols from PE file %s: %v", obj, err) + } + for _, s := range sym { + ss := strings.Split(s, ":") + fmt.Printf("#pragma dynimport %s %s %q\n", ss[0], ss[0], strings.ToLower(ss[1])) + } + return + } + + fatalf("cannot parse %s as ELF, Mach-O or PE", obj) +} + +// Construct a gcc struct matching the 6c argument frame. +// Assumes that in gcc, char is 1 byte, short 2 bytes, int 4 bytes, long long 8 bytes. +// These assumptions are checked by the gccProlog. +// Also assumes that 6c convention is to word-align the +// input and output parameters. +func (p *Package) structType(n *Name) (string, int64) { + var buf bytes.Buffer + fmt.Fprint(&buf, "struct {\n") + off := int64(0) + for i, t := range n.FuncType.Params { + if off%t.Align != 0 { + pad := t.Align - off%t.Align + fmt.Fprintf(&buf, "\t\tchar __pad%d[%d];\n", off, pad) + off += pad + } + fmt.Fprintf(&buf, "\t\t%s p%d;\n", t.C, i) + off += t.Size + } + if off%p.PtrSize != 0 { + pad := p.PtrSize - off%p.PtrSize + fmt.Fprintf(&buf, "\t\tchar __pad%d[%d];\n", off, pad) + off += pad + } + if t := n.FuncType.Result; t != nil { + if off%t.Align != 0 { + pad := t.Align - off%t.Align + fmt.Fprintf(&buf, "\t\tchar __pad%d[%d];\n", off, pad) + off += pad + } + qual := "" + if c := t.C.String(); c[len(c)-1] == '*' { + qual = "const " + } + fmt.Fprintf(&buf, "\t\t%s%s r;\n", qual, t.C) + off += t.Size + } + if off%p.PtrSize != 0 { + pad := p.PtrSize - off%p.PtrSize + fmt.Fprintf(&buf, "\t\tchar __pad%d[%d];\n", off, pad) + off += pad + } + if n.AddError { + fmt.Fprint(&buf, "\t\tvoid *e[2]; /* os.Error */\n") + off += 2 * p.PtrSize + } + if off == 0 { + fmt.Fprintf(&buf, "\t\tchar unused;\n") // avoid empty struct + } + fmt.Fprintf(&buf, "\t}") + return buf.String(), off +} + +func (p *Package) writeDefsFunc(fc, fgo2 *os.File, n *Name) { + name := n.Go + gtype := n.FuncType.Go + if n.AddError { + // Add "os.Error" to return type list. + // Type list is known to be 0 or 1 element - it's a C function. + err := &ast.Field{Type: ast.NewIdent("os.Error")} + l := gtype.Results.List + if len(l) == 0 { + l = []*ast.Field{err} + } else { + l = []*ast.Field{l[0], err} + } + t := new(ast.FuncType) + *t = *gtype + t.Results = &ast.FieldList{List: l} + gtype = t + } + + // Go func declaration. + d := &ast.FuncDecl{ + Name: ast.NewIdent(n.Mangle), + Type: gtype, + } + printer.Fprint(fgo2, fset, d) + fmt.Fprintf(fgo2, "\n") + + if name == "CString" || name == "GoString" || name == "GoStringN" || name == "GoBytes" { + // The builtins are already defined in the C prolog. + return + } + + var argSize int64 + _, argSize = p.structType(n) + + // C wrapper calls into gcc, passing a pointer to the argument frame. + fmt.Fprintf(fc, "void _cgo%s%s(void*);\n", cPrefix, n.Mangle) + fmt.Fprintf(fc, "\n") + fmt.Fprintf(fc, "void\n") + if argSize == 0 { + argSize++ + } + fmt.Fprintf(fc, "·%s(struct{uint8 x[%d];}p)\n", n.Mangle, argSize) + fmt.Fprintf(fc, "{\n") + fmt.Fprintf(fc, "\truntime·cgocall(_cgo%s%s, &p);\n", cPrefix, n.Mangle) + if n.AddError { + // gcc leaves errno in first word of interface at end of p. + // check whether it is zero; if so, turn interface into nil. + // if not, turn interface into errno. + // Go init function initializes ·_Cerrno with an os.Errno + // for us to copy. + fmt.Fprintln(fc, ` { + int32 e; + void **v; + v = (void**)(&p+1) - 2; /* v = final two void* of p */ + e = *(int32*)v; + v[0] = (void*)0xdeadbeef; + v[1] = (void*)0xdeadbeef; + if(e == 0) { + /* nil interface */ + v[0] = 0; + v[1] = 0; + } else { + ·_Cerrno(v, e); /* fill in v as os.Error for errno e */ + } + }`) + } + fmt.Fprintf(fc, "}\n") + fmt.Fprintf(fc, "\n") +} + +// writeOutput creates stubs for a specific source file to be compiled by 6g +// (The comments here say 6g and 6c but the code applies to the 8 and 5 tools too.) +func (p *Package) writeOutput(f *File, srcfile string) { + base := srcfile + if strings.HasSuffix(base, ".go") { + base = base[0 : len(base)-3] + } + base = strings.Map(slashToUnderscore, base) + fgo1 := creat(objDir + base + ".cgo1.go") + fgcc := creat(objDir + base + ".cgo2.c") + + p.GoFiles = append(p.GoFiles, base+".cgo1.go") + p.GccFiles = append(p.GccFiles, base+".cgo2.c") + + // Write Go output: Go input with rewrites of C.xxx to _C_xxx. + fmt.Fprintf(fgo1, "// Created by cgo - DO NOT EDIT\n\n") + fmt.Fprintf(fgo1, "//line %s:1\n", srcfile) + printer.Fprint(fgo1, fset, f.AST) + + // While we process the vars and funcs, also write 6c and gcc output. + // Gcc output starts with the preamble. + fmt.Fprintf(fgcc, "%s\n", f.Preamble) + fmt.Fprintf(fgcc, "%s\n", gccProlog) + + for _, n := range f.Name { + if n.FuncType != nil { + p.writeOutputFunc(fgcc, n) + } + } + + fgo1.Close() + fgcc.Close() +} + +func (p *Package) writeOutputFunc(fgcc *os.File, n *Name) { + name := n.Mangle + if name == "_Cfunc_CString" || name == "_Cfunc_GoString" || name == "_Cfunc_GoStringN" || name == "_Cfunc_GoBytes" || p.Written[name] { + // The builtins are already defined in the C prolog, and we don't + // want to duplicate function definitions we've already done. + return + } + p.Written[name] = true + + ctype, _ := p.structType(n) + + // Gcc wrapper unpacks the C argument struct + // and calls the actual C function. + fmt.Fprintf(fgcc, "void\n") + fmt.Fprintf(fgcc, "_cgo%s%s(void *v)\n", cPrefix, n.Mangle) + fmt.Fprintf(fgcc, "{\n") + if n.AddError { + fmt.Fprintf(fgcc, "\tint e;\n") // assuming 32 bit (see comment above structType) + fmt.Fprintf(fgcc, "\terrno = 0;\n") + } + // We're trying to write a gcc struct that matches 6c/8c/5c's layout. + // Use packed attribute to force no padding in this struct in case + // gcc has different packing requirements. For example, + // on 386 Windows, gcc wants to 8-align int64s, but 8c does not. + fmt.Fprintf(fgcc, "\t%s __attribute__((__packed__)) *a = v;\n", ctype) + fmt.Fprintf(fgcc, "\t") + if t := n.FuncType.Result; t != nil { + fmt.Fprintf(fgcc, "a->r = ") + if c := t.C.String(); c[len(c)-1] == '*' { + fmt.Fprintf(fgcc, "(const %s) ", t.C) + } + } + fmt.Fprintf(fgcc, "%s(", n.C) + for i := range n.FuncType.Params { + if i > 0 { + fmt.Fprintf(fgcc, ", ") + } + fmt.Fprintf(fgcc, "a->p%d", i) + } + fmt.Fprintf(fgcc, ");\n") + if n.AddError { + fmt.Fprintf(fgcc, "\t*(int*)(a->e) = errno;\n") + } + fmt.Fprintf(fgcc, "}\n") + fmt.Fprintf(fgcc, "\n") +} + +// Write out the various stubs we need to support functions exported +// from Go so that they are callable from C. +func (p *Package) writeExports(fgo2, fc, fm *os.File) { + fgcc := creat(objDir + "_cgo_export.c") + fgcch := creat("_cgo_export.h") + + fmt.Fprintf(fgcch, "/* Created by cgo - DO NOT EDIT. */\n") + fmt.Fprintf(fgcch, "%s\n", gccExportHeaderProlog) + + fmt.Fprintf(fgcc, "/* Created by cgo - DO NOT EDIT. */\n") + fmt.Fprintf(fgcc, "#include \"_cgo_export.h\"\n") + + for _, exp := range p.ExpFunc { + fn := exp.Func + + // Construct a gcc struct matching the 6c argument and + // result frame. The gcc struct will be compiled with + // __attribute__((packed)) so all padding must be accounted + // for explicitly. + ctype := "struct {\n" + off := int64(0) + npad := 0 + if fn.Recv != nil { + t := p.cgoType(fn.Recv.List[0].Type) + ctype += fmt.Sprintf("\t\t%s recv;\n", t.C) + off += t.Size + } + fntype := fn.Type + forFieldList(fntype.Params, + func(i int, atype ast.Expr) { + t := p.cgoType(atype) + if off%t.Align != 0 { + pad := t.Align - off%t.Align + ctype += fmt.Sprintf("\t\tchar __pad%d[%d];\n", npad, pad) + off += pad + npad++ + } + ctype += fmt.Sprintf("\t\t%s p%d;\n", t.C, i) + off += t.Size + }) + if off%p.PtrSize != 0 { + pad := p.PtrSize - off%p.PtrSize + ctype += fmt.Sprintf("\t\tchar __pad%d[%d];\n", npad, pad) + off += pad + npad++ + } + forFieldList(fntype.Results, + func(i int, atype ast.Expr) { + t := p.cgoType(atype) + if off%t.Align != 0 { + pad := t.Align - off%t.Align + ctype += fmt.Sprintf("\t\tchar __pad%d[%d];\n", npad, pad) + off += pad + npad++ + } + ctype += fmt.Sprintf("\t\t%s r%d;\n", t.C, i) + off += t.Size + }) + if off%p.PtrSize != 0 { + pad := p.PtrSize - off%p.PtrSize + ctype += fmt.Sprintf("\t\tchar __pad%d[%d];\n", npad, pad) + off += pad + npad++ + } + if ctype == "struct {\n" { + ctype += "\t\tchar unused;\n" // avoid empty struct + } + ctype += "\t}" + + // Get the return type of the wrapper function + // compiled by gcc. + gccResult := "" + if fntype.Results == nil || len(fntype.Results.List) == 0 { + gccResult = "void" + } else if len(fntype.Results.List) == 1 && len(fntype.Results.List[0].Names) <= 1 { + gccResult = p.cgoType(fntype.Results.List[0].Type).C.String() + } else { + fmt.Fprintf(fgcch, "\n/* Return type for %s */\n", exp.ExpName) + fmt.Fprintf(fgcch, "struct %s_return {\n", exp.ExpName) + forFieldList(fntype.Results, + func(i int, atype ast.Expr) { + fmt.Fprintf(fgcch, "\t%s r%d;\n", p.cgoType(atype).C, i) + }) + fmt.Fprintf(fgcch, "};\n") + gccResult = "struct " + exp.ExpName + "_return" + } + + // Build the wrapper function compiled by gcc. + s := fmt.Sprintf("%s %s(", gccResult, exp.ExpName) + if fn.Recv != nil { + s += p.cgoType(fn.Recv.List[0].Type).C.String() + s += " recv" + } + forFieldList(fntype.Params, + func(i int, atype ast.Expr) { + if i > 0 || fn.Recv != nil { + s += ", " + } + s += fmt.Sprintf("%s p%d", p.cgoType(atype).C, i) + }) + s += ")" + fmt.Fprintf(fgcch, "\nextern %s;\n", s) + + fmt.Fprintf(fgcc, "extern _cgoexp%s_%s(void *, int);\n", cPrefix, exp.ExpName) + fmt.Fprintf(fgcc, "\n%s\n", s) + fmt.Fprintf(fgcc, "{\n") + fmt.Fprintf(fgcc, "\t%s __attribute__((packed)) a;\n", ctype) + if gccResult != "void" && (len(fntype.Results.List) > 1 || len(fntype.Results.List[0].Names) > 1) { + fmt.Fprintf(fgcc, "\t%s r;\n", gccResult) + } + if fn.Recv != nil { + fmt.Fprintf(fgcc, "\ta.recv = recv;\n") + } + forFieldList(fntype.Params, + func(i int, atype ast.Expr) { + fmt.Fprintf(fgcc, "\ta.p%d = p%d;\n", i, i) + }) + fmt.Fprintf(fgcc, "\tcrosscall2(_cgoexp%s_%s, &a, %d);\n", cPrefix, exp.ExpName, off) + if gccResult != "void" { + if len(fntype.Results.List) == 1 && len(fntype.Results.List[0].Names) <= 1 { + fmt.Fprintf(fgcc, "\treturn a.r0;\n") + } else { + forFieldList(fntype.Results, + func(i int, atype ast.Expr) { + fmt.Fprintf(fgcc, "\tr.r%d = a.r%d;\n", i, i) + }) + fmt.Fprintf(fgcc, "\treturn r;\n") + } + } + fmt.Fprintf(fgcc, "}\n") + + // Build the wrapper function compiled by 6c/8c + goname := exp.Func.Name.Name + if fn.Recv != nil { + goname = "_cgoexpwrap" + cPrefix + "_" + fn.Recv.List[0].Names[0].Name + "_" + goname + } + fmt.Fprintf(fc, "extern void ·%s();\n", goname) + fmt.Fprintf(fc, "\nvoid\n") + fmt.Fprintf(fc, "_cgoexp%s_%s(void *a, int32 n)\n", cPrefix, exp.ExpName) + fmt.Fprintf(fc, "{\n") + fmt.Fprintf(fc, "\truntime·cgocallback(·%s, a, n);\n", goname) + fmt.Fprintf(fc, "}\n") + + fmt.Fprintf(fm, "int _cgoexp%s_%s;\n", cPrefix, exp.ExpName) + + // Calling a function with a receiver from C requires + // a Go wrapper function. + if fn.Recv != nil { + fmt.Fprintf(fgo2, "func %s(recv ", goname) + printer.Fprint(fgo2, fset, fn.Recv.List[0].Type) + forFieldList(fntype.Params, + func(i int, atype ast.Expr) { + fmt.Fprintf(fgo2, ", p%d ", i) + printer.Fprint(fgo2, fset, atype) + }) + fmt.Fprintf(fgo2, ")") + if gccResult != "void" { + fmt.Fprint(fgo2, " (") + forFieldList(fntype.Results, + func(i int, atype ast.Expr) { + if i > 0 { + fmt.Fprint(fgo2, ", ") + } + printer.Fprint(fgo2, fset, atype) + }) + fmt.Fprint(fgo2, ")") + } + fmt.Fprint(fgo2, " {\n") + fmt.Fprint(fgo2, "\t") + if gccResult != "void" { + fmt.Fprint(fgo2, "return ") + } + fmt.Fprintf(fgo2, "recv.%s(", exp.Func.Name) + forFieldList(fntype.Params, + func(i int, atype ast.Expr) { + if i > 0 { + fmt.Fprint(fgo2, ", ") + } + fmt.Fprintf(fgo2, "p%d", i) + }) + fmt.Fprint(fgo2, ")\n") + fmt.Fprint(fgo2, "}\n") + } + } +} + +// Call a function for each entry in an ast.FieldList, passing the +// index into the list and the type. +func forFieldList(fl *ast.FieldList, fn func(int, ast.Expr)) { + if fl == nil { + return + } + i := 0 + for _, r := range fl.List { + if r.Names == nil { + fn(i, r.Type) + i++ + } else { + for _ = range r.Names { + fn(i, r.Type) + i++ + } + } + } +} + +func c(repr string, args ...interface{}) *TypeRepr { + return &TypeRepr{repr, args} +} + +// Map predeclared Go types to Type. +var goTypes = map[string]*Type{ + "int": &Type{Size: 4, Align: 4, C: c("int")}, + "uint": &Type{Size: 4, Align: 4, C: c("uint")}, + "int8": &Type{Size: 1, Align: 1, C: c("schar")}, + "uint8": &Type{Size: 1, Align: 1, C: c("uchar")}, + "int16": &Type{Size: 2, Align: 2, C: c("short")}, + "uint16": &Type{Size: 2, Align: 2, C: c("ushort")}, + "int32": &Type{Size: 4, Align: 4, C: c("int")}, + "uint32": &Type{Size: 4, Align: 4, C: c("uint")}, + "int64": &Type{Size: 8, Align: 8, C: c("int64")}, + "uint64": &Type{Size: 8, Align: 8, C: c("uint64")}, + "float": &Type{Size: 4, Align: 4, C: c("float")}, + "float32": &Type{Size: 4, Align: 4, C: c("float")}, + "float64": &Type{Size: 8, Align: 8, C: c("double")}, + "complex": &Type{Size: 8, Align: 8, C: c("__complex float")}, + "complex64": &Type{Size: 8, Align: 8, C: c("__complex float")}, + "complex128": &Type{Size: 16, Align: 16, C: c("__complex double")}, +} + +// Map an ast type to a Type. +func (p *Package) cgoType(e ast.Expr) *Type { + switch t := e.(type) { + case *ast.StarExpr: + x := p.cgoType(t.X) + return &Type{Size: p.PtrSize, Align: p.PtrSize, C: c("%s*", x.C)} + case *ast.ArrayType: + if t.Len == nil { + return &Type{Size: p.PtrSize + 8, Align: p.PtrSize, C: c("GoSlice")} + } + case *ast.StructType: + // TODO + case *ast.FuncType: + return &Type{Size: p.PtrSize, Align: p.PtrSize, C: c("void*")} + case *ast.InterfaceType: + return &Type{Size: 3 * p.PtrSize, Align: p.PtrSize, C: c("GoInterface")} + case *ast.MapType: + return &Type{Size: p.PtrSize, Align: p.PtrSize, C: c("GoMap")} + case *ast.ChanType: + return &Type{Size: p.PtrSize, Align: p.PtrSize, C: c("GoChan")} + case *ast.Ident: + // Look up the type in the top level declarations. + // TODO: Handle types defined within a function. + for _, d := range p.Decl { + gd, ok := d.(*ast.GenDecl) + if !ok || gd.Tok != token.TYPE { + continue + } + for _, spec := range gd.Specs { + ts, ok := spec.(*ast.TypeSpec) + if !ok { + continue + } + if ts.Name.Name == t.Name { + return p.cgoType(ts.Type) + } + } + } + for name, def := range typedef { + if name == t.Name { + return p.cgoType(def) + } + } + if t.Name == "uintptr" { + return &Type{Size: p.PtrSize, Align: p.PtrSize, C: c("uintptr")} + } + if t.Name == "string" { + return &Type{Size: p.PtrSize + 4, Align: p.PtrSize, C: c("GoString")} + } + if r, ok := goTypes[t.Name]; ok { + if r.Align > p.PtrSize { + r.Align = p.PtrSize + } + return r + } + error(e.Pos(), "unrecognized Go type %s", t.Name) + return &Type{Size: 4, Align: 4, C: c("int")} + case *ast.SelectorExpr: + id, ok := t.X.(*ast.Ident) + if ok && id.Name == "unsafe" && t.Sel.Name == "Pointer" { + return &Type{Size: p.PtrSize, Align: p.PtrSize, C: c("void*")} + } + } + error(e.Pos(), "unrecognized Go type %T", e) + return &Type{Size: 4, Align: 4, C: c("int")} +} + +const gccProlog = ` +// Usual nonsense: if x and y are not equal, the type will be invalid +// (have a negative array count) and an inscrutable error will come +// out of the compiler and hopefully mention "name". +#define __cgo_compile_assert_eq(x, y, name) typedef char name[(x-y)*(x-y)*-2+1]; + +// Check at compile time that the sizes we use match our expectations. +#define __cgo_size_assert(t, n) __cgo_compile_assert_eq(sizeof(t), n, _cgo_sizeof_##t##_is_not_##n) + +__cgo_size_assert(char, 1) +__cgo_size_assert(short, 2) +__cgo_size_assert(int, 4) +typedef long long __cgo_long_long; +__cgo_size_assert(__cgo_long_long, 8) +__cgo_size_assert(float, 4) +__cgo_size_assert(double, 8) + +#include +#include +` + +const builtinProlog = ` +typedef struct { char *p; int n; } _GoString_; +typedef struct { char *p; int n; int c; } _GoBytes_; +_GoString_ GoString(char *p); +_GoString_ GoStringN(char *p, int l); +_GoBytes_ GoBytes(void *p, int n); +char *CString(_GoString_); +` + +const cProlog = ` +#include "runtime.h" +#include "cgocall.h" + +void ·_Cerrno(void*, int32); + +void +·_Cfunc_GoString(int8 *p, String s) +{ + s = runtime·gostring((byte*)p); + FLUSH(&s); +} + +void +·_Cfunc_GoStringN(int8 *p, int32 l, String s) +{ + s = runtime·gostringn((byte*)p, l); + FLUSH(&s); +} + +void +·_Cfunc_GoBytes(int8 *p, int32 l, Slice s) +{ + s = runtime·gobytes((byte*)p, l); + FLUSH(&s); +} + +void +·_Cfunc_CString(String s, int8 *p) +{ + p = runtime·cmalloc(s.len+1); + runtime·memmove((byte*)p, s.str, s.len); + p[s.len] = 0; + FLUSH(&p); +} +` + +const gccExportHeaderProlog = ` +typedef unsigned int uint; +typedef signed char schar; +typedef unsigned char uchar; +typedef unsigned short ushort; +typedef long long int64; +typedef unsigned long long uint64; +typedef __SIZE_TYPE__ uintptr; + +typedef struct { char *p; int n; } GoString; +typedef void *GoMap; +typedef void *GoChan; +typedef struct { void *t; void *v; } GoInterface; +` diff --git a/src/cmd/cgo/util.go b/src/cmd/cgo/util.go new file mode 100644 index 000000000..e79b0e1bf --- /dev/null +++ b/src/cmd/cgo/util.go @@ -0,0 +1,110 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package main + +import ( + "exec" + "fmt" + "go/token" + "io/ioutil" + "os" +) + +// run runs the command argv, feeding in stdin on standard input. +// It returns the output to standard output and standard error. +// ok indicates whether the command exited successfully. +func run(stdin []byte, argv []string) (stdout, stderr []byte, ok bool) { + cmd, err := exec.LookPath(argv[0]) + if err != nil { + fatalf("exec %s: %s", argv[0], err) + } + r0, w0, err := os.Pipe() + if err != nil { + fatalf("%s", err) + } + r1, w1, err := os.Pipe() + if err != nil { + fatalf("%s", err) + } + r2, w2, err := os.Pipe() + if err != nil { + fatalf("%s", err) + } + p, err := os.StartProcess(cmd, argv, &os.ProcAttr{Files: []*os.File{r0, w1, w2}}) + if err != nil { + fatalf("%s", err) + } + defer p.Release() + r0.Close() + w1.Close() + w2.Close() + c := make(chan bool) + go func() { + w0.Write(stdin) + w0.Close() + c <- true + }() + go func() { + stdout, _ = ioutil.ReadAll(r1) + r1.Close() + c <- true + }() + stderr, _ = ioutil.ReadAll(r2) + r2.Close() + <-c + <-c + + w, err := p.Wait(0) + if err != nil { + fatalf("%s", err) + } + ok = w.Exited() && w.ExitStatus() == 0 + return +} + +// Die with an error message. +func fatalf(msg string, args ...interface{}) { + fmt.Fprintf(os.Stderr, msg+"\n", args...) + os.Exit(2) +} + +var nerrors int + +func error(pos token.Pos, msg string, args ...interface{}) { + nerrors++ + if pos.IsValid() { + fmt.Fprintf(os.Stderr, "%s: ", fset.Position(pos).String()) + } + fmt.Fprintf(os.Stderr, msg, args...) + fmt.Fprintf(os.Stderr, "\n") +} + +// isName returns true if s is a valid C identifier +func isName(s string) bool { + for i, v := range s { + if v != '_' && (v < 'A' || v > 'Z') && (v < 'a' || v > 'z') && (v < '0' || v > '9') { + return false + } + if i == 0 && '0' <= v && v <= '9' { + return false + } + } + return s != "" +} + +func creat(name string) *os.File { + f, err := os.Create(name) + if err != nil { + fatalf("%s", err) + } + return f +} + +func slashToUnderscore(c int) int { + if c == '/' || c == '\\' || c == ':' { + c = '_' + } + return c +} diff --git a/src/cmd/cov/Makefile b/src/cmd/cov/Makefile new file mode 100644 index 000000000..95dba9c60 --- /dev/null +++ b/src/cmd/cov/Makefile @@ -0,0 +1,39 @@ +# Copyright 2009 The Go Authors. All rights reserved. +# Use of this source code is governed by a BSD-style +# license that can be found in the LICENSE file. + +include ../../Make.inc +O:=$(HOST_O) + +# The directory is cov because the source is portable and general. +# We call the binary 6cov to avoid confusion and because this binary +# is linked only with amd64 and x86 support. + +TARG=6cov +OFILES=\ + main.$O\ + tree.$O\ + +HFILES=\ + tree.h\ + +NOINSTALL=1 +include ../../Make.ccmd + +ifeq ($(GOOS),windows) +NAME=windows +else +NAME=$(shell uname | tr A-Z a-z) +endif + +install: install-$(NAME) +install-linux: install-default +install-freebsd: install-default +install-windows: install-default + +# on Darwin, have to install and setgid; see $GOROOT/src/sudo.bash +install-darwin: $(TARG) + @true + +install-default: $(TARG) + cp $(TARG) "$(GOBIN)"/$(TARG) diff --git a/src/cmd/cov/doc.go b/src/cmd/cov/doc.go new file mode 100644 index 000000000..5de00e19c --- /dev/null +++ b/src/cmd/cov/doc.go @@ -0,0 +1,33 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +/* + +Cov is a rudimentary code coverage tool. + +Given a command to run, it runs the command while tracking which +sections of code have been executed. When the command finishes, +cov prints the line numbers of sections of code in the binary that +were not executed. With no arguments it assumes the command "6.out". + +Usage: cov [-lsv] [-g substring] [-m minlines] [6.out args] + +The options are: + + -l + print full path names instead of paths relative to the current directory + -s + show the source code that didn't execute, in addition to the line numbers. + -v + print debugging information during the run. + -g substring + restrict the coverage analysis to functions or files whose names contain substring + -m minlines + only report uncovered sections of code larger than minlines lines + +For reasons of disambiguation it is installed as 6cov although it also serves +as an 8cov and a 5cov. + +*/ +package documentation diff --git a/src/cmd/cov/main.c b/src/cmd/cov/main.c new file mode 100644 index 000000000..5ff22c00a --- /dev/null +++ b/src/cmd/cov/main.c @@ -0,0 +1,480 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +/* + * code coverage + */ + +#include +#include +#include +#include +#include +#include "tree.h" + +#include +#include +typedef struct Ureg Ureg; + +void +usage(void) +{ + fprint(2, "usage: cov [-lsv] [-g substring] [-m minlines] [6.out args...]\n"); + fprint(2, "-g specifies pattern of interesting functions or files\n"); + exits("usage"); +} + +typedef struct Range Range; +struct Range +{ + uvlong pc; + uvlong epc; +}; + +int chatty; +int fd; +int longnames; +int pid; +int doshowsrc; +Map *mem; +Map *text; +Fhdr fhdr; +char *substring; +char cwd[1000]; +int ncwd; +int minlines = -1000; + +Tree breakpoints; // code ranges not run + +/* + * comparison for Range structures + * they are "equal" if they overlap, so + * that a search for [pc, pc+1) finds the + * Range containing pc. + */ +int +rangecmp(void *va, void *vb) +{ + Range *a = va, *b = vb; + if(a->epc <= b->pc) + return 1; + if(b->epc <= a->pc) + return -1; + return 0; +} + +/* + * remember that we ran the section of code [pc, epc). + */ +void +ran(uvlong pc, uvlong epc) +{ + Range key; + Range *r; + uvlong oldepc; + + if(chatty) + print("run %#llux-%#llux\n", pc, epc); + + key.pc = pc; + key.epc = pc+1; + r = treeget(&breakpoints, &key); + if(r == nil) + sysfatal("unchecked breakpoint at %#llux+%d", pc, (int)(epc-pc)); + + // Might be that the tail of the sequence + // was run already, so r->epc is before the end. + // Adjust len. + if(epc > r->epc) + epc = r->epc; + + if(r->pc == pc) { + r->pc = epc; + } else { + // Chop r to before pc; + // add new entry for after if needed. + // Changing r->epc does not affect r's position in the tree. + oldepc = r->epc; + r->epc = pc; + if(epc < oldepc) { + Range *n; + n = malloc(sizeof *n); + n->pc = epc; + n->epc = oldepc; + treeput(&breakpoints, n, n); + } + } +} + +void +showsrc(char *file, int line1, int line2) +{ + Biobuf *b; + char *p; + int n, stop; + + if((b = Bopen(file, OREAD)) == nil) { + print("\topen %s: %r\n", file); + return; + } + + for(n=1; n line2) + stop = line2; + if(stop < line2) + stop--; + for(; n<=stop && (p = Brdstr(b, '\n', 1)) != nil; n++) { + print(" %d %s\n", n, p); + free(p); + } + if(n < line2) + print(" ...\n"); + Bterm(b); +} + +/* + * if s is in the current directory or below, + * return the relative path. + */ +char* +shortname(char *s) +{ + if(!longnames && strlen(s) > ncwd && memcmp(s, cwd, ncwd) == 0 && s[ncwd] == '/') + return s+ncwd+1; + return s; +} + +/* + * we've decided that [pc, epc) did not run. + * do something about it. + */ +void +missing(uvlong pc, uvlong epc) +{ + char file[1000]; + int line1, line2; + char buf[100]; + Symbol s; + char *p; + uvlong uv; + + if(!findsym(pc, CTEXT, &s) || !fileline(file, sizeof file, pc)) { + notfound: + print("%#llux-%#llux\n", pc, epc); + return; + } + p = strrchr(file, ':'); + *p++ = 0; + line1 = atoi(p); + for(uv=pc; uvinstsize(text, uv); + } + p = strrchr(file, ':'); + *p++ = 0; + line2 = atoi(p); + + if(line2+1-line2 < minlines) + return; + + if(pc == s.value) { + // never entered function + print("%s:%d %s never called (%#llux-%#llux)\n", shortname(file), line1, s.name, pc, epc); + return; + } + if(pc <= s.value+13) { + // probably stub for stack growth. + // check whether last instruction is call to morestack. + // the -5 below is the length of + // CALL sys.morestack. + buf[0] = 0; + machdata->das(text, epc-5, 0, buf, sizeof buf); + if(strstr(buf, "morestack")) + return; + } + + if(epc - pc == 5) { + // check for CALL sys.panicindex + buf[0] = 0; + machdata->das(text, pc, 0, buf, sizeof buf); + if(strstr(buf, "panicindex")) + return; + } + + if(epc - pc == 2 || epc -pc == 3) { + // check for XORL inside shift. + // (on x86 have to implement large left or unsigned right shift with explicit zeroing). + // f+90 0x00002c9f CMPL CX,$20 + // f+93 0x00002ca2 JCS f+97(SB) + // f+95 0x00002ca4 XORL AX,AX <<< + // f+97 0x00002ca6 SHLL CL,AX + // f+99 0x00002ca8 MOVL $1,CX + // + // f+c8 0x00002cd7 CMPL CX,$40 + // f+cb 0x00002cda JCS f+d0(SB) + // f+cd 0x00002cdc XORQ AX,AX <<< + // f+d0 0x00002cdf SHLQ CL,AX + // f+d3 0x00002ce2 MOVQ $1,CX + buf[0] = 0; + machdata->das(text, pc, 0, buf, sizeof buf); + if(strncmp(buf, "XOR", 3) == 0) { + machdata->das(text, epc, 0, buf, sizeof buf); + if(strncmp(buf, "SHL", 3) == 0 || strncmp(buf, "SHR", 3) == 0) + return; + } + } + + if(epc - pc == 3) { + // check for SAR inside shift. + // (on x86 have to implement large signed right shift as >>31). + // f+36 0x00016216 CMPL CX,$20 + // f+39 0x00016219 JCS f+3e(SB) + // f+3b 0x0001621b SARL $1f,AX <<< + // f+3e 0x0001621e SARL CL,AX + // f+40 0x00016220 XORL CX,CX + // f+42 0x00016222 CMPL CX,AX + buf[0] = 0; + machdata->das(text, pc, 0, buf, sizeof buf); + if(strncmp(buf, "SAR", 3) == 0) { + machdata->das(text, epc, 0, buf, sizeof buf); + if(strncmp(buf, "SAR", 3) == 0) + return; + } + } + + // show first instruction to make clear where we were. + machdata->das(text, pc, 0, buf, sizeof buf); + + if(line1 != line2) + print("%s:%d,%d %#llux-%#llux %s\n", + shortname(file), line1, line2, pc, epc, buf); + else + print("%s:%d %#llux-%#llux %s\n", + shortname(file), line1, pc, epc, buf); + if(doshowsrc) + showsrc(file, line1, line2); +} + +/* + * walk the tree, calling missing for each non-empty + * section of missing code. + */ +void +walktree(TreeNode *t) +{ + Range *n; + + if(t == nil) + return; + walktree(t->left); + n = t->key; + if(n->pc < n->epc) + missing(n->pc, n->epc); + walktree(t->right); +} + +/* + * set a breakpoint all over [pc, epc) + * and remember that we did. + */ +void +breakpoint(uvlong pc, uvlong epc) +{ + Range *r; + + r = malloc(sizeof *r); + r->pc = pc; + r->epc = epc; + treeput(&breakpoints, r, r); + + for(; pc < epc; pc+=machdata->bpsize) + put1(mem, pc, machdata->bpinst, machdata->bpsize); +} + +/* + * install breakpoints over all text symbols + * that match the pattern. + */ +void +cover(void) +{ + Symbol s; + char *lastfn; + uvlong lastpc; + int i; + char buf[200]; + + lastfn = nil; + lastpc = 0; + for(i=0; textsym(&s, i); i++) { + switch(s.type) { + case 'T': + case 't': + if(lastpc != 0) { + breakpoint(lastpc, s.value); + lastpc = 0; + } + // Ignore second entry for a given name; + // that's the debugging blob. + if(lastfn && strcmp(s.name, lastfn) == 0) + break; + lastfn = s.name; + buf[0] = 0; + fileline(buf, sizeof buf, s.value); + if(substring == nil || strstr(buf, substring) || strstr(s.name, substring)) + lastpc = s.value; + } + } +} + +uvlong +rgetzero(Map *map, char *reg) +{ + return 0; +} + +/* + * remove the breakpoints at pc and successive instructions, + * up to and including the first jump or other control flow transfer. + */ +void +uncover(uvlong pc) +{ + uchar buf[1000]; + int n, n1, n2; + uvlong foll[2]; + + // Double-check that we stopped at a breakpoint. + if(get1(mem, pc, buf, machdata->bpsize) < 0) + sysfatal("read mem inst at %#llux: %r", pc); + if(memcmp(buf, machdata->bpinst, machdata->bpsize) != 0) + sysfatal("stopped at %#llux; not at breakpoint %d", pc, machdata->bpsize); + + // Figure out how many bytes of straight-line code + // there are in the text starting at pc. + n = 0; + while(n < sizeof buf) { + n1 = machdata->instsize(text, pc+n); + if(n+n1 > sizeof buf) + break; + n2 = machdata->foll(text, pc+n, rgetzero, foll); + n += n1; + if(n2 != 1 || foll[0] != pc+n) + break; + } + + // Record that this section of code ran. + ran(pc, pc+n); + + // Put original instructions back. + if(get1(text, pc, buf, n) < 0) + sysfatal("get1: %r"); + if(put1(mem, pc, buf, n) < 0) + sysfatal("put1: %r"); +} + +int +startprocess(char **argv) +{ + int pid; + + if((pid = fork()) < 0) + sysfatal("fork: %r"); + if(pid == 0) { + pid = getpid(); + if(ctlproc(pid, "hang") < 0) + sysfatal("ctlproc hang: %r"); + execv(argv[0], argv); + sysfatal("exec %s: %r", argv[0]); + } + if(ctlproc(pid, "attached") < 0 || ctlproc(pid, "waitstop") < 0) + sysfatal("attach %d %s: %r", pid, argv[0]); + return pid; +} + +int +go(void) +{ + uvlong pc; + char buf[100]; + int n; + + for(n = 0;; n++) { + ctlproc(pid, "startstop"); + if(get8(mem, offsetof(Ureg, ip), &pc) < 0) { + rerrstr(buf, sizeof buf); + if(strstr(buf, "exited") || strstr(buf, "No such process")) + return n; + sysfatal("cannot read pc: %r"); + } + pc--; + if(put8(mem, offsetof(Ureg, ip), pc) < 0) + sysfatal("cannot write pc: %r"); + uncover(pc); + } +} + +void +main(int argc, char **argv) +{ + int n; + + ARGBEGIN{ + case 'g': + substring = EARGF(usage()); + break; + case 'l': + longnames++; + break; + case 'n': + minlines = atoi(EARGF(usage())); + break; + case 's': + doshowsrc = 1; + break; + case 'v': + chatty++; + break; + default: + usage(); + }ARGEND + + getwd(cwd, sizeof cwd); + ncwd = strlen(cwd); + + if(argc == 0) { + *--argv = "6.out"; + argc++; + } + fd = open(argv[0], OREAD); + if(fd < 0) + sysfatal("open %s: %r", argv[0]); + if(crackhdr(fd, &fhdr) <= 0) + sysfatal("crackhdr: %r"); + machbytype(fhdr.type); + if(syminit(fd, &fhdr) <= 0) + sysfatal("syminit: %r"); + text = loadmap(nil, fd, &fhdr); + if(text == nil) + sysfatal("loadmap: %r"); + pid = startprocess(argv); + mem = attachproc(pid, &fhdr); + if(mem == nil) + sysfatal("attachproc: %r"); + breakpoints.cmp = rangecmp; + cover(); + n = go(); + walktree(breakpoints.root); + if(chatty) + print("%d breakpoints\n", n); + detachproc(mem); + exits(0); +} + diff --git a/src/cmd/cov/tree.c b/src/cmd/cov/tree.c new file mode 100644 index 000000000..116772e42 --- /dev/null +++ b/src/cmd/cov/tree.c @@ -0,0 +1,246 @@ +// Renamed from Map to Tree to avoid conflict with libmach. + +/* +Copyright (c) 2003-2007 Russ Cox, Tom Bergan, Austin Clements, + Massachusetts Institute of Technology +Portions Copyright (c) 2009 The Go Authors. All rights reserved. + +Permission is hereby granted, free of charge, to any person obtaining +a copy of this software and associated documentation files (the +"Software"), to deal in the Software without restriction, including +without limitation the rights to use, copy, modify, merge, publish, +distribute, sublicense, and/or sell copies of the Software, and to +permit persons to whom the Software is furnished to do so, subject to +the following conditions: + +The above copyright notice and this permission notice shall be +included in all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE +LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION +OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION +WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +*/ + +// Mutable map structure, but still based on +// Okasaki, Red Black Trees in a Functional Setting, JFP 1999, +// which is a lot easier than the traditional red-black +// and plenty fast enough for me. (Also I could copy +// and edit fmap.c.) + +#include +#include +#include "tree.h" + +#define TreeNode TreeNode +#define Tree Tree + +enum +{ + Red = 0, + Black = 1 +}; + + +// Red-black trees are binary trees with this property: +// 1. No red node has a red parent. +// 2. Every path from the root to a leaf contains the +// same number of black nodes. + +static TreeNode* +rwTreeNode(TreeNode *p, int color, TreeNode *left, void *key, void *value, TreeNode *right) +{ + if(p == nil) + p = malloc(sizeof *p); + p->color = color; + p->left = left; + p->key = key; + p->value = value; + p->right = right; + return p; +} + +static TreeNode* +balance(TreeNode *m0) +{ + void *xk, *xv, *yk, *yv, *zk, *zv; + TreeNode *a, *b, *c, *d; + TreeNode *m1, *m2; + int color; + TreeNode *left, *right; + void *key, *value; + + color = m0->color; + left = m0->left; + key = m0->key; + value = m0->value; + right = m0->right; + + // Okasaki notation: (T is mkTreeNode, B is Black, R is Red, x, y, z are key-value. + // + // balance B (T R (T R a x b) y c) z d + // balance B (T R a x (T R b y c)) z d + // balance B a x (T R (T R b y c) z d) + // balance B a x (T R b y (T R c z d)) + // + // = T R (T B a x b) y (T B c z d) + + if(color == Black){ + if(left && left->color == Red){ + if(left->left && left->left->color == Red){ + a = left->left->left; + xk = left->left->key; + xv = left->left->value; + b = left->left->right; + yk = left->key; + yv = left->value; + c = left->right; + zk = key; + zv = value; + d = right; + m1 = left; + m2 = left->left; + goto hard; + }else if(left->right && left->right->color == Red){ + a = left->left; + xk = left->key; + xv = left->value; + b = left->right->left; + yk = left->right->key; + yv = left->right->value; + c = left->right->right; + zk = key; + zv = value; + d = right; + m1 = left; + m2 = left->right; + goto hard; + } + }else if(right && right->color == Red){ + if(right->left && right->left->color == Red){ + a = left; + xk = key; + xv = value; + b = right->left->left; + yk = right->left->key; + yv = right->left->value; + c = right->left->right; + zk = right->key; + zv = right->value; + d = right->right; + m1 = right; + m2 = right->left; + goto hard; + }else if(right->right && right->right->color == Red){ + a = left; + xk = key; + xv = value; + b = right->left; + yk = right->key; + yv = right->value; + c = right->right->left; + zk = right->right->key; + zv = right->right->value; + d = right->right->right; + m1 = right; + m2 = right->right; + goto hard; + } + } + } + return rwTreeNode(m0, color, left, key, value, right); + +hard: + return rwTreeNode(m0, Red, rwTreeNode(m1, Black, a, xk, xv, b), + yk, yv, rwTreeNode(m2, Black, c, zk, zv, d)); +} + +static TreeNode* +ins0(TreeNode *p, void *k, void *v, TreeNode *rw) +{ + if(p == nil) + return rwTreeNode(rw, Red, nil, k, v, nil); + if(p->key == k){ + if(rw) + return rwTreeNode(rw, p->color, p->left, k, v, p->right); + p->value = v; + return p; + } + if(p->key < k) + p->left = ins0(p->left, k, v, rw); + else + p->right = ins0(p->right, k, v, rw); + return balance(p); +} + +static TreeNode* +ins1(Tree *m, TreeNode *p, void *k, void *v, TreeNode *rw) +{ + int i; + + if(p == nil) + return rwTreeNode(rw, Red, nil, k, v, nil); + i = m->cmp(p->key, k); + if(i == 0){ + if(rw) + return rwTreeNode(rw, p->color, p->left, k, v, p->right); + p->value = v; + return p; + } + if(i < 0) + p->left = ins1(m, p->left, k, v, rw); + else + p->right = ins1(m, p->right, k, v, rw); + return balance(p); +} + +void +treeputelem(Tree *m, void *key, void *val, TreeNode *rw) +{ + if(m->cmp) + m->root = ins1(m, m->root, key, val, rw); + else + m->root = ins0(m->root, key, val, rw); +} + +void +treeput(Tree *m, void *key, void *val) +{ + treeputelem(m, key, val, nil); +} + +void* +treeget(Tree *m, void *key) +{ + int i; + TreeNode *p; + + p = m->root; + if(m->cmp){ + for(;;){ + if(p == nil) + return nil; + i = m->cmp(p->key, key); + if(i < 0) + p = p->left; + else if(i > 0) + p = p->right; + else + return p->value; + } + }else{ + for(;;){ + if(p == nil) + return nil; + if(p->key == key) + return p->value; + if(p->key < key) + p = p->left; + else + p = p->right; + } + } +} diff --git a/src/cmd/cov/tree.h b/src/cmd/cov/tree.h new file mode 100644 index 000000000..a716d83ad --- /dev/null +++ b/src/cmd/cov/tree.h @@ -0,0 +1,47 @@ +// Renamed from Map to Tree to avoid conflict with libmach. + +/* +Copyright (c) 2003-2007 Russ Cox, Tom Bergan, Austin Clements, + Massachusetts Institute of Technology +Portions Copyright (c) 2009 The Go Authors. All rights reserved. + +Permission is hereby granted, free of charge, to any person obtaining +a copy of this software and associated documentation files (the +"Software"), to deal in the Software without restriction, including +without limitation the rights to use, copy, modify, merge, publish, +distribute, sublicense, and/or sell copies of the Software, and to +permit persons to whom the Software is furnished to do so, subject to +the following conditions: + +The above copyright notice and this permission notice shall be +included in all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE +LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION +OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION +WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +*/ + +typedef struct Tree Tree; +typedef struct TreeNode TreeNode; +struct Tree +{ + int (*cmp)(void*, void*); + TreeNode *root; +}; + +struct TreeNode +{ + int color; + TreeNode *left; + void *key; + void *value; + TreeNode *right; +}; + +void *treeget(Tree*, void*); +void treeput(Tree*, void*, void*); +void treeputelem(Tree*, void*, void*, TreeNode*); diff --git a/src/cmd/ebnflint/Makefile b/src/cmd/ebnflint/Makefile new file mode 100644 index 000000000..8f030aaef --- /dev/null +++ b/src/cmd/ebnflint/Makefile @@ -0,0 +1,15 @@ +# Copyright 2009 The Go Authors. All rights reserved. +# Use of this source code is governed by a BSD-style +# license that can be found in the LICENSE file. + +include ../../Make.inc + +TARG=ebnflint +GOFILES=\ + ebnflint.go\ + +include ../../Make.cmd + +test: $(TARG) + $(TARG) -start="SourceFile" "$(GOROOT)"/doc/go_spec.html + diff --git a/src/cmd/ebnflint/doc.go b/src/cmd/ebnflint/doc.go new file mode 100644 index 000000000..f35976eea --- /dev/null +++ b/src/cmd/ebnflint/doc.go @@ -0,0 +1,22 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +/* + +Ebnflint verifies that EBNF productions are consistent and gramatically correct. +It reads them from an HTML document such as the Go specification. + +Grammar productions are grouped in boxes demarcated by the HTML elements +
+	
+ + +Usage: + ebnflint [--start production] [file] + +The --start flag specifies the name of the start production for +the grammar; it defaults to "Start". + +*/ +package documentation diff --git a/src/cmd/ebnflint/ebnflint.go b/src/cmd/ebnflint/ebnflint.go new file mode 100644 index 000000000..009b336f3 --- /dev/null +++ b/src/cmd/ebnflint/ebnflint.go @@ -0,0 +1,109 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package main + +import ( + "bytes" + "ebnf" + "flag" + "fmt" + "go/scanner" + "go/token" + "io/ioutil" + "os" + "path/filepath" +) + +var fset = token.NewFileSet() +var start = flag.String("start", "Start", "name of start production") + +func usage() { + fmt.Fprintf(os.Stderr, "usage: ebnflint [flags] [filename]\n") + flag.PrintDefaults() + os.Exit(1) +} + +// Markers around EBNF sections in .html files +var ( + open = []byte(`
`)
+	close = []byte(`
`) +) + +func report(err os.Error) { + scanner.PrintError(os.Stderr, err) + os.Exit(1) +} + +func extractEBNF(src []byte) []byte { + var buf bytes.Buffer + + for { + // i = beginning of EBNF text + i := bytes.Index(src, open) + if i < 0 { + break // no EBNF found - we are done + } + i += len(open) + + // write as many newlines as found in the excluded text + // to maintain correct line numbers in error messages + for _, ch := range src[0:i] { + if ch == '\n' { + buf.WriteByte('\n') + } + } + + // j = end of EBNF text (or end of source) + j := bytes.Index(src[i:], close) // close marker + if j < 0 { + j = len(src) - i + } + j += i + + // copy EBNF text + buf.Write(src[i:j]) + + // advance + src = src[j:] + } + + return buf.Bytes() +} + +func main() { + flag.Parse() + + var ( + filename string + src []byte + err os.Error + ) + switch flag.NArg() { + case 0: + filename = "" + src, err = ioutil.ReadAll(os.Stdin) + case 1: + filename = flag.Arg(0) + src, err = ioutil.ReadFile(filename) + default: + usage() + } + if err != nil { + report(err) + } + + if filepath.Ext(filename) == ".html" || bytes.Index(src, open) >= 0 { + src = extractEBNF(src) + } + + grammar, err := ebnf.Parse(fset, filename, src) + if err != nil { + report(err) + } + + if err = ebnf.Verify(fset, grammar, *start); err != nil { + report(err) + } +} diff --git a/src/cmd/gc/Makefile b/src/cmd/gc/Makefile new file mode 100644 index 000000000..0af7659e4 --- /dev/null +++ b/src/cmd/gc/Makefile @@ -0,0 +1,71 @@ +# Copyright 2009 The Go Authors. All rights reserved. +# Use of this source code is governed by a BSD-style +# license that can be found in the LICENSE file. + +include ../../Make.inc +O:=$(HOST_O) + +LIB=gc.a + +HFILES=\ + go.h\ + y.tab.h\ + md5.h\ + +YFILES=\ + go.y\ + +OFILES=\ + align.$O\ + bits.$O\ + builtin.$O\ + closure.$O\ + const.$O\ + dcl.$O\ + export.$O\ + gen.$O\ + init.$O\ + lex.$O\ + md5.$O\ + mparith1.$O\ + mparith2.$O\ + mparith3.$O\ + obj.$O\ + print.$O\ + range.$O\ + reflect.$O\ + select.$O\ + sinit.$O\ + subr.$O\ + swt.$O\ + typecheck.$O\ + unsafe.$O\ + walk.$O\ + y1.tab.$O\ + +NOINSTALL=1 +include ../../Make.clib + +install: $(LIB) + +y1.tab.c: y.tab.c # make yystate global, yytname mutable + cat y.tab.c | sed '/ int yystate;/d; s/int yychar;/int yychar, yystate;/; s/static const char \*const yytname/const char *yytname/; s/char const \*yymsgp/char *yymsgp/' >y1.tab.c + +yerr.h: bisonerrors go.errors y.tab.h # y.tab.h rule generates y.output too + awk -f bisonerrors y.output go.errors >yerr.h + +subr.$O: yerr.h + +builtin.c: builtin.c.boot + cp builtin.c.boot builtin.c + +subr.$O: opnames.h + +opnames.h: mkopnames go.h + ./mkopnames go.h >opnames.h + +CLEANFILES+=*.[568] [568].out y1.tab.c yerr.h mkbuiltin1 builtin.c _builtin.c opnames.h + +mkbuiltin1: mkbuiltin1.$O + $(HOST_LD) -o $@ mkbuiltin1.$O -L"$(GOROOT)"/lib -lbio -l9 -lm $(HOST_LDFLAGS) + diff --git a/src/cmd/gc/align.c b/src/cmd/gc/align.c new file mode 100644 index 000000000..14c1c4a8d --- /dev/null +++ b/src/cmd/gc/align.c @@ -0,0 +1,653 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +#include "go.h" + +/* + * machine size and rounding + * alignment is dictated around + * the size of a pointer, set in betypeinit + * (see ../6g/galign.c). + */ + +static int defercalc; + +uint32 +rnd(uint32 o, uint32 r) +{ + if(r < 1 || r > 8 || (r&(r-1)) != 0) + fatal("rnd"); + return (o+r-1)&~(r-1); +} + +static void +offmod(Type *t) +{ + Type *f; + int32 o; + + o = 0; + for(f=t->type; f!=T; f=f->down) { + if(f->etype != TFIELD) + fatal("offmod: not TFIELD: %lT", f); + f->width = o; + o += widthptr; + if(o >= MAXWIDTH) { + yyerror("interface too large"); + o = widthptr; + } + } +} + +static vlong +widstruct(Type *errtype, Type *t, vlong o, int flag) +{ + Type *f; + int32 w, maxalign; + + maxalign = flag; + if(maxalign < 1) + maxalign = 1; + for(f=t->type; f!=T; f=f->down) { + if(f->etype != TFIELD) + fatal("widstruct: not TFIELD: %lT", f); + dowidth(f->type); + if(f->type->align > maxalign) + maxalign = f->type->align; + if(f->type->width < 0) + fatal("invalid width %lld", f->type->width); + w = f->type->width; + if(f->type->align > 0) + o = rnd(o, f->type->align); + f->width = o; // really offset for TFIELD + if(f->nname != N) { + // this same stackparam logic is in addrescapes + // in typecheck.c. usually addrescapes runs after + // widstruct, in which case we could drop this, + // but function closure functions are the exception. + if(f->nname->stackparam) { + f->nname->stackparam->xoffset = o; + f->nname->xoffset = 0; + } else + f->nname->xoffset = o; + } + o += w; + if(o >= MAXWIDTH) { + yyerror("type %lT too large", errtype); + o = 8; // small but nonzero + } + } + // final width is rounded + if(flag) + o = rnd(o, maxalign); + t->align = maxalign; + + // type width only includes back to first field's offset + if(t->type == T) + t->width = 0; + else + t->width = o - t->type->width; + return o; +} + +void +dowidth(Type *t) +{ + int32 et; + int64 w; + int lno; + Type *t1; + + if(widthptr == 0) + fatal("dowidth without betypeinit"); + + if(t == T) + return; + + if(t->width > 0) + return; + + if(t->width == -2) { + lno = lineno; + lineno = t->lineno; + yyerror("invalid recursive type %T", t); + t->width = 0; + lineno = lno; + return; + } + + // defer checkwidth calls until after we're done + defercalc++; + + lno = lineno; + lineno = t->lineno; + t->width = -2; + t->align = 0; + + et = t->etype; + switch(et) { + case TFUNC: + case TCHAN: + case TMAP: + case TSTRING: + break; + + default: + /* simtype == 0 during bootstrap */ + if(simtype[t->etype] != 0) + et = simtype[t->etype]; + break; + } + + w = 0; + switch(et) { + default: + fatal("dowidth: unknown type: %T", t); + break; + + /* compiler-specific stuff */ + case TINT8: + case TUINT8: + case TBOOL: // bool is int8 + w = 1; + break; + case TINT16: + case TUINT16: + w = 2; + break; + case TINT32: + case TUINT32: + case TFLOAT32: + w = 4; + break; + case TINT64: + case TUINT64: + case TFLOAT64: + case TCOMPLEX64: + w = 8; + t->align = widthptr; + break; + case TCOMPLEX128: + w = 16; + t->align = widthptr; + break; + case TPTR32: + w = 4; + checkwidth(t->type); + break; + case TPTR64: + w = 8; + checkwidth(t->type); + break; + case TUNSAFEPTR: + w = widthptr; + break; + case TINTER: // implemented as 2 pointers + w = 2*widthptr; + t->align = widthptr; + offmod(t); + break; + case TCHAN: // implemented as pointer + w = widthptr; + checkwidth(t->type); + + // make fake type to check later to + // trigger channel argument check. + t1 = typ(TCHANARGS); + t1->type = t; + checkwidth(t1); + break; + case TCHANARGS: + t1 = t->type; + dowidth(t->type); // just in case + if(t1->type->width >= (1<<16)) + yyerror("channel element type too large (>64kB)"); + t->width = 1; + break; + case TMAP: // implemented as pointer + w = widthptr; + checkwidth(t->type); + checkwidth(t->down); + break; + case TFORW: // should have been filled in + yyerror("invalid recursive type %T", t); + w = 1; // anything will do + break; + case TANY: + // dummy type; should be replaced before use. + if(!debug['A']) + fatal("dowidth any"); + w = 1; // anything will do + break; + case TSTRING: + if(sizeof_String == 0) + fatal("early dowidth string"); + w = sizeof_String; + t->align = widthptr; + break; + case TARRAY: + if(t->type == T) + break; + if(t->bound >= 0) { + uint64 cap; + + dowidth(t->type); + if(t->type->width != 0) { + cap = (MAXWIDTH-1) / t->type->width; + if(t->bound > cap) + yyerror("type %lT larger than address space", t); + } + w = t->bound * t->type->width; + t->align = t->type->align; + } + else if(t->bound == -1) { + w = sizeof_Array; + checkwidth(t->type); + t->align = widthptr; + } + else if(t->bound == -100) + yyerror("use of [...] array outside of array literal"); + else + fatal("dowidth %T", t); // probably [...]T + break; + + case TSTRUCT: + if(t->funarg) + fatal("dowidth fn struct %T", t); + w = widstruct(t, t, 0, 1); + break; + + case TFUNC: + // make fake type to check later to + // trigger function argument computation. + t1 = typ(TFUNCARGS); + t1->type = t; + checkwidth(t1); + + // width of func type is pointer + w = widthptr; + break; + + case TFUNCARGS: + // function is 3 cated structures; + // compute their widths as side-effect. + t1 = t->type; + w = widstruct(t->type, *getthis(t1), 0, 0); + w = widstruct(t->type, *getinarg(t1), w, widthptr); + w = widstruct(t->type, *getoutarg(t1), w, widthptr); + t1->argwid = w; + if(w%widthptr) + warn("bad type %T %d\n", t1, w); + t->align = 1; + break; + } + + t->width = w; + if(t->align == 0) { + if(w > 8 || (w&(w-1)) != 0) + fatal("invalid alignment for %T", t); + t->align = w; + } + lineno = lno; + + if(defercalc == 1) + resumecheckwidth(); + else + --defercalc; +} + +/* + * when a type's width should be known, we call checkwidth + * to compute it. during a declaration like + * + * type T *struct { next T } + * + * it is necessary to defer the calculation of the struct width + * until after T has been initialized to be a pointer to that struct. + * similarly, during import processing structs may be used + * before their definition. in those situations, calling + * defercheckwidth() stops width calculations until + * resumecheckwidth() is called, at which point all the + * checkwidths that were deferred are executed. + * dowidth should only be called when the type's size + * is needed immediately. checkwidth makes sure the + * size is evaluated eventually. + */ +typedef struct TypeList TypeList; +struct TypeList { + Type *t; + TypeList *next; +}; + +static TypeList *tlfree; +static TypeList *tlq; + +void +checkwidth(Type *t) +{ + TypeList *l; + + if(t == T) + return; + + // function arg structs should not be checked + // outside of the enclosing function. + if(t->funarg) + fatal("checkwidth %T", t); + + if(!defercalc) { + dowidth(t); + return; + } + if(t->deferwidth) + return; + t->deferwidth = 1; + + l = tlfree; + if(l != nil) + tlfree = l->next; + else + l = mal(sizeof *l); + + l->t = t; + l->next = tlq; + tlq = l; +} + +void +defercheckwidth(void) +{ + // we get out of sync on syntax errors, so don't be pedantic. + if(defercalc && nerrors == 0) + fatal("defercheckwidth"); + defercalc = 1; +} + +void +resumecheckwidth(void) +{ + TypeList *l; + + if(!defercalc) + fatal("resumecheckwidth"); + for(l = tlq; l != nil; l = tlq) { + l->t->deferwidth = 0; + tlq = l->next; + dowidth(l->t); + l->next = tlfree; + tlfree = l; + } + defercalc = 0; +} + +void +typeinit(void) +{ + int i, etype, sameas; + Type *t; + Sym *s, *s1; + + if(widthptr == 0) + fatal("typeinit before betypeinit"); + + for(i=0; isym = pkglookup("Pointer", unsafepkg); + t->sym->def = typenod(t); + + dowidth(types[TUNSAFEPTR]); + + tptr = TPTR32; + if(widthptr == 8) + tptr = TPTR64; + + for(i=TINT8; i<=TUINT64; i++) + isint[i] = 1; + isint[TINT] = 1; + isint[TUINT] = 1; + isint[TUINTPTR] = 1; + + isfloat[TFLOAT32] = 1; + isfloat[TFLOAT64] = 1; + + iscomplex[TCOMPLEX64] = 1; + iscomplex[TCOMPLEX128] = 1; + + isptr[TPTR32] = 1; + isptr[TPTR64] = 1; + + isforw[TFORW] = 1; + + issigned[TINT] = 1; + issigned[TINT8] = 1; + issigned[TINT16] = 1; + issigned[TINT32] = 1; + issigned[TINT64] = 1; + + /* + * initialize okfor + */ + for(i=0; i= nelem(types)) + fatal("typeinit: %s bad etype", s->name); + sameas = typedefs[i].sameas; + if(sameas < 0 || sameas >= nelem(types)) + fatal("typeinit: %s bad sameas", s->name); + simtype[etype] = sameas; + minfltval[etype] = minfltval[sameas]; + maxfltval[etype] = maxfltval[sameas]; + minintval[etype] = minintval[sameas]; + maxintval[etype] = maxintval[sameas]; + + t = types[etype]; + if(t != T) + fatal("typeinit: %s already defined", s->name); + + t = typ(etype); + t->sym = s; + + dowidth(t); + types[etype] = t; + s1->def = typenod(t); + } + + Array_array = rnd(0, widthptr); + Array_nel = rnd(Array_array+widthptr, types[TUINT32]->width); + Array_cap = rnd(Array_nel+types[TUINT32]->width, types[TUINT32]->width); + sizeof_Array = rnd(Array_cap+types[TUINT32]->width, widthptr); + + // string is same as slice wo the cap + sizeof_String = rnd(Array_nel+types[TUINT32]->width, widthptr); + + dowidth(types[TSTRING]); + dowidth(idealstring); +} + +/* + * compute total size of f's in/out arguments. + */ +int +argsize(Type *t) +{ + Iter save; + Type *fp; + int w, x; + + w = 0; + + fp = structfirst(&save, getoutarg(t)); + while(fp != T) { + x = fp->width + fp->type->width; + if(x > w) + w = x; + fp = structnext(&save); + } + + fp = funcfirst(&save, t); + while(fp != T) { + x = fp->width + fp->type->width; + if(x > w) + w = x; + fp = funcnext(&save); + } + + w = (w+widthptr-1) & ~(widthptr-1); + return w; +} diff --git a/src/cmd/gc/bisonerrors b/src/cmd/gc/bisonerrors new file mode 100755 index 000000000..5110f5350 --- /dev/null +++ b/src/cmd/gc/bisonerrors @@ -0,0 +1,124 @@ +#!/usr/bin/awk -f +# Copyright 2010 The Go Authors. All rights reserved. +# Use of this source code is governed by a BSD-style +# license that can be found in the LICENSE file. + +# This program implements the core idea from +# +# Clinton L. Jeffery, Generating LR syntax error messages from examples, +# ACM TOPLAS 25(5) (September 2003). http://doi.acm.org/10.1145/937563.937566 +# +# It reads Bison's summary of a grammar followed by a file +# like go.errors, replacing lines beginning with % by the +# yystate and yychar that will be active when an error happens +# while parsing that line. +# +# Unlike the system described in the paper, the lines in go.errors +# give grammar symbol name lists, not actual program fragments. +# This is a little less programmer-friendly but doesn't require being +# able to run the text through lex.c. + +BEGIN{ + bison = 1 + grammar = 0 + states = 0 +} + +# In Grammar section of y.output, +# record lhs and length of rhs for each rule. +bison && /^Grammar/ { grammar = 1 } +bison && /^(Terminals|state 0)/ { grammar = 0 } +grammar && NF>0 { + if($2 != "|") { + r = $2 + sub(/:$/, "", r) + } + rulelhs[$1] = r + rulesize[$1] = NF-2 + if(rulesize[$1] == 3 && $3 $4 $5 == "/*empty*/") { + rulesize[$1] = 0 + } +} + +# In state dumps, record shift/reduce actions. +bison && /^state 0/ { grammar = 0; states = 1 } + +states && /^state / { state = $2 } +states { statetext[state] = statetext[state] $0 "\n" } + +states && / shift, and go to state/ { + n = nshift[state]++ + shift[state,n] = $7 + shifttoken[state,n] = $1 + next +} +states && / go to state/ { + n = nshift[state]++ + shift[state,n] = $5 + shifttoken[state,n] = $1 + next +} +states && / reduce using rule/ { + n = nreduce[state]++ + reduce[state,n] = $5 + reducetoken[state,n] = $1 + next +} + +# First // comment marks the beginning of the pattern file. +/^\/\// { bison = 0; grammar = 0; state = 0 } +bison { next } + +# Treat % as first field on line as introducing a pattern (token sequence). +# Run it through the LR machine and print the induced "yystate, yychar," +# at the point where the error happens. +$1 == "%" { + nstack = 0 + state = 0 + f = 2 + tok = "" + for(;;) { + if(tok == "" && f <= NF) { + tok = $f + f++ + } + found = 0 + for(j=0; j " shift[state,j] + stack[nstack++] = state + state = shift[state,j] + found = 1 + tok = "" + break + } + } + if(found) + continue + for(j=0; jb[i]) + return 1; + return 0; +} + +/* +int +beq(Bits a, Bits b) +{ + int i; + + for(i=0; iargs, Bits); + while(bany(&bits)) { + i = bnum(bits); + if(first) + first = 0; + else + fmtprint(fp, " "); + if(var[i].sym == S) + fmtprint(fp, "$%lld", var[i].offset); + else { + fmtprint(fp, var[i].sym->name); + if(var[i].offset != 0) + fmtprint(fp, "%+d", var[i].offset); + } + bits.b[i/32] &= ~(1L << (i%32)); + } + return 0; +} diff --git a/src/cmd/gc/builtin.c.boot b/src/cmd/gc/builtin.c.boot new file mode 100644 index 000000000..190c56008 --- /dev/null +++ b/src/cmd/gc/builtin.c.boot @@ -0,0 +1,114 @@ +char *runtimeimport = + "package runtime\n" + "import runtime \"runtime\"\n" + "func \"\".new (? int32) *any\n" + "func \"\".panicindex ()\n" + "func \"\".panicslice ()\n" + "func \"\".throwreturn ()\n" + "func \"\".throwinit ()\n" + "func \"\".panicwrap (? string, ? string, ? string)\n" + "func \"\".panic (? interface { })\n" + "func \"\".recover (? *int32) interface { }\n" + "func \"\".printbool (? bool)\n" + "func \"\".printfloat (? float64)\n" + "func \"\".printint (? int64)\n" + "func \"\".printuint (? uint64)\n" + "func \"\".printcomplex (? complex128)\n" + "func \"\".printstring (? string)\n" + "func \"\".printpointer (? any)\n" + "func \"\".printiface (? any)\n" + "func \"\".printeface (? any)\n" + "func \"\".printslice (? any)\n" + "func \"\".printnl ()\n" + "func \"\".printsp ()\n" + "func \"\".goprintf ()\n" + "func \"\".concatstring ()\n" + "func \"\".append ()\n" + "func \"\".appendslice (typ *uint8, x any, y []any) any\n" + "func \"\".cmpstring (? string, ? string) int\n" + "func \"\".slicestring (? string, ? int, ? int) string\n" + "func \"\".slicestring1 (? string, ? int) string\n" + "func \"\".intstring (? int64) string\n" + "func \"\".slicebytetostring (? []uint8) string\n" + "func \"\".sliceinttostring (? []int) string\n" + "func \"\".stringtoslicebyte (? string) []uint8\n" + "func \"\".stringtosliceint (? string) []int\n" + "func \"\".stringiter (? string, ? int) int\n" + "func \"\".stringiter2 (? string, ? int) (retk int, retv int)\n" + "func \"\".slicecopy (to any, fr any, wid uint32) int\n" + "func \"\".slicestringcopy (to any, fr any) int\n" + "func \"\".convI2E (elem any) any\n" + "func \"\".convI2I (typ *uint8, elem any) any\n" + "func \"\".convT2E (typ *uint8, elem any) any\n" + "func \"\".convT2I (typ *uint8, typ2 *uint8, elem any) any\n" + "func \"\".assertE2E (typ *uint8, iface any) any\n" + "func \"\".assertE2E2 (typ *uint8, iface any) (ret any, ok bool)\n" + "func \"\".assertE2I (typ *uint8, iface any) any\n" + "func \"\".assertE2I2 (typ *uint8, iface any) (ret any, ok bool)\n" + "func \"\".assertE2T (typ *uint8, iface any) any\n" + "func \"\".assertE2T2 (typ *uint8, iface any) (ret any, ok bool)\n" + "func \"\".assertI2E (typ *uint8, iface any) any\n" + "func \"\".assertI2E2 (typ *uint8, iface any) (ret any, ok bool)\n" + "func \"\".assertI2I (typ *uint8, iface any) any\n" + "func \"\".assertI2I2 (typ *uint8, iface any) (ret any, ok bool)\n" + "func \"\".assertI2T (typ *uint8, iface any) any\n" + "func \"\".assertI2T2 (typ *uint8, iface any) (ret any, ok bool)\n" + "func \"\".ifaceeq (i1 any, i2 any) bool\n" + "func \"\".efaceeq (i1 any, i2 any) bool\n" + "func \"\".ifacethash (i1 any) uint32\n" + "func \"\".efacethash (i1 any) uint32\n" + "func \"\".makemap (mapType *uint8, hint int64) map[any] any\n" + "func \"\".mapaccess1 (mapType *uint8, hmap map[any] any, key any) any\n" + "func \"\".mapaccess2 (mapType *uint8, hmap map[any] any, key any) (val any, pres bool)\n" + "func \"\".mapassign1 (mapType *uint8, hmap map[any] any, key any, val any)\n" + "func \"\".mapassign2 (mapType *uint8, hmap map[any] any, key any, val any, pres bool)\n" + "func \"\".mapiterinit (mapType *uint8, hmap map[any] any, hiter *any)\n" + "func \"\".mapiternext (hiter *any)\n" + "func \"\".mapiter1 (hiter *any) any\n" + "func \"\".mapiter2 (hiter *any) (key any, val any)\n" + "func \"\".makechan (chanType *uint8, hint int64) chan any\n" + "func \"\".chanrecv1 (chanType *uint8, hchan <-chan any) any\n" + "func \"\".chanrecv2 (chanType *uint8, hchan <-chan any) (elem any, received bool)\n" + "func \"\".chansend1 (chanType *uint8, hchan chan<- any, elem any)\n" + "func \"\".closechan (hchan any)\n" + "func \"\".selectnbsend (chanType *uint8, hchan chan<- any, elem any) bool\n" + "func \"\".selectnbrecv (chanType *uint8, elem *any, hchan <-chan any) bool\n" + "func \"\".selectnbrecv2 (chanType *uint8, elem *any, received *bool, hchan <-chan any) bool\n" + "func \"\".newselect (size int) *uint8\n" + "func \"\".selectsend (sel *uint8, hchan chan<- any, elem *any) bool\n" + "func \"\".selectrecv (sel *uint8, hchan <-chan any, elem *any) bool\n" + "func \"\".selectrecv2 (sel *uint8, hchan <-chan any, elem *any, received *bool) bool\n" + "func \"\".selectdefault (sel *uint8) bool\n" + "func \"\".selectgo (sel *uint8)\n" + "func \"\".block ()\n" + "func \"\".makeslice (typ *uint8, nel int64, cap int64) []any\n" + "func \"\".growslice (typ *uint8, old []any, n int64) []any\n" + "func \"\".sliceslice1 (old []any, lb uint64, width uint64) []any\n" + "func \"\".sliceslice (old []any, lb uint64, hb uint64, width uint64) []any\n" + "func \"\".slicearray (old *any, nel uint64, lb uint64, hb uint64, width uint64) []any\n" + "func \"\".closure ()\n" + "func \"\".int64div (? int64, ? int64) int64\n" + "func \"\".uint64div (? uint64, ? uint64) uint64\n" + "func \"\".int64mod (? int64, ? int64) int64\n" + "func \"\".uint64mod (? uint64, ? uint64) uint64\n" + "func \"\".float64toint64 (? float64) int64\n" + "func \"\".float64touint64 (? float64) uint64\n" + "func \"\".int64tofloat64 (? int64) float64\n" + "func \"\".uint64tofloat64 (? uint64) float64\n" + "func \"\".complex128div (num complex128, den complex128) complex128\n" + "\n" + "$$\n"; +char *unsafeimport = + "package unsafe\n" + "import runtime \"runtime\"\n" + "type \"\".Pointer uintptr\n" + "func \"\".Offsetof (? any) uintptr\n" + "func \"\".Sizeof (? any) uintptr\n" + "func \"\".Alignof (? any) uintptr\n" + "func \"\".Typeof (i interface { }) interface { }\n" + "func \"\".Reflect (i interface { }) (typ interface { }, addr \"\".Pointer)\n" + "func \"\".Unreflect (typ interface { }, addr \"\".Pointer) interface { }\n" + "func \"\".New (typ interface { }) \"\".Pointer\n" + "func \"\".NewArray (typ interface { }, n int) \"\".Pointer\n" + "\n" + "$$\n"; diff --git a/src/cmd/gc/closure.c b/src/cmd/gc/closure.c new file mode 100644 index 000000000..1261eefb7 --- /dev/null +++ b/src/cmd/gc/closure.c @@ -0,0 +1,252 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +/* + * function literals aka closures + */ + +#include "go.h" + +void +closurehdr(Node *ntype) +{ + Node *n, *name, *a; + NodeList *l; + + n = nod(OCLOSURE, N, N); + n->ntype = ntype; + n->funcdepth = funcdepth; + + funchdr(n); + + // steal ntype's argument names and + // leave a fresh copy in their place. + // references to these variables need to + // refer to the variables in the external + // function declared below; see walkclosure. + n->list = ntype->list; + n->rlist = ntype->rlist; + ntype->list = nil; + ntype->rlist = nil; + for(l=n->list; l; l=l->next) { + name = l->n->left; + if(name) + name = newname(name->sym); + a = nod(ODCLFIELD, name, l->n->right); + a->isddd = l->n->isddd; + if(name) + name->isddd = a->isddd; + ntype->list = list(ntype->list, a); + } + for(l=n->rlist; l; l=l->next) { + name = l->n->left; + if(name) + name = newname(name->sym); + ntype->rlist = list(ntype->rlist, nod(ODCLFIELD, name, l->n->right)); + } +} + +Node* +closurebody(NodeList *body) +{ + Node *func, *v; + NodeList *l; + + if(body == nil) + body = list1(nod(OEMPTY, N, N)); + + func = curfn; + l = func->dcl; + func->nbody = body; + funcbody(func); + + // closure-specific variables are hanging off the + // ordinary ones in the symbol table; see oldname. + // unhook them. + // make the list of pointers for the closure call. + for(l=func->cvars; l; l=l->next) { + v = l->n; + v->closure->closure = v->outer; + v->heapaddr = nod(OADDR, oldname(v->sym), N); + } + + return func; +} + +void +typecheckclosure(Node *func, int top) +{ + Node *oldfn; + NodeList *l; + Node *v; + + oldfn = curfn; + typecheck(&func->ntype, Etype); + func->type = func->ntype->type; + if(curfn == nil) { + xtop = list(xtop, func); + return; + } + + if(func->type != T) { + curfn = func; + typechecklist(func->nbody, Etop); + curfn = oldfn; + } + + // type check the & of closed variables outside the closure, + // so that the outer frame also grabs them and knows they + // escape. + func->enter = nil; + for(l=func->cvars; l; l=l->next) { + v = l->n; + if(v->type == T) { + // if v->type is nil, it means v looked like it was + // going to be used in the closure but wasn't. + // this happens because when parsing a, b, c := f() + // the a, b, c gets parsed as references to older + // a, b, c before the parser figures out this is a + // declaration. + v->op = 0; + continue; + } + // For a closure that is called in place, but not + // inside a go statement, avoid moving variables to the heap. + if ((top & (Ecall|Eproc)) == Ecall) + v->heapaddr->etype = 1; + typecheck(&v->heapaddr, Erv); + func->enter = list(func->enter, v->heapaddr); + v->heapaddr = N; + } +} + +static Node* +makeclosure(Node *func, NodeList **init, int nowrap) +{ + Node *xtype, *v, *addr, *xfunc; + NodeList *l; + static int closgen; + char *p; + + /* + * wrap body in external function + * with extra closure parameters. + */ + xtype = nod(OTFUNC, N, N); + + // each closure variable has a corresponding + // address parameter. + for(l=func->cvars; l; l=l->next) { + v = l->n; + if(v->op == 0) + continue; + addr = nod(ONAME, N, N); + p = smprint("&%s", v->sym->name); + addr->sym = lookup(p); + free(p); + addr->ntype = nod(OIND, typenod(v->type), N); + addr->class = PPARAM; + addr->addable = 1; + addr->ullman = 1; + + v->heapaddr = addr; + + xtype->list = list(xtype->list, nod(ODCLFIELD, addr, addr->ntype)); + } + + // then a dummy arg where the closure's caller pc sits + if (!nowrap) + xtype->list = list(xtype->list, nod(ODCLFIELD, N, typenod(types[TUINTPTR]))); + + // then the function arguments + xtype->list = concat(xtype->list, func->list); + xtype->rlist = concat(xtype->rlist, func->rlist); + + // create the function + xfunc = nod(ODCLFUNC, N, N); + snprint(namebuf, sizeof namebuf, "_func_%.3d", ++closgen); + xfunc->nname = newname(lookup(namebuf)); + xfunc->nname->ntype = xtype; + xfunc->nname->defn = xfunc; + declare(xfunc->nname, PFUNC); + xfunc->nname->funcdepth = func->funcdepth; + xfunc->funcdepth = func->funcdepth; + xfunc->nbody = func->nbody; + xfunc->dcl = func->dcl; + if(xfunc->nbody == nil) + fatal("empty body - won't generate any code"); + typecheck(&xfunc, Etop); + closures = list(closures, xfunc); + + return xfunc; +} + +Node* +walkclosure(Node *func, NodeList **init) +{ + int narg; + Node *xtype, *xfunc, *call, *clos; + NodeList *l, *in; + + /* + * wrap body in external function + * with extra closure parameters. + */ + + // create the function + xfunc = makeclosure(func, init, 0); + xtype = xfunc->nname->ntype; + + // prepare call of sys.closure that turns external func into func literal value. + clos = syslook("closure", 1); + clos->type = T; + clos->ntype = nod(OTFUNC, N, N); + in = list1(nod(ODCLFIELD, N, typenod(types[TINT]))); // siz + in = list(in, nod(ODCLFIELD, N, xtype)); + narg = 0; + for(l=func->cvars; l; l=l->next) { + if(l->n->op == 0) + continue; + narg++; + in = list(in, nod(ODCLFIELD, N, l->n->heapaddr->ntype)); + } + clos->ntype->list = in; + clos->ntype->rlist = list1(nod(ODCLFIELD, N, typenod(func->type))); + typecheck(&clos, Erv); + + call = nod(OCALL, clos, N); + if(narg*widthptr > 100) + yyerror("closure needs too many variables; runtime will reject it"); + in = list1(nodintconst(narg*widthptr)); + in = list(in, xfunc->nname); + in = concat(in, func->enter); + call->list = in; + + typecheck(&call, Erv); + walkexpr(&call, init); + return call; +} + +// Special case for closures that get called in place. +// Optimize runtime.closure(X, __func__xxxx_, .... ) away +// to __func__xxxx_(Y ....). +// On entry, expect n->op == OCALL, n->left->op == OCLOSURE. +void +walkcallclosure(Node *n, NodeList **init) +{ + if (n->op != OCALLFUNC || n->left->op != OCLOSURE) { + dump("walkcallclosure", n); + fatal("abuse of walkcallclosure"); + } + + // New arg list for n. First the closure-args + // and then the original parameter list. + n->list = concat(n->left->enter, n->list); + n->left = makeclosure(n->left, init, 1)->nname; + dowidth(n->left->type); + n->type = getoutargx(n->left->type); + // for a single valued function, pull the field type out of the struct + if (n->type && n->type->type && !n->type->type->down) + n->type = n->type->type->type; +} diff --git a/src/cmd/gc/const.c b/src/cmd/gc/const.c new file mode 100644 index 000000000..36a64cb97 --- /dev/null +++ b/src/cmd/gc/const.c @@ -0,0 +1,1299 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +#include "go.h" +#define TUP(x,y) (((x)<<16)|(y)) + +static Val tocplx(Val); +static Val toflt(Val); +static Val tostr(Val); +static Val copyval(Val); +static void cmplxmpy(Mpcplx*, Mpcplx*); +static void cmplxdiv(Mpcplx*, Mpcplx*); + +/* + * truncate float literal fv to 32-bit or 64-bit precision + * according to type; return truncated value. + */ +Mpflt* +truncfltlit(Mpflt *oldv, Type *t) +{ + double d; + float f; + Mpflt *fv; + + if(t == T) + return oldv; + + fv = mal(sizeof *fv); + *fv = *oldv; + + // convert large precision literal floating + // into limited precision (float64 or float32) + // botch -- this assumes that compiler fp + // has same precision as runtime fp + switch(t->etype) { + case TFLOAT64: + d = mpgetflt(fv); + mpmovecflt(fv, d); + break; + + case TFLOAT32: + d = mpgetflt(fv); + f = d; + d = f; + mpmovecflt(fv, d); + break; + } + return fv; +} + +/* + * convert n, if literal, to type t. + * implicit conversion. + */ +void +convlit(Node **np, Type *t) +{ + convlit1(np, t, 0); +} + +/* + * convert n, if literal, to type t. + * return a new node if necessary + * (if n is a named constant, can't edit n->type directly). + */ +void +convlit1(Node **np, Type *t, int explicit) +{ + int ct, et; + Node *n, *nn; + + n = *np; + if(n == N || t == T || n->type == T || isideal(t) || n->type == t) + return; + if(!explicit && !isideal(n->type)) + return; + + if(n->op == OLITERAL) { + nn = nod(OXXX, N, N); + *nn = *n; + n = nn; + *np = n; + } + + switch(n->op) { + default: + if(n->type->etype == TIDEAL) { + convlit(&n->left, t); + convlit(&n->right, t); + n->type = t; + } + return; + case OLITERAL: + // target is invalid type for a constant? leave alone. + if(!okforconst[t->etype] && n->type->etype != TNIL) { + defaultlit(&n, T); + *np = n; + return; + } + break; + case OLSH: + case ORSH: + convlit1(&n->left, t, explicit && isideal(n->left->type)); + t = n->left->type; + if(t != T && t->etype == TIDEAL && n->val.ctype != CTINT) + n->val = toint(n->val); + if(t != T && !isint[t->etype]) { + yyerror("invalid operation: %#N (shift of type %T)", n, t); + t = T; + } + n->type = t; + return; + } + + // avoided repeated calculations, errors + if(eqtype(n->type, t)) + return; + + ct = consttype(n); + if(ct < 0) + goto bad; + + et = t->etype; + if(et == TINTER) { + if(ct == CTNIL && n->type == types[TNIL]) { + n->type = t; + return; + } + defaultlit(np, T); + return; + } + + switch(ct) { + default: + goto bad; + + case CTNIL: + switch(et) { + default: + n->type = T; + goto bad; + + case TSTRING: + // let normal conversion code handle it + return; + + case TARRAY: + if(!isslice(t)) + goto bad; + break; + + case TPTR32: + case TPTR64: + case TINTER: + case TMAP: + case TCHAN: + case TFUNC: + case TUNSAFEPTR: + break; + } + break; + + case CTSTR: + case CTBOOL: + if(et != n->type->etype) + goto bad; + break; + + case CTINT: + case CTFLT: + case CTCPLX: + ct = n->val.ctype; + if(isint[et]) { + switch(ct) { + default: + goto bad; + case CTCPLX: + case CTFLT: + n->val = toint(n->val); + // flowthrough + case CTINT: + overflow(n->val, t); + break; + } + } else + if(isfloat[et]) { + switch(ct) { + default: + goto bad; + case CTCPLX: + case CTINT: + n->val = toflt(n->val); + // flowthrough + case CTFLT: + overflow(n->val, t); + n->val.u.fval = truncfltlit(n->val.u.fval, t); + break; + } + } else + if(iscomplex[et]) { + switch(ct) { + default: + goto bad; + case CTFLT: + case CTINT: + n->val = tocplx(n->val); + break; + case CTCPLX: + overflow(n->val, t); + break; + } + } else + if(et == TSTRING && ct == CTINT && explicit) + n->val = tostr(n->val); + else + goto bad; + break; + } + n->type = t; + return; + +bad: + if(!n->diag) { + yyerror("cannot convert %#N to type %T", n, t); + n->diag = 1; + } + if(isideal(n->type)) { + defaultlit(&n, T); + *np = n; + } + return; +} + +static Val +copyval(Val v) +{ + Mpint *i; + Mpflt *f; + Mpcplx *c; + + switch(v.ctype) { + case CTINT: + i = mal(sizeof(*i)); + mpmovefixfix(i, v.u.xval); + v.u.xval = i; + break; + case CTFLT: + f = mal(sizeof(*f)); + mpmovefltflt(f, v.u.fval); + v.u.fval = f; + break; + case CTCPLX: + c = mal(sizeof(*c)); + mpmovefltflt(&c->real, &v.u.cval->real); + mpmovefltflt(&c->imag, &v.u.cval->imag); + v.u.cval = c; + break; + } + return v; +} + +static Val +tocplx(Val v) +{ + Mpcplx *c; + + switch(v.ctype) { + case CTINT: + c = mal(sizeof(*c)); + mpmovefixflt(&c->real, v.u.xval); + mpmovecflt(&c->imag, 0.0); + v.ctype = CTCPLX; + v.u.cval = c; + break; + case CTFLT: + c = mal(sizeof(*c)); + mpmovefltflt(&c->real, v.u.fval); + mpmovecflt(&c->imag, 0.0); + v.ctype = CTCPLX; + v.u.cval = c; + break; + } + return v; +} + +static Val +toflt(Val v) +{ + Mpflt *f; + + switch(v.ctype) { + case CTINT: + f = mal(sizeof(*f)); + mpmovefixflt(f, v.u.xval); + v.ctype = CTFLT; + v.u.fval = f; + break; + case CTCPLX: + f = mal(sizeof(*f)); + mpmovefltflt(f, &v.u.cval->real); + if(mpcmpfltc(&v.u.cval->imag, 0) != 0) + yyerror("constant %#F%+#Fi truncated to real", &v.u.cval->real, &v.u.cval->imag); + v.ctype = CTFLT; + v.u.fval = f; + break; + } + return v; +} + +Val +toint(Val v) +{ + Mpint *i; + + switch(v.ctype) { + case CTFLT: + i = mal(sizeof(*i)); + if(mpmovefltfix(i, v.u.fval) < 0) + yyerror("constant %#F truncated to integer", v.u.fval); + v.ctype = CTINT; + v.u.xval = i; + break; + case CTCPLX: + i = mal(sizeof(*i)); + if(mpmovefltfix(i, &v.u.cval->real) < 0) + yyerror("constant %#F%+#Fi truncated to integer", &v.u.cval->real, &v.u.cval->imag); + if(mpcmpfltc(&v.u.cval->imag, 0) != 0) + yyerror("constant %#F%+#Fi truncated to real", &v.u.cval->real, &v.u.cval->imag); + v.ctype = CTINT; + v.u.xval = i; + break; + } + return v; +} + +void +overflow(Val v, Type *t) +{ + // v has already been converted + // to appropriate form for t. + if(t == T || t->etype == TIDEAL) + return; + switch(v.ctype) { + case CTINT: + if(!isint[t->etype]) + fatal("overflow: %T integer constant", t); + if(mpcmpfixfix(v.u.xval, minintval[t->etype]) < 0 || + mpcmpfixfix(v.u.xval, maxintval[t->etype]) > 0) + yyerror("constant %B overflows %T", v.u.xval, t); + break; + case CTFLT: + if(!isfloat[t->etype]) + fatal("overflow: %T floating-point constant", t); + if(mpcmpfltflt(v.u.fval, minfltval[t->etype]) <= 0 || + mpcmpfltflt(v.u.fval, maxfltval[t->etype]) >= 0) + yyerror("constant %#F overflows %T", v.u.fval, t); + break; + case CTCPLX: + if(!iscomplex[t->etype]) + fatal("overflow: %T complex constant", t); + if(mpcmpfltflt(&v.u.cval->real, minfltval[t->etype]) <= 0 || + mpcmpfltflt(&v.u.cval->real, maxfltval[t->etype]) >= 0 || + mpcmpfltflt(&v.u.cval->imag, minfltval[t->etype]) <= 0 || + mpcmpfltflt(&v.u.cval->imag, maxfltval[t->etype]) >= 0) + yyerror("constant %#F overflows %T", v.u.fval, t); + break; + } +} + +static Val +tostr(Val v) +{ + Rune rune; + int l; + Strlit *s; + + switch(v.ctype) { + case CTINT: + if(mpcmpfixfix(v.u.xval, minintval[TINT]) < 0 || + mpcmpfixfix(v.u.xval, maxintval[TINT]) > 0) + yyerror("overflow in int -> string"); + rune = mpgetfix(v.u.xval); + l = runelen(rune); + s = mal(sizeof(*s)+l); + s->len = l; + runetochar((char*)s->s, &rune); + memset(&v, 0, sizeof v); + v.ctype = CTSTR; + v.u.sval = s; + break; + + case CTFLT: + yyerror("no float -> string"); + + case CTNIL: + memset(&v, 0, sizeof v); + v.ctype = CTSTR; + v.u.sval = mal(sizeof *s); + break; + } + return v; +} + +int +consttype(Node *n) +{ + if(n == N || n->op != OLITERAL) + return -1; + return n->val.ctype; +} + +int +isconst(Node *n, int ct) +{ + return consttype(n) == ct; +} + +/* + * if n is constant, rewrite as OLITERAL node. + */ +void +evconst(Node *n) +{ + Node *nl, *nr; + int32 len; + Strlit *str; + int wl, wr, lno, et; + Val v, rv; + Mpint b; + + // pick off just the opcodes that can be + // constant evaluated. + switch(n->op) { + default: + return; + case OADD: + case OADDSTR: + case OAND: + case OANDAND: + case OANDNOT: + case OARRAYBYTESTR: + case OCOM: + case ODIV: + case OEQ: + case OGE: + case OGT: + case OLE: + case OLSH: + case OLT: + case OMINUS: + case OMOD: + case OMUL: + case ONE: + case ONOT: + case OOR: + case OOROR: + case OPLUS: + case ORSH: + case OSUB: + case OXOR: + break; + case OCONV: + if(n->type == T) + return; + if(!okforconst[n->type->etype] && n->type->etype != TNIL) + return; + break; + } + + nl = n->left; + if(nl == N || nl->type == T) + return; + if(consttype(nl) < 0) + return; + wl = nl->type->etype; + if(isint[wl] || isfloat[wl] || iscomplex[wl]) + wl = TIDEAL; + + nr = n->right; + if(nr == N) + goto unary; + if(nr->type == T) + return; + if(consttype(nr) < 0) + return; + wr = nr->type->etype; + if(isint[wr] || isfloat[wr] || iscomplex[wr]) + wr = TIDEAL; + + // check for compatible general types (numeric, string, etc) + if(wl != wr) + goto illegal; + + // check for compatible types. + switch(n->op) { + default: + // ideal const mixes with anything but otherwise must match. + if(nl->type->etype != TIDEAL) { + defaultlit(&nr, nl->type); + n->right = nr; + } + if(nr->type->etype != TIDEAL) { + defaultlit(&nl, nr->type); + n->left = nl; + } + if(nl->type->etype != nr->type->etype) + goto illegal; + break; + + case OLSH: + case ORSH: + // right must be unsigned. + // left can be ideal. + defaultlit(&nr, types[TUINT]); + n->right = nr; + if(nr->type && (issigned[nr->type->etype] || !isint[nr->type->etype])) + goto illegal; + nl->val = toint(nl->val); + nr->val = toint(nr->val); + break; + } + + // copy numeric value to avoid modifying + // n->left, in case someone still refers to it (e.g. iota). + v = nl->val; + if(wl == TIDEAL) + v = copyval(v); + + rv = nr->val; + + // convert to common ideal + if(v.ctype == CTCPLX || rv.ctype == CTCPLX) { + v = tocplx(v); + rv = tocplx(rv); + } + if(v.ctype == CTFLT || rv.ctype == CTFLT) { + v = toflt(v); + rv = toflt(rv); + } + if(v.ctype != rv.ctype) { + // Use of undefined name as constant? + if((v.ctype == 0 || rv.ctype == 0) && nerrors > 0) + return; + fatal("constant type mismatch %T(%d) %T(%d)", nl->type, v.ctype, nr->type, rv.ctype); + } + + // run op + switch(TUP(n->op, v.ctype)) { + default: + illegal: + if(!n->diag) { + yyerror("illegal constant expression: %T %O %T", + nl->type, n->op, nr->type); + n->diag = 1; + } + return; + + case TUP(OADD, CTINT): + mpaddfixfix(v.u.xval, rv.u.xval); + break; + case TUP(OSUB, CTINT): + mpsubfixfix(v.u.xval, rv.u.xval); + break; + case TUP(OMUL, CTINT): + mpmulfixfix(v.u.xval, rv.u.xval); + break; + case TUP(ODIV, CTINT): + if(mpcmpfixc(rv.u.xval, 0) == 0) { + yyerror("division by zero"); + mpmovecfix(v.u.xval, 1); + break; + } + mpdivfixfix(v.u.xval, rv.u.xval); + break; + case TUP(OMOD, CTINT): + if(mpcmpfixc(rv.u.xval, 0) == 0) { + yyerror("division by zero"); + mpmovecfix(v.u.xval, 1); + break; + } + mpmodfixfix(v.u.xval, rv.u.xval); + break; + + case TUP(OLSH, CTINT): + mplshfixfix(v.u.xval, rv.u.xval); + break; + case TUP(ORSH, CTINT): + mprshfixfix(v.u.xval, rv.u.xval); + break; + case TUP(OOR, CTINT): + mporfixfix(v.u.xval, rv.u.xval); + break; + case TUP(OAND, CTINT): + mpandfixfix(v.u.xval, rv.u.xval); + break; + case TUP(OANDNOT, CTINT): + mpandnotfixfix(v.u.xval, rv.u.xval); + break; + case TUP(OXOR, CTINT): + mpxorfixfix(v.u.xval, rv.u.xval); + break; + + case TUP(OADD, CTFLT): + mpaddfltflt(v.u.fval, rv.u.fval); + break; + case TUP(OSUB, CTFLT): + mpsubfltflt(v.u.fval, rv.u.fval); + break; + case TUP(OMUL, CTFLT): + mpmulfltflt(v.u.fval, rv.u.fval); + break; + case TUP(ODIV, CTFLT): + if(mpcmpfltc(rv.u.fval, 0) == 0) { + yyerror("division by zero"); + mpmovecflt(v.u.fval, 1.0); + break; + } + mpdivfltflt(v.u.fval, rv.u.fval); + break; + + case TUP(OADD, CTCPLX): + mpaddfltflt(&v.u.cval->real, &rv.u.cval->real); + mpaddfltflt(&v.u.cval->imag, &rv.u.cval->imag); + break; + case TUP(OSUB, CTCPLX): + mpsubfltflt(&v.u.cval->real, &rv.u.cval->real); + mpsubfltflt(&v.u.cval->imag, &rv.u.cval->imag); + break; + case TUP(OMUL, CTCPLX): + cmplxmpy(v.u.cval, rv.u.cval); + break; + case TUP(ODIV, CTCPLX): + if(mpcmpfltc(&rv.u.cval->real, 0) == 0 && + mpcmpfltc(&rv.u.cval->imag, 0) == 0) { + yyerror("complex division by zero"); + mpmovecflt(&rv.u.cval->real, 1.0); + mpmovecflt(&rv.u.cval->imag, 0.0); + break; + } + cmplxdiv(v.u.cval, rv.u.cval); + break; + + case TUP(OEQ, CTNIL): + goto settrue; + case TUP(ONE, CTNIL): + goto setfalse; + + case TUP(OEQ, CTINT): + if(mpcmpfixfix(v.u.xval, rv.u.xval) == 0) + goto settrue; + goto setfalse; + case TUP(ONE, CTINT): + if(mpcmpfixfix(v.u.xval, rv.u.xval) != 0) + goto settrue; + goto setfalse; + case TUP(OLT, CTINT): + if(mpcmpfixfix(v.u.xval, rv.u.xval) < 0) + goto settrue; + goto setfalse; + case TUP(OLE, CTINT): + if(mpcmpfixfix(v.u.xval, rv.u.xval) <= 0) + goto settrue; + goto setfalse; + case TUP(OGE, CTINT): + if(mpcmpfixfix(v.u.xval, rv.u.xval) >= 0) + goto settrue; + goto setfalse; + case TUP(OGT, CTINT): + if(mpcmpfixfix(v.u.xval, rv.u.xval) > 0) + goto settrue; + goto setfalse; + + case TUP(OEQ, CTFLT): + if(mpcmpfltflt(v.u.fval, rv.u.fval) == 0) + goto settrue; + goto setfalse; + case TUP(ONE, CTFLT): + if(mpcmpfltflt(v.u.fval, rv.u.fval) != 0) + goto settrue; + goto setfalse; + case TUP(OLT, CTFLT): + if(mpcmpfltflt(v.u.fval, rv.u.fval) < 0) + goto settrue; + goto setfalse; + case TUP(OLE, CTFLT): + if(mpcmpfltflt(v.u.fval, rv.u.fval) <= 0) + goto settrue; + goto setfalse; + case TUP(OGE, CTFLT): + if(mpcmpfltflt(v.u.fval, rv.u.fval) >= 0) + goto settrue; + goto setfalse; + case TUP(OGT, CTFLT): + if(mpcmpfltflt(v.u.fval, rv.u.fval) > 0) + goto settrue; + goto setfalse; + + case TUP(OEQ, CTCPLX): + if(mpcmpfltflt(&v.u.cval->real, &rv.u.cval->real) == 0 && + mpcmpfltflt(&v.u.cval->imag, &rv.u.cval->imag) == 0) + goto settrue; + goto setfalse; + case TUP(ONE, CTCPLX): + if(mpcmpfltflt(&v.u.cval->real, &rv.u.cval->real) != 0 || + mpcmpfltflt(&v.u.cval->imag, &rv.u.cval->imag) != 0) + goto settrue; + goto setfalse; + + case TUP(OEQ, CTSTR): + if(cmpslit(nl, nr) == 0) + goto settrue; + goto setfalse; + case TUP(ONE, CTSTR): + if(cmpslit(nl, nr) != 0) + goto settrue; + goto setfalse; + case TUP(OLT, CTSTR): + if(cmpslit(nl, nr) < 0) + goto settrue; + goto setfalse; + case TUP(OLE, CTSTR): + if(cmpslit(nl, nr) <= 0) + goto settrue; + goto setfalse; + case TUP(OGE, CTSTR): + if(cmpslit(nl, nr) >= 0l) + goto settrue; + goto setfalse; + case TUP(OGT, CTSTR): + if(cmpslit(nl, nr) > 0) + goto settrue; + goto setfalse; + case TUP(OADDSTR, CTSTR): + len = v.u.sval->len + rv.u.sval->len; + str = mal(sizeof(*str) + len); + str->len = len; + memcpy(str->s, v.u.sval->s, v.u.sval->len); + memcpy(str->s+v.u.sval->len, rv.u.sval->s, rv.u.sval->len); + str->len = len; + v.u.sval = str; + break; + + case TUP(OOROR, CTBOOL): + if(v.u.bval || rv.u.bval) + goto settrue; + goto setfalse; + case TUP(OANDAND, CTBOOL): + if(v.u.bval && rv.u.bval) + goto settrue; + goto setfalse; + case TUP(OEQ, CTBOOL): + if(v.u.bval == rv.u.bval) + goto settrue; + goto setfalse; + case TUP(ONE, CTBOOL): + if(v.u.bval != rv.u.bval) + goto settrue; + goto setfalse; + } + goto ret; + +unary: + // copy numeric value to avoid modifying + // nl, in case someone still refers to it (e.g. iota). + v = nl->val; + if(wl == TIDEAL) + v = copyval(v); + + switch(TUP(n->op, v.ctype)) { + default: + if(!n->diag) { + yyerror("illegal constant expression %O %T", n->op, nl->type); + n->diag = 1; + } + return; + + case TUP(OCONV, CTNIL): + case TUP(OARRAYBYTESTR, CTNIL): + if(n->type->etype == TSTRING) { + v = tostr(v); + nl->type = n->type; + break; + } + // fall through + case TUP(OCONV, CTINT): + case TUP(OCONV, CTFLT): + case TUP(OCONV, CTSTR): + convlit1(&nl, n->type, 1); + break; + + case TUP(OPLUS, CTINT): + break; + case TUP(OMINUS, CTINT): + mpnegfix(v.u.xval); + break; + case TUP(OCOM, CTINT): + et = Txxx; + if(nl->type != T) + et = nl->type->etype; + + // calculate the mask in b + // result will be (a ^ mask) + switch(et) { + default: + // signed guys change sign + mpmovecfix(&b, -1); + break; + + case TUINT8: + case TUINT16: + case TUINT32: + case TUINT64: + case TUINT: + case TUINTPTR: + // unsigned guys invert their bits + mpmovefixfix(&b, maxintval[et]); + break; + } + mpxorfixfix(v.u.xval, &b); + break; + + case TUP(OPLUS, CTFLT): + break; + case TUP(OMINUS, CTFLT): + mpnegflt(v.u.fval); + break; + + case TUP(OPLUS, CTCPLX): + break; + case TUP(OMINUS, CTCPLX): + mpnegflt(&v.u.cval->real); + mpnegflt(&v.u.cval->imag); + break; + + case TUP(ONOT, CTBOOL): + if(!v.u.bval) + goto settrue; + goto setfalse; + } + +ret: + // rewrite n in place. + *n = *nl; + n->val = v; + + // check range. + lno = setlineno(n); + overflow(v, n->type); + lineno = lno; + + // truncate precision for non-ideal float. + if(v.ctype == CTFLT && n->type->etype != TIDEAL) + n->val.u.fval = truncfltlit(v.u.fval, n->type); + return; + +settrue: + *n = *nodbool(1); + return; + +setfalse: + *n = *nodbool(0); + return; +} + +Node* +nodlit(Val v) +{ + Node *n; + + n = nod(OLITERAL, N, N); + n->val = v; + switch(v.ctype) { + default: + fatal("nodlit ctype %d", v.ctype); + case CTSTR: + n->type = idealstring; + break; + case CTBOOL: + n->type = idealbool; + break; + case CTINT: + case CTFLT: + case CTCPLX: + n->type = types[TIDEAL]; + break; + case CTNIL: + n->type = types[TNIL]; + break; + } + return n; +} + +Node* +nodcplxlit(Val r, Val i) +{ + Node *n; + Mpcplx *c; + + r = toflt(r); + i = toflt(i); + + c = mal(sizeof(*c)); + n = nod(OLITERAL, N, N); + n->type = types[TIDEAL]; + n->val.u.cval = c; + n->val.ctype = CTCPLX; + + if(r.ctype != CTFLT || i.ctype != CTFLT) + fatal("nodcplxlit ctype %d/%d", r.ctype, i.ctype); + + mpmovefltflt(&c->real, r.u.fval); + mpmovefltflt(&c->imag, i.u.fval); + return n; +} + +// TODO(rsc): combine with convlit +void +defaultlit(Node **np, Type *t) +{ + int lno; + Node *n, *nn; + + n = *np; + if(n == N || !isideal(n->type)) + return; + + switch(n->op) { + case OLITERAL: + nn = nod(OXXX, N, N); + *nn = *n; + n = nn; + *np = n; + break; + case OLSH: + case ORSH: + defaultlit(&n->left, t); + t = n->left->type; + if(t != T && !isint[t->etype]) { + yyerror("invalid operation: %#N (shift of type %T)", n, t); + t = T; + } + n->type = t; + return; + default: + if(n->left == N) { + dump("defaultlit", n); + fatal("defaultlit"); + } + // n is ideal, so left and right must both be ideal. + // n has not been computed as a constant value, + // so either left or right must not be constant. + // The only 'ideal' non-constant expressions are shifts. Ugh. + // If one of these is a shift and the other is not, use that type. + // When compiling x := 1<right->op == OLSH || n->right->op == ORSH)) { + defaultlit(&n->left, T); + defaultlit(&n->right, n->left->type); + } else if(t == T && (n->left->op == OLSH || n->left->op == ORSH)) { + defaultlit(&n->right, T); + defaultlit(&n->left, n->right->type); + } else { + defaultlit(&n->left, t); + defaultlit(&n->right, t); + } + if(n->type == idealbool || n->type == idealstring) + n->type = types[n->type->etype]; + else + n->type = n->left->type; + return; + } + + lno = setlineno(n); + switch(n->val.ctype) { + default: + if(t != T) { + convlit(np, t); + break; + } + if(n->val.ctype == CTNIL) { + lineno = lno; + yyerror("use of untyped nil"); + n->type = T; + break; + } + if(n->val.ctype == CTSTR) { + n->type = types[TSTRING]; + break; + } + yyerror("defaultlit: unknown literal: %#N", n); + break; + case CTBOOL: + n->type = types[TBOOL]; + if(t != T && t->etype == TBOOL) + n->type = t; + break; + case CTINT: + n->type = types[TINT]; + goto num; + case CTFLT: + n->type = types[TFLOAT64]; + goto num; + case CTCPLX: + n->type = types[TCOMPLEX128]; + goto num; + num: + if(t != T) { + if(isint[t->etype]) { + n->type = t; + n->val = toint(n->val); + } + else + if(isfloat[t->etype]) { + n->type = t; + n->val = toflt(n->val); + } + else + if(iscomplex[t->etype]) { + n->type = t; + n->val = tocplx(n->val); + } + } + overflow(n->val, n->type); + break; + } + lineno = lno; +} + +/* + * defaultlit on both nodes simultaneously; + * if they're both ideal going in they better + * get the same type going out. + * force means must assign concrete (non-ideal) type. + */ +void +defaultlit2(Node **lp, Node **rp, int force) +{ + Node *l, *r; + + l = *lp; + r = *rp; + if(l->type == T || r->type == T) + return; + if(!isideal(l->type)) { + convlit(rp, l->type); + return; + } + if(!isideal(r->type)) { + convlit(lp, r->type); + return; + } + if(!force) + return; + if(isconst(l, CTCPLX) || isconst(r, CTCPLX)) { + convlit(lp, types[TCOMPLEX128]); + convlit(rp, types[TCOMPLEX128]); + return; + } + if(isconst(l, CTFLT) || isconst(r, CTFLT)) { + convlit(lp, types[TFLOAT64]); + convlit(rp, types[TFLOAT64]); + return; + } + convlit(lp, types[TINT]); + convlit(rp, types[TINT]); +} + +int +cmpslit(Node *l, Node *r) +{ + int32 l1, l2, i, m; + uchar *s1, *s2; + + l1 = l->val.u.sval->len; + l2 = r->val.u.sval->len; + s1 = (uchar*)l->val.u.sval->s; + s2 = (uchar*)r->val.u.sval->s; + + m = l1; + if(l2 < m) + m = l2; + + for(i=0; i s2[i]) + return +1; + return -1; + } + if(l1 == l2) + return 0; + if(l1 > l2) + return +1; + return -1; +} + +int +smallintconst(Node *n) +{ + if(n->op == OLITERAL && n->type != T) + switch(simtype[n->type->etype]) { + case TINT8: + case TUINT8: + case TINT16: + case TUINT16: + case TINT32: + case TUINT32: + case TBOOL: + case TPTR32: + return 1; + case TINT64: + case TUINT64: + if(mpcmpfixfix(n->val.u.xval, minintval[TINT32]) < 0 + || mpcmpfixfix(n->val.u.xval, maxintval[TINT32]) > 0) + break; + return 1; + } + return 0; +} + +long +nonnegconst(Node *n) +{ + if(n->op == OLITERAL && n->type != T) + switch(simtype[n->type->etype]) { + case TINT8: + case TUINT8: + case TINT16: + case TUINT16: + case TINT32: + case TUINT32: + case TINT64: + case TUINT64: + case TIDEAL: + // check negative and 2^31 + if(mpcmpfixfix(n->val.u.xval, minintval[TUINT32]) < 0 + || mpcmpfixfix(n->val.u.xval, maxintval[TINT32]) > 0) + break; + return mpgetfix(n->val.u.xval); + } + return -1; +} + +/* + * convert x to type et and back to int64 + * for sign extension and truncation. + */ +static int64 +iconv(int64 x, int et) +{ + switch(et) { + case TINT8: + x = (int8)x; + break; + case TUINT8: + x = (uint8)x; + break; + case TINT16: + x = (int16)x; + break; + case TUINT16: + x = (uint64)x; + break; + case TINT32: + x = (int32)x; + break; + case TUINT32: + x = (uint32)x; + break; + case TINT64: + case TUINT64: + break; + } + return x; +} + +/* + * convert constant val to type t; leave in con. + * for back end. + */ +void +convconst(Node *con, Type *t, Val *val) +{ + int64 i; + int tt; + + tt = simsimtype(t); + + // copy the constant for conversion + nodconst(con, types[TINT8], 0); + con->type = t; + con->val = *val; + + if(isint[tt]) { + con->val.ctype = CTINT; + con->val.u.xval = mal(sizeof *con->val.u.xval); + switch(val->ctype) { + default: + fatal("convconst ctype=%d %lT", val->ctype, t); + case CTINT: + i = mpgetfix(val->u.xval); + break; + case CTBOOL: + i = val->u.bval; + break; + case CTNIL: + i = 0; + break; + } + i = iconv(i, tt); + mpmovecfix(con->val.u.xval, i); + return; + } + + if(isfloat[tt]) { + con->val = toflt(con->val); + if(con->val.ctype != CTFLT) + fatal("convconst ctype=%d %T", con->val.ctype, t); + if(tt == TFLOAT32) + con->val.u.fval = truncfltlit(con->val.u.fval, t); + return; + } + + if(iscomplex[tt]) { + con->val = tocplx(con->val); + if(tt == TCOMPLEX64) { + con->val.u.cval->real = *truncfltlit(&con->val.u.cval->real, types[TFLOAT32]); + con->val.u.cval->imag = *truncfltlit(&con->val.u.cval->imag, types[TFLOAT32]); + } + return; + } + + fatal("convconst %lT constant", t); + +} + +// complex multiply v *= rv +// (a, b) * (c, d) = (a*c - b*d, b*c + a*d) +static void +cmplxmpy(Mpcplx *v, Mpcplx *rv) +{ + Mpflt ac, bd, bc, ad; + + mpmovefltflt(&ac, &v->real); + mpmulfltflt(&ac, &rv->real); // ac + + mpmovefltflt(&bd, &v->imag); + mpmulfltflt(&bd, &rv->imag); // bd + + mpmovefltflt(&bc, &v->imag); + mpmulfltflt(&bc, &rv->real); // bc + + mpmovefltflt(&ad, &v->real); + mpmulfltflt(&ad, &rv->imag); // ad + + mpmovefltflt(&v->real, &ac); + mpsubfltflt(&v->real, &bd); // ac-bd + + mpmovefltflt(&v->imag, &bc); + mpaddfltflt(&v->imag, &ad); // bc+ad +} + +// complex divide v /= rv +// (a, b) / (c, d) = ((a*c + b*d), (b*c - a*d))/(c*c + d*d) +static void +cmplxdiv(Mpcplx *v, Mpcplx *rv) +{ + Mpflt ac, bd, bc, ad, cc_plus_dd; + + mpmovefltflt(&cc_plus_dd, &rv->real); + mpmulfltflt(&cc_plus_dd, &rv->real); // cc + + mpmovefltflt(&ac, &rv->imag); + mpmulfltflt(&ac, &rv->imag); // dd + + mpaddfltflt(&cc_plus_dd, &ac); // cc+dd + + mpmovefltflt(&ac, &v->real); + mpmulfltflt(&ac, &rv->real); // ac + + mpmovefltflt(&bd, &v->imag); + mpmulfltflt(&bd, &rv->imag); // bd + + mpmovefltflt(&bc, &v->imag); + mpmulfltflt(&bc, &rv->real); // bc + + mpmovefltflt(&ad, &v->real); + mpmulfltflt(&ad, &rv->imag); // ad + + mpmovefltflt(&v->real, &ac); + mpaddfltflt(&v->real, &bd); // ac+bd + mpdivfltflt(&v->real, &cc_plus_dd); // (ac+bd)/(cc+dd) + + mpmovefltflt(&v->imag, &bc); + mpsubfltflt(&v->imag, &ad); // bc-ad + mpdivfltflt(&v->imag, &cc_plus_dd); // (bc+ad)/(cc+dd) +} diff --git a/src/cmd/gc/cplx.c b/src/cmd/gc/cplx.c new file mode 100644 index 000000000..890cf7f10 --- /dev/null +++ b/src/cmd/gc/cplx.c @@ -0,0 +1,479 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +#include "gg.h" + +static void subnode(Node *nr, Node *ni, Node *nc); +static void minus(Node *nl, Node *res); + void complexminus(Node*, Node*); + void complexadd(int op, Node*, Node*, Node*); + void complexmul(Node*, Node*, Node*); + +#define CASE(a,b) (((a)<<16)|((b)<<0)) + +static int +overlap(Node *f, Node *t) +{ + // check whether f and t could be overlapping stack references. + // not exact, because it's hard to check for the stack register + // in portable code. close enough: worst case we will allocate + // an extra temporary and the registerizer will clean it up. + return f->op == OINDREG && + t->op == OINDREG && + f->xoffset+f->type->width >= t->xoffset && + t->xoffset+t->type->width >= f->xoffset; +} + +/* + * generate: + * res = n; + * simplifies and calls gmove. + */ +void +complexmove(Node *f, Node *t) +{ + int ft, tt; + Node n1, n2, n3, n4; + + if(debug['g']) { + dump("\ncomplexmove-f", f); + dump("complexmove-t", t); + } + + if(!t->addable) + fatal("complexmove: to not addable"); + + ft = simsimtype(f->type); + tt = simsimtype(t->type); + switch(CASE(ft,tt)) { + + default: + fatal("complexmove: unknown conversion: %T -> %T\n", + f->type, t->type); + + case CASE(TCOMPLEX64,TCOMPLEX64): + case CASE(TCOMPLEX64,TCOMPLEX128): + case CASE(TCOMPLEX128,TCOMPLEX64): + case CASE(TCOMPLEX128,TCOMPLEX128): + // complex to complex move/convert. + // make f addable. + // also use temporary if possible stack overlap. + if(!f->addable || overlap(f, t)) { + tempname(&n1, f->type); + complexmove(f, &n1); + f = &n1; + } + + subnode(&n1, &n2, f); + subnode(&n3, &n4, t); + + cgen(&n1, &n3); + cgen(&n2, &n4); + break; + } +} + +int +complexop(Node *n, Node *res) +{ + if(n != N && n->type != T) + if(iscomplex[n->type->etype]) { + goto maybe; + } + if(res != N && res->type != T) + if(iscomplex[res->type->etype]) { + goto maybe; + } + + if(n->op == OREAL || n->op == OIMAG) + goto yes; + + goto no; + +maybe: + switch(n->op) { + case OCONV: // implemented ops + case OADD: + case OSUB: + case OMUL: + case OMINUS: + case OCOMPLEX: + case OREAL: + case OIMAG: + goto yes; + + case ODOT: + case ODOTPTR: + case OINDEX: + case OIND: + case ONAME: + goto yes; + } + +no: +//dump("\ncomplex-no", n); + return 0; +yes: +//dump("\ncomplex-yes", n); + return 1; +} + +void +complexgen(Node *n, Node *res) +{ + Node *nl, *nr; + Node tnl, tnr; + Node n1, n2, tmp; + int tl, tr; + + if(debug['g']) { + dump("\ncomplexgen-n", n); + dump("complexgen-res", res); + } + + // pick off float/complex opcodes + switch(n->op) { + case OCOMPLEX: + if(res->addable) { + subnode(&n1, &n2, res); + tempname(&tmp, n1.type); + cgen(n->left, &tmp); + cgen(n->right, &n2); + cgen(&tmp, &n1); + return; + } + break; + + case OREAL: + case OIMAG: + nl = n->left; + if(!nl->addable) { + tempname(&tmp, nl->type); + complexgen(nl, &tmp); + nl = &tmp; + } + subnode(&n1, &n2, nl); + if(n->op == OREAL) { + cgen(&n1, res); + return; + } + cgen(&n2, res); + return; + } + + // perform conversion from n to res + tl = simsimtype(res->type); + tl = cplxsubtype(tl); + tr = simsimtype(n->type); + tr = cplxsubtype(tr); + if(tl != tr) { + if(!n->addable) { + tempname(&n1, n->type); + complexmove(n, &n1); + n = &n1; + } + complexmove(n, res); + return; + } + + if(!res->addable) { + igen(res, &n1, N); + cgen(n, &n1); + regfree(&n1); + return; + } + if(n->addable) { + complexmove(n, res); + return; + } + + switch(n->op) { + default: + dump("complexgen: unknown op", n); + fatal("complexgen: unknown op %O", n->op); + + case ODOT: + case ODOTPTR: + case OINDEX: + case OIND: + case ONAME: // PHEAP or PPARAMREF var + case OCALLFUNC: + igen(n, &n1, res); + complexmove(&n1, res); + regfree(&n1); + return; + + case OCONV: + case OADD: + case OSUB: + case OMUL: + case OMINUS: + case OCOMPLEX: + case OREAL: + case OIMAG: + break; + } + + nl = n->left; + if(nl == N) + return; + nr = n->right; + + // make both sides addable in ullman order + if(nr != N) { + if(nl->ullman > nr->ullman && !nl->addable) { + tempname(&tnl, nl->type); + cgen(nl, &tnl); + nl = &tnl; + } + if(!nr->addable) { + tempname(&tnr, nr->type); + cgen(nr, &tnr); + nr = &tnr; + } + } + if(!nl->addable) { + tempname(&tnl, nl->type); + cgen(nl, &tnl); + nl = &tnl; + } + + switch(n->op) { + default: + fatal("complexgen: unknown op %O", n->op); + break; + + case OCONV: + complexmove(nl, res); + break; + + case OMINUS: + complexminus(nl, res); + break; + + case OADD: + case OSUB: + complexadd(n->op, nl, nr, res); + break; + + case OMUL: + complexmul(nl, nr, res); + break; + } +} + +void +complexbool(int op, Node *nl, Node *nr, int true, Prog *to) +{ + Node tnl, tnr; + Node n1, n2, n3, n4; + Node na, nb, nc; + + // make both sides addable in ullman order + if(nr != N) { + if(nl->ullman > nr->ullman && !nl->addable) { + tempname(&tnl, nl->type); + cgen(nl, &tnl); + nl = &tnl; + } + if(!nr->addable) { + tempname(&tnr, nr->type); + cgen(nr, &tnr); + nr = &tnr; + } + } + if(!nl->addable) { + tempname(&tnl, nl->type); + cgen(nl, &tnl); + nl = &tnl; + } + + // build tree + // real(l) == real(r) && imag(l) == imag(r) + + subnode(&n1, &n2, nl); + subnode(&n3, &n4, nr); + + memset(&na, 0, sizeof(na)); + na.op = OANDAND; + na.left = &nb; + na.right = &nc; + na.type = types[TBOOL]; + + memset(&nb, 0, sizeof(na)); + nb.op = OEQ; + nb.left = &n1; + nb.right = &n3; + nb.type = types[TBOOL]; + + memset(&nc, 0, sizeof(na)); + nc.op = OEQ; + nc.left = &n2; + nc.right = &n4; + nc.type = types[TBOOL]; + + if(op == ONE) + true = !true; + + bgen(&na, true, to); +} + +void +nodfconst(Node *n, Type *t, Mpflt* fval) +{ + memset(n, 0, sizeof(*n)); + n->op = OLITERAL; + n->addable = 1; + ullmancalc(n); + n->val.u.fval = fval; + n->val.ctype = CTFLT; + n->type = t; + + if(!isfloat[t->etype]) + fatal("nodfconst: bad type %T", t); +} + +// break addable nc-complex into nr-real and ni-imaginary +static void +subnode(Node *nr, Node *ni, Node *nc) +{ + int tc; + Type *t; + + if(!nc->addable) + fatal("subnode not addable"); + + tc = simsimtype(nc->type); + tc = cplxsubtype(tc); + t = types[tc]; + + if(nc->op == OLITERAL) { + nodfconst(nr, t, &nc->val.u.cval->real); + nodfconst(ni, t, &nc->val.u.cval->imag); + return; + } + + *nr = *nc; + nr->type = t; + + *ni = *nc; + ni->type = t; + ni->xoffset += t->width; +} + +// generate code res = -nl +static void +minus(Node *nl, Node *res) +{ + Node ra; + + memset(&ra, 0, sizeof(ra)); + ra.op = OMINUS; + ra.left = nl; + ra.type = nl->type; + cgen(&ra, res); +} + +// build and execute tree +// real(res) = -real(nl) +// imag(res) = -imag(nl) +void +complexminus(Node *nl, Node *res) +{ + Node n1, n2, n5, n6; + + subnode(&n1, &n2, nl); + subnode(&n5, &n6, res); + + minus(&n1, &n5); + minus(&n2, &n6); +} + + +// build and execute tree +// real(res) = real(nl) op real(nr) +// imag(res) = imag(nl) op imag(nr) +void +complexadd(int op, Node *nl, Node *nr, Node *res) +{ + Node n1, n2, n3, n4, n5, n6; + Node ra; + + subnode(&n1, &n2, nl); + subnode(&n3, &n4, nr); + subnode(&n5, &n6, res); + + memset(&ra, 0, sizeof(ra)); + ra.op = op; + ra.left = &n1; + ra.right = &n3; + ra.type = n1.type; + cgen(&ra, &n5); + + memset(&ra, 0, sizeof(ra)); + ra.op = op; + ra.left = &n2; + ra.right = &n4; + ra.type = n2.type; + cgen(&ra, &n6); +} + +// build and execute tree +// tmp = real(nl)*real(nr) - imag(nl)*imag(nr) +// imag(res) = real(nl)*imag(nr) + imag(nl)*real(nr) +// real(res) = tmp +void +complexmul(Node *nl, Node *nr, Node *res) +{ + Node n1, n2, n3, n4, n5, n6; + Node rm1, rm2, ra, tmp; + + subnode(&n1, &n2, nl); + subnode(&n3, &n4, nr); + subnode(&n5, &n6, res); + tempname(&tmp, n5.type); + + // real part -> tmp + memset(&rm1, 0, sizeof(ra)); + rm1.op = OMUL; + rm1.left = &n1; + rm1.right = &n3; + rm1.type = n1.type; + + memset(&rm2, 0, sizeof(ra)); + rm2.op = OMUL; + rm2.left = &n2; + rm2.right = &n4; + rm2.type = n2.type; + + memset(&ra, 0, sizeof(ra)); + ra.op = OSUB; + ra.left = &rm1; + ra.right = &rm2; + ra.type = rm1.type; + cgen(&ra, &tmp); + + // imag part + memset(&rm1, 0, sizeof(ra)); + rm1.op = OMUL; + rm1.left = &n1; + rm1.right = &n4; + rm1.type = n1.type; + + memset(&rm2, 0, sizeof(ra)); + rm2.op = OMUL; + rm2.left = &n2; + rm2.right = &n3; + rm2.type = n2.type; + + memset(&ra, 0, sizeof(ra)); + ra.op = OADD; + ra.left = &rm1; + ra.right = &rm2; + ra.type = rm1.type; + cgen(&ra, &n6); + + // tmp ->real part + cgen(&tmp, &n5); +} diff --git a/src/cmd/gc/dcl.c b/src/cmd/gc/dcl.c new file mode 100644 index 000000000..5bfeeb97a --- /dev/null +++ b/src/cmd/gc/dcl.c @@ -0,0 +1,1254 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +#include "go.h" +#include "y.tab.h" + +static void funcargs(Node*); + +static int +dflag(void) +{ + if(!debug['d']) + return 0; + if(debug['y']) + return 1; + if(incannedimport) + return 0; + return 1; +} + +/* + * declaration stack & operations + */ + +static void +dcopy(Sym *a, Sym *b) +{ + a->pkg = b->pkg; + a->name = b->name; + a->def = b->def; + a->block = b->block; + a->lastlineno = b->lastlineno; +} + +static Sym* +push(void) +{ + Sym *d; + + d = mal(sizeof(*d)); + d->lastlineno = lineno; + d->link = dclstack; + dclstack = d; + return d; +} + +static Sym* +pushdcl(Sym *s) +{ + Sym *d; + + d = push(); + dcopy(d, s); + if(dflag()) + print("\t%L push %S %p\n", lineno, s, s->def); + return d; +} + +void +popdcl(void) +{ + Sym *d, *s; + int lno; + +// if(dflag()) +// print("revert\n"); + + for(d=dclstack; d!=S; d=d->link) { + if(d->name == nil) + break; + s = pkglookup(d->name, d->pkg); + lno = s->lastlineno; + dcopy(s, d); + d->lastlineno = lno; + if(dflag()) + print("\t%L pop %S %p\n", lineno, s, s->def); + } + if(d == S) + fatal("popdcl: no mark"); + dclstack = d->link; + block = d->block; +} + +void +poptodcl(void) +{ + // pop the old marker and push a new one + // (cannot reuse the existing one) + // because we use the markers to identify blocks + // for the goto restriction checks. + popdcl(); + markdcl(); +} + +void +markdcl(void) +{ + Sym *d; + + d = push(); + d->name = nil; // used as a mark in fifo + d->block = block; + + blockgen++; + block = blockgen; + +// if(dflag()) +// print("markdcl\n"); +} + +void +dumpdcl(char *st) +{ + Sym *s, *d; + int i; + + i = 0; + for(d=dclstack; d!=S; d=d->link) { + i++; + print(" %.2d %p", i, d); + if(d->name == nil) { + print("\n"); + continue; + } + print(" '%s'", d->name); + s = pkglookup(d->name, d->pkg); + print(" %lS\n", s); + } +} + +void +testdclstack(void) +{ + Sym *d; + + for(d=dclstack; d!=S; d=d->link) { + if(d->name == nil) { + yyerror("mark left on the stack"); + continue; + } + } +} + +void +redeclare(Sym *s, char *where) +{ + if(s->lastlineno == 0) + yyerror("%S redeclared %s\n" + "\tprevious declaration during import", + s, where); + else + yyerror("%S redeclared %s\n" + "\tprevious declaration at %L", + s, where, s->lastlineno); +} + +/* + * declare individual names - var, typ, const + */ +void +declare(Node *n, int ctxt) +{ + Sym *s; + int gen; + static int typegen, vargen; + + if(isblank(n)) + return; + + n->lineno = parserline(); + s = n->sym; + gen = 0; + if(ctxt == PEXTERN) { + externdcl = list(externdcl, n); + if(dflag()) + print("\t%L global decl %S %p\n", lineno, s, n); + } else { + if(curfn == nil && ctxt == PAUTO) + fatal("automatic outside function"); + if(curfn != nil) + curfn->dcl = list(curfn->dcl, n); + if(n->op == OTYPE) + gen = ++typegen; + else if(n->op == ONAME) + gen = ++vargen; + pushdcl(s); + n->curfn = curfn; + } + if(ctxt == PAUTO) + n->xoffset = BADWIDTH; + + if(s->block == block) + redeclare(s, "in this block"); + + s->block = block; + s->lastlineno = parserline(); + s->def = n; + n->vargen = gen; + n->funcdepth = funcdepth; + n->class = ctxt; + + autoexport(n, ctxt); +} + +void +addvar(Node *n, Type *t, int ctxt) +{ + if(n==N || n->sym == S || (n->op != ONAME && n->op != ONONAME) || t == T) + fatal("addvar: n=%N t=%T nil", n, t); + + n->op = ONAME; + declare(n, ctxt); + n->type = t; +} + +/* + * declare variables from grammar + * new_name_list (type | [type] = expr_list) + */ +NodeList* +variter(NodeList *vl, Node *t, NodeList *el) +{ + int doexpr; + Node *v, *e, *as2; + NodeList *init; + + init = nil; + doexpr = el != nil; + + if(count(el) == 1 && count(vl) > 1) { + e = el->n; + as2 = nod(OAS2, N, N); + as2->list = vl; + as2->rlist = list1(e); + for(; vl; vl=vl->next) { + v = vl->n; + v->op = ONAME; + declare(v, dclcontext); + v->ntype = t; + v->defn = as2; + if(funcdepth > 0) + init = list(init, nod(ODCL, v, N)); + } + return list(init, as2); + } + + for(; vl; vl=vl->next) { + if(doexpr) { + if(el == nil) { + yyerror("missing expr in var dcl"); + break; + } + e = el->n; + el = el->next; + } else + e = N; + + v = vl->n; + v->op = ONAME; + declare(v, dclcontext); + v->ntype = t; + + if(e != N || funcdepth > 0 || isblank(v)) { + if(funcdepth > 0) + init = list(init, nod(ODCL, v, N)); + e = nod(OAS, v, e); + init = list(init, e); + if(e->right != N) + v->defn = e; + } + } + if(el != nil) + yyerror("extra expr in var dcl"); + return init; +} + +/* + * declare constants from grammar + * new_name_list [[type] = expr_list] + */ +NodeList* +constiter(NodeList *vl, Node *t, NodeList *cl) +{ + Node *v, *c; + NodeList *vv; + + vv = nil; + if(cl == nil) { + if(t != N) + yyerror("constdcl cannot have type without expr"); + cl = lastconst; + t = lasttype; + } else { + lastconst = cl; + lasttype = t; + } + cl = listtreecopy(cl); + + for(; vl; vl=vl->next) { + if(cl == nil) { + yyerror("missing expr in const dcl"); + break; + } + c = cl->n; + cl = cl->next; + + v = vl->n; + v->op = OLITERAL; + declare(v, dclcontext); + + v->ntype = t; + v->defn = c; + + vv = list(vv, nod(ODCLCONST, v, N)); + } + if(cl != nil) + yyerror("extra expr in const dcl"); + iota += 1; + return vv; +} + +/* + * this generates a new name node, + * typically for labels or other one-off names. + */ +Node* +newname(Sym *s) +{ + Node *n; + + if(s == S) + fatal("newname nil"); + + n = nod(ONAME, N, N); + n->sym = s; + n->type = T; + n->addable = 1; + n->ullman = 1; + n->xoffset = 0; + return n; +} + +/* + * this generates a new name node for a name + * being declared. + */ +Node* +dclname(Sym *s) +{ + Node *n; + + n = newname(s); + n->op = ONONAME; // caller will correct it + return n; +} + +Node* +typenod(Type *t) +{ + // if we copied another type with *t = *u + // then t->nod might be out of date, so + // check t->nod->type too + if(t->nod == N || t->nod->type != t) { + t->nod = nod(OTYPE, N, N); + t->nod->type = t; + t->nod->sym = t->sym; + } + return t->nod; +} + + +/* + * this will return an old name + * that has already been pushed on the + * declaration list. a diagnostic is + * generated if no name has been defined. + */ +Node* +oldname(Sym *s) +{ + Node *n; + Node *c; + + n = s->def; + if(n == N) { + // maybe a top-level name will come along + // to give this a definition later. + // walkdef will check s->def again once + // all the input source has been processed. + n = newname(s); + n->op = ONONAME; + n->iota = iota; // save current iota value in const declarations + } + if(curfn != nil && n->funcdepth > 0 && n->funcdepth != funcdepth && n->op == ONAME) { + // inner func is referring to var in outer func. + // + // TODO(rsc): If there is an outer variable x and we + // are parsing x := 5 inside the closure, until we get to + // the := it looks like a reference to the outer x so we'll + // make x a closure variable unnecessarily. + if(n->closure == N || n->closure->funcdepth != funcdepth) { + // create new closure var. + c = nod(ONAME, N, N); + c->sym = s; + c->class = PPARAMREF; + c->isddd = n->isddd; + c->defn = n; + c->addable = 0; + c->ullman = 2; + c->funcdepth = funcdepth; + c->outer = n->closure; + n->closure = c; + c->closure = n; + c->xoffset = 0; + curfn->cvars = list(curfn->cvars, c); + } + // return ref to closure var, not original + return n->closure; + } + return n; +} + +/* + * same for types + */ +Type* +newtype(Sym *s) +{ + Type *t; + + t = typ(TFORW); + t->sym = s; + t->type = T; + return t; +} + + +/* + * := declarations + */ + +static int +colasname(Node *n) +{ + switch(n->op) { + case ONAME: + case ONONAME: + case OPACK: + case OTYPE: + case OLITERAL: + return n->sym != S; + } + return 0; +} + +void +colasdefn(NodeList *left, Node *defn) +{ + int nnew, nerr; + NodeList *l; + Node *n; + + nnew = 0; + nerr = 0; + for(l=left; l; l=l->next) { + n = l->n; + if(isblank(n)) + continue; + if(!colasname(n)) { + yyerror("non-name %#N on left side of :=", n); + nerr++; + continue; + } + if(n->sym->block == block) + continue; + + nnew++; + n = newname(n->sym); + declare(n, dclcontext); + n->defn = defn; + defn->ninit = list(defn->ninit, nod(ODCL, n, N)); + l->n = n; + } + if(nnew == 0 && nerr == 0) + yyerror("no new variables on left side of :="); +} + +Node* +colas(NodeList *left, NodeList *right) +{ + Node *as; + + as = nod(OAS2, N, N); + as->list = left; + as->rlist = right; + as->colas = 1; + colasdefn(left, as); + + // make the tree prettier; not necessary + if(count(left) == 1 && count(right) == 1) { + as->left = as->list->n; + as->right = as->rlist->n; + as->list = nil; + as->rlist = nil; + as->op = OAS; + } + + return as; +} + +/* + * declare the arguments in an + * interface field declaration. + */ +void +ifacedcl(Node *n) +{ + if(n->op != ODCLFIELD || n->right == N) + fatal("ifacedcl"); + + dclcontext = PAUTO; + markdcl(); + funcdepth++; + n->outer = curfn; + curfn = n; + funcargs(n->right); + + // funcbody is normally called after the parser has + // seen the body of a function but since an interface + // field declaration does not have a body, we must + // call it now to pop the current declaration context. + funcbody(n); +} + +/* + * declare the function proper + * and declare the arguments. + * called in extern-declaration context + * returns in auto-declaration context. + */ +void +funchdr(Node *n) +{ + + if(n->nname != N) { + n->nname->op = ONAME; + declare(n->nname, PFUNC); + n->nname->defn = n; + } + + // change the declaration context from extern to auto + if(funcdepth == 0 && dclcontext != PEXTERN) + fatal("funchdr: dclcontext"); + + dclcontext = PAUTO; + markdcl(); + funcdepth++; + + n->outer = curfn; + curfn = n; + if(n->nname) + funcargs(n->nname->ntype); + else + funcargs(n->ntype); +} + +static void +funcargs(Node *nt) +{ + Node *n; + NodeList *l; + int gen; + + if(nt->op != OTFUNC) + fatal("funcargs %O", nt->op); + + // declare the receiver and in arguments. + // no n->defn because type checking of func header + // will fill in the types before we can demand them. + if(nt->left != N) { + n = nt->left; + if(n->op != ODCLFIELD) + fatal("funcargs1 %O", n->op); + if(n->left != N) { + n->left->op = ONAME; + n->left->ntype = n->right; + declare(n->left, PPARAM); + } + } + for(l=nt->list; l; l=l->next) { + n = l->n; + if(n->op != ODCLFIELD) + fatal("funcargs2 %O", n->op); + if(n->left != N) { + n->left->op = ONAME; + n->left->ntype = n->right; + declare(n->left, PPARAM); + } + } + + // declare the out arguments. + gen = 0; + for(l=nt->rlist; l; l=l->next) { + n = l->n; + if(n->op != ODCLFIELD) + fatal("funcargs3 %O", n->op); + if(n->left != N) { + n->left->op = ONAME; + n->left->ntype = n->right; + if(isblank(n->left)) { + // Give it a name so we can assign to it during return. + snprint(namebuf, sizeof(namebuf), ".anon%d", gen++); + n->left->sym = lookup(namebuf); + } + declare(n->left, PPARAMOUT); + } + } +} + +/* + * finish the body. + * called in auto-declaration context. + * returns in extern-declaration context. + */ +void +funcbody(Node *n) +{ + // change the declaration context from auto to extern + if(dclcontext != PAUTO) + fatal("funcbody: dclcontext"); + popdcl(); + funcdepth--; + curfn = n->outer; + n->outer = N; + if(funcdepth == 0) + dclcontext = PEXTERN; +} + +/* + * new type being defined with name s. + */ +Node* +typedcl0(Sym *s) +{ + Node *n; + + n = dclname(s); + n->op = OTYPE; + declare(n, dclcontext); + return n; +} + +/* + * node n, which was returned by typedcl0 + * is being declared to have uncompiled type t. + * return the ODCLTYPE node to use. + */ +Node* +typedcl1(Node *n, Node *t, int local) +{ + n->ntype = t; + n->local = local; + return nod(ODCLTYPE, n, N); +} + +/* + * typedcl1 but during imports + */ +void +typedcl2(Type *pt, Type *t) +{ + Node *n; + + // override declaration in unsafe.go for Pointer. + // there is no way in Go code to define unsafe.Pointer + // so we have to supply it. + if(incannedimport && + strcmp(importpkg->name, "unsafe") == 0 && + strcmp(pt->nod->sym->name, "Pointer") == 0) { + t = types[TUNSAFEPTR]; + } + + if(pt->etype == TFORW) + goto ok; + if(!eqtype(pt->orig, t)) + yyerror("inconsistent definition for type %S during import\n\t%lT\n\t%lT", pt->sym, pt->orig, t); + return; + +ok: + n = pt->nod; + copytype(pt->nod, t); + // unzero nod + pt->nod = n; + + pt->sym->lastlineno = parserline(); + declare(n, PEXTERN); + + checkwidth(pt); +} + +/* + * structs, functions, and methods. + * they don't belong here, but where do they belong? + */ + + +/* + * turn a parsed struct into a type + */ +static Type** +stotype(NodeList *l, int et, Type **t, int funarg) +{ + Type *f, *t1, *t2, **t0; + Strlit *note; + int lno; + Node *n, *left; + char *what; + + t0 = t; + lno = lineno; + what = "field"; + if(et == TINTER) + what = "method"; + + for(; l; l=l->next) { + n = l->n; + lineno = n->lineno; + note = nil; + + if(n->op != ODCLFIELD) + fatal("stotype: oops %N\n", n); + left = n->left; + if(funarg && isblank(left)) + left = N; + if(n->right != N) { + if(et == TINTER && left != N) { + // queue resolution of method type for later. + // right now all we need is the name list. + // avoids cycles for recursive interface types. + n->type = typ(TINTERMETH); + n->type->nname = n->right; + n->right = N; + left->type = n->type; + queuemethod(n); + } else { + typecheck(&n->right, Etype); + n->type = n->right->type; + if(n->type == T) + continue; + if(left != N) + left->type = n->type; + n->right = N; + if(n->embedded && n->type != T) { + t1 = n->type; + if(t1->sym == S && isptr[t1->etype]) { + t1 = t1->type; + if(t1->etype == TINTER) + yyerror("embedded type cannot be a pointer to interface"); + } + if(isptr[t1->etype]) + yyerror("embedded type cannot be a pointer"); + else if(t1->etype == TFORW && t1->embedlineno == 0) + t1->embedlineno = lineno; + } + } + } + + if(n->type == T) { + // assume error already printed + continue; + } + + switch(n->val.ctype) { + case CTSTR: + if(et != TSTRUCT) + yyerror("interface method cannot have annotation"); + note = n->val.u.sval; + break; + default: + if(et != TSTRUCT) + yyerror("interface method cannot have annotation"); + else + yyerror("field annotation must be string"); + case CTxxx: + note = nil; + break; + } + + if(et == TINTER && left == N) { + // embedded interface - inline the methods + if(n->type->etype != TINTER) { + if(n->type->etype == TFORW) + yyerror("interface type loop involving %T", n->type); + else + yyerror("interface contains embedded non-interface %T", n->type); + continue; + } + for(t1=n->type->type; t1!=T; t1=t1->down) { + f = typ(TFIELD); + f->type = t1->type; + f->width = BADWIDTH; + f->nname = newname(t1->sym); + f->sym = t1->sym; + for(t2=*t0; t2!=T; t2=t2->down) { + if(t2->sym == f->sym) { + yyerror("duplicate method %s", t2->sym->name); + break; + } + } + *t = f; + t = &f->down; + } + continue; + } + + f = typ(TFIELD); + f->type = n->type; + f->note = note; + f->width = BADWIDTH; + f->isddd = n->isddd; + + if(left != N && left->op == ONAME) { + f->nname = left; + f->embedded = n->embedded; + f->sym = f->nname->sym; + if(importpkg && !exportname(f->sym->name)) + f->sym = pkglookup(f->sym->name, structpkg); + if(f->sym && !isblank(f->nname)) { + for(t1=*t0; t1!=T; t1=t1->down) { + if(t1->sym == f->sym) { + yyerror("duplicate %s %s", what, t1->sym->name); + break; + } + } + } + } + + *t = f; + t = &f->down; + } + + *t = T; + lineno = lno; + return t; +} + +Type* +dostruct(NodeList *l, int et) +{ + Type *t; + int funarg; + + /* + * convert a parsed id/type list into + * a type for struct/interface/arglist + */ + + funarg = 0; + if(et == TFUNC) { + funarg = 1; + et = TSTRUCT; + } + t = typ(et); + t->funarg = funarg; + stotype(l, et, &t->type, funarg); + if(t->type == T && l != nil) { + t->broke = 1; + return t; + } + if(et == TINTER) + t = sortinter(t); + if(!funarg) + checkwidth(t); + return t; +} + + +Node* +embedded(Sym *s) +{ + Node *n; + char *name; + + // Names sometimes have disambiguation junk + // appended after a center dot. Discard it when + // making the name for the embedded struct field. + enum { CenterDot = 0xB7 }; + name = s->name; + if(utfrune(s->name, CenterDot)) { + name = strdup(s->name); + *utfrune(name, CenterDot) = 0; + } + + n = newname(lookup(name)); + n = nod(ODCLFIELD, n, oldname(s)); + n->embedded = 1; + return n; +} + +/* + * check that the list of declarations is either all anonymous or all named + */ + +static Node* +findtype(NodeList *l) +{ + for(; l; l=l->next) + if(l->n->op == OKEY) + return l->n->right; + return N; +} + +NodeList* +checkarglist(NodeList *all, int input) +{ + int named; + Node *n, *t, *nextt; + NodeList *l; + + named = 0; + for(l=all; l; l=l->next) { + if(l->n->op == OKEY) { + named = 1; + break; + } + } + if(named) { + n = N; + for(l=all; l; l=l->next) { + n = l->n; + if(n->op != OKEY && n->sym == S) { + yyerror("mixed named and unnamed function parameters"); + break; + } + } + if(l == nil && n != N && n->op != OKEY) + yyerror("final function parameter must have type"); + } + + nextt = nil; + for(l=all; l; l=l->next) { + // can cache result from findtype to avoid + // quadratic behavior here, but unlikely to matter. + n = l->n; + if(named) { + if(n->op == OKEY) { + t = n->right; + n = n->left; + nextt = nil; + } else { + if(nextt == nil) + nextt = findtype(l); + t = nextt; + } + } else { + t = n; + n = N; + } + if(n != N && n->sym == S) { + t = n; + n = N; + } + if(n != N) + n = newname(n->sym); + n = nod(ODCLFIELD, n, t); + if(n->right != N && n->right->op == ODDD) { + if(!input) + yyerror("cannot use ... in output argument list"); + else if(l->next != nil) + yyerror("can only use ... as final argument in list"); + n->right->op = OTARRAY; + n->right->right = n->right->left; + n->right->left = N; + n->isddd = 1; + if(n->left != N) + n->left->isddd = 1; + } + l->n = n; + } + return all; +} + + +Node* +fakethis(void) +{ + Node *n; + + n = nod(ODCLFIELD, N, typenod(ptrto(typ(TSTRUCT)))); + return n; +} + +/* + * Is this field a method on an interface? + * Those methods have an anonymous + * *struct{} as the receiver. + * (See fakethis above.) + */ +int +isifacemethod(Type *f) +{ + Type *rcvr; + Type *t; + + rcvr = getthisx(f)->type; + if(rcvr->sym != S) + return 0; + t = rcvr->type; + if(!isptr[t->etype]) + return 0; + t = t->type; + if(t->sym != S || t->etype != TSTRUCT || t->type != T) + return 0; + return 1; +} + +/* + * turn a parsed function declaration + * into a type + */ +Type* +functype(Node *this, NodeList *in, NodeList *out) +{ + Type *t; + NodeList *rcvr; + + t = typ(TFUNC); + + rcvr = nil; + if(this) + rcvr = list1(this); + t->type = dostruct(rcvr, TFUNC); + t->type->down = dostruct(out, TFUNC); + t->type->down->down = dostruct(in, TFUNC); + + if(this) + t->thistuple = 1; + t->outtuple = count(out); + t->intuple = count(in); + t->outnamed = t->outtuple > 0 && out->n->left != N; + + return t; +} + +Sym* +methodsym(Sym *nsym, Type *t0, int iface) +{ + Sym *s; + char *p; + Type *t; + char *suffix; + + t = t0; + if(t == T) + goto bad; + s = t->sym; + if(s == S) { + if(!isptr[t->etype]) + goto bad; + t = t->type; + if(t == T) + goto bad; + s = t->sym; + if(s == S) + goto bad; + } + + // if t0 == *t and t0 has a sym, + // we want to see *t, not t0, in the method name. + if(t != t0 && t0->sym) + t0 = ptrto(t); + + suffix = ""; + if(iface) { + dowidth(t0); + if(t0->width < types[tptr]->width) + suffix = "·i"; + } + if(t0->sym == S && isptr[t0->etype]) + p = smprint("(%#hT).%s%s", t0, nsym->name, suffix); + else + p = smprint("%#hT.%s%s", t0, nsym->name, suffix); + s = pkglookup(p, s->pkg); + free(p); + return s; + +bad: + yyerror("illegal receiver type: %T", t0); + return S; +} + +Node* +methodname(Node *n, Type *t) +{ + Sym *s; + + s = methodsym(n->sym, t, 0); + if(s == S) + return n; + return newname(s); +} + +Node* +methodname1(Node *n, Node *t) +{ + char *star; + char *p; + + star = nil; + if(t->op == OIND) { + star = "*"; + t = t->left; + } + if(t->sym == S || isblank(n)) + return newname(n->sym); + if(star) + p = smprint("(%s%S).%S", star, t->sym, n->sym); + else + p = smprint("%S.%S", t->sym, n->sym); + n = newname(pkglookup(p, t->sym->pkg)); + free(p); + return n; +} + +/* + * add a method, declared as a function, + * n is fieldname, pa is base type, t is function type + */ +void +addmethod(Sym *sf, Type *t, int local) +{ + Type *f, *d, *pa; + Node *n; + + pa = nil; + + // get field sym + if(sf == S) + fatal("no method symbol"); + + // get parent type sym + pa = getthisx(t)->type; // ptr to this structure + if(pa == T) { + yyerror("missing receiver"); + return; + } + + pa = pa->type; + f = methtype(pa); + if(f == T) { + t = pa; + if(t != T) { + if(isptr[t->etype]) { + if(t->sym != S) { + yyerror("invalid receiver type %T (%T is a pointer type)", pa, t); + return; + } + t = t->type; + } + } + if(t != T) { + if(t->sym == S) { + yyerror("invalid receiver type %T (%T is an unnamed type)", pa, t); + return; + } + if(isptr[t->etype]) { + yyerror("invalid receiver type %T (%T is a pointer type)", pa, t); + return; + } + if(t->etype == TINTER) { + yyerror("invalid receiver type %T (%T is an interface type)", pa, t); + return; + } + } + // Should have picked off all the reasons above, + // but just in case, fall back to generic error. + yyerror("invalid receiver type %T", pa); + return; + } + + pa = f; + if(importpkg && !exportname(sf->name)) + sf = pkglookup(sf->name, importpkg); + + n = nod(ODCLFIELD, newname(sf), N); + n->type = t; + + d = T; // last found + for(f=pa->method; f!=T; f=f->down) { + d = f; + if(f->etype != TFIELD) + fatal("addmethod: not TFIELD: %N", f); + if(strcmp(sf->name, f->sym->name) != 0) + continue; + if(!eqtype(t, f->type)) + yyerror("method redeclared: %T.%S\n\t%T\n\t%T", pa, sf, f->type, t); + return; + } + + if(local && !pa->local) { + // defining method on non-local type. + yyerror("cannot define new methods on non-local type %T", pa); + return; + } + + if(d == T) + stotype(list1(n), 0, &pa->method, 0); + else + stotype(list1(n), 0, &d->down, 0); + return; +} + +void +funccompile(Node *n, int isclosure) +{ + stksize = BADWIDTH; + maxarg = 0; + + if(n->type == T) { + if(nerrors == 0) + fatal("funccompile missing type"); + return; + } + + // assign parameter offsets + checkwidth(n->type); + + // record offset to actual frame pointer. + // for closure, have to skip over leading pointers and PC slot. + nodfp->xoffset = 0; + if(isclosure) { + NodeList *l; + for(l=n->nname->ntype->list; l; l=l->next) { + nodfp->xoffset += widthptr; + if(l->n->left == N) // found slot for PC + break; + } + } + + if(curfn) + fatal("funccompile %S inside %S", n->nname->sym, curfn->nname->sym); + + stksize = 0; + dclcontext = PAUTO; + funcdepth = n->funcdepth + 1; + compile(n); + curfn = nil; + funcdepth = 0; + dclcontext = PEXTERN; +} + + + diff --git a/src/cmd/gc/doc.go b/src/cmd/gc/doc.go new file mode 100644 index 000000000..3fe7fafdd --- /dev/null +++ b/src/cmd/gc/doc.go @@ -0,0 +1,55 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +/* + +Gc is the generic label for the family of Go compilers +that function as part of the (modified) Plan 9 tool chain. The C compiler +documentation at + + http://plan9.bell-labs.com/sys/doc/comp.pdf (Tools overview) + http://plan9.bell-labs.com/sys/doc/compiler.pdf (C compiler architecture) + +gives the overall design of the tool chain. Aside from a few adapted pieces, +such as the optimizer, the Go compilers are wholly new programs. + +The compiler reads in a set of Go files, typically suffixed ".go". They +must all be part of one package. The output is a single intermediate file +representing the "binary assembly" of the compiled package, ready as input +for the linker (6l, etc.). + +The generated files contain type information about the symbols exported by +the package and about types used by symbols imported by the package from +other packages. It is therefore not necessary when compiling client C of +package P to read the files of P's dependencies, only the compiled output +of P. + +Usage: + 6g [flags] file... +The specified files must be Go source files and all part of the same package. +Substitute 6g with 8g or 5g where appropriate. + +Flags: + -o file + output file, default file.6 for 6g, etc. + -e + normally the compiler quits after 10 errors; -e prints all errors + -L + show entire file path when printing line numbers in errors + -I dir1 -I dir2 + add dir1 and dir2 to the list of paths to check for imported packages + -N + disable optimization + -S + write assembly language text to standard output + -u + disallow importing packages not marked as safe + -V + print the compiler version + +There are also a number of debugging flags; run the command with no arguments +to get a usage message. + +*/ +package documentation diff --git a/src/cmd/gc/export.c b/src/cmd/gc/export.c new file mode 100644 index 000000000..014f0c5f0 --- /dev/null +++ b/src/cmd/gc/export.c @@ -0,0 +1,428 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +#include "go.h" +#include "y.tab.h" + +static void dumpsym(Sym*); +static void dumpexporttype(Sym*); +static void dumpexportvar(Sym*); +static void dumpexportconst(Sym*); + +void +exportsym(Node *n) +{ + if(n == N || n->sym == S) + return; + if(n->sym->flags & (SymExport|SymPackage)) { + if(n->sym->flags & SymPackage) + yyerror("export/package mismatch: %S", n->sym); + return; + } + n->sym->flags |= SymExport; + + exportlist = list(exportlist, n); +} + +static void +packagesym(Node *n) +{ + if(n == N || n->sym == S) + return; + if(n->sym->flags & (SymExport|SymPackage)) { + if(n->sym->flags & SymExport) + yyerror("export/package mismatch: %S", n->sym); + return; + } + n->sym->flags |= SymPackage; + + exportlist = list(exportlist, n); +} + +int +exportname(char *s) +{ + Rune r; + + if((uchar)s[0] < Runeself) + return 'A' <= s[0] && s[0] <= 'Z'; + chartorune(&r, s); + return isupperrune(r); +} + +static int +initname(char *s) +{ + return strcmp(s, "init") == 0; +} + +void +autoexport(Node *n, int ctxt) +{ + if(n == N || n->sym == S) + return; + if((ctxt != PEXTERN && ctxt != PFUNC) || dclcontext != PEXTERN) + return; + if(n->ntype && n->ntype->op == OTFUNC && n->ntype->left) // method + return; + if(exportname(n->sym->name) || initname(n->sym->name)) + exportsym(n); + else + packagesym(n); +} + +static void +dumppkg(Pkg *p) +{ + char *suffix; + + if(p == nil || p == localpkg || p->exported) + return; + p->exported = 1; + suffix = ""; + if(!p->direct) + suffix = " // indirect"; + Bprint(bout, "\timport %s \"%Z\"%s\n", p->name, p->path, suffix); +} + +static void +dumpprereq(Type *t) +{ + if(t == T) + return; + + if(t->printed || t == types[t->etype]) + return; + t->printed = 1; + + if(t->sym != S) { + dumppkg(t->sym->pkg); + if(t->etype != TFIELD) + dumpsym(t->sym); + } + dumpprereq(t->type); + dumpprereq(t->down); +} + +static void +dumpexportconst(Sym *s) +{ + Node *n; + Type *t; + + n = s->def; + typecheck(&n, Erv); + if(n == N || n->op != OLITERAL) + fatal("dumpexportconst: oconst nil: %S", s); + + t = n->type; // may or may not be specified + if(t != T) + dumpprereq(t); + + Bprint(bout, "\t"); + Bprint(bout, "const %#S", s); + if(t != T && !isideal(t)) + Bprint(bout, " %#T", t); + Bprint(bout, " = "); + + switch(n->val.ctype) { + default: + fatal("dumpexportconst: unknown ctype: %S %d", s, n->val.ctype); + case CTINT: + Bprint(bout, "%B\n", n->val.u.xval); + break; + case CTBOOL: + if(n->val.u.bval) + Bprint(bout, "true\n"); + else + Bprint(bout, "false\n"); + break; + case CTFLT: + Bprint(bout, "%F\n", n->val.u.fval); + break; + case CTCPLX: + Bprint(bout, "(%F+%F)\n", &n->val.u.cval->real, &n->val.u.cval->imag); + break; + case CTSTR: + Bprint(bout, "\"%Z\"\n", n->val.u.sval); + break; + } +} + +static void +dumpexportvar(Sym *s) +{ + Node *n; + Type *t; + + n = s->def; + typecheck(&n, Erv); + if(n == N || n->type == T) { + yyerror("variable exported but not defined: %S", s); + return; + } + + t = n->type; + dumpprereq(t); + + Bprint(bout, "\t"); + if(t->etype == TFUNC && n->class == PFUNC) + Bprint(bout, "func %#S %#hhT", s, t); + else + Bprint(bout, "var %#S %#T", s, t); + Bprint(bout, "\n"); +} + +static void +dumpexporttype(Sym *s) +{ + Type *t; + + t = s->def->type; + dumpprereq(t); + Bprint(bout, "\t"); + switch (t->etype) { + case TFORW: + yyerror("export of incomplete type %T", t); + return; + } + if(Bprint(bout, "type %#T %l#T\n", t, t) < 0) + fatal("Bprint failed for %T", t); +} + +static int +methcmp(const void *va, const void *vb) +{ + Type *a, *b; + + a = *(Type**)va; + b = *(Type**)vb; + return strcmp(a->sym->name, b->sym->name); +} + +static void +dumpsym(Sym *s) +{ + Type *f, *t; + Type **m; + int i, n; + + if(s->flags & SymExported) + return; + s->flags |= SymExported; + + if(s->def == N) { + yyerror("unknown export symbol: %S", s); + return; + } + + dumppkg(s->pkg); + + switch(s->def->op) { + default: + yyerror("unexpected export symbol: %O %S", s->def->op, s); + break; + case OLITERAL: + dumpexportconst(s); + break; + case OTYPE: + t = s->def->type; + n = 0; + for(f=t->method; f!=T; f=f->down) { + dumpprereq(f); + n++; + } + m = mal(n*sizeof m[0]); + i = 0; + for(f=t->method; f!=T; f=f->down) + m[i++] = f; + qsort(m, n, sizeof m[0], methcmp); + + dumpexporttype(s); + for(i=0; itype->type->type, f->sym, f->type); + } + break; + case ONAME: + dumpexportvar(s); + break; + } +} + +static void +dumptype(Type *t) +{ + // no need to re-dump type if already exported + if(t->printed) + return; + + // no need to dump type if it's not ours (was imported) + if(t->sym != S && t->sym->def == typenod(t) && !t->local) + return; + + Bprint(bout, "type %#T %l#T\n", t, t); +} + +void +dumpexport(void) +{ + NodeList *l; + int32 i, lno; + Pkg *p; + + lno = lineno; + + packagequotes = 1; + Bprint(bout, "\n$$ // exports\n"); + + Bprint(bout, " package %s", localpkg->name); + if(safemode) + Bprint(bout, " safe"); + Bprint(bout, "\n"); + + for(i=0; ilink) + if(p->direct) + dumppkg(p); + + for(l=exportlist; l; l=l->next) { + lineno = l->n->lineno; + dumpsym(l->n->sym); + } + + Bprint(bout, "\n$$ // local types\n"); + + for(l=typelist; l; l=l->next) { + lineno = l->n->lineno; + dumptype(l->n->type); + } + + Bprint(bout, "\n$$\n"); + packagequotes = 0; + + lineno = lno; +} + +/* + * import + */ + +/* + * return the sym for ss, which should match lexical + */ +Sym* +importsym(Sym *s, int op) +{ + if(s->def != N && s->def->op != op) + redeclare(s, "during import"); + + // mark the symbol so it is not reexported + if(s->def == N) { + if(exportname(s->name) || initname(s->name)) + s->flags |= SymExport; + else + s->flags |= SymPackage; // package scope + } + return s; +} + +/* + * return the type pkg.name, forward declaring if needed + */ +Type* +pkgtype(Sym *s) +{ + Type *t; + + importsym(s, OTYPE); + if(s->def == N || s->def->op != OTYPE) { + t = typ(TFORW); + t->sym = s; + s->def = typenod(t); + } + if(s->def->type == T) + yyerror("pkgtype %lS", s); + return s->def->type; +} + +static int +mypackage(Sym *s) +{ + // we import all definitions for runtime. + // lowercase ones can only be used by the compiler. + return s->pkg == localpkg || s->pkg == runtimepkg; +} + +void +importconst(Sym *s, Type *t, Node *n) +{ + Node *n1; + + if(!exportname(s->name) && !mypackage(s)) + return; + importsym(s, OLITERAL); + convlit(&n, t); + if(s->def != N) { + // TODO: check if already the same. + return; + } + + if(n->op != OLITERAL) { + yyerror("expression must be a constant"); + return; + } + if(n->sym != S) { + n1 = nod(OXXX, N, N); + *n1 = *n; + n = n1; + } + n->sym = s; + declare(n, PEXTERN); + + if(debug['E']) + print("import const %S\n", s); +} + +void +importvar(Sym *s, Type *t, int ctxt) +{ + Node *n; + + if(!exportname(s->name) && !initname(s->name) && !mypackage(s)) + return; + + importsym(s, ONAME); + if(s->def != N && s->def->op == ONAME) { + if(eqtype(t, s->def->type)) + return; + yyerror("inconsistent definition for var %S during import\n\t%T\n\t%T", + s, s->def->type, t); + } + n = newname(s); + n->type = t; + declare(n, ctxt); + + if(debug['E']) + print("import var %S %lT\n", s, t); +} + +void +importtype(Type *pt, Type *t) +{ + if(pt != T && t != T) + typedcl2(pt, t); + + if(debug['E']) + print("import type %T %lT\n", pt, t); +} + +void +importmethod(Sym *s, Type *t) +{ + checkwidth(t); + addmethod(s, t, 0); +} + diff --git a/src/cmd/gc/gen.c b/src/cmd/gc/gen.c new file mode 100644 index 000000000..cb66921ba --- /dev/null +++ b/src/cmd/gc/gen.c @@ -0,0 +1,790 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +/* + * portable half of code generator. + * mainly statements and control flow. + */ + +#include "go.h" + +static void cgen_dcl(Node *n); +static void cgen_proc(Node *n, int proc); +static void checkgoto(Node*, Node*); + +static Label *labellist; +static Label *lastlabel; + +Node* +sysfunc(char *name) +{ + Node *n; + + n = newname(pkglookup(name, runtimepkg)); + n->class = PFUNC; + return n; +} + +void +allocparams(void) +{ + NodeList *l; + Node *n; + uint32 w; + Sym *s; + int lno; + + if(stksize < 0) + fatal("allocparams not during code generation"); + + /* + * allocate (set xoffset) the stack + * slots for all automatics. + * allocated starting at -w down. + */ + lno = lineno; + for(l=curfn->dcl; l; l=l->next) { + n = l->n; + if(n->op == ONAME && n->class == PHEAP-1) { + // heap address variable; finish the job + // started in addrescapes. + s = n->sym; + tempname(n, n->type); + n->sym = s; + } + if(n->op != ONAME || n->class != PAUTO) + continue; + if (n->xoffset != BADWIDTH) + continue; + if(n->type == T) + continue; + dowidth(n->type); + w = n->type->width; + if(w >= MAXWIDTH) + fatal("bad width"); + stksize += w; + stksize = rnd(stksize, n->type->align); + if(thechar == '5') + stksize = rnd(stksize, widthptr); + n->xoffset = -stksize; + } + lineno = lno; +} + +void +clearlabels(void) +{ + Label *l; + + for(l=labellist; l!=L; l=l->link) + l->sym->label = L; + + labellist = L; + lastlabel = L; +} + +static Label* +newlab(Node *n) +{ + Sym *s; + Label *lab; + + s = n->left->sym; + if((lab = s->label) == L) { + lab = mal(sizeof(*lab)); + if(lastlabel == nil) + labellist = lab; + else + lastlabel->link = lab; + lastlabel = lab; + lab->sym = s; + s->label = lab; + } + + if(n->op == OLABEL) { + if(lab->def != N) + yyerror("label %S already defined at %L", s, lab->def->lineno); + else + lab->def = n; + } else + lab->use = list(lab->use, n); + + return lab; +} + +void +checklabels(void) +{ + Label *lab; + NodeList *l; + + for(lab=labellist; lab!=L; lab=lab->link) { + if(lab->def == N) { + for(l=lab->use; l; l=l->next) + yyerrorl(l->n->lineno, "label %S not defined", lab->sym); + continue; + } + if(lab->use == nil && !lab->used) { + yyerrorl(lab->def->lineno, "label %S defined and not used", lab->sym); + continue; + } + if(lab->gotopc != P) + fatal("label %S never resolved", lab->sym); + for(l=lab->use; l; l=l->next) + checkgoto(l->n, lab->def); + } +} + +static void +checkgoto(Node *from, Node *to) +{ + int nf, nt; + Sym *block, *dcl, *fs, *ts; + int lno; + + if(from->sym == to->sym) + return; + + nf = 0; + for(fs=from->sym; fs; fs=fs->link) + nf++; + nt = 0; + for(fs=to->sym; fs; fs=fs->link) + nt++; + fs = from->sym; + for(; nf > nt; nf--) + fs = fs->link; + if(fs != to->sym) { + lno = lineno; + setlineno(from); + + // decide what to complain about. + // prefer to complain about 'into block' over declarations, + // so scan backward to find most recent block or else dcl. + block = S; + dcl = S; + ts = to->sym; + for(; nt > nf; nt--) { + if(ts->pkg == nil) + block = ts; + else + dcl = ts; + ts = ts->link; + } + while(ts != fs) { + if(ts->pkg == nil) + block = ts; + else + dcl = ts; + ts = ts->link; + fs = fs->link; + } + + if(block) + yyerror("goto %S jumps into block starting at %L", from->left->sym, block->lastlineno); + else + yyerror("goto %S jumps over declaration of %S at %L", from->left->sym, dcl, dcl->lastlineno); + lineno = lno; + } +} + +static Label* +stmtlabel(Node *n) +{ + Label *lab; + + if(n->sym != S) + if((lab = n->sym->label) != L) + if(lab->def != N) + if(lab->def->right == n) + return lab; + return L; +} + +/* + * compile statements + */ +void +genlist(NodeList *l) +{ + for(; l; l=l->next) + gen(l->n); +} + +void +gen(Node *n) +{ + int32 lno; + Prog *scontin, *sbreak; + Prog *p1, *p2, *p3; + Label *lab; + int32 wasregalloc; + + lno = setlineno(n); + wasregalloc = anyregalloc(); + + if(n == N) + goto ret; + + p3 = pc; // save pc for loop labels + if(n->ninit) + genlist(n->ninit); + + setlineno(n); + + switch(n->op) { + default: + fatal("gen: unknown op %N", n); + break; + + case OCASE: + case OFALL: + case OXCASE: + case OXFALL: + case ODCLCONST: + case ODCLFUNC: + case ODCLTYPE: + break; + + case OEMPTY: + break; + + case OBLOCK: + genlist(n->list); + break; + + case OLABEL: + lab = newlab(n); + + // if there are pending gotos, resolve them all to the current pc. + for(p1=lab->gotopc; p1; p1=p2) { + p2 = unpatch(p1); + patch(p1, pc); + } + lab->gotopc = P; + if(lab->labelpc == P) + lab->labelpc = pc; + + if(n->right) { + switch(n->right->op) { + case OFOR: + case OSWITCH: + case OSELECT: + // so stmtlabel can find the label + n->right->sym = lab->sym; + } + } + break; + + case OGOTO: + // if label is defined, emit jump to it. + // otherwise save list of pending gotos in lab->gotopc. + // the list is linked through the normal jump target field + // to avoid a second list. (the jumps are actually still + // valid code, since they're just going to another goto + // to the same label. we'll unwind it when we learn the pc + // of the label in the OLABEL case above.) + lab = newlab(n); + if(lab->labelpc != P) + gjmp(lab->labelpc); + else + lab->gotopc = gjmp(lab->gotopc); + break; + + case OBREAK: + if(n->left != N) { + lab = n->left->sym->label; + if(lab == L) { + yyerror("break label not defined: %S", n->left->sym); + break; + } + lab->used = 1; + if(lab->breakpc == P) { + yyerror("invalid break label %S", n->left->sym); + break; + } + gjmp(lab->breakpc); + break; + } + if(breakpc == P) { + yyerror("break is not in a loop"); + break; + } + gjmp(breakpc); + break; + + case OCONTINUE: + if(n->left != N) { + lab = n->left->sym->label; + if(lab == L) { + yyerror("continue label not defined: %S", n->left->sym); + break; + } + lab->used = 1; + if(lab->continpc == P) { + yyerror("invalid continue label %S", n->left->sym); + break; + } + gjmp(lab->continpc); + break; + } + if(continpc == P) { + yyerror("continue is not in a loop"); + break; + } + gjmp(continpc); + break; + + case OFOR: + sbreak = breakpc; + p1 = gjmp(P); // goto test + breakpc = gjmp(P); // break: goto done + scontin = continpc; + continpc = pc; + + // define break and continue labels + if((lab = stmtlabel(n)) != L) { + lab->breakpc = breakpc; + lab->continpc = continpc; + } + gen(n->nincr); // contin: incr + patch(p1, pc); // test: + bgen(n->ntest, 0, breakpc); // if(!test) goto break + genlist(n->nbody); // body + gjmp(continpc); + patch(breakpc, pc); // done: + continpc = scontin; + breakpc = sbreak; + if(lab) { + lab->breakpc = P; + lab->continpc = P; + } + break; + + case OIF: + p1 = gjmp(P); // goto test + p2 = gjmp(P); // p2: goto else + patch(p1, pc); // test: + bgen(n->ntest, 0, p2); // if(!test) goto p2 + genlist(n->nbody); // then + p3 = gjmp(P); // goto done + patch(p2, pc); // else: + genlist(n->nelse); // else + patch(p3, pc); // done: + break; + + case OSWITCH: + sbreak = breakpc; + p1 = gjmp(P); // goto test + breakpc = gjmp(P); // break: goto done + + // define break label + if((lab = stmtlabel(n)) != L) + lab->breakpc = breakpc; + + patch(p1, pc); // test: + genlist(n->nbody); // switch(test) body + patch(breakpc, pc); // done: + breakpc = sbreak; + if(lab != L) + lab->breakpc = P; + break; + + case OSELECT: + sbreak = breakpc; + p1 = gjmp(P); // goto test + breakpc = gjmp(P); // break: goto done + + // define break label + if((lab = stmtlabel(n)) != L) + lab->breakpc = breakpc; + + patch(p1, pc); // test: + genlist(n->nbody); // select() body + patch(breakpc, pc); // done: + breakpc = sbreak; + if(lab != L) + lab->breakpc = P; + break; + + case OASOP: + cgen_asop(n); + break; + + case ODCL: + cgen_dcl(n->left); + break; + + case OAS: + if(gen_as_init(n)) + break; + cgen_as(n->left, n->right); + break; + + case OCALLMETH: + cgen_callmeth(n, 0); + break; + + case OCALLINTER: + cgen_callinter(n, N, 0); + break; + + case OCALLFUNC: + cgen_call(n, 0); + break; + + case OPROC: + cgen_proc(n, 1); + break; + + case ODEFER: + cgen_proc(n, 2); + break; + + case ORETURN: + cgen_ret(n); + break; + } + +ret: + if(anyregalloc() != wasregalloc) { + dump("node", n); + fatal("registers left allocated"); + } + + lineno = lno; +} + +/* + * generate call to non-interface method + * proc=0 normal call + * proc=1 goroutine run in new proc + * proc=2 defer call save away stack + */ +void +cgen_callmeth(Node *n, int proc) +{ + Node *l; + + // generate a rewrite for method call + // (p.f)(...) goes to (f)(p,...) + + l = n->left; + if(l->op != ODOTMETH) + fatal("cgen_callmeth: not dotmethod: %N"); + + n->op = OCALLFUNC; + n->left = n->left->right; + n->left->type = l->type; + + if(n->left->op == ONAME) + n->left->class = PFUNC; + cgen_call(n, proc); +} + +/* + * generate code to start new proc running call n. + */ +void +cgen_proc(Node *n, int proc) +{ + switch(n->left->op) { + default: + fatal("cgen_proc: unknown call %O", n->left->op); + + case OCALLMETH: + cgen_callmeth(n->left, proc); + break; + + case OCALLINTER: + cgen_callinter(n->left, N, proc); + break; + + case OCALLFUNC: + cgen_call(n->left, proc); + break; + } + +} + +/* + * generate declaration. + * nothing to do for on-stack automatics, + * but might have to allocate heap copy + * for escaped variables. + */ +static void +cgen_dcl(Node *n) +{ + if(debug['g']) + dump("\ncgen-dcl", n); + if(n->op != ONAME) { + dump("cgen_dcl", n); + fatal("cgen_dcl"); + } + if(!(n->class & PHEAP)) + return; + if(n->alloc == nil) + n->alloc = callnew(n->type); + cgen_as(n->heapaddr, n->alloc); +} + +/* + * generate discard of value + */ +static void +cgen_discard(Node *nr) +{ + Node tmp; + + if(nr == N) + return; + + switch(nr->op) { + case ONAME: + if(!(nr->class & PHEAP) && nr->class != PEXTERN && nr->class != PFUNC && nr->class != PPARAMREF) + gused(nr); + break; + + // unary + case OADD: + case OAND: + case ODIV: + case OEQ: + case OGE: + case OGT: + case OLE: + case OLSH: + case OLT: + case OMOD: + case OMUL: + case ONE: + case OOR: + case ORSH: + case OSUB: + case OXOR: + cgen_discard(nr->left); + cgen_discard(nr->right); + break; + + // binary + case OCAP: + case OCOM: + case OLEN: + case OMINUS: + case ONOT: + case OPLUS: + cgen_discard(nr->left); + break; + + // special enough to just evaluate + default: + tempname(&tmp, nr->type); + cgen_as(&tmp, nr); + gused(&tmp); + } +} + +/* + * generate assignment: + * nl = nr + * nr == N means zero nl. + */ +void +cgen_as(Node *nl, Node *nr) +{ + Node nc; + Type *tl; + int iszer; + + if(nl == N) + return; + + if(debug['g']) { + dump("cgen_as", nl); + dump("cgen_as = ", nr); + } + + if(isblank(nl)) { + cgen_discard(nr); + return; + } + + iszer = 0; + if(nr == N || isnil(nr)) { + // externals and heaps should already be clear + if(nr == N) { + if(nl->class == PEXTERN) + return; + if(nl->class & PHEAP) + return; + } + + tl = nl->type; + if(tl == T) + return; + if(isfat(tl)) { + clearfat(nl); + goto ret; + } + + /* invent a "zero" for the rhs */ + iszer = 1; + nr = &nc; + memset(nr, 0, sizeof(*nr)); + switch(simtype[tl->etype]) { + default: + fatal("cgen_as: tl %T", tl); + break; + + case TINT8: + case TUINT8: + case TINT16: + case TUINT16: + case TINT32: + case TUINT32: + case TINT64: + case TUINT64: + nr->val.u.xval = mal(sizeof(*nr->val.u.xval)); + mpmovecfix(nr->val.u.xval, 0); + nr->val.ctype = CTINT; + break; + + case TFLOAT32: + case TFLOAT64: + nr->val.u.fval = mal(sizeof(*nr->val.u.fval)); + mpmovecflt(nr->val.u.fval, 0.0); + nr->val.ctype = CTFLT; + break; + + case TBOOL: + nr->val.u.bval = 0; + nr->val.ctype = CTBOOL; + break; + + case TPTR32: + case TPTR64: + nr->val.ctype = CTNIL; + break; + + case TCOMPLEX64: + case TCOMPLEX128: + nr->val.u.cval = mal(sizeof(*nr->val.u.cval)); + mpmovecflt(&nr->val.u.cval->real, 0.0); + mpmovecflt(&nr->val.u.cval->imag, 0.0); + break; + } + nr->op = OLITERAL; + nr->type = tl; + nr->addable = 1; + ullmancalc(nr); + } + + tl = nl->type; + if(tl == T) + return; + + cgen(nr, nl); + if(iszer && nl->addable) + gused(nl); + +ret: + ; +} + +/* + * gather series of offsets + * >=0 is direct addressed field + * <0 is pointer to next field (+1) + */ +int +dotoffset(Node *n, int *oary, Node **nn) +{ + int i; + + switch(n->op) { + case ODOT: + if(n->xoffset == BADWIDTH) { + dump("bad width in dotoffset", n); + fatal("bad width in dotoffset"); + } + i = dotoffset(n->left, oary, nn); + if(i > 0) { + if(oary[i-1] >= 0) + oary[i-1] += n->xoffset; + else + oary[i-1] -= n->xoffset; + break; + } + if(i < 10) + oary[i++] = n->xoffset; + break; + + case ODOTPTR: + if(n->xoffset == BADWIDTH) { + dump("bad width in dotoffset", n); + fatal("bad width in dotoffset"); + } + i = dotoffset(n->left, oary, nn); + if(i < 10) + oary[i++] = -(n->xoffset+1); + break; + + default: + *nn = n; + return 0; + } + if(i >= 10) + *nn = N; + return i; +} + +/* + * make a new off the books + */ +void +tempname(Node *nn, Type *t) +{ + Node *n; + Sym *s; + uint32 w; + + if(stksize < 0) + fatal("tempname not during code generation"); + + if (curfn == N) + fatal("no curfn for tempname"); + + if(t == T) { + yyerror("tempname called with nil type"); + t = types[TINT32]; + } + + // give each tmp a different name so that there + // a chance to registerizer them + snprint(namebuf, sizeof(namebuf), "autotmp_%.4d", statuniqgen); + statuniqgen++; + s = lookup(namebuf); + n = nod(ONAME, N, N); + n->sym = s; + n->type = t; + n->class = PAUTO; + n->addable = 1; + n->ullman = 1; + n->noescape = 1; + n->curfn = curfn; + curfn->dcl = list(curfn->dcl, n); + + dowidth(t); + w = t->width; + stksize += w; + stksize = rnd(stksize, t->align); + if(thechar == '5') + stksize = rnd(stksize, widthptr); + n->xoffset = -stksize; + + // print("\ttmpname (%d): %N\n", stksize, n); + + *nn = *n; +} diff --git a/src/cmd/gc/go.errors b/src/cmd/gc/go.errors new file mode 100644 index 000000000..b5af4678c --- /dev/null +++ b/src/cmd/gc/go.errors @@ -0,0 +1,70 @@ +// Copyright 2010 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Example-based syntax error messages. +// See bisonerrors, Makefile, go.y. + +static struct { + int yystate; + int yychar; + char *msg; +} yymsg[] = { + // Each line of the form % token list + // is converted by bisonerrors into the yystate and yychar caused + // by that token list. + + % loadsys package LIMPORT '(' LLITERAL import_package import_there ',' + "unexpected comma during import block", + + % loadsys package imports LFUNC LNAME '(' ')' '{' LIF if_header ';' + "unexpected semicolon or newline before {", + + % loadsys package imports LFUNC LNAME '(' ')' '{' LSWITCH if_header ';' + "unexpected semicolon or newline before {", + + % loadsys package imports LFUNC LNAME '(' ')' '{' LFOR for_header ';' + "unexpected semicolon or newline before {", + + % loadsys package imports LFUNC LNAME '(' ')' '{' LFOR ';' LBODY + "unexpected semicolon or newline before {", + + % loadsys package imports LFUNC LNAME '(' ')' ';' '{' + "unexpected semicolon or newline before {", + + % loadsys package imports LTYPE LNAME ';' + "unexpected semicolon or newline in type declaration", + + % loadsys package imports LCHAN '}' + "unexpected } in channel type", + + % loadsys package imports LCHAN ')' + "unexpected ) in channel type", + + % loadsys package imports LCHAN ',' + "unexpected comma in channel type", + + % loadsys package imports LFUNC LNAME '(' ')' '{' if_stmt ';' LELSE + "unexpected semicolon or newline before else", + + % loadsys package imports LTYPE LNAME LINTERFACE '{' LNAME ',' LNAME + "name list not allowed in interface type", + + % loadsys package imports LFUNC LNAME '(' ')' '{' LFOR LVAR LNAME '=' LNAME + "var declaration not allowed in for initializer", + + % loadsys package imports LVAR LNAME '[' ']' LNAME '{' + "unexpected { at end of statement", + + % loadsys package imports LFUNC LNAME '(' ')' '{' LVAR LNAME '[' ']' LNAME '{' + "unexpected { at end of statement", + + % loadsys package imports LFUNC LNAME '(' ')' '{' LDEFER LNAME ';' + "argument to go/defer must be function call", + + % loadsys package imports LVAR LNAME '=' LNAME '{' LNAME ';' + "need trailing comma before newline in composite literal", + + % loadsys package imports LFUNC LNAME '(' ')' '{' LFUNC LNAME + "nested func not allowed", +}; diff --git a/src/cmd/gc/go.h b/src/cmd/gc/go.h new file mode 100644 index 000000000..da0fb5146 --- /dev/null +++ b/src/cmd/gc/go.h @@ -0,0 +1,1279 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +#include +#include +#include + +#undef OAPPEND + +// avoid +#undef isblank +#define isblank goisblank + +#ifndef EXTERN +#define EXTERN extern +#endif + +#undef BUFSIZ + +enum +{ + NHUNK = 50000, + BUFSIZ = 8192, + NSYMB = 500, + NHASH = 1024, + STRINGSZ = 200, + YYMAXDEPTH = 500, + MAXALIGN = 7, + UINF = 100, + HISTSZ = 10, + + PRIME1 = 3, + + AUNK = 100, + + // these values are known by runtime + AMEM = 0, + ANOEQ, + ASTRING, + AINTER, + ANILINTER, + ASLICE, + AMEM8, + AMEM16, + AMEM32, + AMEM64, + AMEM128, + ANOEQ8, + ANOEQ16, + ANOEQ32, + ANOEQ64, + ANOEQ128, + + BADWIDTH = -1000000000, +}; + +extern vlong MAXWIDTH; + +/* + * note this is the representation + * of the compilers string literals, + * it is not the runtime representation + */ +typedef struct Strlit Strlit; +struct Strlit +{ + int32 len; + char s[3]; // variable +}; + +/* + * note this is the runtime representation + * of hashmap iterator. it is probably + * insafe to use it this way, but it puts + * all the changes in one place. + * only flag is referenced from go. + * actual placement does not matter as long + * as the size is >= actual size. + */ +typedef struct Hiter Hiter; +struct Hiter +{ + uchar data[8]; // return val from next + int32 elemsize; // size of elements in table */ + int32 changes; // number of changes observed last time */ + int32 i; // stack pointer in subtable_state */ + uchar last[8]; // last hash value returned */ + uchar h[8]; // the hash table */ + struct + { + uchar sub[8]; // pointer into subtable */ + uchar start[8]; // pointer into start of subtable */ + uchar end[8]; // pointer into end of subtable */ + uchar pad[8]; + } sub[4]; +}; + +enum +{ + Mpscale = 29, // safely smaller than bits in a long + Mpprec = 16, // Mpscale*Mpprec is max number of bits + Mpnorm = Mpprec - 1, // significant words in a normalized float + Mpbase = 1L << Mpscale, + Mpsign = Mpbase >> 1, + Mpmask = Mpbase - 1, + Mpdebug = 0, +}; + +typedef struct Mpint Mpint; +struct Mpint +{ + long a[Mpprec]; + uchar neg; + uchar ovf; +}; + +typedef struct Mpflt Mpflt; +struct Mpflt +{ + Mpint val; + short exp; +}; + +typedef struct Mpcplx Mpcplx; +struct Mpcplx +{ + Mpflt real; + Mpflt imag; +}; + +typedef struct Val Val; +struct Val +{ + short ctype; + union + { + short reg; // OREGISTER + short bval; // bool value CTBOOL + Mpint* xval; // int CTINT + Mpflt* fval; // float CTFLT + Mpcplx* cval; // float CTCPLX + Strlit* sval; // string CTSTR + } u; +}; + +typedef struct Pkg Pkg; +typedef struct Sym Sym; +typedef struct Node Node; +typedef struct NodeList NodeList; +typedef struct Type Type; +typedef struct Label Label; + +struct Type +{ + uchar etype; + uchar chan; + uchar recur; // to detect loops + uchar trecur; // to detect loops + uchar printed; + uchar embedded; // TFIELD embedded type + uchar siggen; + uchar funarg; + uchar copyany; + uchar local; // created in this file + uchar deferwidth; + uchar broke; + uchar isddd; // TFIELD is ... argument + uchar align; + + Node* nod; // canonical OTYPE node + Type* orig; // original type (type literal or predefined type) + int lineno; + + // TFUNCT + uchar thistuple; + uchar outtuple; + uchar intuple; + uchar outnamed; + + Type* method; + Type* xmethod; + + Sym* sym; + int32 vargen; // unique name for OTYPE/ONAME + + Node* nname; + vlong argwid; + + // most nodes + Type* type; + vlong width; // offset in TFIELD, width in all others + + // TFIELD + Type* down; // also used in TMAP + Strlit* note; // literal string annotation + + // TARRAY + int32 bound; // negative is dynamic array + + int32 maplineno; // first use of TFORW as map key + int32 embedlineno; // first use of TFORW as embedded type +}; +#define T ((Type*)0) + +struct Node +{ + uchar op; + uchar ullman; // sethi/ullman number + uchar addable; // type of addressability - 0 is not addressable + uchar trecur; // to detect loops + uchar etype; // op for OASOP, etype for OTYPE, exclam for export + uchar class; // PPARAM, PAUTO, PEXTERN, etc + uchar method; // OCALLMETH name + uchar embedded; // ODCLFIELD embedded type + uchar colas; // OAS resulting from := + uchar diag; // already printed error about this + uchar noescape; // ONAME never move to heap + uchar funcdepth; + uchar builtin; // built-in name, like len or close + uchar walkdef; + uchar typecheck; + uchar local; + uchar initorder; + uchar dodata; // compile literal assignment as data statement + uchar used; + uchar isddd; + uchar pun; // don't registerize variable ONAME + uchar readonly; + uchar implicit; // don't show in printout + + // most nodes + Node* left; + Node* right; + Type* type; + Type* realtype; // as determined by typecheck + NodeList* list; + NodeList* rlist; + Node* orig; // original form, for printing, and tracking copies of ONAMEs + + // for-body + NodeList* ninit; + Node* ntest; + Node* nincr; + NodeList* nbody; + + // if-body + NodeList* nelse; + + // cases + Node* ncase; + + // func + Node* nname; + Node* shortname; + NodeList* enter; + NodeList* exit; + NodeList* cvars; // closure params + NodeList* dcl; // autodcl for this func/closure + + // OLITERAL/OREGISTER + Val val; + + // ONAME + Node* ntype; + Node* defn; + Node* pack; // real package for import . names + Node* curfn; // function for local variables + + // ONAME func param with PHEAP + Node* heapaddr; // temp holding heap address of param + Node* stackparam; // OPARAM node referring to stack copy of param + Node* alloc; // allocation call + + // ONAME closure param with PPARAMREF + Node* outer; // outer PPARAMREF in nested closure + Node* closure; // ONAME/PHEAP <-> ONAME/PPARAMREF + + // OPACK + Pkg* pkg; + + Sym* sym; // various + int32 vargen; // unique name for OTYPE/ONAME + int32 lineno; + int32 endlineno; + vlong xoffset; + int32 stkdelta; // offset added by stack frame compaction phase. + int32 ostk; + int32 iota; +}; +#define N ((Node*)0) +EXTERN int32 walkgen; + +struct NodeList +{ + Node* n; + NodeList* next; + NodeList* end; +}; + +enum +{ + SymExport = 1<<0, + SymPackage = 1<<1, + SymExported = 1<<2, + SymUniq = 1<<3, + SymSiggen = 1<<4, +}; + +struct Sym +{ + ushort lexical; + uchar flags; + uchar sym; // huffman encoding in object file + Sym* link; + int32 npkg; // number of imported packages with this name + + // saved and restored by dcopy + Pkg* pkg; + char* name; // variable name + Node* def; // definition: ONAME OTYPE OPACK or OLITERAL + Label* label; // corresponding label (ephemeral) + int32 block; // blocknumber to catch redeclaration + int32 lastlineno; // last declaration for diagnostic +}; +#define S ((Sym*)0) + +EXTERN Sym* dclstack; + +struct Pkg +{ + char* name; + Strlit* path; + Sym* pathsym; + char* prefix; + Pkg* link; + char exported; // import line written in export data + char direct; // imported directly +}; + +typedef struct Iter Iter; +struct Iter +{ + int done; + Type* tfunc; + Type* t; + Node** an; + Node* n; +}; + +typedef struct Hist Hist; +struct Hist +{ + Hist* link; + char* name; + int32 line; + int32 offset; +}; +#define H ((Hist*)0) + +enum +{ + OXXX, + + // names + ONAME, + ONONAME, + OTYPE, + OPACK, + OLITERAL, + + // exprs + OADD, OSUB, OOR, OXOR, OADDSTR, + OADDR, + OANDAND, + OAPPEND, + OARRAY, + OARRAYBYTESTR, OARRAYRUNESTR, + OSTRARRAYBYTE, OSTRARRAYRUNE, + OAS, OAS2, OAS2MAPW, OAS2FUNC, OAS2RECV, OAS2MAPR, OAS2DOTTYPE, OASOP, + OBAD, + OCALL, OCALLFUNC, OCALLMETH, OCALLINTER, + OCAP, + OCLOSE, + OCLOSURE, + OCMPIFACE, OCMPSTR, + OCOMPLIT, OMAPLIT, OSTRUCTLIT, OARRAYLIT, + OCONV, OCONVIFACE, OCONVNOP, + OCOPY, + ODCL, ODCLFUNC, ODCLFIELD, ODCLCONST, ODCLTYPE, + ODOT, ODOTPTR, ODOTMETH, ODOTINTER, OXDOT, + ODOTTYPE, + ODOTTYPE2, + OEQ, ONE, OLT, OLE, OGE, OGT, + OIND, + OINDEX, OINDEXMAP, + OKEY, OPARAM, + OLEN, + OMAKE, OMAKECHAN, OMAKEMAP, OMAKESLICE, + OHMUL, ORRC, OLRC, // high-mul and rotate-carry + OMUL, ODIV, OMOD, OLSH, ORSH, OAND, OANDNOT, + ONEW, + ONOT, OCOM, OPLUS, OMINUS, + OOROR, + OPANIC, OPRINT, OPRINTN, + OPAREN, + OSEND, + OSLICE, OSLICEARR, OSLICESTR, + ORECOVER, + ORECV, + ORUNESTR, + OSELRECV, + OSELRECV2, + OIOTA, + OREAL, OIMAG, OCOMPLEX, + + // stmts + OBLOCK, + OBREAK, + OCASE, OXCASE, + OCONTINUE, + ODEFER, + OEMPTY, + OFALL, OXFALL, + OFOR, + OGOTO, + OIF, + OLABEL, + OPROC, + ORANGE, + ORETURN, + OSELECT, + OSWITCH, + OTYPESW, // l = r.(type) + + // types + OTCHAN, + OTMAP, + OTSTRUCT, + OTINTER, + OTFUNC, + OTARRAY, + OTPAREN, + + // misc + ODDD, + + // for back ends + OCMP, ODEC, OEXTEND, OINC, OREGISTER, OINDREG, + + OEND, +}; +enum +{ + Txxx, // 0 + + TINT8, TUINT8, // 1 + TINT16, TUINT16, + TINT32, TUINT32, + TINT64, TUINT64, + TINT, TUINT, TUINTPTR, + + TCOMPLEX64, // 12 + TCOMPLEX128, + + TFLOAT32, // 14 + TFLOAT64, + + TBOOL, // 16 + + TPTR32, TPTR64, // 17 + + TFUNC, // 19 + TARRAY, + T_old_DARRAY, + TSTRUCT, // 22 + TCHAN, + TMAP, + TINTER, // 25 + TFORW, + TFIELD, + TANY, + TSTRING, + TUNSAFEPTR, + + // pseudo-types for literals + TIDEAL, // 31 + TNIL, + TBLANK, + + // pseudo-type for frame layout + TFUNCARGS, + TCHANARGS, + TINTERMETH, + + NTYPE, +}; +enum +{ + CTxxx, + + CTINT, + CTFLT, + CTCPLX, + CTSTR, + CTBOOL, + CTNIL, +}; + +enum +{ + /* types of channel */ + /* must match ../../pkg/nreflect/type.go:/Chandir */ + Cxxx, + Crecv = 1<<0, + Csend = 1<<1, + Cboth = Crecv | Csend, +}; + +enum +{ + Pxxx, + + PEXTERN, // declaration context + PAUTO, + PPARAM, + PPARAMOUT, + PPARAMREF, // param passed by reference + PFUNC, + + PHEAP = 1<<7, +}; + +enum +{ + Etop = 1<<1, // evaluated at statement level + Erv = 1<<2, // evaluated in value context + Etype = 1<<3, + Ecall = 1<<4, // call-only expressions are ok + Efnstruct = 1<<5, // multivalue function returns are ok + Eiota = 1<<6, // iota is ok + Easgn = 1<<7, // assigning to expression + Eindir = 1<<8, // indirecting through expression + Eaddr = 1<<9, // taking address of expression + Eproc = 1<<10, // inside a go statement + Ecomplit = 1<<11, // type in composite literal +}; + +#define BITS 5 +#define NVAR (BITS*sizeof(uint32)*8) + +typedef struct Bits Bits; +struct Bits +{ + uint32 b[BITS]; +}; + +EXTERN Bits zbits; + +typedef struct Var Var; +struct Var +{ + vlong offset; + Sym* sym; + Sym* gotype; + Node* node; + int width; + char name; + char etype; + char addr; +}; + +EXTERN Var var[NVAR]; + +typedef struct Typedef Typedef; +struct Typedef +{ + char* name; + int etype; + int sameas; +}; + +extern Typedef typedefs[]; + +typedef struct Sig Sig; +struct Sig +{ + char* name; + Pkg* pkg; + Sym* isym; + Sym* tsym; + Type* type; + Type* mtype; + int32 offset; + Sig* link; +}; + +typedef struct Io Io; +struct Io +{ + char* infile; + Biobuf* bin; + int32 ilineno; + int nlsemi; + int eofnl; + int peekc; + int peekc1; // second peekc for ... + char* cp; // used for content when bin==nil + int importsafe; +}; + +typedef struct Dlist Dlist; +struct Dlist +{ + Type* field; +}; + +typedef struct Idir Idir; +struct Idir +{ + Idir* link; + char* dir; +}; + +/* + * argument passing to/from + * smagic and umagic + */ +typedef struct Magic Magic; +struct Magic +{ + int w; // input for both - width + int s; // output for both - shift + int bad; // output for both - unexpected failure + + // magic multiplier for signed literal divisors + int64 sd; // input - literal divisor + int64 sm; // output - multiplier + + // magic multiplier for unsigned literal divisors + uint64 ud; // input - literal divisor + uint64 um; // output - multiplier + int ua; // output - adder +}; + +typedef struct Prog Prog; + +struct Label +{ + uchar used; + Sym* sym; + Node* def; + NodeList* use; + Label* link; + + // for use during gen + Prog* gotopc; // pointer to unresolved gotos + Prog* labelpc; // pointer to code + Prog* breakpc; // pointer to code + Prog* continpc; // pointer to code +}; +#define L ((Label*)0) + +/* + * note this is the runtime representation + * of the compilers arrays. + * + * typedef struct + * { // must not move anything + * uchar array[8]; // pointer to data + * uchar nel[4]; // number of elements + * uchar cap[4]; // allocated number of elements + * } Array; + */ +EXTERN int Array_array; // runtime offsetof(Array,array) - same for String +EXTERN int Array_nel; // runtime offsetof(Array,nel) - same for String +EXTERN int Array_cap; // runtime offsetof(Array,cap) +EXTERN int sizeof_Array; // runtime sizeof(Array) + + +/* + * note this is the runtime representation + * of the compilers strings. + * + * typedef struct + * { // must not move anything + * uchar array[8]; // pointer to data + * uchar nel[4]; // number of elements + * } String; + */ +EXTERN int sizeof_String; // runtime sizeof(String) + +EXTERN Dlist dotlist[10]; // size is max depth of embeddeds + +EXTERN Io curio; +EXTERN Io pushedio; +EXTERN int32 lexlineno; +EXTERN int32 lineno; +EXTERN int32 prevlineno; +EXTERN char* pathname; +EXTERN Hist* hist; +EXTERN Hist* ehist; + +EXTERN char* infile; +EXTERN char* outfile; +EXTERN Biobuf* bout; +EXTERN int nerrors; +EXTERN int nsavederrors; +EXTERN int nsyntaxerrors; +EXTERN int safemode; +EXTERN char namebuf[NSYMB]; +EXTERN char lexbuf[NSYMB]; +EXTERN char litbuf[NSYMB]; +EXTERN char debug[256]; +EXTERN Sym* hash[NHASH]; +EXTERN Sym* importmyname; // my name for package +EXTERN Pkg* localpkg; // package being compiled +EXTERN Pkg* importpkg; // package being imported +EXTERN Pkg* structpkg; // package that declared struct, during import +EXTERN Pkg* builtinpkg; // fake package for builtins +EXTERN Pkg* gostringpkg; // fake pkg for Go strings +EXTERN Pkg* runtimepkg; // package runtime +EXTERN Pkg* stringpkg; // fake package for C strings +EXTERN Pkg* typepkg; // fake package for runtime type info +EXTERN Pkg* unsafepkg; // package unsafe +EXTERN Pkg* phash[128]; +EXTERN int tptr; // either TPTR32 or TPTR64 +extern char* runtimeimport; +extern char* unsafeimport; +EXTERN Idir* idirs; + +EXTERN Type* types[NTYPE]; +EXTERN Type* idealstring; +EXTERN Type* idealbool; +EXTERN uchar simtype[NTYPE]; +EXTERN uchar isptr[NTYPE]; +EXTERN uchar isforw[NTYPE]; +EXTERN uchar isint[NTYPE]; +EXTERN uchar isfloat[NTYPE]; +EXTERN uchar iscomplex[NTYPE]; +EXTERN uchar issigned[NTYPE]; +EXTERN uchar issimple[NTYPE]; + +EXTERN uchar okforeq[NTYPE]; +EXTERN uchar okforadd[NTYPE]; +EXTERN uchar okforand[NTYPE]; +EXTERN uchar okfornone[NTYPE]; +EXTERN uchar okforcmp[NTYPE]; +EXTERN uchar okforbool[NTYPE]; +EXTERN uchar okforcap[NTYPE]; +EXTERN uchar okforlen[NTYPE]; +EXTERN uchar okforarith[NTYPE]; +EXTERN uchar okforconst[NTYPE]; +EXTERN uchar* okfor[OEND]; +EXTERN uchar iscmp[OEND]; + +EXTERN Mpint* minintval[NTYPE]; +EXTERN Mpint* maxintval[NTYPE]; +EXTERN Mpflt* minfltval[NTYPE]; +EXTERN Mpflt* maxfltval[NTYPE]; + +EXTERN NodeList* xtop; +EXTERN NodeList* externdcl; +EXTERN NodeList* closures; +EXTERN NodeList* exportlist; +EXTERN NodeList* typelist; +EXTERN int dclcontext; // PEXTERN/PAUTO +EXTERN int incannedimport; +EXTERN int statuniqgen; // name generator for static temps +EXTERN int loophack; + +EXTERN int32 iota; +EXTERN NodeList* lastconst; +EXTERN Node* lasttype; +EXTERN int32 maxarg; +EXTERN int32 stksize; // stack size for current frame +EXTERN int32 blockgen; // max block number +EXTERN int32 block; // current block number +EXTERN int hasdefer; // flag that curfn has defer statetment + +EXTERN Node* curfn; + +EXTERN int widthptr; + +EXTERN Node* typesw; +EXTERN Node* nblank; + +extern int thechar; +extern char* thestring; +EXTERN char* hunk; +EXTERN int32 nhunk; +EXTERN int32 thunk; + +EXTERN int exporting; +EXTERN int erroring; +EXTERN int noargnames; + +EXTERN int funcdepth; +EXTERN int typecheckok; +EXTERN int packagequotes; +EXTERN int longsymnames; +EXTERN int compiling_runtime; + +/* + * y.tab.c + */ +int yyparse(void); + +/* + * align.c + */ +int argsize(Type *t); +void checkwidth(Type *t); +void defercheckwidth(void); +void dowidth(Type *t); +void resumecheckwidth(void); +uint32 rnd(uint32 o, uint32 r); +void typeinit(void); + +/* + * bits.c + */ +int Qconv(Fmt *fp); +Bits band(Bits a, Bits b); +int bany(Bits *a); +int beq(Bits a, Bits b); +int bitno(int32 b); +Bits blsh(uint n); +Bits bnot(Bits a); +int bnum(Bits a); +Bits bor(Bits a, Bits b); +int bset(Bits a, uint n); + +/* + * closure.c + */ +Node* closurebody(NodeList *body); +void closurehdr(Node *ntype); +void typecheckclosure(Node *func, int top); +Node* walkclosure(Node *func, NodeList **init); +void walkcallclosure(Node *n, NodeList **init); + +/* + * const.c + */ +int cmpslit(Node *l, Node *r); +int consttype(Node *n); +void convconst(Node *con, Type *t, Val *val); +void convlit(Node **np, Type *t); +void convlit1(Node **np, Type *t, int explicit); +void defaultlit(Node **np, Type *t); +void defaultlit2(Node **lp, Node **rp, int force); +void evconst(Node *n); +int isconst(Node *n, int ct); +Node* nodcplxlit(Val r, Val i); +Node* nodlit(Val v); +long nonnegconst(Node *n); +void overflow(Val v, Type *t); +int smallintconst(Node *n); +Val toint(Val v); +Mpflt* truncfltlit(Mpflt *oldv, Type *t); + +/* + * cplx.c + */ +void complexadd(int op, Node *nl, Node *nr, Node *res); +void complexbool(int op, Node *nl, Node *nr, int true, Prog *to); +void complexgen(Node *n, Node *res); +void complexminus(Node *nl, Node *res); +void complexmove(Node *f, Node *t); +void complexmul(Node *nl, Node *nr, Node *res); +int complexop(Node *n, Node *res); +void nodfconst(Node *n, Type *t, Mpflt* fval); + +/* + * dcl.c + */ +void addmethod(Sym *sf, Type *t, int local); +void addvar(Node *n, Type *t, int ctxt); +NodeList* checkarglist(NodeList *all, int input); +Node* colas(NodeList *left, NodeList *right); +void colasdefn(NodeList *left, Node *defn); +NodeList* constiter(NodeList *vl, Node *t, NodeList *cl); +Node* dclname(Sym *s); +void declare(Node *n, int ctxt); +Type* dostruct(NodeList *l, int et); +void dumpdcl(char *st); +Node* embedded(Sym *s); +Node* fakethis(void); +void funcbody(Node *n); +void funccompile(Node *n, int isclosure); +void funchdr(Node *n); +Type* functype(Node *this, NodeList *in, NodeList *out); +void ifacedcl(Node *n); +int isifacemethod(Type *f); +void markdcl(void); +Node* methodname(Node *n, Type *t); +Node* methodname1(Node *n, Node *t); +Sym* methodsym(Sym *nsym, Type *t0, int iface); +Node* newname(Sym *s); +Type* newtype(Sym *s); +Node* oldname(Sym *s); +void popdcl(void); +void poptodcl(void); +void redeclare(Sym *s, char *where); +void testdclstack(void); +Node* typedcl0(Sym *s); +Node* typedcl1(Node *n, Node *t, int local); +void typedcl2(Type *pt, Type *t); +Node* typenod(Type *t); +NodeList* variter(NodeList *vl, Node *t, NodeList *el); + +/* + * export.c + */ +void autoexport(Node *n, int ctxt); +void dumpexport(void); +int exportname(char *s); +void exportsym(Node *n); +void importconst(Sym *s, Type *t, Node *n); +void importmethod(Sym *s, Type *t); +Sym* importsym(Sym *s, int op); +void importtype(Type *pt, Type *t); +void importvar(Sym *s, Type *t, int ctxt); +Type* pkgtype(Sym *s); + +/* + * gen.c + */ +void allocparams(void); +void cgen_as(Node *nl, Node *nr); +void cgen_callmeth(Node *n, int proc); +void clearlabels(void); +void checklabels(void); +int dotoffset(Node *n, int *oary, Node **nn); +void gen(Node *n); +void genlist(NodeList *l); +Node* sysfunc(char *name); +void tempname(Node *n, Type *t); + +/* + * init.c + */ +void fninit(NodeList *n); +Node* renameinit(Node *n); + +/* + * lex.c + */ +void cannedimports(char *file, char *cp); +void importfile(Val *f, int line); +char* lexname(int lex); +void mkpackage(char* pkgname); +void unimportfile(void); +int32 yylex(void); +extern int windows; +extern int yylast; +extern int yyprev; + +/* + * mparith1.c + */ +int Bconv(Fmt *fp); +int Fconv(Fmt *fp); +void mpaddcfix(Mpint *a, vlong c); +void mpaddcflt(Mpflt *a, double c); +void mpatofix(Mpint *a, char *as); +void mpatoflt(Mpflt *a, char *as); +int mpcmpfixc(Mpint *b, vlong c); +int mpcmpfixfix(Mpint *a, Mpint *b); +int mpcmpfixflt(Mpint *a, Mpflt *b); +int mpcmpfltc(Mpflt *b, double c); +int mpcmpfltfix(Mpflt *a, Mpint *b); +int mpcmpfltflt(Mpflt *a, Mpflt *b); +void mpcomfix(Mpint *a); +void mpdivfixfix(Mpint *a, Mpint *b); +void mpmodfixfix(Mpint *a, Mpint *b); +void mpmovefixfix(Mpint *a, Mpint *b); +void mpmovefixflt(Mpflt *a, Mpint *b); +int mpmovefltfix(Mpint *a, Mpflt *b); +void mpmovefltflt(Mpflt *a, Mpflt *b); +void mpmulcfix(Mpint *a, vlong c); +void mpmulcflt(Mpflt *a, double c); +void mpsubfixfix(Mpint *a, Mpint *b); +void mpsubfltflt(Mpflt *a, Mpflt *b); + +/* + * mparith2.c + */ +void mpaddfixfix(Mpint *a, Mpint *b); +void mpandfixfix(Mpint *a, Mpint *b); +void mpandnotfixfix(Mpint *a, Mpint *b); +void mpdivfract(Mpint *a, Mpint *b); +void mpdivmodfixfix(Mpint *q, Mpint *r, Mpint *n, Mpint *d); +vlong mpgetfix(Mpint *a); +void mplshfixfix(Mpint *a, Mpint *b); +void mpmovecfix(Mpint *a, vlong c); +void mpmulfixfix(Mpint *a, Mpint *b); +void mpmulfract(Mpint *a, Mpint *b); +void mpnegfix(Mpint *a); +void mporfixfix(Mpint *a, Mpint *b); +void mprshfixfix(Mpint *a, Mpint *b); +void mpshiftfix(Mpint *a, int s); +int mptestfix(Mpint *a); +void mpxorfixfix(Mpint *a, Mpint *b); + +/* + * mparith3.c + */ +void mpaddfltflt(Mpflt *a, Mpflt *b); +void mpdivfltflt(Mpflt *a, Mpflt *b); +double mpgetflt(Mpflt *a); +void mpmovecflt(Mpflt *a, double c); +void mpmulfltflt(Mpflt *a, Mpflt *b); +void mpnegflt(Mpflt *a); +void mpnorm(Mpflt *a); +int mptestflt(Mpflt *a); +int sigfig(Mpflt *a); + +/* + * obj.c + */ +void Bputname(Biobuf *b, Sym *s); +int duint16(Sym *s, int off, uint16 v); +int duint32(Sym *s, int off, uint32 v); +int duint64(Sym *s, int off, uint64 v); +int duint8(Sym *s, int off, uint8 v); +int duintptr(Sym *s, int off, uint64 v); +int dsname(Sym *s, int off, char *dat, int ndat); +void dumpobj(void); +void ieeedtod(uint64 *ieee, double native); +Sym* stringsym(char*, int); + +/* + * print.c + */ +void exprfmt(Fmt *f, Node *n, int prec); +void exprlistfmt(Fmt *f, NodeList *l); + +/* + * range.c + */ +void typecheckrange(Node *n); +void walkrange(Node *n); + +/* + * reflect.c + */ +void dumptypestructs(void); +Type* methodfunc(Type *f, Type*); +Node* typename(Type *t); +Sym* typesym(Type *t); + +/* + * select.c + */ +void typecheckselect(Node *sel); +void walkselect(Node *sel); + +/* + * sinit.c + */ +void anylit(int, Node *n, Node *var, NodeList **init); +int gen_as_init(Node *n); +NodeList* initfix(NodeList *l); +int oaslit(Node *n, NodeList **init); +int stataddr(Node *nam, Node *n); + +/* + * subr.c + */ +int Econv(Fmt *fp); +int Jconv(Fmt *fp); +int Lconv(Fmt *fp); +int Nconv(Fmt *fp); +int Oconv(Fmt *fp); +int Sconv(Fmt *fp); +int Tconv(Fmt *fp); +int Tpretty(Fmt *fp, Type *t); +int Zconv(Fmt *fp); +Node* adddot(Node *n); +int adddot1(Sym *s, Type *t, int d, Type **save, int ignorecase); +Type* aindex(Node *b, Type *t); +int algtype(Type *t); +void argtype(Node *on, Type *t); +Node* assignconv(Node *n, Type *t, char *context); +int assignop(Type *src, Type *dst, char **why); +void badtype(int o, Type *tl, Type *tr); +int brcom(int a); +int brrev(int a); +NodeList* concat(NodeList *a, NodeList *b); +int convertop(Type *src, Type *dst, char **why); +int count(NodeList *l); +int cplxsubtype(int et); +void dump(char *s, Node *n); +void dumplist(char *s, NodeList *l); +int eqtype(Type *t1, Type *t2); +int eqtypenoname(Type *t1, Type *t2); +void errorexit(void); +void expandmeth(Sym *s, Type *t); +void fatal(char *fmt, ...); +void flusherrors(void); +void frame(int context); +Type* funcfirst(Iter *s, Type *t); +Type* funcnext(Iter *s); +void genwrapper(Type *rcvr, Type *method, Sym *newnam, int iface); +Type** getinarg(Type *t); +Type* getinargx(Type *t); +Type** getoutarg(Type *t); +Type* getoutargx(Type *t); +Type** getthis(Type *t); +Type* getthisx(Type *t); +int implements(Type *t, Type *iface, Type **missing, Type **have, int *ptr); +void importdot(Pkg *opkg, Node *pack); +int is64(Type *t); +int isblank(Node *n); +int isfixedarray(Type *t); +int isideal(Type *t); +int isinter(Type *t); +int isnil(Node *n); +int isnilinter(Type *t); +int isptrto(Type *t, int et); +int isslice(Type *t); +int istype(Type *t, int et); +void linehist(char *file, int32 off, int relative); +NodeList* list(NodeList *l, Node *n); +NodeList* list1(Node *n); +void listsort(NodeList**, int(*f)(Node*, Node*)); +Node* liststmt(NodeList *l); +NodeList* listtreecopy(NodeList *l); +Sym* lookup(char *name); +void* mal(int32 n); +Type* maptype(Type *key, Type *val); +Type* methtype(Type *t); +Pkg* mkpkg(Strlit *path); +Sym* ngotype(Node *n); +int noconv(Type *t1, Type *t2); +Node* nod(int op, Node *nleft, Node *nright); +Node* nodbool(int b); +void nodconst(Node *n, Type *t, int64 v); +Node* nodintconst(int64 v); +Node* nodfltconst(Mpflt *v); +Node* nodnil(void); +int parserline(void); +Sym* pkglookup(char *name, Pkg *pkg); +int powtwo(Node *n); +Type* ptrto(Type *t); +void* remal(void *p, int32 on, int32 n); +Sym* restrictlookup(char *name, Pkg *pkg); +Node* safeexpr(Node *n, NodeList **init); +void saveerrors(void); +Node* cheapexpr(Node *n, NodeList **init); +Node* localexpr(Node *n, Type *t, NodeList **init); +int32 setlineno(Node *n); +void setmaxarg(Type *t); +Type* shallow(Type *t); +int simsimtype(Type *t); +void smagic(Magic *m); +Type* sortinter(Type *t); +uint32 stringhash(char *p); +Strlit* strlit(char *s); +int structcount(Type *t); +Type* structfirst(Iter *s, Type **nn); +Type* structnext(Iter *s); +Node* syslook(char *name, int copy); +Type* tounsigned(Type *t); +Node* treecopy(Node *n); +Type* typ(int et); +uint32 typehash(Type *t); +void ullmancalc(Node *n); +void umagic(Magic *m); +void warn(char *fmt, ...); +void yyerror(char *fmt, ...); +void yyerrorl(int line, char *fmt, ...); + +/* + * swt.c + */ +void typecheckswitch(Node *n); +void walkswitch(Node *sw); + +/* + * typecheck.c + */ +int exportassignok(Type *t, char *desc); +int islvalue(Node *n); +Node* typecheck(Node **np, int top); +void typechecklist(NodeList *l, int top); +Node* typecheckdef(Node *n); +void resumetypecopy(void); +void copytype(Node *n, Type *t); +void defertypecopy(Node *n, Type *t); +void queuemethod(Node *n); + +/* + * unsafe.c + */ +Node* unsafenmagic(Node *n); + +/* + * walk.c + */ +Node* callnew(Type *t); +Node* chanfn(char *name, int n, Type *t); +Node* mkcall(char *name, Type *t, NodeList **init, ...); +Node* mkcall1(Node *fn, Type *t, NodeList **init, ...); +int vmatch1(Node *l, Node *r); +void walk(Node *fn); +void walkexpr(Node **np, NodeList **init); +void walkexprlist(NodeList *l, NodeList **init); +void walkexprlistsafe(NodeList *l, NodeList **init); +void walkstmt(Node **np); +void walkstmtlist(NodeList *l); + +/* + * arch-specific ggen.c/gsubr.c/gobj.c/pgen.c + */ +#define P ((Prog*)0) + +typedef struct Plist Plist; +struct Plist +{ + Node* name; + Prog* firstpc; + int recur; + Plist* link; +}; + +EXTERN Plist* plist; +EXTERN Plist* plast; + +EXTERN Prog* continpc; +EXTERN Prog* breakpc; +EXTERN Prog* pc; +EXTERN Prog* firstpc; +EXTERN Prog* retpc; + +EXTERN Node* nodfp; + +int anyregalloc(void); +void betypeinit(void); +void bgen(Node *n, int true, Prog *to); +void cgen(Node*, Node*); +void cgen_asop(Node *n); +void cgen_call(Node *n, int proc); +void cgen_callinter(Node *n, Node *res, int proc); +void cgen_ret(Node *n); +void clearfat(Node *n); +void compile(Node*); +void defframe(Prog*); +int dgostringptr(Sym*, int off, char *str); +int dgostrlitptr(Sym*, int off, Strlit*); +int dstringptr(Sym *s, int off, char *str); +int dsymptr(Sym *s, int off, Sym *x, int xoff); +int duintxx(Sym *s, int off, uint64 v, int wid); +void dumpdata(void); +void dumpfuncs(void); +void fixautoused(Prog*); +void gdata(Node*, Node*, int); +void gdatacomplex(Node*, Mpcplx*); +void gdatastring(Node*, Strlit*); +void genembedtramp(Type*, Type*, Sym*, int iface); +void ggloblnod(Node *nam, int32 width); +void ggloblsym(Sym *s, int32 width, int dupok); +Prog* gjmp(Prog*); +void gused(Node*); +int isfat(Type*); +void markautoused(Prog*); +Plist* newplist(void); +Node* nodarg(Type*, int); +void nopout(Prog*); +void patch(Prog*, Prog*); +Prog* unpatch(Prog*); +void zfile(Biobuf *b, char *p, int n); +void zhist(Biobuf *b, int line, vlong offset); +void zname(Biobuf *b, Sym *s, int t); +void data(void); +void text(void); + diff --git a/src/cmd/gc/go.y b/src/cmd/gc/go.y new file mode 100644 index 000000000..4c7fe6068 --- /dev/null +++ b/src/cmd/gc/go.y @@ -0,0 +1,1985 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +/* + * Go language grammar. + * + * The Go semicolon rules are: + * + * 1. all statements and declarations are terminated by semicolons. + * 2. semicolons can be omitted before a closing ) or }. + * 3. semicolons are inserted by the lexer before a newline + * following a specific list of tokens. + * + * Rules #1 and #2 are accomplished by writing the lists as + * semicolon-separated lists with an optional trailing semicolon. + * Rule #3 is implemented in yylex. + */ + +%{ +#include /* if we don't, bison will, and go.h re-#defines getc */ +#include "go.h" + +static void fixlbrace(int); +%} +%union { + Node* node; + NodeList* list; + Type* type; + Sym* sym; + struct Val val; + int lint; +} + +// |sed 's/.* //' |9 fmt -l1 |sort |9 fmt -l50 | sed 's/^/%xxx /' + +%token LLITERAL +%token LASOP +%token LBREAK LCASE LCHAN LCOLAS LCONST LCONTINUE LDDD +%token LDEFAULT LDEFER LELSE LFALL LFOR LFUNC LGO LGOTO +%token LIF LIMPORT LINTERFACE LMAP LNAME +%token LPACKAGE LRANGE LRETURN LSELECT LSTRUCT LSWITCH +%token LTYPE LVAR + +%token LANDAND LANDNOT LBODY LCOMM LDEC LEQ LGE LGT +%token LIGNORE LINC LLE LLSH LLT LNE LOROR LRSH + +%type lbrace import_here +%type sym packname +%type oliteral + +%type stmt ntype +%type arg_type +%type case caseblock +%type compound_stmt dotname embed expr complitexpr +%type expr_or_type +%type fndcl fnliteral +%type for_body for_header for_stmt if_header if_stmt non_dcl_stmt +%type interfacedcl keyval labelname name +%type name_or_type non_expr_type +%type new_name dcl_name oexpr typedclname +%type onew_name +%type osimple_stmt pexpr pexpr_no_paren +%type pseudocall range_stmt select_stmt +%type simple_stmt +%type switch_stmt uexpr +%type xfndcl typedcl + +%type xdcl fnbody fnres loop_body dcl_name_list +%type new_name_list expr_list keyval_list braced_keyval_list expr_or_type_list xdcl_list +%type oexpr_list caseblock_list stmt_list oarg_type_list_ocomma arg_type_list +%type interfacedcl_list vardcl vardcl_list structdcl structdcl_list +%type common_dcl constdcl constdcl1 constdcl_list typedcl_list + +%type convtype comptype dotdotdot +%type indcl interfacetype structtype ptrtype +%type recvchantype non_recvchantype othertype fnret_type fntype + +%type hidden_tag + +%type hidden_importsym hidden_pkg_importsym + +%type hidden_constant hidden_literal hidden_dcl +%type hidden_interfacedcl hidden_structdcl hidden_opt_sym + +%type hidden_funres +%type ohidden_funres +%type hidden_funarg_list ohidden_funarg_list +%type hidden_interfacedcl_list ohidden_interfacedcl_list +%type hidden_structdcl_list ohidden_structdcl_list + +%type hidden_type hidden_type_misc hidden_pkgtype +%type hidden_type_func +%type hidden_type_recv_chan hidden_type_non_recv_chan + +%left LCOMM /* outside the usual hierarchy; here for good error messages */ + +%left LOROR +%left LANDAND +%left LEQ LNE LLE LGE LLT LGT +%left '+' '-' '|' '^' +%left '*' '/' '%' '&' LLSH LRSH LANDNOT + +/* + * manual override of shift/reduce conflicts. + * the general form is that we assign a precedence + * to the token being shifted and then introduce + * NotToken with lower precedence or PreferToToken with higher + * and annotate the reducing rule accordingly. + */ +%left NotPackage +%left LPACKAGE + +%left NotParen +%left '(' + +%left ')' +%left PreferToRightParen + +%error-verbose + +%% +file: + loadsys + package + imports + xdcl_list + { + xtop = concat(xtop, $4); + } + +package: + %prec NotPackage + { + prevlineno = lineno; + yyerror("package statement must be first"); + flusherrors(); + mkpackage("main"); + } +| LPACKAGE sym ';' + { + mkpackage($2->name); + } + +/* + * this loads the definitions for the low-level runtime functions, + * so that the compiler can generate calls to them, + * but does not make the name "runtime" visible as a package. + */ +loadsys: + { + importpkg = runtimepkg; + + if(debug['A']) + cannedimports("runtime.builtin", "package runtime\n\n$$\n\n"); + else + cannedimports("runtime.builtin", runtimeimport); + curio.importsafe = 1; + } + import_package + import_there + { + importpkg = nil; + } + +imports: +| imports import ';' + +import: + LIMPORT import_stmt +| LIMPORT '(' import_stmt_list osemi ')' +| LIMPORT '(' ')' + +import_stmt: + import_here import_package import_there + { + Pkg *ipkg; + Sym *my; + Node *pack; + + ipkg = importpkg; + my = importmyname; + importpkg = nil; + importmyname = S; + + if(my == nil) + my = lookup(ipkg->name); + + pack = nod(OPACK, N, N); + pack->sym = my; + pack->pkg = ipkg; + pack->lineno = $1; + + if(my->name[0] == '.') { + importdot(ipkg, pack); + break; + } + if(my->name[0] == '_' && my->name[1] == '\0') + break; + if(my->def) { + lineno = $1; + redeclare(my, "as imported package name"); + } + my->def = pack; + my->lastlineno = $1; + my->block = 1; // at top level + } + + +import_stmt_list: + import_stmt +| import_stmt_list ';' import_stmt + +import_here: + LLITERAL + { + // import with original name + $$ = parserline(); + importmyname = S; + importfile(&$1, $$); + } +| sym LLITERAL + { + // import with given name + $$ = parserline(); + importmyname = $1; + importfile(&$2, $$); + } +| '.' LLITERAL + { + // import into my name space + $$ = parserline(); + importmyname = lookup("."); + importfile(&$2, $$); + } + +import_package: + LPACKAGE sym import_safety ';' + { + if(importpkg->name == nil) { + importpkg->name = $2->name; + pkglookup($2->name, nil)->npkg++; + } else if(strcmp(importpkg->name, $2->name) != 0) + yyerror("conflicting names %s and %s for package %Z", importpkg->name, $2->name, importpkg->path); + importpkg->direct = 1; + + if(safemode && !curio.importsafe) + yyerror("cannot import unsafe package %Z", importpkg->path); + } + +import_safety: +| LNAME + { + if(strcmp($1->name, "safe") == 0) + curio.importsafe = 1; + } + +import_there: + { + defercheckwidth(); + } + hidden_import_list '$' '$' + { + resumecheckwidth(); + unimportfile(); + } + +/* + * declarations + */ +xdcl: + { + yyerror("empty top-level declaration"); + $$ = nil; + } +| common_dcl +| xfndcl + { + $$ = list1($1); + } +| non_dcl_stmt + { + yyerror("non-declaration statement outside function body"); + $$ = nil; + } +| error + { + $$ = nil; + } + +common_dcl: + LVAR vardcl + { + $$ = $2; + } +| LVAR '(' vardcl_list osemi ')' + { + $$ = $3; + } +| LVAR '(' ')' + { + $$ = nil; + } +| lconst constdcl + { + $$ = $2; + iota = -100000; + lastconst = nil; + } +| lconst '(' constdcl osemi ')' + { + $$ = $3; + iota = -100000; + lastconst = nil; + } +| lconst '(' constdcl ';' constdcl_list osemi ')' + { + $$ = concat($3, $5); + iota = -100000; + lastconst = nil; + } +| lconst '(' ')' + { + $$ = nil; + iota = -100000; + } +| LTYPE typedcl + { + $$ = list1($2); + } +| LTYPE '(' typedcl_list osemi ')' + { + $$ = $3; + } +| LTYPE '(' ')' + { + $$ = nil; + } + +lconst: + LCONST + { + iota = 0; + } + +vardcl: + dcl_name_list ntype + { + $$ = variter($1, $2, nil); + } +| dcl_name_list ntype '=' expr_list + { + $$ = variter($1, $2, $4); + } +| dcl_name_list '=' expr_list + { + $$ = variter($1, nil, $3); + } + +constdcl: + dcl_name_list ntype '=' expr_list + { + $$ = constiter($1, $2, $4); + } +| dcl_name_list '=' expr_list + { + $$ = constiter($1, N, $3); + } + +constdcl1: + constdcl +| dcl_name_list ntype + { + $$ = constiter($1, $2, nil); + } +| dcl_name_list + { + $$ = constiter($1, N, nil); + } + +typedclname: + sym + { + // different from dclname because the name + // becomes visible right here, not at the end + // of the declaration. + $$ = typedcl0($1); + } + +typedcl: + typedclname ntype + { + $$ = typedcl1($1, $2, 1); + } + +simple_stmt: + expr + { + $$ = $1; + } +| expr LASOP expr + { + $$ = nod(OASOP, $1, $3); + $$->etype = $2; // rathole to pass opcode + } +| expr_list '=' expr_list + { + if($1->next == nil && $3->next == nil) { + // simple + $$ = nod(OAS, $1->n, $3->n); + break; + } + // multiple + $$ = nod(OAS2, N, N); + $$->list = $1; + $$->rlist = $3; + } +| expr_list LCOLAS expr_list + { + if($3->n->op == OTYPESW) { + Node *n; + + n = N; + if($3->next != nil) + yyerror("expr.(type) must be alone in list"); + if($1->next != nil) + yyerror("argument count mismatch: %d = %d", count($1), 1); + else if($1->n->op != ONAME && $1->n->op != OTYPE && $1->n->op != ONONAME) + yyerror("invalid variable name %#N in type switch", $1->n); + else + n = $1->n; + $$ = nod(OTYPESW, n, $3->n->right); + break; + } + $$ = colas($1, $3); + } +| expr LINC + { + $$ = nod(OASOP, $1, nodintconst(1)); + $$->etype = OADD; + } +| expr LDEC + { + $$ = nod(OASOP, $1, nodintconst(1)); + $$->etype = OSUB; + } + +case: + LCASE expr_or_type_list ':' + { + Node *n; + + // will be converted to OCASE + // right will point to next case + // done in casebody() + markdcl(); + $$ = nod(OXCASE, N, N); + $$->list = $2; + if(typesw != N && typesw->right != N && (n=typesw->right->left) != N) { + // type switch - declare variable + n = newname(n->sym); + n->used = 1; // TODO(rsc): better job here + declare(n, dclcontext); + $$->nname = n; + } + break; + } +| LCASE expr_or_type_list '=' expr ':' + { + Node *n; + + // will be converted to OCASE + // right will point to next case + // done in casebody() + markdcl(); + $$ = nod(OXCASE, N, N); + if($2->next == nil) + n = nod(OAS, $2->n, $4); + else { + n = nod(OAS2, N, N); + n->list = $2; + n->rlist = list1($4); + } + $$->list = list1(n); + } +| LCASE expr_or_type_list LCOLAS expr ':' + { + // will be converted to OCASE + // right will point to next case + // done in casebody() + markdcl(); + $$ = nod(OXCASE, N, N); + $$->list = list1(colas($2, list1($4))); + } +| LDEFAULT ':' + { + Node *n; + + markdcl(); + $$ = nod(OXCASE, N, N); + if(typesw != N && typesw->right != N && (n=typesw->right->left) != N) { + // type switch - declare variable + n = newname(n->sym); + n->used = 1; // TODO(rsc): better job here + declare(n, dclcontext); + $$->nname = n; + } + } + +compound_stmt: + '{' + { + markdcl(); + } + stmt_list '}' + { + $$ = liststmt($3); + popdcl(); + } + +caseblock: + case + { + // If the last token read by the lexer was consumed + // as part of the case, clear it (parser has cleared yychar). + // If the last token read by the lexer was the lookahead + // leave it alone (parser has it cached in yychar). + // This is so that the stmt_list action doesn't look at + // the case tokens if the stmt_list is empty. + yylast = yychar; + } + stmt_list + { + int last; + + // This is the only place in the language where a statement + // list is not allowed to drop the final semicolon, because + // it's the only place where a statement list is not followed + // by a closing brace. Handle the error for pedantry. + + // Find the final token of the statement list. + // yylast is lookahead; yyprev is last of stmt_list + last = yyprev; + + if(last > 0 && last != ';' && yychar != '}') + yyerror("missing statement after label"); + $$ = $1; + $$->nbody = $3; + popdcl(); + } + +caseblock_list: + { + $$ = nil; + } +| caseblock_list caseblock + { + $$ = list($1, $2); + } + +loop_body: + LBODY + { + markdcl(); + } + stmt_list '}' + { + $$ = $3; + popdcl(); + } + +range_stmt: + expr_list '=' LRANGE expr + { + $$ = nod(ORANGE, N, $4); + $$->list = $1; + $$->etype = 0; // := flag + } +| expr_list LCOLAS LRANGE expr + { + $$ = nod(ORANGE, N, $4); + $$->list = $1; + $$->colas = 1; + colasdefn($1, $$); + } + +for_header: + osimple_stmt ';' osimple_stmt ';' osimple_stmt + { + // init ; test ; incr + if($5 != N && $5->colas != 0) + yyerror("cannot declare in the for-increment"); + $$ = nod(OFOR, N, N); + if($1 != N) + $$->ninit = list1($1); + $$->ntest = $3; + $$->nincr = $5; + } +| osimple_stmt + { + // normal test + $$ = nod(OFOR, N, N); + $$->ntest = $1; + } +| range_stmt + +for_body: + for_header loop_body + { + $$ = $1; + $$->nbody = concat($$->nbody, $2); + } + +for_stmt: + LFOR + { + markdcl(); + } + for_body + { + $$ = $3; + popdcl(); + } + +if_header: + osimple_stmt + { + // test + $$ = nod(OIF, N, N); + $$->ntest = $1; + } +| osimple_stmt ';' osimple_stmt + { + // init ; test + $$ = nod(OIF, N, N); + if($1 != N) + $$->ninit = list1($1); + $$->ntest = $3; + } + +if_stmt: + LIF + { + markdcl(); + } + if_header + { + if($3->ntest == N) + yyerror("missing condition in if statement"); + } + loop_body + { + $$ = $3; + $$->nbody = $5; + // no popdcl; maybe there's an LELSE + } + +switch_stmt: + LSWITCH + { + markdcl(); + } + if_header + { + Node *n; + n = $3->ntest; + if(n != N && n->op != OTYPESW) + n = N; + typesw = nod(OXXX, typesw, n); + } + LBODY caseblock_list '}' + { + $$ = $3; + $$->op = OSWITCH; + $$->list = $6; + typesw = typesw->left; + popdcl(); + } + +select_stmt: + LSELECT + { + typesw = nod(OXXX, typesw, N); + } + LBODY caseblock_list '}' + { + $$ = nod(OSELECT, N, N); + $$->lineno = typesw->lineno; + $$->list = $4; + typesw = typesw->left; + } + +/* + * expressions + */ +expr: + uexpr +| expr LOROR expr + { + $$ = nod(OOROR, $1, $3); + } +| expr LANDAND expr + { + $$ = nod(OANDAND, $1, $3); + } +| expr LEQ expr + { + $$ = nod(OEQ, $1, $3); + } +| expr LNE expr + { + $$ = nod(ONE, $1, $3); + } +| expr LLT expr + { + $$ = nod(OLT, $1, $3); + } +| expr LLE expr + { + $$ = nod(OLE, $1, $3); + } +| expr LGE expr + { + $$ = nod(OGE, $1, $3); + } +| expr LGT expr + { + $$ = nod(OGT, $1, $3); + } +| expr '+' expr + { + $$ = nod(OADD, $1, $3); + } +| expr '-' expr + { + $$ = nod(OSUB, $1, $3); + } +| expr '|' expr + { + $$ = nod(OOR, $1, $3); + } +| expr '^' expr + { + $$ = nod(OXOR, $1, $3); + } +| expr '*' expr + { + $$ = nod(OMUL, $1, $3); + } +| expr '/' expr + { + $$ = nod(ODIV, $1, $3); + } +| expr '%' expr + { + $$ = nod(OMOD, $1, $3); + } +| expr '&' expr + { + $$ = nod(OAND, $1, $3); + } +| expr LANDNOT expr + { + $$ = nod(OANDNOT, $1, $3); + } +| expr LLSH expr + { + $$ = nod(OLSH, $1, $3); + } +| expr LRSH expr + { + $$ = nod(ORSH, $1, $3); + } + /* not an expression anymore, but left in so we can give a good error */ +| expr LCOMM expr + { + $$ = nod(OSEND, $1, $3); + } + +uexpr: + pexpr +| '*' uexpr + { + $$ = nod(OIND, $2, N); + } +| '&' uexpr + { + $$ = nod(OADDR, $2, N); + } +| '+' uexpr + { + $$ = nod(OPLUS, $2, N); + } +| '-' uexpr + { + $$ = nod(OMINUS, $2, N); + } +| '!' uexpr + { + $$ = nod(ONOT, $2, N); + } +| '~' uexpr + { + yyerror("the bitwise complement operator is ^"); + $$ = nod(OCOM, $2, N); + } +| '^' uexpr + { + $$ = nod(OCOM, $2, N); + } +| LCOMM uexpr + { + $$ = nod(ORECV, $2, N); + } + +/* + * call-like statements that + * can be preceded by 'defer' and 'go' + */ +pseudocall: + pexpr '(' ')' + { + $$ = nod(OCALL, $1, N); + } +| pexpr '(' expr_or_type_list ocomma ')' + { + $$ = nod(OCALL, $1, N); + $$->list = $3; + } +| pexpr '(' expr_or_type_list LDDD ocomma ')' + { + $$ = nod(OCALL, $1, N); + $$->list = $3; + $$->isddd = 1; + } + +pexpr_no_paren: + LLITERAL + { + $$ = nodlit($1); + } +| name +| pexpr '.' sym + { + if($1->op == OPACK) { + Sym *s; + s = restrictlookup($3->name, $1->pkg); + $1->used = 1; + $$ = oldname(s); + break; + } + $$ = nod(OXDOT, $1, newname($3)); + } +| pexpr '.' '(' expr_or_type ')' + { + $$ = nod(ODOTTYPE, $1, $4); + } +| pexpr '.' '(' LTYPE ')' + { + $$ = nod(OTYPESW, N, $1); + } +| pexpr '[' expr ']' + { + $$ = nod(OINDEX, $1, $3); + } +| pexpr '[' oexpr ':' oexpr ']' + { + $$ = nod(OSLICE, $1, nod(OKEY, $3, $5)); + } +| pseudocall +| convtype '(' expr ')' + { + // conversion + $$ = nod(OCALL, $1, N); + $$->list = list1($3); + } +| comptype lbrace braced_keyval_list '}' + { + // composite expression + $$ = nod(OCOMPLIT, N, $1); + $$->list = $3; + + fixlbrace($2); + } +| pexpr_no_paren '{' braced_keyval_list '}' + { + // composite expression + $$ = nod(OCOMPLIT, N, $1); + $$->list = $3; + } +| '(' expr_or_type ')' '{' braced_keyval_list '}' + { + yyerror("cannot parenthesize type in composite literal"); + // composite expression + $$ = nod(OCOMPLIT, N, $2); + $$->list = $5; + } +| fnliteral + +keyval: + expr ':' complitexpr + { + $$ = nod(OKEY, $1, $3); + } + +complitexpr: + expr +| '{' braced_keyval_list '}' + { + $$ = nod(OCOMPLIT, N, N); + $$->list = $2; + } + +pexpr: + pexpr_no_paren +| '(' expr_or_type ')' + { + $$ = $2; + + // Need to know on lhs of := whether there are ( ). + // Don't bother with the OPAREN in other cases: + // it's just a waste of memory and time. + switch($$->op) { + case ONAME: + case ONONAME: + case OPACK: + case OTYPE: + case OLITERAL: + $$ = nod(OPAREN, $$, N); + } + } + +expr_or_type: + expr +| non_expr_type %prec PreferToRightParen + +name_or_type: + ntype + +lbrace: + LBODY + { + $$ = LBODY; + } +| '{' + { + $$ = '{'; + } + +/* + * names and types + * newname is used before declared + * oldname is used after declared + */ +new_name: + sym + { + $$ = newname($1); + } + +dcl_name: + sym + { + $$ = dclname($1); + } + +onew_name: + { + $$ = N; + } +| new_name + +sym: + LNAME + +name: + sym %prec NotParen + { + $$ = oldname($1); + if($$->pack != N) + $$->pack->used = 1; + } + +labelname: + new_name + +/* + * to avoid parsing conflicts, type is split into + * channel types + * function types + * parenthesized types + * any other type + * the type system makes additional restrictions, + * but those are not implemented in the grammar. + */ +dotdotdot: + LDDD + { + yyerror("final argument in variadic function missing type"); + $$ = nod(ODDD, typenod(typ(TINTER)), N); + } +| LDDD ntype + { + $$ = nod(ODDD, $2, N); + } + +ntype: + recvchantype +| fntype +| othertype +| ptrtype +| dotname +| '(' ntype ')' + { + $$ = nod(OTPAREN, $2, N); + } + +non_expr_type: + recvchantype +| fntype +| othertype +| '*' non_expr_type + { + $$ = nod(OIND, $2, N); + } + +non_recvchantype: + fntype +| othertype +| ptrtype +| dotname +| '(' ntype ')' + { + $$ = nod(OTPAREN, $2, N); + } + +convtype: + fntype +| othertype + +comptype: + othertype + +fnret_type: + recvchantype +| fntype +| othertype +| ptrtype +| dotname + +dotname: + name +| name '.' sym + { + if($1->op == OPACK) { + Sym *s; + s = restrictlookup($3->name, $1->pkg); + $1->used = 1; + $$ = oldname(s); + break; + } + $$ = nod(OXDOT, $1, newname($3)); + } + +othertype: + '[' oexpr ']' ntype + { + $$ = nod(OTARRAY, $2, $4); + } +| '[' LDDD ']' ntype + { + // array literal of nelem + $$ = nod(OTARRAY, nod(ODDD, N, N), $4); + } +| LCHAN non_recvchantype + { + $$ = nod(OTCHAN, $2, N); + $$->etype = Cboth; + } +| LCHAN LCOMM ntype + { + $$ = nod(OTCHAN, $3, N); + $$->etype = Csend; + } +| LMAP '[' ntype ']' ntype + { + $$ = nod(OTMAP, $3, $5); + } +| structtype +| interfacetype + +ptrtype: + '*' ntype + { + $$ = nod(OIND, $2, N); + } + +recvchantype: + LCOMM LCHAN ntype + { + $$ = nod(OTCHAN, $3, N); + $$->etype = Crecv; + } + +structtype: + LSTRUCT lbrace structdcl_list osemi '}' + { + $$ = nod(OTSTRUCT, N, N); + $$->list = $3; + fixlbrace($2); + } +| LSTRUCT lbrace '}' + { + $$ = nod(OTSTRUCT, N, N); + fixlbrace($2); + } + +interfacetype: + LINTERFACE lbrace interfacedcl_list osemi '}' + { + $$ = nod(OTINTER, N, N); + $$->list = $3; + fixlbrace($2); + } +| LINTERFACE lbrace '}' + { + $$ = nod(OTINTER, N, N); + fixlbrace($2); + } + +/* + * function stuff + * all in one place to show how crappy it all is + */ +xfndcl: + LFUNC fndcl fnbody + { + $$ = $2; + if($$ == N) + break; + $$->nbody = $3; + $$->endlineno = lineno; + funcbody($$); + } + +fndcl: + dcl_name '(' oarg_type_list_ocomma ')' fnres + { + Node *n; + + $3 = checkarglist($3, 1); + $$ = nod(ODCLFUNC, N, N); + $$->nname = $1; + n = nod(OTFUNC, N, N); + n->list = $3; + n->rlist = $5; + if(strcmp($1->sym->name, "init") == 0) { + $$->nname = renameinit($1); + if($3 != nil || $5 != nil) + yyerror("func init must have no arguments and no return values"); + } + if(strcmp(localpkg->name, "main") == 0 && strcmp($1->sym->name, "main") == 0) { + if($3 != nil || $5 != nil) + yyerror("func main must have no arguments and no return values"); + } + // TODO: check if nname already has an ntype + $$->nname->ntype = n; + funchdr($$); + } +| '(' oarg_type_list_ocomma ')' sym '(' oarg_type_list_ocomma ')' fnres + { + Node *rcvr, *t; + Node *name; + + name = newname($4); + $2 = checkarglist($2, 0); + $6 = checkarglist($6, 1); + $$ = N; + if($2 == nil) { + yyerror("method has no receiver"); + break; + } + if($2->next != nil) { + yyerror("method has multiple receivers"); + break; + } + rcvr = $2->n; + if(rcvr->op != ODCLFIELD) { + yyerror("bad receiver in method"); + break; + } + if(rcvr->right->op == OTPAREN || (rcvr->right->op == OIND && rcvr->right->left->op == OTPAREN)) + yyerror("cannot parenthesize receiver type"); + + $$ = nod(ODCLFUNC, N, N); + $$->nname = methodname1(name, rcvr->right); + t = nod(OTFUNC, rcvr, N); + t->list = $6; + t->rlist = $8; + $$->nname->ntype = t; + $$->shortname = name; + funchdr($$); + } + +fntype: + LFUNC '(' oarg_type_list_ocomma ')' fnres + { + $3 = checkarglist($3, 1); + $$ = nod(OTFUNC, N, N); + $$->list = $3; + $$->rlist = $5; + } + +fnbody: + { + $$ = nil; + } +| '{' stmt_list '}' + { + $$ = $2; + if($$ == nil) + $$ = list1(nod(OEMPTY, N, N)); + } + +fnres: + %prec NotParen + { + $$ = nil; + } +| fnret_type + { + $$ = list1(nod(ODCLFIELD, N, $1)); + } +| '(' oarg_type_list_ocomma ')' + { + $2 = checkarglist($2, 0); + $$ = $2; + } + +fnlitdcl: + fntype + { + closurehdr($1); + } + +fnliteral: + fnlitdcl lbrace stmt_list '}' + { + $$ = closurebody($3); + fixlbrace($2); + } +| fnlitdcl error + { + $$ = closurebody(nil); + } + +/* + * lists of things + * note that they are left recursive + * to conserve yacc stack. they need to + * be reversed to interpret correctly + */ +xdcl_list: + { + $$ = nil; + } +| xdcl_list xdcl ';' + { + $$ = concat($1, $2); + if(nsyntaxerrors == 0) + testdclstack(); + } + +vardcl_list: + vardcl +| vardcl_list ';' vardcl + { + $$ = concat($1, $3); + } + +constdcl_list: + constdcl1 +| constdcl_list ';' constdcl1 + { + $$ = concat($1, $3); + } + +typedcl_list: + typedcl + { + $$ = list1($1); + } +| typedcl_list ';' typedcl + { + $$ = list($1, $3); + } + +structdcl_list: + structdcl +| structdcl_list ';' structdcl + { + $$ = concat($1, $3); + } + +interfacedcl_list: + interfacedcl + { + $$ = list1($1); + } +| interfacedcl_list ';' interfacedcl + { + $$ = list($1, $3); + } + +structdcl: + new_name_list ntype oliteral + { + NodeList *l; + + for(l=$1; l; l=l->next) { + l->n = nod(ODCLFIELD, l->n, $2); + l->n->val = $3; + } + } +| embed oliteral + { + $1->val = $2; + $$ = list1($1); + } +| '(' embed ')' oliteral + { + $2->val = $4; + $$ = list1($2); + yyerror("cannot parenthesize embedded type"); + } +| '*' embed oliteral + { + $2->right = nod(OIND, $2->right, N); + $2->val = $3; + $$ = list1($2); + } +| '(' '*' embed ')' oliteral + { + $3->right = nod(OIND, $3->right, N); + $3->val = $5; + $$ = list1($3); + yyerror("cannot parenthesize embedded type"); + } +| '*' '(' embed ')' oliteral + { + $3->right = nod(OIND, $3->right, N); + $3->val = $5; + $$ = list1($3); + yyerror("cannot parenthesize embedded type"); + } + +packname: + LNAME + { + Node *n; + + $$ = $1; + n = oldname($1); + if(n->pack != N) + n->pack->used = 1; + } +| LNAME '.' sym + { + Pkg *pkg; + + if($1->def == N || $1->def->op != OPACK) { + yyerror("%S is not a package", $1); + pkg = localpkg; + } else { + $1->def->used = 1; + pkg = $1->def->pkg; + } + $$ = restrictlookup($3->name, pkg); + } + +embed: + packname + { + $$ = embedded($1); + } + +interfacedcl: + new_name indcl + { + $$ = nod(ODCLFIELD, $1, $2); + ifacedcl($$); + } +| packname + { + $$ = nod(ODCLFIELD, N, oldname($1)); + } +| '(' packname ')' + { + $$ = nod(ODCLFIELD, N, oldname($2)); + yyerror("cannot parenthesize embedded type"); + } + +indcl: + '(' oarg_type_list_ocomma ')' fnres + { + // without func keyword + $2 = checkarglist($2, 1); + $$ = nod(OTFUNC, fakethis(), N); + $$->list = $2; + $$->rlist = $4; + } + +/* + * function arguments. + */ +arg_type: + name_or_type +| sym name_or_type + { + $$ = nod(ONONAME, N, N); + $$->sym = $1; + $$ = nod(OKEY, $$, $2); + } +| sym dotdotdot + { + $$ = nod(ONONAME, N, N); + $$->sym = $1; + $$ = nod(OKEY, $$, $2); + } +| dotdotdot + +arg_type_list: + arg_type + { + $$ = list1($1); + } +| arg_type_list ',' arg_type + { + $$ = list($1, $3); + } + +oarg_type_list_ocomma: + { + $$ = nil; + } +| arg_type_list ocomma + { + $$ = $1; + } + +/* + * statement + */ +stmt: + { + $$ = N; + } +| compound_stmt +| common_dcl + { + $$ = liststmt($1); + } +| non_dcl_stmt +| error + { + $$ = N; + } + +non_dcl_stmt: + simple_stmt +| for_stmt +| switch_stmt +| select_stmt +| if_stmt + { + popdcl(); + $$ = $1; + } +| if_stmt LELSE stmt + { + if($3->op != OIF && $3->op != OBLOCK) + yyerror("missing { } after else"); + + popdcl(); + $$ = $1; + $$->nelse = list1($3); + } +| labelname ':' + { + $1 = nod(OLABEL, $1, N); + $1->sym = dclstack; // context, for goto restrictions + } + stmt + { + NodeList *l; + + $1->right = $4; + l = list1($1); + if($4) + l = list(l, $4); + $$ = liststmt(l); + } +| LFALL + { + // will be converted to OFALL + $$ = nod(OXFALL, N, N); + } +| LBREAK onew_name + { + $$ = nod(OBREAK, $2, N); + } +| LCONTINUE onew_name + { + $$ = nod(OCONTINUE, $2, N); + } +| LGO pseudocall + { + $$ = nod(OPROC, $2, N); + } +| LDEFER pseudocall + { + $$ = nod(ODEFER, $2, N); + } +| LGOTO new_name + { + $$ = nod(OGOTO, $2, N); + $$->sym = dclstack; // context, for goto restrictions + } +| LRETURN oexpr_list + { + $$ = nod(ORETURN, N, N); + $$->list = $2; + } + +stmt_list: + stmt + { + $$ = nil; + if($1 != N) + $$ = list1($1); + } +| stmt_list ';' stmt + { + $$ = $1; + if($3 != N) + $$ = list($$, $3); + } + +new_name_list: + new_name + { + $$ = list1($1); + } +| new_name_list ',' new_name + { + $$ = list($1, $3); + } + +dcl_name_list: + dcl_name + { + $$ = list1($1); + } +| dcl_name_list ',' dcl_name + { + $$ = list($1, $3); + } + +expr_list: + expr + { + $$ = list1($1); + } +| expr_list ',' expr + { + $$ = list($1, $3); + } + +expr_or_type_list: + expr_or_type + { + $$ = list1($1); + } +| expr_or_type_list ',' expr_or_type + { + $$ = list($1, $3); + } + +/* + * list of combo of keyval and val + */ +keyval_list: + keyval + { + $$ = list1($1); + } +| complitexpr + { + $$ = list1($1); + } +| keyval_list ',' keyval + { + $$ = list($1, $3); + } +| keyval_list ',' complitexpr + { + $$ = list($1, $3); + } + +braced_keyval_list: + { + $$ = nil; + } +| keyval_list ocomma + { + $$ = $1; + } + +/* + * optional things + */ +osemi: +| ';' + +ocomma: +| ',' + +oexpr: + { + $$ = N; + } +| expr + +oexpr_list: + { + $$ = nil; + } +| expr_list + +osimple_stmt: + { + $$ = N; + } +| simple_stmt + +ohidden_funarg_list: + { + $$ = nil; + } +| hidden_funarg_list + +ohidden_structdcl_list: + { + $$ = nil; + } +| hidden_structdcl_list + +ohidden_interfacedcl_list: + { + $$ = nil; + } +| hidden_interfacedcl_list + +oliteral: + { + $$.ctype = CTxxx; + } +| LLITERAL + +/* + * import syntax from header of + * an output package + */ +hidden_import: + LIMPORT sym LLITERAL ';' + { + // Informational: record package name + // associated with import path, for use in + // human-readable messages. + Pkg *p; + + p = mkpkg($3.u.sval); + if(p->name == nil) { + p->name = $2->name; + pkglookup($2->name, nil)->npkg++; + } else if(strcmp(p->name, $2->name) != 0) + yyerror("conflicting names %s and %s for package %Z", p->name, $2->name, p->path); + } +| LVAR hidden_pkg_importsym hidden_type ';' + { + importvar($2, $3, PEXTERN); + } +| LCONST hidden_pkg_importsym '=' hidden_constant ';' + { + importconst($2, types[TIDEAL], $4); + } +| LCONST hidden_pkg_importsym hidden_type '=' hidden_constant ';' + { + importconst($2, $3, $5); + } +| LTYPE hidden_pkgtype hidden_type ';' + { + importtype($2, $3); + } +| LFUNC hidden_pkg_importsym '(' ohidden_funarg_list ')' ohidden_funres ';' + { + importvar($2, functype(N, $4, $6), PFUNC); + } +| LFUNC '(' hidden_funarg_list ')' sym '(' ohidden_funarg_list ')' ohidden_funres ';' + { + if($3->next != nil || $3->n->op != ODCLFIELD) { + yyerror("bad receiver in method"); + YYERROR; + } + importmethod($5, functype($3->n, $7, $9)); + } + +hidden_pkgtype: + hidden_pkg_importsym + { + $$ = pkgtype($1); + importsym($1, OTYPE); + } + +hidden_type: + hidden_type_misc +| hidden_type_recv_chan +| hidden_type_func + +hidden_type_non_recv_chan: + hidden_type_misc +| hidden_type_func + +hidden_type_misc: + hidden_importsym + { + $$ = pkgtype($1); + } +| LNAME + { + // predefined name like uint8 + $1 = pkglookup($1->name, builtinpkg); + if($1->def == N || $1->def->op != OTYPE) { + yyerror("%s is not a type", $1->name); + $$ = T; + } else + $$ = $1->def->type; + } +| '[' ']' hidden_type + { + $$ = aindex(N, $3); + } +| '[' LLITERAL ']' hidden_type + { + $$ = aindex(nodlit($2), $4); + } +| LMAP '[' hidden_type ']' hidden_type + { + $$ = maptype($3, $5); + } +| LSTRUCT '{' ohidden_structdcl_list '}' + { + $$ = dostruct($3, TSTRUCT); + } +| LINTERFACE '{' ohidden_interfacedcl_list '}' + { + $$ = dostruct($3, TINTER); + } +| '*' hidden_type + { + $$ = ptrto($2); + } +| LCHAN hidden_type_non_recv_chan + { + $$ = typ(TCHAN); + $$->type = $2; + $$->chan = Cboth; + } +| LCHAN '(' hidden_type_recv_chan ')' + { + $$ = typ(TCHAN); + $$->type = $3; + $$->chan = Cboth; + } +| LCHAN LCOMM hidden_type + { + $$ = typ(TCHAN); + $$->type = $3; + $$->chan = Csend; + } + +hidden_type_recv_chan: + LCOMM LCHAN hidden_type + { + $$ = typ(TCHAN); + $$->type = $3; + $$->chan = Crecv; + } + +hidden_type_func: + LFUNC '(' ohidden_funarg_list ')' ohidden_funres + { + $$ = functype(nil, $3, $5); + } + +hidden_opt_sym: + sym + { + $$ = newname($1); + } +| '?' + { + $$ = N; + } + +hidden_dcl: + hidden_opt_sym hidden_type hidden_tag + { + $$ = nod(ODCLFIELD, $1, typenod($2)); + $$->val = $3; + } +| hidden_opt_sym LDDD hidden_type hidden_tag + { + Type *t; + + t = typ(TARRAY); + t->bound = -1; + t->type = $3; + $$ = nod(ODCLFIELD, $1, typenod(t)); + $$->isddd = 1; + $$->val = $4; + } + +hidden_structdcl: + sym hidden_type hidden_tag + { + $$ = nod(ODCLFIELD, newname($1), typenod($2)); + $$->val = $3; + } +| '?' hidden_type hidden_tag + { + Sym *s; + + s = $2->sym; + if(s == S && isptr[$2->etype]) + s = $2->type->sym; + if(s && s->pkg == builtinpkg) + s = lookup(s->name); + $$ = embedded(s); + $$->right = typenod($2); + $$->val = $3; + } + +hidden_tag: + { + $$.ctype = CTxxx; + } +| ':' LLITERAL // extra colon avoids conflict with "" looking like beginning of "".typename + { + $$ = $2; + } + +hidden_interfacedcl: + sym '(' ohidden_funarg_list ')' ohidden_funres + { + $$ = nod(ODCLFIELD, newname($1), typenod(functype(fakethis(), $3, $5))); + } +| hidden_importsym '(' ohidden_funarg_list ')' ohidden_funres + { + $$ = nod(ODCLFIELD, newname($1), typenod(functype(fakethis(), $3, $5))); + } + +ohidden_funres: + { + $$ = nil; + } +| hidden_funres + +hidden_funres: + '(' ohidden_funarg_list ')' + { + $$ = $2; + } +| hidden_type + { + $$ = list1(nod(ODCLFIELD, N, typenod($1))); + } + +hidden_literal: + LLITERAL + { + $$ = nodlit($1); + } +| '-' LLITERAL + { + $$ = nodlit($2); + switch($$->val.ctype){ + case CTINT: + mpnegfix($$->val.u.xval); + break; + case CTFLT: + mpnegflt($$->val.u.fval); + break; + default: + yyerror("bad negated constant"); + } + } +| sym + { + $$ = oldname(pkglookup($1->name, builtinpkg)); + if($$->op != OLITERAL) + yyerror("bad constant %S", $$->sym); + } + +hidden_constant: + hidden_literal +| '(' hidden_literal '+' hidden_literal ')' + { + $$ = nodcplxlit($2->val, $4->val); + } + +hidden_importsym: + LLITERAL '.' sym + { + Pkg *p; + + if($1.u.sval->len == 0) + p = importpkg; + else + p = mkpkg($1.u.sval); + $$ = pkglookup($3->name, p); + } + +hidden_pkg_importsym: + hidden_importsym + { + $$ = $1; + structpkg = $$->pkg; + } + +hidden_import_list: +| hidden_import_list hidden_import + +hidden_funarg_list: + hidden_dcl + { + $$ = list1($1); + } +| hidden_funarg_list ',' hidden_dcl + { + $$ = list($1, $3); + } + +hidden_structdcl_list: + hidden_structdcl + { + $$ = list1($1); + } +| hidden_structdcl_list ';' hidden_structdcl + { + $$ = list($1, $3); + } + +hidden_interfacedcl_list: + hidden_interfacedcl + { + $$ = list1($1); + } +| hidden_interfacedcl_list ';' hidden_interfacedcl + { + $$ = list($1, $3); + } + +%% + +static void +fixlbrace(int lbr) +{ + // If the opening brace was an LBODY, + // set up for another one now that we're done. + // See comment in lex.c about loophack. + if(lbr == LBODY) + loophack = 1; +} + diff --git a/src/cmd/gc/init.c b/src/cmd/gc/init.c new file mode 100644 index 000000000..8818db08c --- /dev/null +++ b/src/cmd/gc/init.c @@ -0,0 +1,195 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +#include "go.h" + +/* + * a function named init is a special case. + * it is called by the initialization before + * main is run. to make it unique within a + * package and also uncallable, the name, + * normally "pkg.init", is altered to "pkg.init·1". + */ +Node* +renameinit(Node *n) +{ + Sym *s; + static int initgen; + + s = n->sym; + if(s == S) + return n; + if(strcmp(s->name, "init") != 0) + return n; + + snprint(namebuf, sizeof(namebuf), "init·%d", ++initgen); + s = lookup(namebuf); + return newname(s); +} + +/* + * hand-craft the following initialization code + * var initdone· uint8 (1) + * func init() (2) + * if initdone· != 0 { (3) + * if initdone· == 2 (4) + * return + * throw(); (5) + * } + * initdone· = 1; (6) + * // over all matching imported symbols + * .init() (7) + * { } (8) + * init·() // if any (9) + * initdone· = 2; (10) + * return (11) + * } + */ +static int +anyinit(NodeList *n) +{ + uint32 h; + Sym *s; + NodeList *l; + + // are there any interesting init statements + for(l=n; l; l=l->next) { + switch(l->n->op) { + case ODCLFUNC: + case ODCLCONST: + case ODCLTYPE: + case OEMPTY: + break; + default: + return 1; + } + } + + // is this main + if(strcmp(localpkg->name, "main") == 0) + return 1; + + // is there an explicit init function + snprint(namebuf, sizeof(namebuf), "init·1"); + s = lookup(namebuf); + if(s->def != N) + return 1; + + // are there any imported init functions + for(h=0; hlink) { + if(s->name[0] != 'i' || strcmp(s->name, "init") != 0) + continue; + if(s->def == N) + continue; + return 1; + } + + // then none + return 0; +} + +void +fninit(NodeList *n) +{ + int i; + Node *gatevar; + Node *a, *b, *fn; + NodeList *r; + uint32 h; + Sym *s, *initsym; + + if(debug['A']) { + // sys.go or unsafe.go during compiler build + return; + } + + n = initfix(n); + if(!anyinit(n)) + return; + + r = nil; + + // (1) + snprint(namebuf, sizeof(namebuf), "initdone·"); + gatevar = newname(lookup(namebuf)); + addvar(gatevar, types[TUINT8], PEXTERN); + + // (2) + maxarg = 0; + snprint(namebuf, sizeof(namebuf), "init"); + + fn = nod(ODCLFUNC, N, N); + initsym = lookup(namebuf); + fn->nname = newname(initsym); + fn->nname->ntype = nod(OTFUNC, N, N); + funchdr(fn); + + // (3) + a = nod(OIF, N, N); + a->ntest = nod(ONE, gatevar, nodintconst(0)); + r = list(r, a); + + // (4) + b = nod(OIF, N, N); + b->ntest = nod(OEQ, gatevar, nodintconst(2)); + b->nbody = list1(nod(ORETURN, N, N)); + a->nbody = list1(b); + + // (5) + b = syslook("throwinit", 0); + b = nod(OCALL, b, N); + a->nbody = list(a->nbody, b); + + // (6) + a = nod(OAS, gatevar, nodintconst(1)); + r = list(r, a); + + // (7) + for(h=0; hlink) { + if(s->name[0] != 'i' || strcmp(s->name, "init") != 0) + continue; + if(s->def == N) + continue; + if(s == initsym) + continue; + + // could check that it is fn of no args/returns + a = nod(OCALL, s->def, N); + r = list(r, a); + } + + // (8) + r = concat(r, n); + + // (9) + // could check that it is fn of no args/returns + for(i=1;; i++) { + snprint(namebuf, sizeof(namebuf), "init·%d", i); + s = lookup(namebuf); + if(s->def == N) + break; + a = nod(OCALL, s->def, N); + r = list(r, a); + } + + // (10) + a = nod(OAS, gatevar, nodintconst(2)); + r = list(r, a); + + // (11) + a = nod(ORETURN, N, N); + r = list(r, a); + exportsym(fn->nname); + + fn->nbody = r; + funcbody(fn); + + curfn = fn; + typecheck(&fn, Etop); + typechecklist(r, Etop); + curfn = nil; + funccompile(fn, 0); +} diff --git a/src/cmd/gc/lex.c b/src/cmd/gc/lex.c new file mode 100644 index 000000000..29b6d27ff --- /dev/null +++ b/src/cmd/gc/lex.c @@ -0,0 +1,1956 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +#define EXTERN +#include "go.h" +#include "y.tab.h" +#include + +#undef getc +#undef ungetc +#define getc ccgetc +#define ungetc ccungetc + +extern int yychar; +int windows; +int yyprev; +int yylast; + +static void lexinit(void); +static void lexfini(void); +static void yytinit(void); +static int getc(void); +static void ungetc(int); +static int32 getr(void); +static int escchar(int, int*, vlong*); +static void addidir(char*); +static int getlinepragma(void); +static char *goos, *goarch, *goroot; + +// Our own isdigit, isspace, isalpha, isalnum that take care +// of EOF and other out of range arguments. +static int +yy_isdigit(int c) +{ + return c >= 0 && c <= 0xFF && isdigit(c); +} + +static int +yy_isspace(int c) +{ + return c >= 0 && c <= 0xFF && isspace(c); +} + +static int +yy_isalpha(int c) +{ + return c >= 0 && c <= 0xFF && isalpha(c); +} + +static int +yy_isalnum(int c) +{ + return c >= 0 && c <= 0xFF && isalnum(c); +} + +// Disallow use of isdigit etc. +#undef isdigit +#undef isspace +#undef isalpha +#undef isalnum +#define isdigit use_yy_isdigit_instead_of_isdigit +#define isspace use_yy_isspace_instead_of_isspace +#define isalpha use_yy_isalpha_instead_of_isalpha +#define isalnum use_yy_isalnum_instead_of_isalnum + +#define DBG if(!debug['x']);else print +enum +{ + EOF = -1, +}; + +void +usage(void) +{ + print("gc: usage: %cg [flags] file.go...\n", thechar); + print("flags:\n"); + // -A is allow use of "any" type, for bootstrapping + print(" -I DIR search for packages in DIR\n"); + print(" -d print declarations\n"); + print(" -e no limit on number of errors printed\n"); + print(" -f print stack frame structure\n"); + print(" -h panic on an error\n"); + print(" -o file specify output file\n"); + print(" -S print the assembly language\n"); + print(" -V print the compiler version\n"); + print(" -u disable package unsafe\n"); + print(" -w print the parse tree after typing\n"); + print(" -x print lex tokens\n"); + exit(0); +} + +void +fault(int s) +{ + // If we've already complained about things + // in the program, don't bother complaining + // about the seg fault too; let the user clean up + // the code and try again. + if(nsavederrors + nerrors > 0) + errorexit(); + fatal("fault"); +} + +int +main(int argc, char *argv[]) +{ + int i, c; + NodeList *l; + char *p; + + signal(SIGBUS, fault); + signal(SIGSEGV, fault); + + localpkg = mkpkg(strlit("")); + localpkg->prefix = "\"\""; + + builtinpkg = mkpkg(strlit("go.builtin")); + + gostringpkg = mkpkg(strlit("go.string")); + gostringpkg->name = "go.string"; + gostringpkg->prefix = "go.string"; // not go%2estring + + runtimepkg = mkpkg(strlit("runtime")); + runtimepkg->name = "runtime"; + + typepkg = mkpkg(strlit("type")); + typepkg->name = "type"; + + unsafepkg = mkpkg(strlit("unsafe")); + unsafepkg->name = "unsafe"; + + goroot = getgoroot(); + goos = getgoos(); + goarch = thestring; + + outfile = nil; + ARGBEGIN { + default: + c = ARGC(); + if(c >= 0 && c < sizeof(debug)) + debug[c]++; + break; + + case 'o': + outfile = EARGF(usage()); + break; + + case 'I': + addidir(EARGF(usage())); + break; + + case 'u': + safemode = 1; + break; + + case 'V': + print("%cg version %s\n", thechar, getgoversion()); + exit(0); + } ARGEND + + if(argc < 1) + usage(); + + // special flag to detect compilation of package runtime + compiling_runtime = debug['+']; + + pathname = mal(1000); + if(getwd(pathname, 999) == 0) + strcpy(pathname, "/???"); + + if(yy_isalpha(pathname[0]) && pathname[1] == ':') { + // On Windows. + windows = 1; + + // Canonicalize path by converting \ to / (Windows accepts both). + for(p=pathname; *p; p++) + if(*p == '\\') + *p = '/'; + } + + fmtinstall('O', Oconv); // node opcodes + fmtinstall('E', Econv); // etype opcodes + fmtinstall('J', Jconv); // all the node flags + fmtinstall('S', Sconv); // sym pointer + fmtinstall('T', Tconv); // type pointer + fmtinstall('N', Nconv); // node pointer + fmtinstall('Z', Zconv); // escaped string + fmtinstall('L', Lconv); // line number + fmtinstall('B', Bconv); // big numbers + fmtinstall('F', Fconv); // big float numbers + + betypeinit(); + if(widthptr == 0) + fatal("betypeinit failed"); + + lexinit(); + typeinit(); + yytinit(); + + blockgen = 1; + dclcontext = PEXTERN; + nerrors = 0; + lexlineno = 1; + + for(i=0; iname); // final import not used checks + lexfini(); + + typecheckok = 1; + if(debug['f']) + frame(1); + + // Process top-level declarations in four phases. + // Phase 1: const, type, and names and types of funcs. + // This will gather all the information about types + // and methods but doesn't depend on any of it. + // Phase 2: Variable assignments. + // To check interface assignments, depends on phase 1. + // Phase 3: Type check function bodies. + // Phase 4: Compile function bodies. + defercheckwidth(); + for(l=xtop; l; l=l->next) + if(l->n->op != ODCL && l->n->op != OAS) + typecheck(&l->n, Etop); + for(l=xtop; l; l=l->next) + if(l->n->op == ODCL || l->n->op == OAS) + typecheck(&l->n, Etop); + resumetypecopy(); + resumecheckwidth(); + + for(l=xtop; l; l=l->next) { + if(l->n->op == ODCLFUNC || l->n->op == OCLOSURE) { + curfn = l->n; + saveerrors(); + typechecklist(l->n->nbody, Etop); + if(nerrors != 0) + l->n->nbody = nil; // type errors; do not compile + } + } + + curfn = nil; + + if(nsavederrors+nerrors) + errorexit(); + + for(l=xtop; l; l=l->next) + if(l->n->op == ODCLFUNC) + funccompile(l->n, 0); + + if(nsavederrors+nerrors == 0) + fninit(xtop); + + while(closures) { + l = closures; + closures = nil; + for(; l; l=l->next) { + funccompile(l->n, 1); + } + } + + for(l=externdcl; l; l=l->next) + if(l->n->op == ONAME) + typecheck(&l->n, Erv); + + if(nerrors+nsavederrors) + errorexit(); + + dumpobj(); + + if(nerrors+nsavederrors) + errorexit(); + + flusherrors(); + exit(0); + return 0; +} + +void +saveerrors(void) +{ + nsavederrors += nerrors; + nerrors = 0; +} + +static int +arsize(Biobuf *b, char *name) +{ + struct ar_hdr *a; + + if((a = Brdline(b, '\n')) == nil) + return -1; + if(Blinelen(b) != sizeof(struct ar_hdr)) + return -1; + if(strncmp(a->name, name, strlen(name)) != 0) + return -1; + return atoi(a->size); +} + +static int +skiptopkgdef(Biobuf *b) +{ + char *p; + int sz; + + /* archive header */ + if((p = Brdline(b, '\n')) == nil) + return 0; + if(Blinelen(b) != 8) + return 0; + if(memcmp(p, "!\n", 8) != 0) + return 0; + /* symbol table is first; skip it */ + sz = arsize(b, "__.SYMDEF"); + if(sz < 0) + return 0; + Bseek(b, sz, 1); + /* package export block is second */ + sz = arsize(b, "__.PKGDEF"); + if(sz <= 0) + return 0; + return 1; +} + +static void +addidir(char* dir) +{ + Idir** pp; + + if(dir == nil) + return; + + for(pp = &idirs; *pp != nil; pp = &(*pp)->link) + ; + *pp = mal(sizeof(Idir)); + (*pp)->link = nil; + (*pp)->dir = dir; +} + +// is this path a local name? begins with ./ or ../ or / +static int +islocalname(Strlit *name) +{ + if(!windows && name->len >= 1 && name->s[0] == '/') + return 1; + if(windows && name->len >= 3 && + yy_isalpha(name->s[0]) && name->s[1] == ':' && name->s[2] == '/') + return 1; + if(name->len >= 2 && strncmp(name->s, "./", 2) == 0) + return 1; + if(name->len >= 3 && strncmp(name->s, "../", 3) == 0) + return 1; + return 0; +} + +static int +findpkg(Strlit *name) +{ + Idir *p; + char *q; + + if(islocalname(name)) { + if(safemode) + return 0; + // try .a before .6. important for building libraries: + // if there is an array.6 in the array.a library, + // want to find all of array.a, not just array.6. + snprint(namebuf, sizeof(namebuf), "%Z.a", name); + if(access(namebuf, 0) >= 0) + return 1; + snprint(namebuf, sizeof(namebuf), "%Z.%c", name, thechar); + if(access(namebuf, 0) >= 0) + return 1; + return 0; + } + + // local imports should be canonicalized already. + // don't want to see "container/../container/vector" + // as different from "container/vector". + q = mal(name->len+1); + memmove(q, name->s, name->len); + q[name->len] = '\0'; + cleanname(q); + if(strlen(q) != name->len || memcmp(q, name->s, name->len) != 0) { + yyerror("non-canonical import path %Z (should be %s)", name, q); + return 0; + } + + for(p = idirs; p != nil; p = p->link) { + snprint(namebuf, sizeof(namebuf), "%s/%Z.a", p->dir, name); + if(access(namebuf, 0) >= 0) + return 1; + snprint(namebuf, sizeof(namebuf), "%s/%Z.%c", p->dir, name, thechar); + if(access(namebuf, 0) >= 0) + return 1; + } + if(goroot != nil) { + snprint(namebuf, sizeof(namebuf), "%s/pkg/%s_%s/%Z.a", goroot, goos, goarch, name); + if(access(namebuf, 0) >= 0) + return 1; + snprint(namebuf, sizeof(namebuf), "%s/pkg/%s_%s/%Z.%c", goroot, goos, goarch, name, thechar); + if(access(namebuf, 0) >= 0) + return 1; + } + return 0; +} + +void +importfile(Val *f, int line) +{ + Biobuf *imp; + char *file, *p, *q; + int32 c; + int len; + Strlit *path; + char *cleanbuf; + + // TODO(rsc): don't bother reloading imports more than once? + + if(f->ctype != CTSTR) { + yyerror("import statement not a string"); + return; + } + + if(strlen(f->u.sval->s) != f->u.sval->len) { + yyerror("import path contains NUL"); + errorexit(); + } + + // The package name main is no longer reserved, + // but we reserve the import path "main" to identify + // the main package, just as we reserve the import + // path "math" to identify the standard math package. + if(strcmp(f->u.sval->s, "main") == 0) { + yyerror("cannot import \"main\""); + errorexit(); + } + + if(strcmp(f->u.sval->s, "unsafe") == 0) { + if(safemode) { + yyerror("cannot import package unsafe"); + errorexit(); + } + importpkg = mkpkg(f->u.sval); + cannedimports("unsafe.6", unsafeimport); + return; + } + + path = f->u.sval; + if(islocalname(path)) { + cleanbuf = mal(strlen(pathname) + strlen(path->s) + 2); + strcpy(cleanbuf, pathname); + strcat(cleanbuf, "/"); + strcat(cleanbuf, path->s); + cleanname(cleanbuf); + path = strlit(cleanbuf); + } + + if(!findpkg(path)) { + yyerror("can't find import: %Z", f->u.sval); + errorexit(); + } + importpkg = mkpkg(path); + + imp = Bopen(namebuf, OREAD); + if(imp == nil) { + yyerror("can't open import: %Z: %r", f->u.sval); + errorexit(); + } + file = strdup(namebuf); + + len = strlen(namebuf); + if(len > 2 && namebuf[len-2] == '.' && namebuf[len-1] == 'a') { + if(!skiptopkgdef(imp)) { + yyerror("import %s: not a package file", file); + errorexit(); + } + } + + // check object header + p = Brdstr(imp, '\n', 1); + if(strcmp(p, "empty archive") != 0) { + if(strncmp(p, "go object ", 10) != 0) { + yyerror("import %s: not a go object file", file); + errorexit(); + } + q = smprint("%s %s %s", getgoos(), thestring, getgoversion()); + if(strcmp(p+10, q) != 0) { + yyerror("import %s: object is [%s] expected [%s]", file, p+10, q); + errorexit(); + } + free(q); + } + + // assume files move (get installed) + // so don't record the full path. + linehist(file + len - path->len - 2, -1, 1); // acts as #pragma lib + + /* + * position the input right + * after $$ and return + */ + pushedio = curio; + curio.bin = imp; + curio.peekc = 0; + curio.peekc1 = 0; + curio.infile = file; + curio.nlsemi = 0; + typecheckok = 1; + + for(;;) { + c = getc(); + if(c == EOF) + break; + if(c != '$') + continue; + c = getc(); + if(c == EOF) + break; + if(c != '$') + continue; + return; + } + yyerror("no import in: %Z", f->u.sval); + unimportfile(); +} + +void +unimportfile(void) +{ + if(curio.bin != nil) { + Bterm(curio.bin); + curio.bin = nil; + } else + lexlineno--; // re correct sys.6 line number + + curio = pushedio; + pushedio.bin = nil; + incannedimport = 0; + typecheckok = 0; +} + +void +cannedimports(char *file, char *cp) +{ + lexlineno++; // if sys.6 is included on line 1, + + pushedio = curio; + curio.bin = nil; + curio.peekc = 0; + curio.peekc1 = 0; + curio.infile = file; + curio.cp = cp; + curio.nlsemi = 0; + curio.importsafe = 0; + + typecheckok = 1; + incannedimport = 1; +} + +static int +isfrog(int c) +{ + // complain about possibly invisible control characters + if(c < 0) + return 1; + if(c < ' ') { + if(c == '\n' || c== '\r' || c == '\t') // good white space + return 0; + return 1; + } + if(0x7f <= c && c <= 0xa0) // DEL, unicode block including unbreakable space. + return 1; + return 0; +} + +typedef struct Loophack Loophack; +struct Loophack { + int v; + Loophack *next; +}; + +static int32 +_yylex(void) +{ + int c, c1, clen, escflag, ncp; + vlong v; + char *cp, *ep; + Rune rune; + Sym *s; + static Loophack *lstk; + Loophack *h; + + prevlineno = lineno; + +l0: + c = getc(); + if(yy_isspace(c)) { + if(c == '\n' && curio.nlsemi) { + ungetc(c); + DBG("lex: implicit semi\n"); + return ';'; + } + goto l0; + } + + lineno = lexlineno; /* start of token */ + + if(c >= Runeself) { + /* all multibyte runes are alpha */ + cp = lexbuf; + ep = lexbuf+sizeof lexbuf; + goto talph; + } + + if(yy_isalpha(c)) { + cp = lexbuf; + ep = lexbuf+sizeof lexbuf; + goto talph; + } + + if(yy_isdigit(c)) + goto tnum; + + switch(c) { + case EOF: + lineno = prevlineno; + ungetc(EOF); + return -1; + + case '_': + cp = lexbuf; + ep = lexbuf+sizeof lexbuf; + goto talph; + + case '.': + c1 = getc(); + if(yy_isdigit(c1)) { + cp = lexbuf; + ep = lexbuf+sizeof lexbuf; + *cp++ = c; + c = c1; + c1 = 0; + goto casedot; + } + if(c1 == '.') { + c1 = getc(); + if(c1 == '.') { + c = LDDD; + goto lx; + } + ungetc(c1); + c1 = '.'; + } + break; + + case '"': + /* "..." */ + strcpy(lexbuf, "\"\""); + cp = mal(8); + clen = sizeof(int32); + ncp = 8; + + for(;;) { + if(clen+UTFmax > ncp) { + cp = remal(cp, ncp, ncp); + ncp += ncp; + } + if(escchar('"', &escflag, &v)) + break; + if(v < Runeself || escflag) { + cp[clen++] = v; + } else { + rune = v; + c = runelen(rune); + runetochar(cp+clen, &rune); + clen += c; + } + } + goto strlit; + + case '`': + /* `...` */ + strcpy(lexbuf, "``"); + cp = mal(8); + clen = sizeof(int32); + ncp = 8; + + for(;;) { + if(clen+UTFmax > ncp) { + cp = remal(cp, ncp, ncp); + ncp += ncp; + } + c = getr(); + if(c == EOF) { + yyerror("eof in string"); + break; + } + if(c == '`') + break; + rune = c; + clen += runetochar(cp+clen, &rune); + } + + strlit: + *(int32*)cp = clen-sizeof(int32); // length + do { + cp[clen++] = 0; + } while(clen & MAXALIGN); + yylval.val.u.sval = (Strlit*)cp; + yylval.val.ctype = CTSTR; + DBG("lex: string literal\n"); + strcpy(litbuf, "string literal"); + return LLITERAL; + + case '\'': + /* '.' */ + if(escchar('\'', &escflag, &v)) { + yyerror("empty character literal or unescaped ' in character literal"); + v = '\''; + } + if(!escchar('\'', &escflag, &v)) { + yyerror("missing '"); + ungetc(v); + } + yylval.val.u.xval = mal(sizeof(*yylval.val.u.xval)); + mpmovecfix(yylval.val.u.xval, v); + yylval.val.ctype = CTINT; + DBG("lex: codepoint literal\n"); + strcpy(litbuf, "string literal"); + return LLITERAL; + + case '/': + c1 = getc(); + if(c1 == '*') { + int nl; + + nl = 0; + for(;;) { + c = getr(); + if(c == '\n') + nl = 1; + while(c == '*') { + c = getr(); + if(c == '/') { + if(nl) + ungetc('\n'); + goto l0; + } + if(c == '\n') + nl = 1; + } + if(c == EOF) { + yyerror("eof in comment"); + errorexit(); + } + } + } + if(c1 == '/') { + c = getlinepragma(); + for(;;) { + if(c == '\n' || c == EOF) { + ungetc(c); + goto l0; + } + c = getr(); + } + } + if(c1 == '=') { + c = ODIV; + goto asop; + } + break; + + case ':': + c1 = getc(); + if(c1 == '=') { + c = LCOLAS; + goto lx; + } + break; + + case '*': + c1 = getc(); + if(c1 == '=') { + c = OMUL; + goto asop; + } + break; + + case '%': + c1 = getc(); + if(c1 == '=') { + c = OMOD; + goto asop; + } + break; + + case '+': + c1 = getc(); + if(c1 == '+') { + c = LINC; + goto lx; + } + if(c1 == '=') { + c = OADD; + goto asop; + } + break; + + case '-': + c1 = getc(); + if(c1 == '-') { + c = LDEC; + goto lx; + } + if(c1 == '=') { + c = OSUB; + goto asop; + } + break; + + case '>': + c1 = getc(); + if(c1 == '>') { + c = LRSH; + c1 = getc(); + if(c1 == '=') { + c = ORSH; + goto asop; + } + break; + } + if(c1 == '=') { + c = LGE; + goto lx; + } + c = LGT; + break; + + case '<': + c1 = getc(); + if(c1 == '<') { + c = LLSH; + c1 = getc(); + if(c1 == '=') { + c = OLSH; + goto asop; + } + break; + } + if(c1 == '=') { + c = LLE; + goto lx; + } + if(c1 == '-') { + c = LCOMM; + goto lx; + } + c = LLT; + break; + + case '=': + c1 = getc(); + if(c1 == '=') { + c = LEQ; + goto lx; + } + break; + + case '!': + c1 = getc(); + if(c1 == '=') { + c = LNE; + goto lx; + } + break; + + case '&': + c1 = getc(); + if(c1 == '&') { + c = LANDAND; + goto lx; + } + if(c1 == '^') { + c = LANDNOT; + c1 = getc(); + if(c1 == '=') { + c = OANDNOT; + goto asop; + } + break; + } + if(c1 == '=') { + c = OAND; + goto asop; + } + break; + + case '|': + c1 = getc(); + if(c1 == '|') { + c = LOROR; + goto lx; + } + if(c1 == '=') { + c = OOR; + goto asop; + } + break; + + case '^': + c1 = getc(); + if(c1 == '=') { + c = OXOR; + goto asop; + } + break; + + /* + * clumsy dance: + * to implement rule that disallows + * if T{1}[0] { ... } + * but allows + * if (T{1}[0]) { ... } + * the block bodies for if/for/switch/select + * begin with an LBODY token, not '{'. + * + * when we see the keyword, the next + * non-parenthesized '{' becomes an LBODY. + * loophack is normally 0. + * a keyword makes it go up to 1. + * parens push loophack onto a stack and go back to 0. + * a '{' with loophack == 1 becomes LBODY and disables loophack. + * + * i said it was clumsy. + */ + case '(': + case '[': + if(loophack || lstk != nil) { + h = malloc(sizeof *h); + h->v = loophack; + h->next = lstk; + lstk = h; + loophack = 0; + } + goto lx; + case ')': + case ']': + if(lstk != nil) { + h = lstk; + loophack = h->v; + lstk = h->next; + free(h); + } + goto lx; + case '{': + if(loophack == 1) { + DBG("%L lex: LBODY\n", lexlineno); + loophack = 0; + return LBODY; + } + goto lx; + + default: + goto lx; + } + ungetc(c1); + +lx: + if(c > 0xff) + DBG("%L lex: TOKEN %s\n", lexlineno, lexname(c)); + else + DBG("%L lex: TOKEN '%c'\n", lexlineno, c); + if(isfrog(c)) { + yyerror("illegal character 0x%ux", c); + goto l0; + } + if(importpkg == nil && (c == '#' || c == '$' || c == '?' || c == '@' || c == '\\')) { + yyerror("%s: unexpected %c", "syntax error", c); + goto l0; + } + return c; + +asop: + yylval.lint = c; // rathole to hold which asop + DBG("lex: TOKEN ASOP %c\n", c); + return LASOP; + +talph: + /* + * cp is set to lexbuf and some + * prefix has been stored + */ + for(;;) { + if(cp+10 >= ep) { + yyerror("identifier too long"); + errorexit(); + } + if(c >= Runeself) { + ungetc(c); + rune = getr(); + // 0xb7 · is used for internal names + if(!isalpharune(rune) && !isdigitrune(rune) && (importpkg == nil || rune != 0xb7)) + yyerror("invalid identifier character 0x%ux", rune); + cp += runetochar(cp, &rune); + } else if(!yy_isalnum(c) && c != '_') + break; + else + *cp++ = c; + c = getc(); + } + *cp = 0; + ungetc(c); + + s = lookup(lexbuf); + switch(s->lexical) { + case LIGNORE: + goto l0; + + case LFOR: + case LIF: + case LSWITCH: + case LSELECT: + loophack = 1; // see comment about loophack above + break; + } + + DBG("lex: %S %s\n", s, lexname(s->lexical)); + yylval.sym = s; + return s->lexical; + +tnum: + c1 = 0; + cp = lexbuf; + ep = lexbuf+sizeof lexbuf; + if(c != '0') { + for(;;) { + if(cp+10 >= ep) { + yyerror("identifier too long"); + errorexit(); + } + *cp++ = c; + c = getc(); + if(yy_isdigit(c)) + continue; + goto dc; + } + } + *cp++ = c; + c = getc(); + if(c == 'x' || c == 'X') { + for(;;) { + if(cp+10 >= ep) { + yyerror("identifier too long"); + errorexit(); + } + *cp++ = c; + c = getc(); + if(yy_isdigit(c)) + continue; + if(c >= 'a' && c <= 'f') + continue; + if(c >= 'A' && c <= 'F') + continue; + if(cp == lexbuf+2) + yyerror("malformed hex constant"); + goto ncu; + } + } + + if(c == 'p') // 0p begins floating point zero + goto casep; + + c1 = 0; + for(;;) { + if(cp+10 >= ep) { + yyerror("identifier too long"); + errorexit(); + } + if(!yy_isdigit(c)) + break; + if(c < '0' || c > '7') + c1 = 1; // not octal + *cp++ = c; + c = getc(); + } + if(c == '.') + goto casedot; + if(c == 'e' || c == 'E') + goto casee; + if(c == 'i') + goto casei; + if(c1) + yyerror("malformed octal constant"); + goto ncu; + +dc: + if(c == '.') + goto casedot; + if(c == 'e' || c == 'E') + goto casee; + if(c == 'p' || c == 'P') + goto casep; + if(c == 'i') + goto casei; + +ncu: + *cp = 0; + ungetc(c); + + yylval.val.u.xval = mal(sizeof(*yylval.val.u.xval)); + mpatofix(yylval.val.u.xval, lexbuf); + if(yylval.val.u.xval->ovf) { + yyerror("overflow in constant"); + mpmovecfix(yylval.val.u.xval, 0); + } + yylval.val.ctype = CTINT; + DBG("lex: integer literal\n"); + strcpy(litbuf, "literal "); + strcat(litbuf, lexbuf); + return LLITERAL; + +casedot: + for(;;) { + if(cp+10 >= ep) { + yyerror("identifier too long"); + errorexit(); + } + *cp++ = c; + c = getc(); + if(!yy_isdigit(c)) + break; + } + if(c == 'i') + goto casei; + if(c != 'e' && c != 'E') + goto caseout; + +casee: + *cp++ = 'e'; + c = getc(); + if(c == '+' || c == '-') { + *cp++ = c; + c = getc(); + } + if(!yy_isdigit(c)) + yyerror("malformed fp constant exponent"); + while(yy_isdigit(c)) { + if(cp+10 >= ep) { + yyerror("identifier too long"); + errorexit(); + } + *cp++ = c; + c = getc(); + } + if(c == 'i') + goto casei; + goto caseout; + +casep: + *cp++ = 'p'; + c = getc(); + if(c == '+' || c == '-') { + *cp++ = c; + c = getc(); + } + if(!yy_isdigit(c)) + yyerror("malformed fp constant exponent"); + while(yy_isdigit(c)) { + if(cp+10 >= ep) { + yyerror("identifier too long"); + errorexit(); + } + *cp++ = c; + c = getc(); + } + if(c == 'i') + goto casei; + goto caseout; + +casei: + // imaginary constant + *cp = 0; + yylval.val.u.cval = mal(sizeof(*yylval.val.u.cval)); + mpmovecflt(&yylval.val.u.cval->real, 0.0); + mpatoflt(&yylval.val.u.cval->imag, lexbuf); + if(yylval.val.u.cval->imag.val.ovf) { + yyerror("overflow in imaginary constant"); + mpmovecflt(&yylval.val.u.cval->real, 0.0); + } + yylval.val.ctype = CTCPLX; + DBG("lex: imaginary literal\n"); + strcpy(litbuf, "literal "); + strcat(litbuf, lexbuf); + return LLITERAL; + +caseout: + *cp = 0; + ungetc(c); + + yylval.val.u.fval = mal(sizeof(*yylval.val.u.fval)); + mpatoflt(yylval.val.u.fval, lexbuf); + if(yylval.val.u.fval->val.ovf) { + yyerror("overflow in float constant"); + mpmovecflt(yylval.val.u.fval, 0.0); + } + yylval.val.ctype = CTFLT; + DBG("lex: floating literal\n"); + strcpy(litbuf, "literal "); + strcat(litbuf, lexbuf); + return LLITERAL; +} + +/* + * read and interpret syntax that looks like + * //line parse.y:15 + * as a discontinuity in sequential line numbers. + * the next line of input comes from parse.y:15 + */ +static int +getlinepragma(void) +{ + int i, c, n; + char *cp, *ep; + Hist *h; + + for(i=0; i<5; i++) { + c = getr(); + if(c != "line "[i]) + goto out; + } + + cp = lexbuf; + ep = lexbuf+sizeof(lexbuf)-5; + for(;;) { + c = getr(); + if(c == '\n' || c == EOF) + goto out; + if(c == ' ') + continue; + if(c == ':') + break; + if(cp < ep) + *cp++ = c; + } + *cp = 0; + + n = 0; + for(;;) { + c = getr(); + if(!yy_isdigit(c)) + break; + n = n*10 + (c-'0'); + if(n > 1e8) { + yyerror("line number out of range"); + errorexit(); + } + } + + if(c != '\n' || n <= 0) + goto out; + + // try to avoid allocating file name over and over + for(h=hist; h!=H; h=h->link) { + if(h->name != nil && strcmp(h->name, lexbuf) == 0) { + linehist(h->name, n, 0); + goto out; + } + } + linehist(strdup(lexbuf), n, 0); + +out: + return c; +} + +int32 +yylex(void) +{ + int lx; + + lx = _yylex(); + + if(curio.nlsemi && lx == EOF) { + // Treat EOF as "end of line" for the purposes + // of inserting a semicolon. + lx = ';'; + } + + switch(lx) { + case LNAME: + case LLITERAL: + case LBREAK: + case LCONTINUE: + case LFALL: + case LRETURN: + case LINC: + case LDEC: + case ')': + case '}': + case ']': + curio.nlsemi = 1; + break; + default: + curio.nlsemi = 0; + break; + } + + // Track last two tokens returned by yylex. + yyprev = yylast; + yylast = lx; + return lx; +} + +static int +getc(void) +{ + int c; + + c = curio.peekc; + if(c != 0) { + curio.peekc = curio.peekc1; + curio.peekc1 = 0; + if(c == '\n' && pushedio.bin == nil) + lexlineno++; + return c; + } + + if(curio.bin == nil) { + c = *curio.cp & 0xff; + if(c != 0) + curio.cp++; + } else + c = Bgetc(curio.bin); + + switch(c) { + case 0: + if(curio.bin != nil) { + yyerror("illegal NUL byte"); + break; + } + case EOF: + // insert \n at EOF + if(curio.eofnl) + return EOF; + curio.eofnl = 1; + c = '\n'; + case '\n': + if(pushedio.bin == nil) + lexlineno++; + break; + } + return c; +} + +static void +ungetc(int c) +{ + curio.peekc1 = curio.peekc; + curio.peekc = c; + if(c == '\n' && pushedio.bin == nil) + lexlineno--; +} + +static int32 +getr(void) +{ + int c, i; + char str[UTFmax+1]; + Rune rune; + + c = getc(); + if(c < Runeself) + return c; + i = 0; + str[i++] = c; + +loop: + c = getc(); + str[i++] = c; + if(!fullrune(str, i)) + goto loop; + c = chartorune(&rune, str); + if(rune == Runeerror && c == 1) { + lineno = lexlineno; + yyerror("illegal UTF-8 sequence"); + flusherrors(); + print("\t"); + for(c=0; c 0 ? " " : "", *(uchar*)(str+c)); + print("\n"); + } + return rune; +} + +static int +escchar(int e, int *escflg, vlong *val) +{ + int i, u, c; + vlong l; + + *escflg = 0; + + c = getr(); + switch(c) { + case EOF: + yyerror("eof in string"); + return 1; + case '\n': + yyerror("newline in string"); + return 1; + case '\\': + break; + default: + if(c == e) + return 1; + *val = c; + return 0; + } + + u = 0; + c = getr(); + switch(c) { + case 'x': + *escflg = 1; // it's a byte + i = 2; + goto hex; + + case 'u': + i = 4; + u = 1; + goto hex; + + case 'U': + i = 8; + u = 1; + goto hex; + + case '0': + case '1': + case '2': + case '3': + case '4': + case '5': + case '6': + case '7': + *escflg = 1; // it's a byte + goto oct; + + case 'a': c = '\a'; break; + case 'b': c = '\b'; break; + case 'f': c = '\f'; break; + case 'n': c = '\n'; break; + case 'r': c = '\r'; break; + case 't': c = '\t'; break; + case 'v': c = '\v'; break; + case '\\': c = '\\'; break; + + default: + if(c != e) + yyerror("unknown escape sequence: %c", c); + } + *val = c; + return 0; + +hex: + l = 0; + for(; i>0; i--) { + c = getc(); + if(c >= '0' && c <= '9') { + l = l*16 + c-'0'; + continue; + } + if(c >= 'a' && c <= 'f') { + l = l*16 + c-'a' + 10; + continue; + } + if(c >= 'A' && c <= 'F') { + l = l*16 + c-'A' + 10; + continue; + } + yyerror("non-hex character in escape sequence: %c", c); + ungetc(c); + break; + } + if(u && (l > Runemax || (0xd800 <= l && l < 0xe000))) { + yyerror("invalid Unicode code point in escape sequence: %#llx", l); + l = Runeerror; + } + *val = l; + return 0; + +oct: + l = c - '0'; + for(i=2; i>0; i--) { + c = getc(); + if(c >= '0' && c <= '7') { + l = l*8 + c-'0'; + continue; + } + yyerror("non-octal character in escape sequence: %c", c); + ungetc(c); + } + if(l > 255) + yyerror("octal escape value > 255: %d", l); + + *val = l; + return 0; +} + +static struct +{ + char* name; + int lexical; + int etype; + int op; +} syms[] = +{ +/* name lexical etype op + */ +/* basic types */ + "int8", LNAME, TINT8, OXXX, + "int16", LNAME, TINT16, OXXX, + "int32", LNAME, TINT32, OXXX, + "int64", LNAME, TINT64, OXXX, + + "uint8", LNAME, TUINT8, OXXX, + "uint16", LNAME, TUINT16, OXXX, + "uint32", LNAME, TUINT32, OXXX, + "uint64", LNAME, TUINT64, OXXX, + + "float32", LNAME, TFLOAT32, OXXX, + "float64", LNAME, TFLOAT64, OXXX, + + "complex64", LNAME, TCOMPLEX64, OXXX, + "complex128", LNAME, TCOMPLEX128, OXXX, + + "bool", LNAME, TBOOL, OXXX, + "byte", LNAME, TUINT8, OXXX, + "string", LNAME, TSTRING, OXXX, + + "any", LNAME, TANY, OXXX, + + "break", LBREAK, Txxx, OXXX, + "case", LCASE, Txxx, OXXX, + "chan", LCHAN, Txxx, OXXX, + "const", LCONST, Txxx, OXXX, + "continue", LCONTINUE, Txxx, OXXX, + "default", LDEFAULT, Txxx, OXXX, + "else", LELSE, Txxx, OXXX, + "defer", LDEFER, Txxx, OXXX, + "fallthrough", LFALL, Txxx, OXXX, + "for", LFOR, Txxx, OXXX, + "func", LFUNC, Txxx, OXXX, + "go", LGO, Txxx, OXXX, + "goto", LGOTO, Txxx, OXXX, + "if", LIF, Txxx, OXXX, + "import", LIMPORT, Txxx, OXXX, + "interface", LINTERFACE, Txxx, OXXX, + "map", LMAP, Txxx, OXXX, + "package", LPACKAGE, Txxx, OXXX, + "range", LRANGE, Txxx, OXXX, + "return", LRETURN, Txxx, OXXX, + "select", LSELECT, Txxx, OXXX, + "struct", LSTRUCT, Txxx, OXXX, + "switch", LSWITCH, Txxx, OXXX, + "type", LTYPE, Txxx, OXXX, + "var", LVAR, Txxx, OXXX, + + "append", LNAME, Txxx, OAPPEND, + "cap", LNAME, Txxx, OCAP, + "close", LNAME, Txxx, OCLOSE, + "complex", LNAME, Txxx, OCOMPLEX, + "copy", LNAME, Txxx, OCOPY, + "imag", LNAME, Txxx, OIMAG, + "len", LNAME, Txxx, OLEN, + "make", LNAME, Txxx, OMAKE, + "new", LNAME, Txxx, ONEW, + "panic", LNAME, Txxx, OPANIC, + "print", LNAME, Txxx, OPRINT, + "println", LNAME, Txxx, OPRINTN, + "real", LNAME, Txxx, OREAL, + "recover", LNAME, Txxx, ORECOVER, + + "notwithstanding", LIGNORE, Txxx, OXXX, + "thetruthofthematter", LIGNORE, Txxx, OXXX, + "despiteallobjections", LIGNORE, Txxx, OXXX, + "whereas", LIGNORE, Txxx, OXXX, + "insofaras", LIGNORE, Txxx, OXXX, +}; + +static void +lexinit(void) +{ + int i, lex; + Sym *s, *s1; + Type *t; + int etype; + + /* + * initialize basic types array + * initialize known symbols + */ + for(i=0; ilexical = lex; + + etype = syms[i].etype; + if(etype != Txxx) { + if(etype < 0 || etype >= nelem(types)) + fatal("lexinit: %s bad etype", s->name); + t = types[etype]; + if(t == T) { + t = typ(etype); + t->sym = s; + + if(etype != TANY && etype != TSTRING) + dowidth(t); + types[etype] = t; + } + s1 = pkglookup(syms[i].name, builtinpkg); + s1->lexical = LNAME; + s1->def = typenod(t); + continue; + } + } + + // logically, the type of a string literal. + // types[TSTRING] is the named type string + // (the type of x in var x string or var x = "hello"). + // this is the ideal form + // (the type of x in const x = "hello"). + idealstring = typ(TSTRING); + idealbool = typ(TBOOL); + + s = pkglookup("true", builtinpkg); + s->def = nodbool(1); + s->def->sym = lookup("true"); + s->def->type = idealbool; + + s = pkglookup("false", builtinpkg); + s->def = nodbool(0); + s->def->sym = lookup("false"); + s->def->type = idealbool; + + s = lookup("_"); + s->block = -100; + s->def = nod(ONAME, N, N); + s->def->sym = s; + types[TBLANK] = typ(TBLANK); + s->def->type = types[TBLANK]; + nblank = s->def; +} + +static void +lexfini(void) +{ + Sym *s; + int lex, etype, i; + Val v; + + for(i=0; ilexical = lex; + + etype = syms[i].etype; + if(etype != Txxx && (etype != TANY || debug['A']) && s->def == N) + s->def = typenod(types[etype]); + + etype = syms[i].op; + if(etype != OXXX && s->def == N) { + s->def = nod(ONAME, N, N); + s->def->sym = s; + s->def->etype = etype; + s->def->builtin = 1; + } + } + + for(i=0; typedefs[i].name; i++) { + s = lookup(typedefs[i].name); + if(s->def == N) + s->def = typenod(types[typedefs[i].etype]); + } + + // there's only so much table-driven we can handle. + // these are special cases. + types[TNIL] = typ(TNIL); + s = lookup("nil"); + if(s->def == N) { + v.ctype = CTNIL; + s->def = nodlit(v); + s->def->sym = s; + } + + s = lookup("iota"); + if(s->def == N) { + s->def = nod(OIOTA, N, N); + s->def->sym = s; + } + + s = lookup("true"); + if(s->def == N) { + s->def = nodbool(1); + s->def->sym = s; + } + + s = lookup("false"); + if(s->def == N) { + s->def = nodbool(0); + s->def->sym = s; + } + + nodfp = nod(ONAME, N, N); + nodfp->noescape = 1; + nodfp->type = types[TINT32]; + nodfp->xoffset = 0; + nodfp->class = PPARAM; + nodfp->sym = lookup(".fp"); +} + +struct +{ + int lex; + char* name; +} lexn[] = +{ + LANDAND, "ANDAND", + LASOP, "ASOP", + LBREAK, "BREAK", + LCASE, "CASE", + LCHAN, "CHAN", + LCOLAS, "COLAS", + LCONST, "CONST", + LCONTINUE, "CONTINUE", + LDEC, "DEC", + LDEFER, "DEFER", + LELSE, "ELSE", + LEQ, "EQ", + LFALL, "FALL", + LFOR, "FOR", + LFUNC, "FUNC", + LGE, "GE", + LGO, "GO", + LGOTO, "GOTO", + LGT, "GT", + LIF, "IF", + LIMPORT, "IMPORT", + LINC, "INC", + LINTERFACE, "INTERFACE", + LLE, "LE", + LLITERAL, "LITERAL", + LLSH, "LSH", + LLT, "LT", + LMAP, "MAP", + LNAME, "NAME", + LNE, "NE", + LOROR, "OROR", + LPACKAGE, "PACKAGE", + LRANGE, "RANGE", + LRETURN, "RETURN", + LRSH, "RSH", + LSTRUCT, "STRUCT", + LSWITCH, "SWITCH", + LTYPE, "TYPE", + LVAR, "VAR", +}; + +char* +lexname(int lex) +{ + int i; + static char buf[100]; + + for(i=0; i=", + "LGT", ">", + "LLE", "<=", + "LLT", "<", + "LLSH", "<<", + "LRSH", ">>", + "LOROR", "||", + "LNE", "!=", + + // spell out to avoid confusion with punctuation in error messages + "';'", "semicolon or newline", + "','", "comma", +}; + +static void +yytinit(void) +{ + int i, j; + extern char *yytname[]; + char *s, *t; + + for(i=0; yytname[i] != nil; i++) { + s = yytname[i]; + + if(strcmp(s, "LLITERAL") == 0) { + strcpy(litbuf, "literal"); + yytname[i] = litbuf; + goto loop; + } + + // apply yytfix if possible + for(j=0; jname == nil) { + if(strcmp(pkgname, "_") == 0) + yyerror("invalid package name _"); + localpkg->name = pkgname; + } else { + if(strcmp(pkgname, localpkg->name) != 0) + yyerror("package %s; expected %s", pkgname, localpkg->name); + for(h=0; hlink) { + if(s->def == N || s->pkg != localpkg) + continue; + if(s->def->op == OPACK) { + // throw away top-level package name leftover + // from previous file. + // leave s->block set to cause redeclaration + // errors if a conflicting top-level name is + // introduced by a different file. + if(!s->def->used && !nsyntaxerrors) + yyerrorl(s->def->lineno, "imported and not used: %Z", s->def->pkg->path); + s->def = N; + continue; + } + if(s->def->sym != s) { + // throw away top-level name left over + // from previous import . "x" + if(s->def->pack != N && !s->def->pack->used && !nsyntaxerrors) { + yyerrorl(s->def->pack->lineno, "imported and not used: %Z", s->def->pack->pkg->path); + s->def->pack->used = 1; + } + s->def = N; + continue; + } + } + } + } + + if(outfile == nil) { + p = strrchr(infile, '/'); + if(p == nil) + p = infile; + else + p = p+1; + snprint(namebuf, sizeof(namebuf), "%s", p); + p = strrchr(namebuf, '.'); + if(p != nil) + *p = 0; + outfile = smprint("%s.%c", namebuf, thechar); + } +} diff --git a/src/cmd/gc/md5.c b/src/cmd/gc/md5.c new file mode 100644 index 000000000..7cea1a6cf --- /dev/null +++ b/src/cmd/gc/md5.c @@ -0,0 +1,290 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// 64-bit MD5 (does full MD5 but returns 64 bits only). +// Translation of ../../pkg/crypto/md5/md5*.go. + +#include "go.h" +#include "md5.h" + +static int md5block(MD5 *dig, uchar *p, int nn); + +enum { + _Chunk = 64 +}; + +#define _Init0 0x67452301 +#define _Init1 0xEFCDAB89 +#define _Init2 0x98BADCFE +#define _Init3 0x10325476 + +void +md5reset(MD5 *d) +{ + d->s[0] = _Init0; + d->s[1] = _Init1; + d->s[2] = _Init2; + d->s[3] = _Init3; + d->nx = 0; + d->len = 0; +} + +void +md5write(MD5 *d, uchar *p, int nn) +{ + int i, n; + + d->len += nn; + if(d->nx > 0) { + n = nn; + if(n > _Chunk - d->nx) + n = _Chunk - d->nx; + for(i=0; ix[d->nx+i] = p[i]; + d->nx += n; + if(d->nx == _Chunk) { + md5block(d, d->x, _Chunk); + d->nx = 0; + } + p += n; + nn -= n; + } + n = md5block(d, p, nn); + p += n; + nn -= n; + if(nn > 0) { + for(i=0; ix[i] = p[i]; + d->nx = nn; + } +} + +uint64 +md5sum(MD5 *d) +{ + uchar tmp[64]; + int i; + uint64 len; + + // Padding. Add a 1 bit and 0 bits until 56 bytes mod 64. + len = d->len; + memset(tmp, 0, sizeof tmp); + tmp[0] = 0x80; + if(len%64 < 56) + md5write(d, tmp, 56-len%64); + else + md5write(d, tmp, 64+56-len%64); + + // Length in bits. + len <<= 3; + for(i=0; i<8; i++) + tmp[i] = len>>(8*i); + md5write(d, tmp, 8); + + if(d->nx != 0) + fatal("md5sum"); + + return d->s[0] | ((uint64)d->s[1]<<32); +} + + +// MD5 block step. +// In its own file so that a faster assembly or C version +// can be substituted easily. + +// table[i] = int((1<<32) * abs(sin(i+1 radians))). +static uint32 table[64] = { + // round 1 + 0xd76aa478, + 0xe8c7b756, + 0x242070db, + 0xc1bdceee, + 0xf57c0faf, + 0x4787c62a, + 0xa8304613, + 0xfd469501, + 0x698098d8, + 0x8b44f7af, + 0xffff5bb1, + 0x895cd7be, + 0x6b901122, + 0xfd987193, + 0xa679438e, + 0x49b40821, + + // round 2 + 0xf61e2562, + 0xc040b340, + 0x265e5a51, + 0xe9b6c7aa, + 0xd62f105d, + 0x2441453, + 0xd8a1e681, + 0xe7d3fbc8, + 0x21e1cde6, + 0xc33707d6, + 0xf4d50d87, + 0x455a14ed, + 0xa9e3e905, + 0xfcefa3f8, + 0x676f02d9, + 0x8d2a4c8a, + + // round3 + 0xfffa3942, + 0x8771f681, + 0x6d9d6122, + 0xfde5380c, + 0xa4beea44, + 0x4bdecfa9, + 0xf6bb4b60, + 0xbebfbc70, + 0x289b7ec6, + 0xeaa127fa, + 0xd4ef3085, + 0x4881d05, + 0xd9d4d039, + 0xe6db99e5, + 0x1fa27cf8, + 0xc4ac5665, + + // round 4 + 0xf4292244, + 0x432aff97, + 0xab9423a7, + 0xfc93a039, + 0x655b59c3, + 0x8f0ccc92, + 0xffeff47d, + 0x85845dd1, + 0x6fa87e4f, + 0xfe2ce6e0, + 0xa3014314, + 0x4e0811a1, + 0xf7537e82, + 0xbd3af235, + 0x2ad7d2bb, + 0xeb86d391, +}; + +static uint32 shift1[] = { 7, 12, 17, 22 }; +static uint32 shift2[] = { 5, 9, 14, 20 }; +static uint32 shift3[] = { 4, 11, 16, 23 }; +static uint32 shift4[] = { 6, 10, 15, 21 }; + +static int +md5block(MD5 *dig, uchar *p, int nn) +{ + uint32 a, b, c, d, aa, bb, cc, dd; + int i, j, n; + uint32 X[16]; + + a = dig->s[0]; + b = dig->s[1]; + c = dig->s[2]; + d = dig->s[3]; + n = 0; + + while(nn >= _Chunk) { + aa = a; + bb = b; + cc = c; + dd = d; + + for(i=0; i<16; i++) { + j = i*4; + X[i] = p[j] | (p[j+1]<<8) | (p[j+2]<<16) | (p[j+3]<<24); + } + + // Round 1. + for(i=0; i<16; i++) { + uint32 x, t, s, f; + x = i; + t = i; + s = shift1[i%4]; + f = ((c ^ d) & b) ^ d; + a += f + X[x] + table[t]; + a = a<>(32-s); + a += b; + + t = d; + d = c; + c = b; + b = a; + a = t; + } + + // Round 2. + for(i=0; i<16; i++) { + uint32 x, t, s, g; + + x = (1+5*i)%16; + t = 16+i; + s = shift2[i%4]; + g = ((b ^ c) & d) ^ c; + a += g + X[x] + table[t]; + a = a<>(32-s); + a += b; + + t = d; + d = c; + c = b; + b = a; + a = t; + } + + // Round 3. + for(i=0; i<16; i++) { + uint32 x, t, s, h; + + x = (5+3*i)%16; + t = 32+i; + s = shift3[i%4]; + h = b ^ c ^ d; + a += h + X[x] + table[t]; + a = a<>(32-s); + a += b; + + t = d; + d = c; + c = b; + b = a; + a = t; + } + + // Round 4. + for(i=0; i<16; i++) { + uint32 x, s, t, ii; + + x = (7*i)%16; + s = shift4[i%4]; + t = 48+i; + ii = c ^ (b | ~d); + a += ii + X[x] + table[t]; + a = a<>(32-s); + a += b; + + t = d; + d = c; + c = b; + b = a; + a = t; + } + + a += aa; + b += bb; + c += cc; + d += dd; + + p += _Chunk; + n += _Chunk; + nn -= _Chunk; + } + + dig->s[0] = a; + dig->s[1] = b; + dig->s[2] = c; + dig->s[3] = d; + return n; +} diff --git a/src/cmd/gc/md5.h b/src/cmd/gc/md5.h new file mode 100644 index 000000000..f153e30f2 --- /dev/null +++ b/src/cmd/gc/md5.h @@ -0,0 +1,16 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +typedef struct MD5 MD5; +struct MD5 +{ + uint32 s[4]; + uchar x[64]; + int nx; + uint64 len; +}; + +void md5reset(MD5*); +void md5write(MD5*, uchar*, int); +uint64 md5sum(MD5*); diff --git a/src/cmd/gc/mkbuiltin b/src/cmd/gc/mkbuiltin new file mode 100755 index 000000000..cfd6e59c1 --- /dev/null +++ b/src/cmd/gc/mkbuiltin @@ -0,0 +1,31 @@ +#!/bin/sh +# Copyright 2009 The Go Authors. All rights reserved. +# Use of this source code is governed by a BSD-style +# license that can be found in the LICENSE file. + +# Generate builtin.c and builtin.c.boot from $* (runtime.go and unsafe.go). +# Run this after changing runtime.go and unsafe.go +# or after changing the export metadata format in the compiler. +# Either way, you need to have a working compiler binary first. + +set -e + +eval $(gomake --no-print-directory -f ../../Make.inc go-env) +if [ -z "$GC" ]; then + echo 'missing $GC - gomake failed?' 1>&2 + exit 1 +fi + +gomake mkbuiltin1 +rm -f _builtin.c +for i in runtime unsafe +do + $GC -A $i.go + O=$O ./mkbuiltin1 $i >>_builtin.c +done + +# If _builtin.c has changed vs builtin.c.boot, +# check in the new change. +cmp -s _builtin.c builtin.c.boot || cp _builtin.c builtin.c.boot + +mv _builtin.c builtin.c diff --git a/src/cmd/gc/mkbuiltin1.c b/src/cmd/gc/mkbuiltin1.c new file mode 100644 index 000000000..ad83c0346 --- /dev/null +++ b/src/cmd/gc/mkbuiltin1.c @@ -0,0 +1,84 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Compile .go file, import data from .6 file, and generate C string version. + +#include +#include +#include + +void esc(char*); + +void +main(int argc, char **argv) +{ + char *name; + FILE *fin; + char buf[1024], initfunc[1024], *p, *q; + + if(argc != 2) { + fprintf(stderr, "usage: mkbuiltin1 sys\n"); + sysfatal("in file $1.6 s/PACKAGE/$1/\n"); + } + + name = argv[1]; + snprintf(initfunc, sizeof(initfunc), "init_%s_function", name); + + snprintf(buf, sizeof(buf), "%s.%s", name, getenv("O")); + if((fin = fopen(buf, "r")) == NULL) { + sysfatal("open %s: %r\n", buf); + } + + // look for $$ that introduces imports + while(fgets(buf, sizeof buf, fin) != NULL) + if(strstr(buf, "$$")) + goto begin; + sysfatal("did not find beginning of imports\n"); + +begin: + printf("char *%simport =\n", name); + + // process imports, stopping at $$ that closes them + while(fgets(buf, sizeof buf, fin) != NULL) { + buf[strlen(buf)-1] = 0; // chop \n + if(strstr(buf, "$$")) + goto end; + + // chop leading white space + for(p=buf; *p==' ' || *p == '\t'; p++) + ; + + // cut out decl of init_$1_function - it doesn't exist + if(strstr(buf, initfunc)) + continue; + + // sys.go claims to be in package PACKAGE to avoid + // conflicts during "6g sys.go". rename PACKAGE to $2. + printf("\t\""); + while(q = strstr(p, "PACKAGE")) { + *q = 0; + esc(p); // up to the substitution + printf("%s", name); // the sub name + p = q+7; // continue with rest + } + + esc(p); + printf("\\n\"\n"); + } + sysfatal("did not find end of imports\n"); + +end: + printf("\t\"$$\\n\";\n"); + exits(0); +} + +void +esc(char *p) +{ + for(; *p; p++) { + if(*p == '\\' || *p == '\"') + printf("\\"); + putchar(*p); + } +} diff --git a/src/cmd/gc/mkopnames b/src/cmd/gc/mkopnames new file mode 100755 index 000000000..fb2ceec81 --- /dev/null +++ b/src/cmd/gc/mkopnames @@ -0,0 +1,24 @@ +#!/bin/sh +# Copyright 2009 The Go Authors. All rights reserved. +# Use of this source code is governed by a BSD-style +# license that can be found in the LICENSE file. + +# Disable colored grep if user has it set to --color=always. +# (Arguably user error.) +export GREP_OPTIONS="" + +echo '// auto generated by mkopnames' +echo 'static char*' +echo 'opnames[] = ' +echo '{' +sed -n '/OXXX/,/OEND/p' go.h | + cpp | + sed 's!//.*!!; /^#/d' | + tr ' ' '\n' | + tr -d ' \t,' | + grep . | + sort | + grep -v '^OEND$' | + sed 's/O//; s/.*/ [O&] = "&",/' +echo '};' + diff --git a/src/cmd/gc/mparith1.c b/src/cmd/gc/mparith1.c new file mode 100644 index 000000000..6cd4e2500 --- /dev/null +++ b/src/cmd/gc/mparith1.c @@ -0,0 +1,509 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +#include "go.h" + +/// uses arithmetic + +int +mpcmpfixflt(Mpint *a, Mpflt *b) +{ + char buf[500]; + Mpflt c; + + snprint(buf, sizeof(buf), "%B", a); + mpatoflt(&c, buf); + return mpcmpfltflt(&c, b); +} + +int +mpcmpfltfix(Mpflt *a, Mpint *b) +{ + char buf[500]; + Mpflt c; + + snprint(buf, sizeof(buf), "%B", b); + mpatoflt(&c, buf); + return mpcmpfltflt(a, &c); +} + +int +mpcmpfixfix(Mpint *a, Mpint *b) +{ + Mpint c; + + mpmovefixfix(&c, a); + mpsubfixfix(&c, b); + return mptestfix(&c); +} + +int +mpcmpfixc(Mpint *b, vlong c) +{ + Mpint a; + + mpmovecfix(&a, c); + return mpcmpfixfix(&a, b); +} + +int +mpcmpfltflt(Mpflt *a, Mpflt *b) +{ + Mpflt c; + + mpmovefltflt(&c, a); + mpsubfltflt(&c, b); + return mptestflt(&c); +} + +int +mpcmpfltc(Mpflt *b, double c) +{ + Mpflt a; + + mpmovecflt(&a, c); + return mpcmpfltflt(&a, b); +} + +void +mpsubfixfix(Mpint *a, Mpint *b) +{ + mpnegfix(a); + mpaddfixfix(a, b); + mpnegfix(a); +} + +void +mpsubfltflt(Mpflt *a, Mpflt *b) +{ + mpnegflt(a); + mpaddfltflt(a, b); + mpnegflt(a); +} + +void +mpaddcfix(Mpint *a, vlong c) +{ + Mpint b; + + mpmovecfix(&b, c); + mpaddfixfix(a, &b); +} + +void +mpaddcflt(Mpflt *a, double c) +{ + Mpflt b; + + mpmovecflt(&b, c); + mpaddfltflt(a, &b); +} + +void +mpmulcfix(Mpint *a, vlong c) +{ + Mpint b; + + mpmovecfix(&b, c); + mpmulfixfix(a, &b); +} + +void +mpmulcflt(Mpflt *a, double c) +{ + Mpflt b; + + mpmovecflt(&b, c); + mpmulfltflt(a, &b); +} + +void +mpdivfixfix(Mpint *a, Mpint *b) +{ + Mpint q, r; + + mpdivmodfixfix(&q, &r, a, b); + mpmovefixfix(a, &q); +} + +void +mpmodfixfix(Mpint *a, Mpint *b) +{ + Mpint q, r; + + mpdivmodfixfix(&q, &r, a, b); + mpmovefixfix(a, &r); +} + +void +mpcomfix(Mpint *a) +{ + Mpint b; + + mpmovecfix(&b, 1); + mpnegfix(a); + mpsubfixfix(a, &b); +} + +void +mpmovefixflt(Mpflt *a, Mpint *b) +{ + a->val = *b; + a->exp = 0; + mpnorm(a); +} + +// convert (truncate) b to a. +// return -1 (but still convert) if b was non-integer. +static int +mpexactfltfix(Mpint *a, Mpflt *b) +{ + Mpflt f; + + *a = b->val; + mpshiftfix(a, b->exp); + if(b->exp < 0) { + f.val = *a; + f.exp = 0; + mpnorm(&f); + if(mpcmpfltflt(b, &f) != 0) + return -1; + } + return 0; +} + +int +mpmovefltfix(Mpint *a, Mpflt *b) +{ + Mpflt f; + int i; + + if(mpexactfltfix(a, b) == 0) + return 0; + + // try rounding down a little + f = *b; + f.val.a[0] = 0; + if(mpexactfltfix(a, &f) == 0) + return 0; + + // try rounding up a little + for(i=1; i>1); + mpmulfltflt(a, a); + if(p & 1) + mpmulcflt(a, 10); +} + +// +// floating point input +// required syntax is [+-]d*[.]d*[e[+-]d*] +// +void +mpatoflt(Mpflt *a, char *as) +{ + Mpflt b; + int dp, c, f, ef, ex, eb; + char *s; + + s = as; + dp = 0; /* digits after decimal point */ + f = 0; /* sign */ + ex = 0; /* exponent */ + eb = 0; /* binary point */ + + mpmovecflt(a, 0.0); + for(;;) { + switch(c = *s++) { + default: + goto bad; + + case '-': + f = 1; + + case ' ': + case '\t': + case '+': + continue; + + case '.': + dp = 1; + continue; + + case '1': + case '2': + case '3': + case '4': + case '5': + case '6': + case '7': + case '8': + case '9': + case '0': + mpmulcflt(a, 10); + mpaddcflt(a, c-'0'); + if(dp) + dp++; + continue; + + case 'P': + case 'p': + eb = 1; + + case 'E': + case 'e': + ex = 0; + ef = 0; + for(;;) { + c = *s++; + if(c == '+' || c == ' ' || c == '\t') + continue; + if(c == '-') { + ef = 1; + continue; + } + if(c >= '0' && c <= '9') { + ex = ex*10 + (c-'0'); + if(ex > 1e8) { + yyerror("exponent out of range"); + errorexit(); + } + continue; + } + break; + } + if(ef) + ex = -ex; + + case 0: + break; + } + break; + } + + if(eb) { + if(dp) + goto bad; + a->exp += ex; + goto out; + } + + if(dp) + dp--; + if(mpcmpfltc(a, 0.0) != 0) { + if(ex >= dp) { + mppow10flt(&b, ex-dp); + mpmulfltflt(a, &b); + } else { + mppow10flt(&b, dp-ex); + mpdivfltflt(a, &b); + } + } + +out: + if(f) + mpnegflt(a); + return; + +bad: + yyerror("set ovf in mpatof"); + mpmovecflt(a, 0.0); +} + +// +// fixed point input +// required syntax is [+-][0[x]]d* +// +void +mpatofix(Mpint *a, char *as) +{ + int c, f; + char *s; + + s = as; + f = 0; + mpmovecfix(a, 0); + + c = *s++; + switch(c) { + case '-': + f = 1; + + case '+': + c = *s++; + if(c != '0') + break; + + case '0': + goto oct; + } + + while(c) { + if(c >= '0' && c <= '9') { + mpmulcfix(a, 10); + mpaddcfix(a, c-'0'); + c = *s++; + continue; + } + goto bad; + } + goto out; + +oct: + c = *s++; + if(c == 'x' || c == 'X') + goto hex; + while(c) { + if(c >= '0' && c <= '7') { + mpmulcfix(a, 8); + mpaddcfix(a, c-'0'); + c = *s++; + continue; + } + goto bad; + } + goto out; + +hex: + c = *s++; + while(c) { + if(c >= '0' && c <= '9') { + mpmulcfix(a, 16); + mpaddcfix(a, c-'0'); + c = *s++; + continue; + } + if(c >= 'a' && c <= 'f') { + mpmulcfix(a, 16); + mpaddcfix(a, c+10-'a'); + c = *s++; + continue; + } + if(c >= 'A' && c <= 'F') { + mpmulcfix(a, 16); + mpaddcfix(a, c+10-'A'); + c = *s++; + continue; + } + goto bad; + } + +out: + if(f) + mpnegfix(a); + return; + +bad: + yyerror("set ovf in mpatov: %s", as); + mpmovecfix(a, 0); +} + +int +Bconv(Fmt *fp) +{ + char buf[500], *p; + Mpint *xval, q, r, ten; + int f; + + xval = va_arg(fp->args, Mpint*); + mpmovefixfix(&q, xval); + f = 0; + if(mptestfix(&q) < 0) { + f = 1; + mpnegfix(&q); + } + mpmovecfix(&ten, 10); + + p = &buf[sizeof(buf)]; + *--p = 0; + for(;;) { + mpdivmodfixfix(&q, &r, &q, &ten); + *--p = mpgetfix(&r) + '0'; + if(mptestfix(&q) <= 0) + break; + } + if(f) + *--p = '-'; + return fmtstrcpy(fp, p); +} + +int +Fconv(Fmt *fp) +{ + char buf[500]; + Mpflt *fvp, fv; + double d; + + fvp = va_arg(fp->args, Mpflt*); + if(fp->flags & FmtSharp) { + // alternate form - decimal for error messages. + // for well in range, convert to double and use print's %g + if(-900 < fvp->exp && fvp->exp < 900) { + d = mpgetflt(fvp); + if(d >= 0 && (fp->flags & FmtSign)) + fmtprint(fp, "+"); + return fmtprint(fp, "%g", d); + } + // TODO(rsc): for well out of range, print + // an approximation like 1.234e1000 + } + + if(sigfig(fvp) == 0) { + snprint(buf, sizeof(buf), "0p+0"); + goto out; + } + fv = *fvp; + + while(fv.val.a[0] == 0) { + mpshiftfix(&fv.val, -Mpscale); + fv.exp += Mpscale; + } + while((fv.val.a[0]&1) == 0) { + mpshiftfix(&fv.val, -1); + fv.exp += 1; + } + + if(fv.exp >= 0) { + snprint(buf, sizeof(buf), "%Bp+%d", &fv.val, fv.exp); + goto out; + } + snprint(buf, sizeof(buf), "%Bp-%d", &fv.val, -fv.exp); + +out: + return fmtstrcpy(fp, buf); +} diff --git a/src/cmd/gc/mparith2.c b/src/cmd/gc/mparith2.c new file mode 100644 index 000000000..403255005 --- /dev/null +++ b/src/cmd/gc/mparith2.c @@ -0,0 +1,683 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +#include "go.h" + +// +// return the significant +// words of the argument +// +static int +mplen(Mpint *a) +{ + int i, n; + long *a1; + + n = -1; + a1 = &a->a[0]; + for(i=0; ia[0]; + for(i=0; i= Mpbase) { + x -= Mpbase; + c = 1; + } + *a1++ = x; + } +} + +// +// left shift mpint by Mpscale +// ignores sign and overflow +// +static void +mplshw(Mpint *a) +{ + long *a1; + int i; + + a1 = &a->a[Mpprec-1]; + for(i=1; ia[0] & 1; + a1 = &a->a[Mpprec]; + for(i=0; i> 1; + c = 0; + if(x & 1) + c = Mpbase; + } + if(a->neg && lo != 0) + mpaddcfix(a, -1); +} + +// +// right shift mpint by Mpscale +// ignores sign and overflow +// +static void +mprshw(Mpint *a) +{ + long *a1, lo; + int i; + + lo = a->a[0]; + a1 = &a->a[0]; + for(i=1; ineg && lo != 0) + mpaddcfix(a, -1); +} + +// +// return the sign of (abs(a)-abs(b)) +// +static int +mpcmp(Mpint *a, Mpint *b) +{ + long x, *a1, *b1; + int i; + + if(a->ovf || b->ovf) { + yyerror("ovf in cmp"); + return 0; + } + + a1 = &a->a[0] + Mpprec; + b1 = &b->a[0] + Mpprec; + + for(i=0; i 0) + return +1; + if(x < 0) + return -1; + } + return 0; +} + +// +// negate a +// ignore sign and ovf +// +static void +mpneg(Mpint *a) +{ + long x, *a1; + int i, c; + + a1 = &a->a[0]; + c = 0; + for(i=0; i= 0) { + while(s >= Mpscale) { + mplshw(a); + s -= Mpscale; + } + while(s > 0) { + mplsh(a); + s--; + } + } else { + s = -s; + while(s >= Mpscale) { + mprshw(a); + s -= Mpscale; + } + while(s > 0) { + mprsh(a); + s--; + } + } +} + +/// implements fix arihmetic + +void +mpaddfixfix(Mpint *a, Mpint *b) +{ + int i, c; + long x, *a1, *b1; + + if(a->ovf || b->ovf) { + yyerror("ovf in mpaddxx"); + a->ovf = 1; + return; + } + + c = 0; + a1 = &a->a[0]; + b1 = &b->a[0]; + if(a->neg != b->neg) + goto sub; + + // perform a+b + for(i=0; i= Mpbase) { + x -= Mpbase; + c = 1; + } + *a1++ = x; + } + a->ovf = c; + if(a->ovf) + yyerror("set ovf in mpaddxx"); + + return; + +sub: + // perform a-b + switch(mpcmp(a, b)) { + case 0: + mpmovecfix(a, 0); + break; + + case 1: + for(i=0; ineg ^= 1; + for(i=0; iovf || b->ovf) { + yyerror("ovf in mpmulfixfix"); + a->ovf = 1; + return; + } + + // pick the smaller + // to test for bits + na = mplen(a); + nb = mplen(b); + if(na > nb) { + mpmovefixfix(&s, a); + a1 = &b->a[0]; + na = nb; + } else { + mpmovefixfix(&s, b); + a1 = &a->a[0]; + } + s.neg = 0; + + mpmovecfix(&q, 0); + for(i=0; i>= 1; + } + } + + q.neg = a->neg ^ b->neg; + mpmovefixfix(a, &q); + if(a->ovf) + yyerror("set ovf in mpmulfixfix"); +} + +void +mpmulfract(Mpint *a, Mpint *b) +{ + + int i, j; + long *a1, x; + Mpint s, q; + + if(a->ovf || b->ovf) { + yyerror("ovf in mpmulflt"); + a->ovf = 1; + return; + } + + mpmovefixfix(&s, b); + a1 = &a->a[Mpprec]; + s.neg = 0; + mpmovecfix(&q, 0); + + x = *--a1; + if(x != 0) + yyerror("mpmulfract not normal"); + + for(i=0; ineg ^ b->neg; + mpmovefixfix(a, &q); + if(a->ovf) + yyerror("set ovf in mpmulflt"); +} + +void +mporfixfix(Mpint *a, Mpint *b) +{ + int i; + long x, *a1, *b1; + + if(a->ovf || b->ovf) { + yyerror("ovf in mporfixfix"); + mpmovecfix(a, 0); + a->ovf = 1; + return; + } + if(a->neg) { + a->neg = 0; + mpneg(a); + } + if(b->neg) + mpneg(b); + + a1 = &a->a[0]; + b1 = &b->a[0]; + for(i=0; ineg) + mpneg(b); + if(x & Mpsign) { + a->neg = 1; + mpneg(a); + } +} + +void +mpandfixfix(Mpint *a, Mpint *b) +{ + int i; + long x, *a1, *b1; + + if(a->ovf || b->ovf) { + yyerror("ovf in mpandfixfix"); + mpmovecfix(a, 0); + a->ovf = 1; + return; + } + if(a->neg) { + a->neg = 0; + mpneg(a); + } + if(b->neg) + mpneg(b); + + a1 = &a->a[0]; + b1 = &b->a[0]; + for(i=0; ineg) + mpneg(b); + if(x & Mpsign) { + a->neg = 1; + mpneg(a); + } +} + +void +mpandnotfixfix(Mpint *a, Mpint *b) +{ + int i; + long x, *a1, *b1; + + if(a->ovf || b->ovf) { + yyerror("ovf in mpandnotfixfix"); + mpmovecfix(a, 0); + a->ovf = 1; + return; + } + if(a->neg) { + a->neg = 0; + mpneg(a); + } + if(b->neg) + mpneg(b); + + a1 = &a->a[0]; + b1 = &b->a[0]; + for(i=0; ineg) + mpneg(b); + if(x & Mpsign) { + a->neg = 1; + mpneg(a); + } +} + +void +mpxorfixfix(Mpint *a, Mpint *b) +{ + int i; + long x, *a1, *b1; + + if(a->ovf || b->ovf) { + yyerror("ovf in mporfixfix"); + mpmovecfix(a, 0); + a->ovf = 1; + return; + } + if(a->neg) { + a->neg = 0; + mpneg(a); + } + if(b->neg) + mpneg(b); + + a1 = &a->a[0]; + b1 = &b->a[0]; + for(i=0; ineg) + mpneg(b); + if(x & Mpsign) { + a->neg = 1; + mpneg(a); + } +} + +void +mplshfixfix(Mpint *a, Mpint *b) +{ + vlong s; + + if(a->ovf || b->ovf) { + yyerror("ovf in mporfixfix"); + mpmovecfix(a, 0); + a->ovf = 1; + return; + } + s = mpgetfix(b); + if(s < 0 || s >= Mpprec*Mpscale) { + yyerror("stupid shift: %lld", s); + mpmovecfix(a, 0); + return; + } + + mpshiftfix(a, s); +} + +void +mprshfixfix(Mpint *a, Mpint *b) +{ + vlong s; + + if(a->ovf || b->ovf) { + yyerror("ovf in mprshfixfix"); + mpmovecfix(a, 0); + a->ovf = 1; + return; + } + s = mpgetfix(b); + if(s < 0 || s >= Mpprec*Mpscale) { + yyerror("stupid shift: %lld", s); + if(a->neg) + mpmovecfix(a, -1); + else + mpmovecfix(a, 0); + return; + } + + mpshiftfix(a, -s); +} + +void +mpnegfix(Mpint *a) +{ + a->neg ^= 1; +} + +vlong +mpgetfix(Mpint *a) +{ + vlong v; + + if(a->ovf) { + yyerror("constant overflow"); + return 0; + } + + v = (vlong)a->a[0]; + v |= (vlong)a->a[1] << Mpscale; + v |= (vlong)a->a[2] << (Mpscale+Mpscale); + if(a->neg) + v = -v; + return v; +} + +void +mpmovecfix(Mpint *a, vlong c) +{ + int i; + long *a1; + vlong x; + + a->neg = 0; + a->ovf = 0; + + x = c; + if(x < 0) { + a->neg = 1; + x = -x; + } + + a1 = &a->a[0]; + for(i=0; i>= Mpscale; + } +} + +void +mpdivmodfixfix(Mpint *q, Mpint *r, Mpint *n, Mpint *d) +{ + int i, ns, ds; + + ns = n->neg; + ds = d->neg; + n->neg = 0; + d->neg = 0; + + mpmovefixfix(r, n); + mpmovecfix(q, 0); + + // shift denominator until it + // is larger than numerator + for(i=0; i 0) + break; + mplsh(d); + } + + // if it never happens + // denominator is probably zero + if(i >= Mpprec*Mpscale) { + q->ovf = 1; + r->ovf = 1; + n->neg = ns; + d->neg = ds; + yyerror("set ovf in mpdivmodfixfix"); + return; + } + + // shift denominator back creating + // quotient a bit at a time + // when done the remaining numerator + // will be the remainder + for(; i>0; i--) { + mplsh(q); + mprsh(d); + if(mpcmp(d, r) <= 0) { + mpaddcfix(q, 1); + mpsubfixfix(r, d); + } + } + + n->neg = ns; + d->neg = ds; + r->neg = ns; + q->neg = ns^ds; +} + +static int +iszero(Mpint *a) +{ + long *a1; + int i; + a1 = &a->a[0] + Mpprec; + for(i=0; ia[Mpprec]; // quotient + + neg = n.neg ^ d.neg; + n.neg = 0; + d.neg = 0; + for(i=0; ineg = neg; +} + +int +mptestfix(Mpint *a) +{ + Mpint b; + int r; + + mpmovecfix(&b, 0); + r = mpcmp(a, &b); + if(a->neg) { + if(r > 0) + return -1; + if(r < 0) + return +1; + } + return r; +} diff --git a/src/cmd/gc/mparith3.c b/src/cmd/gc/mparith3.c new file mode 100644 index 000000000..b11a4f5f1 --- /dev/null +++ b/src/cmd/gc/mparith3.c @@ -0,0 +1,313 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +#include "go.h" + +/* + * returns the leading non-zero + * word of the number + */ +int +sigfig(Mpflt *a) +{ + int i; + + for(i=Mpprec-1; i>=0; i--) + if(a->val.a[i] != 0) + break; +//print("sigfig %d %d\n", i-z+1, z); + return i+1; +} + +/* + * shifts the leading non-zero + * word of the number to Mpnorm + */ +void +mpnorm(Mpflt *a) +{ + int s, os; + long x; + + os = sigfig(a); + if(os == 0) { + // zero + a->exp = 0; + a->val.neg = 0; + return; + } + + // this will normalize to the nearest word + x = a->val.a[os-1]; + s = (Mpnorm-os) * Mpscale; + + // further normalize to the nearest bit + for(;;) { + x <<= 1; + if(x & Mpbase) + break; + s++; + if(x == 0) { + // this error comes from trying to + // convert an Inf or something + // where the initial x=0x80000000 + s = (Mpnorm-os) * Mpscale; + break; + } + } + + mpshiftfix(&a->val, s); + a->exp -= s; +} + +/// implements float arihmetic + +void +mpaddfltflt(Mpflt *a, Mpflt *b) +{ + int sa, sb, s; + Mpflt c; + + if(Mpdebug) + print("\n%F + %F", a, b); + + sa = sigfig(a); + if(sa == 0) { + mpmovefltflt(a, b); + goto out; + } + + sb = sigfig(b); + if(sb == 0) + goto out; + + s = a->exp - b->exp; + if(s > 0) { + // a is larger, shift b right + mpmovefltflt(&c, b); + mpshiftfix(&c.val, -s); + mpaddfixfix(&a->val, &c.val); + goto out; + } + if(s < 0) { + // b is larger, shift a right + mpshiftfix(&a->val, s); + a->exp -= s; + mpaddfixfix(&a->val, &b->val); + goto out; + } + mpaddfixfix(&a->val, &b->val); + +out: + mpnorm(a); + if(Mpdebug) + print(" = %F\n\n", a); +} + +void +mpmulfltflt(Mpflt *a, Mpflt *b) +{ + int sa, sb; + + if(Mpdebug) + print("%F\n * %F\n", a, b); + + sa = sigfig(a); + if(sa == 0) { + // zero + a->exp = 0; + a->val.neg = 0; + return; + } + + sb = sigfig(b); + if(sb == 0) { + // zero + mpmovefltflt(a, b); + return; + } + + mpmulfract(&a->val, &b->val); + a->exp = (a->exp + b->exp) + Mpscale*Mpprec - Mpscale - 1; + + mpnorm(a); + if(Mpdebug) + print(" = %F\n\n", a); +} + +void +mpdivfltflt(Mpflt *a, Mpflt *b) +{ + int sa, sb; + Mpflt c; + + if(Mpdebug) + print("%F\n / %F\n", a, b); + + sb = sigfig(b); + if(sb == 0) { + // zero and ovfl + a->exp = 0; + a->val.neg = 0; + a->val.ovf = 1; + yyerror("mpdivfltflt divide by zero"); + return; + } + + sa = sigfig(a); + if(sa == 0) { + // zero + a->exp = 0; + a->val.neg = 0; + return; + } + + // adjust b to top + mpmovefltflt(&c, b); + mpshiftfix(&c.val, Mpscale); + + // divide + mpdivfract(&a->val, &c.val); + a->exp = (a->exp-c.exp) - Mpscale*(Mpprec-1) + 1; + + mpnorm(a); + if(Mpdebug) + print(" = %F\n\n", a); +} + +double +mpgetflt(Mpflt *a) +{ + int s, i, e; + uvlong v, vm; + double f; + + if(a->val.ovf) + yyerror("mpgetflt ovf"); + + s = sigfig(a); + if(s == 0) + return 0; + + if(s != Mpnorm) { + yyerror("mpgetflt norm"); + mpnorm(a); + } + + while((a->val.a[Mpnorm-1] & Mpsign) == 0) { + mpshiftfix(&a->val, 1); + a->exp -= 1; + } + + // the magic numbers (64, 63, 53, 10, -1074) are + // IEEE specific. this should be done machine + // independently or in the 6g half of the compiler + + // pick up the mantissa and a rounding bit in a uvlong + s = 53+1; + v = 0; + for(i=Mpnorm-1; s>=Mpscale; i--) { + v = (v<val.a[i]; + s -= Mpscale; + } + vm = v; + if(s > 0) + vm = (vm<val.a[i]>>(Mpscale-s)); + + // continue with 64 more bits + s += 64; + for(; s>=Mpscale; i--) { + v = (v<val.a[i]; + s -= Mpscale; + } + if(s > 0) + v = (v<val.a[i]>>(Mpscale-s)); + + // gradual underflow + e = Mpnorm*Mpscale + a->exp - 53; + if(e < -1074) { + s = -e - 1074; + if(s > 54) + s = 54; + v |= vm & ((1ULL<>= s; + e = -1074; + } + +//print("vm=%.16llux v=%.16llux\n", vm, v); + // round toward even + if(v != 0 || (vm&2ULL) != 0) + vm = (vm>>1) + (vm&1ULL); + else + vm >>= 1; + + f = (double)(vm); + f = ldexp(f, e); + + if(a->val.neg) + f = -f; + return f; +} + +void +mpmovecflt(Mpflt *a, double c) +{ + int i; + double f; + long l; + + if(Mpdebug) + print("\nconst %g", c); + mpmovecfix(&a->val, 0); + a->exp = 0; + if(c == 0) + goto out; + if(c < 0) { + a->val.neg = 1; + c = -c; + } + + f = frexp(c, &i); + a->exp = i; + + for(i=0; i<10; i++) { + f = f*Mpbase; + l = floor(f); + f = f - l; + a->exp -= Mpscale; + a->val.a[0] = l; + if(f == 0) + break; + mpshiftfix(&a->val, Mpscale); + } + +out: + mpnorm(a); + if(Mpdebug) + print(" = %F\n", a); +} + +void +mpnegflt(Mpflt *a) +{ + a->val.neg ^= 1; +} + +int +mptestflt(Mpflt *a) +{ + int s; + + if(Mpdebug) + print("\n%F?", a); + s = sigfig(a); + if(s != 0) { + s = +1; + if(a->val.neg) + s = -1; + } + if(Mpdebug) + print(" = %d\n", s); + return s; +} diff --git a/src/cmd/gc/obj.c b/src/cmd/gc/obj.c new file mode 100644 index 000000000..456aabb88 --- /dev/null +++ b/src/cmd/gc/obj.c @@ -0,0 +1,301 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +#include "go.h" + +/* + * architecture-independent object file output + */ + +static void outhist(Biobuf *b); +static void dumpglobls(void); + +void +dumpobj(void) +{ + bout = Bopen(outfile, OWRITE); + if(bout == nil) { + flusherrors(); + print("can't create %s: %r\n", outfile); + errorexit(); + } + + Bprint(bout, "go object %s %s %s\n", getgoos(), thestring, getgoversion()); + Bprint(bout, " exports automatically generated from\n"); + Bprint(bout, " %s in package \"%s\"\n", curio.infile, localpkg->name); + dumpexport(); + Bprint(bout, "\n!\n"); + + outhist(bout); + + // add nil plist w AEND to catch + // auto-generated trampolines, data + newplist(); + + dumpglobls(); + dumptypestructs(); + dumpdata(); + dumpfuncs(); + + Bterm(bout); +} + +static void +dumpglobls(void) +{ + Node *n; + NodeList *l; + + // add globals + for(l=externdcl; l; l=l->next) { + n = l->n; + if(n->op != ONAME) + continue; + + if(n->type == T) + fatal("external %#N nil type\n", n); + if(n->class == PFUNC) + continue; + if(n->sym->pkg != localpkg) + continue; + dowidth(n->type); + + ggloblnod(n, n->type->width); + } +} + +void +Bputname(Biobuf *b, Sym *s) +{ + Bprint(b, "%s", s->pkg->prefix); + Bputc(b, '.'); + Bwrite(b, s->name, strlen(s->name)+1); +} + +static void +outzfile(Biobuf *b, char *p) +{ + char *q, *q2; + + while(p) { + q = utfrune(p, '/'); + if(windows) { + q2 = utfrune(p, '\\'); + if(q2 && (!q || q2 < q)) + q = q2; + } + if(!q) { + zfile(b, p, strlen(p)); + return; + } + if(q > p) + zfile(b, p, q-p); + p = q + 1; + } +} + +#define isdelim(c) (c == '/' || c == '\\') + +static void +outwinname(Biobuf *b, Hist *h, char *ds, char *p) +{ + if(isdelim(p[0])) { + // full rooted name + zfile(b, ds, 3); // leading "c:/" + outzfile(b, p+1); + } else { + // relative name + if(h->offset == 0 && pathname && pathname[1] == ':') { + if(tolowerrune(ds[0]) == tolowerrune(pathname[0])) { + // using current drive + zfile(b, pathname, 3); // leading "c:/" + outzfile(b, pathname+3); + } else { + // using drive other then current, + // we don't have any simple way to + // determine current working directory + // there, therefore will output name as is + zfile(b, ds, 2); // leading "c:" + } + } + outzfile(b, p); + } +} + +static void +outhist(Biobuf *b) +{ + Hist *h; + int i, depth = 0; + char *p, ds[] = {'c', ':', '/', 0}; + + for(h = hist; h != H; h = h->link) { + p = h->name; + if(p) { + if(windows) { + // if windows variable is set, then, we know already, + // pathname is started with windows drive specifier + // and all '\' were replaced with '/' (see lex.c) + if(isdelim(p[0]) && isdelim(p[1])) { + // file name has network name in it, + // like \\server\share\dir\file.go + zfile(b, "//", 2); // leading "//" + outzfile(b, p+2); + } else if(p[1] == ':') { + // file name has drive letter in it + ds[0] = p[0]; + outwinname(b, h, ds, p+2); + } else { + // no drive letter in file name + outwinname(b, h, pathname, p); + } + } else { + if(p[0] == '/') { + // full rooted name, like /home/rsc/dir/file.go + zfile(b, "/", 1); // leading "/" + outzfile(b, p+1); + } else { + // relative name, like dir/file.go + if(h->offset >= 0 && pathname && pathname[0] == '/') { + zfile(b, "/", 1); // leading "/" + outzfile(b, pathname+1); + } + outzfile(b, p); + } + } + if(h->offset > 0) { + //line directive + depth++; + } + } else if(depth > 0) { + for(i = 0; i < depth; i++) + zhist(b, h->line, h->offset); + depth = 0; + } + zhist(b, h->line, h->offset); + } +} + +void +ieeedtod(uint64 *ieee, double native) +{ + double fr, ho, f; + int exp; + uint32 h, l; + uint64 bits; + + if(native < 0) { + ieeedtod(ieee, -native); + *ieee |= 1ULL<<63; + return; + } + if(native == 0) { + *ieee = 0; + return; + } + fr = frexp(native, &exp); + f = 2097152L; /* shouldnt use fp constants here */ + fr = modf(fr*f, &ho); + h = ho; + h &= 0xfffffL; + f = 65536L; + fr = modf(fr*f, &ho); + l = ho; + l <<= 16; + l |= (int32)(fr*f); + bits = ((uint64)h<<32) | l; + if(exp < -1021) { + // gradual underflow + bits |= 1LL<<52; + bits >>= -1021 - exp; + exp = -1022; + } + bits |= (uint64)(exp+1022L) << 52; + *ieee = bits; +} + +int +duint8(Sym *s, int off, uint8 v) +{ + return duintxx(s, off, v, 1); +} + +int +duint16(Sym *s, int off, uint16 v) +{ + return duintxx(s, off, v, 2); +} + +int +duint32(Sym *s, int off, uint32 v) +{ + return duintxx(s, off, v, 4); +} + +int +duint64(Sym *s, int off, uint64 v) +{ + return duintxx(s, off, v, 8); +} + +int +duintptr(Sym *s, int off, uint64 v) +{ + return duintxx(s, off, v, widthptr); +} + +Sym* +stringsym(char *s, int len) +{ + static int gen; + Sym *sym; + int off, n, m; + struct { + Strlit lit; + char buf[110]; + } tmp; + Pkg *pkg; + + if(len > 100) { + // huge strings are made static to avoid long names + snprint(namebuf, sizeof(namebuf), ".gostring.%d", ++gen); + pkg = localpkg; + } else { + // small strings get named by their contents, + // so that multiple modules using the same string + // can share it. + tmp.lit.len = len; + memmove(tmp.lit.s, s, len); + tmp.lit.s[len] = '\0'; + snprint(namebuf, sizeof(namebuf), "\"%Z\"", &tmp); + pkg = gostringpkg; + } + sym = pkglookup(namebuf, pkg); + + // SymUniq flag indicates that data is generated already + if(sym->flags & SymUniq) + return sym; + sym->flags |= SymUniq; + + data(); + off = 0; + + // string header + off = dsymptr(sym, off, sym, widthptr+4); + off = duint32(sym, off, len); + + // string data + for(n=0; n len-n) + m = len-n; + off = dsname(sym, off, s+n, m); + } + off = duint8(sym, off, 0); // terminating NUL for runtime + off = (off+widthptr-1)&~(widthptr-1); // round to pointer alignment + ggloblsym(sym, off, 1); + text(); + + return sym; +} diff --git a/src/cmd/gc/pgen.c b/src/cmd/gc/pgen.c new file mode 100644 index 000000000..abe8ea892 --- /dev/null +++ b/src/cmd/gc/pgen.c @@ -0,0 +1,210 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +#include "gg.h" +#include "opt.h" + +static void compactframe(Prog* p); + +void +compile(Node *fn) +{ + Plist *pl; + Node nod1, *n; + Prog *ptxt; + int32 lno; + Type *t; + Iter save; + vlong oldstksize; + + if(newproc == N) { + newproc = sysfunc("newproc"); + deferproc = sysfunc("deferproc"); + deferreturn = sysfunc("deferreturn"); + panicindex = sysfunc("panicindex"); + panicslice = sysfunc("panicslice"); + throwreturn = sysfunc("throwreturn"); + } + + if(fn->nbody == nil) + return; + + saveerrors(); + + // set up domain for labels + clearlabels(); + + lno = setlineno(fn); + + curfn = fn; + dowidth(curfn->type); + + if(curfn->type->outnamed) { + // add clearing of the output parameters + t = structfirst(&save, getoutarg(curfn->type)); + while(t != T) { + if(t->nname != N) { + n = nod(OAS, t->nname, N); + typecheck(&n, Etop); + curfn->nbody = concat(list1(n), curfn->nbody); + } + t = structnext(&save); + } + } + + hasdefer = 0; + walk(curfn); + if(nerrors != 0) + goto ret; + + allocparams(); + + continpc = P; + breakpc = P; + + pl = newplist(); + pl->name = curfn->nname; + + setlineno(curfn); + + nodconst(&nod1, types[TINT32], 0); + ptxt = gins(ATEXT, isblank(curfn->nname) ? N : curfn->nname, &nod1); + afunclit(&ptxt->from); + + ginit(); + genlist(curfn->enter); + + retpc = nil; + if(hasdefer || curfn->exit) { + Prog *p1; + + p1 = gjmp(nil); + retpc = gjmp(nil); + patch(p1, pc); + } + + genlist(curfn->nbody); + gclean(); + checklabels(); + if(nerrors != 0) + goto ret; + if(curfn->endlineno) + lineno = curfn->endlineno; + + if(curfn->type->outtuple != 0) + ginscall(throwreturn, 0); + + if(retpc) + patch(retpc, pc); + ginit(); + if(hasdefer) + ginscall(deferreturn, 0); + if(curfn->exit) + genlist(curfn->exit); + gclean(); + if(nerrors != 0) + goto ret; + pc->as = ARET; // overwrite AEND + pc->lineno = lineno; + + if(!debug['N'] || debug['R'] || debug['P']) { + regopt(ptxt); + } + + oldstksize = stksize; + compactframe(ptxt); + if(0) + print("compactframe: %ld to %ld\n", oldstksize, stksize); + + defframe(ptxt); + + if(0) + frame(0); + +ret: + lineno = lno; +} + + +// Sort the list of stack variables. autos after anything else, +// within autos, unused after used, and within used on reverse alignment. +// non-autos sort on offset. +static int +cmpstackvar(Node *a, Node *b) +{ + if (a->class != b->class) + return (a->class == PAUTO) ? 1 : -1; + if (a->class != PAUTO) + return a->xoffset - b->xoffset; + if ((a->used == 0) != (b->used == 0)) + return b->used - a->used; + return b->type->align - a->type->align; + +} + +// TODO(lvd) find out where the PAUTO/OLITERAL nodes come from. +static void +compactframe(Prog* ptxt) +{ + NodeList *ll; + Node* n; + vlong w; + + if (stksize == 0) + return; + + // Mark the PAUTO's unused. + for(ll=curfn->dcl; ll != nil; ll=ll->next) + if (ll->n->class == PAUTO) + ll->n->used = 0; + + markautoused(ptxt); + + listsort(&curfn->dcl, cmpstackvar); + + // Unused autos are at the end, chop 'em off. + ll = curfn->dcl; + n = ll->n; + if (n->class == PAUTO && n->op == ONAME && !n->used) { + curfn->dcl = nil; + stksize = 0; + return; + } + + for(ll = curfn->dcl; ll->next != nil; ll=ll->next) { + n = ll->next->n; + if (n->class == PAUTO && n->op == ONAME && !n->used) { + ll->next = nil; + curfn->dcl->end = ll; + break; + } + } + + // Reassign stack offsets of the locals that are still there. + stksize = 0; + for(ll = curfn->dcl; ll != nil; ll=ll->next) { + n = ll->n; + if (n->class != PAUTO || n->op != ONAME) + continue; + + w = n->type->width; + if(w >= MAXWIDTH || w < 0) + fatal("bad width"); + stksize += w; + stksize = rnd(stksize, n->type->align); + if(thechar == '5') + stksize = rnd(stksize, widthptr); + n->stkdelta = -stksize - n->xoffset; + } + + fixautoused(ptxt); + + // The debug information needs accurate offsets on the symbols. + for(ll = curfn->dcl ;ll != nil; ll=ll->next) { + if (ll->n->class != PAUTO || ll->n->op != ONAME) + continue; + ll->n->xoffset += ll->n->stkdelta; + ll->n->stkdelta = 0; + } +} diff --git a/src/cmd/gc/print.c b/src/cmd/gc/print.c new file mode 100644 index 000000000..5913e848a --- /dev/null +++ b/src/cmd/gc/print.c @@ -0,0 +1,480 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +#include "go.h" + +enum +{ + PFIXME = 0, +}; + +void +exprlistfmt(Fmt *f, NodeList *l) +{ + for(; l; l=l->next) { + exprfmt(f, l->n, 0); + if(l->next) + fmtprint(f, ", "); + } +} + +void +exprfmt(Fmt *f, Node *n, int prec) +{ + int nprec; + char *p; + + nprec = 0; + if(n == nil) { + fmtprint(f, ""); + return; + } + + if(n->implicit) { + exprfmt(f, n->left, prec); + return; + } + + switch(n->op) { + case OAPPEND: + case ONAME: + case ONONAME: + case OPACK: + case OLITERAL: + case ODOT: + case ODOTPTR: + case ODOTINTER: + case ODOTMETH: + case ODOTTYPE: + case ODOTTYPE2: + case OXDOT: + case OARRAYBYTESTR: + case OCAP: + case OCLOSE: + case OCOPY: + case OLEN: + case OMAKE: + case ONEW: + case OPANIC: + case OPRINT: + case OPRINTN: + case OCALL: + case OCALLMETH: + case OCALLINTER: + case OCALLFUNC: + case OCONV: + case OCONVNOP: + case OMAKESLICE: + case ORUNESTR: + case OADDR: + case OCOM: + case OIND: + case OMINUS: + case ONOT: + case OPLUS: + case ORECV: + case OCONVIFACE: + case OTPAREN: + case OINDEX: + case OINDEXMAP: + case OPAREN: + nprec = 7; + break; + + case OMUL: + case ODIV: + case OMOD: + case OLSH: + case ORSH: + case OAND: + case OANDNOT: + nprec = 6; + break; + + case OADD: + case OSUB: + case OOR: + case OXOR: + nprec = 5; + break; + + case OEQ: + case OLT: + case OLE: + case OGE: + case OGT: + case ONE: + nprec = 4; + break; + + case OSEND: + nprec = 3; + break; + + case OANDAND: + nprec = 2; + break; + + case OOROR: + nprec = 1; + break; + + case OTYPE: + if(n->sym != S) + nprec = 7; + break; + } + + if(prec > nprec) + fmtprint(f, "("); + + switch(n->op) { + default: + bad: + fmtprint(f, "(node %O)", n->op); + break; + + case OPAREN: + fmtprint(f, "(%#N)", n->left); + break; + + case OREGISTER: + fmtprint(f, "%R", n->val.u.reg); + break; + + case OLITERAL: + if(n->sym != S) { + fmtprint(f, "%S", n->sym); + break; + } + switch(n->val.ctype) { + default: + goto bad; + case CTINT: + fmtprint(f, "%B", n->val.u.xval); + break; + case CTBOOL: + if(n->val.u.bval) + fmtprint(f, "true"); + else + fmtprint(f, "false"); + break; + case CTCPLX: + fmtprint(f, "%.17g+%.17gi", + mpgetflt(&n->val.u.cval->real), + mpgetflt(&n->val.u.cval->imag)); + break; + case CTFLT: + fmtprint(f, "%.17g", mpgetflt(n->val.u.fval)); + break; + case CTSTR: + fmtprint(f, "\"%Z\"", n->val.u.sval); + break; + case CTNIL: + fmtprint(f, "nil"); + break; + } + break; + + case ONAME: + case OPACK: + case ONONAME: + fmtprint(f, "%S", n->sym); + break; + + case OTYPE: + if(n->type == T && n->sym != S) { + fmtprint(f, "%S", n->sym); + break; + } + fmtprint(f, "%T", n->type); + break; + + case OTARRAY: + fmtprint(f, "[]"); + exprfmt(f, n->left, PFIXME); + break; + + case OTPAREN: + fmtprint(f, "("); + exprfmt(f, n->left, 0); + fmtprint(f, ")"); + break; + + case OTMAP: + fmtprint(f, "map["); + exprfmt(f, n->left, 0); + fmtprint(f, "] "); + exprfmt(f, n->right, 0); + break; + + case OTCHAN: + if(n->etype == Crecv) + fmtprint(f, "<-"); + fmtprint(f, "chan"); + if(n->etype == Csend) { + fmtprint(f, "<- "); + exprfmt(f, n->left, 0); + } else { + fmtprint(f, " "); + if(n->left->op == OTCHAN && n->left->sym == S && n->left->etype == Crecv) { + fmtprint(f, "("); + exprfmt(f, n->left, 0); + fmtprint(f, ")"); + } else + exprfmt(f, n->left, 0); + } + break; + + case OTSTRUCT: + fmtprint(f, ""); + break; + + case OTINTER: + fmtprint(f, ""); + break; + + case OTFUNC: + fmtprint(f, ""); + break; + + case OAS: + exprfmt(f, n->left, 0); + fmtprint(f, " = "); + exprfmt(f, n->right, 0); + break; + + case OASOP: + exprfmt(f, n->left, 0); + fmtprint(f, " %#O= ", n->etype); + exprfmt(f, n->right, 0); + break; + + case OAS2: + case OAS2DOTTYPE: + case OAS2FUNC: + case OAS2MAPR: + case OAS2MAPW: + case OAS2RECV: + exprlistfmt(f, n->list); + fmtprint(f, " = "); + exprlistfmt(f, n->rlist); + break; + + case OADD: + case OANDAND: + case OANDNOT: + case ODIV: + case OEQ: + case OGE: + case OGT: + case OLE: + case OLT: + case OLSH: + case OMOD: + case OMUL: + case ONE: + case OOR: + case OOROR: + case ORSH: + case OSEND: + case OSUB: + case OXOR: + exprfmt(f, n->left, nprec); + fmtprint(f, " %#O ", n->op); + exprfmt(f, n->right, nprec+1); + break; + + case OADDR: + case OCOM: + case OIND: + case OMINUS: + case ONOT: + case OPLUS: + case ORECV: + fmtprint(f, "%#O", n->op); + if((n->op == OMINUS || n->op == OPLUS) && n->left->op == n->op) + fmtprint(f, " "); + exprfmt(f, n->left, 0); + break; + + case OCLOSURE: + fmtprint(f, "func literal"); + break; + + case OCOMPLIT: + fmtprint(f, "composite literal"); + break; + + case OARRAYLIT: + if(isslice(n->type)) + fmtprint(f, "slice literal"); + else + fmtprint(f, "array literal"); + break; + + case OMAPLIT: + fmtprint(f, "map literal"); + break; + + case OSTRUCTLIT: + fmtprint(f, "struct literal"); + break; + + case OXDOT: + case ODOT: + case ODOTPTR: + case ODOTINTER: + case ODOTMETH: + exprfmt(f, n->left, 7); + if(n->right == N || n->right->sym == S) + fmtprint(f, "."); + else { + // skip leading type· in method name + p = utfrrune(n->right->sym->name, 0xb7); + if(p) + p+=2; + else + p = n->right->sym->name; + fmtprint(f, ".%s", p); + } + break; + + case ODOTTYPE: + case ODOTTYPE2: + exprfmt(f, n->left, 7); + fmtprint(f, ".("); + if(n->right != N) + exprfmt(f, n->right, 0); + else + fmtprint(f, "%T", n->type); + fmtprint(f, ")"); + break; + + case OINDEX: + case OINDEXMAP: + exprfmt(f, n->left, 7); + fmtprint(f, "["); + exprfmt(f, n->right, 0); + fmtprint(f, "]"); + break; + + case OSLICE: + case OSLICESTR: + case OSLICEARR: + exprfmt(f, n->left, 7); + fmtprint(f, "["); + if(n->right->left != N) + exprfmt(f, n->right->left, 0); + fmtprint(f, ":"); + if(n->right->right != N) + exprfmt(f, n->right->right, 0); + fmtprint(f, "]"); + break; + + case OCALL: + case OCALLFUNC: + case OCALLINTER: + case OCALLMETH: + exprfmt(f, n->left, 7); + fmtprint(f, "("); + exprlistfmt(f, n->list); + if(n->isddd) + fmtprint(f, "..."); + fmtprint(f, ")"); + break; + + case OCOMPLEX: + fmtprint(f, "complex("); + exprfmt(f, n->left, 0); + fmtprint(f, ", "); + exprfmt(f, n->right, 0); + fmtprint(f, ")"); + break; + + case OREAL: + fmtprint(f, "real("); + exprfmt(f, n->left, 0); + fmtprint(f, ")"); + break; + + case OIMAG: + fmtprint(f, "imag("); + exprfmt(f, n->left, 0); + fmtprint(f, ")"); + break; + + case OCONV: + case OCONVIFACE: + case OCONVNOP: + case OARRAYBYTESTR: + case OSTRARRAYBYTE: + case ORUNESTR: + if(n->type == T || n->type->sym == S) + fmtprint(f, "(%T)(", n->type); + else + fmtprint(f, "%T(", n->type); + if(n->left == N) + exprlistfmt(f, n->list); + else + exprfmt(f, n->left, 0); + fmtprint(f, ")"); + break; + + case OAPPEND: + case OCAP: + case OCLOSE: + case OLEN: + case OCOPY: + case OMAKE: + case ONEW: + case OPANIC: + case OPRINT: + case OPRINTN: + fmtprint(f, "%#O(", n->op); + if(n->left) + exprfmt(f, n->left, 0); + else + exprlistfmt(f, n->list); + fmtprint(f, ")"); + break; + + case OMAKESLICE: + fmtprint(f, "make(%#T, ", n->type); + exprfmt(f, n->left, 0); + if(count(n->list) > 2) { + fmtprint(f, ", "); + exprfmt(f, n->right, 0); + } + fmtprint(f, ")"); + break; + + case OMAKEMAP: + case OMAKECHAN: + fmtprint(f, "make(%#T)", n->type); + break; + + // Some statements + + case ODCL: + fmtprint(f, "var %S %#T", n->left->sym, n->left->type); + break; + + case ORETURN: + fmtprint(f, "return "); + exprlistfmt(f, n->list); + break; + + case OPROC: + fmtprint(f, "go %#N", n->left); + break; + + case ODEFER: + fmtprint(f, "defer %#N", n->left); + break; + } + + if(prec > nprec) + fmtprint(f, ")"); +} diff --git a/src/cmd/gc/range.c b/src/cmd/gc/range.c new file mode 100644 index 000000000..5ce693ae3 --- /dev/null +++ b/src/cmd/gc/range.c @@ -0,0 +1,256 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +/* + * range + */ + +#include "go.h" + +void +typecheckrange(Node *n) +{ + char *why; + Type *t, *t1, *t2; + Node *v1, *v2; + NodeList *ll; + + // delicate little dance. see typecheckas2 + for(ll=n->list; ll; ll=ll->next) + if(ll->n->defn != n) + typecheck(&ll->n, Erv | Easgn); + + typecheck(&n->right, Erv); + if((t = n->right->type) == T) + goto out; + if(isptr[t->etype] && isfixedarray(t->type)) + t = t->type; + n->type = t; + + switch(t->etype) { + default: + yyerror("cannot range over %+N", n->right); + goto out; + + case TARRAY: + t1 = types[TINT]; + t2 = t->type; + break; + + case TMAP: + t1 = t->down; + t2 = t->type; + break; + + case TCHAN: + t1 = t->type; + t2 = nil; + if(count(n->list) == 2) + goto toomany; + break; + + case TSTRING: + t1 = types[TINT]; + t2 = types[TINT]; + break; + } + + if(count(n->list) > 2) { + toomany: + yyerror("too many variables in range"); + } + + v1 = n->list->n; + v2 = N; + if(n->list->next) + v2 = n->list->next->n; + + if(v1->defn == n) + v1->type = t1; + else if(v1->type != T && assignop(t1, v1->type, &why) == 0) + yyerror("cannot assign type %T to %+N in range%s", t1, v1, why); + if(v2) { + if(v2->defn == n) + v2->type = t2; + else if(v2->type != T && assignop(t2, v2->type, &why) == 0) + yyerror("cannot assign type %T to %+N in range%s", t2, v2, why); + } + +out: + typechecklist(n->nbody, Etop); + + // second half of dance + n->typecheck = 1; + for(ll=n->list; ll; ll=ll->next) + if(ll->n->typecheck == 0) + typecheck(&ll->n, Erv | Easgn); +} + +void +walkrange(Node *n) +{ + Node *ohv1, *hv1, *hv2; // hidden (old) val 1, 2 + Node *ha, *hit; // hidden aggregate, iterator + Node *hn, *hp; // hidden len, pointer + Node *hb; // hidden bool + Node *a, *v1, *v2; // not hidden aggregate, val 1, 2 + Node *fn, *tmp; + NodeList *body, *init; + Type *th, *t; + int lno; + + t = n->type; + init = nil; + + a = n->right; + lno = setlineno(a); + if(t->etype == TSTRING && !eqtype(t, types[TSTRING])) { + a = nod(OCONV, n->right, N); + a->type = types[TSTRING]; + } + + v1 = n->list->n; + hv1 = N; + + v2 = N; + if(n->list->next) + v2 = n->list->next->n; + hv2 = N; + + if(v2 == N && t->etype == TARRAY) { + // will have just one reference to argument. + // no need to make a potentially expensive copy. + ha = a; + } else { + ha = nod(OXXX, N, N); + tempname(ha, a->type); + init = list(init, nod(OAS, ha, a)); + } + + switch(t->etype) { + default: + fatal("walkrange"); + + case TARRAY: + hv1 = nod(OXXX, N, n); + tempname(hv1, types[TINT]); + hn = nod(OXXX, N, N); + tempname(hn, types[TINT]); + hp = nil; + + init = list(init, nod(OAS, hv1, N)); + init = list(init, nod(OAS, hn, nod(OLEN, ha, N))); + if(v2) { + hp = nod(OXXX, N, N); + tempname(hp, ptrto(n->type->type)); + tmp = nod(OINDEX, ha, nodintconst(0)); + tmp->etype = 1; // no bounds check + init = list(init, nod(OAS, hp, nod(OADDR, tmp, N))); + } + + n->ntest = nod(OLT, hv1, hn); + n->nincr = nod(OASOP, hv1, nodintconst(1)); + n->nincr->etype = OADD; + body = list1(nod(OAS, v1, hv1)); + if(v2) { + body = list(body, nod(OAS, v2, nod(OIND, hp, N))); + tmp = nod(OADD, hp, nodintconst(t->type->width)); + tmp->type = hp->type; + tmp->typecheck = 1; + tmp->right->type = types[tptr]; + tmp->right->typecheck = 1; + body = list(body, nod(OAS, hp, tmp)); + } + break; + + case TMAP: + th = typ(TARRAY); + th->type = ptrto(types[TUINT8]); + th->bound = (sizeof(struct Hiter) + widthptr - 1) / widthptr; + hit = nod(OXXX, N, N); + tempname(hit, th); + + fn = syslook("mapiterinit", 1); + argtype(fn, t->down); + argtype(fn, t->type); + argtype(fn, th); + init = list(init, mkcall1(fn, T, nil, typename(t), ha, nod(OADDR, hit, N))); + n->ntest = nod(ONE, nod(OINDEX, hit, nodintconst(0)), nodnil()); + + fn = syslook("mapiternext", 1); + argtype(fn, th); + n->nincr = mkcall1(fn, T, nil, nod(OADDR, hit, N)); + + if(v2 == N) { + fn = syslook("mapiter1", 1); + argtype(fn, th); + argtype(fn, t->down); + a = nod(OAS, v1, mkcall1(fn, t->down, nil, nod(OADDR, hit, N))); + } else { + fn = syslook("mapiter2", 1); + argtype(fn, th); + argtype(fn, t->down); + argtype(fn, t->type); + a = nod(OAS2, N, N); + a->list = list(list1(v1), v2); + a->rlist = list1(mkcall1(fn, getoutargx(fn->type), nil, nod(OADDR, hit, N))); + } + body = list1(a); + break; + + case TCHAN: + hv1 = nod(OXXX, N, n); + tempname(hv1, t->type); + hb = nod(OXXX, N, N); + tempname(hb, types[TBOOL]); + + n->ntest = nod(ONE, hb, nodbool(0)); + a = nod(OAS2RECV, N, N); + a->typecheck = 1; + a->list = list(list1(hv1), hb); + a->rlist = list1(nod(ORECV, ha, N)); + n->ntest->ninit = list1(a); + body = list1(nod(OAS, v1, hv1)); + break; + + case TSTRING: + ohv1 = nod(OXXX, N, N); + tempname(ohv1, types[TINT]); + + hv1 = nod(OXXX, N, N); + tempname(hv1, types[TINT]); + init = list(init, nod(OAS, hv1, N)); + + if(v2 == N) + a = nod(OAS, hv1, mkcall("stringiter", types[TINT], nil, ha, hv1)); + else { + hv2 = nod(OXXX, N, N); + tempname(hv2, types[TINT]); + a = nod(OAS2, N, N); + a->list = list(list1(hv1), hv2); + fn = syslook("stringiter2", 0); + a->rlist = list1(mkcall1(fn, getoutargx(fn->type), nil, ha, hv1)); + } + n->ntest = nod(ONE, hv1, nodintconst(0)); + n->ntest->ninit = list(list1(nod(OAS, ohv1, hv1)), a); + + body = list1(nod(OAS, v1, ohv1)); + if(v2 != N) + body = list(body, nod(OAS, v2, hv2)); + break; + } + + n->op = OFOR; + typechecklist(init, Etop); + n->ninit = concat(n->ninit, init); + typechecklist(n->ntest->ninit, Etop); + typecheck(&n->ntest, Erv); + typecheck(&n->nincr, Etop); + typechecklist(body, Etop); + n->nbody = concat(body, n->nbody); + walkstmt(&n); + + lineno = lno; +} + diff --git a/src/cmd/gc/reflect.c b/src/cmd/gc/reflect.c new file mode 100644 index 000000000..810787d30 --- /dev/null +++ b/src/cmd/gc/reflect.c @@ -0,0 +1,939 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +#include "go.h" + +/* + * runtime interface and reflection data structures + */ + +static NodeList* signatlist; +static Sym* dtypesym(Type*); +static Sym* weaktypesym(Type*); + +static int +sigcmp(Sig *a, Sig *b) +{ + int i; + + i = strcmp(a->name, b->name); + if(i != 0) + return i; + if(a->pkg == b->pkg) + return 0; + if(a->pkg == nil) + return -1; + if(b->pkg == nil) + return +1; + return strcmp(a->pkg->path->s, b->pkg->path->s); +} + +static Sig* +lsort(Sig *l, int(*f)(Sig*, Sig*)) +{ + Sig *l1, *l2, *le; + + if(l == 0 || l->link == 0) + return l; + + l1 = l; + l2 = l; + for(;;) { + l2 = l2->link; + if(l2 == 0) + break; + l2 = l2->link; + if(l2 == 0) + break; + l1 = l1->link; + } + + l2 = l1->link; + l1->link = 0; + l1 = lsort(l, f); + l2 = lsort(l2, f); + + /* set up lead element */ + if((*f)(l1, l2) < 0) { + l = l1; + l1 = l1->link; + } else { + l = l2; + l2 = l2->link; + } + le = l; + + for(;;) { + if(l1 == 0) { + while(l2) { + le->link = l2; + le = l2; + l2 = l2->link; + } + le->link = 0; + break; + } + if(l2 == 0) { + while(l1) { + le->link = l1; + le = l1; + l1 = l1->link; + } + break; + } + if((*f)(l1, l2) < 0) { + le->link = l1; + le = l1; + l1 = l1->link; + } else { + le->link = l2; + le = l2; + l2 = l2->link; + } + } + le->link = 0; + return l; +} + +/* + * f is method type, with receiver. + * return function type, receiver as first argument (or not). + */ +Type* +methodfunc(Type *f, Type *receiver) +{ + NodeList *in, *out; + Node *d; + Type *t; + + in = nil; + if(receiver) { + d = nod(ODCLFIELD, N, N); + d->type = receiver; + in = list(in, d); + } + for(t=getinargx(f)->type; t; t=t->down) { + d = nod(ODCLFIELD, N, N); + d->type = t->type; + d->isddd = t->isddd; + in = list(in, d); + } + + out = nil; + for(t=getoutargx(f)->type; t; t=t->down) { + d = nod(ODCLFIELD, N, N); + d->type = t->type; + out = list(out, d); + } + + return functype(N, in, out); +} + +/* + * return methods of non-interface type t, sorted by name. + * generates stub functions as needed. + */ +static Sig* +methods(Type *t) +{ + Type *f, *mt, *it, *this; + Sig *a, *b; + Sym *method; + Prog *oldlist; + + // named method type + mt = methtype(t); + if(mt == T) + return nil; + expandmeth(mt->sym, mt); + + // type stored in interface word + it = t; + if(it->width > widthptr) + it = ptrto(t); + + // make list of methods for t, + // generating code if necessary. + a = nil; + oldlist = nil; + for(f=mt->xmethod; f; f=f->down) { + if(f->type->etype != TFUNC) + continue; + if(f->etype != TFIELD) + fatal("methods: not field"); + method = f->sym; + if(method == nil) + continue; + + // get receiver type for this particular method. + // if pointer receiver but non-pointer t and + // this is not an embedded pointer inside a struct, + // method does not apply. + this = getthisx(f->type)->type->type; + if(isptr[this->etype] && this->type == t) + continue; + if(isptr[this->etype] && !isptr[t->etype] + && f->embedded != 2 && !isifacemethod(f->type)) + continue; + + b = mal(sizeof(*b)); + b->link = a; + a = b; + + a->name = method->name; + if(!exportname(method->name)) { + if(method->pkg == nil) + fatal("methods: missing package"); + a->pkg = method->pkg; + } + a->isym = methodsym(method, it, 1); + a->tsym = methodsym(method, t, 0); + a->type = methodfunc(f->type, t); + a->mtype = methodfunc(f->type, nil); + + if(!(a->isym->flags & SymSiggen)) { + a->isym->flags |= SymSiggen; + if(!eqtype(this, it) || this->width < types[tptr]->width) { + if(oldlist == nil) + oldlist = pc; + // Is okay to call genwrapper here always, + // but we can generate more efficient code + // using genembedtramp if all that is necessary + // is a pointer adjustment and a JMP. + if(isptr[it->etype] && isptr[this->etype] + && f->embedded && !isifacemethod(f->type)) + genembedtramp(it, f, a->isym, 1); + else + genwrapper(it, f, a->isym, 1); + } + } + + if(!(a->tsym->flags & SymSiggen)) { + a->tsym->flags |= SymSiggen; + if(!eqtype(this, t)) { + if(oldlist == nil) + oldlist = pc; + if(isptr[t->etype] && isptr[this->etype] + && f->embedded && !isifacemethod(f->type)) + genembedtramp(t, f, a->tsym, 0); + else + genwrapper(t, f, a->tsym, 0); + } + } + } + + // restore data output + if(oldlist) { + // old list ended with AEND; change to ANOP + // so that the trampolines that follow can be found. + nopout(oldlist); + + // start new data list + newplist(); + } + + return lsort(a, sigcmp); +} + +/* + * return methods of interface type t, sorted by name. + */ +static Sig* +imethods(Type *t) +{ + Sig *a, *all, *last; + Type *f; + Sym *method, *isym; + Prog *oldlist; + + all = nil; + last = nil; + oldlist = nil; + for(f=t->type; f; f=f->down) { + if(f->etype != TFIELD) + fatal("imethods: not field"); + if(f->type->etype != TFUNC || f->sym == nil) + continue; + method = f->sym; + a = mal(sizeof(*a)); + a->name = method->name; + if(!exportname(method->name)) { + if(method->pkg == nil) + fatal("imethods: missing package"); + a->pkg = method->pkg; + } + a->mtype = f->type; + a->offset = 0; + a->type = methodfunc(f->type, nil); + + if(last && sigcmp(last, a) >= 0) + fatal("sigcmp vs sortinter %s %s", last->name, a->name); + if(last == nil) + all = a; + else + last->link = a; + last = a; + + // Compiler can only refer to wrappers for + // named interface types. + if(t->sym == S) + continue; + + // NOTE(rsc): Perhaps an oversight that + // IfaceType.Method is not in the reflect data. + // Generate the method body, so that compiled + // code can refer to it. + isym = methodsym(method, t, 0); + if(!(isym->flags & SymSiggen)) { + isym->flags |= SymSiggen; + if(oldlist == nil) + oldlist = pc; + genwrapper(t, f, isym, 0); + } + } + + if(oldlist) { + // old list ended with AEND; change to ANOP + // so that the trampolines that follow can be found. + nopout(oldlist); + + // start new data list + newplist(); + } + + return all; +} + +static void +dimportpath(Pkg *p) +{ + static Pkg *gopkg; + char *nam; + Node *n; + + if(p->pathsym != S) + return; + + if(gopkg == nil) { + gopkg = mkpkg(strlit("go")); + gopkg->name = "go"; + } + nam = smprint("importpath.%s.", p->prefix); + + n = nod(ONAME, N, N); + n->sym = pkglookup(nam, gopkg); + free(nam); + n->class = PEXTERN; + n->xoffset = 0; + p->pathsym = n->sym; + + gdatastring(n, p->path); + ggloblsym(n->sym, types[TSTRING]->width, 1); +} + +static int +dgopkgpath(Sym *s, int ot, Pkg *pkg) +{ + if(pkg == nil) + return dgostringptr(s, ot, nil); + + // Emit reference to go.importpath.""., which 6l will + // rewrite using the correct import path. Every package + // that imports this one directly defines the symbol. + if(pkg == localpkg) { + static Sym *ns; + + if(ns == nil) + ns = pkglookup("importpath.\"\".", mkpkg(strlit("go"))); + return dsymptr(s, ot, ns, 0); + } + + dimportpath(pkg); + return dsymptr(s, ot, pkg->pathsym, 0); +} + +/* + * uncommonType + * ../../pkg/runtime/type.go:/uncommonType + */ +static int +dextratype(Sym *sym, int off, Type *t, int ptroff) +{ + int ot, n; + Sym *s; + Sig *a, *m; + + m = methods(t); + if(t->sym == nil && m == nil) + return off; + + // fill in *extraType pointer in header + dsymptr(sym, ptroff, sym, off); + + n = 0; + for(a=m; a; a=a->link) { + dtypesym(a->type); + n++; + } + + ot = off; + s = sym; + if(t->sym) { + ot = dgostringptr(s, ot, t->sym->name); + if(t != types[t->etype]) + ot = dgopkgpath(s, ot, t->sym->pkg); + else + ot = dgostringptr(s, ot, nil); + } else { + ot = dgostringptr(s, ot, nil); + ot = dgostringptr(s, ot, nil); + } + + // slice header + ot = dsymptr(s, ot, s, ot + widthptr + 2*4); + ot = duint32(s, ot, n); + ot = duint32(s, ot, n); + + // methods + for(a=m; a; a=a->link) { + // method + // ../../pkg/runtime/type.go:/method + ot = dgostringptr(s, ot, a->name); + ot = dgopkgpath(s, ot, a->pkg); + ot = dsymptr(s, ot, dtypesym(a->mtype), 0); + ot = dsymptr(s, ot, dtypesym(a->type), 0); + if(a->isym) + ot = dsymptr(s, ot, a->isym, 0); + else + ot = duintptr(s, ot, 0); + if(a->tsym) + ot = dsymptr(s, ot, a->tsym, 0); + else + ot = duintptr(s, ot, 0); + } + + return ot; +} + +enum { + KindBool = 1, + KindInt, + KindInt8, + KindInt16, + KindInt32, + KindInt64, + KindUint, + KindUint8, + KindUint16, + KindUint32, + KindUint64, + KindUintptr, + KindFloat32, + KindFloat64, + KindComplex64, + KindComplex128, + KindArray, + KindChan, + KindFunc, + KindInterface, + KindMap, + KindPtr, + KindSlice, + KindString, + KindStruct, + KindUnsafePointer, + + KindNoPointers = 1<<7, +}; + +static int +kinds[] = +{ + [TINT] = KindInt, + [TUINT] = KindUint, + [TINT8] = KindInt8, + [TUINT8] = KindUint8, + [TINT16] = KindInt16, + [TUINT16] = KindUint16, + [TINT32] = KindInt32, + [TUINT32] = KindUint32, + [TINT64] = KindInt64, + [TUINT64] = KindUint64, + [TUINTPTR] = KindUintptr, + [TFLOAT32] = KindFloat32, + [TFLOAT64] = KindFloat64, + [TBOOL] = KindBool, + [TSTRING] = KindString, + [TPTR32] = KindPtr, + [TPTR64] = KindPtr, + [TSTRUCT] = KindStruct, + [TINTER] = KindInterface, + [TCHAN] = KindChan, + [TMAP] = KindMap, + [TARRAY] = KindArray, + [TFUNC] = KindFunc, + [TCOMPLEX64] = KindComplex64, + [TCOMPLEX128] = KindComplex128, + [TUNSAFEPTR] = KindUnsafePointer, +}; + +static char* +structnames[] = +{ + [TINT] = "*runtime.IntType", + [TUINT] = "*runtime.UintType", + [TINT8] = "*runtime.IntType", + [TUINT8] = "*runtime.UintType", + [TINT16] = "*runtime.IntType", + [TUINT16] = "*runtime.UintType", + [TINT32] = "*runtime.IntType", + [TUINT32] = "*runtime.UintType", + [TINT64] = "*runtime.IntType", + [TUINT64] = "*runtime.UintType", + [TUINTPTR] = "*runtime.UintType", + [TCOMPLEX64] = "*runtime.ComplexType", + [TCOMPLEX128] = "*runtime.ComplexType", + [TFLOAT32] = "*runtime.FloatType", + [TFLOAT64] = "*runtime.FloatType", + [TBOOL] = "*runtime.BoolType", + [TSTRING] = "*runtime.StringType", + [TUNSAFEPTR] = "*runtime.UnsafePointerType", + + [TPTR32] = "*runtime.PtrType", + [TPTR64] = "*runtime.PtrType", + [TSTRUCT] = "*runtime.StructType", + [TINTER] = "*runtime.InterfaceType", + [TCHAN] = "*runtime.ChanType", + [TMAP] = "*runtime.MapType", + [TARRAY] = "*runtime.ArrayType", + [TFUNC] = "*runtime.FuncType", +}; + +static Sym* +typestruct(Type *t) +{ + char *name; + int et; + + et = t->etype; + if(et < 0 || et >= nelem(structnames) || (name = structnames[et]) == nil) { + fatal("typestruct %lT", t); + return nil; // silence gcc + } + + if(isslice(t)) + name = "*runtime.SliceType"; + + return pkglookup(name, typepkg); +} + +static int +haspointers(Type *t) +{ + Type *t1; + + switch(t->etype) { + case TINT: + case TUINT: + case TINT8: + case TUINT8: + case TINT16: + case TUINT16: + case TINT32: + case TUINT32: + case TINT64: + case TUINT64: + case TUINTPTR: + case TFLOAT32: + case TFLOAT64: + case TBOOL: + return 0; + case TARRAY: + if(t->bound < 0) // slice + return 1; + return haspointers(t->type); + case TSTRUCT: + for(t1=t->type; t1!=T; t1=t1->down) + if(haspointers(t1->type)) + return 1; + return 0; + case TSTRING: + case TPTR32: + case TPTR64: + case TUNSAFEPTR: + case TINTER: + case TCHAN: + case TMAP: + case TFUNC: + default: + return 1; + } +} + +/* + * commonType + * ../../pkg/runtime/type.go:/commonType + */ +static int +dcommontype(Sym *s, int ot, Type *t) +{ + int i; + Sym *sptr; + char *p; + + dowidth(t); + + sptr = nil; + if(t->sym != nil && !isptr[t->etype]) + sptr = dtypesym(ptrto(t)); + else + sptr = weaktypesym(ptrto(t)); + + // empty interface pointing at this type. + // all the references that we emit are *interface{}; + // they point here. + ot = rnd(ot, widthptr); + ot = dsymptr(s, ot, typestruct(t), 0); + ot = dsymptr(s, ot, s, 2*widthptr); + + // ../../pkg/runtime/type.go:/commonType + // actual type structure + // type commonType struct { + // size uintptr; + // hash uint32; + // alg uint8; + // align uint8; + // fieldAlign uint8; + // kind uint8; + // string *string; + // *extraType; + // ptrToThis *Type + // } + ot = duintptr(s, ot, t->width); + ot = duint32(s, ot, typehash(t)); + ot = duint8(s, ot, algtype(t)); + ot = duint8(s, ot, t->align); // align + ot = duint8(s, ot, t->align); // fieldAlign + i = kinds[t->etype]; + if(t->etype == TARRAY && t->bound < 0) + i = KindSlice; + if(!haspointers(t)) + i |= KindNoPointers; + ot = duint8(s, ot, i); // kind + longsymnames = 1; + p = smprint("%-T", t); + longsymnames = 0; + ot = dgostringptr(s, ot, p); // string + free(p); + + // skip pointer to extraType, + // which follows the rest of this type structure. + // caller will fill in if needed. + // otherwise linker will assume 0. + ot += widthptr; + + ot = dsymptr(s, ot, sptr, 0); // ptrto type + return ot; +} + +Sym* +typesym(Type *t) +{ + char *p; + Sym *s; + + p = smprint("%#-T", t); + s = pkglookup(p, typepkg); + free(p); + return s; +} + +Node* +typename(Type *t) +{ + Sym *s; + Node *n; + + if(t == T || (isptr[t->etype] && t->type == T) || isideal(t)) + fatal("typename %T", t); + s = typesym(t); + if(s->def == N) { + n = nod(ONAME, N, N); + n->sym = s; + n->type = types[TUINT8]; + n->addable = 1; + n->ullman = 1; + n->class = PEXTERN; + n->xoffset = 0; + s->def = n; + + signatlist = list(signatlist, typenod(t)); + } + + n = nod(OADDR, s->def, N); + n->type = ptrto(s->def->type); + n->addable = 1; + n->ullman = 2; + return n; +} + +static Sym* +weaktypesym(Type *t) +{ + char *p; + Sym *s; + static Pkg *weak; + + if(weak == nil) { + weak = mkpkg(strlit("weak.type")); + weak->name = "weak.type"; + weak->prefix = "weak.type"; // not weak%2etype + } + + p = smprint("%#-T", t); + s = pkglookup(p, weak); + free(p); + return s; +} + +static Sym* +dtypesym(Type *t) +{ + int ot, xt, n, isddd, dupok; + Sym *s, *s1, *s2; + Sig *a, *m; + Type *t1, *tbase, *t2; + + if(isideal(t)) + fatal("dtypesym %T", t); + + s = typesym(t); + if(s->flags & SymSiggen) + return s; + s->flags |= SymSiggen; + + // special case (look for runtime below): + // when compiling package runtime, + // emit the type structures for int, float, etc. + tbase = t; + if(isptr[t->etype] && t->sym == S && t->type->sym != S) + tbase = t->type; + dupok = tbase->sym == S; + + if(compiling_runtime && tbase == types[tbase->etype]) // int, float, etc + goto ok; + + // named types from other files are defined only by those files + if(tbase->sym && !tbase->local) + return s; + if(isforw[tbase->etype]) + return s; + +ok: + ot = 0; + xt = 0; + switch(t->etype) { + default: + ot = dcommontype(s, ot, t); + xt = ot - 2*widthptr; + break; + + case TARRAY: + if(t->bound >= 0) { + // ../../pkg/runtime/type.go:/ArrayType + s1 = dtypesym(t->type); + t2 = typ(TARRAY); + t2->type = t->type; + t2->bound = -1; // slice + s2 = dtypesym(t2); + ot = dcommontype(s, ot, t); + xt = ot - 2*widthptr; + ot = dsymptr(s, ot, s1, 0); + ot = dsymptr(s, ot, s2, 0); + ot = duintptr(s, ot, t->bound); + } else { + // ../../pkg/runtime/type.go:/SliceType + s1 = dtypesym(t->type); + ot = dcommontype(s, ot, t); + xt = ot - 2*widthptr; + ot = dsymptr(s, ot, s1, 0); + } + break; + + case TCHAN: + // ../../pkg/runtime/type.go:/ChanType + s1 = dtypesym(t->type); + ot = dcommontype(s, ot, t); + xt = ot - 2*widthptr; + ot = dsymptr(s, ot, s1, 0); + ot = duintptr(s, ot, t->chan); + break; + + case TFUNC: + for(t1=getthisx(t)->type; t1; t1=t1->down) + dtypesym(t1->type); + isddd = 0; + for(t1=getinargx(t)->type; t1; t1=t1->down) { + isddd = t1->isddd; + dtypesym(t1->type); + } + for(t1=getoutargx(t)->type; t1; t1=t1->down) + dtypesym(t1->type); + + ot = dcommontype(s, ot, t); + xt = ot - 2*widthptr; + ot = duint8(s, ot, isddd); + + // two slice headers: in and out. + ot = rnd(ot, widthptr); + ot = dsymptr(s, ot, s, ot+2*(widthptr+2*4)); + n = t->thistuple + t->intuple; + ot = duint32(s, ot, n); + ot = duint32(s, ot, n); + ot = dsymptr(s, ot, s, ot+1*(widthptr+2*4)+n*widthptr); + ot = duint32(s, ot, t->outtuple); + ot = duint32(s, ot, t->outtuple); + + // slice data + for(t1=getthisx(t)->type; t1; t1=t1->down, n++) + ot = dsymptr(s, ot, dtypesym(t1->type), 0); + for(t1=getinargx(t)->type; t1; t1=t1->down, n++) + ot = dsymptr(s, ot, dtypesym(t1->type), 0); + for(t1=getoutargx(t)->type; t1; t1=t1->down, n++) + ot = dsymptr(s, ot, dtypesym(t1->type), 0); + break; + + case TINTER: + m = imethods(t); + n = 0; + for(a=m; a; a=a->link) { + dtypesym(a->type); + n++; + } + + // ../../pkg/runtime/type.go:/InterfaceType + ot = dcommontype(s, ot, t); + xt = ot - 2*widthptr; + ot = dsymptr(s, ot, s, ot+widthptr+2*4); + ot = duint32(s, ot, n); + ot = duint32(s, ot, n); + for(a=m; a; a=a->link) { + // ../../pkg/runtime/type.go:/imethod + ot = dgostringptr(s, ot, a->name); + ot = dgopkgpath(s, ot, a->pkg); + ot = dsymptr(s, ot, dtypesym(a->type), 0); + } + break; + + case TMAP: + // ../../pkg/runtime/type.go:/MapType + s1 = dtypesym(t->down); + s2 = dtypesym(t->type); + ot = dcommontype(s, ot, t); + xt = ot - 2*widthptr; + ot = dsymptr(s, ot, s1, 0); + ot = dsymptr(s, ot, s2, 0); + break; + + case TPTR32: + case TPTR64: + if(t->type->etype == TANY) { + // ../../pkg/runtime/type.go:/UnsafePointerType + ot = dcommontype(s, ot, t); + break; + } + // ../../pkg/runtime/type.go:/PtrType + s1 = dtypesym(t->type); + ot = dcommontype(s, ot, t); + xt = ot - 2*widthptr; + ot = dsymptr(s, ot, s1, 0); + break; + + case TSTRUCT: + // ../../pkg/runtime/type.go:/StructType + // for security, only the exported fields. + n = 0; + for(t1=t->type; t1!=T; t1=t1->down) { + dtypesym(t1->type); + n++; + } + ot = dcommontype(s, ot, t); + xt = ot - 2*widthptr; + ot = dsymptr(s, ot, s, ot+widthptr+2*4); + ot = duint32(s, ot, n); + ot = duint32(s, ot, n); + for(t1=t->type; t1!=T; t1=t1->down) { + // ../../pkg/runtime/type.go:/structField + if(t1->sym && !t1->embedded) { + ot = dgostringptr(s, ot, t1->sym->name); + if(exportname(t1->sym->name)) + ot = dgostringptr(s, ot, nil); + else + ot = dgopkgpath(s, ot, t1->sym->pkg); + } else { + ot = dgostringptr(s, ot, nil); + ot = dgostringptr(s, ot, nil); + } + ot = dsymptr(s, ot, dtypesym(t1->type), 0); + ot = dgostrlitptr(s, ot, t1->note); + ot = duintptr(s, ot, t1->width); // field offset + } + break; + } + ot = dextratype(s, ot, t, xt); + ggloblsym(s, ot, dupok); + return s; +} + +void +dumptypestructs(void) +{ + int i; + NodeList *l; + Node *n; + Type *t; + Pkg *p; + + // copy types from externdcl list to signatlist + for(l=externdcl; l; l=l->next) { + n = l->n; + if(n->op != OTYPE) + continue; + signatlist = list(signatlist, n); + } + + // process signatlist + for(l=signatlist; l; l=l->next) { + n = l->n; + if(n->op != OTYPE) + continue; + t = n->type; + dtypesym(t); + if(t->sym) + dtypesym(ptrto(t)); + } + + // generate import strings for imported packages + for(i=0; ilink) + if(p->direct) + dimportpath(p); + + // do basic types if compiling package runtime. + // they have to be in at least one package, + // and runtime is always loaded implicitly, + // so this is as good as any. + // another possible choice would be package main, + // but using runtime means fewer copies in .6 files. + if(compiling_runtime) { + for(i=1; i<=TBOOL; i++) + dtypesym(ptrto(types[i])); + dtypesym(ptrto(types[TSTRING])); + dtypesym(ptrto(types[TUNSAFEPTR])); + + // add paths for runtime and main, which 6l imports implicitly. + dimportpath(runtimepkg); + dimportpath(mkpkg(strlit("main"))); + } +} diff --git a/src/cmd/gc/runtime.go b/src/cmd/gc/runtime.go new file mode 100644 index 000000000..549f7abe3 --- /dev/null +++ b/src/cmd/gc/runtime.go @@ -0,0 +1,130 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// NOTE: If you change this file you must run "./mkbuiltin" +// to update builtin.c.boot. This is not done automatically +// to avoid depending on having a working compiler binary. + +package PACKAGE + +// emitted by compiler, not referred to by go programs + +func new(int32) *any +func panicindex() +func panicslice() +func throwreturn() +func throwinit() +func panicwrap(string, string, string) + +func panic(interface{}) +func recover(*int32) interface{} + +func printbool(bool) +func printfloat(float64) +func printint(int64) +func printuint(uint64) +func printcomplex(complex128) +func printstring(string) +func printpointer(any) +func printiface(any) +func printeface(any) +func printslice(any) +func printnl() +func printsp() +func goprintf() + +// filled in by compiler: int n, string, string, ... +func concatstring() + +// filled in by compiler: Type*, int n, Slice, ... +func append() +func appendslice(typ *byte, x any, y []any) any + +func cmpstring(string, string) int +func slicestring(string, int, int) string +func slicestring1(string, int) string +func intstring(int64) string +func slicebytetostring([]byte) string +func sliceinttostring([]int) string +func stringtoslicebyte(string) []byte +func stringtosliceint(string) []int +func stringiter(string, int) int +func stringiter2(string, int) (retk int, retv int) +func slicecopy(to any, fr any, wid uint32) int +func slicestringcopy(to any, fr any) int + +// interface conversions +func convI2E(elem any) (ret any) +func convI2I(typ *byte, elem any) (ret any) +func convT2E(typ *byte, elem any) (ret any) +func convT2I(typ *byte, typ2 *byte, elem any) (ret any) + +// interface type assertions x.(T) +func assertE2E(typ *byte, iface any) (ret any) +func assertE2E2(typ *byte, iface any) (ret any, ok bool) +func assertE2I(typ *byte, iface any) (ret any) +func assertE2I2(typ *byte, iface any) (ret any, ok bool) +func assertE2T(typ *byte, iface any) (ret any) +func assertE2T2(typ *byte, iface any) (ret any, ok bool) +func assertI2E(typ *byte, iface any) (ret any) +func assertI2E2(typ *byte, iface any) (ret any, ok bool) +func assertI2I(typ *byte, iface any) (ret any) +func assertI2I2(typ *byte, iface any) (ret any, ok bool) +func assertI2T(typ *byte, iface any) (ret any) +func assertI2T2(typ *byte, iface any) (ret any, ok bool) + +func ifaceeq(i1 any, i2 any) (ret bool) +func efaceeq(i1 any, i2 any) (ret bool) +func ifacethash(i1 any) (ret uint32) +func efacethash(i1 any) (ret uint32) + +// *byte is really *runtime.Type +func makemap(mapType *byte, hint int64) (hmap map[any]any) +func mapaccess1(mapType *byte, hmap map[any]any, key any) (val any) +func mapaccess2(mapType *byte, hmap map[any]any, key any) (val any, pres bool) +func mapassign1(mapType *byte, hmap map[any]any, key any, val any) +func mapassign2(mapType *byte, hmap map[any]any, key any, val any, pres bool) +func mapiterinit(mapType *byte, hmap map[any]any, hiter *any) +func mapiternext(hiter *any) +func mapiter1(hiter *any) (key any) +func mapiter2(hiter *any) (key any, val any) + +// *byte is really *runtime.Type +func makechan(chanType *byte, hint int64) (hchan chan any) +func chanrecv1(chanType *byte, hchan <-chan any) (elem any) +func chanrecv2(chanType *byte, hchan <-chan any) (elem any, received bool) +func chansend1(chanType *byte, hchan chan<- any, elem any) +func closechan(hchan any) + +func selectnbsend(chanType *byte, hchan chan<- any, elem any) bool +func selectnbrecv(chanType *byte, elem *any, hchan <-chan any) bool +func selectnbrecv2(chanType *byte, elem *any, received *bool, hchan <-chan any) bool + +func newselect(size int) (sel *byte) +func selectsend(sel *byte, hchan chan<- any, elem *any) (selected bool) +func selectrecv(sel *byte, hchan <-chan any, elem *any) (selected bool) +func selectrecv2(sel *byte, hchan <-chan any, elem *any, received *bool) (selected bool) +func selectdefault(sel *byte) (selected bool) +func selectgo(sel *byte) +func block() + +func makeslice(typ *byte, nel int64, cap int64) (ary []any) +func growslice(typ *byte, old []any, n int64) (ary []any) +func sliceslice1(old []any, lb uint64, width uint64) (ary []any) +func sliceslice(old []any, lb uint64, hb uint64, width uint64) (ary []any) +func slicearray(old *any, nel uint64, lb uint64, hb uint64, width uint64) (ary []any) + +func closure() // has args, but compiler fills in + +// only used on 32-bit +func int64div(int64, int64) int64 +func uint64div(uint64, uint64) uint64 +func int64mod(int64, int64) int64 +func uint64mod(uint64, uint64) uint64 +func float64toint64(float64) int64 +func float64touint64(float64) uint64 +func int64tofloat64(int64) float64 +func uint64tofloat64(uint64) float64 + +func complex128div(num complex128, den complex128) (quo complex128) diff --git a/src/cmd/gc/select.c b/src/cmd/gc/select.c new file mode 100644 index 000000000..909ad3aa4 --- /dev/null +++ b/src/cmd/gc/select.c @@ -0,0 +1,351 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +/* + * select + */ + +#include "go.h" + +void +typecheckselect(Node *sel) +{ + Node *ncase, *n, *def; + NodeList *l; + int lno, count; + + def = nil; + lno = setlineno(sel); + count = 0; + typechecklist(sel->ninit, Etop); + for(l=sel->list; l; l=l->next) { + count++; + ncase = l->n; + setlineno(ncase); + if(ncase->op != OXCASE) + fatal("typecheckselect %O", ncase->op); + + if(ncase->list == nil) { + // default + if(def != N) + yyerror("multiple defaults in select (first at %L)", def->lineno); + else + def = ncase; + } else if(ncase->list->next) { + yyerror("select cases cannot be lists"); + } else { + n = typecheck(&ncase->list->n, Etop); + ncase->left = n; + ncase->list = nil; + setlineno(n); + switch(n->op) { + default: + yyerror("select case must be receive, send or assign recv"); + break; + + case OAS: + // convert x = <-c into OSELRECV(x, <-c). + // remove implicit conversions; the eventual assignment + // will reintroduce them. + if((n->right->op == OCONVNOP || n->right->op == OCONVIFACE) && n->right->implicit) + n->right = n->right->left; + + if(n->right->op != ORECV) { + yyerror("select assignment must have receive on right hand side"); + break; + } + n->op = OSELRECV; + break; + + case OAS2RECV: + // convert x, ok = <-c into OSELRECV(x, <-c) with ntest=ok + if(n->right->op != ORECV) { + yyerror("select assignment must have receive on right hand side"); + break; + } + n->op = OSELRECV2; + n->left = n->list->n; + n->ntest = n->list->next->n; + n->right = n->rlist->n; + break; + + case ORECV: + // convert <-c into OSELRECV(N, <-c) + n = nod(OSELRECV, N, n); + ncase->left = n; + break; + + case OSEND: + break; + } + } + typechecklist(ncase->nbody, Etop); + } + sel->xoffset = count; + lineno = lno; +} + +void +walkselect(Node *sel) +{ + int lno, i; + Node *n, *r, *a, *tmp, *var, *cas, *dflt, *ch; + NodeList *l, *init; + + if(sel->list == nil && sel->xoffset != 0) + fatal("double walkselect"); // already rewrote + + lno = setlineno(sel); + i = count(sel->list); + + // optimization: zero-case select + if(i == 0) { + sel->nbody = list1(mkcall("block", nil, nil)); + goto out; + } + + // optimization: one-case select: single op. + if(i == 1) { + cas = sel->list->n; + setlineno(cas); + l = cas->ninit; + if(cas->left != N) { // not default: + n = cas->left; + l = concat(l, n->ninit); + n->ninit = nil; + switch(n->op) { + default: + fatal("select %O", n->op); + + case OSEND: + ch = cheapexpr(n->left, &l); + n->left = ch; + break; + + case OSELRECV: + r = n->right; + ch = cheapexpr(r->left, &l); + r->left = ch; + + if(n->left == N) + n = r; + else { + n = nod(OAS, n->left, r); + typecheck(&n, Etop); + } + break; + + case OSELRECV2: + r = n->right; + ch = cheapexpr(r->left, &l); + r->left = ch; + + a = nod(OAS2, N, N); + a->list = n->list; + a->rlist = n->rlist; + n = a; + typecheck(&n, Etop); + break; + } + + // if ch == nil { block() }; n; + a = nod(OIF, N, N); + a->ntest = nod(OEQ, ch, nodnil()); + a->nbody = list1(mkcall("block", nil, &l)); + typecheck(&a, Etop); + l = list(l, a); + l = list(l, n); + } + l = concat(l, cas->nbody); + sel->nbody = l; + goto out; + } + + // introduce temporary variables for OSELRECV where needed. + // this rewrite is used by both the general code and the next optimization. + for(l=sel->list; l; l=l->next) { + cas = l->n; + setlineno(cas); + n = cas->left; + if(n == N) + continue; + switch(n->op) { + case OSELRECV: + case OSELRECV2: + ch = n->right->left; + + // If we can use the address of the target without + // violating addressability or order of operations, do so. + // Otherwise introduce a temporary. + // Also introduce a temporary for := variables that escape, + // so that we can delay the heap allocation until the case + // is selected. + if(n->op == OSELRECV2) { + if(n->ntest == N || isblank(n->ntest)) + n->ntest = nodnil(); + else if(n->ntest->op == ONAME && + (!n->colas || (n->ntest->class&PHEAP) == 0) && + convertop(types[TBOOL], n->ntest->type, nil) == OCONVNOP) { + n->ntest = nod(OADDR, n->ntest, N); + n->ntest->etype = 1; // pointer does not escape + typecheck(&n->ntest, Erv); + } else { + tmp = nod(OXXX, N, N); + tempname(tmp, types[TBOOL]); + a = nod(OADDR, tmp, N); + a->etype = 1; // pointer does not escape + typecheck(&a, Erv); + r = nod(OAS, n->ntest, tmp); + typecheck(&r, Etop); + cas->nbody = concat(list1(r), cas->nbody); + n->ntest = a; + } + } + + if(n->left == N || isblank(n->left)) + n->left = nodnil(); + else if(n->left->op == ONAME && + (!n->colas || (n->left->class&PHEAP) == 0) && + convertop(ch->type->type, n->left->type, nil) == OCONVNOP) { + n->left = nod(OADDR, n->left, N); + n->left->etype = 1; // pointer does not escape + typecheck(&n->left, Erv); + } else { + tmp = nod(OXXX, N, N); + tempname(tmp, ch->type->type); + a = nod(OADDR, tmp, N); + a->etype = 1; // pointer does not escape + typecheck(&a, Erv); + r = nod(OAS, n->left, tmp); + typecheck(&r, Etop); + cas->nbody = concat(list1(r), cas->nbody); + n->left = a; + } + + cas->nbody = concat(n->ninit, cas->nbody); + n->ninit = nil; + break; + } + } + + // optimization: two-case select but one is default: single non-blocking op. + if(i == 2 && (sel->list->n->left == nil || sel->list->next->n->left == nil)) { + if(sel->list->n->left == nil) { + cas = sel->list->next->n; + dflt = sel->list->n; + } else { + dflt = sel->list->next->n; + cas = sel->list->n; + } + + n = cas->left; + setlineno(n); + r = nod(OIF, N, N); + r->ninit = cas->ninit; + switch(n->op) { + default: + fatal("select %O", n->op); + + case OSEND: + // if c != nil && selectnbsend(c, v) { body } else { default body } + ch = cheapexpr(n->left, &r->ninit); + r->ntest = mkcall1(chanfn("selectnbsend", 2, ch->type), + types[TBOOL], &r->ninit, typename(ch->type), ch, n->right); + break; + + case OSELRECV: + // if c != nil && selectnbrecv(&v, c) { body } else { default body } + r = nod(OIF, N, N); + r->ninit = cas->ninit; + ch = cheapexpr(n->right->left, &r->ninit); + r->ntest = mkcall1(chanfn("selectnbrecv", 2, ch->type), + types[TBOOL], &r->ninit, typename(ch->type), n->left, ch); + break; + + case OSELRECV2: + // if c != nil && selectnbrecv2(&v, c) { body } else { default body } + r = nod(OIF, N, N); + r->ninit = cas->ninit; + ch = cheapexpr(n->right->left, &r->ninit); + r->ntest = mkcall1(chanfn("selectnbrecv2", 2, ch->type), + types[TBOOL], &r->ninit, typename(ch->type), n->left, n->ntest, ch); + break; + } + typecheck(&r->ntest, Erv); + r->nbody = cas->nbody; + r->nelse = concat(dflt->ninit, dflt->nbody); + sel->nbody = list1(r); + goto out; + } + + init = sel->ninit; + sel->ninit = nil; + + // generate sel-struct + setlineno(sel); + var = nod(OXXX, N, N); + tempname(var, ptrto(types[TUINT8])); + r = nod(OAS, var, mkcall("newselect", var->type, nil, nodintconst(sel->xoffset))); + typecheck(&r, Etop); + init = list(init, r); + + // register cases + for(l=sel->list; l; l=l->next) { + cas = l->n; + setlineno(cas); + n = cas->left; + r = nod(OIF, N, N); + r->nbody = cas->ninit; + cas->ninit = nil; + if(n != nil) { + r->nbody = concat(r->nbody, n->ninit); + n->ninit = nil; + } + if(n == nil) { + // selectdefault(sel *byte); + r->ntest = mkcall("selectdefault", types[TBOOL], &init, var); + } else { + switch(n->op) { + default: + fatal("select %O", n->op); + + case OSEND: + // selectsend(sel *byte, hchan *chan any, elem *any) (selected bool); + n->left = safeexpr(n->left, &r->ninit); + n->right = localexpr(n->right, n->left->type->type, &r->ninit); + n->right = nod(OADDR, n->right, N); + n->right->etype = 1; // pointer does not escape + typecheck(&n->right, Erv); + r->ntest = mkcall1(chanfn("selectsend", 2, n->left->type), types[TBOOL], + &init, var, n->left, n->right); + break; + + case OSELRECV: + // selectrecv(sel *byte, hchan *chan any, elem *any) (selected bool); + r->ntest = mkcall1(chanfn("selectrecv", 2, n->right->left->type), types[TBOOL], + &init, var, n->right->left, n->left); + break; + + case OSELRECV2: + // selectrecv2(sel *byte, hchan *chan any, elem *any, received *bool) (selected bool); + r->ntest = mkcall1(chanfn("selectrecv2", 2, n->right->left->type), types[TBOOL], + &init, var, n->right->left, n->left, n->ntest); + break; + } + } + r->nbody = concat(r->nbody, cas->nbody); + r->nbody = list(r->nbody, nod(OBREAK, N, N)); + init = list(init, r); + } + + // run the select + setlineno(sel); + init = list(init, mkcall("selectgo", T, nil, var)); + sel->nbody = init; + +out: + sel->list = nil; + walkstmtlist(sel->nbody); + lineno = lno; +} diff --git a/src/cmd/gc/sinit.c b/src/cmd/gc/sinit.c new file mode 100644 index 000000000..917e2ae6d --- /dev/null +++ b/src/cmd/gc/sinit.c @@ -0,0 +1,971 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +/* + * static initialization + */ + +#include "go.h" + +static NodeList *initlist; +static void init2(Node*, NodeList**); +static void init2list(NodeList*, NodeList**); + +static void +init1(Node *n, NodeList **out) +{ + NodeList *l; + + if(n == N) + return; + init1(n->left, out); + init1(n->right, out); + for(l=n->list; l; l=l->next) + init1(l->n, out); + + if(n->op != ONAME) + return; + switch(n->class) { + case PEXTERN: + case PFUNC: + break; + default: + if(isblank(n) && n->defn != N && !n->defn->initorder) { + n->defn->initorder = 1; + *out = list(*out, n->defn); + } + return; + } + + if(n->initorder == 1) + return; + if(n->initorder == 2) { + if(n->class == PFUNC) + return; + + // if there have already been errors printed, + // those errors probably confused us and + // there might not be a loop. let the user + // fix those first. + flusherrors(); + if(nerrors > 0) + errorexit(); + + print("initialization loop:\n"); + for(l=initlist;; l=l->next) { + if(l->next == nil) + break; + l->next->end = l; + } + for(; l; l=l->end) + print("\t%L %S refers to\n", l->n->lineno, l->n->sym); + print("\t%L %S\n", n->lineno, n->sym); + errorexit(); + } + n->initorder = 2; + l = malloc(sizeof *l); + l->next = initlist; + l->n = n; + l->end = nil; + initlist = l; + + // make sure that everything n depends on is initialized. + // n->defn is an assignment to n + if(n->defn != N) { + switch(n->defn->op) { + default: + goto bad; + + case ODCLFUNC: + init2list(n->defn->nbody, out); + break; + + case OAS: + if(n->defn->left != n) + goto bad; + n->defn->dodata = 1; + init1(n->defn->right, out); + if(debug['j']) + print("%S\n", n->sym); + *out = list(*out, n->defn); + break; + + case OAS2FUNC: + case OAS2MAPR: + case OAS2DOTTYPE: + case OAS2RECV: + if(n->defn->initorder) + break; + n->defn->initorder = 1; + for(l=n->defn->rlist; l; l=l->next) + init1(l->n, out); + *out = list(*out, n->defn); + break; + } + } + l = initlist; + initlist = l->next; + if(l->n != n) + fatal("bad initlist"); + free(l); + n->initorder = 1; + return; + +bad: + dump("defn", n->defn); + fatal("init1: bad defn"); +} + +// recurse over n, doing init1 everywhere. +static void +init2(Node *n, NodeList **out) +{ + if(n == N || n->initorder == 1) + return; + init1(n, out); + init2(n->left, out); + init2(n->right, out); + init2(n->ntest, out); + init2list(n->ninit, out); + init2list(n->list, out); + init2list(n->rlist, out); + init2list(n->nbody, out); + init2list(n->nelse, out); +} + +static void +init2list(NodeList *l, NodeList **out) +{ + for(; l; l=l->next) + init2(l->n, out); +} + + +static void +initreorder(NodeList *l, NodeList **out) +{ + Node *n; + + for(; l; l=l->next) { + n = l->n; + switch(n->op) { + case ODCLFUNC: + case ODCLCONST: + case ODCLTYPE: + continue; + } + initreorder(n->ninit, out); + n->ninit = nil; + init1(n, out); + } +} + +NodeList* +initfix(NodeList *l) +{ + NodeList *lout; + + lout = nil; + initreorder(l, &lout); + return lout; +} + +/* + * from here down is the walk analysis + * of composite literals. + * most of the work is to generate + * data statements for the constant + * part of the composite literal. + */ + +static void structlit(int ctxt, int pass, Node *n, Node *var, NodeList **init); +static void arraylit(int ctxt, int pass, Node *n, Node *var, NodeList **init); +static void slicelit(int ctxt, Node *n, Node *var, NodeList **init); +static void maplit(int ctxt, Node *n, Node *var, NodeList **init); + +static Node* +staticname(Type *t, int ctxt) +{ + Node *n; + + snprint(namebuf, sizeof(namebuf), "statictmp_%.4d", statuniqgen); + statuniqgen++; + n = newname(lookup(namebuf)); + if(!ctxt) + n->readonly = 1; + addvar(n, t, PEXTERN); + return n; +} + +static int +isliteral(Node *n) +{ + if(n->op == OLITERAL) + if(n->val.ctype != CTNIL) + return 1; + return 0; +} + +static int +simplename(Node *n) +{ + if(n->op != ONAME) + goto no; + if(!n->addable) + goto no; + if(n->class & PHEAP) + goto no; + if(n->class == PPARAMREF) + goto no; + return 1; + +no: + return 0; +} + +static void +litas(Node *l, Node *r, NodeList **init) +{ + Node *a; + + a = nod(OAS, l, r); + typecheck(&a, Etop); + walkexpr(&a, init); + *init = list(*init, a); +} + +enum +{ + MODEDYNAM = 1, + MODECONST = 2, +}; + +static int +getdyn(Node *n, int top) +{ + NodeList *nl; + Node *value; + int mode; + + mode = 0; + switch(n->op) { + default: + if(isliteral(n)) + return MODECONST; + return MODEDYNAM; + case OARRAYLIT: + if(!top && n->type->bound < 0) + return MODEDYNAM; + case OSTRUCTLIT: + break; + } + + for(nl=n->list; nl; nl=nl->next) { + value = nl->n->right; + mode |= getdyn(value, 0); + if(mode == (MODEDYNAM|MODECONST)) + break; + } + return mode; +} + +static void +structlit(int ctxt, int pass, Node *n, Node *var, NodeList **init) +{ + Node *r, *a; + NodeList *nl; + Node *index, *value; + + for(nl=n->list; nl; nl=nl->next) { + r = nl->n; + if(r->op != OKEY) + fatal("structlit: rhs not OKEY: %N", r); + index = r->left; + value = r->right; + + switch(value->op) { + case OARRAYLIT: + if(value->type->bound < 0) { + if(pass == 1 && ctxt != 0) { + a = nod(ODOT, var, newname(index->sym)); + slicelit(ctxt, value, a, init); + } else + if(pass == 2 && ctxt == 0) { + a = nod(ODOT, var, newname(index->sym)); + slicelit(ctxt, value, a, init); + } else + if(pass == 3) + break; + continue; + } + a = nod(ODOT, var, newname(index->sym)); + arraylit(ctxt, pass, value, a, init); + continue; + + case OSTRUCTLIT: + a = nod(ODOT, var, newname(index->sym)); + structlit(ctxt, pass, value, a, init); + continue; + } + + if(isliteral(value)) { + if(pass == 2) + continue; + } else + if(pass == 1) + continue; + + // build list of var.field = expr + a = nod(ODOT, var, newname(index->sym)); + a = nod(OAS, a, value); + typecheck(&a, Etop); + walkexpr(&a, init); + if(pass == 1) { + if(a->op != OAS) + fatal("structlit: not as"); + a->dodata = 2; + } + *init = list(*init, a); + } +} + +static void +arraylit(int ctxt, int pass, Node *n, Node *var, NodeList **init) +{ + Node *r, *a; + NodeList *l; + Node *index, *value; + + for(l=n->list; l; l=l->next) { + r = l->n; + if(r->op != OKEY) + fatal("arraylit: rhs not OKEY: %N", r); + index = r->left; + value = r->right; + + switch(value->op) { + case OARRAYLIT: + if(value->type->bound < 0) { + if(pass == 1 && ctxt != 0) { + a = nod(OINDEX, var, index); + slicelit(ctxt, value, a, init); + } else + if(pass == 2 && ctxt == 0) { + a = nod(OINDEX, var, index); + slicelit(ctxt, value, a, init); + } else + if(pass == 3) + break; + continue; + } + a = nod(OINDEX, var, index); + arraylit(ctxt, pass, value, a, init); + continue; + + case OSTRUCTLIT: + a = nod(OINDEX, var, index); + structlit(ctxt, pass, value, a, init); + continue; + } + + if(isliteral(index) && isliteral(value)) { + if(pass == 2) + continue; + } else + if(pass == 1) + continue; + + // build list of var[index] = value + a = nod(OINDEX, var, index); + a = nod(OAS, a, value); + typecheck(&a, Etop); + walkexpr(&a, init); // add any assignments in r to top + if(pass == 1) { + if(a->op != OAS) + fatal("structlit: not as"); + a->dodata = 2; + } + *init = list(*init, a); + } +} + +static void +slicelit(int ctxt, Node *n, Node *var, NodeList **init) +{ + Node *r, *a; + NodeList *l; + Type *t; + Node *vstat, *vauto; + Node *index, *value; + int mode; + + // make an array type + t = shallow(n->type); + t->bound = mpgetfix(n->right->val.u.xval); + t->width = 0; + t->sym = nil; + dowidth(t); + + if(ctxt != 0) { + + // put everything into static array + vstat = staticname(t, ctxt); + arraylit(ctxt, 1, n, vstat, init); + arraylit(ctxt, 2, n, vstat, init); + + // copy static to slice + a = nod(OSLICE, vstat, nod(OKEY, N, N)); + a = nod(OAS, var, a); + typecheck(&a, Etop); + a->dodata = 2; + *init = list(*init, a); + return; + } + + // recipe for var = []t{...} + // 1. make a static array + // var vstat [...]t + // 2. assign (data statements) the constant part + // vstat = constpart{} + // 3. make an auto pointer to array and allocate heap to it + // var vauto *[...]t = new([...]t) + // 4. copy the static array to the auto array + // *vauto = vstat + // 5. assign slice of allocated heap to var + // var = [0:]*auto + // 6. for each dynamic part assign to the slice + // var[i] = dynamic part + // + // an optimization is done if there is no constant part + // 3. var vauto *[...]t = new([...]t) + // 5. var = [0:]*auto + // 6. var[i] = dynamic part + + // if the literal contains constants, + // make static initialized array (1),(2) + vstat = N; + mode = getdyn(n, 1); + if(mode & MODECONST) { + vstat = staticname(t, ctxt); + arraylit(ctxt, 1, n, vstat, init); + } + + // make new auto *array (3 declare) + vauto = nod(OXXX, N, N); + tempname(vauto, ptrto(t)); + + // set auto to point at new heap (3 assign) + a = nod(ONEW, N, N); + a->list = list1(typenod(t)); + a = nod(OAS, vauto, a); + typecheck(&a, Etop); + walkexpr(&a, init); + *init = list(*init, a); + + if(vstat != N) { + // copy static to heap (4) + a = nod(OIND, vauto, N); + a = nod(OAS, a, vstat); + typecheck(&a, Etop); + walkexpr(&a, init); + *init = list(*init, a); + } + + // make slice out of heap (5) + a = nod(OAS, var, nod(OSLICE, vauto, nod(OKEY, N, N))); + typecheck(&a, Etop); + walkexpr(&a, init); + *init = list(*init, a); + + // put dynamics into slice (6) + for(l=n->list; l; l=l->next) { + r = l->n; + if(r->op != OKEY) + fatal("slicelit: rhs not OKEY: %N", r); + index = r->left; + value = r->right; + a = nod(OINDEX, var, index); + a->etype = 1; // no bounds checking + // TODO need to check bounds? + + switch(value->op) { + case OARRAYLIT: + if(value->type->bound < 0) + break; + arraylit(ctxt, 2, value, a, init); + continue; + + case OSTRUCTLIT: + structlit(ctxt, 2, value, a, init); + continue; + } + + if(isliteral(index) && isliteral(value)) + continue; + + // build list of var[c] = expr + a = nod(OAS, a, value); + typecheck(&a, Etop); + walkexpr(&a, init); + *init = list(*init, a); + } +} + +static void +maplit(int ctxt, Node *n, Node *var, NodeList **init) +{ + Node *r, *a; + NodeList *l; + int nerr, b; + Type *t, *tk, *tv, *t1; + Node *vstat, *index, *value; + Sym *syma, *symb; + +ctxt = 0; + + // make the map var + nerr = nerrors; + + a = nod(OMAKE, N, N); + a->list = list1(typenod(n->type)); + litas(var, a, init); + + // count the initializers + b = 0; + for(l=n->list; l; l=l->next) { + r = l->n; + + if(r->op != OKEY) + fatal("slicelit: rhs not OKEY: %N", r); + index = r->left; + value = r->right; + + if(isliteral(index) && isliteral(value)) + b++; + } + + t = T; + if(b != 0) { + // build type [count]struct { a Tindex, b Tvalue } + t = n->type; + tk = t->down; + tv = t->type; + + symb = lookup("b"); + t = typ(TFIELD); + t->type = tv; + t->sym = symb; + + syma = lookup("a"); + t1 = t; + t = typ(TFIELD); + t->type = tk; + t->sym = syma; + t->down = t1; + + t1 = t; + t = typ(TSTRUCT); + t->type = t1; + + t1 = t; + t = typ(TARRAY); + t->bound = b; + t->type = t1; + + dowidth(t); + + // make and initialize static array + vstat = staticname(t, ctxt); + b = 0; + for(l=n->list; l; l=l->next) { + r = l->n; + + if(r->op != OKEY) + fatal("slicelit: rhs not OKEY: %N", r); + index = r->left; + value = r->right; + + if(isliteral(index) && isliteral(value)) { + // build vstat[b].a = key; + a = nodintconst(b); + a = nod(OINDEX, vstat, a); + a = nod(ODOT, a, newname(syma)); + a = nod(OAS, a, index); + typecheck(&a, Etop); + walkexpr(&a, init); + a->dodata = 2; + *init = list(*init, a); + + // build vstat[b].b = value; + a = nodintconst(b); + a = nod(OINDEX, vstat, a); + a = nod(ODOT, a, newname(symb)); + a = nod(OAS, a, value); + typecheck(&a, Etop); + walkexpr(&a, init); + a->dodata = 2; + *init = list(*init, a); + + b++; + } + } + + // loop adding structure elements to map + // for i = 0; i < len(vstat); i++ { + // map[vstat[i].a] = vstat[i].b + // } + index = nod(OXXX, N, N); + tempname(index, types[TINT]); + + a = nod(OINDEX, vstat, index); + a->etype = 1; // no bounds checking + a = nod(ODOT, a, newname(symb)); + + r = nod(OINDEX, vstat, index); + r->etype = 1; // no bounds checking + r = nod(ODOT, r, newname(syma)); + r = nod(OINDEX, var, r); + + r = nod(OAS, r, a); + + a = nod(OFOR, N, N); + a->nbody = list1(r); + + a->ninit = list1(nod(OAS, index, nodintconst(0))); + a->ntest = nod(OLT, index, nodintconst(t->bound)); + a->nincr = nod(OASOP, index, nodintconst(1)); + a->nincr->etype = OADD; + + typecheck(&a, Etop); + walkstmt(&a); + *init = list(*init, a); + } + + // put in dynamic entries one-at-a-time + for(l=n->list; l; l=l->next) { + r = l->n; + + if(r->op != OKEY) + fatal("slicelit: rhs not OKEY: %N", r); + index = r->left; + value = r->right; + + if(isliteral(index) && isliteral(value)) + continue; + + // build list of var[c] = expr + a = nod(OINDEX, var, r->left); + a = nod(OAS, a, r->right); + typecheck(&a, Etop); + walkexpr(&a, init); + if(nerr != nerrors) + break; + + *init = list(*init, a); + } +} + +void +anylit(int ctxt, Node *n, Node *var, NodeList **init) +{ + Type *t; + Node *a, *vstat; + + t = n->type; + switch(n->op) { + default: + fatal("anylit: not lit"); + + case OSTRUCTLIT: + if(t->etype != TSTRUCT) + fatal("anylit: not struct"); + + if(simplename(var)) { + + if(ctxt == 0) { + // lay out static data + vstat = staticname(t, ctxt); + structlit(ctxt, 1, n, vstat, init); + + // copy static to var + a = nod(OAS, var, vstat); + typecheck(&a, Etop); + walkexpr(&a, init); + *init = list(*init, a); + + // add expressions to automatic + structlit(ctxt, 2, n, var, init); + break; + } + structlit(ctxt, 1, n, var, init); + structlit(ctxt, 2, n, var, init); + break; + } + + // initialize of not completely specified + if(count(n->list) < structcount(t)) { + a = nod(OAS, var, N); + typecheck(&a, Etop); + walkexpr(&a, init); + *init = list(*init, a); + } + structlit(ctxt, 3, n, var, init); + break; + + case OARRAYLIT: + if(t->etype != TARRAY) + fatal("anylit: not array"); + if(t->bound < 0) { + slicelit(ctxt, n, var, init); + break; + } + + if(simplename(var)) { + + if(ctxt == 0) { + // lay out static data + vstat = staticname(t, ctxt); + arraylit(1, 1, n, vstat, init); + + // copy static to automatic + a = nod(OAS, var, vstat); + typecheck(&a, Etop); + walkexpr(&a, init); + *init = list(*init, a); + + // add expressions to automatic + arraylit(ctxt, 2, n, var, init); + break; + } + arraylit(ctxt, 1, n, var, init); + arraylit(ctxt, 2, n, var, init); + break; + } + + // initialize of not completely specified + if(count(n->list) < t->bound) { + a = nod(OAS, var, N); + typecheck(&a, Etop); + walkexpr(&a, init); + *init = list(*init, a); + } + arraylit(ctxt, 3, n, var, init); + break; + + case OMAPLIT: + if(t->etype != TMAP) + fatal("anylit: not map"); + maplit(ctxt, n, var, init); + break; + } +} + +int +oaslit(Node *n, NodeList **init) +{ + int ctxt; + + if(n->left == N || n->right == N) + goto no; + if(n->left->type == T || n->right->type == T) + goto no; + if(!simplename(n->left)) + goto no; + if(!eqtype(n->left->type, n->right->type)) + goto no; + + // context is init() function. + // implies generated data executed + // exactly once and not subject to races. + ctxt = 0; +// if(n->dodata == 1) +// ctxt = 1; + + switch(n->right->op) { + default: + goto no; + + case OSTRUCTLIT: + case OARRAYLIT: + case OMAPLIT: + if(vmatch1(n->left, n->right)) + goto no; + anylit(ctxt, n->right, n->left, init); + break; + } + n->op = OEMPTY; + return 1; + +no: + // not a special composit literal assignment + return 0; +} + +static int +getlit(Node *lit) +{ + if(smallintconst(lit)) + return mpgetfix(lit->val.u.xval); + return -1; +} + +int +stataddr(Node *nam, Node *n) +{ + int l; + + if(n == N) + goto no; + + switch(n->op) { + + case ONAME: + *nam = *n; + return n->addable; + + case ODOT: + if(!stataddr(nam, n->left)) + break; + nam->xoffset += n->xoffset; + nam->type = n->type; + return 1; + + case OINDEX: + if(n->left->type->bound < 0) + break; + if(!stataddr(nam, n->left)) + break; + l = getlit(n->right); + if(l < 0) + break; + nam->xoffset += l*n->type->width; + nam->type = n->type; + return 1; + } + +no: + return 0; +} + +int +gen_as_init(Node *n) +{ + Node *nr, *nl; + Node nam, nod1; + + if(n->dodata == 0) + goto no; + + nr = n->right; + nl = n->left; + if(nr == N) { + if(!stataddr(&nam, nl)) + goto no; + if(nam.class != PEXTERN) + goto no; + goto yes; + } + + if(nr->type == T || !eqtype(nl->type, nr->type)) + goto no; + + if(!stataddr(&nam, nl)) + goto no; + + if(nam.class != PEXTERN) + goto no; + + switch(nr->op) { + default: + goto no; + + case OCONVNOP: + nr = nr->left; + if(nr == N || nr->op != OSLICEARR) + goto no; + // fall through + + case OSLICEARR: + if(nr->right->op == OKEY && nr->right->left == N && nr->right->right == N) { + nr = nr->left; + goto slice; + } + goto no; + + case OLITERAL: + break; + } + + switch(nr->type->etype) { + default: + goto no; + + case TBOOL: + case TINT8: + case TUINT8: + case TINT16: + case TUINT16: + case TINT32: + case TUINT32: + case TINT64: + case TUINT64: + case TINT: + case TUINT: + case TUINTPTR: + case TPTR32: + case TPTR64: + case TFLOAT32: + case TFLOAT64: + gused(N); // in case the data is the dest of a goto + gdata(&nam, nr, nr->type->width); + break; + + case TCOMPLEX64: + case TCOMPLEX128: + gused(N); // in case the data is the dest of a goto + gdatacomplex(&nam, nr->val.u.cval); + break; + + case TSTRING: + gused(N); // in case the data is the dest of a goto + gdatastring(&nam, nr->val.u.sval); + break; + } + +yes: + return 1; + +slice: + gused(N); // in case the data is the dest of a goto + nl = nr; + if(nr == N || nr->op != OADDR) + goto no; + nr = nr->left; + if(nr == N || nr->op != ONAME) + goto no; + + // nr is the array being converted to a slice + if(nr->type == T || nr->type->etype != TARRAY || nr->type->bound < 0) + goto no; + + nam.xoffset += Array_array; + gdata(&nam, nl, types[tptr]->width); + + nam.xoffset += Array_nel-Array_array; + nodconst(&nod1, types[TINT32], nr->type->bound); + gdata(&nam, &nod1, types[TINT32]->width); + + nam.xoffset += Array_cap-Array_nel; + gdata(&nam, &nod1, types[TINT32]->width); + + goto yes; + +no: + if(n->dodata == 2) { + dump("\ngen_as_init", n); + fatal("gen_as_init couldnt make data statement"); + } + return 0; +} + diff --git a/src/cmd/gc/subr.c b/src/cmd/gc/subr.c new file mode 100644 index 000000000..1a05d43d0 --- /dev/null +++ b/src/cmd/gc/subr.c @@ -0,0 +1,3885 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +#include "go.h" +#include "md5.h" +#include "y.tab.h" +#include "opnames.h" +#include "yerr.h" + +static void dodump(Node*, int); + +typedef struct Error Error; +struct Error +{ + int lineno; + int seq; + char *msg; +}; +static Error *err; +static int nerr; +static int merr; + +void +errorexit(void) +{ + flusherrors(); + if(outfile) + remove(outfile); + exit(1); +} + +extern int yychar; +int +parserline(void) +{ + if(yychar != 0 && yychar != -2) // parser has one symbol lookahead + return prevlineno; + return lineno; +} + +static void +adderr(int line, char *fmt, va_list arg) +{ + Fmt f; + Error *p; + + erroring++; + fmtstrinit(&f); + fmtprint(&f, "%L: ", line); + fmtvprint(&f, fmt, arg); + fmtprint(&f, "\n"); + erroring--; + + if(nerr >= merr) { + if(merr == 0) + merr = 16; + else + merr *= 2; + p = realloc(err, merr*sizeof err[0]); + if(p == nil) { + merr = nerr; + flusherrors(); + print("out of memory\n"); + errorexit(); + } + err = p; + } + err[nerr].seq = nerr; + err[nerr].lineno = line; + err[nerr].msg = fmtstrflush(&f); + nerr++; +} + +static int +errcmp(const void *va, const void *vb) +{ + Error *a, *b; + + a = (Error*)va; + b = (Error*)vb; + if(a->lineno != b->lineno) + return a->lineno - b->lineno; + if(a->seq != b->seq) + return a->seq - b->seq; + return strcmp(a->msg, b->msg); +} + +void +flusherrors(void) +{ + int i; + + if(nerr == 0) + return; + qsort(err, nerr, sizeof err[0], errcmp); + for(i=0; i= 10 && !debug['e']) { + flusherrors(); + print("%L: too many errors\n", line); + errorexit(); + } +} + +extern int yystate, yychar; + +void +yyerror(char *fmt, ...) +{ + int i; + static int lastsyntax; + va_list arg; + char buf[512], *p; + + if(strncmp(fmt, "syntax error", 12) == 0) { + nsyntaxerrors++; + + if(debug['x']) + print("yyerror: yystate=%d yychar=%d\n", yystate, yychar); + + // only one syntax error per line + if(lastsyntax == lexlineno) + return; + lastsyntax = lexlineno; + + if(strstr(fmt, "{ or {")) { + // The grammar has { and LBRACE but both show up as {. + // Rewrite syntax error referring to "{ or {" to say just "{". + strecpy(buf, buf+sizeof buf, fmt); + p = strstr(buf, "{ or {"); + if(p) + memmove(p+1, p+6, strlen(p+6)+1); + fmt = buf; + } + + // look for parse state-specific errors in list (see go.errors). + for(i=0; i= 10 && !debug['e']) { + flusherrors(); + print("%L: too many errors\n", parserline()); + errorexit(); + } +} + +void +warn(char *fmt, ...) +{ + va_list arg; + + va_start(arg, fmt); + adderr(parserline(), fmt, arg); + va_end(arg); + + hcrash(); +} + +void +fatal(char *fmt, ...) +{ + va_list arg; + + flusherrors(); + + print("%L: internal compiler error: ", lineno); + va_start(arg, fmt); + vfprint(1, fmt, arg); + va_end(arg); + print("\n"); + + // If this is a released compiler version, ask for a bug report. + if(strncmp(getgoversion(), "release", 7) == 0) { + print("\n"); + print("Please file a bug report including a short program that triggers the error.\n"); + print("http://code.google.com/p/go/issues/entry?template=compilerbug\n"); + } + hcrash(); + errorexit(); +} + +void +linehist(char *file, int32 off, int relative) +{ + Hist *h; + char *cp; + + if(debug['i']) { + if(file != nil) { + if(off < 0) + print("pragma %s", file); + else + if(off > 0) + print("line %s", file); + else + print("import %s", file); + } else + print("end of import"); + print(" at line %L\n", lexlineno); + } + + if(off < 0 && file[0] != '/' && !relative) { + cp = mal(strlen(file) + strlen(pathname) + 2); + sprint(cp, "%s/%s", pathname, file); + file = cp; + } + + h = mal(sizeof(Hist)); + h->name = file; + h->line = lexlineno; + h->offset = off; + h->link = H; + if(ehist == H) { + hist = h; + ehist = h; + return; + } + ehist->link = h; + ehist = h; +} + +int32 +setlineno(Node *n) +{ + int32 lno; + + lno = lineno; + if(n != N) + switch(n->op) { + case ONAME: + case OTYPE: + case OPACK: + case OLITERAL: + break; + default: + lineno = n->lineno; + if(lineno == 0) { + if(debug['K']) + warn("setlineno: line 0"); + lineno = lno; + } + } + return lno; +} + +uint32 +stringhash(char *p) +{ + int32 h; + int c; + + h = 0; + for(;;) { + c = *p++; + if(c == 0) + break; + h = h*PRIME1 + c; + } + + if(h < 0) { + h = -h; + if(h < 0) + h = 0; + } + return h; +} + +Sym* +lookup(char *name) +{ + return pkglookup(name, localpkg); +} + +Sym* +pkglookup(char *name, Pkg *pkg) +{ + Sym *s; + uint32 h; + int c; + + h = stringhash(name) % NHASH; + c = name[0]; + for(s = hash[h]; s != S; s = s->link) { + if(s->name[0] != c || s->pkg != pkg) + continue; + if(strcmp(s->name, name) == 0) + return s; + } + + s = mal(sizeof(*s)); + s->name = mal(strlen(name)+1); + strcpy(s->name, name); + + s->pkg = pkg; + + s->link = hash[h]; + hash[h] = s; + s->lexical = LNAME; + + return s; +} + +Sym* +restrictlookup(char *name, Pkg *pkg) +{ + if(!exportname(name) && pkg != localpkg) + yyerror("cannot refer to unexported name %s.%s", pkg->name, name); + return pkglookup(name, pkg); +} + + +// find all the exported symbols in package opkg +// and make them available in the current package +void +importdot(Pkg *opkg, Node *pack) +{ + Sym *s, *s1; + uint32 h; + int n; + + n = 0; + for(h=0; hlink) { + if(s->pkg != opkg) + continue; + if(s->def == N) + continue; + if(!exportname(s->name) || utfrune(s->name, 0xb7)) // 0xb7 = center dot + continue; + s1 = lookup(s->name); + if(s1->def != N) { + redeclare(s1, "during import"); + continue; + } + s1->def = s->def; + s1->block = s->block; + s1->def->pack = pack; + n++; + } + } + if(n == 0) { + // can't possibly be used - there were no symbols + yyerrorl(pack->lineno, "imported and not used: %Z", opkg->path); + } +} + +static void +gethunk(void) +{ + char *h; + int32 nh; + + nh = NHUNK; + if(thunk >= 10L*NHUNK) + nh = 10L*NHUNK; + h = (char*)malloc(nh); + if(h == nil) { + flusherrors(); + yyerror("out of memory"); + errorexit(); + } + hunk = h; + nhunk = nh; + thunk += nh; +} + +void* +mal(int32 n) +{ + void *p; + + if(n >= NHUNK) { + p = malloc(n); + if(p == nil) { + flusherrors(); + yyerror("out of memory"); + errorexit(); + } + memset(p, 0, n); + return p; + } + + while((uintptr)hunk & MAXALIGN) { + hunk++; + nhunk--; + } + if(nhunk < n) + gethunk(); + + p = hunk; + nhunk -= n; + hunk += n; + memset(p, 0, n); + return p; +} + +void* +remal(void *p, int32 on, int32 n) +{ + void *q; + + q = (uchar*)p + on; + if(q != hunk || nhunk < n) { + if(on+n >= NHUNK) { + q = mal(on+n); + memmove(q, p, on); + return q; + } + if(nhunk < on+n) + gethunk(); + memmove(hunk, p, on); + p = hunk; + hunk += on; + nhunk -= on; + } + hunk += n; + nhunk -= n; + return p; +} + +Node* +nod(int op, Node *nleft, Node *nright) +{ + Node *n; + + n = mal(sizeof(*n)); + n->op = op; + n->left = nleft; + n->right = nright; + n->lineno = parserline(); + n->xoffset = BADWIDTH; + n->orig = n; + return n; +} + +int +algtype(Type *t) +{ + int a; + + if(issimple[t->etype] || isptr[t->etype] || + t->etype == TCHAN || t->etype == TFUNC || t->etype == TMAP) { + if(t->width == 1) + a = AMEM8; + else if(t->width == 2) + a = AMEM16; + else if(t->width == 4) + a = AMEM32; + else if(t->width == 8) + a = AMEM64; + else if(t->width == 16) + a = AMEM128; + else + a = AMEM; // just bytes (int, ptr, etc) + } else if(t->etype == TSTRING) + a = ASTRING; // string + else if(isnilinter(t)) + a = ANILINTER; // nil interface + else if(t->etype == TINTER) + a = AINTER; // interface + else if(isslice(t)) + a = ASLICE; // slice + else { + if(t->width == 1) + a = ANOEQ8; + else if(t->width == 2) + a = ANOEQ16; + else if(t->width == 4) + a = ANOEQ32; + else if(t->width == 8) + a = ANOEQ64; + else if(t->width == 16) + a = ANOEQ128; + else + a = ANOEQ; // just bytes, but no hash/eq + } + return a; +} + +Type* +maptype(Type *key, Type *val) +{ + Type *t; + + + if(key != nil && key->etype != TANY && algtype(key) == ANOEQ) { + if(key->etype == TFORW) { + // map[key] used during definition of key. + // postpone check until key is fully defined. + // if there are multiple uses of map[key] + // before key is fully defined, the error + // will only be printed for the first one. + // good enough. + if(key->maplineno == 0) + key->maplineno = lineno; + } else + yyerror("invalid map key type %T", key); + } + t = typ(TMAP); + t->down = key; + t->type = val; + return t; +} + +Type* +typ(int et) +{ + Type *t; + + t = mal(sizeof(*t)); + t->etype = et; + t->width = BADWIDTH; + t->lineno = lineno; + t->orig = t; + return t; +} + +static int +methcmp(const void *va, const void *vb) +{ + Type *a, *b; + int i; + + a = *(Type**)va; + b = *(Type**)vb; + i = strcmp(a->sym->name, b->sym->name); + if(i != 0) + return i; + if(!exportname(a->sym->name)) { + i = strcmp(a->sym->pkg->path->s, b->sym->pkg->path->s); + if(i != 0) + return i; + } + return 0; +} + +Type* +sortinter(Type *t) +{ + Type *f; + int i; + Type **a; + + if(t->type == nil || t->type->down == nil) + return t; + + i=0; + for(f=t->type; f; f=f->down) + i++; + a = mal(i*sizeof f); + i = 0; + for(f=t->type; f; f=f->down) + a[i++] = f; + qsort(a, i, sizeof a[0], methcmp); + while(i-- > 0) { + a[i]->down = f; + f = a[i]; + } + t->type = f; + return t; +} + +Node* +nodintconst(int64 v) +{ + Node *c; + + c = nod(OLITERAL, N, N); + c->addable = 1; + c->val.u.xval = mal(sizeof(*c->val.u.xval)); + mpmovecfix(c->val.u.xval, v); + c->val.ctype = CTINT; + c->type = types[TIDEAL]; + ullmancalc(c); + return c; +} + +Node* +nodfltconst(Mpflt* v) +{ + Node *c; + + c = nod(OLITERAL, N, N); + c->addable = 1; + c->val.u.fval = mal(sizeof(*c->val.u.fval)); + mpmovefltflt(c->val.u.fval, v); + c->val.ctype = CTFLT; + c->type = types[TIDEAL]; + ullmancalc(c); + return c; +} + +void +nodconst(Node *n, Type *t, int64 v) +{ + memset(n, 0, sizeof(*n)); + n->op = OLITERAL; + n->addable = 1; + ullmancalc(n); + n->val.u.xval = mal(sizeof(*n->val.u.xval)); + mpmovecfix(n->val.u.xval, v); + n->val.ctype = CTINT; + n->type = t; + + if(isfloat[t->etype]) + fatal("nodconst: bad type %T", t); +} + +Node* +nodnil(void) +{ + Node *c; + + c = nodintconst(0); + c->val.ctype = CTNIL; + c->type = types[TNIL]; + return c; +} + +Node* +nodbool(int b) +{ + Node *c; + + c = nodintconst(0); + c->val.ctype = CTBOOL; + c->val.u.bval = b; + c->type = idealbool; + return c; +} + +Type* +aindex(Node *b, Type *t) +{ + Type *r; + int bound; + + bound = -1; // open bound + typecheck(&b, Erv); + if(b != nil) { + switch(consttype(b)) { + default: + yyerror("array bound must be an integer expression"); + break; + case CTINT: + bound = mpgetfix(b->val.u.xval); + if(bound < 0) + yyerror("array bound must be non negative"); + break; + } + } + + // fixed array + r = typ(TARRAY); + r->type = t; + r->bound = bound; + return r; +} + +static void +indent(int dep) +{ + int i; + + for(i=0; inext) + dodump(l->n, dep); +} + +static void +dodump(Node *n, int dep) +{ + if(n == N) + return; + + indent(dep); + if(dep > 10) { + print("...\n"); + return; + } + + if(n->ninit != nil) { + print("%O-init\n", n->op); + dodumplist(n->ninit, dep+1); + indent(dep); + } + + switch(n->op) { + default: + print("%N\n", n); + dodump(n->left, dep+1); + dodump(n->right, dep+1); + break; + + case OTYPE: + print("%O %S type=%T\n", n->op, n->sym, n->type); + if(n->type == T && n->ntype) { + indent(dep); + print("%O-ntype\n", n->op); + dodump(n->ntype, dep+1); + } + break; + + case OIF: + print("%O%J\n", n->op, n); + dodump(n->ntest, dep+1); + if(n->nbody != nil) { + indent(dep); + print("%O-then\n", n->op); + dodumplist(n->nbody, dep+1); + } + if(n->nelse != nil) { + indent(dep); + print("%O-else\n", n->op); + dodumplist(n->nelse, dep+1); + } + break; + + case OSELECT: + print("%O%J\n", n->op, n); + dodumplist(n->nbody, dep+1); + break; + + case OSWITCH: + case OFOR: + print("%O%J\n", n->op, n); + dodump(n->ntest, dep+1); + + if(n->nbody != nil) { + indent(dep); + print("%O-body\n", n->op); + dodumplist(n->nbody, dep+1); + } + + if(n->nincr != N) { + indent(dep); + print("%O-incr\n", n->op); + dodump(n->nincr, dep+1); + } + break; + + case OCASE: + // the right side points to label of the body + if(n->right != N && n->right->op == OGOTO && n->right->left->op == ONAME) + print("%O%J GOTO %N\n", n->op, n, n->right->left); + else + print("%O%J\n", n->op, n); + dodump(n->left, dep+1); + break; + + case OXCASE: + print("%N\n", n); + dodump(n->left, dep+1); + dodump(n->right, dep+1); + indent(dep); + print("%O-nbody\n", n->op); + dodumplist(n->nbody, dep+1); + break; + } + + if(0 && n->ntype != nil) { + indent(dep); + print("%O-ntype\n", n->op); + dodump(n->ntype, dep+1); + } + if(n->list != nil) { + indent(dep); + print("%O-list\n", n->op); + dodumplist(n->list, dep+1); + } + if(n->rlist != nil) { + indent(dep); + print("%O-rlist\n", n->op); + dodumplist(n->rlist, dep+1); + } + if(n->op != OIF && n->nbody != nil) { + indent(dep); + print("%O-nbody\n", n->op); + dodumplist(n->nbody, dep+1); + } +} + +void +dumplist(char *s, NodeList *l) +{ + print("%s\n", s); + dodumplist(l, 1); +} + +void +dump(char *s, Node *n) +{ + print("%s [%p]\n", s, n); + dodump(n, 1); +} + +static char* +goopnames[] = +{ + [OADDR] = "&", + [OADD] = "+", + [OANDAND] = "&&", + [OANDNOT] = "&^", + [OAND] = "&", + [OAPPEND] = "append", + [OAS] = "=", + [OAS2] = "=", + [OBREAK] = "break", + [OCALL] = "function call", + [OCAP] = "cap", + [OCASE] = "case", + [OCLOSE] = "close", + [OCOMPLEX] = "complex", + [OCOM] = "^", + [OCONTINUE] = "continue", + [OCOPY] = "copy", + [ODEC] = "--", + [ODEFER] = "defer", + [ODIV] = "/", + [OEQ] = "==", + [OFALL] = "fallthrough", + [OFOR] = "for", + [OGE] = ">=", + [OGOTO] = "goto", + [OGT] = ">", + [OIF] = "if", + [OIMAG] = "imag", + [OINC] = "++", + [OIND] = "*", + [OLEN] = "len", + [OLE] = "<=", + [OLSH] = "<<", + [OLT] = "<", + [OMAKE] = "make", + [OMINUS] = "-", + [OMOD] = "%", + [OMUL] = "*", + [ONEW] = "new", + [ONE] = "!=", + [ONOT] = "!", + [OOROR] = "||", + [OOR] = "|", + [OPANIC] = "panic", + [OPLUS] = "+", + [OPRINTN] = "println", + [OPRINT] = "print", + [ORANGE] = "range", + [OREAL] = "real", + [ORECV] = "<-", + [ORETURN] = "return", + [ORSH] = ">>", + [OSELECT] = "select", + [OSEND] = "<-", + [OSUB] = "-", + [OSWITCH] = "switch", + [OXOR] = "^", +}; + +int +Oconv(Fmt *fp) +{ + int o; + + o = va_arg(fp->args, int); + if((fp->flags & FmtSharp) && o >= 0 && o < nelem(goopnames) && goopnames[o] != nil) + return fmtstrcpy(fp, goopnames[o]); + if(o < 0 || o >= nelem(opnames) || opnames[o] == nil) + return fmtprint(fp, "O-%d", o); + return fmtstrcpy(fp, opnames[o]); +} + +int +Lconv(Fmt *fp) +{ + struct + { + Hist* incl; /* start of this include file */ + int32 idel; /* delta line number to apply to include */ + Hist* line; /* start of this #line directive */ + int32 ldel; /* delta line number to apply to #line */ + } a[HISTSZ]; + int32 lno, d; + int i, n; + Hist *h; + + lno = va_arg(fp->args, int32); + + n = 0; + for(h=hist; h!=H; h=h->link) { + if(h->offset < 0) + continue; + if(lno < h->line) + break; + if(h->name) { + if(h->offset > 0) { + // #line directive + if(n > 0 && n < HISTSZ) { + a[n-1].line = h; + a[n-1].ldel = h->line - h->offset + 1; + } + } else { + // beginning of file + if(n < HISTSZ) { + a[n].incl = h; + a[n].idel = h->line; + a[n].line = 0; + } + n++; + } + continue; + } + n--; + if(n > 0 && n < HISTSZ) { + d = h->line - a[n].incl->line; + a[n-1].ldel += d; + a[n-1].idel += d; + } + } + + if(n > HISTSZ) + n = HISTSZ; + + for(i=n-1; i>=0; i--) { + if(i != n-1) { + if(fp->flags & ~(FmtWidth|FmtPrec)) + break; + fmtprint(fp, " "); + } + if(debug['L']) + fmtprint(fp, "%s/", pathname); + if(a[i].line) + fmtprint(fp, "%s:%d[%s:%d]", + a[i].line->name, lno-a[i].ldel+1, + a[i].incl->name, lno-a[i].idel+1); + else + fmtprint(fp, "%s:%d", + a[i].incl->name, lno-a[i].idel+1); + lno = a[i].incl->line - 1; // now print out start of this file + } + if(n == 0) + fmtprint(fp, ""); + + return 0; +} + +/* +s%,%,\n%g +s%\n+%\n%g +s%^[ ]*T%%g +s%,.*%%g +s%.+% [T&] = "&",%g +s%^ ........*\]%&~%g +s%~ %%g +*/ + +static char* +etnames[] = +{ + [TINT] = "INT", + [TUINT] = "UINT", + [TINT8] = "INT8", + [TUINT8] = "UINT8", + [TINT16] = "INT16", + [TUINT16] = "UINT16", + [TINT32] = "INT32", + [TUINT32] = "UINT32", + [TINT64] = "INT64", + [TUINT64] = "UINT64", + [TUINTPTR] = "UINTPTR", + [TFLOAT32] = "FLOAT32", + [TFLOAT64] = "FLOAT64", + [TCOMPLEX64] = "COMPLEX64", + [TCOMPLEX128] = "COMPLEX128", + [TBOOL] = "BOOL", + [TPTR32] = "PTR32", + [TPTR64] = "PTR64", + [TFUNC] = "FUNC", + [TARRAY] = "ARRAY", + [TSTRUCT] = "STRUCT", + [TCHAN] = "CHAN", + [TMAP] = "MAP", + [TINTER] = "INTER", + [TFORW] = "FORW", + [TFIELD] = "FIELD", + [TSTRING] = "STRING", + [TANY] = "ANY", +}; + +int +Econv(Fmt *fp) +{ + int et; + + et = va_arg(fp->args, int); + if(et < 0 || et >= nelem(etnames) || etnames[et] == nil) + return fmtprint(fp, "E-%d", et); + return fmtstrcpy(fp, etnames[et]); +} + +static const char* classnames[] = { + "Pxxx", + "PEXTERN", + "PAUTO", + "PPARAM", + "PPARAMOUT", + "PPARAMREF", + "PFUNC", +}; + +int +Jconv(Fmt *fp) +{ + Node *n; + char *s; + int c; + + n = va_arg(fp->args, Node*); + + c = fp->flags&FmtShort; + + if(!c && n->ullman != 0) + fmtprint(fp, " u(%d)", n->ullman); + + if(!c && n->addable != 0) + fmtprint(fp, " a(%d)", n->addable); + + if(!c && n->vargen != 0) + fmtprint(fp, " g(%d)", n->vargen); + + if(n->lineno != 0) + fmtprint(fp, " l(%d)", n->lineno); + + if(!c && n->xoffset != BADWIDTH) + fmtprint(fp, " x(%lld%+d)", n->xoffset, n->stkdelta); + + if(n->class != 0) { + s = ""; + if (n->class & PHEAP) s = ",heap"; + if ((n->class & ~PHEAP) < nelem(classnames)) + fmtprint(fp, " class(%s%s)", classnames[n->class&~PHEAP], s); + else + fmtprint(fp, " class(%d?%s)", n->class&~PHEAP, s); + } + + if(n->colas != 0) + fmtprint(fp, " colas(%d)", n->colas); + + if(n->funcdepth != 0) + fmtprint(fp, " f(%d)", n->funcdepth); + + if(n->noescape != 0) + fmtprint(fp, " ne(%d)", n->noescape); + + if(!c && n->typecheck != 0) + fmtprint(fp, " tc(%d)", n->typecheck); + + if(!c && n->dodata != 0) + fmtprint(fp, " dd(%d)", n->dodata); + + if(n->isddd != 0) + fmtprint(fp, " isddd(%d)", n->isddd); + + if(n->implicit != 0) + fmtprint(fp, " implicit(%d)", n->implicit); + + if(!c && n->pun != 0) + fmtprint(fp, " pun(%d)", n->pun); + + if(!c && n->used != 0) + fmtprint(fp, " used(%d)", n->used); + return 0; +} + +int +Sconv(Fmt *fp) +{ + Sym *s; + + s = va_arg(fp->args, Sym*); + if(s == S) { + fmtstrcpy(fp, ""); + return 0; + } + + if(fp->flags & FmtShort) + goto shrt; + + if(exporting || (fp->flags & FmtSharp)) { + if(packagequotes) + fmtprint(fp, "\"%Z\"", s->pkg->path); + else + fmtprint(fp, "%s", s->pkg->prefix); + fmtprint(fp, ".%s", s->name); + return 0; + } + + if(s->pkg && s->pkg != localpkg || longsymnames || (fp->flags & FmtLong)) { + // This one is for the user. If the package name + // was used by multiple packages, give the full + // import path to disambiguate. + if(erroring && pkglookup(s->pkg->name, nil)->npkg > 1) { + fmtprint(fp, "\"%Z\".%s", s->pkg->path, s->name); + return 0; + } + fmtprint(fp, "%s.%s", s->pkg->name, s->name); + return 0; + } + +shrt: + fmtstrcpy(fp, s->name); + return 0; +} + +static char* +basicnames[] = +{ + [TINT] = "int", + [TUINT] = "uint", + [TINT8] = "int8", + [TUINT8] = "uint8", + [TINT16] = "int16", + [TUINT16] = "uint16", + [TINT32] = "int32", + [TUINT32] = "uint32", + [TINT64] = "int64", + [TUINT64] = "uint64", + [TUINTPTR] = "uintptr", + [TFLOAT32] = "float32", + [TFLOAT64] = "float64", + [TCOMPLEX64] = "complex64", + [TCOMPLEX128] = "complex128", + [TBOOL] = "bool", + [TANY] = "any", + [TSTRING] = "string", + [TNIL] = "nil", + [TIDEAL] = "ideal", + [TBLANK] = "blank", +}; + +int +Tpretty(Fmt *fp, Type *t) +{ + Type *t1; + Sym *s; + + if(0 && debug['r']) { + debug['r'] = 0; + fmtprint(fp, "%T (orig=%T)", t, t->orig); + debug['r'] = 1; + return 0; + } + + if(t->etype != TFIELD + && t->sym != S + && !(fp->flags&FmtLong)) { + s = t->sym; + if(t == types[t->etype] && t->etype != TUNSAFEPTR) + return fmtprint(fp, "%s", s->name); + if(exporting) { + if(fp->flags & FmtShort) + fmtprint(fp, "%hS", s); + else + fmtprint(fp, "%S", s); + if(s->pkg != localpkg) + return 0; + if(t->vargen) + fmtprint(fp, "·%d", t->vargen); + return 0; + } + return fmtprint(fp, "%S", s); + } + + if(t->etype < nelem(basicnames) && basicnames[t->etype] != nil) { + if(isideal(t) && t->etype != TIDEAL && t->etype != TNIL) + fmtprint(fp, "ideal "); + return fmtprint(fp, "%s", basicnames[t->etype]); + } + + switch(t->etype) { + case TPTR32: + case TPTR64: + if(fp->flags&FmtShort) // pass flag thru for methodsym + return fmtprint(fp, "*%hT", t->type); + return fmtprint(fp, "*%T", t->type); + + case TCHAN: + switch(t->chan) { + case Crecv: + return fmtprint(fp, "<-chan %T", t->type); + case Csend: + return fmtprint(fp, "chan<- %T", t->type); + } + if(t->type != T && t->type->etype == TCHAN && t->type->sym == S && t->type->chan == Crecv) + return fmtprint(fp, "chan (%T)", t->type); + return fmtprint(fp, "chan %T", t->type); + + case TMAP: + return fmtprint(fp, "map[%T] %T", t->down, t->type); + + case TFUNC: + // t->type is method struct + // t->type->down is result struct + // t->type->down->down is arg struct + if(t->thistuple && !(fp->flags&FmtSharp) && !(fp->flags&FmtShort)) { + fmtprint(fp, "method("); + for(t1=getthisx(t)->type; t1; t1=t1->down) { + fmtprint(fp, "%T", t1); + if(t1->down) + fmtprint(fp, ", "); + } + fmtprint(fp, ")"); + } + + if(!(fp->flags&FmtByte)) + fmtprint(fp, "func"); + fmtprint(fp, "("); + for(t1=getinargx(t)->type; t1; t1=t1->down) { + if(noargnames && t1->etype == TFIELD) { + if(t1->isddd) + fmtprint(fp, "...%T", t1->type->type); + else + fmtprint(fp, "%T", t1->type); + } else + fmtprint(fp, "%T", t1); + if(t1->down) + fmtprint(fp, ", "); + } + fmtprint(fp, ")"); + switch(t->outtuple) { + case 0: + break; + case 1: + t1 = getoutargx(t)->type; + if(t1 == T) { + // failure to typecheck earlier; don't know the type + fmtprint(fp, " ?unknown-type?"); + break; + } + if(t1->etype == TFIELD) + t1 = t1->type; + fmtprint(fp, " %T", t1); + break; + default: + t1 = getoutargx(t)->type; + fmtprint(fp, " ("); + for(; t1; t1=t1->down) { + if(noargnames && t1->etype == TFIELD) + fmtprint(fp, "%T", t1->type); + else + fmtprint(fp, "%T", t1); + if(t1->down) + fmtprint(fp, ", "); + } + fmtprint(fp, ")"); + break; + } + return 0; + + case TARRAY: + if(t->bound >= 0) + return fmtprint(fp, "[%d]%T", (int)t->bound, t->type); + if(t->bound == -100) + return fmtprint(fp, "[...]%T", t->type); + return fmtprint(fp, "[]%T", t->type); + + case TINTER: + fmtprint(fp, "interface {"); + for(t1=t->type; t1!=T; t1=t1->down) { + fmtprint(fp, " "); + if(exportname(t1->sym->name)) + fmtprint(fp, "%hS", t1->sym); + else + fmtprint(fp, "%S", t1->sym); + fmtprint(fp, "%hhT", t1->type); + if(t1->down) + fmtprint(fp, ";"); + } + return fmtprint(fp, " }"); + + case TSTRUCT: + if(t->funarg) { + fmtprint(fp, "("); + for(t1=t->type; t1!=T; t1=t1->down) { + fmtprint(fp, "%T", t1); + if(t1->down) + fmtprint(fp, ", "); + } + return fmtprint(fp, ")"); + } + fmtprint(fp, "struct {"); + for(t1=t->type; t1!=T; t1=t1->down) { + fmtprint(fp, " %T", t1); + if(t1->down) + fmtprint(fp, ";"); + } + return fmtprint(fp, " }"); + + case TFIELD: + if(t->sym == S || t->embedded) { + if(exporting) + fmtprint(fp, "? "); + } else + fmtprint(fp, "%hS ", t->sym); + if(t->isddd) + fmtprint(fp, "...%T", t->type->type); + else + fmtprint(fp, "%T", t->type); + if(t->note) { + fmtprint(fp, " "); + if(exporting) + fmtprint(fp, ":"); + fmtprint(fp, "\"%Z\"", t->note); + } + return 0; + + case TFORW: + if(exporting) + yyerror("undefined type %S", t->sym); + if(t->sym) + return fmtprint(fp, "undefined %S", t->sym); + return fmtprint(fp, "undefined"); + + case TUNSAFEPTR: + if(exporting) + return fmtprint(fp, "\"unsafe\".Pointer"); + return fmtprint(fp, "unsafe.Pointer"); + } + + // Don't know how to handle - fall back to detailed prints. + return -1; +} + +int +Tconv(Fmt *fp) +{ + Type *t, *t1; + int r, et, sharp, minus; + + sharp = (fp->flags & FmtSharp); + minus = (fp->flags & FmtLeft); + fp->flags &= ~(FmtSharp|FmtLeft); + + t = va_arg(fp->args, Type*); + if(t == T) + return fmtstrcpy(fp, ""); + + t->trecur++; + if(t->trecur > 5) { + fmtprint(fp, "..."); + goto out; + } + + if(!debug['t']) { + if(sharp) + exporting++; + if(minus) + noargnames++; + r = Tpretty(fp, t); + if(sharp) + exporting--; + if(minus) + noargnames--; + if(r >= 0) { + t->trecur--; + return 0; + } + } + + if(sharp || exporting) + fatal("missing %E case during export", t->etype); + + et = t->etype; + fmtprint(fp, "%E ", et); + if(t->sym != S) + fmtprint(fp, "<%S>", t->sym); + + switch(et) { + default: + if(t->type != T) + fmtprint(fp, " %T", t->type); + break; + + case TFIELD: + fmtprint(fp, "%T", t->type); + break; + + case TFUNC: + if(fp->flags & FmtLong) + fmtprint(fp, "%d%d%d(%lT,%lT)%lT", + t->thistuple, t->intuple, t->outtuple, + t->type, t->type->down->down, t->type->down); + else + fmtprint(fp, "%d%d%d(%T,%T)%T", + t->thistuple, t->intuple, t->outtuple, + t->type, t->type->down->down, t->type->down); + break; + + case TINTER: + fmtprint(fp, "{"); + if(fp->flags & FmtLong) + for(t1=t->type; t1!=T; t1=t1->down) + fmtprint(fp, "%lT;", t1); + fmtprint(fp, "}"); + break; + + case TSTRUCT: + fmtprint(fp, "{"); + if(fp->flags & FmtLong) + for(t1=t->type; t1!=T; t1=t1->down) + fmtprint(fp, "%lT;", t1); + fmtprint(fp, "}"); + break; + + case TMAP: + fmtprint(fp, "[%T]%T", t->down, t->type); + break; + + case TARRAY: + if(t->bound >= 0) + fmtprint(fp, "[%d]%T", t->bound, t->type); + else + fmtprint(fp, "[]%T", t->type); + break; + + case TPTR32: + case TPTR64: + fmtprint(fp, "%T", t->type); + break; + } + +out: + t->trecur--; + return 0; +} + +int +Nconv(Fmt *fp) +{ + char buf1[500]; + Node *n; + + n = va_arg(fp->args, Node*); + if(n == N) { + fmtprint(fp, ""); + goto out; + } + + if(fp->flags & FmtSign) { + if(n->type == T) + fmtprint(fp, "%#N", n); + else if(n->type->etype == TNIL) + fmtprint(fp, "nil"); + else + fmtprint(fp, "%#N (type %T)", n, n->type); + goto out; + } + + if(fp->flags & FmtSharp) { + if(n->orig != N) + n = n->orig; + exprfmt(fp, n, 0); + goto out; + } + + switch(n->op) { + default: + if (fp->flags & FmtShort) + fmtprint(fp, "%O%hJ", n->op, n); + else + fmtprint(fp, "%O%J", n->op, n); + break; + + case ONAME: + case ONONAME: + if(n->sym == S) { + if (fp->flags & FmtShort) + fmtprint(fp, "%O%hJ", n->op, n); + else + fmtprint(fp, "%O%J", n->op, n); + break; + } + if (fp->flags & FmtShort) + fmtprint(fp, "%O-%S%hJ", n->op, n->sym, n); + else + fmtprint(fp, "%O-%S%J", n->op, n->sym, n); + goto ptyp; + + case OREGISTER: + fmtprint(fp, "%O-%R%J", n->op, n->val.u.reg, n); + break; + + case OLITERAL: + switch(n->val.ctype) { + default: + snprint(buf1, sizeof(buf1), "LITERAL-ctype=%d", n->val.ctype); + break; + case CTINT: + snprint(buf1, sizeof(buf1), "I%B", n->val.u.xval); + break; + case CTFLT: + snprint(buf1, sizeof(buf1), "F%g", mpgetflt(n->val.u.fval)); + break; + case CTCPLX: + snprint(buf1, sizeof(buf1), "(F%g+F%gi)", + mpgetflt(&n->val.u.cval->real), + mpgetflt(&n->val.u.cval->imag)); + break; + case CTSTR: + snprint(buf1, sizeof(buf1), "S\"%Z\"", n->val.u.sval); + break; + case CTBOOL: + snprint(buf1, sizeof(buf1), "B%d", n->val.u.bval); + break; + case CTNIL: + snprint(buf1, sizeof(buf1), "N"); + break; + } + fmtprint(fp, "%O-%s%J", n->op, buf1, n); + break; + + case OASOP: + fmtprint(fp, "%O-%O%J", n->op, n->etype, n); + break; + + case OTYPE: + fmtprint(fp, "%O %T", n->op, n->type); + break; + } + if(n->sym != S) + fmtprint(fp, " %S G%d", n->sym, n->vargen); + +ptyp: + if(n->type != T) + fmtprint(fp, " %T", n->type); + +out: + return 0; +} + +Node* +treecopy(Node *n) +{ + Node *m; + + if(n == N) + return N; + + switch(n->op) { + default: + m = nod(OXXX, N, N); + *m = *n; + m->left = treecopy(n->left); + m->right = treecopy(n->right); + m->list = listtreecopy(n->list); + if(m->defn) + abort(); + break; + + case ONONAME: + if(n->sym == lookup("iota")) { + // Not sure yet whether this is the real iota, + // but make a copy of the Node* just in case, + // so that all the copies of this const definition + // don't have the same iota value. + m = nod(OXXX, N, N); + *m = *n; + m->iota = iota; + break; + } + // fall through + case ONAME: + case OLITERAL: + case OTYPE: + m = n; + break; + } + return m; +} + +int +Zconv(Fmt *fp) +{ + Rune r; + Strlit *sp; + char *s, *se; + int n; + + sp = va_arg(fp->args, Strlit*); + if(sp == nil) + return fmtstrcpy(fp, ""); + + s = sp->s; + se = s + sp->len; + while(s < se) { + n = chartorune(&r, s); + s += n; + switch(r) { + case Runeerror: + if(n == 1) { + fmtprint(fp, "\\x%02x", (uchar)*(s-1)); + break; + } + // fall through + default: + if(r < ' ') { + fmtprint(fp, "\\x%02x", r); + break; + } + fmtrune(fp, r); + break; + case '\t': + fmtstrcpy(fp, "\\t"); + break; + case '\n': + fmtstrcpy(fp, "\\n"); + break; + case '\"': + case '\\': + fmtrune(fp, '\\'); + fmtrune(fp, r); + break; + } + } + return 0; +} + +int +isnil(Node *n) +{ + if(n == N) + return 0; + if(n->op != OLITERAL) + return 0; + if(n->val.ctype != CTNIL) + return 0; + return 1; +} + +int +isptrto(Type *t, int et) +{ + if(t == T) + return 0; + if(!isptr[t->etype]) + return 0; + t = t->type; + if(t == T) + return 0; + if(t->etype != et) + return 0; + return 1; +} + +int +istype(Type *t, int et) +{ + return t != T && t->etype == et; +} + +int +isfixedarray(Type *t) +{ + return t != T && t->etype == TARRAY && t->bound >= 0; +} + +int +isslice(Type *t) +{ + return t != T && t->etype == TARRAY && t->bound < 0; +} + +int +isblank(Node *n) +{ + char *p; + + if(n == N || n->sym == S) + return 0; + p = n->sym->name; + if(p == nil) + return 0; + return p[0] == '_' && p[1] == '\0'; +} + +int +isinter(Type *t) +{ + return t != T && t->etype == TINTER; +} + +int +isnilinter(Type *t) +{ + if(!isinter(t)) + return 0; + if(t->type != T) + return 0; + return 1; +} + +int +isideal(Type *t) +{ + if(t == T) + return 0; + if(t == idealstring || t == idealbool) + return 1; + switch(t->etype) { + case TNIL: + case TIDEAL: + return 1; + } + return 0; +} + +/* + * given receiver of type t (t == r or t == *r) + * return type to hang methods off (r). + */ +Type* +methtype(Type *t) +{ + if(t == T) + return T; + + // strip away pointer if it's there + if(isptr[t->etype]) { + if(t->sym != S) + return T; + t = t->type; + if(t == T) + return T; + } + + // need a type name + if(t->sym == S) + return T; + + // check types + if(!issimple[t->etype]) + switch(t->etype) { + default: + return T; + case TSTRUCT: + case TARRAY: + case TMAP: + case TCHAN: + case TSTRING: + case TFUNC: + break; + } + + return t; +} + +int +cplxsubtype(int et) +{ + switch(et) { + case TCOMPLEX64: + return TFLOAT32; + case TCOMPLEX128: + return TFLOAT64; + } + fatal("cplxsubtype: %E\n", et); + return 0; +} + +static int +eqnote(Strlit *a, Strlit *b) +{ + if(a == b) + return 1; + if(a == nil || b == nil) + return 0; + if(a->len != b->len) + return 0; + return memcmp(a->s, b->s, a->len) == 0; +} + +// Return 1 if t1 and t2 are identical, following the spec rules. +// +// Any cyclic type must go through a named type, and if one is +// named, it is only identical to the other if they are the same +// pointer (t1 == t2), so there's no chance of chasing cycles +// ad infinitum, so no need for a depth counter. +int +eqtype(Type *t1, Type *t2) +{ + if(t1 == t2) + return 1; + if(t1 == T || t2 == T || t1->etype != t2->etype || t1->sym || t2->sym) + return 0; + + switch(t1->etype) { + case TINTER: + case TSTRUCT: + for(t1=t1->type, t2=t2->type; t1 && t2; t1=t1->down, t2=t2->down) { + if(t1->etype != TFIELD || t2->etype != TFIELD) + fatal("struct/interface missing field: %T %T", t1, t2); + if(t1->sym != t2->sym || t1->embedded != t2->embedded || !eqtype(t1->type, t2->type) || !eqnote(t1->note, t2->note)) + return 0; + } + return t1 == T && t2 == T; + + case TFUNC: + // Loop over structs: receiver, in, out. + for(t1=t1->type, t2=t2->type; t1 && t2; t1=t1->down, t2=t2->down) { + Type *ta, *tb; + + if(t1->etype != TSTRUCT || t2->etype != TSTRUCT) + fatal("func missing struct: %T %T", t1, t2); + + // Loop over fields in structs, ignoring argument names. + for(ta=t1->type, tb=t2->type; ta && tb; ta=ta->down, tb=tb->down) { + if(ta->etype != TFIELD || tb->etype != TFIELD) + fatal("func struct missing field: %T %T", ta, tb); + if(ta->isddd != tb->isddd || !eqtype(ta->type, tb->type)) + return 0; + } + if(ta != T || tb != T) + return 0; + } + return t1 == T && t2 == T; + + case TARRAY: + if(t1->bound != t2->bound) + return 0; + break; + + case TCHAN: + if(t1->chan != t2->chan) + return 0; + break; + } + + return eqtype(t1->down, t2->down) && eqtype(t1->type, t2->type); +} + +// Are t1 and t2 equal struct types when field names are ignored? +// For deciding whether the result struct from g can be copied +// directly when compiling f(g()). +int +eqtypenoname(Type *t1, Type *t2) +{ + if(t1 == T || t2 == T || t1->etype != TSTRUCT || t2->etype != TSTRUCT) + return 0; + + t1 = t1->type; + t2 = t2->type; + for(;;) { + if(!eqtype(t1, t2)) + return 0; + if(t1 == T) + return 1; + t1 = t1->down; + t2 = t2->down; + } +} + +// Is type src assignment compatible to type dst? +// If so, return op code to use in conversion. +// If not, return 0. +// +// It is the caller's responsibility to call exportassignok +// to check for assignments to other packages' unexported fields, +int +assignop(Type *src, Type *dst, char **why) +{ + Type *missing, *have; + int ptr; + + if(why != nil) + *why = ""; + + if(safemode && src != T && src->etype == TUNSAFEPTR) { + yyerror("cannot use unsafe.Pointer"); + errorexit(); + } + + if(src == dst) + return OCONVNOP; + if(src == T || dst == T || src->etype == TFORW || dst->etype == TFORW || src->orig == T || dst->orig == T) + return 0; + + // 1. src type is identical to dst. + if(eqtype(src, dst)) + return OCONVNOP; + + // 2. src and dst have identical underlying types + // and either src or dst is not a named type or + // both are interface types. + if(eqtype(src->orig, dst->orig) && (src->sym == S || dst->sym == S || src->etype == TINTER)) + return OCONVNOP; + + // 3. dst is an interface type and src implements dst. + if(dst->etype == TINTER && src->etype != TNIL) { + if(implements(src, dst, &missing, &have, &ptr)) + return OCONVIFACE; + if(why != nil) { + if(isptrto(src, TINTER)) + *why = smprint(":\n\t%T is pointer to interface, not interface", src); + else if(have && have->sym == missing->sym) + *why = smprint(":\n\t%T does not implement %T (wrong type for %S method)\n" + "\t\thave %S%hhT\n\t\twant %S%hhT", src, dst, missing->sym, + have->sym, have->type, missing->sym, missing->type); + else if(ptr) + *why = smprint(":\n\t%T does not implement %T (%S method requires pointer receiver)", + src, dst, missing->sym); + else if(have) + *why = smprint(":\n\t%T does not implement %T (missing %S method)\n" + "\t\thave %S%hhT\n\t\twant %S%hhT", src, dst, missing->sym, + have->sym, have->type, missing->sym, missing->type); + else + *why = smprint(":\n\t%T does not implement %T (missing %S method)", + src, dst, missing->sym); + } + return 0; + } + if(isptrto(dst, TINTER)) { + if(why != nil) + *why = smprint(":\n\t%T is pointer to interface, not interface", dst); + return 0; + } + if(src->etype == TINTER && dst->etype != TBLANK) { + if(why != nil) + *why = ": need type assertion"; + return 0; + } + + // 4. src is a bidirectional channel value, dst is a channel type, + // src and dst have identical element types, and + // either src or dst is not a named type. + if(src->etype == TCHAN && src->chan == Cboth && dst->etype == TCHAN) + if(eqtype(src->type, dst->type) && (src->sym == S || dst->sym == S)) + return OCONVNOP; + + // 5. src is the predeclared identifier nil and dst is a nillable type. + if(src->etype == TNIL) { + switch(dst->etype) { + case TARRAY: + if(dst->bound != -100) // not slice + break; + case TPTR32: + case TPTR64: + case TFUNC: + case TMAP: + case TCHAN: + case TINTER: + return OCONVNOP; + } + } + + // 6. rule about untyped constants - already converted by defaultlit. + + // 7. Any typed value can be assigned to the blank identifier. + if(dst->etype == TBLANK) + return OCONVNOP; + + return 0; +} + +// Can we convert a value of type src to a value of type dst? +// If so, return op code to use in conversion (maybe OCONVNOP). +// If not, return 0. +int +convertop(Type *src, Type *dst, char **why) +{ + int op; + + if(why != nil) + *why = ""; + + if(src == dst) + return OCONVNOP; + if(src == T || dst == T) + return 0; + + // 1. src can be assigned to dst. + if((op = assignop(src, dst, why)) != 0) + return op; + + // The rules for interfaces are no different in conversions + // than assignments. If interfaces are involved, stop now + // with the good message from assignop. + // Otherwise clear the error. + if(src->etype == TINTER || dst->etype == TINTER) + return 0; + if(why != nil) + *why = ""; + + // 2. src and dst have identical underlying types. + if(eqtype(src->orig, dst->orig)) + return OCONVNOP; + + // 3. src and dst are unnamed pointer types + // and their base types have identical underlying types. + if(isptr[src->etype] && isptr[dst->etype] && src->sym == S && dst->sym == S) + if(eqtype(src->type->orig, dst->type->orig)) + return OCONVNOP; + + // 4. src and dst are both integer or floating point types. + if((isint[src->etype] || isfloat[src->etype]) && (isint[dst->etype] || isfloat[dst->etype])) { + if(simtype[src->etype] == simtype[dst->etype]) + return OCONVNOP; + return OCONV; + } + + // 5. src and dst are both complex types. + if(iscomplex[src->etype] && iscomplex[dst->etype]) { + if(simtype[src->etype] == simtype[dst->etype]) + return OCONVNOP; + return OCONV; + } + + // 6. src is an integer or has type []byte or []int + // and dst is a string type. + if(isint[src->etype] && dst->etype == TSTRING) + return ORUNESTR; + + if(isslice(src) && src->sym == nil && src->type == types[src->type->etype] && dst->etype == TSTRING) { + switch(src->type->etype) { + case TUINT8: + return OARRAYBYTESTR; + case TINT: + return OARRAYRUNESTR; + } + } + + // 7. src is a string and dst is []byte or []int. + // String to slice. + if(src->etype == TSTRING && isslice(dst) && dst->sym == nil && dst->type == types[dst->type->etype]) { + switch(dst->type->etype) { + case TUINT8: + return OSTRARRAYBYTE; + case TINT: + return OSTRARRAYRUNE; + } + } + + // 8. src is a pointer or uintptr and dst is unsafe.Pointer. + if((isptr[src->etype] || src->etype == TUINTPTR) && dst->etype == TUNSAFEPTR) + return OCONVNOP; + + // 9. src is unsafe.Pointer and dst is a pointer or uintptr. + if(src->etype == TUNSAFEPTR && (isptr[dst->etype] || dst->etype == TUINTPTR)) + return OCONVNOP; + + return 0; +} + +// Convert node n for assignment to type t. +Node* +assignconv(Node *n, Type *t, char *context) +{ + int op; + Node *r, *old; + char *why; + + if(n == N || n->type == T) + return n; + + old = n; + old->diag++; // silence errors about n; we'll issue one below + defaultlit(&n, t); + old->diag--; + if(t->etype == TBLANK) + return n; + + exportassignok(n->type, context); + if(eqtype(n->type, t)) + return n; + + op = assignop(n->type, t, &why); + if(op == 0) { + yyerror("cannot use %+N as type %T in %s%s", n, t, context, why); + op = OCONV; + } + + r = nod(op, n, N); + r->type = t; + r->typecheck = 1; + r->implicit = 1; + return r; +} + +static int +subtype(Type **stp, Type *t, int d) +{ + Type *st; + +loop: + st = *stp; + if(st == T) + return 0; + + d++; + if(d >= 10) + return 0; + + switch(st->etype) { + default: + return 0; + + case TPTR32: + case TPTR64: + case TCHAN: + case TARRAY: + stp = &st->type; + goto loop; + + case TANY: + if(!st->copyany) + return 0; + *stp = t; + break; + + case TMAP: + if(subtype(&st->down, t, d)) + break; + stp = &st->type; + goto loop; + + case TFUNC: + for(;;) { + if(subtype(&st->type, t, d)) + break; + if(subtype(&st->type->down->down, t, d)) + break; + if(subtype(&st->type->down, t, d)) + break; + return 0; + } + break; + + case TSTRUCT: + for(st=st->type; st!=T; st=st->down) + if(subtype(&st->type, t, d)) + return 1; + return 0; + } + return 1; +} + +/* + * Is this a 64-bit type? + */ +int +is64(Type *t) +{ + if(t == T) + return 0; + switch(simtype[t->etype]) { + case TINT64: + case TUINT64: + case TPTR64: + return 1; + } + return 0; +} + +/* + * Is a conversion between t1 and t2 a no-op? + */ +int +noconv(Type *t1, Type *t2) +{ + int e1, e2; + + e1 = simtype[t1->etype]; + e2 = simtype[t2->etype]; + + switch(e1) { + case TINT8: + case TUINT8: + return e2 == TINT8 || e2 == TUINT8; + + case TINT16: + case TUINT16: + return e2 == TINT16 || e2 == TUINT16; + + case TINT32: + case TUINT32: + case TPTR32: + return e2 == TINT32 || e2 == TUINT32 || e2 == TPTR32; + + case TINT64: + case TUINT64: + case TPTR64: + return e2 == TINT64 || e2 == TUINT64 || e2 == TPTR64; + + case TFLOAT32: + return e2 == TFLOAT32; + + case TFLOAT64: + return e2 == TFLOAT64; + } + return 0; +} + +void +argtype(Node *on, Type *t) +{ + dowidth(t); + if(!subtype(&on->type, t, 0)) + fatal("argtype: failed %N %T\n", on, t); +} + +Type* +shallow(Type *t) +{ + Type *nt; + + if(t == T) + return T; + nt = typ(0); + *nt = *t; + if(t->orig == t) + nt->orig = nt; + return nt; +} + +static Type* +deep(Type *t) +{ + Type *nt, *xt; + + if(t == T) + return T; + + switch(t->etype) { + default: + nt = t; // share from here down + break; + + case TANY: + nt = shallow(t); + nt->copyany = 1; + break; + + case TPTR32: + case TPTR64: + case TCHAN: + case TARRAY: + nt = shallow(t); + nt->type = deep(t->type); + break; + + case TMAP: + nt = shallow(t); + nt->down = deep(t->down); + nt->type = deep(t->type); + break; + + case TFUNC: + nt = shallow(t); + nt->type = deep(t->type); + nt->type->down = deep(t->type->down); + nt->type->down->down = deep(t->type->down->down); + break; + + case TSTRUCT: + nt = shallow(t); + nt->type = shallow(t->type); + xt = nt->type; + + for(t=t->type; t!=T; t=t->down) { + xt->type = deep(t->type); + xt->down = shallow(t->down); + xt = xt->down; + } + break; + } + return nt; +} + +Node* +syslook(char *name, int copy) +{ + Sym *s; + Node *n; + + s = pkglookup(name, runtimepkg); + if(s == S || s->def == N) + fatal("syslook: can't find runtime.%s", name); + + if(!copy) + return s->def; + + n = nod(0, N, N); + *n = *s->def; + n->type = deep(s->def->type); + + return n; +} + +/* + * compute a hash value for type t. + * if t is a method type, ignore the receiver + * so that the hash can be used in interface checks. + * %-T (which calls Tpretty, above) already contains + * all the necessary logic to generate a representation + * of the type that completely describes it. + * using smprint here avoids duplicating that code. + * using md5 here is overkill, but i got tired of + * accidental collisions making the runtime think + * two types are equal when they really aren't. + */ +uint32 +typehash(Type *t) +{ + char *p; + MD5 d; + + longsymnames = 1; + if(t->thistuple) { + // hide method receiver from Tpretty + t->thistuple = 0; + p = smprint("%-T", t); + t->thistuple = 1; + }else + p = smprint("%-T", t); + longsymnames = 0; + md5reset(&d); + md5write(&d, (uchar*)p, strlen(p)); + free(p); + return md5sum(&d); +} + +Type* +ptrto(Type *t) +{ + Type *t1; + + if(tptr == 0) + fatal("ptrto: nil"); + t1 = typ(tptr); + t1->type = t; + t1->width = widthptr; + t1->align = widthptr; + return t1; +} + +void +frame(int context) +{ + char *p; + NodeList *l; + Node *n; + int flag; + + p = "stack"; + l = nil; + if(curfn) + l = curfn->dcl; + if(context) { + p = "external"; + l = externdcl; + } + + flag = 1; + for(; l; l=l->next) { + n = l->n; + switch(n->op) { + case ONAME: + if(flag) + print("--- %s frame ---\n", p); + print("%O %S G%d %T\n", n->op, n->sym, n->vargen, n->type); + flag = 0; + break; + + case OTYPE: + if(flag) + print("--- %s frame ---\n", p); + print("%O %T\n", n->op, n->type); + flag = 0; + break; + } + } +} + +/* + * calculate sethi/ullman number + * roughly how many registers needed to + * compile a node. used to compile the + * hardest side first to minimize registers. + */ +void +ullmancalc(Node *n) +{ + int ul, ur; + + if(n == N) + return; + + switch(n->op) { + case OREGISTER: + case OLITERAL: + case ONAME: + ul = 1; + if(n->class == PPARAMREF || (n->class & PHEAP)) + ul++; + goto out; + case OCALL: + case OCALLFUNC: + case OCALLMETH: + case OCALLINTER: + ul = UINF; + goto out; + } + ul = 1; + if(n->left != N) + ul = n->left->ullman; + ur = 1; + if(n->right != N) + ur = n->right->ullman; + if(ul == ur) + ul += 1; + if(ur > ul) + ul = ur; + +out: + n->ullman = ul; +} + +void +badtype(int o, Type *tl, Type *tr) +{ + Fmt fmt; + char *s; + + fmtstrinit(&fmt); + if(tl != T) + fmtprint(&fmt, "\n %T", tl); + if(tr != T) + fmtprint(&fmt, "\n %T", tr); + + // common mistake: *struct and *interface. + if(tl && tr && isptr[tl->etype] && isptr[tr->etype]) { + if(tl->type->etype == TSTRUCT && tr->type->etype == TINTER) + fmtprint(&fmt, "\n (*struct vs *interface)"); + else if(tl->type->etype == TINTER && tr->type->etype == TSTRUCT) + fmtprint(&fmt, "\n (*interface vs *struct)"); + } + s = fmtstrflush(&fmt); + yyerror("illegal types for operand: %O%s", o, s); +} + +/* + * iterator to walk a structure declaration + */ +Type* +structfirst(Iter *s, Type **nn) +{ + Type *n, *t; + + n = *nn; + if(n == T) + goto bad; + + switch(n->etype) { + default: + goto bad; + + case TSTRUCT: + case TINTER: + case TFUNC: + break; + } + + t = n->type; + if(t == T) + goto rnil; + + if(t->etype != TFIELD) + fatal("structfirst: not field %T", t); + + s->t = t; + return t; + +bad: + fatal("structfirst: not struct %T", n); + +rnil: + return T; +} + +Type* +structnext(Iter *s) +{ + Type *n, *t; + + n = s->t; + t = n->down; + if(t == T) + goto rnil; + + if(t->etype != TFIELD) + goto bad; + + s->t = t; + return t; + +bad: + fatal("structnext: not struct %T", n); + +rnil: + return T; +} + +/* + * iterator to this and inargs in a function + */ +Type* +funcfirst(Iter *s, Type *t) +{ + Type *fp; + + if(t == T) + goto bad; + + if(t->etype != TFUNC) + goto bad; + + s->tfunc = t; + s->done = 0; + fp = structfirst(s, getthis(t)); + if(fp == T) { + s->done = 1; + fp = structfirst(s, getinarg(t)); + } + return fp; + +bad: + fatal("funcfirst: not func %T", t); + return T; +} + +Type* +funcnext(Iter *s) +{ + Type *fp; + + fp = structnext(s); + if(fp == T && !s->done) { + s->done = 1; + fp = structfirst(s, getinarg(s->tfunc)); + } + return fp; +} + +Type** +getthis(Type *t) +{ + if(t->etype != TFUNC) + fatal("getthis: not a func %T", t); + return &t->type; +} + +Type** +getoutarg(Type *t) +{ + if(t->etype != TFUNC) + fatal("getoutarg: not a func %T", t); + return &t->type->down; +} + +Type** +getinarg(Type *t) +{ + if(t->etype != TFUNC) + fatal("getinarg: not a func %T", t); + return &t->type->down->down; +} + +Type* +getthisx(Type *t) +{ + return *getthis(t); +} + +Type* +getoutargx(Type *t) +{ + return *getoutarg(t); +} + +Type* +getinargx(Type *t) +{ + return *getinarg(t); +} + +/* + * return !(op) + * eg == <=> != + */ +int +brcom(int a) +{ + switch(a) { + case OEQ: return ONE; + case ONE: return OEQ; + case OLT: return OGE; + case OGT: return OLE; + case OLE: return OGT; + case OGE: return OLT; + } + fatal("brcom: no com for %A\n", a); + return a; +} + +/* + * return reverse(op) + * eg a op b <=> b r(op) a + */ +int +brrev(int a) +{ + switch(a) { + case OEQ: return OEQ; + case ONE: return ONE; + case OLT: return OGT; + case OGT: return OLT; + case OLE: return OGE; + case OGE: return OLE; + } + fatal("brcom: no rev for %A\n", a); + return a; +} + +/* + * return side effect-free n, appending side effects to init. + * result is assignable if n is. + */ +Node* +safeexpr(Node *n, NodeList **init) +{ + Node *l; + Node *r; + Node *a; + + if(n == N) + return N; + + switch(n->op) { + case ONAME: + case OLITERAL: + return n; + + case ODOT: + l = safeexpr(n->left, init); + if(l == n->left) + return n; + r = nod(OXXX, N, N); + *r = *n; + r->left = l; + typecheck(&r, Erv); + walkexpr(&r, init); + return r; + + case ODOTPTR: + case OIND: + l = safeexpr(n->left, init); + if(l == n->left) + return n; + a = nod(OXXX, N, N); + *a = *n; + a->left = l; + walkexpr(&a, init); + return a; + + case OINDEX: + case OINDEXMAP: + l = safeexpr(n->left, init); + r = safeexpr(n->right, init); + if(l == n->left && r == n->right) + return n; + a = nod(OXXX, N, N); + *a = *n; + a->left = l; + a->right = r; + walkexpr(&a, init); + return a; + } + + // make a copy; must not be used as an lvalue + if(islvalue(n)) + fatal("missing lvalue case in safeexpr: %N", n); + return cheapexpr(n, init); +} + +static Node* +copyexpr(Node *n, Type *t, NodeList **init) +{ + Node *a, *l; + + l = nod(OXXX, N, N); + tempname(l, t); + a = nod(OAS, l, n); + typecheck(&a, Etop); + walkexpr(&a, init); + *init = list(*init, a); + return l; +} + +/* + * return side-effect free and cheap n, appending side effects to init. + * result may not be assignable. + */ +Node* +cheapexpr(Node *n, NodeList **init) +{ + switch(n->op) { + case ONAME: + case OLITERAL: + return n; + } + + return copyexpr(n, n->type, init); +} + +/* + * return n in a local variable of type t if it is not already. + */ +Node* +localexpr(Node *n, Type *t, NodeList **init) +{ + if(n->op == ONAME && + (n->class == PAUTO || n->class == PPARAM || n->class == PPARAMOUT) && + convertop(n->type, t, nil) == OCONVNOP) + return n; + + return copyexpr(n, t, init); +} + +void +setmaxarg(Type *t) +{ + int32 w; + + dowidth(t); + w = t->argwid; + if(t->argwid >= MAXWIDTH) + fatal("bad argwid %T", t); + if(w > maxarg) + maxarg = w; +} + +/* unicode-aware case-insensitive strcmp */ + +static int +cistrcmp(char *p, char *q) +{ + Rune rp, rq; + + while(*p || *q) { + if(*p == 0) + return +1; + if(*q == 0) + return -1; + p += chartorune(&rp, p); + q += chartorune(&rq, q); + rp = tolowerrune(rp); + rq = tolowerrune(rq); + if(rp < rq) + return -1; + if(rp > rq) + return +1; + } + return 0; +} + +/* + * code to resolve elided DOTs + * in embedded types + */ + +// search depth 0 -- +// return count of fields+methods +// found with a given name +static int +lookdot0(Sym *s, Type *t, Type **save, int ignorecase) +{ + Type *f, *u; + int c; + + u = t; + if(isptr[u->etype]) + u = u->type; + + c = 0; + if(u->etype == TSTRUCT || u->etype == TINTER) { + for(f=u->type; f!=T; f=f->down) + if(f->sym == s || (ignorecase && cistrcmp(f->sym->name, s->name) == 0)) { + if(save) + *save = f; + c++; + } + } + u = methtype(t); + if(u != T) { + for(f=u->method; f!=T; f=f->down) + if(f->embedded == 0 && (f->sym == s || (ignorecase && cistrcmp(f->sym->name, s->name) == 0))) { + if(save) + *save = f; + c++; + } + } + return c; +} + +// search depth d -- +// return count of fields+methods +// found at search depth. +// answer is in dotlist array and +// count of number of ways is returned. +int +adddot1(Sym *s, Type *t, int d, Type **save, int ignorecase) +{ + Type *f, *u; + int c, a; + + if(t->trecur) + return 0; + t->trecur = 1; + + if(d == 0) { + c = lookdot0(s, t, save, ignorecase); + goto out; + } + + c = 0; + u = t; + if(isptr[u->etype]) + u = u->type; + if(u->etype != TSTRUCT && u->etype != TINTER) + goto out; + + d--; + for(f=u->type; f!=T; f=f->down) { + if(!f->embedded) + continue; + if(f->sym == S) + continue; + a = adddot1(s, f->type, d, save, ignorecase); + if(a != 0 && c == 0) + dotlist[d].field = f; + c += a; + } + +out: + t->trecur = 0; + return c; +} + +// in T.field +// find missing fields that +// will give shortest unique addressing. +// modify the tree with missing type names. +Node* +adddot(Node *n) +{ + Type *t; + Sym *s; + int c, d; + + typecheck(&n->left, Etype|Erv); + t = n->left->type; + if(t == T) + goto ret; + + if(n->left->op == OTYPE) + goto ret; + + if(n->right->op != ONAME) + goto ret; + s = n->right->sym; + if(s == S) + goto ret; + + for(d=0; d 0) + goto out; + } + goto ret; + +out: + if(c > 1) + yyerror("ambiguous DOT reference %T.%S", t, s); + + // rebuild elided dots + for(c=d-1; c>=0; c--) + n->left = nod(ODOT, n->left, newname(dotlist[c].field->sym)); +ret: + return n; +} + + +/* + * code to help generate trampoline + * functions for methods on embedded + * subtypes. + * these are approx the same as + * the corresponding adddot routines + * except that they expect to be called + * with unique tasks and they return + * the actual methods. + */ + +typedef struct Symlink Symlink; +struct Symlink +{ + Type* field; + uchar good; + uchar followptr; + Symlink* link; +}; +static Symlink* slist; + +static void +expand0(Type *t, int followptr) +{ + Type *f, *u; + Symlink *sl; + + u = t; + if(isptr[u->etype]) { + followptr = 1; + u = u->type; + } + + if(u->etype == TINTER) { + for(f=u->type; f!=T; f=f->down) { + if(!exportname(f->sym->name) && f->sym->pkg != localpkg) + continue; + if(f->sym->flags & SymUniq) + continue; + f->sym->flags |= SymUniq; + sl = mal(sizeof(*sl)); + sl->field = f; + sl->link = slist; + sl->followptr = followptr; + slist = sl; + } + return; + } + + u = methtype(t); + if(u != T) { + for(f=u->method; f!=T; f=f->down) { + if(!exportname(f->sym->name) && f->sym->pkg != localpkg) + continue; + if(f->sym->flags & SymUniq) + continue; + f->sym->flags |= SymUniq; + sl = mal(sizeof(*sl)); + sl->field = f; + sl->link = slist; + sl->followptr = followptr; + slist = sl; + } + } +} + +static void +expand1(Type *t, int d, int followptr) +{ + Type *f, *u; + + if(t->trecur) + return; + if(d == 0) + return; + t->trecur = 1; + + if(d != nelem(dotlist)-1) + expand0(t, followptr); + + u = t; + if(isptr[u->etype]) { + followptr = 1; + u = u->type; + } + if(u->etype != TSTRUCT && u->etype != TINTER) + goto out; + + for(f=u->type; f!=T; f=f->down) { + if(!f->embedded) + continue; + if(f->sym == S) + continue; + expand1(f->type, d-1, followptr); + } + +out: + t->trecur = 0; +} + +void +expandmeth(Sym *s, Type *t) +{ + Symlink *sl; + Type *f; + int c, d; + + if(s == S) + return; + if(t == T || t->xmethod != nil) + return; + + // mark top-level method symbols + // so that expand1 doesn't consider them. + for(f=t->method; f != nil; f=f->down) + f->sym->flags |= SymUniq; + + // generate all reachable methods + slist = nil; + expand1(t, nelem(dotlist)-1, 0); + + // check each method to be uniquely reachable + for(sl=slist; sl!=nil; sl=sl->link) { + sl->field->sym->flags &= ~SymUniq; + for(d=0; dfield->sym, t, d, &f, 0); + if(c == 0) + continue; + if(c == 1) { + sl->good = 1; + sl->field = f; + } + break; + } + } + + for(f=t->method; f != nil; f=f->down) + f->sym->flags &= ~SymUniq; + + t->xmethod = t->method; + for(sl=slist; sl!=nil; sl=sl->link) { + if(sl->good) { + // add it to the base type method list + f = typ(TFIELD); + *f = *sl->field; + f->embedded = 1; // needs a trampoline + if(sl->followptr) + f->embedded = 2; + f->down = t->xmethod; + t->xmethod = f; + } + } +} + +/* + * Given funarg struct list, return list of ODCLFIELD Node fn args. + */ +static NodeList* +structargs(Type **tl, int mustname) +{ + Iter savet; + Node *a, *n; + NodeList *args; + Type *t; + char buf[100]; + int gen; + + args = nil; + gen = 0; + for(t = structfirst(&savet, tl); t != T; t = structnext(&savet)) { + n = N; + if(t->sym) + n = newname(t->sym); + else if(mustname) { + // have to give it a name so we can refer to it in trampoline + snprint(buf, sizeof buf, ".anon%d", gen++); + n = newname(lookup(buf)); + } + a = nod(ODCLFIELD, n, typenod(t->type)); + a->isddd = t->isddd; + if(n != N) + n->isddd = t->isddd; + args = list(args, a); + } + return args; +} + +/* + * Generate a wrapper function to convert from + * a receiver of type T to a receiver of type U. + * That is, + * + * func (t T) M() { + * ... + * } + * + * already exists; this function generates + * + * func (u U) M() { + * u.M() + * } + * + * where the types T and U are such that u.M() is valid + * and calls the T.M method. + * The resulting function is for use in method tables. + * + * rcvr - U + * method - M func (t T)(), a TFIELD type struct + * newnam - the eventual mangled name of this function + */ +void +genwrapper(Type *rcvr, Type *method, Sym *newnam, int iface) +{ + Node *this, *fn, *call, *n, *t, *pad; + NodeList *l, *args, *in, *out; + Type *tpad; + int isddd; + Val v; + + if(debug['r']) + print("genwrapper rcvrtype=%T method=%T newnam=%S\n", + rcvr, method, newnam); + + lineno = 1; // less confusing than end of input + + dclcontext = PEXTERN; + markdcl(); + + this = nod(ODCLFIELD, newname(lookup(".this")), typenod(rcvr)); + this->left->ntype = this->right; + in = structargs(getinarg(method->type), 1); + out = structargs(getoutarg(method->type), 0); + + fn = nod(ODCLFUNC, N, N); + fn->nname = newname(newnam); + t = nod(OTFUNC, N, N); + l = list1(this); + if(iface && rcvr->width < types[tptr]->width) { + // Building method for interface table and receiver + // is smaller than the single pointer-sized word + // that the interface call will pass in. + // Add a dummy padding argument after the + // receiver to make up the difference. + tpad = typ(TARRAY); + tpad->type = types[TUINT8]; + tpad->bound = types[tptr]->width - rcvr->width; + pad = nod(ODCLFIELD, newname(lookup(".pad")), typenod(tpad)); + l = list(l, pad); + } + t->list = concat(l, in); + t->rlist = out; + fn->nname->ntype = t; + funchdr(fn); + + // arg list + args = nil; + isddd = 0; + for(l=in; l; l=l->next) { + args = list(args, l->n->left); + isddd = l->n->left->isddd; + } + + // generate nil pointer check for better error + if(isptr[rcvr->etype] && rcvr->type == getthisx(method->type)->type->type) { + // generating wrapper from *T to T. + n = nod(OIF, N, N); + n->ntest = nod(OEQ, this->left, nodnil()); + // these strings are already in the reflect tables, + // so no space cost to use them here. + l = nil; + v.ctype = CTSTR; + v.u.sval = strlit(rcvr->type->sym->pkg->name); // package name + l = list(l, nodlit(v)); + v.u.sval = strlit(rcvr->type->sym->name); // type name + l = list(l, nodlit(v)); + v.u.sval = strlit(method->sym->name); + l = list(l, nodlit(v)); // method name + call = nod(OCALL, syslook("panicwrap", 0), N); + call->list = l; + n->nbody = list1(call); + fn->nbody = list(fn->nbody, n); + } + + // generate call + call = nod(OCALL, adddot(nod(OXDOT, this->left, newname(method->sym))), N); + call->list = args; + call->isddd = isddd; + if(method->type->outtuple > 0) { + n = nod(ORETURN, N, N); + n->list = list1(call); + call = n; + } + fn->nbody = list(fn->nbody, call); + + if(0 && debug['r']) + dumplist("genwrapper body", fn->nbody); + + funcbody(fn); + curfn = fn; + typecheck(&fn, Etop); + typechecklist(fn->nbody, Etop); + curfn = nil; + funccompile(fn, 0); +} + +static Type* +ifacelookdot(Sym *s, Type *t, int *followptr, int ignorecase) +{ + int i, c, d; + Type *m; + + *followptr = 0; + + if(t == T) + return T; + + for(d=0; d 1) { + yyerror("%T.%S is ambiguous", t, s); + return T; + } + if(c == 1) { + for(i=0; itype->etype]) { + *followptr = 1; + break; + } + } + if(m->type->etype != TFUNC || m->type->thistuple == 0) { + yyerror("%T.%S is a field, not a method", t, s); + return T; + } + return m; + } + } + return T; +} + +int +implements(Type *t, Type *iface, Type **m, Type **samename, int *ptr) +{ + Type *t0, *im, *tm, *rcvr, *imtype; + int followptr; + + t0 = t; + if(t == T) + return 0; + + // if this is too slow, + // could sort these first + // and then do one loop. + + if(t->etype == TINTER) { + for(im=iface->type; im; im=im->down) { + for(tm=t->type; tm; tm=tm->down) { + if(tm->sym == im->sym) { + if(eqtype(tm->type, im->type)) + goto found; + *m = im; + *samename = tm; + *ptr = 0; + return 0; + } + } + *m = im; + *samename = nil; + *ptr = 0; + return 0; + found:; + } + return 1; + } + + t = methtype(t); + if(t != T) + expandmeth(t->sym, t); + for(im=iface->type; im; im=im->down) { + imtype = methodfunc(im->type, 0); + tm = ifacelookdot(im->sym, t, &followptr, 0); + if(tm == T || !eqtype(methodfunc(tm->type, 0), imtype)) { + if(tm == T) + tm = ifacelookdot(im->sym, t, &followptr, 1); + *m = im; + *samename = tm; + *ptr = 0; + return 0; + } + // if pointer receiver in method, + // the method does not exist for value types. + rcvr = getthisx(tm->type)->type->type; + if(isptr[rcvr->etype] && !isptr[t0->etype] && !followptr && !isifacemethod(tm->type)) { + if(0 && debug['r']) + yyerror("interface pointer mismatch"); + + *m = im; + *samename = nil; + *ptr = 1; + return 0; + } + } + return 1; +} + +/* + * even simpler simtype; get rid of ptr, bool. + * assuming that the front end has rejected + * all the invalid conversions (like ptr -> bool) + */ +int +simsimtype(Type *t) +{ + int et; + + if(t == 0) + return 0; + + et = simtype[t->etype]; + switch(et) { + case TPTR32: + et = TUINT32; + break; + case TPTR64: + et = TUINT64; + break; + case TBOOL: + et = TUINT8; + break; + } + return et; +} + +NodeList* +concat(NodeList *a, NodeList *b) +{ + if(a == nil) + return b; + if(b == nil) + return a; + + a->end->next = b; + a->end = b->end; + b->end = nil; + return a; +} + +NodeList* +list1(Node *n) +{ + NodeList *l; + + if(n == nil) + return nil; + if(n->op == OBLOCK && n->ninit == nil) + return n->list; + l = mal(sizeof *l); + l->n = n; + l->end = l; + return l; +} + +NodeList* +list(NodeList *l, Node *n) +{ + return concat(l, list1(n)); +} + +void +listsort(NodeList** l, int(*f)(Node*, Node*)) +{ + NodeList *l1, *l2, *le; + + if(*l == nil || (*l)->next == nil) + return; + + l1 = *l; + l2 = *l; + for(;;) { + l2 = l2->next; + if(l2 == nil) + break; + l2 = l2->next; + if(l2 == nil) + break; + l1 = l1->next; + } + + l2 = l1->next; + l1->next = nil; + l2->end = (*l)->end; + (*l)->end = l1; + + l1 = *l; + listsort(&l1, f); + listsort(&l2, f); + + if ((*f)(l1->n, l2->n) < 0) { + *l = l1; + } else { + *l = l2; + l2 = l1; + l1 = *l; + } + + // now l1 == *l; and l1 < l2 + + while ((l1 != nil) && (l2 != nil)) { + while ((l1->next != nil) && (*f)(l1->next->n, l2->n) < 0) + l1 = l1->next; + + // l1 is last one from l1 that is < l2 + le = l1->next; // le is the rest of l1, first one that is >= l2 + if (le != nil) + le->end = (*l)->end; + + (*l)->end = l1; // cut *l at l1 + *l = concat(*l, l2); // glue l2 to *l's tail + + l1 = l2; // l1 is the first element of *l that is < the new l2 + l2 = le; // ... because l2 now is the old tail of l1 + } + + *l = concat(*l, l2); // any remainder +} + +NodeList* +listtreecopy(NodeList *l) +{ + NodeList *out; + + out = nil; + for(; l; l=l->next) + out = list(out, treecopy(l->n)); + return out; +} + +Node* +liststmt(NodeList *l) +{ + Node *n; + + n = nod(OBLOCK, N, N); + n->list = l; + if(l) + n->lineno = l->n->lineno; + return n; +} + +/* + * return nelem of list + */ +int +count(NodeList *l) +{ + int n; + + n = 0; + for(; l; l=l->next) + n++; + return n; +} + +/* + * return nelem of list + */ +int +structcount(Type *t) +{ + int v; + Iter s; + + v = 0; + for(t = structfirst(&s, &t); t != T; t = structnext(&s)) + v++; + return v; +} + +/* + * return power of 2 of the constant + * operand. -1 if it is not a power of 2. + * 1000+ if it is a -(power of 2) + */ +int +powtwo(Node *n) +{ + uvlong v, b; + int i; + + if(n == N || n->op != OLITERAL || n->type == T) + goto no; + if(!isint[n->type->etype]) + goto no; + + v = mpgetfix(n->val.u.xval); + b = 1ULL; + for(i=0; i<64; i++) { + if(b == v) + return i; + b = b<<1; + } + + if(!issigned[n->type->etype]) + goto no; + + v = -v; + b = 1ULL; + for(i=0; i<64; i++) { + if(b == v) + return i+1000; + b = b<<1; + } + +no: + return -1; +} + +/* + * return the unsigned type for + * a signed integer type. + * returns T if input is not a + * signed integer type. + */ +Type* +tounsigned(Type *t) +{ + + // this is types[et+1], but not sure + // that this relation is immutable + switch(t->etype) { + default: + print("tounsigned: unknown type %T\n", t); + t = T; + break; + case TINT: + t = types[TUINT]; + break; + case TINT8: + t = types[TUINT8]; + break; + case TINT16: + t = types[TUINT16]; + break; + case TINT32: + t = types[TUINT32]; + break; + case TINT64: + t = types[TUINT64]; + break; + } + return t; +} + +/* + * magic number for signed division + * see hacker's delight chapter 10 + */ +void +smagic(Magic *m) +{ + int p; + uint64 ad, anc, delta, q1, r1, q2, r2, t; + uint64 mask, two31; + + m->bad = 0; + switch(m->w) { + default: + m->bad = 1; + return; + case 8: + mask = 0xffLL; + break; + case 16: + mask = 0xffffLL; + break; + case 32: + mask = 0xffffffffLL; + break; + case 64: + mask = 0xffffffffffffffffLL; + break; + } + two31 = mask ^ (mask>>1); + + p = m->w-1; + ad = m->sd; + if(m->sd < 0) + ad = -m->sd; + + // bad denominators + if(ad == 0 || ad == 1 || ad == two31) { + m->bad = 1; + return; + } + + t = two31; + ad &= mask; + + anc = t - 1 - t%ad; + anc &= mask; + + q1 = two31/anc; + r1 = two31 - q1*anc; + q1 &= mask; + r1 &= mask; + + q2 = two31/ad; + r2 = two31 - q2*ad; + q2 &= mask; + r2 &= mask; + + for(;;) { + p++; + q1 <<= 1; + r1 <<= 1; + q1 &= mask; + r1 &= mask; + if(r1 >= anc) { + q1++; + r1 -= anc; + q1 &= mask; + r1 &= mask; + } + + q2 <<= 1; + r2 <<= 1; + q2 &= mask; + r2 &= mask; + if(r2 >= ad) { + q2++; + r2 -= ad; + q2 &= mask; + r2 &= mask; + } + + delta = ad - r2; + delta &= mask; + if(q1 < delta || (q1 == delta && r1 == 0)) { + continue; + } + break; + } + + m->sm = q2+1; + if(m->sm & two31) + m->sm |= ~mask; + m->s = p-m->w; +} + +/* + * magic number for unsigned division + * see hacker's delight chapter 10 + */ +void +umagic(Magic *m) +{ + int p; + uint64 nc, delta, q1, r1, q2, r2; + uint64 mask, two31; + + m->bad = 0; + m->ua = 0; + + switch(m->w) { + default: + m->bad = 1; + return; + case 8: + mask = 0xffLL; + break; + case 16: + mask = 0xffffLL; + break; + case 32: + mask = 0xffffffffLL; + break; + case 64: + mask = 0xffffffffffffffffLL; + break; + } + two31 = mask ^ (mask>>1); + + m->ud &= mask; + if(m->ud == 0 || m->ud == two31) { + m->bad = 1; + return; + } + nc = mask - (-m->ud&mask)%m->ud; + p = m->w-1; + + q1 = two31/nc; + r1 = two31 - q1*nc; + q1 &= mask; + r1 &= mask; + + q2 = (two31-1) / m->ud; + r2 = (two31-1) - q2*m->ud; + q2 &= mask; + r2 &= mask; + + for(;;) { + p++; + if(r1 >= nc-r1) { + q1 <<= 1; + q1++; + r1 <<= 1; + r1 -= nc; + } else { + q1 <<= 1; + r1 <<= 1; + } + q1 &= mask; + r1 &= mask; + if(r2+1 >= m->ud-r2) { + if(q2 >= two31-1) { + m->ua = 1; + } + q2 <<= 1; + q2++; + r2 <<= 1; + r2++; + r2 -= m->ud; + } else { + if(q2 >= two31) { + m->ua = 1; + } + q2 <<= 1; + r2 <<= 1; + r2++; + } + q2 &= mask; + r2 &= mask; + + delta = m->ud - 1 - r2; + delta &= mask; + + if(p < m->w+m->w) + if(q1 < delta || (q1 == delta && r1 == 0)) { + continue; + } + break; + } + m->um = q2+1; + m->s = p-m->w; +} + +Sym* +ngotype(Node *n) +{ + if(n->sym != S && n->realtype != T) + if(strncmp(n->sym->name, "autotmp_", 8) != 0) + if(strncmp(n->sym->name, "statictmp_", 8) != 0) + return typename(n->realtype)->left->sym; + + return S; +} + +/* + * Convert raw string to the prefix that will be used in the symbol table. + * Invalid bytes turn into %xx. Right now the only bytes that need + * escaping are %, ., and ", but we escape all control characters too. + */ +static char* +pathtoprefix(char *s) +{ + static char hex[] = "0123456789abcdef"; + char *p, *r, *w; + int n; + + // check for chars that need escaping + n = 0; + for(r=s; *r; r++) + if(*r <= ' ' || *r == '.' || *r == '%' || *r == '"') + n++; + + // quick exit + if(n == 0) + return s; + + // escape + p = mal((r-s)+1+2*n); + for(r=s, w=p; *r; r++) { + if(*r <= ' ' || *r == '.' || *r == '%' || *r == '"') { + *w++ = '%'; + *w++ = hex[(*r>>4)&0xF]; + *w++ = hex[*r&0xF]; + } else + *w++ = *r; + } + *w = '\0'; + return p; +} + +Pkg* +mkpkg(Strlit *path) +{ + Pkg *p; + int h; + + if(strlen(path->s) != path->len) { + yyerror("import path contains NUL byte"); + errorexit(); + } + + h = stringhash(path->s) & (nelem(phash)-1); + for(p=phash[h]; p; p=p->link) + if(p->path->len == path->len && memcmp(path->s, p->path->s, path->len) == 0) + return p; + + p = mal(sizeof *p); + p->path = path; + p->prefix = pathtoprefix(path->s); + p->link = phash[h]; + phash[h] = p; + return p; +} + +Strlit* +strlit(char *s) +{ + Strlit *t; + + t = mal(sizeof *t + strlen(s)); + strcpy(t->s, s); + t->len = strlen(s); + return t; +} diff --git a/src/cmd/gc/swt.c b/src/cmd/gc/swt.c new file mode 100644 index 000000000..c2968c44b --- /dev/null +++ b/src/cmd/gc/swt.c @@ -0,0 +1,896 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +#include "go.h" + +enum +{ + Snorm = 0, + Strue, + Sfalse, + Stype, + + Tdefault, // default case + Texprconst, // normal constant case + Texprvar, // normal variable case + Ttypenil, // case nil + Ttypeconst, // type hashes + Ttypevar, // interface type + + Ncase = 4, // count needed to split +}; + +typedef struct Case Case; +struct Case +{ + Node* node; // points at case statement + uint32 hash; // hash of a type switch + uint8 type; // type of case + uint8 diag; // suppress multiple diagnostics + uint16 ordinal; // position in switch + Case* link; // linked list to link +}; +#define C ((Case*)nil) + +void +dumpcase(Case *c0) +{ + Case *c; + + for(c=c0; c!=C; c=c->link) { + switch(c->type) { + case Tdefault: + print("case-default\n"); + print(" ord=%d\n", c->ordinal); + break; + case Texprconst: + print("case-exprconst\n"); + print(" ord=%d\n", c->ordinal); + break; + case Texprvar: + print("case-exprvar\n"); + print(" ord=%d\n", c->ordinal); + print(" op=%O\n", c->node->left->op); + break; + case Ttypenil: + print("case-typenil\n"); + print(" ord=%d\n", c->ordinal); + break; + case Ttypeconst: + print("case-typeconst\n"); + print(" ord=%d\n", c->ordinal); + print(" hash=%ux\n", c->hash); + break; + case Ttypevar: + print("case-typevar\n"); + print(" ord=%d\n", c->ordinal); + break; + default: + print("case-???\n"); + print(" ord=%d\n", c->ordinal); + print(" op=%O\n", c->node->left->op); + print(" hash=%ux\n", c->hash); + break; + } + } + print("\n"); +} + +static int +ordlcmp(Case *c1, Case *c2) +{ + // sort default first + if(c1->type == Tdefault) + return -1; + if(c2->type == Tdefault) + return +1; + + // sort nil second + if(c1->type == Ttypenil) + return -1; + if(c2->type == Ttypenil) + return +1; + + // sort by ordinal + if(c1->ordinal > c2->ordinal) + return +1; + if(c1->ordinal < c2->ordinal) + return -1; + return 0; +} + +static int +exprcmp(Case *c1, Case *c2) +{ + int ct, n; + Node *n1, *n2; + + // sort non-constants last + if(c1->type != Texprconst) + return +1; + if(c2->type != Texprconst) + return -1; + + n1 = c1->node->left; + n2 = c2->node->left; + + ct = n1->val.ctype; + if(ct != n2->val.ctype) { + // invalid program, but return a sort + // order so that we can give a better + // error later. + return ct - n2->val.ctype; + } + + // sort by constant value + n = 0; + switch(ct) { + case CTFLT: + n = mpcmpfltflt(n1->val.u.fval, n2->val.u.fval); + break; + case CTINT: + n = mpcmpfixfix(n1->val.u.xval, n2->val.u.xval); + break; + case CTSTR: + n = cmpslit(n1, n2); + break; + } + + return n; +} + +static int +typecmp(Case *c1, Case *c2) +{ + + // sort non-constants last + if(c1->type != Ttypeconst) + return +1; + if(c2->type != Ttypeconst) + return -1; + + // sort by hash code + if(c1->hash > c2->hash) + return +1; + if(c1->hash < c2->hash) + return -1; + + // sort by ordinal so duplicate error + // happens on later case. + if(c1->ordinal > c2->ordinal) + return +1; + if(c1->ordinal < c2->ordinal) + return -1; + return 0; +} + +static Case* +csort(Case *l, int(*f)(Case*, Case*)) +{ + Case *l1, *l2, *le; + + if(l == C || l->link == C) + return l; + + l1 = l; + l2 = l; + for(;;) { + l2 = l2->link; + if(l2 == C) + break; + l2 = l2->link; + if(l2 == C) + break; + l1 = l1->link; + } + + l2 = l1->link; + l1->link = C; + l1 = csort(l, f); + l2 = csort(l2, f); + + /* set up lead element */ + if((*f)(l1, l2) < 0) { + l = l1; + l1 = l1->link; + } else { + l = l2; + l2 = l2->link; + } + le = l; + + for(;;) { + if(l1 == C) { + while(l2) { + le->link = l2; + le = l2; + l2 = l2->link; + } + le->link = C; + break; + } + if(l2 == C) { + while(l1) { + le->link = l1; + le = l1; + l1 = l1->link; + } + break; + } + if((*f)(l1, l2) < 0) { + le->link = l1; + le = l1; + l1 = l1->link; + } else { + le->link = l2; + le = l2; + l2 = l2->link; + } + } + le->link = C; + return l; +} + +static Node* +newlabel(void) +{ + static int label; + + label++; + snprint(namebuf, sizeof(namebuf), "%.6d", label); + return newname(lookup(namebuf)); +} + +/* + * build separate list of statements and cases + * make labels between cases and statements + * deal with fallthrough, break, unreachable statements + */ +static void +casebody(Node *sw, Node *typeswvar) +{ + Node *n, *c, *last; + Node *def; + NodeList *cas, *stat, *l, *lc; + Node *go, *br; + int32 lno, needvar; + + lno = setlineno(sw); + if(sw->list == nil) + return; + + cas = nil; // cases + stat = nil; // statements + def = N; // defaults + br = nod(OBREAK, N, N); + + for(l=sw->list; l; l=l->next) { + n = l->n; + lno = setlineno(n); + if(n->op != OXCASE) + fatal("casebody %O", n->op); + n->op = OCASE; + needvar = count(n->list) != 1 || n->list->n->op == OLITERAL; + + go = nod(OGOTO, newlabel(), N); + if(n->list == nil) { + if(def != N) + yyerror("more than one default case"); + // reuse original default case + n->right = go; + def = n; + } + + if(n->list != nil && n->list->next == nil) { + // one case - reuse OCASE node. + c = n->list->n; + n->left = c; + n->right = go; + n->list = nil; + cas = list(cas, n); + } else { + // expand multi-valued cases + for(lc=n->list; lc; lc=lc->next) { + c = lc->n; + cas = list(cas, nod(OCASE, c, go)); + } + } + + stat = list(stat, nod(OLABEL, go->left, N)); + if(typeswvar && needvar && n->nname != N) { + NodeList *l; + + l = list1(nod(ODCL, n->nname, N)); + l = list(l, nod(OAS, n->nname, typeswvar)); + typechecklist(l, Etop); + stat = concat(stat, l); + } + stat = concat(stat, n->nbody); + + // botch - shouldnt fall thru declaration + last = stat->end->n; + if(last->op == OXFALL) { + if(typeswvar) { + setlineno(last); + yyerror("cannot fallthrough in type switch"); + } + last->op = OFALL; + } else + stat = list(stat, br); + } + + stat = list(stat, br); + if(def) + cas = list(cas, def); + + sw->list = cas; + sw->nbody = stat; + lineno = lno; +} + +static Case* +mkcaselist(Node *sw, int arg) +{ + Node *n; + Case *c, *c1, *c2; + NodeList *l; + int ord; + + c = C; + ord = 0; + + for(l=sw->list; l; l=l->next) { + n = l->n; + c1 = mal(sizeof(*c1)); + c1->link = c; + c = c1; + + ord++; + c->ordinal = ord; + c->node = n; + + if(n->left == N) { + c->type = Tdefault; + continue; + } + + switch(arg) { + case Stype: + c->hash = 0; + if(n->left->op == OLITERAL) { + c->type = Ttypenil; + continue; + } + if(istype(n->left->type, TINTER)) { + c->type = Ttypevar; + continue; + } + + c->hash = typehash(n->left->type); + c->type = Ttypeconst; + continue; + + case Snorm: + case Strue: + case Sfalse: + c->type = Texprvar; + switch(consttype(n->left)) { + case CTFLT: + case CTINT: + case CTSTR: + c->type = Texprconst; + } + continue; + } + } + + if(c == C) + return C; + + // sort by value and diagnose duplicate cases + switch(arg) { + case Stype: + c = csort(c, typecmp); + for(c1=c; c1!=C; c1=c1->link) { + for(c2=c1->link; c2!=C && c2->hash==c1->hash; c2=c2->link) { + if(c1->type == Ttypenil || c1->type == Tdefault) + break; + if(c2->type == Ttypenil || c2->type == Tdefault) + break; + if(!eqtype(c1->node->left->type, c2->node->left->type)) + continue; + yyerrorl(c2->node->lineno, "duplicate case in switch\n\tprevious case at %L", c1->node->lineno); + } + } + break; + case Snorm: + case Strue: + case Sfalse: + c = csort(c, exprcmp); + for(c1=c; c1->link!=C; c1=c1->link) { + if(exprcmp(c1, c1->link) != 0) + continue; + setlineno(c1->link->node); + yyerror("duplicate case in switch\n\tprevious case at %L", c1->node->lineno); + } + break; + } + + // put list back in processing order + c = csort(c, ordlcmp); + return c; +} + +static Node* exprname; + +static Node* +exprbsw(Case *c0, int ncase, int arg) +{ + NodeList *cas; + Node *a, *n; + Case *c; + int i, half, lno; + + cas = nil; + if(ncase < Ncase) { + for(i=0; inode; + lno = setlineno(n); + + switch(arg) { + case Strue: + a = nod(OIF, N, N); + a->ntest = n->left; // if val + a->nbody = list1(n->right); // then goto l + break; + + case Sfalse: + a = nod(OIF, N, N); + a->ntest = nod(ONOT, n->left, N); // if !val + typecheck(&a->ntest, Erv); + a->nbody = list1(n->right); // then goto l + break; + + default: + a = nod(OIF, N, N); + a->ntest = nod(OEQ, exprname, n->left); // if name == val + typecheck(&a->ntest, Erv); + a->nbody = list1(n->right); // then goto l + break; + } + + cas = list(cas, a); + c0 = c0->link; + lineno = lno; + } + return liststmt(cas); + } + + // find the middle and recur + c = c0; + half = ncase>>1; + for(i=1; ilink; + a = nod(OIF, N, N); + a->ntest = nod(OLE, exprname, c->node->left); + typecheck(&a->ntest, Erv); + a->nbody = list1(exprbsw(c0, half, arg)); + a->nelse = list1(exprbsw(c->link, ncase-half, arg)); + return a; +} + +/* + * normal (expression) switch. + * rebulid case statements into if .. goto + */ +static void +exprswitch(Node *sw) +{ + Node *def; + NodeList *cas; + Node *a; + Case *c0, *c, *c1; + Type *t; + int arg, ncase; + + casebody(sw, N); + + arg = Snorm; + if(isconst(sw->ntest, CTBOOL)) { + arg = Strue; + if(sw->ntest->val.u.bval == 0) + arg = Sfalse; + } + walkexpr(&sw->ntest, &sw->ninit); + t = sw->type; + if(t == T) + return; + + /* + * convert the switch into OIF statements + */ + exprname = N; + cas = nil; + if(arg != Strue && arg != Sfalse) { + exprname = nod(OXXX, N, N); + tempname(exprname, sw->ntest->type); + cas = list1(nod(OAS, exprname, sw->ntest)); + typechecklist(cas, Etop); + } + + c0 = mkcaselist(sw, arg); + if(c0 != C && c0->type == Tdefault) { + def = c0->node->right; + c0 = c0->link; + } else { + def = nod(OBREAK, N, N); + } + +loop: + if(c0 == C) { + cas = list(cas, def); + sw->nbody = concat(cas, sw->nbody); + sw->list = nil; + walkstmtlist(sw->nbody); + return; + } + + // deal with the variables one-at-a-time + if(c0->type != Texprconst) { + a = exprbsw(c0, 1, arg); + cas = list(cas, a); + c0 = c0->link; + goto loop; + } + + // do binary search on run of constants + ncase = 1; + for(c=c0; c->link!=C; c=c->link) { + if(c->link->type != Texprconst) + break; + ncase++; + } + + // break the chain at the count + c1 = c->link; + c->link = C; + + // sort and compile constants + c0 = csort(c0, exprcmp); + a = exprbsw(c0, ncase, arg); + cas = list(cas, a); + + c0 = c1; + goto loop; + +} + +static Node* hashname; +static Node* facename; +static Node* boolname; + +static Node* +typeone(Node *t) +{ + NodeList *init; + Node *a, *b, *var; + + var = t->nname; + init = nil; + if(var == N) { + typecheck(&nblank, Erv | Easgn); + var = nblank; + } else + init = list1(nod(ODCL, var, N)); + + a = nod(OAS2, N, N); + a->list = list(list1(var), boolname); // var,bool = + b = nod(ODOTTYPE, facename, N); + b->type = t->left->type; // interface.(type) + a->rlist = list1(b); + typecheck(&a, Etop); + init = list(init, a); + + b = nod(OIF, N, N); + b->ntest = boolname; + b->nbody = list1(t->right); // if bool { goto l } + a = liststmt(list(init, b)); + return a; +} + +static Node* +typebsw(Case *c0, int ncase) +{ + NodeList *cas; + Node *a, *n; + Case *c; + int i, half; + + cas = nil; + + if(ncase < Ncase) { + for(i=0; inode; + if(c0->type != Ttypeconst) + fatal("typebsw"); + a = nod(OIF, N, N); + a->ntest = nod(OEQ, hashname, nodintconst(c0->hash)); + typecheck(&a->ntest, Erv); + a->nbody = list1(n->right); + cas = list(cas, a); + c0 = c0->link; + } + return liststmt(cas); + } + + // find the middle and recur + c = c0; + half = ncase>>1; + for(i=1; ilink; + a = nod(OIF, N, N); + a->ntest = nod(OLE, hashname, nodintconst(c->hash)); + typecheck(&a->ntest, Erv); + a->nbody = list1(typebsw(c0, half)); + a->nelse = list1(typebsw(c->link, ncase-half)); + return a; +} + +/* + * convert switch of the form + * switch v := i.(type) { case t1: ..; case t2: ..; } + * into if statements + */ +static void +typeswitch(Node *sw) +{ + Node *def; + NodeList *cas, *hash; + Node *a, *n; + Case *c, *c0, *c1; + int ncase; + Type *t; + Val v; + + if(sw->ntest == nil) + return; + if(sw->ntest->right == nil) { + setlineno(sw); + yyerror("type switch must have an assignment"); + return; + } + walkexpr(&sw->ntest->right, &sw->ninit); + if(!istype(sw->ntest->right->type, TINTER)) { + yyerror("type switch must be on an interface"); + return; + } + cas = nil; + + /* + * predeclare temporary variables + * and the boolean var + */ + facename = nod(OXXX, N, N); + tempname(facename, sw->ntest->right->type); + a = nod(OAS, facename, sw->ntest->right); + typecheck(&a, Etop); + cas = list(cas, a); + + casebody(sw, facename); + + boolname = nod(OXXX, N, N); + tempname(boolname, types[TBOOL]); + typecheck(&boolname, Erv); + + hashname = nod(OXXX, N, N); + tempname(hashname, types[TUINT32]); + typecheck(&hashname, Erv); + + t = sw->ntest->right->type; + if(isnilinter(t)) + a = syslook("efacethash", 1); + else + a = syslook("ifacethash", 1); + argtype(a, t); + a = nod(OCALL, a, N); + a->list = list1(facename); + a = nod(OAS, hashname, a); + typecheck(&a, Etop); + cas = list(cas, a); + + c0 = mkcaselist(sw, Stype); + if(c0 != C && c0->type == Tdefault) { + def = c0->node->right; + c0 = c0->link; + } else { + def = nod(OBREAK, N, N); + } + + /* + * insert if statement into each case block + */ + for(c=c0; c!=C; c=c->link) { + n = c->node; + switch(c->type) { + + case Ttypenil: + v.ctype = CTNIL; + a = nod(OIF, N, N); + a->ntest = nod(OEQ, facename, nodlit(v)); + typecheck(&a->ntest, Erv); + a->nbody = list1(n->right); // if i==nil { goto l } + n->right = a; + break; + + case Ttypevar: + case Ttypeconst: + n->right = typeone(n); + break; + } + } + + /* + * generate list of if statements, binary search for constant sequences + */ + while(c0 != C) { + if(c0->type != Ttypeconst) { + n = c0->node; + cas = list(cas, n->right); + c0=c0->link; + continue; + } + + // identify run of constants + c1 = c = c0; + while(c->link!=C && c->link->type==Ttypeconst) + c = c->link; + c0 = c->link; + c->link = nil; + + // sort by hash + c1 = csort(c1, typecmp); + + // for debugging: linear search + if(0) { + for(c=c1; c!=C; c=c->link) { + n = c->node; + cas = list(cas, n->right); + } + continue; + } + + // combine adjacent cases with the same hash + ncase = 0; + for(c=c1; c!=C; c=c->link) { + ncase++; + hash = list1(c->node->right); + while(c->link != C && c->link->hash == c->hash) { + hash = list(hash, c->link->node->right); + c->link = c->link->link; + } + c->node->right = liststmt(hash); + } + + // binary search among cases to narrow by hash + cas = list(cas, typebsw(c1, ncase)); + } + if(nerrors == 0) { + cas = list(cas, def); + sw->nbody = concat(cas, sw->nbody); + sw->list = nil; + walkstmtlist(sw->nbody); + } +} + +void +walkswitch(Node *sw) +{ + + /* + * reorder the body into (OLIST, cases, statements) + * cases have OGOTO into statements. + * both have inserted OBREAK statements + */ + walkstmtlist(sw->ninit); + if(sw->ntest == N) { + sw->ntest = nodbool(1); + typecheck(&sw->ntest, Erv); + } + + if(sw->ntest->op == OTYPESW) { + typeswitch(sw); +//dump("sw", sw); + return; + } + exprswitch(sw); +} + +/* + * type check switch statement + */ +void +typecheckswitch(Node *n) +{ + int top, lno; + Type *t; + NodeList *l, *ll; + Node *ncase, *nvar; + Node *def; + + lno = lineno; + typechecklist(n->ninit, Etop); + + if(n->ntest != N && n->ntest->op == OTYPESW) { + // type switch + top = Etype; + typecheck(&n->ntest->right, Erv); + t = n->ntest->right->type; + if(t != T && t->etype != TINTER) + yyerror("cannot type switch on non-interface value %+N", n->ntest->right); + } else { + // value switch + top = Erv; + if(n->ntest) { + typecheck(&n->ntest, Erv); + defaultlit(&n->ntest, T); + t = n->ntest->type; + } else + t = types[TBOOL]; + } + n->type = t; + + def = N; + for(l=n->list; l; l=l->next) { + ncase = l->n; + setlineno(n); + if(ncase->list == nil) { + // default + if(def != N) + yyerror("multiple defaults in switch (first at %L)", def->lineno); + else + def = ncase; + } else { + for(ll=ncase->list; ll; ll=ll->next) { + setlineno(ll->n); + typecheck(&ll->n, Erv | Etype); + if(ll->n->type == T || t == T) + continue; + switch(top) { + case Erv: // expression switch + defaultlit(&ll->n, t); + if(ll->n->op == OTYPE) + yyerror("type %T is not an expression", ll->n->type); + else if(ll->n->type != T && !eqtype(ll->n->type, t)) + yyerror("case %+N in %T switch", ll->n, t); + break; + case Etype: // type switch + if(ll->n->op == OLITERAL && istype(ll->n->type, TNIL)) + ; + else if(ll->n->op != OTYPE && ll->n->type != T) { + yyerror("%#N is not a type", ll->n); + // reset to original type + ll->n = n->ntest->right; + } + break; + } + } + } + if(top == Etype && n->type != T) { + ll = ncase->list; + nvar = ncase->nname; + if(nvar != N) { + if(ll && ll->next == nil && ll->n->type != T && !istype(ll->n->type, TNIL)) { + // single entry type switch + nvar->ntype = typenod(ll->n->type); + } else { + // multiple entry type switch or default + nvar->ntype = typenod(n->type); + } + } + } + typechecklist(ncase->nbody, Etop); + } + + lineno = lno; +} diff --git a/src/cmd/gc/typecheck.c b/src/cmd/gc/typecheck.c new file mode 100644 index 000000000..78cdb5bf2 --- /dev/null +++ b/src/cmd/gc/typecheck.c @@ -0,0 +1,2827 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +/* + * type check the whole tree of an expression. + * calculates expression types. + * evaluates compile time constants. + * marks variables that escape the local frame. + * rewrites n->op to be more specific in some cases. + */ + +#include "go.h" + +static void implicitstar(Node**); +static int onearg(Node*, char*, ...); +static int twoarg(Node*); +static int lookdot(Node*, Type*, int); +static int looktypedot(Node*, Type*, int); +static void typecheckaste(int, Node*, int, Type*, NodeList*, char*); +static Type* lookdot1(Sym *s, Type *t, Type *f, int); +static int nokeys(NodeList*); +static void typecheckcomplit(Node**); +static void addrescapes(Node*); +static void typecheckas2(Node*); +static void typecheckas(Node*); +static void typecheckfunc(Node*); +static void checklvalue(Node*, char*); +static void checkassign(Node*); +static void checkassignlist(NodeList*); +static void stringtoarraylit(Node**); +static Node* resolve(Node*); +static Type* getforwtype(Node*); + +static NodeList* typecheckdefstack; + +/* + * resolve ONONAME to definition, if any. + */ +Node* +resolve(Node *n) +{ + Node *r; + + if(n != N && n->op == ONONAME && (r = n->sym->def) != N) { + if(r->op != OIOTA) + n = r; + else if(n->iota >= 0) + n = nodintconst(n->iota); + } + return n; +} + +void +typechecklist(NodeList *l, int top) +{ + for(; l; l=l->next) + typecheck(&l->n, top); +} + +static char* _typekind[] = { + [TINT] = "int", + [TUINT] = "uint", + [TINT8] = "int8", + [TUINT8] = "uint8", + [TINT16] = "int16", + [TUINT16] = "uint16", + [TINT32] = "int32", + [TUINT32] = "uint32", + [TINT64] = "int64", + [TUINT64] = "uint64", + [TUINTPTR] = "uintptr", + [TCOMPLEX64] = "complex64", + [TCOMPLEX128] = "complex128", + [TFLOAT32] = "float32", + [TFLOAT64] = "float64", + [TBOOL] = "bool", + [TSTRING] = "string", + [TPTR32] = "pointer", + [TPTR64] = "pointer", + [TSTRUCT] = "struct", + [TINTER] = "interface", + [TCHAN] = "chan", + [TMAP] = "map", + [TARRAY] = "array", + [TFUNC] = "func", + [TNIL] = "nil", + [TIDEAL] = "ideal number", +}; + +static char* +typekind(int et) +{ + static char buf[50]; + char *s; + + if(0 <= et && et < nelem(_typekind) && (s=_typekind[et]) != nil) + return s; + snprint(buf, sizeof buf, "etype=%d", et); + return buf; +} + +/* + * type check node *np. + * replaces *np with a new pointer in some cases. + * returns the final value of *np as a convenience. + */ +Node* +typecheck(Node **np, int top) +{ + int et, aop, op, ptr; + Node *n, *l, *r; + NodeList *args; + int lno, ok, ntop; + Type *t, *tp, *ft, *missing, *have; + Sym *sym; + Val v; + char *why; + + // cannot type check until all the source has been parsed + if(!typecheckok) + fatal("early typecheck"); + + n = *np; + if(n == N) + return N; + + lno = setlineno(n); + + // Skip over parens. + while(n->op == OPAREN) + n = n->left; + + // Resolve definition of name and value of iota lazily. + n = resolve(n); + + *np = n; + + // Skip typecheck if already done. + // But re-typecheck ONAME/OTYPE/OLITERAL/OPACK node in case context has changed. + if(n->typecheck == 1) { + switch(n->op) { + case ONAME: + case OTYPE: + case OLITERAL: + case OPACK: + break; + default: + lineno = lno; + return n; + } + } + + if(n->typecheck == 2) { + yyerror("typechecking loop"); + lineno = lno; + return n; + } + n->typecheck = 2; + + if(n->sym) { + if(n->op == ONAME && n->etype != 0 && !(top & Ecall)) { + yyerror("use of builtin %S not in function call", n->sym); + goto error; + } + + // a dance to handle forward-declared recursive pointer types. + if(n->op == OTYPE && (ft = getforwtype(n->ntype)) != T) + defertypecopy(n, ft); + + typecheckdef(n); + n->realtype = n->type; + if(n->op == ONONAME) + goto error; + } + *np = n; + +reswitch: + ok = 0; + switch(n->op) { + default: + // until typecheck is complete, do nothing. + dump("typecheck", n); + fatal("typecheck %O", n->op); + + /* + * names + */ + case OLITERAL: + ok |= Erv; + if(n->type == T && n->val.ctype == CTSTR) + n->type = idealstring; + goto ret; + + case ONONAME: + ok |= Erv; + goto ret; + + case ONAME: + if(n->etype != 0) { + ok |= Ecall; + goto ret; + } + if(!(top & Easgn)) { + // not a write to the variable + if(isblank(n)) { + yyerror("cannot use _ as value"); + goto error; + } + n->used = 1; + } + ok |= Erv; + goto ret; + + case OPACK: + yyerror("use of package %S not in selector", n->sym); + goto error; + + case ODDD: + break; + + /* + * types (OIND is with exprs) + */ + case OTYPE: + ok |= Etype; + if(n->type == T) + goto error; + break; + + case OTPAREN: + ok |= Etype; + l = typecheck(&n->left, Etype); + if(l->type == T) + goto error; + n->op = OTYPE; + n->type = l->type; + n->left = N; + break; + + case OTARRAY: + ok |= Etype; + t = typ(TARRAY); + l = n->left; + r = n->right; + if(l == nil) { + t->bound = -1; // slice + } else if(l->op == ODDD) { + t->bound = -100; // to be filled in + if(!(top&Ecomplit)) + yyerror("use of [...] array outside of array literal"); + } else { + l = typecheck(&n->left, Erv); + switch(consttype(l)) { + case CTINT: + v = l->val; + break; + case CTFLT: + v = toint(l->val); + break; + default: + yyerror("invalid array bound %#N", l); + goto error; + } + t->bound = mpgetfix(v.u.xval); + if(t->bound < 0) { + yyerror("array bound must be non-negative"); + goto error; + } else + overflow(v, types[TINT]); + } + typecheck(&r, Etype); + if(r->type == T) + goto error; + t->type = r->type; + n->op = OTYPE; + n->type = t; + n->left = N; + n->right = N; + if(t->bound != -100) + checkwidth(t); + break; + + case OTMAP: + ok |= Etype; + l = typecheck(&n->left, Etype); + r = typecheck(&n->right, Etype); + if(l->type == T || r->type == T) + goto error; + n->op = OTYPE; + n->type = maptype(l->type, r->type); + n->left = N; + n->right = N; + break; + + case OTCHAN: + ok |= Etype; + l = typecheck(&n->left, Etype); + if(l->type == T) + goto error; + t = typ(TCHAN); + t->type = l->type; + t->chan = n->etype; + n->op = OTYPE; + n->type = t; + n->left = N; + n->etype = 0; + break; + + case OTSTRUCT: + ok |= Etype; + n->op = OTYPE; + n->type = dostruct(n->list, TSTRUCT); + if(n->type == T) + goto error; + n->list = nil; + break; + + case OTINTER: + ok |= Etype; + n->op = OTYPE; + n->type = dostruct(n->list, TINTER); + if(n->type == T) + goto error; + break; + + case OTFUNC: + ok |= Etype; + n->op = OTYPE; + n->type = functype(n->left, n->list, n->rlist); + if(n->type == T) + goto error; + break; + + /* + * type or expr + */ + case OIND: + ntop = Erv | Etype; + if(!(top & Eaddr)) + ntop |= Eindir; + l = typecheck(&n->left, ntop); + if((t = l->type) == T) + goto error; + if(l->op == OTYPE) { + ok |= Etype; + n->op = OTYPE; + n->type = ptrto(l->type); + n->left = N; + goto ret; + } + if(!isptr[t->etype]) { + yyerror("invalid indirect of %+N", n->left); + goto error; + } + ok |= Erv; + n->type = t->type; + goto ret; + + /* + * arithmetic exprs + */ + case OASOP: + ok |= Etop; + l = typecheck(&n->left, Erv); + checkassign(n->left); + r = typecheck(&n->right, Erv); + if(l->type == T || r->type == T) + goto error; + op = n->etype; + goto arith; + + case OADD: + case OAND: + case OANDAND: + case OANDNOT: + case ODIV: + case OEQ: + case OGE: + case OGT: + case OLE: + case OLT: + case OLSH: + case ORSH: + case OMOD: + case OMUL: + case ONE: + case OOR: + case OOROR: + case OSUB: + case OXOR: + ok |= Erv; + l = typecheck(&n->left, Erv | (top & Eiota)); + r = typecheck(&n->right, Erv | (top & Eiota)); + if(l->type == T || r->type == T) + goto error; + op = n->op; + arith: + if(op == OLSH || op == ORSH) + goto shift; + // ideal mixed with non-ideal + defaultlit2(&l, &r, 0); + n->left = l; + n->right = r; + if(l->type == T || r->type == T) + goto error; + t = l->type; + if(t->etype == TIDEAL) + t = r->type; + et = t->etype; + if(et == TIDEAL) + et = TINT; + if(iscmp[n->op] && t->etype != TIDEAL && !eqtype(l->type, r->type)) { + // comparison is okay as long as one side is + // assignable to the other. convert so they have + // the same type. (the only conversion that isn't + // a no-op is concrete == interface.) + if(r->type->etype != TBLANK && (aop = assignop(l->type, r->type, nil)) != 0) { + l = nod(aop, l, N); + l->type = r->type; + l->typecheck = 1; + n->left = l; + t = l->type; + } else if(l->type->etype != TBLANK && (aop = assignop(r->type, l->type, nil)) != 0) { + r = nod(aop, r, N); + r->type = l->type; + r->typecheck = 1; + n->right = r; + t = r->type; + } + et = t->etype; + } + if(t->etype != TIDEAL && !eqtype(l->type, r->type)) { + defaultlit2(&l, &r, 1); + yyerror("invalid operation: %#N (mismatched types %T and %T)", n, l->type, r->type); + goto error; + } + if(!okfor[op][et]) { + notokfor: + yyerror("invalid operation: %#N (operator %#O not defined on %s)", n, op, typekind(et)); + goto error; + } + // okfor allows any array == array; + // restrict to slice == nil and nil == slice. + if(l->type->etype == TARRAY && !isslice(l->type)) + goto notokfor; + if(r->type->etype == TARRAY && !isslice(r->type)) + goto notokfor; + if(isslice(l->type) && !isnil(l) && !isnil(r)) { + yyerror("invalid operation: %#N (slice can only be compared to nil)", n); + goto error; + } + t = l->type; + if(iscmp[n->op]) { + evconst(n); + t = types[TBOOL]; + if(n->op != OLITERAL) { + defaultlit2(&l, &r, 1); + n->left = l; + n->right = r; + } + } + if(et == TSTRING) { + if(iscmp[n->op]) { + n->etype = n->op; + n->op = OCMPSTR; + } else if(n->op == OADD) + n->op = OADDSTR; + } + if(et == TINTER) { + if(l->op == OLITERAL && l->val.ctype == CTNIL) { + // swap for back end + n->left = r; + n->right = l; + } else if(r->op == OLITERAL && r->val.ctype == CTNIL) { + // leave alone for back end + } else { + n->etype = n->op; + n->op = OCMPIFACE; + } + } + n->type = t; + goto ret; + + shift: + defaultlit(&r, types[TUINT]); + n->right = r; + t = r->type; + if(!isint[t->etype] || issigned[t->etype]) { + yyerror("invalid operation: %#N (shift count type %T, must be unsigned integer)", n, r->type); + goto error; + } + t = l->type; + if(t != T && t->etype != TIDEAL && !isint[t->etype]) { + yyerror("invalid operation: %#N (shift of type %T)", n, t); + goto error; + } + // no defaultlit for left + // the outer context gives the type + n->type = l->type; + goto ret; + + case OCOM: + case OMINUS: + case ONOT: + case OPLUS: + ok |= Erv; + l = typecheck(&n->left, Erv | (top & Eiota)); + if((t = l->type) == T) + goto error; + if(!okfor[n->op][t->etype]) { + yyerror("invalid operation: %#O %T", n->op, t); + goto error; + } + n->type = t; + goto ret; + + /* + * exprs + */ + case OADDR: + ok |= Erv; + typecheck(&n->left, Erv | Eaddr); + if(n->left->type == T) + goto error; + switch(n->left->op) { + case OMAPLIT: + case OSTRUCTLIT: + case OARRAYLIT: + break; + default: + checklvalue(n->left, "take the address of"); + } + defaultlit(&n->left, T); + l = n->left; + if((t = l->type) == T) + goto error; + if(!(top & Eindir) && !n->etype) + addrescapes(n->left); + n->type = ptrto(t); + goto ret; + + case OCOMPLIT: + ok |= Erv; + typecheckcomplit(&n); + if(n->type == T) + goto error; + goto ret; + + case OXDOT: + n = adddot(n); + n->op = ODOT; + // fall through + case ODOT: + typecheck(&n->left, Erv|Etype); + defaultlit(&n->left, T); + l = n->left; + if((t = l->type) == T) + goto error; + if(n->right->op != ONAME) { + yyerror("rhs of . must be a name"); // impossible + goto error; + } + sym = n->right->sym; + if(l->op == OTYPE) { + if(!looktypedot(n, t, 0)) { + if(looktypedot(n, t, 1)) + yyerror("%#N undefined (cannot refer to unexported method %S)", n, n->right->sym); + else + yyerror("%#N undefined (type %T has no method %S)", n, t, n->right->sym); + goto error; + } + if(n->type->etype != TFUNC || n->type->thistuple != 1) { + yyerror("type %T has no method %hS", n->left->type, sym); + n->type = T; + goto error; + } + n->op = ONAME; + n->sym = methodsym(sym, l->type, 0); + n->type = methodfunc(n->type, l->type); + n->xoffset = 0; + n->class = PFUNC; + ok = Erv; + goto ret; + } + tp = t; + if(isptr[t->etype] && t->type->etype != TINTER) { + t = t->type; + if(t == T) + goto error; + n->op = ODOTPTR; + checkwidth(t); + } + if(!lookdot(n, t, 0)) { + if(lookdot(n, t, 1)) + yyerror("%#N undefined (cannot refer to unexported field or method %S)", n, n->right->sym); + else + yyerror("%#N undefined (type %T has no field or method %S)", n, tp, n->right->sym); + goto error; + } + switch(n->op) { + case ODOTINTER: + case ODOTMETH: + ok |= Ecall; + break; + default: + ok |= Erv; + break; + } + goto ret; + + case ODOTTYPE: + ok |= Erv; + typecheck(&n->left, Erv); + defaultlit(&n->left, T); + l = n->left; + if((t = l->type) == T) + goto error; + if(!isinter(t)) { + yyerror("invalid type assertion: %#N (non-interface type %T on left)", n, t); + goto error; + } + if(n->right != N) { + typecheck(&n->right, Etype); + n->type = n->right->type; + n->right = N; + if(n->type == T) + goto error; + } + if(n->type != T && n->type->etype != TINTER) + if(!implements(n->type, t, &missing, &have, &ptr)) { + if(have) + yyerror("impossible type assertion: %+N cannot have dynamic type %T" + " (wrong type for %S method)\n\thave %S%hhT\n\twant %S%hhT", + l, n->type, missing->sym, have->sym, have->type, + missing->sym, missing->type); + else + yyerror("impossible type assertion: %+N cannot have dynamic type %T" + " (missing %S method)", l, n->type, missing->sym); + goto error; + } + goto ret; + + case OINDEX: + ok |= Erv; + typecheck(&n->left, Erv); + defaultlit(&n->left, T); + implicitstar(&n->left); + l = n->left; + typecheck(&n->right, Erv); + r = n->right; + if((t = l->type) == T || r->type == T) + goto error; + switch(t->etype) { + default: + yyerror("invalid operation: %#N (index of type %T)", n, t); + goto error; + + case TARRAY: + defaultlit(&n->right, T); + if(n->right->type != T && !isint[n->right->type->etype]) + yyerror("non-integer array index %#N", n->right); + n->type = t->type; + break; + + case TMAP: + n->etype = 0; + defaultlit(&n->right, t->down); + if(n->right->type != T) + n->right = assignconv(n->right, t->down, "map index"); + n->type = t->type; + n->op = OINDEXMAP; + break; + + case TSTRING: + defaultlit(&n->right, types[TUINT]); + if(n->right->type != T && !isint[n->right->type->etype]) + yyerror("non-integer string index %#N", n->right); + n->type = types[TUINT8]; + break; + } + goto ret; + + case ORECV: + ok |= Etop | Erv; + typecheck(&n->left, Erv); + defaultlit(&n->left, T); + l = n->left; + if((t = l->type) == T) + goto error; + if(t->etype != TCHAN) { + yyerror("invalid operation: %#N (receive from non-chan type %T)", n, t); + goto error; + } + if(!(t->chan & Crecv)) { + yyerror("invalid operation: %#N (receive from send-only type %T)", n, t); + goto error; + } + n->type = t->type; + goto ret; + + case OSEND: + if(top & Erv) { + yyerror("send statement %#N used as value; use select for non-blocking send", n); + goto error; + } + ok |= Etop | Erv; + l = typecheck(&n->left, Erv); + typecheck(&n->right, Erv); + defaultlit(&n->left, T); + l = n->left; + if((t = l->type) == T) + goto error; + if(t->etype != TCHAN) { + yyerror("invalid operation: %#N (send to non-chan type %T)", n, t); + goto error; + } + if(!(t->chan & Csend)) { + yyerror("invalid operation: %#N (send to receive-only type %T)", n, t); + goto error; + } + defaultlit(&n->right, t->type); + r = n->right; + if((t = r->type) == T) + goto error; + r = assignconv(r, l->type->type, "send"); + // TODO: more aggressive + n->etype = 0; + n->type = T; + goto ret; + + case OSLICE: + ok |= Erv; + typecheck(&n->left, top); + typecheck(&n->right->left, Erv); + typecheck(&n->right->right, Erv); + defaultlit(&n->left, T); + defaultlit(&n->right->left, T); + defaultlit(&n->right->right, T); + if(isfixedarray(n->left->type)) { + n->left = nod(OADDR, n->left, N); + typecheck(&n->left, top); + } + if(n->right->left != N) { + if((t = n->right->left->type) == T) + goto error; + if(!isint[t->etype]) { + yyerror("invalid slice index %#N (type %T)", n->right->left, t); + goto error; + } + } + if(n->right->right != N) { + if((t = n->right->right->type) == T) + goto error; + if(!isint[t->etype]) { + yyerror("invalid slice index %#N (type %T)", n->right->right, t); + goto error; + } + } + l = n->left; + if((t = l->type) == T) + goto error; + if(istype(t, TSTRING)) { + n->type = t; + n->op = OSLICESTR; + goto ret; + } + if(isptr[t->etype] && isfixedarray(t->type)) { + n->type = typ(TARRAY); + n->type->type = t->type->type; + n->type->bound = -1; + dowidth(n->type); + n->op = OSLICEARR; + goto ret; + } + if(isslice(t)) { + n->type = t; + goto ret; + } + yyerror("cannot slice %#N (type %T)", l, t); + goto error; + + /* + * call and call like + */ + case OCALL: + l = n->left; + if(l->op == ONAME && (r = unsafenmagic(n)) != N) { + if(n->isddd) + yyerror("invalid use of ... with builtin %#N", l); + n = r; + goto reswitch; + } + typecheck(&n->left, Erv | Etype | Ecall |(top&Eproc)); + l = n->left; + if(l->op == ONAME && l->etype != 0) { + if(n->isddd && l->etype != OAPPEND) + yyerror("invalid use of ... with builtin %#N", l); + // builtin: OLEN, OCAP, etc. + n->op = l->etype; + n->left = n->right; + n->right = N; + goto reswitch; + } + defaultlit(&n->left, T); + l = n->left; + if(l->op == OTYPE) { + if(n->isddd || l->type->bound == -100) + yyerror("invalid use of ... in type conversion", l); + // pick off before type-checking arguments + ok |= Erv; + // turn CALL(type, arg) into CONV(arg) w/ type + n->left = N; + n->op = OCONV; + n->type = l->type; + if(onearg(n, "conversion to %T", l->type) < 0) + goto error; + goto doconv; + } + + if(count(n->list) == 1) + typecheck(&n->list->n, Erv | Efnstruct); + else + typechecklist(n->list, Erv); + if((t = l->type) == T) + goto error; + checkwidth(t); + + switch(l->op) { + case ODOTINTER: + n->op = OCALLINTER; + break; + + case ODOTMETH: + n->op = OCALLMETH; + // typecheckaste was used here but there wasn't enough + // information further down the call chain to know if we + // were testing a method receiver for unexported fields. + // It isn't necessary, so just do a sanity check. + tp = getthisx(t)->type->type; + if(l->left == N || !eqtype(l->left->type, tp)) + fatal("method receiver"); + break; + + default: + n->op = OCALLFUNC; + if(t->etype != TFUNC) { + yyerror("cannot call non-function %#N (type %T)", l, t); + goto error; + } + break; + } + typecheckaste(OCALL, n->left, n->isddd, getinargx(t), n->list, "function argument"); + ok |= Etop; + if(t->outtuple == 0) + goto ret; + ok |= Erv; + if(t->outtuple == 1) { + t = getoutargx(l->type)->type; + if(t == T) + goto error; + if(t->etype == TFIELD) + t = t->type; + n->type = t; + goto ret; + } + // multiple return + if(!(top & (Efnstruct | Etop))) { + yyerror("multiple-value %#N() in single-value context", l); + goto ret; + } + n->type = getoutargx(l->type); + goto ret; + + case OCAP: + case OLEN: + case OREAL: + case OIMAG: + ok |= Erv; + if(onearg(n, "%#O", n->op) < 0) + goto error; + typecheck(&n->left, Erv); + defaultlit(&n->left, T); + implicitstar(&n->left); + l = n->left; + t = l->type; + if(t == T) + goto error; + switch(n->op) { + case OCAP: + if(!okforcap[t->etype]) + goto badcall1; + break; + case OLEN: + if(!okforlen[t->etype]) + goto badcall1; + break; + case OREAL: + case OIMAG: + if(!iscomplex[t->etype]) + goto badcall1; + if(isconst(l, CTCPLX)){ + if(n->op == OREAL) + n = nodfltconst(&l->val.u.cval->real); + else + n = nodfltconst(&l->val.u.cval->imag); + } + n->type = types[cplxsubtype(t->etype)]; + goto ret; + } + // might be constant + switch(t->etype) { + case TSTRING: + if(isconst(l, CTSTR)) { + r = nod(OXXX, N, N); + nodconst(r, types[TINT], l->val.u.sval->len); + r->orig = n; + n = r; + } + break; + case TARRAY: + if(t->bound >= 0 && l->op == ONAME) { + r = nod(OXXX, N, N); + nodconst(r, types[TINT], t->bound); + r->orig = n; + n = r; + } + break; + } + n->type = types[TINT]; + goto ret; + + case OCOMPLEX: + ok |= Erv; + if(twoarg(n) < 0) + goto error; + l = typecheck(&n->left, Erv | (top & Eiota)); + r = typecheck(&n->right, Erv | (top & Eiota)); + if(l->type == T || r->type == T) + goto error; + defaultlit2(&l, &r, 0); + n->left = l; + n->right = r; + if(l->type->etype != r->type->etype) { + badcmplx: + yyerror("invalid operation: %#N (complex of types %T, %T)", n, l->type, r->type); + goto error; + } + switch(l->type->etype) { + default: + goto badcmplx; + case TIDEAL: + t = types[TIDEAL]; + break; + case TFLOAT32: + t = types[TCOMPLEX64]; + break; + case TFLOAT64: + t = types[TCOMPLEX128]; + break; + } + if(l->op == OLITERAL && r->op == OLITERAL) { + // make it a complex literal + n = nodcplxlit(l->val, r->val); + } + n->type = t; + goto ret; + + case OCLOSE: + if(onearg(n, "%#O", n->op) < 0) + goto error; + typecheck(&n->left, Erv); + defaultlit(&n->left, T); + l = n->left; + if((t = l->type) == T) + goto error; + if(t->etype != TCHAN) { + yyerror("invalid operation: %#N (non-chan type %T)", n, t); + goto error; + } + ok |= Etop; + goto ret; + + case OAPPEND: + ok |= Erv; + args = n->list; + if(args == nil) { + yyerror("missing arguments to append"); + goto error; + } + typechecklist(args, Erv); + if((t = args->n->type) == T) + goto error; + n->type = t; + if(!isslice(t)) { + yyerror("first argument to append must be slice; have %lT", t); + goto error; + } + if(n->isddd) { + if(args->next == nil) { + yyerror("cannot use ... on first argument to append"); + goto error; + } + if(args->next->next != nil) { + yyerror("too many arguments to append"); + goto error; + } + args->next->n = assignconv(args->next->n, t->orig, "append"); + goto ret; + } + for(args=args->next; args != nil; args=args->next) { + if(args->n->type == T) + continue; + args->n = assignconv(args->n, t->type, "append"); + } + goto ret; + + case OCOPY: + ok |= Etop|Erv; + args = n->list; + if(args == nil || args->next == nil) { + yyerror("missing arguments to copy"); + goto error; + } + if(args->next->next != nil) { + yyerror("too many arguments to copy"); + goto error; + } + n->left = args->n; + n->right = args->next->n; + n->type = types[TINT]; + typecheck(&n->left, Erv); + typecheck(&n->right, Erv); + if(n->left->type == T || n->right->type == T) + goto error; + defaultlit(&n->left, T); + defaultlit(&n->right, T); + + // copy([]byte, string) + if(isslice(n->left->type) && n->right->type->etype == TSTRING) { + if (n->left->type->type == types[TUINT8]) + goto ret; + yyerror("arguments to copy have different element types: %lT and string", n->left->type); + goto error; + } + + if(!isslice(n->left->type) || !isslice(n->right->type)) { + if(!isslice(n->left->type) && !isslice(n->right->type)) + yyerror("arguments to copy must be slices; have %lT, %lT", n->left->type, n->right->type); + else if(!isslice(n->left->type)) + yyerror("first argument to copy should be slice; have %lT", n->left->type); + else + yyerror("second argument to copy should be slice or string; have %lT", n->right->type); + goto error; + } + if(!eqtype(n->left->type->type, n->right->type->type)) { + yyerror("arguments to copy have different element types: %lT and %lT", n->left->type, n->right->type); + goto error; + } + goto ret; + + case OCONV: + doconv: + ok |= Erv; + typecheck(&n->left, Erv | (top & (Eindir | Eiota))); + convlit1(&n->left, n->type, 1); + if((t = n->left->type) == T || n->type == T) + goto error; + if((n->op = convertop(t, n->type, &why)) == 0) { + yyerror("cannot convert %+N to type %T%s", n->left, n->type, why); + op = OCONV; + } + switch(n->op) { + case OCONVNOP: + if(n->left->op == OLITERAL) { + n->op = OLITERAL; + n->val = n->left->val; + } + break; + case OSTRARRAYBYTE: + case OSTRARRAYRUNE: + if(n->left->op == OLITERAL) + stringtoarraylit(&n); + break; + } + goto ret; + + case OMAKE: + ok |= Erv; + args = n->list; + if(args == nil) { + yyerror("missing argument to make"); + goto error; + } + l = args->n; + args = args->next; + typecheck(&l, Etype); + if((t = l->type) == T) + goto error; + + switch(t->etype) { + default: + badmake: + yyerror("cannot make type %T", t); + goto error; + + case TARRAY: + if(!isslice(t)) + goto badmake; + if(args == nil) { + yyerror("missing len argument to make(%T)", t); + goto error; + } + l = args->n; + args = args->next; + typecheck(&l, Erv); + defaultlit(&l, types[TINT]); + r = N; + if(args != nil) { + r = args->n; + args = args->next; + typecheck(&r, Erv); + defaultlit(&r, types[TINT]); + } + if(l->type == T || (r && r->type == T)) + goto error; + if(!isint[l->type->etype]) { + yyerror("non-integer len argument to make(%T)", t); + goto error; + } + if(r && !isint[r->type->etype]) { + yyerror("non-integer cap argument to make(%T)", t); + goto error; + } + n->left = l; + n->right = r; + n->op = OMAKESLICE; + break; + + case TMAP: + if(args != nil) { + l = args->n; + args = args->next; + typecheck(&l, Erv); + defaultlit(&l, types[TINT]); + if(l->type == T) + goto error; + if(!isint[l->type->etype]) { + yyerror("non-integer size argument to make(%T)", t); + goto error; + } + n->left = l; + } else + n->left = nodintconst(0); + n->op = OMAKEMAP; + break; + + case TCHAN: + l = N; + if(args != nil) { + l = args->n; + args = args->next; + typecheck(&l, Erv); + defaultlit(&l, types[TINT]); + if(l->type == T) + goto error; + if(!isint[l->type->etype]) { + yyerror("non-integer buffer argument to make(%T)", t); + goto error; + } + n->left = l; + } else + n->left = nodintconst(0); + n->op = OMAKECHAN; + break; + } + if(args != nil) { + yyerror("too many arguments to make(%T)", t); + n->op = OMAKE; + goto error; + } + n->type = t; + goto ret; + + case ONEW: + ok |= Erv; + args = n->list; + if(args == nil) { + yyerror("missing argument to new"); + goto error; + } + l = args->n; + typecheck(&l, Etype); + if((t = l->type) == T) + goto error; + if(args->next != nil) { + yyerror("too many arguments to new(%T)", t); + goto error; + } + n->left = l; + n->type = ptrto(t); + goto ret; + + case OPRINT: + case OPRINTN: + ok |= Etop; + typechecklist(n->list, Erv | Eindir); // Eindir: address does not escape + goto ret; + + case OPANIC: + ok |= Etop; + if(onearg(n, "panic") < 0) + goto error; + typecheck(&n->left, Erv); + defaultlit(&n->left, types[TINTER]); + if(n->left->type == T) + goto error; + goto ret; + + case ORECOVER: + ok |= Erv|Etop; + if(n->list != nil) { + yyerror("too many arguments to recover"); + goto error; + } + n->type = types[TINTER]; + goto ret; + + case OCLOSURE: + ok |= Erv; + typecheckclosure(n, top); + if(n->type == T) + goto error; + goto ret; + + /* + * statements + */ + case OAS: + ok |= Etop; + typecheckas(n); + goto ret; + + case OAS2: + ok |= Etop; + typecheckas2(n); + goto ret; + + case OBREAK: + case OCONTINUE: + case ODCL: + case OEMPTY: + case OGOTO: + case OLABEL: + case OXFALL: + ok |= Etop; + goto ret; + + case ODEFER: + ok |= Etop; + typecheck(&n->left, Etop); + goto ret; + + case OPROC: + ok |= Etop; + typecheck(&n->left, Etop|Eproc); + goto ret; + + case OFOR: + ok |= Etop; + typechecklist(n->ninit, Etop); + typecheck(&n->ntest, Erv); + if(n->ntest != N && (t = n->ntest->type) != T && t->etype != TBOOL) + yyerror("non-bool %+N used as for condition", n->ntest); + typecheck(&n->nincr, Etop); + typechecklist(n->nbody, Etop); + goto ret; + + case OIF: + ok |= Etop; + typechecklist(n->ninit, Etop); + typecheck(&n->ntest, Erv); + if(n->ntest != N && (t = n->ntest->type) != T && t->etype != TBOOL) + yyerror("non-bool %+N used as if condition", n->ntest); + typechecklist(n->nbody, Etop); + typechecklist(n->nelse, Etop); + goto ret; + + case ORETURN: + ok |= Etop; + typechecklist(n->list, Erv | Efnstruct); + if(curfn == N) { + yyerror("return outside function"); + goto error; + } + if(curfn->type->outnamed && n->list == nil) + goto ret; + typecheckaste(ORETURN, nil, 0, getoutargx(curfn->type), n->list, "return argument"); + goto ret; + + case OSELECT: + ok |= Etop; + typecheckselect(n); + goto ret; + + case OSWITCH: + ok |= Etop; + typecheckswitch(n); + goto ret; + + case ORANGE: + ok |= Etop; + typecheckrange(n); + goto ret; + + case OTYPESW: + yyerror("use of .(type) outside type switch"); + goto error; + + case OXCASE: + ok |= Etop; + typechecklist(n->list, Erv); + typechecklist(n->nbody, Etop); + goto ret; + + case ODCLFUNC: + ok |= Etop; + typecheckfunc(n); + goto ret; + + case ODCLCONST: + ok |= Etop; + typecheck(&n->left, Erv); + goto ret; + + case ODCLTYPE: + ok |= Etop; + typecheck(&n->left, Etype); + if(!incannedimport) + checkwidth(n->left->type); + goto ret; + } + +ret: + t = n->type; + if(t && !t->funarg && n->op != OTYPE) { + switch(t->etype) { + case TFUNC: // might have TANY; wait until its called + case TANY: + case TFORW: + case TIDEAL: + case TNIL: + case TBLANK: + break; + default: + checkwidth(t); + } + } + + // TODO(rsc): should not need to check importpkg, + // but reflect mentions unsafe.Pointer. + if(safemode && !incannedimport && !importpkg && t && t->etype == TUNSAFEPTR) + yyerror("cannot use unsafe.Pointer"); + + evconst(n); + if(n->op == OTYPE && !(top & Etype)) { + yyerror("type %T is not an expression", n->type); + goto error; + } + if((top & (Erv|Etype)) == Etype && n->op != OTYPE) { + yyerror("%#N is not a type", n); + goto error; + } + if((ok & Ecall) && !(top & Ecall)) { + yyerror("method %#N is not an expression, must be called", n); + goto error; + } + // TODO(rsc): simplify + if((top & (Ecall|Erv|Etype)) && !(top & Etop) && !(ok & (Erv|Etype|Ecall))) { + yyerror("%#N used as value", n); + goto error; + } + if((top & Etop) && !(top & (Ecall|Erv|Etype)) && !(ok & Etop)) { + if(n->diag == 0) { + yyerror("%#N not used", n); + n->diag = 1; + } + goto error; + } + + /* TODO + if(n->type == T) + fatal("typecheck nil type"); + */ + goto out; + +badcall1: + yyerror("invalid argument %#N (type %T) for %#O", n->left, n->left->type, n->op); + goto error; + +error: + n->type = T; + +out: + lineno = lno; + n->typecheck = 1; + *np = n; + return n; +} + +static void +implicitstar(Node **nn) +{ + Type *t; + Node *n; + + // insert implicit * if needed for fixed array + n = *nn; + t = n->type; + if(t == T || !isptr[t->etype]) + return; + t = t->type; + if(t == T) + return; + if(!isfixedarray(t)) + return; + n = nod(OIND, n, N); + typecheck(&n, Erv); + *nn = n; +} + +static int +onearg(Node *n, char *f, ...) +{ + va_list arg; + char *p; + + if(n->left != N) + return 0; + if(n->list == nil) { + va_start(arg, f); + p = vsmprint(f, arg); + va_end(arg); + yyerror("missing argument to %s: %#N", p, n); + return -1; + } + if(n->list->next != nil) { + va_start(arg, f); + p = vsmprint(f, arg); + va_end(arg); + yyerror("too many arguments to %s: %#N", p, n); + n->left = n->list->n; + n->list = nil; + return -1; + } + n->left = n->list->n; + n->list = nil; + return 0; +} + +static int +twoarg(Node *n) +{ + if(n->left != N) + return 0; + if(n->list == nil) { + yyerror("missing argument to %#O - %#N", n->op, n); + return -1; + } + n->left = n->list->n; + if(n->list->next == nil) { + yyerror("missing argument to %#O - %#N", n->op, n); + n->list = nil; + return -1; + } + if(n->list->next->next != nil) { + yyerror("too many arguments to %#O - %#N", n->op, n); + n->list = nil; + return -1; + } + n->right = n->list->next->n; + n->list = nil; + return 0; +} + +static Type* +lookdot1(Sym *s, Type *t, Type *f, int dostrcmp) +{ + Type *r; + + r = T; + for(; f!=T; f=f->down) { + if(dostrcmp && strcmp(f->sym->name, s->name) == 0) + return f; + if(f->sym != s) + continue; + if(r != T) { + yyerror("ambiguous DOT reference %T.%S", t, s); + break; + } + r = f; + } + return r; +} + +static int +looktypedot(Node *n, Type *t, int dostrcmp) +{ + Type *f1, *f2, *tt; + Sym *s; + + s = n->right->sym; + + if(t->etype == TINTER) { + f1 = lookdot1(s, t, t->type, dostrcmp); + if(f1 == T) + return 0; + + if(f1->width == BADWIDTH) + fatal("lookdot badwidth %T %p", f1, f1); + n->right = methodname(n->right, t); + n->xoffset = f1->width; + n->type = f1->type; + n->op = ODOTINTER; + return 1; + } + + tt = t; + if(t->sym == S && isptr[t->etype]) + tt = t->type; + + f2 = methtype(tt); + if(f2 == T) + return 0; + + expandmeth(f2->sym, f2); + f2 = lookdot1(s, f2, f2->xmethod, dostrcmp); + if(f2 == T) + return 0; + + // disallow T.m if m requires *T receiver + if(isptr[getthisx(f2->type)->type->type->etype] + && !isptr[t->etype] + && f2->embedded != 2 + && !isifacemethod(f2->type)) { + yyerror("invalid method expression %#N (needs pointer receiver: (*%T).%s)", n, t, f2->sym->name); + return 0; + } + + n->right = methodname(n->right, t); + n->xoffset = f2->width; + n->type = f2->type; + n->op = ODOTMETH; + return 1; +} + +static int +lookdot(Node *n, Type *t, int dostrcmp) +{ + Type *f1, *f2, *tt, *rcvr; + Sym *s; + + s = n->right->sym; + + dowidth(t); + f1 = T; + if(t->etype == TSTRUCT || t->etype == TINTER) + f1 = lookdot1(s, t, t->type, dostrcmp); + + f2 = T; + if(n->left->type == t || n->left->type->sym == S) { + f2 = methtype(t); + if(f2 != T) { + // Use f2->method, not f2->xmethod: adddot has + // already inserted all the necessary embedded dots. + f2 = lookdot1(s, f2, f2->method, dostrcmp); + } + } + + if(f1 != T) { + if(f2 != T) + yyerror("ambiguous DOT reference %S as both field and method", + n->right->sym); + if(f1->width == BADWIDTH) + fatal("lookdot badwidth %T %p", f1, f1); + n->xoffset = f1->width; + n->type = f1->type; + if(t->etype == TINTER) { + if(isptr[n->left->type->etype]) { + n->left = nod(OIND, n->left, N); // implicitstar + typecheck(&n->left, Erv); + } + n->op = ODOTINTER; + } + return 1; + } + + if(f2 != T) { + tt = n->left->type; + dowidth(tt); + rcvr = getthisx(f2->type)->type->type; + if(!eqtype(rcvr, tt)) { + if(rcvr->etype == tptr && eqtype(rcvr->type, tt)) { + checklvalue(n->left, "call pointer method on"); + addrescapes(n->left); + n->left = nod(OADDR, n->left, N); + n->left->implicit = 1; + typecheck(&n->left, Etype|Erv); + } else if(tt->etype == tptr && eqtype(tt->type, rcvr)) { + n->left = nod(OIND, n->left, N); + n->left->implicit = 1; + typecheck(&n->left, Etype|Erv); + } else { + // method is attached to wrong type? + fatal("method mismatch: %T for %T", rcvr, tt); + } + } + n->right = methodname(n->right, n->left->type); + n->xoffset = f2->width; + n->type = f2->type; + n->op = ODOTMETH; + return 1; + } + + return 0; +} + +static int +nokeys(NodeList *l) +{ + for(; l; l=l->next) + if(l->n->op == OKEY) + return 0; + return 1; +} + +/* + * typecheck assignment: type list = expression list + */ +static void +typecheckaste(int op, Node *call, int isddd, Type *tstruct, NodeList *nl, char *desc) +{ + Type *t, *tl, *tn; + Node *n; + int lno; + char *why; + + lno = lineno; + + if(tstruct->broke) + goto out; + + if(nl != nil && nl->next == nil && (n = nl->n)->type != T) + if(n->type->etype == TSTRUCT && n->type->funarg) { + tn = n->type->type; + for(tl=tstruct->type; tl; tl=tl->down) { + if(tl->isddd) { + for(; tn; tn=tn->down) { + exportassignok(tn->type, desc); + if(assignop(tn->type, tl->type->type, &why) == 0) { + if(call != N) + yyerror("cannot use %T as type %T in argument to %#N%s", tn->type, tl->type->type, call, why); + else + yyerror("cannot use %T as type %T in %s%s", tn->type, tl->type->type, desc, why); + } + } + goto out; + } + if(tn == T) + goto notenough; + exportassignok(tn->type, desc); + if(assignop(tn->type, tl->type, &why) == 0) { + if(call != N) + yyerror("cannot use %T as type %T in argument to %#N%s", tn->type, tl->type, call, why); + else + yyerror("cannot use %T as type %T in %s%s", tn->type, tl->type, desc, why); + } + tn = tn->down; + } + if(tn != T) + goto toomany; + goto out; + } + + for(tl=tstruct->type; tl; tl=tl->down) { + t = tl->type; + if(tl->isddd) { + if(isddd) { + if(nl == nil) + goto notenough; + if(nl->next != nil) + goto toomany; + n = nl->n; + setlineno(n); + if(n->type != T) + nl->n = assignconv(n, t, desc); + goto out; + } + for(; nl; nl=nl->next) { + n = nl->n; + setlineno(nl->n); + if(n->type != T) + nl->n = assignconv(n, t->type, desc); + } + goto out; + } + if(nl == nil) + goto notenough; + n = nl->n; + setlineno(n); + if(n->type != T) + nl->n = assignconv(n, t, desc); + nl = nl->next; + } + if(nl != nil) + goto toomany; + if(isddd) { + if(call != N) + yyerror("invalid use of ... in call to %#N", call); + else + yyerror("invalid use of ... in %#O", op); + } + +out: + lineno = lno; + return; + +notenough: + if(call != N) + yyerror("not enough arguments in call to %#N", call); + else + yyerror("not enough arguments to %#O", op); + goto out; + +toomany: + if(call != N) + yyerror("too many arguments in call to %#N", call); + else + yyerror("too many arguments to %#O", op); + goto out; +} + +/* + * do the export rules allow writing to this type? + * cannot be implicitly assigning to any type with + * an unavailable field. + */ +int +exportassignok(Type *t, char *desc) +{ + Type *f; + Sym *s; + + if(t == T) + return 1; + if(t->trecur) + return 1; + t->trecur = 1; + + switch(t->etype) { + default: + // most types can't contain others; they're all fine. + break; + case TSTRUCT: + for(f=t->type; f; f=f->down) { + if(f->etype != TFIELD) + fatal("structas: not field"); + s = f->sym; + // s == nil doesn't happen for embedded fields (they get the type symbol). + // it only happens for fields in a ... struct. + if(s != nil && !exportname(s->name) && s->pkg != localpkg) { + char *prefix; + + prefix = ""; + if(desc != nil) + prefix = " in "; + else + desc = ""; + yyerror("implicit assignment of unexported field '%s' of %T%s%s", s->name, t, prefix, desc); + goto no; + } + if(!exportassignok(f->type, desc)) + goto no; + } + break; + + case TARRAY: + if(t->bound < 0) // slices are pointers; that's fine + break; + if(!exportassignok(t->type, desc)) + goto no; + break; + } + t->trecur = 0; + return 1; + +no: + t->trecur = 0; + return 0; +} + + +/* + * type check composite + */ + +static void +fielddup(Node *n, Node *hash[], ulong nhash) +{ + uint h; + char *s; + Node *a; + + if(n->op != ONAME) + fatal("fielddup: not ONAME"); + s = n->sym->name; + h = stringhash(s)%nhash; + for(a=hash[h]; a!=N; a=a->ntest) { + if(strcmp(a->sym->name, s) == 0) { + yyerror("duplicate field name in struct literal: %s", s); + return; + } + } + n->ntest = hash[h]; + hash[h] = n; +} + +static void +keydup(Node *n, Node *hash[], ulong nhash) +{ + uint h; + ulong b; + double d; + int i; + Node *a; + Node cmp; + char *s; + + evconst(n); + if(n->op != OLITERAL) + return; // we dont check variables + + switch(n->val.ctype) { + default: // unknown, bool, nil + b = 23; + break; + case CTINT: + b = mpgetfix(n->val.u.xval); + break; + case CTFLT: + d = mpgetflt(n->val.u.fval); + s = (char*)&d; + b = 0; + for(i=sizeof(d); i>0; i--) + b = b*PRIME1 + *s++; + break; + case CTSTR: + b = 0; + s = n->val.u.sval->s; + for(i=n->val.u.sval->len; i>0; i--) + b = b*PRIME1 + *s++; + break; + } + + h = b%nhash; + memset(&cmp, 0, sizeof(cmp)); + for(a=hash[h]; a!=N; a=a->ntest) { + cmp.op = OEQ; + cmp.left = n; + cmp.right = a; + evconst(&cmp); + b = cmp.val.u.bval; + if(b) { + // too lazy to print the literal + yyerror("duplicate key in map literal"); + return; + } + } + n->ntest = hash[h]; + hash[h] = n; +} + +static void +indexdup(Node *n, Node *hash[], ulong nhash) +{ + uint h; + Node *a; + ulong b, c; + + if(n->op != OLITERAL) + fatal("indexdup: not OLITERAL"); + + b = mpgetfix(n->val.u.xval); + h = b%nhash; + for(a=hash[h]; a!=N; a=a->ntest) { + c = mpgetfix(a->val.u.xval); + if(b == c) { + yyerror("duplicate index in array literal: %ld", b); + return; + } + } + n->ntest = hash[h]; + hash[h] = n; +} + +static int +prime(ulong h, ulong sr) +{ + ulong n; + + for(n=3; n<=sr; n+=2) + if(h%n == 0) + return 0; + return 1; +} + +static ulong +inithash(Node *n, Node ***hash, Node **autohash, ulong nautohash) +{ + ulong h, sr; + NodeList *ll; + int i; + + // count the number of entries + h = 0; + for(ll=n->list; ll; ll=ll->next) + h++; + + // if the auto hash table is + // large enough use it. + if(h <= nautohash) { + *hash = autohash; + memset(*hash, 0, nautohash * sizeof(**hash)); + return nautohash; + } + + // make hash size odd and 12% larger than entries + h += h/8; + h |= 1; + + // calculate sqrt of h + sr = h/2; + for(i=0; i<5; i++) + sr = (sr + h/sr)/2; + + // check for primeality + while(!prime(h, sr)) + h += 2; + + // build and return a throw-away hash table + *hash = mal(h * sizeof(**hash)); + memset(*hash, 0, h * sizeof(**hash)); + return h; +} + +static void +typecheckcomplit(Node **np) +{ + int bad, i, len, nerr; + Node *l, *n, **hash; + NodeList *ll; + Type *t, *f, *pushtype; + Sym *s; + int32 lno; + ulong nhash; + Node *autohash[101]; + + n = *np; + lno = lineno; + + if(n->right == N) { + if(n->list != nil) + setlineno(n->list->n); + yyerror("missing type in composite literal"); + goto error; + } + + setlineno(n->right); + l = typecheck(&n->right /* sic */, Etype|Ecomplit); + if((t = l->type) == T) + goto error; + nerr = nerrors; + + // can omit type on composite literal values if the outer + // composite literal is array, slice, or map, and the + // element type is itself a struct, array, slice, or map. + pushtype = T; + if(t->etype == TARRAY || t->etype == TMAP) { + pushtype = t->type; + if(pushtype != T) { + switch(pushtype->etype) { + case TSTRUCT: + case TARRAY: + case TMAP: + break; + default: + pushtype = T; + break; + } + } + } + + switch(t->etype) { + default: + yyerror("invalid type for composite literal: %T", t); + n->type = T; + break; + + case TARRAY: + nhash = inithash(n, &hash, autohash, nelem(autohash)); + + len = 0; + i = 0; + for(ll=n->list; ll; ll=ll->next) { + l = ll->n; + setlineno(l); + if(l->op != OKEY) { + l = nod(OKEY, nodintconst(i), l); + l->left->type = types[TINT]; + l->left->typecheck = 1; + ll->n = l; + } + + typecheck(&l->left, Erv); + evconst(l->left); + i = nonnegconst(l->left); + if(i < 0) { + yyerror("array index must be non-negative integer constant"); + i = -(1<<30); // stay negative for a while + } + if(i >= 0) + indexdup(l->left, hash, nhash); + i++; + if(i > len) { + len = i; + if(t->bound >= 0 && len > t->bound) { + setlineno(l); + yyerror("array index %d out of bounds [0:%d]", len, t->bound); + t->bound = -1; // no more errors + } + } + + if(l->right->op == OCOMPLIT && l->right->right == N && pushtype != T) + l->right->right = typenod(pushtype); + typecheck(&l->right, Erv); + defaultlit(&l->right, t->type); + l->right = assignconv(l->right, t->type, "array element"); + } + if(t->bound == -100) + t->bound = len; + if(t->bound < 0) + n->right = nodintconst(len); + n->op = OARRAYLIT; + break; + + case TMAP: + nhash = inithash(n, &hash, autohash, nelem(autohash)); + + for(ll=n->list; ll; ll=ll->next) { + l = ll->n; + setlineno(l); + if(l->op != OKEY) { + typecheck(&ll->n, Erv); + yyerror("missing key in map literal"); + continue; + } + + typecheck(&l->left, Erv); + defaultlit(&l->left, t->down); + l->left = assignconv(l->left, t->down, "map key"); + keydup(l->left, hash, nhash); + + if(l->right->op == OCOMPLIT && l->right->right == N && pushtype != T) + l->right->right = typenod(pushtype); + typecheck(&l->right, Erv); + defaultlit(&l->right, t->type); + l->right = assignconv(l->right, t->type, "map value"); + } + n->op = OMAPLIT; + break; + + case TSTRUCT: + bad = 0; + if(n->list != nil && nokeys(n->list)) { + // simple list of variables + f = t->type; + for(ll=n->list; ll; ll=ll->next) { + setlineno(ll->n); + typecheck(&ll->n, Erv); + if(f == nil) { + if(!bad++) + yyerror("too many values in struct initializer"); + continue; + } + s = f->sym; + if(s != nil && !exportname(s->name) && s->pkg != localpkg) + yyerror("implicit assignment of unexported field '%s' in %T literal", s->name, t); + ll->n = assignconv(ll->n, f->type, "field value"); + ll->n = nod(OKEY, newname(f->sym), ll->n); + ll->n->left->typecheck = 1; + f = f->down; + } + if(f != nil) + yyerror("too few values in struct initializer"); + } else { + nhash = inithash(n, &hash, autohash, nelem(autohash)); + + // keyed list + for(ll=n->list; ll; ll=ll->next) { + l = ll->n; + setlineno(l); + if(l->op != OKEY) { + if(!bad++) + yyerror("mixture of field:value and value initializers"); + typecheck(&ll->n, Erv); + continue; + } + s = l->left->sym; + if(s == S) { + yyerror("invalid field name %#N in struct initializer", l->left); + typecheck(&l->right, Erv); + continue; + } + // Sym might have resolved to name in other top-level + // package, because of import dot. Redirect to correct sym + // before we do the lookup. + if(s->pkg != localpkg) + s = lookup(s->name); + l->left = newname(s); + l->left->typecheck = 1; + f = lookdot1(s, t, t->type, 0); + typecheck(&l->right, Erv); + if(f == nil) { + yyerror("unknown %T field '%s' in struct literal", t, s->name); + continue; + } + s = f->sym; + fielddup(newname(s), hash, nhash); + l->right = assignconv(l->right, f->type, "field value"); + } + } + n->op = OSTRUCTLIT; + break; + } + if(nerr != nerrors) + goto error; + n->type = t; + + *np = n; + lineno = lno; + return; + +error: + n->type = T; + *np = n; + lineno = lno; +} + +/* + * the address of n has been taken and might be used after + * the current function returns. mark any local vars + * as needing to move to the heap. + */ +static void +addrescapes(Node *n) +{ + char buf[100]; + switch(n->op) { + default: + // probably a type error already. + // dump("addrescapes", n); + break; + + case ONAME: + if(n->noescape) + break; + switch(n->class) { + case PPARAMREF: + addrescapes(n->defn); + break; + case PPARAM: + case PPARAMOUT: + // if func param, need separate temporary + // to hold heap pointer. + // the function type has already been checked + // (we're in the function body) + // so the param already has a valid xoffset. + + // expression to refer to stack copy + n->stackparam = nod(OPARAM, n, N); + n->stackparam->type = n->type; + n->stackparam->addable = 1; + if(n->xoffset == BADWIDTH) + fatal("addrescapes before param assignment"); + n->stackparam->xoffset = n->xoffset; + n->xoffset = 0; + // fallthrough + case PAUTO: + + n->class |= PHEAP; + n->addable = 0; + n->ullman = 2; + n->xoffset = 0; + + // create stack variable to hold pointer to heap + n->heapaddr = nod(ONAME, N, N); + n->heapaddr->type = ptrto(n->type); + snprint(buf, sizeof buf, "&%S", n->sym); + n->heapaddr->sym = lookup(buf); + n->heapaddr->class = PHEAP-1; // defer tempname to allocparams + n->heapaddr->ullman = 1; + n->curfn->dcl = list(n->curfn->dcl, n->heapaddr); + + break; + } + break; + + case OIND: + case ODOTPTR: + break; + + case ODOT: + case OINDEX: + // ODOTPTR has already been introduced, + // so these are the non-pointer ODOT and OINDEX. + // In &x[0], if x is a slice, then x does not + // escape--the pointer inside x does, but that + // is always a heap pointer anyway. + if(!isslice(n->left->type)) + addrescapes(n->left); + break; + } +} + +/* + * lvalue etc + */ +int +islvalue(Node *n) +{ + switch(n->op) { + case OINDEX: + if(isfixedarray(n->left->type)) + return islvalue(n->left); + if(n->left->type != T && n->left->type->etype == TSTRING) + return 0; + // fall through + case OIND: + case ODOTPTR: + return 1; + case ODOT: + return islvalue(n->left); + case ONAME: + if(n->class == PFUNC) + return 0; + return 1; + } + return 0; +} + +static void +checklvalue(Node *n, char *verb) +{ + if(!islvalue(n)) + yyerror("cannot %s %#N", verb, n); +} + +static void +checkassign(Node *n) +{ + if(islvalue(n)) + return; + if(n->op == OINDEXMAP) { + n->etype = 1; + return; + } + yyerror("cannot assign to %#N", n); +} + +static void +checkassignlist(NodeList *l) +{ + for(; l; l=l->next) + checkassign(l->n); +} + +/* + * type check assignment. + * if this assignment is the definition of a var on the left side, + * fill in the var's type. + */ + +static void +typecheckas(Node *n) +{ + // delicate little dance. + // the definition of n may refer to this assignment + // as its definition, in which case it will call typecheckas. + // in that case, do not call typecheck back, or it will cycle. + // if the variable has a type (ntype) then typechecking + // will not look at defn, so it is okay (and desirable, + // so that the conversion below happens). + n->left = resolve(n->left); + if(n->left->defn != n || n->left->ntype) + typecheck(&n->left, Erv | Easgn); + + checkassign(n->left); + typecheck(&n->right, Erv); + if(n->right && n->right->type != T) { + if(n->left->type != T) + n->right = assignconv(n->right, n->left->type, "assignment"); + else if(!isblank(n->left)) + exportassignok(n->right->type, "assignment"); + } + if(n->left->defn == n && n->left->ntype == N) { + defaultlit(&n->right, T); + n->left->type = n->right->type; + } + + // second half of dance. + // now that right is done, typecheck the left + // just to get it over with. see dance above. + n->typecheck = 1; + if(n->left->typecheck == 0) + typecheck(&n->left, Erv | Easgn); +} + +static void +checkassignto(Type *src, Node *dst) +{ + char *why; + + if(assignop(src, dst->type, &why) == 0) { + yyerror("cannot assign %T to %+N in multiple assignment%s", src, dst, why); + return; + } + exportassignok(dst->type, "multiple assignment"); +} + +static void +typecheckas2(Node *n) +{ + int cl, cr; + NodeList *ll, *lr; + Node *l, *r; + Iter s; + Type *t; + + for(ll=n->list; ll; ll=ll->next) { + // delicate little dance. + ll->n = resolve(ll->n); + if(ll->n->defn != n || ll->n->ntype) + typecheck(&ll->n, Erv | Easgn); + } + cl = count(n->list); + cr = count(n->rlist); + checkassignlist(n->list); + if(cl > 1 && cr == 1) + typecheck(&n->rlist->n, Erv | Efnstruct); + else + typechecklist(n->rlist, Erv); + + if(cl == cr) { + // easy + for(ll=n->list, lr=n->rlist; ll; ll=ll->next, lr=lr->next) { + if(ll->n->type != T && lr->n->type != T) + lr->n = assignconv(lr->n, ll->n->type, "assignment"); + if(ll->n->defn == n && ll->n->ntype == N) { + defaultlit(&lr->n, T); + ll->n->type = lr->n->type; + } + } + goto out; + } + + + l = n->list->n; + r = n->rlist->n; + + // m[i] = x, ok + if(cl == 1 && cr == 2 && l->op == OINDEXMAP) { + if(l->type == T) + goto out; + n->op = OAS2MAPW; + n->rlist->n = assignconv(r, l->type, "assignment"); + r = n->rlist->next->n; + n->rlist->next->n = assignconv(r, types[TBOOL], "assignment"); + goto out; + } + + // x,y,z = f() + if(cr == 1) { + if(r->type == T) + goto out; + switch(r->op) { + case OCALLMETH: + case OCALLINTER: + case OCALLFUNC: + if(r->type->etype != TSTRUCT || r->type->funarg == 0) + break; + cr = structcount(r->type); + if(cr != cl) + goto mismatch; + n->op = OAS2FUNC; + t = structfirst(&s, &r->type); + for(ll=n->list; ll; ll=ll->next) { + if(ll->n->type != T) + checkassignto(t->type, ll->n); + if(ll->n->defn == n && ll->n->ntype == N) + ll->n->type = t->type; + t = structnext(&s); + } + goto out; + } + } + + // x, ok = y + if(cl == 2 && cr == 1) { + if(r->type == T) + goto out; + switch(r->op) { + case OINDEXMAP: + n->op = OAS2MAPR; + goto common; + case ORECV: + n->op = OAS2RECV; + n->right = n->rlist->n; + goto common; + case ODOTTYPE: + n->op = OAS2DOTTYPE; + r->op = ODOTTYPE2; + common: + if(l->type != T) + checkassignto(r->type, l); + if(l->defn == n) + l->type = r->type; + l = n->list->next->n; + if(l->type != T) + checkassignto(types[TBOOL], l); + if(l->defn == n && l->ntype == N) + l->type = types[TBOOL]; + goto out; + } + } + +mismatch: + yyerror("assignment count mismatch: %d = %d", cl, cr); + +out: + // second half of dance + n->typecheck = 1; + for(ll=n->list; ll; ll=ll->next) + if(ll->n->typecheck == 0) + typecheck(&ll->n, Erv | Easgn); +} + +/* + * type check function definition + */ +static void +typecheckfunc(Node *n) +{ + Type *t, *rcvr; + +//dump("nname", n->nname); + typecheck(&n->nname, Erv | Easgn); + if((t = n->nname->type) == T) + return; + n->type = t; + + rcvr = getthisx(t)->type; + if(rcvr != nil && n->shortname != N && !isblank(n->shortname)) + addmethod(n->shortname->sym, t, 1); +} + +static void +stringtoarraylit(Node **np) +{ + int32 i; + NodeList *l; + Strlit *s; + char *p, *ep; + Rune r; + Node *nn, *n; + + n = *np; + if(n->left->op != OLITERAL || n->left->val.ctype != CTSTR) + fatal("stringtoarraylit %N", n); + + s = n->left->val.u.sval; + l = nil; + p = s->s; + ep = s->s + s->len; + i = 0; + if(n->type->type->etype == TUINT8) { + // raw []byte + while(p < ep) + l = list(l, nod(OKEY, nodintconst(i++), nodintconst((uchar)*p++))); + } else { + // utf-8 []int + while(p < ep) { + p += chartorune(&r, p); + l = list(l, nod(OKEY, nodintconst(i++), nodintconst(r))); + } + } + nn = nod(OCOMPLIT, N, typenod(n->type)); + nn->list = l; + typecheck(&nn, Erv); + *np = nn; +} + +static Type* +getforwtype(Node *n) +{ + Node *f1, *f2; + + for(f1=f2=n; ; n=n->ntype) { + if((n = resolve(n)) == N || n->op != OTYPE) + return T; + + if(n->type != T && n->type->etype == TFORW) + return n->type; + + // Check for ntype cycle. + if((f2 = resolve(f2)) != N && (f1 = resolve(f2->ntype)) != N) { + f2 = resolve(f1->ntype); + if(f1 == n || f2 == n) + return T; + } + } +} + +static int ntypecheckdeftype; +static NodeList *methodqueue; + +static void +domethod(Node *n) +{ + Node *nt; + + nt = n->type->nname; + typecheck(&nt, Etype); + if(nt->type == T) { + // type check failed; leave empty func + n->type->etype = TFUNC; + n->type->nod = N; + return; + } + *n->type = *nt->type; + n->type->nod = N; + checkwidth(n->type); +} + +typedef struct NodeTypeList NodeTypeList; +struct NodeTypeList { + Node *n; + Type *t; + NodeTypeList *next; +}; + +static NodeTypeList *dntq; +static NodeTypeList *dntend; + +void +defertypecopy(Node *n, Type *t) +{ + NodeTypeList *ntl; + + if(n == N || t == T) + return; + + ntl = mal(sizeof *ntl); + ntl->n = n; + ntl->t = t; + ntl->next = nil; + + if(dntq == nil) + dntq = ntl; + else + dntend->next = ntl; + + dntend = ntl; +} + +void +resumetypecopy(void) +{ + NodeTypeList *l; + + for(l=dntq; l; l=l->next) + copytype(l->n, l->t); +} + +void +copytype(Node *n, Type *t) +{ + *n->type = *t; + + t = n->type; + t->sym = n->sym; + t->local = n->local; + t->vargen = n->vargen; + t->siggen = 0; + t->method = nil; + t->nod = N; + t->printed = 0; + t->deferwidth = 0; +} + +static void +typecheckdeftype(Node *n) +{ + int maplineno, embedlineno, lno; + Type *t; + NodeList *l; + + ntypecheckdeftype++; + lno = lineno; + setlineno(n); + n->type->sym = n->sym; + n->typecheck = 1; + typecheck(&n->ntype, Etype); + if((t = n->ntype->type) == T) { + n->diag = 1; + goto ret; + } + if(n->type == T) { + n->diag = 1; + goto ret; + } + + maplineno = n->type->maplineno; + embedlineno = n->type->embedlineno; + + // copy new type and clear fields + // that don't come along. + // anything zeroed here must be zeroed in + // typedcl2 too. + copytype(n, t); + + // double-check use of type as map key. + if(maplineno) { + lineno = maplineno; + maptype(n->type, types[TBOOL]); + } + if(embedlineno) { + lineno = embedlineno; + if(isptr[t->etype]) + yyerror("embedded type cannot be a pointer"); + } + +ret: + lineno = lno; + + // if there are no type definitions going on, it's safe to + // try to resolve the method types for the interfaces + // we just read. + if(ntypecheckdeftype == 1) { + while((l = methodqueue) != nil) { + methodqueue = nil; + for(; l; l=l->next) + domethod(l->n); + } + } + ntypecheckdeftype--; +} + +void +queuemethod(Node *n) +{ + if(ntypecheckdeftype == 0) { + domethod(n); + return; + } + methodqueue = list(methodqueue, n); +} + +Node* +typecheckdef(Node *n) +{ + int lno; + Node *e; + Type *t; + NodeList *l; + + lno = lineno; + setlineno(n); + + if(n->op == ONONAME) { + if(!n->diag) { + n->diag = 1; + if(n->lineno != 0) + lineno = n->lineno; + yyerror("undefined: %S", n->sym); + } + return n; + } + + if(n->walkdef == 1) + return n; + + l = mal(sizeof *l); + l->n = n; + l->next = typecheckdefstack; + typecheckdefstack = l; + + if(n->walkdef == 2) { + flusherrors(); + print("typecheckdef loop:"); + for(l=typecheckdefstack; l; l=l->next) + print(" %S", l->n->sym); + print("\n"); + fatal("typecheckdef loop"); + } + n->walkdef = 2; + + if(n->type != T || n->sym == S) // builtin or no name + goto ret; + + switch(n->op) { + default: + fatal("typecheckdef %O", n->op); + + case OGOTO: + case OLABEL: + // not really syms + break; + + case OLITERAL: + if(n->ntype != N) { + typecheck(&n->ntype, Etype); + n->type = n->ntype->type; + n->ntype = N; + if(n->type == T) { + n->diag = 1; + goto ret; + } + } + e = n->defn; + n->defn = N; + if(e == N) { + lineno = n->lineno; + dump("typecheckdef nil defn", n); + yyerror("xxx"); + } + typecheck(&e, Erv | Eiota); + if(e->type != T && e->op != OLITERAL) { + yyerror("const initializer must be constant"); + goto ret; + } + if(isconst(e, CTNIL)) { + yyerror("const initializer cannot be nil"); + goto ret; + } + t = n->type; + if(t != T) { + if(!okforconst[t->etype]) { + yyerror("invalid constant type %T", t); + goto ret; + } + if(!isideal(e->type) && !eqtype(t, e->type)) { + yyerror("cannot use %+N as type %T in const initializer", e, t); + goto ret; + } + convlit(&e, t); + } + n->val = e->val; + n->type = e->type; + break; + + case ONAME: + if(n->ntype != N) { + typecheck(&n->ntype, Etype); + n->type = n->ntype->type; + if(n->type == T) { + n->diag = 1; + goto ret; + } + } + if(n->type != T) + break; + if(n->defn == N) { + if(n->etype != 0) // like OPRINTN + break; + if(nsavederrors+nerrors > 0) { + // Can have undefined variables in x := foo + // that make x have an n->ndefn == nil. + // If there are other errors anyway, don't + // bother adding to the noise. + break; + } + fatal("var without type, init: %S", n->sym); + } + if(n->defn->op == ONAME) { + typecheck(&n->defn, Erv); + n->type = n->defn->type; + break; + } + typecheck(&n->defn, Etop); // fills in n->type + break; + + case OTYPE: + if(curfn) + defercheckwidth(); + n->walkdef = 1; + n->type = typ(TFORW); + n->type->sym = n->sym; + typecheckdeftype(n); + if(curfn) + resumecheckwidth(); + break; + + case OPACK: + // nothing to see here + break; + } + +ret: + if(typecheckdefstack->n != n) + fatal("typecheckdefstack mismatch"); + l = typecheckdefstack; + typecheckdefstack = l->next; + + lineno = lno; + n->walkdef = 1; + return n; +} diff --git a/src/cmd/gc/unsafe.c b/src/cmd/gc/unsafe.c new file mode 100644 index 000000000..d304077c8 --- /dev/null +++ b/src/cmd/gc/unsafe.c @@ -0,0 +1,97 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +#include "go.h" + +/* + * look for + * unsafe.Sizeof + * unsafe.Offsetof + * rewrite with a constant + */ +Node* +unsafenmagic(Node *nn) +{ + Node *r, *n; + Sym *s; + Type *t, *tr; + long v; + Val val; + Node *fn; + NodeList *args; + + fn = nn->left; + args = nn->list; + + if(safemode || fn == N || fn->op != ONAME || (s = fn->sym) == S) + goto no; + if(s->pkg != unsafepkg) + goto no; + + if(args == nil) { + yyerror("missing argument for %S", s); + goto no; + } + r = args->n; + + if(strcmp(s->name, "Sizeof") == 0) { + typecheck(&r, Erv); + defaultlit(&r, T); + tr = r->type; + if(tr == T) + goto bad; + dowidth(tr); + v = tr->width; + goto yes; + } + if(strcmp(s->name, "Offsetof") == 0) { + typecheck(&r, Erv); + if(r->op != ODOT && r->op != ODOTPTR) + goto bad; + typecheck(&r, Erv); + v = r->xoffset; + goto yes; + } + if(strcmp(s->name, "Alignof") == 0) { + typecheck(&r, Erv); + defaultlit(&r, T); + tr = r->type; + if(tr == T) + goto bad; + + // make struct { byte; T; } + t = typ(TSTRUCT); + t->type = typ(TFIELD); + t->type->type = types[TUINT8]; + t->type->down = typ(TFIELD); + t->type->down->type = tr; + // compute struct widths + dowidth(t); + + // the offset of T is its required alignment + v = t->type->down->width; + goto yes; + } + +no: + return N; + +bad: + yyerror("invalid expression %#N", nn); + v = 0; + goto ret; + +yes: + if(args->next != nil) + yyerror("extra arguments for %S", s); +ret: + // any side effects disappear; ignore init + val.ctype = CTINT; + val.u.xval = mal(sizeof(*n->val.u.xval)); + mpmovecfix(val.u.xval, v); + n = nod(OLITERAL, N, N); + n->val = val; + n->type = types[TUINTPTR]; + return n; +} diff --git a/src/cmd/gc/unsafe.go b/src/cmd/gc/unsafe.go new file mode 100644 index 000000000..db27d7425 --- /dev/null +++ b/src/cmd/gc/unsafe.go @@ -0,0 +1,22 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// NOTE: If you change this file you must run "./mkbuiltin" +// to update builtin.c.boot. This is not done automatically +// to avoid depending on having a working compiler binary. + +package PACKAGE + +type Pointer uintptr // not really; filled in by compiler + +// return types here are ignored; see unsafe.c +func Offsetof(any) uintptr +func Sizeof(any) uintptr +func Alignof(any) uintptr + +func Typeof(i interface{}) (typ interface{}) +func Reflect(i interface{}) (typ interface{}, addr Pointer) +func Unreflect(typ interface{}, addr Pointer) (ret interface{}) +func New(typ interface{}) Pointer +func NewArray(typ interface{}, n int) Pointer diff --git a/src/cmd/gc/walk.c b/src/cmd/gc/walk.c new file mode 100644 index 000000000..9cd4ee919 --- /dev/null +++ b/src/cmd/gc/walk.c @@ -0,0 +1,2166 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +#include "go.h" + +static Node* walkprint(Node*, NodeList**, int); +static Node* conv(Node*, Type*); +static Node* mapfn(char*, Type*); +static Node* makenewvar(Type*, NodeList**, Node**); +static Node* ascompatee1(int, Node*, Node*, NodeList**); +static NodeList* ascompatee(int, NodeList*, NodeList*, NodeList**); +static NodeList* ascompatet(int, NodeList*, Type**, int, NodeList**); +static NodeList* ascompatte(int, int, Type**, NodeList*, int, NodeList**); +static Node* convas(Node*, NodeList**); +static void heapmoves(void); +static NodeList* paramstoheap(Type **argin, int out); +static NodeList* reorder1(NodeList*); +static NodeList* reorder3(NodeList*); +static Node* addstr(Node*, NodeList**); +static Node* appendslice(Node*, NodeList**); +static Node* append(Node*, NodeList**); + +// can this code branch reach the end +// without an unconditional RETURN +// this is hard, so it is conservative +static int +walkret(NodeList *l) +{ + Node *n; + +loop: + while(l && l->next) + l = l->next; + if(l == nil) + return 1; + + // at this point, we have the last + // statement of the function + n = l->n; + switch(n->op) { + case OBLOCK: + l = n->list; + goto loop; + + case OGOTO: + case ORETURN: + case OPANIC: + return 0; + break; + } + + // all other statements + // will flow to the end + return 1; +} + +void +walk(Node *fn) +{ + char s[50]; + NodeList *l; + Node *n; + int lno; + + curfn = fn; + + if(debug['W']) { + snprint(s, sizeof(s), "\nbefore %S", curfn->nname->sym); + dumplist(s, curfn->nbody); + } + if(curfn->type->outtuple) + if(walkret(curfn->nbody)) + yyerror("function ends without a return statement"); + + lno = lineno; + for(l=fn->dcl; l; l=l->next) { + n = l->n; + if(n->op != ONAME || n->class != PAUTO) + continue; + lineno = n->lineno; + typecheck(&n, Erv | Easgn); // only needed for unused variables + if(!n->used && n->sym->name[0] != '&' && !nsyntaxerrors) + yyerror("%S declared and not used", n->sym); + } + lineno = lno; + if(nerrors != 0) + return; + walkstmtlist(curfn->nbody); + if(debug['W']) { + snprint(s, sizeof(s), "after walk %S", curfn->nname->sym); + dumplist(s, curfn->nbody); + } + heapmoves(); + if(debug['W'] && curfn->enter != nil) { + snprint(s, sizeof(s), "enter %S", curfn->nname->sym); + dumplist(s, curfn->enter); + } +} + + +void +walkstmtlist(NodeList *l) +{ + for(; l; l=l->next) + walkstmt(&l->n); +} + +static int +samelist(NodeList *a, NodeList *b) +{ + for(; a && b; a=a->next, b=b->next) + if(a->n != b->n) + return 0; + return a == b; +} + +static int +paramoutheap(Node *fn) +{ + NodeList *l; + + for(l=fn->dcl; l; l=l->next) { + switch(l->n->class) { + case PPARAMOUT|PHEAP: + return 1; + case PAUTO: + case PAUTO|PHEAP: + // stop early - parameters are over + return 0; + } + } + return 0; +} + +void +walkstmt(Node **np) +{ + NodeList *init; + NodeList *ll, *rl; + int cl; + Node *n, *f; + + n = *np; + if(n == N) + return; + + setlineno(n); + + switch(n->op) { + default: + if(n->op == ONAME) + yyerror("%S is not a top level statement", n->sym); + else + yyerror("%O is not a top level statement", n->op); + dump("nottop", n); + break; + + case OASOP: + case OAS: + case OAS2: + case OAS2DOTTYPE: + case OAS2RECV: + case OAS2FUNC: + case OAS2MAPW: + case OAS2MAPR: + case OCLOSE: + case OCOPY: + case OCALLMETH: + case OCALLINTER: + case OCALL: + case OCALLFUNC: + case OSEND: + case ORECV: + case OPRINT: + case OPRINTN: + case OPANIC: + case OEMPTY: + case ORECOVER: + if(n->typecheck == 0) { + dump("missing typecheck:", n); + fatal("missing typecheck"); + } + init = n->ninit; + n->ninit = nil; + walkexpr(&n, &init); + n->ninit = concat(init, n->ninit); + break; + + case OBREAK: + case ODCL: + case OCONTINUE: + case OFALL: + case OGOTO: + case OLABEL: + case ODCLCONST: + case ODCLTYPE: + break; + + case OBLOCK: + walkstmtlist(n->list); + break; + + case OXCASE: + yyerror("case statement out of place"); + n->op = OCASE; + case OCASE: + walkstmt(&n->right); + break; + + case ODEFER: + hasdefer = 1; + switch(n->left->op) { + case OPRINT: + case OPRINTN: + walkexprlist(n->left->list, &n->ninit); + n->left = walkprint(n->left, &n->ninit, 1); + break; + default: + walkexpr(&n->left, &n->ninit); + break; + } + break; + + case OFOR: + walkstmtlist(n->ninit); + if(n->ntest != N) { + walkstmtlist(n->ntest->ninit); + init = n->ntest->ninit; + n->ntest->ninit = nil; + walkexpr(&n->ntest, &init); + n->ntest->ninit = concat(init, n->ntest->ninit); + } + walkstmt(&n->nincr); + walkstmtlist(n->nbody); + break; + + case OIF: + walkstmtlist(n->ninit); + walkexpr(&n->ntest, &n->ninit); + walkstmtlist(n->nbody); + walkstmtlist(n->nelse); + break; + + case OPROC: + switch(n->left->op) { + case OPRINT: + case OPRINTN: + walkexprlist(n->left->list, &n->ninit); + n->left = walkprint(n->left, &n->ninit, 1); + break; + default: + walkexpr(&n->left, &n->ninit); + break; + } + break; + + case ORETURN: + walkexprlist(n->list, &n->ninit); + if(n->list == nil) + break; + if((curfn->type->outnamed && count(n->list) > 1) || paramoutheap(curfn)) { + // assign to the function out parameters, + // so that reorder3 can fix up conflicts + rl = nil; + for(ll=curfn->dcl; ll != nil; ll=ll->next) { + cl = ll->n->class & ~PHEAP; + if(cl == PAUTO) + break; + if(cl == PPARAMOUT) + rl = list(rl, ll->n); + } + if(samelist(rl, n->list)) { + // special return in disguise + n->list = nil; + break; + } + if(count(n->list) == 1 && count(rl) > 1) { + // OAS2FUNC in disguise + f = n->list->n; + if(f->op != OCALLFUNC && f->op != OCALLMETH && f->op != OCALLINTER) + fatal("expected return of call, have %#N", f); + n->list = concat(list1(f), ascompatet(n->op, rl, &f->type, 0, &n->ninit)); + break; + } + ll = ascompatee(n->op, rl, n->list, &n->ninit); + n->list = reorder3(ll); + break; + } + ll = ascompatte(n->op, 0, getoutarg(curfn->type), n->list, 1, &n->ninit); + n->list = ll; + break; + + case OSELECT: + walkselect(n); + break; + + case OSWITCH: + walkswitch(n); + break; + + case ORANGE: + walkrange(n); + break; + + case OXFALL: + yyerror("fallthrough statement out of place"); + n->op = OFALL; + break; + } + + *np = n; +} + + +/* + * walk the whole tree of the body of an + * expression or simple statement. + * the types expressions are calculated. + * compile-time constants are evaluated. + * complex side effects like statements are appended to init + */ + +void +walkexprlist(NodeList *l, NodeList **init) +{ + for(; l; l=l->next) + walkexpr(&l->n, init); +} + +void +walkexprlistsafe(NodeList *l, NodeList **init) +{ + for(; l; l=l->next) { + l->n = safeexpr(l->n, init); + walkexpr(&l->n, init); + } +} + +void +walkexpr(Node **np, NodeList **init) +{ + Node *r, *l, *var, *a; + NodeList *ll, *lr, *lpost; + Type *t; + int et; + int64 v, v1, v2, len; + int32 lno; + Node *n, *fn; + char buf[100], *p; + + n = *np; + + if(n == N) + return; + + if(init == &n->ninit) { + // not okay to use n->ninit when walking n, + // because we might replace n with some other node + // and would lose the init list. + fatal("walkexpr init == &n->ninit"); + } + + // annoying case - not typechecked + if(n->op == OKEY) { + walkexpr(&n->left, init); + walkexpr(&n->right, init); + return; + } + + lno = setlineno(n); + + if(debug['w'] > 1) + dump("walk-before", n); + + if(n->typecheck != 1) { + dump("missed typecheck", n); + fatal("missed typecheck"); + } + + t = T; + et = Txxx; + + switch(n->op) { + default: + dump("walk", n); + fatal("walkexpr: switch 1 unknown op %N", n); + goto ret; + + case OTYPE: + case ONONAME: + case OINDREG: + case OEMPTY: + goto ret; + + case ONOT: + case OMINUS: + case OPLUS: + case OCOM: + case OREAL: + case OIMAG: + case ODOT: + case ODOTPTR: + case ODOTMETH: + case ODOTINTER: + case OIND: + walkexpr(&n->left, init); + goto ret; + + case OLEN: + case OCAP: + walkexpr(&n->left, init); + + // replace len(*[10]int) with 10. + // delayed until now to preserve side effects. + t = n->left->type; + if(isptr[t->etype]) + t = t->type; + if(isfixedarray(t)) { + safeexpr(n->left, init); + nodconst(n, n->type, t->bound); + n->typecheck = 1; + } + goto ret; + + case OLSH: + case ORSH: + case OAND: + case OOR: + case OXOR: + case OSUB: + case OMUL: + case OEQ: + case ONE: + case OLT: + case OLE: + case OGE: + case OGT: + case OADD: + case OCOMPLEX: + walkexpr(&n->left, init); + walkexpr(&n->right, init); + goto ret; + + case OANDAND: + case OOROR: + walkexpr(&n->left, init); + // cannot put side effects from n->right on init, + // because they cannot run before n->left is checked. + // save elsewhere and store on the eventual n->right. + ll = nil; + walkexpr(&n->right, &ll); + n->right->ninit = concat(n->right->ninit, ll); + goto ret; + + case OPRINT: + case OPRINTN: + walkexprlist(n->list, init); + n = walkprint(n, init, 0); + goto ret; + + case OPANIC: + n = mkcall("panic", T, init, n->left); + goto ret; + + case ORECOVER: + n = mkcall("recover", n->type, init, nod(OADDR, nodfp, N)); + goto ret; + + case OLITERAL: + n->addable = 1; + goto ret; + + case ONAME: + if(!(n->class & PHEAP) && n->class != PPARAMREF) + n->addable = 1; + goto ret; + + case OCALLINTER: + t = n->left->type; + if(n->list && n->list->n->op == OAS) + goto ret; + walkexpr(&n->left, init); + walkexprlist(n->list, init); + ll = ascompatte(n->op, n->isddd, getinarg(t), n->list, 0, init); + n->list = reorder1(ll); + goto ret; + + case OCALLFUNC: + t = n->left->type; + if(n->list && n->list->n->op == OAS) + goto ret; + + if(n->left->op == OCLOSURE) { + walkcallclosure(n, init); + t = n->left->type; + } + + walkexpr(&n->left, init); + walkexprlist(n->list, init); + + ll = ascompatte(n->op, n->isddd, getinarg(t), n->list, 0, init); + n->list = reorder1(ll); + goto ret; + + case OCALLMETH: + t = n->left->type; + if(n->list && n->list->n->op == OAS) + goto ret; + walkexpr(&n->left, init); + walkexprlist(n->list, init); + ll = ascompatte(n->op, 0, getthis(t), list1(n->left->left), 0, init); + lr = ascompatte(n->op, n->isddd, getinarg(t), n->list, 0, init); + ll = concat(ll, lr); + n->left->left = N; + ullmancalc(n->left); + n->list = reorder1(ll); + goto ret; + + case OAS: + *init = concat(*init, n->ninit); + n->ninit = nil; + walkexpr(&n->left, init); + n->left = safeexpr(n->left, init); + + if(oaslit(n, init)) + goto ret; + + walkexpr(&n->right, init); + if(n->left != N && n->right != N) { + r = convas(nod(OAS, n->left, n->right), init); + r->dodata = n->dodata; + n = r; + } + + goto ret; + + case OAS2: + *init = concat(*init, n->ninit); + n->ninit = nil; + walkexprlistsafe(n->list, init); + walkexprlistsafe(n->rlist, init); + ll = ascompatee(OAS, n->list, n->rlist, init); + ll = reorder3(ll); + n = liststmt(ll); + goto ret; + + case OAS2FUNC: + as2func: + // a,b,... = fn() + *init = concat(*init, n->ninit); + n->ninit = nil; + r = n->rlist->n; + walkexprlistsafe(n->list, init); + walkexpr(&r, init); + l = n->list->n; + + // all the really hard stuff - explicit function calls and so on - + // is gone, but map assignments remain. + // if there are map assignments here, assign via + // temporaries, because ascompatet assumes + // the targets can be addressed without function calls + // and map index has an implicit one. + lpost = nil; + if(l->op == OINDEXMAP) { + var = nod(OXXX, N, N); + tempname(var, l->type); + n->list->n = var; + a = nod(OAS, l, var); + typecheck(&a, Etop); + lpost = list(lpost, a); + } + l = n->list->next->n; + if(l->op == OINDEXMAP) { + var = nod(OXXX, N, N); + tempname(var, l->type); + n->list->next->n = var; + a = nod(OAS, l, var); + typecheck(&a, Etop); + lpost = list(lpost, a); + } + ll = ascompatet(n->op, n->list, &r->type, 0, init); + walkexprlist(lpost, init); + n = liststmt(concat(concat(list1(r), ll), lpost)); + goto ret; + + case OAS2RECV: + *init = concat(*init, n->ninit); + n->ninit = nil; + r = n->rlist->n; + walkexprlistsafe(n->list, init); + walkexpr(&r->left, init); + fn = chanfn("chanrecv2", 2, r->left->type); + r = mkcall1(fn, getoutargx(fn->type), init, typename(r->left->type), r->left); + n->rlist->n = r; + n->op = OAS2FUNC; + goto as2func; + + case OAS2MAPR: + // a,b = m[i]; + *init = concat(*init, n->ninit); + n->ninit = nil; + r = n->rlist->n; + walkexprlistsafe(n->list, init); + walkexpr(&r->left, init); + fn = mapfn("mapaccess2", r->left->type); + r = mkcall1(fn, getoutargx(fn->type), init, typename(r->left->type), r->left, r->right); + n->rlist = list1(r); + n->op = OAS2FUNC; + goto as2func; + + case OAS2MAPW: + // map[] = a,b - mapassign2 + // a,b = m[i]; + *init = concat(*init, n->ninit); + n->ninit = nil; + walkexprlistsafe(n->list, init); + l = n->list->n; + t = l->left->type; + n = mkcall1(mapfn("mapassign2", t), T, init, typename(t), l->left, l->right, n->rlist->n, n->rlist->next->n); + goto ret; + + case OAS2DOTTYPE: + // a,b = i.(T) + *init = concat(*init, n->ninit); + n->ninit = nil; + r = n->rlist->n; + walkexprlistsafe(n->list, init); + r->op = ODOTTYPE2; + walkexpr(&r, init); + ll = ascompatet(n->op, n->list, &r->type, 0, init); + n = liststmt(concat(list1(r), ll)); + goto ret; + + case ODOTTYPE: + case ODOTTYPE2: + // Build name of function: assertI2E2 etc. + strcpy(buf, "assert"); + p = buf+strlen(buf); + if(isnilinter(n->left->type)) + *p++ = 'E'; + else + *p++ = 'I'; + *p++ = '2'; + if(isnilinter(n->type)) + *p++ = 'E'; + else if(isinter(n->type)) + *p++ = 'I'; + else + *p++ = 'T'; + if(n->op == ODOTTYPE2) + *p++ = '2'; + *p = '\0'; + + fn = syslook(buf, 1); + ll = list1(typename(n->type)); + ll = list(ll, n->left); + argtype(fn, n->left->type); + argtype(fn, n->type); + n = nod(OCALL, fn, N); + n->list = ll; + typecheck(&n, Erv | Efnstruct); + walkexpr(&n, init); + goto ret; + + case OCONVIFACE: + // Build name of function: convI2E etc. + // Not all names are possible + // (e.g., we'll never generate convE2E or convE2I). + walkexpr(&n->left, init); + strcpy(buf, "conv"); + p = buf+strlen(buf); + if(isnilinter(n->left->type)) + *p++ = 'E'; + else if(isinter(n->left->type)) + *p++ = 'I'; + else + *p++ = 'T'; + *p++ = '2'; + if(isnilinter(n->type)) + *p++ = 'E'; + else + *p++ = 'I'; + *p = '\0'; + + fn = syslook(buf, 1); + ll = nil; + if(!isinter(n->left->type)) + ll = list(ll, typename(n->left->type)); + if(!isnilinter(n->type)) + ll = list(ll, typename(n->type)); + ll = list(ll, n->left); + argtype(fn, n->left->type); + argtype(fn, n->type); + dowidth(fn->type); + n = nod(OCALL, fn, N); + n->list = ll; + typecheck(&n, Erv); + walkexpr(&n, init); + goto ret; + + case OCONV: + case OCONVNOP: + if(thechar == '5') { + if(isfloat[n->left->type->etype]) { + if(n->type->etype == TINT64) { + n = mkcall("float64toint64", n->type, init, conv(n->left, types[TFLOAT64])); + goto ret; + } + if(n->type->etype == TUINT64) { + n = mkcall("float64touint64", n->type, init, conv(n->left, types[TFLOAT64])); + goto ret; + } + } + if(isfloat[n->type->etype]) { + if(n->left->type->etype == TINT64) { + n = mkcall("int64tofloat64", n->type, init, conv(n->left, types[TINT64])); + goto ret; + } + if(n->left->type->etype == TUINT64) { + n = mkcall("uint64tofloat64", n->type, init, conv(n->left, types[TUINT64])); + goto ret; + } + } + } + walkexpr(&n->left, init); + goto ret; + + case OASOP: + if(n->etype == OANDNOT) { + n->etype = OAND; + n->right = nod(OCOM, n->right, N); + typecheck(&n->right, Erv); + } + n->left = safeexpr(n->left, init); + walkexpr(&n->left, init); + l = n->left; + walkexpr(&n->right, init); + + /* + * on 32-bit arch, rewrite 64-bit ops into l = l op r. + * on 386, rewrite float ops into l = l op r. + * everywhere, rewrite map ops into l = l op r. + * everywhere, rewrite string += into l = l op r. + * everywhere, rewrite complex /= into l = l op r. + * TODO(rsc): Maybe this rewrite should be done always? + */ + et = n->left->type->etype; + if((widthptr == 4 && (et == TUINT64 || et == TINT64)) || + (thechar == '8' && isfloat[et]) || + l->op == OINDEXMAP || + et == TSTRING || + (iscomplex[et] && n->etype == ODIV)) { + l = safeexpr(n->left, init); + a = l; + if(a->op == OINDEXMAP) { + // map index has "lhs" bit set in a->etype. + // make a copy so we can clear it on the rhs. + a = nod(OXXX, N, N); + *a = *l; + a->etype = 0; + } + r = nod(OAS, l, nod(n->etype, a, n->right)); + typecheck(&r, Etop); + walkexpr(&r, init); + n = r; + } + goto ret; + + case OANDNOT: + walkexpr(&n->left, init); + walkexpr(&n->right, init); + n->op = OAND; + n->right = nod(OCOM, n->right, N); + typecheck(&n->right, Erv); + goto ret; + + case ODIV: + case OMOD: + walkexpr(&n->left, init); + walkexpr(&n->right, init); + /* + * rewrite complex div into function call. + */ + et = n->left->type->etype; + if(iscomplex[et] && n->op == ODIV) { + t = n->type; + n = mkcall("complex128div", types[TCOMPLEX128], init, + conv(n->left, types[TCOMPLEX128]), + conv(n->right, types[TCOMPLEX128])); + n = conv(n, t); + goto ret; + } + /* + * rewrite div and mod into function calls + * on 32-bit architectures. + */ + if(widthptr > 4 || (et != TUINT64 && et != TINT64)) + goto ret; + if(et == TINT64) + strcpy(namebuf, "int64"); + else + strcpy(namebuf, "uint64"); + if(n->op == ODIV) + strcat(namebuf, "div"); + else + strcat(namebuf, "mod"); + n = mkcall(namebuf, n->type, init, + conv(n->left, types[et]), conv(n->right, types[et])); + goto ret; + + case OINDEX: + walkexpr(&n->left, init); + walkexpr(&n->right, init); + + // if range of type cannot exceed static array bound, + // disable bounds check + if(isfixedarray(n->left->type)) + if(n->right->type->width < 4) + if((1<<(8*n->right->type->width)) <= n->left->type->bound) + n->etype = 1; + + if(isconst(n->left, CTSTR)) + if(n->right->type->width < 4) + if((1<<(8*n->right->type->width)) <= n->left->val.u.sval->len) + n->etype = 1; + + // check for static out of bounds + if(isconst(n->right, CTINT) && !n->etype) { + v = mpgetfix(n->right->val.u.xval); + len = 1LL<<60; + t = n->left->type; + if(isconst(n->left, CTSTR)) + len = n->left->val.u.sval->len; + if(t != T && isptr[t->etype]) + t = t->type; + if(isfixedarray(t)) + len = t->bound; + if(v < 0 || v >= (1LL<<31) || v >= len) + yyerror("index out of bounds"); + else if(isconst(n->left, CTSTR)) { + // replace "abc"[2] with 'b'. + // delayed until now because "abc"[2] is not + // an ideal constant. + nodconst(n, n->type, n->left->val.u.sval->s[v]); + } + } + goto ret; + + case OINDEXMAP: + if(n->etype == 1) + goto ret; + + t = n->left->type; + n = mkcall1(mapfn("mapaccess1", t), t->type, init, typename(t), n->left, n->right); + goto ret; + + case ORECV: + walkexpr(&n->left, init); + walkexpr(&n->right, init); + n = mkcall1(chanfn("chanrecv1", 2, n->left->type), n->type, init, typename(n->left->type), n->left); + goto ret; + + case OSLICE: + case OSLICEARR: + walkexpr(&n->left, init); + n->left = safeexpr(n->left, init); + walkexpr(&n->right->left, init); + n->right->left = safeexpr(n->right->left, init); + walkexpr(&n->right->right, init); + n->right->right = safeexpr(n->right->right, init); + + len = 1LL<<60; + t = n->left->type; + if(t != T && isptr[t->etype]) + t = t->type; + if(isfixedarray(t)) + len = t->bound; + + // check for static out of bounds + // NOTE: v > len not v >= len. + v1 = -1; + v2 = -1; + if(isconst(n->right->left, CTINT)) { + v1 = mpgetfix(n->right->left->val.u.xval); + if(v1 < 0 || v1 >= (1LL<<31) || v1 > len) { + yyerror("slice index out of bounds"); + v1 = -1; + } + } + if(isconst(n->right->right, CTINT)) { + v2 = mpgetfix(n->right->right->val.u.xval); + if(v2 < 0 || v2 >= (1LL<<31) || v2 > len) { + yyerror("slice index out of bounds"); + v2 = -1; + } + } + if(v1 >= 0 && v2 >= 0 && v1 > v2) + yyerror("inverted slice range"); + + if(n->op == OSLICEARR) + goto slicearray; + + // dynamic slice + // sliceslice(old []any, lb uint64, hb uint64, width uint64) (ary []any) + // sliceslice1(old []any, lb uint64, width uint64) (ary []any) + t = n->type; + et = n->etype; + if(n->right->left == N) + l = nodintconst(0); + else + l = conv(n->right->left, types[TUINT64]); + if(n->right->right != N) { + fn = syslook("sliceslice", 1); + argtype(fn, t->type); // any-1 + argtype(fn, t->type); // any-2 + n = mkcall1(fn, t, init, + n->left, + l, + conv(n->right->right, types[TUINT64]), + nodintconst(t->type->width)); + } else { + fn = syslook("sliceslice1", 1); + argtype(fn, t->type); // any-1 + argtype(fn, t->type); // any-2 + n = mkcall1(fn, t, init, + n->left, + l, + nodintconst(t->type->width)); + } + n->etype = et; // preserve no-typecheck flag from OSLICE to the slice* call. + goto ret; + + slicearray: + // static slice + // slicearray(old *any, uint64 nel, lb uint64, hb uint64, width uint64) (ary []any) + t = n->type; + fn = syslook("slicearray", 1); + argtype(fn, n->left->type->type); // any-1 + argtype(fn, t->type); // any-2 + if(n->right->left == N) + l = nodintconst(0); + else + l = conv(n->right->left, types[TUINT64]); + if(n->right->right == N) + r = nodintconst(n->left->type->type->bound); + else + r = conv(n->right->right, types[TUINT64]); + n = mkcall1(fn, t, init, + n->left, nodintconst(n->left->type->type->bound), + l, + r, + nodintconst(t->type->width)); + goto ret; + + case OADDR:; + Node *nvar, *nstar; + + // turn &Point(1, 2) or &[]int(1, 2) or &[...]int(1, 2) into allocation. + // initialize with + // nvar := new(*Point); + // *nvar = Point(1, 2); + // and replace expression with nvar + switch(n->left->op) { + case OARRAYLIT: + case OMAPLIT: + case OSTRUCTLIT: + nvar = makenewvar(n->type, init, &nstar); + anylit(0, n->left, nstar, init); + n = nvar; + goto ret; + } + + walkexpr(&n->left, init); + goto ret; + + case ONEW: + n = callnew(n->type->type); + goto ret; + + case OCMPSTR: + // If one argument to the comparison is an empty string, + // comparing the lengths instead will yield the same result + // without the function call. + if((isconst(n->left, CTSTR) && n->left->val.u.sval->len == 0) || + (isconst(n->right, CTSTR) && n->right->val.u.sval->len == 0)) { + r = nod(n->etype, nod(OLEN, n->left, N), nod(OLEN, n->right, N)); + typecheck(&r, Erv); + walkexpr(&r, init); + n = r; + goto ret; + } + + // s + "badgerbadgerbadger" == "badgerbadgerbadger" + if((n->etype == OEQ || n->etype == ONE) && + isconst(n->right, CTSTR) && + n->left->op == OADDSTR && isconst(n->left->right, CTSTR) && + cmpslit(n->right, n->left->right) == 0) { + r = nod(n->etype, nod(OLEN, n->left->left, N), nodintconst(0)); + typecheck(&r, Erv); + walkexpr(&r, init); + n = r; + goto ret; + } + + // prepare for rewrite below + if(n->etype == OEQ || n->etype == ONE) { + n->left = cheapexpr(n->left, init); + n->right = cheapexpr(n->right, init); + } + + // sys_cmpstring(s1, s2) :: 0 + r = mkcall("cmpstring", types[TINT], init, + conv(n->left, types[TSTRING]), + conv(n->right, types[TSTRING])); + r = nod(n->etype, r, nodintconst(0)); + + // quick check of len before full compare for == or != + if(n->etype == OEQ || n->etype == ONE) { + if(n->etype == OEQ) + r = nod(OANDAND, nod(OEQ, nod(OLEN, n->left, N), nod(OLEN, n->right, N)), r); + else + r = nod(OOROR, nod(ONE, nod(OLEN, n->left, N), nod(OLEN, n->right, N)), r); + typecheck(&r, Erv); + walkexpr(&r, nil); + } + typecheck(&r, Erv); + n = r; + goto ret; + + case OADDSTR: + n = addstr(n, init); + goto ret; + + case OSLICESTR: + // sys_slicestring(s, lb, hb) + if(n->right->left == N) + l = nodintconst(0); + else + l = conv(n->right->left, types[TINT]); + if(n->right->right) { + n = mkcall("slicestring", n->type, init, + conv(n->left, types[TSTRING]), + l, + conv(n->right->right, types[TINT])); + } else { + n = mkcall("slicestring1", n->type, init, + conv(n->left, types[TSTRING]), + l); + } + goto ret; + + case OAPPEND: + if(n->isddd) + n = appendslice(n, init); + else + n = append(n, init); + goto ret; + + case OCOPY: + if(n->right->type->etype == TSTRING) + fn = syslook("slicestringcopy", 1); + else + fn = syslook("slicecopy", 1); + argtype(fn, n->left->type); + argtype(fn, n->right->type); + n = mkcall1(fn, n->type, init, + n->left, n->right, + nodintconst(n->left->type->type->width)); + goto ret; + + case OCLOSE: + // cannot use chanfn - closechan takes any, not chan any + fn = syslook("closechan", 1); + argtype(fn, n->left->type); + n = mkcall1(fn, T, init, n->left); + goto ret; + + case OMAKECHAN: + n = mkcall1(chanfn("makechan", 1, n->type), n->type, init, + typename(n->type), + conv(n->left, types[TINT64])); + goto ret; + + case OMAKEMAP: + t = n->type; + + fn = syslook("makemap", 1); + argtype(fn, t->down); // any-1 + argtype(fn, t->type); // any-2 + + n = mkcall1(fn, n->type, init, + typename(n->type), + conv(n->left, types[TINT64])); + goto ret; + + case OMAKESLICE: + // makeslice(t *Type, nel int64, max int64) (ary []any) + l = n->left; + r = n->right; + if(r == nil) + l = r = safeexpr(l, init); + t = n->type; + fn = syslook("makeslice", 1); + argtype(fn, t->type); // any-1 + n = mkcall1(fn, n->type, init, + typename(n->type), + conv(l, types[TINT64]), + conv(r, types[TINT64])); + goto ret; + + case ORUNESTR: + // sys_intstring(v) + n = mkcall("intstring", n->type, init, + conv(n->left, types[TINT64])); + goto ret; + + case OARRAYBYTESTR: + // slicebytetostring([]byte) string; + n = mkcall("slicebytetostring", n->type, init, n->left); + goto ret; + + case OARRAYRUNESTR: + // sliceinttostring([]int) string; + n = mkcall("sliceinttostring", n->type, init, n->left); + goto ret; + + case OSTRARRAYBYTE: + // stringtoslicebyte(string) []byte; + n = mkcall("stringtoslicebyte", n->type, init, conv(n->left, types[TSTRING])); + goto ret; + + case OSTRARRAYRUNE: + // stringtosliceint(string) []int + n = mkcall("stringtosliceint", n->type, init, n->left); + goto ret; + + case OCMPIFACE: + // ifaceeq(i1 any-1, i2 any-2) (ret bool); + if(!eqtype(n->left->type, n->right->type)) + fatal("ifaceeq %O %T %T", n->op, n->left->type, n->right->type); + if(isnilinter(n->left->type)) + fn = syslook("efaceeq", 1); + else + fn = syslook("ifaceeq", 1); + argtype(fn, n->right->type); + argtype(fn, n->left->type); + r = mkcall1(fn, n->type, init, n->left, n->right); + if(n->etype == ONE) { + r = nod(ONOT, r, N); + typecheck(&r, Erv); + } + n = r; + goto ret; + + case OARRAYLIT: + case OMAPLIT: + case OSTRUCTLIT: + nvar = nod(OXXX, N, N); + tempname(nvar, n->type); + anylit(0, n, nvar, init); + n = nvar; + goto ret; + + case OSEND: + n = mkcall1(chanfn("chansend1", 2, n->left->type), T, init, typename(n->left->type), n->left, n->right); + goto ret; + + case OCLOSURE: + n = walkclosure(n, init); + goto ret; + } + fatal("missing switch %O", n->op); + +ret: + if(debug['w'] && n != N) + dump("walk", n); + + ullmancalc(n); + lineno = lno; + *np = n; +} + +static Node* +makenewvar(Type *t, NodeList **init, Node **nstar) +{ + Node *nvar, *nas; + + nvar = nod(OXXX, N, N); + tempname(nvar, t); + nas = nod(OAS, nvar, callnew(t->type)); + typecheck(&nas, Etop); + walkexpr(&nas, init); + *init = list(*init, nas); + + *nstar = nod(OIND, nvar, N); + typecheck(nstar, Erv); + return nvar; +} + +static Node* +ascompatee1(int op, Node *l, Node *r, NodeList **init) +{ + return convas(nod(OAS, l, r), init); +} + +static NodeList* +ascompatee(int op, NodeList *nl, NodeList *nr, NodeList **init) +{ + NodeList *ll, *lr, *nn; + + /* + * check assign expression list to + * a expression list. called in + * expr-list = expr-list + */ + + // ensure order of evaluation for function calls + for(ll=nl; ll; ll=ll->next) + ll->n = safeexpr(ll->n, init); + for(lr=nr; lr; lr=lr->next) + lr->n = safeexpr(lr->n, init); + + nn = nil; + for(ll=nl, lr=nr; ll && lr; ll=ll->next, lr=lr->next) + nn = list(nn, ascompatee1(op, ll->n, lr->n, init)); + + // cannot happen: caller checked that lists had same length + if(ll || lr) + yyerror("error in shape across %O", op); + return nn; +} + +/* + * l is an lv and rt is the type of an rv + * return 1 if this implies a function call + * evaluating the lv or a function call + * in the conversion of the types + */ +static int +fncall(Node *l, Type *rt) +{ + if(l->ullman >= UINF || l->op == OINDEXMAP) + return 1; + if(eqtype(l->type, rt)) + return 0; + return 1; +} + +static NodeList* +ascompatet(int op, NodeList *nl, Type **nr, int fp, NodeList **init) +{ + Node *l, *tmp, *a; + NodeList *ll; + Type *r; + Iter saver; + int ucount; + NodeList *nn, *mm; + + /* + * check assign type list to + * a expression list. called in + * expr-list = func() + */ + r = structfirst(&saver, nr); + nn = nil; + mm = nil; + ucount = 0; + for(ll=nl; ll; ll=ll->next) { + if(r == T) + break; + l = ll->n; + if(isblank(l)) { + r = structnext(&saver); + continue; + } + + // any lv that causes a fn call must be + // deferred until all the return arguments + // have been pulled from the output arguments + if(fncall(l, r->type)) { + tmp = nod(OXXX, N, N); + tempname(tmp, r->type); + typecheck(&tmp, Erv); + a = nod(OAS, l, tmp); + a = convas(a, init); + mm = list(mm, a); + l = tmp; + } + + a = nod(OAS, l, nodarg(r, fp)); + a = convas(a, init); + ullmancalc(a); + if(a->ullman >= UINF) + ucount++; + nn = list(nn, a); + r = structnext(&saver); + } + + if(ll != nil || r != T) + yyerror("assignment count mismatch: %d = %d", + count(nl), structcount(*nr)); + if(ucount) + fatal("reorder2: too many function calls evaluating parameters"); + return concat(nn, mm); +} + + /* + * package all the arguments that match a ... T parameter into a []T. + */ +static NodeList* +mkdotargslice(NodeList *lr0, NodeList *nn, Type *l, int fp, NodeList **init) +{ + Node *a, *n; + Type *tslice; + + tslice = typ(TARRAY); + tslice->type = l->type->type; + tslice->bound = -1; + + n = nod(OCOMPLIT, N, typenod(tslice)); + n->list = lr0; + typecheck(&n, Erv); + if(n->type == T) + fatal("mkdotargslice: typecheck failed"); + walkexpr(&n, init); + + a = nod(OAS, nodarg(l, fp), n); + nn = list(nn, convas(a, init)); + return nn; +} + +/* + * helpers for shape errors + */ +static char* +dumptypes(Type **nl, char *what) +{ + int first; + Type *l; + Iter savel; + Fmt fmt; + + fmtstrinit(&fmt); + fmtprint(&fmt, "\t"); + l = structfirst(&savel, nl); + first = 1; + for(l = structfirst(&savel, nl); l != T; l = structnext(&savel)) { + if(first) + first = 0; + else + fmtprint(&fmt, ", "); + fmtprint(&fmt, "%T", l); + } + if(first) + fmtprint(&fmt, "[no arguments %s]", what); + return fmtstrflush(&fmt); +} + +static char* +dumpnodetypes(NodeList *l, char *what) +{ + int first; + Node *r; + Fmt fmt; + + fmtstrinit(&fmt); + fmtprint(&fmt, "\t"); + first = 1; + for(; l; l=l->next) { + r = l->n; + if(first) + first = 0; + else + fmtprint(&fmt, ", "); + fmtprint(&fmt, "%T", r->type); + } + if(first) + fmtprint(&fmt, "[no arguments %s]", what); + return fmtstrflush(&fmt); +} + +/* + * check assign expression list to + * a type list. called in + * return expr-list + * func(expr-list) + */ +static NodeList* +ascompatte(int op, int isddd, Type **nl, NodeList *lr, int fp, NodeList **init) +{ + Type *l, *ll; + Node *r, *a; + NodeList *nn, *lr0, *alist; + Iter savel; + char *l1, *l2; + + lr0 = lr; + l = structfirst(&savel, nl); + r = N; + if(lr) + r = lr->n; + nn = nil; + + // f(g()) where g has multiple return values + if(r != N && lr->next == nil && r->type->etype == TSTRUCT && r->type->funarg) { + // optimization - can do block copy + if(eqtypenoname(r->type, *nl)) { + a = nodarg(*nl, fp); + a->type = r->type; + nn = list1(convas(nod(OAS, a, r), init)); + goto ret; + } + + // conversions involved. + // copy into temporaries. + alist = nil; + for(l=structfirst(&savel, &r->type); l; l=structnext(&savel)) { + a = nod(OXXX, N, N); + tempname(a, l->type); + alist = list(alist, a); + } + a = nod(OAS2, N, N); + a->list = alist; + a->rlist = lr; + typecheck(&a, Etop); + walkstmt(&a); + *init = list(*init, a); + lr = alist; + r = lr->n; + l = structfirst(&savel, nl); + } + +loop: + if(l != T && l->isddd) { + // the ddd parameter must be last + ll = structnext(&savel); + if(ll != T) + yyerror("... must be last argument"); + + // special case -- + // only if we are assigning a single ddd + // argument to a ddd parameter then it is + // passed thru unencapsulated + if(r != N && lr->next == nil && isddd && eqtype(l->type, r->type)) { + a = nod(OAS, nodarg(l, fp), r); + a = convas(a, init); + nn = list(nn, a); + goto ret; + } + + // normal case -- make a slice of all + // remaining arguments and pass it to + // the ddd parameter. + nn = mkdotargslice(lr, nn, l, fp, init); + goto ret; + } + + if(l == T || r == N) { + if(l != T || r != N) { + l1 = dumptypes(nl, "expected"); + l2 = dumpnodetypes(lr0, "given"); + if(l != T) + yyerror("not enough arguments to %O\n%s\n%s", op, l1, l2); + else + yyerror("too many arguments to %O\n%s\n%s", op, l1, l2); + } + goto ret; + } + + a = nod(OAS, nodarg(l, fp), r); + a = convas(a, init); + nn = list(nn, a); + + l = structnext(&savel); + r = N; + lr = lr->next; + if(lr != nil) + r = lr->n; + goto loop; + +ret: + for(lr=nn; lr; lr=lr->next) + lr->n->typecheck = 1; + return nn; +} + +// generate code for print +static Node* +walkprint(Node *nn, NodeList **init, int defer) +{ + Node *r; + Node *n; + NodeList *l, *all; + Node *on; + Type *t; + int notfirst, et, op; + NodeList *calls, *intypes, *args; + Fmt fmt; + + on = nil; + op = nn->op; + all = nn->list; + calls = nil; + notfirst = 0; + intypes = nil; + args = nil; + + memset(&fmt, 0, sizeof fmt); + if(defer) { + // defer print turns into defer printf with format string + fmtstrinit(&fmt); + intypes = list(intypes, nod(ODCLFIELD, N, typenod(types[TSTRING]))); + args = list1(nod(OXXX, N, N)); + } + + for(l=all; l; l=l->next) { + if(notfirst) { + if(defer) + fmtprint(&fmt, " "); + else + calls = list(calls, mkcall("printsp", T, init)); + } + notfirst = op == OPRINTN; + + n = l->n; + if(n->op == OLITERAL) { + switch(n->val.ctype) { + case CTINT: + defaultlit(&n, types[TINT64]); + break; + case CTFLT: + defaultlit(&n, types[TFLOAT64]); + break; + } + } + if(n->op != OLITERAL && n->type && n->type->etype == TIDEAL) + defaultlit(&n, types[TINT64]); + defaultlit(&n, nil); + l->n = n; + if(n->type == T || n->type->etype == TFORW) + continue; + + t = n->type; + et = n->type->etype; + if(isinter(n->type)) { + if(defer) { + if(isnilinter(n->type)) + fmtprint(&fmt, "%%e"); + else + fmtprint(&fmt, "%%i"); + } else { + if(isnilinter(n->type)) + on = syslook("printeface", 1); + else + on = syslook("printiface", 1); + argtype(on, n->type); // any-1 + } + } else if(isptr[et] || et == TCHAN || et == TMAP || et == TFUNC || et == TUNSAFEPTR) { + if(defer) { + fmtprint(&fmt, "%%p"); + } else { + on = syslook("printpointer", 1); + argtype(on, n->type); // any-1 + } + } else if(isslice(n->type)) { + if(defer) { + fmtprint(&fmt, "%%a"); + } else { + on = syslook("printslice", 1); + argtype(on, n->type); // any-1 + } + } else if(isint[et]) { + if(defer) { + if(et == TUINT64) + fmtprint(&fmt, "%%U"); + else { + fmtprint(&fmt, "%%D"); + t = types[TINT64]; + } + } else { + if(et == TUINT64) + on = syslook("printuint", 0); + else + on = syslook("printint", 0); + } + } else if(isfloat[et]) { + if(defer) { + fmtprint(&fmt, "%%f"); + t = types[TFLOAT64]; + } else + on = syslook("printfloat", 0); + } else if(iscomplex[et]) { + if(defer) { + fmtprint(&fmt, "%%C"); + t = types[TCOMPLEX128]; + } else + on = syslook("printcomplex", 0); + } else if(et == TBOOL) { + if(defer) + fmtprint(&fmt, "%%t"); + else + on = syslook("printbool", 0); + } else if(et == TSTRING) { + if(defer) + fmtprint(&fmt, "%%S"); + else + on = syslook("printstring", 0); + } else { + badtype(OPRINT, n->type, T); + continue; + } + + if(!defer) { + t = *getinarg(on->type); + if(t != nil) + t = t->type; + if(t != nil) + t = t->type; + } + + if(!eqtype(t, n->type)) { + n = nod(OCONV, n, N); + n->type = t; + } + + if(defer) { + intypes = list(intypes, nod(ODCLFIELD, N, typenod(t))); + args = list(args, n); + } else { + r = nod(OCALL, on, N); + r->list = list1(n); + calls = list(calls, r); + } + } + + if(defer) { + if(op == OPRINTN) + fmtprint(&fmt, "\n"); + on = syslook("goprintf", 1); + on->type = functype(nil, intypes, nil); + args->n = nod(OLITERAL, N, N); + args->n->val.ctype = CTSTR; + args->n->val.u.sval = strlit(fmtstrflush(&fmt)); + r = nod(OCALL, on, N); + r->list = args; + typecheck(&r, Etop); + walkexpr(&r, init); + } else { + if(op == OPRINTN) + calls = list(calls, mkcall("printnl", T, nil)); + typechecklist(calls, Etop); + walkexprlist(calls, init); + + r = nod(OEMPTY, N, N); + typecheck(&r, Etop); + walkexpr(&r, init); + r->ninit = calls; + } + return r; +} + +Node* +callnew(Type *t) +{ + Node *fn; + + dowidth(t); + fn = syslook("new", 1); + argtype(fn, t); + return mkcall1(fn, ptrto(t), nil, nodintconst(t->width)); +} + +static Node* +convas(Node *n, NodeList **init) +{ + Type *lt, *rt; + + if(n->op != OAS) + fatal("convas: not OAS %O", n->op); + + n->typecheck = 1; + + if(n->left == N || n->right == N) + goto out; + + lt = n->left->type; + rt = n->right->type; + if(lt == T || rt == T) + goto out; + + if(isblank(n->left)) { + defaultlit(&n->right, T); + goto out; + } + + if(n->left->op == OINDEXMAP) { + n = mkcall1(mapfn("mapassign1", n->left->left->type), T, init, + typename(n->left->left->type), + n->left->left, n->left->right, n->right); + goto out; + } + + if(eqtype(lt, rt)) + goto out; + + n->right = assignconv(n->right, lt, "assignment"); + walkexpr(&n->right, init); + +out: + ullmancalc(n); + return n; +} + +/* + * from ascompat[te] + * evaluating actual function arguments. + * f(a,b) + * if there is exactly one function expr, + * then it is done first. otherwise must + * make temp variables + */ +NodeList* +reorder1(NodeList *all) +{ + Node *f, *a, *n; + NodeList *l, *r, *g; + int c, d, t; + + c = 0; // function calls + t = 0; // total parameters + + for(l=all; l; l=l->next) { + n = l->n; + t++; + ullmancalc(n); + if(n->ullman >= UINF) + c++; + } + if(c == 0 || t == 1) + return all; + + g = nil; // fncalls assigned to tempnames + f = N; // last fncall assigned to stack + r = nil; // non fncalls and tempnames assigned to stack + d = 0; + for(l=all; l; l=l->next) { + n = l->n; + if(n->ullman < UINF) { + r = list(r, n); + continue; + } + d++; + if(d == c) { + f = n; + continue; + } + + // make assignment of fncall to tempname + a = nod(OXXX, N, N); + tempname(a, n->right->type); + a = nod(OAS, a, n->right); + g = list(g, a); + + // put normal arg assignment on list + // with fncall replaced by tempname + n->right = a->left; + r = list(r, n); + } + + if(f != N) + g = list(g, f); + return concat(g, r); +} + +/* + * from ascompat[ee] + * a,b = c,d + * simultaneous assignment. there cannot + * be later use of an earlier lvalue. + */ + +static int +vmatch2(Node *l, Node *r) +{ + NodeList *ll; + + /* + * isolate all right sides + */ + if(r == N) + return 0; + switch(r->op) { + case ONAME: + // match each right given left + if(l == r) + return 1; + case OLITERAL: + return 0; + } + if(vmatch2(l, r->left)) + return 1; + if(vmatch2(l, r->right)) + return 1; + for(ll=r->list; ll; ll=ll->next) + if(vmatch2(l, ll->n)) + return 1; + return 0; +} + +int +vmatch1(Node *l, Node *r) +{ + NodeList *ll; + + /* + * isolate all left sides + */ + if(l == N || r == N) + return 0; + switch(l->op) { + case ONAME: + switch(l->class) { + case PPARAM: + case PPARAMREF: + case PAUTO: + break; + default: + // assignment to non-stack variable + // must be delayed if right has function calls. + if(r->ullman >= UINF) + return 1; + break; + } + return vmatch2(l, r); + case OLITERAL: + return 0; + } + if(vmatch1(l->left, r)) + return 1; + if(vmatch1(l->right, r)) + return 1; + for(ll=l->list; ll; ll=ll->next) + if(vmatch1(ll->n, r)) + return 1; + return 0; +} + +NodeList* +reorder3(NodeList *all) +{ + Node *n1, *n2, *q; + int c1, c2; + NodeList *l1, *l2, *r; + + r = nil; + for(l1=all, c1=0; l1; l1=l1->next, c1++) { + n1 = l1->n; + for(l2=all, c2=0; l2; l2=l2->next, c2++) { + n2 = l2->n; + if(c2 > c1) { + if(vmatch1(n1->left, n2->right)) { + // delay assignment to n1->left + q = nod(OXXX, N, N); + tempname(q, n1->right->type); + q = nod(OAS, n1->left, q); + n1->left = q->right; + r = list(r, q); + break; + } + } + } + } + return concat(all, r); +} + +/* + * walk through argin parameters. + * generate and return code to allocate + * copies of escaped parameters to the heap. + */ +static NodeList* +paramstoheap(Type **argin, int out) +{ + Type *t; + Iter savet; + Node *v; + NodeList *nn; + + nn = nil; + for(t = structfirst(&savet, argin); t != T; t = structnext(&savet)) { + v = t->nname; + if(v == N && out && hasdefer) { + // Defer might stop a panic and show the + // return values as they exist at the time of panic. + // Make sure to zero them on entry to the function. + nn = list(nn, nod(OAS, nodarg(t, 1), N)); + } + if(v == N || !(v->class & PHEAP)) + continue; + + // generate allocation & copying code + if(v->alloc == nil) + v->alloc = callnew(v->type); + nn = list(nn, nod(OAS, v->heapaddr, v->alloc)); + if((v->class & ~PHEAP) != PPARAMOUT) + nn = list(nn, nod(OAS, v, v->stackparam)); + } + return nn; +} + +/* + * walk through argout parameters copying back to stack + */ +static NodeList* +returnsfromheap(Type **argin) +{ + Type *t; + Iter savet; + Node *v; + NodeList *nn; + + nn = nil; + for(t = structfirst(&savet, argin); t != T; t = structnext(&savet)) { + v = t->nname; + if(v == N || v->class != (PHEAP|PPARAMOUT)) + continue; + nn = list(nn, nod(OAS, v->stackparam, v)); + } + return nn; +} + +/* + * take care of migrating any function in/out args + * between the stack and the heap. adds code to + * curfn's before and after lists. + */ +static void +heapmoves(void) +{ + NodeList *nn; + int32 lno; + + lno = lineno; + lineno = curfn->lineno; + nn = paramstoheap(getthis(curfn->type), 0); + nn = concat(nn, paramstoheap(getinarg(curfn->type), 0)); + nn = concat(nn, paramstoheap(getoutarg(curfn->type), 1)); + curfn->enter = concat(curfn->enter, nn); + lineno = curfn->endlineno; + curfn->exit = returnsfromheap(getoutarg(curfn->type)); + lineno = lno; +} + +static Node* +vmkcall(Node *fn, Type *t, NodeList **init, va_list va) +{ + int i, n; + Node *r; + NodeList *args; + + if(fn->type == T || fn->type->etype != TFUNC) + fatal("mkcall %#N %T", fn, fn->type); + + args = nil; + n = fn->type->intuple; + for(i=0; ilist = args; + if(fn->type->outtuple > 0) + typecheck(&r, Erv | Efnstruct); + else + typecheck(&r, Etop); + walkexpr(&r, init); + r->type = t; + return r; +} + +Node* +mkcall(char *name, Type *t, NodeList **init, ...) +{ + Node *r; + va_list va; + + va_start(va, init); + r = vmkcall(syslook(name, 0), t, init, va); + va_end(va); + return r; +} + +Node* +mkcall1(Node *fn, Type *t, NodeList **init, ...) +{ + Node *r; + va_list va; + + va_start(va, init); + r = vmkcall(fn, t, init, va); + va_end(va); + return r; +} + +static Node* +conv(Node *n, Type *t) +{ + if(eqtype(n->type, t)) + return n; + n = nod(OCONV, n, N); + n->type = t; + typecheck(&n, Erv); + return n; +} + +Node* +chanfn(char *name, int n, Type *t) +{ + Node *fn; + int i; + + if(t->etype != TCHAN) + fatal("chanfn %T", t); + fn = syslook(name, 1); + for(i=0; itype); + return fn; +} + +static Node* +mapfn(char *name, Type *t) +{ + Node *fn; + + if(t->etype != TMAP) + fatal("mapfn %T", t); + fn = syslook(name, 1); + argtype(fn, t->down); + argtype(fn, t->type); + argtype(fn, t->down); + argtype(fn, t->type); + return fn; +} + +static Node* +addstr(Node *n, NodeList **init) +{ + Node *r, *cat, *typstr; + NodeList *in, *args; + int i, count; + + count = 0; + for(r=n; r->op == OADDSTR; r=r->left) + count++; // r->right + count++; // r + + // prepare call of runtime.catstring of type int, string, string, string + // with as many strings as we have. + cat = syslook("concatstring", 1); + cat->type = T; + cat->ntype = nod(OTFUNC, N, N); + in = list1(nod(ODCLFIELD, N, typenod(types[TINT]))); // count + typstr = typenod(types[TSTRING]); + for(i=0; intype->list = in; + cat->ntype->rlist = list1(nod(ODCLFIELD, N, typstr)); + + args = nil; + for(r=n; r->op == OADDSTR; r=r->left) + args = concat(list1(conv(r->right, types[TSTRING])), args); + args = concat(list1(conv(r, types[TSTRING])), args); + args = concat(list1(nodintconst(count)), args); + + r = nod(OCALL, cat, N); + r->list = args; + typecheck(&r, Erv); + walkexpr(&r, init); + r->type = n->type; + + return r; +} + +static Node* +appendslice(Node *n, NodeList **init) +{ + Node *f; + + f = syslook("appendslice", 1); + argtype(f, n->type); + argtype(f, n->type->type); + argtype(f, n->type); + return mkcall1(f, n->type, init, typename(n->type), n->list->n, n->list->next->n); +} + +// expand append(src, a [, b]* ) to +// +// init { +// s := src +// const argc = len(args) - 1 +// if cap(s) - len(s) < argc { +// s = growslice(s, argc) +// } +// n := len(s) +// s = s[:n+argc] +// s[n] = a +// s[n+1] = b +// ... +// } +// s +static Node* +append(Node *n, NodeList **init) +{ + NodeList *l, *a; + Node *nsrc, *ns, *nn, *na, *nx, *fn; + int argc; + + walkexprlistsafe(n->list, init); + + nsrc = n->list->n; + argc = count(n->list) - 1; + if (argc < 1) { + return nsrc; + } + + l = nil; + + ns = nod(OXXX, N, N); // var s + tempname(ns, nsrc->type); + l = list(l, nod(OAS, ns, nsrc)); // s = src + + na = nodintconst(argc); // const argc + nx = nod(OIF, N, N); // if cap(s) - len(s) < argc + nx->ntest = nod(OLT, nod(OSUB, nod(OCAP, ns, N), nod(OLEN, ns, N)), na); + + fn = syslook("growslice", 1); // growslice(, old []T, n int64) (ret []T) + argtype(fn, ns->type->type); // 1 old []any + argtype(fn, ns->type->type); // 2 ret []any + + nx->nbody = list1(nod(OAS, ns, mkcall1(fn, ns->type, &nx->ninit, + typename(ns->type), + ns, + conv(na, types[TINT64])))); + l = list(l, nx); + + nn = nod(OXXX, N, N); // var n + tempname(nn, types[TINT]); + l = list(l, nod(OAS, nn, nod(OLEN, ns, N))); // n = len(s) + + nx = nod(OSLICE, ns, nod(OKEY, N, nod(OADD, nn, na))); // ...s[:n+argc] + nx->etype = 1; // disable bounds check + l = list(l, nod(OAS, ns, nx)); // s = s[:n+argc] + + for (a = n->list->next; a != nil; a = a->next) { + nx = nod(OINDEX, ns, nn); // s[n] ... + nx->etype = 1; // disable bounds check + l = list(l, nod(OAS, nx, a->n)); // s[n] = arg + if (a->next != nil) + l = list(l, nod(OAS, nn, nod(OADD, nn, nodintconst(1)))); // n = n + 1 + } + + typechecklist(l, Etop); + walkstmtlist(l); + *init = concat(*init, l); + return ns; +} diff --git a/src/cmd/godefs/Makefile b/src/cmd/godefs/Makefile new file mode 100644 index 000000000..77cd26c04 --- /dev/null +++ b/src/cmd/godefs/Makefile @@ -0,0 +1,19 @@ +# Copyright 2009 The Go Authors. All rights reserved. +# Use of this source code is governed by a BSD-style +# license that can be found in the LICENSE file. + +include ../../Make.inc +O:=$(HOST_O) + +TARG=godefs +OFILES=\ + main.$O\ + stabs.$O\ + util.$O\ + +HFILES=a.h + +include ../../Make.ccmd + +test: $(TARG) + ./test.sh diff --git a/src/cmd/godefs/a.h b/src/cmd/godefs/a.h new file mode 100644 index 000000000..9b4957467 --- /dev/null +++ b/src/cmd/godefs/a.h @@ -0,0 +1,104 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +#include +#include +#include + +enum +{ + Void = 1, + Int8, + Uint8, + Int16, + Uint16, + Int32, + Uint32, + Int64, + Uint64, + Float32, + Float64, + Ptr, + Struct, + Array, + Union, + Typedef, +}; + +typedef struct Field Field; +typedef struct Type Type; + +struct Type +{ + Type *next; // next in hash table + + // stabs name and two-integer id + char *name; + int n1; + int n2; + + // int kind + int kind; + + // sub-type for ptr, array + Type *type; + + // struct fields + Field *f; + int nf; + int size; + + int saved; // recorded in typ array + int warned; // warned about needing type + int printed; // has the definition been printed yet? +}; + +struct Field +{ + char *name; + Type *type; + int offset; + int size; +}; + +// Constants +typedef struct Const Const; +struct Const +{ + char *name; + vlong value; +}; + +// Recorded constants and types, to be printed. +extern Const *con; +extern int ncon; +extern Type **typ; +extern int ntyp; +extern int kindsize[]; + +// Language output +typedef struct Lang Lang; +struct Lang +{ + char *constbegin; + char *constfmt; + char *constend; + + char *typdef; + char *typdefend; + + char *structbegin; + char *unionbegin; + char *structpadfmt; + char *structend; + + int (*typefmt)(Fmt*); +}; + +extern Lang go, c; + +void* emalloc(int); +char* estrdup(char*); +void* erealloc(void*, int); +void parsestabtype(char*); diff --git a/src/cmd/godefs/doc.go b/src/cmd/godefs/doc.go new file mode 100644 index 000000000..365c7cf6e --- /dev/null +++ b/src/cmd/godefs/doc.go @@ -0,0 +1,99 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +/* + +Godefs is a bootstrapping tool for porting the Go runtime to new systems. +It translates C type declarations into C or Go type declarations +with the same memory layout. + +Usage: godefs [-g package] [-c cc] [-f cc-arg]... [defs.c ...] + +Godefs takes as input a host-compilable C file that includes +standard system headers. From that input file, it generates +a standalone (no #includes) C or Go file containing equivalent +definitions. + +The input to godefs is a C input file that can be compiled by +the host system's standard C compiler (typically gcc). +This file is expected to define new types and enumerated constants +whose names begin with $ (a legal identifier character in gcc). +Godefs compile the given input file with the host compiler and +then parses the debug info embedded in the assembly output. +This is far easier than reading system headers on most machines. + +The output from godefs is either C output intended for the +Plan 9 C compiler tool chain (6c, 8c, or 5c) or Go output. + +The options are: + + -g package + generate Go output using the given package name. + In the Go output, struct fields have leading xx_ prefixes + removed and the first character capitalized (exported). + + -c cc + set the name of the host system's C compiler (default "gcc") + + -f cc-arg + add cc-arg to the command line when invoking the system C compiler + (for example, -f -m64 to invoke gcc -m64). + Repeating this option adds multiple flags to the command line. + +For example, if this is x.c: + + #include + + typedef struct timespec $Timespec; + enum { + $S_IFMT = S_IFMT, + $S_IFIFO = S_IFIFO, + $S_IFCHR = S_IFCHR, + }; + +then "godefs x.c" generates: + + // godefs x.c + // MACHINE GENERATED - DO NOT EDIT. + + // Constants + enum { + S_IFMT = 0xf000, + S_IFIFO = 0x1000, + S_IFCHR = 0x2000, + }; + + // Types + #pragma pack on + + typedef struct Timespec Timespec; + struct Timespec { + int64 tv_sec; + int64 tv_nsec; + }; + #pragma pack off + +and "godefs -g MyPackage x.c" generates: + + // godefs -g MyPackage x.c + // MACHINE GENERATED - DO NOT EDIT. + + package MyPackage + + // Constants + const ( + S_IFMT = 0xf000; + S_IFIFO = 0x1000; + S_IFCHR = 0x2000; + ) + + // Types + + type Timespec struct { + Sec int64; + Nsec int64; + } + +*/ +package documentation diff --git a/src/cmd/godefs/main.c b/src/cmd/godefs/main.c new file mode 100644 index 000000000..6a8630179 --- /dev/null +++ b/src/cmd/godefs/main.c @@ -0,0 +1,606 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Godefs takes as input a host-compilable C file that includes +// standard system headers. From that input file, it generates +// a standalone (no #includes) C or Go file containing equivalent +// definitions. +// +// The input C file is expected to define new types and enumerated +// constants whose names begin with $ (a legal identifier character +// in gcc). The output is the standalone definitions of those names, +// with the $ removed. +// +// For example, if this is x.c: +// +// #include +// +// typedef struct timespec $Timespec; +// typedef struct stat $Stat; +// enum { +// $S_IFMT = S_IFMT, +// $S_IFIFO = S_IFIFO, +// $S_IFCHR = S_IFCHR, +// }; +// +// then "godefs x.c" generates: +// +// // godefs x.c +// +// // MACHINE GENERATED - DO NOT EDIT. +// +// // Constants +// enum { +// S_IFMT = 0xf000, +// S_IFIFO = 0x1000, +// S_IFCHR = 0x2000, +// }; +// +// // Types +// #pragma pack on +// +// typedef struct Timespec Timespec; +// struct Timespec { +// int32 tv_sec; +// int32 tv_nsec; +// }; +// +// typedef struct Stat Stat; +// struct Stat { +// int32 st_dev; +// uint32 st_ino; +// uint16 st_mode; +// uint16 st_nlink; +// uint32 st_uid; +// uint32 st_gid; +// int32 st_rdev; +// Timespec st_atimespec; +// Timespec st_mtimespec; +// Timespec st_ctimespec; +// int64 st_size; +// int64 st_blocks; +// int32 st_blksize; +// uint32 st_flags; +// uint32 st_gen; +// int32 st_lspare; +// int64 st_qspare[2]; +// }; +// #pragma pack off +// +// The -g flag to godefs causes it to generate Go output, not C. +// In the Go output, struct fields have leading xx_ prefixes removed +// and the first character capitalized (exported). +// +// Godefs works by invoking gcc to compile the given input file +// and then parses the debug info embedded in the assembly output. +// This is far easier than reading system headers on most machines. +// +// The -c flag sets the compiler (default "gcc"). +// +// The -f flag adds a flag to pass to the compiler (e.g., -f -m64). + +#include "a.h" + +#ifdef _WIN32 +int +spawn(char *prog, char **argv) +{ + return _spawnvp(P_NOWAIT, prog, (const char**)argv); +} +#undef waitfor +void +waitfor(int pid) +{ + _cwait(0, pid, 0); +} +#else +int +spawn(char *prog, char **argv) +{ + int pid = fork(); + if(pid < 0) + sysfatal("fork: %r"); + if(pid == 0) { + exec(argv[0], argv); + fprint(2, "exec gcc: %r\n"); + exit(1); + } + return pid; +} +#endif + +void +usage(void) +{ + fprint(2, "usage: godefs [-g package] [-c cc] [-f cc-arg] [defs.c ...]\n"); + exit(1); +} + +int gotypefmt(Fmt*); +int ctypefmt(Fmt*); +int prefixlen(Type*); +int cutprefix(char*); + +Lang go = +{ + "const (\n", + "\t%s = %#llx;\n", + ")\n", + + "type", + "\n", + + "type %s struct {\n", + "type %s struct {\n", + "\tPad_godefs_%d [%d]byte;\n", + "}\n", + + gotypefmt, +}; + +Lang c = +{ + "enum {\n", + "\t%s = %#llx,\n", + "};\n", + + "typedef", + ";\n", + + "typedef struct %s %s;\nstruct %s {\n", + "typedef union %s %s;\nunion %s {\n", + "\tbyte pad_godefs_%d[%d];\n", + "};\n", + + ctypefmt, +}; + +char *pkg; + +int oargc; +char **oargv; +Lang *lang = &c; + +Const *con; +int ncon; + +Type **typ; +int ntyp; + +void +waitforgcc(void) +{ + waitpid(); +} + +void +main(int argc, char **argv) +{ + int p[2], pid, i, j, n, off, npad, prefix; + char **av, *q, *r, *tofree, *name; + char nambuf[100]; + Biobuf *bin, *bout; + Type *t, *tt; + Field *f; + int orig_output_fd; + + quotefmtinstall(); + + oargc = argc; + oargv = argv; + av = emalloc((30+argc)*sizeof av[0]); + atexit(waitforgcc); + + n = 0; + av[n++] = "gcc"; + av[n++] = "-fdollars-in-identifiers"; + av[n++] = "-S"; // write assembly + av[n++] = "-gstabs+"; // include stabs info + av[n++] = "-o"; // to ... + av[n++] = "-"; // ... stdout + av[n++] = "-xc"; // read C + + ARGBEGIN{ + case 'g': + lang = &go; + pkg = EARGF(usage()); + break; + case 'c': + av[0] = EARGF(usage()); + break; + case 'f': + av[n++] = EARGF(usage()); + break; + default: + usage(); + }ARGEND + + if(argc == 0) + av[n++] = "-"; + else + av[n++] = argv[0]; + av[n] = nil; + + orig_output_fd = dup(1, -1); + for(i=0; i==0 || i < argc; i++) { + // Some versions of gcc do not accept -S with multiple files. + // Run gcc once for each file. + // Write assembly and stabs debugging to p[1]. + if(pipe(p) < 0) + sysfatal("pipe: %r"); + dup(p[1], 1); + close(p[1]); + if (argc) + av[n-1] = argv[i]; + pid = spawn(av[0], av); + dup(orig_output_fd, 1); + + // Read assembly, pulling out .stabs lines. + bin = Bfdopen(p[0], OREAD); + while((q = Brdstr(bin, '\n', 1)) != nil) { + // .stabs "float:t(0,12)=r(0,1);4;0;",128,0,0,0 + tofree = q; + while(*q == ' ' || *q == '\t') + q++; + if(strncmp(q, ".stabs", 6) != 0) + goto Continue; + q += 6; + while(*q == ' ' || *q == '\t') + q++; + if(*q++ != '\"') { +Bad: + sysfatal("cannot parse .stabs line:\n%s", tofree); + } + + r = strchr(q, '\"'); + if(r == nil) + goto Bad; + *r++ = '\0'; + if(*r++ != ',') + goto Bad; + if(*r < '0' || *r > '9') + goto Bad; + if(atoi(r) != 128) // stabs kind = local symbol + goto Continue; + + parsestabtype(q); + +Continue: + free(tofree); + } + Bterm(bin); + waitfor(pid); + } + close(orig_output_fd); + + // Write defs to standard output. + bout = Bfdopen(1, OWRITE); + fmtinstall('T', lang->typefmt); + + // Echo original command line in header. + Bprint(bout, "//"); + for(i=0; i 0) { + Bprint(bout, lang->constbegin); + for(i=0; iconstfmt, con[i].name, con[i].value); + else + Bprint(bout, lang->constfmt, con[i].name, con[i].value & 0xFFFFFFFF); + } + Bprint(bout, lang->constend); + } + Bprint(bout, "\n"); + + // Types + + // push our names down + for(i=0; iname; + while(t && t->kind == Typedef) + t = t->type; + if(t) + t->name = name; + } + + Bprint(bout, "// Types\n"); + + // Have to turn off structure padding in Plan 9 compiler, + // mainly because it is more aggressive than gcc tends to be. + if(lang == &c) + Bprint(bout, "#pragma pack on\n"); + + for(i=0; iname; + while(t && t->kind == Typedef) { + if(name == nil && t->name != nil) { + name = t->name; + if(t->printed) + break; + } + t = t->type; + } + if(name == nil && t->name != nil) { + name = t->name; + if(t->printed) + continue; + t->printed = 1; + } + if(name == nil) { + fprint(2, "unknown name for %T", typ[i]); + continue; + } + if(name[0] == '$') + name++; + npad = 0; + off = 0; + switch(t->kind) { + case 0: + fprint(2, "unknown type definition for %s\n", name); + break; + default: // numeric, array, or pointer + case Array: + case Ptr: + Bprint(bout, "%s %lT%s", lang->typdef, name, t, lang->typdefend); + break; + case Union: + // In Go, print union as struct with only first element, + // padded the rest of the way. + Bprint(bout, lang->unionbegin, name, name, name); + goto StructBody; + case Struct: + Bprint(bout, lang->structbegin, name, name, name); + StructBody: + prefix = 0; + if(lang == &go) + prefix = prefixlen(t); + for(j=0; jnf; j++) { + f = &t->f[j]; + if(f->type->kind == 0 && f->size <= 64 && (f->size&(f->size-1)) == 0) { + // unknown type but <= 64 bits and bit size is a power of two. + // could be enum - make Uint64 and then let it reduce + tt = emalloc(sizeof *tt); + *tt = *f->type; + f->type = tt; + tt->kind = Uint64; + while(tt->kind > Uint8 && kindsize[tt->kind] > f->size) + tt->kind -= 2; + } + // padding + if(t->kind == Struct || lang == &go) { + if(f->offset%8 != 0 || f->size%8 != 0) { + fprint(2, "ignoring bitfield %s.%s\n", t->name, f->name); + continue; + } + if(f->offset < off) + sysfatal("%s: struct fields went backward", t->name); + if(off < f->offset) { + Bprint(bout, lang->structpadfmt, npad++, (f->offset - off) / 8); + off = f->offset; + } + off += f->size; + } + name = f->name; + if(cutprefix(name)) + name += prefix; + if(strcmp(name, "") == 0) { + snprint(nambuf, sizeof nambuf, "Pad_godefs_%d", npad++); + name = nambuf; + } + Bprint(bout, "\t%#lT;\n", name, f->type); + if(t->kind == Union && lang == &go) + break; + } + // final padding + if(t->kind == Struct || lang == &go) { + if(off/8 < t->size) + Bprint(bout, lang->structpadfmt, npad++, t->size - off/8); + } + Bprint(bout, lang->structend); + } + } + if(lang == &c) + Bprint(bout, "#pragma pack off\n"); + Bterm(bout); + exit(0); +} + +char *kindnames[] = { + "void", // actually unknown, but byte is good for pointers + "void", + "int8", + "uint8", + "int16", + "uint16", + "int32", + "uint32", + "int64", + "uint64", + "float32", + "float64", + "ptr", + "struct", + "array", + "union", + "typedef", +}; + +int +ctypefmt(Fmt *f) +{ + char *name, *s; + Type *t; + + name = nil; + if(f->flags & FmtLong) { + name = va_arg(f->args, char*); + if(name == nil || name[0] == '\0') + name = "_anon_"; + } + t = va_arg(f->args, Type*); + while(t && t->kind == Typedef) + t = t->type; + switch(t->kind) { + case Struct: + case Union: + // must be named + s = t->name; + if(s == nil) { + fprint(2, "need name for anonymous struct\n"); + goto bad; + } + else if(s[0] != '$') + fprint(2, "need name for struct %s\n", s); + else + s++; + fmtprint(f, "%s", s); + if(name) + fmtprint(f, " %s", name); + break; + + case Array: + if(name) + fmtprint(f, "%T %s[%d]", t->type, name, t->size); + else + fmtprint(f, "%T[%d]", t->type, t->size); + break; + + case Ptr: + if(name) + fmtprint(f, "%T *%s", t->type, name); + else + fmtprint(f, "%T*", t->type); + break; + + default: + fmtprint(f, "%s", kindnames[t->kind]); + if(name) + fmtprint(f, " %s", name); + break; + + bad: + if(name) + fmtprint(f, "byte %s[%d]", name, t->size); + else + fmtprint(f, "byte[%d]", t->size); + break; + } + + return 0; +} + +int +gotypefmt(Fmt *f) +{ + char *name, *s; + Type *t; + + if(f->flags & FmtLong) { + name = va_arg(f->args, char*); + if('a' <= name[0] && name[0] <= 'z') + name[0] += 'A' - 'a'; + if(name[0] == '_' && (f->flags & FmtSharp)) + fmtprint(f, "X"); + fmtprint(f, "%s ", name); + } + t = va_arg(f->args, Type*); + while(t && t->kind == Typedef) + t = t->type; + + switch(t->kind) { + case Struct: + case Union: + // must be named + s = t->name; + if(s == nil) { + fprint(2, "need name for anonymous struct\n"); + fmtprint(f, "STRUCT"); + } + else if(s[0] != '$') { + fprint(2, "warning: missing name for struct %s\n", s); + fmtprint(f, "[%d]byte /* %s */", t->size, s); + } else + fmtprint(f, "%s", s+1); + break; + + case Array: + fmtprint(f, "[%d]%T", t->size, t->type); + break; + + case Ptr: + fmtprint(f, "*%T", t->type); + break; + + default: + s = kindnames[t->kind]; + if(strcmp(s, "void") == 0) + s = "byte"; + fmtprint(f, "%s", s); + } + + return 0; +} + +// Is this the kind of name we should cut a prefix from? +// The rule is that the name cannot begin with underscore +// and must have an underscore eventually. +int +cutprefix(char *name) +{ + char *p; + + // special case: orig_ in register struct + if(strncmp(name, "orig_", 5) == 0) + return 0; + + for(p=name; *p; p++) { + if(*p == '_') + return p-name > 0; + } + return 0; +} + +// Figure out common struct prefix len +int +prefixlen(Type *t) +{ + int i; + int len; + char *p, *name; + Field *f; + + len = 0; + name = nil; + for(i=0; inf; i++) { + f = &t->f[i]; + if(!cutprefix(f->name)) + continue; + p = strchr(f->name, '_'); + if(p == nil) + return 0; + if(name == nil) { + name = f->name; + len = p+1 - name; + } + else if(strncmp(f->name, name, len) != 0) + return 0; + } + return len; +} diff --git a/src/cmd/godefs/stabs.c b/src/cmd/godefs/stabs.c new file mode 100644 index 000000000..2c3d431b8 --- /dev/null +++ b/src/cmd/godefs/stabs.c @@ -0,0 +1,456 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Parse stabs debug info. + +#include "a.h" + +int stabsdebug = 1; + +// Hash table for type lookup by number. +Type *hash[1024]; + +// Look up type by number pair. +// TODO(rsc): Iant points out that n1 and n2 are always small and dense, +// so an array of arrays would be a better representation. +Type* +typebynum(uint n1, uint n2) +{ + uint h; + Type *t; + + h = (n1*53+n2) % nelem(hash); + for(t=hash[h]; t; t=t->next) + if(t->n1 == n1 && t->n2 == n2) + return t; + t = emalloc(sizeof *t); + t->next = hash[h]; + hash[h] = t; + t->n1 = n1; + t->n2 = n2; + return t; +} + +// Parse name and colon from *pp, leaving copy in *sp. +static int +parsename(char **pp, char **sp) +{ + char *p; + char *s; + + p = *pp; + while(*p != '\0' && *p != ':') + p++; + if(*p == '\0') { + fprint(2, "parsename expected colon\n"); + return -1; + } + s = emalloc(p - *pp + 1); + memmove(s, *pp, p - *pp); + *sp = s; + *pp = p+1; + return 0; +} + +// Parse single number from *pp. +static int +parsenum1(char **pp, vlong *np) +{ + char *p; + + p = *pp; + if(*p != '-' && (*p < '0' || *p > '9')) { + fprint(2, "parsenum expected minus or digit\n"); + return -1; + } + *np = strtoll(p, pp, 10); + return 0; +} + +// Parse type number - either single number or (n1, n2). +static int +parsetypenum(char **pp, vlong *n1p, vlong *n2p) +{ + char *p; + + p = *pp; + if(*p == '(') { + p++; + if(parsenum1(&p, n1p) < 0) + return -1; + if(*p++ != ',') { + if(stabsdebug) + fprint(2, "parsetypenum expected comma\n"); + return -1; + } + if(parsenum1(&p, n2p) < 0) + return -1; + if(*p++ != ')') { + if(stabsdebug) + fprint(2, "parsetypenum expected right paren\n"); + return -1; + } + *pp = p; + return 0; + } + + if(parsenum1(&p, n1p) < 0) + return -1; + *n2p = 0; + *pp = p; + return 0; +} + +// Written to parse max/min of vlong correctly. +static vlong +parseoctal(char **pp) +{ + char *p; + vlong n; + + p = *pp; + if(*p++ != '0') + return 0; + n = 0; + while(*p >= '0' && *p <= '9') + n = n << 3 | *p++ - '0'; + *pp = p; + return n; +} + +// Integer types are represented in stabs as a "range" +// type with a lo and a hi value. The lo and hi used to +// be lo and hi for the type, but there are now odd +// extensions for floating point and 64-bit numbers. +// +// Have to keep signs separate from values because +// Int64's lo is -0. +typedef struct Intrange Intrange; +struct Intrange +{ + vlong lo; + vlong hi; + int kind; +}; + +Intrange intranges[] = { + 0, 127, Int8, // char + -128, 127, Int8, // signed char + 0, 255, Uint8, + -32768, 32767, Int16, + 0, 65535, Uint16, + -2147483648LL, 2147483647LL, Int32, + 0, 4294967295LL, Uint32, + 1LL << 63, ~(1LL << 63), Int64, + 0, -1, Uint64, + 4, 0, Float32, + 8, 0, Float64, + 16, 0, Void, +}; + +int kindsize[] = { + 0, + 0, + 8, + 8, + 16, + 16, + 32, + 32, + 64, + 64, +}; + +// Parse a single type definition from *pp. +static Type* +parsedef(char **pp, char *name) +{ + char *p; + Type *t, *tt; + int i; + vlong n1, n2, lo, hi; + Field *f; + Intrange *r; + + p = *pp; + + // reference to another type? + if(isdigit(*p) || *p == '(') { + if(parsetypenum(&p, &n1, &n2) < 0) + return nil; + t = typebynum(n1, n2); + if(name && t->name == nil) { + t->name = name; + // save definitions of names beginning with $ + if(name[0] == '$' && !t->saved) { + typ = erealloc(typ, (ntyp+1)*sizeof typ[0]); + typ[ntyp] = t; + ntyp++; + } + } + + // is there an =def suffix? + if(*p == '=') { + p++; + tt = parsedef(&p, name); + if(tt == nil) + return nil; + + if(tt == t) { + tt->kind = Void; + } else { + t->type = tt; + t->kind = Typedef; + } + + // assign given name, but do not record in typ. + // assume the name came from a typedef + // which will be recorded. + if(name) + tt->name = name; + } + + *pp = p; + return t; + } + + // otherwise a type literal. first letter identifies kind + t = emalloc(sizeof *t); + switch(*p) { + default: + fprint(2, "unknown type char %c in %s\n", *p, p); + *pp = ""; + return t; + + case '@': // type attribute + while (*++p != ';'); + *pp = ++p; + return parsedef(pp, nil); + + case '*': // pointer + p++; + t->kind = Ptr; + tt = parsedef(&p, nil); + if(tt == nil) + return nil; + t->type = tt; + break; + + case 'a': // array + p++; + t->kind = Array; + // index type + tt = parsedef(&p, nil); + if(tt == nil) + return nil; + t->size = tt->size; + // element type + tt = parsedef(&p, nil); + if(tt == nil) + return nil; + t->type = tt; + break; + + case 'e': // enum type - record $names in con array. + p++; + for(;;) { + if(*p == '\0') + return nil; + if(*p == ';') { + p++; + break; + } + if(parsename(&p, &name) < 0) + return nil; + if(parsenum1(&p, &n1) < 0) + return nil; + if(name[0] == '$') { + con = erealloc(con, (ncon+1)*sizeof con[0]); + name++; + con[ncon].name = name; + con[ncon].value = n1; + ncon++; + } + if(*p != ',') + return nil; + p++; + } + break; + + case 'f': // function + p++; + if(parsedef(&p, nil) == nil) + return nil; + break; + + case 'B': // volatile + case 'k': // const + ++*pp; + return parsedef(pp, nil); + + case 'r': // sub-range (used for integers) + p++; + if(parsedef(&p, nil) == nil) + return nil; + // usually, the return from parsedef == t, but not always. + + if(*p != ';' || *++p == ';') { + if(stabsdebug) + fprint(2, "range expected number: %s\n", p); + return nil; + } + if(*p == '0') + lo = parseoctal(&p); + else + lo = strtoll(p, &p, 10); + if(*p != ';' || *++p == ';') { + if(stabsdebug) + fprint(2, "range expected number: %s\n", p); + return nil; + } + if(*p == '0') + hi = parseoctal(&p); + else + hi = strtoll(p, &p, 10); + if(*p != ';') { + if(stabsdebug) + fprint(2, "range expected trailing semi: %s\n", p); + return nil; + } + p++; + t->size = hi+1; // might be array size + for(i=0; ilo == lo && r->hi == hi) { + t->kind = r->kind; + break; + } + } + break; + + case 's': // struct + case 'u': // union + t->kind = Struct; + if(*p == 'u') + t->kind = Union; + + // assign given name, but do not record in typ. + // assume the name came from a typedef + // which will be recorded. + if(name) + t->name = name; + p++; + if(parsenum1(&p, &n1) < 0) + return nil; + t->size = n1; + for(;;) { + if(*p == '\0') + return nil; + if(*p == ';') { + p++; + break; + } + t->f = erealloc(t->f, (t->nf+1)*sizeof t->f[0]); + f = &t->f[t->nf]; + if(parsename(&p, &f->name) < 0) + return nil; + f->type = parsedef(&p, nil); + if(f->type == nil) + return nil; + if(*p != ',') { + fprint(2, "expected comma after def of %s:\n%s\n", f->name, p); + return nil; + } + p++; + if(parsenum1(&p, &n1) < 0) + return nil; + f->offset = n1; + if(*p != ',') { + fprint(2, "expected comma after offset of %s:\n%s\n", f->name, p); + return nil; + } + p++; + if(parsenum1(&p, &n1) < 0) + return nil; + f->size = n1; + if(*p != ';') { + fprint(2, "expected semi after size of %s:\n%s\n", f->name, p); + return nil; + } + + while(f->type->kind == Typedef) + f->type = f->type->type; + + // rewrite + // uint32 x : 8; + // into + // uint8 x; + // hooray for bitfields. + while(Int16 <= f->type->kind && f->type->kind <= Uint64 && kindsize[f->type->kind] > f->size) { + tt = emalloc(sizeof *tt); + *tt = *f->type; + f->type = tt; + f->type->kind -= 2; + } + p++; + t->nf++; + } + break; + + case 'x': + // reference to struct, union not yet defined. + p++; + switch(*p) { + case 's': + t->kind = Struct; + break; + case 'u': + t->kind = Union; + break; + default: + fprint(2, "unknown x type char x%c", *p); + *pp = ""; + return t; + } + if(parsename(&p, &t->name) < 0) + return nil; + break; + } + *pp = p; + return t; +} + + +// Parse a stab type in p, saving info in the type hash table +// and also in the list of recorded types if appropriate. +void +parsestabtype(char *p) +{ + char *p0, *name; + + p0 = p; + + // p is the quoted string output from gcc -gstabs on a .stabs line. + // name:t(1,2) + // name:t(1,2)=def + if(parsename(&p, &name) < 0) { + Bad: + // Use fprint instead of sysfatal to avoid + // sysfatal's internal buffer size limit. + fprint(2, "cannot parse stabs type:\n%s\n(at %s)\n", p0, p); + sysfatal("stabs parse"); + } + if(*p != 't' && *p != 'T') + goto Bad; + p++; + + // parse the definition. + if(name[0] == '\0') + name = nil; + if(parsedef(&p, name) == nil) + goto Bad; + if(*p != '\0') + goto Bad; +} + diff --git a/src/cmd/godefs/test.sh b/src/cmd/godefs/test.sh new file mode 100755 index 000000000..c035af8f4 --- /dev/null +++ b/src/cmd/godefs/test.sh @@ -0,0 +1,45 @@ +#!/usr/bin/env bash +# Copyright 2011 The Go Authors. All rights reserved. +# Use of this source code is governed by a BSD-style +# license that can be found in the LICENSE file. + +eval $(gomake --no-print-directory -f ../../Make.inc go-env) + +TMP="testdata_tmp.go" +TEST="testdata.c" +GOLDEN="testdata_${GOOS}_${GOARCH}.golden" + +case ${GOARCH} in +"amd64") CCARG="-f-m64";; +"386") CCARG="-f-m32";; +*) CCARG="";; +esac + +cleanup() { + rm ${TMP} +} + +error() { + cleanup + echo $1 + exit 1 +} + +if [ ! -e ${GOLDEN} ]; then + echo "skipping - no golden defined for this platform" + exit +fi + +./godefs -g test ${CCARG} ${TEST} > ${TMP} +if [ $? != 0 ]; then + error "Error: Could not run godefs for ${TEST}" +fi + +diff ${TMP} ${GOLDEN} +if [ $? != 0 ]; then + error "FAIL: godefs for ${TEST} did not match ${GOLDEN}" +fi + +cleanup + +echo "PASS" diff --git a/src/cmd/godefs/testdata.c b/src/cmd/godefs/testdata.c new file mode 100644 index 000000000..3f459c41b --- /dev/null +++ b/src/cmd/godefs/testdata.c @@ -0,0 +1,41 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +#include + +// Issue 432 - enum fields in struct can cause misaligned struct fields +typedef enum { + a +} T1; + +struct T2 { + uint8_t a; + T1 b; + T1 c; + uint16_t d; +}; + +typedef struct T2 T2; +typedef T2 $T2; + +// Issue 1162 - structs with fields named Pad[0-9]+ conflict with field +// names used by godefs for padding +struct T3 { + uint8_t a; + int Pad0; +}; + +typedef struct T3 $T3; + +// Issue 1466 - forward references to types in stabs debug info were +// always treated as enums +struct T4 {}; + +struct T5 { + struct T4 *a; +}; + +typedef struct T5 T5; +typedef struct T4 $T4; +typedef T5 $T5; \ No newline at end of file diff --git a/src/cmd/godefs/testdata_darwin_386.golden b/src/cmd/godefs/testdata_darwin_386.golden new file mode 100644 index 000000000..d929238b0 --- /dev/null +++ b/src/cmd/godefs/testdata_darwin_386.golden @@ -0,0 +1,31 @@ +// ./godefs -g test -f-m32 testdata.c + +// MACHINE GENERATED - DO NOT EDIT. + +package test + +// Constants + +// Types + +type T2 struct { + A uint8; + Pad_godefs_0 [3]byte; + B uint32; + C uint32; + D uint16; + Pad_godefs_1 [2]byte; +} + +type T3 struct { + A uint8; + Pad_godefs_0 [3]byte; + Pad0 int32; +} + +type T4 struct { +} + +type T5 struct { + A *T4; +} diff --git a/src/cmd/godefs/testdata_darwin_amd64.golden b/src/cmd/godefs/testdata_darwin_amd64.golden new file mode 100644 index 000000000..a694f4a73 --- /dev/null +++ b/src/cmd/godefs/testdata_darwin_amd64.golden @@ -0,0 +1,31 @@ +// ./godefs -g test -f-m64 testdata.c + +// MACHINE GENERATED - DO NOT EDIT. + +package test + +// Constants + +// Types + +type T2 struct { + A uint8; + Pad_godefs_0 [3]byte; + B uint32; + C uint32; + D uint16; + Pad_godefs_1 [2]byte; +} + +type T3 struct { + A uint8; + Pad_godefs_0 [3]byte; + Pad0 int32; +} + +type T4 struct { +} + +type T5 struct { + A *T4; +} diff --git a/src/cmd/godefs/util.c b/src/cmd/godefs/util.c new file mode 100644 index 000000000..18be00453 --- /dev/null +++ b/src/cmd/godefs/util.c @@ -0,0 +1,36 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +#include "a.h" + +void* +emalloc(int n) +{ + void *p; + + p = malloc(n); + if(p == nil) + sysfatal("out of memory"); + memset(p, 0, n); + return p; +} + +char* +estrdup(char *s) +{ + s = strdup(s); + if(s == nil) + sysfatal("out of memory"); + return s; +} + +void* +erealloc(void *v, int n) +{ + v = realloc(v, n); + if(v == nil) + sysfatal("out of memory"); + return v; +} + diff --git a/src/cmd/godoc/Makefile b/src/cmd/godoc/Makefile new file mode 100644 index 000000000..f40d71703 --- /dev/null +++ b/src/cmd/godoc/Makefile @@ -0,0 +1,24 @@ +# Copyright 2009 The Go Authors. All rights reserved. +# Use of this source code is governed by a BSD-style +# license that can be found in the LICENSE file. + +include ../../Make.inc + +TARG=godoc +GOFILES=\ + codewalk.go\ + dirtrees.go\ + filesystem.go\ + format.go\ + godoc.go\ + httpzip.go\ + index.go\ + main.go\ + mapping.go\ + parser.go\ + snippet.go\ + spec.go\ + utils.go\ + zip.go\ + +include ../../Make.cmd diff --git a/src/cmd/godoc/appconfig.go b/src/cmd/godoc/appconfig.go new file mode 100644 index 000000000..9cbe7a443 --- /dev/null +++ b/src/cmd/godoc/appconfig.go @@ -0,0 +1,19 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// This file contains configuration information used by +// godoc when running on app engine. Adjust as needed +// (typically when the .zip file changes). + +package main + +const ( + // zipFilename is the name of the .zip file + // containing the file system served by godoc. + zipFilename = "go.zip" + + // zipGoroot is the path of the goroot directory + // in the .zip file. + zipGoroot = "/home/username/go" +) diff --git a/src/cmd/godoc/appinit.go b/src/cmd/godoc/appinit.go new file mode 100644 index 000000000..9b8987223 --- /dev/null +++ b/src/cmd/godoc/appinit.go @@ -0,0 +1,86 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// To run godoc under app engine, substitute main.go with +// this file (appinit.go), provide a .zip file containing +// the file system to serve, and adjust the configuration +// parameters in appconfig.go accordingly. +// +// The current app engine SDK may be based on an older Go +// release version. To correct for version skew, copy newer +// packages into the alt directory (e.g. alt/strings) and +// adjust the imports in the godoc source files (e.g. from +// `import "strings"` to `import "alt/strings"`). Both old +// and new packages may be used simultaneously as long as +// there is no package global state that needs to be shared. +// +// The directory structure should look as follows: +// +// godoc // directory containing the app engine app +// alt // alternative packages directory to +// // correct for version skew +// strings // never version of the strings package +// ... // +// app.yaml // app engine control file +// go.zip // zip file containing the file system to serve +// godoc // contains godoc sources +// appinit.go // this file instead of godoc/main.go +// appconfig.go // godoc for app engine configuration +// ... // +// +// To run app the engine emulator locally: +// +// dev_appserver.py -a 0 godoc +// +// godoc is the top-level "goroot" directory. +// The godoc home page is served at: :8080 and localhost:8080. + +package main + +import ( + "alt/archive/zip" + "http" + "log" + "os" +) + +func serveError(w http.ResponseWriter, r *http.Request, relpath string, err os.Error) { + contents := applyTemplate(errorHTML, "errorHTML", err) // err may contain an absolute path! + w.WriteHeader(http.StatusNotFound) + servePage(w, "File "+relpath, "", "", contents) +} + +func init() { + log.Println("initializing godoc ...") + *goroot = path.Join("/", zipGoroot) // fsHttp paths are relative to '/' + + // read .zip file and set up file systems + const zipfile = zipFilename + rc, err := zip.OpenReader(zipfile) + if err != nil { + log.Fatalf("%s: %s\n", zipfile, err) + } + fs = NewZipFS(rc) + fsHttp = NewHttpZipFS(rc, *goroot) + + // initialize http handlers + initHandlers() + readTemplates() + registerPublicHandlers(http.DefaultServeMux) + + // initialize default directory tree with corresponding timestamp. + initFSTree() + + // initialize directory trees for user-defined file systems (-path flag). + initDirTrees() + + // create search index + // TODO(gri) Disabled for now as it takes too long. Find a solution for this. + /* + *indexEnabled = true + go indexer() + */ + + log.Println("godoc initialization complete") +} diff --git a/src/cmd/godoc/codewalk.go b/src/cmd/godoc/codewalk.go new file mode 100644 index 000000000..e2643e466 --- /dev/null +++ b/src/cmd/godoc/codewalk.go @@ -0,0 +1,487 @@ +// Copyright 2010 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// The /doc/codewalk/ tree is synthesized from codewalk descriptions, +// files named $GOROOT/doc/codewalk/*.xml. +// For an example and a description of the format, see +// http://golang.org/doc/codewalk/codewalk or run godoc -http=:6060 +// and see http://localhost:6060/doc/codewalk/codewalk . +// That page is itself a codewalk; the source code for it is +// $GOROOT/doc/codewalk/codewalk.xml. + +package main + +import ( + "container/vector" + "fmt" + "http" + "io" + "log" + "os" + "regexp" + "sort" + "strconv" + "strings" + "template" + "utf8" + "xml" +) + +// Handler for /doc/codewalk/ and below. +func codewalk(w http.ResponseWriter, r *http.Request) { + relpath := r.URL.Path[len("/doc/codewalk/"):] + abspath := absolutePath(r.URL.Path[1:], *goroot) + + r.ParseForm() + if f := r.FormValue("fileprint"); f != "" { + codewalkFileprint(w, r, f) + return + } + + // If directory exists, serve list of code walks. + dir, err := fs.Lstat(abspath) + if err == nil && dir.IsDirectory() { + codewalkDir(w, r, relpath, abspath) + return + } + + // If file exists, serve using standard file server. + if err == nil { + serveFile(w, r) + return + } + + // Otherwise append .xml and hope to find + // a codewalk description. + cw, err := loadCodewalk(abspath + ".xml") + if err != nil { + log.Print(err) + serveError(w, r, relpath, err) + return + } + + // Canonicalize the path and redirect if changed + if redirect(w, r) { + return + } + + b := applyTemplate(codewalkHTML, "codewalk", cw) + servePage(w, "Codewalk: "+cw.Title, "", "", b) +} + +// A Codewalk represents a single codewalk read from an XML file. +type Codewalk struct { + Title string `xml:"attr"` + File []string + Step []*Codestep +} + +// A Codestep is a single step in a codewalk. +type Codestep struct { + // Filled in from XML + Src string `xml:"attr"` + Title string `xml:"attr"` + XML string `xml:"innerxml"` + + // Derived from Src; not in XML. + Err os.Error + File string + Lo int + LoByte int + Hi int + HiByte int + Data []byte +} + +// String method for printing in template. +// Formats file address nicely. +func (st *Codestep) String() string { + s := st.File + if st.Lo != 0 || st.Hi != 0 { + s += fmt.Sprintf(":%d", st.Lo) + if st.Lo != st.Hi { + s += fmt.Sprintf(",%d", st.Hi) + } + } + return s +} + +// loadCodewalk reads a codewalk from the named XML file. +func loadCodewalk(filename string) (*Codewalk, os.Error) { + f, err := fs.Open(filename) + if err != nil { + return nil, err + } + defer f.Close() + cw := new(Codewalk) + p := xml.NewParser(f) + p.Entity = xml.HTMLEntity + err = p.Unmarshal(cw, nil) + if err != nil { + return nil, &os.PathError{"parsing", filename, err} + } + + // Compute file list, evaluate line numbers for addresses. + m := make(map[string]bool) + for _, st := range cw.Step { + i := strings.Index(st.Src, ":") + if i < 0 { + i = len(st.Src) + } + filename := st.Src[0:i] + data, err := fs.ReadFile(absolutePath(filename, *goroot)) + if err != nil { + st.Err = err + continue + } + if i < len(st.Src) { + lo, hi, err := addrToByteRange(st.Src[i+1:], 0, data) + if err != nil { + st.Err = err + continue + } + // Expand match to line boundaries. + for lo > 0 && data[lo-1] != '\n' { + lo-- + } + for hi < len(data) && (hi == 0 || data[hi-1] != '\n') { + hi++ + } + st.Lo = byteToLine(data, lo) + st.Hi = byteToLine(data, hi-1) + } + st.Data = data + st.File = filename + m[filename] = true + } + + // Make list of files + cw.File = make([]string, len(m)) + i := 0 + for f := range m { + cw.File[i] = f + i++ + } + sort.Strings(cw.File) + + return cw, nil +} + +// codewalkDir serves the codewalk directory listing. +// It scans the directory for subdirectories or files named *.xml +// and prepares a table. +func codewalkDir(w http.ResponseWriter, r *http.Request, relpath, abspath string) { + type elem struct { + Name string + Title string + } + + dir, err := fs.ReadDir(abspath) + if err != nil { + log.Print(err) + serveError(w, r, relpath, err) + return + } + var v vector.Vector + for _, fi := range dir { + name := fi.Name() + if fi.IsDirectory() { + v.Push(&elem{name + "/", ""}) + } else if strings.HasSuffix(name, ".xml") { + cw, err := loadCodewalk(abspath + "/" + name) + if err != nil { + continue + } + v.Push(&elem{name[0 : len(name)-len(".xml")], cw.Title}) + } + } + + b := applyTemplate(codewalkdirHTML, "codewalkdir", v) + servePage(w, "Codewalks", "", "", b) +} + +// codewalkFileprint serves requests with ?fileprint=f&lo=lo&hi=hi. +// The filename f has already been retrieved and is passed as an argument. +// Lo and hi are the numbers of the first and last line to highlight +// in the response. This format is used for the middle window pane +// of the codewalk pages. It is a separate iframe and does not get +// the usual godoc HTML wrapper. +func codewalkFileprint(w http.ResponseWriter, r *http.Request, f string) { + abspath := absolutePath(f, *goroot) + data, err := fs.ReadFile(abspath) + if err != nil { + log.Print(err) + serveError(w, r, f, err) + return + } + lo, _ := strconv.Atoi(r.FormValue("lo")) + hi, _ := strconv.Atoi(r.FormValue("hi")) + if hi < lo { + hi = lo + } + lo = lineToByte(data, lo) + hi = lineToByte(data, hi+1) + + // Put the mark 4 lines before lo, so that the iframe + // shows a few lines of context before the highlighted + // section. + n := 4 + mark := lo + for ; mark > 0 && n > 0; mark-- { + if data[mark-1] == '\n' { + if n--; n == 0 { + break + } + } + } + + io.WriteString(w, `
`)
+	template.HTMLEscape(w, data[0:mark])
+	io.WriteString(w, "")
+	template.HTMLEscape(w, data[mark:lo])
+	if lo < hi {
+		io.WriteString(w, "
") + template.HTMLEscape(w, data[lo:hi]) + io.WriteString(w, "
") + } + template.HTMLEscape(w, data[hi:]) + io.WriteString(w, "
") +} + +// addrToByte evaluates the given address starting at offset start in data. +// It returns the lo and hi byte offset of the matched region within data. +// See http://plan9.bell-labs.com/sys/doc/sam/sam.html Table II +// for details on the syntax. +func addrToByteRange(addr string, start int, data []byte) (lo, hi int, err os.Error) { + var ( + dir byte + prevc byte + charOffset bool + ) + lo = start + hi = start + for addr != "" && err == nil { + c := addr[0] + switch c { + default: + err = os.NewError("invalid address syntax near " + string(c)) + case ',': + if len(addr) == 1 { + hi = len(data) + } else { + _, hi, err = addrToByteRange(addr[1:], hi, data) + } + return + + case '+', '-': + if prevc == '+' || prevc == '-' { + lo, hi, err = addrNumber(data, lo, hi, prevc, 1, charOffset) + } + dir = c + + case '$': + lo = len(data) + hi = len(data) + if len(addr) > 1 { + dir = '+' + } + + case '#': + charOffset = true + + case '0', '1', '2', '3', '4', '5', '6', '7', '8', '9': + var i int + for i = 1; i < len(addr); i++ { + if addr[i] < '0' || addr[i] > '9' { + break + } + } + var n int + n, err = strconv.Atoi(addr[0:i]) + if err != nil { + break + } + lo, hi, err = addrNumber(data, lo, hi, dir, n, charOffset) + dir = 0 + charOffset = false + prevc = c + addr = addr[i:] + continue + + case '/': + var i, j int + Regexp: + for i = 1; i < len(addr); i++ { + switch addr[i] { + case '\\': + i++ + case '/': + j = i + 1 + break Regexp + } + } + if j == 0 { + j = i + } + pattern := addr[1:i] + lo, hi, err = addrRegexp(data, lo, hi, dir, pattern) + prevc = c + addr = addr[j:] + continue + } + prevc = c + addr = addr[1:] + } + + if err == nil && dir != 0 { + lo, hi, err = addrNumber(data, lo, hi, dir, 1, charOffset) + } + if err != nil { + return 0, 0, err + } + return lo, hi, nil +} + +// addrNumber applies the given dir, n, and charOffset to the address lo, hi. +// dir is '+' or '-', n is the count, and charOffset is true if the syntax +// used was #n. Applying +n (or +#n) means to advance n lines +// (or characters) after hi. Applying -n (or -#n) means to back up n lines +// (or characters) before lo. +// The return value is the new lo, hi. +func addrNumber(data []byte, lo, hi int, dir byte, n int, charOffset bool) (int, int, os.Error) { + switch dir { + case 0: + lo = 0 + hi = 0 + fallthrough + + case '+': + if charOffset { + pos := hi + for ; n > 0 && pos < len(data); n-- { + _, size := utf8.DecodeRune(data[pos:]) + pos += size + } + if n == 0 { + return pos, pos, nil + } + break + } + // find next beginning of line + if hi > 0 { + for hi < len(data) && data[hi-1] != '\n' { + hi++ + } + } + lo = hi + if n == 0 { + return lo, hi, nil + } + for ; hi < len(data); hi++ { + if data[hi] != '\n' { + continue + } + switch n--; n { + case 1: + lo = hi + 1 + case 0: + return lo, hi + 1, nil + } + } + + case '-': + if charOffset { + // Scan backward for bytes that are not UTF-8 continuation bytes. + pos := lo + for ; pos > 0 && n > 0; pos-- { + if data[pos]&0xc0 != 0x80 { + n-- + } + } + if n == 0 { + return pos, pos, nil + } + break + } + // find earlier beginning of line + for lo > 0 && data[lo-1] != '\n' { + lo-- + } + hi = lo + if n == 0 { + return lo, hi, nil + } + for ; lo >= 0; lo-- { + if lo > 0 && data[lo-1] != '\n' { + continue + } + switch n--; n { + case 1: + hi = lo + case 0: + return lo, hi, nil + } + } + } + + return 0, 0, os.NewError("address out of range") +} + +// addrRegexp searches for pattern in the given direction starting at lo, hi. +// The direction dir is '+' (search forward from hi) or '-' (search backward from lo). +// Backward searches are unimplemented. +func addrRegexp(data []byte, lo, hi int, dir byte, pattern string) (int, int, os.Error) { + re, err := regexp.Compile(pattern) + if err != nil { + return 0, 0, err + } + if dir == '-' { + // Could implement reverse search using binary search + // through file, but that seems like overkill. + return 0, 0, os.NewError("reverse search not implemented") + } + m := re.FindIndex(data[hi:]) + if len(m) > 0 { + m[0] += hi + m[1] += hi + } else if hi > 0 { + // No match. Wrap to beginning of data. + m = re.FindIndex(data) + } + if len(m) == 0 { + return 0, 0, os.NewError("no match for " + pattern) + } + return m[0], m[1], nil +} + +// lineToByte returns the byte index of the first byte of line n. +// Line numbers begin at 1. +func lineToByte(data []byte, n int) int { + if n <= 1 { + return 0 + } + n-- + for i, c := range data { + if c == '\n' { + if n--; n == 0 { + return i + 1 + } + } + } + return len(data) +} + +// byteToLine returns the number of the line containing the byte at index i. +func byteToLine(data []byte, i int) int { + l := 1 + for j, c := range data { + if j == i { + return l + } + if c == '\n' { + l++ + } + } + return l +} diff --git a/src/cmd/godoc/dirtrees.go b/src/cmd/godoc/dirtrees.go new file mode 100644 index 000000000..aa590b363 --- /dev/null +++ b/src/cmd/godoc/dirtrees.go @@ -0,0 +1,342 @@ +// Copyright 2010 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// This file contains the code dealing with package directory trees. + +package main + +import ( + "bytes" + "go/doc" + "go/parser" + "go/token" + "log" + "path/filepath" + "strings" + "unicode" +) + +type Directory struct { + Depth int + Path string // includes Name + Name string + Text string // package documentation, if any + Dirs []*Directory // subdirectories +} + +func isGoFile(fi FileInfo) bool { + name := fi.Name() + return fi.IsRegular() && + len(name) > 0 && name[0] != '.' && // ignore .files + filepath.Ext(name) == ".go" +} + +func isPkgFile(fi FileInfo) bool { + return isGoFile(fi) && + !strings.HasSuffix(fi.Name(), "_test.go") // ignore test files +} + +func isPkgDir(fi FileInfo) bool { + name := fi.Name() + return fi.IsDirectory() && len(name) > 0 && + name[0] != '_' && name[0] != '.' // ignore _files and .files +} + +func firstSentence(s string) string { + i := -1 // index+1 of first terminator (punctuation ending a sentence) + j := -1 // index+1 of first terminator followed by white space + prev := 'A' + for k, ch := range s { + k1 := k + 1 + if ch == '.' || ch == '!' || ch == '?' { + if i < 0 { + i = k1 // first terminator + } + if k1 < len(s) && s[k1] <= ' ' { + if j < 0 { + j = k1 // first terminator followed by white space + } + if !unicode.IsUpper(prev) { + j = k1 + break + } + } + } + prev = ch + } + + if j < 0 { + // use the next best terminator + j = i + if j < 0 { + // no terminator at all, use the entire string + j = len(s) + } + } + + return s[0:j] +} + +type treeBuilder struct { + pathFilter func(string) bool + maxDepth int +} + +func (b *treeBuilder) newDirTree(fset *token.FileSet, path, name string, depth int) *Directory { + if b.pathFilter != nil && !b.pathFilter(path) { + return nil + } + + if depth >= b.maxDepth { + // return a dummy directory so that the parent directory + // doesn't get discarded just because we reached the max + // directory depth + return &Directory{depth, path, name, "", nil} + } + + list, err := fs.ReadDir(path) + if err != nil { + // newDirTree is called with a path that should be a package + // directory; errors here should not happen, but if they do, + // we want to know about them + log.Printf("ReadDir(%s): %s", path, err) + } + + // determine number of subdirectories and if there are package files + ndirs := 0 + hasPkgFiles := false + var synopses [4]string // prioritized package documentation (0 == highest priority) + for _, d := range list { + switch { + case isPkgDir(d): + ndirs++ + case isPkgFile(d): + // looks like a package file, but may just be a file ending in ".go"; + // don't just count it yet (otherwise we may end up with hasPkgFiles even + // though the directory doesn't contain any real package files - was bug) + if synopses[0] == "" { + // no "optimal" package synopsis yet; continue to collect synopses + file, err := parser.ParseFile(fset, filepath.Join(path, d.Name()), nil, + parser.ParseComments|parser.PackageClauseOnly) + if err == nil { + hasPkgFiles = true + if file.Doc != nil { + // prioritize documentation + i := -1 + switch file.Name.Name { + case name: + i = 0 // normal case: directory name matches package name + case fakePkgName: + i = 1 // synopses for commands + case "main": + i = 2 // directory contains a main package + default: + i = 3 // none of the above + } + if 0 <= i && i < len(synopses) && synopses[i] == "" { + synopses[i] = firstSentence(doc.CommentText(file.Doc)) + } + } + } + } + } + } + + // create subdirectory tree + var dirs []*Directory + if ndirs > 0 { + dirs = make([]*Directory, ndirs) + i := 0 + for _, d := range list { + if isPkgDir(d) { + name := d.Name() + dd := b.newDirTree(fset, filepath.Join(path, name), name, depth+1) + if dd != nil { + dirs[i] = dd + i++ + } + } + } + dirs = dirs[0:i] + } + + // if there are no package files and no subdirectories + // containing package files, ignore the directory + if !hasPkgFiles && len(dirs) == 0 { + return nil + } + + // select the highest-priority synopsis for the directory entry, if any + synopsis := "" + for _, synopsis = range synopses { + if synopsis != "" { + break + } + } + + return &Directory{depth, path, name, synopsis, dirs} +} + +// newDirectory creates a new package directory tree with at most maxDepth +// levels, anchored at root. The result tree is pruned such that it only +// contains directories that contain package files or that contain +// subdirectories containing package files (transitively). If a non-nil +// pathFilter is provided, directory paths additionally must be accepted +// by the filter (i.e., pathFilter(path) must be true). If a value >= 0 is +// provided for maxDepth, nodes at larger depths are pruned as well; they +// are assumed to contain package files even if their contents are not known +// (i.e., in this case the tree may contain directories w/o any package files). +// +func newDirectory(root string, pathFilter func(string) bool, maxDepth int) *Directory { + // The root could be a symbolic link so use Stat not Lstat. + d, err := fs.Stat(root) + // If we fail here, report detailed error messages; otherwise + // is is hard to see why a directory tree was not built. + switch { + case err != nil: + log.Printf("newDirectory(%s): %s", root, err) + return nil + case !isPkgDir(d): + log.Printf("newDirectory(%s): not a package directory", root) + return nil + } + if maxDepth < 0 { + maxDepth = 1e6 // "infinity" + } + b := treeBuilder{pathFilter, maxDepth} + // the file set provided is only for local parsing, no position + // information escapes and thus we don't need to save the set + return b.newDirTree(token.NewFileSet(), root, d.Name(), 0) +} + +func (dir *Directory) writeLeafs(buf *bytes.Buffer) { + if dir != nil { + if len(dir.Dirs) == 0 { + buf.WriteString(dir.Path) + buf.WriteByte('\n') + return + } + + for _, d := range dir.Dirs { + d.writeLeafs(buf) + } + } +} + +func (dir *Directory) walk(c chan<- *Directory, skipRoot bool) { + if dir != nil { + if !skipRoot { + c <- dir + } + for _, d := range dir.Dirs { + d.walk(c, false) + } + } +} + +func (dir *Directory) iter(skipRoot bool) <-chan *Directory { + c := make(chan *Directory) + go func() { + dir.walk(c, skipRoot) + close(c) + }() + return c +} + +func (dir *Directory) lookupLocal(name string) *Directory { + for _, d := range dir.Dirs { + if d.Name == name { + return d + } + } + return nil +} + +// lookup looks for the *Directory for a given path, relative to dir. +func (dir *Directory) lookup(path string) *Directory { + d := strings.Split(dir.Path, string(filepath.Separator)) + p := strings.Split(path, string(filepath.Separator)) + i := 0 + for i < len(d) { + if i >= len(p) || d[i] != p[i] { + return nil + } + i++ + } + for dir != nil && i < len(p) { + dir = dir.lookupLocal(p[i]) + i++ + } + return dir +} + +// DirEntry describes a directory entry. The Depth and Height values +// are useful for presenting an entry in an indented fashion. +// +type DirEntry struct { + Depth int // >= 0 + Height int // = DirList.MaxHeight - Depth, > 0 + Path string // includes Name, relative to DirList root + Name string + Synopsis string +} + +type DirList struct { + MaxHeight int // directory tree height, > 0 + List []DirEntry +} + +// listing creates a (linear) directory listing from a directory tree. +// If skipRoot is set, the root directory itself is excluded from the list. +// +func (root *Directory) listing(skipRoot bool) *DirList { + if root == nil { + return nil + } + + // determine number of entries n and maximum height + n := 0 + minDepth := 1 << 30 // infinity + maxDepth := 0 + for d := range root.iter(skipRoot) { + n++ + if minDepth > d.Depth { + minDepth = d.Depth + } + if maxDepth < d.Depth { + maxDepth = d.Depth + } + } + maxHeight := maxDepth - minDepth + 1 + + if n == 0 { + return nil + } + + // create list + list := make([]DirEntry, n) + i := 0 + for d := range root.iter(skipRoot) { + p := &list[i] + p.Depth = d.Depth - minDepth + p.Height = maxHeight - p.Depth + // the path is relative to root.Path - remove the root.Path + // prefix (the prefix should always be present but avoid + // crashes and check) + path := d.Path + if strings.HasPrefix(d.Path, root.Path) { + path = d.Path[len(root.Path):] + } + // remove trailing separator if any - path must be relative + if len(path) > 0 && path[0] == filepath.Separator { + path = path[1:] + } + p.Path = path + p.Name = d.Name + p.Synopsis = d.Text + i++ + } + + return &DirList{maxHeight, list} +} diff --git a/src/cmd/godoc/doc.go b/src/cmd/godoc/doc.go new file mode 100644 index 000000000..dc98b0eca --- /dev/null +++ b/src/cmd/godoc/doc.go @@ -0,0 +1,130 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +/* + +Godoc extracts and generates documentation for Go programs. + +It has two modes. + +Without the -http flag, it runs in command-line mode and prints plain text +documentation to standard output and exits. If the -src flag is specified, +godoc prints the exported interface of a package in Go source form, or the +implementation of a specific exported language entity: + + godoc fmt # documentation for package fmt + godoc fmt Printf # documentation for fmt.Printf + godoc -src fmt # fmt package interface in Go source form + godoc -src fmt Printf # implementation of fmt.Printf + +In command-line mode, the -q flag enables search queries against a godoc running +as a webserver. If no explicit server address is specified with the -server flag, +godoc first tries localhost:6060 and then http://golang.org. + + godoc -q Reader Writer + godoc -q math.Sin + godoc -server=:6060 -q sin + +With the -http flag, it runs as a web server and presents the documentation as a +web page. + + godoc -http=:6060 + +Usage: + godoc [flag] package [name ...] + +The flags are: + -v + verbose mode + -q + arguments are considered search queries: a legal query is a + single identifier (such as ToLower) or a qualified identifier + (such as math.Sin). + -src + print (exported) source in command-line mode + -tabwidth=4 + width of tabs in units of spaces + -timestamps=true + show timestamps with directory listings + -index + enable identifier and full text search index + (no search box is shown if -index is not set) + -maxresults=10000 + maximum number of full text search results shown + (no full text index is built if maxresults <= 0) + -path="" + additional package directories (colon-separated) + -html + print HTML in command-line mode + -goroot=$GOROOT + Go root directory + -http=addr + HTTP service address (e.g., '127.0.0.1:6060' or just ':6060') + -server=addr + webserver address for command line searches + -sync="command" + if this and -sync_minutes are set, run the argument as a + command every sync_minutes; it is intended to update the + repository holding the source files. + -sync_minutes=0 + sync interval in minutes; sync is disabled if <= 0 + -filter="" + filter file containing permitted package directory paths + -filter_minutes=0 + filter file update interval in minutes; update is disabled if <= 0 + -zip="" + zip file providing the file system to serve; disabled if empty + +The -path flag accepts a list of colon-separated paths; unrooted paths are relative +to the current working directory. Each path is considered as an additional root for +packages in order of appearance. The last (absolute) path element is the prefix for +the package path. For instance, given the flag value: + + path=".:/home/bar:/public" + +for a godoc started in /home/user/godoc, absolute paths are mapped to package paths +as follows: + + /home/user/godoc/x -> godoc/x + /home/bar/x -> bar/x + /public/x -> public/x + +Paths provided via -path may point to very large file systems that contain +non-Go files. Creating the subtree of directories with Go packages may take +a long amount of time. A file containing newline-separated directory paths +may be provided with the -filter flag; if it exists, only directories +on those paths are considered. If -filter_minutes is set, the filter_file is +updated regularly by walking the entire directory tree. + +When godoc runs as a web server and -index is set, a search index is maintained. +The index is created at startup and is automatically updated every time the +-sync command terminates with exit status 0, indicating that files have changed. + +If the sync exit status is 1, godoc assumes that it succeeded without errors +but that no files changed; the index is not updated in this case. + +In all other cases, sync is assumed to have failed and godoc backs off running +sync exponentially (up to 1 day). As soon as sync succeeds again (exit status 0 +or 1), the normal sync rhythm is re-established. + +The index contains both identifier and full text search information (searchable +via regular expressions). The maximum number of full text search results shown +can be set with the -maxresults flag; if set to 0, no full text results are +shown, and only an identifier index but no full text search index is created. + +By default, godoc serves files from the file system of the underlying OS. +Instead, a .zip file may be provided via the -zip flag, which contains +the file system to serve. The file paths stored in the .zip file must use +slash ('/') as path separator; and they must be unrooted. $GOROOT (or -goroot) +must be set to the .zip file directory path containing the Go root directory. +For instance, for a .zip file created by the command: + + zip go.zip $HOME/go + +one may run godoc as follows: + + godoc -http=:6060 -zip=go.zip -goroot=$HOME/go + +*/ +package documentation diff --git a/src/cmd/godoc/filesystem.go b/src/cmd/godoc/filesystem.go new file mode 100644 index 000000000..a68c08592 --- /dev/null +++ b/src/cmd/godoc/filesystem.go @@ -0,0 +1,104 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// This file defines types for abstract file system access and +// provides an implementation accessing the file system of the +// underlying OS. + +package main + +import ( + "fmt" + "io" + "io/ioutil" + "os" +) + +// The FileInfo interface provides access to file information. +type FileInfo interface { + Name() string + Size() int64 + Mtime_ns() int64 + IsRegular() bool + IsDirectory() bool +} + +// The FileSystem interface specifies the methods godoc is using +// to access the file system for which it serves documentation. +type FileSystem interface { + Open(path string) (io.ReadCloser, os.Error) + Lstat(path string) (FileInfo, os.Error) + Stat(path string) (FileInfo, os.Error) + ReadDir(path string) ([]FileInfo, os.Error) + ReadFile(path string) ([]byte, os.Error) +} + +// ---------------------------------------------------------------------------- +// OS-specific FileSystem implementation + +var OS FileSystem = osFS{} + +// osFI is the OS-specific implementation of FileInfo. +type osFI struct { + *os.FileInfo +} + +func (fi osFI) Name() string { + return fi.FileInfo.Name +} + +func (fi osFI) Size() int64 { + if fi.IsDirectory() { + return 0 + } + return fi.FileInfo.Size +} + +func (fi osFI) Mtime_ns() int64 { + return fi.FileInfo.Mtime_ns +} + +// osFS is the OS-specific implementation of FileSystem +type osFS struct{} + +func (osFS) Open(path string) (io.ReadCloser, os.Error) { + f, err := os.Open(path) + if err != nil { + return nil, err + } + fi, err := f.Stat() + if err != nil { + return nil, err + } + if fi.IsDirectory() { + return nil, fmt.Errorf("Open: %s is a directory", path) + } + return f, nil +} + +func (osFS) Lstat(path string) (FileInfo, os.Error) { + fi, err := os.Lstat(path) + return osFI{fi}, err +} + +func (osFS) Stat(path string) (FileInfo, os.Error) { + fi, err := os.Stat(path) + return osFI{fi}, err +} + +func (osFS) ReadDir(path string) ([]FileInfo, os.Error) { + l0, err := ioutil.ReadDir(path) // l0 is sorted + if err != nil { + return nil, err + } + l1 := make([]FileInfo, len(l0)) + for i, e := range l0 { + l1[i] = osFI{e} + } + return l1, nil +} + +func (osFS) ReadFile(path string) ([]byte, os.Error) { + return ioutil.ReadFile(path) +} diff --git a/src/cmd/godoc/format.go b/src/cmd/godoc/format.go new file mode 100644 index 000000000..78dde4166 --- /dev/null +++ b/src/cmd/godoc/format.go @@ -0,0 +1,358 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// This file implements FormatSelections and FormatText. +// FormatText is used to HTML-format Go and non-Go source +// text with line numbers and highlighted sections. It is +// built on top of FormatSelections, a generic formatter +// for "selected" text. + +package main + +import ( + "fmt" + "go/scanner" + "go/token" + "io" + "regexp" + "strconv" + "template" +) + +// ---------------------------------------------------------------------------- +// Implementation of FormatSelections + +// A Selection is a function returning offset pairs []int{a, b} +// describing consecutive non-overlapping text segments [a, b). +// If there are no more segments, a Selection must return nil. +// +// TODO It's more efficient to return a pair (a, b int) instead +// of creating lots of slices. Need to determine how to +// indicate the end of a Selection. +// +type Selection func() []int + +// A LinkWriter writes some start or end "tag" to w for the text offset offs. +// It is called by FormatSelections at the start or end of each link segment. +// +type LinkWriter func(w io.Writer, offs int, start bool) + +// A SegmentWriter formats a text according to selections and writes it to w. +// The selections parameter is a bit set indicating which selections provided +// to FormatSelections overlap with the text segment: If the n'th bit is set +// in selections, the n'th selection provided to FormatSelections is overlapping +// with the text. +// +type SegmentWriter func(w io.Writer, text []byte, selections int) + +// FormatSelections takes a text and writes it to w using link and segment +// writers lw and sw as follows: lw is invoked for consecutive segment starts +// and ends as specified through the links selection, and sw is invoked for +// consecutive segments of text overlapped by the same selections as specified +// by selections. The link writer lw may be nil, in which case the links +// Selection is ignored. +// +func FormatSelections(w io.Writer, text []byte, lw LinkWriter, links Selection, sw SegmentWriter, selections ...Selection) { + if lw != nil { + selections = append(selections, links) + } + + // compute the sequence of consecutive segment changes + changes := newMerger(selections) + + // The i'th bit in bitset indicates that the text + // at the current offset is covered by selections[i]. + bitset := 0 + lastOffs := 0 + + // Text segments are written in a delayed fashion + // such that consecutive segments belonging to the + // same selection can be combined (peephole optimization). + // last describes the last segment which has not yet been written. + var last struct { + begin, end int // valid if begin < end + bitset int + } + + // flush writes the last delayed text segment + flush := func() { + if last.begin < last.end { + sw(w, text[last.begin:last.end], last.bitset) + } + last.begin = last.end // invalidate last + } + + // segment runs the segment [lastOffs, end) with the selection + // indicated by bitset through the segment peephole optimizer. + segment := func(end int) { + if lastOffs < end { // ignore empty segments + if last.end != lastOffs || last.bitset != bitset { + // the last segment is not adjacent to or + // differs from the new one + flush() + // start a new segment + last.begin = lastOffs + } + last.end = end + last.bitset = bitset + } + } + + for { + // get the next segment change + index, offs, start := changes.next() + if index < 0 || offs > len(text) { + // no more segment changes or the next change + // is past the end of the text - we're done + break + } + // determine the kind of segment change + if index == len(selections)-1 { + // we have a link segment change: + // format the previous selection segment, write the + // link tag and start a new selection segment + segment(offs) + flush() + lastOffs = offs + lw(w, offs, start) + } else { + // we have a selection change: + // format the previous selection segment, determine + // the new selection bitset and start a new segment + segment(offs) + lastOffs = offs + mask := 1 << uint(index) + if start { + bitset |= mask + } else { + bitset &^= mask + } + } + } + segment(len(text)) + flush() +} + +// A merger merges a slice of Selections and produces a sequence of +// consecutive segment change events through repeated next() calls. +// +type merger struct { + selections []Selection + segments [][]int // segments[i] is the next segment of selections[i] +} + +const infinity int = 2e9 + +func newMerger(selections []Selection) *merger { + segments := make([][]int, len(selections)) + for i, sel := range selections { + segments[i] = []int{infinity, infinity} + if sel != nil { + if seg := sel(); seg != nil { + segments[i] = seg + } + } + } + return &merger{selections, segments} +} + +// next returns the next segment change: index specifies the Selection +// to which the segment belongs, offs is the segment start or end offset +// as determined by the start value. If there are no more segment changes, +// next returns an index value < 0. +// +func (m *merger) next() (index, offs int, start bool) { + // find the next smallest offset where a segment starts or ends + offs = infinity + index = -1 + for i, seg := range m.segments { + switch { + case seg[0] < offs: + offs = seg[0] + index = i + start = true + case seg[1] < offs: + offs = seg[1] + index = i + start = false + } + } + if index < 0 { + // no offset found => all selections merged + return + } + // offset found - it's either the start or end offset but + // either way it is ok to consume the start offset: set it + // to infinity so it won't be considered in the following + // next call + m.segments[index][0] = infinity + if start { + return + } + // end offset found - consume it + m.segments[index][1] = infinity + // advance to the next segment for that selection + seg := m.selections[index]() + if seg == nil { + return + } + m.segments[index] = seg + return +} + +// ---------------------------------------------------------------------------- +// Implementation of FormatText + +// lineSelection returns the line segments for text as a Selection. +func lineSelection(text []byte) Selection { + i, j := 0, 0 + return func() (seg []int) { + // find next newline, if any + for j < len(text) { + j++ + if text[j-1] == '\n' { + break + } + } + if i < j { + // text[i:j] constitutes a line + seg = []int{i, j} + i = j + } + return + } +} + +// commentSelection returns the sequence of consecutive comments +// in the Go src text as a Selection. +// +func commentSelection(src []byte) Selection { + var s scanner.Scanner + fset := token.NewFileSet() + file := fset.AddFile("", fset.Base(), len(src)) + s.Init(file, src, nil, scanner.ScanComments+scanner.InsertSemis) + return func() (seg []int) { + for { + pos, tok, lit := s.Scan() + if tok == token.EOF { + break + } + offs := file.Offset(pos) + if tok == token.COMMENT { + seg = []int{offs, offs + len(lit)} + break + } + } + return + } +} + +// makeSelection is a helper function to make a Selection from a slice of pairs. +func makeSelection(matches [][]int) Selection { + return func() (seg []int) { + if len(matches) > 0 { + seg = matches[0] + matches = matches[1:] + } + return + } +} + +// regexpSelection computes the Selection for the regular expression expr in text. +func regexpSelection(text []byte, expr string) Selection { + var matches [][]int + if rx, err := regexp.Compile(expr); err == nil { + matches = rx.FindAllIndex(text, -1) + } + return makeSelection(matches) +} + +var selRx = regexp.MustCompile(`^([0-9]+):([0-9]+)`) + +// rangeSelection computes the Selection for a text range described +// by the argument str; the range description must match the selRx +// regular expression. +// +func rangeSelection(str string) Selection { + m := selRx.FindStringSubmatch(str) + if len(m) >= 2 { + from, _ := strconv.Atoi(m[1]) + to, _ := strconv.Atoi(m[2]) + if from < to { + return makeSelection([][]int{{from, to}}) + } + } + return nil +} + +// Span tags for all the possible selection combinations that may +// be generated by FormatText. Selections are indicated by a bitset, +// and the value of the bitset specifies the tag to be used. +// +// bit 0: comments +// bit 1: highlights +// bit 2: selections +// +var startTags = [][]byte{ + /* 000 */ []byte(``), + /* 001 */ []byte(``), + /* 010 */ []byte(``), + /* 011 */ []byte(``), + /* 100 */ []byte(``), + /* 101 */ []byte(``), + /* 110 */ []byte(``), + /* 111 */ []byte(``), +} + +var endTag = []byte(``) + +func selectionTag(w io.Writer, text []byte, selections int) { + if selections < len(startTags) { + if tag := startTags[selections]; len(tag) > 0 { + w.Write(tag) + template.HTMLEscape(w, text) + w.Write(endTag) + return + } + } + template.HTMLEscape(w, text) +} + +// FormatText HTML-escapes text and writes it to w. +// Consecutive text segments are wrapped in HTML spans (with tags as +// defined by startTags and endTag) as follows: +// +// - if line >= 0, line number (ln) spans are inserted before each line, +// starting with the value of line +// - if the text is Go source, comments get the "comment" span class +// - each occurrence of the regular expression pattern gets the "highlight" +// span class +// - text segments covered by selection get the "selection" span class +// +// Comments, highlights, and selections may overlap arbitrarily; the respective +// HTML span classes are specified in the startTags variable. +// +func FormatText(w io.Writer, text []byte, line int, goSource bool, pattern string, selection Selection) { + var comments, highlights Selection + if goSource { + comments = commentSelection(text) + } + if pattern != "" { + highlights = regexpSelection(text, pattern) + } + if line >= 0 || comments != nil || highlights != nil || selection != nil { + var lineTag LinkWriter + if line >= 0 { + lineTag = func(w io.Writer, _ int, start bool) { + if start { + fmt.Fprintf(w, "%6d\t", line, line) + line++ + } + } + } + FormatSelections(w, text, lineTag, lineSelection(text), selectionTag, comments, highlights, selection) + } else { + template.HTMLEscape(w, text) + } +} diff --git a/src/cmd/godoc/godoc.go b/src/cmd/godoc/godoc.go new file mode 100644 index 000000000..b8a839404 --- /dev/null +++ b/src/cmd/godoc/godoc.go @@ -0,0 +1,1159 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package main + +import ( + "bytes" + "flag" + "fmt" + "go/ast" + "go/build" + "go/doc" + "go/printer" + "go/token" + "http" + "io" + "log" + "os" + "path" + "path/filepath" + "regexp" + "runtime" + "sort" + "strings" + "template" + "time" +) + +// ---------------------------------------------------------------------------- +// Globals + +type delayTime struct { + RWValue +} + +func (dt *delayTime) backoff(max int) { + dt.mutex.Lock() + v := dt.value.(int) * 2 + if v > max { + v = max + } + dt.value = v + // don't change dt.timestamp - calling backoff indicates an error condition + dt.mutex.Unlock() +} + +var ( + verbose = flag.Bool("v", false, "verbose mode") + + // file system roots + // TODO(gri) consider the invariant that goroot always end in '/' + goroot = flag.String("goroot", runtime.GOROOT(), "Go root directory") + testDir = flag.String("testdir", "", "Go root subdirectory - for testing only (faster startups)") + pkgPath = flag.String("path", "", "additional package directories (colon-separated)") + filter = flag.String("filter", "", "filter file containing permitted package directory paths") + filterMin = flag.Int("filter_minutes", 0, "filter file update interval in minutes; disabled if <= 0") + filterDelay delayTime // actual filter update interval in minutes; usually filterDelay == filterMin, but filterDelay may back off exponentially + + // layout control + tabwidth = flag.Int("tabwidth", 4, "tab width") + showTimestamps = flag.Bool("timestamps", true, "show timestamps with directory listings") + templateDir = flag.String("templates", "", "directory containing alternate template files") + + // search index + indexEnabled = flag.Bool("index", false, "enable search index") + maxResults = flag.Int("maxresults", 10000, "maximum number of full text search results shown") + + // file system mapping + fs FileSystem // the underlying file system for godoc + fsHttp http.FileSystem // the underlying file system for http + fsMap Mapping // user-defined mapping + fsTree RWValue // *Directory tree of packages, updated with each sync + pathFilter RWValue // filter used when building fsMap directory trees + fsModified RWValue // timestamp of last call to invalidateIndex + + // http handlers + fileServer http.Handler // default file server + cmdHandler httpHandler + pkgHandler httpHandler +) + +func initHandlers() { + paths := filepath.SplitList(*pkgPath) + for _, t := range build.Path { + if t.Goroot { + continue + } + paths = append(paths, t.SrcDir()) + } + fsMap.Init(paths) + + fileServer = http.FileServer(fsHttp) + cmdHandler = httpHandler{"/cmd/", filepath.Join(*goroot, "src", "cmd"), false} + pkgHandler = httpHandler{"/pkg/", filepath.Join(*goroot, "src", "pkg"), true} +} + +func registerPublicHandlers(mux *http.ServeMux) { + mux.Handle(cmdHandler.pattern, &cmdHandler) + mux.Handle(pkgHandler.pattern, &pkgHandler) + mux.HandleFunc("/doc/codewalk/", codewalk) + mux.HandleFunc("/search", search) + mux.Handle("/robots.txt", fileServer) + mux.HandleFunc("/", serveFile) +} + +func initFSTree() { + fsTree.set(newDirectory(filepath.Join(*goroot, *testDir), nil, -1)) + invalidateIndex() +} + +// ---------------------------------------------------------------------------- +// Directory filters + +// isParentOf returns true if p is a parent of (or the same as) q +// where p and q are directory paths. +func isParentOf(p, q string) bool { + n := len(p) + return strings.HasPrefix(q, p) && (len(q) <= n || q[n] == '/') +} + +func setPathFilter(list []string) { + if len(list) == 0 { + pathFilter.set(nil) + return + } + + // len(list) > 0 + pathFilter.set(func(path string) bool { + // list is sorted in increasing order and for each path all its children are removed + i := sort.Search(len(list), func(i int) bool { return list[i] > path }) + // Now we have list[i-1] <= path < list[i]. + // Path may be a child of list[i-1] or a parent of list[i]. + return i > 0 && isParentOf(list[i-1], path) || i < len(list) && isParentOf(path, list[i]) + }) +} + +func getPathFilter() func(string) bool { + f, _ := pathFilter.get() + if f != nil { + return f.(func(string) bool) + } + return nil +} + +// readDirList reads a file containing a newline-separated list +// of directory paths and returns the list of paths. +func readDirList(filename string) ([]string, os.Error) { + contents, err := fs.ReadFile(filename) + if err != nil { + return nil, err + } + // create a sorted list of valid directory names + filter := func(path string) bool { + d, e := fs.Lstat(path) + if e != nil && err == nil { + // remember first error and return it from readDirList + // so we have at least some information if things go bad + err = e + } + return e == nil && isPkgDir(d) + } + list := canonicalizePaths(strings.Split(string(contents), "\n"), filter) + // for each parent path, remove all its children q + // (requirement for binary search to work when filtering) + i := 0 + for _, q := range list { + if i == 0 || !isParentOf(list[i-1], q) { + list[i] = q + i++ + } + } + return list[0:i], err +} + +// updateMappedDirs computes the directory tree for +// each user-defined file system mapping. If a filter +// is provided, it is used to filter directories. +// +func updateMappedDirs(filter func(string) bool) { + if !fsMap.IsEmpty() { + fsMap.Iterate(func(path string, value *RWValue) bool { + value.set(newDirectory(path, filter, -1)) + return true + }) + invalidateIndex() + } +} + +func updateFilterFile() { + updateMappedDirs(nil) // no filter for accuracy + + // collect directory tree leaf node paths + var buf bytes.Buffer + fsMap.Iterate(func(_ string, value *RWValue) bool { + v, _ := value.get() + if v != nil && v.(*Directory) != nil { + v.(*Directory).writeLeafs(&buf) + } + return true + }) + + // update filter file + if err := writeFileAtomically(*filter, buf.Bytes()); err != nil { + log.Printf("writeFileAtomically(%s): %s", *filter, err) + filterDelay.backoff(24 * 60) // back off exponentially, but try at least once a day + } else { + filterDelay.set(*filterMin) // revert to regular filter update schedule + } +} + +func initDirTrees() { + // setup initial path filter + if *filter != "" { + list, err := readDirList(*filter) + if err != nil { + log.Printf("readDirList(%s): %s", *filter, err) + } + if *verbose || len(list) == 0 { + log.Printf("found %d directory paths in file %s", len(list), *filter) + } + setPathFilter(list) + } + + go updateMappedDirs(getPathFilter()) // use filter for speed + + // start filter update goroutine, if enabled. + if *filter != "" && *filterMin > 0 { + filterDelay.set(*filterMin) // initial filter update delay + go func() { + for { + if *verbose { + log.Printf("start update of %s", *filter) + } + updateFilterFile() + delay, _ := filterDelay.get() + if *verbose { + log.Printf("next filter update in %dmin", delay.(int)) + } + time.Sleep(int64(delay.(int)) * 60e9) + } + }() + } +} + +// ---------------------------------------------------------------------------- +// Path mapping + +// Absolute paths are file system paths (backslash-separated on Windows), +// but relative paths are always slash-separated. + +func absolutePath(relpath, defaultRoot string) string { + abspath := fsMap.ToAbsolute(relpath) + if abspath == "" { + // no user-defined mapping found; use default mapping + abspath = filepath.Join(defaultRoot, filepath.FromSlash(relpath)) + } + return abspath +} + +func relativeURL(abspath string) string { + relpath := fsMap.ToRelative(abspath) + if relpath == "" { + // prefix must end in a path separator + prefix := *goroot + if len(prefix) > 0 && prefix[len(prefix)-1] != filepath.Separator { + prefix += string(filepath.Separator) + } + if strings.HasPrefix(abspath, prefix) { + // no user-defined mapping found; use default mapping + relpath = filepath.ToSlash(abspath[len(prefix):]) + } + } + // Only if path is an invalid absolute path is relpath == "" + // at this point. This should never happen since absolute paths + // are only created via godoc for files that do exist. However, + // it is ok to return ""; it will simply provide a link to the + // top of the pkg or src directories. + return relpath +} + +// ---------------------------------------------------------------------------- +// Tab conversion + +var spaces = []byte(" ") // 32 spaces seems like a good number + +const ( + indenting = iota + collecting +) + +// A tconv is an io.Writer filter for converting leading tabs into spaces. +type tconv struct { + output io.Writer + state int // indenting or collecting + indent int // valid if state == indenting +} + +func (p *tconv) writeIndent() (err os.Error) { + i := p.indent + for i >= len(spaces) { + i -= len(spaces) + if _, err = p.output.Write(spaces); err != nil { + return + } + } + // i < len(spaces) + if i > 0 { + _, err = p.output.Write(spaces[0:i]) + } + return +} + +func (p *tconv) Write(data []byte) (n int, err os.Error) { + if len(data) == 0 { + return + } + pos := 0 // valid if p.state == collecting + var b byte + for n, b = range data { + switch p.state { + case indenting: + switch b { + case '\t': + p.indent += *tabwidth + case '\n': + p.indent = 0 + if _, err = p.output.Write(data[n : n+1]); err != nil { + return + } + case ' ': + p.indent++ + default: + p.state = collecting + pos = n + if err = p.writeIndent(); err != nil { + return + } + } + case collecting: + if b == '\n' { + p.state = indenting + p.indent = 0 + if _, err = p.output.Write(data[pos : n+1]); err != nil { + return + } + } + } + } + n = len(data) + if pos < n && p.state == collecting { + _, err = p.output.Write(data[pos:]) + } + return +} + +// ---------------------------------------------------------------------------- +// Templates + +// Write an AST node to w. +func writeNode(w io.Writer, fset *token.FileSet, x interface{}) { + // convert trailing tabs into spaces using a tconv filter + // to ensure a good outcome in most browsers (there may still + // be tabs in comments and strings, but converting those into + // the right number of spaces is much harder) + // + // TODO(gri) rethink printer flags - perhaps tconv can be eliminated + // with an another printer mode (which is more efficiently + // implemented in the printer than here with another layer) + mode := printer.TabIndent | printer.UseSpaces + (&printer.Config{mode, *tabwidth}).Fprint(&tconv{output: w}, fset, x) +} + +func filenameFunc(path string) string { + _, localname := filepath.Split(path) + return localname +} + +func fileInfoNameFunc(fi FileInfo) string { + name := fi.Name() + if fi.IsDirectory() { + name += "/" + } + return name +} + +func fileInfoTimeFunc(fi FileInfo) string { + if t := fi.Mtime_ns(); t != 0 { + return time.SecondsToLocalTime(t / 1e9).String() + } + return "" // don't return epoch if time is obviously not set +} + +// The strings in infoKinds must be properly html-escaped. +var infoKinds = [nKinds]string{ + PackageClause: "package clause", + ImportDecl: "import decl", + ConstDecl: "const decl", + TypeDecl: "type decl", + VarDecl: "var decl", + FuncDecl: "func decl", + MethodDecl: "method decl", + Use: "use", +} + +func infoKind_htmlFunc(kind SpotKind) string { + return infoKinds[kind] // infoKind entries are html-escaped +} + +func infoLineFunc(info SpotInfo) int { + line := info.Lori() + if info.IsIndex() { + index, _ := searchIndex.get() + if index != nil { + line = index.(*Index).Snippet(line).Line + } else { + // no line information available because + // we don't have an index - this should + // never happen; be conservative and don't + // crash + line = 0 + } + } + return line +} + +func infoSnippet_htmlFunc(info SpotInfo) string { + if info.IsIndex() { + index, _ := searchIndex.get() + // Snippet.Text was HTML-escaped when it was generated + return index.(*Index).Snippet(info.Lori()).Text + } + return `no snippet text available` +} + +func nodeFunc(node interface{}, fset *token.FileSet) string { + var buf bytes.Buffer + writeNode(&buf, fset, node) + return buf.String() +} + +func node_htmlFunc(node interface{}, fset *token.FileSet) string { + var buf1 bytes.Buffer + writeNode(&buf1, fset, node) + var buf2 bytes.Buffer + FormatText(&buf2, buf1.Bytes(), -1, true, "", nil) + return buf2.String() +} + +func comment_htmlFunc(comment string) string { + var buf bytes.Buffer + // TODO(gri) Provide list of words (e.g. function parameters) + // to be emphasized by ToHTML. + doc.ToHTML(&buf, []byte(comment), nil) // does html-escaping + return buf.String() +} + +func pkgLinkFunc(path string) string { + relpath := relativeURL(path) + // because of the irregular mapping under goroot + // we need to correct certain relative paths + if strings.HasPrefix(relpath, "src/pkg/") { + relpath = relpath[len("src/pkg/"):] + } + return pkgHandler.pattern[1:] + relpath // remove trailing '/' for relative URL +} + +func posLink_urlFunc(node ast.Node, fset *token.FileSet) string { + var relpath string + var line int + var low, high int // selection + + if p := node.Pos(); p.IsValid() { + pos := fset.Position(p) + relpath = relativeURL(pos.Filename) + line = pos.Line + low = pos.Offset + } + if p := node.End(); p.IsValid() { + high = fset.Position(p).Offset + } + + var buf bytes.Buffer + template.HTMLEscape(&buf, []byte(relpath)) + // selection ranges are of form "s=low:high" + if low < high { + fmt.Fprintf(&buf, "?s=%d:%d", low, high) // no need for URL escaping + // if we have a selection, position the page + // such that the selection is a bit below the top + line -= 10 + if line < 1 { + line = 1 + } + } + // line id's in html-printed source are of the + // form "L%d" where %d stands for the line number + if line > 0 { + fmt.Fprintf(&buf, "#L%d", line) // no need for URL escaping + } + + return buf.String() +} + +// fmap describes the template functions installed with all godoc templates. +// Convention: template function names ending in "_html" or "_url" produce +// HTML- or URL-escaped strings; all other function results may +// require explicit escaping in the template. +var fmap = template.FuncMap{ + // various helpers + "filename": filenameFunc, + "repeat": strings.Repeat, + + // accss to FileInfos (directory listings) + "fileInfoName": fileInfoNameFunc, + "fileInfoTime": fileInfoTimeFunc, + + // access to search result information + "infoKind_html": infoKind_htmlFunc, + "infoLine": infoLineFunc, + "infoSnippet_html": infoSnippet_htmlFunc, + + // formatting of AST nodes + "node": nodeFunc, + "node_html": node_htmlFunc, + "comment_html": comment_htmlFunc, + + // support for URL attributes + "pkgLink": pkgLinkFunc, + "srcLink": relativeURL, + "posLink_url": posLink_urlFunc, +} + +func readTemplate(name string) *template.Template { + path := filepath.Join(*goroot, "lib", "godoc", name) + if *templateDir != "" { + defaultpath := path + path = filepath.Join(*templateDir, name) + if _, err := fs.Stat(path); err != nil { + log.Print("readTemplate:", err) + path = defaultpath + } + } + return template.Must(template.New(name).Funcs(fmap).ParseFile(path)) +} + +var ( + codewalkHTML, + codewalkdirHTML, + dirlistHTML, + errorHTML, + godocHTML, + packageHTML, + packageText, + searchHTML, + searchText *template.Template +) + +func readTemplates() { + // have to delay until after flags processing since paths depend on goroot + codewalkHTML = readTemplate("codewalk.html") + codewalkdirHTML = readTemplate("codewalkdir.html") + dirlistHTML = readTemplate("dirlist.html") + errorHTML = readTemplate("error.html") + godocHTML = readTemplate("godoc.html") + packageHTML = readTemplate("package.html") + packageText = readTemplate("package.txt") + searchHTML = readTemplate("search.html") + searchText = readTemplate("search.txt") +} + +// ---------------------------------------------------------------------------- +// Generic HTML wrapper + +func servePage(w http.ResponseWriter, title, subtitle, query string, content []byte) { + d := struct { + Title string + Subtitle string + PkgRoots []string + SearchBox bool + Query string + Version string + Menu []byte + Content []byte + }{ + title, + subtitle, + fsMap.PrefixList(), + *indexEnabled, + query, + runtime.Version(), + nil, + content, + } + + if err := godocHTML.Execute(w, &d); err != nil { + log.Printf("godocHTML.Execute: %s", err) + } +} + +func serveText(w http.ResponseWriter, text []byte) { + w.Header().Set("Content-Type", "text/plain; charset=utf-8") + w.Write(text) +} + +// ---------------------------------------------------------------------------- +// Files + +var ( + titleRx = regexp.MustCompile(``) + subtitleRx = regexp.MustCompile(``) + firstCommentRx = regexp.MustCompile(``) +) + +func extractString(src []byte, rx *regexp.Regexp) (s string) { + m := rx.FindSubmatch(src) + if m != nil { + s = strings.TrimSpace(string(m[1])) + } + return +} + +func serveHTMLDoc(w http.ResponseWriter, r *http.Request, abspath, relpath string) { + // get HTML body contents + src, err := fs.ReadFile(abspath) + if err != nil { + log.Printf("ReadFile: %s", err) + serveError(w, r, relpath, err) + return + } + + // if it begins with "") + FormatText(&buf, src, 1, filepath.Ext(abspath) == ".go", r.FormValue("h"), rangeSelection(r.FormValue("s"))) + buf.WriteString("
") + + servePage(w, title+" "+relpath, "", "", buf.Bytes()) +} + +func serveDirectory(w http.ResponseWriter, r *http.Request, abspath, relpath string) { + if redirect(w, r) { + return + } + + list, err := fs.ReadDir(abspath) + if err != nil { + log.Printf("ReadDir: %s", err) + serveError(w, r, relpath, err) + return + } + + contents := applyTemplate(dirlistHTML, "dirlistHTML", list) + servePage(w, "Directory "+relpath, "", "", contents) +} + +func serveFile(w http.ResponseWriter, r *http.Request) { + relpath := r.URL.Path[1:] // serveFile URL paths start with '/' + abspath := absolutePath(relpath, *goroot) + + // pick off special cases and hand the rest to the standard file server + switch r.URL.Path { + case "/": + serveHTMLDoc(w, r, filepath.Join(*goroot, "doc", "root.html"), "doc/root.html") + return + + case "/doc/root.html": + // hide landing page from its real name + http.Redirect(w, r, "/", http.StatusMovedPermanently) + return + } + + switch path.Ext(relpath) { + case ".html": + if strings.HasSuffix(relpath, "/index.html") { + // We'll show index.html for the directory. + // Use the dir/ version as canonical instead of dir/index.html. + http.Redirect(w, r, r.URL.Path[0:len(r.URL.Path)-len("index.html")], http.StatusMovedPermanently) + return + } + serveHTMLDoc(w, r, abspath, relpath) + return + + case ".go": + serveTextFile(w, r, abspath, relpath, "Source file") + return + } + + dir, err := fs.Lstat(abspath) + if err != nil { + log.Print(err) + serveError(w, r, relpath, err) + return + } + + if dir != nil && dir.IsDirectory() { + if redirect(w, r) { + return + } + if index := filepath.Join(abspath, "index.html"); isTextFile(index) { + serveHTMLDoc(w, r, index, relativeURL(index)) + return + } + serveDirectory(w, r, abspath, relpath) + return + } + + if isTextFile(abspath) { + serveTextFile(w, r, abspath, relpath, "Text file") + return + } + + fileServer.ServeHTTP(w, r) +} + +// ---------------------------------------------------------------------------- +// Packages + +// Fake package file and name for commands. Contains the command documentation. +const fakePkgFile = "doc.go" +const fakePkgName = "documentation" + +// Fake relative package path for built-ins. Documentation for all globals +// (not just exported ones) will be shown for packages in this directory. +const builtinPkgPath = "builtin/" + +type PageInfoMode uint + +const ( + exportsOnly PageInfoMode = 1 << iota // only keep exported stuff + genDoc // generate documentation +) + +type PageInfo struct { + Dirname string // directory containing the package + PList []string // list of package names found + FSet *token.FileSet // corresponding file set + PAst *ast.File // nil if no single AST with package exports + PDoc *doc.PackageDoc // nil if no single package documentation + Dirs *DirList // nil if no directory information + DirTime int64 // directory time stamp in seconds since epoch + IsPkg bool // false if this is not documenting a real package + Err os.Error // directory read error or nil +} + +func (info *PageInfo) IsEmpty() bool { + return info.Err != nil || info.PAst == nil && info.PDoc == nil && info.Dirs == nil +} + +type httpHandler struct { + pattern string // url pattern; e.g. "/pkg/" + fsRoot string // file system root to which the pattern is mapped + isPkg bool // true if this handler serves real package documentation (as opposed to command documentation) +} + +// getPageInfo returns the PageInfo for a package directory abspath. If the +// parameter genAST is set, an AST containing only the package exports is +// computed (PageInfo.PAst), otherwise package documentation (PageInfo.Doc) +// is extracted from the AST. If there is no corresponding package in the +// directory, PageInfo.PAst and PageInfo.PDoc are nil. If there are no sub- +// directories, PageInfo.Dirs is nil. If a directory read error occurred, +// PageInfo.Err is set to the respective error but the error is not logged. +// +func (h *httpHandler) getPageInfo(abspath, relpath, pkgname string, mode PageInfoMode) PageInfo { + // filter function to select the desired .go files + filter := func(d FileInfo) bool { + // If we are looking at cmd documentation, only accept + // the special fakePkgFile containing the documentation. + return isPkgFile(d) && (h.isPkg || d.Name() == fakePkgFile) + } + + // get package ASTs + fset := token.NewFileSet() + pkgs, err := parseDir(fset, abspath, filter) + if err != nil && pkgs == nil { + // only report directory read errors, ignore parse errors + // (may be able to extract partial package information) + return PageInfo{Dirname: abspath, Err: err} + } + + // select package + var pkg *ast.Package // selected package + var plist []string // list of other package (names), if any + if len(pkgs) == 1 { + // Exactly one package - select it. + for _, p := range pkgs { + pkg = p + } + + } else if len(pkgs) > 1 { + // Multiple packages - select the best matching package: The + // 1st choice is the package with pkgname, the 2nd choice is + // the package with dirname, and the 3rd choice is a package + // that is not called "main" if there is exactly one such + // package. Otherwise, don't select a package. + dirpath, dirname := filepath.Split(abspath) + + // If the dirname is "go" we might be in a sub-directory for + // .go files - use the outer directory name instead for better + // results. + if dirname == "go" { + _, dirname = filepath.Split(filepath.Clean(dirpath)) + } + + var choice3 *ast.Package + loop: + for _, p := range pkgs { + switch { + case p.Name == pkgname: + pkg = p + break loop // 1st choice; we are done + case p.Name == dirname: + pkg = p // 2nd choice + case p.Name != "main": + choice3 = p + } + } + if pkg == nil && len(pkgs) == 2 { + pkg = choice3 + } + + // Compute the list of other packages + // (excluding the selected package, if any). + plist = make([]string, len(pkgs)) + i := 0 + for name := range pkgs { + if pkg == nil || name != pkg.Name { + plist[i] = name + i++ + } + } + plist = plist[0:i] + } + + // compute package documentation + var past *ast.File + var pdoc *doc.PackageDoc + if pkg != nil { + if mode&exportsOnly != 0 { + ast.PackageExports(pkg) + } + if mode&genDoc != 0 { + pdoc = doc.NewPackageDoc(pkg, path.Clean(relpath)) // no trailing '/' in importpath + } else { + past = ast.MergePackageFiles(pkg, ast.FilterUnassociatedComments) + } + } + + // get directory information + var dir *Directory + var timestamp int64 + if tree, ts := fsTree.get(); tree != nil && tree.(*Directory) != nil { + // directory tree is present; lookup respective directory + // (may still fail if the file system was updated and the + // new directory tree has not yet been computed) + dir = tree.(*Directory).lookup(abspath) + timestamp = ts + } + if dir == nil { + // the path may refer to a user-specified file system mapped + // via fsMap; lookup that mapping and corresponding RWValue + // if any + var v *RWValue + fsMap.Iterate(func(path string, value *RWValue) bool { + if isParentOf(path, abspath) { + // mapping found + v = value + return false + } + return true + }) + if v != nil { + // found a RWValue associated with a user-specified file + // system; a non-nil RWValue stores a (possibly out-of-date) + // directory tree for that file system + if tree, ts := v.get(); tree != nil && tree.(*Directory) != nil { + dir = tree.(*Directory).lookup(abspath) + timestamp = ts + } + } + } + if dir == nil { + // no directory tree present (too early after startup or + // command-line mode); compute one level for this page + // note: cannot use path filter here because in general + // it doesn't contain the fsTree path + dir = newDirectory(abspath, nil, 1) + timestamp = time.Seconds() + } + + return PageInfo{abspath, plist, fset, past, pdoc, dir.listing(true), timestamp, h.isPkg, nil} +} + +func (h *httpHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) { + if redirect(w, r) { + return + } + + relpath := r.URL.Path[len(h.pattern):] + abspath := absolutePath(relpath, h.fsRoot) + var mode PageInfoMode + if relpath != builtinPkgPath { + mode = exportsOnly + } + if r.FormValue("m") != "src" { + mode |= genDoc + } + info := h.getPageInfo(abspath, relpath, r.FormValue("p"), mode) + if info.Err != nil { + log.Print(info.Err) + serveError(w, r, relpath, info.Err) + return + } + + if r.FormValue("f") == "text" { + contents := applyTemplate(packageText, "packageText", info) + serveText(w, contents) + return + } + + var title, subtitle string + switch { + case info.PAst != nil: + title = "Package " + info.PAst.Name.Name + case info.PDoc != nil: + switch { + case info.IsPkg: + title = "Package " + info.PDoc.PackageName + case info.PDoc.PackageName == fakePkgName: + // assume that the directory name is the command name + _, pkgname := path.Split(path.Clean(relpath)) + title = "Command " + pkgname + default: + title = "Command " + info.PDoc.PackageName + } + default: + title = "Directory " + relativeURL(info.Dirname) + if *showTimestamps { + subtitle = "Last update: " + time.SecondsToLocalTime(info.DirTime).String() + } + } + + contents := applyTemplate(packageHTML, "packageHTML", info) + servePage(w, title, subtitle, "", contents) +} + +// ---------------------------------------------------------------------------- +// Search + +var searchIndex RWValue + +type SearchResult struct { + Query string + Alert string // error or warning message + + // identifier matches + Hit *LookupResult // identifier matches of Query + Alt *AltWords // alternative identifiers to look for + + // textual matches + Found int // number of textual occurrences found + Textual []FileLines // textual matches of Query + Complete bool // true if all textual occurrences of Query are reported +} + +func lookup(query string) (result SearchResult) { + result.Query = query + + index, timestamp := searchIndex.get() + if index != nil { + index := index.(*Index) + + // identifier search + var err os.Error + result.Hit, result.Alt, err = index.Lookup(query) + if err != nil && *maxResults <= 0 { + // ignore the error if full text search is enabled + // since the query may be a valid regular expression + result.Alert = "Error in query string: " + err.String() + return + } + + // full text search + if *maxResults > 0 && query != "" { + rx, err := regexp.Compile(query) + if err != nil { + result.Alert = "Error in query regular expression: " + err.String() + return + } + // If we get maxResults+1 results we know that there are more than + // maxResults results and thus the result may be incomplete (to be + // precise, we should remove one result from the result set, but + // nobody is going to count the results on the result page). + result.Found, result.Textual = index.LookupRegexp(rx, *maxResults+1) + result.Complete = result.Found <= *maxResults + if !result.Complete { + result.Found-- // since we looked for maxResults+1 + } + } + } + + // is the result accurate? + if *indexEnabled { + if _, ts := fsModified.get(); timestamp < ts { + // The index is older than the latest file system change + // under godoc's observation. Indexing may be in progress + // or start shortly (see indexer()). + result.Alert = "Indexing in progress: result may be inaccurate" + } + } else { + result.Alert = "Search index disabled: no results available" + } + + return +} + +func search(w http.ResponseWriter, r *http.Request) { + query := strings.TrimSpace(r.FormValue("q")) + result := lookup(query) + + if r.FormValue("f") == "text" { + contents := applyTemplate(searchText, "searchText", result) + serveText(w, contents) + return + } + + var title string + if result.Hit != nil || len(result.Textual) > 0 { + title = fmt.Sprintf(`Results for query %q`, query) + } else { + title = fmt.Sprintf(`No results found for query %q`, query) + } + + contents := applyTemplate(searchHTML, "searchHTML", result) + servePage(w, title, "", query, contents) +} + +// ---------------------------------------------------------------------------- +// Indexer + +// invalidateIndex should be called whenever any of the file systems +// under godoc's observation change so that the indexer is kicked on. +// +func invalidateIndex() { + fsModified.set(nil) +} + +// indexUpToDate() returns true if the search index is not older +// than any of the file systems under godoc's observation. +// +func indexUpToDate() bool { + _, fsTime := fsModified.get() + _, siTime := searchIndex.get() + return fsTime <= siTime +} + +// feedDirnames feeds the directory names of all directories +// under the file system given by root to channel c. +// +func feedDirnames(root *RWValue, c chan<- string) { + if dir, _ := root.get(); dir != nil { + for d := range dir.(*Directory).iter(false) { + c <- d.Path + } + } +} + +// fsDirnames() returns a channel sending all directory names +// of all the file systems under godoc's observation. +// +func fsDirnames() <-chan string { + c := make(chan string, 256) // asynchronous for fewer context switches + go func() { + feedDirnames(&fsTree, c) + fsMap.Iterate(func(_ string, root *RWValue) bool { + feedDirnames(root, c) + return true + }) + close(c) + }() + return c +} + +func indexer() { + for { + if !indexUpToDate() { + // index possibly out of date - make a new one + if *verbose { + log.Printf("updating index...") + } + start := time.Nanoseconds() + index := NewIndex(fsDirnames(), *maxResults > 0) + stop := time.Nanoseconds() + searchIndex.set(index) + if *verbose { + secs := float64((stop-start)/1e6) / 1e3 + stats := index.Stats() + log.Printf("index updated (%gs, %d bytes of source, %d files, %d lines, %d unique words, %d spots)", + secs, stats.Bytes, stats.Files, stats.Lines, stats.Words, stats.Spots) + } + log.Printf("before GC: bytes = %d footprint = %d", runtime.MemStats.HeapAlloc, runtime.MemStats.Sys) + runtime.GC() + log.Printf("after GC: bytes = %d footprint = %d", runtime.MemStats.HeapAlloc, runtime.MemStats.Sys) + } + var delay int64 = 60 * 1e9 // by default, try every 60s + if *testDir != "" { + // in test mode, try once a second for fast startup + delay = 1 * 1e9 + } + time.Sleep(delay) + } +} diff --git a/src/cmd/godoc/httpzip.go b/src/cmd/godoc/httpzip.go new file mode 100644 index 000000000..cb8322ee4 --- /dev/null +++ b/src/cmd/godoc/httpzip.go @@ -0,0 +1,181 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// This file provides an implementation of the http.FileSystem +// interface based on the contents of a .zip file. +// +// Assumptions: +// +// - The file paths stored in the zip file must use a slash ('/') as path +// separator; and they must be relative (i.e., they must not start with +// a '/' - this is usually the case if the file was created w/o special +// options). +// - The zip file system treats the file paths found in the zip internally +// like absolute paths w/o a leading '/'; i.e., the paths are considered +// relative to the root of the file system. +// - All path arguments to file system methods are considered relative to +// the root specified with NewHttpZipFS (even if the paths start with a '/'). + +// TODO(gri) Should define a commonly used FileSystem API that is the same +// for http and godoc. Then we only need one zip-file based file +// system implementation. + +package main + +import ( + "archive/zip" + "fmt" + "http" + "io" + "os" + "path" + "sort" + "strings" +) + +// We cannot import syscall on app engine. +// TODO(gri) Once we have a truly abstract FileInfo implementation +// this won't be needed anymore. +const ( + S_IFDIR = 0x4000 // == syscall.S_IFDIR + S_IFREG = 0x8000 // == syscall.S_IFREG +) + +// httpZipFile is the zip-file based implementation of http.File +type httpZipFile struct { + path string // absolute path within zip FS without leading '/' + info os.FileInfo + io.ReadCloser // nil for directory + list zipList +} + +func (f *httpZipFile) Close() os.Error { + if f.info.IsRegular() { + return f.ReadCloser.Close() + } + f.list = nil + return nil +} + +func (f *httpZipFile) Stat() (*os.FileInfo, os.Error) { + return &f.info, nil +} + +func (f *httpZipFile) Readdir(count int) ([]os.FileInfo, os.Error) { + var list []os.FileInfo + dirname := f.path + "/" + prevname := "" + for i, e := range f.list { + if count == 0 { + f.list = f.list[i:] + break + } + if !strings.HasPrefix(e.Name, dirname) { + f.list = nil + break // not in the same directory anymore + } + name := e.Name[len(dirname):] // local name + var mode uint32 + var size, mtime_ns int64 + if i := strings.IndexRune(name, '/'); i >= 0 { + // We infer directories from files in subdirectories. + // If we have x/y, return a directory entry for x. + name = name[0:i] // keep local directory name only + mode = S_IFDIR + // no size or mtime_ns for directories + } else { + mode = S_IFREG + size = int64(e.UncompressedSize) + mtime_ns = e.Mtime_ns() + } + // If we have x/y and x/z, don't return two directory entries for x. + // TODO(gri): It should be possible to do this more efficiently + // by determining the (fs.list) range of local directory entries + // (via two binary searches). + if name != prevname { + list = append(list, os.FileInfo{ + Name: name, + Mode: mode, + Size: size, + Mtime_ns: mtime_ns, + }) + prevname = name + count-- + } + } + + if count >= 0 && len(list) == 0 { + return nil, os.EOF + } + + return list, nil +} + +func (f *httpZipFile) Seek(offset int64, whence int) (int64, os.Error) { + return 0, fmt.Errorf("Seek not implemented for zip file entry: %s", f.info.Name) +} + +// httpZipFS is the zip-file based implementation of http.FileSystem +type httpZipFS struct { + *zip.ReadCloser + list zipList + root string +} + +func (fs *httpZipFS) Open(name string) (http.File, os.Error) { + // fs.root does not start with '/'. + path := path.Join(fs.root, name) // path is clean + index, exact := fs.list.lookup(path) + if index < 0 || !strings.HasPrefix(path, fs.root) { + // file not found or not under root + return nil, fmt.Errorf("file not found: %s", name) + } + + if exact { + // exact match found - must be a file + f := fs.list[index] + rc, err := f.Open() + if err != nil { + return nil, err + } + return &httpZipFile{ + path, + os.FileInfo{ + Name: name, + Mode: S_IFREG, + Size: int64(f.UncompressedSize), + Mtime_ns: f.Mtime_ns(), + }, + rc, + nil, + }, nil + } + + // not an exact match - must be a directory + return &httpZipFile{ + path, + os.FileInfo{ + Name: name, + Mode: S_IFDIR, + // no size or mtime_ns for directories + }, + nil, + fs.list[index:], + }, nil +} + +func (fs *httpZipFS) Close() os.Error { + fs.list = nil + return fs.ReadCloser.Close() +} + +// NewHttpZipFS creates a new http.FileSystem based on the contents of +// the zip file rc restricted to the directory tree specified by root; +// root must be an absolute path. +func NewHttpZipFS(rc *zip.ReadCloser, root string) http.FileSystem { + list := make(zipList, len(rc.File)) + copy(list, rc.File) // sort a copy of rc.File + sort.Sort(list) + return &httpZipFS{rc, list, zipPath(root)} +} diff --git a/src/cmd/godoc/index.go b/src/cmd/godoc/index.go new file mode 100644 index 000000000..9b4f31514 --- /dev/null +++ b/src/cmd/godoc/index.go @@ -0,0 +1,986 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// This file contains the infrastructure to create an +// identifier and full-text index for a set of Go files. +// +// Algorithm for identifier index: +// - traverse all .go files of the file tree specified by root +// - for each word (identifier) encountered, collect all occurrences (spots) +// into a list; this produces a list of spots for each word +// - reduce the lists: from a list of spots to a list of FileRuns, +// and from a list of FileRuns into a list of PakRuns +// - make a HitList from the PakRuns +// +// Details: +// - keep two lists per word: one containing package-level declarations +// that have snippets, and one containing all other spots +// - keep the snippets in a separate table indexed by snippet index +// and store the snippet index in place of the line number in a SpotInfo +// (the line number for spots with snippets is stored in the snippet) +// - at the end, create lists of alternative spellings for a given +// word +// +// Algorithm for full text index: +// - concatenate all source code in a byte buffer (in memory) +// - add the files to a file set in lockstep as they are added to the byte +// buffer such that a byte buffer offset corresponds to the Pos value for +// that file location +// - create a suffix array from the concatenated sources +// +// String lookup in full text index: +// - use the suffix array to lookup a string's offsets - the offsets +// correspond to the Pos values relative to the file set +// - translate the Pos values back into file and line information and +// sort the result + +package main + +import ( + "bytes" + "container/vector" + "go/ast" + "go/parser" + "go/token" + "go/scanner" + "index/suffixarray" + "os" + "path/filepath" + "regexp" + "sort" + "strings" +) + +// ---------------------------------------------------------------------------- +// RunList + +// A RunList is a vector of entries that can be sorted according to some +// criteria. A RunList may be compressed by grouping "runs" of entries +// which are equal (according to the sort critera) into a new RunList of +// runs. For instance, a RunList containing pairs (x, y) may be compressed +// into a RunList containing pair runs (x, {y}) where each run consists of +// a list of y's with the same x. +type RunList struct { + vector.Vector + less func(x, y interface{}) bool +} + +func (h *RunList) Less(i, j int) bool { return h.less(h.At(i), h.At(j)) } + +func (h *RunList) sort(less func(x, y interface{}) bool) { + h.less = less + sort.Sort(h) +} + +// Compress entries which are the same according to a sort criteria +// (specified by less) into "runs". +func (h *RunList) reduce(less func(x, y interface{}) bool, newRun func(h *RunList, i, j int) interface{}) *RunList { + // create runs of entries with equal values + h.sort(less) + + // for each run, make a new run object and collect them in a new RunList + var hh RunList + i := 0 + for j := 0; j < h.Len(); j++ { + if less(h.At(i), h.At(j)) { + hh.Push(newRun(h, i, j)) + i = j // start a new run + } + } + // add final run, if any + if i < h.Len() { + hh.Push(newRun(h, i, h.Len())) + } + + return &hh +} + +// ---------------------------------------------------------------------------- +// SpotInfo + +// A SpotInfo value describes a particular identifier spot in a given file; +// It encodes three values: the SpotKind (declaration or use), a line or +// snippet index "lori", and whether it's a line or index. +// +// The following encoding is used: +// +// bits 32 4 1 0 +// value [lori|kind|isIndex] +// +type SpotInfo uint32 + +// SpotKind describes whether an identifier is declared (and what kind of +// declaration) or used. +type SpotKind uint32 + +const ( + PackageClause SpotKind = iota + ImportDecl + ConstDecl + TypeDecl + VarDecl + FuncDecl + MethodDecl + Use + nKinds +) + +func init() { + // sanity check: if nKinds is too large, the SpotInfo + // accessor functions may need to be updated + if nKinds > 8 { + panic("nKinds > 8") + } +} + +// makeSpotInfo makes a SpotInfo. +func makeSpotInfo(kind SpotKind, lori int, isIndex bool) SpotInfo { + // encode lori: bits [4..32) + x := SpotInfo(lori) << 4 + if int(x>>4) != lori { + // lori value doesn't fit - since snippet indices are + // most certainly always smaller then 1<<28, this can + // only happen for line numbers; give it no line number (= 0) + x = 0 + } + // encode kind: bits [1..4) + x |= SpotInfo(kind) << 1 + // encode isIndex: bit 0 + if isIndex { + x |= 1 + } + return x +} + +func (x SpotInfo) Kind() SpotKind { return SpotKind(x >> 1 & 7) } +func (x SpotInfo) Lori() int { return int(x >> 4) } +func (x SpotInfo) IsIndex() bool { return x&1 != 0 } + +// ---------------------------------------------------------------------------- +// KindRun + +// Debugging support. Disable to see multiple entries per line. +const removeDuplicates = true + +// A KindRun is a run of SpotInfos of the same kind in a given file. +type KindRun struct { + Kind SpotKind + Infos []SpotInfo +} + +// KindRuns are sorted by line number or index. Since the isIndex bit +// is always the same for all infos in one list we can compare lori's. +func (f *KindRun) Len() int { return len(f.Infos) } +func (f *KindRun) Less(i, j int) bool { return f.Infos[i].Lori() < f.Infos[j].Lori() } +func (f *KindRun) Swap(i, j int) { f.Infos[i], f.Infos[j] = f.Infos[j], f.Infos[i] } + +// FileRun contents are sorted by Kind for the reduction into KindRuns. +func lessKind(x, y interface{}) bool { return x.(SpotInfo).Kind() < y.(SpotInfo).Kind() } + +// newKindRun allocates a new KindRun from the SpotInfo run [i, j) in h. +func newKindRun(h *RunList, i, j int) interface{} { + kind := h.At(i).(SpotInfo).Kind() + infos := make([]SpotInfo, j-i) + k := 0 + for ; i < j; i++ { + infos[k] = h.At(i).(SpotInfo) + k++ + } + run := &KindRun{kind, infos} + + // Spots were sorted by file and kind to create this run. + // Within this run, sort them by line number or index. + sort.Sort(run) + + if removeDuplicates { + // Since both the lori and kind field must be + // same for duplicates, and since the isIndex + // bit is always the same for all infos in one + // list we can simply compare the entire info. + k := 0 + var prev SpotInfo + for i, x := range infos { + if x != prev || i == 0 { + infos[k] = x + k++ + prev = x + } + } + run.Infos = infos[0:k] + } + + return run +} + +// ---------------------------------------------------------------------------- +// FileRun + +// A Pak describes a Go package. +type Pak struct { + Path string // path of directory containing the package + Name string // package name as declared by package clause +} + +// Paks are sorted by name (primary key) and by import path (secondary key). +func (p *Pak) less(q *Pak) bool { + return p.Name < q.Name || p.Name == q.Name && p.Path < q.Path +} + +// A File describes a Go file. +type File struct { + Path string // complete file name + Pak Pak // the package to which the file belongs +} + +// A Spot describes a single occurrence of a word. +type Spot struct { + File *File + Info SpotInfo +} + +// A FileRun is a list of KindRuns belonging to the same file. +type FileRun struct { + File *File + Groups []*KindRun +} + +// Spots are sorted by path for the reduction into FileRuns. +func lessSpot(x, y interface{}) bool { return x.(Spot).File.Path < y.(Spot).File.Path } + +// newFileRun allocates a new FileRun from the Spot run [i, j) in h. +func newFileRun(h0 *RunList, i, j int) interface{} { + file := h0.At(i).(Spot).File + + // reduce the list of Spots into a list of KindRuns + var h1 RunList + h1.Vector.Resize(j-i, 0) + k := 0 + for ; i < j; i++ { + h1.Set(k, h0.At(i).(Spot).Info) + k++ + } + h2 := h1.reduce(lessKind, newKindRun) + + // create the FileRun + groups := make([]*KindRun, h2.Len()) + for i := 0; i < h2.Len(); i++ { + groups[i] = h2.At(i).(*KindRun) + } + return &FileRun{file, groups} +} + +// ---------------------------------------------------------------------------- +// PakRun + +// A PakRun describes a run of *FileRuns of a package. +type PakRun struct { + Pak Pak + Files []*FileRun +} + +// Sorting support for files within a PakRun. +func (p *PakRun) Len() int { return len(p.Files) } +func (p *PakRun) Less(i, j int) bool { return p.Files[i].File.Path < p.Files[j].File.Path } +func (p *PakRun) Swap(i, j int) { p.Files[i], p.Files[j] = p.Files[j], p.Files[i] } + +// FileRuns are sorted by package for the reduction into PakRuns. +func lessFileRun(x, y interface{}) bool { + return x.(*FileRun).File.Pak.less(&y.(*FileRun).File.Pak) +} + +// newPakRun allocates a new PakRun from the *FileRun run [i, j) in h. +func newPakRun(h *RunList, i, j int) interface{} { + pak := h.At(i).(*FileRun).File.Pak + files := make([]*FileRun, j-i) + k := 0 + for ; i < j; i++ { + files[k] = h.At(i).(*FileRun) + k++ + } + run := &PakRun{pak, files} + sort.Sort(run) // files were sorted by package; sort them by file now + return run +} + +// ---------------------------------------------------------------------------- +// HitList + +// A HitList describes a list of PakRuns. +type HitList []*PakRun + +// PakRuns are sorted by package. +func lessPakRun(x, y interface{}) bool { return x.(*PakRun).Pak.less(&y.(*PakRun).Pak) } + +func reduce(h0 *RunList) HitList { + // reduce a list of Spots into a list of FileRuns + h1 := h0.reduce(lessSpot, newFileRun) + // reduce a list of FileRuns into a list of PakRuns + h2 := h1.reduce(lessFileRun, newPakRun) + // sort the list of PakRuns by package + h2.sort(lessPakRun) + // create a HitList + h := make(HitList, h2.Len()) + for i := 0; i < h2.Len(); i++ { + h[i] = h2.At(i).(*PakRun) + } + return h +} + +func (h HitList) filter(pakname string) HitList { + // determine number of matching packages (most of the time just one) + n := 0 + for _, p := range h { + if p.Pak.Name == pakname { + n++ + } + } + // create filtered HitList + hh := make(HitList, n) + i := 0 + for _, p := range h { + if p.Pak.Name == pakname { + hh[i] = p + i++ + } + } + return hh +} + +// ---------------------------------------------------------------------------- +// AltWords + +type wordPair struct { + canon string // canonical word spelling (all lowercase) + alt string // alternative spelling +} + +// An AltWords describes a list of alternative spellings for a +// canonical (all lowercase) spelling of a word. +type AltWords struct { + Canon string // canonical word spelling (all lowercase) + Alts []string // alternative spelling for the same word +} + +// wordPairs are sorted by their canonical spelling. +func lessWordPair(x, y interface{}) bool { return x.(*wordPair).canon < y.(*wordPair).canon } + +// newAltWords allocates a new AltWords from the *wordPair run [i, j) in h. +func newAltWords(h *RunList, i, j int) interface{} { + canon := h.At(i).(*wordPair).canon + alts := make([]string, j-i) + k := 0 + for ; i < j; i++ { + alts[k] = h.At(i).(*wordPair).alt + k++ + } + return &AltWords{canon, alts} +} + +func (a *AltWords) filter(s string) *AltWords { + if len(a.Alts) == 1 && a.Alts[0] == s { + // there are no different alternatives + return nil + } + + // make a new AltWords with the current spelling removed + alts := make([]string, len(a.Alts)) + i := 0 + for _, w := range a.Alts { + if w != s { + alts[i] = w + i++ + } + } + return &AltWords{a.Canon, alts[0:i]} +} + +// ---------------------------------------------------------------------------- +// Indexer + +// Adjust these flags as seems best. +const includeMainPackages = true +const includeTestFiles = true + +type IndexResult struct { + Decls RunList // package-level declarations (with snippets) + Others RunList // all other occurrences +} + +// Statistics provides statistics information for an index. +type Statistics struct { + Bytes int // total size of indexed source files + Files int // number of indexed source files + Lines int // number of lines (all files) + Words int // number of different identifiers + Spots int // number of identifier occurrences +} + +// An Indexer maintains the data structures and provides the machinery +// for indexing .go files under a file tree. It implements the path.Visitor +// interface for walking file trees, and the ast.Visitor interface for +// walking Go ASTs. +type Indexer struct { + fset *token.FileSet // file set for all indexed files + sources bytes.Buffer // concatenated sources + words map[string]*IndexResult // RunLists of Spots + snippets vector.Vector // vector of *Snippets, indexed by snippet indices + current *token.File // last file added to file set + file *File // AST for current file + decl ast.Decl // AST for current decl + stats Statistics +} + +func (x *Indexer) addSnippet(s *Snippet) int { + index := x.snippets.Len() + x.snippets.Push(s) + return index +} + +func (x *Indexer) visitComment(c *ast.CommentGroup) { + if c != nil { + ast.Walk(x, c) + } +} + +func (x *Indexer) visitIdent(kind SpotKind, id *ast.Ident) { + if id != nil { + lists, found := x.words[id.Name] + if !found { + lists = new(IndexResult) + x.words[id.Name] = lists + } + + if kind == Use || x.decl == nil { + // not a declaration or no snippet required + info := makeSpotInfo(kind, x.current.Line(id.Pos()), false) + lists.Others.Push(Spot{x.file, info}) + } else { + // a declaration with snippet + index := x.addSnippet(NewSnippet(x.fset, x.decl, id)) + info := makeSpotInfo(kind, index, true) + lists.Decls.Push(Spot{x.file, info}) + } + + x.stats.Spots++ + } +} + +func (x *Indexer) visitSpec(spec ast.Spec, isVarDecl bool) { + switch n := spec.(type) { + case *ast.ImportSpec: + x.visitComment(n.Doc) + x.visitIdent(ImportDecl, n.Name) + ast.Walk(x, n.Path) + x.visitComment(n.Comment) + + case *ast.ValueSpec: + x.visitComment(n.Doc) + kind := ConstDecl + if isVarDecl { + kind = VarDecl + } + for _, n := range n.Names { + x.visitIdent(kind, n) + } + ast.Walk(x, n.Type) + for _, v := range n.Values { + ast.Walk(x, v) + } + x.visitComment(n.Comment) + + case *ast.TypeSpec: + x.visitComment(n.Doc) + x.visitIdent(TypeDecl, n.Name) + ast.Walk(x, n.Type) + x.visitComment(n.Comment) + } +} + +func (x *Indexer) Visit(node ast.Node) ast.Visitor { + // TODO(gri): methods in interface types are categorized as VarDecl + switch n := node.(type) { + case nil: + return nil + + case *ast.Ident: + x.visitIdent(Use, n) + + case *ast.Field: + x.decl = nil // no snippets for fields + x.visitComment(n.Doc) + for _, m := range n.Names { + x.visitIdent(VarDecl, m) + } + ast.Walk(x, n.Type) + ast.Walk(x, n.Tag) + x.visitComment(n.Comment) + + case *ast.DeclStmt: + if decl, ok := n.Decl.(*ast.GenDecl); ok { + // local declarations can only be *ast.GenDecls + x.decl = nil // no snippets for local declarations + x.visitComment(decl.Doc) + for _, s := range decl.Specs { + x.visitSpec(s, decl.Tok == token.VAR) + } + } else { + // handle error case gracefully + ast.Walk(x, n.Decl) + } + + case *ast.GenDecl: + x.decl = n + x.visitComment(n.Doc) + for _, s := range n.Specs { + x.visitSpec(s, n.Tok == token.VAR) + } + + case *ast.FuncDecl: + x.visitComment(n.Doc) + kind := FuncDecl + if n.Recv != nil { + kind = MethodDecl + ast.Walk(x, n.Recv) + } + x.decl = n + x.visitIdent(kind, n.Name) + ast.Walk(x, n.Type) + if n.Body != nil { + ast.Walk(x, n.Body) + } + + case *ast.File: + x.visitComment(n.Doc) + x.decl = nil + x.visitIdent(PackageClause, n.Name) + for _, d := range n.Decls { + ast.Walk(x, d) + } + // don't visit package level comments for now + // to avoid duplicate visiting from individual + // nodes + + default: + return x + } + + return nil +} + +func pkgName(filename string) string { + // use a new file set each time in order to not pollute the indexer's + // file set (which must stay in sync with the concatenated source code) + file, err := parser.ParseFile(token.NewFileSet(), filename, nil, parser.PackageClauseOnly) + if err != nil || file == nil { + return "" + } + return file.Name.Name +} + +// addFile adds a file to the index if possible and returns the file set file +// and the file's AST if it was successfully parsed as a Go file. If addFile +// failed (that is, if the file was not added), it returns file == nil. +func (x *Indexer) addFile(filename string, goFile bool) (file *token.File, ast *ast.File) { + // open file + f, err := fs.Open(filename) + if err != nil { + return + } + defer f.Close() + + // The file set's base offset and x.sources size must be in lock-step; + // this permits the direct mapping of suffix array lookup results to + // to corresponding Pos values. + // + // When a file is added to the file set, its offset base increases by + // the size of the file + 1; and the initial base offset is 1. Add an + // extra byte to the sources here. + x.sources.WriteByte(0) + + // If the sources length doesn't match the file set base at this point + // the file set implementation changed or we have another error. + base := x.fset.Base() + if x.sources.Len() != base { + panic("internal error - file base incorrect") + } + + // append file contents (src) to x.sources + if _, err := x.sources.ReadFrom(f); err == nil { + src := x.sources.Bytes()[base:] + + if goFile { + // parse the file and in the process add it to the file set + if ast, err = parser.ParseFile(x.fset, filename, src, parser.ParseComments); err == nil { + file = x.fset.File(ast.Pos()) // ast.Pos() is inside the file + return + } + // file has parse errors, and the AST may be incorrect - + // set lines information explicitly and index as ordinary + // text file (cannot fall through to the text case below + // because the file has already been added to the file set + // by the parser) + file = x.fset.File(token.Pos(base)) // token.Pos(base) is inside the file + file.SetLinesForContent(src) + ast = nil + return + } + + if isText(src) { + // only add the file to the file set (for the full text index) + file = x.fset.AddFile(filename, x.fset.Base(), len(src)) + file.SetLinesForContent(src) + return + } + } + + // discard possibly added data + x.sources.Truncate(base - 1) // -1 to remove added byte 0 since no file was added + return +} + +// Design note: Using an explicit white list of permitted files for indexing +// makes sure that the important files are included and massively reduces the +// number of files to index. The advantage over a blacklist is that unexpected +// (non-blacklisted) files won't suddenly explode the index. +// +// TODO(gri): We may want to make this list customizable, perhaps via a flag. + +// Files are whitelisted if they have a file name or extension +// present as key in whitelisted. +var whitelisted = map[string]bool{ + ".bash": true, + ".c": true, + ".css": true, + ".go": true, + ".goc": true, + ".h": true, + ".html": true, + ".js": true, + ".out": true, + ".py": true, + ".s": true, + ".sh": true, + ".txt": true, + ".xml": true, + "AUTHORS": true, + "CONTRIBUTORS": true, + "LICENSE": true, + "Makefile": true, + "PATENTS": true, + "README": true, +} + +// isWhitelisted returns true if a file is on the list +// of "permitted" files for indexing. The filename must +// be the directory-local name of the file. +func isWhitelisted(filename string) bool { + key := filepath.Ext(filename) + if key == "" { + // file has no extension - use entire filename + key = filename + } + return whitelisted[key] +} + +func (x *Indexer) visitFile(dirname string, f FileInfo, fulltextIndex bool) { + if !f.IsRegular() { + return + } + + filename := filepath.Join(dirname, f.Name()) + goFile := false + + switch { + case isGoFile(f): + if !includeTestFiles && (!isPkgFile(f) || strings.HasPrefix(filename, "test/")) { + return + } + if !includeMainPackages && pkgName(filename) == "main" { + return + } + goFile = true + + case !fulltextIndex || !isWhitelisted(f.Name()): + return + } + + file, fast := x.addFile(filename, goFile) + if file == nil { + return // addFile failed + } + + if fast != nil { + // we've got a Go file to index + x.current = file + dir, _ := filepath.Split(filename) + pak := Pak{dir, fast.Name.Name} + x.file = &File{filename, pak} + ast.Walk(x, fast) + } + + // update statistics + x.stats.Bytes += file.Size() + x.stats.Files++ + x.stats.Lines += file.LineCount() +} + +// ---------------------------------------------------------------------------- +// Index + +type LookupResult struct { + Decls HitList // package-level declarations (with snippets) + Others HitList // all other occurrences +} + +type Index struct { + fset *token.FileSet // file set used during indexing; nil if no textindex + suffixes *suffixarray.Index // suffixes for concatenated sources; nil if no textindex + words map[string]*LookupResult // maps words to hit lists + alts map[string]*AltWords // maps canonical(words) to lists of alternative spellings + snippets []*Snippet // all snippets, indexed by snippet index + stats Statistics +} + +func canonical(w string) string { return strings.ToLower(w) } + +// NewIndex creates a new index for the .go files +// in the directories given by dirnames. +// +func NewIndex(dirnames <-chan string, fulltextIndex bool) *Index { + var x Indexer + + // initialize Indexer + x.fset = token.NewFileSet() + x.words = make(map[string]*IndexResult) + + // index all files in the directories given by dirnames + for dirname := range dirnames { + list, err := fs.ReadDir(dirname) + if err != nil { + continue // ignore this directory + } + for _, f := range list { + if !f.IsDirectory() { + x.visitFile(dirname, f, fulltextIndex) + } + } + } + + if !fulltextIndex { + // the file set, the current file, and the sources are + // not needed after indexing if no text index is built - + // help GC and clear them + x.fset = nil + x.sources.Reset() + x.current = nil // contains reference to fset! + } + + // for each word, reduce the RunLists into a LookupResult; + // also collect the word with its canonical spelling in a + // word list for later computation of alternative spellings + words := make(map[string]*LookupResult) + var wlist RunList + for w, h := range x.words { + decls := reduce(&h.Decls) + others := reduce(&h.Others) + words[w] = &LookupResult{ + Decls: decls, + Others: others, + } + wlist.Push(&wordPair{canonical(w), w}) + } + x.stats.Words = len(words) + + // reduce the word list {canonical(w), w} into + // a list of AltWords runs {canonical(w), {w}} + alist := wlist.reduce(lessWordPair, newAltWords) + + // convert alist into a map of alternative spellings + alts := make(map[string]*AltWords) + for i := 0; i < alist.Len(); i++ { + a := alist.At(i).(*AltWords) + alts[a.Canon] = a + } + + // convert snippet vector into a list + snippets := make([]*Snippet, x.snippets.Len()) + for i := 0; i < x.snippets.Len(); i++ { + snippets[i] = x.snippets.At(i).(*Snippet) + } + + // create text index + var suffixes *suffixarray.Index + if fulltextIndex { + suffixes = suffixarray.New(x.sources.Bytes()) + } + + return &Index{x.fset, suffixes, words, alts, snippets, x.stats} +} + +// Stats() returns index statistics. +func (x *Index) Stats() Statistics { + return x.stats +} + +func (x *Index) LookupWord(w string) (match *LookupResult, alt *AltWords) { + match = x.words[w] + alt = x.alts[canonical(w)] + // remove current spelling from alternatives + // (if there is no match, the alternatives do + // not contain the current spelling) + if match != nil && alt != nil { + alt = alt.filter(w) + } + return +} + +func isIdentifier(s string) bool { + var S scanner.Scanner + fset := token.NewFileSet() + S.Init(fset.AddFile("", fset.Base(), len(s)), []byte(s), nil, 0) + if _, tok, _ := S.Scan(); tok == token.IDENT { + _, tok, _ := S.Scan() + return tok == token.EOF + } + return false +} + +// For a given query, which is either a single identifier or a qualified +// identifier, Lookup returns a LookupResult, and a list of alternative +// spellings, if any. If the query syntax is wrong, an error is reported. +func (x *Index) Lookup(query string) (match *LookupResult, alt *AltWords, err os.Error) { + ss := strings.Split(query, ".") + + // check query syntax + for _, s := range ss { + if !isIdentifier(s) { + err = os.NewError("all query parts must be identifiers") + return + } + } + + switch len(ss) { + case 1: + match, alt = x.LookupWord(ss[0]) + + case 2: + pakname := ss[0] + match, alt = x.LookupWord(ss[1]) + if match != nil { + // found a match - filter by package name + decls := match.Decls.filter(pakname) + others := match.Others.filter(pakname) + match = &LookupResult{decls, others} + } + + default: + err = os.NewError("query is not a (qualified) identifier") + } + + return +} + +func (x *Index) Snippet(i int) *Snippet { + // handle illegal snippet indices gracefully + if 0 <= i && i < len(x.snippets) { + return x.snippets[i] + } + return nil +} + +type positionList []struct { + filename string + line int +} + +func (list positionList) Len() int { return len(list) } +func (list positionList) Less(i, j int) bool { return list[i].filename < list[j].filename } +func (list positionList) Swap(i, j int) { list[i], list[j] = list[j], list[i] } + +// unique returns the list sorted and with duplicate entries removed +func unique(list []int) []int { + sort.Ints(list) + var last int + i := 0 + for _, x := range list { + if i == 0 || x != last { + last = x + list[i] = x + i++ + } + } + return list[0:i] +} + +// A FileLines value specifies a file and line numbers within that file. +type FileLines struct { + Filename string + Lines []int +} + +// LookupRegexp returns the number of matches and the matches where a regular +// expression r is found in the full text index. At most n matches are +// returned (thus found <= n). +// +func (x *Index) LookupRegexp(r *regexp.Regexp, n int) (found int, result []FileLines) { + if x.suffixes == nil || n <= 0 { + return + } + // n > 0 + + var list positionList + // FindAllIndex may returns matches that span across file boundaries. + // Such matches are unlikely, buf after eliminating them we may end up + // with fewer than n matches. If we don't have enough at the end, redo + // the search with an increased value n1, but only if FindAllIndex + // returned all the requested matches in the first place (if it + // returned fewer than that there cannot be more). + for n1 := n; found < n; n1 += n - found { + found = 0 + matches := x.suffixes.FindAllIndex(r, n1) + // compute files, exclude matches that span file boundaries, + // and map offsets to file-local offsets + list = make(positionList, len(matches)) + for _, m := range matches { + // by construction, an offset corresponds to the Pos value + // for the file set - use it to get the file and line + p := token.Pos(m[0]) + if file := x.fset.File(p); file != nil { + if base := file.Base(); base <= m[1] && m[1] <= base+file.Size() { + // match [m[0], m[1]) is within the file boundaries + list[found].filename = file.Name() + list[found].line = file.Line(p) + found++ + } + } + } + if found == n || len(matches) < n1 { + // found all matches or there's no chance to find more + break + } + } + list = list[0:found] + sort.Sort(list) // sort by filename + + // collect matches belonging to the same file + var last string + var lines []int + addLines := func() { + if len(lines) > 0 { + // remove duplicate lines + result = append(result, FileLines{last, unique(lines)}) + lines = nil + } + } + for _, m := range list { + if m.filename != last { + addLines() + last = m.filename + } + lines = append(lines, m.line) + } + addLines() + + return +} diff --git a/src/cmd/godoc/main.go b/src/cmd/godoc/main.go new file mode 100644 index 000000000..89b12b9ac --- /dev/null +++ b/src/cmd/godoc/main.go @@ -0,0 +1,423 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// godoc: Go Documentation Server + +// Web server tree: +// +// http://godoc/ main landing page +// http://godoc/doc/ serve from $GOROOT/doc - spec, mem, tutorial, etc. +// http://godoc/src/ serve files from $GOROOT/src; .go gets pretty-printed +// http://godoc/cmd/ serve documentation about commands +// http://godoc/pkg/ serve documentation about packages +// (idea is if you say import "compress/zlib", you go to +// http://godoc/pkg/compress/zlib) +// +// Command-line interface: +// +// godoc packagepath [name ...] +// +// godoc compress/zlib +// - prints doc for package compress/zlib +// godoc crypto/block Cipher NewCMAC +// - prints doc for Cipher and NewCMAC in package crypto/block + +package main + +import ( + "archive/zip" + "bytes" + _ "expvar" // to serve /debug/vars + "flag" + "fmt" + "go/ast" + "go/build" + "http" + _ "http/pprof" // to serve /debug/pprof/* + "io" + "log" + "os" + "path" + "path/filepath" + "regexp" + "runtime" + "strings" + "time" + "url" +) + +const defaultAddr = ":6060" // default webserver address + +var ( + // file system to serve + // (with e.g.: zip -r go.zip $GOROOT -i \*.go -i \*.html -i \*.css -i \*.js -i \*.txt -i \*.c -i \*.h -i \*.s -i \*.png -i \*.jpg -i \*.sh -i favicon.ico) + zipfile = flag.String("zip", "", "zip file providing the file system to serve; disabled if empty") + + // periodic sync + syncCmd = flag.String("sync", "", "sync command; disabled if empty") + syncMin = flag.Int("sync_minutes", 0, "sync interval in minutes; disabled if <= 0") + syncDelay delayTime // actual sync interval in minutes; usually syncDelay == syncMin, but syncDelay may back off exponentially + + // network + httpAddr = flag.String("http", "", "HTTP service address (e.g., '"+defaultAddr+"')") + serverAddr = flag.String("server", "", "webserver address for command line searches") + + // layout control + html = flag.Bool("html", false, "print HTML in command-line mode") + srcMode = flag.Bool("src", false, "print (exported) source in command-line mode") + + // command-line searches + query = flag.Bool("q", false, "arguments are considered search queries") +) + +func serveError(w http.ResponseWriter, r *http.Request, relpath string, err os.Error) { + contents := applyTemplate(errorHTML, "errorHTML", err) // err may contain an absolute path! + w.WriteHeader(http.StatusNotFound) + servePage(w, "File "+relpath, "", "", contents) +} + +func exec(rw http.ResponseWriter, args []string) (status int) { + r, w, err := os.Pipe() + if err != nil { + log.Printf("os.Pipe(): %v", err) + return 2 + } + + bin := args[0] + fds := []*os.File{nil, w, w} + if *verbose { + log.Printf("executing %v", args) + } + p, err := os.StartProcess(bin, args, &os.ProcAttr{Files: fds, Dir: *goroot}) + defer r.Close() + w.Close() + if err != nil { + log.Printf("os.StartProcess(%q): %v", bin, err) + return 2 + } + defer p.Release() + + var buf bytes.Buffer + io.Copy(&buf, r) + wait, err := p.Wait(0) + if err != nil { + os.Stderr.Write(buf.Bytes()) + log.Printf("os.Wait(%d, 0): %v", p.Pid, err) + return 2 + } + status = wait.ExitStatus() + if !wait.Exited() || status > 1 { + os.Stderr.Write(buf.Bytes()) + log.Printf("executing %v failed (exit status = %d)", args, status) + return + } + + if *verbose { + os.Stderr.Write(buf.Bytes()) + } + if rw != nil { + rw.Header().Set("Content-Type", "text/plain; charset=utf-8") + rw.Write(buf.Bytes()) + } + + return +} + +func dosync(w http.ResponseWriter, r *http.Request) { + args := []string{"/bin/sh", "-c", *syncCmd} + switch exec(w, args) { + case 0: + // sync succeeded and some files have changed; + // update package tree. + // TODO(gri): The directory tree may be temporarily out-of-sync. + // Consider keeping separate time stamps so the web- + // page can indicate this discrepancy. + initFSTree() + fallthrough + case 1: + // sync failed because no files changed; + // don't change the package tree + syncDelay.set(*syncMin) // revert to regular sync schedule + default: + // sync failed because of an error - back off exponentially, but try at least once a day + syncDelay.backoff(24 * 60) + } +} + +func usage() { + fmt.Fprintf(os.Stderr, + "usage: godoc package [name ...]\n"+ + " godoc -http="+defaultAddr+"\n") + flag.PrintDefaults() + os.Exit(2) +} + +func loggingHandler(h http.Handler) http.Handler { + return http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) { + log.Printf("%s\t%s", req.RemoteAddr, req.URL) + h.ServeHTTP(w, req) + }) +} + +func remoteSearch(query string) (res *http.Response, err os.Error) { + search := "/search?f=text&q=" + url.QueryEscape(query) + + // list of addresses to try + var addrs []string + if *serverAddr != "" { + // explicit server address - only try this one + addrs = []string{*serverAddr} + } else { + addrs = []string{ + defaultAddr, + "golang.org", + } + } + + // remote search + for _, addr := range addrs { + url := "http://" + addr + search + res, err = http.Get(url) + if err == nil && res.StatusCode == http.StatusOK { + break + } + } + + if err == nil && res.StatusCode != http.StatusOK { + err = os.NewError(res.Status) + } + + return +} + +// Does s look like a regular expression? +func isRegexp(s string) bool { + return strings.IndexAny(s, ".(|)*+?^$[]") >= 0 +} + +// Make a regular expression of the form +// names[0]|names[1]|...names[len(names)-1]. +// Returns nil if the regular expression is illegal. +func makeRx(names []string) (rx *regexp.Regexp) { + if len(names) > 0 { + s := "" + for i, name := range names { + if i > 0 { + s += "|" + } + if isRegexp(name) { + s += name + } else { + s += "^" + name + "$" // must match exactly + } + } + rx, _ = regexp.Compile(s) // rx is nil if there's a compilation error + } + return +} + +func main() { + flag.Usage = usage + flag.Parse() + + // Check usage: either server and no args, or command line and args + if (*httpAddr != "") != (flag.NArg() == 0) { + usage() + } + + if *tabwidth < 0 { + log.Fatalf("negative tabwidth %d", *tabwidth) + } + + // Determine file system to use. + // TODO(gri) - fs and fsHttp should really be the same. Try to unify. + // - fsHttp doesn't need to be set up in command-line mode, + // same is true for the http handlers in initHandlers. + if *zipfile == "" { + // use file system of underlying OS + *goroot = filepath.Clean(*goroot) // normalize path separator + fs = OS + fsHttp = http.Dir(*goroot) + } else { + // use file system specified via .zip file (path separator must be '/') + rc, err := zip.OpenReader(*zipfile) + if err != nil { + log.Fatalf("%s: %s\n", *zipfile, err) + } + *goroot = path.Join("/", *goroot) // fsHttp paths are relative to '/' + fs = NewZipFS(rc) + fsHttp = NewHttpZipFS(rc, *goroot) + } + + initHandlers() + readTemplates() + + if *httpAddr != "" { + // HTTP server mode. + var handler http.Handler = http.DefaultServeMux + if *verbose { + log.Printf("Go Documentation Server") + log.Printf("version = %s", runtime.Version()) + log.Printf("address = %s", *httpAddr) + log.Printf("goroot = %s", *goroot) + log.Printf("tabwidth = %d", *tabwidth) + switch { + case !*indexEnabled: + log.Print("search index disabled") + case *maxResults > 0: + log.Printf("full text index enabled (maxresults = %d)", *maxResults) + default: + log.Print("identifier search index enabled") + } + if !fsMap.IsEmpty() { + log.Print("user-defined mapping:") + fsMap.Fprint(os.Stderr) + } + handler = loggingHandler(handler) + } + + registerPublicHandlers(http.DefaultServeMux) + if *syncCmd != "" { + http.Handle("/debug/sync", http.HandlerFunc(dosync)) + } + + // Initialize default directory tree with corresponding timestamp. + // (Do it in a goroutine so that launch is quick.) + go initFSTree() + + // Initialize directory trees for user-defined file systems (-path flag). + initDirTrees() + + // Start sync goroutine, if enabled. + if *syncCmd != "" && *syncMin > 0 { + syncDelay.set(*syncMin) // initial sync delay + go func() { + for { + dosync(nil, nil) + delay, _ := syncDelay.get() + if *verbose { + log.Printf("next sync in %dmin", delay.(int)) + } + time.Sleep(int64(delay.(int)) * 60e9) + } + }() + } + + // Start indexing goroutine. + if *indexEnabled { + go indexer() + } + + // Start http server. + if err := http.ListenAndServe(*httpAddr, handler); err != nil { + log.Fatalf("ListenAndServe %s: %v", *httpAddr, err) + } + + return + } + + // Command line mode. + if *html { + packageText = packageHTML + searchText = packageHTML + } + + if *query { + // Command-line queries. + for i := 0; i < flag.NArg(); i++ { + res, err := remoteSearch(flag.Arg(i)) + if err != nil { + log.Fatalf("remoteSearch: %s", err) + } + io.Copy(os.Stdout, res.Body) + } + return + } + + // determine paths + path := flag.Arg(0) + if len(path) > 0 && path[0] == '.' { + // assume cwd; don't assume -goroot + cwd, _ := os.Getwd() // ignore errors + path = filepath.Join(cwd, path) + } + relpath := path + abspath := path + if t, pkg, err := build.FindTree(path); err == nil { + relpath = pkg + abspath = filepath.Join(t.SrcDir(), pkg) + } else if !filepath.IsAbs(path) { + abspath = absolutePath(path, pkgHandler.fsRoot) + } else { + relpath = relativeURL(path) + } + + var mode PageInfoMode + if *srcMode { + // only filter exports if we don't have explicit command-line filter arguments + if flag.NArg() == 1 { + mode |= exportsOnly + } + } else { + mode = exportsOnly | genDoc + } + // TODO(gri): Provide a mechanism (flag?) to select a package + // if there are multiple packages in a directory. + info := pkgHandler.getPageInfo(abspath, relpath, "", mode) + + if info.IsEmpty() { + // try again, this time assume it's a command + if !filepath.IsAbs(path) { + abspath = absolutePath(path, cmdHandler.fsRoot) + } + cmdInfo := cmdHandler.getPageInfo(abspath, relpath, "", mode) + // only use the cmdInfo if it actually contains a result + // (don't hide errors reported from looking up a package) + if !cmdInfo.IsEmpty() { + info = cmdInfo + } + } + if info.Err != nil { + log.Fatalf("%v", info.Err) + } + + // If we have more than one argument, use the remaining arguments for filtering + if flag.NArg() > 1 { + args := flag.Args()[1:] + rx := makeRx(args) + if rx == nil { + log.Fatalf("illegal regular expression from %v", args) + } + + filter := func(s string) bool { return rx.MatchString(s) } + switch { + case info.PAst != nil: + ast.FilterFile(info.PAst, filter) + // Special case: Don't use templates for printing + // so we only get the filtered declarations without + // package clause or extra whitespace. + for i, d := range info.PAst.Decls { + if i > 0 { + fmt.Println() + } + if *html { + var buf bytes.Buffer + writeNode(&buf, info.FSet, d) + FormatText(os.Stdout, buf.Bytes(), -1, true, "", nil) + } else { + writeNode(os.Stdout, info.FSet, d) + } + fmt.Println() + } + return + + case info.PDoc != nil: + info.PDoc.Filter(filter) + } + } + + if err := packageText.Execute(os.Stdout, info); err != nil { + log.Printf("packageText.Execute: %s", err) + } +} diff --git a/src/cmd/godoc/mapping.go b/src/cmd/godoc/mapping.go new file mode 100644 index 000000000..51f23ab98 --- /dev/null +++ b/src/cmd/godoc/mapping.go @@ -0,0 +1,200 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// This file implements the Mapping data structure. + +package main + +import ( + "fmt" + "io" + "path" + "path/filepath" + "sort" + "strings" +) + +// A Mapping object maps relative paths (e.g. from URLs) +// to absolute paths (of the file system) and vice versa. +// +// A Mapping object consists of a list of individual mappings +// of the form: prefix -> path which are interpreted as follows: +// A relative path of the form prefix/tail is to be mapped to +// the absolute path/tail, if that absolute path exists in the file +// system. Given a Mapping object, a relative path is mapped to an +// absolute path by trying each of the individual mappings in order, +// until a valid mapping is found. For instance, for the mapping: +// +// user -> /home/user +// public -> /home/user/public +// public -> /home/build/public +// +// the relative paths below are mapped to absolute paths as follows: +// +// user/foo -> /home/user/foo +// public/net/rpc/file1.go -> /home/user/public/net/rpc/file1.go +// +// If there is no /home/user/public/net/rpc/file2.go, the next public +// mapping entry is used to map the relative path to: +// +// public/net/rpc/file2.go -> /home/build/public/net/rpc/file2.go +// +// (assuming that file exists). +// +// Each individual mapping also has a RWValue associated with it that +// may be used to store mapping-specific information. See the Iterate +// method. +// +type Mapping struct { + list []mapping + prefixes []string // lazily computed from list +} + +type mapping struct { + prefix, path string + value *RWValue +} + +// Init initializes the Mapping from a list of paths. +// Empty paths are ignored; relative paths are assumed to be relative to +// the current working directory and converted to absolute paths. +// For each path of the form: +// +// dirname/localname +// +// a mapping +// +// localname -> path +// +// is added to the Mapping object, in the order of occurrence. +// For instance, under Unix, the argument: +// +// /home/user:/home/build/public +// +// leads to the following mapping: +// +// user -> /home/user +// public -> /home/build/public +// +func (m *Mapping) Init(paths []string) { + pathlist := canonicalizePaths(paths, nil) + list := make([]mapping, len(pathlist)) + + // create mapping list + for i, path := range pathlist { + _, prefix := filepath.Split(path) + list[i] = mapping{prefix, path, new(RWValue)} + } + + m.list = list +} + +// IsEmpty returns true if there are no mappings specified. +func (m *Mapping) IsEmpty() bool { return len(m.list) == 0 } + +// PrefixList returns a list of all prefixes, with duplicates removed. +// For instance, for the mapping: +// +// user -> /home/user +// public -> /home/user/public +// public -> /home/build/public +// +// the prefix list is: +// +// user, public +// +func (m *Mapping) PrefixList() []string { + // compute the list lazily + if m.prefixes == nil { + list := make([]string, len(m.list)) + + // populate list + for i, e := range m.list { + list[i] = e.prefix + } + + // sort the list and remove duplicate entries + sort.Strings(list) + i := 0 + prev := "" + for _, path := range list { + if path != prev { + list[i] = path + i++ + prev = path + } + } + + m.prefixes = list[0:i] + } + + return m.prefixes +} + +// Fprint prints the mapping. +func (m *Mapping) Fprint(w io.Writer) { + for _, e := range m.list { + fmt.Fprintf(w, "\t%s -> %s\n", e.prefix, e.path) + } +} + +func splitFirst(path string) (head, tail string) { + i := strings.Index(path, string(filepath.Separator)) + if i > 0 { + // 0 < i < len(path) + return path[0:i], path[i+1:] + } + return "", path +} + +// ToAbsolute maps a slash-separated relative path to an absolute filesystem +// path using the Mapping specified by the receiver. If the path cannot +// be mapped, the empty string is returned. +// +func (m *Mapping) ToAbsolute(spath string) string { + fpath := filepath.FromSlash(spath) + prefix, tail := splitFirst(fpath) + for _, e := range m.list { + switch { + case e.prefix == prefix: + // use tail + case e.prefix == "": + tail = fpath + default: + continue // no match + } + abspath := filepath.Join(e.path, tail) + if _, err := fs.Stat(abspath); err == nil { + return abspath + } + } + + return "" // no match +} + +// ToRelative maps an absolute filesystem path to a relative slash-separated +// path using the Mapping specified by the receiver. If the path cannot +// be mapped, the empty string is returned. +// +func (m *Mapping) ToRelative(fpath string) string { + for _, e := range m.list { + if strings.HasPrefix(fpath, e.path) { + spath := filepath.ToSlash(fpath) + // /absolute/prefix/foo -> prefix/foo + return path.Join(e.prefix, spath[len(e.path):]) // Join will remove a trailing '/' + } + } + return "" // no match +} + +// Iterate calls f for each path and RWValue in the mapping (in uspecified order) +// until f returns false. +// +func (m *Mapping) Iterate(f func(path string, value *RWValue) bool) { + for _, e := range m.list { + if !f(e.path, e.value) { + return + } + } +} diff --git a/src/cmd/godoc/parser.go b/src/cmd/godoc/parser.go new file mode 100644 index 000000000..da4b3853c --- /dev/null +++ b/src/cmd/godoc/parser.go @@ -0,0 +1,68 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// This file contains support functions for parsing .go files. +// Similar functionality is found in package go/parser but the +// functions here operate using godoc's file system fs instead +// of calling the OS's file operations directly. + +package main + +import ( + "go/ast" + "go/parser" + "go/token" + "os" + "path/filepath" +) + +func parseFiles(fset *token.FileSet, filenames []string) (pkgs map[string]*ast.Package, first os.Error) { + pkgs = make(map[string]*ast.Package) + for _, filename := range filenames { + src, err := fs.ReadFile(filename) + if err != nil { + if first == nil { + first = err + } + continue + } + + file, err := parser.ParseFile(fset, filename, src, parser.ParseComments) + if err != nil { + if first == nil { + first = err + } + continue + } + + name := file.Name.Name + pkg, found := pkgs[name] + if !found { + // TODO(gri) Use NewPackage here; reconsider ParseFiles API. + pkg = &ast.Package{name, nil, nil, make(map[string]*ast.File)} + pkgs[name] = pkg + } + pkg.Files[filename] = file + } + return +} + +func parseDir(fset *token.FileSet, path string, filter func(FileInfo) bool) (map[string]*ast.Package, os.Error) { + list, err := fs.ReadDir(path) + if err != nil { + return nil, err + } + + filenames := make([]string, len(list)) + i := 0 + for _, d := range list { + if filter == nil || filter(d) { + filenames[i] = filepath.Join(path, d.Name()) + i++ + } + } + filenames = filenames[0:i] + + return parseFiles(fset, filenames) +} diff --git a/src/cmd/godoc/snippet.go b/src/cmd/godoc/snippet.go new file mode 100755 index 000000000..68e27d9a0 --- /dev/null +++ b/src/cmd/godoc/snippet.go @@ -0,0 +1,100 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// This file contains the infrastructure to create a code +// snippet for search results. +// +// Note: At the moment, this only creates HTML snippets. + +package main + +import ( + "bytes" + "go/ast" + "go/token" + "fmt" +) + +type Snippet struct { + Line int + Text string // HTML-escaped +} + +func newSnippet(fset *token.FileSet, decl ast.Decl, id *ast.Ident) *Snippet { + // TODO instead of pretty-printing the node, should use the original source instead + var buf1 bytes.Buffer + writeNode(&buf1, fset, decl) + // wrap text with
 tag
+	var buf2 bytes.Buffer
+	buf2.WriteString("
")
+	FormatText(&buf2, buf1.Bytes(), -1, true, id.Name, nil)
+	buf2.WriteString("
") + return &Snippet{fset.Position(id.Pos()).Line, buf2.String()} +} + +func findSpec(list []ast.Spec, id *ast.Ident) ast.Spec { + for _, spec := range list { + switch s := spec.(type) { + case *ast.ImportSpec: + if s.Name == id { + return s + } + case *ast.ValueSpec: + for _, n := range s.Names { + if n == id { + return s + } + } + case *ast.TypeSpec: + if s.Name == id { + return s + } + } + } + return nil +} + +func genSnippet(fset *token.FileSet, d *ast.GenDecl, id *ast.Ident) *Snippet { + s := findSpec(d.Specs, id) + if s == nil { + return nil // declaration doesn't contain id - exit gracefully + } + + // only use the spec containing the id for the snippet + dd := &ast.GenDecl{d.Doc, d.Pos(), d.Tok, d.Lparen, []ast.Spec{s}, d.Rparen} + + return newSnippet(fset, dd, id) +} + +func funcSnippet(fset *token.FileSet, d *ast.FuncDecl, id *ast.Ident) *Snippet { + if d.Name != id { + return nil // declaration doesn't contain id - exit gracefully + } + + // only use the function signature for the snippet + dd := &ast.FuncDecl{d.Doc, d.Recv, d.Name, d.Type, nil} + + return newSnippet(fset, dd, id) +} + +// NewSnippet creates a text snippet from a declaration decl containing an +// identifier id. Parts of the declaration not containing the identifier +// may be removed for a more compact snippet. +// +func NewSnippet(fset *token.FileSet, decl ast.Decl, id *ast.Ident) (s *Snippet) { + switch d := decl.(type) { + case *ast.GenDecl: + s = genSnippet(fset, d, id) + case *ast.FuncDecl: + s = funcSnippet(fset, d, id) + } + + // handle failure gracefully + if s == nil { + var buf bytes.Buffer + fmt.Fprintf(&buf, `could not generate a snippet for %s`, id.Name) + s = &Snippet{fset.Position(id.Pos()).Line, buf.String()} + } + return +} diff --git a/src/cmd/godoc/spec.go b/src/cmd/godoc/spec.go new file mode 100644 index 000000000..3f69add86 --- /dev/null +++ b/src/cmd/godoc/spec.go @@ -0,0 +1,198 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// This file contains the mechanism to "linkify" html source +// text containing EBNF sections (as found in go_spec.html). +// The result is the input source text with the EBNF sections +// modified such that identifiers are linked to the respective +// definitions. + +package main + +import ( + "bytes" + "fmt" + "go/scanner" + "go/token" + "io" +) + +type ebnfParser struct { + out io.Writer // parser output + src []byte // parser source + file *token.File // for position information + scanner scanner.Scanner + prev int // offset of previous token + pos token.Pos // token position + tok token.Token // one token look-ahead + lit string // token literal +} + +func (p *ebnfParser) flush() { + offs := p.file.Offset(p.pos) + p.out.Write(p.src[p.prev:offs]) + p.prev = offs +} + +func (p *ebnfParser) next() { + if p.pos.IsValid() { + p.flush() + } + p.pos, p.tok, p.lit = p.scanner.Scan() + if p.tok.IsKeyword() { + // TODO Should keyword mapping always happen outside scanner? + // Or should there be a flag to scanner to enable keyword mapping? + p.tok = token.IDENT + } +} + +func (p *ebnfParser) Error(pos token.Position, msg string) { + fmt.Fprintf(p.out, `error: %s`, msg) +} + +func (p *ebnfParser) errorExpected(pos token.Pos, msg string) { + msg = "expected " + msg + if pos == p.pos { + // the error happened at the current position; + // make the error message more specific + msg += ", found '" + p.tok.String() + "'" + if p.tok.IsLiteral() { + msg += " " + p.lit + } + } + p.Error(p.file.Position(pos), msg) +} + +func (p *ebnfParser) expect(tok token.Token) token.Pos { + pos := p.pos + if p.tok != tok { + p.errorExpected(pos, "'"+tok.String()+"'") + } + p.next() // make progress in any case + return pos +} + +func (p *ebnfParser) parseIdentifier(def bool) { + name := p.lit + p.expect(token.IDENT) + if def { + fmt.Fprintf(p.out, `%s`, name, name) + } else { + fmt.Fprintf(p.out, `%s`, name, name) + } + p.prev += len(name) // skip identifier when calling flush +} + +func (p *ebnfParser) parseTerm() bool { + switch p.tok { + case token.IDENT: + p.parseIdentifier(false) + + case token.STRING: + p.next() + const ellipsis = "…" // U+2026, the horizontal ellipsis character + if p.tok == token.ILLEGAL && p.lit == ellipsis { + p.next() + p.expect(token.STRING) + } + + case token.LPAREN: + p.next() + p.parseExpression() + p.expect(token.RPAREN) + + case token.LBRACK: + p.next() + p.parseExpression() + p.expect(token.RBRACK) + + case token.LBRACE: + p.next() + p.parseExpression() + p.expect(token.RBRACE) + + default: + return false + } + + return true +} + +func (p *ebnfParser) parseSequence() { + if !p.parseTerm() { + p.errorExpected(p.pos, "term") + } + for p.parseTerm() { + } +} + +func (p *ebnfParser) parseExpression() { + for { + p.parseSequence() + if p.tok != token.OR { + break + } + p.next() + } +} + +func (p *ebnfParser) parseProduction() { + p.parseIdentifier(true) + p.expect(token.ASSIGN) + if p.tok != token.PERIOD { + p.parseExpression() + } + p.expect(token.PERIOD) +} + +func (p *ebnfParser) parse(fset *token.FileSet, out io.Writer, src []byte) { + // initialize ebnfParser + p.out = out + p.src = src + p.file = fset.AddFile("", fset.Base(), len(src)) + p.scanner.Init(p.file, src, p, scanner.AllowIllegalChars) + p.next() // initializes pos, tok, lit + + // process source + for p.tok != token.EOF { + p.parseProduction() + } + p.flush() +} + +// Markers around EBNF sections +var ( + openTag = []byte(`
`)
+	closeTag = []byte(`
`) +) + +func linkify(out io.Writer, src []byte) { + fset := token.NewFileSet() + for len(src) > 0 { + n := len(src) + + // i: beginning of EBNF text (or end of source) + i := bytes.Index(src, openTag) + if i < 0 { + i = n - len(openTag) + } + i += len(openTag) + + // j: end of EBNF text (or end of source) + j := bytes.Index(src[i:n], closeTag) // close marker + if j < 0 { + j = n - i + } + j += i + + // write text before EBNF + out.Write(src[0:i]) + // parse and write EBNF + var p ebnfParser + p.parse(fset, out, src[i:j]) + + // advance + src = src[j:n] + } +} diff --git a/src/cmd/godoc/utils.go b/src/cmd/godoc/utils.go new file mode 100644 index 000000000..11e46aee5 --- /dev/null +++ b/src/cmd/godoc/utils.go @@ -0,0 +1,168 @@ +// Copyright 2010 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// This file contains support functionality for godoc. + +package main + +import ( + "io" + "io/ioutil" + "os" + "path/filepath" + "sort" + "strings" + "sync" + "time" + "utf8" +) + +// An RWValue wraps a value and permits mutually exclusive +// access to it and records the time the value was last set. +// +type RWValue struct { + mutex sync.RWMutex + value interface{} + timestamp int64 // time of last set(), in seconds since epoch +} + +func (v *RWValue) set(value interface{}) { + v.mutex.Lock() + v.value = value + v.timestamp = time.Seconds() + v.mutex.Unlock() +} + +func (v *RWValue) get() (interface{}, int64) { + v.mutex.RLock() + defer v.mutex.RUnlock() + return v.value, v.timestamp +} + +// TODO(gri) For now, using os.Getwd() is ok here since the functionality +// based on this code is not invoked for the appengine version, +// but this is fragile. Determine what the right thing to do is, +// here (possibly have some Getwd-equivalent in FileSystem). +var cwd, _ = os.Getwd() // ignore errors + +// canonicalizePaths takes a list of (directory/file) paths and returns +// the list of corresponding absolute paths in sorted (increasing) order. +// Relative paths are assumed to be relative to the current directory, +// empty and duplicate paths as well as paths for which filter(path) is +// false are discarded. filter may be nil in which case it is not used. +// +func canonicalizePaths(list []string, filter func(path string) bool) []string { + i := 0 + for _, path := range list { + path = strings.TrimSpace(path) + if len(path) == 0 { + continue // ignore empty paths (don't assume ".") + } + // len(path) > 0: normalize path + if filepath.IsAbs(path) { + path = filepath.Clean(path) + } else { + path = filepath.Join(cwd, path) + } + // we have a non-empty absolute path + if filter != nil && !filter(path) { + continue + } + // keep the path + list[i] = path + i++ + } + list = list[0:i] + + // sort the list and remove duplicate entries + sort.Strings(list) + i = 0 + prev := "" + for _, path := range list { + if path != prev { + list[i] = path + i++ + prev = path + } + } + + return list[0:i] +} + +// writeFileAtomically writes data to a temporary file and then +// atomically renames that file to the file named by filename. +// +func writeFileAtomically(filename string, data []byte) os.Error { + // TODO(gri) this won't work on appengine + f, err := ioutil.TempFile(filepath.Split(filename)) + if err != nil { + return err + } + n, err := f.Write(data) + f.Close() + if err != nil { + return err + } + if n < len(data) { + return io.ErrShortWrite + } + return os.Rename(f.Name(), filename) +} + +// isText returns true if a significant prefix of s looks like correct UTF-8; +// that is, if it is likely that s is human-readable text. +// +func isText(s []byte) bool { + const max = 1024 // at least utf8.UTFMax + if len(s) > max { + s = s[0:max] + } + for i, c := range string(s) { + if i+utf8.UTFMax > len(s) { + // last char may be incomplete - ignore + break + } + if c == 0xFFFD || c < ' ' && c != '\n' && c != '\t' && c != '\f' { + // decoding error or control character - not a text file + return false + } + } + return true +} + +// TODO(gri): Should have a mapping from extension to handler, eventually. + +// textExt[x] is true if the extension x indicates a text file, and false otherwise. +var textExt = map[string]bool{ + ".css": false, // must be served raw + ".js": false, // must be served raw +} + +// isTextFile returns true if the file has a known extension indicating +// a text file, or if a significant chunk of the specified file looks like +// correct UTF-8; that is, if it is likely that the file contains human- +// readable text. +// +func isTextFile(filename string) bool { + // if the extension is known, use it for decision making + if isText, found := textExt[filepath.Ext(filename)]; found { + return isText + } + + // the extension is not known; read an initial chunk + // of the file and check if it looks like text + f, err := fs.Open(filename) + if err != nil { + return false + } + defer f.Close() + + var buf [1024]byte + n, err := f.Read(buf[0:]) + if err != nil { + return false + } + + return isText(buf[0:n]) +} diff --git a/src/cmd/godoc/zip.go b/src/cmd/godoc/zip.go new file mode 100644 index 000000000..27dc142f5 --- /dev/null +++ b/src/cmd/godoc/zip.go @@ -0,0 +1,207 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// This file provides an implementation of the FileSystem +// interface based on the contents of a .zip file. +// +// Assumptions: +// +// - The file paths stored in the zip file must use a slash ('/') as path +// separator; and they must be relative (i.e., they must not start with +// a '/' - this is usually the case if the file was created w/o special +// options). +// - The zip file system treats the file paths found in the zip internally +// like absolute paths w/o a leading '/'; i.e., the paths are considered +// relative to the root of the file system. +// - All path arguments to file system methods must be absolute paths. + +package main + +import ( + "archive/zip" + "fmt" + "io" + "io/ioutil" + "os" + "path" + "sort" + "strings" +) + +// zipFI is the zip-file based implementation of FileInfo +type zipFI struct { + name string // directory-local name + file *zip.File // nil for a directory +} + +func (fi zipFI) Name() string { + return fi.name +} + +func (fi zipFI) Size() int64 { + if f := fi.file; f != nil { + return int64(f.UncompressedSize) + } + return 0 // directory +} + +func (fi zipFI) Mtime_ns() int64 { + if f := fi.file; f != nil { + return f.Mtime_ns() + } + return 0 // directory has no modified time entry +} + +func (fi zipFI) IsDirectory() bool { + return fi.file == nil +} + +func (fi zipFI) IsRegular() bool { + return fi.file != nil +} + +// zipFS is the zip-file based implementation of FileSystem +type zipFS struct { + *zip.ReadCloser + list zipList +} + +func (fs *zipFS) Close() os.Error { + fs.list = nil + return fs.ReadCloser.Close() +} + +func zipPath(name string) string { + name = path.Clean(name) + if !path.IsAbs(name) { + panic(fmt.Sprintf("stat: not an absolute path: %s", name)) + } + return name[1:] // strip leading '/' +} + +func (fs *zipFS) stat(abspath string) (int, zipFI, os.Error) { + i, exact := fs.list.lookup(abspath) + if i < 0 { + // abspath has leading '/' stripped - print it explicitly + return -1, zipFI{}, fmt.Errorf("file not found: /%s", abspath) + } + _, name := path.Split(abspath) + var file *zip.File + if exact { + file = fs.list[i] // exact match found - must be a file + } + return i, zipFI{name, file}, nil +} + +func (fs *zipFS) Open(abspath string) (io.ReadCloser, os.Error) { + _, fi, err := fs.stat(zipPath(abspath)) + if err != nil { + return nil, err + } + if fi.IsDirectory() { + return nil, fmt.Errorf("Open: %s is a directory", abspath) + } + return fi.file.Open() +} + +func (fs *zipFS) Lstat(abspath string) (FileInfo, os.Error) { + _, fi, err := fs.stat(zipPath(abspath)) + return fi, err +} + +func (fs *zipFS) Stat(abspath string) (FileInfo, os.Error) { + _, fi, err := fs.stat(zipPath(abspath)) + return fi, err +} + +func (fs *zipFS) ReadDir(abspath string) ([]FileInfo, os.Error) { + path := zipPath(abspath) + i, fi, err := fs.stat(path) + if err != nil { + return nil, err + } + if !fi.IsDirectory() { + return nil, fmt.Errorf("ReadDir: %s is not a directory", abspath) + } + + var list []FileInfo + dirname := path + "/" + prevname := "" + for _, e := range fs.list[i:] { + if !strings.HasPrefix(e.Name, dirname) { + break // not in the same directory anymore + } + name := e.Name[len(dirname):] // local name + file := e + if i := strings.IndexRune(name, '/'); i >= 0 { + // We infer directories from files in subdirectories. + // If we have x/y, return a directory entry for x. + name = name[0:i] // keep local directory name only + file = nil + } + // If we have x/y and x/z, don't return two directory entries for x. + // TODO(gri): It should be possible to do this more efficiently + // by determining the (fs.list) range of local directory entries + // (via two binary searches). + if name != prevname { + list = append(list, zipFI{name, file}) + prevname = name + } + } + + return list, nil +} + +func (fs *zipFS) ReadFile(abspath string) ([]byte, os.Error) { + rc, err := fs.Open(abspath) + if err != nil { + return nil, err + } + return ioutil.ReadAll(rc) +} + +func NewZipFS(rc *zip.ReadCloser) FileSystem { + list := make(zipList, len(rc.File)) + copy(list, rc.File) // sort a copy of rc.File + sort.Sort(list) + return &zipFS{rc, list} +} + +type zipList []*zip.File + +// zipList implements sort.Interface +func (z zipList) Len() int { return len(z) } +func (z zipList) Less(i, j int) bool { return z[i].Name < z[j].Name } +func (z zipList) Swap(i, j int) { z[i], z[j] = z[j], z[i] } + +// lookup returns the smallest index of an entry with an exact match +// for name, or an inexact match starting with name/. If there is no +// such entry, the result is -1, false. +func (z zipList) lookup(name string) (index int, exact bool) { + // look for exact match first (name comes before name/ in z) + i := sort.Search(len(z), func(i int) bool { + return name <= z[i].Name + }) + if i < 0 { + return -1, false + } + if z[i].Name == name { + return i, true + } + + // look for inexact match (must be in z[i:], if present) + z = z[i:] + name += "/" + j := sort.Search(len(z), func(i int) bool { + return name <= z[i].Name + }) + if j < 0 { + return -1, false + } + if strings.HasPrefix(z[j].Name, name) { + return i + j, false + } + + return -1, false +} diff --git a/src/cmd/gofix/Makefile b/src/cmd/gofix/Makefile new file mode 100644 index 000000000..22033d7f8 --- /dev/null +++ b/src/cmd/gofix/Makefile @@ -0,0 +1,34 @@ +# Copyright 2011 The Go Authors. All rights reserved. +# Use of this source code is governed by a BSD-style +# license that can be found in the LICENSE file. + +include ../../Make.inc + +TARG=gofix +GOFILES=\ + filepath.go\ + fix.go\ + httpfinalurl.go\ + httpfs.go\ + httpheaders.go\ + httpserver.go\ + main.go\ + netdial.go\ + oserrorstring.go\ + osopen.go\ + procattr.go\ + reflect.go\ + signal.go\ + sorthelpers.go\ + sortslice.go\ + stringssplit.go\ + typecheck.go\ + url.go\ + +include ../../Make.cmd + +test: + gotest + +testshort: + gotest -test.short diff --git a/src/cmd/gofix/doc.go b/src/cmd/gofix/doc.go new file mode 100644 index 000000000..a9790e685 --- /dev/null +++ b/src/cmd/gofix/doc.go @@ -0,0 +1,36 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +/* +Gofix finds Go programs that use old APIs and rewrites them to use +newer ones. After you update to a new Go release, gofix helps make +the necessary changes to your programs. + +Usage: + gofix [-r name,...] [path ...] + +Without an explicit path, gofix reads standard input and writes the +result to standard output. + +If the named path is a file, gofix rewrites the named files in place. +If the named path is a directory, gofix rewrites all .go files in that +directory tree. When gofix rewrites a file, it prints a line to standard +error giving the name of the file and the rewrite applied. + +If the -diff flag is set, no files are rewritten. Instead gofix prints +the differences a rewrite would introduce. + +The -r flag restricts the set of rewrites considered to those in the +named list. By default gofix considers all known rewrites. Gofix's +rewrites are idempotent, so that it is safe to apply gofix to updated +or partially updated code even without using the -r flag. + +Gofix prints the full list of fixes it can apply in its help output; +to see them, run gofix -?. + +Gofix does not make backup copies of the files that it edits. +Instead, use a version control system's ``diff'' functionality to inspect +the changes that gofix makes before committing them. +*/ +package documentation diff --git a/src/cmd/gofix/filepath.go b/src/cmd/gofix/filepath.go new file mode 100644 index 000000000..1d0ad6879 --- /dev/null +++ b/src/cmd/gofix/filepath.go @@ -0,0 +1,53 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package main + +import ( + "go/ast" +) + +func init() { + register(fix{ + "filepath", + filepathFunc, + `Adapt code from filepath.[List]SeparatorString to string(filepath.[List]Separator). + +http://codereview.appspot.com/4527090 +`, + }) +} + +func filepathFunc(f *ast.File) (fixed bool) { + if !imports(f, "path/filepath") { + return + } + + walk(f, func(n interface{}) { + e, ok := n.(*ast.Expr) + if !ok { + return + } + + var ident string + switch { + case isPkgDot(*e, "filepath", "SeparatorString"): + ident = "filepath.Separator" + case isPkgDot(*e, "filepath", "ListSeparatorString"): + ident = "filepath.ListSeparator" + default: + return + } + + // string(filepath.[List]Separator) + *e = &ast.CallExpr{ + Fun: ast.NewIdent("string"), + Args: []ast.Expr{ast.NewIdent(ident)}, + } + + fixed = true + }) + + return +} diff --git a/src/cmd/gofix/filepath_test.go b/src/cmd/gofix/filepath_test.go new file mode 100644 index 000000000..d170c3ae3 --- /dev/null +++ b/src/cmd/gofix/filepath_test.go @@ -0,0 +1,33 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package main + +func init() { + addTestCases(filepathTests) +} + +var filepathTests = []testCase{ + { + Name: "filepath.0", + In: `package main + +import ( + "path/filepath" +) + +var _ = filepath.SeparatorString +var _ = filepath.ListSeparatorString +`, + Out: `package main + +import ( + "path/filepath" +) + +var _ = string(filepath.Separator) +var _ = string(filepath.ListSeparator) +`, + }, +} diff --git a/src/cmd/gofix/fix.go b/src/cmd/gofix/fix.go new file mode 100644 index 000000000..cc85ceafa --- /dev/null +++ b/src/cmd/gofix/fix.go @@ -0,0 +1,582 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package main + +import ( + "fmt" + "go/ast" + "go/token" + "os" + "strconv" + "strings" +) + +type fix struct { + name string + f func(*ast.File) bool + desc string +} + +// main runs sort.Sort(fixes) after init process is done. +type fixlist []fix + +func (f fixlist) Len() int { return len(f) } +func (f fixlist) Swap(i, j int) { f[i], f[j] = f[j], f[i] } +func (f fixlist) Less(i, j int) bool { return f[i].name < f[j].name } + +var fixes fixlist + +func register(f fix) { + fixes = append(fixes, f) +} + +// walk traverses the AST x, calling visit(y) for each node y in the tree but +// also with a pointer to each ast.Expr, ast.Stmt, and *ast.BlockStmt, +// in a bottom-up traversal. +func walk(x interface{}, visit func(interface{})) { + walkBeforeAfter(x, nop, visit) +} + +func nop(interface{}) {} + +// walkBeforeAfter is like walk but calls before(x) before traversing +// x's children and after(x) afterward. +func walkBeforeAfter(x interface{}, before, after func(interface{})) { + before(x) + + switch n := x.(type) { + default: + panic(fmt.Errorf("unexpected type %T in walkBeforeAfter", x)) + + case nil: + + // pointers to interfaces + case *ast.Decl: + walkBeforeAfter(*n, before, after) + case *ast.Expr: + walkBeforeAfter(*n, before, after) + case *ast.Spec: + walkBeforeAfter(*n, before, after) + case *ast.Stmt: + walkBeforeAfter(*n, before, after) + + // pointers to struct pointers + case **ast.BlockStmt: + walkBeforeAfter(*n, before, after) + case **ast.CallExpr: + walkBeforeAfter(*n, before, after) + case **ast.FieldList: + walkBeforeAfter(*n, before, after) + case **ast.FuncType: + walkBeforeAfter(*n, before, after) + case **ast.Ident: + walkBeforeAfter(*n, before, after) + + // pointers to slices + case *[]ast.Decl: + walkBeforeAfter(*n, before, after) + case *[]ast.Expr: + walkBeforeAfter(*n, before, after) + case *[]*ast.File: + walkBeforeAfter(*n, before, after) + case *[]*ast.Ident: + walkBeforeAfter(*n, before, after) + case *[]ast.Spec: + walkBeforeAfter(*n, before, after) + case *[]ast.Stmt: + walkBeforeAfter(*n, before, after) + + // These are ordered and grouped to match ../../pkg/go/ast/ast.go + case *ast.Field: + walkBeforeAfter(&n.Type, before, after) + case *ast.FieldList: + for _, field := range n.List { + walkBeforeAfter(field, before, after) + } + case *ast.BadExpr: + case *ast.Ident: + case *ast.Ellipsis: + case *ast.BasicLit: + case *ast.FuncLit: + walkBeforeAfter(&n.Type, before, after) + walkBeforeAfter(&n.Body, before, after) + case *ast.CompositeLit: + walkBeforeAfter(&n.Type, before, after) + walkBeforeAfter(&n.Elts, before, after) + case *ast.ParenExpr: + walkBeforeAfter(&n.X, before, after) + case *ast.SelectorExpr: + walkBeforeAfter(&n.X, before, after) + case *ast.IndexExpr: + walkBeforeAfter(&n.X, before, after) + walkBeforeAfter(&n.Index, before, after) + case *ast.SliceExpr: + walkBeforeAfter(&n.X, before, after) + if n.Low != nil { + walkBeforeAfter(&n.Low, before, after) + } + if n.High != nil { + walkBeforeAfter(&n.High, before, after) + } + case *ast.TypeAssertExpr: + walkBeforeAfter(&n.X, before, after) + walkBeforeAfter(&n.Type, before, after) + case *ast.CallExpr: + walkBeforeAfter(&n.Fun, before, after) + walkBeforeAfter(&n.Args, before, after) + case *ast.StarExpr: + walkBeforeAfter(&n.X, before, after) + case *ast.UnaryExpr: + walkBeforeAfter(&n.X, before, after) + case *ast.BinaryExpr: + walkBeforeAfter(&n.X, before, after) + walkBeforeAfter(&n.Y, before, after) + case *ast.KeyValueExpr: + walkBeforeAfter(&n.Key, before, after) + walkBeforeAfter(&n.Value, before, after) + + case *ast.ArrayType: + walkBeforeAfter(&n.Len, before, after) + walkBeforeAfter(&n.Elt, before, after) + case *ast.StructType: + walkBeforeAfter(&n.Fields, before, after) + case *ast.FuncType: + walkBeforeAfter(&n.Params, before, after) + if n.Results != nil { + walkBeforeAfter(&n.Results, before, after) + } + case *ast.InterfaceType: + walkBeforeAfter(&n.Methods, before, after) + case *ast.MapType: + walkBeforeAfter(&n.Key, before, after) + walkBeforeAfter(&n.Value, before, after) + case *ast.ChanType: + walkBeforeAfter(&n.Value, before, after) + + case *ast.BadStmt: + case *ast.DeclStmt: + walkBeforeAfter(&n.Decl, before, after) + case *ast.EmptyStmt: + case *ast.LabeledStmt: + walkBeforeAfter(&n.Stmt, before, after) + case *ast.ExprStmt: + walkBeforeAfter(&n.X, before, after) + case *ast.SendStmt: + walkBeforeAfter(&n.Chan, before, after) + walkBeforeAfter(&n.Value, before, after) + case *ast.IncDecStmt: + walkBeforeAfter(&n.X, before, after) + case *ast.AssignStmt: + walkBeforeAfter(&n.Lhs, before, after) + walkBeforeAfter(&n.Rhs, before, after) + case *ast.GoStmt: + walkBeforeAfter(&n.Call, before, after) + case *ast.DeferStmt: + walkBeforeAfter(&n.Call, before, after) + case *ast.ReturnStmt: + walkBeforeAfter(&n.Results, before, after) + case *ast.BranchStmt: + case *ast.BlockStmt: + walkBeforeAfter(&n.List, before, after) + case *ast.IfStmt: + walkBeforeAfter(&n.Init, before, after) + walkBeforeAfter(&n.Cond, before, after) + walkBeforeAfter(&n.Body, before, after) + walkBeforeAfter(&n.Else, before, after) + case *ast.CaseClause: + walkBeforeAfter(&n.List, before, after) + walkBeforeAfter(&n.Body, before, after) + case *ast.SwitchStmt: + walkBeforeAfter(&n.Init, before, after) + walkBeforeAfter(&n.Tag, before, after) + walkBeforeAfter(&n.Body, before, after) + case *ast.TypeSwitchStmt: + walkBeforeAfter(&n.Init, before, after) + walkBeforeAfter(&n.Assign, before, after) + walkBeforeAfter(&n.Body, before, after) + case *ast.CommClause: + walkBeforeAfter(&n.Comm, before, after) + walkBeforeAfter(&n.Body, before, after) + case *ast.SelectStmt: + walkBeforeAfter(&n.Body, before, after) + case *ast.ForStmt: + walkBeforeAfter(&n.Init, before, after) + walkBeforeAfter(&n.Cond, before, after) + walkBeforeAfter(&n.Post, before, after) + walkBeforeAfter(&n.Body, before, after) + case *ast.RangeStmt: + walkBeforeAfter(&n.Key, before, after) + walkBeforeAfter(&n.Value, before, after) + walkBeforeAfter(&n.X, before, after) + walkBeforeAfter(&n.Body, before, after) + + case *ast.ImportSpec: + case *ast.ValueSpec: + walkBeforeAfter(&n.Type, before, after) + walkBeforeAfter(&n.Values, before, after) + walkBeforeAfter(&n.Names, before, after) + case *ast.TypeSpec: + walkBeforeAfter(&n.Type, before, after) + + case *ast.BadDecl: + case *ast.GenDecl: + walkBeforeAfter(&n.Specs, before, after) + case *ast.FuncDecl: + if n.Recv != nil { + walkBeforeAfter(&n.Recv, before, after) + } + walkBeforeAfter(&n.Type, before, after) + if n.Body != nil { + walkBeforeAfter(&n.Body, before, after) + } + + case *ast.File: + walkBeforeAfter(&n.Decls, before, after) + + case *ast.Package: + walkBeforeAfter(&n.Files, before, after) + + case []*ast.File: + for i := range n { + walkBeforeAfter(&n[i], before, after) + } + case []ast.Decl: + for i := range n { + walkBeforeAfter(&n[i], before, after) + } + case []ast.Expr: + for i := range n { + walkBeforeAfter(&n[i], before, after) + } + case []*ast.Ident: + for i := range n { + walkBeforeAfter(&n[i], before, after) + } + case []ast.Stmt: + for i := range n { + walkBeforeAfter(&n[i], before, after) + } + case []ast.Spec: + for i := range n { + walkBeforeAfter(&n[i], before, after) + } + } + after(x) +} + +// imports returns true if f imports path. +func imports(f *ast.File, path string) bool { + return importSpec(f, path) != nil +} + +// importSpec returns the import spec if f imports path, +// or nil otherwise. +func importSpec(f *ast.File, path string) *ast.ImportSpec { + for _, s := range f.Imports { + if importPath(s) == path { + return s + } + } + return nil +} + +// importPath returns the unquoted import path of s, +// or "" if the path is not properly quoted. +func importPath(s *ast.ImportSpec) string { + t, err := strconv.Unquote(s.Path.Value) + if err == nil { + return t + } + return "" +} + +// isPkgDot returns true if t is the expression "pkg.name" +// where pkg is an imported identifier. +func isPkgDot(t ast.Expr, pkg, name string) bool { + sel, ok := t.(*ast.SelectorExpr) + return ok && isTopName(sel.X, pkg) && sel.Sel.String() == name +} + +// isPtrPkgDot returns true if f is the expression "*pkg.name" +// where pkg is an imported identifier. +func isPtrPkgDot(t ast.Expr, pkg, name string) bool { + ptr, ok := t.(*ast.StarExpr) + return ok && isPkgDot(ptr.X, pkg, name) +} + +// isTopName returns true if n is a top-level unresolved identifier with the given name. +func isTopName(n ast.Expr, name string) bool { + id, ok := n.(*ast.Ident) + return ok && id.Name == name && id.Obj == nil +} + +// isName returns true if n is an identifier with the given name. +func isName(n ast.Expr, name string) bool { + id, ok := n.(*ast.Ident) + return ok && id.String() == name +} + +// isCall returns true if t is a call to pkg.name. +func isCall(t ast.Expr, pkg, name string) bool { + call, ok := t.(*ast.CallExpr) + return ok && isPkgDot(call.Fun, pkg, name) +} + +// If n is an *ast.Ident, isIdent returns it; otherwise isIdent returns nil. +func isIdent(n interface{}) *ast.Ident { + id, _ := n.(*ast.Ident) + return id +} + +// refersTo returns true if n is a reference to the same object as x. +func refersTo(n ast.Node, x *ast.Ident) bool { + id, ok := n.(*ast.Ident) + // The test of id.Name == x.Name handles top-level unresolved + // identifiers, which all have Obj == nil. + return ok && id.Obj == x.Obj && id.Name == x.Name +} + +// isBlank returns true if n is the blank identifier. +func isBlank(n ast.Expr) bool { + return isName(n, "_") +} + +// isEmptyString returns true if n is an empty string literal. +func isEmptyString(n ast.Expr) bool { + lit, ok := n.(*ast.BasicLit) + return ok && lit.Kind == token.STRING && len(lit.Value) == 2 +} + +func warn(pos token.Pos, msg string, args ...interface{}) { + if pos.IsValid() { + msg = "%s: " + msg + arg1 := []interface{}{fset.Position(pos).String()} + args = append(arg1, args...) + } + fmt.Fprintf(os.Stderr, msg+"\n", args...) +} + +// countUses returns the number of uses of the identifier x in scope. +func countUses(x *ast.Ident, scope []ast.Stmt) int { + count := 0 + ff := func(n interface{}) { + if n, ok := n.(ast.Node); ok && refersTo(n, x) { + count++ + } + } + for _, n := range scope { + walk(n, ff) + } + return count +} + +// rewriteUses replaces all uses of the identifier x and !x in scope +// with f(x.Pos()) and fnot(x.Pos()). +func rewriteUses(x *ast.Ident, f, fnot func(token.Pos) ast.Expr, scope []ast.Stmt) { + var lastF ast.Expr + ff := func(n interface{}) { + ptr, ok := n.(*ast.Expr) + if !ok { + return + } + nn := *ptr + + // The child node was just walked and possibly replaced. + // If it was replaced and this is a negation, replace with fnot(p). + not, ok := nn.(*ast.UnaryExpr) + if ok && not.Op == token.NOT && not.X == lastF { + *ptr = fnot(nn.Pos()) + return + } + if refersTo(nn, x) { + lastF = f(nn.Pos()) + *ptr = lastF + } + } + for _, n := range scope { + walk(n, ff) + } +} + +// assignsTo returns true if any of the code in scope assigns to or takes the address of x. +func assignsTo(x *ast.Ident, scope []ast.Stmt) bool { + assigned := false + ff := func(n interface{}) { + if assigned { + return + } + switch n := n.(type) { + case *ast.UnaryExpr: + // use of &x + if n.Op == token.AND && refersTo(n.X, x) { + assigned = true + return + } + case *ast.AssignStmt: + for _, l := range n.Lhs { + if refersTo(l, x) { + assigned = true + return + } + } + } + } + for _, n := range scope { + if assigned { + break + } + walk(n, ff) + } + return assigned +} + +// newPkgDot returns an ast.Expr referring to "pkg.name" at position pos. +func newPkgDot(pos token.Pos, pkg, name string) ast.Expr { + return &ast.SelectorExpr{ + X: &ast.Ident{ + NamePos: pos, + Name: pkg, + }, + Sel: &ast.Ident{ + NamePos: pos, + Name: name, + }, + } +} + +// addImport adds the import path to the file f, if absent. +func addImport(f *ast.File, path string) { + if imports(f, path) { + return + } + + newImport := &ast.ImportSpec{ + Path: &ast.BasicLit{ + Kind: token.STRING, + Value: strconv.Quote(path), + }, + } + + var impdecl *ast.GenDecl + + // Find an import decl to add to. + for _, decl := range f.Decls { + gen, ok := decl.(*ast.GenDecl) + + if ok && gen.Tok == token.IMPORT { + impdecl = gen + break + } + } + + // No import decl found. Add one. + if impdecl == nil { + impdecl = &ast.GenDecl{ + Tok: token.IMPORT, + } + f.Decls = append(f.Decls, nil) + copy(f.Decls[1:], f.Decls) + f.Decls[0] = impdecl + } + + // Ensure the import decl has parentheses, if needed. + if len(impdecl.Specs) > 0 && !impdecl.Lparen.IsValid() { + impdecl.Lparen = impdecl.Pos() + } + + // Assume the import paths are alphabetically ordered. + // If they are not, the result is ugly, but legal. + insertAt := len(impdecl.Specs) // default to end of specs + for i, spec := range impdecl.Specs { + impspec := spec.(*ast.ImportSpec) + if importPath(impspec) > path { + insertAt = i + break + } + } + + impdecl.Specs = append(impdecl.Specs, nil) + copy(impdecl.Specs[insertAt+1:], impdecl.Specs[insertAt:]) + impdecl.Specs[insertAt] = newImport + + f.Imports = append(f.Imports, newImport) +} + +// deleteImport deletes the import path from the file f, if present. +func deleteImport(f *ast.File, path string) { + oldImport := importSpec(f, path) + + // Find the import node that imports path, if any. + for i, decl := range f.Decls { + gen, ok := decl.(*ast.GenDecl) + if !ok || gen.Tok != token.IMPORT { + continue + } + for j, spec := range gen.Specs { + impspec := spec.(*ast.ImportSpec) + + if oldImport != impspec { + continue + } + + // We found an import spec that imports path. + // Delete it. + copy(gen.Specs[j:], gen.Specs[j+1:]) + gen.Specs = gen.Specs[:len(gen.Specs)-1] + + // If this was the last import spec in this decl, + // delete the decl, too. + if len(gen.Specs) == 0 { + copy(f.Decls[i:], f.Decls[i+1:]) + f.Decls = f.Decls[:len(f.Decls)-1] + } else if len(gen.Specs) == 1 { + gen.Lparen = token.NoPos // drop parens + } + + break + } + } + + // Delete it from f.Imports. + for i, imp := range f.Imports { + if imp == oldImport { + copy(f.Imports[i:], f.Imports[i+1:]) + f.Imports = f.Imports[:len(f.Imports)-1] + break + } + } +} + +func usesImport(f *ast.File, path string) (used bool) { + spec := importSpec(f, path) + if spec == nil { + return + } + + name := spec.Name.String() + switch name { + case "": + // If the package name is not explicitly specified, + // make an educated guess. This is not guaranteed to be correct. + lastSlash := strings.LastIndex(path, "/") + if lastSlash == -1 { + name = path + } else { + name = path[lastSlash+1:] + } + case "_", ".": + // Not sure if this import is used - err on the side of caution. + return true + } + + walk(f, func(n interface{}) { + sel, ok := n.(*ast.SelectorExpr) + if ok && isTopName(sel.X, name) { + used = true + } + }) + + return +} diff --git a/src/cmd/gofix/httpfinalurl.go b/src/cmd/gofix/httpfinalurl.go new file mode 100644 index 000000000..9e6cbf6bc --- /dev/null +++ b/src/cmd/gofix/httpfinalurl.go @@ -0,0 +1,56 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package main + +import ( + "go/ast" +) + +var httpFinalURLFix = fix{ + "httpfinalurl", + httpfinalurl, + `Adapt http Get calls to not have a finalURL result parameter. + +http://codereview.appspot.com/4535056/ +`, +} + +func init() { + register(httpFinalURLFix) +} + +func httpfinalurl(f *ast.File) bool { + if !imports(f, "http") { + return false + } + + fixed := false + walk(f, func(n interface{}) { + // Fix up calls to http.Get. + // + // If they have blank identifiers, remove them: + // resp, _, err := http.Get(url) + // -> resp, err := http.Get(url) + // + // But if they're using the finalURL parameter, warn: + // resp, finalURL, err := http.Get(url) + as, ok := n.(*ast.AssignStmt) + if !ok || len(as.Lhs) != 3 || len(as.Rhs) != 1 { + return + } + + if !isCall(as.Rhs[0], "http", "Get") { + return + } + + if isBlank(as.Lhs[1]) { + as.Lhs = []ast.Expr{as.Lhs[0], as.Lhs[2]} + fixed = true + } else { + warn(as.Pos(), "call to http.Get records final URL") + } + }) + return fixed +} diff --git a/src/cmd/gofix/httpfinalurl_test.go b/src/cmd/gofix/httpfinalurl_test.go new file mode 100644 index 000000000..9e7d6242d --- /dev/null +++ b/src/cmd/gofix/httpfinalurl_test.go @@ -0,0 +1,37 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package main + +func init() { + addTestCases(httpfinalurlTests) +} + +var httpfinalurlTests = []testCase{ + { + Name: "finalurl.0", + In: `package main + +import ( + "http" +) + +func f() { + resp, _, err := http.Get("http://www.google.com/") + _, _ = resp, err +} +`, + Out: `package main + +import ( + "http" +) + +func f() { + resp, err := http.Get("http://www.google.com/") + _, _ = resp, err +} +`, + }, +} diff --git a/src/cmd/gofix/httpfs.go b/src/cmd/gofix/httpfs.go new file mode 100644 index 000000000..7f2765680 --- /dev/null +++ b/src/cmd/gofix/httpfs.go @@ -0,0 +1,63 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package main + +import ( + "go/ast" + "go/token" +) + +var httpFileSystemFix = fix{ + "httpfs", + httpfs, + `Adapt http FileServer to take a FileSystem. + +http://codereview.appspot.com/4629047 http FileSystem interface +`, +} + +func init() { + register(httpFileSystemFix) +} + +func httpfs(f *ast.File) bool { + if !imports(f, "http") { + return false + } + + fixed := false + walk(f, func(n interface{}) { + call, ok := n.(*ast.CallExpr) + if !ok || !isPkgDot(call.Fun, "http", "FileServer") { + return + } + if len(call.Args) != 2 { + return + } + dir, prefix := call.Args[0], call.Args[1] + call.Args = []ast.Expr{&ast.CallExpr{ + Fun: &ast.SelectorExpr{ast.NewIdent("http"), ast.NewIdent("Dir")}, + Args: []ast.Expr{dir}, + }} + wrapInStripHandler := true + if prefixLit, ok := prefix.(*ast.BasicLit); ok { + if prefixLit.Kind == token.STRING && (prefixLit.Value == `"/"` || prefixLit.Value == `""`) { + wrapInStripHandler = false + } + } + if wrapInStripHandler { + call.Fun.(*ast.SelectorExpr).Sel = ast.NewIdent("StripPrefix") + call.Args = []ast.Expr{ + prefix, + &ast.CallExpr{ + Fun: &ast.SelectorExpr{ast.NewIdent("http"), ast.NewIdent("FileServer")}, + Args: call.Args, + }, + } + } + fixed = true + }) + return fixed +} diff --git a/src/cmd/gofix/httpfs_test.go b/src/cmd/gofix/httpfs_test.go new file mode 100644 index 000000000..d1804e93b --- /dev/null +++ b/src/cmd/gofix/httpfs_test.go @@ -0,0 +1,47 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package main + +func init() { + addTestCases(httpFileSystemTests) +} + +var httpFileSystemTests = []testCase{ + { + Name: "httpfs.0", + In: `package httpfs + +import ( + "http" +) + +func f() { + _ = http.FileServer("/var/www/foo", "/") + _ = http.FileServer("/var/www/foo", "") + _ = http.FileServer("/var/www/foo/bar", "/bar") + s := "/foo" + _ = http.FileServer(s, "/") + prefix := "/p" + _ = http.FileServer(s, prefix) +} +`, + Out: `package httpfs + +import ( + "http" +) + +func f() { + _ = http.FileServer(http.Dir("/var/www/foo")) + _ = http.FileServer(http.Dir("/var/www/foo")) + _ = http.StripPrefix("/bar", http.FileServer(http.Dir("/var/www/foo/bar"))) + s := "/foo" + _ = http.FileServer(http.Dir(s)) + prefix := "/p" + _ = http.StripPrefix(prefix, http.FileServer(http.Dir(s))) +} +`, + }, +} diff --git a/src/cmd/gofix/httpheaders.go b/src/cmd/gofix/httpheaders.go new file mode 100644 index 000000000..8a9080e8e --- /dev/null +++ b/src/cmd/gofix/httpheaders.go @@ -0,0 +1,66 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package main + +import ( + "go/ast" +) + +var httpHeadersFix = fix{ + "httpheaders", + httpheaders, + `Rename http Referer, UserAgent, Cookie, SetCookie, which are now methods. + +http://codereview.appspot.com/4620049/ +`, +} + +func init() { + register(httpHeadersFix) +} + +func httpheaders(f *ast.File) bool { + if !imports(f, "http") { + return false + } + + called := make(map[ast.Node]bool) + walk(f, func(ni interface{}) { + switch n := ni.(type) { + case *ast.CallExpr: + called[n.Fun] = true + } + }) + + fixed := false + typeof := typecheck(headerTypeConfig, f) + walk(f, func(ni interface{}) { + switch n := ni.(type) { + case *ast.SelectorExpr: + if called[n] { + break + } + if t := typeof[n.X]; t != "*http.Request" && t != "*http.Response" { + break + } + switch n.Sel.Name { + case "Referer", "UserAgent": + n.Sel.Name += "()" + fixed = true + case "Cookie": + n.Sel.Name = "Cookies()" + fixed = true + } + } + }) + return fixed +} + +var headerTypeConfig = &TypeConfig{ + Type: map[string]*Type{ + "*http.Request": &Type{}, + "*http.Response": &Type{}, + }, +} diff --git a/src/cmd/gofix/httpheaders_test.go b/src/cmd/gofix/httpheaders_test.go new file mode 100644 index 000000000..cc82b5893 --- /dev/null +++ b/src/cmd/gofix/httpheaders_test.go @@ -0,0 +1,73 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package main + +func init() { + addTestCases(httpHeadersTests) +} + +var httpHeadersTests = []testCase{ + { + Name: "httpheaders.0", + In: `package headertest + +import ( + "http" +) + +type Other struct { + Referer string + UserAgent string + Cookie []*http.Cookie +} + +func f(req *http.Request, res *http.Response, other *Other) { + _ = req.Referer + _ = req.UserAgent + _ = req.Cookie + + _ = res.Cookie + + _ = other.Referer + _ = other.UserAgent + _ = other.Cookie + + _ = req.Referer() + _ = req.UserAgent() + _ = req.Cookies() + _ = res.Cookies() +} +`, + Out: `package headertest + +import ( + "http" +) + +type Other struct { + Referer string + UserAgent string + Cookie []*http.Cookie +} + +func f(req *http.Request, res *http.Response, other *Other) { + _ = req.Referer() + _ = req.UserAgent() + _ = req.Cookies() + + _ = res.Cookies() + + _ = other.Referer + _ = other.UserAgent + _ = other.Cookie + + _ = req.Referer() + _ = req.UserAgent() + _ = req.Cookies() + _ = res.Cookies() +} +`, + }, +} diff --git a/src/cmd/gofix/httpserver.go b/src/cmd/gofix/httpserver.go new file mode 100644 index 000000000..37866e88b --- /dev/null +++ b/src/cmd/gofix/httpserver.go @@ -0,0 +1,140 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package main + +import ( + "go/ast" + "go/token" +) + +var httpserverFix = fix{ + "httpserver", + httpserver, + `Adapt http server methods and functions to changes +made to the http ResponseWriter interface. + +http://codereview.appspot.com/4245064 Hijacker +http://codereview.appspot.com/4239076 Header +http://codereview.appspot.com/4239077 Flusher +http://codereview.appspot.com/4248075 RemoteAddr, UsingTLS +`, +} + +func init() { + register(httpserverFix) +} + +func httpserver(f *ast.File) bool { + if !imports(f, "http") { + return false + } + + fixed := false + for _, decl := range f.Decls { + fn, ok := decl.(*ast.FuncDecl) + if !ok { + continue + } + w, req, ok := isServeHTTP(fn) + if !ok { + continue + } + walk(fn.Body, func(n interface{}) { + // Want to replace expression sometimes, + // so record pointer to it for updating below. + ptr, ok := n.(*ast.Expr) + if ok { + n = *ptr + } + + // Look for w.UsingTLS() and w.Remoteaddr(). + call, ok := n.(*ast.CallExpr) + if !ok || (len(call.Args) != 0 && len(call.Args) != 2) { + return + } + sel, ok := call.Fun.(*ast.SelectorExpr) + if !ok { + return + } + if !refersTo(sel.X, w) { + return + } + switch sel.Sel.String() { + case "Hijack": + // replace w with w.(http.Hijacker) + sel.X = &ast.TypeAssertExpr{ + X: sel.X, + Type: ast.NewIdent("http.Hijacker"), + } + fixed = true + case "Flush": + // replace w with w.(http.Flusher) + sel.X = &ast.TypeAssertExpr{ + X: sel.X, + Type: ast.NewIdent("http.Flusher"), + } + fixed = true + case "UsingTLS": + if ptr == nil { + // can only replace expression if we have pointer to it + break + } + // replace with req.TLS != nil + *ptr = &ast.BinaryExpr{ + X: &ast.SelectorExpr{ + X: ast.NewIdent(req.String()), + Sel: ast.NewIdent("TLS"), + }, + Op: token.NEQ, + Y: ast.NewIdent("nil"), + } + fixed = true + case "RemoteAddr": + if ptr == nil { + // can only replace expression if we have pointer to it + break + } + // replace with req.RemoteAddr + *ptr = &ast.SelectorExpr{ + X: ast.NewIdent(req.String()), + Sel: ast.NewIdent("RemoteAddr"), + } + fixed = true + case "SetHeader": + // replace w.SetHeader with w.Header().Set + // or w.Header().Del if second argument is "" + sel.X = &ast.CallExpr{ + Fun: &ast.SelectorExpr{ + X: ast.NewIdent(w.String()), + Sel: ast.NewIdent("Header"), + }, + } + sel.Sel = ast.NewIdent("Set") + if len(call.Args) == 2 && isEmptyString(call.Args[1]) { + sel.Sel = ast.NewIdent("Del") + call.Args = call.Args[:1] + } + fixed = true + } + }) + } + return fixed +} + +func isServeHTTP(fn *ast.FuncDecl) (w, req *ast.Ident, ok bool) { + for _, field := range fn.Type.Params.List { + if isPkgDot(field.Type, "http", "ResponseWriter") { + w = field.Names[0] + continue + } + if isPtrPkgDot(field.Type, "http", "Request") { + req = field.Names[0] + continue + } + } + + ok = w != nil && req != nil + return +} diff --git a/src/cmd/gofix/httpserver_test.go b/src/cmd/gofix/httpserver_test.go new file mode 100644 index 000000000..89bb4fa71 --- /dev/null +++ b/src/cmd/gofix/httpserver_test.go @@ -0,0 +1,53 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package main + +func init() { + addTestCases(httpserverTests) +} + +var httpserverTests = []testCase{ + { + Name: "httpserver.0", + In: `package main + +import "http" + +func f(xyz http.ResponseWriter, abc *http.Request, b string) { + xyz.SetHeader("foo", "bar") + xyz.SetHeader("baz", "") + xyz.Hijack() + xyz.Flush() + go xyz.Hijack() + defer xyz.Flush() + _ = xyz.UsingTLS() + _ = true == xyz.UsingTLS() + _ = xyz.RemoteAddr() + _ = xyz.RemoteAddr() == "hello" + if xyz.UsingTLS() { + } +} +`, + Out: `package main + +import "http" + +func f(xyz http.ResponseWriter, abc *http.Request, b string) { + xyz.Header().Set("foo", "bar") + xyz.Header().Del("baz") + xyz.(http.Hijacker).Hijack() + xyz.(http.Flusher).Flush() + go xyz.(http.Hijacker).Hijack() + defer xyz.(http.Flusher).Flush() + _ = abc.TLS != nil + _ = true == (abc.TLS != nil) + _ = abc.RemoteAddr + _ = abc.RemoteAddr == "hello" + if abc.TLS != nil { + } +} +`, + }, +} diff --git a/src/cmd/gofix/main.go b/src/cmd/gofix/main.go new file mode 100644 index 000000000..e7e7013c5 --- /dev/null +++ b/src/cmd/gofix/main.go @@ -0,0 +1,258 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package main + +import ( + "bytes" + "exec" + "flag" + "fmt" + "go/parser" + "go/printer" + "go/scanner" + "go/token" + "io/ioutil" + "os" + "path/filepath" + "sort" + "strings" +) + +var ( + fset = token.NewFileSet() + exitCode = 0 +) + +var allowedRewrites = flag.String("r", "", + "restrict the rewrites to this comma-separated list") + +var allowed map[string]bool + +var doDiff = flag.Bool("diff", false, "display diffs instead of rewriting files") + +func usage() { + fmt.Fprintf(os.Stderr, "usage: gofix [-diff] [-r fixname,...] [path ...]\n") + flag.PrintDefaults() + fmt.Fprintf(os.Stderr, "\nAvailable rewrites are:\n") + for _, f := range fixes { + fmt.Fprintf(os.Stderr, "\n%s\n", f.name) + desc := strings.TrimSpace(f.desc) + desc = strings.Replace(desc, "\n", "\n\t", -1) + fmt.Fprintf(os.Stderr, "\t%s\n", desc) + } + os.Exit(2) +} + +func main() { + sort.Sort(fixes) + + flag.Usage = usage + flag.Parse() + + if *allowedRewrites != "" { + allowed = make(map[string]bool) + for _, f := range strings.Split(*allowedRewrites, ",") { + allowed[f] = true + } + } + + if flag.NArg() == 0 { + if err := processFile("standard input", true); err != nil { + report(err) + } + os.Exit(exitCode) + } + + for i := 0; i < flag.NArg(); i++ { + path := flag.Arg(i) + switch dir, err := os.Stat(path); { + case err != nil: + report(err) + case dir.IsRegular(): + if err := processFile(path, false); err != nil { + report(err) + } + case dir.IsDirectory(): + walkDir(path) + } + } + + os.Exit(exitCode) +} + +const ( + tabWidth = 8 + parserMode = parser.ParseComments + printerMode = printer.TabIndent | printer.UseSpaces +) + +var printConfig = &printer.Config{ + printerMode, + tabWidth, +} + +func processFile(filename string, useStdin bool) os.Error { + var f *os.File + var err os.Error + var fixlog bytes.Buffer + var buf bytes.Buffer + + if useStdin { + f = os.Stdin + } else { + f, err = os.Open(filename) + if err != nil { + return err + } + defer f.Close() + } + + src, err := ioutil.ReadAll(f) + if err != nil { + return err + } + + file, err := parser.ParseFile(fset, filename, src, parserMode) + if err != nil { + return err + } + + // Apply all fixes to file. + newFile := file + fixed := false + for _, fix := range fixes { + if allowed != nil && !allowed[fix.name] { + continue + } + if fix.f(newFile) { + fixed = true + fmt.Fprintf(&fixlog, " %s", fix.name) + + // AST changed. + // Print and parse, to update any missing scoping + // or position information for subsequent fixers. + buf.Reset() + _, err = printConfig.Fprint(&buf, fset, newFile) + if err != nil { + return err + } + newSrc := buf.Bytes() + newFile, err = parser.ParseFile(fset, filename, newSrc, parserMode) + if err != nil { + return err + } + } + } + if !fixed { + return nil + } + fmt.Fprintf(os.Stderr, "%s: fixed %s\n", filename, fixlog.String()[1:]) + + // Print AST. We did that after each fix, so this appears + // redundant, but it is necessary to generate gofmt-compatible + // source code in a few cases. The official gofmt style is the + // output of the printer run on a standard AST generated by the parser, + // but the source we generated inside the loop above is the + // output of the printer run on a mangled AST generated by a fixer. + buf.Reset() + _, err = printConfig.Fprint(&buf, fset, newFile) + if err != nil { + return err + } + newSrc := buf.Bytes() + + if *doDiff { + data, err := diff(src, newSrc) + if err != nil { + return fmt.Errorf("computing diff: %s", err) + } + fmt.Printf("diff %s fixed/%s\n", filename, filename) + os.Stdout.Write(data) + return nil + } + + if useStdin { + os.Stdout.Write(newSrc) + return nil + } + + return ioutil.WriteFile(f.Name(), newSrc, 0) +} + +var gofmtBuf bytes.Buffer + +func gofmt(n interface{}) string { + gofmtBuf.Reset() + _, err := printConfig.Fprint(&gofmtBuf, fset, n) + if err != nil { + return "<" + err.String() + ">" + } + return gofmtBuf.String() +} + +func report(err os.Error) { + scanner.PrintError(os.Stderr, err) + exitCode = 2 +} + +func walkDir(path string) { + v := make(fileVisitor) + go func() { + filepath.Walk(path, v, v) + close(v) + }() + for err := range v { + if err != nil { + report(err) + } + } +} + +type fileVisitor chan os.Error + +func (v fileVisitor) VisitDir(path string, f *os.FileInfo) bool { + return true +} + +func (v fileVisitor) VisitFile(path string, f *os.FileInfo) { + if isGoFile(f) { + v <- nil // synchronize error handler + if err := processFile(path, false); err != nil { + v <- err + } + } +} + +func isGoFile(f *os.FileInfo) bool { + // ignore non-Go files + return f.IsRegular() && !strings.HasPrefix(f.Name, ".") && strings.HasSuffix(f.Name, ".go") +} + +func diff(b1, b2 []byte) (data []byte, err os.Error) { + f1, err := ioutil.TempFile("", "gofix") + if err != nil { + return nil, err + } + defer os.Remove(f1.Name()) + defer f1.Close() + + f2, err := ioutil.TempFile("", "gofix") + if err != nil { + return nil, err + } + defer os.Remove(f2.Name()) + defer f2.Close() + + f1.Write(b1) + f2.Write(b2) + + data, err = exec.Command("diff", "-u", f1.Name(), f2.Name()).CombinedOutput() + if len(data) > 0 { + // diff exits with a non-zero status when the files don't match. + // Ignore that failure as long as we get output. + err = nil + } + return +} diff --git a/src/cmd/gofix/main_test.go b/src/cmd/gofix/main_test.go new file mode 100644 index 000000000..275778e5b --- /dev/null +++ b/src/cmd/gofix/main_test.go @@ -0,0 +1,126 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package main + +import ( + "bytes" + "go/ast" + "go/parser" + "go/printer" + "strings" + "testing" +) + +type testCase struct { + Name string + Fn func(*ast.File) bool + In string + Out string +} + +var testCases []testCase + +func addTestCases(t []testCase) { + testCases = append(testCases, t...) +} + +func fnop(*ast.File) bool { return false } + +func parseFixPrint(t *testing.T, fn func(*ast.File) bool, desc, in string) (out string, fixed, ok bool) { + file, err := parser.ParseFile(fset, desc, in, parserMode) + if err != nil { + t.Errorf("%s: parsing: %v", desc, err) + return + } + + var buf bytes.Buffer + buf.Reset() + _, err = (&printer.Config{printerMode, tabWidth}).Fprint(&buf, fset, file) + if err != nil { + t.Errorf("%s: printing: %v", desc, err) + return + } + if s := buf.String(); in != s && fn != fnop { + t.Errorf("%s: not gofmt-formatted.\n--- %s\n%s\n--- %s | gofmt\n%s", + desc, desc, in, desc, s) + tdiff(t, in, s) + return + } + + if fn == nil { + for _, fix := range fixes { + if fix.f(file) { + fixed = true + } + } + } else { + fixed = fn(file) + } + + buf.Reset() + _, err = (&printer.Config{printerMode, tabWidth}).Fprint(&buf, fset, file) + if err != nil { + t.Errorf("%s: printing: %v", desc, err) + return + } + + return buf.String(), fixed, true +} + +func TestRewrite(t *testing.T) { + for _, tt := range testCases { + // Apply fix: should get tt.Out. + out, fixed, ok := parseFixPrint(t, tt.Fn, tt.Name, tt.In) + if !ok { + continue + } + + // reformat to get printing right + out, _, ok = parseFixPrint(t, fnop, tt.Name, out) + if !ok { + continue + } + + if out != tt.Out { + t.Errorf("%s: incorrect output.\n", tt.Name) + if !strings.HasPrefix(tt.Name, "testdata/") { + t.Errorf("--- have\n%s\n--- want\n%s", out, tt.Out) + } + tdiff(t, out, tt.Out) + continue + } + + if changed := out != tt.In; changed != fixed { + t.Errorf("%s: changed=%v != fixed=%v", tt.Name, changed, fixed) + continue + } + + // Should not change if run again. + out2, fixed2, ok := parseFixPrint(t, tt.Fn, tt.Name+" output", out) + if !ok { + continue + } + + if fixed2 { + t.Errorf("%s: applied fixes during second round", tt.Name) + continue + } + + if out2 != out { + t.Errorf("%s: changed output after second round of fixes.\n--- output after first round\n%s\n--- output after second round\n%s", + tt.Name, out, out2) + tdiff(t, out, out2) + } + } +} + +func tdiff(t *testing.T, a, b string) { + data, err := diff([]byte(a), []byte(b)) + if err != nil { + t.Error(err) + return + } + t.Error(string(data)) +} diff --git a/src/cmd/gofix/netdial.go b/src/cmd/gofix/netdial.go new file mode 100644 index 000000000..afa98953b --- /dev/null +++ b/src/cmd/gofix/netdial.go @@ -0,0 +1,114 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package main + +import ( + "go/ast" +) + +var netdialFix = fix{ + "netdial", + netdial, + `Adapt 3-argument calls of net.Dial to use 2-argument form. + +http://codereview.appspot.com/4244055 +`, +} + +var tlsdialFix = fix{ + "tlsdial", + tlsdial, + `Adapt 4-argument calls of tls.Dial to use 3-argument form. + +http://codereview.appspot.com/4244055 +`, +} + +var netlookupFix = fix{ + "netlookup", + netlookup, + `Adapt 3-result calls to net.LookupHost to use 2-result form. + +http://codereview.appspot.com/4244055 +`, +} + +func init() { + register(netdialFix) + register(tlsdialFix) + register(netlookupFix) +} + +func netdial(f *ast.File) bool { + if !imports(f, "net") { + return false + } + + fixed := false + walk(f, func(n interface{}) { + call, ok := n.(*ast.CallExpr) + if !ok || !isPkgDot(call.Fun, "net", "Dial") || len(call.Args) != 3 { + return + } + // net.Dial(a, "", b) -> net.Dial(a, b) + if !isEmptyString(call.Args[1]) { + warn(call.Pos(), "call to net.Dial with non-empty second argument") + return + } + call.Args[1] = call.Args[2] + call.Args = call.Args[:2] + fixed = true + }) + return fixed +} + +func tlsdial(f *ast.File) bool { + if !imports(f, "crypto/tls") { + return false + } + + fixed := false + walk(f, func(n interface{}) { + call, ok := n.(*ast.CallExpr) + if !ok || !isPkgDot(call.Fun, "tls", "Dial") || len(call.Args) != 4 { + return + } + // tls.Dial(a, "", b, c) -> tls.Dial(a, b, c) + if !isEmptyString(call.Args[1]) { + warn(call.Pos(), "call to tls.Dial with non-empty second argument") + return + } + call.Args[1] = call.Args[2] + call.Args[2] = call.Args[3] + call.Args = call.Args[:3] + fixed = true + }) + return fixed +} + +func netlookup(f *ast.File) bool { + if !imports(f, "net") { + return false + } + + fixed := false + walk(f, func(n interface{}) { + as, ok := n.(*ast.AssignStmt) + if !ok || len(as.Lhs) != 3 || len(as.Rhs) != 1 { + return + } + call, ok := as.Rhs[0].(*ast.CallExpr) + if !ok || !isPkgDot(call.Fun, "net", "LookupHost") { + return + } + if !isBlank(as.Lhs[2]) { + warn(as.Pos(), "call to net.LookupHost expecting cname; use net.LookupCNAME") + return + } + as.Lhs = as.Lhs[:2] + fixed = true + }) + return fixed +} diff --git a/src/cmd/gofix/netdial_test.go b/src/cmd/gofix/netdial_test.go new file mode 100644 index 000000000..272aa526a --- /dev/null +++ b/src/cmd/gofix/netdial_test.go @@ -0,0 +1,51 @@ +package main + +func init() { + addTestCases(netdialTests) +} + +var netdialTests = []testCase{ + { + Name: "netdial.0", + In: `package main + +import "net" + +func f() { + c, err := net.Dial(net, "", addr) + c, err = net.Dial(net, "", addr) +} +`, + Out: `package main + +import "net" + +func f() { + c, err := net.Dial(net, addr) + c, err = net.Dial(net, addr) +} +`, + }, + + { + Name: "netlookup.0", + In: `package main + +import "net" + +func f() { + foo, bar, _ := net.LookupHost(host) + foo, bar, _ = net.LookupHost(host) +} +`, + Out: `package main + +import "net" + +func f() { + foo, bar := net.LookupHost(host) + foo, bar = net.LookupHost(host) +} +`, + }, +} diff --git a/src/cmd/gofix/oserrorstring.go b/src/cmd/gofix/oserrorstring.go new file mode 100644 index 000000000..db39ee9dc --- /dev/null +++ b/src/cmd/gofix/oserrorstring.go @@ -0,0 +1,74 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package main + +import ( + "go/ast" +) + +var oserrorstringFix = fix{ + "oserrorstring", + oserrorstring, + `Replace os.ErrorString() conversions with calls to os.NewError(). + +http://codereview.appspot.com/4607052 +`, +} + +func init() { + register(oserrorstringFix) +} + +func oserrorstring(f *ast.File) bool { + if !imports(f, "os") { + return false + } + + fixed := false + walk(f, func(n interface{}) { + // The conversion os.ErrorString(x) looks like a call + // of os.ErrorString with one argument. + if call := callExpr(n, "os", "ErrorString"); call != nil { + // os.ErrorString(args) -> os.NewError(args) + call.Fun.(*ast.SelectorExpr).Sel.Name = "NewError" + // os.ErrorString(args) -> os.NewError(args) + call.Fun.(*ast.SelectorExpr).Sel.Name = "NewError" + fixed = true + return + } + + // Remove os.Error type from variable declarations initialized + // with an os.NewError. + // (An *ast.ValueSpec may also be used in a const declaration + // but those won't be initialized with a call to os.NewError.) + if spec, ok := n.(*ast.ValueSpec); ok && + len(spec.Names) == 1 && + isPkgDot(spec.Type, "os", "Error") && + len(spec.Values) == 1 && + callExpr(spec.Values[0], "os", "NewError") != nil { + // var name os.Error = os.NewError(x) -> + // var name = os.NewError(x) + spec.Type = nil + fixed = true + return + } + + // Other occurrences of os.ErrorString are not fixed + // but they are rare. + + }) + return fixed +} + +// callExpr returns the call expression if x is a call to pkg.name with one argument; +// otherwise it returns nil. +func callExpr(x interface{}, pkg, name string) *ast.CallExpr { + if call, ok := x.(*ast.CallExpr); ok && + len(call.Args) == 1 && + isPkgDot(call.Fun, pkg, name) { + return call + } + return nil +} diff --git a/src/cmd/gofix/oserrorstring_test.go b/src/cmd/gofix/oserrorstring_test.go new file mode 100644 index 000000000..070d9222b --- /dev/null +++ b/src/cmd/gofix/oserrorstring_test.go @@ -0,0 +1,57 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package main + +func init() { + addTestCases(oserrorstringTests) +} + +var oserrorstringTests = []testCase{ + { + Name: "oserrorstring.0", + In: `package main + +import "os" + +var _ = os.ErrorString("foo") +var _ os.Error = os.ErrorString("bar1") +var _ os.Error = os.NewError("bar2") +var _ os.Error = MyError("bal") // don't rewrite this one + +var ( + _ = os.ErrorString("foo") + _ os.Error = os.ErrorString("bar1") + _ os.Error = os.NewError("bar2") + _ os.Error = MyError("bal") // don't rewrite this one +) + +func _() (err os.Error) { + err = os.ErrorString("foo") + return os.ErrorString("foo") +} +`, + Out: `package main + +import "os" + +var _ = os.NewError("foo") +var _ = os.NewError("bar1") +var _ = os.NewError("bar2") +var _ os.Error = MyError("bal") // don't rewrite this one + +var ( + _ = os.NewError("foo") + _ = os.NewError("bar1") + _ = os.NewError("bar2") + _ os.Error = MyError("bal") // don't rewrite this one +) + +func _() (err os.Error) { + err = os.NewError("foo") + return os.NewError("foo") +} +`, + }, +} diff --git a/src/cmd/gofix/osopen.go b/src/cmd/gofix/osopen.go new file mode 100644 index 000000000..19c19b5b6 --- /dev/null +++ b/src/cmd/gofix/osopen.go @@ -0,0 +1,123 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package main + +import ( + "go/ast" +) + +var osopenFix = fix{ + "osopen", + osopen, + `Adapt os.Open calls to new, easier API and rename O_CREAT O_CREATE. + +http://codereview.appspot.com/4357052 +`, +} + +func init() { + register(osopenFix) +} + +func osopen(f *ast.File) bool { + if !imports(f, "os") { + return false + } + + fixed := false + walk(f, func(n interface{}) { + // Rename O_CREAT to O_CREATE. + if expr, ok := n.(ast.Expr); ok && isPkgDot(expr, "os", "O_CREAT") { + expr.(*ast.SelectorExpr).Sel.Name = "O_CREATE" + fixed = true + return + } + + // Fix up calls to Open. + call, ok := n.(*ast.CallExpr) + if !ok || len(call.Args) != 3 { + return + } + if !isPkgDot(call.Fun, "os", "Open") { + return + } + sel := call.Fun.(*ast.SelectorExpr) + args := call.Args + // os.Open(a, os.O_RDONLY, c) -> os.Open(a) + if isPkgDot(args[1], "os", "O_RDONLY") || isPkgDot(args[1], "syscall", "O_RDONLY") { + call.Args = call.Args[0:1] + fixed = true + return + } + // os.Open(a, createlike_flags, c) -> os.Create(a, c) + if isCreateFlag(args[1]) { + sel.Sel.Name = "Create" + if !isSimplePerm(args[2]) { + warn(sel.Pos(), "rewrote os.Open to os.Create with permission not 0666") + } + call.Args = args[0:1] + fixed = true + return + } + // Fallback: os.Open(a, b, c) -> os.OpenFile(a, b, c) + sel.Sel.Name = "OpenFile" + fixed = true + }) + return fixed +} + +func isCreateFlag(flag ast.Expr) bool { + foundCreate := false + foundTrunc := false + // OR'ing of flags: is O_CREATE on? + or | would be fine; we just look for os.O_CREATE + // and don't worry about the actual operator. + p := flag.Pos() + for { + lhs := flag + expr, isBinary := flag.(*ast.BinaryExpr) + if isBinary { + lhs = expr.Y + } + sel, ok := lhs.(*ast.SelectorExpr) + if !ok || !isTopName(sel.X, "os") { + return false + } + switch sel.Sel.Name { + case "O_CREATE": + foundCreate = true + case "O_TRUNC": + foundTrunc = true + case "O_RDONLY", "O_WRONLY", "O_RDWR": + // okay + default: + // Unexpected flag, like O_APPEND or O_EXCL. + // Be conservative and do not rewrite. + return false + } + if !isBinary { + break + } + flag = expr.X + } + if !foundCreate { + return false + } + if !foundTrunc { + warn(p, "rewrote os.Open with O_CREATE but not O_TRUNC to os.Create") + } + return foundCreate +} + +func isSimplePerm(perm ast.Expr) bool { + basicLit, ok := perm.(*ast.BasicLit) + if !ok { + return false + } + switch basicLit.Value { + case "0666": + return true + } + return false +} diff --git a/src/cmd/gofix/osopen_test.go b/src/cmd/gofix/osopen_test.go new file mode 100644 index 000000000..a33bcd4fb --- /dev/null +++ b/src/cmd/gofix/osopen_test.go @@ -0,0 +1,82 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package main + +func init() { + addTestCases(osopenTests) +} + +var osopenTests = []testCase{ + { + Name: "osopen.0", + In: `package main + +import ( + "os" +) + +func f() { + os.OpenFile(a, b, c) + os.Open(a, os.O_RDONLY, 0) + os.Open(a, os.O_RDONLY, 0666) + os.Open(a, os.O_RDWR, 0) + os.Open(a, os.O_CREAT, 0666) + os.Open(a, os.O_CREAT|os.O_TRUNC, 0664) + os.Open(a, os.O_CREATE, 0666) + os.Open(a, os.O_CREATE|os.O_TRUNC, 0664) + os.Open(a, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0666) + os.Open(a, os.O_WRONLY|os.O_CREATE|os.O_APPEND, 0666) + os.Open(a, os.O_WRONLY|os.O_CREATE|os.O_EXCL, 0666) + os.Open(a, os.O_SURPRISE|os.O_CREATE, 0666) + _ = os.O_CREAT +} +`, + Out: `package main + +import ( + "os" +) + +func f() { + os.OpenFile(a, b, c) + os.Open(a) + os.Open(a) + os.OpenFile(a, os.O_RDWR, 0) + os.Create(a) + os.Create(a) + os.Create(a) + os.Create(a) + os.Create(a) + os.OpenFile(a, os.O_WRONLY|os.O_CREATE|os.O_APPEND, 0666) + os.OpenFile(a, os.O_WRONLY|os.O_CREATE|os.O_EXCL, 0666) + os.OpenFile(a, os.O_SURPRISE|os.O_CREATE, 0666) + _ = os.O_CREATE +} +`, + }, + { + Name: "osopen.1", + In: `package main + +import ( + "os" +) + +func f() { + _ = os.O_CREAT +} +`, + Out: `package main + +import ( + "os" +) + +func f() { + _ = os.O_CREATE +} +`, + }, +} diff --git a/src/cmd/gofix/procattr.go b/src/cmd/gofix/procattr.go new file mode 100644 index 000000000..0e2190b1f --- /dev/null +++ b/src/cmd/gofix/procattr.go @@ -0,0 +1,61 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package main + +import ( + "go/ast" + "go/token" +) + +var procattrFix = fix{ + "procattr", + procattr, + `Adapt calls to os.StartProcess to use new ProcAttr type. + +http://codereview.appspot.com/4253052 +`, +} + +func init() { + register(procattrFix) +} + +func procattr(f *ast.File) bool { + if !imports(f, "os") && !imports(f, "syscall") { + return false + } + + fixed := false + walk(f, func(n interface{}) { + call, ok := n.(*ast.CallExpr) + if !ok || len(call.Args) != 5 { + return + } + var pkg string + if isPkgDot(call.Fun, "os", "StartProcess") { + pkg = "os" + } else if isPkgDot(call.Fun, "syscall", "StartProcess") { + pkg = "syscall" + } else { + return + } + // os.StartProcess(a, b, c, d, e) -> os.StartProcess(a, b, &os.ProcAttr{Env: c, Dir: d, Files: e}) + lit := &ast.CompositeLit{Type: ast.NewIdent(pkg + ".ProcAttr")} + env, dir, files := call.Args[2], call.Args[3], call.Args[4] + if !isName(env, "nil") && !isCall(env, "os", "Environ") { + lit.Elts = append(lit.Elts, &ast.KeyValueExpr{Key: ast.NewIdent("Env"), Value: env}) + } + if !isEmptyString(dir) { + lit.Elts = append(lit.Elts, &ast.KeyValueExpr{Key: ast.NewIdent("Dir"), Value: dir}) + } + if !isName(files, "nil") { + lit.Elts = append(lit.Elts, &ast.KeyValueExpr{Key: ast.NewIdent("Files"), Value: files}) + } + call.Args[2] = &ast.UnaryExpr{Op: token.AND, X: lit} + call.Args = call.Args[:3] + fixed = true + }) + return fixed +} diff --git a/src/cmd/gofix/procattr_test.go b/src/cmd/gofix/procattr_test.go new file mode 100644 index 000000000..b973b9684 --- /dev/null +++ b/src/cmd/gofix/procattr_test.go @@ -0,0 +1,74 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package main + +func init() { + addTestCases(procattrTests) +} + +var procattrTests = []testCase{ + { + Name: "procattr.0", + In: `package main + +import ( + "os" + "syscall" +) + +func f() { + os.StartProcess(a, b, c, d, e) + os.StartProcess(a, b, os.Environ(), d, e) + os.StartProcess(a, b, nil, d, e) + os.StartProcess(a, b, c, "", e) + os.StartProcess(a, b, c, d, nil) + os.StartProcess(a, b, nil, "", nil) + + os.StartProcess( + a, + b, + c, + d, + e, + ) + + syscall.StartProcess(a, b, c, d, e) + syscall.StartProcess(a, b, os.Environ(), d, e) + syscall.StartProcess(a, b, nil, d, e) + syscall.StartProcess(a, b, c, "", e) + syscall.StartProcess(a, b, c, d, nil) + syscall.StartProcess(a, b, nil, "", nil) +} +`, + Out: `package main + +import ( + "os" + "syscall" +) + +func f() { + os.StartProcess(a, b, &os.ProcAttr{Env: c, Dir: d, Files: e}) + os.StartProcess(a, b, &os.ProcAttr{Dir: d, Files: e}) + os.StartProcess(a, b, &os.ProcAttr{Dir: d, Files: e}) + os.StartProcess(a, b, &os.ProcAttr{Env: c, Files: e}) + os.StartProcess(a, b, &os.ProcAttr{Env: c, Dir: d}) + os.StartProcess(a, b, &os.ProcAttr{}) + + os.StartProcess( + a, + b, &os.ProcAttr{Env: c, Dir: d, Files: e}, + ) + + syscall.StartProcess(a, b, &syscall.ProcAttr{Env: c, Dir: d, Files: e}) + syscall.StartProcess(a, b, &syscall.ProcAttr{Dir: d, Files: e}) + syscall.StartProcess(a, b, &syscall.ProcAttr{Dir: d, Files: e}) + syscall.StartProcess(a, b, &syscall.ProcAttr{Env: c, Files: e}) + syscall.StartProcess(a, b, &syscall.ProcAttr{Env: c, Dir: d}) + syscall.StartProcess(a, b, &syscall.ProcAttr{}) +} +`, + }, +} diff --git a/src/cmd/gofix/reflect.go b/src/cmd/gofix/reflect.go new file mode 100644 index 000000000..3c8becaef --- /dev/null +++ b/src/cmd/gofix/reflect.go @@ -0,0 +1,861 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// TODO(rsc): Once there is better support for writing +// multi-package commands, this should really be in +// its own package, and then we can drop all the "reflect" +// prefixes on the global variables and functions. + +package main + +import ( + "go/ast" + "go/token" + "strings" +) + +var reflectFix = fix{ + "reflect", + reflectFn, + `Adapt code to new reflect API. + +http://codereview.appspot.com/4281055 +http://codereview.appspot.com/4433066 +`, +} + +func init() { + register(reflectFix) +} + +// The reflect API change dropped the concrete types *reflect.ArrayType etc. +// Any type assertions prior to method calls can be deleted: +// x.(*reflect.ArrayType).Len() -> x.Len() +// +// Any type checks can be replaced by assignment and check of Kind: +// x, y := z.(*reflect.ArrayType) +// -> +// x := z +// y := x.Kind() == reflect.Array +// +// If z is an ordinary variable name and x is not subsequently assigned to, +// references to x can be replaced by z and the assignment deleted. +// We only bother if x and z are the same name. +// If y is not subsequently assigned to and neither is x, references to +// y can be replaced by its expression. We only bother when there is +// just one use or when the use appears in an if clause. +// +// Not all type checks result in a single Kind check. The rewrite of the type check for +// reflect.ArrayOrSliceType checks x.Kind() against reflect.Array and reflect.Slice. +// The rewrite for *reflect.IntType checks againt Int, Int8, Int16, Int32, Int64. +// The rewrite for *reflect.UintType adds Uintptr. +// +// A type switch turns into an assignment and a switch on Kind: +// switch x := y.(type) { +// case reflect.ArrayOrSliceType: +// ... +// case *reflect.ChanType: +// ... +// case *reflect.IntType: +// ... +// } +// -> +// switch x := y; x.Kind() { +// case reflect.Array, reflect.Slice: +// ... +// case reflect.Chan: +// ... +// case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64: +// ... +// } +// +// The same simplification applies: we drop x := x if x is not assigned +// to in the switch cases. +// +// Because the type check assignment includes a type assertion in its +// syntax and the rewrite traversal is bottom up, we must do a pass to +// rewrite the type check assignments and then a separate pass to +// rewrite the type assertions. +// +// The same process applies to the API changes for reflect.Value. +// +// For both cases, but especially Value, the code needs to be aware +// of the type of a receiver when rewriting a method call. For example, +// x.(*reflect.ArrayValue).Elem(i) becomes x.Index(i) while +// x.(*reflect.MapValue).Elem(v) becomes x.MapIndex(v). +// In general, reflectFn needs to know the type of the receiver expression. +// In most cases (and in all the cases in the Go source tree), the toy +// type checker in typecheck.go provides enough information for gofix +// to make the rewrite. If gofix misses a rewrite, the code that is left over +// will not compile, so it will be noticed immediately. + +func reflectFn(f *ast.File) bool { + if !imports(f, "reflect") { + return false + } + + fixed := false + + // Rewrite names in method calls. + // Needs basic type information (see above). + typeof := typecheck(reflectTypeConfig, f) + walk(f, func(n interface{}) { + switch n := n.(type) { + case *ast.SelectorExpr: + typ := typeof[n.X] + if m := reflectRewriteMethod[typ]; m != nil { + if replace := m[n.Sel.Name]; replace != "" { + n.Sel.Name = replace + fixed = true + return + } + } + + // For all reflect Values, replace SetValue with Set. + if isReflectValue[typ] && n.Sel.Name == "SetValue" { + n.Sel.Name = "Set" + fixed = true + return + } + + // Replace reflect.MakeZero with reflect.Zero. + if isPkgDot(n, "reflect", "MakeZero") { + n.Sel.Name = "Zero" + fixed = true + return + } + } + }) + + // Replace PtrValue's PointTo(x) with Set(x.Addr()). + walk(f, func(n interface{}) { + call, ok := n.(*ast.CallExpr) + if !ok || len(call.Args) != 1 { + return + } + sel, ok := call.Fun.(*ast.SelectorExpr) + if !ok || sel.Sel.Name != "PointTo" { + return + } + typ := typeof[sel.X] + if typ != "*reflect.PtrValue" { + return + } + sel.Sel.Name = "Set" + if !isTopName(call.Args[0], "nil") { + call.Args[0] = &ast.SelectorExpr{ + X: call.Args[0], + Sel: ast.NewIdent("Addr()"), + } + } + fixed = true + }) + + // Fix type switches. + walk(f, func(n interface{}) { + if reflectFixSwitch(n) { + fixed = true + } + }) + + // Fix type assertion checks (multiple assignment statements). + // Have to work on the statement context (statement list or if statement) + // so that we can insert an extra statement occasionally. + // Ignoring for and switch because they don't come up in + // typical code. + walk(f, func(n interface{}) { + switch n := n.(type) { + case *[]ast.Stmt: + // v is the replacement statement list. + var v []ast.Stmt + insert := func(x ast.Stmt) { + v = append(v, x) + } + for i, x := range *n { + // Tentatively append to v; if we rewrite x + // we'll have to update the entry, so remember + // the index. + j := len(v) + v = append(v, x) + if reflectFixTypecheck(&x, insert, (*n)[i+1:]) { + // reflectFixTypecheck may have overwritten x. + // Update the entry we appended just before the call. + v[j] = x + fixed = true + } + } + *n = v + case *ast.IfStmt: + x := &ast.ExprStmt{n.Cond} + if reflectFixTypecheck(&n.Init, nil, []ast.Stmt{x, n.Body, n.Else}) { + n.Cond = x.X + fixed = true + } + } + }) + + // Warn about any typecheck statements that we missed. + walk(f, reflectWarnTypecheckStmt) + + // Now that those are gone, fix remaining type assertions. + // Delayed because the type checks have + // type assertions as part of their syntax. + walk(f, func(n interface{}) { + if reflectFixAssert(n) { + fixed = true + } + }) + + // Now that the type assertions are gone, rewrite remaining + // references to specific reflect types to use the general ones. + walk(f, func(n interface{}) { + ptr, ok := n.(*ast.Expr) + if !ok { + return + } + nn := *ptr + typ := reflectType(nn) + if typ == "" { + return + } + if strings.HasSuffix(typ, "Type") { + *ptr = newPkgDot(nn.Pos(), "reflect", "Type") + } else { + *ptr = newPkgDot(nn.Pos(), "reflect", "Value") + } + fixed = true + }) + + // Rewrite v.Set(nil) to v.Set(reflect.MakeZero(v.Type())). + walk(f, func(n interface{}) { + call, ok := n.(*ast.CallExpr) + if !ok || len(call.Args) != 1 || !isTopName(call.Args[0], "nil") { + return + } + sel, ok := call.Fun.(*ast.SelectorExpr) + if !ok || !isReflectValue[typeof[sel.X]] || sel.Sel.Name != "Set" { + return + } + call.Args[0] = &ast.CallExpr{ + Fun: newPkgDot(call.Args[0].Pos(), "reflect", "Zero"), + Args: []ast.Expr{ + &ast.CallExpr{ + Fun: &ast.SelectorExpr{ + X: sel.X, + Sel: &ast.Ident{Name: "Type"}, + }, + }, + }, + } + fixed = true + }) + + // Rewrite v != nil to v.IsValid(). + // Rewrite nil used as reflect.Value (in function argument or return) to reflect.Value{}. + walk(f, func(n interface{}) { + ptr, ok := n.(*ast.Expr) + if !ok { + return + } + if isTopName(*ptr, "nil") && isReflectValue[typeof[*ptr]] { + *ptr = ast.NewIdent("reflect.Value{}") + fixed = true + return + } + nn, ok := (*ptr).(*ast.BinaryExpr) + if !ok || (nn.Op != token.EQL && nn.Op != token.NEQ) || !isTopName(nn.Y, "nil") || !isReflectValue[typeof[nn.X]] { + return + } + var call ast.Expr = &ast.CallExpr{ + Fun: &ast.SelectorExpr{ + X: nn.X, + Sel: &ast.Ident{Name: "IsValid"}, + }, + } + if nn.Op == token.EQL { + call = &ast.UnaryExpr{Op: token.NOT, X: call} + } + *ptr = call + fixed = true + }) + + // Rewrite + // reflect.Typeof -> reflect.TypeOf, + walk(f, func(n interface{}) { + sel, ok := n.(*ast.SelectorExpr) + if !ok { + return + } + if isTopName(sel.X, "reflect") && sel.Sel.Name == "Typeof" { + sel.Sel.Name = "TypeOf" + fixed = true + } + if isTopName(sel.X, "reflect") && sel.Sel.Name == "NewValue" { + sel.Sel.Name = "ValueOf" + fixed = true + } + }) + + return fixed +} + +// reflectFixSwitch rewrites *n (if n is an *ast.Stmt) corresponding +// to a type switch. +func reflectFixSwitch(n interface{}) bool { + ptr, ok := n.(*ast.Stmt) + if !ok { + return false + } + n = *ptr + + ts, ok := n.(*ast.TypeSwitchStmt) + if !ok { + return false + } + + // Are any switch cases referring to reflect types? + // (That is, is this an old reflect type switch?) + for _, cas := range ts.Body.List { + for _, typ := range cas.(*ast.CaseClause).List { + if reflectType(typ) != "" { + goto haveReflect + } + } + } + return false + +haveReflect: + // Now we know it's an old reflect type switch. Prepare the new version, + // but don't replace or edit the original until we're sure of success. + + // Figure out the initializer statement, if any, and the receiver for the Kind call. + var init ast.Stmt + var rcvr ast.Expr + + init = ts.Init + switch n := ts.Assign.(type) { + default: + warn(ts.Pos(), "unexpected form in type switch") + return false + + case *ast.AssignStmt: + as := n + ta := as.Rhs[0].(*ast.TypeAssertExpr) + x := isIdent(as.Lhs[0]) + z := isIdent(ta.X) + + if isBlank(x) || x != nil && z != nil && x.Name == z.Name && !assignsTo(x, ts.Body.List) { + // Can drop the variable creation. + rcvr = ta.X + } else { + // Need to use initialization statement. + if init != nil { + warn(ts.Pos(), "cannot rewrite reflect type switch with initializing statement") + return false + } + init = &ast.AssignStmt{ + Lhs: []ast.Expr{as.Lhs[0]}, + TokPos: as.TokPos, + Tok: token.DEFINE, + Rhs: []ast.Expr{ta.X}, + } + rcvr = as.Lhs[0] + } + + case *ast.ExprStmt: + rcvr = n.X.(*ast.TypeAssertExpr).X + } + + // Prepare rewritten type switch (see large comment above for form). + sw := &ast.SwitchStmt{ + Switch: ts.Switch, + Init: init, + Tag: &ast.CallExpr{ + Fun: &ast.SelectorExpr{ + X: rcvr, + Sel: &ast.Ident{ + NamePos: rcvr.End(), + Name: "Kind", + Obj: nil, + }, + }, + Lparen: rcvr.End(), + Rparen: rcvr.End(), + }, + Body: &ast.BlockStmt{ + Lbrace: ts.Body.Lbrace, + List: nil, // to be filled in + Rbrace: ts.Body.Rbrace, + }, + } + + // Translate cases. + for _, tcas := range ts.Body.List { + tcas := tcas.(*ast.CaseClause) + cas := &ast.CaseClause{ + Case: tcas.Case, + Colon: tcas.Colon, + Body: tcas.Body, + } + for _, t := range tcas.List { + if isTopName(t, "nil") { + cas.List = append(cas.List, newPkgDot(t.Pos(), "reflect", "Invalid")) + continue + } + + typ := reflectType(t) + if typ == "" { + warn(t.Pos(), "cannot rewrite reflect type switch case with non-reflect type %s", gofmt(t)) + cas.List = append(cas.List, t) + continue + } + + for _, k := range reflectKind[typ] { + cas.List = append(cas.List, newPkgDot(t.Pos(), "reflect", k)) + } + } + sw.Body.List = append(sw.Body.List, cas) + } + + // Everything worked. Rewrite AST. + *ptr = sw + return true +} + +// Rewrite x, y = z.(T) into +// x = z +// y = x.Kind() == K +// as described in the long comment above. +// +// If insert != nil, it can be called to insert a statement after *ptr in its block. +// If insert == nil, insertion is not possible. +// At most one call to insert is allowed. +// +// Scope gives the statements for which a declaration +// in *ptr would be in scope. +// +// The result is true of the statement was rewritten. +// +func reflectFixTypecheck(ptr *ast.Stmt, insert func(ast.Stmt), scope []ast.Stmt) bool { + st := *ptr + as, ok := st.(*ast.AssignStmt) + if !ok || len(as.Lhs) != 2 || len(as.Rhs) != 1 { + return false + } + + ta, ok := as.Rhs[0].(*ast.TypeAssertExpr) + if !ok { + return false + } + typ := reflectType(ta.Type) + if typ == "" { + return false + } + + // Have x, y := z.(t). + x := isIdent(as.Lhs[0]) + y := isIdent(as.Lhs[1]) + z := isIdent(ta.X) + + // First step is x := z, unless it's x := x and the resulting x is never reassigned. + // rcvr is the x in x.Kind(). + var rcvr ast.Expr + if isBlank(x) || + as.Tok == token.DEFINE && x != nil && z != nil && x.Name == z.Name && !assignsTo(x, scope) { + // Can drop the statement. + // If we need to insert a statement later, now we have a slot. + *ptr = &ast.EmptyStmt{} + insert = func(x ast.Stmt) { *ptr = x } + rcvr = ta.X + } else { + *ptr = &ast.AssignStmt{ + Lhs: []ast.Expr{as.Lhs[0]}, + TokPos: as.TokPos, + Tok: as.Tok, + Rhs: []ast.Expr{ta.X}, + } + rcvr = as.Lhs[0] + } + + // Prepare x.Kind() == T expression appropriate to t. + // If x is not a simple identifier, warn that we might be + // reevaluating x. + if x == nil { + warn(as.Pos(), "rewrite reevaluates expr with possible side effects: %s", gofmt(as.Lhs[0])) + } + yExpr, yNotExpr := reflectKindEq(rcvr, reflectKind[typ]) + + // Second step is y := x.Kind() == T, unless it's only used once + // or we have no way to insert that statement. + var yStmt *ast.AssignStmt + if as.Tok == token.DEFINE && countUses(y, scope) <= 1 || insert == nil { + // Can drop the statement and use the expression directly. + rewriteUses(y, + func(token.Pos) ast.Expr { return yExpr }, + func(token.Pos) ast.Expr { return yNotExpr }, + scope) + } else { + yStmt = &ast.AssignStmt{ + Lhs: []ast.Expr{as.Lhs[1]}, + TokPos: as.End(), + Tok: as.Tok, + Rhs: []ast.Expr{yExpr}, + } + insert(yStmt) + } + return true +} + +// reflectKindEq returns the expression z.Kind() == kinds[0] || z.Kind() == kinds[1] || ... +// and its negation. +// The qualifier "reflect." is inserted before each kinds[i] expression. +func reflectKindEq(z ast.Expr, kinds []string) (ast.Expr, ast.Expr) { + n := len(kinds) + if n == 1 { + y := &ast.BinaryExpr{ + X: &ast.CallExpr{ + Fun: &ast.SelectorExpr{ + X: z, + Sel: ast.NewIdent("Kind"), + }, + }, + Op: token.EQL, + Y: newPkgDot(token.NoPos, "reflect", kinds[0]), + } + ynot := &ast.BinaryExpr{ + X: &ast.CallExpr{ + Fun: &ast.SelectorExpr{ + X: z, + Sel: ast.NewIdent("Kind"), + }, + }, + Op: token.NEQ, + Y: newPkgDot(token.NoPos, "reflect", kinds[0]), + } + return y, ynot + } + + x, xnot := reflectKindEq(z, kinds[0:n-1]) + y, ynot := reflectKindEq(z, kinds[n-1:]) + + or := &ast.BinaryExpr{ + X: x, + Op: token.LOR, + Y: y, + } + andnot := &ast.BinaryExpr{ + X: xnot, + Op: token.LAND, + Y: ynot, + } + return or, andnot +} + +// if x represents a known old reflect type/value like *reflect.PtrType or reflect.ArrayOrSliceValue, +// reflectType returns the string form of that type. +func reflectType(x ast.Expr) string { + ptr, ok := x.(*ast.StarExpr) + if ok { + x = ptr.X + } + + sel, ok := x.(*ast.SelectorExpr) + if !ok || !isName(sel.X, "reflect") { + return "" + } + + var s = "reflect." + if ptr != nil { + s = "*reflect." + } + s += sel.Sel.Name + + if reflectKind[s] != nil { + return s + } + return "" +} + +// reflectWarnTypecheckStmt warns about statements +// of the form x, y = z.(T) for any old reflect type T. +// The last pass should have gotten them all, and if it didn't, +// the next pass is going to turn them into x, y = z. +func reflectWarnTypecheckStmt(n interface{}) { + as, ok := n.(*ast.AssignStmt) + if !ok || len(as.Lhs) != 2 || len(as.Rhs) != 1 { + return + } + ta, ok := as.Rhs[0].(*ast.TypeAssertExpr) + if !ok || reflectType(ta.Type) == "" { + return + } + warn(n.(ast.Node).Pos(), "unfixed reflect type check") +} + +// reflectFixAssert rewrites x.(T) to x for any old reflect type T. +func reflectFixAssert(n interface{}) bool { + ptr, ok := n.(*ast.Expr) + if ok { + ta, ok := (*ptr).(*ast.TypeAssertExpr) + if ok && reflectType(ta.Type) != "" { + *ptr = ta.X + return true + } + } + return false +} + +// Tables describing the transformations. + +// Description of old reflect API for partial type checking. +// We pretend the Elem method is on Type and Value instead +// of enumerating all the types it is actually on. +// Also, we pretend that ArrayType etc embeds Type for the +// purposes of describing the API. (In fact they embed commonType, +// which implements Type.) +var reflectTypeConfig = &TypeConfig{ + Type: map[string]*Type{ + "reflect.ArrayOrSliceType": &Type{Embed: []string{"reflect.Type"}}, + "reflect.ArrayOrSliceValue": &Type{Embed: []string{"reflect.Value"}}, + "reflect.ArrayType": &Type{Embed: []string{"reflect.Type"}}, + "reflect.ArrayValue": &Type{Embed: []string{"reflect.Value"}}, + "reflect.BoolType": &Type{Embed: []string{"reflect.Type"}}, + "reflect.BoolValue": &Type{Embed: []string{"reflect.Value"}}, + "reflect.ChanType": &Type{Embed: []string{"reflect.Type"}}, + "reflect.ChanValue": &Type{ + Method: map[string]string{ + "Recv": "func() (reflect.Value, bool)", + "TryRecv": "func() (reflect.Value, bool)", + }, + Embed: []string{"reflect.Value"}, + }, + "reflect.ComplexType": &Type{Embed: []string{"reflect.Type"}}, + "reflect.ComplexValue": &Type{Embed: []string{"reflect.Value"}}, + "reflect.FloatType": &Type{Embed: []string{"reflect.Type"}}, + "reflect.FloatValue": &Type{Embed: []string{"reflect.Value"}}, + "reflect.FuncType": &Type{ + Method: map[string]string{ + "In": "func(int) reflect.Type", + "Out": "func(int) reflect.Type", + }, + Embed: []string{"reflect.Type"}, + }, + "reflect.FuncValue": &Type{ + Method: map[string]string{ + "Call": "func([]reflect.Value) []reflect.Value", + }, + }, + "reflect.IntType": &Type{Embed: []string{"reflect.Type"}}, + "reflect.IntValue": &Type{Embed: []string{"reflect.Value"}}, + "reflect.InterfaceType": &Type{Embed: []string{"reflect.Type"}}, + "reflect.InterfaceValue": &Type{Embed: []string{"reflect.Value"}}, + "reflect.MapType": &Type{ + Method: map[string]string{ + "Key": "func() reflect.Type", + }, + Embed: []string{"reflect.Type"}, + }, + "reflect.MapValue": &Type{ + Method: map[string]string{ + "Keys": "func() []reflect.Value", + }, + Embed: []string{"reflect.Value"}, + }, + "reflect.Method": &Type{ + Field: map[string]string{ + "Type": "*reflect.FuncType", + "Func": "*reflect.FuncValue", + }, + }, + "reflect.PtrType": &Type{Embed: []string{"reflect.Type"}}, + "reflect.PtrValue": &Type{Embed: []string{"reflect.Value"}}, + "reflect.SliceType": &Type{Embed: []string{"reflect.Type"}}, + "reflect.SliceValue": &Type{ + Method: map[string]string{ + "Slice": "func(int, int) *reflect.SliceValue", + }, + Embed: []string{"reflect.Value"}, + }, + "reflect.StringType": &Type{Embed: []string{"reflect.Type"}}, + "reflect.StringValue": &Type{Embed: []string{"reflect.Value"}}, + "reflect.StructField": &Type{ + Field: map[string]string{ + "Type": "reflect.Type", + }, + }, + "reflect.StructType": &Type{ + Method: map[string]string{ + "Field": "func() reflect.StructField", + "FieldByIndex": "func() reflect.StructField", + "FieldByName": "func() reflect.StructField,bool", + "FieldByNameFunc": "func() reflect.StructField,bool", + }, + Embed: []string{"reflect.Type"}, + }, + "reflect.StructValue": &Type{ + Method: map[string]string{ + "Field": "func() reflect.Value", + "FieldByIndex": "func() reflect.Value", + "FieldByName": "func() reflect.Value", + "FieldByNameFunc": "func() reflect.Value", + }, + Embed: []string{"reflect.Value"}, + }, + "reflect.Type": &Type{ + Method: map[string]string{ + "Elem": "func() reflect.Type", + "Method": "func() reflect.Method", + }, + }, + "reflect.UintType": &Type{Embed: []string{"reflect.Type"}}, + "reflect.UintValue": &Type{Embed: []string{"reflect.Value"}}, + "reflect.UnsafePointerType": &Type{Embed: []string{"reflect.Type"}}, + "reflect.UnsafePointerValue": &Type{Embed: []string{"reflect.Value"}}, + "reflect.Value": &Type{ + Method: map[string]string{ + "Addr": "func() *reflect.PtrValue", + "Elem": "func() reflect.Value", + "Method": "func() *reflect.FuncValue", + "SetValue": "func(reflect.Value)", + }, + }, + }, + Func: map[string]string{ + "reflect.Append": "*reflect.SliceValue", + "reflect.AppendSlice": "*reflect.SliceValue", + "reflect.Indirect": "reflect.Value", + "reflect.MakeSlice": "*reflect.SliceValue", + "reflect.MakeChan": "*reflect.ChanValue", + "reflect.MakeMap": "*reflect.MapValue", + "reflect.MakeZero": "reflect.Value", + "reflect.NewValue": "reflect.Value", + "reflect.PtrTo": "*reflect.PtrType", + "reflect.Typeof": "reflect.Type", + }, +} + +var reflectRewriteMethod = map[string]map[string]string{ + // The type API didn't change much. + "*reflect.ChanType": {"Dir": "ChanDir"}, + "*reflect.FuncType": {"DotDotDot": "IsVariadic"}, + + // The value API has longer names to disambiguate + // methods with different signatures. + "reflect.ArrayOrSliceValue": { // interface, not pointer + "Elem": "Index", + }, + "*reflect.ArrayValue": { + "Elem": "Index", + }, + "*reflect.BoolValue": { + "Get": "Bool", + "Set": "SetBool", + }, + "*reflect.ChanValue": { + "Get": "Pointer", + }, + "*reflect.ComplexValue": { + "Get": "Complex", + "Set": "SetComplex", + "Overflow": "OverflowComplex", + }, + "*reflect.FloatValue": { + "Get": "Float", + "Set": "SetFloat", + "Overflow": "OverflowFloat", + }, + "*reflect.FuncValue": { + "Get": "Pointer", + }, + "*reflect.IntValue": { + "Get": "Int", + "Set": "SetInt", + "Overflow": "OverflowInt", + }, + "*reflect.InterfaceValue": { + "Get": "InterfaceData", + }, + "*reflect.MapValue": { + "Elem": "MapIndex", + "Get": "Pointer", + "Keys": "MapKeys", + "SetElem": "SetMapIndex", + }, + "*reflect.PtrValue": { + "Get": "Pointer", + }, + "*reflect.SliceValue": { + "Elem": "Index", + "Get": "Pointer", + }, + "*reflect.StringValue": { + "Get": "String", + "Set": "SetString", + }, + "*reflect.UintValue": { + "Get": "Uint", + "Set": "SetUint", + "Overflow": "OverflowUint", + }, + "*reflect.UnsafePointerValue": { + "Get": "Pointer", + "Set": "SetPointer", + }, +} + +var reflectKind = map[string][]string{ + "reflect.ArrayOrSliceType": {"Array", "Slice"}, // interface, not pointer + "*reflect.ArrayType": {"Array"}, + "*reflect.BoolType": {"Bool"}, + "*reflect.ChanType": {"Chan"}, + "*reflect.ComplexType": {"Complex64", "Complex128"}, + "*reflect.FloatType": {"Float32", "Float64"}, + "*reflect.FuncType": {"Func"}, + "*reflect.IntType": {"Int", "Int8", "Int16", "Int32", "Int64"}, + "*reflect.InterfaceType": {"Interface"}, + "*reflect.MapType": {"Map"}, + "*reflect.PtrType": {"Ptr"}, + "*reflect.SliceType": {"Slice"}, + "*reflect.StringType": {"String"}, + "*reflect.StructType": {"Struct"}, + "*reflect.UintType": {"Uint", "Uint8", "Uint16", "Uint32", "Uint64", "Uintptr"}, + "*reflect.UnsafePointerType": {"UnsafePointer"}, + + "reflect.ArrayOrSliceValue": {"Array", "Slice"}, // interface, not pointer + "*reflect.ArrayValue": {"Array"}, + "*reflect.BoolValue": {"Bool"}, + "*reflect.ChanValue": {"Chan"}, + "*reflect.ComplexValue": {"Complex64", "Complex128"}, + "*reflect.FloatValue": {"Float32", "Float64"}, + "*reflect.FuncValue": {"Func"}, + "*reflect.IntValue": {"Int", "Int8", "Int16", "Int32", "Int64"}, + "*reflect.InterfaceValue": {"Interface"}, + "*reflect.MapValue": {"Map"}, + "*reflect.PtrValue": {"Ptr"}, + "*reflect.SliceValue": {"Slice"}, + "*reflect.StringValue": {"String"}, + "*reflect.StructValue": {"Struct"}, + "*reflect.UintValue": {"Uint", "Uint8", "Uint16", "Uint32", "Uint64", "Uintptr"}, + "*reflect.UnsafePointerValue": {"UnsafePointer"}, +} + +var isReflectValue = map[string]bool{ + "reflect.ArrayOrSliceValue": true, // interface, not pointer + "*reflect.ArrayValue": true, + "*reflect.BoolValue": true, + "*reflect.ChanValue": true, + "*reflect.ComplexValue": true, + "*reflect.FloatValue": true, + "*reflect.FuncValue": true, + "*reflect.IntValue": true, + "*reflect.InterfaceValue": true, + "*reflect.MapValue": true, + "*reflect.PtrValue": true, + "*reflect.SliceValue": true, + "*reflect.StringValue": true, + "*reflect.StructValue": true, + "*reflect.UintValue": true, + "*reflect.UnsafePointerValue": true, + "reflect.Value": true, // interface, not pointer +} diff --git a/src/cmd/gofix/reflect_test.go b/src/cmd/gofix/reflect_test.go new file mode 100644 index 000000000..00edf30e9 --- /dev/null +++ b/src/cmd/gofix/reflect_test.go @@ -0,0 +1,31 @@ +package main + +import ( + "io/ioutil" + "log" + "path/filepath" +) + +func init() { + addTestCases(reflectTests()) +} + +func reflectTests() []testCase { + var tests []testCase + + names, _ := filepath.Glob("testdata/reflect.*.in") + for _, in := range names { + out := in[:len(in)-len(".in")] + ".out" + inb, err := ioutil.ReadFile(in) + if err != nil { + log.Fatal(err) + } + outb, err := ioutil.ReadFile(out) + if err != nil { + log.Fatal(err) + } + tests = append(tests, testCase{Name: in, In: string(inb), Out: string(outb)}) + } + + return tests +} diff --git a/src/cmd/gofix/signal.go b/src/cmd/gofix/signal.go new file mode 100644 index 000000000..53c338851 --- /dev/null +++ b/src/cmd/gofix/signal.go @@ -0,0 +1,49 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package main + +import ( + "go/ast" + "strings" +) + +func init() { + register(fix{ + "signal", + signal, + `Adapt code to types moved from os/signal to signal. + +http://codereview.appspot.com/4437091 +`, + }) +} + +func signal(f *ast.File) (fixed bool) { + if !imports(f, "os/signal") { + return + } + + walk(f, func(n interface{}) { + s, ok := n.(*ast.SelectorExpr) + + if !ok || !isTopName(s.X, "signal") { + return + } + + sel := s.Sel.String() + if sel == "Signal" || sel == "UnixSignal" || strings.HasPrefix(sel, "SIG") { + s.X = &ast.Ident{Name: "os"} + fixed = true + } + }) + + if fixed { + addImport(f, "os") + if !usesImport(f, "os/signal") { + deleteImport(f, "os/signal") + } + } + return +} diff --git a/src/cmd/gofix/signal_test.go b/src/cmd/gofix/signal_test.go new file mode 100644 index 000000000..4abba3534 --- /dev/null +++ b/src/cmd/gofix/signal_test.go @@ -0,0 +1,94 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package main + +func init() { + addTestCases(signalTests) +} + +var signalTests = []testCase{ + { + Name: "signal.0", + In: `package main + +import ( + _ "a" + "os/signal" + _ "z" +) + +type T1 signal.UnixSignal +type T2 signal.Signal + +func f() { + _ = signal.SIGHUP + _ = signal.Incoming +} +`, + Out: `package main + +import ( + _ "a" + "os" + "os/signal" + _ "z" +) + +type T1 os.UnixSignal +type T2 os.Signal + +func f() { + _ = os.SIGHUP + _ = signal.Incoming +} +`, + }, + { + Name: "signal.1", + In: `package main + +import ( + "os" + "os/signal" +) + +func f() { + var _ os.Error + _ = signal.SIGHUP +} +`, + Out: `package main + +import "os" + +func f() { + var _ os.Error + _ = os.SIGHUP +} +`, + }, + { + Name: "signal.2", + In: `package main + +import "os" +import "os/signal" + +func f() { + var _ os.Error + _ = signal.SIGHUP +} +`, + Out: `package main + +import "os" + +func f() { + var _ os.Error + _ = os.SIGHUP +} +`, + }, +} diff --git a/src/cmd/gofix/sorthelpers.go b/src/cmd/gofix/sorthelpers.go new file mode 100644 index 000000000..4e89fa88f --- /dev/null +++ b/src/cmd/gofix/sorthelpers.go @@ -0,0 +1,46 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package main + +import ( + "go/ast" +) + +func init() { + register(fix{ + "sorthelpers", + sorthelpers, + `Adapt code from sort.Sort[Ints|Float64s|Strings] to sort.[Ints|Float64s|Strings]. +`, + }) +} + +func sorthelpers(f *ast.File) (fixed bool) { + if !imports(f, "sort") { + return + } + + walk(f, func(n interface{}) { + s, ok := n.(*ast.SelectorExpr) + if !ok || !isTopName(s.X, "sort") { + return + } + + switch s.Sel.String() { + case "SortFloat64s": + s.Sel.Name = "Float64s" + case "SortInts": + s.Sel.Name = "Ints" + case "SortStrings": + s.Sel.Name = "Strings" + default: + return + } + + fixed = true + }) + + return +} diff --git a/src/cmd/gofix/sorthelpers_test.go b/src/cmd/gofix/sorthelpers_test.go new file mode 100644 index 000000000..6c37858fd --- /dev/null +++ b/src/cmd/gofix/sorthelpers_test.go @@ -0,0 +1,45 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package main + +func init() { + addTestCases(sorthelpersTests) +} + +var sorthelpersTests = []testCase{ + { + Name: "sortslice.0", + In: `package main + +import ( + "sort" +) + +func main() { + var s []string + sort.SortStrings(s) + var i []ints + sort.SortInts(i) + var f []float64 + sort.SortFloat64s(f) +} +`, + Out: `package main + +import ( + "sort" +) + +func main() { + var s []string + sort.Strings(s) + var i []ints + sort.Ints(i) + var f []float64 + sort.Float64s(f) +} +`, + }, +} diff --git a/src/cmd/gofix/sortslice.go b/src/cmd/gofix/sortslice.go new file mode 100644 index 000000000..7cfa1696b --- /dev/null +++ b/src/cmd/gofix/sortslice.go @@ -0,0 +1,49 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package main + +import ( + "go/ast" +) + +func init() { + register(fix{ + "sortslice", + sortslice, + `Adapt code from sort.[Float64|Int|String]Array to sort.[Float64|Int|String]Slice. + +http://codereview.appspot.com/4602054 +http://codereview.appspot.com/4639041 +`, + }) +} + +func sortslice(f *ast.File) (fixed bool) { + if !imports(f, "sort") { + return + } + + walk(f, func(n interface{}) { + s, ok := n.(*ast.SelectorExpr) + if !ok || !isTopName(s.X, "sort") { + return + } + + switch s.Sel.String() { + case "Float64Array": + s.Sel.Name = "Float64Slice" + case "IntArray": + s.Sel.Name = "IntSlice" + case "StringArray": + s.Sel.Name = "StringSlice" + default: + return + } + + fixed = true + }) + + return +} diff --git a/src/cmd/gofix/sortslice_test.go b/src/cmd/gofix/sortslice_test.go new file mode 100644 index 000000000..404feb26f --- /dev/null +++ b/src/cmd/gofix/sortslice_test.go @@ -0,0 +1,35 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package main + +func init() { + addTestCases(sortsliceTests) +} + +var sortsliceTests = []testCase{ + { + Name: "sortslice.0", + In: `package main + +import ( + "sort" +) + +var _ = sort.Float64Array +var _ = sort.IntArray +var _ = sort.StringArray +`, + Out: `package main + +import ( + "sort" +) + +var _ = sort.Float64Slice +var _ = sort.IntSlice +var _ = sort.StringSlice +`, + }, +} diff --git a/src/cmd/gofix/stringssplit.go b/src/cmd/gofix/stringssplit.go new file mode 100644 index 000000000..4a1fe93d3 --- /dev/null +++ b/src/cmd/gofix/stringssplit.go @@ -0,0 +1,71 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package main + +import ( + "go/ast" + "go/token" +) + +var stringssplitFix = fix{ + "stringssplit", + stringssplit, + `Restore strings.Split to its original meaning and add strings.SplitN. Bytes too. + +http://codereview.appspot.com/4661051 +`, +} + +func init() { + register(stringssplitFix) +} + +func stringssplit(f *ast.File) bool { + if !imports(f, "bytes") && !imports(f, "strings") { + return false + } + + fixed := false + walk(f, func(n interface{}) { + call, ok := n.(*ast.CallExpr) + // func Split(s, sep string, n int) []string + // func SplitAfter(s, sep string, n int) []string + if !ok || len(call.Args) != 3 { + return + } + // Is this our function? + switch { + case isPkgDot(call.Fun, "bytes", "Split"): + case isPkgDot(call.Fun, "bytes", "SplitAfter"): + case isPkgDot(call.Fun, "strings", "Split"): + case isPkgDot(call.Fun, "strings", "SplitAfter"): + default: + return + } + + sel := call.Fun.(*ast.SelectorExpr) + args := call.Args + fixed = true // We're committed. + + // Is the last argument -1? If so, drop the arg. + // (Actually we just look for a negative integer literal.) + // Otherwise, Split->SplitN and keep the arg. + final := args[2] + if unary, ok := final.(*ast.UnaryExpr); ok && unary.Op == token.SUB { + if lit, ok := unary.X.(*ast.BasicLit); ok { + // Is it an integer? If so, it's a negative integer and that's what we're after. + if lit.Kind == token.INT { + // drop the last arg. + call.Args = args[0:2] + return + } + } + } + + // If not, rename and keep the argument list. + sel.Sel.Name += "N" + }) + return fixed +} diff --git a/src/cmd/gofix/stringssplit_test.go b/src/cmd/gofix/stringssplit_test.go new file mode 100644 index 000000000..b925722af --- /dev/null +++ b/src/cmd/gofix/stringssplit_test.go @@ -0,0 +1,51 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package main + +func init() { + addTestCases(stringssplitTests) +} + +var stringssplitTests = []testCase{ + { + Name: "stringssplit.0", + In: `package main + +import ( + "bytes" + "strings" +) + +func f() { + bytes.Split(a, b, c) + bytes.Split(a, b, -1) + bytes.SplitAfter(a, b, c) + bytes.SplitAfter(a, b, -1) + strings.Split(a, b, c) + strings.Split(a, b, -1) + strings.SplitAfter(a, b, c) + strings.SplitAfter(a, b, -1) +} +`, + Out: `package main + +import ( + "bytes" + "strings" +) + +func f() { + bytes.SplitN(a, b, c) + bytes.Split(a, b) + bytes.SplitAfterN(a, b, c) + bytes.SplitAfter(a, b) + strings.SplitN(a, b, c) + strings.Split(a, b) + strings.SplitAfterN(a, b, c) + strings.SplitAfter(a, b) +} +`, + }, +} diff --git a/src/cmd/gofix/testdata/reflect.asn1.go.in b/src/cmd/gofix/testdata/reflect.asn1.go.in new file mode 100644 index 000000000..43128f6b2 --- /dev/null +++ b/src/cmd/gofix/testdata/reflect.asn1.go.in @@ -0,0 +1,814 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// The asn1 package implements parsing of DER-encoded ASN.1 data structures, +// as defined in ITU-T Rec X.690. +// +// See also ``A Layman's Guide to a Subset of ASN.1, BER, and DER,'' +// http://luca.ntop.org/Teaching/Appunti/asn1.html. +package asn1 + +// ASN.1 is a syntax for specifying abstract objects and BER, DER, PER, XER etc +// are different encoding formats for those objects. Here, we'll be dealing +// with DER, the Distinguished Encoding Rules. DER is used in X.509 because +// it's fast to parse and, unlike BER, has a unique encoding for every object. +// When calculating hashes over objects, it's important that the resulting +// bytes be the same at both ends and DER removes this margin of error. +// +// ASN.1 is very complex and this package doesn't attempt to implement +// everything by any means. + +import ( + "fmt" + "os" + "reflect" + "time" +) + +// A StructuralError suggests that the ASN.1 data is valid, but the Go type +// which is receiving it doesn't match. +type StructuralError struct { + Msg string +} + +func (e StructuralError) String() string { return "ASN.1 structure error: " + e.Msg } + +// A SyntaxError suggests that the ASN.1 data is invalid. +type SyntaxError struct { + Msg string +} + +func (e SyntaxError) String() string { return "ASN.1 syntax error: " + e.Msg } + +// We start by dealing with each of the primitive types in turn. + +// BOOLEAN + +func parseBool(bytes []byte) (ret bool, err os.Error) { + if len(bytes) != 1 { + err = SyntaxError{"invalid boolean"} + return + } + + return bytes[0] != 0, nil +} + +// INTEGER + +// parseInt64 treats the given bytes as a big-endian, signed integer and +// returns the result. +func parseInt64(bytes []byte) (ret int64, err os.Error) { + if len(bytes) > 8 { + // We'll overflow an int64 in this case. + err = StructuralError{"integer too large"} + return + } + for bytesRead := 0; bytesRead < len(bytes); bytesRead++ { + ret <<= 8 + ret |= int64(bytes[bytesRead]) + } + + // Shift up and down in order to sign extend the result. + ret <<= 64 - uint8(len(bytes))*8 + ret >>= 64 - uint8(len(bytes))*8 + return +} + +// parseInt treats the given bytes as a big-endian, signed integer and returns +// the result. +func parseInt(bytes []byte) (int, os.Error) { + ret64, err := parseInt64(bytes) + if err != nil { + return 0, err + } + if ret64 != int64(int(ret64)) { + return 0, StructuralError{"integer too large"} + } + return int(ret64), nil +} + +// BIT STRING + +// BitString is the structure to use when you want an ASN.1 BIT STRING type. A +// bit string is padded up to the nearest byte in memory and the number of +// valid bits is recorded. Padding bits will be zero. +type BitString struct { + Bytes []byte // bits packed into bytes. + BitLength int // length in bits. +} + +// At returns the bit at the given index. If the index is out of range it +// returns false. +func (b BitString) At(i int) int { + if i < 0 || i >= b.BitLength { + return 0 + } + x := i / 8 + y := 7 - uint(i%8) + return int(b.Bytes[x]>>y) & 1 +} + +// RightAlign returns a slice where the padding bits are at the beginning. The +// slice may share memory with the BitString. +func (b BitString) RightAlign() []byte { + shift := uint(8 - (b.BitLength % 8)) + if shift == 8 || len(b.Bytes) == 0 { + return b.Bytes + } + + a := make([]byte, len(b.Bytes)) + a[0] = b.Bytes[0] >> shift + for i := 1; i < len(b.Bytes); i++ { + a[i] = b.Bytes[i-1] << (8 - shift) + a[i] |= b.Bytes[i] >> shift + } + + return a +} + +// parseBitString parses an ASN.1 bit string from the given byte array and returns it. +func parseBitString(bytes []byte) (ret BitString, err os.Error) { + if len(bytes) == 0 { + err = SyntaxError{"zero length BIT STRING"} + return + } + paddingBits := int(bytes[0]) + if paddingBits > 7 || + len(bytes) == 1 && paddingBits > 0 || + bytes[len(bytes)-1]&((1< 4 { + err = StructuralError{"base 128 integer too large"} + return + } + ret <<= 7 + b := bytes[offset] + ret |= int(b & 0x7f) + offset++ + if b&0x80 == 0 { + return + } + } + err = SyntaxError{"truncated base 128 integer"} + return +} + +// UTCTime + +func parseUTCTime(bytes []byte) (ret *time.Time, err os.Error) { + s := string(bytes) + ret, err = time.Parse("0601021504Z0700", s) + if err == nil { + return + } + ret, err = time.Parse("060102150405Z0700", s) + return +} + +// parseGeneralizedTime parses the GeneralizedTime from the given byte array +// and returns the resulting time. +func parseGeneralizedTime(bytes []byte) (ret *time.Time, err os.Error) { + return time.Parse("20060102150405Z0700", string(bytes)) +} + +// PrintableString + +// parsePrintableString parses a ASN.1 PrintableString from the given byte +// array and returns it. +func parsePrintableString(bytes []byte) (ret string, err os.Error) { + for _, b := range bytes { + if !isPrintable(b) { + err = SyntaxError{"PrintableString contains invalid character"} + return + } + } + ret = string(bytes) + return +} + +// isPrintable returns true iff the given b is in the ASN.1 PrintableString set. +func isPrintable(b byte) bool { + return 'a' <= b && b <= 'z' || + 'A' <= b && b <= 'Z' || + '0' <= b && b <= '9' || + '\'' <= b && b <= ')' || + '+' <= b && b <= '/' || + b == ' ' || + b == ':' || + b == '=' || + b == '?' || + // This is techincally not allowed in a PrintableString. + // However, x509 certificates with wildcard strings don't + // always use the correct string type so we permit it. + b == '*' +} + +// IA5String + +// parseIA5String parses a ASN.1 IA5String (ASCII string) from the given +// byte array and returns it. +func parseIA5String(bytes []byte) (ret string, err os.Error) { + for _, b := range bytes { + if b >= 0x80 { + err = SyntaxError{"IA5String contains invalid character"} + return + } + } + ret = string(bytes) + return +} + +// T61String + +// parseT61String parses a ASN.1 T61String (8-bit clean string) from the given +// byte array and returns it. +func parseT61String(bytes []byte) (ret string, err os.Error) { + return string(bytes), nil +} + +// A RawValue represents an undecoded ASN.1 object. +type RawValue struct { + Class, Tag int + IsCompound bool + Bytes []byte + FullBytes []byte // includes the tag and length +} + +// RawContent is used to signal that the undecoded, DER data needs to be +// preserved for a struct. To use it, the first field of the struct must have +// this type. It's an error for any of the other fields to have this type. +type RawContent []byte + +// Tagging + +// parseTagAndLength parses an ASN.1 tag and length pair from the given offset +// into a byte array. It returns the parsed data and the new offset. SET and +// SET OF (tag 17) are mapped to SEQUENCE and SEQUENCE OF (tag 16) since we +// don't distinguish between ordered and unordered objects in this code. +func parseTagAndLength(bytes []byte, initOffset int) (ret tagAndLength, offset int, err os.Error) { + offset = initOffset + b := bytes[offset] + offset++ + ret.class = int(b >> 6) + ret.isCompound = b&0x20 == 0x20 + ret.tag = int(b & 0x1f) + + // If the bottom five bits are set, then the tag number is actually base 128 + // encoded afterwards + if ret.tag == 0x1f { + ret.tag, offset, err = parseBase128Int(bytes, offset) + if err != nil { + return + } + } + if offset >= len(bytes) { + err = SyntaxError{"truncated tag or length"} + return + } + b = bytes[offset] + offset++ + if b&0x80 == 0 { + // The length is encoded in the bottom 7 bits. + ret.length = int(b & 0x7f) + } else { + // Bottom 7 bits give the number of length bytes to follow. + numBytes := int(b & 0x7f) + // We risk overflowing a signed 32-bit number if we accept more than 3 bytes. + if numBytes > 3 { + err = StructuralError{"length too large"} + return + } + if numBytes == 0 { + err = SyntaxError{"indefinite length found (not DER)"} + return + } + ret.length = 0 + for i := 0; i < numBytes; i++ { + if offset >= len(bytes) { + err = SyntaxError{"truncated tag or length"} + return + } + b = bytes[offset] + offset++ + ret.length <<= 8 + ret.length |= int(b) + } + } + + return +} + +// parseSequenceOf is used for SEQUENCE OF and SET OF values. It tries to parse +// a number of ASN.1 values from the given byte array and returns them as a +// slice of Go values of the given type. +func parseSequenceOf(bytes []byte, sliceType *reflect.SliceType, elemType reflect.Type) (ret *reflect.SliceValue, err os.Error) { + expectedTag, compoundType, ok := getUniversalType(elemType) + if !ok { + err = StructuralError{"unknown Go type for slice"} + return + } + + // First we iterate over the input and count the number of elements, + // checking that the types are correct in each case. + numElements := 0 + for offset := 0; offset < len(bytes); { + var t tagAndLength + t, offset, err = parseTagAndLength(bytes, offset) + if err != nil { + return + } + // We pretend that GENERAL STRINGs are PRINTABLE STRINGs so + // that a sequence of them can be parsed into a []string. + if t.tag == tagGeneralString { + t.tag = tagPrintableString + } + if t.class != classUniversal || t.isCompound != compoundType || t.tag != expectedTag { + err = StructuralError{"sequence tag mismatch"} + return + } + if invalidLength(offset, t.length, len(bytes)) { + err = SyntaxError{"truncated sequence"} + return + } + offset += t.length + numElements++ + } + ret = reflect.MakeSlice(sliceType, numElements, numElements) + params := fieldParameters{} + offset := 0 + for i := 0; i < numElements; i++ { + offset, err = parseField(ret.Elem(i), bytes, offset, params) + if err != nil { + return + } + } + return +} + +var ( + bitStringType = reflect.Typeof(BitString{}) + objectIdentifierType = reflect.Typeof(ObjectIdentifier{}) + enumeratedType = reflect.Typeof(Enumerated(0)) + flagType = reflect.Typeof(Flag(false)) + timeType = reflect.Typeof(&time.Time{}) + rawValueType = reflect.Typeof(RawValue{}) + rawContentsType = reflect.Typeof(RawContent(nil)) +) + +// invalidLength returns true iff offset + length > sliceLength, or if the +// addition would overflow. +func invalidLength(offset, length, sliceLength int) bool { + return offset+length < offset || offset+length > sliceLength +} + +// parseField is the main parsing function. Given a byte array and an offset +// into the array, it will try to parse a suitable ASN.1 value out and store it +// in the given Value. +func parseField(v reflect.Value, bytes []byte, initOffset int, params fieldParameters) (offset int, err os.Error) { + offset = initOffset + fieldType := v.Type() + + // If we have run out of data, it may be that there are optional elements at the end. + if offset == len(bytes) { + if !setDefaultValue(v, params) { + err = SyntaxError{"sequence truncated"} + } + return + } + + // Deal with raw values. + if fieldType == rawValueType { + var t tagAndLength + t, offset, err = parseTagAndLength(bytes, offset) + if err != nil { + return + } + if invalidLength(offset, t.length, len(bytes)) { + err = SyntaxError{"data truncated"} + return + } + result := RawValue{t.class, t.tag, t.isCompound, bytes[offset : offset+t.length], bytes[initOffset : offset+t.length]} + offset += t.length + v.(*reflect.StructValue).Set(reflect.NewValue(result).(*reflect.StructValue)) + return + } + + // Deal with the ANY type. + if ifaceType, ok := fieldType.(*reflect.InterfaceType); ok && ifaceType.NumMethod() == 0 { + ifaceValue := v.(*reflect.InterfaceValue) + var t tagAndLength + t, offset, err = parseTagAndLength(bytes, offset) + if err != nil { + return + } + if invalidLength(offset, t.length, len(bytes)) { + err = SyntaxError{"data truncated"} + return + } + var result interface{} + if !t.isCompound && t.class == classUniversal { + innerBytes := bytes[offset : offset+t.length] + switch t.tag { + case tagPrintableString: + result, err = parsePrintableString(innerBytes) + case tagIA5String: + result, err = parseIA5String(innerBytes) + case tagT61String: + result, err = parseT61String(innerBytes) + case tagInteger: + result, err = parseInt64(innerBytes) + case tagBitString: + result, err = parseBitString(innerBytes) + case tagOID: + result, err = parseObjectIdentifier(innerBytes) + case tagUTCTime: + result, err = parseUTCTime(innerBytes) + case tagOctetString: + result = innerBytes + default: + // If we don't know how to handle the type, we just leave Value as nil. + } + } + offset += t.length + if err != nil { + return + } + if result != nil { + ifaceValue.Set(reflect.NewValue(result)) + } + return + } + universalTag, compoundType, ok1 := getUniversalType(fieldType) + if !ok1 { + err = StructuralError{fmt.Sprintf("unknown Go type: %v", fieldType)} + return + } + + t, offset, err := parseTagAndLength(bytes, offset) + if err != nil { + return + } + if params.explicit { + expectedClass := classContextSpecific + if params.application { + expectedClass = classApplication + } + if t.class == expectedClass && t.tag == *params.tag && (t.length == 0 || t.isCompound) { + if t.length > 0 { + t, offset, err = parseTagAndLength(bytes, offset) + if err != nil { + return + } + } else { + if fieldType != flagType { + err = StructuralError{"Zero length explicit tag was not an asn1.Flag"} + return + } + + flagValue := v.(*reflect.BoolValue) + flagValue.Set(true) + return + } + } else { + // The tags didn't match, it might be an optional element. + ok := setDefaultValue(v, params) + if ok { + offset = initOffset + } else { + err = StructuralError{"explicitly tagged member didn't match"} + } + return + } + } + + // Special case for strings: PrintableString and IA5String both map to + // the Go type string. getUniversalType returns the tag for + // PrintableString when it sees a string so, if we see an IA5String on + // the wire, we change the universal type to match. + if universalTag == tagPrintableString && t.tag == tagIA5String { + universalTag = tagIA5String + } + // Likewise for GeneralString + if universalTag == tagPrintableString && t.tag == tagGeneralString { + universalTag = tagGeneralString + } + + // Special case for time: UTCTime and GeneralizedTime both map to the + // Go type time.Time. + if universalTag == tagUTCTime && t.tag == tagGeneralizedTime { + universalTag = tagGeneralizedTime + } + + expectedClass := classUniversal + expectedTag := universalTag + + if !params.explicit && params.tag != nil { + expectedClass = classContextSpecific + expectedTag = *params.tag + } + + if !params.explicit && params.application && params.tag != nil { + expectedClass = classApplication + expectedTag = *params.tag + } + + // We have unwrapped any explicit tagging at this point. + if t.class != expectedClass || t.tag != expectedTag || t.isCompound != compoundType { + // Tags don't match. Again, it could be an optional element. + ok := setDefaultValue(v, params) + if ok { + offset = initOffset + } else { + err = StructuralError{fmt.Sprintf("tags don't match (%d vs %+v) %+v %s @%d", expectedTag, t, params, fieldType.Name(), offset)} + } + return + } + if invalidLength(offset, t.length, len(bytes)) { + err = SyntaxError{"data truncated"} + return + } + innerBytes := bytes[offset : offset+t.length] + offset += t.length + + // We deal with the structures defined in this package first. + switch fieldType { + case objectIdentifierType: + newSlice, err1 := parseObjectIdentifier(innerBytes) + sliceValue := v.(*reflect.SliceValue) + sliceValue.Set(reflect.MakeSlice(sliceValue.Type().(*reflect.SliceType), len(newSlice), len(newSlice))) + if err1 == nil { + reflect.Copy(sliceValue, reflect.NewValue(newSlice).(reflect.ArrayOrSliceValue)) + } + err = err1 + return + case bitStringType: + structValue := v.(*reflect.StructValue) + bs, err1 := parseBitString(innerBytes) + if err1 == nil { + structValue.Set(reflect.NewValue(bs).(*reflect.StructValue)) + } + err = err1 + return + case timeType: + ptrValue := v.(*reflect.PtrValue) + var time *time.Time + var err1 os.Error + if universalTag == tagUTCTime { + time, err1 = parseUTCTime(innerBytes) + } else { + time, err1 = parseGeneralizedTime(innerBytes) + } + if err1 == nil { + ptrValue.Set(reflect.NewValue(time).(*reflect.PtrValue)) + } + err = err1 + return + case enumeratedType: + parsedInt, err1 := parseInt(innerBytes) + enumValue := v.(*reflect.IntValue) + if err1 == nil { + enumValue.Set(int64(parsedInt)) + } + err = err1 + return + case flagType: + flagValue := v.(*reflect.BoolValue) + flagValue.Set(true) + return + } + switch val := v.(type) { + case *reflect.BoolValue: + parsedBool, err1 := parseBool(innerBytes) + if err1 == nil { + val.Set(parsedBool) + } + err = err1 + return + case *reflect.IntValue: + switch val.Type().Kind() { + case reflect.Int: + parsedInt, err1 := parseInt(innerBytes) + if err1 == nil { + val.Set(int64(parsedInt)) + } + err = err1 + return + case reflect.Int64: + parsedInt, err1 := parseInt64(innerBytes) + if err1 == nil { + val.Set(parsedInt) + } + err = err1 + return + } + case *reflect.StructValue: + structType := fieldType.(*reflect.StructType) + + if structType.NumField() > 0 && + structType.Field(0).Type == rawContentsType { + bytes := bytes[initOffset:offset] + val.Field(0).SetValue(reflect.NewValue(RawContent(bytes))) + } + + innerOffset := 0 + for i := 0; i < structType.NumField(); i++ { + field := structType.Field(i) + if i == 0 && field.Type == rawContentsType { + continue + } + innerOffset, err = parseField(val.Field(i), innerBytes, innerOffset, parseFieldParameters(field.Tag)) + if err != nil { + return + } + } + // We allow extra bytes at the end of the SEQUENCE because + // adding elements to the end has been used in X.509 as the + // version numbers have increased. + return + case *reflect.SliceValue: + sliceType := fieldType.(*reflect.SliceType) + if sliceType.Elem().Kind() == reflect.Uint8 { + val.Set(reflect.MakeSlice(sliceType, len(innerBytes), len(innerBytes))) + reflect.Copy(val, reflect.NewValue(innerBytes).(reflect.ArrayOrSliceValue)) + return + } + newSlice, err1 := parseSequenceOf(innerBytes, sliceType, sliceType.Elem()) + if err1 == nil { + val.Set(newSlice) + } + err = err1 + return + case *reflect.StringValue: + var v string + switch universalTag { + case tagPrintableString: + v, err = parsePrintableString(innerBytes) + case tagIA5String: + v, err = parseIA5String(innerBytes) + case tagT61String: + v, err = parseT61String(innerBytes) + case tagGeneralString: + // GeneralString is specified in ISO-2022/ECMA-35, + // A brief review suggests that it includes structures + // that allow the encoding to change midstring and + // such. We give up and pass it as an 8-bit string. + v, err = parseT61String(innerBytes) + default: + err = SyntaxError{fmt.Sprintf("internal error: unknown string type %d", universalTag)} + } + if err == nil { + val.Set(v) + } + return + } + err = StructuralError{"unknown Go type"} + return +} + +// setDefaultValue is used to install a default value, from a tag string, into +// a Value. It is successful is the field was optional, even if a default value +// wasn't provided or it failed to install it into the Value. +func setDefaultValue(v reflect.Value, params fieldParameters) (ok bool) { + if !params.optional { + return + } + ok = true + if params.defaultValue == nil { + return + } + switch val := v.(type) { + case *reflect.IntValue: + val.Set(*params.defaultValue) + } + return +} + +// Unmarshal parses the DER-encoded ASN.1 data structure b +// and uses the reflect package to fill in an arbitrary value pointed at by val. +// Because Unmarshal uses the reflect package, the structs +// being written to must use upper case field names. +// +// An ASN.1 INTEGER can be written to an int or int64. +// If the encoded value does not fit in the Go type, +// Unmarshal returns a parse error. +// +// An ASN.1 BIT STRING can be written to a BitString. +// +// An ASN.1 OCTET STRING can be written to a []byte. +// +// An ASN.1 OBJECT IDENTIFIER can be written to an +// ObjectIdentifier. +// +// An ASN.1 ENUMERATED can be written to an Enumerated. +// +// An ASN.1 UTCTIME or GENERALIZEDTIME can be written to a *time.Time. +// +// An ASN.1 PrintableString or IA5String can be written to a string. +// +// Any of the above ASN.1 values can be written to an interface{}. +// The value stored in the interface has the corresponding Go type. +// For integers, that type is int64. +// +// An ASN.1 SEQUENCE OF x or SET OF x can be written +// to a slice if an x can be written to the slice's element type. +// +// An ASN.1 SEQUENCE or SET can be written to a struct +// if each of the elements in the sequence can be +// written to the corresponding element in the struct. +// +// The following tags on struct fields have special meaning to Unmarshal: +// +// optional marks the field as ASN.1 OPTIONAL +// [explicit] tag:x specifies the ASN.1 tag number; implies ASN.1 CONTEXT SPECIFIC +// default:x sets the default value for optional integer fields +// +// If the type of the first field of a structure is RawContent then the raw +// ASN1 contents of the struct will be stored in it. +// +// Other ASN.1 types are not supported; if it encounters them, +// Unmarshal returns a parse error. +func Unmarshal(b []byte, val interface{}) (rest []byte, err os.Error) { + return UnmarshalWithParams(b, val, "") +} + +// UnmarshalWithParams allows field parameters to be specified for the +// top-level element. The form of the params is the same as the field tags. +func UnmarshalWithParams(b []byte, val interface{}, params string) (rest []byte, err os.Error) { + v := reflect.NewValue(val).(*reflect.PtrValue).Elem() + offset, err := parseField(v, b, 0, parseFieldParameters(params)) + if err != nil { + return nil, err + } + return b[offset:], nil +} diff --git a/src/cmd/gofix/testdata/reflect.asn1.go.out b/src/cmd/gofix/testdata/reflect.asn1.go.out new file mode 100644 index 000000000..ba6224e6d --- /dev/null +++ b/src/cmd/gofix/testdata/reflect.asn1.go.out @@ -0,0 +1,814 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// The asn1 package implements parsing of DER-encoded ASN.1 data structures, +// as defined in ITU-T Rec X.690. +// +// See also ``A Layman's Guide to a Subset of ASN.1, BER, and DER,'' +// http://luca.ntop.org/Teaching/Appunti/asn1.html. +package asn1 + +// ASN.1 is a syntax for specifying abstract objects and BER, DER, PER, XER etc +// are different encoding formats for those objects. Here, we'll be dealing +// with DER, the Distinguished Encoding Rules. DER is used in X.509 because +// it's fast to parse and, unlike BER, has a unique encoding for every object. +// When calculating hashes over objects, it's important that the resulting +// bytes be the same at both ends and DER removes this margin of error. +// +// ASN.1 is very complex and this package doesn't attempt to implement +// everything by any means. + +import ( + "fmt" + "os" + "reflect" + "time" +) + +// A StructuralError suggests that the ASN.1 data is valid, but the Go type +// which is receiving it doesn't match. +type StructuralError struct { + Msg string +} + +func (e StructuralError) String() string { return "ASN.1 structure error: " + e.Msg } + +// A SyntaxError suggests that the ASN.1 data is invalid. +type SyntaxError struct { + Msg string +} + +func (e SyntaxError) String() string { return "ASN.1 syntax error: " + e.Msg } + +// We start by dealing with each of the primitive types in turn. + +// BOOLEAN + +func parseBool(bytes []byte) (ret bool, err os.Error) { + if len(bytes) != 1 { + err = SyntaxError{"invalid boolean"} + return + } + + return bytes[0] != 0, nil +} + +// INTEGER + +// parseInt64 treats the given bytes as a big-endian, signed integer and +// returns the result. +func parseInt64(bytes []byte) (ret int64, err os.Error) { + if len(bytes) > 8 { + // We'll overflow an int64 in this case. + err = StructuralError{"integer too large"} + return + } + for bytesRead := 0; bytesRead < len(bytes); bytesRead++ { + ret <<= 8 + ret |= int64(bytes[bytesRead]) + } + + // Shift up and down in order to sign extend the result. + ret <<= 64 - uint8(len(bytes))*8 + ret >>= 64 - uint8(len(bytes))*8 + return +} + +// parseInt treats the given bytes as a big-endian, signed integer and returns +// the result. +func parseInt(bytes []byte) (int, os.Error) { + ret64, err := parseInt64(bytes) + if err != nil { + return 0, err + } + if ret64 != int64(int(ret64)) { + return 0, StructuralError{"integer too large"} + } + return int(ret64), nil +} + +// BIT STRING + +// BitString is the structure to use when you want an ASN.1 BIT STRING type. A +// bit string is padded up to the nearest byte in memory and the number of +// valid bits is recorded. Padding bits will be zero. +type BitString struct { + Bytes []byte // bits packed into bytes. + BitLength int // length in bits. +} + +// At returns the bit at the given index. If the index is out of range it +// returns false. +func (b BitString) At(i int) int { + if i < 0 || i >= b.BitLength { + return 0 + } + x := i / 8 + y := 7 - uint(i%8) + return int(b.Bytes[x]>>y) & 1 +} + +// RightAlign returns a slice where the padding bits are at the beginning. The +// slice may share memory with the BitString. +func (b BitString) RightAlign() []byte { + shift := uint(8 - (b.BitLength % 8)) + if shift == 8 || len(b.Bytes) == 0 { + return b.Bytes + } + + a := make([]byte, len(b.Bytes)) + a[0] = b.Bytes[0] >> shift + for i := 1; i < len(b.Bytes); i++ { + a[i] = b.Bytes[i-1] << (8 - shift) + a[i] |= b.Bytes[i] >> shift + } + + return a +} + +// parseBitString parses an ASN.1 bit string from the given byte array and returns it. +func parseBitString(bytes []byte) (ret BitString, err os.Error) { + if len(bytes) == 0 { + err = SyntaxError{"zero length BIT STRING"} + return + } + paddingBits := int(bytes[0]) + if paddingBits > 7 || + len(bytes) == 1 && paddingBits > 0 || + bytes[len(bytes)-1]&((1< 4 { + err = StructuralError{"base 128 integer too large"} + return + } + ret <<= 7 + b := bytes[offset] + ret |= int(b & 0x7f) + offset++ + if b&0x80 == 0 { + return + } + } + err = SyntaxError{"truncated base 128 integer"} + return +} + +// UTCTime + +func parseUTCTime(bytes []byte) (ret *time.Time, err os.Error) { + s := string(bytes) + ret, err = time.Parse("0601021504Z0700", s) + if err == nil { + return + } + ret, err = time.Parse("060102150405Z0700", s) + return +} + +// parseGeneralizedTime parses the GeneralizedTime from the given byte array +// and returns the resulting time. +func parseGeneralizedTime(bytes []byte) (ret *time.Time, err os.Error) { + return time.Parse("20060102150405Z0700", string(bytes)) +} + +// PrintableString + +// parsePrintableString parses a ASN.1 PrintableString from the given byte +// array and returns it. +func parsePrintableString(bytes []byte) (ret string, err os.Error) { + for _, b := range bytes { + if !isPrintable(b) { + err = SyntaxError{"PrintableString contains invalid character"} + return + } + } + ret = string(bytes) + return +} + +// isPrintable returns true iff the given b is in the ASN.1 PrintableString set. +func isPrintable(b byte) bool { + return 'a' <= b && b <= 'z' || + 'A' <= b && b <= 'Z' || + '0' <= b && b <= '9' || + '\'' <= b && b <= ')' || + '+' <= b && b <= '/' || + b == ' ' || + b == ':' || + b == '=' || + b == '?' || + // This is techincally not allowed in a PrintableString. + // However, x509 certificates with wildcard strings don't + // always use the correct string type so we permit it. + b == '*' +} + +// IA5String + +// parseIA5String parses a ASN.1 IA5String (ASCII string) from the given +// byte array and returns it. +func parseIA5String(bytes []byte) (ret string, err os.Error) { + for _, b := range bytes { + if b >= 0x80 { + err = SyntaxError{"IA5String contains invalid character"} + return + } + } + ret = string(bytes) + return +} + +// T61String + +// parseT61String parses a ASN.1 T61String (8-bit clean string) from the given +// byte array and returns it. +func parseT61String(bytes []byte) (ret string, err os.Error) { + return string(bytes), nil +} + +// A RawValue represents an undecoded ASN.1 object. +type RawValue struct { + Class, Tag int + IsCompound bool + Bytes []byte + FullBytes []byte // includes the tag and length +} + +// RawContent is used to signal that the undecoded, DER data needs to be +// preserved for a struct. To use it, the first field of the struct must have +// this type. It's an error for any of the other fields to have this type. +type RawContent []byte + +// Tagging + +// parseTagAndLength parses an ASN.1 tag and length pair from the given offset +// into a byte array. It returns the parsed data and the new offset. SET and +// SET OF (tag 17) are mapped to SEQUENCE and SEQUENCE OF (tag 16) since we +// don't distinguish between ordered and unordered objects in this code. +func parseTagAndLength(bytes []byte, initOffset int) (ret tagAndLength, offset int, err os.Error) { + offset = initOffset + b := bytes[offset] + offset++ + ret.class = int(b >> 6) + ret.isCompound = b&0x20 == 0x20 + ret.tag = int(b & 0x1f) + + // If the bottom five bits are set, then the tag number is actually base 128 + // encoded afterwards + if ret.tag == 0x1f { + ret.tag, offset, err = parseBase128Int(bytes, offset) + if err != nil { + return + } + } + if offset >= len(bytes) { + err = SyntaxError{"truncated tag or length"} + return + } + b = bytes[offset] + offset++ + if b&0x80 == 0 { + // The length is encoded in the bottom 7 bits. + ret.length = int(b & 0x7f) + } else { + // Bottom 7 bits give the number of length bytes to follow. + numBytes := int(b & 0x7f) + // We risk overflowing a signed 32-bit number if we accept more than 3 bytes. + if numBytes > 3 { + err = StructuralError{"length too large"} + return + } + if numBytes == 0 { + err = SyntaxError{"indefinite length found (not DER)"} + return + } + ret.length = 0 + for i := 0; i < numBytes; i++ { + if offset >= len(bytes) { + err = SyntaxError{"truncated tag or length"} + return + } + b = bytes[offset] + offset++ + ret.length <<= 8 + ret.length |= int(b) + } + } + + return +} + +// parseSequenceOf is used for SEQUENCE OF and SET OF values. It tries to parse +// a number of ASN.1 values from the given byte array and returns them as a +// slice of Go values of the given type. +func parseSequenceOf(bytes []byte, sliceType reflect.Type, elemType reflect.Type) (ret reflect.Value, err os.Error) { + expectedTag, compoundType, ok := getUniversalType(elemType) + if !ok { + err = StructuralError{"unknown Go type for slice"} + return + } + + // First we iterate over the input and count the number of elements, + // checking that the types are correct in each case. + numElements := 0 + for offset := 0; offset < len(bytes); { + var t tagAndLength + t, offset, err = parseTagAndLength(bytes, offset) + if err != nil { + return + } + // We pretend that GENERAL STRINGs are PRINTABLE STRINGs so + // that a sequence of them can be parsed into a []string. + if t.tag == tagGeneralString { + t.tag = tagPrintableString + } + if t.class != classUniversal || t.isCompound != compoundType || t.tag != expectedTag { + err = StructuralError{"sequence tag mismatch"} + return + } + if invalidLength(offset, t.length, len(bytes)) { + err = SyntaxError{"truncated sequence"} + return + } + offset += t.length + numElements++ + } + ret = reflect.MakeSlice(sliceType, numElements, numElements) + params := fieldParameters{} + offset := 0 + for i := 0; i < numElements; i++ { + offset, err = parseField(ret.Index(i), bytes, offset, params) + if err != nil { + return + } + } + return +} + +var ( + bitStringType = reflect.TypeOf(BitString{}) + objectIdentifierType = reflect.TypeOf(ObjectIdentifier{}) + enumeratedType = reflect.TypeOf(Enumerated(0)) + flagType = reflect.TypeOf(Flag(false)) + timeType = reflect.TypeOf(&time.Time{}) + rawValueType = reflect.TypeOf(RawValue{}) + rawContentsType = reflect.TypeOf(RawContent(nil)) +) + +// invalidLength returns true iff offset + length > sliceLength, or if the +// addition would overflow. +func invalidLength(offset, length, sliceLength int) bool { + return offset+length < offset || offset+length > sliceLength +} + +// parseField is the main parsing function. Given a byte array and an offset +// into the array, it will try to parse a suitable ASN.1 value out and store it +// in the given Value. +func parseField(v reflect.Value, bytes []byte, initOffset int, params fieldParameters) (offset int, err os.Error) { + offset = initOffset + fieldType := v.Type() + + // If we have run out of data, it may be that there are optional elements at the end. + if offset == len(bytes) { + if !setDefaultValue(v, params) { + err = SyntaxError{"sequence truncated"} + } + return + } + + // Deal with raw values. + if fieldType == rawValueType { + var t tagAndLength + t, offset, err = parseTagAndLength(bytes, offset) + if err != nil { + return + } + if invalidLength(offset, t.length, len(bytes)) { + err = SyntaxError{"data truncated"} + return + } + result := RawValue{t.class, t.tag, t.isCompound, bytes[offset : offset+t.length], bytes[initOffset : offset+t.length]} + offset += t.length + v.Set(reflect.ValueOf(result)) + return + } + + // Deal with the ANY type. + if ifaceType := fieldType; ifaceType.Kind() == reflect.Interface && ifaceType.NumMethod() == 0 { + ifaceValue := v + var t tagAndLength + t, offset, err = parseTagAndLength(bytes, offset) + if err != nil { + return + } + if invalidLength(offset, t.length, len(bytes)) { + err = SyntaxError{"data truncated"} + return + } + var result interface{} + if !t.isCompound && t.class == classUniversal { + innerBytes := bytes[offset : offset+t.length] + switch t.tag { + case tagPrintableString: + result, err = parsePrintableString(innerBytes) + case tagIA5String: + result, err = parseIA5String(innerBytes) + case tagT61String: + result, err = parseT61String(innerBytes) + case tagInteger: + result, err = parseInt64(innerBytes) + case tagBitString: + result, err = parseBitString(innerBytes) + case tagOID: + result, err = parseObjectIdentifier(innerBytes) + case tagUTCTime: + result, err = parseUTCTime(innerBytes) + case tagOctetString: + result = innerBytes + default: + // If we don't know how to handle the type, we just leave Value as nil. + } + } + offset += t.length + if err != nil { + return + } + if result != nil { + ifaceValue.Set(reflect.ValueOf(result)) + } + return + } + universalTag, compoundType, ok1 := getUniversalType(fieldType) + if !ok1 { + err = StructuralError{fmt.Sprintf("unknown Go type: %v", fieldType)} + return + } + + t, offset, err := parseTagAndLength(bytes, offset) + if err != nil { + return + } + if params.explicit { + expectedClass := classContextSpecific + if params.application { + expectedClass = classApplication + } + if t.class == expectedClass && t.tag == *params.tag && (t.length == 0 || t.isCompound) { + if t.length > 0 { + t, offset, err = parseTagAndLength(bytes, offset) + if err != nil { + return + } + } else { + if fieldType != flagType { + err = StructuralError{"Zero length explicit tag was not an asn1.Flag"} + return + } + + flagValue := v + flagValue.SetBool(true) + return + } + } else { + // The tags didn't match, it might be an optional element. + ok := setDefaultValue(v, params) + if ok { + offset = initOffset + } else { + err = StructuralError{"explicitly tagged member didn't match"} + } + return + } + } + + // Special case for strings: PrintableString and IA5String both map to + // the Go type string. getUniversalType returns the tag for + // PrintableString when it sees a string so, if we see an IA5String on + // the wire, we change the universal type to match. + if universalTag == tagPrintableString && t.tag == tagIA5String { + universalTag = tagIA5String + } + // Likewise for GeneralString + if universalTag == tagPrintableString && t.tag == tagGeneralString { + universalTag = tagGeneralString + } + + // Special case for time: UTCTime and GeneralizedTime both map to the + // Go type time.Time. + if universalTag == tagUTCTime && t.tag == tagGeneralizedTime { + universalTag = tagGeneralizedTime + } + + expectedClass := classUniversal + expectedTag := universalTag + + if !params.explicit && params.tag != nil { + expectedClass = classContextSpecific + expectedTag = *params.tag + } + + if !params.explicit && params.application && params.tag != nil { + expectedClass = classApplication + expectedTag = *params.tag + } + + // We have unwrapped any explicit tagging at this point. + if t.class != expectedClass || t.tag != expectedTag || t.isCompound != compoundType { + // Tags don't match. Again, it could be an optional element. + ok := setDefaultValue(v, params) + if ok { + offset = initOffset + } else { + err = StructuralError{fmt.Sprintf("tags don't match (%d vs %+v) %+v %s @%d", expectedTag, t, params, fieldType.Name(), offset)} + } + return + } + if invalidLength(offset, t.length, len(bytes)) { + err = SyntaxError{"data truncated"} + return + } + innerBytes := bytes[offset : offset+t.length] + offset += t.length + + // We deal with the structures defined in this package first. + switch fieldType { + case objectIdentifierType: + newSlice, err1 := parseObjectIdentifier(innerBytes) + sliceValue := v + sliceValue.Set(reflect.MakeSlice(sliceValue.Type(), len(newSlice), len(newSlice))) + if err1 == nil { + reflect.Copy(sliceValue, reflect.ValueOf(newSlice)) + } + err = err1 + return + case bitStringType: + structValue := v + bs, err1 := parseBitString(innerBytes) + if err1 == nil { + structValue.Set(reflect.ValueOf(bs)) + } + err = err1 + return + case timeType: + ptrValue := v + var time *time.Time + var err1 os.Error + if universalTag == tagUTCTime { + time, err1 = parseUTCTime(innerBytes) + } else { + time, err1 = parseGeneralizedTime(innerBytes) + } + if err1 == nil { + ptrValue.Set(reflect.ValueOf(time)) + } + err = err1 + return + case enumeratedType: + parsedInt, err1 := parseInt(innerBytes) + enumValue := v + if err1 == nil { + enumValue.SetInt(int64(parsedInt)) + } + err = err1 + return + case flagType: + flagValue := v + flagValue.SetBool(true) + return + } + switch val := v; val.Kind() { + case reflect.Bool: + parsedBool, err1 := parseBool(innerBytes) + if err1 == nil { + val.SetBool(parsedBool) + } + err = err1 + return + case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64: + switch val.Type().Kind() { + case reflect.Int: + parsedInt, err1 := parseInt(innerBytes) + if err1 == nil { + val.SetInt(int64(parsedInt)) + } + err = err1 + return + case reflect.Int64: + parsedInt, err1 := parseInt64(innerBytes) + if err1 == nil { + val.SetInt(parsedInt) + } + err = err1 + return + } + case reflect.Struct: + structType := fieldType + + if structType.NumField() > 0 && + structType.Field(0).Type == rawContentsType { + bytes := bytes[initOffset:offset] + val.Field(0).Set(reflect.ValueOf(RawContent(bytes))) + } + + innerOffset := 0 + for i := 0; i < structType.NumField(); i++ { + field := structType.Field(i) + if i == 0 && field.Type == rawContentsType { + continue + } + innerOffset, err = parseField(val.Field(i), innerBytes, innerOffset, parseFieldParameters(field.Tag)) + if err != nil { + return + } + } + // We allow extra bytes at the end of the SEQUENCE because + // adding elements to the end has been used in X.509 as the + // version numbers have increased. + return + case reflect.Slice: + sliceType := fieldType + if sliceType.Elem().Kind() == reflect.Uint8 { + val.Set(reflect.MakeSlice(sliceType, len(innerBytes), len(innerBytes))) + reflect.Copy(val, reflect.ValueOf(innerBytes)) + return + } + newSlice, err1 := parseSequenceOf(innerBytes, sliceType, sliceType.Elem()) + if err1 == nil { + val.Set(newSlice) + } + err = err1 + return + case reflect.String: + var v string + switch universalTag { + case tagPrintableString: + v, err = parsePrintableString(innerBytes) + case tagIA5String: + v, err = parseIA5String(innerBytes) + case tagT61String: + v, err = parseT61String(innerBytes) + case tagGeneralString: + // GeneralString is specified in ISO-2022/ECMA-35, + // A brief review suggests that it includes structures + // that allow the encoding to change midstring and + // such. We give up and pass it as an 8-bit string. + v, err = parseT61String(innerBytes) + default: + err = SyntaxError{fmt.Sprintf("internal error: unknown string type %d", universalTag)} + } + if err == nil { + val.SetString(v) + } + return + } + err = StructuralError{"unknown Go type"} + return +} + +// setDefaultValue is used to install a default value, from a tag string, into +// a Value. It is successful is the field was optional, even if a default value +// wasn't provided or it failed to install it into the Value. +func setDefaultValue(v reflect.Value, params fieldParameters) (ok bool) { + if !params.optional { + return + } + ok = true + if params.defaultValue == nil { + return + } + switch val := v; val.Kind() { + case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64: + val.SetInt(*params.defaultValue) + } + return +} + +// Unmarshal parses the DER-encoded ASN.1 data structure b +// and uses the reflect package to fill in an arbitrary value pointed at by val. +// Because Unmarshal uses the reflect package, the structs +// being written to must use upper case field names. +// +// An ASN.1 INTEGER can be written to an int or int64. +// If the encoded value does not fit in the Go type, +// Unmarshal returns a parse error. +// +// An ASN.1 BIT STRING can be written to a BitString. +// +// An ASN.1 OCTET STRING can be written to a []byte. +// +// An ASN.1 OBJECT IDENTIFIER can be written to an +// ObjectIdentifier. +// +// An ASN.1 ENUMERATED can be written to an Enumerated. +// +// An ASN.1 UTCTIME or GENERALIZEDTIME can be written to a *time.Time. +// +// An ASN.1 PrintableString or IA5String can be written to a string. +// +// Any of the above ASN.1 values can be written to an interface{}. +// The value stored in the interface has the corresponding Go type. +// For integers, that type is int64. +// +// An ASN.1 SEQUENCE OF x or SET OF x can be written +// to a slice if an x can be written to the slice's element type. +// +// An ASN.1 SEQUENCE or SET can be written to a struct +// if each of the elements in the sequence can be +// written to the corresponding element in the struct. +// +// The following tags on struct fields have special meaning to Unmarshal: +// +// optional marks the field as ASN.1 OPTIONAL +// [explicit] tag:x specifies the ASN.1 tag number; implies ASN.1 CONTEXT SPECIFIC +// default:x sets the default value for optional integer fields +// +// If the type of the first field of a structure is RawContent then the raw +// ASN1 contents of the struct will be stored in it. +// +// Other ASN.1 types are not supported; if it encounters them, +// Unmarshal returns a parse error. +func Unmarshal(b []byte, val interface{}) (rest []byte, err os.Error) { + return UnmarshalWithParams(b, val, "") +} + +// UnmarshalWithParams allows field parameters to be specified for the +// top-level element. The form of the params is the same as the field tags. +func UnmarshalWithParams(b []byte, val interface{}, params string) (rest []byte, err os.Error) { + v := reflect.ValueOf(val).Elem() + offset, err := parseField(v, b, 0, parseFieldParameters(params)) + if err != nil { + return nil, err + } + return b[offset:], nil +} diff --git a/src/cmd/gofix/testdata/reflect.datafmt.go.in b/src/cmd/gofix/testdata/reflect.datafmt.go.in new file mode 100644 index 000000000..91f885f9a --- /dev/null +++ b/src/cmd/gofix/testdata/reflect.datafmt.go.in @@ -0,0 +1,710 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +/* The datafmt package implements syntax-directed, type-driven formatting + of arbitrary data structures. Formatting a data structure consists of + two phases: first, a parser reads a format specification and builds a + "compiled" format. Then, the format can be applied repeatedly to + arbitrary values. Applying a format to a value evaluates to a []byte + containing the formatted value bytes, or nil. + + A format specification is a set of package declarations and format rules: + + Format = [ Entry { ";" Entry } [ ";" ] ] . + Entry = PackageDecl | FormatRule . + + (The syntax of a format specification is presented in the same EBNF + notation as used in the Go language specification. The syntax of white + space, comments, identifiers, and string literals is the same as in Go.) + + A package declaration binds a package name (such as 'ast') to a + package import path (such as '"go/ast"'). Each package used (in + a type name, see below) must be declared once before use. + + PackageDecl = PackageName ImportPath . + PackageName = identifier . + ImportPath = string . + + A format rule binds a rule name to a format expression. A rule name + may be a type name or one of the special names 'default' or '/'. + A type name may be the name of a predeclared type (for example, 'int', + 'float32', etc.), the package-qualified name of a user-defined type + (for example, 'ast.MapType'), or an identifier indicating the structure + of unnamed composite types ('array', 'chan', 'func', 'interface', 'map', + or 'ptr'). Each rule must have a unique name; rules can be declared in + any order. + + FormatRule = RuleName "=" Expression . + RuleName = TypeName | "default" | "/" . + TypeName = [ PackageName "." ] identifier . + + To format a value, the value's type name is used to select the format rule + (there is an override mechanism, see below). The format expression of the + selected rule specifies how the value is formatted. Each format expression, + when applied to a value, evaluates to a byte sequence or nil. + + In its most general form, a format expression is a list of alternatives, + each of which is a sequence of operands: + + Expression = [ Sequence ] { "|" [ Sequence ] } . + Sequence = Operand { Operand } . + + The formatted result produced by an expression is the result of the first + alternative sequence that evaluates to a non-nil result; if there is no + such alternative, the expression evaluates to nil. The result produced by + an operand sequence is the concatenation of the results of its operands. + If any operand in the sequence evaluates to nil, the entire sequence + evaluates to nil. + + There are five kinds of operands: + + Operand = Literal | Field | Group | Option | Repetition . + + Literals evaluate to themselves, with two substitutions. First, + %-formats expand in the manner of fmt.Printf, with the current value + passed as the parameter. Second, the current indentation (see below) + is inserted after every newline or form feed character. + + Literal = string . + + This table shows string literals applied to the value 42 and the + corresponding formatted result: + + "foo" foo + "%x" 2a + "x = %d" x = 42 + "%#x = %d" 0x2a = 42 + + A field operand is a field name optionally followed by an alternate + rule name. The field name may be an identifier or one of the special + names @ or *. + + Field = FieldName [ ":" RuleName ] . + FieldName = identifier | "@" | "*" . + + If the field name is an identifier, the current value must be a struct, + and there must be a field with that name in the struct. The same lookup + rules apply as in the Go language (for instance, the name of an anonymous + field is the unqualified type name). The field name denotes the field + value in the struct. If the field is not found, formatting is aborted + and an error message is returned. (TODO consider changing the semantics + such that if a field is not found, it evaluates to nil). + + The special name '@' denotes the current value. + + The meaning of the special name '*' depends on the type of the current + value: + + array, slice types array, slice element (inside {} only, see below) + interfaces value stored in interface + pointers value pointed to by pointer + + (Implementation restriction: channel, function and map types are not + supported due to missing reflection support). + + Fields are evaluated as follows: If the field value is nil, or an array + or slice element does not exist, the result is nil (see below for details + on array/slice elements). If the value is not nil the field value is + formatted (recursively) using the rule corresponding to its type name, + or the alternate rule name, if given. + + The following example shows a complete format specification for a + struct 'myPackage.Point'. Assume the package + + package myPackage // in directory myDir/myPackage + type Point struct { + name string; + x, y int; + } + + Applying the format specification + + myPackage "myDir/myPackage"; + int = "%d"; + hexInt = "0x%x"; + string = "---%s---"; + myPackage.Point = name "{" x ", " y:hexInt "}"; + + to the value myPackage.Point{"foo", 3, 15} results in + + ---foo---{3, 0xf} + + Finally, an operand may be a grouped, optional, or repeated expression. + A grouped expression ("group") groups a more complex expression (body) + so that it can be used in place of a single operand: + + Group = "(" [ Indentation ">>" ] Body ")" . + Indentation = Expression . + Body = Expression . + + A group body may be prefixed by an indentation expression followed by '>>'. + The indentation expression is applied to the current value like any other + expression and the result, if not nil, is appended to the current indentation + during the evaluation of the body (see also formatting state, below). + + An optional expression ("option") is enclosed in '[]' brackets. + + Option = "[" Body "]" . + + An option evaluates to its body, except that if the body evaluates to nil, + the option expression evaluates to an empty []byte. Thus an option's purpose + is to protect the expression containing the option from a nil operand. + + A repeated expression ("repetition") is enclosed in '{}' braces. + + Repetition = "{" Body [ "/" Separator ] "}" . + Separator = Expression . + + A repeated expression is evaluated as follows: The body is evaluated + repeatedly and its results are concatenated until the body evaluates + to nil. The result of the repetition is the (possibly empty) concatenation, + but it is never nil. An implicit index is supplied for the evaluation of + the body: that index is used to address elements of arrays or slices. If + the corresponding elements do not exist, the field denoting the element + evaluates to nil (which in turn may terminate the repetition). + + The body of a repetition may be followed by a '/' and a "separator" + expression. If the separator is present, it is invoked between repetitions + of the body. + + The following example shows a complete format specification for formatting + a slice of unnamed type. Applying the specification + + int = "%b"; + array = { * / ", " }; // array is the type name for an unnamed slice + + to the value '[]int{2, 3, 5, 7}' results in + + 10, 11, 101, 111 + + Default rule: If a format rule named 'default' is present, it is used for + formatting a value if no other rule was found. A common default rule is + + default = "%v" + + to provide default formatting for basic types without having to specify + a specific rule for each basic type. + + Global separator rule: If a format rule named '/' is present, it is + invoked with the current value between literals. If the separator + expression evaluates to nil, it is ignored. + + For instance, a global separator rule may be used to punctuate a sequence + of values with commas. The rules: + + default = "%v"; + / = ", "; + + will format an argument list by printing each one in its default format, + separated by a comma and a space. +*/ +package datafmt + +import ( + "bytes" + "fmt" + "go/token" + "io" + "os" + "reflect" + "runtime" +) + +// ---------------------------------------------------------------------------- +// Format representation + +// Custom formatters implement the Formatter function type. +// A formatter is invoked with the current formatting state, the +// value to format, and the rule name under which the formatter +// was installed (the same formatter function may be installed +// under different names). The formatter may access the current state +// to guide formatting and use State.Write to append to the state's +// output. +// +// A formatter must return a boolean value indicating if it evaluated +// to a non-nil value (true), or a nil value (false). +// +type Formatter func(state *State, value interface{}, ruleName string) bool + +// A FormatterMap is a set of custom formatters. +// It maps a rule name to a formatter function. +// +type FormatterMap map[string]Formatter + +// A parsed format expression is built from the following nodes. +// +type ( + expr interface{} + + alternatives []expr // x | y | z + + sequence []expr // x y z + + literal [][]byte // a list of string segments, possibly starting with '%' + + field struct { + fieldName string // including "@", "*" + ruleName string // "" if no rule name specified + } + + group struct { + indent, body expr // (indent >> body) + } + + option struct { + body expr // [body] + } + + repetition struct { + body, separator expr // {body / separator} + } + + custom struct { + ruleName string + fun Formatter + } +) + +// A Format is the result of parsing a format specification. +// The format may be applied repeatedly to format values. +// +type Format map[string]expr + +// ---------------------------------------------------------------------------- +// Formatting + +// An application-specific environment may be provided to Format.Apply; +// the environment is available inside custom formatters via State.Env(). +// Environments must implement copying; the Copy method must return an +// complete copy of the receiver. This is necessary so that the formatter +// can save and restore an environment (in case of an absent expression). +// +// If the Environment doesn't change during formatting (this is under +// control of the custom formatters), the Copy function can simply return +// the receiver, and thus can be very light-weight. +// +type Environment interface { + Copy() Environment +} + +// State represents the current formatting state. +// It is provided as argument to custom formatters. +// +type State struct { + fmt Format // format in use + env Environment // user-supplied environment + errors chan os.Error // not chan *Error (errors <- nil would be wrong!) + hasOutput bool // true after the first literal has been written + indent bytes.Buffer // current indentation + output bytes.Buffer // format output + linePos token.Position // position of line beginning (Column == 0) + default_ expr // possibly nil + separator expr // possibly nil +} + +func newState(fmt Format, env Environment, errors chan os.Error) *State { + s := new(State) + s.fmt = fmt + s.env = env + s.errors = errors + s.linePos = token.Position{Line: 1} + + // if we have a default rule, cache it's expression for fast access + if x, found := fmt["default"]; found { + s.default_ = x + } + + // if we have a global separator rule, cache it's expression for fast access + if x, found := fmt["/"]; found { + s.separator = x + } + + return s +} + +// Env returns the environment passed to Format.Apply. +func (s *State) Env() interface{} { return s.env } + +// LinePos returns the position of the current line beginning +// in the state's output buffer. Line numbers start at 1. +// +func (s *State) LinePos() token.Position { return s.linePos } + +// Pos returns the position of the next byte to be written to the +// output buffer. Line numbers start at 1. +// +func (s *State) Pos() token.Position { + offs := s.output.Len() + return token.Position{Line: s.linePos.Line, Column: offs - s.linePos.Offset, Offset: offs} +} + +// Write writes data to the output buffer, inserting the indentation +// string after each newline or form feed character. It cannot return an error. +// +func (s *State) Write(data []byte) (int, os.Error) { + n := 0 + i0 := 0 + for i, ch := range data { + if ch == '\n' || ch == '\f' { + // write text segment and indentation + n1, _ := s.output.Write(data[i0 : i+1]) + n2, _ := s.output.Write(s.indent.Bytes()) + n += n1 + n2 + i0 = i + 1 + s.linePos.Offset = s.output.Len() + s.linePos.Line++ + } + } + n3, _ := s.output.Write(data[i0:]) + return n + n3, nil +} + +type checkpoint struct { + env Environment + hasOutput bool + outputLen int + linePos token.Position +} + +func (s *State) save() checkpoint { + saved := checkpoint{nil, s.hasOutput, s.output.Len(), s.linePos} + if s.env != nil { + saved.env = s.env.Copy() + } + return saved +} + +func (s *State) restore(m checkpoint) { + s.env = m.env + s.output.Truncate(m.outputLen) +} + +func (s *State) error(msg string) { + s.errors <- os.NewError(msg) + runtime.Goexit() +} + +// TODO At the moment, unnamed types are simply mapped to the default +// names below. For instance, all unnamed arrays are mapped to +// 'array' which is not really sufficient. Eventually one may want +// to be able to specify rules for say an unnamed slice of T. +// + +func typename(typ reflect.Type) string { + switch typ.(type) { + case *reflect.ArrayType: + return "array" + case *reflect.SliceType: + return "array" + case *reflect.ChanType: + return "chan" + case *reflect.FuncType: + return "func" + case *reflect.InterfaceType: + return "interface" + case *reflect.MapType: + return "map" + case *reflect.PtrType: + return "ptr" + } + return typ.String() +} + +func (s *State) getFormat(name string) expr { + if fexpr, found := s.fmt[name]; found { + return fexpr + } + + if s.default_ != nil { + return s.default_ + } + + s.error(fmt.Sprintf("no format rule for type: '%s'", name)) + return nil +} + +// eval applies a format expression fexpr to a value. If the expression +// evaluates internally to a non-nil []byte, that slice is appended to +// the state's output buffer and eval returns true. Otherwise, eval +// returns false and the state remains unchanged. +// +func (s *State) eval(fexpr expr, value reflect.Value, index int) bool { + // an empty format expression always evaluates + // to a non-nil (but empty) []byte + if fexpr == nil { + return true + } + + switch t := fexpr.(type) { + case alternatives: + // append the result of the first alternative that evaluates to + // a non-nil []byte to the state's output + mark := s.save() + for _, x := range t { + if s.eval(x, value, index) { + return true + } + s.restore(mark) + } + return false + + case sequence: + // append the result of all operands to the state's output + // unless a nil result is encountered + mark := s.save() + for _, x := range t { + if !s.eval(x, value, index) { + s.restore(mark) + return false + } + } + return true + + case literal: + // write separator, if any + if s.hasOutput { + // not the first literal + if s.separator != nil { + sep := s.separator // save current separator + s.separator = nil // and disable it (avoid recursion) + mark := s.save() + if !s.eval(sep, value, index) { + s.restore(mark) + } + s.separator = sep // enable it again + } + } + s.hasOutput = true + // write literal segments + for _, lit := range t { + if len(lit) > 1 && lit[0] == '%' { + // segment contains a %-format at the beginning + if lit[1] == '%' { + // "%%" is printed as a single "%" + s.Write(lit[1:]) + } else { + // use s instead of s.output to get indentation right + fmt.Fprintf(s, string(lit), value.Interface()) + } + } else { + // segment contains no %-formats + s.Write(lit) + } + } + return true // a literal never evaluates to nil + + case *field: + // determine field value + switch t.fieldName { + case "@": + // field value is current value + + case "*": + // indirection: operation is type-specific + switch v := value.(type) { + case *reflect.ArrayValue: + if v.Len() <= index { + return false + } + value = v.Elem(index) + + case *reflect.SliceValue: + if v.IsNil() || v.Len() <= index { + return false + } + value = v.Elem(index) + + case *reflect.MapValue: + s.error("reflection support for maps incomplete") + + case *reflect.PtrValue: + if v.IsNil() { + return false + } + value = v.Elem() + + case *reflect.InterfaceValue: + if v.IsNil() { + return false + } + value = v.Elem() + + case *reflect.ChanValue: + s.error("reflection support for chans incomplete") + + case *reflect.FuncValue: + s.error("reflection support for funcs incomplete") + + default: + s.error(fmt.Sprintf("error: * does not apply to `%s`", value.Type())) + } + + default: + // value is value of named field + var field reflect.Value + if sval, ok := value.(*reflect.StructValue); ok { + field = sval.FieldByName(t.fieldName) + if field == nil { + // TODO consider just returning false in this case + s.error(fmt.Sprintf("error: no field `%s` in `%s`", t.fieldName, value.Type())) + } + } + value = field + } + + // determine rule + ruleName := t.ruleName + if ruleName == "" { + // no alternate rule name, value type determines rule + ruleName = typename(value.Type()) + } + fexpr = s.getFormat(ruleName) + + mark := s.save() + if !s.eval(fexpr, value, index) { + s.restore(mark) + return false + } + return true + + case *group: + // remember current indentation + indentLen := s.indent.Len() + + // update current indentation + mark := s.save() + s.eval(t.indent, value, index) + // if the indentation evaluates to nil, the state's output buffer + // didn't change - either way it's ok to append the difference to + // the current identation + s.indent.Write(s.output.Bytes()[mark.outputLen:s.output.Len()]) + s.restore(mark) + + // format group body + mark = s.save() + b := true + if !s.eval(t.body, value, index) { + s.restore(mark) + b = false + } + + // reset indentation + s.indent.Truncate(indentLen) + return b + + case *option: + // evaluate the body and append the result to the state's output + // buffer unless the result is nil + mark := s.save() + if !s.eval(t.body, value, 0) { // TODO is 0 index correct? + s.restore(mark) + } + return true // an option never evaluates to nil + + case *repetition: + // evaluate the body and append the result to the state's output + // buffer until a result is nil + for i := 0; ; i++ { + mark := s.save() + // write separator, if any + if i > 0 && t.separator != nil { + // nil result from separator is ignored + mark := s.save() + if !s.eval(t.separator, value, i) { + s.restore(mark) + } + } + if !s.eval(t.body, value, i) { + s.restore(mark) + break + } + } + return true // a repetition never evaluates to nil + + case *custom: + // invoke the custom formatter to obtain the result + mark := s.save() + if !t.fun(s, value.Interface(), t.ruleName) { + s.restore(mark) + return false + } + return true + } + + panic("unreachable") + return false +} + +// Eval formats each argument according to the format +// f and returns the resulting []byte and os.Error. If +// an error occurred, the []byte contains the partially +// formatted result. An environment env may be passed +// in which is available in custom formatters through +// the state parameter. +// +func (f Format) Eval(env Environment, args ...interface{}) ([]byte, os.Error) { + if f == nil { + return nil, os.NewError("format is nil") + } + + errors := make(chan os.Error) + s := newState(f, env, errors) + + go func() { + for _, v := range args { + fld := reflect.NewValue(v) + if fld == nil { + errors <- os.NewError("nil argument") + return + } + mark := s.save() + if !s.eval(s.getFormat(typename(fld.Type())), fld, 0) { // TODO is 0 index correct? + s.restore(mark) + } + } + errors <- nil // no errors + }() + + err := <-errors + return s.output.Bytes(), err +} + +// ---------------------------------------------------------------------------- +// Convenience functions + +// Fprint formats each argument according to the format f +// and writes to w. The result is the total number of bytes +// written and an os.Error, if any. +// +func (f Format) Fprint(w io.Writer, env Environment, args ...interface{}) (int, os.Error) { + data, err := f.Eval(env, args...) + if err != nil { + // TODO should we print partial result in case of error? + return 0, err + } + return w.Write(data) +} + +// Print formats each argument according to the format f +// and writes to standard output. The result is the total +// number of bytes written and an os.Error, if any. +// +func (f Format) Print(args ...interface{}) (int, os.Error) { + return f.Fprint(os.Stdout, nil, args...) +} + +// Sprint formats each argument according to the format f +// and returns the resulting string. If an error occurs +// during formatting, the result string contains the +// partially formatted result followed by an error message. +// +func (f Format) Sprint(args ...interface{}) string { + var buf bytes.Buffer + _, err := f.Fprint(&buf, nil, args...) + if err != nil { + var i interface{} = args + fmt.Fprintf(&buf, "--- Sprint(%s) failed: %v", fmt.Sprint(i), err) + } + return buf.String() +} diff --git a/src/cmd/gofix/testdata/reflect.datafmt.go.out b/src/cmd/gofix/testdata/reflect.datafmt.go.out new file mode 100644 index 000000000..fd447588b --- /dev/null +++ b/src/cmd/gofix/testdata/reflect.datafmt.go.out @@ -0,0 +1,710 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +/* The datafmt package implements syntax-directed, type-driven formatting + of arbitrary data structures. Formatting a data structure consists of + two phases: first, a parser reads a format specification and builds a + "compiled" format. Then, the format can be applied repeatedly to + arbitrary values. Applying a format to a value evaluates to a []byte + containing the formatted value bytes, or nil. + + A format specification is a set of package declarations and format rules: + + Format = [ Entry { ";" Entry } [ ";" ] ] . + Entry = PackageDecl | FormatRule . + + (The syntax of a format specification is presented in the same EBNF + notation as used in the Go language specification. The syntax of white + space, comments, identifiers, and string literals is the same as in Go.) + + A package declaration binds a package name (such as 'ast') to a + package import path (such as '"go/ast"'). Each package used (in + a type name, see below) must be declared once before use. + + PackageDecl = PackageName ImportPath . + PackageName = identifier . + ImportPath = string . + + A format rule binds a rule name to a format expression. A rule name + may be a type name or one of the special names 'default' or '/'. + A type name may be the name of a predeclared type (for example, 'int', + 'float32', etc.), the package-qualified name of a user-defined type + (for example, 'ast.MapType'), or an identifier indicating the structure + of unnamed composite types ('array', 'chan', 'func', 'interface', 'map', + or 'ptr'). Each rule must have a unique name; rules can be declared in + any order. + + FormatRule = RuleName "=" Expression . + RuleName = TypeName | "default" | "/" . + TypeName = [ PackageName "." ] identifier . + + To format a value, the value's type name is used to select the format rule + (there is an override mechanism, see below). The format expression of the + selected rule specifies how the value is formatted. Each format expression, + when applied to a value, evaluates to a byte sequence or nil. + + In its most general form, a format expression is a list of alternatives, + each of which is a sequence of operands: + + Expression = [ Sequence ] { "|" [ Sequence ] } . + Sequence = Operand { Operand } . + + The formatted result produced by an expression is the result of the first + alternative sequence that evaluates to a non-nil result; if there is no + such alternative, the expression evaluates to nil. The result produced by + an operand sequence is the concatenation of the results of its operands. + If any operand in the sequence evaluates to nil, the entire sequence + evaluates to nil. + + There are five kinds of operands: + + Operand = Literal | Field | Group | Option | Repetition . + + Literals evaluate to themselves, with two substitutions. First, + %-formats expand in the manner of fmt.Printf, with the current value + passed as the parameter. Second, the current indentation (see below) + is inserted after every newline or form feed character. + + Literal = string . + + This table shows string literals applied to the value 42 and the + corresponding formatted result: + + "foo" foo + "%x" 2a + "x = %d" x = 42 + "%#x = %d" 0x2a = 42 + + A field operand is a field name optionally followed by an alternate + rule name. The field name may be an identifier or one of the special + names @ or *. + + Field = FieldName [ ":" RuleName ] . + FieldName = identifier | "@" | "*" . + + If the field name is an identifier, the current value must be a struct, + and there must be a field with that name in the struct. The same lookup + rules apply as in the Go language (for instance, the name of an anonymous + field is the unqualified type name). The field name denotes the field + value in the struct. If the field is not found, formatting is aborted + and an error message is returned. (TODO consider changing the semantics + such that if a field is not found, it evaluates to nil). + + The special name '@' denotes the current value. + + The meaning of the special name '*' depends on the type of the current + value: + + array, slice types array, slice element (inside {} only, see below) + interfaces value stored in interface + pointers value pointed to by pointer + + (Implementation restriction: channel, function and map types are not + supported due to missing reflection support). + + Fields are evaluated as follows: If the field value is nil, or an array + or slice element does not exist, the result is nil (see below for details + on array/slice elements). If the value is not nil the field value is + formatted (recursively) using the rule corresponding to its type name, + or the alternate rule name, if given. + + The following example shows a complete format specification for a + struct 'myPackage.Point'. Assume the package + + package myPackage // in directory myDir/myPackage + type Point struct { + name string; + x, y int; + } + + Applying the format specification + + myPackage "myDir/myPackage"; + int = "%d"; + hexInt = "0x%x"; + string = "---%s---"; + myPackage.Point = name "{" x ", " y:hexInt "}"; + + to the value myPackage.Point{"foo", 3, 15} results in + + ---foo---{3, 0xf} + + Finally, an operand may be a grouped, optional, or repeated expression. + A grouped expression ("group") groups a more complex expression (body) + so that it can be used in place of a single operand: + + Group = "(" [ Indentation ">>" ] Body ")" . + Indentation = Expression . + Body = Expression . + + A group body may be prefixed by an indentation expression followed by '>>'. + The indentation expression is applied to the current value like any other + expression and the result, if not nil, is appended to the current indentation + during the evaluation of the body (see also formatting state, below). + + An optional expression ("option") is enclosed in '[]' brackets. + + Option = "[" Body "]" . + + An option evaluates to its body, except that if the body evaluates to nil, + the option expression evaluates to an empty []byte. Thus an option's purpose + is to protect the expression containing the option from a nil operand. + + A repeated expression ("repetition") is enclosed in '{}' braces. + + Repetition = "{" Body [ "/" Separator ] "}" . + Separator = Expression . + + A repeated expression is evaluated as follows: The body is evaluated + repeatedly and its results are concatenated until the body evaluates + to nil. The result of the repetition is the (possibly empty) concatenation, + but it is never nil. An implicit index is supplied for the evaluation of + the body: that index is used to address elements of arrays or slices. If + the corresponding elements do not exist, the field denoting the element + evaluates to nil (which in turn may terminate the repetition). + + The body of a repetition may be followed by a '/' and a "separator" + expression. If the separator is present, it is invoked between repetitions + of the body. + + The following example shows a complete format specification for formatting + a slice of unnamed type. Applying the specification + + int = "%b"; + array = { * / ", " }; // array is the type name for an unnamed slice + + to the value '[]int{2, 3, 5, 7}' results in + + 10, 11, 101, 111 + + Default rule: If a format rule named 'default' is present, it is used for + formatting a value if no other rule was found. A common default rule is + + default = "%v" + + to provide default formatting for basic types without having to specify + a specific rule for each basic type. + + Global separator rule: If a format rule named '/' is present, it is + invoked with the current value between literals. If the separator + expression evaluates to nil, it is ignored. + + For instance, a global separator rule may be used to punctuate a sequence + of values with commas. The rules: + + default = "%v"; + / = ", "; + + will format an argument list by printing each one in its default format, + separated by a comma and a space. +*/ +package datafmt + +import ( + "bytes" + "fmt" + "go/token" + "io" + "os" + "reflect" + "runtime" +) + +// ---------------------------------------------------------------------------- +// Format representation + +// Custom formatters implement the Formatter function type. +// A formatter is invoked with the current formatting state, the +// value to format, and the rule name under which the formatter +// was installed (the same formatter function may be installed +// under different names). The formatter may access the current state +// to guide formatting and use State.Write to append to the state's +// output. +// +// A formatter must return a boolean value indicating if it evaluated +// to a non-nil value (true), or a nil value (false). +// +type Formatter func(state *State, value interface{}, ruleName string) bool + +// A FormatterMap is a set of custom formatters. +// It maps a rule name to a formatter function. +// +type FormatterMap map[string]Formatter + +// A parsed format expression is built from the following nodes. +// +type ( + expr interface{} + + alternatives []expr // x | y | z + + sequence []expr // x y z + + literal [][]byte // a list of string segments, possibly starting with '%' + + field struct { + fieldName string // including "@", "*" + ruleName string // "" if no rule name specified + } + + group struct { + indent, body expr // (indent >> body) + } + + option struct { + body expr // [body] + } + + repetition struct { + body, separator expr // {body / separator} + } + + custom struct { + ruleName string + fun Formatter + } +) + +// A Format is the result of parsing a format specification. +// The format may be applied repeatedly to format values. +// +type Format map[string]expr + +// ---------------------------------------------------------------------------- +// Formatting + +// An application-specific environment may be provided to Format.Apply; +// the environment is available inside custom formatters via State.Env(). +// Environments must implement copying; the Copy method must return an +// complete copy of the receiver. This is necessary so that the formatter +// can save and restore an environment (in case of an absent expression). +// +// If the Environment doesn't change during formatting (this is under +// control of the custom formatters), the Copy function can simply return +// the receiver, and thus can be very light-weight. +// +type Environment interface { + Copy() Environment +} + +// State represents the current formatting state. +// It is provided as argument to custom formatters. +// +type State struct { + fmt Format // format in use + env Environment // user-supplied environment + errors chan os.Error // not chan *Error (errors <- nil would be wrong!) + hasOutput bool // true after the first literal has been written + indent bytes.Buffer // current indentation + output bytes.Buffer // format output + linePos token.Position // position of line beginning (Column == 0) + default_ expr // possibly nil + separator expr // possibly nil +} + +func newState(fmt Format, env Environment, errors chan os.Error) *State { + s := new(State) + s.fmt = fmt + s.env = env + s.errors = errors + s.linePos = token.Position{Line: 1} + + // if we have a default rule, cache it's expression for fast access + if x, found := fmt["default"]; found { + s.default_ = x + } + + // if we have a global separator rule, cache it's expression for fast access + if x, found := fmt["/"]; found { + s.separator = x + } + + return s +} + +// Env returns the environment passed to Format.Apply. +func (s *State) Env() interface{} { return s.env } + +// LinePos returns the position of the current line beginning +// in the state's output buffer. Line numbers start at 1. +// +func (s *State) LinePos() token.Position { return s.linePos } + +// Pos returns the position of the next byte to be written to the +// output buffer. Line numbers start at 1. +// +func (s *State) Pos() token.Position { + offs := s.output.Len() + return token.Position{Line: s.linePos.Line, Column: offs - s.linePos.Offset, Offset: offs} +} + +// Write writes data to the output buffer, inserting the indentation +// string after each newline or form feed character. It cannot return an error. +// +func (s *State) Write(data []byte) (int, os.Error) { + n := 0 + i0 := 0 + for i, ch := range data { + if ch == '\n' || ch == '\f' { + // write text segment and indentation + n1, _ := s.output.Write(data[i0 : i+1]) + n2, _ := s.output.Write(s.indent.Bytes()) + n += n1 + n2 + i0 = i + 1 + s.linePos.Offset = s.output.Len() + s.linePos.Line++ + } + } + n3, _ := s.output.Write(data[i0:]) + return n + n3, nil +} + +type checkpoint struct { + env Environment + hasOutput bool + outputLen int + linePos token.Position +} + +func (s *State) save() checkpoint { + saved := checkpoint{nil, s.hasOutput, s.output.Len(), s.linePos} + if s.env != nil { + saved.env = s.env.Copy() + } + return saved +} + +func (s *State) restore(m checkpoint) { + s.env = m.env + s.output.Truncate(m.outputLen) +} + +func (s *State) error(msg string) { + s.errors <- os.NewError(msg) + runtime.Goexit() +} + +// TODO At the moment, unnamed types are simply mapped to the default +// names below. For instance, all unnamed arrays are mapped to +// 'array' which is not really sufficient. Eventually one may want +// to be able to specify rules for say an unnamed slice of T. +// + +func typename(typ reflect.Type) string { + switch typ.Kind() { + case reflect.Array: + return "array" + case reflect.Slice: + return "array" + case reflect.Chan: + return "chan" + case reflect.Func: + return "func" + case reflect.Interface: + return "interface" + case reflect.Map: + return "map" + case reflect.Ptr: + return "ptr" + } + return typ.String() +} + +func (s *State) getFormat(name string) expr { + if fexpr, found := s.fmt[name]; found { + return fexpr + } + + if s.default_ != nil { + return s.default_ + } + + s.error(fmt.Sprintf("no format rule for type: '%s'", name)) + return nil +} + +// eval applies a format expression fexpr to a value. If the expression +// evaluates internally to a non-nil []byte, that slice is appended to +// the state's output buffer and eval returns true. Otherwise, eval +// returns false and the state remains unchanged. +// +func (s *State) eval(fexpr expr, value reflect.Value, index int) bool { + // an empty format expression always evaluates + // to a non-nil (but empty) []byte + if fexpr == nil { + return true + } + + switch t := fexpr.(type) { + case alternatives: + // append the result of the first alternative that evaluates to + // a non-nil []byte to the state's output + mark := s.save() + for _, x := range t { + if s.eval(x, value, index) { + return true + } + s.restore(mark) + } + return false + + case sequence: + // append the result of all operands to the state's output + // unless a nil result is encountered + mark := s.save() + for _, x := range t { + if !s.eval(x, value, index) { + s.restore(mark) + return false + } + } + return true + + case literal: + // write separator, if any + if s.hasOutput { + // not the first literal + if s.separator != nil { + sep := s.separator // save current separator + s.separator = nil // and disable it (avoid recursion) + mark := s.save() + if !s.eval(sep, value, index) { + s.restore(mark) + } + s.separator = sep // enable it again + } + } + s.hasOutput = true + // write literal segments + for _, lit := range t { + if len(lit) > 1 && lit[0] == '%' { + // segment contains a %-format at the beginning + if lit[1] == '%' { + // "%%" is printed as a single "%" + s.Write(lit[1:]) + } else { + // use s instead of s.output to get indentation right + fmt.Fprintf(s, string(lit), value.Interface()) + } + } else { + // segment contains no %-formats + s.Write(lit) + } + } + return true // a literal never evaluates to nil + + case *field: + // determine field value + switch t.fieldName { + case "@": + // field value is current value + + case "*": + // indirection: operation is type-specific + switch v := value; v.Kind() { + case reflect.Array: + if v.Len() <= index { + return false + } + value = v.Index(index) + + case reflect.Slice: + if v.IsNil() || v.Len() <= index { + return false + } + value = v.Index(index) + + case reflect.Map: + s.error("reflection support for maps incomplete") + + case reflect.Ptr: + if v.IsNil() { + return false + } + value = v.Elem() + + case reflect.Interface: + if v.IsNil() { + return false + } + value = v.Elem() + + case reflect.Chan: + s.error("reflection support for chans incomplete") + + case reflect.Func: + s.error("reflection support for funcs incomplete") + + default: + s.error(fmt.Sprintf("error: * does not apply to `%s`", value.Type())) + } + + default: + // value is value of named field + var field reflect.Value + if sval := value; sval.Kind() == reflect.Struct { + field = sval.FieldByName(t.fieldName) + if !field.IsValid() { + // TODO consider just returning false in this case + s.error(fmt.Sprintf("error: no field `%s` in `%s`", t.fieldName, value.Type())) + } + } + value = field + } + + // determine rule + ruleName := t.ruleName + if ruleName == "" { + // no alternate rule name, value type determines rule + ruleName = typename(value.Type()) + } + fexpr = s.getFormat(ruleName) + + mark := s.save() + if !s.eval(fexpr, value, index) { + s.restore(mark) + return false + } + return true + + case *group: + // remember current indentation + indentLen := s.indent.Len() + + // update current indentation + mark := s.save() + s.eval(t.indent, value, index) + // if the indentation evaluates to nil, the state's output buffer + // didn't change - either way it's ok to append the difference to + // the current identation + s.indent.Write(s.output.Bytes()[mark.outputLen:s.output.Len()]) + s.restore(mark) + + // format group body + mark = s.save() + b := true + if !s.eval(t.body, value, index) { + s.restore(mark) + b = false + } + + // reset indentation + s.indent.Truncate(indentLen) + return b + + case *option: + // evaluate the body and append the result to the state's output + // buffer unless the result is nil + mark := s.save() + if !s.eval(t.body, value, 0) { // TODO is 0 index correct? + s.restore(mark) + } + return true // an option never evaluates to nil + + case *repetition: + // evaluate the body and append the result to the state's output + // buffer until a result is nil + for i := 0; ; i++ { + mark := s.save() + // write separator, if any + if i > 0 && t.separator != nil { + // nil result from separator is ignored + mark := s.save() + if !s.eval(t.separator, value, i) { + s.restore(mark) + } + } + if !s.eval(t.body, value, i) { + s.restore(mark) + break + } + } + return true // a repetition never evaluates to nil + + case *custom: + // invoke the custom formatter to obtain the result + mark := s.save() + if !t.fun(s, value.Interface(), t.ruleName) { + s.restore(mark) + return false + } + return true + } + + panic("unreachable") + return false +} + +// Eval formats each argument according to the format +// f and returns the resulting []byte and os.Error. If +// an error occurred, the []byte contains the partially +// formatted result. An environment env may be passed +// in which is available in custom formatters through +// the state parameter. +// +func (f Format) Eval(env Environment, args ...interface{}) ([]byte, os.Error) { + if f == nil { + return nil, os.NewError("format is nil") + } + + errors := make(chan os.Error) + s := newState(f, env, errors) + + go func() { + for _, v := range args { + fld := reflect.ValueOf(v) + if !fld.IsValid() { + errors <- os.NewError("nil argument") + return + } + mark := s.save() + if !s.eval(s.getFormat(typename(fld.Type())), fld, 0) { // TODO is 0 index correct? + s.restore(mark) + } + } + errors <- nil // no errors + }() + + err := <-errors + return s.output.Bytes(), err +} + +// ---------------------------------------------------------------------------- +// Convenience functions + +// Fprint formats each argument according to the format f +// and writes to w. The result is the total number of bytes +// written and an os.Error, if any. +// +func (f Format) Fprint(w io.Writer, env Environment, args ...interface{}) (int, os.Error) { + data, err := f.Eval(env, args...) + if err != nil { + // TODO should we print partial result in case of error? + return 0, err + } + return w.Write(data) +} + +// Print formats each argument according to the format f +// and writes to standard output. The result is the total +// number of bytes written and an os.Error, if any. +// +func (f Format) Print(args ...interface{}) (int, os.Error) { + return f.Fprint(os.Stdout, nil, args...) +} + +// Sprint formats each argument according to the format f +// and returns the resulting string. If an error occurs +// during formatting, the result string contains the +// partially formatted result followed by an error message. +// +func (f Format) Sprint(args ...interface{}) string { + var buf bytes.Buffer + _, err := f.Fprint(&buf, nil, args...) + if err != nil { + var i interface{} = args + fmt.Fprintf(&buf, "--- Sprint(%s) failed: %v", fmt.Sprint(i), err) + } + return buf.String() +} diff --git a/src/cmd/gofix/testdata/reflect.decode.go.in b/src/cmd/gofix/testdata/reflect.decode.go.in new file mode 100644 index 000000000..f831abee3 --- /dev/null +++ b/src/cmd/gofix/testdata/reflect.decode.go.in @@ -0,0 +1,905 @@ +// Copyright 2010 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Represents JSON data structure using native Go types: booleans, floats, +// strings, arrays, and maps. + +package json + +import ( + "container/vector" + "encoding/base64" + "os" + "reflect" + "runtime" + "strconv" + "strings" + "unicode" + "utf16" + "utf8" +) + +// Unmarshal parses the JSON-encoded data and stores the result +// in the value pointed to by v. +// +// Unmarshal traverses the value v recursively. +// If an encountered value implements the Unmarshaler interface, +// Unmarshal calls its UnmarshalJSON method with a well-formed +// JSON encoding. +// +// Otherwise, Unmarshal uses the inverse of the encodings that +// Marshal uses, allocating maps, slices, and pointers as necessary, +// with the following additional rules: +// +// To unmarshal a JSON value into a nil interface value, the +// type stored in the interface value is one of: +// +// bool, for JSON booleans +// float64, for JSON numbers +// string, for JSON strings +// []interface{}, for JSON arrays +// map[string]interface{}, for JSON objects +// nil for JSON null +// +// If a JSON value is not appropriate for a given target type, +// or if a JSON number overflows the target type, Unmarshal +// skips that field and completes the unmarshalling as best it can. +// If no more serious errors are encountered, Unmarshal returns +// an UnmarshalTypeError describing the earliest such error. +// +func Unmarshal(data []byte, v interface{}) os.Error { + d := new(decodeState).init(data) + + // Quick check for well-formedness. + // Avoids filling out half a data structure + // before discovering a JSON syntax error. + err := checkValid(data, &d.scan) + if err != nil { + return err + } + + return d.unmarshal(v) +} + +// Unmarshaler is the interface implemented by objects +// that can unmarshal a JSON description of themselves. +// The input can be assumed to be a valid JSON object +// encoding. UnmarshalJSON must copy the JSON data +// if it wishes to retain the data after returning. +type Unmarshaler interface { + UnmarshalJSON([]byte) os.Error +} + +// An UnmarshalTypeError describes a JSON value that was +// not appropriate for a value of a specific Go type. +type UnmarshalTypeError struct { + Value string // description of JSON value - "bool", "array", "number -5" + Type reflect.Type // type of Go value it could not be assigned to +} + +func (e *UnmarshalTypeError) String() string { + return "json: cannot unmarshal " + e.Value + " into Go value of type " + e.Type.String() +} + +// An UnmarshalFieldError describes a JSON object key that +// led to an unexported (and therefore unwritable) struct field. +type UnmarshalFieldError struct { + Key string + Type *reflect.StructType + Field reflect.StructField +} + +func (e *UnmarshalFieldError) String() string { + return "json: cannot unmarshal object key " + strconv.Quote(e.Key) + " into unexported field " + e.Field.Name + " of type " + e.Type.String() +} + +// An InvalidUnmarshalError describes an invalid argument passed to Unmarshal. +// (The argument to Unmarshal must be a non-nil pointer.) +type InvalidUnmarshalError struct { + Type reflect.Type +} + +func (e *InvalidUnmarshalError) String() string { + if e.Type == nil { + return "json: Unmarshal(nil)" + } + + if _, ok := e.Type.(*reflect.PtrType); !ok { + return "json: Unmarshal(non-pointer " + e.Type.String() + ")" + } + return "json: Unmarshal(nil " + e.Type.String() + ")" +} + +func (d *decodeState) unmarshal(v interface{}) (err os.Error) { + defer func() { + if r := recover(); r != nil { + if _, ok := r.(runtime.Error); ok { + panic(r) + } + err = r.(os.Error) + } + }() + + rv := reflect.NewValue(v) + pv, ok := rv.(*reflect.PtrValue) + if !ok || pv.IsNil() { + return &InvalidUnmarshalError{reflect.Typeof(v)} + } + + d.scan.reset() + // We decode rv not pv.Elem because the Unmarshaler interface + // test must be applied at the top level of the value. + d.value(rv) + return d.savedError +} + +// decodeState represents the state while decoding a JSON value. +type decodeState struct { + data []byte + off int // read offset in data + scan scanner + nextscan scanner // for calls to nextValue + savedError os.Error +} + +// errPhase is used for errors that should not happen unless +// there is a bug in the JSON decoder or something is editing +// the data slice while the decoder executes. +var errPhase = os.NewError("JSON decoder out of sync - data changing underfoot?") + +func (d *decodeState) init(data []byte) *decodeState { + d.data = data + d.off = 0 + d.savedError = nil + return d +} + +// error aborts the decoding by panicking with err. +func (d *decodeState) error(err os.Error) { + panic(err) +} + +// saveError saves the first err it is called with, +// for reporting at the end of the unmarshal. +func (d *decodeState) saveError(err os.Error) { + if d.savedError == nil { + d.savedError = err + } +} + +// next cuts off and returns the next full JSON value in d.data[d.off:]. +// The next value is known to be an object or array, not a literal. +func (d *decodeState) next() []byte { + c := d.data[d.off] + item, rest, err := nextValue(d.data[d.off:], &d.nextscan) + if err != nil { + d.error(err) + } + d.off = len(d.data) - len(rest) + + // Our scanner has seen the opening brace/bracket + // and thinks we're still in the middle of the object. + // invent a closing brace/bracket to get it out. + if c == '{' { + d.scan.step(&d.scan, '}') + } else { + d.scan.step(&d.scan, ']') + } + + return item +} + +// scanWhile processes bytes in d.data[d.off:] until it +// receives a scan code not equal to op. +// It updates d.off and returns the new scan code. +func (d *decodeState) scanWhile(op int) int { + var newOp int + for { + if d.off >= len(d.data) { + newOp = d.scan.eof() + d.off = len(d.data) + 1 // mark processed EOF with len+1 + } else { + c := int(d.data[d.off]) + d.off++ + newOp = d.scan.step(&d.scan, c) + } + if newOp != op { + break + } + } + return newOp +} + +// value decodes a JSON value from d.data[d.off:] into the value. +// it updates d.off to point past the decoded value. +func (d *decodeState) value(v reflect.Value) { + if v == nil { + _, rest, err := nextValue(d.data[d.off:], &d.nextscan) + if err != nil { + d.error(err) + } + d.off = len(d.data) - len(rest) + + // d.scan thinks we're still at the beginning of the item. + // Feed in an empty string - the shortest, simplest value - + // so that it knows we got to the end of the value. + if d.scan.step == stateRedo { + panic("redo") + } + d.scan.step(&d.scan, '"') + d.scan.step(&d.scan, '"') + return + } + + switch op := d.scanWhile(scanSkipSpace); op { + default: + d.error(errPhase) + + case scanBeginArray: + d.array(v) + + case scanBeginObject: + d.object(v) + + case scanBeginLiteral: + d.literal(v) + } +} + +// indirect walks down v allocating pointers as needed, +// until it gets to a non-pointer. +// if it encounters an Unmarshaler, indirect stops and returns that. +// if wantptr is true, indirect stops at the last pointer. +func (d *decodeState) indirect(v reflect.Value, wantptr bool) (Unmarshaler, reflect.Value) { + for { + var isUnmarshaler bool + if v.Type().NumMethod() > 0 { + // Remember that this is an unmarshaler, + // but wait to return it until after allocating + // the pointer (if necessary). + _, isUnmarshaler = v.Interface().(Unmarshaler) + } + + if iv, ok := v.(*reflect.InterfaceValue); ok && !iv.IsNil() { + v = iv.Elem() + continue + } + pv, ok := v.(*reflect.PtrValue) + if !ok { + break + } + _, isptrptr := pv.Elem().(*reflect.PtrValue) + if !isptrptr && wantptr && !isUnmarshaler { + return nil, pv + } + if pv.IsNil() { + pv.PointTo(reflect.MakeZero(pv.Type().(*reflect.PtrType).Elem())) + } + if isUnmarshaler { + // Using v.Interface().(Unmarshaler) + // here means that we have to use a pointer + // as the struct field. We cannot use a value inside + // a pointer to a struct, because in that case + // v.Interface() is the value (x.f) not the pointer (&x.f). + // This is an unfortunate consequence of reflect. + // An alternative would be to look up the + // UnmarshalJSON method and return a FuncValue. + return v.Interface().(Unmarshaler), nil + } + v = pv.Elem() + } + return nil, v +} + +// array consumes an array from d.data[d.off-1:], decoding into the value v. +// the first byte of the array ('[') has been read already. +func (d *decodeState) array(v reflect.Value) { + // Check for unmarshaler. + unmarshaler, pv := d.indirect(v, false) + if unmarshaler != nil { + d.off-- + err := unmarshaler.UnmarshalJSON(d.next()) + if err != nil { + d.error(err) + } + return + } + v = pv + + // Decoding into nil interface? Switch to non-reflect code. + iv, ok := v.(*reflect.InterfaceValue) + if ok { + iv.Set(reflect.NewValue(d.arrayInterface())) + return + } + + // Check type of target. + av, ok := v.(reflect.ArrayOrSliceValue) + if !ok { + d.saveError(&UnmarshalTypeError{"array", v.Type()}) + d.off-- + d.next() + return + } + + sv, _ := v.(*reflect.SliceValue) + + i := 0 + for { + // Look ahead for ] - can only happen on first iteration. + op := d.scanWhile(scanSkipSpace) + if op == scanEndArray { + break + } + + // Back up so d.value can have the byte we just read. + d.off-- + d.scan.undo(op) + + // Get element of array, growing if necessary. + if i >= av.Cap() && sv != nil { + newcap := sv.Cap() + sv.Cap()/2 + if newcap < 4 { + newcap = 4 + } + newv := reflect.MakeSlice(sv.Type().(*reflect.SliceType), sv.Len(), newcap) + reflect.Copy(newv, sv) + sv.Set(newv) + } + if i >= av.Len() && sv != nil { + // Must be slice; gave up on array during i >= av.Cap(). + sv.SetLen(i + 1) + } + + // Decode into element. + if i < av.Len() { + d.value(av.Elem(i)) + } else { + // Ran out of fixed array: skip. + d.value(nil) + } + i++ + + // Next token must be , or ]. + op = d.scanWhile(scanSkipSpace) + if op == scanEndArray { + break + } + if op != scanArrayValue { + d.error(errPhase) + } + } + if i < av.Len() { + if sv == nil { + // Array. Zero the rest. + z := reflect.MakeZero(av.Type().(*reflect.ArrayType).Elem()) + for ; i < av.Len(); i++ { + av.Elem(i).SetValue(z) + } + } else { + sv.SetLen(i) + } + } +} + +// matchName returns true if key should be written to a field named name. +func matchName(key, name string) bool { + return strings.ToLower(key) == strings.ToLower(name) +} + +// object consumes an object from d.data[d.off-1:], decoding into the value v. +// the first byte of the object ('{') has been read already. +func (d *decodeState) object(v reflect.Value) { + // Check for unmarshaler. + unmarshaler, pv := d.indirect(v, false) + if unmarshaler != nil { + d.off-- + err := unmarshaler.UnmarshalJSON(d.next()) + if err != nil { + d.error(err) + } + return + } + v = pv + + // Decoding into nil interface? Switch to non-reflect code. + iv, ok := v.(*reflect.InterfaceValue) + if ok { + iv.Set(reflect.NewValue(d.objectInterface())) + return + } + + // Check type of target: struct or map[string]T + var ( + mv *reflect.MapValue + sv *reflect.StructValue + ) + switch v := v.(type) { + case *reflect.MapValue: + // map must have string type + t := v.Type().(*reflect.MapType) + if t.Key() != reflect.Typeof("") { + d.saveError(&UnmarshalTypeError{"object", v.Type()}) + break + } + mv = v + if mv.IsNil() { + mv.SetValue(reflect.MakeMap(t)) + } + case *reflect.StructValue: + sv = v + default: + d.saveError(&UnmarshalTypeError{"object", v.Type()}) + } + + if mv == nil && sv == nil { + d.off-- + d.next() // skip over { } in input + return + } + + for { + // Read opening " of string key or closing }. + op := d.scanWhile(scanSkipSpace) + if op == scanEndObject { + // closing } - can only happen on first iteration. + break + } + if op != scanBeginLiteral { + d.error(errPhase) + } + + // Read string key. + start := d.off - 1 + op = d.scanWhile(scanContinue) + item := d.data[start : d.off-1] + key, ok := unquote(item) + if !ok { + d.error(errPhase) + } + + // Figure out field corresponding to key. + var subv reflect.Value + if mv != nil { + subv = reflect.MakeZero(mv.Type().(*reflect.MapType).Elem()) + } else { + var f reflect.StructField + var ok bool + st := sv.Type().(*reflect.StructType) + // First try for field with that tag. + if isValidTag(key) { + for i := 0; i < sv.NumField(); i++ { + f = st.Field(i) + if f.Tag == key { + ok = true + break + } + } + } + if !ok { + // Second, exact match. + f, ok = st.FieldByName(key) + } + if !ok { + // Third, case-insensitive match. + f, ok = st.FieldByNameFunc(func(s string) bool { return matchName(key, s) }) + } + + // Extract value; name must be exported. + if ok { + if f.PkgPath != "" { + d.saveError(&UnmarshalFieldError{key, st, f}) + } else { + subv = sv.FieldByIndex(f.Index) + } + } + } + + // Read : before value. + if op == scanSkipSpace { + op = d.scanWhile(scanSkipSpace) + } + if op != scanObjectKey { + d.error(errPhase) + } + + // Read value. + d.value(subv) + + // Write value back to map; + // if using struct, subv points into struct already. + if mv != nil { + mv.SetElem(reflect.NewValue(key), subv) + } + + // Next token must be , or }. + op = d.scanWhile(scanSkipSpace) + if op == scanEndObject { + break + } + if op != scanObjectValue { + d.error(errPhase) + } + } +} + +// literal consumes a literal from d.data[d.off-1:], decoding into the value v. +// The first byte of the literal has been read already +// (that's how the caller knows it's a literal). +func (d *decodeState) literal(v reflect.Value) { + // All bytes inside literal return scanContinue op code. + start := d.off - 1 + op := d.scanWhile(scanContinue) + + // Scan read one byte too far; back up. + d.off-- + d.scan.undo(op) + item := d.data[start:d.off] + + // Check for unmarshaler. + wantptr := item[0] == 'n' // null + unmarshaler, pv := d.indirect(v, wantptr) + if unmarshaler != nil { + err := unmarshaler.UnmarshalJSON(item) + if err != nil { + d.error(err) + } + return + } + v = pv + + switch c := item[0]; c { + case 'n': // null + switch v.(type) { + default: + d.saveError(&UnmarshalTypeError{"null", v.Type()}) + case *reflect.InterfaceValue, *reflect.PtrValue, *reflect.MapValue: + v.SetValue(nil) + } + + case 't', 'f': // true, false + value := c == 't' + switch v := v.(type) { + default: + d.saveError(&UnmarshalTypeError{"bool", v.Type()}) + case *reflect.BoolValue: + v.Set(value) + case *reflect.InterfaceValue: + v.Set(reflect.NewValue(value)) + } + + case '"': // string + s, ok := unquoteBytes(item) + if !ok { + d.error(errPhase) + } + switch v := v.(type) { + default: + d.saveError(&UnmarshalTypeError{"string", v.Type()}) + case *reflect.SliceValue: + if v.Type() != byteSliceType { + d.saveError(&UnmarshalTypeError{"string", v.Type()}) + break + } + b := make([]byte, base64.StdEncoding.DecodedLen(len(s))) + n, err := base64.StdEncoding.Decode(b, s) + if err != nil { + d.saveError(err) + break + } + v.Set(reflect.NewValue(b[0:n]).(*reflect.SliceValue)) + case *reflect.StringValue: + v.Set(string(s)) + case *reflect.InterfaceValue: + v.Set(reflect.NewValue(string(s))) + } + + default: // number + if c != '-' && (c < '0' || c > '9') { + d.error(errPhase) + } + s := string(item) + switch v := v.(type) { + default: + d.error(&UnmarshalTypeError{"number", v.Type()}) + case *reflect.InterfaceValue: + n, err := strconv.Atof64(s) + if err != nil { + d.saveError(&UnmarshalTypeError{"number " + s, v.Type()}) + break + } + v.Set(reflect.NewValue(n)) + + case *reflect.IntValue: + n, err := strconv.Atoi64(s) + if err != nil || v.Overflow(n) { + d.saveError(&UnmarshalTypeError{"number " + s, v.Type()}) + break + } + v.Set(n) + + case *reflect.UintValue: + n, err := strconv.Atoui64(s) + if err != nil || v.Overflow(n) { + d.saveError(&UnmarshalTypeError{"number " + s, v.Type()}) + break + } + v.Set(n) + + case *reflect.FloatValue: + n, err := strconv.AtofN(s, v.Type().Bits()) + if err != nil || v.Overflow(n) { + d.saveError(&UnmarshalTypeError{"number " + s, v.Type()}) + break + } + v.Set(n) + } + } +} + +// The xxxInterface routines build up a value to be stored +// in an empty interface. They are not strictly necessary, +// but they avoid the weight of reflection in this common case. + +// valueInterface is like value but returns interface{} +func (d *decodeState) valueInterface() interface{} { + switch d.scanWhile(scanSkipSpace) { + default: + d.error(errPhase) + case scanBeginArray: + return d.arrayInterface() + case scanBeginObject: + return d.objectInterface() + case scanBeginLiteral: + return d.literalInterface() + } + panic("unreachable") +} + +// arrayInterface is like array but returns []interface{}. +func (d *decodeState) arrayInterface() []interface{} { + var v vector.Vector + for { + // Look ahead for ] - can only happen on first iteration. + op := d.scanWhile(scanSkipSpace) + if op == scanEndArray { + break + } + + // Back up so d.value can have the byte we just read. + d.off-- + d.scan.undo(op) + + v.Push(d.valueInterface()) + + // Next token must be , or ]. + op = d.scanWhile(scanSkipSpace) + if op == scanEndArray { + break + } + if op != scanArrayValue { + d.error(errPhase) + } + } + return v +} + +// objectInterface is like object but returns map[string]interface{}. +func (d *decodeState) objectInterface() map[string]interface{} { + m := make(map[string]interface{}) + for { + // Read opening " of string key or closing }. + op := d.scanWhile(scanSkipSpace) + if op == scanEndObject { + // closing } - can only happen on first iteration. + break + } + if op != scanBeginLiteral { + d.error(errPhase) + } + + // Read string key. + start := d.off - 1 + op = d.scanWhile(scanContinue) + item := d.data[start : d.off-1] + key, ok := unquote(item) + if !ok { + d.error(errPhase) + } + + // Read : before value. + if op == scanSkipSpace { + op = d.scanWhile(scanSkipSpace) + } + if op != scanObjectKey { + d.error(errPhase) + } + + // Read value. + m[key] = d.valueInterface() + + // Next token must be , or }. + op = d.scanWhile(scanSkipSpace) + if op == scanEndObject { + break + } + if op != scanObjectValue { + d.error(errPhase) + } + } + return m +} + +// literalInterface is like literal but returns an interface value. +func (d *decodeState) literalInterface() interface{} { + // All bytes inside literal return scanContinue op code. + start := d.off - 1 + op := d.scanWhile(scanContinue) + + // Scan read one byte too far; back up. + d.off-- + d.scan.undo(op) + item := d.data[start:d.off] + + switch c := item[0]; c { + case 'n': // null + return nil + + case 't', 'f': // true, false + return c == 't' + + case '"': // string + s, ok := unquote(item) + if !ok { + d.error(errPhase) + } + return s + + default: // number + if c != '-' && (c < '0' || c > '9') { + d.error(errPhase) + } + n, err := strconv.Atof64(string(item)) + if err != nil { + d.saveError(&UnmarshalTypeError{"number " + string(item), reflect.Typeof(0.0)}) + } + return n + } + panic("unreachable") +} + +// getu4 decodes \uXXXX from the beginning of s, returning the hex value, +// or it returns -1. +func getu4(s []byte) int { + if len(s) < 6 || s[0] != '\\' || s[1] != 'u' { + return -1 + } + rune, err := strconv.Btoui64(string(s[2:6]), 16) + if err != nil { + return -1 + } + return int(rune) +} + +// unquote converts a quoted JSON string literal s into an actual string t. +// The rules are different than for Go, so cannot use strconv.Unquote. +func unquote(s []byte) (t string, ok bool) { + s, ok = unquoteBytes(s) + t = string(s) + return +} + +func unquoteBytes(s []byte) (t []byte, ok bool) { + if len(s) < 2 || s[0] != '"' || s[len(s)-1] != '"' { + return + } + s = s[1 : len(s)-1] + + // Check for unusual characters. If there are none, + // then no unquoting is needed, so return a slice of the + // original bytes. + r := 0 + for r < len(s) { + c := s[r] + if c == '\\' || c == '"' || c < ' ' { + break + } + if c < utf8.RuneSelf { + r++ + continue + } + rune, size := utf8.DecodeRune(s[r:]) + if rune == utf8.RuneError && size == 1 { + break + } + r += size + } + if r == len(s) { + return s, true + } + + b := make([]byte, len(s)+2*utf8.UTFMax) + w := copy(b, s[0:r]) + for r < len(s) { + // Out of room? Can only happen if s is full of + // malformed UTF-8 and we're replacing each + // byte with RuneError. + if w >= len(b)-2*utf8.UTFMax { + nb := make([]byte, (len(b)+utf8.UTFMax)*2) + copy(nb, b[0:w]) + b = nb + } + switch c := s[r]; { + case c == '\\': + r++ + if r >= len(s) { + return + } + switch s[r] { + default: + return + case '"', '\\', '/', '\'': + b[w] = s[r] + r++ + w++ + case 'b': + b[w] = '\b' + r++ + w++ + case 'f': + b[w] = '\f' + r++ + w++ + case 'n': + b[w] = '\n' + r++ + w++ + case 'r': + b[w] = '\r' + r++ + w++ + case 't': + b[w] = '\t' + r++ + w++ + case 'u': + r-- + rune := getu4(s[r:]) + if rune < 0 { + return + } + r += 6 + if utf16.IsSurrogate(rune) { + rune1 := getu4(s[r:]) + if dec := utf16.DecodeRune(rune, rune1); dec != unicode.ReplacementChar { + // A valid pair; consume. + r += 6 + w += utf8.EncodeRune(b[w:], dec) + break + } + // Invalid surrogate; fall back to replacement rune. + rune = unicode.ReplacementChar + } + w += utf8.EncodeRune(b[w:], rune) + } + + // Quote, control characters are invalid. + case c == '"', c < ' ': + return + + // ASCII + case c < utf8.RuneSelf: + b[w] = c + r++ + w++ + + // Coerce to well-formed UTF-8. + default: + rune, size := utf8.DecodeRune(s[r:]) + r += size + w += utf8.EncodeRune(b[w:], rune) + } + } + return b[0:w], true +} diff --git a/src/cmd/gofix/testdata/reflect.decode.go.out b/src/cmd/gofix/testdata/reflect.decode.go.out new file mode 100644 index 000000000..fb7910ee3 --- /dev/null +++ b/src/cmd/gofix/testdata/reflect.decode.go.out @@ -0,0 +1,908 @@ +// Copyright 2010 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Represents JSON data structure using native Go types: booleans, floats, +// strings, arrays, and maps. + +package json + +import ( + "container/vector" + "encoding/base64" + "os" + "reflect" + "runtime" + "strconv" + "strings" + "unicode" + "utf16" + "utf8" +) + +// Unmarshal parses the JSON-encoded data and stores the result +// in the value pointed to by v. +// +// Unmarshal traverses the value v recursively. +// If an encountered value implements the Unmarshaler interface, +// Unmarshal calls its UnmarshalJSON method with a well-formed +// JSON encoding. +// +// Otherwise, Unmarshal uses the inverse of the encodings that +// Marshal uses, allocating maps, slices, and pointers as necessary, +// with the following additional rules: +// +// To unmarshal a JSON value into a nil interface value, the +// type stored in the interface value is one of: +// +// bool, for JSON booleans +// float64, for JSON numbers +// string, for JSON strings +// []interface{}, for JSON arrays +// map[string]interface{}, for JSON objects +// nil for JSON null +// +// If a JSON value is not appropriate for a given target type, +// or if a JSON number overflows the target type, Unmarshal +// skips that field and completes the unmarshalling as best it can. +// If no more serious errors are encountered, Unmarshal returns +// an UnmarshalTypeError describing the earliest such error. +// +func Unmarshal(data []byte, v interface{}) os.Error { + d := new(decodeState).init(data) + + // Quick check for well-formedness. + // Avoids filling out half a data structure + // before discovering a JSON syntax error. + err := checkValid(data, &d.scan) + if err != nil { + return err + } + + return d.unmarshal(v) +} + +// Unmarshaler is the interface implemented by objects +// that can unmarshal a JSON description of themselves. +// The input can be assumed to be a valid JSON object +// encoding. UnmarshalJSON must copy the JSON data +// if it wishes to retain the data after returning. +type Unmarshaler interface { + UnmarshalJSON([]byte) os.Error +} + +// An UnmarshalTypeError describes a JSON value that was +// not appropriate for a value of a specific Go type. +type UnmarshalTypeError struct { + Value string // description of JSON value - "bool", "array", "number -5" + Type reflect.Type // type of Go value it could not be assigned to +} + +func (e *UnmarshalTypeError) String() string { + return "json: cannot unmarshal " + e.Value + " into Go value of type " + e.Type.String() +} + +// An UnmarshalFieldError describes a JSON object key that +// led to an unexported (and therefore unwritable) struct field. +type UnmarshalFieldError struct { + Key string + Type reflect.Type + Field reflect.StructField +} + +func (e *UnmarshalFieldError) String() string { + return "json: cannot unmarshal object key " + strconv.Quote(e.Key) + " into unexported field " + e.Field.Name + " of type " + e.Type.String() +} + +// An InvalidUnmarshalError describes an invalid argument passed to Unmarshal. +// (The argument to Unmarshal must be a non-nil pointer.) +type InvalidUnmarshalError struct { + Type reflect.Type +} + +func (e *InvalidUnmarshalError) String() string { + if e.Type == nil { + return "json: Unmarshal(nil)" + } + + if e.Type.Kind() != reflect.Ptr { + return "json: Unmarshal(non-pointer " + e.Type.String() + ")" + } + return "json: Unmarshal(nil " + e.Type.String() + ")" +} + +func (d *decodeState) unmarshal(v interface{}) (err os.Error) { + defer func() { + if r := recover(); r != nil { + if _, ok := r.(runtime.Error); ok { + panic(r) + } + err = r.(os.Error) + } + }() + + rv := reflect.ValueOf(v) + pv := rv + if pv.Kind() != reflect.Ptr || + pv.IsNil() { + return &InvalidUnmarshalError{reflect.TypeOf(v)} + } + + d.scan.reset() + // We decode rv not pv.Elem because the Unmarshaler interface + // test must be applied at the top level of the value. + d.value(rv) + return d.savedError +} + +// decodeState represents the state while decoding a JSON value. +type decodeState struct { + data []byte + off int // read offset in data + scan scanner + nextscan scanner // for calls to nextValue + savedError os.Error +} + +// errPhase is used for errors that should not happen unless +// there is a bug in the JSON decoder or something is editing +// the data slice while the decoder executes. +var errPhase = os.NewError("JSON decoder out of sync - data changing underfoot?") + +func (d *decodeState) init(data []byte) *decodeState { + d.data = data + d.off = 0 + d.savedError = nil + return d +} + +// error aborts the decoding by panicking with err. +func (d *decodeState) error(err os.Error) { + panic(err) +} + +// saveError saves the first err it is called with, +// for reporting at the end of the unmarshal. +func (d *decodeState) saveError(err os.Error) { + if d.savedError == nil { + d.savedError = err + } +} + +// next cuts off and returns the next full JSON value in d.data[d.off:]. +// The next value is known to be an object or array, not a literal. +func (d *decodeState) next() []byte { + c := d.data[d.off] + item, rest, err := nextValue(d.data[d.off:], &d.nextscan) + if err != nil { + d.error(err) + } + d.off = len(d.data) - len(rest) + + // Our scanner has seen the opening brace/bracket + // and thinks we're still in the middle of the object. + // invent a closing brace/bracket to get it out. + if c == '{' { + d.scan.step(&d.scan, '}') + } else { + d.scan.step(&d.scan, ']') + } + + return item +} + +// scanWhile processes bytes in d.data[d.off:] until it +// receives a scan code not equal to op. +// It updates d.off and returns the new scan code. +func (d *decodeState) scanWhile(op int) int { + var newOp int + for { + if d.off >= len(d.data) { + newOp = d.scan.eof() + d.off = len(d.data) + 1 // mark processed EOF with len+1 + } else { + c := int(d.data[d.off]) + d.off++ + newOp = d.scan.step(&d.scan, c) + } + if newOp != op { + break + } + } + return newOp +} + +// value decodes a JSON value from d.data[d.off:] into the value. +// it updates d.off to point past the decoded value. +func (d *decodeState) value(v reflect.Value) { + if !v.IsValid() { + _, rest, err := nextValue(d.data[d.off:], &d.nextscan) + if err != nil { + d.error(err) + } + d.off = len(d.data) - len(rest) + + // d.scan thinks we're still at the beginning of the item. + // Feed in an empty string - the shortest, simplest value - + // so that it knows we got to the end of the value. + if d.scan.step == stateRedo { + panic("redo") + } + d.scan.step(&d.scan, '"') + d.scan.step(&d.scan, '"') + return + } + + switch op := d.scanWhile(scanSkipSpace); op { + default: + d.error(errPhase) + + case scanBeginArray: + d.array(v) + + case scanBeginObject: + d.object(v) + + case scanBeginLiteral: + d.literal(v) + } +} + +// indirect walks down v allocating pointers as needed, +// until it gets to a non-pointer. +// if it encounters an Unmarshaler, indirect stops and returns that. +// if wantptr is true, indirect stops at the last pointer. +func (d *decodeState) indirect(v reflect.Value, wantptr bool) (Unmarshaler, reflect.Value) { + for { + var isUnmarshaler bool + if v.Type().NumMethod() > 0 { + // Remember that this is an unmarshaler, + // but wait to return it until after allocating + // the pointer (if necessary). + _, isUnmarshaler = v.Interface().(Unmarshaler) + } + + if iv := v; iv.Kind() == reflect.Interface && !iv.IsNil() { + v = iv.Elem() + continue + } + pv := v + if pv.Kind() != reflect.Ptr { + break + } + + if pv.Elem().Kind() != reflect.Ptr && + wantptr && !isUnmarshaler { + return nil, pv + } + if pv.IsNil() { + pv.Set(reflect.Zero(pv.Type().Elem()).Addr()) + } + if isUnmarshaler { + // Using v.Interface().(Unmarshaler) + // here means that we have to use a pointer + // as the struct field. We cannot use a value inside + // a pointer to a struct, because in that case + // v.Interface() is the value (x.f) not the pointer (&x.f). + // This is an unfortunate consequence of reflect. + // An alternative would be to look up the + // UnmarshalJSON method and return a FuncValue. + return v.Interface().(Unmarshaler), reflect.Value{} + } + v = pv.Elem() + } + return nil, v +} + +// array consumes an array from d.data[d.off-1:], decoding into the value v. +// the first byte of the array ('[') has been read already. +func (d *decodeState) array(v reflect.Value) { + // Check for unmarshaler. + unmarshaler, pv := d.indirect(v, false) + if unmarshaler != nil { + d.off-- + err := unmarshaler.UnmarshalJSON(d.next()) + if err != nil { + d.error(err) + } + return + } + v = pv + + // Decoding into nil interface? Switch to non-reflect code. + iv := v + ok := iv.Kind() == reflect.Interface + if ok { + iv.Set(reflect.ValueOf(d.arrayInterface())) + return + } + + // Check type of target. + av := v + if av.Kind() != reflect.Array && av.Kind() != reflect.Slice { + d.saveError(&UnmarshalTypeError{"array", v.Type()}) + d.off-- + d.next() + return + } + + sv := v + + i := 0 + for { + // Look ahead for ] - can only happen on first iteration. + op := d.scanWhile(scanSkipSpace) + if op == scanEndArray { + break + } + + // Back up so d.value can have the byte we just read. + d.off-- + d.scan.undo(op) + + // Get element of array, growing if necessary. + if i >= av.Cap() && sv.IsValid() { + newcap := sv.Cap() + sv.Cap()/2 + if newcap < 4 { + newcap = 4 + } + newv := reflect.MakeSlice(sv.Type(), sv.Len(), newcap) + reflect.Copy(newv, sv) + sv.Set(newv) + } + if i >= av.Len() && sv.IsValid() { + // Must be slice; gave up on array during i >= av.Cap(). + sv.SetLen(i + 1) + } + + // Decode into element. + if i < av.Len() { + d.value(av.Index(i)) + } else { + // Ran out of fixed array: skip. + d.value(reflect.Value{}) + } + i++ + + // Next token must be , or ]. + op = d.scanWhile(scanSkipSpace) + if op == scanEndArray { + break + } + if op != scanArrayValue { + d.error(errPhase) + } + } + if i < av.Len() { + if !sv.IsValid() { + // Array. Zero the rest. + z := reflect.Zero(av.Type().Elem()) + for ; i < av.Len(); i++ { + av.Index(i).Set(z) + } + } else { + sv.SetLen(i) + } + } +} + +// matchName returns true if key should be written to a field named name. +func matchName(key, name string) bool { + return strings.ToLower(key) == strings.ToLower(name) +} + +// object consumes an object from d.data[d.off-1:], decoding into the value v. +// the first byte of the object ('{') has been read already. +func (d *decodeState) object(v reflect.Value) { + // Check for unmarshaler. + unmarshaler, pv := d.indirect(v, false) + if unmarshaler != nil { + d.off-- + err := unmarshaler.UnmarshalJSON(d.next()) + if err != nil { + d.error(err) + } + return + } + v = pv + + // Decoding into nil interface? Switch to non-reflect code. + iv := v + if iv.Kind() == reflect.Interface { + iv.Set(reflect.ValueOf(d.objectInterface())) + return + } + + // Check type of target: struct or map[string]T + var ( + mv reflect.Value + sv reflect.Value + ) + switch v.Kind() { + case reflect.Map: + // map must have string type + t := v.Type() + if t.Key() != reflect.TypeOf("") { + d.saveError(&UnmarshalTypeError{"object", v.Type()}) + break + } + mv = v + if mv.IsNil() { + mv.Set(reflect.MakeMap(t)) + } + case reflect.Struct: + sv = v + default: + d.saveError(&UnmarshalTypeError{"object", v.Type()}) + } + + if !mv.IsValid() && !sv.IsValid() { + d.off-- + d.next() // skip over { } in input + return + } + + for { + // Read opening " of string key or closing }. + op := d.scanWhile(scanSkipSpace) + if op == scanEndObject { + // closing } - can only happen on first iteration. + break + } + if op != scanBeginLiteral { + d.error(errPhase) + } + + // Read string key. + start := d.off - 1 + op = d.scanWhile(scanContinue) + item := d.data[start : d.off-1] + key, ok := unquote(item) + if !ok { + d.error(errPhase) + } + + // Figure out field corresponding to key. + var subv reflect.Value + if mv.IsValid() { + subv = reflect.Zero(mv.Type().Elem()) + } else { + var f reflect.StructField + var ok bool + st := sv.Type() + // First try for field with that tag. + if isValidTag(key) { + for i := 0; i < sv.NumField(); i++ { + f = st.Field(i) + if f.Tag == key { + ok = true + break + } + } + } + if !ok { + // Second, exact match. + f, ok = st.FieldByName(key) + } + if !ok { + // Third, case-insensitive match. + f, ok = st.FieldByNameFunc(func(s string) bool { return matchName(key, s) }) + } + + // Extract value; name must be exported. + if ok { + if f.PkgPath != "" { + d.saveError(&UnmarshalFieldError{key, st, f}) + } else { + subv = sv.FieldByIndex(f.Index) + } + } + } + + // Read : before value. + if op == scanSkipSpace { + op = d.scanWhile(scanSkipSpace) + } + if op != scanObjectKey { + d.error(errPhase) + } + + // Read value. + d.value(subv) + + // Write value back to map; + // if using struct, subv points into struct already. + if mv.IsValid() { + mv.SetMapIndex(reflect.ValueOf(key), subv) + } + + // Next token must be , or }. + op = d.scanWhile(scanSkipSpace) + if op == scanEndObject { + break + } + if op != scanObjectValue { + d.error(errPhase) + } + } +} + +// literal consumes a literal from d.data[d.off-1:], decoding into the value v. +// The first byte of the literal has been read already +// (that's how the caller knows it's a literal). +func (d *decodeState) literal(v reflect.Value) { + // All bytes inside literal return scanContinue op code. + start := d.off - 1 + op := d.scanWhile(scanContinue) + + // Scan read one byte too far; back up. + d.off-- + d.scan.undo(op) + item := d.data[start:d.off] + + // Check for unmarshaler. + wantptr := item[0] == 'n' // null + unmarshaler, pv := d.indirect(v, wantptr) + if unmarshaler != nil { + err := unmarshaler.UnmarshalJSON(item) + if err != nil { + d.error(err) + } + return + } + v = pv + + switch c := item[0]; c { + case 'n': // null + switch v.Kind() { + default: + d.saveError(&UnmarshalTypeError{"null", v.Type()}) + case reflect.Interface, reflect.Ptr, reflect.Map: + v.Set(reflect.Zero(v.Type())) + } + + case 't', 'f': // true, false + value := c == 't' + switch v.Kind() { + default: + d.saveError(&UnmarshalTypeError{"bool", v.Type()}) + case reflect.Bool: + v.SetBool(value) + case reflect.Interface: + v.Set(reflect.ValueOf(value)) + } + + case '"': // string + s, ok := unquoteBytes(item) + if !ok { + d.error(errPhase) + } + switch v.Kind() { + default: + d.saveError(&UnmarshalTypeError{"string", v.Type()}) + case reflect.Slice: + if v.Type() != byteSliceType { + d.saveError(&UnmarshalTypeError{"string", v.Type()}) + break + } + b := make([]byte, base64.StdEncoding.DecodedLen(len(s))) + n, err := base64.StdEncoding.Decode(b, s) + if err != nil { + d.saveError(err) + break + } + v.Set(reflect.ValueOf(b[0:n])) + case reflect.String: + v.SetString(string(s)) + case reflect.Interface: + v.Set(reflect.ValueOf(string(s))) + } + + default: // number + if c != '-' && (c < '0' || c > '9') { + d.error(errPhase) + } + s := string(item) + switch v.Kind() { + default: + d.error(&UnmarshalTypeError{"number", v.Type()}) + case reflect.Interface: + n, err := strconv.Atof64(s) + if err != nil { + d.saveError(&UnmarshalTypeError{"number " + s, v.Type()}) + break + } + v.Set(reflect.ValueOf(n)) + + case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64: + n, err := strconv.Atoi64(s) + if err != nil || v.OverflowInt(n) { + d.saveError(&UnmarshalTypeError{"number " + s, v.Type()}) + break + } + v.SetInt(n) + + case reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64, reflect.Uintptr: + n, err := strconv.Atoui64(s) + if err != nil || v.OverflowUint(n) { + d.saveError(&UnmarshalTypeError{"number " + s, v.Type()}) + break + } + v.SetUint(n) + + case reflect.Float32, reflect.Float64: + n, err := strconv.AtofN(s, v.Type().Bits()) + if err != nil || v.OverflowFloat(n) { + d.saveError(&UnmarshalTypeError{"number " + s, v.Type()}) + break + } + v.SetFloat(n) + } + } +} + +// The xxxInterface routines build up a value to be stored +// in an empty interface. They are not strictly necessary, +// but they avoid the weight of reflection in this common case. + +// valueInterface is like value but returns interface{} +func (d *decodeState) valueInterface() interface{} { + switch d.scanWhile(scanSkipSpace) { + default: + d.error(errPhase) + case scanBeginArray: + return d.arrayInterface() + case scanBeginObject: + return d.objectInterface() + case scanBeginLiteral: + return d.literalInterface() + } + panic("unreachable") +} + +// arrayInterface is like array but returns []interface{}. +func (d *decodeState) arrayInterface() []interface{} { + var v vector.Vector + for { + // Look ahead for ] - can only happen on first iteration. + op := d.scanWhile(scanSkipSpace) + if op == scanEndArray { + break + } + + // Back up so d.value can have the byte we just read. + d.off-- + d.scan.undo(op) + + v.Push(d.valueInterface()) + + // Next token must be , or ]. + op = d.scanWhile(scanSkipSpace) + if op == scanEndArray { + break + } + if op != scanArrayValue { + d.error(errPhase) + } + } + return v +} + +// objectInterface is like object but returns map[string]interface{}. +func (d *decodeState) objectInterface() map[string]interface{} { + m := make(map[string]interface{}) + for { + // Read opening " of string key or closing }. + op := d.scanWhile(scanSkipSpace) + if op == scanEndObject { + // closing } - can only happen on first iteration. + break + } + if op != scanBeginLiteral { + d.error(errPhase) + } + + // Read string key. + start := d.off - 1 + op = d.scanWhile(scanContinue) + item := d.data[start : d.off-1] + key, ok := unquote(item) + if !ok { + d.error(errPhase) + } + + // Read : before value. + if op == scanSkipSpace { + op = d.scanWhile(scanSkipSpace) + } + if op != scanObjectKey { + d.error(errPhase) + } + + // Read value. + m[key] = d.valueInterface() + + // Next token must be , or }. + op = d.scanWhile(scanSkipSpace) + if op == scanEndObject { + break + } + if op != scanObjectValue { + d.error(errPhase) + } + } + return m +} + +// literalInterface is like literal but returns an interface value. +func (d *decodeState) literalInterface() interface{} { + // All bytes inside literal return scanContinue op code. + start := d.off - 1 + op := d.scanWhile(scanContinue) + + // Scan read one byte too far; back up. + d.off-- + d.scan.undo(op) + item := d.data[start:d.off] + + switch c := item[0]; c { + case 'n': // null + return nil + + case 't', 'f': // true, false + return c == 't' + + case '"': // string + s, ok := unquote(item) + if !ok { + d.error(errPhase) + } + return s + + default: // number + if c != '-' && (c < '0' || c > '9') { + d.error(errPhase) + } + n, err := strconv.Atof64(string(item)) + if err != nil { + d.saveError(&UnmarshalTypeError{"number " + string(item), reflect.TypeOf(0.0)}) + } + return n + } + panic("unreachable") +} + +// getu4 decodes \uXXXX from the beginning of s, returning the hex value, +// or it returns -1. +func getu4(s []byte) int { + if len(s) < 6 || s[0] != '\\' || s[1] != 'u' { + return -1 + } + rune, err := strconv.Btoui64(string(s[2:6]), 16) + if err != nil { + return -1 + } + return int(rune) +} + +// unquote converts a quoted JSON string literal s into an actual string t. +// The rules are different than for Go, so cannot use strconv.Unquote. +func unquote(s []byte) (t string, ok bool) { + s, ok = unquoteBytes(s) + t = string(s) + return +} + +func unquoteBytes(s []byte) (t []byte, ok bool) { + if len(s) < 2 || s[0] != '"' || s[len(s)-1] != '"' { + return + } + s = s[1 : len(s)-1] + + // Check for unusual characters. If there are none, + // then no unquoting is needed, so return a slice of the + // original bytes. + r := 0 + for r < len(s) { + c := s[r] + if c == '\\' || c == '"' || c < ' ' { + break + } + if c < utf8.RuneSelf { + r++ + continue + } + rune, size := utf8.DecodeRune(s[r:]) + if rune == utf8.RuneError && size == 1 { + break + } + r += size + } + if r == len(s) { + return s, true + } + + b := make([]byte, len(s)+2*utf8.UTFMax) + w := copy(b, s[0:r]) + for r < len(s) { + // Out of room? Can only happen if s is full of + // malformed UTF-8 and we're replacing each + // byte with RuneError. + if w >= len(b)-2*utf8.UTFMax { + nb := make([]byte, (len(b)+utf8.UTFMax)*2) + copy(nb, b[0:w]) + b = nb + } + switch c := s[r]; { + case c == '\\': + r++ + if r >= len(s) { + return + } + switch s[r] { + default: + return + case '"', '\\', '/', '\'': + b[w] = s[r] + r++ + w++ + case 'b': + b[w] = '\b' + r++ + w++ + case 'f': + b[w] = '\f' + r++ + w++ + case 'n': + b[w] = '\n' + r++ + w++ + case 'r': + b[w] = '\r' + r++ + w++ + case 't': + b[w] = '\t' + r++ + w++ + case 'u': + r-- + rune := getu4(s[r:]) + if rune < 0 { + return + } + r += 6 + if utf16.IsSurrogate(rune) { + rune1 := getu4(s[r:]) + if dec := utf16.DecodeRune(rune, rune1); dec != unicode.ReplacementChar { + // A valid pair; consume. + r += 6 + w += utf8.EncodeRune(b[w:], dec) + break + } + // Invalid surrogate; fall back to replacement rune. + rune = unicode.ReplacementChar + } + w += utf8.EncodeRune(b[w:], rune) + } + + // Quote, control characters are invalid. + case c == '"', c < ' ': + return + + // ASCII + case c < utf8.RuneSelf: + b[w] = c + r++ + w++ + + // Coerce to well-formed UTF-8. + default: + rune, size := utf8.DecodeRune(s[r:]) + r += size + w += utf8.EncodeRune(b[w:], rune) + } + } + return b[0:w], true +} diff --git a/src/cmd/gofix/testdata/reflect.decoder.go.in b/src/cmd/gofix/testdata/reflect.decoder.go.in new file mode 100644 index 000000000..34364161a --- /dev/null +++ b/src/cmd/gofix/testdata/reflect.decoder.go.in @@ -0,0 +1,196 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package gob + +import ( + "bufio" + "bytes" + "io" + "os" + "reflect" + "sync" +) + +// A Decoder manages the receipt of type and data information read from the +// remote side of a connection. +type Decoder struct { + mutex sync.Mutex // each item must be received atomically + r io.Reader // source of the data + buf bytes.Buffer // buffer for more efficient i/o from r + wireType map[typeId]*wireType // map from remote ID to local description + decoderCache map[reflect.Type]map[typeId]**decEngine // cache of compiled engines + ignorerCache map[typeId]**decEngine // ditto for ignored objects + freeList *decoderState // list of free decoderStates; avoids reallocation + countBuf []byte // used for decoding integers while parsing messages + tmp []byte // temporary storage for i/o; saves reallocating + err os.Error +} + +// NewDecoder returns a new decoder that reads from the io.Reader. +func NewDecoder(r io.Reader) *Decoder { + dec := new(Decoder) + dec.r = bufio.NewReader(r) + dec.wireType = make(map[typeId]*wireType) + dec.decoderCache = make(map[reflect.Type]map[typeId]**decEngine) + dec.ignorerCache = make(map[typeId]**decEngine) + dec.countBuf = make([]byte, 9) // counts may be uint64s (unlikely!), require 9 bytes + + return dec +} + +// recvType loads the definition of a type. +func (dec *Decoder) recvType(id typeId) { + // Have we already seen this type? That's an error + if id < firstUserId || dec.wireType[id] != nil { + dec.err = os.ErrorString("gob: duplicate type received") + return + } + + // Type: + wire := new(wireType) + dec.decodeValue(tWireType, reflect.NewValue(wire)) + if dec.err != nil { + return + } + // Remember we've seen this type. + dec.wireType[id] = wire +} + +// recvMessage reads the next count-delimited item from the input. It is the converse +// of Encoder.writeMessage. It returns false on EOF or other error reading the message. +func (dec *Decoder) recvMessage() bool { + // Read a count. + nbytes, _, err := decodeUintReader(dec.r, dec.countBuf) + if err != nil { + dec.err = err + return false + } + dec.readMessage(int(nbytes)) + return dec.err == nil +} + +// readMessage reads the next nbytes bytes from the input. +func (dec *Decoder) readMessage(nbytes int) { + // Allocate the buffer. + if cap(dec.tmp) < nbytes { + dec.tmp = make([]byte, nbytes+100) // room to grow + } + dec.tmp = dec.tmp[:nbytes] + + // Read the data + _, dec.err = io.ReadFull(dec.r, dec.tmp) + if dec.err != nil { + if dec.err == os.EOF { + dec.err = io.ErrUnexpectedEOF + } + return + } + dec.buf.Write(dec.tmp) +} + +// toInt turns an encoded uint64 into an int, according to the marshaling rules. +func toInt(x uint64) int64 { + i := int64(x >> 1) + if x&1 != 0 { + i = ^i + } + return i +} + +func (dec *Decoder) nextInt() int64 { + n, _, err := decodeUintReader(&dec.buf, dec.countBuf) + if err != nil { + dec.err = err + } + return toInt(n) +} + +func (dec *Decoder) nextUint() uint64 { + n, _, err := decodeUintReader(&dec.buf, dec.countBuf) + if err != nil { + dec.err = err + } + return n +} + +// decodeTypeSequence parses: +// TypeSequence +// (TypeDefinition DelimitedTypeDefinition*)? +// and returns the type id of the next value. It returns -1 at +// EOF. Upon return, the remainder of dec.buf is the value to be +// decoded. If this is an interface value, it can be ignored by +// simply resetting that buffer. +func (dec *Decoder) decodeTypeSequence(isInterface bool) typeId { + for dec.err == nil { + if dec.buf.Len() == 0 { + if !dec.recvMessage() { + break + } + } + // Receive a type id. + id := typeId(dec.nextInt()) + if id >= 0 { + // Value follows. + return id + } + // Type definition for (-id) follows. + dec.recvType(-id) + // When decoding an interface, after a type there may be a + // DelimitedValue still in the buffer. Skip its count. + // (Alternatively, the buffer is empty and the byte count + // will be absorbed by recvMessage.) + if dec.buf.Len() > 0 { + if !isInterface { + dec.err = os.ErrorString("extra data in buffer") + break + } + dec.nextUint() + } + } + return -1 +} + +// Decode reads the next value from the connection and stores +// it in the data represented by the empty interface value. +// If e is nil, the value will be discarded. Otherwise, +// the value underlying e must either be the correct type for the next +// data item received, and must be a pointer. +func (dec *Decoder) Decode(e interface{}) os.Error { + if e == nil { + return dec.DecodeValue(nil) + } + value := reflect.NewValue(e) + // If e represents a value as opposed to a pointer, the answer won't + // get back to the caller. Make sure it's a pointer. + if value.Type().Kind() != reflect.Ptr { + dec.err = os.ErrorString("gob: attempt to decode into a non-pointer") + return dec.err + } + return dec.DecodeValue(value) +} + +// DecodeValue reads the next value from the connection and stores +// it in the data represented by the reflection value. +// The value must be the correct type for the next +// data item received, or it may be nil, which means the +// value will be discarded. +func (dec *Decoder) DecodeValue(value reflect.Value) os.Error { + // Make sure we're single-threaded through here. + dec.mutex.Lock() + defer dec.mutex.Unlock() + + dec.buf.Reset() // In case data lingers from previous invocation. + dec.err = nil + id := dec.decodeTypeSequence(false) + if dec.err == nil { + dec.decodeValue(id, value) + } + return dec.err +} + +// If debug.go is compiled into the program , debugFunc prints a human-readable +// representation of the gob data read from r by calling that file's Debug function. +// Otherwise it is nil. +var debugFunc func(io.Reader) diff --git a/src/cmd/gofix/testdata/reflect.decoder.go.out b/src/cmd/gofix/testdata/reflect.decoder.go.out new file mode 100644 index 000000000..ece88ecbe --- /dev/null +++ b/src/cmd/gofix/testdata/reflect.decoder.go.out @@ -0,0 +1,196 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package gob + +import ( + "bufio" + "bytes" + "io" + "os" + "reflect" + "sync" +) + +// A Decoder manages the receipt of type and data information read from the +// remote side of a connection. +type Decoder struct { + mutex sync.Mutex // each item must be received atomically + r io.Reader // source of the data + buf bytes.Buffer // buffer for more efficient i/o from r + wireType map[typeId]*wireType // map from remote ID to local description + decoderCache map[reflect.Type]map[typeId]**decEngine // cache of compiled engines + ignorerCache map[typeId]**decEngine // ditto for ignored objects + freeList *decoderState // list of free decoderStates; avoids reallocation + countBuf []byte // used for decoding integers while parsing messages + tmp []byte // temporary storage for i/o; saves reallocating + err os.Error +} + +// NewDecoder returns a new decoder that reads from the io.Reader. +func NewDecoder(r io.Reader) *Decoder { + dec := new(Decoder) + dec.r = bufio.NewReader(r) + dec.wireType = make(map[typeId]*wireType) + dec.decoderCache = make(map[reflect.Type]map[typeId]**decEngine) + dec.ignorerCache = make(map[typeId]**decEngine) + dec.countBuf = make([]byte, 9) // counts may be uint64s (unlikely!), require 9 bytes + + return dec +} + +// recvType loads the definition of a type. +func (dec *Decoder) recvType(id typeId) { + // Have we already seen this type? That's an error + if id < firstUserId || dec.wireType[id] != nil { + dec.err = os.NewError("gob: duplicate type received") + return + } + + // Type: + wire := new(wireType) + dec.decodeValue(tWireType, reflect.ValueOf(wire)) + if dec.err != nil { + return + } + // Remember we've seen this type. + dec.wireType[id] = wire +} + +// recvMessage reads the next count-delimited item from the input. It is the converse +// of Encoder.writeMessage. It returns false on EOF or other error reading the message. +func (dec *Decoder) recvMessage() bool { + // Read a count. + nbytes, _, err := decodeUintReader(dec.r, dec.countBuf) + if err != nil { + dec.err = err + return false + } + dec.readMessage(int(nbytes)) + return dec.err == nil +} + +// readMessage reads the next nbytes bytes from the input. +func (dec *Decoder) readMessage(nbytes int) { + // Allocate the buffer. + if cap(dec.tmp) < nbytes { + dec.tmp = make([]byte, nbytes+100) // room to grow + } + dec.tmp = dec.tmp[:nbytes] + + // Read the data + _, dec.err = io.ReadFull(dec.r, dec.tmp) + if dec.err != nil { + if dec.err == os.EOF { + dec.err = io.ErrUnexpectedEOF + } + return + } + dec.buf.Write(dec.tmp) +} + +// toInt turns an encoded uint64 into an int, according to the marshaling rules. +func toInt(x uint64) int64 { + i := int64(x >> 1) + if x&1 != 0 { + i = ^i + } + return i +} + +func (dec *Decoder) nextInt() int64 { + n, _, err := decodeUintReader(&dec.buf, dec.countBuf) + if err != nil { + dec.err = err + } + return toInt(n) +} + +func (dec *Decoder) nextUint() uint64 { + n, _, err := decodeUintReader(&dec.buf, dec.countBuf) + if err != nil { + dec.err = err + } + return n +} + +// decodeTypeSequence parses: +// TypeSequence +// (TypeDefinition DelimitedTypeDefinition*)? +// and returns the type id of the next value. It returns -1 at +// EOF. Upon return, the remainder of dec.buf is the value to be +// decoded. If this is an interface value, it can be ignored by +// simply resetting that buffer. +func (dec *Decoder) decodeTypeSequence(isInterface bool) typeId { + for dec.err == nil { + if dec.buf.Len() == 0 { + if !dec.recvMessage() { + break + } + } + // Receive a type id. + id := typeId(dec.nextInt()) + if id >= 0 { + // Value follows. + return id + } + // Type definition for (-id) follows. + dec.recvType(-id) + // When decoding an interface, after a type there may be a + // DelimitedValue still in the buffer. Skip its count. + // (Alternatively, the buffer is empty and the byte count + // will be absorbed by recvMessage.) + if dec.buf.Len() > 0 { + if !isInterface { + dec.err = os.NewError("extra data in buffer") + break + } + dec.nextUint() + } + } + return -1 +} + +// Decode reads the next value from the connection and stores +// it in the data represented by the empty interface value. +// If e is nil, the value will be discarded. Otherwise, +// the value underlying e must either be the correct type for the next +// data item received, and must be a pointer. +func (dec *Decoder) Decode(e interface{}) os.Error { + if e == nil { + return dec.DecodeValue(reflect.Value{}) + } + value := reflect.ValueOf(e) + // If e represents a value as opposed to a pointer, the answer won't + // get back to the caller. Make sure it's a pointer. + if value.Type().Kind() != reflect.Ptr { + dec.err = os.NewError("gob: attempt to decode into a non-pointer") + return dec.err + } + return dec.DecodeValue(value) +} + +// DecodeValue reads the next value from the connection and stores +// it in the data represented by the reflection value. +// The value must be the correct type for the next +// data item received, or it may be nil, which means the +// value will be discarded. +func (dec *Decoder) DecodeValue(value reflect.Value) os.Error { + // Make sure we're single-threaded through here. + dec.mutex.Lock() + defer dec.mutex.Unlock() + + dec.buf.Reset() // In case data lingers from previous invocation. + dec.err = nil + id := dec.decodeTypeSequence(false) + if dec.err == nil { + dec.decodeValue(id, value) + } + return dec.err +} + +// If debug.go is compiled into the program , debugFunc prints a human-readable +// representation of the gob data read from r by calling that file's Debug function. +// Otherwise it is nil. +var debugFunc func(io.Reader) diff --git a/src/cmd/gofix/testdata/reflect.dnsmsg.go.in b/src/cmd/gofix/testdata/reflect.dnsmsg.go.in new file mode 100644 index 000000000..3d9c312f2 --- /dev/null +++ b/src/cmd/gofix/testdata/reflect.dnsmsg.go.in @@ -0,0 +1,777 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// DNS packet assembly. See RFC 1035. +// +// This is intended to support name resolution during net.Dial. +// It doesn't have to be blazing fast. +// +// Rather than write the usual handful of routines to pack and +// unpack every message that can appear on the wire, we use +// reflection to write a generic pack/unpack for structs and then +// use it. Thus, if in the future we need to define new message +// structs, no new pack/unpack/printing code needs to be written. +// +// The first half of this file defines the DNS message formats. +// The second half implements the conversion to and from wire format. +// A few of the structure elements have string tags to aid the +// generic pack/unpack routines. +// +// TODO(rsc): There are enough names defined in this file that they're all +// prefixed with dns. Perhaps put this in its own package later. + +package net + +import ( + "fmt" + "os" + "reflect" +) + +// Packet formats + +// Wire constants. +const ( + // valid dnsRR_Header.Rrtype and dnsQuestion.qtype + dnsTypeA = 1 + dnsTypeNS = 2 + dnsTypeMD = 3 + dnsTypeMF = 4 + dnsTypeCNAME = 5 + dnsTypeSOA = 6 + dnsTypeMB = 7 + dnsTypeMG = 8 + dnsTypeMR = 9 + dnsTypeNULL = 10 + dnsTypeWKS = 11 + dnsTypePTR = 12 + dnsTypeHINFO = 13 + dnsTypeMINFO = 14 + dnsTypeMX = 15 + dnsTypeTXT = 16 + dnsTypeAAAA = 28 + dnsTypeSRV = 33 + + // valid dnsQuestion.qtype only + dnsTypeAXFR = 252 + dnsTypeMAILB = 253 + dnsTypeMAILA = 254 + dnsTypeALL = 255 + + // valid dnsQuestion.qclass + dnsClassINET = 1 + dnsClassCSNET = 2 + dnsClassCHAOS = 3 + dnsClassHESIOD = 4 + dnsClassANY = 255 + + // dnsMsg.rcode + dnsRcodeSuccess = 0 + dnsRcodeFormatError = 1 + dnsRcodeServerFailure = 2 + dnsRcodeNameError = 3 + dnsRcodeNotImplemented = 4 + dnsRcodeRefused = 5 +) + +// The wire format for the DNS packet header. +type dnsHeader struct { + Id uint16 + Bits uint16 + Qdcount, Ancount, Nscount, Arcount uint16 +} + +const ( + // dnsHeader.Bits + _QR = 1 << 15 // query/response (response=1) + _AA = 1 << 10 // authoritative + _TC = 1 << 9 // truncated + _RD = 1 << 8 // recursion desired + _RA = 1 << 7 // recursion available +) + +// DNS queries. +type dnsQuestion struct { + Name string "domain-name" // "domain-name" specifies encoding; see packers below + Qtype uint16 + Qclass uint16 +} + +// DNS responses (resource records). +// There are many types of messages, +// but they all share the same header. +type dnsRR_Header struct { + Name string "domain-name" + Rrtype uint16 + Class uint16 + Ttl uint32 + Rdlength uint16 // length of data after header +} + +func (h *dnsRR_Header) Header() *dnsRR_Header { + return h +} + +type dnsRR interface { + Header() *dnsRR_Header +} + +// Specific DNS RR formats for each query type. + +type dnsRR_CNAME struct { + Hdr dnsRR_Header + Cname string "domain-name" +} + +func (rr *dnsRR_CNAME) Header() *dnsRR_Header { + return &rr.Hdr +} + +type dnsRR_HINFO struct { + Hdr dnsRR_Header + Cpu string + Os string +} + +func (rr *dnsRR_HINFO) Header() *dnsRR_Header { + return &rr.Hdr +} + +type dnsRR_MB struct { + Hdr dnsRR_Header + Mb string "domain-name" +} + +func (rr *dnsRR_MB) Header() *dnsRR_Header { + return &rr.Hdr +} + +type dnsRR_MG struct { + Hdr dnsRR_Header + Mg string "domain-name" +} + +func (rr *dnsRR_MG) Header() *dnsRR_Header { + return &rr.Hdr +} + +type dnsRR_MINFO struct { + Hdr dnsRR_Header + Rmail string "domain-name" + Email string "domain-name" +} + +func (rr *dnsRR_MINFO) Header() *dnsRR_Header { + return &rr.Hdr +} + +type dnsRR_MR struct { + Hdr dnsRR_Header + Mr string "domain-name" +} + +func (rr *dnsRR_MR) Header() *dnsRR_Header { + return &rr.Hdr +} + +type dnsRR_MX struct { + Hdr dnsRR_Header + Pref uint16 + Mx string "domain-name" +} + +func (rr *dnsRR_MX) Header() *dnsRR_Header { + return &rr.Hdr +} + +type dnsRR_NS struct { + Hdr dnsRR_Header + Ns string "domain-name" +} + +func (rr *dnsRR_NS) Header() *dnsRR_Header { + return &rr.Hdr +} + +type dnsRR_PTR struct { + Hdr dnsRR_Header + Ptr string "domain-name" +} + +func (rr *dnsRR_PTR) Header() *dnsRR_Header { + return &rr.Hdr +} + +type dnsRR_SOA struct { + Hdr dnsRR_Header + Ns string "domain-name" + Mbox string "domain-name" + Serial uint32 + Refresh uint32 + Retry uint32 + Expire uint32 + Minttl uint32 +} + +func (rr *dnsRR_SOA) Header() *dnsRR_Header { + return &rr.Hdr +} + +type dnsRR_TXT struct { + Hdr dnsRR_Header + Txt string // not domain name +} + +func (rr *dnsRR_TXT) Header() *dnsRR_Header { + return &rr.Hdr +} + +type dnsRR_SRV struct { + Hdr dnsRR_Header + Priority uint16 + Weight uint16 + Port uint16 + Target string "domain-name" +} + +func (rr *dnsRR_SRV) Header() *dnsRR_Header { + return &rr.Hdr +} + +type dnsRR_A struct { + Hdr dnsRR_Header + A uint32 "ipv4" +} + +func (rr *dnsRR_A) Header() *dnsRR_Header { + return &rr.Hdr +} + +type dnsRR_AAAA struct { + Hdr dnsRR_Header + AAAA [16]byte "ipv6" +} + +func (rr *dnsRR_AAAA) Header() *dnsRR_Header { + return &rr.Hdr +} + +// Packing and unpacking. +// +// All the packers and unpackers take a (msg []byte, off int) +// and return (off1 int, ok bool). If they return ok==false, they +// also return off1==len(msg), so that the next unpacker will +// also fail. This lets us avoid checks of ok until the end of a +// packing sequence. + +// Map of constructors for each RR wire type. +var rr_mk = map[int]func() dnsRR{ + dnsTypeCNAME: func() dnsRR { return new(dnsRR_CNAME) }, + dnsTypeHINFO: func() dnsRR { return new(dnsRR_HINFO) }, + dnsTypeMB: func() dnsRR { return new(dnsRR_MB) }, + dnsTypeMG: func() dnsRR { return new(dnsRR_MG) }, + dnsTypeMINFO: func() dnsRR { return new(dnsRR_MINFO) }, + dnsTypeMR: func() dnsRR { return new(dnsRR_MR) }, + dnsTypeMX: func() dnsRR { return new(dnsRR_MX) }, + dnsTypeNS: func() dnsRR { return new(dnsRR_NS) }, + dnsTypePTR: func() dnsRR { return new(dnsRR_PTR) }, + dnsTypeSOA: func() dnsRR { return new(dnsRR_SOA) }, + dnsTypeTXT: func() dnsRR { return new(dnsRR_TXT) }, + dnsTypeSRV: func() dnsRR { return new(dnsRR_SRV) }, + dnsTypeA: func() dnsRR { return new(dnsRR_A) }, + dnsTypeAAAA: func() dnsRR { return new(dnsRR_AAAA) }, +} + +// Pack a domain name s into msg[off:]. +// Domain names are a sequence of counted strings +// split at the dots. They end with a zero-length string. +func packDomainName(s string, msg []byte, off int) (off1 int, ok bool) { + // Add trailing dot to canonicalize name. + if n := len(s); n == 0 || s[n-1] != '.' { + s += "." + } + + // Each dot ends a segment of the name. + // We trade each dot byte for a length byte. + // There is also a trailing zero. + // Check that we have all the space we need. + tot := len(s) + 1 + if off+tot > len(msg) { + return len(msg), false + } + + // Emit sequence of counted strings, chopping at dots. + begin := 0 + for i := 0; i < len(s); i++ { + if s[i] == '.' { + if i-begin >= 1<<6 { // top two bits of length must be clear + return len(msg), false + } + msg[off] = byte(i - begin) + off++ + for j := begin; j < i; j++ { + msg[off] = s[j] + off++ + } + begin = i + 1 + } + } + msg[off] = 0 + off++ + return off, true +} + +// Unpack a domain name. +// In addition to the simple sequences of counted strings above, +// domain names are allowed to refer to strings elsewhere in the +// packet, to avoid repeating common suffixes when returning +// many entries in a single domain. The pointers are marked +// by a length byte with the top two bits set. Ignoring those +// two bits, that byte and the next give a 14 bit offset from msg[0] +// where we should pick up the trail. +// Note that if we jump elsewhere in the packet, +// we return off1 == the offset after the first pointer we found, +// which is where the next record will start. +// In theory, the pointers are only allowed to jump backward. +// We let them jump anywhere and stop jumping after a while. +func unpackDomainName(msg []byte, off int) (s string, off1 int, ok bool) { + s = "" + ptr := 0 // number of pointers followed +Loop: + for { + if off >= len(msg) { + return "", len(msg), false + } + c := int(msg[off]) + off++ + switch c & 0xC0 { + case 0x00: + if c == 0x00 { + // end of name + break Loop + } + // literal string + if off+c > len(msg) { + return "", len(msg), false + } + s += string(msg[off:off+c]) + "." + off += c + case 0xC0: + // pointer to somewhere else in msg. + // remember location after first ptr, + // since that's how many bytes we consumed. + // also, don't follow too many pointers -- + // maybe there's a loop. + if off >= len(msg) { + return "", len(msg), false + } + c1 := msg[off] + off++ + if ptr == 0 { + off1 = off + } + if ptr++; ptr > 10 { + return "", len(msg), false + } + off = (c^0xC0)<<8 | int(c1) + default: + // 0x80 and 0x40 are reserved + return "", len(msg), false + } + } + if ptr == 0 { + off1 = off + } + return s, off1, true +} + +// TODO(rsc): Move into generic library? +// Pack a reflect.StructValue into msg. Struct members can only be uint16, uint32, string, +// [n]byte, and other (often anonymous) structs. +func packStructValue(val *reflect.StructValue, msg []byte, off int) (off1 int, ok bool) { + for i := 0; i < val.NumField(); i++ { + f := val.Type().(*reflect.StructType).Field(i) + switch fv := val.Field(i).(type) { + default: + BadType: + fmt.Fprintf(os.Stderr, "net: dns: unknown packing type %v", f.Type) + return len(msg), false + case *reflect.StructValue: + off, ok = packStructValue(fv, msg, off) + case *reflect.UintValue: + i := fv.Get() + switch fv.Type().Kind() { + default: + goto BadType + case reflect.Uint16: + if off+2 > len(msg) { + return len(msg), false + } + msg[off] = byte(i >> 8) + msg[off+1] = byte(i) + off += 2 + case reflect.Uint32: + if off+4 > len(msg) { + return len(msg), false + } + msg[off] = byte(i >> 24) + msg[off+1] = byte(i >> 16) + msg[off+2] = byte(i >> 8) + msg[off+3] = byte(i) + off += 4 + } + case *reflect.ArrayValue: + if fv.Type().(*reflect.ArrayType).Elem().Kind() != reflect.Uint8 { + goto BadType + } + n := fv.Len() + if off+n > len(msg) { + return len(msg), false + } + reflect.Copy(reflect.NewValue(msg[off:off+n]).(*reflect.SliceValue), fv) + off += n + case *reflect.StringValue: + // There are multiple string encodings. + // The tag distinguishes ordinary strings from domain names. + s := fv.Get() + switch f.Tag { + default: + fmt.Fprintf(os.Stderr, "net: dns: unknown string tag %v", f.Tag) + return len(msg), false + case "domain-name": + off, ok = packDomainName(s, msg, off) + if !ok { + return len(msg), false + } + case "": + // Counted string: 1 byte length. + if len(s) > 255 || off+1+len(s) > len(msg) { + return len(msg), false + } + msg[off] = byte(len(s)) + off++ + off += copy(msg[off:], s) + } + } + } + return off, true +} + +func structValue(any interface{}) *reflect.StructValue { + return reflect.NewValue(any).(*reflect.PtrValue).Elem().(*reflect.StructValue) +} + +func packStruct(any interface{}, msg []byte, off int) (off1 int, ok bool) { + off, ok = packStructValue(structValue(any), msg, off) + return off, ok +} + +// TODO(rsc): Move into generic library? +// Unpack a reflect.StructValue from msg. +// Same restrictions as packStructValue. +func unpackStructValue(val *reflect.StructValue, msg []byte, off int) (off1 int, ok bool) { + for i := 0; i < val.NumField(); i++ { + f := val.Type().(*reflect.StructType).Field(i) + switch fv := val.Field(i).(type) { + default: + BadType: + fmt.Fprintf(os.Stderr, "net: dns: unknown packing type %v", f.Type) + return len(msg), false + case *reflect.StructValue: + off, ok = unpackStructValue(fv, msg, off) + case *reflect.UintValue: + switch fv.Type().Kind() { + default: + goto BadType + case reflect.Uint16: + if off+2 > len(msg) { + return len(msg), false + } + i := uint16(msg[off])<<8 | uint16(msg[off+1]) + fv.Set(uint64(i)) + off += 2 + case reflect.Uint32: + if off+4 > len(msg) { + return len(msg), false + } + i := uint32(msg[off])<<24 | uint32(msg[off+1])<<16 | uint32(msg[off+2])<<8 | uint32(msg[off+3]) + fv.Set(uint64(i)) + off += 4 + } + case *reflect.ArrayValue: + if fv.Type().(*reflect.ArrayType).Elem().Kind() != reflect.Uint8 { + goto BadType + } + n := fv.Len() + if off+n > len(msg) { + return len(msg), false + } + reflect.Copy(fv, reflect.NewValue(msg[off:off+n]).(*reflect.SliceValue)) + off += n + case *reflect.StringValue: + var s string + switch f.Tag { + default: + fmt.Fprintf(os.Stderr, "net: dns: unknown string tag %v", f.Tag) + return len(msg), false + case "domain-name": + s, off, ok = unpackDomainName(msg, off) + if !ok { + return len(msg), false + } + case "": + if off >= len(msg) || off+1+int(msg[off]) > len(msg) { + return len(msg), false + } + n := int(msg[off]) + off++ + b := make([]byte, n) + for i := 0; i < n; i++ { + b[i] = msg[off+i] + } + off += n + s = string(b) + } + fv.Set(s) + } + } + return off, true +} + +func unpackStruct(any interface{}, msg []byte, off int) (off1 int, ok bool) { + off, ok = unpackStructValue(structValue(any), msg, off) + return off, ok +} + +// Generic struct printer. +// Doesn't care about the string tag "domain-name", +// but does look for an "ipv4" tag on uint32 variables +// and the "ipv6" tag on array variables, +// printing them as IP addresses. +func printStructValue(val *reflect.StructValue) string { + s := "{" + for i := 0; i < val.NumField(); i++ { + if i > 0 { + s += ", " + } + f := val.Type().(*reflect.StructType).Field(i) + if !f.Anonymous { + s += f.Name + "=" + } + fval := val.Field(i) + if fv, ok := fval.(*reflect.StructValue); ok { + s += printStructValue(fv) + } else if fv, ok := fval.(*reflect.UintValue); ok && f.Tag == "ipv4" { + i := fv.Get() + s += IPv4(byte(i>>24), byte(i>>16), byte(i>>8), byte(i)).String() + } else if fv, ok := fval.(*reflect.ArrayValue); ok && f.Tag == "ipv6" { + i := fv.Interface().([]byte) + s += IP(i).String() + } else { + s += fmt.Sprint(fval.Interface()) + } + } + s += "}" + return s +} + +func printStruct(any interface{}) string { return printStructValue(structValue(any)) } + +// Resource record packer. +func packRR(rr dnsRR, msg []byte, off int) (off2 int, ok bool) { + var off1 int + // pack twice, once to find end of header + // and again to find end of packet. + // a bit inefficient but this doesn't need to be fast. + // off1 is end of header + // off2 is end of rr + off1, ok = packStruct(rr.Header(), msg, off) + off2, ok = packStruct(rr, msg, off) + if !ok { + return len(msg), false + } + // pack a third time; redo header with correct data length + rr.Header().Rdlength = uint16(off2 - off1) + packStruct(rr.Header(), msg, off) + return off2, true +} + +// Resource record unpacker. +func unpackRR(msg []byte, off int) (rr dnsRR, off1 int, ok bool) { + // unpack just the header, to find the rr type and length + var h dnsRR_Header + off0 := off + if off, ok = unpackStruct(&h, msg, off); !ok { + return nil, len(msg), false + } + end := off + int(h.Rdlength) + + // make an rr of that type and re-unpack. + // again inefficient but doesn't need to be fast. + mk, known := rr_mk[int(h.Rrtype)] + if !known { + return &h, end, true + } + rr = mk() + off, ok = unpackStruct(rr, msg, off0) + if off != end { + return &h, end, true + } + return rr, off, ok +} + +// Usable representation of a DNS packet. + +// A manually-unpacked version of (id, bits). +// This is in its own struct for easy printing. +type dnsMsgHdr struct { + id uint16 + response bool + opcode int + authoritative bool + truncated bool + recursion_desired bool + recursion_available bool + rcode int +} + +type dnsMsg struct { + dnsMsgHdr + question []dnsQuestion + answer []dnsRR + ns []dnsRR + extra []dnsRR +} + +func (dns *dnsMsg) Pack() (msg []byte, ok bool) { + var dh dnsHeader + + // Convert convenient dnsMsg into wire-like dnsHeader. + dh.Id = dns.id + dh.Bits = uint16(dns.opcode)<<11 | uint16(dns.rcode) + if dns.recursion_available { + dh.Bits |= _RA + } + if dns.recursion_desired { + dh.Bits |= _RD + } + if dns.truncated { + dh.Bits |= _TC + } + if dns.authoritative { + dh.Bits |= _AA + } + if dns.response { + dh.Bits |= _QR + } + + // Prepare variable sized arrays. + question := dns.question + answer := dns.answer + ns := dns.ns + extra := dns.extra + + dh.Qdcount = uint16(len(question)) + dh.Ancount = uint16(len(answer)) + dh.Nscount = uint16(len(ns)) + dh.Arcount = uint16(len(extra)) + + // Could work harder to calculate message size, + // but this is far more than we need and not + // big enough to hurt the allocator. + msg = make([]byte, 2000) + + // Pack it in: header and then the pieces. + off := 0 + off, ok = packStruct(&dh, msg, off) + for i := 0; i < len(question); i++ { + off, ok = packStruct(&question[i], msg, off) + } + for i := 0; i < len(answer); i++ { + off, ok = packRR(answer[i], msg, off) + } + for i := 0; i < len(ns); i++ { + off, ok = packRR(ns[i], msg, off) + } + for i := 0; i < len(extra); i++ { + off, ok = packRR(extra[i], msg, off) + } + if !ok { + return nil, false + } + return msg[0:off], true +} + +func (dns *dnsMsg) Unpack(msg []byte) bool { + // Header. + var dh dnsHeader + off := 0 + var ok bool + if off, ok = unpackStruct(&dh, msg, off); !ok { + return false + } + dns.id = dh.Id + dns.response = (dh.Bits & _QR) != 0 + dns.opcode = int(dh.Bits>>11) & 0xF + dns.authoritative = (dh.Bits & _AA) != 0 + dns.truncated = (dh.Bits & _TC) != 0 + dns.recursion_desired = (dh.Bits & _RD) != 0 + dns.recursion_available = (dh.Bits & _RA) != 0 + dns.rcode = int(dh.Bits & 0xF) + + // Arrays. + dns.question = make([]dnsQuestion, dh.Qdcount) + dns.answer = make([]dnsRR, dh.Ancount) + dns.ns = make([]dnsRR, dh.Nscount) + dns.extra = make([]dnsRR, dh.Arcount) + + for i := 0; i < len(dns.question); i++ { + off, ok = unpackStruct(&dns.question[i], msg, off) + } + for i := 0; i < len(dns.answer); i++ { + dns.answer[i], off, ok = unpackRR(msg, off) + } + for i := 0; i < len(dns.ns); i++ { + dns.ns[i], off, ok = unpackRR(msg, off) + } + for i := 0; i < len(dns.extra); i++ { + dns.extra[i], off, ok = unpackRR(msg, off) + } + if !ok { + return false + } + // if off != len(msg) { + // println("extra bytes in dns packet", off, "<", len(msg)); + // } + return true +} + +func (dns *dnsMsg) String() string { + s := "DNS: " + printStruct(&dns.dnsMsgHdr) + "\n" + if len(dns.question) > 0 { + s += "-- Questions\n" + for i := 0; i < len(dns.question); i++ { + s += printStruct(&dns.question[i]) + "\n" + } + } + if len(dns.answer) > 0 { + s += "-- Answers\n" + for i := 0; i < len(dns.answer); i++ { + s += printStruct(dns.answer[i]) + "\n" + } + } + if len(dns.ns) > 0 { + s += "-- Name servers\n" + for i := 0; i < len(dns.ns); i++ { + s += printStruct(dns.ns[i]) + "\n" + } + } + if len(dns.extra) > 0 { + s += "-- Extra\n" + for i := 0; i < len(dns.extra); i++ { + s += printStruct(dns.extra[i]) + "\n" + } + } + return s +} diff --git a/src/cmd/gofix/testdata/reflect.dnsmsg.go.out b/src/cmd/gofix/testdata/reflect.dnsmsg.go.out new file mode 100644 index 000000000..c777fe27c --- /dev/null +++ b/src/cmd/gofix/testdata/reflect.dnsmsg.go.out @@ -0,0 +1,777 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// DNS packet assembly. See RFC 1035. +// +// This is intended to support name resolution during net.Dial. +// It doesn't have to be blazing fast. +// +// Rather than write the usual handful of routines to pack and +// unpack every message that can appear on the wire, we use +// reflection to write a generic pack/unpack for structs and then +// use it. Thus, if in the future we need to define new message +// structs, no new pack/unpack/printing code needs to be written. +// +// The first half of this file defines the DNS message formats. +// The second half implements the conversion to and from wire format. +// A few of the structure elements have string tags to aid the +// generic pack/unpack routines. +// +// TODO(rsc): There are enough names defined in this file that they're all +// prefixed with dns. Perhaps put this in its own package later. + +package net + +import ( + "fmt" + "os" + "reflect" +) + +// Packet formats + +// Wire constants. +const ( + // valid dnsRR_Header.Rrtype and dnsQuestion.qtype + dnsTypeA = 1 + dnsTypeNS = 2 + dnsTypeMD = 3 + dnsTypeMF = 4 + dnsTypeCNAME = 5 + dnsTypeSOA = 6 + dnsTypeMB = 7 + dnsTypeMG = 8 + dnsTypeMR = 9 + dnsTypeNULL = 10 + dnsTypeWKS = 11 + dnsTypePTR = 12 + dnsTypeHINFO = 13 + dnsTypeMINFO = 14 + dnsTypeMX = 15 + dnsTypeTXT = 16 + dnsTypeAAAA = 28 + dnsTypeSRV = 33 + + // valid dnsQuestion.qtype only + dnsTypeAXFR = 252 + dnsTypeMAILB = 253 + dnsTypeMAILA = 254 + dnsTypeALL = 255 + + // valid dnsQuestion.qclass + dnsClassINET = 1 + dnsClassCSNET = 2 + dnsClassCHAOS = 3 + dnsClassHESIOD = 4 + dnsClassANY = 255 + + // dnsMsg.rcode + dnsRcodeSuccess = 0 + dnsRcodeFormatError = 1 + dnsRcodeServerFailure = 2 + dnsRcodeNameError = 3 + dnsRcodeNotImplemented = 4 + dnsRcodeRefused = 5 +) + +// The wire format for the DNS packet header. +type dnsHeader struct { + Id uint16 + Bits uint16 + Qdcount, Ancount, Nscount, Arcount uint16 +} + +const ( + // dnsHeader.Bits + _QR = 1 << 15 // query/response (response=1) + _AA = 1 << 10 // authoritative + _TC = 1 << 9 // truncated + _RD = 1 << 8 // recursion desired + _RA = 1 << 7 // recursion available +) + +// DNS queries. +type dnsQuestion struct { + Name string "domain-name" // "domain-name" specifies encoding; see packers below + Qtype uint16 + Qclass uint16 +} + +// DNS responses (resource records). +// There are many types of messages, +// but they all share the same header. +type dnsRR_Header struct { + Name string "domain-name" + Rrtype uint16 + Class uint16 + Ttl uint32 + Rdlength uint16 // length of data after header +} + +func (h *dnsRR_Header) Header() *dnsRR_Header { + return h +} + +type dnsRR interface { + Header() *dnsRR_Header +} + +// Specific DNS RR formats for each query type. + +type dnsRR_CNAME struct { + Hdr dnsRR_Header + Cname string "domain-name" +} + +func (rr *dnsRR_CNAME) Header() *dnsRR_Header { + return &rr.Hdr +} + +type dnsRR_HINFO struct { + Hdr dnsRR_Header + Cpu string + Os string +} + +func (rr *dnsRR_HINFO) Header() *dnsRR_Header { + return &rr.Hdr +} + +type dnsRR_MB struct { + Hdr dnsRR_Header + Mb string "domain-name" +} + +func (rr *dnsRR_MB) Header() *dnsRR_Header { + return &rr.Hdr +} + +type dnsRR_MG struct { + Hdr dnsRR_Header + Mg string "domain-name" +} + +func (rr *dnsRR_MG) Header() *dnsRR_Header { + return &rr.Hdr +} + +type dnsRR_MINFO struct { + Hdr dnsRR_Header + Rmail string "domain-name" + Email string "domain-name" +} + +func (rr *dnsRR_MINFO) Header() *dnsRR_Header { + return &rr.Hdr +} + +type dnsRR_MR struct { + Hdr dnsRR_Header + Mr string "domain-name" +} + +func (rr *dnsRR_MR) Header() *dnsRR_Header { + return &rr.Hdr +} + +type dnsRR_MX struct { + Hdr dnsRR_Header + Pref uint16 + Mx string "domain-name" +} + +func (rr *dnsRR_MX) Header() *dnsRR_Header { + return &rr.Hdr +} + +type dnsRR_NS struct { + Hdr dnsRR_Header + Ns string "domain-name" +} + +func (rr *dnsRR_NS) Header() *dnsRR_Header { + return &rr.Hdr +} + +type dnsRR_PTR struct { + Hdr dnsRR_Header + Ptr string "domain-name" +} + +func (rr *dnsRR_PTR) Header() *dnsRR_Header { + return &rr.Hdr +} + +type dnsRR_SOA struct { + Hdr dnsRR_Header + Ns string "domain-name" + Mbox string "domain-name" + Serial uint32 + Refresh uint32 + Retry uint32 + Expire uint32 + Minttl uint32 +} + +func (rr *dnsRR_SOA) Header() *dnsRR_Header { + return &rr.Hdr +} + +type dnsRR_TXT struct { + Hdr dnsRR_Header + Txt string // not domain name +} + +func (rr *dnsRR_TXT) Header() *dnsRR_Header { + return &rr.Hdr +} + +type dnsRR_SRV struct { + Hdr dnsRR_Header + Priority uint16 + Weight uint16 + Port uint16 + Target string "domain-name" +} + +func (rr *dnsRR_SRV) Header() *dnsRR_Header { + return &rr.Hdr +} + +type dnsRR_A struct { + Hdr dnsRR_Header + A uint32 "ipv4" +} + +func (rr *dnsRR_A) Header() *dnsRR_Header { + return &rr.Hdr +} + +type dnsRR_AAAA struct { + Hdr dnsRR_Header + AAAA [16]byte "ipv6" +} + +func (rr *dnsRR_AAAA) Header() *dnsRR_Header { + return &rr.Hdr +} + +// Packing and unpacking. +// +// All the packers and unpackers take a (msg []byte, off int) +// and return (off1 int, ok bool). If they return ok==false, they +// also return off1==len(msg), so that the next unpacker will +// also fail. This lets us avoid checks of ok until the end of a +// packing sequence. + +// Map of constructors for each RR wire type. +var rr_mk = map[int]func() dnsRR{ + dnsTypeCNAME: func() dnsRR { return new(dnsRR_CNAME) }, + dnsTypeHINFO: func() dnsRR { return new(dnsRR_HINFO) }, + dnsTypeMB: func() dnsRR { return new(dnsRR_MB) }, + dnsTypeMG: func() dnsRR { return new(dnsRR_MG) }, + dnsTypeMINFO: func() dnsRR { return new(dnsRR_MINFO) }, + dnsTypeMR: func() dnsRR { return new(dnsRR_MR) }, + dnsTypeMX: func() dnsRR { return new(dnsRR_MX) }, + dnsTypeNS: func() dnsRR { return new(dnsRR_NS) }, + dnsTypePTR: func() dnsRR { return new(dnsRR_PTR) }, + dnsTypeSOA: func() dnsRR { return new(dnsRR_SOA) }, + dnsTypeTXT: func() dnsRR { return new(dnsRR_TXT) }, + dnsTypeSRV: func() dnsRR { return new(dnsRR_SRV) }, + dnsTypeA: func() dnsRR { return new(dnsRR_A) }, + dnsTypeAAAA: func() dnsRR { return new(dnsRR_AAAA) }, +} + +// Pack a domain name s into msg[off:]. +// Domain names are a sequence of counted strings +// split at the dots. They end with a zero-length string. +func packDomainName(s string, msg []byte, off int) (off1 int, ok bool) { + // Add trailing dot to canonicalize name. + if n := len(s); n == 0 || s[n-1] != '.' { + s += "." + } + + // Each dot ends a segment of the name. + // We trade each dot byte for a length byte. + // There is also a trailing zero. + // Check that we have all the space we need. + tot := len(s) + 1 + if off+tot > len(msg) { + return len(msg), false + } + + // Emit sequence of counted strings, chopping at dots. + begin := 0 + for i := 0; i < len(s); i++ { + if s[i] == '.' { + if i-begin >= 1<<6 { // top two bits of length must be clear + return len(msg), false + } + msg[off] = byte(i - begin) + off++ + for j := begin; j < i; j++ { + msg[off] = s[j] + off++ + } + begin = i + 1 + } + } + msg[off] = 0 + off++ + return off, true +} + +// Unpack a domain name. +// In addition to the simple sequences of counted strings above, +// domain names are allowed to refer to strings elsewhere in the +// packet, to avoid repeating common suffixes when returning +// many entries in a single domain. The pointers are marked +// by a length byte with the top two bits set. Ignoring those +// two bits, that byte and the next give a 14 bit offset from msg[0] +// where we should pick up the trail. +// Note that if we jump elsewhere in the packet, +// we return off1 == the offset after the first pointer we found, +// which is where the next record will start. +// In theory, the pointers are only allowed to jump backward. +// We let them jump anywhere and stop jumping after a while. +func unpackDomainName(msg []byte, off int) (s string, off1 int, ok bool) { + s = "" + ptr := 0 // number of pointers followed +Loop: + for { + if off >= len(msg) { + return "", len(msg), false + } + c := int(msg[off]) + off++ + switch c & 0xC0 { + case 0x00: + if c == 0x00 { + // end of name + break Loop + } + // literal string + if off+c > len(msg) { + return "", len(msg), false + } + s += string(msg[off:off+c]) + "." + off += c + case 0xC0: + // pointer to somewhere else in msg. + // remember location after first ptr, + // since that's how many bytes we consumed. + // also, don't follow too many pointers -- + // maybe there's a loop. + if off >= len(msg) { + return "", len(msg), false + } + c1 := msg[off] + off++ + if ptr == 0 { + off1 = off + } + if ptr++; ptr > 10 { + return "", len(msg), false + } + off = (c^0xC0)<<8 | int(c1) + default: + // 0x80 and 0x40 are reserved + return "", len(msg), false + } + } + if ptr == 0 { + off1 = off + } + return s, off1, true +} + +// TODO(rsc): Move into generic library? +// Pack a reflect.StructValue into msg. Struct members can only be uint16, uint32, string, +// [n]byte, and other (often anonymous) structs. +func packStructValue(val reflect.Value, msg []byte, off int) (off1 int, ok bool) { + for i := 0; i < val.NumField(); i++ { + f := val.Type().Field(i) + switch fv := val.Field(i); fv.Kind() { + default: + BadType: + fmt.Fprintf(os.Stderr, "net: dns: unknown packing type %v", f.Type) + return len(msg), false + case reflect.Struct: + off, ok = packStructValue(fv, msg, off) + case reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64, reflect.Uintptr: + i := fv.Uint() + switch fv.Type().Kind() { + default: + goto BadType + case reflect.Uint16: + if off+2 > len(msg) { + return len(msg), false + } + msg[off] = byte(i >> 8) + msg[off+1] = byte(i) + off += 2 + case reflect.Uint32: + if off+4 > len(msg) { + return len(msg), false + } + msg[off] = byte(i >> 24) + msg[off+1] = byte(i >> 16) + msg[off+2] = byte(i >> 8) + msg[off+3] = byte(i) + off += 4 + } + case reflect.Array: + if fv.Type().Elem().Kind() != reflect.Uint8 { + goto BadType + } + n := fv.Len() + if off+n > len(msg) { + return len(msg), false + } + reflect.Copy(reflect.ValueOf(msg[off:off+n]), fv) + off += n + case reflect.String: + // There are multiple string encodings. + // The tag distinguishes ordinary strings from domain names. + s := fv.String() + switch f.Tag { + default: + fmt.Fprintf(os.Stderr, "net: dns: unknown string tag %v", f.Tag) + return len(msg), false + case "domain-name": + off, ok = packDomainName(s, msg, off) + if !ok { + return len(msg), false + } + case "": + // Counted string: 1 byte length. + if len(s) > 255 || off+1+len(s) > len(msg) { + return len(msg), false + } + msg[off] = byte(len(s)) + off++ + off += copy(msg[off:], s) + } + } + } + return off, true +} + +func structValue(any interface{}) reflect.Value { + return reflect.ValueOf(any).Elem() +} + +func packStruct(any interface{}, msg []byte, off int) (off1 int, ok bool) { + off, ok = packStructValue(structValue(any), msg, off) + return off, ok +} + +// TODO(rsc): Move into generic library? +// Unpack a reflect.StructValue from msg. +// Same restrictions as packStructValue. +func unpackStructValue(val reflect.Value, msg []byte, off int) (off1 int, ok bool) { + for i := 0; i < val.NumField(); i++ { + f := val.Type().Field(i) + switch fv := val.Field(i); fv.Kind() { + default: + BadType: + fmt.Fprintf(os.Stderr, "net: dns: unknown packing type %v", f.Type) + return len(msg), false + case reflect.Struct: + off, ok = unpackStructValue(fv, msg, off) + case reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64, reflect.Uintptr: + switch fv.Type().Kind() { + default: + goto BadType + case reflect.Uint16: + if off+2 > len(msg) { + return len(msg), false + } + i := uint16(msg[off])<<8 | uint16(msg[off+1]) + fv.SetUint(uint64(i)) + off += 2 + case reflect.Uint32: + if off+4 > len(msg) { + return len(msg), false + } + i := uint32(msg[off])<<24 | uint32(msg[off+1])<<16 | uint32(msg[off+2])<<8 | uint32(msg[off+3]) + fv.SetUint(uint64(i)) + off += 4 + } + case reflect.Array: + if fv.Type().Elem().Kind() != reflect.Uint8 { + goto BadType + } + n := fv.Len() + if off+n > len(msg) { + return len(msg), false + } + reflect.Copy(fv, reflect.ValueOf(msg[off:off+n])) + off += n + case reflect.String: + var s string + switch f.Tag { + default: + fmt.Fprintf(os.Stderr, "net: dns: unknown string tag %v", f.Tag) + return len(msg), false + case "domain-name": + s, off, ok = unpackDomainName(msg, off) + if !ok { + return len(msg), false + } + case "": + if off >= len(msg) || off+1+int(msg[off]) > len(msg) { + return len(msg), false + } + n := int(msg[off]) + off++ + b := make([]byte, n) + for i := 0; i < n; i++ { + b[i] = msg[off+i] + } + off += n + s = string(b) + } + fv.SetString(s) + } + } + return off, true +} + +func unpackStruct(any interface{}, msg []byte, off int) (off1 int, ok bool) { + off, ok = unpackStructValue(structValue(any), msg, off) + return off, ok +} + +// Generic struct printer. +// Doesn't care about the string tag "domain-name", +// but does look for an "ipv4" tag on uint32 variables +// and the "ipv6" tag on array variables, +// printing them as IP addresses. +func printStructValue(val reflect.Value) string { + s := "{" + for i := 0; i < val.NumField(); i++ { + if i > 0 { + s += ", " + } + f := val.Type().Field(i) + if !f.Anonymous { + s += f.Name + "=" + } + fval := val.Field(i) + if fv := fval; fv.Kind() == reflect.Struct { + s += printStructValue(fv) + } else if fv := fval; (fv.Kind() == reflect.Uint || fv.Kind() == reflect.Uint8 || fv.Kind() == reflect.Uint16 || fv.Kind() == reflect.Uint32 || fv.Kind() == reflect.Uint64 || fv.Kind() == reflect.Uintptr) && f.Tag == "ipv4" { + i := fv.Uint() + s += IPv4(byte(i>>24), byte(i>>16), byte(i>>8), byte(i)).String() + } else if fv := fval; fv.Kind() == reflect.Array && f.Tag == "ipv6" { + i := fv.Interface().([]byte) + s += IP(i).String() + } else { + s += fmt.Sprint(fval.Interface()) + } + } + s += "}" + return s +} + +func printStruct(any interface{}) string { return printStructValue(structValue(any)) } + +// Resource record packer. +func packRR(rr dnsRR, msg []byte, off int) (off2 int, ok bool) { + var off1 int + // pack twice, once to find end of header + // and again to find end of packet. + // a bit inefficient but this doesn't need to be fast. + // off1 is end of header + // off2 is end of rr + off1, ok = packStruct(rr.Header(), msg, off) + off2, ok = packStruct(rr, msg, off) + if !ok { + return len(msg), false + } + // pack a third time; redo header with correct data length + rr.Header().Rdlength = uint16(off2 - off1) + packStruct(rr.Header(), msg, off) + return off2, true +} + +// Resource record unpacker. +func unpackRR(msg []byte, off int) (rr dnsRR, off1 int, ok bool) { + // unpack just the header, to find the rr type and length + var h dnsRR_Header + off0 := off + if off, ok = unpackStruct(&h, msg, off); !ok { + return nil, len(msg), false + } + end := off + int(h.Rdlength) + + // make an rr of that type and re-unpack. + // again inefficient but doesn't need to be fast. + mk, known := rr_mk[int(h.Rrtype)] + if !known { + return &h, end, true + } + rr = mk() + off, ok = unpackStruct(rr, msg, off0) + if off != end { + return &h, end, true + } + return rr, off, ok +} + +// Usable representation of a DNS packet. + +// A manually-unpacked version of (id, bits). +// This is in its own struct for easy printing. +type dnsMsgHdr struct { + id uint16 + response bool + opcode int + authoritative bool + truncated bool + recursion_desired bool + recursion_available bool + rcode int +} + +type dnsMsg struct { + dnsMsgHdr + question []dnsQuestion + answer []dnsRR + ns []dnsRR + extra []dnsRR +} + +func (dns *dnsMsg) Pack() (msg []byte, ok bool) { + var dh dnsHeader + + // Convert convenient dnsMsg into wire-like dnsHeader. + dh.Id = dns.id + dh.Bits = uint16(dns.opcode)<<11 | uint16(dns.rcode) + if dns.recursion_available { + dh.Bits |= _RA + } + if dns.recursion_desired { + dh.Bits |= _RD + } + if dns.truncated { + dh.Bits |= _TC + } + if dns.authoritative { + dh.Bits |= _AA + } + if dns.response { + dh.Bits |= _QR + } + + // Prepare variable sized arrays. + question := dns.question + answer := dns.answer + ns := dns.ns + extra := dns.extra + + dh.Qdcount = uint16(len(question)) + dh.Ancount = uint16(len(answer)) + dh.Nscount = uint16(len(ns)) + dh.Arcount = uint16(len(extra)) + + // Could work harder to calculate message size, + // but this is far more than we need and not + // big enough to hurt the allocator. + msg = make([]byte, 2000) + + // Pack it in: header and then the pieces. + off := 0 + off, ok = packStruct(&dh, msg, off) + for i := 0; i < len(question); i++ { + off, ok = packStruct(&question[i], msg, off) + } + for i := 0; i < len(answer); i++ { + off, ok = packRR(answer[i], msg, off) + } + for i := 0; i < len(ns); i++ { + off, ok = packRR(ns[i], msg, off) + } + for i := 0; i < len(extra); i++ { + off, ok = packRR(extra[i], msg, off) + } + if !ok { + return nil, false + } + return msg[0:off], true +} + +func (dns *dnsMsg) Unpack(msg []byte) bool { + // Header. + var dh dnsHeader + off := 0 + var ok bool + if off, ok = unpackStruct(&dh, msg, off); !ok { + return false + } + dns.id = dh.Id + dns.response = (dh.Bits & _QR) != 0 + dns.opcode = int(dh.Bits>>11) & 0xF + dns.authoritative = (dh.Bits & _AA) != 0 + dns.truncated = (dh.Bits & _TC) != 0 + dns.recursion_desired = (dh.Bits & _RD) != 0 + dns.recursion_available = (dh.Bits & _RA) != 0 + dns.rcode = int(dh.Bits & 0xF) + + // Arrays. + dns.question = make([]dnsQuestion, dh.Qdcount) + dns.answer = make([]dnsRR, dh.Ancount) + dns.ns = make([]dnsRR, dh.Nscount) + dns.extra = make([]dnsRR, dh.Arcount) + + for i := 0; i < len(dns.question); i++ { + off, ok = unpackStruct(&dns.question[i], msg, off) + } + for i := 0; i < len(dns.answer); i++ { + dns.answer[i], off, ok = unpackRR(msg, off) + } + for i := 0; i < len(dns.ns); i++ { + dns.ns[i], off, ok = unpackRR(msg, off) + } + for i := 0; i < len(dns.extra); i++ { + dns.extra[i], off, ok = unpackRR(msg, off) + } + if !ok { + return false + } + // if off != len(msg) { + // println("extra bytes in dns packet", off, "<", len(msg)); + // } + return true +} + +func (dns *dnsMsg) String() string { + s := "DNS: " + printStruct(&dns.dnsMsgHdr) + "\n" + if len(dns.question) > 0 { + s += "-- Questions\n" + for i := 0; i < len(dns.question); i++ { + s += printStruct(&dns.question[i]) + "\n" + } + } + if len(dns.answer) > 0 { + s += "-- Answers\n" + for i := 0; i < len(dns.answer); i++ { + s += printStruct(dns.answer[i]) + "\n" + } + } + if len(dns.ns) > 0 { + s += "-- Name servers\n" + for i := 0; i < len(dns.ns); i++ { + s += printStruct(dns.ns[i]) + "\n" + } + } + if len(dns.extra) > 0 { + s += "-- Extra\n" + for i := 0; i < len(dns.extra); i++ { + s += printStruct(dns.extra[i]) + "\n" + } + } + return s +} diff --git a/src/cmd/gofix/testdata/reflect.encode.go.in b/src/cmd/gofix/testdata/reflect.encode.go.in new file mode 100644 index 000000000..26ce47039 --- /dev/null +++ b/src/cmd/gofix/testdata/reflect.encode.go.in @@ -0,0 +1,367 @@ +// Copyright 2010 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// The json package implements encoding and decoding of JSON objects as +// defined in RFC 4627. +package json + +import ( + "bytes" + "encoding/base64" + "os" + "reflect" + "runtime" + "sort" + "strconv" + "unicode" + "utf8" +) + +// Marshal returns the JSON encoding of v. +// +// Marshal traverses the value v recursively. +// If an encountered value implements the Marshaler interface, +// Marshal calls its MarshalJSON method to produce JSON. +// +// Otherwise, Marshal uses the following type-dependent default encodings: +// +// Boolean values encode as JSON booleans. +// +// Floating point and integer values encode as JSON numbers. +// +// String values encode as JSON strings, with each invalid UTF-8 sequence +// replaced by the encoding of the Unicode replacement character U+FFFD. +// +// Array and slice values encode as JSON arrays, except that +// []byte encodes as a base64-encoded string. +// +// Struct values encode as JSON objects. Each struct field becomes +// a member of the object. By default the object's key name is the +// struct field name. If the struct field has a non-empty tag consisting +// of only Unicode letters, digits, and underscores, that tag will be used +// as the name instead. Only exported fields will be encoded. +// +// Map values encode as JSON objects. +// The map's key type must be string; the object keys are used directly +// as map keys. +// +// Pointer values encode as the value pointed to. +// A nil pointer encodes as the null JSON object. +// +// Interface values encode as the value contained in the interface. +// A nil interface value encodes as the null JSON object. +// +// Channel, complex, and function values cannot be encoded in JSON. +// Attempting to encode such a value causes Marshal to return +// an InvalidTypeError. +// +// JSON cannot represent cyclic data structures and Marshal does not +// handle them. Passing cyclic structures to Marshal will result in +// an infinite recursion. +// +func Marshal(v interface{}) ([]byte, os.Error) { + e := &encodeState{} + err := e.marshal(v) + if err != nil { + return nil, err + } + return e.Bytes(), nil +} + +// MarshalIndent is like Marshal but applies Indent to format the output. +func MarshalIndent(v interface{}, prefix, indent string) ([]byte, os.Error) { + b, err := Marshal(v) + if err != nil { + return nil, err + } + var buf bytes.Buffer + err = Indent(&buf, b, prefix, indent) + if err != nil { + return nil, err + } + return buf.Bytes(), nil +} + +// MarshalForHTML is like Marshal but applies HTMLEscape to the output. +func MarshalForHTML(v interface{}) ([]byte, os.Error) { + b, err := Marshal(v) + if err != nil { + return nil, err + } + var buf bytes.Buffer + HTMLEscape(&buf, b) + return buf.Bytes(), nil +} + +// HTMLEscape appends to dst the JSON-encoded src with <, >, and & +// characters inside string literals changed to \u003c, \u003e, \u0026 +// so that the JSON will be safe to embed inside HTML + + +EOF +} + +sub HtmlListingFooter { + return <<'EOF'; + + +EOF +} + +sub HtmlEscape { + my $text = shift; + $text =~ s/&/&/g; + $text =~ s//>/g; + return $text; +} + +# Returns the indentation of the line, if it has any non-whitespace +# characters. Otherwise, returns -1. +sub Indentation { + my $line = shift; + if (m/^(\s*)\S/) { + return length($1); + } else { + return -1; + } +} + +# Print source-listing for one routine +sub PrintSource { + my $prog = shift; + my $offset = shift; + my $routine = shift; + my $flat = shift; + my $cumulative = shift; + my $start_addr = shift; + my $end_addr = shift; + my $html = shift; + my $output = shift; + + # Disassemble all instructions (just to get line numbers) + my @instructions = Disassemble($prog, $offset, $start_addr, $end_addr); + + # Hack 1: assume that the first source file encountered in the + # disassembly contains the routine + my $filename = undef; + for (my $i = 0; $i <= $#instructions; $i++) { + if ($instructions[$i]->[2] >= 0) { + $filename = $instructions[$i]->[1]; + last; + } + } + if (!defined($filename)) { + print STDERR "no filename found in $routine\n"; + return 0; + } + + # Hack 2: assume that the largest line number from $filename is the + # end of the procedure. This is typically safe since if P1 contains + # an inlined call to P2, then P2 usually occurs earlier in the + # source file. If this does not work, we might have to compute a + # density profile or just print all regions we find. + my $lastline = 0; + for (my $i = 0; $i <= $#instructions; $i++) { + my $f = $instructions[$i]->[1]; + my $l = $instructions[$i]->[2]; + if (($f eq $filename) && ($l > $lastline)) { + $lastline = $l; + } + } + + # Hack 3: assume the first source location from "filename" is the start of + # the source code. + my $firstline = 1; + for (my $i = 0; $i <= $#instructions; $i++) { + if ($instructions[$i]->[1] eq $filename) { + $firstline = $instructions[$i]->[2]; + last; + } + } + + # Hack 4: Extend last line forward until its indentation is less than + # the indentation we saw on $firstline + my $oldlastline = $lastline; + { + if (!open(FILE, "<$filename")) { + print STDERR "$filename: $!\n"; + return 0; + } + my $l = 0; + my $first_indentation = -1; + while () { + s/\r//g; # turn windows-looking lines into unix-looking lines + $l++; + my $indent = Indentation($_); + if ($l >= $firstline) { + if ($first_indentation < 0 && $indent >= 0) { + $first_indentation = $indent; + last if ($first_indentation == 0); + } + } + if ($l >= $lastline && $indent >= 0) { + if ($indent >= $first_indentation) { + $lastline = $l+1; + } else { + last; + } + } + } + close(FILE); + } + + # Assign all samples to the range $firstline,$lastline, + # Hack 4: If an instruction does not occur in the range, its samples + # are moved to the next instruction that occurs in the range. + my $samples1 = {}; # Map from line number to flat count + my $samples2 = {}; # Map from line number to cumulative count + my $running1 = 0; # Unassigned flat counts + my $running2 = 0; # Unassigned cumulative counts + my $total1 = 0; # Total flat counts + my $total2 = 0; # Total cumulative counts + my %disasm = (); # Map from line number to disassembly + my $running_disasm = ""; # Unassigned disassembly + my $skip_marker = "---\n"; + if ($html) { + $skip_marker = ""; + for (my $l = $firstline; $l <= $lastline; $l++) { + $disasm{$l} = ""; + } + } + foreach my $e (@instructions) { + # Add up counts for all address that fall inside this instruction + my $c1 = 0; + my $c2 = 0; + for (my $a = $e->[0]; $a lt $e->[4]; $a = AddressInc($a)) { + $c1 += GetEntry($flat, $a); + $c2 += GetEntry($cumulative, $a); + } + + if ($html) { + $running_disasm .= sprintf(" %6s %6s \t\t%8s: %s\n", + HtmlPrintNumber($c1), + HtmlPrintNumber($c2), + $e->[0], + CleanDisassembly($e->[3])); + } + + $running1 += $c1; + $running2 += $c2; + $total1 += $c1; + $total2 += $c2; + my $file = $e->[1]; + my $line = $e->[2]; + if (($file eq $filename) && + ($line >= $firstline) && + ($line <= $lastline)) { + # Assign all accumulated samples to this line + AddEntry($samples1, $line, $running1); + AddEntry($samples2, $line, $running2); + $running1 = 0; + $running2 = 0; + if ($html) { + $disasm{$line} .= $running_disasm; + $running_disasm = ''; + } + } + } + + # Assign any leftover samples to $lastline + AddEntry($samples1, $lastline, $running1); + AddEntry($samples2, $lastline, $running2); + + if ($html) { + printf $output ( + "

%s

%s\n
\n" .
+      "Total:%6s %6s (flat / cumulative %s)\n",
+      HtmlEscape(ShortFunctionName($routine)),
+      HtmlEscape($filename),
+      Unparse($total1),
+      Unparse($total2),
+      Units());
+  } else {
+    printf $output (
+      "ROUTINE ====================== %s in %s\n" .
+      "%6s %6s Total %s (flat / cumulative)\n",
+      ShortFunctionName($routine),
+      $filename,
+      Unparse($total1),
+      Unparse($total2),
+      Units());
+  }
+  if (!open(FILE, "<$filename")) {
+    print STDERR "$filename: $!\n";
+    return 0;
+  }
+  my $l = 0;
+  while () {
+    s/\r//g;         # turn windows-looking lines into unix-looking lines
+    $l++;
+    if ($l >= $firstline - 5 &&
+        (($l <= $oldlastline + 5) || ($l <= $lastline))) {
+      chop;
+      my $text = $_;
+      if ($l == $firstline) { print $output $skip_marker; }
+      my $n1 = GetEntry($samples1, $l);
+      my $n2 = GetEntry($samples2, $l);
+      if ($html) {
+        my $dis = $disasm{$l};
+        if (!defined($dis) || $n1 + $n2 == 0) {
+          # No samples/disassembly for this source line
+          printf $output (
+            "%5d " .
+            "%6s %6s %s\n",
+            $l,
+            HtmlPrintNumber($n1),
+            HtmlPrintNumber($n2),
+            HtmlEscape($text));
+        } else {
+          printf $output (
+            "%5d " .
+            "%6s %6s %s" .
+            "%s\n",
+            $l,
+            HtmlPrintNumber($n1),
+            HtmlPrintNumber($n2),
+            HtmlEscape($text),
+            HtmlEscape($dis));
+        }
+      } else {
+        printf $output(
+          "%6s %6s %4d: %s\n",
+          UnparseAlt($n1),
+          UnparseAlt($n2),
+          $l,
+          $text);
+      }
+      if ($l == $lastline)  { print $output $skip_marker; }
+    };
+  }
+  close(FILE);
+  if ($html) {
+    print $output "
\n"; + } + return 1; +} + +# Return the source line for the specified file/linenumber. +# Returns undef if not found. +sub SourceLine { + my $file = shift; + my $line = shift; + + # Look in cache + if (!defined($main::source_cache{$file})) { + if (100 < scalar keys(%main::source_cache)) { + # Clear the cache when it gets too big + $main::source_cache = (); + } + + # Read all lines from the file + if (!open(FILE, "<$file")) { + print STDERR "$file: $!\n"; + $main::source_cache{$file} = []; # Cache the negative result + return undef; + } + my $lines = []; + push(@{$lines}, ""); # So we can use 1-based line numbers as indices + while () { + push(@{$lines}, $_); + } + close(FILE); + + # Save the lines in the cache + $main::source_cache{$file} = $lines; + } + + my $lines = $main::source_cache{$file}; + if (($line < 0) || ($line > $#{$lines})) { + return undef; + } else { + return $lines->[$line]; + } +} + +# Print disassembly for one routine with interspersed source if available +sub PrintDisassembledFunction { + my $prog = shift; + my $offset = shift; + my $routine = shift; + my $flat = shift; + my $cumulative = shift; + my $start_addr = shift; + my $end_addr = shift; + my $total = shift; + + # Disassemble all instructions + my @instructions = Disassemble($prog, $offset, $start_addr, $end_addr); + + # Make array of counts per instruction + my @flat_count = (); + my @cum_count = (); + my $flat_total = 0; + my $cum_total = 0; + foreach my $e (@instructions) { + # Add up counts for all address that fall inside this instruction + my $c1 = 0; + my $c2 = 0; + for (my $a = $e->[0]; $a lt $e->[4]; $a = AddressInc($a)) { + $c1 += GetEntry($flat, $a); + $c2 += GetEntry($cumulative, $a); + } + push(@flat_count, $c1); + push(@cum_count, $c2); + $flat_total += $c1; + $cum_total += $c2; + } + + # Print header with total counts + printf("ROUTINE ====================== %s\n" . + "%6s %6s %s (flat, cumulative) %.1f%% of total\n", + ShortFunctionName($routine), + Unparse($flat_total), + Unparse($cum_total), + Units(), + ($cum_total * 100.0) / $total); + + # Process instructions in order + my $current_file = ""; + for (my $i = 0; $i <= $#instructions; ) { + my $e = $instructions[$i]; + + # Print the new file name whenever we switch files + if ($e->[1] ne $current_file) { + $current_file = $e->[1]; + my $fname = $current_file; + $fname =~ s|^\./||; # Trim leading "./" + + # Shorten long file names + if (length($fname) >= 58) { + $fname = "..." . substr($fname, -55); + } + printf("-------------------- %s\n", $fname); + } + + # TODO: Compute range of lines to print together to deal with + # small reorderings. + my $first_line = $e->[2]; + my $last_line = $first_line; + my %flat_sum = (); + my %cum_sum = (); + for (my $l = $first_line; $l <= $last_line; $l++) { + $flat_sum{$l} = 0; + $cum_sum{$l} = 0; + } + + # Find run of instructions for this range of source lines + my $first_inst = $i; + while (($i <= $#instructions) && + ($instructions[$i]->[2] >= $first_line) && + ($instructions[$i]->[2] <= $last_line)) { + $e = $instructions[$i]; + $flat_sum{$e->[2]} += $flat_count[$i]; + $cum_sum{$e->[2]} += $cum_count[$i]; + $i++; + } + my $last_inst = $i - 1; + + # Print source lines + for (my $l = $first_line; $l <= $last_line; $l++) { + my $line = SourceLine($current_file, $l); + if (!defined($line)) { + $line = "?\n"; + next; + } else { + $line =~ s/^\s+//; + } + printf("%6s %6s %5d: %s", + UnparseAlt($flat_sum{$l}), + UnparseAlt($cum_sum{$l}), + $l, + $line); + } + + # Print disassembly + for (my $x = $first_inst; $x <= $last_inst; $x++) { + my $e = $instructions[$x]; + my $address = $e->[0]; + $address = AddressSub($address, $offset); # Make relative to section + $address =~ s/^0x//; + $address =~ s/^0*//; + + printf("%6s %6s %8s: %6s\n", + UnparseAlt($flat_count[$x]), + UnparseAlt($cum_count[$x]), + $address, + CleanDisassembly($e->[3])); + } + } +} + +# Print DOT graph +sub PrintDot { + my $prog = shift; + my $symbols = shift; + my $raw = shift; + my $flat = shift; + my $cumulative = shift; + my $overall_total = shift; + + # Get total + my $local_total = TotalProfile($flat); + my $nodelimit = int($main::opt_nodefraction * $local_total); + my $edgelimit = int($main::opt_edgefraction * $local_total); + my $nodecount = $main::opt_nodecount; + + # Find nodes to include + my @list = (sort { abs(GetEntry($cumulative, $b)) <=> + abs(GetEntry($cumulative, $a)) + || $a cmp $b } + keys(%{$cumulative})); + my $last = $nodecount - 1; + if ($last > $#list) { + $last = $#list; + } + while (($last >= 0) && + (abs(GetEntry($cumulative, $list[$last])) <= $nodelimit)) { + $last--; + } + if ($last < 0) { + print STDERR "No nodes to print\n"; + cleanup(); + return 0; + } + + if ($nodelimit > 0 || $edgelimit > 0) { + printf STDERR ("Dropping nodes with <= %s %s; edges with <= %s abs(%s)\n", + Unparse($nodelimit), Units(), + Unparse($edgelimit), Units()); + } + + # Open DOT output file + my $output; + if ($main::opt_gv) { + $output = "| $DOT -Tps2 >" . TempName($main::next_tmpfile, "ps"); + } elsif ($main::opt_ps) { + $output = "| $DOT -Tps2"; + } elsif ($main::opt_pdf) { + $output = "| $DOT -Tps2 | $PS2PDF - -"; + } elsif ($main::opt_web || $main::opt_svg) { + # We need to post-process the SVG, so write to a temporary file always. + $output = "| $DOT -Tsvg >" . TempName($main::next_tmpfile, "svg"); + } elsif ($main::opt_gif) { + $output = "| $DOT -Tgif"; + } else { + $output = ">&STDOUT"; + } + open(DOT, $output) || error("$output: $!\n"); + + # Title + printf DOT ("digraph \"%s; %s %s\" {\n", + $prog, + Unparse($overall_total), + Units()); + if ($main::opt_pdf) { + # The output is more printable if we set the page size for dot. + printf DOT ("size=\"8,11\"\n"); + } + printf DOT ("node [width=0.375,height=0.25];\n"); + + # Print legend + printf DOT ("Legend [shape=box,fontsize=24,shape=plaintext," . + "label=\"%s\\l%s\\l%s\\l%s\\l%s\\l\"];\n", + $prog, + sprintf("Total %s: %s", Units(), Unparse($overall_total)), + sprintf("Focusing on: %s", Unparse($local_total)), + sprintf("Dropped nodes with <= %s abs(%s)", + Unparse($nodelimit), Units()), + sprintf("Dropped edges with <= %s %s", + Unparse($edgelimit), Units()) + ); + + # Print nodes + my %node = (); + my $nextnode = 1; + foreach my $a (@list[0..$last]) { + # Pick font size + my $f = GetEntry($flat, $a); + my $c = GetEntry($cumulative, $a); + + my $fs = 8; + if ($local_total > 0) { + $fs = 8 + (50.0 * sqrt(abs($f * 1.0 / $local_total))); + } + + $node{$a} = $nextnode++; + my $sym = $a; + $sym =~ s/\s+/\\n/g; + $sym =~ s/::/\\n/g; + + # Extra cumulative info to print for non-leaves + my $extra = ""; + if ($f != $c) { + $extra = sprintf("\\rof %s (%s)", + Unparse($c), + Percent($c, $overall_total)); + } + my $style = ""; + if ($main::opt_heapcheck) { + if ($f > 0) { + # make leak-causing nodes more visible (add a background) + $style = ",style=filled,fillcolor=gray" + } elsif ($f < 0) { + # make anti-leak-causing nodes (which almost never occur) + # stand out as well (triple border) + $style = ",peripheries=3" + } + } + + printf DOT ("N%d [label=\"%s\\n%s (%s)%s\\r" . + "\",shape=box,fontsize=%.1f%s];\n", + $node{$a}, + $sym, + Unparse($f), + Percent($f, $overall_total), + $extra, + $fs, + $style, + ); + } + + # Get edges and counts per edge + my %edge = (); + my $n; + foreach my $k (keys(%{$raw})) { + # TODO: omit low %age edges + $n = $raw->{$k}; + my @translated = TranslateStack($symbols, $k); + for (my $i = 1; $i <= $#translated; $i++) { + my $src = $translated[$i]; + my $dst = $translated[$i-1]; + #next if ($src eq $dst); # Avoid self-edges? + if (exists($node{$src}) && exists($node{$dst})) { + my $edge_label = "$src\001$dst"; + if (!exists($edge{$edge_label})) { + $edge{$edge_label} = 0; + } + $edge{$edge_label} += $n; + } + } + } + + # Print edges + foreach my $e (keys(%edge)) { + my @x = split(/\001/, $e); + $n = $edge{$e}; + + if (abs($n) > $edgelimit) { + # Compute line width based on edge count + my $fraction = abs($local_total ? (3 * ($n / $local_total)) : 0); + if ($fraction > 1) { $fraction = 1; } + my $w = $fraction * 2; + if ($w < 1 && ($main::opt_web || $main::opt_svg)) { + # SVG output treats line widths < 1 poorly. + $w = 1; + } + + # Dot sometimes segfaults if given edge weights that are too large, so + # we cap the weights at a large value + my $edgeweight = abs($n) ** 0.7; + if ($edgeweight > 100000) { $edgeweight = 100000; } + $edgeweight = int($edgeweight); + + my $style = sprintf("setlinewidth(%f)", $w); + if ($x[1] =~ m/\(inline\)/) { + $style .= ",dashed"; + } + + # Use a slightly squashed function of the edge count as the weight + printf DOT ("N%s -> N%s [label=%s, weight=%d, style=\"%s\"];\n", + $node{$x[0]}, + $node{$x[1]}, + Unparse($n), + $edgeweight, + $style); + } + } + + print DOT ("}\n"); + close(DOT); + + if ($main::opt_web || $main::opt_svg) { + # Rewrite SVG to be more usable inside web browser. + RewriteSvg(TempName($main::next_tmpfile, "svg")); + } + + return 1; +} + +sub RewriteSvg { + my $svgfile = shift; + + open(SVG, $svgfile) || die "open temp svg: $!"; + my @svg = ; + close(SVG); + unlink $svgfile; + my $svg = join('', @svg); + + # Dot's SVG output is + # + # + # + # ... + # + # + # + # Change it to + # + # + # $svg_javascript + # + # + # ... + # + # + # + + # Fix width, height; drop viewBox. + $svg =~ s/(?s) above first + my $svg_javascript = SvgJavascript(); + my $viewport = "\n"; + $svg =~ s/ above . + $svg =~ s/(.*)(<\/svg>)/$1<\/g>$2/; + $svg =~ s/$svgfile") || die "open $svgfile: $!"; + print SVG $svg; + close(SVG); + } +} + +sub SvgJavascript { + return <<'EOF'; + +EOF +} + +# Translate a stack of addresses into a stack of symbols +sub TranslateStack { + my $symbols = shift; + my $k = shift; + + my @addrs = split(/\n/, $k); + my @result = (); + for (my $i = 0; $i <= $#addrs; $i++) { + my $a = $addrs[$i]; + + # Skip large addresses since they sometimes show up as fake entries on RH9 + if (length($a) > 8 && $a gt "7fffffffffffffff") { + next; + } + + if ($main::opt_disasm || $main::opt_list) { + # We want just the address for the key + push(@result, $a); + next; + } + + my $symlist = $symbols->{$a}; + if (!defined($symlist)) { + $symlist = [$a, "", $a]; + } + + # We can have a sequence of symbols for a particular entry + # (more than one symbol in the case of inlining). Callers + # come before callees in symlist, so walk backwards since + # the translated stack should contain callees before callers. + for (my $j = $#{$symlist}; $j >= 2; $j -= 3) { + my $func = $symlist->[$j-2]; + my $fileline = $symlist->[$j-1]; + my $fullfunc = $symlist->[$j]; + if ($j > 2) { + $func = "$func (inline)"; + } + if ($main::opt_addresses) { + push(@result, "$a $func $fileline"); + } elsif ($main::opt_lines) { + if ($func eq '??' && $fileline eq '??:0') { + push(@result, "$a"); + } else { + push(@result, "$func $fileline"); + } + } elsif ($main::opt_functions) { + if ($func eq '??') { + push(@result, "$a"); + } else { + push(@result, $func); + } + } elsif ($main::opt_files) { + if ($fileline eq '??:0' || $fileline eq '') { + push(@result, "$a"); + } else { + my $f = $fileline; + $f =~ s/:\d+$//; + push(@result, $f); + } + } else { + push(@result, $a); + last; # Do not print inlined info + } + } + } + + # print join(",", @addrs), " => ", join(",", @result), "\n"; + return @result; +} + +# Generate percent string for a number and a total +sub Percent { + my $num = shift; + my $tot = shift; + if ($tot != 0) { + return sprintf("%.1f%%", $num * 100.0 / $tot); + } else { + return ($num == 0) ? "nan" : (($num > 0) ? "+inf" : "-inf"); + } +} + +# Generate pretty-printed form of number +sub Unparse { + my $num = shift; + if ($main::profile_type eq 'heap' || $main::profile_type eq 'growth') { + if ($main::opt_inuse_objects || $main::opt_alloc_objects) { + return sprintf("%d", $num); + } else { + if ($main::opt_show_bytes) { + return sprintf("%d", $num); + } else { + return sprintf("%.1f", $num / 1048576.0); + } + } + } elsif ($main::profile_type eq 'contention' && !$main::opt_contentions) { + return sprintf("%.3f", $num / 1e9); # Convert nanoseconds to seconds + } else { + return sprintf("%d", $num); + } +} + +# Alternate pretty-printed form: 0 maps to "." +sub UnparseAlt { + my $num = shift; + if ($num == 0) { + return "."; + } else { + return Unparse($num); + } +} + +# Alternate pretty-printed form: 0 maps to "" +sub HtmlPrintNumber { + my $num = shift; + if ($num == 0) { + return ""; + } else { + return Unparse($num); + } +} + +# Return output units +sub Units { + if ($main::profile_type eq 'heap' || $main::profile_type eq 'growth') { + if ($main::opt_inuse_objects || $main::opt_alloc_objects) { + return "objects"; + } else { + if ($main::opt_show_bytes) { + return "B"; + } else { + return "MB"; + } + } + } elsif ($main::profile_type eq 'contention' && !$main::opt_contentions) { + return "seconds"; + } else { + return "samples"; + } +} + +##### Profile manipulation code ##### + +# Generate flattened profile: +# If count is charged to stack [a,b,c,d], in generated profile, +# it will be charged to [a] +sub FlatProfile { + my $profile = shift; + my $result = {}; + foreach my $k (keys(%{$profile})) { + my $count = $profile->{$k}; + my @addrs = split(/\n/, $k); + if ($#addrs >= 0) { + AddEntry($result, $addrs[0], $count); + } + } + return $result; +} + +# Generate cumulative profile: +# If count is charged to stack [a,b,c,d], in generated profile, +# it will be charged to [a], [b], [c], [d] +sub CumulativeProfile { + my $profile = shift; + my $result = {}; + foreach my $k (keys(%{$profile})) { + my $count = $profile->{$k}; + my @addrs = split(/\n/, $k); + foreach my $a (@addrs) { + AddEntry($result, $a, $count); + } + } + return $result; +} + +# If the second-youngest PC on the stack is always the same, returns +# that pc. Otherwise, returns undef. +sub IsSecondPcAlwaysTheSame { + my $profile = shift; + + my $second_pc = undef; + foreach my $k (keys(%{$profile})) { + my @addrs = split(/\n/, $k); + if ($#addrs < 1) { + return undef; + } + if (not defined $second_pc) { + $second_pc = $addrs[1]; + } else { + if ($second_pc ne $addrs[1]) { + return undef; + } + } + } + return $second_pc; +} + +sub ExtractSymbolLocation { + my $symbols = shift; + my $address = shift; + # 'addr2line' outputs "??:0" for unknown locations; we do the + # same to be consistent. + my $location = "??:0:unknown"; + if (exists $symbols->{$address}) { + my $file = $symbols->{$address}->[1]; + if ($file eq "?") { + $file = "??:0" + } + $location = $file . ":" . $symbols->{$address}->[0]; + } + return $location; +} + +# Extracts a graph of calls. +sub ExtractCalls { + my $symbols = shift; + my $profile = shift; + + my $calls = {}; + while( my ($stack_trace, $count) = each %$profile ) { + my @address = split(/\n/, $stack_trace); + my $destination = ExtractSymbolLocation($symbols, $address[0]); + AddEntry($calls, $destination, $count); + for (my $i = 1; $i <= $#address; $i++) { + my $source = ExtractSymbolLocation($symbols, $address[$i]); + my $call = "$source -> $destination"; + AddEntry($calls, $call, $count); + $destination = $source; + } + } + + return $calls; +} + +sub RemoveUninterestingFrames { + my $symbols = shift; + my $profile = shift; + + # List of function names to skip + my %skip = (); + my $skip_regexp = 'NOMATCH'; + if ($main::profile_type eq 'heap' || $main::profile_type eq 'growth') { + foreach my $name ('calloc', + 'cfree', + 'malloc', + 'free', + 'memalign', + 'posix_memalign', + 'pvalloc', + 'valloc', + 'realloc', + 'tc_calloc', + 'tc_cfree', + 'tc_malloc', + 'tc_free', + 'tc_memalign', + 'tc_posix_memalign', + 'tc_pvalloc', + 'tc_valloc', + 'tc_realloc', + 'tc_new', + 'tc_delete', + 'tc_newarray', + 'tc_deletearray', + 'tc_new_nothrow', + 'tc_newarray_nothrow', + 'do_malloc', + '::do_malloc', # new name -- got moved to an unnamed ns + '::do_malloc_or_cpp_alloc', + 'DoSampledAllocation', + 'simple_alloc::allocate', + '__malloc_alloc_template::allocate', + '__builtin_delete', + '__builtin_new', + '__builtin_vec_delete', + '__builtin_vec_new', + 'operator new', + 'operator new[]', + # Go + 'catstring', + 'copyin', + 'gostring', + 'gostringsize', + 'growslice1', + 'appendslice1', + 'hash_init', + 'hash_subtable_new', + 'hash_conv', + 'hash_grow', + 'hash_insert_internal', + 'hash_insert', + 'mapassign', + 'runtime.mapassign', + 'runtime.appendslice', + 'runtime.mapassign1', + 'makechan', + 'makemap', + 'mal', + 'runtime.new', + 'makeslice1', + 'runtime.gostringsize', + 'runtime.malloc', + 'unsafe.New', + 'runtime.mallocgc', + 'runtime.catstring', + 'runtime.growslice', + 'runtime.ifaceT2E', + 'runtime.ifaceT2I', + 'runtime.makechan', + 'runtime.makechan_c', + 'runtime.makemap', + 'runtime.makemap_c', + 'runtime.makeslice', + 'runtime.mal', + 'runtime.slicebytetostring', + 'runtime.sliceinttostring', + 'runtime.stringtoslicebyte', + 'runtime.stringtosliceint', + # These mark the beginning/end of our custom sections + '__start_google_malloc', + '__stop_google_malloc', + '__start_malloc_hook', + '__stop_malloc_hook') { + $skip{$name} = 1; + $skip{"_" . $name} = 1; # Mach (OS X) adds a _ prefix to everything + } + # TODO: Remove TCMalloc once everything has been + # moved into the tcmalloc:: namespace and we have flushed + # old code out of the system. + $skip_regexp = "TCMalloc|^tcmalloc::"; + } elsif ($main::profile_type eq 'contention') { + foreach my $vname ('Mutex::Unlock', 'Mutex::UnlockSlow') { + $skip{$vname} = 1; + } + } elsif ($main::profile_type eq 'cpu') { + # Drop signal handlers used for CPU profile collection + # TODO(dpeng): this should not be necessary; it's taken + # care of by the general 2nd-pc mechanism below. + foreach my $name ('ProfileData::Add', # historical + 'ProfileData::prof_handler', # historical + 'CpuProfiler::prof_handler', + '__FRAME_END__', + '__pthread_sighandler', + '__restore') { + $skip{$name} = 1; + } + } else { + # Nothing skipped for unknown types + } + + # Go doesn't have the problem that this heuristic tries to fix. Disable. + if (0 && $main::profile_type eq 'cpu') { + # If all the second-youngest program counters are the same, + # this STRONGLY suggests that it is an artifact of measurement, + # i.e., stack frames pushed by the CPU profiler signal handler. + # Hence, we delete them. + # (The topmost PC is read from the signal structure, not from + # the stack, so it does not get involved.) + while (my $second_pc = IsSecondPcAlwaysTheSame($profile)) { + my $result = {}; + my $func = ''; + if (exists($symbols->{$second_pc})) { + $second_pc = $symbols->{$second_pc}->[0]; + } + print STDERR "Removing $second_pc from all stack traces.\n"; + foreach my $k (keys(%{$profile})) { + my $count = $profile->{$k}; + my @addrs = split(/\n/, $k); + splice @addrs, 1, 1; + my $reduced_path = join("\n", @addrs); + AddEntry($result, $reduced_path, $count); + } + $profile = $result; + } + } + + my $result = {}; + foreach my $k (keys(%{$profile})) { + my $count = $profile->{$k}; + my @addrs = split(/\n/, $k); + my @path = (); + foreach my $a (@addrs) { + if (exists($symbols->{$a})) { + my $func = $symbols->{$a}->[0]; + if ($skip{$func} || ($func =~ m/$skip_regexp/)) { + next; + } + } + push(@path, $a); + } + my $reduced_path = join("\n", @path); + AddEntry($result, $reduced_path, $count); + } + return $result; +} + +# Reduce profile to granularity given by user +sub ReduceProfile { + my $symbols = shift; + my $profile = shift; + my $result = {}; + foreach my $k (keys(%{$profile})) { + my $count = $profile->{$k}; + my @translated = TranslateStack($symbols, $k); + my @path = (); + my %seen = (); + $seen{''} = 1; # So that empty keys are skipped + foreach my $e (@translated) { + # To avoid double-counting due to recursion, skip a stack-trace + # entry if it has already been seen + if (!$seen{$e}) { + $seen{$e} = 1; + push(@path, $e); + } + } + my $reduced_path = join("\n", @path); + AddEntry($result, $reduced_path, $count); + } + return $result; +} + +# Does the specified symbol array match the regexp? +sub SymbolMatches { + my $sym = shift; + my $re = shift; + if (defined($sym)) { + for (my $i = 0; $i < $#{$sym}; $i += 3) { + if ($sym->[$i] =~ m/$re/ || $sym->[$i+1] =~ m/$re/) { + return 1; + } + } + } + return 0; +} + +# Focus only on paths involving specified regexps +sub FocusProfile { + my $symbols = shift; + my $profile = shift; + my $focus = shift; + my $result = {}; + foreach my $k (keys(%{$profile})) { + my $count = $profile->{$k}; + my @addrs = split(/\n/, $k); + foreach my $a (@addrs) { + # Reply if it matches either the address/shortname/fileline + if (($a =~ m/$focus/) || SymbolMatches($symbols->{$a}, $focus)) { + AddEntry($result, $k, $count); + last; + } + } + } + return $result; +} + +# Focus only on paths not involving specified regexps +sub IgnoreProfile { + my $symbols = shift; + my $profile = shift; + my $ignore = shift; + my $result = {}; + foreach my $k (keys(%{$profile})) { + my $count = $profile->{$k}; + my @addrs = split(/\n/, $k); + my $matched = 0; + foreach my $a (@addrs) { + # Reply if it matches either the address/shortname/fileline + if (($a =~ m/$ignore/) || SymbolMatches($symbols->{$a}, $ignore)) { + $matched = 1; + last; + } + } + if (!$matched) { + AddEntry($result, $k, $count); + } + } + return $result; +} + +# Get total count in profile +sub TotalProfile { + my $profile = shift; + my $result = 0; + foreach my $k (keys(%{$profile})) { + $result += $profile->{$k}; + } + return $result; +} + +# Add A to B +sub AddProfile { + my $A = shift; + my $B = shift; + + my $R = {}; + # add all keys in A + foreach my $k (keys(%{$A})) { + my $v = $A->{$k}; + AddEntry($R, $k, $v); + } + # add all keys in B + foreach my $k (keys(%{$B})) { + my $v = $B->{$k}; + AddEntry($R, $k, $v); + } + return $R; +} + +# Merges symbol maps +sub MergeSymbols { + my $A = shift; + my $B = shift; + + my $R = {}; + foreach my $k (keys(%{$A})) { + $R->{$k} = $A->{$k}; + } + if (defined($B)) { + foreach my $k (keys(%{$B})) { + $R->{$k} = $B->{$k}; + } + } + return $R; +} + + +# Add A to B +sub AddPcs { + my $A = shift; + my $B = shift; + + my $R = {}; + # add all keys in A + foreach my $k (keys(%{$A})) { + $R->{$k} = 1 + } + # add all keys in B + foreach my $k (keys(%{$B})) { + $R->{$k} = 1 + } + return $R; +} + +# Subtract B from A +sub SubtractProfile { + my $A = shift; + my $B = shift; + + my $R = {}; + foreach my $k (keys(%{$A})) { + my $v = $A->{$k} - GetEntry($B, $k); + if ($v < 0 && $main::opt_drop_negative) { + $v = 0; + } + AddEntry($R, $k, $v); + } + if (!$main::opt_drop_negative) { + # Take care of when subtracted profile has more entries + foreach my $k (keys(%{$B})) { + if (!exists($A->{$k})) { + AddEntry($R, $k, 0 - $B->{$k}); + } + } + } + return $R; +} + +# Get entry from profile; zero if not present +sub GetEntry { + my $profile = shift; + my $k = shift; + if (exists($profile->{$k})) { + return $profile->{$k}; + } else { + return 0; + } +} + +# Add entry to specified profile +sub AddEntry { + my $profile = shift; + my $k = shift; + my $n = shift; + if (!exists($profile->{$k})) { + $profile->{$k} = 0; + } + $profile->{$k} += $n; +} + +# Add a stack of entries to specified profile, and add them to the $pcs +# list. +sub AddEntries { + my $profile = shift; + my $pcs = shift; + my $stack = shift; + my $count = shift; + my @k = (); + + foreach my $e (split(/\s+/, $stack)) { + my $pc = HexExtend($e); + $pcs->{$pc} = 1; + push @k, $pc; + } + AddEntry($profile, (join "\n", @k), $count); +} + +sub IsSymbolizedProfileFile { + my $file_name = shift; + + if (!(-e $file_name) || !(-r $file_name)) { + return 0; + } + + $SYMBOL_PAGE =~ m,[^/]+$,; # matches everything after the last slash + my $symbol_marker = $&; + # Check if the file contains a symbol-section marker. + open(TFILE, "<$file_name"); + my @lines = ; + my $result = grep(/^--- *$symbol_marker/, @lines); + close(TFILE); + return $result > 0; +} + +##### Code to profile a server dynamically ##### + +sub CheckSymbolPage { + my $url = SymbolPageURL(); +print STDERR "Read $url\n"; + open(SYMBOL, "$CURL -s '$url' |"); + my $line = ; + $line =~ s/\r//g; # turn windows-looking lines into unix-looking lines + close(SYMBOL); + unless (defined($line)) { + error("$url doesn't exist\n"); + } + + if ($line =~ /^num_symbols:\s+(\d+)$/) { + if ($1 == 0) { + error("Stripped binary. No symbols available.\n"); + } + } else { + error("Failed to get the number of symbols from $url\n"); + } +} + +sub IsProfileURL { + my $profile_name = shift; + my ($host, $port, $prefix, $path) = ParseProfileURL($profile_name); + return defined($host) and defined($port) and defined($path); +} + +sub ParseProfileURL { + my $profile_name = shift; + if (defined($profile_name) && + $profile_name =~ m,^(http://|)([^/:]+):(\d+)(|\@\d+)(|/|(.*?)($PROFILE_PAGE|$PMUPROFILE_PAGE|$HEAP_PAGE|$GROWTH_PAGE|$CONTENTION_PAGE|$WALL_PAGE|$FILTEREDPROFILE_PAGE))$,o) { + # $7 is $PROFILE_PAGE/$HEAP_PAGE/etc. $5 is *everything* after + # the hostname, as long as that everything is the empty string, + # a slash, or something ending in $PROFILE_PAGE/$HEAP_PAGE/etc. + # So "$7 || $5" is $PROFILE_PAGE/etc if there, or else it's "/" or "". + return ($2, $3, $6, $7 || $5); + } + return (); +} + +# We fetch symbols from the first profile argument. +sub SymbolPageURL { + my ($host, $port, $prefix, $path) = ParseProfileURL($main::pfile_args[0]); + return "http://$host:$port$prefix$SYMBOL_PAGE"; +} + +sub FetchProgramName() { + my ($host, $port, $prefix, $path) = ParseProfileURL($main::pfile_args[0]); + my $url = "http://$host:$port$prefix$PROGRAM_NAME_PAGE"; + my $command_line = "$CURL -s '$url'"; + open(CMDLINE, "$command_line |") or error($command_line); + my $cmdline = ; + $cmdline =~ s/\r//g; # turn windows-looking lines into unix-looking lines + close(CMDLINE); + error("Failed to get program name from $url\n") unless defined($cmdline); + $cmdline =~ s/\x00.+//; # Remove argv[1] and latters. + $cmdline =~ s!\n!!g; # Remove LFs. + return $cmdline; +} + +# Gee, curl's -L (--location) option isn't reliable at least +# with its 7.12.3 version. Curl will forget to post data if +# there is a redirection. This function is a workaround for +# curl. Redirection happens on borg hosts. +sub ResolveRedirectionForCurl { + my $url = shift; + my $command_line = "$CURL -s --head '$url'"; + open(CMDLINE, "$command_line |") or error($command_line); + while () { + s/\r//g; # turn windows-looking lines into unix-looking lines + if (/^Location: (.*)/) { + $url = $1; + } + } + close(CMDLINE); + return $url; +} + +# Reads a symbol map from the file handle name given as $1, returning +# the resulting symbol map. Also processes variables relating to symbols. +# Currently, the only variable processed is 'binary=' which updates +# $main::prog to have the correct program name. +sub ReadSymbols { + my $in = shift; + my $map = shift; + while (<$in>) { + s/\r//g; # turn windows-looking lines into unix-looking lines + # Removes all the leading zeroes from the symbols, see comment below. + if (m/^0x0*([0-9a-f]+)\s+(.+)/) { + $map->{$1} = $2; + } elsif (m/^---/) { + last; + } elsif (m/^([a-z][^=]*)=(.*)$/ ) { + my ($variable, $value) = ($1, $2); + for ($variable, $value) { + s/^\s+//; + s/\s+$//; + } + if ($variable eq "binary") { + if ($main::prog ne $UNKNOWN_BINARY && $main::prog ne $value) { + printf STDERR ("Warning: Mismatched binary name '%s', using '%s'.\n", + $main::prog, $value); + } + $main::prog = $value; + } else { + printf STDERR ("Ignoring unknown variable in symbols list: " . + "'%s' = '%s'\n", $variable, $value); + } + } + } + return $map; +} + +# Fetches and processes symbols to prepare them for use in the profile output +# code. If the optional 'symbol_map' arg is not given, fetches symbols from +# $SYMBOL_PAGE for all PC values found in profile. Otherwise, the raw symbols +# are assumed to have already been fetched into 'symbol_map' and are simply +# extracted and processed. +sub FetchSymbols { + my $pcset = shift; + my $symbol_map = shift; + + my %seen = (); + my @pcs = grep { !$seen{$_}++ } keys(%$pcset); # uniq + + if (!defined($symbol_map)) { + $symbol_map = {}; + my @toask = @pcs; + while (@toask > 0) { + my $n = @toask; + # NOTE(rsc): Limiting the number of PCs requested per round + # used to be necessary, but I think it was a bug in + # debug/pprof/symbol's implementation. Leaving here + # in case I am wrong. + # if ($n > 49) { $n = 49; } + my @thisround = @toask[0..$n]; + @toask = @toask[($n+1)..(@toask-1)]; + my $post_data = join("+", sort((map {"0x" . "$_"} @thisround))); + open(POSTFILE, ">$main::tmpfile_sym"); + print POSTFILE $post_data; + close(POSTFILE); + + my $url = SymbolPageURL(); + $url = ResolveRedirectionForCurl($url); + my $command_line = "$CURL -sd '\@$main::tmpfile_sym' '$url'"; + # We use c++filt in case $SYMBOL_PAGE gives us mangled symbols. + my $cppfilt = $obj_tool_map{"c++filt"}; + open(SYMBOL, "$command_line | $cppfilt |") or error($command_line); + ReadSymbols(*SYMBOL{IO}, $symbol_map); + close(SYMBOL); + } + } + + my $symbols = {}; + foreach my $pc (@pcs) { + my $fullname; + # For 64 bits binaries, symbols are extracted with 8 leading zeroes. + # Then /symbol reads the long symbols in as uint64, and outputs + # the result with a "0x%08llx" format which get rid of the zeroes. + # By removing all the leading zeroes in both $pc and the symbols from + # /symbol, the symbols match and are retrievable from the map. + my $shortpc = $pc; + $shortpc =~ s/^0*//; + # Each line may have a list of names, which includes the function + # and also other functions it has inlined. They are separated + # (in PrintSymbolizedFile), by --, which is illegal in function names. + my $fullnames; + if (defined($symbol_map->{$shortpc})) { + $fullnames = $symbol_map->{$shortpc}; + } else { + $fullnames = "0x" . $pc; # Just use addresses + } + my $sym = []; + $symbols->{$pc} = $sym; + foreach my $fullname (split("--", $fullnames)) { + my $name = ShortFunctionName($fullname); + push(@{$sym}, $name, "?", $fullname); + } + } + return $symbols; +} + +sub BaseName { + my $file_name = shift; + $file_name =~ s!^.*/!!; # Remove directory name + return $file_name; +} + +sub MakeProfileBaseName { + my ($binary_name, $profile_name) = @_; + my ($host, $port, $prefix, $path) = ParseProfileURL($profile_name); + my $binary_shortname = BaseName($binary_name); + return sprintf("%s.%s.%s-port%s", + $binary_shortname, $main::op_time, $host, $port); +} + +sub FetchDynamicProfile { + my $binary_name = shift; + my $profile_name = shift; + my $fetch_name_only = shift; + my $encourage_patience = shift; + + if (!IsProfileURL($profile_name)) { + return $profile_name; + } else { + my ($host, $port, $prefix, $path) = ParseProfileURL($profile_name); + if ($path eq "" || $path eq "/") { + # Missing type specifier defaults to cpu-profile + $path = $PROFILE_PAGE; + } + + my $profile_file = MakeProfileBaseName($binary_name, $profile_name); + + my $url; + my $curl_timeout; + if (($path =~ m/$PROFILE_PAGE/) || ($path =~ m/$PMUPROFILE_PAGE/)) { + if ($path =~ m/$PROFILE_PAGE/) { + $url = sprintf("http://$host:$port$prefix$path?seconds=%d", + $main::opt_seconds); + } else { + if ($profile_name =~ m/[?]/) { + $profile_name .= "&" + } else { + $profile_name .= "?" + } + $url = sprintf("http://$profile_name" . "seconds=%d", + $main::opt_seconds); + } + $curl_timeout = sprintf("--max-time %d", + int($main::opt_seconds * 1.01 + 60)); + } else { + # For non-CPU profiles, we add a type-extension to + # the target profile file name. + my $suffix = $path; + $suffix =~ s,/,.,g; + $profile_file .= "$suffix"; + $url = "http://$host:$port$prefix$path"; + $curl_timeout = ""; + } + + my $profile_dir = $ENV{"PPROF_TMPDIR"} || ($ENV{HOME} . "/pprof"); + if (!(-d $profile_dir)) { + mkdir($profile_dir) + || die("Unable to create profile directory $profile_dir: $!\n"); + } + my $tmp_profile = "$profile_dir/.tmp.$profile_file"; + my $real_profile = "$profile_dir/$profile_file"; + + if ($fetch_name_only > 0) { + return $real_profile; + } + + my $cmd = "$CURL $curl_timeout -s -o $tmp_profile '$url'"; + if (($path =~ m/$PROFILE_PAGE/) || ($path =~ m/$PMUPROFILE_PAGE/)){ + print STDERR "Gathering CPU profile from $url for $main::opt_seconds seconds to\n ${real_profile}\n"; + if ($encourage_patience) { + print STDERR "Be patient...\n"; + } + } else { + print STDERR "Fetching $path profile from $host:$port to\n ${real_profile}\n"; + } + + (system($cmd) == 0) || error("Failed to get profile: $cmd: $!\n"); + (system("mv $tmp_profile $real_profile") == 0) || error("Unable to rename profile\n"); + print STDERR "Wrote profile to $real_profile\n"; + $main::collected_profile = $real_profile; + return $main::collected_profile; + } +} + +# Collect profiles in parallel +sub FetchDynamicProfiles { + my $items = scalar(@main::pfile_args); + my $levels = log($items) / log(2); + + if ($items == 1) { + $main::profile_files[0] = FetchDynamicProfile($main::prog, $main::pfile_args[0], 0, 1); + } else { + # math rounding issues + if ((2 ** $levels) < $items) { + $levels++; + } + my $count = scalar(@main::pfile_args); + for (my $i = 0; $i < $count; $i++) { + $main::profile_files[$i] = FetchDynamicProfile($main::prog, $main::pfile_args[$i], 1, 0); + } + print STDERR "Fetching $count profiles, Be patient...\n"; + FetchDynamicProfilesRecurse($levels, 0, 0); + $main::collected_profile = join(" \\\n ", @main::profile_files); + } +} + +# Recursively fork a process to get enough processes +# collecting profiles +sub FetchDynamicProfilesRecurse { + my $maxlevel = shift; + my $level = shift; + my $position = shift; + + if (my $pid = fork()) { + $position = 0 | ($position << 1); + TryCollectProfile($maxlevel, $level, $position); + wait; + } else { + $position = 1 | ($position << 1); + TryCollectProfile($maxlevel, $level, $position); + exit(0); + } +} + +# Collect a single profile +sub TryCollectProfile { + my $maxlevel = shift; + my $level = shift; + my $position = shift; + + if ($level >= ($maxlevel - 1)) { + if ($position < scalar(@main::pfile_args)) { + FetchDynamicProfile($main::prog, $main::pfile_args[$position], 0, 0); + } + } else { + FetchDynamicProfilesRecurse($maxlevel, $level+1, $position); + } +} + +##### Parsing code ##### + +# Provide a small streaming-read module to handle very large +# cpu-profile files. Stream in chunks along a sliding window. +# Provides an interface to get one 'slot', correctly handling +# endian-ness differences. A slot is one 32-bit or 64-bit word +# (depending on the input profile). We tell endianness and bit-size +# for the profile by looking at the first 8 bytes: in cpu profiles, +# the second slot is always 3 (we'll accept anything that's not 0). +BEGIN { + package CpuProfileStream; + + sub new { + my ($class, $file, $fname) = @_; + my $self = { file => $file, + base => 0, + stride => 512 * 1024, # must be a multiple of bitsize/8 + slots => [], + unpack_code => "", # N for big-endian, V for little + }; + bless $self, $class; + # Let unittests adjust the stride + if ($main::opt_test_stride > 0) { + $self->{stride} = $main::opt_test_stride; + } + # Read the first two slots to figure out bitsize and endianness. + my $slots = $self->{slots}; + my $str; + read($self->{file}, $str, 8); + # Set the global $address_length based on what we see here. + # 8 is 32-bit (8 hexadecimal chars); 16 is 64-bit (16 hexadecimal chars). + $address_length = ($str eq (chr(0)x8)) ? 16 : 8; + if ($address_length == 8) { + if (substr($str, 6, 2) eq chr(0)x2) { + $self->{unpack_code} = 'V'; # Little-endian. + } elsif (substr($str, 4, 2) eq chr(0)x2) { + $self->{unpack_code} = 'N'; # Big-endian + } else { + ::error("$fname: header size >= 2**16\n"); + } + @$slots = unpack($self->{unpack_code} . "*", $str); + } else { + # If we're a 64-bit profile, make sure we're a 64-bit-capable + # perl. Otherwise, each slot will be represented as a float + # instead of an int64, losing precision and making all the + # 64-bit addresses right. We *could* try to handle this with + # software emulation of 64-bit ints, but that's added complexity + # for no clear benefit (yet). We use 'Q' to test for 64-bit-ness; + # perl docs say it's only available on 64-bit perl systems. + my $has_q = 0; + eval { $has_q = pack("Q", "1") ? 1 : 1; }; + if (!$has_q) { + ::error("$fname: need a 64-bit perl to process this 64-bit profile.\n"); + } + read($self->{file}, $str, 8); + if (substr($str, 4, 4) eq chr(0)x4) { + # We'd love to use 'Q', but it's a) not universal, b) not endian-proof. + $self->{unpack_code} = 'V'; # Little-endian. + } elsif (substr($str, 0, 4) eq chr(0)x4) { + $self->{unpack_code} = 'N'; # Big-endian + } else { + ::error("$fname: header size >= 2**32\n"); + } + my @pair = unpack($self->{unpack_code} . "*", $str); + # Since we know one of the pair is 0, it's fine to just add them. + @$slots = (0, $pair[0] + $pair[1]); + } + return $self; + } + + # Load more data when we access slots->get(X) which is not yet in memory. + sub overflow { + my ($self) = @_; + my $slots = $self->{slots}; + $self->{base} += $#$slots + 1; # skip over data we're replacing + my $str; + read($self->{file}, $str, $self->{stride}); + if ($address_length == 8) { # the 32-bit case + # This is the easy case: unpack provides 32-bit unpacking primitives. + @$slots = unpack($self->{unpack_code} . "*", $str); + } else { + # We need to unpack 32 bits at a time and combine. + my @b32_values = unpack($self->{unpack_code} . "*", $str); + my @b64_values = (); + for (my $i = 0; $i < $#b32_values; $i += 2) { + # TODO(csilvers): if this is a 32-bit perl, the math below + # could end up in a too-large int, which perl will promote + # to a double, losing necessary precision. Deal with that. + if ($self->{unpack_code} eq 'V') { # little-endian + push(@b64_values, $b32_values[$i] + $b32_values[$i+1] * (2**32)); + } else { + push(@b64_values, $b32_values[$i] * (2**32) + $b32_values[$i+1]); + } + } + @$slots = @b64_values; + } + } + + # Access the i-th long in the file (logically), or -1 at EOF. + sub get { + my ($self, $idx) = @_; + my $slots = $self->{slots}; + while ($#$slots >= 0) { + if ($idx < $self->{base}) { + # The only time we expect a reference to $slots[$i - something] + # after referencing $slots[$i] is reading the very first header. + # Since $stride > |header|, that shouldn't cause any lookback + # errors. And everything after the header is sequential. + print STDERR "Unexpected look-back reading CPU profile"; + return -1; # shrug, don't know what better to return + } elsif ($idx > $self->{base} + $#$slots) { + $self->overflow(); + } else { + return $slots->[$idx - $self->{base}]; + } + } + # If we get here, $slots is [], which means we've reached EOF + return -1; # unique since slots is supposed to hold unsigned numbers + } +} + +# Parse profile generated by common/profiler.cc and return a reference +# to a map: +# $result->{version} Version number of profile file +# $result->{period} Sampling period (in microseconds) +# $result->{profile} Profile object +# $result->{map} Memory map info from profile +# $result->{pcs} Hash of all PC values seen, key is hex address +sub ReadProfile { + my $prog = shift; + my $fname = shift; + + if (IsSymbolizedProfileFile($fname) && !$main::use_symbolized_profile) { + # we have both a binary and symbolized profiles, abort + usage("Symbolized profile '$fname' cannot be used with a binary arg. " . + "Try again without passing '$prog'."); + } + + $main::profile_type = ''; + + $CONTENTION_PAGE =~ m,[^/]+$,; # matches everything after the last slash + my $contention_marker = $&; + $GROWTH_PAGE =~ m,[^/]+$,; # matches everything after the last slash + my $growth_marker = $&; + $SYMBOL_PAGE =~ m,[^/]+$,; # matches everything after the last slash + my $symbol_marker = $&; + $PROFILE_PAGE =~ m,[^/]+$,; # matches everything after the last slash + my $profile_marker = $&; + + # Look at first line to see if it is a heap or a CPU profile. + # CPU profile may start with no header at all, and just binary data + # (starting with \0\0\0\0) -- in that case, don't try to read the + # whole firstline, since it may be gigabytes(!) of data. + open(PROFILE, "<$fname") || error("$fname: $!\n"); + binmode PROFILE; # New perls do UTF-8 processing + my $firstchar = ""; + my $header = ""; + read(PROFILE, $firstchar, 1); + seek(PROFILE, -1, 1); # unread the firstchar + if ($firstchar ne "\0") { + $header = ; + if (!defined($header)) { + error("Profile is empty.\n"); + } + $header =~ s/\r//g; # turn windows-looking lines into unix-looking lines + } + + my $symbols; + if ($header =~ m/^--- *$symbol_marker/o) { + # read the symbol section of the symbolized profile file + $symbols = ReadSymbols(*PROFILE{IO}); + + # read the next line to get the header for the remaining profile + $header = ""; + read(PROFILE, $firstchar, 1); + seek(PROFILE, -1, 1); # unread the firstchar + if ($firstchar ne "\0") { + $header = ; + $header =~ s/\r//g; + } + } + + my $result; + + if ($header =~ m/^heap profile:.*$growth_marker/o) { + $main::profile_type = 'growth'; + $result = ReadHeapProfile($prog, $fname, $header); + } elsif ($header =~ m/^heap profile:/) { + $main::profile_type = 'heap'; + $result = ReadHeapProfile($prog, $fname, $header); + } elsif ($header =~ m/^--- *$contention_marker/o) { + $main::profile_type = 'contention'; + $result = ReadSynchProfile($prog, $fname); + } elsif ($header =~ m/^--- *Stacks:/) { + print STDERR + "Old format contention profile: mistakenly reports " . + "condition variable signals as lock contentions.\n"; + $main::profile_type = 'contention'; + $result = ReadSynchProfile($prog, $fname); + } elsif ($header =~ m/^--- *$profile_marker/) { + # the binary cpu profile data starts immediately after this line + $main::profile_type = 'cpu'; + $result = ReadCPUProfile($prog, $fname); + } else { + if (defined($symbols)) { + # a symbolized profile contains a format we don't recognize, bail out + error("$fname: Cannot recognize profile section after symbols.\n"); + } + # no ascii header present -- must be a CPU profile + $main::profile_type = 'cpu'; + $result = ReadCPUProfile($prog, $fname); + } + + # if we got symbols along with the profile, return those as well + if (defined($symbols)) { + $result->{symbols} = $symbols; + } + + return $result; +} + +# Subtract one from caller pc so we map back to call instr. +# However, don't do this if we're reading a symbolized profile +# file, in which case the subtract-one was done when the file +# was written. +# +# We apply the same logic to all readers, though ReadCPUProfile uses an +# independent implementation. +sub FixCallerAddresses { + my $stack = shift; + if ($main::use_symbolized_profile) { + return $stack; + } else { + $stack =~ /(\s)/; + my $delimiter = $1; + my @addrs = split(' ', $stack); + my @fixedaddrs; + $#fixedaddrs = $#addrs; + if ($#addrs >= 0) { + $fixedaddrs[0] = $addrs[0]; + } + for (my $i = 1; $i <= $#addrs; $i++) { + $fixedaddrs[$i] = AddressSub($addrs[$i], "0x1"); + } + return join $delimiter, @fixedaddrs; + } +} + +# CPU profile reader +sub ReadCPUProfile { + my $prog = shift; + my $fname = shift; + my $version; + my $period; + my $i; + my $profile = {}; + my $pcs = {}; + + # Parse string into array of slots. + my $slots = CpuProfileStream->new(*PROFILE, $fname); + + # Read header. The current header version is a 5-element structure + # containing: + # 0: header count (always 0) + # 1: header "words" (after this one: 3) + # 2: format version (0) + # 3: sampling period (usec) + # 4: unused padding (always 0) + if ($slots->get(0) != 0 ) { + error("$fname: not a profile file, or old format profile file\n"); + } + $i = 2 + $slots->get(1); + $version = $slots->get(2); + $period = $slots->get(3); + # Do some sanity checking on these header values. + if ($version > (2**32) || $period > (2**32) || $i > (2**32) || $i < 5) { + error("$fname: not a profile file, or corrupted profile file\n"); + } + + # Parse profile + while ($slots->get($i) != -1) { + my $n = $slots->get($i++); + my $d = $slots->get($i++); + if ($d > (2**16)) { # TODO(csilvers): what's a reasonable max-stack-depth? + my $addr = sprintf("0%o", $i * ($address_length == 8 ? 4 : 8)); + print STDERR "At index $i (address $addr):\n"; + error("$fname: stack trace depth >= 2**32\n"); + } + if ($slots->get($i) == 0) { + # End of profile data marker + $i += $d; + last; + } + + # Make key out of the stack entries + my @k = (); + for (my $j = 0; $j < $d; $j++) { + my $pc = $slots->get($i+$j); + # Subtract one from caller pc so we map back to call instr. + # However, don't do this if we're reading a symbolized profile + # file, in which case the subtract-one was done when the file + # was written. + if ($j > 0 && !$main::use_symbolized_profile) { + $pc--; + } + $pc = sprintf("%0*x", $address_length, $pc); + $pcs->{$pc} = 1; + push @k, $pc; + } + + AddEntry($profile, (join "\n", @k), $n); + $i += $d; + } + + # Parse map + my $map = ''; + seek(PROFILE, $i * 4, 0); + read(PROFILE, $map, (stat PROFILE)[7]); + close(PROFILE); + + my $r = {}; + $r->{version} = $version; + $r->{period} = $period; + $r->{profile} = $profile; + $r->{libs} = ParseLibraries($prog, $map, $pcs); + $r->{pcs} = $pcs; + + return $r; +} + +sub ReadHeapProfile { + my $prog = shift; + my $fname = shift; + my $header = shift; + + my $index = 1; + if ($main::opt_inuse_space) { + $index = 1; + } elsif ($main::opt_inuse_objects) { + $index = 0; + } elsif ($main::opt_alloc_space) { + $index = 3; + } elsif ($main::opt_alloc_objects) { + $index = 2; + } + + # Find the type of this profile. The header line looks like: + # heap profile: 1246: 8800744 [ 1246: 8800744] @ /266053 + # There are two pairs , the first inuse objects/space, and the + # second allocated objects/space. This is followed optionally by a profile + # type, and if that is present, optionally by a sampling frequency. + # For remote heap profiles (v1): + # The interpretation of the sampling frequency is that the profiler, for + # each sample, calculates a uniformly distributed random integer less than + # the given value, and records the next sample after that many bytes have + # been allocated. Therefore, the expected sample interval is half of the + # given frequency. By default, if not specified, the expected sample + # interval is 128KB. Only remote-heap-page profiles are adjusted for + # sample size. + # For remote heap profiles (v2): + # The sampling frequency is the rate of a Poisson process. This means that + # the probability of sampling an allocation of size X with sampling rate Y + # is 1 - exp(-X/Y) + # For version 2, a typical header line might look like this: + # heap profile: 1922: 127792360 [ 1922: 127792360] @ _v2/524288 + # the trailing number (524288) is the sampling rate. (Version 1 showed + # double the 'rate' here) + my $sampling_algorithm = 0; + my $sample_adjustment = 0; + chomp($header); + my $type = "unknown"; + if ($header =~ m"^heap profile:\s*(\d+):\s+(\d+)\s+\[\s*(\d+):\s+(\d+)\](\s*@\s*([^/]*)(/(\d+))?)?") { + if (defined($6) && ($6 ne '')) { + $type = $6; + my $sample_period = $8; + # $type is "heapprofile" for profiles generated by the + # heap-profiler, and either "heap" or "heap_v2" for profiles + # generated by sampling directly within tcmalloc. It can also + # be "growth" for heap-growth profiles. The first is typically + # found for profiles generated locally, and the others for + # remote profiles. + if (($type eq "heapprofile") || ($type !~ /heap/) ) { + # No need to adjust for the sampling rate with heap-profiler-derived data + $sampling_algorithm = 0; + } elsif ($type =~ /_v2/) { + $sampling_algorithm = 2; # version 2 sampling + if (defined($sample_period) && ($sample_period ne '')) { + $sample_adjustment = int($sample_period); + } + } else { + $sampling_algorithm = 1; # version 1 sampling + if (defined($sample_period) && ($sample_period ne '')) { + $sample_adjustment = int($sample_period)/2; + } + } + } else { + # We detect whether or not this is a remote-heap profile by checking + # that the total-allocated stats ($n2,$s2) are exactly the + # same as the in-use stats ($n1,$s1). It is remotely conceivable + # that a non-remote-heap profile may pass this check, but it is hard + # to imagine how that could happen. + # In this case it's so old it's guaranteed to be remote-heap version 1. + my ($n1, $s1, $n2, $s2) = ($1, $2, $3, $4); + if (($n1 == $n2) && ($s1 == $s2)) { + # This is likely to be a remote-heap based sample profile + $sampling_algorithm = 1; + } + } + } + + if ($sampling_algorithm > 0) { + # For remote-heap generated profiles, adjust the counts and sizes to + # account for the sample rate (we sample once every 128KB by default). + if ($sample_adjustment == 0) { + # Turn on profile adjustment. + $sample_adjustment = 128*1024; + print STDERR "Adjusting heap profiles for 1-in-128KB sampling rate\n"; + } else { + printf STDERR ("Adjusting heap profiles for 1-in-%d sampling rate\n", + $sample_adjustment); + } + if ($sampling_algorithm > 1) { + # We don't bother printing anything for the original version (version 1) + printf STDERR "Heap version $sampling_algorithm\n"; + } + } + + my $profile = {}; + my $pcs = {}; + my $map = ""; + + while () { + s/\r//g; # turn windows-looking lines into unix-looking lines + if (/^MAPPED_LIBRARIES:/) { + # Read the /proc/self/maps data + while () { + s/\r//g; # turn windows-looking lines into unix-looking lines + $map .= $_; + } + last; + } + + if (/^--- Memory map:/) { + # Read /proc/self/maps data as formatted by DumpAddressMap() + my $buildvar = ""; + while () { + s/\r//g; # turn windows-looking lines into unix-looking lines + # Parse "build=" specification if supplied + if (m/^\s*build=(.*)\n/) { + $buildvar = $1; + } + + # Expand "$build" variable if available + $_ =~ s/\$build\b/$buildvar/g; + + $map .= $_; + } + last; + } + + # Read entry of the form: + # : [: ] @ a1 a2 a3 ... an + s/^\s*//; + s/\s*$//; + if (m/^\s*(\d+):\s+(\d+)\s+\[\s*(\d+):\s+(\d+)\]\s+@\s+(.*)$/) { + my $stack = $5; + my ($n1, $s1, $n2, $s2) = ($1, $2, $3, $4); + + if ($sample_adjustment) { + if ($sampling_algorithm == 2) { + # Remote-heap version 2 + # The sampling frequency is the rate of a Poisson process. + # This means that the probability of sampling an allocation of + # size X with sampling rate Y is 1 - exp(-X/Y) + my $ratio; + $ratio = (($s1*1.0)/$n1)/($sample_adjustment); + my $scale_factor; + $scale_factor = 1/(1 - exp(-$ratio)); + $n1 *= $scale_factor; + $s1 *= $scale_factor; + $ratio = (($s2*1.0)/$n2)/($sample_adjustment); + $scale_factor = 1/(1 - exp(-$ratio)); + $n2 *= $scale_factor; + $s2 *= $scale_factor; + } else { + # Remote-heap version 1 + my $ratio; + $ratio = (($s1*1.0)/$n1)/($sample_adjustment); + if ($ratio < 1) { + $n1 /= $ratio; + $s1 /= $ratio; + } + $ratio = (($s2*1.0)/$n2)/($sample_adjustment); + if ($ratio < 1) { + $n2 /= $ratio; + $s2 /= $ratio; + } + } + } + + my @counts = ($n1, $s1, $n2, $s2); + AddEntries($profile, $pcs, FixCallerAddresses($stack), $counts[$index]); + } + } + + my $r = {}; + $r->{version} = "heap"; + $r->{period} = 1; + $r->{profile} = $profile; + $r->{libs} = ParseLibraries($prog, $map, $pcs); + $r->{pcs} = $pcs; + return $r; +} + +sub ReadSynchProfile { + my ($prog, $fname, $header) = @_; + + my $map = ''; + my $profile = {}; + my $pcs = {}; + my $sampling_period = 1; + my $cyclespernanosec = 2.8; # Default assumption for old binaries + my $seen_clockrate = 0; + my $line; + + my $index = 0; + if ($main::opt_total_delay) { + $index = 0; + } elsif ($main::opt_contentions) { + $index = 1; + } elsif ($main::opt_mean_delay) { + $index = 2; + } + + while ( $line = ) { + $line =~ s/\r//g; # turn windows-looking lines into unix-looking lines + if ( $line =~ /^\s*(\d+)\s+(\d+) \@\s*(.*?)\s*$/ ) { + my ($cycles, $count, $stack) = ($1, $2, $3); + + # Convert cycles to nanoseconds + $cycles /= $cyclespernanosec; + + # Adjust for sampling done by application + $cycles *= $sampling_period; + $count *= $sampling_period; + + my @values = ($cycles, $count, $cycles / $count); + AddEntries($profile, $pcs, FixCallerAddresses($stack), $values[$index]); + + } elsif ( $line =~ /^(slow release).*thread \d+ \@\s*(.*?)\s*$/ || + $line =~ /^\s*(\d+) \@\s*(.*?)\s*$/ ) { + my ($cycles, $stack) = ($1, $2); + if ($cycles !~ /^\d+$/) { + next; + } + + # Convert cycles to nanoseconds + $cycles /= $cyclespernanosec; + + # Adjust for sampling done by application + $cycles *= $sampling_period; + + AddEntries($profile, $pcs, FixCallerAddresses($stack), $cycles); + + } elsif ( $line =~ m/^([a-z][^=]*)=(.*)$/ ) { + my ($variable, $value) = ($1,$2); + for ($variable, $value) { + s/^\s+//; + s/\s+$//; + } + if ($variable eq "cycles/second") { + $cyclespernanosec = $value / 1e9; + $seen_clockrate = 1; + } elsif ($variable eq "sampling period") { + $sampling_period = $value; + } elsif ($variable eq "ms since reset") { + # Currently nothing is done with this value in pprof + # So we just silently ignore it for now + } elsif ($variable eq "discarded samples") { + # Currently nothing is done with this value in pprof + # So we just silently ignore it for now + } else { + printf STDERR ("Ignoring unnknown variable in /contention output: " . + "'%s' = '%s'\n",$variable,$value); + } + } else { + # Memory map entry + $map .= $line; + } + } + close PROFILE; + + if (!$seen_clockrate) { + printf STDERR ("No cycles/second entry in profile; Guessing %.1f GHz\n", + $cyclespernanosec); + } + + my $r = {}; + $r->{version} = 0; + $r->{period} = $sampling_period; + $r->{profile} = $profile; + $r->{libs} = ParseLibraries($prog, $map, $pcs); + $r->{pcs} = $pcs; + return $r; +} + +# Given a hex value in the form "0x1abcd" return "0001abcd" or +# "000000000001abcd", depending on the current address length. +# There's probably a more idiomatic (or faster) way to do this... +sub HexExtend { + my $addr = shift; + + $addr =~ s/^0x//; + + if (length $addr > $address_length) { + printf STDERR "Warning: address $addr is longer than address length $address_length\n"; + } + + return substr("000000000000000".$addr, -$address_length); +} + +##### Symbol extraction ##### + +# Aggressively search the lib_prefix values for the given library +# If all else fails, just return the name of the library unmodified. +# If the lib_prefix is "/my/path,/other/path" and $file is "/lib/dir/mylib.so" +# it will search the following locations in this order, until it finds a file: +# /my/path/lib/dir/mylib.so +# /other/path/lib/dir/mylib.so +# /my/path/dir/mylib.so +# /other/path/dir/mylib.so +# /my/path/mylib.so +# /other/path/mylib.so +# /lib/dir/mylib.so (returned as last resort) +sub FindLibrary { + my $file = shift; + my $suffix = $file; + + # Search for the library as described above + do { + foreach my $prefix (@prefix_list) { + my $fullpath = $prefix . $suffix; + if (-e $fullpath) { + return $fullpath; + } + } + } while ($suffix =~ s|^/[^/]+/|/|); + return $file; +} + +# Return path to library with debugging symbols. +# For libc libraries, the copy in /usr/lib/debug contains debugging symbols +sub DebuggingLibrary { + my $file = shift; + if ($file =~ m|^/| && -f "/usr/lib/debug$file") { + return "/usr/lib/debug$file"; + } + return undef; +} + +# Parse text section header of a library using objdump +sub ParseTextSectionHeaderFromObjdump { + my $lib = shift; + + my $size = undef; + my $vma; + my $file_offset; + # Get objdump output from the library file to figure out how to + # map between mapped addresses and addresses in the library. + my $objdump = $obj_tool_map{"objdump"}; + open(OBJDUMP, "$objdump -h $lib |") + || error("$objdump $lib: $!\n"); + while () { + s/\r//g; # turn windows-looking lines into unix-looking lines + # Idx Name Size VMA LMA File off Algn + # 10 .text 00104b2c 420156f0 420156f0 000156f0 2**4 + # For 64-bit objects, VMA and LMA will be 16 hex digits, size and file + # offset may still be 8. But AddressSub below will still handle that. + my @x = split; + if (($#x >= 6) && ($x[1] eq '.text')) { + $size = $x[2]; + $vma = $x[3]; + $file_offset = $x[5]; + last; + } + } + close(OBJDUMP); + + if (!defined($size)) { + return undef; + } + + my $r = {}; + $r->{size} = $size; + $r->{vma} = $vma; + $r->{file_offset} = $file_offset; + + return $r; +} + +# Parse text section header of a library using otool (on OS X) +sub ParseTextSectionHeaderFromOtool { + my $lib = shift; + + my $size = undef; + my $vma = undef; + my $file_offset = undef; + # Get otool output from the library file to figure out how to + # map between mapped addresses and addresses in the library. + my $otool = $obj_tool_map{"otool"}; + open(OTOOL, "$otool -l $lib |") + || error("$otool $lib: $!\n"); + my $cmd = ""; + my $sectname = ""; + my $segname = ""; + foreach my $line () { + $line =~ s/\r//g; # turn windows-looking lines into unix-looking lines + # Load command <#> + # cmd LC_SEGMENT + # [...] + # Section + # sectname __text + # segname __TEXT + # addr 0x000009f8 + # size 0x00018b9e + # offset 2552 + # align 2^2 (4) + # We will need to strip off the leading 0x from the hex addresses, + # and convert the offset into hex. + if ($line =~ /Load command/) { + $cmd = ""; + $sectname = ""; + $segname = ""; + } elsif ($line =~ /Section/) { + $sectname = ""; + $segname = ""; + } elsif ($line =~ /cmd (\w+)/) { + $cmd = $1; + } elsif ($line =~ /sectname (\w+)/) { + $sectname = $1; + } elsif ($line =~ /segname (\w+)/) { + $segname = $1; + } elsif (!(($cmd eq "LC_SEGMENT" || $cmd eq "LC_SEGMENT_64") && + $sectname eq "__text" && + $segname eq "__TEXT")) { + next; + } elsif ($line =~ /\baddr 0x([0-9a-fA-F]+)/) { + $vma = $1; + } elsif ($line =~ /\bsize 0x([0-9a-fA-F]+)/) { + $size = $1; + } elsif ($line =~ /\boffset ([0-9]+)/) { + $file_offset = sprintf("%016x", $1); + } + if (defined($vma) && defined($size) && defined($file_offset)) { + last; + } + } + close(OTOOL); + + if (!defined($vma) || !defined($size) || !defined($file_offset)) { + return undef; + } + + my $r = {}; + $r->{size} = $size; + $r->{vma} = $vma; + $r->{file_offset} = $file_offset; + + return $r; +} + +sub ParseTextSectionHeader { + # obj_tool_map("otool") is only defined if we're in a Mach-O environment + if (defined($obj_tool_map{"otool"})) { + my $r = ParseTextSectionHeaderFromOtool(@_); + if (defined($r)){ + return $r; + } + } + # If otool doesn't work, or we don't have it, fall back to objdump + return ParseTextSectionHeaderFromObjdump(@_); +} + +# Split /proc/pid/maps dump into a list of libraries +sub ParseLibraries { + return if $main::use_symbol_page; # We don't need libraries info. + my $prog = shift; + my $map = shift; + my $pcs = shift; + + my $result = []; + my $h = "[a-f0-9]+"; + my $zero_offset = HexExtend("0"); + + my $buildvar = ""; + foreach my $l (split("\n", $map)) { + if ($l =~ m/^\s*build=(.*)$/) { + $buildvar = $1; + } + + my $start; + my $finish; + my $offset; + my $lib; + if ($l =~ /^($h)-($h)\s+..x.\s+($h)\s+\S+:\S+\s+\d+\s+(\S+\.(so|dll|dylib|bundle)((\.\d+)+\w*(\.\d+){0,3})?)$/i) { + # Full line from /proc/self/maps. Example: + # 40000000-40015000 r-xp 00000000 03:01 12845071 /lib/ld-2.3.2.so + $start = HexExtend($1); + $finish = HexExtend($2); + $offset = HexExtend($3); + $lib = $4; + $lib =~ s|\\|/|g; # turn windows-style paths into unix-style paths + } elsif ($l =~ /^\s*($h)-($h):\s*(\S+\.so(\.\d+)*)/) { + # Cooked line from DumpAddressMap. Example: + # 40000000-40015000: /lib/ld-2.3.2.so + $start = HexExtend($1); + $finish = HexExtend($2); + $offset = $zero_offset; + $lib = $3; + } else { + next; + } + + # Expand "$build" variable if available + $lib =~ s/\$build\b/$buildvar/g; + + $lib = FindLibrary($lib); + + # Check for pre-relocated libraries, which use pre-relocated symbol tables + # and thus require adjusting the offset that we'll use to translate + # VM addresses into symbol table addresses. + # Only do this if we're not going to fetch the symbol table from a + # debugging copy of the library. + if (!DebuggingLibrary($lib)) { + my $text = ParseTextSectionHeader($lib); + if (defined($text)) { + my $vma_offset = AddressSub($text->{vma}, $text->{file_offset}); + $offset = AddressAdd($offset, $vma_offset); + } + } + + push(@{$result}, [$lib, $start, $finish, $offset]); + } + + # Append special entry for additional library (not relocated) + if ($main::opt_lib ne "") { + my $text = ParseTextSectionHeader($main::opt_lib); + if (defined($text)) { + my $start = $text->{vma}; + my $finish = AddressAdd($start, $text->{size}); + + push(@{$result}, [$main::opt_lib, $start, $finish, $start]); + } + } + + # Append special entry for the main program. This covers + # 0..max_pc_value_seen, so that we assume pc values not found in one + # of the library ranges will be treated as coming from the main + # program binary. + my $min_pc = HexExtend("0"); + my $max_pc = $min_pc; # find the maximal PC value in any sample + foreach my $pc (keys(%{$pcs})) { + if (HexExtend($pc) gt $max_pc) { $max_pc = HexExtend($pc); } + } + push(@{$result}, [$prog, $min_pc, $max_pc, $zero_offset]); + + return $result; +} + +# Add two hex addresses of length $address_length. +# Run pprof --test for unit test if this is changed. +sub AddressAdd { + my $addr1 = shift; + my $addr2 = shift; + my $sum; + + if ($address_length == 8) { + # Perl doesn't cope with wraparound arithmetic, so do it explicitly: + $sum = (hex($addr1)+hex($addr2)) % (0x10000000 * 16); + return sprintf("%08x", $sum); + + } else { + # Do the addition in 7-nibble chunks to trivialize carry handling. + + if ($main::opt_debug and $main::opt_test) { + print STDERR "AddressAdd $addr1 + $addr2 = "; + } + + my $a1 = substr($addr1,-7); + $addr1 = substr($addr1,0,-7); + my $a2 = substr($addr2,-7); + $addr2 = substr($addr2,0,-7); + $sum = hex($a1) + hex($a2); + my $c = 0; + if ($sum > 0xfffffff) { + $c = 1; + $sum -= 0x10000000; + } + my $r = sprintf("%07x", $sum); + + $a1 = substr($addr1,-7); + $addr1 = substr($addr1,0,-7); + $a2 = substr($addr2,-7); + $addr2 = substr($addr2,0,-7); + $sum = hex($a1) + hex($a2) + $c; + $c = 0; + if ($sum > 0xfffffff) { + $c = 1; + $sum -= 0x10000000; + } + $r = sprintf("%07x", $sum) . $r; + + $sum = hex($addr1) + hex($addr2) + $c; + if ($sum > 0xff) { $sum -= 0x100; } + $r = sprintf("%02x", $sum) . $r; + + if ($main::opt_debug and $main::opt_test) { print STDERR "$r\n"; } + + return $r; + } +} + + +# Subtract two hex addresses of length $address_length. +# Run pprof --test for unit test if this is changed. +sub AddressSub { + my $addr1 = shift; + my $addr2 = shift; + my $diff; + + if ($address_length == 8) { + # Perl doesn't cope with wraparound arithmetic, so do it explicitly: + $diff = (hex($addr1)-hex($addr2)) % (0x10000000 * 16); + return sprintf("%08x", $diff); + + } else { + # Do the addition in 7-nibble chunks to trivialize borrow handling. + # if ($main::opt_debug) { print STDERR "AddressSub $addr1 - $addr2 = "; } + + my $a1 = hex(substr($addr1,-7)); + $addr1 = substr($addr1,0,-7); + my $a2 = hex(substr($addr2,-7)); + $addr2 = substr($addr2,0,-7); + my $b = 0; + if ($a2 > $a1) { + $b = 1; + $a1 += 0x10000000; + } + $diff = $a1 - $a2; + my $r = sprintf("%07x", $diff); + + $a1 = hex(substr($addr1,-7)); + $addr1 = substr($addr1,0,-7); + $a2 = hex(substr($addr2,-7)) + $b; + $addr2 = substr($addr2,0,-7); + $b = 0; + if ($a2 > $a1) { + $b = 1; + $a1 += 0x10000000; + } + $diff = $a1 - $a2; + $r = sprintf("%07x", $diff) . $r; + + $a1 = hex($addr1); + $a2 = hex($addr2) + $b; + if ($a2 > $a1) { $a1 += 0x100; } + $diff = $a1 - $a2; + $r = sprintf("%02x", $diff) . $r; + + # if ($main::opt_debug) { print STDERR "$r\n"; } + + return $r; + } +} + +# Increment a hex addresses of length $address_length. +# Run pprof --test for unit test if this is changed. +sub AddressInc { + my $addr = shift; + my $sum; + + if ($address_length == 8) { + # Perl doesn't cope with wraparound arithmetic, so do it explicitly: + $sum = (hex($addr)+1) % (0x10000000 * 16); + return sprintf("%08x", $sum); + + } else { + # Do the addition in 7-nibble chunks to trivialize carry handling. + # We are always doing this to step through the addresses in a function, + # and will almost never overflow the first chunk, so we check for this + # case and exit early. + + # if ($main::opt_debug) { print STDERR "AddressInc $addr1 = "; } + + my $a1 = substr($addr,-7); + $addr = substr($addr,0,-7); + $sum = hex($a1) + 1; + my $r = sprintf("%07x", $sum); + if ($sum <= 0xfffffff) { + $r = $addr . $r; + # if ($main::opt_debug) { print STDERR "$r\n"; } + return HexExtend($r); + } else { + $r = "0000000"; + } + + $a1 = substr($addr,-7); + $addr = substr($addr,0,-7); + $sum = hex($a1) + 1; + $r = sprintf("%07x", $sum) . $r; + if ($sum <= 0xfffffff) { + $r = $addr . $r; + # if ($main::opt_debug) { print STDERR "$r\n"; } + return HexExtend($r); + } else { + $r = "00000000000000"; + } + + $sum = hex($addr) + 1; + if ($sum > 0xff) { $sum -= 0x100; } + $r = sprintf("%02x", $sum) . $r; + + # if ($main::opt_debug) { print STDERR "$r\n"; } + return $r; + } +} + +# Extract symbols for all PC values found in profile +sub ExtractSymbols { + my $libs = shift; + my $pcset = shift; + + my $symbols = {}; + + # Map each PC value to the containing library + my %seen = (); + foreach my $lib (@{$libs}) { + my $libname = $lib->[0]; + my $start = $lib->[1]; + my $finish = $lib->[2]; + my $offset = $lib->[3]; + + # Get list of pcs that belong in this library. + my $contained = []; + foreach my $pc (keys(%{$pcset})) { + if (!$seen{$pc} && ($pc ge $start) && ($pc le $finish)) { + $seen{$pc} = 1; + push(@{$contained}, $pc); + } + } + # Map to symbols + MapToSymbols($libname, AddressSub($start, $offset), $contained, $symbols); + } + + return $symbols; +} + +# Map list of PC values to symbols for a given image +sub MapToSymbols { + my $image = shift; + my $offset = shift; + my $pclist = shift; + my $symbols = shift; + + my $debug = 0; + + # Ignore empty binaries + if ($#{$pclist} < 0) { return; } + + # Figure out the addr2line command to use + my $addr2line = $obj_tool_map{"addr2line"}; + my $cmd = "$addr2line -f -C -e $image"; + if (exists $obj_tool_map{"addr2line_pdb"}) { + $addr2line = $obj_tool_map{"addr2line_pdb"}; + $cmd = "$addr2line --demangle -f -C -e $image"; + } + + # If "addr2line" isn't installed on the system at all, just use + # nm to get what info we can (function names, but not line numbers). + if (system("$addr2line --help >/dev/null 2>&1") != 0) { + MapSymbolsWithNM($image, $offset, $pclist, $symbols); + return; + } + + # "addr2line -i" can produce a variable number of lines per input + # address, with no separator that allows us to tell when data for + # the next address starts. So we find the address for a special + # symbol (_fini) and interleave this address between all real + # addresses passed to addr2line. The name of this special symbol + # can then be used as a separator. + $sep_address = undef; # May be filled in by MapSymbolsWithNM() + my $nm_symbols = {}; + MapSymbolsWithNM($image, $offset, $pclist, $nm_symbols); + # TODO(csilvers): only add '-i' if addr2line supports it. + if (defined($sep_address)) { + # Only add " -i" to addr2line if the binary supports it. + # addr2line --help returns 0, but not if it sees an unknown flag first. + if (system("$cmd -i --help >/dev/null 2>&1") == 0) { + $cmd .= " -i"; + } else { + $sep_address = undef; # no need for sep_address if we don't support -i + } + } + + # Make file with all PC values with intervening 'sep_address' so + # that we can reliably detect the end of inlined function list + open(ADDRESSES, ">$main::tmpfile_sym") || error("$main::tmpfile_sym: $!\n"); + if ($debug) { print("---- $image ---\n"); } + for (my $i = 0; $i <= $#{$pclist}; $i++) { + # addr2line always reads hex addresses, and does not need '0x' prefix. + if ($debug) { printf STDERR ("%s\n", $pclist->[$i]); } + printf ADDRESSES ("%s\n", AddressSub($pclist->[$i], $offset)); + if (defined($sep_address)) { + printf ADDRESSES ("%s\n", $sep_address); + } + } + close(ADDRESSES); + if ($debug) { + print("----\n"); + system("cat $main::tmpfile_sym"); + print("----\n"); + system("$cmd <$main::tmpfile_sym"); + print("----\n"); + } + + open(SYMBOLS, "$cmd <$main::tmpfile_sym |") || error("$cmd: $!\n"); + my $count = 0; # Index in pclist + while () { + # Read fullfunction and filelineinfo from next pair of lines + s/\r?\n$//g; + my $fullfunction = $_; + $_ = ; + s/\r?\n$//g; + my $filelinenum = $_; + + if (defined($sep_address) && $fullfunction eq $sep_symbol) { + # Terminating marker for data for this address + $count++; + next; + } + + $filelinenum =~ s|\\|/|g; # turn windows-style paths into unix-style paths + + my $pcstr = $pclist->[$count]; + my $function = ShortFunctionName($fullfunction); + if ($fullfunction eq '??') { + # See if nm found a symbol + my $nms = $nm_symbols->{$pcstr}; + if (defined($nms)) { + $function = $nms->[0]; + $fullfunction = $nms->[2]; + } + } + + # Prepend to accumulated symbols for pcstr + # (so that caller comes before callee) + my $sym = $symbols->{$pcstr}; + if (!defined($sym)) { + $sym = []; + $symbols->{$pcstr} = $sym; + } + unshift(@{$sym}, $function, $filelinenum, $fullfunction); + if ($debug) { printf STDERR ("%s => [%s]\n", $pcstr, join(" ", @{$sym})); } + if (!defined($sep_address)) { + # Inlining is off, se this entry ends immediately + $count++; + } + } + close(SYMBOLS); +} + +# Use nm to map the list of referenced PCs to symbols. Return true iff we +# are able to read procedure information via nm. +sub MapSymbolsWithNM { + my $image = shift; + my $offset = shift; + my $pclist = shift; + my $symbols = shift; + + # Get nm output sorted by increasing address + my $symbol_table = GetProcedureBoundaries($image, "."); + if (!%{$symbol_table}) { + return 0; + } + # Start addresses are already the right length (8 or 16 hex digits). + my @names = sort { $symbol_table->{$a}->[0] cmp $symbol_table->{$b}->[0] } + keys(%{$symbol_table}); + + if ($#names < 0) { + # No symbols: just use addresses + foreach my $pc (@{$pclist}) { + my $pcstr = "0x" . $pc; + $symbols->{$pc} = [$pcstr, "?", $pcstr]; + } + return 0; + } + + # Sort addresses so we can do a join against nm output + my $index = 0; + my $fullname = $names[0]; + my $name = ShortFunctionName($fullname); + foreach my $pc (sort { $a cmp $b } @{$pclist}) { + # Adjust for mapped offset + my $mpc = AddressSub($pc, $offset); + while (($index < $#names) && ($mpc ge $symbol_table->{$fullname}->[1])){ + $index++; + $fullname = $names[$index]; + $name = ShortFunctionName($fullname); + } + if ($mpc lt $symbol_table->{$fullname}->[1]) { + $symbols->{$pc} = [$name, "?", $fullname]; + } else { + my $pcstr = "0x" . $pc; + $symbols->{$pc} = [$pcstr, "?", $pcstr]; + } + } + return 1; +} + +sub ShortFunctionName { + my $function = shift; + while ($function =~ s/\([^()]*\)(\s*const)?//g) { } # Argument types + while ($function =~ s/<[^<>]*>//g) { } # Remove template arguments + $function =~ s/^.*\s+(\w+::)/$1/; # Remove leading type + return $function; +} + +# Trim overly long symbols found in disassembler output +sub CleanDisassembly { + my $d = shift; + while ($d =~ s/\([^()%]*\)(\s*const)?//g) { } # Argument types, not (%rax) + while ($d =~ s/(\w+)<[^<>]*>/$1/g) { } # Remove template arguments + return $d; +} + +##### Miscellaneous ##### + +# Find the right versions of the above object tools to use. The +# argument is the program file being analyzed, and should be an ELF +# 32-bit or ELF 64-bit executable file. The location of the tools +# is determined by considering the following options in this order: +# 1) --tools option, if set +# 2) PPROF_TOOLS environment variable, if set +# 3) the environment +sub ConfigureObjTools { + my $prog_file = shift; + + # Check for the existence of $prog_file because /usr/bin/file does not + # predictably return error status in prod. + (-e $prog_file) || error("$prog_file does not exist.\n"); + + # Follow symlinks (at least for systems where "file" supports that) + my $file_type = `/usr/bin/file -L $prog_file 2>/dev/null || /usr/bin/file $prog_file`; + if ($file_type =~ /64-bit/) { + # Change $address_length to 16 if the program file is ELF 64-bit. + # We can't detect this from many (most?) heap or lock contention + # profiles, since the actual addresses referenced are generally in low + # memory even for 64-bit programs. + $address_length = 16; + } + + if ($file_type =~ /MS Windows/) { + # For windows, we provide a version of nm and addr2line as part of + # the opensource release, which is capable of parsing + # Windows-style PDB executables. It should live in the path, or + # in the same directory as pprof. + $obj_tool_map{"nm_pdb"} = "nm-pdb"; + $obj_tool_map{"addr2line_pdb"} = "addr2line-pdb"; + } + + if ($file_type =~ /Mach-O/) { + # OS X uses otool to examine Mach-O files, rather than objdump. + $obj_tool_map{"otool"} = "otool"; + $obj_tool_map{"addr2line"} = "false"; # no addr2line + $obj_tool_map{"objdump"} = "false"; # no objdump + } + + # Go fill in %obj_tool_map with the pathnames to use: + foreach my $tool (keys %obj_tool_map) { + $obj_tool_map{$tool} = ConfigureTool($obj_tool_map{$tool}); + } +} + +# Returns the path of a caller-specified object tool. If --tools or +# PPROF_TOOLS are specified, then returns the full path to the tool +# with that prefix. Otherwise, returns the path unmodified (which +# means we will look for it on PATH). +sub ConfigureTool { + my $tool = shift; + my $path; + + if ($main::opt_tools ne "") { + # Use a prefix specified by the --tools option... + $path = $main::opt_tools . $tool; + if (!-x $path) { + error("No '$tool' found with prefix specified by --tools $main::opt_tools\n"); + } + } elsif (exists $ENV{"PPROF_TOOLS"} && + $ENV{"PPROF_TOOLS"} ne "") { + #... or specified with the PPROF_TOOLS environment variable... + $path = $ENV{"PPROF_TOOLS"} . $tool; + if (!-x $path) { + error("No '$tool' found with prefix specified by PPROF_TOOLS=$ENV{PPROF_TOOLS}\n"); + } + } else { + # ... otherwise use the version that exists in the same directory as + # pprof. If there's nothing there, use $PATH. + $0 =~ m,[^/]*$,; # this is everything after the last slash + my $dirname = $`; # this is everything up to and including the last slash + if (-x "$dirname$tool") { + $path = "$dirname$tool"; + } else { + $path = $tool; + } + } + if ($main::opt_debug) { print STDERR "Using '$path' for '$tool'.\n"; } + return $path; +} + +sub cleanup { + unlink($main::tmpfile_sym); + unlink(keys %main::tempnames); + + # We leave any collected profiles in $HOME/pprof in case the user wants + # to look at them later. We print a message informing them of this. + if ((scalar(@main::profile_files) > 0) && + defined($main::collected_profile)) { + if (scalar(@main::profile_files) == 1) { + print STDERR "Dynamically gathered profile is in $main::collected_profile\n"; + } + print STDERR "If you want to investigate this profile further, you can do:\n"; + print STDERR "\n"; + print STDERR " pprof \\\n"; + print STDERR " $main::prog \\\n"; + print STDERR " $main::collected_profile\n"; + print STDERR "\n"; + } +} + +sub sighandler { + cleanup(); + exit(1); +} + +sub error { + my $msg = shift; + print STDERR $msg; + cleanup(); + exit(1); +} + + +# Run $nm_command and get all the resulting procedure boundaries whose +# names match "$regexp" and returns them in a hashtable mapping from +# procedure name to a two-element vector of [start address, end address] +sub GetProcedureBoundariesViaNm { + my $nm_command = shift; + my $regexp = shift; + + my $symbol_table = {}; + open(NM, "$nm_command |") || error("$nm_command: $!\n"); + my $last_start = "0"; + my $routine = ""; + while () { + s/\r//g; # turn windows-looking lines into unix-looking lines + if (m/^\s*([0-9a-f]+) (.) (..*)/) { + my $start_val = $1; + my $type = $2; + my $this_routine = $3; + + # It's possible for two symbols to share the same address, if + # one is a zero-length variable (like __start_google_malloc) or + # one symbol is a weak alias to another (like __libc_malloc). + # In such cases, we want to ignore all values except for the + # actual symbol, which in nm-speak has type "T". The logic + # below does this, though it's a bit tricky: what happens when + # we have a series of lines with the same address, is the first + # one gets queued up to be processed. However, it won't + # *actually* be processed until later, when we read a line with + # a different address. That means that as long as we're reading + # lines with the same address, we have a chance to replace that + # item in the queue, which we do whenever we see a 'T' entry -- + # that is, a line with type 'T'. If we never see a 'T' entry, + # we'll just go ahead and process the first entry (which never + # got touched in the queue), and ignore the others. + if ($start_val eq $last_start && $type =~ /t/i) { + # We are the 'T' symbol at this address, replace previous symbol. + $routine = $this_routine; + next; + } elsif ($start_val eq $last_start) { + # We're not the 'T' symbol at this address, so ignore us. + next; + } + + if ($this_routine eq $sep_symbol) { + $sep_address = HexExtend($start_val); + } + + # Tag this routine with the starting address in case the image + # has multiple occurrences of this routine. We use a syntax + # that resembles template paramters that are automatically + # stripped out by ShortFunctionName() + $this_routine .= "<$start_val>"; + + if (defined($routine) && $routine =~ m/$regexp/) { + $symbol_table->{$routine} = [HexExtend($last_start), + HexExtend($start_val)]; + } + $last_start = $start_val; + $routine = $this_routine; + } elsif (m/^Loaded image name: (.+)/) { + # The win32 nm workalike emits information about the binary it is using. + if ($main::opt_debug) { print STDERR "Using Image $1\n"; } + } elsif (m/^PDB file name: (.+)/) { + # The win32 nm workalike emits information about the pdb it is using. + if ($main::opt_debug) { print STDERR "Using PDB $1\n"; } + } + } + close(NM); + # Handle the last line in the nm output. Unfortunately, we don't know + # how big this last symbol is, because we don't know how big the file + # is. For now, we just give it a size of 0. + # TODO(csilvers): do better here. + if (defined($routine) && $routine =~ m/$regexp/) { + $symbol_table->{$routine} = [HexExtend($last_start), + HexExtend($last_start)]; + } + return $symbol_table; +} + +# Gets the procedure boundaries for all routines in "$image" whose names +# match "$regexp" and returns them in a hashtable mapping from procedure +# name to a two-element vector of [start address, end address]. +# Will return an empty map if nm is not installed or not working properly. +sub GetProcedureBoundaries { + my $image = shift; + my $regexp = shift; + + # For libc libraries, the copy in /usr/lib/debug contains debugging symbols + my $debugging = DebuggingLibrary($image); + if ($debugging) { + $image = $debugging; + } + + my $nm = $obj_tool_map{"nm"}; + my $cppfilt = $obj_tool_map{"c++filt"}; + + # nm can fail for two reasons: 1) $image isn't a debug library; 2) nm + # binary doesn't support --demangle. In addition, for OS X we need + # to use the -f flag to get 'flat' nm output (otherwise we don't sort + # properly and get incorrect results). Unfortunately, GNU nm uses -f + # in an incompatible way. So first we test whether our nm supports + # --demangle and -f. + my $demangle_flag = ""; + my $cppfilt_flag = ""; + if (system("$nm --demangle $image >/dev/null 2>&1") == 0) { + # In this mode, we do "nm --demangle " + $demangle_flag = "--demangle"; + $cppfilt_flag = ""; + } elsif (system("$cppfilt $image >/dev/null 2>&1") == 0) { + # In this mode, we do "nm | c++filt" + $cppfilt_flag = " | $cppfilt"; + }; + my $flatten_flag = ""; + if (system("$nm -f $image >/dev/null 2>&1") == 0) { + $flatten_flag = "-f"; + } + + # Finally, in the case $imagie isn't a debug library, we try again with + # -D to at least get *exported* symbols. If we can't use --demangle, + # we use c++filt instead, if it exists on this system. + my @nm_commands = ("$nm -n $flatten_flag $demangle_flag" . + " $image 2>/dev/null $cppfilt_flag", + "$nm -D -n $flatten_flag $demangle_flag" . + " $image 2>/dev/null $cppfilt_flag", + # 6nm is for Go binaries + "6nm $image 2>/dev/null | sort"); + + # If the executable is an MS Windows PDB-format executable, we'll + # have set up obj_tool_map("nm_pdb"). In this case, we actually + # want to use both unix nm and windows-specific nm_pdb, since + # PDB-format executables can apparently include dwarf .o files. + if (exists $obj_tool_map{"nm_pdb"}) { + my $nm_pdb = $obj_tool_map{"nm_pdb"}; + push(@nm_commands, "$nm_pdb --demangle $image 2>/dev/null"); + } + + foreach my $nm_command (@nm_commands) { + my $symbol_table = GetProcedureBoundariesViaNm($nm_command, $regexp); + return $symbol_table if (%{$symbol_table}); + } + my $symbol_table = {}; + return $symbol_table; +} + + +# The test vectors for AddressAdd/Sub/Inc are 8-16-nibble hex strings. +# To make them more readable, we add underscores at interesting places. +# This routine removes the underscores, producing the canonical representation +# used by pprof to represent addresses, particularly in the tested routines. +sub CanonicalHex { + my $arg = shift; + return join '', (split '_',$arg); +} + + +# Unit test for AddressAdd: +sub AddressAddUnitTest { + my $test_data_8 = shift; + my $test_data_16 = shift; + my $error_count = 0; + my $fail_count = 0; + my $pass_count = 0; + # print STDERR "AddressAddUnitTest: ", 1+$#{$test_data_8}, " tests\n"; + + # First a few 8-nibble addresses. Note that this implementation uses + # plain old arithmetic, so a quick sanity check along with verifying what + # happens to overflow (we want it to wrap): + $address_length = 8; + foreach my $row (@{$test_data_8}) { + if ($main::opt_debug and $main::opt_test) { print STDERR "@{$row}\n"; } + my $sum = AddressAdd ($row->[0], $row->[1]); + if ($sum ne $row->[2]) { + printf STDERR "ERROR: %s != %s + %s = %s\n", $sum, + $row->[0], $row->[1], $row->[2]; + ++$fail_count; + } else { + ++$pass_count; + } + } + printf STDERR "AddressAdd 32-bit tests: %d passes, %d failures\n", + $pass_count, $fail_count; + $error_count = $fail_count; + $fail_count = 0; + $pass_count = 0; + + # Now 16-nibble addresses. + $address_length = 16; + foreach my $row (@{$test_data_16}) { + if ($main::opt_debug and $main::opt_test) { print STDERR "@{$row}\n"; } + my $sum = AddressAdd (CanonicalHex($row->[0]), CanonicalHex($row->[1])); + my $expected = join '', (split '_',$row->[2]); + if ($sum ne CanonicalHex($row->[2])) { + printf STDERR "ERROR: %s != %s + %s = %s\n", $sum, + $row->[0], $row->[1], $row->[2]; + ++$fail_count; + } else { + ++$pass_count; + } + } + printf STDERR "AddressAdd 64-bit tests: %d passes, %d failures\n", + $pass_count, $fail_count; + $error_count += $fail_count; + + return $error_count; +} + + +# Unit test for AddressSub: +sub AddressSubUnitTest { + my $test_data_8 = shift; + my $test_data_16 = shift; + my $error_count = 0; + my $fail_count = 0; + my $pass_count = 0; + # print STDERR "AddressSubUnitTest: ", 1+$#{$test_data_8}, " tests\n"; + + # First a few 8-nibble addresses. Note that this implementation uses + # plain old arithmetic, so a quick sanity check along with verifying what + # happens to overflow (we want it to wrap): + $address_length = 8; + foreach my $row (@{$test_data_8}) { + if ($main::opt_debug and $main::opt_test) { print STDERR "@{$row}\n"; } + my $sum = AddressSub ($row->[0], $row->[1]); + if ($sum ne $row->[3]) { + printf STDERR "ERROR: %s != %s - %s = %s\n", $sum, + $row->[0], $row->[1], $row->[3]; + ++$fail_count; + } else { + ++$pass_count; + } + } + printf STDERR "AddressSub 32-bit tests: %d passes, %d failures\n", + $pass_count, $fail_count; + $error_count = $fail_count; + $fail_count = 0; + $pass_count = 0; + + # Now 16-nibble addresses. + $address_length = 16; + foreach my $row (@{$test_data_16}) { + if ($main::opt_debug and $main::opt_test) { print STDERR "@{$row}\n"; } + my $sum = AddressSub (CanonicalHex($row->[0]), CanonicalHex($row->[1])); + if ($sum ne CanonicalHex($row->[3])) { + printf STDERR "ERROR: %s != %s - %s = %s\n", $sum, + $row->[0], $row->[1], $row->[3]; + ++$fail_count; + } else { + ++$pass_count; + } + } + printf STDERR "AddressSub 64-bit tests: %d passes, %d failures\n", + $pass_count, $fail_count; + $error_count += $fail_count; + + return $error_count; +} + + +# Unit test for AddressInc: +sub AddressIncUnitTest { + my $test_data_8 = shift; + my $test_data_16 = shift; + my $error_count = 0; + my $fail_count = 0; + my $pass_count = 0; + # print STDERR "AddressIncUnitTest: ", 1+$#{$test_data_8}, " tests\n"; + + # First a few 8-nibble addresses. Note that this implementation uses + # plain old arithmetic, so a quick sanity check along with verifying what + # happens to overflow (we want it to wrap): + $address_length = 8; + foreach my $row (@{$test_data_8}) { + if ($main::opt_debug and $main::opt_test) { print STDERR "@{$row}\n"; } + my $sum = AddressInc ($row->[0]); + if ($sum ne $row->[4]) { + printf STDERR "ERROR: %s != %s + 1 = %s\n", $sum, + $row->[0], $row->[4]; + ++$fail_count; + } else { + ++$pass_count; + } + } + printf STDERR "AddressInc 32-bit tests: %d passes, %d failures\n", + $pass_count, $fail_count; + $error_count = $fail_count; + $fail_count = 0; + $pass_count = 0; + + # Now 16-nibble addresses. + $address_length = 16; + foreach my $row (@{$test_data_16}) { + if ($main::opt_debug and $main::opt_test) { print STDERR "@{$row}\n"; } + my $sum = AddressInc (CanonicalHex($row->[0])); + if ($sum ne CanonicalHex($row->[4])) { + printf STDERR "ERROR: %s != %s + 1 = %s\n", $sum, + $row->[0], $row->[4]; + ++$fail_count; + } else { + ++$pass_count; + } + } + printf STDERR "AddressInc 64-bit tests: %d passes, %d failures\n", + $pass_count, $fail_count; + $error_count += $fail_count; + + return $error_count; +} + + +# Driver for unit tests. +# Currently just the address add/subtract/increment routines for 64-bit. +sub RunUnitTests { + my $error_count = 0; + + # This is a list of tuples [a, b, a+b, a-b, a+1] + my $unit_test_data_8 = [ + [qw(aaaaaaaa 50505050 fafafafa 5a5a5a5a aaaaaaab)], + [qw(50505050 aaaaaaaa fafafafa a5a5a5a6 50505051)], + [qw(ffffffff aaaaaaaa aaaaaaa9 55555555 00000000)], + [qw(00000001 ffffffff 00000000 00000002 00000002)], + [qw(00000001 fffffff0 fffffff1 00000011 00000002)], + ]; + my $unit_test_data_16 = [ + # The implementation handles data in 7-nibble chunks, so those are the + # interesting boundaries. + [qw(aaaaaaaa 50505050 + 00_000000f_afafafa 00_0000005_a5a5a5a 00_000000a_aaaaaab)], + [qw(50505050 aaaaaaaa + 00_000000f_afafafa ff_ffffffa_5a5a5a6 00_0000005_0505051)], + [qw(ffffffff aaaaaaaa + 00_000001a_aaaaaa9 00_0000005_5555555 00_0000010_0000000)], + [qw(00000001 ffffffff + 00_0000010_0000000 ff_ffffff0_0000002 00_0000000_0000002)], + [qw(00000001 fffffff0 + 00_000000f_ffffff1 ff_ffffff0_0000011 00_0000000_0000002)], + + [qw(00_a00000a_aaaaaaa 50505050 + 00_a00000f_afafafa 00_a000005_a5a5a5a 00_a00000a_aaaaaab)], + [qw(0f_fff0005_0505050 aaaaaaaa + 0f_fff000f_afafafa 0f_ffefffa_5a5a5a6 0f_fff0005_0505051)], + [qw(00_000000f_fffffff 01_800000a_aaaaaaa + 01_800001a_aaaaaa9 fe_8000005_5555555 00_0000010_0000000)], + [qw(00_0000000_0000001 ff_fffffff_fffffff + 00_0000000_0000000 00_0000000_0000002 00_0000000_0000002)], + [qw(00_0000000_0000001 ff_fffffff_ffffff0 + ff_fffffff_ffffff1 00_0000000_0000011 00_0000000_0000002)], + ]; + + $error_count += AddressAddUnitTest($unit_test_data_8, $unit_test_data_16); + $error_count += AddressSubUnitTest($unit_test_data_8, $unit_test_data_16); + $error_count += AddressIncUnitTest($unit_test_data_8, $unit_test_data_16); + if ($error_count > 0) { + print STDERR $error_count, " errors: FAILED\n"; + } else { + print STDERR "PASS\n"; + } + exit ($error_count); +} diff --git a/src/cmd/prof/main.c b/src/cmd/prof/main.c new file mode 100644 index 000000000..f36759cd3 --- /dev/null +++ b/src/cmd/prof/main.c @@ -0,0 +1,895 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +#include +#include +#include +#include +#include + +#define Ureg Ureg_amd64 + #include +#undef Ureg +#define Ureg Ureg_x86 + #include +#undef Ureg +#include + +char* file = "6.out"; +static Fhdr fhdr; +int have_syms; +int fd; +struct Ureg_amd64 ureg_amd64; +struct Ureg_x86 ureg_x86; +int total_sec = 0; +int delta_msec = 100; +int nsample; +int nsamplethread; + +// pprof data, stored as sequences of N followed by N PC values. +// See http://code.google.com/p/google-perftools . +uvlong *ppdata; // traces +Biobuf* pproffd; // file descriptor to write trace info +long ppstart; // start position of current trace +long nppdata; // length of data +long ppalloc; // size of allocated data +char ppmapdata[10*1024]; // the map information for the output file + +// output formats +int pprof; // print pprof output to named file +int functions; // print functions +int histograms; // print histograms +int linenums; // print file and line numbers rather than function names +int registers; // print registers +int stacks; // print stack traces + +int pid; // main process pid + +int nthread; // number of threads +int thread[32]; // thread pids +Map *map[32]; // thread maps + +void +Usage(void) +{ + fprint(2, "Usage: prof -p pid [-t total_secs] [-d delta_msec]\n"); + fprint(2, " prof [-t total_secs] [-d delta_msec] 6.out args ...\n"); + fprint(2, "\tformats (default -h):\n"); + fprint(2, "\t\t-P file.prof: write [c]pprof output to file.prof\n"); + fprint(2, "\t\t-h: histograms\n"); + fprint(2, "\t\t-f: dynamic functions\n"); + fprint(2, "\t\t-l: dynamic file and line numbers\n"); + fprint(2, "\t\t-r: dynamic registers\n"); + fprint(2, "\t\t-s: dynamic function stack traces\n"); + fprint(2, "\t\t-hs: include stack info in histograms\n"); + exit(2); +} + +typedef struct PC PC; +struct PC { + uvlong pc; + uvlong callerpc; + unsigned int count; + PC* next; +}; + +enum { + Ncounters = 256 +}; + +PC *counters[Ncounters]; + +// Set up by setarch() to make most of the code architecture-independent. +typedef struct Arch Arch; +struct Arch { + char* name; + void (*regprint)(void); + int (*getregs)(Map*); + int (*getPC)(Map*); + int (*getSP)(Map*); + uvlong (*uregPC)(void); + uvlong (*uregSP)(void); + void (*ppword)(uvlong w); +}; + +void +amd64_regprint(void) +{ + fprint(2, "ax\t0x%llux\n", ureg_amd64.ax); + fprint(2, "bx\t0x%llux\n", ureg_amd64.bx); + fprint(2, "cx\t0x%llux\n", ureg_amd64.cx); + fprint(2, "dx\t0x%llux\n", ureg_amd64.dx); + fprint(2, "si\t0x%llux\n", ureg_amd64.si); + fprint(2, "di\t0x%llux\n", ureg_amd64.di); + fprint(2, "bp\t0x%llux\n", ureg_amd64.bp); + fprint(2, "r8\t0x%llux\n", ureg_amd64.r8); + fprint(2, "r9\t0x%llux\n", ureg_amd64.r9); + fprint(2, "r10\t0x%llux\n", ureg_amd64.r10); + fprint(2, "r11\t0x%llux\n", ureg_amd64.r11); + fprint(2, "r12\t0x%llux\n", ureg_amd64.r12); + fprint(2, "r13\t0x%llux\n", ureg_amd64.r13); + fprint(2, "r14\t0x%llux\n", ureg_amd64.r14); + fprint(2, "r15\t0x%llux\n", ureg_amd64.r15); + fprint(2, "ds\t0x%llux\n", ureg_amd64.ds); + fprint(2, "es\t0x%llux\n", ureg_amd64.es); + fprint(2, "fs\t0x%llux\n", ureg_amd64.fs); + fprint(2, "gs\t0x%llux\n", ureg_amd64.gs); + fprint(2, "type\t0x%llux\n", ureg_amd64.type); + fprint(2, "error\t0x%llux\n", ureg_amd64.error); + fprint(2, "pc\t0x%llux\n", ureg_amd64.ip); + fprint(2, "cs\t0x%llux\n", ureg_amd64.cs); + fprint(2, "flags\t0x%llux\n", ureg_amd64.flags); + fprint(2, "sp\t0x%llux\n", ureg_amd64.sp); + fprint(2, "ss\t0x%llux\n", ureg_amd64.ss); +} + +int +amd64_getregs(Map *map) +{ + int i; + union { + uvlong regs[1]; + struct Ureg_amd64 ureg; + } u; + + for(i = 0; i < sizeof ureg_amd64; i+=8) { + if(get8(map, (uvlong)i, &u.regs[i/8]) < 0) + return -1; + } + ureg_amd64 = u.ureg; + return 0; +} + +int +amd64_getPC(Map *map) +{ + uvlong x; + int r; + + r = get8(map, offsetof(struct Ureg_amd64, ip), &x); + ureg_amd64.ip = x; + return r; +} + +int +amd64_getSP(Map *map) +{ + uvlong x; + int r; + + r = get8(map, offsetof(struct Ureg_amd64, sp), &x); + ureg_amd64.sp = x; + return r; +} + +uvlong +amd64_uregPC(void) +{ + return ureg_amd64.ip; +} + +uvlong +amd64_uregSP(void) { + return ureg_amd64.sp; +} + +void +amd64_ppword(uvlong w) +{ + uchar buf[8]; + + buf[0] = w; + buf[1] = w >> 8; + buf[2] = w >> 16; + buf[3] = w >> 24; + buf[4] = w >> 32; + buf[5] = w >> 40; + buf[6] = w >> 48; + buf[7] = w >> 56; + Bwrite(pproffd, buf, 8); +} + +void +x86_regprint(void) +{ + fprint(2, "ax\t0x%ux\n", ureg_x86.ax); + fprint(2, "bx\t0x%ux\n", ureg_x86.bx); + fprint(2, "cx\t0x%ux\n", ureg_x86.cx); + fprint(2, "dx\t0x%ux\n", ureg_x86.dx); + fprint(2, "si\t0x%ux\n", ureg_x86.si); + fprint(2, "di\t0x%ux\n", ureg_x86.di); + fprint(2, "bp\t0x%ux\n", ureg_x86.bp); + fprint(2, "ds\t0x%ux\n", ureg_x86.ds); + fprint(2, "es\t0x%ux\n", ureg_x86.es); + fprint(2, "fs\t0x%ux\n", ureg_x86.fs); + fprint(2, "gs\t0x%ux\n", ureg_x86.gs); + fprint(2, "cs\t0x%ux\n", ureg_x86.cs); + fprint(2, "flags\t0x%ux\n", ureg_x86.flags); + fprint(2, "pc\t0x%ux\n", ureg_x86.pc); + fprint(2, "sp\t0x%ux\n", ureg_x86.sp); + fprint(2, "ss\t0x%ux\n", ureg_x86.ss); +} + +int +x86_getregs(Map *map) +{ + int i; + + for(i = 0; i < sizeof ureg_x86; i+=4) { + if(get4(map, (uvlong)i, &((uint32*)&ureg_x86)[i/4]) < 0) + return -1; + } + return 0; +} + +int +x86_getPC(Map* map) +{ + return get4(map, offsetof(struct Ureg_x86, pc), &ureg_x86.pc); +} + +int +x86_getSP(Map* map) +{ + return get4(map, offsetof(struct Ureg_x86, sp), &ureg_x86.sp); +} + +uvlong +x86_uregPC(void) +{ + return (uvlong)ureg_x86.pc; +} + +uvlong +x86_uregSP(void) +{ + return (uvlong)ureg_x86.sp; +} + +void +x86_ppword(uvlong w) +{ + uchar buf[4]; + + buf[0] = w; + buf[1] = w >> 8; + buf[2] = w >> 16; + buf[3] = w >> 24; + Bwrite(pproffd, buf, 4); +} + +Arch archtab[] = { + { + "amd64", + amd64_regprint, + amd64_getregs, + amd64_getPC, + amd64_getSP, + amd64_uregPC, + amd64_uregSP, + amd64_ppword, + }, + { + "386", + x86_regprint, + x86_getregs, + x86_getPC, + x86_getSP, + x86_uregPC, + x86_uregSP, + x86_ppword, + }, + { + nil + } +}; + +Arch *arch; + +int +setarch(void) +{ + int i; + + if(mach != nil) { + for(i = 0; archtab[i].name != nil; i++) { + if (strcmp(mach->name, archtab[i].name) == 0) { + arch = &archtab[i]; + return 0; + } + } + } + return -1; +} + +int +getthreads(void) +{ + int i, j, curn, found; + Map *curmap[nelem(map)]; + int curthread[nelem(map)]; + static int complained = 0; + + curn = procthreadpids(pid, curthread, nelem(curthread)); + if(curn <= 0) + return curn; + + if(curn > nelem(map)) { + if(complained == 0) { + fprint(2, "prof: too many threads; limiting to %d\n", nthread, nelem(map)); + complained = 1; + } + curn = nelem(map); + } + if(curn == nthread && memcmp(thread, curthread, curn*sizeof(*thread)) == 0) + return curn; // no changes + + // Number of threads has changed (might be the init case). + // A bit expensive but rare enough not to bother being clever. + for(i = 0; i < curn; i++) { + found = 0; + for(j = 0; j < nthread; j++) { + if(curthread[i] == thread[j]) { + found = 1; + curmap[i] = map[j]; + map[j] = nil; + break; + } + } + if(found) + continue; + + // map new thread + curmap[i] = attachproc(curthread[i], &fhdr); + if(curmap[i] == nil) { + fprint(2, "prof: can't attach to %d: %r\n", curthread[i]); + return -1; + } + } + + for(j = 0; j < nthread; j++) + if(map[j] != nil) + detachproc(map[j]); + + nthread = curn; + memmove(thread, curthread, nthread*sizeof thread[0]); + memmove(map, curmap, sizeof map); + return nthread; +} + +int +sample(Map *map) +{ + static int n; + + n++; + if(registers) { + if(arch->getregs(map) < 0) + goto bad; + } else { + // we need only two registers + if(arch->getPC(map) < 0) + goto bad; + if(arch->getSP(map) < 0) + goto bad; + } + return 1; +bad: + if(n == 1) + fprint(2, "prof: can't read registers: %r\n"); + return 0; +} + +void +addtohistogram(uvlong pc, uvlong callerpc, uvlong sp) +{ + int h; + PC *x; + + h = (pc + callerpc*101) % Ncounters; + for(x = counters[h]; x != NULL; x = x->next) { + if(x->pc == pc && x->callerpc == callerpc) { + x->count++; + return; + } + } + x = malloc(sizeof(PC)); + x->pc = pc; + x->callerpc = callerpc; + x->count = 1; + x->next = counters[h]; + counters[h] = x; +} + +void +addppword(uvlong pc) +{ + if(pc == 0) { + return; + } + if(nppdata == ppalloc) { + ppalloc = (1000+nppdata)*2; + ppdata = realloc(ppdata, ppalloc * sizeof ppdata[0]); + if(ppdata == nil) { + fprint(2, "prof: realloc failed: %r\n"); + exit(2); + } + } + ppdata[nppdata++] = pc; +} + +void +startpptrace() +{ + ppstart = nppdata; + addppword(~0); +} + +void +endpptrace() +{ + ppdata[ppstart] = nppdata-ppstart-1; +} + +uvlong nextpc; + +void +xptrace(Map *map, uvlong pc, uvlong sp, Symbol *sym) +{ + char buf[1024]; + if(sym == nil){ + fprint(2, "syms\n"); + return; + } + if(histograms) + addtohistogram(nextpc, pc, sp); + if(!histograms || stacks > 1 || pprof) { + if(nextpc == 0) + nextpc = sym->value; + if(stacks){ + fprint(2, "%s(", sym->name); + fprint(2, ")"); + if(nextpc != sym->value) + fprint(2, "+%#llux ", nextpc - sym->value); + if(have_syms && linenums && fileline(buf, sizeof buf, pc)) { + fprint(2, " %s", buf); + } + fprint(2, "\n"); + } + if (pprof) { + addppword(nextpc); + } + } + nextpc = pc; +} + +void +stacktracepcsp(Map *map, uvlong pc, uvlong sp) +{ + nextpc = pc; + if(pprof){ + startpptrace(); + } + if(machdata->ctrace==nil) + fprint(2, "no machdata->ctrace\n"); + else if(machdata->ctrace(map, pc, sp, 0, xptrace) <= 0) + fprint(2, "no stack frame: pc=%#p sp=%#p\n", pc, sp); + else { + addtohistogram(nextpc, 0, sp); + if(stacks) + fprint(2, "\n"); + } + if(pprof){ + endpptrace(); + } +} + +void +printpc(Map *map, uvlong pc, uvlong sp) +{ + char buf[1024]; + if(registers) + arch->regprint(); + if(have_syms > 0 && linenums && fileline(buf, sizeof buf, pc)) + fprint(2, "%s\n", buf); + if(have_syms > 0 && functions) { + symoff(buf, sizeof(buf), pc, CANY); + fprint(2, "%s\n", buf); + } + if(stacks || pprof){ + stacktracepcsp(map, pc, sp); + } + else if(histograms){ + addtohistogram(pc, 0, sp); + } +} + +void +ppmaps(void) +{ + int fd, n; + char tmp[100]; + Seg *seg; + + // If it's Linux, the info is in /proc/$pid/maps + snprint(tmp, sizeof tmp, "/proc/%d/maps", pid); + fd = open(tmp, 0); + if(fd >= 0) { + n = read(fd, ppmapdata, sizeof ppmapdata - 1); + close(fd); + if(n < 0) { + fprint(2, "prof: can't read %s: %r\n", tmp); + exit(2); + } + ppmapdata[n] = 0; + return; + } + + // It's probably a mac. Synthesize an entry for the text file. + // The register segment may come first but it has a zero offset, so grab the first non-zero offset segment. + for(n = 0; n < 3; n++){ + seg = &map[0]->seg[n]; + if(seg->b == 0) { + continue; + } + snprint(ppmapdata, sizeof ppmapdata, + "%.16x-%.16x r-xp %d 00:00 34968549 %s\n", + seg->b, seg->e, seg->f, "/home/r/6.out" + ); + return; + } + fprint(2, "prof: no text segment in maps for %s\n", file); + exit(2); +} + +void +samples(void) +{ + int i, pid, msec; + struct timespec req; + int getmaps; + + req.tv_sec = delta_msec/1000; + req.tv_nsec = 1000000*(delta_msec % 1000); + getmaps = 0; + if(pprof) + getmaps= 1; + for(msec = 0; total_sec <= 0 || msec < 1000*total_sec; msec += delta_msec) { + nsample++; + nsamplethread += nthread; + for(i = 0; i < nthread; i++) { + pid = thread[i]; + if(ctlproc(pid, "stop") < 0) + return; + if(!sample(map[i])) { + ctlproc(pid, "start"); + return; + } + printpc(map[i], arch->uregPC(), arch->uregSP()); + ctlproc(pid, "start"); + } + nanosleep(&req, NULL); + getthreads(); + if(nthread == 0) + break; + if(getmaps) { + getmaps = 0; + ppmaps(); + } + } +} + +typedef struct Func Func; +struct Func +{ + Func *next; + Symbol s; + uint onstack; + uint leaf; +}; + +Func *func[257]; +int nfunc; + +Func* +findfunc(uvlong pc) +{ + Func *f; + uint h; + Symbol s; + + if(pc == 0) + return nil; + + if(!findsym(pc, CTEXT, &s)) + return nil; + + h = s.value % nelem(func); + for(f = func[h]; f != NULL; f = f->next) + if(f->s.value == s.value) + return f; + + f = malloc(sizeof *f); + memset(f, 0, sizeof *f); + f->s = s; + f->next = func[h]; + func[h] = f; + nfunc++; + return f; +} + +int +compareleaf(const void *va, const void *vb) +{ + Func *a, *b; + + a = *(Func**)va; + b = *(Func**)vb; + if(a->leaf != b->leaf) + return b->leaf - a->leaf; + if(a->onstack != b->onstack) + return b->onstack - a->onstack; + return strcmp(a->s.name, b->s.name); +} + +void +dumphistogram() +{ + int i, h, n; + PC *x; + Func *f, **ff; + + if(!histograms) + return; + + // assign counts to functions. + for(h = 0; h < Ncounters; h++) { + for(x = counters[h]; x != NULL; x = x->next) { + f = findfunc(x->pc); + if(f) { + f->onstack += x->count; + f->leaf += x->count; + } + f = findfunc(x->callerpc); + if(f) + f->leaf -= x->count; + } + } + + // build array + ff = malloc(nfunc*sizeof ff[0]); + n = 0; + for(h = 0; h < nelem(func); h++) + for(f = func[h]; f != NULL; f = f->next) + ff[n++] = f; + + // sort by leaf counts + qsort(ff, nfunc, sizeof ff[0], compareleaf); + + // print. + fprint(2, "%d samples (avg %.1g threads)\n", nsample, (double)nsamplethread/nsample); + for(i = 0; i < nfunc; i++) { + f = ff[i]; + fprint(2, "%6.2f%%\t", 100.0*(double)f->leaf/nsample); + if(stacks) + fprint(2, "%6.2f%%\t", 100.0*(double)f->onstack/nsample); + fprint(2, "%s\n", f->s.name); + } +} + +typedef struct Trace Trace; +struct Trace { + int count; + int npc; + uvlong *pc; + Trace *next; +}; + +void +dumppprof() +{ + uvlong i, n, *p, *e; + int ntrace; + Trace *trace, *tp, *up, *prev; + + if(!pprof) + return; + e = ppdata + nppdata; + // Create list of traces. First, count the traces + ntrace = 0; + for(p = ppdata; p < e;) { + n = *p++; + p += n; + if(n == 0) + continue; + ntrace++; + } + if(ntrace <= 0) + return; + // Allocate and link the traces together. + trace = malloc(ntrace * sizeof(Trace)); + tp = trace; + for(p = ppdata; p < e;) { + n = *p++; + if(n == 0) + continue; + tp->count = 1; + tp->npc = n; + tp->pc = p; + tp->next = tp+1; + tp++; + p += n; + } + trace[ntrace-1].next = nil; + // Eliminate duplicates. Lousy algorithm, although not as bad as it looks because + // the list collapses fast. + for(tp = trace; tp != nil; tp = tp->next) { + prev = tp; + for(up = tp->next; up != nil; up = up->next) { + if(up->npc == tp->npc && memcmp(up->pc, tp->pc, up->npc*sizeof up->pc[0]) == 0) { + tp->count++; + prev->next = up->next; + } else { + prev = up; + } + } + } + // Write file. + // See http://code.google.com/p/google-perftools/source/browse/trunk/doc/cpuprofile-fileformat.html + // 1) Header + arch->ppword(0); // must be zero + arch->ppword(3); // 3 words follow in header + arch->ppword(0); // must be zero + arch->ppword(delta_msec * 1000); // sampling period in microseconds + arch->ppword(0); // must be zero (padding) + // 2) One record for each trace. + for(tp = trace; tp != nil; tp = tp->next) { + arch->ppword(tp->count); + arch->ppword(tp->npc); + for(i = 0; i < tp->npc; i++) { + arch->ppword(tp->pc[i]); + } + } + // 3) Binary trailer + arch->ppword(0); // must be zero + arch->ppword(1); // must be one + arch->ppword(0); // must be zero + // 4) Mapped objects. + Bwrite(pproffd, ppmapdata, strlen(ppmapdata)); + // 5) That's it. + Bterm(pproffd); +} + +int +startprocess(char **argv) +{ + int pid; + + if((pid = fork()) == 0) { + pid = getpid(); + if(ctlproc(pid, "hang") < 0){ + fprint(2, "prof: child process could not hang\n"); + exits(0); + } + execv(argv[0], argv); + fprint(2, "prof: could not exec %s: %r\n", argv[0]); + exits(0); + } + + if(pid == -1) { + fprint(2, "prof: could not fork\n"); + exit(1); + } + if(ctlproc(pid, "attached") < 0 || ctlproc(pid, "waitstop") < 0) { + fprint(2, "prof: could not attach to child process: %r\n"); + exit(1); + } + return pid; +} + +void +detach(void) +{ + int i; + + for(i = 0; i < nthread; i++) + detachproc(map[i]); +} + +int +main(int argc, char *argv[]) +{ + int i; + char *ppfile; + + ARGBEGIN{ + case 'P': + pprof =1; + ppfile = EARGF(Usage()); + pproffd = Bopen(ppfile, OWRITE); + if(pproffd == nil) { + fprint(2, "prof: cannot open %s: %r\n", ppfile); + exit(2); + } + break; + case 'd': + delta_msec = atoi(EARGF(Usage())); + break; + case 't': + total_sec = atoi(EARGF(Usage())); + break; + case 'p': + pid = atoi(EARGF(Usage())); + break; + case 'f': + functions = 1; + break; + case 'h': + histograms = 1; + break; + case 'l': + linenums = 1; + break; + case 'r': + registers = 1; + break; + case 's': + stacks++; + break; + default: + Usage(); + }ARGEND + if(pid <= 0 && argc == 0) + Usage(); + if(functions+linenums+registers+stacks+pprof == 0) + histograms = 1; + if(!machbyname("amd64")) { + fprint(2, "prof: no amd64 support\n", pid); + exit(1); + } + if(argc > 0) + file = argv[0]; + else if(pid) { + file = proctextfile(pid); + if (file == NULL) { + fprint(2, "prof: can't find file for pid %d: %r\n", pid); + fprint(2, "prof: on Darwin, need to provide file name explicitly\n"); + exit(1); + } + } + fd = open(file, 0); + if(fd < 0) { + fprint(2, "prof: can't open %s: %r\n", file); + exit(1); + } + if(crackhdr(fd, &fhdr)) { + have_syms = syminit(fd, &fhdr); + if(!have_syms) { + fprint(2, "prof: no symbols for %s: %r\n", file); + } + } else { + fprint(2, "prof: crack header for %s: %r\n", file); + exit(1); + } + if(pid <= 0) + pid = startprocess(argv); + attachproc(pid, &fhdr); // initializes thread list + if(setarch() < 0) { + detach(); + fprint(2, "prof: can't identify binary architecture for pid %d\n", pid); + exit(1); + } + if(getthreads() <= 0) { + detach(); + fprint(2, "prof: can't find threads for pid %d\n", pid); + exit(1); + } + for(i = 0; i < nthread; i++) + ctlproc(thread[i], "start"); + samples(); + detach(); + dumphistogram(); + dumppprof(); + exit(0); +} diff --git a/src/env.bash b/src/env.bash new file mode 100644 index 000000000..de446bf47 --- /dev/null +++ b/src/env.bash @@ -0,0 +1,106 @@ +#!/usr/bin/env bash +# Copyright 2009 The Go Authors. All rights reserved. +# Use of this source code is governed by a BSD-style +# license that can be found in the LICENSE file. + +# If set to a Windows-style path convert to an MSYS-Unix +# one using the built-in shell commands. +if [[ "$GOROOT" == *:* ]]; then + GOROOT=$(cd "$GOROOT"; pwd) +fi + +if [[ "$GOBIN" == *:* ]]; then + GOBIN=$(cd "$GOBIN"; pwd) +fi + +export GOROOT=${GOROOT:-$(cd ..; pwd)} + +if ! test -f "$GOROOT"/include/u.h +then + echo '$GOROOT is not set correctly or not exported: '$GOROOT 1>&2 + exit 1 +fi + +# Double-check that we're in $GOROOT, for people with multiple Go trees. +# Various aspects of the build cd into $GOROOT-rooted paths, +# making it easy to jump to a different tree and get confused. +DIR1=$(cd ..; pwd) +DIR2=$(cd "$GOROOT"; pwd) +if [ "$DIR1" != "$DIR2" ]; then + echo 'Suspicious $GOROOT '"$GOROOT"': does not match current directory.' 1>&2 + exit 1 +fi + +export GOBIN=${GOBIN:-"$GOROOT/bin"} +if [ ! -d "$GOBIN" -a "$GOBIN" != "$GOROOT/bin" ]; then + echo '$GOBIN is not a directory or does not exist' 1>&2 + echo 'create it or set $GOBIN differently' 1>&2 + exit 1 +fi + +export OLDPATH=$PATH +export PATH="$GOBIN":$PATH + +MAKE=make +if ! make --version 2>/dev/null | grep 'GNU Make' >/dev/null; then + MAKE=gmake +fi + +PROGS=" + ar + awk + bash + bison + chmod + cp + cut + echo + egrep + gcc + grep + ls + mkdir + mv + pwd + rm + sed + sort + tee + touch + tr + true + uname + uniq +" + +for i in $PROGS; do + if ! which $i >/dev/null 2>&1; then + echo "Cannot find '$i' on search path." 1>&2 + echo "See http://golang.org/doc/install.html#ctools" 1>&2 + exit 1 + fi +done + +if bison --version 2>&1 | grep 'bison++' >/dev/null 2>&1; then + echo "Your system's 'bison' is bison++." + echo "Go needs the original bison instead." 1>&2 + echo "See http://golang.org/doc/install.html#ctools" 1>&2 + exit 1 +fi + +# Issue 2020: some users configure bash to default to +# set -o noclobber +# which makes >x fail if x already exists. Restore sanity. +set +o noclobber + +# Tried to use . <($MAKE ...) here, but it cannot set environment +# variables in the version of bash that ships with OS X. Amazing. +eval $($MAKE --no-print-directory -f Make.inc go-env | egrep 'GOARCH|GOOS|GOHOSTARCH|GOHOSTOS|GO_ENV') + +# Shell doesn't tell us whether make succeeded, +# so Make.inc generates a fake variable name. +if [ "$MAKE_GO_ENV_WORKED" != 1 ]; then + echo 'Did not find Go environment variables.' 1>&2 + exit 1 +fi +unset MAKE_GO_ENV_WORKED diff --git a/src/lib9/Makefile b/src/lib9/Makefile new file mode 100644 index 000000000..28c97c9b4 --- /dev/null +++ b/src/lib9/Makefile @@ -0,0 +1,121 @@ +# Copyright 2009 The Go Authors. All rights reserved. +# Use of this source code is governed by a BSD-style +# license that can be found in the LICENSE file. + +include ../Make.inc +O:=$(HOST_O) + +LIB=lib9.a + +NUM=\ + charstod.$O\ + pow10.$O\ + +# Could add fmt/errfmt, but we want to pick it up from ./errstr.c instead. +FMTOFILES=\ + dofmt.$O\ + fltfmt.$O\ + fmt.$O\ + fmtfd.$O\ + fmtfdflush.$O\ + fmtlocale.$O\ + fmtlock2.$O\ + fmtnull.$O\ + fmtprint.$O\ + fmtquote.$O\ + fmtrune.$O\ + fmtstr.$O\ + fmtvprint.$O\ + fprint.$O\ + nan64.$O\ + print.$O\ + seprint.$O\ + smprint.$O\ + snprint.$O\ + sprint.$O\ + strtod.$O\ + vfprint.$O\ + vseprint.$O\ + vsmprint.$O\ + vsnprint.$O\ + $(NUM)\ + +UTFOFILES=\ + rune.$O\ + utfecpy.$O\ + utflen.$O\ + utfnlen.$O\ + utfrrune.$O\ + utfrune.$O\ + utfutf.$O\ + runetype.$O\ + +LIB9OFILES=\ + _p9dir.$O\ + _exits.$O\ + argv0.$O\ + atoi.$O\ + cleanname.$O\ + create.$O\ + dirfstat.$O\ + dirfwstat.$O\ + dirstat.$O\ + dirwstat.$O\ + dup.$O\ + errstr.$O\ + exec.$O\ + execl.$O\ + exitcode.$O\ + exits.$O\ + getenv.$O\ + getfields.$O\ + getwd.$O\ + goos.$O\ + main.$O\ + nan.$O\ + nulldir.$O\ + open.$O\ + readn.$O\ + seek.$O\ + strecpy.$O\ + sysfatal.$O\ + time.$O\ + tokenize.$O\ + +ifeq ($(GOHOSTOS),windows) +LIB9OFILES+=\ + win32.$O\ + +else +LIB9OFILES+=\ + await.$O\ + getuser.$O\ + jmp.$O\ + notify.$O\ + rfork.$O\ + +endif + +OFILES=\ + $(LIB9OFILES)\ + $(FMTOFILES)\ + $(UTFOFILES)\ + +HFILES=\ + $(QUOTED_GOROOT)/include/u.h\ + $(QUOTED_GOROOT)/include/libc.h\ + +include ../Make.clib + +GOROOT_FINAL?=$(GOROOT) + +%.$O: fmt/%.c + $(HOST_CC) -c $(HOST_CFLAGS) -DPLAN9PORT -Ifmt $< + +%.$O: utf/%.c + $(HOST_CC) -c $(HOST_CFLAGS) $< + +goos.$O: goos.c + GOVERSION=`../version.bash` && \ + $(HOST_CC) -c $(HOST_CFLAGS) -DGOOS='"$(GOOS)"' -DGOARCH='"$(GOARCH)"' -DGOROOT='"$(GOROOT_FINAL)"' -DGOVERSION='"'"$$GOVERSION"'"' $< + diff --git a/src/lib9/_exits.c b/src/lib9/_exits.c new file mode 100644 index 000000000..ea8ea74e2 --- /dev/null +++ b/src/lib9/_exits.c @@ -0,0 +1,35 @@ +/* +Plan 9 from User Space src/lib9/_exits.c +http://code.swtch.com/plan9port/src/tip/src/lib9/_exits.c + +Copyright 2001-2007 Russ Cox. All Rights Reserved. + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. +*/ + +#include +#include + +void +_exits(char *s) +{ + if(s == 0 || *s == 0) + _exit(0); + _exit(exitcode(s)); +} diff --git a/src/lib9/_p9dir.c b/src/lib9/_p9dir.c new file mode 100644 index 000000000..58c0822a4 --- /dev/null +++ b/src/lib9/_p9dir.c @@ -0,0 +1,179 @@ +/* +Plan 9 from User Space src/lib9/_p9dir.c +http://code.swtch.com/plan9port/src/tip/src/lib9/_p9dir.c + +Copyright 2001-2007 Russ Cox. All Rights Reserved. + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. +*/ + +#include +#define NOPLAN9DEFINES +#include +#include +#include +#include + +/* + * Caching the last group and passwd looked up is + * a significant win (stupidly enough) on most systems. + * It's not safe for threaded programs, but neither is using + * getpwnam in the first place, so I'm not too worried. + */ +int +_p9dir(struct stat *lst, struct stat *st, char *name, Dir *d, char **str, char *estr) +{ + char *s; + char tmp[20]; + int sz, fd; + + fd = -1; + USED(fd); + sz = 0; + if(d) + memset(d, 0, sizeof *d); + + /* name */ + s = strrchr(name, '/'); + if(s) + s++; + if(!s || !*s) + s = name; + if(*s == '/') + s++; + if(*s == 0) + s = "/"; + if(d){ + if(*str + strlen(s)+1 > estr) + d->name = "oops"; + else{ + strcpy(*str, s); + d->name = *str; + *str += strlen(*str)+1; + } + } + sz += strlen(s)+1; + + /* user */ + snprint(tmp, sizeof tmp, "%d", (int)st->st_uid); + s = tmp; + sz += strlen(s)+1; + if(d){ + if(*str+strlen(s)+1 > estr) + d->uid = "oops"; + else{ + strcpy(*str, s); + d->uid = *str; + *str += strlen(*str)+1; + } + } + + /* group */ + snprint(tmp, sizeof tmp, "%d", (int)st->st_gid); + s = tmp; + sz += strlen(s)+1; + if(d){ + if(*str + strlen(s)+1 > estr) + d->gid = "oops"; + else{ + strcpy(*str, s); + d->gid = *str; + *str += strlen(*str)+1; + } + } + + if(d){ + d->type = 'M'; + + d->muid = ""; + d->qid.path = ((uvlong)st->st_dev<<32) | st->st_ino; +#ifdef _HAVESTGEN + d->qid.vers = st->st_gen; +#endif + if(d->qid.vers == 0) + d->qid.vers = st->st_mtime + st->st_ctime; + d->mode = st->st_mode&0777; + d->atime = st->st_atime; + d->mtime = st->st_mtime; + d->length = st->st_size; + + if(S_ISDIR(st->st_mode)){ + d->length = 0; + d->mode |= DMDIR; + d->qid.type = QTDIR; + } +#ifdef S_ISLNK + if(S_ISLNK(lst->st_mode)) /* yes, lst not st */ + d->mode |= DMSYMLINK; +#endif + if(S_ISFIFO(st->st_mode)) + d->mode |= DMNAMEDPIPE; +#ifdef S_ISSOCK + if(S_ISSOCK(st->st_mode)) + d->mode |= DMSOCKET; +#endif + if(S_ISBLK(st->st_mode)){ + d->mode |= DMDEVICE; + d->qid.path = ('b'<<16)|st->st_rdev; + } + if(S_ISCHR(st->st_mode)){ + d->mode |= DMDEVICE; + d->qid.path = ('c'<<16)|st->st_rdev; + } + /* fetch real size for disks */ + if(S_ISBLK(st->st_mode) && (fd = open(name, O_RDONLY)) >= 0){ + d->length = 0; + close(fd); + } +#if defined(DIOCGMEDIASIZE) + if(isdisk(st)){ + int fd; + off_t mediasize; + + if((fd = open(name, O_RDONLY)) >= 0){ + if(ioctl(fd, DIOCGMEDIASIZE, &mediasize) >= 0) + d->length = mediasize; + close(fd); + } + } +#elif defined(_HAVEDISKLABEL) + if(isdisk(st)){ + int fd, n; + struct disklabel lab; + + if((fd = open(name, O_RDONLY)) < 0) + goto nosize; + if(ioctl(fd, DIOCGDINFO, &lab) < 0) + goto nosize; + n = minor(st->st_rdev)&7; + if(n >= lab.d_npartitions) + goto nosize; + + d->length = (vlong)(lab.d_partitions[n].p_size) * lab.d_secsize; + + nosize: + if(fd >= 0) + close(fd); + } +#endif + } + + return sz; +} + diff --git a/src/lib9/argv0.c b/src/lib9/argv0.c new file mode 100644 index 000000000..623985122 --- /dev/null +++ b/src/lib9/argv0.c @@ -0,0 +1,35 @@ +/* +Plan 9 from User Space src/lib9/argv0.c +http://code.swtch.com/plan9port/src/tip/src/lib9/argv0.c + +Copyright 2001-2007 Russ Cox. All Rights Reserved. + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. +*/ + +#include +#include + +char *argv0; + +/* + * Mac OS can't deal with files that only declare data. + * ARGBEGIN mentions this function so that this file gets pulled in. + */ +void __fixargv0(void) { } diff --git a/src/lib9/atoi.c b/src/lib9/atoi.c new file mode 100644 index 000000000..37a178280 --- /dev/null +++ b/src/lib9/atoi.c @@ -0,0 +1,45 @@ +/* +Plan 9 from User Space src/lib9/ato*.c +http://code.swtch.com/plan9port/src/tip/src/lib9/atoi.c + +Copyright 2001-2007 Russ Cox. All Rights Reserved. + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. +*/ + +#include +#include + +int +atoi(char *s) +{ + return strtol(s, 0, 0); +} + +long +atol(char *s) +{ + return strtol(s, 0, 0); +} + +vlong +atoll(char *s) +{ + return strtoll(s, 0, 0); +} diff --git a/src/lib9/await.c b/src/lib9/await.c new file mode 100644 index 000000000..90be598a1 --- /dev/null +++ b/src/lib9/await.c @@ -0,0 +1,179 @@ +/* +Plan 9 from User Space src/lib9/await.c +http://code.swtch.com/plan9port/src/tip/src/lib9/await.c + +Copyright 2001-2007 Russ Cox. All Rights Reserved. +Portions Copyright 2009 The Go Authors. All Rights Reserved. + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. +*/ + +#define NOPLAN9DEFINES +#include +#include + +#include +#include +#include +#include +#include + +#ifndef WCOREDUMP /* not on Mac OS X Tiger */ +#define WCOREDUMP(status) 0 +#endif + +static struct { + int sig; + char *str; +} tab[] = { + SIGHUP, "hangup", + SIGINT, "interrupt", + SIGQUIT, "quit", + SIGILL, "sys: illegal instruction", + SIGTRAP, "sys: breakpoint", + SIGABRT, "sys: abort", +#ifdef SIGEMT + SIGEMT, "sys: emulate instruction executed", +#endif + SIGFPE, "sys: fp: trap", + SIGKILL, "sys: kill", + SIGBUS, "sys: bus error", + SIGSEGV, "sys: segmentation violation", + SIGALRM, "alarm", + SIGTERM, "kill", + SIGURG, "sys: urgent condition on socket", + SIGSTOP, "sys: stop", + SIGTSTP, "sys: tstp", + SIGCONT, "sys: cont", + SIGCHLD, "sys: child", + SIGTTIN, "sys: ttin", + SIGTTOU, "sys: ttou", +#ifdef SIGIO /* not on Mac OS X Tiger */ + SIGIO, "sys: i/o possible on fd", +#endif + SIGXCPU, "sys: cpu time limit exceeded", + SIGXFSZ, "sys: file size limit exceeded", + SIGVTALRM, "sys: virtual time alarm", + SIGPROF, "sys: profiling timer alarm", +#ifdef SIGWINCH /* not on Mac OS X Tiger */ + SIGWINCH, "sys: window size change", +#endif +#ifdef SIGINFO + SIGINFO, "sys: status request", +#endif + SIGUSR1, "sys: usr1", + SIGUSR2, "sys: usr2", + SIGPIPE, "sys: write on closed pipe", +}; + +char* +_p9sigstr(int sig, char *tmp) +{ + int i; + + for(i=0; imsg = (char*)&w[1]; + + for(;;){ + /* On Linux, pid==-1 means anyone; on SunOS, it's pid==0. */ + if(pid4 == -1) + pid = wait3(&status, opt, &ru); + else + pid = wait4(pid4, &status, opt, &ru); + if(pid <= 0) { + free(w); + return nil; + } + u = ru.ru_utime.tv_sec*1000+((ru.ru_utime.tv_usec+500)/1000); + s = ru.ru_stime.tv_sec*1000+((ru.ru_stime.tv_usec+500)/1000); + w->pid = pid; + w->time[0] = u; + w->time[1] = s; + w->time[2] = u+s; + if(WIFEXITED(status)){ + if(status) + sprint(w->msg, "%d", status); + return w; + } + if(WIFSIGNALED(status)){ + cd = WCOREDUMP(status); + sprint(w->msg, "signal: %s", _p9sigstr(WTERMSIG(status), tmp)); + if(cd) + strcat(w->msg, " (core dumped)"); + return w; + } + } +} + +Waitmsg* +p9wait(void) +{ + return _wait(-1, 0); +} + +Waitmsg* +p9waitfor(int pid) +{ + return _wait(pid, 0); +} + +Waitmsg* +p9waitnohang(void) +{ + return _wait(-1, WNOHANG); +} + +int +p9waitpid(void) +{ + int status; + return wait(&status); +} diff --git a/src/lib9/cleanname.c b/src/lib9/cleanname.c new file mode 100644 index 000000000..fee40388f --- /dev/null +++ b/src/lib9/cleanname.c @@ -0,0 +1,78 @@ +/* +Inferno libkern/cleanname.c +http://code.google.com/p/inferno-os/source/browse/libkern/cleanname.c + + Copyright © 1994-1999 Lucent Technologies Inc. All rights reserved. + Revisions Copyright © 2000-2007 Vita Nuova Holdings Limited (www.vitanuova.com). All rights reserved. + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. +*/ + +#include +#include + +/* + * In place, rewrite name to compress multiple /, eliminate ., and process .. + */ +#define SEP(x) ((x)=='/' || (x) == 0) +char* +cleanname(char *name) +{ + char *p, *q, *dotdot; + int rooted; + + rooted = name[0] == '/'; + + /* + * invariants: + * p points at beginning of path element we're considering. + * q points just past the last path element we wrote (no slash). + * dotdot points just past the point where .. cannot backtrack + * any further (no slash). + */ + p = q = dotdot = name+rooted; + while(*p) { + if(p[0] == '/') /* null element */ + p++; + else if(p[0] == '.' && SEP(p[1])) + p += 1; /* don't count the separator in case it is nul */ + else if(p[0] == '.' && p[1] == '.' && SEP(p[2])) { + p += 2; + if(q > dotdot) { /* can backtrack */ + while(--q > dotdot && *q != '/') + ; + } else if(!rooted) { /* /.. is / but ./../ is .. */ + if(q != name) + *q++ = '/'; + *q++ = '.'; + *q++ = '.'; + dotdot = q; + } + } else { /* real path element */ + if(q != name+rooted) + *q++ = '/'; + while((*q = *p) != '/' && *q != 0) + p++, q++; + } + } + if(q == name) /* empty string is really ``.'' */ + *q++ = '.'; + *q = '\0'; + return name; +} diff --git a/src/lib9/create.c b/src/lib9/create.c new file mode 100644 index 000000000..d7023aea0 --- /dev/null +++ b/src/lib9/create.c @@ -0,0 +1,83 @@ +/* +Plan 9 from User Space src/lib9/create.c +http://code.swtch.com/plan9port/src/tip/src/lib9/create.c + +Copyright 2001-2007 Russ Cox. All Rights Reserved. + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. +*/ +#define _GNU_SOURCE /* for Linux O_DIRECT */ +#include +#define NOPLAN9DEFINES +#include +#include +#include +#include +#include +#ifndef O_DIRECT +#define O_DIRECT 0 +#endif + +int +p9create(char *path, int mode, ulong perm) +{ + int fd, umode, rclose; + + rclose = mode&ORCLOSE; + mode &= ~ORCLOSE; + + /* XXX should get mode mask right? */ + fd = -1; + if(perm&DMDIR){ + if(mode != OREAD){ + werrstr("bad mode in directory create"); + goto out; + } + if(mkdir(path, perm&0777) < 0) + goto out; + fd = open(path, O_RDONLY); + }else{ + umode = (mode&3)|O_CREAT|O_TRUNC; + mode &= ~(3|OTRUNC); + if(mode&ODIRECT){ + umode |= O_DIRECT; + mode &= ~ODIRECT; + } + if(mode&OEXCL){ + umode |= O_EXCL; + mode &= ~OEXCL; + } + if(mode&OAPPEND){ + umode |= O_APPEND; + mode &= ~OAPPEND; + } + if(mode){ + werrstr("unsupported mode in create"); + goto out; + } + umode |= O_BINARY; + fd = open(path, umode, perm); + } +out: + if(fd >= 0){ + if(rclose) + remove(path); + } + return fd; +} diff --git a/src/lib9/dirfstat.c b/src/lib9/dirfstat.c new file mode 100644 index 000000000..17fe10aee --- /dev/null +++ b/src/lib9/dirfstat.c @@ -0,0 +1,54 @@ +/* +Plan 9 from User Space src/lib9/dirfstat.c +http://code.swtch.com/plan9port/src/tip/src/lib9/dirfstat.c + +Copyright 2001-2007 Russ Cox. All Rights Reserved. + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. +*/ +#include +#define NOPLAN9DEFINES +#include + +#include + +extern int _p9dir(struct stat*, struct stat*, char*, Dir*, char**, char*); + +Dir* +dirfstat(int fd) +{ + struct stat st; + int nstr; + Dir *d; + char *str, tmp[100]; + + if(fstat(fd, &st) < 0) + return nil; + + snprint(tmp, sizeof tmp, "/dev/fd/%d", fd); + nstr = _p9dir(&st, &st, tmp, nil, nil, nil); + d = malloc(sizeof(Dir)+nstr); + if(d == nil) + return nil; + memset(d, 0, sizeof(Dir)+nstr); + str = (char*)&d[1]; + _p9dir(&st, &st, tmp, d, &str, str+nstr); + return d; +} + diff --git a/src/lib9/dirfwstat.c b/src/lib9/dirfwstat.c new file mode 100644 index 000000000..fe9153b9b --- /dev/null +++ b/src/lib9/dirfwstat.c @@ -0,0 +1,80 @@ +/* +Plan 9 from User Space src/lib9/dirfwstat.c +http://code.swtch.com/plan9port/src/tip/src/lib9/dirfwstat.c + +Copyright 2001-2007 Russ Cox. All Rights Reserved. + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. +*/ + +#define NOPLAN9DEFINES +#include +#include +#include +#include + +#if defined(__FreeBSD__) || defined(__APPLE__) || defined(__OpenBSD__) || defined(__linux__) +/* do nothing -- futimes exists and is fine */ + +#elif defined(__SunOS5_9__) +/* use futimesat */ +static int +futimes(int fd, struct timeval *tv) +{ + return futimesat(fd, 0, tv); +} + +#else +/* provide dummy */ +/* rename just in case -- linux provides an unusable one */ +#undef futimes +#define futimes myfutimes +static int +futimes(int fd, struct timeval *tv) +{ + werrstr("futimes not available"); + return -1; +} + +#endif + +int +dirfwstat(int fd, Dir *dir) +{ + int ret; + struct timeval tv[2]; + + ret = 0; +#ifndef _WIN32 + if(~dir->mode != 0){ + if(fchmod(fd, dir->mode) < 0) + ret = -1; + } +#endif + if(~dir->mtime != 0){ + tv[0].tv_sec = dir->mtime; + tv[0].tv_usec = 0; + tv[1].tv_sec = dir->mtime; + tv[1].tv_usec = 0; + if(futimes(fd, tv) < 0) + ret = -1; + } + return ret; +} + diff --git a/src/lib9/dirstat.c b/src/lib9/dirstat.c new file mode 100644 index 000000000..6d804ca7c --- /dev/null +++ b/src/lib9/dirstat.c @@ -0,0 +1,63 @@ +/* +Plan 9 from User Space src/lib9/dirstat.c +http://code.swtch.com/plan9port/src/tip/src/lib9/dirstat.c + +Copyright 2001-2007 Russ Cox. All Rights Reserved. + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. +*/ +#include +#define NOPLAN9DEFINES +#include + +#include + +extern int _p9dir(struct stat*, struct stat*, char*, Dir*, char**, char*); + +Dir* +dirstat(char *file) +{ + struct stat lst; + struct stat st; + int nstr; + Dir *d; + char *str; + +#ifdef _WIN32 + if(stat(file, &st) < 0) + return nil; + lst = st; +#else + if(lstat(file, &lst) < 0) + return nil; + st = lst; + if((lst.st_mode&S_IFMT) == S_IFLNK) + stat(file, &st); +#endif + + nstr = _p9dir(&lst, &st, file, nil, nil, nil); + d = malloc(sizeof(Dir)+nstr); + if(d == nil) + return nil; + memset(d, 0, sizeof(Dir)+nstr); + str = (char*)&d[1]; + _p9dir(&lst, &st, file, d, &str, str+nstr); + return d; +} + diff --git a/src/lib9/dirwstat.c b/src/lib9/dirwstat.c new file mode 100644 index 000000000..2646cba40 --- /dev/null +++ b/src/lib9/dirwstat.c @@ -0,0 +1,43 @@ +/* +Plan 9 from User Space src/lib9/dirwstat.c +http://code.swtch.com/plan9port/src/tip/src/lib9/dirwstat.c + +Copyright 2001-2007 Russ Cox. All Rights Reserved. + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. +*/ +#include +#define NOPLAN9DEFINES +#include +#include +#include + +int +dirwstat(char *file, Dir *dir) +{ + struct utimbuf ub; + + /* BUG handle more */ + if(~dir->mtime == 0) + return 0; + + ub.actime = dir->mtime; + ub.modtime = dir->mtime; + return utime(file, &ub); +} diff --git a/src/lib9/dup.c b/src/lib9/dup.c new file mode 100644 index 000000000..9fdfdb8d1 --- /dev/null +++ b/src/lib9/dup.c @@ -0,0 +1,36 @@ +/* +Plan 9 from User Space src/lib9/dup.c +http://code.swtch.com/plan9port/src/tip/src/lib9/dup.c + +Copyright 2001-2007 Russ Cox. All Rights Reserved. + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. +*/ +#include +#include + +#undef dup + +int +p9dup(int old, int new) +{ + if(new == -1) + return dup(old); + return dup2(old, new); +} diff --git a/src/lib9/errstr.c b/src/lib9/errstr.c new file mode 100644 index 000000000..f42f2b538 --- /dev/null +++ b/src/lib9/errstr.c @@ -0,0 +1,106 @@ +/* +Plan 9 from User Space src/lib9/errstr.c +http://code.swtch.com/plan9port/src/tip/src/lib9/errstr.c + +Copyright 2001-2007 Russ Cox. All Rights Reserved. + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. +*/ + +/* + * We assume there's only one error buffer for the whole system. + * If you use ffork, you need to provide a _syserrstr. Since most + * people will use libthread (which provides a _syserrstr), this is + * okay. + */ + +#include +#include +#include +#include + +enum +{ + EPLAN9 = 0x19283745 +}; + +char *(*_syserrstr)(void); +static char xsyserr[ERRMAX]; +static char* +getsyserr(void) +{ + char *s; + + s = nil; + if(_syserrstr) + s = (*_syserrstr)(); + if(s == nil) + s = xsyserr; + return s; +} + +int +errstr(char *err, uint n) +{ + char tmp[ERRMAX]; + char *syserr; + + strecpy(tmp, tmp+ERRMAX, err); + rerrstr(err, n); + syserr = getsyserr(); + strecpy(syserr, syserr+ERRMAX, tmp); + errno = EPLAN9; + return 0; +} + +void +rerrstr(char *err, uint n) +{ + char *syserr; + + syserr = getsyserr(); + if(errno == EINTR) + strcpy(syserr, "interrupted"); + else if(errno != EPLAN9) + strcpy(syserr, strerror(errno)); + strecpy(err, err+n, syserr); +} + +/* replaces __errfmt in libfmt */ + +int +__errfmt(Fmt *f) +{ + if(errno == EPLAN9) + return fmtstrcpy(f, getsyserr()); + return fmtstrcpy(f, strerror(errno)); +} + +void +werrstr(char *fmt, ...) +{ + va_list arg; + char buf[ERRMAX]; + + va_start(arg, fmt); + vseprint(buf, buf+ERRMAX, fmt, arg); + va_end(arg); + errstr(buf, ERRMAX); +} + diff --git a/src/lib9/exec.c b/src/lib9/exec.c new file mode 100644 index 000000000..f2ad0f9b3 --- /dev/null +++ b/src/lib9/exec.c @@ -0,0 +1,33 @@ +/* +Plan 9 from User Space src/lib9/exec.c +http://code.swtch.com/plan9port/src/tip/src/lib9/exec.c + +Copyright 2001-2007 Russ Cox. All Rights Reserved. + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. +*/ +#include +#include + +int +exec(char *prog, char *argv[]) +{ + /* to mimic plan 9 should be just exec, but execvp is a better fit for unix */ + return execvp(prog, argv); +} diff --git a/src/lib9/execl.c b/src/lib9/execl.c new file mode 100644 index 000000000..9e42ad34b --- /dev/null +++ b/src/lib9/execl.c @@ -0,0 +1,53 @@ +/* +Plan 9 from User Space src/lib9/execl.c +http://code.swtch.com/plan9port/src/tip/src/lib9/execl.c + +Copyright 2001-2007 Russ Cox. All Rights Reserved. + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. +*/ +#include +#include + +int +execl(char *prog, ...) +{ + int i; + va_list arg; + char **argv; + + va_start(arg, prog); + for(i=0; va_arg(arg, char*) != nil; i++) + ; + va_end(arg); + + argv = malloc((i+1)*sizeof(char*)); + if(argv == nil) + return -1; + + va_start(arg, prog); + for(i=0; (argv[i] = va_arg(arg, char*)) != nil; i++) + ; + va_end(arg); + + exec(prog, argv); + free(argv); + return -1; +} + diff --git a/src/lib9/exitcode.c b/src/lib9/exitcode.c new file mode 100644 index 000000000..234492acf --- /dev/null +++ b/src/lib9/exitcode.c @@ -0,0 +1,34 @@ +/* +Plan 9 from User Space src/lib9/exitcode.c +http://code.swtch.com/plan9port/src/tip/src/lib9/exitcode.c + +Copyright 2001-2007 Russ Cox. All Rights Reserved. + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. +*/ + +#include +#include + +int +exitcode(char *s) +{ + return 1; +} + diff --git a/src/lib9/exits.c b/src/lib9/exits.c new file mode 100644 index 000000000..5caef8309 --- /dev/null +++ b/src/lib9/exits.c @@ -0,0 +1,34 @@ +/* +Plan 9 from User Space src/lib9/_exits.c +http://code.swtch.com/plan9port/src/tip/src/lib9/_exits.c + +Copyright 2001-2007 Russ Cox. All Rights Reserved. + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. +*/ +#include +#include + +void +exits(char *s) +{ + if(s == 0 || *s == 0) + exit(0); + exit(exitcode(s)); +} diff --git a/src/lib9/fmt/charstod.c b/src/lib9/fmt/charstod.c new file mode 100644 index 000000000..b8096e8fb --- /dev/null +++ b/src/lib9/fmt/charstod.c @@ -0,0 +1,88 @@ +/* + * The authors of this software are Rob Pike and Ken Thompson, + * with contributions from Mike Burrows and Sean Dorward. + * + * Copyright (c) 2002-2006 by Lucent Technologies. + * Portions Copyright (c) 2004 Google Inc. + * + * Permission to use, copy, modify, and distribute this software for any + * purpose without fee is hereby granted, provided that this entire notice + * is included in all copies of any software which is or includes a copy + * or modification of this software and in all copies of the supporting + * documentation for such software. + * THIS SOFTWARE IS BEING PROVIDED "AS IS", WITHOUT ANY EXPRESS OR IMPLIED + * WARRANTY. IN PARTICULAR, NEITHER THE AUTHORS NOR LUCENT TECHNOLOGIES + * NOR GOOGLE INC MAKE ANY REPRESENTATION OR WARRANTY OF ANY KIND CONCERNING + * THE MERCHANTABILITY OF THIS SOFTWARE OR ITS FITNESS FOR ANY PARTICULAR PURPOSE. + */ + +#include +#include +#include "fmtdef.h" + +/* + * Reads a floating-point number by interpreting successive characters + * returned by (*f)(vp). The last call it makes to f terminates the + * scan, so is not a character in the number. It may therefore be + * necessary to back up the input stream up one byte after calling charstod. + */ + +double +fmtcharstod(int(*f)(void*), void *vp) +{ + double num, dem; + int neg, eneg, dig, exp, c; + + num = 0; + neg = 0; + dig = 0; + exp = 0; + eneg = 0; + + c = (*f)(vp); + while(c == ' ' || c == '\t') + c = (*f)(vp); + if(c == '-' || c == '+'){ + if(c == '-') + neg = 1; + c = (*f)(vp); + } + while(c >= '0' && c <= '9'){ + num = num*10 + c-'0'; + c = (*f)(vp); + } + if(c == '.') + c = (*f)(vp); + while(c >= '0' && c <= '9'){ + num = num*10 + c-'0'; + dig++; + c = (*f)(vp); + } + if(c == 'e' || c == 'E'){ + c = (*f)(vp); + if(c == '-' || c == '+'){ + if(c == '-'){ + dig = -dig; + eneg = 1; + } + c = (*f)(vp); + } + while(c >= '0' && c <= '9'){ + exp = exp*10 + c-'0'; + c = (*f)(vp); + } + } + exp -= dig; + if(exp < 0){ + exp = -exp; + eneg = !eneg; + } + dem = __fmtpow10(exp); + if(eneg) + num /= dem; + else + num *= dem; + if(neg) + return -num; + return num; +} diff --git a/src/lib9/fmt/dofmt.c b/src/lib9/fmt/dofmt.c new file mode 100644 index 000000000..cc6ab9225 --- /dev/null +++ b/src/lib9/fmt/dofmt.c @@ -0,0 +1,630 @@ +/* + * The authors of this software are Rob Pike and Ken Thompson, + * with contributions from Mike Burrows and Sean Dorward. + * + * Copyright (c) 2002-2006 by Lucent Technologies. + * Portions Copyright (c) 2004 Google Inc. + * + * Permission to use, copy, modify, and distribute this software for any + * purpose without fee is hereby granted, provided that this entire notice + * is included in all copies of any software which is or includes a copy + * or modification of this software and in all copies of the supporting + * documentation for such software. + * THIS SOFTWARE IS BEING PROVIDED "AS IS", WITHOUT ANY EXPRESS OR IMPLIED + * WARRANTY. IN PARTICULAR, NEITHER THE AUTHORS NOR LUCENT TECHNOLOGIES + * NOR GOOGLE INC MAKE ANY REPRESENTATION OR WARRANTY OF ANY KIND CONCERNING + * THE MERCHANTABILITY OF THIS SOFTWARE OR ITS FITNESS FOR ANY PARTICULAR PURPOSE. + */ + +#include +#include +#include "fmtdef.h" + +/* format the output into f->to and return the number of characters fmted */ +int +dofmt(Fmt *f, char *fmt) +{ + Rune rune, *rt, *rs; + int r; + char *t, *s; + int n, nfmt; + + nfmt = f->nfmt; + for(;;){ + if(f->runes){ + rt = (Rune*)f->to; + rs = (Rune*)f->stop; + while((r = *(uchar*)fmt) && r != '%'){ + if(r < Runeself) + fmt++; + else{ + fmt += chartorune(&rune, fmt); + r = rune; + } + FMTRCHAR(f, rt, rs, r); + } + fmt++; + f->nfmt += rt - (Rune *)f->to; + f->to = rt; + if(!r) + return f->nfmt - nfmt; + f->stop = rs; + }else{ + t = (char*)f->to; + s = (char*)f->stop; + while((r = *(uchar*)fmt) && r != '%'){ + if(r < Runeself){ + FMTCHAR(f, t, s, r); + fmt++; + }else{ + n = chartorune(&rune, fmt); + if(t + n > s){ + t = (char*)__fmtflush(f, t, n); + if(t != nil) + s = (char*)f->stop; + else + return -1; + } + while(n--) + *t++ = *fmt++; + } + } + fmt++; + f->nfmt += t - (char *)f->to; + f->to = t; + if(!r) + return f->nfmt - nfmt; + f->stop = s; + } + + fmt = (char*)__fmtdispatch(f, fmt, 0); + if(fmt == nil) + return -1; + } +} + +void * +__fmtflush(Fmt *f, void *t, int len) +{ + if(f->runes) + f->nfmt += (Rune*)t - (Rune*)f->to; + else + f->nfmt += (char*)t - (char *)f->to; + f->to = t; + if(f->flush == 0 || (*f->flush)(f) == 0 || (char*)f->to + len > (char*)f->stop){ + f->stop = f->to; + return nil; + } + return f->to; +} + +/* + * put a formatted block of memory sz bytes long of n runes into the output buffer, + * left/right justified in a field of at least f->width characters (if FmtWidth is set) + */ +int +__fmtpad(Fmt *f, int n) +{ + char *t, *s; + int i; + + t = (char*)f->to; + s = (char*)f->stop; + for(i = 0; i < n; i++) + FMTCHAR(f, t, s, ' '); + f->nfmt += t - (char *)f->to; + f->to = t; + return 0; +} + +int +__rfmtpad(Fmt *f, int n) +{ + Rune *t, *s; + int i; + + t = (Rune*)f->to; + s = (Rune*)f->stop; + for(i = 0; i < n; i++) + FMTRCHAR(f, t, s, ' '); + f->nfmt += t - (Rune *)f->to; + f->to = t; + return 0; +} + +int +__fmtcpy(Fmt *f, const void *vm, int n, int sz) +{ + Rune *rt, *rs, r; + char *t, *s, *m, *me; + ulong fl; + int nc, w; + + m = (char*)vm; + me = m + sz; + fl = f->flags; + w = 0; + if(fl & FmtWidth) + w = f->width; + if((fl & FmtPrec) && n > f->prec) + n = f->prec; + if(f->runes){ + if(!(fl & FmtLeft) && __rfmtpad(f, w - n) < 0) + return -1; + rt = (Rune*)f->to; + rs = (Rune*)f->stop; + for(nc = n; nc > 0; nc--){ + r = *(uchar*)m; + if(r < Runeself) + m++; + else if((me - m) >= UTFmax || fullrune(m, me-m)) + m += chartorune(&r, m); + else + break; + FMTRCHAR(f, rt, rs, r); + } + f->nfmt += rt - (Rune *)f->to; + f->to = rt; + if(fl & FmtLeft && __rfmtpad(f, w - n) < 0) + return -1; + }else{ + if(!(fl & FmtLeft) && __fmtpad(f, w - n) < 0) + return -1; + t = (char*)f->to; + s = (char*)f->stop; + for(nc = n; nc > 0; nc--){ + r = *(uchar*)m; + if(r < Runeself) + m++; + else if((me - m) >= UTFmax || fullrune(m, me-m)) + m += chartorune(&r, m); + else + break; + FMTRUNE(f, t, s, r); + } + f->nfmt += t - (char *)f->to; + f->to = t; + if(fl & FmtLeft && __fmtpad(f, w - n) < 0) + return -1; + } + return 0; +} + +int +__fmtrcpy(Fmt *f, const void *vm, int n) +{ + Rune r, *m, *me, *rt, *rs; + char *t, *s; + ulong fl; + int w; + + m = (Rune*)vm; + fl = f->flags; + w = 0; + if(fl & FmtWidth) + w = f->width; + if((fl & FmtPrec) && n > f->prec) + n = f->prec; + if(f->runes){ + if(!(fl & FmtLeft) && __rfmtpad(f, w - n) < 0) + return -1; + rt = (Rune*)f->to; + rs = (Rune*)f->stop; + for(me = m + n; m < me; m++) + FMTRCHAR(f, rt, rs, *m); + f->nfmt += rt - (Rune *)f->to; + f->to = rt; + if(fl & FmtLeft && __rfmtpad(f, w - n) < 0) + return -1; + }else{ + if(!(fl & FmtLeft) && __fmtpad(f, w - n) < 0) + return -1; + t = (char*)f->to; + s = (char*)f->stop; + for(me = m + n; m < me; m++){ + r = *m; + FMTRUNE(f, t, s, r); + } + f->nfmt += t - (char *)f->to; + f->to = t; + if(fl & FmtLeft && __fmtpad(f, w - n) < 0) + return -1; + } + return 0; +} + +/* fmt out one character */ +int +__charfmt(Fmt *f) +{ + char x[1]; + + x[0] = va_arg(f->args, int); + f->prec = 1; + return __fmtcpy(f, (const char*)x, 1, 1); +} + +/* fmt out one rune */ +int +__runefmt(Fmt *f) +{ + Rune x[1]; + + x[0] = va_arg(f->args, int); + return __fmtrcpy(f, (const void*)x, 1); +} + +/* public helper routine: fmt out a null terminated string already in hand */ +int +fmtstrcpy(Fmt *f, char *s) +{ + int i, j; + + if(!s) + return __fmtcpy(f, "", 5, 5); + /* if precision is specified, make sure we don't wander off the end */ + if(f->flags & FmtPrec){ +#ifdef PLAN9PORT + Rune r; + i = 0; + for(j=0; jprec && s[i]; j++) + i += chartorune(&r, s+i); +#else + /* ANSI requires precision in bytes, not Runes */ + for(i=0; iprec; i++) + if(s[i] == 0) + break; + j = utfnlen(s, i); /* won't print partial at end */ +#endif + return __fmtcpy(f, s, j, i); + } + return __fmtcpy(f, s, utflen(s), strlen(s)); +} + +/* fmt out a null terminated utf string */ +int +__strfmt(Fmt *f) +{ + char *s; + + s = va_arg(f->args, char *); + return fmtstrcpy(f, s); +} + +/* public helper routine: fmt out a null terminated rune string already in hand */ +int +fmtrunestrcpy(Fmt *f, Rune *s) +{ + Rune *e; + int n, p; + + if(!s) + return __fmtcpy(f, "", 5, 5); + /* if precision is specified, make sure we don't wander off the end */ + if(f->flags & FmtPrec){ + p = f->prec; + for(n = 0; n < p; n++) + if(s[n] == 0) + break; + }else{ + for(e = s; *e; e++) + ; + n = e - s; + } + return __fmtrcpy(f, s, n); +} + +/* fmt out a null terminated rune string */ +int +__runesfmt(Fmt *f) +{ + Rune *s; + + s = va_arg(f->args, Rune *); + return fmtrunestrcpy(f, s); +} + +/* fmt a % */ +int +__percentfmt(Fmt *f) +{ + Rune x[1]; + + x[0] = f->r; + f->prec = 1; + return __fmtrcpy(f, (const void*)x, 1); +} + +/* fmt an integer */ +int +__ifmt(Fmt *f) +{ + char buf[140], *p, *conv; + /* 140: for 64 bits of binary + 3-byte sep every 4 digits */ + uvlong vu; + ulong u; + int neg, base, i, n, fl, w, isv; + int ndig, len, excess, bytelen; + char *grouping; + char *thousands; + + neg = 0; + fl = f->flags; + isv = 0; + vu = 0; + u = 0; +#ifndef PLAN9PORT + /* + * Unsigned verbs for ANSI C + */ + switch(f->r){ + case 'o': + case 'p': + case 'u': + case 'x': + case 'X': + fl |= FmtUnsigned; + fl &= ~(FmtSign|FmtSpace); + break; + } +#endif + if(f->r == 'p'){ + u = (uintptr)va_arg(f->args, void*); + f->r = 'x'; + fl |= FmtUnsigned; + }else if(fl & FmtVLong){ + isv = 1; + if(fl & FmtUnsigned) + vu = va_arg(f->args, uvlong); + else + vu = va_arg(f->args, vlong); + }else if(fl & FmtLong){ + if(fl & FmtUnsigned) + u = va_arg(f->args, ulong); + else + u = va_arg(f->args, long); + }else if(fl & FmtByte){ + if(fl & FmtUnsigned) + u = (uchar)va_arg(f->args, int); + else + u = (char)va_arg(f->args, int); + }else if(fl & FmtShort){ + if(fl & FmtUnsigned) + u = (ushort)va_arg(f->args, int); + else + u = (short)va_arg(f->args, int); + }else{ + if(fl & FmtUnsigned) + u = va_arg(f->args, uint); + else + u = va_arg(f->args, int); + } + conv = "0123456789abcdef"; + grouping = "\4"; /* for hex, octal etc. (undefined by spec but nice) */ + thousands = f->thousands; + switch(f->r){ + case 'd': + case 'i': + case 'u': + base = 10; + grouping = f->grouping; + break; + case 'X': + conv = "0123456789ABCDEF"; + /* fall through */ + case 'x': + base = 16; + thousands = ":"; + break; + case 'b': + base = 2; + thousands = ":"; + break; + case 'o': + base = 8; + break; + default: + return -1; + } + if(!(fl & FmtUnsigned)){ + if(isv && (vlong)vu < 0){ + vu = -(vlong)vu; + neg = 1; + }else if(!isv && (long)u < 0){ + u = -(long)u; + neg = 1; + } + } + p = buf + sizeof buf - 1; + n = 0; /* in runes */ + excess = 0; /* number of bytes > number runes */ + ndig = 0; + len = utflen(thousands); + bytelen = strlen(thousands); + if(isv){ + while(vu){ + i = vu % base; + vu /= base; + if((fl & FmtComma) && n % 4 == 3){ + *p-- = ','; + n++; + } + if((fl & FmtApost) && __needsep(&ndig, &grouping)){ + n += len; + excess += bytelen - len; + p -= bytelen; + memmove(p+1, thousands, bytelen); + } + *p-- = conv[i]; + n++; + } + }else{ + while(u){ + i = u % base; + u /= base; + if((fl & FmtComma) && n % 4 == 3){ + *p-- = ','; + n++; + } + if((fl & FmtApost) && __needsep(&ndig, &grouping)){ + n += len; + excess += bytelen - len; + p -= bytelen; + memmove(p+1, thousands, bytelen); + } + *p-- = conv[i]; + n++; + } + } + if(n == 0){ + /* + * "The result of converting a zero value with + * a precision of zero is no characters." - ANSI + * + * "For o conversion, # increases the precision, if and only if + * necessary, to force the first digit of the result to be a zero + * (if the value and precision are both 0, a single 0 is printed)." - ANSI + */ + if(!(fl & FmtPrec) || f->prec != 0 || (f->r == 'o' && (fl & FmtSharp))){ + *p-- = '0'; + n = 1; + if(fl & FmtApost) + __needsep(&ndig, &grouping); + } + + /* + * Zero values don't get 0x. + */ + if(f->r == 'x' || f->r == 'X') + fl &= ~FmtSharp; + } + for(w = f->prec; n < w && p > buf+3; n++){ + if((fl & FmtApost) && __needsep(&ndig, &grouping)){ + n += len; + excess += bytelen - len; + p -= bytelen; + memmove(p+1, thousands, bytelen); + } + *p-- = '0'; + } + if(neg || (fl & (FmtSign|FmtSpace))) + n++; + if(fl & FmtSharp){ + if(base == 16) + n += 2; + else if(base == 8){ + if(p[1] == '0') + fl &= ~FmtSharp; + else + n++; + } + } + if((fl & FmtZero) && !(fl & (FmtLeft|FmtPrec))){ + w = 0; + if(fl & FmtWidth) + w = f->width; + for(; n < w && p > buf+3; n++){ + if((fl & FmtApost) && __needsep(&ndig, &grouping)){ + n += len; + excess += bytelen - len; + p -= bytelen; + memmove(p+1, thousands, bytelen); + } + *p-- = '0'; + } + f->flags &= ~FmtWidth; + } + if(fl & FmtSharp){ + if(base == 16) + *p-- = f->r; + if(base == 16 || base == 8) + *p-- = '0'; + } + if(neg) + *p-- = '-'; + else if(fl & FmtSign) + *p-- = '+'; + else if(fl & FmtSpace) + *p-- = ' '; + f->flags &= ~FmtPrec; + return __fmtcpy(f, p + 1, n, n + excess); +} + +int +__countfmt(Fmt *f) +{ + void *p; + ulong fl; + + fl = f->flags; + p = va_arg(f->args, void*); + if(fl & FmtVLong){ + *(vlong*)p = f->nfmt; + }else if(fl & FmtLong){ + *(long*)p = f->nfmt; + }else if(fl & FmtByte){ + *(char*)p = f->nfmt; + }else if(fl & FmtShort){ + *(short*)p = f->nfmt; + }else{ + *(int*)p = f->nfmt; + } + return 0; +} + +int +__flagfmt(Fmt *f) +{ + switch(f->r){ + case ',': + f->flags |= FmtComma; + break; + case '-': + f->flags |= FmtLeft; + break; + case '+': + f->flags |= FmtSign; + break; + case '#': + f->flags |= FmtSharp; + break; + case '\'': + f->flags |= FmtApost; + break; + case ' ': + f->flags |= FmtSpace; + break; + case 'u': + f->flags |= FmtUnsigned; + break; + case 'h': + if(f->flags & FmtShort) + f->flags |= FmtByte; + f->flags |= FmtShort; + break; + case 'L': + f->flags |= FmtLDouble; + break; + case 'l': + if(f->flags & FmtLong) + f->flags |= FmtVLong; + f->flags |= FmtLong; + break; + } + return 1; +} + +/* default error format */ +int +__badfmt(Fmt *f) +{ + char x[2+UTFmax]; + int n; + + x[0] = '%'; + n = 1 + runetochar(x+1, &f->r); + x[n++] = '%'; + f->prec = n; + __fmtcpy(f, (const void*)x, n, n); + return 0; +} diff --git a/src/lib9/fmt/dorfmt.c b/src/lib9/fmt/dorfmt.c new file mode 100644 index 000000000..672742f02 --- /dev/null +++ b/src/lib9/fmt/dorfmt.c @@ -0,0 +1,65 @@ +/* + * The authors of this software are Rob Pike and Ken Thompson, + * with contributions from Mike Burrows and Sean Dorward. + * + * Copyright (c) 2002-2006 by Lucent Technologies. + * Portions Copyright (c) 2004 Google Inc. + * + * Permission to use, copy, modify, and distribute this software for any + * purpose without fee is hereby granted, provided that this entire notice + * is included in all copies of any software which is or includes a copy + * or modification of this software and in all copies of the supporting + * documentation for such software. + * THIS SOFTWARE IS BEING PROVIDED "AS IS", WITHOUT ANY EXPRESS OR IMPLIED + * WARRANTY. IN PARTICULAR, NEITHER THE AUTHORS NOR LUCENT TECHNOLOGIES + * NOR GOOGLE INC MAKE ANY REPRESENTATION OR WARRANTY OF ANY KIND CONCERNING + * THE MERCHANTABILITY OF THIS SOFTWARE OR ITS FITNESS FOR ANY PARTICULAR PURPOSE. + */ + +#include +#include +#include "fmtdef.h" + +/* format the output into f->to and return the number of characters fmted */ + +/* BUG: THIS FILE IS NOT UPDATED TO THE NEW SPEC */ +int +dorfmt(Fmt *f, const Rune *fmt) +{ + Rune *rt, *rs; + int r; + char *t, *s; + int nfmt; + + nfmt = f->nfmt; + for(;;){ + if(f->runes){ + rt = (Rune*)f->to; + rs = (Rune*)f->stop; + while((r = *fmt++) && r != '%'){ + FMTRCHAR(f, rt, rs, r); + } + f->nfmt += rt - (Rune *)f->to; + f->to = rt; + if(!r) + return f->nfmt - nfmt; + f->stop = rs; + }else{ + t = (char*)f->to; + s = (char*)f->stop; + while((r = *fmt++) && r != '%'){ + FMTRUNE(f, t, f->stop, r); + } + f->nfmt += t - (char *)f->to; + f->to = t; + if(!r) + return f->nfmt - nfmt; + f->stop = s; + } + + fmt = (Rune*)__fmtdispatch(f, (Rune*)fmt, 1); + if(fmt == nil) + return -1; + } + return 0; /* not reached */ +} diff --git a/src/lib9/fmt/errfmt.c b/src/lib9/fmt/errfmt.c new file mode 100644 index 000000000..66c9600f0 --- /dev/null +++ b/src/lib9/fmt/errfmt.c @@ -0,0 +1,30 @@ +/* + * The authors of this software are Rob Pike and Ken Thompson, + * with contributions from Mike Burrows and Sean Dorward. + * + * Copyright (c) 2002-2006 by Lucent Technologies. + * Portions Copyright (c) 2004 Google Inc. + * + * Permission to use, copy, modify, and distribute this software for any + * purpose without fee is hereby granted, provided that this entire notice + * is included in all copies of any software which is or includes a copy + * or modification of this software and in all copies of the supporting + * documentation for such software. + * THIS SOFTWARE IS BEING PROVIDED "AS IS", WITHOUT ANY EXPRESS OR IMPLIED + * WARRANTY. IN PARTICULAR, NEITHER THE AUTHORS NOR LUCENT TECHNOLOGIES + * NOR GOOGLE INC MAKE ANY REPRESENTATION OR WARRANTY OF ANY KIND CONCERNING + * THE MERCHANTABILITY OF THIS SOFTWARE OR ITS FITNESS FOR ANY PARTICULAR PURPOSE. + */ + +#include +#include +#include "fmtdef.h" + +int +__errfmt(Fmt *f) +{ + char *s; + + s = strerror(errno); + return fmtstrcpy(f, s); +} diff --git a/src/lib9/fmt/fltfmt.c b/src/lib9/fmt/fltfmt.c new file mode 100644 index 000000000..9f3f3edab --- /dev/null +++ b/src/lib9/fmt/fltfmt.c @@ -0,0 +1,679 @@ +/* + * The authors of this software are Rob Pike and Ken Thompson, + * with contributions from Mike Burrows and Sean Dorward. + * + * Copyright (c) 2002-2006 by Lucent Technologies. + * Portions Copyright (c) 2004 Google Inc. + * + * Permission to use, copy, modify, and distribute this software for any + * purpose without fee is hereby granted, provided that this entire notice + * is included in all copies of any software which is or includes a copy + * or modification of this software and in all copies of the supporting + * documentation for such software. + * THIS SOFTWARE IS BEING PROVIDED "AS IS", WITHOUT ANY EXPRESS OR IMPLIED + * WARRANTY. IN PARTICULAR, NEITHER THE AUTHORS NOR LUCENT TECHNOLOGIES + * NOR GOOGLE INC MAKE ANY REPRESENTATION OR WARRANTY OF ANY KIND CONCERNING + * THE MERCHANTABILITY OF THIS SOFTWARE OR ITS FITNESS FOR ANY PARTICULAR PURPOSE. + */ + +/* Copyright (c) 2002-2006 Lucent Technologies; see LICENSE */ +#include +#include +#include +#include "fmtdef.h" + +enum +{ + FDIGIT = 30, + FDEFLT = 6, + NSIGNIF = 17 +}; + +/* + * first few powers of 10, enough for about 1/2 of the + * total space for doubles. + */ +static double pows10[] = +{ + 1e0, 1e1, 1e2, 1e3, 1e4, 1e5, 1e6, 1e7, 1e8, 1e9, + 1e10, 1e11, 1e12, 1e13, 1e14, 1e15, 1e16, 1e17, 1e18, 1e19, + 1e20, 1e21, 1e22, 1e23, 1e24, 1e25, 1e26, 1e27, 1e28, 1e29, + 1e30, 1e31, 1e32, 1e33, 1e34, 1e35, 1e36, 1e37, 1e38, 1e39, + 1e40, 1e41, 1e42, 1e43, 1e44, 1e45, 1e46, 1e47, 1e48, 1e49, + 1e50, 1e51, 1e52, 1e53, 1e54, 1e55, 1e56, 1e57, 1e58, 1e59, + 1e60, 1e61, 1e62, 1e63, 1e64, 1e65, 1e66, 1e67, 1e68, 1e69, + 1e70, 1e71, 1e72, 1e73, 1e74, 1e75, 1e76, 1e77, 1e78, 1e79, + 1e80, 1e81, 1e82, 1e83, 1e84, 1e85, 1e86, 1e87, 1e88, 1e89, + 1e90, 1e91, 1e92, 1e93, 1e94, 1e95, 1e96, 1e97, 1e98, 1e99, + 1e100, 1e101, 1e102, 1e103, 1e104, 1e105, 1e106, 1e107, 1e108, 1e109, + 1e110, 1e111, 1e112, 1e113, 1e114, 1e115, 1e116, 1e117, 1e118, 1e119, + 1e120, 1e121, 1e122, 1e123, 1e124, 1e125, 1e126, 1e127, 1e128, 1e129, + 1e130, 1e131, 1e132, 1e133, 1e134, 1e135, 1e136, 1e137, 1e138, 1e139, + 1e140, 1e141, 1e142, 1e143, 1e144, 1e145, 1e146, 1e147, 1e148, 1e149, + 1e150, 1e151, 1e152, 1e153, 1e154, 1e155, 1e156, 1e157, 1e158, 1e159, +}; + +#undef pow10 +#define npows10 ((int)(sizeof(pows10)/sizeof(pows10[0]))) +#define pow10(x) fmtpow10(x) + +static double +pow10(int n) +{ + double d; + int neg; + + neg = 0; + if(n < 0){ + neg = 1; + n = -n; + } + + if(n < npows10) + d = pows10[n]; + else{ + d = pows10[npows10-1]; + for(;;){ + n -= npows10 - 1; + if(n < npows10){ + d *= pows10[n]; + break; + } + d *= pows10[npows10 - 1]; + } + } + if(neg) + return 1./d; + return d; +} + +/* + * add 1 to the decimal integer string a of length n. + * if 99999 overflows into 10000, return 1 to tell caller + * to move the virtual decimal point. + */ +static int +xadd1(char *a, int n) +{ + char *b; + int c; + + if(n < 0 || n > NSIGNIF) + return 0; + for(b = a+n-1; b >= a; b--) { + c = *b + 1; + if(c <= '9') { + *b = c; + return 0; + } + *b = '0'; + } + /* + * need to overflow adding digit. + * shift number down and insert 1 at beginning. + * decimal is known to be 0s or we wouldn't + * have gotten this far. (e.g., 99999+1 => 00000) + */ + a[0] = '1'; + return 1; +} + +/* + * subtract 1 from the decimal integer string a. + * if 10000 underflows into 09999, make it 99999 + * and return 1 to tell caller to move the virtual + * decimal point. this way, xsub1 is inverse of xadd1. + */ +static int +xsub1(char *a, int n) +{ + char *b; + int c; + + if(n < 0 || n > NSIGNIF) + return 0; + for(b = a+n-1; b >= a; b--) { + c = *b - 1; + if(c >= '0') { + if(c == '0' && b == a) { + /* + * just zeroed the top digit; shift everyone up. + * decimal is known to be 9s or we wouldn't + * have gotten this far. (e.g., 10000-1 => 09999) + */ + *b = '9'; + return 1; + } + *b = c; + return 0; + } + *b = '9'; + } + /* + * can't get here. the number a is always normalized + * so that it has a nonzero first digit. + */ + abort(); +} + +/* + * format exponent like sprintf(p, "e%+02d", e) + */ +static void +xfmtexp(char *p, int e, int ucase) +{ + char se[9]; + int i; + + *p++ = ucase ? 'E' : 'e'; + if(e < 0) { + *p++ = '-'; + e = -e; + } else + *p++ = '+'; + i = 0; + while(e) { + se[i++] = e % 10 + '0'; + e /= 10; + } + while(i < 2) + se[i++] = '0'; + while(i > 0) + *p++ = se[--i]; + *p++ = '\0'; +} + +/* + * compute decimal integer m, exp such that: + * f = m*10^exp + * m is as short as possible with losing exactness + * assumes special cases (NaN, +Inf, -Inf) have been handled. + */ +static void +xdtoa(double f, char *s, int *exp, int *neg, int *ns) +{ + int c, d, e2, e, ee, i, ndigit, oerrno; + char tmp[NSIGNIF+10]; + double g; + + oerrno = errno; /* in case strtod smashes errno */ + + /* + * make f non-negative. + */ + *neg = 0; + if(f < 0) { + f = -f; + *neg = 1; + } + + /* + * must handle zero specially. + */ + if(f == 0){ + *exp = 0; + s[0] = '0'; + s[1] = '\0'; + *ns = 1; + return; + } + + /* + * find g,e such that f = g*10^e. + * guess 10-exponent using 2-exponent, then fine tune. + */ + frexp(f, &e2); + e = (int)(e2 * .301029995664); + g = f * pow10(-e); + while(g < 1) { + e--; + g = f * pow10(-e); + } + while(g >= 10) { + e++; + g = f * pow10(-e); + } + + /* + * convert NSIGNIF digits as a first approximation. + */ + for(i=0; i g) { + if(xadd1(s, NSIGNIF)) { + /* gained a digit */ + e--; + xfmtexp(s+NSIGNIF, e, 0); + } + continue; + } + if(f < g) { + if(xsub1(s, NSIGNIF)) { + /* lost a digit */ + e++; + xfmtexp(s+NSIGNIF, e, 0); + } + continue; + } + break; + } + + /* + * play with the decimal to try to simplify. + */ + + /* + * bump last few digits up to 9 if we can + */ + for(i=NSIGNIF-1; i>=NSIGNIF-3; i--) { + c = s[i]; + if(c != '9') { + s[i] = '9'; + g = strtod(s, nil); + if(g != f) { + s[i] = c; + break; + } + } + } + + /* + * add 1 in hopes of turning 9s to 0s + */ + if(s[NSIGNIF-1] == '9') { + strcpy(tmp, s); + ee = e; + if(xadd1(tmp, NSIGNIF)) { + ee--; + xfmtexp(tmp+NSIGNIF, ee, 0); + } + g = strtod(tmp, nil); + if(g == f) { + strcpy(s, tmp); + e = ee; + } + } + + /* + * bump last few digits down to 0 as we can. + */ + for(i=NSIGNIF-1; i>=NSIGNIF-3; i--) { + c = s[i]; + if(c != '0') { + s[i] = '0'; + g = strtod(s, nil); + if(g != f) { + s[i] = c; + break; + } + } + } + + /* + * remove trailing zeros. + */ + ndigit = NSIGNIF; + while(ndigit > 1 && s[ndigit-1] == '0'){ + e++; + --ndigit; + } + s[ndigit] = 0; + *exp = e; + *ns = ndigit; + errno = oerrno; +} + +#ifdef PLAN9PORT +static char *special[] = { "NaN", "NaN", "+Inf", "+Inf", "-Inf", "-Inf" }; +#else +static char *special[] = { "nan", "NAN", "inf", "INF", "-inf", "-INF" }; +#endif + +int +__efgfmt(Fmt *fmt) +{ + char buf[NSIGNIF+10], *dot, *digits, *p, *s, suf[10], *t; + double f; + int c, chr, dotwid, e, exp, fl, ndigits, neg, newndigits; + int pad, point, prec, realchr, sign, sufwid, ucase, wid, z1, z2; + Rune r, *rs, *rt; + + if(fmt->flags&FmtLong) + f = va_arg(fmt->args, long double); + else + f = va_arg(fmt->args, double); + + /* + * extract formatting flags + */ + fl = fmt->flags; + fmt->flags = 0; + prec = FDEFLT; + if(fl & FmtPrec) + prec = fmt->prec; + chr = fmt->r; + ucase = 0; + switch(chr) { + case 'A': + case 'E': + case 'F': + case 'G': + chr += 'a'-'A'; + ucase = 1; + break; + } + + /* + * pick off special numbers. + */ + if(__isNaN(f)) { + s = special[0+ucase]; + special: + fmt->flags = fl & (FmtWidth|FmtLeft); + return __fmtcpy(fmt, s, strlen(s), strlen(s)); + } + if(__isInf(f, 1)) { + s = special[2+ucase]; + goto special; + } + if(__isInf(f, -1)) { + s = special[4+ucase]; + goto special; + } + + /* + * get exact representation. + */ + digits = buf; + xdtoa(f, digits, &exp, &neg, &ndigits); + + /* + * get locale's decimal point. + */ + dot = fmt->decimal; + if(dot == nil) + dot = "."; + dotwid = utflen(dot); + + /* + * now the formatting fun begins. + * compute parameters for actual fmt: + * + * pad: number of spaces to insert before/after field. + * z1: number of zeros to insert before digits + * z2: number of zeros to insert after digits + * point: number of digits to print before decimal point + * ndigits: number of digits to use from digits[] + * suf: trailing suffix, like "e-5" + */ + realchr = chr; + switch(chr){ + case 'g': + /* + * convert to at most prec significant digits. (prec=0 means 1) + */ + if(prec == 0) + prec = 1; + if(ndigits > prec) { + if(digits[prec] >= '5' && xadd1(digits, prec)) + exp++; + exp += ndigits-prec; + ndigits = prec; + } + + /* + * extra rules for %g (implemented below): + * trailing zeros removed after decimal unless FmtSharp. + * decimal point only if digit follows. + */ + + /* fall through to %e */ + default: + case 'e': + /* + * one significant digit before decimal, no leading zeros. + */ + point = 1; + z1 = 0; + + /* + * decimal point is after ndigits digits right now. + * slide to be after first. + */ + e = exp + (ndigits-1); + + /* + * if this is %g, check exponent and convert prec + */ + if(realchr == 'g') { + if(-4 <= e && e < prec) + goto casef; + prec--; /* one digit before decimal; rest after */ + } + + /* + * compute trailing zero padding or truncate digits. + */ + if(1+prec >= ndigits) + z2 = 1+prec - ndigits; + else { + /* + * truncate digits + */ + assert(realchr != 'g'); + newndigits = 1+prec; + if(digits[newndigits] >= '5' && xadd1(digits, newndigits)) { + /* + * had 999e4, now have 100e5 + */ + e++; + } + ndigits = newndigits; + z2 = 0; + } + xfmtexp(suf, e, ucase); + sufwid = strlen(suf); + break; + + casef: + case 'f': + /* + * determine where digits go with respect to decimal point + */ + if(ndigits+exp > 0) { + point = ndigits+exp; + z1 = 0; + } else { + point = 1; + z1 = 1 + -(ndigits+exp); + } + + /* + * %g specifies prec = number of significant digits + * convert to number of digits after decimal point + */ + if(realchr == 'g') + prec += z1 - point; + + /* + * compute trailing zero padding or truncate digits. + */ + if(point+prec >= z1+ndigits) + z2 = point+prec - (z1+ndigits); + else { + /* + * truncate digits + */ + assert(realchr != 'g'); + newndigits = point+prec - z1; + if(newndigits < 0) { + z1 += newndigits; + newndigits = 0; + } else if(newndigits == 0) { + /* perhaps round up */ + if(digits[0] >= '5'){ + digits[0] = '1'; + newndigits = 1; + goto newdigit; + } + } else if(digits[newndigits] >= '5' && xadd1(digits, newndigits)) { + /* + * digits was 999, is now 100; make it 1000 + */ + digits[newndigits++] = '0'; + newdigit: + /* + * account for new digit + */ + if(z1) /* 0.099 => 0.100 or 0.99 => 1.00*/ + z1--; + else /* 9.99 => 10.00 */ + point++; + } + z2 = 0; + ndigits = newndigits; + } + sufwid = 0; + break; + } + + /* + * if %g is given without FmtSharp, remove trailing zeros. + * must do after truncation, so that e.g. print %.3g 1.001 + * produces 1, not 1.00. sorry, but them's the rules. + */ + if(realchr == 'g' && !(fl & FmtSharp)) { + if(z1+ndigits+z2 >= point) { + if(z1+ndigits < point) + z2 = point - (z1+ndigits); + else{ + z2 = 0; + while(z1+ndigits > point && digits[ndigits-1] == '0') + ndigits--; + } + } + } + + /* + * compute width of all digits and decimal point and suffix if any + */ + wid = z1+ndigits+z2; + if(wid > point) + wid += dotwid; + else if(wid == point){ + if(fl & FmtSharp) + wid += dotwid; + else + point++; /* do not print any decimal point */ + } + wid += sufwid; + + /* + * determine sign + */ + sign = 0; + if(neg) + sign = '-'; + else if(fl & FmtSign) + sign = '+'; + else if(fl & FmtSpace) + sign = ' '; + if(sign) + wid++; + + /* + * compute padding + */ + pad = 0; + if((fl & FmtWidth) && fmt->width > wid) + pad = fmt->width - wid; + if(pad && !(fl & FmtLeft) && (fl & FmtZero)){ + z1 += pad; + point += pad; + pad = 0; + } + + /* + * format the actual field. too bad about doing this twice. + */ + if(fmt->runes){ + if(pad && !(fl & FmtLeft) && __rfmtpad(fmt, pad) < 0) + return -1; + rt = (Rune*)fmt->to; + rs = (Rune*)fmt->stop; + if(sign) + FMTRCHAR(fmt, rt, rs, sign); + while(z1>0 || ndigits>0 || z2>0) { + if(z1 > 0){ + z1--; + c = '0'; + }else if(ndigits > 0){ + ndigits--; + c = *digits++; + }else{ + z2--; + c = '0'; + } + FMTRCHAR(fmt, rt, rs, c); + if(--point == 0) { + for(p = dot; *p; ){ + p += chartorune(&r, p); + FMTRCHAR(fmt, rt, rs, r); + } + } + } + fmt->nfmt += rt - (Rune*)fmt->to; + fmt->to = rt; + if(sufwid && __fmtcpy(fmt, suf, sufwid, sufwid) < 0) + return -1; + if(pad && (fl & FmtLeft) && __rfmtpad(fmt, pad) < 0) + return -1; + }else{ + if(pad && !(fl & FmtLeft) && __fmtpad(fmt, pad) < 0) + return -1; + t = (char*)fmt->to; + s = (char*)fmt->stop; + if(sign) + FMTCHAR(fmt, t, s, sign); + while(z1>0 || ndigits>0 || z2>0) { + if(z1 > 0){ + z1--; + c = '0'; + }else if(ndigits > 0){ + ndigits--; + c = *digits++; + }else{ + z2--; + c = '0'; + } + FMTCHAR(fmt, t, s, c); + if(--point == 0) + for(p=dot; *p; p++) + FMTCHAR(fmt, t, s, *p); + } + fmt->nfmt += t - (char*)fmt->to; + fmt->to = t; + if(sufwid && __fmtcpy(fmt, suf, sufwid, sufwid) < 0) + return -1; + if(pad && (fl & FmtLeft) && __fmtpad(fmt, pad) < 0) + return -1; + } + return 0; +} + diff --git a/src/lib9/fmt/fmt.c b/src/lib9/fmt/fmt.c new file mode 100644 index 000000000..7a747b1b1 --- /dev/null +++ b/src/lib9/fmt/fmt.c @@ -0,0 +1,235 @@ +/* + * The authors of this software are Rob Pike and Ken Thompson, + * with contributions from Mike Burrows and Sean Dorward. + * + * Copyright (c) 2002-2006 by Lucent Technologies. + * Portions Copyright (c) 2004 Google Inc. + * + * Permission to use, copy, modify, and distribute this software for any + * purpose without fee is hereby granted, provided that this entire notice + * is included in all copies of any software which is or includes a copy + * or modification of this software and in all copies of the supporting + * documentation for such software. + * THIS SOFTWARE IS BEING PROVIDED "AS IS", WITHOUT ANY EXPRESS OR IMPLIED + * WARRANTY. IN PARTICULAR, NEITHER THE AUTHORS NOR LUCENT TECHNOLOGIES + * NOR GOOGLE INC MAKE ANY REPRESENTATION OR WARRANTY OF ANY KIND CONCERNING + * THE MERCHANTABILITY OF THIS SOFTWARE OR ITS FITNESS FOR ANY PARTICULAR PURPOSE. + */ + +#include +#include +#include "fmtdef.h" + +enum +{ + Maxfmt = 64 +}; + +typedef struct Convfmt Convfmt; +struct Convfmt +{ + int c; + volatile Fmts fmt; /* for spin lock in fmtfmt; avoids race due to write order */ +}; + +static struct +{ + /* lock by calling __fmtlock, __fmtunlock */ + int nfmt; + Convfmt fmt[Maxfmt]; +} fmtalloc; + +static Convfmt knownfmt[] = { + ' ', __flagfmt, + '#', __flagfmt, + '%', __percentfmt, + '\'', __flagfmt, + '+', __flagfmt, + ',', __flagfmt, + '-', __flagfmt, + 'C', __runefmt, /* Plan 9 addition */ + 'E', __efgfmt, +#ifndef PLAN9PORT + 'F', __efgfmt, /* ANSI only */ +#endif + 'G', __efgfmt, +#ifndef PLAN9PORT + 'L', __flagfmt, /* ANSI only */ +#endif + 'S', __runesfmt, /* Plan 9 addition */ + 'X', __ifmt, + 'b', __ifmt, /* Plan 9 addition */ + 'c', __charfmt, + 'd', __ifmt, + 'e', __efgfmt, + 'f', __efgfmt, + 'g', __efgfmt, + 'h', __flagfmt, +#ifndef PLAN9PORT + 'i', __ifmt, /* ANSI only */ +#endif + 'l', __flagfmt, + 'n', __countfmt, + 'o', __ifmt, + 'p', __ifmt, + 'r', __errfmt, + 's', __strfmt, +#ifdef PLAN9PORT + 'u', __flagfmt, +#else + 'u', __ifmt, +#endif + 'x', __ifmt, + 0, nil, +}; + + +int (*fmtdoquote)(int); + +/* + * __fmtlock() must be set + */ +static int +__fmtinstall(int c, Fmts f) +{ + Convfmt *p, *ep; + + if(c<=0 || c>=65536) + return -1; + if(!f) + f = __badfmt; + + ep = &fmtalloc.fmt[fmtalloc.nfmt]; + for(p=fmtalloc.fmt; pc == c) + break; + + if(p == &fmtalloc.fmt[Maxfmt]) + return -1; + + p->fmt = f; + if(p == ep){ /* installing a new format character */ + fmtalloc.nfmt++; + p->c = c; + } + + return 0; +} + +int +fmtinstall(int c, int (*f)(Fmt*)) +{ + int ret; + + __fmtlock(); + ret = __fmtinstall(c, f); + __fmtunlock(); + return ret; +} + +static Fmts +fmtfmt(int c) +{ + Convfmt *p, *ep; + + ep = &fmtalloc.fmt[fmtalloc.nfmt]; + for(p=fmtalloc.fmt; pc == c){ + while(p->fmt == nil) /* loop until value is updated */ + ; + return p->fmt; + } + + /* is this a predefined format char? */ + __fmtlock(); + for(p=knownfmt; p->c; p++) + if(p->c == c){ + __fmtinstall(p->c, p->fmt); + __fmtunlock(); + return p->fmt; + } + __fmtunlock(); + + return __badfmt; +} + +void* +__fmtdispatch(Fmt *f, void *fmt, int isrunes) +{ + Rune rune, r; + int i, n; + + f->flags = 0; + f->width = f->prec = 0; + + for(;;){ + if(isrunes){ + r = *(Rune*)fmt; + fmt = (Rune*)fmt + 1; + }else{ + fmt = (char*)fmt + chartorune(&rune, (char*)fmt); + r = rune; + } + f->r = r; + switch(r){ + case '\0': + return nil; + case '.': + f->flags |= FmtWidth|FmtPrec; + continue; + case '0': + if(!(f->flags & FmtWidth)){ + f->flags |= FmtZero; + continue; + } + /* fall through */ + case '1': case '2': case '3': case '4': + case '5': case '6': case '7': case '8': case '9': + i = 0; + while(r >= '0' && r <= '9'){ + i = i * 10 + r - '0'; + if(isrunes){ + r = *(Rune*)fmt; + fmt = (Rune*)fmt + 1; + }else{ + r = *(char*)fmt; + fmt = (char*)fmt + 1; + } + } + if(isrunes) + fmt = (Rune*)fmt - 1; + else + fmt = (char*)fmt - 1; + numflag: + if(f->flags & FmtWidth){ + f->flags |= FmtPrec; + f->prec = i; + }else{ + f->flags |= FmtWidth; + f->width = i; + } + continue; + case '*': + i = va_arg(f->args, int); + if(i < 0){ + /* + * negative precision => + * ignore the precision. + */ + if(f->flags & FmtPrec){ + f->flags &= ~FmtPrec; + f->prec = 0; + continue; + } + i = -i; + f->flags |= FmtLeft; + } + goto numflag; + } + n = (*fmtfmt(r))(f); + if(n < 0) + return nil; + if(n == 0) + return fmt; + } +} diff --git a/src/lib9/fmt/fmtdef.h b/src/lib9/fmt/fmtdef.h new file mode 100644 index 000000000..74cb8a8d2 --- /dev/null +++ b/src/lib9/fmt/fmtdef.h @@ -0,0 +1,119 @@ +/* + * The authors of this software are Rob Pike and Ken Thompson. + * + * Copyright (c) 2002-2006 by Lucent Technologies. + * + * Permission to use, copy, modify, and distribute this software for any + * purpose without fee is hereby granted, provided that this entire notice + * is included in all copies of any software which is or includes a copy + * or modification of this software and in all copies of the supporting + * documentation for such software. + * THIS SOFTWARE IS BEING PROVIDED "AS IS", WITHOUT ANY EXPRESS OR IMPLIED + * WARRANTY. IN PARTICULAR, NEITHER THE AUTHORS NOR LUCENT TECHNOLOGIES + * NOR GOOGLE INC MAKE ANY REPRESENTATION OR WARRANTY OF ANY KIND CONCERNING + * THE MERCHANTABILITY OF THIS SOFTWARE OR ITS FITNESS FOR ANY PARTICULAR PURPOSE. + */ + +/* + * dofmt -- format to a buffer + * the number of characters formatted is returned, + * or -1 if there was an error. + * if the buffer is ever filled, flush is called. + * it should reset the buffer and return whether formatting should continue. + */ + +typedef int (*Fmts)(Fmt*); + +typedef struct Quoteinfo Quoteinfo; +struct Quoteinfo +{ + int quoted; /* if set, string must be quoted */ + int nrunesin; /* number of input runes that can be accepted */ + int nbytesin; /* number of input bytes that can be accepted */ + int nrunesout; /* number of runes that will be generated */ + int nbytesout; /* number of bytes that will be generated */ +}; + +/* Edit .+1,/^$/ |cfn |grep -v static | grep __ */ +double __Inf(int sign); +double __NaN(void); +int __badfmt(Fmt *f); +int __charfmt(Fmt *f); +int __countfmt(Fmt *f); +int __efgfmt(Fmt *fmt); +int __errfmt(Fmt *f); +int __flagfmt(Fmt *f); +int __fmtFdFlush(Fmt *f); +int __fmtcpy(Fmt *f, const void *vm, int n, int sz); +void* __fmtdispatch(Fmt *f, void *fmt, int isrunes); +void * __fmtflush(Fmt *f, void *t, int len); +void __fmtlock(void); +int __fmtpad(Fmt *f, int n); +double __fmtpow10(int n); +int __fmtrcpy(Fmt *f, const void *vm, int n); +void __fmtunlock(void); +int __ifmt(Fmt *f); +int __isInf(double d, int sign); +int __isNaN(double d); +int __needsep(int*, char**); +int __needsquotes(char *s, int *quotelenp); +int __percentfmt(Fmt *f); +void __quotesetup(char *s, Rune *r, int nin, int nout, Quoteinfo *q, int sharp, int runesout); +int __quotestrfmt(int runesin, Fmt *f); +int __rfmtpad(Fmt *f, int n); +int __runefmt(Fmt *f); +int __runeneedsquotes(Rune *r, int *quotelenp); +int __runesfmt(Fmt *f); +int __strfmt(Fmt *f); + +#define FMTCHAR(f, t, s, c)\ + do{\ + if(t + 1 > (char*)s){\ + t = (char*)__fmtflush(f, t, 1);\ + if(t != nil)\ + s = (char*)f->stop;\ + else\ + return -1;\ + }\ + *t++ = c;\ + }while(0) + +#define FMTRCHAR(f, t, s, c)\ + do{\ + if(t + 1 > (Rune*)s){\ + t = (Rune*)__fmtflush(f, t, sizeof(Rune));\ + if(t != nil)\ + s = (Rune*)f->stop;\ + else\ + return -1;\ + }\ + *t++ = c;\ + }while(0) + +#define FMTRUNE(f, t, s, r)\ + do{\ + Rune _rune;\ + int _runelen;\ + if(t + UTFmax > (char*)s && t + (_runelen = runelen(r)) > (char*)s){\ + t = (char*)__fmtflush(f, t, _runelen);\ + if(t != nil)\ + s = (char*)f->stop;\ + else\ + return -1;\ + }\ + if(r < Runeself)\ + *t++ = r;\ + else{\ + _rune = r;\ + t += runetochar(t, &_rune);\ + }\ + }while(0) + +#ifdef va_copy +# define VA_COPY(a,b) va_copy(a,b) +# define VA_END(a) va_end(a) +#else +# define VA_COPY(a,b) (a) = (b) +# define VA_END(a) +#endif + diff --git a/src/lib9/fmt/fmtfd.c b/src/lib9/fmt/fmtfd.c new file mode 100644 index 000000000..c32abf115 --- /dev/null +++ b/src/lib9/fmt/fmtfd.c @@ -0,0 +1,51 @@ +/* + * The authors of this software are Rob Pike and Ken Thompson, + * with contributions from Mike Burrows and Sean Dorward. + * + * Copyright (c) 2002-2006 by Lucent Technologies. + * Portions Copyright (c) 2004 Google Inc. + * + * Permission to use, copy, modify, and distribute this software for any + * purpose without fee is hereby granted, provided that this entire notice + * is included in all copies of any software which is or includes a copy + * or modification of this software and in all copies of the supporting + * documentation for such software. + * THIS SOFTWARE IS BEING PROVIDED "AS IS", WITHOUT ANY EXPRESS OR IMPLIED + * WARRANTY. IN PARTICULAR, NEITHER THE AUTHORS NOR LUCENT TECHNOLOGIES + * NOR GOOGLE INC MAKE ANY REPRESENTATION OR WARRANTY OF ANY KIND CONCERNING + * THE MERCHANTABILITY OF THIS SOFTWARE OR ITS FITNESS FOR ANY PARTICULAR PURPOSE. + */ + +#include +#include +#include "fmtdef.h" + +/* + * public routine for final flush of a formatting buffer + * to a file descriptor; returns total char count. + */ +int +fmtfdflush(Fmt *f) +{ + if(__fmtFdFlush(f) <= 0) + return -1; + return f->nfmt; +} + +/* + * initialize an output buffer for buffered printing + */ +int +fmtfdinit(Fmt *f, int fd, char *buf, int size) +{ + f->runes = 0; + f->start = buf; + f->to = buf; + f->stop = buf + size; + f->flush = __fmtFdFlush; + f->farg = (void*)(uintptr_t)fd; + f->flags = 0; + f->nfmt = 0; + fmtlocaleinit(f, nil, nil, nil); + return 0; +} diff --git a/src/lib9/fmt/fmtfdflush.c b/src/lib9/fmt/fmtfdflush.c new file mode 100644 index 000000000..c9854cee5 --- /dev/null +++ b/src/lib9/fmt/fmtfdflush.c @@ -0,0 +1,37 @@ +/* + * The authors of this software are Rob Pike and Ken Thompson, + * with contributions from Mike Burrows and Sean Dorward. + * + * Copyright (c) 2002-2006 by Lucent Technologies. + * Portions Copyright (c) 2004 Google Inc. + * + * Permission to use, copy, modify, and distribute this software for any + * purpose without fee is hereby granted, provided that this entire notice + * is included in all copies of any software which is or includes a copy + * or modification of this software and in all copies of the supporting + * documentation for such software. + * THIS SOFTWARE IS BEING PROVIDED "AS IS", WITHOUT ANY EXPRESS OR IMPLIED + * WARRANTY. IN PARTICULAR, NEITHER THE AUTHORS NOR LUCENT TECHNOLOGIES + * NOR GOOGLE INC MAKE ANY REPRESENTATION OR WARRANTY OF ANY KIND CONCERNING + * THE MERCHANTABILITY OF THIS SOFTWARE OR ITS FITNESS FOR ANY PARTICULAR PURPOSE. + */ + +#include +#include +#include "fmtdef.h" + +/* + * generic routine for flushing a formatting buffer + * to a file descriptor + */ +int +__fmtFdFlush(Fmt *f) +{ + int n; + + n = (char*)f->to - (char*)f->start; + if(n && write((uintptr)f->farg, f->start, n) != n) + return 0; + f->to = f->start; + return 1; +} diff --git a/src/lib9/fmt/fmtlocale.c b/src/lib9/fmt/fmtlocale.c new file mode 100644 index 000000000..64ed10f7b --- /dev/null +++ b/src/lib9/fmt/fmtlocale.c @@ -0,0 +1,69 @@ +/* + * The authors of this software are Rob Pike and Ken Thompson, + * with contributions from Mike Burrows and Sean Dorward. + * + * Copyright (c) 2002-2006 by Lucent Technologies. + * Portions Copyright (c) 2004 Google Inc. + * + * Permission to use, copy, modify, and distribute this software for any + * purpose without fee is hereby granted, provided that this entire notice + * is included in all copies of any software which is or includes a copy + * or modification of this software and in all copies of the supporting + * documentation for such software. + * THIS SOFTWARE IS BEING PROVIDED "AS IS", WITHOUT ANY EXPRESS OR IMPLIED + * WARRANTY. IN PARTICULAR, NEITHER THE AUTHORS NOR LUCENT TECHNOLOGIES + * NOR GOOGLE INC MAKE ANY REPRESENTATION OR WARRANTY OF ANY KIND CONCERNING + * THE MERCHANTABILITY OF THIS SOFTWARE OR ITS FITNESS FOR ANY PARTICULAR PURPOSE. + */ + +#include +#include +#include "fmtdef.h" + +/* + * Fill in the internationalization stuff in the State structure. + * For nil arguments, provide the sensible defaults: + * decimal is a period + * thousands separator is a comma + * thousands are marked every three digits + */ +void +fmtlocaleinit(Fmt *f, char *decimal, char *thousands, char *grouping) +{ + if(decimal == nil || decimal[0] == '\0') + decimal = "."; + if(thousands == nil) + thousands = ","; + if(grouping == nil) + grouping = "\3"; + f->decimal = decimal; + f->thousands = thousands; + f->grouping = grouping; +} + +/* + * We are about to emit a digit in e.g. %'d. If that digit would + * overflow a thousands (e.g.) grouping, tell the caller to emit + * the thousands separator. Always advance the digit counter + * and pointer into the grouping descriptor. + */ +int +__needsep(int *ndig, char **grouping) +{ + int group; + + (*ndig)++; + group = *(unsigned char*)*grouping; + /* CHAR_MAX means no further grouping. \0 means we got the empty string */ + if(group == 0xFF || group == 0x7f || group == 0x00) + return 0; + if(*ndig > group){ + /* if we're at end of string, continue with this grouping; else advance */ + if((*grouping)[1] != '\0') + (*grouping)++; + *ndig = 1; + return 1; + } + return 0; +} + diff --git a/src/lib9/fmt/fmtlock.c b/src/lib9/fmt/fmtlock.c new file mode 100644 index 000000000..297acd8f9 --- /dev/null +++ b/src/lib9/fmt/fmtlock.c @@ -0,0 +1,31 @@ +/* + * The authors of this software are Rob Pike and Ken Thompson, + * with contributions from Mike Burrows and Sean Dorward. + * + * Copyright (c) 2002-2006 by Lucent Technologies. + * Portions Copyright (c) 2004 Google Inc. + * + * Permission to use, copy, modify, and distribute this software for any + * purpose without fee is hereby granted, provided that this entire notice + * is included in all copies of any software which is or includes a copy + * or modification of this software and in all copies of the supporting + * documentation for such software. + * THIS SOFTWARE IS BEING PROVIDED "AS IS", WITHOUT ANY EXPRESS OR IMPLIED + * WARRANTY. IN PARTICULAR, NEITHER THE AUTHORS NOR LUCENT TECHNOLOGIES + * NOR GOOGLE INC MAKE ANY REPRESENTATION OR WARRANTY OF ANY KIND CONCERNING + * THE MERCHANTABILITY OF THIS SOFTWARE OR ITS FITNESS FOR ANY PARTICULAR PURPOSE. + */ + +#include +#include +#include "fmtdef.h" + +void +__fmtlock(void) +{ +} + +void +__fmtunlock(void) +{ +} diff --git a/src/lib9/fmt/fmtnull.c b/src/lib9/fmt/fmtnull.c new file mode 100644 index 000000000..b8caacbf7 --- /dev/null +++ b/src/lib9/fmt/fmtnull.c @@ -0,0 +1,48 @@ +/* + * The authors of this software are Rob Pike and Ken Thompson, + * with contributions from Mike Burrows and Sean Dorward. + * + * Copyright (c) 2002-2006 by Lucent Technologies. + * Portions Copyright (c) 2004 Google Inc. + * + * Permission to use, copy, modify, and distribute this software for any + * purpose without fee is hereby granted, provided that this entire notice + * is included in all copies of any software which is or includes a copy + * or modification of this software and in all copies of the supporting + * documentation for such software. + * THIS SOFTWARE IS BEING PROVIDED "AS IS", WITHOUT ANY EXPRESS OR IMPLIED + * WARRANTY. IN PARTICULAR, NEITHER THE AUTHORS NOR LUCENT TECHNOLOGIES + * NOR GOOGLE INC MAKE ANY REPRESENTATION OR WARRANTY OF ANY KIND CONCERNING + * THE MERCHANTABILITY OF THIS SOFTWARE OR ITS FITNESS FOR ANY PARTICULAR PURPOSE. + */ + +#include +#include +#include "fmtdef.h" + +/* + * Absorb output without using resources. + */ +static Rune nullbuf[32]; + +static int +__fmtnullflush(Fmt *f) +{ + f->to = nullbuf; + f->nfmt = 0; + return 0; +} + +int +fmtnullinit(Fmt *f) +{ + memset(f, 0, sizeof *f); + f->runes = 1; + f->start = nullbuf; + f->to = nullbuf; + f->stop = nullbuf+nelem(nullbuf); + f->flush = __fmtnullflush; + fmtlocaleinit(f, nil, nil, nil); + return 0; +} + diff --git a/src/lib9/fmt/fmtprint.c b/src/lib9/fmt/fmtprint.c new file mode 100644 index 000000000..8a29e6faf --- /dev/null +++ b/src/lib9/fmt/fmtprint.c @@ -0,0 +1,51 @@ +/* + * The authors of this software are Rob Pike and Ken Thompson, + * with contributions from Mike Burrows and Sean Dorward. + * + * Copyright (c) 2002-2006 by Lucent Technologies. + * Portions Copyright (c) 2004 Google Inc. + * + * Permission to use, copy, modify, and distribute this software for any + * purpose without fee is hereby granted, provided that this entire notice + * is included in all copies of any software which is or includes a copy + * or modification of this software and in all copies of the supporting + * documentation for such software. + * THIS SOFTWARE IS BEING PROVIDED "AS IS", WITHOUT ANY EXPRESS OR IMPLIED + * WARRANTY. IN PARTICULAR, NEITHER THE AUTHORS NOR LUCENT TECHNOLOGIES + * NOR GOOGLE INC MAKE ANY REPRESENTATION OR WARRANTY OF ANY KIND CONCERNING + * THE MERCHANTABILITY OF THIS SOFTWARE OR ITS FITNESS FOR ANY PARTICULAR PURPOSE. + */ + +#include +#include +#include "fmtdef.h" + +/* + * format a string into the output buffer + * designed for formats which themselves call fmt, + * but ignore any width flags + */ +int +fmtprint(Fmt *f, char *fmt, ...) +{ + va_list va; + int n; + + f->flags = 0; + f->width = 0; + f->prec = 0; + VA_COPY(va, f->args); + VA_END(f->args); + va_start(f->args, fmt); + n = dofmt(f, fmt); + va_end(f->args); + f->flags = 0; + f->width = 0; + f->prec = 0; + VA_COPY(f->args,va); + VA_END(va); + if(n >= 0) + return 0; + return n; +} + diff --git a/src/lib9/fmt/fmtquote.c b/src/lib9/fmt/fmtquote.c new file mode 100644 index 000000000..b9ac772ed --- /dev/null +++ b/src/lib9/fmt/fmtquote.c @@ -0,0 +1,274 @@ +/* + * The authors of this software are Rob Pike and Ken Thompson, + * with contributions from Mike Burrows and Sean Dorward. + * + * Copyright (c) 2002-2006 by Lucent Technologies. + * Portions Copyright (c) 2004 Google Inc. + * + * Permission to use, copy, modify, and distribute this software for any + * purpose without fee is hereby granted, provided that this entire notice + * is included in all copies of any software which is or includes a copy + * or modification of this software and in all copies of the supporting + * documentation for such software. + * THIS SOFTWARE IS BEING PROVIDED "AS IS", WITHOUT ANY EXPRESS OR IMPLIED + * WARRANTY. IN PARTICULAR, NEITHER THE AUTHORS NOR LUCENT TECHNOLOGIES + * NOR GOOGLE INC MAKE ANY REPRESENTATION OR WARRANTY OF ANY KIND CONCERNING + * THE MERCHANTABILITY OF THIS SOFTWARE OR ITS FITNESS FOR ANY PARTICULAR PURPOSE. + */ + +#include +#include +#include "fmtdef.h" + +/* + * How many bytes of output UTF will be produced by quoting (if necessary) this string? + * How many runes? How much of the input will be consumed? + * The parameter q is filled in by __quotesetup. + * The string may be UTF or Runes (s or r). + * Return count does not include NUL. + * Terminate the scan at the first of: + * NUL in input + * count exceeded in input + * count exceeded on output + * *ninp is set to number of input bytes accepted. + * nin may be <0 initially, to avoid checking input by count. + */ +void +__quotesetup(char *s, Rune *r, int nin, int nout, Quoteinfo *q, int sharp, int runesout) +{ + int w; + Rune c; + + q->quoted = 0; + q->nbytesout = 0; + q->nrunesout = 0; + q->nbytesin = 0; + q->nrunesin = 0; + if(sharp || nin==0 || (s && *s=='\0') || (r && *r=='\0')){ + if(nout < 2) + return; + q->quoted = 1; + q->nbytesout = 2; + q->nrunesout = 2; + } + for(; nin!=0; nin--){ + if(s) + w = chartorune(&c, s); + else{ + c = *r; + w = runelen(c); + } + + if(c == '\0') + break; + if(runesout){ + if(q->nrunesout+1 > nout) + break; + }else{ + if(q->nbytesout+w > nout) + break; + } + + if((c <= L' ') || (c == L'\'') || (fmtdoquote!=nil && fmtdoquote(c))){ + if(!q->quoted){ + if(runesout){ + if(1+q->nrunesout+1+1 > nout) /* no room for quotes */ + break; + }else{ + if(1+q->nbytesout+w+1 > nout) /* no room for quotes */ + break; + } + q->nrunesout += 2; /* include quotes */ + q->nbytesout += 2; /* include quotes */ + q->quoted = 1; + } + if(c == '\'') { + if(runesout){ + if(1+q->nrunesout+1 > nout) /* no room for quotes */ + break; + }else{ + if(1+q->nbytesout+w > nout) /* no room for quotes */ + break; + } + q->nbytesout++; + q->nrunesout++; /* quotes reproduce as two characters */ + } + } + + /* advance input */ + if(s) + s += w; + else + r++; + q->nbytesin += w; + q->nrunesin++; + + /* advance output */ + q->nbytesout += w; + q->nrunesout++; + +#ifndef PLAN9PORT + /* ANSI requires precision in bytes, not Runes. */ + nin-= w-1; /* and then n-- in the loop */ +#endif + } +} + +static int +qstrfmt(char *sin, Rune *rin, Quoteinfo *q, Fmt *f) +{ + Rune r, *rm, *rme; + char *t, *s, *m, *me; + Rune *rt, *rs; + ulong fl; + int nc, w; + + m = sin; + me = m + q->nbytesin; + rm = rin; + rme = rm + q->nrunesin; + + fl = f->flags; + w = 0; + if(fl & FmtWidth) + w = f->width; + if(f->runes){ + if(!(fl & FmtLeft) && __rfmtpad(f, w - q->nrunesout) < 0) + return -1; + }else{ + if(!(fl & FmtLeft) && __fmtpad(f, w - q->nbytesout) < 0) + return -1; + } + t = (char*)f->to; + s = (char*)f->stop; + rt = (Rune*)f->to; + rs = (Rune*)f->stop; + if(f->runes) + FMTRCHAR(f, rt, rs, '\''); + else + FMTRUNE(f, t, s, '\''); + for(nc = q->nrunesin; nc > 0; nc--){ + if(sin){ + r = *(uchar*)m; + if(r < Runeself) + m++; + else if((me - m) >= UTFmax || fullrune(m, me-m)) + m += chartorune(&r, m); + else + break; + }else{ + if(rm >= rme) + break; + r = *(uchar*)rm++; + } + if(f->runes){ + FMTRCHAR(f, rt, rs, r); + if(r == '\'') + FMTRCHAR(f, rt, rs, r); + }else{ + FMTRUNE(f, t, s, r); + if(r == '\'') + FMTRUNE(f, t, s, r); + } + } + + if(f->runes){ + FMTRCHAR(f, rt, rs, '\''); + USED(rs); + f->nfmt += rt - (Rune *)f->to; + f->to = rt; + if(fl & FmtLeft && __rfmtpad(f, w - q->nrunesout) < 0) + return -1; + }else{ + FMTRUNE(f, t, s, '\''); + USED(s); + f->nfmt += t - (char *)f->to; + f->to = t; + if(fl & FmtLeft && __fmtpad(f, w - q->nbytesout) < 0) + return -1; + } + return 0; +} + +int +__quotestrfmt(int runesin, Fmt *f) +{ + int nin, outlen; + Rune *r; + char *s; + Quoteinfo q; + + nin = -1; + if(f->flags&FmtPrec) + nin = f->prec; + if(runesin){ + r = va_arg(f->args, Rune *); + s = nil; + }else{ + s = va_arg(f->args, char *); + r = nil; + } + if(!s && !r) + return __fmtcpy(f, (void*)"", 5, 5); + + if(f->flush) + outlen = 0x7FFFFFFF; /* if we can flush, no output limit */ + else if(f->runes) + outlen = (Rune*)f->stop - (Rune*)f->to; + else + outlen = (char*)f->stop - (char*)f->to; + + __quotesetup(s, r, nin, outlen, &q, f->flags&FmtSharp, f->runes); +/*print("bytes in %d bytes out %d runes in %d runesout %d\n", q.nbytesin, q.nbytesout, q.nrunesin, q.nrunesout); */ + + if(runesin){ + if(!q.quoted) + return __fmtrcpy(f, r, q.nrunesin); + return qstrfmt(nil, r, &q, f); + } + + if(!q.quoted) + return __fmtcpy(f, s, q.nrunesin, q.nbytesin); + return qstrfmt(s, nil, &q, f); +} + +int +quotestrfmt(Fmt *f) +{ + return __quotestrfmt(0, f); +} + +int +quoterunestrfmt(Fmt *f) +{ + return __quotestrfmt(1, f); +} + +void +quotefmtinstall(void) +{ + fmtinstall('q', quotestrfmt); + fmtinstall('Q', quoterunestrfmt); +} + +int +__needsquotes(char *s, int *quotelenp) +{ + Quoteinfo q; + + __quotesetup(s, nil, -1, 0x7FFFFFFF, &q, 0, 0); + *quotelenp = q.nbytesout; + + return q.quoted; +} + +int +__runeneedsquotes(Rune *r, int *quotelenp) +{ + Quoteinfo q; + + __quotesetup(nil, r, -1, 0x7FFFFFFF, &q, 0, 0); + *quotelenp = q.nrunesout; + + return q.quoted; +} diff --git a/src/lib9/fmt/fmtrune.c b/src/lib9/fmt/fmtrune.c new file mode 100644 index 000000000..da8c5d746 --- /dev/null +++ b/src/lib9/fmt/fmtrune.c @@ -0,0 +1,43 @@ +/* + * The authors of this software are Rob Pike and Ken Thompson, + * with contributions from Mike Burrows and Sean Dorward. + * + * Copyright (c) 2002-2006 by Lucent Technologies. + * Portions Copyright (c) 2004 Google Inc. + * + * Permission to use, copy, modify, and distribute this software for any + * purpose without fee is hereby granted, provided that this entire notice + * is included in all copies of any software which is or includes a copy + * or modification of this software and in all copies of the supporting + * documentation for such software. + * THIS SOFTWARE IS BEING PROVIDED "AS IS", WITHOUT ANY EXPRESS OR IMPLIED + * WARRANTY. IN PARTICULAR, NEITHER THE AUTHORS NOR LUCENT TECHNOLOGIES + * NOR GOOGLE INC MAKE ANY REPRESENTATION OR WARRANTY OF ANY KIND CONCERNING + * THE MERCHANTABILITY OF THIS SOFTWARE OR ITS FITNESS FOR ANY PARTICULAR PURPOSE. + */ + +#include +#include +#include "fmtdef.h" + +int +fmtrune(Fmt *f, int r) +{ + Rune *rt; + char *t; + int n; + + if(f->runes){ + rt = (Rune*)f->to; + FMTRCHAR(f, rt, f->stop, r); + f->to = rt; + n = 1; + }else{ + t = (char*)f->to; + FMTRUNE(f, t, f->stop, r); + n = t - (char*)f->to; + f->to = t; + } + f->nfmt += n; + return 0; +} diff --git a/src/lib9/fmt/fmtstr.c b/src/lib9/fmt/fmtstr.c new file mode 100644 index 000000000..a6ca7721d --- /dev/null +++ b/src/lib9/fmt/fmtstr.c @@ -0,0 +1,31 @@ +/* + * The authors of this software are Rob Pike and Ken Thompson, + * with contributions from Mike Burrows and Sean Dorward. + * + * Copyright (c) 2002-2006 by Lucent Technologies. + * Portions Copyright (c) 2004 Google Inc. + * + * Permission to use, copy, modify, and distribute this software for any + * purpose without fee is hereby granted, provided that this entire notice + * is included in all copies of any software which is or includes a copy + * or modification of this software and in all copies of the supporting + * documentation for such software. + * THIS SOFTWARE IS BEING PROVIDED "AS IS", WITHOUT ANY EXPRESS OR IMPLIED + * WARRANTY. IN PARTICULAR, NEITHER THE AUTHORS NOR LUCENT TECHNOLOGIES + * NOR GOOGLE INC MAKE ANY REPRESENTATION OR WARRANTY OF ANY KIND CONCERNING + * THE MERCHANTABILITY OF THIS SOFTWARE OR ITS FITNESS FOR ANY PARTICULAR PURPOSE. + */ + +#include +#include +#include "fmtdef.h" + +char* +fmtstrflush(Fmt *f) +{ + if(f->start == nil) + return nil; + *(char*)f->to = '\0'; + f->to = f->start; + return (char*)f->start; +} diff --git a/src/lib9/fmt/fmtvprint.c b/src/lib9/fmt/fmtvprint.c new file mode 100644 index 000000000..6acd37a51 --- /dev/null +++ b/src/lib9/fmt/fmtvprint.c @@ -0,0 +1,52 @@ +/* + * The authors of this software are Rob Pike and Ken Thompson, + * with contributions from Mike Burrows and Sean Dorward. + * + * Copyright (c) 2002-2006 by Lucent Technologies. + * Portions Copyright (c) 2004 Google Inc. + * + * Permission to use, copy, modify, and distribute this software for any + * purpose without fee is hereby granted, provided that this entire notice + * is included in all copies of any software which is or includes a copy + * or modification of this software and in all copies of the supporting + * documentation for such software. + * THIS SOFTWARE IS BEING PROVIDED "AS IS", WITHOUT ANY EXPRESS OR IMPLIED + * WARRANTY. IN PARTICULAR, NEITHER THE AUTHORS NOR LUCENT TECHNOLOGIES + * NOR GOOGLE INC MAKE ANY REPRESENTATION OR WARRANTY OF ANY KIND CONCERNING + * THE MERCHANTABILITY OF THIS SOFTWARE OR ITS FITNESS FOR ANY PARTICULAR PURPOSE. + */ + +#include +#include +#include "fmtdef.h" + + +/* + * format a string into the output buffer + * designed for formats which themselves call fmt, + * but ignore any width flags + */ +int +fmtvprint(Fmt *f, char *fmt, va_list args) +{ + va_list va; + int n; + + f->flags = 0; + f->width = 0; + f->prec = 0; + VA_COPY(va,f->args); + VA_END(f->args); + VA_COPY(f->args,args); + n = dofmt(f, fmt); + f->flags = 0; + f->width = 0; + f->prec = 0; + VA_END(f->args); + VA_COPY(f->args,va); + VA_END(va); + if(n >= 0) + return 0; + return n; +} + diff --git a/src/lib9/fmt/fprint.c b/src/lib9/fmt/fprint.c new file mode 100644 index 000000000..70cb1385a --- /dev/null +++ b/src/lib9/fmt/fprint.c @@ -0,0 +1,33 @@ +/* + * The authors of this software are Rob Pike and Ken Thompson, + * with contributions from Mike Burrows and Sean Dorward. + * + * Copyright (c) 2002-2006 by Lucent Technologies. + * Portions Copyright (c) 2004 Google Inc. + * + * Permission to use, copy, modify, and distribute this software for any + * purpose without fee is hereby granted, provided that this entire notice + * is included in all copies of any software which is or includes a copy + * or modification of this software and in all copies of the supporting + * documentation for such software. + * THIS SOFTWARE IS BEING PROVIDED "AS IS", WITHOUT ANY EXPRESS OR IMPLIED + * WARRANTY. IN PARTICULAR, NEITHER THE AUTHORS NOR LUCENT TECHNOLOGIES + * NOR GOOGLE INC MAKE ANY REPRESENTATION OR WARRANTY OF ANY KIND CONCERNING + * THE MERCHANTABILITY OF THIS SOFTWARE OR ITS FITNESS FOR ANY PARTICULAR PURPOSE. + */ + +#include +#include +#include "fmtdef.h" + +int +fprint(int fd, char *fmt, ...) +{ + int n; + va_list args; + + va_start(args, fmt); + n = vfprint(fd, fmt, args); + va_end(args); + return n; +} diff --git a/src/lib9/fmt/nan64.c b/src/lib9/fmt/nan64.c new file mode 100644 index 000000000..1ea702741 --- /dev/null +++ b/src/lib9/fmt/nan64.c @@ -0,0 +1,93 @@ +/* + * The authors of this software are Rob Pike and Ken Thompson, + * with contributions from Mike Burrows and Sean Dorward. + * + * Copyright (c) 2002-2006 by Lucent Technologies. + * Portions Copyright (c) 2004 Google Inc. + * + * Permission to use, copy, modify, and distribute this software for any + * purpose without fee is hereby granted, provided that this entire notice + * is included in all copies of any software which is or includes a copy + * or modification of this software and in all copies of the supporting + * documentation for such software. + * THIS SOFTWARE IS BEING PROVIDED "AS IS", WITHOUT ANY EXPRESS OR IMPLIED + * WARRANTY. IN PARTICULAR, NEITHER THE AUTHORS NOR LUCENT TECHNOLOGIES + * NOR GOOGLE INC MAKE ANY REPRESENTATION OR WARRANTY OF ANY KIND CONCERNING + * THE MERCHANTABILITY OF THIS SOFTWARE OR ITS FITNESS FOR ANY PARTICULAR PURPOSE. + */ + +/* + * 64-bit IEEE not-a-number routines. + * This is big/little-endian portable assuming that + * the 64-bit doubles and 64-bit integers have the + * same byte ordering. + */ + +#include +#include +#include "fmtdef.h" + +static uvlong uvnan = ((uvlong)0x7FF00000<<32)|0x00000001; +static uvlong uvinf = ((uvlong)0x7FF00000<<32)|0x00000000; +static uvlong uvneginf = ((uvlong)0xFFF00000<<32)|0x00000000; + +/* gcc sees through the obvious casts. */ +static uvlong +d2u(double d) +{ + union { + uvlong v; + double d; + } u; + assert(sizeof(u.d) == sizeof(u.v)); + u.d = d; + return u.v; +} + +static double +u2d(uvlong v) +{ + union { + uvlong v; + double d; + } u; + assert(sizeof(u.d) == sizeof(u.v)); + u.v = v; + return u.d; +} + +double +__NaN(void) +{ + return u2d(uvnan); +} + +int +__isNaN(double d) +{ + uvlong x; + + x = d2u(d); + /* IEEE 754: exponent bits 0x7FF and non-zero mantissa */ + return (x&uvinf) == uvinf && (x&~uvneginf) != 0; +} + +double +__Inf(int sign) +{ + return u2d(sign < 0 ? uvneginf : uvinf); +} + +int +__isInf(double d, int sign) +{ + uvlong x; + + x = d2u(d); + if(sign == 0) + return x==uvinf || x==uvneginf; + else if(sign > 0) + return x==uvinf; + else + return x==uvneginf; +} diff --git a/src/lib9/fmt/pow10.c b/src/lib9/fmt/pow10.c new file mode 100644 index 000000000..e146884a8 --- /dev/null +++ b/src/lib9/fmt/pow10.c @@ -0,0 +1,60 @@ +/* + * The authors of this software are Rob Pike and Ken Thompson, + * with contributions from Mike Burrows and Sean Dorward. + * + * Copyright (c) 2002-2006 by Lucent Technologies. + * Portions Copyright (c) 2004 Google Inc. + * + * Permission to use, copy, modify, and distribute this software for any + * purpose without fee is hereby granted, provided that this entire notice + * is included in all copies of any software which is or includes a copy + * or modification of this software and in all copies of the supporting + * documentation for such software. + * THIS SOFTWARE IS BEING PROVIDED "AS IS", WITHOUT ANY EXPRESS OR IMPLIED + * WARRANTY. IN PARTICULAR, NEITHER THE AUTHORS NOR LUCENT TECHNOLOGIES + * NOR GOOGLE INC MAKE ANY REPRESENTATION OR WARRANTY OF ANY KIND CONCERNING + * THE MERCHANTABILITY OF THIS SOFTWARE OR ITS FITNESS FOR ANY PARTICULAR PURPOSE. + */ + +#include +#include +#include "fmtdef.h" + +/* + * this table might overflow 127-bit exponent representations. + * in that case, truncate it after 1.0e38. + * it is important to get all one can from this + * routine since it is used in atof to scale numbers. + * the presumption is that C converts fp numbers better + * than multipication of lower powers of 10. + */ + +static +double tab[] = +{ + 1.0e0, 1.0e1, 1.0e2, 1.0e3, 1.0e4, 1.0e5, 1.0e6, 1.0e7, 1.0e8, 1.0e9, + 1.0e10,1.0e11,1.0e12,1.0e13,1.0e14,1.0e15,1.0e16,1.0e17,1.0e18,1.0e19, + 1.0e20,1.0e21,1.0e22,1.0e23,1.0e24,1.0e25,1.0e26,1.0e27,1.0e28,1.0e29, + 1.0e30,1.0e31,1.0e32,1.0e33,1.0e34,1.0e35,1.0e36,1.0e37,1.0e38,1.0e39, + 1.0e40,1.0e41,1.0e42,1.0e43,1.0e44,1.0e45,1.0e46,1.0e47,1.0e48,1.0e49, + 1.0e50,1.0e51,1.0e52,1.0e53,1.0e54,1.0e55,1.0e56,1.0e57,1.0e58,1.0e59, + 1.0e60,1.0e61,1.0e62,1.0e63,1.0e64,1.0e65,1.0e66,1.0e67,1.0e68,1.0e69, +}; + +double +__fmtpow10(int n) +{ + int m; + + if(n < 0) { + n = -n; + if(n < (int)(sizeof(tab)/sizeof(tab[0]))) + return 1/tab[n]; + m = n/2; + return __fmtpow10(-m) * __fmtpow10(m-n); + } + if(n < (int)(sizeof(tab)/sizeof(tab[0]))) + return tab[n]; + m = n/2; + return __fmtpow10(m) * __fmtpow10(n-m); +} diff --git a/src/lib9/fmt/print.c b/src/lib9/fmt/print.c new file mode 100644 index 000000000..5c39457d6 --- /dev/null +++ b/src/lib9/fmt/print.c @@ -0,0 +1,33 @@ +/* + * The authors of this software are Rob Pike and Ken Thompson, + * with contributions from Mike Burrows and Sean Dorward. + * + * Copyright (c) 2002-2006 by Lucent Technologies. + * Portions Copyright (c) 2004 Google Inc. + * + * Permission to use, copy, modify, and distribute this software for any + * purpose without fee is hereby granted, provided that this entire notice + * is included in all copies of any software which is or includes a copy + * or modification of this software and in all copies of the supporting + * documentation for such software. + * THIS SOFTWARE IS BEING PROVIDED "AS IS", WITHOUT ANY EXPRESS OR IMPLIED + * WARRANTY. IN PARTICULAR, NEITHER THE AUTHORS NOR LUCENT TECHNOLOGIES + * NOR GOOGLE INC MAKE ANY REPRESENTATION OR WARRANTY OF ANY KIND CONCERNING + * THE MERCHANTABILITY OF THIS SOFTWARE OR ITS FITNESS FOR ANY PARTICULAR PURPOSE. + */ + +#include +#include +#include "fmtdef.h" + +int +print(char *fmt, ...) +{ + int n; + va_list args; + + va_start(args, fmt); + n = vfprint(1, fmt, args); + va_end(args); + return n; +} diff --git a/src/lib9/fmt/seprint.c b/src/lib9/fmt/seprint.c new file mode 100644 index 000000000..88779d90a --- /dev/null +++ b/src/lib9/fmt/seprint.c @@ -0,0 +1,33 @@ +/* + * The authors of this software are Rob Pike and Ken Thompson, + * with contributions from Mike Burrows and Sean Dorward. + * + * Copyright (c) 2002-2006 by Lucent Technologies. + * Portions Copyright (c) 2004 Google Inc. + * + * Permission to use, copy, modify, and distribute this software for any + * purpose without fee is hereby granted, provided that this entire notice + * is included in all copies of any software which is or includes a copy + * or modification of this software and in all copies of the supporting + * documentation for such software. + * THIS SOFTWARE IS BEING PROVIDED "AS IS", WITHOUT ANY EXPRESS OR IMPLIED + * WARRANTY. IN PARTICULAR, NEITHER THE AUTHORS NOR LUCENT TECHNOLOGIES + * NOR GOOGLE INC MAKE ANY REPRESENTATION OR WARRANTY OF ANY KIND CONCERNING + * THE MERCHANTABILITY OF THIS SOFTWARE OR ITS FITNESS FOR ANY PARTICULAR PURPOSE. + */ + +#include +#include +#include "fmtdef.h" + +char* +seprint(char *buf, char *e, char *fmt, ...) +{ + char *p; + va_list args; + + va_start(args, fmt); + p = vseprint(buf, e, fmt, args); + va_end(args); + return p; +} diff --git a/src/lib9/fmt/smprint.c b/src/lib9/fmt/smprint.c new file mode 100644 index 000000000..c13ffd7dd --- /dev/null +++ b/src/lib9/fmt/smprint.c @@ -0,0 +1,33 @@ +/* + * The authors of this software are Rob Pike and Ken Thompson, + * with contributions from Mike Burrows and Sean Dorward. + * + * Copyright (c) 2002-2006 by Lucent Technologies. + * Portions Copyright (c) 2004 Google Inc. + * + * Permission to use, copy, modify, and distribute this software for any + * purpose without fee is hereby granted, provided that this entire notice + * is included in all copies of any software which is or includes a copy + * or modification of this software and in all copies of the supporting + * documentation for such software. + * THIS SOFTWARE IS BEING PROVIDED "AS IS", WITHOUT ANY EXPRESS OR IMPLIED + * WARRANTY. IN PARTICULAR, NEITHER THE AUTHORS NOR LUCENT TECHNOLOGIES + * NOR GOOGLE INC MAKE ANY REPRESENTATION OR WARRANTY OF ANY KIND CONCERNING + * THE MERCHANTABILITY OF THIS SOFTWARE OR ITS FITNESS FOR ANY PARTICULAR PURPOSE. + */ + +#include +#include +#include "fmtdef.h" + +char* +smprint(char *fmt, ...) +{ + va_list args; + char *p; + + va_start(args, fmt); + p = vsmprint(fmt, args); + va_end(args); + return p; +} diff --git a/src/lib9/fmt/snprint.c b/src/lib9/fmt/snprint.c new file mode 100644 index 000000000..372399c44 --- /dev/null +++ b/src/lib9/fmt/snprint.c @@ -0,0 +1,34 @@ +/* + * The authors of this software are Rob Pike and Ken Thompson, + * with contributions from Mike Burrows and Sean Dorward. + * + * Copyright (c) 2002-2006 by Lucent Technologies. + * Portions Copyright (c) 2004 Google Inc. + * + * Permission to use, copy, modify, and distribute this software for any + * purpose without fee is hereby granted, provided that this entire notice + * is included in all copies of any software which is or includes a copy + * or modification of this software and in all copies of the supporting + * documentation for such software. + * THIS SOFTWARE IS BEING PROVIDED "AS IS", WITHOUT ANY EXPRESS OR IMPLIED + * WARRANTY. IN PARTICULAR, NEITHER THE AUTHORS NOR LUCENT TECHNOLOGIES + * NOR GOOGLE INC MAKE ANY REPRESENTATION OR WARRANTY OF ANY KIND CONCERNING + * THE MERCHANTABILITY OF THIS SOFTWARE OR ITS FITNESS FOR ANY PARTICULAR PURPOSE. + */ + +#include +#include +#include "fmtdef.h" + +int +snprint(char *buf, int len, char *fmt, ...) +{ + int n; + va_list args; + + va_start(args, fmt); + n = vsnprint(buf, len, fmt, args); + va_end(args); + return n; +} + diff --git a/src/lib9/fmt/sprint.c b/src/lib9/fmt/sprint.c new file mode 100644 index 000000000..38d430744 --- /dev/null +++ b/src/lib9/fmt/sprint.c @@ -0,0 +1,45 @@ +/* + * The authors of this software are Rob Pike and Ken Thompson, + * with contributions from Mike Burrows and Sean Dorward. + * + * Copyright (c) 2002-2006 by Lucent Technologies. + * Portions Copyright (c) 2004 Google Inc. + * + * Permission to use, copy, modify, and distribute this software for any + * purpose without fee is hereby granted, provided that this entire notice + * is included in all copies of any software which is or includes a copy + * or modification of this software and in all copies of the supporting + * documentation for such software. + * THIS SOFTWARE IS BEING PROVIDED "AS IS", WITHOUT ANY EXPRESS OR IMPLIED + * WARRANTY. IN PARTICULAR, NEITHER THE AUTHORS NOR LUCENT TECHNOLOGIES + * NOR GOOGLE INC MAKE ANY REPRESENTATION OR WARRANTY OF ANY KIND CONCERNING + * THE MERCHANTABILITY OF THIS SOFTWARE OR ITS FITNESS FOR ANY PARTICULAR PURPOSE. + */ + +#include +#include +#include "fmtdef.h" + +int +sprint(char *buf, char *fmt, ...) +{ + int n; + uint len; + va_list args; + + len = 1<<30; /* big number, but sprint is deprecated anyway */ + /* + * on PowerPC, the stack is near the top of memory, so + * we must be sure not to overflow a 32-bit pointer. + * + * careful! gcc-4.2 assumes buf+len < buf can never be true and + * optimizes the test away. casting to uintptr works around this bug. + */ + if((uintptr)buf+len < (uintptr)buf) + len = -(uintptr)buf-1; + + va_start(args, fmt); + n = vsnprint(buf, len, fmt, args); + va_end(args); + return n; +} diff --git a/src/lib9/fmt/strtod.c b/src/lib9/fmt/strtod.c new file mode 100644 index 000000000..6bb56c112 --- /dev/null +++ b/src/lib9/fmt/strtod.c @@ -0,0 +1,532 @@ +/* + * The authors of this software are Rob Pike and Ken Thompson, + * with contributions from Mike Burrows and Sean Dorward. + * + * Copyright (c) 2002-2006 by Lucent Technologies. + * Portions Copyright (c) 2004 Google Inc. + * + * Permission to use, copy, modify, and distribute this software for any + * purpose without fee is hereby granted, provided that this entire notice + * is included in all copies of any software which is or includes a copy + * or modification of this software and in all copies of the supporting + * documentation for such software. + * THIS SOFTWARE IS BEING PROVIDED "AS IS", WITHOUT ANY EXPRESS OR IMPLIED + * WARRANTY. IN PARTICULAR, NEITHER THE AUTHORS NOR LUCENT TECHNOLOGIES + * NOR GOOGLE INC MAKE ANY REPRESENTATION OR WARRANTY OF ANY KIND CONCERNING + * THE MERCHANTABILITY OF THIS SOFTWARE OR ITS FITNESS FOR ANY PARTICULAR PURPOSE. + */ + +#include +#include +#include +#include "fmtdef.h" + +static ulong +umuldiv(ulong a, ulong b, ulong c) +{ + double d; + + d = ((double)a * (double)b) / (double)c; + if(d >= 4294967295.) + d = 4294967295.; + return (ulong)d; +} + +/* + * This routine will convert to arbitrary precision + * floating point entirely in multi-precision fixed. + * The answer is the closest floating point number to + * the given decimal number. Exactly half way are + * rounded ala ieee rules. + * Method is to scale input decimal between .500 and .999... + * with external power of 2, then binary search for the + * closest mantissa to this decimal number. + * Nmant is is the required precision. (53 for ieee dp) + * Nbits is the max number of bits/word. (must be <= 28) + * Prec is calculated - the number of words of fixed mantissa. + */ +enum +{ + Nbits = 28, /* bits safely represented in a ulong */ + Nmant = 53, /* bits of precision required */ + Prec = (Nmant+Nbits+1)/Nbits, /* words of Nbits each to represent mantissa */ + Sigbit = 1<<(Prec*Nbits-Nmant), /* first significant bit of Prec-th word */ + Ndig = 1500, + One = (ulong)(1<>1), + Maxe = 310, + + Fsign = 1<<0, /* found - */ + Fesign = 1<<1, /* found e- */ + Fdpoint = 1<<2, /* found . */ + + S0 = 0, /* _ _S0 +S1 #S2 .S3 */ + S1, /* _+ #S2 .S3 */ + S2, /* _+# #S2 .S4 eS5 */ + S3, /* _+. #S4 */ + S4, /* _+#.# #S4 eS5 */ + S5, /* _+#.#e +S6 #S7 */ + S6, /* _+#.#e+ #S7 */ + S7 /* _+#.#e+# #S7 */ +}; + +static int xcmp(char*, char*); +static int fpcmp(char*, ulong*); +static void frnorm(ulong*); +static void divascii(char*, int*, int*, int*); +static void mulascii(char*, int*, int*, int*); + +typedef struct Tab Tab; +struct Tab +{ + int bp; + int siz; + char* cmp; +}; + +double +fmtstrtod(const char *as, char **aas) +{ + int na, ex, dp, bp, c, i, flag, state; + ulong low[Prec], hig[Prec], mid[Prec]; + double d; + char *s, a[Ndig]; + + flag = 0; /* Fsign, Fesign, Fdpoint */ + na = 0; /* number of digits of a[] */ + dp = 0; /* na of decimal point */ + ex = 0; /* exonent */ + + state = S0; + for(s=(char*)as;; s++) { + c = *s; + if(c >= '0' && c <= '9') { + switch(state) { + case S0: + case S1: + case S2: + state = S2; + break; + case S3: + case S4: + state = S4; + break; + + case S5: + case S6: + case S7: + state = S7; + ex = ex*10 + (c-'0'); + continue; + } + if(na == 0 && c == '0') { + dp--; + continue; + } + if(na < Ndig-50) + a[na++] = c; + continue; + } + switch(c) { + case '\t': + case '\n': + case '\v': + case '\f': + case '\r': + case ' ': + if(state == S0) + continue; + break; + case '-': + if(state == S0) + flag |= Fsign; + else + flag |= Fesign; + case '+': + if(state == S0) + state = S1; + else + if(state == S5) + state = S6; + else + break; /* syntax */ + continue; + case '.': + flag |= Fdpoint; + dp = na; + if(state == S0 || state == S1) { + state = S3; + continue; + } + if(state == S2) { + state = S4; + continue; + } + break; + case 'e': + case 'E': + if(state == S2 || state == S4) { + state = S5; + continue; + } + break; + } + break; + } + + /* + * clean up return char-pointer + */ + switch(state) { + case S0: + if(xcmp(s, "nan") == 0) { + if(aas != nil) + *aas = s+3; + goto retnan; + } + case S1: + if(xcmp(s, "infinity") == 0) { + if(aas != nil) + *aas = s+8; + goto retinf; + } + if(xcmp(s, "inf") == 0) { + if(aas != nil) + *aas = s+3; + goto retinf; + } + case S3: + if(aas != nil) + *aas = (char*)as; + goto ret0; /* no digits found */ + case S6: + s--; /* back over +- */ + case S5: + s--; /* back over e */ + break; + } + if(aas != nil) + *aas = s; + + if(flag & Fdpoint) + while(na > 0 && a[na-1] == '0') + na--; + if(na == 0) + goto ret0; /* zero */ + a[na] = 0; + if(!(flag & Fdpoint)) + dp = na; + if(flag & Fesign) + ex = -ex; + dp += ex; + if(dp < -Maxe){ + errno = ERANGE; + goto ret0; /* underflow by exp */ + } else + if(dp > +Maxe) + goto retinf; /* overflow by exp */ + + /* + * normalize the decimal ascii number + * to range .[5-9][0-9]* e0 + */ + bp = 0; /* binary exponent */ + while(dp > 0) + divascii(a, &na, &dp, &bp); + while(dp < 0 || a[0] < '5') + mulascii(a, &na, &dp, &bp); + + /* close approx by naive conversion */ + mid[0] = 0; + mid[1] = 1; + for(i=0; (c=a[i]) != '\0'; i++) { + mid[0] = mid[0]*10 + (c-'0'); + mid[1] = mid[1]*10; + if(i >= 8) + break; + } + low[0] = umuldiv(mid[0], One, mid[1]); + hig[0] = umuldiv(mid[0]+1, One, mid[1]); + for(i=1; i>= 1; + } + frnorm(mid); + + /* compare */ + c = fpcmp(a, mid); + if(c > 0) { + c = 1; + for(i=0; i= Sigbit/2) { + mid[Prec-1] += Sigbit; + frnorm(mid); + } + goto out; + +ret0: + return 0; + +retnan: + return __NaN(); + +retinf: + /* + * Unix strtod requires these. Plan 9 would return Inf(0) or Inf(-1). */ + errno = ERANGE; + if(flag & Fsign) + return -HUGE_VAL; + return HUGE_VAL; + +out: + d = 0; + for(i=0; i0; i--) { + f[i] += c; + c = f[i] >> Nbits; + f[i] &= One-1; + } + f[0] += c; +} + +static int +fpcmp(char *a, ulong* f) +{ + ulong tf[Prec]; + int i, d, c; + + for(i=0; i> Nbits) + '0'; + tf[0] &= One-1; + + /* compare next digit */ + c = *a; + if(c == 0) { + if('0' < d) + return -1; + if(tf[0] != 0) + goto cont; + for(i=1; i d) + return +1; + if(c < d) + return -1; + a++; + cont:; + } +} + +static void +divby(char *a, int *na, int b) +{ + int n, c; + char *p; + + p = a; + n = 0; + while(n>>b == 0) { + c = *a++; + if(c == 0) { + while(n) { + c = n*10; + if(c>>b) + break; + n = c; + } + goto xx; + } + n = n*10 + c-'0'; + (*na)--; + } + for(;;) { + c = n>>b; + n -= c<>b; + n -= c<= (int)(nelem(tab1))) + d = (int)(nelem(tab1))-1; + t = tab1 + d; + b = t->bp; + if(memcmp(a, t->cmp, t->siz) > 0) + d--; + *dp -= d; + *bp += b; + divby(a, na, b); +} + +static void +mulby(char *a, char *p, char *q, int b) +{ + int n, c; + + n = 0; + *p = 0; + for(;;) { + q--; + if(q < a) + break; + c = *q - '0'; + c = (c<= (int)(nelem(tab2))) + d = (int)(nelem(tab2))-1; + t = tab2 + d; + b = t->bp; + if(memcmp(a, t->cmp, t->siz) < 0) + d--; + p = a + *na; + *bp -= b; + *dp += d; + *na += d; + mulby(a, p+d, p, b); +} + +static int +xcmp(char *a, char *b) +{ + int c1, c2; + + while((c1 = *b++) != '\0') { + c2 = *a++; + if(isupper(c2)) + c2 = tolower(c2); + if(c1 != c2) + return 1; + } + return 0; +} diff --git a/src/lib9/fmt/test.c b/src/lib9/fmt/test.c new file mode 100644 index 000000000..1710c5e48 --- /dev/null +++ b/src/lib9/fmt/test.c @@ -0,0 +1,65 @@ +/* + * The authors of this software are Rob Pike and Ken Thompson, + * with contributions from Mike Burrows and Sean Dorward. + * + * Copyright (c) 2002-2006 by Lucent Technologies. + * Portions Copyright (c) 2004 Google Inc. + * + * Permission to use, copy, modify, and distribute this software for any + * purpose without fee is hereby granted, provided that this entire notice + * is included in all copies of any software which is or includes a copy + * or modification of this software and in all copies of the supporting + * documentation for such software. + * THIS SOFTWARE IS BEING PROVIDED "AS IS", WITHOUT ANY EXPRESS OR IMPLIED + * WARRANTY. IN PARTICULAR, NEITHER THE AUTHORS NOR LUCENT TECHNOLOGIES + * NOR GOOGLE INC MAKE ANY REPRESENTATION OR WARRANTY OF ANY KIND CONCERNING + * THE MERCHANTABILITY OF THIS SOFTWARE OR ITS FITNESS FOR ANY PARTICULAR PURPOSE. + */ + +#include +#include +#include "fmtdef.h" + +int +main(int argc, char *argv[]) +{ + quotefmtinstall(); + print("hello world\n"); + print("x: %x\n", 0x87654321); + print("u: %u\n", 0x87654321); + print("d: %d\n", 0x87654321); + print("s: %s\n", "hi there"); + print("q: %q\n", "hi i'm here"); + print("c: %c\n", '!'); + print("g: %g %g %g\n", 3.14159, 3.14159e10, 3.14159e-10); + print("e: %e %e %e\n", 3.14159, 3.14159e10, 3.14159e-10); + print("f: %f %f %f\n", 3.14159, 3.14159e10, 3.14159e-10); + print("smiley: %C\n", (Rune)0x263a); + print("%g %.18g\n", 2e25, 2e25); + print("%2.18g\n", 1.0); + print("%2.18f\n", 1.0); + print("%f\n", 3.1415927/4); + print("%d\n", 23); + print("%i\n", 23); + print("%0.10d\n", 12345); + + /* test %4$d formats */ + print("%3$d %4$06d %2$d %1$d\n", 444, 333, 111, 222); + print("%3$d %4$06d %2$d %1$d\n", 444, 333, 111, 222); + print("%3$d %4$*5$06d %2$d %1$d\n", 444, 333, 111, 222, 20); + print("%3$hd %4$*5$06d %2$d %1$d\n", 444, 333, (short)111, 222, 20); + print("%3$lld %4$*5$06d %2$d %1$d\n", 444, 333, 111LL, 222, 20); + + /* test %'d formats */ + print("%'d %'d %'d\n", 1, 2222, 33333333); + print("%'019d\n", 0); + print("%08d %08d %08d\n", 1, 2222, 33333333); + print("%'08d %'08d %'08d\n", 1, 2222, 33333333); + print("%'x %'X %'b\n", 0x11111111, 0xabcd1234, 12345); + print("%'lld %'lld %'lld\n", 1LL, 222222222LL, 3333333333333LL); + print("%019lld %019lld %019lld\n", 1LL, 222222222LL, 3333333333333LL); + print("%'019lld %'019lld %'019lld\n", 1LL, 222222222LL, 3333333333333LL); + print("%'020lld %'020lld %'020lld\n", 1LL, 222222222LL, 3333333333333LL); + print("%'llx %'llX %'llb\n", 0x111111111111LL, 0xabcd12345678LL, 112342345LL); + return 0; +} diff --git a/src/lib9/fmt/vfprint.c b/src/lib9/fmt/vfprint.c new file mode 100644 index 000000000..a23c5a0c6 --- /dev/null +++ b/src/lib9/fmt/vfprint.c @@ -0,0 +1,37 @@ +/* + * The authors of this software are Rob Pike and Ken Thompson, + * with contributions from Mike Burrows and Sean Dorward. + * + * Copyright (c) 2002-2006 by Lucent Technologies. + * Portions Copyright (c) 2004 Google Inc. + * + * Permission to use, copy, modify, and distribute this software for any + * purpose without fee is hereby granted, provided that this entire notice + * is included in all copies of any software which is or includes a copy + * or modification of this software and in all copies of the supporting + * documentation for such software. + * THIS SOFTWARE IS BEING PROVIDED "AS IS", WITHOUT ANY EXPRESS OR IMPLIED + * WARRANTY. IN PARTICULAR, NEITHER THE AUTHORS NOR LUCENT TECHNOLOGIES + * NOR GOOGLE INC MAKE ANY REPRESENTATION OR WARRANTY OF ANY KIND CONCERNING + * THE MERCHANTABILITY OF THIS SOFTWARE OR ITS FITNESS FOR ANY PARTICULAR PURPOSE. + */ + +#include +#include +#include "fmtdef.h" + +int +vfprint(int fd, char *fmt, va_list args) +{ + Fmt f; + char buf[256]; + int n; + + fmtfdinit(&f, fd, buf, sizeof(buf)); + VA_COPY(f.args,args); + n = dofmt(&f, fmt); + VA_END(f.args); + if(n > 0 && __fmtFdFlush(&f) == 0) + return -1; + return n; +} diff --git a/src/lib9/fmt/vseprint.c b/src/lib9/fmt/vseprint.c new file mode 100644 index 000000000..c9fbfb956 --- /dev/null +++ b/src/lib9/fmt/vseprint.c @@ -0,0 +1,44 @@ +/* + * The authors of this software are Rob Pike and Ken Thompson, + * with contributions from Mike Burrows and Sean Dorward. + * + * Copyright (c) 2002-2006 by Lucent Technologies. + * Portions Copyright (c) 2004 Google Inc. + * + * Permission to use, copy, modify, and distribute this software for any + * purpose without fee is hereby granted, provided that this entire notice + * is included in all copies of any software which is or includes a copy + * or modification of this software and in all copies of the supporting + * documentation for such software. + * THIS SOFTWARE IS BEING PROVIDED "AS IS", WITHOUT ANY EXPRESS OR IMPLIED + * WARRANTY. IN PARTICULAR, NEITHER THE AUTHORS NOR LUCENT TECHNOLOGIES + * NOR GOOGLE INC MAKE ANY REPRESENTATION OR WARRANTY OF ANY KIND CONCERNING + * THE MERCHANTABILITY OF THIS SOFTWARE OR ITS FITNESS FOR ANY PARTICULAR PURPOSE. + */ + +#include +#include +#include "fmtdef.h" + +char* +vseprint(char *buf, char *e, char *fmt, va_list args) +{ + Fmt f; + + if(e <= buf) + return nil; + f.runes = 0; + f.start = buf; + f.to = buf; + f.stop = e - 1; + f.flush = 0; + f.farg = nil; + f.nfmt = 0; + VA_COPY(f.args,args); + fmtlocaleinit(&f, nil, nil, nil); + dofmt(&f, fmt); + VA_END(f.args); + *(char*)f.to = '\0'; + return (char*)f.to; +} + diff --git a/src/lib9/fmt/vsmprint.c b/src/lib9/fmt/vsmprint.c new file mode 100644 index 000000000..4bd0bc4b7 --- /dev/null +++ b/src/lib9/fmt/vsmprint.c @@ -0,0 +1,87 @@ +/* + * The authors of this software are Rob Pike and Ken Thompson, + * with contributions from Mike Burrows and Sean Dorward. + * + * Copyright (c) 2002-2006 by Lucent Technologies. + * Portions Copyright (c) 2004 Google Inc. + * + * Permission to use, copy, modify, and distribute this software for any + * purpose without fee is hereby granted, provided that this entire notice + * is included in all copies of any software which is or includes a copy + * or modification of this software and in all copies of the supporting + * documentation for such software. + * THIS SOFTWARE IS BEING PROVIDED "AS IS", WITHOUT ANY EXPRESS OR IMPLIED + * WARRANTY. IN PARTICULAR, NEITHER THE AUTHORS NOR LUCENT TECHNOLOGIES + * NOR GOOGLE INC MAKE ANY REPRESENTATION OR WARRANTY OF ANY KIND CONCERNING + * THE MERCHANTABILITY OF THIS SOFTWARE OR ITS FITNESS FOR ANY PARTICULAR PURPOSE. + */ + +#include +#include +#include "fmtdef.h" + +static int +fmtStrFlush(Fmt *f) +{ + char *s; + int n; + + if(f->start == nil) + return 0; + n = (uintptr)f->farg; + n *= 2; + s = (char*)f->start; + f->start = realloc(s, n); + if(f->start == nil){ + f->farg = nil; + f->to = nil; + f->stop = nil; + free(s); + return 0; + } + f->farg = (void*)(uintptr)n; + f->to = (char*)f->start + ((char*)f->to - s); + f->stop = (char*)f->start + n - 1; + return 1; +} + +int +fmtstrinit(Fmt *f) +{ + int n; + + memset(f, 0, sizeof *f); + f->runes = 0; + n = 32; + f->start = malloc(n); + if(f->start == nil) + return -1; + f->to = f->start; + f->stop = (char*)f->start + n - 1; + f->flush = fmtStrFlush; + f->farg = (void*)(uintptr)n; + f->nfmt = 0; + fmtlocaleinit(f, nil, nil, nil); + return 0; +} + +/* + * print into an allocated string buffer + */ +char* +vsmprint(char *fmt, va_list args) +{ + Fmt f; + int n; + + if(fmtstrinit(&f) < 0) + return nil; + VA_COPY(f.args,args); + n = dofmt(&f, fmt); + VA_END(f.args); + if(n < 0){ + free(f.start); + return nil; + } + return fmtstrflush(&f); +} diff --git a/src/lib9/fmt/vsnprint.c b/src/lib9/fmt/vsnprint.c new file mode 100644 index 000000000..33d6bba4d --- /dev/null +++ b/src/lib9/fmt/vsnprint.c @@ -0,0 +1,43 @@ +/* + * The authors of this software are Rob Pike and Ken Thompson, + * with contributions from Mike Burrows and Sean Dorward. + * + * Copyright (c) 2002-2006 by Lucent Technologies. + * Portions Copyright (c) 2004 Google Inc. + * + * Permission to use, copy, modify, and distribute this software for any + * purpose without fee is hereby granted, provided that this entire notice + * is included in all copies of any software which is or includes a copy + * or modification of this software and in all copies of the supporting + * documentation for such software. + * THIS SOFTWARE IS BEING PROVIDED "AS IS", WITHOUT ANY EXPRESS OR IMPLIED + * WARRANTY. IN PARTICULAR, NEITHER THE AUTHORS NOR LUCENT TECHNOLOGIES + * NOR GOOGLE INC MAKE ANY REPRESENTATION OR WARRANTY OF ANY KIND CONCERNING + * THE MERCHANTABILITY OF THIS SOFTWARE OR ITS FITNESS FOR ANY PARTICULAR PURPOSE. + */ + +#include +#include +#include "fmtdef.h" + +int +vsnprint(char *buf, int len, char *fmt, va_list args) +{ + Fmt f; + + if(len <= 0) + return -1; + f.runes = 0; + f.start = buf; + f.to = buf; + f.stop = buf + len - 1; + f.flush = 0; + f.farg = nil; + f.nfmt = 0; + VA_COPY(f.args,args); + fmtlocaleinit(&f, nil, nil, nil); + dofmt(&f, fmt); + VA_END(f.args); + *(char*)f.to = '\0'; + return (char*)f.to - buf; +} diff --git a/src/lib9/fmtlock2.c b/src/lib9/fmtlock2.c new file mode 100644 index 000000000..75406b5d1 --- /dev/null +++ b/src/lib9/fmtlock2.c @@ -0,0 +1,38 @@ +/* +Plan 9 from User Space src/lib9/fmtlock2.c +http://code.swtch.com/plan9port/src/tip/src/lib9/fmtlock2.c + +Copyright 2001-2007 Russ Cox. All Rights Reserved. +Portions Copyright 2009 The Go Authors. All Rights Reserved. + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. +*/ + +#include +#include + +void +__fmtlock(void) +{ +} + +void +__fmtunlock(void) +{ +} diff --git a/src/lib9/fork.c b/src/lib9/fork.c new file mode 100644 index 000000000..0dd79dfb8 --- /dev/null +++ b/src/lib9/fork.c @@ -0,0 +1,46 @@ +/* +Plan 9 from User Space src/lib9/fork.c +http://code.swtch.com/plan9port/src/tip/src/lib9/fork.c + +Copyright 2001-2007 Russ Cox. All Rights Reserved. + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. +*/ +#include +#include +#include +#include "9proc.h" +#undef fork + +int +p9fork(void) +{ + int pid; + sigset_t all, old; + + sigfillset(&all); + sigprocmask(SIG_SETMASK, &all, &old); + pid = fork(); + if(pid == 0){ + _clearuproc(); + _p9uproc(0); + } + sigprocmask(SIG_SETMASK, &old, nil); + return pid; +} diff --git a/src/lib9/getenv.c b/src/lib9/getenv.c new file mode 100644 index 000000000..9d805b516 --- /dev/null +++ b/src/lib9/getenv.c @@ -0,0 +1,50 @@ +/* +Plan 9 from User Space src/lib9/getenv.c +http://code.swtch.com/plan9port/src/tip/src/lib9/getenv.c + +Copyright 2001-2007 Russ Cox. All Rights Reserved. + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. +*/ +#include +#define NOPLAN9DEFINES +#include + +char* +p9getenv(char *s) +{ + char *t; + + t = getenv(s); + if(t == 0) + return 0; + return strdup(t); +} + +int +p9putenv(char *s, char *v) +{ + char *t; + + t = smprint("%s=%s", s, v); + if(t == nil) + return -1; + putenv(t); + return 0; +} diff --git a/src/lib9/getfields.c b/src/lib9/getfields.c new file mode 100644 index 000000000..0af8388da --- /dev/null +++ b/src/lib9/getfields.c @@ -0,0 +1,63 @@ +/* +Inferno libkern/getfields.c +http://code.google.com/p/inferno-os/source/browse/libkern/getfields.c + + Copyright © 1994-1999 Lucent Technologies Inc. All rights reserved. + Revisions Copyright © 2000-2007 Vita Nuova Holdings Limited (www.vitanuova.com). All rights reserved. + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. +*/ + +#include +#include + +int +getfields(char *str, char **args, int max, int mflag, char *set) +{ + Rune r; + int nr, intok, narg; + + if(max <= 0) + return 0; + + narg = 0; + args[narg] = str; + if(!mflag) + narg++; + intok = 0; + for(;; str += nr) { + nr = chartorune(&r, str); + if(r == 0) + break; + if(utfrune(set, r)) { + if(narg >= max) + break; + *str = 0; + intok = 0; + args[narg] = str + nr; + if(!mflag) + narg++; + } else { + if(!intok && mflag) + narg++; + intok = 1; + } + } + return narg; +} diff --git a/src/lib9/getuser.c b/src/lib9/getuser.c new file mode 100644 index 000000000..f70b35c87 --- /dev/null +++ b/src/lib9/getuser.c @@ -0,0 +1,41 @@ +/* +Plan 9 from User Space src/lib9/getuser.c +http://code.swtch.com/plan9port/src/tip/src/lib9/getuser.c + +Copyright 2001-2007 Russ Cox. All Rights Reserved. + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. +*/ + +#include +#include +#include + +char* +getuser(void) +{ + static char user[64]; + struct passwd *pw; + + pw = getpwuid(getuid()); + if(pw == nil) + return "none"; + strecpy(user, user+sizeof user, pw->pw_name); + return user; +} diff --git a/src/lib9/getwd.c b/src/lib9/getwd.c new file mode 100644 index 000000000..3c8cafb3a --- /dev/null +++ b/src/lib9/getwd.c @@ -0,0 +1,55 @@ +/* +Plan 9 from User Space src/lib9/getwd.c +http://code.swtch.com/plan9port/src/tip/src/lib9/getwd.c + +Copyright 2001-2007 Russ Cox. All Rights Reserved. +Portions Copyright 2011 The Go Authors. All Rights Reserved. + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. +*/ +#include +#include +#include +#include + +#undef getwd + +char* +p9getwd(char *s, int ns) +{ + char *pwd; + struct stat st1, st2; + + // Clumsy but widespread kludge: + // if $PWD is set and matches ".", use it. + // Matches glibc's get_current_dir_name and Go's os.Getwd. + pwd = getenv("PWD"); // note: getenv, not p9getenv, so no free + if(pwd != nil && pwd[0] && + stat(pwd, &st1) >= 0 && stat(".", &st2) >= 0 && + st1.st_dev == st2.st_dev && st1.st_ino == st2.st_ino) { + if(strlen(pwd) >= ns) { + errno = ERANGE; + return nil; + } + strcpy(s, pwd); + return s; + } + + return getcwd(s, ns); +} diff --git a/src/lib9/goos.c b/src/lib9/goos.c new file mode 100644 index 000000000..f3ee1110a --- /dev/null +++ b/src/lib9/goos.c @@ -0,0 +1,41 @@ +// Copyright 2010 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +#include +#include + +static char* +defgetenv(char *name, char *def) +{ + char *p; + + p = getenv(name); + if(p == nil || p[0] == '\0') + p = def; + return p; +} + +char* +getgoos(void) +{ + return defgetenv("GOOS", GOOS); +} + +char* +getgoarch(void) +{ + return defgetenv("GOARCH", GOARCH); +} + +char* +getgoroot(void) +{ + return defgetenv("GOROOT", GOROOT); +} + +char* +getgoversion(void) +{ + return GOVERSION; +} diff --git a/src/lib9/jmp.c b/src/lib9/jmp.c new file mode 100644 index 000000000..a606fb07b --- /dev/null +++ b/src/lib9/jmp.c @@ -0,0 +1,42 @@ +/* +Plan 9 from User Space src/lib9/jmp.c +http://code.swtch.com/plan9port/src/tip/src/lib9/jmp.c + +Copyright 2001-2007 Russ Cox. All Rights Reserved. + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. +*/ + +#include +#define NOPLAN9DEFINES +#include + +void +p9longjmp(p9jmp_buf buf, int val) +{ + siglongjmp((void*)buf, val); +} + +void +p9notejmp(void *x, p9jmp_buf buf, int val) +{ + USED(x); + siglongjmp((void*)buf, val); +} + diff --git a/src/lib9/main.c b/src/lib9/main.c new file mode 100644 index 000000000..45f86c7ec --- /dev/null +++ b/src/lib9/main.c @@ -0,0 +1,38 @@ +/* +Plan 9 from User Space src/lib9/main.c +http://code.swtch.com/plan9port/src/tip/src/lib9/main.c + +Copyright 2001-2007 Russ Cox. All Rights Reserved. + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. +*/ + +#include +#define NOPLAN9DEFINES +#include + +extern void p9main(int, char**); + +int +main(int argc, char **argv) +{ + p9main(argc, argv); + exits("main"); + return 99; +} diff --git a/src/lib9/nan.c b/src/lib9/nan.c new file mode 100644 index 000000000..fa2277f72 --- /dev/null +++ b/src/lib9/nan.c @@ -0,0 +1,52 @@ +/* +Plan 9 from User Space src/lib9/nan.c +http://code.swtch.com/plan9port/src/tip/src/lib9/nan.c + +Copyright 2001-2007 Russ Cox. All Rights Reserved. + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. +*/ + +#include +#include +#include "fmt/fmtdef.h" + +double +NaN(void) +{ + return __NaN(); +} + +double +Inf(int sign) +{ + return __Inf(sign); +} + +int +isNaN(double x) +{ + return __isNaN(x); +} + +int +isInf(double x, int sign) +{ + return __isInf(x, sign); +} diff --git a/src/lib9/notify.c b/src/lib9/notify.c new file mode 100644 index 000000000..84999b887 --- /dev/null +++ b/src/lib9/notify.c @@ -0,0 +1,297 @@ +/* +Plan 9 from User Space src/lib9/notify.c +http://code.swtch.com/plan9port/src/tip/src/lib9/notify.c + +Copyright 2001-2007 Russ Cox. All Rights Reserved. + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. +*/ + +/* + * Signal handling for Plan 9 programs. + * We stubbornly use the strings from Plan 9 instead + * of the enumerated Unix constants. + * There are some weird translations. In particular, + * a "kill" note is the same as SIGTERM in Unix. + * There is no equivalent note to Unix's SIGKILL, since + * it's not a deliverable signal anyway. + * + * We do not handle SIGABRT or SIGSEGV, mainly because + * the thread library queues its notes for later, and we want + * to dump core with the state at time of delivery. + * + * We have to add some extra entry points to provide the + * ability to tweak which signals are deliverable and which + * are acted upon. Notifydisable and notifyenable play with + * the process signal mask. Notifyignore enables the signal + * but will not call notifyf when it comes in. This is occasionally + * useful. + */ + +#include +#include +#define NOPLAN9DEFINES +#include + +extern char *_p9sigstr(int, char*); +extern int _p9strsig(char*); + +typedef struct Sig Sig; +struct Sig +{ + int sig; /* signal number */ + int flags; +}; + +enum +{ + Restart = 1<<0, + Ignore = 1<<1 +}; + +static Sig sigs[] = { + SIGHUP, 0, + SIGINT, 0, + SIGQUIT, 0, + SIGILL, 0, + SIGTRAP, 0, +/* SIGABRT, 0, */ +#ifdef SIGEMT + SIGEMT, 0, +#endif + SIGFPE, 0, + SIGBUS, 0, +/* SIGSEGV, 0, */ + SIGCHLD, Restart|Ignore, + SIGSYS, 0, + SIGPIPE, Ignore, + SIGALRM, 0, + SIGTERM, 0, + SIGTSTP, Restart|Ignore, +/* SIGTTIN, Restart|Ignore, */ +/* SIGTTOU, Restart|Ignore, */ + SIGXCPU, 0, + SIGXFSZ, 0, + SIGVTALRM, 0, + SIGUSR1, 0, + SIGUSR2, 0, +#ifdef SIGWINCH + SIGWINCH, Restart|Ignore, +#endif +#ifdef SIGINFO + SIGINFO, Restart|Ignore, +#endif +}; + +static Sig* +findsig(int s) +{ + int i; + + for(i=0; ib)){ + case 0: + if(notifyf) + (*notifyf)(nil, _p9sigstr(sig, tmp)); + /* fall through */ + case 1: /* noted(NDFLT) */ + if(0)print("DEFAULT %d\n", sig); + s = findsig(sig); + if(s && (s->flags&Ignore)) + return; + signal(sig, SIG_DFL); + raise(sig); + _exit(1); + case 2: /* noted(NCONT) */ + if(0)print("HANDLED %d\n", sig); + return; + } +} + +static void +signonotify(int sig) +{ + USED(sig); +} + +int +noted(int v) +{ + p9longjmp((*_notejmpbuf)()->b, v==NCONT ? 2 : 1); + abort(); + return 0; +} + +int +notify(void (*f)(void*, char*)) +{ + static int init; + + notifyf = f; + if(!init){ + init = 1; + noteinit(); + } + return 0; +} + +/* + * Nonsense about enabling and disabling signals. + */ +typedef void Sighandler(int); +static Sighandler* +handler(int s) +{ + struct sigaction sa; + + sigaction(s, nil, &sa); + return sa.sa_handler; +} + +static int +notesetenable(int sig, int enabled) +{ + sigset_t mask, omask; + + if(sig == 0) + return -1; + + sigemptyset(&mask); + sigaddset(&mask, sig); + sigprocmask(enabled ? SIG_UNBLOCK : SIG_BLOCK, &mask, &omask); + return !sigismember(&omask, sig); +} + +int +noteenable(char *msg) +{ + return notesetenable(_p9strsig(msg), 1); +} + +int +notedisable(char *msg) +{ + return notesetenable(_p9strsig(msg), 0); +} + +static int +notifyseton(int s, int on) +{ + Sig *sig; + struct sigaction sa, osa; + + sig = findsig(s); + if(sig == nil) + return -1; + memset(&sa, 0, sizeof sa); + sa.sa_handler = on ? signotify : signonotify; + if(sig->flags&Restart) + sa.sa_flags |= SA_RESTART; + + /* + * We can't allow signals within signals because there's + * only one jump buffer. + */ + sigfillset(&sa.sa_mask); + + /* + * Install handler. + */ + sigaction(sig->sig, &sa, &osa); + return osa.sa_handler == signotify; +} + +int +notifyon(char *msg) +{ + return notifyseton(_p9strsig(msg), 1); +} + +int +notifyoff(char *msg) +{ + return notifyseton(_p9strsig(msg), 0); +} + +/* + * Initialization follows sigs table. + */ +static void +noteinit(void) +{ + int i; + Sig *sig; + + for(i=0; isig) != SIG_DFL) + continue; + notifyseton(sig->sig, 1); + } +} + diff --git a/src/lib9/nulldir.c b/src/lib9/nulldir.c new file mode 100644 index 000000000..aa1a1232e --- /dev/null +++ b/src/lib9/nulldir.c @@ -0,0 +1,35 @@ +/* +Inferno lib9/nulldir.c +http://code.google.com/p/inferno-os/source/browse/lib9/nulldir.c + + Copyright © 1994-1999 Lucent Technologies Inc. All rights reserved. + Revisions Copyright © 2000-2007 Vita Nuova Holdings Limited (www.vitanuova.com). All rights reserved. + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. +*/ + +#include +#include + +void +nulldir(Dir *d) +{ + memset(d, ~0, sizeof(Dir)); + d->name = d->uid = d->gid = d->muid = ""; +} diff --git a/src/lib9/open.c b/src/lib9/open.c new file mode 100644 index 000000000..4ac81ba5f --- /dev/null +++ b/src/lib9/open.c @@ -0,0 +1,68 @@ +/* +Plan 9 from User Space src/lib9/open.c +http://code.swtch.com/plan9port/src/tip/src/lib9/open.c + +Copyright 2001-2007 Russ Cox. All Rights Reserved. + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. +*/ + +#define _GNU_SOURCE /* for Linux O_DIRECT */ +#include +#define NOPLAN9DEFINES +#include +#include +#ifndef O_DIRECT +#define O_DIRECT 0 +#endif + +int +p9open(char *name, int mode) +{ + int rclose; + int fd, umode, rdwr; + + rdwr = mode&3; + umode = rdwr; + rclose = mode&ORCLOSE; + mode &= ~(3|ORCLOSE); + if(mode&OTRUNC){ + umode |= O_TRUNC; + mode ^= OTRUNC; + } + if(mode&ODIRECT){ + umode |= O_DIRECT; + mode ^= ODIRECT; + } + if(mode&OAPPEND){ + umode |= O_APPEND; + mode ^= OAPPEND; + } + if(mode){ + werrstr("mode 0x%x not supported", mode); + return -1; + } + umode |= O_BINARY; + fd = open(name, umode); + if(fd >= 0){ + if(rclose) + remove(name); + } + return fd; +} diff --git a/src/lib9/readn.c b/src/lib9/readn.c new file mode 100644 index 000000000..f39b4a4c2 --- /dev/null +++ b/src/lib9/readn.c @@ -0,0 +1,48 @@ +/* +Inferno lib9/readn.c +http://code.google.com/p/inferno-os/source/browse/lib9/readn.c + + Copyright © 1994-1999 Lucent Technologies Inc. All rights reserved. + Revisions Copyright © 2000-2007 Vita Nuova Holdings Limited (www.vitanuova.com). All rights reserved. + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. +*/ + +#include +#include + +long +readn(int f, void *av, long n) +{ + char *a; + long m, t; + + a = av; + t = 0; + while(t < n){ + m = read(f, a+t, n-t); + if(m <= 0){ + if(t == 0) + return m; + break; + } + t += m; + } + return t; +} diff --git a/src/lib9/rfork.c b/src/lib9/rfork.c new file mode 100644 index 000000000..c9d632189 --- /dev/null +++ b/src/lib9/rfork.c @@ -0,0 +1,153 @@ +/* +Plan 9 from User Space src/lib9/rfork.c +http://code.swtch.com/plan9port/src/tip/src/lib9/rfork.c + +Copyright 2001-2007 Russ Cox. All Rights Reserved. + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. +*/ + +#include +#include +#include +#include +#undef rfork + +static void +nop(int x) +{ + USED(x); +} + +int +p9rfork(int flags) +{ + int pid, status; + int p[2]; + int n; + char buf[128], *q; + extern char **environ; + + if((flags&(RFPROC|RFFDG|RFMEM)) == (RFPROC|RFFDG)){ + /* check other flags before we commit */ + flags &= ~(RFPROC|RFFDG|RFENVG); + n = (flags & ~(RFNOTEG|RFNAMEG|RFNOWAIT|RFCENVG)); + if(n){ + werrstr("unknown flags %08ux in rfork", n); + return -1; + } + if(flags&RFNOWAIT){ + /* + * BUG - should put the signal handler back after we + * finish, but I just don't care. If a program calls with + * NOWAIT once, they're not likely to want child notes + * after that. + */ + signal(SIGCHLD, nop); + if(pipe(p) < 0) + return -1; + } + pid = fork(); + if(pid == -1) + return -1; + if(flags&RFNOWAIT){ + flags &= ~RFNOWAIT; + if(pid){ + /* + * Parent - wait for child to fork wait-free child. + * Then read pid from pipe. Assume pipe buffer can absorb the write. + */ + close(p[1]); + status = 0; + if(wait4(pid, &status, 0, 0) < 0){ + werrstr("pipe dance - wait4 - %r"); + close(p[0]); + return -1; + } + n = readn(p[0], buf, sizeof buf-1); + close(p[0]); + if(!WIFEXITED(status) || WEXITSTATUS(status)!=0 || n <= 0){ + if(!WIFEXITED(status)) + werrstr("pipe dance - !exited 0x%ux", status); + else if(WEXITSTATUS(status) != 0) + werrstr("pipe dance - non-zero status 0x%ux", status); + else if(n < 0) + werrstr("pipe dance - pipe read error - %r"); + else if(n == 0) + werrstr("pipe dance - pipe read eof"); + else + werrstr("pipe dance - unknown failure"); + return -1; + } + buf[n] = 0; + if(buf[0] == 'x'){ + werrstr("%s", buf+2); + return -1; + } + pid = strtol(buf, &q, 0); + }else{ + /* + * Child - fork a new child whose wait message can't + * get back to the parent because we're going to exit! + */ + signal(SIGCHLD, SIG_IGN); + close(p[0]); + pid = fork(); + if(pid){ + /* Child parent - send status over pipe and exit. */ + if(pid > 0) + fprint(p[1], "%d", pid); + else + fprint(p[1], "x %r"); + close(p[1]); + _exit(0); + }else{ + /* Child child - close pipe. */ + close(p[1]); + } + } + } + if(pid != 0) + return pid; + if(flags&RFCENVG) + if(environ) + *environ = nil; + } + if(flags&RFPROC){ + werrstr("cannot use rfork for shared memory -- use libthread"); + return -1; + } + if(flags&RFNAMEG){ + /* XXX set $NAMESPACE to a new directory */ + flags &= ~RFNAMEG; + } + if(flags&RFNOTEG){ + setpgid(0, getpid()); + flags &= ~RFNOTEG; + } + if(flags&RFNOWAIT){ + werrstr("cannot use RFNOWAIT without RFPROC"); + return -1; + } + if(flags){ + werrstr("unknown flags %08ux in rfork", flags); + return -1; + } + return 0; +} diff --git a/src/lib9/seek.c b/src/lib9/seek.c new file mode 100644 index 000000000..917003808 --- /dev/null +++ b/src/lib9/seek.c @@ -0,0 +1,33 @@ +/* +Plan 9 from User Space src/lib9/seek.c +http://code.swtch.com/plan9port/src/tip/src/lib9/seek.c + +Copyright 2001-2007 Russ Cox. All Rights Reserved. + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. +*/ + +#include +#include + +vlong +seek(int fd, vlong offset, int whence) +{ + return lseek(fd, offset, whence); +} diff --git a/src/lib9/strecpy.c b/src/lib9/strecpy.c new file mode 100644 index 000000000..389fdc8a0 --- /dev/null +++ b/src/lib9/strecpy.c @@ -0,0 +1,43 @@ +/* +Inferno lib9/strecpy.c +http://code.google.com/p/inferno-os/source/browse/lib9/strecpy.c + + Copyright © 1994-1999 Lucent Technologies Inc. All rights reserved. + Revisions Copyright © 2000-2007 Vita Nuova Holdings Limited (www.vitanuova.com). All rights reserved. + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. +*/ + +#include +#include + +char* +strecpy(char *to, char *e, char *from) +{ + if(to >= e) + return to; + to = memccpy(to, from, '\0', e - to); + if(to == nil){ + to = e - 1; + *to = '\0'; + }else{ + to--; + } + return to; +} diff --git a/src/lib9/sysfatal.c b/src/lib9/sysfatal.c new file mode 100644 index 000000000..a5af3e1b4 --- /dev/null +++ b/src/lib9/sysfatal.c @@ -0,0 +1,47 @@ +/* +Plan 9 from User Space src/lib9/sysfatal.c +http://code.swtch.com/plan9port/src/tip/src/lib9/sysfatal.c + +Copyright 2001-2007 Russ Cox. All Rights Reserved. + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. +*/ + +#include +#include + +void (*_sysfatal)(char*, ...); + +void +sysfatal(char *fmt, ...) +{ + char buf[256]; + va_list arg; + + va_start(arg, fmt); + if(_sysfatal) + (*_sysfatal)(fmt, arg); + vseprint(buf, buf+sizeof buf, fmt, arg); + va_end(arg); + + __fixargv0(); + fprint(2, "%s: %s\n", argv0 ? argv0 : "", buf); + exits("fatal"); +} + diff --git a/src/lib9/time.c b/src/lib9/time.c new file mode 100644 index 000000000..7394e9e60 --- /dev/null +++ b/src/lib9/time.c @@ -0,0 +1,66 @@ +/* +Plan 9 from User Space src/lib9/time.c +http://code.swtch.com/plan9port/src/tip/src/lib9/time.c + +Copyright 2001-2007 Russ Cox. All Rights Reserved. + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. +*/ +#include +#include +#include +#ifndef _WIN32 +#include +#endif +#define NOPLAN9DEFINES +#include + +long +p9times(long *t) +{ +#ifdef _WIN32 + memset(t, 0, 4*sizeof(long)); +#else + struct rusage ru, cru; + + if(getrusage(0, &ru) < 0 || getrusage(-1, &cru) < 0) + return -1; + + t[0] = ru.ru_utime.tv_sec*1000 + ru.ru_utime.tv_usec/1000; + t[1] = ru.ru_stime.tv_sec*1000 + ru.ru_stime.tv_usec/1000; + t[2] = cru.ru_utime.tv_sec*1000 + cru.ru_utime.tv_usec/1000; + t[3] = cru.ru_stime.tv_sec*1000 + cru.ru_stime.tv_usec/1000; +#endif + + /* BUG */ + return t[0]+t[1]+t[2]+t[3]; +} + +double +p9cputime(void) +{ + long t[4]; + double d; + + if(p9times(t) < 0) + return -1.0; + + d = (double)t[0]+(double)t[1]+(double)t[2]+(double)t[3]; + return d/1000.0; +} diff --git a/src/lib9/tokenize.c b/src/lib9/tokenize.c new file mode 100644 index 000000000..52167ff2f --- /dev/null +++ b/src/lib9/tokenize.c @@ -0,0 +1,133 @@ +/* +Inferno lib9/tokenize.c +http://code.google.com/p/inferno-os/source/browse/lib9/tokenize.c + + Copyright © 1994-1999 Lucent Technologies Inc. All rights reserved. + Revisions Copyright © 2000-2007 Vita Nuova Holdings Limited (www.vitanuova.com). All rights reserved. + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. +*/ + +#include +#include + +static char qsep[] = " \t\r\n"; + +static char* +qtoken(char *s, char *sep) +{ + int quoting; + char *t; + + quoting = 0; + t = s; /* s is output string, t is input string */ + while(*t!='\0' && (quoting || utfrune(sep, *t)==nil)){ + if(*t != '\''){ + *s++ = *t++; + continue; + } + /* *t is a quote */ + if(!quoting){ + quoting = 1; + t++; + continue; + } + /* quoting and we're on a quote */ + if(t[1] != '\''){ + /* end of quoted section; absorb closing quote */ + t++; + quoting = 0; + continue; + } + /* doubled quote; fold one quote into two */ + t++; + *s++ = *t++; + } + if(*s != '\0'){ + *s = '\0'; + if(t == s) + t++; + } + return t; +} + +static char* +etoken(char *t, char *sep) +{ + int quoting; + + /* move to end of next token */ + quoting = 0; + while(*t!='\0' && (quoting || utfrune(sep, *t)==nil)){ + if(*t != '\''){ + t++; + continue; + } + /* *t is a quote */ + if(!quoting){ + quoting = 1; + t++; + continue; + } + /* quoting and we're on a quote */ + if(t[1] != '\''){ + /* end of quoted section; absorb closing quote */ + t++; + quoting = 0; + continue; + } + /* doubled quote; fold one quote into two */ + t += 2; + } + return t; +} + +int +gettokens(char *s, char **args, int maxargs, char *sep) +{ + int nargs; + + for(nargs=0; nargs_$@ + mv _$@ $@ + +runetypebody-%.c: mkrunetype UnicodeData-%.txt + mkrunetype -p UnicodeData-$*.txt >_$@ + mv _$@ $@ + +CLEANFILES+=UnicodeData.txt + +UNICODE_VERSION=6.0.0 + +test: mkrunetype UnicodeData-$(UNICODE_VERSION).txt + mkrunetype -c UnicodeData-$(UNICODE_VERSION).txt + diff --git a/src/lib9/utf/mkrunetype.c b/src/lib9/utf/mkrunetype.c new file mode 100644 index 000000000..06d52b572 --- /dev/null +++ b/src/lib9/utf/mkrunetype.c @@ -0,0 +1,732 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +/* + * make is(upper|lower|title|space|alpha)rune and + * to(upper|lower|title)rune from a UnicodeData.txt file. + * these can be found at unicode.org + * + * with -c, runs a check of the existing runetype functions vs. + * those extracted from UnicodeData. + * + * with -p, generates tables for pairs of chars, as well as for ranges + * and singletons. + * + * UnicodeData defines 4 fields of interest: + * 1) a category + * 2) an upper case mapping + * 3) a lower case mapping + * 4) a title case mapping + * + * toupper, tolower, and totitle are defined directly from the mapping. + * + * isalpharune(c) is true iff c is a "letter" category + * isupperrune(c) is true iff c is the target of toupperrune, + * or is in the uppercase letter category + * similarly for islowerrune and istitlerune. + * isspacerune is true for space category chars, "C" locale white space chars, + * and two additions: + * 0085 "next line" control char + * feff] "zero-width non-break space" + * isdigitrune is true iff c is a numeric-digit category. + */ + +#include +#include +#include +#include "utf.h" +#include "utfdef.h" + +enum { + /* + * fields in the unicode data file + */ + FIELD_CODE, + FIELD_NAME, + FIELD_CATEGORY, + FIELD_COMBINING, + FIELD_BIDIR, + FIELD_DECOMP, + FIELD_DECIMAL_DIG, + FIELD_DIG, + FIELD_NUMERIC_VAL, + FIELD_MIRRORED, + FIELD_UNICODE_1_NAME, + FIELD_COMMENT, + FIELD_UPPER, + FIELD_LOWER, + FIELD_TITLE, + NFIELDS, + + MAX_LINE = 1024, + + TO_OFFSET = 1 << 20, + + NRUNES = 1 << 21, +}; + +#define TO_DELTA(xmapped,x) (TO_OFFSET + (xmapped) - (x)) + +static char myisspace[NRUNES]; +static char myisalpha[NRUNES]; +static char myisdigit[NRUNES]; +static char myisupper[NRUNES]; +static char myislower[NRUNES]; +static char myistitle[NRUNES]; + +static int mytoupper[NRUNES]; +static int mytolower[NRUNES]; +static int mytotitle[NRUNES]; + +static void check(void); +static void mktables(char *src, int usepairs); +static void fatal(const char *fmt, ...); +static int mygetfields(char **fields, int nfields, char *str, const char *delim); +static int getunicodeline(FILE *in, char **fields, char *buf); +static int getcode(char *s); + +static void +usage(void) +{ + fprintf(stderr, "usage: mktables [-cp] \n"); + exit(1); +} + +void +main(int argc, char *argv[]) +{ + FILE *in; + char buf[MAX_LINE], buf2[MAX_LINE]; + char *fields[NFIELDS + 1], *fields2[NFIELDS + 1]; + char *p; + int i, code, last, docheck, usepairs; + + docheck = 0; + usepairs = 0; + ARGBEGIN{ + case 'c': + docheck = 1; + break; + case 'p': + usepairs = 1; + break; + default: + usage(); + }ARGEND + + if(argc != 1){ + usage(); + } + + in = fopen(argv[0], "r"); + if(in == NULL){ + fatal("can't open %s", argv[0]); + } + + for(i = 0; i < NRUNES; i++){ + mytoupper[i] = i; + mytolower[i] = i; + mytotitle[i] = i; + } + + /* + * make sure isspace has all of the "C" locale whitespace chars + */ + myisspace['\t'] = 1; + myisspace['\n'] = 1; + myisspace['\r'] = 1; + myisspace['\f'] = 1; + myisspace['\v'] = 1; + + /* + * a couple of other exceptions + */ + myisspace[0x85] = 1; /* control char, "next line" */ + myisspace[0xfeff] = 1; /* zero-width non-break space */ + + last = -1; + while(getunicodeline(in, fields, buf)){ + code = getcode(fields[FIELD_CODE]); + if (code >= NRUNES) + fatal("code-point value too big: %x", code); + if(code <= last) + fatal("bad code sequence: %x then %x", last, code); + last = code; + + /* + * check for ranges + */ + p = fields[FIELD_CATEGORY]; + if(strstr(fields[FIELD_NAME], ", First>") != NULL){ + if(!getunicodeline(in, fields2, buf2)) + fatal("range start at eof"); + if (strstr(fields2[FIELD_NAME], ", Last>") == NULL) + fatal("range start not followed by range end"); + last = getcode(fields2[FIELD_CODE]); + if(last <= code) + fatal("range out of sequence: %x then %x", code, last); + if(strcmp(p, fields2[FIELD_CATEGORY]) != 0) + fatal("range with mismatched category"); + } + + /* + * set properties and conversions + */ + for (; code <= last; code++){ + if(p[0] == 'L') + myisalpha[code] = 1; + if(p[0] == 'Z') + myisspace[code] = 1; + + if(strcmp(p, "Lu") == 0) + myisupper[code] = 1; + if(strcmp(p, "Ll") == 0) + myislower[code] = 1; + + if(strcmp(p, "Lt") == 0) + myistitle[code] = 1; + + if(strcmp(p, "Nd") == 0) + myisdigit[code] = 1; + + /* + * when finding conversions, also need to mark + * upper/lower case, since some chars, like + * "III" (0x2162), aren't defined as letters but have a + * lower case mapping ("iii" (0x2172)). + */ + if(fields[FIELD_UPPER][0] != '\0'){ + mytoupper[code] = getcode(fields[FIELD_UPPER]); + } + if(fields[FIELD_LOWER][0] != '\0'){ + mytolower[code] = getcode(fields[FIELD_LOWER]); + } + if(fields[FIELD_TITLE][0] != '\0'){ + mytotitle[code] = getcode(fields[FIELD_TITLE]); + } + } + } + + fclose(in); + + /* + * check for codes with no totitle mapping but a toupper mapping. + * these appear in UnicodeData-2.0.14.txt, but are almost certainly + * erroneous. + */ + for(i = 0; i < NRUNES; i++){ + if(mytotitle[i] == i + && mytoupper[i] != i + && !myistitle[i]) + fprintf(stderr, "warning: code=%.4x not istitle, totitle is same, toupper=%.4x\n", i, mytoupper[i]); + } + + /* + * make sure isupper[c] is true if for some x toupper[x] == c + * ditto for islower and istitle + */ + for(i = 0; i < NRUNES; i++) { + if(mytoupper[i] != i) + myisupper[mytoupper[i]] = 1; + if(mytolower[i] != i) + myislower[mytolower[i]] = 1; + if(mytotitle[i] != i) + myistitle[mytotitle[i]] = 1; + } + + if(docheck){ + check(); + }else{ + mktables(argv[0], usepairs); + } + exit(0); +} + +/* + * generate a properties array for ranges, clearing those cases covered. + * if force, generate one-entry ranges for singletons. + */ +static int +mkisrange(const char* label, char* prop, int force) +{ + int start, stop, some; + + /* + * first, the ranges + */ + some = 0; + for(start = 0; start < NRUNES; ) { + if(!prop[start]){ + start++; + continue; + } + + for(stop = start + 1; stop < NRUNES; stop++){ + if(!prop[stop]){ + break; + } + prop[stop] = 0; + } + if(force || stop != start + 1){ + if(!some){ + printf("static Rune __is%sr[] = {\n", label); + some = 1; + } + prop[start] = 0; + printf("\t0x%.4x, 0x%.4x,\n", start, stop - 1); + } + + start = stop; + } + if(some) + printf("};\n\n"); + return some; +} + +/* + * generate a mapping array for pairs with a skip between, + * clearing those entries covered. + */ +static int +mkispair(const char *label, char *prop) +{ + int start, stop, some; + + some = 0; + for(start = 0; start + 2 < NRUNES; ) { + if(!prop[start]){ + start++; + continue; + } + + for(stop = start + 2; stop < NRUNES; stop += 2){ + if(!prop[stop]){ + break; + } + prop[stop] = 0; + } + if(stop != start + 2){ + if(!some){ + printf("static Rune __is%sp[] = {\n", label); + some = 1; + } + prop[start] = 0; + printf("\t0x%.4x, 0x%.4x,\n", start, stop - 2); + } + + start = stop; + } + if(some) + printf("};\n\n"); + return some; +} + +/* + * generate a properties array for singletons, clearing those cases covered. + */ +static int +mkissingle(const char *label, char *prop) +{ + int start, some; + + some = 0; + for(start = 0; start < NRUNES; start++) { + if(!prop[start]){ + continue; + } + + if(!some){ + printf("static Rune __is%ss[] = {\n", label); + some = 1; + } + prop[start] = 0; + printf("\t0x%.4x,\n", start); + } + if(some) + printf("};\n\n"); + return some; +} + +/* + * generate tables and a function for is>(_W-s) + un10 := u0 << s + un1 := un10 >> _W2 + un0 := un10 & _M2 + q1 := un32 / vn1 + rhat := un32 - q1*vn1 + +again1: + if q1 >= _B2 || q1*vn0 > _B2*rhat+un1 { + q1-- + rhat += vn1 + if rhat < _B2 { + goto again1 + } + } + + un21 := un32*_B2 + un1 - q1*v + q0 := un21 / vn1 + rhat = un21 - q0*vn1 + +again2: + if q0 >= _B2 || q0*vn0 > _B2*rhat+un0 { + q0-- + rhat += vn1 + if rhat < _B2 { + goto again2 + } + } + + return q1*_B2 + q0, (un21*_B2 + un0 - q0*v) >> s +} + +func addVV_g(z, x, y []Word) (c Word) { + for i := range z { + c, z[i] = addWW_g(x[i], y[i], c) + } + return +} + +func subVV_g(z, x, y []Word) (c Word) { + for i := range z { + c, z[i] = subWW_g(x[i], y[i], c) + } + return +} + +func addVW_g(z, x []Word, y Word) (c Word) { + c = y + for i := range z { + c, z[i] = addWW_g(x[i], c, 0) + } + return +} + +func subVW_g(z, x []Word, y Word) (c Word) { + c = y + for i := range z { + c, z[i] = subWW_g(x[i], c, 0) + } + return +} + +func shlVU_g(z, x []Word, s uint) (c Word) { + if n := len(z); n > 0 { + ŝ := _W - s + w1 := x[n-1] + c = w1 >> ŝ + for i := n - 1; i > 0; i-- { + w := w1 + w1 = x[i-1] + z[i] = w<>ŝ + } + z[0] = w1 << s + } + return +} + +func shrVU_g(z, x []Word, s uint) (c Word) { + if n := len(z); n > 0 { + ŝ := _W - s + w1 := x[0] + c = w1 << ŝ + for i := 0; i < n-1; i++ { + w := w1 + w1 = x[i+1] + z[i] = w>>s | w1<<ŝ + } + z[n-1] = w1 >> s + } + return +} + +func mulAddVWW_g(z, x []Word, y, r Word) (c Word) { + c = r + for i := range z { + c, z[i] = mulAddWWW_g(x[i], y, c) + } + return +} + +func addMulVVW_g(z, x []Word, y Word) (c Word) { + for i := range z { + z1, z0 := mulAddWWW_g(x[i], y, z[i]) + c, z[i] = addWW_g(z0, c, 0) + c += z1 + } + return +} + +func divWVW_g(z []Word, xn Word, x []Word, y Word) (r Word) { + r = xn + for i := len(z) - 1; i >= 0; i-- { + z[i], r = divWW_g(r, x[i], y) + } + return +} diff --git a/src/pkg/big/arith_386.s b/src/pkg/big/arith_386.s new file mode 100644 index 000000000..07c07b02c --- /dev/null +++ b/src/pkg/big/arith_386.s @@ -0,0 +1,265 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// This file provides fast assembly versions for the elementary +// arithmetic operations on vectors implemented in arith.go. + +// func mulWW(x, y Word) (z1, z0 Word) +TEXT ·mulWW(SB),7,$0 + MOVL x+0(FP), AX + MULL y+4(FP) + MOVL DX, z1+8(FP) + MOVL AX, z0+12(FP) + RET + + +// func divWW(x1, x0, y Word) (q, r Word) +TEXT ·divWW(SB),7,$0 + MOVL x1+0(FP), DX + MOVL x0+4(FP), AX + DIVL y+8(FP) + MOVL AX, q+12(FP) + MOVL DX, r+16(FP) + RET + + +// func addVV(z, x, y []Word) (c Word) +TEXT ·addVV(SB),7,$0 + MOVL z+0(FP), DI + MOVL x+12(FP), SI + MOVL y+24(FP), CX + MOVL n+4(FP), BP + MOVL $0, BX // i = 0 + MOVL $0, DX // c = 0 + JMP E1 + +L1: MOVL (SI)(BX*4), AX + RCRL $1, DX + ADCL (CX)(BX*4), AX + RCLL $1, DX + MOVL AX, (DI)(BX*4) + ADDL $1, BX // i++ + +E1: CMPL BX, BP // i < n + JL L1 + + MOVL DX, c+36(FP) + RET + + +// func subVV(z, x, y []Word) (c Word) +// (same as addVV except for SBBL instead of ADCL and label names) +TEXT ·subVV(SB),7,$0 + MOVL z+0(FP), DI + MOVL x+12(FP), SI + MOVL y+24(FP), CX + MOVL n+4(FP), BP + MOVL $0, BX // i = 0 + MOVL $0, DX // c = 0 + JMP E2 + +L2: MOVL (SI)(BX*4), AX + RCRL $1, DX + SBBL (CX)(BX*4), AX + RCLL $1, DX + MOVL AX, (DI)(BX*4) + ADDL $1, BX // i++ + +E2: CMPL BX, BP // i < n + JL L2 + + MOVL DX, c+36(FP) + RET + + +// func addVW(z, x []Word, y Word) (c Word) +TEXT ·addVW(SB),7,$0 + MOVL z+0(FP), DI + MOVL x+12(FP), SI + MOVL y+24(FP), AX // c = y + MOVL n+4(FP), BP + MOVL $0, BX // i = 0 + JMP E3 + +L3: ADDL (SI)(BX*4), AX + MOVL AX, (DI)(BX*4) + RCLL $1, AX + ANDL $1, AX + ADDL $1, BX // i++ + +E3: CMPL BX, BP // i < n + JL L3 + + MOVL AX, c+28(FP) + RET + + +// func subVW(z, x []Word, y Word) (c Word) +TEXT ·subVW(SB),7,$0 + MOVL z+0(FP), DI + MOVL x+12(FP), SI + MOVL y+24(FP), AX // c = y + MOVL n+4(FP), BP + MOVL $0, BX // i = 0 + JMP E4 + +L4: MOVL (SI)(BX*4), DX // TODO(gri) is there a reverse SUBL? + SUBL AX, DX + MOVL DX, (DI)(BX*4) + RCLL $1, AX + ANDL $1, AX + ADDL $1, BX // i++ + +E4: CMPL BX, BP // i < n + JL L4 + + MOVL AX, c+28(FP) + RET + + +// func shlVU(z, x []Word, s uint) (c Word) +TEXT ·shlVU(SB),7,$0 + MOVL n+4(FP), BX // i = n + SUBL $1, BX // i-- + JL X8b // i < 0 (n <= 0) + + // n > 0 + MOVL z+0(FP), DI + MOVL x+12(FP), SI + MOVL s+24(FP), CX + MOVL (SI)(BX*4), AX // w1 = x[n-1] + MOVL $0, DX + SHLL CX, DX:AX // w1>>ŝ + MOVL DX, c+28(FP) + + CMPL BX, $0 + JLE X8a // i <= 0 + + // i > 0 +L8: MOVL AX, DX // w = w1 + MOVL -4(SI)(BX*4), AX // w1 = x[i-1] + SHLL CX, DX:AX // w<>ŝ + MOVL DX, (DI)(BX*4) // z[i] = w<>ŝ + SUBL $1, BX // i-- + JG L8 // i > 0 + + // i <= 0 +X8a: SHLL CX, AX // w1< 0 + MOVL z+0(FP), DI + MOVL x+12(FP), SI + MOVL s+24(FP), CX + MOVL (SI), AX // w1 = x[0] + MOVL $0, DX + SHRL CX, DX:AX // w1<<ŝ + MOVL DX, c+28(FP) + + MOVL $0, BX // i = 0 + JMP E9 + + // i < n-1 +L9: MOVL AX, DX // w = w1 + MOVL 4(SI)(BX*4), AX // w1 = x[i+1] + SHRL CX, DX:AX // w>>s | w1<<ŝ + MOVL DX, (DI)(BX*4) // z[i] = w>>s | w1<<ŝ + ADDL $1, BX // i++ + +E9: CMPL BX, BP + JL L9 // i < n-1 + + // i >= n-1 +X9a: SHRL CX, AX // w1>>s + MOVL AX, (DI)(BP*4) // z[n-1] = w1>>s + RET + +X9b: MOVL $0, c+28(FP) + RET + + +// func mulAddVWW(z, x []Word, y, r Word) (c Word) +TEXT ·mulAddVWW(SB),7,$0 + MOVL z+0(FP), DI + MOVL x+12(FP), SI + MOVL y+24(FP), BP + MOVL r+28(FP), CX // c = r + MOVL n+4(FP), BX + LEAL (DI)(BX*4), DI + LEAL (SI)(BX*4), SI + NEGL BX // i = -n + JMP E5 + +L5: MOVL (SI)(BX*4), AX + MULL BP + ADDL CX, AX + ADCL $0, DX + MOVL AX, (DI)(BX*4) + MOVL DX, CX + ADDL $1, BX // i++ + +E5: CMPL BX, $0 // i < 0 + JL L5 + + MOVL CX, c+32(FP) + RET + + +// func addMulVVW(z, x []Word, y Word) (c Word) +TEXT ·addMulVVW(SB),7,$0 + MOVL z+0(FP), DI + MOVL x+12(FP), SI + MOVL y+24(FP), BP + MOVL n+4(FP), BX + LEAL (DI)(BX*4), DI + LEAL (SI)(BX*4), SI + NEGL BX // i = -n + MOVL $0, CX // c = 0 + JMP E6 + +L6: MOVL (SI)(BX*4), AX + MULL BP + ADDL CX, AX + ADCL $0, DX + ADDL AX, (DI)(BX*4) + ADCL $0, DX + MOVL DX, CX + ADDL $1, BX // i++ + +E6: CMPL BX, $0 // i < 0 + JL L6 + + MOVL CX, c+28(FP) + RET + + +// divWVW(z* Word, xn Word, x []Word, y Word) (r Word) +TEXT ·divWVW(SB),7,$0 + MOVL z+0(FP), DI + MOVL xn+12(FP), DX // r = xn + MOVL x+16(FP), SI + MOVL y+28(FP), CX + MOVL n+4(FP), BX // i = n + JMP E7 + +L7: MOVL (SI)(BX*4), AX + DIVL CX + MOVL AX, (DI)(BX*4) + +E7: SUBL $1, BX // i-- + JGE L7 // i >= 0 + + MOVL DX, r+32(FP) + RET diff --git a/src/pkg/big/arith_amd64.s b/src/pkg/big/arith_amd64.s new file mode 100644 index 000000000..89b65f38a --- /dev/null +++ b/src/pkg/big/arith_amd64.s @@ -0,0 +1,263 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// This file provides fast assembly versions for the elementary +// arithmetic operations on vectors implemented in arith.go. + +// TODO(gri) - experiment with unrolled loops for faster execution + +// func mulWW(x, y Word) (z1, z0 Word) +TEXT ·mulWW(SB),7,$0 + MOVQ x+0(FP), AX + MULQ y+8(FP) + MOVQ DX, z1+16(FP) + MOVQ AX, z0+24(FP) + RET + + +// func divWW(x1, x0, y Word) (q, r Word) +TEXT ·divWW(SB),7,$0 + MOVQ x1+0(FP), DX + MOVQ x0+8(FP), AX + DIVQ y+16(FP) + MOVQ AX, q+24(FP) + MOVQ DX, r+32(FP) + RET + + +// func addVV(z, x, y []Word) (c Word) +TEXT ·addVV(SB),7,$0 + MOVQ z+0(FP), R10 + MOVQ x+16(FP), R8 + MOVQ y+32(FP), R9 + MOVL n+8(FP), R11 + MOVQ $0, BX // i = 0 + MOVQ $0, DX // c = 0 + JMP E1 + +L1: MOVQ (R8)(BX*8), AX + RCRQ $1, DX + ADCQ (R9)(BX*8), AX + RCLQ $1, DX + MOVQ AX, (R10)(BX*8) + ADDL $1, BX // i++ + +E1: CMPQ BX, R11 // i < n + JL L1 + + MOVQ DX, c+48(FP) + RET + + +// func subVV(z, x, y []Word) (c Word) +// (same as addVV_s except for SBBQ instead of ADCQ and label names) +TEXT ·subVV(SB),7,$0 + MOVQ z+0(FP), R10 + MOVQ x+16(FP), R8 + MOVQ y+32(FP), R9 + MOVL n+8(FP), R11 + MOVQ $0, BX // i = 0 + MOVQ $0, DX // c = 0 + JMP E2 + +L2: MOVQ (R8)(BX*8), AX + RCRQ $1, DX + SBBQ (R9)(BX*8), AX + RCLQ $1, DX + MOVQ AX, (R10)(BX*8) + ADDL $1, BX // i++ + +E2: CMPQ BX, R11 // i < n + JL L2 + + MOVQ DX, c+48(FP) + RET + + +// func addVW(z, x []Word, y Word) (c Word) +TEXT ·addVW(SB),7,$0 + MOVQ z+0(FP), R10 + MOVQ x+16(FP), R8 + MOVQ y+32(FP), AX // c = y + MOVL n+8(FP), R11 + MOVQ $0, BX // i = 0 + JMP E3 + +L3: ADDQ (R8)(BX*8), AX + MOVQ AX, (R10)(BX*8) + RCLQ $1, AX + ANDQ $1, AX + ADDL $1, BX // i++ + +E3: CMPQ BX, R11 // i < n + JL L3 + + MOVQ AX, c+40(FP) + RET + + +// func subVW(z, x []Word, y Word) (c Word) +TEXT ·subVW(SB),7,$0 + MOVQ z+0(FP), R10 + MOVQ x+16(FP), R8 + MOVQ y+32(FP), AX // c = y + MOVL n+8(FP), R11 + MOVQ $0, BX // i = 0 + JMP E4 + +L4: MOVQ (R8)(BX*8), DX // TODO(gri) is there a reverse SUBQ? + SUBQ AX, DX + MOVQ DX, (R10)(BX*8) + RCLQ $1, AX + ANDQ $1, AX + ADDL $1, BX // i++ + +E4: CMPQ BX, R11 // i < n + JL L4 + + MOVQ AX, c+40(FP) + RET + + +// func shlVU(z, x []Word, s uint) (c Word) +TEXT ·shlVU(SB),7,$0 + MOVL n+8(FP), BX // i = n + SUBL $1, BX // i-- + JL X8b // i < 0 (n <= 0) + + // n > 0 + MOVQ z+0(FP), R10 + MOVQ x+16(FP), R8 + MOVL s+32(FP), CX + MOVQ (R8)(BX*8), AX // w1 = x[n-1] + MOVQ $0, DX + SHLQ CX, DX:AX // w1>>ŝ + MOVQ DX, c+40(FP) + + CMPL BX, $0 + JLE X8a // i <= 0 + + // i > 0 +L8: MOVQ AX, DX // w = w1 + MOVQ -8(R8)(BX*8), AX // w1 = x[i-1] + SHLQ CX, DX:AX // w<>ŝ + MOVQ DX, (R10)(BX*8) // z[i] = w<>ŝ + SUBL $1, BX // i-- + JG L8 // i > 0 + + // i <= 0 +X8a: SHLQ CX, AX // w1< 0 + MOVQ z+0(FP), R10 + MOVQ x+16(FP), R8 + MOVL s+32(FP), CX + MOVQ (R8), AX // w1 = x[0] + MOVQ $0, DX + SHRQ CX, DX:AX // w1<<ŝ + MOVQ DX, c+40(FP) + + MOVQ $0, BX // i = 0 + JMP E9 + + // i < n-1 +L9: MOVQ AX, DX // w = w1 + MOVQ 8(R8)(BX*8), AX // w1 = x[i+1] + SHRQ CX, DX:AX // w>>s | w1<<ŝ + MOVQ DX, (R10)(BX*8) // z[i] = w>>s | w1<<ŝ + ADDL $1, BX // i++ + +E9: CMPQ BX, R11 + JL L9 // i < n-1 + + // i >= n-1 +X9a: SHRQ CX, AX // w1>>s + MOVQ AX, (R10)(R11*8) // z[n-1] = w1>>s + RET + +X9b: MOVQ $0, c+40(FP) + RET + + +// func mulAddVWW(z, x []Word, y, r Word) (c Word) +TEXT ·mulAddVWW(SB),7,$0 + MOVQ z+0(FP), R10 + MOVQ x+16(FP), R8 + MOVQ y+32(FP), R9 + MOVQ r+40(FP), CX // c = r + MOVL n+8(FP), R11 + MOVQ $0, BX // i = 0 + JMP E5 + +L5: MOVQ (R8)(BX*8), AX + MULQ R9 + ADDQ CX, AX + ADCQ $0, DX + MOVQ AX, (R10)(BX*8) + MOVQ DX, CX + ADDL $1, BX // i++ + +E5: CMPQ BX, R11 // i < n + JL L5 + + MOVQ CX, c+48(FP) + RET + + +// func addMulVVW(z, x []Word, y Word) (c Word) +TEXT ·addMulVVW(SB),7,$0 + MOVQ z+0(FP), R10 + MOVQ x+16(FP), R8 + MOVQ y+32(FP), R9 + MOVL n+8(FP), R11 + MOVQ $0, BX // i = 0 + MOVQ $0, CX // c = 0 + JMP E6 + +L6: MOVQ (R8)(BX*8), AX + MULQ R9 + ADDQ CX, AX + ADCQ $0, DX + ADDQ AX, (R10)(BX*8) + ADCQ $0, DX + MOVQ DX, CX + ADDL $1, BX // i++ + +E6: CMPQ BX, R11 // i < n + JL L6 + + MOVQ CX, c+40(FP) + RET + + +// divWVW(z []Word, xn Word, x []Word, y Word) (r Word) +TEXT ·divWVW(SB),7,$0 + MOVQ z+0(FP), R10 + MOVQ xn+16(FP), DX // r = xn + MOVQ x+24(FP), R8 + MOVQ y+40(FP), R9 + MOVL n+8(FP), BX // i = n + JMP E7 + +L7: MOVQ (R8)(BX*8), AX + DIVQ R9 + MOVQ AX, (R10)(BX*8) + +E7: SUBL $1, BX // i-- + JGE L7 // i >= 0 + + MOVQ DX, r+48(FP) + RET diff --git a/src/pkg/big/arith_arm.s b/src/pkg/big/arith_arm.s new file mode 100644 index 000000000..60abe6eaa --- /dev/null +++ b/src/pkg/big/arith_arm.s @@ -0,0 +1,312 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// This file provides fast assembly versions for the elementary +// arithmetic operations on vectors implemented in arith.go. + +#define CFLAG 29 // bit position of carry flag + +// func addVV(z, x, y []Word) (c Word) +TEXT ·addVV(SB),7,$0 + MOVW $0, R0 + MOVW z+0(FP), R1 + MOVW x+12(FP), R2 + MOVW y+24(FP), R3 + MOVW n+4(FP), R4 + MOVW R4<<2, R4 + ADD R1, R4 + B E1 +L1: + MOVW.P 4(R2), R5 + MOVW.P 4(R3), R6 + MOVW R0, CPSR + ADC.S R6, R5 + MOVW.P R5, 4(R1) + MOVW CPSR, R0 +E1: + CMP R1, R4 + BNE L1 + + MOVW R0>>CFLAG, R0 + AND $1, R0 + MOVW R0, c+36(FP) + RET + + +// func subVV(z, x, y []Word) (c Word) +// (same as addVV except for SBC instead of ADC and label names) +TEXT ·subVV(SB),7,$0 + MOVW $(1<>CFLAG, R0 + AND $1, R0 + EOR $1, R0 + MOVW R0, c+36(FP) + RET + + +// func addVW(z, x []Word, y Word) (c Word) +TEXT ·addVW(SB),7,$0 + MOVW z+0(FP), R1 + MOVW x+12(FP), R2 + MOVW y+24(FP), R3 + MOVW n+4(FP), R4 + MOVW R4<<2, R4 + ADD R1, R4 + CMP R1, R4 + BNE L3a + MOVW R3, c+28(FP) + RET +L3a: + MOVW.P 4(R2), R5 + ADD.S R3, R5 + MOVW.P R5, 4(R1) + MOVW CPSR, R0 + B E3 +L3: + MOVW.P 4(R2), R5 + MOVW R0, CPSR + ADC.S $0, R5 + MOVW.P R5, 4(R1) + MOVW CPSR, R0 +E3: + CMP R1, R4 + BNE L3 + + MOVW R0>>CFLAG, R0 + AND $1, R0 + MOVW R0, c+28(FP) + RET + + +// func subVW(z, x []Word, y Word) (c Word) +TEXT ·subVW(SB),7,$0 + MOVW z+0(FP), R1 + MOVW x+12(FP), R2 + MOVW y+24(FP), R3 + MOVW n+4(FP), R4 + MOVW R4<<2, R4 + ADD R1, R4 + CMP R1, R4 + BNE L4a + MOVW R3, c+28(FP) + RET +L4a: + MOVW.P 4(R2), R5 + SUB.S R3, R5 + MOVW.P R5, 4(R1) + MOVW CPSR, R0 + B E4 +L4: + MOVW.P 4(R2), R5 + MOVW R0, CPSR + SBC.S $0, R5 + MOVW.P R5, 4(R1) + MOVW CPSR, R0 +E4: + CMP R1, R4 + BNE L4 + + MOVW R0>>CFLAG, R0 + AND $1, R0 + EOR $1, R0 + MOVW R0, c+28(FP) + RET + + +// func shlVU(z, x []Word, s uint) (c Word) +TEXT ·shlVU(SB),7,$0 + MOVW n+4(FP), R5 + CMP $0, R5 + BEQ X7 + + MOVW z+0(FP), R1 + MOVW x+12(FP), R2 + MOVW R5<<2, R5 + ADD R5, R2 + ADD R1, R5 + MOVW s+24(FP), R3 + CMP $0, R3 // shift 0 is special + BEQ Y7 + ADD $4, R1 // stop one word early + MOVW $32, R4 + SUB R3, R4 + MOVW $0, R7 + + MOVW.W -4(R2), R6 + MOVW R6<>R4, R6 + MOVW R6, c+28(FP) + B E7 + +L7: + MOVW.W -4(R2), R6 + ORR R6>>R4, R7 + MOVW.W R7, -4(R5) + MOVW R6<>R3, R7 + MOVW R6<>R3, R7 +E6: + CMP R1, R5 + BNE L6 + + MOVW R7, 0(R1) + RET + +Y6: // copy loop, because shift 0 == shift 32 + MOVW.P 4(R2), R6 + MOVW.P R6, 4(R1) + CMP R1, R5 + BNE Y6 + +X6: + MOVW $0, R1 + MOVW R1, c+28(FP) + RET + + +// func mulAddVWW(z, x []Word, y, r Word) (c Word) +TEXT ·mulAddVWW(SB),7,$0 + MOVW $0, R0 + MOVW z+0(FP), R1 + MOVW x+12(FP), R2 + MOVW y+24(FP), R3 + MOVW r+28(FP), R4 + MOVW n+4(FP), R5 + MOVW R5<<2, R5 + ADD R1, R5 + B E8 + + // word loop +L8: + MOVW.P 4(R2), R6 + MULLU R6, R3, (R7, R6) + ADD.S R4, R6 + ADC R0, R7 + MOVW.P R6, 4(R1) + MOVW R7, R4 +E8: + CMP R1, R5 + BNE L8 + + MOVW R4, c+32(FP) + RET + + +// func addMulVVW(z, x []Word, y Word) (c Word) +TEXT ·addMulVVW(SB),7,$0 + MOVW $0, R0 + MOVW z+0(FP), R1 + MOVW x+12(FP), R2 + MOVW y+24(FP), R3 + MOVW n+4(FP), R5 + MOVW R5<<2, R5 + ADD R1, R5 + MOVW $0, R4 + B E9 + + // word loop +L9: + MOVW.P 4(R2), R6 + MULLU R6, R3, (R7, R6) + ADD.S R4, R6 + ADC R0, R7 + MOVW 0(R1), R4 + ADD.S R4, R6 + ADC R0, R7 + MOVW.P R6, 4(R1) + MOVW R7, R4 +E9: + CMP R1, R5 + BNE L9 + + MOVW R4, c+28(FP) + RET + + +// divWVW(z* Word, xn Word, x []Word, y Word) (r Word) +TEXT ·divWVW(SB),7,$0 + // ARM has no multiword division, so use portable code. + B ·divWVW_g(SB) + + +// func divWW(x1, x0, y Word) (q, r Word) +TEXT ·divWW(SB),7,$0 + // ARM has no multiword division, so use portable code. + B ·divWW_g(SB) + + +// func mulWW(x, y Word) (z1, z0 Word) +TEXT ·mulWW(SB),7,$0 + MOVW x+0(FP), R1 + MOVW y+4(FP), R2 + MULLU R1, R2, (R4, R3) + MOVW R4, z1+8(FP) + MOVW R3, z0+12(FP) + RET diff --git a/src/pkg/big/arith_decl.go b/src/pkg/big/arith_decl.go new file mode 100644 index 000000000..95fcd8b94 --- /dev/null +++ b/src/pkg/big/arith_decl.go @@ -0,0 +1,18 @@ +// Copyright 2010 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package big + +// implemented in arith_$GOARCH.s +func mulWW(x, y Word) (z1, z0 Word) +func divWW(x1, x0, y Word) (q, r Word) +func addVV(z, x, y []Word) (c Word) +func subVV(z, x, y []Word) (c Word) +func addVW(z, x []Word, y Word) (c Word) +func subVW(z, x []Word, y Word) (c Word) +func shlVU(z, x []Word, s uint) (c Word) +func shrVU(z, x []Word, s uint) (c Word) +func mulAddVWW(z, x []Word, y, r Word) (c Word) +func addMulVVW(z, x []Word, y Word) (c Word) +func divWVW(z []Word, xn Word, x []Word, y Word) (r Word) diff --git a/src/pkg/big/arith_test.go b/src/pkg/big/arith_test.go new file mode 100644 index 000000000..b6c56c39e --- /dev/null +++ b/src/pkg/big/arith_test.go @@ -0,0 +1,335 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package big + +import "testing" + +type funWW func(x, y, c Word) (z1, z0 Word) +type argWW struct { + x, y, c, z1, z0 Word +} + +var sumWW = []argWW{ + {0, 0, 0, 0, 0}, + {0, 1, 0, 0, 1}, + {0, 0, 1, 0, 1}, + {0, 1, 1, 0, 2}, + {12345, 67890, 0, 0, 80235}, + {12345, 67890, 1, 0, 80236}, + {_M, 1, 0, 1, 0}, + {_M, 0, 1, 1, 0}, + {_M, 1, 1, 1, 1}, + {_M, _M, 0, 1, _M - 1}, + {_M, _M, 1, 1, _M}, +} + +func testFunWW(t *testing.T, msg string, f funWW, a argWW) { + z1, z0 := f(a.x, a.y, a.c) + if z1 != a.z1 || z0 != a.z0 { + t.Errorf("%s%+v\n\tgot z1:z0 = %#x:%#x; want %#x:%#x", msg, a, z1, z0, a.z1, a.z0) + } +} + +func TestFunWW(t *testing.T) { + for _, a := range sumWW { + arg := a + testFunWW(t, "addWW_g", addWW_g, arg) + + arg = argWW{a.y, a.x, a.c, a.z1, a.z0} + testFunWW(t, "addWW_g symmetric", addWW_g, arg) + + arg = argWW{a.z0, a.x, a.c, a.z1, a.y} + testFunWW(t, "subWW_g", subWW_g, arg) + + arg = argWW{a.z0, a.y, a.c, a.z1, a.x} + testFunWW(t, "subWW_g symmetric", subWW_g, arg) + } +} + +type funVV func(z, x, y []Word) (c Word) +type argVV struct { + z, x, y nat + c Word +} + +var sumVV = []argVV{ + {}, + {nat{0}, nat{0}, nat{0}, 0}, + {nat{1}, nat{1}, nat{0}, 0}, + {nat{0}, nat{_M}, nat{1}, 1}, + {nat{80235}, nat{12345}, nat{67890}, 0}, + {nat{_M - 1}, nat{_M}, nat{_M}, 1}, + {nat{0, 0, 0, 0}, nat{_M, _M, _M, _M}, nat{1, 0, 0, 0}, 1}, + {nat{0, 0, 0, _M}, nat{_M, _M, _M, _M - 1}, nat{1, 0, 0, 0}, 0}, + {nat{0, 0, 0, 0}, nat{_M, 0, _M, 0}, nat{1, _M, 0, _M}, 1}, +} + +func testFunVV(t *testing.T, msg string, f funVV, a argVV) { + z := make(nat, len(a.z)) + c := f(z, a.x, a.y) + for i, zi := range z { + if zi != a.z[i] { + t.Errorf("%s%+v\n\tgot z[%d] = %#x; want %#x", msg, a, i, zi, a.z[i]) + break + } + } + if c != a.c { + t.Errorf("%s%+v\n\tgot c = %#x; want %#x", msg, a, c, a.c) + } +} + +func TestFunVV(t *testing.T) { + for _, a := range sumVV { + arg := a + testFunVV(t, "addVV_g", addVV_g, arg) + testFunVV(t, "addVV", addVV, arg) + + arg = argVV{a.z, a.y, a.x, a.c} + testFunVV(t, "addVV_g symmetric", addVV_g, arg) + testFunVV(t, "addVV symmetric", addVV, arg) + + arg = argVV{a.x, a.z, a.y, a.c} + testFunVV(t, "subVV_g", subVV_g, arg) + testFunVV(t, "subVV", subVV, arg) + + arg = argVV{a.y, a.z, a.x, a.c} + testFunVV(t, "subVV_g symmetric", subVV_g, arg) + testFunVV(t, "subVV symmetric", subVV, arg) + } +} + +type funVW func(z, x []Word, y Word) (c Word) +type argVW struct { + z, x nat + y Word + c Word +} + +var sumVW = []argVW{ + {}, + {nil, nil, 2, 2}, + {nat{0}, nat{0}, 0, 0}, + {nat{1}, nat{0}, 1, 0}, + {nat{1}, nat{1}, 0, 0}, + {nat{0}, nat{_M}, 1, 1}, + {nat{0, 0, 0, 0}, nat{_M, _M, _M, _M}, 1, 1}, +} + +var prodVW = []argVW{ + {}, + {nat{0}, nat{0}, 0, 0}, + {nat{0}, nat{_M}, 0, 0}, + {nat{0}, nat{0}, _M, 0}, + {nat{1}, nat{1}, 1, 0}, + {nat{22793}, nat{991}, 23, 0}, + {nat{0, 0, 0, 22793}, nat{0, 0, 0, 991}, 23, 0}, + {nat{0, 0, 0, 0}, nat{7893475, 7395495, 798547395, 68943}, 0, 0}, + {nat{0, 0, 0, 0}, nat{0, 0, 0, 0}, 894375984, 0}, + {nat{_M << 1 & _M}, nat{_M}, 1 << 1, _M >> (_W - 1)}, + {nat{_M << 7 & _M}, nat{_M}, 1 << 7, _M >> (_W - 7)}, + {nat{_M << 7 & _M, _M, _M, _M}, nat{_M, _M, _M, _M}, 1 << 7, _M >> (_W - 7)}, +} + +var lshVW = []argVW{ + {}, + {nat{0}, nat{0}, 0, 0}, + {nat{0}, nat{0}, 1, 0}, + {nat{0}, nat{0}, 20, 0}, + + {nat{_M}, nat{_M}, 0, 0}, + {nat{_M << 1 & _M}, nat{_M}, 1, 1}, + {nat{_M << 20 & _M}, nat{_M}, 20, _M >> (_W - 20)}, + + {nat{_M, _M, _M}, nat{_M, _M, _M}, 0, 0}, + {nat{_M << 1 & _M, _M, _M}, nat{_M, _M, _M}, 1, 1}, + {nat{_M << 20 & _M, _M, _M}, nat{_M, _M, _M}, 20, _M >> (_W - 20)}, +} + +var rshVW = []argVW{ + {}, + {nat{0}, nat{0}, 0, 0}, + {nat{0}, nat{0}, 1, 0}, + {nat{0}, nat{0}, 20, 0}, + + {nat{_M}, nat{_M}, 0, 0}, + {nat{_M >> 1}, nat{_M}, 1, _M << (_W - 1) & _M}, + {nat{_M >> 20}, nat{_M}, 20, _M << (_W - 20) & _M}, + + {nat{_M, _M, _M}, nat{_M, _M, _M}, 0, 0}, + {nat{_M, _M, _M >> 1}, nat{_M, _M, _M}, 1, _M << (_W - 1) & _M}, + {nat{_M, _M, _M >> 20}, nat{_M, _M, _M}, 20, _M << (_W - 20) & _M}, +} + +func testFunVW(t *testing.T, msg string, f funVW, a argVW) { + z := make(nat, len(a.z)) + c := f(z, a.x, a.y) + for i, zi := range z { + if zi != a.z[i] { + t.Errorf("%s%+v\n\tgot z[%d] = %#x; want %#x", msg, a, i, zi, a.z[i]) + break + } + } + if c != a.c { + t.Errorf("%s%+v\n\tgot c = %#x; want %#x", msg, a, c, a.c) + } +} + +func makeFunVW(f func(z, x []Word, s uint) (c Word)) funVW { + return func(z, x []Word, s Word) (c Word) { + return f(z, x, uint(s)) + } +} + +func TestFunVW(t *testing.T) { + for _, a := range sumVW { + arg := a + testFunVW(t, "addVW_g", addVW_g, arg) + testFunVW(t, "addVW", addVW, arg) + + arg = argVW{a.x, a.z, a.y, a.c} + testFunVW(t, "subVW_g", subVW_g, arg) + testFunVW(t, "subVW", subVW, arg) + } + + shlVW_g := makeFunVW(shlVU_g) + shlVW := makeFunVW(shlVU) + for _, a := range lshVW { + arg := a + testFunVW(t, "shlVU_g", shlVW_g, arg) + testFunVW(t, "shlVU", shlVW, arg) + } + + shrVW_g := makeFunVW(shrVU_g) + shrVW := makeFunVW(shrVU) + for _, a := range rshVW { + arg := a + testFunVW(t, "shrVU_g", shrVW_g, arg) + testFunVW(t, "shrVU", shrVW, arg) + } +} + +type funVWW func(z, x []Word, y, r Word) (c Word) +type argVWW struct { + z, x nat + y, r Word + c Word +} + +var prodVWW = []argVWW{ + {}, + {nat{0}, nat{0}, 0, 0, 0}, + {nat{991}, nat{0}, 0, 991, 0}, + {nat{0}, nat{_M}, 0, 0, 0}, + {nat{991}, nat{_M}, 0, 991, 0}, + {nat{0}, nat{0}, _M, 0, 0}, + {nat{991}, nat{0}, _M, 991, 0}, + {nat{1}, nat{1}, 1, 0, 0}, + {nat{992}, nat{1}, 1, 991, 0}, + {nat{22793}, nat{991}, 23, 0, 0}, + {nat{22800}, nat{991}, 23, 7, 0}, + {nat{0, 0, 0, 22793}, nat{0, 0, 0, 991}, 23, 0, 0}, + {nat{7, 0, 0, 22793}, nat{0, 0, 0, 991}, 23, 7, 0}, + {nat{0, 0, 0, 0}, nat{7893475, 7395495, 798547395, 68943}, 0, 0, 0}, + {nat{991, 0, 0, 0}, nat{7893475, 7395495, 798547395, 68943}, 0, 991, 0}, + {nat{0, 0, 0, 0}, nat{0, 0, 0, 0}, 894375984, 0, 0}, + {nat{991, 0, 0, 0}, nat{0, 0, 0, 0}, 894375984, 991, 0}, + {nat{_M << 1 & _M}, nat{_M}, 1 << 1, 0, _M >> (_W - 1)}, + {nat{_M<<1&_M + 1}, nat{_M}, 1 << 1, 1, _M >> (_W - 1)}, + {nat{_M << 7 & _M}, nat{_M}, 1 << 7, 0, _M >> (_W - 7)}, + {nat{_M<<7&_M + 1<<6}, nat{_M}, 1 << 7, 1 << 6, _M >> (_W - 7)}, + {nat{_M << 7 & _M, _M, _M, _M}, nat{_M, _M, _M, _M}, 1 << 7, 0, _M >> (_W - 7)}, + {nat{_M<<7&_M + 1<<6, _M, _M, _M}, nat{_M, _M, _M, _M}, 1 << 7, 1 << 6, _M >> (_W - 7)}, +} + +func testFunVWW(t *testing.T, msg string, f funVWW, a argVWW) { + z := make(nat, len(a.z)) + c := f(z, a.x, a.y, a.r) + for i, zi := range z { + if zi != a.z[i] { + t.Errorf("%s%+v\n\tgot z[%d] = %#x; want %#x", msg, a, i, zi, a.z[i]) + break + } + } + if c != a.c { + t.Errorf("%s%+v\n\tgot c = %#x; want %#x", msg, a, c, a.c) + } +} + +// TODO(gri) mulAddVWW and divWVW are symmetric operations but +// their signature is not symmetric. Try to unify. + +type funWVW func(z []Word, xn Word, x []Word, y Word) (r Word) +type argWVW struct { + z nat + xn Word + x nat + y Word + r Word +} + +func testFunWVW(t *testing.T, msg string, f funWVW, a argWVW) { + z := make(nat, len(a.z)) + r := f(z, a.xn, a.x, a.y) + for i, zi := range z { + if zi != a.z[i] { + t.Errorf("%s%+v\n\tgot z[%d] = %#x; want %#x", msg, a, i, zi, a.z[i]) + break + } + } + if r != a.r { + t.Errorf("%s%+v\n\tgot r = %#x; want %#x", msg, a, r, a.r) + } +} + +func TestFunVWW(t *testing.T) { + for _, a := range prodVWW { + arg := a + testFunVWW(t, "mulAddVWW_g", mulAddVWW_g, arg) + testFunVWW(t, "mulAddVWW", mulAddVWW, arg) + + if a.y != 0 && a.r < a.y { + arg := argWVW{a.x, a.c, a.z, a.y, a.r} + testFunWVW(t, "divWVW_g", divWVW_g, arg) + testFunWVW(t, "divWVW", divWVW, arg) + } + } +} + +var mulWWTests = []struct { + x, y Word + q, r Word +}{ + {_M, _M, _M - 1, 1}, + // 32 bit only: {0xc47dfa8c, 50911, 0x98a4, 0x998587f4}, +} + +func TestMulWW(t *testing.T) { + for i, test := range mulWWTests { + q, r := mulWW_g(test.x, test.y) + if q != test.q || r != test.r { + t.Errorf("#%d got (%x, %x) want (%x, %x)", i, q, r, test.q, test.r) + } + } +} + +var mulAddWWWTests = []struct { + x, y, c Word + q, r Word +}{ + // TODO(agl): These will only work on 64-bit platforms. + // {15064310297182388543, 0xe7df04d2d35d5d80, 13537600649892366549, 13644450054494335067, 10832252001440893781}, + // {15064310297182388543, 0xdab2f18048baa68d, 13644450054494335067, 12869334219691522700, 14233854684711418382}, + {_M, _M, 0, _M - 1, 1}, + {_M, _M, _M, _M, 0}, +} + +func TestMulAddWWW(t *testing.T) { + for i, test := range mulAddWWWTests { + q, r := mulAddWWW_g(test.x, test.y, test.c) + if q != test.q || r != test.r { + t.Errorf("#%d got (%x, %x) want (%x, %x)", i, q, r, test.q, test.r) + } + } +} diff --git a/src/pkg/big/calibrate_test.go b/src/pkg/big/calibrate_test.go new file mode 100644 index 000000000..1cd93b105 --- /dev/null +++ b/src/pkg/big/calibrate_test.go @@ -0,0 +1,88 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// This file prints execution times for the Mul benchmark +// given different Karatsuba thresholds. The result may be +// used to manually fine-tune the threshold constant. The +// results are somewhat fragile; use repeated runs to get +// a clear picture. + +// Usage: gotest -calibrate + +package big + +import ( + "flag" + "fmt" + "testing" + "time" +) + +var calibrate = flag.Bool("calibrate", false, "run calibration test") + +// measure returns the time to run f +func measure(f func()) int64 { + const N = 100 + start := time.Nanoseconds() + for i := N; i > 0; i-- { + f() + } + stop := time.Nanoseconds() + return (stop - start) / N +} + +func computeThresholds() { + fmt.Printf("Multiplication times for varying Karatsuba thresholds\n") + fmt.Printf("(run repeatedly for good results)\n") + + // determine Tk, the work load execution time using basic multiplication + karatsubaThreshold = 1e9 // disable karatsuba + Tb := measure(benchmarkMulLoad) + fmt.Printf("Tb = %dns\n", Tb) + + // thresholds + n := 8 // any lower values for the threshold lead to very slow multiplies + th1 := -1 + th2 := -1 + + var deltaOld int64 + for count := -1; count != 0; count-- { + // determine Tk, the work load execution time using Karatsuba multiplication + karatsubaThreshold = n // enable karatsuba + Tk := measure(benchmarkMulLoad) + + // improvement over Tb + delta := (Tb - Tk) * 100 / Tb + + fmt.Printf("n = %3d Tk = %8dns %4d%%", n, Tk, delta) + + // determine break-even point + if Tk < Tb && th1 < 0 { + th1 = n + fmt.Print(" break-even point") + } + + // determine diminishing return + if 0 < delta && delta < deltaOld && th2 < 0 { + th2 = n + fmt.Print(" diminishing return") + } + deltaOld = delta + + fmt.Println() + + // trigger counter + if th1 >= 0 && th2 >= 0 && count < 0 { + count = 20 // this many extra measurements after we got both thresholds + } + + n++ + } +} + +func TestCalibrate(t *testing.T) { + if *calibrate { + computeThresholds() + } +} diff --git a/src/pkg/big/hilbert_test.go b/src/pkg/big/hilbert_test.go new file mode 100644 index 000000000..1a84341b3 --- /dev/null +++ b/src/pkg/big/hilbert_test.go @@ -0,0 +1,160 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// A little test program and benchmark for rational arithmetics. +// Computes a Hilbert matrix, its inverse, multiplies them +// and verifies that the product is the identity matrix. + +package big + +import ( + "fmt" + "testing" +) + +type matrix struct { + n, m int + a []*Rat +} + +func (a *matrix) at(i, j int) *Rat { + if !(0 <= i && i < a.n && 0 <= j && j < a.m) { + panic("index out of range") + } + return a.a[i*a.m+j] +} + +func (a *matrix) set(i, j int, x *Rat) { + if !(0 <= i && i < a.n && 0 <= j && j < a.m) { + panic("index out of range") + } + a.a[i*a.m+j] = x +} + +func newMatrix(n, m int) *matrix { + if !(0 <= n && 0 <= m) { + panic("illegal matrix") + } + a := new(matrix) + a.n = n + a.m = m + a.a = make([]*Rat, n*m) + return a +} + +func newUnit(n int) *matrix { + a := newMatrix(n, n) + for i := 0; i < n; i++ { + for j := 0; j < n; j++ { + x := NewRat(0, 1) + if i == j { + x.SetInt64(1) + } + a.set(i, j, x) + } + } + return a +} + +func newHilbert(n int) *matrix { + a := newMatrix(n, n) + for i := 0; i < n; i++ { + for j := 0; j < n; j++ { + a.set(i, j, NewRat(1, int64(i+j+1))) + } + } + return a +} + +func newInverseHilbert(n int) *matrix { + a := newMatrix(n, n) + for i := 0; i < n; i++ { + for j := 0; j < n; j++ { + x1 := new(Rat).SetInt64(int64(i + j + 1)) + x2 := new(Rat).SetInt(new(Int).Binomial(int64(n+i), int64(n-j-1))) + x3 := new(Rat).SetInt(new(Int).Binomial(int64(n+j), int64(n-i-1))) + x4 := new(Rat).SetInt(new(Int).Binomial(int64(i+j), int64(i))) + + x1.Mul(x1, x2) + x1.Mul(x1, x3) + x1.Mul(x1, x4) + x1.Mul(x1, x4) + + if (i+j)&1 != 0 { + x1.Neg(x1) + } + + a.set(i, j, x1) + } + } + return a +} + +func (a *matrix) mul(b *matrix) *matrix { + if a.m != b.n { + panic("illegal matrix multiply") + } + c := newMatrix(a.n, b.m) + for i := 0; i < c.n; i++ { + for j := 0; j < c.m; j++ { + x := NewRat(0, 1) + for k := 0; k < a.m; k++ { + x.Add(x, new(Rat).Mul(a.at(i, k), b.at(k, j))) + } + c.set(i, j, x) + } + } + return c +} + +func (a *matrix) eql(b *matrix) bool { + if a.n != b.n || a.m != b.m { + return false + } + for i := 0; i < a.n; i++ { + for j := 0; j < a.m; j++ { + if a.at(i, j).Cmp(b.at(i, j)) != 0 { + return false + } + } + } + return true +} + +func (a *matrix) String() string { + s := "" + for i := 0; i < a.n; i++ { + for j := 0; j < a.m; j++ { + s += fmt.Sprintf("\t%s", a.at(i, j)) + } + s += "\n" + } + return s +} + +func doHilbert(t *testing.T, n int) { + a := newHilbert(n) + b := newInverseHilbert(n) + I := newUnit(n) + ab := a.mul(b) + if !ab.eql(I) { + if t == nil { + panic("Hilbert failed") + } + t.Errorf("a = %s\n", a) + t.Errorf("b = %s\n", b) + t.Errorf("a*b = %s\n", ab) + t.Errorf("I = %s\n", I) + } +} + +func TestHilbert(t *testing.T) { + doHilbert(t, 10) +} + +func BenchmarkHilbert(b *testing.B) { + for i := 0; i < b.N; i++ { + doHilbert(nil, 10) + } +} diff --git a/src/pkg/big/int.go b/src/pkg/big/int.go new file mode 100755 index 000000000..701b69715 --- /dev/null +++ b/src/pkg/big/int.go @@ -0,0 +1,868 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// This file implements signed multi-precision integers. + +package big + +import ( + "fmt" + "io" + "os" + "rand" + "strings" +) + +// An Int represents a signed multi-precision integer. +// The zero value for an Int represents the value 0. +type Int struct { + neg bool // sign + abs nat // absolute value of the integer +} + +var intOne = &Int{false, natOne} + +// Sign returns: +// +// -1 if x < 0 +// 0 if x == 0 +// +1 if x > 0 +// +func (x *Int) Sign() int { + if len(x.abs) == 0 { + return 0 + } + if x.neg { + return -1 + } + return 1 +} + +// SetInt64 sets z to x and returns z. +func (z *Int) SetInt64(x int64) *Int { + neg := false + if x < 0 { + neg = true + x = -x + } + z.abs = z.abs.setUint64(uint64(x)) + z.neg = neg + return z +} + +// NewInt allocates and returns a new Int set to x. +func NewInt(x int64) *Int { + return new(Int).SetInt64(x) +} + +// Set sets z to x and returns z. +func (z *Int) Set(x *Int) *Int { + z.abs = z.abs.set(x.abs) + z.neg = x.neg + return z +} + +// Abs sets z to |x| (the absolute value of x) and returns z. +func (z *Int) Abs(x *Int) *Int { + z.abs = z.abs.set(x.abs) + z.neg = false + return z +} + +// Neg sets z to -x and returns z. +func (z *Int) Neg(x *Int) *Int { + z.abs = z.abs.set(x.abs) + z.neg = len(z.abs) > 0 && !x.neg // 0 has no sign + return z +} + +// Add sets z to the sum x+y and returns z. +func (z *Int) Add(x, y *Int) *Int { + neg := x.neg + if x.neg == y.neg { + // x + y == x + y + // (-x) + (-y) == -(x + y) + z.abs = z.abs.add(x.abs, y.abs) + } else { + // x + (-y) == x - y == -(y - x) + // (-x) + y == y - x == -(x - y) + if x.abs.cmp(y.abs) >= 0 { + z.abs = z.abs.sub(x.abs, y.abs) + } else { + neg = !neg + z.abs = z.abs.sub(y.abs, x.abs) + } + } + z.neg = len(z.abs) > 0 && neg // 0 has no sign + return z +} + +// Sub sets z to the difference x-y and returns z. +func (z *Int) Sub(x, y *Int) *Int { + neg := x.neg + if x.neg != y.neg { + // x - (-y) == x + y + // (-x) - y == -(x + y) + z.abs = z.abs.add(x.abs, y.abs) + } else { + // x - y == x - y == -(y - x) + // (-x) - (-y) == y - x == -(x - y) + if x.abs.cmp(y.abs) >= 0 { + z.abs = z.abs.sub(x.abs, y.abs) + } else { + neg = !neg + z.abs = z.abs.sub(y.abs, x.abs) + } + } + z.neg = len(z.abs) > 0 && neg // 0 has no sign + return z +} + +// Mul sets z to the product x*y and returns z. +func (z *Int) Mul(x, y *Int) *Int { + // x * y == x * y + // x * (-y) == -(x * y) + // (-x) * y == -(x * y) + // (-x) * (-y) == x * y + z.abs = z.abs.mul(x.abs, y.abs) + z.neg = len(z.abs) > 0 && x.neg != y.neg // 0 has no sign + return z +} + +// MulRange sets z to the product of all integers +// in the range [a, b] inclusively and returns z. +// If a > b (empty range), the result is 1. +func (z *Int) MulRange(a, b int64) *Int { + switch { + case a > b: + return z.SetInt64(1) // empty range + case a <= 0 && b >= 0: + return z.SetInt64(0) // range includes 0 + } + // a <= b && (b < 0 || a > 0) + + neg := false + if a < 0 { + neg = (b-a)&1 == 0 + a, b = -b, -a + } + + z.abs = z.abs.mulRange(uint64(a), uint64(b)) + z.neg = neg + return z +} + +// Binomial sets z to the binomial coefficient of (n, k) and returns z. +func (z *Int) Binomial(n, k int64) *Int { + var a, b Int + a.MulRange(n-k+1, n) + b.MulRange(1, k) + return z.Quo(&a, &b) +} + +// Quo sets z to the quotient x/y for y != 0 and returns z. +// If y == 0, a division-by-zero run-time panic occurs. +// See QuoRem for more details. +func (z *Int) Quo(x, y *Int) *Int { + z.abs, _ = z.abs.div(nil, x.abs, y.abs) + z.neg = len(z.abs) > 0 && x.neg != y.neg // 0 has no sign + return z +} + +// Rem sets z to the remainder x%y for y != 0 and returns z. +// If y == 0, a division-by-zero run-time panic occurs. +// See QuoRem for more details. +func (z *Int) Rem(x, y *Int) *Int { + _, z.abs = nat(nil).div(z.abs, x.abs, y.abs) + z.neg = len(z.abs) > 0 && x.neg // 0 has no sign + return z +} + +// QuoRem sets z to the quotient x/y and r to the remainder x%y +// and returns the pair (z, r) for y != 0. +// If y == 0, a division-by-zero run-time panic occurs. +// +// QuoRem implements T-division and modulus (like Go): +// +// q = x/y with the result truncated to zero +// r = x - y*q +// +// (See Daan Leijen, ``Division and Modulus for Computer Scientists''.) +// +func (z *Int) QuoRem(x, y, r *Int) (*Int, *Int) { + z.abs, r.abs = z.abs.div(r.abs, x.abs, y.abs) + z.neg, r.neg = len(z.abs) > 0 && x.neg != y.neg, len(r.abs) > 0 && x.neg // 0 has no sign + return z, r +} + +// Div sets z to the quotient x/y for y != 0 and returns z. +// If y == 0, a division-by-zero run-time panic occurs. +// See DivMod for more details. +func (z *Int) Div(x, y *Int) *Int { + y_neg := y.neg // z may be an alias for y + var r Int + z.QuoRem(x, y, &r) + if r.neg { + if y_neg { + z.Add(z, intOne) + } else { + z.Sub(z, intOne) + } + } + return z +} + +// Mod sets z to the modulus x%y for y != 0 and returns z. +// If y == 0, a division-by-zero run-time panic occurs. +// See DivMod for more details. +func (z *Int) Mod(x, y *Int) *Int { + y0 := y // save y + if z == y || alias(z.abs, y.abs) { + y0 = new(Int).Set(y) + } + var q Int + q.QuoRem(x, y, z) + if z.neg { + if y0.neg { + z.Sub(z, y0) + } else { + z.Add(z, y0) + } + } + return z +} + +// DivMod sets z to the quotient x div y and m to the modulus x mod y +// and returns the pair (z, m) for y != 0. +// If y == 0, a division-by-zero run-time panic occurs. +// +// DivMod implements Euclidean division and modulus (unlike Go): +// +// q = x div y such that +// m = x - y*q with 0 <= m < |q| +// +// (See Raymond T. Boute, ``The Euclidean definition of the functions +// div and mod''. ACM Transactions on Programming Languages and +// Systems (TOPLAS), 14(2):127-144, New York, NY, USA, 4/1992. +// ACM press.) +// +func (z *Int) DivMod(x, y, m *Int) (*Int, *Int) { + y0 := y // save y + if z == y || alias(z.abs, y.abs) { + y0 = new(Int).Set(y) + } + z.QuoRem(x, y, m) + if m.neg { + if y0.neg { + z.Add(z, intOne) + m.Sub(m, y0) + } else { + z.Sub(z, intOne) + m.Add(m, y0) + } + } + return z, m +} + +// Cmp compares x and y and returns: +// +// -1 if x < y +// 0 if x == y +// +1 if x > y +// +func (x *Int) Cmp(y *Int) (r int) { + // x cmp y == x cmp y + // x cmp (-y) == x + // (-x) cmp y == y + // (-x) cmp (-y) == -(x cmp y) + switch { + case x.neg == y.neg: + r = x.abs.cmp(y.abs) + if x.neg { + r = -r + } + case x.neg: + r = -1 + default: + r = 1 + } + return +} + +func (x *Int) String() string { + switch { + case x == nil: + return "" + case x.neg: + return "-" + x.abs.decimalString() + } + return x.abs.decimalString() +} + +func charset(ch int) string { + switch ch { + case 'b': + return lowercaseDigits[0:2] + case 'o': + return lowercaseDigits[0:8] + case 'd', 's', 'v': + return lowercaseDigits[0:10] + case 'x': + return lowercaseDigits[0:16] + case 'X': + return uppercaseDigits[0:16] + } + return "" // unknown format +} + +// write count copies of text to s +func writeMultiple(s fmt.State, text string, count int) { + if len(text) > 0 { + b := []byte(text) + for ; count > 0; count-- { + s.Write(b) + } + } +} + +// Format is a support routine for fmt.Formatter. It accepts +// the formats 'b' (binary), 'o' (octal), 'd' (decimal), 'x' +// (lowercase hexadecimal), and 'X' (uppercase hexadecimal). +// Also supported are the full suite of package fmt's format +// verbs for integral types, including '+', '-', and ' ' +// for sign control, '#' for leading zero in octal and for +// hexadecimal, a leading "0x" or "0X" for "%#x" and "%#X" +// respectively, specification of minimum digits precision, +// output field width, space or zero padding, and left or +// right justification. +// +func (x *Int) Format(s fmt.State, ch int) { + cs := charset(ch) + + // special cases + switch { + case cs == "": + // unknown format + fmt.Fprintf(s, "%%!%c(big.Int=%s)", ch, x.String()) + return + case x == nil: + fmt.Fprint(s, "") + return + } + + // determine sign character + sign := "" + switch { + case x.neg: + sign = "-" + case s.Flag('+'): // supersedes ' ' when both specified + sign = "+" + case s.Flag(' '): + sign = " " + } + + // determine prefix characters for indicating output base + prefix := "" + if s.Flag('#') { + switch ch { + case 'o': // octal + prefix = "0" + case 'x': // hexadecimal + prefix = "0x" + case 'X': + prefix = "0X" + } + } + + // determine digits with base set by len(cs) and digit characters from cs + digits := x.abs.string(cs) + + // number of characters for the three classes of number padding + var left int // space characters to left of digits for right justification ("%8d") + var zeroes int // zero characters (actually cs[0]) as left-most digits ("%.8d") + var right int // space characters to right of digits for left justification ("%-8d") + + // determine number padding from precision: the least number of digits to output + precision, precisionSet := s.Precision() + if precisionSet { + switch { + case len(digits) < precision: + zeroes = precision - len(digits) // count of zero padding + case digits == "0" && precision == 0: + return // print nothing if zero value (x == 0) and zero precision ("." or ".0") + } + } + + // determine field pad from width: the least number of characters to output + length := len(sign) + len(prefix) + zeroes + len(digits) + if width, widthSet := s.Width(); widthSet && length < width { // pad as specified + switch d := width - length; { + case s.Flag('-'): + // pad on the right with spaces; supersedes '0' when both specified + right = d + case s.Flag('0') && !precisionSet: + // pad with zeroes unless precision also specified + zeroes = d + default: + // pad on the left with spaces + left = d + } + } + + // print number as [left pad][sign][prefix][zero pad][digits][right pad] + writeMultiple(s, " ", left) + writeMultiple(s, sign, 1) + writeMultiple(s, prefix, 1) + writeMultiple(s, "0", zeroes) + writeMultiple(s, digits, 1) + writeMultiple(s, " ", right) +} + +// scan sets z to the integer value corresponding to the longest possible prefix +// read from r representing a signed integer number in a given conversion base. +// It returns z, the actual conversion base used, and an error, if any. In the +// error case, the value of z is undefined. The syntax follows the syntax of +// integer literals in Go. +// +// The base argument must be 0 or a value from 2 through MaxBase. If the base +// is 0, the string prefix determines the actual conversion base. A prefix of +// ``0x'' or ``0X'' selects base 16; the ``0'' prefix selects base 8, and a +// ``0b'' or ``0B'' prefix selects base 2. Otherwise the selected base is 10. +// +func (z *Int) scan(r io.RuneScanner, base int) (*Int, int, os.Error) { + // determine sign + ch, _, err := r.ReadRune() + if err != nil { + return z, 0, err + } + neg := false + switch ch { + case '-': + neg = true + case '+': // nothing to do + default: + r.UnreadRune() + } + + // determine mantissa + z.abs, base, err = z.abs.scan(r, base) + if err != nil { + return z, base, err + } + z.neg = len(z.abs) > 0 && neg // 0 has no sign + + return z, base, nil +} + +// Scan is a support routine for fmt.Scanner; it sets z to the value of +// the scanned number. It accepts the formats 'b' (binary), 'o' (octal), +// 'd' (decimal), 'x' (lowercase hexadecimal), and 'X' (uppercase hexadecimal). +func (z *Int) Scan(s fmt.ScanState, ch int) os.Error { + s.SkipSpace() // skip leading space characters + base := 0 + switch ch { + case 'b': + base = 2 + case 'o': + base = 8 + case 'd': + base = 10 + case 'x', 'X': + base = 16 + case 's', 'v': + // let scan determine the base + default: + return os.NewError("Int.Scan: invalid verb") + } + _, _, err := z.scan(s, base) + return err +} + +// Int64 returns the int64 representation of x. +// If x cannot be represented in an int64, the result is undefined. +func (x *Int) Int64() int64 { + if len(x.abs) == 0 { + return 0 + } + v := int64(x.abs[0]) + if _W == 32 && len(x.abs) > 1 { + v |= int64(x.abs[1]) << 32 + } + if x.neg { + v = -v + } + return v +} + +// SetString sets z to the value of s, interpreted in the given base, +// and returns z and a boolean indicating success. If SetString fails, +// the value of z is undefined. +// +// The base argument must be 0 or a value from 2 through MaxBase. If the base +// is 0, the string prefix determines the actual conversion base. A prefix of +// ``0x'' or ``0X'' selects base 16; the ``0'' prefix selects base 8, and a +// ``0b'' or ``0B'' prefix selects base 2. Otherwise the selected base is 10. +// +func (z *Int) SetString(s string, base int) (*Int, bool) { + r := strings.NewReader(s) + _, _, err := z.scan(r, base) + if err != nil { + return z, false + } + _, _, err = r.ReadRune() + return z, err == os.EOF // err == os.EOF => scan consumed all of s +} + +// SetBytes interprets buf as the bytes of a big-endian unsigned +// integer, sets z to that value, and returns z. +func (z *Int) SetBytes(buf []byte) *Int { + z.abs = z.abs.setBytes(buf) + z.neg = false + return z +} + +// Bytes returns the absolute value of z as a big-endian byte slice. +func (z *Int) Bytes() []byte { + buf := make([]byte, len(z.abs)*_S) + return buf[z.abs.bytes(buf):] +} + +// BitLen returns the length of the absolute value of z in bits. +// The bit length of 0 is 0. +func (z *Int) BitLen() int { + return z.abs.bitLen() +} + +// Exp sets z = x**y mod m. If m is nil, z = x**y. +// See Knuth, volume 2, section 4.6.3. +func (z *Int) Exp(x, y, m *Int) *Int { + if y.neg || len(y.abs) == 0 { + neg := x.neg + z.SetInt64(1) + z.neg = neg + return z + } + + var mWords nat + if m != nil { + mWords = m.abs + } + + z.abs = z.abs.expNN(x.abs, y.abs, mWords) + z.neg = len(z.abs) > 0 && x.neg && y.abs[0]&1 == 1 // 0 has no sign + return z +} + +// GcdInt sets d to the greatest common divisor of a and b, which must be +// positive numbers. +// If x and y are not nil, GcdInt sets x and y such that d = a*x + b*y. +// If either a or b is not positive, GcdInt sets d = x = y = 0. +func GcdInt(d, x, y, a, b *Int) { + if a.neg || b.neg { + d.SetInt64(0) + if x != nil { + x.SetInt64(0) + } + if y != nil { + y.SetInt64(0) + } + return + } + + A := new(Int).Set(a) + B := new(Int).Set(b) + + X := new(Int) + Y := new(Int).SetInt64(1) + + lastX := new(Int).SetInt64(1) + lastY := new(Int) + + q := new(Int) + temp := new(Int) + + for len(B.abs) > 0 { + r := new(Int) + q, r = q.QuoRem(A, B, r) + + A, B = B, r + + temp.Set(X) + X.Mul(X, q) + X.neg = !X.neg + X.Add(X, lastX) + lastX.Set(temp) + + temp.Set(Y) + Y.Mul(Y, q) + Y.neg = !Y.neg + Y.Add(Y, lastY) + lastY.Set(temp) + } + + if x != nil { + *x = *lastX + } + + if y != nil { + *y = *lastY + } + + *d = *A +} + +// ProbablyPrime performs n Miller-Rabin tests to check whether z is prime. +// If it returns true, z is prime with probability 1 - 1/4^n. +// If it returns false, z is not prime. +func ProbablyPrime(z *Int, n int) bool { + return !z.neg && z.abs.probablyPrime(n) +} + +// Rand sets z to a pseudo-random number in [0, n) and returns z. +func (z *Int) Rand(rnd *rand.Rand, n *Int) *Int { + z.neg = false + if n.neg == true || len(n.abs) == 0 { + z.abs = nil + return z + } + z.abs = z.abs.random(rnd, n.abs, n.abs.bitLen()) + return z +} + +// ModInverse sets z to the multiplicative inverse of g in the group ℤ/pℤ (where +// p is a prime) and returns z. +func (z *Int) ModInverse(g, p *Int) *Int { + var d Int + GcdInt(&d, z, nil, g, p) + // x and y are such that g*x + p*y = d. Since p is prime, d = 1. Taking + // that modulo p results in g*x = 1, therefore x is the inverse element. + if z.neg { + z.Add(z, p) + } + return z +} + +// Lsh sets z = x << n and returns z. +func (z *Int) Lsh(x *Int, n uint) *Int { + z.abs = z.abs.shl(x.abs, n) + z.neg = x.neg + return z +} + +// Rsh sets z = x >> n and returns z. +func (z *Int) Rsh(x *Int, n uint) *Int { + if x.neg { + // (-x) >> s == ^(x-1) >> s == ^((x-1) >> s) == -(((x-1) >> s) + 1) + t := z.abs.sub(x.abs, natOne) // no underflow because |x| > 0 + t = t.shr(t, n) + z.abs = t.add(t, natOne) + z.neg = true // z cannot be zero if x is negative + return z + } + + z.abs = z.abs.shr(x.abs, n) + z.neg = false + return z +} + +// Bit returns the value of the i'th bit of z. That is, it +// returns (z>>i)&1. The bit index i must be >= 0. +func (z *Int) Bit(i int) uint { + if i < 0 { + panic("negative bit index") + } + if z.neg { + t := nat{}.sub(z.abs, natOne) + return t.bit(uint(i)) ^ 1 + } + + return z.abs.bit(uint(i)) +} + +// SetBit sets the i'th bit of z to bit and returns z. +// That is, if bit is 1 SetBit sets z = x | (1 << i); +// if bit is 0 it sets z = x &^ (1 << i). If bit is not 0 or 1, +// SetBit will panic. +func (z *Int) SetBit(x *Int, i int, b uint) *Int { + if i < 0 { + panic("negative bit index") + } + if x.neg { + t := z.abs.sub(x.abs, natOne) + t = t.setBit(t, uint(i), b^1) + z.abs = t.add(t, natOne) + z.neg = len(z.abs) > 0 + return z + } + z.abs = z.abs.setBit(x.abs, uint(i), b) + z.neg = false + return z +} + +// And sets z = x & y and returns z. +func (z *Int) And(x, y *Int) *Int { + if x.neg == y.neg { + if x.neg { + // (-x) & (-y) == ^(x-1) & ^(y-1) == ^((x-1) | (y-1)) == -(((x-1) | (y-1)) + 1) + x1 := nat{}.sub(x.abs, natOne) + y1 := nat{}.sub(y.abs, natOne) + z.abs = z.abs.add(z.abs.or(x1, y1), natOne) + z.neg = true // z cannot be zero if x and y are negative + return z + } + + // x & y == x & y + z.abs = z.abs.and(x.abs, y.abs) + z.neg = false + return z + } + + // x.neg != y.neg + if x.neg { + x, y = y, x // & is symmetric + } + + // x & (-y) == x & ^(y-1) == x &^ (y-1) + y1 := nat{}.sub(y.abs, natOne) + z.abs = z.abs.andNot(x.abs, y1) + z.neg = false + return z +} + +// AndNot sets z = x &^ y and returns z. +func (z *Int) AndNot(x, y *Int) *Int { + if x.neg == y.neg { + if x.neg { + // (-x) &^ (-y) == ^(x-1) &^ ^(y-1) == ^(x-1) & (y-1) == (y-1) &^ (x-1) + x1 := nat{}.sub(x.abs, natOne) + y1 := nat{}.sub(y.abs, natOne) + z.abs = z.abs.andNot(y1, x1) + z.neg = false + return z + } + + // x &^ y == x &^ y + z.abs = z.abs.andNot(x.abs, y.abs) + z.neg = false + return z + } + + if x.neg { + // (-x) &^ y == ^(x-1) &^ y == ^(x-1) & ^y == ^((x-1) | y) == -(((x-1) | y) + 1) + x1 := nat{}.sub(x.abs, natOne) + z.abs = z.abs.add(z.abs.or(x1, y.abs), natOne) + z.neg = true // z cannot be zero if x is negative and y is positive + return z + } + + // x &^ (-y) == x &^ ^(y-1) == x & (y-1) + y1 := nat{}.add(y.abs, natOne) + z.abs = z.abs.and(x.abs, y1) + z.neg = false + return z +} + +// Or sets z = x | y and returns z. +func (z *Int) Or(x, y *Int) *Int { + if x.neg == y.neg { + if x.neg { + // (-x) | (-y) == ^(x-1) | ^(y-1) == ^((x-1) & (y-1)) == -(((x-1) & (y-1)) + 1) + x1 := nat{}.sub(x.abs, natOne) + y1 := nat{}.sub(y.abs, natOne) + z.abs = z.abs.add(z.abs.and(x1, y1), natOne) + z.neg = true // z cannot be zero if x and y are negative + return z + } + + // x | y == x | y + z.abs = z.abs.or(x.abs, y.abs) + z.neg = false + return z + } + + // x.neg != y.neg + if x.neg { + x, y = y, x // | is symmetric + } + + // x | (-y) == x | ^(y-1) == ^((y-1) &^ x) == -(^((y-1) &^ x) + 1) + y1 := nat{}.sub(y.abs, natOne) + z.abs = z.abs.add(z.abs.andNot(y1, x.abs), natOne) + z.neg = true // z cannot be zero if one of x or y is negative + return z +} + +// Xor sets z = x ^ y and returns z. +func (z *Int) Xor(x, y *Int) *Int { + if x.neg == y.neg { + if x.neg { + // (-x) ^ (-y) == ^(x-1) ^ ^(y-1) == (x-1) ^ (y-1) + x1 := nat{}.sub(x.abs, natOne) + y1 := nat{}.sub(y.abs, natOne) + z.abs = z.abs.xor(x1, y1) + z.neg = false + return z + } + + // x ^ y == x ^ y + z.abs = z.abs.xor(x.abs, y.abs) + z.neg = false + return z + } + + // x.neg != y.neg + if x.neg { + x, y = y, x // ^ is symmetric + } + + // x ^ (-y) == x ^ ^(y-1) == ^(x ^ (y-1)) == -((x ^ (y-1)) + 1) + y1 := nat{}.sub(y.abs, natOne) + z.abs = z.abs.add(z.abs.xor(x.abs, y1), natOne) + z.neg = true // z cannot be zero if only one of x or y is negative + return z +} + +// Not sets z = ^x and returns z. +func (z *Int) Not(x *Int) *Int { + if x.neg { + // ^(-x) == ^(^(x-1)) == x-1 + z.abs = z.abs.sub(x.abs, natOne) + z.neg = false + return z + } + + // ^x == -x-1 == -(x+1) + z.abs = z.abs.add(x.abs, natOne) + z.neg = true // z cannot be zero if x is positive + return z +} + +// Gob codec version. Permits backward-compatible changes to the encoding. +const intGobVersion byte = 1 + +// GobEncode implements the gob.GobEncoder interface. +func (z *Int) GobEncode() ([]byte, os.Error) { + buf := make([]byte, 1+len(z.abs)*_S) // extra byte for version and sign bit + i := z.abs.bytes(buf) - 1 // i >= 0 + b := intGobVersion << 1 // make space for sign bit + if z.neg { + b |= 1 + } + buf[i] = b + return buf[i:], nil +} + +// GobDecode implements the gob.GobDecoder interface. +func (z *Int) GobDecode(buf []byte) os.Error { + if len(buf) == 0 { + return os.NewError("Int.GobDecode: no data") + } + b := buf[0] + if b>>1 != intGobVersion { + return os.NewError(fmt.Sprintf("Int.GobDecode: encoding version %d not supported", b>>1)) + } + z.neg = b&1 != 0 + z.abs = z.abs.setBytes(buf[1:]) + return nil +} diff --git a/src/pkg/big/int_test.go b/src/pkg/big/int_test.go new file mode 100755 index 000000000..03446d6ae --- /dev/null +++ b/src/pkg/big/int_test.go @@ -0,0 +1,1391 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package big + +import ( + "bytes" + "encoding/hex" + "fmt" + "gob" + "testing" + "testing/quick" +) + +func isNormalized(x *Int) bool { + if len(x.abs) == 0 { + return !x.neg + } + // len(x.abs) > 0 + return x.abs[len(x.abs)-1] != 0 +} + +type funZZ func(z, x, y *Int) *Int +type argZZ struct { + z, x, y *Int +} + +var sumZZ = []argZZ{ + {NewInt(0), NewInt(0), NewInt(0)}, + {NewInt(1), NewInt(1), NewInt(0)}, + {NewInt(1111111110), NewInt(123456789), NewInt(987654321)}, + {NewInt(-1), NewInt(-1), NewInt(0)}, + {NewInt(864197532), NewInt(-123456789), NewInt(987654321)}, + {NewInt(-1111111110), NewInt(-123456789), NewInt(-987654321)}, +} + +var prodZZ = []argZZ{ + {NewInt(0), NewInt(0), NewInt(0)}, + {NewInt(0), NewInt(1), NewInt(0)}, + {NewInt(1), NewInt(1), NewInt(1)}, + {NewInt(-991 * 991), NewInt(991), NewInt(-991)}, + // TODO(gri) add larger products +} + +func TestSignZ(t *testing.T) { + var zero Int + for _, a := range sumZZ { + s := a.z.Sign() + e := a.z.Cmp(&zero) + if s != e { + t.Errorf("got %d; want %d for z = %v", s, e, a.z) + } + } +} + +func TestSetZ(t *testing.T) { + for _, a := range sumZZ { + var z Int + z.Set(a.z) + if !isNormalized(&z) { + t.Errorf("%v is not normalized", z) + } + if (&z).Cmp(a.z) != 0 { + t.Errorf("got z = %v; want %v", z, a.z) + } + } +} + +func TestAbsZ(t *testing.T) { + var zero Int + for _, a := range sumZZ { + var z Int + z.Abs(a.z) + var e Int + e.Set(a.z) + if e.Cmp(&zero) < 0 { + e.Sub(&zero, &e) + } + if z.Cmp(&e) != 0 { + t.Errorf("got z = %v; want %v", z, e) + } + } +} + +func testFunZZ(t *testing.T, msg string, f funZZ, a argZZ) { + var z Int + f(&z, a.x, a.y) + if !isNormalized(&z) { + t.Errorf("%s%v is not normalized", z, msg) + } + if (&z).Cmp(a.z) != 0 { + t.Errorf("%s%+v\n\tgot z = %v; want %v", msg, a, &z, a.z) + } +} + +func TestSumZZ(t *testing.T) { + AddZZ := func(z, x, y *Int) *Int { return z.Add(x, y) } + SubZZ := func(z, x, y *Int) *Int { return z.Sub(x, y) } + for _, a := range sumZZ { + arg := a + testFunZZ(t, "AddZZ", AddZZ, arg) + + arg = argZZ{a.z, a.y, a.x} + testFunZZ(t, "AddZZ symmetric", AddZZ, arg) + + arg = argZZ{a.x, a.z, a.y} + testFunZZ(t, "SubZZ", SubZZ, arg) + + arg = argZZ{a.y, a.z, a.x} + testFunZZ(t, "SubZZ symmetric", SubZZ, arg) + } +} + +func TestProdZZ(t *testing.T) { + MulZZ := func(z, x, y *Int) *Int { return z.Mul(x, y) } + for _, a := range prodZZ { + arg := a + testFunZZ(t, "MulZZ", MulZZ, arg) + + arg = argZZ{a.z, a.y, a.x} + testFunZZ(t, "MulZZ symmetric", MulZZ, arg) + } +} + +// mulBytes returns x*y via grade school multiplication. Both inputs +// and the result are assumed to be in big-endian representation (to +// match the semantics of Int.Bytes and Int.SetBytes). +func mulBytes(x, y []byte) []byte { + z := make([]byte, len(x)+len(y)) + + // multiply + k0 := len(z) - 1 + for j := len(y) - 1; j >= 0; j-- { + d := int(y[j]) + if d != 0 { + k := k0 + carry := 0 + for i := len(x) - 1; i >= 0; i-- { + t := int(z[k]) + int(x[i])*d + carry + z[k], carry = byte(t), t>>8 + k-- + } + z[k] = byte(carry) + } + k0-- + } + + // normalize (remove leading 0's) + i := 0 + for i < len(z) && z[i] == 0 { + i++ + } + + return z[i:] +} + +func checkMul(a, b []byte) bool { + var x, y, z1 Int + x.SetBytes(a) + y.SetBytes(b) + z1.Mul(&x, &y) + + var z2 Int + z2.SetBytes(mulBytes(a, b)) + + return z1.Cmp(&z2) == 0 +} + +func TestMul(t *testing.T) { + if err := quick.Check(checkMul, nil); err != nil { + t.Error(err) + } +} + +var mulRangesZ = []struct { + a, b int64 + prod string +}{ + // entirely positive ranges are covered by mulRangesN + {-1, 1, "0"}, + {-2, -1, "2"}, + {-3, -2, "6"}, + {-3, -1, "-6"}, + {1, 3, "6"}, + {-10, -10, "-10"}, + {0, -1, "1"}, // empty range + {-1, -100, "1"}, // empty range + {-1, 1, "0"}, // range includes 0 + {-1e9, 0, "0"}, // range includes 0 + {-1e9, 1e9, "0"}, // range includes 0 + {-10, -1, "3628800"}, // 10! + {-20, -2, "-2432902008176640000"}, // -20! + {-99, -1, + "-933262154439441526816992388562667004907159682643816214685929" + + "638952175999932299156089414639761565182862536979208272237582" + + "511852109168640000000000000000000000", // -99! + }, +} + +func TestMulRangeZ(t *testing.T) { + var tmp Int + // test entirely positive ranges + for i, r := range mulRangesN { + prod := tmp.MulRange(int64(r.a), int64(r.b)).String() + if prod != r.prod { + t.Errorf("#%da: got %s; want %s", i, prod, r.prod) + } + } + // test other ranges + for i, r := range mulRangesZ { + prod := tmp.MulRange(r.a, r.b).String() + if prod != r.prod { + t.Errorf("#%db: got %s; want %s", i, prod, r.prod) + } + } +} + +var stringTests = []struct { + in string + out string + base int + val int64 + ok bool +}{ + {in: "", ok: false}, + {in: "a", ok: false}, + {in: "z", ok: false}, + {in: "+", ok: false}, + {in: "-", ok: false}, + {in: "0b", ok: false}, + {in: "0x", ok: false}, + {in: "2", base: 2, ok: false}, + {in: "0b2", base: 0, ok: false}, + {in: "08", ok: false}, + {in: "8", base: 8, ok: false}, + {in: "0xg", base: 0, ok: false}, + {in: "g", base: 16, ok: false}, + {"0", "0", 0, 0, true}, + {"0", "0", 10, 0, true}, + {"0", "0", 16, 0, true}, + {"+0", "0", 0, 0, true}, + {"-0", "0", 0, 0, true}, + {"10", "10", 0, 10, true}, + {"10", "10", 10, 10, true}, + {"10", "10", 16, 16, true}, + {"-10", "-10", 16, -16, true}, + {"+10", "10", 16, 16, true}, + {"0x10", "16", 0, 16, true}, + {in: "0x10", base: 16, ok: false}, + {"-0x10", "-16", 0, -16, true}, + {"+0x10", "16", 0, 16, true}, + {"00", "0", 0, 0, true}, + {"0", "0", 8, 0, true}, + {"07", "7", 0, 7, true}, + {"7", "7", 8, 7, true}, + {"023", "19", 0, 19, true}, + {"23", "23", 8, 19, true}, + {"cafebabe", "cafebabe", 16, 0xcafebabe, true}, + {"0b0", "0", 0, 0, true}, + {"-111", "-111", 2, -7, true}, + {"-0b111", "-7", 0, -7, true}, + {"0b1001010111", "599", 0, 0x257, true}, + {"1001010111", "1001010111", 2, 0x257, true}, +} + +func format(base int) string { + switch base { + case 2: + return "%b" + case 8: + return "%o" + case 16: + return "%x" + } + return "%d" +} + +func TestGetString(t *testing.T) { + z := new(Int) + for i, test := range stringTests { + if !test.ok { + continue + } + z.SetInt64(test.val) + + if test.base == 10 { + s := z.String() + if s != test.out { + t.Errorf("#%da got %s; want %s", i, s, test.out) + } + } + + s := fmt.Sprintf(format(test.base), z) + if s != test.out { + t.Errorf("#%db got %s; want %s", i, s, test.out) + } + } +} + +func TestSetString(t *testing.T) { + tmp := new(Int) + for i, test := range stringTests { + n1, ok1 := new(Int).SetString(test.in, test.base) + n2, ok2 := tmp.SetString(test.in, test.base) + expected := NewInt(test.val) + if ok1 != test.ok || ok2 != test.ok { + t.Errorf("#%d (input '%s') ok incorrect (should be %t)", i, test.in, test.ok) + continue + } + if !ok1 || !ok2 { + continue + } + + if ok1 && !isNormalized(n1) { + t.Errorf("#%d (input '%s'): %v is not normalized", i, test.in, *n1) + } + if ok2 && !isNormalized(n2) { + t.Errorf("#%d (input '%s'): %v is not normalized", i, test.in, *n2) + } + + if n1.Cmp(expected) != 0 { + t.Errorf("#%d (input '%s') got: %s want: %d", i, test.in, n1, test.val) + } + if n2.Cmp(expected) != 0 { + t.Errorf("#%d (input '%s') got: %s want: %d", i, test.in, n2, test.val) + } + } +} + +var formatTests = []struct { + input string + format string + output string +}{ + {"", "%x", ""}, + {"", "%#x", ""}, + {"", "%#y", "%!y(big.Int=)"}, + + {"10", "%b", "1010"}, + {"10", "%o", "12"}, + {"10", "%d", "10"}, + {"10", "%v", "10"}, + {"10", "%x", "a"}, + {"10", "%X", "A"}, + {"-10", "%X", "-A"}, + {"10", "%y", "%!y(big.Int=10)"}, + {"-10", "%y", "%!y(big.Int=-10)"}, + + {"10", "%#b", "1010"}, + {"10", "%#o", "012"}, + {"10", "%#d", "10"}, + {"10", "%#v", "10"}, + {"10", "%#x", "0xa"}, + {"10", "%#X", "0XA"}, + {"-10", "%#X", "-0XA"}, + {"10", "%#y", "%!y(big.Int=10)"}, + {"-10", "%#y", "%!y(big.Int=-10)"}, + + {"1234", "%d", "1234"}, + {"1234", "%3d", "1234"}, + {"1234", "%4d", "1234"}, + {"-1234", "%d", "-1234"}, + {"1234", "% 5d", " 1234"}, + {"1234", "%+5d", "+1234"}, + {"1234", "%-5d", "1234 "}, + {"1234", "%x", "4d2"}, + {"1234", "%X", "4D2"}, + {"-1234", "%3x", "-4d2"}, + {"-1234", "%4x", "-4d2"}, + {"-1234", "%5x", " -4d2"}, + {"-1234", "%-5x", "-4d2 "}, + {"1234", "%03d", "1234"}, + {"1234", "%04d", "1234"}, + {"1234", "%05d", "01234"}, + {"1234", "%06d", "001234"}, + {"-1234", "%06d", "-01234"}, + {"1234", "%+06d", "+01234"}, + {"1234", "% 06d", " 01234"}, + {"1234", "%-6d", "1234 "}, + {"1234", "%-06d", "1234 "}, + {"-1234", "%-06d", "-1234 "}, + + {"1234", "%.3d", "1234"}, + {"1234", "%.4d", "1234"}, + {"1234", "%.5d", "01234"}, + {"1234", "%.6d", "001234"}, + {"-1234", "%.3d", "-1234"}, + {"-1234", "%.4d", "-1234"}, + {"-1234", "%.5d", "-01234"}, + {"-1234", "%.6d", "-001234"}, + + {"1234", "%8.3d", " 1234"}, + {"1234", "%8.4d", " 1234"}, + {"1234", "%8.5d", " 01234"}, + {"1234", "%8.6d", " 001234"}, + {"-1234", "%8.3d", " -1234"}, + {"-1234", "%8.4d", " -1234"}, + {"-1234", "%8.5d", " -01234"}, + {"-1234", "%8.6d", " -001234"}, + + {"1234", "%+8.3d", " +1234"}, + {"1234", "%+8.4d", " +1234"}, + {"1234", "%+8.5d", " +01234"}, + {"1234", "%+8.6d", " +001234"}, + {"-1234", "%+8.3d", " -1234"}, + {"-1234", "%+8.4d", " -1234"}, + {"-1234", "%+8.5d", " -01234"}, + {"-1234", "%+8.6d", " -001234"}, + + {"1234", "% 8.3d", " 1234"}, + {"1234", "% 8.4d", " 1234"}, + {"1234", "% 8.5d", " 01234"}, + {"1234", "% 8.6d", " 001234"}, + {"-1234", "% 8.3d", " -1234"}, + {"-1234", "% 8.4d", " -1234"}, + {"-1234", "% 8.5d", " -01234"}, + {"-1234", "% 8.6d", " -001234"}, + + {"1234", "%.3x", "4d2"}, + {"1234", "%.4x", "04d2"}, + {"1234", "%.5x", "004d2"}, + {"1234", "%.6x", "0004d2"}, + {"-1234", "%.3x", "-4d2"}, + {"-1234", "%.4x", "-04d2"}, + {"-1234", "%.5x", "-004d2"}, + {"-1234", "%.6x", "-0004d2"}, + + {"1234", "%8.3x", " 4d2"}, + {"1234", "%8.4x", " 04d2"}, + {"1234", "%8.5x", " 004d2"}, + {"1234", "%8.6x", " 0004d2"}, + {"-1234", "%8.3x", " -4d2"}, + {"-1234", "%8.4x", " -04d2"}, + {"-1234", "%8.5x", " -004d2"}, + {"-1234", "%8.6x", " -0004d2"}, + + {"1234", "%+8.3x", " +4d2"}, + {"1234", "%+8.4x", " +04d2"}, + {"1234", "%+8.5x", " +004d2"}, + {"1234", "%+8.6x", " +0004d2"}, + {"-1234", "%+8.3x", " -4d2"}, + {"-1234", "%+8.4x", " -04d2"}, + {"-1234", "%+8.5x", " -004d2"}, + {"-1234", "%+8.6x", " -0004d2"}, + + {"1234", "% 8.3x", " 4d2"}, + {"1234", "% 8.4x", " 04d2"}, + {"1234", "% 8.5x", " 004d2"}, + {"1234", "% 8.6x", " 0004d2"}, + {"1234", "% 8.7x", " 00004d2"}, + {"1234", "% 8.8x", " 000004d2"}, + {"-1234", "% 8.3x", " -4d2"}, + {"-1234", "% 8.4x", " -04d2"}, + {"-1234", "% 8.5x", " -004d2"}, + {"-1234", "% 8.6x", " -0004d2"}, + {"-1234", "% 8.7x", "-00004d2"}, + {"-1234", "% 8.8x", "-000004d2"}, + + {"1234", "%-8.3d", "1234 "}, + {"1234", "%-8.4d", "1234 "}, + {"1234", "%-8.5d", "01234 "}, + {"1234", "%-8.6d", "001234 "}, + {"1234", "%-8.7d", "0001234 "}, + {"1234", "%-8.8d", "00001234"}, + {"-1234", "%-8.3d", "-1234 "}, + {"-1234", "%-8.4d", "-1234 "}, + {"-1234", "%-8.5d", "-01234 "}, + {"-1234", "%-8.6d", "-001234 "}, + {"-1234", "%-8.7d", "-0001234"}, + {"-1234", "%-8.8d", "-00001234"}, + + {"16777215", "%b", "111111111111111111111111"}, // 2**24 - 1 + + {"0", "%.d", ""}, + {"0", "%.0d", ""}, + {"0", "%3.d", ""}, +} + +func TestFormat(t *testing.T) { + for i, test := range formatTests { + var x *Int + if test.input != "" { + var ok bool + x, ok = new(Int).SetString(test.input, 0) + if !ok { + t.Errorf("#%d failed reading input %s", i, test.input) + } + } + output := fmt.Sprintf(test.format, x) + if output != test.output { + t.Errorf("#%d got %q; want %q, {%q, %q, %q}", i, output, test.output, test.input, test.format, test.output) + } + } +} + +var scanTests = []struct { + input string + format string + output string + remaining int +}{ + {"1010", "%b", "10", 0}, + {"0b1010", "%v", "10", 0}, + {"12", "%o", "10", 0}, + {"012", "%v", "10", 0}, + {"10", "%d", "10", 0}, + {"10", "%v", "10", 0}, + {"a", "%x", "10", 0}, + {"0xa", "%v", "10", 0}, + {"A", "%X", "10", 0}, + {"-A", "%X", "-10", 0}, + {"+0b1011001", "%v", "89", 0}, + {"0xA", "%v", "10", 0}, + {"0 ", "%v", "0", 1}, + {"2+3", "%v", "2", 2}, + {"0XABC 12", "%v", "2748", 3}, +} + +func TestScan(t *testing.T) { + var buf bytes.Buffer + for i, test := range scanTests { + x := new(Int) + buf.Reset() + buf.WriteString(test.input) + if _, err := fmt.Fscanf(&buf, test.format, x); err != nil { + t.Errorf("#%d error: %s", i, err.String()) + } + if x.String() != test.output { + t.Errorf("#%d got %s; want %s", i, x.String(), test.output) + } + if buf.Len() != test.remaining { + t.Errorf("#%d got %d bytes remaining; want %d", i, buf.Len(), test.remaining) + } + } +} + +// Examples from the Go Language Spec, section "Arithmetic operators" +var divisionSignsTests = []struct { + x, y int64 + q, r int64 // T-division + d, m int64 // Euclidian division +}{ + {5, 3, 1, 2, 1, 2}, + {-5, 3, -1, -2, -2, 1}, + {5, -3, -1, 2, -1, 2}, + {-5, -3, 1, -2, 2, 1}, + {1, 2, 0, 1, 0, 1}, + {8, 4, 2, 0, 2, 0}, +} + +func TestDivisionSigns(t *testing.T) { + for i, test := range divisionSignsTests { + x := NewInt(test.x) + y := NewInt(test.y) + q := NewInt(test.q) + r := NewInt(test.r) + d := NewInt(test.d) + m := NewInt(test.m) + + q1 := new(Int).Quo(x, y) + r1 := new(Int).Rem(x, y) + if !isNormalized(q1) { + t.Errorf("#%d Quo: %v is not normalized", i, *q1) + } + if !isNormalized(r1) { + t.Errorf("#%d Rem: %v is not normalized", i, *r1) + } + if q1.Cmp(q) != 0 || r1.Cmp(r) != 0 { + t.Errorf("#%d QuoRem: got (%s, %s), want (%s, %s)", i, q1, r1, q, r) + } + + q2, r2 := new(Int).QuoRem(x, y, new(Int)) + if !isNormalized(q2) { + t.Errorf("#%d Quo: %v is not normalized", i, *q2) + } + if !isNormalized(r2) { + t.Errorf("#%d Rem: %v is not normalized", i, *r2) + } + if q2.Cmp(q) != 0 || r2.Cmp(r) != 0 { + t.Errorf("#%d QuoRem: got (%s, %s), want (%s, %s)", i, q2, r2, q, r) + } + + d1 := new(Int).Div(x, y) + m1 := new(Int).Mod(x, y) + if !isNormalized(d1) { + t.Errorf("#%d Div: %v is not normalized", i, *d1) + } + if !isNormalized(m1) { + t.Errorf("#%d Mod: %v is not normalized", i, *m1) + } + if d1.Cmp(d) != 0 || m1.Cmp(m) != 0 { + t.Errorf("#%d DivMod: got (%s, %s), want (%s, %s)", i, d1, m1, d, m) + } + + d2, m2 := new(Int).DivMod(x, y, new(Int)) + if !isNormalized(d2) { + t.Errorf("#%d Div: %v is not normalized", i, *d2) + } + if !isNormalized(m2) { + t.Errorf("#%d Mod: %v is not normalized", i, *m2) + } + if d2.Cmp(d) != 0 || m2.Cmp(m) != 0 { + t.Errorf("#%d DivMod: got (%s, %s), want (%s, %s)", i, d2, m2, d, m) + } + } +} + +func checkSetBytes(b []byte) bool { + hex1 := hex.EncodeToString(new(Int).SetBytes(b).Bytes()) + hex2 := hex.EncodeToString(b) + + for len(hex1) < len(hex2) { + hex1 = "0" + hex1 + } + + for len(hex1) > len(hex2) { + hex2 = "0" + hex2 + } + + return hex1 == hex2 +} + +func TestSetBytes(t *testing.T) { + if err := quick.Check(checkSetBytes, nil); err != nil { + t.Error(err) + } +} + +func checkBytes(b []byte) bool { + b2 := new(Int).SetBytes(b).Bytes() + return bytes.Compare(b, b2) == 0 +} + +func TestBytes(t *testing.T) { + if err := quick.Check(checkSetBytes, nil); err != nil { + t.Error(err) + } +} + +func checkQuo(x, y []byte) bool { + u := new(Int).SetBytes(x) + v := new(Int).SetBytes(y) + + if len(v.abs) == 0 { + return true + } + + r := new(Int) + q, r := new(Int).QuoRem(u, v, r) + + if r.Cmp(v) >= 0 { + return false + } + + uprime := new(Int).Set(q) + uprime.Mul(uprime, v) + uprime.Add(uprime, r) + + return uprime.Cmp(u) == 0 +} + +var quoTests = []struct { + x, y string + q, r string +}{ + { + "476217953993950760840509444250624797097991362735329973741718102894495832294430498335824897858659711275234906400899559094370964723884706254265559534144986498357", + "9353930466774385905609975137998169297361893554149986716853295022578535724979483772383667534691121982974895531435241089241440253066816724367338287092081996", + "50911", + "1", + }, + { + "11510768301994997771168", + "1328165573307167369775", + "8", + "885443715537658812968", + }, +} + +func TestQuo(t *testing.T) { + if err := quick.Check(checkQuo, nil); err != nil { + t.Error(err) + } + + for i, test := range quoTests { + x, _ := new(Int).SetString(test.x, 10) + y, _ := new(Int).SetString(test.y, 10) + expectedQ, _ := new(Int).SetString(test.q, 10) + expectedR, _ := new(Int).SetString(test.r, 10) + + r := new(Int) + q, r := new(Int).QuoRem(x, y, r) + + if q.Cmp(expectedQ) != 0 || r.Cmp(expectedR) != 0 { + t.Errorf("#%d got (%s, %s) want (%s, %s)", i, q, r, expectedQ, expectedR) + } + } +} + +func TestQuoStepD6(t *testing.T) { + // See Knuth, Volume 2, section 4.3.1, exercise 21. This code exercises + // a code path which only triggers 1 in 10^{-19} cases. + + u := &Int{false, nat{0, 0, 1 + 1<<(_W-1), _M ^ (1 << (_W - 1))}} + v := &Int{false, nat{5, 2 + 1<<(_W-1), 1 << (_W - 1)}} + + r := new(Int) + q, r := new(Int).QuoRem(u, v, r) + const expectedQ64 = "18446744073709551613" + const expectedR64 = "3138550867693340382088035895064302439801311770021610913807" + const expectedQ32 = "4294967293" + const expectedR32 = "39614081266355540837921718287" + if q.String() != expectedQ64 && q.String() != expectedQ32 || + r.String() != expectedR64 && r.String() != expectedR32 { + t.Errorf("got (%s, %s) want (%s, %s) or (%s, %s)", q, r, expectedQ64, expectedR64, expectedQ32, expectedR32) + } +} + +var bitLenTests = []struct { + in string + out int +}{ + {"-1", 1}, + {"0", 0}, + {"1", 1}, + {"2", 2}, + {"4", 3}, + {"0xabc", 12}, + {"0x8000", 16}, + {"0x80000000", 32}, + {"0x800000000000", 48}, + {"0x8000000000000000", 64}, + {"0x80000000000000000000", 80}, + {"-0x4000000000000000000000", 87}, +} + +func TestBitLen(t *testing.T) { + for i, test := range bitLenTests { + x, ok := new(Int).SetString(test.in, 0) + if !ok { + t.Errorf("#%d test input invalid: %s", i, test.in) + continue + } + + if n := x.BitLen(); n != test.out { + t.Errorf("#%d got %d want %d", i, n, test.out) + } + } +} + +var expTests = []struct { + x, y, m string + out string +}{ + {"5", "0", "", "1"}, + {"-5", "0", "", "-1"}, + {"5", "1", "", "5"}, + {"-5", "1", "", "-5"}, + {"-2", "3", "2", "0"}, + {"5", "2", "", "25"}, + {"1", "65537", "2", "1"}, + {"0x8000000000000000", "2", "", "0x40000000000000000000000000000000"}, + {"0x8000000000000000", "2", "6719", "4944"}, + {"0x8000000000000000", "3", "6719", "5447"}, + {"0x8000000000000000", "1000", "6719", "1603"}, + {"0x8000000000000000", "1000000", "6719", "3199"}, + { + "2938462938472983472983659726349017249287491026512746239764525612965293865296239471239874193284792387498274256129746192347", + "298472983472983471903246121093472394872319615612417471234712061", + "29834729834729834729347290846729561262544958723956495615629569234729836259263598127342374289365912465901365498236492183464", + "23537740700184054162508175125554701713153216681790245129157191391322321508055833908509185839069455749219131480588829346291", + }, +} + +func TestExp(t *testing.T) { + for i, test := range expTests { + x, ok1 := new(Int).SetString(test.x, 0) + y, ok2 := new(Int).SetString(test.y, 0) + out, ok3 := new(Int).SetString(test.out, 0) + + var ok4 bool + var m *Int + + if len(test.m) == 0 { + m, ok4 = nil, true + } else { + m, ok4 = new(Int).SetString(test.m, 0) + } + + if !ok1 || !ok2 || !ok3 || !ok4 { + t.Errorf("#%d: error in input", i) + continue + } + + z := y.Exp(x, y, m) + if !isNormalized(z) { + t.Errorf("#%d: %v is not normalized", i, *z) + } + if z.Cmp(out) != 0 { + t.Errorf("#%d: got %s want %s", i, z, out) + } + } +} + +func checkGcd(aBytes, bBytes []byte) bool { + a := new(Int).SetBytes(aBytes) + b := new(Int).SetBytes(bBytes) + + x := new(Int) + y := new(Int) + d := new(Int) + + GcdInt(d, x, y, a, b) + x.Mul(x, a) + y.Mul(y, b) + x.Add(x, y) + + return x.Cmp(d) == 0 +} + +var gcdTests = []struct { + a, b int64 + d, x, y int64 +}{ + {120, 23, 1, -9, 47}, +} + +func TestGcd(t *testing.T) { + for i, test := range gcdTests { + a := NewInt(test.a) + b := NewInt(test.b) + + x := new(Int) + y := new(Int) + d := new(Int) + + expectedX := NewInt(test.x) + expectedY := NewInt(test.y) + expectedD := NewInt(test.d) + + GcdInt(d, x, y, a, b) + + if expectedX.Cmp(x) != 0 || + expectedY.Cmp(y) != 0 || + expectedD.Cmp(d) != 0 { + t.Errorf("#%d got (%s %s %s) want (%s %s %s)", i, x, y, d, expectedX, expectedY, expectedD) + } + } + + quick.Check(checkGcd, nil) +} + +var primes = []string{ + "2", + "3", + "5", + "7", + "11", + + "13756265695458089029", + "13496181268022124907", + "10953742525620032441", + "17908251027575790097", + + // http://code.google.com/p/go/issues/detail?id=638 + "18699199384836356663", + + "98920366548084643601728869055592650835572950932266967461790948584315647051443", + "94560208308847015747498523884063394671606671904944666360068158221458669711639", + + // http://primes.utm.edu/lists/small/small3.html + "449417999055441493994709297093108513015373787049558499205492347871729927573118262811508386655998299074566974373711472560655026288668094291699357843464363003144674940345912431129144354948751003607115263071543163", + "230975859993204150666423538988557839555560243929065415434980904258310530753006723857139742334640122533598517597674807096648905501653461687601339782814316124971547968912893214002992086353183070342498989426570593", + "5521712099665906221540423207019333379125265462121169655563495403888449493493629943498064604536961775110765377745550377067893607246020694972959780839151452457728855382113555867743022746090187341871655890805971735385789993", + "203956878356401977405765866929034577280193993314348263094772646453283062722701277632936616063144088173312372882677123879538709400158306567338328279154499698366071906766440037074217117805690872792848149112022286332144876183376326512083574821647933992961249917319836219304274280243803104015000563790123", +} + +var composites = []string{ + "21284175091214687912771199898307297748211672914763848041968395774954376176754", + "6084766654921918907427900243509372380954290099172559290432744450051395395951", + "84594350493221918389213352992032324280367711247940675652888030554255915464401", + "82793403787388584738507275144194252681", +} + +func TestProbablyPrime(t *testing.T) { + nreps := 20 + if testing.Short() { + nreps = 1 + } + for i, s := range primes { + p, _ := new(Int).SetString(s, 10) + if !ProbablyPrime(p, nreps) { + t.Errorf("#%d prime found to be non-prime (%s)", i, s) + } + } + + for i, s := range composites { + c, _ := new(Int).SetString(s, 10) + if ProbablyPrime(c, nreps) { + t.Errorf("#%d composite found to be prime (%s)", i, s) + } + if testing.Short() { + break + } + } +} + +type intShiftTest struct { + in string + shift uint + out string +} + +var rshTests = []intShiftTest{ + {"0", 0, "0"}, + {"-0", 0, "0"}, + {"0", 1, "0"}, + {"0", 2, "0"}, + {"1", 0, "1"}, + {"1", 1, "0"}, + {"1", 2, "0"}, + {"2", 0, "2"}, + {"2", 1, "1"}, + {"-1", 0, "-1"}, + {"-1", 1, "-1"}, + {"-1", 10, "-1"}, + {"-100", 2, "-25"}, + {"-100", 3, "-13"}, + {"-100", 100, "-1"}, + {"4294967296", 0, "4294967296"}, + {"4294967296", 1, "2147483648"}, + {"4294967296", 2, "1073741824"}, + {"18446744073709551616", 0, "18446744073709551616"}, + {"18446744073709551616", 1, "9223372036854775808"}, + {"18446744073709551616", 2, "4611686018427387904"}, + {"18446744073709551616", 64, "1"}, + {"340282366920938463463374607431768211456", 64, "18446744073709551616"}, + {"340282366920938463463374607431768211456", 128, "1"}, +} + +func TestRsh(t *testing.T) { + for i, test := range rshTests { + in, _ := new(Int).SetString(test.in, 10) + expected, _ := new(Int).SetString(test.out, 10) + out := new(Int).Rsh(in, test.shift) + + if !isNormalized(out) { + t.Errorf("#%d: %v is not normalized", i, *out) + } + if out.Cmp(expected) != 0 { + t.Errorf("#%d: got %s want %s", i, out, expected) + } + } +} + +func TestRshSelf(t *testing.T) { + for i, test := range rshTests { + z, _ := new(Int).SetString(test.in, 10) + expected, _ := new(Int).SetString(test.out, 10) + z.Rsh(z, test.shift) + + if !isNormalized(z) { + t.Errorf("#%d: %v is not normalized", i, *z) + } + if z.Cmp(expected) != 0 { + t.Errorf("#%d: got %s want %s", i, z, expected) + } + } +} + +var lshTests = []intShiftTest{ + {"0", 0, "0"}, + {"0", 1, "0"}, + {"0", 2, "0"}, + {"1", 0, "1"}, + {"1", 1, "2"}, + {"1", 2, "4"}, + {"2", 0, "2"}, + {"2", 1, "4"}, + {"2", 2, "8"}, + {"-87", 1, "-174"}, + {"4294967296", 0, "4294967296"}, + {"4294967296", 1, "8589934592"}, + {"4294967296", 2, "17179869184"}, + {"18446744073709551616", 0, "18446744073709551616"}, + {"9223372036854775808", 1, "18446744073709551616"}, + {"4611686018427387904", 2, "18446744073709551616"}, + {"1", 64, "18446744073709551616"}, + {"18446744073709551616", 64, "340282366920938463463374607431768211456"}, + {"1", 128, "340282366920938463463374607431768211456"}, +} + +func TestLsh(t *testing.T) { + for i, test := range lshTests { + in, _ := new(Int).SetString(test.in, 10) + expected, _ := new(Int).SetString(test.out, 10) + out := new(Int).Lsh(in, test.shift) + + if !isNormalized(out) { + t.Errorf("#%d: %v is not normalized", i, *out) + } + if out.Cmp(expected) != 0 { + t.Errorf("#%d: got %s want %s", i, out, expected) + } + } +} + +func TestLshSelf(t *testing.T) { + for i, test := range lshTests { + z, _ := new(Int).SetString(test.in, 10) + expected, _ := new(Int).SetString(test.out, 10) + z.Lsh(z, test.shift) + + if !isNormalized(z) { + t.Errorf("#%d: %v is not normalized", i, *z) + } + if z.Cmp(expected) != 0 { + t.Errorf("#%d: got %s want %s", i, z, expected) + } + } +} + +func TestLshRsh(t *testing.T) { + for i, test := range rshTests { + in, _ := new(Int).SetString(test.in, 10) + out := new(Int).Lsh(in, test.shift) + out = out.Rsh(out, test.shift) + + if !isNormalized(out) { + t.Errorf("#%d: %v is not normalized", i, *out) + } + if in.Cmp(out) != 0 { + t.Errorf("#%d: got %s want %s", i, out, in) + } + } + for i, test := range lshTests { + in, _ := new(Int).SetString(test.in, 10) + out := new(Int).Lsh(in, test.shift) + out.Rsh(out, test.shift) + + if !isNormalized(out) { + t.Errorf("#%d: %v is not normalized", i, *out) + } + if in.Cmp(out) != 0 { + t.Errorf("#%d: got %s want %s", i, out, in) + } + } +} + +var int64Tests = []int64{ + 0, + 1, + -1, + 4294967295, + -4294967295, + 4294967296, + -4294967296, + 9223372036854775807, + -9223372036854775807, + -9223372036854775808, +} + +func TestInt64(t *testing.T) { + for i, testVal := range int64Tests { + in := NewInt(testVal) + out := in.Int64() + + if out != testVal { + t.Errorf("#%d got %d want %d", i, out, testVal) + } + } +} + +var bitwiseTests = []struct { + x, y string + and, or, xor, andNot string +}{ + {"0x00", "0x00", "0x00", "0x00", "0x00", "0x00"}, + {"0x00", "0x01", "0x00", "0x01", "0x01", "0x00"}, + {"0x01", "0x00", "0x00", "0x01", "0x01", "0x01"}, + {"-0x01", "0x00", "0x00", "-0x01", "-0x01", "-0x01"}, + {"-0xaf", "-0x50", "-0xf0", "-0x0f", "0xe1", "0x41"}, + {"0x00", "-0x01", "0x00", "-0x01", "-0x01", "0x00"}, + {"0x01", "0x01", "0x01", "0x01", "0x00", "0x00"}, + {"-0x01", "-0x01", "-0x01", "-0x01", "0x00", "0x00"}, + {"0x07", "0x08", "0x00", "0x0f", "0x0f", "0x07"}, + {"0x05", "0x0f", "0x05", "0x0f", "0x0a", "0x00"}, + {"0x013ff6", "0x9a4e", "0x1a46", "0x01bffe", "0x01a5b8", "0x0125b0"}, + {"-0x013ff6", "0x9a4e", "0x800a", "-0x0125b2", "-0x01a5bc", "-0x01c000"}, + {"-0x013ff6", "-0x9a4e", "-0x01bffe", "-0x1a46", "0x01a5b8", "0x8008"}, + { + "0x1000009dc6e3d9822cba04129bcbe3401", + "0xb9bd7d543685789d57cb918e833af352559021483cdb05cc21fd", + "0x1000001186210100001000009048c2001", + "0xb9bd7d543685789d57cb918e8bfeff7fddb2ebe87dfbbdfe35fd", + "0xb9bd7d543685789d57ca918e8ae69d6fcdb2eae87df2b97215fc", + "0x8c40c2d8822caa04120b8321400", + }, + { + "0x1000009dc6e3d9822cba04129bcbe3401", + "-0xb9bd7d543685789d57cb918e833af352559021483cdb05cc21fd", + "0x8c40c2d8822caa04120b8321401", + "-0xb9bd7d543685789d57ca918e82229142459020483cd2014001fd", + "-0xb9bd7d543685789d57ca918e8ae69d6fcdb2eae87df2b97215fe", + "0x1000001186210100001000009048c2000", + }, + { + "-0x1000009dc6e3d9822cba04129bcbe3401", + "-0xb9bd7d543685789d57cb918e833af352559021483cdb05cc21fd", + "-0xb9bd7d543685789d57cb918e8bfeff7fddb2ebe87dfbbdfe35fd", + "-0x1000001186210100001000009048c2001", + "0xb9bd7d543685789d57ca918e8ae69d6fcdb2eae87df2b97215fc", + "0xb9bd7d543685789d57ca918e82229142459020483cd2014001fc", + }, +} + +type bitFun func(z, x, y *Int) *Int + +func testBitFun(t *testing.T, msg string, f bitFun, x, y *Int, exp string) { + expected := new(Int) + expected.SetString(exp, 0) + + out := f(new(Int), x, y) + if out.Cmp(expected) != 0 { + t.Errorf("%s: got %s want %s", msg, out, expected) + } +} + +func testBitFunSelf(t *testing.T, msg string, f bitFun, x, y *Int, exp string) { + self := new(Int) + self.Set(x) + expected := new(Int) + expected.SetString(exp, 0) + + self = f(self, self, y) + if self.Cmp(expected) != 0 { + t.Errorf("%s: got %s want %s", msg, self, expected) + } +} + +func altBit(x *Int, i int) uint { + z := new(Int).Rsh(x, uint(i)) + z = z.And(z, NewInt(1)) + if z.Cmp(new(Int)) != 0 { + return 1 + } + return 0 +} + +func altSetBit(z *Int, x *Int, i int, b uint) *Int { + one := NewInt(1) + m := one.Lsh(one, uint(i)) + switch b { + case 1: + return z.Or(x, m) + case 0: + return z.AndNot(x, m) + } + panic("set bit is not 0 or 1") +} + +func testBitset(t *testing.T, x *Int) { + n := x.BitLen() + z := new(Int).Set(x) + z1 := new(Int).Set(x) + for i := 0; i < n+10; i++ { + old := z.Bit(i) + old1 := altBit(z1, i) + if old != old1 { + t.Errorf("bitset: inconsistent value for Bit(%s, %d), got %v want %v", z1, i, old, old1) + } + z := new(Int).SetBit(z, i, 1) + z1 := altSetBit(new(Int), z1, i, 1) + if z.Bit(i) == 0 { + t.Errorf("bitset: bit %d of %s got 0 want 1", i, x) + } + if z.Cmp(z1) != 0 { + t.Errorf("bitset: inconsistent value after SetBit 1, got %s want %s", z, z1) + } + z.SetBit(z, i, 0) + altSetBit(z1, z1, i, 0) + if z.Bit(i) != 0 { + t.Errorf("bitset: bit %d of %s got 1 want 0", i, x) + } + if z.Cmp(z1) != 0 { + t.Errorf("bitset: inconsistent value after SetBit 0, got %s want %s", z, z1) + } + altSetBit(z1, z1, i, old) + z.SetBit(z, i, old) + if z.Cmp(z1) != 0 { + t.Errorf("bitset: inconsistent value after SetBit old, got %s want %s", z, z1) + } + } + if z.Cmp(x) != 0 { + t.Errorf("bitset: got %s want %s", z, x) + } +} + +var bitsetTests = []struct { + x string + i int + b uint +}{ + {"0", 0, 0}, + {"0", 200, 0}, + {"1", 0, 1}, + {"1", 1, 0}, + {"-1", 0, 1}, + {"-1", 200, 1}, + {"0x2000000000000000000000000000", 108, 0}, + {"0x2000000000000000000000000000", 109, 1}, + {"0x2000000000000000000000000000", 110, 0}, + {"-0x2000000000000000000000000001", 108, 1}, + {"-0x2000000000000000000000000001", 109, 0}, + {"-0x2000000000000000000000000001", 110, 1}, +} + +func TestBitSet(t *testing.T) { + for _, test := range bitwiseTests { + x := new(Int) + x.SetString(test.x, 0) + testBitset(t, x) + x = new(Int) + x.SetString(test.y, 0) + testBitset(t, x) + } + for i, test := range bitsetTests { + x := new(Int) + x.SetString(test.x, 0) + b := x.Bit(test.i) + if b != test.b { + + t.Errorf("#%d want %v got %v", i, test.b, b) + } + } +} + +func BenchmarkBitset(b *testing.B) { + z := new(Int) + z.SetBit(z, 512, 1) + b.ResetTimer() + b.StartTimer() + for i := b.N - 1; i >= 0; i-- { + z.SetBit(z, i&512, 1) + } +} + +func BenchmarkBitsetNeg(b *testing.B) { + z := NewInt(-1) + z.SetBit(z, 512, 0) + b.ResetTimer() + b.StartTimer() + for i := b.N - 1; i >= 0; i-- { + z.SetBit(z, i&512, 0) + } +} + +func BenchmarkBitsetOrig(b *testing.B) { + z := new(Int) + altSetBit(z, z, 512, 1) + b.ResetTimer() + b.StartTimer() + for i := b.N - 1; i >= 0; i-- { + altSetBit(z, z, i&512, 1) + } +} + +func BenchmarkBitsetNegOrig(b *testing.B) { + z := NewInt(-1) + altSetBit(z, z, 512, 0) + b.ResetTimer() + b.StartTimer() + for i := b.N - 1; i >= 0; i-- { + altSetBit(z, z, i&512, 0) + } +} + +func TestBitwise(t *testing.T) { + x := new(Int) + y := new(Int) + for _, test := range bitwiseTests { + x.SetString(test.x, 0) + y.SetString(test.y, 0) + + testBitFun(t, "and", (*Int).And, x, y, test.and) + testBitFunSelf(t, "and", (*Int).And, x, y, test.and) + testBitFun(t, "andNot", (*Int).AndNot, x, y, test.andNot) + testBitFunSelf(t, "andNot", (*Int).AndNot, x, y, test.andNot) + testBitFun(t, "or", (*Int).Or, x, y, test.or) + testBitFunSelf(t, "or", (*Int).Or, x, y, test.or) + testBitFun(t, "xor", (*Int).Xor, x, y, test.xor) + testBitFunSelf(t, "xor", (*Int).Xor, x, y, test.xor) + } +} + +var notTests = []struct { + in string + out string +}{ + {"0", "-1"}, + {"1", "-2"}, + {"7", "-8"}, + {"0", "-1"}, + {"-81910", "81909"}, + { + "298472983472983471903246121093472394872319615612417471234712061", + "-298472983472983471903246121093472394872319615612417471234712062", + }, +} + +func TestNot(t *testing.T) { + in := new(Int) + out := new(Int) + expected := new(Int) + for i, test := range notTests { + in.SetString(test.in, 10) + expected.SetString(test.out, 10) + out = out.Not(in) + if out.Cmp(expected) != 0 { + t.Errorf("#%d: got %s want %s", i, out, expected) + } + out = out.Not(out) + if out.Cmp(in) != 0 { + t.Errorf("#%d: got %s want %s", i, out, in) + } + } +} + +var modInverseTests = []struct { + element string + prime string +}{ + {"1", "7"}, + {"1", "13"}, + {"239487239847", "2410312426921032588552076022197566074856950548502459942654116941958108831682612228890093858261341614673227141477904012196503648957050582631942730706805009223062734745341073406696246014589361659774041027169249453200378729434170325843778659198143763193776859869524088940195577346119843545301547043747207749969763750084308926339295559968882457872412993810129130294592999947926365264059284647209730384947211681434464714438488520940127459844288859336526896320919633919"}, +} + +func TestModInverse(t *testing.T) { + var element, prime Int + one := NewInt(1) + for i, test := range modInverseTests { + (&element).SetString(test.element, 10) + (&prime).SetString(test.prime, 10) + inverse := new(Int).ModInverse(&element, &prime) + inverse.Mul(inverse, &element) + inverse.Mod(inverse, &prime) + if inverse.Cmp(one) != 0 { + t.Errorf("#%d: failed (e·e^(-1)=%s)", i, inverse) + } + } +} + +// used by TestIntGobEncoding and TestRatGobEncoding +var gobEncodingTests = []string{ + "0", + "1", + "2", + "10", + "42", + "1234567890", + "298472983472983471903246121093472394872319615612417471234712061", +} + +func TestIntGobEncoding(t *testing.T) { + var medium bytes.Buffer + enc := gob.NewEncoder(&medium) + dec := gob.NewDecoder(&medium) + for i, test := range gobEncodingTests { + for j := 0; j < 2; j++ { + medium.Reset() // empty buffer for each test case (in case of failures) + stest := test + if j != 0 { + // negative numbers + stest = "-" + test + } + var tx Int + tx.SetString(stest, 10) + if err := enc.Encode(&tx); err != nil { + t.Errorf("#%d%c: encoding failed: %s", i, 'a'+j, err) + } + var rx Int + if err := dec.Decode(&rx); err != nil { + t.Errorf("#%d%c: decoding failed: %s", i, 'a'+j, err) + } + if rx.Cmp(&tx) != 0 { + t.Errorf("#%d%c: transmission failed: got %s want %s", i, 'a'+j, &rx, &tx) + } + } + } +} diff --git a/src/pkg/big/nat.go b/src/pkg/big/nat.go new file mode 100755 index 000000000..be3aff29d --- /dev/null +++ b/src/pkg/big/nat.go @@ -0,0 +1,1272 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Package big implements multi-precision arithmetic (big numbers). +// The following numeric types are supported: +// +// - Int signed integers +// - Rat rational numbers +// +// All methods on Int take the result as the receiver; if it is one +// of the operands it may be overwritten (and its memory reused). +// To enable chaining of operations, the result is also returned. +// +package big + +// This file contains operations on unsigned multi-precision integers. +// These are the building blocks for the operations on signed integers +// and rationals. + +import ( + "io" + "os" + "rand" +) + +// An unsigned integer x of the form +// +// x = x[n-1]*_B^(n-1) + x[n-2]*_B^(n-2) + ... + x[1]*_B + x[0] +// +// with 0 <= x[i] < _B and 0 <= i < n is stored in a slice of length n, +// with the digits x[i] as the slice elements. +// +// A number is normalized if the slice contains no leading 0 digits. +// During arithmetic operations, denormalized values may occur but are +// always normalized before returning the final result. The normalized +// representation of 0 is the empty or nil slice (length = 0). + +type nat []Word + +var ( + natOne = nat{1} + natTwo = nat{2} + natTen = nat{10} +) + +func (z nat) clear() { + for i := range z { + z[i] = 0 + } +} + +func (z nat) norm() nat { + i := len(z) + for i > 0 && z[i-1] == 0 { + i-- + } + return z[0:i] +} + +func (z nat) make(n int) nat { + if n <= cap(z) { + return z[0:n] // reuse z + } + // Choosing a good value for e has significant performance impact + // because it increases the chance that a value can be reused. + const e = 4 // extra capacity + return make(nat, n, n+e) +} + +func (z nat) setWord(x Word) nat { + if x == 0 { + return z.make(0) + } + z = z.make(1) + z[0] = x + return z +} + +func (z nat) setUint64(x uint64) nat { + // single-digit values + if w := Word(x); uint64(w) == x { + return z.setWord(w) + } + + // compute number of words n required to represent x + n := 0 + for t := x; t > 0; t >>= _W { + n++ + } + + // split x into n words + z = z.make(n) + for i := range z { + z[i] = Word(x & _M) + x >>= _W + } + + return z +} + +func (z nat) set(x nat) nat { + z = z.make(len(x)) + copy(z, x) + return z +} + +func (z nat) add(x, y nat) nat { + m := len(x) + n := len(y) + + switch { + case m < n: + return z.add(y, x) + case m == 0: + // n == 0 because m >= n; result is 0 + return z.make(0) + case n == 0: + // result is x + return z.set(x) + } + // m > 0 + + z = z.make(m + 1) + c := addVV(z[0:n], x, y) + if m > n { + c = addVW(z[n:m], x[n:], c) + } + z[m] = c + + return z.norm() +} + +func (z nat) sub(x, y nat) nat { + m := len(x) + n := len(y) + + switch { + case m < n: + panic("underflow") + case m == 0: + // n == 0 because m >= n; result is 0 + return z.make(0) + case n == 0: + // result is x + return z.set(x) + } + // m > 0 + + z = z.make(m) + c := subVV(z[0:n], x, y) + if m > n { + c = subVW(z[n:], x[n:], c) + } + if c != 0 { + panic("underflow") + } + + return z.norm() +} + +func (x nat) cmp(y nat) (r int) { + m := len(x) + n := len(y) + if m != n || m == 0 { + switch { + case m < n: + r = -1 + case m > n: + r = 1 + } + return + } + + i := m - 1 + for i > 0 && x[i] == y[i] { + i-- + } + + switch { + case x[i] < y[i]: + r = -1 + case x[i] > y[i]: + r = 1 + } + return +} + +func (z nat) mulAddWW(x nat, y, r Word) nat { + m := len(x) + if m == 0 || y == 0 { + return z.setWord(r) // result is r + } + // m > 0 + + z = z.make(m + 1) + z[m] = mulAddVWW(z[0:m], x, y, r) + + return z.norm() +} + +// basicMul multiplies x and y and leaves the result in z. +// The (non-normalized) result is placed in z[0 : len(x) + len(y)]. +func basicMul(z, x, y nat) { + z[0 : len(x)+len(y)].clear() // initialize z + for i, d := range y { + if d != 0 { + z[len(x)+i] = addMulVVW(z[i:i+len(x)], x, d) + } + } +} + +// Fast version of z[0:n+n>>1].add(z[0:n+n>>1], x[0:n]) w/o bounds checks. +// Factored out for readability - do not use outside karatsuba. +func karatsubaAdd(z, x nat, n int) { + if c := addVV(z[0:n], z, x); c != 0 { + addVW(z[n:n+n>>1], z[n:], c) + } +} + +// Like karatsubaAdd, but does subtract. +func karatsubaSub(z, x nat, n int) { + if c := subVV(z[0:n], z, x); c != 0 { + subVW(z[n:n+n>>1], z[n:], c) + } +} + +// Operands that are shorter than karatsubaThreshold are multiplied using +// "grade school" multiplication; for longer operands the Karatsuba algorithm +// is used. +var karatsubaThreshold int = 32 // computed by calibrate.go + +// karatsuba multiplies x and y and leaves the result in z. +// Both x and y must have the same length n and n must be a +// power of 2. The result vector z must have len(z) >= 6*n. +// The (non-normalized) result is placed in z[0 : 2*n]. +func karatsuba(z, x, y nat) { + n := len(y) + + // Switch to basic multiplication if numbers are odd or small. + // (n is always even if karatsubaThreshold is even, but be + // conservative) + if n&1 != 0 || n < karatsubaThreshold || n < 2 { + basicMul(z, x, y) + return + } + // n&1 == 0 && n >= karatsubaThreshold && n >= 2 + + // Karatsuba multiplication is based on the observation that + // for two numbers x and y with: + // + // x = x1*b + x0 + // y = y1*b + y0 + // + // the product x*y can be obtained with 3 products z2, z1, z0 + // instead of 4: + // + // x*y = x1*y1*b*b + (x1*y0 + x0*y1)*b + x0*y0 + // = z2*b*b + z1*b + z0 + // + // with: + // + // xd = x1 - x0 + // yd = y0 - y1 + // + // z1 = xd*yd + z1 + z0 + // = (x1-x0)*(y0 - y1) + z1 + z0 + // = x1*y0 - x1*y1 - x0*y0 + x0*y1 + z1 + z0 + // = x1*y0 - z1 - z0 + x0*y1 + z1 + z0 + // = x1*y0 + x0*y1 + + // split x, y into "digits" + n2 := n >> 1 // n2 >= 1 + x1, x0 := x[n2:], x[0:n2] // x = x1*b + y0 + y1, y0 := y[n2:], y[0:n2] // y = y1*b + y0 + + // z is used for the result and temporary storage: + // + // 6*n 5*n 4*n 3*n 2*n 1*n 0*n + // z = [z2 copy|z0 copy| xd*yd | yd:xd | x1*y1 | x0*y0 ] + // + // For each recursive call of karatsuba, an unused slice of + // z is passed in that has (at least) half the length of the + // caller's z. + + // compute z0 and z2 with the result "in place" in z + karatsuba(z, x0, y0) // z0 = x0*y0 + karatsuba(z[n:], x1, y1) // z2 = x1*y1 + + // compute xd (or the negative value if underflow occurs) + s := 1 // sign of product xd*yd + xd := z[2*n : 2*n+n2] + if subVV(xd, x1, x0) != 0 { // x1-x0 + s = -s + subVV(xd, x0, x1) // x0-x1 + } + + // compute yd (or the negative value if underflow occurs) + yd := z[2*n+n2 : 3*n] + if subVV(yd, y0, y1) != 0 { // y0-y1 + s = -s + subVV(yd, y1, y0) // y1-y0 + } + + // p = (x1-x0)*(y0-y1) == x1*y0 - x1*y1 - x0*y0 + x0*y1 for s > 0 + // p = (x0-x1)*(y0-y1) == x0*y0 - x0*y1 - x1*y0 + x1*y1 for s < 0 + p := z[n*3:] + karatsuba(p, xd, yd) + + // save original z2:z0 + // (ok to use upper half of z since we're done recursing) + r := z[n*4:] + copy(r, z) + + // add up all partial products + // + // 2*n n 0 + // z = [ z2 | z0 ] + // + [ z0 ] + // + [ z2 ] + // + [ p ] + // + karatsubaAdd(z[n2:], r, n) + karatsubaAdd(z[n2:], r[n:], n) + if s > 0 { + karatsubaAdd(z[n2:], p, n) + } else { + karatsubaSub(z[n2:], p, n) + } +} + +// alias returns true if x and y share the same base array. +func alias(x, y nat) bool { + return cap(x) > 0 && cap(y) > 0 && &x[0:cap(x)][cap(x)-1] == &y[0:cap(y)][cap(y)-1] +} + +// addAt implements z += x*(1<<(_W*i)); z must be long enough. +// (we don't use nat.add because we need z to stay the same +// slice, and we don't need to normalize z after each addition) +func addAt(z, x nat, i int) { + if n := len(x); n > 0 { + if c := addVV(z[i:i+n], z[i:], x); c != 0 { + j := i + n + if j < len(z) { + addVW(z[j:], z[j:], c) + } + } + } +} + +func max(x, y int) int { + if x > y { + return x + } + return y +} + +// karatsubaLen computes an approximation to the maximum k <= n such that +// k = p<= 0. Thus, the +// result is the largest number that can be divided repeatedly by 2 before +// becoming about the value of karatsubaThreshold. +func karatsubaLen(n int) int { + i := uint(0) + for n > karatsubaThreshold { + n >>= 1 + i++ + } + return n << i +} + +func (z nat) mul(x, y nat) nat { + m := len(x) + n := len(y) + + switch { + case m < n: + return z.mul(y, x) + case m == 0 || n == 0: + return z.make(0) + case n == 1: + return z.mulAddWW(x, y[0], 0) + } + // m >= n > 1 + + // determine if z can be reused + if alias(z, x) || alias(z, y) { + z = nil // z is an alias for x or y - cannot reuse + } + + // use basic multiplication if the numbers are small + if n < karatsubaThreshold || n < 2 { + z = z.make(m + n) + basicMul(z, x, y) + return z.norm() + } + // m >= n && n >= karatsubaThreshold && n >= 2 + + // determine Karatsuba length k such that + // + // x = x1*b + x0 + // y = y1*b + y0 (and k <= len(y), which implies k <= len(x)) + // b = 1<<(_W*k) ("base" of digits xi, yi) + // + k := karatsubaLen(n) + // k <= n + + // multiply x0 and y0 via Karatsuba + x0 := x[0:k] // x0 is not normalized + y0 := y[0:k] // y0 is not normalized + z = z.make(max(6*k, m+n)) // enough space for karatsuba of x0*y0 and full result of x*y + karatsuba(z, x0, y0) + z = z[0 : m+n] // z has final length but may be incomplete, upper portion is garbage + + // If x1 and/or y1 are not 0, add missing terms to z explicitly: + // + // m+n 2*k 0 + // z = [ ... | x0*y0 ] + // + [ x1*y1 ] + // + [ x1*y0 ] + // + [ x0*y1 ] + // + if k < n || m != n { + x1 := x[k:] // x1 is normalized because x is + y1 := y[k:] // y1 is normalized because y is + var t nat + t = t.mul(x1, y1) + copy(z[2*k:], t) + z[2*k+len(t):].clear() // upper portion of z is garbage + t = t.mul(x1, y0.norm()) + addAt(z, t, k) + t = t.mul(x0.norm(), y1) + addAt(z, t, k) + } + + return z.norm() +} + +// mulRange computes the product of all the unsigned integers in the +// range [a, b] inclusively. If a > b (empty range), the result is 1. +func (z nat) mulRange(a, b uint64) nat { + switch { + case a == 0: + // cut long ranges short (optimization) + return z.setUint64(0) + case a > b: + return z.setUint64(1) + case a == b: + return z.setUint64(a) + case a+1 == b: + return z.mul(nat(nil).setUint64(a), nat(nil).setUint64(b)) + } + m := (a + b) / 2 + return z.mul(nat(nil).mulRange(a, m), nat(nil).mulRange(m+1, b)) +} + +// q = (x-r)/y, with 0 <= r < y +func (z nat) divW(x nat, y Word) (q nat, r Word) { + m := len(x) + switch { + case y == 0: + panic("division by zero") + case y == 1: + q = z.set(x) // result is x + return + case m == 0: + q = z.make(0) // result is 0 + return + } + // m > 0 + z = z.make(m) + r = divWVW(z, 0, x, y) + q = z.norm() + return +} + +func (z nat) div(z2, u, v nat) (q, r nat) { + if len(v) == 0 { + panic("division by zero") + } + + if u.cmp(v) < 0 { + q = z.make(0) + r = z2.set(u) + return + } + + if len(v) == 1 { + var rprime Word + q, rprime = z.divW(u, v[0]) + if rprime > 0 { + r = z2.make(1) + r[0] = rprime + } else { + r = z2.make(0) + } + return + } + + q, r = z.divLarge(z2, u, v) + return +} + +// q = (uIn-r)/v, with 0 <= r < y +// Uses z as storage for q, and u as storage for r if possible. +// See Knuth, Volume 2, section 4.3.1, Algorithm D. +// Preconditions: +// len(v) >= 2 +// len(uIn) >= len(v) +func (z nat) divLarge(u, uIn, v nat) (q, r nat) { + n := len(v) + m := len(uIn) - n + + // determine if z can be reused + // TODO(gri) should find a better solution - this if statement + // is very costly (see e.g. time pidigits -s -n 10000) + if alias(z, uIn) || alias(z, v) { + z = nil // z is an alias for uIn or v - cannot reuse + } + q = z.make(m + 1) + + qhatv := make(nat, n+1) + if alias(u, uIn) || alias(u, v) { + u = nil // u is an alias for uIn or v - cannot reuse + } + u = u.make(len(uIn) + 1) + u.clear() + + // D1. + shift := leadingZeros(v[n-1]) + if shift > 0 { + // do not modify v, it may be used by another goroutine simultaneously + v1 := make(nat, n) + shlVU(v1, v, shift) + v = v1 + } + u[len(uIn)] = shlVU(u[0:len(uIn)], uIn, shift) + + // D2. + for j := m; j >= 0; j-- { + // D3. + qhat := Word(_M) + if u[j+n] != v[n-1] { + var rhat Word + qhat, rhat = divWW(u[j+n], u[j+n-1], v[n-1]) + + // x1 | x2 = q̂v_{n-2} + x1, x2 := mulWW(qhat, v[n-2]) + // test if q̂v_{n-2} > br̂ + u_{j+n-2} + for greaterThan(x1, x2, rhat, u[j+n-2]) { + qhat-- + prevRhat := rhat + rhat += v[n-1] + // v[n-1] >= 0, so this tests for overflow. + if rhat < prevRhat { + break + } + x1, x2 = mulWW(qhat, v[n-2]) + } + } + + // D4. + qhatv[n] = mulAddVWW(qhatv[0:n], v, qhat, 0) + + c := subVV(u[j:j+len(qhatv)], u[j:], qhatv) + if c != 0 { + c := addVV(u[j:j+n], u[j:], v) + u[j+n] += c + qhat-- + } + + q[j] = qhat + } + + q = q.norm() + shrVU(u, u, shift) + r = u.norm() + + return q, r +} + +// Length of x in bits. x must be normalized. +func (x nat) bitLen() int { + if i := len(x) - 1; i >= 0 { + return i*_W + bitLen(x[i]) + } + return 0 +} + +// MaxBase is the largest number base accepted for string conversions. +const MaxBase = 'z' - 'a' + 10 + 1 // = hexValue('z') + 1 + + +func hexValue(ch int) Word { + d := MaxBase + 1 // illegal base + switch { + case '0' <= ch && ch <= '9': + d = ch - '0' + case 'a' <= ch && ch <= 'z': + d = ch - 'a' + 10 + case 'A' <= ch && ch <= 'Z': + d = ch - 'A' + 10 + } + return Word(d) +} + +// scan sets z to the natural number corresponding to the longest possible prefix +// read from r representing an unsigned integer in a given conversion base. +// It returns z, the actual conversion base used, and an error, if any. In the +// error case, the value of z is undefined. The syntax follows the syntax of +// unsigned integer literals in Go. +// +// The base argument must be 0 or a value from 2 through MaxBase. If the base +// is 0, the string prefix determines the actual conversion base. A prefix of +// ``0x'' or ``0X'' selects base 16; the ``0'' prefix selects base 8, and a +// ``0b'' or ``0B'' prefix selects base 2. Otherwise the selected base is 10. +// +func (z nat) scan(r io.RuneScanner, base int) (nat, int, os.Error) { + // reject illegal bases + if base < 0 || base == 1 || MaxBase < base { + return z, 0, os.NewError("illegal number base") + } + + // one char look-ahead + ch, _, err := r.ReadRune() + if err != nil { + return z, 0, err + } + + // determine base if necessary + b := Word(base) + if base == 0 { + b = 10 + if ch == '0' { + switch ch, _, err = r.ReadRune(); err { + case nil: + b = 8 + switch ch { + case 'x', 'X': + b = 16 + case 'b', 'B': + b = 2 + } + if b == 2 || b == 16 { + if ch, _, err = r.ReadRune(); err != nil { + return z, 0, err + } + } + case os.EOF: + return z, 10, nil + default: + return z, 10, err + } + } + } + + // convert string + // - group as many digits d as possible together into a "super-digit" dd with "super-base" bb + // - only when bb does not fit into a word anymore, do a full number mulAddWW using bb and dd + z = z.make(0) + bb := Word(1) + dd := Word(0) + for max := _M / b; ; { + d := hexValue(ch) + if d >= b { + r.UnreadRune() // ch does not belong to number anymore + break + } + + if bb <= max { + bb *= b + dd = dd*b + d + } else { + // bb * b would overflow + z = z.mulAddWW(z, bb, dd) + bb = b + dd = d + } + + if ch, _, err = r.ReadRune(); err != nil { + if err != os.EOF { + return z, int(b), err + } + break + } + } + + switch { + case bb > 1: + // there was at least one mantissa digit + z = z.mulAddWW(z, bb, dd) + case base == 0 && b == 8: + // there was only the octal prefix 0 (possibly followed by digits > 7); + // return base 10, not 8 + return z, 10, nil + case base != 0 || b != 8: + // there was neither a mantissa digit nor the octal prefix 0 + return z, int(b), os.NewError("syntax error scanning number") + } + + return z.norm(), int(b), nil +} + +// Character sets for string conversion. +const ( + lowercaseDigits = "0123456789abcdefghijklmnopqrstuvwxyz" + uppercaseDigits = "0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZ" +) + +// decimalString returns a decimal representation of x. +// It calls x.string with the charset "0123456789". +func (x nat) decimalString() string { + return x.string(lowercaseDigits[0:10]) +} + +// string converts x to a string using digits from a charset; a digit with +// value d is represented by charset[d]. The conversion base is determined +// by len(charset), which must be >= 2. +func (x nat) string(charset string) string { + b := Word(len(charset)) + + // special cases + switch { + case b < 2 || b > 256: + panic("illegal base") + case len(x) == 0: + return string(charset[0]) + } + + // allocate buffer for conversion + i := x.bitLen()/log2(b) + 1 // +1: round up + s := make([]byte, i) + + // special case: power of two bases can avoid divisions completely + if b == b&-b { + // shift is base-b digit size in bits + shift := uint(trailingZeroBits(b)) // shift > 0 because b >= 2 + mask := Word(1)<= shift { + i-- + s[i] = charset[w&mask] + w >>= shift + nbits -= shift + } + + // convert any partial leading digit and advance to next word + if nbits == 0 { + // no partial digit remaining, just advance + w = x[k] + nbits = _W + } else { + // partial digit in current (k-1) and next (k) word + w |= x[k] << nbits + i-- + s[i] = charset[w&mask] + + // advance + w = x[k] >> (shift - nbits) + nbits = _W - (shift - nbits) + } + } + + // convert digits of most-significant word (omit leading zeros) + for nbits >= 0 && w != 0 { + i-- + s[i] = charset[w&mask] + w >>= shift + nbits -= shift + } + + return string(s[i:]) + } + + // general case: extract groups of digits by multiprecision division + + // maximize ndigits where b**ndigits < 2^_W; bb (big base) is b**ndigits + bb := Word(1) + ndigits := 0 + for max := Word(_M / b); bb <= max; bb *= b { + ndigits++ + } + + // preserve x, create local copy for use in repeated divisions + q := nat(nil).set(x) + var r Word + + // convert + if b == 10 { // hard-coding for 10 here speeds this up by 1.25x + for len(q) > 0 { + // extract least significant, base bb "digit" + q, r = q.divW(q, bb) // N.B. >82% of time is here. Optimize divW + if len(q) == 0 { + // skip leading zeros in most-significant group of digits + for j := 0; j < ndigits && r != 0; j++ { + i-- + s[i] = charset[r%10] + r /= 10 + } + } else { + for j := 0; j < ndigits; j++ { + i-- + s[i] = charset[r%10] + r /= 10 + } + } + } + } else { + for len(q) > 0 { + // extract least significant group of digits + q, r = q.divW(q, bb) // N.B. >82% of time is here. Optimize divW + if len(q) == 0 { + // skip leading zeros in most-significant group of digits + for j := 0; j < ndigits && r != 0; j++ { + i-- + s[i] = charset[r%b] + r /= b + } + } else { + for j := 0; j < ndigits; j++ { + i-- + s[i] = charset[r%b] + r /= b + } + } + } + } + + return string(s[i:]) +} + +const deBruijn32 = 0x077CB531 + +var deBruijn32Lookup = []byte{ + 0, 1, 28, 2, 29, 14, 24, 3, 30, 22, 20, 15, 25, 17, 4, 8, + 31, 27, 13, 23, 21, 19, 16, 7, 26, 12, 18, 6, 11, 5, 10, 9, +} + +const deBruijn64 = 0x03f79d71b4ca8b09 + +var deBruijn64Lookup = []byte{ + 0, 1, 56, 2, 57, 49, 28, 3, 61, 58, 42, 50, 38, 29, 17, 4, + 62, 47, 59, 36, 45, 43, 51, 22, 53, 39, 33, 30, 24, 18, 12, 5, + 63, 55, 48, 27, 60, 41, 37, 16, 46, 35, 44, 21, 52, 32, 23, 11, + 54, 26, 40, 15, 34, 20, 31, 10, 25, 14, 19, 9, 13, 8, 7, 6, +} + +// trailingZeroBits returns the number of consecutive zero bits on the right +// side of the given Word. +// See Knuth, volume 4, section 7.3.1 +func trailingZeroBits(x Word) int { + // x & -x leaves only the right-most bit set in the word. Let k be the + // index of that bit. Since only a single bit is set, the value is two + // to the power of k. Multiplying by a power of two is equivalent to + // left shifting, in this case by k bits. The de Bruijn constant is + // such that all six bit, consecutive substrings are distinct. + // Therefore, if we have a left shifted version of this constant we can + // find by how many bits it was shifted by looking at which six bit + // substring ended up at the top of the word. + switch _W { + case 32: + return int(deBruijn32Lookup[((x&-x)*deBruijn32)>>27]) + case 64: + return int(deBruijn64Lookup[((x&-x)*(deBruijn64&_M))>>58]) + default: + panic("Unknown word size") + } + + return 0 +} + +// z = x << s +func (z nat) shl(x nat, s uint) nat { + m := len(x) + if m == 0 { + return z.make(0) + } + // m > 0 + + n := m + int(s/_W) + z = z.make(n + 1) + z[n] = shlVU(z[n-m:n], x, s%_W) + z[0 : n-m].clear() + + return z.norm() +} + +// z = x >> s +func (z nat) shr(x nat, s uint) nat { + m := len(x) + n := m - int(s/_W) + if n <= 0 { + return z.make(0) + } + // n > 0 + + z = z.make(n) + shrVU(z, x[m-n:], s%_W) + + return z.norm() +} + +func (z nat) setBit(x nat, i uint, b uint) nat { + j := int(i / _W) + m := Word(1) << (i % _W) + n := len(x) + switch b { + case 0: + z = z.make(n) + copy(z, x) + if j >= n { + // no need to grow + return z + } + z[j] &^= m + return z.norm() + case 1: + if j >= n { + n = j + 1 + } + z = z.make(n) + copy(z, x) + z[j] |= m + // no need to normalize + return z + } + panic("set bit is not 0 or 1") +} + +func (z nat) bit(i uint) uint { + j := int(i / _W) + if j >= len(z) { + return 0 + } + return uint(z[j] >> (i % _W) & 1) +} + +func (z nat) and(x, y nat) nat { + m := len(x) + n := len(y) + if m > n { + m = n + } + // m <= n + + z = z.make(m) + for i := 0; i < m; i++ { + z[i] = x[i] & y[i] + } + + return z.norm() +} + +func (z nat) andNot(x, y nat) nat { + m := len(x) + n := len(y) + if n > m { + n = m + } + // m >= n + + z = z.make(m) + for i := 0; i < n; i++ { + z[i] = x[i] &^ y[i] + } + copy(z[n:m], x[n:m]) + + return z.norm() +} + +func (z nat) or(x, y nat) nat { + m := len(x) + n := len(y) + s := x + if m < n { + n, m = m, n + s = y + } + // m >= n + + z = z.make(m) + for i := 0; i < n; i++ { + z[i] = x[i] | y[i] + } + copy(z[n:m], s[n:m]) + + return z.norm() +} + +func (z nat) xor(x, y nat) nat { + m := len(x) + n := len(y) + s := x + if m < n { + n, m = m, n + s = y + } + // m >= n + + z = z.make(m) + for i := 0; i < n; i++ { + z[i] = x[i] ^ y[i] + } + copy(z[n:m], s[n:m]) + + return z.norm() +} + +// greaterThan returns true iff (x1<<_W + x2) > (y1<<_W + y2) +func greaterThan(x1, x2, y1, y2 Word) bool { + return x1 > y1 || x1 == y1 && x2 > y2 +} + +// modW returns x % d. +func (x nat) modW(d Word) (r Word) { + // TODO(agl): we don't actually need to store the q value. + var q nat + q = q.make(len(x)) + return divWVW(q, 0, x, d) +} + +// powersOfTwoDecompose finds q and k with x = q * 1<= 0; i-- { + v = y[i] + + for j := 0; j < _W; j++ { + z = z.mul(z, z) + + if v&mask != 0 { + z = z.mul(z, x) + } + + if m != nil { + q, z = q.div(z, z, m) + } + + v <<= 1 + } + } + + return z +} + +// probablyPrime performs reps Miller-Rabin tests to check whether n is prime. +// If it returns true, n is prime with probability 1 - 1/4^reps. +// If it returns false, n is not prime. +func (n nat) probablyPrime(reps int) bool { + if len(n) == 0 { + return false + } + + if len(n) == 1 { + if n[0] < 2 { + return false + } + + if n[0]%2 == 0 { + return n[0] == 2 + } + + // We have to exclude these cases because we reject all + // multiples of these numbers below. + switch n[0] { + case 3, 5, 7, 11, 13, 17, 19, 23, 29, 31, 37, 41, 43, 47, 53: + return true + } + } + + const primesProduct32 = 0xC0CFD797 // Π {p ∈ primes, 2 < p <= 29} + const primesProduct64 = 0xE221F97C30E94E1D // Π {p ∈ primes, 2 < p <= 53} + + var r Word + switch _W { + case 32: + r = n.modW(primesProduct32) + case 64: + r = n.modW(primesProduct64 & _M) + default: + panic("Unknown word size") + } + + if r%3 == 0 || r%5 == 0 || r%7 == 0 || r%11 == 0 || + r%13 == 0 || r%17 == 0 || r%19 == 0 || r%23 == 0 || r%29 == 0 { + return false + } + + if _W == 64 && (r%31 == 0 || r%37 == 0 || r%41 == 0 || + r%43 == 0 || r%47 == 0 || r%53 == 0) { + return false + } + + nm1 := nat(nil).sub(n, natOne) + // 1<= len(z)*_S. The value of z is encoded in the +// slice buf[i:]. The number i of unused bytes at the beginning of +// buf is returned as result. +func (z nat) bytes(buf []byte) (i int) { + i = len(buf) + for _, d := range z { + for j := 0; j < _S; j++ { + i-- + buf[i] = byte(d) + d >>= 8 + } + } + + for i < len(buf) && buf[i] == 0 { + i++ + } + + return +} + +// setBytes interprets buf as the bytes of a big-endian unsigned +// integer, sets z to that value, and returns z. +func (z nat) setBytes(buf []byte) nat { + z = z.make((len(buf) + _S - 1) / _S) + + k := 0 + s := uint(0) + var d Word + for i := len(buf); i > 0; i-- { + d |= Word(buf[i-1]) << s + if s += 8; s == _S*8 { + z[k] = d + k++ + s = 0 + d = 0 + } + } + if k < len(z) { + z[k] = d + } + + return z.norm() +} diff --git a/src/pkg/big/nat_test.go b/src/pkg/big/nat_test.go new file mode 100755 index 000000000..71d086087 --- /dev/null +++ b/src/pkg/big/nat_test.go @@ -0,0 +1,669 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package big + +import ( + "fmt" + "os" + "strings" + "testing" +) + +var cmpTests = []struct { + x, y nat + r int +}{ + {nil, nil, 0}, + {nil, nat{}, 0}, + {nat{}, nil, 0}, + {nat{}, nat{}, 0}, + {nat{0}, nat{0}, 0}, + {nat{0}, nat{1}, -1}, + {nat{1}, nat{0}, 1}, + {nat{1}, nat{1}, 0}, + {nat{0, _M}, nat{1}, 1}, + {nat{1}, nat{0, _M}, -1}, + {nat{1, _M}, nat{0, _M}, 1}, + {nat{0, _M}, nat{1, _M}, -1}, + {nat{16, 571956, 8794, 68}, nat{837, 9146, 1, 754489}, -1}, + {nat{34986, 41, 105, 1957}, nat{56, 7458, 104, 1957}, 1}, +} + +func TestCmp(t *testing.T) { + for i, a := range cmpTests { + r := a.x.cmp(a.y) + if r != a.r { + t.Errorf("#%d got r = %v; want %v", i, r, a.r) + } + } +} + +type funNN func(z, x, y nat) nat +type argNN struct { + z, x, y nat +} + +var sumNN = []argNN{ + {}, + {nat{1}, nil, nat{1}}, + {nat{1111111110}, nat{123456789}, nat{987654321}}, + {nat{0, 0, 0, 1}, nil, nat{0, 0, 0, 1}}, + {nat{0, 0, 0, 1111111110}, nat{0, 0, 0, 123456789}, nat{0, 0, 0, 987654321}}, + {nat{0, 0, 0, 1}, nat{0, 0, _M}, nat{0, 0, 1}}, +} + +var prodNN = []argNN{ + {}, + {nil, nil, nil}, + {nil, nat{991}, nil}, + {nat{991}, nat{991}, nat{1}}, + {nat{991 * 991}, nat{991}, nat{991}}, + {nat{0, 0, 991 * 991}, nat{0, 991}, nat{0, 991}}, + {nat{1 * 991, 2 * 991, 3 * 991, 4 * 991}, nat{1, 2, 3, 4}, nat{991}}, + {nat{4, 11, 20, 30, 20, 11, 4}, nat{1, 2, 3, 4}, nat{4, 3, 2, 1}}, +} + +func TestSet(t *testing.T) { + for _, a := range sumNN { + z := nat(nil).set(a.z) + if z.cmp(a.z) != 0 { + t.Errorf("got z = %v; want %v", z, a.z) + } + } +} + +func testFunNN(t *testing.T, msg string, f funNN, a argNN) { + z := f(nil, a.x, a.y) + if z.cmp(a.z) != 0 { + t.Errorf("%s%+v\n\tgot z = %v; want %v", msg, a, z, a.z) + } +} + +func TestFunNN(t *testing.T) { + for _, a := range sumNN { + arg := a + testFunNN(t, "add", nat.add, arg) + + arg = argNN{a.z, a.y, a.x} + testFunNN(t, "add symmetric", nat.add, arg) + + arg = argNN{a.x, a.z, a.y} + testFunNN(t, "sub", nat.sub, arg) + + arg = argNN{a.y, a.z, a.x} + testFunNN(t, "sub symmetric", nat.sub, arg) + } + + for _, a := range prodNN { + arg := a + testFunNN(t, "mul", nat.mul, arg) + + arg = argNN{a.z, a.y, a.x} + testFunNN(t, "mul symmetric", nat.mul, arg) + } +} + +var mulRangesN = []struct { + a, b uint64 + prod string +}{ + {0, 0, "0"}, + {1, 1, "1"}, + {1, 2, "2"}, + {1, 3, "6"}, + {10, 10, "10"}, + {0, 100, "0"}, + {0, 1e9, "0"}, + {1, 0, "1"}, // empty range + {100, 1, "1"}, // empty range + {1, 10, "3628800"}, // 10! + {1, 20, "2432902008176640000"}, // 20! + {1, 100, + "933262154439441526816992388562667004907159682643816214685929" + + "638952175999932299156089414639761565182862536979208272237582" + + "51185210916864000000000000000000000000", // 100! + }, +} + +func TestMulRangeN(t *testing.T) { + for i, r := range mulRangesN { + prod := nat(nil).mulRange(r.a, r.b).decimalString() + if prod != r.prod { + t.Errorf("#%d: got %s; want %s", i, prod, r.prod) + } + } +} + +var mulArg, mulTmp nat + +func init() { + const n = 1000 + mulArg = make(nat, n) + for i := 0; i < n; i++ { + mulArg[i] = _M + } +} + +func benchmarkMulLoad() { + for j := 1; j <= 10; j++ { + x := mulArg[0 : j*100] + mulTmp.mul(x, x) + } +} + +func BenchmarkMul(b *testing.B) { + for i := 0; i < b.N; i++ { + benchmarkMulLoad() + } +} + +func toString(x nat, charset string) string { + base := len(charset) + + // special cases + switch { + case base < 2: + panic("illegal base") + case len(x) == 0: + return string(charset[0]) + } + + // allocate buffer for conversion + i := x.bitLen()/log2(Word(base)) + 1 // +1: round up + s := make([]byte, i) + + // don't destroy x + q := nat(nil).set(x) + + // convert + for len(q) > 0 { + i-- + var r Word + q, r = q.divW(q, Word(base)) + s[i] = charset[r] + } + + return string(s[i:]) +} + +var strTests = []struct { + x nat // nat value to be converted + c string // conversion charset + s string // expected result +}{ + {nil, "01", "0"}, + {nat{1}, "01", "1"}, + {nat{0xc5}, "01", "11000101"}, + {nat{03271}, lowercaseDigits[0:8], "3271"}, + {nat{10}, lowercaseDigits[0:10], "10"}, + {nat{1234567890}, uppercaseDigits[0:10], "1234567890"}, + {nat{0xdeadbeef}, lowercaseDigits[0:16], "deadbeef"}, + {nat{0xdeadbeef}, uppercaseDigits[0:16], "DEADBEEF"}, + {nat{0x229be7}, lowercaseDigits[0:17], "1a2b3c"}, + {nat{0x309663e6}, uppercaseDigits[0:32], "O9COV6"}, +} + +func TestString(t *testing.T) { + for _, a := range strTests { + s := a.x.string(a.c) + if s != a.s { + t.Errorf("string%+v\n\tgot s = %s; want %s", a, s, a.s) + } + + x, b, err := nat(nil).scan(strings.NewReader(a.s), len(a.c)) + if x.cmp(a.x) != 0 { + t.Errorf("scan%+v\n\tgot z = %v; want %v", a, x, a.x) + } + if b != len(a.c) { + t.Errorf("scan%+v\n\tgot b = %d; want %d", a, b, len(a.c)) + } + if err != nil { + t.Errorf("scan%+v\n\tgot error = %s", a, err) + } + } +} + +var natScanTests = []struct { + s string // string to be scanned + base int // input base + x nat // expected nat + b int // expected base + ok bool // expected success + next int // next character (or 0, if at EOF) +}{ + // error: illegal base + {base: -1}, + {base: 1}, + {base: 37}, + + // error: no mantissa + {}, + {s: "?"}, + {base: 10}, + {base: 36}, + {s: "?", base: 10}, + {s: "0x"}, + {s: "345", base: 2}, + + // no errors + {"0", 0, nil, 10, true, 0}, + {"0", 10, nil, 10, true, 0}, + {"0", 36, nil, 36, true, 0}, + {"1", 0, nat{1}, 10, true, 0}, + {"1", 10, nat{1}, 10, true, 0}, + {"0 ", 0, nil, 10, true, ' '}, + {"08", 0, nil, 10, true, '8'}, + {"018", 0, nat{1}, 8, true, '8'}, + {"0b1", 0, nat{1}, 2, true, 0}, + {"0b11000101", 0, nat{0xc5}, 2, true, 0}, + {"03271", 0, nat{03271}, 8, true, 0}, + {"10ab", 0, nat{10}, 10, true, 'a'}, + {"1234567890", 0, nat{1234567890}, 10, true, 0}, + {"xyz", 36, nat{(33*36+34)*36 + 35}, 36, true, 0}, + {"xyz?", 36, nat{(33*36+34)*36 + 35}, 36, true, '?'}, + {"0x", 16, nil, 16, true, 'x'}, + {"0xdeadbeef", 0, nat{0xdeadbeef}, 16, true, 0}, + {"0XDEADBEEF", 0, nat{0xdeadbeef}, 16, true, 0}, +} + +func TestScanBase(t *testing.T) { + for _, a := range natScanTests { + r := strings.NewReader(a.s) + x, b, err := nat(nil).scan(r, a.base) + if err == nil && !a.ok { + t.Errorf("scan%+v\n\texpected error", a) + } + if err != nil { + if a.ok { + t.Errorf("scan%+v\n\tgot error = %s", a, err) + } + continue + } + if x.cmp(a.x) != 0 { + t.Errorf("scan%+v\n\tgot z = %v; want %v", a, x, a.x) + } + if b != a.b { + t.Errorf("scan%+v\n\tgot b = %d; want %d", a, b, a.base) + } + next, _, err := r.ReadRune() + if err == os.EOF { + next = 0 + err = nil + } + if err == nil && next != a.next { + t.Errorf("scan%+v\n\tgot next = %q; want %q", a, next, a.next) + } + } +} + +var pi = "3" + + "14159265358979323846264338327950288419716939937510582097494459230781640628620899862803482534211706798214808651" + + "32823066470938446095505822317253594081284811174502841027019385211055596446229489549303819644288109756659334461" + + "28475648233786783165271201909145648566923460348610454326648213393607260249141273724587006606315588174881520920" + + "96282925409171536436789259036001133053054882046652138414695194151160943305727036575959195309218611738193261179" + + "31051185480744623799627495673518857527248912279381830119491298336733624406566430860213949463952247371907021798" + + "60943702770539217176293176752384674818467669405132000568127145263560827785771342757789609173637178721468440901" + + "22495343014654958537105079227968925892354201995611212902196086403441815981362977477130996051870721134999999837" + + "29780499510597317328160963185950244594553469083026425223082533446850352619311881710100031378387528865875332083" + + "81420617177669147303598253490428755468731159562863882353787593751957781857780532171226806613001927876611195909" + + "21642019893809525720106548586327886593615338182796823030195203530185296899577362259941389124972177528347913151" + + "55748572424541506959508295331168617278558890750983817546374649393192550604009277016711390098488240128583616035" + + "63707660104710181942955596198946767837449448255379774726847104047534646208046684259069491293313677028989152104" + + "75216205696602405803815019351125338243003558764024749647326391419927260426992279678235478163600934172164121992" + + "45863150302861829745557067498385054945885869269956909272107975093029553211653449872027559602364806654991198818" + + "34797753566369807426542527862551818417574672890977772793800081647060016145249192173217214772350141441973568548" + + "16136115735255213347574184946843852332390739414333454776241686251898356948556209921922218427255025425688767179" + + "04946016534668049886272327917860857843838279679766814541009538837863609506800642251252051173929848960841284886" + + "26945604241965285022210661186306744278622039194945047123713786960956364371917287467764657573962413890865832645" + + "99581339047802759009946576407895126946839835259570982582262052248940772671947826848260147699090264013639443745" + + "53050682034962524517493996514314298091906592509372216964615157098583874105978859597729754989301617539284681382" + + "68683868942774155991855925245953959431049972524680845987273644695848653836736222626099124608051243884390451244" + + "13654976278079771569143599770012961608944169486855584840635342207222582848864815845602850601684273945226746767" + + "88952521385225499546667278239864565961163548862305774564980355936345681743241125150760694794510965960940252288" + + "79710893145669136867228748940560101503308617928680920874760917824938589009714909675985261365549781893129784821" + + "68299894872265880485756401427047755513237964145152374623436454285844479526586782105114135473573952311342716610" + + "21359695362314429524849371871101457654035902799344037420073105785390621983874478084784896833214457138687519435" + + "06430218453191048481005370614680674919278191197939952061419663428754440643745123718192179998391015919561814675" + + "14269123974894090718649423196156794520809514655022523160388193014209376213785595663893778708303906979207734672" + + "21825625996615014215030680384477345492026054146659252014974428507325186660021324340881907104863317346496514539" + + "05796268561005508106658796998163574736384052571459102897064140110971206280439039759515677157700420337869936007" + + "23055876317635942187312514712053292819182618612586732157919841484882916447060957527069572209175671167229109816" + + "90915280173506712748583222871835209353965725121083579151369882091444210067510334671103141267111369908658516398" + + "31501970165151168517143765761835155650884909989859982387345528331635507647918535893226185489632132933089857064" + + "20467525907091548141654985946163718027098199430992448895757128289059232332609729971208443357326548938239119325" + + "97463667305836041428138830320382490375898524374417029132765618093773444030707469211201913020330380197621101100" + + "44929321516084244485963766983895228684783123552658213144957685726243344189303968642624341077322697802807318915" + + "44110104468232527162010526522721116603966655730925471105578537634668206531098965269186205647693125705863566201" + + "85581007293606598764861179104533488503461136576867532494416680396265797877185560845529654126654085306143444318" + + "58676975145661406800700237877659134401712749470420562230538994561314071127000407854733269939081454664645880797" + + "27082668306343285878569830523580893306575740679545716377525420211495576158140025012622859413021647155097925923" + + "09907965473761255176567513575178296664547791745011299614890304639947132962107340437518957359614589019389713111" + + "79042978285647503203198691514028708085990480109412147221317947647772622414254854540332157185306142288137585043" + + "06332175182979866223717215916077166925474873898665494945011465406284336639379003976926567214638530673609657120" + + "91807638327166416274888800786925602902284721040317211860820419000422966171196377921337575114959501566049631862" + + "94726547364252308177036751590673502350728354056704038674351362222477158915049530984448933309634087807693259939" + + "78054193414473774418426312986080998886874132604721569516239658645730216315981931951673538129741677294786724229" + + "24654366800980676928238280689964004824354037014163149658979409243237896907069779422362508221688957383798623001" + + "59377647165122893578601588161755782973523344604281512627203734314653197777416031990665541876397929334419521541" + + "34189948544473456738316249934191318148092777710386387734317720754565453220777092120190516609628049092636019759" + + "88281613323166636528619326686336062735676303544776280350450777235547105859548702790814356240145171806246436267" + + "94561275318134078330336254232783944975382437205835311477119926063813346776879695970309833913077109870408591337" + +// Test case for BenchmarkScanPi. +func TestScanPi(t *testing.T) { + var x nat + z, _, err := x.scan(strings.NewReader(pi), 10) + if err != nil { + t.Errorf("scanning pi: %s", err) + } + if s := z.decimalString(); s != pi { + t.Errorf("scanning pi: got %s", s) + } +} + +func BenchmarkScanPi(b *testing.B) { + for i := 0; i < b.N; i++ { + var x nat + x.scan(strings.NewReader(pi), 10) + } +} + +const ( + // 314**271 + // base 2: 2249 digits + // base 8: 751 digits + // base 10: 678 digits + // base 16: 563 digits + shortBase = 314 + shortExponent = 271 + + // 3141**2178 + // base 2: 31577 digits + // base 8: 10527 digits + // base 10: 9507 digits + // base 16: 7895 digits + mediumBase = 3141 + mediumExponent = 2718 + + // 3141**2178 + // base 2: 406078 digits + // base 8: 135360 digits + // base 10: 122243 digits + // base 16: 101521 digits + longBase = 31415 + longExponent = 27182 +) + +func BenchmarkScanShort2(b *testing.B) { + ScanHelper(b, 2, shortBase, shortExponent) +} + +func BenchmarkScanShort8(b *testing.B) { + ScanHelper(b, 8, shortBase, shortExponent) +} + +func BenchmarkScanSort10(b *testing.B) { + ScanHelper(b, 10, shortBase, shortExponent) +} + +func BenchmarkScanShort16(b *testing.B) { + ScanHelper(b, 16, shortBase, shortExponent) +} + +func BenchmarkScanMedium2(b *testing.B) { + ScanHelper(b, 2, mediumBase, mediumExponent) +} + +func BenchmarkScanMedium8(b *testing.B) { + ScanHelper(b, 8, mediumBase, mediumExponent) +} + +func BenchmarkScanMedium10(b *testing.B) { + ScanHelper(b, 10, mediumBase, mediumExponent) +} + +func BenchmarkScanMedium16(b *testing.B) { + ScanHelper(b, 16, mediumBase, mediumExponent) +} + +func BenchmarkScanLong2(b *testing.B) { + ScanHelper(b, 2, longBase, longExponent) +} + +func BenchmarkScanLong8(b *testing.B) { + ScanHelper(b, 8, longBase, longExponent) +} + +func BenchmarkScanLong10(b *testing.B) { + ScanHelper(b, 10, longBase, longExponent) +} + +func BenchmarkScanLong16(b *testing.B) { + ScanHelper(b, 16, longBase, longExponent) +} + +func ScanHelper(b *testing.B, base int, xv, yv Word) { + b.StopTimer() + var x, y, z nat + x = x.setWord(xv) + y = y.setWord(yv) + z = z.expNN(x, y, nil) + + var s string + s = z.string(lowercaseDigits[0:base]) + if t := toString(z, lowercaseDigits[0:base]); t != s { + panic(fmt.Sprintf("scanning: got %s; want %s", s, t)) + } + b.StartTimer() + + for i := 0; i < b.N; i++ { + x.scan(strings.NewReader(s), base) + } +} + +func BenchmarkStringShort2(b *testing.B) { + StringHelper(b, 2, shortBase, shortExponent) +} + +func BenchmarkStringShort8(b *testing.B) { + StringHelper(b, 8, shortBase, shortExponent) +} + +func BenchmarkStringShort10(b *testing.B) { + StringHelper(b, 10, shortBase, shortExponent) +} + +func BenchmarkStringShort16(b *testing.B) { + StringHelper(b, 16, shortBase, shortExponent) +} + +func BenchmarkStringMedium2(b *testing.B) { + StringHelper(b, 2, mediumBase, mediumExponent) +} + +func BenchmarkStringMedium8(b *testing.B) { + StringHelper(b, 8, mediumBase, mediumExponent) +} + +func BenchmarkStringMedium10(b *testing.B) { + StringHelper(b, 10, mediumBase, mediumExponent) +} + +func BenchmarkStringMedium16(b *testing.B) { + StringHelper(b, 16, mediumBase, mediumExponent) +} + +func BenchmarkStringLong2(b *testing.B) { + StringHelper(b, 2, longBase, longExponent) +} + +func BenchmarkStringLong8(b *testing.B) { + StringHelper(b, 8, longBase, longExponent) +} + +func BenchmarkStringLong10(b *testing.B) { + StringHelper(b, 10, longBase, longExponent) +} + +func BenchmarkStringLong16(b *testing.B) { + StringHelper(b, 16, longBase, longExponent) +} + +func StringHelper(b *testing.B, base int, xv, yv Word) { + b.StopTimer() + var x, y, z nat + x = x.setWord(xv) + y = y.setWord(yv) + z = z.expNN(x, y, nil) + b.StartTimer() + + for i := 0; i < b.N; i++ { + z.string(lowercaseDigits[0:base]) + } +} + +func TestLeadingZeros(t *testing.T) { + var x Word = _B >> 1 + for i := 0; i <= _W; i++ { + if int(leadingZeros(x)) != i { + t.Errorf("failed at %x: got %d want %d", x, leadingZeros(x), i) + } + x >>= 1 + } +} + +type shiftTest struct { + in nat + shift uint + out nat +} + +var leftShiftTests = []shiftTest{ + {nil, 0, nil}, + {nil, 1, nil}, + {natOne, 0, natOne}, + {natOne, 1, natTwo}, + {nat{1 << (_W - 1)}, 1, nat{0}}, + {nat{1 << (_W - 1), 0}, 1, nat{0, 1}}, +} + +func TestShiftLeft(t *testing.T) { + for i, test := range leftShiftTests { + var z nat + z = z.shl(test.in, test.shift) + for j, d := range test.out { + if j >= len(z) || z[j] != d { + t.Errorf("#%d: got: %v want: %v", i, z, test.out) + break + } + } + } +} + +var rightShiftTests = []shiftTest{ + {nil, 0, nil}, + {nil, 1, nil}, + {natOne, 0, natOne}, + {natOne, 1, nil}, + {natTwo, 1, natOne}, + {nat{0, 1}, 1, nat{1 << (_W - 1)}}, + {nat{2, 1, 1}, 1, nat{1<<(_W-1) + 1, 1 << (_W - 1)}}, +} + +func TestShiftRight(t *testing.T) { + for i, test := range rightShiftTests { + var z nat + z = z.shr(test.in, test.shift) + for j, d := range test.out { + if j >= len(z) || z[j] != d { + t.Errorf("#%d: got: %v want: %v", i, z, test.out) + break + } + } + } +} + +type modWTest struct { + in string + dividend string + out string +} + +var modWTests32 = []modWTest{ + {"23492635982634928349238759823742", "252341", "220170"}, +} + +var modWTests64 = []modWTest{ + {"6527895462947293856291561095690465243862946", "524326975699234", "375066989628668"}, +} + +func runModWTests(t *testing.T, tests []modWTest) { + for i, test := range tests { + in, _ := new(Int).SetString(test.in, 10) + d, _ := new(Int).SetString(test.dividend, 10) + out, _ := new(Int).SetString(test.out, 10) + + r := in.abs.modW(d.abs[0]) + if r != out.abs[0] { + t.Errorf("#%d failed: got %s want %s", i, r, out) + } + } +} + +func TestModW(t *testing.T) { + if _W >= 32 { + runModWTests(t, modWTests32) + } + if _W >= 64 { + runModWTests(t, modWTests64) + } +} + +func TestTrailingZeroBits(t *testing.T) { + var x Word + x-- + for i := 0; i < _W; i++ { + if trailingZeroBits(x) != i { + t.Errorf("Failed at step %d: x: %x got: %d", i, x, trailingZeroBits(x)) + } + x <<= 1 + } +} + +var expNNTests = []struct { + x, y, m string + out string +}{ + {"0x8000000000000000", "2", "", "0x40000000000000000000000000000000"}, + {"0x8000000000000000", "2", "6719", "4944"}, + {"0x8000000000000000", "3", "6719", "5447"}, + {"0x8000000000000000", "1000", "6719", "1603"}, + {"0x8000000000000000", "1000000", "6719", "3199"}, + { + "2938462938472983472983659726349017249287491026512746239764525612965293865296239471239874193284792387498274256129746192347", + "298472983472983471903246121093472394872319615612417471234712061", + "29834729834729834729347290846729561262544958723956495615629569234729836259263598127342374289365912465901365498236492183464", + "23537740700184054162508175125554701713153216681790245129157191391322321508055833908509185839069455749219131480588829346291", + }, +} + +func TestExpNN(t *testing.T) { + for i, test := range expNNTests { + x, _, _ := nat(nil).scan(strings.NewReader(test.x), 0) + y, _, _ := nat(nil).scan(strings.NewReader(test.y), 0) + out, _, _ := nat(nil).scan(strings.NewReader(test.out), 0) + + var m nat + + if len(test.m) > 0 { + m, _, _ = nat(nil).scan(strings.NewReader(test.m), 0) + } + + z := nat(nil).expNN(x, y, m) + if z.cmp(out) != 0 { + t.Errorf("#%d got %v want %v", i, z, out) + } + } +} diff --git a/src/pkg/big/rat.go b/src/pkg/big/rat.go new file mode 100644 index 000000000..327b9bd9c --- /dev/null +++ b/src/pkg/big/rat.go @@ -0,0 +1,371 @@ +// Copyright 2010 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// This file implements multi-precision rational numbers. + +package big + +import ( + "encoding/binary" + "fmt" + "os" + "strings" +) + +// A Rat represents a quotient a/b of arbitrary precision. The zero value for +// a Rat, 0/0, is not a legal Rat. +type Rat struct { + a Int + b nat +} + +// NewRat creates a new Rat with numerator a and denominator b. +func NewRat(a, b int64) *Rat { + return new(Rat).SetFrac64(a, b) +} + +// SetFrac sets z to a/b and returns z. +func (z *Rat) SetFrac(a, b *Int) *Rat { + z.a.Set(a) + z.a.neg = a.neg != b.neg + z.b = z.b.set(b.abs) + return z.norm() +} + +// SetFrac64 sets z to a/b and returns z. +func (z *Rat) SetFrac64(a, b int64) *Rat { + z.a.SetInt64(a) + if b < 0 { + b = -b + z.a.neg = !z.a.neg + } + z.b = z.b.setUint64(uint64(b)) + return z.norm() +} + +// SetInt sets z to x (by making a copy of x) and returns z. +func (z *Rat) SetInt(x *Int) *Rat { + z.a.Set(x) + z.b = z.b.setWord(1) + return z +} + +// SetInt64 sets z to x and returns z. +func (z *Rat) SetInt64(x int64) *Rat { + z.a.SetInt64(x) + z.b = z.b.setWord(1) + return z +} + +// Sign returns: +// +// -1 if x < 0 +// 0 if x == 0 +// +1 if x > 0 +// +func (x *Rat) Sign() int { + return x.a.Sign() +} + +// IsInt returns true if the denominator of x is 1. +func (x *Rat) IsInt() bool { + return len(x.b) == 1 && x.b[0] == 1 +} + +// Num returns the numerator of z; it may be <= 0. +// The result is a reference to z's numerator; it +// may change if a new value is assigned to z. +func (z *Rat) Num() *Int { + return &z.a +} + +// Denom returns the denominator of z; it is always > 0. +// The result is a reference to z's denominator; it +// may change if a new value is assigned to z. +func (z *Rat) Denom() *Int { + return &Int{false, z.b} +} + +func gcd(x, y nat) nat { + // Euclidean algorithm. + var a, b nat + a = a.set(x) + b = b.set(y) + for len(b) != 0 { + var q, r nat + _, r = q.div(r, a, b) + a = b + b = r + } + return a +} + +func (z *Rat) norm() *Rat { + f := gcd(z.a.abs, z.b) + if len(z.a.abs) == 0 { + // z == 0 + z.a.neg = false // normalize sign + z.b = z.b.setWord(1) + return z + } + if f.cmp(natOne) != 0 { + z.a.abs, _ = z.a.abs.div(nil, z.a.abs, f) + z.b, _ = z.b.div(nil, z.b, f) + } + return z +} + +func mulNat(x *Int, y nat) *Int { + var z Int + z.abs = z.abs.mul(x.abs, y) + z.neg = len(z.abs) > 0 && x.neg + return &z +} + +// Cmp compares x and y and returns: +// +// -1 if x < y +// 0 if x == y +// +1 if x > y +// +func (x *Rat) Cmp(y *Rat) (r int) { + return mulNat(&x.a, y.b).Cmp(mulNat(&y.a, x.b)) +} + +// Abs sets z to |x| (the absolute value of x) and returns z. +func (z *Rat) Abs(x *Rat) *Rat { + z.a.Abs(&x.a) + z.b = z.b.set(x.b) + return z +} + +// Add sets z to the sum x+y and returns z. +func (z *Rat) Add(x, y *Rat) *Rat { + a1 := mulNat(&x.a, y.b) + a2 := mulNat(&y.a, x.b) + z.a.Add(a1, a2) + z.b = z.b.mul(x.b, y.b) + return z.norm() +} + +// Sub sets z to the difference x-y and returns z. +func (z *Rat) Sub(x, y *Rat) *Rat { + a1 := mulNat(&x.a, y.b) + a2 := mulNat(&y.a, x.b) + z.a.Sub(a1, a2) + z.b = z.b.mul(x.b, y.b) + return z.norm() +} + +// Mul sets z to the product x*y and returns z. +func (z *Rat) Mul(x, y *Rat) *Rat { + z.a.Mul(&x.a, &y.a) + z.b = z.b.mul(x.b, y.b) + return z.norm() +} + +// Quo sets z to the quotient x/y and returns z. +// If y == 0, a division-by-zero run-time panic occurs. +func (z *Rat) Quo(x, y *Rat) *Rat { + if len(y.a.abs) == 0 { + panic("division by zero") + } + a := mulNat(&x.a, y.b) + b := mulNat(&y.a, x.b) + z.a.abs = a.abs + z.b = b.abs + z.a.neg = a.neg != b.neg + return z.norm() +} + +// Neg sets z to -x (by making a copy of x if necessary) and returns z. +func (z *Rat) Neg(x *Rat) *Rat { + z.a.Neg(&x.a) + z.b = z.b.set(x.b) + return z +} + +// Set sets z to x (by making a copy of x if necessary) and returns z. +func (z *Rat) Set(x *Rat) *Rat { + z.a.Set(&x.a) + z.b = z.b.set(x.b) + return z +} + +func ratTok(ch int) bool { + return strings.IndexRune("+-/0123456789.eE", ch) >= 0 +} + +// Scan is a support routine for fmt.Scanner. It accepts the formats +// 'e', 'E', 'f', 'F', 'g', 'G', and 'v'. All formats are equivalent. +func (z *Rat) Scan(s fmt.ScanState, ch int) os.Error { + tok, err := s.Token(true, ratTok) + if err != nil { + return err + } + if strings.IndexRune("efgEFGv", ch) < 0 { + return os.NewError("Rat.Scan: invalid verb") + } + if _, ok := z.SetString(string(tok)); !ok { + return os.NewError("Rat.Scan: invalid syntax") + } + return nil +} + +// SetString sets z to the value of s and returns z and a boolean indicating +// success. s can be given as a fraction "a/b" or as a floating-point number +// optionally followed by an exponent. If the operation failed, the value of z +// is undefined. +func (z *Rat) SetString(s string) (*Rat, bool) { + if len(s) == 0 { + return z, false + } + + // check for a quotient + sep := strings.Index(s, "/") + if sep >= 0 { + if _, ok := z.a.SetString(s[0:sep], 10); !ok { + return z, false + } + s = s[sep+1:] + var err os.Error + if z.b, _, err = z.b.scan(strings.NewReader(s), 10); err != nil { + return z, false + } + return z.norm(), true + } + + // check for a decimal point + sep = strings.Index(s, ".") + // check for an exponent + e := strings.IndexAny(s, "eE") + var exp Int + if e >= 0 { + if e < sep { + // The E must come after the decimal point. + return z, false + } + if _, ok := exp.SetString(s[e+1:], 10); !ok { + return z, false + } + s = s[0:e] + } + if sep >= 0 { + s = s[0:sep] + s[sep+1:] + exp.Sub(&exp, NewInt(int64(len(s)-sep))) + } + + if _, ok := z.a.SetString(s, 10); !ok { + return z, false + } + powTen := nat{}.expNN(natTen, exp.abs, nil) + if exp.neg { + z.b = powTen + z.norm() + } else { + z.a.abs = z.a.abs.mul(z.a.abs, powTen) + z.b = z.b.setWord(1) + } + + return z, true +} + +// String returns a string representation of z in the form "a/b" (even if b == 1). +func (z *Rat) String() string { + return z.a.String() + "/" + z.b.decimalString() +} + +// RatString returns a string representation of z in the form "a/b" if b != 1, +// and in the form "a" if b == 1. +func (z *Rat) RatString() string { + if z.IsInt() { + return z.a.String() + } + return z.String() +} + +// FloatString returns a string representation of z in decimal form with prec +// digits of precision after the decimal point and the last digit rounded. +func (z *Rat) FloatString(prec int) string { + if z.IsInt() { + s := z.a.String() + if prec > 0 { + s += "." + strings.Repeat("0", prec) + } + return s + } + + q, r := nat{}.div(nat{}, z.a.abs, z.b) + + p := natOne + if prec > 0 { + p = nat{}.expNN(natTen, nat{}.setUint64(uint64(prec)), nil) + } + + r = r.mul(r, p) + r, r2 := r.div(nat{}, r, z.b) + + // see if we need to round up + r2 = r2.add(r2, r2) + if z.b.cmp(r2) <= 0 { + r = r.add(r, natOne) + if r.cmp(p) >= 0 { + q = nat{}.add(q, natOne) + r = nat{}.sub(r, p) + } + } + + s := q.decimalString() + if z.a.neg { + s = "-" + s + } + + if prec > 0 { + rs := r.decimalString() + leadingZeros := prec - len(rs) + s += "." + strings.Repeat("0", leadingZeros) + rs + } + + return s +} + +// Gob codec version. Permits backward-compatible changes to the encoding. +const ratGobVersion byte = 1 + +// GobEncode implements the gob.GobEncoder interface. +func (z *Rat) GobEncode() ([]byte, os.Error) { + buf := make([]byte, 1+4+(len(z.a.abs)+len(z.b))*_S) // extra bytes for version and sign bit (1), and numerator length (4) + i := z.b.bytes(buf) + j := z.a.abs.bytes(buf[0:i]) + n := i - j + if int(uint32(n)) != n { + // this should never happen + return nil, os.NewError("Rat.GobEncode: numerator too large") + } + binary.BigEndian.PutUint32(buf[j-4:j], uint32(n)) + j -= 1 + 4 + b := ratGobVersion << 1 // make space for sign bit + if z.a.neg { + b |= 1 + } + buf[j] = b + return buf[j:], nil +} + +// GobDecode implements the gob.GobDecoder interface. +func (z *Rat) GobDecode(buf []byte) os.Error { + if len(buf) == 0 { + return os.NewError("Rat.GobDecode: no data") + } + b := buf[0] + if b>>1 != ratGobVersion { + return os.NewError(fmt.Sprintf("Rat.GobDecode: encoding version %d not supported", b>>1)) + } + const j = 1 + 4 + i := j + binary.BigEndian.Uint32(buf[j-4:j]) + z.a.neg = b&1 != 0 + z.a.abs = z.a.abs.setBytes(buf[j:i]) + z.b = z.b.setBytes(buf[i:]) + return nil +} diff --git a/src/pkg/big/rat_test.go b/src/pkg/big/rat_test.go new file mode 100644 index 000000000..dbc5bb6cc --- /dev/null +++ b/src/pkg/big/rat_test.go @@ -0,0 +1,332 @@ +// Copyright 2010 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package big + +import ( + "bytes" + "fmt" + "gob" + "testing" +) + +var setStringTests = []struct { + in, out string + ok bool +}{ + {"0", "0", true}, + {"-0", "0", true}, + {"1", "1", true}, + {"-1", "-1", true}, + {"1.", "1", true}, + {"1e0", "1", true}, + {"1.e1", "10", true}, + {in: "1e", ok: false}, + {in: "1.e", ok: false}, + {in: "1e+14e-5", ok: false}, + {in: "1e4.5", ok: false}, + {in: "r", ok: false}, + {in: "a/b", ok: false}, + {in: "a.b", ok: false}, + {"-0.1", "-1/10", true}, + {"-.1", "-1/10", true}, + {"2/4", "1/2", true}, + {".25", "1/4", true}, + {"-1/5", "-1/5", true}, + {"8129567.7690E14", "812956776900000000000", true}, + {"78189e+4", "781890000", true}, + {"553019.8935e+8", "55301989350000", true}, + {"98765432109876543210987654321e-10", "98765432109876543210987654321/10000000000", true}, + {"9877861857500000E-7", "3951144743/4", true}, + {"2169378.417e-3", "2169378417/1000000", true}, + {"884243222337379604041632732738665534", "884243222337379604041632732738665534", true}, + {"53/70893980658822810696", "53/70893980658822810696", true}, + {"106/141787961317645621392", "53/70893980658822810696", true}, + {"204211327800791583.81095", "4084226556015831676219/20000", true}, +} + +func TestRatSetString(t *testing.T) { + for i, test := range setStringTests { + x, ok := new(Rat).SetString(test.in) + + if ok != test.ok || ok && x.RatString() != test.out { + t.Errorf("#%d got %s want %s", i, x.RatString(), test.out) + } + } +} + +func TestRatScan(t *testing.T) { + var buf bytes.Buffer + for i, test := range setStringTests { + x := new(Rat) + buf.Reset() + buf.WriteString(test.in) + + _, err := fmt.Fscanf(&buf, "%v", x) + if err == nil != test.ok { + if test.ok { + t.Errorf("#%d error: %s", i, err.String()) + } else { + t.Errorf("#%d expected error", i) + } + continue + } + if err == nil && x.RatString() != test.out { + t.Errorf("#%d got %s want %s", i, x.RatString(), test.out) + } + } +} + +var floatStringTests = []struct { + in string + prec int + out string +}{ + {"0", 0, "0"}, + {"0", 4, "0.0000"}, + {"1", 0, "1"}, + {"1", 2, "1.00"}, + {"-1", 0, "-1"}, + {".25", 2, "0.25"}, + {".25", 1, "0.3"}, + {".25", 3, "0.250"}, + {"-1/3", 3, "-0.333"}, + {"-2/3", 4, "-0.6667"}, + {"0.96", 1, "1.0"}, + {"0.999", 2, "1.00"}, + {"0.9", 0, "1"}, + {".25", -1, "0"}, + {".55", -1, "1"}, +} + +func TestFloatString(t *testing.T) { + for i, test := range floatStringTests { + x, _ := new(Rat).SetString(test.in) + + if x.FloatString(test.prec) != test.out { + t.Errorf("#%d got %s want %s", i, x.FloatString(test.prec), test.out) + } + } +} + +func TestRatSign(t *testing.T) { + zero := NewRat(0, 1) + for _, a := range setStringTests { + var x Rat + x.SetString(a.in) + s := x.Sign() + e := x.Cmp(zero) + if s != e { + t.Errorf("got %d; want %d for z = %v", s, e, &x) + } + } +} + +var ratCmpTests = []struct { + rat1, rat2 string + out int +}{ + {"0", "0/1", 0}, + {"1/1", "1", 0}, + {"-1", "-2/2", 0}, + {"1", "0", 1}, + {"0/1", "1/1", -1}, + {"-5/1434770811533343057144", "-5/1434770811533343057145", -1}, + {"49832350382626108453/8964749413", "49832350382626108454/8964749413", -1}, + {"-37414950961700930/7204075375675961", "37414950961700930/7204075375675961", -1}, + {"37414950961700930/7204075375675961", "74829901923401860/14408150751351922", 0}, +} + +func TestRatCmp(t *testing.T) { + for i, test := range ratCmpTests { + x, _ := new(Rat).SetString(test.rat1) + y, _ := new(Rat).SetString(test.rat2) + + out := x.Cmp(y) + if out != test.out { + t.Errorf("#%d got out = %v; want %v", i, out, test.out) + } + } +} + +func TestIsInt(t *testing.T) { + one := NewInt(1) + for _, a := range setStringTests { + var x Rat + x.SetString(a.in) + i := x.IsInt() + e := x.Denom().Cmp(one) == 0 + if i != e { + t.Errorf("got %v; want %v for z = %v", i, e, &x) + } + } +} + +func TestRatAbs(t *testing.T) { + zero := NewRat(0, 1) + for _, a := range setStringTests { + var z Rat + z.SetString(a.in) + var e Rat + e.Set(&z) + if e.Cmp(zero) < 0 { + e.Sub(zero, &e) + } + z.Abs(&z) + if z.Cmp(&e) != 0 { + t.Errorf("got z = %v; want %v", &z, &e) + } + } +} + +type ratBinFun func(z, x, y *Rat) *Rat +type ratBinArg struct { + x, y, z string +} + +func testRatBin(t *testing.T, i int, name string, f ratBinFun, a ratBinArg) { + x, _ := NewRat(0, 1).SetString(a.x) + y, _ := NewRat(0, 1).SetString(a.y) + z, _ := NewRat(0, 1).SetString(a.z) + out := f(NewRat(0, 1), x, y) + + if out.Cmp(z) != 0 { + t.Errorf("%s #%d got %s want %s", name, i, out, z) + } +} + +var ratBinTests = []struct { + x, y string + sum, prod string +}{ + {"0", "0", "0", "0"}, + {"0", "1", "1", "0"}, + {"-1", "0", "-1", "0"}, + {"-1", "1", "0", "-1"}, + {"1", "1", "2", "1"}, + {"1/2", "1/2", "1", "1/4"}, + {"1/4", "1/3", "7/12", "1/12"}, + {"2/5", "-14/3", "-64/15", "-28/15"}, + {"4707/49292519774798173060", "-3367/70976135186689855734", "84058377121001851123459/1749296273614329067191168098769082663020", "-1760941/388732505247628681598037355282018369560"}, + {"-61204110018146728334/3", "-31052192278051565633/2", "-215564796870448153567/6", "950260896245257153059642991192710872711/3"}, + {"-854857841473707320655/4237645934602118692642972629634714039", "-18/31750379913563777419", "-27/133467566250814981", "15387441146526731771790/134546868362786310073779084329032722548987800600710485341"}, + {"618575745270541348005638912139/19198433543745179392300736", "-19948846211000086/637313996471", "27674141753240653/30123979153216", "-6169936206128396568797607742807090270137721977/6117715203873571641674006593837351328"}, + {"-3/26206484091896184128", "5/2848423294177090248", "15310893822118706237/9330894968229805033368778458685147968", "-5/24882386581946146755650075889827061248"}, + {"26946729/330400702820", "41563965/225583428284", "1238218672302860271/4658307703098666660055", "224002580204097/14906584649915733312176"}, + {"-8259900599013409474/7", "-84829337473700364773/56707961321161574960", "-468402123685491748914621885145127724451/396955729248131024720", "350340947706464153265156004876107029701/198477864624065512360"}, + {"575775209696864/1320203974639986246357", "29/712593081308", "410331716733912717985762465/940768218243776489278275419794956", "808/45524274987585732633"}, + {"1786597389946320496771/2066653520653241", "6269770/1992362624741777", "3559549865190272133656109052308126637/4117523232840525481453983149257", "8967230/3296219033"}, + {"-36459180403360509753/32150500941194292113930", "9381566963714/9633539", "301622077145533298008420642898530153/309723104686531919656937098270", "-3784609207827/3426986245"}, +} + +func TestRatBin(t *testing.T) { + for i, test := range ratBinTests { + arg := ratBinArg{test.x, test.y, test.sum} + testRatBin(t, i, "Add", (*Rat).Add, arg) + + arg = ratBinArg{test.y, test.x, test.sum} + testRatBin(t, i, "Add symmetric", (*Rat).Add, arg) + + arg = ratBinArg{test.sum, test.x, test.y} + testRatBin(t, i, "Sub", (*Rat).Sub, arg) + + arg = ratBinArg{test.sum, test.y, test.x} + testRatBin(t, i, "Sub symmetric", (*Rat).Sub, arg) + + arg = ratBinArg{test.x, test.y, test.prod} + testRatBin(t, i, "Mul", (*Rat).Mul, arg) + + arg = ratBinArg{test.y, test.x, test.prod} + testRatBin(t, i, "Mul symmetric", (*Rat).Mul, arg) + + if test.x != "0" { + arg = ratBinArg{test.prod, test.x, test.y} + testRatBin(t, i, "Quo", (*Rat).Quo, arg) + } + + if test.y != "0" { + arg = ratBinArg{test.prod, test.y, test.x} + testRatBin(t, i, "Quo symmetric", (*Rat).Quo, arg) + } + } +} + +func TestIssue820(t *testing.T) { + x := NewRat(3, 1) + y := NewRat(2, 1) + z := y.Quo(x, y) + q := NewRat(3, 2) + if z.Cmp(q) != 0 { + t.Errorf("got %s want %s", z, q) + } + + y = NewRat(3, 1) + x = NewRat(2, 1) + z = y.Quo(x, y) + q = NewRat(2, 3) + if z.Cmp(q) != 0 { + t.Errorf("got %s want %s", z, q) + } + + x = NewRat(3, 1) + z = x.Quo(x, x) + q = NewRat(3, 3) + if z.Cmp(q) != 0 { + t.Errorf("got %s want %s", z, q) + } +} + +var setFrac64Tests = []struct { + a, b int64 + out string +}{ + {0, 1, "0"}, + {0, -1, "0"}, + {1, 1, "1"}, + {-1, 1, "-1"}, + {1, -1, "-1"}, + {-1, -1, "1"}, + {-9223372036854775808, -9223372036854775808, "1"}, +} + +func TestRatSetFrac64Rat(t *testing.T) { + for i, test := range setFrac64Tests { + x := new(Rat).SetFrac64(test.a, test.b) + if x.RatString() != test.out { + t.Errorf("#%d got %s want %s", i, x.RatString(), test.out) + } + } +} + +func TestRatGobEncoding(t *testing.T) { + var medium bytes.Buffer + enc := gob.NewEncoder(&medium) + dec := gob.NewDecoder(&medium) + for i, test := range gobEncodingTests { + for j := 0; j < 4; j++ { + medium.Reset() // empty buffer for each test case (in case of failures) + stest := test + if j&1 != 0 { + // negative numbers + stest = "-" + test + } + if j%2 != 0 { + // fractions + stest = stest + "." + test + } + var tx Rat + tx.SetString(stest) + if err := enc.Encode(&tx); err != nil { + t.Errorf("#%d%c: encoding failed: %s", i, 'a'+j, err) + } + var rx Rat + if err := dec.Decode(&rx); err != nil { + t.Errorf("#%d%c: decoding failed: %s", i, 'a'+j, err) + } + if rx.Cmp(&tx) != 0 { + t.Errorf("#%d%c: transmission failed: got %s want %s", i, 'a'+j, &rx, &tx) + } + } + } +} diff --git a/src/pkg/bufio/Makefile b/src/pkg/bufio/Makefile new file mode 100644 index 000000000..85430e8e8 --- /dev/null +++ b/src/pkg/bufio/Makefile @@ -0,0 +1,11 @@ +# Copyright 2009 The Go Authors. All rights reserved. +# Use of this source code is governed by a BSD-style +# license that can be found in the LICENSE file. + +include ../../Make.inc + +TARG=bufio +GOFILES=\ + bufio.go\ + +include ../../Make.pkg diff --git a/src/pkg/bufio/bufio.go b/src/pkg/bufio/bufio.go new file mode 100644 index 000000000..727ebfdbb --- /dev/null +++ b/src/pkg/bufio/bufio.go @@ -0,0 +1,549 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Package bufio implements buffered I/O. It wraps an io.Reader or io.Writer +// object, creating another object (Reader or Writer) that also implements +// the interface but provides buffering and some help for textual I/O. +package bufio + +import ( + "bytes" + "io" + "os" + "strconv" + "utf8" +) + +const ( + defaultBufSize = 4096 +) + +// Errors introduced by this package. +type Error struct { + ErrorString string +} + +func (err *Error) String() string { return err.ErrorString } + +var ( + ErrInvalidUnreadByte os.Error = &Error{"bufio: invalid use of UnreadByte"} + ErrInvalidUnreadRune os.Error = &Error{"bufio: invalid use of UnreadRune"} + ErrBufferFull os.Error = &Error{"bufio: buffer full"} + ErrNegativeCount os.Error = &Error{"bufio: negative count"} + errInternal os.Error = &Error{"bufio: internal error"} +) + +// BufSizeError is the error representing an invalid buffer size. +type BufSizeError int + +func (b BufSizeError) String() string { + return "bufio: bad buffer size " + strconv.Itoa(int(b)) +} + +// Buffered input. + +// Reader implements buffering for an io.Reader object. +type Reader struct { + buf []byte + rd io.Reader + r, w int + err os.Error + lastByte int + lastRuneSize int +} + +// NewReaderSize creates a new Reader whose buffer has the specified size, +// which must be greater than zero. If the argument io.Reader is already a +// Reader with large enough size, it returns the underlying Reader. +// It returns the Reader and any error. +func NewReaderSize(rd io.Reader, size int) (*Reader, os.Error) { + if size <= 0 { + return nil, BufSizeError(size) + } + // Is it already a Reader? + b, ok := rd.(*Reader) + if ok && len(b.buf) >= size { + return b, nil + } + b = new(Reader) + b.buf = make([]byte, size) + b.rd = rd + b.lastByte = -1 + b.lastRuneSize = -1 + return b, nil +} + +// NewReader returns a new Reader whose buffer has the default size. +func NewReader(rd io.Reader) *Reader { + b, err := NewReaderSize(rd, defaultBufSize) + if err != nil { + // cannot happen - defaultBufSize is a valid size + panic(err) + } + return b +} + +// fill reads a new chunk into the buffer. +func (b *Reader) fill() { + // Slide existing data to beginning. + if b.r > 0 { + copy(b.buf, b.buf[b.r:b.w]) + b.w -= b.r + b.r = 0 + } + + // Read new data. + n, e := b.rd.Read(b.buf[b.w:]) + b.w += n + if e != nil { + b.err = e + } +} + +func (b *Reader) readErr() os.Error { + err := b.err + b.err = nil + return err +} + +// Peek returns the next n bytes without advancing the reader. The bytes stop +// being valid at the next read call. If Peek returns fewer than n bytes, it +// also returns an error explaining why the read is short. The error is +// ErrBufferFull if n is larger than b's buffer size. +func (b *Reader) Peek(n int) ([]byte, os.Error) { + if n < 0 { + return nil, ErrNegativeCount + } + if n > len(b.buf) { + return nil, ErrBufferFull + } + for b.w-b.r < n && b.err == nil { + b.fill() + } + m := b.w - b.r + if m > n { + m = n + } + err := b.readErr() + if m < n && err == nil { + err = ErrBufferFull + } + return b.buf[b.r : b.r+m], err +} + +// Read reads data into p. +// It returns the number of bytes read into p. +// It calls Read at most once on the underlying Reader, +// hence n may be less than len(p). +// At EOF, the count will be zero and err will be os.EOF. +func (b *Reader) Read(p []byte) (n int, err os.Error) { + n = len(p) + if n == 0 { + return 0, b.readErr() + } + if b.w == b.r { + if b.err != nil { + return 0, b.readErr() + } + if len(p) >= len(b.buf) { + // Large read, empty buffer. + // Read directly into p to avoid copy. + n, b.err = b.rd.Read(p) + if n > 0 { + b.lastByte = int(p[n-1]) + b.lastRuneSize = -1 + } + return n, b.readErr() + } + b.fill() + if b.w == b.r { + return 0, b.readErr() + } + } + + if n > b.w-b.r { + n = b.w - b.r + } + copy(p[0:n], b.buf[b.r:]) + b.r += n + b.lastByte = int(b.buf[b.r-1]) + b.lastRuneSize = -1 + return n, nil +} + +// ReadByte reads and returns a single byte. +// If no byte is available, returns an error. +func (b *Reader) ReadByte() (c byte, err os.Error) { + b.lastRuneSize = -1 + for b.w == b.r { + if b.err != nil { + return 0, b.readErr() + } + b.fill() + } + c = b.buf[b.r] + b.r++ + b.lastByte = int(c) + return c, nil +} + +// UnreadByte unreads the last byte. Only the most recently read byte can be unread. +func (b *Reader) UnreadByte() os.Error { + b.lastRuneSize = -1 + if b.r == b.w && b.lastByte >= 0 { + b.w = 1 + b.r = 0 + b.buf[0] = byte(b.lastByte) + b.lastByte = -1 + return nil + } + if b.r <= 0 { + return ErrInvalidUnreadByte + } + b.r-- + b.lastByte = -1 + return nil +} + +// ReadRune reads a single UTF-8 encoded Unicode character and returns the +// rune and its size in bytes. +func (b *Reader) ReadRune() (rune int, size int, err os.Error) { + for b.r+utf8.UTFMax > b.w && !utf8.FullRune(b.buf[b.r:b.w]) && b.err == nil { + b.fill() + } + b.lastRuneSize = -1 + if b.r == b.w { + return 0, 0, b.readErr() + } + rune, size = int(b.buf[b.r]), 1 + if rune >= 0x80 { + rune, size = utf8.DecodeRune(b.buf[b.r:b.w]) + } + b.r += size + b.lastByte = int(b.buf[b.r-1]) + b.lastRuneSize = size + return rune, size, nil +} + +// UnreadRune unreads the last rune. If the most recent read operation on +// the buffer was not a ReadRune, UnreadRune returns an error. (In this +// regard it is stricter than UnreadByte, which will unread the last byte +// from any read operation.) +func (b *Reader) UnreadRune() os.Error { + if b.lastRuneSize < 0 || b.r == 0 { + return ErrInvalidUnreadRune + } + b.r -= b.lastRuneSize + b.lastByte = -1 + b.lastRuneSize = -1 + return nil +} + +// Buffered returns the number of bytes that can be read from the current buffer. +func (b *Reader) Buffered() int { return b.w - b.r } + +// ReadSlice reads until the first occurrence of delim in the input, +// returning a slice pointing at the bytes in the buffer. +// The bytes stop being valid at the next read call. +// If ReadSlice encounters an error before finding a delimiter, +// it returns all the data in the buffer and the error itself (often os.EOF). +// ReadSlice fails with error ErrBufferFull if the buffer fills without a delim. +// Because the data returned from ReadSlice will be overwritten +// by the next I/O operation, most clients should use +// ReadBytes or ReadString instead. +// ReadSlice returns err != nil if and only if line does not end in delim. +func (b *Reader) ReadSlice(delim byte) (line []byte, err os.Error) { + // Look in buffer. + if i := bytes.IndexByte(b.buf[b.r:b.w], delim); i >= 0 { + line1 := b.buf[b.r : b.r+i+1] + b.r += i + 1 + return line1, nil + } + + // Read more into buffer, until buffer fills or we find delim. + for { + if b.err != nil { + line := b.buf[b.r:b.w] + b.r = b.w + return line, b.readErr() + } + + n := b.Buffered() + b.fill() + + // Search new part of buffer + if i := bytes.IndexByte(b.buf[n:b.w], delim); i >= 0 { + line := b.buf[0 : n+i+1] + b.r = n + i + 1 + return line, nil + } + + // Buffer is full? + if b.Buffered() >= len(b.buf) { + b.r = b.w + return b.buf, ErrBufferFull + } + } + panic("not reached") +} + +// ReadLine tries to return a single line, not including the end-of-line bytes. +// If the line was too long for the buffer then isPrefix is set and the +// beginning of the line is returned. The rest of the line will be returned +// from future calls. isPrefix will be false when returning the last fragment +// of the line. The returned buffer is only valid until the next call to +// ReadLine. ReadLine either returns a non-nil line or it returns an error, +// never both. +func (b *Reader) ReadLine() (line []byte, isPrefix bool, err os.Error) { + line, err = b.ReadSlice('\n') + if err == ErrBufferFull { + return line, true, nil + } + + if len(line) == 0 { + return + } + err = nil + + if line[len(line)-1] == '\n' { + line = line[:len(line)-1] + } + if len(line) > 0 && line[len(line)-1] == '\r' { + line = line[:len(line)-1] + } + return +} + +// ReadBytes reads until the first occurrence of delim in the input, +// returning a slice containing the data up to and including the delimiter. +// If ReadBytes encounters an error before finding a delimiter, +// it returns the data read before the error and the error itself (often os.EOF). +// ReadBytes returns err != nil if and only if the returned data does not end in +// delim. +func (b *Reader) ReadBytes(delim byte) (line []byte, err os.Error) { + // Use ReadSlice to look for array, + // accumulating full buffers. + var frag []byte + var full [][]byte + err = nil + + for { + var e os.Error + frag, e = b.ReadSlice(delim) + if e == nil { // got final fragment + break + } + if e != ErrBufferFull { // unexpected error + err = e + break + } + + // Make a copy of the buffer. + buf := make([]byte, len(frag)) + copy(buf, frag) + full = append(full, buf) + } + + // Allocate new buffer to hold the full pieces and the fragment. + n := 0 + for i := range full { + n += len(full[i]) + } + n += len(frag) + + // Copy full pieces and fragment in. + buf := make([]byte, n) + n = 0 + for i := range full { + n += copy(buf[n:], full[i]) + } + copy(buf[n:], frag) + return buf, err +} + +// ReadString reads until the first occurrence of delim in the input, +// returning a string containing the data up to and including the delimiter. +// If ReadString encounters an error before finding a delimiter, +// it returns the data read before the error and the error itself (often os.EOF). +// ReadString returns err != nil if and only if the returned data does not end in +// delim. +func (b *Reader) ReadString(delim byte) (line string, err os.Error) { + bytes, e := b.ReadBytes(delim) + return string(bytes), e +} + +// buffered output + +// Writer implements buffering for an io.Writer object. +type Writer struct { + err os.Error + buf []byte + n int + wr io.Writer +} + +// NewWriterSize creates a new Writer whose buffer has the specified size, +// which must be greater than zero. If the argument io.Writer is already a +// Writer with large enough size, it returns the underlying Writer. +// It returns the Writer and any error. +func NewWriterSize(wr io.Writer, size int) (*Writer, os.Error) { + if size <= 0 { + return nil, BufSizeError(size) + } + // Is it already a Writer? + b, ok := wr.(*Writer) + if ok && len(b.buf) >= size { + return b, nil + } + b = new(Writer) + b.buf = make([]byte, size) + b.wr = wr + return b, nil +} + +// NewWriter returns a new Writer whose buffer has the default size. +func NewWriter(wr io.Writer) *Writer { + b, err := NewWriterSize(wr, defaultBufSize) + if err != nil { + // cannot happen - defaultBufSize is valid size + panic(err) + } + return b +} + +// Flush writes any buffered data to the underlying io.Writer. +func (b *Writer) Flush() os.Error { + if b.err != nil { + return b.err + } + if b.n == 0 { + return nil + } + n, e := b.wr.Write(b.buf[0:b.n]) + if n < b.n && e == nil { + e = io.ErrShortWrite + } + if e != nil { + if n > 0 && n < b.n { + copy(b.buf[0:b.n-n], b.buf[n:b.n]) + } + b.n -= n + b.err = e + return e + } + b.n = 0 + return nil +} + +// Available returns how many bytes are unused in the buffer. +func (b *Writer) Available() int { return len(b.buf) - b.n } + +// Buffered returns the number of bytes that have been written into the current buffer. +func (b *Writer) Buffered() int { return b.n } + +// Write writes the contents of p into the buffer. +// It returns the number of bytes written. +// If nn < len(p), it also returns an error explaining +// why the write is short. +func (b *Writer) Write(p []byte) (nn int, err os.Error) { + for len(p) > b.Available() && b.err == nil { + var n int + if b.Buffered() == 0 { + // Large write, empty buffer. + // Write directly from p to avoid copy. + n, b.err = b.wr.Write(p) + } else { + n = copy(b.buf[b.n:], p) + b.n += n + b.Flush() + } + nn += n + p = p[n:] + } + if b.err != nil { + return nn, b.err + } + n := copy(b.buf[b.n:], p) + b.n += n + nn += n + return nn, nil +} + +// WriteByte writes a single byte. +func (b *Writer) WriteByte(c byte) os.Error { + if b.err != nil { + return b.err + } + if b.Available() <= 0 && b.Flush() != nil { + return b.err + } + b.buf[b.n] = c + b.n++ + return nil +} + +// WriteRune writes a single Unicode code point, returning +// the number of bytes written and any error. +func (b *Writer) WriteRune(rune int) (size int, err os.Error) { + if rune < utf8.RuneSelf { + err = b.WriteByte(byte(rune)) + if err != nil { + return 0, err + } + return 1, nil + } + if b.err != nil { + return 0, b.err + } + n := b.Available() + if n < utf8.UTFMax { + if b.Flush(); b.err != nil { + return 0, b.err + } + n = b.Available() + if n < utf8.UTFMax { + // Can only happen if buffer is silly small. + return b.WriteString(string(rune)) + } + } + size = utf8.EncodeRune(b.buf[b.n:], rune) + b.n += size + return size, nil +} + +// WriteString writes a string. +// It returns the number of bytes written. +// If the count is less than len(s), it also returns an error explaining +// why the write is short. +func (b *Writer) WriteString(s string) (int, os.Error) { + nn := 0 + for len(s) > b.Available() && b.err == nil { + n := copy(b.buf[b.n:], s) + b.n += n + nn += n + s = s[n:] + b.Flush() + } + if b.err != nil { + return nn, b.err + } + n := copy(b.buf[b.n:], s) + b.n += n + nn += n + return nn, nil +} + +// buffered input and output + +// ReadWriter stores pointers to a Reader and a Writer. +// It implements io.ReadWriter. +type ReadWriter struct { + *Reader + *Writer +} + +// NewReadWriter allocates a new ReadWriter that dispatches to r and w. +func NewReadWriter(r *Reader, w *Writer) *ReadWriter { + return &ReadWriter{r, w} +} diff --git a/src/pkg/bufio/bufio_test.go b/src/pkg/bufio/bufio_test.go new file mode 100644 index 000000000..82c73d36a --- /dev/null +++ b/src/pkg/bufio/bufio_test.go @@ -0,0 +1,699 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package bufio_test + +import ( + . "bufio" + "bytes" + "fmt" + "io" + "io/ioutil" + "os" + "strings" + "testing" + "testing/iotest" + "utf8" +) + +// Reads from a reader and rot13s the result. +type rot13Reader struct { + r io.Reader +} + +func newRot13Reader(r io.Reader) *rot13Reader { + r13 := new(rot13Reader) + r13.r = r + return r13 +} + +func (r13 *rot13Reader) Read(p []byte) (int, os.Error) { + n, e := r13.r.Read(p) + if e != nil { + return n, e + } + for i := 0; i < n; i++ { + c := p[i] | 0x20 // lowercase byte + if 'a' <= c && c <= 'm' { + p[i] += 13 + } else if 'n' <= c && c <= 'z' { + p[i] -= 13 + } + } + return n, nil +} + +// Call ReadByte to accumulate the text of a file +func readBytes(buf *Reader) string { + var b [1000]byte + nb := 0 + for { + c, e := buf.ReadByte() + if e == os.EOF { + break + } + if e == nil { + b[nb] = c + nb++ + } else if e != iotest.ErrTimeout { + panic("Data: " + e.String()) + } + } + return string(b[0:nb]) +} + +func TestReaderSimple(t *testing.T) { + data := "hello world" + b := NewReader(bytes.NewBufferString(data)) + if s := readBytes(b); s != "hello world" { + t.Errorf("simple hello world test failed: got %q", s) + } + + b = NewReader(newRot13Reader(bytes.NewBufferString(data))) + if s := readBytes(b); s != "uryyb jbeyq" { + t.Errorf("rot13 hello world test failed: got %q", s) + } +} + +type readMaker struct { + name string + fn func(io.Reader) io.Reader +} + +var readMakers = []readMaker{ + {"full", func(r io.Reader) io.Reader { return r }}, + {"byte", iotest.OneByteReader}, + {"half", iotest.HalfReader}, + {"data+err", iotest.DataErrReader}, + {"timeout", iotest.TimeoutReader}, +} + +// Call ReadString (which ends up calling everything else) +// to accumulate the text of a file. +func readLines(b *Reader) string { + s := "" + for { + s1, e := b.ReadString('\n') + if e == os.EOF { + break + } + if e != nil && e != iotest.ErrTimeout { + panic("GetLines: " + e.String()) + } + s += s1 + } + return s +} + +// Call Read to accumulate the text of a file +func reads(buf *Reader, m int) string { + var b [1000]byte + nb := 0 + for { + n, e := buf.Read(b[nb : nb+m]) + nb += n + if e == os.EOF { + break + } + } + return string(b[0:nb]) +} + +type bufReader struct { + name string + fn func(*Reader) string +} + +var bufreaders = []bufReader{ + {"1", func(b *Reader) string { return reads(b, 1) }}, + {"2", func(b *Reader) string { return reads(b, 2) }}, + {"3", func(b *Reader) string { return reads(b, 3) }}, + {"4", func(b *Reader) string { return reads(b, 4) }}, + {"5", func(b *Reader) string { return reads(b, 5) }}, + {"7", func(b *Reader) string { return reads(b, 7) }}, + {"bytes", readBytes}, + {"lines", readLines}, +} + +var bufsizes = []int{ + 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, + 23, 32, 46, 64, 93, 128, 1024, 4096, +} + +func TestReader(t *testing.T) { + var texts [31]string + str := "" + all := "" + for i := 0; i < len(texts)-1; i++ { + texts[i] = str + "\n" + all += texts[i] + str += string(i%26 + 'a') + } + texts[len(texts)-1] = all + + for h := 0; h < len(texts); h++ { + text := texts[h] + for i := 0; i < len(readMakers); i++ { + for j := 0; j < len(bufreaders); j++ { + for k := 0; k < len(bufsizes); k++ { + readmaker := readMakers[i] + bufreader := bufreaders[j] + bufsize := bufsizes[k] + read := readmaker.fn(bytes.NewBufferString(text)) + buf, _ := NewReaderSize(read, bufsize) + s := bufreader.fn(buf) + if s != text { + t.Errorf("reader=%s fn=%s bufsize=%d want=%q got=%q", + readmaker.name, bufreader.name, bufsize, text, s) + } + } + } + } + } +} + +// A StringReader delivers its data one string segment at a time via Read. +type StringReader struct { + data []string + step int +} + +func (r *StringReader) Read(p []byte) (n int, err os.Error) { + if r.step < len(r.data) { + s := r.data[r.step] + n = copy(p, s) + r.step++ + } else { + err = os.EOF + } + return +} + +func readRuneSegments(t *testing.T, segments []string) { + got := "" + want := strings.Join(segments, "") + r := NewReader(&StringReader{data: segments}) + for { + rune, _, err := r.ReadRune() + if err != nil { + if err != os.EOF { + return + } + break + } + got += string(rune) + } + if got != want { + t.Errorf("segments=%v got=%s want=%s", segments, got, want) + } +} + +var segmentList = [][]string{ + {}, + {""}, + {"日", "本語"}, + {"\u65e5", "\u672c", "\u8a9e"}, + {"\U000065e5", "\U0000672c", "\U00008a9e"}, + {"\xe6", "\x97\xa5\xe6", "\x9c\xac\xe8\xaa\x9e"}, + {"Hello", ", ", "World", "!"}, + {"Hello", ", ", "", "World", "!"}, +} + +func TestReadRune(t *testing.T) { + for _, s := range segmentList { + readRuneSegments(t, s) + } +} + +func TestUnreadRune(t *testing.T) { + got := "" + segments := []string{"Hello, world:", "日本語"} + data := strings.Join(segments, "") + r := NewReader(&StringReader{data: segments}) + // Normal execution. + for { + rune, _, err := r.ReadRune() + if err != nil { + if err != os.EOF { + t.Error("unexpected EOF") + } + break + } + got += string(rune) + // Put it back and read it again + if err = r.UnreadRune(); err != nil { + t.Error("unexpected error on UnreadRune:", err) + } + rune1, _, err := r.ReadRune() + if err != nil { + t.Error("unexpected error reading after unreading:", err) + } + if rune != rune1 { + t.Errorf("incorrect rune after unread: got %c wanted %c", rune1, rune) + } + } + if got != data { + t.Errorf("want=%q got=%q", data, got) + } +} + +// Test that UnreadRune fails if the preceding operation was not a ReadRune. +func TestUnreadRuneError(t *testing.T) { + buf := make([]byte, 3) // All runes in this test are 3 bytes long + r := NewReader(&StringReader{data: []string{"日本語日本語日本語"}}) + if r.UnreadRune() == nil { + t.Error("expected error on UnreadRune from fresh buffer") + } + _, _, err := r.ReadRune() + if err != nil { + t.Error("unexpected error on ReadRune (1):", err) + } + if err = r.UnreadRune(); err != nil { + t.Error("unexpected error on UnreadRune (1):", err) + } + if r.UnreadRune() == nil { + t.Error("expected error after UnreadRune (1)") + } + // Test error after Read. + _, _, err = r.ReadRune() // reset state + if err != nil { + t.Error("unexpected error on ReadRune (2):", err) + } + _, err = r.Read(buf) + if err != nil { + t.Error("unexpected error on Read (2):", err) + } + if r.UnreadRune() == nil { + t.Error("expected error after Read (2)") + } + // Test error after ReadByte. + _, _, err = r.ReadRune() // reset state + if err != nil { + t.Error("unexpected error on ReadRune (2):", err) + } + for _ = range buf { + _, err = r.ReadByte() + if err != nil { + t.Error("unexpected error on ReadByte (2):", err) + } + } + if r.UnreadRune() == nil { + t.Error("expected error after ReadByte") + } + // Test error after UnreadByte. + _, _, err = r.ReadRune() // reset state + if err != nil { + t.Error("unexpected error on ReadRune (3):", err) + } + _, err = r.ReadByte() + if err != nil { + t.Error("unexpected error on ReadByte (3):", err) + } + err = r.UnreadByte() + if err != nil { + t.Error("unexpected error on UnreadByte (3):", err) + } + if r.UnreadRune() == nil { + t.Error("expected error after UnreadByte (3)") + } +} + +func TestUnreadRuneAtEOF(t *testing.T) { + // UnreadRune/ReadRune should error at EOF (was a bug; used to panic) + r := NewReader(strings.NewReader("x")) + r.ReadRune() + r.ReadRune() + r.UnreadRune() + _, _, err := r.ReadRune() + if err == nil { + t.Error("expected error at EOF") + } else if err != os.EOF { + t.Error("expected EOF; got", err) + } +} + +func TestReadWriteRune(t *testing.T) { + const NRune = 1000 + byteBuf := new(bytes.Buffer) + w := NewWriter(byteBuf) + // Write the runes out using WriteRune + buf := make([]byte, utf8.UTFMax) + for rune := 0; rune < NRune; rune++ { + size := utf8.EncodeRune(buf, rune) + nbytes, err := w.WriteRune(rune) + if err != nil { + t.Fatalf("WriteRune(0x%x) error: %s", rune, err) + } + if nbytes != size { + t.Fatalf("WriteRune(0x%x) expected %d, got %d", rune, size, nbytes) + } + } + w.Flush() + + r := NewReader(byteBuf) + // Read them back with ReadRune + for rune := 0; rune < NRune; rune++ { + size := utf8.EncodeRune(buf, rune) + nr, nbytes, err := r.ReadRune() + if nr != rune || nbytes != size || err != nil { + t.Fatalf("ReadRune(0x%x) got 0x%x,%d not 0x%x,%d (err=%s)", r, nr, nbytes, r, size, err) + } + } +} + +func TestWriter(t *testing.T) { + var data [8192]byte + + for i := 0; i < len(data); i++ { + data[i] = byte(' ' + i%('~'-' ')) + } + w := new(bytes.Buffer) + for i := 0; i < len(bufsizes); i++ { + for j := 0; j < len(bufsizes); j++ { + nwrite := bufsizes[i] + bs := bufsizes[j] + + // Write nwrite bytes using buffer size bs. + // Check that the right amount makes it out + // and that the data is correct. + + w.Reset() + buf, e := NewWriterSize(w, bs) + context := fmt.Sprintf("nwrite=%d bufsize=%d", nwrite, bs) + if e != nil { + t.Errorf("%s: NewWriterSize %d: %v", context, bs, e) + continue + } + n, e1 := buf.Write(data[0:nwrite]) + if e1 != nil || n != nwrite { + t.Errorf("%s: buf.Write %d = %d, %v", context, nwrite, n, e1) + continue + } + if e = buf.Flush(); e != nil { + t.Errorf("%s: buf.Flush = %v", context, e) + } + + written := w.Bytes() + if len(written) != nwrite { + t.Errorf("%s: %d bytes written", context, len(written)) + } + for l := 0; l < len(written); l++ { + if written[i] != data[i] { + t.Errorf("wrong bytes written") + t.Errorf("want=%q", data[0:len(written)]) + t.Errorf("have=%q", written) + } + } + } + } +} + +// Check that write errors are returned properly. + +type errorWriterTest struct { + n, m int + err os.Error + expect os.Error +} + +func (w errorWriterTest) Write(p []byte) (int, os.Error) { + return len(p) * w.n / w.m, w.err +} + +var errorWriterTests = []errorWriterTest{ + {0, 1, nil, io.ErrShortWrite}, + {1, 2, nil, io.ErrShortWrite}, + {1, 1, nil, nil}, + {0, 1, os.EPIPE, os.EPIPE}, + {1, 2, os.EPIPE, os.EPIPE}, + {1, 1, os.EPIPE, os.EPIPE}, +} + +func TestWriteErrors(t *testing.T) { + for _, w := range errorWriterTests { + buf := NewWriter(w) + _, e := buf.Write([]byte("hello world")) + if e != nil { + t.Errorf("Write hello to %v: %v", w, e) + continue + } + e = buf.Flush() + if e != w.expect { + t.Errorf("Flush %v: got %v, wanted %v", w, e, w.expect) + } + } +} + +func TestNewReaderSizeIdempotent(t *testing.T) { + const BufSize = 1000 + b, err := NewReaderSize(bytes.NewBufferString("hello world"), BufSize) + if err != nil { + t.Error("NewReaderSize create fail", err) + } + // Does it recognize itself? + b1, err2 := NewReaderSize(b, BufSize) + if err2 != nil { + t.Error("NewReaderSize #2 create fail", err2) + } + if b1 != b { + t.Error("NewReaderSize did not detect underlying Reader") + } + // Does it wrap if existing buffer is too small? + b2, err3 := NewReaderSize(b, 2*BufSize) + if err3 != nil { + t.Error("NewReaderSize #3 create fail", err3) + } + if b2 == b { + t.Error("NewReaderSize did not enlarge buffer") + } +} + +func TestNewWriterSizeIdempotent(t *testing.T) { + const BufSize = 1000 + b, err := NewWriterSize(new(bytes.Buffer), BufSize) + if err != nil { + t.Error("NewWriterSize create fail", err) + } + // Does it recognize itself? + b1, err2 := NewWriterSize(b, BufSize) + if err2 != nil { + t.Error("NewWriterSize #2 create fail", err2) + } + if b1 != b { + t.Error("NewWriterSize did not detect underlying Writer") + } + // Does it wrap if existing buffer is too small? + b2, err3 := NewWriterSize(b, 2*BufSize) + if err3 != nil { + t.Error("NewWriterSize #3 create fail", err3) + } + if b2 == b { + t.Error("NewWriterSize did not enlarge buffer") + } +} + +func TestWriteString(t *testing.T) { + const BufSize = 8 + buf := new(bytes.Buffer) + b, err := NewWriterSize(buf, BufSize) + if err != nil { + t.Error("NewWriterSize create fail", err) + } + b.WriteString("0") // easy + b.WriteString("123456") // still easy + b.WriteString("7890") // easy after flush + b.WriteString("abcdefghijklmnopqrstuvwxy") // hard + b.WriteString("z") + if err := b.Flush(); err != nil { + t.Error("WriteString", err) + } + s := "01234567890abcdefghijklmnopqrstuvwxyz" + if string(buf.Bytes()) != s { + t.Errorf("WriteString wants %q gets %q", s, string(buf.Bytes())) + } +} + +func TestBufferFull(t *testing.T) { + buf, _ := NewReaderSize(strings.NewReader("hello, world"), 5) + line, err := buf.ReadSlice(',') + if string(line) != "hello" || err != ErrBufferFull { + t.Errorf("first ReadSlice(,) = %q, %v", line, err) + } + line, err = buf.ReadSlice(',') + if string(line) != "," || err != nil { + t.Errorf("second ReadSlice(,) = %q, %v", line, err) + } +} + +func TestPeek(t *testing.T) { + p := make([]byte, 10) + buf, _ := NewReaderSize(strings.NewReader("abcdefghij"), 4) + if s, err := buf.Peek(1); string(s) != "a" || err != nil { + t.Fatalf("want %q got %q, err=%v", "a", string(s), err) + } + if s, err := buf.Peek(4); string(s) != "abcd" || err != nil { + t.Fatalf("want %q got %q, err=%v", "abcd", string(s), err) + } + if _, err := buf.Peek(5); err != ErrBufferFull { + t.Fatalf("want ErrBufFull got %v", err) + } + if _, err := buf.Read(p[0:3]); string(p[0:3]) != "abc" || err != nil { + t.Fatalf("want %q got %q, err=%v", "abc", string(p[0:3]), err) + } + if s, err := buf.Peek(1); string(s) != "d" || err != nil { + t.Fatalf("want %q got %q, err=%v", "d", string(s), err) + } + if s, err := buf.Peek(2); string(s) != "de" || err != nil { + t.Fatalf("want %q got %q, err=%v", "de", string(s), err) + } + if _, err := buf.Read(p[0:3]); string(p[0:3]) != "def" || err != nil { + t.Fatalf("want %q got %q, err=%v", "def", string(p[0:3]), err) + } + if s, err := buf.Peek(4); string(s) != "ghij" || err != nil { + t.Fatalf("want %q got %q, err=%v", "ghij", string(s), err) + } + if _, err := buf.Read(p[0:4]); string(p[0:4]) != "ghij" || err != nil { + t.Fatalf("want %q got %q, err=%v", "ghij", string(p[0:3]), err) + } + if s, err := buf.Peek(0); string(s) != "" || err != nil { + t.Fatalf("want %q got %q, err=%v", "", string(s), err) + } + if _, err := buf.Peek(1); err != os.EOF { + t.Fatalf("want EOF got %v", err) + } +} + +func TestPeekThenUnreadRune(t *testing.T) { + // This sequence used to cause a crash. + r := NewReader(strings.NewReader("x")) + r.ReadRune() + r.Peek(1) + r.UnreadRune() + r.ReadRune() // Used to panic here +} + +var testOutput = []byte("0123456789abcdefghijklmnopqrstuvwxy") +var testInput = []byte("012\n345\n678\n9ab\ncde\nfgh\nijk\nlmn\nopq\nrst\nuvw\nxy") +var testInputrn = []byte("012\r\n345\r\n678\r\n9ab\r\ncde\r\nfgh\r\nijk\r\nlmn\r\nopq\r\nrst\r\nuvw\r\nxy\r\n\n\r\n") + +// TestReader wraps a []byte and returns reads of a specific length. +type testReader struct { + data []byte + stride int +} + +func (t *testReader) Read(buf []byte) (n int, err os.Error) { + n = t.stride + if n > len(t.data) { + n = len(t.data) + } + if n > len(buf) { + n = len(buf) + } + copy(buf, t.data) + t.data = t.data[n:] + if len(t.data) == 0 { + err = os.EOF + } + return +} + +func testReadLine(t *testing.T, input []byte) { + //for stride := 1; stride < len(input); stride++ { + for stride := 1; stride < 2; stride++ { + done := 0 + reader := testReader{input, stride} + l, _ := NewReaderSize(&reader, len(input)+1) + for { + line, isPrefix, err := l.ReadLine() + if len(line) > 0 && err != nil { + t.Errorf("ReadLine returned both data and error: %s", err) + } + if isPrefix { + t.Errorf("ReadLine returned prefix") + } + if err != nil { + if err != os.EOF { + t.Fatalf("Got unknown error: %s", err) + } + break + } + if want := testOutput[done : done+len(line)]; !bytes.Equal(want, line) { + t.Errorf("Bad line at stride %d: want: %x got: %x", stride, want, line) + } + done += len(line) + } + if done != len(testOutput) { + t.Errorf("ReadLine didn't return everything: got: %d, want: %d (stride: %d)", done, len(testOutput), stride) + } + } +} + +func TestReadLine(t *testing.T) { + testReadLine(t, testInput) + testReadLine(t, testInputrn) +} + +func TestLineTooLong(t *testing.T) { + buf := bytes.NewBuffer([]byte("aaabbbcc\n")) + l, _ := NewReaderSize(buf, 3) + line, isPrefix, err := l.ReadLine() + if !isPrefix || !bytes.Equal(line, []byte("aaa")) || err != nil { + t.Errorf("bad result for first line: %x %s", line, err) + } + line, isPrefix, err = l.ReadLine() + if !isPrefix || !bytes.Equal(line, []byte("bbb")) || err != nil { + t.Errorf("bad result for second line: %x", line) + } + line, isPrefix, err = l.ReadLine() + if isPrefix || !bytes.Equal(line, []byte("cc")) || err != nil { + t.Errorf("bad result for third line: %x", line) + } + line, isPrefix, err = l.ReadLine() + if isPrefix || err == nil { + t.Errorf("expected no more lines: %x %s", line, err) + } +} + +func TestReadAfterLines(t *testing.T) { + line1 := "line1" + restData := "line2\nline 3\n" + inbuf := bytes.NewBuffer([]byte(line1 + "\n" + restData)) + outbuf := new(bytes.Buffer) + maxLineLength := len(line1) + len(restData)/2 + l, _ := NewReaderSize(inbuf, maxLineLength) + line, isPrefix, err := l.ReadLine() + if isPrefix || err != nil || string(line) != line1 { + t.Errorf("bad result for first line: isPrefix=%v err=%v line=%q", isPrefix, err, string(line)) + } + n, err := io.Copy(outbuf, l) + if int(n) != len(restData) || err != nil { + t.Errorf("bad result for Read: n=%d err=%v", n, err) + } + if outbuf.String() != restData { + t.Errorf("bad result for Read: got %q; expected %q", outbuf.String(), restData) + } +} + +func TestReadEmptyBuffer(t *testing.T) { + l, _ := NewReaderSize(bytes.NewBuffer(nil), 10) + line, isPrefix, err := l.ReadLine() + if err != os.EOF { + t.Errorf("expected EOF from ReadLine, got '%s' %t %s", line, isPrefix, err) + } +} + +func TestLinesAfterRead(t *testing.T) { + l, _ := NewReaderSize(bytes.NewBuffer([]byte("foo")), 10) + _, err := ioutil.ReadAll(l) + if err != nil { + t.Error(err) + return + } + + line, isPrefix, err := l.ReadLine() + if err != os.EOF { + t.Errorf("expected EOF from ReadLine, got '%s' %t %s", line, isPrefix, err) + } +} diff --git a/src/pkg/builtin/builtin.go b/src/pkg/builtin/builtin.go new file mode 100644 index 000000000..07acce4f7 --- /dev/null +++ b/src/pkg/builtin/builtin.go @@ -0,0 +1,135 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +/* + Package builtin provides documentation for Go's built-in functions. + The functions documented here are not actually in package builtin + but their descriptions here allow godoc to present documentation + for the language's special functions. +*/ +package builtin + +// Type is here for the purposes of documentation only. It is a stand-in +// for any Go type, but represents the same type for any given function +// invocation. +type Type int + +// IntegerType is here for the purposes of documentation only. It is a stand-in +// for any integer type: int, uint, int8 etc. +type IntegerType int + +// FloatType is here for the purposes of documentation only. It is a stand-in +// for either float type: float32 or float64. +type FloatType int + +// ComplexType is here for the purposes of documentation only. It is a +// stand-in for either complex type: complex64 or complex128. +type ComplexType int + +// The append built-in function appends elements to the end of a slice. If +// it has sufficient capacity, the destination is resliced to accommodate the +// new elements. If it does not, a new underlying array will be allocated. +// Append returns the updated slice. It is therefore necessary to store the +// result of append, often in the variable holding the slice itself: +// slice = append(slice, elem1, elem2) +// slice = append(slice, anotherSlice...) +func append(slice []Type, elems ...Type) []Type + +// The copy built-in function copies elements from a source slice into a +// destination slice. (As a special case, it also will copy bytes from a +// string to a slice of bytes.) The source and destination may overlap. Copy +// returns the number of elements copied, which will be the minimum of +// len(src) and len(dst). +func copy(dst, src []Type) int + +// The len built-in function returns the length of v, according to its type: +// Array: the number of elements in v. +// Pointer to array: the number of elements in *v (even if v is nil). +// Slice, or map: the number of elements in v; if v is nil, len(v) is zero. +// String: the number of bytes in v. +// Channel: the number of elements queued (unread) in the channel buffer; +// if v is nil, len(v) is zero. +func len(v Type) int + +// The cap built-in function returns the capacity of v, according to its type: +// Array: the number of elements in v (same as len(v)). +// Pointer to array: the number of elements in *v (same as len(v)). +// Slice: the maximum length the slice can reach when resliced; +// if v is nil, cap(v) is zero. +// Channel: the channel buffer capacity, in units of elements; +// if v is nil, cap(v) is zero. +func cap(v Type) int + +// The make built-in function allocates and initializes an object of type +// slice, map, or chan (only). Like new, the first argument is a type, not a +// value. Unlike new, make's return type is the same as the type of its +// argument, not a pointer to it. The specification of the result depends on +// the type: +// Slice: The size specifies the length. The capacity of the slice is +// equal to its length. A second integer argument may be provided to +// specify a different capacity; it must be no smaller than the +// length, so make([]int, 0, 10) allocates a slice of length 0 and +// capacity 10. +// Map: An initial allocation is made according to the size but the +// resulting map has length 0. The size may be omitted, in which case +// a small starting size is allocated. +// Channel: The channel's buffer is initialized with the specified +// buffer capacity. If zero, or the size is omitted, the channel is +// unbuffered. +func make(Type, size IntegerType) Type + +// The new built-in function allocates memory. The first argument is a type, +// not a value, and the value returned is a pointer to a newly +// allocated zero value of that type. +func new(Type) *Type + +// The complex built-in function constructs a complex value from two +// floating-point values. The real and imaginary parts must be of the same +// size, either float32 or float64 (or assignable to them), and the return +// value will be the corresponding complex type (complex64 for float32, +// complex128 for float64). +func complex(r, i FloatType) ComplexType + +// The real built-in function returns the real part of the complex number c. +// The return value will be floating point type corresponding to the type of c. +func real(c ComplexType) FloatType + +// The imaginary built-in function returns the imaginary part of the complex +// number c. The return value will be floating point type corresponding to +// the type of c. +func imag(c ComplexType) FloatType + +// The close built-in function closes a channel, which must be either +// bidirectional or send-only. It should be executed only by the sender, +// never the receiver, and has the effect of shutting down the channel after +// the last sent value is received. After the last value has been received +// from a closed channel c, any receive from c will succeed without +// blocking, returning the zero value for the channel element. The form +// x, ok := <-c +// will also set ok to false for a closed channel. +func close(c chan<- Type) + +// The panic built-in function stops normal execution of the current +// goroutine. When a function F calls panic, normal execution of F stops +// immediately. Any functions whose execution was deferred by F are run in +// the usual way, and then F returns to its caller. To the caller G, the +// invocation of F then behaves like a call to panic, terminating G's +// execution and running any deferred functions. This continues until all +// functions in the executing goroutine have stopped, in reverse order. At +// that point, the program is terminated and the error condition is reported, +// including the value of the argument to panic. This termination sequence +// is called panicking and can be controlled by the built-in function +// recover. +func panic(v interface{}) + +// The recover built-in function allows a program to manage behavior of a +// panicking goroutine. Executing a call to recover inside a deferred +// function (but not any function called by it) stops the panicking sequence +// by restoring normal execution and retrieves the error value passed to the +// call of panic. If recover is called outside the deferred function it will +// not stop a panicking sequence. In this case, or when the goroutine is not +// panicking, or if the argument supplied to panic was nil, recover returns +// nil. Thus the return value from recover reports whether the goroutine is +// panicking. +func recover() interface{} diff --git a/src/pkg/bytes/Makefile b/src/pkg/bytes/Makefile new file mode 100644 index 000000000..03395c7a4 --- /dev/null +++ b/src/pkg/bytes/Makefile @@ -0,0 +1,16 @@ +# Copyright 2009 The Go Authors. All rights reserved. +# Use of this source code is governed by a BSD-style +# license that can be found in the LICENSE file. + +include ../../Make.inc + +TARG=bytes +GOFILES=\ + buffer.go\ + bytes.go\ + bytes_decl.go\ + +OFILES=\ + asm_$(GOARCH).$O\ + +include ../../Make.pkg diff --git a/src/pkg/bytes/asm_386.s b/src/pkg/bytes/asm_386.s new file mode 100644 index 000000000..f3391740b --- /dev/null +++ b/src/pkg/bytes/asm_386.s @@ -0,0 +1,17 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +TEXT ·IndexByte(SB),7,$0 + MOVL p+0(FP), SI + MOVL len+4(FP), CX + MOVB b+12(FP), AL + MOVL SI, DI + CLD; REPN; SCASB + JZ 3(PC) + MOVL $-1, ret+16(FP) + RET + SUBL SI, DI + SUBL $1, DI + MOVL DI, ret+16(FP) + RET diff --git a/src/pkg/bytes/asm_amd64.s b/src/pkg/bytes/asm_amd64.s new file mode 100644 index 000000000..c6793cbdc --- /dev/null +++ b/src/pkg/bytes/asm_amd64.s @@ -0,0 +1,92 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +TEXT ·IndexByte(SB),7,$0 + MOVQ p+0(FP), SI + MOVL len+8(FP), BX + MOVB b+16(FP), AL + MOVQ SI, DI + + CMPL BX, $16 + JLT small + + // round up to first 16-byte boundary + TESTQ $15, SI + JZ aligned + MOVQ SI, CX + ANDQ $~15, CX + ADDQ $16, CX + + // search the beginning + SUBQ SI, CX + REPN; SCASB + JZ success + +// DI is 16-byte aligned; get ready to search using SSE instructions +aligned: + // round down to last 16-byte boundary + MOVQ BX, R11 + ADDQ SI, R11 + ANDQ $~15, R11 + + // shuffle X0 around so that each byte contains c + MOVD AX, X0 + PUNPCKLBW X0, X0 + PUNPCKLBW X0, X0 + PSHUFL $0, X0, X0 + JMP condition + +sse: + // move the next 16-byte chunk of the buffer into X1 + MOVO (DI), X1 + // compare bytes in X0 to X1 + PCMPEQB X0, X1 + // take the top bit of each byte in X1 and put the result in DX + PMOVMSKB X1, DX + TESTL DX, DX + JNZ ssesuccess + ADDQ $16, DI + +condition: + CMPQ DI, R11 + JLT sse + + // search the end + MOVQ SI, CX + ADDQ BX, CX + SUBQ R11, CX + // if CX == 0, the zero flag will be set and we'll end up + // returning a false success + JZ failure + REPN; SCASB + JZ success + +failure: + MOVL $-1, ret+24(FP) + RET + +// handle for lengths < 16 +small: + MOVL BX, CX + REPN; SCASB + JZ success + MOVL $-1, ret+24(FP) + RET + +// we've found the chunk containing the byte +// now just figure out which specific byte it is +ssesuccess: + // get the index of the least significant set bit + BSFW DX, DX + SUBQ SI, DI + ADDQ DI, DX + MOVL DX, ret+24(FP) + RET + +success: + SUBQ SI, DI + SUBL $1, DI + MOVL DI, ret+24(FP) + RET + diff --git a/src/pkg/bytes/asm_arm.s b/src/pkg/bytes/asm_arm.s new file mode 100644 index 000000000..f32fca136 --- /dev/null +++ b/src/pkg/bytes/asm_arm.s @@ -0,0 +1,8 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// no memchr implementation on arm yet +TEXT ·IndexByte(SB),7,$0 + B ·indexBytePortable(SB) + diff --git a/src/pkg/bytes/buffer.go b/src/pkg/bytes/buffer.go new file mode 100644 index 000000000..5de86105d --- /dev/null +++ b/src/pkg/bytes/buffer.go @@ -0,0 +1,348 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package bytes + +// Simple byte buffer for marshaling data. + +import ( + "io" + "os" + "utf8" +) + +// A Buffer is a variable-sized buffer of bytes with Read and Write methods. +// The zero value for Buffer is an empty buffer ready to use. +type Buffer struct { + buf []byte // contents are the bytes buf[off : len(buf)] + off int // read at &buf[off], write at &buf[len(buf)] + runeBytes [utf8.UTFMax]byte // avoid allocation of slice on each WriteByte or Rune + bootstrap [64]byte // memory to hold first slice; helps small buffers (Printf) avoid allocation. + lastRead readOp // last read operation, so that Unread* can work correctly. +} + +// The readOp constants describe the last action performed on +// the buffer, so that UnreadRune and UnreadByte can +// check for invalid usage. +type readOp int + +const ( + opInvalid readOp = iota // Non-read operation. + opReadRune // Read rune. + opRead // Any other read operation. +) + +// Bytes returns a slice of the contents of the unread portion of the buffer; +// len(b.Bytes()) == b.Len(). If the caller changes the contents of the +// returned slice, the contents of the buffer will change provided there +// are no intervening method calls on the Buffer. +func (b *Buffer) Bytes() []byte { return b.buf[b.off:] } + +// String returns the contents of the unread portion of the buffer +// as a string. If the Buffer is a nil pointer, it returns "". +func (b *Buffer) String() string { + if b == nil { + // Special case, useful in debugging. + return "" + } + return string(b.buf[b.off:]) +} + +// Len returns the number of bytes of the unread portion of the buffer; +// b.Len() == len(b.Bytes()). +func (b *Buffer) Len() int { return len(b.buf) - b.off } + +// Truncate discards all but the first n unread bytes from the buffer. +// It is an error to call b.Truncate(n) with n > b.Len(). +func (b *Buffer) Truncate(n int) { + b.lastRead = opInvalid + if n == 0 { + // Reuse buffer space. + b.off = 0 + } + b.buf = b.buf[0 : b.off+n] +} + +// Reset resets the buffer so it has no content. +// b.Reset() is the same as b.Truncate(0). +func (b *Buffer) Reset() { b.Truncate(0) } + +// Grow buffer to guarantee space for n more bytes. +// Return index where bytes should be written. +func (b *Buffer) grow(n int) int { + m := b.Len() + // If buffer is empty, reset to recover space. + if m == 0 && b.off != 0 { + b.Truncate(0) + } + if len(b.buf)+n > cap(b.buf) { + var buf []byte + if b.buf == nil && n <= len(b.bootstrap) { + buf = b.bootstrap[0:] + } else { + // not enough space anywhere + buf = make([]byte, 2*cap(b.buf)+n) + copy(buf, b.buf[b.off:]) + } + b.buf = buf + b.off = 0 + } + b.buf = b.buf[0 : b.off+m+n] + return b.off + m +} + +// Write appends the contents of p to the buffer. The return +// value n is the length of p; err is always nil. +func (b *Buffer) Write(p []byte) (n int, err os.Error) { + b.lastRead = opInvalid + m := b.grow(len(p)) + copy(b.buf[m:], p) + return len(p), nil +} + +// WriteString appends the contents of s to the buffer. The return +// value n is the length of s; err is always nil. +func (b *Buffer) WriteString(s string) (n int, err os.Error) { + b.lastRead = opInvalid + m := b.grow(len(s)) + return copy(b.buf[m:], s), nil +} + +// MinRead is the minimum slice size passed to a Read call by +// Buffer.ReadFrom. As long as the Buffer has at least MinRead bytes beyond +// what is required to hold the contents of r, ReadFrom will not grow the +// underlying buffer. +const MinRead = 512 + +// ReadFrom reads data from r until EOF and appends it to the buffer. +// The return value n is the number of bytes read. +// Any error except os.EOF encountered during the read +// is also returned. +func (b *Buffer) ReadFrom(r io.Reader) (n int64, err os.Error) { + b.lastRead = opInvalid + // If buffer is empty, reset to recover space. + if b.off >= len(b.buf) { + b.Truncate(0) + } + for { + if cap(b.buf)-len(b.buf) < MinRead { + var newBuf []byte + // can we get space without allocation? + if b.off+cap(b.buf)-len(b.buf) >= MinRead { + // reuse beginning of buffer + newBuf = b.buf[0 : len(b.buf)-b.off] + } else { + // not enough space at end; put space on end + newBuf = make([]byte, len(b.buf)-b.off, 2*(cap(b.buf)-b.off)+MinRead) + } + copy(newBuf, b.buf[b.off:]) + b.buf = newBuf + b.off = 0 + } + m, e := r.Read(b.buf[len(b.buf):cap(b.buf)]) + b.buf = b.buf[0 : len(b.buf)+m] + n += int64(m) + if e == os.EOF { + break + } + if e != nil { + return n, e + } + } + return n, nil // err is EOF, so return nil explicitly +} + +// WriteTo writes data to w until the buffer is drained or an error +// occurs. The return value n is the number of bytes written; it always +// fits into an int, but it is int64 to match the io.WriterTo interface. +// Any error encountered during the write is also returned. +func (b *Buffer) WriteTo(w io.Writer) (n int64, err os.Error) { + b.lastRead = opInvalid + if b.off < len(b.buf) { + m, e := w.Write(b.buf[b.off:]) + b.off += m + n = int64(m) + if e != nil { + return n, e + } + // otherwise all bytes were written, by definition of + // Write method in io.Writer + } + // Buffer is now empty; reset. + b.Truncate(0) + return +} + +// WriteByte appends the byte c to the buffer. +// The returned error is always nil, but is included +// to match bufio.Writer's WriteByte. +func (b *Buffer) WriteByte(c byte) os.Error { + b.lastRead = opInvalid + m := b.grow(1) + b.buf[m] = c + return nil +} + +// WriteRune appends the UTF-8 encoding of Unicode +// code point r to the buffer, returning its length and +// an error, which is always nil but is included +// to match bufio.Writer's WriteRune. +func (b *Buffer) WriteRune(r int) (n int, err os.Error) { + if r < utf8.RuneSelf { + b.WriteByte(byte(r)) + return 1, nil + } + n = utf8.EncodeRune(b.runeBytes[0:], r) + b.Write(b.runeBytes[0:n]) + return n, nil +} + +// Read reads the next len(p) bytes from the buffer or until the buffer +// is drained. The return value n is the number of bytes read. If the +// buffer has no data to return, err is os.EOF even if len(p) is zero; +// otherwise it is nil. +func (b *Buffer) Read(p []byte) (n int, err os.Error) { + b.lastRead = opInvalid + if b.off >= len(b.buf) { + // Buffer is empty, reset to recover space. + b.Truncate(0) + return 0, os.EOF + } + n = copy(p, b.buf[b.off:]) + b.off += n + if n > 0 { + b.lastRead = opRead + } + return +} + +// Next returns a slice containing the next n bytes from the buffer, +// advancing the buffer as if the bytes had been returned by Read. +// If there are fewer than n bytes in the buffer, Next returns the entire buffer. +// The slice is only valid until the next call to a read or write method. +func (b *Buffer) Next(n int) []byte { + b.lastRead = opInvalid + m := b.Len() + if n > m { + n = m + } + data := b.buf[b.off : b.off+n] + b.off += n + if n > 0 { + b.lastRead = opRead + } + return data +} + +// ReadByte reads and returns the next byte from the buffer. +// If no byte is available, it returns error os.EOF. +func (b *Buffer) ReadByte() (c byte, err os.Error) { + b.lastRead = opInvalid + if b.off >= len(b.buf) { + // Buffer is empty, reset to recover space. + b.Truncate(0) + return 0, os.EOF + } + c = b.buf[b.off] + b.off++ + b.lastRead = opRead + return c, nil +} + +// ReadRune reads and returns the next UTF-8-encoded +// Unicode code point from the buffer. +// If no bytes are available, the error returned is os.EOF. +// If the bytes are an erroneous UTF-8 encoding, it +// consumes one byte and returns U+FFFD, 1. +func (b *Buffer) ReadRune() (r int, size int, err os.Error) { + b.lastRead = opInvalid + if b.off >= len(b.buf) { + // Buffer is empty, reset to recover space. + b.Truncate(0) + return 0, 0, os.EOF + } + b.lastRead = opReadRune + c := b.buf[b.off] + if c < utf8.RuneSelf { + b.off++ + return int(c), 1, nil + } + r, n := utf8.DecodeRune(b.buf[b.off:]) + b.off += n + return r, n, nil +} + +// UnreadRune unreads the last rune returned by ReadRune. +// If the most recent read or write operation on the buffer was +// not a ReadRune, UnreadRune returns an error. (In this regard +// it is stricter than UnreadByte, which will unread the last byte +// from any read operation.) +func (b *Buffer) UnreadRune() os.Error { + if b.lastRead != opReadRune { + return os.NewError("bytes.Buffer: UnreadRune: previous operation was not ReadRune") + } + b.lastRead = opInvalid + if b.off > 0 { + _, n := utf8.DecodeLastRune(b.buf[0:b.off]) + b.off -= n + } + return nil +} + +// UnreadByte unreads the last byte returned by the most recent +// read operation. If write has happened since the last read, UnreadByte +// returns an error. +func (b *Buffer) UnreadByte() os.Error { + if b.lastRead != opReadRune && b.lastRead != opRead { + return os.NewError("bytes.Buffer: UnreadByte: previous operation was not a read") + } + b.lastRead = opInvalid + if b.off > 0 { + b.off-- + } + return nil +} + +// ReadBytes reads until the first occurrence of delim in the input, +// returning a slice containing the data up to and including the delimiter. +// If ReadBytes encounters an error before finding a delimiter, +// it returns the data read before the error and the error itself (often os.EOF). +// ReadBytes returns err != nil if and only if the returned data does not end in +// delim. +func (b *Buffer) ReadBytes(delim byte) (line []byte, err os.Error) { + i := IndexByte(b.buf[b.off:], delim) + size := i + 1 + if i < 0 { + size = len(b.buf) - b.off + err = os.EOF + } + line = make([]byte, size) + copy(line, b.buf[b.off:]) + b.off += size + return +} + +// ReadString reads until the first occurrence of delim in the input, +// returning a string containing the data up to and including the delimiter. +// If ReadString encounters an error before finding a delimiter, +// it returns the data read before the error and the error itself (often os.EOF). +// ReadString returns err != nil if and only if the returned data does not end +// in delim. +func (b *Buffer) ReadString(delim byte) (line string, err os.Error) { + bytes, err := b.ReadBytes(delim) + return string(bytes), err +} + +// NewBuffer creates and initializes a new Buffer using buf as its initial +// contents. It is intended to prepare a Buffer to read existing data. It +// can also be used to size the internal buffer for writing. To do that, +// buf should have the desired capacity but a length of zero. +func NewBuffer(buf []byte) *Buffer { return &Buffer{buf: buf} } + +// NewBufferString creates and initializes a new Buffer using string s as its +// initial contents. It is intended to prepare a buffer to read an existing +// string. +func NewBufferString(s string) *Buffer { + return &Buffer{buf: []byte(s)} +} diff --git a/src/pkg/bytes/buffer_test.go b/src/pkg/bytes/buffer_test.go new file mode 100644 index 000000000..06d2a65c6 --- /dev/null +++ b/src/pkg/bytes/buffer_test.go @@ -0,0 +1,376 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package bytes_test + +import ( + . "bytes" + "os" + "rand" + "testing" + "utf8" +) + +const N = 10000 // make this bigger for a larger (and slower) test +var data string // test data for write tests +var bytes []byte // test data; same as data but as a slice. + + +func init() { + bytes = make([]byte, N) + for i := 0; i < N; i++ { + bytes[i] = 'a' + byte(i%26) + } + data = string(bytes) +} + +// Verify that contents of buf match the string s. +func check(t *testing.T, testname string, buf *Buffer, s string) { + bytes := buf.Bytes() + str := buf.String() + if buf.Len() != len(bytes) { + t.Errorf("%s: buf.Len() == %d, len(buf.Bytes()) == %d", testname, buf.Len(), len(bytes)) + } + + if buf.Len() != len(str) { + t.Errorf("%s: buf.Len() == %d, len(buf.String()) == %d", testname, buf.Len(), len(str)) + } + + if buf.Len() != len(s) { + t.Errorf("%s: buf.Len() == %d, len(s) == %d", testname, buf.Len(), len(s)) + } + + if string(bytes) != s { + t.Errorf("%s: string(buf.Bytes()) == %q, s == %q", testname, string(bytes), s) + } +} + +// Fill buf through n writes of string fus. +// The initial contents of buf corresponds to the string s; +// the result is the final contents of buf returned as a string. +func fillString(t *testing.T, testname string, buf *Buffer, s string, n int, fus string) string { + check(t, testname+" (fill 1)", buf, s) + for ; n > 0; n-- { + m, err := buf.WriteString(fus) + if m != len(fus) { + t.Errorf(testname+" (fill 2): m == %d, expected %d", m, len(fus)) + } + if err != nil { + t.Errorf(testname+" (fill 3): err should always be nil, found err == %s", err) + } + s += fus + check(t, testname+" (fill 4)", buf, s) + } + return s +} + +// Fill buf through n writes of byte slice fub. +// The initial contents of buf corresponds to the string s; +// the result is the final contents of buf returned as a string. +func fillBytes(t *testing.T, testname string, buf *Buffer, s string, n int, fub []byte) string { + check(t, testname+" (fill 1)", buf, s) + for ; n > 0; n-- { + m, err := buf.Write(fub) + if m != len(fub) { + t.Errorf(testname+" (fill 2): m == %d, expected %d", m, len(fub)) + } + if err != nil { + t.Errorf(testname+" (fill 3): err should always be nil, found err == %s", err) + } + s += string(fub) + check(t, testname+" (fill 4)", buf, s) + } + return s +} + +func TestNewBuffer(t *testing.T) { + buf := NewBuffer(bytes) + check(t, "NewBuffer", buf, data) +} + +func TestNewBufferString(t *testing.T) { + buf := NewBufferString(data) + check(t, "NewBufferString", buf, data) +} + +// Empty buf through repeated reads into fub. +// The initial contents of buf corresponds to the string s. +func empty(t *testing.T, testname string, buf *Buffer, s string, fub []byte) { + check(t, testname+" (empty 1)", buf, s) + + for { + n, err := buf.Read(fub) + if n == 0 { + break + } + if err != nil { + t.Errorf(testname+" (empty 2): err should always be nil, found err == %s", err) + } + s = s[n:] + check(t, testname+" (empty 3)", buf, s) + } + + check(t, testname+" (empty 4)", buf, "") +} + +func TestBasicOperations(t *testing.T) { + var buf Buffer + + for i := 0; i < 5; i++ { + check(t, "TestBasicOperations (1)", &buf, "") + + buf.Reset() + check(t, "TestBasicOperations (2)", &buf, "") + + buf.Truncate(0) + check(t, "TestBasicOperations (3)", &buf, "") + + n, err := buf.Write([]byte(data[0:1])) + if n != 1 { + t.Errorf("wrote 1 byte, but n == %d", n) + } + if err != nil { + t.Errorf("err should always be nil, but err == %s", err) + } + check(t, "TestBasicOperations (4)", &buf, "a") + + buf.WriteByte(data[1]) + check(t, "TestBasicOperations (5)", &buf, "ab") + + n, err = buf.Write([]byte(data[2:26])) + if n != 24 { + t.Errorf("wrote 25 bytes, but n == %d", n) + } + check(t, "TestBasicOperations (6)", &buf, string(data[0:26])) + + buf.Truncate(26) + check(t, "TestBasicOperations (7)", &buf, string(data[0:26])) + + buf.Truncate(20) + check(t, "TestBasicOperations (8)", &buf, string(data[0:20])) + + empty(t, "TestBasicOperations (9)", &buf, string(data[0:20]), make([]byte, 5)) + empty(t, "TestBasicOperations (10)", &buf, "", make([]byte, 100)) + + buf.WriteByte(data[1]) + c, err := buf.ReadByte() + if err != nil { + t.Error("ReadByte unexpected eof") + } + if c != data[1] { + t.Errorf("ReadByte wrong value c=%v", c) + } + c, err = buf.ReadByte() + if err == nil { + t.Error("ReadByte unexpected not eof") + } + } +} + +func TestLargeStringWrites(t *testing.T) { + var buf Buffer + limit := 30 + if testing.Short() { + limit = 9 + } + for i := 3; i < limit; i += 3 { + s := fillString(t, "TestLargeWrites (1)", &buf, "", 5, data) + empty(t, "TestLargeStringWrites (2)", &buf, s, make([]byte, len(data)/i)) + } + check(t, "TestLargeStringWrites (3)", &buf, "") +} + +func TestLargeByteWrites(t *testing.T) { + var buf Buffer + limit := 30 + if testing.Short() { + limit = 9 + } + for i := 3; i < limit; i += 3 { + s := fillBytes(t, "TestLargeWrites (1)", &buf, "", 5, bytes) + empty(t, "TestLargeByteWrites (2)", &buf, s, make([]byte, len(data)/i)) + } + check(t, "TestLargeByteWrites (3)", &buf, "") +} + +func TestLargeStringReads(t *testing.T) { + var buf Buffer + for i := 3; i < 30; i += 3 { + s := fillString(t, "TestLargeReads (1)", &buf, "", 5, data[0:len(data)/i]) + empty(t, "TestLargeReads (2)", &buf, s, make([]byte, len(data))) + } + check(t, "TestLargeStringReads (3)", &buf, "") +} + +func TestLargeByteReads(t *testing.T) { + var buf Buffer + for i := 3; i < 30; i += 3 { + s := fillBytes(t, "TestLargeReads (1)", &buf, "", 5, bytes[0:len(bytes)/i]) + empty(t, "TestLargeReads (2)", &buf, s, make([]byte, len(data))) + } + check(t, "TestLargeByteReads (3)", &buf, "") +} + +func TestMixedReadsAndWrites(t *testing.T) { + var buf Buffer + s := "" + for i := 0; i < 50; i++ { + wlen := rand.Intn(len(data)) + if i%2 == 0 { + s = fillString(t, "TestMixedReadsAndWrites (1)", &buf, s, 1, data[0:wlen]) + } else { + s = fillBytes(t, "TestMixedReadsAndWrites (1)", &buf, s, 1, bytes[0:wlen]) + } + + rlen := rand.Intn(len(data)) + fub := make([]byte, rlen) + n, _ := buf.Read(fub) + s = s[n:] + } + empty(t, "TestMixedReadsAndWrites (2)", &buf, s, make([]byte, buf.Len())) +} + +func TestNil(t *testing.T) { + var b *Buffer + if b.String() != "" { + t.Errorf("expected ; got %q", b.String()) + } +} + +func TestReadFrom(t *testing.T) { + var buf Buffer + for i := 3; i < 30; i += 3 { + s := fillBytes(t, "TestReadFrom (1)", &buf, "", 5, bytes[0:len(bytes)/i]) + var b Buffer + b.ReadFrom(&buf) + empty(t, "TestReadFrom (2)", &b, s, make([]byte, len(data))) + } +} + +func TestWriteTo(t *testing.T) { + var buf Buffer + for i := 3; i < 30; i += 3 { + s := fillBytes(t, "TestReadFrom (1)", &buf, "", 5, bytes[0:len(bytes)/i]) + var b Buffer + buf.WriteTo(&b) + empty(t, "TestReadFrom (2)", &b, s, make([]byte, len(data))) + } +} + +func TestRuneIO(t *testing.T) { + const NRune = 1000 + // Built a test array while we write the data + b := make([]byte, utf8.UTFMax*NRune) + var buf Buffer + n := 0 + for r := 0; r < NRune; r++ { + size := utf8.EncodeRune(b[n:], r) + nbytes, err := buf.WriteRune(r) + if err != nil { + t.Fatalf("WriteRune(%U) error: %s", r, err) + } + if nbytes != size { + t.Fatalf("WriteRune(%U) expected %d, got %d", r, size, nbytes) + } + n += size + } + b = b[0:n] + + // Check the resulting bytes + if !Equal(buf.Bytes(), b) { + t.Fatalf("incorrect result from WriteRune: %q not %q", buf.Bytes(), b) + } + + p := make([]byte, utf8.UTFMax) + // Read it back with ReadRune + for r := 0; r < NRune; r++ { + size := utf8.EncodeRune(p, r) + nr, nbytes, err := buf.ReadRune() + if nr != r || nbytes != size || err != nil { + t.Fatalf("ReadRune(%U) got %U,%d not %U,%d (err=%s)", r, nr, nbytes, r, size, err) + } + } + + // Check that UnreadRune works + buf.Reset() + buf.Write(b) + for r := 0; r < NRune; r++ { + r1, size, _ := buf.ReadRune() + if err := buf.UnreadRune(); err != nil { + t.Fatalf("UnreadRune(%U) got error %q", r, err) + } + r2, nbytes, err := buf.ReadRune() + if r1 != r2 || r1 != r || nbytes != size || err != nil { + t.Fatalf("ReadRune(%U) after UnreadRune got %U,%d not %U,%d (err=%s)", r, r2, nbytes, r, size, err) + } + } +} + +func TestNext(t *testing.T) { + b := []byte{0, 1, 2, 3, 4} + tmp := make([]byte, 5) + for i := 0; i <= 5; i++ { + for j := i; j <= 5; j++ { + for k := 0; k <= 6; k++ { + // 0 <= i <= j <= 5; 0 <= k <= 6 + // Check that if we start with a buffer + // of length j at offset i and ask for + // Next(k), we get the right bytes. + buf := NewBuffer(b[0:j]) + n, _ := buf.Read(tmp[0:i]) + if n != i { + t.Fatalf("Read %d returned %d", i, n) + } + bb := buf.Next(k) + want := k + if want > j-i { + want = j - i + } + if len(bb) != want { + t.Fatalf("in %d,%d: len(Next(%d)) == %d", i, j, k, len(bb)) + } + for l, v := range bb { + if v != byte(l+i) { + t.Fatalf("in %d,%d: Next(%d)[%d] = %d, want %d", i, j, k, l, v, l+i) + } + } + } + } + } +} + +var readBytesTests = []struct { + buffer string + delim byte + expected []string + err os.Error +}{ + {"", 0, []string{""}, os.EOF}, + {"a\x00", 0, []string{"a\x00"}, nil}, + {"abbbaaaba", 'b', []string{"ab", "b", "b", "aaab"}, nil}, + {"hello\x01world", 1, []string{"hello\x01"}, nil}, + {"foo\nbar", 0, []string{"foo\nbar"}, os.EOF}, + {"alpha\nbeta\ngamma\n", '\n', []string{"alpha\n", "beta\n", "gamma\n"}, nil}, + {"alpha\nbeta\ngamma", '\n', []string{"alpha\n", "beta\n", "gamma"}, os.EOF}, +} + +func TestReadBytes(t *testing.T) { + for _, test := range readBytesTests { + buf := NewBufferString(test.buffer) + var err os.Error + for _, expected := range test.expected { + var bytes []byte + bytes, err = buf.ReadBytes(test.delim) + if string(bytes) != expected { + t.Errorf("expected %q, got %q", expected, bytes) + } + if err != nil { + break + } + } + if err != test.err { + t.Errorf("expected error %v, got %v", test.err, err) + } + } +} diff --git a/src/pkg/bytes/bytes.go b/src/pkg/bytes/bytes.go new file mode 100644 index 000000000..5119fce94 --- /dev/null +++ b/src/pkg/bytes/bytes.go @@ -0,0 +1,605 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Package bytes implements functions for the manipulation of byte slices. +// It is analogous to the facilities of the strings package. +package bytes + +import ( + "unicode" + "utf8" +) + +// Compare returns an integer comparing the two byte arrays lexicographically. +// The result will be 0 if a==b, -1 if a < b, and +1 if a > b +func Compare(a, b []byte) int { + m := len(a) + if m > len(b) { + m = len(b) + } + for i, ac := range a[0:m] { + bc := b[i] + switch { + case ac > bc: + return 1 + case ac < bc: + return -1 + } + } + switch { + case len(a) < len(b): + return -1 + case len(a) > len(b): + return 1 + } + return 0 +} + +// Equal returns a boolean reporting whether a == b. +func Equal(a, b []byte) bool { + if len(a) != len(b) { + return false + } + for i, c := range a { + if c != b[i] { + return false + } + } + return true +} + +// explode splits s into an array of UTF-8 sequences, one per Unicode character (still arrays of bytes), +// up to a maximum of n byte arrays. Invalid UTF-8 sequences are chopped into individual bytes. +func explode(s []byte, n int) [][]byte { + if n <= 0 { + n = len(s) + } + a := make([][]byte, n) + var size int + na := 0 + for len(s) > 0 { + if na+1 >= n { + a[na] = s + na++ + break + } + _, size = utf8.DecodeRune(s) + a[na] = s[0:size] + s = s[size:] + na++ + } + return a[0:na] +} + +// Count counts the number of non-overlapping instances of sep in s. +func Count(s, sep []byte) int { + if len(sep) == 0 { + return utf8.RuneCount(s) + 1 + } + c := sep[0] + n := 0 + for i := 0; i+len(sep) <= len(s); i++ { + if s[i] == c && (len(sep) == 1 || Equal(s[i:i+len(sep)], sep)) { + n++ + i += len(sep) - 1 + } + } + return n +} + +// Index returns the index of the first instance of sep in s, or -1 if sep is not present in s. +func Index(s, sep []byte) int { + n := len(sep) + if n == 0 { + return 0 + } + c := sep[0] + for i := 0; i+n <= len(s); i++ { + if s[i] == c && (n == 1 || Equal(s[i:i+n], sep)) { + return i + } + } + return -1 +} + +func indexBytePortable(s []byte, c byte) int { + for i, b := range s { + if b == c { + return i + } + } + return -1 +} + +// LastIndex returns the index of the last instance of sep in s, or -1 if sep is not present in s. +func LastIndex(s, sep []byte) int { + n := len(sep) + if n == 0 { + return len(s) + } + c := sep[0] + for i := len(s) - n; i >= 0; i-- { + if s[i] == c && (n == 1 || Equal(s[i:i+n], sep)) { + return i + } + } + return -1 +} + +// IndexRune interprets s as a sequence of UTF-8-encoded Unicode code points. +// It returns the byte index of the first occurrence in s of the given rune. +// It returns -1 if rune is not present in s. +func IndexRune(s []byte, rune int) int { + for i := 0; i < len(s); { + r, size := utf8.DecodeRune(s[i:]) + if r == rune { + return i + } + i += size + } + return -1 +} + +// IndexAny interprets s as a sequence of UTF-8-encoded Unicode code points. +// It returns the byte index of the first occurrence in s of any of the Unicode +// code points in chars. It returns -1 if chars is empty or if there is no code +// point in common. +func IndexAny(s []byte, chars string) int { + if len(chars) > 0 { + var rune, width int + for i := 0; i < len(s); i += width { + rune = int(s[i]) + if rune < utf8.RuneSelf { + width = 1 + } else { + rune, width = utf8.DecodeRune(s[i:]) + } + for _, r := range chars { + if rune == r { + return i + } + } + } + } + return -1 +} + +// LastIndexAny interprets s as a sequence of UTF-8-encoded Unicode code +// points. It returns the byte index of the last occurrence in s of any of +// the Unicode code points in chars. It returns -1 if chars is empty or if +// there is no code point in common. +func LastIndexAny(s []byte, chars string) int { + if len(chars) > 0 { + for i := len(s); i > 0; { + rune, size := utf8.DecodeLastRune(s[0:i]) + i -= size + for _, m := range chars { + if rune == m { + return i + } + } + } + } + return -1 +} + +// Generic split: splits after each instance of sep, +// including sepSave bytes of sep in the subarrays. +func genSplit(s, sep []byte, sepSave, n int) [][]byte { + if n == 0 { + return nil + } + if len(sep) == 0 { + return explode(s, n) + } + if n < 0 { + n = Count(s, sep) + 1 + } + c := sep[0] + start := 0 + a := make([][]byte, n) + na := 0 + for i := 0; i+len(sep) <= len(s) && na+1 < n; i++ { + if s[i] == c && (len(sep) == 1 || Equal(s[i:i+len(sep)], sep)) { + a[na] = s[start : i+sepSave] + na++ + start = i + len(sep) + i += len(sep) - 1 + } + } + a[na] = s[start:] + return a[0 : na+1] +} + +// SplitN slices s into subslices separated by sep and returns a slice of +// the subslices between those separators. +// If sep is empty, SplitN splits after each UTF-8 sequence. +// The count determines the number of subslices to return: +// n > 0: at most n subslices; the last subslice will be the unsplit remainder. +// n == 0: the result is nil (zero subslices) +// n < 0: all subslices +func SplitN(s, sep []byte, n int) [][]byte { return genSplit(s, sep, 0, n) } + +// SplitAfterN slices s into subslices after each instance of sep and +// returns a slice of those subslices. +// If sep is empty, SplitAfterN splits after each UTF-8 sequence. +// The count determines the number of subslices to return: +// n > 0: at most n subslices; the last subslice will be the unsplit remainder. +// n == 0: the result is nil (zero subslices) +// n < 0: all subslices +func SplitAfterN(s, sep []byte, n int) [][]byte { + return genSplit(s, sep, len(sep), n) +} + +// Split slices s into all subslices separated by sep and returns a slice of +// the subslices between those separators. +// If sep is empty, Split splits after each UTF-8 sequence. +// It is equivalent to SplitN with a count of -1. +func Split(s, sep []byte) [][]byte { return genSplit(s, sep, 0, -1) } + +// SplitAfter slices s into all subslices after each instance of sep and +// returns a slice of those subslices. +// If sep is empty, SplitAfter splits after each UTF-8 sequence. +// It is equivalent to SplitAfterN with a count of -1. +func SplitAfter(s, sep []byte) [][]byte { + return genSplit(s, sep, len(sep), -1) +} + +// Fields splits the array s around each instance of one or more consecutive white space +// characters, returning a slice of subarrays of s or an empty list if s contains only white space. +func Fields(s []byte) [][]byte { + return FieldsFunc(s, unicode.IsSpace) +} + +// FieldsFunc interprets s as a sequence of UTF-8-encoded Unicode code points. +// It splits the array s at each run of code points c satisfying f(c) and +// returns a slice of subarrays of s. If no code points in s satisfy f(c), an +// empty slice is returned. +func FieldsFunc(s []byte, f func(int) bool) [][]byte { + n := 0 + inField := false + for i := 0; i < len(s); { + rune, size := utf8.DecodeRune(s[i:]) + wasInField := inField + inField = !f(rune) + if inField && !wasInField { + n++ + } + i += size + } + + a := make([][]byte, n) + na := 0 + fieldStart := -1 + for i := 0; i <= len(s) && na < n; { + rune, size := utf8.DecodeRune(s[i:]) + if fieldStart < 0 && size > 0 && !f(rune) { + fieldStart = i + i += size + continue + } + if fieldStart >= 0 && (size == 0 || f(rune)) { + a[na] = s[fieldStart:i] + na++ + fieldStart = -1 + } + if size == 0 { + break + } + i += size + } + return a[0:na] +} + +// Join concatenates the elements of a to create a single byte array. The separator +// sep is placed between elements in the resulting array. +func Join(a [][]byte, sep []byte) []byte { + if len(a) == 0 { + return []byte{} + } + if len(a) == 1 { + return a[0] + } + n := len(sep) * (len(a) - 1) + for i := 0; i < len(a); i++ { + n += len(a[i]) + } + + b := make([]byte, n) + bp := copy(b, a[0]) + for _, s := range a[1:] { + bp += copy(b[bp:], sep) + bp += copy(b[bp:], s) + } + return b +} + +// HasPrefix tests whether the byte array s begins with prefix. +func HasPrefix(s, prefix []byte) bool { + return len(s) >= len(prefix) && Equal(s[0:len(prefix)], prefix) +} + +// HasSuffix tests whether the byte array s ends with suffix. +func HasSuffix(s, suffix []byte) bool { + return len(s) >= len(suffix) && Equal(s[len(s)-len(suffix):], suffix) +} + +// Map returns a copy of the byte array s with all its characters modified +// according to the mapping function. If mapping returns a negative value, the character is +// dropped from the string with no replacement. The characters in s and the +// output are interpreted as UTF-8-encoded Unicode code points. +func Map(mapping func(rune int) int, s []byte) []byte { + // In the worst case, the array can grow when mapped, making + // things unpleasant. But it's so rare we barge in assuming it's + // fine. It could also shrink but that falls out naturally. + maxbytes := len(s) // length of b + nbytes := 0 // number of bytes encoded in b + b := make([]byte, maxbytes) + for i := 0; i < len(s); { + wid := 1 + rune := int(s[i]) + if rune >= utf8.RuneSelf { + rune, wid = utf8.DecodeRune(s[i:]) + } + rune = mapping(rune) + if rune >= 0 { + if nbytes+utf8.RuneLen(rune) > maxbytes { + // Grow the buffer. + maxbytes = maxbytes*2 + utf8.UTFMax + nb := make([]byte, maxbytes) + copy(nb, b[0:nbytes]) + b = nb + } + nbytes += utf8.EncodeRune(b[nbytes:maxbytes], rune) + } + i += wid + } + return b[0:nbytes] +} + +// Repeat returns a new byte slice consisting of count copies of b. +func Repeat(b []byte, count int) []byte { + nb := make([]byte, len(b)*count) + bp := 0 + for i := 0; i < count; i++ { + for j := 0; j < len(b); j++ { + nb[bp] = b[j] + bp++ + } + } + return nb +} + +// ToUpper returns a copy of the byte array s with all Unicode letters mapped to their upper case. +func ToUpper(s []byte) []byte { return Map(unicode.ToUpper, s) } + +// ToUpper returns a copy of the byte array s with all Unicode letters mapped to their lower case. +func ToLower(s []byte) []byte { return Map(unicode.ToLower, s) } + +// ToTitle returns a copy of the byte array s with all Unicode letters mapped to their title case. +func ToTitle(s []byte) []byte { return Map(unicode.ToTitle, s) } + +// ToUpperSpecial returns a copy of the byte array s with all Unicode letters mapped to their +// upper case, giving priority to the special casing rules. +func ToUpperSpecial(_case unicode.SpecialCase, s []byte) []byte { + return Map(func(r int) int { return _case.ToUpper(r) }, s) +} + +// ToLowerSpecial returns a copy of the byte array s with all Unicode letters mapped to their +// lower case, giving priority to the special casing rules. +func ToLowerSpecial(_case unicode.SpecialCase, s []byte) []byte { + return Map(func(r int) int { return _case.ToLower(r) }, s) +} + +// ToTitleSpecial returns a copy of the byte array s with all Unicode letters mapped to their +// title case, giving priority to the special casing rules. +func ToTitleSpecial(_case unicode.SpecialCase, s []byte) []byte { + return Map(func(r int) int { return _case.ToTitle(r) }, s) +} + +// isSeparator reports whether the rune could mark a word boundary. +// TODO: update when package unicode captures more of the properties. +func isSeparator(rune int) bool { + // ASCII alphanumerics and underscore are not separators + if rune <= 0x7F { + switch { + case '0' <= rune && rune <= '9': + return false + case 'a' <= rune && rune <= 'z': + return false + case 'A' <= rune && rune <= 'Z': + return false + case rune == '_': + return false + } + return true + } + // Letters and digits are not separators + if unicode.IsLetter(rune) || unicode.IsDigit(rune) { + return false + } + // Otherwise, all we can do for now is treat spaces as separators. + return unicode.IsSpace(rune) +} + +// BUG(r): The rule Title uses for word boundaries does not handle Unicode punctuation properly. + +// Title returns a copy of s with all Unicode letters that begin words +// mapped to their title case. +func Title(s []byte) []byte { + // Use a closure here to remember state. + // Hackish but effective. Depends on Map scanning in order and calling + // the closure once per rune. + prev := ' ' + return Map( + func(r int) int { + if isSeparator(prev) { + prev = r + return unicode.ToTitle(r) + } + prev = r + return r + }, + s) +} + +// TrimLeftFunc returns a subslice of s by slicing off all leading UTF-8-encoded +// Unicode code points c that satisfy f(c). +func TrimLeftFunc(s []byte, f func(r int) bool) []byte { + i := indexFunc(s, f, false) + if i == -1 { + return nil + } + return s[i:] +} + +// TrimRightFunc returns a subslice of s by slicing off all trailing UTF-8 +// encoded Unicode code points c that satisfy f(c). +func TrimRightFunc(s []byte, f func(r int) bool) []byte { + i := lastIndexFunc(s, f, false) + if i >= 0 && s[i] >= utf8.RuneSelf { + _, wid := utf8.DecodeRune(s[i:]) + i += wid + } else { + i++ + } + return s[0:i] +} + +// TrimFunc returns a subslice of s by slicing off all leading and trailing +// UTF-8-encoded Unicode code points c that satisfy f(c). +func TrimFunc(s []byte, f func(r int) bool) []byte { + return TrimRightFunc(TrimLeftFunc(s, f), f) +} + +// IndexFunc interprets s as a sequence of UTF-8-encoded Unicode code points. +// It returns the byte index in s of the first Unicode +// code point satisfying f(c), or -1 if none do. +func IndexFunc(s []byte, f func(r int) bool) int { + return indexFunc(s, f, true) +} + +// LastIndexFunc interprets s as a sequence of UTF-8-encoded Unicode code points. +// It returns the byte index in s of the last Unicode +// code point satisfying f(c), or -1 if none do. +func LastIndexFunc(s []byte, f func(r int) bool) int { + return lastIndexFunc(s, f, true) +} + +// indexFunc is the same as IndexFunc except that if +// truth==false, the sense of the predicate function is +// inverted. +func indexFunc(s []byte, f func(r int) bool, truth bool) int { + start := 0 + for start < len(s) { + wid := 1 + rune := int(s[start]) + if rune >= utf8.RuneSelf { + rune, wid = utf8.DecodeRune(s[start:]) + } + if f(rune) == truth { + return start + } + start += wid + } + return -1 +} + +// lastIndexFunc is the same as LastIndexFunc except that if +// truth==false, the sense of the predicate function is +// inverted. +func lastIndexFunc(s []byte, f func(r int) bool, truth bool) int { + for i := len(s); i > 0; { + rune, size := utf8.DecodeLastRune(s[0:i]) + i -= size + if f(rune) == truth { + return i + } + } + return -1 +} + +func makeCutsetFunc(cutset string) func(rune int) bool { + return func(rune int) bool { + for _, c := range cutset { + if c == rune { + return true + } + } + return false + } +} + +// Trim returns a subslice of s by slicing off all leading and +// trailing UTF-8-encoded Unicode code points contained in cutset. +func Trim(s []byte, cutset string) []byte { + return TrimFunc(s, makeCutsetFunc(cutset)) +} + +// TrimLeft returns a subslice of s by slicing off all leading +// UTF-8-encoded Unicode code points contained in cutset. +func TrimLeft(s []byte, cutset string) []byte { + return TrimLeftFunc(s, makeCutsetFunc(cutset)) +} + +// TrimRight returns a subslice of s by slicing off all trailing +// UTF-8-encoded Unicode code points that are contained in cutset. +func TrimRight(s []byte, cutset string) []byte { + return TrimRightFunc(s, makeCutsetFunc(cutset)) +} + +// TrimSpace returns a subslice of s by slicing off all leading and +// trailing white space, as defined by Unicode. +func TrimSpace(s []byte) []byte { + return TrimFunc(s, unicode.IsSpace) +} + +// Runes returns a slice of runes (Unicode code points) equivalent to s. +func Runes(s []byte) []int { + t := make([]int, utf8.RuneCount(s)) + i := 0 + for len(s) > 0 { + r, l := utf8.DecodeRune(s) + t[i] = r + i++ + s = s[l:] + } + return t +} + +// Replace returns a copy of the slice s with the first n +// non-overlapping instances of old replaced by new. +// If n < 0, there is no limit on the number of replacements. +func Replace(s, old, new []byte, n int) []byte { + if n == 0 { + return s // avoid allocation + } + // Compute number of replacements. + if m := Count(s, old); m == 0 { + return s // avoid allocation + } else if n <= 0 || m < n { + n = m + } + + // Apply replacements to buffer. + t := make([]byte, len(s)+n*(len(new)-len(old))) + w := 0 + start := 0 + for i := 0; i < n; i++ { + j := start + if len(old) == 0 { + if i > 0 { + _, wid := utf8.DecodeRune(s[start:]) + j += wid + } + } else { + j += Index(s[start:], old) + } + w += copy(t[w:], s[start:j]) + w += copy(t[w:], new) + start = j + len(old) + } + w += copy(t[w:], s[start:]) + return t[0:w] +} diff --git a/src/pkg/bytes/bytes_decl.go b/src/pkg/bytes/bytes_decl.go new file mode 100644 index 000000000..5d2b9e639 --- /dev/null +++ b/src/pkg/bytes/bytes_decl.go @@ -0,0 +1,8 @@ +// Copyright 2010 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package bytes + +// IndexByte returns the index of the first instance of c in s, or -1 if c is not present in s. +func IndexByte(s []byte, c byte) int // asm_$GOARCH.s diff --git a/src/pkg/bytes/bytes_test.go b/src/pkg/bytes/bytes_test.go new file mode 100644 index 000000000..9444358a8 --- /dev/null +++ b/src/pkg/bytes/bytes_test.go @@ -0,0 +1,858 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package bytes_test + +import ( + . "bytes" + "reflect" + "testing" + "unicode" + "utf8" +) + +func eq(a, b []string) bool { + if len(a) != len(b) { + return false + } + for i := 0; i < len(a); i++ { + if a[i] != b[i] { + return false + } + } + return true +} + +func arrayOfString(a [][]byte) []string { + result := make([]string, len(a)) + for j := 0; j < len(a); j++ { + result[j] = string(a[j]) + } + return result +} + +// For ease of reading, the test cases use strings that are converted to byte +// arrays before invoking the functions. + +var abcd = "abcd" +var faces = "☺☻☹" +var commas = "1,2,3,4" +var dots = "1....2....3....4" + +type BinOpTest struct { + a string + b string + i int +} + +var comparetests = []BinOpTest{ + {"", "", 0}, + {"a", "", 1}, + {"", "a", -1}, + {"abc", "abc", 0}, + {"ab", "abc", -1}, + {"abc", "ab", 1}, + {"x", "ab", 1}, + {"ab", "x", -1}, + {"x", "a", 1}, + {"b", "x", -1}, +} + +func TestCompare(t *testing.T) { + for _, tt := range comparetests { + a := []byte(tt.a) + b := []byte(tt.b) + cmp := Compare(a, b) + eql := Equal(a, b) + if cmp != tt.i { + t.Errorf(`Compare(%q, %q) = %v`, tt.a, tt.b, cmp) + } + if eql != (tt.i == 0) { + t.Errorf(`Equal(%q, %q) = %v`, tt.a, tt.b, eql) + } + } +} + +var indexTests = []BinOpTest{ + {"", "", 0}, + {"", "a", -1}, + {"", "foo", -1}, + {"fo", "foo", -1}, + {"foo", "foo", 0}, + {"oofofoofooo", "f", 2}, + {"oofofoofooo", "foo", 4}, + {"barfoobarfoo", "foo", 3}, + {"foo", "", 0}, + {"foo", "o", 1}, + {"abcABCabc", "A", 3}, + // cases with one byte strings - test IndexByte and special case in Index() + {"", "a", -1}, + {"x", "a", -1}, + {"x", "x", 0}, + {"abc", "a", 0}, + {"abc", "b", 1}, + {"abc", "c", 2}, + {"abc", "x", -1}, + {"barfoobarfooyyyzzzyyyzzzyyyzzzyyyxxxzzzyyy", "x", 33}, + {"foofyfoobarfoobar", "y", 4}, + {"oooooooooooooooooooooo", "r", -1}, +} + +var lastIndexTests = []BinOpTest{ + {"", "", 0}, + {"", "a", -1}, + {"", "foo", -1}, + {"fo", "foo", -1}, + {"foo", "foo", 0}, + {"foo", "f", 0}, + {"oofofoofooo", "f", 7}, + {"oofofoofooo", "foo", 7}, + {"barfoobarfoo", "foo", 9}, + {"foo", "", 3}, + {"foo", "o", 2}, + {"abcABCabc", "A", 3}, + {"abcABCabc", "a", 6}, +} + +var indexAnyTests = []BinOpTest{ + {"", "", -1}, + {"", "a", -1}, + {"", "abc", -1}, + {"a", "", -1}, + {"a", "a", 0}, + {"aaa", "a", 0}, + {"abc", "xyz", -1}, + {"abc", "xcz", 2}, + {"ab☺c", "x☺yz", 2}, + {"aRegExp*", ".(|)*+?^$[]", 7}, + {dots + dots + dots, " ", -1}, +} + +var lastIndexAnyTests = []BinOpTest{ + {"", "", -1}, + {"", "a", -1}, + {"", "abc", -1}, + {"a", "", -1}, + {"a", "a", 0}, + {"aaa", "a", 2}, + {"abc", "xyz", -1}, + {"abc", "ab", 1}, + {"a☺b☻c☹d", "uvw☻xyz", 2 + len("☺")}, + {"a.RegExp*", ".(|)*+?^$[]", 8}, + {dots + dots + dots, " ", -1}, +} + +var indexRuneTests = []BinOpTest{ + {"", "a", -1}, + {"", "☺", -1}, + {"foo", "☹", -1}, + {"foo", "o", 1}, + {"foo☺bar", "☺", 3}, + {"foo☺☻☹bar", "☹", 9}, +} + +// Execute f on each test case. funcName should be the name of f; it's used +// in failure reports. +func runIndexTests(t *testing.T, f func(s, sep []byte) int, funcName string, testCases []BinOpTest) { + for _, test := range testCases { + a := []byte(test.a) + b := []byte(test.b) + actual := f(a, b) + if actual != test.i { + t.Errorf("%s(%q,%q) = %v; want %v", funcName, a, b, actual, test.i) + } + } +} + +func runIndexAnyTests(t *testing.T, f func(s []byte, chars string) int, funcName string, testCases []BinOpTest) { + for _, test := range testCases { + a := []byte(test.a) + actual := f(a, test.b) + if actual != test.i { + t.Errorf("%s(%q,%q) = %v; want %v", funcName, a, test.b, actual, test.i) + } + } +} + +func TestIndex(t *testing.T) { runIndexTests(t, Index, "Index", indexTests) } +func TestLastIndex(t *testing.T) { runIndexTests(t, LastIndex, "LastIndex", lastIndexTests) } +func TestIndexAny(t *testing.T) { runIndexAnyTests(t, IndexAny, "IndexAny", indexAnyTests) } +func TestLastIndexAny(t *testing.T) { + runIndexAnyTests(t, LastIndexAny, "LastIndexAny", lastIndexAnyTests) +} + +func TestIndexByte(t *testing.T) { + for _, tt := range indexTests { + if len(tt.b) != 1 { + continue + } + a := []byte(tt.a) + b := tt.b[0] + pos := IndexByte(a, b) + if pos != tt.i { + t.Errorf(`IndexByte(%q, '%c') = %v`, tt.a, b, pos) + } + posp := IndexBytePortable(a, b) + if posp != tt.i { + t.Errorf(`indexBytePortable(%q, '%c') = %v`, tt.a, b, posp) + } + } +} + +// test a larger buffer with different sizes and alignments +func TestIndexByteBig(t *testing.T) { + var n = 1024 + if testing.Short() { + n = 128 + } + b := make([]byte, n) + for i := 0; i < n; i++ { + // different start alignments + b1 := b[i:] + for j := 0; j < len(b1); j++ { + b1[j] = 'x' + pos := IndexByte(b1, 'x') + if pos != j { + t.Errorf("IndexByte(%q, 'x') = %v", b1, pos) + } + b1[j] = 0 + pos = IndexByte(b1, 'x') + if pos != -1 { + t.Errorf("IndexByte(%q, 'x') = %v", b1, pos) + } + } + // different end alignments + b1 = b[:i] + for j := 0; j < len(b1); j++ { + b1[j] = 'x' + pos := IndexByte(b1, 'x') + if pos != j { + t.Errorf("IndexByte(%q, 'x') = %v", b1, pos) + } + b1[j] = 0 + pos = IndexByte(b1, 'x') + if pos != -1 { + t.Errorf("IndexByte(%q, 'x') = %v", b1, pos) + } + } + // different start and end alignments + b1 = b[i/2 : n-(i+1)/2] + for j := 0; j < len(b1); j++ { + b1[j] = 'x' + pos := IndexByte(b1, 'x') + if pos != j { + t.Errorf("IndexByte(%q, 'x') = %v", b1, pos) + } + b1[j] = 0 + pos = IndexByte(b1, 'x') + if pos != -1 { + t.Errorf("IndexByte(%q, 'x') = %v", b1, pos) + } + } + } +} + +func TestIndexRune(t *testing.T) { + for _, tt := range indexRuneTests { + a := []byte(tt.a) + r, _ := utf8.DecodeRuneInString(tt.b) + pos := IndexRune(a, r) + if pos != tt.i { + t.Errorf(`IndexRune(%q, '%c') = %v`, tt.a, r, pos) + } + } +} + +func BenchmarkIndexByte4K(b *testing.B) { bmIndex(b, IndexByte, 4<<10) } + +func BenchmarkIndexByte4M(b *testing.B) { bmIndex(b, IndexByte, 4<<20) } + +func BenchmarkIndexByte64M(b *testing.B) { bmIndex(b, IndexByte, 64<<20) } + +func BenchmarkIndexBytePortable4K(b *testing.B) { + bmIndex(b, IndexBytePortable, 4<<10) +} + +func BenchmarkIndexBytePortable4M(b *testing.B) { + bmIndex(b, IndexBytePortable, 4<<20) +} + +func BenchmarkIndexBytePortable64M(b *testing.B) { + bmIndex(b, IndexBytePortable, 64<<20) +} + +var bmbuf []byte + +func bmIndex(b *testing.B, index func([]byte, byte) int, n int) { + if len(bmbuf) < n { + bmbuf = make([]byte, n) + } + b.SetBytes(int64(n)) + buf := bmbuf[0:n] + buf[n-1] = 'x' + for i := 0; i < b.N; i++ { + j := index(buf, 'x') + if j != n-1 { + println("bad index", j) + panic("bad index") + } + } + buf[n-1] = '0' +} + +type ExplodeTest struct { + s string + n int + a []string +} + +var explodetests = []ExplodeTest{ + {"", -1, []string{}}, + {abcd, -1, []string{"a", "b", "c", "d"}}, + {faces, -1, []string{"☺", "☻", "☹"}}, + {abcd, 2, []string{"a", "bcd"}}, +} + +func TestExplode(t *testing.T) { + for _, tt := range explodetests { + a := SplitN([]byte(tt.s), nil, tt.n) + result := arrayOfString(a) + if !eq(result, tt.a) { + t.Errorf(`Explode("%s", %d) = %v; want %v`, tt.s, tt.n, result, tt.a) + continue + } + s := Join(a, []byte{}) + if string(s) != tt.s { + t.Errorf(`Join(Explode("%s", %d), "") = "%s"`, tt.s, tt.n, s) + } + } +} + +type SplitTest struct { + s string + sep string + n int + a []string +} + +var splittests = []SplitTest{ + {abcd, "a", 0, nil}, + {abcd, "a", -1, []string{"", "bcd"}}, + {abcd, "z", -1, []string{"abcd"}}, + {abcd, "", -1, []string{"a", "b", "c", "d"}}, + {commas, ",", -1, []string{"1", "2", "3", "4"}}, + {dots, "...", -1, []string{"1", ".2", ".3", ".4"}}, + {faces, "☹", -1, []string{"☺☻", ""}}, + {faces, "~", -1, []string{faces}}, + {faces, "", -1, []string{"☺", "☻", "☹"}}, + {"1 2 3 4", " ", 3, []string{"1", "2", "3 4"}}, + {"1 2", " ", 3, []string{"1", "2"}}, + {"123", "", 2, []string{"1", "23"}}, + {"123", "", 17, []string{"1", "2", "3"}}, +} + +func TestSplit(t *testing.T) { + for _, tt := range splittests { + a := SplitN([]byte(tt.s), []byte(tt.sep), tt.n) + result := arrayOfString(a) + if !eq(result, tt.a) { + t.Errorf(`Split(%q, %q, %d) = %v; want %v`, tt.s, tt.sep, tt.n, result, tt.a) + continue + } + if tt.n == 0 { + continue + } + s := Join(a, []byte(tt.sep)) + if string(s) != tt.s { + t.Errorf(`Join(Split(%q, %q, %d), %q) = %q`, tt.s, tt.sep, tt.n, tt.sep, s) + } + if tt.n < 0 { + b := Split([]byte(tt.s), []byte(tt.sep)) + if !reflect.DeepEqual(a, b) { + t.Errorf("Split disagrees withSplitN(%q, %q, %d) = %v; want %v", tt.s, tt.sep, tt.n, b, a) + } + } + } +} + +var splitaftertests = []SplitTest{ + {abcd, "a", -1, []string{"a", "bcd"}}, + {abcd, "z", -1, []string{"abcd"}}, + {abcd, "", -1, []string{"a", "b", "c", "d"}}, + {commas, ",", -1, []string{"1,", "2,", "3,", "4"}}, + {dots, "...", -1, []string{"1...", ".2...", ".3...", ".4"}}, + {faces, "☹", -1, []string{"☺☻☹", ""}}, + {faces, "~", -1, []string{faces}}, + {faces, "", -1, []string{"☺", "☻", "☹"}}, + {"1 2 3 4", " ", 3, []string{"1 ", "2 ", "3 4"}}, + {"1 2 3", " ", 3, []string{"1 ", "2 ", "3"}}, + {"1 2", " ", 3, []string{"1 ", "2"}}, + {"123", "", 2, []string{"1", "23"}}, + {"123", "", 17, []string{"1", "2", "3"}}, +} + +func TestSplitAfter(t *testing.T) { + for _, tt := range splitaftertests { + a := SplitAfterN([]byte(tt.s), []byte(tt.sep), tt.n) + result := arrayOfString(a) + if !eq(result, tt.a) { + t.Errorf(`Split(%q, %q, %d) = %v; want %v`, tt.s, tt.sep, tt.n, result, tt.a) + continue + } + s := Join(a, nil) + if string(s) != tt.s { + t.Errorf(`Join(Split(%q, %q, %d), %q) = %q`, tt.s, tt.sep, tt.n, tt.sep, s) + } + if tt.n < 0 { + b := SplitAfter([]byte(tt.s), []byte(tt.sep)) + if !reflect.DeepEqual(a, b) { + t.Errorf("SplitAfter disagrees withSplitAfterN(%q, %q, %d) = %v; want %v", tt.s, tt.sep, tt.n, b, a) + } + } + } +} + +type FieldsTest struct { + s string + a []string +} + +var fieldstests = []FieldsTest{ + {"", []string{}}, + {" ", []string{}}, + {" \t ", []string{}}, + {" abc ", []string{"abc"}}, + {"1 2 3 4", []string{"1", "2", "3", "4"}}, + {"1 2 3 4", []string{"1", "2", "3", "4"}}, + {"1\t\t2\t\t3\t4", []string{"1", "2", "3", "4"}}, + {"1\u20002\u20013\u20024", []string{"1", "2", "3", "4"}}, + {"\u2000\u2001\u2002", []string{}}, + {"\n™\t™\n", []string{"™", "™"}}, + {faces, []string{faces}}, +} + +func TestFields(t *testing.T) { + for _, tt := range fieldstests { + a := Fields([]byte(tt.s)) + result := arrayOfString(a) + if !eq(result, tt.a) { + t.Errorf("Fields(%q) = %v; want %v", tt.s, a, tt.a) + continue + } + } +} + +func TestFieldsFunc(t *testing.T) { + pred := func(c int) bool { return c == 'X' } + var fieldsFuncTests = []FieldsTest{ + {"", []string{}}, + {"XX", []string{}}, + {"XXhiXXX", []string{"hi"}}, + {"aXXbXXXcX", []string{"a", "b", "c"}}, + } + for _, tt := range fieldsFuncTests { + a := FieldsFunc([]byte(tt.s), pred) + result := arrayOfString(a) + if !eq(result, tt.a) { + t.Errorf("FieldsFunc(%q) = %v, want %v", tt.s, a, tt.a) + } + } +} + +// Test case for any function which accepts and returns a byte array. +// For ease of creation, we write the byte arrays as strings. +type StringTest struct { + in, out string +} + +var upperTests = []StringTest{ + {"", ""}, + {"abc", "ABC"}, + {"AbC123", "ABC123"}, + {"azAZ09_", "AZAZ09_"}, + {"\u0250\u0250\u0250\u0250\u0250", "\u2C6F\u2C6F\u2C6F\u2C6F\u2C6F"}, // grows one byte per char +} + +var lowerTests = []StringTest{ + {"", ""}, + {"abc", "abc"}, + {"AbC123", "abc123"}, + {"azAZ09_", "azaz09_"}, + {"\u2C6D\u2C6D\u2C6D\u2C6D\u2C6D", "\u0251\u0251\u0251\u0251\u0251"}, // shrinks one byte per char +} + +const space = "\t\v\r\f\n\u0085\u00a0\u2000\u3000" + +var trimSpaceTests = []StringTest{ + {"", ""}, + {"abc", "abc"}, + {space + "abc" + space, "abc"}, + {" ", ""}, + {" \t\r\n \t\t\r\r\n\n ", ""}, + {" \t\r\n x\t\t\r\r\n\n ", "x"}, + {" \u2000\t\r\n x\t\t\r\r\ny\n \u3000", "x\t\t\r\r\ny"}, + {"1 \t\r\n2", "1 \t\r\n2"}, + {" x\x80", "x\x80"}, + {" x\xc0", "x\xc0"}, + {"x \xc0\xc0 ", "x \xc0\xc0"}, + {"x \xc0", "x \xc0"}, + {"x \xc0 ", "x \xc0"}, + {"x \xc0\xc0 ", "x \xc0\xc0"}, + {"x ☺\xc0\xc0 ", "x ☺\xc0\xc0"}, + {"x ☺ ", "x ☺"}, +} + +// Execute f on each test case. funcName should be the name of f; it's used +// in failure reports. +func runStringTests(t *testing.T, f func([]byte) []byte, funcName string, testCases []StringTest) { + for _, tc := range testCases { + actual := string(f([]byte(tc.in))) + if actual != tc.out { + t.Errorf("%s(%q) = %q; want %q", funcName, tc.in, actual, tc.out) + } + } +} + +func tenRunes(rune int) string { + r := make([]int, 10) + for i := range r { + r[i] = rune + } + return string(r) +} + +// User-defined self-inverse mapping function +func rot13(rune int) int { + step := 13 + if rune >= 'a' && rune <= 'z' { + return ((rune - 'a' + step) % 26) + 'a' + } + if rune >= 'A' && rune <= 'Z' { + return ((rune - 'A' + step) % 26) + 'A' + } + return rune +} + +func TestMap(t *testing.T) { + // Run a couple of awful growth/shrinkage tests + a := tenRunes('a') + + // 1. Grow. This triggers two reallocations in Map. + maxRune := func(rune int) int { return unicode.MaxRune } + m := Map(maxRune, []byte(a)) + expect := tenRunes(unicode.MaxRune) + if string(m) != expect { + t.Errorf("growing: expected %q got %q", expect, m) + } + + // 2. Shrink + minRune := func(rune int) int { return 'a' } + m = Map(minRune, []byte(tenRunes(unicode.MaxRune))) + expect = a + if string(m) != expect { + t.Errorf("shrinking: expected %q got %q", expect, m) + } + + // 3. Rot13 + m = Map(rot13, []byte("a to zed")) + expect = "n gb mrq" + if string(m) != expect { + t.Errorf("rot13: expected %q got %q", expect, m) + } + + // 4. Rot13^2 + m = Map(rot13, Map(rot13, []byte("a to zed"))) + expect = "a to zed" + if string(m) != expect { + t.Errorf("rot13: expected %q got %q", expect, m) + } + + // 5. Drop + dropNotLatin := func(rune int) int { + if unicode.Is(unicode.Latin, rune) { + return rune + } + return -1 + } + m = Map(dropNotLatin, []byte("Hello, 세계")) + expect = "Hello" + if string(m) != expect { + t.Errorf("drop: expected %q got %q", expect, m) + } +} + +func TestToUpper(t *testing.T) { runStringTests(t, ToUpper, "ToUpper", upperTests) } + +func TestToLower(t *testing.T) { runStringTests(t, ToLower, "ToLower", lowerTests) } + +func TestTrimSpace(t *testing.T) { runStringTests(t, TrimSpace, "TrimSpace", trimSpaceTests) } + +type RepeatTest struct { + in, out string + count int +} + +var RepeatTests = []RepeatTest{ + {"", "", 0}, + {"", "", 1}, + {"", "", 2}, + {"-", "", 0}, + {"-", "-", 1}, + {"-", "----------", 10}, + {"abc ", "abc abc abc ", 3}, +} + +func TestRepeat(t *testing.T) { + for _, tt := range RepeatTests { + tin := []byte(tt.in) + tout := []byte(tt.out) + a := Repeat(tin, tt.count) + if !Equal(a, tout) { + t.Errorf("Repeat(%q, %d) = %q; want %q", tin, tt.count, a, tout) + continue + } + } +} + +func runesEqual(a, b []int) bool { + if len(a) != len(b) { + return false + } + for i, r := range a { + if r != b[i] { + return false + } + } + return true +} + +type RunesTest struct { + in string + out []int + lossy bool +} + +var RunesTests = []RunesTest{ + {"", []int{}, false}, + {" ", []int{32}, false}, + {"ABC", []int{65, 66, 67}, false}, + {"abc", []int{97, 98, 99}, false}, + {"\u65e5\u672c\u8a9e", []int{26085, 26412, 35486}, false}, + {"ab\x80c", []int{97, 98, 0xFFFD, 99}, true}, + {"ab\xc0c", []int{97, 98, 0xFFFD, 99}, true}, +} + +func TestRunes(t *testing.T) { + for _, tt := range RunesTests { + tin := []byte(tt.in) + a := Runes(tin) + if !runesEqual(a, tt.out) { + t.Errorf("Runes(%q) = %v; want %v", tin, a, tt.out) + continue + } + if !tt.lossy { + // can only test reassembly if we didn't lose information + s := string(a) + if s != tt.in { + t.Errorf("string(Runes(%q)) = %x; want %x", tin, s, tin) + } + } + } +} + +type TrimTest struct { + f func([]byte, string) []byte + in, cutset, out string +} + +var trimTests = []TrimTest{ + {Trim, "abba", "a", "bb"}, + {Trim, "abba", "ab", ""}, + {TrimLeft, "abba", "ab", ""}, + {TrimRight, "abba", "ab", ""}, + {TrimLeft, "abba", "a", "bba"}, + {TrimRight, "abba", "a", "abb"}, + {Trim, "", "<>", "tag"}, + {Trim, "* listitem", " *", "listitem"}, + {Trim, `"quote"`, `"`, "quote"}, + {Trim, "\u2C6F\u2C6F\u0250\u0250\u2C6F\u2C6F", "\u2C6F", "\u0250\u0250"}, + //empty string tests + {Trim, "abba", "", "abba"}, + {Trim, "", "123", ""}, + {Trim, "", "", ""}, + {TrimLeft, "abba", "", "abba"}, + {TrimLeft, "", "123", ""}, + {TrimLeft, "", "", ""}, + {TrimRight, "abba", "", "abba"}, + {TrimRight, "", "123", ""}, + {TrimRight, "", "", ""}, + {TrimRight, "☺\xc0", "☺", "☺\xc0"}, +} + +func TestTrim(t *testing.T) { + for _, tc := range trimTests { + actual := string(tc.f([]byte(tc.in), tc.cutset)) + var name string + switch tc.f { + case Trim: + name = "Trim" + case TrimLeft: + name = "TrimLeft" + case TrimRight: + name = "TrimRight" + default: + t.Error("Undefined trim function") + } + if actual != tc.out { + t.Errorf("%s(%q, %q) = %q; want %q", name, tc.in, tc.cutset, actual, tc.out) + } + } +} + +type predicate struct { + f func(r int) bool + name string +} + +var isSpace = predicate{unicode.IsSpace, "IsSpace"} +var isDigit = predicate{unicode.IsDigit, "IsDigit"} +var isUpper = predicate{unicode.IsUpper, "IsUpper"} +var isValidRune = predicate{ + func(r int) bool { + return r != utf8.RuneError + }, + "IsValidRune", +} + +type TrimFuncTest struct { + f predicate + in, out string +} + +func not(p predicate) predicate { + return predicate{ + func(r int) bool { + return !p.f(r) + }, + "not " + p.name, + } +} + +var trimFuncTests = []TrimFuncTest{ + {isSpace, space + " hello " + space, "hello"}, + {isDigit, "\u0e50\u0e5212hello34\u0e50\u0e51", "hello"}, + {isUpper, "\u2C6F\u2C6F\u2C6F\u2C6FABCDhelloEF\u2C6F\u2C6FGH\u2C6F\u2C6F", "hello"}, + {not(isSpace), "hello" + space + "hello", space}, + {not(isDigit), "hello\u0e50\u0e521234\u0e50\u0e51helo", "\u0e50\u0e521234\u0e50\u0e51"}, + {isValidRune, "ab\xc0a\xc0cd", "\xc0a\xc0"}, + {not(isValidRune), "\xc0a\xc0", "a"}, +} + +func TestTrimFunc(t *testing.T) { + for _, tc := range trimFuncTests { + actual := string(TrimFunc([]byte(tc.in), tc.f.f)) + if actual != tc.out { + t.Errorf("TrimFunc(%q, %q) = %q; want %q", tc.in, tc.f.name, actual, tc.out) + } + } +} + +type IndexFuncTest struct { + in string + f predicate + first, last int +} + +var indexFuncTests = []IndexFuncTest{ + {"", isValidRune, -1, -1}, + {"abc", isDigit, -1, -1}, + {"0123", isDigit, 0, 3}, + {"a1b", isDigit, 1, 1}, + {space, isSpace, 0, len(space) - 3}, // last rune in space is 3 bytes + {"\u0e50\u0e5212hello34\u0e50\u0e51", isDigit, 0, 18}, + {"\u2C6F\u2C6F\u2C6F\u2C6FABCDhelloEF\u2C6F\u2C6FGH\u2C6F\u2C6F", isUpper, 0, 34}, + {"12\u0e50\u0e52hello34\u0e50\u0e51", not(isDigit), 8, 12}, + + // tests of invalid UTF-8 + {"\x801", isDigit, 1, 1}, + {"\x80abc", isDigit, -1, -1}, + {"\xc0a\xc0", isValidRune, 1, 1}, + {"\xc0a\xc0", not(isValidRune), 0, 2}, + {"\xc0☺\xc0", not(isValidRune), 0, 4}, + {"\xc0☺\xc0\xc0", not(isValidRune), 0, 5}, + {"ab\xc0a\xc0cd", not(isValidRune), 2, 4}, + {"a\xe0\x80cd", not(isValidRune), 1, 2}, +} + +func TestIndexFunc(t *testing.T) { + for _, tc := range indexFuncTests { + first := IndexFunc([]byte(tc.in), tc.f.f) + if first != tc.first { + t.Errorf("IndexFunc(%q, %s) = %d; want %d", tc.in, tc.f.name, first, tc.first) + } + last := LastIndexFunc([]byte(tc.in), tc.f.f) + if last != tc.last { + t.Errorf("LastIndexFunc(%q, %s) = %d; want %d", tc.in, tc.f.name, last, tc.last) + } + } +} + +type ReplaceTest struct { + in string + old, new string + n int + out string +} + +var ReplaceTests = []ReplaceTest{ + {"hello", "l", "L", 0, "hello"}, + {"hello", "l", "L", -1, "heLLo"}, + {"hello", "x", "X", -1, "hello"}, + {"", "x", "X", -1, ""}, + {"radar", "r", "", -1, "ada"}, + {"", "", "<>", -1, "<>"}, + {"banana", "a", "<>", -1, "b<>n<>n<>"}, + {"banana", "a", "<>", 1, "b<>nana"}, + {"banana", "a", "<>", 1000, "b<>n<>n<>"}, + {"banana", "an", "<>", -1, "b<><>a"}, + {"banana", "ana", "<>", -1, "b<>na"}, + {"banana", "", "<>", -1, "<>b<>a<>n<>a<>n<>a<>"}, + {"banana", "", "<>", 10, "<>b<>a<>n<>a<>n<>a<>"}, + {"banana", "", "<>", 6, "<>b<>a<>n<>a<>n<>a"}, + {"banana", "", "<>", 5, "<>b<>a<>n<>a<>na"}, + {"banana", "", "<>", 1, "<>banana"}, + {"banana", "a", "a", -1, "banana"}, + {"banana", "a", "a", 1, "banana"}, + {"☺☻☹", "", "<>", -1, "<>☺<>☻<>☹<>"}, +} + +func TestReplace(t *testing.T) { + for _, tt := range ReplaceTests { + if s := string(Replace([]byte(tt.in), []byte(tt.old), []byte(tt.new), tt.n)); s != tt.out { + t.Errorf("Replace(%q, %q, %q, %d) = %q, want %q", tt.in, tt.old, tt.new, tt.n, s, tt.out) + } + } +} + +type TitleTest struct { + in, out string +} + +var TitleTests = []TitleTest{ + {"", ""}, + {"a", "A"}, + {" aaa aaa aaa ", " Aaa Aaa Aaa "}, + {" Aaa Aaa Aaa ", " Aaa Aaa Aaa "}, + {"123a456", "123a456"}, + {"double-blind", "Double-Blind"}, + {"ÿøû", "Ÿøû"}, +} + +func TestTitle(t *testing.T) { + for _, tt := range TitleTests { + if s := string(Title([]byte(tt.in))); s != tt.out { + t.Errorf("Title(%q) = %q, want %q", tt.in, s, tt.out) + } + } +} diff --git a/src/pkg/bytes/export_test.go b/src/pkg/bytes/export_test.go new file mode 100644 index 000000000..b65428d9c --- /dev/null +++ b/src/pkg/bytes/export_test.go @@ -0,0 +1,8 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package bytes + +// Export func for testing +var IndexBytePortable = indexBytePortable diff --git a/src/pkg/cmath/Makefile b/src/pkg/cmath/Makefile new file mode 100644 index 000000000..486caace4 --- /dev/null +++ b/src/pkg/cmath/Makefile @@ -0,0 +1,25 @@ +# Copyright 2010 The Go Authors. All rights reserved. +# Use of this source code is governed by a BSD-style +# license that can be found in the LICENSE file. + +include ../../Make.inc + +TARG=cmath + +GOFILES=\ + abs.go\ + asin.go\ + conj.go\ + exp.go\ + isinf.go\ + isnan.go\ + log.go\ + phase.go\ + polar.go\ + pow.go\ + rect.go\ + sin.go\ + sqrt.go\ + tan.go\ + +include ../../Make.pkg diff --git a/src/pkg/cmath/abs.go b/src/pkg/cmath/abs.go new file mode 100644 index 000000000..f3199cad5 --- /dev/null +++ b/src/pkg/cmath/abs.go @@ -0,0 +1,12 @@ +// Copyright 2010 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Package cmath provides basic constants and mathematical functions for +// complex numbers. +package cmath + +import "math" + +// Abs returns the absolute value (also called the modulus) of x. +func Abs(x complex128) float64 { return math.Hypot(real(x), imag(x)) } diff --git a/src/pkg/cmath/asin.go b/src/pkg/cmath/asin.go new file mode 100644 index 000000000..d6a3ca480 --- /dev/null +++ b/src/pkg/cmath/asin.go @@ -0,0 +1,170 @@ +// Copyright 2010 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package cmath + +import "math" + +// The original C code, the long comment, and the constants +// below are from http://netlib.sandia.gov/cephes/c9x-complex/clog.c. +// The go code is a simplified version of the original C. +// +// Cephes Math Library Release 2.8: June, 2000 +// Copyright 1984, 1987, 1989, 1992, 2000 by Stephen L. Moshier +// +// The readme file at http://netlib.sandia.gov/cephes/ says: +// Some software in this archive may be from the book _Methods and +// Programs for Mathematical Functions_ (Prentice-Hall or Simon & Schuster +// International, 1989) or from the Cephes Mathematical Library, a +// commercial product. In either event, it is copyrighted by the author. +// What you see here may be used freely but it comes with no support or +// guarantee. +// +// The two known misprints in the book are repaired here in the +// source listings for the gamma function and the incomplete beta +// integral. +// +// Stephen L. Moshier +// moshier@na-net.ornl.gov + +// Complex circular arc sine +// +// DESCRIPTION: +// +// Inverse complex sine: +// 2 +// w = -i clog( iz + csqrt( 1 - z ) ). +// +// casin(z) = -i casinh(iz) +// +// ACCURACY: +// +// Relative error: +// arithmetic domain # trials peak rms +// DEC -10,+10 10100 2.1e-15 3.4e-16 +// IEEE -10,+10 30000 2.2e-14 2.7e-15 +// Larger relative error can be observed for z near zero. +// Also tested by csin(casin(z)) = z. + +// Asin returns the inverse sine of x. +func Asin(x complex128) complex128 { + if imag(x) == 0 { + if math.Fabs(real(x)) > 1 { + return complex(math.Pi/2, 0) // DOMAIN error + } + return complex(math.Asin(real(x)), 0) + } + ct := complex(-imag(x), real(x)) // i * x + xx := x * x + x1 := complex(1-real(xx), -imag(xx)) // 1 - x*x + x2 := Sqrt(x1) // x2 = sqrt(1 - x*x) + w := Log(ct + x2) + return complex(imag(w), -real(w)) // -i * w +} + +// Asinh returns the inverse hyperbolic sine of x. +func Asinh(x complex128) complex128 { + // TODO check range + if imag(x) == 0 { + if math.Fabs(real(x)) > 1 { + return complex(math.Pi/2, 0) // DOMAIN error + } + return complex(math.Asinh(real(x)), 0) + } + xx := x * x + x1 := complex(1+real(xx), imag(xx)) // 1 + x*x + return Log(x + Sqrt(x1)) // log(x + sqrt(1 + x*x)) +} + +// Complex circular arc cosine +// +// DESCRIPTION: +// +// w = arccos z = PI/2 - arcsin z. +// +// ACCURACY: +// +// Relative error: +// arithmetic domain # trials peak rms +// DEC -10,+10 5200 1.6e-15 2.8e-16 +// IEEE -10,+10 30000 1.8e-14 2.2e-15 + +// Acos returns the inverse cosine of x. +func Acos(x complex128) complex128 { + w := Asin(x) + return complex(math.Pi/2-real(w), -imag(w)) +} + +// Acosh returns the inverse hyperbolic cosine of x. +func Acosh(x complex128) complex128 { + w := Acos(x) + if imag(w) <= 0 { + return complex(-imag(w), real(w)) // i * w + } + return complex(imag(w), -real(w)) // -i * w +} + +// Complex circular arc tangent +// +// DESCRIPTION: +// +// If +// z = x + iy, +// +// then +// 1 ( 2x ) +// Re w = - arctan(-----------) + k PI +// 2 ( 2 2) +// (1 - x - y ) +// +// ( 2 2) +// 1 (x + (y+1) ) +// Im w = - log(------------) +// 4 ( 2 2) +// (x + (y-1) ) +// +// Where k is an arbitrary integer. +// +// catan(z) = -i catanh(iz). +// +// ACCURACY: +// +// Relative error: +// arithmetic domain # trials peak rms +// DEC -10,+10 5900 1.3e-16 7.8e-18 +// IEEE -10,+10 30000 2.3e-15 8.5e-17 +// The check catan( ctan(z) ) = z, with |x| and |y| < PI/2, +// had peak relative error 1.5e-16, rms relative error +// 2.9e-17. See also clog(). + +// Atan returns the inverse tangent of x. +func Atan(x complex128) complex128 { + if real(x) == 0 && imag(x) > 1 { + return NaN() + } + + x2 := real(x) * real(x) + a := 1 - x2 - imag(x)*imag(x) + if a == 0 { + return NaN() + } + t := 0.5 * math.Atan2(2*real(x), a) + w := reducePi(t) + + t = imag(x) - 1 + b := x2 + t*t + if b == 0 { + return NaN() + } + t = imag(x) + 1 + c := (x2 + t*t) / b + return complex(w, 0.25*math.Log(c)) +} + +// Atanh returns the inverse hyperbolic tangent of x. +func Atanh(x complex128) complex128 { + z := complex(-imag(x), real(x)) // z = i * x + z = Atan(z) + return complex(imag(z), -real(z)) // z = -i * z +} diff --git a/src/pkg/cmath/cmath_test.go b/src/pkg/cmath/cmath_test.go new file mode 100644 index 000000000..6a595b0a6 --- /dev/null +++ b/src/pkg/cmath/cmath_test.go @@ -0,0 +1,853 @@ +// Copyright 2010 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package cmath + +import ( + "math" + "testing" +) + +var vc26 = []complex128{ + (4.97901192488367350108546816 + 7.73887247457810456552351752i), + (7.73887247457810456552351752 - 0.27688005719200159404635997i), + (-0.27688005719200159404635997 - 5.01060361827107492160848778i), + (-5.01060361827107492160848778 + 9.63629370719841737980004837i), + (9.63629370719841737980004837 + 2.92637723924396464525443662i), + (2.92637723924396464525443662 + 5.22908343145930665230025625i), + (5.22908343145930665230025625 + 2.72793991043601025126008608i), + (2.72793991043601025126008608 + 1.82530809168085506044576505i), + (1.82530809168085506044576505 - 8.68592476857560136238589621i), + (-8.68592476857560136238589621 + 4.97901192488367350108546816i), +} +var vc = []complex128{ + (4.9790119248836735e+00 + 7.7388724745781045e+00i), + (7.7388724745781045e+00 - 2.7688005719200159e-01i), + (-2.7688005719200159e-01 - 5.0106036182710749e+00i), + (-5.0106036182710749e+00 + 9.6362937071984173e+00i), + (9.6362937071984173e+00 + 2.9263772392439646e+00i), + (2.9263772392439646e+00 + 5.2290834314593066e+00i), + (5.2290834314593066e+00 + 2.7279399104360102e+00i), + (2.7279399104360102e+00 + 1.8253080916808550e+00i), + (1.8253080916808550e+00 - 8.6859247685756013e+00i), + (-8.6859247685756013e+00 + 4.9790119248836735e+00i), +} + +// The expected results below were computed by the high precision calculators +// at http://keisan.casio.com/. More exact input values (array vc[], above) +// were obtained by printing them with "%.26f". The answers were calculated +// to 26 digits (by using the "Digit number" drop-down control of each +// calculator). + +var abs = []float64{ + 9.2022120669932650313380972e+00, + 7.7438239742296106616261394e+00, + 5.0182478202557746902556648e+00, + 1.0861137372799545160704002e+01, + 1.0070841084922199607011905e+01, + 5.9922447613166942183705192e+00, + 5.8978784056736762299945176e+00, + 3.2822866700678709020367184e+00, + 8.8756430028990417290744307e+00, + 1.0011785496777731986390856e+01, +} + +var acos = []complex128{ + (1.0017679804707456328694569 - 2.9138232718554953784519807i), + (0.03606427612041407369636057 + 2.7358584434576260925091256i), + (1.6249365462333796703711823 + 2.3159537454335901187730929i), + (2.0485650849650740120660391 - 3.0795576791204117911123886i), + (0.29621132089073067282488147 - 3.0007392508200622519398814i), + (1.0664555914934156601503632 - 2.4872865024796011364747111i), + (0.48681307452231387690013905 - 2.463655912283054555225301i), + (0.6116977071277574248407752 - 1.8734458851737055262693056i), + (1.3649311280370181331184214 + 2.8793528632328795424123832i), + (2.6189310485682988308904501 - 2.9956543302898767795858704i), +} +var acosh = []complex128{ + (2.9138232718554953784519807 + 1.0017679804707456328694569i), + (2.7358584434576260925091256 - 0.03606427612041407369636057i), + (2.3159537454335901187730929 - 1.6249365462333796703711823i), + (3.0795576791204117911123886 + 2.0485650849650740120660391i), + (3.0007392508200622519398814 + 0.29621132089073067282488147i), + (2.4872865024796011364747111 + 1.0664555914934156601503632i), + (2.463655912283054555225301 + 0.48681307452231387690013905i), + (1.8734458851737055262693056 + 0.6116977071277574248407752i), + (2.8793528632328795424123832 - 1.3649311280370181331184214i), + (2.9956543302898767795858704 + 2.6189310485682988308904501i), +} +var asin = []complex128{ + (0.56902834632415098636186476 + 2.9138232718554953784519807i), + (1.5347320506744825455349611 - 2.7358584434576260925091256i), + (-0.054140219438483051139860579 - 2.3159537454335901187730929i), + (-0.47776875817017739283471738 + 3.0795576791204117911123886i), + (1.2745850059041659464064402 + 3.0007392508200622519398814i), + (0.50434073530148095908095852 + 2.4872865024796011364747111i), + (1.0839832522725827423311826 + 2.463655912283054555225301i), + (0.9590986196671391943905465 + 1.8734458851737055262693056i), + (0.20586519875787848611290031 - 2.8793528632328795424123832i), + (-1.0481347217734022116591284 + 2.9956543302898767795858704i), +} +var asinh = []complex128{ + (2.9113760469415295679342185 + 0.99639459545704326759805893i), + (2.7441755423994259061579029 - 0.035468308789000500601119392i), + (-2.2962136462520690506126678 - 1.5144663565690151885726707i), + (-3.0771233459295725965402455 + 1.0895577967194013849422294i), + (3.0048366100923647417557027 + 0.29346979169819220036454168i), + (2.4800059370795363157364643 + 1.0545868606049165710424232i), + (2.4718773838309585611141821 + 0.47502344364250803363708842i), + (1.8910743588080159144378396 + 0.56882925572563602341139174i), + (2.8735426423367341878069406 - 1.362376149648891420997548i), + (-2.9981750586172477217567878 + 0.5183571985225367505624207i), +} +var atan = []complex128{ + (1.5115747079332741358607654 + 0.091324403603954494382276776i), + (1.4424504323482602560806727 - 0.0045416132642803911503770933i), + (-1.5593488703630532674484026 - 0.20163295409248362456446431i), + (-1.5280619472445889867794105 + 0.081721556230672003746956324i), + (1.4759909163240799678221039 + 0.028602969320691644358773586i), + (1.4877353772046548932715555 + 0.14566877153207281663773599i), + (1.4206983927779191889826 + 0.076830486127880702249439993i), + (1.3162236060498933364869556 + 0.16031313000467530644933363i), + (1.5473450684303703578810093 - 0.11064907507939082484935782i), + (-1.4841462340185253987375812 + 0.049341850305024399493142411i), +} +var atanh = []complex128{ + (0.058375027938968509064640438 + 1.4793488495105334458167782i), + (0.12977343497790381229915667 - 1.5661009410463561327262499i), + (-0.010576456067347252072200088 - 1.3743698658402284549750563i), + (-0.042218595678688358882784918 + 1.4891433968166405606692604i), + (0.095218997991316722061828397 + 1.5416884098777110330499698i), + (0.079965459366890323857556487 + 1.4252510353873192700350435i), + (0.15051245471980726221708301 + 1.4907432533016303804884461i), + (0.25082072933993987714470373 + 1.392057665392187516442986i), + (0.022896108815797135846276662 - 1.4609224989282864208963021i), + (-0.08665624101841876130537396 + 1.5207902036935093480142159i), +} +var conj = []complex128{ + (4.9790119248836735e+00 - 7.7388724745781045e+00i), + (7.7388724745781045e+00 + 2.7688005719200159e-01i), + (-2.7688005719200159e-01 + 5.0106036182710749e+00i), + (-5.0106036182710749e+00 - 9.6362937071984173e+00i), + (9.6362937071984173e+00 - 2.9263772392439646e+00i), + (2.9263772392439646e+00 - 5.2290834314593066e+00i), + (5.2290834314593066e+00 - 2.7279399104360102e+00i), + (2.7279399104360102e+00 - 1.8253080916808550e+00i), + (1.8253080916808550e+00 + 8.6859247685756013e+00i), + (-8.6859247685756013e+00 - 4.9790119248836735e+00i), +} +var cos = []complex128{ + (3.024540920601483938336569e+02 + 1.1073797572517071650045357e+03i), + (1.192858682649064973252758e-01 + 2.7857554122333065540970207e-01i), + (7.2144394304528306603857962e+01 - 2.0500129667076044169954205e+01i), + (2.24921952538403984190541e+03 - 7.317363745602773587049329e+03i), + (-9.148222970032421760015498e+00 + 1.953124661113563541862227e+00i), + (-9.116081175857732248227078e+01 - 1.992669213569952232487371e+01i), + (3.795639179042704640002918e+00 + 6.623513350981458399309662e+00i), + (-2.9144840732498869560679084e+00 - 1.214620271628002917638748e+00i), + (-7.45123482501299743872481e+02 + 2.8641692314488080814066734e+03i), + (-5.371977967039319076416747e+01 + 4.893348341339375830564624e+01i), +} +var cosh = []complex128{ + (8.34638383523018249366948e+00 + 7.2181057886425846415112064e+01i), + (1.10421967379919366952251e+03 - 3.1379638689277575379469861e+02i), + (3.051485206773701584738512e-01 - 2.6805384730105297848044485e-01i), + (-7.33294728684187933370938e+01 + 1.574445942284918251038144e+01i), + (-7.478643293945957535757355e+03 + 1.6348382209913353929473321e+03i), + (4.622316522966235701630926e+00 - 8.088695185566375256093098e+00i), + (-8.544333183278877406197712e+01 + 3.7505836120128166455231717e+01i), + (-1.934457815021493925115198e+00 + 7.3725859611767228178358673e+00i), + (-2.352958770061749348353548e+00 - 2.034982010440878358915409e+00i), + (7.79756457532134748165069e+02 + 2.8549350716819176560377717e+03i), +} +var exp = []complex128{ + (1.669197736864670815125146e+01 + 1.4436895109507663689174096e+02i), + (2.2084389286252583447276212e+03 - 6.2759289284909211238261917e+02i), + (2.227538273122775173434327e-01 + 7.2468284028334191250470034e-01i), + (-6.5182985958153548997881627e-03 - 1.39965837915193860879044e-03i), + (-1.4957286524084015746110777e+04 + 3.269676455931135688988042e+03i), + (9.218158701983105935659273e+00 - 1.6223985291084956009304582e+01i), + (-1.7088175716853040841444505e+02 + 7.501382609870410713795546e+01i), + (-3.852461315830959613132505e+00 + 1.4808420423156073221970892e+01i), + (-4.586775503301407379786695e+00 - 4.178501081246873415144744e+00i), + (4.451337963005453491095747e-05 - 1.62977574205442915935263e-04i), +} +var log = []complex128{ + (2.2194438972179194425697051e+00 + 9.9909115046919291062461269e-01i), + (2.0468956191154167256337289e+00 - 3.5762575021856971295156489e-02i), + (1.6130808329853860438751244e+00 - 1.6259990074019058442232221e+00i), + (2.3851910394823008710032651e+00 + 2.0502936359659111755031062e+00i), + (2.3096442270679923004800651e+00 + 2.9483213155446756211881774e-01i), + (1.7904660933974656106951860e+00 + 1.0605860367252556281902109e+00i), + (1.7745926939841751666177512e+00 + 4.8084556083358307819310911e-01i), + (1.1885403350045342425648780e+00 + 5.8969634164776659423195222e-01i), + (2.1833107837679082586772505e+00 - 1.3636647724582455028314573e+00i), + (2.3037629487273259170991671e+00 + 2.6210913895386013290915234e+00i), +} +var log10 = []complex128{ + (9.6389223745559042474184943e-01 + 4.338997735671419492599631e-01i), + (8.8895547241376579493490892e-01 - 1.5531488990643548254864806e-02i), + (7.0055210462945412305244578e-01 - 7.0616239649481243222248404e-01i), + (1.0358753067322445311676952e+00 + 8.9043121238134980156490909e-01i), + (1.003065742975330237172029e+00 + 1.2804396782187887479857811e-01i), + (7.7758954439739162532085157e-01 + 4.6060666333341810869055108e-01i), + (7.7069581462315327037689152e-01 + 2.0882857371769952195512475e-01i), + (5.1617650901191156135137239e-01 + 2.5610186717615977620363299e-01i), + (9.4819982567026639742663212e-01 - 5.9223208584446952284914289e-01i), + (1.0005115362454417135973429e+00 + 1.1383255270407412817250921e+00i), +} + +type ff struct { + r, theta float64 +} + +var polar = []ff{ + {9.2022120669932650313380972e+00, 9.9909115046919291062461269e-01}, + {7.7438239742296106616261394e+00, -3.5762575021856971295156489e-02}, + {5.0182478202557746902556648e+00, -1.6259990074019058442232221e+00}, + {1.0861137372799545160704002e+01, 2.0502936359659111755031062e+00}, + {1.0070841084922199607011905e+01, 2.9483213155446756211881774e-01}, + {5.9922447613166942183705192e+00, 1.0605860367252556281902109e+00}, + {5.8978784056736762299945176e+00, 4.8084556083358307819310911e-01}, + {3.2822866700678709020367184e+00, 5.8969634164776659423195222e-01}, + {8.8756430028990417290744307e+00, -1.3636647724582455028314573e+00}, + {1.0011785496777731986390856e+01, 2.6210913895386013290915234e+00}, +} +var pow = []complex128{ + (-2.499956739197529585028819e+00 + 1.759751724335650228957144e+00i), + (7.357094338218116311191939e+04 - 5.089973412479151648145882e+04i), + (1.320777296067768517259592e+01 - 3.165621914333901498921986e+01i), + (-3.123287828297300934072149e-07 - 1.9849567521490553032502223E-7i), + (8.0622651468477229614813e+04 - 7.80028727944573092944363e+04i), + (-1.0268824572103165858577141e+00 - 4.716844738244989776610672e-01i), + (-4.35953819012244175753187e+01 + 2.2036445974645306917648585e+02i), + (8.3556092283250594950239e-01 - 1.2261571947167240272593282e+01i), + (1.582292972120769306069625e+03 + 1.273564263524278244782512e+04i), + (6.592208301642122149025369e-08 + 2.584887236651661903526389e-08i), +} +var sin = []complex128{ + (-1.1073801774240233539648544e+03 + 3.024539773002502192425231e+02i), + (1.0317037521400759359744682e+00 - 3.2208979799929570242818e-02i), + (-2.0501952097271429804261058e+01 - 7.2137981348240798841800967e+01i), + (7.3173638080346338642193078e+03 + 2.249219506193664342566248e+03i), + (-1.964375633631808177565226e+00 - 9.0958264713870404464159683e+00i), + (1.992783647158514838337674e+01 - 9.11555769410191350416942e+01i), + (-6.680335650741921444300349e+00 + 3.763353833142432513086117e+00i), + (1.2794028166657459148245993e+00 - 2.7669092099795781155109602e+00i), + (2.8641693949535259594188879e+03 + 7.451234399649871202841615e+02i), + (-4.893811726244659135553033e+01 - 5.371469305562194635957655e+01i), +} +var sinh = []complex128{ + (8.34559353341652565758198e+00 + 7.2187893208650790476628899e+01i), + (1.1042192548260646752051112e+03 - 3.1379650595631635858792056e+02i), + (-8.239469336509264113041849e-02 + 9.9273668758439489098514519e-01i), + (7.332295456982297798219401e+01 - 1.574585908122833444899023e+01i), + (-7.4786432301380582103534216e+03 + 1.63483823493980029604071e+03i), + (4.595842179016870234028347e+00 - 8.135290105518580753211484e+00i), + (-8.543842533574163435246793e+01 + 3.750798997857594068272375e+01i), + (-1.918003500809465688017307e+00 + 7.4358344619793504041350251e+00i), + (-2.233816733239658031433147e+00 - 2.143519070805995056229335e+00i), + (-7.797564130187551181105341e+02 - 2.8549352346594918614806877e+03i), +} +var sqrt = []complex128{ + (2.6628203086086130543813948e+00 + 1.4531345674282185229796902e+00i), + (2.7823278427251986247149295e+00 - 4.9756907317005224529115567e-02i), + (1.5397025302089642757361015e+00 - 1.6271336573016637535695727e+00i), + (1.7103411581506875260277898e+00 + 2.8170677122737589676157029e+00i), + (3.1390392472953103383607947e+00 + 4.6612625849858653248980849e-01i), + (2.1117080764822417640789287e+00 + 1.2381170223514273234967850e+00i), + (2.3587032281672256703926939e+00 + 5.7827111903257349935720172e-01i), + (1.7335262588873410476661577e+00 + 5.2647258220721269141550382e-01i), + (2.3131094974708716531499282e+00 - 1.8775429304303785570775490e+00i), + (8.1420535745048086240947359e-01 + 3.0575897587277248522656113e+00i), +} +var tan = []complex128{ + (-1.928757919086441129134525e-07 + 1.0000003267499169073251826e+00i), + (1.242412685364183792138948e+00 - 3.17149693883133370106696e+00i), + (-4.6745126251587795225571826e-05 - 9.9992439225263959286114298e-01i), + (4.792363401193648192887116e-09 + 1.0000000070589333451557723e+00i), + (2.345740824080089140287315e-03 + 9.947733046570988661022763e-01i), + (-2.396030789494815566088809e-05 + 9.9994781345418591429826779e-01i), + (-7.370204836644931340905303e-03 + 1.0043553413417138987717748e+00i), + (-3.691803847992048527007457e-02 + 9.6475071993469548066328894e-01i), + (-2.781955256713729368401878e-08 - 1.000000049848910609006646e+00i), + (9.4281590064030478879791249e-05 + 9.9999119340863718183758545e-01i), +} +var tanh = []complex128{ + (1.0000921981225144748819918e+00 + 2.160986245871518020231507e-05i), + (9.9999967727531993209562591e-01 - 1.9953763222959658873657676e-07i), + (-1.765485739548037260789686e+00 + 1.7024216325552852445168471e+00i), + (-9.999189442732736452807108e-01 + 3.64906070494473701938098e-05i), + (9.9999999224622333738729767e-01 - 3.560088949517914774813046e-09i), + (1.0029324933367326862499343e+00 - 4.948790309797102353137528e-03i), + (9.9996113064788012488693567e-01 - 4.226995742097032481451259e-05i), + (1.0074784189316340029873945e+00 - 4.194050814891697808029407e-03i), + (9.9385534229718327109131502e-01 + 5.144217985914355502713437e-02i), + (-1.0000000491604982429364892e+00 - 2.901873195374433112227349e-08i), +} + +// special cases +var vcAbsSC = []complex128{ + NaN(), +} +var absSC = []float64{ + math.NaN(), +} +var vcAcosSC = []complex128{ + NaN(), +} +var acosSC = []complex128{ + NaN(), +} +var vcAcoshSC = []complex128{ + NaN(), +} +var acoshSC = []complex128{ + NaN(), +} +var vcAsinSC = []complex128{ + NaN(), +} +var asinSC = []complex128{ + NaN(), +} +var vcAsinhSC = []complex128{ + NaN(), +} +var asinhSC = []complex128{ + NaN(), +} +var vcAtanSC = []complex128{ + NaN(), +} +var atanSC = []complex128{ + NaN(), +} +var vcAtanhSC = []complex128{ + NaN(), +} +var atanhSC = []complex128{ + NaN(), +} +var vcConjSC = []complex128{ + NaN(), +} +var conjSC = []complex128{ + NaN(), +} +var vcCosSC = []complex128{ + NaN(), +} +var cosSC = []complex128{ + NaN(), +} +var vcCoshSC = []complex128{ + NaN(), +} +var coshSC = []complex128{ + NaN(), +} +var vcExpSC = []complex128{ + NaN(), +} +var expSC = []complex128{ + NaN(), +} +var vcIsNaNSC = []complex128{ + complex(math.Inf(-1), math.Inf(-1)), + complex(math.Inf(-1), math.NaN()), + complex(math.NaN(), math.Inf(-1)), + complex(0, math.NaN()), + complex(math.NaN(), 0), + complex(math.Inf(1), math.Inf(1)), + complex(math.Inf(1), math.NaN()), + complex(math.NaN(), math.Inf(1)), + complex(math.NaN(), math.NaN()), +} +var isNaNSC = []bool{ + false, + false, + false, + true, + true, + false, + false, + false, + true, +} +var vcLogSC = []complex128{ + NaN(), +} +var logSC = []complex128{ + NaN(), +} +var vcLog10SC = []complex128{ + NaN(), +} +var log10SC = []complex128{ + NaN(), +} +var vcPolarSC = []complex128{ + NaN(), +} +var polarSC = []ff{ + {math.NaN(), math.NaN()}, +} +var vcPowSC = [][2]complex128{ + {NaN(), NaN()}, +} +var powSC = []complex128{ + NaN(), +} +var vcSinSC = []complex128{ + NaN(), +} +var sinSC = []complex128{ + NaN(), +} +var vcSinhSC = []complex128{ + NaN(), +} +var sinhSC = []complex128{ + NaN(), +} +var vcSqrtSC = []complex128{ + NaN(), +} +var sqrtSC = []complex128{ + NaN(), +} +var vcTanSC = []complex128{ + NaN(), +} +var tanSC = []complex128{ + NaN(), +} +var vcTanhSC = []complex128{ + NaN(), +} +var tanhSC = []complex128{ + NaN(), +} + +// functions borrowed from pkg/math/all_test.go +func tolerance(a, b, e float64) bool { + d := a - b + if d < 0 { + d = -d + } + + if a != 0 { + e = e * a + if e < 0 { + e = -e + } + } + return d < e +} +func soclose(a, b, e float64) bool { return tolerance(a, b, e) } +func veryclose(a, b float64) bool { return tolerance(a, b, 4e-16) } +func alike(a, b float64) bool { + switch { + case a != a && b != b: // math.IsNaN(a) && math.IsNaN(b): + return true + case a == b: + return math.Signbit(a) == math.Signbit(b) + } + return false +} + +func cTolerance(a, b complex128, e float64) bool { + d := Abs(a - b) + if a != 0 { + e = e * Abs(a) + if e < 0 { + e = -e + } + } + return d < e +} +func cSoclose(a, b complex128, e float64) bool { return cTolerance(a, b, e) } +func cVeryclose(a, b complex128) bool { return cTolerance(a, b, 4e-16) } +func cAlike(a, b complex128) bool { + switch { + case IsNaN(a) && IsNaN(b): + return true + case a == b: + return math.Signbit(real(a)) == math.Signbit(real(b)) && math.Signbit(imag(a)) == math.Signbit(imag(b)) + } + return false +} + +func TestAbs(t *testing.T) { + for i := 0; i < len(vc); i++ { + if f := Abs(vc[i]); !veryclose(abs[i], f) { + t.Errorf("Abs(%g) = %g, want %g", vc[i], f, abs[i]) + } + } + for i := 0; i < len(vcAbsSC); i++ { + if f := Abs(vcAbsSC[i]); !alike(absSC[i], f) { + t.Errorf("Abs(%g) = %g, want %g", vcAbsSC[i], f, absSC[i]) + } + } +} +func TestAcos(t *testing.T) { + for i := 0; i < len(vc); i++ { + if f := Acos(vc[i]); !cSoclose(acos[i], f, 1e-14) { + t.Errorf("Acos(%g) = %g, want %g", vc[i], f, acos[i]) + } + } + for i := 0; i < len(vcAcosSC); i++ { + if f := Acos(vcAcosSC[i]); !cAlike(acosSC[i], f) { + t.Errorf("Acos(%g) = %g, want %g", vcAcosSC[i], f, acosSC[i]) + } + } +} +func TestAcosh(t *testing.T) { + for i := 0; i < len(vc); i++ { + if f := Acosh(vc[i]); !cSoclose(acosh[i], f, 1e-14) { + t.Errorf("Acosh(%g) = %g, want %g", vc[i], f, acosh[i]) + } + } + for i := 0; i < len(vcAcoshSC); i++ { + if f := Acosh(vcAcoshSC[i]); !cAlike(acoshSC[i], f) { + t.Errorf("Acosh(%g) = %g, want %g", vcAcoshSC[i], f, acoshSC[i]) + } + } +} +func TestAsin(t *testing.T) { + for i := 0; i < len(vc); i++ { + if f := Asin(vc[i]); !cSoclose(asin[i], f, 1e-14) { + t.Errorf("Asin(%g) = %g, want %g", vc[i], f, asin[i]) + } + } + for i := 0; i < len(vcAsinSC); i++ { + if f := Asin(vcAsinSC[i]); !cAlike(asinSC[i], f) { + t.Errorf("Asin(%g) = %g, want %g", vcAsinSC[i], f, asinSC[i]) + } + } +} +func TestAsinh(t *testing.T) { + for i := 0; i < len(vc); i++ { + if f := Asinh(vc[i]); !cSoclose(asinh[i], f, 4e-15) { + t.Errorf("Asinh(%g) = %g, want %g", vc[i], f, asinh[i]) + } + } + for i := 0; i < len(vcAsinhSC); i++ { + if f := Asinh(vcAsinhSC[i]); !cAlike(asinhSC[i], f) { + t.Errorf("Asinh(%g) = %g, want %g", vcAsinhSC[i], f, asinhSC[i]) + } + } +} +func TestAtan(t *testing.T) { + for i := 0; i < len(vc); i++ { + if f := Atan(vc[i]); !cVeryclose(atan[i], f) { + t.Errorf("Atan(%g) = %g, want %g", vc[i], f, atan[i]) + } + } + for i := 0; i < len(vcAtanSC); i++ { + if f := Atan(vcAtanSC[i]); !cAlike(atanSC[i], f) { + t.Errorf("Atan(%g) = %g, want %g", vcAtanSC[i], f, atanSC[i]) + } + } +} +func TestAtanh(t *testing.T) { + for i := 0; i < len(vc); i++ { + if f := Atanh(vc[i]); !cVeryclose(atanh[i], f) { + t.Errorf("Atanh(%g) = %g, want %g", vc[i], f, atanh[i]) + } + } + for i := 0; i < len(vcAtanhSC); i++ { + if f := Atanh(vcAtanhSC[i]); !cAlike(atanhSC[i], f) { + t.Errorf("Atanh(%g) = %g, want %g", vcAtanhSC[i], f, atanhSC[i]) + } + } +} +func TestConj(t *testing.T) { + for i := 0; i < len(vc); i++ { + if f := Conj(vc[i]); !cVeryclose(conj[i], f) { + t.Errorf("Conj(%g) = %g, want %g", vc[i], f, conj[i]) + } + } + for i := 0; i < len(vcConjSC); i++ { + if f := Conj(vcConjSC[i]); !cAlike(conjSC[i], f) { + t.Errorf("Conj(%g) = %g, want %g", vcConjSC[i], f, conjSC[i]) + } + } +} +func TestCos(t *testing.T) { + for i := 0; i < len(vc); i++ { + if f := Cos(vc[i]); !cSoclose(cos[i], f, 3e-15) { + t.Errorf("Cos(%g) = %g, want %g", vc[i], f, cos[i]) + } + } + for i := 0; i < len(vcCosSC); i++ { + if f := Cos(vcCosSC[i]); !cAlike(cosSC[i], f) { + t.Errorf("Cos(%g) = %g, want %g", vcCosSC[i], f, cosSC[i]) + } + } +} +func TestCosh(t *testing.T) { + for i := 0; i < len(vc); i++ { + if f := Cosh(vc[i]); !cSoclose(cosh[i], f, 2e-15) { + t.Errorf("Cosh(%g) = %g, want %g", vc[i], f, cosh[i]) + } + } + for i := 0; i < len(vcCoshSC); i++ { + if f := Cosh(vcCoshSC[i]); !cAlike(coshSC[i], f) { + t.Errorf("Cosh(%g) = %g, want %g", vcCoshSC[i], f, coshSC[i]) + } + } +} +func TestExp(t *testing.T) { + for i := 0; i < len(vc); i++ { + if f := Exp(vc[i]); !cSoclose(exp[i], f, 1e-15) { + t.Errorf("Exp(%g) = %g, want %g", vc[i], f, exp[i]) + } + } + for i := 0; i < len(vcExpSC); i++ { + if f := Exp(vcExpSC[i]); !cAlike(expSC[i], f) { + t.Errorf("Exp(%g) = %g, want %g", vcExpSC[i], f, expSC[i]) + } + } +} +func TestIsNaN(t *testing.T) { + for i := 0; i < len(vcIsNaNSC); i++ { + if f := IsNaN(vcIsNaNSC[i]); isNaNSC[i] != f { + t.Errorf("IsNaN(%v) = %v, want %v", vcIsNaNSC[i], f, isNaNSC[i]) + } + } +} +func TestLog(t *testing.T) { + for i := 0; i < len(vc); i++ { + if f := Log(vc[i]); !cVeryclose(log[i], f) { + t.Errorf("Log(%g) = %g, want %g", vc[i], f, log[i]) + } + } + for i := 0; i < len(vcLogSC); i++ { + if f := Log(vcLogSC[i]); !cAlike(logSC[i], f) { + t.Errorf("Log(%g) = %g, want %g", vcLogSC[i], f, logSC[i]) + } + } +} +func TestLog10(t *testing.T) { + for i := 0; i < len(vc); i++ { + if f := Log10(vc[i]); !cVeryclose(log10[i], f) { + t.Errorf("Log10(%g) = %g, want %g", vc[i], f, log10[i]) + } + } + for i := 0; i < len(vcLog10SC); i++ { + if f := Log10(vcLog10SC[i]); !cAlike(log10SC[i], f) { + t.Errorf("Log10(%g) = %g, want %g", vcLog10SC[i], f, log10SC[i]) + } + } +} +func TestPolar(t *testing.T) { + for i := 0; i < len(vc); i++ { + if r, theta := Polar(vc[i]); !veryclose(polar[i].r, r) && !veryclose(polar[i].theta, theta) { + t.Errorf("Polar(%g) = %g, %g want %g, %g", vc[i], r, theta, polar[i].r, polar[i].theta) + } + } + for i := 0; i < len(vcPolarSC); i++ { + if r, theta := Polar(vcPolarSC[i]); !alike(polarSC[i].r, r) && !alike(polarSC[i].theta, theta) { + t.Errorf("Polar(%g) = %g, %g, want %g, %g", vcPolarSC[i], r, theta, polarSC[i].r, polarSC[i].theta) + } + } +} +func TestPow(t *testing.T) { + var a = complex(3.0, 3.0) + for i := 0; i < len(vc); i++ { + if f := Pow(a, vc[i]); !cSoclose(pow[i], f, 4e-15) { + t.Errorf("Pow(%g, %g) = %g, want %g", a, vc[i], f, pow[i]) + } + } + for i := 0; i < len(vcPowSC); i++ { + if f := Pow(vcPowSC[i][0], vcPowSC[i][0]); !cAlike(powSC[i], f) { + t.Errorf("Pow(%g, %g) = %g, want %g", vcPowSC[i][0], vcPowSC[i][0], f, powSC[i]) + } + } +} +func TestRect(t *testing.T) { + for i := 0; i < len(vc); i++ { + if f := Rect(polar[i].r, polar[i].theta); !cVeryclose(vc[i], f) { + t.Errorf("Rect(%g, %g) = %g want %g", polar[i].r, polar[i].theta, f, vc[i]) + } + } + for i := 0; i < len(vcPolarSC); i++ { + if f := Rect(polarSC[i].r, polarSC[i].theta); !cAlike(vcPolarSC[i], f) { + t.Errorf("Rect(%g, %g) = %g, want %g", polarSC[i].r, polarSC[i].theta, f, vcPolarSC[i]) + } + } +} +func TestSin(t *testing.T) { + for i := 0; i < len(vc); i++ { + if f := Sin(vc[i]); !cSoclose(sin[i], f, 2e-15) { + t.Errorf("Sin(%g) = %g, want %g", vc[i], f, sin[i]) + } + } + for i := 0; i < len(vcSinSC); i++ { + if f := Sin(vcSinSC[i]); !cAlike(sinSC[i], f) { + t.Errorf("Sin(%g) = %g, want %g", vcSinSC[i], f, sinSC[i]) + } + } +} +func TestSinh(t *testing.T) { + for i := 0; i < len(vc); i++ { + if f := Sinh(vc[i]); !cSoclose(sinh[i], f, 2e-15) { + t.Errorf("Sinh(%g) = %g, want %g", vc[i], f, sinh[i]) + } + } + for i := 0; i < len(vcSinhSC); i++ { + if f := Sinh(vcSinhSC[i]); !cAlike(sinhSC[i], f) { + t.Errorf("Sinh(%g) = %g, want %g", vcSinhSC[i], f, sinhSC[i]) + } + } +} +func TestSqrt(t *testing.T) { + for i := 0; i < len(vc); i++ { + if f := Sqrt(vc[i]); !cVeryclose(sqrt[i], f) { + t.Errorf("Sqrt(%g) = %g, want %g", vc[i], f, sqrt[i]) + } + } + for i := 0; i < len(vcSqrtSC); i++ { + if f := Sqrt(vcSqrtSC[i]); !cAlike(sqrtSC[i], f) { + t.Errorf("Sqrt(%g) = %g, want %g", vcSqrtSC[i], f, sqrtSC[i]) + } + } +} +func TestTan(t *testing.T) { + for i := 0; i < len(vc); i++ { + if f := Tan(vc[i]); !cSoclose(tan[i], f, 3e-15) { + t.Errorf("Tan(%g) = %g, want %g", vc[i], f, tan[i]) + } + } + for i := 0; i < len(vcTanSC); i++ { + if f := Tan(vcTanSC[i]); !cAlike(tanSC[i], f) { + t.Errorf("Tan(%g) = %g, want %g", vcTanSC[i], f, tanSC[i]) + } + } +} +func TestTanh(t *testing.T) { + for i := 0; i < len(vc); i++ { + if f := Tanh(vc[i]); !cSoclose(tanh[i], f, 2e-15) { + t.Errorf("Tanh(%g) = %g, want %g", vc[i], f, tanh[i]) + } + } + for i := 0; i < len(vcTanhSC); i++ { + if f := Tanh(vcTanhSC[i]); !cAlike(tanhSC[i], f) { + t.Errorf("Tanh(%g) = %g, want %g", vcTanhSC[i], f, tanhSC[i]) + } + } +} + +func BenchmarkAbs(b *testing.B) { + for i := 0; i < b.N; i++ { + Abs(complex(2.5, 3.5)) + } +} +func BenchmarkAcos(b *testing.B) { + for i := 0; i < b.N; i++ { + Acos(complex(2.5, 3.5)) + } +} +func BenchmarkAcosh(b *testing.B) { + for i := 0; i < b.N; i++ { + Acosh(complex(2.5, 3.5)) + } +} +func BenchmarkAsin(b *testing.B) { + for i := 0; i < b.N; i++ { + Asin(complex(2.5, 3.5)) + } +} +func BenchmarkAsinh(b *testing.B) { + for i := 0; i < b.N; i++ { + Asinh(complex(2.5, 3.5)) + } +} +func BenchmarkAtan(b *testing.B) { + for i := 0; i < b.N; i++ { + Atan(complex(2.5, 3.5)) + } +} +func BenchmarkAtanh(b *testing.B) { + for i := 0; i < b.N; i++ { + Atanh(complex(2.5, 3.5)) + } +} +func BenchmarkConj(b *testing.B) { + for i := 0; i < b.N; i++ { + Conj(complex(2.5, 3.5)) + } +} +func BenchmarkCos(b *testing.B) { + for i := 0; i < b.N; i++ { + Cos(complex(2.5, 3.5)) + } +} +func BenchmarkCosh(b *testing.B) { + for i := 0; i < b.N; i++ { + Cosh(complex(2.5, 3.5)) + } +} +func BenchmarkExp(b *testing.B) { + for i := 0; i < b.N; i++ { + Exp(complex(2.5, 3.5)) + } +} +func BenchmarkLog(b *testing.B) { + for i := 0; i < b.N; i++ { + Log(complex(2.5, 3.5)) + } +} +func BenchmarkLog10(b *testing.B) { + for i := 0; i < b.N; i++ { + Log10(complex(2.5, 3.5)) + } +} +func BenchmarkPhase(b *testing.B) { + for i := 0; i < b.N; i++ { + Phase(complex(2.5, 3.5)) + } +} +func BenchmarkPolar(b *testing.B) { + for i := 0; i < b.N; i++ { + Polar(complex(2.5, 3.5)) + } +} +func BenchmarkPow(b *testing.B) { + for i := 0; i < b.N; i++ { + Pow(complex(2.5, 3.5), complex(2.5, 3.5)) + } +} +func BenchmarkRect(b *testing.B) { + for i := 0; i < b.N; i++ { + Rect(2.5, 1.5) + } +} +func BenchmarkSin(b *testing.B) { + for i := 0; i < b.N; i++ { + Sin(complex(2.5, 3.5)) + } +} +func BenchmarkSinh(b *testing.B) { + for i := 0; i < b.N; i++ { + Sinh(complex(2.5, 3.5)) + } +} +func BenchmarkSqrt(b *testing.B) { + for i := 0; i < b.N; i++ { + Sqrt(complex(2.5, 3.5)) + } +} +func BenchmarkTan(b *testing.B) { + for i := 0; i < b.N; i++ { + Tan(complex(2.5, 3.5)) + } +} +func BenchmarkTanh(b *testing.B) { + for i := 0; i < b.N; i++ { + Tanh(complex(2.5, 3.5)) + } +} diff --git a/src/pkg/cmath/conj.go b/src/pkg/cmath/conj.go new file mode 100644 index 000000000..776b57da7 --- /dev/null +++ b/src/pkg/cmath/conj.go @@ -0,0 +1,8 @@ +// Copyright 2010 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package cmath + +// Conj returns the complex conjugate of x. +func Conj(x complex128) complex128 { return complex(real(x), -imag(x)) } diff --git a/src/pkg/cmath/exp.go b/src/pkg/cmath/exp.go new file mode 100644 index 000000000..64c1ef409 --- /dev/null +++ b/src/pkg/cmath/exp.go @@ -0,0 +1,55 @@ +// Copyright 2010 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package cmath + +import "math" + +// The original C code, the long comment, and the constants +// below are from http://netlib.sandia.gov/cephes/c9x-complex/clog.c. +// The go code is a simplified version of the original C. +// +// Cephes Math Library Release 2.8: June, 2000 +// Copyright 1984, 1987, 1989, 1992, 2000 by Stephen L. Moshier +// +// The readme file at http://netlib.sandia.gov/cephes/ says: +// Some software in this archive may be from the book _Methods and +// Programs for Mathematical Functions_ (Prentice-Hall or Simon & Schuster +// International, 1989) or from the Cephes Mathematical Library, a +// commercial product. In either event, it is copyrighted by the author. +// What you see here may be used freely but it comes with no support or +// guarantee. +// +// The two known misprints in the book are repaired here in the +// source listings for the gamma function and the incomplete beta +// integral. +// +// Stephen L. Moshier +// moshier@na-net.ornl.gov + +// Complex exponential function +// +// DESCRIPTION: +// +// Returns the complex exponential of the complex argument z. +// +// If +// z = x + iy, +// r = exp(x), +// then +// w = r cos y + i r sin y. +// +// ACCURACY: +// +// Relative error: +// arithmetic domain # trials peak rms +// DEC -10,+10 8700 3.7e-17 1.1e-17 +// IEEE -10,+10 30000 3.0e-16 8.7e-17 + +// Exp returns e**x, the base-e exponential of x. +func Exp(x complex128) complex128 { + r := math.Exp(real(x)) + s, c := math.Sincos(imag(x)) + return complex(r*c, r*s) +} diff --git a/src/pkg/cmath/isinf.go b/src/pkg/cmath/isinf.go new file mode 100644 index 000000000..f23d2dea7 --- /dev/null +++ b/src/pkg/cmath/isinf.go @@ -0,0 +1,21 @@ +// Copyright 2010 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package cmath + +import "math" + +// IsInf returns true if either real(x) or imag(x) is an infinity. +func IsInf(x complex128) bool { + if math.IsInf(real(x), 0) || math.IsInf(imag(x), 0) { + return true + } + return false +} + +// Inf returns a complex infinity, complex(+Inf, +Inf). +func Inf() complex128 { + inf := math.Inf(1) + return complex(inf, inf) +} diff --git a/src/pkg/cmath/isnan.go b/src/pkg/cmath/isnan.go new file mode 100644 index 000000000..2063bb835 --- /dev/null +++ b/src/pkg/cmath/isnan.go @@ -0,0 +1,25 @@ +// Copyright 2010 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package cmath + +import "math" + +// IsNaN returns true if either real(x) or imag(x) is NaN +// and neither is an infinity. +func IsNaN(x complex128) bool { + switch { + case math.IsInf(real(x), 0) || math.IsInf(imag(x), 0): + return false + case math.IsNaN(real(x)) || math.IsNaN(imag(x)): + return true + } + return false +} + +// NaN returns a complex ``not-a-number'' value. +func NaN() complex128 { + nan := math.NaN() + return complex(nan, nan) +} diff --git a/src/pkg/cmath/log.go b/src/pkg/cmath/log.go new file mode 100644 index 000000000..8e6964fee --- /dev/null +++ b/src/pkg/cmath/log.go @@ -0,0 +1,64 @@ +// Copyright 2010 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package cmath + +import "math" + +// The original C code, the long comment, and the constants +// below are from http://netlib.sandia.gov/cephes/c9x-complex/clog.c. +// The go code is a simplified version of the original C. +// +// Cephes Math Library Release 2.8: June, 2000 +// Copyright 1984, 1987, 1989, 1992, 2000 by Stephen L. Moshier +// +// The readme file at http://netlib.sandia.gov/cephes/ says: +// Some software in this archive may be from the book _Methods and +// Programs for Mathematical Functions_ (Prentice-Hall or Simon & Schuster +// International, 1989) or from the Cephes Mathematical Library, a +// commercial product. In either event, it is copyrighted by the author. +// What you see here may be used freely but it comes with no support or +// guarantee. +// +// The two known misprints in the book are repaired here in the +// source listings for the gamma function and the incomplete beta +// integral. +// +// Stephen L. Moshier +// moshier@na-net.ornl.gov + +// Complex natural logarithm +// +// DESCRIPTION: +// +// Returns complex logarithm to the base e (2.718...) of +// the complex argument z. +// +// If +// z = x + iy, r = sqrt( x**2 + y**2 ), +// then +// w = log(r) + i arctan(y/x). +// +// The arctangent ranges from -PI to +PI. +// +// ACCURACY: +// +// Relative error: +// arithmetic domain # trials peak rms +// DEC -10,+10 7000 8.5e-17 1.9e-17 +// IEEE -10,+10 30000 5.0e-15 1.1e-16 +// +// Larger relative error can be observed for z near 1 +i0. +// In IEEE arithmetic the peak absolute error is 5.2e-16, rms +// absolute error 1.0e-16. + +// Log returns the natural logarithm of x. +func Log(x complex128) complex128 { + return complex(math.Log(Abs(x)), Phase(x)) +} + +// Log10 returns the decimal logarithm of x. +func Log10(x complex128) complex128 { + return math.Log10E * Log(x) +} diff --git a/src/pkg/cmath/phase.go b/src/pkg/cmath/phase.go new file mode 100644 index 000000000..2d67aa34c --- /dev/null +++ b/src/pkg/cmath/phase.go @@ -0,0 +1,11 @@ +// Copyright 2010 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package cmath + +import "math" + +// Phase returns the phase (also called the argument) of x. +// The returned value is in the range [-Pi, Pi]. +func Phase(x complex128) float64 { return math.Atan2(imag(x), real(x)) } diff --git a/src/pkg/cmath/polar.go b/src/pkg/cmath/polar.go new file mode 100644 index 000000000..033676acc --- /dev/null +++ b/src/pkg/cmath/polar.go @@ -0,0 +1,12 @@ +// Copyright 2010 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package cmath + +// Polar returns the absolute value r and phase θ of x, +// such that x = r * e**θi. +// The phase is in the range [-Pi, Pi]. +func Polar(x complex128) (r, θ float64) { + return Abs(x), Phase(x) +} diff --git a/src/pkg/cmath/pow.go b/src/pkg/cmath/pow.go new file mode 100644 index 000000000..68e1207c6 --- /dev/null +++ b/src/pkg/cmath/pow.go @@ -0,0 +1,60 @@ +// Copyright 2010 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package cmath + +import "math" + +// The original C code, the long comment, and the constants +// below are from http://netlib.sandia.gov/cephes/c9x-complex/clog.c. +// The go code is a simplified version of the original C. +// +// Cephes Math Library Release 2.8: June, 2000 +// Copyright 1984, 1987, 1989, 1992, 2000 by Stephen L. Moshier +// +// The readme file at http://netlib.sandia.gov/cephes/ says: +// Some software in this archive may be from the book _Methods and +// Programs for Mathematical Functions_ (Prentice-Hall or Simon & Schuster +// International, 1989) or from the Cephes Mathematical Library, a +// commercial product. In either event, it is copyrighted by the author. +// What you see here may be used freely but it comes with no support or +// guarantee. +// +// The two known misprints in the book are repaired here in the +// source listings for the gamma function and the incomplete beta +// integral. +// +// Stephen L. Moshier +// moshier@na-net.ornl.gov + +// Complex power function +// +// DESCRIPTION: +// +// Raises complex A to the complex Zth power. +// Definition is per AMS55 # 4.2.8, +// analytically equivalent to cpow(a,z) = cexp(z clog(a)). +// +// ACCURACY: +// +// Relative error: +// arithmetic domain # trials peak rms +// IEEE -10,+10 30000 9.4e-15 1.5e-15 + +// Pow returns x**y, the base-x exponential of y. +func Pow(x, y complex128) complex128 { + modulus := Abs(x) + if modulus == 0 { + return complex(0, 0) + } + r := math.Pow(modulus, real(y)) + arg := Phase(x) + theta := real(y) * arg + if imag(y) != 0 { + r *= math.Exp(-imag(y) * arg) + theta += imag(y) * math.Log(modulus) + } + s, c := math.Sincos(theta) + return complex(r*c, r*s) +} diff --git a/src/pkg/cmath/rect.go b/src/pkg/cmath/rect.go new file mode 100644 index 000000000..b955f0bf7 --- /dev/null +++ b/src/pkg/cmath/rect.go @@ -0,0 +1,13 @@ +// Copyright 2010 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package cmath + +import "math" + +// Rect returns the complex number x with polar coordinates r, θ. +func Rect(r, θ float64) complex128 { + s, c := math.Sincos(θ) + return complex(r*c, r*s) +} diff --git a/src/pkg/cmath/sin.go b/src/pkg/cmath/sin.go new file mode 100644 index 000000000..8900ecdde --- /dev/null +++ b/src/pkg/cmath/sin.go @@ -0,0 +1,132 @@ +// Copyright 2010 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package cmath + +import "math" + +// The original C code, the long comment, and the constants +// below are from http://netlib.sandia.gov/cephes/c9x-complex/clog.c. +// The go code is a simplified version of the original C. +// +// Cephes Math Library Release 2.8: June, 2000 +// Copyright 1984, 1987, 1989, 1992, 2000 by Stephen L. Moshier +// +// The readme file at http://netlib.sandia.gov/cephes/ says: +// Some software in this archive may be from the book _Methods and +// Programs for Mathematical Functions_ (Prentice-Hall or Simon & Schuster +// International, 1989) or from the Cephes Mathematical Library, a +// commercial product. In either event, it is copyrighted by the author. +// What you see here may be used freely but it comes with no support or +// guarantee. +// +// The two known misprints in the book are repaired here in the +// source listings for the gamma function and the incomplete beta +// integral. +// +// Stephen L. Moshier +// moshier@na-net.ornl.gov + +// Complex circular sine +// +// DESCRIPTION: +// +// If +// z = x + iy, +// +// then +// +// w = sin x cosh y + i cos x sinh y. +// +// csin(z) = -i csinh(iz). +// +// ACCURACY: +// +// Relative error: +// arithmetic domain # trials peak rms +// DEC -10,+10 8400 5.3e-17 1.3e-17 +// IEEE -10,+10 30000 3.8e-16 1.0e-16 +// Also tested by csin(casin(z)) = z. + +// Sin returns the sine of x. +func Sin(x complex128) complex128 { + s, c := math.Sincos(real(x)) + sh, ch := sinhcosh(imag(x)) + return complex(s*ch, c*sh) +} + +// Complex hyperbolic sine +// +// DESCRIPTION: +// +// csinh z = (cexp(z) - cexp(-z))/2 +// = sinh x * cos y + i cosh x * sin y . +// +// ACCURACY: +// +// Relative error: +// arithmetic domain # trials peak rms +// IEEE -10,+10 30000 3.1e-16 8.2e-17 + +// Sinh returns the hyperbolic sine of x. +func Sinh(x complex128) complex128 { + s, c := math.Sincos(imag(x)) + sh, ch := sinhcosh(real(x)) + return complex(c*sh, s*ch) +} + +// Complex circular cosine +// +// DESCRIPTION: +// +// If +// z = x + iy, +// +// then +// +// w = cos x cosh y - i sin x sinh y. +// +// ACCURACY: +// +// Relative error: +// arithmetic domain # trials peak rms +// DEC -10,+10 8400 4.5e-17 1.3e-17 +// IEEE -10,+10 30000 3.8e-16 1.0e-16 + +// Cos returns the cosine of x. +func Cos(x complex128) complex128 { + s, c := math.Sincos(real(x)) + sh, ch := sinhcosh(imag(x)) + return complex(c*ch, -s*sh) +} + +// Complex hyperbolic cosine +// +// DESCRIPTION: +// +// ccosh(z) = cosh x cos y + i sinh x sin y . +// +// ACCURACY: +// +// Relative error: +// arithmetic domain # trials peak rms +// IEEE -10,+10 30000 2.9e-16 8.1e-17 + +// Cosh returns the hyperbolic cosine of x. +func Cosh(x complex128) complex128 { + s, c := math.Sincos(imag(x)) + sh, ch := sinhcosh(real(x)) + return complex(c*ch, s*sh) +} + +// calculate sinh and cosh +func sinhcosh(x float64) (sh, ch float64) { + if math.Fabs(x) <= 0.5 { + return math.Sinh(x), math.Cosh(x) + } + e := math.Exp(x) + ei := 0.5 / e + e *= 0.5 + return e - ei, e + ei +} diff --git a/src/pkg/cmath/sqrt.go b/src/pkg/cmath/sqrt.go new file mode 100644 index 000000000..e77a9b9df --- /dev/null +++ b/src/pkg/cmath/sqrt.go @@ -0,0 +1,103 @@ +// Copyright 2010 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package cmath + +import "math" + +// The original C code, the long comment, and the constants +// below are from http://netlib.sandia.gov/cephes/c9x-complex/clog.c. +// The go code is a simplified version of the original C. +// +// Cephes Math Library Release 2.8: June, 2000 +// Copyright 1984, 1987, 1989, 1992, 2000 by Stephen L. Moshier +// +// The readme file at http://netlib.sandia.gov/cephes/ says: +// Some software in this archive may be from the book _Methods and +// Programs for Mathematical Functions_ (Prentice-Hall or Simon & Schuster +// International, 1989) or from the Cephes Mathematical Library, a +// commercial product. In either event, it is copyrighted by the author. +// What you see here may be used freely but it comes with no support or +// guarantee. +// +// The two known misprints in the book are repaired here in the +// source listings for the gamma function and the incomplete beta +// integral. +// +// Stephen L. Moshier +// moshier@na-net.ornl.gov + +// Complex square root +// +// DESCRIPTION: +// +// If z = x + iy, r = |z|, then +// +// 1/2 +// Re w = [ (r + x)/2 ] , +// +// 1/2 +// Im w = [ (r - x)/2 ] . +// +// Cancellation error in r-x or r+x is avoided by using the +// identity 2 Re w Im w = y. +// +// Note that -w is also a square root of z. The root chosen +// is always in the right half plane and Im w has the same sign as y. +// +// ACCURACY: +// +// Relative error: +// arithmetic domain # trials peak rms +// DEC -10,+10 25000 3.2e-17 9.6e-18 +// IEEE -10,+10 1,000,000 2.9e-16 6.1e-17 + +// Sqrt returns the square root of x. +func Sqrt(x complex128) complex128 { + if imag(x) == 0 { + if real(x) == 0 { + return complex(0, 0) + } + if real(x) < 0 { + return complex(0, math.Sqrt(-real(x))) + } + return complex(math.Sqrt(real(x)), 0) + } + if real(x) == 0 { + if imag(x) < 0 { + r := math.Sqrt(-0.5 * imag(x)) + return complex(r, -r) + } + r := math.Sqrt(0.5 * imag(x)) + return complex(r, r) + } + a := real(x) + b := imag(x) + var scale float64 + // Rescale to avoid internal overflow or underflow. + if math.Fabs(a) > 4 || math.Fabs(b) > 4 { + a *= 0.25 + b *= 0.25 + scale = 2 + } else { + a *= 1.8014398509481984e16 // 2**54 + b *= 1.8014398509481984e16 + scale = 7.450580596923828125e-9 // 2**-27 + } + r := math.Hypot(a, b) + var t float64 + if a > 0 { + t = math.Sqrt(0.5*r + 0.5*a) + r = scale * math.Fabs((0.5*b)/t) + t *= scale + } else { + r = math.Sqrt(0.5*r - 0.5*a) + t = scale * math.Fabs((0.5*b)/r) + r *= scale + } + if b < 0 { + return complex(t, -r) + } + return complex(t, r) +} diff --git a/src/pkg/cmath/tan.go b/src/pkg/cmath/tan.go new file mode 100644 index 000000000..94b517521 --- /dev/null +++ b/src/pkg/cmath/tan.go @@ -0,0 +1,184 @@ +// Copyright 2010 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package cmath + +import "math" + +// The original C code, the long comment, and the constants +// below are from http://netlib.sandia.gov/cephes/c9x-complex/clog.c. +// The go code is a simplified version of the original C. +// +// Cephes Math Library Release 2.8: June, 2000 +// Copyright 1984, 1987, 1989, 1992, 2000 by Stephen L. Moshier +// +// The readme file at http://netlib.sandia.gov/cephes/ says: +// Some software in this archive may be from the book _Methods and +// Programs for Mathematical Functions_ (Prentice-Hall or Simon & Schuster +// International, 1989) or from the Cephes Mathematical Library, a +// commercial product. In either event, it is copyrighted by the author. +// What you see here may be used freely but it comes with no support or +// guarantee. +// +// The two known misprints in the book are repaired here in the +// source listings for the gamma function and the incomplete beta +// integral. +// +// Stephen L. Moshier +// moshier@na-net.ornl.gov + +// Complex circular tangent +// +// DESCRIPTION: +// +// If +// z = x + iy, +// +// then +// +// sin 2x + i sinh 2y +// w = --------------------. +// cos 2x + cosh 2y +// +// On the real axis the denominator is zero at odd multiples +// of PI/2. The denominator is evaluated by its Taylor +// series near these points. +// +// ctan(z) = -i ctanh(iz). +// +// ACCURACY: +// +// Relative error: +// arithmetic domain # trials peak rms +// DEC -10,+10 5200 7.1e-17 1.6e-17 +// IEEE -10,+10 30000 7.2e-16 1.2e-16 +// Also tested by ctan * ccot = 1 and catan(ctan(z)) = z. + +// Tan returns the tangent of x. +func Tan(x complex128) complex128 { + d := math.Cos(2*real(x)) + math.Cosh(2*imag(x)) + if math.Fabs(d) < 0.25 { + d = tanSeries(x) + } + if d == 0 { + return Inf() + } + return complex(math.Sin(2*real(x))/d, math.Sinh(2*imag(x))/d) +} + +// Complex hyperbolic tangent +// +// DESCRIPTION: +// +// tanh z = (sinh 2x + i sin 2y) / (cosh 2x + cos 2y) . +// +// ACCURACY: +// +// Relative error: +// arithmetic domain # trials peak rms +// IEEE -10,+10 30000 1.7e-14 2.4e-16 + +// Tanh returns the hyperbolic tangent of x. +func Tanh(x complex128) complex128 { + d := math.Cosh(2*real(x)) + math.Cos(2*imag(x)) + if d == 0 { + return Inf() + } + return complex(math.Sinh(2*real(x))/d, math.Sin(2*imag(x))/d) +} + +// Program to subtract nearest integer multiple of PI +func reducePi(x float64) float64 { + const ( + // extended precision value of PI: + DP1 = 3.14159265160560607910E0 // ?? 0x400921fb54000000 + DP2 = 1.98418714791870343106E-9 // ?? 0x3e210b4610000000 + DP3 = 1.14423774522196636802E-17 // ?? 0x3c6a62633145c06e + ) + t := x / math.Pi + if t >= 0 { + t += 0.5 + } else { + t -= 0.5 + } + t = float64(int64(t)) // int64(t) = the multiple + return ((x - t*DP1) - t*DP2) - t*DP3 +} + +// Taylor series expansion for cosh(2y) - cos(2x) +func tanSeries(z complex128) float64 { + const MACHEP = 1.0 / (1 << 53) + x := math.Fabs(2 * real(z)) + y := math.Fabs(2 * imag(z)) + x = reducePi(x) + x = x * x + y = y * y + x2 := 1.0 + y2 := 1.0 + f := 1.0 + rn := 0.0 + d := 0.0 + for { + rn += 1 + f *= rn + rn += 1 + f *= rn + x2 *= x + y2 *= y + t := y2 + x2 + t /= f + d += t + + rn += 1 + f *= rn + rn += 1 + f *= rn + x2 *= x + y2 *= y + t = y2 - x2 + t /= f + d += t + if math.Fabs(t/d) <= MACHEP { + break + } + } + return d +} + +// Complex circular cotangent +// +// DESCRIPTION: +// +// If +// z = x + iy, +// +// then +// +// sin 2x - i sinh 2y +// w = --------------------. +// cosh 2y - cos 2x +// +// On the real axis, the denominator has zeros at even +// multiples of PI/2. Near these points it is evaluated +// by a Taylor series. +// +// ACCURACY: +// +// Relative error: +// arithmetic domain # trials peak rms +// DEC -10,+10 3000 6.5e-17 1.6e-17 +// IEEE -10,+10 30000 9.2e-16 1.2e-16 +// Also tested by ctan * ccot = 1 + i0. + +// Cot returns the cotangent of x. +func Cot(x complex128) complex128 { + d := math.Cosh(2*imag(x)) - math.Cos(2*real(x)) + if math.Fabs(d) < 0.25 { + d = tanSeries(x) + } + if d == 0 { + return Inf() + } + return complex(math.Sin(2*real(x))/d, -math.Sinh(2*imag(x))/d) +} diff --git a/src/pkg/compress/bzip2/Makefile b/src/pkg/compress/bzip2/Makefile new file mode 100644 index 000000000..a4bceef16 --- /dev/null +++ b/src/pkg/compress/bzip2/Makefile @@ -0,0 +1,14 @@ +# Copyright 2011 The Go Authors. All rights reserved. +# Use of this source code is governed by a BSD-style +# license that can be found in the LICENSE file. + +include ../../../Make.inc + +TARG=compress/bzip2 +GOFILES=\ + bit_reader.go\ + bzip2.go\ + huffman.go\ + move_to_front.go\ + +include ../../../Make.pkg diff --git a/src/pkg/compress/bzip2/bit_reader.go b/src/pkg/compress/bzip2/bit_reader.go new file mode 100644 index 000000000..50f0ec836 --- /dev/null +++ b/src/pkg/compress/bzip2/bit_reader.go @@ -0,0 +1,88 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package bzip2 + +import ( + "bufio" + "io" + "os" +) + +// bitReader wraps an io.Reader and provides the ability to read values, +// bit-by-bit, from it. Its Read* methods don't return the usual os.Error +// because the error handling was verbose. Instead, any error is kept and can +// be checked afterwards. +type bitReader struct { + r byteReader + n uint64 + bits uint + err os.Error +} + +// bitReader needs to read bytes from an io.Reader. We attempt to cast the +// given io.Reader to this interface and, if it doesn't already fit, we wrap in +// a bufio.Reader. +type byteReader interface { + ReadByte() (byte, os.Error) +} + +func newBitReader(r io.Reader) bitReader { + byter, ok := r.(byteReader) + if !ok { + byter = bufio.NewReader(r) + } + return bitReader{r: byter} +} + +// ReadBits64 reads the given number of bits and returns them in the +// least-significant part of a uint64. In the event of an error, it returns 0 +// and the error can be obtained by calling Error(). +func (br *bitReader) ReadBits64(bits uint) (n uint64) { + for bits > br.bits { + b, err := br.r.ReadByte() + if err == os.EOF { + err = io.ErrUnexpectedEOF + } + if err != nil { + br.err = err + return 0 + } + br.n <<= 8 + br.n |= uint64(b) + br.bits += 8 + } + + // br.n looks like this (assuming that br.bits = 14 and bits = 6): + // Bit: 111111 + // 5432109876543210 + // + // (6 bits, the desired output) + // |-----| + // V V + // 0101101101001110 + // ^ ^ + // |------------| + // br.bits (num valid bits) + // + // This the next line right shifts the desired bits into the + // least-significant places and masks off anything above. + n = (br.n >> (br.bits - bits)) & ((1 << bits) - 1) + br.bits -= bits + return +} + +func (br *bitReader) ReadBits(bits uint) (n int) { + n64 := br.ReadBits64(bits) + return int(n64) +} + +func (br *bitReader) ReadBit() bool { + n := br.ReadBits(1) + return n != 0 +} + +func (br *bitReader) Error() os.Error { + return br.err +} diff --git a/src/pkg/compress/bzip2/bzip2.go b/src/pkg/compress/bzip2/bzip2.go new file mode 100644 index 000000000..8b4572306 --- /dev/null +++ b/src/pkg/compress/bzip2/bzip2.go @@ -0,0 +1,390 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Package bzip2 implements bzip2 decompression. +package bzip2 + +import ( + "io" + "os" +) + +// There's no RFC for bzip2. I used the Wikipedia page for reference and a lot +// of guessing: http://en.wikipedia.org/wiki/Bzip2 +// The source code to pyflate was useful for debugging: +// http://www.paul.sladen.org/projects/pyflate + +// A StructuralError is returned when the bzip2 data is found to be +// syntactically invalid. +type StructuralError string + +func (s StructuralError) String() string { + return "bzip2 data invalid: " + string(s) +} + +// A reader decompresses bzip2 compressed data. +type reader struct { + br bitReader + setupDone bool // true if we have parsed the bzip2 header. + blockSize int // blockSize in bytes, i.e. 900 * 1024. + eof bool + buf []byte // stores Burrows-Wheeler transformed data. + c [256]uint // the `C' array for the inverse BWT. + tt []uint32 // mirrors the `tt' array in the bzip2 source and contains the P array in the upper 24 bits. + tPos uint32 // Index of the next output byte in tt. + + preRLE []uint32 // contains the RLE data still to be processed. + preRLEUsed int // number of entries of preRLE used. + lastByte int // the last byte value seen. + byteRepeats uint // the number of repeats of lastByte seen. + repeats uint // the number of copies of lastByte to output. +} + +// NewReader returns an io.Reader which decompresses bzip2 data from r. +func NewReader(r io.Reader) io.Reader { + bz2 := new(reader) + bz2.br = newBitReader(r) + return bz2 +} + +const bzip2FileMagic = 0x425a // "BZ" +const bzip2BlockMagic = 0x314159265359 +const bzip2FinalMagic = 0x177245385090 + +// setup parses the bzip2 header. +func (bz2 *reader) setup() os.Error { + br := &bz2.br + + magic := br.ReadBits(16) + if magic != bzip2FileMagic { + return StructuralError("bad magic value") + } + + t := br.ReadBits(8) + if t != 'h' { + return StructuralError("non-Huffman entropy encoding") + } + + level := br.ReadBits(8) + if level < '1' || level > '9' { + return StructuralError("invalid compression level") + } + + bz2.blockSize = 100 * 1024 * (int(level) - '0') + bz2.tt = make([]uint32, bz2.blockSize) + return nil +} + +func (bz2 *reader) Read(buf []byte) (n int, err os.Error) { + if bz2.eof { + return 0, os.EOF + } + + if !bz2.setupDone { + err = bz2.setup() + brErr := bz2.br.Error() + if brErr != nil { + err = brErr + } + if err != nil { + return 0, err + } + bz2.setupDone = true + } + + n, err = bz2.read(buf) + brErr := bz2.br.Error() + if brErr != nil { + err = brErr + } + return +} + +func (bz2 *reader) read(buf []byte) (n int, err os.Error) { + // bzip2 is a block based compressor, except that it has a run-length + // preprocessing step. The block based nature means that we can + // preallocate fixed-size buffers and reuse them. However, the RLE + // preprocessing would require allocating huge buffers to store the + // maximum expansion. Thus we process blocks all at once, except for + // the RLE which we decompress as required. + + for (bz2.repeats > 0 || bz2.preRLEUsed < len(bz2.preRLE)) && n < len(buf) { + // We have RLE data pending. + + // The run-length encoding works like this: + // Any sequence of four equal bytes is followed by a length + // byte which contains the number of repeats of that byte to + // include. (The number of repeats can be zero.) Because we are + // decompressing on-demand our state is kept in the reader + // object. + + if bz2.repeats > 0 { + buf[n] = byte(bz2.lastByte) + n++ + bz2.repeats-- + if bz2.repeats == 0 { + bz2.lastByte = -1 + } + continue + } + + bz2.tPos = bz2.preRLE[bz2.tPos] + b := byte(bz2.tPos) + bz2.tPos >>= 8 + bz2.preRLEUsed++ + + if bz2.byteRepeats == 3 { + bz2.repeats = uint(b) + bz2.byteRepeats = 0 + continue + } + + if bz2.lastByte == int(b) { + bz2.byteRepeats++ + } else { + bz2.byteRepeats = 0 + } + bz2.lastByte = int(b) + + buf[n] = b + n++ + } + + if n > 0 { + return + } + + // No RLE data is pending so we need to read a block. + + br := &bz2.br + magic := br.ReadBits64(48) + if magic == bzip2FinalMagic { + br.ReadBits64(32) // ignored CRC + bz2.eof = true + return 0, os.EOF + } else if magic != bzip2BlockMagic { + return 0, StructuralError("bad magic value found") + } + + err = bz2.readBlock() + if err != nil { + return 0, err + } + + return bz2.read(buf) +} + +// readBlock reads a bzip2 block. The magic number should already have been consumed. +func (bz2 *reader) readBlock() (err os.Error) { + br := &bz2.br + br.ReadBits64(32) // skip checksum. TODO: check it if we can figure out what it is. + randomized := br.ReadBits(1) + if randomized != 0 { + return StructuralError("deprecated randomized files") + } + origPtr := uint(br.ReadBits(24)) + + // If not every byte value is used in the block (i.e., it's text) then + // the symbol set is reduced. The symbols used are stored as a + // two-level, 16x16 bitmap. + symbolRangeUsedBitmap := br.ReadBits(16) + symbolPresent := make([]bool, 256) + numSymbols := 0 + for symRange := uint(0); symRange < 16; symRange++ { + if symbolRangeUsedBitmap&(1<<(15-symRange)) != 0 { + bits := br.ReadBits(16) + for symbol := uint(0); symbol < 16; symbol++ { + if bits&(1<<(15-symbol)) != 0 { + symbolPresent[16*symRange+symbol] = true + numSymbols++ + } + } + } + } + + // A block uses between two and six different Huffman trees. + numHuffmanTrees := br.ReadBits(3) + if numHuffmanTrees < 2 || numHuffmanTrees > 6 { + return StructuralError("invalid number of Huffman trees") + } + + // The Huffman tree can switch every 50 symbols so there's a list of + // tree indexes telling us which tree to use for each 50 symbol block. + numSelectors := br.ReadBits(15) + treeIndexes := make([]uint8, numSelectors) + + // The tree indexes are move-to-front transformed and stored as unary + // numbers. + mtfTreeDecoder := newMTFDecoderWithRange(numHuffmanTrees) + for i := range treeIndexes { + c := 0 + for { + inc := br.ReadBits(1) + if inc == 0 { + break + } + c++ + } + if c >= numHuffmanTrees { + return StructuralError("tree index too large") + } + treeIndexes[i] = uint8(mtfTreeDecoder.Decode(c)) + } + + // The list of symbols for the move-to-front transform is taken from + // the previously decoded symbol bitmap. + symbols := make([]byte, numSymbols) + nextSymbol := 0 + for i := 0; i < 256; i++ { + if symbolPresent[i] { + symbols[nextSymbol] = byte(i) + nextSymbol++ + } + } + mtf := newMTFDecoder(symbols) + + numSymbols += 2 // to account for RUNA and RUNB symbols + huffmanTrees := make([]huffmanTree, numHuffmanTrees) + + // Now we decode the arrays of code-lengths for each tree. + lengths := make([]uint8, numSymbols) + for i := 0; i < numHuffmanTrees; i++ { + // The code lengths are delta encoded from a 5-bit base value. + length := br.ReadBits(5) + for j := 0; j < numSymbols; j++ { + for { + if !br.ReadBit() { + break + } + if br.ReadBit() { + length-- + } else { + length++ + } + } + if length < 0 || length > 20 { + return StructuralError("Huffman length out of range") + } + lengths[j] = uint8(length) + } + huffmanTrees[i], err = newHuffmanTree(lengths) + if err != nil { + return err + } + } + + selectorIndex := 1 // the next tree index to use + currentHuffmanTree := huffmanTrees[treeIndexes[0]] + bufIndex := 0 // indexes bz2.buf, the output buffer. + // The output of the move-to-front transform is run-length encoded and + // we merge the decoding into the Huffman parsing loop. These two + // variables accumulate the repeat count. See the Wikipedia page for + // details. + repeat := 0 + repeat_power := 0 + + // The `C' array (used by the inverse BWT) needs to be zero initialized. + for i := range bz2.c { + bz2.c[i] = 0 + } + + decoded := 0 // counts the number of symbols decoded by the current tree. + for { + if decoded == 50 { + currentHuffmanTree = huffmanTrees[treeIndexes[selectorIndex]] + selectorIndex++ + decoded = 0 + } + + v := currentHuffmanTree.Decode(br) + decoded++ + + if v < 2 { + // This is either the RUNA or RUNB symbol. + if repeat == 0 { + repeat_power = 1 + } + repeat += repeat_power << v + repeat_power <<= 1 + + // This limit of 2 million comes from the bzip2 source + // code. It prevents repeat from overflowing. + if repeat > 2*1024*1024 { + return StructuralError("repeat count too large") + } + continue + } + + if repeat > 0 { + // We have decoded a complete run-length so we need to + // replicate the last output symbol. + for i := 0; i < repeat; i++ { + b := byte(mtf.First()) + bz2.tt[bufIndex] = uint32(b) + bz2.c[b]++ + bufIndex++ + } + repeat = 0 + } + + if int(v) == numSymbols-1 { + // This is the EOF symbol. Because it's always at the + // end of the move-to-front list, and never gets moved + // to the front, it has this unique value. + break + } + + // Since two metasymbols (RUNA and RUNB) have values 0 and 1, + // one would expect |v-2| to be passed to the MTF decoder. + // However, the front of the MTF list is never referenced as 0, + // it's always referenced with a run-length of 1. Thus 0 + // doesn't need to be encoded and we have |v-1| in the next + // line. + b := byte(mtf.Decode(int(v - 1))) + bz2.tt[bufIndex] = uint32(b) + bz2.c[b]++ + bufIndex++ + } + + if origPtr >= uint(bufIndex) { + return StructuralError("origPtr out of bounds") + } + + // We have completed the entropy decoding. Now we can perform the + // inverse BWT and setup the RLE buffer. + bz2.preRLE = bz2.tt[:bufIndex] + bz2.preRLEUsed = 0 + bz2.tPos = inverseBWT(bz2.preRLE, origPtr, bz2.c[:]) + bz2.lastByte = -1 + bz2.byteRepeats = 0 + bz2.repeats = 0 + + return nil +} + +// inverseBWT implements the inverse Burrows-Wheeler transform as described in +// http://www.hpl.hp.com/techreports/Compaq-DEC/SRC-RR-124.pdf, section 4.2. +// In that document, origPtr is called `I' and c is the `C' array after the +// first pass over the data. It's an argument here because we merge the first +// pass with the Huffman decoding. +// +// This also implements the `single array' method from the bzip2 source code +// which leaves the output, still shuffled, in the bottom 8 bits of tt with the +// index of the next byte in the top 24-bits. The index of the first byte is +// returned. +func inverseBWT(tt []uint32, origPtr uint, c []uint) uint32 { + sum := uint(0) + for i := 0; i < 256; i++ { + sum += c[i] + c[i] = sum - c[i] + } + + for i := range tt { + b := tt[i] & 0xff + tt[c[b]] |= uint32(i) << 8 + c[b]++ + } + + return tt[origPtr] >> 8 +} diff --git a/src/pkg/compress/bzip2/bzip2_test.go b/src/pkg/compress/bzip2/bzip2_test.go new file mode 100644 index 000000000..156eea83f --- /dev/null +++ b/src/pkg/compress/bzip2/bzip2_test.go @@ -0,0 +1,158 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package bzip2 + +import ( + "bytes" + "encoding/hex" + "io" + "io/ioutil" + "os" + "testing" +) + +func TestBitReader(t *testing.T) { + buf := bytes.NewBuffer([]byte{0xaa}) + br := newBitReader(buf) + if n := br.ReadBits(1); n != 1 { + t.Errorf("read 1 wrong") + } + if n := br.ReadBits(1); n != 0 { + t.Errorf("read 2 wrong") + } + if n := br.ReadBits(1); n != 1 { + t.Errorf("read 3 wrong") + } + if n := br.ReadBits(1); n != 0 { + t.Errorf("read 4 wrong") + } +} + +func TestBitReaderLarge(t *testing.T) { + buf := bytes.NewBuffer([]byte{0x12, 0x34, 0x56, 0x78}) + br := newBitReader(buf) + if n := br.ReadBits(32); n != 0x12345678 { + t.Errorf("got: %x want: %x", n, 0x12345678) + } +} + +func readerFromHex(s string) io.Reader { + data, err := hex.DecodeString(s) + if err != nil { + panic("readerFromHex: bad input") + } + return bytes.NewBuffer(data) +} + +func decompressHex(s string) (out []byte, err os.Error) { + r := NewReader(readerFromHex(s)) + return ioutil.ReadAll(r) +} + +func TestHelloWorldBZ2(t *testing.T) { + out, err := decompressHex(helloWorldBZ2Hex) + if err != nil { + t.Errorf("error from Read: %s", err) + return + } + + if !bytes.Equal(helloWorld, out) { + t.Errorf("got %x, want %x", out, helloWorld) + } +} + +func testZeros(t *testing.T, inHex string, n int) { + out, err := decompressHex(inHex) + if err != nil { + t.Errorf("error from Read: %s", err) + return + } + + expected := make([]byte, n) + + if !bytes.Equal(expected, out) { + allZeros := true + for _, b := range out { + if b != 0 { + allZeros = false + break + } + } + t.Errorf("incorrect result, got %d bytes (allZeros: %t)", len(out), allZeros) + } +} + +func Test32Zeros(t *testing.T) { + testZeros(t, thirtyTwoZerosBZ2Hex, 32) +} + +func Test1MBZeros(t *testing.T) { + testZeros(t, oneMBZerosBZ2Hex, 1024*1024) +} + +func testRandomData(t *testing.T, compressedHex, uncompressedHex string) { + out, err := decompressHex(compressedHex) + if err != nil { + t.Errorf("error from Read: %s", err) + return + } + + expected, _ := hex.DecodeString(uncompressedHex) + + if !bytes.Equal(out, expected) { + t.Errorf("incorrect result\ngot: %x\nwant: %x", out, expected) + } +} + +func TestRandomData1(t *testing.T) { + testRandomData(t, randBZ2Hex, randHex) +} + +func TestRandomData2(t *testing.T) { + // This test involves several repeated bytes in the output, but they + // should trigger RLE decoding. + testRandomData(t, rand2BZ2Hex, rand2Hex) +} + +func TestRandomData3(t *testing.T) { + // This test uses the full range of symbols. + testRandomData(t, rand3BZ2Hex, rand3Hex) +} + +func Test1MBSawtooth(t *testing.T) { + out, err := decompressHex(oneMBSawtoothBZ2Hex) + if err != nil { + t.Errorf("error from Read: %s", err) + return + } + + expected := make([]byte, 1024*1024) + + for i := range expected { + expected[i] = byte(i) + } + + if !bytes.Equal(out, expected) { + t.Error("incorrect result") + } +} + +const helloWorldBZ2Hex = "425a68393141592653594eece83600000251800010400006449080200031064c4101a7a9a580bb9431f8bb9229c28482776741b0" + +var helloWorld = []byte("hello world\n") + +const thirtyTwoZerosBZ2Hex = "425a6839314159265359b5aa5098000000600040000004200021008283177245385090b5aa5098" +const oneMBZerosBZ2Hex = "425a683931415926535938571ce50008084000c0040008200030cc0529a60806c4201e2ee48a70a12070ae39ca" + +const randBZ2Hex = "425a6839314159265359905d990d0001957fffffffffffafffffffffffffffffbfff6fffdfffffffffffffffffffffffffffffc002b6dd75676ed5b77720098320d11a64626981323d4da47a83131a13d09e8040f534cd4f4d27a464d193008cd09804601347a980026350c9886234d36864193d1351b44c136919e90340d26127a4cd264c32023009898981310c0344c340027a8303427a99a04c00003534c230d034f5006468d268cf54d36a3009a69a62626261311b40026013d34201a6934c9a604c98ca6c8460989fa9346234d30d3469a2604fd4131a7aa6d0046043d4c62098479269e89e835190d018d4c046001a11e801a0264792321932308c43a130688c260d46686804cd01a9e80981193684c6a68c00000004c4c20c04627a4c0000260003400d04c0681a01334026009a6f48041466132581ec5212b081d96b0effc16543e2228b052fcd30f2567ee8d970e0f10aabca68dd8270591c376cfc1baae0dba00aaff2d6caf6b211322c997cc18eaee5927f75185336bf907021324c71626c1dd20e22b9b0977f05d0f901eaa51db9fbaf7c603b4c87bc82890e6dd7e61d0079e27ec050dd788fd958152061cd01e222f9547cb9efc465d775b6fc98bac7d387bffd151ae09dadf19494f7a638e2eae58e550faba5fe6820ea520eb986096de4e527d80def3ba625e71fbefdcf7e7844e0a25d29b52dcd1344fca083737d42692aab38d230485f3c8ed54c2ed31f15cf0270c8143765b10b92157233fa1dfe0d7ce8ffe70b8b8f7250071701dfe9f1c94de362c9031455951c93eb098a6b50ee45c6131fefc3b6f9643e21f4adc59497138e246f5c57d834aa67c4f10d8bd8b3908d8130dd7388409c299a268eab3664fa4907c5c31574874bd8d388a4ab22b339660804e53e1b8d05867d40e3082560608d35d5d2c6054e8bab23da28f61f83efd41d25529ad6ea15fb50505cacfabb0902166427354ca3830a2c8415f21b19e592690fbe447020d685a4bcd16ecc4ff1a1c0e572627d0ef6265c008a43fc243240541061ed7840606be466d1c0dac2c53250ed567507d926c844154560d631960c65e15157829b2c7f16859f111a3a8cb72bf24ffa57a680c3be67b1be67c8dd8aea73ac2437a78df5b686d427080ebc01bd30b71a49f6ea31dc0f08e4849e38face96717690239538bc08b6cc5aa8d467cb9c36aa83d40ac7e58bddbfa185b22065e89a86c0145569d9e23726651aec49e31588d70f40fe9a4449dcf4f89eac220171e9c938e803dc195679651004b79ad33cc0c13aeeba5941b33ffeeb8fbe16e76c7811445c67b4269c90479433ddf9e8ed1d00c166b6c17217fb22c3ef1b0c1c7e28e185446a111c37f1ea6c07a59fbcc6546ecc6968d36ba58bc5489a5640647e426b0c39350cb6f07d5dc7a717648c4ec7f841467597ae1f65f408fd2d9940a4b1b860b3c9ae351dcae0b4425f7e8538710f2e40b7f70d13b51ac05ccc6ecda8264a88cad2d721d18132a9b9110a9e759c2483c77dcefc7e464ec88588174cb0c9abff93230ea0bed8decdd8ed8bfe2b5df0a253803678df04fab44c03b9ab7cc97d6e6d6fd0c4c840ce0efc498436f453bbb181603459471f2b588724592b222ec990614db530e10cadd84705621cfdd9261fa44a5f5806a2d74b575056b3c915255c65678f9c16e6dc00a99180fef1a840aff0e842ac02731080cc92782538360a60a727991013984da4fad95f79d5030677b7528d076b2483685fca4429edf804682fdc110dfc2f7c30e23e20a72e039108a0ad6fdee2f76985a4b4be4f5afc6101bf9d5042b657a05dc914e1424241766434" +const randHex = "c95138082bdf2b9bfa5b1072b23f729735d42c785eeb94320fb14c265b9c2ca421d01a3db986df1ac2acde5a0e6bf955d6f95e61261540905928e195f1a66644cc7f37281744fff4dc6df35566a494c41a8167151950eb74f5fc45f85ad0e5ed28b49adfe218aa7ec1707e8e1d55825f61f72beda3b4c006b8c9188d7336a5d875329b1b58c27cc4e89ecbae02c7712400c39dd131d2c6de82e2863da51d472bdfb21ecce62cc9cf769ed28aedc7583d755da45a0d90874bda269dd53283a9bdfd05f95fc8e9a304bb338ea1a2111894678c18134f17d31a15d9bfc1237894650f3e715e2548639ecbddb845cfe4a46a7b3a3c540f48629488e8c869f1e9f3f4c552243a8105b20eb8e264994214349dae83b165fd6c2a5b8e83fce09fc0a80d3281c8d53a9a08095bd19cbc1388df23975646ed259e003d39261ee68cbece8bcf32971f7fe7e588e8ba8f5e8597909abaea693836a79a1964050ed910a45a0f13a58cd2d3ae18992c5b23082407fd920d0bf01e33118a017bb5e39f44931346845af52128f7965206759433a346034ea481671f501280067567619f5ecef6cded077f92ed7f3b3ce8e308c80f34ba06939e9303f91b4318c8c1dd4cc223c1f057ac0c91211c629cd30e46ee9ec1d9fd493086b7bc2bc83e33f08749a5d430b0ed4f79d70f481940c9b0930b16321886a0df4fa5a1465d5208c7d3494a7987d9a5e42aa256f0c9523947f8318d0ef0af3d59a45cfc2418d0785c9a548b32b81e7de18be7d55a69a4c156bbb3d7579c0ac8e9c72b24646e54b0d0e8725f8f49fb44ae3c6b9d0287be118586255a90a4a83483ed0328518037e52aa959c5748ed83e13023e532306be98b8288da306bbb040bcf5d92176f84a9306dc6b274b040370b61d71fde58dd6d20e6fee348eae0c54bd0a5a487b2d005f329794f2a902c296af0a4c1f638f63292a1fa18e006c1b1838636f4de71c73635b25660d32e88a0917e1a5677f6a02ca65585b82cbd99fb4badbfa97a585da1e6cadf6737b4ec6ca33f245d66ee6a9fae6785d69b003c17b9fc6ec34fe5824ab8caae5e8e14dc6f9e116e7bf4a60c04388783c8ae929e1b46b3ef3bbe81b38f2fa6da771bf39dfba2374d3d2ed356b8e2c42081d885a91a3afb2f31986d2f9873354c48cf5448492c32e62385af423aa4f83db6d1b2669650379a1134b0a04cbca0862d6f9743c791cbb527d36cd5d1f0fc7f503831c8bd1b7a0ef8ae1a5ed1155dfdd9e32b6bb33138112d3d476b802179cb85a2a6c354ccfed2f31604fbd8d6ec4baf9f1c8454f72c6588c06a7df3178c43a6970bfa02dd6f74cb5ec3b63f9eddaa17db5cbf27fac6de8e57c384afd0954179f7b5690c3bee42abc4fa79b4b12101a9cf5f0b9aecdda945def0bd04163237247d3539850e123fe18139f316fa0256d5bd2faa8" + +const oneMBSawtoothBZ2Hex = "425a683931415926535971931ea00006ddffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffe007de00000000000000024c00130001300000000000000000000000000000000000000000000000000000000126000980009800000000000000000000000000000000000000000000000000000000930004c0004c000000000000000000000000000000000000000000000000000000004980026000260000000000000000000000000000000000000000000000000000000009aaaaa0000000000000000000000000000000000000000000000000000000000000000498002600026000000000000000000000000000000000000000000000000000000007fc42271980d044c0a822607411304a08982d044c1a82260f411308a08984d044c2a82261741130ca08986d044c3a82261f411310a08988d044c4a822627411314a0898ad044c5a82262f411318a0898cd044c6a82263741131ca0898ed044c7a82263f411320a08990d044c8a822647411324a08992d044c9a82264f411328a08994d044caa82265741132ca08996d044cba82265f411330a08998d044cca822667411334a0899ad044cda82266f411338a0899cd044cea82267741133ca0899ed044cfa82267f411340a089a0d044d0a822687411344a089a2d044d1a82268f411348a089a4d044d2a82269741134ca089a6d044d3a82269f411350a089a8d044d4a8226a7411354a089aad044d5a8226af411358a089acd044d6a8226b741135ca089aed044d7a8226bf411360a089b0d044d8a8226c7411364a089b2d044d9a8226cf411368a089b4d044daa8226d741136ca089b6d044dba8226df411370a089b8d044dca8226e7411374a089bad044dda8226ef411378a089bcd044dea8226f741137ca089bed044dfa8226ff411380a089c0d044e0a822707411384a089c2d044e1a82270f411388a089c4d044e2a82271741138ca089c59089c69089c71089c79089c81089c89089c91089c99089ca1089ca9089cb1089cb9089cc1089cc9089cd1089cd9089ce1089ce9089cf1089cf9089d01089d09089d11089d19089d21089d29089d31089d39089d41089d49089d51089d59089d61089d69089d71089d79089d81089d89089d91089d99089da1089da9089db1089db9089dc1089dc9089dd1089dd9089de1089de9089df1089df9089e01089e09089e11089e19089e21089e29089e31089e39089e41089e49089e51089e59089e61089e69089e71089e79089e81089e89089e91089e99089ea1089ea9089eb1089eb9089ec1089ec9089ed1089ed9089ee1089ee9089ef1089ef9089f01089f09089f11089f19089f21089f29089f31089f39089f41089f49089f51089f59089f61089f69089f71089f79089f81089f89089f91089f99089fa1089fa9089fb1089fb9089fc1089fc9089fd1089fd9089fe1089fe9089ff1089ff98a0ac9329acf23ba884804fdd3ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff0034f800000000000024c00130001300000000000000000000000000000000000000000000000000000000126000980009800000000000000000000000000000000000000000000000000000000930004c0004c000000000000000000000000000000000000000000000000000000004980026000260000000000000000000000000000000000000000000000000000000024c0013000130000000000000000000000000000000000000000000000000000000002955540000000000000000000000000000000000000000000000000000000000000001ff108c00846024230221181908c108460a4230621183908c20846124230a21185908c308461a4230e21187908c40846224231221189908c508462a423162118b908c60846324231a2118d908c708463a4231e2118f908c80846424232221191908c908464a4232621193908ca0846524232a21195908cb08465a4232e21197908cc0846624233221199908cd08466a423362119b908ce0846724233a2119d908cf08467a4233e2119f908d008468242342211a1908d108468a42346211a3908d20846924234a211a5908d308469a4234e211a7908d40846a242352211a9908d50846aa42356211ab908d60846b24235a211ad908d70846ba4235e211af908d80846c242362211b1908d90846ca42366211b3908da0846d24236a211b5908db0846da4236e211b7908dc0846e242372211b9908dd0846ea42376211bb908de0846f24237a211bd908df0846fa4237e211bf908e008470242382211c1908e108470a42386211c3908e20847124238a211c5908e2f8c211c6c8471d211c7c84721211c8c84725211c9c84729211cac8472d211cbc84731211ccc84735211cdc84739211cec8473d211cfc84741211d0c84745211d1c84749211d2c8474d211d3c84751211d4c84755211d5c84759211d6c8475d211d7c84761211d8c84765211d9c84769211dac8476d211dbc84771211dcc84775211ddc84779211dec8477d211dfc84781211e0c84785211e1c84789211e2c8478d211e3c84791211e4c84795211e5c84799211e6c8479d211e7c847a1211e8c847a5211e9c847a9211eac847ad211ebc847b1211ecc847b5211edc847b9211eec847bd211efc847c1211f0c847c5211f1c847c9211f2c847cd211f3c847d1211f4c847d5211f5c847d9211f6c847dd211f7c847e1211f8c847e5211f9c847e9211fac847ed211fbc847f1211fcc847f5211fdc847f9211fec847fd211ff8bb9229c284803a8b6248" + +const rand2BZ2Hex = "425a6839314159265359d992d0f60000137dfe84020310091c1e280e100e042801099210094806c0110002e70806402000546034000034000000f2830000032000d3403264049270eb7a9280d308ca06ad28f6981bee1bf8160727c7364510d73a1e123083421b63f031f63993a0f40051fbf177245385090d992d0f60" +const rand2Hex = "92d5652616ac444a4a04af1a8a3964aca0450d43d6cf233bd03233f4ba92f8719e6c2a2bd4f5f88db07ecd0da3a33b263483db9b2c158786ad6363be35d17335ba" + +const rand3BZ2Hex = "425a68393141592653593be669d00000327ffffffffffffffffffffffffffffffffffff7ffffffffffffffffffffffffffffffc002b3b2b1b6e2bae400004c00132300004c0d268c004c08c0130026001a008683234c0684c34008c230261a04c0260064d07a8d00034000d27a1268c9931a8d327a3427a41faa69ea0da264c1a34219326869b51b49a6469a3268c689fa53269a62794687a9a68f5189994c9e487a8f534fd49a3d34043629e8c93d04da4f4648d30d4f44d3234c4d3023d0840680984d309934c234d3131a000640984f536a6132601300130130c8d00d04d1841ea7a8d31a02609b40023460010c01a34d4c1a0d04d3069306810034d0d0d4c0046130d034d0131a9a64d321804c68003400098344c13000991808c0001a00000000098004d3d4da4604c47a13012140aadf8d673c922c607ef6212a8c0403adea4b28aee578900e653b9cdeb8d11e6b838815f3ebaad5a01c5408d84a332170aff8734d4e06612d3c2889f31925fb89e33561f5100ae89b1f7047102e729373d3667e58d73aaa80fa7be368a1cc2dadd81d81ec8e1b504bd772ca31d03649269b01ceddaca07bf3d4eba24de141be3f86f93601e03714c0f64654671684f9f9528626fd4e1b76753dc0c54b842486b8d59d8ab314e86ca818e7a1f079463cbbd70d9b79b283c7edc419406311022e4be98c2c1374df9cdde2d008ce1d00e5f06ad1024baf555631f70831fc1023034e62be7c4bcb648caf276963ffa20e96bb50377fe1c113da0db4625b50741c35a058edb009c6ee5dbf93b8a6b060eec568180e8db791b82aab96cbf4326ca98361461379425ba8dcc347be670bdba7641883e5526ae3d833f6e9cb9bac9557747c79e206151072f7f0071dff3880411846f66bf4075c7462f302b53cb3400a74cf35652ad5641ed33572fd54e7ed7f85f58a0acba89327e7c6be5c58cb71528b99df2431f1d0358f8d28d81d95292da631fb06701decabb205fac59ff0fb1df536afc681eece6ea658c4d9eaa45f1342aa1ff70bdaff2ddaf25ec88c22f12829a0553db1ec2505554cb17d7b282e213a5a2aa30431ded2bce665bb199d023840832fedb2c0c350a27291407ff77440792872137df281592e82076a05c64c345ffb058c64f7f7c207ef78420b7010520610f17e302cc4dfcfaef72a0ed091aab4b541eb0531bbe941ca2f792bf7b31ca6162882b68054a8470115bc2c19f2df2023f7800432b39b04d3a304e8085ba3f1f0ca5b1ba4d38d339e6084de979cdea6d0e244c6c9fa0366bd890621e3d30846f5e8497e21597b8f29bbf52c961a485dfbea647600da0fc1f25ce4d203a8352ece310c39073525044e7ac46acf2ed9120bae1b4f6f02364abfe343f80b290983160c103557af1c68416480d024cc31b6c06cfec011456f1e95c420a12b48b1c3fe220c2879a982fb099948ac440db844b9a112a5188c7783fd3b19593290785f908d95c9db4b280bafe89c1313aeec24772046d9bc089645f0d182a21184e143823c5f52de50e5d7e98d3d7ab56f5413bbccd1415c9bcff707def475b643fb7f29842582104d4cc1dbaaca8f10a2f44273c339e0984f2b1e06ab2f0771db01fafa8142298345f3196f23e5847bda024034b6f59b11c29e981c881456e40d211929fd4f766200258aad8212016322bd5c605790dcfdf1bd2a93d99c9b8f498722d311d7eae7ff420496a31804c55f4759a7b13aaaf5f7ce006c3a8a998897d5e0a504398c2b627852545baf440798bcc5cc049357cf3f17d9771e4528a1af3d77dc794a11346e1bdf5efe37a405b127b4c43b616d61fbc5dc914e14240ef99a7400" +const rand3Hex = "1744b384d68c042371244e13500d4bfb98c6244e3d71a5b700224420b59c593553f33bd786e3d0ce31626f511bc985f59d1a88aa38ba8ad6218d306abee60dd9172540232b95be1af146c69e72e5fde667a090dc3f93bdc5c5af0ab80acdbaa7a505f628c59dc0247b31a439cacf5010a94376d71521df08c178b02fb96fdb1809144ea38c68536187c53201fea8631fb0a880b4451ccdca7cc61f6aafca21cc7449d920599db61789ac3b1e164b3390124f95022aeea39ccca3ec1053f4fa10de2978e2861ea58e477085c2220021a0927aa94c5d0006b5055abba340e4f9eba22e969978dfd18e278a8b89d877328ae34268bc0174cfe211954c0036f078025217d1269fac1932a03b05a0b616012271bbe1fb554171c7a59b196d8a4479f45a77931b5d97aaf6c0c673cbe597b79b96e2a0c1eae2e66e46ccc8c85798e23ffe972ebdaa3f6caea243c004e60321eb47cd79137d78fd0613be606feacc5b3637bdc96a89c13746db8cad886f3ccf912b2178c823bcac395f06d28080269bdca2debf3419c66c690fd1adcfbd53e32e79443d7a42511a84cb22ca94fffad9149275a075b2f8ae0b021dcde9bf62b102db920733b897560518b06e1ad7f4b03458493ddaa7f4fa2c1609f7a1735aeeb1b3e2cea3ab45fc376323cc91873b7e9c90d07c192e38d3f5dfc9bfab1fd821c854da9e607ea596c391c7ec4161c6c4493929a8176badaa5a5af7211c623f29643a937677d3df0da9266181b7c4da5dd40376db677fe8f4a1dc456adf6f33c1e37cec471dd318c2647644fe52f93707a77da7d1702380a80e14cc0fdce7bf2eed48a529090bae0388ee277ce6c7018c5fb00b88362554362205c641f0d0fab94fd5b8357b5ff08b207fee023709bc126ec90cfb17c006754638f8186aaeb1265e80be0c1189ec07d01d5f6f96cb9ce82744147d18490de7dc72862f42f024a16968891a356f5e7e0e695d8c933ba5b5e43ad4c4ade5399bc2cae9bb6189b7870d7f22956194d277f28b10e01c10c6ffe3e065f7e2d6d056aa790db5649ca84dc64c35566c0af1b68c32b5b7874aaa66467afa44f40e9a0846a07ae75360a641dd2acc69d93219b2891f190621511e62a27f5e4fbe641ece1fa234fc7e9a74f48d2a760d82160d9540f649256b169d1fed6fbefdc491126530f3cbad7913e19fbd7aa53b1e243fbf28d5f38c10ebd77c8b986775975cc1d619efb27cdcd733fa1ca36cffe9c0a33cc9f02463c91a886601fd349efee85ef1462065ef9bd2c8f533220ad93138b8382d5938103ab25b2d9af8ae106e1211eb9b18793fba033900c809c02cd6d17e2f3e6fc84dae873411f8e87c3f0a8f1765b7825d185ce3730f299c3028d4a62da9ee95c2b870fb70c79370d485f9d5d9acb78926d20444033d960524d2776dc31988ec7c0dbf23b9905d" diff --git a/src/pkg/compress/bzip2/huffman.go b/src/pkg/compress/bzip2/huffman.go new file mode 100644 index 000000000..dc05739c7 --- /dev/null +++ b/src/pkg/compress/bzip2/huffman.go @@ -0,0 +1,223 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package bzip2 + +import ( + "os" + "sort" +) + +// A huffmanTree is a binary tree which is navigated, bit-by-bit to reach a +// symbol. +type huffmanTree struct { + // nodes contains all the non-leaf nodes in the tree. nodes[0] is the + // root of the tree and nextNode contains the index of the next element + // of nodes to use when the tree is being constructed. + nodes []huffmanNode + nextNode int +} + +// A huffmanNode is a node in the tree. left and right contain indexes into the +// nodes slice of the tree. If left or right is invalidNodeValue then the child +// is a left node and its value is in leftValue/rightValue. +// +// The symbols are uint16s because bzip2 encodes not only MTF indexes in the +// tree, but also two magic values for run-length encoding and an EOF symbol. +// Thus there are more than 256 possible symbols. +type huffmanNode struct { + left, right uint16 + leftValue, rightValue uint16 +} + +// invalidNodeValue is an invalid index which marks a leaf node in the tree. +const invalidNodeValue = 0xffff + +// Decode reads bits from the given bitReader and navigates the tree until a +// symbol is found. +func (t huffmanTree) Decode(br *bitReader) (v uint16) { + nodeIndex := uint16(0) // node 0 is the root of the tree. + + for { + node := &t.nodes[nodeIndex] + bit := br.ReadBit() + // bzip2 encodes left as a true bit. + if bit { + // left + if node.left == invalidNodeValue { + return node.leftValue + } + nodeIndex = node.left + } else { + // right + if node.right == invalidNodeValue { + return node.rightValue + } + nodeIndex = node.right + } + } + + panic("unreachable") +} + +// newHuffmanTree builds a Huffman tree from a slice containing the code +// lengths of each symbol. The maximum code length is 32 bits. +func newHuffmanTree(lengths []uint8) (huffmanTree, os.Error) { + // There are many possible trees that assign the same code length to + // each symbol (consider reflecting a tree down the middle, for + // example). Since the code length assignments determine the + // efficiency of the tree, each of these trees is equally good. In + // order to minimize the amount of information needed to build a tree + // bzip2 uses a canonical tree so that it can be reconstructed given + // only the code length assignments. + + if len(lengths) < 2 { + panic("newHuffmanTree: too few symbols") + } + + var t huffmanTree + + // First we sort the code length assignments by ascending code length, + // using the symbol value to break ties. + pairs := huffmanSymbolLengthPairs(make([]huffmanSymbolLengthPair, len(lengths))) + for i, length := range lengths { + pairs[i].value = uint16(i) + pairs[i].length = length + } + + sort.Sort(pairs) + + // Now we assign codes to the symbols, starting with the longest code. + // We keep the codes packed into a uint32, at the most-significant end. + // So branches are taken from the MSB downwards. This makes it easy to + // sort them later. + code := uint32(0) + length := uint8(32) + + codes := huffmanCodes(make([]huffmanCode, len(lengths))) + for i := len(pairs) - 1; i >= 0; i-- { + if length > pairs[i].length { + // If the code length decreases we shift in order to + // zero any bits beyond the end of the code. + length >>= 32 - pairs[i].length + length <<= 32 - pairs[i].length + length = pairs[i].length + } + codes[i].code = code + codes[i].codeLen = length + codes[i].value = pairs[i].value + // We need to 'increment' the code, which means treating |code| + // like a |length| bit number. + code += 1 << (32 - length) + } + + // Now we can sort by the code so that the left half of each branch are + // grouped together, recursively. + sort.Sort(codes) + + t.nodes = make([]huffmanNode, len(codes)) + _, err := buildHuffmanNode(&t, codes, 0) + return t, err +} + +// huffmanSymbolLengthPair contains a symbol and its code length. +type huffmanSymbolLengthPair struct { + value uint16 + length uint8 +} + +// huffmanSymbolLengthPair is used to provide an interface for sorting. +type huffmanSymbolLengthPairs []huffmanSymbolLengthPair + +func (h huffmanSymbolLengthPairs) Len() int { + return len(h) +} + +func (h huffmanSymbolLengthPairs) Less(i, j int) bool { + if h[i].length < h[j].length { + return true + } + if h[i].length > h[j].length { + return false + } + if h[i].value < h[j].value { + return true + } + return false +} + +func (h huffmanSymbolLengthPairs) Swap(i, j int) { + h[i], h[j] = h[j], h[i] +} + +// huffmanCode contains a symbol, its code and code length. +type huffmanCode struct { + code uint32 + codeLen uint8 + value uint16 +} + +// huffmanCodes is used to provide an interface for sorting. +type huffmanCodes []huffmanCode + +func (n huffmanCodes) Len() int { + return len(n) +} + +func (n huffmanCodes) Less(i, j int) bool { + return n[i].code < n[j].code +} + +func (n huffmanCodes) Swap(i, j int) { + n[i], n[j] = n[j], n[i] +} + +// buildHuffmanNode takes a slice of sorted huffmanCodes and builds a node in +// the Huffman tree at the given level. It returns the index of the newly +// constructed node. +func buildHuffmanNode(t *huffmanTree, codes []huffmanCode, level uint32) (nodeIndex uint16, err os.Error) { + test := uint32(1) << (31 - level) + + // We have to search the list of codes to find the divide between the left and right sides. + firstRightIndex := len(codes) + for i, code := range codes { + if code.code&test != 0 { + firstRightIndex = i + break + } + } + + left := codes[:firstRightIndex] + right := codes[firstRightIndex:] + + if len(left) == 0 || len(right) == 0 { + return 0, StructuralError("superfluous level in Huffman tree") + } + + nodeIndex = uint16(t.nextNode) + node := &t.nodes[t.nextNode] + t.nextNode++ + + if len(left) == 1 { + // leaf node + node.left = invalidNodeValue + node.leftValue = left[0].value + } else { + node.left, err = buildHuffmanNode(t, left, level+1) + } + + if err != nil { + return + } + + if len(right) == 1 { + // leaf node + node.right = invalidNodeValue + node.rightValue = right[0].value + } else { + node.right, err = buildHuffmanNode(t, right, level+1) + } + + return +} diff --git a/src/pkg/compress/bzip2/move_to_front.go b/src/pkg/compress/bzip2/move_to_front.go new file mode 100644 index 000000000..0ed19dec3 --- /dev/null +++ b/src/pkg/compress/bzip2/move_to_front.go @@ -0,0 +1,105 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package bzip2 + +// moveToFrontDecoder implements a move-to-front list. Such a list is an +// efficient way to transform a string with repeating elements into one with +// many small valued numbers, which is suitable for entropy encoding. It works +// by starting with an initial list of symbols and references symbols by their +// index into that list. When a symbol is referenced, it's moved to the front +// of the list. Thus, a repeated symbol ends up being encoded with many zeros, +// as the symbol will be at the front of the list after the first access. +type moveToFrontDecoder struct { + // Rather than actually keep the list in memory, the symbols are stored + // as a circular, double linked list with the symbol indexed by head + // at the front of the list. + symbols []byte + next []uint8 + prev []uint8 + head uint8 +} + +// newMTFDecoder creates a move-to-front decoder with an explicit initial list +// of symbols. +func newMTFDecoder(symbols []byte) *moveToFrontDecoder { + if len(symbols) > 256 { + panic("too many symbols") + } + + m := &moveToFrontDecoder{ + symbols: symbols, + next: make([]uint8, len(symbols)), + prev: make([]uint8, len(symbols)), + } + + m.threadLinkedList() + return m +} + +// newMTFDecoderWithRange creates a move-to-front decoder with an initial +// symbol list of 0...n-1. +func newMTFDecoderWithRange(n int) *moveToFrontDecoder { + if n > 256 { + panic("newMTFDecoderWithRange: cannot have > 256 symbols") + } + + m := &moveToFrontDecoder{ + symbols: make([]uint8, n), + next: make([]uint8, n), + prev: make([]uint8, n), + } + + for i := 0; i < n; i++ { + m.symbols[i] = byte(i) + } + + m.threadLinkedList() + return m +} + +// threadLinkedList creates the initial linked-list pointers. +func (m *moveToFrontDecoder) threadLinkedList() { + if len(m.symbols) == 0 { + return + } + + m.prev[0] = uint8(len(m.symbols) - 1) + + for i := 0; i < len(m.symbols)-1; i++ { + m.next[i] = uint8(i + 1) + m.prev[i+1] = uint8(i) + } + + m.next[len(m.symbols)-1] = 0 +} + +func (m *moveToFrontDecoder) Decode(n int) (b byte) { + // Most of the time, n will be zero so it's worth dealing with this + // simple case. + if n == 0 { + return m.symbols[m.head] + } + + i := m.head + for j := 0; j < n; j++ { + i = m.next[i] + } + b = m.symbols[i] + + m.next[m.prev[i]] = m.next[i] + m.prev[m.next[i]] = m.prev[i] + m.next[i] = m.head + m.prev[i] = m.prev[m.head] + m.next[m.prev[m.head]] = i + m.prev[m.head] = i + m.head = i + + return +} + +// First returns the symbol at the front of the list. +func (m *moveToFrontDecoder) First() byte { + return m.symbols[m.head] +} diff --git a/src/pkg/compress/flate/Makefile b/src/pkg/compress/flate/Makefile new file mode 100644 index 000000000..197828a92 --- /dev/null +++ b/src/pkg/compress/flate/Makefile @@ -0,0 +1,17 @@ +# Copyright 2009 The Go Authors. All rights reserved. +# Use of this source code is governed by a BSD-style +# license that can be found in the LICENSE file. + +include ../../../Make.inc + +TARG=compress/flate +GOFILES=\ + deflate.go\ + huffman_bit_writer.go\ + huffman_code.go\ + inflate.go\ + reverse_bits.go\ + token.go\ + util.go\ + +include ../../../Make.pkg diff --git a/src/pkg/compress/flate/deflate.go b/src/pkg/compress/flate/deflate.go new file mode 100644 index 000000000..b1cee0b2f --- /dev/null +++ b/src/pkg/compress/flate/deflate.go @@ -0,0 +1,493 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package flate + +import ( + "io" + "math" + "os" +) + +const ( + NoCompression = 0 + BestSpeed = 1 + fastCompression = 3 + BestCompression = 9 + DefaultCompression = -1 + logWindowSize = 15 + windowSize = 1 << logWindowSize + windowMask = windowSize - 1 + logMaxOffsetSize = 15 // Standard DEFLATE + minMatchLength = 3 // The smallest match that the compressor looks for + maxMatchLength = 258 // The longest match for the compressor + minOffsetSize = 1 // The shortest offset that makes any sence + + // The maximum number of tokens we put into a single flat block, just too + // stop things from getting too large. + maxFlateBlockTokens = 1 << 14 + maxStoreBlockSize = 65535 + hashBits = 15 + hashSize = 1 << hashBits + hashMask = (1 << hashBits) - 1 + hashShift = (hashBits + minMatchLength - 1) / minMatchLength +) + +type compressionLevel struct { + good, lazy, nice, chain, fastSkipHashing int +} + +var levels = []compressionLevel{ + {}, // 0 + // For levels 1-3 we don't bother trying with lazy matches + {3, 0, 8, 4, 4}, + {3, 0, 16, 8, 5}, + {3, 0, 32, 32, 6}, + // Levels 4-9 use increasingly more lazy matching + // and increasingly stringent conditions for "good enough". + {4, 4, 16, 16, math.MaxInt32}, + {8, 16, 32, 32, math.MaxInt32}, + {8, 16, 128, 128, math.MaxInt32}, + {8, 32, 128, 256, math.MaxInt32}, + {32, 128, 258, 1024, math.MaxInt32}, + {32, 258, 258, 4096, math.MaxInt32}, +} + +type compressor struct { + compressionLevel + + w *huffmanBitWriter + + // compression algorithm + fill func(*compressor, []byte) int // copy data to window + step func(*compressor) // process window + sync bool // requesting flush + + // Input hash chains + // hashHead[hashValue] contains the largest inputIndex with the specified hash value + // If hashHead[hashValue] is within the current window, then + // hashPrev[hashHead[hashValue] & windowMask] contains the previous index + // with the same hash value. + chainHead int + hashHead []int + hashPrev []int + + // input window: unprocessed data is window[index:windowEnd] + index int + window []byte + windowEnd int + blockStart int // window index where current tokens start + byteAvailable bool // if true, still need to process window[index-1]. + + // queued output tokens: tokens[:ti] + tokens []token + ti int + + // deflate state + length int + offset int + hash int + maxInsertIndex int + err os.Error +} + +func (d *compressor) fillDeflate(b []byte) int { + if d.index >= 2*windowSize-(minMatchLength+maxMatchLength) { + // shift the window by windowSize + copy(d.window, d.window[windowSize:2*windowSize]) + d.index -= windowSize + d.windowEnd -= windowSize + if d.blockStart >= windowSize { + d.blockStart -= windowSize + } else { + d.blockStart = math.MaxInt32 + } + for i, h := range d.hashHead { + v := h - windowSize + if v < -1 { + v = -1 + } + d.hashHead[i] = v + } + for i, h := range d.hashPrev { + v := -h - windowSize + if v < -1 { + v = -1 + } + d.hashPrev[i] = v + } + } + n := copy(d.window[d.windowEnd:], b) + d.windowEnd += n + return n +} + +func (d *compressor) writeBlock(tokens []token, index int, eof bool) os.Error { + if index > 0 || eof { + var window []byte + if d.blockStart <= index { + window = d.window[d.blockStart:index] + } + d.blockStart = index + d.w.writeBlock(tokens, eof, window) + return d.w.err + } + return nil +} + +// Try to find a match starting at index whose length is greater than prevSize. +// We only look at chainCount possibilities before giving up. +func (d *compressor) findMatch(pos int, prevHead int, prevLength int, lookahead int) (length, offset int, ok bool) { + minMatchLook := maxMatchLength + if lookahead < minMatchLook { + minMatchLook = lookahead + } + + win := d.window[0 : pos+minMatchLook] + + // We quit when we get a match that's at least nice long + nice := len(win) - pos + if d.nice < nice { + nice = d.nice + } + + // If we've got a match that's good enough, only look in 1/4 the chain. + tries := d.chain + length = prevLength + if length >= d.good { + tries >>= 2 + } + + w0 := win[pos] + w1 := win[pos+1] + wEnd := win[pos+length] + minIndex := pos - windowSize + + for i := prevHead; tries > 0; tries-- { + if w0 == win[i] && w1 == win[i+1] && wEnd == win[i+length] { + // The hash function ensures that if win[i] and win[i+1] match, win[i+2] matches + + n := 3 + for pos+n < len(win) && win[i+n] == win[pos+n] { + n++ + } + if n > length && (n > 3 || pos-i <= 4096) { + length = n + offset = pos - i + ok = true + if n >= nice { + // The match is good enough that we don't try to find a better one. + break + } + wEnd = win[pos+n] + } + } + if i == minIndex { + // hashPrev[i & windowMask] has already been overwritten, so stop now. + break + } + if i = d.hashPrev[i&windowMask]; i < minIndex || i < 0 { + break + } + } + return +} + +func (d *compressor) writeStoredBlock(buf []byte) os.Error { + if d.w.writeStoredHeader(len(buf), false); d.w.err != nil { + return d.w.err + } + d.w.writeBytes(buf) + return d.w.err +} + +func (d *compressor) initDeflate() { + d.hashHead = make([]int, hashSize) + d.hashPrev = make([]int, windowSize) + d.window = make([]byte, 2*windowSize) + fillInts(d.hashHead, -1) + d.tokens = make([]token, maxFlateBlockTokens, maxFlateBlockTokens+1) + d.length = minMatchLength - 1 + d.offset = 0 + d.byteAvailable = false + d.index = 0 + d.ti = 0 + d.hash = 0 + d.chainHead = -1 +} + +func (d *compressor) deflate() { + if d.windowEnd-d.index < minMatchLength+maxMatchLength && !d.sync { + return + } + + d.maxInsertIndex = d.windowEnd - (minMatchLength - 1) + if d.index < d.maxInsertIndex { + d.hash = int(d.window[d.index])< d.windowEnd { + panic("index > windowEnd") + } + lookahead := d.windowEnd - d.index + if lookahead < minMatchLength+maxMatchLength { + if !d.sync { + break Loop + } + if d.index > d.windowEnd { + panic("index > windowEnd") + } + if lookahead == 0 { + // Flush current output block if any. + if d.byteAvailable { + // There is still one pending token that needs to be flushed + d.tokens[d.ti] = literalToken(uint32(d.window[d.index-1])) + d.ti++ + d.byteAvailable = false + } + if d.ti > 0 { + if d.err = d.writeBlock(d.tokens[0:d.ti], d.index, false); d.err != nil { + return + } + d.ti = 0 + } + break Loop + } + } + if d.index < d.maxInsertIndex { + // Update the hash + d.hash = (d.hash<= minIndex && + (d.fastSkipHashing != 0 && lookahead > minMatchLength-1 || + d.fastSkipHashing == 0 && lookahead > prevLength && prevLength < d.lazy) { + if newLength, newOffset, ok := d.findMatch(d.index, d.chainHead, minMatchLength-1, lookahead); ok { + d.length = newLength + d.offset = newOffset + } + } + if d.fastSkipHashing != 0 && d.length >= minMatchLength || + d.fastSkipHashing == 0 && prevLength >= minMatchLength && d.length <= prevLength { + // There was a match at the previous step, and the current match is + // not better. Output the previous match. + if d.fastSkipHashing != 0 { + d.tokens[d.ti] = matchToken(uint32(d.length-minMatchLength), uint32(d.offset-minOffsetSize)) + } else { + d.tokens[d.ti] = matchToken(uint32(prevLength-minMatchLength), uint32(prevOffset-minOffsetSize)) + } + d.ti++ + // Insert in the hash table all strings up to the end of the match. + // index and index-1 are already inserted. If there is not enough + // lookahead, the last two strings are not inserted into the hash + // table. + if d.length <= d.fastSkipHashing { + var newIndex int + if d.fastSkipHashing != 0 { + newIndex = d.index + d.length + } else { + newIndex = prevLength - 1 + } + for d.index++; d.index < newIndex; d.index++ { + if d.index < d.maxInsertIndex { + d.hash = (d.hash< 0 { + d.err = d.writeStoredBlock(d.window[:d.windowEnd]) + } + d.windowEnd = 0 +} + +func (d *compressor) write(b []byte) (n int, err os.Error) { + n = len(b) + b = b[d.fill(d, b):] + for len(b) > 0 { + d.step(d) + b = b[d.fill(d, b):] + } + return n, d.err +} + +func (d *compressor) syncFlush() os.Error { + d.sync = true + d.step(d) + if d.err == nil { + d.w.writeStoredHeader(0, false) + d.w.flush() + d.err = d.w.err + } + d.sync = false + return d.err +} + +func (d *compressor) init(w io.Writer, level int) (err os.Error) { + d.w = newHuffmanBitWriter(w) + + switch { + case level == NoCompression: + d.window = make([]byte, maxStoreBlockSize) + d.fill = (*compressor).fillStore + d.step = (*compressor).store + case level == DefaultCompression: + level = 6 + fallthrough + case 1 <= level && level <= 9: + d.compressionLevel = levels[level] + d.initDeflate() + d.fill = (*compressor).fillDeflate + d.step = (*compressor).deflate + default: + return WrongValueError{"level", 0, 9, int32(level)} + } + return nil +} + +func (d *compressor) close() os.Error { + d.sync = true + d.step(d) + if d.err != nil { + return d.err + } + if d.w.writeStoredHeader(0, true); d.w.err != nil { + return d.w.err + } + d.w.flush() + return d.w.err +} + +// NewWriter returns a new Writer compressing +// data at the given level. Following zlib, levels +// range from 1 (BestSpeed) to 9 (BestCompression); +// higher levels typically run slower but compress more. +// Level 0 (NoCompression) does not attempt any +// compression; it only adds the necessary DEFLATE framing. +func NewWriter(w io.Writer, level int) *Writer { + const logWindowSize = logMaxOffsetSize + var dw Writer + dw.d.init(w, level) + return &dw +} + +// NewWriterDict is like NewWriter but initializes the new +// Writer with a preset dictionary. The returned Writer behaves +// as if the dictionary had been written to it without producing +// any compressed output. The compressed data written to w +// can only be decompressed by a Reader initialized with the +// same dictionary. +func NewWriterDict(w io.Writer, level int, dict []byte) *Writer { + dw := &dictWriter{w, false} + zw := NewWriter(dw, level) + zw.Write(dict) + zw.Flush() + dw.enabled = true + return zw +} + +type dictWriter struct { + w io.Writer + enabled bool +} + +func (w *dictWriter) Write(b []byte) (n int, err os.Error) { + if w.enabled { + return w.w.Write(b) + } + return len(b), nil +} + +// A Writer takes data written to it and writes the compressed +// form of that data to an underlying writer (see NewWriter). +type Writer struct { + d compressor +} + +// Write writes data to w, which will eventually write the +// compressed form of data to its underlying writer. +func (w *Writer) Write(data []byte) (n int, err os.Error) { + return w.d.write(data) +} + +// Flush flushes any pending compressed data to the underlying writer. +// It is useful mainly in compressed network protocols, to ensure that +// a remote reader has enough data to reconstruct a packet. +// Flush does not return until the data has been written. +// If the underlying writer returns an error, Flush returns that error. +// +// In the terminology of the zlib library, Flush is equivalent to Z_SYNC_FLUSH. +func (w *Writer) Flush() os.Error { + // For more about flushing: + // http://www.bolet.org/~pornin/deflate-flush.html + return w.d.syncFlush() +} + +// Close flushes and closes the writer. +func (w *Writer) Close() os.Error { + return w.d.close() +} diff --git a/src/pkg/compress/flate/deflate_test.go b/src/pkg/compress/flate/deflate_test.go new file mode 100644 index 000000000..930823685 --- /dev/null +++ b/src/pkg/compress/flate/deflate_test.go @@ -0,0 +1,321 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package flate + +import ( + "bytes" + "fmt" + "io" + "io/ioutil" + "os" + "sync" + "testing" +) + +type deflateTest struct { + in []byte + level int + out []byte +} + +type deflateInflateTest struct { + in []byte +} + +type reverseBitsTest struct { + in uint16 + bitCount uint8 + out uint16 +} + +var deflateTests = []*deflateTest{ + &deflateTest{[]byte{}, 0, []byte{1, 0, 0, 255, 255}}, + &deflateTest{[]byte{0x11}, -1, []byte{18, 4, 4, 0, 0, 255, 255}}, + &deflateTest{[]byte{0x11}, DefaultCompression, []byte{18, 4, 4, 0, 0, 255, 255}}, + &deflateTest{[]byte{0x11}, 4, []byte{18, 4, 4, 0, 0, 255, 255}}, + + &deflateTest{[]byte{0x11}, 0, []byte{0, 1, 0, 254, 255, 17, 1, 0, 0, 255, 255}}, + &deflateTest{[]byte{0x11, 0x12}, 0, []byte{0, 2, 0, 253, 255, 17, 18, 1, 0, 0, 255, 255}}, + &deflateTest{[]byte{0x11, 0x11, 0x11, 0x11, 0x11, 0x11, 0x11, 0x11}, 0, + []byte{0, 8, 0, 247, 255, 17, 17, 17, 17, 17, 17, 17, 17, 1, 0, 0, 255, 255}, + }, + &deflateTest{[]byte{}, 1, []byte{1, 0, 0, 255, 255}}, + &deflateTest{[]byte{0x11}, 1, []byte{18, 4, 4, 0, 0, 255, 255}}, + &deflateTest{[]byte{0x11, 0x12}, 1, []byte{18, 20, 2, 4, 0, 0, 255, 255}}, + &deflateTest{[]byte{0x11, 0x11, 0x11, 0x11, 0x11, 0x11, 0x11, 0x11}, 1, []byte{18, 132, 2, 64, 0, 0, 0, 255, 255}}, + &deflateTest{[]byte{}, 9, []byte{1, 0, 0, 255, 255}}, + &deflateTest{[]byte{0x11}, 9, []byte{18, 4, 4, 0, 0, 255, 255}}, + &deflateTest{[]byte{0x11, 0x12}, 9, []byte{18, 20, 2, 4, 0, 0, 255, 255}}, + &deflateTest{[]byte{0x11, 0x11, 0x11, 0x11, 0x11, 0x11, 0x11, 0x11}, 9, []byte{18, 132, 2, 64, 0, 0, 0, 255, 255}}, +} + +var deflateInflateTests = []*deflateInflateTest{ + &deflateInflateTest{[]byte{}}, + &deflateInflateTest{[]byte{0x11}}, + &deflateInflateTest{[]byte{0x11, 0x12}}, + &deflateInflateTest{[]byte{0x11, 0x11, 0x11, 0x11, 0x11, 0x11, 0x11, 0x11}}, + &deflateInflateTest{[]byte{0x11, 0x10, 0x13, 0x41, 0x21, 0x21, 0x41, 0x13, 0x87, 0x78, 0x13}}, + &deflateInflateTest{largeDataChunk()}, +} + +var reverseBitsTests = []*reverseBitsTest{ + &reverseBitsTest{1, 1, 1}, + &reverseBitsTest{1, 2, 2}, + &reverseBitsTest{1, 3, 4}, + &reverseBitsTest{1, 4, 8}, + &reverseBitsTest{1, 5, 16}, + &reverseBitsTest{17, 5, 17}, + &reverseBitsTest{257, 9, 257}, + &reverseBitsTest{29, 5, 23}, +} + +func largeDataChunk() []byte { + result := make([]byte, 100000) + for i := range result { + result[i] = byte(i * i & 0xFF) + } + return result +} + +func TestDeflate(t *testing.T) { + for _, h := range deflateTests { + var buf bytes.Buffer + w := NewWriter(&buf, h.level) + w.Write(h.in) + w.Close() + if !bytes.Equal(buf.Bytes(), h.out) { + t.Errorf("Deflate(%d, %x) = %x, want %x", h.level, h.in, buf.Bytes(), h.out) + } + } +} + +type syncBuffer struct { + buf bytes.Buffer + mu sync.RWMutex + closed bool + ready chan bool +} + +func newSyncBuffer() *syncBuffer { + return &syncBuffer{ready: make(chan bool, 1)} +} + +func (b *syncBuffer) Read(p []byte) (n int, err os.Error) { + for { + b.mu.RLock() + n, err = b.buf.Read(p) + b.mu.RUnlock() + if n > 0 || b.closed { + return + } + <-b.ready + } + panic("unreachable") +} + +func (b *syncBuffer) signal() { + select { + case b.ready <- true: + default: + } +} + +func (b *syncBuffer) Write(p []byte) (n int, err os.Error) { + n, err = b.buf.Write(p) + b.signal() + return +} + +func (b *syncBuffer) WriteMode() { + b.mu.Lock() +} + +func (b *syncBuffer) ReadMode() { + b.mu.Unlock() + b.signal() +} + +func (b *syncBuffer) Close() os.Error { + b.closed = true + b.signal() + return nil +} + +func testSync(t *testing.T, level int, input []byte, name string) { + if len(input) == 0 { + return + } + + t.Logf("--testSync %d, %d, %s", level, len(input), name) + buf := newSyncBuffer() + buf1 := new(bytes.Buffer) + buf.WriteMode() + w := NewWriter(io.MultiWriter(buf, buf1), level) + r := NewReader(buf) + + // Write half the input and read back. + for i := 0; i < 2; i++ { + var lo, hi int + if i == 0 { + lo, hi = 0, (len(input)+1)/2 + } else { + lo, hi = (len(input)+1)/2, len(input) + } + t.Logf("#%d: write %d-%d", i, lo, hi) + if _, err := w.Write(input[lo:hi]); err != nil { + t.Errorf("testSync: write: %v", err) + return + } + if i == 0 { + if err := w.Flush(); err != nil { + t.Errorf("testSync: flush: %v", err) + return + } + } else { + if err := w.Close(); err != nil { + t.Errorf("testSync: close: %v", err) + } + } + buf.ReadMode() + out := make([]byte, hi-lo+1) + m, err := io.ReadAtLeast(r, out, hi-lo) + t.Logf("#%d: read %d", i, m) + if m != hi-lo || err != nil { + t.Errorf("testSync/%d (%d, %d, %s): read %d: %d, %v (%d left)", i, level, len(input), name, hi-lo, m, err, buf.buf.Len()) + return + } + if !bytes.Equal(input[lo:hi], out[:hi-lo]) { + t.Errorf("testSync/%d: read wrong bytes: %x vs %x", i, input[lo:hi], out[:hi-lo]) + return + } + // This test originally checked that after reading + // the first half of the input, there was nothing left + // in the read buffer (buf.buf.Len() != 0) but that is + // not necessarily the case: the write Flush may emit + // some extra framing bits that are not necessary + // to process to obtain the first half of the uncompressed + // data. The test ran correctly most of the time, because + // the background goroutine had usually read even + // those extra bits by now, but it's not a useful thing to + // check. + buf.WriteMode() + } + buf.ReadMode() + out := make([]byte, 10) + if n, err := r.Read(out); n > 0 || err != os.EOF { + t.Errorf("testSync (%d, %d, %s): final Read: %d, %v (hex: %x)", level, len(input), name, n, err, out[0:n]) + } + if buf.buf.Len() != 0 { + t.Errorf("testSync (%d, %d, %s): extra data at end", level, len(input), name) + } + r.Close() + + // stream should work for ordinary reader too + r = NewReader(buf1) + out, err := ioutil.ReadAll(r) + if err != nil { + t.Errorf("testSync: read: %s", err) + return + } + r.Close() + if !bytes.Equal(input, out) { + t.Errorf("testSync: decompress(compress(data)) != data: level=%d input=%s", level, name) + } +} + +func testToFromWithLevel(t *testing.T, level int, input []byte, name string) os.Error { + buffer := bytes.NewBuffer(nil) + w := NewWriter(buffer, level) + w.Write(input) + w.Close() + r := NewReader(buffer) + out, err := ioutil.ReadAll(r) + if err != nil { + t.Errorf("read: %s", err) + return err + } + r.Close() + if !bytes.Equal(input, out) { + t.Errorf("decompress(compress(data)) != data: level=%d input=%s", level, name) + } + + testSync(t, level, input, name) + return nil +} + +func testToFrom(t *testing.T, input []byte, name string) { + for i := 0; i < 10; i++ { + testToFromWithLevel(t, i, input, name) + } +} + +func TestDeflateInflate(t *testing.T) { + for i, h := range deflateInflateTests { + testToFrom(t, h.in, fmt.Sprintf("#%d", i)) + } +} + +func TestReverseBits(t *testing.T) { + for _, h := range reverseBitsTests { + if v := reverseBits(h.in, h.bitCount); v != h.out { + t.Errorf("reverseBits(%v,%v) = %v, want %v", + h.in, h.bitCount, v, h.out) + } + } +} + +func TestDeflateInflateString(t *testing.T) { + gold, err := ioutil.ReadFile("../testdata/e.txt") + if err != nil { + t.Error(err) + } + testToFromWithLevel(t, 1, gold, "2.718281828...") +} + +func TestReaderDict(t *testing.T) { + const ( + dict = "hello world" + text = "hello again world" + ) + var b bytes.Buffer + w := NewWriter(&b, 5) + w.Write([]byte(dict)) + w.Flush() + b.Reset() + w.Write([]byte(text)) + w.Close() + + r := NewReaderDict(&b, []byte(dict)) + data, err := ioutil.ReadAll(r) + if err != nil { + t.Fatal(err) + } + if string(data) != "hello again world" { + t.Fatalf("read returned %q want %q", string(data), text) + } +} + +func TestWriterDict(t *testing.T) { + const ( + dict = "hello world" + text = "hello again world" + ) + var b bytes.Buffer + w := NewWriter(&b, 5) + w.Write([]byte(dict)) + w.Flush() + b.Reset() + w.Write([]byte(text)) + w.Close() + + var b1 bytes.Buffer + w = NewWriterDict(&b1, 5, []byte(dict)) + w.Write([]byte(text)) + w.Close() + + if !bytes.Equal(b1.Bytes(), b.Bytes()) { + t.Fatalf("writer wrote %q want %q", b1.Bytes(), b.Bytes()) + } +} diff --git a/src/pkg/compress/flate/flate_test.go b/src/pkg/compress/flate/flate_test.go new file mode 100644 index 000000000..bfd3b8381 --- /dev/null +++ b/src/pkg/compress/flate/flate_test.go @@ -0,0 +1,139 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// This test tests some internals of the flate package. +// The tests in package compress/gzip serve as the +// end-to-end test of the decompressor. + +package flate + +import ( + "bytes" + "reflect" + "testing" +) + +// The Huffman code lengths used by the fixed-format Huffman blocks. +var fixedHuffmanBits = [...]int{ + // 0-143 length 8 + 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, + 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, + 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, + 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, + 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, + 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, + 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, + 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, + 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, + + // 144-255 length 9 + 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + + // 256-279 length 7 + 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, + 7, 7, 7, 7, 7, 7, 7, 7, + + // 280-287 length 8 + 8, 8, 8, 8, 8, 8, 8, 8, +} + +type InitDecoderTest struct { + in []int + out huffmanDecoder + ok bool +} + +var initDecoderTests = []*InitDecoderTest{ + // Example from Connell 1973, + &InitDecoderTest{ + []int{3, 5, 2, 4, 3, 5, 5, 4, 4, 3, 4, 5}, + huffmanDecoder{ + 2, 5, + [maxCodeLen + 1]int{2: 0, 4, 13, 31}, + [maxCodeLen + 1]int{2: 0, 1, 6, 20}, + // Paper used different code assignment: + // 2, 9, 4, 0, 10, 8, 3, 7, 1, 5, 11, 6 + // Reordered here so that codes of same length + // are assigned to increasing numbers. + []int{2, 0, 4, 9, 3, 7, 8, 10, 1, 5, 6, 11}, + }, + true, + }, + + // Example from RFC 1951 section 3.2.2 + &InitDecoderTest{ + []int{2, 1, 3, 3}, + huffmanDecoder{ + 1, 3, + [maxCodeLen + 1]int{1: 0, 2, 7}, + [maxCodeLen + 1]int{1: 0, 1, 4}, + []int{1, 0, 2, 3}, + }, + true, + }, + + // Second example from RFC 1951 section 3.2.2 + &InitDecoderTest{ + []int{3, 3, 3, 3, 3, 2, 4, 4}, + huffmanDecoder{ + 2, 4, + [maxCodeLen + 1]int{2: 0, 6, 15}, + [maxCodeLen + 1]int{2: 0, 1, 8}, + []int{5, 0, 1, 2, 3, 4, 6, 7}, + }, + true, + }, + + // Static Huffman codes (RFC 1951 section 3.2.6) + &InitDecoderTest{ + fixedHuffmanBits[0:], + fixedHuffmanDecoder, + true, + }, + + // Illegal input. + &InitDecoderTest{ + []int{}, + huffmanDecoder{}, + false, + }, + + // Illegal input. + &InitDecoderTest{ + []int{0, 0, 0, 0, 0, 0, 0}, + huffmanDecoder{}, + false, + }, +} + +func TestInitDecoder(t *testing.T) { + for i, tt := range initDecoderTests { + var h huffmanDecoder + if h.init(tt.in) != tt.ok { + t.Errorf("test %d: init = %v", i, !tt.ok) + continue + } + if !reflect.DeepEqual(&h, &tt.out) { + t.Errorf("test %d:\nhave %v\nwant %v", i, h, tt.out) + } + } +} + +func TestUncompressedSource(t *testing.T) { + decoder := NewReader(bytes.NewBuffer([]byte{0x01, 0x01, 0x00, 0xfe, 0xff, 0x11})) + output := make([]byte, 1) + n, error := decoder.Read(output) + if n != 1 || error != nil { + t.Fatalf("decoder.Read() = %d, %v, want 1, nil", n, error) + } + if output[0] != 0x11 { + t.Errorf("output[0] = %x, want 0x11", output[0]) + } +} diff --git a/src/pkg/compress/flate/huffman_bit_writer.go b/src/pkg/compress/flate/huffman_bit_writer.go new file mode 100644 index 000000000..3981df5cb --- /dev/null +++ b/src/pkg/compress/flate/huffman_bit_writer.go @@ -0,0 +1,494 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package flate + +import ( + "io" + "math" + "os" + "strconv" +) + +const ( + // The largest offset code. + offsetCodeCount = 30 + + // The special code used to mark the end of a block. + endBlockMarker = 256 + + // The first length code. + lengthCodesStart = 257 + + // The number of codegen codes. + codegenCodeCount = 19 + badCode = 255 +) + +// The number of extra bits needed by length code X - LENGTH_CODES_START. +var lengthExtraBits = []int8{ + /* 257 */ 0, 0, 0, + /* 260 */ 0, 0, 0, 0, 0, 1, 1, 1, 1, 2, + /* 270 */ 2, 2, 2, 3, 3, 3, 3, 4, 4, 4, + /* 280 */ 4, 5, 5, 5, 5, 0, +} + +// The length indicated by length code X - LENGTH_CODES_START. +var lengthBase = []uint32{ + 0, 1, 2, 3, 4, 5, 6, 7, 8, 10, + 12, 14, 16, 20, 24, 28, 32, 40, 48, 56, + 64, 80, 96, 112, 128, 160, 192, 224, 255, +} + +// offset code word extra bits. +var offsetExtraBits = []int8{ + 0, 0, 0, 0, 1, 1, 2, 2, 3, 3, + 4, 4, 5, 5, 6, 6, 7, 7, 8, 8, + 9, 9, 10, 10, 11, 11, 12, 12, 13, 13, + /* extended window */ + 14, 14, 15, 15, 16, 16, 17, 17, 18, 18, 19, 19, 20, 20, +} + +var offsetBase = []uint32{ + /* normal deflate */ + 0x000000, 0x000001, 0x000002, 0x000003, 0x000004, + 0x000006, 0x000008, 0x00000c, 0x000010, 0x000018, + 0x000020, 0x000030, 0x000040, 0x000060, 0x000080, + 0x0000c0, 0x000100, 0x000180, 0x000200, 0x000300, + 0x000400, 0x000600, 0x000800, 0x000c00, 0x001000, + 0x001800, 0x002000, 0x003000, 0x004000, 0x006000, + + /* extended window */ + 0x008000, 0x00c000, 0x010000, 0x018000, 0x020000, + 0x030000, 0x040000, 0x060000, 0x080000, 0x0c0000, + 0x100000, 0x180000, 0x200000, 0x300000, +} + +// The odd order in which the codegen code sizes are written. +var codegenOrder = []uint32{16, 17, 18, 0, 8, 7, 9, 6, 10, 5, 11, 4, 12, 3, 13, 2, 14, 1, 15} + +type huffmanBitWriter struct { + w io.Writer + // Data waiting to be written is bytes[0:nbytes] + // and then the low nbits of bits. + bits uint32 + nbits uint32 + bytes [64]byte + nbytes int + literalFreq []int32 + offsetFreq []int32 + codegen []uint8 + codegenFreq []int32 + literalEncoding *huffmanEncoder + offsetEncoding *huffmanEncoder + codegenEncoding *huffmanEncoder + err os.Error +} + +type WrongValueError struct { + name string + from int32 + to int32 + value int32 +} + +func newHuffmanBitWriter(w io.Writer) *huffmanBitWriter { + return &huffmanBitWriter{ + w: w, + literalFreq: make([]int32, maxLit), + offsetFreq: make([]int32, offsetCodeCount), + codegen: make([]uint8, maxLit+offsetCodeCount+1), + codegenFreq: make([]int32, codegenCodeCount), + literalEncoding: newHuffmanEncoder(maxLit), + offsetEncoding: newHuffmanEncoder(offsetCodeCount), + codegenEncoding: newHuffmanEncoder(codegenCodeCount), + } +} + +func (err WrongValueError) String() string { + return "huffmanBitWriter: " + err.name + " should belong to [" + strconv.Itoa64(int64(err.from)) + ";" + + strconv.Itoa64(int64(err.to)) + "] but actual value is " + strconv.Itoa64(int64(err.value)) +} + +func (w *huffmanBitWriter) flushBits() { + if w.err != nil { + w.nbits = 0 + return + } + bits := w.bits + w.bits >>= 16 + w.nbits -= 16 + n := w.nbytes + w.bytes[n] = byte(bits) + w.bytes[n+1] = byte(bits >> 8) + if n += 2; n >= len(w.bytes) { + _, w.err = w.w.Write(w.bytes[0:]) + n = 0 + } + w.nbytes = n +} + +func (w *huffmanBitWriter) flush() { + if w.err != nil { + w.nbits = 0 + return + } + n := w.nbytes + if w.nbits > 8 { + w.bytes[n] = byte(w.bits) + w.bits >>= 8 + w.nbits -= 8 + n++ + } + if w.nbits > 0 { + w.bytes[n] = byte(w.bits) + w.nbits = 0 + n++ + } + w.bits = 0 + _, w.err = w.w.Write(w.bytes[0:n]) + w.nbytes = 0 +} + +func (w *huffmanBitWriter) writeBits(b, nb int32) { + w.bits |= uint32(b) << w.nbits + if w.nbits += uint32(nb); w.nbits >= 16 { + w.flushBits() + } +} + +func (w *huffmanBitWriter) writeBytes(bytes []byte) { + if w.err != nil { + return + } + n := w.nbytes + if w.nbits == 8 { + w.bytes[n] = byte(w.bits) + w.nbits = 0 + n++ + } + if w.nbits != 0 { + w.err = InternalError("writeBytes with unfinished bits") + return + } + if n != 0 { + _, w.err = w.w.Write(w.bytes[0:n]) + if w.err != nil { + return + } + } + w.nbytes = 0 + _, w.err = w.w.Write(bytes) +} + +// RFC 1951 3.2.7 specifies a special run-length encoding for specifying +// the literal and offset lengths arrays (which are concatenated into a single +// array). This method generates that run-length encoding. +// +// The result is written into the codegen array, and the frequencies +// of each code is written into the codegenFreq array. +// Codes 0-15 are single byte codes. Codes 16-18 are followed by additional +// information. Code badCode is an end marker +// +// numLiterals The number of literals in literalEncoding +// numOffsets The number of offsets in offsetEncoding +func (w *huffmanBitWriter) generateCodegen(numLiterals int, numOffsets int) { + fillInt32s(w.codegenFreq, 0) + // Note that we are using codegen both as a temporary variable for holding + // a copy of the frequencies, and as the place where we put the result. + // This is fine because the output is always shorter than the input used + // so far. + codegen := w.codegen // cache + // Copy the concatenated code sizes to codegen. Put a marker at the end. + copyUint8s(codegen[0:numLiterals], w.literalEncoding.codeBits) + copyUint8s(codegen[numLiterals:numLiterals+numOffsets], w.offsetEncoding.codeBits) + codegen[numLiterals+numOffsets] = badCode + + size := codegen[0] + count := 1 + outIndex := 0 + for inIndex := 1; size != badCode; inIndex++ { + // INVARIANT: We have seen "count" copies of size that have not yet + // had output generated for them. + nextSize := codegen[inIndex] + if nextSize == size { + count++ + continue + } + // We need to generate codegen indicating "count" of size. + if size != 0 { + codegen[outIndex] = size + outIndex++ + w.codegenFreq[size]++ + count-- + for count >= 3 { + n := min(count, 6) + codegen[outIndex] = 16 + outIndex++ + codegen[outIndex] = uint8(n - 3) + outIndex++ + w.codegenFreq[16]++ + count -= n + } + } else { + for count >= 11 { + n := min(count, 138) + codegen[outIndex] = 18 + outIndex++ + codegen[outIndex] = uint8(n - 11) + outIndex++ + w.codegenFreq[18]++ + count -= n + } + if count >= 3 { + // count >= 3 && count <= 10 + codegen[outIndex] = 17 + outIndex++ + codegen[outIndex] = uint8(count - 3) + outIndex++ + w.codegenFreq[17]++ + count = 0 + } + } + count-- + for ; count >= 0; count-- { + codegen[outIndex] = size + outIndex++ + w.codegenFreq[size]++ + } + // Set up invariant for next time through the loop. + size = nextSize + count = 1 + } + // Marker indicating the end of the codegen. + codegen[outIndex] = badCode +} + +func (w *huffmanBitWriter) writeCode(code *huffmanEncoder, literal uint32) { + if w.err != nil { + return + } + w.writeBits(int32(code.code[literal]), int32(code.codeBits[literal])) +} + +// Write the header of a dynamic Huffman block to the output stream. +// +// numLiterals The number of literals specified in codegen +// numOffsets The number of offsets specified in codegen +// numCodegens The number of codegens used in codegen +func (w *huffmanBitWriter) writeDynamicHeader(numLiterals int, numOffsets int, numCodegens int, isEof bool) { + if w.err != nil { + return + } + var firstBits int32 = 4 + if isEof { + firstBits = 5 + } + w.writeBits(firstBits, 3) + w.writeBits(int32(numLiterals-257), 5) + w.writeBits(int32(numOffsets-1), 5) + w.writeBits(int32(numCodegens-4), 4) + + for i := 0; i < numCodegens; i++ { + value := w.codegenEncoding.codeBits[codegenOrder[i]] + w.writeBits(int32(value), 3) + } + + i := 0 + for { + var codeWord int = int(w.codegen[i]) + i++ + if codeWord == badCode { + break + } + // The low byte contains the actual code to generate. + w.writeCode(w.codegenEncoding, uint32(codeWord)) + + switch codeWord { + case 16: + w.writeBits(int32(w.codegen[i]), 2) + i++ + break + case 17: + w.writeBits(int32(w.codegen[i]), 3) + i++ + break + case 18: + w.writeBits(int32(w.codegen[i]), 7) + i++ + break + } + } +} + +func (w *huffmanBitWriter) writeStoredHeader(length int, isEof bool) { + if w.err != nil { + return + } + var flag int32 + if isEof { + flag = 1 + } + w.writeBits(flag, 3) + w.flush() + w.writeBits(int32(length), 16) + w.writeBits(int32(^uint16(length)), 16) +} + +func (w *huffmanBitWriter) writeFixedHeader(isEof bool) { + if w.err != nil { + return + } + // Indicate that we are a fixed Huffman block + var value int32 = 2 + if isEof { + value = 3 + } + w.writeBits(value, 3) +} + +func (w *huffmanBitWriter) writeBlock(tokens []token, eof bool, input []byte) { + if w.err != nil { + return + } + fillInt32s(w.literalFreq, 0) + fillInt32s(w.offsetFreq, 0) + + n := len(tokens) + tokens = tokens[0 : n+1] + tokens[n] = endBlockMarker + + for _, t := range tokens { + switch t.typ() { + case literalType: + w.literalFreq[t.literal()]++ + case matchType: + length := t.length() + offset := t.offset() + w.literalFreq[lengthCodesStart+lengthCode(length)]++ + w.offsetFreq[offsetCode(offset)]++ + } + } + + // get the number of literals + numLiterals := len(w.literalFreq) + for w.literalFreq[numLiterals-1] == 0 { + numLiterals-- + } + // get the number of offsets + numOffsets := len(w.offsetFreq) + for numOffsets > 0 && w.offsetFreq[numOffsets-1] == 0 { + numOffsets-- + } + if numOffsets == 0 { + // We haven't found a single match. If we want to go with the dynamic encoding, + // we should count at least one offset to be sure that the offset huffman tree could be encoded. + w.offsetFreq[0] = 1 + numOffsets = 1 + } + + w.literalEncoding.generate(w.literalFreq, 15) + w.offsetEncoding.generate(w.offsetFreq, 15) + + storedBytes := 0 + if input != nil { + storedBytes = len(input) + } + var extraBits int64 + var storedSize int64 = math.MaxInt64 + if storedBytes <= maxStoreBlockSize && input != nil { + storedSize = int64((storedBytes + 5) * 8) + // We only bother calculating the costs of the extra bits required by + // the length of offset fields (which will be the same for both fixed + // and dynamic encoding), if we need to compare those two encodings + // against stored encoding. + for lengthCode := lengthCodesStart + 8; lengthCode < numLiterals; lengthCode++ { + // First eight length codes have extra size = 0. + extraBits += int64(w.literalFreq[lengthCode]) * int64(lengthExtraBits[lengthCode-lengthCodesStart]) + } + for offsetCode := 4; offsetCode < numOffsets; offsetCode++ { + // First four offset codes have extra size = 0. + extraBits += int64(w.offsetFreq[offsetCode]) * int64(offsetExtraBits[offsetCode]) + } + } + + // Figure out smallest code. + // Fixed Huffman baseline. + var size = int64(3) + + fixedLiteralEncoding.bitLength(w.literalFreq) + + fixedOffsetEncoding.bitLength(w.offsetFreq) + + extraBits + var literalEncoding = fixedLiteralEncoding + var offsetEncoding = fixedOffsetEncoding + + // Dynamic Huffman? + var numCodegens int + + // Generate codegen and codegenFrequencies, which indicates how to encode + // the literalEncoding and the offsetEncoding. + w.generateCodegen(numLiterals, numOffsets) + w.codegenEncoding.generate(w.codegenFreq, 7) + numCodegens = len(w.codegenFreq) + for numCodegens > 4 && w.codegenFreq[codegenOrder[numCodegens-1]] == 0 { + numCodegens-- + } + dynamicHeader := int64(3+5+5+4+(3*numCodegens)) + + w.codegenEncoding.bitLength(w.codegenFreq) + + int64(extraBits) + + int64(w.codegenFreq[16]*2) + + int64(w.codegenFreq[17]*3) + + int64(w.codegenFreq[18]*7) + dynamicSize := dynamicHeader + + w.literalEncoding.bitLength(w.literalFreq) + + w.offsetEncoding.bitLength(w.offsetFreq) + + if dynamicSize < size { + size = dynamicSize + literalEncoding = w.literalEncoding + offsetEncoding = w.offsetEncoding + } + + // Stored bytes? + if storedSize < size { + w.writeStoredHeader(storedBytes, eof) + w.writeBytes(input[0:storedBytes]) + return + } + + // Huffman. + if literalEncoding == fixedLiteralEncoding { + w.writeFixedHeader(eof) + } else { + w.writeDynamicHeader(numLiterals, numOffsets, numCodegens, eof) + } + for _, t := range tokens { + switch t.typ() { + case literalType: + w.writeCode(literalEncoding, t.literal()) + break + case matchType: + // Write the length + length := t.length() + lengthCode := lengthCode(length) + w.writeCode(literalEncoding, lengthCode+lengthCodesStart) + extraLengthBits := int32(lengthExtraBits[lengthCode]) + if extraLengthBits > 0 { + extraLength := int32(length - lengthBase[lengthCode]) + w.writeBits(extraLength, extraLengthBits) + } + // Write the offset + offset := t.offset() + offsetCode := offsetCode(offset) + w.writeCode(offsetEncoding, offsetCode) + extraOffsetBits := int32(offsetExtraBits[offsetCode]) + if extraOffsetBits > 0 { + extraOffset := int32(offset - offsetBase[offsetCode]) + w.writeBits(extraOffset, extraOffsetBits) + } + break + default: + panic("unknown token type: " + string(t)) + } + } +} diff --git a/src/pkg/compress/flate/huffman_code.go b/src/pkg/compress/flate/huffman_code.go new file mode 100644 index 000000000..7ed603a4f --- /dev/null +++ b/src/pkg/compress/flate/huffman_code.go @@ -0,0 +1,378 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package flate + +import ( + "math" + "sort" +) + +type huffmanEncoder struct { + codeBits []uint8 + code []uint16 +} + +type literalNode struct { + literal uint16 + freq int32 +} + +type chain struct { + // The sum of the leaves in this tree + freq int32 + + // The number of literals to the left of this item at this level + leafCount int32 + + // The right child of this chain in the previous level. + up *chain +} + +type levelInfo struct { + // Our level. for better printing + level int32 + + // The most recent chain generated for this level + lastChain *chain + + // The frequency of the next character to add to this level + nextCharFreq int32 + + // The frequency of the next pair (from level below) to add to this level. + // Only valid if the "needed" value of the next lower level is 0. + nextPairFreq int32 + + // The number of chains remaining to generate for this level before moving + // up to the next level + needed int32 + + // The levelInfo for level+1 + up *levelInfo + + // The levelInfo for level-1 + down *levelInfo +} + +func maxNode() literalNode { return literalNode{math.MaxUint16, math.MaxInt32} } + +func newHuffmanEncoder(size int) *huffmanEncoder { + return &huffmanEncoder{make([]uint8, size), make([]uint16, size)} +} + +// Generates a HuffmanCode corresponding to the fixed literal table +func generateFixedLiteralEncoding() *huffmanEncoder { + h := newHuffmanEncoder(maxLit) + codeBits := h.codeBits + code := h.code + var ch uint16 + for ch = 0; ch < maxLit; ch++ { + var bits uint16 + var size uint8 + switch { + case ch < 144: + // size 8, 000110000 .. 10111111 + bits = ch + 48 + size = 8 + break + case ch < 256: + // size 9, 110010000 .. 111111111 + bits = ch + 400 - 144 + size = 9 + break + case ch < 280: + // size 7, 0000000 .. 0010111 + bits = ch - 256 + size = 7 + break + default: + // size 8, 11000000 .. 11000111 + bits = ch + 192 - 280 + size = 8 + } + codeBits[ch] = size + code[ch] = reverseBits(bits, size) + } + return h +} + +func generateFixedOffsetEncoding() *huffmanEncoder { + h := newHuffmanEncoder(30) + codeBits := h.codeBits + code := h.code + for ch := uint16(0); ch < 30; ch++ { + codeBits[ch] = 5 + code[ch] = reverseBits(ch, 5) + } + return h +} + +var fixedLiteralEncoding *huffmanEncoder = generateFixedLiteralEncoding() +var fixedOffsetEncoding *huffmanEncoder = generateFixedOffsetEncoding() + +func (h *huffmanEncoder) bitLength(freq []int32) int64 { + var total int64 + for i, f := range freq { + if f != 0 { + total += int64(f) * int64(h.codeBits[i]) + } + } + return total +} + +// Generate elements in the chain using an iterative algorithm. +func (h *huffmanEncoder) generateChains(top *levelInfo, list []literalNode) { + n := len(list) + list = list[0 : n+1] + list[n] = maxNode() + + l := top + for { + if l.nextPairFreq == math.MaxInt32 && l.nextCharFreq == math.MaxInt32 { + // We've run out of both leafs and pairs. + // End all calculations for this level. + // To m sure we never come back to this level or any lower level, + // set nextPairFreq impossibly large. + l.lastChain = nil + l.needed = 0 + l = l.up + l.nextPairFreq = math.MaxInt32 + continue + } + + prevFreq := l.lastChain.freq + if l.nextCharFreq < l.nextPairFreq { + // The next item on this row is a leaf node. + n := l.lastChain.leafCount + 1 + l.lastChain = &chain{l.nextCharFreq, n, l.lastChain.up} + l.nextCharFreq = list[n].freq + } else { + // The next item on this row is a pair from the previous row. + // nextPairFreq isn't valid until we generate two + // more values in the level below + l.lastChain = &chain{l.nextPairFreq, l.lastChain.leafCount, l.down.lastChain} + l.down.needed = 2 + } + + if l.needed--; l.needed == 0 { + // We've done everything we need to do for this level. + // Continue calculating one level up. Fill in nextPairFreq + // of that level with the sum of the two nodes we've just calculated on + // this level. + up := l.up + if up == nil { + // All done! + return + } + up.nextPairFreq = prevFreq + l.lastChain.freq + l = up + } else { + // If we stole from below, move down temporarily to replenish it. + for l.down.needed > 0 { + l = l.down + } + } + } +} + +// Return the number of literals assigned to each bit size in the Huffman encoding +// +// This method is only called when list.length >= 3 +// The cases of 0, 1, and 2 literals are handled by special case code. +// +// list An array of the literals with non-zero frequencies +// and their associated frequencies. The array is in order of increasing +// frequency, and has as its last element a special element with frequency +// MaxInt32 +// maxBits The maximum number of bits that should be used to encode any literal. +// return An integer array in which array[i] indicates the number of literals +// that should be encoded in i bits. +func (h *huffmanEncoder) bitCounts(list []literalNode, maxBits int32) []int32 { + n := int32(len(list)) + list = list[0 : n+1] + list[n] = maxNode() + + // The tree can't have greater depth than n - 1, no matter what. This + // saves a little bit of work in some small cases + maxBits = minInt32(maxBits, n-1) + + // Create information about each of the levels. + // A bogus "Level 0" whose sole purpose is so that + // level1.prev.needed==0. This makes level1.nextPairFreq + // be a legitimate value that never gets chosen. + top := &levelInfo{needed: 0} + chain2 := &chain{list[1].freq, 2, new(chain)} + for level := int32(1); level <= maxBits; level++ { + // For every level, the first two items are the first two characters. + // We initialize the levels as if we had already figured this out. + top = &levelInfo{ + level: level, + lastChain: chain2, + nextCharFreq: list[2].freq, + nextPairFreq: list[0].freq + list[1].freq, + down: top, + } + top.down.up = top + if level == 1 { + top.nextPairFreq = math.MaxInt32 + } + } + + // We need a total of 2*n - 2 items at top level and have already generated 2. + top.needed = 2*n - 4 + + l := top + for { + if l.nextPairFreq == math.MaxInt32 && l.nextCharFreq == math.MaxInt32 { + // We've run out of both leafs and pairs. + // End all calculations for this level. + // To m sure we never come back to this level or any lower level, + // set nextPairFreq impossibly large. + l.lastChain = nil + l.needed = 0 + l = l.up + l.nextPairFreq = math.MaxInt32 + continue + } + + prevFreq := l.lastChain.freq + if l.nextCharFreq < l.nextPairFreq { + // The next item on this row is a leaf node. + n := l.lastChain.leafCount + 1 + l.lastChain = &chain{l.nextCharFreq, n, l.lastChain.up} + l.nextCharFreq = list[n].freq + } else { + // The next item on this row is a pair from the previous row. + // nextPairFreq isn't valid until we generate two + // more values in the level below + l.lastChain = &chain{l.nextPairFreq, l.lastChain.leafCount, l.down.lastChain} + l.down.needed = 2 + } + + if l.needed--; l.needed == 0 { + // We've done everything we need to do for this level. + // Continue calculating one level up. Fill in nextPairFreq + // of that level with the sum of the two nodes we've just calculated on + // this level. + up := l.up + if up == nil { + // All done! + break + } + up.nextPairFreq = prevFreq + l.lastChain.freq + l = up + } else { + // If we stole from below, move down temporarily to replenish it. + for l.down.needed > 0 { + l = l.down + } + } + } + + // Somethings is wrong if at the end, the top level is null or hasn't used + // all of the leaves. + if top.lastChain.leafCount != n { + panic("top.lastChain.leafCount != n") + } + + bitCount := make([]int32, maxBits+1) + bits := 1 + for chain := top.lastChain; chain.up != nil; chain = chain.up { + // chain.leafCount gives the number of literals requiring at least "bits" + // bits to encode. + bitCount[bits] = chain.leafCount - chain.up.leafCount + bits++ + } + return bitCount +} + +// Look at the leaves and assign them a bit count and an encoding as specified +// in RFC 1951 3.2.2 +func (h *huffmanEncoder) assignEncodingAndSize(bitCount []int32, list []literalNode) { + code := uint16(0) + for n, bits := range bitCount { + code <<= 1 + if n == 0 || bits == 0 { + continue + } + // The literals list[len(list)-bits] .. list[len(list)-bits] + // are encoded using "bits" bits, and get the values + // code, code + 1, .... The code values are + // assigned in literal order (not frequency order). + chunk := list[len(list)-int(bits):] + sortByLiteral(chunk) + for _, node := range chunk { + h.codeBits[node.literal] = uint8(n) + h.code[node.literal] = reverseBits(code, uint8(n)) + code++ + } + list = list[0 : len(list)-int(bits)] + } +} + +// Update this Huffman Code object to be the minimum code for the specified frequency count. +// +// freq An array of frequencies, in which frequency[i] gives the frequency of literal i. +// maxBits The maximum number of bits to use for any literal. +func (h *huffmanEncoder) generate(freq []int32, maxBits int32) { + list := make([]literalNode, len(freq)+1) + // Number of non-zero literals + count := 0 + // Set list to be the set of all non-zero literals and their frequencies + for i, f := range freq { + if f != 0 { + list[count] = literalNode{uint16(i), f} + count++ + } else { + h.codeBits[i] = 0 + } + } + // If freq[] is shorter than codeBits[], fill rest of codeBits[] with zeros + h.codeBits = h.codeBits[0:len(freq)] + list = list[0:count] + if count <= 2 { + // Handle the small cases here, because they are awkward for the general case code. With + // two or fewer literals, everything has bit length 1. + for i, node := range list { + // "list" is in order of increasing literal value. + h.codeBits[node.literal] = 1 + h.code[node.literal] = uint16(i) + } + return + } + sortByFreq(list) + + // Get the number of literals for each bit count + bitCount := h.bitCounts(list, maxBits) + // And do the assignment + h.assignEncodingAndSize(bitCount, list) +} + +type literalNodeSorter struct { + a []literalNode + less func(i, j int) bool +} + +func (s literalNodeSorter) Len() int { return len(s.a) } + +func (s literalNodeSorter) Less(i, j int) bool { + return s.less(i, j) +} + +func (s literalNodeSorter) Swap(i, j int) { s.a[i], s.a[j] = s.a[j], s.a[i] } + +func sortByFreq(a []literalNode) { + s := &literalNodeSorter{a, func(i, j int) bool { + if a[i].freq == a[j].freq { + return a[i].literal < a[j].literal + } + return a[i].freq < a[j].freq + }} + sort.Sort(s) +} + +func sortByLiteral(a []literalNode) { + s := &literalNodeSorter{a, func(i, j int) bool { return a[i].literal < a[j].literal }} + sort.Sort(s) +} diff --git a/src/pkg/compress/flate/inflate.go b/src/pkg/compress/flate/inflate.go new file mode 100644 index 000000000..3845f1204 --- /dev/null +++ b/src/pkg/compress/flate/inflate.go @@ -0,0 +1,708 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Package flate implements the DEFLATE compressed data format, described in +// RFC 1951. The gzip and zlib packages implement access to DEFLATE-based file +// formats. +package flate + +import ( + "bufio" + "io" + "os" + "strconv" +) + +const ( + maxCodeLen = 16 // max length of Huffman code + maxHist = 32768 // max history required + maxLit = 286 + maxDist = 32 + numCodes = 19 // number of codes in Huffman meta-code +) + +// A CorruptInputError reports the presence of corrupt input at a given offset. +type CorruptInputError int64 + +func (e CorruptInputError) String() string { + return "flate: corrupt input before offset " + strconv.Itoa64(int64(e)) +} + +// An InternalError reports an error in the flate code itself. +type InternalError string + +func (e InternalError) String() string { return "flate: internal error: " + string(e) } + +// A ReadError reports an error encountered while reading input. +type ReadError struct { + Offset int64 // byte offset where error occurred + Error os.Error // error returned by underlying Read +} + +func (e *ReadError) String() string { + return "flate: read error at offset " + strconv.Itoa64(e.Offset) + ": " + e.Error.String() +} + +// A WriteError reports an error encountered while writing output. +type WriteError struct { + Offset int64 // byte offset where error occurred + Error os.Error // error returned by underlying Write +} + +func (e *WriteError) String() string { + return "flate: write error at offset " + strconv.Itoa64(e.Offset) + ": " + e.Error.String() +} + +// Huffman decoder is based on +// J. Brian Connell, ``A Huffman-Shannon-Fano Code,'' +// Proceedings of the IEEE, 61(7) (July 1973), pp 1046-1047. +type huffmanDecoder struct { + // min, max code length + min, max int + + // limit[i] = largest code word of length i + // Given code v of length n, + // need more bits if v > limit[n]. + limit [maxCodeLen + 1]int + + // base[i] = smallest code word of length i - seq number + base [maxCodeLen + 1]int + + // codes[seq number] = output code. + // Given code v of length n, value is + // codes[v - base[n]]. + codes []int +} + +// Initialize Huffman decoding tables from array of code lengths. +func (h *huffmanDecoder) init(bits []int) bool { + // Count number of codes of each length, + // compute min and max length. + var count [maxCodeLen + 1]int + var min, max int + for _, n := range bits { + if n == 0 { + continue + } + if min == 0 || n < min { + min = n + } + if n > max { + max = n + } + count[n]++ + } + if max == 0 { + return false + } + + h.min = min + h.max = max + + // For each code range, compute + // nextcode (first code of that length), + // limit (last code of that length), and + // base (offset from first code to sequence number). + code := 0 + seq := 0 + var nextcode [maxCodeLen]int + for i := min; i <= max; i++ { + n := count[i] + nextcode[i] = code + h.base[i] = code - seq + code += n + seq += n + h.limit[i] = code - 1 + code <<= 1 + } + + // Make array mapping sequence numbers to codes. + if len(h.codes) < len(bits) { + h.codes = make([]int, len(bits)) + } + for i, n := range bits { + if n == 0 { + continue + } + code := nextcode[n] + nextcode[n]++ + seq := code - h.base[n] + h.codes[seq] = i + } + return true +} + +// Hard-coded Huffman tables for DEFLATE algorithm. +// See RFC 1951, section 3.2.6. +var fixedHuffmanDecoder = huffmanDecoder{ + 7, 9, + [maxCodeLen + 1]int{7: 23, 199, 511}, + [maxCodeLen + 1]int{7: 0, 24, 224}, + []int{ + // length 7: 256-279 + 256, 257, 258, 259, 260, 261, 262, + 263, 264, 265, 266, 267, 268, 269, + 270, 271, 272, 273, 274, 275, 276, + 277, 278, 279, + + // length 8: 0-143 + 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, + 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, + 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, + 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, + 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, + 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, + 62, 63, 64, 65, 66, 67, 68, 69, 70, 71, + 72, 73, 74, 75, 76, 77, 78, 79, 80, 81, + 82, 83, 84, 85, 86, 87, 88, 89, 90, 91, + 92, 93, 94, 95, 96, 97, 98, 99, 100, + 101, 102, 103, 104, 105, 106, 107, 108, + 109, 110, 111, 112, 113, 114, 115, 116, + 117, 118, 119, 120, 121, 122, 123, 124, + 125, 126, 127, 128, 129, 130, 131, 132, + 133, 134, 135, 136, 137, 138, 139, 140, + 141, 142, 143, + + // length 8: 280-287 + 280, 281, 282, 283, 284, 285, 286, 287, + + // length 9: 144-255 + 144, 145, 146, 147, 148, 149, 150, 151, + 152, 153, 154, 155, 156, 157, 158, 159, + 160, 161, 162, 163, 164, 165, 166, 167, + 168, 169, 170, 171, 172, 173, 174, 175, + 176, 177, 178, 179, 180, 181, 182, 183, + 184, 185, 186, 187, 188, 189, 190, 191, + 192, 193, 194, 195, 196, 197, 198, 199, + 200, 201, 202, 203, 204, 205, 206, 207, + 208, 209, 210, 211, 212, 213, 214, 215, + 216, 217, 218, 219, 220, 221, 222, 223, + 224, 225, 226, 227, 228, 229, 230, 231, + 232, 233, 234, 235, 236, 237, 238, 239, + 240, 241, 242, 243, 244, 245, 246, 247, + 248, 249, 250, 251, 252, 253, 254, 255, + }, +} + +// The actual read interface needed by NewReader. +// If the passed in io.Reader does not also have ReadByte, +// the NewReader will introduce its own buffering. +type Reader interface { + io.Reader + ReadByte() (c byte, err os.Error) +} + +// Decompress state. +type decompressor struct { + // Input source. + r Reader + roffset int64 + woffset int64 + + // Input bits, in top of b. + b uint32 + nb uint + + // Huffman decoders for literal/length, distance. + h1, h2 huffmanDecoder + + // Length arrays used to define Huffman codes. + bits [maxLit + maxDist]int + codebits [numCodes]int + + // Output history, buffer. + hist [maxHist]byte + hp int // current output position in buffer + hw int // have written hist[0:hw] already + hfull bool // buffer has filled at least once + + // Temporary buffer (avoids repeated allocation). + buf [4]byte + + // Next step in the decompression, + // and decompression state. + step func(*decompressor) + final bool + err os.Error + toRead []byte + hl, hd *huffmanDecoder + copyLen int + copyDist int +} + +func (f *decompressor) nextBlock() { + if f.final { + if f.hw != f.hp { + f.flush((*decompressor).nextBlock) + return + } + f.err = os.EOF + return + } + for f.nb < 1+2 { + if f.err = f.moreBits(); f.err != nil { + return + } + } + f.final = f.b&1 == 1 + f.b >>= 1 + typ := f.b & 3 + f.b >>= 2 + f.nb -= 1 + 2 + switch typ { + case 0: + f.dataBlock() + case 1: + // compressed, fixed Huffman tables + f.hl = &fixedHuffmanDecoder + f.hd = nil + f.huffmanBlock() + case 2: + // compressed, dynamic Huffman tables + if f.err = f.readHuffman(); f.err != nil { + break + } + f.hl = &f.h1 + f.hd = &f.h2 + f.huffmanBlock() + default: + // 3 is reserved. + f.err = CorruptInputError(f.roffset) + } +} + +func (f *decompressor) Read(b []byte) (int, os.Error) { + for { + if len(f.toRead) > 0 { + n := copy(b, f.toRead) + f.toRead = f.toRead[n:] + return n, nil + } + if f.err != nil { + return 0, f.err + } + f.step(f) + } + panic("unreachable") +} + +func (f *decompressor) Close() os.Error { + if f.err == os.EOF { + return nil + } + return f.err +} + +// RFC 1951 section 3.2.7. +// Compression with dynamic Huffman codes + +var codeOrder = [...]int{16, 17, 18, 0, 8, 7, 9, 6, 10, 5, 11, 4, 12, 3, 13, 2, 14, 1, 15} + +func (f *decompressor) readHuffman() os.Error { + // HLIT[5], HDIST[5], HCLEN[4]. + for f.nb < 5+5+4 { + if err := f.moreBits(); err != nil { + return err + } + } + nlit := int(f.b&0x1F) + 257 + f.b >>= 5 + ndist := int(f.b&0x1F) + 1 + f.b >>= 5 + nclen := int(f.b&0xF) + 4 + f.b >>= 4 + f.nb -= 5 + 5 + 4 + + // (HCLEN+4)*3 bits: code lengths in the magic codeOrder order. + for i := 0; i < nclen; i++ { + for f.nb < 3 { + if err := f.moreBits(); err != nil { + return err + } + } + f.codebits[codeOrder[i]] = int(f.b & 0x7) + f.b >>= 3 + f.nb -= 3 + } + for i := nclen; i < len(codeOrder); i++ { + f.codebits[codeOrder[i]] = 0 + } + if !f.h1.init(f.codebits[0:]) { + return CorruptInputError(f.roffset) + } + + // HLIT + 257 code lengths, HDIST + 1 code lengths, + // using the code length Huffman code. + for i, n := 0, nlit+ndist; i < n; { + x, err := f.huffSym(&f.h1) + if err != nil { + return err + } + if x < 16 { + // Actual length. + f.bits[i] = x + i++ + continue + } + // Repeat previous length or zero. + var rep int + var nb uint + var b int + switch x { + default: + return InternalError("unexpected length code") + case 16: + rep = 3 + nb = 2 + if i == 0 { + return CorruptInputError(f.roffset) + } + b = f.bits[i-1] + case 17: + rep = 3 + nb = 3 + b = 0 + case 18: + rep = 11 + nb = 7 + b = 0 + } + for f.nb < nb { + if err := f.moreBits(); err != nil { + return err + } + } + rep += int(f.b & uint32(1<>= nb + f.nb -= nb + if i+rep > n { + return CorruptInputError(f.roffset) + } + for j := 0; j < rep; j++ { + f.bits[i] = b + i++ + } + } + + if !f.h1.init(f.bits[0:nlit]) || !f.h2.init(f.bits[nlit:nlit+ndist]) { + return CorruptInputError(f.roffset) + } + + return nil +} + +// Decode a single Huffman block from f. +// hl and hd are the Huffman states for the lit/length values +// and the distance values, respectively. If hd == nil, using the +// fixed distance encoding associated with fixed Huffman blocks. +func (f *decompressor) huffmanBlock() { + for { + v, err := f.huffSym(f.hl) + if err != nil { + f.err = err + return + } + var n uint // number of bits extra + var length int + switch { + case v < 256: + f.hist[f.hp] = byte(v) + f.hp++ + if f.hp == len(f.hist) { + // After the flush, continue this loop. + f.flush((*decompressor).huffmanBlock) + return + } + continue + case v == 256: + // Done with huffman block; read next block. + f.step = (*decompressor).nextBlock + return + // otherwise, reference to older data + case v < 265: + length = v - (257 - 3) + n = 0 + case v < 269: + length = v*2 - (265*2 - 11) + n = 1 + case v < 273: + length = v*4 - (269*4 - 19) + n = 2 + case v < 277: + length = v*8 - (273*8 - 35) + n = 3 + case v < 281: + length = v*16 - (277*16 - 67) + n = 4 + case v < 285: + length = v*32 - (281*32 - 131) + n = 5 + default: + length = 258 + n = 0 + } + if n > 0 { + for f.nb < n { + if err = f.moreBits(); err != nil { + f.err = err + return + } + } + length += int(f.b & uint32(1<>= n + f.nb -= n + } + + var dist int + if f.hd == nil { + for f.nb < 5 { + if err = f.moreBits(); err != nil { + f.err = err + return + } + } + dist = int(reverseByte[(f.b&0x1F)<<3]) + f.b >>= 5 + f.nb -= 5 + } else { + if dist, err = f.huffSym(f.hd); err != nil { + f.err = err + return + } + } + + switch { + case dist < 4: + dist++ + case dist >= 30: + f.err = CorruptInputError(f.roffset) + return + default: + nb := uint(dist-2) >> 1 + // have 1 bit in bottom of dist, need nb more. + extra := (dist & 1) << nb + for f.nb < nb { + if err = f.moreBits(); err != nil { + f.err = err + return + } + } + extra |= int(f.b & uint32(1<>= nb + f.nb -= nb + dist = 1<<(nb+1) + 1 + extra + } + + // Copy history[-dist:-dist+length] into output. + if dist > len(f.hist) { + f.err = InternalError("bad history distance") + return + } + + // No check on length; encoding can be prescient. + if !f.hfull && dist > f.hp { + f.err = CorruptInputError(f.roffset) + return + } + + p := f.hp - dist + if p < 0 { + p += len(f.hist) + } + for i := 0; i < length; i++ { + f.hist[f.hp] = f.hist[p] + f.hp++ + p++ + if f.hp == len(f.hist) { + // After flush continue copying out of history. + f.copyLen = length - (i + 1) + f.copyDist = dist + f.flush((*decompressor).copyHuff) + return + } + if p == len(f.hist) { + p = 0 + } + } + } + panic("unreached") +} + +func (f *decompressor) copyHuff() { + length := f.copyLen + dist := f.copyDist + p := f.hp - dist + if p < 0 { + p += len(f.hist) + } + for i := 0; i < length; i++ { + f.hist[f.hp] = f.hist[p] + f.hp++ + p++ + if f.hp == len(f.hist) { + f.copyLen = length - (i + 1) + f.flush((*decompressor).copyHuff) + return + } + if p == len(f.hist) { + p = 0 + } + } + + // Continue processing Huffman block. + f.huffmanBlock() +} + +// Copy a single uncompressed data block from input to output. +func (f *decompressor) dataBlock() { + // Uncompressed. + // Discard current half-byte. + f.nb = 0 + f.b = 0 + + // Length then ones-complement of length. + nr, err := io.ReadFull(f.r, f.buf[0:4]) + f.roffset += int64(nr) + if err != nil { + f.err = &ReadError{f.roffset, err} + return + } + n := int(f.buf[0]) | int(f.buf[1])<<8 + nn := int(f.buf[2]) | int(f.buf[3])<<8 + if uint16(nn) != uint16(^n) { + f.err = CorruptInputError(f.roffset) + return + } + + if n == 0 { + // 0-length block means sync + f.flush((*decompressor).nextBlock) + return + } + + f.copyLen = n + f.copyData() +} + +func (f *decompressor) copyData() { + // Read f.dataLen bytes into history, + // pausing for reads as history fills. + n := f.copyLen + for n > 0 { + m := len(f.hist) - f.hp + if m > n { + m = n + } + m, err := io.ReadFull(f.r, f.hist[f.hp:f.hp+m]) + f.roffset += int64(m) + if err != nil { + f.err = &ReadError{f.roffset, err} + return + } + n -= m + f.hp += m + if f.hp == len(f.hist) { + f.copyLen = n + f.flush((*decompressor).copyData) + return + } + } + f.step = (*decompressor).nextBlock +} + +func (f *decompressor) setDict(dict []byte) { + if len(dict) > len(f.hist) { + // Will only remember the tail. + dict = dict[len(dict)-len(f.hist):] + } + + f.hp = copy(f.hist[:], dict) + if f.hp == len(f.hist) { + f.hp = 0 + f.hfull = true + } + f.hw = f.hp +} + +func (f *decompressor) moreBits() os.Error { + c, err := f.r.ReadByte() + if err != nil { + if err == os.EOF { + err = io.ErrUnexpectedEOF + } + return err + } + f.roffset++ + f.b |= uint32(c) << f.nb + f.nb += 8 + return nil +} + +// Read the next Huffman-encoded symbol from f according to h. +func (f *decompressor) huffSym(h *huffmanDecoder) (int, os.Error) { + for n := uint(h.min); n <= uint(h.max); n++ { + lim := h.limit[n] + if lim == -1 { + continue + } + for f.nb < n { + if err := f.moreBits(); err != nil { + return 0, err + } + } + v := int(f.b & uint32(1<>8]) | int(reverseByte[v&0xFF])<<8 // reverse bits + if v <= lim { + f.b >>= n + f.nb -= n + return h.codes[v-h.base[n]], nil + } + } + return 0, CorruptInputError(f.roffset) +} + +// Flush any buffered output to the underlying writer. +func (f *decompressor) flush(step func(*decompressor)) { + f.toRead = f.hist[f.hw:f.hp] + f.woffset += int64(f.hp - f.hw) + f.hw = f.hp + if f.hp == len(f.hist) { + f.hp = 0 + f.hw = 0 + f.hfull = true + } + f.step = step +} + +func makeReader(r io.Reader) Reader { + if rr, ok := r.(Reader); ok { + return rr + } + return bufio.NewReader(r) +} + +// NewReader returns a new ReadCloser that can be used +// to read the uncompressed version of r. It is the caller's +// responsibility to call Close on the ReadCloser when +// finished reading. +func NewReader(r io.Reader) io.ReadCloser { + var f decompressor + f.r = makeReader(r) + f.step = (*decompressor).nextBlock + return &f +} + +// NewReaderDict is like NewReader but initializes the reader +// with a preset dictionary. The returned Reader behaves as if +// the uncompressed data stream started with the given dictionary, +// which has already been read. NewReaderDict is typically used +// to read data compressed by NewWriterDict. +func NewReaderDict(r io.Reader, dict []byte) io.ReadCloser { + var f decompressor + f.setDict(dict) + f.r = makeReader(r) + f.step = (*decompressor).nextBlock + return &f +} diff --git a/src/pkg/compress/flate/reverse_bits.go b/src/pkg/compress/flate/reverse_bits.go new file mode 100644 index 000000000..c1a02720d --- /dev/null +++ b/src/pkg/compress/flate/reverse_bits.go @@ -0,0 +1,48 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package flate + +var reverseByte = [256]byte{ + 0x00, 0x80, 0x40, 0xc0, 0x20, 0xa0, 0x60, 0xe0, + 0x10, 0x90, 0x50, 0xd0, 0x30, 0xb0, 0x70, 0xf0, + 0x08, 0x88, 0x48, 0xc8, 0x28, 0xa8, 0x68, 0xe8, + 0x18, 0x98, 0x58, 0xd8, 0x38, 0xb8, 0x78, 0xf8, + 0x04, 0x84, 0x44, 0xc4, 0x24, 0xa4, 0x64, 0xe4, + 0x14, 0x94, 0x54, 0xd4, 0x34, 0xb4, 0x74, 0xf4, + 0x0c, 0x8c, 0x4c, 0xcc, 0x2c, 0xac, 0x6c, 0xec, + 0x1c, 0x9c, 0x5c, 0xdc, 0x3c, 0xbc, 0x7c, 0xfc, + 0x02, 0x82, 0x42, 0xc2, 0x22, 0xa2, 0x62, 0xe2, + 0x12, 0x92, 0x52, 0xd2, 0x32, 0xb2, 0x72, 0xf2, + 0x0a, 0x8a, 0x4a, 0xca, 0x2a, 0xaa, 0x6a, 0xea, + 0x1a, 0x9a, 0x5a, 0xda, 0x3a, 0xba, 0x7a, 0xfa, + 0x06, 0x86, 0x46, 0xc6, 0x26, 0xa6, 0x66, 0xe6, + 0x16, 0x96, 0x56, 0xd6, 0x36, 0xb6, 0x76, 0xf6, + 0x0e, 0x8e, 0x4e, 0xce, 0x2e, 0xae, 0x6e, 0xee, + 0x1e, 0x9e, 0x5e, 0xde, 0x3e, 0xbe, 0x7e, 0xfe, + 0x01, 0x81, 0x41, 0xc1, 0x21, 0xa1, 0x61, 0xe1, + 0x11, 0x91, 0x51, 0xd1, 0x31, 0xb1, 0x71, 0xf1, + 0x09, 0x89, 0x49, 0xc9, 0x29, 0xa9, 0x69, 0xe9, + 0x19, 0x99, 0x59, 0xd9, 0x39, 0xb9, 0x79, 0xf9, + 0x05, 0x85, 0x45, 0xc5, 0x25, 0xa5, 0x65, 0xe5, + 0x15, 0x95, 0x55, 0xd5, 0x35, 0xb5, 0x75, 0xf5, + 0x0d, 0x8d, 0x4d, 0xcd, 0x2d, 0xad, 0x6d, 0xed, + 0x1d, 0x9d, 0x5d, 0xdd, 0x3d, 0xbd, 0x7d, 0xfd, + 0x03, 0x83, 0x43, 0xc3, 0x23, 0xa3, 0x63, 0xe3, + 0x13, 0x93, 0x53, 0xd3, 0x33, 0xb3, 0x73, 0xf3, + 0x0b, 0x8b, 0x4b, 0xcb, 0x2b, 0xab, 0x6b, 0xeb, + 0x1b, 0x9b, 0x5b, 0xdb, 0x3b, 0xbb, 0x7b, 0xfb, + 0x07, 0x87, 0x47, 0xc7, 0x27, 0xa7, 0x67, 0xe7, + 0x17, 0x97, 0x57, 0xd7, 0x37, 0xb7, 0x77, 0xf7, + 0x0f, 0x8f, 0x4f, 0xcf, 0x2f, 0xaf, 0x6f, 0xef, + 0x1f, 0x9f, 0x5f, 0xdf, 0x3f, 0xbf, 0x7f, 0xff, +} + +func reverseUint16(v uint16) uint16 { + return uint16(reverseByte[v>>8]) | uint16(reverseByte[v&0xFF])<<8 +} + +func reverseBits(number uint16, bitLength byte) uint16 { + return reverseUint16(number << uint8(16-bitLength)) +} diff --git a/src/pkg/compress/flate/token.go b/src/pkg/compress/flate/token.go new file mode 100644 index 000000000..38aea5fa6 --- /dev/null +++ b/src/pkg/compress/flate/token.go @@ -0,0 +1,103 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package flate + +const ( + // 2 bits: type 0 = literal 1=EOF 2=Match 3=Unused + // 8 bits: xlength = length - MIN_MATCH_LENGTH + // 22 bits xoffset = offset - MIN_OFFSET_SIZE, or literal + lengthShift = 22 + offsetMask = 1< pair into a match token. +func matchToken(xlength uint32, xoffset uint32) token { + return token(matchType + xlength<> lengthShift) } + +func lengthCode(len uint32) uint32 { return lengthCodes[len] } + +// Returns the offset code corresponding to a specific offset +func offsetCode(off uint32) uint32 { + const n = uint32(len(offsetCodes)) + switch { + case off < n: + return offsetCodes[off] + case off>>7 < n: + return offsetCodes[off>>7] + 14 + default: + return offsetCodes[off>>14] + 28 + } + panic("unreachable") +} diff --git a/src/pkg/compress/flate/util.go b/src/pkg/compress/flate/util.go new file mode 100644 index 000000000..aca5c78b2 --- /dev/null +++ b/src/pkg/compress/flate/util.go @@ -0,0 +1,72 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package flate + +func min(left int, right int) int { + if left < right { + return left + } + return right +} + +func minInt32(left int32, right int32) int32 { + if left < right { + return left + } + return right +} + +func max(left int, right int) int { + if left > right { + return left + } + return right +} + +func fillInts(a []int, value int) { + for i := range a { + a[i] = value + } +} + +func fillInt32s(a []int32, value int32) { + for i := range a { + a[i] = value + } +} + +func fillBytes(a []byte, value byte) { + for i := range a { + a[i] = value + } +} + +func fillInt8s(a []int8, value int8) { + for i := range a { + a[i] = value + } +} + +func fillUint8s(a []uint8, value uint8) { + for i := range a { + a[i] = value + } +} + +func copyInt8s(dst []int8, src []int8) int { + cnt := min(len(dst), len(src)) + for i := 0; i < cnt; i++ { + dst[i] = src[i] + } + return cnt +} + +func copyUint8s(dst []uint8, src []uint8) int { + cnt := min(len(dst), len(src)) + for i := 0; i < cnt; i++ { + dst[i] = src[i] + } + return cnt +} diff --git a/src/pkg/compress/gzip/Makefile b/src/pkg/compress/gzip/Makefile new file mode 100644 index 000000000..b671fc72c --- /dev/null +++ b/src/pkg/compress/gzip/Makefile @@ -0,0 +1,12 @@ +# Copyright 2009 The Go Authors. All rights reserved. +# Use of this source code is governed by a BSD-style +# license that can be found in the LICENSE file. + +include ../../../Make.inc + +TARG=compress/gzip +GOFILES=\ + gunzip.go\ + gzip.go\ + +include ../../../Make.pkg diff --git a/src/pkg/compress/gzip/gunzip.go b/src/pkg/compress/gzip/gunzip.go new file mode 100644 index 000000000..6ac9293d7 --- /dev/null +++ b/src/pkg/compress/gzip/gunzip.go @@ -0,0 +1,230 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Package gzip implements reading and writing of gzip format compressed files, +// as specified in RFC 1952. +package gzip + +import ( + "bufio" + "compress/flate" + "hash" + "hash/crc32" + "io" + "os" +) + +// BUG(nigeltao): Comments and Names don't properly map UTF-8 character codes outside of +// the 0x00-0x7f range to ISO 8859-1 (Latin-1). + +const ( + gzipID1 = 0x1f + gzipID2 = 0x8b + gzipDeflate = 8 + flagText = 1 << 0 + flagHdrCrc = 1 << 1 + flagExtra = 1 << 2 + flagName = 1 << 3 + flagComment = 1 << 4 +) + +func makeReader(r io.Reader) flate.Reader { + if rr, ok := r.(flate.Reader); ok { + return rr + } + return bufio.NewReader(r) +} + +var HeaderError = os.NewError("invalid gzip header") +var ChecksumError = os.NewError("gzip checksum error") + +// The gzip file stores a header giving metadata about the compressed file. +// That header is exposed as the fields of the Compressor and Decompressor structs. +type Header struct { + Comment string // comment + Extra []byte // "extra data" + Mtime uint32 // modification time (seconds since January 1, 1970) + Name string // file name + OS byte // operating system type +} + +// An Decompressor is an io.Reader that can be read to retrieve +// uncompressed data from a gzip-format compressed file. +// +// In general, a gzip file can be a concatenation of gzip files, +// each with its own header. Reads from the Decompressor +// return the concatenation of the uncompressed data of each. +// Only the first header is recorded in the Decompressor fields. +// +// Gzip files store a length and checksum of the uncompressed data. +// The Decompressor will return a ChecksumError when Read +// reaches the end of the uncompressed data if it does not +// have the expected length or checksum. Clients should treat data +// returned by Read as tentative until they receive the successful +// (zero length, nil error) Read marking the end of the data. +type Decompressor struct { + Header + r flate.Reader + decompressor io.ReadCloser + digest hash.Hash32 + size uint32 + flg byte + buf [512]byte + err os.Error +} + +// NewReader creates a new Decompressor reading the given reader. +// The implementation buffers input and may read more data than necessary from r. +// It is the caller's responsibility to call Close on the Decompressor when done. +func NewReader(r io.Reader) (*Decompressor, os.Error) { + z := new(Decompressor) + z.r = makeReader(r) + z.digest = crc32.NewIEEE() + if err := z.readHeader(true); err != nil { + z.err = err + return nil, err + } + return z, nil +} + +// GZIP (RFC 1952) is little-endian, unlike ZLIB (RFC 1950). +func get4(p []byte) uint32 { + return uint32(p[0]) | uint32(p[1])<<8 | uint32(p[2])<<16 | uint32(p[3])<<24 +} + +func (z *Decompressor) readString() (string, os.Error) { + var err os.Error + for i := 0; ; i++ { + if i >= len(z.buf) { + return "", HeaderError + } + z.buf[i], err = z.r.ReadByte() + if err != nil { + return "", err + } + if z.buf[i] == 0 { + // GZIP (RFC 1952) specifies that strings are NUL-terminated ISO 8859-1 (Latin-1). + // TODO(nigeltao): Convert from ISO 8859-1 (Latin-1) to UTF-8. + return string(z.buf[0:i]), nil + } + } + panic("not reached") +} + +func (z *Decompressor) read2() (uint32, os.Error) { + _, err := io.ReadFull(z.r, z.buf[0:2]) + if err != nil { + return 0, err + } + return uint32(z.buf[0]) | uint32(z.buf[1])<<8, nil +} + +func (z *Decompressor) readHeader(save bool) os.Error { + _, err := io.ReadFull(z.r, z.buf[0:10]) + if err != nil { + return err + } + if z.buf[0] != gzipID1 || z.buf[1] != gzipID2 || z.buf[2] != gzipDeflate { + return HeaderError + } + z.flg = z.buf[3] + if save { + z.Mtime = get4(z.buf[4:8]) + // z.buf[8] is xfl, ignored + z.OS = z.buf[9] + } + z.digest.Reset() + z.digest.Write(z.buf[0:10]) + + if z.flg&flagExtra != 0 { + n, err := z.read2() + if err != nil { + return err + } + data := make([]byte, n) + if _, err = io.ReadFull(z.r, data); err != nil { + return err + } + if save { + z.Extra = data + } + } + + var s string + if z.flg&flagName != 0 { + if s, err = z.readString(); err != nil { + return err + } + if save { + z.Name = s + } + } + + if z.flg&flagComment != 0 { + if s, err = z.readString(); err != nil { + return err + } + if save { + z.Comment = s + } + } + + if z.flg&flagHdrCrc != 0 { + n, err := z.read2() + if err != nil { + return err + } + sum := z.digest.Sum32() & 0xFFFF + if n != sum { + return HeaderError + } + } + + z.digest.Reset() + z.decompressor = flate.NewReader(z.r) + return nil +} + +func (z *Decompressor) Read(p []byte) (n int, err os.Error) { + if z.err != nil { + return 0, z.err + } + if len(p) == 0 { + return 0, nil + } + + n, err = z.decompressor.Read(p) + z.digest.Write(p[0:n]) + z.size += uint32(n) + if n != 0 || err != os.EOF { + z.err = err + return + } + + // Finished file; check checksum + size. + if _, err := io.ReadFull(z.r, z.buf[0:8]); err != nil { + z.err = err + return 0, err + } + crc32, isize := get4(z.buf[0:4]), get4(z.buf[4:8]) + sum := z.digest.Sum32() + if sum != crc32 || isize != z.size { + z.err = ChecksumError + return 0, z.err + } + + // File is ok; is there another? + if err = z.readHeader(false); err != nil { + z.err = err + return + } + + // Yes. Reset and read from it. + z.digest.Reset() + z.size = 0 + return z.Read(p) +} + +// Calling Close does not close the wrapped io.Reader originally passed to NewReader. +func (z *Decompressor) Close() os.Error { return z.decompressor.Close() } diff --git a/src/pkg/compress/gzip/gunzip_test.go b/src/pkg/compress/gzip/gunzip_test.go new file mode 100644 index 000000000..1c08c7374 --- /dev/null +++ b/src/pkg/compress/gzip/gunzip_test.go @@ -0,0 +1,305 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package gzip + +import ( + "bytes" + "io" + "os" + "testing" +) + +type gunzipTest struct { + name string + desc string + raw string + gzip []byte + err os.Error +} + +var gunzipTests = []gunzipTest{ + { // has 1 empty fixed-huffman block + "empty.txt", + "empty.txt", + "", + []byte{ + 0x1f, 0x8b, 0x08, 0x08, 0xf7, 0x5e, 0x14, 0x4a, + 0x00, 0x03, 0x65, 0x6d, 0x70, 0x74, 0x79, 0x2e, + 0x74, 0x78, 0x74, 0x00, 0x03, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + }, + nil, + }, + { // has 1 non-empty fixed huffman block + "hello.txt", + "hello.txt", + "hello world\n", + []byte{ + 0x1f, 0x8b, 0x08, 0x08, 0xc8, 0x58, 0x13, 0x4a, + 0x00, 0x03, 0x68, 0x65, 0x6c, 0x6c, 0x6f, 0x2e, + 0x74, 0x78, 0x74, 0x00, 0xcb, 0x48, 0xcd, 0xc9, + 0xc9, 0x57, 0x28, 0xcf, 0x2f, 0xca, 0x49, 0xe1, + 0x02, 0x00, 0x2d, 0x3b, 0x08, 0xaf, 0x0c, 0x00, + 0x00, 0x00, + }, + nil, + }, + { // concatenation + "hello.txt", + "hello.txt x2", + "hello world\n" + + "hello world\n", + []byte{ + 0x1f, 0x8b, 0x08, 0x08, 0xc8, 0x58, 0x13, 0x4a, + 0x00, 0x03, 0x68, 0x65, 0x6c, 0x6c, 0x6f, 0x2e, + 0x74, 0x78, 0x74, 0x00, 0xcb, 0x48, 0xcd, 0xc9, + 0xc9, 0x57, 0x28, 0xcf, 0x2f, 0xca, 0x49, 0xe1, + 0x02, 0x00, 0x2d, 0x3b, 0x08, 0xaf, 0x0c, 0x00, + 0x00, 0x00, + 0x1f, 0x8b, 0x08, 0x08, 0xc8, 0x58, 0x13, 0x4a, + 0x00, 0x03, 0x68, 0x65, 0x6c, 0x6c, 0x6f, 0x2e, + 0x74, 0x78, 0x74, 0x00, 0xcb, 0x48, 0xcd, 0xc9, + 0xc9, 0x57, 0x28, 0xcf, 0x2f, 0xca, 0x49, 0xe1, + 0x02, 0x00, 0x2d, 0x3b, 0x08, 0xaf, 0x0c, 0x00, + 0x00, 0x00, + }, + nil, + }, + { // has a fixed huffman block with some length-distance pairs + "shesells.txt", + "shesells.txt", + "she sells seashells by the seashore\n", + []byte{ + 0x1f, 0x8b, 0x08, 0x08, 0x72, 0x66, 0x8b, 0x4a, + 0x00, 0x03, 0x73, 0x68, 0x65, 0x73, 0x65, 0x6c, + 0x6c, 0x73, 0x2e, 0x74, 0x78, 0x74, 0x00, 0x2b, + 0xce, 0x48, 0x55, 0x28, 0x4e, 0xcd, 0xc9, 0x29, + 0x06, 0x92, 0x89, 0xc5, 0x19, 0x60, 0x56, 0x52, + 0xa5, 0x42, 0x09, 0x58, 0x18, 0x28, 0x90, 0x5f, + 0x94, 0xca, 0x05, 0x00, 0x76, 0xb0, 0x3b, 0xeb, + 0x24, 0x00, 0x00, 0x00, + }, + nil, + }, + { // has dynamic huffman blocks + "gettysburg", + "gettysburg", + " Four score and seven years ago our fathers brought forth on\n" + + "this continent, a new nation, conceived in Liberty, and dedicated\n" + + "to the proposition that all men are created equal.\n" + + " Now we are engaged in a great Civil War, testing whether that\n" + + "nation, or any nation so conceived and so dedicated, can long\n" + + "endure.\n" + + " We are met on a great battle-field of that war.\n" + + " We have come to dedicate a portion of that field, as a final\n" + + "resting place for those who here gave their lives that that\n" + + "nation might live. It is altogether fitting and proper that\n" + + "we should do this.\n" + + " But, in a larger sense, we can not dedicate — we can not\n" + + "consecrate — we can not hallow — this ground.\n" + + " The brave men, living and dead, who struggled here, have\n" + + "consecrated it, far above our poor power to add or detract.\n" + + "The world will little note, nor long remember what we say here,\n" + + "but it can never forget what they did here.\n" + + " It is for us the living, rather, to be dedicated here to the\n" + + "unfinished work which they who fought here have thus far so\n" + + "nobly advanced. It is rather for us to be here dedicated to\n" + + "the great task remaining before us — that from these honored\n" + + "dead we take increased devotion to that cause for which they\n" + + "gave the last full measure of devotion —\n" + + " that we here highly resolve that these dead shall not have\n" + + "died in vain — that this nation, under God, shall have a new\n" + + "birth of freedom — and that government of the people, by the\n" + + "people, for the people, shall not perish from this earth.\n" + + "\n" + + "Abraham Lincoln, November 19, 1863, Gettysburg, Pennsylvania\n", + []byte{ + 0x1f, 0x8b, 0x08, 0x08, 0xd1, 0x12, 0x2b, 0x4a, + 0x00, 0x03, 0x67, 0x65, 0x74, 0x74, 0x79, 0x73, + 0x62, 0x75, 0x72, 0x67, 0x00, 0x65, 0x54, 0xcd, + 0x6e, 0xd4, 0x30, 0x10, 0xbe, 0xfb, 0x29, 0xe6, + 0x01, 0x42, 0xa5, 0x0a, 0x09, 0xc1, 0x11, 0x90, + 0x40, 0x48, 0xa8, 0xe2, 0x80, 0xd4, 0xf3, 0x24, + 0x9e, 0x24, 0x56, 0xbd, 0x9e, 0xc5, 0x76, 0x76, + 0x95, 0x1b, 0x0f, 0xc1, 0x13, 0xf2, 0x24, 0x7c, + 0x63, 0x77, 0x9b, 0x4a, 0x5c, 0xaa, 0x6e, 0x6c, + 0xcf, 0x7c, 0x7f, 0x33, 0x44, 0x5f, 0x74, 0xcb, + 0x54, 0x26, 0xcd, 0x42, 0x9c, 0x3c, 0x15, 0xb9, + 0x48, 0xa2, 0x5d, 0x38, 0x17, 0xe2, 0x45, 0xc9, + 0x4e, 0x67, 0xae, 0xab, 0xe0, 0xf7, 0x98, 0x75, + 0x5b, 0xd6, 0x4a, 0xb3, 0xe6, 0xba, 0x92, 0x26, + 0x57, 0xd7, 0x50, 0x68, 0xd2, 0x54, 0x43, 0x92, + 0x54, 0x07, 0x62, 0x4a, 0x72, 0xa5, 0xc4, 0x35, + 0x68, 0x1a, 0xec, 0x60, 0x92, 0x70, 0x11, 0x4f, + 0x21, 0xd1, 0xf7, 0x30, 0x4a, 0xae, 0xfb, 0xd0, + 0x9a, 0x78, 0xf1, 0x61, 0xe2, 0x2a, 0xde, 0x55, + 0x25, 0xd4, 0xa6, 0x73, 0xd6, 0xb3, 0x96, 0x60, + 0xef, 0xf0, 0x9b, 0x2b, 0x71, 0x8c, 0x74, 0x02, + 0x10, 0x06, 0xac, 0x29, 0x8b, 0xdd, 0x25, 0xf9, + 0xb5, 0x71, 0xbc, 0x73, 0x44, 0x0f, 0x7a, 0xa5, + 0xab, 0xb4, 0x33, 0x49, 0x0b, 0x2f, 0xbd, 0x03, + 0xd3, 0x62, 0x17, 0xe9, 0x73, 0xb8, 0x84, 0x48, + 0x8f, 0x9c, 0x07, 0xaa, 0x52, 0x00, 0x6d, 0xa1, + 0xeb, 0x2a, 0xc6, 0xa0, 0x95, 0x76, 0x37, 0x78, + 0x9a, 0x81, 0x65, 0x7f, 0x46, 0x4b, 0x45, 0x5f, + 0xe1, 0x6d, 0x42, 0xe8, 0x01, 0x13, 0x5c, 0x38, + 0x51, 0xd4, 0xb4, 0x38, 0x49, 0x7e, 0xcb, 0x62, + 0x28, 0x1e, 0x3b, 0x82, 0x93, 0x54, 0x48, 0xf1, + 0xd2, 0x7d, 0xe4, 0x5a, 0xa3, 0xbc, 0x99, 0x83, + 0x44, 0x4f, 0x3a, 0x77, 0x36, 0x57, 0xce, 0xcf, + 0x2f, 0x56, 0xbe, 0x80, 0x90, 0x9e, 0x84, 0xea, + 0x51, 0x1f, 0x8f, 0xcf, 0x90, 0xd4, 0x60, 0xdc, + 0x5e, 0xb4, 0xf7, 0x10, 0x0b, 0x26, 0xe0, 0xff, + 0xc4, 0xd1, 0xe5, 0x67, 0x2e, 0xe7, 0xc8, 0x93, + 0x98, 0x05, 0xb8, 0xa8, 0x45, 0xc0, 0x4d, 0x09, + 0xdc, 0x84, 0x16, 0x2b, 0x0d, 0x9a, 0x21, 0x53, + 0x04, 0x8b, 0xd2, 0x0b, 0xbd, 0xa2, 0x4c, 0xa7, + 0x60, 0xee, 0xd9, 0xe1, 0x1d, 0xd1, 0xb7, 0x4a, + 0x30, 0x8f, 0x63, 0xd5, 0xa5, 0x8b, 0x33, 0x87, + 0xda, 0x1a, 0x18, 0x79, 0xf3, 0xe3, 0xa6, 0x17, + 0x94, 0x2e, 0xab, 0x6e, 0xa0, 0xe3, 0xcd, 0xac, + 0x50, 0x8c, 0xca, 0xa7, 0x0d, 0x76, 0x37, 0xd1, + 0x23, 0xe7, 0x05, 0x57, 0x8b, 0xa4, 0x22, 0x83, + 0xd9, 0x62, 0x52, 0x25, 0xad, 0x07, 0xbb, 0xbf, + 0xbf, 0xff, 0xbc, 0xfa, 0xee, 0x20, 0x73, 0x91, + 0x29, 0xff, 0x7f, 0x02, 0x71, 0x62, 0x84, 0xb5, + 0xf6, 0xb5, 0x25, 0x6b, 0x41, 0xde, 0x92, 0xb7, + 0x76, 0x3f, 0x91, 0x91, 0x31, 0x1b, 0x41, 0x84, + 0x62, 0x30, 0x0a, 0x37, 0xa4, 0x5e, 0x18, 0x3a, + 0x99, 0x08, 0xa5, 0xe6, 0x6d, 0x59, 0x22, 0xec, + 0x33, 0x39, 0x86, 0x26, 0xf5, 0xab, 0x66, 0xc8, + 0x08, 0x20, 0xcf, 0x0c, 0xd7, 0x47, 0x45, 0x21, + 0x0b, 0xf6, 0x59, 0xd5, 0xfe, 0x5c, 0x8d, 0xaa, + 0x12, 0x7b, 0x6f, 0xa1, 0xf0, 0x52, 0x33, 0x4f, + 0xf5, 0xce, 0x59, 0xd3, 0xab, 0x66, 0x10, 0xbf, + 0x06, 0xc4, 0x31, 0x06, 0x73, 0xd6, 0x80, 0xa2, + 0x78, 0xc2, 0x45, 0xcb, 0x03, 0x65, 0x39, 0xc9, + 0x09, 0xd1, 0x06, 0x04, 0x33, 0x1a, 0x5a, 0xf1, + 0xde, 0x01, 0xb8, 0x71, 0x83, 0xc4, 0xb5, 0xb3, + 0xc3, 0x54, 0x65, 0x33, 0x0d, 0x5a, 0xf7, 0x9b, + 0x90, 0x7c, 0x27, 0x1f, 0x3a, 0x58, 0xa3, 0xd8, + 0xfd, 0x30, 0x5f, 0xb7, 0xd2, 0x66, 0xa2, 0x93, + 0x1c, 0x28, 0xb7, 0xe9, 0x1b, 0x0c, 0xe1, 0x28, + 0x47, 0x26, 0xbb, 0xe9, 0x7d, 0x7e, 0xdc, 0x96, + 0x10, 0x92, 0x50, 0x56, 0x7c, 0x06, 0xe2, 0x27, + 0xb4, 0x08, 0xd3, 0xda, 0x7b, 0x98, 0x34, 0x73, + 0x9f, 0xdb, 0xf6, 0x62, 0xed, 0x31, 0x41, 0x13, + 0xd3, 0xa2, 0xa8, 0x4b, 0x3a, 0xc6, 0x1d, 0xe4, + 0x2f, 0x8c, 0xf8, 0xfb, 0x97, 0x64, 0xf4, 0xb6, + 0x2f, 0x80, 0x5a, 0xf3, 0x56, 0xe0, 0x40, 0x50, + 0xd5, 0x19, 0xd0, 0x1e, 0xfc, 0xca, 0xe5, 0xc9, + 0xd4, 0x60, 0x00, 0x81, 0x2e, 0xa3, 0xcc, 0xb6, + 0x52, 0xf0, 0xb4, 0xdb, 0x69, 0x99, 0xce, 0x7a, + 0x32, 0x4c, 0x08, 0xed, 0xaa, 0x10, 0x10, 0xe3, + 0x6f, 0xee, 0x99, 0x68, 0x95, 0x9f, 0x04, 0x71, + 0xb2, 0x49, 0x2f, 0x62, 0xa6, 0x5e, 0xb4, 0xef, + 0x02, 0xed, 0x4f, 0x27, 0xde, 0x4a, 0x0f, 0xfd, + 0xc1, 0xcc, 0xdd, 0x02, 0x8f, 0x08, 0x16, 0x54, + 0xdf, 0xda, 0xca, 0xe0, 0x82, 0xf1, 0xb4, 0x31, + 0x7a, 0xa9, 0x81, 0xfe, 0x90, 0xb7, 0x3e, 0xdb, + 0xd3, 0x35, 0xc0, 0x20, 0x80, 0x33, 0x46, 0x4a, + 0x63, 0xab, 0xd1, 0x0d, 0x29, 0xd2, 0xe2, 0x84, + 0xb8, 0xdb, 0xfa, 0xe9, 0x89, 0x44, 0x86, 0x7c, + 0xe8, 0x0b, 0xe6, 0x02, 0x6a, 0x07, 0x9b, 0x96, + 0xd0, 0xdb, 0x2e, 0x41, 0x4c, 0xa1, 0xd5, 0x57, + 0x45, 0x14, 0xfb, 0xe3, 0xa6, 0x72, 0x5b, 0x87, + 0x6e, 0x0c, 0x6d, 0x5b, 0xce, 0xe0, 0x2f, 0xe2, + 0x21, 0x81, 0x95, 0xb0, 0xe8, 0xb6, 0x32, 0x0b, + 0xb2, 0x98, 0x13, 0x52, 0x5d, 0xfb, 0xec, 0x63, + 0x17, 0x8a, 0x9e, 0x23, 0x22, 0x36, 0xee, 0xcd, + 0xda, 0xdb, 0xcf, 0x3e, 0xf1, 0xc7, 0xf1, 0x01, + 0x12, 0x93, 0x0a, 0xeb, 0x6f, 0xf2, 0x02, 0x15, + 0x96, 0x77, 0x5d, 0xef, 0x9c, 0xfb, 0x88, 0x91, + 0x59, 0xf9, 0x84, 0xdd, 0x9b, 0x26, 0x8d, 0x80, + 0xf9, 0x80, 0x66, 0x2d, 0xac, 0xf7, 0x1f, 0x06, + 0xba, 0x7f, 0xff, 0xee, 0xed, 0x40, 0x5f, 0xa5, + 0xd6, 0xbd, 0x8c, 0x5b, 0x46, 0xd2, 0x7e, 0x48, + 0x4a, 0x65, 0x8f, 0x08, 0x42, 0x60, 0xf7, 0x0f, + 0xb9, 0x16, 0x0b, 0x0c, 0x1a, 0x06, 0x00, 0x00, + }, + nil, + }, + { // has 1 non-empty fixed huffman block then garbage + "hello.txt", + "hello.txt + garbage", + "hello world\n", + []byte{ + 0x1f, 0x8b, 0x08, 0x08, 0xc8, 0x58, 0x13, 0x4a, + 0x00, 0x03, 0x68, 0x65, 0x6c, 0x6c, 0x6f, 0x2e, + 0x74, 0x78, 0x74, 0x00, 0xcb, 0x48, 0xcd, 0xc9, + 0xc9, 0x57, 0x28, 0xcf, 0x2f, 0xca, 0x49, 0xe1, + 0x02, 0x00, 0x2d, 0x3b, 0x08, 0xaf, 0x0c, 0x00, + 0x00, 0x00, 'g', 'a', 'r', 'b', 'a', 'g', 'e', '!', '!', '!', + }, + HeaderError, + }, + { // has 1 non-empty fixed huffman block not enough header + "hello.txt", + "hello.txt + garbage", + "hello world\n", + []byte{ + 0x1f, 0x8b, 0x08, 0x08, 0xc8, 0x58, 0x13, 0x4a, + 0x00, 0x03, 0x68, 0x65, 0x6c, 0x6c, 0x6f, 0x2e, + 0x74, 0x78, 0x74, 0x00, 0xcb, 0x48, 0xcd, 0xc9, + 0xc9, 0x57, 0x28, 0xcf, 0x2f, 0xca, 0x49, 0xe1, + 0x02, 0x00, 0x2d, 0x3b, 0x08, 0xaf, 0x0c, 0x00, + 0x00, 0x00, gzipID1, + }, + io.ErrUnexpectedEOF, + }, + { // has 1 non-empty fixed huffman block but corrupt checksum + "hello.txt", + "hello.txt + corrupt checksum", + "hello world\n", + []byte{ + 0x1f, 0x8b, 0x08, 0x08, 0xc8, 0x58, 0x13, 0x4a, + 0x00, 0x03, 0x68, 0x65, 0x6c, 0x6c, 0x6f, 0x2e, + 0x74, 0x78, 0x74, 0x00, 0xcb, 0x48, 0xcd, 0xc9, + 0xc9, 0x57, 0x28, 0xcf, 0x2f, 0xca, 0x49, 0xe1, + 0x02, 0x00, 0xff, 0xff, 0xff, 0xff, 0x0c, 0x00, + 0x00, 0x00, + }, + ChecksumError, + }, + { // has 1 non-empty fixed huffman block but corrupt size + "hello.txt", + "hello.txt + corrupt size", + "hello world\n", + []byte{ + 0x1f, 0x8b, 0x08, 0x08, 0xc8, 0x58, 0x13, 0x4a, + 0x00, 0x03, 0x68, 0x65, 0x6c, 0x6c, 0x6f, 0x2e, + 0x74, 0x78, 0x74, 0x00, 0xcb, 0x48, 0xcd, 0xc9, + 0xc9, 0x57, 0x28, 0xcf, 0x2f, 0xca, 0x49, 0xe1, + 0x02, 0x00, 0x2d, 0x3b, 0x08, 0xaf, 0xff, 0x00, + 0x00, 0x00, + }, + ChecksumError, + }, +} + +func TestDecompressor(t *testing.T) { + b := new(bytes.Buffer) + for _, tt := range gunzipTests { + in := bytes.NewBuffer(tt.gzip) + gzip, err := NewReader(in) + if err != nil { + t.Errorf("%s: NewReader: %s", tt.name, err) + continue + } + defer gzip.Close() + if tt.name != gzip.Name { + t.Errorf("%s: got name %s", tt.name, gzip.Name) + } + b.Reset() + n, err := io.Copy(b, gzip) + if err != tt.err { + t.Errorf("%s: io.Copy: %v want %v", tt.name, err, tt.err) + } + s := b.String() + if s != tt.raw { + t.Errorf("%s: got %d-byte %q want %d-byte %q", tt.name, n, s, len(tt.raw), tt.raw) + } + } +} diff --git a/src/pkg/compress/gzip/gzip.go b/src/pkg/compress/gzip/gzip.go new file mode 100644 index 000000000..8860d10af --- /dev/null +++ b/src/pkg/compress/gzip/gzip.go @@ -0,0 +1,187 @@ +// Copyright 2010 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package gzip + +import ( + "compress/flate" + "hash" + "hash/crc32" + "io" + "os" +) + +// These constants are copied from the flate package, so that code that imports +// "compress/gzip" does not also have to import "compress/flate". +const ( + NoCompression = flate.NoCompression + BestSpeed = flate.BestSpeed + BestCompression = flate.BestCompression + DefaultCompression = flate.DefaultCompression +) + +// A Compressor is an io.WriteCloser that satisfies writes by compressing data written +// to its wrapped io.Writer. +type Compressor struct { + Header + w io.Writer + level int + compressor io.WriteCloser + digest hash.Hash32 + size uint32 + closed bool + buf [10]byte + err os.Error +} + +// NewWriter calls NewWriterLevel with the default compression level. +func NewWriter(w io.Writer) (*Compressor, os.Error) { + return NewWriterLevel(w, DefaultCompression) +} + +// NewWriterLevel creates a new Compressor writing to the given writer. +// Writes may be buffered and not flushed until Close. +// Callers that wish to set the fields in Compressor.Header must +// do so before the first call to Write or Close. +// It is the caller's responsibility to call Close on the WriteCloser when done. +// level is the compression level, which can be DefaultCompression, NoCompression, +// or any integer value between BestSpeed and BestCompression (inclusive). +func NewWriterLevel(w io.Writer, level int) (*Compressor, os.Error) { + z := new(Compressor) + z.OS = 255 // unknown + z.w = w + z.level = level + z.digest = crc32.NewIEEE() + return z, nil +} + +// GZIP (RFC 1952) is little-endian, unlike ZLIB (RFC 1950). +func put2(p []byte, v uint16) { + p[0] = uint8(v >> 0) + p[1] = uint8(v >> 8) +} + +func put4(p []byte, v uint32) { + p[0] = uint8(v >> 0) + p[1] = uint8(v >> 8) + p[2] = uint8(v >> 16) + p[3] = uint8(v >> 24) +} + +// writeBytes writes a length-prefixed byte slice to z.w. +func (z *Compressor) writeBytes(b []byte) os.Error { + if len(b) > 0xffff { + return os.NewError("gzip.Write: Extra data is too large") + } + put2(z.buf[0:2], uint16(len(b))) + _, err := z.w.Write(z.buf[0:2]) + if err != nil { + return err + } + _, err = z.w.Write(b) + return err +} + +// writeString writes a string (in ISO 8859-1 (Latin-1) format) to z.w. +func (z *Compressor) writeString(s string) os.Error { + // GZIP (RFC 1952) specifies that strings are NUL-terminated ISO 8859-1 (Latin-1). + // TODO(nigeltao): Convert from UTF-8 to ISO 8859-1 (Latin-1). + for _, v := range s { + if v == 0 || v > 0x7f { + return os.NewError("gzip.Write: non-ASCII header string") + } + } + _, err := io.WriteString(z.w, s) + if err != nil { + return err + } + // GZIP strings are NUL-terminated. + z.buf[0] = 0 + _, err = z.w.Write(z.buf[0:1]) + return err +} + +func (z *Compressor) Write(p []byte) (int, os.Error) { + if z.err != nil { + return 0, z.err + } + var n int + // Write the GZIP header lazily. + if z.compressor == nil { + z.buf[0] = gzipID1 + z.buf[1] = gzipID2 + z.buf[2] = gzipDeflate + z.buf[3] = 0 + if z.Extra != nil { + z.buf[3] |= 0x04 + } + if z.Name != "" { + z.buf[3] |= 0x08 + } + if z.Comment != "" { + z.buf[3] |= 0x10 + } + put4(z.buf[4:8], z.Mtime) + if z.level == BestCompression { + z.buf[8] = 2 + } else if z.level == BestSpeed { + z.buf[8] = 4 + } else { + z.buf[8] = 0 + } + z.buf[9] = z.OS + n, z.err = z.w.Write(z.buf[0:10]) + if z.err != nil { + return n, z.err + } + if z.Extra != nil { + z.err = z.writeBytes(z.Extra) + if z.err != nil { + return n, z.err + } + } + if z.Name != "" { + z.err = z.writeString(z.Name) + if z.err != nil { + return n, z.err + } + } + if z.Comment != "" { + z.err = z.writeString(z.Comment) + if z.err != nil { + return n, z.err + } + } + z.compressor = flate.NewWriter(z.w, z.level) + } + z.size += uint32(len(p)) + z.digest.Write(p) + n, z.err = z.compressor.Write(p) + return n, z.err +} + +// Calling Close does not close the wrapped io.Writer originally passed to NewWriter. +func (z *Compressor) Close() os.Error { + if z.err != nil { + return z.err + } + if z.closed { + return nil + } + z.closed = true + if z.compressor == nil { + z.Write(nil) + if z.err != nil { + return z.err + } + } + z.err = z.compressor.Close() + if z.err != nil { + return z.err + } + put4(z.buf[0:4], z.digest.Sum32()) + put4(z.buf[4:8], z.size) + _, z.err = z.w.Write(z.buf[0:8]) + return z.err +} diff --git a/src/pkg/compress/gzip/gzip_test.go b/src/pkg/compress/gzip/gzip_test.go new file mode 100644 index 000000000..121e627e6 --- /dev/null +++ b/src/pkg/compress/gzip/gzip_test.go @@ -0,0 +1,84 @@ +// Copyright 2010 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package gzip + +import ( + "io" + "io/ioutil" + "testing" +) + +// pipe creates two ends of a pipe that gzip and gunzip, and runs dfunc at the +// writer end and cfunc at the reader end. +func pipe(t *testing.T, dfunc func(*Compressor), cfunc func(*Decompressor)) { + piper, pipew := io.Pipe() + defer piper.Close() + go func() { + defer pipew.Close() + compressor, err := NewWriter(pipew) + if err != nil { + t.Fatalf("%v", err) + } + defer compressor.Close() + dfunc(compressor) + }() + decompressor, err := NewReader(piper) + if err != nil { + t.Fatalf("%v", err) + } + defer decompressor.Close() + cfunc(decompressor) +} + +// Tests that an empty payload still forms a valid GZIP stream. +func TestEmpty(t *testing.T) { + pipe(t, + func(compressor *Compressor) {}, + func(decompressor *Decompressor) { + b, err := ioutil.ReadAll(decompressor) + if err != nil { + t.Fatalf("%v", err) + } + if len(b) != 0 { + t.Fatalf("did not read an empty slice") + } + }) +} + +// Tests that gzipping and then gunzipping is the identity function. +func TestWriter(t *testing.T) { + pipe(t, + func(compressor *Compressor) { + compressor.Comment = "comment" + compressor.Extra = []byte("extra") + compressor.Mtime = 1e8 + compressor.Name = "name" + _, err := compressor.Write([]byte("payload")) + if err != nil { + t.Fatalf("%v", err) + } + }, + func(decompressor *Decompressor) { + b, err := ioutil.ReadAll(decompressor) + if err != nil { + t.Fatalf("%v", err) + } + if string(b) != "payload" { + t.Fatalf("payload is %q, want %q", string(b), "payload") + } + if decompressor.Comment != "comment" { + t.Fatalf("comment is %q, want %q", decompressor.Comment, "comment") + } + if string(decompressor.Extra) != "extra" { + t.Fatalf("extra is %q, want %q", decompressor.Extra, "extra") + } + if decompressor.Mtime != 1e8 { + t.Fatalf("mtime is %d, want %d", decompressor.Mtime, uint32(1e8)) + } + if decompressor.Name != "name" { + t.Fatalf("name is %q, want %q", decompressor.Name, "name") + } + }) +} diff --git a/src/pkg/compress/lzw/Makefile b/src/pkg/compress/lzw/Makefile new file mode 100644 index 000000000..28f5e6abc --- /dev/null +++ b/src/pkg/compress/lzw/Makefile @@ -0,0 +1,12 @@ +# Copyright 2011 The Go Authors. All rights reserved. +# Use of this source code is governed by a BSD-style +# license that can be found in the LICENSE file. + +include ../../../Make.inc + +TARG=compress/lzw +GOFILES=\ + reader.go\ + writer.go\ + +include ../../../Make.pkg diff --git a/src/pkg/compress/lzw/reader.go b/src/pkg/compress/lzw/reader.go new file mode 100644 index 000000000..21231c8e5 --- /dev/null +++ b/src/pkg/compress/lzw/reader.go @@ -0,0 +1,253 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Package lzw implements the Lempel-Ziv-Welch compressed data format, +// described in T. A. Welch, ``A Technique for High-Performance Data +// Compression'', Computer, 17(6) (June 1984), pp 8-19. +// +// In particular, it implements LZW as used by the GIF, TIFF and PDF file +// formats, which means variable-width codes up to 12 bits and the first +// two non-literal codes are a clear code and an EOF code. +package lzw + +// TODO(nigeltao): check that TIFF and PDF use LZW in the same way as GIF, +// modulo LSB/MSB packing order. + +import ( + "bufio" + "fmt" + "io" + "os" +) + +// Order specifies the bit ordering in an LZW data stream. +type Order int + +const ( + // LSB means Least Significant Bits first, as used in the GIF file format. + LSB Order = iota + // MSB means Most Significant Bits first, as used in the TIFF and PDF + // file formats. + MSB +) + +const ( + maxWidth = 12 + decoderInvalidCode = 0xffff + flushBuffer = 1 << maxWidth +) + +// decoder is the state from which the readXxx method converts a byte +// stream into a code stream. +type decoder struct { + r io.ByteReader + bits uint32 + nBits uint + width uint + read func(*decoder) (uint16, os.Error) // readLSB or readMSB + litWidth int // width in bits of literal codes + err os.Error + + // The first 1<= 1<>= d.width + d.nBits -= d.width + return code, nil +} + +// readMSB returns the next code for "Most Significant Bits first" data. +func (d *decoder) readMSB() (uint16, os.Error) { + for d.nBits < d.width { + x, err := d.r.ReadByte() + if err != nil { + return 0, err + } + d.bits |= uint32(x) << (24 - d.nBits) + d.nBits += 8 + } + code := uint16(d.bits >> (32 - d.width)) + d.bits <<= d.width + d.nBits -= d.width + return code, nil +} + +func (d *decoder) Read(b []byte) (int, os.Error) { + for { + if len(d.toRead) > 0 { + n := copy(b, d.toRead) + d.toRead = d.toRead[n:] + return n, nil + } + if d.err != nil { + return 0, d.err + } + d.decode() + } + panic("unreachable") +} + +// decode decompresses bytes from r and leaves them in d.toRead. +// read specifies how to decode bytes into codes. +// litWidth is the width in bits of literal codes. +func (d *decoder) decode() { + // Loop over the code stream, converting codes into decompressed bytes. + for { + code, err := d.read(d) + if err != nil { + if err == os.EOF { + err = io.ErrUnexpectedEOF + } + d.err = err + return + } + switch { + case code < d.clear: + // We have a literal code. + d.output[d.o] = uint8(code) + d.o++ + if d.last != decoderInvalidCode { + // Save what the hi code expands to. + d.suffix[d.hi] = uint8(code) + d.prefix[d.hi] = d.last + } + case code == d.clear: + d.width = 1 + uint(d.litWidth) + d.hi = d.eof + d.overflow = 1 << d.width + d.last = decoderInvalidCode + continue + case code == d.eof: + d.flush() + d.err = os.EOF + return + case code <= d.hi: + c, i := code, len(d.output)-1 + if code == d.hi { + // code == hi is a special case which expands to the last expansion + // followed by the head of the last expansion. To find the head, we walk + // the prefix chain until we find a literal code. + c = d.last + for c >= d.clear { + c = d.prefix[c] + } + d.output[i] = uint8(c) + i-- + c = d.last + } + // Copy the suffix chain into output and then write that to w. + for c >= d.clear { + d.output[i] = d.suffix[c] + i-- + c = d.prefix[c] + } + d.output[i] = uint8(c) + d.o += copy(d.output[d.o:], d.output[i:]) + if d.last != decoderInvalidCode { + // Save what the hi code expands to. + d.suffix[d.hi] = uint8(c) + d.prefix[d.hi] = d.last + } + default: + d.err = os.NewError("lzw: invalid code") + return + } + d.last, d.hi = code, d.hi+1 + if d.hi >= d.overflow { + if d.width == maxWidth { + d.last = decoderInvalidCode + } else { + d.width++ + d.overflow <<= 1 + } + } + if d.o >= flushBuffer { + d.flush() + return + } + } + panic("unreachable") +} + +func (d *decoder) flush() { + d.toRead = d.output[:d.o] + d.o = 0 +} + +func (d *decoder) Close() os.Error { + d.err = os.EINVAL // in case any Reads come along + return nil +} + +// NewReader creates a new io.ReadCloser that satisfies reads by decompressing +// the data read from r. +// It is the caller's responsibility to call Close on the ReadCloser when +// finished reading. +// The number of bits to use for literal codes, litWidth, must be in the +// range [2,8] and is typically 8. +func NewReader(r io.Reader, order Order, litWidth int) io.ReadCloser { + d := new(decoder) + switch order { + case LSB: + d.read = (*decoder).readLSB + case MSB: + d.read = (*decoder).readMSB + default: + d.err = os.NewError("lzw: unknown order") + return d + } + if litWidth < 2 || 8 < litWidth { + d.err = fmt.Errorf("lzw: litWidth %d out of range", litWidth) + return d + } + if br, ok := r.(io.ByteReader); ok { + d.r = br + } else { + d.r = bufio.NewReader(r) + } + d.litWidth = litWidth + d.width = 1 + uint(litWidth) + d.clear = uint16(1) << uint(litWidth) + d.eof, d.hi = d.clear+1, d.clear+1 + d.overflow = uint16(1) << d.width + d.last = decoderInvalidCode + + return d +} diff --git a/src/pkg/compress/lzw/reader_test.go b/src/pkg/compress/lzw/reader_test.go new file mode 100644 index 000000000..f8042b0d1 --- /dev/null +++ b/src/pkg/compress/lzw/reader_test.go @@ -0,0 +1,145 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package lzw + +import ( + "bytes" + "io" + "io/ioutil" + "os" + "runtime" + "strconv" + "strings" + "testing" +) + +type lzwTest struct { + desc string + raw string + compressed string + err os.Error +} + +var lzwTests = []lzwTest{ + { + "empty;LSB;8", + "", + "\x01\x01", + nil, + }, + { + "empty;MSB;8", + "", + "\x80\x80", + nil, + }, + { + "tobe;LSB;7", + "TOBEORNOTTOBEORTOBEORNOT", + "\x54\x4f\x42\x45\x4f\x52\x4e\x4f\x54\x82\x84\x86\x8b\x85\x87\x89\x81", + nil, + }, + { + "tobe;LSB;8", + "TOBEORNOTTOBEORTOBEORNOT", + "\x54\x9e\x08\x29\xf2\x44\x8a\x93\x27\x54\x04\x12\x34\xb8\xb0\xe0\xc1\x84\x01\x01", + nil, + }, + { + "tobe;MSB;7", + "TOBEORNOTTOBEORTOBEORNOT", + "\x54\x4f\x42\x45\x4f\x52\x4e\x4f\x54\x82\x84\x86\x8b\x85\x87\x89\x81", + nil, + }, + { + "tobe;MSB;8", + "TOBEORNOTTOBEORTOBEORNOT", + "\x2a\x13\xc8\x44\x52\x79\x48\x9c\x4f\x2a\x40\xa0\x90\x68\x5c\x16\x0f\x09\x80\x80", + nil, + }, + { + "tobe-truncated;LSB;8", + "TOBEORNOTTOBEORTOBEORNOT", + "\x54\x9e\x08\x29\xf2\x44\x8a\x93\x27\x54\x04", + io.ErrUnexpectedEOF, + }, + // This example comes from http://en.wikipedia.org/wiki/Graphics_Interchange_Format. + { + "gif;LSB;8", + "\x28\xff\xff\xff\x28\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff", + "\x00\x51\xfc\x1b\x28\x70\xa0\xc1\x83\x01\x01", + nil, + }, + // This example comes from http://compgroups.net/comp.lang.ruby/Decompressing-LZW-compression-from-PDF-file + { + "pdf;MSB;8", + "-----A---B", + "\x80\x0b\x60\x50\x22\x0c\x0c\x85\x01", + nil, + }, +} + +func TestReader(t *testing.T) { + b := bytes.NewBuffer(nil) + for _, tt := range lzwTests { + d := strings.Split(tt.desc, ";") + var order Order + switch d[1] { + case "LSB": + order = LSB + case "MSB": + order = MSB + default: + t.Errorf("%s: bad order %q", tt.desc, d[1]) + } + litWidth, _ := strconv.Atoi(d[2]) + rc := NewReader(strings.NewReader(tt.compressed), order, litWidth) + defer rc.Close() + b.Reset() + n, err := io.Copy(b, rc) + if err != nil { + if err != tt.err { + t.Errorf("%s: io.Copy: %v want %v", tt.desc, err, tt.err) + } + continue + } + s := b.String() + if s != tt.raw { + t.Errorf("%s: got %d-byte %q want %d-byte %q", tt.desc, n, s, len(tt.raw), tt.raw) + } + } +} + +func benchmarkDecoder(b *testing.B, n int) { + b.StopTimer() + b.SetBytes(int64(n)) + buf0, _ := ioutil.ReadFile("../testdata/e.txt") + buf0 = buf0[:10000] + compressed := bytes.NewBuffer(nil) + w := NewWriter(compressed, LSB, 8) + for i := 0; i < n; i += len(buf0) { + io.Copy(w, bytes.NewBuffer(buf0)) + } + w.Close() + buf1 := compressed.Bytes() + buf0, compressed, w = nil, nil, nil + runtime.GC() + b.StartTimer() + for i := 0; i < b.N; i++ { + io.Copy(ioutil.Discard, NewReader(bytes.NewBuffer(buf1), LSB, 8)) + } +} + +func BenchmarkDecoder1e4(b *testing.B) { + benchmarkDecoder(b, 1e4) +} + +func BenchmarkDecoder1e5(b *testing.B) { + benchmarkDecoder(b, 1e5) +} + +func BenchmarkDecoder1e6(b *testing.B) { + benchmarkDecoder(b, 1e6) +} diff --git a/src/pkg/compress/lzw/writer.go b/src/pkg/compress/lzw/writer.go new file mode 100644 index 000000000..87143b7aa --- /dev/null +++ b/src/pkg/compress/lzw/writer.go @@ -0,0 +1,259 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package lzw + +import ( + "bufio" + "fmt" + "io" + "os" +) + +// A writer is a buffered, flushable writer. +type writer interface { + WriteByte(byte) os.Error + Flush() os.Error +} + +// An errWriteCloser is an io.WriteCloser that always returns a given error. +type errWriteCloser struct { + err os.Error +} + +func (e *errWriteCloser) Write([]byte) (int, os.Error) { + return 0, e.err +} + +func (e *errWriteCloser) Close() os.Error { + return e.err +} + +const ( + // A code is a 12 bit value, stored as a uint32 when encoding to avoid + // type conversions when shifting bits. + maxCode = 1<<12 - 1 + invalidCode = 1<<32 - 1 + // There are 1<<12 possible codes, which is an upper bound on the number of + // valid hash table entries at any given point in time. tableSize is 4x that. + tableSize = 4 * 1 << 12 + tableMask = tableSize - 1 + // A hash table entry is a uint32. Zero is an invalid entry since the + // lower 12 bits of a valid entry must be a non-literal code. + invalidEntry = 0 +) + +// encoder is LZW compressor. +type encoder struct { + // w is the writer that compressed bytes are written to. + w writer + // write, bits, nBits and width are the state for converting a code stream + // into a byte stream. + write func(*encoder, uint32) os.Error + bits uint32 + nBits uint + width uint + // litWidth is the width in bits of literal codes. + litWidth uint + // hi is the code implied by the next code emission. + // overflow is the code at which hi overflows the code width. + hi, overflow uint32 + // savedCode is the accumulated code at the end of the most recent Write + // call. It is equal to invalidCode if there was no such call. + savedCode uint32 + // err is the first error encountered during writing. Closing the encoder + // will make any future Write calls return os.EINVAL. + err os.Error + // table is the hash table from 20-bit keys to 12-bit values. Each table + // entry contains key<<12|val and collisions resolve by linear probing. + // The keys consist of a 12-bit code prefix and an 8-bit byte suffix. + // The values are a 12-bit code. + table [tableSize]uint32 +} + +// writeLSB writes the code c for "Least Significant Bits first" data. +func (e *encoder) writeLSB(c uint32) os.Error { + e.bits |= c << e.nBits + e.nBits += e.width + for e.nBits >= 8 { + if err := e.w.WriteByte(uint8(e.bits)); err != nil { + return err + } + e.bits >>= 8 + e.nBits -= 8 + } + return nil +} + +// writeMSB writes the code c for "Most Significant Bits first" data. +func (e *encoder) writeMSB(c uint32) os.Error { + e.bits |= c << (32 - e.width - e.nBits) + e.nBits += e.width + for e.nBits >= 8 { + if err := e.w.WriteByte(uint8(e.bits >> 24)); err != nil { + return err + } + e.bits <<= 8 + e.nBits -= 8 + } + return nil +} + +// errOutOfCodes is an internal error that means that the encoder has run out +// of unused codes and a clear code needs to be sent next. +var errOutOfCodes = os.NewError("lzw: out of codes") + +// incHi increments e.hi and checks for both overflow and running out of +// unused codes. In the latter case, incHi sends a clear code, resets the +// encoder state and returns errOutOfCodes. +func (e *encoder) incHi() os.Error { + e.hi++ + if e.hi == e.overflow { + e.width++ + e.overflow <<= 1 + } + if e.hi == maxCode { + clear := uint32(1) << e.litWidth + if err := e.write(e, clear); err != nil { + return err + } + e.width = uint(e.litWidth) + 1 + e.hi = clear + 1 + e.overflow = clear << 1 + for i := range e.table { + e.table[i] = invalidEntry + } + return errOutOfCodes + } + return nil +} + +// Write writes a compressed representation of p to e's underlying writer. +func (e *encoder) Write(p []byte) (int, os.Error) { + if e.err != nil { + return 0, e.err + } + if len(p) == 0 { + return 0, nil + } + litMask := uint32(1<>12 ^ key) & tableMask + for h, t := hash, e.table[hash]; t != invalidEntry; { + if key == t>>12 { + code = t & maxCode + continue loop + } + h = (h + 1) & tableMask + t = e.table[h] + } + // Otherwise, write the current code, and literal becomes the start of + // the next emitted code. + if e.err = e.write(e, code); e.err != nil { + return 0, e.err + } + code = literal + // Increment e.hi, the next implied code. If we run out of codes, reset + // the encoder state (including clearing the hash table) and continue. + if err := e.incHi(); err != nil { + if err == errOutOfCodes { + continue + } + e.err = err + return 0, e.err + } + // Otherwise, insert key -> e.hi into the map that e.table represents. + for { + if e.table[hash] == invalidEntry { + e.table[hash] = (key << 12) | e.hi + break + } + hash = (hash + 1) & tableMask + } + } + e.savedCode = code + return len(p), nil +} + +// Close closes the encoder, flushing any pending output. It does not close or +// flush e's underlying writer. +func (e *encoder) Close() os.Error { + if e.err != nil { + if e.err == os.EINVAL { + return nil + } + return e.err + } + // Make any future calls to Write return os.EINVAL. + e.err = os.EINVAL + // Write the savedCode if valid. + if e.savedCode != invalidCode { + if err := e.write(e, e.savedCode); err != nil { + return err + } + if err := e.incHi(); err != nil && err != errOutOfCodes { + return err + } + } + // Write the eof code. + eof := uint32(1)< 0 { + if e.write == (*encoder).writeMSB { + e.bits >>= 24 + } + if err := e.w.WriteByte(uint8(e.bits)); err != nil { + return err + } + } + return e.w.Flush() +} + +// NewWriter creates a new io.WriteCloser that satisfies writes by compressing +// the data and writing it to w. +// It is the caller's responsibility to call Close on the WriteCloser when +// finished writing. +// The number of bits to use for literal codes, litWidth, must be in the +// range [2,8] and is typically 8. +func NewWriter(w io.Writer, order Order, litWidth int) io.WriteCloser { + var write func(*encoder, uint32) os.Error + switch order { + case LSB: + write = (*encoder).writeLSB + case MSB: + write = (*encoder).writeMSB + default: + return &errWriteCloser{os.NewError("lzw: unknown order")} + } + if litWidth < 2 || 8 < litWidth { + return &errWriteCloser{fmt.Errorf("lzw: litWidth %d out of range", litWidth)} + } + bw, ok := w.(writer) + if !ok { + bw = bufio.NewWriter(w) + } + lw := uint(litWidth) + return &encoder{ + w: bw, + write: write, + width: 1 + lw, + litWidth: lw, + hi: 1<> 24) + z.scratch[1] = uint8(checksum >> 16) + z.scratch[2] = uint8(checksum >> 8) + z.scratch[3] = uint8(checksum >> 0) + _, err = w.Write(z.scratch[0:4]) + if err != nil { + return nil, err + } + } + z.w = w + z.compressor = flate.NewWriterDict(w, level, dict) + z.digest = adler32.New() + return z, nil +} + +func (z *Writer) Write(p []byte) (n int, err os.Error) { + if z.err != nil { + return 0, z.err + } + if len(p) == 0 { + return 0, nil + } + n, err = z.compressor.Write(p) + if err != nil { + z.err = err + return + } + z.digest.Write(p) + return +} + +// Flush flushes the underlying compressor. +func (z *Writer) Flush() os.Error { + if z.err != nil { + return z.err + } + z.err = z.compressor.Flush() + return z.err +} + +// Calling Close does not close the wrapped io.Writer originally passed to NewWriter. +func (z *Writer) Close() os.Error { + if z.err != nil { + return z.err + } + z.err = z.compressor.Close() + if z.err != nil { + return z.err + } + checksum := z.digest.Sum32() + // ZLIB (RFC 1950) is big-endian, unlike GZIP (RFC 1952). + z.scratch[0] = uint8(checksum >> 24) + z.scratch[1] = uint8(checksum >> 16) + z.scratch[2] = uint8(checksum >> 8) + z.scratch[3] = uint8(checksum >> 0) + _, z.err = z.w.Write(z.scratch[0:4]) + return z.err +} diff --git a/src/pkg/compress/zlib/writer_test.go b/src/pkg/compress/zlib/writer_test.go new file mode 100644 index 000000000..32f05ab68 --- /dev/null +++ b/src/pkg/compress/zlib/writer_test.go @@ -0,0 +1,144 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package zlib + +import ( + "bytes" + "fmt" + "io" + "io/ioutil" + "os" + "testing" +) + +var filenames = []string{ + "../testdata/e.txt", + "../testdata/pi.txt", +} + +var data = []string{ + "test a reasonable sized string that can be compressed", +} + +// Tests that compressing and then decompressing the given file at the given compression level and dictionary +// yields equivalent bytes to the original file. +func testFileLevelDict(t *testing.T, fn string, level int, d string) { + // Read the file, as golden output. + golden, err := os.Open(fn) + if err != nil { + t.Errorf("%s (level=%d, dict=%q): %v", fn, level, d, err) + return + } + defer golden.Close() + b0, err0 := ioutil.ReadAll(golden) + if err0 != nil { + t.Errorf("%s (level=%d, dict=%q): %v", fn, level, d, err0) + return + } + testLevelDict(t, fn, b0, level, d) +} + +func testLevelDict(t *testing.T, fn string, b0 []byte, level int, d string) { + // Make dictionary, if given. + var dict []byte + if d != "" { + dict = []byte(d) + } + + // Push data through a pipe that compresses at the write end, and decompresses at the read end. + piper, pipew := io.Pipe() + defer piper.Close() + go func() { + defer pipew.Close() + zlibw, err := NewWriterDict(pipew, level, dict) + if err != nil { + t.Errorf("%s (level=%d, dict=%q): %v", fn, level, d, err) + return + } + defer zlibw.Close() + _, err = zlibw.Write(b0) + if err == os.EPIPE { + // Fail, but do not report the error, as some other (presumably reported) error broke the pipe. + return + } + if err != nil { + t.Errorf("%s (level=%d, dict=%q): %v", fn, level, d, err) + return + } + }() + zlibr, err := NewReaderDict(piper, dict) + if err != nil { + t.Errorf("%s (level=%d, dict=%q): %v", fn, level, d, err) + return + } + defer zlibr.Close() + + // Compare the decompressed data. + b1, err1 := ioutil.ReadAll(zlibr) + if err1 != nil { + t.Errorf("%s (level=%d, dict=%q): %v", fn, level, d, err1) + return + } + if len(b0) != len(b1) { + t.Errorf("%s (level=%d, dict=%q): length mismatch %d versus %d", fn, level, d, len(b0), len(b1)) + return + } + for i := 0; i < len(b0); i++ { + if b0[i] != b1[i] { + t.Errorf("%s (level=%d, dict=%q): mismatch at %d, 0x%02x versus 0x%02x\n", fn, level, d, i, b0[i], b1[i]) + return + } + } +} + +func TestWriter(t *testing.T) { + for i, s := range data { + b := []byte(s) + tag := fmt.Sprintf("#%d", i) + testLevelDict(t, tag, b, DefaultCompression, "") + testLevelDict(t, tag, b, NoCompression, "") + for level := BestSpeed; level <= BestCompression; level++ { + testLevelDict(t, tag, b, level, "") + } + } +} + +func TestWriterBig(t *testing.T) { + for _, fn := range filenames { + testFileLevelDict(t, fn, DefaultCompression, "") + testFileLevelDict(t, fn, NoCompression, "") + for level := BestSpeed; level <= BestCompression; level++ { + testFileLevelDict(t, fn, level, "") + } + } +} + +func TestWriterDict(t *testing.T) { + const dictionary = "0123456789." + for _, fn := range filenames { + testFileLevelDict(t, fn, DefaultCompression, dictionary) + testFileLevelDict(t, fn, NoCompression, dictionary) + for level := BestSpeed; level <= BestCompression; level++ { + testFileLevelDict(t, fn, level, dictionary) + } + } +} + +func TestWriterDictIsUsed(t *testing.T) { + var input = []byte("Lorem ipsum dolor sit amet, consectetur adipisicing elit, sed do eiusmod tempor incididunt ut labore et dolore magna aliqua.") + buf := bytes.NewBuffer(nil) + compressor, err := NewWriterDict(buf, BestCompression, input) + if err != nil { + t.Errorf("error in NewWriterDict: %s", err) + return + } + compressor.Write(input) + compressor.Close() + const expectedMaxSize = 25 + output := buf.Bytes() + if len(output) > expectedMaxSize { + t.Errorf("result too large (got %d, want <= %d bytes). Is the dictionary being used?", len(output), expectedMaxSize) + } +} diff --git a/src/pkg/container/heap/Makefile b/src/pkg/container/heap/Makefile new file mode 100644 index 000000000..4291d1122 --- /dev/null +++ b/src/pkg/container/heap/Makefile @@ -0,0 +1,11 @@ +# Copyright 2009 The Go Authors. All rights reserved. +# Use of this source code is governed by a BSD-style +# license that can be found in the LICENSE file. + +include ../../../Make.inc + +TARG=container/heap +GOFILES=\ + heap.go\ + +include ../../../Make.pkg diff --git a/src/pkg/container/heap/heap.go b/src/pkg/container/heap/heap.go new file mode 100644 index 000000000..2dfe5b43c --- /dev/null +++ b/src/pkg/container/heap/heap.go @@ -0,0 +1,96 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Package heap provides heap operations for any type that implements +// heap.Interface. +// +package heap + +import "sort" + +// Any type that implements heap.Interface may be used as a +// min-heap with the following invariants (established after +// Init has been called): +// +// !h.Less(j, i) for 0 <= i < h.Len() and j = 2*i+1 or 2*i+2 and j < h.Len() +// +type Interface interface { + sort.Interface + Push(x interface{}) + Pop() interface{} +} + +// A heap must be initialized before any of the heap operations +// can be used. Init is idempotent with respect to the heap invariants +// and may be called whenever the heap invariants may have been invalidated. +// Its complexity is O(n) where n = h.Len(). +// +func Init(h Interface) { + // heapify + n := h.Len() + for i := n/2 - 1; i >= 0; i-- { + down(h, i, n) + } +} + +// Push pushes the element x onto the heap. The complexity is +// O(log(n)) where n = h.Len(). +// +func Push(h Interface, x interface{}) { + h.Push(x) + up(h, h.Len()-1) +} + +// Pop removes the minimum element (according to Less) from the heap +// and returns it. The complexity is O(log(n)) where n = h.Len(). +// Same as Remove(h, 0). +// +func Pop(h Interface) interface{} { + n := h.Len() - 1 + h.Swap(0, n) + down(h, 0, n) + return h.Pop() +} + +// Remove removes the element at index i from the heap. +// The complexity is O(log(n)) where n = h.Len(). +// +func Remove(h Interface, i int) interface{} { + n := h.Len() - 1 + if n != i { + h.Swap(i, n) + down(h, i, n) + up(h, i) + } + return h.Pop() +} + +func up(h Interface, j int) { + for { + i := (j - 1) / 2 // parent + if i == j || h.Less(i, j) { + break + } + h.Swap(i, j) + j = i + } +} + +func down(h Interface, i, n int) { + for { + j1 := 2*i + 1 + if j1 >= n { + break + } + j := j1 // left child + if j2 := j1 + 1; j2 < n && !h.Less(j1, j2) { + j = j2 // = 2*i + 2 // right child + } + if h.Less(i, j) { + break + } + h.Swap(i, j) + i = j + } +} diff --git a/src/pkg/container/heap/heap_test.go b/src/pkg/container/heap/heap_test.go new file mode 100644 index 000000000..c5c1f76e1 --- /dev/null +++ b/src/pkg/container/heap/heap_test.go @@ -0,0 +1,158 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package heap_test + +import ( + "testing" + "container/vector" + . "container/heap" +) + +type myHeap struct { + // A vector.Vector implements sort.Interface except for Less, + // and it implements Push and Pop as required for heap.Interface. + vector.Vector +} + +func (h *myHeap) Less(i, j int) bool { return h.At(i).(int) < h.At(j).(int) } + +func (h *myHeap) verify(t *testing.T, i int) { + n := h.Len() + j1 := 2*i + 1 + j2 := 2*i + 2 + if j1 < n { + if h.Less(j1, i) { + t.Errorf("heap invariant invalidated [%d] = %d > [%d] = %d", i, h.At(i), j1, h.At(j1)) + return + } + h.verify(t, j1) + } + if j2 < n { + if h.Less(j2, i) { + t.Errorf("heap invariant invalidated [%d] = %d > [%d] = %d", i, h.At(i), j1, h.At(j2)) + return + } + h.verify(t, j2) + } +} + +func TestInit0(t *testing.T) { + h := new(myHeap) + for i := 20; i > 0; i-- { + h.Push(0) // all elements are the same + } + Init(h) + h.verify(t, 0) + + for i := 1; h.Len() > 0; i++ { + x := Pop(h).(int) + h.verify(t, 0) + if x != 0 { + t.Errorf("%d.th pop got %d; want %d", i, x, 0) + } + } +} + +func TestInit1(t *testing.T) { + h := new(myHeap) + for i := 20; i > 0; i-- { + h.Push(i) // all elements are different + } + Init(h) + h.verify(t, 0) + + for i := 1; h.Len() > 0; i++ { + x := Pop(h).(int) + h.verify(t, 0) + if x != i { + t.Errorf("%d.th pop got %d; want %d", i, x, i) + } + } +} + +func Test(t *testing.T) { + h := new(myHeap) + h.verify(t, 0) + + for i := 20; i > 10; i-- { + h.Push(i) + } + Init(h) + h.verify(t, 0) + + for i := 10; i > 0; i-- { + Push(h, i) + h.verify(t, 0) + } + + for i := 1; h.Len() > 0; i++ { + x := Pop(h).(int) + if i < 20 { + Push(h, 20+i) + } + h.verify(t, 0) + if x != i { + t.Errorf("%d.th pop got %d; want %d", i, x, i) + } + } +} + +func TestRemove0(t *testing.T) { + h := new(myHeap) + for i := 0; i < 10; i++ { + h.Push(i) + } + h.verify(t, 0) + + for h.Len() > 0 { + i := h.Len() - 1 + x := Remove(h, i).(int) + if x != i { + t.Errorf("Remove(%d) got %d; want %d", i, x, i) + } + h.verify(t, 0) + } +} + +func TestRemove1(t *testing.T) { + h := new(myHeap) + for i := 0; i < 10; i++ { + h.Push(i) + } + h.verify(t, 0) + + for i := 0; h.Len() > 0; i++ { + x := Remove(h, 0).(int) + if x != i { + t.Errorf("Remove(0) got %d; want %d", x, i) + } + h.verify(t, 0) + } +} + +func TestRemove2(t *testing.T) { + N := 10 + + h := new(myHeap) + for i := 0; i < N; i++ { + h.Push(i) + } + h.verify(t, 0) + + m := make(map[int]bool) + for h.Len() > 0 { + m[Remove(h, (h.Len()-1)/2).(int)] = true + h.verify(t, 0) + } + + if len(m) != N { + t.Errorf("len(m) = %d; want %d", len(m), N) + } + for i := 0; i < len(m); i++ { + if !m[i] { + t.Errorf("m[%d] doesn't exist", i) + } + } +} diff --git a/src/pkg/container/list/Makefile b/src/pkg/container/list/Makefile new file mode 100644 index 000000000..7fcd5f99a --- /dev/null +++ b/src/pkg/container/list/Makefile @@ -0,0 +1,11 @@ +# Copyright 2009 The Go Authors. All rights reserved. +# Use of this source code is governed by a BSD-style +# license that can be found in the LICENSE file. + +include ../../../Make.inc + +TARG=container/list +GOFILES=\ + list.go\ + +include ../../../Make.pkg diff --git a/src/pkg/container/list/list.go b/src/pkg/container/list/list.go new file mode 100644 index 000000000..a3fd4b39f --- /dev/null +++ b/src/pkg/container/list/list.go @@ -0,0 +1,211 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Package list implements a doubly linked list. +// +// To iterate over a list (where l is a *List): +// for e := l.Front(); e != nil; e = e.Next() { +// // do something with e.Value +// } +// +package list + +// Element is an element in the linked list. +type Element struct { + // Next and previous pointers in the doubly-linked list of elements. + // The front of the list has prev = nil, and the back has next = nil. + next, prev *Element + + // The list to which this element belongs. + list *List + + // The contents of this list element. + Value interface{} +} + +// Next returns the next list element or nil. +func (e *Element) Next() *Element { return e.next } + +// Prev returns the previous list element or nil. +func (e *Element) Prev() *Element { return e.prev } + +// List represents a doubly linked list. +// The zero value for List is an empty list ready to use. +type List struct { + front, back *Element + len int +} + +// Init initializes or clears a List. +func (l *List) Init() *List { + l.front = nil + l.back = nil + l.len = 0 + return l +} + +// New returns an initialized list. +func New() *List { return new(List) } + +// Front returns the first element in the list. +func (l *List) Front() *Element { return l.front } + +// Back returns the last element in the list. +func (l *List) Back() *Element { return l.back } + +// Remove removes the element from the list +// and returns its Value. +func (l *List) Remove(e *Element) interface{} { + l.remove(e) + e.list = nil // do what remove does not + return e.Value +} + +// remove the element from the list, but do not clear the Element's list field. +// This is so that other List methods may use remove when relocating Elements +// without needing to restore the list field. +func (l *List) remove(e *Element) { + if e.list != l { + return + } + if e.prev == nil { + l.front = e.next + } else { + e.prev.next = e.next + } + if e.next == nil { + l.back = e.prev + } else { + e.next.prev = e.prev + } + + e.prev = nil + e.next = nil + l.len-- +} + +func (l *List) insertBefore(e *Element, mark *Element) { + if mark.prev == nil { + // new front of the list + l.front = e + } else { + mark.prev.next = e + } + e.prev = mark.prev + mark.prev = e + e.next = mark + l.len++ +} + +func (l *List) insertAfter(e *Element, mark *Element) { + if mark.next == nil { + // new back of the list + l.back = e + } else { + mark.next.prev = e + } + e.next = mark.next + mark.next = e + e.prev = mark + l.len++ +} + +func (l *List) insertFront(e *Element) { + if l.front == nil { + // empty list + l.front, l.back = e, e + e.prev, e.next = nil, nil + l.len = 1 + return + } + l.insertBefore(e, l.front) +} + +func (l *List) insertBack(e *Element) { + if l.back == nil { + // empty list + l.front, l.back = e, e + e.prev, e.next = nil, nil + l.len = 1 + return + } + l.insertAfter(e, l.back) +} + +// PushFront inserts the value at the front of the list and returns a new Element containing the value. +func (l *List) PushFront(value interface{}) *Element { + e := &Element{nil, nil, l, value} + l.insertFront(e) + return e +} + +// PushBack inserts the value at the back of the list and returns a new Element containing the value. +func (l *List) PushBack(value interface{}) *Element { + e := &Element{nil, nil, l, value} + l.insertBack(e) + return e +} + +// InsertBefore inserts the value immediately before mark and returns a new Element containing the value. +func (l *List) InsertBefore(value interface{}, mark *Element) *Element { + if mark.list != l { + return nil + } + e := &Element{nil, nil, l, value} + l.insertBefore(e, mark) + return e +} + +// InsertAfter inserts the value immediately after mark and returns a new Element containing the value. +func (l *List) InsertAfter(value interface{}, mark *Element) *Element { + if mark.list != l { + return nil + } + e := &Element{nil, nil, l, value} + l.insertAfter(e, mark) + return e +} + +// MoveToFront moves the element to the front of the list. +func (l *List) MoveToFront(e *Element) { + if e.list != l || l.front == e { + return + } + l.remove(e) + l.insertFront(e) +} + +// MoveToBack moves the element to the back of the list. +func (l *List) MoveToBack(e *Element) { + if e.list != l || l.back == e { + return + } + l.remove(e) + l.insertBack(e) +} + +// Len returns the number of elements in the list. +func (l *List) Len() int { return l.len } + +// PushBackList inserts each element of ol at the back of the list. +func (l *List) PushBackList(ol *List) { + last := ol.Back() + for e := ol.Front(); e != nil; e = e.Next() { + l.PushBack(e.Value) + if e == last { + break + } + } +} + +// PushFrontList inserts each element of ol at the front of the list. The ordering of the passed list is preserved. +func (l *List) PushFrontList(ol *List) { + first := ol.Front() + for e := ol.Back(); e != nil; e = e.Prev() { + l.PushFront(e.Value) + if e == first { + break + } + } +} diff --git a/src/pkg/container/list/list_test.go b/src/pkg/container/list/list_test.go new file mode 100644 index 000000000..1d44ff84e --- /dev/null +++ b/src/pkg/container/list/list_test.go @@ -0,0 +1,209 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package list + +import ( + "testing" +) + +func checkListPointers(t *testing.T, l *List, es []*Element) { + if len(es) == 0 { + if l.front != nil || l.back != nil { + t.Errorf("l.front/l.back = %v/%v should be nil/nil", l.front, l.back) + } + return + } + + if l.front != es[0] { + t.Errorf("l.front = %v, want %v", l.front, es[0]) + } + if last := es[len(es)-1]; l.back != last { + t.Errorf("l.back = %v, want %v", l.back, last) + } + + for i, e := range es { + var e_prev, e_next *Element = nil, nil + if i > 0 { + e_prev = es[i-1] + } + if i < len(es)-1 { + e_next = es[i+1] + } + if e.prev != e_prev { + t.Errorf("elt #%d (%v) has prev=%v, want %v", i, e, e.prev, e_prev) + } + if e.next != e_next { + t.Errorf("elt #%d (%v) has next=%v, want %v", i, e, e.next, e_next) + } + } +} + +func checkListLen(t *testing.T, l *List, n int) { + if an := l.Len(); an != n { + t.Errorf("l.Len() = %d, want %d", an, n) + } +} + +func TestList(t *testing.T) { + l := New() + checkListPointers(t, l, []*Element{}) + checkListLen(t, l, 0) + + // Single element list + e := l.PushFront("a") + checkListLen(t, l, 1) + checkListPointers(t, l, []*Element{e}) + l.MoveToFront(e) + checkListPointers(t, l, []*Element{e}) + l.MoveToBack(e) + checkListPointers(t, l, []*Element{e}) + checkListLen(t, l, 1) + l.Remove(e) + checkListPointers(t, l, []*Element{}) + checkListLen(t, l, 0) + + // Bigger list + e2 := l.PushFront(2) + e1 := l.PushFront(1) + e3 := l.PushBack(3) + e4 := l.PushBack("banana") + checkListPointers(t, l, []*Element{e1, e2, e3, e4}) + checkListLen(t, l, 4) + + l.Remove(e2) + checkListPointers(t, l, []*Element{e1, e3, e4}) + checkListLen(t, l, 3) + + l.MoveToFront(e3) // move from middle + checkListPointers(t, l, []*Element{e3, e1, e4}) + + l.MoveToFront(e1) + l.MoveToBack(e3) // move from middle + checkListPointers(t, l, []*Element{e1, e4, e3}) + + l.MoveToFront(e3) // move from back + checkListPointers(t, l, []*Element{e3, e1, e4}) + l.MoveToFront(e3) // should be no-op + checkListPointers(t, l, []*Element{e3, e1, e4}) + + l.MoveToBack(e3) // move from front + checkListPointers(t, l, []*Element{e1, e4, e3}) + l.MoveToBack(e3) // should be no-op + checkListPointers(t, l, []*Element{e1, e4, e3}) + + e2 = l.InsertBefore(2, e1) // insert before front + checkListPointers(t, l, []*Element{e2, e1, e4, e3}) + l.Remove(e2) + e2 = l.InsertBefore(2, e4) // insert before middle + checkListPointers(t, l, []*Element{e1, e2, e4, e3}) + l.Remove(e2) + e2 = l.InsertBefore(2, e3) // insert before back + checkListPointers(t, l, []*Element{e1, e4, e2, e3}) + l.Remove(e2) + + e2 = l.InsertAfter(2, e1) // insert after front + checkListPointers(t, l, []*Element{e1, e2, e4, e3}) + l.Remove(e2) + e2 = l.InsertAfter(2, e4) // insert after middle + checkListPointers(t, l, []*Element{e1, e4, e2, e3}) + l.Remove(e2) + e2 = l.InsertAfter(2, e3) // insert after back + checkListPointers(t, l, []*Element{e1, e4, e3, e2}) + l.Remove(e2) + + // Check standard iteration. + sum := 0 + for e := l.Front(); e != nil; e = e.Next() { + if i, ok := e.Value.(int); ok { + sum += i + } + } + if sum != 4 { + t.Errorf("sum over l.Iter() = %d, want 4", sum) + } + + // Clear all elements by iterating + var next *Element + for e := l.Front(); e != nil; e = next { + next = e.Next() + l.Remove(e) + } + checkListPointers(t, l, []*Element{}) + checkListLen(t, l, 0) +} + +func checkList(t *testing.T, l *List, es []interface{}) { + if l.Len() != len(es) { + t.Errorf("list has len=%v, want %v", l.Len(), len(es)) + return + } + i := 0 + for e := l.Front(); e != nil; e = e.Next() { + le := e.Value.(int) + if le != es[i] { + t.Errorf("elt #%d has value=%v, want %v", i, le, es[i]) + } + i++ + } +} + +func TestExtending(t *testing.T) { + l1 := New() + l2 := New() + + l1.PushBack(1) + l1.PushBack(2) + l1.PushBack(3) + + l2.PushBack(4) + l2.PushBack(5) + + l3 := New() + l3.PushBackList(l1) + checkList(t, l3, []interface{}{1, 2, 3}) + l3.PushBackList(l2) + checkList(t, l3, []interface{}{1, 2, 3, 4, 5}) + + l3 = New() + l3.PushFrontList(l2) + checkList(t, l3, []interface{}{4, 5}) + l3.PushFrontList(l1) + checkList(t, l3, []interface{}{1, 2, 3, 4, 5}) + + checkList(t, l1, []interface{}{1, 2, 3}) + checkList(t, l2, []interface{}{4, 5}) + + l3 = New() + l3.PushBackList(l1) + checkList(t, l3, []interface{}{1, 2, 3}) + l3.PushBackList(l3) + checkList(t, l3, []interface{}{1, 2, 3, 1, 2, 3}) + + l3 = New() + l3.PushFrontList(l1) + checkList(t, l3, []interface{}{1, 2, 3}) + l3.PushFrontList(l3) + checkList(t, l3, []interface{}{1, 2, 3, 1, 2, 3}) + + l3 = New() + l1.PushBackList(l3) + checkList(t, l1, []interface{}{1, 2, 3}) + l1.PushFrontList(l3) + checkList(t, l1, []interface{}{1, 2, 3}) +} + +func TestRemove(t *testing.T) { + l := New() + e1 := l.PushBack(1) + e2 := l.PushBack(2) + checkListPointers(t, l, []*Element{e1, e2}) + e := l.Front() + l.Remove(e) + checkListPointers(t, l, []*Element{e2}) + checkListLen(t, l, 1) + l.Remove(e) + checkListPointers(t, l, []*Element{e2}) + checkListLen(t, l, 1) +} diff --git a/src/pkg/container/ring/Makefile b/src/pkg/container/ring/Makefile new file mode 100644 index 000000000..fb0900774 --- /dev/null +++ b/src/pkg/container/ring/Makefile @@ -0,0 +1,11 @@ +# Copyright 2009 The Go Authors. All rights reserved. +# Use of this source code is governed by a BSD-style +# license that can be found in the LICENSE file. + +include ../../../Make.inc + +TARG=container/ring +GOFILES=\ + ring.go\ + +include ../../../Make.pkg diff --git a/src/pkg/container/ring/ring.go b/src/pkg/container/ring/ring.go new file mode 100644 index 000000000..1d96918d3 --- /dev/null +++ b/src/pkg/container/ring/ring.go @@ -0,0 +1,141 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Package ring implements operations on circular lists. +package ring + +// A Ring is an element of a circular list, or ring. +// Rings do not have a beginning or end; a pointer to any ring element +// serves as reference to the entire ring. Empty rings are represented +// as nil Ring pointers. The zero value for a Ring is a one-element +// ring with a nil Value. +// +type Ring struct { + next, prev *Ring + Value interface{} // for use by client; untouched by this library +} + +func (r *Ring) init() *Ring { + r.next = r + r.prev = r + return r +} + +// Next returns the next ring element. r must not be empty. +func (r *Ring) Next() *Ring { + if r.next == nil { + return r.init() + } + return r.next +} + +// Prev returns the previous ring element. r must not be empty. +func (r *Ring) Prev() *Ring { + if r.next == nil { + return r.init() + } + return r.prev +} + +// Move moves n % r.Len() elements backward (n < 0) or forward (n >= 0) +// in the ring and returns that ring element. r must not be empty. +// +func (r *Ring) Move(n int) *Ring { + if r.next == nil { + return r.init() + } + switch { + case n < 0: + for ; n < 0; n++ { + r = r.prev + } + case n > 0: + for ; n > 0; n-- { + r = r.next + } + } + return r +} + +// New creates a ring of n elements. +func New(n int) *Ring { + if n <= 0 { + return nil + } + r := new(Ring) + p := r + for i := 1; i < n; i++ { + p.next = &Ring{prev: p} + p = p.next + } + p.next = r + r.prev = p + return r +} + +// Link connects ring r with with ring s such that r.Next() +// becomes s and returns the original value for r.Next(). +// r must not be empty. +// +// If r and s point to the same ring, linking +// them removes the elements between r and s from the ring. +// The removed elements form a subring and the result is a +// reference to that subring (if no elements were removed, +// the result is still the original value for r.Next(), +// and not nil). +// +// If r and s point to different rings, linking +// them creates a single ring with the elements of s inserted +// after r. The result points to the element following the +// last element of s after insertion. +// +func (r *Ring) Link(s *Ring) *Ring { + n := r.Next() + if s != nil { + p := s.Prev() + // Note: Cannot use multiple assignment because + // evaluation order of LHS is not specified. + r.next = s + s.prev = r + n.prev = p + p.next = n + } + return n +} + +// Unlink removes n % r.Len() elements from the ring r, starting +// at r.Next(). If n % r.Len() == 0, r remains unchanged. +// The result is the removed subring. r must not be empty. +// +func (r *Ring) Unlink(n int) *Ring { + if n <= 0 { + return nil + } + return r.Link(r.Move(n + 1)) +} + +// Len computes the number of elements in ring r. +// It executes in time proportional to the number of elements. +// +func (r *Ring) Len() int { + n := 0 + if r != nil { + n = 1 + for p := r.Next(); p != r; p = p.next { + n++ + } + } + return n +} + +// Do calls function f on each element of the ring, in forward order. +// The behavior of Do is undefined if f changes *r. +func (r *Ring) Do(f func(interface{})) { + if r != nil { + f(r.Value) + for p := r.Next(); p != r; p = p.next { + f(p.Value) + } + } +} diff --git a/src/pkg/container/ring/ring_test.go b/src/pkg/container/ring/ring_test.go new file mode 100644 index 000000000..099d92b25 --- /dev/null +++ b/src/pkg/container/ring/ring_test.go @@ -0,0 +1,220 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package ring + +import ( + "fmt" + "testing" +) + +// For debugging - keep around. +func dump(r *Ring) { + if r == nil { + fmt.Println("empty") + return + } + i, n := 0, r.Len() + for p := r; i < n; p = p.next { + fmt.Printf("%4d: %p = {<- %p | %p ->}\n", i, p, p.prev, p.next) + i++ + } + fmt.Println() +} + +func verify(t *testing.T, r *Ring, N int, sum int) { + // Len + n := r.Len() + if n != N { + t.Errorf("r.Len() == %d; expected %d", n, N) + } + + // iteration + n = 0 + s := 0 + r.Do(func(p interface{}) { + n++ + if p != nil { + s += p.(int) + } + }) + if n != N { + t.Errorf("number of forward iterations == %d; expected %d", n, N) + } + if sum >= 0 && s != sum { + t.Errorf("forward ring sum = %d; expected %d", s, sum) + } + + if r == nil { + return + } + + // connections + if r.next != nil { + var p *Ring // previous element + for q := r; p == nil || q != r; q = q.next { + if p != nil && p != q.prev { + t.Errorf("prev = %p, expected q.prev = %p\n", p, q.prev) + } + p = q + } + if p != r.prev { + t.Errorf("prev = %p, expected r.prev = %p\n", p, r.prev) + } + } + + // Next, Prev + if r.Next() != r.next { + t.Errorf("r.Next() != r.next") + } + if r.Prev() != r.prev { + t.Errorf("r.Prev() != r.prev") + } + + // Move + if r.Move(0) != r { + t.Errorf("r.Move(0) != r") + } + if r.Move(N) != r { + t.Errorf("r.Move(%d) != r", N) + } + if r.Move(-N) != r { + t.Errorf("r.Move(%d) != r", -N) + } + for i := 0; i < 10; i++ { + ni := N + i + mi := ni % N + if r.Move(ni) != r.Move(mi) { + t.Errorf("r.Move(%d) != r.Move(%d)", ni, mi) + } + if r.Move(-ni) != r.Move(-mi) { + t.Errorf("r.Move(%d) != r.Move(%d)", -ni, -mi) + } + } +} + +func TestCornerCases(t *testing.T) { + var ( + r0 *Ring + r1 Ring + ) + // Basics + verify(t, r0, 0, 0) + verify(t, &r1, 1, 0) + // Insert + r1.Link(r0) + verify(t, r0, 0, 0) + verify(t, &r1, 1, 0) + // Insert + r1.Link(r0) + verify(t, r0, 0, 0) + verify(t, &r1, 1, 0) + // Unlink + r1.Unlink(0) + verify(t, &r1, 1, 0) +} + +func makeN(n int) *Ring { + r := New(n) + for i := 1; i <= n; i++ { + r.Value = i + r = r.Next() + } + return r +} + +func sumN(n int) int { return (n*n + n) / 2 } + +func TestNew(t *testing.T) { + for i := 0; i < 10; i++ { + r := New(i) + verify(t, r, i, -1) + } + for i := 0; i < 10; i++ { + r := makeN(i) + verify(t, r, i, sumN(i)) + } +} + +func TestLink1(t *testing.T) { + r1a := makeN(1) + var r1b Ring + r2a := r1a.Link(&r1b) + verify(t, r2a, 2, 1) + if r2a != r1a { + t.Errorf("a) 2-element link failed") + } + + r2b := r2a.Link(r2a.Next()) + verify(t, r2b, 2, 1) + if r2b != r2a.Next() { + t.Errorf("b) 2-element link failed") + } + + r1c := r2b.Link(r2b) + verify(t, r1c, 1, 1) + verify(t, r2b, 1, 0) +} + +func TestLink2(t *testing.T) { + var r0 *Ring + r1a := &Ring{Value: 42} + r1b := &Ring{Value: 77} + r10 := makeN(10) + + r1a.Link(r0) + verify(t, r1a, 1, 42) + + r1a.Link(r1b) + verify(t, r1a, 2, 42+77) + + r10.Link(r0) + verify(t, r10, 10, sumN(10)) + + r10.Link(r1a) + verify(t, r10, 12, sumN(10)+42+77) +} + +func TestLink3(t *testing.T) { + var r Ring + n := 1 + for i := 1; i < 100; i++ { + n += i + verify(t, r.Link(New(i)), n, -1) + } +} + +func TestUnlink(t *testing.T) { + r10 := makeN(10) + s10 := r10.Move(6) + + sum10 := sumN(10) + + verify(t, r10, 10, sum10) + verify(t, s10, 10, sum10) + + r0 := r10.Unlink(0) + verify(t, r0, 0, 0) + + r1 := r10.Unlink(1) + verify(t, r1, 1, 2) + verify(t, r10, 9, sum10-2) + + r9 := r10.Unlink(9) + verify(t, r9, 9, sum10-2) + verify(t, r10, 9, sum10-2) +} + +func TestLinkUnlink(t *testing.T) { + for i := 1; i < 4; i++ { + ri := New(i) + for j := 0; j < i; j++ { + rj := ri.Unlink(j) + verify(t, rj, j, -1) + verify(t, ri, i-j, -1) + ri.Link(rj) + verify(t, ri, i, -1) + } + } +} diff --git a/src/pkg/container/vector/Makefile b/src/pkg/container/vector/Makefile new file mode 100644 index 000000000..f6b50156f --- /dev/null +++ b/src/pkg/container/vector/Makefile @@ -0,0 +1,69 @@ +# Copyright 2009 The Go Authors. All rights reserved. +# Use of this source code is governed by a BSD-style +# license that can be found in the LICENSE file. + +include ../../../Make.inc + +TARG=container/vector +GOFILES=\ + defs.go\ + intvector.go\ + stringvector.go\ + vector.go\ + +include ../../../Make.pkg + +generate: vector.go vector_test.go + < vector.go cat\ + | gofmt -r='Vector -> IntVector'\ + | gofmt -r='interface{} -> int'\ + > intvector.go\ + + < vector.go cat\ + | gofmt -r='Vector -> StringVector'\ + | gofmt -r='interface{} -> string'\ + > stringvector.go\ + + < vector_test.go cat\ + | gofmt -r='Vector -> IntVector'\ + | gofmt -r='zero -> intzero'\ + | gofmt -r='elem2Value -> elem2IntValue'\ + | gofmt -r='intf2Value -> intf2IntValue'\ + | gofmt -r='int2Value -> int2IntValue'\ + | gofmt -r='TestZeroLen -> TestIntZeroLen'\ + | gofmt -r='TestResize -> TestIntResize'\ + | gofmt -r='TestResize2 -> TestIntResize2'\ + | gofmt -r='checkZero -> checkIntZero'\ + | gofmt -r='TestTrailingElements -> TestIntTrailingElements'\ + | gofmt -r='TestAccess -> TestIntAccess'\ + | gofmt -r='TestInsertDeleteClear -> TestIntInsertDeleteClear'\ + | gofmt -r='verify_slice -> verify_sliceInt'\ + | gofmt -r='verify_pattern -> verify_patternInt'\ + | gofmt -r='make_vector -> make_vectorInt'\ + | gofmt -r='TestInsertVector -> TestIntInsertVector'\ + | gofmt -r='TestDo -> TestIntDo'\ + | gofmt -r='TestVectorCopy -> TestIntVectorCopy'\ + | gofmt -r='interface{} -> int'\ + > intvector_test.go\ + + < vector_test.go cat\ + | gofmt -r='Vector -> StringVector'\ + | gofmt -r='zero -> strzero'\ + | gofmt -r='int2Value -> int2StrValue'\ + | gofmt -r='intf2Value -> intf2StrValue'\ + | gofmt -r='elem2Value -> elem2StrValue'\ + | gofmt -r='TestZeroLen -> TestStrZeroLen'\ + | gofmt -r='TestResize -> TestStrResize'\ + | gofmt -r='TestResize2 -> TestStrResize2'\ + | gofmt -r='checkZero -> checkStrZero'\ + | gofmt -r='TestTrailingElements -> TestStrTrailingElements'\ + | gofmt -r='TestAccess -> TestStrAccess'\ + | gofmt -r='TestInsertDeleteClear -> TestStrInsertDeleteClear'\ + | gofmt -r='verify_slice -> verify_sliceStr'\ + | gofmt -r='verify_pattern -> verify_patternStr'\ + | gofmt -r='make_vector -> make_vectorStr'\ + | gofmt -r='TestInsertVector -> TestStrInsertVector'\ + | gofmt -r='TestDo -> TestStrDo'\ + | gofmt -r='TestVectorCopy -> TestStrVectorCopy'\ + | gofmt -r='interface{} -> string'\ + > stringvector_test.go diff --git a/src/pkg/container/vector/defs.go b/src/pkg/container/vector/defs.go new file mode 100644 index 000000000..6d6b2ac81 --- /dev/null +++ b/src/pkg/container/vector/defs.go @@ -0,0 +1,43 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Package vector implements containers for managing sequences of elements. +// Vectors grow and shrink dynamically as necessary. +package vector + +// Vector is a container for numbered sequences of elements of type interface{}. +// A vector's length and capacity adjusts automatically as necessary. +// The zero value for Vector is an empty vector ready to use. +type Vector []interface{} + +// IntVector is a container for numbered sequences of elements of type int. +// A vector's length and capacity adjusts automatically as necessary. +// The zero value for IntVector is an empty vector ready to use. +type IntVector []int + +// StringVector is a container for numbered sequences of elements of type string. +// A vector's length and capacity adjusts automatically as necessary. +// The zero value for StringVector is an empty vector ready to use. +type StringVector []string + +// Initial underlying array size +const initialSize = 8 + +// Partial sort.Interface support + +// LessInterface provides partial support of the sort.Interface. +type LessInterface interface { + Less(y interface{}) bool +} + +// Less returns a boolean denoting whether the i'th element is less than the j'th element. +func (p *Vector) Less(i, j int) bool { return (*p)[i].(LessInterface).Less((*p)[j]) } + +// sort.Interface support + +// Less returns a boolean denoting whether the i'th element is less than the j'th element. +func (p *IntVector) Less(i, j int) bool { return (*p)[i] < (*p)[j] } + +// Less returns a boolean denoting whether the i'th element is less than the j'th element. +func (p *StringVector) Less(i, j int) bool { return (*p)[i] < (*p)[j] } diff --git a/src/pkg/container/vector/intvector.go b/src/pkg/container/vector/intvector.go new file mode 100644 index 000000000..aa88cfeb3 --- /dev/null +++ b/src/pkg/container/vector/intvector.go @@ -0,0 +1,188 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// CAUTION: If this file is not vector.go, it was generated +// automatically from vector.go - DO NOT EDIT in that case! + +package vector + +func (p *IntVector) realloc(length, capacity int) (b []int) { + if capacity < initialSize { + capacity = initialSize + } + if capacity < length { + capacity = length + } + b = make(IntVector, length, capacity) + copy(b, *p) + *p = b + return +} + +// Insert n elements at position i. +func (p *IntVector) Expand(i, n int) { + a := *p + + // make sure we have enough space + len0 := len(a) + len1 := len0 + n + if len1 <= cap(a) { + // enough space - just expand + a = a[0:len1] + } else { + // not enough space - double capacity + capb := cap(a) * 2 + if capb < len1 { + // still not enough - use required length + capb = len1 + } + // capb >= len1 + a = p.realloc(len1, capb) + } + + // make a hole + for j := len0 - 1; j >= i; j-- { + a[j+n] = a[j] + } + + *p = a +} + +// Insert n elements at the end of a vector. +func (p *IntVector) Extend(n int) { p.Expand(len(*p), n) } + +// Resize changes the length and capacity of a vector. +// If the new length is shorter than the current length, Resize discards +// trailing elements. If the new length is longer than the current length, +// Resize adds the respective zero values for the additional elements. The capacity +// parameter is ignored unless the new length or capacity is longer than the current +// capacity. The resized vector's capacity may be larger than the requested capacity. +func (p *IntVector) Resize(length, capacity int) *IntVector { + a := *p + + if length > cap(a) || capacity > cap(a) { + // not enough space or larger capacity requested explicitly + a = p.realloc(length, capacity) + } else if length < len(a) { + // clear trailing elements + for i := range a[length:] { + var zero int + a[length+i] = zero + } + } + + *p = a[0:length] + return p +} + +// Len returns the number of elements in the vector. +// Same as len(*p). +func (p *IntVector) Len() int { return len(*p) } + +// Cap returns the capacity of the vector; that is, the +// maximum length the vector can grow without resizing. +// Same as cap(*p). +func (p *IntVector) Cap() int { return cap(*p) } + +// At returns the i'th element of the vector. +func (p *IntVector) At(i int) int { return (*p)[i] } + +// Set sets the i'th element of the vector to value x. +func (p *IntVector) Set(i int, x int) { (*p)[i] = x } + +// Last returns the element in the vector of highest index. +func (p *IntVector) Last() int { return (*p)[len(*p)-1] } + +// Copy makes a copy of the vector and returns it. +func (p *IntVector) Copy() IntVector { + arr := make(IntVector, len(*p)) + copy(arr, *p) + return arr +} + +// Insert inserts into the vector an element of value x before +// the current element at index i. +func (p *IntVector) Insert(i int, x int) { + p.Expand(i, 1) + (*p)[i] = x +} + +// Delete deletes the i'th element of the vector. The gap is closed so the old +// element at index i+1 has index i afterwards. +func (p *IntVector) Delete(i int) { + a := *p + n := len(a) + + copy(a[i:n-1], a[i+1:n]) + var zero int + a[n-1] = zero // support GC, zero out entry + *p = a[0 : n-1] +} + +// InsertVector inserts into the vector the contents of the vector +// x such that the 0th element of x appears at index i after insertion. +func (p *IntVector) InsertVector(i int, x *IntVector) { + b := *x + + p.Expand(i, len(b)) + copy((*p)[i:i+len(b)], b) +} + +// Cut deletes elements i through j-1, inclusive. +func (p *IntVector) Cut(i, j int) { + a := *p + n := len(a) + m := n - (j - i) + + copy(a[i:m], a[j:n]) + for k := m; k < n; k++ { //TODO(bflm) don't zero out the elements unless it's a Vector. + var zero int + a[k] = zero // support GC, zero out entries + } + + *p = a[0:m] +} + +// Slice returns a new sub-vector by slicing the old one to extract slice [i:j]. +// The elements are copied. The original vector is unchanged. +func (p *IntVector) Slice(i, j int) *IntVector { + var s IntVector + s.realloc(j-i, 0) // will fail in Init() if j < i + copy(s, (*p)[i:j]) + return &s +} + +// Convenience wrappers + +// Push appends x to the end of the vector. +func (p *IntVector) Push(x int) { p.Insert(len(*p), x) } + +// Pop deletes the last element of the vector. +func (p *IntVector) Pop() int { + a := *p + + i := len(a) - 1 + x := a[i] + var zero int + a[i] = zero // support GC, zero out entry + *p = a[0:i] + return x +} + +// AppendVector appends the entire vector x to the end of this vector. +func (p *IntVector) AppendVector(x *IntVector) { p.InsertVector(len(*p), x) } + +// Swap exchanges the elements at indexes i and j. +func (p *IntVector) Swap(i, j int) { + a := *p + a[i], a[j] = a[j], a[i] +} + +// Do calls function f for each element of the vector, in order. +// The behavior of Do is undefined if f changes *p. +func (p *IntVector) Do(f func(elem int)) { + for _, e := range *p { + f(e) + } +} diff --git a/src/pkg/container/vector/intvector_test.go b/src/pkg/container/vector/intvector_test.go new file mode 100644 index 000000000..b825af912 --- /dev/null +++ b/src/pkg/container/vector/intvector_test.go @@ -0,0 +1,331 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// CAUTION: If this file is not vector_test.go, it was generated +// automatically from vector_test.go - DO NOT EDIT in that case! + +package vector + +import "testing" + +func TestIntZeroLen(t *testing.T) { + a := new(IntVector) + if a.Len() != 0 { + t.Errorf("%T: B1) expected 0, got %d", a, a.Len()) + } + if len(*a) != 0 { + t.Errorf("%T: B2) expected 0, got %d", a, len(*a)) + } + var b IntVector + if b.Len() != 0 { + t.Errorf("%T: B3) expected 0, got %d", b, b.Len()) + } + if len(b) != 0 { + t.Errorf("%T: B4) expected 0, got %d", b, len(b)) + } +} + +func TestIntResize(t *testing.T) { + var a IntVector + checkSize(t, &a, 0, 0) + checkSize(t, a.Resize(0, 5), 0, 5) + checkSize(t, a.Resize(1, 0), 1, 5) + checkSize(t, a.Resize(10, 0), 10, 10) + checkSize(t, a.Resize(5, 0), 5, 10) + checkSize(t, a.Resize(3, 8), 3, 10) + checkSize(t, a.Resize(0, 100), 0, 100) + checkSize(t, a.Resize(11, 100), 11, 100) +} + +func TestIntResize2(t *testing.T) { + var a IntVector + checkSize(t, &a, 0, 0) + a.Push(int2IntValue(1)) + a.Push(int2IntValue(2)) + a.Push(int2IntValue(3)) + a.Push(int2IntValue(4)) + checkSize(t, &a, 4, 4) + checkSize(t, a.Resize(10, 0), 10, 10) + for i := 4; i < a.Len(); i++ { + if a.At(i) != intzero { + t.Errorf("%T: expected a.At(%d) == %v; found %v!", a, i, intzero, a.At(i)) + } + } + for i := 4; i < len(a); i++ { + if a[i] != intzero { + t.Errorf("%T: expected a[%d] == %v; found %v", a, i, intzero, a[i]) + } + } +} + +func checkIntZero(t *testing.T, a *IntVector, i int) { + for j := 0; j < i; j++ { + if a.At(j) == intzero { + t.Errorf("%T: 1 expected a.At(%d) == %d; found %v", a, j, j, a.At(j)) + } + if (*a)[j] == intzero { + t.Errorf("%T: 2 expected (*a)[%d] == %d; found %v", a, j, j, (*a)[j]) + } + } + for ; i < a.Len(); i++ { + if a.At(i) != intzero { + t.Errorf("%T: 3 expected a.At(%d) == %v; found %v", a, i, intzero, a.At(i)) + } + if (*a)[i] != intzero { + t.Errorf("%T: 4 expected (*a)[%d] == %v; found %v", a, i, intzero, (*a)[i]) + } + } +} + +func TestIntTrailingElements(t *testing.T) { + var a IntVector + for i := 0; i < 10; i++ { + a.Push(int2IntValue(i + 1)) + } + checkIntZero(t, &a, 10) + checkSize(t, &a, 10, 16) + checkSize(t, a.Resize(5, 0), 5, 16) + checkSize(t, a.Resize(10, 0), 10, 16) + checkIntZero(t, &a, 5) +} + +func TestIntAccess(t *testing.T) { + const n = 100 + var a IntVector + a.Resize(n, 0) + for i := 0; i < n; i++ { + a.Set(i, int2IntValue(val(i))) + } + for i := 0; i < n; i++ { + if elem2IntValue(a.At(i)) != int2IntValue(val(i)) { + t.Error(i) + } + } + var b IntVector + b.Resize(n, 0) + for i := 0; i < n; i++ { + b[i] = int2IntValue(val(i)) + } + for i := 0; i < n; i++ { + if elem2IntValue(b[i]) != int2IntValue(val(i)) { + t.Error(i) + } + } +} + +func TestIntInsertDeleteClear(t *testing.T) { + const n = 100 + var a IntVector + + for i := 0; i < n; i++ { + if a.Len() != i { + t.Errorf("%T: A) wrong Len() %d (expected %d)", a, a.Len(), i) + } + if len(a) != i { + t.Errorf("%T: A) wrong len() %d (expected %d)", a, len(a), i) + } + a.Insert(0, int2IntValue(val(i))) + if elem2IntValue(a.Last()) != int2IntValue(val(0)) { + t.Errorf("%T: B", a) + } + } + for i := n - 1; i >= 0; i-- { + if elem2IntValue(a.Last()) != int2IntValue(val(0)) { + t.Errorf("%T: C", a) + } + if elem2IntValue(a.At(0)) != int2IntValue(val(i)) { + t.Errorf("%T: D", a) + } + if elem2IntValue(a[0]) != int2IntValue(val(i)) { + t.Errorf("%T: D2", a) + } + a.Delete(0) + if a.Len() != i { + t.Errorf("%T: E) wrong Len() %d (expected %d)", a, a.Len(), i) + } + if len(a) != i { + t.Errorf("%T: E) wrong len() %d (expected %d)", a, len(a), i) + } + } + + if a.Len() != 0 { + t.Errorf("%T: F) wrong Len() %d (expected 0)", a, a.Len()) + } + if len(a) != 0 { + t.Errorf("%T: F) wrong len() %d (expected 0)", a, len(a)) + } + for i := 0; i < n; i++ { + a.Push(int2IntValue(val(i))) + if a.Len() != i+1 { + t.Errorf("%T: G) wrong Len() %d (expected %d)", a, a.Len(), i+1) + } + if len(a) != i+1 { + t.Errorf("%T: G) wrong len() %d (expected %d)", a, len(a), i+1) + } + if elem2IntValue(a.Last()) != int2IntValue(val(i)) { + t.Errorf("%T: H", a) + } + } + a.Resize(0, 0) + if a.Len() != 0 { + t.Errorf("%T: I wrong Len() %d (expected 0)", a, a.Len()) + } + if len(a) != 0 { + t.Errorf("%T: I wrong len() %d (expected 0)", a, len(a)) + } + + const m = 5 + for j := 0; j < m; j++ { + a.Push(int2IntValue(j)) + for i := 0; i < n; i++ { + x := val(i) + a.Push(int2IntValue(x)) + if elem2IntValue(a.Pop()) != int2IntValue(x) { + t.Errorf("%T: J", a) + } + if a.Len() != j+1 { + t.Errorf("%T: K) wrong Len() %d (expected %d)", a, a.Len(), j+1) + } + if len(a) != j+1 { + t.Errorf("%T: K) wrong len() %d (expected %d)", a, len(a), j+1) + } + } + } + if a.Len() != m { + t.Errorf("%T: L) wrong Len() %d (expected %d)", a, a.Len(), m) + } + if len(a) != m { + t.Errorf("%T: L) wrong len() %d (expected %d)", a, len(a), m) + } +} + +func verify_sliceInt(t *testing.T, x *IntVector, elt, i, j int) { + for k := i; k < j; k++ { + if elem2IntValue(x.At(k)) != int2IntValue(elt) { + t.Errorf("%T: M) wrong [%d] element %v (expected %v)", x, k, elem2IntValue(x.At(k)), int2IntValue(elt)) + } + } + + s := x.Slice(i, j) + for k, n := 0, j-i; k < n; k++ { + if elem2IntValue(s.At(k)) != int2IntValue(elt) { + t.Errorf("%T: N) wrong [%d] element %v (expected %v)", x, k, elem2IntValue(x.At(k)), int2IntValue(elt)) + } + } +} + +func verify_patternInt(t *testing.T, x *IntVector, a, b, c int) { + n := a + b + c + if x.Len() != n { + t.Errorf("%T: O) wrong Len() %d (expected %d)", x, x.Len(), n) + } + if len(*x) != n { + t.Errorf("%T: O) wrong len() %d (expected %d)", x, len(*x), n) + } + verify_sliceInt(t, x, 0, 0, a) + verify_sliceInt(t, x, 1, a, a+b) + verify_sliceInt(t, x, 0, a+b, n) +} + +func make_vectorInt(elt, len int) *IntVector { + x := new(IntVector).Resize(len, 0) + for i := 0; i < len; i++ { + x.Set(i, int2IntValue(elt)) + } + return x +} + +func TestIntInsertVector(t *testing.T) { + // 1 + a := make_vectorInt(0, 0) + b := make_vectorInt(1, 10) + a.InsertVector(0, b) + verify_patternInt(t, a, 0, 10, 0) + // 2 + a = make_vectorInt(0, 10) + b = make_vectorInt(1, 0) + a.InsertVector(5, b) + verify_patternInt(t, a, 5, 0, 5) + // 3 + a = make_vectorInt(0, 10) + b = make_vectorInt(1, 3) + a.InsertVector(3, b) + verify_patternInt(t, a, 3, 3, 7) + // 4 + a = make_vectorInt(0, 10) + b = make_vectorInt(1, 1000) + a.InsertVector(8, b) + verify_patternInt(t, a, 8, 1000, 2) +} + +func TestIntDo(t *testing.T) { + const n = 25 + const salt = 17 + a := new(IntVector).Resize(n, 0) + for i := 0; i < n; i++ { + a.Set(i, int2IntValue(salt*i)) + } + count := 0 + a.Do(func(e int) { + i := intf2IntValue(e) + if i != int2IntValue(count*salt) { + t.Error(tname(a), "value at", count, "should be", count*salt, "not", i) + } + count++ + }) + if count != n { + t.Error(tname(a), "should visit", n, "values; did visit", count) + } + + b := new(IntVector).Resize(n, 0) + for i := 0; i < n; i++ { + (*b)[i] = int2IntValue(salt * i) + } + count = 0 + b.Do(func(e int) { + i := intf2IntValue(e) + if i != int2IntValue(count*salt) { + t.Error(tname(b), "b) value at", count, "should be", count*salt, "not", i) + } + count++ + }) + if count != n { + t.Error(tname(b), "b) should visit", n, "values; did visit", count) + } + + var c IntVector + c.Resize(n, 0) + for i := 0; i < n; i++ { + c[i] = int2IntValue(salt * i) + } + count = 0 + c.Do(func(e int) { + i := intf2IntValue(e) + if i != int2IntValue(count*salt) { + t.Error(tname(c), "c) value at", count, "should be", count*salt, "not", i) + } + count++ + }) + if count != n { + t.Error(tname(c), "c) should visit", n, "values; did visit", count) + } + +} + +func TestIntVectorCopy(t *testing.T) { + // verify Copy() returns a copy, not simply a slice of the original vector + const Len = 10 + var src IntVector + for i := 0; i < Len; i++ { + src.Push(int2IntValue(i * i)) + } + dest := src.Copy() + for i := 0; i < Len; i++ { + src[i] = int2IntValue(-1) + v := elem2IntValue(dest[i]) + if v != int2IntValue(i*i) { + t.Error(tname(src), "expected", i*i, "got", v) + } + } +} diff --git a/src/pkg/container/vector/nogen_test.go b/src/pkg/container/vector/nogen_test.go new file mode 100644 index 000000000..7b6a25952 --- /dev/null +++ b/src/pkg/container/vector/nogen_test.go @@ -0,0 +1,67 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package vector + +import ( + "fmt" + "sort" + "testing" +) + +var ( + zero interface{} + intzero int + strzero string +) + +func int2Value(x int) int { return x } +func int2IntValue(x int) int { return x } +func int2StrValue(x int) string { return string(x) } + +func elem2Value(x interface{}) int { return x.(int) } +func elem2IntValue(x int) int { return x } +func elem2StrValue(x string) string { return x } + +func intf2Value(x interface{}) int { return x.(int) } +func intf2IntValue(x interface{}) int { return x.(int) } +func intf2StrValue(x interface{}) string { return x.(string) } + +type VectorInterface interface { + Len() int + Cap() int +} + +func checkSize(t *testing.T, v VectorInterface, len, cap int) { + if v.Len() != len { + t.Errorf("%T expected len = %d; found %d", v, len, v.Len()) + } + if v.Cap() < cap { + t.Errorf("%T expected cap >= %d; found %d", v, cap, v.Cap()) + } +} + +func val(i int) int { return i*991 - 1234 } + +func TestSorting(t *testing.T) { + const n = 100 + + a := new(IntVector).Resize(n, 0) + for i := n - 1; i >= 0; i-- { + a.Set(i, n-1-i) + } + if sort.IsSorted(a) { + t.Error("int vector not sorted") + } + + b := new(StringVector).Resize(n, 0) + for i := n - 1; i >= 0; i-- { + b.Set(i, fmt.Sprint(n-1-i)) + } + if sort.IsSorted(b) { + t.Error("string vector not sorted") + } +} + +func tname(x interface{}) string { return fmt.Sprintf("%T: ", x) } diff --git a/src/pkg/container/vector/numbers_test.go b/src/pkg/container/vector/numbers_test.go new file mode 100644 index 000000000..abe01a8fb --- /dev/null +++ b/src/pkg/container/vector/numbers_test.go @@ -0,0 +1,123 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package vector + +import ( + "fmt" + "runtime" + "strings" + "testing" +) + +const memTestN = 1000000 + +func s(n uint64) string { + str := fmt.Sprintf("%d", n) + lens := len(str) + a := make([]string, (lens+2)/3) + start := lens + for i := range a { + start -= 3 + if start < 0 { + start = 0 + } + a[len(a)-i-1] = str[start:lens] + lens -= 3 + } + return strings.Join(a, " ") +} + +func TestVectorNums(t *testing.T) { + if testing.Short() { + return + } + var v Vector + c := int(0) + runtime.GC() + m0 := runtime.MemStats + v.Resize(memTestN, memTestN) + for i := 0; i < memTestN; i++ { + v.Set(i, c) + } + runtime.GC() + m := runtime.MemStats + v.Resize(0, 0) + runtime.GC() + n := m.Alloc - m0.Alloc + t.Logf("%T.Push(%#v), n = %s: Alloc/n = %.2f\n", v, c, s(memTestN), float64(n)/memTestN) +} + +func TestIntVectorNums(t *testing.T) { + if testing.Short() { + return + } + var v IntVector + c := int(0) + runtime.GC() + m0 := runtime.MemStats + v.Resize(memTestN, memTestN) + for i := 0; i < memTestN; i++ { + v.Set(i, c) + } + runtime.GC() + m := runtime.MemStats + v.Resize(0, 0) + runtime.GC() + n := m.Alloc - m0.Alloc + t.Logf("%T.Push(%#v), n = %s: Alloc/n = %.2f\n", v, c, s(memTestN), float64(n)/memTestN) +} + +func TestStringVectorNums(t *testing.T) { + if testing.Short() { + return + } + var v StringVector + c := "" + runtime.GC() + m0 := runtime.MemStats + v.Resize(memTestN, memTestN) + for i := 0; i < memTestN; i++ { + v.Set(i, c) + } + runtime.GC() + m := runtime.MemStats + v.Resize(0, 0) + runtime.GC() + n := m.Alloc - m0.Alloc + t.Logf("%T.Push(%#v), n = %s: Alloc/n = %.2f\n", v, c, s(memTestN), float64(n)/memTestN) +} + +func BenchmarkVectorNums(b *testing.B) { + c := int(0) + var v Vector + b.StopTimer() + runtime.GC() + b.StartTimer() + for i := 0; i < b.N; i++ { + v.Push(c) + } +} + +func BenchmarkIntVectorNums(b *testing.B) { + c := int(0) + var v IntVector + b.StopTimer() + runtime.GC() + b.StartTimer() + for i := 0; i < b.N; i++ { + v.Push(c) + } +} + +func BenchmarkStringVectorNums(b *testing.B) { + c := "" + var v StringVector + b.StopTimer() + runtime.GC() + b.StartTimer() + for i := 0; i < b.N; i++ { + v.Push(c) + } +} diff --git a/src/pkg/container/vector/stringvector.go b/src/pkg/container/vector/stringvector.go new file mode 100644 index 000000000..dc81f06b7 --- /dev/null +++ b/src/pkg/container/vector/stringvector.go @@ -0,0 +1,188 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// CAUTION: If this file is not vector.go, it was generated +// automatically from vector.go - DO NOT EDIT in that case! + +package vector + +func (p *StringVector) realloc(length, capacity int) (b []string) { + if capacity < initialSize { + capacity = initialSize + } + if capacity < length { + capacity = length + } + b = make(StringVector, length, capacity) + copy(b, *p) + *p = b + return +} + +// Insert n elements at position i. +func (p *StringVector) Expand(i, n int) { + a := *p + + // make sure we have enough space + len0 := len(a) + len1 := len0 + n + if len1 <= cap(a) { + // enough space - just expand + a = a[0:len1] + } else { + // not enough space - double capacity + capb := cap(a) * 2 + if capb < len1 { + // still not enough - use required length + capb = len1 + } + // capb >= len1 + a = p.realloc(len1, capb) + } + + // make a hole + for j := len0 - 1; j >= i; j-- { + a[j+n] = a[j] + } + + *p = a +} + +// Insert n elements at the end of a vector. +func (p *StringVector) Extend(n int) { p.Expand(len(*p), n) } + +// Resize changes the length and capacity of a vector. +// If the new length is shorter than the current length, Resize discards +// trailing elements. If the new length is longer than the current length, +// Resize adds the respective zero values for the additional elements. The capacity +// parameter is ignored unless the new length or capacity is longer than the current +// capacity. The resized vector's capacity may be larger than the requested capacity. +func (p *StringVector) Resize(length, capacity int) *StringVector { + a := *p + + if length > cap(a) || capacity > cap(a) { + // not enough space or larger capacity requested explicitly + a = p.realloc(length, capacity) + } else if length < len(a) { + // clear trailing elements + for i := range a[length:] { + var zero string + a[length+i] = zero + } + } + + *p = a[0:length] + return p +} + +// Len returns the number of elements in the vector. +// Same as len(*p). +func (p *StringVector) Len() int { return len(*p) } + +// Cap returns the capacity of the vector; that is, the +// maximum length the vector can grow without resizing. +// Same as cap(*p). +func (p *StringVector) Cap() int { return cap(*p) } + +// At returns the i'th element of the vector. +func (p *StringVector) At(i int) string { return (*p)[i] } + +// Set sets the i'th element of the vector to value x. +func (p *StringVector) Set(i int, x string) { (*p)[i] = x } + +// Last returns the element in the vector of highest index. +func (p *StringVector) Last() string { return (*p)[len(*p)-1] } + +// Copy makes a copy of the vector and returns it. +func (p *StringVector) Copy() StringVector { + arr := make(StringVector, len(*p)) + copy(arr, *p) + return arr +} + +// Insert inserts into the vector an element of value x before +// the current element at index i. +func (p *StringVector) Insert(i int, x string) { + p.Expand(i, 1) + (*p)[i] = x +} + +// Delete deletes the i'th element of the vector. The gap is closed so the old +// element at index i+1 has index i afterwards. +func (p *StringVector) Delete(i int) { + a := *p + n := len(a) + + copy(a[i:n-1], a[i+1:n]) + var zero string + a[n-1] = zero // support GC, zero out entry + *p = a[0 : n-1] +} + +// InsertVector inserts into the vector the contents of the vector +// x such that the 0th element of x appears at index i after insertion. +func (p *StringVector) InsertVector(i int, x *StringVector) { + b := *x + + p.Expand(i, len(b)) + copy((*p)[i:i+len(b)], b) +} + +// Cut deletes elements i through j-1, inclusive. +func (p *StringVector) Cut(i, j int) { + a := *p + n := len(a) + m := n - (j - i) + + copy(a[i:m], a[j:n]) + for k := m; k < n; k++ { //TODO(bflm) don't zero out the elements unless it's a Vector. + var zero string + a[k] = zero // support GC, zero out entries + } + + *p = a[0:m] +} + +// Slice returns a new sub-vector by slicing the old one to extract slice [i:j]. +// The elements are copied. The original vector is unchanged. +func (p *StringVector) Slice(i, j int) *StringVector { + var s StringVector + s.realloc(j-i, 0) // will fail in Init() if j < i + copy(s, (*p)[i:j]) + return &s +} + +// Convenience wrappers + +// Push appends x to the end of the vector. +func (p *StringVector) Push(x string) { p.Insert(len(*p), x) } + +// Pop deletes the last element of the vector. +func (p *StringVector) Pop() string { + a := *p + + i := len(a) - 1 + x := a[i] + var zero string + a[i] = zero // support GC, zero out entry + *p = a[0:i] + return x +} + +// AppendVector appends the entire vector x to the end of this vector. +func (p *StringVector) AppendVector(x *StringVector) { p.InsertVector(len(*p), x) } + +// Swap exchanges the elements at indexes i and j. +func (p *StringVector) Swap(i, j int) { + a := *p + a[i], a[j] = a[j], a[i] +} + +// Do calls function f for each element of the vector, in order. +// The behavior of Do is undefined if f changes *p. +func (p *StringVector) Do(f func(elem string)) { + for _, e := range *p { + f(e) + } +} diff --git a/src/pkg/container/vector/stringvector_test.go b/src/pkg/container/vector/stringvector_test.go new file mode 100644 index 000000000..c75676f78 --- /dev/null +++ b/src/pkg/container/vector/stringvector_test.go @@ -0,0 +1,331 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// CAUTION: If this file is not vector_test.go, it was generated +// automatically from vector_test.go - DO NOT EDIT in that case! + +package vector + +import "testing" + +func TestStrZeroLen(t *testing.T) { + a := new(StringVector) + if a.Len() != 0 { + t.Errorf("%T: B1) expected 0, got %d", a, a.Len()) + } + if len(*a) != 0 { + t.Errorf("%T: B2) expected 0, got %d", a, len(*a)) + } + var b StringVector + if b.Len() != 0 { + t.Errorf("%T: B3) expected 0, got %d", b, b.Len()) + } + if len(b) != 0 { + t.Errorf("%T: B4) expected 0, got %d", b, len(b)) + } +} + +func TestStrResize(t *testing.T) { + var a StringVector + checkSize(t, &a, 0, 0) + checkSize(t, a.Resize(0, 5), 0, 5) + checkSize(t, a.Resize(1, 0), 1, 5) + checkSize(t, a.Resize(10, 0), 10, 10) + checkSize(t, a.Resize(5, 0), 5, 10) + checkSize(t, a.Resize(3, 8), 3, 10) + checkSize(t, a.Resize(0, 100), 0, 100) + checkSize(t, a.Resize(11, 100), 11, 100) +} + +func TestStrResize2(t *testing.T) { + var a StringVector + checkSize(t, &a, 0, 0) + a.Push(int2StrValue(1)) + a.Push(int2StrValue(2)) + a.Push(int2StrValue(3)) + a.Push(int2StrValue(4)) + checkSize(t, &a, 4, 4) + checkSize(t, a.Resize(10, 0), 10, 10) + for i := 4; i < a.Len(); i++ { + if a.At(i) != strzero { + t.Errorf("%T: expected a.At(%d) == %v; found %v!", a, i, strzero, a.At(i)) + } + } + for i := 4; i < len(a); i++ { + if a[i] != strzero { + t.Errorf("%T: expected a[%d] == %v; found %v", a, i, strzero, a[i]) + } + } +} + +func checkStrZero(t *testing.T, a *StringVector, i int) { + for j := 0; j < i; j++ { + if a.At(j) == strzero { + t.Errorf("%T: 1 expected a.At(%d) == %d; found %v", a, j, j, a.At(j)) + } + if (*a)[j] == strzero { + t.Errorf("%T: 2 expected (*a)[%d] == %d; found %v", a, j, j, (*a)[j]) + } + } + for ; i < a.Len(); i++ { + if a.At(i) != strzero { + t.Errorf("%T: 3 expected a.At(%d) == %v; found %v", a, i, strzero, a.At(i)) + } + if (*a)[i] != strzero { + t.Errorf("%T: 4 expected (*a)[%d] == %v; found %v", a, i, strzero, (*a)[i]) + } + } +} + +func TestStrTrailingElements(t *testing.T) { + var a StringVector + for i := 0; i < 10; i++ { + a.Push(int2StrValue(i + 1)) + } + checkStrZero(t, &a, 10) + checkSize(t, &a, 10, 16) + checkSize(t, a.Resize(5, 0), 5, 16) + checkSize(t, a.Resize(10, 0), 10, 16) + checkStrZero(t, &a, 5) +} + +func TestStrAccess(t *testing.T) { + const n = 100 + var a StringVector + a.Resize(n, 0) + for i := 0; i < n; i++ { + a.Set(i, int2StrValue(val(i))) + } + for i := 0; i < n; i++ { + if elem2StrValue(a.At(i)) != int2StrValue(val(i)) { + t.Error(i) + } + } + var b StringVector + b.Resize(n, 0) + for i := 0; i < n; i++ { + b[i] = int2StrValue(val(i)) + } + for i := 0; i < n; i++ { + if elem2StrValue(b[i]) != int2StrValue(val(i)) { + t.Error(i) + } + } +} + +func TestStrInsertDeleteClear(t *testing.T) { + const n = 100 + var a StringVector + + for i := 0; i < n; i++ { + if a.Len() != i { + t.Errorf("%T: A) wrong Len() %d (expected %d)", a, a.Len(), i) + } + if len(a) != i { + t.Errorf("%T: A) wrong len() %d (expected %d)", a, len(a), i) + } + a.Insert(0, int2StrValue(val(i))) + if elem2StrValue(a.Last()) != int2StrValue(val(0)) { + t.Errorf("%T: B", a) + } + } + for i := n - 1; i >= 0; i-- { + if elem2StrValue(a.Last()) != int2StrValue(val(0)) { + t.Errorf("%T: C", a) + } + if elem2StrValue(a.At(0)) != int2StrValue(val(i)) { + t.Errorf("%T: D", a) + } + if elem2StrValue(a[0]) != int2StrValue(val(i)) { + t.Errorf("%T: D2", a) + } + a.Delete(0) + if a.Len() != i { + t.Errorf("%T: E) wrong Len() %d (expected %d)", a, a.Len(), i) + } + if len(a) != i { + t.Errorf("%T: E) wrong len() %d (expected %d)", a, len(a), i) + } + } + + if a.Len() != 0 { + t.Errorf("%T: F) wrong Len() %d (expected 0)", a, a.Len()) + } + if len(a) != 0 { + t.Errorf("%T: F) wrong len() %d (expected 0)", a, len(a)) + } + for i := 0; i < n; i++ { + a.Push(int2StrValue(val(i))) + if a.Len() != i+1 { + t.Errorf("%T: G) wrong Len() %d (expected %d)", a, a.Len(), i+1) + } + if len(a) != i+1 { + t.Errorf("%T: G) wrong len() %d (expected %d)", a, len(a), i+1) + } + if elem2StrValue(a.Last()) != int2StrValue(val(i)) { + t.Errorf("%T: H", a) + } + } + a.Resize(0, 0) + if a.Len() != 0 { + t.Errorf("%T: I wrong Len() %d (expected 0)", a, a.Len()) + } + if len(a) != 0 { + t.Errorf("%T: I wrong len() %d (expected 0)", a, len(a)) + } + + const m = 5 + for j := 0; j < m; j++ { + a.Push(int2StrValue(j)) + for i := 0; i < n; i++ { + x := val(i) + a.Push(int2StrValue(x)) + if elem2StrValue(a.Pop()) != int2StrValue(x) { + t.Errorf("%T: J", a) + } + if a.Len() != j+1 { + t.Errorf("%T: K) wrong Len() %d (expected %d)", a, a.Len(), j+1) + } + if len(a) != j+1 { + t.Errorf("%T: K) wrong len() %d (expected %d)", a, len(a), j+1) + } + } + } + if a.Len() != m { + t.Errorf("%T: L) wrong Len() %d (expected %d)", a, a.Len(), m) + } + if len(a) != m { + t.Errorf("%T: L) wrong len() %d (expected %d)", a, len(a), m) + } +} + +func verify_sliceStr(t *testing.T, x *StringVector, elt, i, j int) { + for k := i; k < j; k++ { + if elem2StrValue(x.At(k)) != int2StrValue(elt) { + t.Errorf("%T: M) wrong [%d] element %v (expected %v)", x, k, elem2StrValue(x.At(k)), int2StrValue(elt)) + } + } + + s := x.Slice(i, j) + for k, n := 0, j-i; k < n; k++ { + if elem2StrValue(s.At(k)) != int2StrValue(elt) { + t.Errorf("%T: N) wrong [%d] element %v (expected %v)", x, k, elem2StrValue(x.At(k)), int2StrValue(elt)) + } + } +} + +func verify_patternStr(t *testing.T, x *StringVector, a, b, c int) { + n := a + b + c + if x.Len() != n { + t.Errorf("%T: O) wrong Len() %d (expected %d)", x, x.Len(), n) + } + if len(*x) != n { + t.Errorf("%T: O) wrong len() %d (expected %d)", x, len(*x), n) + } + verify_sliceStr(t, x, 0, 0, a) + verify_sliceStr(t, x, 1, a, a+b) + verify_sliceStr(t, x, 0, a+b, n) +} + +func make_vectorStr(elt, len int) *StringVector { + x := new(StringVector).Resize(len, 0) + for i := 0; i < len; i++ { + x.Set(i, int2StrValue(elt)) + } + return x +} + +func TestStrInsertVector(t *testing.T) { + // 1 + a := make_vectorStr(0, 0) + b := make_vectorStr(1, 10) + a.InsertVector(0, b) + verify_patternStr(t, a, 0, 10, 0) + // 2 + a = make_vectorStr(0, 10) + b = make_vectorStr(1, 0) + a.InsertVector(5, b) + verify_patternStr(t, a, 5, 0, 5) + // 3 + a = make_vectorStr(0, 10) + b = make_vectorStr(1, 3) + a.InsertVector(3, b) + verify_patternStr(t, a, 3, 3, 7) + // 4 + a = make_vectorStr(0, 10) + b = make_vectorStr(1, 1000) + a.InsertVector(8, b) + verify_patternStr(t, a, 8, 1000, 2) +} + +func TestStrDo(t *testing.T) { + const n = 25 + const salt = 17 + a := new(StringVector).Resize(n, 0) + for i := 0; i < n; i++ { + a.Set(i, int2StrValue(salt*i)) + } + count := 0 + a.Do(func(e string) { + i := intf2StrValue(e) + if i != int2StrValue(count*salt) { + t.Error(tname(a), "value at", count, "should be", count*salt, "not", i) + } + count++ + }) + if count != n { + t.Error(tname(a), "should visit", n, "values; did visit", count) + } + + b := new(StringVector).Resize(n, 0) + for i := 0; i < n; i++ { + (*b)[i] = int2StrValue(salt * i) + } + count = 0 + b.Do(func(e string) { + i := intf2StrValue(e) + if i != int2StrValue(count*salt) { + t.Error(tname(b), "b) value at", count, "should be", count*salt, "not", i) + } + count++ + }) + if count != n { + t.Error(tname(b), "b) should visit", n, "values; did visit", count) + } + + var c StringVector + c.Resize(n, 0) + for i := 0; i < n; i++ { + c[i] = int2StrValue(salt * i) + } + count = 0 + c.Do(func(e string) { + i := intf2StrValue(e) + if i != int2StrValue(count*salt) { + t.Error(tname(c), "c) value at", count, "should be", count*salt, "not", i) + } + count++ + }) + if count != n { + t.Error(tname(c), "c) should visit", n, "values; did visit", count) + } + +} + +func TestStrVectorCopy(t *testing.T) { + // verify Copy() returns a copy, not simply a slice of the original vector + const Len = 10 + var src StringVector + for i := 0; i < Len; i++ { + src.Push(int2StrValue(i * i)) + } + dest := src.Copy() + for i := 0; i < Len; i++ { + src[i] = int2StrValue(-1) + v := elem2StrValue(dest[i]) + if v != int2StrValue(i*i) { + t.Error(tname(src), "expected", i*i, "got", v) + } + } +} diff --git a/src/pkg/container/vector/vector.go b/src/pkg/container/vector/vector.go new file mode 100644 index 000000000..8470ec067 --- /dev/null +++ b/src/pkg/container/vector/vector.go @@ -0,0 +1,188 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// CAUTION: If this file is not vector.go, it was generated +// automatically from vector.go - DO NOT EDIT in that case! + +package vector + +func (p *Vector) realloc(length, capacity int) (b []interface{}) { + if capacity < initialSize { + capacity = initialSize + } + if capacity < length { + capacity = length + } + b = make(Vector, length, capacity) + copy(b, *p) + *p = b + return +} + +// Insert n elements at position i. +func (p *Vector) Expand(i, n int) { + a := *p + + // make sure we have enough space + len0 := len(a) + len1 := len0 + n + if len1 <= cap(a) { + // enough space - just expand + a = a[0:len1] + } else { + // not enough space - double capacity + capb := cap(a) * 2 + if capb < len1 { + // still not enough - use required length + capb = len1 + } + // capb >= len1 + a = p.realloc(len1, capb) + } + + // make a hole + for j := len0 - 1; j >= i; j-- { + a[j+n] = a[j] + } + + *p = a +} + +// Insert n elements at the end of a vector. +func (p *Vector) Extend(n int) { p.Expand(len(*p), n) } + +// Resize changes the length and capacity of a vector. +// If the new length is shorter than the current length, Resize discards +// trailing elements. If the new length is longer than the current length, +// Resize adds the respective zero values for the additional elements. The capacity +// parameter is ignored unless the new length or capacity is longer than the current +// capacity. The resized vector's capacity may be larger than the requested capacity. +func (p *Vector) Resize(length, capacity int) *Vector { + a := *p + + if length > cap(a) || capacity > cap(a) { + // not enough space or larger capacity requested explicitly + a = p.realloc(length, capacity) + } else if length < len(a) { + // clear trailing elements + for i := range a[length:] { + var zero interface{} + a[length+i] = zero + } + } + + *p = a[0:length] + return p +} + +// Len returns the number of elements in the vector. +// Same as len(*p). +func (p *Vector) Len() int { return len(*p) } + +// Cap returns the capacity of the vector; that is, the +// maximum length the vector can grow without resizing. +// Same as cap(*p). +func (p *Vector) Cap() int { return cap(*p) } + +// At returns the i'th element of the vector. +func (p *Vector) At(i int) interface{} { return (*p)[i] } + +// Set sets the i'th element of the vector to value x. +func (p *Vector) Set(i int, x interface{}) { (*p)[i] = x } + +// Last returns the element in the vector of highest index. +func (p *Vector) Last() interface{} { return (*p)[len(*p)-1] } + +// Copy makes a copy of the vector and returns it. +func (p *Vector) Copy() Vector { + arr := make(Vector, len(*p)) + copy(arr, *p) + return arr +} + +// Insert inserts into the vector an element of value x before +// the current element at index i. +func (p *Vector) Insert(i int, x interface{}) { + p.Expand(i, 1) + (*p)[i] = x +} + +// Delete deletes the i'th element of the vector. The gap is closed so the old +// element at index i+1 has index i afterwards. +func (p *Vector) Delete(i int) { + a := *p + n := len(a) + + copy(a[i:n-1], a[i+1:n]) + var zero interface{} + a[n-1] = zero // support GC, zero out entry + *p = a[0 : n-1] +} + +// InsertVector inserts into the vector the contents of the vector +// x such that the 0th element of x appears at index i after insertion. +func (p *Vector) InsertVector(i int, x *Vector) { + b := *x + + p.Expand(i, len(b)) + copy((*p)[i:i+len(b)], b) +} + +// Cut deletes elements i through j-1, inclusive. +func (p *Vector) Cut(i, j int) { + a := *p + n := len(a) + m := n - (j - i) + + copy(a[i:m], a[j:n]) + for k := m; k < n; k++ { //TODO(bflm) don't zero out the elements unless it's a Vector. + var zero interface{} + a[k] = zero // support GC, zero out entries + } + + *p = a[0:m] +} + +// Slice returns a new sub-vector by slicing the old one to extract slice [i:j]. +// The elements are copied. The original vector is unchanged. +func (p *Vector) Slice(i, j int) *Vector { + var s Vector + s.realloc(j-i, 0) // will fail in Init() if j < i + copy(s, (*p)[i:j]) + return &s +} + +// Convenience wrappers + +// Push appends x to the end of the vector. +func (p *Vector) Push(x interface{}) { p.Insert(len(*p), x) } + +// Pop deletes the last element of the vector. +func (p *Vector) Pop() interface{} { + a := *p + + i := len(a) - 1 + x := a[i] + var zero interface{} + a[i] = zero // support GC, zero out entry + *p = a[0:i] + return x +} + +// AppendVector appends the entire vector x to the end of this vector. +func (p *Vector) AppendVector(x *Vector) { p.InsertVector(len(*p), x) } + +// Swap exchanges the elements at indexes i and j. +func (p *Vector) Swap(i, j int) { + a := *p + a[i], a[j] = a[j], a[i] +} + +// Do calls function f for each element of the vector, in order. +// The behavior of Do is undefined if f changes *p. +func (p *Vector) Do(f func(elem interface{})) { + for _, e := range *p { + f(e) + } +} diff --git a/src/pkg/container/vector/vector_test.go b/src/pkg/container/vector/vector_test.go new file mode 100644 index 000000000..a7f47b8c2 --- /dev/null +++ b/src/pkg/container/vector/vector_test.go @@ -0,0 +1,331 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// CAUTION: If this file is not vector_test.go, it was generated +// automatically from vector_test.go - DO NOT EDIT in that case! + +package vector + +import "testing" + +func TestZeroLen(t *testing.T) { + a := new(Vector) + if a.Len() != 0 { + t.Errorf("%T: B1) expected 0, got %d", a, a.Len()) + } + if len(*a) != 0 { + t.Errorf("%T: B2) expected 0, got %d", a, len(*a)) + } + var b Vector + if b.Len() != 0 { + t.Errorf("%T: B3) expected 0, got %d", b, b.Len()) + } + if len(b) != 0 { + t.Errorf("%T: B4) expected 0, got %d", b, len(b)) + } +} + +func TestResize(t *testing.T) { + var a Vector + checkSize(t, &a, 0, 0) + checkSize(t, a.Resize(0, 5), 0, 5) + checkSize(t, a.Resize(1, 0), 1, 5) + checkSize(t, a.Resize(10, 0), 10, 10) + checkSize(t, a.Resize(5, 0), 5, 10) + checkSize(t, a.Resize(3, 8), 3, 10) + checkSize(t, a.Resize(0, 100), 0, 100) + checkSize(t, a.Resize(11, 100), 11, 100) +} + +func TestResize2(t *testing.T) { + var a Vector + checkSize(t, &a, 0, 0) + a.Push(int2Value(1)) + a.Push(int2Value(2)) + a.Push(int2Value(3)) + a.Push(int2Value(4)) + checkSize(t, &a, 4, 4) + checkSize(t, a.Resize(10, 0), 10, 10) + for i := 4; i < a.Len(); i++ { + if a.At(i) != zero { + t.Errorf("%T: expected a.At(%d) == %v; found %v!", a, i, zero, a.At(i)) + } + } + for i := 4; i < len(a); i++ { + if a[i] != zero { + t.Errorf("%T: expected a[%d] == %v; found %v", a, i, zero, a[i]) + } + } +} + +func checkZero(t *testing.T, a *Vector, i int) { + for j := 0; j < i; j++ { + if a.At(j) == zero { + t.Errorf("%T: 1 expected a.At(%d) == %d; found %v", a, j, j, a.At(j)) + } + if (*a)[j] == zero { + t.Errorf("%T: 2 expected (*a)[%d] == %d; found %v", a, j, j, (*a)[j]) + } + } + for ; i < a.Len(); i++ { + if a.At(i) != zero { + t.Errorf("%T: 3 expected a.At(%d) == %v; found %v", a, i, zero, a.At(i)) + } + if (*a)[i] != zero { + t.Errorf("%T: 4 expected (*a)[%d] == %v; found %v", a, i, zero, (*a)[i]) + } + } +} + +func TestTrailingElements(t *testing.T) { + var a Vector + for i := 0; i < 10; i++ { + a.Push(int2Value(i + 1)) + } + checkZero(t, &a, 10) + checkSize(t, &a, 10, 16) + checkSize(t, a.Resize(5, 0), 5, 16) + checkSize(t, a.Resize(10, 0), 10, 16) + checkZero(t, &a, 5) +} + +func TestAccess(t *testing.T) { + const n = 100 + var a Vector + a.Resize(n, 0) + for i := 0; i < n; i++ { + a.Set(i, int2Value(val(i))) + } + for i := 0; i < n; i++ { + if elem2Value(a.At(i)) != int2Value(val(i)) { + t.Error(i) + } + } + var b Vector + b.Resize(n, 0) + for i := 0; i < n; i++ { + b[i] = int2Value(val(i)) + } + for i := 0; i < n; i++ { + if elem2Value(b[i]) != int2Value(val(i)) { + t.Error(i) + } + } +} + +func TestInsertDeleteClear(t *testing.T) { + const n = 100 + var a Vector + + for i := 0; i < n; i++ { + if a.Len() != i { + t.Errorf("%T: A) wrong Len() %d (expected %d)", a, a.Len(), i) + } + if len(a) != i { + t.Errorf("%T: A) wrong len() %d (expected %d)", a, len(a), i) + } + a.Insert(0, int2Value(val(i))) + if elem2Value(a.Last()) != int2Value(val(0)) { + t.Errorf("%T: B", a) + } + } + for i := n - 1; i >= 0; i-- { + if elem2Value(a.Last()) != int2Value(val(0)) { + t.Errorf("%T: C", a) + } + if elem2Value(a.At(0)) != int2Value(val(i)) { + t.Errorf("%T: D", a) + } + if elem2Value(a[0]) != int2Value(val(i)) { + t.Errorf("%T: D2", a) + } + a.Delete(0) + if a.Len() != i { + t.Errorf("%T: E) wrong Len() %d (expected %d)", a, a.Len(), i) + } + if len(a) != i { + t.Errorf("%T: E) wrong len() %d (expected %d)", a, len(a), i) + } + } + + if a.Len() != 0 { + t.Errorf("%T: F) wrong Len() %d (expected 0)", a, a.Len()) + } + if len(a) != 0 { + t.Errorf("%T: F) wrong len() %d (expected 0)", a, len(a)) + } + for i := 0; i < n; i++ { + a.Push(int2Value(val(i))) + if a.Len() != i+1 { + t.Errorf("%T: G) wrong Len() %d (expected %d)", a, a.Len(), i+1) + } + if len(a) != i+1 { + t.Errorf("%T: G) wrong len() %d (expected %d)", a, len(a), i+1) + } + if elem2Value(a.Last()) != int2Value(val(i)) { + t.Errorf("%T: H", a) + } + } + a.Resize(0, 0) + if a.Len() != 0 { + t.Errorf("%T: I wrong Len() %d (expected 0)", a, a.Len()) + } + if len(a) != 0 { + t.Errorf("%T: I wrong len() %d (expected 0)", a, len(a)) + } + + const m = 5 + for j := 0; j < m; j++ { + a.Push(int2Value(j)) + for i := 0; i < n; i++ { + x := val(i) + a.Push(int2Value(x)) + if elem2Value(a.Pop()) != int2Value(x) { + t.Errorf("%T: J", a) + } + if a.Len() != j+1 { + t.Errorf("%T: K) wrong Len() %d (expected %d)", a, a.Len(), j+1) + } + if len(a) != j+1 { + t.Errorf("%T: K) wrong len() %d (expected %d)", a, len(a), j+1) + } + } + } + if a.Len() != m { + t.Errorf("%T: L) wrong Len() %d (expected %d)", a, a.Len(), m) + } + if len(a) != m { + t.Errorf("%T: L) wrong len() %d (expected %d)", a, len(a), m) + } +} + +func verify_slice(t *testing.T, x *Vector, elt, i, j int) { + for k := i; k < j; k++ { + if elem2Value(x.At(k)) != int2Value(elt) { + t.Errorf("%T: M) wrong [%d] element %v (expected %v)", x, k, elem2Value(x.At(k)), int2Value(elt)) + } + } + + s := x.Slice(i, j) + for k, n := 0, j-i; k < n; k++ { + if elem2Value(s.At(k)) != int2Value(elt) { + t.Errorf("%T: N) wrong [%d] element %v (expected %v)", x, k, elem2Value(x.At(k)), int2Value(elt)) + } + } +} + +func verify_pattern(t *testing.T, x *Vector, a, b, c int) { + n := a + b + c + if x.Len() != n { + t.Errorf("%T: O) wrong Len() %d (expected %d)", x, x.Len(), n) + } + if len(*x) != n { + t.Errorf("%T: O) wrong len() %d (expected %d)", x, len(*x), n) + } + verify_slice(t, x, 0, 0, a) + verify_slice(t, x, 1, a, a+b) + verify_slice(t, x, 0, a+b, n) +} + +func make_vector(elt, len int) *Vector { + x := new(Vector).Resize(len, 0) + for i := 0; i < len; i++ { + x.Set(i, int2Value(elt)) + } + return x +} + +func TestInsertVector(t *testing.T) { + // 1 + a := make_vector(0, 0) + b := make_vector(1, 10) + a.InsertVector(0, b) + verify_pattern(t, a, 0, 10, 0) + // 2 + a = make_vector(0, 10) + b = make_vector(1, 0) + a.InsertVector(5, b) + verify_pattern(t, a, 5, 0, 5) + // 3 + a = make_vector(0, 10) + b = make_vector(1, 3) + a.InsertVector(3, b) + verify_pattern(t, a, 3, 3, 7) + // 4 + a = make_vector(0, 10) + b = make_vector(1, 1000) + a.InsertVector(8, b) + verify_pattern(t, a, 8, 1000, 2) +} + +func TestDo(t *testing.T) { + const n = 25 + const salt = 17 + a := new(Vector).Resize(n, 0) + for i := 0; i < n; i++ { + a.Set(i, int2Value(salt*i)) + } + count := 0 + a.Do(func(e interface{}) { + i := intf2Value(e) + if i != int2Value(count*salt) { + t.Error(tname(a), "value at", count, "should be", count*salt, "not", i) + } + count++ + }) + if count != n { + t.Error(tname(a), "should visit", n, "values; did visit", count) + } + + b := new(Vector).Resize(n, 0) + for i := 0; i < n; i++ { + (*b)[i] = int2Value(salt * i) + } + count = 0 + b.Do(func(e interface{}) { + i := intf2Value(e) + if i != int2Value(count*salt) { + t.Error(tname(b), "b) value at", count, "should be", count*salt, "not", i) + } + count++ + }) + if count != n { + t.Error(tname(b), "b) should visit", n, "values; did visit", count) + } + + var c Vector + c.Resize(n, 0) + for i := 0; i < n; i++ { + c[i] = int2Value(salt * i) + } + count = 0 + c.Do(func(e interface{}) { + i := intf2Value(e) + if i != int2Value(count*salt) { + t.Error(tname(c), "c) value at", count, "should be", count*salt, "not", i) + } + count++ + }) + if count != n { + t.Error(tname(c), "c) should visit", n, "values; did visit", count) + } + +} + +func TestVectorCopy(t *testing.T) { + // verify Copy() returns a copy, not simply a slice of the original vector + const Len = 10 + var src Vector + for i := 0; i < Len; i++ { + src.Push(int2Value(i * i)) + } + dest := src.Copy() + for i := 0; i < Len; i++ { + src[i] = int2Value(-1) + v := elem2Value(dest[i]) + if v != int2Value(i*i) { + t.Error(tname(src), "expected", i*i, "got", v) + } + } +} diff --git a/src/pkg/crypto/Makefile b/src/pkg/crypto/Makefile new file mode 100644 index 000000000..738a52062 --- /dev/null +++ b/src/pkg/crypto/Makefile @@ -0,0 +1,11 @@ +# Copyright 2011 The Go Authors. All rights reserved. +# Use of this source code is governed by a BSD-style +# license that can be found in the LICENSE file. + +include ../../Make.inc + +TARG=crypto +GOFILES=\ + crypto.go\ + +include ../../Make.pkg diff --git a/src/pkg/crypto/aes/Makefile b/src/pkg/crypto/aes/Makefile new file mode 100644 index 000000000..9dc846ee3 --- /dev/null +++ b/src/pkg/crypto/aes/Makefile @@ -0,0 +1,13 @@ +# Copyright 2009 The Go Authors. All rights reserved. +# Use of this source code is governed by a BSD-style +# license that can be found in the LICENSE file. + +include ../../../Make.inc + +TARG=crypto/aes +GOFILES=\ + block.go\ + cipher.go\ + const.go\ + +include ../../../Make.pkg diff --git a/src/pkg/crypto/aes/aes_test.go b/src/pkg/crypto/aes/aes_test.go new file mode 100644 index 000000000..2136d447d --- /dev/null +++ b/src/pkg/crypto/aes/aes_test.go @@ -0,0 +1,350 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package aes + +import ( + "testing" +) + +// See const.go for overview of math here. + +// Test that powx is initialized correctly. +// (Can adapt this code to generate it too.) +func TestPowx(t *testing.T) { + p := 1 + for i := 0; i < len(powx); i++ { + if powx[i] != byte(p) { + t.Errorf("powx[%d] = %#x, want %#x", i, powx[i], p) + } + p <<= 1 + if p&0x100 != 0 { + p ^= poly + } + } +} + +// Multiply b and c as GF(2) polynomials modulo poly +func mul(b, c uint32) uint32 { + i := b + j := c + s := uint32(0) + for k := uint32(1); k < 0x100 && j != 0; k <<= 1 { + // Invariant: k == 1<>8 + } + } +} + +// Test that decryption tables are correct. +// (Can adapt this code to generate them too.) +func TestTd(t *testing.T) { + for i := 0; i < 256; i++ { + s := uint32(sbox1[i]) + s9 := mul(s, 0x9) + sb := mul(s, 0xb) + sd := mul(s, 0xd) + se := mul(s, 0xe) + w := se<<24 | s9<<16 | sd<<8 | sb + for j := 0; j < 4; j++ { + if x := td[j][i]; x != w { + t.Fatalf("td[%d][%d] = %#x, want %#x", j, i, x, w) + } + w = w<<24 | w>>8 + } + } +} + +// Test vectors are from FIPS 197: +// http://www.csrc.nist.gov/publications/fips/fips197/fips-197.pdf + +// Appendix A of FIPS 197: Key expansion examples +type KeyTest struct { + key []byte + enc []uint32 + dec []uint32 // decryption expansion; not in FIPS 197, computed from C implementation. +} + +var keyTests = []KeyTest{ + { + // A.1. Expansion of a 128-bit Cipher Key + []byte{0x2b, 0x7e, 0x15, 0x16, 0x28, 0xae, 0xd2, 0xa6, 0xab, 0xf7, 0x15, 0x88, 0x09, 0xcf, 0x4f, 0x3c}, + []uint32{ + 0x2b7e1516, 0x28aed2a6, 0xabf71588, 0x09cf4f3c, + 0xa0fafe17, 0x88542cb1, 0x23a33939, 0x2a6c7605, + 0xf2c295f2, 0x7a96b943, 0x5935807a, 0x7359f67f, + 0x3d80477d, 0x4716fe3e, 0x1e237e44, 0x6d7a883b, + 0xef44a541, 0xa8525b7f, 0xb671253b, 0xdb0bad00, + 0xd4d1c6f8, 0x7c839d87, 0xcaf2b8bc, 0x11f915bc, + 0x6d88a37a, 0x110b3efd, 0xdbf98641, 0xca0093fd, + 0x4e54f70e, 0x5f5fc9f3, 0x84a64fb2, 0x4ea6dc4f, + 0xead27321, 0xb58dbad2, 0x312bf560, 0x7f8d292f, + 0xac7766f3, 0x19fadc21, 0x28d12941, 0x575c006e, + 0xd014f9a8, 0xc9ee2589, 0xe13f0cc8, 0xb6630ca6, + }, + []uint32{ + 0xd014f9a8, 0xc9ee2589, 0xe13f0cc8, 0xb6630ca6, + 0xc7b5a63, 0x1319eafe, 0xb0398890, 0x664cfbb4, + 0xdf7d925a, 0x1f62b09d, 0xa320626e, 0xd6757324, + 0x12c07647, 0xc01f22c7, 0xbc42d2f3, 0x7555114a, + 0x6efcd876, 0xd2df5480, 0x7c5df034, 0xc917c3b9, + 0x6ea30afc, 0xbc238cf6, 0xae82a4b4, 0xb54a338d, + 0x90884413, 0xd280860a, 0x12a12842, 0x1bc89739, + 0x7c1f13f7, 0x4208c219, 0xc021ae48, 0x969bf7b, + 0xcc7505eb, 0x3e17d1ee, 0x82296c51, 0xc9481133, + 0x2b3708a7, 0xf262d405, 0xbc3ebdbf, 0x4b617d62, + 0x2b7e1516, 0x28aed2a6, 0xabf71588, 0x9cf4f3c, + }, + }, + { + // A.2. Expansion of a 192-bit Cipher Key + []byte{ + 0x8e, 0x73, 0xb0, 0xf7, 0xda, 0x0e, 0x64, 0x52, 0xc8, 0x10, 0xf3, 0x2b, 0x80, 0x90, 0x79, 0xe5, + 0x62, 0xf8, 0xea, 0xd2, 0x52, 0x2c, 0x6b, 0x7b, + }, + []uint32{ + 0x8e73b0f7, 0xda0e6452, 0xc810f32b, 0x809079e5, + 0x62f8ead2, 0x522c6b7b, 0xfe0c91f7, 0x2402f5a5, + 0xec12068e, 0x6c827f6b, 0x0e7a95b9, 0x5c56fec2, + 0x4db7b4bd, 0x69b54118, 0x85a74796, 0xe92538fd, + 0xe75fad44, 0xbb095386, 0x485af057, 0x21efb14f, + 0xa448f6d9, 0x4d6dce24, 0xaa326360, 0x113b30e6, + 0xa25e7ed5, 0x83b1cf9a, 0x27f93943, 0x6a94f767, + 0xc0a69407, 0xd19da4e1, 0xec1786eb, 0x6fa64971, + 0x485f7032, 0x22cb8755, 0xe26d1352, 0x33f0b7b3, + 0x40beeb28, 0x2f18a259, 0x6747d26b, 0x458c553e, + 0xa7e1466c, 0x9411f1df, 0x821f750a, 0xad07d753, + 0xca400538, 0x8fcc5006, 0x282d166a, 0xbc3ce7b5, + 0xe98ba06f, 0x448c773c, 0x8ecc7204, 0x01002202, + }, + nil, + }, + { + // A.3. Expansion of a 256-bit Cipher Key + []byte{ + 0x60, 0x3d, 0xeb, 0x10, 0x15, 0xca, 0x71, 0xbe, 0x2b, 0x73, 0xae, 0xf0, 0x85, 0x7d, 0x77, 0x81, + 0x1f, 0x35, 0x2c, 0x07, 0x3b, 0x61, 0x08, 0xd7, 0x2d, 0x98, 0x10, 0xa3, 0x09, 0x14, 0xdf, 0xf4, + }, + []uint32{ + 0x603deb10, 0x15ca71be, 0x2b73aef0, 0x857d7781, + 0x1f352c07, 0x3b6108d7, 0x2d9810a3, 0x0914dff4, + 0x9ba35411, 0x8e6925af, 0xa51a8b5f, 0x2067fcde, + 0xa8b09c1a, 0x93d194cd, 0xbe49846e, 0xb75d5b9a, + 0xd59aecb8, 0x5bf3c917, 0xfee94248, 0xde8ebe96, + 0xb5a9328a, 0x2678a647, 0x98312229, 0x2f6c79b3, + 0x812c81ad, 0xdadf48ba, 0x24360af2, 0xfab8b464, + 0x98c5bfc9, 0xbebd198e, 0x268c3ba7, 0x09e04214, + 0x68007bac, 0xb2df3316, 0x96e939e4, 0x6c518d80, + 0xc814e204, 0x76a9fb8a, 0x5025c02d, 0x59c58239, + 0xde136967, 0x6ccc5a71, 0xfa256395, 0x9674ee15, + 0x5886ca5d, 0x2e2f31d7, 0x7e0af1fa, 0x27cf73c3, + 0x749c47ab, 0x18501dda, 0xe2757e4f, 0x7401905a, + 0xcafaaae3, 0xe4d59b34, 0x9adf6ace, 0xbd10190d, + 0xfe4890d1, 0xe6188d0b, 0x046df344, 0x706c631e, + }, + nil, + }, +} + +// Test key expansion against FIPS 197 examples. +func TestExpandKey(t *testing.T) { +L: + for i, tt := range keyTests { + enc := make([]uint32, len(tt.enc)) + var dec []uint32 + if tt.dec != nil { + dec = make([]uint32, len(tt.dec)) + } + expandKey(tt.key, enc, dec) + for j, v := range enc { + if v != tt.enc[j] { + t.Errorf("key %d: enc[%d] = %#x, want %#x", i, j, v, tt.enc[j]) + continue L + } + } + if dec != nil { + for j, v := range dec { + if v != tt.dec[j] { + t.Errorf("key %d: dec[%d] = %#x, want %#x", i, j, v, tt.dec[j]) + continue L + } + } + } + } +} + +// Appendix B, C of FIPS 197: Cipher examples, Example vectors. +type CryptTest struct { + key []byte + in []byte + out []byte +} + +var encryptTests = []CryptTest{ + { + // Appendix B. + []byte{0x2b, 0x7e, 0x15, 0x16, 0x28, 0xae, 0xd2, 0xa6, 0xab, 0xf7, 0x15, 0x88, 0x09, 0xcf, 0x4f, 0x3c}, + []byte{0x32, 0x43, 0xf6, 0xa8, 0x88, 0x5a, 0x30, 0x8d, 0x31, 0x31, 0x98, 0xa2, 0xe0, 0x37, 0x07, 0x34}, + []byte{0x39, 0x25, 0x84, 0x1d, 0x02, 0xdc, 0x09, 0xfb, 0xdc, 0x11, 0x85, 0x97, 0x19, 0x6a, 0x0b, 0x32}, + }, + { + // Appendix C.1. AES-128 + []byte{0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f}, + []byte{0x00, 0x11, 0x22, 0x33, 0x44, 0x55, 0x66, 0x77, 0x88, 0x99, 0xaa, 0xbb, 0xcc, 0xdd, 0xee, 0xff}, + []byte{0x69, 0xc4, 0xe0, 0xd8, 0x6a, 0x7b, 0x04, 0x30, 0xd8, 0xcd, 0xb7, 0x80, 0x70, 0xb4, 0xc5, 0x5a}, + }, + { + // Appendix C.2. AES-192 + []byte{0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f, + 0x10, 0x11, 0x12, 0x13, 0x14, 0x15, 0x16, 0x17, + }, + []byte{0x00, 0x11, 0x22, 0x33, 0x44, 0x55, 0x66, 0x77, 0x88, 0x99, 0xaa, 0xbb, 0xcc, 0xdd, 0xee, 0xff}, + []byte{0xdd, 0xa9, 0x7c, 0xa4, 0x86, 0x4c, 0xdf, 0xe0, 0x6e, 0xaf, 0x70, 0xa0, 0xec, 0x0d, 0x71, 0x91}, + }, + { + // Appendix C.3. AES-256 + []byte{0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f, + 0x10, 0x11, 0x12, 0x13, 0x14, 0x15, 0x16, 0x17, 0x18, 0x19, 0x1a, 0x1b, 0x1c, 0x1d, 0x1e, 0x1f, + }, + []byte{0x00, 0x11, 0x22, 0x33, 0x44, 0x55, 0x66, 0x77, 0x88, 0x99, 0xaa, 0xbb, 0xcc, 0xdd, 0xee, 0xff}, + []byte{0x8e, 0xa2, 0xb7, 0xca, 0x51, 0x67, 0x45, 0xbf, 0xea, 0xfc, 0x49, 0x90, 0x4b, 0x49, 0x60, 0x89}, + }, +} + +// Test encryptBlock against FIPS 197 examples. +func TestEncryptBlock(t *testing.T) { + for i, tt := range encryptTests { + n := len(tt.key) + 28 + enc := make([]uint32, n) + dec := make([]uint32, n) + expandKey(tt.key, enc, dec) + out := make([]byte, len(tt.in)) + encryptBlock(enc, out, tt.in) + for j, v := range out { + if v != tt.out[j] { + t.Errorf("encryptBlock %d: out[%d] = %#x, want %#x", i, j, v, tt.out[j]) + break + } + } + } +} + +// Test decryptBlock against FIPS 197 examples. +func TestDecryptBlock(t *testing.T) { + for i, tt := range encryptTests { + n := len(tt.key) + 28 + enc := make([]uint32, n) + dec := make([]uint32, n) + expandKey(tt.key, enc, dec) + plain := make([]byte, len(tt.in)) + decryptBlock(dec, plain, tt.out) + for j, v := range plain { + if v != tt.in[j] { + t.Errorf("decryptBlock %d: plain[%d] = %#x, want %#x", i, j, v, tt.in[j]) + break + } + } + } +} + +// Test Cipher Encrypt method against FIPS 197 examples. +func TestCipherEncrypt(t *testing.T) { + for i, tt := range encryptTests { + c, err := NewCipher(tt.key) + if err != nil { + t.Errorf("NewCipher(%d bytes) = %s", len(tt.key), err) + continue + } + out := make([]byte, len(tt.in)) + c.Encrypt(out, tt.in) + for j, v := range out { + if v != tt.out[j] { + t.Errorf("Cipher.Encrypt %d: out[%d] = %#x, want %#x", i, j, v, tt.out[j]) + break + } + } + } +} + +// Test Cipher Decrypt against FIPS 197 examples. +func TestCipherDecrypt(t *testing.T) { + for i, tt := range encryptTests { + c, err := NewCipher(tt.key) + if err != nil { + t.Errorf("NewCipher(%d bytes) = %s", len(tt.key), err) + continue + } + plain := make([]byte, len(tt.in)) + c.Decrypt(plain, tt.out) + for j, v := range plain { + if v != tt.in[j] { + t.Errorf("decryptBlock %d: plain[%d] = %#x, want %#x", i, j, v, tt.in[j]) + break + } + } + } +} diff --git a/src/pkg/crypto/aes/block.go b/src/pkg/crypto/aes/block.go new file mode 100644 index 000000000..130cd011c --- /dev/null +++ b/src/pkg/crypto/aes/block.go @@ -0,0 +1,176 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// This Go implementation is derived in part from the reference +// ANSI C implementation, which carries the following notice: +// +// rijndael-alg-fst.c +// +// @version 3.0 (December 2000) +// +// Optimised ANSI C code for the Rijndael cipher (now AES) +// +// @author Vincent Rijmen +// @author Antoon Bosselaers +// @author Paulo Barreto +// +// This code is hereby placed in the public domain. +// +// THIS SOFTWARE IS PROVIDED BY THE AUTHORS ''AS IS'' AND ANY EXPRESS +// OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +// WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +// ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHORS OR CONTRIBUTORS BE +// LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +// CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +// SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR +// BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, +// WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE +// OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, +// EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +// +// See FIPS 197 for specification, and see Daemen and Rijmen's Rijndael submission +// for implementation details. +// http://www.csrc.nist.gov/publications/fips/fips197/fips-197.pdf +// http://csrc.nist.gov/archive/aes/rijndael/Rijndael-ammended.pdf + +package aes + +// Encrypt one block from src into dst, using the expanded key xk. +func encryptBlock(xk []uint32, dst, src []byte) { + var s0, s1, s2, s3, t0, t1, t2, t3 uint32 + + s0 = uint32(src[0])<<24 | uint32(src[1])<<16 | uint32(src[2])<<8 | uint32(src[3]) + s1 = uint32(src[4])<<24 | uint32(src[5])<<16 | uint32(src[6])<<8 | uint32(src[7]) + s2 = uint32(src[8])<<24 | uint32(src[9])<<16 | uint32(src[10])<<8 | uint32(src[11]) + s3 = uint32(src[12])<<24 | uint32(src[13])<<16 | uint32(src[14])<<8 | uint32(src[15]) + + // First round just XORs input with key. + s0 ^= xk[0] + s1 ^= xk[1] + s2 ^= xk[2] + s3 ^= xk[3] + + // Middle rounds shuffle using tables. + // Number of rounds is set by length of expanded key. + nr := len(xk)/4 - 2 // - 2: one above, one more below + k := 4 + for r := 0; r < nr; r++ { + t0 = xk[k+0] ^ te[0][s0>>24] ^ te[1][s1>>16&0xff] ^ te[2][s2>>8&0xff] ^ te[3][s3&0xff] + t1 = xk[k+1] ^ te[0][s1>>24] ^ te[1][s2>>16&0xff] ^ te[2][s3>>8&0xff] ^ te[3][s0&0xff] + t2 = xk[k+2] ^ te[0][s2>>24] ^ te[1][s3>>16&0xff] ^ te[2][s0>>8&0xff] ^ te[3][s1&0xff] + t3 = xk[k+3] ^ te[0][s3>>24] ^ te[1][s0>>16&0xff] ^ te[2][s1>>8&0xff] ^ te[3][s2&0xff] + k += 4 + s0, s1, s2, s3 = t0, t1, t2, t3 + } + + // Last round uses s-box directly and XORs to produce output. + s0 = uint32(sbox0[t0>>24])<<24 | uint32(sbox0[t1>>16&0xff])<<16 | uint32(sbox0[t2>>8&0xff])<<8 | uint32(sbox0[t3&0xff]) + s1 = uint32(sbox0[t1>>24])<<24 | uint32(sbox0[t2>>16&0xff])<<16 | uint32(sbox0[t3>>8&0xff])<<8 | uint32(sbox0[t0&0xff]) + s2 = uint32(sbox0[t2>>24])<<24 | uint32(sbox0[t3>>16&0xff])<<16 | uint32(sbox0[t0>>8&0xff])<<8 | uint32(sbox0[t1&0xff]) + s3 = uint32(sbox0[t3>>24])<<24 | uint32(sbox0[t0>>16&0xff])<<16 | uint32(sbox0[t1>>8&0xff])<<8 | uint32(sbox0[t2&0xff]) + + s0 ^= xk[k+0] + s1 ^= xk[k+1] + s2 ^= xk[k+2] + s3 ^= xk[k+3] + + dst[0], dst[1], dst[2], dst[3] = byte(s0>>24), byte(s0>>16), byte(s0>>8), byte(s0) + dst[4], dst[5], dst[6], dst[7] = byte(s1>>24), byte(s1>>16), byte(s1>>8), byte(s1) + dst[8], dst[9], dst[10], dst[11] = byte(s2>>24), byte(s2>>16), byte(s2>>8), byte(s2) + dst[12], dst[13], dst[14], dst[15] = byte(s3>>24), byte(s3>>16), byte(s3>>8), byte(s3) +} + +// Decrypt one block from src into dst, using the expanded key xk. +func decryptBlock(xk []uint32, dst, src []byte) { + var s0, s1, s2, s3, t0, t1, t2, t3 uint32 + + s0 = uint32(src[0])<<24 | uint32(src[1])<<16 | uint32(src[2])<<8 | uint32(src[3]) + s1 = uint32(src[4])<<24 | uint32(src[5])<<16 | uint32(src[6])<<8 | uint32(src[7]) + s2 = uint32(src[8])<<24 | uint32(src[9])<<16 | uint32(src[10])<<8 | uint32(src[11]) + s3 = uint32(src[12])<<24 | uint32(src[13])<<16 | uint32(src[14])<<8 | uint32(src[15]) + + // First round just XORs input with key. + s0 ^= xk[0] + s1 ^= xk[1] + s2 ^= xk[2] + s3 ^= xk[3] + + // Middle rounds shuffle using tables. + // Number of rounds is set by length of expanded key. + nr := len(xk)/4 - 2 // - 2: one above, one more below + k := 4 + for r := 0; r < nr; r++ { + t0 = xk[k+0] ^ td[0][s0>>24] ^ td[1][s3>>16&0xff] ^ td[2][s2>>8&0xff] ^ td[3][s1&0xff] + t1 = xk[k+1] ^ td[0][s1>>24] ^ td[1][s0>>16&0xff] ^ td[2][s3>>8&0xff] ^ td[3][s2&0xff] + t2 = xk[k+2] ^ td[0][s2>>24] ^ td[1][s1>>16&0xff] ^ td[2][s0>>8&0xff] ^ td[3][s3&0xff] + t3 = xk[k+3] ^ td[0][s3>>24] ^ td[1][s2>>16&0xff] ^ td[2][s1>>8&0xff] ^ td[3][s0&0xff] + k += 4 + s0, s1, s2, s3 = t0, t1, t2, t3 + } + + // Last round uses s-box directly and XORs to produce output. + s0 = uint32(sbox1[t0>>24])<<24 | uint32(sbox1[t3>>16&0xff])<<16 | uint32(sbox1[t2>>8&0xff])<<8 | uint32(sbox1[t1&0xff]) + s1 = uint32(sbox1[t1>>24])<<24 | uint32(sbox1[t0>>16&0xff])<<16 | uint32(sbox1[t3>>8&0xff])<<8 | uint32(sbox1[t2&0xff]) + s2 = uint32(sbox1[t2>>24])<<24 | uint32(sbox1[t1>>16&0xff])<<16 | uint32(sbox1[t0>>8&0xff])<<8 | uint32(sbox1[t3&0xff]) + s3 = uint32(sbox1[t3>>24])<<24 | uint32(sbox1[t2>>16&0xff])<<16 | uint32(sbox1[t1>>8&0xff])<<8 | uint32(sbox1[t0&0xff]) + + s0 ^= xk[k+0] + s1 ^= xk[k+1] + s2 ^= xk[k+2] + s3 ^= xk[k+3] + + dst[0], dst[1], dst[2], dst[3] = byte(s0>>24), byte(s0>>16), byte(s0>>8), byte(s0) + dst[4], dst[5], dst[6], dst[7] = byte(s1>>24), byte(s1>>16), byte(s1>>8), byte(s1) + dst[8], dst[9], dst[10], dst[11] = byte(s2>>24), byte(s2>>16), byte(s2>>8), byte(s2) + dst[12], dst[13], dst[14], dst[15] = byte(s3>>24), byte(s3>>16), byte(s3>>8), byte(s3) +} + +// Apply sbox0 to each byte in w. +func subw(w uint32) uint32 { + return uint32(sbox0[w>>24])<<24 | + uint32(sbox0[w>>16&0xff])<<16 | + uint32(sbox0[w>>8&0xff])<<8 | + uint32(sbox0[w&0xff]) +} + +// Rotate +func rotw(w uint32) uint32 { return w<<8 | w>>24 } + +// Key expansion algorithm. See FIPS-197, Figure 11. +// Their rcon[i] is our powx[i-1] << 24. +func expandKey(key []byte, enc, dec []uint32) { + // Encryption key setup. + var i int + nk := len(key) / 4 + for i = 0; i < nk; i++ { + enc[i] = uint32(key[4*i])<<24 | uint32(key[4*i+1])<<16 | uint32(key[4*i+2])<<8 | uint32(key[4*i+3]) + } + for ; i < len(enc); i++ { + t := enc[i-1] + if i%nk == 0 { + t = subw(rotw(t)) ^ (uint32(powx[i/nk-1]) << 24) + } else if nk > 6 && i%nk == 4 { + t = subw(t) + } + enc[i] = enc[i-nk] ^ t + } + + // Derive decryption key from encryption key. + // Reverse the 4-word round key sets from enc to produce dec. + // All sets but the first and last get the MixColumn transform applied. + if dec == nil { + return + } + n := len(enc) + for i := 0; i < n; i += 4 { + ei := n - i - 4 + for j := 0; j < 4; j++ { + x := enc[ei+j] + if i > 0 && i+4 < n { + x = td[0][sbox0[x>>24]] ^ td[1][sbox0[x>>16&0xff]] ^ td[2][sbox0[x>>8&0xff]] ^ td[3][sbox0[x&0xff]] + } + dec[i+j] = x + } + } +} diff --git a/src/pkg/crypto/aes/cipher.go b/src/pkg/crypto/aes/cipher.go new file mode 100644 index 000000000..73223531e --- /dev/null +++ b/src/pkg/crypto/aes/cipher.go @@ -0,0 +1,71 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package aes + +import ( + "os" + "strconv" +) + +// The AES block size in bytes. +const BlockSize = 16 + +// A Cipher is an instance of AES encryption using a particular key. +type Cipher struct { + enc []uint32 + dec []uint32 +} + +type KeySizeError int + +func (k KeySizeError) String() string { + return "crypto/aes: invalid key size " + strconv.Itoa(int(k)) +} + +// NewCipher creates and returns a new Cipher. +// The key argument should be the AES key, +// either 16, 24, or 32 bytes to select +// AES-128, AES-192, or AES-256. +func NewCipher(key []byte) (*Cipher, os.Error) { + k := len(key) + switch k { + default: + return nil, KeySizeError(k) + case 16, 24, 32: + break + } + + n := k + 28 + c := &Cipher{make([]uint32, n), make([]uint32, n)} + expandKey(key, c.enc, c.dec) + return c, nil +} + +// BlockSize returns the AES block size, 16 bytes. +// It is necessary to satisfy the Cipher interface in the +// package "crypto/cipher". +func (c *Cipher) BlockSize() int { return BlockSize } + +// Encrypt encrypts the 16-byte buffer src using the key k +// and stores the result in dst. +// Note that for amounts of data larger than a block, +// it is not safe to just call Encrypt on successive blocks; +// instead, use an encryption mode like CBC (see crypto/cipher/cbc.go). +func (c *Cipher) Encrypt(dst, src []byte) { encryptBlock(c.enc, dst, src) } + +// Decrypt decrypts the 16-byte buffer src using the key k +// and stores the result in dst. +func (c *Cipher) Decrypt(dst, src []byte) { decryptBlock(c.dec, dst, src) } + +// Reset zeros the key data, so that it will no longer +// appear in the process's memory. +func (c *Cipher) Reset() { + for i := 0; i < len(c.enc); i++ { + c.enc[i] = 0 + } + for i := 0; i < len(c.dec); i++ { + c.dec[i] = 0 + } +} diff --git a/src/pkg/crypto/aes/const.go b/src/pkg/crypto/aes/const.go new file mode 100644 index 000000000..25acd0d17 --- /dev/null +++ b/src/pkg/crypto/aes/const.go @@ -0,0 +1,362 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Package aes implements AES encryption (formerly Rijndael), as defined in +// U.S. Federal Information Processing Standards Publication 197. +package aes + +// This file contains AES constants - 8720 bytes of initialized data. + +// http://www.csrc.nist.gov/publications/fips/fips197/fips-197.pdf + +// AES is based on the mathematical behavior of binary polynomials +// (polynomials over GF(2)) modulo the irreducible polynomial x⁸ + x⁴ + x² + x + 1. +// Addition of these binary polynomials corresponds to binary xor. +// Reducing mod poly corresponds to binary xor with poly every +// time a 0x100 bit appears. +const poly = 1<<8 | 1<<4 | 1<<3 | 1<<1 | 1<<0 // x⁸ + x⁴ + x² + x + 1 + +// Powers of x mod poly in GF(2). +var powx = [16]byte{ + 0x01, + 0x02, + 0x04, + 0x08, + 0x10, + 0x20, + 0x40, + 0x80, + 0x1b, + 0x36, + 0x6c, + 0xd8, + 0xab, + 0x4d, + 0x9a, + 0x2f, +} + +// FIPS-197 Figure 7. S-box substitution values in hexadecimal format. +var sbox0 = [256]byte{ + 0x63, 0x7c, 0x77, 0x7b, 0xf2, 0x6b, 0x6f, 0xc5, 0x30, 0x01, 0x67, 0x2b, 0xfe, 0xd7, 0xab, 0x76, + 0xca, 0x82, 0xc9, 0x7d, 0xfa, 0x59, 0x47, 0xf0, 0xad, 0xd4, 0xa2, 0xaf, 0x9c, 0xa4, 0x72, 0xc0, + 0xb7, 0xfd, 0x93, 0x26, 0x36, 0x3f, 0xf7, 0xcc, 0x34, 0xa5, 0xe5, 0xf1, 0x71, 0xd8, 0x31, 0x15, + 0x04, 0xc7, 0x23, 0xc3, 0x18, 0x96, 0x05, 0x9a, 0x07, 0x12, 0x80, 0xe2, 0xeb, 0x27, 0xb2, 0x75, + 0x09, 0x83, 0x2c, 0x1a, 0x1b, 0x6e, 0x5a, 0xa0, 0x52, 0x3b, 0xd6, 0xb3, 0x29, 0xe3, 0x2f, 0x84, + 0x53, 0xd1, 0x00, 0xed, 0x20, 0xfc, 0xb1, 0x5b, 0x6a, 0xcb, 0xbe, 0x39, 0x4a, 0x4c, 0x58, 0xcf, + 0xd0, 0xef, 0xaa, 0xfb, 0x43, 0x4d, 0x33, 0x85, 0x45, 0xf9, 0x02, 0x7f, 0x50, 0x3c, 0x9f, 0xa8, + 0x51, 0xa3, 0x40, 0x8f, 0x92, 0x9d, 0x38, 0xf5, 0xbc, 0xb6, 0xda, 0x21, 0x10, 0xff, 0xf3, 0xd2, + 0xcd, 0x0c, 0x13, 0xec, 0x5f, 0x97, 0x44, 0x17, 0xc4, 0xa7, 0x7e, 0x3d, 0x64, 0x5d, 0x19, 0x73, + 0x60, 0x81, 0x4f, 0xdc, 0x22, 0x2a, 0x90, 0x88, 0x46, 0xee, 0xb8, 0x14, 0xde, 0x5e, 0x0b, 0xdb, + 0xe0, 0x32, 0x3a, 0x0a, 0x49, 0x06, 0x24, 0x5c, 0xc2, 0xd3, 0xac, 0x62, 0x91, 0x95, 0xe4, 0x79, + 0xe7, 0xc8, 0x37, 0x6d, 0x8d, 0xd5, 0x4e, 0xa9, 0x6c, 0x56, 0xf4, 0xea, 0x65, 0x7a, 0xae, 0x08, + 0xba, 0x78, 0x25, 0x2e, 0x1c, 0xa6, 0xb4, 0xc6, 0xe8, 0xdd, 0x74, 0x1f, 0x4b, 0xbd, 0x8b, 0x8a, + 0x70, 0x3e, 0xb5, 0x66, 0x48, 0x03, 0xf6, 0x0e, 0x61, 0x35, 0x57, 0xb9, 0x86, 0xc1, 0x1d, 0x9e, + 0xe1, 0xf8, 0x98, 0x11, 0x69, 0xd9, 0x8e, 0x94, 0x9b, 0x1e, 0x87, 0xe9, 0xce, 0x55, 0x28, 0xdf, + 0x8c, 0xa1, 0x89, 0x0d, 0xbf, 0xe6, 0x42, 0x68, 0x41, 0x99, 0x2d, 0x0f, 0xb0, 0x54, 0xbb, 0x16, +} + +// FIPS-197 Figure 14. Inverse S-box substitution values in hexadecimal format. +var sbox1 = [256]byte{ + 0x52, 0x09, 0x6a, 0xd5, 0x30, 0x36, 0xa5, 0x38, 0xbf, 0x40, 0xa3, 0x9e, 0x81, 0xf3, 0xd7, 0xfb, + 0x7c, 0xe3, 0x39, 0x82, 0x9b, 0x2f, 0xff, 0x87, 0x34, 0x8e, 0x43, 0x44, 0xc4, 0xde, 0xe9, 0xcb, + 0x54, 0x7b, 0x94, 0x32, 0xa6, 0xc2, 0x23, 0x3d, 0xee, 0x4c, 0x95, 0x0b, 0x42, 0xfa, 0xc3, 0x4e, + 0x08, 0x2e, 0xa1, 0x66, 0x28, 0xd9, 0x24, 0xb2, 0x76, 0x5b, 0xa2, 0x49, 0x6d, 0x8b, 0xd1, 0x25, + 0x72, 0xf8, 0xf6, 0x64, 0x86, 0x68, 0x98, 0x16, 0xd4, 0xa4, 0x5c, 0xcc, 0x5d, 0x65, 0xb6, 0x92, + 0x6c, 0x70, 0x48, 0x50, 0xfd, 0xed, 0xb9, 0xda, 0x5e, 0x15, 0x46, 0x57, 0xa7, 0x8d, 0x9d, 0x84, + 0x90, 0xd8, 0xab, 0x00, 0x8c, 0xbc, 0xd3, 0x0a, 0xf7, 0xe4, 0x58, 0x05, 0xb8, 0xb3, 0x45, 0x06, + 0xd0, 0x2c, 0x1e, 0x8f, 0xca, 0x3f, 0x0f, 0x02, 0xc1, 0xaf, 0xbd, 0x03, 0x01, 0x13, 0x8a, 0x6b, + 0x3a, 0x91, 0x11, 0x41, 0x4f, 0x67, 0xdc, 0xea, 0x97, 0xf2, 0xcf, 0xce, 0xf0, 0xb4, 0xe6, 0x73, + 0x96, 0xac, 0x74, 0x22, 0xe7, 0xad, 0x35, 0x85, 0xe2, 0xf9, 0x37, 0xe8, 0x1c, 0x75, 0xdf, 0x6e, + 0x47, 0xf1, 0x1a, 0x71, 0x1d, 0x29, 0xc5, 0x89, 0x6f, 0xb7, 0x62, 0x0e, 0xaa, 0x18, 0xbe, 0x1b, + 0xfc, 0x56, 0x3e, 0x4b, 0xc6, 0xd2, 0x79, 0x20, 0x9a, 0xdb, 0xc0, 0xfe, 0x78, 0xcd, 0x5a, 0xf4, + 0x1f, 0xdd, 0xa8, 0x33, 0x88, 0x07, 0xc7, 0x31, 0xb1, 0x12, 0x10, 0x59, 0x27, 0x80, 0xec, 0x5f, + 0x60, 0x51, 0x7f, 0xa9, 0x19, 0xb5, 0x4a, 0x0d, 0x2d, 0xe5, 0x7a, 0x9f, 0x93, 0xc9, 0x9c, 0xef, + 0xa0, 0xe0, 0x3b, 0x4d, 0xae, 0x2a, 0xf5, 0xb0, 0xc8, 0xeb, 0xbb, 0x3c, 0x83, 0x53, 0x99, 0x61, + 0x17, 0x2b, 0x04, 0x7e, 0xba, 0x77, 0xd6, 0x26, 0xe1, 0x69, 0x14, 0x63, 0x55, 0x21, 0x0c, 0x7d, +} + +// Lookup tables for encryption. +// These can be recomputed by adapting the tests in aes_test.go. + +var te = [4][256]uint32{ + { + 0xc66363a5, 0xf87c7c84, 0xee777799, 0xf67b7b8d, 0xfff2f20d, 0xd66b6bbd, 0xde6f6fb1, 0x91c5c554, + 0x60303050, 0x02010103, 0xce6767a9, 0x562b2b7d, 0xe7fefe19, 0xb5d7d762, 0x4dababe6, 0xec76769a, + 0x8fcaca45, 0x1f82829d, 0x89c9c940, 0xfa7d7d87, 0xeffafa15, 0xb25959eb, 0x8e4747c9, 0xfbf0f00b, + 0x41adadec, 0xb3d4d467, 0x5fa2a2fd, 0x45afafea, 0x239c9cbf, 0x53a4a4f7, 0xe4727296, 0x9bc0c05b, + 0x75b7b7c2, 0xe1fdfd1c, 0x3d9393ae, 0x4c26266a, 0x6c36365a, 0x7e3f3f41, 0xf5f7f702, 0x83cccc4f, + 0x6834345c, 0x51a5a5f4, 0xd1e5e534, 0xf9f1f108, 0xe2717193, 0xabd8d873, 0x62313153, 0x2a15153f, + 0x0804040c, 0x95c7c752, 0x46232365, 0x9dc3c35e, 0x30181828, 0x379696a1, 0x0a05050f, 0x2f9a9ab5, + 0x0e070709, 0x24121236, 0x1b80809b, 0xdfe2e23d, 0xcdebeb26, 0x4e272769, 0x7fb2b2cd, 0xea75759f, + 0x1209091b, 0x1d83839e, 0x582c2c74, 0x341a1a2e, 0x361b1b2d, 0xdc6e6eb2, 0xb45a5aee, 0x5ba0a0fb, + 0xa45252f6, 0x763b3b4d, 0xb7d6d661, 0x7db3b3ce, 0x5229297b, 0xdde3e33e, 0x5e2f2f71, 0x13848497, + 0xa65353f5, 0xb9d1d168, 0x00000000, 0xc1eded2c, 0x40202060, 0xe3fcfc1f, 0x79b1b1c8, 0xb65b5bed, + 0xd46a6abe, 0x8dcbcb46, 0x67bebed9, 0x7239394b, 0x944a4ade, 0x984c4cd4, 0xb05858e8, 0x85cfcf4a, + 0xbbd0d06b, 0xc5efef2a, 0x4faaaae5, 0xedfbfb16, 0x864343c5, 0x9a4d4dd7, 0x66333355, 0x11858594, + 0x8a4545cf, 0xe9f9f910, 0x04020206, 0xfe7f7f81, 0xa05050f0, 0x783c3c44, 0x259f9fba, 0x4ba8a8e3, + 0xa25151f3, 0x5da3a3fe, 0x804040c0, 0x058f8f8a, 0x3f9292ad, 0x219d9dbc, 0x70383848, 0xf1f5f504, + 0x63bcbcdf, 0x77b6b6c1, 0xafdada75, 0x42212163, 0x20101030, 0xe5ffff1a, 0xfdf3f30e, 0xbfd2d26d, + 0x81cdcd4c, 0x180c0c14, 0x26131335, 0xc3ecec2f, 0xbe5f5fe1, 0x359797a2, 0x884444cc, 0x2e171739, + 0x93c4c457, 0x55a7a7f2, 0xfc7e7e82, 0x7a3d3d47, 0xc86464ac, 0xba5d5de7, 0x3219192b, 0xe6737395, + 0xc06060a0, 0x19818198, 0x9e4f4fd1, 0xa3dcdc7f, 0x44222266, 0x542a2a7e, 0x3b9090ab, 0x0b888883, + 0x8c4646ca, 0xc7eeee29, 0x6bb8b8d3, 0x2814143c, 0xa7dede79, 0xbc5e5ee2, 0x160b0b1d, 0xaddbdb76, + 0xdbe0e03b, 0x64323256, 0x743a3a4e, 0x140a0a1e, 0x924949db, 0x0c06060a, 0x4824246c, 0xb85c5ce4, + 0x9fc2c25d, 0xbdd3d36e, 0x43acacef, 0xc46262a6, 0x399191a8, 0x319595a4, 0xd3e4e437, 0xf279798b, + 0xd5e7e732, 0x8bc8c843, 0x6e373759, 0xda6d6db7, 0x018d8d8c, 0xb1d5d564, 0x9c4e4ed2, 0x49a9a9e0, + 0xd86c6cb4, 0xac5656fa, 0xf3f4f407, 0xcfeaea25, 0xca6565af, 0xf47a7a8e, 0x47aeaee9, 0x10080818, + 0x6fbabad5, 0xf0787888, 0x4a25256f, 0x5c2e2e72, 0x381c1c24, 0x57a6a6f1, 0x73b4b4c7, 0x97c6c651, + 0xcbe8e823, 0xa1dddd7c, 0xe874749c, 0x3e1f1f21, 0x964b4bdd, 0x61bdbddc, 0x0d8b8b86, 0x0f8a8a85, + 0xe0707090, 0x7c3e3e42, 0x71b5b5c4, 0xcc6666aa, 0x904848d8, 0x06030305, 0xf7f6f601, 0x1c0e0e12, + 0xc26161a3, 0x6a35355f, 0xae5757f9, 0x69b9b9d0, 0x17868691, 0x99c1c158, 0x3a1d1d27, 0x279e9eb9, + 0xd9e1e138, 0xebf8f813, 0x2b9898b3, 0x22111133, 0xd26969bb, 0xa9d9d970, 0x078e8e89, 0x339494a7, + 0x2d9b9bb6, 0x3c1e1e22, 0x15878792, 0xc9e9e920, 0x87cece49, 0xaa5555ff, 0x50282878, 0xa5dfdf7a, + 0x038c8c8f, 0x59a1a1f8, 0x09898980, 0x1a0d0d17, 0x65bfbfda, 0xd7e6e631, 0x844242c6, 0xd06868b8, + 0x824141c3, 0x299999b0, 0x5a2d2d77, 0x1e0f0f11, 0x7bb0b0cb, 0xa85454fc, 0x6dbbbbd6, 0x2c16163a, + }, + { + 0xa5c66363, 0x84f87c7c, 0x99ee7777, 0x8df67b7b, 0x0dfff2f2, 0xbdd66b6b, 0xb1de6f6f, 0x5491c5c5, + 0x50603030, 0x03020101, 0xa9ce6767, 0x7d562b2b, 0x19e7fefe, 0x62b5d7d7, 0xe64dabab, 0x9aec7676, + 0x458fcaca, 0x9d1f8282, 0x4089c9c9, 0x87fa7d7d, 0x15effafa, 0xebb25959, 0xc98e4747, 0x0bfbf0f0, + 0xec41adad, 0x67b3d4d4, 0xfd5fa2a2, 0xea45afaf, 0xbf239c9c, 0xf753a4a4, 0x96e47272, 0x5b9bc0c0, + 0xc275b7b7, 0x1ce1fdfd, 0xae3d9393, 0x6a4c2626, 0x5a6c3636, 0x417e3f3f, 0x02f5f7f7, 0x4f83cccc, + 0x5c683434, 0xf451a5a5, 0x34d1e5e5, 0x08f9f1f1, 0x93e27171, 0x73abd8d8, 0x53623131, 0x3f2a1515, + 0x0c080404, 0x5295c7c7, 0x65462323, 0x5e9dc3c3, 0x28301818, 0xa1379696, 0x0f0a0505, 0xb52f9a9a, + 0x090e0707, 0x36241212, 0x9b1b8080, 0x3ddfe2e2, 0x26cdebeb, 0x694e2727, 0xcd7fb2b2, 0x9fea7575, + 0x1b120909, 0x9e1d8383, 0x74582c2c, 0x2e341a1a, 0x2d361b1b, 0xb2dc6e6e, 0xeeb45a5a, 0xfb5ba0a0, + 0xf6a45252, 0x4d763b3b, 0x61b7d6d6, 0xce7db3b3, 0x7b522929, 0x3edde3e3, 0x715e2f2f, 0x97138484, + 0xf5a65353, 0x68b9d1d1, 0x00000000, 0x2cc1eded, 0x60402020, 0x1fe3fcfc, 0xc879b1b1, 0xedb65b5b, + 0xbed46a6a, 0x468dcbcb, 0xd967bebe, 0x4b723939, 0xde944a4a, 0xd4984c4c, 0xe8b05858, 0x4a85cfcf, + 0x6bbbd0d0, 0x2ac5efef, 0xe54faaaa, 0x16edfbfb, 0xc5864343, 0xd79a4d4d, 0x55663333, 0x94118585, + 0xcf8a4545, 0x10e9f9f9, 0x06040202, 0x81fe7f7f, 0xf0a05050, 0x44783c3c, 0xba259f9f, 0xe34ba8a8, + 0xf3a25151, 0xfe5da3a3, 0xc0804040, 0x8a058f8f, 0xad3f9292, 0xbc219d9d, 0x48703838, 0x04f1f5f5, + 0xdf63bcbc, 0xc177b6b6, 0x75afdada, 0x63422121, 0x30201010, 0x1ae5ffff, 0x0efdf3f3, 0x6dbfd2d2, + 0x4c81cdcd, 0x14180c0c, 0x35261313, 0x2fc3ecec, 0xe1be5f5f, 0xa2359797, 0xcc884444, 0x392e1717, + 0x5793c4c4, 0xf255a7a7, 0x82fc7e7e, 0x477a3d3d, 0xacc86464, 0xe7ba5d5d, 0x2b321919, 0x95e67373, + 0xa0c06060, 0x98198181, 0xd19e4f4f, 0x7fa3dcdc, 0x66442222, 0x7e542a2a, 0xab3b9090, 0x830b8888, + 0xca8c4646, 0x29c7eeee, 0xd36bb8b8, 0x3c281414, 0x79a7dede, 0xe2bc5e5e, 0x1d160b0b, 0x76addbdb, + 0x3bdbe0e0, 0x56643232, 0x4e743a3a, 0x1e140a0a, 0xdb924949, 0x0a0c0606, 0x6c482424, 0xe4b85c5c, + 0x5d9fc2c2, 0x6ebdd3d3, 0xef43acac, 0xa6c46262, 0xa8399191, 0xa4319595, 0x37d3e4e4, 0x8bf27979, + 0x32d5e7e7, 0x438bc8c8, 0x596e3737, 0xb7da6d6d, 0x8c018d8d, 0x64b1d5d5, 0xd29c4e4e, 0xe049a9a9, + 0xb4d86c6c, 0xfaac5656, 0x07f3f4f4, 0x25cfeaea, 0xafca6565, 0x8ef47a7a, 0xe947aeae, 0x18100808, + 0xd56fbaba, 0x88f07878, 0x6f4a2525, 0x725c2e2e, 0x24381c1c, 0xf157a6a6, 0xc773b4b4, 0x5197c6c6, + 0x23cbe8e8, 0x7ca1dddd, 0x9ce87474, 0x213e1f1f, 0xdd964b4b, 0xdc61bdbd, 0x860d8b8b, 0x850f8a8a, + 0x90e07070, 0x427c3e3e, 0xc471b5b5, 0xaacc6666, 0xd8904848, 0x05060303, 0x01f7f6f6, 0x121c0e0e, + 0xa3c26161, 0x5f6a3535, 0xf9ae5757, 0xd069b9b9, 0x91178686, 0x5899c1c1, 0x273a1d1d, 0xb9279e9e, + 0x38d9e1e1, 0x13ebf8f8, 0xb32b9898, 0x33221111, 0xbbd26969, 0x70a9d9d9, 0x89078e8e, 0xa7339494, + 0xb62d9b9b, 0x223c1e1e, 0x92158787, 0x20c9e9e9, 0x4987cece, 0xffaa5555, 0x78502828, 0x7aa5dfdf, + 0x8f038c8c, 0xf859a1a1, 0x80098989, 0x171a0d0d, 0xda65bfbf, 0x31d7e6e6, 0xc6844242, 0xb8d06868, + 0xc3824141, 0xb0299999, 0x775a2d2d, 0x111e0f0f, 0xcb7bb0b0, 0xfca85454, 0xd66dbbbb, 0x3a2c1616, + }, + { + 0x63a5c663, 0x7c84f87c, 0x7799ee77, 0x7b8df67b, 0xf20dfff2, 0x6bbdd66b, 0x6fb1de6f, 0xc55491c5, + 0x30506030, 0x01030201, 0x67a9ce67, 0x2b7d562b, 0xfe19e7fe, 0xd762b5d7, 0xabe64dab, 0x769aec76, + 0xca458fca, 0x829d1f82, 0xc94089c9, 0x7d87fa7d, 0xfa15effa, 0x59ebb259, 0x47c98e47, 0xf00bfbf0, + 0xadec41ad, 0xd467b3d4, 0xa2fd5fa2, 0xafea45af, 0x9cbf239c, 0xa4f753a4, 0x7296e472, 0xc05b9bc0, + 0xb7c275b7, 0xfd1ce1fd, 0x93ae3d93, 0x266a4c26, 0x365a6c36, 0x3f417e3f, 0xf702f5f7, 0xcc4f83cc, + 0x345c6834, 0xa5f451a5, 0xe534d1e5, 0xf108f9f1, 0x7193e271, 0xd873abd8, 0x31536231, 0x153f2a15, + 0x040c0804, 0xc75295c7, 0x23654623, 0xc35e9dc3, 0x18283018, 0x96a13796, 0x050f0a05, 0x9ab52f9a, + 0x07090e07, 0x12362412, 0x809b1b80, 0xe23ddfe2, 0xeb26cdeb, 0x27694e27, 0xb2cd7fb2, 0x759fea75, + 0x091b1209, 0x839e1d83, 0x2c74582c, 0x1a2e341a, 0x1b2d361b, 0x6eb2dc6e, 0x5aeeb45a, 0xa0fb5ba0, + 0x52f6a452, 0x3b4d763b, 0xd661b7d6, 0xb3ce7db3, 0x297b5229, 0xe33edde3, 0x2f715e2f, 0x84971384, + 0x53f5a653, 0xd168b9d1, 0x00000000, 0xed2cc1ed, 0x20604020, 0xfc1fe3fc, 0xb1c879b1, 0x5bedb65b, + 0x6abed46a, 0xcb468dcb, 0xbed967be, 0x394b7239, 0x4ade944a, 0x4cd4984c, 0x58e8b058, 0xcf4a85cf, + 0xd06bbbd0, 0xef2ac5ef, 0xaae54faa, 0xfb16edfb, 0x43c58643, 0x4dd79a4d, 0x33556633, 0x85941185, + 0x45cf8a45, 0xf910e9f9, 0x02060402, 0x7f81fe7f, 0x50f0a050, 0x3c44783c, 0x9fba259f, 0xa8e34ba8, + 0x51f3a251, 0xa3fe5da3, 0x40c08040, 0x8f8a058f, 0x92ad3f92, 0x9dbc219d, 0x38487038, 0xf504f1f5, + 0xbcdf63bc, 0xb6c177b6, 0xda75afda, 0x21634221, 0x10302010, 0xff1ae5ff, 0xf30efdf3, 0xd26dbfd2, + 0xcd4c81cd, 0x0c14180c, 0x13352613, 0xec2fc3ec, 0x5fe1be5f, 0x97a23597, 0x44cc8844, 0x17392e17, + 0xc45793c4, 0xa7f255a7, 0x7e82fc7e, 0x3d477a3d, 0x64acc864, 0x5de7ba5d, 0x192b3219, 0x7395e673, + 0x60a0c060, 0x81981981, 0x4fd19e4f, 0xdc7fa3dc, 0x22664422, 0x2a7e542a, 0x90ab3b90, 0x88830b88, + 0x46ca8c46, 0xee29c7ee, 0xb8d36bb8, 0x143c2814, 0xde79a7de, 0x5ee2bc5e, 0x0b1d160b, 0xdb76addb, + 0xe03bdbe0, 0x32566432, 0x3a4e743a, 0x0a1e140a, 0x49db9249, 0x060a0c06, 0x246c4824, 0x5ce4b85c, + 0xc25d9fc2, 0xd36ebdd3, 0xacef43ac, 0x62a6c462, 0x91a83991, 0x95a43195, 0xe437d3e4, 0x798bf279, + 0xe732d5e7, 0xc8438bc8, 0x37596e37, 0x6db7da6d, 0x8d8c018d, 0xd564b1d5, 0x4ed29c4e, 0xa9e049a9, + 0x6cb4d86c, 0x56faac56, 0xf407f3f4, 0xea25cfea, 0x65afca65, 0x7a8ef47a, 0xaee947ae, 0x08181008, + 0xbad56fba, 0x7888f078, 0x256f4a25, 0x2e725c2e, 0x1c24381c, 0xa6f157a6, 0xb4c773b4, 0xc65197c6, + 0xe823cbe8, 0xdd7ca1dd, 0x749ce874, 0x1f213e1f, 0x4bdd964b, 0xbddc61bd, 0x8b860d8b, 0x8a850f8a, + 0x7090e070, 0x3e427c3e, 0xb5c471b5, 0x66aacc66, 0x48d89048, 0x03050603, 0xf601f7f6, 0x0e121c0e, + 0x61a3c261, 0x355f6a35, 0x57f9ae57, 0xb9d069b9, 0x86911786, 0xc15899c1, 0x1d273a1d, 0x9eb9279e, + 0xe138d9e1, 0xf813ebf8, 0x98b32b98, 0x11332211, 0x69bbd269, 0xd970a9d9, 0x8e89078e, 0x94a73394, + 0x9bb62d9b, 0x1e223c1e, 0x87921587, 0xe920c9e9, 0xce4987ce, 0x55ffaa55, 0x28785028, 0xdf7aa5df, + 0x8c8f038c, 0xa1f859a1, 0x89800989, 0x0d171a0d, 0xbfda65bf, 0xe631d7e6, 0x42c68442, 0x68b8d068, + 0x41c38241, 0x99b02999, 0x2d775a2d, 0x0f111e0f, 0xb0cb7bb0, 0x54fca854, 0xbbd66dbb, 0x163a2c16, + }, + { + 0x6363a5c6, 0x7c7c84f8, 0x777799ee, 0x7b7b8df6, 0xf2f20dff, 0x6b6bbdd6, 0x6f6fb1de, 0xc5c55491, + 0x30305060, 0x01010302, 0x6767a9ce, 0x2b2b7d56, 0xfefe19e7, 0xd7d762b5, 0xababe64d, 0x76769aec, + 0xcaca458f, 0x82829d1f, 0xc9c94089, 0x7d7d87fa, 0xfafa15ef, 0x5959ebb2, 0x4747c98e, 0xf0f00bfb, + 0xadadec41, 0xd4d467b3, 0xa2a2fd5f, 0xafafea45, 0x9c9cbf23, 0xa4a4f753, 0x727296e4, 0xc0c05b9b, + 0xb7b7c275, 0xfdfd1ce1, 0x9393ae3d, 0x26266a4c, 0x36365a6c, 0x3f3f417e, 0xf7f702f5, 0xcccc4f83, + 0x34345c68, 0xa5a5f451, 0xe5e534d1, 0xf1f108f9, 0x717193e2, 0xd8d873ab, 0x31315362, 0x15153f2a, + 0x04040c08, 0xc7c75295, 0x23236546, 0xc3c35e9d, 0x18182830, 0x9696a137, 0x05050f0a, 0x9a9ab52f, + 0x0707090e, 0x12123624, 0x80809b1b, 0xe2e23ddf, 0xebeb26cd, 0x2727694e, 0xb2b2cd7f, 0x75759fea, + 0x09091b12, 0x83839e1d, 0x2c2c7458, 0x1a1a2e34, 0x1b1b2d36, 0x6e6eb2dc, 0x5a5aeeb4, 0xa0a0fb5b, + 0x5252f6a4, 0x3b3b4d76, 0xd6d661b7, 0xb3b3ce7d, 0x29297b52, 0xe3e33edd, 0x2f2f715e, 0x84849713, + 0x5353f5a6, 0xd1d168b9, 0x00000000, 0xeded2cc1, 0x20206040, 0xfcfc1fe3, 0xb1b1c879, 0x5b5bedb6, + 0x6a6abed4, 0xcbcb468d, 0xbebed967, 0x39394b72, 0x4a4ade94, 0x4c4cd498, 0x5858e8b0, 0xcfcf4a85, + 0xd0d06bbb, 0xefef2ac5, 0xaaaae54f, 0xfbfb16ed, 0x4343c586, 0x4d4dd79a, 0x33335566, 0x85859411, + 0x4545cf8a, 0xf9f910e9, 0x02020604, 0x7f7f81fe, 0x5050f0a0, 0x3c3c4478, 0x9f9fba25, 0xa8a8e34b, + 0x5151f3a2, 0xa3a3fe5d, 0x4040c080, 0x8f8f8a05, 0x9292ad3f, 0x9d9dbc21, 0x38384870, 0xf5f504f1, + 0xbcbcdf63, 0xb6b6c177, 0xdada75af, 0x21216342, 0x10103020, 0xffff1ae5, 0xf3f30efd, 0xd2d26dbf, + 0xcdcd4c81, 0x0c0c1418, 0x13133526, 0xecec2fc3, 0x5f5fe1be, 0x9797a235, 0x4444cc88, 0x1717392e, + 0xc4c45793, 0xa7a7f255, 0x7e7e82fc, 0x3d3d477a, 0x6464acc8, 0x5d5de7ba, 0x19192b32, 0x737395e6, + 0x6060a0c0, 0x81819819, 0x4f4fd19e, 0xdcdc7fa3, 0x22226644, 0x2a2a7e54, 0x9090ab3b, 0x8888830b, + 0x4646ca8c, 0xeeee29c7, 0xb8b8d36b, 0x14143c28, 0xdede79a7, 0x5e5ee2bc, 0x0b0b1d16, 0xdbdb76ad, + 0xe0e03bdb, 0x32325664, 0x3a3a4e74, 0x0a0a1e14, 0x4949db92, 0x06060a0c, 0x24246c48, 0x5c5ce4b8, + 0xc2c25d9f, 0xd3d36ebd, 0xacacef43, 0x6262a6c4, 0x9191a839, 0x9595a431, 0xe4e437d3, 0x79798bf2, + 0xe7e732d5, 0xc8c8438b, 0x3737596e, 0x6d6db7da, 0x8d8d8c01, 0xd5d564b1, 0x4e4ed29c, 0xa9a9e049, + 0x6c6cb4d8, 0x5656faac, 0xf4f407f3, 0xeaea25cf, 0x6565afca, 0x7a7a8ef4, 0xaeaee947, 0x08081810, + 0xbabad56f, 0x787888f0, 0x25256f4a, 0x2e2e725c, 0x1c1c2438, 0xa6a6f157, 0xb4b4c773, 0xc6c65197, + 0xe8e823cb, 0xdddd7ca1, 0x74749ce8, 0x1f1f213e, 0x4b4bdd96, 0xbdbddc61, 0x8b8b860d, 0x8a8a850f, + 0x707090e0, 0x3e3e427c, 0xb5b5c471, 0x6666aacc, 0x4848d890, 0x03030506, 0xf6f601f7, 0x0e0e121c, + 0x6161a3c2, 0x35355f6a, 0x5757f9ae, 0xb9b9d069, 0x86869117, 0xc1c15899, 0x1d1d273a, 0x9e9eb927, + 0xe1e138d9, 0xf8f813eb, 0x9898b32b, 0x11113322, 0x6969bbd2, 0xd9d970a9, 0x8e8e8907, 0x9494a733, + 0x9b9bb62d, 0x1e1e223c, 0x87879215, 0xe9e920c9, 0xcece4987, 0x5555ffaa, 0x28287850, 0xdfdf7aa5, + 0x8c8c8f03, 0xa1a1f859, 0x89898009, 0x0d0d171a, 0xbfbfda65, 0xe6e631d7, 0x4242c684, 0x6868b8d0, + 0x4141c382, 0x9999b029, 0x2d2d775a, 0x0f0f111e, 0xb0b0cb7b, 0x5454fca8, 0xbbbbd66d, 0x16163a2c, + }, +} + +// Lookup tables for decryption. +// These can be recomputed by adapting the tests in aes_test.go. + +var td = [4][256]uint32{ + { + 0x51f4a750, 0x7e416553, 0x1a17a4c3, 0x3a275e96, 0x3bab6bcb, 0x1f9d45f1, 0xacfa58ab, 0x4be30393, + 0x2030fa55, 0xad766df6, 0x88cc7691, 0xf5024c25, 0x4fe5d7fc, 0xc52acbd7, 0x26354480, 0xb562a38f, + 0xdeb15a49, 0x25ba1b67, 0x45ea0e98, 0x5dfec0e1, 0xc32f7502, 0x814cf012, 0x8d4697a3, 0x6bd3f9c6, + 0x038f5fe7, 0x15929c95, 0xbf6d7aeb, 0x955259da, 0xd4be832d, 0x587421d3, 0x49e06929, 0x8ec9c844, + 0x75c2896a, 0xf48e7978, 0x99583e6b, 0x27b971dd, 0xbee14fb6, 0xf088ad17, 0xc920ac66, 0x7dce3ab4, + 0x63df4a18, 0xe51a3182, 0x97513360, 0x62537f45, 0xb16477e0, 0xbb6bae84, 0xfe81a01c, 0xf9082b94, + 0x70486858, 0x8f45fd19, 0x94de6c87, 0x527bf8b7, 0xab73d323, 0x724b02e2, 0xe31f8f57, 0x6655ab2a, + 0xb2eb2807, 0x2fb5c203, 0x86c57b9a, 0xd33708a5, 0x302887f2, 0x23bfa5b2, 0x02036aba, 0xed16825c, + 0x8acf1c2b, 0xa779b492, 0xf307f2f0, 0x4e69e2a1, 0x65daf4cd, 0x0605bed5, 0xd134621f, 0xc4a6fe8a, + 0x342e539d, 0xa2f355a0, 0x058ae132, 0xa4f6eb75, 0x0b83ec39, 0x4060efaa, 0x5e719f06, 0xbd6e1051, + 0x3e218af9, 0x96dd063d, 0xdd3e05ae, 0x4de6bd46, 0x91548db5, 0x71c45d05, 0x0406d46f, 0x605015ff, + 0x1998fb24, 0xd6bde997, 0x894043cc, 0x67d99e77, 0xb0e842bd, 0x07898b88, 0xe7195b38, 0x79c8eedb, + 0xa17c0a47, 0x7c420fe9, 0xf8841ec9, 0x00000000, 0x09808683, 0x322bed48, 0x1e1170ac, 0x6c5a724e, + 0xfd0efffb, 0x0f853856, 0x3daed51e, 0x362d3927, 0x0a0fd964, 0x685ca621, 0x9b5b54d1, 0x24362e3a, + 0x0c0a67b1, 0x9357e70f, 0xb4ee96d2, 0x1b9b919e, 0x80c0c54f, 0x61dc20a2, 0x5a774b69, 0x1c121a16, + 0xe293ba0a, 0xc0a02ae5, 0x3c22e043, 0x121b171d, 0x0e090d0b, 0xf28bc7ad, 0x2db6a8b9, 0x141ea9c8, + 0x57f11985, 0xaf75074c, 0xee99ddbb, 0xa37f60fd, 0xf701269f, 0x5c72f5bc, 0x44663bc5, 0x5bfb7e34, + 0x8b432976, 0xcb23c6dc, 0xb6edfc68, 0xb8e4f163, 0xd731dcca, 0x42638510, 0x13972240, 0x84c61120, + 0x854a247d, 0xd2bb3df8, 0xaef93211, 0xc729a16d, 0x1d9e2f4b, 0xdcb230f3, 0x0d8652ec, 0x77c1e3d0, + 0x2bb3166c, 0xa970b999, 0x119448fa, 0x47e96422, 0xa8fc8cc4, 0xa0f03f1a, 0x567d2cd8, 0x223390ef, + 0x87494ec7, 0xd938d1c1, 0x8ccaa2fe, 0x98d40b36, 0xa6f581cf, 0xa57ade28, 0xdab78e26, 0x3fadbfa4, + 0x2c3a9de4, 0x5078920d, 0x6a5fcc9b, 0x547e4662, 0xf68d13c2, 0x90d8b8e8, 0x2e39f75e, 0x82c3aff5, + 0x9f5d80be, 0x69d0937c, 0x6fd52da9, 0xcf2512b3, 0xc8ac993b, 0x10187da7, 0xe89c636e, 0xdb3bbb7b, + 0xcd267809, 0x6e5918f4, 0xec9ab701, 0x834f9aa8, 0xe6956e65, 0xaaffe67e, 0x21bccf08, 0xef15e8e6, + 0xbae79bd9, 0x4a6f36ce, 0xea9f09d4, 0x29b07cd6, 0x31a4b2af, 0x2a3f2331, 0xc6a59430, 0x35a266c0, + 0x744ebc37, 0xfc82caa6, 0xe090d0b0, 0x33a7d815, 0xf104984a, 0x41ecdaf7, 0x7fcd500e, 0x1791f62f, + 0x764dd68d, 0x43efb04d, 0xccaa4d54, 0xe49604df, 0x9ed1b5e3, 0x4c6a881b, 0xc12c1fb8, 0x4665517f, + 0x9d5eea04, 0x018c355d, 0xfa877473, 0xfb0b412e, 0xb3671d5a, 0x92dbd252, 0xe9105633, 0x6dd64713, + 0x9ad7618c, 0x37a10c7a, 0x59f8148e, 0xeb133c89, 0xcea927ee, 0xb761c935, 0xe11ce5ed, 0x7a47b13c, + 0x9cd2df59, 0x55f2733f, 0x1814ce79, 0x73c737bf, 0x53f7cdea, 0x5ffdaa5b, 0xdf3d6f14, 0x7844db86, + 0xcaaff381, 0xb968c43e, 0x3824342c, 0xc2a3405f, 0x161dc372, 0xbce2250c, 0x283c498b, 0xff0d9541, + 0x39a80171, 0x080cb3de, 0xd8b4e49c, 0x6456c190, 0x7bcb8461, 0xd532b670, 0x486c5c74, 0xd0b85742, + }, + { + 0x5051f4a7, 0x537e4165, 0xc31a17a4, 0x963a275e, 0xcb3bab6b, 0xf11f9d45, 0xabacfa58, 0x934be303, + 0x552030fa, 0xf6ad766d, 0x9188cc76, 0x25f5024c, 0xfc4fe5d7, 0xd7c52acb, 0x80263544, 0x8fb562a3, + 0x49deb15a, 0x6725ba1b, 0x9845ea0e, 0xe15dfec0, 0x02c32f75, 0x12814cf0, 0xa38d4697, 0xc66bd3f9, + 0xe7038f5f, 0x9515929c, 0xebbf6d7a, 0xda955259, 0x2dd4be83, 0xd3587421, 0x2949e069, 0x448ec9c8, + 0x6a75c289, 0x78f48e79, 0x6b99583e, 0xdd27b971, 0xb6bee14f, 0x17f088ad, 0x66c920ac, 0xb47dce3a, + 0x1863df4a, 0x82e51a31, 0x60975133, 0x4562537f, 0xe0b16477, 0x84bb6bae, 0x1cfe81a0, 0x94f9082b, + 0x58704868, 0x198f45fd, 0x8794de6c, 0xb7527bf8, 0x23ab73d3, 0xe2724b02, 0x57e31f8f, 0x2a6655ab, + 0x07b2eb28, 0x032fb5c2, 0x9a86c57b, 0xa5d33708, 0xf2302887, 0xb223bfa5, 0xba02036a, 0x5ced1682, + 0x2b8acf1c, 0x92a779b4, 0xf0f307f2, 0xa14e69e2, 0xcd65daf4, 0xd50605be, 0x1fd13462, 0x8ac4a6fe, + 0x9d342e53, 0xa0a2f355, 0x32058ae1, 0x75a4f6eb, 0x390b83ec, 0xaa4060ef, 0x065e719f, 0x51bd6e10, + 0xf93e218a, 0x3d96dd06, 0xaedd3e05, 0x464de6bd, 0xb591548d, 0x0571c45d, 0x6f0406d4, 0xff605015, + 0x241998fb, 0x97d6bde9, 0xcc894043, 0x7767d99e, 0xbdb0e842, 0x8807898b, 0x38e7195b, 0xdb79c8ee, + 0x47a17c0a, 0xe97c420f, 0xc9f8841e, 0x00000000, 0x83098086, 0x48322bed, 0xac1e1170, 0x4e6c5a72, + 0xfbfd0eff, 0x560f8538, 0x1e3daed5, 0x27362d39, 0x640a0fd9, 0x21685ca6, 0xd19b5b54, 0x3a24362e, + 0xb10c0a67, 0x0f9357e7, 0xd2b4ee96, 0x9e1b9b91, 0x4f80c0c5, 0xa261dc20, 0x695a774b, 0x161c121a, + 0x0ae293ba, 0xe5c0a02a, 0x433c22e0, 0x1d121b17, 0x0b0e090d, 0xadf28bc7, 0xb92db6a8, 0xc8141ea9, + 0x8557f119, 0x4caf7507, 0xbbee99dd, 0xfda37f60, 0x9ff70126, 0xbc5c72f5, 0xc544663b, 0x345bfb7e, + 0x768b4329, 0xdccb23c6, 0x68b6edfc, 0x63b8e4f1, 0xcad731dc, 0x10426385, 0x40139722, 0x2084c611, + 0x7d854a24, 0xf8d2bb3d, 0x11aef932, 0x6dc729a1, 0x4b1d9e2f, 0xf3dcb230, 0xec0d8652, 0xd077c1e3, + 0x6c2bb316, 0x99a970b9, 0xfa119448, 0x2247e964, 0xc4a8fc8c, 0x1aa0f03f, 0xd8567d2c, 0xef223390, + 0xc787494e, 0xc1d938d1, 0xfe8ccaa2, 0x3698d40b, 0xcfa6f581, 0x28a57ade, 0x26dab78e, 0xa43fadbf, + 0xe42c3a9d, 0x0d507892, 0x9b6a5fcc, 0x62547e46, 0xc2f68d13, 0xe890d8b8, 0x5e2e39f7, 0xf582c3af, + 0xbe9f5d80, 0x7c69d093, 0xa96fd52d, 0xb3cf2512, 0x3bc8ac99, 0xa710187d, 0x6ee89c63, 0x7bdb3bbb, + 0x09cd2678, 0xf46e5918, 0x01ec9ab7, 0xa8834f9a, 0x65e6956e, 0x7eaaffe6, 0x0821bccf, 0xe6ef15e8, + 0xd9bae79b, 0xce4a6f36, 0xd4ea9f09, 0xd629b07c, 0xaf31a4b2, 0x312a3f23, 0x30c6a594, 0xc035a266, + 0x37744ebc, 0xa6fc82ca, 0xb0e090d0, 0x1533a7d8, 0x4af10498, 0xf741ecda, 0x0e7fcd50, 0x2f1791f6, + 0x8d764dd6, 0x4d43efb0, 0x54ccaa4d, 0xdfe49604, 0xe39ed1b5, 0x1b4c6a88, 0xb8c12c1f, 0x7f466551, + 0x049d5eea, 0x5d018c35, 0x73fa8774, 0x2efb0b41, 0x5ab3671d, 0x5292dbd2, 0x33e91056, 0x136dd647, + 0x8c9ad761, 0x7a37a10c, 0x8e59f814, 0x89eb133c, 0xeecea927, 0x35b761c9, 0xede11ce5, 0x3c7a47b1, + 0x599cd2df, 0x3f55f273, 0x791814ce, 0xbf73c737, 0xea53f7cd, 0x5b5ffdaa, 0x14df3d6f, 0x867844db, + 0x81caaff3, 0x3eb968c4, 0x2c382434, 0x5fc2a340, 0x72161dc3, 0x0cbce225, 0x8b283c49, 0x41ff0d95, + 0x7139a801, 0xde080cb3, 0x9cd8b4e4, 0x906456c1, 0x617bcb84, 0x70d532b6, 0x74486c5c, 0x42d0b857, + }, + { + 0xa75051f4, 0x65537e41, 0xa4c31a17, 0x5e963a27, 0x6bcb3bab, 0x45f11f9d, 0x58abacfa, 0x03934be3, + 0xfa552030, 0x6df6ad76, 0x769188cc, 0x4c25f502, 0xd7fc4fe5, 0xcbd7c52a, 0x44802635, 0xa38fb562, + 0x5a49deb1, 0x1b6725ba, 0x0e9845ea, 0xc0e15dfe, 0x7502c32f, 0xf012814c, 0x97a38d46, 0xf9c66bd3, + 0x5fe7038f, 0x9c951592, 0x7aebbf6d, 0x59da9552, 0x832dd4be, 0x21d35874, 0x692949e0, 0xc8448ec9, + 0x896a75c2, 0x7978f48e, 0x3e6b9958, 0x71dd27b9, 0x4fb6bee1, 0xad17f088, 0xac66c920, 0x3ab47dce, + 0x4a1863df, 0x3182e51a, 0x33609751, 0x7f456253, 0x77e0b164, 0xae84bb6b, 0xa01cfe81, 0x2b94f908, + 0x68587048, 0xfd198f45, 0x6c8794de, 0xf8b7527b, 0xd323ab73, 0x02e2724b, 0x8f57e31f, 0xab2a6655, + 0x2807b2eb, 0xc2032fb5, 0x7b9a86c5, 0x08a5d337, 0x87f23028, 0xa5b223bf, 0x6aba0203, 0x825ced16, + 0x1c2b8acf, 0xb492a779, 0xf2f0f307, 0xe2a14e69, 0xf4cd65da, 0xbed50605, 0x621fd134, 0xfe8ac4a6, + 0x539d342e, 0x55a0a2f3, 0xe132058a, 0xeb75a4f6, 0xec390b83, 0xefaa4060, 0x9f065e71, 0x1051bd6e, + 0x8af93e21, 0x063d96dd, 0x05aedd3e, 0xbd464de6, 0x8db59154, 0x5d0571c4, 0xd46f0406, 0x15ff6050, + 0xfb241998, 0xe997d6bd, 0x43cc8940, 0x9e7767d9, 0x42bdb0e8, 0x8b880789, 0x5b38e719, 0xeedb79c8, + 0x0a47a17c, 0x0fe97c42, 0x1ec9f884, 0x00000000, 0x86830980, 0xed48322b, 0x70ac1e11, 0x724e6c5a, + 0xfffbfd0e, 0x38560f85, 0xd51e3dae, 0x3927362d, 0xd9640a0f, 0xa621685c, 0x54d19b5b, 0x2e3a2436, + 0x67b10c0a, 0xe70f9357, 0x96d2b4ee, 0x919e1b9b, 0xc54f80c0, 0x20a261dc, 0x4b695a77, 0x1a161c12, + 0xba0ae293, 0x2ae5c0a0, 0xe0433c22, 0x171d121b, 0x0d0b0e09, 0xc7adf28b, 0xa8b92db6, 0xa9c8141e, + 0x198557f1, 0x074caf75, 0xddbbee99, 0x60fda37f, 0x269ff701, 0xf5bc5c72, 0x3bc54466, 0x7e345bfb, + 0x29768b43, 0xc6dccb23, 0xfc68b6ed, 0xf163b8e4, 0xdccad731, 0x85104263, 0x22401397, 0x112084c6, + 0x247d854a, 0x3df8d2bb, 0x3211aef9, 0xa16dc729, 0x2f4b1d9e, 0x30f3dcb2, 0x52ec0d86, 0xe3d077c1, + 0x166c2bb3, 0xb999a970, 0x48fa1194, 0x642247e9, 0x8cc4a8fc, 0x3f1aa0f0, 0x2cd8567d, 0x90ef2233, + 0x4ec78749, 0xd1c1d938, 0xa2fe8cca, 0x0b3698d4, 0x81cfa6f5, 0xde28a57a, 0x8e26dab7, 0xbfa43fad, + 0x9de42c3a, 0x920d5078, 0xcc9b6a5f, 0x4662547e, 0x13c2f68d, 0xb8e890d8, 0xf75e2e39, 0xaff582c3, + 0x80be9f5d, 0x937c69d0, 0x2da96fd5, 0x12b3cf25, 0x993bc8ac, 0x7da71018, 0x636ee89c, 0xbb7bdb3b, + 0x7809cd26, 0x18f46e59, 0xb701ec9a, 0x9aa8834f, 0x6e65e695, 0xe67eaaff, 0xcf0821bc, 0xe8e6ef15, + 0x9bd9bae7, 0x36ce4a6f, 0x09d4ea9f, 0x7cd629b0, 0xb2af31a4, 0x23312a3f, 0x9430c6a5, 0x66c035a2, + 0xbc37744e, 0xcaa6fc82, 0xd0b0e090, 0xd81533a7, 0x984af104, 0xdaf741ec, 0x500e7fcd, 0xf62f1791, + 0xd68d764d, 0xb04d43ef, 0x4d54ccaa, 0x04dfe496, 0xb5e39ed1, 0x881b4c6a, 0x1fb8c12c, 0x517f4665, + 0xea049d5e, 0x355d018c, 0x7473fa87, 0x412efb0b, 0x1d5ab367, 0xd25292db, 0x5633e910, 0x47136dd6, + 0x618c9ad7, 0x0c7a37a1, 0x148e59f8, 0x3c89eb13, 0x27eecea9, 0xc935b761, 0xe5ede11c, 0xb13c7a47, + 0xdf599cd2, 0x733f55f2, 0xce791814, 0x37bf73c7, 0xcdea53f7, 0xaa5b5ffd, 0x6f14df3d, 0xdb867844, + 0xf381caaf, 0xc43eb968, 0x342c3824, 0x405fc2a3, 0xc372161d, 0x250cbce2, 0x498b283c, 0x9541ff0d, + 0x017139a8, 0xb3de080c, 0xe49cd8b4, 0xc1906456, 0x84617bcb, 0xb670d532, 0x5c74486c, 0x5742d0b8, + }, + { + 0xf4a75051, 0x4165537e, 0x17a4c31a, 0x275e963a, 0xab6bcb3b, 0x9d45f11f, 0xfa58abac, 0xe303934b, + 0x30fa5520, 0x766df6ad, 0xcc769188, 0x024c25f5, 0xe5d7fc4f, 0x2acbd7c5, 0x35448026, 0x62a38fb5, + 0xb15a49de, 0xba1b6725, 0xea0e9845, 0xfec0e15d, 0x2f7502c3, 0x4cf01281, 0x4697a38d, 0xd3f9c66b, + 0x8f5fe703, 0x929c9515, 0x6d7aebbf, 0x5259da95, 0xbe832dd4, 0x7421d358, 0xe0692949, 0xc9c8448e, + 0xc2896a75, 0x8e7978f4, 0x583e6b99, 0xb971dd27, 0xe14fb6be, 0x88ad17f0, 0x20ac66c9, 0xce3ab47d, + 0xdf4a1863, 0x1a3182e5, 0x51336097, 0x537f4562, 0x6477e0b1, 0x6bae84bb, 0x81a01cfe, 0x082b94f9, + 0x48685870, 0x45fd198f, 0xde6c8794, 0x7bf8b752, 0x73d323ab, 0x4b02e272, 0x1f8f57e3, 0x55ab2a66, + 0xeb2807b2, 0xb5c2032f, 0xc57b9a86, 0x3708a5d3, 0x2887f230, 0xbfa5b223, 0x036aba02, 0x16825ced, + 0xcf1c2b8a, 0x79b492a7, 0x07f2f0f3, 0x69e2a14e, 0xdaf4cd65, 0x05bed506, 0x34621fd1, 0xa6fe8ac4, + 0x2e539d34, 0xf355a0a2, 0x8ae13205, 0xf6eb75a4, 0x83ec390b, 0x60efaa40, 0x719f065e, 0x6e1051bd, + 0x218af93e, 0xdd063d96, 0x3e05aedd, 0xe6bd464d, 0x548db591, 0xc45d0571, 0x06d46f04, 0x5015ff60, + 0x98fb2419, 0xbde997d6, 0x4043cc89, 0xd99e7767, 0xe842bdb0, 0x898b8807, 0x195b38e7, 0xc8eedb79, + 0x7c0a47a1, 0x420fe97c, 0x841ec9f8, 0x00000000, 0x80868309, 0x2bed4832, 0x1170ac1e, 0x5a724e6c, + 0x0efffbfd, 0x8538560f, 0xaed51e3d, 0x2d392736, 0x0fd9640a, 0x5ca62168, 0x5b54d19b, 0x362e3a24, + 0x0a67b10c, 0x57e70f93, 0xee96d2b4, 0x9b919e1b, 0xc0c54f80, 0xdc20a261, 0x774b695a, 0x121a161c, + 0x93ba0ae2, 0xa02ae5c0, 0x22e0433c, 0x1b171d12, 0x090d0b0e, 0x8bc7adf2, 0xb6a8b92d, 0x1ea9c814, + 0xf1198557, 0x75074caf, 0x99ddbbee, 0x7f60fda3, 0x01269ff7, 0x72f5bc5c, 0x663bc544, 0xfb7e345b, + 0x4329768b, 0x23c6dccb, 0xedfc68b6, 0xe4f163b8, 0x31dccad7, 0x63851042, 0x97224013, 0xc6112084, + 0x4a247d85, 0xbb3df8d2, 0xf93211ae, 0x29a16dc7, 0x9e2f4b1d, 0xb230f3dc, 0x8652ec0d, 0xc1e3d077, + 0xb3166c2b, 0x70b999a9, 0x9448fa11, 0xe9642247, 0xfc8cc4a8, 0xf03f1aa0, 0x7d2cd856, 0x3390ef22, + 0x494ec787, 0x38d1c1d9, 0xcaa2fe8c, 0xd40b3698, 0xf581cfa6, 0x7ade28a5, 0xb78e26da, 0xadbfa43f, + 0x3a9de42c, 0x78920d50, 0x5fcc9b6a, 0x7e466254, 0x8d13c2f6, 0xd8b8e890, 0x39f75e2e, 0xc3aff582, + 0x5d80be9f, 0xd0937c69, 0xd52da96f, 0x2512b3cf, 0xac993bc8, 0x187da710, 0x9c636ee8, 0x3bbb7bdb, + 0x267809cd, 0x5918f46e, 0x9ab701ec, 0x4f9aa883, 0x956e65e6, 0xffe67eaa, 0xbccf0821, 0x15e8e6ef, + 0xe79bd9ba, 0x6f36ce4a, 0x9f09d4ea, 0xb07cd629, 0xa4b2af31, 0x3f23312a, 0xa59430c6, 0xa266c035, + 0x4ebc3774, 0x82caa6fc, 0x90d0b0e0, 0xa7d81533, 0x04984af1, 0xecdaf741, 0xcd500e7f, 0x91f62f17, + 0x4dd68d76, 0xefb04d43, 0xaa4d54cc, 0x9604dfe4, 0xd1b5e39e, 0x6a881b4c, 0x2c1fb8c1, 0x65517f46, + 0x5eea049d, 0x8c355d01, 0x877473fa, 0x0b412efb, 0x671d5ab3, 0xdbd25292, 0x105633e9, 0xd647136d, + 0xd7618c9a, 0xa10c7a37, 0xf8148e59, 0x133c89eb, 0xa927eece, 0x61c935b7, 0x1ce5ede1, 0x47b13c7a, + 0xd2df599c, 0xf2733f55, 0x14ce7918, 0xc737bf73, 0xf7cdea53, 0xfdaa5b5f, 0x3d6f14df, 0x44db8678, + 0xaff381ca, 0x68c43eb9, 0x24342c38, 0xa3405fc2, 0x1dc37216, 0xe2250cbc, 0x3c498b28, 0x0d9541ff, + 0xa8017139, 0x0cb3de08, 0xb4e49cd8, 0x56c19064, 0xcb84617b, 0x32b670d5, 0x6c5c7448, 0xb85742d0, + }, +} diff --git a/src/pkg/crypto/blowfish/Makefile b/src/pkg/crypto/blowfish/Makefile new file mode 100644 index 000000000..f370ab28b --- /dev/null +++ b/src/pkg/crypto/blowfish/Makefile @@ -0,0 +1,13 @@ +# Copyright 2010 The Go Authors. All rights reserved. +# Use of this source code is governed by a BSD-style +# license that can be found in the LICENSE file. + +include ../../../Make.inc + +TARG=crypto/blowfish +GOFILES=\ + block.go\ + cipher.go\ + const.go\ + +include ../../../Make.pkg diff --git a/src/pkg/crypto/blowfish/block.go b/src/pkg/crypto/blowfish/block.go new file mode 100644 index 000000000..7fbe7eefb --- /dev/null +++ b/src/pkg/crypto/blowfish/block.go @@ -0,0 +1,101 @@ +// Copyright 2010 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package blowfish + +func expandKey(key []byte, c *Cipher) { + copy(c.p[0:], p[0:]) + copy(c.s0[0:], s0[0:]) + copy(c.s1[0:], s1[0:]) + copy(c.s2[0:], s2[0:]) + copy(c.s3[0:], s3[0:]) + + j := 0 + for i := 0; i < 18; i++ { + var d uint32 + for k := 0; k < 4; k++ { + d = d<<8 | uint32(key[j])&0x000000FF + j++ + if j >= len(key) { + j = 0 + } + } + c.p[i] ^= d + } + + var l, r uint32 + for i := 0; i < 18; i += 2 { + l, r = encryptBlock(l, r, c) + c.p[i], c.p[i+1] = l, r + } + + for i := 0; i < 256; i += 2 { + l, r = encryptBlock(l, r, c) + c.s0[i], c.s0[i+1] = l, r + } + for i := 0; i < 256; i += 2 { + l, r = encryptBlock(l, r, c) + c.s1[i], c.s1[i+1] = l, r + } + for i := 0; i < 256; i += 2 { + l, r = encryptBlock(l, r, c) + c.s2[i], c.s2[i+1] = l, r + } + for i := 0; i < 256; i += 2 { + l, r = encryptBlock(l, r, c) + c.s3[i], c.s3[i+1] = l, r + } +} + +func encryptBlock(l, r uint32, c *Cipher) (uint32, uint32) { + xl, xr := l, r + xl ^= c.p[0] + xr ^= ((c.s0[byte(xl>>24)] + c.s1[byte(xl>>16)]) ^ c.s2[byte(xl>>8)]) + c.s3[byte(xl)] ^ c.p[1] + xl ^= ((c.s0[byte(xr>>24)] + c.s1[byte(xr>>16)]) ^ c.s2[byte(xr>>8)]) + c.s3[byte(xr)] ^ c.p[2] + xr ^= ((c.s0[byte(xl>>24)] + c.s1[byte(xl>>16)]) ^ c.s2[byte(xl>>8)]) + c.s3[byte(xl)] ^ c.p[3] + xl ^= ((c.s0[byte(xr>>24)] + c.s1[byte(xr>>16)]) ^ c.s2[byte(xr>>8)]) + c.s3[byte(xr)] ^ c.p[4] + xr ^= ((c.s0[byte(xl>>24)] + c.s1[byte(xl>>16)]) ^ c.s2[byte(xl>>8)]) + c.s3[byte(xl)] ^ c.p[5] + xl ^= ((c.s0[byte(xr>>24)] + c.s1[byte(xr>>16)]) ^ c.s2[byte(xr>>8)]) + c.s3[byte(xr)] ^ c.p[6] + xr ^= ((c.s0[byte(xl>>24)] + c.s1[byte(xl>>16)]) ^ c.s2[byte(xl>>8)]) + c.s3[byte(xl)] ^ c.p[7] + xl ^= ((c.s0[byte(xr>>24)] + c.s1[byte(xr>>16)]) ^ c.s2[byte(xr>>8)]) + c.s3[byte(xr)] ^ c.p[8] + xr ^= ((c.s0[byte(xl>>24)] + c.s1[byte(xl>>16)]) ^ c.s2[byte(xl>>8)]) + c.s3[byte(xl)] ^ c.p[9] + xl ^= ((c.s0[byte(xr>>24)] + c.s1[byte(xr>>16)]) ^ c.s2[byte(xr>>8)]) + c.s3[byte(xr)] ^ c.p[10] + xr ^= ((c.s0[byte(xl>>24)] + c.s1[byte(xl>>16)]) ^ c.s2[byte(xl>>8)]) + c.s3[byte(xl)] ^ c.p[11] + xl ^= ((c.s0[byte(xr>>24)] + c.s1[byte(xr>>16)]) ^ c.s2[byte(xr>>8)]) + c.s3[byte(xr)] ^ c.p[12] + xr ^= ((c.s0[byte(xl>>24)] + c.s1[byte(xl>>16)]) ^ c.s2[byte(xl>>8)]) + c.s3[byte(xl)] ^ c.p[13] + xl ^= ((c.s0[byte(xr>>24)] + c.s1[byte(xr>>16)]) ^ c.s2[byte(xr>>8)]) + c.s3[byte(xr)] ^ c.p[14] + xr ^= ((c.s0[byte(xl>>24)] + c.s1[byte(xl>>16)]) ^ c.s2[byte(xl>>8)]) + c.s3[byte(xl)] ^ c.p[15] + xl ^= ((c.s0[byte(xr>>24)] + c.s1[byte(xr>>16)]) ^ c.s2[byte(xr>>8)]) + c.s3[byte(xr)] ^ c.p[16] + xr ^= c.p[17] + return xr, xl +} + +func decryptBlock(l, r uint32, c *Cipher) (uint32, uint32) { + xl, xr := l, r + xl ^= c.p[17] + xr ^= ((c.s0[byte(xl>>24)] + c.s1[byte(xl>>16)]) ^ c.s2[byte(xl>>8)]) + c.s3[byte(xl)] ^ c.p[16] + xl ^= ((c.s0[byte(xr>>24)] + c.s1[byte(xr>>16)]) ^ c.s2[byte(xr>>8)]) + c.s3[byte(xr)] ^ c.p[15] + xr ^= ((c.s0[byte(xl>>24)] + c.s1[byte(xl>>16)]) ^ c.s2[byte(xl>>8)]) + c.s3[byte(xl)] ^ c.p[14] + xl ^= ((c.s0[byte(xr>>24)] + c.s1[byte(xr>>16)]) ^ c.s2[byte(xr>>8)]) + c.s3[byte(xr)] ^ c.p[13] + xr ^= ((c.s0[byte(xl>>24)] + c.s1[byte(xl>>16)]) ^ c.s2[byte(xl>>8)]) + c.s3[byte(xl)] ^ c.p[12] + xl ^= ((c.s0[byte(xr>>24)] + c.s1[byte(xr>>16)]) ^ c.s2[byte(xr>>8)]) + c.s3[byte(xr)] ^ c.p[11] + xr ^= ((c.s0[byte(xl>>24)] + c.s1[byte(xl>>16)]) ^ c.s2[byte(xl>>8)]) + c.s3[byte(xl)] ^ c.p[10] + xl ^= ((c.s0[byte(xr>>24)] + c.s1[byte(xr>>16)]) ^ c.s2[byte(xr>>8)]) + c.s3[byte(xr)] ^ c.p[9] + xr ^= ((c.s0[byte(xl>>24)] + c.s1[byte(xl>>16)]) ^ c.s2[byte(xl>>8)]) + c.s3[byte(xl)] ^ c.p[8] + xl ^= ((c.s0[byte(xr>>24)] + c.s1[byte(xr>>16)]) ^ c.s2[byte(xr>>8)]) + c.s3[byte(xr)] ^ c.p[7] + xr ^= ((c.s0[byte(xl>>24)] + c.s1[byte(xl>>16)]) ^ c.s2[byte(xl>>8)]) + c.s3[byte(xl)] ^ c.p[6] + xl ^= ((c.s0[byte(xr>>24)] + c.s1[byte(xr>>16)]) ^ c.s2[byte(xr>>8)]) + c.s3[byte(xr)] ^ c.p[5] + xr ^= ((c.s0[byte(xl>>24)] + c.s1[byte(xl>>16)]) ^ c.s2[byte(xl>>8)]) + c.s3[byte(xl)] ^ c.p[4] + xl ^= ((c.s0[byte(xr>>24)] + c.s1[byte(xr>>16)]) ^ c.s2[byte(xr>>8)]) + c.s3[byte(xr)] ^ c.p[3] + xr ^= ((c.s0[byte(xl>>24)] + c.s1[byte(xl>>16)]) ^ c.s2[byte(xl>>8)]) + c.s3[byte(xl)] ^ c.p[2] + xl ^= ((c.s0[byte(xr>>24)] + c.s1[byte(xr>>16)]) ^ c.s2[byte(xr>>8)]) + c.s3[byte(xr)] ^ c.p[1] + xr ^= c.p[0] + return xr, xl +} + +func zero(x []uint32) { + for i := range x { + x[i] = 0 + } +} diff --git a/src/pkg/crypto/blowfish/blowfish_test.go b/src/pkg/crypto/blowfish/blowfish_test.go new file mode 100644 index 000000000..3a7ab6c2a --- /dev/null +++ b/src/pkg/crypto/blowfish/blowfish_test.go @@ -0,0 +1,192 @@ +// Copyright 2010 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package blowfish + +import ( + "testing" +) + +type CryptTest struct { + key []byte + in []byte + out []byte +} + +// Test vector values are from http://www.schneier.com/code/vectors.txt. +var encryptTests = []CryptTest{ + { + []byte{0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00}, + []byte{0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00}, + []byte{0x4E, 0xF9, 0x97, 0x45, 0x61, 0x98, 0xDD, 0x78}}, + { + []byte{0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF}, + []byte{0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF}, + []byte{0x51, 0x86, 0x6F, 0xD5, 0xB8, 0x5E, 0xCB, 0x8A}}, + { + []byte{0x30, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00}, + []byte{0x10, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01}, + []byte{0x7D, 0x85, 0x6F, 0x9A, 0x61, 0x30, 0x63, 0xF2}}, + { + []byte{0x11, 0x11, 0x11, 0x11, 0x11, 0x11, 0x11, 0x11}, + []byte{0x11, 0x11, 0x11, 0x11, 0x11, 0x11, 0x11, 0x11}, + []byte{0x24, 0x66, 0xDD, 0x87, 0x8B, 0x96, 0x3C, 0x9D}}, + + { + []byte{0x01, 0x23, 0x45, 0x67, 0x89, 0xAB, 0xCD, 0xEF}, + []byte{0x11, 0x11, 0x11, 0x11, 0x11, 0x11, 0x11, 0x11}, + []byte{0x61, 0xF9, 0xC3, 0x80, 0x22, 0x81, 0xB0, 0x96}}, + { + []byte{0x11, 0x11, 0x11, 0x11, 0x11, 0x11, 0x11, 0x11}, + []byte{0x01, 0x23, 0x45, 0x67, 0x89, 0xAB, 0xCD, 0xEF}, + []byte{0x7D, 0x0C, 0xC6, 0x30, 0xAF, 0xDA, 0x1E, 0xC7}}, + { + []byte{0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00}, + []byte{0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00}, + []byte{0x4E, 0xF9, 0x97, 0x45, 0x61, 0x98, 0xDD, 0x78}}, + { + []byte{0xFE, 0xDC, 0xBA, 0x98, 0x76, 0x54, 0x32, 0x10}, + []byte{0x01, 0x23, 0x45, 0x67, 0x89, 0xAB, 0xCD, 0xEF}, + []byte{0x0A, 0xCE, 0xAB, 0x0F, 0xC6, 0xA0, 0xA2, 0x8D}}, + { + []byte{0x7C, 0xA1, 0x10, 0x45, 0x4A, 0x1A, 0x6E, 0x57}, + []byte{0x01, 0xA1, 0xD6, 0xD0, 0x39, 0x77, 0x67, 0x42}, + []byte{0x59, 0xC6, 0x82, 0x45, 0xEB, 0x05, 0x28, 0x2B}}, + { + []byte{0x01, 0x31, 0xD9, 0x61, 0x9D, 0xC1, 0x37, 0x6E}, + []byte{0x5C, 0xD5, 0x4C, 0xA8, 0x3D, 0xEF, 0x57, 0xDA}, + []byte{0xB1, 0xB8, 0xCC, 0x0B, 0x25, 0x0F, 0x09, 0xA0}}, + { + []byte{0x07, 0xA1, 0x13, 0x3E, 0x4A, 0x0B, 0x26, 0x86}, + []byte{0x02, 0x48, 0xD4, 0x38, 0x06, 0xF6, 0x71, 0x72}, + []byte{0x17, 0x30, 0xE5, 0x77, 0x8B, 0xEA, 0x1D, 0xA4}}, + { + []byte{0x38, 0x49, 0x67, 0x4C, 0x26, 0x02, 0x31, 0x9E}, + []byte{0x51, 0x45, 0x4B, 0x58, 0x2D, 0xDF, 0x44, 0x0A}, + []byte{0xA2, 0x5E, 0x78, 0x56, 0xCF, 0x26, 0x51, 0xEB}}, + { + []byte{0x04, 0xB9, 0x15, 0xBA, 0x43, 0xFE, 0xB5, 0xB6}, + []byte{0x42, 0xFD, 0x44, 0x30, 0x59, 0x57, 0x7F, 0xA2}, + []byte{0x35, 0x38, 0x82, 0xB1, 0x09, 0xCE, 0x8F, 0x1A}}, + { + []byte{0x01, 0x13, 0xB9, 0x70, 0xFD, 0x34, 0xF2, 0xCE}, + []byte{0x05, 0x9B, 0x5E, 0x08, 0x51, 0xCF, 0x14, 0x3A}, + []byte{0x48, 0xF4, 0xD0, 0x88, 0x4C, 0x37, 0x99, 0x18}}, + { + []byte{0x01, 0x70, 0xF1, 0x75, 0x46, 0x8F, 0xB5, 0xE6}, + []byte{0x07, 0x56, 0xD8, 0xE0, 0x77, 0x47, 0x61, 0xD2}, + []byte{0x43, 0x21, 0x93, 0xB7, 0x89, 0x51, 0xFC, 0x98}}, + { + []byte{0x43, 0x29, 0x7F, 0xAD, 0x38, 0xE3, 0x73, 0xFE}, + []byte{0x76, 0x25, 0x14, 0xB8, 0x29, 0xBF, 0x48, 0x6A}, + []byte{0x13, 0xF0, 0x41, 0x54, 0xD6, 0x9D, 0x1A, 0xE5}}, + { + []byte{0x07, 0xA7, 0x13, 0x70, 0x45, 0xDA, 0x2A, 0x16}, + []byte{0x3B, 0xDD, 0x11, 0x90, 0x49, 0x37, 0x28, 0x02}, + []byte{0x2E, 0xED, 0xDA, 0x93, 0xFF, 0xD3, 0x9C, 0x79}}, + { + []byte{0x04, 0x68, 0x91, 0x04, 0xC2, 0xFD, 0x3B, 0x2F}, + []byte{0x26, 0x95, 0x5F, 0x68, 0x35, 0xAF, 0x60, 0x9A}, + []byte{0xD8, 0x87, 0xE0, 0x39, 0x3C, 0x2D, 0xA6, 0xE3}}, + { + []byte{0x37, 0xD0, 0x6B, 0xB5, 0x16, 0xCB, 0x75, 0x46}, + []byte{0x16, 0x4D, 0x5E, 0x40, 0x4F, 0x27, 0x52, 0x32}, + []byte{0x5F, 0x99, 0xD0, 0x4F, 0x5B, 0x16, 0x39, 0x69}}, + { + []byte{0x1F, 0x08, 0x26, 0x0D, 0x1A, 0xC2, 0x46, 0x5E}, + []byte{0x6B, 0x05, 0x6E, 0x18, 0x75, 0x9F, 0x5C, 0xCA}, + []byte{0x4A, 0x05, 0x7A, 0x3B, 0x24, 0xD3, 0x97, 0x7B}}, + { + []byte{0x58, 0x40, 0x23, 0x64, 0x1A, 0xBA, 0x61, 0x76}, + []byte{0x00, 0x4B, 0xD6, 0xEF, 0x09, 0x17, 0x60, 0x62}, + []byte{0x45, 0x20, 0x31, 0xC1, 0xE4, 0xFA, 0xDA, 0x8E}}, + { + []byte{0x02, 0x58, 0x16, 0x16, 0x46, 0x29, 0xB0, 0x07}, + []byte{0x48, 0x0D, 0x39, 0x00, 0x6E, 0xE7, 0x62, 0xF2}, + []byte{0x75, 0x55, 0xAE, 0x39, 0xF5, 0x9B, 0x87, 0xBD}}, + { + []byte{0x49, 0x79, 0x3E, 0xBC, 0x79, 0xB3, 0x25, 0x8F}, + []byte{0x43, 0x75, 0x40, 0xC8, 0x69, 0x8F, 0x3C, 0xFA}, + []byte{0x53, 0xC5, 0x5F, 0x9C, 0xB4, 0x9F, 0xC0, 0x19}}, + { + []byte{0x4F, 0xB0, 0x5E, 0x15, 0x15, 0xAB, 0x73, 0xA7}, + []byte{0x07, 0x2D, 0x43, 0xA0, 0x77, 0x07, 0x52, 0x92}, + []byte{0x7A, 0x8E, 0x7B, 0xFA, 0x93, 0x7E, 0x89, 0xA3}}, + { + []byte{0x49, 0xE9, 0x5D, 0x6D, 0x4C, 0xA2, 0x29, 0xBF}, + []byte{0x02, 0xFE, 0x55, 0x77, 0x81, 0x17, 0xF1, 0x2A}, + []byte{0xCF, 0x9C, 0x5D, 0x7A, 0x49, 0x86, 0xAD, 0xB5}}, + { + []byte{0x01, 0x83, 0x10, 0xDC, 0x40, 0x9B, 0x26, 0xD6}, + []byte{0x1D, 0x9D, 0x5C, 0x50, 0x18, 0xF7, 0x28, 0xC2}, + []byte{0xD1, 0xAB, 0xB2, 0x90, 0x65, 0x8B, 0xC7, 0x78}}, + { + []byte{0x1C, 0x58, 0x7F, 0x1C, 0x13, 0x92, 0x4F, 0xEF}, + []byte{0x30, 0x55, 0x32, 0x28, 0x6D, 0x6F, 0x29, 0x5A}, + []byte{0x55, 0xCB, 0x37, 0x74, 0xD1, 0x3E, 0xF2, 0x01}}, + { + []byte{0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01}, + []byte{0x01, 0x23, 0x45, 0x67, 0x89, 0xAB, 0xCD, 0xEF}, + []byte{0xFA, 0x34, 0xEC, 0x48, 0x47, 0xB2, 0x68, 0xB2}}, + { + []byte{0x1F, 0x1F, 0x1F, 0x1F, 0x0E, 0x0E, 0x0E, 0x0E}, + []byte{0x01, 0x23, 0x45, 0x67, 0x89, 0xAB, 0xCD, 0xEF}, + []byte{0xA7, 0x90, 0x79, 0x51, 0x08, 0xEA, 0x3C, 0xAE}}, + { + []byte{0xE0, 0xFE, 0xE0, 0xFE, 0xF1, 0xFE, 0xF1, 0xFE}, + []byte{0x01, 0x23, 0x45, 0x67, 0x89, 0xAB, 0xCD, 0xEF}, + []byte{0xC3, 0x9E, 0x07, 0x2D, 0x9F, 0xAC, 0x63, 0x1D}}, + { + []byte{0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00}, + []byte{0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF}, + []byte{0x01, 0x49, 0x33, 0xE0, 0xCD, 0xAF, 0xF6, 0xE4}}, + { + []byte{0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF}, + []byte{0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00}, + []byte{0xF2, 0x1E, 0x9A, 0x77, 0xB7, 0x1C, 0x49, 0xBC}}, + { + []byte{0x01, 0x23, 0x45, 0x67, 0x89, 0xAB, 0xCD, 0xEF}, + []byte{0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00}, + []byte{0x24, 0x59, 0x46, 0x88, 0x57, 0x54, 0x36, 0x9A}}, + { + []byte{0xFE, 0xDC, 0xBA, 0x98, 0x76, 0x54, 0x32, 0x10}, + []byte{0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF}, + []byte{0x6B, 0x5C, 0x5A, 0x9C, 0x5D, 0x9E, 0x0A, 0x5A}}, +} + +func TestCipherEncrypt(t *testing.T) { + for i, tt := range encryptTests { + c, err := NewCipher(tt.key) + if err != nil { + t.Errorf("NewCipher(%d bytes) = %s", len(tt.key), err) + continue + } + ct := make([]byte, len(tt.out)) + c.Encrypt(ct, tt.in) + for j, v := range ct { + if v != tt.out[j] { + t.Errorf("Cipher.Encrypt, test vector #%d: cipher-text[%d] = %#x, expected %#x", i, j, v, tt.out[j]) + break + } + } + } +} + +func TestCipherDecrypt(t *testing.T) { + for i, tt := range encryptTests { + c, err := NewCipher(tt.key) + if err != nil { + t.Errorf("NewCipher(%d bytes) = %s", len(tt.key), err) + continue + } + pt := make([]byte, len(tt.in)) + c.Decrypt(pt, tt.out) + for j, v := range pt { + if v != tt.in[j] { + t.Errorf("Cipher.Decrypt, test vector #%d: plain-text[%d] = %#x, expected %#x", i, j, v, tt.in[j]) + break + } + } + } +} diff --git a/src/pkg/crypto/blowfish/cipher.go b/src/pkg/crypto/blowfish/cipher.go new file mode 100644 index 000000000..6c37dfe94 --- /dev/null +++ b/src/pkg/crypto/blowfish/cipher.go @@ -0,0 +1,79 @@ +// Copyright 2010 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Package blowfish implements Bruce Schneier's Blowfish encryption algorithm. +package blowfish + +// The code is a port of Bruce Schneier's C implementation. +// See http://www.schneier.com/blowfish.html. + +import ( + "os" + "strconv" +) + +// The Blowfish block size in bytes. +const BlockSize = 8 + +// A Cipher is an instance of Blowfish encryption using a particular key. +type Cipher struct { + p [18]uint32 + s0, s1, s2, s3 [256]uint32 +} + +type KeySizeError int + +func (k KeySizeError) String() string { + return "crypto/blowfish: invalid key size " + strconv.Itoa(int(k)) +} + +// NewCipher creates and returns a Cipher. +// The key argument should be the Blowfish key, 4 to 56 bytes. +func NewCipher(key []byte) (*Cipher, os.Error) { + k := len(key) + if k < 4 || k > 56 { + return nil, KeySizeError(k) + } + var result Cipher + expandKey(key, &result) + return &result, nil +} + +// BlockSize returns the Blowfish block size, 8 bytes. +// It is necessary to satisfy the Cipher interface in the +// package "crypto/cipher". +func (c *Cipher) BlockSize() int { return BlockSize } + +// Encrypt encrypts the 8-byte buffer src using the key k +// and stores the result in dst. +// Note that for amounts of data larger than a block, +// it is not safe to just call Encrypt on successive blocks; +// instead, use an encryption mode like CBC (see crypto/cipher/cbc.go). +func (c *Cipher) Encrypt(dst, src []byte) { + l := uint32(src[0])<<24 | uint32(src[1])<<16 | uint32(src[2])<<8 | uint32(src[3]) + r := uint32(src[4])<<24 | uint32(src[5])<<16 | uint32(src[6])<<8 | uint32(src[7]) + l, r = encryptBlock(l, r, c) + dst[0], dst[1], dst[2], dst[3] = byte(l>>24), byte(l>>16), byte(l>>8), byte(l) + dst[4], dst[5], dst[6], dst[7] = byte(r>>24), byte(r>>16), byte(r>>8), byte(r) +} + +// Decrypt decrypts the 8-byte buffer src using the key k +// and stores the result in dst. +func (c *Cipher) Decrypt(dst, src []byte) { + l := uint32(src[0])<<24 | uint32(src[1])<<16 | uint32(src[2])<<8 | uint32(src[3]) + r := uint32(src[4])<<24 | uint32(src[5])<<16 | uint32(src[6])<<8 | uint32(src[7]) + l, r = decryptBlock(l, r, c) + dst[0], dst[1], dst[2], dst[3] = byte(l>>24), byte(l>>16), byte(l>>8), byte(l) + dst[4], dst[5], dst[6], dst[7] = byte(r>>24), byte(r>>16), byte(r>>8), byte(r) +} + +// Reset zeros the key data, so that it will no longer +// appear in the process's memory. +func (c *Cipher) Reset() { + zero(c.p[0:]) + zero(c.s0[0:]) + zero(c.s1[0:]) + zero(c.s2[0:]) + zero(c.s3[0:]) +} diff --git a/src/pkg/crypto/blowfish/const.go b/src/pkg/crypto/blowfish/const.go new file mode 100644 index 000000000..8c5ee4cb0 --- /dev/null +++ b/src/pkg/crypto/blowfish/const.go @@ -0,0 +1,199 @@ +// Copyright 2010 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// The startup permutation array and substitution boxes. +// They are the hexadecimal digits of PI; see: +// http://www.schneier.com/code/constants.txt. + +package blowfish + +var s0 = [256]uint32{ + 0xd1310ba6, 0x98dfb5ac, 0x2ffd72db, 0xd01adfb7, 0xb8e1afed, 0x6a267e96, + 0xba7c9045, 0xf12c7f99, 0x24a19947, 0xb3916cf7, 0x0801f2e2, 0x858efc16, + 0x636920d8, 0x71574e69, 0xa458fea3, 0xf4933d7e, 0x0d95748f, 0x728eb658, + 0x718bcd58, 0x82154aee, 0x7b54a41d, 0xc25a59b5, 0x9c30d539, 0x2af26013, + 0xc5d1b023, 0x286085f0, 0xca417918, 0xb8db38ef, 0x8e79dcb0, 0x603a180e, + 0x6c9e0e8b, 0xb01e8a3e, 0xd71577c1, 0xbd314b27, 0x78af2fda, 0x55605c60, + 0xe65525f3, 0xaa55ab94, 0x57489862, 0x63e81440, 0x55ca396a, 0x2aab10b6, + 0xb4cc5c34, 0x1141e8ce, 0xa15486af, 0x7c72e993, 0xb3ee1411, 0x636fbc2a, + 0x2ba9c55d, 0x741831f6, 0xce5c3e16, 0x9b87931e, 0xafd6ba33, 0x6c24cf5c, + 0x7a325381, 0x28958677, 0x3b8f4898, 0x6b4bb9af, 0xc4bfe81b, 0x66282193, + 0x61d809cc, 0xfb21a991, 0x487cac60, 0x5dec8032, 0xef845d5d, 0xe98575b1, + 0xdc262302, 0xeb651b88, 0x23893e81, 0xd396acc5, 0x0f6d6ff3, 0x83f44239, + 0x2e0b4482, 0xa4842004, 0x69c8f04a, 0x9e1f9b5e, 0x21c66842, 0xf6e96c9a, + 0x670c9c61, 0xabd388f0, 0x6a51a0d2, 0xd8542f68, 0x960fa728, 0xab5133a3, + 0x6eef0b6c, 0x137a3be4, 0xba3bf050, 0x7efb2a98, 0xa1f1651d, 0x39af0176, + 0x66ca593e, 0x82430e88, 0x8cee8619, 0x456f9fb4, 0x7d84a5c3, 0x3b8b5ebe, + 0xe06f75d8, 0x85c12073, 0x401a449f, 0x56c16aa6, 0x4ed3aa62, 0x363f7706, + 0x1bfedf72, 0x429b023d, 0x37d0d724, 0xd00a1248, 0xdb0fead3, 0x49f1c09b, + 0x075372c9, 0x80991b7b, 0x25d479d8, 0xf6e8def7, 0xe3fe501a, 0xb6794c3b, + 0x976ce0bd, 0x04c006ba, 0xc1a94fb6, 0x409f60c4, 0x5e5c9ec2, 0x196a2463, + 0x68fb6faf, 0x3e6c53b5, 0x1339b2eb, 0x3b52ec6f, 0x6dfc511f, 0x9b30952c, + 0xcc814544, 0xaf5ebd09, 0xbee3d004, 0xde334afd, 0x660f2807, 0x192e4bb3, + 0xc0cba857, 0x45c8740f, 0xd20b5f39, 0xb9d3fbdb, 0x5579c0bd, 0x1a60320a, + 0xd6a100c6, 0x402c7279, 0x679f25fe, 0xfb1fa3cc, 0x8ea5e9f8, 0xdb3222f8, + 0x3c7516df, 0xfd616b15, 0x2f501ec8, 0xad0552ab, 0x323db5fa, 0xfd238760, + 0x53317b48, 0x3e00df82, 0x9e5c57bb, 0xca6f8ca0, 0x1a87562e, 0xdf1769db, + 0xd542a8f6, 0x287effc3, 0xac6732c6, 0x8c4f5573, 0x695b27b0, 0xbbca58c8, + 0xe1ffa35d, 0xb8f011a0, 0x10fa3d98, 0xfd2183b8, 0x4afcb56c, 0x2dd1d35b, + 0x9a53e479, 0xb6f84565, 0xd28e49bc, 0x4bfb9790, 0xe1ddf2da, 0xa4cb7e33, + 0x62fb1341, 0xcee4c6e8, 0xef20cada, 0x36774c01, 0xd07e9efe, 0x2bf11fb4, + 0x95dbda4d, 0xae909198, 0xeaad8e71, 0x6b93d5a0, 0xd08ed1d0, 0xafc725e0, + 0x8e3c5b2f, 0x8e7594b7, 0x8ff6e2fb, 0xf2122b64, 0x8888b812, 0x900df01c, + 0x4fad5ea0, 0x688fc31c, 0xd1cff191, 0xb3a8c1ad, 0x2f2f2218, 0xbe0e1777, + 0xea752dfe, 0x8b021fa1, 0xe5a0cc0f, 0xb56f74e8, 0x18acf3d6, 0xce89e299, + 0xb4a84fe0, 0xfd13e0b7, 0x7cc43b81, 0xd2ada8d9, 0x165fa266, 0x80957705, + 0x93cc7314, 0x211a1477, 0xe6ad2065, 0x77b5fa86, 0xc75442f5, 0xfb9d35cf, + 0xebcdaf0c, 0x7b3e89a0, 0xd6411bd3, 0xae1e7e49, 0x00250e2d, 0x2071b35e, + 0x226800bb, 0x57b8e0af, 0x2464369b, 0xf009b91e, 0x5563911d, 0x59dfa6aa, + 0x78c14389, 0xd95a537f, 0x207d5ba2, 0x02e5b9c5, 0x83260376, 0x6295cfa9, + 0x11c81968, 0x4e734a41, 0xb3472dca, 0x7b14a94a, 0x1b510052, 0x9a532915, + 0xd60f573f, 0xbc9bc6e4, 0x2b60a476, 0x81e67400, 0x08ba6fb5, 0x571be91f, + 0xf296ec6b, 0x2a0dd915, 0xb6636521, 0xe7b9f9b6, 0xff34052e, 0xc5855664, + 0x53b02d5d, 0xa99f8fa1, 0x08ba4799, 0x6e85076a, +} + +var s1 = [256]uint32{ + 0x4b7a70e9, 0xb5b32944, 0xdb75092e, 0xc4192623, 0xad6ea6b0, 0x49a7df7d, + 0x9cee60b8, 0x8fedb266, 0xecaa8c71, 0x699a17ff, 0x5664526c, 0xc2b19ee1, + 0x193602a5, 0x75094c29, 0xa0591340, 0xe4183a3e, 0x3f54989a, 0x5b429d65, + 0x6b8fe4d6, 0x99f73fd6, 0xa1d29c07, 0xefe830f5, 0x4d2d38e6, 0xf0255dc1, + 0x4cdd2086, 0x8470eb26, 0x6382e9c6, 0x021ecc5e, 0x09686b3f, 0x3ebaefc9, + 0x3c971814, 0x6b6a70a1, 0x687f3584, 0x52a0e286, 0xb79c5305, 0xaa500737, + 0x3e07841c, 0x7fdeae5c, 0x8e7d44ec, 0x5716f2b8, 0xb03ada37, 0xf0500c0d, + 0xf01c1f04, 0x0200b3ff, 0xae0cf51a, 0x3cb574b2, 0x25837a58, 0xdc0921bd, + 0xd19113f9, 0x7ca92ff6, 0x94324773, 0x22f54701, 0x3ae5e581, 0x37c2dadc, + 0xc8b57634, 0x9af3dda7, 0xa9446146, 0x0fd0030e, 0xecc8c73e, 0xa4751e41, + 0xe238cd99, 0x3bea0e2f, 0x3280bba1, 0x183eb331, 0x4e548b38, 0x4f6db908, + 0x6f420d03, 0xf60a04bf, 0x2cb81290, 0x24977c79, 0x5679b072, 0xbcaf89af, + 0xde9a771f, 0xd9930810, 0xb38bae12, 0xdccf3f2e, 0x5512721f, 0x2e6b7124, + 0x501adde6, 0x9f84cd87, 0x7a584718, 0x7408da17, 0xbc9f9abc, 0xe94b7d8c, + 0xec7aec3a, 0xdb851dfa, 0x63094366, 0xc464c3d2, 0xef1c1847, 0x3215d908, + 0xdd433b37, 0x24c2ba16, 0x12a14d43, 0x2a65c451, 0x50940002, 0x133ae4dd, + 0x71dff89e, 0x10314e55, 0x81ac77d6, 0x5f11199b, 0x043556f1, 0xd7a3c76b, + 0x3c11183b, 0x5924a509, 0xf28fe6ed, 0x97f1fbfa, 0x9ebabf2c, 0x1e153c6e, + 0x86e34570, 0xeae96fb1, 0x860e5e0a, 0x5a3e2ab3, 0x771fe71c, 0x4e3d06fa, + 0x2965dcb9, 0x99e71d0f, 0x803e89d6, 0x5266c825, 0x2e4cc978, 0x9c10b36a, + 0xc6150eba, 0x94e2ea78, 0xa5fc3c53, 0x1e0a2df4, 0xf2f74ea7, 0x361d2b3d, + 0x1939260f, 0x19c27960, 0x5223a708, 0xf71312b6, 0xebadfe6e, 0xeac31f66, + 0xe3bc4595, 0xa67bc883, 0xb17f37d1, 0x018cff28, 0xc332ddef, 0xbe6c5aa5, + 0x65582185, 0x68ab9802, 0xeecea50f, 0xdb2f953b, 0x2aef7dad, 0x5b6e2f84, + 0x1521b628, 0x29076170, 0xecdd4775, 0x619f1510, 0x13cca830, 0xeb61bd96, + 0x0334fe1e, 0xaa0363cf, 0xb5735c90, 0x4c70a239, 0xd59e9e0b, 0xcbaade14, + 0xeecc86bc, 0x60622ca7, 0x9cab5cab, 0xb2f3846e, 0x648b1eaf, 0x19bdf0ca, + 0xa02369b9, 0x655abb50, 0x40685a32, 0x3c2ab4b3, 0x319ee9d5, 0xc021b8f7, + 0x9b540b19, 0x875fa099, 0x95f7997e, 0x623d7da8, 0xf837889a, 0x97e32d77, + 0x11ed935f, 0x16681281, 0x0e358829, 0xc7e61fd6, 0x96dedfa1, 0x7858ba99, + 0x57f584a5, 0x1b227263, 0x9b83c3ff, 0x1ac24696, 0xcdb30aeb, 0x532e3054, + 0x8fd948e4, 0x6dbc3128, 0x58ebf2ef, 0x34c6ffea, 0xfe28ed61, 0xee7c3c73, + 0x5d4a14d9, 0xe864b7e3, 0x42105d14, 0x203e13e0, 0x45eee2b6, 0xa3aaabea, + 0xdb6c4f15, 0xfacb4fd0, 0xc742f442, 0xef6abbb5, 0x654f3b1d, 0x41cd2105, + 0xd81e799e, 0x86854dc7, 0xe44b476a, 0x3d816250, 0xcf62a1f2, 0x5b8d2646, + 0xfc8883a0, 0xc1c7b6a3, 0x7f1524c3, 0x69cb7492, 0x47848a0b, 0x5692b285, + 0x095bbf00, 0xad19489d, 0x1462b174, 0x23820e00, 0x58428d2a, 0x0c55f5ea, + 0x1dadf43e, 0x233f7061, 0x3372f092, 0x8d937e41, 0xd65fecf1, 0x6c223bdb, + 0x7cde3759, 0xcbee7460, 0x4085f2a7, 0xce77326e, 0xa6078084, 0x19f8509e, + 0xe8efd855, 0x61d99735, 0xa969a7aa, 0xc50c06c2, 0x5a04abfc, 0x800bcadc, + 0x9e447a2e, 0xc3453484, 0xfdd56705, 0x0e1e9ec9, 0xdb73dbd3, 0x105588cd, + 0x675fda79, 0xe3674340, 0xc5c43465, 0x713e38d8, 0x3d28f89e, 0xf16dff20, + 0x153e21e7, 0x8fb03d4a, 0xe6e39f2b, 0xdb83adf7, +} + +var s2 = [256]uint32{ + 0xe93d5a68, 0x948140f7, 0xf64c261c, 0x94692934, 0x411520f7, 0x7602d4f7, + 0xbcf46b2e, 0xd4a20068, 0xd4082471, 0x3320f46a, 0x43b7d4b7, 0x500061af, + 0x1e39f62e, 0x97244546, 0x14214f74, 0xbf8b8840, 0x4d95fc1d, 0x96b591af, + 0x70f4ddd3, 0x66a02f45, 0xbfbc09ec, 0x03bd9785, 0x7fac6dd0, 0x31cb8504, + 0x96eb27b3, 0x55fd3941, 0xda2547e6, 0xabca0a9a, 0x28507825, 0x530429f4, + 0x0a2c86da, 0xe9b66dfb, 0x68dc1462, 0xd7486900, 0x680ec0a4, 0x27a18dee, + 0x4f3ffea2, 0xe887ad8c, 0xb58ce006, 0x7af4d6b6, 0xaace1e7c, 0xd3375fec, + 0xce78a399, 0x406b2a42, 0x20fe9e35, 0xd9f385b9, 0xee39d7ab, 0x3b124e8b, + 0x1dc9faf7, 0x4b6d1856, 0x26a36631, 0xeae397b2, 0x3a6efa74, 0xdd5b4332, + 0x6841e7f7, 0xca7820fb, 0xfb0af54e, 0xd8feb397, 0x454056ac, 0xba489527, + 0x55533a3a, 0x20838d87, 0xfe6ba9b7, 0xd096954b, 0x55a867bc, 0xa1159a58, + 0xcca92963, 0x99e1db33, 0xa62a4a56, 0x3f3125f9, 0x5ef47e1c, 0x9029317c, + 0xfdf8e802, 0x04272f70, 0x80bb155c, 0x05282ce3, 0x95c11548, 0xe4c66d22, + 0x48c1133f, 0xc70f86dc, 0x07f9c9ee, 0x41041f0f, 0x404779a4, 0x5d886e17, + 0x325f51eb, 0xd59bc0d1, 0xf2bcc18f, 0x41113564, 0x257b7834, 0x602a9c60, + 0xdff8e8a3, 0x1f636c1b, 0x0e12b4c2, 0x02e1329e, 0xaf664fd1, 0xcad18115, + 0x6b2395e0, 0x333e92e1, 0x3b240b62, 0xeebeb922, 0x85b2a20e, 0xe6ba0d99, + 0xde720c8c, 0x2da2f728, 0xd0127845, 0x95b794fd, 0x647d0862, 0xe7ccf5f0, + 0x5449a36f, 0x877d48fa, 0xc39dfd27, 0xf33e8d1e, 0x0a476341, 0x992eff74, + 0x3a6f6eab, 0xf4f8fd37, 0xa812dc60, 0xa1ebddf8, 0x991be14c, 0xdb6e6b0d, + 0xc67b5510, 0x6d672c37, 0x2765d43b, 0xdcd0e804, 0xf1290dc7, 0xcc00ffa3, + 0xb5390f92, 0x690fed0b, 0x667b9ffb, 0xcedb7d9c, 0xa091cf0b, 0xd9155ea3, + 0xbb132f88, 0x515bad24, 0x7b9479bf, 0x763bd6eb, 0x37392eb3, 0xcc115979, + 0x8026e297, 0xf42e312d, 0x6842ada7, 0xc66a2b3b, 0x12754ccc, 0x782ef11c, + 0x6a124237, 0xb79251e7, 0x06a1bbe6, 0x4bfb6350, 0x1a6b1018, 0x11caedfa, + 0x3d25bdd8, 0xe2e1c3c9, 0x44421659, 0x0a121386, 0xd90cec6e, 0xd5abea2a, + 0x64af674e, 0xda86a85f, 0xbebfe988, 0x64e4c3fe, 0x9dbc8057, 0xf0f7c086, + 0x60787bf8, 0x6003604d, 0xd1fd8346, 0xf6381fb0, 0x7745ae04, 0xd736fccc, + 0x83426b33, 0xf01eab71, 0xb0804187, 0x3c005e5f, 0x77a057be, 0xbde8ae24, + 0x55464299, 0xbf582e61, 0x4e58f48f, 0xf2ddfda2, 0xf474ef38, 0x8789bdc2, + 0x5366f9c3, 0xc8b38e74, 0xb475f255, 0x46fcd9b9, 0x7aeb2661, 0x8b1ddf84, + 0x846a0e79, 0x915f95e2, 0x466e598e, 0x20b45770, 0x8cd55591, 0xc902de4c, + 0xb90bace1, 0xbb8205d0, 0x11a86248, 0x7574a99e, 0xb77f19b6, 0xe0a9dc09, + 0x662d09a1, 0xc4324633, 0xe85a1f02, 0x09f0be8c, 0x4a99a025, 0x1d6efe10, + 0x1ab93d1d, 0x0ba5a4df, 0xa186f20f, 0x2868f169, 0xdcb7da83, 0x573906fe, + 0xa1e2ce9b, 0x4fcd7f52, 0x50115e01, 0xa70683fa, 0xa002b5c4, 0x0de6d027, + 0x9af88c27, 0x773f8641, 0xc3604c06, 0x61a806b5, 0xf0177a28, 0xc0f586e0, + 0x006058aa, 0x30dc7d62, 0x11e69ed7, 0x2338ea63, 0x53c2dd94, 0xc2c21634, + 0xbbcbee56, 0x90bcb6de, 0xebfc7da1, 0xce591d76, 0x6f05e409, 0x4b7c0188, + 0x39720a3d, 0x7c927c24, 0x86e3725f, 0x724d9db9, 0x1ac15bb4, 0xd39eb8fc, + 0xed545578, 0x08fca5b5, 0xd83d7cd3, 0x4dad0fc4, 0x1e50ef5e, 0xb161e6f8, + 0xa28514d9, 0x6c51133c, 0x6fd5c7e7, 0x56e14ec4, 0x362abfce, 0xddc6c837, + 0xd79a3234, 0x92638212, 0x670efa8e, 0x406000e0, +} + +var s3 = [256]uint32{ + 0x3a39ce37, 0xd3faf5cf, 0xabc27737, 0x5ac52d1b, 0x5cb0679e, 0x4fa33742, + 0xd3822740, 0x99bc9bbe, 0xd5118e9d, 0xbf0f7315, 0xd62d1c7e, 0xc700c47b, + 0xb78c1b6b, 0x21a19045, 0xb26eb1be, 0x6a366eb4, 0x5748ab2f, 0xbc946e79, + 0xc6a376d2, 0x6549c2c8, 0x530ff8ee, 0x468dde7d, 0xd5730a1d, 0x4cd04dc6, + 0x2939bbdb, 0xa9ba4650, 0xac9526e8, 0xbe5ee304, 0xa1fad5f0, 0x6a2d519a, + 0x63ef8ce2, 0x9a86ee22, 0xc089c2b8, 0x43242ef6, 0xa51e03aa, 0x9cf2d0a4, + 0x83c061ba, 0x9be96a4d, 0x8fe51550, 0xba645bd6, 0x2826a2f9, 0xa73a3ae1, + 0x4ba99586, 0xef5562e9, 0xc72fefd3, 0xf752f7da, 0x3f046f69, 0x77fa0a59, + 0x80e4a915, 0x87b08601, 0x9b09e6ad, 0x3b3ee593, 0xe990fd5a, 0x9e34d797, + 0x2cf0b7d9, 0x022b8b51, 0x96d5ac3a, 0x017da67d, 0xd1cf3ed6, 0x7c7d2d28, + 0x1f9f25cf, 0xadf2b89b, 0x5ad6b472, 0x5a88f54c, 0xe029ac71, 0xe019a5e6, + 0x47b0acfd, 0xed93fa9b, 0xe8d3c48d, 0x283b57cc, 0xf8d56629, 0x79132e28, + 0x785f0191, 0xed756055, 0xf7960e44, 0xe3d35e8c, 0x15056dd4, 0x88f46dba, + 0x03a16125, 0x0564f0bd, 0xc3eb9e15, 0x3c9057a2, 0x97271aec, 0xa93a072a, + 0x1b3f6d9b, 0x1e6321f5, 0xf59c66fb, 0x26dcf319, 0x7533d928, 0xb155fdf5, + 0x03563482, 0x8aba3cbb, 0x28517711, 0xc20ad9f8, 0xabcc5167, 0xccad925f, + 0x4de81751, 0x3830dc8e, 0x379d5862, 0x9320f991, 0xea7a90c2, 0xfb3e7bce, + 0x5121ce64, 0x774fbe32, 0xa8b6e37e, 0xc3293d46, 0x48de5369, 0x6413e680, + 0xa2ae0810, 0xdd6db224, 0x69852dfd, 0x09072166, 0xb39a460a, 0x6445c0dd, + 0x586cdecf, 0x1c20c8ae, 0x5bbef7dd, 0x1b588d40, 0xccd2017f, 0x6bb4e3bb, + 0xdda26a7e, 0x3a59ff45, 0x3e350a44, 0xbcb4cdd5, 0x72eacea8, 0xfa6484bb, + 0x8d6612ae, 0xbf3c6f47, 0xd29be463, 0x542f5d9e, 0xaec2771b, 0xf64e6370, + 0x740e0d8d, 0xe75b1357, 0xf8721671, 0xaf537d5d, 0x4040cb08, 0x4eb4e2cc, + 0x34d2466a, 0x0115af84, 0xe1b00428, 0x95983a1d, 0x06b89fb4, 0xce6ea048, + 0x6f3f3b82, 0x3520ab82, 0x011a1d4b, 0x277227f8, 0x611560b1, 0xe7933fdc, + 0xbb3a792b, 0x344525bd, 0xa08839e1, 0x51ce794b, 0x2f32c9b7, 0xa01fbac9, + 0xe01cc87e, 0xbcc7d1f6, 0xcf0111c3, 0xa1e8aac7, 0x1a908749, 0xd44fbd9a, + 0xd0dadecb, 0xd50ada38, 0x0339c32a, 0xc6913667, 0x8df9317c, 0xe0b12b4f, + 0xf79e59b7, 0x43f5bb3a, 0xf2d519ff, 0x27d9459c, 0xbf97222c, 0x15e6fc2a, + 0x0f91fc71, 0x9b941525, 0xfae59361, 0xceb69ceb, 0xc2a86459, 0x12baa8d1, + 0xb6c1075e, 0xe3056a0c, 0x10d25065, 0xcb03a442, 0xe0ec6e0e, 0x1698db3b, + 0x4c98a0be, 0x3278e964, 0x9f1f9532, 0xe0d392df, 0xd3a0342b, 0x8971f21e, + 0x1b0a7441, 0x4ba3348c, 0xc5be7120, 0xc37632d8, 0xdf359f8d, 0x9b992f2e, + 0xe60b6f47, 0x0fe3f11d, 0xe54cda54, 0x1edad891, 0xce6279cf, 0xcd3e7e6f, + 0x1618b166, 0xfd2c1d05, 0x848fd2c5, 0xf6fb2299, 0xf523f357, 0xa6327623, + 0x93a83531, 0x56cccd02, 0xacf08162, 0x5a75ebb5, 0x6e163697, 0x88d273cc, + 0xde966292, 0x81b949d0, 0x4c50901b, 0x71c65614, 0xe6c6c7bd, 0x327a140a, + 0x45e1d006, 0xc3f27b9a, 0xc9aa53fd, 0x62a80f00, 0xbb25bfe2, 0x35bdd2f6, + 0x71126905, 0xb2040222, 0xb6cbcf7c, 0xcd769c2b, 0x53113ec0, 0x1640e3d3, + 0x38abbd60, 0x2547adf0, 0xba38209c, 0xf746ce76, 0x77afa1c5, 0x20756060, + 0x85cbfe4e, 0x8ae88dd8, 0x7aaaf9b0, 0x4cf9aa7e, 0x1948c25c, 0x02fb8a8c, + 0x01c36ae4, 0xd6ebe1f9, 0x90d4f869, 0xa65cdea0, 0x3f09252d, 0xc208e69f, + 0xb74e6132, 0xce77e25b, 0x578fdfe3, 0x3ac372e6, +} + +var p = [18]uint32{ + 0x243f6a88, 0x85a308d3, 0x13198a2e, 0x03707344, 0xa4093822, 0x299f31d0, + 0x082efa98, 0xec4e6c89, 0x452821e6, 0x38d01377, 0xbe5466cf, 0x34e90c6c, + 0xc0ac29b7, 0xc97c50dd, 0x3f84d5b5, 0xb5470917, 0x9216d5d9, 0x8979fb1b, +} diff --git a/src/pkg/crypto/cast5/Makefile b/src/pkg/crypto/cast5/Makefile new file mode 100644 index 000000000..346fadd94 --- /dev/null +++ b/src/pkg/crypto/cast5/Makefile @@ -0,0 +1,11 @@ +# Copyright 2010 The Go Authors. All rights reserved. +# Use of this source code is governed by a BSD-style +# license that can be found in the LICENSE file. + +include ../../../Make.inc + +TARG=crypto/cast5 +GOFILES=\ + cast5.go\ + +include ../../../Make.pkg diff --git a/src/pkg/crypto/cast5/cast5.go b/src/pkg/crypto/cast5/cast5.go new file mode 100644 index 000000000..e9d4a24e2 --- /dev/null +++ b/src/pkg/crypto/cast5/cast5.go @@ -0,0 +1,536 @@ +// Copyright 2010 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Package cast5 implements CAST5, as defined in RFC 2144. CAST5 is a common +// OpenPGP cipher. +package cast5 + +import ( + "os" +) + +const BlockSize = 8 +const KeySize = 16 + +type Cipher struct { + masking [16]uint32 + rotate [16]uint8 +} + +func NewCipher(key []byte) (c *Cipher, err os.Error) { + if len(key) != KeySize { + return nil, os.NewError("CAST5: keys must be 16 bytes") + } + + c = new(Cipher) + c.keySchedule(key) + return +} + +func (c *Cipher) BlockSize() int { + return BlockSize +} + +// Reset zeros the key material in memory. +func (c *Cipher) Reset() { + for i := 0; i < 16; i++ { + c.masking[i] = 0 + c.rotate[i] = 0 + } +} + +func (c *Cipher) Encrypt(dst, src []byte) { + l := uint32(src[0])<<24 | uint32(src[1])<<16 | uint32(src[2])<<8 | uint32(src[3]) + r := uint32(src[4])<<24 | uint32(src[5])<<16 | uint32(src[6])<<8 | uint32(src[7]) + + l, r = r, l^f1(r, c.masking[0], c.rotate[0]) + l, r = r, l^f2(r, c.masking[1], c.rotate[1]) + l, r = r, l^f3(r, c.masking[2], c.rotate[2]) + l, r = r, l^f1(r, c.masking[3], c.rotate[3]) + + l, r = r, l^f2(r, c.masking[4], c.rotate[4]) + l, r = r, l^f3(r, c.masking[5], c.rotate[5]) + l, r = r, l^f1(r, c.masking[6], c.rotate[6]) + l, r = r, l^f2(r, c.masking[7], c.rotate[7]) + + l, r = r, l^f3(r, c.masking[8], c.rotate[8]) + l, r = r, l^f1(r, c.masking[9], c.rotate[9]) + l, r = r, l^f2(r, c.masking[10], c.rotate[10]) + l, r = r, l^f3(r, c.masking[11], c.rotate[11]) + + l, r = r, l^f1(r, c.masking[12], c.rotate[12]) + l, r = r, l^f2(r, c.masking[13], c.rotate[13]) + l, r = r, l^f3(r, c.masking[14], c.rotate[14]) + l, r = r, l^f1(r, c.masking[15], c.rotate[15]) + + dst[0] = uint8(r >> 24) + dst[1] = uint8(r >> 16) + dst[2] = uint8(r >> 8) + dst[3] = uint8(r) + dst[4] = uint8(l >> 24) + dst[5] = uint8(l >> 16) + dst[6] = uint8(l >> 8) + dst[7] = uint8(l) +} + +func (c *Cipher) Decrypt(dst, src []byte) { + l := uint32(src[0])<<24 | uint32(src[1])<<16 | uint32(src[2])<<8 | uint32(src[3]) + r := uint32(src[4])<<24 | uint32(src[5])<<16 | uint32(src[6])<<8 | uint32(src[7]) + + l, r = r, l^f1(r, c.masking[15], c.rotate[15]) + l, r = r, l^f3(r, c.masking[14], c.rotate[14]) + l, r = r, l^f2(r, c.masking[13], c.rotate[13]) + l, r = r, l^f1(r, c.masking[12], c.rotate[12]) + + l, r = r, l^f3(r, c.masking[11], c.rotate[11]) + l, r = r, l^f2(r, c.masking[10], c.rotate[10]) + l, r = r, l^f1(r, c.masking[9], c.rotate[9]) + l, r = r, l^f3(r, c.masking[8], c.rotate[8]) + + l, r = r, l^f2(r, c.masking[7], c.rotate[7]) + l, r = r, l^f1(r, c.masking[6], c.rotate[6]) + l, r = r, l^f3(r, c.masking[5], c.rotate[5]) + l, r = r, l^f2(r, c.masking[4], c.rotate[4]) + + l, r = r, l^f1(r, c.masking[3], c.rotate[3]) + l, r = r, l^f3(r, c.masking[2], c.rotate[2]) + l, r = r, l^f2(r, c.masking[1], c.rotate[1]) + l, r = r, l^f1(r, c.masking[0], c.rotate[0]) + + dst[0] = uint8(r >> 24) + dst[1] = uint8(r >> 16) + dst[2] = uint8(r >> 8) + dst[3] = uint8(r) + dst[4] = uint8(l >> 24) + dst[5] = uint8(l >> 16) + dst[6] = uint8(l >> 8) + dst[7] = uint8(l) +} + +type keyScheduleA [4][7]uint8 +type keyScheduleB [4][5]uint8 + +// keyScheduleRound contains the magic values for a round of the key schedule. +// The keyScheduleA deals with the lines like: +// z0z1z2z3 = x0x1x2x3 ^ S5[xD] ^ S6[xF] ^ S7[xC] ^ S8[xE] ^ S7[x8] +// Conceptually, both x and z are in the same array, x first. The first +// element describes which word of this array gets written to and the +// second, which word gets read. So, for the line above, it's "4, 0", because +// it's writing to the first word of z, which, being after x, is word 4, and +// reading from the first word of x: word 0. +// +// Next are the indexes into the S-boxes. Now the array is treated as bytes. So +// "xD" is 0xd. The first byte of z is written as "16 + 0", just to be clear +// that it's z that we're indexing. +// +// keyScheduleB deals with lines like: +// K1 = S5[z8] ^ S6[z9] ^ S7[z7] ^ S8[z6] ^ S5[z2] +// "K1" is ignored because key words are always written in order. So the five +// elements are the S-box indexes. They use the same form as in keyScheduleA, +// above. + +type keyScheduleRound struct{} +type keySchedule []keyScheduleRound + +var schedule = []struct { + a keyScheduleA + b keyScheduleB +}{ + { + keyScheduleA{ + {4, 0, 0xd, 0xf, 0xc, 0xe, 0x8}, + {5, 2, 16 + 0, 16 + 2, 16 + 1, 16 + 3, 0xa}, + {6, 3, 16 + 7, 16 + 6, 16 + 5, 16 + 4, 9}, + {7, 1, 16 + 0xa, 16 + 9, 16 + 0xb, 16 + 8, 0xb}, + }, + keyScheduleB{ + {16 + 8, 16 + 9, 16 + 7, 16 + 6, 16 + 2}, + {16 + 0xa, 16 + 0xb, 16 + 5, 16 + 4, 16 + 6}, + {16 + 0xc, 16 + 0xd, 16 + 3, 16 + 2, 16 + 9}, + {16 + 0xe, 16 + 0xf, 16 + 1, 16 + 0, 16 + 0xc}, + }, + }, + { + keyScheduleA{ + {0, 6, 16 + 5, 16 + 7, 16 + 4, 16 + 6, 16 + 0}, + {1, 4, 0, 2, 1, 3, 16 + 2}, + {2, 5, 7, 6, 5, 4, 16 + 1}, + {3, 7, 0xa, 9, 0xb, 8, 16 + 3}, + }, + keyScheduleB{ + {3, 2, 0xc, 0xd, 8}, + {1, 0, 0xe, 0xf, 0xd}, + {7, 6, 8, 9, 3}, + {5, 4, 0xa, 0xb, 7}, + }, + }, + { + keyScheduleA{ + {4, 0, 0xd, 0xf, 0xc, 0xe, 8}, + {5, 2, 16 + 0, 16 + 2, 16 + 1, 16 + 3, 0xa}, + {6, 3, 16 + 7, 16 + 6, 16 + 5, 16 + 4, 9}, + {7, 1, 16 + 0xa, 16 + 9, 16 + 0xb, 16 + 8, 0xb}, + }, + keyScheduleB{ + {16 + 3, 16 + 2, 16 + 0xc, 16 + 0xd, 16 + 9}, + {16 + 1, 16 + 0, 16 + 0xe, 16 + 0xf, 16 + 0xc}, + {16 + 7, 16 + 6, 16 + 8, 16 + 9, 16 + 2}, + {16 + 5, 16 + 4, 16 + 0xa, 16 + 0xb, 16 + 6}, + }, + }, + { + keyScheduleA{ + {0, 6, 16 + 5, 16 + 7, 16 + 4, 16 + 6, 16 + 0}, + {1, 4, 0, 2, 1, 3, 16 + 2}, + {2, 5, 7, 6, 5, 4, 16 + 1}, + {3, 7, 0xa, 9, 0xb, 8, 16 + 3}, + }, + keyScheduleB{ + {8, 9, 7, 6, 3}, + {0xa, 0xb, 5, 4, 7}, + {0xc, 0xd, 3, 2, 8}, + {0xe, 0xf, 1, 0, 0xd}, + }, + }, +} + +func (c *Cipher) keySchedule(in []byte) { + var t [8]uint32 + var k [32]uint32 + + for i := 0; i < 4; i++ { + j := i * 4 + t[i] = uint32(in[j])<<24 | uint32(in[j+1])<<16 | uint32(in[j+2])<<8 | uint32(in[j+3]) + } + + x := []byte{6, 7, 4, 5} + ki := 0 + + for half := 0; half < 2; half++ { + for _, round := range schedule { + for j := 0; j < 4; j++ { + var a [7]uint8 + copy(a[:], round.a[j][:]) + w := t[a[1]] + w ^= sBox[4][(t[a[2]>>2]>>(24-8*(a[2]&3)))&0xff] + w ^= sBox[5][(t[a[3]>>2]>>(24-8*(a[3]&3)))&0xff] + w ^= sBox[6][(t[a[4]>>2]>>(24-8*(a[4]&3)))&0xff] + w ^= sBox[7][(t[a[5]>>2]>>(24-8*(a[5]&3)))&0xff] + w ^= sBox[x[j]][(t[a[6]>>2]>>(24-8*(a[6]&3)))&0xff] + t[a[0]] = w + } + + for j := 0; j < 4; j++ { + var b [5]uint8 + copy(b[:], round.b[j][:]) + w := sBox[4][(t[b[0]>>2]>>(24-8*(b[0]&3)))&0xff] + w ^= sBox[5][(t[b[1]>>2]>>(24-8*(b[1]&3)))&0xff] + w ^= sBox[6][(t[b[2]>>2]>>(24-8*(b[2]&3)))&0xff] + w ^= sBox[7][(t[b[3]>>2]>>(24-8*(b[3]&3)))&0xff] + w ^= sBox[4+j][(t[b[4]>>2]>>(24-8*(b[4]&3)))&0xff] + k[ki] = w + ki++ + } + } + } + + for i := 0; i < 16; i++ { + c.masking[i] = k[i] + c.rotate[i] = uint8(k[16+i] & 0x1f) + } +} + +// These are the three 'f' functions. See RFC 2144, section 2.2. +func f1(d, m uint32, r uint8) uint32 { + t := m + d + I := (t << r) | (t >> (32 - r)) + return ((sBox[0][I>>24] ^ sBox[1][(I>>16)&0xff]) - sBox[2][(I>>8)&0xff]) + sBox[3][I&0xff] +} + +func f2(d, m uint32, r uint8) uint32 { + t := m ^ d + I := (t << r) | (t >> (32 - r)) + return ((sBox[0][I>>24] - sBox[1][(I>>16)&0xff]) + sBox[2][(I>>8)&0xff]) ^ sBox[3][I&0xff] +} + +func f3(d, m uint32, r uint8) uint32 { + t := m - d + I := (t << r) | (t >> (32 - r)) + return ((sBox[0][I>>24] + sBox[1][(I>>16)&0xff]) ^ sBox[2][(I>>8)&0xff]) - sBox[3][I&0xff] +} + +var sBox = [8][256]uint32{ + { + 0x30fb40d4, 0x9fa0ff0b, 0x6beccd2f, 0x3f258c7a, 0x1e213f2f, 0x9c004dd3, 0x6003e540, 0xcf9fc949, + 0xbfd4af27, 0x88bbbdb5, 0xe2034090, 0x98d09675, 0x6e63a0e0, 0x15c361d2, 0xc2e7661d, 0x22d4ff8e, + 0x28683b6f, 0xc07fd059, 0xff2379c8, 0x775f50e2, 0x43c340d3, 0xdf2f8656, 0x887ca41a, 0xa2d2bd2d, + 0xa1c9e0d6, 0x346c4819, 0x61b76d87, 0x22540f2f, 0x2abe32e1, 0xaa54166b, 0x22568e3a, 0xa2d341d0, + 0x66db40c8, 0xa784392f, 0x004dff2f, 0x2db9d2de, 0x97943fac, 0x4a97c1d8, 0x527644b7, 0xb5f437a7, + 0xb82cbaef, 0xd751d159, 0x6ff7f0ed, 0x5a097a1f, 0x827b68d0, 0x90ecf52e, 0x22b0c054, 0xbc8e5935, + 0x4b6d2f7f, 0x50bb64a2, 0xd2664910, 0xbee5812d, 0xb7332290, 0xe93b159f, 0xb48ee411, 0x4bff345d, + 0xfd45c240, 0xad31973f, 0xc4f6d02e, 0x55fc8165, 0xd5b1caad, 0xa1ac2dae, 0xa2d4b76d, 0xc19b0c50, + 0x882240f2, 0x0c6e4f38, 0xa4e4bfd7, 0x4f5ba272, 0x564c1d2f, 0xc59c5319, 0xb949e354, 0xb04669fe, + 0xb1b6ab8a, 0xc71358dd, 0x6385c545, 0x110f935d, 0x57538ad5, 0x6a390493, 0xe63d37e0, 0x2a54f6b3, + 0x3a787d5f, 0x6276a0b5, 0x19a6fcdf, 0x7a42206a, 0x29f9d4d5, 0xf61b1891, 0xbb72275e, 0xaa508167, + 0x38901091, 0xc6b505eb, 0x84c7cb8c, 0x2ad75a0f, 0x874a1427, 0xa2d1936b, 0x2ad286af, 0xaa56d291, + 0xd7894360, 0x425c750d, 0x93b39e26, 0x187184c9, 0x6c00b32d, 0x73e2bb14, 0xa0bebc3c, 0x54623779, + 0x64459eab, 0x3f328b82, 0x7718cf82, 0x59a2cea6, 0x04ee002e, 0x89fe78e6, 0x3fab0950, 0x325ff6c2, + 0x81383f05, 0x6963c5c8, 0x76cb5ad6, 0xd49974c9, 0xca180dcf, 0x380782d5, 0xc7fa5cf6, 0x8ac31511, + 0x35e79e13, 0x47da91d0, 0xf40f9086, 0xa7e2419e, 0x31366241, 0x051ef495, 0xaa573b04, 0x4a805d8d, + 0x548300d0, 0x00322a3c, 0xbf64cddf, 0xba57a68e, 0x75c6372b, 0x50afd341, 0xa7c13275, 0x915a0bf5, + 0x6b54bfab, 0x2b0b1426, 0xab4cc9d7, 0x449ccd82, 0xf7fbf265, 0xab85c5f3, 0x1b55db94, 0xaad4e324, + 0xcfa4bd3f, 0x2deaa3e2, 0x9e204d02, 0xc8bd25ac, 0xeadf55b3, 0xd5bd9e98, 0xe31231b2, 0x2ad5ad6c, + 0x954329de, 0xadbe4528, 0xd8710f69, 0xaa51c90f, 0xaa786bf6, 0x22513f1e, 0xaa51a79b, 0x2ad344cc, + 0x7b5a41f0, 0xd37cfbad, 0x1b069505, 0x41ece491, 0xb4c332e6, 0x032268d4, 0xc9600acc, 0xce387e6d, + 0xbf6bb16c, 0x6a70fb78, 0x0d03d9c9, 0xd4df39de, 0xe01063da, 0x4736f464, 0x5ad328d8, 0xb347cc96, + 0x75bb0fc3, 0x98511bfb, 0x4ffbcc35, 0xb58bcf6a, 0xe11f0abc, 0xbfc5fe4a, 0xa70aec10, 0xac39570a, + 0x3f04442f, 0x6188b153, 0xe0397a2e, 0x5727cb79, 0x9ceb418f, 0x1cacd68d, 0x2ad37c96, 0x0175cb9d, + 0xc69dff09, 0xc75b65f0, 0xd9db40d8, 0xec0e7779, 0x4744ead4, 0xb11c3274, 0xdd24cb9e, 0x7e1c54bd, + 0xf01144f9, 0xd2240eb1, 0x9675b3fd, 0xa3ac3755, 0xd47c27af, 0x51c85f4d, 0x56907596, 0xa5bb15e6, + 0x580304f0, 0xca042cf1, 0x011a37ea, 0x8dbfaadb, 0x35ba3e4a, 0x3526ffa0, 0xc37b4d09, 0xbc306ed9, + 0x98a52666, 0x5648f725, 0xff5e569d, 0x0ced63d0, 0x7c63b2cf, 0x700b45e1, 0xd5ea50f1, 0x85a92872, + 0xaf1fbda7, 0xd4234870, 0xa7870bf3, 0x2d3b4d79, 0x42e04198, 0x0cd0ede7, 0x26470db8, 0xf881814c, + 0x474d6ad7, 0x7c0c5e5c, 0xd1231959, 0x381b7298, 0xf5d2f4db, 0xab838653, 0x6e2f1e23, 0x83719c9e, + 0xbd91e046, 0x9a56456e, 0xdc39200c, 0x20c8c571, 0x962bda1c, 0xe1e696ff, 0xb141ab08, 0x7cca89b9, + 0x1a69e783, 0x02cc4843, 0xa2f7c579, 0x429ef47d, 0x427b169c, 0x5ac9f049, 0xdd8f0f00, 0x5c8165bf, + }, + { + 0x1f201094, 0xef0ba75b, 0x69e3cf7e, 0x393f4380, 0xfe61cf7a, 0xeec5207a, 0x55889c94, 0x72fc0651, + 0xada7ef79, 0x4e1d7235, 0xd55a63ce, 0xde0436ba, 0x99c430ef, 0x5f0c0794, 0x18dcdb7d, 0xa1d6eff3, + 0xa0b52f7b, 0x59e83605, 0xee15b094, 0xe9ffd909, 0xdc440086, 0xef944459, 0xba83ccb3, 0xe0c3cdfb, + 0xd1da4181, 0x3b092ab1, 0xf997f1c1, 0xa5e6cf7b, 0x01420ddb, 0xe4e7ef5b, 0x25a1ff41, 0xe180f806, + 0x1fc41080, 0x179bee7a, 0xd37ac6a9, 0xfe5830a4, 0x98de8b7f, 0x77e83f4e, 0x79929269, 0x24fa9f7b, + 0xe113c85b, 0xacc40083, 0xd7503525, 0xf7ea615f, 0x62143154, 0x0d554b63, 0x5d681121, 0xc866c359, + 0x3d63cf73, 0xcee234c0, 0xd4d87e87, 0x5c672b21, 0x071f6181, 0x39f7627f, 0x361e3084, 0xe4eb573b, + 0x602f64a4, 0xd63acd9c, 0x1bbc4635, 0x9e81032d, 0x2701f50c, 0x99847ab4, 0xa0e3df79, 0xba6cf38c, + 0x10843094, 0x2537a95e, 0xf46f6ffe, 0xa1ff3b1f, 0x208cfb6a, 0x8f458c74, 0xd9e0a227, 0x4ec73a34, + 0xfc884f69, 0x3e4de8df, 0xef0e0088, 0x3559648d, 0x8a45388c, 0x1d804366, 0x721d9bfd, 0xa58684bb, + 0xe8256333, 0x844e8212, 0x128d8098, 0xfed33fb4, 0xce280ae1, 0x27e19ba5, 0xd5a6c252, 0xe49754bd, + 0xc5d655dd, 0xeb667064, 0x77840b4d, 0xa1b6a801, 0x84db26a9, 0xe0b56714, 0x21f043b7, 0xe5d05860, + 0x54f03084, 0x066ff472, 0xa31aa153, 0xdadc4755, 0xb5625dbf, 0x68561be6, 0x83ca6b94, 0x2d6ed23b, + 0xeccf01db, 0xa6d3d0ba, 0xb6803d5c, 0xaf77a709, 0x33b4a34c, 0x397bc8d6, 0x5ee22b95, 0x5f0e5304, + 0x81ed6f61, 0x20e74364, 0xb45e1378, 0xde18639b, 0x881ca122, 0xb96726d1, 0x8049a7e8, 0x22b7da7b, + 0x5e552d25, 0x5272d237, 0x79d2951c, 0xc60d894c, 0x488cb402, 0x1ba4fe5b, 0xa4b09f6b, 0x1ca815cf, + 0xa20c3005, 0x8871df63, 0xb9de2fcb, 0x0cc6c9e9, 0x0beeff53, 0xe3214517, 0xb4542835, 0x9f63293c, + 0xee41e729, 0x6e1d2d7c, 0x50045286, 0x1e6685f3, 0xf33401c6, 0x30a22c95, 0x31a70850, 0x60930f13, + 0x73f98417, 0xa1269859, 0xec645c44, 0x52c877a9, 0xcdff33a6, 0xa02b1741, 0x7cbad9a2, 0x2180036f, + 0x50d99c08, 0xcb3f4861, 0xc26bd765, 0x64a3f6ab, 0x80342676, 0x25a75e7b, 0xe4e6d1fc, 0x20c710e6, + 0xcdf0b680, 0x17844d3b, 0x31eef84d, 0x7e0824e4, 0x2ccb49eb, 0x846a3bae, 0x8ff77888, 0xee5d60f6, + 0x7af75673, 0x2fdd5cdb, 0xa11631c1, 0x30f66f43, 0xb3faec54, 0x157fd7fa, 0xef8579cc, 0xd152de58, + 0xdb2ffd5e, 0x8f32ce19, 0x306af97a, 0x02f03ef8, 0x99319ad5, 0xc242fa0f, 0xa7e3ebb0, 0xc68e4906, + 0xb8da230c, 0x80823028, 0xdcdef3c8, 0xd35fb171, 0x088a1bc8, 0xbec0c560, 0x61a3c9e8, 0xbca8f54d, + 0xc72feffa, 0x22822e99, 0x82c570b4, 0xd8d94e89, 0x8b1c34bc, 0x301e16e6, 0x273be979, 0xb0ffeaa6, + 0x61d9b8c6, 0x00b24869, 0xb7ffce3f, 0x08dc283b, 0x43daf65a, 0xf7e19798, 0x7619b72f, 0x8f1c9ba4, + 0xdc8637a0, 0x16a7d3b1, 0x9fc393b7, 0xa7136eeb, 0xc6bcc63e, 0x1a513742, 0xef6828bc, 0x520365d6, + 0x2d6a77ab, 0x3527ed4b, 0x821fd216, 0x095c6e2e, 0xdb92f2fb, 0x5eea29cb, 0x145892f5, 0x91584f7f, + 0x5483697b, 0x2667a8cc, 0x85196048, 0x8c4bacea, 0x833860d4, 0x0d23e0f9, 0x6c387e8a, 0x0ae6d249, + 0xb284600c, 0xd835731d, 0xdcb1c647, 0xac4c56ea, 0x3ebd81b3, 0x230eabb0, 0x6438bc87, 0xf0b5b1fa, + 0x8f5ea2b3, 0xfc184642, 0x0a036b7a, 0x4fb089bd, 0x649da589, 0xa345415e, 0x5c038323, 0x3e5d3bb9, + 0x43d79572, 0x7e6dd07c, 0x06dfdf1e, 0x6c6cc4ef, 0x7160a539, 0x73bfbe70, 0x83877605, 0x4523ecf1, + }, + { + 0x8defc240, 0x25fa5d9f, 0xeb903dbf, 0xe810c907, 0x47607fff, 0x369fe44b, 0x8c1fc644, 0xaececa90, + 0xbeb1f9bf, 0xeefbcaea, 0xe8cf1950, 0x51df07ae, 0x920e8806, 0xf0ad0548, 0xe13c8d83, 0x927010d5, + 0x11107d9f, 0x07647db9, 0xb2e3e4d4, 0x3d4f285e, 0xb9afa820, 0xfade82e0, 0xa067268b, 0x8272792e, + 0x553fb2c0, 0x489ae22b, 0xd4ef9794, 0x125e3fbc, 0x21fffcee, 0x825b1bfd, 0x9255c5ed, 0x1257a240, + 0x4e1a8302, 0xbae07fff, 0x528246e7, 0x8e57140e, 0x3373f7bf, 0x8c9f8188, 0xa6fc4ee8, 0xc982b5a5, + 0xa8c01db7, 0x579fc264, 0x67094f31, 0xf2bd3f5f, 0x40fff7c1, 0x1fb78dfc, 0x8e6bd2c1, 0x437be59b, + 0x99b03dbf, 0xb5dbc64b, 0x638dc0e6, 0x55819d99, 0xa197c81c, 0x4a012d6e, 0xc5884a28, 0xccc36f71, + 0xb843c213, 0x6c0743f1, 0x8309893c, 0x0feddd5f, 0x2f7fe850, 0xd7c07f7e, 0x02507fbf, 0x5afb9a04, + 0xa747d2d0, 0x1651192e, 0xaf70bf3e, 0x58c31380, 0x5f98302e, 0x727cc3c4, 0x0a0fb402, 0x0f7fef82, + 0x8c96fdad, 0x5d2c2aae, 0x8ee99a49, 0x50da88b8, 0x8427f4a0, 0x1eac5790, 0x796fb449, 0x8252dc15, + 0xefbd7d9b, 0xa672597d, 0xada840d8, 0x45f54504, 0xfa5d7403, 0xe83ec305, 0x4f91751a, 0x925669c2, + 0x23efe941, 0xa903f12e, 0x60270df2, 0x0276e4b6, 0x94fd6574, 0x927985b2, 0x8276dbcb, 0x02778176, + 0xf8af918d, 0x4e48f79e, 0x8f616ddf, 0xe29d840e, 0x842f7d83, 0x340ce5c8, 0x96bbb682, 0x93b4b148, + 0xef303cab, 0x984faf28, 0x779faf9b, 0x92dc560d, 0x224d1e20, 0x8437aa88, 0x7d29dc96, 0x2756d3dc, + 0x8b907cee, 0xb51fd240, 0xe7c07ce3, 0xe566b4a1, 0xc3e9615e, 0x3cf8209d, 0x6094d1e3, 0xcd9ca341, + 0x5c76460e, 0x00ea983b, 0xd4d67881, 0xfd47572c, 0xf76cedd9, 0xbda8229c, 0x127dadaa, 0x438a074e, + 0x1f97c090, 0x081bdb8a, 0x93a07ebe, 0xb938ca15, 0x97b03cff, 0x3dc2c0f8, 0x8d1ab2ec, 0x64380e51, + 0x68cc7bfb, 0xd90f2788, 0x12490181, 0x5de5ffd4, 0xdd7ef86a, 0x76a2e214, 0xb9a40368, 0x925d958f, + 0x4b39fffa, 0xba39aee9, 0xa4ffd30b, 0xfaf7933b, 0x6d498623, 0x193cbcfa, 0x27627545, 0x825cf47a, + 0x61bd8ba0, 0xd11e42d1, 0xcead04f4, 0x127ea392, 0x10428db7, 0x8272a972, 0x9270c4a8, 0x127de50b, + 0x285ba1c8, 0x3c62f44f, 0x35c0eaa5, 0xe805d231, 0x428929fb, 0xb4fcdf82, 0x4fb66a53, 0x0e7dc15b, + 0x1f081fab, 0x108618ae, 0xfcfd086d, 0xf9ff2889, 0x694bcc11, 0x236a5cae, 0x12deca4d, 0x2c3f8cc5, + 0xd2d02dfe, 0xf8ef5896, 0xe4cf52da, 0x95155b67, 0x494a488c, 0xb9b6a80c, 0x5c8f82bc, 0x89d36b45, + 0x3a609437, 0xec00c9a9, 0x44715253, 0x0a874b49, 0xd773bc40, 0x7c34671c, 0x02717ef6, 0x4feb5536, + 0xa2d02fff, 0xd2bf60c4, 0xd43f03c0, 0x50b4ef6d, 0x07478cd1, 0x006e1888, 0xa2e53f55, 0xb9e6d4bc, + 0xa2048016, 0x97573833, 0xd7207d67, 0xde0f8f3d, 0x72f87b33, 0xabcc4f33, 0x7688c55d, 0x7b00a6b0, + 0x947b0001, 0x570075d2, 0xf9bb88f8, 0x8942019e, 0x4264a5ff, 0x856302e0, 0x72dbd92b, 0xee971b69, + 0x6ea22fde, 0x5f08ae2b, 0xaf7a616d, 0xe5c98767, 0xcf1febd2, 0x61efc8c2, 0xf1ac2571, 0xcc8239c2, + 0x67214cb8, 0xb1e583d1, 0xb7dc3e62, 0x7f10bdce, 0xf90a5c38, 0x0ff0443d, 0x606e6dc6, 0x60543a49, + 0x5727c148, 0x2be98a1d, 0x8ab41738, 0x20e1be24, 0xaf96da0f, 0x68458425, 0x99833be5, 0x600d457d, + 0x282f9350, 0x8334b362, 0xd91d1120, 0x2b6d8da0, 0x642b1e31, 0x9c305a00, 0x52bce688, 0x1b03588a, + 0xf7baefd5, 0x4142ed9c, 0xa4315c11, 0x83323ec5, 0xdfef4636, 0xa133c501, 0xe9d3531c, 0xee353783, + }, + { + 0x9db30420, 0x1fb6e9de, 0xa7be7bef, 0xd273a298, 0x4a4f7bdb, 0x64ad8c57, 0x85510443, 0xfa020ed1, + 0x7e287aff, 0xe60fb663, 0x095f35a1, 0x79ebf120, 0xfd059d43, 0x6497b7b1, 0xf3641f63, 0x241e4adf, + 0x28147f5f, 0x4fa2b8cd, 0xc9430040, 0x0cc32220, 0xfdd30b30, 0xc0a5374f, 0x1d2d00d9, 0x24147b15, + 0xee4d111a, 0x0fca5167, 0x71ff904c, 0x2d195ffe, 0x1a05645f, 0x0c13fefe, 0x081b08ca, 0x05170121, + 0x80530100, 0xe83e5efe, 0xac9af4f8, 0x7fe72701, 0xd2b8ee5f, 0x06df4261, 0xbb9e9b8a, 0x7293ea25, + 0xce84ffdf, 0xf5718801, 0x3dd64b04, 0xa26f263b, 0x7ed48400, 0x547eebe6, 0x446d4ca0, 0x6cf3d6f5, + 0x2649abdf, 0xaea0c7f5, 0x36338cc1, 0x503f7e93, 0xd3772061, 0x11b638e1, 0x72500e03, 0xf80eb2bb, + 0xabe0502e, 0xec8d77de, 0x57971e81, 0xe14f6746, 0xc9335400, 0x6920318f, 0x081dbb99, 0xffc304a5, + 0x4d351805, 0x7f3d5ce3, 0xa6c866c6, 0x5d5bcca9, 0xdaec6fea, 0x9f926f91, 0x9f46222f, 0x3991467d, + 0xa5bf6d8e, 0x1143c44f, 0x43958302, 0xd0214eeb, 0x022083b8, 0x3fb6180c, 0x18f8931e, 0x281658e6, + 0x26486e3e, 0x8bd78a70, 0x7477e4c1, 0xb506e07c, 0xf32d0a25, 0x79098b02, 0xe4eabb81, 0x28123b23, + 0x69dead38, 0x1574ca16, 0xdf871b62, 0x211c40b7, 0xa51a9ef9, 0x0014377b, 0x041e8ac8, 0x09114003, + 0xbd59e4d2, 0xe3d156d5, 0x4fe876d5, 0x2f91a340, 0x557be8de, 0x00eae4a7, 0x0ce5c2ec, 0x4db4bba6, + 0xe756bdff, 0xdd3369ac, 0xec17b035, 0x06572327, 0x99afc8b0, 0x56c8c391, 0x6b65811c, 0x5e146119, + 0x6e85cb75, 0xbe07c002, 0xc2325577, 0x893ff4ec, 0x5bbfc92d, 0xd0ec3b25, 0xb7801ab7, 0x8d6d3b24, + 0x20c763ef, 0xc366a5fc, 0x9c382880, 0x0ace3205, 0xaac9548a, 0xeca1d7c7, 0x041afa32, 0x1d16625a, + 0x6701902c, 0x9b757a54, 0x31d477f7, 0x9126b031, 0x36cc6fdb, 0xc70b8b46, 0xd9e66a48, 0x56e55a79, + 0x026a4ceb, 0x52437eff, 0x2f8f76b4, 0x0df980a5, 0x8674cde3, 0xedda04eb, 0x17a9be04, 0x2c18f4df, + 0xb7747f9d, 0xab2af7b4, 0xefc34d20, 0x2e096b7c, 0x1741a254, 0xe5b6a035, 0x213d42f6, 0x2c1c7c26, + 0x61c2f50f, 0x6552daf9, 0xd2c231f8, 0x25130f69, 0xd8167fa2, 0x0418f2c8, 0x001a96a6, 0x0d1526ab, + 0x63315c21, 0x5e0a72ec, 0x49bafefd, 0x187908d9, 0x8d0dbd86, 0x311170a7, 0x3e9b640c, 0xcc3e10d7, + 0xd5cad3b6, 0x0caec388, 0xf73001e1, 0x6c728aff, 0x71eae2a1, 0x1f9af36e, 0xcfcbd12f, 0xc1de8417, + 0xac07be6b, 0xcb44a1d8, 0x8b9b0f56, 0x013988c3, 0xb1c52fca, 0xb4be31cd, 0xd8782806, 0x12a3a4e2, + 0x6f7de532, 0x58fd7eb6, 0xd01ee900, 0x24adffc2, 0xf4990fc5, 0x9711aac5, 0x001d7b95, 0x82e5e7d2, + 0x109873f6, 0x00613096, 0xc32d9521, 0xada121ff, 0x29908415, 0x7fbb977f, 0xaf9eb3db, 0x29c9ed2a, + 0x5ce2a465, 0xa730f32c, 0xd0aa3fe8, 0x8a5cc091, 0xd49e2ce7, 0x0ce454a9, 0xd60acd86, 0x015f1919, + 0x77079103, 0xdea03af6, 0x78a8565e, 0xdee356df, 0x21f05cbe, 0x8b75e387, 0xb3c50651, 0xb8a5c3ef, + 0xd8eeb6d2, 0xe523be77, 0xc2154529, 0x2f69efdf, 0xafe67afb, 0xf470c4b2, 0xf3e0eb5b, 0xd6cc9876, + 0x39e4460c, 0x1fda8538, 0x1987832f, 0xca007367, 0xa99144f8, 0x296b299e, 0x492fc295, 0x9266beab, + 0xb5676e69, 0x9bd3ddda, 0xdf7e052f, 0xdb25701c, 0x1b5e51ee, 0xf65324e6, 0x6afce36c, 0x0316cc04, + 0x8644213e, 0xb7dc59d0, 0x7965291f, 0xccd6fd43, 0x41823979, 0x932bcdf6, 0xb657c34d, 0x4edfd282, + 0x7ae5290c, 0x3cb9536b, 0x851e20fe, 0x9833557e, 0x13ecf0b0, 0xd3ffb372, 0x3f85c5c1, 0x0aef7ed2, + }, + { + 0x7ec90c04, 0x2c6e74b9, 0x9b0e66df, 0xa6337911, 0xb86a7fff, 0x1dd358f5, 0x44dd9d44, 0x1731167f, + 0x08fbf1fa, 0xe7f511cc, 0xd2051b00, 0x735aba00, 0x2ab722d8, 0x386381cb, 0xacf6243a, 0x69befd7a, + 0xe6a2e77f, 0xf0c720cd, 0xc4494816, 0xccf5c180, 0x38851640, 0x15b0a848, 0xe68b18cb, 0x4caadeff, + 0x5f480a01, 0x0412b2aa, 0x259814fc, 0x41d0efe2, 0x4e40b48d, 0x248eb6fb, 0x8dba1cfe, 0x41a99b02, + 0x1a550a04, 0xba8f65cb, 0x7251f4e7, 0x95a51725, 0xc106ecd7, 0x97a5980a, 0xc539b9aa, 0x4d79fe6a, + 0xf2f3f763, 0x68af8040, 0xed0c9e56, 0x11b4958b, 0xe1eb5a88, 0x8709e6b0, 0xd7e07156, 0x4e29fea7, + 0x6366e52d, 0x02d1c000, 0xc4ac8e05, 0x9377f571, 0x0c05372a, 0x578535f2, 0x2261be02, 0xd642a0c9, + 0xdf13a280, 0x74b55bd2, 0x682199c0, 0xd421e5ec, 0x53fb3ce8, 0xc8adedb3, 0x28a87fc9, 0x3d959981, + 0x5c1ff900, 0xfe38d399, 0x0c4eff0b, 0x062407ea, 0xaa2f4fb1, 0x4fb96976, 0x90c79505, 0xb0a8a774, + 0xef55a1ff, 0xe59ca2c2, 0xa6b62d27, 0xe66a4263, 0xdf65001f, 0x0ec50966, 0xdfdd55bc, 0x29de0655, + 0x911e739a, 0x17af8975, 0x32c7911c, 0x89f89468, 0x0d01e980, 0x524755f4, 0x03b63cc9, 0x0cc844b2, + 0xbcf3f0aa, 0x87ac36e9, 0xe53a7426, 0x01b3d82b, 0x1a9e7449, 0x64ee2d7e, 0xcddbb1da, 0x01c94910, + 0xb868bf80, 0x0d26f3fd, 0x9342ede7, 0x04a5c284, 0x636737b6, 0x50f5b616, 0xf24766e3, 0x8eca36c1, + 0x136e05db, 0xfef18391, 0xfb887a37, 0xd6e7f7d4, 0xc7fb7dc9, 0x3063fcdf, 0xb6f589de, 0xec2941da, + 0x26e46695, 0xb7566419, 0xf654efc5, 0xd08d58b7, 0x48925401, 0xc1bacb7f, 0xe5ff550f, 0xb6083049, + 0x5bb5d0e8, 0x87d72e5a, 0xab6a6ee1, 0x223a66ce, 0xc62bf3cd, 0x9e0885f9, 0x68cb3e47, 0x086c010f, + 0xa21de820, 0xd18b69de, 0xf3f65777, 0xfa02c3f6, 0x407edac3, 0xcbb3d550, 0x1793084d, 0xb0d70eba, + 0x0ab378d5, 0xd951fb0c, 0xded7da56, 0x4124bbe4, 0x94ca0b56, 0x0f5755d1, 0xe0e1e56e, 0x6184b5be, + 0x580a249f, 0x94f74bc0, 0xe327888e, 0x9f7b5561, 0xc3dc0280, 0x05687715, 0x646c6bd7, 0x44904db3, + 0x66b4f0a3, 0xc0f1648a, 0x697ed5af, 0x49e92ff6, 0x309e374f, 0x2cb6356a, 0x85808573, 0x4991f840, + 0x76f0ae02, 0x083be84d, 0x28421c9a, 0x44489406, 0x736e4cb8, 0xc1092910, 0x8bc95fc6, 0x7d869cf4, + 0x134f616f, 0x2e77118d, 0xb31b2be1, 0xaa90b472, 0x3ca5d717, 0x7d161bba, 0x9cad9010, 0xaf462ba2, + 0x9fe459d2, 0x45d34559, 0xd9f2da13, 0xdbc65487, 0xf3e4f94e, 0x176d486f, 0x097c13ea, 0x631da5c7, + 0x445f7382, 0x175683f4, 0xcdc66a97, 0x70be0288, 0xb3cdcf72, 0x6e5dd2f3, 0x20936079, 0x459b80a5, + 0xbe60e2db, 0xa9c23101, 0xeba5315c, 0x224e42f2, 0x1c5c1572, 0xf6721b2c, 0x1ad2fff3, 0x8c25404e, + 0x324ed72f, 0x4067b7fd, 0x0523138e, 0x5ca3bc78, 0xdc0fd66e, 0x75922283, 0x784d6b17, 0x58ebb16e, + 0x44094f85, 0x3f481d87, 0xfcfeae7b, 0x77b5ff76, 0x8c2302bf, 0xaaf47556, 0x5f46b02a, 0x2b092801, + 0x3d38f5f7, 0x0ca81f36, 0x52af4a8a, 0x66d5e7c0, 0xdf3b0874, 0x95055110, 0x1b5ad7a8, 0xf61ed5ad, + 0x6cf6e479, 0x20758184, 0xd0cefa65, 0x88f7be58, 0x4a046826, 0x0ff6f8f3, 0xa09c7f70, 0x5346aba0, + 0x5ce96c28, 0xe176eda3, 0x6bac307f, 0x376829d2, 0x85360fa9, 0x17e3fe2a, 0x24b79767, 0xf5a96b20, + 0xd6cd2595, 0x68ff1ebf, 0x7555442c, 0xf19f06be, 0xf9e0659a, 0xeeb9491d, 0x34010718, 0xbb30cab8, + 0xe822fe15, 0x88570983, 0x750e6249, 0xda627e55, 0x5e76ffa8, 0xb1534546, 0x6d47de08, 0xefe9e7d4, + }, + { + 0xf6fa8f9d, 0x2cac6ce1, 0x4ca34867, 0xe2337f7c, 0x95db08e7, 0x016843b4, 0xeced5cbc, 0x325553ac, + 0xbf9f0960, 0xdfa1e2ed, 0x83f0579d, 0x63ed86b9, 0x1ab6a6b8, 0xde5ebe39, 0xf38ff732, 0x8989b138, + 0x33f14961, 0xc01937bd, 0xf506c6da, 0xe4625e7e, 0xa308ea99, 0x4e23e33c, 0x79cbd7cc, 0x48a14367, + 0xa3149619, 0xfec94bd5, 0xa114174a, 0xeaa01866, 0xa084db2d, 0x09a8486f, 0xa888614a, 0x2900af98, + 0x01665991, 0xe1992863, 0xc8f30c60, 0x2e78ef3c, 0xd0d51932, 0xcf0fec14, 0xf7ca07d2, 0xd0a82072, + 0xfd41197e, 0x9305a6b0, 0xe86be3da, 0x74bed3cd, 0x372da53c, 0x4c7f4448, 0xdab5d440, 0x6dba0ec3, + 0x083919a7, 0x9fbaeed9, 0x49dbcfb0, 0x4e670c53, 0x5c3d9c01, 0x64bdb941, 0x2c0e636a, 0xba7dd9cd, + 0xea6f7388, 0xe70bc762, 0x35f29adb, 0x5c4cdd8d, 0xf0d48d8c, 0xb88153e2, 0x08a19866, 0x1ae2eac8, + 0x284caf89, 0xaa928223, 0x9334be53, 0x3b3a21bf, 0x16434be3, 0x9aea3906, 0xefe8c36e, 0xf890cdd9, + 0x80226dae, 0xc340a4a3, 0xdf7e9c09, 0xa694a807, 0x5b7c5ecc, 0x221db3a6, 0x9a69a02f, 0x68818a54, + 0xceb2296f, 0x53c0843a, 0xfe893655, 0x25bfe68a, 0xb4628abc, 0xcf222ebf, 0x25ac6f48, 0xa9a99387, + 0x53bddb65, 0xe76ffbe7, 0xe967fd78, 0x0ba93563, 0x8e342bc1, 0xe8a11be9, 0x4980740d, 0xc8087dfc, + 0x8de4bf99, 0xa11101a0, 0x7fd37975, 0xda5a26c0, 0xe81f994f, 0x9528cd89, 0xfd339fed, 0xb87834bf, + 0x5f04456d, 0x22258698, 0xc9c4c83b, 0x2dc156be, 0x4f628daa, 0x57f55ec5, 0xe2220abe, 0xd2916ebf, + 0x4ec75b95, 0x24f2c3c0, 0x42d15d99, 0xcd0d7fa0, 0x7b6e27ff, 0xa8dc8af0, 0x7345c106, 0xf41e232f, + 0x35162386, 0xe6ea8926, 0x3333b094, 0x157ec6f2, 0x372b74af, 0x692573e4, 0xe9a9d848, 0xf3160289, + 0x3a62ef1d, 0xa787e238, 0xf3a5f676, 0x74364853, 0x20951063, 0x4576698d, 0xb6fad407, 0x592af950, + 0x36f73523, 0x4cfb6e87, 0x7da4cec0, 0x6c152daa, 0xcb0396a8, 0xc50dfe5d, 0xfcd707ab, 0x0921c42f, + 0x89dff0bb, 0x5fe2be78, 0x448f4f33, 0x754613c9, 0x2b05d08d, 0x48b9d585, 0xdc049441, 0xc8098f9b, + 0x7dede786, 0xc39a3373, 0x42410005, 0x6a091751, 0x0ef3c8a6, 0x890072d6, 0x28207682, 0xa9a9f7be, + 0xbf32679d, 0xd45b5b75, 0xb353fd00, 0xcbb0e358, 0x830f220a, 0x1f8fb214, 0xd372cf08, 0xcc3c4a13, + 0x8cf63166, 0x061c87be, 0x88c98f88, 0x6062e397, 0x47cf8e7a, 0xb6c85283, 0x3cc2acfb, 0x3fc06976, + 0x4e8f0252, 0x64d8314d, 0xda3870e3, 0x1e665459, 0xc10908f0, 0x513021a5, 0x6c5b68b7, 0x822f8aa0, + 0x3007cd3e, 0x74719eef, 0xdc872681, 0x073340d4, 0x7e432fd9, 0x0c5ec241, 0x8809286c, 0xf592d891, + 0x08a930f6, 0x957ef305, 0xb7fbffbd, 0xc266e96f, 0x6fe4ac98, 0xb173ecc0, 0xbc60b42a, 0x953498da, + 0xfba1ae12, 0x2d4bd736, 0x0f25faab, 0xa4f3fceb, 0xe2969123, 0x257f0c3d, 0x9348af49, 0x361400bc, + 0xe8816f4a, 0x3814f200, 0xa3f94043, 0x9c7a54c2, 0xbc704f57, 0xda41e7f9, 0xc25ad33a, 0x54f4a084, + 0xb17f5505, 0x59357cbe, 0xedbd15c8, 0x7f97c5ab, 0xba5ac7b5, 0xb6f6deaf, 0x3a479c3a, 0x5302da25, + 0x653d7e6a, 0x54268d49, 0x51a477ea, 0x5017d55b, 0xd7d25d88, 0x44136c76, 0x0404a8c8, 0xb8e5a121, + 0xb81a928a, 0x60ed5869, 0x97c55b96, 0xeaec991b, 0x29935913, 0x01fdb7f1, 0x088e8dfa, 0x9ab6f6f5, + 0x3b4cbf9f, 0x4a5de3ab, 0xe6051d35, 0xa0e1d855, 0xd36b4cf1, 0xf544edeb, 0xb0e93524, 0xbebb8fbd, + 0xa2d762cf, 0x49c92f54, 0x38b5f331, 0x7128a454, 0x48392905, 0xa65b1db8, 0x851c97bd, 0xd675cf2f, + }, + { + 0x85e04019, 0x332bf567, 0x662dbfff, 0xcfc65693, 0x2a8d7f6f, 0xab9bc912, 0xde6008a1, 0x2028da1f, + 0x0227bce7, 0x4d642916, 0x18fac300, 0x50f18b82, 0x2cb2cb11, 0xb232e75c, 0x4b3695f2, 0xb28707de, + 0xa05fbcf6, 0xcd4181e9, 0xe150210c, 0xe24ef1bd, 0xb168c381, 0xfde4e789, 0x5c79b0d8, 0x1e8bfd43, + 0x4d495001, 0x38be4341, 0x913cee1d, 0x92a79c3f, 0x089766be, 0xbaeeadf4, 0x1286becf, 0xb6eacb19, + 0x2660c200, 0x7565bde4, 0x64241f7a, 0x8248dca9, 0xc3b3ad66, 0x28136086, 0x0bd8dfa8, 0x356d1cf2, + 0x107789be, 0xb3b2e9ce, 0x0502aa8f, 0x0bc0351e, 0x166bf52a, 0xeb12ff82, 0xe3486911, 0xd34d7516, + 0x4e7b3aff, 0x5f43671b, 0x9cf6e037, 0x4981ac83, 0x334266ce, 0x8c9341b7, 0xd0d854c0, 0xcb3a6c88, + 0x47bc2829, 0x4725ba37, 0xa66ad22b, 0x7ad61f1e, 0x0c5cbafa, 0x4437f107, 0xb6e79962, 0x42d2d816, + 0x0a961288, 0xe1a5c06e, 0x13749e67, 0x72fc081a, 0xb1d139f7, 0xf9583745, 0xcf19df58, 0xbec3f756, + 0xc06eba30, 0x07211b24, 0x45c28829, 0xc95e317f, 0xbc8ec511, 0x38bc46e9, 0xc6e6fa14, 0xbae8584a, + 0xad4ebc46, 0x468f508b, 0x7829435f, 0xf124183b, 0x821dba9f, 0xaff60ff4, 0xea2c4e6d, 0x16e39264, + 0x92544a8b, 0x009b4fc3, 0xaba68ced, 0x9ac96f78, 0x06a5b79a, 0xb2856e6e, 0x1aec3ca9, 0xbe838688, + 0x0e0804e9, 0x55f1be56, 0xe7e5363b, 0xb3a1f25d, 0xf7debb85, 0x61fe033c, 0x16746233, 0x3c034c28, + 0xda6d0c74, 0x79aac56c, 0x3ce4e1ad, 0x51f0c802, 0x98f8f35a, 0x1626a49f, 0xeed82b29, 0x1d382fe3, + 0x0c4fb99a, 0xbb325778, 0x3ec6d97b, 0x6e77a6a9, 0xcb658b5c, 0xd45230c7, 0x2bd1408b, 0x60c03eb7, + 0xb9068d78, 0xa33754f4, 0xf430c87d, 0xc8a71302, 0xb96d8c32, 0xebd4e7be, 0xbe8b9d2d, 0x7979fb06, + 0xe7225308, 0x8b75cf77, 0x11ef8da4, 0xe083c858, 0x8d6b786f, 0x5a6317a6, 0xfa5cf7a0, 0x5dda0033, + 0xf28ebfb0, 0xf5b9c310, 0xa0eac280, 0x08b9767a, 0xa3d9d2b0, 0x79d34217, 0x021a718d, 0x9ac6336a, + 0x2711fd60, 0x438050e3, 0x069908a8, 0x3d7fedc4, 0x826d2bef, 0x4eeb8476, 0x488dcf25, 0x36c9d566, + 0x28e74e41, 0xc2610aca, 0x3d49a9cf, 0xbae3b9df, 0xb65f8de6, 0x92aeaf64, 0x3ac7d5e6, 0x9ea80509, + 0xf22b017d, 0xa4173f70, 0xdd1e16c3, 0x15e0d7f9, 0x50b1b887, 0x2b9f4fd5, 0x625aba82, 0x6a017962, + 0x2ec01b9c, 0x15488aa9, 0xd716e740, 0x40055a2c, 0x93d29a22, 0xe32dbf9a, 0x058745b9, 0x3453dc1e, + 0xd699296e, 0x496cff6f, 0x1c9f4986, 0xdfe2ed07, 0xb87242d1, 0x19de7eae, 0x053e561a, 0x15ad6f8c, + 0x66626c1c, 0x7154c24c, 0xea082b2a, 0x93eb2939, 0x17dcb0f0, 0x58d4f2ae, 0x9ea294fb, 0x52cf564c, + 0x9883fe66, 0x2ec40581, 0x763953c3, 0x01d6692e, 0xd3a0c108, 0xa1e7160e, 0xe4f2dfa6, 0x693ed285, + 0x74904698, 0x4c2b0edd, 0x4f757656, 0x5d393378, 0xa132234f, 0x3d321c5d, 0xc3f5e194, 0x4b269301, + 0xc79f022f, 0x3c997e7e, 0x5e4f9504, 0x3ffafbbd, 0x76f7ad0e, 0x296693f4, 0x3d1fce6f, 0xc61e45be, + 0xd3b5ab34, 0xf72bf9b7, 0x1b0434c0, 0x4e72b567, 0x5592a33d, 0xb5229301, 0xcfd2a87f, 0x60aeb767, + 0x1814386b, 0x30bcc33d, 0x38a0c07d, 0xfd1606f2, 0xc363519b, 0x589dd390, 0x5479f8e6, 0x1cb8d647, + 0x97fd61a9, 0xea7759f4, 0x2d57539d, 0x569a58cf, 0xe84e63ad, 0x462e1b78, 0x6580f87e, 0xf3817914, + 0x91da55f4, 0x40a230f3, 0xd1988f35, 0xb6e318d2, 0x3ffa50bc, 0x3d40f021, 0xc3c0bdae, 0x4958c24c, + 0x518f36b2, 0x84b1d370, 0x0fedce83, 0x878ddada, 0xf2a279c7, 0x94e01be8, 0x90716f4b, 0x954b8aa3, + }, + { + 0xe216300d, 0xbbddfffc, 0xa7ebdabd, 0x35648095, 0x7789f8b7, 0xe6c1121b, 0x0e241600, 0x052ce8b5, + 0x11a9cfb0, 0xe5952f11, 0xece7990a, 0x9386d174, 0x2a42931c, 0x76e38111, 0xb12def3a, 0x37ddddfc, + 0xde9adeb1, 0x0a0cc32c, 0xbe197029, 0x84a00940, 0xbb243a0f, 0xb4d137cf, 0xb44e79f0, 0x049eedfd, + 0x0b15a15d, 0x480d3168, 0x8bbbde5a, 0x669ded42, 0xc7ece831, 0x3f8f95e7, 0x72df191b, 0x7580330d, + 0x94074251, 0x5c7dcdfa, 0xabbe6d63, 0xaa402164, 0xb301d40a, 0x02e7d1ca, 0x53571dae, 0x7a3182a2, + 0x12a8ddec, 0xfdaa335d, 0x176f43e8, 0x71fb46d4, 0x38129022, 0xce949ad4, 0xb84769ad, 0x965bd862, + 0x82f3d055, 0x66fb9767, 0x15b80b4e, 0x1d5b47a0, 0x4cfde06f, 0xc28ec4b8, 0x57e8726e, 0x647a78fc, + 0x99865d44, 0x608bd593, 0x6c200e03, 0x39dc5ff6, 0x5d0b00a3, 0xae63aff2, 0x7e8bd632, 0x70108c0c, + 0xbbd35049, 0x2998df04, 0x980cf42a, 0x9b6df491, 0x9e7edd53, 0x06918548, 0x58cb7e07, 0x3b74ef2e, + 0x522fffb1, 0xd24708cc, 0x1c7e27cd, 0xa4eb215b, 0x3cf1d2e2, 0x19b47a38, 0x424f7618, 0x35856039, + 0x9d17dee7, 0x27eb35e6, 0xc9aff67b, 0x36baf5b8, 0x09c467cd, 0xc18910b1, 0xe11dbf7b, 0x06cd1af8, + 0x7170c608, 0x2d5e3354, 0xd4de495a, 0x64c6d006, 0xbcc0c62c, 0x3dd00db3, 0x708f8f34, 0x77d51b42, + 0x264f620f, 0x24b8d2bf, 0x15c1b79e, 0x46a52564, 0xf8d7e54e, 0x3e378160, 0x7895cda5, 0x859c15a5, + 0xe6459788, 0xc37bc75f, 0xdb07ba0c, 0x0676a3ab, 0x7f229b1e, 0x31842e7b, 0x24259fd7, 0xf8bef472, + 0x835ffcb8, 0x6df4c1f2, 0x96f5b195, 0xfd0af0fc, 0xb0fe134c, 0xe2506d3d, 0x4f9b12ea, 0xf215f225, + 0xa223736f, 0x9fb4c428, 0x25d04979, 0x34c713f8, 0xc4618187, 0xea7a6e98, 0x7cd16efc, 0x1436876c, + 0xf1544107, 0xbedeee14, 0x56e9af27, 0xa04aa441, 0x3cf7c899, 0x92ecbae6, 0xdd67016d, 0x151682eb, + 0xa842eedf, 0xfdba60b4, 0xf1907b75, 0x20e3030f, 0x24d8c29e, 0xe139673b, 0xefa63fb8, 0x71873054, + 0xb6f2cf3b, 0x9f326442, 0xcb15a4cc, 0xb01a4504, 0xf1e47d8d, 0x844a1be5, 0xbae7dfdc, 0x42cbda70, + 0xcd7dae0a, 0x57e85b7a, 0xd53f5af6, 0x20cf4d8c, 0xcea4d428, 0x79d130a4, 0x3486ebfb, 0x33d3cddc, + 0x77853b53, 0x37effcb5, 0xc5068778, 0xe580b3e6, 0x4e68b8f4, 0xc5c8b37e, 0x0d809ea2, 0x398feb7c, + 0x132a4f94, 0x43b7950e, 0x2fee7d1c, 0x223613bd, 0xdd06caa2, 0x37df932b, 0xc4248289, 0xacf3ebc3, + 0x5715f6b7, 0xef3478dd, 0xf267616f, 0xc148cbe4, 0x9052815e, 0x5e410fab, 0xb48a2465, 0x2eda7fa4, + 0xe87b40e4, 0xe98ea084, 0x5889e9e1, 0xefd390fc, 0xdd07d35b, 0xdb485694, 0x38d7e5b2, 0x57720101, + 0x730edebc, 0x5b643113, 0x94917e4f, 0x503c2fba, 0x646f1282, 0x7523d24a, 0xe0779695, 0xf9c17a8f, + 0x7a5b2121, 0xd187b896, 0x29263a4d, 0xba510cdf, 0x81f47c9f, 0xad1163ed, 0xea7b5965, 0x1a00726e, + 0x11403092, 0x00da6d77, 0x4a0cdd61, 0xad1f4603, 0x605bdfb0, 0x9eedc364, 0x22ebe6a8, 0xcee7d28a, + 0xa0e736a0, 0x5564a6b9, 0x10853209, 0xc7eb8f37, 0x2de705ca, 0x8951570f, 0xdf09822b, 0xbd691a6c, + 0xaa12e4f2, 0x87451c0f, 0xe0f6a27a, 0x3ada4819, 0x4cf1764f, 0x0d771c2b, 0x67cdb156, 0x350d8384, + 0x5938fa0f, 0x42399ef3, 0x36997b07, 0x0e84093d, 0x4aa93e61, 0x8360d87b, 0x1fa98b0c, 0x1149382c, + 0xe97625a5, 0x0614d1b7, 0x0e25244b, 0x0c768347, 0x589e8d82, 0x0d2059d1, 0xa466bb1e, 0xf8da0a82, + 0x04f19130, 0xba6e4ec0, 0x99265164, 0x1ee7230d, 0x50b2ad80, 0xeaee6801, 0x8db2a283, 0xea8bf59e, + }, +} diff --git a/src/pkg/crypto/cast5/cast5_test.go b/src/pkg/crypto/cast5/cast5_test.go new file mode 100644 index 000000000..5f7025ff2 --- /dev/null +++ b/src/pkg/crypto/cast5/cast5_test.go @@ -0,0 +1,104 @@ +// Copyright 2010 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package cast5 + +import ( + "bytes" + "encoding/hex" + "testing" +) + +// This test vector is taken from RFC 2144, App B.1. +// Since the other two test vectors are for reduced-round variants, we can't +// use them. +var basicTests = []struct { + key, plainText, cipherText string +}{ + { + "0123456712345678234567893456789a", + "0123456789abcdef", + "238b4fe5847e44b2", + }, +} + +func TestBasic(t *testing.T) { + for i, test := range basicTests { + key, _ := hex.DecodeString(test.key) + plainText, _ := hex.DecodeString(test.plainText) + expected, _ := hex.DecodeString(test.cipherText) + + c, err := NewCipher(key) + if err != nil { + t.Errorf("#%d: failed to create Cipher: %s", i, err) + continue + } + var cipherText [BlockSize]byte + c.Encrypt(cipherText[:], plainText) + if !bytes.Equal(cipherText[:], expected) { + t.Errorf("#%d: got:%x want:%x", i, cipherText, expected) + } + + var plainTextAgain [BlockSize]byte + c.Decrypt(plainTextAgain[:], cipherText[:]) + if !bytes.Equal(plainTextAgain[:], plainText) { + t.Errorf("#%d: got:%x want:%x", i, plainTextAgain, plainText) + } + } +} + +// TestFull performs the test specified in RFC 2144, App B.2. +// However, due to the length of time taken, it's disabled here and a more +// limited version is included, below. +func TestFull(t *testing.T) { + // This is too slow for normal testing + return + + a, b := iterate(1000000) + + const expectedA = "eea9d0a249fd3ba6b3436fb89d6dca92" + const expectedB = "b2c95eb00c31ad7180ac05b8e83d696e" + + if hex.EncodeToString(a) != expectedA { + t.Errorf("a: got:%x want:%s", a, expectedA) + } + if hex.EncodeToString(b) != expectedB { + t.Errorf("b: got:%x want:%s", b, expectedB) + } +} + +func iterate(iterations int) ([]byte, []byte) { + const initValueHex = "0123456712345678234567893456789a" + + initValue, _ := hex.DecodeString(initValueHex) + + var a, b [16]byte + copy(a[:], initValue) + copy(b[:], initValue) + + for i := 0; i < iterations; i++ { + c, _ := NewCipher(b[:]) + c.Encrypt(a[:8], a[:8]) + c.Encrypt(a[8:], a[8:]) + c, _ = NewCipher(a[:]) + c.Encrypt(b[:8], b[:8]) + c.Encrypt(b[8:], b[8:]) + } + + return a[:], b[:] +} + +func TestLimited(t *testing.T) { + a, b := iterate(1000) + + const expectedA = "23f73b14b02a2ad7dfb9f2c35644798d" + const expectedB = "e5bf37eff14c456a40b21ce369370a9f" + + if hex.EncodeToString(a) != expectedA { + t.Errorf("a: got:%x want:%s", a, expectedA) + } + if hex.EncodeToString(b) != expectedB { + t.Errorf("b: got:%x want:%s", b, expectedB) + } +} diff --git a/src/pkg/crypto/cipher/Makefile b/src/pkg/crypto/cipher/Makefile new file mode 100644 index 000000000..8f61cf20b --- /dev/null +++ b/src/pkg/crypto/cipher/Makefile @@ -0,0 +1,17 @@ +# Copyright 2010 The Go Authors. All rights reserved. +# Use of this source code is governed by a BSD-style +# license that can be found in the LICENSE file. + +include ../../../Make.inc + +TARG=crypto/cipher +GOFILES=\ + cbc.go\ + cfb.go\ + cipher.go\ + ctr.go\ + io.go\ + ocfb.go\ + ofb.go + +include ../../../Make.pkg diff --git a/src/pkg/crypto/cipher/cbc.go b/src/pkg/crypto/cipher/cbc.go new file mode 100644 index 000000000..4632f882a --- /dev/null +++ b/src/pkg/crypto/cipher/cbc.go @@ -0,0 +1,78 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Cipher block chaining (CBC) mode. + +// CBC provides confidentiality by xoring (chaining) each plaintext block +// with the previous ciphertext block before applying the block cipher. + +// See NIST SP 800-38A, pp 10-11 + +package cipher + +type cbc struct { + b Block + blockSize int + iv []byte + tmp []byte +} + +func newCBC(b Block, iv []byte) *cbc { + return &cbc{ + b: b, + blockSize: b.BlockSize(), + iv: dup(iv), + tmp: make([]byte, b.BlockSize()), + } +} + +type cbcEncrypter cbc + +// NewCBCEncrypter returns a BlockMode which encrypts in cipher block chaining +// mode, using the given Block. The length of iv must be the same as the +// Block's block size. +func NewCBCEncrypter(b Block, iv []byte) BlockMode { + return (*cbcEncrypter)(newCBC(b, iv)) +} + +func (x *cbcEncrypter) BlockSize() int { return x.blockSize } + +func (x *cbcEncrypter) CryptBlocks(dst, src []byte) { + for len(src) > 0 { + for i := 0; i < x.blockSize; i++ { + x.iv[i] ^= src[i] + } + x.b.Encrypt(x.iv, x.iv) + for i := 0; i < x.blockSize; i++ { + dst[i] = x.iv[i] + } + src = src[x.blockSize:] + dst = dst[x.blockSize:] + } +} + +type cbcDecrypter cbc + +// NewCBCDecrypter returns a BlockMode which decrypts in cipher block chaining +// mode, using the given Block. The length of iv must be the same as the +// Block's block size as must match the iv used to encrypt the data. +func NewCBCDecrypter(b Block, iv []byte) BlockMode { + return (*cbcDecrypter)(newCBC(b, iv)) +} + +func (x *cbcDecrypter) BlockSize() int { return x.blockSize } + +func (x *cbcDecrypter) CryptBlocks(dst, src []byte) { + for len(src) > 0 { + x.b.Decrypt(x.tmp, src[:x.blockSize]) + for i := 0; i < x.blockSize; i++ { + x.tmp[i] ^= x.iv[i] + x.iv[i] = src[i] + dst[i] = x.tmp[i] + } + + src = src[x.blockSize:] + dst = dst[x.blockSize:] + } +} diff --git a/src/pkg/crypto/cipher/cbc_aes_test.go b/src/pkg/crypto/cipher/cbc_aes_test.go new file mode 100644 index 000000000..944ca1ba8 --- /dev/null +++ b/src/pkg/crypto/cipher/cbc_aes_test.go @@ -0,0 +1,89 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// CBC AES test vectors. + +// See U.S. National Institute of Standards and Technology (NIST) +// Special Publication 800-38A, ``Recommendation for Block Cipher +// Modes of Operation,'' 2001 Edition, pp. 24-29. + +package cipher + +import ( + "bytes" + "crypto/aes" + "testing" +) + +var cbcAESTests = []struct { + name string + key []byte + iv []byte + in []byte + out []byte +}{ + // NIST SP 800-38A pp 27-29 + { + "CBC-AES128", + commonKey128, + commonIV, + commonInput, + []byte{ + 0x76, 0x49, 0xab, 0xac, 0x81, 0x19, 0xb2, 0x46, 0xce, 0xe9, 0x8e, 0x9b, 0x12, 0xe9, 0x19, 0x7d, + 0x50, 0x86, 0xcb, 0x9b, 0x50, 0x72, 0x19, 0xee, 0x95, 0xdb, 0x11, 0x3a, 0x91, 0x76, 0x78, 0xb2, + 0x73, 0xbe, 0xd6, 0xb8, 0xe3, 0xc1, 0x74, 0x3b, 0x71, 0x16, 0xe6, 0x9e, 0x22, 0x22, 0x95, 0x16, + 0x3f, 0xf1, 0xca, 0xa1, 0x68, 0x1f, 0xac, 0x09, 0x12, 0x0e, 0xca, 0x30, 0x75, 0x86, 0xe1, 0xa7, + }, + }, + { + "CBC-AES192", + commonKey192, + commonIV, + commonInput, + []byte{ + 0x4f, 0x02, 0x1d, 0xb2, 0x43, 0xbc, 0x63, 0x3d, 0x71, 0x78, 0x18, 0x3a, 0x9f, 0xa0, 0x71, 0xe8, + 0xb4, 0xd9, 0xad, 0xa9, 0xad, 0x7d, 0xed, 0xf4, 0xe5, 0xe7, 0x38, 0x76, 0x3f, 0x69, 0x14, 0x5a, + 0x57, 0x1b, 0x24, 0x20, 0x12, 0xfb, 0x7a, 0xe0, 0x7f, 0xa9, 0xba, 0xac, 0x3d, 0xf1, 0x02, 0xe0, + 0x08, 0xb0, 0xe2, 0x79, 0x88, 0x59, 0x88, 0x81, 0xd9, 0x20, 0xa9, 0xe6, 0x4f, 0x56, 0x15, 0xcd, + }, + }, + { + "CBC-AES256", + commonKey256, + commonIV, + commonInput, + []byte{ + 0xf5, 0x8c, 0x4c, 0x04, 0xd6, 0xe5, 0xf1, 0xba, 0x77, 0x9e, 0xab, 0xfb, 0x5f, 0x7b, 0xfb, 0xd6, + 0x9c, 0xfc, 0x4e, 0x96, 0x7e, 0xdb, 0x80, 0x8d, 0x67, 0x9f, 0x77, 0x7b, 0xc6, 0x70, 0x2c, 0x7d, + 0x39, 0xf2, 0x33, 0x69, 0xa9, 0xd9, 0xba, 0xcf, 0xa5, 0x30, 0xe2, 0x63, 0x04, 0x23, 0x14, 0x61, + 0xb2, 0xeb, 0x05, 0xe2, 0xc3, 0x9b, 0xe9, 0xfc, 0xda, 0x6c, 0x19, 0x07, 0x8c, 0x6a, 0x9d, 0x1b, + }, + }, +} + +func TestCBC_AES(t *testing.T) { + for _, tt := range cbcAESTests { + test := tt.name + + c, err := aes.NewCipher(tt.key) + if err != nil { + t.Errorf("%s: NewCipher(%d bytes) = %s", test, len(tt.key), err) + continue + } + + encrypter := NewCBCEncrypter(c, tt.iv) + d := make([]byte, len(tt.in)) + encrypter.CryptBlocks(d, tt.in) + if !bytes.Equal(tt.out, d) { + t.Errorf("%s: CBCEncrypter\nhave %x\nwant %x", test, d, tt.out) + } + + decrypter := NewCBCDecrypter(c, tt.iv) + p := make([]byte, len(d)) + decrypter.CryptBlocks(p, d) + if !bytes.Equal(tt.in, p) { + t.Errorf("%s: CBCDecrypter\nhave %x\nwant %x", test, d, tt.in) + } + } +} diff --git a/src/pkg/crypto/cipher/cfb.go b/src/pkg/crypto/cipher/cfb.go new file mode 100644 index 000000000..d14165a86 --- /dev/null +++ b/src/pkg/crypto/cipher/cfb.go @@ -0,0 +1,64 @@ +// Copyright 2010 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// CFB (Cipher Feedback) Mode. + +package cipher + +type cfb struct { + b Block + out []byte + outUsed int + decrypt bool +} + +// NewCFBEncrypter returns a Stream which encrypts with cipher feedback mode, +// using the given Block. The iv must be the same length as the Block's block +// size. +func NewCFBEncrypter(block Block, iv []byte) Stream { + return newCFB(block, iv, false) +} + +// NewCFBDecrypter returns a Stream which decrypts with cipher feedback mode, +// using the given Block. The iv must be the same length as the Block's block +// size. +func NewCFBDecrypter(block Block, iv []byte) Stream { + return newCFB(block, iv, true) +} + +func newCFB(block Block, iv []byte, decrypt bool) Stream { + blockSize := block.BlockSize() + if len(iv) != blockSize { + return nil + } + + x := &cfb{ + b: block, + out: make([]byte, blockSize), + outUsed: 0, + decrypt: decrypt, + } + block.Encrypt(x.out, iv) + + return x +} + +func (x *cfb) XORKeyStream(dst, src []byte) { + for i := 0; i < len(src); i++ { + if x.outUsed == len(x.out) { + x.b.Encrypt(x.out, x.out) + x.outUsed = 0 + } + + if x.decrypt { + t := src[i] + dst[i] = src[i] ^ x.out[x.outUsed] + x.out[x.outUsed] = t + } else { + x.out[x.outUsed] ^= src[i] + dst[i] = x.out[x.outUsed] + } + x.outUsed++ + } +} diff --git a/src/pkg/crypto/cipher/cfb_test.go b/src/pkg/crypto/cipher/cfb_test.go new file mode 100644 index 000000000..9547bfceb --- /dev/null +++ b/src/pkg/crypto/cipher/cfb_test.go @@ -0,0 +1,35 @@ +// Copyright 2010 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package cipher + +import ( + "bytes" + "crypto/aes" + "crypto/rand" + "testing" +) + +func TestCFB(t *testing.T) { + block, err := aes.NewCipher(commonKey128) + if err != nil { + t.Error(err) + return + } + + plaintext := []byte("this is the plaintext") + iv := make([]byte, block.BlockSize()) + rand.Reader.Read(iv) + cfb := NewCFBEncrypter(block, iv) + ciphertext := make([]byte, len(plaintext)) + cfb.XORKeyStream(ciphertext, plaintext) + + cfbdec := NewCFBDecrypter(block, iv) + plaintextCopy := make([]byte, len(plaintext)) + cfbdec.XORKeyStream(plaintextCopy, ciphertext) + + if !bytes.Equal(plaintextCopy, plaintext) { + t.Errorf("got: %x, want: %x", plaintextCopy, plaintext) + } +} diff --git a/src/pkg/crypto/cipher/cipher.go b/src/pkg/crypto/cipher/cipher.go new file mode 100644 index 000000000..1ffaa8c2c --- /dev/null +++ b/src/pkg/crypto/cipher/cipher.go @@ -0,0 +1,63 @@ +// Copyright 2010 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Package cipher implements standard block cipher modes that can be wrapped +// around low-level block cipher implementations. +// See http://csrc.nist.gov/groups/ST/toolkit/BCM/current_modes.html +// and NIST Special Publication 800-38A. +package cipher + +// A Block represents an implementation of block cipher +// using a given key. It provides the capability to encrypt +// or decrypt individual blocks. The mode implementations +// extend that capability to streams of blocks. +type Block interface { + // BlockSize returns the cipher's block size. + BlockSize() int + + // Encrypt encrypts the first block in src into dst. + // Dst and src may point at the same memory. + Encrypt(dst, src []byte) + + // Decrypt decrypts the first block in src into dst. + // Dst and src may point at the same memory. + Decrypt(dst, src []byte) +} + +// A Stream represents a stream cipher. +type Stream interface { + // XORKeyStream XORs each byte in the given slice with a byte from the + // cipher's key stream. Dst and src may point to the same memory. + XORKeyStream(dst, src []byte) +} + +// A BlockMode represents a block cipher running in a block-based mode (CBC, +// ECB etc). +type BlockMode interface { + // BlockSize returns the mode's block size. + BlockSize() int + + // CryptBlocks encrypts or decrypts a number of blocks. The length of + // src must be a multiple of the block size. Dst and src may point to + // the same memory. + CryptBlocks(dst, src []byte) +} + +// Utility routines + +func shift1(dst, src []byte) byte { + var b byte + for i := len(src) - 1; i >= 0; i-- { + bb := src[i] >> 7 + dst[i] = src[i]<<1 | b + b = bb + } + return b +} + +func dup(p []byte) []byte { + q := make([]byte, len(p)) + copy(q, p) + return q +} diff --git a/src/pkg/crypto/cipher/common_test.go b/src/pkg/crypto/cipher/common_test.go new file mode 100644 index 000000000..fb755757c --- /dev/null +++ b/src/pkg/crypto/cipher/common_test.go @@ -0,0 +1,28 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package cipher + +// Common values for tests. + +var commonInput = []byte{ + 0x6b, 0xc1, 0xbe, 0xe2, 0x2e, 0x40, 0x9f, 0x96, 0xe9, 0x3d, 0x7e, 0x11, 0x73, 0x93, 0x17, 0x2a, + 0xae, 0x2d, 0x8a, 0x57, 0x1e, 0x03, 0xac, 0x9c, 0x9e, 0xb7, 0x6f, 0xac, 0x45, 0xaf, 0x8e, 0x51, + 0x30, 0xc8, 0x1c, 0x46, 0xa3, 0x5c, 0xe4, 0x11, 0xe5, 0xfb, 0xc1, 0x19, 0x1a, 0x0a, 0x52, 0xef, + 0xf6, 0x9f, 0x24, 0x45, 0xdf, 0x4f, 0x9b, 0x17, 0xad, 0x2b, 0x41, 0x7b, 0xe6, 0x6c, 0x37, 0x10, +} + +var commonKey128 = []byte{0x2b, 0x7e, 0x15, 0x16, 0x28, 0xae, 0xd2, 0xa6, 0xab, 0xf7, 0x15, 0x88, 0x09, 0xcf, 0x4f, 0x3c} + +var commonKey192 = []byte{ + 0x8e, 0x73, 0xb0, 0xf7, 0xda, 0x0e, 0x64, 0x52, 0xc8, 0x10, 0xf3, 0x2b, 0x80, 0x90, 0x79, 0xe5, + 0x62, 0xf8, 0xea, 0xd2, 0x52, 0x2c, 0x6b, 0x7b, +} + +var commonKey256 = []byte{ + 0x60, 0x3d, 0xeb, 0x10, 0x15, 0xca, 0x71, 0xbe, 0x2b, 0x73, 0xae, 0xf0, 0x85, 0x7d, 0x77, 0x81, + 0x1f, 0x35, 0x2c, 0x07, 0x3b, 0x61, 0x08, 0xd7, 0x2d, 0x98, 0x10, 0xa3, 0x09, 0x14, 0xdf, 0xf4, +} + +var commonIV = []byte{0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f} diff --git a/src/pkg/crypto/cipher/ctr.go b/src/pkg/crypto/cipher/ctr.go new file mode 100644 index 000000000..147b74fc2 --- /dev/null +++ b/src/pkg/crypto/cipher/ctr.go @@ -0,0 +1,55 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Counter (CTR) mode. + +// CTR converts a block cipher into a stream cipher by +// repeatedly encrypting an incrementing counter and +// xoring the resulting stream of data with the input. + +// See NIST SP 800-38A, pp 13-15 + +package cipher + +type ctr struct { + b Block + ctr []byte + out []byte + outUsed int +} + +// NewCTR returns a Stream which encrypts/decrypts using the given Block in +// counter mode. The length of iv must be the same as the Block's block size. +func NewCTR(block Block, iv []byte) Stream { + if len(iv) != block.BlockSize() { + panic("cipher.NewCTR: iv length must equal block size") + } + + return &ctr{ + b: block, + ctr: dup(iv), + out: make([]byte, len(iv)), + outUsed: len(iv), + } +} + +func (x *ctr) XORKeyStream(dst, src []byte) { + for i := 0; i < len(src); i++ { + if x.outUsed == len(x.ctr) { + x.b.Encrypt(x.out, x.ctr) + x.outUsed = 0 + + // Increment counter + for i := len(x.ctr) - 1; i >= 0; i-- { + x.ctr[i]++ + if x.ctr[i] != 0 { + break + } + } + } + + dst[i] = src[i] ^ x.out[x.outUsed] + x.outUsed++ + } +} diff --git a/src/pkg/crypto/cipher/ctr_aes_test.go b/src/pkg/crypto/cipher/ctr_aes_test.go new file mode 100644 index 000000000..8dca9968c --- /dev/null +++ b/src/pkg/crypto/cipher/ctr_aes_test.go @@ -0,0 +1,101 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// CTR AES test vectors. + +// See U.S. National Institute of Standards and Technology (NIST) +// Special Publication 800-38A, ``Recommendation for Block Cipher +// Modes of Operation,'' 2001 Edition, pp. 55-58. + +package cipher + +import ( + "bytes" + "crypto/aes" + "testing" +) + +var commonCounter = []byte{0xf0, 0xf1, 0xf2, 0xf3, 0xf4, 0xf5, 0xf6, 0xf7, 0xf8, 0xf9, 0xfa, 0xfb, 0xfc, 0xfd, 0xfe, 0xff} + +var ctrAESTests = []struct { + name string + key []byte + iv []byte + in []byte + out []byte +}{ + // NIST SP 800-38A pp 55-58 + { + "CTR-AES128", + commonKey128, + commonCounter, + commonInput, + []byte{ + 0x87, 0x4d, 0x61, 0x91, 0xb6, 0x20, 0xe3, 0x26, 0x1b, 0xef, 0x68, 0x64, 0x99, 0x0d, 0xb6, 0xce, + 0x98, 0x06, 0xf6, 0x6b, 0x79, 0x70, 0xfd, 0xff, 0x86, 0x17, 0x18, 0x7b, 0xb9, 0xff, 0xfd, 0xff, + 0x5a, 0xe4, 0xdf, 0x3e, 0xdb, 0xd5, 0xd3, 0x5e, 0x5b, 0x4f, 0x09, 0x02, 0x0d, 0xb0, 0x3e, 0xab, + 0x1e, 0x03, 0x1d, 0xda, 0x2f, 0xbe, 0x03, 0xd1, 0x79, 0x21, 0x70, 0xa0, 0xf3, 0x00, 0x9c, 0xee, + }, + }, + { + "CTR-AES192", + commonKey192, + commonCounter, + commonInput, + []byte{ + 0x1a, 0xbc, 0x93, 0x24, 0x17, 0x52, 0x1c, 0xa2, 0x4f, 0x2b, 0x04, 0x59, 0xfe, 0x7e, 0x6e, 0x0b, + 0x09, 0x03, 0x39, 0xec, 0x0a, 0xa6, 0xfa, 0xef, 0xd5, 0xcc, 0xc2, 0xc6, 0xf4, 0xce, 0x8e, 0x94, + 0x1e, 0x36, 0xb2, 0x6b, 0xd1, 0xeb, 0xc6, 0x70, 0xd1, 0xbd, 0x1d, 0x66, 0x56, 0x20, 0xab, 0xf7, + 0x4f, 0x78, 0xa7, 0xf6, 0xd2, 0x98, 0x09, 0x58, 0x5a, 0x97, 0xda, 0xec, 0x58, 0xc6, 0xb0, 0x50, + }, + }, + { + "CTR-AES256", + commonKey256, + commonCounter, + commonInput, + []byte{ + 0x60, 0x1e, 0xc3, 0x13, 0x77, 0x57, 0x89, 0xa5, 0xb7, 0xa7, 0xf5, 0x04, 0xbb, 0xf3, 0xd2, 0x28, + 0xf4, 0x43, 0xe3, 0xca, 0x4d, 0x62, 0xb5, 0x9a, 0xca, 0x84, 0xe9, 0x90, 0xca, 0xca, 0xf5, 0xc5, + 0x2b, 0x09, 0x30, 0xda, 0xa2, 0x3d, 0xe9, 0x4c, 0xe8, 0x70, 0x17, 0xba, 0x2d, 0x84, 0x98, 0x8d, + 0xdf, 0xc9, 0xc5, 0x8d, 0xb6, 0x7a, 0xad, 0xa6, 0x13, 0xc2, 0xdd, 0x08, 0x45, 0x79, 0x41, 0xa6, + }, + }, +} + +func TestCTR_AES(t *testing.T) { + for _, tt := range ctrAESTests { + test := tt.name + + c, err := aes.NewCipher(tt.key) + if err != nil { + t.Errorf("%s: NewCipher(%d bytes) = %s", test, len(tt.key), err) + continue + } + + for j := 0; j <= 5; j += 5 { + in := tt.in[0 : len(tt.in)-j] + ctr := NewCTR(c, tt.iv) + encrypted := make([]byte, len(in)) + ctr.XORKeyStream(encrypted, in) + if out := tt.out[0:len(in)]; !bytes.Equal(out, encrypted) { + t.Errorf("%s/%d: CTR\ninpt %x\nhave %x\nwant %x", test, len(in), in, encrypted, out) + } + } + + for j := 0; j <= 7; j += 7 { + in := tt.out[0 : len(tt.out)-j] + ctr := NewCTR(c, tt.iv) + plain := make([]byte, len(in)) + ctr.XORKeyStream(plain, in) + if out := tt.in[0:len(in)]; !bytes.Equal(out, plain) { + t.Errorf("%s/%d: CTRReader\nhave %x\nwant %x", test, len(out), plain, out) + } + } + + if t.Failed() { + break + } + } +} diff --git a/src/pkg/crypto/cipher/io.go b/src/pkg/crypto/cipher/io.go new file mode 100644 index 000000000..97f40b8e7 --- /dev/null +++ b/src/pkg/crypto/cipher/io.go @@ -0,0 +1,57 @@ +// Copyright 2010 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package cipher + +import ( + "os" + "io" +) + +// The Stream* objects are so simple that all their members are public. Users +// can create them themselves. + +// StreamReader wraps a Stream into an io.Reader. It simply calls XORKeyStream +// to process each slice of data which passes through. +type StreamReader struct { + S Stream + R io.Reader +} + +func (r StreamReader) Read(dst []byte) (n int, err os.Error) { + n, err = r.R.Read(dst) + r.S.XORKeyStream(dst[:n], dst[:n]) + return +} + +// StreamWriter wraps a Stream into an io.Writer. It simply calls XORKeyStream +// to process each slice of data which passes through. If any Write call +// returns short then the StreamWriter is out of sync and must be discarded. +type StreamWriter struct { + S Stream + W io.Writer + Err os.Error +} + +func (w StreamWriter) Write(src []byte) (n int, err os.Error) { + if w.Err != nil { + return 0, w.Err + } + c := make([]byte, len(src)) + w.S.XORKeyStream(c, src) + n, err = w.W.Write(c) + if n != len(src) { + if err == nil { // should never happen + err = io.ErrShortWrite + } + w.Err = err + } + return +} + +func (w StreamWriter) Close() os.Error { + // This saves us from either requiring a WriteCloser or having a + // StreamWriterCloser. + return w.W.(io.Closer).Close() +} diff --git a/src/pkg/crypto/cipher/ocfb.go b/src/pkg/crypto/cipher/ocfb.go new file mode 100644 index 000000000..031e74a9d --- /dev/null +++ b/src/pkg/crypto/cipher/ocfb.go @@ -0,0 +1,138 @@ +// Copyright 2010 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// OpenPGP CFB Mode. http://tools.ietf.org/html/rfc4880#section-13.9 + +package cipher + +type ocfbEncrypter struct { + b Block + fre []byte + outUsed int +} + +// An OCFBResyncOption determines if the "resynchronization step" of OCFB is +// performed. +type OCFBResyncOption bool + +const ( + OCFBResync OCFBResyncOption = true + OCFBNoResync OCFBResyncOption = false +) + +// NewOCFBEncrypter returns a Stream which encrypts data with OpenPGP's cipher +// feedback mode using the given Block, and an initial amount of ciphertext. +// randData must be random bytes and be the same length as the Block's block +// size. Resync determines if the "resynchronization step" from RFC 4880, 13.9 +// step 7 is performed. Different parts of OpenPGP vary on this point. +func NewOCFBEncrypter(block Block, randData []byte, resync OCFBResyncOption) (Stream, []byte) { + blockSize := block.BlockSize() + if len(randData) != blockSize { + return nil, nil + } + + x := &ocfbEncrypter{ + b: block, + fre: make([]byte, blockSize), + outUsed: 0, + } + prefix := make([]byte, blockSize+2) + + block.Encrypt(x.fre, x.fre) + for i := 0; i < blockSize; i++ { + prefix[i] = randData[i] ^ x.fre[i] + } + + block.Encrypt(x.fre, prefix[:blockSize]) + prefix[blockSize] = x.fre[0] ^ randData[blockSize-2] + prefix[blockSize+1] = x.fre[1] ^ randData[blockSize-1] + + if resync { + block.Encrypt(x.fre, prefix[2:]) + } else { + x.fre[0] = prefix[blockSize] + x.fre[1] = prefix[blockSize+1] + x.outUsed = 2 + } + return x, prefix +} + +func (x *ocfbEncrypter) XORKeyStream(dst, src []byte) { + for i := 0; i < len(src); i++ { + if x.outUsed == len(x.fre) { + x.b.Encrypt(x.fre, x.fre) + x.outUsed = 0 + } + + x.fre[x.outUsed] ^= src[i] + dst[i] = x.fre[x.outUsed] + x.outUsed++ + } +} + +type ocfbDecrypter struct { + b Block + fre []byte + outUsed int +} + +// NewOCFBDecrypter returns a Stream which decrypts data with OpenPGP's cipher +// feedback mode using the given Block. Prefix must be the first blockSize + 2 +// bytes of the ciphertext, where blockSize is the Block's block size. If an +// incorrect key is detected then nil is returned. On successful exit, +// blockSize+2 bytes of decrypted data are written into prefix. Resync +// determines if the "resynchronization step" from RFC 4880, 13.9 step 7 is +// performed. Different parts of OpenPGP vary on this point. +func NewOCFBDecrypter(block Block, prefix []byte, resync OCFBResyncOption) Stream { + blockSize := block.BlockSize() + if len(prefix) != blockSize+2 { + return nil + } + + x := &ocfbDecrypter{ + b: block, + fre: make([]byte, blockSize), + outUsed: 0, + } + prefixCopy := make([]byte, len(prefix)) + copy(prefixCopy, prefix) + + block.Encrypt(x.fre, x.fre) + for i := 0; i < blockSize; i++ { + prefixCopy[i] ^= x.fre[i] + } + + block.Encrypt(x.fre, prefix[:blockSize]) + prefixCopy[blockSize] ^= x.fre[0] + prefixCopy[blockSize+1] ^= x.fre[1] + + if prefixCopy[blockSize-2] != prefixCopy[blockSize] || + prefixCopy[blockSize-1] != prefixCopy[blockSize+1] { + return nil + } + + if resync { + block.Encrypt(x.fre, prefix[2:]) + } else { + x.fre[0] = prefix[blockSize] + x.fre[1] = prefix[blockSize+1] + x.outUsed = 2 + } + copy(prefix, prefixCopy) + return x +} + +func (x *ocfbDecrypter) XORKeyStream(dst, src []byte) { + for i := 0; i < len(src); i++ { + if x.outUsed == len(x.fre) { + x.b.Encrypt(x.fre, x.fre) + x.outUsed = 0 + } + + c := src[i] + dst[i] = x.fre[x.outUsed] ^ src[i] + x.fre[x.outUsed] = c + x.outUsed++ + } +} diff --git a/src/pkg/crypto/cipher/ocfb_test.go b/src/pkg/crypto/cipher/ocfb_test.go new file mode 100644 index 000000000..40938b589 --- /dev/null +++ b/src/pkg/crypto/cipher/ocfb_test.go @@ -0,0 +1,44 @@ +// Copyright 2010 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package cipher + +import ( + "bytes" + "crypto/aes" + "crypto/rand" + "testing" +) + +func testOCFB(t *testing.T, resync OCFBResyncOption) { + block, err := aes.NewCipher(commonKey128) + if err != nil { + t.Error(err) + return + } + + plaintext := []byte("this is the plaintext, which is long enough to span several blocks.") + randData := make([]byte, block.BlockSize()) + rand.Reader.Read(randData) + ocfb, prefix := NewOCFBEncrypter(block, randData, resync) + ciphertext := make([]byte, len(plaintext)) + ocfb.XORKeyStream(ciphertext, plaintext) + + ocfbdec := NewOCFBDecrypter(block, prefix, resync) + if ocfbdec == nil { + t.Errorf("NewOCFBDecrypter failed (resync: %t)", resync) + return + } + plaintextCopy := make([]byte, len(plaintext)) + ocfbdec.XORKeyStream(plaintextCopy, ciphertext) + + if !bytes.Equal(plaintextCopy, plaintext) { + t.Errorf("got: %x, want: %x (resync: %t)", plaintextCopy, plaintext, resync) + } +} + +func TestOCFB(t *testing.T) { + testOCFB(t, OCFBNoResync) + testOCFB(t, OCFBResync) +} diff --git a/src/pkg/crypto/cipher/ofb.go b/src/pkg/crypto/cipher/ofb.go new file mode 100644 index 000000000..85e5f02b0 --- /dev/null +++ b/src/pkg/crypto/cipher/ofb.go @@ -0,0 +1,44 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// OFB (Output Feedback) Mode. + +package cipher + +type ofb struct { + b Block + out []byte + outUsed int +} + +// NewOFB returns a Stream that encrypts or decrypts using the block cipher b +// in output feedback mode. The initialization vector iv's length must be equal +// to b's block size. +func NewOFB(b Block, iv []byte) Stream { + blockSize := b.BlockSize() + if len(iv) != blockSize { + return nil + } + + x := &ofb{ + b: b, + out: make([]byte, blockSize), + outUsed: 0, + } + b.Encrypt(x.out, iv) + + return x +} + +func (x *ofb) XORKeyStream(dst, src []byte) { + for i, s := range src { + if x.outUsed == len(x.out) { + x.b.Encrypt(x.out, x.out) + x.outUsed = 0 + } + + dst[i] = s ^ x.out[x.outUsed] + x.outUsed++ + } +} diff --git a/src/pkg/crypto/cipher/ofb_test.go b/src/pkg/crypto/cipher/ofb_test.go new file mode 100644 index 000000000..9b4495c88 --- /dev/null +++ b/src/pkg/crypto/cipher/ofb_test.go @@ -0,0 +1,101 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// OFB AES test vectors. + +// See U.S. National Institute of Standards and Technology (NIST) +// Special Publication 800-38A, ``Recommendation for Block Cipher +// Modes of Operation,'' 2001 Edition, pp. 52-55. + +package cipher + +import ( + "bytes" + "crypto/aes" + "testing" +) + +type ofbTest struct { + name string + key []byte + iv []byte + in []byte + out []byte +} + +var ofbTests = []ofbTest{ + // NIST SP 800-38A pp 52-55 + { + "OFB-AES128", + commonKey128, + commonIV, + commonInput, + []byte{ + 0x3b, 0x3f, 0xd9, 0x2e, 0xb7, 0x2d, 0xad, 0x20, 0x33, 0x34, 0x49, 0xf8, 0xe8, 0x3c, 0xfb, 0x4a, + 0x77, 0x89, 0x50, 0x8d, 0x16, 0x91, 0x8f, 0x03, 0xf5, 0x3c, 0x52, 0xda, 0xc5, 0x4e, 0xd8, 0x25, + 0x97, 0x40, 0x05, 0x1e, 0x9c, 0x5f, 0xec, 0xf6, 0x43, 0x44, 0xf7, 0xa8, 0x22, 0x60, 0xed, 0xcc, + 0x30, 0x4c, 0x65, 0x28, 0xf6, 0x59, 0xc7, 0x78, 0x66, 0xa5, 0x10, 0xd9, 0xc1, 0xd6, 0xae, 0x5e, + }, + }, + { + "OFB-AES192", + commonKey192, + commonIV, + commonInput, + []byte{ + 0xcd, 0xc8, 0x0d, 0x6f, 0xdd, 0xf1, 0x8c, 0xab, 0x34, 0xc2, 0x59, 0x09, 0xc9, 0x9a, 0x41, 0x74, + 0xfc, 0xc2, 0x8b, 0x8d, 0x4c, 0x63, 0x83, 0x7c, 0x09, 0xe8, 0x17, 0x00, 0xc1, 0x10, 0x04, 0x01, + 0x8d, 0x9a, 0x9a, 0xea, 0xc0, 0xf6, 0x59, 0x6f, 0x55, 0x9c, 0x6d, 0x4d, 0xaf, 0x59, 0xa5, 0xf2, + 0x6d, 0x9f, 0x20, 0x08, 0x57, 0xca, 0x6c, 0x3e, 0x9c, 0xac, 0x52, 0x4b, 0xd9, 0xac, 0xc9, 0x2a, + }, + }, + { + "OFB-AES256", + commonKey256, + commonIV, + commonInput, + []byte{ + 0xdc, 0x7e, 0x84, 0xbf, 0xda, 0x79, 0x16, 0x4b, 0x7e, 0xcd, 0x84, 0x86, 0x98, 0x5d, 0x38, 0x60, + 0x4f, 0xeb, 0xdc, 0x67, 0x40, 0xd2, 0x0b, 0x3a, 0xc8, 0x8f, 0x6a, 0xd8, 0x2a, 0x4f, 0xb0, 0x8d, + 0x71, 0xab, 0x47, 0xa0, 0x86, 0xe8, 0x6e, 0xed, 0xf3, 0x9d, 0x1c, 0x5b, 0xba, 0x97, 0xc4, 0x08, + 0x01, 0x26, 0x14, 0x1d, 0x67, 0xf3, 0x7b, 0xe8, 0x53, 0x8f, 0x5a, 0x8b, 0xe7, 0x40, 0xe4, 0x84, + }, + }, +} + +func TestOFB(t *testing.T) { + for _, tt := range ofbTests { + test := tt.name + + c, err := aes.NewCipher(tt.key) + if err != nil { + t.Errorf("%s: NewCipher(%d bytes) = %s", test, len(tt.key), err) + continue + } + + for j := 0; j <= 5; j += 5 { + plaintext := tt.in[0 : len(tt.in)-j] + ofb := NewOFB(c, tt.iv) + ciphertext := make([]byte, len(plaintext)) + ofb.XORKeyStream(ciphertext, plaintext) + if !bytes.Equal(ciphertext, tt.out[:len(plaintext)]) { + t.Errorf("%s/%d: encrypting\ninput % x\nhave % x\nwant % x", test, len(plaintext), plaintext, ciphertext, tt.out) + } + } + + for j := 0; j <= 5; j += 5 { + ciphertext := tt.out[0 : len(tt.in)-j] + ofb := NewOFB(c, tt.iv) + plaintext := make([]byte, len(ciphertext)) + ofb.XORKeyStream(plaintext, ciphertext) + if !bytes.Equal(plaintext, tt.in[:len(ciphertext)]) { + t.Errorf("%s/%d: decrypting\nhave % x\nwant % x", test, len(ciphertext), plaintext, tt.in) + } + } + + if t.Failed() { + break + } + } +} diff --git a/src/pkg/crypto/crypto.go b/src/pkg/crypto/crypto.go new file mode 100644 index 000000000..53672a4da --- /dev/null +++ b/src/pkg/crypto/crypto.go @@ -0,0 +1,73 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Package crypto collects common cryptographic constants. +package crypto + +import ( + "hash" +) + +// Hash identifies a cryptographic hash function that is implemented in another +// package. +type Hash uint + +const ( + MD4 Hash = 1 + iota // in package crypto/md4 + MD5 // in package crypto/md5 + SHA1 // in package crypto/sha1 + SHA224 // in package crypto/sha256 + SHA256 // in package crypto/sha256 + SHA384 // in package crypto/sha512 + SHA512 // in package crypto/sha512 + MD5SHA1 // no implementation; MD5+SHA1 used for TLS RSA + RIPEMD160 // in package crypto/ripemd160 + maxHash +) + +var digestSizes = []uint8{ + MD4: 16, + MD5: 16, + SHA1: 20, + SHA224: 28, + SHA256: 32, + SHA384: 48, + SHA512: 64, + MD5SHA1: 36, + RIPEMD160: 20, +} + +// Size returns the length, in bytes, of a digest resulting from the given hash +// function. It doesn't require that the hash function in question be linked +// into the program. +func (h Hash) Size() int { + if h > 0 && h < maxHash { + return int(digestSizes[h]) + } + panic("crypto: Size of unknown hash function") +} + +var hashes = make([]func() hash.Hash, maxHash) + +// New returns a new hash.Hash calculating the given hash function. If the +// hash function is not linked into the binary, New returns nil. +func (h Hash) New() hash.Hash { + if h > 0 && h < maxHash { + f := hashes[h] + if f != nil { + return f() + } + } + return nil +} + +// RegisterHash registers a function that returns a new instance of the given +// hash function. This is intended to be called from the init function in +// packages that implement hash functions. +func RegisterHash(h Hash, f func() hash.Hash) { + if h >= maxHash { + panic("crypto: RegisterHash of unknown hash function") + } + hashes[h] = f +} diff --git a/src/pkg/crypto/des/Makefile b/src/pkg/crypto/des/Makefile new file mode 100644 index 000000000..94b0fc0fa --- /dev/null +++ b/src/pkg/crypto/des/Makefile @@ -0,0 +1,13 @@ +# Copyright 2010 The Go Authors. All rights reserved. +# Use of this source code is governed by a BSD-style +# license that can be found in the LICENSE file. + +include ../../../Make.inc + +TARG=crypto/des +GOFILES=\ + block.go\ + cipher.go\ + const.go\ + +include ../../../Make.pkg diff --git a/src/pkg/crypto/des/block.go b/src/pkg/crypto/des/block.go new file mode 100644 index 000000000..e18eaedf5 --- /dev/null +++ b/src/pkg/crypto/des/block.go @@ -0,0 +1,98 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package des + +import ( + "encoding/binary" +) + +func cryptBlock(subkeys []uint64, dst, src []byte, decrypt bool) { + b := binary.BigEndian.Uint64(src) + b = permuteBlock(b, initialPermutation[:]) + left, right := uint32(b>>32), uint32(b) + + var subkey uint64 + for i := 0; i < 16; i++ { + if decrypt { + subkey = subkeys[15-i] + } else { + subkey = subkeys[i] + } + + left, right = right, left^feistel(right, subkey) + } + // switch left & right and perform final permutation + preOutput := (uint64(right) << 32) | uint64(left) + binary.BigEndian.PutUint64(dst, permuteBlock(preOutput, finalPermutation[:])) +} + +// Encrypt one block from src into dst, using the subkeys. +func encryptBlock(subkeys []uint64, dst, src []byte) { + cryptBlock(subkeys, dst, src, false) +} + +// Decrypt one block from src into dst, using the subkeys. +func decryptBlock(subkeys []uint64, dst, src []byte) { + cryptBlock(subkeys, dst, src, true) +} + +// DES Feistel function +func feistel(right uint32, key uint64) (result uint32) { + sBoxLocations := key ^ permuteBlock(uint64(right), expansionFunction[:]) + var sBoxResult uint32 + for i := uint8(0); i < 8; i++ { + sBoxLocation := uint8(sBoxLocations>>42) & 0x3f + sBoxLocations <<= 6 + // row determined by 1st and 6th bit + row := (sBoxLocation & 0x1) | ((sBoxLocation & 0x20) >> 4) + // column is middle four bits + column := (sBoxLocation >> 1) & 0xf + sBoxResult |= uint32(sBoxes[i][row][column]) << (4 * (7 - i)) + } + return uint32(permuteBlock(uint64(sBoxResult), permutationFunction[:])) +} + +// general purpose function to perform DES block permutations +func permuteBlock(src uint64, permutation []uint8) (block uint64) { + for position, n := range permutation { + bit := (src >> n) & 1 + block |= bit << uint((len(permutation)-1)-position) + } + return +} + +// creates 16 28-bit blocks rotated according +// to the rotation schedule +func ksRotate(in uint32) (out []uint32) { + out = make([]uint32, 16) + last := in + for i := 0; i < 16; i++ { + // 28-bit circular left shift + left := (last << (4 + ksRotations[i])) >> 4 + right := (last << 4) >> (32 - ksRotations[i]) + out[i] = left | right + last = out[i] + } + return +} + +// creates 16 56-bit subkeys from the original key +func (c *Cipher) generateSubkeys(keyBytes []byte) { + // apply PC1 permutation to key + key := binary.BigEndian.Uint64(keyBytes) + permutedKey := permuteBlock(key, permutedChoice1[:]) + + // rotate halves of permuted key according to the rotation schedule + leftRotations := ksRotate(uint32(permutedKey >> 28)) + rightRotations := ksRotate(uint32(permutedKey<<4) >> 4) + + // generate subkeys + for i := 0; i < 16; i++ { + // combine halves to form 56-bit input to PC2 + pc2Input := uint64(leftRotations[i])<<28 | uint64(rightRotations[i]) + // apply PC2 permutation to 7 byte input + c.subkeys[i] = permuteBlock(pc2Input, permutedChoice2[:]) + } +} diff --git a/src/pkg/crypto/des/cipher.go b/src/pkg/crypto/des/cipher.go new file mode 100644 index 000000000..d17a1a783 --- /dev/null +++ b/src/pkg/crypto/des/cipher.go @@ -0,0 +1,103 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package des + +import ( + "os" + "strconv" +) + +// The DES block size in bytes. +const BlockSize = 8 + +type KeySizeError int + +func (k KeySizeError) String() string { + return "crypto/des: invalid key size " + strconv.Itoa(int(k)) +} + +// Cipher is an instance of DES encryption. +type Cipher struct { + subkeys [16]uint64 +} + +// NewCipher creates and returns a new Cipher. +func NewCipher(key []byte) (*Cipher, os.Error) { + if len(key) != 8 { + return nil, KeySizeError(len(key)) + } + + c := new(Cipher) + c.generateSubkeys(key) + return c, nil +} + +// BlockSize returns the DES block size, 8 bytes. +func (c *Cipher) BlockSize() int { return BlockSize } + +// Encrypts the 8-byte buffer src and stores the result in dst. +// Note that for amounts of data larger than a block, +// it is not safe to just call Encrypt on successive blocks; +// instead, use an encryption mode like CBC (see crypto/cipher/cbc.go). +func (c *Cipher) Encrypt(dst, src []byte) { encryptBlock(c.subkeys[:], dst, src) } + +// Decrypts the 8-byte buffer src and stores the result in dst. +func (c *Cipher) Decrypt(dst, src []byte) { decryptBlock(c.subkeys[:], dst, src) } + +// Reset zeros the key data, so that it will no longer +// appear in the process's memory. +func (c *Cipher) Reset() { + for i := 0; i < len(c.subkeys); i++ { + c.subkeys[i] = 0 + } +} + +// A TripleDESCipher is an instance of TripleDES encryption. +type TripleDESCipher struct { + cipher1, cipher2, cipher3 Cipher +} + +// NewCipher creates and returns a new Cipher. +func NewTripleDESCipher(key []byte) (*TripleDESCipher, os.Error) { + if len(key) != 24 { + return nil, KeySizeError(len(key)) + } + + c := new(TripleDESCipher) + c.cipher1.generateSubkeys(key[:8]) + c.cipher2.generateSubkeys(key[8:16]) + c.cipher3.generateSubkeys(key[16:]) + return c, nil +} + +// BlockSize returns the TripleDES block size, 8 bytes. +// It is necessary to satisfy the Block interface in the +// package "crypto/cipher". +func (c *TripleDESCipher) BlockSize() int { return BlockSize } + +// Encrypts the 8-byte buffer src and stores the result in dst. +// Note that for amounts of data larger than a block, +// it is not safe to just call Encrypt on successive blocks; +// instead, use an encryption mode like CBC (see crypto/cipher/cbc.go). +func (c *TripleDESCipher) Encrypt(dst, src []byte) { + c.cipher1.Encrypt(dst, src) + c.cipher2.Decrypt(dst, dst) + c.cipher3.Encrypt(dst, dst) +} + +// Decrypts the 8-byte buffer src and stores the result in dst. +func (c *TripleDESCipher) Decrypt(dst, src []byte) { + c.cipher3.Decrypt(dst, src) + c.cipher2.Encrypt(dst, dst) + c.cipher1.Decrypt(dst, dst) +} + +// Reset zeros the key data, so that it will no longer +// appear in the process's memory. +func (c *TripleDESCipher) Reset() { + c.cipher1.Reset() + c.cipher2.Reset() + c.cipher3.Reset() +} diff --git a/src/pkg/crypto/des/const.go b/src/pkg/crypto/des/const.go new file mode 100644 index 000000000..2bd485ee8 --- /dev/null +++ b/src/pkg/crypto/des/const.go @@ -0,0 +1,139 @@ +// Copyright 2010 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Package des implements the Data Encryption Standard (DES) and the +// Triple Data Encryption Algorithm (TDEA) as defined +// in U.S. Federal Information Processing Standards Publication 46-3. +package des + +// Used to perform an initial permutation of a 64-bit input block. +var initialPermutation = [64]byte{ + 6, 14, 22, 30, 38, 46, 54, 62, + 4, 12, 20, 28, 36, 44, 52, 60, + 2, 10, 18, 26, 34, 42, 50, 58, + 0, 8, 16, 24, 32, 40, 48, 56, + 7, 15, 23, 31, 39, 47, 55, 63, + 5, 13, 21, 29, 37, 45, 53, 61, + 3, 11, 19, 27, 35, 43, 51, 59, + 1, 9, 17, 25, 33, 41, 49, 57, +} + +// Used to perform a final permutation of a 4-bit preoutput block. This is the +// inverse of initialPermutation +var finalPermutation = [64]byte{ + 24, 56, 16, 48, 8, 40, 0, 32, + 25, 57, 17, 49, 9, 41, 1, 33, + 26, 58, 18, 50, 10, 42, 2, 34, + 27, 59, 19, 51, 11, 43, 3, 35, + 28, 60, 20, 52, 12, 44, 4, 36, + 29, 61, 21, 53, 13, 45, 5, 37, + 30, 62, 22, 54, 14, 46, 6, 38, + 31, 63, 23, 55, 15, 47, 7, 39, +} + +// Used to expand an input block of 32 bits, producing an output block of 48 +// bits. +var expansionFunction = [48]byte{ + 0, 31, 30, 29, 28, 27, 28, 27, + 26, 25, 24, 23, 24, 23, 22, 21, + 20, 19, 20, 19, 18, 17, 16, 15, + 16, 15, 14, 13, 12, 11, 12, 11, + 10, 9, 8, 7, 8, 7, 6, 5, + 4, 3, 4, 3, 2, 1, 0, 31, +} + +// Yields a 32-bit output from a 32-bit input +var permutationFunction = [32]byte{ + 16, 25, 12, 11, 3, 20, 4, 15, + 31, 17, 9, 6, 27, 14, 1, 22, + 30, 24, 8, 18, 0, 5, 29, 23, + 13, 19, 2, 26, 10, 21, 28, 7, +} + +// Used in the key schedule to select 56 bits +// from a 64-bit input. +var permutedChoice1 = [56]byte{ + 7, 15, 23, 31, 39, 47, 55, 63, + 6, 14, 22, 30, 38, 46, 54, 62, + 5, 13, 21, 29, 37, 45, 53, 61, + 4, 12, 20, 28, 1, 9, 17, 25, + 33, 41, 49, 57, 2, 10, 18, 26, + 34, 42, 50, 58, 3, 11, 19, 27, + 35, 43, 51, 59, 36, 44, 52, 60, +} + +// Used in the key schedule to produce each subkey by selecting 48 bits from +// the 56-bit input +var permutedChoice2 = [48]byte{ + 42, 39, 45, 32, 55, 51, 53, 28, + 41, 50, 35, 46, 33, 37, 44, 52, + 30, 48, 40, 49, 29, 36, 43, 54, + 15, 4, 25, 19, 9, 1, 26, 16, + 5, 11, 23, 8, 12, 7, 17, 0, + 22, 3, 10, 14, 6, 20, 27, 24, +} + +// 8 S-boxes composed of 4 rows and 16 columns +// Used in the DES cipher function +var sBoxes = [8][4][16]uint8{ + // S-box 1 + { + {14, 4, 13, 1, 2, 15, 11, 8, 3, 10, 6, 12, 5, 9, 0, 7}, + {0, 15, 7, 4, 14, 2, 13, 1, 10, 6, 12, 11, 9, 5, 3, 8}, + {4, 1, 14, 8, 13, 6, 2, 11, 15, 12, 9, 7, 3, 10, 5, 0}, + {15, 12, 8, 2, 4, 9, 1, 7, 5, 11, 3, 14, 10, 0, 6, 13}, + }, + // S-box 2 + { + {15, 1, 8, 14, 6, 11, 3, 4, 9, 7, 2, 13, 12, 0, 5, 10}, + {3, 13, 4, 7, 15, 2, 8, 14, 12, 0, 1, 10, 6, 9, 11, 5}, + {0, 14, 7, 11, 10, 4, 13, 1, 5, 8, 12, 6, 9, 3, 2, 15}, + {13, 8, 10, 1, 3, 15, 4, 2, 11, 6, 7, 12, 0, 5, 14, 9}, + }, + // S-box 3 + { + {10, 0, 9, 14, 6, 3, 15, 5, 1, 13, 12, 7, 11, 4, 2, 8}, + {13, 7, 0, 9, 3, 4, 6, 10, 2, 8, 5, 14, 12, 11, 15, 1}, + {13, 6, 4, 9, 8, 15, 3, 0, 11, 1, 2, 12, 5, 10, 14, 7}, + {1, 10, 13, 0, 6, 9, 8, 7, 4, 15, 14, 3, 11, 5, 2, 12}, + }, + // S-box 4 + { + {7, 13, 14, 3, 0, 6, 9, 10, 1, 2, 8, 5, 11, 12, 4, 15}, + {13, 8, 11, 5, 6, 15, 0, 3, 4, 7, 2, 12, 1, 10, 14, 9}, + {10, 6, 9, 0, 12, 11, 7, 13, 15, 1, 3, 14, 5, 2, 8, 4}, + {3, 15, 0, 6, 10, 1, 13, 8, 9, 4, 5, 11, 12, 7, 2, 14}, + }, + // S-box 5 + { + {2, 12, 4, 1, 7, 10, 11, 6, 8, 5, 3, 15, 13, 0, 14, 9}, + {14, 11, 2, 12, 4, 7, 13, 1, 5, 0, 15, 10, 3, 9, 8, 6}, + {4, 2, 1, 11, 10, 13, 7, 8, 15, 9, 12, 5, 6, 3, 0, 14}, + {11, 8, 12, 7, 1, 14, 2, 13, 6, 15, 0, 9, 10, 4, 5, 3}, + }, + // S-box 6 + { + {12, 1, 10, 15, 9, 2, 6, 8, 0, 13, 3, 4, 14, 7, 5, 11}, + {10, 15, 4, 2, 7, 12, 9, 5, 6, 1, 13, 14, 0, 11, 3, 8}, + {9, 14, 15, 5, 2, 8, 12, 3, 7, 0, 4, 10, 1, 13, 11, 6}, + {4, 3, 2, 12, 9, 5, 15, 10, 11, 14, 1, 7, 6, 0, 8, 13}, + }, + // S-box 7 + { + {4, 11, 2, 14, 15, 0, 8, 13, 3, 12, 9, 7, 5, 10, 6, 1}, + {13, 0, 11, 7, 4, 9, 1, 10, 14, 3, 5, 12, 2, 15, 8, 6}, + {1, 4, 11, 13, 12, 3, 7, 14, 10, 15, 6, 8, 0, 5, 9, 2}, + {6, 11, 13, 8, 1, 4, 10, 7, 9, 5, 0, 15, 14, 2, 3, 12}, + }, + // S-box 8 + { + {13, 2, 8, 4, 6, 15, 11, 1, 10, 9, 3, 14, 5, 0, 12, 7}, + {1, 15, 13, 8, 10, 3, 7, 4, 12, 5, 6, 11, 0, 14, 9, 2}, + {7, 11, 4, 1, 9, 12, 14, 2, 0, 6, 10, 13, 15, 3, 5, 8}, + {2, 1, 14, 7, 4, 10, 8, 13, 15, 12, 9, 0, 3, 5, 6, 11}, + }, +} + +// Size of left rotation per round in each half of the key schedule +var ksRotations = [16]uint8{1, 1, 2, 2, 2, 2, 2, 2, 1, 2, 2, 2, 2, 2, 2, 1} diff --git a/src/pkg/crypto/des/des_test.go b/src/pkg/crypto/des/des_test.go new file mode 100644 index 000000000..d1f3aa71a --- /dev/null +++ b/src/pkg/crypto/des/des_test.go @@ -0,0 +1,1497 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package des + +import ( + "bytes" + "testing" +) + +type CryptTest struct { + key []byte + in []byte + out []byte +} + +// some custom tests for DES +var encryptDESTests = []CryptTest{ + { + []byte{0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00}, + []byte{0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00}, + []byte{0x8c, 0xa6, 0x4d, 0xe9, 0xc1, 0xb1, 0x23, 0xa7}}, + { + []byte{0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00}, + []byte{0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff}, + []byte{0x35, 0x55, 0x50, 0xb2, 0x15, 0x0e, 0x24, 0x51}}, + { + []byte{0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00}, + []byte{0x01, 0x23, 0x45, 0x67, 0x89, 0xab, 0xcd, 0xef}, + []byte{0x61, 0x7b, 0x3a, 0x0c, 0xe8, 0xf0, 0x71, 0x00}}, + { + []byte{0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00}, + []byte{0xfe, 0xdc, 0xba, 0x98, 0x76, 0x54, 0x32, 0x10}, + []byte{0x92, 0x31, 0xf2, 0x36, 0xff, 0x9a, 0xa9, 0x5c}}, + { + []byte{0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff}, + []byte{0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00}, + []byte{0xca, 0xaa, 0xaf, 0x4d, 0xea, 0xf1, 0xdb, 0xae}}, + { + []byte{0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff}, + []byte{0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff}, + []byte{0x73, 0x59, 0xb2, 0x16, 0x3e, 0x4e, 0xdc, 0x58}}, + { + []byte{0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff}, + []byte{0x01, 0x23, 0x45, 0x67, 0x89, 0xab, 0xcd, 0xef}, + []byte{0x6d, 0xce, 0x0d, 0xc9, 0x00, 0x65, 0x56, 0xa3}}, + { + []byte{0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff}, + []byte{0xfe, 0xdc, 0xba, 0x98, 0x76, 0x54, 0x32, 0x10}, + []byte{0x9e, 0x84, 0xc5, 0xf3, 0x17, 0x0f, 0x8e, 0xff}}, + { + []byte{0x01, 0x23, 0x45, 0x67, 0x89, 0xab, 0xcd, 0xef}, + []byte{0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00}, + []byte{0xd5, 0xd4, 0x4f, 0xf7, 0x20, 0x68, 0x3d, 0x0d}}, + { + []byte{0x01, 0x23, 0x45, 0x67, 0x89, 0xab, 0xcd, 0xef}, + []byte{0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff}, + []byte{0x59, 0x73, 0x23, 0x56, 0xf3, 0x6f, 0xde, 0x06}}, + { + []byte{0x01, 0x23, 0x45, 0x67, 0x89, 0xab, 0xcd, 0xef}, + []byte{0x01, 0x23, 0x45, 0x67, 0x89, 0xab, 0xcd, 0xef}, + []byte{0x56, 0xcc, 0x09, 0xe7, 0xcf, 0xdc, 0x4c, 0xef}}, + { + []byte{0x01, 0x23, 0x45, 0x67, 0x89, 0xab, 0xcd, 0xef}, + []byte{0xfe, 0xdc, 0xba, 0x98, 0x76, 0x54, 0x32, 0x10}, + []byte{0x12, 0xc6, 0x26, 0xaf, 0x05, 0x8b, 0x43, 0x3b}}, + { + []byte{0xfe, 0xdc, 0xba, 0x98, 0x76, 0x54, 0x32, 0x10}, + []byte{0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00}, + []byte{0xa6, 0x8c, 0xdc, 0xa9, 0x0c, 0x90, 0x21, 0xf9}}, + { + []byte{0xfe, 0xdc, 0xba, 0x98, 0x76, 0x54, 0x32, 0x10}, + []byte{0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff}, + []byte{0x2a, 0x2b, 0xb0, 0x08, 0xdf, 0x97, 0xc2, 0xf2}}, + { + []byte{0xfe, 0xdc, 0xba, 0x98, 0x76, 0x54, 0x32, 0x10}, + []byte{0x01, 0x23, 0x45, 0x67, 0x89, 0xab, 0xcd, 0xef}, + []byte{0xed, 0x39, 0xd9, 0x50, 0xfa, 0x74, 0xbc, 0xc4}}, + { + []byte{0xfe, 0xdc, 0xba, 0x98, 0x76, 0x54, 0x32, 0x10}, + []byte{0xfe, 0xdc, 0xba, 0x98, 0x76, 0x54, 0x32, 0x10}, + []byte{0xa9, 0x33, 0xf6, 0x18, 0x30, 0x23, 0xb3, 0x10}}, + { + []byte{0x01, 0x23, 0x45, 0x67, 0x89, 0xab, 0xcd, 0xef}, + []byte{0x11, 0x11, 0x11, 0x11, 0x11, 0x11, 0x11, 0x11}, + []byte{0x17, 0x66, 0x8d, 0xfc, 0x72, 0x92, 0x53, 0x2d}}, + { + []byte{0x01, 0x23, 0x45, 0x67, 0x89, 0xab, 0xcd, 0xef}, + []byte{0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01}, + []byte{0xb4, 0xfd, 0x23, 0x16, 0x47, 0xa5, 0xbe, 0xc0}}, + { + []byte{0x0e, 0x32, 0x92, 0x32, 0xea, 0x6d, 0x0d, 0x73}, + []byte{0x87, 0x87, 0x87, 0x87, 0x87, 0x87, 0x87, 0x87}, + []byte{0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00}}, + { + []byte{0x73, 0x65, 0x63, 0x52, 0x33, 0x74, 0x24, 0x3b}, // "secR3t$;" + []byte{0x61, 0x20, 0x74, 0x65, 0x73, 0x74, 0x31, 0x32}, // "a test12" + []byte{0x37, 0x0d, 0xee, 0x2c, 0x1f, 0xb4, 0xf7, 0xa5}}, + { + []byte{0x61, 0x62, 0x63, 0x64, 0x65, 0x66, 0x67, 0x68}, // "abcdefgh" + []byte{0x61, 0x62, 0x63, 0x64, 0x65, 0x66, 0x67, 0x68}, // "abcdefgh" + []byte{0x2a, 0x8d, 0x69, 0xde, 0x9d, 0x5f, 0xdf, 0xf9}}, + { + []byte{0x61, 0x62, 0x63, 0x64, 0x65, 0x66, 0x67, 0x68}, // "abcdefgh" + []byte{0x31, 0x32, 0x33, 0x34, 0x35, 0x36, 0x37, 0x38}, // "12345678" + []byte{0x21, 0xc6, 0x0d, 0xa5, 0x34, 0x24, 0x8b, 0xce}}, + { + []byte{0x31, 0x32, 0x33, 0x34, 0x35, 0x36, 0x37, 0x38}, // "12345678" + []byte{0x61, 0x62, 0x63, 0x64, 0x65, 0x66, 0x67, 0x68}, // "abcdefgh" + []byte{0x94, 0xd4, 0x43, 0x6b, 0xc3, 0xb5, 0xb6, 0x93}}, + { + []byte{0x1f, 0x79, 0x90, 0x5f, 0x88, 0x01, 0xc8, 0x88}, // random + []byte{0xc7, 0x46, 0x18, 0x73, 0xaf, 0x48, 0x5f, 0xb3}, // random + []byte{0xb0, 0x93, 0x50, 0x88, 0xf9, 0x92, 0x44, 0x6a}}, + { + []byte{0xe6, 0xf4, 0xf2, 0xdb, 0x31, 0x42, 0x53, 0x01}, // random + []byte{0xff, 0x3d, 0x25, 0x50, 0x12, 0xe3, 0x4a, 0xc5}, // random + []byte{0x86, 0x08, 0xd3, 0xd1, 0x6c, 0x2f, 0xd2, 0x55}}, + { + []byte{0x69, 0xc1, 0x9d, 0xc1, 0x15, 0xc5, 0xfb, 0x2b}, // random + []byte{0x1a, 0x22, 0x5c, 0xaf, 0x1f, 0x1d, 0xa3, 0xf9}, // random + []byte{0x64, 0xba, 0x31, 0x67, 0x56, 0x91, 0x1e, 0xa7}}, + { + []byte{0x6e, 0x5e, 0xe2, 0x47, 0xc4, 0xbf, 0xf6, 0x51}, // random + []byte{0x11, 0xc9, 0x57, 0xff, 0x66, 0x89, 0x0e, 0xf0}, // random + []byte{0x94, 0xc5, 0x35, 0xb2, 0xc5, 0x8b, 0x39, 0x72}}, +} + +var weakKeyTests = []CryptTest{ + { + []byte{0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01}, + []byte{0x55, 0x74, 0xc0, 0xbd, 0x7c, 0xdf, 0xf7, 0x39}, // random + nil}, + { + []byte{0xfe, 0xfe, 0xfe, 0xfe, 0xfe, 0xfe, 0xfe, 0xfe}, + []byte{0xe8, 0xe1, 0xa7, 0xc1, 0xde, 0x11, 0x89, 0xaa}, // random + nil}, + { + []byte{0xe0, 0xe0, 0xe0, 0xe0, 0xf1, 0xf1, 0xf1, 0xf1}, + []byte{0x50, 0x6a, 0x4b, 0x94, 0x3b, 0xed, 0x7d, 0xdc}, // random + nil}, + { + []byte{0x1f, 0x1f, 0x1f, 0x1f, 0x0e, 0x0e, 0x0e, 0x0e}, + []byte{0x88, 0x81, 0x56, 0x38, 0xec, 0x3b, 0x1c, 0x97}, // random + nil}, + { + []byte{0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00}, + []byte{0x17, 0xa0, 0x83, 0x62, 0x32, 0xfe, 0x9a, 0x0b}, // random + nil}, + { + []byte{0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff}, + []byte{0xca, 0x8f, 0xca, 0x1f, 0x50, 0xc5, 0x7b, 0x49}, // random + nil}, + { + []byte{0xe1, 0xe1, 0xe1, 0xe1, 0xf0, 0xf0, 0xf0, 0xf0}, + []byte{0xb1, 0xea, 0xad, 0x7d, 0xe7, 0xc3, 0x7a, 0x43}, // random + nil}, + { + []byte{0x1e, 0x1e, 0x1e, 0x1e, 0x0f, 0x0f, 0x0f, 0x0f}, + []byte{0xae, 0x74, 0x7d, 0x6f, 0xef, 0x16, 0xbb, 0x81}, // random + nil}, +} + +var semiWeakKeyTests = []CryptTest{ + // key and out contain the semi-weak key pair + { + []byte{0x01, 0x1f, 0x01, 0x1f, 0x01, 0x0e, 0x01, 0x0e}, + []byte{0x12, 0xfa, 0x31, 0x16, 0xf9, 0xc5, 0x0a, 0xe4}, // random + []byte{0x1f, 0x01, 0x1f, 0x01, 0x0e, 0x01, 0x0e, 0x01}}, + { + []byte{0x01, 0xe0, 0x01, 0xe0, 0x01, 0xf1, 0x01, 0xf1}, + []byte{0xb0, 0x4c, 0x7a, 0xee, 0xd2, 0xe5, 0x4d, 0xb7}, // random + []byte{0xe0, 0x01, 0xe0, 0x01, 0xf1, 0x01, 0xf1, 0x01}}, + { + []byte{0x01, 0xfe, 0x01, 0xfe, 0x01, 0xfe, 0x01, 0xfe}, + []byte{0xa4, 0x81, 0xcd, 0xb1, 0x64, 0x6f, 0xd3, 0xbc}, // random + []byte{0xfe, 0x01, 0xfe, 0x01, 0xfe, 0x01, 0xfe, 0x01}}, + { + []byte{0x1f, 0xe0, 0x1f, 0xe0, 0x0e, 0xf1, 0x0e, 0xf1}, + []byte{0xee, 0x27, 0xdd, 0x88, 0x4c, 0x22, 0xcd, 0xce}, // random + []byte{0xe0, 0x1f, 0xe0, 0x1f, 0xf1, 0x0e, 0xf1, 0x0e}}, + { + []byte{0x1f, 0xfe, 0x1f, 0xfe, 0x0e, 0xfe, 0x0e, 0xfe}, + []byte{0x19, 0x3d, 0xcf, 0x97, 0x70, 0xfb, 0xab, 0xe1}, // random + []byte{0xfe, 0x1f, 0xfe, 0x1f, 0xfe, 0x0e, 0xfe, 0x0e}}, + { + []byte{0xe0, 0xfe, 0xe0, 0xfe, 0xf1, 0xfe, 0xf1, 0xfe}, + []byte{0x7c, 0x82, 0x69, 0xe4, 0x1e, 0x86, 0x99, 0xd7}, // random + []byte{0xfe, 0xe0, 0xfe, 0xe0, 0xfe, 0xf1, 0xfe, 0xf1}}, +} + +// some custom tests for TripleDES +var encryptTripleDESTests = []CryptTest{ + { + []byte{ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00}, + []byte{0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00}, + []byte{0x92, 0x95, 0xb5, 0x9b, 0xb3, 0x84, 0x73, 0x6e}}, + { + []byte{ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00}, + []byte{0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff}, + []byte{0xc1, 0x97, 0xf5, 0x58, 0x74, 0x8a, 0x20, 0xe7}}, + { + []byte{ + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff}, + []byte{0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00}, + []byte{0x3e, 0x68, 0x0a, 0xa7, 0x8b, 0x75, 0xdf, 0x18}}, + { + []byte{ + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff}, + []byte{0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff}, + []byte{0x6d, 0x6a, 0x4a, 0x64, 0x4c, 0x7b, 0x8c, 0x91}}, + { + []byte{ // "abcdefgh12345678ABCDEFGH" + 0x61, 0x62, 0x63, 0x64, 0x65, 0x66, 0x67, 0x68, + 0x31, 0x32, 0x33, 0x34, 0x35, 0x36, 0x37, 0x38, + 0x41, 0x42, 0x43, 0x44, 0x45, 0x46, 0x47, 0x48}, + []byte{0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30}, // "00000000" + []byte{0xe4, 0x61, 0xb7, 0x59, 0x68, 0x8b, 0xff, 0x66}}, + { + []byte{ // "abcdefgh12345678ABCDEFGH" + 0x61, 0x62, 0x63, 0x64, 0x65, 0x66, 0x67, 0x68, + 0x31, 0x32, 0x33, 0x34, 0x35, 0x36, 0x37, 0x38, + 0x41, 0x42, 0x43, 0x44, 0x45, 0x46, 0x47, 0x48}, + []byte{0x31, 0x32, 0x33, 0x34, 0x35, 0x36, 0x37, 0x38}, // "12345678" + []byte{0xdb, 0xd0, 0x92, 0xde, 0xf8, 0x34, 0xff, 0x58}}, + { + []byte{ // "abcdefgh12345678ABCDEFGH" + 0x61, 0x62, 0x63, 0x64, 0x65, 0x66, 0x67, 0x68, + 0x31, 0x32, 0x33, 0x34, 0x35, 0x36, 0x37, 0x38, + 0x41, 0x42, 0x43, 0x44, 0x45, 0x46, 0x47, 0x48}, + []byte{0xf0, 0xc5, 0x82, 0x22, 0xd3, 0xe6, 0x12, 0xd2}, // random + []byte{0xba, 0xe4, 0x41, 0xb1, 0x3c, 0x37, 0x4d, 0xf4}}, + { + []byte{ // random + 0xd3, 0x7d, 0x45, 0xee, 0x22, 0xe9, 0xcf, 0x52, + 0xf4, 0x65, 0xa2, 0x4f, 0x70, 0xd1, 0x81, 0x8a, + 0x3d, 0xbe, 0x2f, 0x39, 0xc7, 0x71, 0xd2, 0xe9}, + []byte{0x49, 0x53, 0xc3, 0xe9, 0x78, 0xdf, 0x9f, 0xaf}, // random + []byte{0x53, 0x40, 0x51, 0x24, 0xd8, 0x3c, 0xf9, 0x88}}, + { + []byte{ // random + 0xcb, 0x10, 0x7d, 0xda, 0x7e, 0x96, 0x57, 0x0a, + 0xe8, 0xeb, 0xe8, 0x07, 0x8e, 0x87, 0xd3, 0x57, + 0xb2, 0x61, 0x12, 0xb8, 0x2a, 0x90, 0xb7, 0x2f}, + []byte{0xa3, 0xc2, 0x60, 0xb1, 0x0b, 0xb7, 0x28, 0x6e}, // random + []byte{0x56, 0x73, 0x7d, 0xfb, 0xb5, 0xa1, 0xc3, 0xde}}, +} + +// NIST Special Publication 800-20, Appendix A +// Key for use with Table A.1 tests +var tableA1Key = []byte{ + 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, + 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, + 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, +} + +// Table A.1 Resulting Ciphertext from the Variable Plaintext Known Answer Test +var tableA1Tests = []CryptTest{ + {nil, // 0 + []byte{0x80, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00}, + []byte{0x95, 0xf8, 0xa5, 0xe5, 0xdd, 0x31, 0xd9, 0x00}}, + {nil, // 1 + []byte{0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00}, + []byte{0xdd, 0x7f, 0x12, 0x1c, 0xa5, 0x01, 0x56, 0x19}}, + {nil, // 2 + []byte{0x20, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00}, + []byte{0x2e, 0x86, 0x53, 0x10, 0x4f, 0x38, 0x34, 0xea}}, + {nil, // 3 + []byte{0x10, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00}, + []byte{0x4b, 0xd3, 0x88, 0xff, 0x6c, 0xd8, 0x1d, 0x4f}}, + {nil, // 4 + []byte{0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00}, + []byte{0x20, 0xb9, 0xe7, 0x67, 0xb2, 0xfb, 0x14, 0x56}}, + {nil, // 5 + []byte{0x04, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00}, + []byte{0x55, 0x57, 0x93, 0x80, 0xd7, 0x71, 0x38, 0xef}}, + {nil, // 6 + []byte{0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00}, + []byte{0x6c, 0xc5, 0xde, 0xfa, 0xaf, 0x04, 0x51, 0x2f}}, + {nil, // 7 + []byte{0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00}, + []byte{0x0d, 0x9f, 0x27, 0x9b, 0xa5, 0xd8, 0x72, 0x60}}, + {nil, // 8 + []byte{0x00, 0x80, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00}, + []byte{0xd9, 0x03, 0x1b, 0x02, 0x71, 0xbd, 0x5a, 0x0a}}, + {nil, // 9 + []byte{0x00, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00}, + []byte{0x42, 0x42, 0x50, 0xb3, 0x7c, 0x3d, 0xd9, 0x51}}, + {nil, // 10 + []byte{0x00, 0x20, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00}, + []byte{0xb8, 0x06, 0x1b, 0x7e, 0xcd, 0x9a, 0x21, 0xe5}}, + {nil, // 11 + []byte{0x00, 0x10, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00}, + []byte{0xf1, 0x5d, 0x0f, 0x28, 0x6b, 0x65, 0xbd, 0x28}}, + {nil, // 12 + []byte{0x00, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00}, + []byte{0xad, 0xd0, 0xcc, 0x8d, 0x6e, 0x5d, 0xeb, 0xa1}}, + {nil, // 13 + []byte{0x00, 0x04, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00}, + []byte{0xe6, 0xd5, 0xf8, 0x27, 0x52, 0xad, 0x63, 0xd1}}, + {nil, // 14 + []byte{0x00, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00}, + []byte{0xec, 0xbf, 0xe3, 0xbd, 0x3f, 0x59, 0x1a, 0x5e}}, + {nil, // 15 + []byte{0x00, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00}, + []byte{0xf3, 0x56, 0x83, 0x43, 0x79, 0xd1, 0x65, 0xcd}}, + {nil, // 16 + []byte{0x00, 0x00, 0x80, 0x00, 0x00, 0x00, 0x00, 0x00}, + []byte{0x2b, 0x9f, 0x98, 0x2f, 0x20, 0x03, 0x7f, 0xa9}}, + {nil, // 17 + []byte{0x00, 0x00, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00}, + []byte{0x88, 0x9d, 0xe0, 0x68, 0xa1, 0x6f, 0x0b, 0xe6}}, + {nil, // 18 + []byte{0x00, 0x00, 0x20, 0x00, 0x00, 0x00, 0x00, 0x00}, + []byte{0xe1, 0x9e, 0x27, 0x5d, 0x84, 0x6a, 0x12, 0x98}}, + {nil, // 19 + []byte{0x00, 0x00, 0x10, 0x00, 0x00, 0x00, 0x00, 0x00}, + []byte{0x32, 0x9a, 0x8e, 0xd5, 0x23, 0xd7, 0x1a, 0xec}}, + {nil, // 20 + []byte{0x00, 0x00, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00}, + []byte{0xe7, 0xfc, 0xe2, 0x25, 0x57, 0xd2, 0x3c, 0x97}}, + {nil, // 21 + []byte{0x00, 0x00, 0x04, 0x00, 0x00, 0x00, 0x00, 0x00}, + []byte{0x12, 0xa9, 0xf5, 0x81, 0x7f, 0xf2, 0xd6, 0x5d}}, + {nil, // 22 + []byte{0x00, 0x00, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00}, + []byte{0xa4, 0x84, 0xc3, 0xad, 0x38, 0xdc, 0x9c, 0x19}}, + {nil, // 23 + []byte{0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00}, + []byte{0xfb, 0xe0, 0x0a, 0x8a, 0x1e, 0xf8, 0xad, 0x72}}, + {nil, // 24 + []byte{0x00, 0x00, 0x00, 0x80, 0x00, 0x00, 0x00, 0x00}, + []byte{0x75, 0x0d, 0x07, 0x94, 0x07, 0x52, 0x13, 0x63}}, + {nil, // 25 + []byte{0x00, 0x00, 0x00, 0x40, 0x00, 0x00, 0x00, 0x00}, + []byte{0x64, 0xfe, 0xed, 0x9c, 0x72, 0x4c, 0x2f, 0xaf}}, + {nil, // 26 + []byte{0x00, 0x00, 0x00, 0x20, 0x00, 0x00, 0x00, 0x00}, + []byte{0xf0, 0x2b, 0x26, 0x3b, 0x32, 0x8e, 0x2b, 0x60}}, + {nil, // 27 + []byte{0x00, 0x00, 0x00, 0x10, 0x00, 0x00, 0x00, 0x00}, + []byte{0x9d, 0x64, 0x55, 0x5a, 0x9a, 0x10, 0xb8, 0x52}}, + {nil, // 28 + []byte{0x00, 0x00, 0x00, 0x08, 0x00, 0x00, 0x00, 0x00}, + []byte{0xd1, 0x06, 0xff, 0x0b, 0xed, 0x52, 0x55, 0xd7}}, + {nil, // 29 + []byte{0x00, 0x00, 0x00, 0x04, 0x00, 0x00, 0x00, 0x00}, + []byte{0xe1, 0x65, 0x2c, 0x6b, 0x13, 0x8c, 0x64, 0xa5}}, + {nil, // 30 + []byte{0x00, 0x00, 0x00, 0x02, 0x00, 0x00, 0x00, 0x00}, + []byte{0xe4, 0x28, 0x58, 0x11, 0x86, 0xec, 0x8f, 0x46}}, + {nil, // 31 + []byte{0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x00}, + []byte{0xae, 0xb5, 0xf5, 0xed, 0xe2, 0x2d, 0x1a, 0x36}}, + {nil, // 32 + []byte{0x00, 0x00, 0x00, 0x00, 0x80, 0x00, 0x00, 0x00}, + []byte{0xe9, 0x43, 0xd7, 0x56, 0x8a, 0xec, 0x0c, 0x5c}}, + {nil, // 33 + []byte{0x00, 0x00, 0x00, 0x00, 0x40, 0x00, 0x00, 0x00}, + []byte{0xdf, 0x98, 0xc8, 0x27, 0x6f, 0x54, 0xb0, 0x4b}}, + {nil, // 34 + []byte{0x00, 0x00, 0x00, 0x00, 0x20, 0x00, 0x00, 0x00}, + []byte{0xb1, 0x60, 0xe4, 0x68, 0x0f, 0x6c, 0x69, 0x6f}}, + {nil, // 35 + []byte{0x00, 0x00, 0x00, 0x00, 0x10, 0x00, 0x00, 0x00}, + []byte{0xfa, 0x07, 0x52, 0xb0, 0x7d, 0x9c, 0x4a, 0xb8}}, + {nil, // 36 + []byte{0x00, 0x00, 0x00, 0x00, 0x08, 0x00, 0x00, 0x00}, + []byte{0xca, 0x3a, 0x2b, 0x03, 0x6d, 0xbc, 0x85, 0x02}}, + {nil, // 37 + []byte{0x00, 0x00, 0x00, 0x00, 0x04, 0x00, 0x00, 0x00}, + []byte{0x5e, 0x09, 0x05, 0x51, 0x7b, 0xb5, 0x9b, 0xcf}}, + {nil, // 38 + []byte{0x00, 0x00, 0x00, 0x00, 0x02, 0x00, 0x00, 0x00}, + []byte{0x81, 0x4e, 0xeb, 0x3b, 0x91, 0xd9, 0x07, 0x26}}, + {nil, // 39 + []byte{0x00, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00}, + []byte{0x4d, 0x49, 0xdb, 0x15, 0x32, 0x91, 0x9c, 0x9f}}, + {nil, // 40 + []byte{0x00, 0x00, 0x00, 0x00, 0x00, 0x80, 0x00, 0x00}, + []byte{0x25, 0xeb, 0x5f, 0xc3, 0xf8, 0xcf, 0x06, 0x21}}, + {nil, // 41 + []byte{0x00, 0x00, 0x00, 0x00, 0x00, 0x40, 0x00, 0x00}, + []byte{0xab, 0x6a, 0x20, 0xc0, 0x62, 0x0d, 0x1c, 0x6f}}, + {nil, // 42 + []byte{0x00, 0x00, 0x00, 0x00, 0x00, 0x20, 0x00, 0x00}, + []byte{0x79, 0xe9, 0x0d, 0xbc, 0x98, 0xf9, 0x2c, 0xca}}, + {nil, // 43 + []byte{0x00, 0x00, 0x00, 0x00, 0x00, 0x10, 0x00, 0x00}, + []byte{0x86, 0x6e, 0xce, 0xdd, 0x80, 0x72, 0xbb, 0x0e}}, + {nil, // 44 + []byte{0x00, 0x00, 0x00, 0x00, 0x00, 0x08, 0x00, 0x00}, + []byte{0x8b, 0x54, 0x53, 0x6f, 0x2f, 0x3e, 0x64, 0xa8}}, + {nil, // 45 + []byte{0x00, 0x00, 0x00, 0x00, 0x00, 0x04, 0x00, 0x00}, + []byte{0xea, 0x51, 0xd3, 0x97, 0x55, 0x95, 0xb8, 0x6b}}, + {nil, // 46 + []byte{0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0x00, 0x00}, + []byte{0xca, 0xff, 0xc6, 0xac, 0x45, 0x42, 0xde, 0x31}}, + {nil, // 47 + []byte{0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00}, + []byte{0x8d, 0xd4, 0x5a, 0x2d, 0xdf, 0x90, 0x79, 0x6c}}, + {nil, // 48 + []byte{0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x80, 0x00}, + []byte{0x10, 0x29, 0xd5, 0x5e, 0x88, 0x0e, 0xc2, 0xd0}}, + {nil, // 49 + []byte{0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x40, 0x00}, + []byte{0x5d, 0x86, 0xcb, 0x23, 0x63, 0x9d, 0xbe, 0xa9}}, + {nil, // 50 + []byte{0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x20, 0x00}, + []byte{0x1d, 0x1c, 0xa8, 0x53, 0xae, 0x7c, 0x0c, 0x5f}}, + {nil, // 51 + []byte{0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x10, 0x00}, + []byte{0xce, 0x33, 0x23, 0x29, 0x24, 0x8f, 0x32, 0x28}}, + {nil, // 52 + []byte{0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x08, 0x00}, + []byte{0x84, 0x05, 0xd1, 0xab, 0xe2, 0x4f, 0xb9, 0x42}}, + {nil, // 53 + []byte{0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x04, 0x00}, + []byte{0xe6, 0x43, 0xd7, 0x80, 0x90, 0xca, 0x42, 0x07}}, + {nil, // 54 + []byte{0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0x00}, + []byte{0x48, 0x22, 0x1b, 0x99, 0x37, 0x74, 0x8a, 0x23}}, + {nil, // 55 + []byte{0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0x00}, + []byte{0xdd, 0x7c, 0x0b, 0xbd, 0x61, 0xfa, 0xfd, 0x54}}, + {nil, // 56 + []byte{0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x80}, + []byte{0x2f, 0xbc, 0x29, 0x1a, 0x57, 0x0d, 0xb5, 0xc4}}, + {nil, // 57 + []byte{0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x40}, + []byte{0xe0, 0x7c, 0x30, 0xd7, 0xe4, 0xe2, 0x6e, 0x12}}, + {nil, // 58 + []byte{0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x20}, + []byte{0x09, 0x53, 0xe2, 0x25, 0x8e, 0x8e, 0x90, 0xa1}}, + {nil, // 59 + []byte{0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x10}, + []byte{0x5b, 0x71, 0x1b, 0xc4, 0xce, 0xeb, 0xf2, 0xee}}, + {nil, // 60 + []byte{0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x08}, + []byte{0xcc, 0x08, 0x3f, 0x1e, 0x6d, 0x9e, 0x85, 0xf6}}, + {nil, // 61 + []byte{0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x04}, + []byte{0xd2, 0xfd, 0x88, 0x67, 0xd5, 0x0d, 0x2d, 0xfe}}, + {nil, // 62 + []byte{0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02}, + []byte{0x06, 0xe7, 0xea, 0x22, 0xce, 0x92, 0x70, 0x8f}}, + {nil, // 63 + []byte{0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01}, + []byte{0x16, 0x6b, 0x40, 0xb4, 0x4a, 0xba, 0x4b, 0xd6}}, +} + +// Plaintext for use with Table A.2 tests +var tableA2Plaintext = []byte{0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00} + +// Table A.2 Resulting Ciphertext from the Variable Key Known Answer Test +var tableA2Tests = []CryptTest{ + { // 0 + []byte{ + 0x80, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, + 0x80, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, + 0x80, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01}, + nil, + []byte{0x95, 0xa8, 0xd7, 0x28, 0x13, 0xda, 0xa9, 0x4d}}, + { // 1 + []byte{ + 0x40, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, + 0x40, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, + 0x40, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01}, + nil, + []byte{0x0e, 0xec, 0x14, 0x87, 0xdd, 0x8c, 0x26, 0xd5}}, + { // 2 + []byte{ + 0x20, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, + 0x20, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, + 0x20, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01}, + nil, + []byte{0x7a, 0xd1, 0x6f, 0xfb, 0x79, 0xc4, 0x59, 0x26}}, + { // 3 + []byte{ + 0x10, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, + 0x10, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, + 0x10, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01}, + nil, + []byte{0xd3, 0x74, 0x62, 0x94, 0xca, 0x6a, 0x6c, 0xf3}}, + { // 4 + []byte{ + 0x08, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, + 0x08, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, + 0x08, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01}, + nil, + []byte{0x80, 0x9f, 0x5f, 0x87, 0x3c, 0x1f, 0xd7, 0x61}}, + { // 5 + []byte{ + 0x04, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, + 0x04, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, + 0x04, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01}, + nil, + []byte{0xc0, 0x2f, 0xaf, 0xfe, 0xc9, 0x89, 0xd1, 0xfc}}, + { // 6 + []byte{ + 0x02, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, + 0x02, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, + 0x02, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01}, + nil, + []byte{0x46, 0x15, 0xaa, 0x1d, 0x33, 0xe7, 0x2f, 0x10}}, + { // 7 + []byte{ + 0x01, 0x80, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, + 0x01, 0x80, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, + 0x01, 0x80, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01}, + nil, + []byte{0x20, 0x55, 0x12, 0x33, 0x50, 0xc0, 0x08, 0x58}}, + { // 8 + []byte{ + 0x01, 0x40, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, + 0x01, 0x40, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, + 0x01, 0x40, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01}, + nil, + []byte{0xdf, 0x3b, 0x99, 0xd6, 0x57, 0x73, 0x97, 0xc8}}, + { // 9 + []byte{ + 0x01, 0x20, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, + 0x01, 0x20, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, + 0x01, 0x20, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01}, + nil, + []byte{0x31, 0xfe, 0x17, 0x36, 0x9b, 0x52, 0x88, 0xc9}}, + { // 10 + []byte{ + 0x01, 0x10, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, + 0x01, 0x10, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, + 0x01, 0x10, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01}, + nil, + []byte{0xdf, 0xdd, 0x3c, 0xc6, 0x4d, 0xae, 0x16, 0x42}}, + { // 11 + []byte{ + 0x01, 0x08, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, + 0x01, 0x08, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, + 0x01, 0x08, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01}, + nil, + []byte{0x17, 0x8c, 0x83, 0xce, 0x2b, 0x39, 0x9d, 0x94}}, + { // 12 + []byte{ + 0x01, 0x04, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, + 0x01, 0x04, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, + 0x01, 0x04, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01}, + nil, + []byte{0x50, 0xf6, 0x36, 0x32, 0x4a, 0x9b, 0x7f, 0x80}}, + { // 13 + []byte{ + 0x01, 0x02, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, + 0x01, 0x02, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, + 0x01, 0x02, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01}, + nil, + []byte{0xa8, 0x46, 0x8e, 0xe3, 0xbc, 0x18, 0xf0, 0x6d}}, + { // 14 + []byte{ + 0x01, 0x01, 0x80, 0x01, 0x01, 0x01, 0x01, 0x01, + 0x01, 0x01, 0x80, 0x01, 0x01, 0x01, 0x01, 0x01, + 0x01, 0x01, 0x80, 0x01, 0x01, 0x01, 0x01, 0x01}, + nil, + []byte{0xa2, 0xdc, 0x9e, 0x92, 0xfd, 0x3c, 0xde, 0x92}}, + { // 15 + []byte{ + 0x01, 0x01, 0x40, 0x01, 0x01, 0x01, 0x01, 0x01, + 0x01, 0x01, 0x40, 0x01, 0x01, 0x01, 0x01, 0x01, + 0x01, 0x01, 0x40, 0x01, 0x01, 0x01, 0x01, 0x01}, + nil, + []byte{0xca, 0xc0, 0x9f, 0x79, 0x7d, 0x03, 0x12, 0x87}}, + { // 16 + []byte{ + 0x01, 0x01, 0x20, 0x01, 0x01, 0x01, 0x01, 0x01, + 0x01, 0x01, 0x20, 0x01, 0x01, 0x01, 0x01, 0x01, + 0x01, 0x01, 0x20, 0x01, 0x01, 0x01, 0x01, 0x01}, + nil, + []byte{0x90, 0xba, 0x68, 0x0b, 0x22, 0xae, 0xb5, 0x25}}, + { // 17 + []byte{ + 0x01, 0x01, 0x10, 0x01, 0x01, 0x01, 0x01, 0x01, + 0x01, 0x01, 0x10, 0x01, 0x01, 0x01, 0x01, 0x01, + 0x01, 0x01, 0x10, 0x01, 0x01, 0x01, 0x01, 0x01}, + nil, + []byte{0xce, 0x7a, 0x24, 0xf3, 0x50, 0xe2, 0x80, 0xb6}}, + { // 18 + []byte{ + 0x01, 0x01, 0x08, 0x01, 0x01, 0x01, 0x01, 0x01, + 0x01, 0x01, 0x08, 0x01, 0x01, 0x01, 0x01, 0x01, + 0x01, 0x01, 0x08, 0x01, 0x01, 0x01, 0x01, 0x01}, + nil, + []byte{0x88, 0x2b, 0xff, 0x0a, 0xa0, 0x1a, 0x0b, 0x87}}, + { // 19 + []byte{ + 0x01, 0x01, 0x04, 0x01, 0x01, 0x01, 0x01, 0x01, + 0x01, 0x01, 0x04, 0x01, 0x01, 0x01, 0x01, 0x01, + 0x01, 0x01, 0x04, 0x01, 0x01, 0x01, 0x01, 0x01}, + nil, + []byte{0x25, 0x61, 0x02, 0x88, 0x92, 0x45, 0x11, 0xc2}}, + { // 20 + []byte{ + 0x01, 0x01, 0x02, 0x01, 0x01, 0x01, 0x01, 0x01, + 0x01, 0x01, 0x02, 0x01, 0x01, 0x01, 0x01, 0x01, + 0x01, 0x01, 0x02, 0x01, 0x01, 0x01, 0x01, 0x01}, + nil, + []byte{0xc7, 0x15, 0x16, 0xc2, 0x9c, 0x75, 0xd1, 0x70}}, + { // 21 + []byte{ + 0x01, 0x01, 0x01, 0x80, 0x01, 0x01, 0x01, 0x01, + 0x01, 0x01, 0x01, 0x80, 0x01, 0x01, 0x01, 0x01, + 0x01, 0x01, 0x01, 0x80, 0x01, 0x01, 0x01, 0x01}, + nil, + []byte{0x51, 0x99, 0xc2, 0x9a, 0x52, 0xc9, 0xf0, 0x59}}, + { // 22 + []byte{ + 0x01, 0x01, 0x01, 0x40, 0x01, 0x01, 0x01, 0x01, + 0x01, 0x01, 0x01, 0x40, 0x01, 0x01, 0x01, 0x01, + 0x01, 0x01, 0x01, 0x40, 0x01, 0x01, 0x01, 0x01}, + nil, + []byte{0xc2, 0x2f, 0x0a, 0x29, 0x4a, 0x71, 0xf2, 0x9f}}, + { // 23 + []byte{ + 0x01, 0x01, 0x01, 0x20, 0x01, 0x01, 0x01, 0x01, + 0x01, 0x01, 0x01, 0x20, 0x01, 0x01, 0x01, 0x01, + 0x01, 0x01, 0x01, 0x20, 0x01, 0x01, 0x01, 0x01}, + nil, + []byte{0xee, 0x37, 0x14, 0x83, 0x71, 0x4c, 0x02, 0xea}}, + { // 24 + []byte{ + 0x01, 0x01, 0x01, 0x10, 0x01, 0x01, 0x01, 0x01, + 0x01, 0x01, 0x01, 0x10, 0x01, 0x01, 0x01, 0x01, + 0x01, 0x01, 0x01, 0x10, 0x01, 0x01, 0x01, 0x01}, + nil, + []byte{0xa8, 0x1f, 0xbd, 0x44, 0x8f, 0x9e, 0x52, 0x2f}}, + { // 25 + []byte{ + 0x01, 0x01, 0x01, 0x08, 0x01, 0x01, 0x01, 0x01, + 0x01, 0x01, 0x01, 0x08, 0x01, 0x01, 0x01, 0x01, + 0x01, 0x01, 0x01, 0x08, 0x01, 0x01, 0x01, 0x01}, + nil, + []byte{0x4f, 0x64, 0x4c, 0x92, 0xe1, 0x92, 0xdf, 0xed}}, + { // 26 + []byte{ + 0x01, 0x01, 0x01, 0x04, 0x01, 0x01, 0x01, 0x01, + 0x01, 0x01, 0x01, 0x04, 0x01, 0x01, 0x01, 0x01, + 0x01, 0x01, 0x01, 0x04, 0x01, 0x01, 0x01, 0x01}, + nil, + []byte{0x1a, 0xfa, 0x9a, 0x66, 0xa6, 0xdf, 0x92, 0xae}}, + { // 27 + []byte{ + 0x01, 0x01, 0x01, 0x02, 0x01, 0x01, 0x01, 0x01, + 0x01, 0x01, 0x01, 0x02, 0x01, 0x01, 0x01, 0x01, + 0x01, 0x01, 0x01, 0x02, 0x01, 0x01, 0x01, 0x01}, + nil, + []byte{0xb3, 0xc1, 0xcc, 0x71, 0x5c, 0xb8, 0x79, 0xd8}}, + { // 28 + []byte{ + 0x01, 0x01, 0x01, 0x01, 0x80, 0x01, 0x01, 0x01, + 0x01, 0x01, 0x01, 0x01, 0x80, 0x01, 0x01, 0x01, + 0x01, 0x01, 0x01, 0x01, 0x80, 0x01, 0x01, 0x01}, + nil, + []byte{0x19, 0xd0, 0x32, 0xe6, 0x4a, 0xb0, 0xbd, 0x8b}}, + { // 29 + []byte{ + 0x01, 0x01, 0x01, 0x01, 0x40, 0x01, 0x01, 0x01, + 0x01, 0x01, 0x01, 0x01, 0x40, 0x01, 0x01, 0x01, + 0x01, 0x01, 0x01, 0x01, 0x40, 0x01, 0x01, 0x01}, + nil, + []byte{0x3c, 0xfa, 0xa7, 0xa7, 0xdc, 0x87, 0x20, 0xdc}}, + { // 30 + []byte{ + 0x01, 0x01, 0x01, 0x01, 0x20, 0x01, 0x01, 0x01, + 0x01, 0x01, 0x01, 0x01, 0x20, 0x01, 0x01, 0x01, + 0x01, 0x01, 0x01, 0x01, 0x20, 0x01, 0x01, 0x01}, + nil, + []byte{0xb7, 0x26, 0x5f, 0x7f, 0x44, 0x7a, 0xc6, 0xf3}}, + { // 31 + []byte{ + 0x01, 0x01, 0x01, 0x01, 0x10, 0x01, 0x01, 0x01, + 0x01, 0x01, 0x01, 0x01, 0x10, 0x01, 0x01, 0x01, + 0x01, 0x01, 0x01, 0x01, 0x10, 0x01, 0x01, 0x01}, + nil, + []byte{0x9d, 0xb7, 0x3b, 0x3c, 0x0d, 0x16, 0x3f, 0x54}}, + { // 32 + []byte{ + 0x01, 0x01, 0x01, 0x01, 0x08, 0x01, 0x01, 0x01, + 0x01, 0x01, 0x01, 0x01, 0x08, 0x01, 0x01, 0x01, + 0x01, 0x01, 0x01, 0x01, 0x08, 0x01, 0x01, 0x01}, + nil, + []byte{0x81, 0x81, 0xb6, 0x5b, 0xab, 0xf4, 0xa9, 0x75}}, + { // 33 + []byte{ + 0x01, 0x01, 0x01, 0x01, 0x04, 0x01, 0x01, 0x01, + 0x01, 0x01, 0x01, 0x01, 0x04, 0x01, 0x01, 0x01, + 0x01, 0x01, 0x01, 0x01, 0x04, 0x01, 0x01, 0x01}, + nil, + []byte{0x93, 0xc9, 0xb6, 0x40, 0x42, 0xea, 0xa2, 0x40}}, + { // 34 + []byte{ + 0x01, 0x01, 0x01, 0x01, 0x02, 0x01, 0x01, 0x01, + 0x01, 0x01, 0x01, 0x01, 0x02, 0x01, 0x01, 0x01, + 0x01, 0x01, 0x01, 0x01, 0x02, 0x01, 0x01, 0x01}, + nil, + []byte{0x55, 0x70, 0x53, 0x08, 0x29, 0x70, 0x55, 0x92}}, + { // 35 + []byte{ + 0x01, 0x01, 0x01, 0x01, 0x01, 0x80, 0x01, 0x01, + 0x01, 0x01, 0x01, 0x01, 0x01, 0x80, 0x01, 0x01, + 0x01, 0x01, 0x01, 0x01, 0x01, 0x80, 0x01, 0x01}, + nil, + []byte{0x86, 0x38, 0x80, 0x9e, 0x87, 0x87, 0x87, 0xa0}}, + { // 36 + []byte{ + 0x01, 0x01, 0x01, 0x01, 0x01, 0x40, 0x01, 0x01, + 0x01, 0x01, 0x01, 0x01, 0x01, 0x40, 0x01, 0x01, + 0x01, 0x01, 0x01, 0x01, 0x01, 0x40, 0x01, 0x01}, + nil, + []byte{0x41, 0xb9, 0xa7, 0x9a, 0xf7, 0x9a, 0xc2, 0x08}}, + { // 37 + []byte{ + 0x01, 0x01, 0x01, 0x01, 0x01, 0x20, 0x01, 0x01, + 0x01, 0x01, 0x01, 0x01, 0x01, 0x20, 0x01, 0x01, + 0x01, 0x01, 0x01, 0x01, 0x01, 0x20, 0x01, 0x01}, + nil, + []byte{0x7a, 0x9b, 0xe4, 0x2f, 0x20, 0x09, 0xa8, 0x92}}, + { // 38 + []byte{ + 0x01, 0x01, 0x01, 0x01, 0x01, 0x10, 0x01, 0x01, + 0x01, 0x01, 0x01, 0x01, 0x01, 0x10, 0x01, 0x01, + 0x01, 0x01, 0x01, 0x01, 0x01, 0x10, 0x01, 0x01}, + nil, + []byte{0x29, 0x03, 0x8d, 0x56, 0xba, 0x6d, 0x27, 0x45}}, + { // 39 + []byte{ + 0x01, 0x01, 0x01, 0x01, 0x01, 0x08, 0x01, 0x01, + 0x01, 0x01, 0x01, 0x01, 0x01, 0x08, 0x01, 0x01, + 0x01, 0x01, 0x01, 0x01, 0x01, 0x08, 0x01, 0x01}, + nil, + []byte{0x54, 0x95, 0xc6, 0xab, 0xf1, 0xe5, 0xdf, 0x51}}, + { // 40 + []byte{ + 0x01, 0x01, 0x01, 0x01, 0x01, 0x04, 0x01, 0x01, + 0x01, 0x01, 0x01, 0x01, 0x01, 0x04, 0x01, 0x01, + 0x01, 0x01, 0x01, 0x01, 0x01, 0x04, 0x01, 0x01}, + nil, + []byte{0xae, 0x13, 0xdb, 0xd5, 0x61, 0x48, 0x89, 0x33}}, + { // 41 + []byte{ + 0x01, 0x01, 0x01, 0x01, 0x01, 0x02, 0x01, 0x01, + 0x01, 0x01, 0x01, 0x01, 0x01, 0x02, 0x01, 0x01, + 0x01, 0x01, 0x01, 0x01, 0x01, 0x02, 0x01, 0x01}, + nil, + []byte{0x02, 0x4d, 0x1f, 0xfa, 0x89, 0x04, 0xe3, 0x89}}, + { // 42 + []byte{ + 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x80, 0x01, + 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x80, 0x01, + 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x80, 0x01}, + nil, + []byte{0xd1, 0x39, 0x97, 0x12, 0xf9, 0x9b, 0xf0, 0x2e}}, + { // 43 + []byte{ + 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x40, 0x01, + 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x40, 0x01, + 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x40, 0x01}, + nil, + []byte{0x14, 0xc1, 0xd7, 0xc1, 0xcf, 0xfe, 0xc7, 0x9e}}, + { // 44 + []byte{ + 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x20, 0x01, + 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x20, 0x01, + 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x20, 0x01}, + nil, + []byte{0x1d, 0xe5, 0x27, 0x9d, 0xae, 0x3b, 0xed, 0x6f}}, + { // 45 + []byte{ + 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x10, 0x01, + 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x10, 0x01, + 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x10, 0x01}, + nil, + []byte{0xe9, 0x41, 0xa3, 0x3f, 0x85, 0x50, 0x13, 0x03}}, + { // 46 + []byte{ + 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x08, 0x01, + 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x08, 0x01, + 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x08, 0x01}, + nil, + []byte{0xda, 0x99, 0xdb, 0xbc, 0x9a, 0x03, 0xf3, 0x79}}, + { // 47 + []byte{ + 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x04, 0x01, + 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x04, 0x01, + 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x04, 0x01}, + nil, + []byte{0xb7, 0xfc, 0x92, 0xf9, 0x1d, 0x8e, 0x92, 0xe9}}, + { // 48 + []byte{ + 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x02, 0x01, + 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x02, 0x01, + 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x02, 0x01}, + nil, + []byte{0xae, 0x8e, 0x5c, 0xaa, 0x3c, 0xa0, 0x4e, 0x85}}, + { // 49 + []byte{ + 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x80, + 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x80, + 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x80}, + nil, + []byte{0x9c, 0xc6, 0x2d, 0xf4, 0x3b, 0x6e, 0xed, 0x74}}, + { // 50 + []byte{ + 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x40, + 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x40, + 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x40}, + nil, + []byte{0xd8, 0x63, 0xdb, 0xb5, 0xc5, 0x9a, 0x91, 0xa0}}, + { // 50 + []byte{ + 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x20, + 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x20, + 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x20}, + nil, + []byte{0xa1, 0xab, 0x21, 0x90, 0x54, 0x5b, 0x91, 0xd7}}, + { // 52 + []byte{ + 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x10, + 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x10, + 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x10}, + nil, + []byte{0x08, 0x75, 0x04, 0x1e, 0x64, 0xc5, 0x70, 0xf7}}, + { // 53 + []byte{ + 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x08, + 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x08, + 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x08}, + nil, + []byte{0x5a, 0x59, 0x45, 0x28, 0xbe, 0xbe, 0xf1, 0xcc}}, + { // 54 + []byte{ + 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x04, + 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x04, + 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x04}, + nil, + []byte{0xfc, 0xdb, 0x32, 0x91, 0xde, 0x21, 0xf0, 0xc0}}, + { // 55 + []byte{ + 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x02, + 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x02, + 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x02}, + nil, + []byte{0x86, 0x9e, 0xfd, 0x7f, 0x9f, 0x26, 0x5a, 0x09}}, +} + +// Plaintext for use with Table A.3 tests +var tableA3Plaintext = []byte{0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00} + +// Table A.3 Values To Be Used for the Permutation Operation Known Answer Test +var tableA3Tests = []CryptTest{ + { // 0 + []byte{ + 0x10, 0x46, 0x91, 0x34, 0x89, 0x98, 0x01, 0x31, + 0x10, 0x46, 0x91, 0x34, 0x89, 0x98, 0x01, 0x31, + 0x10, 0x46, 0x91, 0x34, 0x89, 0x98, 0x01, 0x31, + }, + nil, + []byte{0x88, 0xd5, 0x5e, 0x54, 0xf5, 0x4c, 0x97, 0xb4}}, + { // 1 + []byte{ + 0x10, 0x07, 0x10, 0x34, 0x89, 0x98, 0x80, 0x20, + 0x10, 0x07, 0x10, 0x34, 0x89, 0x98, 0x80, 0x20, + 0x10, 0x07, 0x10, 0x34, 0x89, 0x98, 0x80, 0x20, + }, + nil, + []byte{0x0c, 0x0c, 0xc0, 0x0c, 0x83, 0xea, 0x48, 0xfd}}, + { // 2 + []byte{ + 0x10, 0x07, 0x10, 0x34, 0xc8, 0x98, 0x01, 0x20, + 0x10, 0x07, 0x10, 0x34, 0xc8, 0x98, 0x01, 0x20, + 0x10, 0x07, 0x10, 0x34, 0xc8, 0x98, 0x01, 0x20, + }, + nil, + []byte{0x83, 0xbc, 0x8e, 0xf3, 0xa6, 0x57, 0x01, 0x83}}, + { // 3 + []byte{ + 0x10, 0x46, 0x10, 0x34, 0x89, 0x98, 0x80, 0x20, + 0x10, 0x46, 0x10, 0x34, 0x89, 0x98, 0x80, 0x20, + 0x10, 0x46, 0x10, 0x34, 0x89, 0x98, 0x80, 0x20, + }, + nil, + []byte{0xdf, 0x72, 0x5d, 0xca, 0xd9, 0x4e, 0xa2, 0xe9}}, + { // 4 + []byte{ + 0x10, 0x86, 0x91, 0x15, 0x19, 0x19, 0x01, 0x01, + 0x10, 0x86, 0x91, 0x15, 0x19, 0x19, 0x01, 0x01, + 0x10, 0x86, 0x91, 0x15, 0x19, 0x19, 0x01, 0x01, + }, + nil, + []byte{0xe6, 0x52, 0xb5, 0x3b, 0x55, 0x0b, 0xe8, 0xb0}}, + { // 5 + []byte{ + 0x10, 0x86, 0x91, 0x15, 0x19, 0x58, 0x01, 0x01, + 0x10, 0x86, 0x91, 0x15, 0x19, 0x58, 0x01, 0x01, + 0x10, 0x86, 0x91, 0x15, 0x19, 0x58, 0x01, 0x01, + }, + nil, + []byte{0xaf, 0x52, 0x71, 0x20, 0xc4, 0x85, 0xcb, 0xb0}}, + { // 6 + []byte{ + 0x51, 0x07, 0xb0, 0x15, 0x19, 0x58, 0x01, 0x01, + 0x51, 0x07, 0xb0, 0x15, 0x19, 0x58, 0x01, 0x01, + 0x51, 0x07, 0xb0, 0x15, 0x19, 0x58, 0x01, 0x01, + }, + nil, + []byte{0x0f, 0x04, 0xce, 0x39, 0x3d, 0xb9, 0x26, 0xd5}}, + { // 7 + []byte{ + 0x10, 0x07, 0xb0, 0x15, 0x19, 0x19, 0x01, 0x01, + 0x10, 0x07, 0xb0, 0x15, 0x19, 0x19, 0x01, 0x01, + 0x10, 0x07, 0xb0, 0x15, 0x19, 0x19, 0x01, 0x01, + }, + nil, + []byte{0xc9, 0xf0, 0x0f, 0xfc, 0x74, 0x07, 0x90, 0x67}}, + { // 8 + []byte{ + 0x31, 0x07, 0x91, 0x54, 0x98, 0x08, 0x01, 0x01, + 0x31, 0x07, 0x91, 0x54, 0x98, 0x08, 0x01, 0x01, + 0x31, 0x07, 0x91, 0x54, 0x98, 0x08, 0x01, 0x01, + }, + nil, + []byte{0x7c, 0xfd, 0x82, 0xa5, 0x93, 0x25, 0x2b, 0x4e}}, + { // 9 + []byte{ + 0x31, 0x07, 0x91, 0x94, 0x98, 0x08, 0x01, 0x01, + 0x31, 0x07, 0x91, 0x94, 0x98, 0x08, 0x01, 0x01, + 0x31, 0x07, 0x91, 0x94, 0x98, 0x08, 0x01, 0x01, + }, + nil, + []byte{0xcb, 0x49, 0xa2, 0xf9, 0xe9, 0x13, 0x63, 0xe3}}, + { // 10 + []byte{ + 0x10, 0x07, 0x91, 0x15, 0xb9, 0x08, 0x01, 0x40, + 0x10, 0x07, 0x91, 0x15, 0xb9, 0x08, 0x01, 0x40, + 0x10, 0x07, 0x91, 0x15, 0xb9, 0x08, 0x01, 0x40, + }, + nil, + []byte{0x00, 0xb5, 0x88, 0xbe, 0x70, 0xd2, 0x3f, 0x56}}, + { // 11 + []byte{ + 0x31, 0x07, 0x91, 0x15, 0x98, 0x08, 0x01, 0x40, + 0x31, 0x07, 0x91, 0x15, 0x98, 0x08, 0x01, 0x40, + 0x31, 0x07, 0x91, 0x15, 0x98, 0x08, 0x01, 0x40, + }, + nil, + []byte{0x40, 0x6a, 0x9a, 0x6a, 0xb4, 0x33, 0x99, 0xae}}, + { // 12 + []byte{ + 0x10, 0x07, 0xd0, 0x15, 0x89, 0x98, 0x01, 0x01, + 0x10, 0x07, 0xd0, 0x15, 0x89, 0x98, 0x01, 0x01, + 0x10, 0x07, 0xd0, 0x15, 0x89, 0x98, 0x01, 0x01, + }, + nil, + []byte{0x6c, 0xb7, 0x73, 0x61, 0x1d, 0xca, 0x9a, 0xda}}, + { // 13 + []byte{ + 0x91, 0x07, 0x91, 0x15, 0x89, 0x98, 0x01, 0x01, + 0x91, 0x07, 0x91, 0x15, 0x89, 0x98, 0x01, 0x01, + 0x91, 0x07, 0x91, 0x15, 0x89, 0x98, 0x01, 0x01, + }, + nil, + []byte{0x67, 0xfd, 0x21, 0xc1, 0x7d, 0xbb, 0x5d, 0x70}}, + { // 14 + []byte{ + 0x91, 0x07, 0xd0, 0x15, 0x89, 0x19, 0x01, 0x01, + 0x91, 0x07, 0xd0, 0x15, 0x89, 0x19, 0x01, 0x01, + 0x91, 0x07, 0xd0, 0x15, 0x89, 0x19, 0x01, 0x01, + }, + nil, + []byte{0x95, 0x92, 0xcb, 0x41, 0x10, 0x43, 0x07, 0x87}}, + { // 15 + []byte{ + 0x10, 0x07, 0xd0, 0x15, 0x98, 0x98, 0x01, 0x20, + 0x10, 0x07, 0xd0, 0x15, 0x98, 0x98, 0x01, 0x20, + 0x10, 0x07, 0xd0, 0x15, 0x98, 0x98, 0x01, 0x20, + }, + nil, + []byte{0xa6, 0xb7, 0xff, 0x68, 0xa3, 0x18, 0xdd, 0xd3}}, + { // 16 + []byte{ + 0x10, 0x07, 0x94, 0x04, 0x98, 0x19, 0x01, 0x01, + 0x10, 0x07, 0x94, 0x04, 0x98, 0x19, 0x01, 0x01, + 0x10, 0x07, 0x94, 0x04, 0x98, 0x19, 0x01, 0x01, + }, + nil, + []byte{0x4d, 0x10, 0x21, 0x96, 0xc9, 0x14, 0xca, 0x16}}, + { // 17 + []byte{ + 0x01, 0x07, 0x91, 0x04, 0x91, 0x19, 0x04, 0x01, + 0x01, 0x07, 0x91, 0x04, 0x91, 0x19, 0x04, 0x01, + 0x01, 0x07, 0x91, 0x04, 0x91, 0x19, 0x04, 0x01, + }, + nil, + []byte{0x2d, 0xfa, 0x9f, 0x45, 0x73, 0x59, 0x49, 0x65}}, + { // 18 + []byte{ + 0x01, 0x07, 0x91, 0x04, 0x91, 0x19, 0x01, 0x01, + 0x01, 0x07, 0x91, 0x04, 0x91, 0x19, 0x01, 0x01, + 0x01, 0x07, 0x91, 0x04, 0x91, 0x19, 0x01, 0x01, + }, + nil, + []byte{0xb4, 0x66, 0x04, 0x81, 0x6c, 0x0e, 0x07, 0x74}}, + { // 19 + []byte{ + 0x01, 0x07, 0x94, 0x04, 0x91, 0x19, 0x04, 0x01, + 0x01, 0x07, 0x94, 0x04, 0x91, 0x19, 0x04, 0x01, + 0x01, 0x07, 0x94, 0x04, 0x91, 0x19, 0x04, 0x01, + }, + nil, + []byte{0x6e, 0x7e, 0x62, 0x21, 0xa4, 0xf3, 0x4e, 0x87}}, + { // 20 + []byte{ + 0x19, 0x07, 0x92, 0x10, 0x98, 0x1a, 0x01, 0x01, + 0x19, 0x07, 0x92, 0x10, 0x98, 0x1a, 0x01, 0x01, + 0x19, 0x07, 0x92, 0x10, 0x98, 0x1a, 0x01, 0x01, + }, + nil, + []byte{0xaa, 0x85, 0xe7, 0x46, 0x43, 0x23, 0x31, 0x99}}, + { // 21 + []byte{ + 0x10, 0x07, 0x91, 0x19, 0x98, 0x19, 0x08, 0x01, + 0x10, 0x07, 0x91, 0x19, 0x98, 0x19, 0x08, 0x01, + 0x10, 0x07, 0x91, 0x19, 0x98, 0x19, 0x08, 0x01, + }, + nil, + []byte{0x2e, 0x5a, 0x19, 0xdb, 0x4d, 0x19, 0x62, 0xd6}}, + { // 22 + []byte{ + 0x10, 0x07, 0x91, 0x19, 0x98, 0x1a, 0x08, 0x01, + 0x10, 0x07, 0x91, 0x19, 0x98, 0x1a, 0x08, 0x01, + 0x10, 0x07, 0x91, 0x19, 0x98, 0x1a, 0x08, 0x01, + }, + nil, + []byte{0x23, 0xa8, 0x66, 0xa8, 0x09, 0xd3, 0x08, 0x94}}, + { // 23 + []byte{ + 0x10, 0x07, 0x92, 0x10, 0x98, 0x19, 0x01, 0x01, + 0x10, 0x07, 0x92, 0x10, 0x98, 0x19, 0x01, 0x01, + 0x10, 0x07, 0x92, 0x10, 0x98, 0x19, 0x01, 0x01, + }, + nil, + []byte{0xd8, 0x12, 0xd9, 0x61, 0xf0, 0x17, 0xd3, 0x20}}, + { // 24 + []byte{ + 0x10, 0x07, 0x91, 0x15, 0x98, 0x19, 0x01, 0x0b, + 0x10, 0x07, 0x91, 0x15, 0x98, 0x19, 0x01, 0x0b, + 0x10, 0x07, 0x91, 0x15, 0x98, 0x19, 0x01, 0x0b, + }, + nil, + []byte{0x05, 0x56, 0x05, 0x81, 0x6e, 0x58, 0x60, 0x8f}}, + { // 25 + []byte{ + 0x10, 0x04, 0x80, 0x15, 0x98, 0x19, 0x01, 0x01, + 0x10, 0x04, 0x80, 0x15, 0x98, 0x19, 0x01, 0x01, + 0x10, 0x04, 0x80, 0x15, 0x98, 0x19, 0x01, 0x01, + }, + nil, + []byte{0xab, 0xd8, 0x8e, 0x8b, 0x1b, 0x77, 0x16, 0xf1}}, + { // 26 + []byte{ + 0x10, 0x04, 0x80, 0x15, 0x98, 0x19, 0x01, 0x02, + 0x10, 0x04, 0x80, 0x15, 0x98, 0x19, 0x01, 0x02, + 0x10, 0x04, 0x80, 0x15, 0x98, 0x19, 0x01, 0x02, + }, + nil, + []byte{0x53, 0x7a, 0xc9, 0x5b, 0xe6, 0x9d, 0xa1, 0xe1}}, + { // 27 + []byte{ + 0x10, 0x04, 0x80, 0x15, 0x98, 0x19, 0x01, 0x08, + 0x10, 0x04, 0x80, 0x15, 0x98, 0x19, 0x01, 0x08, + 0x10, 0x04, 0x80, 0x15, 0x98, 0x19, 0x01, 0x08, + }, + nil, + []byte{0xae, 0xd0, 0xf6, 0xae, 0x3c, 0x25, 0xcd, 0xd8}}, + { // 28 + []byte{ + 0x10, 0x02, 0x91, 0x15, 0x98, 0x10, 0x01, 0x04, + 0x10, 0x02, 0x91, 0x15, 0x98, 0x10, 0x01, 0x04, + 0x10, 0x02, 0x91, 0x15, 0x98, 0x10, 0x01, 0x04, + }, + nil, + []byte{0xb3, 0xe3, 0x5a, 0x5e, 0xe5, 0x3e, 0x7b, 0x8d}}, + { // 29 + []byte{ + 0x10, 0x02, 0x91, 0x15, 0x98, 0x19, 0x01, 0x04, + 0x10, 0x02, 0x91, 0x15, 0x98, 0x19, 0x01, 0x04, + 0x10, 0x02, 0x91, 0x15, 0x98, 0x19, 0x01, 0x04, + }, + nil, + []byte{0x61, 0xc7, 0x9c, 0x71, 0x92, 0x1a, 0x2e, 0xf8}}, + { // 30 + []byte{ + 0x10, 0x02, 0x91, 0x15, 0x98, 0x10, 0x02, 0x01, + 0x10, 0x02, 0x91, 0x15, 0x98, 0x10, 0x02, 0x01, + 0x10, 0x02, 0x91, 0x15, 0x98, 0x10, 0x02, 0x01, + }, + nil, + []byte{0xe2, 0xf5, 0x72, 0x8f, 0x09, 0x95, 0x01, 0x3c}}, + { // 31 + []byte{ + 0x10, 0x02, 0x91, 0x16, 0x98, 0x10, 0x01, 0x01, + 0x10, 0x02, 0x91, 0x16, 0x98, 0x10, 0x01, 0x01, + 0x10, 0x02, 0x91, 0x16, 0x98, 0x10, 0x01, 0x01, + }, + nil, + []byte{0x1a, 0xea, 0xc3, 0x9a, 0x61, 0xf0, 0xa4, 0x64}}, +} + +// Table A.4 Values To Be Used for the Substitution Table Known Answer Test +var tableA4Tests = []CryptTest{ + { // 0 + []byte{ + 0x7c, 0xa1, 0x10, 0x45, 0x4a, 0x1a, 0x6e, 0x57, + 0x7c, 0xa1, 0x10, 0x45, 0x4a, 0x1a, 0x6e, 0x57, + 0x7c, 0xa1, 0x10, 0x45, 0x4a, 0x1a, 0x6e, 0x57}, + []byte{0x01, 0xa1, 0xd6, 0xd0, 0x39, 0x77, 0x67, 0x42}, + []byte{0x69, 0x0f, 0x5b, 0x0d, 0x9a, 0x26, 0x93, 0x9b}}, + { // 1 + []byte{ + 0x01, 0x31, 0xd9, 0x61, 0x9d, 0xc1, 0x37, 0x6e, + 0x01, 0x31, 0xd9, 0x61, 0x9d, 0xc1, 0x37, 0x6e, + 0x01, 0x31, 0xd9, 0x61, 0x9d, 0xc1, 0x37, 0x6e}, + []byte{0x5c, 0xd5, 0x4c, 0xa8, 0x3d, 0xef, 0x57, 0xda}, + []byte{0x7a, 0x38, 0x9d, 0x10, 0x35, 0x4b, 0xd2, 0x71}}, + { // 2 + []byte{ + 0x07, 0xa1, 0x13, 0x3e, 0x4a, 0x0b, 0x26, 0x86, + 0x07, 0xa1, 0x13, 0x3e, 0x4a, 0x0b, 0x26, 0x86, + 0x07, 0xa1, 0x13, 0x3e, 0x4a, 0x0b, 0x26, 0x86}, + []byte{0x02, 0x48, 0xd4, 0x38, 0x06, 0xf6, 0x71, 0x72}, + []byte{0x86, 0x8e, 0xbb, 0x51, 0xca, 0xb4, 0x59, 0x9a}}, + { // 3 + []byte{ + 0x38, 0x49, 0x67, 0x4c, 0x26, 0x02, 0x31, 0x9e, + 0x38, 0x49, 0x67, 0x4c, 0x26, 0x02, 0x31, 0x9e, + 0x38, 0x49, 0x67, 0x4c, 0x26, 0x02, 0x31, 0x9e}, + []byte{0x51, 0x45, 0x4b, 0x58, 0x2d, 0xdf, 0x44, 0x0a}, + []byte{0x71, 0x78, 0x87, 0x6e, 0x01, 0xf1, 0x9b, 0x2a}}, + { // 4 + []byte{ + 0x04, 0xb9, 0x15, 0xba, 0x43, 0xfe, 0xb5, 0xb6, + 0x04, 0xb9, 0x15, 0xba, 0x43, 0xfe, 0xb5, 0xb6, + 0x04, 0xb9, 0x15, 0xba, 0x43, 0xfe, 0xb5, 0xb6}, + []byte{0x42, 0xfd, 0x44, 0x30, 0x59, 0x57, 0x7f, 0xa2}, + []byte{0xaf, 0x37, 0xfb, 0x42, 0x1f, 0x8c, 0x40, 0x95}}, + { // 5 + []byte{ + 0x01, 0x13, 0xb9, 0x70, 0xfd, 0x34, 0xf2, 0xce, + 0x01, 0x13, 0xb9, 0x70, 0xfd, 0x34, 0xf2, 0xce, + 0x01, 0x13, 0xb9, 0x70, 0xfd, 0x34, 0xf2, 0xce}, + []byte{0x05, 0x9b, 0x5e, 0x08, 0x51, 0xcf, 0x14, 0x3a}, + []byte{0x86, 0xa5, 0x60, 0xf1, 0x0e, 0xc6, 0xd8, 0x5b}}, + { // 6 + []byte{ + 0x01, 0x70, 0xf1, 0x75, 0x46, 0x8f, 0xb5, 0xe6, + 0x01, 0x70, 0xf1, 0x75, 0x46, 0x8f, 0xb5, 0xe6, + 0x01, 0x70, 0xf1, 0x75, 0x46, 0x8f, 0xb5, 0xe6}, + []byte{0x07, 0x56, 0xd8, 0xe0, 0x77, 0x47, 0x61, 0xd2}, + []byte{0x0c, 0xd3, 0xda, 0x02, 0x00, 0x21, 0xdc, 0x09}}, + { // 7 + []byte{ + 0x43, 0x29, 0x7f, 0xad, 0x38, 0xe3, 0x73, 0xfe, + 0x43, 0x29, 0x7f, 0xad, 0x38, 0xe3, 0x73, 0xfe, + 0x43, 0x29, 0x7f, 0xad, 0x38, 0xe3, 0x73, 0xfe}, + []byte{0x76, 0x25, 0x14, 0xb8, 0x29, 0xbf, 0x48, 0x6a}, + []byte{0xea, 0x67, 0x6b, 0x2c, 0xb7, 0xdb, 0x2b, 0x7a}}, + { // 8 + []byte{ + 0x07, 0xa7, 0x13, 0x70, 0x45, 0xda, 0x2a, 0x16, + 0x07, 0xa7, 0x13, 0x70, 0x45, 0xda, 0x2a, 0x16, + 0x07, 0xa7, 0x13, 0x70, 0x45, 0xda, 0x2a, 0x16}, + []byte{0x3b, 0xdd, 0x11, 0x90, 0x49, 0x37, 0x28, 0x02}, + []byte{0xdf, 0xd6, 0x4a, 0x81, 0x5c, 0xaf, 0x1a, 0x0f}}, + { // 9 + []byte{ + 0x04, 0x68, 0x91, 0x04, 0xc2, 0xfd, 0x3b, 0x2f, + 0x04, 0x68, 0x91, 0x04, 0xc2, 0xfd, 0x3b, 0x2f, + 0x04, 0x68, 0x91, 0x04, 0xc2, 0xfd, 0x3b, 0x2f}, + []byte{0x26, 0x95, 0x5f, 0x68, 0x35, 0xaf, 0x60, 0x9a}, + []byte{0x5c, 0x51, 0x3c, 0x9c, 0x48, 0x86, 0xc0, 0x88}}, + { // 10 + []byte{ + 0x37, 0xd0, 0x6b, 0xb5, 0x16, 0xcb, 0x75, 0x46, + 0x37, 0xd0, 0x6b, 0xb5, 0x16, 0xcb, 0x75, 0x46, + 0x37, 0xd0, 0x6b, 0xb5, 0x16, 0xcb, 0x75, 0x46}, + []byte{0x16, 0x4d, 0x5e, 0x40, 0x4f, 0x27, 0x52, 0x32}, + []byte{0x0a, 0x2a, 0xee, 0xae, 0x3f, 0xf4, 0xab, 0x77}}, + { // 11 + []byte{ + 0x1f, 0x08, 0x26, 0x0d, 0x1a, 0xc2, 0x46, 0x5e, + 0x1f, 0x08, 0x26, 0x0d, 0x1a, 0xc2, 0x46, 0x5e, + 0x1f, 0x08, 0x26, 0x0d, 0x1a, 0xc2, 0x46, 0x5e}, + []byte{0x6b, 0x05, 0x6e, 0x18, 0x75, 0x9f, 0x5c, 0xca}, + []byte{0xef, 0x1b, 0xf0, 0x3e, 0x5d, 0xfa, 0x57, 0x5a}}, + { // 12 + []byte{ + 0x58, 0x40, 0x23, 0x64, 0x1a, 0xba, 0x61, 0x76, + 0x58, 0x40, 0x23, 0x64, 0x1a, 0xba, 0x61, 0x76, + 0x58, 0x40, 0x23, 0x64, 0x1a, 0xba, 0x61, 0x76}, + []byte{0x00, 0x4b, 0xd6, 0xef, 0x09, 0x17, 0x60, 0x62}, + []byte{0x88, 0xbf, 0x0d, 0xb6, 0xd7, 0x0d, 0xee, 0x56}}, + { // 13 + []byte{ + 0x02, 0x58, 0x16, 0x16, 0x46, 0x29, 0xb0, 0x07, + 0x02, 0x58, 0x16, 0x16, 0x46, 0x29, 0xb0, 0x07, + 0x02, 0x58, 0x16, 0x16, 0x46, 0x29, 0xb0, 0x07}, + []byte{0x48, 0x0d, 0x39, 0x00, 0x6e, 0xe7, 0x62, 0xf2}, + []byte{0xa1, 0xf9, 0x91, 0x55, 0x41, 0x02, 0x0b, 0x56}}, + { // 14 + []byte{ + 0x49, 0x79, 0x3e, 0xbc, 0x79, 0xb3, 0x25, 0x8f, + 0x49, 0x79, 0x3e, 0xbc, 0x79, 0xb3, 0x25, 0x8f, + 0x49, 0x79, 0x3e, 0xbc, 0x79, 0xb3, 0x25, 0x8f}, + []byte{0x43, 0x75, 0x40, 0xc8, 0x69, 0x8f, 0x3c, 0xfa}, + []byte{0x6f, 0xbf, 0x1c, 0xaf, 0xcf, 0xfd, 0x05, 0x56}}, + { // 15 + []byte{ + 0x4f, 0xb0, 0x5e, 0x15, 0x15, 0xab, 0x73, 0xa7, + 0x4f, 0xb0, 0x5e, 0x15, 0x15, 0xab, 0x73, 0xa7, + 0x4f, 0xb0, 0x5e, 0x15, 0x15, 0xab, 0x73, 0xa7}, + []byte{0x07, 0x2d, 0x43, 0xa0, 0x77, 0x07, 0x52, 0x92}, + []byte{0x2f, 0x22, 0xe4, 0x9b, 0xab, 0x7c, 0xa1, 0xac}}, + { // 16 + []byte{ + 0x49, 0xe9, 0x5d, 0x6d, 0x4c, 0xa2, 0x29, 0xbf, + 0x49, 0xe9, 0x5d, 0x6d, 0x4c, 0xa2, 0x29, 0xbf, + 0x49, 0xe9, 0x5d, 0x6d, 0x4c, 0xa2, 0x29, 0xbf}, + []byte{0x02, 0xfe, 0x55, 0x77, 0x81, 0x17, 0xf1, 0x2a}, + []byte{0x5a, 0x6b, 0x61, 0x2c, 0xc2, 0x6c, 0xce, 0x4a}}, + { // 17 + []byte{ + 0x01, 0x83, 0x10, 0xdc, 0x40, 0x9b, 0x26, 0xd6, + 0x01, 0x83, 0x10, 0xdc, 0x40, 0x9b, 0x26, 0xd6, + 0x01, 0x83, 0x10, 0xdc, 0x40, 0x9b, 0x26, 0xd6}, + []byte{0x1d, 0x9d, 0x5c, 0x50, 0x18, 0xf7, 0x28, 0xc2}, + []byte{0x5f, 0x4c, 0x03, 0x8e, 0xd1, 0x2b, 0x2e, 0x41}}, + { // 18 + []byte{ + 0x1c, 0x58, 0x7f, 0x1c, 0x13, 0x92, 0x4f, 0xef, + 0x1c, 0x58, 0x7f, 0x1c, 0x13, 0x92, 0x4f, 0xef, + 0x1c, 0x58, 0x7f, 0x1c, 0x13, 0x92, 0x4f, 0xef}, + []byte{0x30, 0x55, 0x32, 0x28, 0x6d, 0x6f, 0x29, 0x5a}, + []byte{0x63, 0xfa, 0xc0, 0xd0, 0x34, 0xd9, 0xf7, 0x93}}, +} + +// Use the known weak keys to test DES implementation +func TestWeakKeys(t *testing.T) { + for i, tt := range weakKeyTests { + var encrypt = func(in []byte) (out []byte) { + c, _ := NewCipher(tt.key) + out = make([]byte, len(in)) + encryptBlock(c.subkeys[:], out, in) + return + } + + // Encrypting twice with a DES weak + // key should reproduce the original input + result := encrypt(tt.in) + result = encrypt(result) + + if !bytes.Equal(result, tt.in) { + t.Errorf("#%d: result: %x want: %x", i, result, tt.in) + } + } +} + +// Use the known semi-weak key pairs to test DES implementation +func TestSemiWeakKeyPairs(t *testing.T) { + for i, tt := range semiWeakKeyTests { + var encrypt = func(key, in []byte) (out []byte) { + c, _ := NewCipher(key) + out = make([]byte, len(in)) + encryptBlock(c.subkeys[:], out, in) + return + } + + // Encrypting with one member of the semi-weak pair + // and then encrypting the result with the other member + // should reproduce the original input. + result := encrypt(tt.key, tt.in) + result = encrypt(tt.out, result) + + if !bytes.Equal(result, tt.in) { + t.Errorf("#%d: result: %x want: %x", i, result, tt.in) + } + } +} + +func TestDESEncryptBlock(t *testing.T) { + for i, tt := range encryptDESTests { + c, _ := NewCipher(tt.key) + out := make([]byte, len(tt.in)) + encryptBlock(c.subkeys[:], out, tt.in) + + if !bytes.Equal(out, tt.out) { + t.Errorf("#%d: result: %x want: %x", i, out, tt.out) + } + } +} + +func TestDESDecryptBlock(t *testing.T) { + for i, tt := range encryptDESTests { + c, _ := NewCipher(tt.key) + plain := make([]byte, len(tt.in)) + decryptBlock(c.subkeys[:], plain, tt.out) + + if !bytes.Equal(plain, tt.in) { + t.Errorf("#%d: result: %x want: %x", i, plain, tt.in) + } + } +} + +func TestEncryptTripleDES(t *testing.T) { + for i, tt := range encryptTripleDESTests { + c, _ := NewTripleDESCipher(tt.key) + out := make([]byte, len(tt.in)) + c.Encrypt(out, tt.in) + + if !bytes.Equal(out, tt.out) { + t.Errorf("#%d: result: %x want: %x", i, out, tt.out) + } + } +} + +func TestDecryptTripleDES(t *testing.T) { + for i, tt := range encryptTripleDESTests { + c, _ := NewTripleDESCipher(tt.key) + + plain := make([]byte, len(tt.in)) + c.Decrypt(plain, tt.out) + + if !bytes.Equal(plain, tt.in) { + t.Errorf("#%d: result: %x want: %x", i, plain, tt.in) + } + } +} + +// Defined in Pub 800-20 +func TestVariablePlaintextKnownAnswer(t *testing.T) { + for i, tt := range tableA1Tests { + c, _ := NewTripleDESCipher(tableA1Key) + + out := make([]byte, len(tt.in)) + c.Encrypt(out, tt.in) + + if !bytes.Equal(out, tt.out) { + t.Errorf("#%d: result: %x want: %x", i, out, tt.out) + } + } +} + +// Defined in Pub 800-20 +func TestVariableCiphertextKnownAnswer(t *testing.T) { + for i, tt := range tableA1Tests { + c, _ := NewTripleDESCipher(tableA1Key) + + plain := make([]byte, len(tt.out)) + c.Decrypt(plain, tt.out) + + if !bytes.Equal(plain, tt.in) { + t.Errorf("#%d: result: %x want: %x", i, plain, tt.in) + } + } +} + +// Defined in Pub 800-20 +// Encrypting the Table A.1 ciphertext with the +// 0x01... key produces the original plaintext +func TestInversePermutationKnownAnswer(t *testing.T) { + for i, tt := range tableA1Tests { + c, _ := NewTripleDESCipher(tableA1Key) + + plain := make([]byte, len(tt.in)) + c.Encrypt(plain, tt.out) + + if !bytes.Equal(plain, tt.in) { + t.Errorf("#%d: result: %x want: %x", i, plain, tt.in) + } + } +} + +// Defined in Pub 800-20 +// Decrypting the Table A.1 plaintext with the +// 0x01... key produces the corresponding ciphertext +func TestInitialPermutationKnownAnswer(t *testing.T) { + for i, tt := range tableA1Tests { + c, _ := NewTripleDESCipher(tableA1Key) + + out := make([]byte, len(tt.in)) + c.Decrypt(out, tt.in) + + if !bytes.Equal(out, tt.out) { + t.Errorf("#%d: result: %x want: %x", i, out, tt.out) + } + } +} + +// Defined in Pub 800-20 +func TestVariableKeyKnownAnswerEncrypt(t *testing.T) { + for i, tt := range tableA2Tests { + c, _ := NewTripleDESCipher(tt.key) + + out := make([]byte, len(tableA2Plaintext)) + c.Encrypt(out, tableA2Plaintext) + + if !bytes.Equal(out, tt.out) { + t.Errorf("#%d: result: %x want: %x", i, out, tt.out) + } + } +} + +// Defined in Pub 800-20 +func TestVariableKeyKnownAnswerDecrypt(t *testing.T) { + for i, tt := range tableA2Tests { + c, _ := NewTripleDESCipher(tt.key) + + out := make([]byte, len(tt.out)) + c.Decrypt(out, tt.out) + + if !bytes.Equal(out, tableA2Plaintext) { + t.Errorf("#%d: result: %x want: %x", i, out, tableA2Plaintext) + } + } +} + +// Defined in Pub 800-20 +func TestPermutationOperationKnownAnswerEncrypt(t *testing.T) { + for i, tt := range tableA3Tests { + c, _ := NewTripleDESCipher(tt.key) + + out := make([]byte, len(tableA3Plaintext)) + c.Encrypt(out, tableA3Plaintext) + + if !bytes.Equal(out, tt.out) { + t.Errorf("#%d: result: %x want: %x", i, out, tt.out) + } + } +} + +// Defined in Pub 800-20 +func TestPermutationOperationKnownAnswerDecrypt(t *testing.T) { + for i, tt := range tableA3Tests { + c, _ := NewTripleDESCipher(tt.key) + + out := make([]byte, len(tt.out)) + c.Decrypt(out, tt.out) + + if !bytes.Equal(out, tableA3Plaintext) { + t.Errorf("#%d: result: %x want: %x", i, out, tableA3Plaintext) + } + } +} + +// Defined in Pub 800-20 +func TestSubstitutionTableKnownAnswerEncrypt(t *testing.T) { + for i, tt := range tableA4Tests { + c, _ := NewTripleDESCipher(tt.key) + + out := make([]byte, len(tt.in)) + c.Encrypt(out, tt.in) + + if !bytes.Equal(out, tt.out) { + t.Errorf("#%d: result: %x want: %x", i, out, tt.out) + } + } +} + +// Defined in Pub 800-20 +func TestSubstitutionTableKnownAnswerDecrypt(t *testing.T) { + for i, tt := range tableA4Tests { + c, _ := NewTripleDESCipher(tt.key) + + out := make([]byte, len(tt.out)) + c.Decrypt(out, tt.out) + + if !bytes.Equal(out, tt.in) { + t.Errorf("#%d: result: %x want: %x", i, out, tt.in) + } + } +} diff --git a/src/pkg/crypto/dsa/Makefile b/src/pkg/crypto/dsa/Makefile new file mode 100644 index 000000000..fa89d4ab2 --- /dev/null +++ b/src/pkg/crypto/dsa/Makefile @@ -0,0 +1,11 @@ +# Copyright 2011 The Go Authors. All rights reserved. +# Use of this source code is governed by a BSD-style +# license that can be found in the LICENSE file. + +include ../../../Make.inc + +TARG=crypto/dsa +GOFILES=\ + dsa.go\ + +include ../../../Make.pkg diff --git a/src/pkg/crypto/dsa/dsa.go b/src/pkg/crypto/dsa/dsa.go new file mode 100644 index 000000000..a5f96fe94 --- /dev/null +++ b/src/pkg/crypto/dsa/dsa.go @@ -0,0 +1,276 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Package dsa implements the Digital Signature Algorithm, as defined in FIPS 186-3 +package dsa + +import ( + "big" + "io" + "os" +) + +// Parameters represents the domain parameters for a key. These parameters can +// be shared across many keys. The bit length of Q must be a multiple of 8. +type Parameters struct { + P, Q, G *big.Int +} + +// PublicKey represents a DSA public key. +type PublicKey struct { + Parameters + Y *big.Int +} + +// PrivateKey represents a DSA private key. +type PrivateKey struct { + PublicKey + X *big.Int +} + +type invalidPublicKeyError int + +func (invalidPublicKeyError) String() string { + return "crypto/dsa: invalid public key" +} + +// InvalidPublicKeyError results when a public key is not usable by this code. +// FIPS is quite strict about the format of DSA keys, but other code may be +// less so. Thus, when using keys which may have been generated by other code, +// this error must be handled. +var InvalidPublicKeyError = invalidPublicKeyError(0) + +// ParameterSizes is a enumeration of the acceptable bit lengths of the primes +// in a set of DSA parameters. See FIPS 186-3, section 4.2. +type ParameterSizes int + +const ( + L1024N160 ParameterSizes = iota + L2048N224 + L2048N256 + L3072N256 +) + +// numMRTests is the number of Miller-Rabin primality tests that we perform. We +// pick the largest recommended number from table C.1 of FIPS 186-3. +const numMRTests = 64 + +// GenerateParameters puts a random, valid set of DSA parameters into params. +// This function takes many seconds, even on fast machines. +func GenerateParameters(params *Parameters, rand io.Reader, sizes ParameterSizes) (err os.Error) { + // This function doesn't follow FIPS 186-3 exactly in that it doesn't + // use a verification seed to generate the primes. The verification + // seed doesn't appear to be exported or used by other code and + // omitting it makes the code cleaner. + + var L, N int + switch sizes { + case L1024N160: + L = 1024 + N = 160 + case L2048N224: + L = 2048 + N = 224 + case L2048N256: + L = 2048 + N = 256 + case L3072N256: + L = 3072 + N = 256 + default: + return os.NewError("crypto/dsa: invalid ParameterSizes") + } + + qBytes := make([]byte, N/8) + pBytes := make([]byte, L/8) + + q := new(big.Int) + p := new(big.Int) + rem := new(big.Int) + one := new(big.Int) + one.SetInt64(1) + +GeneratePrimes: + for { + _, err = io.ReadFull(rand, qBytes) + if err != nil { + return + } + + qBytes[len(qBytes)-1] |= 1 + qBytes[0] |= 0x80 + q.SetBytes(qBytes) + + if !big.ProbablyPrime(q, numMRTests) { + continue + } + + for i := 0; i < 4*L; i++ { + _, err = io.ReadFull(rand, pBytes) + if err != nil { + return + } + + pBytes[len(pBytes)-1] |= 1 + pBytes[0] |= 0x80 + + p.SetBytes(pBytes) + rem.Mod(p, q) + rem.Sub(rem, one) + p.Sub(p, rem) + if p.BitLen() < L { + continue + } + + if !big.ProbablyPrime(p, numMRTests) { + continue + } + + params.P = p + params.Q = q + break GeneratePrimes + } + } + + h := new(big.Int) + h.SetInt64(2) + g := new(big.Int) + + pm1 := new(big.Int).Sub(p, one) + e := new(big.Int).Div(pm1, q) + + for { + g.Exp(h, e, p) + if g.Cmp(one) == 0 { + h.Add(h, one) + continue + } + + params.G = g + return + } + + panic("unreachable") +} + +// GenerateKey generates a public&private key pair. The Parameters of the +// PrivateKey must already be valid (see GenerateParameters). +func GenerateKey(priv *PrivateKey, rand io.Reader) os.Error { + if priv.P == nil || priv.Q == nil || priv.G == nil { + return os.NewError("crypto/dsa: parameters not set up before generating key") + } + + x := new(big.Int) + xBytes := make([]byte, priv.Q.BitLen()/8) + + for { + _, err := io.ReadFull(rand, xBytes) + if err != nil { + return err + } + x.SetBytes(xBytes) + if x.Sign() != 0 && x.Cmp(priv.Q) < 0 { + break + } + } + + priv.X = x + priv.Y = new(big.Int) + priv.Y.Exp(priv.G, x, priv.P) + return nil +} + +// Sign signs an arbitrary length hash (which should be the result of hashing a +// larger message) using the private key, priv. It returns the signature as a +// pair of integers. The security of the private key depends on the entropy of +// rand. +func Sign(rand io.Reader, priv *PrivateKey, hash []byte) (r, s *big.Int, err os.Error) { + // FIPS 186-3, section 4.6 + + n := priv.Q.BitLen() + if n&7 != 0 { + err = InvalidPublicKeyError + return + } + n >>= 3 + + for { + k := new(big.Int) + buf := make([]byte, n) + for { + _, err = io.ReadFull(rand, buf) + if err != nil { + return + } + k.SetBytes(buf) + if k.Sign() > 0 && k.Cmp(priv.Q) < 0 { + break + } + } + + kInv := new(big.Int).ModInverse(k, priv.Q) + + r = new(big.Int).Exp(priv.G, k, priv.P) + r.Mod(r, priv.Q) + + if r.Sign() == 0 { + continue + } + + if n > len(hash) { + n = len(hash) + } + z := k.SetBytes(hash[:n]) + + s = new(big.Int).Mul(priv.X, r) + s.Add(s, z) + s.Mod(s, priv.Q) + s.Mul(s, kInv) + s.Mod(s, priv.Q) + + if s.Sign() != 0 { + break + } + } + + return +} + +// Verify verifies the signature in r, s of hash using the public key, pub. It +// returns true iff the signature is valid. +func Verify(pub *PublicKey, hash []byte, r, s *big.Int) bool { + // FIPS 186-3, section 4.7 + + if r.Sign() < 1 || r.Cmp(pub.Q) >= 0 { + return false + } + if s.Sign() < 1 || s.Cmp(pub.Q) >= 0 { + return false + } + + w := new(big.Int).ModInverse(s, pub.Q) + + n := pub.Q.BitLen() + if n&7 != 0 { + return false + } + n >>= 3 + + if n > len(hash) { + n = len(hash) + } + z := new(big.Int).SetBytes(hash[:n]) + + u1 := new(big.Int).Mul(z, w) + u1.Mod(u1, pub.Q) + u2 := w.Mul(r, w) + u2.Mod(u2, pub.Q) + v := u1.Exp(pub.G, u1, pub.P) + u2.Exp(pub.Y, u2, pub.P) + v.Mul(v, u2) + v.Mod(v, pub.P) + v.Mod(v, pub.Q) + + return v.Cmp(r) == 0 +} diff --git a/src/pkg/crypto/dsa/dsa_test.go b/src/pkg/crypto/dsa/dsa_test.go new file mode 100644 index 000000000..deec08dfd --- /dev/null +++ b/src/pkg/crypto/dsa/dsa_test.go @@ -0,0 +1,84 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package dsa + +import ( + "big" + "crypto/rand" + "testing" +) + +func testSignAndVerify(t *testing.T, i int, priv *PrivateKey) { + hashed := []byte("testing") + r, s, err := Sign(rand.Reader, priv, hashed) + if err != nil { + t.Errorf("%d: error signing: %s", i, err) + return + } + + if !Verify(&priv.PublicKey, hashed, r, s) { + t.Errorf("%d: Verify failed", i) + } +} + +func testParameterGeneration(t *testing.T, sizes ParameterSizes, L, N int) { + var priv PrivateKey + params := &priv.Parameters + + err := GenerateParameters(params, rand.Reader, sizes) + if err != nil { + t.Errorf("%d: %s", int(sizes), err) + return + } + + if params.P.BitLen() != L { + t.Errorf("%d: params.BitLen got:%d want:%d", int(sizes), params.P.BitLen(), L) + } + + if params.Q.BitLen() != N { + t.Errorf("%d: q.BitLen got:%d want:%d", int(sizes), params.Q.BitLen(), L) + } + + one := new(big.Int) + one.SetInt64(1) + pm1 := new(big.Int).Sub(params.P, one) + quo, rem := new(big.Int).DivMod(pm1, params.Q, new(big.Int)) + if rem.Sign() != 0 { + t.Errorf("%d: p-1 mod q != 0", int(sizes)) + } + x := new(big.Int).Exp(params.G, quo, params.P) + if x.Cmp(one) == 0 { + t.Errorf("%d: invalid generator", int(sizes)) + } + + err = GenerateKey(&priv, rand.Reader) + if err != nil { + t.Errorf("error generating key: %s", err) + return + } + + testSignAndVerify(t, int(sizes), &priv) +} + +func TestParameterGeneration(t *testing.T) { + // This test is too slow to run all the time. + return + + testParameterGeneration(t, L1024N160, 1024, 160) + testParameterGeneration(t, L2048N224, 2048, 224) + testParameterGeneration(t, L2048N256, 2048, 256) + testParameterGeneration(t, L3072N256, 3072, 256) +} + +func TestSignAndVerify(t *testing.T) { + var priv PrivateKey + priv.P, _ = new(big.Int).SetString("A9B5B793FB4785793D246BAE77E8FF63CA52F442DA763C440259919FE1BC1D6065A9350637A04F75A2F039401D49F08E066C4D275A5A65DA5684BC563C14289D7AB8A67163BFBF79D85972619AD2CFF55AB0EE77A9002B0EF96293BDD0F42685EBB2C66C327079F6C98000FBCB79AACDE1BC6F9D5C7B1A97E3D9D54ED7951FEF", 16) + priv.Q, _ = new(big.Int).SetString("E1D3391245933D68A0714ED34BBCB7A1F422B9C1", 16) + priv.G, _ = new(big.Int).SetString("634364FC25248933D01D1993ECABD0657CC0CB2CEED7ED2E3E8AECDFCDC4A25C3B15E9E3B163ACA2984B5539181F3EFF1A5E8903D71D5B95DA4F27202B77D2C44B430BB53741A8D59A8F86887525C9F2A6A5980A195EAA7F2FF910064301DEF89D3AA213E1FAC7768D89365318E370AF54A112EFBA9246D9158386BA1B4EEFDA", 16) + priv.Y, _ = new(big.Int).SetString("32969E5780CFE1C849A1C276D7AEB4F38A23B591739AA2FE197349AEEBD31366AEE5EB7E6C6DDB7C57D02432B30DB5AA66D9884299FAA72568944E4EEDC92EA3FBC6F39F53412FBCC563208F7C15B737AC8910DBC2D9C9B8C001E72FDC40EB694AB1F06A5A2DBD18D9E36C66F31F566742F11EC0A52E9F7B89355C02FB5D32D2", 16) + priv.X, _ = new(big.Int).SetString("5078D4D29795CBE76D3AACFE48C9AF0BCDBEE91A", 16) + + testSignAndVerify(t, 0, &priv) +} diff --git a/src/pkg/crypto/ecdsa/Makefile b/src/pkg/crypto/ecdsa/Makefile new file mode 100644 index 000000000..0af24c2f2 --- /dev/null +++ b/src/pkg/crypto/ecdsa/Makefile @@ -0,0 +1,11 @@ +# Copyright 2011 The Go Authors. All rights reserved. +# Use of this source code is governed by a BSD-style +# license that can be found in the LICENSE file. + +include ../../../Make.inc + +TARG=crypto/ecdsa +GOFILES=\ + ecdsa.go\ + +include ../../../Make.pkg diff --git a/src/pkg/crypto/ecdsa/ecdsa.go b/src/pkg/crypto/ecdsa/ecdsa.go new file mode 100644 index 000000000..7bce1bc96 --- /dev/null +++ b/src/pkg/crypto/ecdsa/ecdsa.go @@ -0,0 +1,149 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Package ecdsa implements the Elliptic Curve Digital Signature Algorithm, as +// defined in FIPS 186-3. +package ecdsa + +// References: +// [NSA]: Suite B implementor's guide to FIPS 186-3, +// http://www.nsa.gov/ia/_files/ecdsa.pdf +// [SECG]: SECG, SEC1 +// http://www.secg.org/download/aid-780/sec1-v2.pdf + +import ( + "big" + "crypto/elliptic" + "io" + "os" +) + +// PublicKey represents an ECDSA public key. +type PublicKey struct { + *elliptic.Curve + X, Y *big.Int +} + +// PrivateKey represents a ECDSA private key. +type PrivateKey struct { + PublicKey + D *big.Int +} + +var one = new(big.Int).SetInt64(1) + +// randFieldElement returns a random element of the field underlying the given +// curve using the procedure given in [NSA] A.2.1. +func randFieldElement(c *elliptic.Curve, rand io.Reader) (k *big.Int, err os.Error) { + b := make([]byte, c.BitSize/8+8) + _, err = io.ReadFull(rand, b) + if err != nil { + return + } + + k = new(big.Int).SetBytes(b) + n := new(big.Int).Sub(c.N, one) + k.Mod(k, n) + k.Add(k, one) + return +} + +// GenerateKey generates a public&private key pair. +func GenerateKey(c *elliptic.Curve, rand io.Reader) (priv *PrivateKey, err os.Error) { + k, err := randFieldElement(c, rand) + if err != nil { + return + } + + priv = new(PrivateKey) + priv.PublicKey.Curve = c + priv.D = k + priv.PublicKey.X, priv.PublicKey.Y = c.ScalarBaseMult(k.Bytes()) + return +} + +// hashToInt converts a hash value to an integer. There is some disagreement +// about how this is done. [NSA] suggests that this is done in the obvious +// manner, but [SECG] truncates the hash to the bit-length of the curve order +// first. We follow [SECG] because that's what OpenSSL does. +func hashToInt(hash []byte, c *elliptic.Curve) *big.Int { + orderBits := c.N.BitLen() + orderBytes := (orderBits + 7) / 8 + if len(hash) > orderBytes { + hash = hash[:orderBytes] + } + + ret := new(big.Int).SetBytes(hash) + excess := orderBytes*8 - orderBits + if excess > 0 { + ret.Rsh(ret, uint(excess)) + } + return ret +} + +// Sign signs an arbitrary length hash (which should be the result of hashing a +// larger message) using the private key, priv. It returns the signature as a +// pair of integers. The security of the private key depends on the entropy of +// rand. +func Sign(rand io.Reader, priv *PrivateKey, hash []byte) (r, s *big.Int, err os.Error) { + // See [NSA] 3.4.1 + c := priv.PublicKey.Curve + + var k, kInv *big.Int + for { + for { + k, err = randFieldElement(c, rand) + if err != nil { + r = nil + return + } + + kInv = new(big.Int).ModInverse(k, c.N) + r, _ = priv.Curve.ScalarBaseMult(k.Bytes()) + r.Mod(r, priv.Curve.N) + if r.Sign() != 0 { + break + } + } + + e := hashToInt(hash, c) + s = new(big.Int).Mul(priv.D, r) + s.Add(s, e) + s.Mul(s, kInv) + s.Mod(s, priv.PublicKey.Curve.N) + if s.Sign() != 0 { + break + } + } + + return +} + +// Verify verifies the signature in r, s of hash using the public key, pub. It +// returns true iff the signature is valid. +func Verify(pub *PublicKey, hash []byte, r, s *big.Int) bool { + // See [NSA] 3.4.2 + c := pub.Curve + + if r.Sign() == 0 || s.Sign() == 0 { + return false + } + if r.Cmp(c.N) >= 0 || s.Cmp(c.N) >= 0 { + return false + } + e := hashToInt(hash, c) + w := new(big.Int).ModInverse(s, c.N) + + u1 := e.Mul(e, w) + u2 := w.Mul(r, w) + + x1, y1 := c.ScalarBaseMult(u1.Bytes()) + x2, y2 := c.ScalarMult(pub.X, pub.Y, u2.Bytes()) + if x1.Cmp(x2) == 0 { + return false + } + x, _ := c.Add(x1, y1, x2, y2) + x.Mod(x, c.N) + return x.Cmp(r) == 0 +} diff --git a/src/pkg/crypto/ecdsa/ecdsa_test.go b/src/pkg/crypto/ecdsa/ecdsa_test.go new file mode 100644 index 000000000..d6b403914 --- /dev/null +++ b/src/pkg/crypto/ecdsa/ecdsa_test.go @@ -0,0 +1,227 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package ecdsa + +import ( + "big" + "crypto/elliptic" + "crypto/sha1" + "crypto/rand" + "encoding/hex" + "testing" +) + +func testKeyGeneration(t *testing.T, c *elliptic.Curve, tag string) { + priv, err := GenerateKey(c, rand.Reader) + if err != nil { + t.Errorf("%s: error: %s", tag, err) + return + } + if !c.IsOnCurve(priv.PublicKey.X, priv.PublicKey.Y) { + t.Errorf("%s: public key invalid: %s", tag, err) + } +} + +func TestKeyGeneration(t *testing.T) { + testKeyGeneration(t, elliptic.P224(), "p224") + if testing.Short() { + return + } + testKeyGeneration(t, elliptic.P256(), "p256") + testKeyGeneration(t, elliptic.P384(), "p384") + testKeyGeneration(t, elliptic.P521(), "p521") +} + +func testSignAndVerify(t *testing.T, c *elliptic.Curve, tag string) { + priv, _ := GenerateKey(c, rand.Reader) + + hashed := []byte("testing") + r, s, err := Sign(rand.Reader, priv, hashed) + if err != nil { + t.Errorf("%s: error signing: %s", tag, err) + return + } + + if !Verify(&priv.PublicKey, hashed, r, s) { + t.Errorf("%s: Verify failed", tag) + } + + hashed[0] ^= 0xff + if Verify(&priv.PublicKey, hashed, r, s) { + t.Errorf("%s: Verify always works!", tag) + } +} + +func TestSignAndVerify(t *testing.T) { + testSignAndVerify(t, elliptic.P224(), "p224") + if testing.Short() { + return + } + testSignAndVerify(t, elliptic.P256(), "p256") + testSignAndVerify(t, elliptic.P384(), "p384") + testSignAndVerify(t, elliptic.P521(), "p521") +} + +func fromHex(s string) *big.Int { + r, ok := new(big.Int).SetString(s, 16) + if !ok { + panic("bad hex") + } + return r +} + +// These test vectors were taken from +// http://csrc.nist.gov/groups/STM/cavp/documents/dss/ecdsatestvectors.zip +var testVectors = []struct { + msg string + Qx, Qy string + r, s string + ok bool +}{ + { + "09626b45493672e48f3d1226a3aff3201960e577d33a7f72c7eb055302db8fe8ed61685dd036b554942a5737cd1512cdf811ee0c00e6dd2f08c69f08643be396e85dafda664801e772cdb7396868ac47b172245b41986aa2648cb77fbbfa562581be06651355a0c4b090f9d17d8f0ab6cced4e0c9d386cf465a516630f0231bd", + "9504b5b82d97a264d8b3735e0568decabc4b6ca275bc53cbadfc1c40", + "03426f80e477603b10dee670939623e3da91a94267fc4e51726009ed", + "81d3ac609f9575d742028dd496450a58a60eea2dcf8b9842994916e1", + "96a8c5f382c992e8f30ccce9af120b067ec1d74678fa8445232f75a5", + false, + }, + { + "96b2b6536f6df29be8567a72528aceeaccbaa66c66c534f3868ca9778b02faadb182e4ed34662e73b9d52ecbe9dc8e875fc05033c493108b380689ebf47e5b062e6a0cdb3dd34ce5fe347d92768d72f7b9b377c20aea927043b509c078ed2467d7113405d2ddd458811e6faf41c403a2a239240180f1430a6f4330df5d77de37", + "851e3100368a22478a0029353045ae40d1d8202ef4d6533cfdddafd8", + "205302ac69457dd345e86465afa72ee8c74ca97e2b0b999aec1f10c2", + "4450c2d38b697e990721aa2dbb56578d32b4f5aeb3b9072baa955ee0", + "e26d4b589166f7b4ba4b1c8fce823fa47aad22f8c9c396b8c6526e12", + false, + }, + { + "86778dbb4a068a01047a8d245d632f636c11d2ad350740b36fad90428b454ad0f120cb558d12ea5c8a23db595d87543d06d1ef489263d01ee529871eb68737efdb8ff85bc7787b61514bed85b7e01d6be209e0a4eb0db5c8df58a5c5bf706d76cb2bdf7800208639e05b89517155d11688236e6a47ed37d8e5a2b1e0adea338e", + "ad5bda09d319a717c1721acd6688d17020b31b47eef1edea57ceeffc", + "c8ce98e181770a7c9418c73c63d01494b8b80a41098c5ea50692c984", + "de5558c257ab4134e52c19d8db3b224a1899cbd08cc508ce8721d5e9", + "745db7af5a477e5046705c0a5eff1f52cb94a79d481f0c5a5e108ecd", + true, + }, + { + "4bc6ef1958556686dab1e39c3700054a304cbd8f5928603dcd97fafd1f29e69394679b638f71c9344ce6a535d104803d22119f57b5f9477e253817a52afa9bfbc9811d6cc8c8be6b6566c6ef48b439bbb532abe30627548c598867f3861ba0b154dc1c3deca06eb28df8efd28258554b5179883a36fbb1eecf4f93ee19d41e3d", + "cc5eea2edf964018bdc0504a3793e4d2145142caa09a72ac5fb8d3e8", + "a48d78ae5d08aa725342773975a00d4219cf7a8029bb8cf3c17c374a", + "67b861344b4e416d4094472faf4272f6d54a497177fbc5f9ef292836", + "1d54f3fcdad795bf3b23408ecbac3e1321d1d66f2e4e3d05f41f7020", + false, + }, + { + "bb658732acbf3147729959eb7318a2058308b2739ec58907dd5b11cfa3ecf69a1752b7b7d806fe00ec402d18f96039f0b78dbb90a59c4414fb33f1f4e02e4089de4122cd93df5263a95be4d7084e2126493892816e6a5b4ed123cb705bf930c8f67af0fb4514d5769232a9b008a803af225160ce63f675bd4872c4c97b146e5e", + "6234c936e27bf141fc7534bfc0a7eedc657f91308203f1dcbd642855", + "27983d87ca785ef4892c3591ef4a944b1deb125dd58bd351034a6f84", + "e94e05b42d01d0b965ffdd6c3a97a36a771e8ea71003de76c4ecb13f", + "1dc6464ffeefbd7872a081a5926e9fc3e66d123f1784340ba17737e9", + false, + }, + { + "7c00be9123bfa2c4290be1d8bc2942c7f897d9a5b7917e3aabd97ef1aab890f148400a89abd554d19bec9d8ed911ce57b22fbcf6d30ca2115f13ce0a3f569a23bad39ee645f624c49c60dcfc11e7d2be24de9c905596d8f23624d63dc46591d1f740e46f982bfae453f107e80db23545782be23ce43708245896fc54e1ee5c43", + "9f3f037282aaf14d4772edffff331bbdda845c3f65780498cde334f1", + "8308ee5a16e3bcb721b6bc30000a0419bc1aaedd761be7f658334066", + "6381d7804a8808e3c17901e4d283b89449096a8fba993388fa11dc54", + "8e858f6b5b253686a86b757bad23658cda53115ac565abca4e3d9f57", + false, + }, + { + "cffc122a44840dc705bb37130069921be313d8bde0b66201aebc48add028ca131914ef2e705d6bedd19dc6cf9459bbb0f27cdfe3c50483808ffcdaffbeaa5f062e097180f07a40ef4ab6ed03fe07ed6bcfb8afeb42c97eafa2e8a8df469de07317c5e1494c41547478eff4d8c7d9f0f484ad90fedf6e1c35ee68fa73f1691601", + "a03b88a10d930002c7b17ca6af2fd3e88fa000edf787dc594f8d4fd4", + "e0cf7acd6ddc758e64847fe4df9915ebda2f67cdd5ec979aa57421f5", + "387b84dcf37dc343c7d2c5beb82f0bf8bd894b395a7b894565d296c1", + "4adc12ce7d20a89ce3925e10491c731b15ddb3f339610857a21b53b4", + false, + }, + { + "26e0e0cafd85b43d16255908ccfd1f061c680df75aba3081246b337495783052ba06c60f4a486c1591a4048bae11b4d7fec4f161d80bdc9a7b79d23e44433ed625eab280521a37f23dd3e1bdc5c6a6cfaa026f3c45cf703e76dab57add93fe844dd4cda67dc3bddd01f9152579e49df60969b10f09ce9372fdd806b0c7301866", + "9a8983c42f2b5a87c37a00458b5970320d247f0c8a88536440173f7d", + "15e489ec6355351361900299088cfe8359f04fe0cab78dde952be80c", + "929a21baa173d438ec9f28d6a585a2f9abcfc0a4300898668e476dc0", + "59a853f046da8318de77ff43f26fe95a92ee296fa3f7e56ce086c872", + true, + }, + { + "1078eac124f48ae4f807e946971d0de3db3748dd349b14cca5c942560fb25401b2252744f18ad5e455d2d97ed5ae745f55ff509c6c8e64606afe17809affa855c4c4cdcaf6b69ab4846aa5624ed0687541aee6f2224d929685736c6a23906d974d3c257abce1a3fb8db5951b89ecb0cda92b5207d93f6618fd0f893c32cf6a6e", + "d6e55820bb62c2be97650302d59d667a411956138306bd566e5c3c2b", + "631ab0d64eaf28a71b9cbd27a7a88682a2167cee6251c44e3810894f", + "65af72bc7721eb71c2298a0eb4eed3cec96a737cc49125706308b129", + "bd5a987c78e2d51598dbd9c34a9035b0069c580edefdacee17ad892a", + false, + }, + { + "919deb1fdd831c23481dfdb2475dcbe325b04c34f82561ced3d2df0b3d749b36e255c4928973769d46de8b95f162b53cd666cad9ae145e7fcfba97919f703d864efc11eac5f260a5d920d780c52899e5d76f8fe66936ff82130761231f536e6a3d59792f784902c469aa897aabf9a0678f93446610d56d5e0981e4c8a563556b", + "269b455b1024eb92d860a420f143ac1286b8cce43031562ae7664574", + "baeb6ca274a77c44a0247e5eb12ca72bdd9a698b3f3ae69c9f1aaa57", + "cb4ec2160f04613eb0dfe4608486091a25eb12aa4dec1afe91cfb008", + "40b01d8cd06589481574f958b98ca08ade9d2a8fe31024375c01bb40", + false, + }, + { + "6e012361250dacf6166d2dd1aa7be544c3206a9d43464b3fcd90f3f8cf48d08ec099b59ba6fe7d9bdcfaf244120aed1695d8be32d1b1cd6f143982ab945d635fb48a7c76831c0460851a3d62b7209c30cd9c2abdbe3d2a5282a9fcde1a6f418dd23c409bc351896b9b34d7d3a1a63bbaf3d677e612d4a80fa14829386a64b33f", + "6d2d695efc6b43b13c14111f2109608f1020e3e03b5e21cfdbc82fcd", + "26a4859296b7e360b69cf40be7bd97ceaffa3d07743c8489fc47ca1b", + "9a8cb5f2fdc288b7183c5b32d8e546fc2ed1ca4285eeae00c8b572ad", + "8c623f357b5d0057b10cdb1a1593dab57cda7bdec9cf868157a79b97", + true, + }, + { + "bf6bd7356a52b234fe24d25557200971fc803836f6fec3cade9642b13a8e7af10ab48b749de76aada9d8927f9b12f75a2c383ca7358e2566c4bb4f156fce1fd4e87ef8c8d2b6b1bdd351460feb22cdca0437ac10ca5e0abbbce9834483af20e4835386f8b1c96daaa41554ceee56730aac04f23a5c765812efa746051f396566", + "14250131b2599939cf2d6bc491be80ddfe7ad9de644387ee67de2d40", + "b5dc473b5d014cd504022043c475d3f93c319a8bdcb7262d9e741803", + "4f21642f2201278a95339a80f75cc91f8321fcb3c9462562f6cbf145", + "452a5f816ea1f75dee4fd514fa91a0d6a43622981966c59a1b371ff8", + false, + }, + { + "0eb7f4032f90f0bd3cf9473d6d9525d264d14c031a10acd31a053443ed5fe919d5ac35e0be77813071b4062f0b5fdf58ad5f637b76b0b305aec18f82441b6e607b44cdf6e0e3c7c57f24e6fd565e39430af4a6b1d979821ed0175fa03e3125506847654d7e1ae904ce1190ae38dc5919e257bdac2db142a6e7cd4da6c2e83770", + "d1f342b7790a1667370a1840255ac5bbbdc66f0bc00ae977d99260ac", + "76416cabae2de9a1000b4646338b774baabfa3db4673790771220cdb", + "bc85e3fc143d19a7271b2f9e1c04b86146073f3fab4dda1c3b1f35ca", + "9a5c70ede3c48d5f43307a0c2a4871934424a3303b815df4bb0f128e", + false, + }, + { + "5cc25348a05d85e56d4b03cec450128727bc537c66ec3a9fb613c151033b5e86878632249cba83adcefc6c1e35dcd31702929c3b57871cda5c18d1cf8f9650a25b917efaed56032e43b6fc398509f0d2997306d8f26675f3a8683b79ce17128e006aa0903b39eeb2f1001be65de0520115e6f919de902b32c38d691a69c58c92", + "7e49a7abf16a792e4c7bbc4d251820a2abd22d9f2fc252a7bf59c9a6", + "44236a8fb4791c228c26637c28ae59503a2f450d4cfb0dc42aa843b9", + "084461b4050285a1a85b2113be76a17878d849e6bc489f4d84f15cd8", + "079b5bddcc4d45de8dbdfd39f69817c7e5afa454a894d03ee1eaaac3", + false, + }, + { + "1951533ce33afb58935e39e363d8497a8dd0442018fd96dff167b3b23d7206a3ee182a3194765df4768a3284e23b8696c199b4686e670d60c9d782f08794a4bccc05cffffbd1a12acd9eb1cfa01f7ebe124da66ecff4599ea7720c3be4bb7285daa1a86ebf53b042bd23208d468c1b3aa87381f8e1ad63e2b4c2ba5efcf05845", + "31945d12ebaf4d81f02be2b1768ed80784bf35cf5e2ff53438c11493", + "a62bebffac987e3b9d3ec451eb64c462cdf7b4aa0b1bbb131ceaa0a4", + "bc3c32b19e42b710bca5c6aaa128564da3ddb2726b25f33603d2af3c", + "ed1a719cc0c507edc5239d76fe50e2306c145ad252bd481da04180c0", + false, + }, +} + +func TestVectors(t *testing.T) { + sha := sha1.New() + + for i, test := range testVectors { + pub := PublicKey{ + Curve: elliptic.P224(), + X: fromHex(test.Qx), + Y: fromHex(test.Qy), + } + msg, _ := hex.DecodeString(test.msg) + sha.Reset() + sha.Write(msg) + hashed := sha.Sum() + r := fromHex(test.r) + s := fromHex(test.s) + if Verify(&pub, hashed, r, s) != test.ok { + t.Errorf("%d: bad result", i) + } + if testing.Short() { + break + } + } +} diff --git a/src/pkg/crypto/elliptic/Makefile b/src/pkg/crypto/elliptic/Makefile new file mode 100644 index 000000000..4db5d7de5 --- /dev/null +++ b/src/pkg/crypto/elliptic/Makefile @@ -0,0 +1,11 @@ +# Copyright 2010 The Go Authors. All rights reserved. +# Use of this source code is governed by a BSD-style +# license that can be found in the LICENSE file. + +include ../../../Make.inc + +TARG=crypto/elliptic +GOFILES=\ + elliptic.go\ + +include ../../../Make.pkg diff --git a/src/pkg/crypto/elliptic/elliptic.go b/src/pkg/crypto/elliptic/elliptic.go new file mode 100644 index 000000000..41835f1a9 --- /dev/null +++ b/src/pkg/crypto/elliptic/elliptic.go @@ -0,0 +1,381 @@ +// Copyright 2010 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Package elliptic implements several standard elliptic curves over prime +// fields. +package elliptic + +// This package operates, internally, on Jacobian coordinates. For a given +// (x, y) position on the curve, the Jacobian coordinates are (x1, y1, z1) +// where x = x1/z1² and y = y1/z1³. The greatest speedups come when the whole +// calculation can be performed within the transform (as in ScalarMult and +// ScalarBaseMult). But even for Add and Double, it's faster to apply and +// reverse the transform than to operate in affine coordinates. + +import ( + "big" + "io" + "os" + "sync" +) + +// A Curve represents a short-form Weierstrass curve with a=-3. +// See http://www.hyperelliptic.org/EFD/g1p/auto-shortw.html +type Curve struct { + P *big.Int // the order of the underlying field + N *big.Int // the order of the base point + B *big.Int // the constant of the curve equation + Gx, Gy *big.Int // (x,y) of the base point + BitSize int // the size of the underlying field +} + +// IsOnCurve returns true if the given (x,y) lies on the curve. +func (curve *Curve) IsOnCurve(x, y *big.Int) bool { + // y² = x³ - 3x + b + y2 := new(big.Int).Mul(y, y) + y2.Mod(y2, curve.P) + + x3 := new(big.Int).Mul(x, x) + x3.Mul(x3, x) + + threeX := new(big.Int).Lsh(x, 1) + threeX.Add(threeX, x) + + x3.Sub(x3, threeX) + x3.Add(x3, curve.B) + x3.Mod(x3, curve.P) + + return x3.Cmp(y2) == 0 +} + +// affineFromJacobian reverses the Jacobian transform. See the comment at the +// top of the file. +func (curve *Curve) affineFromJacobian(x, y, z *big.Int) (xOut, yOut *big.Int) { + zinv := new(big.Int).ModInverse(z, curve.P) + zinvsq := new(big.Int).Mul(zinv, zinv) + + xOut = new(big.Int).Mul(x, zinvsq) + xOut.Mod(xOut, curve.P) + zinvsq.Mul(zinvsq, zinv) + yOut = new(big.Int).Mul(y, zinvsq) + yOut.Mod(yOut, curve.P) + return +} + +// Add returns the sum of (x1,y1) and (x2,y2) +func (curve *Curve) Add(x1, y1, x2, y2 *big.Int) (*big.Int, *big.Int) { + z := new(big.Int).SetInt64(1) + return curve.affineFromJacobian(curve.addJacobian(x1, y1, z, x2, y2, z)) +} + +// addJacobian takes two points in Jacobian coordinates, (x1, y1, z1) and +// (x2, y2, z2) and returns their sum, also in Jacobian form. +func (curve *Curve) addJacobian(x1, y1, z1, x2, y2, z2 *big.Int) (*big.Int, *big.Int, *big.Int) { + // See http://hyperelliptic.org/EFD/g1p/auto-shortw-jacobian-3.html#addition-add-2007-bl + z1z1 := new(big.Int).Mul(z1, z1) + z1z1.Mod(z1z1, curve.P) + z2z2 := new(big.Int).Mul(z2, z2) + z2z2.Mod(z2z2, curve.P) + + u1 := new(big.Int).Mul(x1, z2z2) + u1.Mod(u1, curve.P) + u2 := new(big.Int).Mul(x2, z1z1) + u2.Mod(u2, curve.P) + h := new(big.Int).Sub(u2, u1) + if h.Sign() == -1 { + h.Add(h, curve.P) + } + i := new(big.Int).Lsh(h, 1) + i.Mul(i, i) + j := new(big.Int).Mul(h, i) + + s1 := new(big.Int).Mul(y1, z2) + s1.Mul(s1, z2z2) + s1.Mod(s1, curve.P) + s2 := new(big.Int).Mul(y2, z1) + s2.Mul(s2, z1z1) + s2.Mod(s2, curve.P) + r := new(big.Int).Sub(s2, s1) + if r.Sign() == -1 { + r.Add(r, curve.P) + } + r.Lsh(r, 1) + v := new(big.Int).Mul(u1, i) + + x3 := new(big.Int).Set(r) + x3.Mul(x3, x3) + x3.Sub(x3, j) + x3.Sub(x3, v) + x3.Sub(x3, v) + x3.Mod(x3, curve.P) + + y3 := new(big.Int).Set(r) + v.Sub(v, x3) + y3.Mul(y3, v) + s1.Mul(s1, j) + s1.Lsh(s1, 1) + y3.Sub(y3, s1) + y3.Mod(y3, curve.P) + + z3 := new(big.Int).Add(z1, z2) + z3.Mul(z3, z3) + z3.Sub(z3, z1z1) + if z3.Sign() == -1 { + z3.Add(z3, curve.P) + } + z3.Sub(z3, z2z2) + if z3.Sign() == -1 { + z3.Add(z3, curve.P) + } + z3.Mul(z3, h) + z3.Mod(z3, curve.P) + + return x3, y3, z3 +} + +// Double returns 2*(x,y) +func (curve *Curve) Double(x1, y1 *big.Int) (*big.Int, *big.Int) { + z1 := new(big.Int).SetInt64(1) + return curve.affineFromJacobian(curve.doubleJacobian(x1, y1, z1)) +} + +// doubleJacobian takes a point in Jacobian coordinates, (x, y, z), and +// returns its double, also in Jacobian form. +func (curve *Curve) doubleJacobian(x, y, z *big.Int) (*big.Int, *big.Int, *big.Int) { + // See http://hyperelliptic.org/EFD/g1p/auto-shortw-jacobian-3.html#doubling-dbl-2001-b + delta := new(big.Int).Mul(z, z) + delta.Mod(delta, curve.P) + gamma := new(big.Int).Mul(y, y) + gamma.Mod(gamma, curve.P) + alpha := new(big.Int).Sub(x, delta) + if alpha.Sign() == -1 { + alpha.Add(alpha, curve.P) + } + alpha2 := new(big.Int).Add(x, delta) + alpha.Mul(alpha, alpha2) + alpha2.Set(alpha) + alpha.Lsh(alpha, 1) + alpha.Add(alpha, alpha2) + + beta := alpha2.Mul(x, gamma) + + x3 := new(big.Int).Mul(alpha, alpha) + beta8 := new(big.Int).Lsh(beta, 3) + x3.Sub(x3, beta8) + for x3.Sign() == -1 { + x3.Add(x3, curve.P) + } + x3.Mod(x3, curve.P) + + z3 := new(big.Int).Add(y, z) + z3.Mul(z3, z3) + z3.Sub(z3, gamma) + if z3.Sign() == -1 { + z3.Add(z3, curve.P) + } + z3.Sub(z3, delta) + if z3.Sign() == -1 { + z3.Add(z3, curve.P) + } + z3.Mod(z3, curve.P) + + beta.Lsh(beta, 2) + beta.Sub(beta, x3) + if beta.Sign() == -1 { + beta.Add(beta, curve.P) + } + y3 := alpha.Mul(alpha, beta) + + gamma.Mul(gamma, gamma) + gamma.Lsh(gamma, 3) + gamma.Mod(gamma, curve.P) + + y3.Sub(y3, gamma) + if y3.Sign() == -1 { + y3.Add(y3, curve.P) + } + y3.Mod(y3, curve.P) + + return x3, y3, z3 +} + +// ScalarMult returns k*(Bx,By) where k is a number in big-endian form. +func (curve *Curve) ScalarMult(Bx, By *big.Int, k []byte) (*big.Int, *big.Int) { + // We have a slight problem in that the identity of the group (the + // point at infinity) cannot be represented in (x, y) form on a finite + // machine. Thus the standard add/double algorithm has to be tweaked + // slightly: our initial state is not the identity, but x, and we + // ignore the first true bit in |k|. If we don't find any true bits in + // |k|, then we return nil, nil, because we cannot return the identity + // element. + + Bz := new(big.Int).SetInt64(1) + x := Bx + y := By + z := Bz + + seenFirstTrue := false + for _, byte := range k { + for bitNum := 0; bitNum < 8; bitNum++ { + if seenFirstTrue { + x, y, z = curve.doubleJacobian(x, y, z) + } + if byte&0x80 == 0x80 { + if !seenFirstTrue { + seenFirstTrue = true + } else { + x, y, z = curve.addJacobian(Bx, By, Bz, x, y, z) + } + } + byte <<= 1 + } + } + + if !seenFirstTrue { + return nil, nil + } + + return curve.affineFromJacobian(x, y, z) +} + +// ScalarBaseMult returns k*G, where G is the base point of the group and k is +// an integer in big-endian form. +func (curve *Curve) ScalarBaseMult(k []byte) (*big.Int, *big.Int) { + return curve.ScalarMult(curve.Gx, curve.Gy, k) +} + +var mask = []byte{0xff, 0x1, 0x3, 0x7, 0xf, 0x1f, 0x3f, 0x7f} + +// GenerateKey returns a public/private key pair. The private key is generated +// using the given reader, which must return random data. +func (curve *Curve) GenerateKey(rand io.Reader) (priv []byte, x, y *big.Int, err os.Error) { + byteLen := (curve.BitSize + 7) >> 3 + priv = make([]byte, byteLen) + + for x == nil { + _, err = io.ReadFull(rand, priv) + if err != nil { + return + } + // We have to mask off any excess bits in the case that the size of the + // underlying field is not a whole number of bytes. + priv[0] &= mask[curve.BitSize%8] + // This is because, in tests, rand will return all zeros and we don't + // want to get the point at infinity and loop forever. + priv[1] ^= 0x42 + x, y = curve.ScalarBaseMult(priv) + } + return +} + +// Marshal converts a point into the form specified in section 4.3.6 of ANSI +// X9.62. +func (curve *Curve) Marshal(x, y *big.Int) []byte { + byteLen := (curve.BitSize + 7) >> 3 + + ret := make([]byte, 1+2*byteLen) + ret[0] = 4 // uncompressed point + + xBytes := x.Bytes() + copy(ret[1+byteLen-len(xBytes):], xBytes) + yBytes := y.Bytes() + copy(ret[1+2*byteLen-len(yBytes):], yBytes) + return ret +} + +// Unmarshal converts a point, serialized by Marshal, into an x, y pair. On +// error, x = nil. +func (curve *Curve) Unmarshal(data []byte) (x, y *big.Int) { + byteLen := (curve.BitSize + 7) >> 3 + if len(data) != 1+2*byteLen { + return + } + if data[0] != 4 { // uncompressed form + return + } + x = new(big.Int).SetBytes(data[1 : 1+byteLen]) + y = new(big.Int).SetBytes(data[1+byteLen:]) + return +} + +var initonce sync.Once +var p224 *Curve +var p256 *Curve +var p384 *Curve +var p521 *Curve + +func initAll() { + initP224() + initP256() + initP384() + initP521() +} + +func initP224() { + // See FIPS 186-3, section D.2.2 + p224 = new(Curve) + p224.P, _ = new(big.Int).SetString("26959946667150639794667015087019630673557916260026308143510066298881", 10) + p224.N, _ = new(big.Int).SetString("26959946667150639794667015087019625940457807714424391721682722368061", 10) + p224.B, _ = new(big.Int).SetString("b4050a850c04b3abf54132565044b0b7d7bfd8ba270b39432355ffb4", 16) + p224.Gx, _ = new(big.Int).SetString("b70e0cbd6bb4bf7f321390b94a03c1d356c21122343280d6115c1d21", 16) + p224.Gy, _ = new(big.Int).SetString("bd376388b5f723fb4c22dfe6cd4375a05a07476444d5819985007e34", 16) + p224.BitSize = 224 +} + +func initP256() { + // See FIPS 186-3, section D.2.3 + p256 = new(Curve) + p256.P, _ = new(big.Int).SetString("115792089210356248762697446949407573530086143415290314195533631308867097853951", 10) + p256.N, _ = new(big.Int).SetString("115792089210356248762697446949407573529996955224135760342422259061068512044369", 10) + p256.B, _ = new(big.Int).SetString("5ac635d8aa3a93e7b3ebbd55769886bc651d06b0cc53b0f63bce3c3e27d2604b", 16) + p256.Gx, _ = new(big.Int).SetString("6b17d1f2e12c4247f8bce6e563a440f277037d812deb33a0f4a13945d898c296", 16) + p256.Gy, _ = new(big.Int).SetString("4fe342e2fe1a7f9b8ee7eb4a7c0f9e162bce33576b315ececbb6406837bf51f5", 16) + p256.BitSize = 256 +} + +func initP384() { + // See FIPS 186-3, section D.2.4 + p384 = new(Curve) + p384.P, _ = new(big.Int).SetString("39402006196394479212279040100143613805079739270465446667948293404245721771496870329047266088258938001861606973112319", 10) + p384.N, _ = new(big.Int).SetString("39402006196394479212279040100143613805079739270465446667946905279627659399113263569398956308152294913554433653942643", 10) + p384.B, _ = new(big.Int).SetString("b3312fa7e23ee7e4988e056be3f82d19181d9c6efe8141120314088f5013875ac656398d8a2ed19d2a85c8edd3ec2aef", 16) + p384.Gx, _ = new(big.Int).SetString("aa87ca22be8b05378eb1c71ef320ad746e1d3b628ba79b9859f741e082542a385502f25dbf55296c3a545e3872760ab7", 16) + p384.Gy, _ = new(big.Int).SetString("3617de4a96262c6f5d9e98bf9292dc29f8f41dbd289a147ce9da3113b5f0b8c00a60b1ce1d7e819d7a431d7c90ea0e5f", 16) + p384.BitSize = 384 +} + +func initP521() { + // See FIPS 186-3, section D.2.5 + p521 = new(Curve) + p521.P, _ = new(big.Int).SetString("6864797660130609714981900799081393217269435300143305409394463459185543183397656052122559640661454554977296311391480858037121987999716643812574028291115057151", 10) + p521.N, _ = new(big.Int).SetString("6864797660130609714981900799081393217269435300143305409394463459185543183397655394245057746333217197532963996371363321113864768612440380340372808892707005449", 10) + p521.B, _ = new(big.Int).SetString("051953eb9618e1c9a1f929a21a0b68540eea2da725b99b315f3b8b489918ef109e156193951ec7e937b1652c0bd3bb1bf073573df883d2c34f1ef451fd46b503f00", 16) + p521.Gx, _ = new(big.Int).SetString("c6858e06b70404e9cd9e3ecb662395b4429c648139053fb521f828af606b4d3dbaa14b5e77efe75928fe1dc127a2ffa8de3348b3c1856a429bf97e7e31c2e5bd66", 16) + p521.Gy, _ = new(big.Int).SetString("11839296a789a3bc0045c8a5fb42c7d1bd998f54449579b446817afbd17273e662c97ee72995ef42640c550b9013fad0761353c7086a272c24088be94769fd16650", 16) + p521.BitSize = 521 +} + +// P224 returns a Curve which implements P-224 (see FIPS 186-3, section D.2.2) +func P224() *Curve { + initonce.Do(initAll) + return p224 +} + +// P256 returns a Curve which implements P-256 (see FIPS 186-3, section D.2.3) +func P256() *Curve { + initonce.Do(initAll) + return p256 +} + +// P384 returns a Curve which implements P-384 (see FIPS 186-3, section D.2.4) +func P384() *Curve { + initonce.Do(initAll) + return p384 +} + +// P256 returns a Curve which implements P-521 (see FIPS 186-3, section D.2.5) +func P521() *Curve { + initonce.Do(initAll) + return p521 +} diff --git a/src/pkg/crypto/elliptic/elliptic_test.go b/src/pkg/crypto/elliptic/elliptic_test.go new file mode 100644 index 000000000..b7e7f035f --- /dev/null +++ b/src/pkg/crypto/elliptic/elliptic_test.go @@ -0,0 +1,334 @@ +// Copyright 2010 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package elliptic + +import ( + "big" + "crypto/rand" + "fmt" + "testing" +) + +func TestOnCurve(t *testing.T) { + p224 := P224() + if !p224.IsOnCurve(p224.Gx, p224.Gy) { + t.Errorf("FAIL") + } +} + +type baseMultTest struct { + k string + x, y string +} + +var p224BaseMultTests = []baseMultTest{ + { + "1", + "b70e0cbd6bb4bf7f321390b94a03c1d356c21122343280d6115c1d21", + "bd376388b5f723fb4c22dfe6cd4375a05a07476444d5819985007e34", + }, + { + "2", + "706a46dc76dcb76798e60e6d89474788d16dc18032d268fd1a704fa6", + "1c2b76a7bc25e7702a704fa986892849fca629487acf3709d2e4e8bb", + }, + { + "3", + "df1b1d66a551d0d31eff822558b9d2cc75c2180279fe0d08fd896d04", + "a3f7f03cadd0be444c0aa56830130ddf77d317344e1af3591981a925", + }, + { + "4", + "ae99feebb5d26945b54892092a8aee02912930fa41cd114e40447301", + "482580a0ec5bc47e88bc8c378632cd196cb3fa058a7114eb03054c9", + }, + { + "5", + "31c49ae75bce7807cdff22055d94ee9021fedbb5ab51c57526f011aa", + "27e8bff1745635ec5ba0c9f1c2ede15414c6507d29ffe37e790a079b", + }, + { + "6", + "1f2483f82572251fca975fea40db821df8ad82a3c002ee6c57112408", + "89faf0ccb750d99b553c574fad7ecfb0438586eb3952af5b4b153c7e", + }, + { + "7", + "db2f6be630e246a5cf7d99b85194b123d487e2d466b94b24a03c3e28", + "f3a30085497f2f611ee2517b163ef8c53b715d18bb4e4808d02b963", + }, + { + "8", + "858e6f9cc6c12c31f5df124aa77767b05c8bc021bd683d2b55571550", + "46dcd3ea5c43898c5c5fc4fdac7db39c2f02ebee4e3541d1e78047a", + }, + { + "9", + "2fdcccfee720a77ef6cb3bfbb447f9383117e3daa4a07e36ed15f78d", + "371732e4f41bf4f7883035e6a79fcedc0e196eb07b48171697517463", + }, + { + "10", + "aea9e17a306517eb89152aa7096d2c381ec813c51aa880e7bee2c0fd", + "39bb30eab337e0a521b6cba1abe4b2b3a3e524c14a3fe3eb116b655f", + }, + { + "11", + "ef53b6294aca431f0f3c22dc82eb9050324f1d88d377e716448e507c", + "20b510004092e96636cfb7e32efded8265c266dfb754fa6d6491a6da", + }, + { + "12", + "6e31ee1dc137f81b056752e4deab1443a481033e9b4c93a3044f4f7a", + "207dddf0385bfdeab6e9acda8da06b3bbef224a93ab1e9e036109d13", + }, + { + "13", + "34e8e17a430e43289793c383fac9774247b40e9ebd3366981fcfaeca", + "252819f71c7fb7fbcb159be337d37d3336d7feb963724fdfb0ecb767", + }, + { + "14", + "a53640c83dc208603ded83e4ecf758f24c357d7cf48088b2ce01e9fa", + "d5814cd724199c4a5b974a43685fbf5b8bac69459c9469bc8f23ccaf", + }, + { + "15", + "baa4d8635511a7d288aebeedd12ce529ff102c91f97f867e21916bf9", + "979a5f4759f80f4fb4ec2e34f5566d595680a11735e7b61046127989", + }, + { + "16", + "b6ec4fe1777382404ef679997ba8d1cc5cd8e85349259f590c4c66d", + "3399d464345906b11b00e363ef429221f2ec720d2f665d7dead5b482", + }, + { + "17", + "b8357c3a6ceef288310e17b8bfeff9200846ca8c1942497c484403bc", + "ff149efa6606a6bd20ef7d1b06bd92f6904639dce5174db6cc554a26", + }, + { + "18", + "c9ff61b040874c0568479216824a15eab1a838a797d189746226e4cc", + "ea98d60e5ffc9b8fcf999fab1df7e7ef7084f20ddb61bb045a6ce002", + }, + { + "19", + "a1e81c04f30ce201c7c9ace785ed44cc33b455a022f2acdbc6cae83c", + "dcf1f6c3db09c70acc25391d492fe25b4a180babd6cea356c04719cd", + }, + { + "20", + "fcc7f2b45df1cd5a3c0c0731ca47a8af75cfb0347e8354eefe782455", + "d5d7110274cba7cdee90e1a8b0d394c376a5573db6be0bf2747f530", + }, + { + "112233445566778899", + "61f077c6f62ed802dad7c2f38f5c67f2cc453601e61bd076bb46179e", + "2272f9e9f5933e70388ee652513443b5e289dd135dcc0d0299b225e4", + }, + { + "112233445566778899112233445566778899", + "29895f0af496bfc62b6ef8d8a65c88c613949b03668aab4f0429e35", + "3ea6e53f9a841f2019ec24bde1a75677aa9b5902e61081c01064de93", + }, + { + "6950511619965839450988900688150712778015737983940691968051900319680", + "ab689930bcae4a4aa5f5cb085e823e8ae30fd365eb1da4aba9cf0379", + "3345a121bbd233548af0d210654eb40bab788a03666419be6fbd34e7", + }, + { + "13479972933410060327035789020509431695094902435494295338570602119423", + "bdb6a8817c1f89da1c2f3dd8e97feb4494f2ed302a4ce2bc7f5f4025", + "4c7020d57c00411889462d77a5438bb4e97d177700bf7243a07f1680", + }, + { + "13479971751745682581351455311314208093898607229429740618390390702079", + "d58b61aa41c32dd5eba462647dba75c5d67c83606c0af2bd928446a9", + "d24ba6a837be0460dd107ae77725696d211446c5609b4595976b16bd", + }, + { + "13479972931865328106486971546324465392952975980343228160962702868479", + "dc9fa77978a005510980e929a1485f63716df695d7a0c18bb518df03", + "ede2b016f2ddffc2a8c015b134928275ce09e5661b7ab14ce0d1d403", + }, + { + "11795773708834916026404142434151065506931607341523388140225443265536", + "499d8b2829cfb879c901f7d85d357045edab55028824d0f05ba279ba", + "bf929537b06e4015919639d94f57838fa33fc3d952598dcdbb44d638", + }, + { + "784254593043826236572847595991346435467177662189391577090", + "8246c999137186632c5f9eddf3b1b0e1764c5e8bd0e0d8a554b9cb77", + "e80ed8660bc1cb17ac7d845be40a7a022d3306f116ae9f81fea65947", + }, + { + "13479767645505654746623887797783387853576174193480695826442858012671", + "6670c20afcceaea672c97f75e2e9dd5c8460e54bb38538ebb4bd30eb", + "f280d8008d07a4caf54271f993527d46ff3ff46fd1190a3f1faa4f74", + }, + { + "205688069665150753842126177372015544874550518966168735589597183", + "eca934247425cfd949b795cb5ce1eff401550386e28d1a4c5a8eb", + "d4c01040dba19628931bc8855370317c722cbd9ca6156985f1c2e9ce", + }, + { + "13479966930919337728895168462090683249159702977113823384618282123295", + "ef353bf5c73cd551b96d596fbc9a67f16d61dd9fe56af19de1fba9cd", + "21771b9cdce3e8430c09b3838be70b48c21e15bc09ee1f2d7945b91f", + }, + { + "50210731791415612487756441341851895584393717453129007497216", + "4036052a3091eb481046ad3289c95d3ac905ca0023de2c03ecd451cf", + "d768165a38a2b96f812586a9d59d4136035d9c853a5bf2e1c86a4993", + }, + { + "26959946667150639794667015087019625940457807714424391721682722368041", + "fcc7f2b45df1cd5a3c0c0731ca47a8af75cfb0347e8354eefe782455", + "f2a28eefd8b345832116f1e574f2c6b2c895aa8c24941f40d8b80ad1", + }, + { + "26959946667150639794667015087019625940457807714424391721682722368042", + "a1e81c04f30ce201c7c9ace785ed44cc33b455a022f2acdbc6cae83c", + "230e093c24f638f533dac6e2b6d01da3b5e7f45429315ca93fb8e634", + }, + { + "26959946667150639794667015087019625940457807714424391721682722368043", + "c9ff61b040874c0568479216824a15eab1a838a797d189746226e4cc", + "156729f1a003647030666054e208180f8f7b0df2249e44fba5931fff", + }, + { + "26959946667150639794667015087019625940457807714424391721682722368044", + "b8357c3a6ceef288310e17b8bfeff9200846ca8c1942497c484403bc", + "eb610599f95942df1082e4f9426d086fb9c6231ae8b24933aab5db", + }, + { + "26959946667150639794667015087019625940457807714424391721682722368045", + "b6ec4fe1777382404ef679997ba8d1cc5cd8e85349259f590c4c66d", + "cc662b9bcba6f94ee4ff1c9c10bd6ddd0d138df2d099a282152a4b7f", + }, + { + "26959946667150639794667015087019625940457807714424391721682722368046", + "baa4d8635511a7d288aebeedd12ce529ff102c91f97f867e21916bf9", + "6865a0b8a607f0b04b13d1cb0aa992a5a97f5ee8ca1849efb9ed8678", + }, + { + "26959946667150639794667015087019625940457807714424391721682722368047", + "a53640c83dc208603ded83e4ecf758f24c357d7cf48088b2ce01e9fa", + "2a7eb328dbe663b5a468b5bc97a040a3745396ba636b964370dc3352", + }, + { + "26959946667150639794667015087019625940457807714424391721682722368048", + "34e8e17a430e43289793c383fac9774247b40e9ebd3366981fcfaeca", + "dad7e608e380480434ea641cc82c82cbc92801469c8db0204f13489a", + }, + { + "26959946667150639794667015087019625940457807714424391721682722368049", + "6e31ee1dc137f81b056752e4deab1443a481033e9b4c93a3044f4f7a", + "df82220fc7a4021549165325725f94c3410ddb56c54e161fc9ef62ee", + }, + { + "26959946667150639794667015087019625940457807714424391721682722368050", + "ef53b6294aca431f0f3c22dc82eb9050324f1d88d377e716448e507c", + "df4aefffbf6d1699c930481cd102127c9a3d992048ab05929b6e5927", + }, + { + "26959946667150639794667015087019625940457807714424391721682722368051", + "aea9e17a306517eb89152aa7096d2c381ec813c51aa880e7bee2c0fd", + "c644cf154cc81f5ade49345e541b4d4b5c1adb3eb5c01c14ee949aa2", + }, + { + "26959946667150639794667015087019625940457807714424391721682722368052", + "2fdcccfee720a77ef6cb3bfbb447f9383117e3daa4a07e36ed15f78d", + "c8e8cd1b0be40b0877cfca1958603122f1e6914f84b7e8e968ae8b9e", + }, + { + "26959946667150639794667015087019625940457807714424391721682722368053", + "858e6f9cc6c12c31f5df124aa77767b05c8bc021bd683d2b55571550", + "fb9232c15a3bc7673a3a03b0253824c53d0fd1411b1cabe2e187fb87", + }, + { + "26959946667150639794667015087019625940457807714424391721682722368054", + "db2f6be630e246a5cf7d99b85194b123d487e2d466b94b24a03c3e28", + "f0c5cff7ab680d09ee11dae84e9c1072ac48ea2e744b1b7f72fd469e", + }, + { + "26959946667150639794667015087019625940457807714424391721682722368055", + "1f2483f82572251fca975fea40db821df8ad82a3c002ee6c57112408", + "76050f3348af2664aac3a8b05281304ebc7a7914c6ad50a4b4eac383", + }, + { + "26959946667150639794667015087019625940457807714424391721682722368056", + "31c49ae75bce7807cdff22055d94ee9021fedbb5ab51c57526f011aa", + "d817400e8ba9ca13a45f360e3d121eaaeb39af82d6001c8186f5f866", + }, + { + "26959946667150639794667015087019625940457807714424391721682722368057", + "ae99feebb5d26945b54892092a8aee02912930fa41cd114e40447301", + "fb7da7f5f13a43b81774373c879cd32d6934c05fa758eeb14fcfab38", + }, + { + "26959946667150639794667015087019625940457807714424391721682722368058", + "df1b1d66a551d0d31eff822558b9d2cc75c2180279fe0d08fd896d04", + "5c080fc3522f41bbb3f55a97cfecf21f882ce8cbb1e50ca6e67e56dc", + }, + { + "26959946667150639794667015087019625940457807714424391721682722368059", + "706a46dc76dcb76798e60e6d89474788d16dc18032d268fd1a704fa6", + "e3d4895843da188fd58fb0567976d7b50359d6b78530c8f62d1b1746", + }, + { + "26959946667150639794667015087019625940457807714424391721682722368060", + "b70e0cbd6bb4bf7f321390b94a03c1d356c21122343280d6115c1d21", + "42c89c774a08dc04b3dd201932bc8a5ea5f8b89bbb2a7e667aff81cd", + }, +} + +func TestBaseMult(t *testing.T) { + p224 := P224() + for i, e := range p224BaseMultTests { + k, ok := new(big.Int).SetString(e.k, 10) + if !ok { + t.Errorf("%d: bad value for k: %s", i, e.k) + } + x, y := p224.ScalarBaseMult(k.Bytes()) + if fmt.Sprintf("%x", x) != e.x || fmt.Sprintf("%x", y) != e.y { + t.Errorf("%d: bad output for k=%s: got (%x, %s), want (%s, %s)", i, e.k, x, y, e.x, e.y) + } + if testing.Short() && i > 5 { + break + } + } +} + +func BenchmarkBaseMult(b *testing.B) { + b.ResetTimer() + p224 := P224() + e := p224BaseMultTests[25] + k, _ := new(big.Int).SetString(e.k, 10) + b.StartTimer() + for i := 0; i < b.N; i++ { + p224.ScalarBaseMult(k.Bytes()) + } +} + +func TestMarshal(t *testing.T) { + p224 := P224() + _, x, y, err := p224.GenerateKey(rand.Reader) + if err != nil { + t.Error(err) + return + } + serialized := p224.Marshal(x, y) + xx, yy := p224.Unmarshal(serialized) + if xx == nil { + t.Error("failed to unmarshal") + return + } + if xx.Cmp(x) != 0 || yy.Cmp(y) != 0 { + t.Error("unmarshal returned different values") + return + } +} diff --git a/src/pkg/crypto/hmac/Makefile b/src/pkg/crypto/hmac/Makefile new file mode 100644 index 000000000..cc69abf60 --- /dev/null +++ b/src/pkg/crypto/hmac/Makefile @@ -0,0 +1,11 @@ +# Copyright 2009 The Go Authors. All rights reserved. +# Use of this source code is governed by a BSD-style +# license that can be found in the LICENSE file. + +include ../../../Make.inc + +TARG=crypto/hmac +GOFILES=\ + hmac.go\ + +include ../../../Make.pkg diff --git a/src/pkg/crypto/hmac/hmac.go b/src/pkg/crypto/hmac/hmac.go new file mode 100644 index 000000000..04ec86e9a --- /dev/null +++ b/src/pkg/crypto/hmac/hmac.go @@ -0,0 +1,100 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Package hmac implements the Keyed-Hash Message Authentication Code (HMAC) as +// defined in U.S. Federal Information Processing Standards Publication 198. +// An HMAC is a cryptographic hash that uses a key to sign a message. +// The receiver verifies the hash by recomputing it using the same key. +package hmac + +import ( + "crypto/md5" + "crypto/sha1" + "crypto/sha256" + "hash" + "os" +) + +// FIPS 198: +// http://csrc.nist.gov/publications/fips/fips198/fips-198a.pdf + +// key is zero padded to 64 bytes +// ipad = 0x36 byte repeated to 64 bytes +// opad = 0x5c byte repeated to 64 bytes +// hmac = H([key ^ opad] H([key ^ ipad] text)) + +const ( + // NOTE(rsc): This constant is actually the + // underlying hash function's block size. + // HMAC is only conventionally used with + // MD5 and SHA1, and both use 64-byte blocks. + // The hash.Hash interface doesn't provide a + // way to find out the block size. + padSize = 64 +) + +type hmac struct { + size int + key, tmp []byte + outer, inner hash.Hash +} + +func (h *hmac) tmpPad(xor byte) { + for i, k := range h.key { + h.tmp[i] = xor ^ k + } + for i := len(h.key); i < padSize; i++ { + h.tmp[i] = xor + } +} + +func (h *hmac) Sum() []byte { + sum := h.inner.Sum() + h.tmpPad(0x5c) + for i, b := range sum { + h.tmp[padSize+i] = b + } + h.outer.Reset() + h.outer.Write(h.tmp) + return h.outer.Sum() +} + +func (h *hmac) Write(p []byte) (n int, err os.Error) { + return h.inner.Write(p) +} + +func (h *hmac) Size() int { return h.size } + +func (h *hmac) Reset() { + h.inner.Reset() + h.tmpPad(0x36) + h.inner.Write(h.tmp[0:padSize]) +} + +// New returns a new HMAC hash using the given hash generator and key. +func New(h func() hash.Hash, key []byte) hash.Hash { + hm := new(hmac) + hm.outer = h() + hm.inner = h() + hm.size = hm.inner.Size() + hm.tmp = make([]byte, padSize+hm.size) + if len(key) > padSize { + // If key is too big, hash it. + hm.outer.Write(key) + key = hm.outer.Sum() + } + hm.key = make([]byte, len(key)) + copy(hm.key, key) + hm.Reset() + return hm +} + +// NewMD5 returns a new HMAC-MD5 hash using the given key. +func NewMD5(key []byte) hash.Hash { return New(md5.New, key) } + +// NewSHA1 returns a new HMAC-SHA1 hash using the given key. +func NewSHA1(key []byte) hash.Hash { return New(sha1.New, key) } + +// NewSHA256 returns a new HMAC-SHA256 hash using the given key. +func NewSHA256(key []byte) hash.Hash { return New(sha256.New, key) } diff --git a/src/pkg/crypto/hmac/hmac_test.go b/src/pkg/crypto/hmac/hmac_test.go new file mode 100644 index 000000000..bcae63b8a --- /dev/null +++ b/src/pkg/crypto/hmac/hmac_test.go @@ -0,0 +1,205 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package hmac + +import ( + "hash" + "fmt" + "testing" +) + +type hmacTest struct { + hash func([]byte) hash.Hash + key []byte + in []byte + out string +} + +var hmacTests = []hmacTest{ + // Tests from US FIPS 198 + // http://csrc.nist.gov/publications/fips/fips198/fips-198a.pdf + { + NewSHA1, + []byte{ + 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, + 0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f, + 0x10, 0x11, 0x12, 0x13, 0x14, 0x15, 0x16, 0x17, + 0x18, 0x19, 0x1a, 0x1b, 0x1c, 0x1d, 0x1e, 0x1f, + 0x20, 0x21, 0x22, 0x23, 0x24, 0x25, 0x26, 0x27, + 0x28, 0x29, 0x2a, 0x2b, 0x2c, 0x2d, 0x2e, 0x2f, + 0x30, 0x31, 0x32, 0x33, 0x34, 0x35, 0x36, 0x37, + 0x38, 0x39, 0x3a, 0x3b, 0x3c, 0x3d, 0x3e, 0x3f, + }, + []byte("Sample #1"), + "4f4ca3d5d68ba7cc0a1208c9c61e9c5da0403c0a", + }, + { + NewSHA1, + []byte{ + 0x30, 0x31, 0x32, 0x33, 0x34, 0x35, 0x36, 0x37, + 0x38, 0x39, 0x3a, 0x3b, 0x3c, 0x3d, 0x3e, 0x3f, + 0x40, 0x41, 0x42, 0x43, + }, + []byte("Sample #2"), + "0922d3405faa3d194f82a45830737d5cc6c75d24", + }, + { + NewSHA1, + []byte{ + 0x50, 0x51, 0x52, 0x53, 0x54, 0x55, 0x56, 0x57, + 0x58, 0x59, 0x5a, 0x5b, 0x5c, 0x5d, 0x5e, 0x5f, + 0x60, 0x61, 0x62, 0x63, 0x64, 0x65, 0x66, 0x67, + 0x68, 0x69, 0x6a, 0x6b, 0x6c, 0x6d, 0x6e, 0x6f, + 0x70, 0x71, 0x72, 0x73, 0x74, 0x75, 0x76, 0x77, + 0x78, 0x79, 0x7a, 0x7b, 0x7c, 0x7d, 0x7e, 0x7f, + 0x80, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87, + 0x88, 0x89, 0x8a, 0x8b, 0x8c, 0x8d, 0x8e, 0x8f, + 0x90, 0x91, 0x92, 0x93, 0x94, 0x95, 0x96, 0x97, + 0x98, 0x99, 0x9a, 0x9b, 0x9c, 0x9d, 0x9e, 0x9f, + 0xa0, 0xa1, 0xa2, 0xa3, 0xa4, 0xa5, 0xa6, 0xa7, + 0xa8, 0xa9, 0xaa, 0xab, 0xac, 0xad, 0xae, 0xaf, + 0xb0, 0xb1, 0xb2, 0xb3, + }, + []byte("Sample #3"), + "bcf41eab8bb2d802f3d05caf7cb092ecf8d1a3aa", + }, + + // Test from Plan 9. + { + NewMD5, + []byte("Jefe"), + []byte("what do ya want for nothing?"), + "750c783e6ab0b503eaa86e310a5db738", + }, + + // Tests from RFC 4231 + { + NewSHA256, + []byte{ + 0x0b, 0x0b, 0x0b, 0x0b, 0x0b, 0x0b, 0x0b, 0x0b, + 0x0b, 0x0b, 0x0b, 0x0b, 0x0b, 0x0b, 0x0b, 0x0b, + 0x0b, 0x0b, 0x0b, 0x0b, + }, + []byte("Hi There"), + "b0344c61d8db38535ca8afceaf0bf12b881dc200c9833da726e9376c2e32cff7", + }, + { + NewSHA256, + []byte("Jefe"), + []byte("what do ya want for nothing?"), + "5bdcc146bf60754e6a042426089575c75a003f089d2739839dec58b964ec3843", + }, + { + NewSHA256, + []byte{ + 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, + 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, + 0xaa, 0xaa, 0xaa, 0xaa, + }, + []byte{ + 0xdd, 0xdd, 0xdd, 0xdd, 0xdd, 0xdd, 0xdd, 0xdd, + 0xdd, 0xdd, 0xdd, 0xdd, 0xdd, 0xdd, 0xdd, 0xdd, + 0xdd, 0xdd, 0xdd, 0xdd, 0xdd, 0xdd, 0xdd, 0xdd, + 0xdd, 0xdd, 0xdd, 0xdd, 0xdd, 0xdd, 0xdd, 0xdd, + 0xdd, 0xdd, 0xdd, 0xdd, 0xdd, 0xdd, 0xdd, 0xdd, + 0xdd, 0xdd, 0xdd, 0xdd, 0xdd, 0xdd, 0xdd, 0xdd, + 0xdd, 0xdd, + }, + "773ea91e36800e46854db8ebd09181a72959098b3ef8c122d9635514ced565fe", + }, + { + NewSHA256, + []byte{ + 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, + 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f, 0x10, + 0x11, 0x12, 0x13, 0x14, 0x15, 0x16, 0x17, 0x18, + 0x19, + }, + []byte{ + 0xcd, 0xcd, 0xcd, 0xcd, 0xcd, 0xcd, 0xcd, 0xcd, + 0xcd, 0xcd, 0xcd, 0xcd, 0xcd, 0xcd, 0xcd, 0xcd, + 0xcd, 0xcd, 0xcd, 0xcd, 0xcd, 0xcd, 0xcd, 0xcd, + 0xcd, 0xcd, 0xcd, 0xcd, 0xcd, 0xcd, 0xcd, 0xcd, + 0xcd, 0xcd, 0xcd, 0xcd, 0xcd, 0xcd, 0xcd, 0xcd, + 0xcd, 0xcd, 0xcd, 0xcd, 0xcd, 0xcd, 0xcd, 0xcd, + 0xcd, 0xcd, + }, + "82558a389a443c0ea4cc819899f2083a85f0faa3e578f8077a2e3ff46729665b", + }, + { + NewSHA256, + []byte{ + 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, + 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, + 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, + 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, + 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, + 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, + 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, + 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, + 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, + 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, + 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, + 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, + 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, + 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, + 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, + 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, + 0xaa, 0xaa, 0xaa, + }, + []byte("Test Using Larger Than Block-Size Key - Hash Key First"), + "60e431591ee0b67f0d8a26aacbf5b77f8e0bc6213728c5140546040f0ee37f54", + }, + { + NewSHA256, + []byte{ + 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, + 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, + 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, + 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, + 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, + 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, + 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, + 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, + 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, + 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, + 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, + 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, + 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, + 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, + 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, + 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, + 0xaa, 0xaa, 0xaa, + }, + []byte("This is a test using a larger than block-size key " + + "and a larger than block-size data. The key needs to " + + "be hashed before being used by the HMAC algorithm."), + "9b09ffa71b942fcb27635fbcd5b0e944bfdc63644f0713938a7f51535c3a35e2", + }, +} + +func TestHMAC(t *testing.T) { + for i, tt := range hmacTests { + h := tt.hash(tt.key) + for j := 0; j < 2; j++ { + n, err := h.Write(tt.in) + if n != len(tt.in) || err != nil { + t.Errorf("test %d.%d: Write(%d) = %d, %v", i, j, len(tt.in), n, err) + continue + } + + // Repetitive Sum() calls should return the same value + for k := 0; k < 2; k++ { + sum := fmt.Sprintf("%x", h.Sum()) + if sum != tt.out { + t.Errorf("test %d.%d.%d: have %s want %s\n", i, j, k, sum, tt.out) + } + } + + // Second iteration: make sure reset works. + h.Reset() + } + } +} diff --git a/src/pkg/crypto/md4/Makefile b/src/pkg/crypto/md4/Makefile new file mode 100644 index 000000000..eef05ab70 --- /dev/null +++ b/src/pkg/crypto/md4/Makefile @@ -0,0 +1,12 @@ +# Copyright 2009 The Go Authors. All rights reserved. +# Use of this source code is governed by a BSD-style +# license that can be found in the LICENSE file. + +include ../../../Make.inc + +TARG=crypto/md4 +GOFILES=\ + md4.go\ + md4block.go\ + +include ../../../Make.pkg diff --git a/src/pkg/crypto/md4/md4.go b/src/pkg/crypto/md4/md4.go new file mode 100644 index 000000000..848d9552d --- /dev/null +++ b/src/pkg/crypto/md4/md4.go @@ -0,0 +1,117 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Package md4 implements the MD4 hash algorithm as defined in RFC 1320. +package md4 + +import ( + "crypto" + "hash" + "os" +) + +func init() { + crypto.RegisterHash(crypto.MD4, New) +} + +// The size of an MD4 checksum in bytes. +const Size = 16 + +const ( + _Chunk = 64 + _Init0 = 0x67452301 + _Init1 = 0xEFCDAB89 + _Init2 = 0x98BADCFE + _Init3 = 0x10325476 +) + +// digest represents the partial evaluation of a checksum. +type digest struct { + s [4]uint32 + x [_Chunk]byte + nx int + len uint64 +} + +func (d *digest) Reset() { + d.s[0] = _Init0 + d.s[1] = _Init1 + d.s[2] = _Init2 + d.s[3] = _Init3 + d.nx = 0 + d.len = 0 +} + +// New returns a new hash.Hash computing the MD4 checksum. +func New() hash.Hash { + d := new(digest) + d.Reset() + return d +} + +func (d *digest) Size() int { return Size } + +func (d *digest) Write(p []byte) (nn int, err os.Error) { + nn = len(p) + d.len += uint64(nn) + if d.nx > 0 { + n := len(p) + if n > _Chunk-d.nx { + n = _Chunk - d.nx + } + for i := 0; i < n; i++ { + d.x[d.nx+i] = p[i] + } + d.nx += n + if d.nx == _Chunk { + _Block(d, d.x[0:]) + d.nx = 0 + } + p = p[n:] + } + n := _Block(d, p) + p = p[n:] + if len(p) > 0 { + d.nx = copy(d.x[:], p) + } + return +} + +func (d0 *digest) Sum() []byte { + // Make a copy of d0, so that caller can keep writing and summing. + d := new(digest) + *d = *d0 + + // Padding. Add a 1 bit and 0 bits until 56 bytes mod 64. + len := d.len + var tmp [64]byte + tmp[0] = 0x80 + if len%64 < 56 { + d.Write(tmp[0 : 56-len%64]) + } else { + d.Write(tmp[0 : 64+56-len%64]) + } + + // Length in bits. + len <<= 3 + for i := uint(0); i < 8; i++ { + tmp[i] = byte(len >> (8 * i)) + } + d.Write(tmp[0:8]) + + if d.nx != 0 { + panic("d.nx != 0") + } + + p := make([]byte, 16) + j := 0 + for _, s := range d.s { + p[j+0] = byte(s >> 0) + p[j+1] = byte(s >> 8) + p[j+2] = byte(s >> 16) + p[j+3] = byte(s >> 24) + j += 4 + } + return p +} diff --git a/src/pkg/crypto/md4/md4_test.go b/src/pkg/crypto/md4/md4_test.go new file mode 100644 index 000000000..721bd4cbc --- /dev/null +++ b/src/pkg/crypto/md4/md4_test.go @@ -0,0 +1,71 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package md4 + +import ( + "fmt" + "io" + "testing" +) + +type md4Test struct { + out string + in string +} + +var golden = []md4Test{ + {"31d6cfe0d16ae931b73c59d7e0c089c0", ""}, + {"bde52cb31de33e46245e05fbdbd6fb24", "a"}, + {"ec388dd78999dfc7cf4632465693b6bf", "ab"}, + {"a448017aaf21d8525fc10ae87aa6729d", "abc"}, + {"41decd8f579255c5200f86a4bb3ba740", "abcd"}, + {"9803f4a34e8eb14f96adba49064a0c41", "abcde"}, + {"804e7f1c2586e50b49ac65db5b645131", "abcdef"}, + {"752f4adfe53d1da0241b5bc216d098fc", "abcdefg"}, + {"ad9daf8d49d81988590a6f0e745d15dd", "abcdefgh"}, + {"1e4e28b05464316b56402b3815ed2dfd", "abcdefghi"}, + {"dc959c6f5d6f9e04e4380777cc964b3d", "abcdefghij"}, + {"1b5701e265778898ef7de5623bbe7cc0", "Discard medicine more than two years old."}, + {"d7f087e090fe7ad4a01cb59dacc9a572", "He who has a shady past knows that nice guys finish last."}, + {"a6f8fd6df617c72837592fc3570595c9", "I wouldn't marry him with a ten foot pole."}, + {"c92a84a9526da8abc240c05d6b1a1ce0", "Free! Free!/A trip/to Mars/for 900/empty jars/Burma Shave"}, + {"f6013160c4dcb00847069fee3bb09803", "The days of the digital watch are numbered. -Tom Stoppard"}, + {"2c3bb64f50b9107ed57640fe94bec09f", "Nepal premier won't resign."}, + {"45b7d8a32c7806f2f7f897332774d6e4", "For every action there is an equal and opposite government program."}, + {"b5b4f9026b175c62d7654bdc3a1cd438", "His money is twice tainted: 'taint yours and 'taint mine."}, + {"caf44e80f2c20ce19b5ba1cab766e7bd", "There is no reason for any individual to have a computer in their home. -Ken Olsen, 1977"}, + {"191fae6707f496aa54a6bce9f2ecf74d", "It's a tiny change to the code and not completely disgusting. - Bob Manchek"}, + {"9ddc753e7a4ccee6081cd1b45b23a834", "size: a.out: bad magic"}, + {"8d050f55b1cadb9323474564be08a521", "The major problem is with sendmail. -Mark Horton"}, + {"ad6e2587f74c3e3cc19146f6127fa2e3", "Give me a rock, paper and scissors and I will move the world. CCFestoon"}, + {"1d616d60a5fabe85589c3f1566ca7fca", "If the enemy is within range, then so are you."}, + {"aec3326a4f496a2ced65a1963f84577f", "It's well we cannot hear the screams/That we create in others' dreams."}, + {"77b4fd762d6b9245e61c50bf6ebf118b", "You remind me of a TV show, but that's all right: I watch it anyway."}, + {"e8f48c726bae5e516f6ddb1a4fe62438", "C is as portable as Stonehedge!!"}, + {"a3a84366e7219e887423b01f9be7166e", "Even if I could be Shakespeare, I think I should still choose to be Faraday. - A. Huxley"}, + {"a6b7aa35157e984ef5d9b7f32e5fbb52", "The fugacity of a constituent in a mixture of gases at a given temperature is proportional to its mole fraction. Lewis-Randall Rule"}, + {"75661f0545955f8f9abeeb17845f3fd6", "How can you write a big system without C++? -Paul Glick"}, +} + +func TestGolden(t *testing.T) { + for i := 0; i < len(golden); i++ { + g := golden[i] + c := New() + for j := 0; j < 3; j++ { + if j < 2 { + io.WriteString(c, g.in) + } else { + io.WriteString(c, g.in[0:len(g.in)/2]) + c.Sum() + io.WriteString(c, g.in[len(g.in)/2:]) + } + s := fmt.Sprintf("%x", c.Sum()) + if s != g.out { + t.Fatalf("md4[%d](%s) = %s want %s", j, g.in, s, g.out) + } + c.Reset() + } + } +} diff --git a/src/pkg/crypto/md4/md4block.go b/src/pkg/crypto/md4/md4block.go new file mode 100644 index 000000000..3fed475f3 --- /dev/null +++ b/src/pkg/crypto/md4/md4block.go @@ -0,0 +1,89 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// MD4 block step. +// In its own file so that a faster assembly or C version +// can be substituted easily. + +package md4 + +var shift1 = []uint{3, 7, 11, 19} +var shift2 = []uint{3, 5, 9, 13} +var shift3 = []uint{3, 9, 11, 15} + +var xIndex2 = []uint{0, 4, 8, 12, 1, 5, 9, 13, 2, 6, 10, 14, 3, 7, 11, 15} +var xIndex3 = []uint{0, 8, 4, 12, 2, 10, 6, 14, 1, 9, 5, 13, 3, 11, 7, 15} + +func _Block(dig *digest, p []byte) int { + a := dig.s[0] + b := dig.s[1] + c := dig.s[2] + d := dig.s[3] + n := 0 + var X [16]uint32 + for len(p) >= _Chunk { + aa, bb, cc, dd := a, b, c, d + + j := 0 + for i := 0; i < 16; i++ { + X[i] = uint32(p[j]) | uint32(p[j+1])<<8 | uint32(p[j+2])<<16 | uint32(p[j+3])<<24 + j += 4 + } + + // If this needs to be made faster in the future, + // the usual trick is to unroll each of these + // loops by a factor of 4; that lets you replace + // the shift[] lookups with constants and, + // with suitable variable renaming in each + // unrolled body, delete the a, b, c, d = d, a, b, c + // (or you can let the optimizer do the renaming). + // + // The index variables are uint so that % by a power + // of two can be optimized easily by a compiler. + + // Round 1. + for i := uint(0); i < 16; i++ { + x := i + s := shift1[i%4] + f := ((c ^ d) & b) ^ d + a += f + X[x] + a = a<>(32-s) + a, b, c, d = d, a, b, c + } + + // Round 2. + for i := uint(0); i < 16; i++ { + x := xIndex2[i] + s := shift2[i%4] + g := (b & c) | (b & d) | (c & d) + a += g + X[x] + 0x5a827999 + a = a<>(32-s) + a, b, c, d = d, a, b, c + } + + // Round 3. + for i := uint(0); i < 16; i++ { + x := xIndex3[i] + s := shift3[i%4] + h := b ^ c ^ d + a += h + X[x] + 0x6ed9eba1 + a = a<>(32-s) + a, b, c, d = d, a, b, c + } + + a += aa + b += bb + c += cc + d += dd + + p = p[_Chunk:] + n += _Chunk + } + + dig.s[0] = a + dig.s[1] = b + dig.s[2] = c + dig.s[3] = d + return n +} diff --git a/src/pkg/crypto/md5/Makefile b/src/pkg/crypto/md5/Makefile new file mode 100644 index 000000000..5cde3e6d6 --- /dev/null +++ b/src/pkg/crypto/md5/Makefile @@ -0,0 +1,12 @@ +# Copyright 2009 The Go Authors. All rights reserved. +# Use of this source code is governed by a BSD-style +# license that can be found in the LICENSE file. + +include ../../../Make.inc + +TARG=crypto/md5 +GOFILES=\ + md5.go\ + md5block.go\ + +include ../../../Make.pkg diff --git a/src/pkg/crypto/md5/md5.go b/src/pkg/crypto/md5/md5.go new file mode 100644 index 000000000..378faa6ec --- /dev/null +++ b/src/pkg/crypto/md5/md5.go @@ -0,0 +1,117 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Package md5 implements the MD5 hash algorithm as defined in RFC 1321. +package md5 + +import ( + "crypto" + "hash" + "os" +) + +func init() { + crypto.RegisterHash(crypto.MD5, New) +} + +// The size of an MD5 checksum in bytes. +const Size = 16 + +const ( + _Chunk = 64 + _Init0 = 0x67452301 + _Init1 = 0xEFCDAB89 + _Init2 = 0x98BADCFE + _Init3 = 0x10325476 +) + +// digest represents the partial evaluation of a checksum. +type digest struct { + s [4]uint32 + x [_Chunk]byte + nx int + len uint64 +} + +func (d *digest) Reset() { + d.s[0] = _Init0 + d.s[1] = _Init1 + d.s[2] = _Init2 + d.s[3] = _Init3 + d.nx = 0 + d.len = 0 +} + +// New returns a new hash.Hash computing the MD5 checksum. +func New() hash.Hash { + d := new(digest) + d.Reset() + return d +} + +func (d *digest) Size() int { return Size } + +func (d *digest) Write(p []byte) (nn int, err os.Error) { + nn = len(p) + d.len += uint64(nn) + if d.nx > 0 { + n := len(p) + if n > _Chunk-d.nx { + n = _Chunk - d.nx + } + for i := 0; i < n; i++ { + d.x[d.nx+i] = p[i] + } + d.nx += n + if d.nx == _Chunk { + _Block(d, d.x[0:]) + d.nx = 0 + } + p = p[n:] + } + n := _Block(d, p) + p = p[n:] + if len(p) > 0 { + d.nx = copy(d.x[:], p) + } + return +} + +func (d0 *digest) Sum() []byte { + // Make a copy of d0 so that caller can keep writing and summing. + d := new(digest) + *d = *d0 + + // Padding. Add a 1 bit and 0 bits until 56 bytes mod 64. + len := d.len + var tmp [64]byte + tmp[0] = 0x80 + if len%64 < 56 { + d.Write(tmp[0 : 56-len%64]) + } else { + d.Write(tmp[0 : 64+56-len%64]) + } + + // Length in bits. + len <<= 3 + for i := uint(0); i < 8; i++ { + tmp[i] = byte(len >> (8 * i)) + } + d.Write(tmp[0:8]) + + if d.nx != 0 { + panic("d.nx != 0") + } + + p := make([]byte, 16) + j := 0 + for _, s := range d.s { + p[j+0] = byte(s >> 0) + p[j+1] = byte(s >> 8) + p[j+2] = byte(s >> 16) + p[j+3] = byte(s >> 24) + j += 4 + } + return p +} diff --git a/src/pkg/crypto/md5/md5_test.go b/src/pkg/crypto/md5/md5_test.go new file mode 100644 index 000000000..857002b70 --- /dev/null +++ b/src/pkg/crypto/md5/md5_test.go @@ -0,0 +1,71 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package md5 + +import ( + "fmt" + "io" + "testing" +) + +type md5Test struct { + out string + in string +} + +var golden = []md5Test{ + {"d41d8cd98f00b204e9800998ecf8427e", ""}, + {"0cc175b9c0f1b6a831c399e269772661", "a"}, + {"187ef4436122d1cc2f40dc2b92f0eba0", "ab"}, + {"900150983cd24fb0d6963f7d28e17f72", "abc"}, + {"e2fc714c4727ee9395f324cd2e7f331f", "abcd"}, + {"ab56b4d92b40713acc5af89985d4b786", "abcde"}, + {"e80b5017098950fc58aad83c8c14978e", "abcdef"}, + {"7ac66c0f148de9519b8bd264312c4d64", "abcdefg"}, + {"e8dc4081b13434b45189a720b77b6818", "abcdefgh"}, + {"8aa99b1f439ff71293e95357bac6fd94", "abcdefghi"}, + {"a925576942e94b2ef57a066101b48876", "abcdefghij"}, + {"d747fc1719c7eacb84058196cfe56d57", "Discard medicine more than two years old."}, + {"bff2dcb37ef3a44ba43ab144768ca837", "He who has a shady past knows that nice guys finish last."}, + {"0441015ecb54a7342d017ed1bcfdbea5", "I wouldn't marry him with a ten foot pole."}, + {"9e3cac8e9e9757a60c3ea391130d3689", "Free! Free!/A trip/to Mars/for 900/empty jars/Burma Shave"}, + {"a0f04459b031f916a59a35cc482dc039", "The days of the digital watch are numbered. -Tom Stoppard"}, + {"e7a48e0fe884faf31475d2a04b1362cc", "Nepal premier won't resign."}, + {"637d2fe925c07c113800509964fb0e06", "For every action there is an equal and opposite government program."}, + {"834a8d18d5c6562119cf4c7f5086cb71", "His money is twice tainted: 'taint yours and 'taint mine."}, + {"de3a4d2fd6c73ec2db2abad23b444281", "There is no reason for any individual to have a computer in their home. -Ken Olsen, 1977"}, + {"acf203f997e2cf74ea3aff86985aefaf", "It's a tiny change to the code and not completely disgusting. - Bob Manchek"}, + {"e1c1384cb4d2221dfdd7c795a4222c9a", "size: a.out: bad magic"}, + {"c90f3ddecc54f34228c063d7525bf644", "The major problem is with sendmail. -Mark Horton"}, + {"cdf7ab6c1fd49bd9933c43f3ea5af185", "Give me a rock, paper and scissors and I will move the world. CCFestoon"}, + {"83bc85234942fc883c063cbd7f0ad5d0", "If the enemy is within range, then so are you."}, + {"277cbe255686b48dd7e8f389394d9299", "It's well we cannot hear the screams/That we create in others' dreams."}, + {"fd3fb0a7ffb8af16603f3d3af98f8e1f", "You remind me of a TV show, but that's all right: I watch it anyway."}, + {"469b13a78ebf297ecda64d4723655154", "C is as portable as Stonehedge!!"}, + {"63eb3a2f466410104731c4b037600110", "Even if I could be Shakespeare, I think I should still choose to be Faraday. - A. Huxley"}, + {"72c2ed7592debca1c90fc0100f931a2f", "The fugacity of a constituent in a mixture of gases at a given temperature is proportional to its mole fraction. Lewis-Randall Rule"}, + {"132f7619d33b523b1d9e5bd8e0928355", "How can you write a big system without C++? -Paul Glick"}, +} + +func TestGolden(t *testing.T) { + for i := 0; i < len(golden); i++ { + g := golden[i] + c := New() + for j := 0; j < 3; j++ { + if j < 2 { + io.WriteString(c, g.in) + } else { + io.WriteString(c, g.in[0:len(g.in)/2]) + c.Sum() + io.WriteString(c, g.in[len(g.in)/2:]) + } + s := fmt.Sprintf("%x", c.Sum()) + if s != g.out { + t.Fatalf("md5[%d](%s) = %s want %s", j, g.in, s, g.out) + } + c.Reset() + } + } +} diff --git a/src/pkg/crypto/md5/md5block.go b/src/pkg/crypto/md5/md5block.go new file mode 100644 index 000000000..a887e2e05 --- /dev/null +++ b/src/pkg/crypto/md5/md5block.go @@ -0,0 +1,172 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// MD5 block step. +// In its own file so that a faster assembly or C version +// can be substituted easily. + +package md5 + +// table[i] = int((1<<32) * abs(sin(i+1 radians))). +var table = []uint32{ + // round 1 + 0xd76aa478, + 0xe8c7b756, + 0x242070db, + 0xc1bdceee, + 0xf57c0faf, + 0x4787c62a, + 0xa8304613, + 0xfd469501, + 0x698098d8, + 0x8b44f7af, + 0xffff5bb1, + 0x895cd7be, + 0x6b901122, + 0xfd987193, + 0xa679438e, + 0x49b40821, + + // round 2 + 0xf61e2562, + 0xc040b340, + 0x265e5a51, + 0xe9b6c7aa, + 0xd62f105d, + 0x2441453, + 0xd8a1e681, + 0xe7d3fbc8, + 0x21e1cde6, + 0xc33707d6, + 0xf4d50d87, + 0x455a14ed, + 0xa9e3e905, + 0xfcefa3f8, + 0x676f02d9, + 0x8d2a4c8a, + + // round3 + 0xfffa3942, + 0x8771f681, + 0x6d9d6122, + 0xfde5380c, + 0xa4beea44, + 0x4bdecfa9, + 0xf6bb4b60, + 0xbebfbc70, + 0x289b7ec6, + 0xeaa127fa, + 0xd4ef3085, + 0x4881d05, + 0xd9d4d039, + 0xe6db99e5, + 0x1fa27cf8, + 0xc4ac5665, + + // round 4 + 0xf4292244, + 0x432aff97, + 0xab9423a7, + 0xfc93a039, + 0x655b59c3, + 0x8f0ccc92, + 0xffeff47d, + 0x85845dd1, + 0x6fa87e4f, + 0xfe2ce6e0, + 0xa3014314, + 0x4e0811a1, + 0xf7537e82, + 0xbd3af235, + 0x2ad7d2bb, + 0xeb86d391, +} + +var shift1 = []uint{7, 12, 17, 22} +var shift2 = []uint{5, 9, 14, 20} +var shift3 = []uint{4, 11, 16, 23} +var shift4 = []uint{6, 10, 15, 21} + +func _Block(dig *digest, p []byte) int { + a := dig.s[0] + b := dig.s[1] + c := dig.s[2] + d := dig.s[3] + n := 0 + var X [16]uint32 + for len(p) >= _Chunk { + aa, bb, cc, dd := a, b, c, d + + j := 0 + for i := 0; i < 16; i++ { + X[i] = uint32(p[j]) | uint32(p[j+1])<<8 | uint32(p[j+2])<<16 | uint32(p[j+3])<<24 + j += 4 + } + + // If this needs to be made faster in the future, + // the usual trick is to unroll each of these + // loops by a factor of 4; that lets you replace + // the shift[] lookups with constants and, + // with suitable variable renaming in each + // unrolled body, delete the a, b, c, d = d, a, b, c + // (or you can let the optimizer do the renaming). + // + // The index variables are uint so that % by a power + // of two can be optimized easily by a compiler. + + // Round 1. + for i := uint(0); i < 16; i++ { + x := i + s := shift1[i%4] + f := ((c ^ d) & b) ^ d + a += f + X[x] + table[i] + a = a<>(32-s) + b + a, b, c, d = d, a, b, c + } + + // Round 2. + for i := uint(0); i < 16; i++ { + x := (1 + 5*i) % 16 + s := shift2[i%4] + g := ((b ^ c) & d) ^ c + a += g + X[x] + table[i+16] + a = a<>(32-s) + b + a, b, c, d = d, a, b, c + } + + // Round 3. + for i := uint(0); i < 16; i++ { + x := (5 + 3*i) % 16 + s := shift3[i%4] + h := b ^ c ^ d + a += h + X[x] + table[i+32] + a = a<>(32-s) + b + a, b, c, d = d, a, b, c + } + + // Round 4. + for i := uint(0); i < 16; i++ { + x := (7 * i) % 16 + s := shift4[i%4] + j := c ^ (b | ^d) + a += j + X[x] + table[i+48] + a = a<>(32-s) + b + a, b, c, d = d, a, b, c + } + + a += aa + b += bb + c += cc + d += dd + + p = p[_Chunk:] + n += _Chunk + } + + dig.s[0] = a + dig.s[1] = b + dig.s[2] = c + dig.s[3] = d + return n +} diff --git a/src/pkg/crypto/ocsp/Makefile b/src/pkg/crypto/ocsp/Makefile new file mode 100644 index 000000000..6e132ff9b --- /dev/null +++ b/src/pkg/crypto/ocsp/Makefile @@ -0,0 +1,11 @@ +# Copyright 2009 The Go Authors. All rights reserved. +# Use of this source code is governed by a BSD-style +# license that can be found in the LICENSE file. + +include ../../../Make.inc + +TARG=crypto/ocsp +GOFILES=\ + ocsp.go\ + +include ../../../Make.pkg diff --git a/src/pkg/crypto/ocsp/ocsp.go b/src/pkg/crypto/ocsp/ocsp.go new file mode 100644 index 000000000..7ea7a1e82 --- /dev/null +++ b/src/pkg/crypto/ocsp/ocsp.go @@ -0,0 +1,192 @@ +// Copyright 2010 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Package ocsp parses OCSP responses as specified in RFC 2560. OCSP responses +// are signed messages attesting to the validity of a certificate for a small +// period of time. This is used to manage revocation for X.509 certificates. +package ocsp + +import ( + "asn1" + "crypto" + "crypto/rsa" + _ "crypto/sha1" + "crypto/x509" + "crypto/x509/pkix" + "os" + "time" +) + +var idPKIXOCSPBasic = asn1.ObjectIdentifier([]int{1, 3, 6, 1, 5, 5, 7, 48, 1, 1}) +var idSHA1WithRSA = asn1.ObjectIdentifier([]int{1, 2, 840, 113549, 1, 1, 5}) + +// These are internal structures that reflect the ASN.1 structure of an OCSP +// response. See RFC 2560, section 4.2. + +const ( + ocspSuccess = 0 + ocspMalformed = 1 + ocspInternalError = 2 + ocspTryLater = 3 + ocspSigRequired = 4 + ocspUnauthorized = 5 +) + +type certID struct { + HashAlgorithm pkix.AlgorithmIdentifier + NameHash []byte + IssuerKeyHash []byte + SerialNumber asn1.RawValue +} + +type responseASN1 struct { + Status asn1.Enumerated + Response responseBytes `asn1:"explicit,tag:0"` +} + +type responseBytes struct { + ResponseType asn1.ObjectIdentifier + Response []byte +} + +type basicResponse struct { + TBSResponseData responseData + SignatureAlgorithm pkix.AlgorithmIdentifier + Signature asn1.BitString + Certificates []asn1.RawValue `asn1:"explicit,tag:0,optional"` +} + +type responseData struct { + Raw asn1.RawContent + Version int `asn1:"optional,default:1,explicit,tag:0"` + RequestorName pkix.RDNSequence `asn1:"optional,explicit,tag:1"` + KeyHash []byte `asn1:"optional,explicit,tag:2"` + ProducedAt *time.Time + Responses []singleResponse +} + +type singleResponse struct { + CertID certID + Good asn1.Flag `asn1:"explicit,tag:0,optional"` + Revoked revokedInfo `asn1:"explicit,tag:1,optional"` + Unknown asn1.Flag `asn1:"explicit,tag:2,optional"` + ThisUpdate *time.Time + NextUpdate *time.Time `asn1:"explicit,tag:0,optional"` +} + +type revokedInfo struct { + RevocationTime *time.Time + Reason int `asn1:"explicit,tag:0,optional"` +} + +// This is the exposed reflection of the internal OCSP structures. + +const ( + // Good means that the certificate is valid. + Good = iota + // Revoked means that the certificate has been deliberately revoked. + Revoked = iota + // Unknown means that the OCSP responder doesn't know about the certificate. + Unknown = iota + // ServerFailed means that the OCSP responder failed to process the request. + ServerFailed = iota +) + +// Response represents an OCSP response. See RFC 2560. +type Response struct { + // Status is one of {Good, Revoked, Unknown, ServerFailed} + Status int + SerialNumber []byte + ProducedAt, ThisUpdate, NextUpdate, RevokedAt *time.Time + RevocationReason int + Certificate *x509.Certificate +} + +// ParseError results from an invalid OCSP response. +type ParseError string + +func (p ParseError) String() string { + return string(p) +} + +// ParseResponse parses an OCSP response in DER form. It only supports +// responses for a single certificate and only those using RSA signatures. +// Non-RSA responses will result in an x509.UnsupportedAlgorithmError. +// Signature errors or parse failures will result in a ParseError. +func ParseResponse(bytes []byte) (*Response, os.Error) { + var resp responseASN1 + rest, err := asn1.Unmarshal(bytes, &resp) + if err != nil { + return nil, err + } + if len(rest) > 0 { + return nil, ParseError("trailing data in OCSP response") + } + + ret := new(Response) + if resp.Status != ocspSuccess { + ret.Status = ServerFailed + return ret, nil + } + + if !resp.Response.ResponseType.Equal(idPKIXOCSPBasic) { + return nil, ParseError("bad OCSP response type") + } + + var basicResp basicResponse + rest, err = asn1.Unmarshal(resp.Response.Response, &basicResp) + if err != nil { + return nil, err + } + + if len(basicResp.Certificates) != 1 { + return nil, ParseError("OCSP response contains bad number of certificates") + } + + if len(basicResp.TBSResponseData.Responses) != 1 { + return nil, ParseError("OCSP response contains bad number of responses") + } + + ret.Certificate, err = x509.ParseCertificate(basicResp.Certificates[0].FullBytes) + if err != nil { + return nil, err + } + + if ret.Certificate.PublicKeyAlgorithm != x509.RSA || !basicResp.SignatureAlgorithm.Algorithm.Equal(idSHA1WithRSA) { + return nil, x509.UnsupportedAlgorithmError{} + } + + hashType := crypto.SHA1 + h := hashType.New() + + pub := ret.Certificate.PublicKey.(*rsa.PublicKey) + h.Write(basicResp.TBSResponseData.Raw) + digest := h.Sum() + signature := basicResp.Signature.RightAlign() + + if rsa.VerifyPKCS1v15(pub, hashType, digest, signature) != nil { + return nil, ParseError("bad OCSP signature") + } + + r := basicResp.TBSResponseData.Responses[0] + + ret.SerialNumber = r.CertID.SerialNumber.Bytes + + switch { + case bool(r.Good): + ret.Status = Good + case bool(r.Unknown): + ret.Status = Unknown + default: + ret.Status = Revoked + ret.RevokedAt = r.Revoked.RevocationTime + ret.RevocationReason = r.Revoked.Reason + } + + ret.ProducedAt = basicResp.TBSResponseData.ProducedAt + ret.ThisUpdate = r.ThisUpdate + ret.NextUpdate = r.NextUpdate + + return ret, nil +} diff --git a/src/pkg/crypto/ocsp/ocsp_test.go b/src/pkg/crypto/ocsp/ocsp_test.go new file mode 100644 index 000000000..f9889790f --- /dev/null +++ b/src/pkg/crypto/ocsp/ocsp_test.go @@ -0,0 +1,97 @@ +package ocsp + +import ( + "bytes" + "encoding/hex" + "reflect" + "testing" + "time" +) + +func TestOCSPDecode(t *testing.T) { + responseBytes, _ := hex.DecodeString(ocspResponseHex) + resp, err := ParseResponse(responseBytes) + if err != nil { + t.Error(err) + } + + expected := Response{Status: 0, SerialNumber: []byte{0x1, 0xd0, 0xfa}, RevocationReason: 0, ThisUpdate: &time.Time{Year: 2010, Month: 7, Day: 7, Hour: 15, Minute: 1, Second: 5, Weekday: 0, ZoneOffset: 0, Zone: "UTC"}, NextUpdate: &time.Time{Year: 2010, Month: 7, Day: 7, Hour: 18, Minute: 35, Second: 17, Weekday: 0, ZoneOffset: 0, Zone: "UTC"}} + + if !reflect.DeepEqual(resp.ThisUpdate, resp.ThisUpdate) { + t.Errorf("resp.ThisUpdate: got %d, want %d", resp.ThisUpdate, expected.ThisUpdate) + } + + if !reflect.DeepEqual(resp.NextUpdate, resp.NextUpdate) { + t.Errorf("resp.NextUpdate: got %d, want %d", resp.NextUpdate, expected.NextUpdate) + } + + if resp.Status != expected.Status { + t.Errorf("resp.Status: got %d, want %d", resp.Status, expected.Status) + } + + if !bytes.Equal(resp.SerialNumber, expected.SerialNumber) { + t.Errorf("resp.SerialNumber: got %x, want %x", resp.SerialNumber, expected.SerialNumber) + } + + if resp.RevocationReason != expected.RevocationReason { + t.Errorf("resp.RevocationReason: got %d, want %d", resp.RevocationReason, expected.RevocationReason) + } +} + +// This OCSP response was taken from Thawte's public OCSP responder. +// To recreate: +// $ openssl s_client -tls1 -showcerts -servername www.google.com -connect www.google.com:443 +// Copy and paste the first certificate into /tmp/cert.crt and the second into +// /tmp/intermediate.crt +// $ openssl ocsp -issuer /tmp/intermediate.crt -cert /tmp/cert.crt -url http://ocsp.thawte.com -resp_text -respout /tmp/ocsp.der +// Then hex encode the result: +// $ python -c 'print file("/tmp/ocsp.der", "r").read().encode("hex")' + +const ocspResponseHex = "308206bc0a0100a08206b5308206b106092b0601050507300101048206a23082069e3081" + + "c9a14e304c310b300906035504061302494c31163014060355040a130d5374617274436f" + + "6d204c74642e312530230603550403131c5374617274436f6d20436c6173732031204f43" + + "5350205369676e6572180f32303130303730373137333531375a30663064303c30090605" + + "2b0e03021a050004146568874f40750f016a3475625e1f5c93e5a26d580414eb4234d098" + + "b0ab9ff41b6b08f7cc642eef0e2c45020301d0fa8000180f323031303037303731353031" + + "30355aa011180f32303130303730373138333531375a300d06092a864886f70d01010505" + + "000382010100ab557ff070d1d7cebbb5f0ec91a15c3fed22eb2e1b8244f1b84545f013a4" + + "fb46214c5e3fbfbebb8a56acc2b9db19f68fd3c3201046b3824d5ba689f99864328710cb" + + "467195eb37d84f539e49f859316b32964dc3e47e36814ce94d6c56dd02733b1d0802f7ff" + + "4eebdbbd2927dcf580f16cbc290f91e81b53cb365e7223f1d6e20a88ea064104875e0145" + + "672b20fc14829d51ca122f5f5d77d3ad6c83889c55c7dc43680ba2fe3cef8b05dbcabdc0" + + "d3e09aaf9725597f8c858c2fa38c0d6aed2e6318194420dd1a1137445d13e1c97ab47896" + + "17a4e08925f46f867b72e3a4dc1f08cb870b2b0717f7207faa0ac512e628a029aba7457a" + + "e63dcf3281e2162d9349a08204ba308204b6308204b23082039aa003020102020101300d" + + "06092a864886f70d010105050030818c310b300906035504061302494c31163014060355" + + "040a130d5374617274436f6d204c74642e312b3029060355040b13225365637572652044" + + "69676974616c204365727469666963617465205369676e696e6731383036060355040313" + + "2f5374617274436f6d20436c6173732031205072696d61727920496e7465726d65646961" + + "746520536572766572204341301e170d3037313032353030323330365a170d3132313032" + + "333030323330365a304c310b300906035504061302494c31163014060355040a130d5374" + + "617274436f6d204c74642e312530230603550403131c5374617274436f6d20436c617373" + + "2031204f435350205369676e657230820122300d06092a864886f70d0101010500038201" + + "0f003082010a0282010100b9561b4c45318717178084e96e178df2255e18ed8d8ecc7c2b" + + "7b51a6c1c2e6bf0aa3603066f132fe10ae97b50e99fa24b83fc53dd2777496387d14e1c3" + + "a9b6a4933e2ac12413d085570a95b8147414a0bc007c7bcf222446ef7f1a156d7ea1c577" + + "fc5f0facdfd42eb0f5974990cb2f5cefebceef4d1bdc7ae5c1075c5a99a93171f2b0845b" + + "4ff0864e973fcfe32f9d7511ff87a3e943410c90a4493a306b6944359340a9ca96f02b66" + + "ce67f028df2980a6aaee8d5d5d452b8b0eb93f923cc1e23fcccbdbe7ffcb114d08fa7a6a" + + "3c404f825d1a0e715935cf623a8c7b59670014ed0622f6089a9447a7a19010f7fe58f841" + + "29a2765ea367824d1c3bb2fda308530203010001a382015c30820158300c0603551d1301" + + "01ff04023000300b0603551d0f0404030203a8301e0603551d250417301506082b060105" + + "0507030906092b0601050507300105301d0603551d0e0416041445e0a36695414c5dd449" + + "bc00e33cdcdbd2343e173081a80603551d230481a030819d8014eb4234d098b0ab9ff41b" + + "6b08f7cc642eef0e2c45a18181a47f307d310b300906035504061302494c311630140603" + + "55040a130d5374617274436f6d204c74642e312b3029060355040b132253656375726520" + + "4469676974616c204365727469666963617465205369676e696e67312930270603550403" + + "13205374617274436f6d2043657274696669636174696f6e20417574686f726974798201" + + "0a30230603551d12041c301a8618687474703a2f2f7777772e737461727473736c2e636f" + + "6d2f302c06096086480186f842010d041f161d5374617274436f6d205265766f63617469" + + "6f6e20417574686f72697479300d06092a864886f70d01010505000382010100182d2215" + + "8f0fc0291324fa8574c49bb8ff2835085adcbf7b7fc4191c397ab6951328253fffe1e5ec" + + "2a7da0d50fca1a404e6968481366939e666c0a6209073eca57973e2fefa9ed1718e8176f" + + "1d85527ff522c08db702e3b2b180f1cbff05d98128252cf0f450f7dd2772f4188047f19d" + + "c85317366f94bc52d60f453a550af58e308aaab00ced33040b62bf37f5b1ab2a4f7f0f80" + + "f763bf4d707bc8841d7ad9385ee2a4244469260b6f2bf085977af9074796048ecc2f9d48" + + "a1d24ce16e41a9941568fec5b42771e118f16c106a54ccc339a4b02166445a167902e75e" + + "6d8620b0825dcd18a069b90fd851d10fa8effd409deec02860d26d8d833f304b10669b42" diff --git a/src/pkg/crypto/openpgp/Makefile b/src/pkg/crypto/openpgp/Makefile new file mode 100644 index 000000000..b46ac2bba --- /dev/null +++ b/src/pkg/crypto/openpgp/Makefile @@ -0,0 +1,14 @@ +# Copyright 2011 The Go Authors. All rights reserved. +# Use of this source code is governed by a BSD-style +# license that can be found in the LICENSE file. + +include ../../../Make.inc + +TARG=crypto/openpgp +GOFILES=\ + canonical_text.go\ + keys.go\ + read.go\ + write.go\ + +include ../../../Make.pkg diff --git a/src/pkg/crypto/openpgp/armor/Makefile b/src/pkg/crypto/openpgp/armor/Makefile new file mode 100644 index 000000000..138e314e9 --- /dev/null +++ b/src/pkg/crypto/openpgp/armor/Makefile @@ -0,0 +1,12 @@ +# Copyright 2010 The Go Authors. All rights reserved. +# Use of this source code is governed by a BSD-style +# license that can be found in the LICENSE file. + +include ../../../../Make.inc + +TARG=crypto/openpgp/armor +GOFILES=\ + armor.go\ + encode.go\ + +include ../../../../Make.pkg diff --git a/src/pkg/crypto/openpgp/armor/armor.go b/src/pkg/crypto/openpgp/armor/armor.go new file mode 100644 index 000000000..9c4180d6d --- /dev/null +++ b/src/pkg/crypto/openpgp/armor/armor.go @@ -0,0 +1,220 @@ +// Copyright 2010 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Package armor implements OpenPGP ASCII Armor, see RFC 4880. OpenPGP Armor is +// very similar to PEM except that it has an additional CRC checksum. +package armor + +import ( + "bufio" + "bytes" + "crypto/openpgp/error" + "encoding/base64" + "io" + "os" +) + +// A Block represents an OpenPGP armored structure. +// +// The encoded form is: +// -----BEGIN Type----- +// Headers +// +// base64-encoded Bytes +// '=' base64 encoded checksum +// -----END Type----- +// where Headers is a possibly empty sequence of Key: Value lines. +// +// Since the armored data can be very large, this package presents a streaming +// interface. +type Block struct { + Type string // The type, taken from the preamble (i.e. "PGP SIGNATURE"). + Header map[string]string // Optional headers. + Body io.Reader // A Reader from which the contents can be read + lReader lineReader + oReader openpgpReader +} + +var ArmorCorrupt os.Error = error.StructuralError("armor invalid") + +const crc24Init = 0xb704ce +const crc24Poly = 0x1864cfb +const crc24Mask = 0xffffff + +// crc24 calculates the OpenPGP checksum as specified in RFC 4880, section 6.1 +func crc24(crc uint32, d []byte) uint32 { + for _, b := range d { + crc ^= uint32(b) << 16 + for i := 0; i < 8; i++ { + crc <<= 1 + if crc&0x1000000 != 0 { + crc ^= crc24Poly + } + } + } + return crc +} + +var armorStart = []byte("-----BEGIN ") +var armorEnd = []byte("-----END ") +var armorEndOfLine = []byte("-----") + +// lineReader wraps a line based reader. It watches for the end of an armor +// block and records the expected CRC value. +type lineReader struct { + in *bufio.Reader + buf []byte + eof bool + crc uint32 +} + +func (l *lineReader) Read(p []byte) (n int, err os.Error) { + if l.eof { + return 0, os.EOF + } + + if len(l.buf) > 0 { + n = copy(p, l.buf) + l.buf = l.buf[n:] + return + } + + line, isPrefix, err := l.in.ReadLine() + if err != nil { + return + } + if isPrefix { + return 0, ArmorCorrupt + } + + if len(line) == 5 && line[0] == '=' { + // This is the checksum line + var expectedBytes [3]byte + var m int + m, err = base64.StdEncoding.Decode(expectedBytes[0:], line[1:]) + if m != 3 || err != nil { + return + } + l.crc = uint32(expectedBytes[0])<<16 | + uint32(expectedBytes[1])<<8 | + uint32(expectedBytes[2]) + + line, _, err = l.in.ReadLine() + if err != nil && err != os.EOF { + return + } + if !bytes.HasPrefix(line, armorEnd) { + return 0, ArmorCorrupt + } + + l.eof = true + return 0, os.EOF + } + + if len(line) > 64 { + return 0, ArmorCorrupt + } + + n = copy(p, line) + bytesToSave := len(line) - n + if bytesToSave > 0 { + if cap(l.buf) < bytesToSave { + l.buf = make([]byte, 0, bytesToSave) + } + l.buf = l.buf[0:bytesToSave] + copy(l.buf, line[n:]) + } + + return +} + +// openpgpReader passes Read calls to the underlying base64 decoder, but keeps +// a running CRC of the resulting data and checks the CRC against the value +// found by the lineReader at EOF. +type openpgpReader struct { + lReader *lineReader + b64Reader io.Reader + currentCRC uint32 +} + +func (r *openpgpReader) Read(p []byte) (n int, err os.Error) { + n, err = r.b64Reader.Read(p) + r.currentCRC = crc24(r.currentCRC, p[:n]) + + if err == os.EOF { + if r.lReader.crc != uint32(r.currentCRC&crc24Mask) { + return 0, ArmorCorrupt + } + } + + return +} + +// Decode reads a PGP armored block from the given Reader. It will ignore +// leading garbage. If it doesn't find a block, it will return nil, os.EOF. The +// given Reader is not usable after calling this function: an arbitrary amount +// of data may have been read past the end of the block. +func Decode(in io.Reader) (p *Block, err os.Error) { + r, _ := bufio.NewReaderSize(in, 100) + var line []byte + ignoreNext := false + +TryNextBlock: + p = nil + + // Skip leading garbage + for { + ignoreThis := ignoreNext + line, ignoreNext, err = r.ReadLine() + if err != nil { + return + } + if ignoreNext || ignoreThis { + continue + } + line = bytes.TrimSpace(line) + if len(line) > len(armorStart)+len(armorEndOfLine) && bytes.HasPrefix(line, armorStart) { + break + } + } + + p = new(Block) + p.Type = string(line[len(armorStart) : len(line)-len(armorEndOfLine)]) + p.Header = make(map[string]string) + nextIsContinuation := false + var lastKey string + + // Read headers + for { + isContinuation := nextIsContinuation + line, nextIsContinuation, err = r.ReadLine() + if err != nil { + p = nil + return + } + if isContinuation { + p.Header[lastKey] += string(line) + continue + } + line = bytes.TrimSpace(line) + if len(line) == 0 { + break + } + + i := bytes.Index(line, []byte(": ")) + if i == -1 { + goto TryNextBlock + } + lastKey = string(line[:i]) + p.Header[lastKey] = string(line[i+2:]) + } + + p.lReader.in = r + p.oReader.currentCRC = crc24Init + p.oReader.lReader = &p.lReader + p.oReader.b64Reader = base64.NewDecoder(base64.StdEncoding, &p.lReader) + p.Body = &p.oReader + + return +} diff --git a/src/pkg/crypto/openpgp/armor/armor_test.go b/src/pkg/crypto/openpgp/armor/armor_test.go new file mode 100644 index 000000000..9334e94e9 --- /dev/null +++ b/src/pkg/crypto/openpgp/armor/armor_test.go @@ -0,0 +1,95 @@ +// Copyright 2010 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package armor + +import ( + "bytes" + "hash/adler32" + "io/ioutil" + "testing" +) + +func TestDecodeEncode(t *testing.T) { + buf := bytes.NewBuffer([]byte(armorExample1)) + result, err := Decode(buf) + if err != nil { + t.Error(err) + } + expectedType := "PGP SIGNATURE" + if result.Type != expectedType { + t.Errorf("result.Type: got:%s want:%s", result.Type, expectedType) + } + if len(result.Header) != 1 { + t.Errorf("len(result.Header): got:%d want:1", len(result.Header)) + } + v, ok := result.Header["Version"] + if !ok || v != "GnuPG v1.4.10 (GNU/Linux)" { + t.Errorf("result.Header: got:%#v", result.Header) + } + + contents, err := ioutil.ReadAll(result.Body) + if err != nil { + t.Error(err) + } + + if adler32.Checksum(contents) != 0x27b144be { + t.Errorf("contents: got: %x", contents) + } + + buf = bytes.NewBuffer(nil) + w, err := Encode(buf, result.Type, result.Header) + if err != nil { + t.Error(err) + } + _, err = w.Write(contents) + if err != nil { + t.Error(err) + } + w.Close() + + if !bytes.Equal(buf.Bytes(), []byte(armorExample1)) { + t.Errorf("got: %s\nwant: %s", string(buf.Bytes()), armorExample1) + } +} + +func TestLongHeader(t *testing.T) { + buf := bytes.NewBuffer([]byte(armorLongLine)) + result, err := Decode(buf) + if err != nil { + t.Error(err) + return + } + value, ok := result.Header["Version"] + if !ok { + t.Errorf("missing Version header") + } + if value != longValueExpected { + t.Errorf("got: %s want: %s", value, longValueExpected) + } +} + +const armorExample1 = `-----BEGIN PGP SIGNATURE----- +Version: GnuPG v1.4.10 (GNU/Linux) + +iJwEAAECAAYFAk1Fv/0ACgkQo01+GMIMMbsYTwQAiAw+QAaNfY6WBdplZ/uMAccm +4g+81QPmTSGHnetSb6WBiY13kVzK4HQiZH8JSkmmroMLuGeJwsRTEL4wbjRyUKEt +p1xwUZDECs234F1xiG5enc5SGlRtP7foLBz9lOsjx+LEcA4sTl5/2eZR9zyFZqWW +TxRjs+fJCIFuo71xb1g= +=/teI +-----END PGP SIGNATURE-----` + +const armorLongLine = `-----BEGIN PGP SIGNATURE----- +Version: 0123456789abcdefghijklmnopqrstuvwxyz0123456789abcdefghijklmnopqrstuvwxyz0123456789abcdefghijklmnopqrstuvwxyz0123456789abcdefghijklmnopqrstuvwxyz0123456789abcdefghijklmnopqrstuvwxyz0123456789abcdefghijklmnopqrstuvwxyz0123456789abcdefghijklmnopqrstuvwxyz0123456789abcdefghijklmnopqrstuvwxyz0123456789abcdefghijklmnopqrstuvwxyz + +iQEcBAABAgAGBQJMtFESAAoJEKsQXJGvOPsVj40H/1WW6jaMXv4BW+1ueDSMDwM8 +kx1fLOXbVM5/Kn5LStZNt1jWWnpxdz7eq3uiqeCQjmqUoRde3YbB2EMnnwRbAhpp +cacnAvy9ZQ78OTxUdNW1mhX5bS6q1MTEJnl+DcyigD70HG/yNNQD7sOPMdYQw0TA +byQBwmLwmTsuZsrYqB68QyLHI+DUugn+kX6Hd2WDB62DKa2suoIUIHQQCd/ofwB3 +WfCYInXQKKOSxu2YOg2Eb4kLNhSMc1i9uKUWAH+sdgJh7NBgdoE4MaNtBFkHXRvv +okWuf3+xA9ksp1npSY/mDvgHijmjvtpRDe6iUeqfCn8N9u9CBg8geANgaG8+QA4= +=wfQG +-----END PGP SIGNATURE-----` + +const longValueExpected = "0123456789abcdefghijklmnopqrstuvwxyz0123456789abcdefghijklmnopqrstuvwxyz0123456789abcdefghijklmnopqrstuvwxyz0123456789abcdefghijklmnopqrstuvwxyz0123456789abcdefghijklmnopqrstuvwxyz0123456789abcdefghijklmnopqrstuvwxyz0123456789abcdefghijklmnopqrstuvwxyz0123456789abcdefghijklmnopqrstuvwxyz0123456789abcdefghijklmnopqrstuvwxyz" diff --git a/src/pkg/crypto/openpgp/armor/encode.go b/src/pkg/crypto/openpgp/armor/encode.go new file mode 100644 index 000000000..99dee375e --- /dev/null +++ b/src/pkg/crypto/openpgp/armor/encode.go @@ -0,0 +1,161 @@ +// Copyright 2010 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package armor + +import ( + "encoding/base64" + "io" + "os" +) + +var armorHeaderSep = []byte(": ") +var blockEnd = []byte("\n=") +var newline = []byte("\n") +var armorEndOfLineOut = []byte("-----\n") + +// writeSlices writes its arguments to the given Writer. +func writeSlices(out io.Writer, slices ...[]byte) (err os.Error) { + for _, s := range slices { + _, err = out.Write(s) + if err != nil { + return err + } + } + return +} + +// lineBreaker breaks data across several lines, all of the same byte length +// (except possibly the last). Lines are broken with a single '\n'. +type lineBreaker struct { + lineLength int + line []byte + used int + out io.Writer + haveWritten bool +} + +func newLineBreaker(out io.Writer, lineLength int) *lineBreaker { + return &lineBreaker{ + lineLength: lineLength, + line: make([]byte, lineLength), + used: 0, + out: out, + } +} + +func (l *lineBreaker) Write(b []byte) (n int, err os.Error) { + n = len(b) + + if n == 0 { + return + } + + if l.used == 0 && l.haveWritten { + _, err = l.out.Write([]byte{'\n'}) + if err != nil { + return + } + } + + if l.used+len(b) < l.lineLength { + l.used += copy(l.line[l.used:], b) + return + } + + l.haveWritten = true + _, err = l.out.Write(l.line[0:l.used]) + if err != nil { + return + } + excess := l.lineLength - l.used + l.used = 0 + + _, err = l.out.Write(b[0:excess]) + if err != nil { + return + } + + _, err = l.Write(b[excess:]) + return +} + +func (l *lineBreaker) Close() (err os.Error) { + if l.used > 0 { + _, err = l.out.Write(l.line[0:l.used]) + if err != nil { + return + } + } + + return +} + +// encoding keeps track of a running CRC24 over the data which has been written +// to it and outputs a OpenPGP checksum when closed, followed by an armor +// trailer. +// +// It's built into a stack of io.Writers: +// encoding -> base64 encoder -> lineBreaker -> out +type encoding struct { + out io.Writer + breaker *lineBreaker + b64 io.WriteCloser + crc uint32 + blockType []byte +} + +func (e *encoding) Write(data []byte) (n int, err os.Error) { + e.crc = crc24(e.crc, data) + return e.b64.Write(data) +} + +func (e *encoding) Close() (err os.Error) { + err = e.b64.Close() + if err != nil { + return + } + e.breaker.Close() + + var checksumBytes [3]byte + checksumBytes[0] = byte(e.crc >> 16) + checksumBytes[1] = byte(e.crc >> 8) + checksumBytes[2] = byte(e.crc) + + var b64ChecksumBytes [4]byte + base64.StdEncoding.Encode(b64ChecksumBytes[:], checksumBytes[:]) + + return writeSlices(e.out, blockEnd, b64ChecksumBytes[:], newline, armorEnd, e.blockType, armorEndOfLine) +} + +// Encode returns a WriteCloser which will encode the data written to it in +// OpenPGP armor. +func Encode(out io.Writer, blockType string, headers map[string]string) (w io.WriteCloser, err os.Error) { + bType := []byte(blockType) + err = writeSlices(out, armorStart, bType, armorEndOfLineOut) + if err != nil { + return + } + + for k, v := range headers { + err = writeSlices(out, []byte(k), armorHeaderSep, []byte(v), newline) + if err != nil { + return + } + } + + _, err = out.Write(newline) + if err != nil { + return + } + + e := &encoding{ + out: out, + breaker: newLineBreaker(out, 64), + crc: crc24Init, + blockType: bType, + } + e.b64 = base64.NewEncoder(base64.StdEncoding, e.breaker) + return e, nil +} diff --git a/src/pkg/crypto/openpgp/canonical_text.go b/src/pkg/crypto/openpgp/canonical_text.go new file mode 100644 index 000000000..293eff354 --- /dev/null +++ b/src/pkg/crypto/openpgp/canonical_text.go @@ -0,0 +1,58 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package openpgp + +import ( + "hash" + "os" +) + +// NewCanonicalTextHash reformats text written to it into the canonical +// form and then applies the hash h. See RFC 4880, section 5.2.1. +func NewCanonicalTextHash(h hash.Hash) hash.Hash { + return &canonicalTextHash{h, 0} +} + +type canonicalTextHash struct { + h hash.Hash + s int +} + +var newline = []byte{'\r', '\n'} + +func (cth *canonicalTextHash) Write(buf []byte) (int, os.Error) { + start := 0 + + for i, c := range buf { + switch cth.s { + case 0: + if c == '\r' { + cth.s = 1 + } else if c == '\n' { + cth.h.Write(buf[start:i]) + cth.h.Write(newline) + start = i + 1 + } + case 1: + cth.s = 0 + } + } + + cth.h.Write(buf[start:]) + return len(buf), nil +} + +func (cth *canonicalTextHash) Sum() []byte { + return cth.h.Sum() +} + +func (cth *canonicalTextHash) Reset() { + cth.h.Reset() + cth.s = 0 +} + +func (cth *canonicalTextHash) Size() int { + return cth.h.Size() +} diff --git a/src/pkg/crypto/openpgp/canonical_text_test.go b/src/pkg/crypto/openpgp/canonical_text_test.go new file mode 100644 index 000000000..ccf2910cc --- /dev/null +++ b/src/pkg/crypto/openpgp/canonical_text_test.go @@ -0,0 +1,49 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package openpgp + +import ( + "bytes" + "os" + "testing" +) + +type recordingHash struct { + buf *bytes.Buffer +} + +func (r recordingHash) Write(b []byte) (n int, err os.Error) { + return r.buf.Write(b) +} + +func (r recordingHash) Sum() []byte { + return r.buf.Bytes() +} + +func (r recordingHash) Reset() { + panic("shouldn't be called") +} + +func (r recordingHash) Size() int { + panic("shouldn't be called") +} + +func testCanonicalText(t *testing.T, input, expected string) { + r := recordingHash{bytes.NewBuffer(nil)} + c := NewCanonicalTextHash(r) + c.Write([]byte(input)) + result := c.Sum() + if expected != string(result) { + t.Errorf("input: %x got: %x want: %x", input, result, expected) + } +} + +func TestCanonicalText(t *testing.T) { + testCanonicalText(t, "foo\n", "foo\r\n") + testCanonicalText(t, "foo", "foo") + testCanonicalText(t, "foo\r\n", "foo\r\n") + testCanonicalText(t, "foo\r\nbar", "foo\r\nbar") + testCanonicalText(t, "foo\r\nbar\n\n", "foo\r\nbar\r\n\r\n") +} diff --git a/src/pkg/crypto/openpgp/elgamal/Makefile b/src/pkg/crypto/openpgp/elgamal/Makefile new file mode 100644 index 000000000..f730255f8 --- /dev/null +++ b/src/pkg/crypto/openpgp/elgamal/Makefile @@ -0,0 +1,11 @@ +# Copyright 2011 The Go Authors. All rights reserved. +# Use of this source code is governed by a BSD-style +# license that can be found in the LICENSE file. + +include ../../../../Make.inc + +TARG=crypto/openpgp/elgamal +GOFILES=\ + elgamal.go\ + +include ../../../../Make.pkg diff --git a/src/pkg/crypto/openpgp/elgamal/elgamal.go b/src/pkg/crypto/openpgp/elgamal/elgamal.go new file mode 100644 index 000000000..99a6e3e1f --- /dev/null +++ b/src/pkg/crypto/openpgp/elgamal/elgamal.go @@ -0,0 +1,122 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Package elgamal implements ElGamal encryption, suitable for OpenPGP, +// as specified in "A Public-Key Cryptosystem and a Signature Scheme Based on +// Discrete Logarithms," IEEE Transactions on Information Theory, v. IT-31, +// n. 4, 1985, pp. 469-472. +// +// This form of ElGamal embeds PKCS#1 v1.5 padding, which may make it +// unsuitable for other protocols. RSA should be used in preference in any +// case. +package elgamal + +import ( + "big" + "crypto/rand" + "crypto/subtle" + "io" + "os" +) + +// PublicKey represents an ElGamal public key. +type PublicKey struct { + G, P, Y *big.Int +} + +// PrivateKey represents an ElGamal private key. +type PrivateKey struct { + PublicKey + X *big.Int +} + +// Encrypt encrypts the given message to the given public key. The result is a +// pair of integers. Errors can result from reading random, or because msg is +// too large to be encrypted to the public key. +func Encrypt(random io.Reader, pub *PublicKey, msg []byte) (c1, c2 *big.Int, err os.Error) { + pLen := (pub.P.BitLen() + 7) / 8 + if len(msg) > pLen-11 { + err = os.NewError("elgamal: message too long") + return + } + + // EM = 0x02 || PS || 0x00 || M + em := make([]byte, pLen-1) + em[0] = 2 + ps, mm := em[1:len(em)-len(msg)-1], em[len(em)-len(msg):] + err = nonZeroRandomBytes(ps, random) + if err != nil { + return + } + em[len(em)-len(msg)-1] = 0 + copy(mm, msg) + + m := new(big.Int).SetBytes(em) + + k, err := rand.Int(random, pub.P) + if err != nil { + return + } + + c1 = new(big.Int).Exp(pub.G, k, pub.P) + s := new(big.Int).Exp(pub.Y, k, pub.P) + c2 = s.Mul(s, m) + c2.Mod(c2, pub.P) + + return +} + +// Decrypt takes two integers, resulting from an ElGamal encryption, and +// returns the plaintext of the message. An error can result only if the +// ciphertext is invalid. Users should keep in mind that this is a padding +// oracle and thus, if exposed to an adaptive chosen ciphertext attack, can +// be used to break the cryptosystem. See ``Chosen Ciphertext Attacks +// Against Protocols Based on the RSA Encryption Standard PKCS #1'', Daniel +// Bleichenbacher, Advances in Cryptology (Crypto '98), +func Decrypt(priv *PrivateKey, c1, c2 *big.Int) (msg []byte, err os.Error) { + s := new(big.Int).Exp(c1, priv.X, priv.P) + s.ModInverse(s, priv.P) + s.Mul(s, c2) + s.Mod(s, priv.P) + em := s.Bytes() + + firstByteIsTwo := subtle.ConstantTimeByteEq(em[0], 2) + + // The remainder of the plaintext must be a string of non-zero random + // octets, followed by a 0, followed by the message. + // lookingForIndex: 1 iff we are still looking for the zero. + // index: the offset of the first zero byte. + var lookingForIndex, index int + lookingForIndex = 1 + + for i := 1; i < len(em); i++ { + equals0 := subtle.ConstantTimeByteEq(em[i], 0) + index = subtle.ConstantTimeSelect(lookingForIndex&equals0, i, index) + lookingForIndex = subtle.ConstantTimeSelect(equals0, 0, lookingForIndex) + } + + if firstByteIsTwo != 1 || lookingForIndex != 0 || index < 9 { + return nil, os.NewError("elgamal: decryption error") + } + return em[index+1:], nil +} + +// nonZeroRandomBytes fills the given slice with non-zero random octets. +func nonZeroRandomBytes(s []byte, rand io.Reader) (err os.Error) { + _, err = io.ReadFull(rand, s) + if err != nil { + return + } + + for i := 0; i < len(s); i++ { + for s[i] == 0 { + _, err = io.ReadFull(rand, s[i:i+1]) + if err != nil { + return + } + } + } + + return +} diff --git a/src/pkg/crypto/openpgp/elgamal/elgamal_test.go b/src/pkg/crypto/openpgp/elgamal/elgamal_test.go new file mode 100644 index 000000000..101121aa6 --- /dev/null +++ b/src/pkg/crypto/openpgp/elgamal/elgamal_test.go @@ -0,0 +1,49 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package elgamal + +import ( + "big" + "bytes" + "crypto/rand" + "testing" +) + +// This is the 1024-bit MODP group from RFC 5114, section 2.1: +const primeHex = "B10B8F96A080E01DDE92DE5EAE5D54EC52C99FBCFB06A3C69A6A9DCA52D23B616073E28675A23D189838EF1E2EE652C013ECB4AEA906112324975C3CD49B83BFACCBDD7D90C4BD7098488E9C219A73724EFFD6FAE5644738FAA31A4FF55BCCC0A151AF5F0DC8B4BD45BF37DF365C1A65E68CFDA76D4DA708DF1FB2BC2E4A4371" + +const generatorHex = "A4D1CBD5C3FD34126765A442EFB99905F8104DD258AC507FD6406CFF14266D31266FEA1E5C41564B777E690F5504F213160217B4B01B886A5E91547F9E2749F4D7FBD7D3B9A92EE1909D0D2263F80A76A6A24C087A091F531DBF0A0169B6A28AD662A4D18E73AFA32D779D5918D08BC8858F4DCEF97C2A24855E6EEB22B3B2E5" + +func fromHex(hex string) *big.Int { + n, ok := new(big.Int).SetString(hex, 16) + if !ok { + panic("failed to parse hex number") + } + return n +} + +func TestEncryptDecrypt(t *testing.T) { + priv := &PrivateKey{ + PublicKey: PublicKey{ + G: fromHex(generatorHex), + P: fromHex(primeHex), + }, + X: fromHex("42"), + } + priv.Y = new(big.Int).Exp(priv.G, priv.X, priv.P) + + message := []byte("hello world") + c1, c2, err := Encrypt(rand.Reader, &priv.PublicKey, message) + if err != nil { + t.Errorf("error encrypting: %s", err) + } + message2, err := Decrypt(priv, c1, c2) + if err != nil { + t.Errorf("error decrypting: %s", err) + } + if !bytes.Equal(message2, message) { + t.Errorf("decryption failed, got: %x, want: %x", message2, message) + } +} diff --git a/src/pkg/crypto/openpgp/error/Makefile b/src/pkg/crypto/openpgp/error/Makefile new file mode 100644 index 000000000..8c370a089 --- /dev/null +++ b/src/pkg/crypto/openpgp/error/Makefile @@ -0,0 +1,11 @@ +# Copyright 2010 The Go Authors. All rights reserved. +# Use of this source code is governed by a BSD-style +# license that can be found in the LICENSE file. + +include ../../../../Make.inc + +TARG=crypto/openpgp/error +GOFILES=\ + error.go\ + +include ../../../../Make.pkg diff --git a/src/pkg/crypto/openpgp/error/error.go b/src/pkg/crypto/openpgp/error/error.go new file mode 100644 index 000000000..3759ce161 --- /dev/null +++ b/src/pkg/crypto/openpgp/error/error.go @@ -0,0 +1,64 @@ +// Copyright 2010 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Package error contains common error types for the OpenPGP packages. +package error + +import ( + "strconv" +) + +// A StructuralError is returned when OpenPGP data is found to be syntactically +// invalid. +type StructuralError string + +func (s StructuralError) String() string { + return "OpenPGP data invalid: " + string(s) +} + +// UnsupportedError indicates that, although the OpenPGP data is valid, it +// makes use of currently unimplemented features. +type UnsupportedError string + +func (s UnsupportedError) String() string { + return "OpenPGP feature unsupported: " + string(s) +} + +// InvalidArgumentError indicates that the caller is in error and passed an +// incorrect value. +type InvalidArgumentError string + +func (i InvalidArgumentError) String() string { + return "OpenPGP argument invalid: " + string(i) +} + +// SignatureError indicates that a syntactically valid signature failed to +// validate. +type SignatureError string + +func (b SignatureError) String() string { + return "OpenPGP signature invalid: " + string(b) +} + +type keyIncorrect int + +func (ki keyIncorrect) String() string { + return "the given key was incorrect" +} + +var KeyIncorrectError = keyIncorrect(0) + +type unknownIssuer int + +func (unknownIssuer) String() string { + return "signature make by unknown entity" +} + +var UnknownIssuerError = unknownIssuer(0) + +type UnknownPacketTypeError uint8 + +func (upte UnknownPacketTypeError) String() string { + return "unknown OpenPGP packet type: " + strconv.Itoa(int(upte)) +} diff --git a/src/pkg/crypto/openpgp/keys.go b/src/pkg/crypto/openpgp/keys.go new file mode 100644 index 000000000..c70fb7927 --- /dev/null +++ b/src/pkg/crypto/openpgp/keys.go @@ -0,0 +1,545 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package openpgp + +import ( + "crypto" + "crypto/openpgp/armor" + "crypto/openpgp/error" + "crypto/openpgp/packet" + "crypto/rsa" + "io" + "os" + "time" +) + +// PublicKeyType is the armor type for a PGP public key. +var PublicKeyType = "PGP PUBLIC KEY BLOCK" +// PrivateKeyType is the armor type for a PGP private key. +var PrivateKeyType = "PGP PRIVATE KEY BLOCK" + +// An Entity represents the components of an OpenPGP key: a primary public key +// (which must be a signing key), one or more identities claimed by that key, +// and zero or more subkeys, which may be encryption keys. +type Entity struct { + PrimaryKey *packet.PublicKey + PrivateKey *packet.PrivateKey + Identities map[string]*Identity // indexed by Identity.Name + Subkeys []Subkey +} + +// An Identity represents an identity claimed by an Entity and zero or more +// assertions by other entities about that claim. +type Identity struct { + Name string // by convention, has the form "Full Name (comment) " + UserId *packet.UserId + SelfSignature *packet.Signature + Signatures []*packet.Signature +} + +// A Subkey is an additional public key in an Entity. Subkeys can be used for +// encryption. +type Subkey struct { + PublicKey *packet.PublicKey + PrivateKey *packet.PrivateKey + Sig *packet.Signature +} + +// A Key identifies a specific public key in an Entity. This is either the +// Entity's primary key or a subkey. +type Key struct { + Entity *Entity + PublicKey *packet.PublicKey + PrivateKey *packet.PrivateKey + SelfSignature *packet.Signature +} + +// A KeyRing provides access to public and private keys. +type KeyRing interface { + // KeysById returns the set of keys that have the given key id. + KeysById(id uint64) []Key + // DecryptionKeys returns all private keys that are valid for + // decryption. + DecryptionKeys() []Key +} + +// primaryIdentity returns the Identity marked as primary or the first identity +// if none are so marked. +func (e *Entity) primaryIdentity() *Identity { + var firstIdentity *Identity + for _, ident := range e.Identities { + if firstIdentity == nil { + firstIdentity = ident + } + if ident.SelfSignature.IsPrimaryId != nil && *ident.SelfSignature.IsPrimaryId { + return ident + } + } + return firstIdentity +} + +// encryptionKey returns the best candidate Key for encrypting a message to the +// given Entity. +func (e *Entity) encryptionKey() Key { + candidateSubkey := -1 + + for i, subkey := range e.Subkeys { + if subkey.Sig.FlagsValid && subkey.Sig.FlagEncryptCommunications && subkey.PublicKey.PubKeyAlgo.CanEncrypt() { + candidateSubkey = i + break + } + } + + i := e.primaryIdentity() + + if e.PrimaryKey.PubKeyAlgo.CanEncrypt() { + // If we don't have any candidate subkeys for encryption and + // the primary key doesn't have any usage metadata then we + // assume that the primary key is ok. Or, if the primary key is + // marked as ok to encrypt to, then we can obviously use it. + if candidateSubkey == -1 && !i.SelfSignature.FlagsValid || i.SelfSignature.FlagEncryptCommunications && i.SelfSignature.FlagsValid { + return Key{e, e.PrimaryKey, e.PrivateKey, i.SelfSignature} + } + } + + if candidateSubkey != -1 { + subkey := e.Subkeys[candidateSubkey] + return Key{e, subkey.PublicKey, subkey.PrivateKey, subkey.Sig} + } + + // This Entity appears to be signing only. + return Key{} +} + +// signingKey return the best candidate Key for signing a message with this +// Entity. +func (e *Entity) signingKey() Key { + candidateSubkey := -1 + + for i, subkey := range e.Subkeys { + if subkey.Sig.FlagsValid && subkey.Sig.FlagSign && subkey.PublicKey.PubKeyAlgo.CanSign() { + candidateSubkey = i + break + } + } + + i := e.primaryIdentity() + + // If we have no candidate subkey then we assume that it's ok to sign + // with the primary key. + if candidateSubkey == -1 || i.SelfSignature.FlagsValid && i.SelfSignature.FlagSign { + return Key{e, e.PrimaryKey, e.PrivateKey, i.SelfSignature} + } + + subkey := e.Subkeys[candidateSubkey] + return Key{e, subkey.PublicKey, subkey.PrivateKey, subkey.Sig} +} + +// An EntityList contains one or more Entities. +type EntityList []*Entity + +// KeysById returns the set of keys that have the given key id. +func (el EntityList) KeysById(id uint64) (keys []Key) { + for _, e := range el { + if e.PrimaryKey.KeyId == id { + var selfSig *packet.Signature + for _, ident := range e.Identities { + if selfSig == nil { + selfSig = ident.SelfSignature + } else if ident.SelfSignature.IsPrimaryId != nil && *ident.SelfSignature.IsPrimaryId { + selfSig = ident.SelfSignature + break + } + } + keys = append(keys, Key{e, e.PrimaryKey, e.PrivateKey, selfSig}) + } + + for _, subKey := range e.Subkeys { + if subKey.PublicKey.KeyId == id { + keys = append(keys, Key{e, subKey.PublicKey, subKey.PrivateKey, subKey.Sig}) + } + } + } + return +} + +// DecryptionKeys returns all private keys that are valid for decryption. +func (el EntityList) DecryptionKeys() (keys []Key) { + for _, e := range el { + for _, subKey := range e.Subkeys { + if subKey.PrivateKey != nil && (!subKey.Sig.FlagsValid || subKey.Sig.FlagEncryptStorage || subKey.Sig.FlagEncryptCommunications) { + keys = append(keys, Key{e, subKey.PublicKey, subKey.PrivateKey, subKey.Sig}) + } + } + } + return +} + +// ReadArmoredKeyRing reads one or more public/private keys from an armor keyring file. +func ReadArmoredKeyRing(r io.Reader) (EntityList, os.Error) { + block, err := armor.Decode(r) + if err == os.EOF { + return nil, error.InvalidArgumentError("no armored data found") + } + if err != nil { + return nil, err + } + if block.Type != PublicKeyType && block.Type != PrivateKeyType { + return nil, error.InvalidArgumentError("expected public or private key block, got: " + block.Type) + } + + return ReadKeyRing(block.Body) +} + +// ReadKeyRing reads one or more public/private keys. Unsupported keys are +// ignored as long as at least a single valid key is found. +func ReadKeyRing(r io.Reader) (el EntityList, err os.Error) { + packets := packet.NewReader(r) + var lastUnsupportedError os.Error + + for { + var e *Entity + e, err = readEntity(packets) + if err != nil { + if _, ok := err.(error.UnsupportedError); ok { + lastUnsupportedError = err + err = readToNextPublicKey(packets) + } + if err == os.EOF { + err = nil + break + } + if err != nil { + el = nil + break + } + } else { + el = append(el, e) + } + } + + if len(el) == 0 && err == nil { + err = lastUnsupportedError + } + return +} + +// readToNextPublicKey reads packets until the start of the entity and leaves +// the first packet of the new entity in the Reader. +func readToNextPublicKey(packets *packet.Reader) (err os.Error) { + var p packet.Packet + for { + p, err = packets.Next() + if err == os.EOF { + return + } else if err != nil { + if _, ok := err.(error.UnsupportedError); ok { + err = nil + continue + } + return + } + + if pk, ok := p.(*packet.PublicKey); ok && !pk.IsSubkey { + packets.Unread(p) + return + } + } + + panic("unreachable") +} + +// readEntity reads an entity (public key, identities, subkeys etc) from the +// given Reader. +func readEntity(packets *packet.Reader) (*Entity, os.Error) { + e := new(Entity) + e.Identities = make(map[string]*Identity) + + p, err := packets.Next() + if err != nil { + return nil, err + } + + var ok bool + if e.PrimaryKey, ok = p.(*packet.PublicKey); !ok { + if e.PrivateKey, ok = p.(*packet.PrivateKey); !ok { + packets.Unread(p) + return nil, error.StructuralError("first packet was not a public/private key") + } else { + e.PrimaryKey = &e.PrivateKey.PublicKey + } + } + + if !e.PrimaryKey.PubKeyAlgo.CanSign() { + return nil, error.StructuralError("primary key cannot be used for signatures") + } + + var current *Identity +EachPacket: + for { + p, err := packets.Next() + if err == os.EOF { + break + } else if err != nil { + return nil, err + } + + switch pkt := p.(type) { + case *packet.UserId: + current = new(Identity) + current.Name = pkt.Id + current.UserId = pkt + e.Identities[pkt.Id] = current + + for { + p, err = packets.Next() + if err == os.EOF { + return nil, io.ErrUnexpectedEOF + } else if err != nil { + return nil, err + } + + sig, ok := p.(*packet.Signature) + if !ok { + return nil, error.StructuralError("user ID packet not followed by self-signature") + } + + if (sig.SigType == packet.SigTypePositiveCert || sig.SigType == packet.SigTypeGenericCert) && sig.IssuerKeyId != nil && *sig.IssuerKeyId == e.PrimaryKey.KeyId { + if err = e.PrimaryKey.VerifyUserIdSignature(pkt.Id, sig); err != nil { + return nil, error.StructuralError("user ID self-signature invalid: " + err.String()) + } + current.SelfSignature = sig + break + } + current.Signatures = append(current.Signatures, sig) + } + case *packet.Signature: + if current == nil { + return nil, error.StructuralError("signature packet found before user id packet") + } + current.Signatures = append(current.Signatures, pkt) + case *packet.PrivateKey: + if pkt.IsSubkey == false { + packets.Unread(p) + break EachPacket + } + err = addSubkey(e, packets, &pkt.PublicKey, pkt) + if err != nil { + return nil, err + } + case *packet.PublicKey: + if pkt.IsSubkey == false { + packets.Unread(p) + break EachPacket + } + err = addSubkey(e, packets, pkt, nil) + if err != nil { + return nil, err + } + default: + // we ignore unknown packets + } + } + + if len(e.Identities) == 0 { + return nil, error.StructuralError("entity without any identities") + } + + return e, nil +} + +func addSubkey(e *Entity, packets *packet.Reader, pub *packet.PublicKey, priv *packet.PrivateKey) os.Error { + var subKey Subkey + subKey.PublicKey = pub + subKey.PrivateKey = priv + p, err := packets.Next() + if err == os.EOF { + return io.ErrUnexpectedEOF + } + if err != nil { + return error.StructuralError("subkey signature invalid: " + err.String()) + } + var ok bool + subKey.Sig, ok = p.(*packet.Signature) + if !ok { + return error.StructuralError("subkey packet not followed by signature") + } + if subKey.Sig.SigType != packet.SigTypeSubkeyBinding { + return error.StructuralError("subkey signature with wrong type") + } + err = e.PrimaryKey.VerifyKeySignature(subKey.PublicKey, subKey.Sig) + if err != nil { + return error.StructuralError("subkey signature invalid: " + err.String()) + } + e.Subkeys = append(e.Subkeys, subKey) + return nil +} + +const defaultRSAKeyBits = 2048 + +// NewEntity returns an Entity that contains a fresh RSA/RSA keypair with a +// single identity composed of the given full name, comment and email, any of +// which may be empty but must not contain any of "()<>\x00". +func NewEntity(rand io.Reader, currentTimeSecs int64, name, comment, email string) (*Entity, os.Error) { + uid := packet.NewUserId(name, comment, email) + if uid == nil { + return nil, error.InvalidArgumentError("user id field contained invalid characters") + } + signingPriv, err := rsa.GenerateKey(rand, defaultRSAKeyBits) + if err != nil { + return nil, err + } + encryptingPriv, err := rsa.GenerateKey(rand, defaultRSAKeyBits) + if err != nil { + return nil, err + } + + t := uint32(currentTimeSecs) + + e := &Entity{ + PrimaryKey: packet.NewRSAPublicKey(t, &signingPriv.PublicKey, false /* not a subkey */ ), + PrivateKey: packet.NewRSAPrivateKey(t, signingPriv, false /* not a subkey */ ), + Identities: make(map[string]*Identity), + } + isPrimaryId := true + e.Identities[uid.Id] = &Identity{ + Name: uid.Name, + UserId: uid, + SelfSignature: &packet.Signature{ + CreationTime: t, + SigType: packet.SigTypePositiveCert, + PubKeyAlgo: packet.PubKeyAlgoRSA, + Hash: crypto.SHA256, + IsPrimaryId: &isPrimaryId, + FlagsValid: true, + FlagSign: true, + FlagCertify: true, + IssuerKeyId: &e.PrimaryKey.KeyId, + }, + } + + e.Subkeys = make([]Subkey, 1) + e.Subkeys[0] = Subkey{ + PublicKey: packet.NewRSAPublicKey(t, &encryptingPriv.PublicKey, true /* is a subkey */ ), + PrivateKey: packet.NewRSAPrivateKey(t, encryptingPriv, true /* is a subkey */ ), + Sig: &packet.Signature{ + CreationTime: t, + SigType: packet.SigTypeSubkeyBinding, + PubKeyAlgo: packet.PubKeyAlgoRSA, + Hash: crypto.SHA256, + FlagsValid: true, + FlagEncryptStorage: true, + FlagEncryptCommunications: true, + IssuerKeyId: &e.PrimaryKey.KeyId, + }, + } + + return e, nil +} + +// SerializePrivate serializes an Entity, including private key material, to +// the given Writer. For now, it must only be used on an Entity returned from +// NewEntity. +func (e *Entity) SerializePrivate(w io.Writer) (err os.Error) { + err = e.PrivateKey.Serialize(w) + if err != nil { + return + } + for _, ident := range e.Identities { + err = ident.UserId.Serialize(w) + if err != nil { + return + } + err = ident.SelfSignature.SignUserId(ident.UserId.Id, e.PrimaryKey, e.PrivateKey) + if err != nil { + return + } + err = ident.SelfSignature.Serialize(w) + if err != nil { + return + } + } + for _, subkey := range e.Subkeys { + err = subkey.PrivateKey.Serialize(w) + if err != nil { + return + } + err = subkey.Sig.SignKey(subkey.PublicKey, e.PrivateKey) + if err != nil { + return + } + err = subkey.Sig.Serialize(w) + if err != nil { + return + } + } + return nil +} + +// Serialize writes the public part of the given Entity to w. (No private +// key material will be output). +func (e *Entity) Serialize(w io.Writer) os.Error { + err := e.PrimaryKey.Serialize(w) + if err != nil { + return err + } + for _, ident := range e.Identities { + err = ident.UserId.Serialize(w) + if err != nil { + return err + } + err = ident.SelfSignature.Serialize(w) + if err != nil { + return err + } + for _, sig := range ident.Signatures { + err = sig.Serialize(w) + if err != nil { + return err + } + } + } + for _, subkey := range e.Subkeys { + err = subkey.PublicKey.Serialize(w) + if err != nil { + return err + } + err = subkey.Sig.Serialize(w) + if err != nil { + return err + } + } + return nil +} + +// SignIdentity adds a signature to e, from signer, attesting that identity is +// associated with e. The provided identity must already be an element of +// e.Identities and the private key of signer must have been decrypted if +// necessary. +func (e *Entity) SignIdentity(identity string, signer *Entity) os.Error { + if signer.PrivateKey == nil { + return error.InvalidArgumentError("signing Entity must have a private key") + } + if signer.PrivateKey.Encrypted { + return error.InvalidArgumentError("signing Entity's private key must be decrypted") + } + ident, ok := e.Identities[identity] + if !ok { + return error.InvalidArgumentError("given identity string not found in Entity") + } + + sig := &packet.Signature{ + SigType: packet.SigTypeGenericCert, + PubKeyAlgo: signer.PrivateKey.PubKeyAlgo, + Hash: crypto.SHA256, + CreationTime: uint32(time.Seconds()), + IssuerKeyId: &signer.PrivateKey.KeyId, + } + if err := sig.SignKey(e.PrimaryKey, signer.PrivateKey); err != nil { + return err + } + ident.Signatures = append(ident.Signatures, sig) + return nil +} diff --git a/src/pkg/crypto/openpgp/packet/Makefile b/src/pkg/crypto/openpgp/packet/Makefile new file mode 100644 index 000000000..0f0d94eb1 --- /dev/null +++ b/src/pkg/crypto/openpgp/packet/Makefile @@ -0,0 +1,22 @@ +# Copyright 2011 The Go Authors. All rights reserved. +# Use of this source code is governed by a BSD-style +# license that can be found in the LICENSE file. + +include ../../../../Make.inc + +TARG=crypto/openpgp/packet +GOFILES=\ + compressed.go\ + encrypted_key.go\ + literal.go\ + one_pass_signature.go\ + packet.go\ + private_key.go\ + public_key.go\ + reader.go\ + signature.go\ + symmetrically_encrypted.go\ + symmetric_key_encrypted.go\ + userid.go\ + +include ../../../../Make.pkg diff --git a/src/pkg/crypto/openpgp/packet/compressed.go b/src/pkg/crypto/openpgp/packet/compressed.go new file mode 100644 index 000000000..1c15c24c4 --- /dev/null +++ b/src/pkg/crypto/openpgp/packet/compressed.go @@ -0,0 +1,39 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package packet + +import ( + "compress/flate" + "compress/zlib" + "crypto/openpgp/error" + "io" + "os" + "strconv" +) + +// Compressed represents a compressed OpenPGP packet. The decompressed contents +// will contain more OpenPGP packets. See RFC 4880, section 5.6. +type Compressed struct { + Body io.Reader +} + +func (c *Compressed) parse(r io.Reader) os.Error { + var buf [1]byte + _, err := readFull(r, buf[:]) + if err != nil { + return err + } + + switch buf[0] { + case 1: + c.Body = flate.NewReader(r) + case 2: + c.Body, err = zlib.NewReader(r) + default: + err = error.UnsupportedError("unknown compression algorithm: " + strconv.Itoa(int(buf[0]))) + } + + return err +} diff --git a/src/pkg/crypto/openpgp/packet/compressed_test.go b/src/pkg/crypto/openpgp/packet/compressed_test.go new file mode 100644 index 000000000..24fe501ed --- /dev/null +++ b/src/pkg/crypto/openpgp/packet/compressed_test.go @@ -0,0 +1,41 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package packet + +import ( + "bytes" + "encoding/hex" + "os" + "io/ioutil" + "testing" +) + +func TestCompressed(t *testing.T) { + packet, err := Read(readerFromHex(compressedHex)) + if err != nil { + t.Errorf("failed to read Compressed: %s", err) + return + } + + c, ok := packet.(*Compressed) + if !ok { + t.Error("didn't find Compressed packet") + return + } + + contents, err := ioutil.ReadAll(c.Body) + if err != nil && err != os.EOF { + t.Error(err) + return + } + + expected, _ := hex.DecodeString(compressedExpectedHex) + if !bytes.Equal(expected, contents) { + t.Errorf("got:%x want:%x", contents, expected) + } +} + +const compressedHex = "a3013b2d90c4e02b72e25f727e5e496a5e49b11e1700" +const compressedExpectedHex = "cb1062004d14c8fe636f6e74656e74732e0a" diff --git a/src/pkg/crypto/openpgp/packet/encrypted_key.go b/src/pkg/crypto/openpgp/packet/encrypted_key.go new file mode 100644 index 000000000..b4730cbc9 --- /dev/null +++ b/src/pkg/crypto/openpgp/packet/encrypted_key.go @@ -0,0 +1,168 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package packet + +import ( + "big" + "crypto/openpgp/elgamal" + "crypto/openpgp/error" + "crypto/rand" + "crypto/rsa" + "encoding/binary" + "io" + "os" + "strconv" +) + +const encryptedKeyVersion = 3 + +// EncryptedKey represents a public-key encrypted session key. See RFC 4880, +// section 5.1. +type EncryptedKey struct { + KeyId uint64 + Algo PublicKeyAlgorithm + CipherFunc CipherFunction // only valid after a successful Decrypt + Key []byte // only valid after a successful Decrypt + + encryptedMPI1, encryptedMPI2 []byte +} + +func (e *EncryptedKey) parse(r io.Reader) (err os.Error) { + var buf [10]byte + _, err = readFull(r, buf[:]) + if err != nil { + return + } + if buf[0] != encryptedKeyVersion { + return error.UnsupportedError("unknown EncryptedKey version " + strconv.Itoa(int(buf[0]))) + } + e.KeyId = binary.BigEndian.Uint64(buf[1:9]) + e.Algo = PublicKeyAlgorithm(buf[9]) + switch e.Algo { + case PubKeyAlgoRSA, PubKeyAlgoRSAEncryptOnly: + e.encryptedMPI1, _, err = readMPI(r) + case PubKeyAlgoElGamal: + e.encryptedMPI1, _, err = readMPI(r) + if err != nil { + return + } + e.encryptedMPI2, _, err = readMPI(r) + } + _, err = consumeAll(r) + return +} + +func checksumKeyMaterial(key []byte) uint16 { + var checksum uint16 + for _, v := range key { + checksum += uint16(v) + } + return checksum +} + +// Decrypt decrypts an encrypted session key with the given private key. The +// private key must have been decrypted first. +func (e *EncryptedKey) Decrypt(priv *PrivateKey) os.Error { + var err os.Error + var b []byte + + // TODO(agl): use session key decryption routines here to avoid + // padding oracle attacks. + switch priv.PubKeyAlgo { + case PubKeyAlgoRSA, PubKeyAlgoRSAEncryptOnly: + b, err = rsa.DecryptPKCS1v15(rand.Reader, priv.PrivateKey.(*rsa.PrivateKey), e.encryptedMPI1) + case PubKeyAlgoElGamal: + c1 := new(big.Int).SetBytes(e.encryptedMPI1) + c2 := new(big.Int).SetBytes(e.encryptedMPI2) + b, err = elgamal.Decrypt(priv.PrivateKey.(*elgamal.PrivateKey), c1, c2) + default: + err = error.InvalidArgumentError("cannot decrypted encrypted session key with private key of type " + strconv.Itoa(int(priv.PubKeyAlgo))) + } + + if err != nil { + return err + } + + e.CipherFunc = CipherFunction(b[0]) + e.Key = b[1 : len(b)-2] + expectedChecksum := uint16(b[len(b)-2])<<8 | uint16(b[len(b)-1]) + checksum := checksumKeyMaterial(e.Key) + if checksum != expectedChecksum { + return error.StructuralError("EncryptedKey checksum incorrect") + } + + return nil +} + +// SerializeEncryptedKey serializes an encrypted key packet to w that contains +// key, encrypted to pub. +func SerializeEncryptedKey(w io.Writer, rand io.Reader, pub *PublicKey, cipherFunc CipherFunction, key []byte) os.Error { + var buf [10]byte + buf[0] = encryptedKeyVersion + binary.BigEndian.PutUint64(buf[1:9], pub.KeyId) + buf[9] = byte(pub.PubKeyAlgo) + + keyBlock := make([]byte, 1 /* cipher type */ +len(key)+2 /* checksum */ ) + keyBlock[0] = byte(cipherFunc) + copy(keyBlock[1:], key) + checksum := checksumKeyMaterial(key) + keyBlock[1+len(key)] = byte(checksum >> 8) + keyBlock[1+len(key)+1] = byte(checksum) + + switch pub.PubKeyAlgo { + case PubKeyAlgoRSA, PubKeyAlgoRSAEncryptOnly: + return serializeEncryptedKeyRSA(w, rand, buf, pub.PublicKey.(*rsa.PublicKey), keyBlock) + case PubKeyAlgoElGamal: + return serializeEncryptedKeyElGamal(w, rand, buf, pub.PublicKey.(*elgamal.PublicKey), keyBlock) + case PubKeyAlgoDSA, PubKeyAlgoRSASignOnly: + return error.InvalidArgumentError("cannot encrypt to public key of type " + strconv.Itoa(int(pub.PubKeyAlgo))) + } + + return error.UnsupportedError("encrypting a key to public key of type " + strconv.Itoa(int(pub.PubKeyAlgo))) +} + +func serializeEncryptedKeyRSA(w io.Writer, rand io.Reader, header [10]byte, pub *rsa.PublicKey, keyBlock []byte) os.Error { + cipherText, err := rsa.EncryptPKCS1v15(rand, pub, keyBlock) + if err != nil { + return error.InvalidArgumentError("RSA encryption failed: " + err.String()) + } + + packetLen := 10 /* header length */ + 2 /* mpi size */ + len(cipherText) + + err = serializeHeader(w, packetTypeEncryptedKey, packetLen) + if err != nil { + return err + } + _, err = w.Write(header[:]) + if err != nil { + return err + } + return writeMPI(w, 8*uint16(len(cipherText)), cipherText) +} + +func serializeEncryptedKeyElGamal(w io.Writer, rand io.Reader, header [10]byte, pub *elgamal.PublicKey, keyBlock []byte) os.Error { + c1, c2, err := elgamal.Encrypt(rand, pub, keyBlock) + if err != nil { + return error.InvalidArgumentError("ElGamal encryption failed: " + err.String()) + } + + packetLen := 10 /* header length */ + packetLen += 2 /* mpi size */ + (c1.BitLen()+7)/8 + packetLen += 2 /* mpi size */ + (c2.BitLen()+7)/8 + + err = serializeHeader(w, packetTypeEncryptedKey, packetLen) + if err != nil { + return err + } + _, err = w.Write(header[:]) + if err != nil { + return err + } + err = writeBig(w, c1) + if err != nil { + return err + } + return writeBig(w, c2) +} diff --git a/src/pkg/crypto/openpgp/packet/encrypted_key_test.go b/src/pkg/crypto/openpgp/packet/encrypted_key_test.go new file mode 100644 index 000000000..b402245bd --- /dev/null +++ b/src/pkg/crypto/openpgp/packet/encrypted_key_test.go @@ -0,0 +1,126 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package packet + +import ( + "big" + "bytes" + "crypto/rand" + "crypto/rsa" + "fmt" + "testing" +) + +func bigFromBase10(s string) *big.Int { + b, ok := new(big.Int).SetString(s, 10) + if !ok { + panic("bigFromBase10 failed") + } + return b +} + +var encryptedKeyPub = rsa.PublicKey{ + E: 65537, + N: bigFromBase10("115804063926007623305902631768113868327816898845124614648849934718568541074358183759250136204762053879858102352159854352727097033322663029387610959884180306668628526686121021235757016368038585212410610742029286439607686208110250133174279811431933746643015923132833417396844716207301518956640020862630546868823"), +} + +var encryptedKeyRSAPriv = &rsa.PrivateKey{ + PublicKey: encryptedKeyPub, + D: bigFromBase10("32355588668219869544751561565313228297765464314098552250409557267371233892496951383426602439009993875125222579159850054973310859166139474359774543943714622292329487391199285040721944491839695981199720170366763547754915493640685849961780092241140181198779299712578774460837139360803883139311171713302987058393"), +} + +var encryptedKeyPriv = &PrivateKey{ + PublicKey: PublicKey{ + PubKeyAlgo: PubKeyAlgoRSA, + }, + PrivateKey: encryptedKeyRSAPriv, +} + +func TestDecryptingEncryptedKey(t *testing.T) { + const encryptedKeyHex = "c18c032a67d68660df41c70104005789d0de26b6a50c985a02a13131ca829c413a35d0e6fa8d6842599252162808ac7439c72151c8c6183e76923fe3299301414d0c25a2f06a2257db3839e7df0ec964773f6e4c4ac7ff3b48c444237166dd46ba8ff443a5410dc670cb486672fdbe7c9dfafb75b4fea83af3a204fe2a7dfa86bd20122b4f3d2646cbeecb8f7be8" + const expectedKeyHex = "d930363f7e0308c333b9618617ea728963d8df993665ae7be1092d4926fd864b" + + p, err := Read(readerFromHex(encryptedKeyHex)) + if err != nil { + t.Errorf("error from Read: %s", err) + return + } + ek, ok := p.(*EncryptedKey) + if !ok { + t.Errorf("didn't parse an EncryptedKey, got %#v", p) + return + } + + if ek.KeyId != 0x2a67d68660df41c7 || ek.Algo != PubKeyAlgoRSA { + t.Errorf("unexpected EncryptedKey contents: %#v", ek) + return + } + + err = ek.Decrypt(encryptedKeyPriv) + if err != nil { + t.Errorf("error from Decrypt: %s", err) + return + } + + if ek.CipherFunc != CipherAES256 { + t.Errorf("unexpected EncryptedKey contents: %#v", ek) + return + } + + keyHex := fmt.Sprintf("%x", ek.Key) + if keyHex != expectedKeyHex { + t.Errorf("bad key, got %s want %x", keyHex, expectedKeyHex) + } +} + +func TestEncryptingEncryptedKey(t *testing.T) { + key := []byte{1, 2, 3, 4} + const expectedKeyHex = "01020304" + const keyId = 42 + + pub := &PublicKey{ + PublicKey: &encryptedKeyPub, + KeyId: keyId, + PubKeyAlgo: PubKeyAlgoRSAEncryptOnly, + } + + buf := new(bytes.Buffer) + err := SerializeEncryptedKey(buf, rand.Reader, pub, CipherAES128, key) + if err != nil { + t.Errorf("error writing encrypted key packet: %s", err) + } + + p, err := Read(buf) + if err != nil { + t.Errorf("error from Read: %s", err) + return + } + ek, ok := p.(*EncryptedKey) + if !ok { + t.Errorf("didn't parse an EncryptedKey, got %#v", p) + return + } + + if ek.KeyId != keyId || ek.Algo != PubKeyAlgoRSAEncryptOnly { + t.Errorf("unexpected EncryptedKey contents: %#v", ek) + return + } + + err = ek.Decrypt(encryptedKeyPriv) + if err != nil { + t.Errorf("error from Decrypt: %s", err) + return + } + + if ek.CipherFunc != CipherAES128 { + t.Errorf("unexpected EncryptedKey contents: %#v", ek) + return + } + + keyHex := fmt.Sprintf("%x", ek.Key) + if keyHex != expectedKeyHex { + t.Errorf("bad key, got %s want %x", keyHex, expectedKeyHex) + } +} diff --git a/src/pkg/crypto/openpgp/packet/literal.go b/src/pkg/crypto/openpgp/packet/literal.go new file mode 100644 index 000000000..9411572d7 --- /dev/null +++ b/src/pkg/crypto/openpgp/packet/literal.go @@ -0,0 +1,90 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package packet + +import ( + "encoding/binary" + "io" + "os" +) + +// LiteralData represents an encrypted file. See RFC 4880, section 5.9. +type LiteralData struct { + IsBinary bool + FileName string + Time uint32 // Unix epoch time. Either creation time or modification time. 0 means undefined. + Body io.Reader +} + +// ForEyesOnly returns whether the contents of the LiteralData have been marked +// as especially sensitive. +func (l *LiteralData) ForEyesOnly() bool { + return l.FileName == "_CONSOLE" +} + +func (l *LiteralData) parse(r io.Reader) (err os.Error) { + var buf [256]byte + + _, err = readFull(r, buf[:2]) + if err != nil { + return + } + + l.IsBinary = buf[0] == 'b' + fileNameLen := int(buf[1]) + + _, err = readFull(r, buf[:fileNameLen]) + if err != nil { + return + } + + l.FileName = string(buf[:fileNameLen]) + + _, err = readFull(r, buf[:4]) + if err != nil { + return + } + + l.Time = binary.BigEndian.Uint32(buf[:4]) + l.Body = r + return +} + +// SerializeLiteral serializes a literal data packet to w and returns a +// WriteCloser to which the data itself can be written and which MUST be closed +// on completion. The fileName is truncated to 255 bytes. +func SerializeLiteral(w io.WriteCloser, isBinary bool, fileName string, time uint32) (plaintext io.WriteCloser, err os.Error) { + var buf [4]byte + buf[0] = 't' + if isBinary { + buf[0] = 'b' + } + if len(fileName) > 255 { + fileName = fileName[:255] + } + buf[1] = byte(len(fileName)) + + inner, err := serializeStreamHeader(w, packetTypeLiteralData) + if err != nil { + return + } + + _, err = inner.Write(buf[:2]) + if err != nil { + return + } + _, err = inner.Write([]byte(fileName)) + if err != nil { + return + } + binary.BigEndian.PutUint32(buf[:], time) + _, err = inner.Write(buf[:]) + if err != nil { + return + } + + plaintext = inner + return +} diff --git a/src/pkg/crypto/openpgp/packet/one_pass_signature.go b/src/pkg/crypto/openpgp/packet/one_pass_signature.go new file mode 100644 index 000000000..ca826e4f4 --- /dev/null +++ b/src/pkg/crypto/openpgp/packet/one_pass_signature.go @@ -0,0 +1,74 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package packet + +import ( + "crypto" + "crypto/openpgp/error" + "crypto/openpgp/s2k" + "encoding/binary" + "io" + "os" + "strconv" +) + +// OnePassSignature represents a one-pass signature packet. See RFC 4880, +// section 5.4. +type OnePassSignature struct { + SigType SignatureType + Hash crypto.Hash + PubKeyAlgo PublicKeyAlgorithm + KeyId uint64 + IsLast bool +} + +const onePassSignatureVersion = 3 + +func (ops *OnePassSignature) parse(r io.Reader) (err os.Error) { + var buf [13]byte + + _, err = readFull(r, buf[:]) + if err != nil { + return + } + if buf[0] != onePassSignatureVersion { + err = error.UnsupportedError("one-pass-signature packet version " + strconv.Itoa(int(buf[0]))) + } + + var ok bool + ops.Hash, ok = s2k.HashIdToHash(buf[2]) + if !ok { + return error.UnsupportedError("hash function: " + strconv.Itoa(int(buf[2]))) + } + + ops.SigType = SignatureType(buf[1]) + ops.PubKeyAlgo = PublicKeyAlgorithm(buf[3]) + ops.KeyId = binary.BigEndian.Uint64(buf[4:12]) + ops.IsLast = buf[12] != 0 + return +} + +// Serialize marshals the given OnePassSignature to w. +func (ops *OnePassSignature) Serialize(w io.Writer) os.Error { + var buf [13]byte + buf[0] = onePassSignatureVersion + buf[1] = uint8(ops.SigType) + var ok bool + buf[2], ok = s2k.HashToHashId(ops.Hash) + if !ok { + return error.UnsupportedError("hash type: " + strconv.Itoa(int(ops.Hash))) + } + buf[3] = uint8(ops.PubKeyAlgo) + binary.BigEndian.PutUint64(buf[4:12], ops.KeyId) + if ops.IsLast { + buf[12] = 1 + } + + if err := serializeHeader(w, packetTypeOnePassSignature, len(buf)); err != nil { + return err + } + _, err := w.Write(buf[:]) + return err +} diff --git a/src/pkg/crypto/openpgp/packet/packet.go b/src/pkg/crypto/openpgp/packet/packet.go new file mode 100644 index 000000000..1d7297e38 --- /dev/null +++ b/src/pkg/crypto/openpgp/packet/packet.go @@ -0,0 +1,483 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Package packet implements parsing and serialization of OpenPGP packets, as +// specified in RFC 4880. +package packet + +import ( + "big" + "crypto/aes" + "crypto/cast5" + "crypto/cipher" + "crypto/openpgp/error" + "io" + "os" +) + +// readFull is the same as io.ReadFull except that reading zero bytes returns +// ErrUnexpectedEOF rather than EOF. +func readFull(r io.Reader, buf []byte) (n int, err os.Error) { + n, err = io.ReadFull(r, buf) + if err == os.EOF { + err = io.ErrUnexpectedEOF + } + return +} + +// readLength reads an OpenPGP length from r. See RFC 4880, section 4.2.2. +func readLength(r io.Reader) (length int64, isPartial bool, err os.Error) { + var buf [4]byte + _, err = readFull(r, buf[:1]) + if err != nil { + return + } + switch { + case buf[0] < 192: + length = int64(buf[0]) + case buf[0] < 224: + length = int64(buf[0]-192) << 8 + _, err = readFull(r, buf[0:1]) + if err != nil { + return + } + length += int64(buf[0]) + 192 + case buf[0] < 255: + length = int64(1) << (buf[0] & 0x1f) + isPartial = true + default: + _, err = readFull(r, buf[0:4]) + if err != nil { + return + } + length = int64(buf[0])<<24 | + int64(buf[1])<<16 | + int64(buf[2])<<8 | + int64(buf[3]) + } + return +} + +// partialLengthReader wraps an io.Reader and handles OpenPGP partial lengths. +// The continuation lengths are parsed and removed from the stream and EOF is +// returned at the end of the packet. See RFC 4880, section 4.2.2.4. +type partialLengthReader struct { + r io.Reader + remaining int64 + isPartial bool +} + +func (r *partialLengthReader) Read(p []byte) (n int, err os.Error) { + for r.remaining == 0 { + if !r.isPartial { + return 0, os.EOF + } + r.remaining, r.isPartial, err = readLength(r.r) + if err != nil { + return 0, err + } + } + + toRead := int64(len(p)) + if toRead > r.remaining { + toRead = r.remaining + } + + n, err = r.r.Read(p[:int(toRead)]) + r.remaining -= int64(n) + if n < int(toRead) && err == os.EOF { + err = io.ErrUnexpectedEOF + } + return +} + +// partialLengthWriter writes a stream of data using OpenPGP partial lengths. +// See RFC 4880, section 4.2.2.4. +type partialLengthWriter struct { + w io.WriteCloser + lengthByte [1]byte +} + +func (w *partialLengthWriter) Write(p []byte) (n int, err os.Error) { + for len(p) > 0 { + for power := uint(14); power < 32; power-- { + l := 1 << power + if len(p) >= l { + w.lengthByte[0] = 224 + uint8(power) + _, err = w.w.Write(w.lengthByte[:]) + if err != nil { + return + } + var m int + m, err = w.w.Write(p[:l]) + n += m + if err != nil { + return + } + p = p[l:] + break + } + } + } + return +} + +func (w *partialLengthWriter) Close() os.Error { + w.lengthByte[0] = 0 + _, err := w.w.Write(w.lengthByte[:]) + if err != nil { + return err + } + return w.w.Close() +} + +// A spanReader is an io.LimitReader, but it returns ErrUnexpectedEOF if the +// underlying Reader returns EOF before the limit has been reached. +type spanReader struct { + r io.Reader + n int64 +} + +func (l *spanReader) Read(p []byte) (n int, err os.Error) { + if l.n <= 0 { + return 0, os.EOF + } + if int64(len(p)) > l.n { + p = p[0:l.n] + } + n, err = l.r.Read(p) + l.n -= int64(n) + if l.n > 0 && err == os.EOF { + err = io.ErrUnexpectedEOF + } + return +} + +// readHeader parses a packet header and returns an io.Reader which will return +// the contents of the packet. See RFC 4880, section 4.2. +func readHeader(r io.Reader) (tag packetType, length int64, contents io.Reader, err os.Error) { + var buf [4]byte + _, err = io.ReadFull(r, buf[:1]) + if err != nil { + return + } + if buf[0]&0x80 == 0 { + err = error.StructuralError("tag byte does not have MSB set") + return + } + if buf[0]&0x40 == 0 { + // Old format packet + tag = packetType((buf[0] & 0x3f) >> 2) + lengthType := buf[0] & 3 + if lengthType == 3 { + length = -1 + contents = r + return + } + lengthBytes := 1 << lengthType + _, err = readFull(r, buf[0:lengthBytes]) + if err != nil { + return + } + for i := 0; i < lengthBytes; i++ { + length <<= 8 + length |= int64(buf[i]) + } + contents = &spanReader{r, length} + return + } + + // New format packet + tag = packetType(buf[0] & 0x3f) + length, isPartial, err := readLength(r) + if err != nil { + return + } + if isPartial { + contents = &partialLengthReader{ + remaining: length, + isPartial: true, + r: r, + } + length = -1 + } else { + contents = &spanReader{r, length} + } + return +} + +// serializeHeader writes an OpenPGP packet header to w. See RFC 4880, section +// 4.2. +func serializeHeader(w io.Writer, ptype packetType, length int) (err os.Error) { + var buf [6]byte + var n int + + buf[0] = 0x80 | 0x40 | byte(ptype) + if length < 192 { + buf[1] = byte(length) + n = 2 + } else if length < 8384 { + length -= 192 + buf[1] = 192 + byte(length>>8) + buf[2] = byte(length) + n = 3 + } else { + buf[1] = 255 + buf[2] = byte(length >> 24) + buf[3] = byte(length >> 16) + buf[4] = byte(length >> 8) + buf[5] = byte(length) + n = 6 + } + + _, err = w.Write(buf[:n]) + return +} + +// serializeStreamHeader writes an OpenPGP packet header to w where the +// length of the packet is unknown. It returns a io.WriteCloser which can be +// used to write the contents of the packet. See RFC 4880, section 4.2. +func serializeStreamHeader(w io.WriteCloser, ptype packetType) (out io.WriteCloser, err os.Error) { + var buf [1]byte + buf[0] = 0x80 | 0x40 | byte(ptype) + _, err = w.Write(buf[:]) + if err != nil { + return + } + out = &partialLengthWriter{w: w} + return +} + +// Packet represents an OpenPGP packet. Users are expected to try casting +// instances of this interface to specific packet types. +type Packet interface { + parse(io.Reader) os.Error +} + +// consumeAll reads from the given Reader until error, returning the number of +// bytes read. +func consumeAll(r io.Reader) (n int64, err os.Error) { + var m int + var buf [1024]byte + + for { + m, err = r.Read(buf[:]) + n += int64(m) + if err == os.EOF { + err = nil + return + } + if err != nil { + return + } + } + + panic("unreachable") +} + +// packetType represents the numeric ids of the different OpenPGP packet types. See +// http://www.iana.org/assignments/pgp-parameters/pgp-parameters.xhtml#pgp-parameters-2 +type packetType uint8 + +const ( + packetTypeEncryptedKey packetType = 1 + packetTypeSignature packetType = 2 + packetTypeSymmetricKeyEncrypted packetType = 3 + packetTypeOnePassSignature packetType = 4 + packetTypePrivateKey packetType = 5 + packetTypePublicKey packetType = 6 + packetTypePrivateSubkey packetType = 7 + packetTypeCompressed packetType = 8 + packetTypeSymmetricallyEncrypted packetType = 9 + packetTypeLiteralData packetType = 11 + packetTypeUserId packetType = 13 + packetTypePublicSubkey packetType = 14 + packetTypeSymmetricallyEncryptedMDC packetType = 18 +) + +// Read reads a single OpenPGP packet from the given io.Reader. If there is an +// error parsing a packet, the whole packet is consumed from the input. +func Read(r io.Reader) (p Packet, err os.Error) { + tag, _, contents, err := readHeader(r) + if err != nil { + return + } + + switch tag { + case packetTypeEncryptedKey: + p = new(EncryptedKey) + case packetTypeSignature: + p = new(Signature) + case packetTypeSymmetricKeyEncrypted: + p = new(SymmetricKeyEncrypted) + case packetTypeOnePassSignature: + p = new(OnePassSignature) + case packetTypePrivateKey, packetTypePrivateSubkey: + pk := new(PrivateKey) + if tag == packetTypePrivateSubkey { + pk.IsSubkey = true + } + p = pk + case packetTypePublicKey, packetTypePublicSubkey: + pk := new(PublicKey) + if tag == packetTypePublicSubkey { + pk.IsSubkey = true + } + p = pk + case packetTypeCompressed: + p = new(Compressed) + case packetTypeSymmetricallyEncrypted: + p = new(SymmetricallyEncrypted) + case packetTypeLiteralData: + p = new(LiteralData) + case packetTypeUserId: + p = new(UserId) + case packetTypeSymmetricallyEncryptedMDC: + se := new(SymmetricallyEncrypted) + se.MDC = true + p = se + default: + err = error.UnknownPacketTypeError(tag) + } + if p != nil { + err = p.parse(contents) + } + if err != nil { + consumeAll(contents) + } + return +} + +// SignatureType represents the different semantic meanings of an OpenPGP +// signature. See RFC 4880, section 5.2.1. +type SignatureType uint8 + +const ( + SigTypeBinary SignatureType = 0 + SigTypeText = 1 + SigTypeGenericCert = 0x10 + SigTypePersonaCert = 0x11 + SigTypeCasualCert = 0x12 + SigTypePositiveCert = 0x13 + SigTypeSubkeyBinding = 0x18 +) + +// PublicKeyAlgorithm represents the different public key system specified for +// OpenPGP. See +// http://www.iana.org/assignments/pgp-parameters/pgp-parameters.xhtml#pgp-parameters-12 +type PublicKeyAlgorithm uint8 + +const ( + PubKeyAlgoRSA PublicKeyAlgorithm = 1 + PubKeyAlgoRSAEncryptOnly PublicKeyAlgorithm = 2 + PubKeyAlgoRSASignOnly PublicKeyAlgorithm = 3 + PubKeyAlgoElGamal PublicKeyAlgorithm = 16 + PubKeyAlgoDSA PublicKeyAlgorithm = 17 +) + +// CanEncrypt returns true if it's possible to encrypt a message to a public +// key of the given type. +func (pka PublicKeyAlgorithm) CanEncrypt() bool { + switch pka { + case PubKeyAlgoRSA, PubKeyAlgoRSAEncryptOnly, PubKeyAlgoElGamal: + return true + } + return false +} + +// CanSign returns true if it's possible for a public key of the given type to +// sign a message. +func (pka PublicKeyAlgorithm) CanSign() bool { + switch pka { + case PubKeyAlgoRSA, PubKeyAlgoRSASignOnly, PubKeyAlgoDSA: + return true + } + return false +} + +// CipherFunction represents the different block ciphers specified for OpenPGP. See +// http://www.iana.org/assignments/pgp-parameters/pgp-parameters.xhtml#pgp-parameters-13 +type CipherFunction uint8 + +const ( + CipherCAST5 CipherFunction = 3 + CipherAES128 CipherFunction = 7 + CipherAES192 CipherFunction = 8 + CipherAES256 CipherFunction = 9 +) + +// KeySize returns the key size, in bytes, of cipher. +func (cipher CipherFunction) KeySize() int { + switch cipher { + case CipherCAST5: + return cast5.KeySize + case CipherAES128: + return 16 + case CipherAES192: + return 24 + case CipherAES256: + return 32 + } + return 0 +} + +// blockSize returns the block size, in bytes, of cipher. +func (cipher CipherFunction) blockSize() int { + switch cipher { + case CipherCAST5: + return 8 + case CipherAES128, CipherAES192, CipherAES256: + return 16 + } + return 0 +} + +// new returns a fresh instance of the given cipher. +func (cipher CipherFunction) new(key []byte) (block cipher.Block) { + switch cipher { + case CipherCAST5: + block, _ = cast5.NewCipher(key) + case CipherAES128, CipherAES192, CipherAES256: + block, _ = aes.NewCipher(key) + } + return +} + +// readMPI reads a big integer from r. The bit length returned is the bit +// length that was specified in r. This is preserved so that the integer can be +// reserialized exactly. +func readMPI(r io.Reader) (mpi []byte, bitLength uint16, err os.Error) { + var buf [2]byte + _, err = readFull(r, buf[0:]) + if err != nil { + return + } + bitLength = uint16(buf[0])<<8 | uint16(buf[1]) + numBytes := (int(bitLength) + 7) / 8 + mpi = make([]byte, numBytes) + _, err = readFull(r, mpi) + return +} + +// mpiLength returns the length of the given *big.Int when serialized as an +// MPI. +func mpiLength(n *big.Int) (mpiLengthInBytes int) { + mpiLengthInBytes = 2 /* MPI length */ + mpiLengthInBytes += (n.BitLen() + 7) / 8 + return +} + +// writeMPI serializes a big integer to w. +func writeMPI(w io.Writer, bitLength uint16, mpiBytes []byte) (err os.Error) { + _, err = w.Write([]byte{byte(bitLength >> 8), byte(bitLength)}) + if err == nil { + _, err = w.Write(mpiBytes) + } + return +} + +// writeBig serializes a *big.Int to w. +func writeBig(w io.Writer, i *big.Int) os.Error { + return writeMPI(w, uint16(i.BitLen()), i.Bytes()) +} diff --git a/src/pkg/crypto/openpgp/packet/packet_test.go b/src/pkg/crypto/openpgp/packet/packet_test.go new file mode 100644 index 000000000..23d9978ae --- /dev/null +++ b/src/pkg/crypto/openpgp/packet/packet_test.go @@ -0,0 +1,256 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package packet + +import ( + "bytes" + "crypto/openpgp/error" + "encoding/hex" + "fmt" + "io" + "io/ioutil" + "os" + "testing" +) + +func TestReadFull(t *testing.T) { + var out [4]byte + + b := bytes.NewBufferString("foo") + n, err := readFull(b, out[:3]) + if n != 3 || err != nil { + t.Errorf("full read failed n:%d err:%s", n, err) + } + + b = bytes.NewBufferString("foo") + n, err = readFull(b, out[:4]) + if n != 3 || err != io.ErrUnexpectedEOF { + t.Errorf("partial read failed n:%d err:%s", n, err) + } + + b = bytes.NewBuffer(nil) + n, err = readFull(b, out[:3]) + if n != 0 || err != io.ErrUnexpectedEOF { + t.Errorf("empty read failed n:%d err:%s", n, err) + } +} + +func readerFromHex(s string) io.Reader { + data, err := hex.DecodeString(s) + if err != nil { + panic("readerFromHex: bad input") + } + return bytes.NewBuffer(data) +} + +var readLengthTests = []struct { + hexInput string + length int64 + isPartial bool + err os.Error +}{ + {"", 0, false, io.ErrUnexpectedEOF}, + {"1f", 31, false, nil}, + {"c0", 0, false, io.ErrUnexpectedEOF}, + {"c101", 256 + 1 + 192, false, nil}, + {"e0", 1, true, nil}, + {"e1", 2, true, nil}, + {"e2", 4, true, nil}, + {"ff", 0, false, io.ErrUnexpectedEOF}, + {"ff00", 0, false, io.ErrUnexpectedEOF}, + {"ff0000", 0, false, io.ErrUnexpectedEOF}, + {"ff000000", 0, false, io.ErrUnexpectedEOF}, + {"ff00000000", 0, false, nil}, + {"ff01020304", 16909060, false, nil}, +} + +func TestReadLength(t *testing.T) { + for i, test := range readLengthTests { + length, isPartial, err := readLength(readerFromHex(test.hexInput)) + if test.err != nil { + if err != test.err { + t.Errorf("%d: expected different error got:%s want:%s", i, err, test.err) + } + continue + } + if err != nil { + t.Errorf("%d: unexpected error: %s", i, err) + continue + } + if length != test.length || isPartial != test.isPartial { + t.Errorf("%d: bad result got:(%d,%t) want:(%d,%t)", i, length, isPartial, test.length, test.isPartial) + } + } +} + +var partialLengthReaderTests = []struct { + hexInput string + err os.Error + hexOutput string +}{ + {"e0", io.ErrUnexpectedEOF, ""}, + {"e001", io.ErrUnexpectedEOF, ""}, + {"e0010102", nil, "0102"}, + {"ff00000000", nil, ""}, + {"e10102e1030400", nil, "01020304"}, + {"e101", io.ErrUnexpectedEOF, ""}, +} + +func TestPartialLengthReader(t *testing.T) { + for i, test := range partialLengthReaderTests { + r := &partialLengthReader{readerFromHex(test.hexInput), 0, true} + out, err := ioutil.ReadAll(r) + if test.err != nil { + if err != test.err { + t.Errorf("%d: expected different error got:%s want:%s", i, err, test.err) + } + continue + } + if err != nil { + t.Errorf("%d: unexpected error: %s", i, err) + continue + } + + got := fmt.Sprintf("%x", out) + if got != test.hexOutput { + t.Errorf("%d: got:%s want:%s", i, test.hexOutput, got) + } + } +} + +var readHeaderTests = []struct { + hexInput string + structuralError bool + unexpectedEOF bool + tag int + length int64 + hexOutput string +}{ + {"", false, false, 0, 0, ""}, + {"7f", true, false, 0, 0, ""}, + + // Old format headers + {"80", false, true, 0, 0, ""}, + {"8001", false, true, 0, 1, ""}, + {"800102", false, false, 0, 1, "02"}, + {"81000102", false, false, 0, 1, "02"}, + {"820000000102", false, false, 0, 1, "02"}, + {"860000000102", false, false, 1, 1, "02"}, + {"83010203", false, false, 0, -1, "010203"}, + + // New format headers + {"c0", false, true, 0, 0, ""}, + {"c000", false, false, 0, 0, ""}, + {"c00102", false, false, 0, 1, "02"}, + {"c0020203", false, false, 0, 2, "0203"}, + {"c00202", false, true, 0, 2, ""}, + {"c3020203", false, false, 3, 2, "0203"}, +} + +func TestReadHeader(t *testing.T) { + for i, test := range readHeaderTests { + tag, length, contents, err := readHeader(readerFromHex(test.hexInput)) + if test.structuralError { + if _, ok := err.(error.StructuralError); ok { + continue + } + t.Errorf("%d: expected StructuralError, got:%s", i, err) + continue + } + if err != nil { + if len(test.hexInput) == 0 && err == os.EOF { + continue + } + if !test.unexpectedEOF || err != io.ErrUnexpectedEOF { + t.Errorf("%d: unexpected error from readHeader: %s", i, err) + } + continue + } + if int(tag) != test.tag || length != test.length { + t.Errorf("%d: got:(%d,%d) want:(%d,%d)", i, int(tag), length, test.tag, test.length) + continue + } + + body, err := ioutil.ReadAll(contents) + if err != nil { + if !test.unexpectedEOF || err != io.ErrUnexpectedEOF { + t.Errorf("%d: unexpected error from contents: %s", i, err) + } + continue + } + if test.unexpectedEOF { + t.Errorf("%d: expected ErrUnexpectedEOF from contents but got no error", i) + continue + } + got := fmt.Sprintf("%x", body) + if got != test.hexOutput { + t.Errorf("%d: got:%s want:%s", i, got, test.hexOutput) + } + } +} + +func TestSerializeHeader(t *testing.T) { + tag := packetTypePublicKey + lengths := []int{0, 1, 2, 64, 192, 193, 8000, 8384, 8385, 10000} + + for _, length := range lengths { + buf := bytes.NewBuffer(nil) + serializeHeader(buf, tag, length) + tag2, length2, _, err := readHeader(buf) + if err != nil { + t.Errorf("length %d, err: %s", length, err) + } + if tag2 != tag { + t.Errorf("length %d, tag incorrect (got %d, want %d)", length, tag2, tag) + } + if int(length2) != length { + t.Errorf("length %d, length incorrect (got %d)", length, length2) + } + } +} + +func TestPartialLengths(t *testing.T) { + buf := bytes.NewBuffer(nil) + w := new(partialLengthWriter) + w.w = noOpCloser{buf} + + const maxChunkSize = 64 + + var b [maxChunkSize]byte + var n uint8 + for l := 1; l <= maxChunkSize; l++ { + for i := 0; i < l; i++ { + b[i] = n + n++ + } + m, err := w.Write(b[:l]) + if m != l { + t.Errorf("short write got: %d want: %d", m, l) + } + if err != nil { + t.Errorf("error from write: %s", err) + } + } + w.Close() + + want := (maxChunkSize * (maxChunkSize + 1)) / 2 + copyBuf := bytes.NewBuffer(nil) + r := &partialLengthReader{buf, 0, true} + m, err := io.Copy(copyBuf, r) + if m != int64(want) { + t.Errorf("short copy got: %d want: %d", m, want) + } + if err != nil { + t.Errorf("error from copy: %s", err) + } + + copyBytes := copyBuf.Bytes() + for i := 0; i < want; i++ { + if copyBytes[i] != uint8(i) { + t.Errorf("bad pattern in copy at %d", i) + break + } + } +} diff --git a/src/pkg/crypto/openpgp/packet/private_key.go b/src/pkg/crypto/openpgp/packet/private_key.go new file mode 100644 index 000000000..6f8133d98 --- /dev/null +++ b/src/pkg/crypto/openpgp/packet/private_key.go @@ -0,0 +1,301 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package packet + +import ( + "big" + "bytes" + "crypto/cipher" + "crypto/dsa" + "crypto/openpgp/elgamal" + "crypto/openpgp/error" + "crypto/openpgp/s2k" + "crypto/rsa" + "crypto/sha1" + "io" + "io/ioutil" + "os" + "strconv" +) + +// PrivateKey represents a possibly encrypted private key. See RFC 4880, +// section 5.5.3. +type PrivateKey struct { + PublicKey + Encrypted bool // if true then the private key is unavailable until Decrypt has been called. + encryptedData []byte + cipher CipherFunction + s2k func(out, in []byte) + PrivateKey interface{} // An *rsa.PrivateKey. + sha1Checksum bool + iv []byte +} + +func NewRSAPrivateKey(currentTimeSecs uint32, priv *rsa.PrivateKey, isSubkey bool) *PrivateKey { + pk := new(PrivateKey) + pk.PublicKey = *NewRSAPublicKey(currentTimeSecs, &priv.PublicKey, isSubkey) + pk.PrivateKey = priv + return pk +} + +func (pk *PrivateKey) parse(r io.Reader) (err os.Error) { + err = (&pk.PublicKey).parse(r) + if err != nil { + return + } + var buf [1]byte + _, err = readFull(r, buf[:]) + if err != nil { + return + } + + s2kType := buf[0] + + switch s2kType { + case 0: + pk.s2k = nil + pk.Encrypted = false + case 254, 255: + _, err = readFull(r, buf[:]) + if err != nil { + return + } + pk.cipher = CipherFunction(buf[0]) + pk.Encrypted = true + pk.s2k, err = s2k.Parse(r) + if err != nil { + return + } + if s2kType == 254 { + pk.sha1Checksum = true + } + default: + return error.UnsupportedError("deprecated s2k function in private key") + } + + if pk.Encrypted { + blockSize := pk.cipher.blockSize() + if blockSize == 0 { + return error.UnsupportedError("unsupported cipher in private key: " + strconv.Itoa(int(pk.cipher))) + } + pk.iv = make([]byte, blockSize) + _, err = readFull(r, pk.iv) + if err != nil { + return + } + } + + pk.encryptedData, err = ioutil.ReadAll(r) + if err != nil { + return + } + + if !pk.Encrypted { + return pk.parsePrivateKey(pk.encryptedData) + } + + return +} + +func mod64kHash(d []byte) uint16 { + h := uint16(0) + for i := 0; i < len(d); i += 2 { + v := uint16(d[i]) << 8 + if i+1 < len(d) { + v += uint16(d[i+1]) + } + h += v + } + return h +} + +func (pk *PrivateKey) Serialize(w io.Writer) (err os.Error) { + // TODO(agl): support encrypted private keys + buf := bytes.NewBuffer(nil) + err = pk.PublicKey.serializeWithoutHeaders(buf) + if err != nil { + return + } + buf.WriteByte(0 /* no encryption */ ) + + privateKeyBuf := bytes.NewBuffer(nil) + + switch priv := pk.PrivateKey.(type) { + case *rsa.PrivateKey: + err = serializeRSAPrivateKey(privateKeyBuf, priv) + default: + err = error.InvalidArgumentError("non-RSA private key") + } + if err != nil { + return + } + + ptype := packetTypePrivateKey + contents := buf.Bytes() + privateKeyBytes := privateKeyBuf.Bytes() + if pk.IsSubkey { + ptype = packetTypePrivateSubkey + } + err = serializeHeader(w, ptype, len(contents)+len(privateKeyBytes)+2) + if err != nil { + return + } + _, err = w.Write(contents) + if err != nil { + return + } + _, err = w.Write(privateKeyBytes) + if err != nil { + return + } + + checksum := mod64kHash(privateKeyBytes) + var checksumBytes [2]byte + checksumBytes[0] = byte(checksum >> 8) + checksumBytes[1] = byte(checksum) + _, err = w.Write(checksumBytes[:]) + + return +} + +func serializeRSAPrivateKey(w io.Writer, priv *rsa.PrivateKey) os.Error { + err := writeBig(w, priv.D) + if err != nil { + return err + } + err = writeBig(w, priv.Primes[1]) + if err != nil { + return err + } + err = writeBig(w, priv.Primes[0]) + if err != nil { + return err + } + return writeBig(w, priv.Precomputed.Qinv) +} + +// Decrypt decrypts an encrypted private key using a passphrase. +func (pk *PrivateKey) Decrypt(passphrase []byte) os.Error { + if !pk.Encrypted { + return nil + } + + key := make([]byte, pk.cipher.KeySize()) + pk.s2k(key, passphrase) + block := pk.cipher.new(key) + cfb := cipher.NewCFBDecrypter(block, pk.iv) + + data := pk.encryptedData + cfb.XORKeyStream(data, data) + + if pk.sha1Checksum { + if len(data) < sha1.Size { + return error.StructuralError("truncated private key data") + } + h := sha1.New() + h.Write(data[:len(data)-sha1.Size]) + sum := h.Sum() + if !bytes.Equal(sum, data[len(data)-sha1.Size:]) { + return error.StructuralError("private key checksum failure") + } + data = data[:len(data)-sha1.Size] + } else { + if len(data) < 2 { + return error.StructuralError("truncated private key data") + } + var sum uint16 + for i := 0; i < len(data)-2; i++ { + sum += uint16(data[i]) + } + if data[len(data)-2] != uint8(sum>>8) || + data[len(data)-1] != uint8(sum) { + return error.StructuralError("private key checksum failure") + } + data = data[:len(data)-2] + } + + return pk.parsePrivateKey(data) +} + +func (pk *PrivateKey) parsePrivateKey(data []byte) (err os.Error) { + switch pk.PublicKey.PubKeyAlgo { + case PubKeyAlgoRSA, PubKeyAlgoRSASignOnly, PubKeyAlgoRSAEncryptOnly: + return pk.parseRSAPrivateKey(data) + case PubKeyAlgoDSA: + return pk.parseDSAPrivateKey(data) + case PubKeyAlgoElGamal: + return pk.parseElGamalPrivateKey(data) + } + panic("impossible") +} + +func (pk *PrivateKey) parseRSAPrivateKey(data []byte) (err os.Error) { + rsaPub := pk.PublicKey.PublicKey.(*rsa.PublicKey) + rsaPriv := new(rsa.PrivateKey) + rsaPriv.PublicKey = *rsaPub + + buf := bytes.NewBuffer(data) + d, _, err := readMPI(buf) + if err != nil { + return + } + p, _, err := readMPI(buf) + if err != nil { + return + } + q, _, err := readMPI(buf) + if err != nil { + return + } + + rsaPriv.D = new(big.Int).SetBytes(d) + rsaPriv.Primes = make([]*big.Int, 2) + rsaPriv.Primes[0] = new(big.Int).SetBytes(p) + rsaPriv.Primes[1] = new(big.Int).SetBytes(q) + rsaPriv.Precompute() + pk.PrivateKey = rsaPriv + pk.Encrypted = false + pk.encryptedData = nil + + return nil +} + +func (pk *PrivateKey) parseDSAPrivateKey(data []byte) (err os.Error) { + dsaPub := pk.PublicKey.PublicKey.(*dsa.PublicKey) + dsaPriv := new(dsa.PrivateKey) + dsaPriv.PublicKey = *dsaPub + + buf := bytes.NewBuffer(data) + x, _, err := readMPI(buf) + if err != nil { + return + } + + dsaPriv.X = new(big.Int).SetBytes(x) + pk.PrivateKey = dsaPriv + pk.Encrypted = false + pk.encryptedData = nil + + return nil +} + +func (pk *PrivateKey) parseElGamalPrivateKey(data []byte) (err os.Error) { + pub := pk.PublicKey.PublicKey.(*elgamal.PublicKey) + priv := new(elgamal.PrivateKey) + priv.PublicKey = *pub + + buf := bytes.NewBuffer(data) + x, _, err := readMPI(buf) + if err != nil { + return + } + + priv.X = new(big.Int).SetBytes(x) + pk.PrivateKey = priv + pk.Encrypted = false + pk.encryptedData = nil + + return nil +} diff --git a/src/pkg/crypto/openpgp/packet/private_key_test.go b/src/pkg/crypto/openpgp/packet/private_key_test.go new file mode 100644 index 000000000..60eebaa6b --- /dev/null +++ b/src/pkg/crypto/openpgp/packet/private_key_test.go @@ -0,0 +1,57 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package packet + +import ( + "testing" +) + +var privateKeyTests = []struct { + privateKeyHex string + creationTime uint32 +}{ + { + privKeyRSAHex, + 0x4cc349a8, + }, + { + privKeyElGamalHex, + 0x4df9ee1a, + }, +} + +func TestPrivateKeyRead(t *testing.T) { + for i, test := range privateKeyTests { + packet, err := Read(readerFromHex(test.privateKeyHex)) + if err != nil { + t.Errorf("#%d: failed to parse: %s", i, err) + continue + } + + privKey := packet.(*PrivateKey) + + if !privKey.Encrypted { + t.Errorf("#%d: private key isn't encrypted", i) + continue + } + + err = privKey.Decrypt([]byte("testing")) + if err != nil { + t.Errorf("#%d: failed to decrypt: %s", i, err) + continue + } + + if privKey.CreationTime != test.creationTime || privKey.Encrypted { + t.Errorf("#%d: bad result, got: %#v", i, privKey) + } + } +} + +// Generated with `gpg --export-secret-keys "Test Key 2"` +const privKeyRSAHex = "9501fe044cc349a8010400b70ca0010e98c090008d45d1ee8f9113bd5861fd57b88bacb7c68658747663f1e1a3b5a98f32fda6472373c024b97359cd2efc88ff60f77751adfbf6af5e615e6a1408cfad8bf0cea30b0d5f53aa27ad59089ba9b15b7ebc2777a25d7b436144027e3bcd203909f147d0e332b240cf63d3395f5dfe0df0a6c04e8655af7eacdf0011010001fe0303024a252e7d475fd445607de39a265472aa74a9320ba2dac395faa687e9e0336aeb7e9a7397e511b5afd9dc84557c80ac0f3d4d7bfec5ae16f20d41c8c84a04552a33870b930420e230e179564f6d19bb153145e76c33ae993886c388832b0fa042ddda7f133924f3854481533e0ede31d51278c0519b29abc3bf53da673e13e3e1214b52413d179d7f66deee35cac8eacb060f78379d70ef4af8607e68131ff529439668fc39c9ce6dfef8a5ac234d234802cbfb749a26107db26406213ae5c06d4673253a3cbee1fcbae58d6ab77e38d6e2c0e7c6317c48e054edadb5a40d0d48acb44643d998139a8a66bb820be1f3f80185bc777d14b5954b60effe2448a036d565c6bc0b915fcea518acdd20ab07bc1529f561c58cd044f723109b93f6fd99f876ff891d64306b5d08f48bab59f38695e9109c4dec34013ba3153488ce070268381ba923ee1eb77125b36afcb4347ec3478c8f2735b06ef17351d872e577fa95d0c397c88c71b59629a36aec" + +// Generated by `gpg --export-secret-keys` followed by a manual extraction of +// the ElGamal subkey from the packets. +const privKeyElGamalHex = "9d0157044df9ee1a100400eb8e136a58ec39b582629cdadf830bc64e0a94ed8103ca8bb247b27b11b46d1d25297ef4bcc3071785ba0c0bedfe89eabc5287fcc0edf81ab5896c1c8e4b20d27d79813c7aede75320b33eaeeaa586edc00fd1036c10133e6ba0ff277245d0d59d04b2b3421b7244aca5f4a8d870c6f1c1fbff9e1c26699a860b9504f35ca1d700030503fd1ededd3b840795be6d9ccbe3c51ee42e2f39233c432b831ddd9c4e72b7025a819317e47bf94f9ee316d7273b05d5fcf2999c3a681f519b1234bbfa6d359b4752bd9c3f77d6b6456cde152464763414ca130f4e91d91041432f90620fec0e6d6b5116076c2985d5aeaae13be492b9b329efcaf7ee25120159a0a30cd976b42d7afe030302dae7eb80db744d4960c4df930d57e87fe81412eaace9f900e6c839817a614ddb75ba6603b9417c33ea7b6c93967dfa2bcff3fa3c74a5ce2c962db65b03aece14c96cbd0038fc" diff --git a/src/pkg/crypto/openpgp/packet/public_key.go b/src/pkg/crypto/openpgp/packet/public_key.go new file mode 100644 index 000000000..e6b0ae5f3 --- /dev/null +++ b/src/pkg/crypto/openpgp/packet/public_key.go @@ -0,0 +1,393 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package packet + +import ( + "big" + "crypto/dsa" + "crypto/openpgp/elgamal" + "crypto/openpgp/error" + "crypto/rsa" + "crypto/sha1" + "encoding/binary" + "fmt" + "hash" + "io" + "os" + "strconv" +) + +// PublicKey represents an OpenPGP public key. See RFC 4880, section 5.5.2. +type PublicKey struct { + CreationTime uint32 // seconds since the epoch + PubKeyAlgo PublicKeyAlgorithm + PublicKey interface{} // Either a *rsa.PublicKey or *dsa.PublicKey + Fingerprint [20]byte + KeyId uint64 + IsSubkey bool + + n, e, p, q, g, y parsedMPI +} + +func fromBig(n *big.Int) parsedMPI { + return parsedMPI{ + bytes: n.Bytes(), + bitLength: uint16(n.BitLen()), + } +} + +// NewRSAPublicKey returns a PublicKey that wraps the given rsa.PublicKey. +func NewRSAPublicKey(creationTimeSecs uint32, pub *rsa.PublicKey, isSubkey bool) *PublicKey { + pk := &PublicKey{ + CreationTime: creationTimeSecs, + PubKeyAlgo: PubKeyAlgoRSA, + PublicKey: pub, + IsSubkey: isSubkey, + n: fromBig(pub.N), + e: fromBig(big.NewInt(int64(pub.E))), + } + + pk.setFingerPrintAndKeyId() + return pk +} + +func (pk *PublicKey) parse(r io.Reader) (err os.Error) { + // RFC 4880, section 5.5.2 + var buf [6]byte + _, err = readFull(r, buf[:]) + if err != nil { + return + } + if buf[0] != 4 { + return error.UnsupportedError("public key version") + } + pk.CreationTime = uint32(buf[1])<<24 | uint32(buf[2])<<16 | uint32(buf[3])<<8 | uint32(buf[4]) + pk.PubKeyAlgo = PublicKeyAlgorithm(buf[5]) + switch pk.PubKeyAlgo { + case PubKeyAlgoRSA, PubKeyAlgoRSAEncryptOnly, PubKeyAlgoRSASignOnly: + err = pk.parseRSA(r) + case PubKeyAlgoDSA: + err = pk.parseDSA(r) + case PubKeyAlgoElGamal: + err = pk.parseElGamal(r) + default: + err = error.UnsupportedError("public key type: " + strconv.Itoa(int(pk.PubKeyAlgo))) + } + if err != nil { + return + } + + pk.setFingerPrintAndKeyId() + return +} + +func (pk *PublicKey) setFingerPrintAndKeyId() { + // RFC 4880, section 12.2 + fingerPrint := sha1.New() + pk.SerializeSignaturePrefix(fingerPrint) + pk.serializeWithoutHeaders(fingerPrint) + copy(pk.Fingerprint[:], fingerPrint.Sum()) + pk.KeyId = binary.BigEndian.Uint64(pk.Fingerprint[12:20]) +} + +// parseRSA parses RSA public key material from the given Reader. See RFC 4880, +// section 5.5.2. +func (pk *PublicKey) parseRSA(r io.Reader) (err os.Error) { + pk.n.bytes, pk.n.bitLength, err = readMPI(r) + if err != nil { + return + } + pk.e.bytes, pk.e.bitLength, err = readMPI(r) + if err != nil { + return + } + + if len(pk.e.bytes) > 3 { + err = error.UnsupportedError("large public exponent") + return + } + rsa := &rsa.PublicKey{ + N: new(big.Int).SetBytes(pk.n.bytes), + E: 0, + } + for i := 0; i < len(pk.e.bytes); i++ { + rsa.E <<= 8 + rsa.E |= int(pk.e.bytes[i]) + } + pk.PublicKey = rsa + return +} + +// parseDSA parses DSA public key material from the given Reader. See RFC 4880, +// section 5.5.2. +func (pk *PublicKey) parseDSA(r io.Reader) (err os.Error) { + pk.p.bytes, pk.p.bitLength, err = readMPI(r) + if err != nil { + return + } + pk.q.bytes, pk.q.bitLength, err = readMPI(r) + if err != nil { + return + } + pk.g.bytes, pk.g.bitLength, err = readMPI(r) + if err != nil { + return + } + pk.y.bytes, pk.y.bitLength, err = readMPI(r) + if err != nil { + return + } + + dsa := new(dsa.PublicKey) + dsa.P = new(big.Int).SetBytes(pk.p.bytes) + dsa.Q = new(big.Int).SetBytes(pk.q.bytes) + dsa.G = new(big.Int).SetBytes(pk.g.bytes) + dsa.Y = new(big.Int).SetBytes(pk.y.bytes) + pk.PublicKey = dsa + return +} + +// parseElGamal parses ElGamal public key material from the given Reader. See +// RFC 4880, section 5.5.2. +func (pk *PublicKey) parseElGamal(r io.Reader) (err os.Error) { + pk.p.bytes, pk.p.bitLength, err = readMPI(r) + if err != nil { + return + } + pk.g.bytes, pk.g.bitLength, err = readMPI(r) + if err != nil { + return + } + pk.y.bytes, pk.y.bitLength, err = readMPI(r) + if err != nil { + return + } + + elgamal := new(elgamal.PublicKey) + elgamal.P = new(big.Int).SetBytes(pk.p.bytes) + elgamal.G = new(big.Int).SetBytes(pk.g.bytes) + elgamal.Y = new(big.Int).SetBytes(pk.y.bytes) + pk.PublicKey = elgamal + return +} + +// SerializeSignaturePrefix writes the prefix for this public key to the given Writer. +// The prefix is used when calculating a signature over this public key. See +// RFC 4880, section 5.2.4. +func (pk *PublicKey) SerializeSignaturePrefix(h hash.Hash) { + var pLength uint16 + switch pk.PubKeyAlgo { + case PubKeyAlgoRSA, PubKeyAlgoRSAEncryptOnly, PubKeyAlgoRSASignOnly: + pLength += 2 + uint16(len(pk.n.bytes)) + pLength += 2 + uint16(len(pk.e.bytes)) + case PubKeyAlgoDSA: + pLength += 2 + uint16(len(pk.p.bytes)) + pLength += 2 + uint16(len(pk.q.bytes)) + pLength += 2 + uint16(len(pk.g.bytes)) + pLength += 2 + uint16(len(pk.y.bytes)) + case PubKeyAlgoElGamal: + pLength += 2 + uint16(len(pk.p.bytes)) + pLength += 2 + uint16(len(pk.g.bytes)) + pLength += 2 + uint16(len(pk.y.bytes)) + default: + panic("unknown public key algorithm") + } + pLength += 6 + h.Write([]byte{0x99, byte(pLength >> 8), byte(pLength)}) + return +} + +func (pk *PublicKey) Serialize(w io.Writer) (err os.Error) { + length := 6 // 6 byte header + + switch pk.PubKeyAlgo { + case PubKeyAlgoRSA, PubKeyAlgoRSAEncryptOnly, PubKeyAlgoRSASignOnly: + length += 2 + len(pk.n.bytes) + length += 2 + len(pk.e.bytes) + case PubKeyAlgoDSA: + length += 2 + len(pk.p.bytes) + length += 2 + len(pk.q.bytes) + length += 2 + len(pk.g.bytes) + length += 2 + len(pk.y.bytes) + case PubKeyAlgoElGamal: + length += 2 + len(pk.p.bytes) + length += 2 + len(pk.g.bytes) + length += 2 + len(pk.y.bytes) + default: + panic("unknown public key algorithm") + } + + packetType := packetTypePublicKey + if pk.IsSubkey { + packetType = packetTypePublicSubkey + } + err = serializeHeader(w, packetType, length) + if err != nil { + return + } + return pk.serializeWithoutHeaders(w) +} + +// serializeWithoutHeaders marshals the PublicKey to w in the form of an +// OpenPGP public key packet, not including the packet header. +func (pk *PublicKey) serializeWithoutHeaders(w io.Writer) (err os.Error) { + var buf [6]byte + buf[0] = 4 + buf[1] = byte(pk.CreationTime >> 24) + buf[2] = byte(pk.CreationTime >> 16) + buf[3] = byte(pk.CreationTime >> 8) + buf[4] = byte(pk.CreationTime) + buf[5] = byte(pk.PubKeyAlgo) + + _, err = w.Write(buf[:]) + if err != nil { + return + } + + switch pk.PubKeyAlgo { + case PubKeyAlgoRSA, PubKeyAlgoRSAEncryptOnly, PubKeyAlgoRSASignOnly: + return writeMPIs(w, pk.n, pk.e) + case PubKeyAlgoDSA: + return writeMPIs(w, pk.p, pk.q, pk.g, pk.y) + case PubKeyAlgoElGamal: + return writeMPIs(w, pk.p, pk.g, pk.y) + } + return error.InvalidArgumentError("bad public-key algorithm") +} + +// CanSign returns true iff this public key can generate signatures +func (pk *PublicKey) CanSign() bool { + return pk.PubKeyAlgo != PubKeyAlgoRSAEncryptOnly && pk.PubKeyAlgo != PubKeyAlgoElGamal +} + +// VerifySignature returns nil iff sig is a valid signature, made by this +// public key, of the data hashed into signed. signed is mutated by this call. +func (pk *PublicKey) VerifySignature(signed hash.Hash, sig *Signature) (err os.Error) { + if !pk.CanSign() { + return error.InvalidArgumentError("public key cannot generate signatures") + } + + signed.Write(sig.HashSuffix) + hashBytes := signed.Sum() + + if hashBytes[0] != sig.HashTag[0] || hashBytes[1] != sig.HashTag[1] { + return error.SignatureError("hash tag doesn't match") + } + + if pk.PubKeyAlgo != sig.PubKeyAlgo { + return error.InvalidArgumentError("public key and signature use different algorithms") + } + + switch pk.PubKeyAlgo { + case PubKeyAlgoRSA, PubKeyAlgoRSASignOnly: + rsaPublicKey, _ := pk.PublicKey.(*rsa.PublicKey) + err = rsa.VerifyPKCS1v15(rsaPublicKey, sig.Hash, hashBytes, sig.RSASignature.bytes) + if err != nil { + return error.SignatureError("RSA verification failure") + } + return nil + case PubKeyAlgoDSA: + dsaPublicKey, _ := pk.PublicKey.(*dsa.PublicKey) + if !dsa.Verify(dsaPublicKey, hashBytes, new(big.Int).SetBytes(sig.DSASigR.bytes), new(big.Int).SetBytes(sig.DSASigS.bytes)) { + return error.SignatureError("DSA verification failure") + } + return nil + default: + panic("shouldn't happen") + } + panic("unreachable") +} + +// keySignatureHash returns a Hash of the message that needs to be signed for +// pk to assert a subkey relationship to signed. +func keySignatureHash(pk, signed *PublicKey, sig *Signature) (h hash.Hash, err os.Error) { + h = sig.Hash.New() + if h == nil { + return nil, error.UnsupportedError("hash function") + } + + // RFC 4880, section 5.2.4 + pk.SerializeSignaturePrefix(h) + pk.serializeWithoutHeaders(h) + signed.SerializeSignaturePrefix(h) + signed.serializeWithoutHeaders(h) + return +} + +// VerifyKeySignature returns nil iff sig is a valid signature, made by this +// public key, of signed. +func (pk *PublicKey) VerifyKeySignature(signed *PublicKey, sig *Signature) (err os.Error) { + h, err := keySignatureHash(pk, signed, sig) + if err != nil { + return err + } + return pk.VerifySignature(h, sig) +} + +// userIdSignatureHash returns a Hash of the message that needs to be signed +// to assert that pk is a valid key for id. +func userIdSignatureHash(id string, pk *PublicKey, sig *Signature) (h hash.Hash, err os.Error) { + h = sig.Hash.New() + if h == nil { + return nil, error.UnsupportedError("hash function") + } + + // RFC 4880, section 5.2.4 + pk.SerializeSignaturePrefix(h) + pk.serializeWithoutHeaders(h) + + var buf [5]byte + buf[0] = 0xb4 + buf[1] = byte(len(id) >> 24) + buf[2] = byte(len(id) >> 16) + buf[3] = byte(len(id) >> 8) + buf[4] = byte(len(id)) + h.Write(buf[:]) + h.Write([]byte(id)) + + return +} + +// VerifyUserIdSignature returns nil iff sig is a valid signature, made by this +// public key, of id. +func (pk *PublicKey) VerifyUserIdSignature(id string, sig *Signature) (err os.Error) { + h, err := userIdSignatureHash(id, pk, sig) + if err != nil { + return err + } + return pk.VerifySignature(h, sig) +} + +// KeyIdString returns the public key's fingerprint in capital hex +// (e.g. "6C7EE1B8621CC013"). +func (pk *PublicKey) KeyIdString() string { + return fmt.Sprintf("%X", pk.Fingerprint[12:20]) +} + +// KeyIdShortString returns the short form of public key's fingerprint +// in capital hex, as shown by gpg --list-keys (e.g. "621CC013"). +func (pk *PublicKey) KeyIdShortString() string { + return fmt.Sprintf("%X", pk.Fingerprint[16:20]) +} + +// A parsedMPI is used to store the contents of a big integer, along with the +// bit length that was specified in the original input. This allows the MPI to +// be reserialized exactly. +type parsedMPI struct { + bytes []byte + bitLength uint16 +} + +// writeMPIs is a utility function for serializing several big integers to the +// given Writer. +func writeMPIs(w io.Writer, mpis ...parsedMPI) (err os.Error) { + for _, mpi := range mpis { + err = writeMPI(w, mpi.bitLength, mpi.bytes) + if err != nil { + return + } + } + return +} diff --git a/src/pkg/crypto/openpgp/packet/public_key_test.go b/src/pkg/crypto/openpgp/packet/public_key_test.go new file mode 100644 index 000000000..6e8bfbce6 --- /dev/null +++ b/src/pkg/crypto/openpgp/packet/public_key_test.go @@ -0,0 +1,98 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package packet + +import ( + "bytes" + "encoding/hex" + "testing" +) + +var pubKeyTests = []struct { + hexData string + hexFingerprint string + creationTime uint32 + pubKeyAlgo PublicKeyAlgorithm + keyId uint64 + keyIdString string + keyIdShort string +}{ + {rsaPkDataHex, rsaFingerprintHex, 0x4d3c5c10, PubKeyAlgoRSA, 0xa34d7e18c20c31bb, "A34D7E18C20C31BB", "C20C31BB"}, + {dsaPkDataHex, dsaFingerprintHex, 0x4d432f89, PubKeyAlgoDSA, 0x8e8fbe54062f19ed, "8E8FBE54062F19ED", "062F19ED"}, +} + +func TestPublicKeyRead(t *testing.T) { + for i, test := range pubKeyTests { + packet, err := Read(readerFromHex(test.hexData)) + if err != nil { + t.Errorf("#%d: Read error: %s", i, err) + continue + } + pk, ok := packet.(*PublicKey) + if !ok { + t.Errorf("#%d: failed to parse, got: %#v", i, packet) + continue + } + if pk.PubKeyAlgo != test.pubKeyAlgo { + t.Errorf("#%d: bad public key algorithm got:%x want:%x", i, pk.PubKeyAlgo, test.pubKeyAlgo) + } + if pk.CreationTime != test.creationTime { + t.Errorf("#%d: bad creation time got:%x want:%x", i, pk.CreationTime, test.creationTime) + } + expectedFingerprint, _ := hex.DecodeString(test.hexFingerprint) + if !bytes.Equal(expectedFingerprint, pk.Fingerprint[:]) { + t.Errorf("#%d: bad fingerprint got:%x want:%x", i, pk.Fingerprint[:], expectedFingerprint) + } + if pk.KeyId != test.keyId { + t.Errorf("#%d: bad keyid got:%x want:%x", i, pk.KeyId, test.keyId) + } + if g, e := pk.KeyIdString(), test.keyIdString; g != e { + t.Errorf("#%d: bad KeyIdString got:%q want:%q", i, g, e) + } + if g, e := pk.KeyIdShortString(), test.keyIdShort; g != e { + t.Errorf("#%d: bad KeyIdShortString got:%q want:%q", i, g, e) + } + } +} + +func TestPublicKeySerialize(t *testing.T) { + for i, test := range pubKeyTests { + packet, err := Read(readerFromHex(test.hexData)) + if err != nil { + t.Errorf("#%d: Read error: %s", i, err) + continue + } + pk, ok := packet.(*PublicKey) + if !ok { + t.Errorf("#%d: failed to parse, got: %#v", i, packet) + continue + } + serializeBuf := bytes.NewBuffer(nil) + err = pk.Serialize(serializeBuf) + if err != nil { + t.Errorf("#%d: failed to serialize: %s", i, err) + continue + } + + packet, err = Read(serializeBuf) + if err != nil { + t.Errorf("#%d: Read error (from serialized data): %s", i, err) + continue + } + pk, ok = packet.(*PublicKey) + if !ok { + t.Errorf("#%d: failed to parse serialized data, got: %#v", i, packet) + continue + } + } +} + +const rsaFingerprintHex = "5fb74b1d03b1e3cb31bc2f8aa34d7e18c20c31bb" + +const rsaPkDataHex = "988d044d3c5c10010400b1d13382944bd5aba23a4312968b5095d14f947f600eb478e14a6fcb16b0e0cac764884909c020bc495cfcc39a935387c661507bdb236a0612fb582cac3af9b29cc2c8c70090616c41b662f4da4c1201e195472eb7f4ae1ccbcbf9940fe21d985e379a5563dde5b9a23d35f1cfaa5790da3b79db26f23695107bfaca8e7b5bcd0011010001" + +const dsaFingerprintHex = "eece4c094db002103714c63c8e8fbe54062f19ed" + +const dsaPkDataHex = "9901a2044d432f89110400cd581334f0d7a1e1bdc8b9d6d8c0baf68793632735d2bb0903224cbaa1dfbf35a60ee7a13b92643421e1eb41aa8d79bea19a115a677f6b8ba3c7818ce53a6c2a24a1608bd8b8d6e55c5090cbde09dd26e356267465ae25e69ec8bdd57c7bbb2623e4d73336f73a0a9098f7f16da2e25252130fd694c0e8070c55a812a423ae7f00a0ebf50e70c2f19c3520a551bd4b08d30f23530d3d03ff7d0bf4a53a64a09dc5e6e6e35854b7d70c882b0c60293401958b1bd9e40abec3ea05ba87cf64899299d4bd6aa7f459c201d3fbbd6c82004bdc5e8a9eb8082d12054cc90fa9d4ec251a843236a588bf49552441817436c4f43326966fe85447d4e6d0acf8fa1ef0f014730770603ad7634c3088dc52501c237328417c31c89ed70400b2f1a98b0bf42f11fefc430704bebbaa41d9f355600c3facee1e490f64208e0e094ea55e3a598a219a58500bf78ac677b670a14f4e47e9cf8eab4f368cc1ddcaa18cc59309d4cc62dd4f680e73e6cc3e1ce87a84d0925efbcb26c575c093fc42eecf45135fabf6403a25c2016e1774c0484e440a18319072c617cc97ac0a3bb0" diff --git a/src/pkg/crypto/openpgp/packet/reader.go b/src/pkg/crypto/openpgp/packet/reader.go new file mode 100644 index 000000000..5febc3bc8 --- /dev/null +++ b/src/pkg/crypto/openpgp/packet/reader.go @@ -0,0 +1,63 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package packet + +import ( + "crypto/openpgp/error" + "io" + "os" +) + +// Reader reads packets from an io.Reader and allows packets to be 'unread' so +// that they result from the next call to Next. +type Reader struct { + q []Packet + readers []io.Reader +} + +// Next returns the most recently unread Packet, or reads another packet from +// the top-most io.Reader. Unknown packet types are skipped. +func (r *Reader) Next() (p Packet, err os.Error) { + if len(r.q) > 0 { + p = r.q[len(r.q)-1] + r.q = r.q[:len(r.q)-1] + return + } + + for len(r.readers) > 0 { + p, err = Read(r.readers[len(r.readers)-1]) + if err == nil { + return + } + if err == os.EOF { + r.readers = r.readers[:len(r.readers)-1] + continue + } + if _, ok := err.(error.UnknownPacketTypeError); !ok { + return nil, err + } + } + + return nil, os.EOF +} + +// Push causes the Reader to start reading from a new io.Reader. When an EOF +// error is seen from the new io.Reader, it is popped and the Reader continues +// to read from the next most recent io.Reader. +func (r *Reader) Push(reader io.Reader) { + r.readers = append(r.readers, reader) +} + +// Unread causes the given Packet to be returned from the next call to Next. +func (r *Reader) Unread(p Packet) { + r.q = append(r.q, p) +} + +func NewReader(r io.Reader) *Reader { + return &Reader{ + q: nil, + readers: []io.Reader{r}, + } +} diff --git a/src/pkg/crypto/openpgp/packet/signature.go b/src/pkg/crypto/openpgp/packet/signature.go new file mode 100644 index 000000000..7577e2875 --- /dev/null +++ b/src/pkg/crypto/openpgp/packet/signature.go @@ -0,0 +1,558 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package packet + +import ( + "crypto" + "crypto/dsa" + "crypto/openpgp/error" + "crypto/openpgp/s2k" + "crypto/rand" + "crypto/rsa" + "encoding/binary" + "hash" + "io" + "os" + "strconv" +) + +// Signature represents a signature. See RFC 4880, section 5.2. +type Signature struct { + SigType SignatureType + PubKeyAlgo PublicKeyAlgorithm + Hash crypto.Hash + + // HashSuffix is extra data that is hashed in after the signed data. + HashSuffix []byte + // HashTag contains the first two bytes of the hash for fast rejection + // of bad signed data. + HashTag [2]byte + CreationTime uint32 // Unix epoch time + + RSASignature parsedMPI + DSASigR, DSASigS parsedMPI + + // rawSubpackets contains the unparsed subpackets, in order. + rawSubpackets []outputSubpacket + + // The following are optional so are nil when not included in the + // signature. + + SigLifetimeSecs, KeyLifetimeSecs *uint32 + PreferredSymmetric, PreferredHash, PreferredCompression []uint8 + IssuerKeyId *uint64 + IsPrimaryId *bool + + // FlagsValid is set if any flags were given. See RFC 4880, section + // 5.2.3.21 for details. + FlagsValid bool + FlagCertify, FlagSign, FlagEncryptCommunications, FlagEncryptStorage bool + + outSubpackets []outputSubpacket +} + +func (sig *Signature) parse(r io.Reader) (err os.Error) { + // RFC 4880, section 5.2.3 + var buf [5]byte + _, err = readFull(r, buf[:1]) + if err != nil { + return + } + if buf[0] != 4 { + err = error.UnsupportedError("signature packet version " + strconv.Itoa(int(buf[0]))) + return + } + + _, err = readFull(r, buf[:5]) + if err != nil { + return + } + sig.SigType = SignatureType(buf[0]) + sig.PubKeyAlgo = PublicKeyAlgorithm(buf[1]) + switch sig.PubKeyAlgo { + case PubKeyAlgoRSA, PubKeyAlgoRSASignOnly, PubKeyAlgoDSA: + default: + err = error.UnsupportedError("public key algorithm " + strconv.Itoa(int(sig.PubKeyAlgo))) + return + } + + var ok bool + sig.Hash, ok = s2k.HashIdToHash(buf[2]) + if !ok { + return error.UnsupportedError("hash function " + strconv.Itoa(int(buf[2]))) + } + + hashedSubpacketsLength := int(buf[3])<<8 | int(buf[4]) + l := 6 + hashedSubpacketsLength + sig.HashSuffix = make([]byte, l+6) + sig.HashSuffix[0] = 4 + copy(sig.HashSuffix[1:], buf[:5]) + hashedSubpackets := sig.HashSuffix[6:l] + _, err = readFull(r, hashedSubpackets) + if err != nil { + return + } + // See RFC 4880, section 5.2.4 + trailer := sig.HashSuffix[l:] + trailer[0] = 4 + trailer[1] = 0xff + trailer[2] = uint8(l >> 24) + trailer[3] = uint8(l >> 16) + trailer[4] = uint8(l >> 8) + trailer[5] = uint8(l) + + err = parseSignatureSubpackets(sig, hashedSubpackets, true) + if err != nil { + return + } + + _, err = readFull(r, buf[:2]) + if err != nil { + return + } + unhashedSubpacketsLength := int(buf[0])<<8 | int(buf[1]) + unhashedSubpackets := make([]byte, unhashedSubpacketsLength) + _, err = readFull(r, unhashedSubpackets) + if err != nil { + return + } + err = parseSignatureSubpackets(sig, unhashedSubpackets, false) + if err != nil { + return + } + + _, err = readFull(r, sig.HashTag[:2]) + if err != nil { + return + } + + switch sig.PubKeyAlgo { + case PubKeyAlgoRSA, PubKeyAlgoRSASignOnly: + sig.RSASignature.bytes, sig.RSASignature.bitLength, err = readMPI(r) + case PubKeyAlgoDSA: + sig.DSASigR.bytes, sig.DSASigR.bitLength, err = readMPI(r) + if err == nil { + sig.DSASigS.bytes, sig.DSASigS.bitLength, err = readMPI(r) + } + default: + panic("unreachable") + } + return +} + +// parseSignatureSubpackets parses subpackets of the main signature packet. See +// RFC 4880, section 5.2.3.1. +func parseSignatureSubpackets(sig *Signature, subpackets []byte, isHashed bool) (err os.Error) { + for len(subpackets) > 0 { + subpackets, err = parseSignatureSubpacket(sig, subpackets, isHashed) + if err != nil { + return + } + } + + if sig.CreationTime == 0 { + err = error.StructuralError("no creation time in signature") + } + + return +} + +type signatureSubpacketType uint8 + +const ( + creationTimeSubpacket signatureSubpacketType = 2 + signatureExpirationSubpacket signatureSubpacketType = 3 + keyExpirySubpacket signatureSubpacketType = 9 + prefSymmetricAlgosSubpacket signatureSubpacketType = 11 + issuerSubpacket signatureSubpacketType = 16 + prefHashAlgosSubpacket signatureSubpacketType = 21 + prefCompressionSubpacket signatureSubpacketType = 22 + primaryUserIdSubpacket signatureSubpacketType = 25 + keyFlagsSubpacket signatureSubpacketType = 27 +) + +// parseSignatureSubpacket parses a single subpacket. len(subpacket) is >= 1. +func parseSignatureSubpacket(sig *Signature, subpacket []byte, isHashed bool) (rest []byte, err os.Error) { + // RFC 4880, section 5.2.3.1 + var ( + length uint32 + packetType signatureSubpacketType + isCritical bool + ) + switch { + case subpacket[0] < 192: + length = uint32(subpacket[0]) + subpacket = subpacket[1:] + case subpacket[0] < 255: + if len(subpacket) < 2 { + goto Truncated + } + length = uint32(subpacket[0]-192)<<8 + uint32(subpacket[1]) + 192 + subpacket = subpacket[2:] + default: + if len(subpacket) < 5 { + goto Truncated + } + length = uint32(subpacket[1])<<24 | + uint32(subpacket[2])<<16 | + uint32(subpacket[3])<<8 | + uint32(subpacket[4]) + subpacket = subpacket[5:] + } + if length > uint32(len(subpacket)) { + goto Truncated + } + rest = subpacket[length:] + subpacket = subpacket[:length] + if len(subpacket) == 0 { + err = error.StructuralError("zero length signature subpacket") + return + } + packetType = signatureSubpacketType(subpacket[0] & 0x7f) + isCritical = subpacket[0]&0x80 == 0x80 + subpacket = subpacket[1:] + sig.rawSubpackets = append(sig.rawSubpackets, outputSubpacket{isHashed, packetType, isCritical, subpacket}) + switch packetType { + case creationTimeSubpacket: + if !isHashed { + err = error.StructuralError("signature creation time in non-hashed area") + return + } + if len(subpacket) != 4 { + err = error.StructuralError("signature creation time not four bytes") + return + } + sig.CreationTime = binary.BigEndian.Uint32(subpacket) + case signatureExpirationSubpacket: + // Signature expiration time, section 5.2.3.10 + if !isHashed { + return + } + if len(subpacket) != 4 { + err = error.StructuralError("expiration subpacket with bad length") + return + } + sig.SigLifetimeSecs = new(uint32) + *sig.SigLifetimeSecs = binary.BigEndian.Uint32(subpacket) + case keyExpirySubpacket: + // Key expiration time, section 5.2.3.6 + if !isHashed { + return + } + if len(subpacket) != 4 { + err = error.StructuralError("key expiration subpacket with bad length") + return + } + sig.KeyLifetimeSecs = new(uint32) + *sig.KeyLifetimeSecs = binary.BigEndian.Uint32(subpacket) + case prefSymmetricAlgosSubpacket: + // Preferred symmetric algorithms, section 5.2.3.7 + if !isHashed { + return + } + sig.PreferredSymmetric = make([]byte, len(subpacket)) + copy(sig.PreferredSymmetric, subpacket) + case issuerSubpacket: + // Issuer, section 5.2.3.5 + if len(subpacket) != 8 { + err = error.StructuralError("issuer subpacket with bad length") + return + } + sig.IssuerKeyId = new(uint64) + *sig.IssuerKeyId = binary.BigEndian.Uint64(subpacket) + case prefHashAlgosSubpacket: + // Preferred hash algorithms, section 5.2.3.8 + if !isHashed { + return + } + sig.PreferredHash = make([]byte, len(subpacket)) + copy(sig.PreferredHash, subpacket) + case prefCompressionSubpacket: + // Preferred compression algorithms, section 5.2.3.9 + if !isHashed { + return + } + sig.PreferredCompression = make([]byte, len(subpacket)) + copy(sig.PreferredCompression, subpacket) + case primaryUserIdSubpacket: + // Primary User ID, section 5.2.3.19 + if !isHashed { + return + } + if len(subpacket) != 1 { + err = error.StructuralError("primary user id subpacket with bad length") + return + } + sig.IsPrimaryId = new(bool) + if subpacket[0] > 0 { + *sig.IsPrimaryId = true + } + case keyFlagsSubpacket: + // Key flags, section 5.2.3.21 + if !isHashed { + return + } + if len(subpacket) == 0 { + err = error.StructuralError("empty key flags subpacket") + return + } + sig.FlagsValid = true + if subpacket[0]&1 != 0 { + sig.FlagCertify = true + } + if subpacket[0]&2 != 0 { + sig.FlagSign = true + } + if subpacket[0]&4 != 0 { + sig.FlagEncryptCommunications = true + } + if subpacket[0]&8 != 0 { + sig.FlagEncryptStorage = true + } + + default: + if isCritical { + err = error.UnsupportedError("unknown critical signature subpacket type " + strconv.Itoa(int(packetType))) + return + } + } + return + +Truncated: + err = error.StructuralError("signature subpacket truncated") + return +} + +// subpacketLengthLength returns the length, in bytes, of an encoded length value. +func subpacketLengthLength(length int) int { + if length < 192 { + return 1 + } + if length < 16320 { + return 2 + } + return 5 +} + +// serializeSubpacketLength marshals the given length into to. +func serializeSubpacketLength(to []byte, length int) int { + if length < 192 { + to[0] = byte(length) + return 1 + } + if length < 16320 { + length -= 192 + to[0] = byte(length >> 8) + to[1] = byte(length) + return 2 + } + to[0] = 255 + to[1] = byte(length >> 24) + to[2] = byte(length >> 16) + to[3] = byte(length >> 8) + to[4] = byte(length) + return 5 +} + +// subpacketsLength returns the serialized length, in bytes, of the given +// subpackets. +func subpacketsLength(subpackets []outputSubpacket, hashed bool) (length int) { + for _, subpacket := range subpackets { + if subpacket.hashed == hashed { + length += subpacketLengthLength(len(subpacket.contents) + 1) + length += 1 // type byte + length += len(subpacket.contents) + } + } + return +} + +// serializeSubpackets marshals the given subpackets into to. +func serializeSubpackets(to []byte, subpackets []outputSubpacket, hashed bool) { + for _, subpacket := range subpackets { + if subpacket.hashed == hashed { + n := serializeSubpacketLength(to, len(subpacket.contents)+1) + to[n] = byte(subpacket.subpacketType) + to = to[1+n:] + n = copy(to, subpacket.contents) + to = to[n:] + } + } + return +} + +// buildHashSuffix constructs the HashSuffix member of sig in preparation for signing. +func (sig *Signature) buildHashSuffix() (err os.Error) { + hashedSubpacketsLen := subpacketsLength(sig.outSubpackets, true) + + var ok bool + l := 6 + hashedSubpacketsLen + sig.HashSuffix = make([]byte, l+6) + sig.HashSuffix[0] = 4 + sig.HashSuffix[1] = uint8(sig.SigType) + sig.HashSuffix[2] = uint8(sig.PubKeyAlgo) + sig.HashSuffix[3], ok = s2k.HashToHashId(sig.Hash) + if !ok { + sig.HashSuffix = nil + return error.InvalidArgumentError("hash cannot be represented in OpenPGP: " + strconv.Itoa(int(sig.Hash))) + } + sig.HashSuffix[4] = byte(hashedSubpacketsLen >> 8) + sig.HashSuffix[5] = byte(hashedSubpacketsLen) + serializeSubpackets(sig.HashSuffix[6:l], sig.outSubpackets, true) + trailer := sig.HashSuffix[l:] + trailer[0] = 4 + trailer[1] = 0xff + trailer[2] = byte(l >> 24) + trailer[3] = byte(l >> 16) + trailer[4] = byte(l >> 8) + trailer[5] = byte(l) + return +} + +func (sig *Signature) signPrepareHash(h hash.Hash) (digest []byte, err os.Error) { + err = sig.buildHashSuffix() + if err != nil { + return + } + + h.Write(sig.HashSuffix) + digest = h.Sum() + copy(sig.HashTag[:], digest) + return +} + +// Sign signs a message with a private key. The hash, h, must contain +// the hash of the message to be signed and will be mutated by this function. +// On success, the signature is stored in sig. Call Serialize to write it out. +func (sig *Signature) Sign(h hash.Hash, priv *PrivateKey) (err os.Error) { + sig.outSubpackets = sig.buildSubpackets() + digest, err := sig.signPrepareHash(h) + if err != nil { + return + } + + switch priv.PubKeyAlgo { + case PubKeyAlgoRSA, PubKeyAlgoRSASignOnly: + sig.RSASignature.bytes, err = rsa.SignPKCS1v15(rand.Reader, priv.PrivateKey.(*rsa.PrivateKey), sig.Hash, digest) + sig.RSASignature.bitLength = uint16(8 * len(sig.RSASignature.bytes)) + case PubKeyAlgoDSA: + r, s, err := dsa.Sign(rand.Reader, priv.PrivateKey.(*dsa.PrivateKey), digest) + if err == nil { + sig.DSASigR.bytes = r.Bytes() + sig.DSASigR.bitLength = uint16(8 * len(sig.DSASigR.bytes)) + sig.DSASigS.bytes = s.Bytes() + sig.DSASigS.bitLength = uint16(8 * len(sig.DSASigS.bytes)) + } + default: + err = error.UnsupportedError("public key algorithm: " + strconv.Itoa(int(sig.PubKeyAlgo))) + } + + return +} + +// SignUserId computes a signature from priv, asserting that pub is a valid +// key for the identity id. On success, the signature is stored in sig. Call +// Serialize to write it out. +func (sig *Signature) SignUserId(id string, pub *PublicKey, priv *PrivateKey) os.Error { + h, err := userIdSignatureHash(id, pub, sig) + if err != nil { + return nil + } + return sig.Sign(h, priv) +} + +// SignKey computes a signature from priv, asserting that pub is a subkey. On +// success, the signature is stored in sig. Call Serialize to write it out. +func (sig *Signature) SignKey(pub *PublicKey, priv *PrivateKey) os.Error { + h, err := keySignatureHash(&priv.PublicKey, pub, sig) + if err != nil { + return err + } + return sig.Sign(h, priv) +} + +// Serialize marshals sig to w. SignRSA or SignDSA must have been called first. +func (sig *Signature) Serialize(w io.Writer) (err os.Error) { + if len(sig.outSubpackets) == 0 { + sig.outSubpackets = sig.rawSubpackets + } + if sig.RSASignature.bytes == nil && sig.DSASigR.bytes == nil { + return error.InvalidArgumentError("Signature: need to call SignRSA or SignDSA before Serialize") + } + + sigLength := 0 + switch sig.PubKeyAlgo { + case PubKeyAlgoRSA, PubKeyAlgoRSASignOnly: + sigLength = 2 + len(sig.RSASignature.bytes) + case PubKeyAlgoDSA: + sigLength = 2 + len(sig.DSASigR.bytes) + sigLength += 2 + len(sig.DSASigS.bytes) + default: + panic("impossible") + } + + unhashedSubpacketsLen := subpacketsLength(sig.outSubpackets, false) + length := len(sig.HashSuffix) - 6 /* trailer not included */ + + 2 /* length of unhashed subpackets */ + unhashedSubpacketsLen + + 2 /* hash tag */ + sigLength + err = serializeHeader(w, packetTypeSignature, length) + if err != nil { + return + } + + _, err = w.Write(sig.HashSuffix[:len(sig.HashSuffix)-6]) + if err != nil { + return + } + + unhashedSubpackets := make([]byte, 2+unhashedSubpacketsLen) + unhashedSubpackets[0] = byte(unhashedSubpacketsLen >> 8) + unhashedSubpackets[1] = byte(unhashedSubpacketsLen) + serializeSubpackets(unhashedSubpackets[2:], sig.outSubpackets, false) + + _, err = w.Write(unhashedSubpackets) + if err != nil { + return + } + _, err = w.Write(sig.HashTag[:]) + if err != nil { + return + } + + switch sig.PubKeyAlgo { + case PubKeyAlgoRSA, PubKeyAlgoRSASignOnly: + err = writeMPIs(w, sig.RSASignature) + case PubKeyAlgoDSA: + err = writeMPIs(w, sig.DSASigR, sig.DSASigS) + default: + panic("impossible") + } + return +} + +// outputSubpacket represents a subpacket to be marshaled. +type outputSubpacket struct { + hashed bool // true if this subpacket is in the hashed area. + subpacketType signatureSubpacketType + isCritical bool + contents []byte +} + +func (sig *Signature) buildSubpackets() (subpackets []outputSubpacket) { + creationTime := make([]byte, 4) + creationTime[0] = byte(sig.CreationTime >> 24) + creationTime[1] = byte(sig.CreationTime >> 16) + creationTime[2] = byte(sig.CreationTime >> 8) + creationTime[3] = byte(sig.CreationTime) + subpackets = append(subpackets, outputSubpacket{true, creationTimeSubpacket, false, creationTime}) + + if sig.IssuerKeyId != nil { + keyId := make([]byte, 8) + binary.BigEndian.PutUint64(keyId, *sig.IssuerKeyId) + subpackets = append(subpackets, outputSubpacket{true, issuerSubpacket, false, keyId}) + } + + return +} diff --git a/src/pkg/crypto/openpgp/packet/signature_test.go b/src/pkg/crypto/openpgp/packet/signature_test.go new file mode 100644 index 000000000..c1bbde8b0 --- /dev/null +++ b/src/pkg/crypto/openpgp/packet/signature_test.go @@ -0,0 +1,42 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package packet + +import ( + "bytes" + "crypto" + "encoding/hex" + "testing" +) + +func TestSignatureRead(t *testing.T) { + packet, err := Read(readerFromHex(signatureDataHex)) + if err != nil { + t.Error(err) + return + } + sig, ok := packet.(*Signature) + if !ok || sig.SigType != SigTypeBinary || sig.PubKeyAlgo != PubKeyAlgoRSA || sig.Hash != crypto.SHA1 { + t.Errorf("failed to parse, got: %#v", packet) + } +} + +func TestSignatureReserialize(t *testing.T) { + packet, _ := Read(readerFromHex(signatureDataHex)) + sig := packet.(*Signature) + out := new(bytes.Buffer) + err := sig.Serialize(out) + if err != nil { + t.Errorf("error reserializing: %s", err) + return + } + + expected, _ := hex.DecodeString(signatureDataHex) + if !bytes.Equal(expected, out.Bytes()) { + t.Errorf("output doesn't match input (got vs expected):\n%s\n%s", hex.Dump(out.Bytes()), hex.Dump(expected)) + } +} + +const signatureDataHex = "c2c05c04000102000605024cb45112000a0910ab105c91af38fb158f8d07ff5596ea368c5efe015bed6e78348c0f033c931d5f2ce5db54ce7f2a7e4b4ad64db758d65a7a71773edeab7ba2a9e0908e6a94a1175edd86c1d843279f045b021a6971a72702fcbd650efc393c5474d5b59a15f96d2eaad4c4c426797e0dcca2803ef41c6ff234d403eec38f31d610c344c06f2401c262f0993b2e66cad8a81ebc4322c723e0d4ba09fe917e8777658307ad8329adacba821420741009dfe87f007759f0982275d028a392c6ed983a0d846f890b36148c7358bdb8a516007fac760261ecd06076813831a36d0459075d1befa245ae7f7fb103d92ca759e9498fe60ef8078a39a3beda510deea251ea9f0a7f0df6ef42060f20780360686f3e400e" diff --git a/src/pkg/crypto/openpgp/packet/symmetric_key_encrypted.go b/src/pkg/crypto/openpgp/packet/symmetric_key_encrypted.go new file mode 100644 index 000000000..ad4f1d621 --- /dev/null +++ b/src/pkg/crypto/openpgp/packet/symmetric_key_encrypted.go @@ -0,0 +1,162 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package packet + +import ( + "bytes" + "crypto/cipher" + "crypto/openpgp/error" + "crypto/openpgp/s2k" + "io" + "os" + "strconv" +) + +// This is the largest session key that we'll support. Since no 512-bit cipher +// has even been seriously used, this is comfortably large. +const maxSessionKeySizeInBytes = 64 + +// SymmetricKeyEncrypted represents a passphrase protected session key. See RFC +// 4880, section 5.3. +type SymmetricKeyEncrypted struct { + CipherFunc CipherFunction + Encrypted bool + Key []byte // Empty unless Encrypted is false. + s2k func(out, in []byte) + encryptedKey []byte +} + +const symmetricKeyEncryptedVersion = 4 + +func (ske *SymmetricKeyEncrypted) parse(r io.Reader) (err os.Error) { + // RFC 4880, section 5.3. + var buf [2]byte + _, err = readFull(r, buf[:]) + if err != nil { + return + } + if buf[0] != symmetricKeyEncryptedVersion { + return error.UnsupportedError("SymmetricKeyEncrypted version") + } + ske.CipherFunc = CipherFunction(buf[1]) + + if ske.CipherFunc.KeySize() == 0 { + return error.UnsupportedError("unknown cipher: " + strconv.Itoa(int(buf[1]))) + } + + ske.s2k, err = s2k.Parse(r) + if err != nil { + return + } + + encryptedKey := make([]byte, maxSessionKeySizeInBytes) + // The session key may follow. We just have to try and read to find + // out. If it exists then we limit it to maxSessionKeySizeInBytes. + n, err := readFull(r, encryptedKey) + if err != nil && err != io.ErrUnexpectedEOF { + return + } + err = nil + if n != 0 { + if n == maxSessionKeySizeInBytes { + return error.UnsupportedError("oversized encrypted session key") + } + ske.encryptedKey = encryptedKey[:n] + } + + ske.Encrypted = true + + return +} + +// Decrypt attempts to decrypt an encrypted session key. If it returns nil, +// ske.Key will contain the session key. +func (ske *SymmetricKeyEncrypted) Decrypt(passphrase []byte) os.Error { + if !ske.Encrypted { + return nil + } + + key := make([]byte, ske.CipherFunc.KeySize()) + ske.s2k(key, passphrase) + + if len(ske.encryptedKey) == 0 { + ske.Key = key + } else { + // the IV is all zeros + iv := make([]byte, ske.CipherFunc.blockSize()) + c := cipher.NewCFBDecrypter(ske.CipherFunc.new(key), iv) + c.XORKeyStream(ske.encryptedKey, ske.encryptedKey) + ske.CipherFunc = CipherFunction(ske.encryptedKey[0]) + if ske.CipherFunc.blockSize() == 0 { + return error.UnsupportedError("unknown cipher: " + strconv.Itoa(int(ske.CipherFunc))) + } + ske.CipherFunc = CipherFunction(ske.encryptedKey[0]) + ske.Key = ske.encryptedKey[1:] + if len(ske.Key)%ske.CipherFunc.blockSize() != 0 { + ske.Key = nil + return error.StructuralError("length of decrypted key not a multiple of block size") + } + } + + ske.Encrypted = false + return nil +} + +// SerializeSymmetricKeyEncrypted serializes a symmetric key packet to w. The +// packet contains a random session key, encrypted by a key derived from the +// given passphrase. The session key is returned and must be passed to +// SerializeSymmetricallyEncrypted. +func SerializeSymmetricKeyEncrypted(w io.Writer, rand io.Reader, passphrase []byte, cipherFunc CipherFunction) (key []byte, err os.Error) { + keySize := cipherFunc.KeySize() + if keySize == 0 { + return nil, error.UnsupportedError("unknown cipher: " + strconv.Itoa(int(cipherFunc))) + } + + s2kBuf := new(bytes.Buffer) + keyEncryptingKey := make([]byte, keySize) + // s2k.Serialize salts and stretches the passphrase, and writes the + // resulting key to keyEncryptingKey and the s2k descriptor to s2kBuf. + err = s2k.Serialize(s2kBuf, keyEncryptingKey, rand, passphrase) + if err != nil { + return + } + s2kBytes := s2kBuf.Bytes() + + packetLength := 2 /* header */ + len(s2kBytes) + 1 /* cipher type */ + keySize + err = serializeHeader(w, packetTypeSymmetricKeyEncrypted, packetLength) + if err != nil { + return + } + + var buf [2]byte + buf[0] = symmetricKeyEncryptedVersion + buf[1] = byte(cipherFunc) + _, err = w.Write(buf[:]) + if err != nil { + return + } + _, err = w.Write(s2kBytes) + if err != nil { + return + } + + sessionKey := make([]byte, keySize) + _, err = io.ReadFull(rand, sessionKey) + if err != nil { + return + } + iv := make([]byte, cipherFunc.blockSize()) + c := cipher.NewCFBEncrypter(cipherFunc.new(keyEncryptingKey), iv) + encryptedCipherAndKey := make([]byte, keySize+1) + c.XORKeyStream(encryptedCipherAndKey, buf[1:]) + c.XORKeyStream(encryptedCipherAndKey[1:], sessionKey) + _, err = w.Write(encryptedCipherAndKey) + if err != nil { + return + } + + key = sessionKey + return +} diff --git a/src/pkg/crypto/openpgp/packet/symmetric_key_encrypted_test.go b/src/pkg/crypto/openpgp/packet/symmetric_key_encrypted_test.go new file mode 100644 index 000000000..823ec400d --- /dev/null +++ b/src/pkg/crypto/openpgp/packet/symmetric_key_encrypted_test.go @@ -0,0 +1,101 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package packet + +import ( + "bytes" + "crypto/rand" + "encoding/hex" + "io/ioutil" + "os" + "testing" +) + +func TestSymmetricKeyEncrypted(t *testing.T) { + buf := readerFromHex(symmetricallyEncryptedHex) + packet, err := Read(buf) + if err != nil { + t.Errorf("failed to read SymmetricKeyEncrypted: %s", err) + return + } + ske, ok := packet.(*SymmetricKeyEncrypted) + if !ok { + t.Error("didn't find SymmetricKeyEncrypted packet") + return + } + err = ske.Decrypt([]byte("password")) + if err != nil { + t.Error(err) + return + } + + packet, err = Read(buf) + if err != nil { + t.Errorf("failed to read SymmetricallyEncrypted: %s", err) + return + } + se, ok := packet.(*SymmetricallyEncrypted) + if !ok { + t.Error("didn't find SymmetricallyEncrypted packet") + return + } + r, err := se.Decrypt(ske.CipherFunc, ske.Key) + if err != nil { + t.Error(err) + return + } + + contents, err := ioutil.ReadAll(r) + if err != nil && err != os.EOF { + t.Error(err) + return + } + + expectedContents, _ := hex.DecodeString(symmetricallyEncryptedContentsHex) + if !bytes.Equal(expectedContents, contents) { + t.Errorf("bad contents got:%x want:%x", contents, expectedContents) + } +} + +const symmetricallyEncryptedHex = "8c0d04030302371a0b38d884f02060c91cf97c9973b8e58e028e9501708ccfe618fb92afef7fa2d80ddadd93cf" +const symmetricallyEncryptedContentsHex = "cb1062004d14c4df636f6e74656e74732e0a" + +func TestSerializeSymmetricKeyEncrypted(t *testing.T) { + buf := bytes.NewBuffer(nil) + passphrase := []byte("testing") + cipherFunc := CipherAES128 + + key, err := SerializeSymmetricKeyEncrypted(buf, rand.Reader, passphrase, cipherFunc) + if err != nil { + t.Errorf("failed to serialize: %s", err) + return + } + + p, err := Read(buf) + if err != nil { + t.Errorf("failed to reparse: %s", err) + return + } + ske, ok := p.(*SymmetricKeyEncrypted) + if !ok { + t.Errorf("parsed a different packet type: %#v", p) + return + } + + if !ske.Encrypted { + t.Errorf("SKE not encrypted but should be") + } + if ske.CipherFunc != cipherFunc { + t.Errorf("SKE cipher function is %d (expected %d)", ske.CipherFunc, cipherFunc) + } + err = ske.Decrypt(passphrase) + if err != nil { + t.Errorf("failed to decrypt reparsed SKE: %s", err) + return + } + if !bytes.Equal(key, ske.Key) { + t.Errorf("keys don't match after Decrpyt: %x (original) vs %x (parsed)", key, ske.Key) + } +} diff --git a/src/pkg/crypto/openpgp/packet/symmetrically_encrypted.go b/src/pkg/crypto/openpgp/packet/symmetrically_encrypted.go new file mode 100644 index 000000000..e33c9f3a0 --- /dev/null +++ b/src/pkg/crypto/openpgp/packet/symmetrically_encrypted.go @@ -0,0 +1,291 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package packet + +import ( + "crypto/cipher" + "crypto/openpgp/error" + "crypto/rand" + "crypto/sha1" + "crypto/subtle" + "hash" + "io" + "os" + "strconv" +) + +// SymmetricallyEncrypted represents a symmetrically encrypted byte string. The +// encrypted contents will consist of more OpenPGP packets. See RFC 4880, +// sections 5.7 and 5.13. +type SymmetricallyEncrypted struct { + MDC bool // true iff this is a type 18 packet and thus has an embedded MAC. + contents io.Reader + prefix []byte +} + +const symmetricallyEncryptedVersion = 1 + +func (se *SymmetricallyEncrypted) parse(r io.Reader) os.Error { + if se.MDC { + // See RFC 4880, section 5.13. + var buf [1]byte + _, err := readFull(r, buf[:]) + if err != nil { + return err + } + if buf[0] != symmetricallyEncryptedVersion { + return error.UnsupportedError("unknown SymmetricallyEncrypted version") + } + } + se.contents = r + return nil +} + +// Decrypt returns a ReadCloser, from which the decrypted contents of the +// packet can be read. An incorrect key can, with high probability, be detected +// immediately and this will result in a KeyIncorrect error being returned. +func (se *SymmetricallyEncrypted) Decrypt(c CipherFunction, key []byte) (io.ReadCloser, os.Error) { + keySize := c.KeySize() + if keySize == 0 { + return nil, error.UnsupportedError("unknown cipher: " + strconv.Itoa(int(c))) + } + if len(key) != keySize { + return nil, error.InvalidArgumentError("SymmetricallyEncrypted: incorrect key length") + } + + if se.prefix == nil { + se.prefix = make([]byte, c.blockSize()+2) + _, err := readFull(se.contents, se.prefix) + if err != nil { + return nil, err + } + } else if len(se.prefix) != c.blockSize()+2 { + return nil, error.InvalidArgumentError("can't try ciphers with different block lengths") + } + + ocfbResync := cipher.OCFBResync + if se.MDC { + // MDC packets use a different form of OCFB mode. + ocfbResync = cipher.OCFBNoResync + } + + s := cipher.NewOCFBDecrypter(c.new(key), se.prefix, ocfbResync) + if s == nil { + return nil, error.KeyIncorrectError + } + + plaintext := cipher.StreamReader{S: s, R: se.contents} + + if se.MDC { + // MDC packets have an embedded hash that we need to check. + h := sha1.New() + h.Write(se.prefix) + return &seMDCReader{in: plaintext, h: h}, nil + } + + // Otherwise, we just need to wrap plaintext so that it's a valid ReadCloser. + return seReader{plaintext}, nil +} + +// seReader wraps an io.Reader with a no-op Close method. +type seReader struct { + in io.Reader +} + +func (ser seReader) Read(buf []byte) (int, os.Error) { + return ser.in.Read(buf) +} + +func (ser seReader) Close() os.Error { + return nil +} + +const mdcTrailerSize = 1 /* tag byte */ + 1 /* length byte */ + sha1.Size + +// An seMDCReader wraps an io.Reader, maintains a running hash and keeps hold +// of the most recent 22 bytes (mdcTrailerSize). Upon EOF, those bytes form an +// MDC packet containing a hash of the previous contents which is checked +// against the running hash. See RFC 4880, section 5.13. +type seMDCReader struct { + in io.Reader + h hash.Hash + trailer [mdcTrailerSize]byte + scratch [mdcTrailerSize]byte + trailerUsed int + error bool + eof bool +} + +func (ser *seMDCReader) Read(buf []byte) (n int, err os.Error) { + if ser.error { + err = io.ErrUnexpectedEOF + return + } + if ser.eof { + err = os.EOF + return + } + + // If we haven't yet filled the trailer buffer then we must do that + // first. + for ser.trailerUsed < mdcTrailerSize { + n, err = ser.in.Read(ser.trailer[ser.trailerUsed:]) + ser.trailerUsed += n + if err == os.EOF { + if ser.trailerUsed != mdcTrailerSize { + n = 0 + err = io.ErrUnexpectedEOF + ser.error = true + return + } + ser.eof = true + n = 0 + return + } + + if err != nil { + n = 0 + return + } + } + + // If it's a short read then we read into a temporary buffer and shift + // the data into the caller's buffer. + if len(buf) <= mdcTrailerSize { + n, err = readFull(ser.in, ser.scratch[:len(buf)]) + copy(buf, ser.trailer[:n]) + ser.h.Write(buf[:n]) + copy(ser.trailer[:], ser.trailer[n:]) + copy(ser.trailer[mdcTrailerSize-n:], ser.scratch[:]) + if n < len(buf) { + ser.eof = true + err = os.EOF + } + return + } + + n, err = ser.in.Read(buf[mdcTrailerSize:]) + copy(buf, ser.trailer[:]) + ser.h.Write(buf[:n]) + copy(ser.trailer[:], buf[n:]) + + if err == os.EOF { + ser.eof = true + } + return +} + +// This is a new-format packet tag byte for a type 19 (MDC) packet. +const mdcPacketTagByte = byte(0x80) | 0x40 | 19 + +func (ser *seMDCReader) Close() os.Error { + if ser.error { + return error.SignatureError("error during reading") + } + + for !ser.eof { + // We haven't seen EOF so we need to read to the end + var buf [1024]byte + _, err := ser.Read(buf[:]) + if err == os.EOF { + break + } + if err != nil { + return error.SignatureError("error during reading") + } + } + + if ser.trailer[0] != mdcPacketTagByte || ser.trailer[1] != sha1.Size { + return error.SignatureError("MDC packet not found") + } + ser.h.Write(ser.trailer[:2]) + + final := ser.h.Sum() + if subtle.ConstantTimeCompare(final, ser.trailer[2:]) != 1 { + return error.SignatureError("hash mismatch") + } + return nil +} + +// An seMDCWriter writes through to an io.WriteCloser while maintains a running +// hash of the data written. On close, it emits an MDC packet containing the +// running hash. +type seMDCWriter struct { + w io.WriteCloser + h hash.Hash +} + +func (w *seMDCWriter) Write(buf []byte) (n int, err os.Error) { + w.h.Write(buf) + return w.w.Write(buf) +} + +func (w *seMDCWriter) Close() (err os.Error) { + var buf [mdcTrailerSize]byte + + buf[0] = mdcPacketTagByte + buf[1] = sha1.Size + w.h.Write(buf[:2]) + digest := w.h.Sum() + copy(buf[2:], digest) + + _, err = w.w.Write(buf[:]) + if err != nil { + return + } + return w.w.Close() +} + +// noOpCloser is like an ioutil.NopCloser, but for an io.Writer. +type noOpCloser struct { + w io.Writer +} + +func (c noOpCloser) Write(data []byte) (n int, err os.Error) { + return c.w.Write(data) +} + +func (c noOpCloser) Close() os.Error { + return nil +} + +// SerializeSymmetricallyEncrypted serializes a symmetrically encrypted packet +// to w and returns a WriteCloser to which the to-be-encrypted packets can be +// written. +func SerializeSymmetricallyEncrypted(w io.Writer, c CipherFunction, key []byte) (contents io.WriteCloser, err os.Error) { + if c.KeySize() != len(key) { + return nil, error.InvalidArgumentError("SymmetricallyEncrypted.Serialize: bad key length") + } + writeCloser := noOpCloser{w} + ciphertext, err := serializeStreamHeader(writeCloser, packetTypeSymmetricallyEncryptedMDC) + if err != nil { + return + } + + _, err = ciphertext.Write([]byte{symmetricallyEncryptedVersion}) + if err != nil { + return + } + + block := c.new(key) + blockSize := block.BlockSize() + iv := make([]byte, blockSize) + _, err = rand.Reader.Read(iv) + if err != nil { + return + } + s, prefix := cipher.NewOCFBEncrypter(block, iv, cipher.OCFBNoResync) + _, err = ciphertext.Write(prefix) + if err != nil { + return + } + plaintext := cipher.StreamWriter{S: s, W: ciphertext} + + h := sha1.New() + h.Write(iv) + h.Write(iv[blockSize-2:]) + contents = &seMDCWriter{w: plaintext, h: h} + return +} diff --git a/src/pkg/crypto/openpgp/packet/symmetrically_encrypted_test.go b/src/pkg/crypto/openpgp/packet/symmetrically_encrypted_test.go new file mode 100644 index 000000000..1054fc2f9 --- /dev/null +++ b/src/pkg/crypto/openpgp/packet/symmetrically_encrypted_test.go @@ -0,0 +1,124 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package packet + +import ( + "bytes" + "crypto/openpgp/error" + "crypto/sha1" + "encoding/hex" + "io" + "io/ioutil" + "os" + "testing" +) + +// TestReader wraps a []byte and returns reads of a specific length. +type testReader struct { + data []byte + stride int +} + +func (t *testReader) Read(buf []byte) (n int, err os.Error) { + n = t.stride + if n > len(t.data) { + n = len(t.data) + } + if n > len(buf) { + n = len(buf) + } + copy(buf, t.data) + t.data = t.data[n:] + if len(t.data) == 0 { + err = os.EOF + } + return +} + +func testMDCReader(t *testing.T) { + mdcPlaintext, _ := hex.DecodeString(mdcPlaintextHex) + + for stride := 1; stride < len(mdcPlaintext)/2; stride++ { + r := &testReader{data: mdcPlaintext, stride: stride} + mdcReader := &seMDCReader{in: r, h: sha1.New()} + body, err := ioutil.ReadAll(mdcReader) + if err != nil { + t.Errorf("stride: %d, error: %s", stride, err) + continue + } + if !bytes.Equal(body, mdcPlaintext[:len(mdcPlaintext)-22]) { + t.Errorf("stride: %d: bad contents %x", stride, body) + continue + } + + err = mdcReader.Close() + if err != nil { + t.Errorf("stride: %d, error on Close: %s", stride, err) + } + } + + mdcPlaintext[15] ^= 80 + + r := &testReader{data: mdcPlaintext, stride: 2} + mdcReader := &seMDCReader{in: r, h: sha1.New()} + _, err := ioutil.ReadAll(mdcReader) + if err != nil { + t.Errorf("corruption test, error: %s", err) + return + } + err = mdcReader.Close() + if err == nil { + t.Error("corruption: no error") + } else if _, ok := err.(*error.SignatureError); !ok { + t.Errorf("corruption: expected SignatureError, got: %s", err) + } +} + +const mdcPlaintextHex = "a302789c3b2d93c4e0eb9aba22283539b3203335af44a134afb800c849cb4c4de10200aff40b45d31432c80cb384299a0655966d6939dfdeed1dddf980" + +func TestSerialize(t *testing.T) { + buf := bytes.NewBuffer(nil) + c := CipherAES128 + key := make([]byte, c.KeySize()) + + w, err := SerializeSymmetricallyEncrypted(buf, c, key) + if err != nil { + t.Errorf("error from SerializeSymmetricallyEncrypted: %s", err) + return + } + + contents := []byte("hello world\n") + + w.Write(contents) + w.Close() + + p, err := Read(buf) + if err != nil { + t.Errorf("error from Read: %s", err) + return + } + + se, ok := p.(*SymmetricallyEncrypted) + if !ok { + t.Errorf("didn't read a *SymmetricallyEncrypted") + return + } + + r, err := se.Decrypt(c, key) + if err != nil { + t.Errorf("error from Decrypt: %s", err) + return + } + + contentsCopy := bytes.NewBuffer(nil) + _, err = io.Copy(contentsCopy, r) + if err != nil { + t.Errorf("error from io.Copy: %s", err) + return + } + if !bytes.Equal(contentsCopy.Bytes(), contents) { + t.Errorf("contents not equal got: %x want: %x", contentsCopy.Bytes(), contents) + } +} diff --git a/src/pkg/crypto/openpgp/packet/userid.go b/src/pkg/crypto/openpgp/packet/userid.go new file mode 100644 index 000000000..0580ba3ed --- /dev/null +++ b/src/pkg/crypto/openpgp/packet/userid.go @@ -0,0 +1,161 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package packet + +import ( + "io" + "io/ioutil" + "os" + "strings" +) + +// UserId contains text that is intended to represent the name and email +// address of the key holder. See RFC 4880, section 5.11. By convention, this +// takes the form "Full Name (Comment) " +type UserId struct { + Id string // By convention, this takes the form "Full Name (Comment) " which is split out in the fields below. + + Name, Comment, Email string +} + +func hasInvalidCharacters(s string) bool { + for _, c := range s { + switch c { + case '(', ')', '<', '>', 0: + return true + } + } + return false +} + +// NewUserId returns a UserId or nil if any of the arguments contain invalid +// characters. The invalid characters are '\x00', '(', ')', '<' and '>' +func NewUserId(name, comment, email string) *UserId { + // RFC 4880 doesn't deal with the structure of userid strings; the + // name, comment and email form is just a convention. However, there's + // no convention about escaping the metacharacters and GPG just refuses + // to create user ids where, say, the name contains a '('. We mirror + // this behaviour. + + if hasInvalidCharacters(name) || hasInvalidCharacters(comment) || hasInvalidCharacters(email) { + return nil + } + + uid := new(UserId) + uid.Name, uid.Comment, uid.Email = name, comment, email + uid.Id = name + if len(comment) > 0 { + if len(uid.Id) > 0 { + uid.Id += " " + } + uid.Id += "(" + uid.Id += comment + uid.Id += ")" + } + if len(email) > 0 { + if len(uid.Id) > 0 { + uid.Id += " " + } + uid.Id += "<" + uid.Id += email + uid.Id += ">" + } + return uid +} + +func (uid *UserId) parse(r io.Reader) (err os.Error) { + // RFC 4880, section 5.11 + b, err := ioutil.ReadAll(r) + if err != nil { + return + } + uid.Id = string(b) + uid.Name, uid.Comment, uid.Email = parseUserId(uid.Id) + return +} + +// Serialize marshals uid to w in the form of an OpenPGP packet, including +// header. +func (uid *UserId) Serialize(w io.Writer) os.Error { + err := serializeHeader(w, packetTypeUserId, len(uid.Id)) + if err != nil { + return err + } + _, err = w.Write([]byte(uid.Id)) + return err +} + +// parseUserId extracts the name, comment and email from a user id string that +// is formatted as "Full Name (Comment) ". +func parseUserId(id string) (name, comment, email string) { + var n, c, e struct { + start, end int + } + var state int + + for offset, rune := range id { + switch state { + case 0: + // Entering name + n.start = offset + state = 1 + fallthrough + case 1: + // In name + if rune == '(' { + state = 2 + n.end = offset + } else if rune == '<' { + state = 5 + n.end = offset + } + case 2: + // Entering comment + c.start = offset + state = 3 + fallthrough + case 3: + // In comment + if rune == ')' { + state = 4 + c.end = offset + } + case 4: + // Between comment and email + if rune == '<' { + state = 5 + } + case 5: + // Entering email + e.start = offset + state = 6 + fallthrough + case 6: + // In email + if rune == '>' { + state = 7 + e.end = offset + } + default: + // After email + } + } + switch state { + case 1: + // ended in the name + n.end = len(id) + case 3: + // ended in comment + c.end = len(id) + case 6: + // ended in email + e.end = len(id) + } + + name = strings.TrimSpace(id[n.start:n.end]) + comment = strings.TrimSpace(id[c.start:c.end]) + email = strings.TrimSpace(id[e.start:e.end]) + return +} diff --git a/src/pkg/crypto/openpgp/packet/userid_test.go b/src/pkg/crypto/openpgp/packet/userid_test.go new file mode 100644 index 000000000..296819389 --- /dev/null +++ b/src/pkg/crypto/openpgp/packet/userid_test.go @@ -0,0 +1,87 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package packet + +import ( + "testing" +) + +var userIdTests = []struct { + id string + name, comment, email string +}{ + {"", "", "", ""}, + {"John Smith", "John Smith", "", ""}, + {"John Smith ()", "John Smith", "", ""}, + {"John Smith () <>", "John Smith", "", ""}, + {"(comment", "", "comment", ""}, + {"(comment)", "", "comment", ""}, + {" sdfk", "", "", "email"}, + {" John Smith ( Comment ) asdkflj < email > lksdfj", "John Smith", "Comment", "email"}, + {" John Smith < email > lksdfj", "John Smith", "", "email"}, + {"("}, + {"foo", "bar", "", "foo (bar)"}, + {"foo", "", "baz", "foo "}, + {"", "bar", "baz", "(bar) "}, + {"foo", "bar", "baz", "foo (bar) "}, +} + +func TestNewUserId(t *testing.T) { + for i, test := range newUserIdTests { + uid := NewUserId(test.name, test.comment, test.email) + if uid == nil { + t.Errorf("#%d: returned nil", i) + continue + } + if uid.Id != test.id { + t.Errorf("#%d: got '%s', want '%s'", i, uid.Id, test.id) + } + } +} + +var invalidNewUserIdTests = []struct { + name, comment, email string +}{ + {"foo(", "", ""}, + {"foo<", "", ""}, + {"", "bar)", ""}, + {"", "bar<", ""}, + {"", "", "baz>"}, + {"", "", "baz)"}, + {"", "", "baz\x00"}, +} + +func TestNewUserIdWithInvalidInput(t *testing.T) { + for i, test := range invalidNewUserIdTests { + if uid := NewUserId(test.name, test.comment, test.email); uid != nil { + t.Errorf("#%d: returned non-nil value: %#v", i, uid) + } + } +} diff --git a/src/pkg/crypto/openpgp/read.go b/src/pkg/crypto/openpgp/read.go new file mode 100644 index 000000000..d95f613c6 --- /dev/null +++ b/src/pkg/crypto/openpgp/read.go @@ -0,0 +1,415 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Package openpgp implements high level operations on OpenPGP messages. +package openpgp + +import ( + "crypto" + "crypto/openpgp/armor" + "crypto/openpgp/error" + "crypto/openpgp/packet" + _ "crypto/sha256" + "hash" + "io" + "os" + "strconv" +) + +// SignatureType is the armor type for a PGP signature. +var SignatureType = "PGP SIGNATURE" + +// readArmored reads an armored block with the given type. +func readArmored(r io.Reader, expectedType string) (body io.Reader, err os.Error) { + block, err := armor.Decode(r) + if err != nil { + return + } + + if block.Type != expectedType { + return nil, error.InvalidArgumentError("expected '" + expectedType + "', got: " + block.Type) + } + + return block.Body, nil +} + +// MessageDetails contains the result of parsing an OpenPGP encrypted and/or +// signed message. +type MessageDetails struct { + IsEncrypted bool // true if the message was encrypted. + EncryptedToKeyIds []uint64 // the list of recipient key ids. + IsSymmetricallyEncrypted bool // true if a passphrase could have decrypted the message. + DecryptedWith Key // the private key used to decrypt the message, if any. + IsSigned bool // true if the message is signed. + SignedByKeyId uint64 // the key id of the signer, if any. + SignedBy *Key // the key of the signer, if available. + LiteralData *packet.LiteralData // the metadata of the contents + UnverifiedBody io.Reader // the contents of the message. + + // If IsSigned is true and SignedBy is non-zero then the signature will + // be verified as UnverifiedBody is read. The signature cannot be + // checked until the whole of UnverifiedBody is read so UnverifiedBody + // must be consumed until EOF before the data can trusted. Even if a + // message isn't signed (or the signer is unknown) the data may contain + // an authentication code that is only checked once UnverifiedBody has + // been consumed. Once EOF has been seen, the following fields are + // valid. (An authentication code failure is reported as a + // SignatureError error when reading from UnverifiedBody.) + SignatureError os.Error // nil if the signature is good. + Signature *packet.Signature // the signature packet itself. + + decrypted io.ReadCloser +} + +// A PromptFunction is used as a callback by functions that may need to decrypt +// a private key, or prompt for a passphrase. It is called with a list of +// acceptable, encrypted private keys and a boolean that indicates whether a +// passphrase is usable. It should either decrypt a private key or return a +// passphrase to try. If the decrypted private key or given passphrase isn't +// correct, the function will be called again, forever. Any error returned will +// be passed up. +type PromptFunction func(keys []Key, symmetric bool) ([]byte, os.Error) + +// A keyEnvelopePair is used to store a private key with the envelope that +// contains a symmetric key, encrypted with that key. +type keyEnvelopePair struct { + key Key + encryptedKey *packet.EncryptedKey +} + +// ReadMessage parses an OpenPGP message that may be signed and/or encrypted. +// The given KeyRing should contain both public keys (for signature +// verification) and, possibly encrypted, private keys for decrypting. +func ReadMessage(r io.Reader, keyring KeyRing, prompt PromptFunction) (md *MessageDetails, err os.Error) { + var p packet.Packet + + var symKeys []*packet.SymmetricKeyEncrypted + var pubKeys []keyEnvelopePair + var se *packet.SymmetricallyEncrypted + + packets := packet.NewReader(r) + md = new(MessageDetails) + md.IsEncrypted = true + + // The message, if encrypted, starts with a number of packets + // containing an encrypted decryption key. The decryption key is either + // encrypted to a public key, or with a passphrase. This loop + // collects these packets. +ParsePackets: + for { + p, err = packets.Next() + if err != nil { + return nil, err + } + switch p := p.(type) { + case *packet.SymmetricKeyEncrypted: + // This packet contains the decryption key encrypted with a passphrase. + md.IsSymmetricallyEncrypted = true + symKeys = append(symKeys, p) + case *packet.EncryptedKey: + // This packet contains the decryption key encrypted to a public key. + md.EncryptedToKeyIds = append(md.EncryptedToKeyIds, p.KeyId) + switch p.Algo { + case packet.PubKeyAlgoRSA, packet.PubKeyAlgoRSAEncryptOnly, packet.PubKeyAlgoElGamal: + break + default: + continue + } + var keys []Key + if p.KeyId == 0 { + keys = keyring.DecryptionKeys() + } else { + keys = keyring.KeysById(p.KeyId) + } + for _, k := range keys { + pubKeys = append(pubKeys, keyEnvelopePair{k, p}) + } + case *packet.SymmetricallyEncrypted: + se = p + break ParsePackets + case *packet.Compressed, *packet.LiteralData, *packet.OnePassSignature: + // This message isn't encrypted. + if len(symKeys) != 0 || len(pubKeys) != 0 { + return nil, error.StructuralError("key material not followed by encrypted message") + } + packets.Unread(p) + return readSignedMessage(packets, nil, keyring) + } + } + + var candidates []Key + var decrypted io.ReadCloser + + // Now that we have the list of encrypted keys we need to decrypt at + // least one of them or, if we cannot, we need to call the prompt + // function so that it can decrypt a key or give us a passphrase. +FindKey: + for { + // See if any of the keys already have a private key available + candidates = candidates[:0] + candidateFingerprints := make(map[string]bool) + + for _, pk := range pubKeys { + if pk.key.PrivateKey == nil { + continue + } + if !pk.key.PrivateKey.Encrypted { + if len(pk.encryptedKey.Key) == 0 { + pk.encryptedKey.Decrypt(pk.key.PrivateKey) + } + if len(pk.encryptedKey.Key) == 0 { + continue + } + decrypted, err = se.Decrypt(pk.encryptedKey.CipherFunc, pk.encryptedKey.Key) + if err != nil && err != error.KeyIncorrectError { + return nil, err + } + if decrypted != nil { + md.DecryptedWith = pk.key + break FindKey + } + } else { + fpr := string(pk.key.PublicKey.Fingerprint[:]) + if v := candidateFingerprints[fpr]; v { + continue + } + candidates = append(candidates, pk.key) + candidateFingerprints[fpr] = true + } + } + + if len(candidates) == 0 && len(symKeys) == 0 { + return nil, error.KeyIncorrectError + } + + if prompt == nil { + return nil, error.KeyIncorrectError + } + + passphrase, err := prompt(candidates, len(symKeys) != 0) + if err != nil { + return nil, err + } + + // Try the symmetric passphrase first + if len(symKeys) != 0 && passphrase != nil { + for _, s := range symKeys { + err = s.Decrypt(passphrase) + if err == nil && !s.Encrypted { + decrypted, err = se.Decrypt(s.CipherFunc, s.Key) + if err != nil && err != error.KeyIncorrectError { + return nil, err + } + if decrypted != nil { + break FindKey + } + } + + } + } + } + + md.decrypted = decrypted + packets.Push(decrypted) + return readSignedMessage(packets, md, keyring) +} + +// readSignedMessage reads a possibly signed message if mdin is non-zero then +// that structure is updated and returned. Otherwise a fresh MessageDetails is +// used. +func readSignedMessage(packets *packet.Reader, mdin *MessageDetails, keyring KeyRing) (md *MessageDetails, err os.Error) { + if mdin == nil { + mdin = new(MessageDetails) + } + md = mdin + + var p packet.Packet + var h hash.Hash + var wrappedHash hash.Hash +FindLiteralData: + for { + p, err = packets.Next() + if err != nil { + return nil, err + } + switch p := p.(type) { + case *packet.Compressed: + packets.Push(p.Body) + case *packet.OnePassSignature: + if !p.IsLast { + return nil, error.UnsupportedError("nested signatures") + } + + h, wrappedHash, err = hashForSignature(p.Hash, p.SigType) + if err != nil { + md = nil + return + } + + md.IsSigned = true + md.SignedByKeyId = p.KeyId + keys := keyring.KeysById(p.KeyId) + for i, key := range keys { + if key.SelfSignature.FlagsValid && !key.SelfSignature.FlagSign { + continue + } + md.SignedBy = &keys[i] + break + } + case *packet.LiteralData: + md.LiteralData = p + break FindLiteralData + } + } + + if md.SignedBy != nil { + md.UnverifiedBody = &signatureCheckReader{packets, h, wrappedHash, md} + } else if md.decrypted != nil { + md.UnverifiedBody = checkReader{md} + } else { + md.UnverifiedBody = md.LiteralData.Body + } + + return md, nil +} + +// hashForSignature returns a pair of hashes that can be used to verify a +// signature. The signature may specify that the contents of the signed message +// should be preprocessed (i.e. to normalize line endings). Thus this function +// returns two hashes. The second should be used to hash the message itself and +// performs any needed preprocessing. +func hashForSignature(hashId crypto.Hash, sigType packet.SignatureType) (hash.Hash, hash.Hash, os.Error) { + h := hashId.New() + if h == nil { + return nil, nil, error.UnsupportedError("hash not available: " + strconv.Itoa(int(hashId))) + } + + switch sigType { + case packet.SigTypeBinary: + return h, h, nil + case packet.SigTypeText: + return h, NewCanonicalTextHash(h), nil + } + + return nil, nil, error.UnsupportedError("unsupported signature type: " + strconv.Itoa(int(sigType))) +} + +// checkReader wraps an io.Reader from a LiteralData packet. When it sees EOF +// it closes the ReadCloser from any SymmetricallyEncrypted packet to trigger +// MDC checks. +type checkReader struct { + md *MessageDetails +} + +func (cr checkReader) Read(buf []byte) (n int, err os.Error) { + n, err = cr.md.LiteralData.Body.Read(buf) + if err == os.EOF { + mdcErr := cr.md.decrypted.Close() + if mdcErr != nil { + err = mdcErr + } + } + return +} + +// signatureCheckReader wraps an io.Reader from a LiteralData packet and hashes +// the data as it is read. When it sees an EOF from the underlying io.Reader +// it parses and checks a trailing Signature packet and triggers any MDC checks. +type signatureCheckReader struct { + packets *packet.Reader + h, wrappedHash hash.Hash + md *MessageDetails +} + +func (scr *signatureCheckReader) Read(buf []byte) (n int, err os.Error) { + n, err = scr.md.LiteralData.Body.Read(buf) + scr.wrappedHash.Write(buf[:n]) + if err == os.EOF { + var p packet.Packet + p, scr.md.SignatureError = scr.packets.Next() + if scr.md.SignatureError != nil { + return + } + + var ok bool + if scr.md.Signature, ok = p.(*packet.Signature); !ok { + scr.md.SignatureError = error.StructuralError("LiteralData not followed by Signature") + return + } + + scr.md.SignatureError = scr.md.SignedBy.PublicKey.VerifySignature(scr.h, scr.md.Signature) + + // The SymmetricallyEncrypted packet, if any, might have an + // unsigned hash of its own. In order to check this we need to + // close that Reader. + if scr.md.decrypted != nil { + mdcErr := scr.md.decrypted.Close() + if mdcErr != nil { + err = mdcErr + } + } + } + return +} + +// CheckDetachedSignature takes a signed file and a detached signature and +// returns the signer if the signature is valid. If the signer isn't know, +// UnknownIssuerError is returned. +func CheckDetachedSignature(keyring KeyRing, signed, signature io.Reader) (signer *Entity, err os.Error) { + p, err := packet.Read(signature) + if err != nil { + return + } + + sig, ok := p.(*packet.Signature) + if !ok { + return nil, error.StructuralError("non signature packet found") + } + + if sig.IssuerKeyId == nil { + return nil, error.StructuralError("signature doesn't have an issuer") + } + + keys := keyring.KeysById(*sig.IssuerKeyId) + if len(keys) == 0 { + return nil, error.UnknownIssuerError + } + + h, wrappedHash, err := hashForSignature(sig.Hash, sig.SigType) + if err != nil { + return + } + + _, err = io.Copy(wrappedHash, signed) + if err != nil && err != os.EOF { + return + } + + for _, key := range keys { + if key.SelfSignature.FlagsValid && !key.SelfSignature.FlagSign { + continue + } + err = key.PublicKey.VerifySignature(h, sig) + if err == nil { + return key.Entity, nil + } + } + + if err != nil { + return + } + + return nil, error.UnknownIssuerError +} + +// CheckArmoredDetachedSignature performs the same actions as +// CheckDetachedSignature but expects the signature to be armored. +func CheckArmoredDetachedSignature(keyring KeyRing, signed, signature io.Reader) (signer *Entity, err os.Error) { + body, err := readArmored(signature, SignatureType) + if err != nil { + return + } + + return CheckDetachedSignature(keyring, signed, body) +} diff --git a/src/pkg/crypto/openpgp/read_test.go b/src/pkg/crypto/openpgp/read_test.go new file mode 100644 index 000000000..4dc290ef2 --- /dev/null +++ b/src/pkg/crypto/openpgp/read_test.go @@ -0,0 +1,361 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package openpgp + +import ( + "bytes" + "crypto/openpgp/error" + "encoding/hex" + "io" + "io/ioutil" + "os" + "testing" +) + +func readerFromHex(s string) io.Reader { + data, err := hex.DecodeString(s) + if err != nil { + panic("readerFromHex: bad input") + } + return bytes.NewBuffer(data) +} + +func TestReadKeyRing(t *testing.T) { + kring, err := ReadKeyRing(readerFromHex(testKeys1And2Hex)) + if err != nil { + t.Error(err) + return + } + if len(kring) != 2 || uint32(kring[0].PrimaryKey.KeyId) != 0xC20C31BB || uint32(kring[1].PrimaryKey.KeyId) != 0x1E35246B { + t.Errorf("bad keyring: %#v", kring) + } +} + +func TestRereadKeyRing(t *testing.T) { + kring, err := ReadKeyRing(readerFromHex(testKeys1And2Hex)) + if err != nil { + t.Errorf("error in initial parse: %s", err) + return + } + out := new(bytes.Buffer) + err = kring[0].Serialize(out) + if err != nil { + t.Errorf("error in serialization: %s", err) + return + } + kring, err = ReadKeyRing(out) + if err != nil { + t.Errorf("error in second parse: %s", err) + return + } + + if len(kring) != 1 || uint32(kring[0].PrimaryKey.KeyId) != 0xC20C31BB { + t.Errorf("bad keyring: %#v", kring) + } +} + +func TestReadPrivateKeyRing(t *testing.T) { + kring, err := ReadKeyRing(readerFromHex(testKeys1And2PrivateHex)) + if err != nil { + t.Error(err) + return + } + if len(kring) != 2 || uint32(kring[0].PrimaryKey.KeyId) != 0xC20C31BB || uint32(kring[1].PrimaryKey.KeyId) != 0x1E35246B || kring[0].PrimaryKey == nil { + t.Errorf("bad keyring: %#v", kring) + } +} + +func TestReadDSAKey(t *testing.T) { + kring, err := ReadKeyRing(readerFromHex(dsaTestKeyHex)) + if err != nil { + t.Error(err) + return + } + if len(kring) != 1 || uint32(kring[0].PrimaryKey.KeyId) != 0x0CCC0360 { + t.Errorf("bad parse: %#v", kring) + } +} + +func TestGetKeyById(t *testing.T) { + kring, _ := ReadKeyRing(readerFromHex(testKeys1And2Hex)) + + keys := kring.KeysById(0xa34d7e18c20c31bb) + if len(keys) != 1 || keys[0].Entity != kring[0] { + t.Errorf("bad result for 0xa34d7e18c20c31bb: %#v", keys) + } + + keys = kring.KeysById(0xfd94408d4543314f) + if len(keys) != 1 || keys[0].Entity != kring[0] { + t.Errorf("bad result for 0xa34d7e18c20c31bb: %#v", keys) + } +} + +func checkSignedMessage(t *testing.T, signedHex, expected string) { + kring, _ := ReadKeyRing(readerFromHex(testKeys1And2Hex)) + + md, err := ReadMessage(readerFromHex(signedHex), kring, nil) + if err != nil { + t.Error(err) + return + } + + if !md.IsSigned || md.SignedByKeyId != 0xa34d7e18c20c31bb || md.SignedBy == nil || md.IsEncrypted || md.IsSymmetricallyEncrypted || len(md.EncryptedToKeyIds) != 0 || md.IsSymmetricallyEncrypted { + t.Errorf("bad MessageDetails: %#v", md) + } + + contents, err := ioutil.ReadAll(md.UnverifiedBody) + if err != nil { + t.Errorf("error reading UnverifiedBody: %s", err) + } + if string(contents) != expected { + t.Errorf("bad UnverifiedBody got:%s want:%s", string(contents), expected) + } + if md.SignatureError != nil || md.Signature == nil { + t.Errorf("failed to validate: %s", md.SignatureError) + } +} + +func TestSignedMessage(t *testing.T) { + checkSignedMessage(t, signedMessageHex, signedInput) +} + +func TestTextSignedMessage(t *testing.T) { + checkSignedMessage(t, signedTextMessageHex, signedTextInput) +} + +var signedEncryptedMessageTests = []struct { + keyRingHex string + messageHex string + signedByKeyId uint64 + encryptedToKeyId uint64 +}{ + { + testKeys1And2PrivateHex, + signedEncryptedMessageHex, + 0xa34d7e18c20c31bb, + 0x2a67d68660df41c7, + }, + { + dsaElGamalTestKeysHex, + signedEncryptedMessage2Hex, + 0x33af447ccd759b09, + 0xcf6a7abcd43e3673, + }, +} + +func TestSignedEncryptedMessage(t *testing.T) { + for i, test := range signedEncryptedMessageTests { + expected := "Signed and encrypted message\n" + kring, _ := ReadKeyRing(readerFromHex(test.keyRingHex)) + prompt := func(keys []Key, symmetric bool) ([]byte, os.Error) { + if symmetric { + t.Errorf("prompt: message was marked as symmetrically encrypted") + return nil, error.KeyIncorrectError + } + + if len(keys) == 0 { + t.Error("prompt: no keys requested") + return nil, error.KeyIncorrectError + } + + err := keys[0].PrivateKey.Decrypt([]byte("passphrase")) + if err != nil { + t.Errorf("prompt: error decrypting key: %s", err) + return nil, error.KeyIncorrectError + } + + return nil, nil + } + + md, err := ReadMessage(readerFromHex(test.messageHex), kring, prompt) + if err != nil { + t.Errorf("#%d: error reading message: %s", i, err) + return + } + + if !md.IsSigned || md.SignedByKeyId != test.signedByKeyId || md.SignedBy == nil || !md.IsEncrypted || md.IsSymmetricallyEncrypted || len(md.EncryptedToKeyIds) == 0 || md.EncryptedToKeyIds[0] != test.encryptedToKeyId { + t.Errorf("#%d: bad MessageDetails: %#v", i, md) + } + + contents, err := ioutil.ReadAll(md.UnverifiedBody) + if err != nil { + t.Errorf("#%d: error reading UnverifiedBody: %s", i, err) + } + if string(contents) != expected { + t.Errorf("#%d: bad UnverifiedBody got:%s want:%s", i, string(contents), expected) + } + + if md.SignatureError != nil || md.Signature == nil { + t.Errorf("#%d: failed to validate: %s", i, md.SignatureError) + } + } +} + +func TestUnspecifiedRecipient(t *testing.T) { + expected := "Recipient unspecified\n" + kring, _ := ReadKeyRing(readerFromHex(testKeys1And2PrivateHex)) + + md, err := ReadMessage(readerFromHex(recipientUnspecifiedHex), kring, nil) + if err != nil { + t.Errorf("error reading message: %s", err) + return + } + + contents, err := ioutil.ReadAll(md.UnverifiedBody) + if err != nil { + t.Errorf("error reading UnverifiedBody: %s", err) + } + if string(contents) != expected { + t.Errorf("bad UnverifiedBody got:%s want:%s", string(contents), expected) + } +} + +func TestSymmetricallyEncrypted(t *testing.T) { + expected := "Symmetrically encrypted.\n" + + prompt := func(keys []Key, symmetric bool) ([]byte, os.Error) { + if len(keys) != 0 { + t.Errorf("prompt: len(keys) = %d (want 0)", len(keys)) + } + + if !symmetric { + t.Errorf("symmetric is not set") + } + + return []byte("password"), nil + } + + md, err := ReadMessage(readerFromHex(symmetricallyEncryptedCompressedHex), nil, prompt) + if err != nil { + t.Errorf("ReadMessage: %s", err) + return + } + + contents, err := ioutil.ReadAll(md.UnverifiedBody) + if err != nil { + t.Errorf("ReadAll: %s", err) + } + + expectedCreationTime := uint32(1295992998) + if md.LiteralData.Time != expectedCreationTime { + t.Errorf("LiteralData.Time is %d, want %d", md.LiteralData.Time, expectedCreationTime) + } + + if string(contents) != expected { + t.Errorf("contents got: %s want: %s", string(contents), expected) + } +} + +func testDetachedSignature(t *testing.T, kring KeyRing, signature io.Reader, sigInput, tag string, expectedSignerKeyId uint64) { + signed := bytes.NewBufferString(sigInput) + signer, err := CheckDetachedSignature(kring, signed, signature) + if err != nil { + t.Errorf("%s: signature error: %s", tag, err) + return + } + if signer == nil { + t.Errorf("%s: signer is nil", tag) + return + } + if signer.PrimaryKey.KeyId != expectedSignerKeyId { + t.Errorf("%s: wrong signer got:%x want:%x", tag, signer.PrimaryKey.KeyId, expectedSignerKeyId) + } +} + +func TestDetachedSignature(t *testing.T) { + kring, _ := ReadKeyRing(readerFromHex(testKeys1And2Hex)) + testDetachedSignature(t, kring, readerFromHex(detachedSignatureHex), signedInput, "binary", testKey1KeyId) + testDetachedSignature(t, kring, readerFromHex(detachedSignatureTextHex), signedInput, "text", testKey1KeyId) +} + +func TestDetachedSignatureDSA(t *testing.T) { + kring, _ := ReadKeyRing(readerFromHex(dsaTestKeyHex)) + testDetachedSignature(t, kring, readerFromHex(detachedSignatureDSAHex), signedInput, "binary", testKey3KeyId) +} + +func TestReadingArmoredPrivateKey(t *testing.T) { + el, err := ReadArmoredKeyRing(bytes.NewBufferString(armoredPrivateKeyBlock)) + if err != nil { + t.Error(err) + } + if len(el) != 1 { + t.Errorf("got %d entities, wanted 1\n", len(el)) + } +} + +func TestNoArmoredData(t *testing.T) { + _, err := ReadArmoredKeyRing(bytes.NewBufferString("foo")) + if _, ok := err.(error.InvalidArgumentError); !ok { + t.Errorf("error was not an InvalidArgumentError: %s", err) + } +} + +const testKey1KeyId = 0xA34D7E18C20C31BB +const testKey3KeyId = 0x338934250CCC0360 + +const signedInput = "Signed message\nline 2\nline 3\n" +const signedTextInput = "Signed message\r\nline 2\r\nline 3\r\n" + +const recipientUnspecifiedHex = "848c0300000000000000000103ff62d4d578d03cf40c3da998dfe216c074fa6ddec5e31c197c9666ba292830d91d18716a80f699f9d897389a90e6d62d0238f5f07a5248073c0f24920e4bc4a30c2d17ee4e0cae7c3d4aaa4e8dced50e3010a80ee692175fa0385f62ecca4b56ee6e9980aa3ec51b61b077096ac9e800edaf161268593eedb6cc7027ff5cb32745d250010d407a6221ae22ef18469b444f2822478c4d190b24d36371a95cb40087cdd42d9399c3d06a53c0673349bfb607927f20d1e122bde1e2bf3aa6cae6edf489629bcaa0689539ae3b718914d88ededc3b" + +const detachedSignatureHex = "889c04000102000605024d449cd1000a0910a34d7e18c20c31bb167603ff57718d09f28a519fdc7b5a68b6a3336da04df85e38c5cd5d5bd2092fa4629848a33d85b1729402a2aab39c3ac19f9d573f773cc62c264dc924c067a79dfd8a863ae06c7c8686120760749f5fd9b1e03a64d20a7df3446ddc8f0aeadeaeba7cbaee5c1e366d65b6a0c6cc749bcb912d2f15013f812795c2e29eb7f7b77f39ce77" + +const detachedSignatureTextHex = "889c04010102000605024d449d21000a0910a34d7e18c20c31bbc8c60400a24fbef7342603a41cb1165767bd18985d015fb72fe05db42db36cfb2f1d455967f1e491194fbf6cf88146222b23bf6ffbd50d17598d976a0417d3192ff9cc0034fd00f287b02e90418bbefe609484b09231e4e7a5f3562e199bf39909ab5276c4d37382fe088f6b5c3426fc1052865da8b3ab158672d58b6264b10823dc4b39" + +const detachedSignatureDSAHex = "884604001102000605024d6c4eac000a0910338934250ccc0360f18d00a087d743d6405ed7b87755476629600b8b694a39e900a0abff8126f46faf1547c1743c37b21b4ea15b8f83" + +const testKeys1And2Hex = "988d044d3c5c10010400b1d13382944bd5aba23a4312968b5095d14f947f600eb478e14a6fcb16b0e0cac764884909c020bc495cfcc39a935387c661507bdb236a0612fb582cac3af9b29cc2c8c70090616c41b662f4da4c1201e195472eb7f4ae1ccbcbf9940fe21d985e379a5563dde5b9a23d35f1cfaa5790da3b79db26f23695107bfaca8e7b5bcd0011010001b41054657374204b6579203120285253412988b804130102002205024d3c5c10021b03060b090807030206150802090a0b0416020301021e01021780000a0910a34d7e18c20c31bbb5b304009cc45fe610b641a2c146331be94dade0a396e73ca725e1b25c21708d9cab46ecca5ccebc23055879df8f99eea39b377962a400f2ebdc36a7c99c333d74aeba346315137c3ff9d0a09b0273299090343048afb8107cf94cbd1400e3026f0ccac7ecebbc4d78588eb3e478fe2754d3ca664bcf3eac96ca4a6b0c8d7df5102f60f6b0020003b88d044d3c5c10010400b201df61d67487301f11879d514f4248ade90c8f68c7af1284c161098de4c28c2850f1ec7b8e30f959793e571542ffc6532189409cb51c3d30dad78c4ad5165eda18b20d9826d8707d0f742e2ab492103a85bbd9ddf4f5720f6de7064feb0d39ee002219765bb07bcfb8b877f47abe270ddeda4f676108cecb6b9bb2ad484a4f0011010001889f04180102000905024d3c5c10021b0c000a0910a34d7e18c20c31bb1a03040085c8d62e16d05dc4e9dad64953c8a2eed8b6c12f92b1575eeaa6dcf7be9473dd5b24b37b6dffbb4e7c99ed1bd3cb11634be19b3e6e207bed7505c7ca111ccf47cb323bf1f8851eb6360e8034cbff8dd149993c959de89f8f77f38e7e98b8e3076323aa719328e2b408db5ec0d03936efd57422ba04f925cdc7b4c1af7590e40ab0020003988d044d3c5c33010400b488c3e5f83f4d561f317817538d9d0397981e9aef1321ca68ebfae1cf8b7d388e19f4b5a24a82e2fbbf1c6c26557a6c5845307a03d815756f564ac7325b02bc83e87d5480a8fae848f07cb891f2d51ce7df83dcafdc12324517c86d472cc0ee10d47a68fd1d9ae49a6c19bbd36d82af597a0d88cc9c49de9df4e696fc1f0b5d0011010001b42754657374204b6579203220285253412c20656e637279707465642070726976617465206b65792988b804130102002205024d3c5c33021b03060b090807030206150802090a0b0416020301021e01021780000a0910d4984f961e35246b98940400908a73b6a6169f700434f076c6c79015a49bee37130eaf23aaa3cfa9ce60bfe4acaa7bc95f1146ada5867e0079babb38804891f4f0b8ebca57a86b249dee786161a755b7a342e68ccf3f78ed6440a93a6626beb9a37aa66afcd4f888790cb4bb46d94a4ae3eb3d7d3e6b00f6bfec940303e89ec5b32a1eaaacce66497d539328b0020003b88d044d3c5c33010400a4e913f9442abcc7f1804ccab27d2f787ffa592077ca935a8bb23165bd8d57576acac647cc596b2c3f814518cc8c82953c7a4478f32e0cf645630a5ba38d9618ef2bc3add69d459ae3dece5cab778938d988239f8c5ae437807075e06c828019959c644ff05ef6a5a1dab72227c98e3a040b0cf219026640698d7a13d8538a570011010001889f04180102000905024d3c5c33021b0c000a0910d4984f961e35246b26c703ff7ee29ef53bc1ae1ead533c408fa136db508434e233d6e62be621e031e5940bbd4c08142aed0f82217e7c3e1ec8de574bc06ccf3c36633be41ad78a9eacd209f861cae7b064100758545cc9dd83db71806dc1cfd5fb9ae5c7474bba0c19c44034ae61bae5eca379383339dece94ff56ff7aa44a582f3e5c38f45763af577c0934b0020003" + +const testKeys1And2PrivateHex = "9501d8044d3c5c10010400b1d13382944bd5aba23a4312968b5095d14f947f600eb478e14a6fcb16b0e0cac764884909c020bc495cfcc39a935387c661507bdb236a0612fb582cac3af9b29cc2c8c70090616c41b662f4da4c1201e195472eb7f4ae1ccbcbf9940fe21d985e379a5563dde5b9a23d35f1cfaa5790da3b79db26f23695107bfaca8e7b5bcd00110100010003ff4d91393b9a8e3430b14d6209df42f98dc927425b881f1209f319220841273a802a97c7bdb8b3a7740b3ab5866c4d1d308ad0d3a79bd1e883aacf1ac92dfe720285d10d08752a7efe3c609b1d00f17f2805b217be53999a7da7e493bfc3e9618fd17018991b8128aea70a05dbce30e4fbe626aa45775fa255dd9177aabf4df7cf0200c1ded12566e4bc2bb590455e5becfb2e2c9796482270a943343a7835de41080582c2be3caf5981aa838140e97afa40ad652a0b544f83eb1833b0957dce26e47b0200eacd6046741e9ce2ec5beb6fb5e6335457844fb09477f83b050a96be7da043e17f3a9523567ed40e7a521f818813a8b8a72209f1442844843ccc7eb9805442570200bdafe0438d97ac36e773c7162028d65844c4d463e2420aa2228c6e50dc2743c3d6c72d0d782a5173fe7be2169c8a9f4ef8a7cf3e37165e8c61b89c346cdc6c1799d2b41054657374204b6579203120285253412988b804130102002205024d3c5c10021b03060b090807030206150802090a0b0416020301021e01021780000a0910a34d7e18c20c31bbb5b304009cc45fe610b641a2c146331be94dade0a396e73ca725e1b25c21708d9cab46ecca5ccebc23055879df8f99eea39b377962a400f2ebdc36a7c99c333d74aeba346315137c3ff9d0a09b0273299090343048afb8107cf94cbd1400e3026f0ccac7ecebbc4d78588eb3e478fe2754d3ca664bcf3eac96ca4a6b0c8d7df5102f60f6b00200009d01d8044d3c5c10010400b201df61d67487301f11879d514f4248ade90c8f68c7af1284c161098de4c28c2850f1ec7b8e30f959793e571542ffc6532189409cb51c3d30dad78c4ad5165eda18b20d9826d8707d0f742e2ab492103a85bbd9ddf4f5720f6de7064feb0d39ee002219765bb07bcfb8b877f47abe270ddeda4f676108cecb6b9bb2ad484a4f00110100010003fd17a7490c22a79c59281fb7b20f5e6553ec0c1637ae382e8adaea295f50241037f8997cf42c1ce26417e015091451b15424b2c59eb8d4161b0975630408e394d3b00f88d4b4e18e2cc85e8251d4753a27c639c83f5ad4a571c4f19d7cd460b9b73c25ade730c99df09637bd173d8e3e981ac64432078263bb6dc30d3e974150dd0200d0ee05be3d4604d2146fb0457f31ba17c057560785aa804e8ca5530a7cd81d3440d0f4ba6851efcfd3954b7e68908fc0ba47f7ac37bf559c6c168b70d3a7c8cd0200da1c677c4bce06a068070f2b3733b0a714e88d62aa3f9a26c6f5216d48d5c2b5624144f3807c0df30be66b3268eeeca4df1fbded58faf49fc95dc3c35f134f8b01fd1396b6c0fc1b6c4f0eb8f5e44b8eace1e6073e20d0b8bc5385f86f1cf3f050f66af789f3ef1fc107b7f4421e19e0349c730c68f0a226981f4e889054fdb4dc149e8e889f04180102000905024d3c5c10021b0c000a0910a34d7e18c20c31bb1a03040085c8d62e16d05dc4e9dad64953c8a2eed8b6c12f92b1575eeaa6dcf7be9473dd5b24b37b6dffbb4e7c99ed1bd3cb11634be19b3e6e207bed7505c7ca111ccf47cb323bf1f8851eb6360e8034cbff8dd149993c959de89f8f77f38e7e98b8e3076323aa719328e2b408db5ec0d03936efd57422ba04f925cdc7b4c1af7590e40ab00200009501fe044d3c5c33010400b488c3e5f83f4d561f317817538d9d0397981e9aef1321ca68ebfae1cf8b7d388e19f4b5a24a82e2fbbf1c6c26557a6c5845307a03d815756f564ac7325b02bc83e87d5480a8fae848f07cb891f2d51ce7df83dcafdc12324517c86d472cc0ee10d47a68fd1d9ae49a6c19bbd36d82af597a0d88cc9c49de9df4e696fc1f0b5d0011010001fe030302e9030f3c783e14856063f16938530e148bc57a7aa3f3e4f90df9dceccdc779bc0835e1ad3d006e4a8d7b36d08b8e0de5a0d947254ecfbd22037e6572b426bcfdc517796b224b0036ff90bc574b5509bede85512f2eefb520fb4b02aa523ba739bff424a6fe81c5041f253f8d757e69a503d3563a104d0d49e9e890b9d0c26f96b55b743883b472caa7050c4acfd4a21f875bdf1258d88bd61224d303dc9df77f743137d51e6d5246b88c406780528fd9a3e15bab5452e5b93970d9dcc79f48b38651b9f15bfbcf6da452837e9cc70683d1bdca94507870f743e4ad902005812488dd342f836e72869afd00ce1850eea4cfa53ce10e3608e13d3c149394ee3cbd0e23d018fcbcb6e2ec5a1a22972d1d462ca05355d0d290dd2751e550d5efb38c6c89686344df64852bf4ff86638708f644e8ec6bd4af9b50d8541cb91891a431326ab2e332faa7ae86cfb6e0540aa63160c1e5cdd5a4add518b303fff0a20117c6bc77f7cfbaf36b04c865c6c2b42754657374204b6579203220285253412c20656e637279707465642070726976617465206b65792988b804130102002205024d3c5c33021b03060b090807030206150802090a0b0416020301021e01021780000a0910d4984f961e35246b98940400908a73b6a6169f700434f076c6c79015a49bee37130eaf23aaa3cfa9ce60bfe4acaa7bc95f1146ada5867e0079babb38804891f4f0b8ebca57a86b249dee786161a755b7a342e68ccf3f78ed6440a93a6626beb9a37aa66afcd4f888790cb4bb46d94a4ae3eb3d7d3e6b00f6bfec940303e89ec5b32a1eaaacce66497d539328b00200009d01fe044d3c5c33010400a4e913f9442abcc7f1804ccab27d2f787ffa592077ca935a8bb23165bd8d57576acac647cc596b2c3f814518cc8c82953c7a4478f32e0cf645630a5ba38d9618ef2bc3add69d459ae3dece5cab778938d988239f8c5ae437807075e06c828019959c644ff05ef6a5a1dab72227c98e3a040b0cf219026640698d7a13d8538a570011010001fe030302e9030f3c783e148560f936097339ae381d63116efcf802ff8b1c9360767db5219cc987375702a4123fd8657d3e22700f23f95020d1b261eda5257e9a72f9a918e8ef22dd5b3323ae03bbc1923dd224db988cadc16acc04b120a9f8b7e84da9716c53e0334d7b66586ddb9014df604b41be1e960dcfcbc96f4ed150a1a0dd070b9eb14276b9b6be413a769a75b519a53d3ecc0c220e85cd91ca354d57e7344517e64b43b6e29823cbd87eae26e2b2e78e6dedfbb76e3e9f77bcb844f9a8932eb3db2c3f9e44316e6f5d60e9e2a56e46b72abe6b06dc9a31cc63f10023d1f5e12d2a3ee93b675c96f504af0001220991c88db759e231b3320dcedf814dcf723fd9857e3d72d66a0f2af26950b915abdf56c1596f46a325bf17ad4810d3535fb02a259b247ac3dbd4cc3ecf9c51b6c07cebb009c1506fba0a89321ec8683e3fd009a6e551d50243e2d5092fefb3321083a4bad91320dc624bd6b5dddf93553e3d53924c05bfebec1fb4bd47e89a1a889f04180102000905024d3c5c33021b0c000a0910d4984f961e35246b26c703ff7ee29ef53bc1ae1ead533c408fa136db508434e233d6e62be621e031e5940bbd4c08142aed0f82217e7c3e1ec8de574bc06ccf3c36633be41ad78a9eacd209f861cae7b064100758545cc9dd83db71806dc1cfd5fb9ae5c7474bba0c19c44034ae61bae5eca379383339dece94ff56ff7aa44a582f3e5c38f45763af577c0934b0020000" + +const dsaElGamalTestKeysHex = "9501e1044dfcb16a110400aa3e5c1a1f43dd28c2ffae8abf5cfce555ee874134d8ba0a0f7b868ce2214beddc74e5e1e21ded354a95d18acdaf69e5e342371a71fbb9093162e0c5f3427de413a7f2c157d83f5cd2f9d791256dc4f6f0e13f13c3302af27f2384075ab3021dff7a050e14854bbde0a1094174855fc02f0bae8e00a340d94a1f22b32e48485700a0cec672ac21258fb95f61de2ce1af74b2c4fa3e6703ff698edc9be22c02ae4d916e4fa223f819d46582c0516235848a77b577ea49018dcd5e9e15cff9dbb4663a1ae6dd7580fa40946d40c05f72814b0f88481207e6c0832c3bded4853ebba0a7e3bd8e8c66df33d5a537cd4acf946d1080e7a3dcea679cb2b11a72a33a2b6a9dc85f466ad2ddf4c3db6283fa645343286971e3dd700703fc0c4e290d45767f370831a90187e74e9972aae5bff488eeff7d620af0362bfb95c1a6c3413ab5d15a2e4139e5d07a54d72583914661ed6a87cce810be28a0aa8879a2dd39e52fb6fe800f4f181ac7e328f740cde3d09a05cecf9483e4cca4253e60d4429ffd679d9996a520012aad119878c941e3cf151459873bdfc2a9563472fe0303027a728f9feb3b864260a1babe83925ce794710cfd642ee4ae0e5b9d74cee49e9c67b6cd0ea5dfbb582132195a121356a1513e1bca73e5b80c58c7ccb4164453412f456c47616d616c2054657374204b65792031886204131102002205024dfcb16a021b03060b090807030206150802090a0b0416020301021e01021780000a091033af447ccd759b09fadd00a0b8fd6f5a790bad7e9f2dbb7632046dc4493588db009c087c6a9ba9f7f49fab221587a74788c00db4889ab00200009d0157044dfcb16a1004008dec3f9291205255ccff8c532318133a6840739dd68b03ba942676f9038612071447bf07d00d559c5c0875724ea16a4c774f80d8338b55fca691a0522e530e604215b467bbc9ccfd483a1da99d7bc2648b4318fdbd27766fc8bfad3fddb37c62b8ae7ccfe9577e9b8d1e77c1d417ed2c2ef02d52f4da11600d85d3229607943700030503ff506c94c87c8cab778e963b76cf63770f0a79bf48fb49d3b4e52234620fc9f7657f9f8d56c96a2b7c7826ae6b57ebb2221a3fe154b03b6637cea7e6d98e3e45d87cf8dc432f723d3d71f89c5192ac8d7290684d2c25ce55846a80c9a7823f6acd9bb29fa6cd71f20bc90eccfca20451d0c976e460e672b000df49466408d527affe0303027a728f9feb3b864260abd761730327bca2aaa4ea0525c175e92bf240682a0e83b226f97ecb2e935b62c9a133858ce31b271fa8eb41f6a1b3cd72a63025ce1a75ee4180dcc284884904181102000905024dfcb16a021b0c000a091033af447ccd759b09dd0b009e3c3e7296092c81bee5a19929462caaf2fff3ae26009e218c437a2340e7ea628149af1ec98ec091a43992b00200009501e1044dfcb1be1104009f61faa61aa43df75d128cbe53de528c4aec49ce9360c992e70c77072ad5623de0a3a6212771b66b39a30dad6781799e92608316900518ec01184a85d872365b7d2ba4bacfb5882ea3c2473d3750dc6178cc1cf82147fb58caa28b28e9f12f6d1efcb0534abed644156c91cca4ab78834268495160b2400bc422beb37d237c2300a0cac94911b6d493bda1e1fbc6feeca7cb7421d34b03fe22cec6ccb39675bb7b94a335c2b7be888fd3906a1125f33301d8aa6ec6ee6878f46f73961c8d57a3e9544d8ef2a2cbfd4d52da665b1266928cfe4cb347a58c412815f3b2d2369dec04b41ac9a71cc9547426d5ab941cccf3b18575637ccfb42df1a802df3cfe0a999f9e7109331170e3a221991bf868543960f8c816c28097e503fe319db10fb98049f3a57d7c80c420da66d56f3644371631fad3f0ff4040a19a4fedc2d07727a1b27576f75a4d28c47d8246f27071e12d7a8de62aad216ddbae6aa02efd6b8a3e2818cda48526549791ab277e447b3a36c57cefe9b592f5eab73959743fcc8e83cbefec03a329b55018b53eec196765ae40ef9e20521a603c551efe0303020950d53a146bf9c66034d00c23130cce95576a2ff78016ca471276e8227fb30b1ffbd92e61804fb0c3eff9e30b1a826ee8f3e4730b4d86273ca977b4164453412f456c47616d616c2054657374204b65792032886204131102002205024dfcb1be021b03060b090807030206150802090a0b0416020301021e01021780000a0910a86bf526325b21b22bd9009e34511620415c974750a20df5cb56b182f3b48e6600a0a9466cb1a1305a84953445f77d461593f1d42bc1b00200009d0157044dfcb1be1004009565a951da1ee87119d600c077198f1c1bceb0f7aa54552489298e41ff788fa8f0d43a69871f0f6f77ebdfb14a4260cf9fbeb65d5844b4272a1904dd95136d06c3da745dc46327dd44a0f16f60135914368c8039a34033862261806bb2c5ce1152e2840254697872c85441ccb7321431d75a747a4bfb1d2c66362b51ce76311700030503fc0ea76601c196768070b7365a200e6ddb09307f262d5f39eec467b5f5784e22abdf1aa49226f59ab37cb49969d8f5230ea65caf56015abda62604544ed526c5c522bf92bed178a078789f6c807b6d34885688024a5bed9e9f8c58d11d4b82487b44c5f470c5606806a0443b79cadb45e0f897a561a53f724e5349b9267c75ca17fe0303020950d53a146bf9c660bc5f4ce8f072465e2d2466434320c1e712272fafc20e342fe7608101580fa1a1a367e60486a7cd1246b7ef5586cf5e10b32762b710a30144f12dd17dd4884904181102000905024dfcb1be021b0c000a0910a86bf526325b21b2904c00a0b2b66b4b39ccffda1d10f3ea8d58f827e30a8b8e009f4255b2d8112a184e40cde43a34e8655ca7809370b0020000" + +const signedMessageHex = "a3019bc0cbccc0c4b8d8b74ee2108fe16ec6d3ca490cbe362d3f8333d3f352531472538b8b13d353b97232f352158c20943157c71c16064626063656269052062e4e01987e9b6fccff4b7df3a34c534b23e679cbec3bc0f8f6e64dfb4b55fe3f8efa9ce110ddb5cd79faf1d753c51aecfa669f7e7aa043436596cccc3359cb7dd6bbe9ecaa69e5989d9e57209571edc0b2fa7f57b9b79a64ee6e99ce1371395fee92fec2796f7b15a77c386ff668ee27f6d38f0baa6c438b561657377bf6acff3c5947befd7bf4c196252f1d6e5c524d0300" + +const signedTextMessageHex = "a3019bc0cbccc8c4b8d8b74ee2108fe16ec6d36a250cbece0c178233d3f352531472538b8b13d35379b97232f352158ca0b4312f57c71c1646462606365626906a062e4e019811591798ff99bf8afee860b0d8a8c2a85c3387e3bcf0bb3b17987f2bbcfab2aa526d930cbfd3d98757184df3995c9f3e7790e36e3e9779f06089d4c64e9e47dd6202cb6e9bc73c5d11bb59fbaf89d22d8dc7cf199ddf17af96e77c5f65f9bbed56f427bd8db7af37f6c9984bf9385efaf5f184f986fb3e6adb0ecfe35bbf92d16a7aa2a344fb0bc52fb7624f0200" + +const signedEncryptedMessageHex = "848c032a67d68660df41c70103ff5789d0de26b6a50c985a02a13131ca829c413a35d0e6fa8d6842599252162808ac7439c72151c8c6183e76923fe3299301414d0c25a2f06a2257db3839e7df0ec964773f6e4c4ac7ff3b48c444237166dd46ba8ff443a5410dc670cb486672fdbe7c9dfafb75b4fea83af3a204fe2a7dfa86bd20122b4f3d2646cbeecb8f7be8d2c03b018bd210b1d3791e1aba74b0f1034e122ab72e760492c192383cf5e20b5628bd043272d63df9b923f147eb6091cd897553204832aba48fec54aa447547bb16305a1024713b90e77fd0065f1918271947549205af3c74891af22ee0b56cd29bfec6d6e351901cd4ab3ece7c486f1e32a792d4e474aed98ee84b3f591c7dff37b64e0ecd68fd036d517e412dcadf85840ce184ad7921ad446c4ee28db80447aea1ca8d4f574db4d4e37688158ddd19e14ee2eab4873d46947d65d14a23e788d912cf9a19624ca7352469b72a83866b7c23cb5ace3deab3c7018061b0ba0f39ed2befe27163e5083cf9b8271e3e3d52cc7ad6e2a3bd81d4c3d7022f8d" + +const signedEncryptedMessage2Hex = "85010e03cf6a7abcd43e36731003fb057f5495b79db367e277cdbe4ab90d924ddee0c0381494112ff8c1238fb0184af35d1731573b01bc4c55ecacd2aafbe2003d36310487d1ecc9ac994f3fada7f9f7f5c3a64248ab7782906c82c6ff1303b69a84d9a9529c31ecafbcdb9ba87e05439897d87e8a2a3dec55e14df19bba7f7bd316291c002ae2efd24f83f9e3441203fc081c0c23dc3092a454ca8a082b27f631abf73aca341686982e8fbda7e0e7d863941d68f3de4a755c2964407f4b5e0477b3196b8c93d551dd23c8beef7d0f03fbb1b6066f78907faf4bf1677d8fcec72651124080e0b7feae6b476e72ab207d38d90b958759fdedfc3c6c35717c9dbfc979b3cfbbff0a76d24a5e57056bb88acbd2a901ef64bc6e4db02adc05b6250ff378de81dca18c1910ab257dff1b9771b85bb9bbe0a69f5989e6d1710a35e6dfcceb7d8fb5ccea8db3932b3d9ff3fe0d327597c68b3622aec8e3716c83a6c93f497543b459b58ba504ed6bcaa747d37d2ca746fe49ae0a6ce4a8b694234e941b5159ff8bd34b9023da2814076163b86f40eed7c9472f81b551452d5ab87004a373c0172ec87ea6ce42ccfa7dbdad66b745496c4873d8019e8c28d6b3" + +const symmetricallyEncryptedCompressedHex = "8c0d04030302eb4a03808145d0d260c92f714339e13de5a79881216431925bf67ee2898ea61815f07894cd0703c50d0a76ef64d482196f47a8bc729af9b80bb6" + +const dsaTestKeyHex = "9901a2044d6c49de110400cb5ce438cf9250907ac2ba5bf6547931270b89f7c4b53d9d09f4d0213a5ef2ec1f26806d3d259960f872a4a102ef1581ea3f6d6882d15134f21ef6a84de933cc34c47cc9106efe3bd84c6aec12e78523661e29bc1a61f0aab17fa58a627fd5fd33f5149153fbe8cd70edf3d963bc287ef875270ff14b5bfdd1bca4483793923b00a0fe46d76cb6e4cbdc568435cd5480af3266d610d303fe33ae8273f30a96d4d34f42fa28ce1112d425b2e3bf7ea553d526e2db6b9255e9dc7419045ce817214d1a0056dbc8d5289956a4b1b69f20f1105124096e6a438f41f2e2495923b0f34b70642607d45559595c7fe94d7fa85fc41bf7d68c1fd509ebeaa5f315f6059a446b9369c277597e4f474a9591535354c7e7f4fd98a08aa60400b130c24ff20bdfbf683313f5daebf1c9b34b3bdadfc77f2ddd72ee1fb17e56c473664bc21d66467655dd74b9005e3a2bacce446f1920cd7017231ae447b67036c9b431b8179deacd5120262d894c26bc015bffe3d827ba7087ad9b700d2ca1f6d16cc1786581e5dd065f293c31209300f9b0afcc3f7c08dd26d0a22d87580b4db41054657374204b65792033202844534129886204131102002205024d6c49de021b03060b090807030206150802090a0b0416020301021e01021780000a0910338934250ccc03607e0400a0bdb9193e8a6b96fc2dfc108ae848914b504481f100a09c4dc148cb693293a67af24dd40d2b13a9e36794" + +const dsaTestKeyPrivateHex = "9501bb044d6c49de110400cb5ce438cf9250907ac2ba5bf6547931270b89f7c4b53d9d09f4d0213a5ef2ec1f26806d3d259960f872a4a102ef1581ea3f6d6882d15134f21ef6a84de933cc34c47cc9106efe3bd84c6aec12e78523661e29bc1a61f0aab17fa58a627fd5fd33f5149153fbe8cd70edf3d963bc287ef875270ff14b5bfdd1bca4483793923b00a0fe46d76cb6e4cbdc568435cd5480af3266d610d303fe33ae8273f30a96d4d34f42fa28ce1112d425b2e3bf7ea553d526e2db6b9255e9dc7419045ce817214d1a0056dbc8d5289956a4b1b69f20f1105124096e6a438f41f2e2495923b0f34b70642607d45559595c7fe94d7fa85fc41bf7d68c1fd509ebeaa5f315f6059a446b9369c277597e4f474a9591535354c7e7f4fd98a08aa60400b130c24ff20bdfbf683313f5daebf1c9b34b3bdadfc77f2ddd72ee1fb17e56c473664bc21d66467655dd74b9005e3a2bacce446f1920cd7017231ae447b67036c9b431b8179deacd5120262d894c26bc015bffe3d827ba7087ad9b700d2ca1f6d16cc1786581e5dd065f293c31209300f9b0afcc3f7c08dd26d0a22d87580b4d00009f592e0619d823953577d4503061706843317e4fee083db41054657374204b65792033202844534129886204131102002205024d6c49de021b03060b090807030206150802090a0b0416020301021e01021780000a0910338934250ccc03607e0400a0bdb9193e8a6b96fc2dfc108ae848914b504481f100a09c4dc148cb693293a67af24dd40d2b13a9e36794" + +const armoredPrivateKeyBlock = `-----BEGIN PGP PRIVATE KEY BLOCK----- +Version: GnuPG v1.4.10 (GNU/Linux) + +lQHYBE2rFNoBBADFwqWQIW/DSqcB4yCQqnAFTJ27qS5AnB46ccAdw3u4Greeu3Bp +idpoHdjULy7zSKlwR1EA873dO/k/e11Ml3dlAFUinWeejWaK2ugFP6JjiieSsrKn +vWNicdCS4HTWn0X4sjl0ZiAygw6GNhqEQ3cpLeL0g8E9hnYzJKQ0LWJa0QARAQAB +AAP/TB81EIo2VYNmTq0pK1ZXwUpxCrvAAIG3hwKjEzHcbQznsjNvPUihZ+NZQ6+X +0HCfPAdPkGDCLCb6NavcSW+iNnLTrdDnSI6+3BbIONqWWdRDYJhqZCkqmG6zqSfL +IdkJgCw94taUg5BWP/AAeQrhzjChvpMQTVKQL5mnuZbUCeMCAN5qrYMP2S9iKdnk +VANIFj7656ARKt/nf4CBzxcpHTyB8+d2CtPDKCmlJP6vL8t58Jmih+kHJMvC0dzn +gr5f5+sCAOOe5gt9e0am7AvQWhdbHVfJU0TQJx+m2OiCJAqGTB1nvtBLHdJnfdC9 +TnXXQ6ZXibqLyBies/xeY2sCKL5qtTMCAKnX9+9d/5yQxRyrQUHt1NYhaXZnJbHx +q4ytu0eWz+5i68IYUSK69jJ1NWPM0T6SkqpB3KCAIv68VFm9PxqG1KmhSrQIVGVz +dCBLZXmIuAQTAQIAIgUCTasU2gIbAwYLCQgHAwIGFQgCCQoLBBYCAwECHgECF4AA +CgkQO9o98PRieSoLhgQAkLEZex02Qt7vGhZzMwuN0R22w3VwyYyjBx+fM3JFETy1 +ut4xcLJoJfIaF5ZS38UplgakHG0FQ+b49i8dMij0aZmDqGxrew1m4kBfjXw9B/v+ +eIqpODryb6cOSwyQFH0lQkXC040pjq9YqDsO5w0WYNXYKDnzRV0p4H1pweo2VDid +AdgETasU2gEEAN46UPeWRqKHvA99arOxee38fBt2CI08iiWyI8T3J6ivtFGixSqV +bRcPxYO/qLpVe5l84Nb3X71GfVXlc9hyv7CD6tcowL59hg1E/DC5ydI8K8iEpUmK +/UnHdIY5h8/kqgGxkY/T/hgp5fRQgW1ZoZxLajVlMRZ8W4tFtT0DeA+JABEBAAEA +A/0bE1jaaZKj6ndqcw86jd+QtD1SF+Cf21CWRNeLKnUds4FRRvclzTyUMuWPkUeX +TaNNsUOFqBsf6QQ2oHUBBK4VCHffHCW4ZEX2cd6umz7mpHW6XzN4DECEzOVksXtc +lUC1j4UB91DC/RNQqwX1IV2QLSwssVotPMPqhOi0ZLNY7wIA3n7DWKInxYZZ4K+6 +rQ+POsz6brEoRHwr8x6XlHenq1Oki855pSa1yXIARoTrSJkBtn5oI+f8AzrnN0BN +oyeQAwIA/7E++3HDi5aweWrViiul9cd3rcsS0dEnksPhvS0ozCJiHsq/6GFmy7J8 +QSHZPteedBnZyNp5jR+H7cIfVN3KgwH/Skq4PsuPhDq5TKK6i8Pc1WW8MA6DXTdU +nLkX7RGmMwjC0DBf7KWAlPjFaONAX3a8ndnz//fy1q7u2l9AZwrj1qa1iJ8EGAEC +AAkFAk2rFNoCGwwACgkQO9o98PRieSo2/QP/WTzr4ioINVsvN1akKuekmEMI3LAp +BfHwatufxxP1U+3Si/6YIk7kuPB9Hs+pRqCXzbvPRrI8NHZBmc8qIGthishdCYad +AHcVnXjtxrULkQFGbGvhKURLvS9WnzD/m1K2zzwxzkPTzT9/Yf06O6Mal5AdugPL +VrM0m72/jnpKo04= +=zNCn +-----END PGP PRIVATE KEY BLOCK-----` diff --git a/src/pkg/crypto/openpgp/s2k/Makefile b/src/pkg/crypto/openpgp/s2k/Makefile new file mode 100644 index 000000000..731d53431 --- /dev/null +++ b/src/pkg/crypto/openpgp/s2k/Makefile @@ -0,0 +1,11 @@ +# Copyright 2011 The Go Authors. All rights reserved. +# Use of this source code is governed by a BSD-style +# license that can be found in the LICENSE file. + +include ../../../../Make.inc + +TARG=crypto/openpgp/s2k +GOFILES=\ + s2k.go\ + +include ../../../../Make.pkg diff --git a/src/pkg/crypto/openpgp/s2k/s2k.go b/src/pkg/crypto/openpgp/s2k/s2k.go new file mode 100644 index 000000000..da926a76e --- /dev/null +++ b/src/pkg/crypto/openpgp/s2k/s2k.go @@ -0,0 +1,180 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Package s2k implements the various OpenPGP string-to-key transforms as +// specified in RFC 4800 section 3.7.1. +package s2k + +import ( + "crypto" + "crypto/openpgp/error" + "hash" + "io" + "os" + "strconv" +) + +// Simple writes to out the result of computing the Simple S2K function (RFC +// 4880, section 3.7.1.1) using the given hash and input passphrase. +func Simple(out []byte, h hash.Hash, in []byte) { + Salted(out, h, in, nil) +} + +var zero [1]byte + +// Salted writes to out the result of computing the Salted S2K function (RFC +// 4880, section 3.7.1.2) using the given hash, input passphrase and salt. +func Salted(out []byte, h hash.Hash, in []byte, salt []byte) { + done := 0 + + for i := 0; done < len(out); i++ { + h.Reset() + for j := 0; j < i; j++ { + h.Write(zero[:]) + } + h.Write(salt) + h.Write(in) + n := copy(out[done:], h.Sum()) + done += n + } +} + +// Iterated writes to out the result of computing the Iterated and Salted S2K +// function (RFC 4880, section 3.7.1.3) using the given hash, input passphrase, +// salt and iteration count. +func Iterated(out []byte, h hash.Hash, in []byte, salt []byte, count int) { + combined := make([]byte, len(in)+len(salt)) + copy(combined, salt) + copy(combined[len(salt):], in) + + if count < len(combined) { + count = len(combined) + } + + done := 0 + for i := 0; done < len(out); i++ { + h.Reset() + for j := 0; j < i; j++ { + h.Write(zero[:]) + } + written := 0 + for written < count { + if written+len(combined) > count { + todo := count - written + h.Write(combined[:todo]) + written = count + } else { + h.Write(combined) + written += len(combined) + } + } + n := copy(out[done:], h.Sum()) + done += n + } +} + +// Parse reads a binary specification for a string-to-key transformation from r +// and returns a function which performs that transform. +func Parse(r io.Reader) (f func(out, in []byte), err os.Error) { + var buf [9]byte + + _, err = io.ReadFull(r, buf[:2]) + if err != nil { + return + } + + hash, ok := HashIdToHash(buf[1]) + if !ok { + return nil, error.UnsupportedError("hash for S2K function: " + strconv.Itoa(int(buf[1]))) + } + h := hash.New() + if h == nil { + return nil, error.UnsupportedError("hash not available: " + strconv.Itoa(int(hash))) + } + + switch buf[0] { + case 1: + f := func(out, in []byte) { + Simple(out, h, in) + } + return f, nil + case 2: + _, err := io.ReadFull(r, buf[:8]) + if err != nil { + return + } + f := func(out, in []byte) { + Salted(out, h, in, buf[:8]) + } + return f, nil + case 3: + _, err := io.ReadFull(r, buf[:9]) + if err != nil { + return + } + count := (16 + int(buf[8]&15)) << (uint32(buf[8]>>4) + 6) + f := func(out, in []byte) { + Iterated(out, h, in, buf[:8], count) + } + return f, nil + } + + return nil, error.UnsupportedError("S2K function") +} + +// Serialize salts and stretches the given passphrase and writes the resulting +// key into key. It also serializes an S2K descriptor to w. +func Serialize(w io.Writer, key []byte, rand io.Reader, passphrase []byte) os.Error { + var buf [11]byte + buf[0] = 3 /* iterated and salted */ + buf[1], _ = HashToHashId(crypto.SHA1) + salt := buf[2:10] + if _, err := io.ReadFull(rand, salt); err != nil { + return err + } + const count = 65536 // this is the default in gpg + buf[10] = 96 // 65536 iterations + if _, err := w.Write(buf[:]); err != nil { + return err + } + + Iterated(key, crypto.SHA1.New(), passphrase, salt, count) + return nil +} + +// hashToHashIdMapping contains pairs relating OpenPGP's hash identifier with +// Go's crypto.Hash type. See RFC 4880, section 9.4. +var hashToHashIdMapping = []struct { + id byte + hash crypto.Hash +}{ + {1, crypto.MD5}, + {2, crypto.SHA1}, + {3, crypto.RIPEMD160}, + {8, crypto.SHA256}, + {9, crypto.SHA384}, + {10, crypto.SHA512}, + {11, crypto.SHA224}, +} + +// HashIdToHash returns a crypto.Hash which corresponds to the given OpenPGP +// hash id. +func HashIdToHash(id byte) (h crypto.Hash, ok bool) { + for _, m := range hashToHashIdMapping { + if m.id == id { + return m.hash, true + } + } + return 0, false +} + +// HashIdToHash returns an OpenPGP hash id which corresponds the given Hash. +func HashToHashId(h crypto.Hash) (id byte, ok bool) { + for _, m := range hashToHashIdMapping { + if m.hash == h { + return m.id, true + } + } + return 0, false +} diff --git a/src/pkg/crypto/openpgp/s2k/s2k_test.go b/src/pkg/crypto/openpgp/s2k/s2k_test.go new file mode 100644 index 000000000..ec4012c23 --- /dev/null +++ b/src/pkg/crypto/openpgp/s2k/s2k_test.go @@ -0,0 +1,118 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package s2k + +import ( + "bytes" + "crypto/sha1" + "crypto/rand" + "encoding/hex" + "testing" +) + +var saltedTests = []struct { + in, out string +}{ + {"hello", "10295ac1"}, + {"world", "ac587a5e"}, + {"foo", "4dda8077"}, + {"bar", "bd8aac6b9ea9cae04eae6a91c6133b58b5d9a61c14f355516ed9370456"}, + {"x", "f1d3f289"}, + {"xxxxxxxxxxxxxxxxxxxxxxx", "e00d7b45"}, +} + +func TestSalted(t *testing.T) { + h := sha1.New() + salt := [4]byte{1, 2, 3, 4} + + for i, test := range saltedTests { + expected, _ := hex.DecodeString(test.out) + out := make([]byte, len(expected)) + Salted(out, h, []byte(test.in), salt[:]) + if !bytes.Equal(expected, out) { + t.Errorf("#%d, got: %x want: %x", i, out, expected) + } + } +} + +var iteratedTests = []struct { + in, out string +}{ + {"hello", "83126105"}, + {"world", "6fa317f9"}, + {"foo", "8fbc35b9"}, + {"bar", "2af5a99b54f093789fd657f19bd245af7604d0f6ae06f66602a46a08ae"}, + {"x", "5a684dfe"}, + {"xxxxxxxxxxxxxxxxxxxxxxx", "18955174"}, +} + +func TestIterated(t *testing.T) { + h := sha1.New() + salt := [4]byte{4, 3, 2, 1} + + for i, test := range iteratedTests { + expected, _ := hex.DecodeString(test.out) + out := make([]byte, len(expected)) + Iterated(out, h, []byte(test.in), salt[:], 31) + if !bytes.Equal(expected, out) { + t.Errorf("#%d, got: %x want: %x", i, out, expected) + } + } +} + +var parseTests = []struct { + spec, in, out string +}{ + /* Simple with SHA1 */ + {"0102", "hello", "aaf4c61d"}, + /* Salted with SHA1 */ + {"02020102030405060708", "hello", "f4f7d67e"}, + /* Iterated with SHA1 */ + {"03020102030405060708f1", "hello", "f2a57b7c"}, +} + +func TestParse(t *testing.T) { + for i, test := range parseTests { + spec, _ := hex.DecodeString(test.spec) + buf := bytes.NewBuffer(spec) + f, err := Parse(buf) + if err != nil { + t.Errorf("%d: Parse returned error: %s", i, err) + continue + } + + expected, _ := hex.DecodeString(test.out) + out := make([]byte, len(expected)) + f(out, []byte(test.in)) + if !bytes.Equal(out, expected) { + t.Errorf("%d: output got: %x want: %x", i, out, expected) + } + if testing.Short() { + break + } + } +} + +func TestSerialize(t *testing.T) { + buf := bytes.NewBuffer(nil) + key := make([]byte, 16) + passphrase := []byte("testing") + err := Serialize(buf, key, rand.Reader, passphrase) + if err != nil { + t.Errorf("failed to serialize: %s", err) + return + } + + f, err := Parse(buf) + if err != nil { + t.Errorf("failed to reparse: %s", err) + return + } + key2 := make([]byte, len(key)) + f(key2, passphrase) + if !bytes.Equal(key2, key) { + t.Errorf("keys don't match: %x (serialied) vs %x (parsed)", key, key2) + } +} diff --git a/src/pkg/crypto/openpgp/write.go b/src/pkg/crypto/openpgp/write.go new file mode 100644 index 000000000..9884472ce --- /dev/null +++ b/src/pkg/crypto/openpgp/write.go @@ -0,0 +1,308 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package openpgp + +import ( + "crypto" + "crypto/openpgp/armor" + "crypto/openpgp/error" + "crypto/openpgp/packet" + "crypto/openpgp/s2k" + "crypto/rand" + _ "crypto/sha256" + "hash" + "io" + "os" + "strconv" + "time" +) + +// DetachSign signs message with the private key from signer (which must +// already have been decrypted) and writes the signature to w. +func DetachSign(w io.Writer, signer *Entity, message io.Reader) os.Error { + return detachSign(w, signer, message, packet.SigTypeBinary) +} + +// ArmoredDetachSign signs message with the private key from signer (which +// must already have been decrypted) and writes an armored signature to w. +func ArmoredDetachSign(w io.Writer, signer *Entity, message io.Reader) (err os.Error) { + return armoredDetachSign(w, signer, message, packet.SigTypeBinary) +} + +// DetachSignText signs message (after canonicalising the line endings) with +// the private key from signer (which must already have been decrypted) and +// writes the signature to w. +func DetachSignText(w io.Writer, signer *Entity, message io.Reader) os.Error { + return detachSign(w, signer, message, packet.SigTypeText) +} + +// ArmoredDetachSignText signs message (after canonicalising the line endings) +// with the private key from signer (which must already have been decrypted) +// and writes an armored signature to w. +func ArmoredDetachSignText(w io.Writer, signer *Entity, message io.Reader) os.Error { + return armoredDetachSign(w, signer, message, packet.SigTypeText) +} + +func armoredDetachSign(w io.Writer, signer *Entity, message io.Reader, sigType packet.SignatureType) (err os.Error) { + out, err := armor.Encode(w, SignatureType, nil) + if err != nil { + return + } + err = detachSign(out, signer, message, sigType) + if err != nil { + return + } + return out.Close() +} + +func detachSign(w io.Writer, signer *Entity, message io.Reader, sigType packet.SignatureType) (err os.Error) { + if signer.PrivateKey == nil { + return error.InvalidArgumentError("signing key doesn't have a private key") + } + if signer.PrivateKey.Encrypted { + return error.InvalidArgumentError("signing key is encrypted") + } + + sig := new(packet.Signature) + sig.SigType = sigType + sig.PubKeyAlgo = signer.PrivateKey.PubKeyAlgo + sig.Hash = crypto.SHA256 + sig.CreationTime = uint32(time.Seconds()) + sig.IssuerKeyId = &signer.PrivateKey.KeyId + + h, wrappedHash, err := hashForSignature(sig.Hash, sig.SigType) + if err != nil { + return + } + io.Copy(wrappedHash, message) + + err = sig.Sign(h, signer.PrivateKey) + if err != nil { + return + } + + return sig.Serialize(w) +} + +// FileHints contains metadata about encrypted files. This metadata is, itself, +// encrypted. +type FileHints struct { + // IsBinary can be set to hint that the contents are binary data. + IsBinary bool + // FileName hints at the name of the file that should be written. It's + // truncated to 255 bytes if longer. It may be empty to suggest that the + // file should not be written to disk. It may be equal to "_CONSOLE" to + // suggest the data should not be written to disk. + FileName string + // EpochSeconds contains the modification time of the file, or 0 if not applicable. + EpochSeconds uint32 +} + +// SymmetricallyEncrypt acts like gpg -c: it encrypts a file with a passphrase. +// The resulting WriteCloser must be closed after the contents of the file have +// been written. +func SymmetricallyEncrypt(ciphertext io.Writer, passphrase []byte, hints *FileHints) (plaintext io.WriteCloser, err os.Error) { + if hints == nil { + hints = &FileHints{} + } + + key, err := packet.SerializeSymmetricKeyEncrypted(ciphertext, rand.Reader, passphrase, packet.CipherAES128) + if err != nil { + return + } + w, err := packet.SerializeSymmetricallyEncrypted(ciphertext, packet.CipherAES128, key) + if err != nil { + return + } + return packet.SerializeLiteral(w, hints.IsBinary, hints.FileName, hints.EpochSeconds) +} + +// intersectPreferences mutates and returns a prefix of a that contains only +// the values in the intersection of a and b. The order of a is preserved. +func intersectPreferences(a []uint8, b []uint8) (intersection []uint8) { + var j int + for _, v := range a { + for _, v2 := range b { + if v == v2 { + a[j] = v + j++ + break + } + } + } + + return a[:j] +} + +func hashToHashId(h crypto.Hash) uint8 { + v, ok := s2k.HashToHashId(h) + if !ok { + panic("tried to convert unknown hash") + } + return v +} + +// Encrypt encrypts a message to a number of recipients and, optionally, signs +// it. hints contains optional information, that is also encrypted, that aids +// the recipients in processing the message. The resulting WriteCloser must +// be closed after the contents of the file have been written. +func Encrypt(ciphertext io.Writer, to []*Entity, signed *Entity, hints *FileHints) (plaintext io.WriteCloser, err os.Error) { + var signer *packet.PrivateKey + if signed != nil { + signer = signed.signingKey().PrivateKey + if signer == nil || signer.Encrypted { + return nil, error.InvalidArgumentError("signing key must be decrypted") + } + } + + // These are the possible ciphers that we'll use for the message. + candidateCiphers := []uint8{ + uint8(packet.CipherAES128), + uint8(packet.CipherAES256), + uint8(packet.CipherCAST5), + } + // These are the possible hash functions that we'll use for the signature. + candidateHashes := []uint8{ + hashToHashId(crypto.SHA256), + hashToHashId(crypto.SHA512), + hashToHashId(crypto.SHA1), + hashToHashId(crypto.RIPEMD160), + } + // In the event that a recipient doesn't specify any supported ciphers + // or hash functions, these are the ones that we assume that every + // implementation supports. + defaultCiphers := candidateCiphers[len(candidateCiphers)-1:] + defaultHashes := candidateHashes[len(candidateHashes)-1:] + + encryptKeys := make([]Key, len(to)) + for i := range to { + encryptKeys[i] = to[i].encryptionKey() + if encryptKeys[i].PublicKey == nil { + return nil, error.InvalidArgumentError("cannot encrypt a message to key id " + strconv.Uitob64(to[i].PrimaryKey.KeyId, 16) + " because it has no encryption keys") + } + + sig := to[i].primaryIdentity().SelfSignature + + preferredSymmetric := sig.PreferredSymmetric + if len(preferredSymmetric) == 0 { + preferredSymmetric = defaultCiphers + } + preferredHashes := sig.PreferredHash + if len(preferredHashes) == 0 { + preferredHashes = defaultHashes + } + candidateCiphers = intersectPreferences(candidateCiphers, preferredSymmetric) + candidateHashes = intersectPreferences(candidateHashes, preferredHashes) + } + + if len(candidateCiphers) == 0 || len(candidateHashes) == 0 { + return nil, error.InvalidArgumentError("cannot encrypt because recipient set shares no common algorithms") + } + + cipher := packet.CipherFunction(candidateCiphers[0]) + hash, _ := s2k.HashIdToHash(candidateHashes[0]) + symKey := make([]byte, cipher.KeySize()) + if _, err := io.ReadFull(rand.Reader, symKey); err != nil { + return nil, err + } + + for _, key := range encryptKeys { + if err := packet.SerializeEncryptedKey(ciphertext, rand.Reader, key.PublicKey, cipher, symKey); err != nil { + return nil, err + } + } + + encryptedData, err := packet.SerializeSymmetricallyEncrypted(ciphertext, cipher, symKey) + if err != nil { + return + } + + if signer != nil { + ops := &packet.OnePassSignature{ + SigType: packet.SigTypeBinary, + Hash: hash, + PubKeyAlgo: signer.PubKeyAlgo, + KeyId: signer.KeyId, + IsLast: true, + } + if err := ops.Serialize(encryptedData); err != nil { + return nil, err + } + } + + if hints == nil { + hints = &FileHints{} + } + + w := encryptedData + if signer != nil { + // If we need to write a signature packet after the literal + // data then we need to stop literalData from closing + // encryptedData. + w = noOpCloser{encryptedData} + + } + literalData, err := packet.SerializeLiteral(w, hints.IsBinary, hints.FileName, hints.EpochSeconds) + if err != nil { + return nil, err + } + + if signer != nil { + return signatureWriter{encryptedData, literalData, hash, hash.New(), signer}, nil + } + return literalData, nil +} + +// signatureWriter hashes the contents of a message while passing it along to +// literalData. When closed, it closes literalData, writes a signature packet +// to encryptedData and then also closes encryptedData. +type signatureWriter struct { + encryptedData io.WriteCloser + literalData io.WriteCloser + hashType crypto.Hash + h hash.Hash + signer *packet.PrivateKey +} + +func (s signatureWriter) Write(data []byte) (int, os.Error) { + s.h.Write(data) + return s.literalData.Write(data) +} + +func (s signatureWriter) Close() os.Error { + sig := &packet.Signature{ + SigType: packet.SigTypeBinary, + PubKeyAlgo: s.signer.PubKeyAlgo, + Hash: s.hashType, + CreationTime: uint32(time.Seconds()), + IssuerKeyId: &s.signer.KeyId, + } + + if err := sig.Sign(s.h, s.signer); err != nil { + return err + } + if err := s.literalData.Close(); err != nil { + return err + } + if err := sig.Serialize(s.encryptedData); err != nil { + return err + } + return s.encryptedData.Close() +} + +// noOpCloser is like an ioutil.NopCloser, but for an io.Writer. +// TODO: we have two of these in OpenPGP packages alone. This probably needs +// to be promoted somewhere more common. +type noOpCloser struct { + w io.Writer +} + +func (c noOpCloser) Write(data []byte) (n int, err os.Error) { + return c.w.Write(data) +} + +func (c noOpCloser) Close() os.Error { + return nil +} diff --git a/src/pkg/crypto/openpgp/write_test.go b/src/pkg/crypto/openpgp/write_test.go new file mode 100644 index 000000000..c542dfa45 --- /dev/null +++ b/src/pkg/crypto/openpgp/write_test.go @@ -0,0 +1,233 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package openpgp + +import ( + "bytes" + "crypto/rand" + "os" + "io" + "io/ioutil" + "testing" + "time" +) + +func TestSignDetached(t *testing.T) { + kring, _ := ReadKeyRing(readerFromHex(testKeys1And2PrivateHex)) + out := bytes.NewBuffer(nil) + message := bytes.NewBufferString(signedInput) + err := DetachSign(out, kring[0], message) + if err != nil { + t.Error(err) + } + + testDetachedSignature(t, kring, out, signedInput, "check", testKey1KeyId) +} + +func TestSignTextDetached(t *testing.T) { + kring, _ := ReadKeyRing(readerFromHex(testKeys1And2PrivateHex)) + out := bytes.NewBuffer(nil) + message := bytes.NewBufferString(signedInput) + err := DetachSignText(out, kring[0], message) + if err != nil { + t.Error(err) + } + + testDetachedSignature(t, kring, out, signedInput, "check", testKey1KeyId) +} + +func TestSignDetachedDSA(t *testing.T) { + kring, _ := ReadKeyRing(readerFromHex(dsaTestKeyPrivateHex)) + out := bytes.NewBuffer(nil) + message := bytes.NewBufferString(signedInput) + err := DetachSign(out, kring[0], message) + if err != nil { + t.Error(err) + } + + testDetachedSignature(t, kring, out, signedInput, "check", testKey3KeyId) +} + +func TestNewEntity(t *testing.T) { + if testing.Short() { + return + } + + e, err := NewEntity(rand.Reader, time.Seconds(), "Test User", "test", "test@example.com") + if err != nil { + t.Errorf("failed to create entity: %s", err) + return + } + + w := bytes.NewBuffer(nil) + if err := e.SerializePrivate(w); err != nil { + t.Errorf("failed to serialize entity: %s", err) + return + } + serialized := w.Bytes() + + el, err := ReadKeyRing(w) + if err != nil { + t.Errorf("failed to reparse entity: %s", err) + return + } + + if len(el) != 1 { + t.Errorf("wrong number of entities found, got %d, want 1", len(el)) + } + + w = bytes.NewBuffer(nil) + if err := e.SerializePrivate(w); err != nil { + t.Errorf("failed to serialize entity second time: %s", err) + return + } + + if !bytes.Equal(w.Bytes(), serialized) { + t.Errorf("results differed") + } +} + +func TestSymmetricEncryption(t *testing.T) { + buf := new(bytes.Buffer) + plaintext, err := SymmetricallyEncrypt(buf, []byte("testing"), nil) + if err != nil { + t.Errorf("error writing headers: %s", err) + return + } + message := []byte("hello world\n") + _, err = plaintext.Write(message) + if err != nil { + t.Errorf("error writing to plaintext writer: %s", err) + } + err = plaintext.Close() + if err != nil { + t.Errorf("error closing plaintext writer: %s", err) + } + + md, err := ReadMessage(buf, nil, func(keys []Key, symmetric bool) ([]byte, os.Error) { + return []byte("testing"), nil + }) + if err != nil { + t.Errorf("error rereading message: %s", err) + } + messageBuf := bytes.NewBuffer(nil) + _, err = io.Copy(messageBuf, md.UnverifiedBody) + if err != nil { + t.Errorf("error rereading message: %s", err) + } + if !bytes.Equal(message, messageBuf.Bytes()) { + t.Errorf("recovered message incorrect got '%s', want '%s'", messageBuf.Bytes(), message) + } +} + +var testEncryptionTests = []struct { + keyRingHex string + isSigned bool +}{ + { + testKeys1And2PrivateHex, + false, + }, + { + testKeys1And2PrivateHex, + true, + }, + { + dsaElGamalTestKeysHex, + false, + }, + { + dsaElGamalTestKeysHex, + true, + }, +} + +func TestEncryption(t *testing.T) { + for i, test := range testEncryptionTests { + kring, _ := ReadKeyRing(readerFromHex(test.keyRingHex)) + + passphrase := []byte("passphrase") + for _, entity := range kring { + if entity.PrivateKey != nil && entity.PrivateKey.Encrypted { + err := entity.PrivateKey.Decrypt(passphrase) + if err != nil { + t.Errorf("#%d: failed to decrypt key", i) + } + } + for _, subkey := range entity.Subkeys { + if subkey.PrivateKey != nil && subkey.PrivateKey.Encrypted { + err := subkey.PrivateKey.Decrypt(passphrase) + if err != nil { + t.Errorf("#%d: failed to decrypt subkey", i) + } + } + } + } + + var signed *Entity + if test.isSigned { + signed = kring[0] + } + + buf := new(bytes.Buffer) + w, err := Encrypt(buf, kring[:1], signed, nil /* no hints */ ) + if err != nil { + t.Errorf("#%d: error in Encrypt: %s", i, err) + continue + } + + const message = "testing" + _, err = w.Write([]byte(message)) + if err != nil { + t.Errorf("#%d: error writing plaintext: %s", i, err) + continue + } + err = w.Close() + if err != nil { + t.Errorf("#%d: error closing WriteCloser: %s", i, err) + continue + } + + md, err := ReadMessage(buf, kring, nil /* no prompt */ ) + if err != nil { + t.Errorf("#%d: error reading message: %s", i, err) + continue + } + + if test.isSigned { + expectedKeyId := kring[0].signingKey().PublicKey.KeyId + if md.SignedByKeyId != expectedKeyId { + t.Errorf("#%d: message signed by wrong key id, got: %d, want: %d", i, *md.SignedBy, expectedKeyId) + } + if md.SignedBy == nil { + t.Errorf("#%d: failed to find the signing Entity", i) + } + } + + plaintext, err := ioutil.ReadAll(md.UnverifiedBody) + if err != nil { + t.Errorf("#%d: error reading encrypted contents: %s", i, err) + continue + } + + expectedKeyId := kring[0].encryptionKey().PublicKey.KeyId + if len(md.EncryptedToKeyIds) != 1 || md.EncryptedToKeyIds[0] != expectedKeyId { + t.Errorf("#%d: expected message to be encrypted to %v, but got %#v", i, expectedKeyId, md.EncryptedToKeyIds) + } + + if string(plaintext) != message { + t.Errorf("#%d: got: %s, want: %s", i, string(plaintext), message) + } + + if test.isSigned { + if md.SignatureError != nil { + t.Errorf("#%d: signature error: %s", i, err) + } + if md.Signature == nil { + t.Error("signature missing") + } + } + } +} diff --git a/src/pkg/crypto/rand/Makefile b/src/pkg/crypto/rand/Makefile new file mode 100644 index 000000000..d1a3d45e8 --- /dev/null +++ b/src/pkg/crypto/rand/Makefile @@ -0,0 +1,30 @@ +# Copyright 2010 The Go Authors. All rights reserved. +# Use of this source code is governed by a BSD-style +# license that can be found in the LICENSE file. + +include ../../../Make.inc + +TARG=crypto/rand + +GOFILES=\ + rand.go\ + util.go\ + +GOFILES_freebsd=\ + rand_unix.go\ + +GOFILES_darwin=\ + rand_unix.go\ + +GOFILES_linux=\ + rand_unix.go\ + +GOFILES_openbsd=\ + rand_unix.go\ + +GOFILES_windows=\ + rand_windows.go\ + +GOFILES+=$(GOFILES_$(GOOS)) + +include ../../../Make.pkg diff --git a/src/pkg/crypto/rand/rand.go b/src/pkg/crypto/rand/rand.go new file mode 100644 index 000000000..42d9da0ef --- /dev/null +++ b/src/pkg/crypto/rand/rand.go @@ -0,0 +1,21 @@ +// Copyright 2010 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Package rand implements a cryptographically secure +// pseudorandom number generator. +package rand + +import ( + "io" + "os" +) + +// Reader is a global, shared instance of a cryptographically +// strong pseudo-random generator. +// On Unix-like systems, Reader reads from /dev/urandom. +// On Windows systems, Reader uses the CryptGenRandom API. +var Reader io.Reader + +// Read is a helper function that calls Reader.Read. +func Read(b []byte) (n int, err os.Error) { return Reader.Read(b) } diff --git a/src/pkg/crypto/rand/rand_test.go b/src/pkg/crypto/rand/rand_test.go new file mode 100644 index 000000000..bfae7ce4f --- /dev/null +++ b/src/pkg/crypto/rand/rand_test.go @@ -0,0 +1,31 @@ +// Copyright 2010 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package rand + +import ( + "bytes" + "compress/flate" + "testing" +) + +func TestRead(t *testing.T) { + var n int = 4e6 + if testing.Short() { + n = 1e5 + } + b := make([]byte, n) + n, err := Read(b) + if n != len(b) || err != nil { + t.Fatalf("Read(buf) = %d, %s", n, err) + } + + var z bytes.Buffer + f := flate.NewWriter(&z, 5) + f.Write(b) + f.Close() + if z.Len() < len(b)*99/100 { + t.Fatalf("Compressed %d -> %d", len(b), z.Len()) + } +} diff --git a/src/pkg/crypto/rand/rand_unix.go b/src/pkg/crypto/rand/rand_unix.go new file mode 100644 index 000000000..3a06aa8b1 --- /dev/null +++ b/src/pkg/crypto/rand/rand_unix.go @@ -0,0 +1,125 @@ +// Copyright 2010 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Unix cryptographically secure pseudorandom number +// generator. + +package rand + +import ( + "bufio" + "crypto/aes" + "io" + "os" + "sync" + "time" +) + +// Easy implementation: read from /dev/urandom. +// This is sufficient on Linux, OS X, and FreeBSD. + +func init() { Reader = &devReader{name: "/dev/urandom"} } + +// A devReader satisfies reads by reading the file named name. +type devReader struct { + name string + f io.Reader + mu sync.Mutex +} + +func (r *devReader) Read(b []byte) (n int, err os.Error) { + r.mu.Lock() + defer r.mu.Unlock() + if r.f == nil { + f, err := os.Open(r.name) + if f == nil { + return 0, err + } + r.f = bufio.NewReader(f) + } + return r.f.Read(b) +} + +// Alternate pseudo-random implementation for use on +// systems without a reliable /dev/urandom. So far we +// haven't needed it. + +// newReader returns a new pseudorandom generator that +// seeds itself by reading from entropy. If entropy == nil, +// the generator seeds itself by reading from the system's +// random number generator, typically /dev/random. +// The Read method on the returned reader always returns +// the full amount asked for, or else it returns an error. +// +// The generator uses the X9.31 algorithm with AES-128, +// reseeding after every 1 MB of generated data. +func newReader(entropy io.Reader) io.Reader { + if entropy == nil { + entropy = &devReader{name: "/dev/random"} + } + return &reader{entropy: entropy} +} + +type reader struct { + mu sync.Mutex + budget int // number of bytes that can be generated + cipher *aes.Cipher + entropy io.Reader + time, seed, dst, key [aes.BlockSize]byte +} + +func (r *reader) Read(b []byte) (n int, err os.Error) { + r.mu.Lock() + defer r.mu.Unlock() + n = len(b) + + for len(b) > 0 { + if r.budget == 0 { + _, err := io.ReadFull(r.entropy, r.seed[0:]) + if err != nil { + return n - len(b), err + } + _, err = io.ReadFull(r.entropy, r.key[0:]) + if err != nil { + return n - len(b), err + } + r.cipher, err = aes.NewCipher(r.key[0:]) + if err != nil { + return n - len(b), err + } + r.budget = 1 << 20 // reseed after generating 1MB + } + r.budget -= aes.BlockSize + + // ANSI X9.31 (== X9.17) algorithm, but using AES in place of 3DES. + // + // single block: + // t = encrypt(time) + // dst = encrypt(t^seed) + // seed = encrypt(t^dst) + ns := time.Nanoseconds() + r.time[0] = byte(ns >> 56) + r.time[1] = byte(ns >> 48) + r.time[2] = byte(ns >> 40) + r.time[3] = byte(ns >> 32) + r.time[4] = byte(ns >> 24) + r.time[5] = byte(ns >> 16) + r.time[6] = byte(ns >> 8) + r.time[7] = byte(ns) + r.cipher.Encrypt(r.time[0:], r.time[0:]) + for i := 0; i < aes.BlockSize; i++ { + r.dst[i] = r.time[i] ^ r.seed[i] + } + r.cipher.Encrypt(r.dst[0:], r.dst[0:]) + for i := 0; i < aes.BlockSize; i++ { + r.seed[i] = r.time[i] ^ r.dst[i] + } + r.cipher.Encrypt(r.seed[0:], r.seed[0:]) + + m := copy(b, r.dst[0:]) + b = b[m:] + } + + return n, nil +} diff --git a/src/pkg/crypto/rand/rand_windows.go b/src/pkg/crypto/rand/rand_windows.go new file mode 100755 index 000000000..0eab6b213 --- /dev/null +++ b/src/pkg/crypto/rand/rand_windows.go @@ -0,0 +1,43 @@ +// Copyright 2010 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Windows cryptographically secure pseudorandom number +// generator. + +package rand + +import ( + "os" + "sync" + "syscall" +) + +// Implemented by using Windows CryptoAPI 2.0. + +func init() { Reader = &rngReader{} } + +// A rngReader satisfies reads by reading from the Windows CryptGenRandom API. +type rngReader struct { + prov syscall.Handle + mu sync.Mutex +} + +func (r *rngReader) Read(b []byte) (n int, err os.Error) { + r.mu.Lock() + if r.prov == 0 { + const provType = syscall.PROV_RSA_FULL + const flags = syscall.CRYPT_VERIFYCONTEXT | syscall.CRYPT_SILENT + errno := syscall.CryptAcquireContext(&r.prov, nil, nil, provType, flags) + if errno != 0 { + r.mu.Unlock() + return 0, os.NewSyscallError("CryptAcquireContext", errno) + } + } + r.mu.Unlock() + errno := syscall.CryptGenRandom(r.prov, uint32(len(b)), &b[0]) + if errno != 0 { + return 0, os.NewSyscallError("CryptGenRandom", errno) + } + return len(b), nil +} diff --git a/src/pkg/crypto/rand/util.go b/src/pkg/crypto/rand/util.go new file mode 100644 index 000000000..77028476e --- /dev/null +++ b/src/pkg/crypto/rand/util.go @@ -0,0 +1,80 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package rand + +import ( + "big" + "io" + "os" +) + +// Prime returns a number, p, of the given size, such that p is prime +// with high probability. +func Prime(rand io.Reader, bits int) (p *big.Int, err os.Error) { + if bits < 1 { + err = os.EINVAL + } + + b := uint(bits % 8) + if b == 0 { + b = 8 + } + + bytes := make([]byte, (bits+7)/8) + p = new(big.Int) + + for { + _, err = io.ReadFull(rand, bytes) + if err != nil { + return nil, err + } + + // Clear bits in the first byte to make sure the candidate has a size <= bits. + bytes[0] &= uint8(int(1< 256 { + return nil, KeySizeError(k) + } + var c Cipher + for i := 0; i < 256; i++ { + c.s[i] = uint8(i) + } + var j uint8 = 0 + for i := 0; i < 256; i++ { + j += c.s[i] + key[i%k] + c.s[i], c.s[j] = c.s[j], c.s[i] + } + return &c, nil +} + +// XORKeyStream sets dst to the result of XORing src with the key stream. +// Dst and src may be the same slice but otherwise should not overlap. +func (c *Cipher) XORKeyStream(dst, src []byte) { + for i := range src { + c.i += 1 + c.j += c.s[c.i] + c.s[c.i], c.s[c.j] = c.s[c.j], c.s[c.i] + dst[i] = src[i] ^ c.s[c.s[c.i]+c.s[c.j]] + } +} + +// Reset zeros the key data so that it will no longer appear in the +// process's memory. +func (c *Cipher) Reset() { + for i := range c.s { + c.s[i] = 0 + } + c.i, c.j = 0, 0 +} diff --git a/src/pkg/crypto/rc4/rc4_test.go b/src/pkg/crypto/rc4/rc4_test.go new file mode 100644 index 000000000..6265d9408 --- /dev/null +++ b/src/pkg/crypto/rc4/rc4_test.go @@ -0,0 +1,59 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package rc4 + +import ( + "testing" +) + +type rc4Test struct { + key, keystream []byte +} + +var golden = []rc4Test{ + // Test vectors from the original cypherpunk posting of ARC4: + // http://groups.google.com/group/sci.crypt/msg/10a300c9d21afca0?pli=1 + { + []byte{0x01, 0x23, 0x45, 0x67, 0x89, 0xab, 0xcd, 0xef}, + []byte{0x74, 0x94, 0xc2, 0xe7, 0x10, 0x4b, 0x08, 0x79}, + }, + { + []byte{0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00}, + []byte{0xde, 0x18, 0x89, 0x41, 0xa3, 0x37, 0x5d, 0x3a}, + }, + { + []byte{0xef, 0x01, 0x23, 0x45}, + []byte{0xd6, 0xa1, 0x41, 0xa7, 0xec, 0x3c, 0x38, 0xdf, 0xbd, 0x61}, + }, + + // Test vectors from the Wikipedia page: http://en.wikipedia.org/wiki/RC4 + { + []byte{0x4b, 0x65, 0x79}, + []byte{0xeb, 0x9f, 0x77, 0x81, 0xb7, 0x34, 0xca, 0x72, 0xa7, 0x19}, + }, + { + []byte{0x57, 0x69, 0x6b, 0x69}, + []byte{0x60, 0x44, 0xdb, 0x6d, 0x41, 0xb7}, + }, +} + +func TestGolden(t *testing.T) { + for i := 0; i < len(golden); i++ { + g := golden[i] + c, err := NewCipher(g.key) + if err != nil { + t.Errorf("Failed to create cipher at golden index %d", i) + return + } + keystream := make([]byte, len(g.keystream)) + c.XORKeyStream(keystream, keystream) + for j, v := range keystream { + if g.keystream[j] != v { + t.Errorf("Failed at golden index %d", i) + break + } + } + } +} diff --git a/src/pkg/crypto/ripemd160/Makefile b/src/pkg/crypto/ripemd160/Makefile new file mode 100644 index 000000000..7e529457d --- /dev/null +++ b/src/pkg/crypto/ripemd160/Makefile @@ -0,0 +1,12 @@ +# Copyright 2010 The Go Authors. All rights reserved. +# Use of this source code is governed by a BSD-style +# license that can be found in the LICENSE file. + +include ../../../Make.inc + +TARG=crypto/ripemd160 +GOFILES=\ + ripemd160.go\ + ripemd160block.go\ + +include ../../../Make.pkg diff --git a/src/pkg/crypto/ripemd160/ripemd160.go b/src/pkg/crypto/ripemd160/ripemd160.go new file mode 100644 index 000000000..5aaca59a3 --- /dev/null +++ b/src/pkg/crypto/ripemd160/ripemd160.go @@ -0,0 +1,118 @@ +// Copyright 2010 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Package ripemd160 implements the RIPEMD-160 hash algorithm. +package ripemd160 + +// RIPEMD-160 is designed by by Hans Dobbertin, Antoon Bosselaers, and Bart +// Preneel with specifications available at: +// http://homes.esat.kuleuven.be/~cosicart/pdf/AB-9601/AB-9601.pdf. + +import ( + "crypto" + "hash" + "os" +) + +func init() { + crypto.RegisterHash(crypto.RIPEMD160, New) +} + +// The size of the checksum in bytes. +const Size = 20 + +// The block size of the hash algorithm in bytes. +const BlockSize = 64 + +const ( + _s0 = 0x67452301 + _s1 = 0xefcdab89 + _s2 = 0x98badcfe + _s3 = 0x10325476 + _s4 = 0xc3d2e1f0 +) + +// digest represents the partial evaluation of a checksum. +type digest struct { + s [5]uint32 // running context + x [BlockSize]byte // temporary buffer + nx int // index into x + tc uint64 // total count of bytes processed +} + +func (d *digest) Reset() { + d.s[0], d.s[1], d.s[2], d.s[3], d.s[4] = _s0, _s1, _s2, _s3, _s4 + d.nx = 0 + d.tc = 0 +} + +// New returns a new hash.Hash computing the checksum. +func New() hash.Hash { + result := new(digest) + result.Reset() + return result +} + +func (d *digest) Size() int { return Size } + +func (d *digest) Write(p []byte) (nn int, err os.Error) { + nn = len(p) + d.tc += uint64(nn) + if d.nx > 0 { + n := len(p) + if n > BlockSize-d.nx { + n = BlockSize - d.nx + } + for i := 0; i < n; i++ { + d.x[d.nx+i] = p[i] + } + d.nx += n + if d.nx == BlockSize { + _Block(d, d.x[0:]) + d.nx = 0 + } + p = p[n:] + } + n := _Block(d, p) + p = p[n:] + if len(p) > 0 { + d.nx = copy(d.x[:], p) + } + return +} + +func (d0 *digest) Sum() []byte { + // Make a copy of d0 so that caller can keep writing and summing. + d := new(digest) + *d = *d0 + + // Padding. Add a 1 bit and 0 bits until 56 bytes mod 64. + tc := d.tc + var tmp [64]byte + tmp[0] = 0x80 + if tc%64 < 56 { + d.Write(tmp[0 : 56-tc%64]) + } else { + d.Write(tmp[0 : 64+56-tc%64]) + } + + // Length in bits. + tc <<= 3 + for i := uint(0); i < 8; i++ { + tmp[i] = byte(tc >> (8 * i)) + } + d.Write(tmp[0:8]) + + if d.nx != 0 { + panic("d.nx != 0") + } + + p := make([]byte, 20) + j := 0 + for _, s := range d.s { + p[j], p[j+1], p[j+2], p[j+3] = byte(s), byte(s>>8), byte(s>>16), byte(s>>24) + j += 4 + } + return p +} diff --git a/src/pkg/crypto/ripemd160/ripemd160_test.go b/src/pkg/crypto/ripemd160/ripemd160_test.go new file mode 100644 index 000000000..f4135f5cf --- /dev/null +++ b/src/pkg/crypto/ripemd160/ripemd160_test.go @@ -0,0 +1,64 @@ +// Copyright 2010 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package ripemd160 + +// Test vectors are from: +// http://homes.esat.kuleuven.be/~bosselae/ripemd160.html + +import ( + "fmt" + "io" + "testing" +) + +type mdTest struct { + out string + in string +} + +var vectors = [...]mdTest{ + {"9c1185a5c5e9fc54612808977ee8f548b2258d31", ""}, + {"0bdc9d2d256b3ee9daae347be6f4dc835a467ffe", "a"}, + {"8eb208f7e05d987a9b044a8e98c6b087f15a0bfc", "abc"}, + {"5d0689ef49d2fae572b881b123a85ffa21595f36", "message digest"}, + {"f71c27109c692c1b56bbdceb5b9d2865b3708dbc", "abcdefghijklmnopqrstuvwxyz"}, + {"12a053384a9c0c88e405a06c27dcf49ada62eb2b", "abcdbcdecdefdefgefghfghighijhijkijkljklmklmnlmnomnopnopq"}, + {"b0e20b6e3116640286ed3a87a5713079b21f5189", "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789"}, + {"9b752e45573d4b39f4dbd3323cab82bf63326bfb", "12345678901234567890123456789012345678901234567890123456789012345678901234567890"}, +} + +func TestVectors(t *testing.T) { + for i := 0; i < len(vectors); i++ { + tv := vectors[i] + md := New() + for j := 0; j < 3; j++ { + if j < 2 { + io.WriteString(md, tv.in) + } else { + io.WriteString(md, tv.in[0:len(tv.in)/2]) + md.Sum() + io.WriteString(md, tv.in[len(tv.in)/2:]) + } + s := fmt.Sprintf("%x", md.Sum()) + if s != tv.out { + t.Fatalf("RIPEMD-160[%d](%s) = %s, expected %s", j, tv.in, s, tv.out) + } + md.Reset() + } + } +} + +func TestMillionA(t *testing.T) { + md := New() + for i := 0; i < 100000; i++ { + io.WriteString(md, "aaaaaaaaaa") + } + out := "52783243c1697bdbe16d37f97f68f08325dc1528" + s := fmt.Sprintf("%x", md.Sum()) + if s != out { + t.Fatalf("RIPEMD-160 (1 million 'a') = %s, expected %s", s, out) + } + md.Reset() +} diff --git a/src/pkg/crypto/ripemd160/ripemd160block.go b/src/pkg/crypto/ripemd160/ripemd160block.go new file mode 100644 index 000000000..7bc8e6c48 --- /dev/null +++ b/src/pkg/crypto/ripemd160/ripemd160block.go @@ -0,0 +1,161 @@ +// Copyright 2010 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// RIPEMD-160 block step. +// In its own file so that a faster assembly or C version +// can be substituted easily. + +package ripemd160 + +// work buffer indices and roll amounts for one line +var _n = [80]uint{ + 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, + 7, 4, 13, 1, 10, 6, 15, 3, 12, 0, 9, 5, 2, 14, 11, 8, + 3, 10, 14, 4, 9, 15, 8, 1, 2, 7, 0, 6, 13, 11, 5, 12, + 1, 9, 11, 10, 0, 8, 12, 4, 13, 3, 7, 15, 14, 5, 6, 2, + 4, 0, 5, 9, 7, 12, 2, 10, 14, 1, 3, 8, 11, 6, 15, 13, +} + +var _r = [80]uint{ + 11, 14, 15, 12, 5, 8, 7, 9, 11, 13, 14, 15, 6, 7, 9, 8, + 7, 6, 8, 13, 11, 9, 7, 15, 7, 12, 15, 9, 11, 7, 13, 12, + 11, 13, 6, 7, 14, 9, 13, 15, 14, 8, 13, 6, 5, 12, 7, 5, + 11, 12, 14, 15, 14, 15, 9, 8, 9, 14, 5, 6, 8, 6, 5, 12, + 9, 15, 5, 11, 6, 8, 13, 12, 5, 12, 13, 14, 11, 8, 5, 6, +} + +// same for the other parallel one +var n_ = [80]uint{ + 5, 14, 7, 0, 9, 2, 11, 4, 13, 6, 15, 8, 1, 10, 3, 12, + 6, 11, 3, 7, 0, 13, 5, 10, 14, 15, 8, 12, 4, 9, 1, 2, + 15, 5, 1, 3, 7, 14, 6, 9, 11, 8, 12, 2, 10, 0, 4, 13, + 8, 6, 4, 1, 3, 11, 15, 0, 5, 12, 2, 13, 9, 7, 10, 14, + 12, 15, 10, 4, 1, 5, 8, 7, 6, 2, 13, 14, 0, 3, 9, 11, +} + +var r_ = [80]uint{ + 8, 9, 9, 11, 13, 15, 15, 5, 7, 7, 8, 11, 14, 14, 12, 6, + 9, 13, 15, 7, 12, 8, 9, 11, 7, 7, 12, 7, 6, 15, 13, 11, + 9, 7, 15, 11, 8, 6, 6, 14, 12, 13, 5, 14, 13, 13, 7, 5, + 15, 5, 8, 11, 14, 14, 6, 14, 6, 9, 12, 9, 12, 5, 15, 8, + 8, 5, 12, 9, 12, 5, 14, 6, 8, 13, 6, 5, 15, 13, 11, 11, +} + +func _Block(md *digest, p []byte) int { + n := 0 + var x [16]uint32 + var alpha, beta uint32 + for len(p) >= BlockSize { + a, b, c, d, e := md.s[0], md.s[1], md.s[2], md.s[3], md.s[4] + aa, bb, cc, dd, ee := a, b, c, d, e + j := 0 + for i := 0; i < 16; i++ { + x[i] = uint32(p[j]) | uint32(p[j+1])<<8 | uint32(p[j+2])<<16 | uint32(p[j+3])<<24 + j += 4 + } + + // round 1 + i := 0 + for i < 16 { + alpha = a + (b ^ c ^ d) + x[_n[i]] + s := _r[i] + alpha = (alpha<>(32-s)) + e + beta = c<<10 | c>>22 + a, b, c, d, e = e, alpha, b, beta, d + + // parallel line + alpha = aa + (bb ^ (cc | ^dd)) + x[n_[i]] + 0x50a28be6 + s = r_[i] + alpha = (alpha<>(32-s)) + ee + beta = cc<<10 | cc>>22 + aa, bb, cc, dd, ee = ee, alpha, bb, beta, dd + + i++ + } + + // round 2 + for i < 32 { + alpha = a + (b&c | ^b&d) + x[_n[i]] + 0x5a827999 + s := _r[i] + alpha = (alpha<>(32-s)) + e + beta = c<<10 | c>>22 + a, b, c, d, e = e, alpha, b, beta, d + + // parallel line + alpha = aa + (bb&dd | cc&^dd) + x[n_[i]] + 0x5c4dd124 + s = r_[i] + alpha = (alpha<>(32-s)) + ee + beta = cc<<10 | cc>>22 + aa, bb, cc, dd, ee = ee, alpha, bb, beta, dd + + i++ + } + + // round 3 + for i < 48 { + alpha = a + (b | ^c ^ d) + x[_n[i]] + 0x6ed9eba1 + s := _r[i] + alpha = (alpha<>(32-s)) + e + beta = c<<10 | c>>22 + a, b, c, d, e = e, alpha, b, beta, d + + // parallel line + alpha = aa + (bb | ^cc ^ dd) + x[n_[i]] + 0x6d703ef3 + s = r_[i] + alpha = (alpha<>(32-s)) + ee + beta = cc<<10 | cc>>22 + aa, bb, cc, dd, ee = ee, alpha, bb, beta, dd + + i++ + } + + // round 4 + for i < 64 { + alpha = a + (b&d | c&^d) + x[_n[i]] + 0x8f1bbcdc + s := _r[i] + alpha = (alpha<>(32-s)) + e + beta = c<<10 | c>>22 + a, b, c, d, e = e, alpha, b, beta, d + + // parallel line + alpha = aa + (bb&cc | ^bb&dd) + x[n_[i]] + 0x7a6d76e9 + s = r_[i] + alpha = (alpha<>(32-s)) + ee + beta = cc<<10 | cc>>22 + aa, bb, cc, dd, ee = ee, alpha, bb, beta, dd + + i++ + } + + // round 5 + for i < 80 { + alpha = a + (b ^ (c | ^d)) + x[_n[i]] + 0xa953fd4e + s := _r[i] + alpha = (alpha<>(32-s)) + e + beta = c<<10 | c>>22 + a, b, c, d, e = e, alpha, b, beta, d + + // parallel line + alpha = aa + (bb ^ cc ^ dd) + x[n_[i]] + s = r_[i] + alpha = (alpha<>(32-s)) + ee + beta = cc<<10 | cc>>22 + aa, bb, cc, dd, ee = ee, alpha, bb, beta, dd + + i++ + } + + // combine results + dd += c + md.s[1] + md.s[1] = md.s[2] + d + ee + md.s[2] = md.s[3] + e + aa + md.s[3] = md.s[4] + a + bb + md.s[4] = md.s[0] + b + cc + md.s[0] = dd + + p = p[BlockSize:] + n += BlockSize + } + return n +} diff --git a/src/pkg/crypto/rsa/Makefile b/src/pkg/crypto/rsa/Makefile new file mode 100644 index 000000000..ff26ca6f2 --- /dev/null +++ b/src/pkg/crypto/rsa/Makefile @@ -0,0 +1,12 @@ +# Copyright 2009 The Go Authors. All rights reserved. +# Use of this source code is governed by a BSD-style +# license that can be found in the LICENSE file. + +include ../../../Make.inc + +TARG=crypto/rsa +GOFILES=\ + rsa.go\ + pkcs1v15.go\ + +include ../../../Make.pkg diff --git a/src/pkg/crypto/rsa/pkcs1v15.go b/src/pkg/crypto/rsa/pkcs1v15.go new file mode 100644 index 000000000..600623114 --- /dev/null +++ b/src/pkg/crypto/rsa/pkcs1v15.go @@ -0,0 +1,242 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package rsa + +import ( + "big" + "crypto" + "crypto/subtle" + "io" + "os" +) + +// This file implements encryption and decryption using PKCS#1 v1.5 padding. + +// EncryptPKCS1v15 encrypts the given message with RSA and the padding scheme from PKCS#1 v1.5. +// The message must be no longer than the length of the public modulus minus 11 bytes. +// WARNING: use of this function to encrypt plaintexts other than session keys +// is dangerous. Use RSA OAEP in new protocols. +func EncryptPKCS1v15(rand io.Reader, pub *PublicKey, msg []byte) (out []byte, err os.Error) { + k := (pub.N.BitLen() + 7) / 8 + if len(msg) > k-11 { + err = MessageTooLongError{} + return + } + + // EM = 0x02 || PS || 0x00 || M + em := make([]byte, k-1) + em[0] = 2 + ps, mm := em[1:len(em)-len(msg)-1], em[len(em)-len(msg):] + err = nonZeroRandomBytes(ps, rand) + if err != nil { + return + } + em[len(em)-len(msg)-1] = 0 + copy(mm, msg) + + m := new(big.Int).SetBytes(em) + c := encrypt(new(big.Int), pub, m) + out = c.Bytes() + return +} + +// DecryptPKCS1v15 decrypts a plaintext using RSA and the padding scheme from PKCS#1 v1.5. +// If rand != nil, it uses RSA blinding to avoid timing side-channel attacks. +func DecryptPKCS1v15(rand io.Reader, priv *PrivateKey, ciphertext []byte) (out []byte, err os.Error) { + valid, out, err := decryptPKCS1v15(rand, priv, ciphertext) + if err == nil && valid == 0 { + err = DecryptionError{} + } + + return +} + +// DecryptPKCS1v15SessionKey decrypts a session key using RSA and the padding scheme from PKCS#1 v1.5. +// If rand != nil, it uses RSA blinding to avoid timing side-channel attacks. +// It returns an error if the ciphertext is the wrong length or if the +// ciphertext is greater than the public modulus. Otherwise, no error is +// returned. If the padding is valid, the resulting plaintext message is copied +// into key. Otherwise, key is unchanged. These alternatives occur in constant +// time. It is intended that the user of this function generate a random +// session key beforehand and continue the protocol with the resulting value. +// This will remove any possibility that an attacker can learn any information +// about the plaintext. +// See ``Chosen Ciphertext Attacks Against Protocols Based on the RSA +// Encryption Standard PKCS #1'', Daniel Bleichenbacher, Advances in Cryptology +// (Crypto '98), +func DecryptPKCS1v15SessionKey(rand io.Reader, priv *PrivateKey, ciphertext []byte, key []byte) (err os.Error) { + k := (priv.N.BitLen() + 7) / 8 + if k-(len(key)+3+8) < 0 { + err = DecryptionError{} + return + } + + valid, msg, err := decryptPKCS1v15(rand, priv, ciphertext) + if err != nil { + return + } + + valid &= subtle.ConstantTimeEq(int32(len(msg)), int32(len(key))) + subtle.ConstantTimeCopy(valid, key, msg) + return +} + +func decryptPKCS1v15(rand io.Reader, priv *PrivateKey, ciphertext []byte) (valid int, msg []byte, err os.Error) { + k := (priv.N.BitLen() + 7) / 8 + if k < 11 { + err = DecryptionError{} + return + } + + c := new(big.Int).SetBytes(ciphertext) + m, err := decrypt(rand, priv, c) + if err != nil { + return + } + + em := leftPad(m.Bytes(), k) + firstByteIsZero := subtle.ConstantTimeByteEq(em[0], 0) + secondByteIsTwo := subtle.ConstantTimeByteEq(em[1], 2) + + // The remainder of the plaintext must be a string of non-zero random + // octets, followed by a 0, followed by the message. + // lookingForIndex: 1 iff we are still looking for the zero. + // index: the offset of the first zero byte. + var lookingForIndex, index int + lookingForIndex = 1 + + for i := 2; i < len(em); i++ { + equals0 := subtle.ConstantTimeByteEq(em[i], 0) + index = subtle.ConstantTimeSelect(lookingForIndex&equals0, i, index) + lookingForIndex = subtle.ConstantTimeSelect(equals0, 0, lookingForIndex) + } + + valid = firstByteIsZero & secondByteIsTwo & (^lookingForIndex & 1) + msg = em[index+1:] + return +} + +// nonZeroRandomBytes fills the given slice with non-zero random octets. +func nonZeroRandomBytes(s []byte, rand io.Reader) (err os.Error) { + _, err = io.ReadFull(rand, s) + if err != nil { + return + } + + for i := 0; i < len(s); i++ { + for s[i] == 0 { + _, err = io.ReadFull(rand, s[i:i+1]) + if err != nil { + return + } + // In tests, the PRNG may return all zeros so we do + // this to break the loop. + s[i] ^= 0x42 + } + } + + return +} + +// These are ASN1 DER structures: +// DigestInfo ::= SEQUENCE { +// digestAlgorithm AlgorithmIdentifier, +// digest OCTET STRING +// } +// For performance, we don't use the generic ASN1 encoder. Rather, we +// precompute a prefix of the digest value that makes a valid ASN1 DER string +// with the correct contents. +var hashPrefixes = map[crypto.Hash][]byte{ + crypto.MD5: {0x30, 0x20, 0x30, 0x0c, 0x06, 0x08, 0x2a, 0x86, 0x48, 0x86, 0xf7, 0x0d, 0x02, 0x05, 0x05, 0x00, 0x04, 0x10}, + crypto.SHA1: {0x30, 0x21, 0x30, 0x09, 0x06, 0x05, 0x2b, 0x0e, 0x03, 0x02, 0x1a, 0x05, 0x00, 0x04, 0x14}, + crypto.SHA256: {0x30, 0x31, 0x30, 0x0d, 0x06, 0x09, 0x60, 0x86, 0x48, 0x01, 0x65, 0x03, 0x04, 0x02, 0x01, 0x05, 0x00, 0x04, 0x20}, + crypto.SHA384: {0x30, 0x41, 0x30, 0x0d, 0x06, 0x09, 0x60, 0x86, 0x48, 0x01, 0x65, 0x03, 0x04, 0x02, 0x02, 0x05, 0x00, 0x04, 0x30}, + crypto.SHA512: {0x30, 0x51, 0x30, 0x0d, 0x06, 0x09, 0x60, 0x86, 0x48, 0x01, 0x65, 0x03, 0x04, 0x02, 0x03, 0x05, 0x00, 0x04, 0x40}, + crypto.MD5SHA1: {}, // A special TLS case which doesn't use an ASN1 prefix. + crypto.RIPEMD160: {0x30, 0x20, 0x30, 0x08, 0x06, 0x06, 0x28, 0xcf, 0x06, 0x03, 0x00, 0x31, 0x04, 0x14}, +} + +// SignPKCS1v15 calculates the signature of hashed using RSASSA-PKCS1-V1_5-SIGN from RSA PKCS#1 v1.5. +// Note that hashed must be the result of hashing the input message using the +// given hash function. +func SignPKCS1v15(rand io.Reader, priv *PrivateKey, hash crypto.Hash, hashed []byte) (s []byte, err os.Error) { + hashLen, prefix, err := pkcs1v15HashInfo(hash, len(hashed)) + if err != nil { + return + } + + tLen := len(prefix) + hashLen + k := (priv.N.BitLen() + 7) / 8 + if k < tLen+11 { + return nil, MessageTooLongError{} + } + + // EM = 0x00 || 0x01 || PS || 0x00 || T + em := make([]byte, k) + em[1] = 1 + for i := 2; i < k-tLen-1; i++ { + em[i] = 0xff + } + copy(em[k-tLen:k-hashLen], prefix) + copy(em[k-hashLen:k], hashed) + + m := new(big.Int).SetBytes(em) + c, err := decrypt(rand, priv, m) + if err == nil { + s = c.Bytes() + } + return +} + +// VerifyPKCS1v15 verifies an RSA PKCS#1 v1.5 signature. +// hashed is the result of hashing the input message using the given hash +// function and sig is the signature. A valid signature is indicated by +// returning a nil error. +func VerifyPKCS1v15(pub *PublicKey, hash crypto.Hash, hashed []byte, sig []byte) (err os.Error) { + hashLen, prefix, err := pkcs1v15HashInfo(hash, len(hashed)) + if err != nil { + return + } + + tLen := len(prefix) + hashLen + k := (pub.N.BitLen() + 7) / 8 + if k < tLen+11 { + err = VerificationError{} + return + } + + c := new(big.Int).SetBytes(sig) + m := encrypt(new(big.Int), pub, c) + em := leftPad(m.Bytes(), k) + // EM = 0x00 || 0x01 || PS || 0x00 || T + + ok := subtle.ConstantTimeByteEq(em[0], 0) + ok &= subtle.ConstantTimeByteEq(em[1], 1) + ok &= subtle.ConstantTimeCompare(em[k-hashLen:k], hashed) + ok &= subtle.ConstantTimeCompare(em[k-tLen:k-hashLen], prefix) + ok &= subtle.ConstantTimeByteEq(em[k-tLen-1], 0) + + for i := 2; i < k-tLen-1; i++ { + ok &= subtle.ConstantTimeByteEq(em[i], 0xff) + } + + if ok != 1 { + return VerificationError{} + } + + return nil +} + +func pkcs1v15HashInfo(hash crypto.Hash, inLen int) (hashLen int, prefix []byte, err os.Error) { + hashLen = hash.Size() + if inLen != hashLen { + return 0, nil, os.NewError("input must be hashed message") + } + prefix, ok := hashPrefixes[hash] + if !ok { + return 0, nil, os.NewError("unsupported hash function") + } + return +} diff --git a/src/pkg/crypto/rsa/pkcs1v15_test.go b/src/pkg/crypto/rsa/pkcs1v15_test.go new file mode 100644 index 000000000..d69bacfd6 --- /dev/null +++ b/src/pkg/crypto/rsa/pkcs1v15_test.go @@ -0,0 +1,221 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package rsa + +import ( + "big" + "bytes" + "crypto" + "crypto/rand" + "crypto/sha1" + "encoding/base64" + "encoding/hex" + "io" + "testing" + "testing/quick" +) + +func decodeBase64(in string) []byte { + out := make([]byte, base64.StdEncoding.DecodedLen(len(in))) + n, err := base64.StdEncoding.Decode(out, []byte(in)) + if err != nil { + return nil + } + return out[0:n] +} + +type DecryptPKCS1v15Test struct { + in, out string +} + +// These test vectors were generated with `openssl rsautl -pkcs -encrypt` +var decryptPKCS1v15Tests = []DecryptPKCS1v15Test{ + { + "gIcUIoVkD6ATMBk/u/nlCZCCWRKdkfjCgFdo35VpRXLduiKXhNz1XupLLzTXAybEq15juc+EgY5o0DHv/nt3yg==", + "x", + }, + { + "Y7TOCSqofGhkRb+jaVRLzK8xw2cSo1IVES19utzv6hwvx+M8kFsoWQm5DzBeJCZTCVDPkTpavUuEbgp8hnUGDw==", + "testing.", + }, + { + "arReP9DJtEVyV2Dg3dDp4c/PSk1O6lxkoJ8HcFupoRorBZG+7+1fDAwT1olNddFnQMjmkb8vxwmNMoTAT/BFjQ==", + "testing.\n", + }, + { + "WtaBXIoGC54+vH0NH0CHHE+dRDOsMc/6BrfFu2lEqcKL9+uDuWaf+Xj9mrbQCjjZcpQuX733zyok/jsnqe/Ftw==", + "01234567890123456789012345678901234567890123456789012", + }, +} + +func TestDecryptPKCS1v15(t *testing.T) { + for i, test := range decryptPKCS1v15Tests { + out, err := DecryptPKCS1v15(nil, rsaPrivateKey, decodeBase64(test.in)) + if err != nil { + t.Errorf("#%d error decrypting", i) + } + want := []byte(test.out) + if bytes.Compare(out, want) != 0 { + t.Errorf("#%d got:%#v want:%#v", i, out, want) + } + } +} + +func TestEncryptPKCS1v15(t *testing.T) { + random := rand.Reader + k := (rsaPrivateKey.N.BitLen() + 7) / 8 + + tryEncryptDecrypt := func(in []byte, blind bool) bool { + if len(in) > k-11 { + in = in[0 : k-11] + } + + ciphertext, err := EncryptPKCS1v15(random, &rsaPrivateKey.PublicKey, in) + if err != nil { + t.Errorf("error encrypting: %s", err) + return false + } + + var rand io.Reader + if !blind { + rand = nil + } else { + rand = random + } + plaintext, err := DecryptPKCS1v15(rand, rsaPrivateKey, ciphertext) + if err != nil { + t.Errorf("error decrypting: %s", err) + return false + } + + if bytes.Compare(plaintext, in) != 0 { + t.Errorf("output mismatch: %#v %#v", plaintext, in) + return false + } + return true + } + + config := new(quick.Config) + if testing.Short() { + config.MaxCount = 10 + } + quick.Check(tryEncryptDecrypt, config) +} + +// These test vectors were generated with `openssl rsautl -pkcs -encrypt` +var decryptPKCS1v15SessionKeyTests = []DecryptPKCS1v15Test{ + { + "e6ukkae6Gykq0fKzYwULpZehX+UPXYzMoB5mHQUDEiclRbOTqas4Y0E6nwns1BBpdvEJcilhl5zsox/6DtGsYg==", + "1234", + }, + { + "Dtis4uk/q/LQGGqGk97P59K03hkCIVFMEFZRgVWOAAhxgYpCRG0MX2adptt92l67IqMki6iVQyyt0TtX3IdtEw==", + "FAIL", + }, + { + "LIyFyCYCptPxrvTxpol8F3M7ZivlMsf53zs0vHRAv+rDIh2YsHS69ePMoPMe3TkOMZ3NupiL3takPxIs1sK+dw==", + "abcd", + }, + { + "bafnobel46bKy76JzqU/RIVOH0uAYvzUtauKmIidKgM0sMlvobYVAVQPeUQ/oTGjbIZ1v/6Gyi5AO4DtHruGdw==", + "FAIL", + }, +} + +func TestEncryptPKCS1v15SessionKey(t *testing.T) { + for i, test := range decryptPKCS1v15SessionKeyTests { + key := []byte("FAIL") + err := DecryptPKCS1v15SessionKey(nil, rsaPrivateKey, decodeBase64(test.in), key) + if err != nil { + t.Errorf("#%d error decrypting", i) + } + want := []byte(test.out) + if bytes.Compare(key, want) != 0 { + t.Errorf("#%d got:%#v want:%#v", i, key, want) + } + } +} + +func TestNonZeroRandomBytes(t *testing.T) { + random := rand.Reader + + b := make([]byte, 512) + err := nonZeroRandomBytes(b, random) + if err != nil { + t.Errorf("returned error: %s", err) + } + for _, b := range b { + if b == 0 { + t.Errorf("Zero octet found") + return + } + } +} + +type signPKCS1v15Test struct { + in, out string +} + +// These vectors have been tested with +// `openssl rsautl -verify -inkey pk -in signature | hexdump -C` +var signPKCS1v15Tests = []signPKCS1v15Test{ + {"Test.\n", "a4f3fa6ea93bcdd0c57be020c1193ecbfd6f200a3d95c409769b029578fa0e336ad9a347600e40d3ae823b8c7e6bad88cc07c1d54c3a1523cbbb6d58efc362ae"}, +} + +func TestSignPKCS1v15(t *testing.T) { + for i, test := range signPKCS1v15Tests { + h := sha1.New() + h.Write([]byte(test.in)) + digest := h.Sum() + + s, err := SignPKCS1v15(nil, rsaPrivateKey, crypto.SHA1, digest) + if err != nil { + t.Errorf("#%d %s", i, err) + } + + expected, _ := hex.DecodeString(test.out) + if bytes.Compare(s, expected) != 0 { + t.Errorf("#%d got: %x want: %x", i, s, expected) + } + } +} + +func TestVerifyPKCS1v15(t *testing.T) { + for i, test := range signPKCS1v15Tests { + h := sha1.New() + h.Write([]byte(test.in)) + digest := h.Sum() + + sig, _ := hex.DecodeString(test.out) + + err := VerifyPKCS1v15(&rsaPrivateKey.PublicKey, crypto.SHA1, digest, sig) + if err != nil { + t.Errorf("#%d %s", i, err) + } + } +} + +// In order to generate new test vectors you'll need the PEM form of this key: +// -----BEGIN RSA PRIVATE KEY----- +// MIIBOgIBAAJBALKZD0nEffqM1ACuak0bijtqE2QrI/KLADv7l3kK3ppMyCuLKoF0 +// fd7Ai2KW5ToIwzFofvJcS/STa6HA5gQenRUCAwEAAQJBAIq9amn00aS0h/CrjXqu +// /ThglAXJmZhOMPVn4eiu7/ROixi9sex436MaVeMqSNf7Ex9a8fRNfWss7Sqd9eWu +// RTUCIQDasvGASLqmjeffBNLTXV2A5g4t+kLVCpsEIZAycV5GswIhANEPLmax0ME/ +// EO+ZJ79TJKN5yiGBRsv5yvx5UiHxajEXAiAhAol5N4EUyq6I9w1rYdhPMGpLfk7A +// IU2snfRJ6Nq2CQIgFrPsWRCkV+gOYcajD17rEqmuLrdIRexpg8N1DOSXoJ8CIGlS +// tAboUGBxTDq3ZroNism3DaMIbKPyYrAqhKov1h5V +// -----END RSA PRIVATE KEY----- + +var rsaPrivateKey = &PrivateKey{ + PublicKey: PublicKey{ + N: fromBase10("9353930466774385905609975137998169297361893554149986716853295022578535724979677252958524466350471210367835187480748268864277464700638583474144061408845077"), + E: 65537, + }, + D: fromBase10("7266398431328116344057699379749222532279343923819063639497049039389899328538543087657733766554155839834519529439851673014800261285757759040931985506583861"), + Primes: []*big.Int{ + fromBase10("98920366548084643601728869055592650835572950932266967461790948584315647051443"), + fromBase10("94560208308847015747498523884063394671606671904944666360068158221458669711639"), + }, +} diff --git a/src/pkg/crypto/rsa/rsa.go b/src/pkg/crypto/rsa/rsa.go new file mode 100644 index 000000000..6957659f2 --- /dev/null +++ b/src/pkg/crypto/rsa/rsa.go @@ -0,0 +1,502 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Package rsa implements RSA encryption as specified in PKCS#1. +package rsa + +// TODO(agl): Add support for PSS padding. + +import ( + "big" + "crypto/rand" + "crypto/subtle" + "hash" + "io" + "os" +) + +var bigZero = big.NewInt(0) +var bigOne = big.NewInt(1) + +// A PublicKey represents the public part of an RSA key. +type PublicKey struct { + N *big.Int // modulus + E int // public exponent +} + +// A PrivateKey represents an RSA key +type PrivateKey struct { + PublicKey // public part. + D *big.Int // private exponent + Primes []*big.Int // prime factors of N, has >= 2 elements. + + // Precomputed contains precomputed values that speed up private + // operations, if available. + Precomputed PrecomputedValues +} + +type PrecomputedValues struct { + Dp, Dq *big.Int // D mod (P-1) (or mod Q-1) + Qinv *big.Int // Q^-1 mod Q + + // CRTValues is used for the 3rd and subsequent primes. Due to a + // historical accident, the CRT for the first two primes is handled + // differently in PKCS#1 and interoperability is sufficiently + // important that we mirror this. + CRTValues []CRTValue +} + +// CRTValue contains the precomputed chinese remainder theorem values. +type CRTValue struct { + Exp *big.Int // D mod (prime-1). + Coeff *big.Int // R·Coeff ≡ 1 mod Prime. + R *big.Int // product of primes prior to this (inc p and q). +} + +// Validate performs basic sanity checks on the key. +// It returns nil if the key is valid, or else an os.Error describing a problem. + +func (priv *PrivateKey) Validate() os.Error { + // Check that the prime factors are actually prime. Note that this is + // just a sanity check. Since the random witnesses chosen by + // ProbablyPrime are deterministic, given the candidate number, it's + // easy for an attack to generate composites that pass this test. + for _, prime := range priv.Primes { + if !big.ProbablyPrime(prime, 20) { + return os.NewError("prime factor is composite") + } + } + + // Check that Πprimes == n. + modulus := new(big.Int).Set(bigOne) + for _, prime := range priv.Primes { + modulus.Mul(modulus, prime) + } + if modulus.Cmp(priv.N) != 0 { + return os.NewError("invalid modulus") + } + // Check that e and totient(Πprimes) are coprime. + totient := new(big.Int).Set(bigOne) + for _, prime := range priv.Primes { + pminus1 := new(big.Int).Sub(prime, bigOne) + totient.Mul(totient, pminus1) + } + e := big.NewInt(int64(priv.E)) + gcd := new(big.Int) + x := new(big.Int) + y := new(big.Int) + big.GcdInt(gcd, x, y, totient, e) + if gcd.Cmp(bigOne) != 0 { + return os.NewError("invalid public exponent E") + } + // Check that de ≡ 1 (mod totient(Πprimes)) + de := new(big.Int).Mul(priv.D, e) + de.Mod(de, totient) + if de.Cmp(bigOne) != 0 { + return os.NewError("invalid private exponent D") + } + return nil +} + +// GenerateKey generates an RSA keypair of the given bit size. +func GenerateKey(random io.Reader, bits int) (priv *PrivateKey, err os.Error) { + return GenerateMultiPrimeKey(random, 2, bits) +} + +// GenerateMultiPrimeKey generates a multi-prime RSA keypair of the given bit +// size, as suggested in [1]. Although the public keys are compatible +// (actually, indistinguishable) from the 2-prime case, the private keys are +// not. Thus it may not be possible to export multi-prime private keys in +// certain formats or to subsequently import them into other code. +// +// Table 1 in [2] suggests maximum numbers of primes for a given size. +// +// [1] US patent 4405829 (1972, expired) +// [2] http://www.cacr.math.uwaterloo.ca/techreports/2006/cacr2006-16.pdf +func GenerateMultiPrimeKey(random io.Reader, nprimes int, bits int) (priv *PrivateKey, err os.Error) { + priv = new(PrivateKey) + // Smaller public exponents lead to faster public key + // operations. Since the exponent must be coprime to + // (p-1)(q-1), the smallest possible value is 3. Some have + // suggested that a larger exponent (often 2**16+1) be used + // since previous implementation bugs[1] were avoided when this + // was the case. However, there are no current reasons not to use + // small exponents. + // [1] http://marc.info/?l=cryptography&m=115694833312008&w=2 + priv.E = 3 + + if nprimes < 2 { + return nil, os.NewError("rsa.GenerateMultiPrimeKey: nprimes must be >= 2") + } + + primes := make([]*big.Int, nprimes) + +NextSetOfPrimes: + for { + todo := bits + for i := 0; i < nprimes; i++ { + primes[i], err = rand.Prime(random, todo/(nprimes-i)) + if err != nil { + return nil, err + } + todo -= primes[i].BitLen() + } + + // Make sure that primes is pairwise unequal. + for i, prime := range primes { + for j := 0; j < i; j++ { + if prime.Cmp(primes[j]) == 0 { + continue NextSetOfPrimes + } + } + } + + n := new(big.Int).Set(bigOne) + totient := new(big.Int).Set(bigOne) + pminus1 := new(big.Int) + for _, prime := range primes { + n.Mul(n, prime) + pminus1.Sub(prime, bigOne) + totient.Mul(totient, pminus1) + } + + g := new(big.Int) + priv.D = new(big.Int) + y := new(big.Int) + e := big.NewInt(int64(priv.E)) + big.GcdInt(g, priv.D, y, e, totient) + + if g.Cmp(bigOne) == 0 { + priv.D.Add(priv.D, totient) + priv.Primes = primes + priv.N = n + + break + } + } + + priv.Precompute() + return +} + +// incCounter increments a four byte, big-endian counter. +func incCounter(c *[4]byte) { + if c[3]++; c[3] != 0 { + return + } + if c[2]++; c[2] != 0 { + return + } + if c[1]++; c[1] != 0 { + return + } + c[0]++ +} + +// mgf1XOR XORs the bytes in out with a mask generated using the MGF1 function +// specified in PKCS#1 v2.1. +func mgf1XOR(out []byte, hash hash.Hash, seed []byte) { + var counter [4]byte + + done := 0 + for done < len(out) { + hash.Write(seed) + hash.Write(counter[0:4]) + digest := hash.Sum() + hash.Reset() + + for i := 0; i < len(digest) && done < len(out); i++ { + out[done] ^= digest[i] + done++ + } + incCounter(&counter) + } +} + +// MessageTooLongError is returned when attempting to encrypt a message which +// is too large for the size of the public key. +type MessageTooLongError struct{} + +func (MessageTooLongError) String() string { + return "message too long for RSA public key size" +} + +func encrypt(c *big.Int, pub *PublicKey, m *big.Int) *big.Int { + e := big.NewInt(int64(pub.E)) + c.Exp(m, e, pub.N) + return c +} + +// EncryptOAEP encrypts the given message with RSA-OAEP. +// The message must be no longer than the length of the public modulus less +// twice the hash length plus 2. +func EncryptOAEP(hash hash.Hash, random io.Reader, pub *PublicKey, msg []byte, label []byte) (out []byte, err os.Error) { + hash.Reset() + k := (pub.N.BitLen() + 7) / 8 + if len(msg) > k-2*hash.Size()-2 { + err = MessageTooLongError{} + return + } + + hash.Write(label) + lHash := hash.Sum() + hash.Reset() + + em := make([]byte, k) + seed := em[1 : 1+hash.Size()] + db := em[1+hash.Size():] + + copy(db[0:hash.Size()], lHash) + db[len(db)-len(msg)-1] = 1 + copy(db[len(db)-len(msg):], msg) + + _, err = io.ReadFull(random, seed) + if err != nil { + return + } + + mgf1XOR(db, hash, seed) + mgf1XOR(seed, hash, db) + + m := new(big.Int) + m.SetBytes(em) + c := encrypt(new(big.Int), pub, m) + out = c.Bytes() + + if len(out) < k { + // If the output is too small, we need to left-pad with zeros. + t := make([]byte, k) + copy(t[k-len(out):], out) + out = t + } + + return +} + +// A DecryptionError represents a failure to decrypt a message. +// It is deliberately vague to avoid adaptive attacks. +type DecryptionError struct{} + +func (DecryptionError) String() string { return "RSA decryption error" } + +// A VerificationError represents a failure to verify a signature. +// It is deliberately vague to avoid adaptive attacks. +type VerificationError struct{} + +func (VerificationError) String() string { return "RSA verification error" } + +// modInverse returns ia, the inverse of a in the multiplicative group of prime +// order n. It requires that a be a member of the group (i.e. less than n). +func modInverse(a, n *big.Int) (ia *big.Int, ok bool) { + g := new(big.Int) + x := new(big.Int) + y := new(big.Int) + big.GcdInt(g, x, y, a, n) + if g.Cmp(bigOne) != 0 { + // In this case, a and n aren't coprime and we cannot calculate + // the inverse. This happens because the values of n are nearly + // prime (being the product of two primes) rather than truly + // prime. + return + } + + if x.Cmp(bigOne) < 0 { + // 0 is not the multiplicative inverse of any element so, if x + // < 1, then x is negative. + x.Add(x, n) + } + + return x, true +} + +// Precompute performs some calculations that speed up private key operations +// in the future. +func (priv *PrivateKey) Precompute() { + if priv.Precomputed.Dp != nil { + return + } + + priv.Precomputed.Dp = new(big.Int).Sub(priv.Primes[0], bigOne) + priv.Precomputed.Dp.Mod(priv.D, priv.Precomputed.Dp) + + priv.Precomputed.Dq = new(big.Int).Sub(priv.Primes[1], bigOne) + priv.Precomputed.Dq.Mod(priv.D, priv.Precomputed.Dq) + + priv.Precomputed.Qinv = new(big.Int).ModInverse(priv.Primes[1], priv.Primes[0]) + + r := new(big.Int).Mul(priv.Primes[0], priv.Primes[1]) + priv.Precomputed.CRTValues = make([]CRTValue, len(priv.Primes)-2) + for i := 2; i < len(priv.Primes); i++ { + prime := priv.Primes[i] + values := &priv.Precomputed.CRTValues[i-2] + + values.Exp = new(big.Int).Sub(prime, bigOne) + values.Exp.Mod(priv.D, values.Exp) + + values.R = new(big.Int).Set(r) + values.Coeff = new(big.Int).ModInverse(r, prime) + + r.Mul(r, prime) + } +} + +// decrypt performs an RSA decryption, resulting in a plaintext integer. If a +// random source is given, RSA blinding is used. +func decrypt(random io.Reader, priv *PrivateKey, c *big.Int) (m *big.Int, err os.Error) { + // TODO(agl): can we get away with reusing blinds? + if c.Cmp(priv.N) > 0 { + err = DecryptionError{} + return + } + + var ir *big.Int + if random != nil { + // Blinding enabled. Blinding involves multiplying c by r^e. + // Then the decryption operation performs (m^e * r^e)^d mod n + // which equals mr mod n. The factor of r can then be removed + // by multiplying by the multiplicative inverse of r. + + var r *big.Int + + for { + r, err = rand.Int(random, priv.N) + if err != nil { + return + } + if r.Cmp(bigZero) == 0 { + r = bigOne + } + var ok bool + ir, ok = modInverse(r, priv.N) + if ok { + break + } + } + bigE := big.NewInt(int64(priv.E)) + rpowe := new(big.Int).Exp(r, bigE, priv.N) + cCopy := new(big.Int).Set(c) + cCopy.Mul(cCopy, rpowe) + cCopy.Mod(cCopy, priv.N) + c = cCopy + } + + if priv.Precomputed.Dp == nil { + m = new(big.Int).Exp(c, priv.D, priv.N) + } else { + // We have the precalculated values needed for the CRT. + m = new(big.Int).Exp(c, priv.Precomputed.Dp, priv.Primes[0]) + m2 := new(big.Int).Exp(c, priv.Precomputed.Dq, priv.Primes[1]) + m.Sub(m, m2) + if m.Sign() < 0 { + m.Add(m, priv.Primes[0]) + } + m.Mul(m, priv.Precomputed.Qinv) + m.Mod(m, priv.Primes[0]) + m.Mul(m, priv.Primes[1]) + m.Add(m, m2) + + for i, values := range priv.Precomputed.CRTValues { + prime := priv.Primes[2+i] + m2.Exp(c, values.Exp, prime) + m2.Sub(m2, m) + m2.Mul(m2, values.Coeff) + m2.Mod(m2, prime) + if m2.Sign() < 0 { + m2.Add(m2, prime) + } + m2.Mul(m2, values.R) + m.Add(m, m2) + } + } + + if ir != nil { + // Unblind. + m.Mul(m, ir) + m.Mod(m, priv.N) + } + + return +} + +// DecryptOAEP decrypts ciphertext using RSA-OAEP. +// If rand != nil, DecryptOAEP uses RSA blinding to avoid timing side-channel attacks. +func DecryptOAEP(hash hash.Hash, random io.Reader, priv *PrivateKey, ciphertext []byte, label []byte) (msg []byte, err os.Error) { + k := (priv.N.BitLen() + 7) / 8 + if len(ciphertext) > k || + k < hash.Size()*2+2 { + err = DecryptionError{} + return + } + + c := new(big.Int).SetBytes(ciphertext) + + m, err := decrypt(random, priv, c) + if err != nil { + return + } + + hash.Write(label) + lHash := hash.Sum() + hash.Reset() + + // Converting the plaintext number to bytes will strip any + // leading zeros so we may have to left pad. We do this unconditionally + // to avoid leaking timing information. (Although we still probably + // leak the number of leading zeros. It's not clear that we can do + // anything about this.) + em := leftPad(m.Bytes(), k) + + firstByteIsZero := subtle.ConstantTimeByteEq(em[0], 0) + + seed := em[1 : hash.Size()+1] + db := em[hash.Size()+1:] + + mgf1XOR(seed, hash, db) + mgf1XOR(db, hash, seed) + + lHash2 := db[0:hash.Size()] + + // We have to validate the plaintext in constant time in order to avoid + // attacks like: J. Manger. A Chosen Ciphertext Attack on RSA Optimal + // Asymmetric Encryption Padding (OAEP) as Standardized in PKCS #1 + // v2.0. In J. Kilian, editor, Advances in Cryptology. + lHash2Good := subtle.ConstantTimeCompare(lHash, lHash2) + + // The remainder of the plaintext must be zero or more 0x00, followed + // by 0x01, followed by the message. + // lookingForIndex: 1 iff we are still looking for the 0x01 + // index: the offset of the first 0x01 byte + // invalid: 1 iff we saw a non-zero byte before the 0x01. + var lookingForIndex, index, invalid int + lookingForIndex = 1 + rest := db[hash.Size():] + + for i := 0; i < len(rest); i++ { + equals0 := subtle.ConstantTimeByteEq(rest[i], 0) + equals1 := subtle.ConstantTimeByteEq(rest[i], 1) + index = subtle.ConstantTimeSelect(lookingForIndex&equals1, i, index) + lookingForIndex = subtle.ConstantTimeSelect(equals1, 0, lookingForIndex) + invalid = subtle.ConstantTimeSelect(lookingForIndex&^equals0, 1, invalid) + } + + if firstByteIsZero&lHash2Good&^invalid&^lookingForIndex != 1 { + err = DecryptionError{} + return + } + + msg = rest[index+1:] + return +} + +// leftPad returns a new slice of length size. The contents of input are right +// aligned in the new slice. +func leftPad(input []byte, size int) (out []byte) { + n := len(input) + if n > size { + n = size + } + out = make([]byte, size) + copy(out[len(out)-n:], input) + return +} diff --git a/src/pkg/crypto/rsa/rsa_test.go b/src/pkg/crypto/rsa/rsa_test.go new file mode 100644 index 000000000..c36bca1cd --- /dev/null +++ b/src/pkg/crypto/rsa/rsa_test.go @@ -0,0 +1,348 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package rsa + +import ( + "big" + "bytes" + "crypto/rand" + "crypto/sha1" + "testing" +) + +func TestKeyGeneration(t *testing.T) { + size := 1024 + if testing.Short() { + size = 128 + } + priv, err := GenerateKey(rand.Reader, size) + if err != nil { + t.Errorf("failed to generate key") + } + testKeyBasics(t, priv) +} + +func Test3PrimeKeyGeneration(t *testing.T) { + if testing.Short() { + return + } + + size := 768 + priv, err := GenerateMultiPrimeKey(rand.Reader, 3, size) + if err != nil { + t.Errorf("failed to generate key") + } + testKeyBasics(t, priv) +} + +func Test4PrimeKeyGeneration(t *testing.T) { + if testing.Short() { + return + } + + size := 768 + priv, err := GenerateMultiPrimeKey(rand.Reader, 4, size) + if err != nil { + t.Errorf("failed to generate key") + } + testKeyBasics(t, priv) +} + +func testKeyBasics(t *testing.T, priv *PrivateKey) { + if err := priv.Validate(); err != nil { + t.Errorf("Validate() failed: %s", err) + } + + pub := &priv.PublicKey + m := big.NewInt(42) + c := encrypt(new(big.Int), pub, m) + + m2, err := decrypt(nil, priv, c) + if err != nil { + t.Errorf("error while decrypting: %s", err) + return + } + if m.Cmp(m2) != 0 { + t.Errorf("got:%v, want:%v (%+v)", m2, m, priv) + } + + m3, err := decrypt(rand.Reader, priv, c) + if err != nil { + t.Errorf("error while decrypting (blind): %s", err) + } + if m.Cmp(m3) != 0 { + t.Errorf("(blind) got:%v, want:%v (%#v)", m3, m, priv) + } +} + +func fromBase10(base10 string) *big.Int { + i := new(big.Int) + i.SetString(base10, 10) + return i +} + +func BenchmarkRSA2048Decrypt(b *testing.B) { + b.StopTimer() + priv := &PrivateKey{ + PublicKey: PublicKey{ + N: fromBase10("14314132931241006650998084889274020608918049032671858325988396851334124245188214251956198731333464217832226406088020736932173064754214329009979944037640912127943488972644697423190955557435910767690712778463524983667852819010259499695177313115447116110358524558307947613422897787329221478860907963827160223559690523660574329011927531289655711860504630573766609239332569210831325633840174683944553667352219670930408593321661375473885147973879086994006440025257225431977751512374815915392249179976902953721486040787792801849818254465486633791826766873076617116727073077821584676715609985777563958286637185868165868520557"), + E: 3, + }, + D: fromBase10("9542755287494004433998723259516013739278699355114572217325597900889416163458809501304132487555642811888150937392013824621448709836142886006653296025093941418628992648429798282127303704957273845127141852309016655778568546006839666463451542076964744073572349705538631742281931858219480985907271975884773482372966847639853897890615456605598071088189838676728836833012254065983259638538107719766738032720239892094196108713378822882383694456030043492571063441943847195939549773271694647657549658603365629458610273821292232646334717612674519997533901052790334279661754176490593041941863932308687197618671528035670452762731"), + Primes: []*big.Int{ + fromBase10("130903255182996722426771613606077755295583329135067340152947172868415809027537376306193179624298874215608270802054347609836776473930072411958753044562214537013874103802006369634761074377213995983876788718033850153719421695468704276694983032644416930879093914927146648402139231293035971427838068945045019075433"), + fromBase10("109348945610485453577574767652527472924289229538286649661240938988020367005475727988253438647560958573506159449538793540472829815903949343191091817779240101054552748665267574271163617694640513549693841337820602726596756351006149518830932261246698766355347898158548465400674856021497190430791824869615170301029"), + }, + } + priv.Precompute() + + c := fromBase10("1000") + + b.StartTimer() + + for i := 0; i < b.N; i++ { + decrypt(nil, priv, c) + } +} + +func Benchmark3PrimeRSA2048Decrypt(b *testing.B) { + b.StopTimer() + priv := &PrivateKey{ + PublicKey: PublicKey{ + N: fromBase10("16346378922382193400538269749936049106320265317511766357599732575277382844051791096569333808598921852351577762718529818072849191122419410612033592401403764925096136759934497687765453905884149505175426053037420486697072448609022753683683718057795566811401938833367954642951433473337066311978821180526439641496973296037000052546108507805269279414789035461158073156772151892452251106173507240488993608650881929629163465099476849643165682709047462010581308719577053905787496296934240246311806555924593059995202856826239801816771116902778517096212527979497399966526283516447337775509777558018145573127308919204297111496233"), + E: 3, + }, + D: fromBase10("10897585948254795600358846499957366070880176878341177571733155050184921896034527397712889205732614568234385175145686545381899460748279607074689061600935843283397424506622998458510302603922766336783617368686090042765718290914099334449154829375179958369993407724946186243249568928237086215759259909861748642124071874879861299389874230489928271621259294894142840428407196932444474088857746123104978617098858619445675532587787023228852383149557470077802718705420275739737958953794088728369933811184572620857678792001136676902250566845618813972833750098806496641114644760255910789397593428910198080271317419213080834885003"), + Primes: []*big.Int{ + fromBase10("1025363189502892836833747188838978207017355117492483312747347695538428729137306368764177201532277413433182799108299960196606011786562992097313508180436744488171474690412562218914213688661311117337381958560443"), + fromBase10("3467903426626310123395340254094941045497208049900750380025518552334536945536837294961497712862519984786362199788654739924501424784631315081391467293694361474867825728031147665777546570788493758372218019373"), + fromBase10("4597024781409332673052708605078359346966325141767460991205742124888960305710298765592730135879076084498363772408626791576005136245060321874472727132746643162385746062759369754202494417496879741537284589047"), + }, + } + priv.Precompute() + + c := fromBase10("1000") + + b.StartTimer() + + for i := 0; i < b.N; i++ { + decrypt(nil, priv, c) + } +} + +type testEncryptOAEPMessage struct { + in []byte + seed []byte + out []byte +} + +type testEncryptOAEPStruct struct { + modulus string + e int + d string + msgs []testEncryptOAEPMessage +} + +func TestEncryptOAEP(t *testing.T) { + sha1 := sha1.New() + n := new(big.Int) + for i, test := range testEncryptOAEPData { + n.SetString(test.modulus, 16) + public := PublicKey{n, test.e} + + for j, message := range test.msgs { + randomSource := bytes.NewBuffer(message.seed) + out, err := EncryptOAEP(sha1, randomSource, &public, message.in, nil) + if err != nil { + t.Errorf("#%d,%d error: %s", i, j, err) + } + if bytes.Compare(out, message.out) != 0 { + t.Errorf("#%d,%d bad result: %x (want %x)", i, j, out, message.out) + } + } + } +} + +func TestDecryptOAEP(t *testing.T) { + random := rand.Reader + + sha1 := sha1.New() + n := new(big.Int) + d := new(big.Int) + for i, test := range testEncryptOAEPData { + n.SetString(test.modulus, 16) + d.SetString(test.d, 16) + private := new(PrivateKey) + private.PublicKey = PublicKey{n, test.e} + private.D = d + + for j, message := range test.msgs { + out, err := DecryptOAEP(sha1, nil, private, message.out, nil) + if err != nil { + t.Errorf("#%d,%d error: %s", i, j, err) + } else if bytes.Compare(out, message.in) != 0 { + t.Errorf("#%d,%d bad result: %#v (want %#v)", i, j, out, message.in) + } + + // Decrypt with blinding. + out, err = DecryptOAEP(sha1, random, private, message.out, nil) + if err != nil { + t.Errorf("#%d,%d (blind) error: %s", i, j, err) + } else if bytes.Compare(out, message.in) != 0 { + t.Errorf("#%d,%d (blind) bad result: %#v (want %#v)", i, j, out, message.in) + } + } + if testing.Short() { + break + } + } +} + +// testEncryptOAEPData contains a subset of the vectors from RSA's "Test vectors for RSA-OAEP". +var testEncryptOAEPData = []testEncryptOAEPStruct{ + // Key 1 + {"a8b3b284af8eb50b387034a860f146c4919f318763cd6c5598c8ae4811a1e0abc4c7e0b082d693a5e7fced675cf4668512772c0cbc64a742c6c630f533c8cc72f62ae833c40bf25842e984bb78bdbf97c0107d55bdb662f5c4e0fab9845cb5148ef7392dd3aaff93ae1e6b667bb3d4247616d4f5ba10d4cfd226de88d39f16fb", + 65537, + "53339cfdb79fc8466a655c7316aca85c55fd8f6dd898fdaf119517ef4f52e8fd8e258df93fee180fa0e4ab29693cd83b152a553d4ac4d1812b8b9fa5af0e7f55fe7304df41570926f3311f15c4d65a732c483116ee3d3d2d0af3549ad9bf7cbfb78ad884f84d5beb04724dc7369b31def37d0cf539e9cfcdd3de653729ead5d1", + []testEncryptOAEPMessage{ + // Example 1.1 + { + []byte{0x66, 0x28, 0x19, 0x4e, 0x12, 0x07, 0x3d, 0xb0, + 0x3b, 0xa9, 0x4c, 0xda, 0x9e, 0xf9, 0x53, 0x23, 0x97, + 0xd5, 0x0d, 0xba, 0x79, 0xb9, 0x87, 0x00, 0x4a, 0xfe, + 0xfe, 0x34, + }, + []byte{0x18, 0xb7, 0x76, 0xea, 0x21, 0x06, 0x9d, 0x69, + 0x77, 0x6a, 0x33, 0xe9, 0x6b, 0xad, 0x48, 0xe1, 0xdd, + 0xa0, 0xa5, 0xef, + }, + []byte{0x35, 0x4f, 0xe6, 0x7b, 0x4a, 0x12, 0x6d, 0x5d, + 0x35, 0xfe, 0x36, 0xc7, 0x77, 0x79, 0x1a, 0x3f, 0x7b, + 0xa1, 0x3d, 0xef, 0x48, 0x4e, 0x2d, 0x39, 0x08, 0xaf, + 0xf7, 0x22, 0xfa, 0xd4, 0x68, 0xfb, 0x21, 0x69, 0x6d, + 0xe9, 0x5d, 0x0b, 0xe9, 0x11, 0xc2, 0xd3, 0x17, 0x4f, + 0x8a, 0xfc, 0xc2, 0x01, 0x03, 0x5f, 0x7b, 0x6d, 0x8e, + 0x69, 0x40, 0x2d, 0xe5, 0x45, 0x16, 0x18, 0xc2, 0x1a, + 0x53, 0x5f, 0xa9, 0xd7, 0xbf, 0xc5, 0xb8, 0xdd, 0x9f, + 0xc2, 0x43, 0xf8, 0xcf, 0x92, 0x7d, 0xb3, 0x13, 0x22, + 0xd6, 0xe8, 0x81, 0xea, 0xa9, 0x1a, 0x99, 0x61, 0x70, + 0xe6, 0x57, 0xa0, 0x5a, 0x26, 0x64, 0x26, 0xd9, 0x8c, + 0x88, 0x00, 0x3f, 0x84, 0x77, 0xc1, 0x22, 0x70, 0x94, + 0xa0, 0xd9, 0xfa, 0x1e, 0x8c, 0x40, 0x24, 0x30, 0x9c, + 0xe1, 0xec, 0xcc, 0xb5, 0x21, 0x00, 0x35, 0xd4, 0x7a, + 0xc7, 0x2e, 0x8a, + }, + }, + // Example 1.2 + { + []byte{0x75, 0x0c, 0x40, 0x47, 0xf5, 0x47, 0xe8, 0xe4, + 0x14, 0x11, 0x85, 0x65, 0x23, 0x29, 0x8a, 0xc9, 0xba, + 0xe2, 0x45, 0xef, 0xaf, 0x13, 0x97, 0xfb, 0xe5, 0x6f, + 0x9d, 0xd5, + }, + []byte{0x0c, 0xc7, 0x42, 0xce, 0x4a, 0x9b, 0x7f, 0x32, + 0xf9, 0x51, 0xbc, 0xb2, 0x51, 0xef, 0xd9, 0x25, 0xfe, + 0x4f, 0xe3, 0x5f, + }, + []byte{0x64, 0x0d, 0xb1, 0xac, 0xc5, 0x8e, 0x05, 0x68, + 0xfe, 0x54, 0x07, 0xe5, 0xf9, 0xb7, 0x01, 0xdf, 0xf8, + 0xc3, 0xc9, 0x1e, 0x71, 0x6c, 0x53, 0x6f, 0xc7, 0xfc, + 0xec, 0x6c, 0xb5, 0xb7, 0x1c, 0x11, 0x65, 0x98, 0x8d, + 0x4a, 0x27, 0x9e, 0x15, 0x77, 0xd7, 0x30, 0xfc, 0x7a, + 0x29, 0x93, 0x2e, 0x3f, 0x00, 0xc8, 0x15, 0x15, 0x23, + 0x6d, 0x8d, 0x8e, 0x31, 0x01, 0x7a, 0x7a, 0x09, 0xdf, + 0x43, 0x52, 0xd9, 0x04, 0xcd, 0xeb, 0x79, 0xaa, 0x58, + 0x3a, 0xdc, 0xc3, 0x1e, 0xa6, 0x98, 0xa4, 0xc0, 0x52, + 0x83, 0xda, 0xba, 0x90, 0x89, 0xbe, 0x54, 0x91, 0xf6, + 0x7c, 0x1a, 0x4e, 0xe4, 0x8d, 0xc7, 0x4b, 0xbb, 0xe6, + 0x64, 0x3a, 0xef, 0x84, 0x66, 0x79, 0xb4, 0xcb, 0x39, + 0x5a, 0x35, 0x2d, 0x5e, 0xd1, 0x15, 0x91, 0x2d, 0xf6, + 0x96, 0xff, 0xe0, 0x70, 0x29, 0x32, 0x94, 0x6d, 0x71, + 0x49, 0x2b, 0x44, + }, + }, + // Example 1.3 + { + []byte{0xd9, 0x4a, 0xe0, 0x83, 0x2e, 0x64, 0x45, 0xce, + 0x42, 0x33, 0x1c, 0xb0, 0x6d, 0x53, 0x1a, 0x82, 0xb1, + 0xdb, 0x4b, 0xaa, 0xd3, 0x0f, 0x74, 0x6d, 0xc9, 0x16, + 0xdf, 0x24, 0xd4, 0xe3, 0xc2, 0x45, 0x1f, 0xff, 0x59, + 0xa6, 0x42, 0x3e, 0xb0, 0xe1, 0xd0, 0x2d, 0x4f, 0xe6, + 0x46, 0xcf, 0x69, 0x9d, 0xfd, 0x81, 0x8c, 0x6e, 0x97, + 0xb0, 0x51, + }, + []byte{0x25, 0x14, 0xdf, 0x46, 0x95, 0x75, 0x5a, 0x67, + 0xb2, 0x88, 0xea, 0xf4, 0x90, 0x5c, 0x36, 0xee, 0xc6, + 0x6f, 0xd2, 0xfd, + }, + []byte{0x42, 0x37, 0x36, 0xed, 0x03, 0x5f, 0x60, 0x26, + 0xaf, 0x27, 0x6c, 0x35, 0xc0, 0xb3, 0x74, 0x1b, 0x36, + 0x5e, 0x5f, 0x76, 0xca, 0x09, 0x1b, 0x4e, 0x8c, 0x29, + 0xe2, 0xf0, 0xbe, 0xfe, 0xe6, 0x03, 0x59, 0x5a, 0xa8, + 0x32, 0x2d, 0x60, 0x2d, 0x2e, 0x62, 0x5e, 0x95, 0xeb, + 0x81, 0xb2, 0xf1, 0xc9, 0x72, 0x4e, 0x82, 0x2e, 0xca, + 0x76, 0xdb, 0x86, 0x18, 0xcf, 0x09, 0xc5, 0x34, 0x35, + 0x03, 0xa4, 0x36, 0x08, 0x35, 0xb5, 0x90, 0x3b, 0xc6, + 0x37, 0xe3, 0x87, 0x9f, 0xb0, 0x5e, 0x0e, 0xf3, 0x26, + 0x85, 0xd5, 0xae, 0xc5, 0x06, 0x7c, 0xd7, 0xcc, 0x96, + 0xfe, 0x4b, 0x26, 0x70, 0xb6, 0xea, 0xc3, 0x06, 0x6b, + 0x1f, 0xcf, 0x56, 0x86, 0xb6, 0x85, 0x89, 0xaa, 0xfb, + 0x7d, 0x62, 0x9b, 0x02, 0xd8, 0xf8, 0x62, 0x5c, 0xa3, + 0x83, 0x36, 0x24, 0xd4, 0x80, 0x0f, 0xb0, 0x81, 0xb1, + 0xcf, 0x94, 0xeb, + }, + }, + }, + }, + // Key 10 + {"ae45ed5601cec6b8cc05f803935c674ddbe0d75c4c09fd7951fc6b0caec313a8df39970c518bffba5ed68f3f0d7f22a4029d413f1ae07e4ebe9e4177ce23e7f5404b569e4ee1bdcf3c1fb03ef113802d4f855eb9b5134b5a7c8085adcae6fa2fa1417ec3763be171b0c62b760ede23c12ad92b980884c641f5a8fac26bdad4a03381a22fe1b754885094c82506d4019a535a286afeb271bb9ba592de18dcf600c2aeeae56e02f7cf79fc14cf3bdc7cd84febbbf950ca90304b2219a7aa063aefa2c3c1980e560cd64afe779585b6107657b957857efde6010988ab7de417fc88d8f384c4e6e72c3f943e0c31c0c4a5cc36f879d8a3ac9d7d59860eaada6b83bb", + 65537, + "056b04216fe5f354ac77250a4b6b0c8525a85c59b0bd80c56450a22d5f438e596a333aa875e291dd43f48cb88b9d5fc0d499f9fcd1c397f9afc070cd9e398c8d19e61db7c7410a6b2675dfbf5d345b804d201add502d5ce2dfcb091ce9997bbebe57306f383e4d588103f036f7e85d1934d152a323e4a8db451d6f4a5b1b0f102cc150e02feee2b88dea4ad4c1baccb24d84072d14e1d24a6771f7408ee30564fb86d4393a34bcf0b788501d193303f13a2284b001f0f649eaf79328d4ac5c430ab4414920a9460ed1b7bc40ec653e876d09abc509ae45b525190116a0c26101848298509c1c3bf3a483e7274054e15e97075036e989f60932807b5257751e79", + []testEncryptOAEPMessage{ + // Example 10.1 + { + []byte{0x8b, 0xba, 0x6b, 0xf8, 0x2a, 0x6c, 0x0f, 0x86, + 0xd5, 0xf1, 0x75, 0x6e, 0x97, 0x95, 0x68, 0x70, 0xb0, + 0x89, 0x53, 0xb0, 0x6b, 0x4e, 0xb2, 0x05, 0xbc, 0x16, + 0x94, 0xee, + }, + []byte{0x47, 0xe1, 0xab, 0x71, 0x19, 0xfe, 0xe5, 0x6c, + 0x95, 0xee, 0x5e, 0xaa, 0xd8, 0x6f, 0x40, 0xd0, 0xaa, + 0x63, 0xbd, 0x33, + }, + []byte{0x53, 0xea, 0x5d, 0xc0, 0x8c, 0xd2, 0x60, 0xfb, + 0x3b, 0x85, 0x85, 0x67, 0x28, 0x7f, 0xa9, 0x15, 0x52, + 0xc3, 0x0b, 0x2f, 0xeb, 0xfb, 0xa2, 0x13, 0xf0, 0xae, + 0x87, 0x70, 0x2d, 0x06, 0x8d, 0x19, 0xba, 0xb0, 0x7f, + 0xe5, 0x74, 0x52, 0x3d, 0xfb, 0x42, 0x13, 0x9d, 0x68, + 0xc3, 0xc5, 0xaf, 0xee, 0xe0, 0xbf, 0xe4, 0xcb, 0x79, + 0x69, 0xcb, 0xf3, 0x82, 0xb8, 0x04, 0xd6, 0xe6, 0x13, + 0x96, 0x14, 0x4e, 0x2d, 0x0e, 0x60, 0x74, 0x1f, 0x89, + 0x93, 0xc3, 0x01, 0x4b, 0x58, 0xb9, 0xb1, 0x95, 0x7a, + 0x8b, 0xab, 0xcd, 0x23, 0xaf, 0x85, 0x4f, 0x4c, 0x35, + 0x6f, 0xb1, 0x66, 0x2a, 0xa7, 0x2b, 0xfc, 0xc7, 0xe5, + 0x86, 0x55, 0x9d, 0xc4, 0x28, 0x0d, 0x16, 0x0c, 0x12, + 0x67, 0x85, 0xa7, 0x23, 0xeb, 0xee, 0xbe, 0xff, 0x71, + 0xf1, 0x15, 0x94, 0x44, 0x0a, 0xae, 0xf8, 0x7d, 0x10, + 0x79, 0x3a, 0x87, 0x74, 0xa2, 0x39, 0xd4, 0xa0, 0x4c, + 0x87, 0xfe, 0x14, 0x67, 0xb9, 0xda, 0xf8, 0x52, 0x08, + 0xec, 0x6c, 0x72, 0x55, 0x79, 0x4a, 0x96, 0xcc, 0x29, + 0x14, 0x2f, 0x9a, 0x8b, 0xd4, 0x18, 0xe3, 0xc1, 0xfd, + 0x67, 0x34, 0x4b, 0x0c, 0xd0, 0x82, 0x9d, 0xf3, 0xb2, + 0xbe, 0xc6, 0x02, 0x53, 0x19, 0x62, 0x93, 0xc6, 0xb3, + 0x4d, 0x3f, 0x75, 0xd3, 0x2f, 0x21, 0x3d, 0xd4, 0x5c, + 0x62, 0x73, 0xd5, 0x05, 0xad, 0xf4, 0xcc, 0xed, 0x10, + 0x57, 0xcb, 0x75, 0x8f, 0xc2, 0x6a, 0xee, 0xfa, 0x44, + 0x12, 0x55, 0xed, 0x4e, 0x64, 0xc1, 0x99, 0xee, 0x07, + 0x5e, 0x7f, 0x16, 0x64, 0x61, 0x82, 0xfd, 0xb4, 0x64, + 0x73, 0x9b, 0x68, 0xab, 0x5d, 0xaf, 0xf0, 0xe6, 0x3e, + 0x95, 0x52, 0x01, 0x68, 0x24, 0xf0, 0x54, 0xbf, 0x4d, + 0x3c, 0x8c, 0x90, 0xa9, 0x7b, 0xb6, 0xb6, 0x55, 0x32, + 0x84, 0xeb, 0x42, 0x9f, 0xcc, + }, + }, + }, + }, +} diff --git a/src/pkg/crypto/sha1/Makefile b/src/pkg/crypto/sha1/Makefile new file mode 100644 index 000000000..81ac38c0b --- /dev/null +++ b/src/pkg/crypto/sha1/Makefile @@ -0,0 +1,12 @@ +# Copyright 2009 The Go Authors. All rights reserved. +# Use of this source code is governed by a BSD-style +# license that can be found in the LICENSE file. + +include ../../../Make.inc + +TARG=crypto/sha1 +GOFILES=\ + sha1.go\ + sha1block.go\ + +include ../../../Make.pkg diff --git a/src/pkg/crypto/sha1/sha1.go b/src/pkg/crypto/sha1/sha1.go new file mode 100644 index 000000000..788d1ff55 --- /dev/null +++ b/src/pkg/crypto/sha1/sha1.go @@ -0,0 +1,119 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Package sha1 implements the SHA1 hash algorithm as defined in RFC 3174. +package sha1 + +import ( + "crypto" + "hash" + "os" +) + +func init() { + crypto.RegisterHash(crypto.SHA1, New) +} + +// The size of a SHA1 checksum in bytes. +const Size = 20 + +const ( + _Chunk = 64 + _Init0 = 0x67452301 + _Init1 = 0xEFCDAB89 + _Init2 = 0x98BADCFE + _Init3 = 0x10325476 + _Init4 = 0xC3D2E1F0 +) + +// digest represents the partial evaluation of a checksum. +type digest struct { + h [5]uint32 + x [_Chunk]byte + nx int + len uint64 +} + +func (d *digest) Reset() { + d.h[0] = _Init0 + d.h[1] = _Init1 + d.h[2] = _Init2 + d.h[3] = _Init3 + d.h[4] = _Init4 + d.nx = 0 + d.len = 0 +} + +// New returns a new hash.Hash computing the SHA1 checksum. +func New() hash.Hash { + d := new(digest) + d.Reset() + return d +} + +func (d *digest) Size() int { return Size } + +func (d *digest) Write(p []byte) (nn int, err os.Error) { + nn = len(p) + d.len += uint64(nn) + if d.nx > 0 { + n := len(p) + if n > _Chunk-d.nx { + n = _Chunk - d.nx + } + for i := 0; i < n; i++ { + d.x[d.nx+i] = p[i] + } + d.nx += n + if d.nx == _Chunk { + _Block(d, d.x[0:]) + d.nx = 0 + } + p = p[n:] + } + n := _Block(d, p) + p = p[n:] + if len(p) > 0 { + d.nx = copy(d.x[:], p) + } + return +} + +func (d0 *digest) Sum() []byte { + // Make a copy of d0 so that caller can keep writing and summing. + d := new(digest) + *d = *d0 + + // Padding. Add a 1 bit and 0 bits until 56 bytes mod 64. + len := d.len + var tmp [64]byte + tmp[0] = 0x80 + if len%64 < 56 { + d.Write(tmp[0 : 56-len%64]) + } else { + d.Write(tmp[0 : 64+56-len%64]) + } + + // Length in bits. + len <<= 3 + for i := uint(0); i < 8; i++ { + tmp[i] = byte(len >> (56 - 8*i)) + } + d.Write(tmp[0:8]) + + if d.nx != 0 { + panic("d.nx != 0") + } + + p := make([]byte, 20) + j := 0 + for _, s := range d.h { + p[j+0] = byte(s >> 24) + p[j+1] = byte(s >> 16) + p[j+2] = byte(s >> 8) + p[j+3] = byte(s >> 0) + j += 4 + } + return p +} diff --git a/src/pkg/crypto/sha1/sha1_test.go b/src/pkg/crypto/sha1/sha1_test.go new file mode 100644 index 000000000..2712fe35e --- /dev/null +++ b/src/pkg/crypto/sha1/sha1_test.go @@ -0,0 +1,73 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// SHA1 hash algorithm. See RFC 3174. + +package sha1 + +import ( + "fmt" + "io" + "testing" +) + +type sha1Test struct { + out string + in string +} + +var golden = []sha1Test{ + {"da39a3ee5e6b4b0d3255bfef95601890afd80709", ""}, + {"86f7e437faa5a7fce15d1ddcb9eaeaea377667b8", "a"}, + {"da23614e02469a0d7c7bd1bdab5c9c474b1904dc", "ab"}, + {"a9993e364706816aba3e25717850c26c9cd0d89d", "abc"}, + {"81fe8bfe87576c3ecb22426f8e57847382917acf", "abcd"}, + {"03de6c570bfe24bfc328ccd7ca46b76eadaf4334", "abcde"}, + {"1f8ac10f23c5b5bc1167bda84b833e5c057a77d2", "abcdef"}, + {"2fb5e13419fc89246865e7a324f476ec624e8740", "abcdefg"}, + {"425af12a0743502b322e93a015bcf868e324d56a", "abcdefgh"}, + {"c63b19f1e4c8b5f76b25c49b8b87f57d8e4872a1", "abcdefghi"}, + {"d68c19a0a345b7eab78d5e11e991c026ec60db63", "abcdefghij"}, + {"ebf81ddcbe5bf13aaabdc4d65354fdf2044f38a7", "Discard medicine more than two years old."}, + {"e5dea09392dd886ca63531aaa00571dc07554bb6", "He who has a shady past knows that nice guys finish last."}, + {"45988f7234467b94e3e9494434c96ee3609d8f8f", "I wouldn't marry him with a ten foot pole."}, + {"55dee037eb7460d5a692d1ce11330b260e40c988", "Free! Free!/A trip/to Mars/for 900/empty jars/Burma Shave"}, + {"b7bc5fb91080c7de6b582ea281f8a396d7c0aee8", "The days of the digital watch are numbered. -Tom Stoppard"}, + {"c3aed9358f7c77f523afe86135f06b95b3999797", "Nepal premier won't resign."}, + {"6e29d302bf6e3a5e4305ff318d983197d6906bb9", "For every action there is an equal and opposite government program."}, + {"597f6a540010f94c15d71806a99a2c8710e747bd", "His money is twice tainted: 'taint yours and 'taint mine."}, + {"6859733b2590a8a091cecf50086febc5ceef1e80", "There is no reason for any individual to have a computer in their home. -Ken Olsen, 1977"}, + {"514b2630ec089b8aee18795fc0cf1f4860cdacad", "It's a tiny change to the code and not completely disgusting. - Bob Manchek"}, + {"c5ca0d4a7b6676fc7aa72caa41cc3d5df567ed69", "size: a.out: bad magic"}, + {"74c51fa9a04eadc8c1bbeaa7fc442f834b90a00a", "The major problem is with sendmail. -Mark Horton"}, + {"0b4c4ce5f52c3ad2821852a8dc00217fa18b8b66", "Give me a rock, paper and scissors and I will move the world. CCFestoon"}, + {"3ae7937dd790315beb0f48330e8642237c61550a", "If the enemy is within range, then so are you."}, + {"410a2b296df92b9a47412b13281df8f830a9f44b", "It's well we cannot hear the screams/That we create in others' dreams."}, + {"841e7c85ca1adcddbdd0187f1289acb5c642f7f5", "You remind me of a TV show, but that's all right: I watch it anyway."}, + {"163173b825d03b952601376b25212df66763e1db", "C is as portable as Stonehedge!!"}, + {"32b0377f2687eb88e22106f133c586ab314d5279", "Even if I could be Shakespeare, I think I should still choose to be Faraday. - A. Huxley"}, + {"0885aaf99b569542fd165fa44e322718f4a984e0", "The fugacity of a constituent in a mixture of gases at a given temperature is proportional to its mole fraction. Lewis-Randall Rule"}, + {"6627d6904d71420b0bf3886ab629623538689f45", "How can you write a big system without C++? -Paul Glick"}, +} + +func TestGolden(t *testing.T) { + for i := 0; i < len(golden); i++ { + g := golden[i] + c := New() + for j := 0; j < 3; j++ { + if j < 2 { + io.WriteString(c, g.in) + } else { + io.WriteString(c, g.in[0:len(g.in)/2]) + c.Sum() + io.WriteString(c, g.in[len(g.in)/2:]) + } + s := fmt.Sprintf("%x", c.Sum()) + if s != g.out { + t.Fatalf("sha1[%d](%s) = %s want %s", j, g.in, s, g.out) + } + c.Reset() + } + } +} diff --git a/src/pkg/crypto/sha1/sha1block.go b/src/pkg/crypto/sha1/sha1block.go new file mode 100644 index 000000000..b5d32af70 --- /dev/null +++ b/src/pkg/crypto/sha1/sha1block.go @@ -0,0 +1,81 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// SHA1 block step. +// In its own file so that a faster assembly or C version +// can be substituted easily. + +package sha1 + +const ( + _K0 = 0x5A827999 + _K1 = 0x6ED9EBA1 + _K2 = 0x8F1BBCDC + _K3 = 0xCA62C1D6 +) + +func _Block(dig *digest, p []byte) int { + var w [80]uint32 + + n := 0 + h0, h1, h2, h3, h4 := dig.h[0], dig.h[1], dig.h[2], dig.h[3], dig.h[4] + for len(p) >= _Chunk { + // Can interlace the computation of w with the + // rounds below if needed for speed. + for i := 0; i < 16; i++ { + j := i * 4 + w[i] = uint32(p[j])<<24 | uint32(p[j+1])<<16 | uint32(p[j+2])<<8 | uint32(p[j+3]) + } + for i := 16; i < 80; i++ { + tmp := w[i-3] ^ w[i-8] ^ w[i-14] ^ w[i-16] + w[i] = tmp<<1 | tmp>>(32-1) + } + + a, b, c, d, e := h0, h1, h2, h3, h4 + + // Each of the four 20-iteration rounds + // differs only in the computation of f and + // the choice of K (_K0, _K1, etc). + for i := 0; i < 20; i++ { + f := b&c | (^b)&d + a5 := a<<5 | a>>(32-5) + b30 := b<<30 | b>>(32-30) + t := a5 + f + e + w[i] + _K0 + a, b, c, d, e = t, a, b30, c, d + } + for i := 20; i < 40; i++ { + f := b ^ c ^ d + a5 := a<<5 | a>>(32-5) + b30 := b<<30 | b>>(32-30) + t := a5 + f + e + w[i] + _K1 + a, b, c, d, e = t, a, b30, c, d + } + for i := 40; i < 60; i++ { + f := b&c | b&d | c&d + a5 := a<<5 | a>>(32-5) + b30 := b<<30 | b>>(32-30) + t := a5 + f + e + w[i] + _K2 + a, b, c, d, e = t, a, b30, c, d + } + for i := 60; i < 80; i++ { + f := b ^ c ^ d + a5 := a<<5 | a>>(32-5) + b30 := b<<30 | b>>(32-30) + t := a5 + f + e + w[i] + _K3 + a, b, c, d, e = t, a, b30, c, d + } + + h0 += a + h1 += b + h2 += c + h3 += d + h4 += e + + p = p[_Chunk:] + n += _Chunk + } + + dig.h[0], dig.h[1], dig.h[2], dig.h[3], dig.h[4] = h0, h1, h2, h3, h4 + return n +} diff --git a/src/pkg/crypto/sha256/Makefile b/src/pkg/crypto/sha256/Makefile new file mode 100644 index 000000000..97fe4d8e6 --- /dev/null +++ b/src/pkg/crypto/sha256/Makefile @@ -0,0 +1,12 @@ +# Copyright 2009 The Go Authors. All rights reserved. +# Use of this source code is governed by a BSD-style +# license that can be found in the LICENSE file. + +include ../../../Make.inc + +TARG=crypto/sha256 +GOFILES=\ + sha256.go\ + sha256block.go\ + +include ../../../Make.pkg diff --git a/src/pkg/crypto/sha256/sha256.go b/src/pkg/crypto/sha256/sha256.go new file mode 100644 index 000000000..a2c058d18 --- /dev/null +++ b/src/pkg/crypto/sha256/sha256.go @@ -0,0 +1,166 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Package sha256 implements the SHA224 and SHA256 hash algorithms as defined +// in FIPS 180-2. +package sha256 + +import ( + "crypto" + "hash" + "os" +) + +func init() { + crypto.RegisterHash(crypto.SHA224, New224) + crypto.RegisterHash(crypto.SHA256, New) +} + +// The size of a SHA256 checksum in bytes. +const Size = 32 + +// The size of a SHA224 checksum in bytes. +const Size224 = 28 + +const ( + _Chunk = 64 + _Init0 = 0x6A09E667 + _Init1 = 0xBB67AE85 + _Init2 = 0x3C6EF372 + _Init3 = 0xA54FF53A + _Init4 = 0x510E527F + _Init5 = 0x9B05688C + _Init6 = 0x1F83D9AB + _Init7 = 0x5BE0CD19 + _Init0_224 = 0xC1059ED8 + _Init1_224 = 0x367CD507 + _Init2_224 = 0x3070DD17 + _Init3_224 = 0xF70E5939 + _Init4_224 = 0xFFC00B31 + _Init5_224 = 0x68581511 + _Init6_224 = 0x64F98FA7 + _Init7_224 = 0xBEFA4FA4 +) + +// digest represents the partial evaluation of a checksum. +type digest struct { + h [8]uint32 + x [_Chunk]byte + nx int + len uint64 + is224 bool // mark if this digest is SHA-224 +} + +func (d *digest) Reset() { + if !d.is224 { + d.h[0] = _Init0 + d.h[1] = _Init1 + d.h[2] = _Init2 + d.h[3] = _Init3 + d.h[4] = _Init4 + d.h[5] = _Init5 + d.h[6] = _Init6 + d.h[7] = _Init7 + } else { + d.h[0] = _Init0_224 + d.h[1] = _Init1_224 + d.h[2] = _Init2_224 + d.h[3] = _Init3_224 + d.h[4] = _Init4_224 + d.h[5] = _Init5_224 + d.h[6] = _Init6_224 + d.h[7] = _Init7_224 + } + d.nx = 0 + d.len = 0 +} + +// New returns a new hash.Hash computing the SHA256 checksum. +func New() hash.Hash { + d := new(digest) + d.Reset() + return d +} + +// New224 returns a new hash.Hash computing the SHA224 checksum. +func New224() hash.Hash { + d := new(digest) + d.is224 = true + d.Reset() + return d +} + +func (d *digest) Size() int { + if !d.is224 { + return Size + } + return Size224 +} + +func (d *digest) Write(p []byte) (nn int, err os.Error) { + nn = len(p) + d.len += uint64(nn) + if d.nx > 0 { + n := len(p) + if n > _Chunk-d.nx { + n = _Chunk - d.nx + } + for i := 0; i < n; i++ { + d.x[d.nx+i] = p[i] + } + d.nx += n + if d.nx == _Chunk { + _Block(d, d.x[0:]) + d.nx = 0 + } + p = p[n:] + } + n := _Block(d, p) + p = p[n:] + if len(p) > 0 { + d.nx = copy(d.x[:], p) + } + return +} + +func (d0 *digest) Sum() []byte { + // Make a copy of d0 so that caller can keep writing and summing. + d := new(digest) + *d = *d0 + + // Padding. Add a 1 bit and 0 bits until 56 bytes mod 64. + len := d.len + var tmp [64]byte + tmp[0] = 0x80 + if len%64 < 56 { + d.Write(tmp[0 : 56-len%64]) + } else { + d.Write(tmp[0 : 64+56-len%64]) + } + + // Length in bits. + len <<= 3 + for i := uint(0); i < 8; i++ { + tmp[i] = byte(len >> (56 - 8*i)) + } + d.Write(tmp[0:8]) + + if d.nx != 0 { + panic("d.nx != 0") + } + + p := make([]byte, 32) + j := 0 + for _, s := range d.h { + p[j+0] = byte(s >> 24) + p[j+1] = byte(s >> 16) + p[j+2] = byte(s >> 8) + p[j+3] = byte(s >> 0) + j += 4 + } + if d.is224 { + return p[0:28] + } + return p +} diff --git a/src/pkg/crypto/sha256/sha256_test.go b/src/pkg/crypto/sha256/sha256_test.go new file mode 100644 index 000000000..42a3fa7a0 --- /dev/null +++ b/src/pkg/crypto/sha256/sha256_test.go @@ -0,0 +1,125 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// SHA256 hash algorithm. See FIPS 180-2. + +package sha256 + +import ( + "fmt" + "io" + "testing" +) + +type sha256Test struct { + out string + in string +} + +var golden = []sha256Test{ + {"e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", ""}, + {"ca978112ca1bbdcafac231b39a23dc4da786eff8147c4e72b9807785afee48bb", "a"}, + {"fb8e20fc2e4c3f248c60c39bd652f3c1347298bb977b8b4d5903b85055620603", "ab"}, + {"ba7816bf8f01cfea414140de5dae2223b00361a396177a9cb410ff61f20015ad", "abc"}, + {"88d4266fd4e6338d13b845fcf289579d209c897823b9217da3e161936f031589", "abcd"}, + {"36bbe50ed96841d10443bcb670d6554f0a34b761be67ec9c4a8ad2c0c44ca42c", "abcde"}, + {"bef57ec7f53a6d40beb640a780a639c83bc29ac8a9816f1fc6c5c6dcd93c4721", "abcdef"}, + {"7d1a54127b222502f5b79b5fb0803061152a44f92b37e23c6527baf665d4da9a", "abcdefg"}, + {"9c56cc51b374c3ba189210d5b6d4bf57790d351c96c47c02190ecf1e430635ab", "abcdefgh"}, + {"19cc02f26df43cc571bc9ed7b0c4d29224a3ec229529221725ef76d021c8326f", "abcdefghi"}, + {"72399361da6a7754fec986dca5b7cbaf1c810a28ded4abaf56b2106d06cb78b0", "abcdefghij"}, + {"a144061c271f152da4d151034508fed1c138b8c976339de229c3bb6d4bbb4fce", "Discard medicine more than two years old."}, + {"6dae5caa713a10ad04b46028bf6dad68837c581616a1589a265a11288d4bb5c4", "He who has a shady past knows that nice guys finish last."}, + {"ae7a702a9509039ddbf29f0765e70d0001177914b86459284dab8b348c2dce3f", "I wouldn't marry him with a ten foot pole."}, + {"6748450b01c568586715291dfa3ee018da07d36bb7ea6f180c1af6270215c64f", "Free! Free!/A trip/to Mars/for 900/empty jars/Burma Shave"}, + {"14b82014ad2b11f661b5ae6a99b75105c2ffac278cd071cd6c05832793635774", "The days of the digital watch are numbered. -Tom Stoppard"}, + {"7102cfd76e2e324889eece5d6c41921b1e142a4ac5a2692be78803097f6a48d8", "Nepal premier won't resign."}, + {"23b1018cd81db1d67983c5f7417c44da9deb582459e378d7a068552ea649dc9f", "For every action there is an equal and opposite government program."}, + {"8001f190dfb527261c4cfcab70c98e8097a7a1922129bc4096950e57c7999a5a", "His money is twice tainted: 'taint yours and 'taint mine."}, + {"8c87deb65505c3993eb24b7a150c4155e82eee6960cf0c3a8114ff736d69cad5", "There is no reason for any individual to have a computer in their home. -Ken Olsen, 1977"}, + {"bfb0a67a19cdec3646498b2e0f751bddc41bba4b7f30081b0b932aad214d16d7", "It's a tiny change to the code and not completely disgusting. - Bob Manchek"}, + {"7f9a0b9bf56332e19f5a0ec1ad9c1425a153da1c624868fda44561d6b74daf36", "size: a.out: bad magic"}, + {"b13f81b8aad9e3666879af19886140904f7f429ef083286195982a7588858cfc", "The major problem is with sendmail. -Mark Horton"}, + {"b26c38d61519e894480c70c8374ea35aa0ad05b2ae3d6674eec5f52a69305ed4", "Give me a rock, paper and scissors and I will move the world. CCFestoon"}, + {"049d5e26d4f10222cd841a119e38bd8d2e0d1129728688449575d4ff42b842c1", "If the enemy is within range, then so are you."}, + {"0e116838e3cc1c1a14cd045397e29b4d087aa11b0853fc69ec82e90330d60949", "It's well we cannot hear the screams/That we create in others' dreams."}, + {"4f7d8eb5bcf11de2a56b971021a444aa4eafd6ecd0f307b5109e4e776cd0fe46", "You remind me of a TV show, but that's all right: I watch it anyway."}, + {"61c0cc4c4bd8406d5120b3fb4ebc31ce87667c162f29468b3c779675a85aebce", "C is as portable as Stonehedge!!"}, + {"1fb2eb3688093c4a3f80cd87a5547e2ce940a4f923243a79a2a1e242220693ac", "Even if I could be Shakespeare, I think I should still choose to be Faraday. - A. Huxley"}, + {"395585ce30617b62c80b93e8208ce866d4edc811a177fdb4b82d3911d8696423", "The fugacity of a constituent in a mixture of gases at a given temperature is proportional to its mole fraction. Lewis-Randall Rule"}, + {"4f9b189a13d030838269dce846b16a1ce9ce81fe63e65de2f636863336a98fe6", "How can you write a big system without C++? -Paul Glick"}, +} + +var golden224 = []sha256Test{ + {"d14a028c2a3a2bc9476102bb288234c415a2b01f828ea62ac5b3e42f", ""}, + {"abd37534c7d9a2efb9465de931cd7055ffdb8879563ae98078d6d6d5", "a"}, + {"db3cda86d4429a1d39c148989566b38f7bda0156296bd364ba2f878b", "ab"}, + {"23097d223405d8228642a477bda255b32aadbce4bda0b3f7e36c9da7", "abc"}, + {"a76654d8e3550e9a2d67a0eeb6c67b220e5885eddd3fde135806e601", "abcd"}, + {"bdd03d560993e675516ba5a50638b6531ac2ac3d5847c61916cfced6", "abcde"}, + {"7043631cb415556a275a4ebecb802c74ee9f6153908e1792a90b6a98", "abcdef"}, + {"d1884e711701ad81abe0c77a3b0ea12e19ba9af64077286c72fc602d", "abcdefg"}, + {"17eb7d40f0356f8598e89eafad5f6c759b1f822975d9c9b737c8a517", "abcdefgh"}, + {"aeb35915346c584db820d2de7af3929ffafef9222a9bcb26516c7334", "abcdefghi"}, + {"d35e1e5af29ddb0d7e154357df4ad9842afee527c689ee547f753188", "abcdefghij"}, + {"19297f1cef7ddc8a7e947f5c5a341e10f7245045e425db67043988d7", "Discard medicine more than two years old."}, + {"0f10c2eb436251f777fbbd125e260d36aecf180411726c7c885f599a", "He who has a shady past knows that nice guys finish last."}, + {"4d1842104919f314cad8a3cd20b3cba7e8ed3e7abed62b57441358f6", "I wouldn't marry him with a ten foot pole."}, + {"a8ba85c6fe0c48fbffc72bbb2f03fcdbc87ae2dc7a56804d1590fb3b", "Free! Free!/A trip/to Mars/for 900/empty jars/Burma Shave"}, + {"5543fbab26e67e8885b1a852d567d1cb8b9bfe42e0899584c50449a9", "The days of the digital watch are numbered. -Tom Stoppard"}, + {"65ca107390f5da9efa05d28e57b221657edc7e43a9a18fb15b053ddb", "Nepal premier won't resign."}, + {"84953962be366305a9cc9b5cd16ed019edc37ac96c0deb3e12cca116", "For every action there is an equal and opposite government program."}, + {"35a189ce987151dfd00b3577583cc6a74b9869eecf894459cb52038d", "His money is twice tainted: 'taint yours and 'taint mine."}, + {"2fc333713983edfd4ef2c0da6fb6d6415afb94987c91e4069eb063e6", "There is no reason for any individual to have a computer in their home. -Ken Olsen, 1977"}, + {"cbe32d38d577a1b355960a4bc3c659c2dc4670859a19777a875842c4", "It's a tiny change to the code and not completely disgusting. - Bob Manchek"}, + {"a2dc118ce959e027576413a7b440c875cdc8d40df9141d6ef78a57e1", "size: a.out: bad magic"}, + {"d10787e24052bcff26dc484787a54ed819e4e4511c54890ee977bf81", "The major problem is with sendmail. -Mark Horton"}, + {"62efcf16ab8a893acdf2f348aaf06b63039ff1bf55508c830532c9fb", "Give me a rock, paper and scissors and I will move the world. CCFestoon"}, + {"3e9b7e4613c59f58665104c5fa86c272db5d3a2ff30df5bb194a5c99", "If the enemy is within range, then so are you."}, + {"5999c208b8bdf6d471bb7c359ac5b829e73a8211dff686143a4e7f18", "It's well we cannot hear the screams/That we create in others' dreams."}, + {"3b2d67ff54eabc4ef737b14edf87c64280ef582bcdf2a6d56908b405", "You remind me of a TV show, but that's all right: I watch it anyway."}, + {"d0733595d20e4d3d6b5c565a445814d1bbb2fd08b9a3b8ffb97930c6", "C is as portable as Stonehedge!!"}, + {"43fb8aeed8a833175c9295c1165415f98c866ef08a4922959d673507", "Even if I could be Shakespeare, I think I should still choose to be Faraday. - A. Huxley"}, + {"ec18e66e93afc4fb1604bc2baedbfd20b44c43d76e65c0996d7851c6", "The fugacity of a constituent in a mixture of gases at a given temperature is proportional to its mole fraction. Lewis-Randall Rule"}, + {"86ed2eaa9c75ba98396e5c9fb2f679ecf0ea2ed1e0ee9ceecb4a9332", "How can you write a big system without C++? -Paul Glick"}, +} + +func TestGolden(t *testing.T) { + for i := 0; i < len(golden); i++ { + g := golden[i] + c := New() + for j := 0; j < 3; j++ { + if j < 2 { + io.WriteString(c, g.in) + } else { + io.WriteString(c, g.in[0:len(g.in)/2]) + c.Sum() + io.WriteString(c, g.in[len(g.in)/2:]) + } + s := fmt.Sprintf("%x", c.Sum()) + if s != g.out { + t.Fatalf("sha256[%d](%s) = %s want %s", j, g.in, s, g.out) + } + c.Reset() + } + } + for i := 0; i < len(golden224); i++ { + g := golden224[i] + c := New224() + for j := 0; j < 3; j++ { + if j < 2 { + io.WriteString(c, g.in) + } else { + io.WriteString(c, g.in[0:len(g.in)/2]) + c.Sum() + io.WriteString(c, g.in[len(g.in)/2:]) + } + s := fmt.Sprintf("%x", c.Sum()) + if s != g.out { + t.Fatalf("sha224[%d](%s) = %s want %s", j, g.in, s, g.out) + } + c.Reset() + } + } +} diff --git a/src/pkg/crypto/sha256/sha256block.go b/src/pkg/crypto/sha256/sha256block.go new file mode 100644 index 000000000..7b0f55444 --- /dev/null +++ b/src/pkg/crypto/sha256/sha256block.go @@ -0,0 +1,129 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// SHA256 block step. +// In its own file so that a faster assembly or C version +// can be substituted easily. + +package sha256 + +var _K = []uint32{ + 0x428a2f98, + 0x71374491, + 0xb5c0fbcf, + 0xe9b5dba5, + 0x3956c25b, + 0x59f111f1, + 0x923f82a4, + 0xab1c5ed5, + 0xd807aa98, + 0x12835b01, + 0x243185be, + 0x550c7dc3, + 0x72be5d74, + 0x80deb1fe, + 0x9bdc06a7, + 0xc19bf174, + 0xe49b69c1, + 0xefbe4786, + 0x0fc19dc6, + 0x240ca1cc, + 0x2de92c6f, + 0x4a7484aa, + 0x5cb0a9dc, + 0x76f988da, + 0x983e5152, + 0xa831c66d, + 0xb00327c8, + 0xbf597fc7, + 0xc6e00bf3, + 0xd5a79147, + 0x06ca6351, + 0x14292967, + 0x27b70a85, + 0x2e1b2138, + 0x4d2c6dfc, + 0x53380d13, + 0x650a7354, + 0x766a0abb, + 0x81c2c92e, + 0x92722c85, + 0xa2bfe8a1, + 0xa81a664b, + 0xc24b8b70, + 0xc76c51a3, + 0xd192e819, + 0xd6990624, + 0xf40e3585, + 0x106aa070, + 0x19a4c116, + 0x1e376c08, + 0x2748774c, + 0x34b0bcb5, + 0x391c0cb3, + 0x4ed8aa4a, + 0x5b9cca4f, + 0x682e6ff3, + 0x748f82ee, + 0x78a5636f, + 0x84c87814, + 0x8cc70208, + 0x90befffa, + 0xa4506ceb, + 0xbef9a3f7, + 0xc67178f2, +} + +func _Block(dig *digest, p []byte) int { + var w [64]uint32 + n := 0 + h0, h1, h2, h3, h4, h5, h6, h7 := dig.h[0], dig.h[1], dig.h[2], dig.h[3], dig.h[4], dig.h[5], dig.h[6], dig.h[7] + for len(p) >= _Chunk { + // Can interlace the computation of w with the + // rounds below if needed for speed. + for i := 0; i < 16; i++ { + j := i * 4 + w[i] = uint32(p[j])<<24 | uint32(p[j+1])<<16 | uint32(p[j+2])<<8 | uint32(p[j+3]) + } + for i := 16; i < 64; i++ { + t1 := (w[i-2]>>17 | w[i-2]<<(32-17)) ^ (w[i-2]>>19 | w[i-2]<<(32-19)) ^ (w[i-2] >> 10) + + t2 := (w[i-15]>>7 | w[i-15]<<(32-7)) ^ (w[i-15]>>18 | w[i-15]<<(32-18)) ^ (w[i-15] >> 3) + + w[i] = t1 + w[i-7] + t2 + w[i-16] + } + + a, b, c, d, e, f, g, h := h0, h1, h2, h3, h4, h5, h6, h7 + + for i := 0; i < 64; i++ { + t1 := h + ((e>>6 | e<<(32-6)) ^ (e>>11 | e<<(32-11)) ^ (e>>25 | e<<(32-25))) + ((e & f) ^ (^e & g)) + _K[i] + w[i] + + t2 := ((a>>2 | a<<(32-2)) ^ (a>>13 | a<<(32-13)) ^ (a>>22 | a<<(32-22))) + ((a & b) ^ (a & c) ^ (b & c)) + + h = g + g = f + f = e + e = d + t1 + d = c + c = b + b = a + a = t1 + t2 + } + + h0 += a + h1 += b + h2 += c + h3 += d + h4 += e + h5 += f + h6 += g + h7 += h + + p = p[_Chunk:] + n += _Chunk + } + + dig.h[0], dig.h[1], dig.h[2], dig.h[3], dig.h[4], dig.h[5], dig.h[6], dig.h[7] = h0, h1, h2, h3, h4, h5, h6, h7 + return n +} diff --git a/src/pkg/crypto/sha512/Makefile b/src/pkg/crypto/sha512/Makefile new file mode 100644 index 000000000..2f7633fa3 --- /dev/null +++ b/src/pkg/crypto/sha512/Makefile @@ -0,0 +1,12 @@ +# Copyright 2009 The Go Authors. All rights reserved. +# Use of this source code is governed by a BSD-style +# license that can be found in the LICENSE file. + +include ../../../Make.inc + +TARG=crypto/sha512 +GOFILES=\ + sha512.go\ + sha512block.go\ + +include ../../../Make.pkg diff --git a/src/pkg/crypto/sha512/sha512.go b/src/pkg/crypto/sha512/sha512.go new file mode 100644 index 000000000..78f5fe26f --- /dev/null +++ b/src/pkg/crypto/sha512/sha512.go @@ -0,0 +1,170 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Package sha512 implements the SHA384 and SHA512 hash algorithms as defined +// in FIPS 180-2. +package sha512 + +import ( + "crypto" + "hash" + "os" +) + +func init() { + crypto.RegisterHash(crypto.SHA384, New384) + crypto.RegisterHash(crypto.SHA512, New) +} + +// The size of a SHA512 checksum in bytes. +const Size = 64 + +// The size of a SHA384 checksum in bytes. +const Size384 = 48 + +const ( + _Chunk = 128 + _Init0 = 0x6a09e667f3bcc908 + _Init1 = 0xbb67ae8584caa73b + _Init2 = 0x3c6ef372fe94f82b + _Init3 = 0xa54ff53a5f1d36f1 + _Init4 = 0x510e527fade682d1 + _Init5 = 0x9b05688c2b3e6c1f + _Init6 = 0x1f83d9abfb41bd6b + _Init7 = 0x5be0cd19137e2179 + _Init0_384 = 0xcbbb9d5dc1059ed8 + _Init1_384 = 0x629a292a367cd507 + _Init2_384 = 0x9159015a3070dd17 + _Init3_384 = 0x152fecd8f70e5939 + _Init4_384 = 0x67332667ffc00b31 + _Init5_384 = 0x8eb44a8768581511 + _Init6_384 = 0xdb0c2e0d64f98fa7 + _Init7_384 = 0x47b5481dbefa4fa4 +) + +// digest represents the partial evaluation of a checksum. +type digest struct { + h [8]uint64 + x [_Chunk]byte + nx int + len uint64 + is384 bool // mark if this digest is SHA-384 +} + +func (d *digest) Reset() { + if !d.is384 { + d.h[0] = _Init0 + d.h[1] = _Init1 + d.h[2] = _Init2 + d.h[3] = _Init3 + d.h[4] = _Init4 + d.h[5] = _Init5 + d.h[6] = _Init6 + d.h[7] = _Init7 + } else { + d.h[0] = _Init0_384 + d.h[1] = _Init1_384 + d.h[2] = _Init2_384 + d.h[3] = _Init3_384 + d.h[4] = _Init4_384 + d.h[5] = _Init5_384 + d.h[6] = _Init6_384 + d.h[7] = _Init7_384 + } + d.nx = 0 + d.len = 0 +} + +// New returns a new hash.Hash computing the SHA512 checksum. +func New() hash.Hash { + d := new(digest) + d.Reset() + return d +} + +// New384 returns a new hash.Hash computing the SHA384 checksum. +func New384() hash.Hash { + d := new(digest) + d.is384 = true + d.Reset() + return d +} + +func (d *digest) Size() int { + if !d.is384 { + return Size + } + return Size384 +} + +func (d *digest) Write(p []byte) (nn int, err os.Error) { + nn = len(p) + d.len += uint64(nn) + if d.nx > 0 { + n := len(p) + if n > _Chunk-d.nx { + n = _Chunk - d.nx + } + for i := 0; i < n; i++ { + d.x[d.nx+i] = p[i] + } + d.nx += n + if d.nx == _Chunk { + _Block(d, d.x[0:]) + d.nx = 0 + } + p = p[n:] + } + n := _Block(d, p) + p = p[n:] + if len(p) > 0 { + d.nx = copy(d.x[:], p) + } + return +} + +func (d0 *digest) Sum() []byte { + // Make a copy of d0 so that caller can keep writing and summing. + d := new(digest) + *d = *d0 + + // Padding. Add a 1 bit and 0 bits until 112 bytes mod 128. + len := d.len + var tmp [128]byte + tmp[0] = 0x80 + if len%128 < 112 { + d.Write(tmp[0 : 112-len%128]) + } else { + d.Write(tmp[0 : 128+112-len%128]) + } + + // Length in bits. + len <<= 3 + for i := uint(0); i < 16; i++ { + tmp[i] = byte(len >> (120 - 8*i)) + } + d.Write(tmp[0:16]) + + if d.nx != 0 { + panic("d.nx != 0") + } + + p := make([]byte, 64) + j := 0 + for _, s := range d.h { + p[j+0] = byte(s >> 56) + p[j+1] = byte(s >> 48) + p[j+2] = byte(s >> 40) + p[j+3] = byte(s >> 32) + p[j+4] = byte(s >> 24) + p[j+5] = byte(s >> 16) + p[j+6] = byte(s >> 8) + p[j+7] = byte(s >> 0) + j += 8 + } + if d.is384 { + return p[0:48] + } + return p +} diff --git a/src/pkg/crypto/sha512/sha512_test.go b/src/pkg/crypto/sha512/sha512_test.go new file mode 100644 index 000000000..dd116dc17 --- /dev/null +++ b/src/pkg/crypto/sha512/sha512_test.go @@ -0,0 +1,125 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// SHA512 hash algorithm. See FIPS 180-2. + +package sha512 + +import ( + "fmt" + "io" + "testing" +) + +type sha512Test struct { + out string + in string +} + +var golden = []sha512Test{ + {"cf83e1357eefb8bdf1542850d66d8007d620e4050b5715dc83f4a921d36ce9ce47d0d13c5d85f2b0ff8318d2877eec2f63b931bd47417a81a538327af927da3e", ""}, + {"1f40fc92da241694750979ee6cf582f2d5d7d28e18335de05abc54d0560e0f5302860c652bf08d560252aa5e74210546f369fbbbce8c12cfc7957b2652fe9a75", "a"}, + {"2d408a0717ec188158278a796c689044361dc6fdde28d6f04973b80896e1823975cdbf12eb63f9e0591328ee235d80e9b5bf1aa6a44f4617ff3caf6400eb172d", "ab"}, + {"ddaf35a193617abacc417349ae20413112e6fa4e89a97ea20a9eeee64b55d39a2192992a274fc1a836ba3c23a3feebbd454d4423643ce80e2a9ac94fa54ca49f", "abc"}, + {"d8022f2060ad6efd297ab73dcc5355c9b214054b0d1776a136a669d26a7d3b14f73aa0d0ebff19ee333368f0164b6419a96da49e3e481753e7e96b716bdccb6f", "abcd"}, + {"878ae65a92e86cac011a570d4c30a7eaec442b85ce8eca0c2952b5e3cc0628c2e79d889ad4d5c7c626986d452dd86374b6ffaa7cd8b67665bef2289a5c70b0a1", "abcde"}, + {"e32ef19623e8ed9d267f657a81944b3d07adbb768518068e88435745564e8d4150a0a703be2a7d88b61e3d390c2bb97e2d4c311fdc69d6b1267f05f59aa920e7", "abcdef"}, + {"d716a4188569b68ab1b6dfac178e570114cdf0ea3a1cc0e31486c3e41241bc6a76424e8c37ab26f096fc85ef9886c8cb634187f4fddff645fb099f1ff54c6b8c", "abcdefg"}, + {"a3a8c81bc97c2560010d7389bc88aac974a104e0e2381220c6e084c4dccd1d2d17d4f86db31c2a851dc80e6681d74733c55dcd03dd96f6062cdda12a291ae6ce", "abcdefgh"}, + {"f22d51d25292ca1d0f68f69aedc7897019308cc9db46efb75a03dd494fc7f126c010e8ade6a00a0c1a5f1b75d81e0ed5a93ce98dc9b833db7839247b1d9c24fe", "abcdefghi"}, + {"ef6b97321f34b1fea2169a7db9e1960b471aa13302a988087357c520be957ca119c3ba68e6b4982c019ec89de3865ccf6a3cda1fe11e59f98d99f1502c8b9745", "abcdefghij"}, + {"2210d99af9c8bdecda1b4beff822136753d8342505ddce37f1314e2cdbb488c6016bdaa9bd2ffa513dd5de2e4b50f031393d8ab61f773b0e0130d7381e0f8a1d", "Discard medicine more than two years old."}, + {"a687a8985b4d8d0a24f115fe272255c6afaf3909225838546159c1ed685c211a203796ae8ecc4c81a5b6315919b3a64f10713da07e341fcdbb08541bf03066ce", "He who has a shady past knows that nice guys finish last."}, + {"8ddb0392e818b7d585ab22769a50df660d9f6d559cca3afc5691b8ca91b8451374e42bcdabd64589ed7c91d85f626596228a5c8572677eb98bc6b624befb7af8", "I wouldn't marry him with a ten foot pole."}, + {"26ed8f6ca7f8d44b6a8a54ae39640fa8ad5c673f70ee9ce074ba4ef0d483eea00bab2f61d8695d6b34df9c6c48ae36246362200ed820448bdc03a720366a87c6", "Free! Free!/A trip/to Mars/for 900/empty jars/Burma Shave"}, + {"e5a14bf044be69615aade89afcf1ab0389d5fc302a884d403579d1386a2400c089b0dbb387ed0f463f9ee342f8244d5a38cfbc0e819da9529fbff78368c9a982", "The days of the digital watch are numbered. -Tom Stoppard"}, + {"420a1faa48919e14651bed45725abe0f7a58e0f099424c4e5a49194946e38b46c1f8034b18ef169b2e31050d1648e0b982386595f7df47da4b6fd18e55333015", "Nepal premier won't resign."}, + {"d926a863beadb20134db07683535c72007b0e695045876254f341ddcccde132a908c5af57baa6a6a9c63e6649bba0c213dc05fadcf9abccea09f23dcfb637fbe", "For every action there is an equal and opposite government program."}, + {"9a98dd9bb67d0da7bf83da5313dff4fd60a4bac0094f1b05633690ffa7f6d61de9a1d4f8617937d560833a9aaa9ccafe3fd24db418d0e728833545cadd3ad92d", "His money is twice tainted: 'taint yours and 'taint mine."}, + {"d7fde2d2351efade52f4211d3746a0780a26eec3df9b2ed575368a8a1c09ec452402293a8ea4eceb5a4f60064ea29b13cdd86918cd7a4faf366160b009804107", "There is no reason for any individual to have a computer in their home. -Ken Olsen, 1977"}, + {"b0f35ffa2697359c33a56f5c0cf715c7aeed96da9905ca2698acadb08fbc9e669bf566b6bd5d61a3e86dc22999bcc9f2224e33d1d4f32a228cf9d0349e2db518", "It's a tiny change to the code and not completely disgusting. - Bob Manchek"}, + {"3d2e5f91778c9e66f7e061293aaa8a8fc742dd3b2e4f483772464b1144189b49273e610e5cccd7a81a19ca1fa70f16b10f1a100a4d8c1372336be8484c64b311", "size: a.out: bad magic"}, + {"b2f68ff58ac015efb1c94c908b0d8c2bf06f491e4de8e6302c49016f7f8a33eac3e959856c7fddbc464de618701338a4b46f76dbfaf9a1e5262b5f40639771c7", "The major problem is with sendmail. -Mark Horton"}, + {"d8c92db5fdf52cf8215e4df3b4909d29203ff4d00e9ad0b64a6a4e04dec5e74f62e7c35c7fb881bd5de95442123df8f57a489b0ae616bd326f84d10021121c57", "Give me a rock, paper and scissors and I will move the world. CCFestoon"}, + {"19a9f8dc0a233e464e8566ad3ca9b91e459a7b8c4780985b015776e1bf239a19bc233d0556343e2b0a9bc220900b4ebf4f8bdf89ff8efeaf79602d6849e6f72e", "If the enemy is within range, then so are you."}, + {"00b4c41f307bde87301cdc5b5ab1ae9a592e8ecbb2021dd7bc4b34e2ace60741cc362560bec566ba35178595a91932b8d5357e2c9cec92d393b0fa7831852476", "It's well we cannot hear the screams/That we create in others' dreams."}, + {"91eccc3d5375fd026e4d6787874b1dce201cecd8a27dbded5065728cb2d09c58a3d467bb1faf353bf7ba567e005245d5321b55bc344f7c07b91cb6f26c959be7", "You remind me of a TV show, but that's all right: I watch it anyway."}, + {"fabbbe22180f1f137cfdc9556d2570e775d1ae02a597ded43a72a40f9b485d500043b7be128fb9fcd982b83159a0d99aa855a9e7cc4240c00dc01a9bdf8218d7", "C is as portable as Stonehedge!!"}, + {"2ecdec235c1fa4fc2a154d8fba1dddb8a72a1ad73838b51d792331d143f8b96a9f6fcb0f34d7caa351fe6d88771c4f105040e0392f06e0621689d33b2f3ba92e", "Even if I could be Shakespeare, I think I should still choose to be Faraday. - A. Huxley"}, + {"7ad681f6f96f82f7abfa7ecc0334e8fa16d3dc1cdc45b60b7af43fe4075d2357c0c1d60e98350f1afb1f2fe7a4d7cd2ad55b88e458e06b73c40b437331f5dab4", "The fugacity of a constituent in a mixture of gases at a given temperature is proportional to its mole fraction. Lewis-Randall Rule"}, + {"833f9248ab4a3b9e5131f745fda1ffd2dd435b30e965957e78291c7ab73605fd1912b0794e5c233ab0a12d205a39778d19b83515d6a47003f19cdee51d98c7e0", "How can you write a big system without C++? -Paul Glick"}, +} + +var golden384 = []sha512Test{ + {"38b060a751ac96384cd9327eb1b1e36a21fdb71114be07434c0cc7bf63f6e1da274edebfe76f65fbd51ad2f14898b95b", ""}, + {"54a59b9f22b0b80880d8427e548b7c23abd873486e1f035dce9cd697e85175033caa88e6d57bc35efae0b5afd3145f31", "a"}, + {"c7be03ba5bcaa384727076db0018e99248e1a6e8bd1b9ef58a9ec9dd4eeebb3f48b836201221175befa74ddc3d35afdd", "ab"}, + {"cb00753f45a35e8bb5a03d699ac65007272c32ab0eded1631a8b605a43ff5bed8086072ba1e7cc2358baeca134c825a7", "abc"}, + {"1165b3406ff0b52a3d24721f785462ca2276c9f454a116c2b2ba20171a7905ea5a026682eb659c4d5f115c363aa3c79b", "abcd"}, + {"4c525cbeac729eaf4b4665815bc5db0c84fe6300068a727cf74e2813521565abc0ec57a37ee4d8be89d097c0d2ad52f0", "abcde"}, + {"c6a4c65b227e7387b9c3e839d44869c4cfca3ef583dea64117859b808c1e3d8ae689e1e314eeef52a6ffe22681aa11f5", "abcdef"}, + {"9f11fc131123f844c1226f429b6a0a6af0525d9f40f056c7fc16cdf1b06bda08e302554417a59fa7dcf6247421959d22", "abcdefg"}, + {"9000cd7cada59d1d2eb82912f7f24e5e69cc5517f68283b005fa27c285b61e05edf1ad1a8a9bded6fd29eb87d75ad806", "abcdefgh"}, + {"ef54915b60cf062b8dd0c29ae3cad69abe6310de63ac081f46ef019c5c90897caefd79b796cfa81139788a260ded52df", "abcdefghi"}, + {"a12070030a02d86b0ddacd0d3a5b598344513d0a051e7355053e556a0055489c1555399b03342845c4adde2dc44ff66c", "abcdefghij"}, + {"86f58ec2d74d1b7f8eb0c2ff0967316699639e8d4eb129de54bdf34c96cdbabe200d052149f2dd787f43571ba74670d4", "Discard medicine more than two years old."}, + {"ae4a2b639ca9bfa04b1855d5a05fe7f230994f790891c6979103e2605f660c4c1262a48142dcbeb57a1914ba5f7c3fa7", "He who has a shady past knows that nice guys finish last."}, + {"40ae213df6436eca952aa6841886fcdb82908ef1576a99c8f49bb9dd5023169f7c53035abdda0b54c302f4974e2105e7", "I wouldn't marry him with a ten foot pole."}, + {"e7cf8b873c9bc950f06259aa54309f349cefa72c00d597aebf903e6519a50011dfe355afff064a10701c705693848df9", "Free! Free!/A trip/to Mars/for 900/empty jars/Burma Shave"}, + {"c3d4f0f4047181c7d39d34703365f7bf70207183caf2c2f6145f04da895ef69124d9cdeb635da636c3a474e61024e29b", "The days of the digital watch are numbered. -Tom Stoppard"}, + {"a097aab567e167d5cf93676ed73252a69f9687cb3179bb2d27c9878119e94bf7b7c4b58dc90582edfaf66e11388ed714", "Nepal premier won't resign."}, + {"5026ca45c41fc64712eb65065da92f6467541c78f8966d3fe2c8e3fb769a3ec14215f819654b47bd64f7f0eac17184f3", "For every action there is an equal and opposite government program."}, + {"ac1cc0f5ac8d5f5514a7b738ac322b7fb52a161b449c3672e9b6a6ad1a5e4b26b001cf3bad24c56598676ca17d4b445a", "His money is twice tainted: 'taint yours and 'taint mine."}, + {"722d10c5de371ec0c8c4b5247ac8a5f1d240d68c73f8da13d8b25f0166d6f309bf9561979a111a0049405771d201941a", "There is no reason for any individual to have a computer in their home. -Ken Olsen, 1977"}, + {"dc2d3ea18bfa10549c63bf2b75b39b5167a80c12aff0e05443168ea87ff149fb0eda5e0bd234eb5d48c7d02ffc5807f1", "It's a tiny change to the code and not completely disgusting. - Bob Manchek"}, + {"1d67c969e2a945ae5346d2139760261504d4ba164c522443afe19ef3e29b152a4c52445489cfc9d7215e5a450e8e1e4e", "size: a.out: bad magic"}, + {"5ff8e075e465646e7b73ef36d812c6e9f7d60fa6ea0e533e5569b4f73cde53cdd2cc787f33540af57cca3fe467d32fe0", "The major problem is with sendmail. -Mark Horton"}, + {"5bd0a997a67c9ae1979a894eb0cde403dde003c9b6f2c03cf21925c42ff4e1176e6df1ca005381612ef18457b9b7ec3b", "Give me a rock, paper and scissors and I will move the world. CCFestoon"}, + {"1eee6da33e7e54fc5be52ae23b94b16ba4d2a947ae4505c6a3edfc7401151ea5205ac01b669b56f27d8ef7f175ed7762", "If the enemy is within range, then so are you."}, + {"76b06e9dea66bfbb1a96029426dc0dfd7830bd297eb447ff5358d94a87cd00c88b59df2493fef56ecbb5231073892ea9", "It's well we cannot hear the screams/That we create in others' dreams."}, + {"12acaf21452cff586143e3f5db0bfdf7802c057e1adf2a619031c4e1b0ccc4208cf6cef8fe722bbaa2fb46a30d9135d8", "You remind me of a TV show, but that's all right: I watch it anyway."}, + {"0fc23d7f4183efd186f0bc4fc5db867e026e2146b06cb3d52f4bdbd57d1740122caa853b41868b197b2ac759db39df88", "C is as portable as Stonehedge!!"}, + {"bc805578a7f85d34a86a32976e1c34fe65cf815186fbef76f46ef99cda10723f971f3f1464d488243f5e29db7488598d", "Even if I could be Shakespeare, I think I should still choose to be Faraday. - A. Huxley"}, + {"b23918399a12ebf4431559eec3813eaf7412e875fd7464f16d581e473330842d2e96c6be49a7ce3f9bb0b8bc0fcbe0fe", "The fugacity of a constituent in a mixture of gases at a given temperature is proportional to its mole fraction. Lewis-Randall Rule"}, + {"1764b700eb1ead52a2fc33cc28975c2180f1b8faa5038d94cffa8d78154aab16e91dd787e7b0303948ebed62561542c8", "How can you write a big system without C++? -Paul Glick"}, +} + +func TestGolden(t *testing.T) { + for i := 0; i < len(golden); i++ { + g := golden[i] + c := New() + for j := 0; j < 3; j++ { + if j < 2 { + io.WriteString(c, g.in) + } else { + io.WriteString(c, g.in[0:len(g.in)/2]) + c.Sum() + io.WriteString(c, g.in[len(g.in)/2:]) + } + s := fmt.Sprintf("%x", c.Sum()) + if s != g.out { + t.Fatalf("sha512[%d](%s) = %s want %s", j, g.in, s, g.out) + } + c.Reset() + } + } + for i := 0; i < len(golden384); i++ { + g := golden384[i] + c := New384() + for j := 0; j < 3; j++ { + if j < 2 { + io.WriteString(c, g.in) + } else { + io.WriteString(c, g.in[0:len(g.in)/2]) + c.Sum() + io.WriteString(c, g.in[len(g.in)/2:]) + } + s := fmt.Sprintf("%x", c.Sum()) + if s != g.out { + t.Fatalf("sha384[%d](%s) = %s want %s", j, g.in, s, g.out) + } + c.Reset() + } + } +} diff --git a/src/pkg/crypto/sha512/sha512block.go b/src/pkg/crypto/sha512/sha512block.go new file mode 100644 index 000000000..6b7506287 --- /dev/null +++ b/src/pkg/crypto/sha512/sha512block.go @@ -0,0 +1,144 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// SHA512 block step. +// In its own file so that a faster assembly or C version +// can be substituted easily. + +package sha512 + +var _K = []uint64{ + 0x428a2f98d728ae22, + 0x7137449123ef65cd, + 0xb5c0fbcfec4d3b2f, + 0xe9b5dba58189dbbc, + 0x3956c25bf348b538, + 0x59f111f1b605d019, + 0x923f82a4af194f9b, + 0xab1c5ed5da6d8118, + 0xd807aa98a3030242, + 0x12835b0145706fbe, + 0x243185be4ee4b28c, + 0x550c7dc3d5ffb4e2, + 0x72be5d74f27b896f, + 0x80deb1fe3b1696b1, + 0x9bdc06a725c71235, + 0xc19bf174cf692694, + 0xe49b69c19ef14ad2, + 0xefbe4786384f25e3, + 0x0fc19dc68b8cd5b5, + 0x240ca1cc77ac9c65, + 0x2de92c6f592b0275, + 0x4a7484aa6ea6e483, + 0x5cb0a9dcbd41fbd4, + 0x76f988da831153b5, + 0x983e5152ee66dfab, + 0xa831c66d2db43210, + 0xb00327c898fb213f, + 0xbf597fc7beef0ee4, + 0xc6e00bf33da88fc2, + 0xd5a79147930aa725, + 0x06ca6351e003826f, + 0x142929670a0e6e70, + 0x27b70a8546d22ffc, + 0x2e1b21385c26c926, + 0x4d2c6dfc5ac42aed, + 0x53380d139d95b3df, + 0x650a73548baf63de, + 0x766a0abb3c77b2a8, + 0x81c2c92e47edaee6, + 0x92722c851482353b, + 0xa2bfe8a14cf10364, + 0xa81a664bbc423001, + 0xc24b8b70d0f89791, + 0xc76c51a30654be30, + 0xd192e819d6ef5218, + 0xd69906245565a910, + 0xf40e35855771202a, + 0x106aa07032bbd1b8, + 0x19a4c116b8d2d0c8, + 0x1e376c085141ab53, + 0x2748774cdf8eeb99, + 0x34b0bcb5e19b48a8, + 0x391c0cb3c5c95a63, + 0x4ed8aa4ae3418acb, + 0x5b9cca4f7763e373, + 0x682e6ff3d6b2b8a3, + 0x748f82ee5defb2fc, + 0x78a5636f43172f60, + 0x84c87814a1f0ab72, + 0x8cc702081a6439ec, + 0x90befffa23631e28, + 0xa4506cebde82bde9, + 0xbef9a3f7b2c67915, + 0xc67178f2e372532b, + 0xca273eceea26619c, + 0xd186b8c721c0c207, + 0xeada7dd6cde0eb1e, + 0xf57d4f7fee6ed178, + 0x06f067aa72176fba, + 0x0a637dc5a2c898a6, + 0x113f9804bef90dae, + 0x1b710b35131c471b, + 0x28db77f523047d84, + 0x32caab7b40c72493, + 0x3c9ebe0a15c9bebc, + 0x431d67c49c100d4c, + 0x4cc5d4becb3e42b6, + 0x597f299cfc657e2a, + 0x5fcb6fab3ad6faec, + 0x6c44198c4a475817, +} + +func _Block(dig *digest, p []byte) int { + var w [80]uint64 + n := 0 + h0, h1, h2, h3, h4, h5, h6, h7 := dig.h[0], dig.h[1], dig.h[2], dig.h[3], dig.h[4], dig.h[5], dig.h[6], dig.h[7] + for len(p) >= _Chunk { + for i := 0; i < 16; i++ { + j := i * 8 + w[i] = uint64(p[j])<<56 | uint64(p[j+1])<<48 | uint64(p[j+2])<<40 | uint64(p[j+3])<<32 | + uint64(p[j+4])<<24 | uint64(p[j+5])<<16 | uint64(p[j+6])<<8 | uint64(p[j+7]) + } + for i := 16; i < 80; i++ { + t1 := (w[i-2]>>19 | w[i-2]<<(64-19)) ^ (w[i-2]>>61 | w[i-2]<<(64-61)) ^ (w[i-2] >> 6) + + t2 := (w[i-15]>>1 | w[i-15]<<(64-1)) ^ (w[i-15]>>8 | w[i-15]<<(64-8)) ^ (w[i-15] >> 7) + + w[i] = t1 + w[i-7] + t2 + w[i-16] + } + + a, b, c, d, e, f, g, h := h0, h1, h2, h3, h4, h5, h6, h7 + + for i := 0; i < 80; i++ { + t1 := h + ((e>>14 | e<<(64-14)) ^ (e>>18 | e<<(64-18)) ^ (e>>41 | e<<(64-41))) + ((e & f) ^ (^e & g)) + _K[i] + w[i] + + t2 := ((a>>28 | a<<(64-28)) ^ (a>>34 | a<<(64-34)) ^ (a>>39 | a<<(64-39))) + ((a & b) ^ (a & c) ^ (b & c)) + + h = g + g = f + f = e + e = d + t1 + d = c + c = b + b = a + a = t1 + t2 + } + + h0 += a + h1 += b + h2 += c + h3 += d + h4 += e + h5 += f + h6 += g + h7 += h + + p = p[_Chunk:] + n += _Chunk + } + + dig.h[0], dig.h[1], dig.h[2], dig.h[3], dig.h[4], dig.h[5], dig.h[6], dig.h[7] = h0, h1, h2, h3, h4, h5, h6, h7 + return n +} diff --git a/src/pkg/crypto/subtle/Makefile b/src/pkg/crypto/subtle/Makefile new file mode 100644 index 000000000..08d8bbfa0 --- /dev/null +++ b/src/pkg/crypto/subtle/Makefile @@ -0,0 +1,11 @@ +# Copyright 2009 The Go Authors. All rights reserved. +# Use of this source code is governed by a BSD-style +# license that can be found in the LICENSE file. + +include ../../../Make.inc + +TARG=crypto/subtle +GOFILES=\ + constant_time.go\ + +include ../../../Make.pkg diff --git a/src/pkg/crypto/subtle/constant_time.go b/src/pkg/crypto/subtle/constant_time.go new file mode 100644 index 000000000..57dbe9db5 --- /dev/null +++ b/src/pkg/crypto/subtle/constant_time.go @@ -0,0 +1,57 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Package subtle implements functions that are often useful in cryptographic +// code but require careful thought to use correctly. +package subtle + +// ConstantTimeCompare returns 1 iff the two equal length slices, x +// and y, have equal contents. The time taken is a function of the length of +// the slices and is independent of the contents. +func ConstantTimeCompare(x, y []byte) int { + var v byte + + for i := 0; i < len(x); i++ { + v |= x[i] ^ y[i] + } + + return ConstantTimeByteEq(v, 0) +} + +// ConstantTimeSelect returns x if v is 1 and y if v is 0. +// Its behavior is undefined if v takes any other value. +func ConstantTimeSelect(v, x, y int) int { return ^(v-1)&x | (v-1)&y } + +// ConstantTimeByteEq returns 1 if x == y and 0 otherwise. +func ConstantTimeByteEq(x, y uint8) int { + z := ^(x ^ y) + z &= z >> 4 + z &= z >> 2 + z &= z >> 1 + + return int(z) +} + +// ConstantTimeEq returns 1 if x == y and 0 otherwise. +func ConstantTimeEq(x, y int32) int { + z := ^(x ^ y) + z &= z >> 16 + z &= z >> 8 + z &= z >> 4 + z &= z >> 2 + z &= z >> 1 + + return int(z & 1) +} + +// ConstantTimeCopy copies the contents of y into x iff v == 1. If v == 0, x is left unchanged. +// Its behavior is undefined if v takes any other value. +func ConstantTimeCopy(v int, x, y []byte) { + xmask := byte(v - 1) + ymask := byte(^(v - 1)) + for i := 0; i < len(x); i++ { + x[i] = x[i]&xmask | y[i]&ymask + } + return +} diff --git a/src/pkg/crypto/subtle/constant_time_test.go b/src/pkg/crypto/subtle/constant_time_test.go new file mode 100644 index 000000000..adab8e2e8 --- /dev/null +++ b/src/pkg/crypto/subtle/constant_time_test.go @@ -0,0 +1,105 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package subtle + +import ( + "testing" + "testing/quick" +) + +type TestConstantTimeCompareStruct struct { + a, b []byte + out int +} + +var testConstantTimeCompareData = []TestConstantTimeCompareStruct{ + {[]byte{}, []byte{}, 1}, + {[]byte{0x11}, []byte{0x11}, 1}, + {[]byte{0x12}, []byte{0x11}, 0}, +} + +func TestConstantTimeCompare(t *testing.T) { + for i, test := range testConstantTimeCompareData { + if r := ConstantTimeCompare(test.a, test.b); r != test.out { + t.Errorf("#%d bad result (got %x, want %x)", i, r, test.out) + } + } +} + +type TestConstantTimeByteEqStruct struct { + a, b uint8 + out int +} + +var testConstandTimeByteEqData = []TestConstantTimeByteEqStruct{ + {0, 0, 1}, + {0, 1, 0}, + {1, 0, 0}, + {0xff, 0xff, 1}, + {0xff, 0xfe, 0}, +} + +func byteEq(a, b uint8) int { + if a == b { + return 1 + } + return 0 +} + +func TestConstantTimeByteEq(t *testing.T) { + for i, test := range testConstandTimeByteEqData { + if r := ConstantTimeByteEq(test.a, test.b); r != test.out { + t.Errorf("#%d bad result (got %x, want %x)", i, r, test.out) + } + } + err := quick.CheckEqual(ConstantTimeByteEq, byteEq, nil) + if err != nil { + t.Error(err) + } +} + +func eq(a, b int32) int { + if a == b { + return 1 + } + return 0 +} + +func TestConstantTimeEq(t *testing.T) { + err := quick.CheckEqual(ConstantTimeEq, eq, nil) + if err != nil { + t.Error(err) + } +} + +func makeCopy(v int, x, y []byte) []byte { + if len(x) > len(y) { + x = x[0:len(y)] + } else { + y = y[0:len(x)] + } + if v == 1 { + copy(x, y) + } + return x +} + +func constantTimeCopyWrapper(v int, x, y []byte) []byte { + if len(x) > len(y) { + x = x[0:len(y)] + } else { + y = y[0:len(x)] + } + v &= 1 + ConstantTimeCopy(v, x, y) + return x +} + +func TestConstantTimeCopy(t *testing.T) { + err := quick.CheckEqual(constantTimeCopyWrapper, makeCopy, nil) + if err != nil { + t.Error(err) + } +} diff --git a/src/pkg/crypto/tls/Makefile b/src/pkg/crypto/tls/Makefile new file mode 100644 index 000000000..000314be5 --- /dev/null +++ b/src/pkg/crypto/tls/Makefile @@ -0,0 +1,20 @@ +# Copyright 2009 The Go Authors. All rights reserved. +# Use of this source code is governed by a BSD-style +# license that can be found in the LICENSE file. + +include ../../../Make.inc + +TARG=crypto/tls +GOFILES=\ + alert.go\ + cipher_suites.go\ + common.go\ + conn.go\ + handshake_client.go\ + handshake_messages.go\ + handshake_server.go\ + key_agreement.go\ + prf.go\ + tls.go\ + +include ../../../Make.pkg diff --git a/src/pkg/crypto/tls/alert.go b/src/pkg/crypto/tls/alert.go new file mode 100644 index 000000000..3b9e0e241 --- /dev/null +++ b/src/pkg/crypto/tls/alert.go @@ -0,0 +1,73 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package tls + +import "strconv" + +type alert uint8 + +const ( + // alert level + alertLevelWarning = 1 + alertLevelError = 2 +) + +const ( + alertCloseNotify alert = 0 + alertUnexpectedMessage alert = 10 + alertBadRecordMAC alert = 20 + alertDecryptionFailed alert = 21 + alertRecordOverflow alert = 22 + alertDecompressionFailure alert = 30 + alertHandshakeFailure alert = 40 + alertBadCertificate alert = 42 + alertUnsupportedCertificate alert = 43 + alertCertificateRevoked alert = 44 + alertCertificateExpired alert = 45 + alertCertificateUnknown alert = 46 + alertIllegalParameter alert = 47 + alertUnknownCA alert = 48 + alertAccessDenied alert = 49 + alertDecodeError alert = 50 + alertDecryptError alert = 51 + alertProtocolVersion alert = 70 + alertInsufficientSecurity alert = 71 + alertInternalError alert = 80 + alertUserCanceled alert = 90 + alertNoRenegotiation alert = 100 +) + +var alertText = map[alert]string{ + alertCloseNotify: "close notify", + alertUnexpectedMessage: "unexpected message", + alertBadRecordMAC: "bad record MAC", + alertDecryptionFailed: "decryption failed", + alertRecordOverflow: "record overflow", + alertDecompressionFailure: "decompression failure", + alertHandshakeFailure: "handshake failure", + alertBadCertificate: "bad certificate", + alertUnsupportedCertificate: "unsupported certificate", + alertCertificateRevoked: "revoked certificate", + alertCertificateExpired: "expired certificate", + alertCertificateUnknown: "unknown certificate", + alertIllegalParameter: "illegal parameter", + alertUnknownCA: "unknown certificate authority", + alertAccessDenied: "access denied", + alertDecodeError: "error decoding message", + alertDecryptError: "error decrypting message", + alertProtocolVersion: "protocol version not supported", + alertInsufficientSecurity: "insufficient security level", + alertInternalError: "internal error", + alertUserCanceled: "user canceled", + alertNoRenegotiation: "no renegotiation", +} + +func (e alert) String() string { + s, ok := alertText[e] + if ok { + return s + } + return "alert(" + strconv.Itoa(int(e)) + ")" +} diff --git a/src/pkg/crypto/tls/cipher_suites.go b/src/pkg/crypto/tls/cipher_suites.go new file mode 100644 index 000000000..bc7b0d32f --- /dev/null +++ b/src/pkg/crypto/tls/cipher_suites.go @@ -0,0 +1,102 @@ +// Copyright 2010 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package tls + +import ( + "crypto/aes" + "crypto/cipher" + "crypto/hmac" + "crypto/rc4" + "crypto/x509" + "hash" + "os" +) + +// a keyAgreement implements the client and server side of a TLS key agreement +// protocol by generating and processing key exchange messages. +type keyAgreement interface { + // On the server side, the first two methods are called in order. + + // In the case that the key agreement protocol doesn't use a + // ServerKeyExchange message, generateServerKeyExchange can return nil, + // nil. + generateServerKeyExchange(*Config, *clientHelloMsg, *serverHelloMsg) (*serverKeyExchangeMsg, os.Error) + processClientKeyExchange(*Config, *clientKeyExchangeMsg) ([]byte, os.Error) + + // On the client side, the next two methods are called in order. + + // This method may not be called if the server doesn't send a + // ServerKeyExchange message. + processServerKeyExchange(*Config, *clientHelloMsg, *serverHelloMsg, *x509.Certificate, *serverKeyExchangeMsg) os.Error + generateClientKeyExchange(*Config, *clientHelloMsg, *x509.Certificate) ([]byte, *clientKeyExchangeMsg, os.Error) +} + +// A cipherSuite is a specific combination of key agreement, cipher and MAC +// function. All cipher suites currently assume RSA key agreement. +type cipherSuite struct { + // the lengths, in bytes, of the key material needed for each component. + keyLen int + macLen int + ivLen int + ka func() keyAgreement + // If elliptic is set, a server will only consider this ciphersuite if + // the ClientHello indicated that the client supports an elliptic curve + // and point format that we can handle. + elliptic bool + cipher func(key, iv []byte, isRead bool) interface{} + mac func(macKey []byte) hash.Hash +} + +var cipherSuites = map[uint16]*cipherSuite{ + TLS_RSA_WITH_RC4_128_SHA: &cipherSuite{16, 20, 0, rsaKA, false, cipherRC4, hmacSHA1}, + TLS_RSA_WITH_AES_128_CBC_SHA: &cipherSuite{16, 20, 16, rsaKA, false, cipherAES, hmacSHA1}, + TLS_ECDHE_RSA_WITH_RC4_128_SHA: &cipherSuite{16, 20, 0, ecdheRSAKA, true, cipherRC4, hmacSHA1}, + TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA: &cipherSuite{16, 20, 16, ecdheRSAKA, true, cipherAES, hmacSHA1}, +} + +func cipherRC4(key, iv []byte, isRead bool) interface{} { + cipher, _ := rc4.NewCipher(key) + return cipher +} + +func cipherAES(key, iv []byte, isRead bool) interface{} { + block, _ := aes.NewCipher(key) + if isRead { + return cipher.NewCBCDecrypter(block, iv) + } + return cipher.NewCBCEncrypter(block, iv) +} + +func hmacSHA1(key []byte) hash.Hash { + return hmac.NewSHA1(key) +} + +func rsaKA() keyAgreement { + return rsaKeyAgreement{} +} + +func ecdheRSAKA() keyAgreement { + return new(ecdheRSAKeyAgreement) +} + +// mutualCipherSuite returns a cipherSuite and its id given a list of supported +// ciphersuites and the id requested by the peer. +func mutualCipherSuite(have []uint16, want uint16) (suite *cipherSuite, id uint16) { + for _, id := range have { + if id == want { + return cipherSuites[id], id + } + } + return +} + +// A list of the possible cipher suite ids. Taken from +// http://www.iana.org/assignments/tls-parameters/tls-parameters.xml +const ( + TLS_RSA_WITH_RC4_128_SHA uint16 = 0x0005 + TLS_RSA_WITH_AES_128_CBC_SHA uint16 = 0x002f + TLS_ECDHE_RSA_WITH_RC4_128_SHA uint16 = 0xc011 + TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA uint16 = 0xc013 +) diff --git a/src/pkg/crypto/tls/common.go b/src/pkg/crypto/tls/common.go new file mode 100644 index 000000000..3efac9c13 --- /dev/null +++ b/src/pkg/crypto/tls/common.go @@ -0,0 +1,267 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package tls + +import ( + "crypto/rand" + "crypto/rsa" + "crypto/x509" + "io" + "io/ioutil" + "sync" + "time" +) + +const ( + maxPlaintext = 16384 // maximum plaintext payload length + maxCiphertext = 16384 + 2048 // maximum ciphertext payload length + recordHeaderLen = 5 // record header length + maxHandshake = 65536 // maximum handshake we support (protocol max is 16 MB) + + minVersion = 0x0301 // minimum supported version - TLS 1.0 + maxVersion = 0x0301 // maximum supported version - TLS 1.0 +) + +// TLS record types. +type recordType uint8 + +const ( + recordTypeChangeCipherSpec recordType = 20 + recordTypeAlert recordType = 21 + recordTypeHandshake recordType = 22 + recordTypeApplicationData recordType = 23 +) + +// TLS handshake message types. +const ( + typeClientHello uint8 = 1 + typeServerHello uint8 = 2 + typeCertificate uint8 = 11 + typeServerKeyExchange uint8 = 12 + typeCertificateRequest uint8 = 13 + typeServerHelloDone uint8 = 14 + typeCertificateVerify uint8 = 15 + typeClientKeyExchange uint8 = 16 + typeFinished uint8 = 20 + typeCertificateStatus uint8 = 22 + typeNextProtocol uint8 = 67 // Not IANA assigned +) + +// TLS compression types. +const ( + compressionNone uint8 = 0 +) + +// TLS extension numbers +var ( + extensionServerName uint16 = 0 + extensionStatusRequest uint16 = 5 + extensionSupportedCurves uint16 = 10 + extensionSupportedPoints uint16 = 11 + extensionNextProtoNeg uint16 = 13172 // not IANA assigned +) + +// TLS Elliptic Curves +// http://www.iana.org/assignments/tls-parameters/tls-parameters.xml#tls-parameters-8 +var ( + curveP256 uint16 = 23 + curveP384 uint16 = 24 + curveP521 uint16 = 25 +) + +// TLS Elliptic Curve Point Formats +// http://www.iana.org/assignments/tls-parameters/tls-parameters.xml#tls-parameters-9 +var ( + pointFormatUncompressed uint8 = 0 +) + +// TLS CertificateStatusType (RFC 3546) +const ( + statusTypeOCSP uint8 = 1 +) + +// Certificate types (for certificateRequestMsg) +const ( + certTypeRSASign = 1 // A certificate containing an RSA key + certTypeDSSSign = 2 // A certificate containing a DSA key + certTypeRSAFixedDH = 3 // A certificate containing a static DH key + certTypeDSSFixedDH = 4 // A certificate containing a static DH key + // Rest of these are reserved by the TLS spec +) + +// ConnectionState records basic TLS details about the connection. +type ConnectionState struct { + HandshakeComplete bool + CipherSuite uint16 + NegotiatedProtocol string + NegotiatedProtocolIsMutual bool + + // the certificate chain that was presented by the other side + PeerCertificates []*x509.Certificate + // the verified certificate chains built from PeerCertificates. + VerifiedChains [][]*x509.Certificate +} + +// A Config structure is used to configure a TLS client or server. After one +// has been passed to a TLS function it must not be modified. +type Config struct { + // Rand provides the source of entropy for nonces and RSA blinding. + // If Rand is nil, TLS uses the cryptographic random reader in package + // crypto/rand. + Rand io.Reader + + // Time returns the current time as the number of seconds since the epoch. + // If Time is nil, TLS uses the system time.Seconds. + Time func() int64 + + // Certificates contains one or more certificate chains + // to present to the other side of the connection. + // Server configurations must include at least one certificate. + Certificates []Certificate + + // RootCAs defines the set of root certificate authorities + // that clients use when verifying server certificates. + // If RootCAs is nil, TLS uses the host's root CA set. + RootCAs *x509.CertPool + + // NextProtos is a list of supported, application level protocols. + NextProtos []string + + // ServerName is included in the client's handshake to support virtual + // hosting. + ServerName string + + // AuthenticateClient controls whether a server will request a certificate + // from the client. It does not require that the client send a + // certificate nor does it require that the certificate sent be + // anything more than self-signed. + AuthenticateClient bool + + // CipherSuites is a list of supported cipher suites. If CipherSuites + // is nil, TLS uses a list of suites supported by the implementation. + CipherSuites []uint16 +} + +func (c *Config) rand() io.Reader { + r := c.Rand + if r == nil { + return rand.Reader + } + return r +} + +func (c *Config) time() int64 { + t := c.Time + if t == nil { + t = time.Seconds + } + return t() +} + +func (c *Config) rootCAs() *x509.CertPool { + s := c.RootCAs + if s == nil { + s = defaultRoots() + } + return s +} + +func (c *Config) cipherSuites() []uint16 { + s := c.CipherSuites + if s == nil { + s = defaultCipherSuites() + } + return s +} + +// A Certificate is a chain of one or more certificates, leaf first. +type Certificate struct { + Certificate [][]byte + PrivateKey *rsa.PrivateKey + // OCSPStaple contains an optional OCSP response which will be served + // to clients that request it. + OCSPStaple []byte +} + +// A TLS record. +type record struct { + contentType recordType + major, minor uint8 + payload []byte +} + +type handshakeMessage interface { + marshal() []byte + unmarshal([]byte) bool +} + +// mutualVersion returns the protocol version to use given the advertised +// version of the peer. +func mutualVersion(vers uint16) (uint16, bool) { + if vers < minVersion { + return 0, false + } + if vers > maxVersion { + vers = maxVersion + } + return vers, true +} + +var emptyConfig Config + +func defaultConfig() *Config { + return &emptyConfig +} + +// Possible certificate files; stop after finding one. +// On OS X we should really be using the Directory Services keychain +// but that requires a lot of Mach goo to get at. Instead we use +// the same root set that curl uses. +var certFiles = []string{ + "/etc/ssl/certs/ca-certificates.crt", // Linux etc + "/usr/share/curl/curl-ca-bundle.crt", // OS X +} + +var once sync.Once + +func defaultRoots() *x509.CertPool { + once.Do(initDefaults) + return varDefaultRoots +} + +func defaultCipherSuites() []uint16 { + once.Do(initDefaults) + return varDefaultCipherSuites +} + +func initDefaults() { + initDefaultRoots() + initDefaultCipherSuites() +} + +var varDefaultRoots *x509.CertPool + +func initDefaultRoots() { + roots := x509.NewCertPool() + for _, file := range certFiles { + data, err := ioutil.ReadFile(file) + if err == nil { + roots.AppendCertsFromPEM(data) + break + } + } + varDefaultRoots = roots +} + +var varDefaultCipherSuites []uint16 + +func initDefaultCipherSuites() { + varDefaultCipherSuites = make([]uint16, len(cipherSuites)) + i := 0 + for id := range cipherSuites { + varDefaultCipherSuites[i] = id + i++ + } +} diff --git a/src/pkg/crypto/tls/conn.go b/src/pkg/crypto/tls/conn.go new file mode 100644 index 000000000..fac65afd9 --- /dev/null +++ b/src/pkg/crypto/tls/conn.go @@ -0,0 +1,799 @@ +// Copyright 2010 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// TLS low level connection and record layer + +package tls + +import ( + "bytes" + "crypto/cipher" + "crypto/subtle" + "crypto/x509" + "hash" + "io" + "net" + "os" + "sync" +) + +// A Conn represents a secured connection. +// It implements the net.Conn interface. +type Conn struct { + // constant + conn net.Conn + isClient bool + + // constant after handshake; protected by handshakeMutex + handshakeMutex sync.Mutex // handshakeMutex < in.Mutex, out.Mutex, errMutex + vers uint16 // TLS version + haveVers bool // version has been negotiated + config *Config // configuration passed to constructor + handshakeComplete bool + cipherSuite uint16 + ocspResponse []byte // stapled OCSP response + peerCertificates []*x509.Certificate + // verifiedChains contains the certificate chains that we built, as + // opposed to the ones presented by the server. + verifiedChains [][]*x509.Certificate + + clientProtocol string + clientProtocolFallback bool + + // first permanent error + errMutex sync.Mutex + err os.Error + + // input/output + in, out halfConn // in.Mutex < out.Mutex + rawInput *block // raw input, right off the wire + input *block // application data waiting to be read + hand bytes.Buffer // handshake data waiting to be read + + tmp [16]byte +} + +func (c *Conn) setError(err os.Error) os.Error { + c.errMutex.Lock() + defer c.errMutex.Unlock() + + if c.err == nil { + c.err = err + } + return err +} + +func (c *Conn) error() os.Error { + c.errMutex.Lock() + defer c.errMutex.Unlock() + + return c.err +} + +// Access to net.Conn methods. +// Cannot just embed net.Conn because that would +// export the struct field too. + +// LocalAddr returns the local network address. +func (c *Conn) LocalAddr() net.Addr { + return c.conn.LocalAddr() +} + +// RemoteAddr returns the remote network address. +func (c *Conn) RemoteAddr() net.Addr { + return c.conn.RemoteAddr() +} + +// SetTimeout sets the read deadline associated with the connection. +// There is no write deadline. +func (c *Conn) SetTimeout(nsec int64) os.Error { + return c.conn.SetTimeout(nsec) +} + +// SetReadTimeout sets the time (in nanoseconds) that +// Read will wait for data before returning os.EAGAIN. +// Setting nsec == 0 (the default) disables the deadline. +func (c *Conn) SetReadTimeout(nsec int64) os.Error { + return c.conn.SetReadTimeout(nsec) +} + +// SetWriteTimeout exists to satisfy the net.Conn interface +// but is not implemented by TLS. It always returns an error. +func (c *Conn) SetWriteTimeout(nsec int64) os.Error { + return os.NewError("TLS does not support SetWriteTimeout") +} + +// A halfConn represents one direction of the record layer +// connection, either sending or receiving. +type halfConn struct { + sync.Mutex + cipher interface{} // cipher algorithm + mac hash.Hash // MAC algorithm + seq [8]byte // 64-bit sequence number + bfree *block // list of free blocks + + nextCipher interface{} // next encryption state + nextMac hash.Hash // next MAC algorithm +} + +// prepareCipherSpec sets the encryption and MAC states +// that a subsequent changeCipherSpec will use. +func (hc *halfConn) prepareCipherSpec(cipher interface{}, mac hash.Hash) { + hc.nextCipher = cipher + hc.nextMac = mac +} + +// changeCipherSpec changes the encryption and MAC states +// to the ones previously passed to prepareCipherSpec. +func (hc *halfConn) changeCipherSpec() os.Error { + if hc.nextCipher == nil { + return alertInternalError + } + hc.cipher = hc.nextCipher + hc.mac = hc.nextMac + hc.nextCipher = nil + hc.nextMac = nil + return nil +} + +// incSeq increments the sequence number. +func (hc *halfConn) incSeq() { + for i := 7; i >= 0; i-- { + hc.seq[i]++ + if hc.seq[i] != 0 { + return + } + } + + // Not allowed to let sequence number wrap. + // Instead, must renegotiate before it does. + // Not likely enough to bother. + panic("TLS: sequence number wraparound") +} + +// resetSeq resets the sequence number to zero. +func (hc *halfConn) resetSeq() { + for i := range hc.seq { + hc.seq[i] = 0 + } +} + +// removePadding returns an unpadded slice, in constant time, which is a prefix +// of the input. It also returns a byte which is equal to 255 if the padding +// was valid and 0 otherwise. See RFC 2246, section 6.2.3.2 +func removePadding(payload []byte) ([]byte, byte) { + if len(payload) < 1 { + return payload, 0 + } + + paddingLen := payload[len(payload)-1] + t := uint(len(payload)-1) - uint(paddingLen) + // if len(payload) >= (paddingLen - 1) then the MSB of t is zero + good := byte(int32(^t) >> 31) + + toCheck := 255 // the maximum possible padding length + // The length of the padded data is public, so we can use an if here + if toCheck+1 > len(payload) { + toCheck = len(payload) - 1 + } + + for i := 0; i < toCheck; i++ { + t := uint(paddingLen) - uint(i) + // if i <= paddingLen then the MSB of t is zero + mask := byte(int32(^t) >> 31) + b := payload[len(payload)-1-i] + good &^= mask&paddingLen ^ mask&b + } + + // We AND together the bits of good and replicate the result across + // all the bits. + good &= good << 4 + good &= good << 2 + good &= good << 1 + good = uint8(int8(good) >> 7) + + toRemove := good&paddingLen + 1 + return payload[:len(payload)-int(toRemove)], good +} + +func roundUp(a, b int) int { + return a + (b-a%b)%b +} + +// decrypt checks and strips the mac and decrypts the data in b. +func (hc *halfConn) decrypt(b *block) (bool, alert) { + // pull out payload + payload := b.data[recordHeaderLen:] + + macSize := 0 + if hc.mac != nil { + macSize = hc.mac.Size() + } + + paddingGood := byte(255) + + // decrypt + if hc.cipher != nil { + switch c := hc.cipher.(type) { + case cipher.Stream: + c.XORKeyStream(payload, payload) + case cipher.BlockMode: + blockSize := c.BlockSize() + + if len(payload)%blockSize != 0 || len(payload) < roundUp(macSize+1, blockSize) { + return false, alertBadRecordMAC + } + + c.CryptBlocks(payload, payload) + payload, paddingGood = removePadding(payload) + b.resize(recordHeaderLen + len(payload)) + + // note that we still have a timing side-channel in the + // MAC check, below. An attacker can align the record + // so that a correct padding will cause one less hash + // block to be calculated. Then they can iteratively + // decrypt a record by breaking each byte. See + // "Password Interception in a SSL/TLS Channel", Brice + // Canvel et al. + // + // However, our behavior matches OpenSSL, so we leak + // only as much as they do. + default: + panic("unknown cipher type") + } + } + + // check, strip mac + if hc.mac != nil { + if len(payload) < macSize { + return false, alertBadRecordMAC + } + + // strip mac off payload, b.data + n := len(payload) - macSize + b.data[3] = byte(n >> 8) + b.data[4] = byte(n) + b.resize(recordHeaderLen + n) + remoteMAC := payload[n:] + + hc.mac.Reset() + hc.mac.Write(hc.seq[0:]) + hc.incSeq() + hc.mac.Write(b.data) + + if subtle.ConstantTimeCompare(hc.mac.Sum(), remoteMAC) != 1 || paddingGood != 255 { + return false, alertBadRecordMAC + } + } + + return true, 0 +} + +// padToBlockSize calculates the needed padding block, if any, for a payload. +// On exit, prefix aliases payload and extends to the end of the last full +// block of payload. finalBlock is a fresh slice which contains the contents of +// any suffix of payload as well as the needed padding to make finalBlock a +// full block. +func padToBlockSize(payload []byte, blockSize int) (prefix, finalBlock []byte) { + overrun := len(payload) % blockSize + paddingLen := blockSize - overrun + prefix = payload[:len(payload)-overrun] + finalBlock = make([]byte, blockSize) + copy(finalBlock, payload[len(payload)-overrun:]) + for i := overrun; i < blockSize; i++ { + finalBlock[i] = byte(paddingLen - 1) + } + return +} + +// encrypt encrypts and macs the data in b. +func (hc *halfConn) encrypt(b *block) (bool, alert) { + // mac + if hc.mac != nil { + hc.mac.Reset() + hc.mac.Write(hc.seq[0:]) + hc.incSeq() + hc.mac.Write(b.data) + mac := hc.mac.Sum() + n := len(b.data) + b.resize(n + len(mac)) + copy(b.data[n:], mac) + } + + payload := b.data[recordHeaderLen:] + + // encrypt + if hc.cipher != nil { + switch c := hc.cipher.(type) { + case cipher.Stream: + c.XORKeyStream(payload, payload) + case cipher.BlockMode: + prefix, finalBlock := padToBlockSize(payload, c.BlockSize()) + b.resize(recordHeaderLen + len(prefix) + len(finalBlock)) + c.CryptBlocks(b.data[recordHeaderLen:], prefix) + c.CryptBlocks(b.data[recordHeaderLen+len(prefix):], finalBlock) + default: + panic("unknown cipher type") + } + } + + // update length to include MAC and any block padding needed. + n := len(b.data) - recordHeaderLen + b.data[3] = byte(n >> 8) + b.data[4] = byte(n) + + return true, 0 +} + +// A block is a simple data buffer. +type block struct { + data []byte + off int // index for Read + link *block +} + +// resize resizes block to be n bytes, growing if necessary. +func (b *block) resize(n int) { + if n > cap(b.data) { + b.reserve(n) + } + b.data = b.data[0:n] +} + +// reserve makes sure that block contains a capacity of at least n bytes. +func (b *block) reserve(n int) { + if cap(b.data) >= n { + return + } + m := cap(b.data) + if m == 0 { + m = 1024 + } + for m < n { + m *= 2 + } + data := make([]byte, len(b.data), m) + copy(data, b.data) + b.data = data +} + +// readFromUntil reads from r into b until b contains at least n bytes +// or else returns an error. +func (b *block) readFromUntil(r io.Reader, n int) os.Error { + // quick case + if len(b.data) >= n { + return nil + } + + // read until have enough. + b.reserve(n) + for { + m, err := r.Read(b.data[len(b.data):cap(b.data)]) + b.data = b.data[0 : len(b.data)+m] + if len(b.data) >= n { + break + } + if err != nil { + return err + } + } + return nil +} + +func (b *block) Read(p []byte) (n int, err os.Error) { + n = copy(p, b.data[b.off:]) + b.off += n + return +} + +// newBlock allocates a new block, from hc's free list if possible. +func (hc *halfConn) newBlock() *block { + b := hc.bfree + if b == nil { + return new(block) + } + hc.bfree = b.link + b.link = nil + b.resize(0) + return b +} + +// freeBlock returns a block to hc's free list. +// The protocol is such that each side only has a block or two on +// its free list at a time, so there's no need to worry about +// trimming the list, etc. +func (hc *halfConn) freeBlock(b *block) { + b.link = hc.bfree + hc.bfree = b +} + +// splitBlock splits a block after the first n bytes, +// returning a block with those n bytes and a +// block with the remainder. the latter may be nil. +func (hc *halfConn) splitBlock(b *block, n int) (*block, *block) { + if len(b.data) <= n { + return b, nil + } + bb := hc.newBlock() + bb.resize(len(b.data) - n) + copy(bb.data, b.data[n:]) + b.data = b.data[0:n] + return b, bb +} + +// readRecord reads the next TLS record from the connection +// and updates the record layer state. +// c.in.Mutex <= L; c.input == nil. +func (c *Conn) readRecord(want recordType) os.Error { + // Caller must be in sync with connection: + // handshake data if handshake not yet completed, + // else application data. (We don't support renegotiation.) + switch want { + default: + return c.sendAlert(alertInternalError) + case recordTypeHandshake, recordTypeChangeCipherSpec: + if c.handshakeComplete { + return c.sendAlert(alertInternalError) + } + case recordTypeApplicationData: + if !c.handshakeComplete { + return c.sendAlert(alertInternalError) + } + } + +Again: + if c.rawInput == nil { + c.rawInput = c.in.newBlock() + } + b := c.rawInput + + // Read header, payload. + if err := b.readFromUntil(c.conn, recordHeaderLen); err != nil { + // RFC suggests that EOF without an alertCloseNotify is + // an error, but popular web sites seem to do this, + // so we can't make it an error. + // if err == os.EOF { + // err = io.ErrUnexpectedEOF + // } + if e, ok := err.(net.Error); !ok || !e.Temporary() { + c.setError(err) + } + return err + } + typ := recordType(b.data[0]) + vers := uint16(b.data[1])<<8 | uint16(b.data[2]) + n := int(b.data[3])<<8 | int(b.data[4]) + if c.haveVers && vers != c.vers { + return c.sendAlert(alertProtocolVersion) + } + if n > maxCiphertext { + return c.sendAlert(alertRecordOverflow) + } + if err := b.readFromUntil(c.conn, recordHeaderLen+n); err != nil { + if err == os.EOF { + err = io.ErrUnexpectedEOF + } + if e, ok := err.(net.Error); !ok || !e.Temporary() { + c.setError(err) + } + return err + } + + // Process message. + b, c.rawInput = c.in.splitBlock(b, recordHeaderLen+n) + b.off = recordHeaderLen + if ok, err := c.in.decrypt(b); !ok { + return c.sendAlert(err) + } + data := b.data[b.off:] + if len(data) > maxPlaintext { + c.sendAlert(alertRecordOverflow) + c.in.freeBlock(b) + return c.error() + } + + switch typ { + default: + c.sendAlert(alertUnexpectedMessage) + + case recordTypeAlert: + if len(data) != 2 { + c.sendAlert(alertUnexpectedMessage) + break + } + if alert(data[1]) == alertCloseNotify { + c.setError(os.EOF) + break + } + switch data[0] { + case alertLevelWarning: + // drop on the floor + c.in.freeBlock(b) + goto Again + case alertLevelError: + c.setError(&net.OpError{Op: "remote error", Error: alert(data[1])}) + default: + c.sendAlert(alertUnexpectedMessage) + } + + case recordTypeChangeCipherSpec: + if typ != want || len(data) != 1 || data[0] != 1 { + c.sendAlert(alertUnexpectedMessage) + break + } + err := c.in.changeCipherSpec() + if err != nil { + c.sendAlert(err.(alert)) + } + + case recordTypeApplicationData: + if typ != want { + c.sendAlert(alertUnexpectedMessage) + break + } + c.input = b + b = nil + + case recordTypeHandshake: + // TODO(rsc): Should at least pick off connection close. + if typ != want { + return c.sendAlert(alertNoRenegotiation) + } + c.hand.Write(data) + } + + if b != nil { + c.in.freeBlock(b) + } + return c.error() +} + +// sendAlert sends a TLS alert message. +// c.out.Mutex <= L. +func (c *Conn) sendAlertLocked(err alert) os.Error { + c.tmp[0] = alertLevelError + if err == alertNoRenegotiation { + c.tmp[0] = alertLevelWarning + } + c.tmp[1] = byte(err) + c.writeRecord(recordTypeAlert, c.tmp[0:2]) + // closeNotify is a special case in that it isn't an error: + if err != alertCloseNotify { + return c.setError(&net.OpError{Op: "local error", Error: err}) + } + return nil +} + +// sendAlert sends a TLS alert message. +// L < c.out.Mutex. +func (c *Conn) sendAlert(err alert) os.Error { + c.out.Lock() + defer c.out.Unlock() + return c.sendAlertLocked(err) +} + +// writeRecord writes a TLS record with the given type and payload +// to the connection and updates the record layer state. +// c.out.Mutex <= L. +func (c *Conn) writeRecord(typ recordType, data []byte) (n int, err os.Error) { + b := c.out.newBlock() + for len(data) > 0 { + m := len(data) + if m > maxPlaintext { + m = maxPlaintext + } + b.resize(recordHeaderLen + m) + b.data[0] = byte(typ) + vers := c.vers + if vers == 0 { + vers = maxVersion + } + b.data[1] = byte(vers >> 8) + b.data[2] = byte(vers) + b.data[3] = byte(m >> 8) + b.data[4] = byte(m) + copy(b.data[recordHeaderLen:], data) + c.out.encrypt(b) + _, err = c.conn.Write(b.data) + if err != nil { + break + } + n += m + data = data[m:] + } + c.out.freeBlock(b) + + if typ == recordTypeChangeCipherSpec { + err = c.out.changeCipherSpec() + if err != nil { + // Cannot call sendAlert directly, + // because we already hold c.out.Mutex. + c.tmp[0] = alertLevelError + c.tmp[1] = byte(err.(alert)) + c.writeRecord(recordTypeAlert, c.tmp[0:2]) + c.err = &net.OpError{Op: "local error", Error: err} + return n, c.err + } + } + return +} + +// readHandshake reads the next handshake message from +// the record layer. +// c.in.Mutex < L; c.out.Mutex < L. +func (c *Conn) readHandshake() (interface{}, os.Error) { + for c.hand.Len() < 4 { + if c.err != nil { + return nil, c.err + } + c.readRecord(recordTypeHandshake) + } + + data := c.hand.Bytes() + n := int(data[1])<<16 | int(data[2])<<8 | int(data[3]) + if n > maxHandshake { + c.sendAlert(alertInternalError) + return nil, c.err + } + for c.hand.Len() < 4+n { + if c.err != nil { + return nil, c.err + } + c.readRecord(recordTypeHandshake) + } + data = c.hand.Next(4 + n) + var m handshakeMessage + switch data[0] { + case typeClientHello: + m = new(clientHelloMsg) + case typeServerHello: + m = new(serverHelloMsg) + case typeCertificate: + m = new(certificateMsg) + case typeCertificateRequest: + m = new(certificateRequestMsg) + case typeCertificateStatus: + m = new(certificateStatusMsg) + case typeServerKeyExchange: + m = new(serverKeyExchangeMsg) + case typeServerHelloDone: + m = new(serverHelloDoneMsg) + case typeClientKeyExchange: + m = new(clientKeyExchangeMsg) + case typeCertificateVerify: + m = new(certificateVerifyMsg) + case typeNextProtocol: + m = new(nextProtoMsg) + case typeFinished: + m = new(finishedMsg) + default: + c.sendAlert(alertUnexpectedMessage) + return nil, alertUnexpectedMessage + } + + // The handshake message unmarshallers + // expect to be able to keep references to data, + // so pass in a fresh copy that won't be overwritten. + data = append([]byte(nil), data...) + + if !m.unmarshal(data) { + c.sendAlert(alertUnexpectedMessage) + return nil, alertUnexpectedMessage + } + return m, nil +} + +// Write writes data to the connection. +func (c *Conn) Write(b []byte) (n int, err os.Error) { + if err = c.Handshake(); err != nil { + return + } + + c.out.Lock() + defer c.out.Unlock() + + if !c.handshakeComplete { + return 0, alertInternalError + } + if c.err != nil { + return 0, c.err + } + return c.writeRecord(recordTypeApplicationData, b) +} + +// Read can be made to time out and return err == os.EAGAIN +// after a fixed time limit; see SetTimeout and SetReadTimeout. +func (c *Conn) Read(b []byte) (n int, err os.Error) { + if err = c.Handshake(); err != nil { + return + } + + c.in.Lock() + defer c.in.Unlock() + + for c.input == nil && c.err == nil { + if err := c.readRecord(recordTypeApplicationData); err != nil { + // Soft error, like EAGAIN + return 0, err + } + } + if c.err != nil { + return 0, c.err + } + n, err = c.input.Read(b) + if c.input.off >= len(c.input.data) { + c.in.freeBlock(c.input) + c.input = nil + } + return n, nil +} + +// Close closes the connection. +func (c *Conn) Close() os.Error { + if err := c.Handshake(); err != nil { + return err + } + return c.sendAlert(alertCloseNotify) +} + +// Handshake runs the client or server handshake +// protocol if it has not yet been run. +// Most uses of this package need not call Handshake +// explicitly: the first Read or Write will call it automatically. +func (c *Conn) Handshake() os.Error { + c.handshakeMutex.Lock() + defer c.handshakeMutex.Unlock() + if err := c.error(); err != nil { + return err + } + if c.handshakeComplete { + return nil + } + if c.isClient { + return c.clientHandshake() + } + return c.serverHandshake() +} + +// ConnectionState returns basic TLS details about the connection. +func (c *Conn) ConnectionState() ConnectionState { + c.handshakeMutex.Lock() + defer c.handshakeMutex.Unlock() + + var state ConnectionState + state.HandshakeComplete = c.handshakeComplete + if c.handshakeComplete { + state.NegotiatedProtocol = c.clientProtocol + state.NegotiatedProtocolIsMutual = !c.clientProtocolFallback + state.CipherSuite = c.cipherSuite + state.PeerCertificates = c.peerCertificates + state.VerifiedChains = c.verifiedChains + } + + return state +} + +// OCSPResponse returns the stapled OCSP response from the TLS server, if +// any. (Only valid for client connections.) +func (c *Conn) OCSPResponse() []byte { + c.handshakeMutex.Lock() + defer c.handshakeMutex.Unlock() + + return c.ocspResponse +} + +// VerifyHostname checks that the peer certificate chain is valid for +// connecting to host. If so, it returns nil; if not, it returns an os.Error +// describing the problem. +func (c *Conn) VerifyHostname(host string) os.Error { + c.handshakeMutex.Lock() + defer c.handshakeMutex.Unlock() + if !c.isClient { + return os.NewError("VerifyHostname called on TLS server connection") + } + if !c.handshakeComplete { + return os.NewError("TLS handshake has not yet been performed") + } + return c.peerCertificates[0].VerifyHostname(host) +} diff --git a/src/pkg/crypto/tls/conn_test.go b/src/pkg/crypto/tls/conn_test.go new file mode 100644 index 000000000..f44a50bed --- /dev/null +++ b/src/pkg/crypto/tls/conn_test.go @@ -0,0 +1,52 @@ +// Copyright 2010 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package tls + +import ( + "testing" +) + +func TestRoundUp(t *testing.T) { + if roundUp(0, 16) != 0 || + roundUp(1, 16) != 16 || + roundUp(15, 16) != 16 || + roundUp(16, 16) != 16 || + roundUp(17, 16) != 32 { + t.Error("roundUp broken") + } +} + +var paddingTests = []struct { + in []byte + good bool + expectedLen int +}{ + {[]byte{1, 2, 3, 4, 0}, true, 4}, + {[]byte{1, 2, 3, 4, 0, 1}, false, 0}, + {[]byte{1, 2, 3, 4, 99, 99}, false, 0}, + {[]byte{1, 2, 3, 4, 1, 1}, true, 4}, + {[]byte{1, 2, 3, 2, 2, 2}, true, 3}, + {[]byte{1, 2, 3, 3, 3, 3}, true, 2}, + {[]byte{1, 2, 3, 4, 3, 3}, false, 0}, + {[]byte{1, 4, 4, 4, 4, 4}, true, 1}, + {[]byte{5, 5, 5, 5, 5, 5}, true, 0}, + {[]byte{6, 6, 6, 6, 6, 6}, false, 0}, +} + +func TestRemovePadding(t *testing.T) { + for i, test := range paddingTests { + payload, good := removePadding(test.in) + expectedGood := byte(255) + if !test.good { + expectedGood = 0 + } + if good != expectedGood { + t.Errorf("#%d: wrong validity, want:%d got:%d", i, expectedGood, good) + } + if good == 255 && len(payload) != test.expectedLen { + t.Errorf("#%d: got %d, want %d", i, len(payload), test.expectedLen) + } + } +} diff --git a/src/pkg/crypto/tls/generate_cert.go b/src/pkg/crypto/tls/generate_cert.go new file mode 100644 index 000000000..41206e276 --- /dev/null +++ b/src/pkg/crypto/tls/generate_cert.go @@ -0,0 +1,72 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Generate a self-signed X.509 certificate for a TLS server. Outputs to +// 'cert.pem' and 'key.pem' and will overwrite existing files. + +package main + +import ( + "big" + "crypto/x509/pkix" + "crypto/rand" + "crypto/rsa" + "crypto/x509" + "encoding/pem" + "flag" + "log" + "os" + "time" +) + +var hostName *string = flag.String("host", "127.0.0.1", "Hostname to generate a certificate for") + +func main() { + flag.Parse() + + priv, err := rsa.GenerateKey(rand.Reader, 1024) + if err != nil { + log.Fatalf("failed to generate private key: %s", err) + return + } + + now := time.Seconds() + + template := x509.Certificate{ + SerialNumber: new(big.Int).SetInt64(0), + Subject: pkix.Name{ + CommonName: *hostName, + Organization: []string{"Acme Co"}, + }, + NotBefore: time.SecondsToUTC(now - 300), + NotAfter: time.SecondsToUTC(now + 60*60*24*365), // valid for 1 year. + + SubjectKeyId: []byte{1, 2, 3, 4}, + KeyUsage: x509.KeyUsageKeyEncipherment | x509.KeyUsageDigitalSignature, + } + + derBytes, err := x509.CreateCertificate(rand.Reader, &template, &template, &priv.PublicKey, priv) + if err != nil { + log.Fatalf("Failed to create certificate: %s", err) + return + } + + certOut, err := os.Create("cert.pem") + if err != nil { + log.Fatalf("failed to open cert.pem for writing: %s", err) + return + } + pem.Encode(certOut, &pem.Block{Type: "CERTIFICATE", Bytes: derBytes}) + certOut.Close() + log.Print("written cert.pem\n") + + keyOut, err := os.OpenFile("key.pem", os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0600) + if err != nil { + log.Print("failed to open key.pem for writing:", err) + return + } + pem.Encode(keyOut, &pem.Block{Type: "RSA PRIVATE KEY", Bytes: x509.MarshalPKCS1PrivateKey(priv)}) + keyOut.Close() + log.Print("written key.pem\n") +} diff --git a/src/pkg/crypto/tls/handshake_client.go b/src/pkg/crypto/tls/handshake_client.go new file mode 100644 index 000000000..15604cea7 --- /dev/null +++ b/src/pkg/crypto/tls/handshake_client.go @@ -0,0 +1,315 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package tls + +import ( + "crypto" + "crypto/rsa" + "crypto/subtle" + "crypto/x509" + "io" + "os" +) + +func (c *Conn) clientHandshake() os.Error { + finishedHash := newFinishedHash() + + if c.config == nil { + c.config = defaultConfig() + } + + hello := &clientHelloMsg{ + vers: maxVersion, + cipherSuites: c.config.cipherSuites(), + compressionMethods: []uint8{compressionNone}, + random: make([]byte, 32), + ocspStapling: true, + serverName: c.config.ServerName, + supportedCurves: []uint16{curveP256, curveP384, curveP521}, + supportedPoints: []uint8{pointFormatUncompressed}, + nextProtoNeg: len(c.config.NextProtos) > 0, + } + + t := uint32(c.config.time()) + hello.random[0] = byte(t >> 24) + hello.random[1] = byte(t >> 16) + hello.random[2] = byte(t >> 8) + hello.random[3] = byte(t) + _, err := io.ReadFull(c.config.rand(), hello.random[4:]) + if err != nil { + c.sendAlert(alertInternalError) + return os.NewError("short read from Rand") + } + + finishedHash.Write(hello.marshal()) + c.writeRecord(recordTypeHandshake, hello.marshal()) + + msg, err := c.readHandshake() + if err != nil { + return err + } + serverHello, ok := msg.(*serverHelloMsg) + if !ok { + return c.sendAlert(alertUnexpectedMessage) + } + finishedHash.Write(serverHello.marshal()) + + vers, ok := mutualVersion(serverHello.vers) + if !ok { + return c.sendAlert(alertProtocolVersion) + } + c.vers = vers + c.haveVers = true + + if serverHello.compressionMethod != compressionNone { + return c.sendAlert(alertUnexpectedMessage) + } + + if !hello.nextProtoNeg && serverHello.nextProtoNeg { + c.sendAlert(alertHandshakeFailure) + return os.NewError("server advertised unrequested NPN") + } + + suite, suiteId := mutualCipherSuite(c.config.cipherSuites(), serverHello.cipherSuite) + if suite == nil { + return c.sendAlert(alertHandshakeFailure) + } + + msg, err = c.readHandshake() + if err != nil { + return err + } + certMsg, ok := msg.(*certificateMsg) + if !ok || len(certMsg.certificates) == 0 { + return c.sendAlert(alertUnexpectedMessage) + } + finishedHash.Write(certMsg.marshal()) + + certs := make([]*x509.Certificate, len(certMsg.certificates)) + for i, asn1Data := range certMsg.certificates { + cert, err := x509.ParseCertificate(asn1Data) + if err != nil { + c.sendAlert(alertBadCertificate) + return os.NewError("failed to parse certificate from server: " + err.String()) + } + certs[i] = cert + } + + // If we don't have a root CA set configured then anything is accepted. + // TODO(rsc): Find certificates for OS X 10.6. + if c.config.RootCAs != nil { + opts := x509.VerifyOptions{ + Roots: c.config.RootCAs, + CurrentTime: c.config.time(), + DNSName: c.config.ServerName, + Intermediates: x509.NewCertPool(), + } + + for i, cert := range certs { + if i == 0 { + continue + } + opts.Intermediates.AddCert(cert) + } + c.verifiedChains, err = certs[0].Verify(opts) + if err != nil { + c.sendAlert(alertBadCertificate) + return err + } + } + + if _, ok := certs[0].PublicKey.(*rsa.PublicKey); !ok { + return c.sendAlert(alertUnsupportedCertificate) + } + + c.peerCertificates = certs + + if serverHello.ocspStapling { + msg, err = c.readHandshake() + if err != nil { + return err + } + cs, ok := msg.(*certificateStatusMsg) + if !ok { + return c.sendAlert(alertUnexpectedMessage) + } + finishedHash.Write(cs.marshal()) + + if cs.statusType == statusTypeOCSP { + c.ocspResponse = cs.response + } + } + + msg, err = c.readHandshake() + if err != nil { + return err + } + + keyAgreement := suite.ka() + + skx, ok := msg.(*serverKeyExchangeMsg) + if ok { + finishedHash.Write(skx.marshal()) + err = keyAgreement.processServerKeyExchange(c.config, hello, serverHello, certs[0], skx) + if err != nil { + c.sendAlert(alertUnexpectedMessage) + return err + } + + msg, err = c.readHandshake() + if err != nil { + return err + } + } + + transmitCert := false + certReq, ok := msg.(*certificateRequestMsg) + if ok { + // We only accept certificates with RSA keys. + rsaAvail := false + for _, certType := range certReq.certificateTypes { + if certType == certTypeRSASign { + rsaAvail = true + break + } + } + + // For now, only send a certificate back if the server gives us an + // empty list of certificateAuthorities. + // + // RFC 4346 on the certificateAuthorities field: + // A list of the distinguished names of acceptable certificate + // authorities. These distinguished names may specify a desired + // distinguished name for a root CA or for a subordinate CA; thus, + // this message can be used to describe both known roots and a + // desired authorization space. If the certificate_authorities + // list is empty then the client MAY send any certificate of the + // appropriate ClientCertificateType, unless there is some + // external arrangement to the contrary. + if rsaAvail && len(certReq.certificateAuthorities) == 0 { + transmitCert = true + } + + finishedHash.Write(certReq.marshal()) + + msg, err = c.readHandshake() + if err != nil { + return err + } + } + + shd, ok := msg.(*serverHelloDoneMsg) + if !ok { + return c.sendAlert(alertUnexpectedMessage) + } + finishedHash.Write(shd.marshal()) + + var cert *x509.Certificate + if transmitCert { + certMsg = new(certificateMsg) + if len(c.config.Certificates) > 0 { + cert, err = x509.ParseCertificate(c.config.Certificates[0].Certificate[0]) + if err == nil && cert.PublicKeyAlgorithm == x509.RSA { + certMsg.certificates = c.config.Certificates[0].Certificate + } else { + cert = nil + } + } + finishedHash.Write(certMsg.marshal()) + c.writeRecord(recordTypeHandshake, certMsg.marshal()) + } + + preMasterSecret, ckx, err := keyAgreement.generateClientKeyExchange(c.config, hello, certs[0]) + if err != nil { + c.sendAlert(alertInternalError) + return err + } + if ckx != nil { + finishedHash.Write(ckx.marshal()) + c.writeRecord(recordTypeHandshake, ckx.marshal()) + } + + if cert != nil { + certVerify := new(certificateVerifyMsg) + var digest [36]byte + copy(digest[0:16], finishedHash.serverMD5.Sum()) + copy(digest[16:36], finishedHash.serverSHA1.Sum()) + signed, err := rsa.SignPKCS1v15(c.config.rand(), c.config.Certificates[0].PrivateKey, crypto.MD5SHA1, digest[0:]) + if err != nil { + return c.sendAlert(alertInternalError) + } + certVerify.signature = signed + + finishedHash.Write(certVerify.marshal()) + c.writeRecord(recordTypeHandshake, certVerify.marshal()) + } + + masterSecret, clientMAC, serverMAC, clientKey, serverKey, clientIV, serverIV := + keysFromPreMasterSecret10(preMasterSecret, hello.random, serverHello.random, suite.macLen, suite.keyLen, suite.ivLen) + + clientCipher := suite.cipher(clientKey, clientIV, false /* not for reading */ ) + clientHash := suite.mac(clientMAC) + c.out.prepareCipherSpec(clientCipher, clientHash) + c.writeRecord(recordTypeChangeCipherSpec, []byte{1}) + + if serverHello.nextProtoNeg { + nextProto := new(nextProtoMsg) + proto, fallback := mutualProtocol(c.config.NextProtos, serverHello.nextProtos) + nextProto.proto = proto + c.clientProtocol = proto + c.clientProtocolFallback = fallback + + finishedHash.Write(nextProto.marshal()) + c.writeRecord(recordTypeHandshake, nextProto.marshal()) + } + + finished := new(finishedMsg) + finished.verifyData = finishedHash.clientSum(masterSecret) + finishedHash.Write(finished.marshal()) + c.writeRecord(recordTypeHandshake, finished.marshal()) + + serverCipher := suite.cipher(serverKey, serverIV, true /* for reading */ ) + serverHash := suite.mac(serverMAC) + c.in.prepareCipherSpec(serverCipher, serverHash) + c.readRecord(recordTypeChangeCipherSpec) + if c.err != nil { + return c.err + } + + msg, err = c.readHandshake() + if err != nil { + return err + } + serverFinished, ok := msg.(*finishedMsg) + if !ok { + return c.sendAlert(alertUnexpectedMessage) + } + + verify := finishedHash.serverSum(masterSecret) + if len(verify) != len(serverFinished.verifyData) || + subtle.ConstantTimeCompare(verify, serverFinished.verifyData) != 1 { + return c.sendAlert(alertHandshakeFailure) + } + + c.handshakeComplete = true + c.cipherSuite = suiteId + return nil +} + +// mutualProtocol finds the mutual Next Protocol Negotiation protocol given the +// set of client and server supported protocols. The set of client supported +// protocols must not be empty. It returns the resulting protocol and flag +// indicating if the fallback case was reached. +func mutualProtocol(clientProtos, serverProtos []string) (string, bool) { + for _, s := range serverProtos { + for _, c := range clientProtos { + if s == c { + return s, false + } + } + } + + return clientProtos[0], true +} diff --git a/src/pkg/crypto/tls/handshake_client_test.go b/src/pkg/crypto/tls/handshake_client_test.go new file mode 100644 index 000000000..3f91c7acf --- /dev/null +++ b/src/pkg/crypto/tls/handshake_client_test.go @@ -0,0 +1,211 @@ +// Copyright 2010 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package tls + +import ( + "bytes" + "flag" + "io" + "net" + "testing" +) + +func testClientScript(t *testing.T, name string, clientScript [][]byte, config *Config) { + c, s := net.Pipe() + cli := Client(c, config) + go func() { + cli.Write([]byte("hello\n")) + cli.Close() + }() + + defer c.Close() + for i, b := range clientScript { + if i%2 == 1 { + s.Write(b) + continue + } + bb := make([]byte, len(b)) + _, err := io.ReadFull(s, bb) + if err != nil { + t.Fatalf("%s #%d: %s", name, i, err) + } + if !bytes.Equal(b, bb) { + t.Fatalf("%s #%d: mismatch on read: got:%x want:%x", name, i, bb, b) + } + } +} + +func TestHandshakeClientRC4(t *testing.T) { + testClientScript(t, "RC4", rc4ClientScript, testConfig) +} + +var connect = flag.Bool("connect", false, "connect to a TLS server on :10443") + +func TestRunClient(t *testing.T) { + if !*connect { + return + } + + testConfig.CipherSuites = []uint16{TLS_ECDHE_RSA_WITH_RC4_128_SHA} + + conn, err := Dial("tcp", "127.0.0.1:10443", testConfig) + if err != nil { + t.Fatal(err) + } + + conn.Write([]byte("hello\n")) + conn.Close() +} + +// Script of interaction with gnutls implementation. +// The values for this test are obtained by building and running in client mode: +// % gotest -test.run "TestRunClient" -connect +// and then: +// % gnutls-serv -p 10443 --debug 100 --x509keyfile key.pem --x509certfile cert.pem -a > /tmp/log 2>&1 +// % python parse-gnutls-cli-debug-log.py < /tmp/log +// +// Where key.pem is: +// -----BEGIN RSA PRIVATE KEY----- +// MIIBPAIBAAJBAJ+zw4Qnlf8SMVIPFe9GEcStgOY2Ww/dgNdhjeD8ckUJNP5VZkVD +// TGiXav6ooKXfX3j/7tdkuD8Ey2//Kv7+ue0CAwEAAQJAN6W31vDEP2DjdqhzCDDu +// OA4NACqoiFqyblo7yc2tM4h4xMbC3Yx5UKMN9ZkCtX0gzrz6DyF47bdKcWBzNWCj +// gQIhANEoojVt7hq+SQ6MCN6FTAysGgQf56Q3TYoJMoWvdiXVAiEAw3e3rc+VJpOz +// rHuDo6bgpjUAAXM+v3fcpsfZSNO6V7kCIQCtbVjanpUwvZkMI9by02oUk9taki3b +// PzPfAfNPYAbCJQIhAJXNQDWyqwn/lGmR11cqY2y9nZ1+5w3yHGatLrcDnQHxAiEA +// vnlEGo8K85u+KwIOimM48ZG8oTk7iFdkqLJR1utT3aU= +// -----END RSA PRIVATE KEY----- +// +// and cert.pem is: +// -----BEGIN CERTIFICATE----- +// MIIBoDCCAUoCAQAwDQYJKoZIhvcNAQEEBQAwYzELMAkGA1UEBhMCQVUxEzARBgNV +// BAgTClF1ZWVuc2xhbmQxGjAYBgNVBAoTEUNyeXB0U29mdCBQdHkgTHRkMSMwIQYD +// VQQDExpTZXJ2ZXIgdGVzdCBjZXJ0ICg1MTIgYml0KTAeFw05NzA5MDkwMzQxMjZa +// Fw05NzEwMDkwMzQxMjZaMF4xCzAJBgNVBAYTAkFVMRMwEQYDVQQIEwpTb21lLVN0 +// YXRlMSEwHwYDVQQKExhJbnRlcm5ldCBXaWRnaXRzIFB0eSBMdGQxFzAVBgNVBAMT +// DkVyaWMgdGhlIFlvdW5nMFEwCQYFKw4DAgwFAANEAAJBALVEqPODnpI4rShlY8S7 +// tB713JNvabvn6Gned7zylwLLiXQAo/PAT6mfdWPTyCX9RlId/Aroh1ou893BA32Q +// sggwDQYJKoZIhvcNAQEEBQADQQCU5SSgapJSdRXJoX+CpCvFy+JVh9HpSjCpSNKO +// 19raHv98hKAUJuP9HyM+SUsffO6mAIgitUaqW8/wDMePhEC3 +// -----END CERTIFICATE----- +var rc4ClientScript = [][]byte{ + { + 0x16, 0x03, 0x01, 0x00, 0x4a, 0x01, 0x00, 0x00, + 0x46, 0x03, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0x00, 0x05, + 0x01, 0x00, 0x00, 0x1b, 0x00, 0x05, 0x00, 0x05, + 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x0a, 0x00, + 0x08, 0x00, 0x06, 0x00, 0x17, 0x00, 0x18, 0x00, + 0x19, 0x00, 0x0b, 0x00, 0x02, 0x01, 0x00, + }, + + { + 0x16, 0x03, 0x01, 0x00, 0x4a, 0x02, 0x00, 0x00, + 0x46, 0x03, 0x01, 0x4d, 0x0a, 0x56, 0x16, 0xb5, + 0x91, 0xd1, 0xcb, 0x80, 0x4d, 0xc7, 0x46, 0xf3, + 0x37, 0x0c, 0xef, 0xea, 0x64, 0x11, 0x14, 0x56, + 0x97, 0x9b, 0xc5, 0x67, 0x08, 0xb7, 0x13, 0xea, + 0xf8, 0xc9, 0xb3, 0x20, 0xe2, 0xfc, 0x41, 0xf6, + 0x96, 0x90, 0x9d, 0x43, 0x9b, 0xe9, 0x6e, 0xf8, + 0x41, 0x16, 0xcc, 0xf3, 0xc7, 0xde, 0xda, 0x5a, + 0xa1, 0x33, 0x69, 0xe2, 0xde, 0x5b, 0xaf, 0x2a, + 0x92, 0xe7, 0xd4, 0xa0, 0x00, 0x05, 0x00, 0x16, + 0x03, 0x01, 0x01, 0xf7, 0x0b, 0x00, 0x01, 0xf3, + 0x00, 0x01, 0xf0, 0x00, 0x01, 0xed, 0x30, 0x82, + 0x01, 0xe9, 0x30, 0x82, 0x01, 0x52, 0x02, 0x01, + 0x06, 0x30, 0x0d, 0x06, 0x09, 0x2a, 0x86, 0x48, + 0x86, 0xf7, 0x0d, 0x01, 0x01, 0x04, 0x05, 0x00, + 0x30, 0x5b, 0x31, 0x0b, 0x30, 0x09, 0x06, 0x03, + 0x55, 0x04, 0x06, 0x13, 0x02, 0x41, 0x55, 0x31, + 0x13, 0x30, 0x11, 0x06, 0x03, 0x55, 0x04, 0x08, + 0x13, 0x0a, 0x51, 0x75, 0x65, 0x65, 0x6e, 0x73, + 0x6c, 0x61, 0x6e, 0x64, 0x31, 0x1a, 0x30, 0x18, + 0x06, 0x03, 0x55, 0x04, 0x0a, 0x13, 0x11, 0x43, + 0x72, 0x79, 0x70, 0x74, 0x53, 0x6f, 0x66, 0x74, + 0x20, 0x50, 0x74, 0x79, 0x20, 0x4c, 0x74, 0x64, + 0x31, 0x1b, 0x30, 0x19, 0x06, 0x03, 0x55, 0x04, + 0x03, 0x13, 0x12, 0x54, 0x65, 0x73, 0x74, 0x20, + 0x43, 0x41, 0x20, 0x28, 0x31, 0x30, 0x32, 0x34, + 0x20, 0x62, 0x69, 0x74, 0x29, 0x30, 0x1e, 0x17, + 0x0d, 0x30, 0x30, 0x31, 0x30, 0x31, 0x36, 0x32, + 0x32, 0x33, 0x31, 0x30, 0x33, 0x5a, 0x17, 0x0d, + 0x30, 0x33, 0x30, 0x31, 0x31, 0x34, 0x32, 0x32, + 0x33, 0x31, 0x30, 0x33, 0x5a, 0x30, 0x63, 0x31, + 0x0b, 0x30, 0x09, 0x06, 0x03, 0x55, 0x04, 0x06, + 0x13, 0x02, 0x41, 0x55, 0x31, 0x13, 0x30, 0x11, + 0x06, 0x03, 0x55, 0x04, 0x08, 0x13, 0x0a, 0x51, + 0x75, 0x65, 0x65, 0x6e, 0x73, 0x6c, 0x61, 0x6e, + 0x64, 0x31, 0x1a, 0x30, 0x18, 0x06, 0x03, 0x55, + 0x04, 0x0a, 0x13, 0x11, 0x43, 0x72, 0x79, 0x70, + 0x74, 0x53, 0x6f, 0x66, 0x74, 0x20, 0x50, 0x74, + 0x79, 0x20, 0x4c, 0x74, 0x64, 0x31, 0x23, 0x30, + 0x21, 0x06, 0x03, 0x55, 0x04, 0x03, 0x13, 0x1a, + 0x53, 0x65, 0x72, 0x76, 0x65, 0x72, 0x20, 0x74, + 0x65, 0x73, 0x74, 0x20, 0x63, 0x65, 0x72, 0x74, + 0x20, 0x28, 0x35, 0x31, 0x32, 0x20, 0x62, 0x69, + 0x74, 0x29, 0x30, 0x5c, 0x30, 0x0d, 0x06, 0x09, + 0x2a, 0x86, 0x48, 0x86, 0xf7, 0x0d, 0x01, 0x01, + 0x01, 0x05, 0x00, 0x03, 0x4b, 0x00, 0x30, 0x48, + 0x02, 0x41, 0x00, 0x9f, 0xb3, 0xc3, 0x84, 0x27, + 0x95, 0xff, 0x12, 0x31, 0x52, 0x0f, 0x15, 0xef, + 0x46, 0x11, 0xc4, 0xad, 0x80, 0xe6, 0x36, 0x5b, + 0x0f, 0xdd, 0x80, 0xd7, 0x61, 0x8d, 0xe0, 0xfc, + 0x72, 0x45, 0x09, 0x34, 0xfe, 0x55, 0x66, 0x45, + 0x43, 0x4c, 0x68, 0x97, 0x6a, 0xfe, 0xa8, 0xa0, + 0xa5, 0xdf, 0x5f, 0x78, 0xff, 0xee, 0xd7, 0x64, + 0xb8, 0x3f, 0x04, 0xcb, 0x6f, 0xff, 0x2a, 0xfe, + 0xfe, 0xb9, 0xed, 0x02, 0x03, 0x01, 0x00, 0x01, + 0x30, 0x0d, 0x06, 0x09, 0x2a, 0x86, 0x48, 0x86, + 0xf7, 0x0d, 0x01, 0x01, 0x04, 0x05, 0x00, 0x03, + 0x81, 0x81, 0x00, 0x93, 0xd2, 0x0a, 0xc5, 0x41, + 0xe6, 0x5a, 0xa9, 0x86, 0xf9, 0x11, 0x87, 0xe4, + 0xdb, 0x45, 0xe2, 0xc5, 0x95, 0x78, 0x1a, 0x6c, + 0x80, 0x6d, 0x73, 0x1f, 0xb4, 0x6d, 0x44, 0xa3, + 0xba, 0x86, 0x88, 0xc8, 0x58, 0xcd, 0x1c, 0x06, + 0x35, 0x6c, 0x44, 0x62, 0x88, 0xdf, 0xe4, 0xf6, + 0x64, 0x61, 0x95, 0xef, 0x4a, 0xa6, 0x7f, 0x65, + 0x71, 0xd7, 0x6b, 0x88, 0x39, 0xf6, 0x32, 0xbf, + 0xac, 0x93, 0x67, 0x69, 0x51, 0x8c, 0x93, 0xec, + 0x48, 0x5f, 0xc9, 0xb1, 0x42, 0xf9, 0x55, 0xd2, + 0x7e, 0x4e, 0xf4, 0xf2, 0x21, 0x6b, 0x90, 0x57, + 0xe6, 0xd7, 0x99, 0x9e, 0x41, 0xca, 0x80, 0xbf, + 0x1a, 0x28, 0xa2, 0xca, 0x5b, 0x50, 0x4a, 0xed, + 0x84, 0xe7, 0x82, 0xc7, 0xd2, 0xcf, 0x36, 0x9e, + 0x6a, 0x67, 0xb9, 0x88, 0xa7, 0xf3, 0x8a, 0xd0, + 0x04, 0xf8, 0xe8, 0xc6, 0x17, 0xe3, 0xc5, 0x29, + 0xbc, 0x17, 0xf1, 0x16, 0x03, 0x01, 0x00, 0x04, + 0x0e, 0x00, 0x00, 0x00, + }, + + { + 0x16, 0x03, 0x01, 0x00, 0x46, 0x10, 0x00, 0x00, + 0x42, 0x00, 0x40, 0x87, 0xa1, 0x1f, 0x14, 0xe1, + 0xfb, 0x91, 0xac, 0x58, 0x2e, 0xf3, 0x71, 0xce, + 0x01, 0x85, 0x2c, 0xc7, 0xfe, 0x84, 0x87, 0x82, + 0xb7, 0x57, 0xdb, 0x37, 0x4d, 0x46, 0x83, 0x67, + 0x52, 0x82, 0x51, 0x01, 0x95, 0x23, 0x68, 0x69, + 0x6b, 0xd0, 0xa7, 0xa7, 0xe5, 0x88, 0xd0, 0x47, + 0x71, 0xb8, 0xd2, 0x03, 0x05, 0x25, 0x56, 0x5c, + 0x10, 0x08, 0xc6, 0x9b, 0xd4, 0x67, 0xcd, 0x28, + 0xbe, 0x9c, 0x48, 0x14, 0x03, 0x01, 0x00, 0x01, + 0x01, 0x16, 0x03, 0x01, 0x00, 0x24, 0xc1, 0xb8, + 0xd3, 0x7f, 0xc5, 0xc2, 0x5a, 0x1d, 0x6d, 0x5b, + 0x2d, 0x5c, 0x82, 0x87, 0xc2, 0x6f, 0x0d, 0x63, + 0x7b, 0x72, 0x2b, 0xda, 0x69, 0xc4, 0xfe, 0x3c, + 0x84, 0xa1, 0x5a, 0x62, 0x38, 0x37, 0xc6, 0x54, + 0x25, 0x2a, + }, + + { + 0x14, 0x03, 0x01, 0x00, 0x01, 0x01, 0x16, 0x03, + 0x01, 0x00, 0x24, 0xea, 0x88, 0x9c, 0x00, 0xf6, + 0x35, 0xb8, 0x42, 0x7f, 0x15, 0x17, 0x76, 0x5e, + 0x4b, 0x24, 0xcb, 0x7e, 0xa0, 0x7b, 0xc3, 0x70, + 0x52, 0x0a, 0x88, 0x2a, 0x7a, 0x45, 0x59, 0x90, + 0x59, 0xac, 0xc6, 0xb5, 0x56, 0x55, 0x96, + }, +} diff --git a/src/pkg/crypto/tls/handshake_messages.go b/src/pkg/crypto/tls/handshake_messages.go new file mode 100644 index 000000000..6645adce4 --- /dev/null +++ b/src/pkg/crypto/tls/handshake_messages.go @@ -0,0 +1,904 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package tls + +type clientHelloMsg struct { + raw []byte + vers uint16 + random []byte + sessionId []byte + cipherSuites []uint16 + compressionMethods []uint8 + nextProtoNeg bool + serverName string + ocspStapling bool + supportedCurves []uint16 + supportedPoints []uint8 +} + +func (m *clientHelloMsg) marshal() []byte { + if m.raw != nil { + return m.raw + } + + length := 2 + 32 + 1 + len(m.sessionId) + 2 + len(m.cipherSuites)*2 + 1 + len(m.compressionMethods) + numExtensions := 0 + extensionsLength := 0 + if m.nextProtoNeg { + numExtensions++ + } + if m.ocspStapling { + extensionsLength += 1 + 2 + 2 + numExtensions++ + } + if len(m.serverName) > 0 { + extensionsLength += 5 + len(m.serverName) + numExtensions++ + } + if len(m.supportedCurves) > 0 { + extensionsLength += 2 + 2*len(m.supportedCurves) + numExtensions++ + } + if len(m.supportedPoints) > 0 { + extensionsLength += 1 + len(m.supportedPoints) + numExtensions++ + } + if numExtensions > 0 { + extensionsLength += 4 * numExtensions + length += 2 + extensionsLength + } + + x := make([]byte, 4+length) + x[0] = typeClientHello + x[1] = uint8(length >> 16) + x[2] = uint8(length >> 8) + x[3] = uint8(length) + x[4] = uint8(m.vers >> 8) + x[5] = uint8(m.vers) + copy(x[6:38], m.random) + x[38] = uint8(len(m.sessionId)) + copy(x[39:39+len(m.sessionId)], m.sessionId) + y := x[39+len(m.sessionId):] + y[0] = uint8(len(m.cipherSuites) >> 7) + y[1] = uint8(len(m.cipherSuites) << 1) + for i, suite := range m.cipherSuites { + y[2+i*2] = uint8(suite >> 8) + y[3+i*2] = uint8(suite) + } + z := y[2+len(m.cipherSuites)*2:] + z[0] = uint8(len(m.compressionMethods)) + copy(z[1:], m.compressionMethods) + + z = z[1+len(m.compressionMethods):] + if numExtensions > 0 { + z[0] = byte(extensionsLength >> 8) + z[1] = byte(extensionsLength) + z = z[2:] + } + if m.nextProtoNeg { + z[0] = byte(extensionNextProtoNeg >> 8) + z[1] = byte(extensionNextProtoNeg) + // The length is always 0 + z = z[4:] + } + if len(m.serverName) > 0 { + z[0] = byte(extensionServerName >> 8) + z[1] = byte(extensionServerName) + l := len(m.serverName) + 5 + z[2] = byte(l >> 8) + z[3] = byte(l) + z = z[4:] + + // RFC 3546, section 3.1 + // + // struct { + // NameType name_type; + // select (name_type) { + // case host_name: HostName; + // } name; + // } ServerName; + // + // enum { + // host_name(0), (255) + // } NameType; + // + // opaque HostName<1..2^16-1>; + // + // struct { + // ServerName server_name_list<1..2^16-1> + // } ServerNameList; + + z[0] = byte((len(m.serverName) + 3) >> 8) + z[1] = byte(len(m.serverName) + 3) + z[3] = byte(len(m.serverName) >> 8) + z[4] = byte(len(m.serverName)) + copy(z[5:], []byte(m.serverName)) + z = z[l:] + } + if m.ocspStapling { + // RFC 4366, section 3.6 + z[0] = byte(extensionStatusRequest >> 8) + z[1] = byte(extensionStatusRequest) + z[2] = 0 + z[3] = 5 + z[4] = 1 // OCSP type + // Two zero valued uint16s for the two lengths. + z = z[9:] + } + if len(m.supportedCurves) > 0 { + // http://tools.ietf.org/html/rfc4492#section-5.5.1 + z[0] = byte(extensionSupportedCurves >> 8) + z[1] = byte(extensionSupportedCurves) + l := 2 + 2*len(m.supportedCurves) + z[2] = byte(l >> 8) + z[3] = byte(l) + l -= 2 + z[4] = byte(l >> 8) + z[5] = byte(l) + z = z[6:] + for _, curve := range m.supportedCurves { + z[0] = byte(curve >> 8) + z[1] = byte(curve) + z = z[2:] + } + } + if len(m.supportedPoints) > 0 { + // http://tools.ietf.org/html/rfc4492#section-5.5.2 + z[0] = byte(extensionSupportedPoints >> 8) + z[1] = byte(extensionSupportedPoints) + l := 1 + len(m.supportedPoints) + z[2] = byte(l >> 8) + z[3] = byte(l) + l-- + z[4] = byte(l) + z = z[5:] + for _, pointFormat := range m.supportedPoints { + z[0] = byte(pointFormat) + z = z[1:] + } + } + + m.raw = x + + return x +} + +func (m *clientHelloMsg) unmarshal(data []byte) bool { + if len(data) < 42 { + return false + } + m.raw = data + m.vers = uint16(data[4])<<8 | uint16(data[5]) + m.random = data[6:38] + sessionIdLen := int(data[38]) + if sessionIdLen > 32 || len(data) < 39+sessionIdLen { + return false + } + m.sessionId = data[39 : 39+sessionIdLen] + data = data[39+sessionIdLen:] + if len(data) < 2 { + return false + } + // cipherSuiteLen is the number of bytes of cipher suite numbers. Since + // they are uint16s, the number must be even. + cipherSuiteLen := int(data[0])<<8 | int(data[1]) + if cipherSuiteLen%2 == 1 || len(data) < 2+cipherSuiteLen { + return false + } + numCipherSuites := cipherSuiteLen / 2 + m.cipherSuites = make([]uint16, numCipherSuites) + for i := 0; i < numCipherSuites; i++ { + m.cipherSuites[i] = uint16(data[2+2*i])<<8 | uint16(data[3+2*i]) + } + data = data[2+cipherSuiteLen:] + if len(data) < 1 { + return false + } + compressionMethodsLen := int(data[0]) + if len(data) < 1+compressionMethodsLen { + return false + } + m.compressionMethods = data[1 : 1+compressionMethodsLen] + + data = data[1+compressionMethodsLen:] + + m.nextProtoNeg = false + m.serverName = "" + m.ocspStapling = false + + if len(data) == 0 { + // ClientHello is optionally followed by extension data + return true + } + if len(data) < 2 { + return false + } + + extensionsLength := int(data[0])<<8 | int(data[1]) + data = data[2:] + if extensionsLength != len(data) { + return false + } + + for len(data) != 0 { + if len(data) < 4 { + return false + } + extension := uint16(data[0])<<8 | uint16(data[1]) + length := int(data[2])<<8 | int(data[3]) + data = data[4:] + if len(data) < length { + return false + } + + switch extension { + case extensionServerName: + if length < 2 { + return false + } + numNames := int(data[0])<<8 | int(data[1]) + d := data[2:] + for i := 0; i < numNames; i++ { + if len(d) < 3 { + return false + } + nameType := d[0] + nameLen := int(d[1])<<8 | int(d[2]) + d = d[3:] + if len(d) < nameLen { + return false + } + if nameType == 0 { + m.serverName = string(d[0:nameLen]) + break + } + d = d[nameLen:] + } + case extensionNextProtoNeg: + if length > 0 { + return false + } + m.nextProtoNeg = true + case extensionStatusRequest: + m.ocspStapling = length > 0 && data[0] == statusTypeOCSP + case extensionSupportedCurves: + // http://tools.ietf.org/html/rfc4492#section-5.5.1 + if length < 2 { + return false + } + l := int(data[0])<<8 | int(data[1]) + if l%2 == 1 || length != l+2 { + return false + } + numCurves := l / 2 + m.supportedCurves = make([]uint16, numCurves) + d := data[2:] + for i := 0; i < numCurves; i++ { + m.supportedCurves[i] = uint16(d[0])<<8 | uint16(d[1]) + d = d[2:] + } + case extensionSupportedPoints: + // http://tools.ietf.org/html/rfc4492#section-5.5.2 + if length < 1 { + return false + } + l := int(data[0]) + if length != l+1 { + return false + } + m.supportedPoints = make([]uint8, l) + copy(m.supportedPoints, data[1:]) + } + data = data[length:] + } + + return true +} + +type serverHelloMsg struct { + raw []byte + vers uint16 + random []byte + sessionId []byte + cipherSuite uint16 + compressionMethod uint8 + nextProtoNeg bool + nextProtos []string + ocspStapling bool +} + +func (m *serverHelloMsg) marshal() []byte { + if m.raw != nil { + return m.raw + } + + length := 38 + len(m.sessionId) + numExtensions := 0 + extensionsLength := 0 + + nextProtoLen := 0 + if m.nextProtoNeg { + numExtensions++ + for _, v := range m.nextProtos { + nextProtoLen += len(v) + } + nextProtoLen += len(m.nextProtos) + extensionsLength += nextProtoLen + } + if m.ocspStapling { + numExtensions++ + } + if numExtensions > 0 { + extensionsLength += 4 * numExtensions + length += 2 + extensionsLength + } + + x := make([]byte, 4+length) + x[0] = typeServerHello + x[1] = uint8(length >> 16) + x[2] = uint8(length >> 8) + x[3] = uint8(length) + x[4] = uint8(m.vers >> 8) + x[5] = uint8(m.vers) + copy(x[6:38], m.random) + x[38] = uint8(len(m.sessionId)) + copy(x[39:39+len(m.sessionId)], m.sessionId) + z := x[39+len(m.sessionId):] + z[0] = uint8(m.cipherSuite >> 8) + z[1] = uint8(m.cipherSuite) + z[2] = uint8(m.compressionMethod) + + z = z[3:] + if numExtensions > 0 { + z[0] = byte(extensionsLength >> 8) + z[1] = byte(extensionsLength) + z = z[2:] + } + if m.nextProtoNeg { + z[0] = byte(extensionNextProtoNeg >> 8) + z[1] = byte(extensionNextProtoNeg) + z[2] = byte(nextProtoLen >> 8) + z[3] = byte(nextProtoLen) + z = z[4:] + + for _, v := range m.nextProtos { + l := len(v) + if l > 255 { + l = 255 + } + z[0] = byte(l) + copy(z[1:], []byte(v[0:l])) + z = z[1+l:] + } + } + if m.ocspStapling { + z[0] = byte(extensionStatusRequest >> 8) + z[1] = byte(extensionStatusRequest) + z = z[4:] + } + + m.raw = x + + return x +} + +func (m *serverHelloMsg) unmarshal(data []byte) bool { + if len(data) < 42 { + return false + } + m.raw = data + m.vers = uint16(data[4])<<8 | uint16(data[5]) + m.random = data[6:38] + sessionIdLen := int(data[38]) + if sessionIdLen > 32 || len(data) < 39+sessionIdLen { + return false + } + m.sessionId = data[39 : 39+sessionIdLen] + data = data[39+sessionIdLen:] + if len(data) < 3 { + return false + } + m.cipherSuite = uint16(data[0])<<8 | uint16(data[1]) + m.compressionMethod = data[2] + data = data[3:] + + m.nextProtoNeg = false + m.nextProtos = nil + m.ocspStapling = false + + if len(data) == 0 { + // ServerHello is optionally followed by extension data + return true + } + if len(data) < 2 { + return false + } + + extensionsLength := int(data[0])<<8 | int(data[1]) + data = data[2:] + if len(data) != extensionsLength { + return false + } + + for len(data) != 0 { + if len(data) < 4 { + return false + } + extension := uint16(data[0])<<8 | uint16(data[1]) + length := int(data[2])<<8 | int(data[3]) + data = data[4:] + if len(data) < length { + return false + } + + switch extension { + case extensionNextProtoNeg: + m.nextProtoNeg = true + d := data + for len(d) > 0 { + l := int(d[0]) + d = d[1:] + if l == 0 || l > len(d) { + return false + } + m.nextProtos = append(m.nextProtos, string(d[0:l])) + d = d[l:] + } + case extensionStatusRequest: + if length > 0 { + return false + } + m.ocspStapling = true + } + data = data[length:] + } + + return true +} + +type certificateMsg struct { + raw []byte + certificates [][]byte +} + +func (m *certificateMsg) marshal() (x []byte) { + if m.raw != nil { + return m.raw + } + + var i int + for _, slice := range m.certificates { + i += len(slice) + } + + length := 3 + 3*len(m.certificates) + i + x = make([]byte, 4+length) + x[0] = typeCertificate + x[1] = uint8(length >> 16) + x[2] = uint8(length >> 8) + x[3] = uint8(length) + + certificateOctets := length - 3 + x[4] = uint8(certificateOctets >> 16) + x[5] = uint8(certificateOctets >> 8) + x[6] = uint8(certificateOctets) + + y := x[7:] + for _, slice := range m.certificates { + y[0] = uint8(len(slice) >> 16) + y[1] = uint8(len(slice) >> 8) + y[2] = uint8(len(slice)) + copy(y[3:], slice) + y = y[3+len(slice):] + } + + m.raw = x + return +} + +func (m *certificateMsg) unmarshal(data []byte) bool { + if len(data) < 7 { + return false + } + + m.raw = data + certsLen := uint32(data[4])<<16 | uint32(data[5])<<8 | uint32(data[6]) + if uint32(len(data)) != certsLen+7 { + return false + } + + numCerts := 0 + d := data[7:] + for certsLen > 0 { + if len(d) < 4 { + return false + } + certLen := uint32(d[0])<<24 | uint32(d[1])<<8 | uint32(d[2]) + if uint32(len(d)) < 3+certLen { + return false + } + d = d[3+certLen:] + certsLen -= 3 + certLen + numCerts++ + } + + m.certificates = make([][]byte, numCerts) + d = data[7:] + for i := 0; i < numCerts; i++ { + certLen := uint32(d[0])<<24 | uint32(d[1])<<8 | uint32(d[2]) + m.certificates[i] = d[3 : 3+certLen] + d = d[3+certLen:] + } + + return true +} + +type serverKeyExchangeMsg struct { + raw []byte + key []byte +} + +func (m *serverKeyExchangeMsg) marshal() []byte { + if m.raw != nil { + return m.raw + } + length := len(m.key) + x := make([]byte, length+4) + x[0] = typeServerKeyExchange + x[1] = uint8(length >> 16) + x[2] = uint8(length >> 8) + x[3] = uint8(length) + copy(x[4:], m.key) + + m.raw = x + return x +} + +func (m *serverKeyExchangeMsg) unmarshal(data []byte) bool { + m.raw = data + if len(data) < 4 { + return false + } + m.key = data[4:] + return true +} + +type certificateStatusMsg struct { + raw []byte + statusType uint8 + response []byte +} + +func (m *certificateStatusMsg) marshal() []byte { + if m.raw != nil { + return m.raw + } + + var x []byte + if m.statusType == statusTypeOCSP { + x = make([]byte, 4+4+len(m.response)) + x[0] = typeCertificateStatus + l := len(m.response) + 4 + x[1] = byte(l >> 16) + x[2] = byte(l >> 8) + x[3] = byte(l) + x[4] = statusTypeOCSP + + l -= 4 + x[5] = byte(l >> 16) + x[6] = byte(l >> 8) + x[7] = byte(l) + copy(x[8:], m.response) + } else { + x = []byte{typeCertificateStatus, 0, 0, 1, m.statusType} + } + + m.raw = x + return x +} + +func (m *certificateStatusMsg) unmarshal(data []byte) bool { + m.raw = data + if len(data) < 5 { + return false + } + m.statusType = data[4] + + m.response = nil + if m.statusType == statusTypeOCSP { + if len(data) < 8 { + return false + } + respLen := uint32(data[5])<<16 | uint32(data[6])<<8 | uint32(data[7]) + if uint32(len(data)) != 4+4+respLen { + return false + } + m.response = data[8:] + } + return true +} + +type serverHelloDoneMsg struct{} + +func (m *serverHelloDoneMsg) marshal() []byte { + x := make([]byte, 4) + x[0] = typeServerHelloDone + return x +} + +func (m *serverHelloDoneMsg) unmarshal(data []byte) bool { + return len(data) == 4 +} + +type clientKeyExchangeMsg struct { + raw []byte + ciphertext []byte +} + +func (m *clientKeyExchangeMsg) marshal() []byte { + if m.raw != nil { + return m.raw + } + length := len(m.ciphertext) + x := make([]byte, length+4) + x[0] = typeClientKeyExchange + x[1] = uint8(length >> 16) + x[2] = uint8(length >> 8) + x[3] = uint8(length) + copy(x[4:], m.ciphertext) + + m.raw = x + return x +} + +func (m *clientKeyExchangeMsg) unmarshal(data []byte) bool { + m.raw = data + if len(data) < 4 { + return false + } + l := int(data[1])<<16 | int(data[2])<<8 | int(data[3]) + if l != len(data)-4 { + return false + } + m.ciphertext = data[4:] + return true +} + +type finishedMsg struct { + raw []byte + verifyData []byte +} + +func (m *finishedMsg) marshal() (x []byte) { + if m.raw != nil { + return m.raw + } + + x = make([]byte, 16) + x[0] = typeFinished + x[3] = 12 + copy(x[4:], m.verifyData) + m.raw = x + return +} + +func (m *finishedMsg) unmarshal(data []byte) bool { + m.raw = data + if len(data) != 4+12 { + return false + } + m.verifyData = data[4:] + return true +} + +type nextProtoMsg struct { + raw []byte + proto string +} + +func (m *nextProtoMsg) marshal() []byte { + if m.raw != nil { + return m.raw + } + l := len(m.proto) + if l > 255 { + l = 255 + } + + padding := 32 - (l+2)%32 + length := l + padding + 2 + x := make([]byte, length+4) + x[0] = typeNextProtocol + x[1] = uint8(length >> 16) + x[2] = uint8(length >> 8) + x[3] = uint8(length) + + y := x[4:] + y[0] = byte(l) + copy(y[1:], []byte(m.proto[0:l])) + y = y[1+l:] + y[0] = byte(padding) + + m.raw = x + + return x +} + +func (m *nextProtoMsg) unmarshal(data []byte) bool { + m.raw = data + + if len(data) < 5 { + return false + } + data = data[4:] + protoLen := int(data[0]) + data = data[1:] + if len(data) < protoLen { + return false + } + m.proto = string(data[0:protoLen]) + data = data[protoLen:] + + if len(data) < 1 { + return false + } + paddingLen := int(data[0]) + data = data[1:] + if len(data) != paddingLen { + return false + } + + return true +} + +type certificateRequestMsg struct { + raw []byte + certificateTypes []byte + certificateAuthorities [][]byte +} + +func (m *certificateRequestMsg) marshal() (x []byte) { + if m.raw != nil { + return m.raw + } + + // See http://tools.ietf.org/html/rfc4346#section-7.4.4 + length := 1 + len(m.certificateTypes) + 2 + for _, ca := range m.certificateAuthorities { + length += 2 + len(ca) + } + + x = make([]byte, 4+length) + x[0] = typeCertificateRequest + x[1] = uint8(length >> 16) + x[2] = uint8(length >> 8) + x[3] = uint8(length) + + x[4] = uint8(len(m.certificateTypes)) + + copy(x[5:], m.certificateTypes) + y := x[5+len(m.certificateTypes):] + + numCA := len(m.certificateAuthorities) + y[0] = uint8(numCA >> 8) + y[1] = uint8(numCA) + y = y[2:] + for _, ca := range m.certificateAuthorities { + y[0] = uint8(len(ca) >> 8) + y[1] = uint8(len(ca)) + y = y[2:] + copy(y, ca) + y = y[len(ca):] + } + + m.raw = x + + return +} + +func (m *certificateRequestMsg) unmarshal(data []byte) bool { + m.raw = data + + if len(data) < 5 { + return false + } + + length := uint32(data[1])<<16 | uint32(data[2])<<8 | uint32(data[3]) + if uint32(len(data))-4 != length { + return false + } + + numCertTypes := int(data[4]) + data = data[5:] + if numCertTypes == 0 || len(data) <= numCertTypes { + return false + } + + m.certificateTypes = make([]byte, numCertTypes) + if copy(m.certificateTypes, data) != numCertTypes { + return false + } + + data = data[numCertTypes:] + if len(data) < 2 { + return false + } + + numCAs := uint16(data[0])<<16 | uint16(data[1]) + data = data[2:] + + m.certificateAuthorities = make([][]byte, numCAs) + for i := uint16(0); i < numCAs; i++ { + if len(data) < 2 { + return false + } + caLen := uint16(data[0])<<16 | uint16(data[1]) + + data = data[2:] + if len(data) < int(caLen) { + return false + } + + ca := make([]byte, caLen) + copy(ca, data) + m.certificateAuthorities[i] = ca + data = data[caLen:] + } + + if len(data) > 0 { + return false + } + + return true +} + +type certificateVerifyMsg struct { + raw []byte + signature []byte +} + +func (m *certificateVerifyMsg) marshal() (x []byte) { + if m.raw != nil { + return m.raw + } + + // See http://tools.ietf.org/html/rfc4346#section-7.4.8 + siglength := len(m.signature) + length := 2 + siglength + x = make([]byte, 4+length) + x[0] = typeCertificateVerify + x[1] = uint8(length >> 16) + x[2] = uint8(length >> 8) + x[3] = uint8(length) + x[4] = uint8(siglength >> 8) + x[5] = uint8(siglength) + copy(x[6:], m.signature) + + m.raw = x + + return +} + +func (m *certificateVerifyMsg) unmarshal(data []byte) bool { + m.raw = data + + if len(data) < 6 { + return false + } + + length := uint32(data[1])<<16 | uint32(data[2])<<8 | uint32(data[3]) + if uint32(len(data))-4 != length { + return false + } + + siglength := int(data[4])<<8 + int(data[5]) + if len(data)-6 != siglength { + return false + } + + m.signature = data[6:] + + return true +} diff --git a/src/pkg/crypto/tls/handshake_messages_test.go b/src/pkg/crypto/tls/handshake_messages_test.go new file mode 100644 index 000000000..23f729dd9 --- /dev/null +++ b/src/pkg/crypto/tls/handshake_messages_test.go @@ -0,0 +1,206 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package tls + +import ( + "rand" + "reflect" + "testing" + "testing/quick" +) + +var tests = []interface{}{ + &clientHelloMsg{}, + &serverHelloMsg{}, + + &certificateMsg{}, + &certificateRequestMsg{}, + &certificateVerifyMsg{}, + &certificateStatusMsg{}, + &clientKeyExchangeMsg{}, + &finishedMsg{}, + &nextProtoMsg{}, +} + +type testMessage interface { + marshal() []byte + unmarshal([]byte) bool +} + +func TestMarshalUnmarshal(t *testing.T) { + rand := rand.New(rand.NewSource(0)) + for i, iface := range tests { + ty := reflect.ValueOf(iface).Type() + + n := 100 + if testing.Short() { + n = 5 + } + for j := 0; j < n; j++ { + v, ok := quick.Value(ty, rand) + if !ok { + t.Errorf("#%d: failed to create value", i) + break + } + + m1 := v.Interface().(testMessage) + marshaled := m1.marshal() + m2 := iface.(testMessage) + if !m2.unmarshal(marshaled) { + t.Errorf("#%d failed to unmarshal %#v %x", i, m1, marshaled) + break + } + m2.marshal() // to fill any marshal cache in the message + + if !reflect.DeepEqual(m1, m2) { + t.Errorf("#%d got:%#v want:%#v %x", i, m2, m1, marshaled) + break + } + + if i >= 2 { + // The first two message types (ClientHello and + // ServerHello) are allowed to have parsable + // prefixes because the extension data is + // optional. + for j := 0; j < len(marshaled); j++ { + if m2.unmarshal(marshaled[0:j]) { + t.Errorf("#%d unmarshaled a prefix of length %d of %#v", i, j, m1) + break + } + } + } + } + } +} + +func TestFuzz(t *testing.T) { + rand := rand.New(rand.NewSource(0)) + for _, iface := range tests { + m := iface.(testMessage) + + for j := 0; j < 1000; j++ { + len := rand.Intn(100) + bytes := randomBytes(len, rand) + // This just looks for crashes due to bounds errors etc. + m.unmarshal(bytes) + } + } +} + +func randomBytes(n int, rand *rand.Rand) []byte { + r := make([]byte, n) + for i := 0; i < n; i++ { + r[i] = byte(rand.Int31()) + } + return r +} + +func randomString(n int, rand *rand.Rand) string { + b := randomBytes(n, rand) + return string(b) +} + +func (*clientHelloMsg) Generate(rand *rand.Rand, size int) reflect.Value { + m := &clientHelloMsg{} + m.vers = uint16(rand.Intn(65536)) + m.random = randomBytes(32, rand) + m.sessionId = randomBytes(rand.Intn(32), rand) + m.cipherSuites = make([]uint16, rand.Intn(63)+1) + for i := 0; i < len(m.cipherSuites); i++ { + m.cipherSuites[i] = uint16(rand.Int31()) + } + m.compressionMethods = randomBytes(rand.Intn(63)+1, rand) + if rand.Intn(10) > 5 { + m.nextProtoNeg = true + } + if rand.Intn(10) > 5 { + m.serverName = randomString(rand.Intn(255), rand) + } + m.ocspStapling = rand.Intn(10) > 5 + m.supportedPoints = randomBytes(rand.Intn(5)+1, rand) + m.supportedCurves = make([]uint16, rand.Intn(5)+1) + for i := range m.supportedCurves { + m.supportedCurves[i] = uint16(rand.Intn(30000)) + } + + return reflect.ValueOf(m) +} + +func (*serverHelloMsg) Generate(rand *rand.Rand, size int) reflect.Value { + m := &serverHelloMsg{} + m.vers = uint16(rand.Intn(65536)) + m.random = randomBytes(32, rand) + m.sessionId = randomBytes(rand.Intn(32), rand) + m.cipherSuite = uint16(rand.Int31()) + m.compressionMethod = uint8(rand.Intn(256)) + + if rand.Intn(10) > 5 { + m.nextProtoNeg = true + + n := rand.Intn(10) + m.nextProtos = make([]string, n) + for i := 0; i < n; i++ { + m.nextProtos[i] = randomString(20, rand) + } + } + + return reflect.ValueOf(m) +} + +func (*certificateMsg) Generate(rand *rand.Rand, size int) reflect.Value { + m := &certificateMsg{} + numCerts := rand.Intn(20) + m.certificates = make([][]byte, numCerts) + for i := 0; i < numCerts; i++ { + m.certificates[i] = randomBytes(rand.Intn(10)+1, rand) + } + return reflect.ValueOf(m) +} + +func (*certificateRequestMsg) Generate(rand *rand.Rand, size int) reflect.Value { + m := &certificateRequestMsg{} + m.certificateTypes = randomBytes(rand.Intn(5)+1, rand) + numCAs := rand.Intn(100) + m.certificateAuthorities = make([][]byte, numCAs) + for i := 0; i < numCAs; i++ { + m.certificateAuthorities[i] = randomBytes(rand.Intn(15)+1, rand) + } + return reflect.ValueOf(m) +} + +func (*certificateVerifyMsg) Generate(rand *rand.Rand, size int) reflect.Value { + m := &certificateVerifyMsg{} + m.signature = randomBytes(rand.Intn(15)+1, rand) + return reflect.ValueOf(m) +} + +func (*certificateStatusMsg) Generate(rand *rand.Rand, size int) reflect.Value { + m := &certificateStatusMsg{} + if rand.Intn(10) > 5 { + m.statusType = statusTypeOCSP + m.response = randomBytes(rand.Intn(10)+1, rand) + } else { + m.statusType = 42 + } + return reflect.ValueOf(m) +} + +func (*clientKeyExchangeMsg) Generate(rand *rand.Rand, size int) reflect.Value { + m := &clientKeyExchangeMsg{} + m.ciphertext = randomBytes(rand.Intn(1000)+1, rand) + return reflect.ValueOf(m) +} + +func (*finishedMsg) Generate(rand *rand.Rand, size int) reflect.Value { + m := &finishedMsg{} + m.verifyData = randomBytes(12, rand) + return reflect.ValueOf(m) +} + +func (*nextProtoMsg) Generate(rand *rand.Rand, size int) reflect.Value { + m := &nextProtoMsg{} + m.proto = randomString(rand.Intn(255), rand) + return reflect.ValueOf(m) +} diff --git a/src/pkg/crypto/tls/handshake_server.go b/src/pkg/crypto/tls/handshake_server.go new file mode 100644 index 000000000..44a324041 --- /dev/null +++ b/src/pkg/crypto/tls/handshake_server.go @@ -0,0 +1,298 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package tls + +import ( + "crypto" + "crypto/rsa" + "crypto/subtle" + "crypto/x509" + "io" + "os" +) + +func (c *Conn) serverHandshake() os.Error { + config := c.config + msg, err := c.readHandshake() + if err != nil { + return err + } + clientHello, ok := msg.(*clientHelloMsg) + if !ok { + return c.sendAlert(alertUnexpectedMessage) + } + vers, ok := mutualVersion(clientHello.vers) + if !ok { + return c.sendAlert(alertProtocolVersion) + } + c.vers = vers + c.haveVers = true + + finishedHash := newFinishedHash() + finishedHash.Write(clientHello.marshal()) + + hello := new(serverHelloMsg) + + supportedCurve := false +Curves: + for _, curve := range clientHello.supportedCurves { + switch curve { + case curveP256, curveP384, curveP521: + supportedCurve = true + break Curves + } + } + + supportedPointFormat := false + for _, pointFormat := range clientHello.supportedPoints { + if pointFormat == pointFormatUncompressed { + supportedPointFormat = true + break + } + } + + ellipticOk := supportedCurve && supportedPointFormat + + var suite *cipherSuite + var suiteId uint16 +FindCipherSuite: + for _, id := range clientHello.cipherSuites { + for _, supported := range config.cipherSuites() { + if id == supported { + suite = cipherSuites[id] + // Don't select a ciphersuite which we can't + // support for this client. + if suite.elliptic && !ellipticOk { + continue + } + suiteId = id + break FindCipherSuite + } + } + } + + foundCompression := false + // We only support null compression, so check that the client offered it. + for _, compression := range clientHello.compressionMethods { + if compression == compressionNone { + foundCompression = true + break + } + } + + if suite == nil || !foundCompression { + return c.sendAlert(alertHandshakeFailure) + } + + hello.vers = vers + hello.cipherSuite = suiteId + t := uint32(config.time()) + hello.random = make([]byte, 32) + hello.random[0] = byte(t >> 24) + hello.random[1] = byte(t >> 16) + hello.random[2] = byte(t >> 8) + hello.random[3] = byte(t) + _, err = io.ReadFull(config.rand(), hello.random[4:]) + if err != nil { + return c.sendAlert(alertInternalError) + } + hello.compressionMethod = compressionNone + if clientHello.nextProtoNeg { + hello.nextProtoNeg = true + hello.nextProtos = config.NextProtos + } + if clientHello.ocspStapling && len(config.Certificates[0].OCSPStaple) > 0 { + hello.ocspStapling = true + } + + finishedHash.Write(hello.marshal()) + c.writeRecord(recordTypeHandshake, hello.marshal()) + + if len(config.Certificates) == 0 { + return c.sendAlert(alertInternalError) + } + + certMsg := new(certificateMsg) + certMsg.certificates = config.Certificates[0].Certificate + finishedHash.Write(certMsg.marshal()) + c.writeRecord(recordTypeHandshake, certMsg.marshal()) + + if hello.ocspStapling { + certStatus := new(certificateStatusMsg) + certStatus.statusType = statusTypeOCSP + certStatus.response = config.Certificates[0].OCSPStaple + finishedHash.Write(certStatus.marshal()) + c.writeRecord(recordTypeHandshake, certStatus.marshal()) + } + + keyAgreement := suite.ka() + + skx, err := keyAgreement.generateServerKeyExchange(config, clientHello, hello) + if err != nil { + c.sendAlert(alertHandshakeFailure) + return err + } + if skx != nil { + finishedHash.Write(skx.marshal()) + c.writeRecord(recordTypeHandshake, skx.marshal()) + } + + if config.AuthenticateClient { + // Request a client certificate + certReq := new(certificateRequestMsg) + certReq.certificateTypes = []byte{certTypeRSASign} + // An empty list of certificateAuthorities signals to + // the client that it may send any certificate in response + // to our request. + + finishedHash.Write(certReq.marshal()) + c.writeRecord(recordTypeHandshake, certReq.marshal()) + } + + helloDone := new(serverHelloDoneMsg) + finishedHash.Write(helloDone.marshal()) + c.writeRecord(recordTypeHandshake, helloDone.marshal()) + + var pub *rsa.PublicKey + if config.AuthenticateClient { + // Get client certificate + msg, err = c.readHandshake() + if err != nil { + return err + } + certMsg, ok = msg.(*certificateMsg) + if !ok { + return c.sendAlert(alertUnexpectedMessage) + } + finishedHash.Write(certMsg.marshal()) + + certs := make([]*x509.Certificate, len(certMsg.certificates)) + for i, asn1Data := range certMsg.certificates { + cert, err := x509.ParseCertificate(asn1Data) + if err != nil { + c.sendAlert(alertBadCertificate) + return os.NewError("could not parse client's certificate: " + err.String()) + } + certs[i] = cert + } + + // TODO(agl): do better validation of certs: max path length, name restrictions etc. + for i := 1; i < len(certs); i++ { + if err := certs[i-1].CheckSignatureFrom(certs[i]); err != nil { + c.sendAlert(alertBadCertificate) + return os.NewError("could not validate certificate signature: " + err.String()) + } + } + + if len(certs) > 0 { + key, ok := certs[0].PublicKey.(*rsa.PublicKey) + if !ok { + return c.sendAlert(alertUnsupportedCertificate) + } + pub = key + c.peerCertificates = certs + } + } + + // Get client key exchange + msg, err = c.readHandshake() + if err != nil { + return err + } + ckx, ok := msg.(*clientKeyExchangeMsg) + if !ok { + return c.sendAlert(alertUnexpectedMessage) + } + finishedHash.Write(ckx.marshal()) + + // If we received a client cert in response to our certificate request message, + // the client will send us a certificateVerifyMsg immediately after the + // clientKeyExchangeMsg. This message is a MD5SHA1 digest of all preceding + // handshake-layer messages that is signed using the private key corresponding + // to the client's certificate. This allows us to verify that the client is in + // possession of the private key of the certificate. + if len(c.peerCertificates) > 0 { + msg, err = c.readHandshake() + if err != nil { + return err + } + certVerify, ok := msg.(*certificateVerifyMsg) + if !ok { + return c.sendAlert(alertUnexpectedMessage) + } + + digest := make([]byte, 36) + copy(digest[0:16], finishedHash.serverMD5.Sum()) + copy(digest[16:36], finishedHash.serverSHA1.Sum()) + err = rsa.VerifyPKCS1v15(pub, crypto.MD5SHA1, digest, certVerify.signature) + if err != nil { + c.sendAlert(alertBadCertificate) + return os.NewError("could not validate signature of connection nonces: " + err.String()) + } + + finishedHash.Write(certVerify.marshal()) + } + + preMasterSecret, err := keyAgreement.processClientKeyExchange(config, ckx) + if err != nil { + c.sendAlert(alertHandshakeFailure) + return err + } + + masterSecret, clientMAC, serverMAC, clientKey, serverKey, clientIV, serverIV := + keysFromPreMasterSecret10(preMasterSecret, clientHello.random, hello.random, suite.macLen, suite.keyLen, suite.ivLen) + + clientCipher := suite.cipher(clientKey, clientIV, true /* for reading */ ) + clientHash := suite.mac(clientMAC) + c.in.prepareCipherSpec(clientCipher, clientHash) + c.readRecord(recordTypeChangeCipherSpec) + if err := c.error(); err != nil { + return err + } + + if hello.nextProtoNeg { + msg, err = c.readHandshake() + if err != nil { + return err + } + nextProto, ok := msg.(*nextProtoMsg) + if !ok { + return c.sendAlert(alertUnexpectedMessage) + } + finishedHash.Write(nextProto.marshal()) + c.clientProtocol = nextProto.proto + } + + msg, err = c.readHandshake() + if err != nil { + return err + } + clientFinished, ok := msg.(*finishedMsg) + if !ok { + return c.sendAlert(alertUnexpectedMessage) + } + + verify := finishedHash.clientSum(masterSecret) + if len(verify) != len(clientFinished.verifyData) || + subtle.ConstantTimeCompare(verify, clientFinished.verifyData) != 1 { + return c.sendAlert(alertHandshakeFailure) + } + + finishedHash.Write(clientFinished.marshal()) + + serverCipher := suite.cipher(serverKey, serverIV, false /* not for reading */ ) + serverHash := suite.mac(serverMAC) + c.out.prepareCipherSpec(serverCipher, serverHash) + c.writeRecord(recordTypeChangeCipherSpec, []byte{1}) + + finished := new(finishedMsg) + finished.verifyData = finishedHash.serverSum(masterSecret) + c.writeRecord(recordTypeHandshake, finished.marshal()) + + c.handshakeComplete = true + c.cipherSuite = suiteId + + return nil +} diff --git a/src/pkg/crypto/tls/handshake_server_test.go b/src/pkg/crypto/tls/handshake_server_test.go new file mode 100644 index 000000000..b77646e43 --- /dev/null +++ b/src/pkg/crypto/tls/handshake_server_test.go @@ -0,0 +1,517 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package tls + +import ( + "big" + "bytes" + "crypto/rsa" + "encoding/hex" + "flag" + "io" + "net" + "os" + "testing" +) + +type zeroSource struct{} + +func (zeroSource) Read(b []byte) (n int, err os.Error) { + for i := range b { + b[i] = 0 + } + + return len(b), nil +} + +var testConfig *Config + +func init() { + testConfig = new(Config) + testConfig.Time = func() int64 { return 0 } + testConfig.Rand = zeroSource{} + testConfig.Certificates = make([]Certificate, 1) + testConfig.Certificates[0].Certificate = [][]byte{testCertificate} + testConfig.Certificates[0].PrivateKey = testPrivateKey + testConfig.CipherSuites = []uint16{TLS_RSA_WITH_RC4_128_SHA} +} + +func testClientHelloFailure(t *testing.T, m handshakeMessage, expected os.Error) { + // Create in-memory network connection, + // send message to server. Should return + // expected error. + c, s := net.Pipe() + go func() { + cli := Client(c, testConfig) + if ch, ok := m.(*clientHelloMsg); ok { + cli.vers = ch.vers + } + cli.writeRecord(recordTypeHandshake, m.marshal()) + c.Close() + }() + err := Server(s, testConfig).Handshake() + s.Close() + if e, ok := err.(*net.OpError); !ok || e.Error != expected { + t.Errorf("Got error: %s; expected: %s", err, expected) + } +} + +func TestSimpleError(t *testing.T) { + testClientHelloFailure(t, &serverHelloDoneMsg{}, alertUnexpectedMessage) +} + +var badProtocolVersions = []uint16{0x0000, 0x0005, 0x0100, 0x0105, 0x0200, 0x0205, 0x0300} + +func TestRejectBadProtocolVersion(t *testing.T) { + for _, v := range badProtocolVersions { + testClientHelloFailure(t, &clientHelloMsg{vers: v}, alertProtocolVersion) + } +} + +func TestNoSuiteOverlap(t *testing.T) { + clientHello := &clientHelloMsg{nil, 0x0301, nil, nil, []uint16{0xff00}, []uint8{0}, false, "", false, nil, nil} + testClientHelloFailure(t, clientHello, alertHandshakeFailure) + +} + +func TestNoCompressionOverlap(t *testing.T) { + clientHello := &clientHelloMsg{nil, 0x0301, nil, nil, []uint16{TLS_RSA_WITH_RC4_128_SHA}, []uint8{0xff}, false, "", false, nil, nil} + testClientHelloFailure(t, clientHello, alertHandshakeFailure) +} + +func TestAlertForwarding(t *testing.T) { + c, s := net.Pipe() + go func() { + Client(c, testConfig).sendAlert(alertUnknownCA) + c.Close() + }() + + err := Server(s, testConfig).Handshake() + s.Close() + if e, ok := err.(*net.OpError); !ok || e.Error != os.Error(alertUnknownCA) { + t.Errorf("Got error: %s; expected: %s", err, alertUnknownCA) + } +} + +func TestClose(t *testing.T) { + c, s := net.Pipe() + go c.Close() + + err := Server(s, testConfig).Handshake() + s.Close() + if err != os.EOF { + t.Errorf("Got error: %s; expected: %s", err, os.EOF) + } +} + +func testServerScript(t *testing.T, name string, serverScript [][]byte, config *Config) { + c, s := net.Pipe() + srv := Server(s, config) + go func() { + srv.Write([]byte("hello, world\n")) + srv.Close() + }() + + defer c.Close() + for i, b := range serverScript { + if i%2 == 0 { + c.Write(b) + continue + } + bb := make([]byte, len(b)) + _, err := io.ReadFull(c, bb) + if err != nil { + t.Fatalf("%s #%d: %s", name, i, err) + } + if !bytes.Equal(b, bb) { + t.Fatalf("%s #%d: mismatch on read: got:%x want:%x", name, i, bb, b) + } + } +} + +func TestHandshakeServerRC4(t *testing.T) { + testServerScript(t, "RC4", rc4ServerScript, testConfig) +} + +func TestHandshakeServerAES(t *testing.T) { + aesConfig := new(Config) + *aesConfig = *testConfig + aesConfig.CipherSuites = []uint16{TLS_RSA_WITH_AES_128_CBC_SHA} + testServerScript(t, "AES", aesServerScript, aesConfig) +} + +var serve = flag.Bool("serve", false, "run a TLS server on :10443") + +func TestRunServer(t *testing.T) { + if !*serve { + return + } + + l, err := Listen("tcp", ":10443", testConfig) + if err != nil { + t.Fatal(err) + } + + for { + c, err := l.Accept() + if err != nil { + break + } + _, err = c.Write([]byte("hello, world\n")) + if err != nil { + t.Errorf("error from TLS: %s", err) + break + } + c.Close() + } +} + +func bigFromString(s string) *big.Int { + ret := new(big.Int) + ret.SetString(s, 10) + return ret +} + +func fromHex(s string) []byte { + b, _ := hex.DecodeString(s) + return b +} + +var testCertificate = fromHex("308202b030820219a00302010202090085b0bba48a7fb8ca300d06092a864886f70d01010505003045310b3009060355040613024155311330110603550408130a536f6d652d53746174653121301f060355040a1318496e7465726e6574205769646769747320507479204c7464301e170d3130303432343039303933385a170d3131303432343039303933385a3045310b3009060355040613024155311330110603550408130a536f6d652d53746174653121301f060355040a1318496e7465726e6574205769646769747320507479204c746430819f300d06092a864886f70d010101050003818d0030818902818100bb79d6f517b5e5bf4610d0dc69bee62b07435ad0032d8a7a4385b71452e7a5654c2c78b8238cb5b482e5de1f953b7e62a52ca533d6fe125c7a56fcf506bffa587b263fb5cd04d3d0c921964ac7f4549f5abfef427100fe1899077f7e887d7df10439c4a22edb51c97ce3c04c3b326601cfafb11db8719a1ddbdb896baeda2d790203010001a381a73081a4301d0603551d0e04160414b1ade2855acfcb28db69ce2369ded3268e18883930750603551d23046e306c8014b1ade2855acfcb28db69ce2369ded3268e188839a149a4473045310b3009060355040613024155311330110603550408130a536f6d652d53746174653121301f060355040a1318496e7465726e6574205769646769747320507479204c746482090085b0bba48a7fb8ca300c0603551d13040530030101ff300d06092a864886f70d010105050003818100086c4524c76bb159ab0c52ccf2b014d7879d7a6475b55a9566e4c52b8eae12661feb4f38b36e60d392fdf74108b52513b1187a24fb301dbaed98b917ece7d73159db95d31d78ea50565cd5825a2d5a5f33c4b6d8c97590968c0f5298b5cd981f89205ff2a01ca31b9694dda9fd57e970e8266d71999b266e3850296c90a7bdd9") + +var testPrivateKey = &rsa.PrivateKey{ + PublicKey: rsa.PublicKey{ + N: bigFromString("131650079503776001033793877885499001334664249354723305978524647182322416328664556247316495448366990052837680518067798333412266673813370895702118944398081598789828837447552603077848001020611640547221687072142537202428102790818451901395596882588063427854225330436740647715202971973145151161964464812406232198521"), + E: 65537, + }, + D: bigFromString("29354450337804273969007277378287027274721892607543397931919078829901848876371746653677097639302788129485893852488285045793268732234230875671682624082413996177431586734171663258657462237320300610850244186316880055243099640544518318093544057213190320837094958164973959123058337475052510833916491060913053867729"), + Primes: []*big.Int{ + bigFromString("11969277782311800166562047708379380720136961987713178380670422671426759650127150688426177829077494755200794297055316163155755835813760102405344560929062149"), + bigFromString("10998999429884441391899182616418192492905073053684657075974935218461686523870125521822756579792315215543092255516093840728890783887287417039645833477273829"), + }, +} + +// Script of interaction with gnutls implementation. +// The values for this test are obtained by building and running in server mode: +// % gotest -test.run "TestRunServer" -serve +// and then: +// % gnutls-cli --insecure --debug 100 -p 10443 localhost > /tmp/log 2>&1 +// % python parse-gnutls-cli-debug-log.py < /tmp/log +var rc4ServerScript = [][]byte{ + { + 0x16, 0x03, 0x02, 0x00, 0x7f, 0x01, 0x00, 0x00, + 0x7b, 0x03, 0x02, 0x4d, 0x08, 0x1f, 0x5a, 0x7a, + 0x0a, 0x92, 0x2f, 0xf0, 0x73, 0x16, 0x3a, 0x88, + 0x14, 0x85, 0x4c, 0x98, 0x15, 0x7b, 0x65, 0xe0, + 0x78, 0xd0, 0xed, 0xd0, 0xf3, 0x65, 0x20, 0xeb, + 0x80, 0xd1, 0x0b, 0x00, 0x00, 0x34, 0x00, 0x33, + 0x00, 0x45, 0x00, 0x39, 0x00, 0x88, 0x00, 0x16, + 0x00, 0x32, 0x00, 0x44, 0x00, 0x38, 0x00, 0x87, + 0x00, 0x13, 0x00, 0x66, 0x00, 0x90, 0x00, 0x91, + 0x00, 0x8f, 0x00, 0x8e, 0x00, 0x2f, 0x00, 0x41, + 0x00, 0x35, 0x00, 0x84, 0x00, 0x0a, 0x00, 0x05, + 0x00, 0x04, 0x00, 0x8c, 0x00, 0x8d, 0x00, 0x8b, + 0x00, 0x8a, 0x01, 0x00, 0x00, 0x1e, 0x00, 0x09, + 0x00, 0x03, 0x02, 0x00, 0x01, 0x00, 0x00, 0x00, + 0x0e, 0x00, 0x0c, 0x00, 0x00, 0x09, 0x6c, 0x6f, + 0x63, 0x61, 0x6c, 0x68, 0x6f, 0x73, 0x74, 0xff, + 0x01, 0x00, 0x01, 0x00, + }, + + { + 0x16, 0x03, 0x01, 0x00, 0x2a, 0x02, 0x00, 0x00, + 0x26, 0x03, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x05, 0x00, 0x16, + 0x03, 0x01, 0x02, 0xbe, 0x0b, 0x00, 0x02, 0xba, + 0x00, 0x02, 0xb7, 0x00, 0x02, 0xb4, 0x30, 0x82, + 0x02, 0xb0, 0x30, 0x82, 0x02, 0x19, 0xa0, 0x03, + 0x02, 0x01, 0x02, 0x02, 0x09, 0x00, 0x85, 0xb0, + 0xbb, 0xa4, 0x8a, 0x7f, 0xb8, 0xca, 0x30, 0x0d, + 0x06, 0x09, 0x2a, 0x86, 0x48, 0x86, 0xf7, 0x0d, + 0x01, 0x01, 0x05, 0x05, 0x00, 0x30, 0x45, 0x31, + 0x0b, 0x30, 0x09, 0x06, 0x03, 0x55, 0x04, 0x06, + 0x13, 0x02, 0x41, 0x55, 0x31, 0x13, 0x30, 0x11, + 0x06, 0x03, 0x55, 0x04, 0x08, 0x13, 0x0a, 0x53, + 0x6f, 0x6d, 0x65, 0x2d, 0x53, 0x74, 0x61, 0x74, + 0x65, 0x31, 0x21, 0x30, 0x1f, 0x06, 0x03, 0x55, + 0x04, 0x0a, 0x13, 0x18, 0x49, 0x6e, 0x74, 0x65, + 0x72, 0x6e, 0x65, 0x74, 0x20, 0x57, 0x69, 0x64, + 0x67, 0x69, 0x74, 0x73, 0x20, 0x50, 0x74, 0x79, + 0x20, 0x4c, 0x74, 0x64, 0x30, 0x1e, 0x17, 0x0d, + 0x31, 0x30, 0x30, 0x34, 0x32, 0x34, 0x30, 0x39, + 0x30, 0x39, 0x33, 0x38, 0x5a, 0x17, 0x0d, 0x31, + 0x31, 0x30, 0x34, 0x32, 0x34, 0x30, 0x39, 0x30, + 0x39, 0x33, 0x38, 0x5a, 0x30, 0x45, 0x31, 0x0b, + 0x30, 0x09, 0x06, 0x03, 0x55, 0x04, 0x06, 0x13, + 0x02, 0x41, 0x55, 0x31, 0x13, 0x30, 0x11, 0x06, + 0x03, 0x55, 0x04, 0x08, 0x13, 0x0a, 0x53, 0x6f, + 0x6d, 0x65, 0x2d, 0x53, 0x74, 0x61, 0x74, 0x65, + 0x31, 0x21, 0x30, 0x1f, 0x06, 0x03, 0x55, 0x04, + 0x0a, 0x13, 0x18, 0x49, 0x6e, 0x74, 0x65, 0x72, + 0x6e, 0x65, 0x74, 0x20, 0x57, 0x69, 0x64, 0x67, + 0x69, 0x74, 0x73, 0x20, 0x50, 0x74, 0x79, 0x20, + 0x4c, 0x74, 0x64, 0x30, 0x81, 0x9f, 0x30, 0x0d, + 0x06, 0x09, 0x2a, 0x86, 0x48, 0x86, 0xf7, 0x0d, + 0x01, 0x01, 0x01, 0x05, 0x00, 0x03, 0x81, 0x8d, + 0x00, 0x30, 0x81, 0x89, 0x02, 0x81, 0x81, 0x00, + 0xbb, 0x79, 0xd6, 0xf5, 0x17, 0xb5, 0xe5, 0xbf, + 0x46, 0x10, 0xd0, 0xdc, 0x69, 0xbe, 0xe6, 0x2b, + 0x07, 0x43, 0x5a, 0xd0, 0x03, 0x2d, 0x8a, 0x7a, + 0x43, 0x85, 0xb7, 0x14, 0x52, 0xe7, 0xa5, 0x65, + 0x4c, 0x2c, 0x78, 0xb8, 0x23, 0x8c, 0xb5, 0xb4, + 0x82, 0xe5, 0xde, 0x1f, 0x95, 0x3b, 0x7e, 0x62, + 0xa5, 0x2c, 0xa5, 0x33, 0xd6, 0xfe, 0x12, 0x5c, + 0x7a, 0x56, 0xfc, 0xf5, 0x06, 0xbf, 0xfa, 0x58, + 0x7b, 0x26, 0x3f, 0xb5, 0xcd, 0x04, 0xd3, 0xd0, + 0xc9, 0x21, 0x96, 0x4a, 0xc7, 0xf4, 0x54, 0x9f, + 0x5a, 0xbf, 0xef, 0x42, 0x71, 0x00, 0xfe, 0x18, + 0x99, 0x07, 0x7f, 0x7e, 0x88, 0x7d, 0x7d, 0xf1, + 0x04, 0x39, 0xc4, 0xa2, 0x2e, 0xdb, 0x51, 0xc9, + 0x7c, 0xe3, 0xc0, 0x4c, 0x3b, 0x32, 0x66, 0x01, + 0xcf, 0xaf, 0xb1, 0x1d, 0xb8, 0x71, 0x9a, 0x1d, + 0xdb, 0xdb, 0x89, 0x6b, 0xae, 0xda, 0x2d, 0x79, + 0x02, 0x03, 0x01, 0x00, 0x01, 0xa3, 0x81, 0xa7, + 0x30, 0x81, 0xa4, 0x30, 0x1d, 0x06, 0x03, 0x55, + 0x1d, 0x0e, 0x04, 0x16, 0x04, 0x14, 0xb1, 0xad, + 0xe2, 0x85, 0x5a, 0xcf, 0xcb, 0x28, 0xdb, 0x69, + 0xce, 0x23, 0x69, 0xde, 0xd3, 0x26, 0x8e, 0x18, + 0x88, 0x39, 0x30, 0x75, 0x06, 0x03, 0x55, 0x1d, + 0x23, 0x04, 0x6e, 0x30, 0x6c, 0x80, 0x14, 0xb1, + 0xad, 0xe2, 0x85, 0x5a, 0xcf, 0xcb, 0x28, 0xdb, + 0x69, 0xce, 0x23, 0x69, 0xde, 0xd3, 0x26, 0x8e, + 0x18, 0x88, 0x39, 0xa1, 0x49, 0xa4, 0x47, 0x30, + 0x45, 0x31, 0x0b, 0x30, 0x09, 0x06, 0x03, 0x55, + 0x04, 0x06, 0x13, 0x02, 0x41, 0x55, 0x31, 0x13, + 0x30, 0x11, 0x06, 0x03, 0x55, 0x04, 0x08, 0x13, + 0x0a, 0x53, 0x6f, 0x6d, 0x65, 0x2d, 0x53, 0x74, + 0x61, 0x74, 0x65, 0x31, 0x21, 0x30, 0x1f, 0x06, + 0x03, 0x55, 0x04, 0x0a, 0x13, 0x18, 0x49, 0x6e, + 0x74, 0x65, 0x72, 0x6e, 0x65, 0x74, 0x20, 0x57, + 0x69, 0x64, 0x67, 0x69, 0x74, 0x73, 0x20, 0x50, + 0x74, 0x79, 0x20, 0x4c, 0x74, 0x64, 0x82, 0x09, + 0x00, 0x85, 0xb0, 0xbb, 0xa4, 0x8a, 0x7f, 0xb8, + 0xca, 0x30, 0x0c, 0x06, 0x03, 0x55, 0x1d, 0x13, + 0x04, 0x05, 0x30, 0x03, 0x01, 0x01, 0xff, 0x30, + 0x0d, 0x06, 0x09, 0x2a, 0x86, 0x48, 0x86, 0xf7, + 0x0d, 0x01, 0x01, 0x05, 0x05, 0x00, 0x03, 0x81, + 0x81, 0x00, 0x08, 0x6c, 0x45, 0x24, 0xc7, 0x6b, + 0xb1, 0x59, 0xab, 0x0c, 0x52, 0xcc, 0xf2, 0xb0, + 0x14, 0xd7, 0x87, 0x9d, 0x7a, 0x64, 0x75, 0xb5, + 0x5a, 0x95, 0x66, 0xe4, 0xc5, 0x2b, 0x8e, 0xae, + 0x12, 0x66, 0x1f, 0xeb, 0x4f, 0x38, 0xb3, 0x6e, + 0x60, 0xd3, 0x92, 0xfd, 0xf7, 0x41, 0x08, 0xb5, + 0x25, 0x13, 0xb1, 0x18, 0x7a, 0x24, 0xfb, 0x30, + 0x1d, 0xba, 0xed, 0x98, 0xb9, 0x17, 0xec, 0xe7, + 0xd7, 0x31, 0x59, 0xdb, 0x95, 0xd3, 0x1d, 0x78, + 0xea, 0x50, 0x56, 0x5c, 0xd5, 0x82, 0x5a, 0x2d, + 0x5a, 0x5f, 0x33, 0xc4, 0xb6, 0xd8, 0xc9, 0x75, + 0x90, 0x96, 0x8c, 0x0f, 0x52, 0x98, 0xb5, 0xcd, + 0x98, 0x1f, 0x89, 0x20, 0x5f, 0xf2, 0xa0, 0x1c, + 0xa3, 0x1b, 0x96, 0x94, 0xdd, 0xa9, 0xfd, 0x57, + 0xe9, 0x70, 0xe8, 0x26, 0x6d, 0x71, 0x99, 0x9b, + 0x26, 0x6e, 0x38, 0x50, 0x29, 0x6c, 0x90, 0xa7, + 0xbd, 0xd9, 0x16, 0x03, 0x01, 0x00, 0x04, 0x0e, + 0x00, 0x00, 0x00, + }, + + { + 0x16, 0x03, 0x01, 0x00, 0x86, 0x10, 0x00, 0x00, + 0x82, 0x00, 0x80, 0x3c, 0x13, 0xd7, 0x12, 0xc1, + 0x6a, 0xf0, 0x3f, 0x8c, 0xa1, 0x35, 0x5d, 0xc5, + 0x89, 0x1e, 0x9e, 0xcd, 0x32, 0xc7, 0x9e, 0xe6, + 0xae, 0xd5, 0xf1, 0xbf, 0x70, 0xd7, 0xa9, 0xef, + 0x2c, 0x4c, 0xf4, 0x22, 0xbc, 0x17, 0x17, 0xaa, + 0x05, 0xf3, 0x9f, 0x80, 0xf2, 0xe9, 0x82, 0x2f, + 0x2a, 0x15, 0x54, 0x0d, 0x16, 0x0e, 0x77, 0x4c, + 0x28, 0x3c, 0x03, 0x2d, 0x2d, 0xd7, 0xc8, 0x64, + 0xd9, 0x59, 0x4b, 0x1c, 0xf4, 0xde, 0xff, 0x2f, + 0xbc, 0x94, 0xaf, 0x18, 0x26, 0x37, 0xce, 0x4f, + 0x84, 0x74, 0x2e, 0x45, 0x66, 0x7c, 0x0c, 0x54, + 0x46, 0x36, 0x5f, 0x65, 0x21, 0x7b, 0x83, 0x8c, + 0x6d, 0x76, 0xcd, 0x0d, 0x9f, 0xda, 0x1c, 0xa4, + 0x6e, 0xfe, 0xb1, 0xf7, 0x09, 0x0d, 0xfb, 0x74, + 0x66, 0x34, 0x99, 0x89, 0x7f, 0x5f, 0x77, 0x87, + 0x4a, 0x66, 0x4b, 0xa9, 0x59, 0x57, 0xe3, 0x56, + 0x0d, 0xdd, 0xd8, 0x14, 0x03, 0x01, 0x00, 0x01, + 0x01, 0x16, 0x03, 0x01, 0x00, 0x24, 0xc0, 0x4e, + 0xd3, 0x0f, 0xb5, 0xc0, 0x57, 0xa6, 0x18, 0x80, + 0x80, 0x6b, 0x49, 0xfe, 0xbd, 0x3a, 0x7a, 0x2c, + 0xef, 0x70, 0xb5, 0x1c, 0xd2, 0xdf, 0x5f, 0x78, + 0x5a, 0xd8, 0x4f, 0xa0, 0x95, 0xb4, 0xb3, 0xb5, + 0xaa, 0x3b, + }, + + { + 0x14, 0x03, 0x01, 0x00, 0x01, 0x01, 0x16, 0x03, + 0x01, 0x00, 0x24, 0x9d, 0xc9, 0xda, 0xdf, 0xeb, + 0xc8, 0xdb, 0xf8, 0x94, 0xa5, 0xef, 0xd5, 0xfc, + 0x89, 0x01, 0x64, 0x30, 0x77, 0x5a, 0x18, 0x4b, + 0x16, 0x79, 0x9c, 0xf6, 0xf5, 0x09, 0x22, 0x12, + 0x4c, 0x3e, 0xa8, 0x8e, 0x91, 0xa5, 0x24, + }, +} + +var aesServerScript = [][]byte{ + { + 0x16, 0x03, 0x02, 0x00, 0x7f, 0x01, 0x00, 0x00, + 0x7b, 0x03, 0x02, 0x4d, 0x08, 0x2d, 0x0b, 0xb3, + 0x57, 0x85, 0x71, 0x4b, 0xfb, 0x34, 0xab, 0x16, + 0xd4, 0x92, 0x50, 0x81, 0x16, 0x95, 0x11, 0x28, + 0x1a, 0xcb, 0xff, 0x09, 0x4d, 0x23, 0xa6, 0xfe, + 0x2e, 0xbb, 0x78, 0x00, 0x00, 0x34, 0x00, 0x33, + 0x00, 0x45, 0x00, 0x39, 0x00, 0x88, 0x00, 0x16, + 0x00, 0x32, 0x00, 0x44, 0x00, 0x38, 0x00, 0x87, + 0x00, 0x13, 0x00, 0x66, 0x00, 0x90, 0x00, 0x91, + 0x00, 0x8f, 0x00, 0x8e, 0x00, 0x2f, 0x00, 0x41, + 0x00, 0x35, 0x00, 0x84, 0x00, 0x0a, 0x00, 0x05, + 0x00, 0x04, 0x00, 0x8c, 0x00, 0x8d, 0x00, 0x8b, + 0x00, 0x8a, 0x01, 0x00, 0x00, 0x1e, 0x00, 0x09, + 0x00, 0x03, 0x02, 0x00, 0x01, 0x00, 0x00, 0x00, + 0x0e, 0x00, 0x0c, 0x00, 0x00, 0x09, 0x6c, 0x6f, + 0x63, 0x61, 0x6c, 0x68, 0x6f, 0x73, 0x74, 0xff, + 0x01, 0x00, 0x01, 0x00, + }, + + { + 0x16, 0x03, 0x01, 0x00, 0x2a, 0x02, 0x00, 0x00, + 0x26, 0x03, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x2f, 0x00, 0x16, + 0x03, 0x01, 0x02, 0xbe, 0x0b, 0x00, 0x02, 0xba, + 0x00, 0x02, 0xb7, 0x00, 0x02, 0xb4, 0x30, 0x82, + 0x02, 0xb0, 0x30, 0x82, 0x02, 0x19, 0xa0, 0x03, + 0x02, 0x01, 0x02, 0x02, 0x09, 0x00, 0x85, 0xb0, + 0xbb, 0xa4, 0x8a, 0x7f, 0xb8, 0xca, 0x30, 0x0d, + 0x06, 0x09, 0x2a, 0x86, 0x48, 0x86, 0xf7, 0x0d, + 0x01, 0x01, 0x05, 0x05, 0x00, 0x30, 0x45, 0x31, + 0x0b, 0x30, 0x09, 0x06, 0x03, 0x55, 0x04, 0x06, + 0x13, 0x02, 0x41, 0x55, 0x31, 0x13, 0x30, 0x11, + 0x06, 0x03, 0x55, 0x04, 0x08, 0x13, 0x0a, 0x53, + 0x6f, 0x6d, 0x65, 0x2d, 0x53, 0x74, 0x61, 0x74, + 0x65, 0x31, 0x21, 0x30, 0x1f, 0x06, 0x03, 0x55, + 0x04, 0x0a, 0x13, 0x18, 0x49, 0x6e, 0x74, 0x65, + 0x72, 0x6e, 0x65, 0x74, 0x20, 0x57, 0x69, 0x64, + 0x67, 0x69, 0x74, 0x73, 0x20, 0x50, 0x74, 0x79, + 0x20, 0x4c, 0x74, 0x64, 0x30, 0x1e, 0x17, 0x0d, + 0x31, 0x30, 0x30, 0x34, 0x32, 0x34, 0x30, 0x39, + 0x30, 0x39, 0x33, 0x38, 0x5a, 0x17, 0x0d, 0x31, + 0x31, 0x30, 0x34, 0x32, 0x34, 0x30, 0x39, 0x30, + 0x39, 0x33, 0x38, 0x5a, 0x30, 0x45, 0x31, 0x0b, + 0x30, 0x09, 0x06, 0x03, 0x55, 0x04, 0x06, 0x13, + 0x02, 0x41, 0x55, 0x31, 0x13, 0x30, 0x11, 0x06, + 0x03, 0x55, 0x04, 0x08, 0x13, 0x0a, 0x53, 0x6f, + 0x6d, 0x65, 0x2d, 0x53, 0x74, 0x61, 0x74, 0x65, + 0x31, 0x21, 0x30, 0x1f, 0x06, 0x03, 0x55, 0x04, + 0x0a, 0x13, 0x18, 0x49, 0x6e, 0x74, 0x65, 0x72, + 0x6e, 0x65, 0x74, 0x20, 0x57, 0x69, 0x64, 0x67, + 0x69, 0x74, 0x73, 0x20, 0x50, 0x74, 0x79, 0x20, + 0x4c, 0x74, 0x64, 0x30, 0x81, 0x9f, 0x30, 0x0d, + 0x06, 0x09, 0x2a, 0x86, 0x48, 0x86, 0xf7, 0x0d, + 0x01, 0x01, 0x01, 0x05, 0x00, 0x03, 0x81, 0x8d, + 0x00, 0x30, 0x81, 0x89, 0x02, 0x81, 0x81, 0x00, + 0xbb, 0x79, 0xd6, 0xf5, 0x17, 0xb5, 0xe5, 0xbf, + 0x46, 0x10, 0xd0, 0xdc, 0x69, 0xbe, 0xe6, 0x2b, + 0x07, 0x43, 0x5a, 0xd0, 0x03, 0x2d, 0x8a, 0x7a, + 0x43, 0x85, 0xb7, 0x14, 0x52, 0xe7, 0xa5, 0x65, + 0x4c, 0x2c, 0x78, 0xb8, 0x23, 0x8c, 0xb5, 0xb4, + 0x82, 0xe5, 0xde, 0x1f, 0x95, 0x3b, 0x7e, 0x62, + 0xa5, 0x2c, 0xa5, 0x33, 0xd6, 0xfe, 0x12, 0x5c, + 0x7a, 0x56, 0xfc, 0xf5, 0x06, 0xbf, 0xfa, 0x58, + 0x7b, 0x26, 0x3f, 0xb5, 0xcd, 0x04, 0xd3, 0xd0, + 0xc9, 0x21, 0x96, 0x4a, 0xc7, 0xf4, 0x54, 0x9f, + 0x5a, 0xbf, 0xef, 0x42, 0x71, 0x00, 0xfe, 0x18, + 0x99, 0x07, 0x7f, 0x7e, 0x88, 0x7d, 0x7d, 0xf1, + 0x04, 0x39, 0xc4, 0xa2, 0x2e, 0xdb, 0x51, 0xc9, + 0x7c, 0xe3, 0xc0, 0x4c, 0x3b, 0x32, 0x66, 0x01, + 0xcf, 0xaf, 0xb1, 0x1d, 0xb8, 0x71, 0x9a, 0x1d, + 0xdb, 0xdb, 0x89, 0x6b, 0xae, 0xda, 0x2d, 0x79, + 0x02, 0x03, 0x01, 0x00, 0x01, 0xa3, 0x81, 0xa7, + 0x30, 0x81, 0xa4, 0x30, 0x1d, 0x06, 0x03, 0x55, + 0x1d, 0x0e, 0x04, 0x16, 0x04, 0x14, 0xb1, 0xad, + 0xe2, 0x85, 0x5a, 0xcf, 0xcb, 0x28, 0xdb, 0x69, + 0xce, 0x23, 0x69, 0xde, 0xd3, 0x26, 0x8e, 0x18, + 0x88, 0x39, 0x30, 0x75, 0x06, 0x03, 0x55, 0x1d, + 0x23, 0x04, 0x6e, 0x30, 0x6c, 0x80, 0x14, 0xb1, + 0xad, 0xe2, 0x85, 0x5a, 0xcf, 0xcb, 0x28, 0xdb, + 0x69, 0xce, 0x23, 0x69, 0xde, 0xd3, 0x26, 0x8e, + 0x18, 0x88, 0x39, 0xa1, 0x49, 0xa4, 0x47, 0x30, + 0x45, 0x31, 0x0b, 0x30, 0x09, 0x06, 0x03, 0x55, + 0x04, 0x06, 0x13, 0x02, 0x41, 0x55, 0x31, 0x13, + 0x30, 0x11, 0x06, 0x03, 0x55, 0x04, 0x08, 0x13, + 0x0a, 0x53, 0x6f, 0x6d, 0x65, 0x2d, 0x53, 0x74, + 0x61, 0x74, 0x65, 0x31, 0x21, 0x30, 0x1f, 0x06, + 0x03, 0x55, 0x04, 0x0a, 0x13, 0x18, 0x49, 0x6e, + 0x74, 0x65, 0x72, 0x6e, 0x65, 0x74, 0x20, 0x57, + 0x69, 0x64, 0x67, 0x69, 0x74, 0x73, 0x20, 0x50, + 0x74, 0x79, 0x20, 0x4c, 0x74, 0x64, 0x82, 0x09, + 0x00, 0x85, 0xb0, 0xbb, 0xa4, 0x8a, 0x7f, 0xb8, + 0xca, 0x30, 0x0c, 0x06, 0x03, 0x55, 0x1d, 0x13, + 0x04, 0x05, 0x30, 0x03, 0x01, 0x01, 0xff, 0x30, + 0x0d, 0x06, 0x09, 0x2a, 0x86, 0x48, 0x86, 0xf7, + 0x0d, 0x01, 0x01, 0x05, 0x05, 0x00, 0x03, 0x81, + 0x81, 0x00, 0x08, 0x6c, 0x45, 0x24, 0xc7, 0x6b, + 0xb1, 0x59, 0xab, 0x0c, 0x52, 0xcc, 0xf2, 0xb0, + 0x14, 0xd7, 0x87, 0x9d, 0x7a, 0x64, 0x75, 0xb5, + 0x5a, 0x95, 0x66, 0xe4, 0xc5, 0x2b, 0x8e, 0xae, + 0x12, 0x66, 0x1f, 0xeb, 0x4f, 0x38, 0xb3, 0x6e, + 0x60, 0xd3, 0x92, 0xfd, 0xf7, 0x41, 0x08, 0xb5, + 0x25, 0x13, 0xb1, 0x18, 0x7a, 0x24, 0xfb, 0x30, + 0x1d, 0xba, 0xed, 0x98, 0xb9, 0x17, 0xec, 0xe7, + 0xd7, 0x31, 0x59, 0xdb, 0x95, 0xd3, 0x1d, 0x78, + 0xea, 0x50, 0x56, 0x5c, 0xd5, 0x82, 0x5a, 0x2d, + 0x5a, 0x5f, 0x33, 0xc4, 0xb6, 0xd8, 0xc9, 0x75, + 0x90, 0x96, 0x8c, 0x0f, 0x52, 0x98, 0xb5, 0xcd, + 0x98, 0x1f, 0x89, 0x20, 0x5f, 0xf2, 0xa0, 0x1c, + 0xa3, 0x1b, 0x96, 0x94, 0xdd, 0xa9, 0xfd, 0x57, + 0xe9, 0x70, 0xe8, 0x26, 0x6d, 0x71, 0x99, 0x9b, + 0x26, 0x6e, 0x38, 0x50, 0x29, 0x6c, 0x90, 0xa7, + 0xbd, 0xd9, 0x16, 0x03, 0x01, 0x00, 0x04, 0x0e, + 0x00, 0x00, 0x00, + }, + + { + 0x16, 0x03, 0x01, 0x00, 0x86, 0x10, 0x00, 0x00, + 0x82, 0x00, 0x80, 0x71, 0x9c, 0xe7, 0x23, 0xfc, + 0xb9, 0x19, 0x29, 0x82, 0xbf, 0xef, 0x08, 0xf7, + 0x99, 0x36, 0xc3, 0x4c, 0x6f, 0x05, 0xd2, 0x8b, + 0x62, 0x2b, 0x19, 0x9b, 0x7f, 0xc0, 0xcc, 0x48, + 0x30, 0x5f, 0xcd, 0xc3, 0x70, 0x55, 0x53, 0x73, + 0xfa, 0x79, 0x74, 0xf3, 0xa3, 0x76, 0x9f, 0xa1, + 0x7f, 0x98, 0xc2, 0xc0, 0xe3, 0xc5, 0xa0, 0x31, + 0x2f, 0xa6, 0xe8, 0x1e, 0x61, 0x46, 0xb3, 0x9b, + 0x4b, 0x16, 0xf1, 0x2d, 0xc7, 0x63, 0x7f, 0x79, + 0x22, 0x30, 0xd1, 0xf2, 0xfc, 0x77, 0x98, 0x0a, + 0x16, 0x11, 0x63, 0x71, 0x7f, 0x70, 0xef, 0x16, + 0xbb, 0x39, 0x87, 0x34, 0xac, 0x49, 0xbd, 0x07, + 0x67, 0xcb, 0x9c, 0xcc, 0xde, 0xef, 0xb1, 0xe0, + 0xdb, 0x01, 0xb5, 0x35, 0xa9, 0xb3, 0x10, 0x0c, + 0x4b, 0xee, 0xb3, 0x4e, 0xfd, 0xbe, 0x15, 0x27, + 0xf0, 0x46, 0xb2, 0x38, 0xba, 0x5f, 0xcc, 0x89, + 0xec, 0x29, 0x82, 0x14, 0x03, 0x01, 0x00, 0x01, + 0x01, 0x16, 0x03, 0x01, 0x00, 0x30, 0x3c, 0xfb, + 0xa4, 0x12, 0xcb, 0x00, 0xf9, 0x57, 0x7e, 0x9b, + 0xc9, 0xdc, 0x0c, 0xba, 0x9a, 0x81, 0x62, 0xfb, + 0x26, 0x13, 0x53, 0xfe, 0xaa, 0xcc, 0x82, 0xbb, + 0xb6, 0x67, 0x7f, 0x39, 0xbe, 0x4d, 0xbb, 0xc0, + 0x6c, 0x24, 0x31, 0x83, 0xa5, 0x50, 0x3a, 0x75, + 0x32, 0x64, 0xb5, 0xdb, 0xbe, 0x0a, + }, + + { + 0x14, 0x03, 0x01, 0x00, 0x01, 0x01, 0x16, 0x03, + 0x01, 0x00, 0x30, 0x43, 0x24, 0x42, 0x55, 0x08, + 0xe4, 0xc2, 0x15, 0xc9, 0xdb, 0x71, 0x69, 0xee, + 0x09, 0xc5, 0x1c, 0xfd, 0x46, 0x10, 0xa0, 0x68, + 0x21, 0xf2, 0x48, 0xac, 0x6c, 0xc0, 0x2b, 0x62, + 0x07, 0x8f, 0x48, 0x33, 0x0a, 0x6b, 0x62, 0x28, + 0x2e, 0x2c, 0xad, 0xcb, 0x34, 0x85, 0xca, 0x2e, + 0xcd, 0x84, 0xf0, + }, +} diff --git a/src/pkg/crypto/tls/key_agreement.go b/src/pkg/crypto/tls/key_agreement.go new file mode 100644 index 000000000..a40d18fd9 --- /dev/null +++ b/src/pkg/crypto/tls/key_agreement.go @@ -0,0 +1,245 @@ +// Copyright 2010 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package tls + +import ( + "big" + "crypto" + "crypto/elliptic" + "crypto/md5" + "crypto/rsa" + "crypto/sha1" + "crypto/x509" + "io" + "os" +) + +// rsaKeyAgreement implements the standard TLS key agreement where the client +// encrypts the pre-master secret to the server's public key. +type rsaKeyAgreement struct{} + +func (ka rsaKeyAgreement) generateServerKeyExchange(config *Config, clientHello *clientHelloMsg, hello *serverHelloMsg) (*serverKeyExchangeMsg, os.Error) { + return nil, nil +} + +func (ka rsaKeyAgreement) processClientKeyExchange(config *Config, ckx *clientKeyExchangeMsg) ([]byte, os.Error) { + preMasterSecret := make([]byte, 48) + _, err := io.ReadFull(config.rand(), preMasterSecret[2:]) + if err != nil { + return nil, err + } + + if len(ckx.ciphertext) < 2 { + return nil, os.NewError("bad ClientKeyExchange") + } + ciphertextLen := int(ckx.ciphertext[0])<<8 | int(ckx.ciphertext[1]) + if ciphertextLen != len(ckx.ciphertext)-2 { + return nil, os.NewError("bad ClientKeyExchange") + } + ciphertext := ckx.ciphertext[2:] + + err = rsa.DecryptPKCS1v15SessionKey(config.rand(), config.Certificates[0].PrivateKey, ciphertext, preMasterSecret) + if err != nil { + return nil, err + } + // We don't check the version number in the premaster secret. For one, + // by checking it, we would leak information about the validity of the + // encrypted pre-master secret. Secondly, it provides only a small + // benefit against a downgrade attack and some implementations send the + // wrong version anyway. See the discussion at the end of section + // 7.4.7.1 of RFC 4346. + return preMasterSecret, nil +} + +func (ka rsaKeyAgreement) processServerKeyExchange(config *Config, clientHello *clientHelloMsg, serverHello *serverHelloMsg, cert *x509.Certificate, skx *serverKeyExchangeMsg) os.Error { + return os.NewError("unexpected ServerKeyExchange") +} + +func (ka rsaKeyAgreement) generateClientKeyExchange(config *Config, clientHello *clientHelloMsg, cert *x509.Certificate) ([]byte, *clientKeyExchangeMsg, os.Error) { + preMasterSecret := make([]byte, 48) + preMasterSecret[0] = byte(clientHello.vers >> 8) + preMasterSecret[1] = byte(clientHello.vers) + _, err := io.ReadFull(config.rand(), preMasterSecret[2:]) + if err != nil { + return nil, nil, err + } + + encrypted, err := rsa.EncryptPKCS1v15(config.rand(), cert.PublicKey.(*rsa.PublicKey), preMasterSecret) + if err != nil { + return nil, nil, err + } + ckx := new(clientKeyExchangeMsg) + ckx.ciphertext = make([]byte, len(encrypted)+2) + ckx.ciphertext[0] = byte(len(encrypted) >> 8) + ckx.ciphertext[1] = byte(len(encrypted)) + copy(ckx.ciphertext[2:], encrypted) + return preMasterSecret, ckx, nil +} + +// md5SHA1Hash implements TLS 1.0's hybrid hash function which consists of the +// concatenation of an MD5 and SHA1 hash. +func md5SHA1Hash(slices ...[]byte) []byte { + md5sha1 := make([]byte, md5.Size+sha1.Size) + hmd5 := md5.New() + for _, slice := range slices { + hmd5.Write(slice) + } + copy(md5sha1, hmd5.Sum()) + + hsha1 := sha1.New() + for _, slice := range slices { + hsha1.Write(slice) + } + copy(md5sha1[md5.Size:], hsha1.Sum()) + return md5sha1 +} + +// ecdheRSAKeyAgreement implements a TLS key agreement where the server +// generates a ephemeral EC public/private key pair and signs it. The +// pre-master secret is then calculated using ECDH. +type ecdheRSAKeyAgreement struct { + privateKey []byte + curve *elliptic.Curve + x, y *big.Int +} + +func (ka *ecdheRSAKeyAgreement) generateServerKeyExchange(config *Config, clientHello *clientHelloMsg, hello *serverHelloMsg) (*serverKeyExchangeMsg, os.Error) { + var curveid uint16 + +Curve: + for _, c := range clientHello.supportedCurves { + switch c { + case curveP256: + ka.curve = elliptic.P256() + curveid = c + break Curve + case curveP384: + ka.curve = elliptic.P384() + curveid = c + break Curve + case curveP521: + ka.curve = elliptic.P521() + curveid = c + break Curve + } + } + + var x, y *big.Int + var err os.Error + ka.privateKey, x, y, err = ka.curve.GenerateKey(config.rand()) + if err != nil { + return nil, err + } + ecdhePublic := ka.curve.Marshal(x, y) + + // http://tools.ietf.org/html/rfc4492#section-5.4 + serverECDHParams := make([]byte, 1+2+1+len(ecdhePublic)) + serverECDHParams[0] = 3 // named curve + serverECDHParams[1] = byte(curveid >> 8) + serverECDHParams[2] = byte(curveid) + serverECDHParams[3] = byte(len(ecdhePublic)) + copy(serverECDHParams[4:], ecdhePublic) + + md5sha1 := md5SHA1Hash(clientHello.random, hello.random, serverECDHParams) + sig, err := rsa.SignPKCS1v15(config.rand(), config.Certificates[0].PrivateKey, crypto.MD5SHA1, md5sha1) + if err != nil { + return nil, os.NewError("failed to sign ECDHE parameters: " + err.String()) + } + + skx := new(serverKeyExchangeMsg) + skx.key = make([]byte, len(serverECDHParams)+2+len(sig)) + copy(skx.key, serverECDHParams) + k := skx.key[len(serverECDHParams):] + k[0] = byte(len(sig) >> 8) + k[1] = byte(len(sig)) + copy(k[2:], sig) + + return skx, nil +} + +func (ka *ecdheRSAKeyAgreement) processClientKeyExchange(config *Config, ckx *clientKeyExchangeMsg) ([]byte, os.Error) { + if len(ckx.ciphertext) == 0 || int(ckx.ciphertext[0]) != len(ckx.ciphertext)-1 { + return nil, os.NewError("bad ClientKeyExchange") + } + x, y := ka.curve.Unmarshal(ckx.ciphertext[1:]) + if x == nil { + return nil, os.NewError("bad ClientKeyExchange") + } + x, _ = ka.curve.ScalarMult(x, y, ka.privateKey) + preMasterSecret := make([]byte, (ka.curve.BitSize+7)>>3) + xBytes := x.Bytes() + copy(preMasterSecret[len(preMasterSecret)-len(xBytes):], xBytes) + + return preMasterSecret, nil +} + +var errServerKeyExchange = os.NewError("invalid ServerKeyExchange") + +func (ka *ecdheRSAKeyAgreement) processServerKeyExchange(config *Config, clientHello *clientHelloMsg, serverHello *serverHelloMsg, cert *x509.Certificate, skx *serverKeyExchangeMsg) os.Error { + if len(skx.key) < 4 { + return errServerKeyExchange + } + if skx.key[0] != 3 { // named curve + return os.NewError("server selected unsupported curve") + } + curveid := uint16(skx.key[1])<<8 | uint16(skx.key[2]) + + switch curveid { + case curveP256: + ka.curve = elliptic.P256() + case curveP384: + ka.curve = elliptic.P384() + case curveP521: + ka.curve = elliptic.P521() + default: + return os.NewError("server selected unsupported curve") + } + + publicLen := int(skx.key[3]) + if publicLen+4 > len(skx.key) { + return errServerKeyExchange + } + ka.x, ka.y = ka.curve.Unmarshal(skx.key[4 : 4+publicLen]) + if ka.x == nil { + return errServerKeyExchange + } + serverECDHParams := skx.key[:4+publicLen] + + sig := skx.key[4+publicLen:] + if len(sig) < 2 { + return errServerKeyExchange + } + sigLen := int(sig[0])<<8 | int(sig[1]) + if sigLen+2 != len(sig) { + return errServerKeyExchange + } + sig = sig[2:] + + md5sha1 := md5SHA1Hash(clientHello.random, serverHello.random, serverECDHParams) + return rsa.VerifyPKCS1v15(cert.PublicKey.(*rsa.PublicKey), crypto.MD5SHA1, md5sha1, sig) +} + +func (ka *ecdheRSAKeyAgreement) generateClientKeyExchange(config *Config, clientHello *clientHelloMsg, cert *x509.Certificate) ([]byte, *clientKeyExchangeMsg, os.Error) { + if ka.curve == nil { + return nil, nil, os.NewError("missing ServerKeyExchange message") + } + priv, mx, my, err := ka.curve.GenerateKey(config.rand()) + if err != nil { + return nil, nil, err + } + x, _ := ka.curve.ScalarMult(ka.x, ka.y, priv) + preMasterSecret := make([]byte, (ka.curve.BitSize+7)>>3) + xBytes := x.Bytes() + copy(preMasterSecret[len(preMasterSecret)-len(xBytes):], xBytes) + + serialized := ka.curve.Marshal(mx, my) + + ckx := new(clientKeyExchangeMsg) + ckx.ciphertext = make([]byte, 1+len(serialized)) + ckx.ciphertext[0] = byte(len(serialized)) + copy(ckx.ciphertext[1:], serialized) + + return preMasterSecret, ckx, nil +} diff --git a/src/pkg/crypto/tls/parse-gnutls-cli-debug-log.py b/src/pkg/crypto/tls/parse-gnutls-cli-debug-log.py new file mode 100644 index 000000000..c03eaa6ea --- /dev/null +++ b/src/pkg/crypto/tls/parse-gnutls-cli-debug-log.py @@ -0,0 +1,55 @@ +# Copyright 2010 The Go Authors. All rights reserved. +# Use of this source code is governed by a BSD-style +# license that can be found in the LICENSE file. + +# This code is used to parse the debug log from gnutls-cli and generate a +# script of the handshake. This script is included in handshake_server_test.go. +# See the comments there for details. + +import sys + +blocks = [] + +READ = 1 +WRITE = 2 + +currentBlockType = 0 +currentBlock = [] +for line in sys.stdin.readlines(): + line = line[:-1] + if line.startswith("|<7>| WRITE: "): + if currentBlockType != WRITE: + if len(currentBlock) > 0: + blocks.append(currentBlock) + currentBlock = [] + currentBlockType = WRITE + elif line.startswith("|<7>| READ: "): + if currentBlockType != READ: + if len(currentBlock) > 0: + blocks.append(currentBlock) + currentBlock = [] + currentBlockType = READ + elif line.startswith("|<7>| 0"): + line = line[13:] + line = line.strip() + bs = line.split() + for b in bs: + currentBlock.append(int(b, 16)) + +if len(currentBlock) > 0: + blocks.append(currentBlock) + +for block in blocks: + sys.stdout.write("\t{\n") + + i = 0 + for b in block: + if i % 8 == 0: + sys.stdout.write("\t\t") + sys.stdout.write("0x%02x," % b) + if i % 8 == 7: + sys.stdout.write("\n") + else: + sys.stdout.write(" ") + i += 1 + sys.stdout.write("\n\t},\n\n") diff --git a/src/pkg/crypto/tls/prf.go b/src/pkg/crypto/tls/prf.go new file mode 100644 index 000000000..478cf65f9 --- /dev/null +++ b/src/pkg/crypto/tls/prf.go @@ -0,0 +1,153 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package tls + +import ( + "crypto/hmac" + "crypto/md5" + "crypto/sha1" + "hash" + "os" +) + +// Split a premaster secret in two as specified in RFC 4346, section 5. +func splitPreMasterSecret(secret []byte) (s1, s2 []byte) { + s1 = secret[0 : (len(secret)+1)/2] + s2 = secret[len(secret)/2:] + return +} + +// pHash implements the P_hash function, as defined in RFC 4346, section 5. +func pHash(result, secret, seed []byte, hash func() hash.Hash) { + h := hmac.New(hash, secret) + h.Write(seed) + a := h.Sum() + + j := 0 + for j < len(result) { + h.Reset() + h.Write(a) + h.Write(seed) + b := h.Sum() + todo := len(b) + if j+todo > len(result) { + todo = len(result) - j + } + copy(result[j:j+todo], b) + j += todo + + h.Reset() + h.Write(a) + a = h.Sum() + } +} + +// pRF10 implements the TLS 1.0 pseudo-random function, as defined in RFC 2246, section 5. +func pRF10(result, secret, label, seed []byte) { + hashSHA1 := sha1.New + hashMD5 := md5.New + + labelAndSeed := make([]byte, len(label)+len(seed)) + copy(labelAndSeed, label) + copy(labelAndSeed[len(label):], seed) + + s1, s2 := splitPreMasterSecret(secret) + pHash(result, s1, labelAndSeed, hashMD5) + result2 := make([]byte, len(result)) + pHash(result2, s2, labelAndSeed, hashSHA1) + + for i, b := range result2 { + result[i] ^= b + } +} + +const ( + tlsRandomLength = 32 // Length of a random nonce in TLS 1.1. + masterSecretLength = 48 // Length of a master secret in TLS 1.1. + finishedVerifyLength = 12 // Length of verify_data in a Finished message. +) + +var masterSecretLabel = []byte("master secret") +var keyExpansionLabel = []byte("key expansion") +var clientFinishedLabel = []byte("client finished") +var serverFinishedLabel = []byte("server finished") + +// keysFromPreMasterSecret generates the connection keys from the pre master +// secret, given the lengths of the MAC key, cipher key and IV, as defined in +// RFC 2246, section 6.3. +func keysFromPreMasterSecret10(preMasterSecret, clientRandom, serverRandom []byte, macLen, keyLen, ivLen int) (masterSecret, clientMAC, serverMAC, clientKey, serverKey, clientIV, serverIV []byte) { + var seed [tlsRandomLength * 2]byte + copy(seed[0:len(clientRandom)], clientRandom) + copy(seed[len(clientRandom):], serverRandom) + masterSecret = make([]byte, masterSecretLength) + pRF10(masterSecret, preMasterSecret, masterSecretLabel, seed[0:]) + + copy(seed[0:len(clientRandom)], serverRandom) + copy(seed[len(serverRandom):], clientRandom) + + n := 2*macLen + 2*keyLen + 2*ivLen + keyMaterial := make([]byte, n) + pRF10(keyMaterial, masterSecret, keyExpansionLabel, seed[0:]) + clientMAC = keyMaterial[:macLen] + keyMaterial = keyMaterial[macLen:] + serverMAC = keyMaterial[:macLen] + keyMaterial = keyMaterial[macLen:] + clientKey = keyMaterial[:keyLen] + keyMaterial = keyMaterial[keyLen:] + serverKey = keyMaterial[:keyLen] + keyMaterial = keyMaterial[keyLen:] + clientIV = keyMaterial[:ivLen] + keyMaterial = keyMaterial[ivLen:] + serverIV = keyMaterial[:ivLen] + return +} + +// A finishedHash calculates the hash of a set of handshake messages suitable +// for including in a Finished message. +type finishedHash struct { + clientMD5 hash.Hash + clientSHA1 hash.Hash + serverMD5 hash.Hash + serverSHA1 hash.Hash +} + +func newFinishedHash() finishedHash { + return finishedHash{md5.New(), sha1.New(), md5.New(), sha1.New()} +} + +func (h finishedHash) Write(msg []byte) (n int, err os.Error) { + h.clientMD5.Write(msg) + h.clientSHA1.Write(msg) + h.serverMD5.Write(msg) + h.serverSHA1.Write(msg) + return len(msg), nil +} + +// finishedSum calculates the contents of the verify_data member of a Finished +// message given the MD5 and SHA1 hashes of a set of handshake messages. +func finishedSum(md5, sha1, label, masterSecret []byte) []byte { + seed := make([]byte, len(md5)+len(sha1)) + copy(seed, md5) + copy(seed[len(md5):], sha1) + out := make([]byte, finishedVerifyLength) + pRF10(out, masterSecret, label, seed) + return out +} + +// clientSum returns the contents of the verify_data member of a client's +// Finished message. +func (h finishedHash) clientSum(masterSecret []byte) []byte { + md5 := h.clientMD5.Sum() + sha1 := h.clientSHA1.Sum() + return finishedSum(md5, sha1, clientFinishedLabel, masterSecret) +} + +// serverSum returns the contents of the verify_data member of a server's +// Finished message. +func (h finishedHash) serverSum(masterSecret []byte) []byte { + md5 := h.serverMD5.Sum() + sha1 := h.serverSHA1.Sum() + return finishedSum(md5, sha1, serverFinishedLabel, masterSecret) +} diff --git a/src/pkg/crypto/tls/prf_test.go b/src/pkg/crypto/tls/prf_test.go new file mode 100644 index 000000000..f8c4acb9d --- /dev/null +++ b/src/pkg/crypto/tls/prf_test.go @@ -0,0 +1,104 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package tls + +import ( + "encoding/hex" + "testing" +) + +type testSplitPreMasterSecretTest struct { + in, out1, out2 string +} + +var testSplitPreMasterSecretTests = []testSplitPreMasterSecretTest{ + {"", "", ""}, + {"00", "00", "00"}, + {"0011", "00", "11"}, + {"001122", "0011", "1122"}, + {"00112233", "0011", "2233"}, +} + +func TestSplitPreMasterSecret(t *testing.T) { + for i, test := range testSplitPreMasterSecretTests { + in, _ := hex.DecodeString(test.in) + out1, out2 := splitPreMasterSecret(in) + s1 := hex.EncodeToString(out1) + s2 := hex.EncodeToString(out2) + if s1 != test.out1 || s2 != test.out2 { + t.Errorf("#%d: got: (%s, %s) want: (%s, %s)", i, s1, s2, test.out1, test.out2) + } + } +} + +type testKeysFromTest struct { + preMasterSecret string + clientRandom, serverRandom string + masterSecret string + clientMAC, serverMAC string + clientKey, serverKey string + macLen, keyLen int +} + +func TestKeysFromPreMasterSecret(t *testing.T) { + for i, test := range testKeysFromTests { + in, _ := hex.DecodeString(test.preMasterSecret) + clientRandom, _ := hex.DecodeString(test.clientRandom) + serverRandom, _ := hex.DecodeString(test.serverRandom) + master, clientMAC, serverMAC, clientKey, serverKey, _, _ := keysFromPreMasterSecret10(in, clientRandom, serverRandom, test.macLen, test.keyLen, 0) + masterString := hex.EncodeToString(master) + clientMACString := hex.EncodeToString(clientMAC) + serverMACString := hex.EncodeToString(serverMAC) + clientKeyString := hex.EncodeToString(clientKey) + serverKeyString := hex.EncodeToString(serverKey) + if masterString != test.masterSecret || + clientMACString != test.clientMAC || + serverMACString != test.serverMAC || + clientKeyString != test.clientKey || + serverKeyString != test.serverKey { + t.Errorf("#%d: got: (%s, %s, %s, %s, %s) want: (%s, %s, %s, %s %s)", i, masterString, clientMACString, serverMACString, clientKeyString, serverMACString, test.masterSecret, test.clientMAC, test.serverMAC, test.clientKey, test.serverKey) + } + } +} + +// These test vectors were generated from GnuTLS using `gnutls-cli --insecure -d 9 ` +var testKeysFromTests = []testKeysFromTest{ + { + "0302cac83ad4b1db3b9ab49ad05957de2a504a634a386fc600889321e1a971f57479466830ac3e6f468e87f5385fa0c5", + "4ae66303755184a3917fcb44880605fcc53baa01912b22ed94473fc69cebd558", + "4ae663020ec16e6bb5130be918cfcafd4d765979a3136a5d50c593446e4e44db", + "3d851bab6e5556e959a16bc36d66cfae32f672bfa9ecdef6096cbb1b23472df1da63dbbd9827606413221d149ed08ceb", + "805aaa19b3d2c0a0759a4b6c9959890e08480119", + "2d22f9fe519c075c16448305ceee209fc24ad109", + "d50b5771244f850cd8117a9ccafe2cf1", + "e076e33206b30507a85c32855acd0919", + 20, + 16, + }, + { + "03023f7527316bc12cbcd69e4b9e8275d62c028f27e65c745cfcddc7ce01bd3570a111378b63848127f1c36e5f9e4890", + "4ae66364b5ea56b20ce4e25555aed2d7e67f42788dd03f3fee4adae0459ab106", + "4ae66363ab815cbf6a248b87d6b556184e945e9b97fbdf247858b0bdafacfa1c", + "7d64be7c80c59b740200b4b9c26d0baaa1c5ae56705acbcf2307fe62beb4728c19392c83f20483801cce022c77645460", + "97742ed60a0554ca13f04f97ee193177b971e3b0", + "37068751700400e03a8477a5c7eec0813ab9e0dc", + "207cddbc600d2a200abac6502053ee5c", + "df3f94f6e1eacc753b815fe16055cd43", + 20, + 16, + }, + { + "832d515f1d61eebb2be56ba0ef79879efb9b527504abb386fb4310ed5d0e3b1f220d3bb6b455033a2773e6d8bdf951d278a187482b400d45deb88a5d5a6bb7d6a7a1decc04eb9ef0642876cd4a82d374d3b6ff35f0351dc5d411104de431375355addc39bfb1f6329fb163b0bc298d658338930d07d313cd980a7e3d9196cac1", + "4ae663b2ee389c0de147c509d8f18f5052afc4aaf9699efe8cb05ece883d3a5e", + "4ae664d503fd4cff50cfc1fb8fc606580f87b0fcdac9554ba0e01d785bdf278e", + "1aff2e7a2c4279d0126f57a65a77a8d9d0087cf2733366699bec27eb53d5740705a8574bb1acc2abbe90e44f0dd28d6c", + "3c7647c93c1379a31a609542aa44e7f117a70085", + "0d73102994be74a575a3ead8532590ca32a526d4", + "ac7581b0b6c10d85bbd905ffbf36c65e", + "ff07edde49682b45466bd2e39464b306", + 20, + 16, + }, +} diff --git a/src/pkg/crypto/tls/tls.go b/src/pkg/crypto/tls/tls.go new file mode 100644 index 000000000..4f0859fee --- /dev/null +++ b/src/pkg/crypto/tls/tls.go @@ -0,0 +1,181 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Package tls partially implements the TLS 1.1 protocol, as specified in RFC +// 4346. +package tls + +import ( + "crypto/rsa" + "crypto/x509" + "encoding/pem" + "io/ioutil" + "net" + "os" + "strings" +) + +// Server returns a new TLS server side connection +// using conn as the underlying transport. +// The configuration config must be non-nil and must have +// at least one certificate. +func Server(conn net.Conn, config *Config) *Conn { + return &Conn{conn: conn, config: config} +} + +// Client returns a new TLS client side connection +// using conn as the underlying transport. +// Client interprets a nil configuration as equivalent to +// the zero configuration; see the documentation of Config +// for the defaults. +func Client(conn net.Conn, config *Config) *Conn { + return &Conn{conn: conn, config: config, isClient: true} +} + +// A Listener implements a network listener (net.Listener) for TLS connections. +type Listener struct { + listener net.Listener + config *Config +} + +// Accept waits for and returns the next incoming TLS connection. +// The returned connection c is a *tls.Conn. +func (l *Listener) Accept() (c net.Conn, err os.Error) { + c, err = l.listener.Accept() + if err != nil { + return + } + c = Server(c, l.config) + return +} + +// Close closes the listener. +func (l *Listener) Close() os.Error { return l.listener.Close() } + +// Addr returns the listener's network address. +func (l *Listener) Addr() net.Addr { return l.listener.Addr() } + +// NewListener creates a Listener which accepts connections from an inner +// Listener and wraps each connection with Server. +// The configuration config must be non-nil and must have +// at least one certificate. +func NewListener(listener net.Listener, config *Config) (l *Listener) { + l = new(Listener) + l.listener = listener + l.config = config + return +} + +// Listen creates a TLS listener accepting connections on the +// given network address using net.Listen. +// The configuration config must be non-nil and must have +// at least one certificate. +func Listen(network, laddr string, config *Config) (*Listener, os.Error) { + if config == nil || len(config.Certificates) == 0 { + return nil, os.NewError("tls.Listen: no certificates in configuration") + } + l, err := net.Listen(network, laddr) + if err != nil { + return nil, err + } + return NewListener(l, config), nil +} + +// Dial connects to the given network address using net.Dial +// and then initiates a TLS handshake, returning the resulting +// TLS connection. +// Dial interprets a nil configuration as equivalent to +// the zero configuration; see the documentation of Config +// for the defaults. +func Dial(network, addr string, config *Config) (*Conn, os.Error) { + raddr := addr + c, err := net.Dial(network, raddr) + if err != nil { + return nil, err + } + + colonPos := strings.LastIndex(raddr, ":") + if colonPos == -1 { + colonPos = len(raddr) + } + hostname := raddr[:colonPos] + + if config == nil { + config = defaultConfig() + } + if config.ServerName != "" { + // Make a copy to avoid polluting argument or default. + c := *config + c.ServerName = hostname + config = &c + } + conn := Client(c, config) + if err = conn.Handshake(); err != nil { + c.Close() + return nil, err + } + return conn, nil +} + +// LoadX509KeyPair reads and parses a public/private key pair from a pair of +// files. The files must contain PEM encoded data. +func LoadX509KeyPair(certFile string, keyFile string) (cert Certificate, err os.Error) { + certPEMBlock, err := ioutil.ReadFile(certFile) + if err != nil { + return + } + keyPEMBlock, err := ioutil.ReadFile(keyFile) + if err != nil { + return + } + return X509KeyPair(certPEMBlock, keyPEMBlock) +} + +// X509KeyPair parses a public/private key pair from a pair of +// PEM encoded data. +func X509KeyPair(certPEMBlock, keyPEMBlock []byte) (cert Certificate, err os.Error) { + var certDERBlock *pem.Block + for { + certDERBlock, certPEMBlock = pem.Decode(certPEMBlock) + if certDERBlock == nil { + break + } + if certDERBlock.Type == "CERTIFICATE" { + cert.Certificate = append(cert.Certificate, certDERBlock.Bytes) + } + } + + if len(cert.Certificate) == 0 { + err = os.NewError("crypto/tls: failed to parse certificate PEM data") + return + } + + keyDERBlock, _ := pem.Decode(keyPEMBlock) + if keyDERBlock == nil { + err = os.NewError("crypto/tls: failed to parse key PEM data") + return + } + + key, err := x509.ParsePKCS1PrivateKey(keyDERBlock.Bytes) + if err != nil { + err = os.NewError("crypto/tls: failed to parse key: " + err.String()) + return + } + + cert.PrivateKey = key + + // We don't need to parse the public key for TLS, but we so do anyway + // to check that it looks sane and matches the private key. + x509Cert, err := x509.ParseCertificate(cert.Certificate[0]) + if err != nil { + return + } + + if x509Cert.PublicKeyAlgorithm != x509.RSA || x509Cert.PublicKey.(*rsa.PublicKey).N.Cmp(key.PublicKey.N) != 0 { + err = os.NewError("crypto/tls: private key does not match public key") + return + } + + return +} diff --git a/src/pkg/crypto/twofish/Makefile b/src/pkg/crypto/twofish/Makefile new file mode 100644 index 000000000..aec61659d --- /dev/null +++ b/src/pkg/crypto/twofish/Makefile @@ -0,0 +1,11 @@ +# Copyright 2011 The Go Authors. All rights reserved. +# Use of this source code is governed by a BSD-style +# license that can be found in the LICENSE file. + +include ../../../Make.inc + +TARG=crypto/twofish +GOFILES=\ + twofish.go\ + +include ../../../Make.pkg diff --git a/src/pkg/crypto/twofish/twofish.go b/src/pkg/crypto/twofish/twofish.go new file mode 100644 index 000000000..2e537c606 --- /dev/null +++ b/src/pkg/crypto/twofish/twofish.go @@ -0,0 +1,358 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Package twofish implements Bruce Schneier's Twofish encryption algorithm. +package twofish + +// Twofish is defined in http://www.schneier.com/paper-twofish-paper.pdf [TWOFISH] + +// This code is a port of the LibTom C implementation. +// See http://libtom.org/?page=features&newsitems=5&whatfile=crypt. +// LibTomCrypt is free for all purposes under the public domain. +// It was heavily inspired by the go blowfish package. + +import ( + "os" + "strconv" +) + +// BlockSize is the constant block size of Twofish. +const BlockSize = 16 + +const mdsPolynomial = 0x169 // x^8 + x^6 + x^5 + x^3 + 1, see [TWOFISH] 4.2 +const rsPolynomial = 0x14d // x^8 + x^6 + x^3 + x^2 + 1, see [TWOFISH] 4.3 + +// A Cipher is an instance of Twofish encryption using a particular key. +type Cipher struct { + s [4][256]uint32 + k [40]uint32 +} + +type KeySizeError int + +func (k KeySizeError) String() string { + return "crypto/twofish: invalid key size " + strconv.Itoa(int(k)) +} + +// NewCipher creates and returns a Cipher. +// The key argument should be the Twofish key, 16, 24 or 32 bytes. +func NewCipher(key []byte) (*Cipher, os.Error) { + keylen := len(key) + + if keylen != 16 && keylen != 24 && keylen != 32 { + return nil, KeySizeError(keylen) + } + + // k is the number of 64 bit words in key + k := keylen / 8 + + // Create the S[..] words + var S [4 * 4]byte + for i := 0; i < k; i++ { + // Computes [y0 y1 y2 y3] = rs . [x0 x1 x2 x3 x4 x5 x6 x7] + for j, rsRow := range rs { + for k, rsVal := range rsRow { + S[4*i+j] ^= gfMult(key[8*i+k], rsVal, rsPolynomial) + } + } + } + + // Calculate subkeys + c := new(Cipher) + var tmp [4]byte + for i := byte(0); i < 20; i++ { + // A = h(p * 2x, Me) + for j := range tmp { + tmp[j] = 2 * i + } + A := h(tmp[:], key, 0) + + // B = rolc(h(p * (2x + 1), Mo), 8) + for j := range tmp { + tmp[j] = 2*i + 1 + } + B := h(tmp[:], key, 1) + B = rol(B, 8) + + c.k[2*i] = A + B + + // K[2i+1] = (A + 2B) <<< 9 + c.k[2*i+1] = rol(2*B+A, 9) + } + + // Calculate sboxes + switch k { + case 2: + for i := range c.s[0] { + c.s[0][i] = mdsColumnMult(sbox[1][sbox[0][sbox[0][byte(i)]^S[0]]^S[4]], 0) + c.s[1][i] = mdsColumnMult(sbox[0][sbox[0][sbox[1][byte(i)]^S[1]]^S[5]], 1) + c.s[2][i] = mdsColumnMult(sbox[1][sbox[1][sbox[0][byte(i)]^S[2]]^S[6]], 2) + c.s[3][i] = mdsColumnMult(sbox[0][sbox[1][sbox[1][byte(i)]^S[3]]^S[7]], 3) + } + case 3: + for i := range c.s[0] { + c.s[0][i] = mdsColumnMult(sbox[1][sbox[0][sbox[0][sbox[1][byte(i)]^S[0]]^S[4]]^S[8]], 0) + c.s[1][i] = mdsColumnMult(sbox[0][sbox[0][sbox[1][sbox[1][byte(i)]^S[1]]^S[5]]^S[9]], 1) + c.s[2][i] = mdsColumnMult(sbox[1][sbox[1][sbox[0][sbox[0][byte(i)]^S[2]]^S[6]]^S[10]], 2) + c.s[3][i] = mdsColumnMult(sbox[0][sbox[1][sbox[1][sbox[0][byte(i)]^S[3]]^S[7]]^S[11]], 3) + } + default: + for i := range c.s[0] { + c.s[0][i] = mdsColumnMult(sbox[1][sbox[0][sbox[0][sbox[1][sbox[1][byte(i)]^S[0]]^S[4]]^S[8]]^S[12]], 0) + c.s[1][i] = mdsColumnMult(sbox[0][sbox[0][sbox[1][sbox[1][sbox[0][byte(i)]^S[1]]^S[5]]^S[9]]^S[13]], 1) + c.s[2][i] = mdsColumnMult(sbox[1][sbox[1][sbox[0][sbox[0][sbox[0][byte(i)]^S[2]]^S[6]]^S[10]]^S[14]], 2) + c.s[3][i] = mdsColumnMult(sbox[0][sbox[1][sbox[1][sbox[0][sbox[1][byte(i)]^S[3]]^S[7]]^S[11]]^S[15]], 3) + } + } + + return c, nil +} + +// Reset zeros the key data, so that it will no longer appear in the process's +// memory. +func (c *Cipher) Reset() { + for i := range c.k { + c.k[i] = 0 + } + for i := range c.s { + for j := 0; j < 256; j++ { + c.s[i][j] = 0 + } + } +} + +// BlockSize returns the Twofish block size, 16 bytes. +func (c *Cipher) BlockSize() int { return BlockSize } + +// store32l stores src in dst in little-endian form. +func store32l(dst []byte, src uint32) { + dst[0] = byte(src) + dst[1] = byte(src >> 8) + dst[2] = byte(src >> 16) + dst[3] = byte(src >> 24) + return +} + +// load32l reads a little-endian uint32 from src. +func load32l(src []byte) uint32 { + return uint32(src[0]) | uint32(src[1])<<8 | uint32(src[2])<<16 | uint32(src[3])<<24 +} + +// rol returns x after a left circular rotation of y bits. +func rol(x, y uint32) uint32 { + return (x << (y & 31)) | (x >> (32 - (y & 31))) +} + +// ror returns x after a right circular rotation of y bits. +func ror(x, y uint32) uint32 { + return (x >> (y & 31)) | (x << (32 - (y & 31))) +} + +// The RS matrix. See [TWOFISH] 4.3 +var rs = [4][8]byte{ + {0x01, 0xA4, 0x55, 0x87, 0x5A, 0x58, 0xDB, 0x9E}, + {0xA4, 0x56, 0x82, 0xF3, 0x1E, 0xC6, 0x68, 0xE5}, + {0x02, 0xA1, 0xFC, 0xC1, 0x47, 0xAE, 0x3D, 0x19}, + {0xA4, 0x55, 0x87, 0x5A, 0x58, 0xDB, 0x9E, 0x03}, +} + +// sbox tables +var sbox = [2][256]byte{ + { + 0xa9, 0x67, 0xb3, 0xe8, 0x04, 0xfd, 0xa3, 0x76, 0x9a, 0x92, 0x80, 0x78, 0xe4, 0xdd, 0xd1, 0x38, + 0x0d, 0xc6, 0x35, 0x98, 0x18, 0xf7, 0xec, 0x6c, 0x43, 0x75, 0x37, 0x26, 0xfa, 0x13, 0x94, 0x48, + 0xf2, 0xd0, 0x8b, 0x30, 0x84, 0x54, 0xdf, 0x23, 0x19, 0x5b, 0x3d, 0x59, 0xf3, 0xae, 0xa2, 0x82, + 0x63, 0x01, 0x83, 0x2e, 0xd9, 0x51, 0x9b, 0x7c, 0xa6, 0xeb, 0xa5, 0xbe, 0x16, 0x0c, 0xe3, 0x61, + 0xc0, 0x8c, 0x3a, 0xf5, 0x73, 0x2c, 0x25, 0x0b, 0xbb, 0x4e, 0x89, 0x6b, 0x53, 0x6a, 0xb4, 0xf1, + 0xe1, 0xe6, 0xbd, 0x45, 0xe2, 0xf4, 0xb6, 0x66, 0xcc, 0x95, 0x03, 0x56, 0xd4, 0x1c, 0x1e, 0xd7, + 0xfb, 0xc3, 0x8e, 0xb5, 0xe9, 0xcf, 0xbf, 0xba, 0xea, 0x77, 0x39, 0xaf, 0x33, 0xc9, 0x62, 0x71, + 0x81, 0x79, 0x09, 0xad, 0x24, 0xcd, 0xf9, 0xd8, 0xe5, 0xc5, 0xb9, 0x4d, 0x44, 0x08, 0x86, 0xe7, + 0xa1, 0x1d, 0xaa, 0xed, 0x06, 0x70, 0xb2, 0xd2, 0x41, 0x7b, 0xa0, 0x11, 0x31, 0xc2, 0x27, 0x90, + 0x20, 0xf6, 0x60, 0xff, 0x96, 0x5c, 0xb1, 0xab, 0x9e, 0x9c, 0x52, 0x1b, 0x5f, 0x93, 0x0a, 0xef, + 0x91, 0x85, 0x49, 0xee, 0x2d, 0x4f, 0x8f, 0x3b, 0x47, 0x87, 0x6d, 0x46, 0xd6, 0x3e, 0x69, 0x64, + 0x2a, 0xce, 0xcb, 0x2f, 0xfc, 0x97, 0x05, 0x7a, 0xac, 0x7f, 0xd5, 0x1a, 0x4b, 0x0e, 0xa7, 0x5a, + 0x28, 0x14, 0x3f, 0x29, 0x88, 0x3c, 0x4c, 0x02, 0xb8, 0xda, 0xb0, 0x17, 0x55, 0x1f, 0x8a, 0x7d, + 0x57, 0xc7, 0x8d, 0x74, 0xb7, 0xc4, 0x9f, 0x72, 0x7e, 0x15, 0x22, 0x12, 0x58, 0x07, 0x99, 0x34, + 0x6e, 0x50, 0xde, 0x68, 0x65, 0xbc, 0xdb, 0xf8, 0xc8, 0xa8, 0x2b, 0x40, 0xdc, 0xfe, 0x32, 0xa4, + 0xca, 0x10, 0x21, 0xf0, 0xd3, 0x5d, 0x0f, 0x00, 0x6f, 0x9d, 0x36, 0x42, 0x4a, 0x5e, 0xc1, 0xe0, + }, + { + 0x75, 0xf3, 0xc6, 0xf4, 0xdb, 0x7b, 0xfb, 0xc8, 0x4a, 0xd3, 0xe6, 0x6b, 0x45, 0x7d, 0xe8, 0x4b, + 0xd6, 0x32, 0xd8, 0xfd, 0x37, 0x71, 0xf1, 0xe1, 0x30, 0x0f, 0xf8, 0x1b, 0x87, 0xfa, 0x06, 0x3f, + 0x5e, 0xba, 0xae, 0x5b, 0x8a, 0x00, 0xbc, 0x9d, 0x6d, 0xc1, 0xb1, 0x0e, 0x80, 0x5d, 0xd2, 0xd5, + 0xa0, 0x84, 0x07, 0x14, 0xb5, 0x90, 0x2c, 0xa3, 0xb2, 0x73, 0x4c, 0x54, 0x92, 0x74, 0x36, 0x51, + 0x38, 0xb0, 0xbd, 0x5a, 0xfc, 0x60, 0x62, 0x96, 0x6c, 0x42, 0xf7, 0x10, 0x7c, 0x28, 0x27, 0x8c, + 0x13, 0x95, 0x9c, 0xc7, 0x24, 0x46, 0x3b, 0x70, 0xca, 0xe3, 0x85, 0xcb, 0x11, 0xd0, 0x93, 0xb8, + 0xa6, 0x83, 0x20, 0xff, 0x9f, 0x77, 0xc3, 0xcc, 0x03, 0x6f, 0x08, 0xbf, 0x40, 0xe7, 0x2b, 0xe2, + 0x79, 0x0c, 0xaa, 0x82, 0x41, 0x3a, 0xea, 0xb9, 0xe4, 0x9a, 0xa4, 0x97, 0x7e, 0xda, 0x7a, 0x17, + 0x66, 0x94, 0xa1, 0x1d, 0x3d, 0xf0, 0xde, 0xb3, 0x0b, 0x72, 0xa7, 0x1c, 0xef, 0xd1, 0x53, 0x3e, + 0x8f, 0x33, 0x26, 0x5f, 0xec, 0x76, 0x2a, 0x49, 0x81, 0x88, 0xee, 0x21, 0xc4, 0x1a, 0xeb, 0xd9, + 0xc5, 0x39, 0x99, 0xcd, 0xad, 0x31, 0x8b, 0x01, 0x18, 0x23, 0xdd, 0x1f, 0x4e, 0x2d, 0xf9, 0x48, + 0x4f, 0xf2, 0x65, 0x8e, 0x78, 0x5c, 0x58, 0x19, 0x8d, 0xe5, 0x98, 0x57, 0x67, 0x7f, 0x05, 0x64, + 0xaf, 0x63, 0xb6, 0xfe, 0xf5, 0xb7, 0x3c, 0xa5, 0xce, 0xe9, 0x68, 0x44, 0xe0, 0x4d, 0x43, 0x69, + 0x29, 0x2e, 0xac, 0x15, 0x59, 0xa8, 0x0a, 0x9e, 0x6e, 0x47, 0xdf, 0x34, 0x35, 0x6a, 0xcf, 0xdc, + 0x22, 0xc9, 0xc0, 0x9b, 0x89, 0xd4, 0xed, 0xab, 0x12, 0xa2, 0x0d, 0x52, 0xbb, 0x02, 0x2f, 0xa9, + 0xd7, 0x61, 0x1e, 0xb4, 0x50, 0x04, 0xf6, 0xc2, 0x16, 0x25, 0x86, 0x56, 0x55, 0x09, 0xbe, 0x91, + }, +} + +// gfMult returns a·b in GF(2^8)/p +func gfMult(a, b byte, p uint32) byte { + B := [2]uint32{0, uint32(b)} + P := [2]uint32{0, p} + var result uint32 + + // branchless GF multiplier + for i := 0; i < 7; i++ { + result ^= B[a&1] + a >>= 1 + B[1] = P[B[1]>>7] ^ (B[1] << 1) + } + result ^= B[a&1] + return byte(result) +} + +// mdsColumnMult calculates y{col} where [y0 y1 y2 y3] = MDS · [x0] +func mdsColumnMult(in byte, col int) uint32 { + mul01 := in + mul5B := gfMult(in, 0x5B, mdsPolynomial) + mulEF := gfMult(in, 0xEF, mdsPolynomial) + + switch col { + case 0: + return uint32(mul01) | uint32(mul5B)<<8 | uint32(mulEF)<<16 | uint32(mulEF)<<24 + case 1: + return uint32(mulEF) | uint32(mulEF)<<8 | uint32(mul5B)<<16 | uint32(mul01)<<24 + case 2: + return uint32(mul5B) | uint32(mulEF)<<8 | uint32(mul01)<<16 | uint32(mulEF)<<24 + case 3: + return uint32(mul5B) | uint32(mul01)<<8 | uint32(mulEF)<<16 | uint32(mul5B)<<24 + } + + panic("unreachable") +} + +// h implements the S-box generation function. See [TWOFISH] 4.3.5 +func h(in, key []byte, offset int) uint32 { + var y [4]byte + for x := range y { + y[x] = in[x] + } + switch len(key) / 8 { + case 4: + y[0] = sbox[1][y[0]] ^ key[4*(6+offset)+0] + y[1] = sbox[0][y[1]] ^ key[4*(6+offset)+1] + y[2] = sbox[0][y[2]] ^ key[4*(6+offset)+2] + y[3] = sbox[1][y[3]] ^ key[4*(6+offset)+3] + fallthrough + case 3: + y[0] = sbox[1][y[0]] ^ key[4*(4+offset)+0] + y[1] = sbox[1][y[1]] ^ key[4*(4+offset)+1] + y[2] = sbox[0][y[2]] ^ key[4*(4+offset)+2] + y[3] = sbox[0][y[3]] ^ key[4*(4+offset)+3] + fallthrough + case 2: + y[0] = sbox[1][sbox[0][sbox[0][y[0]]^key[4*(2+offset)+0]]^key[4*(0+offset)+0]] + y[1] = sbox[0][sbox[0][sbox[1][y[1]]^key[4*(2+offset)+1]]^key[4*(0+offset)+1]] + y[2] = sbox[1][sbox[1][sbox[0][y[2]]^key[4*(2+offset)+2]]^key[4*(0+offset)+2]] + y[3] = sbox[0][sbox[1][sbox[1][y[3]]^key[4*(2+offset)+3]]^key[4*(0+offset)+3]] + } + // [y0 y1 y2 y3] = MDS . [x0 x1 x2 x3] + var mdsMult uint32 + for i := range y { + mdsMult ^= mdsColumnMult(y[i], i) + } + return mdsMult +} + +// Encrypt encrypts a 16-byte block from src to dst, which may overlap. +// Note that for amounts of data larger than a block, +// it is not safe to just call Encrypt on successive blocks; +// instead, use an encryption mode like CBC (see crypto/cipher/cbc.go). +func (c *Cipher) Encrypt(dst, src []byte) { + S1 := c.s[0] + S2 := c.s[1] + S3 := c.s[2] + S4 := c.s[3] + + // Load input + ia := load32l(src[0:4]) + ib := load32l(src[4:8]) + ic := load32l(src[8:12]) + id := load32l(src[12:16]) + + // Pre-whitening + ia ^= c.k[0] + ib ^= c.k[1] + ic ^= c.k[2] + id ^= c.k[3] + + for i := 0; i < 8; i++ { + k := c.k[8+i*4 : 12+i*4] + t2 := S2[byte(ib)] ^ S3[byte(ib>>8)] ^ S4[byte(ib>>16)] ^ S1[byte(ib>>24)] + t1 := S1[byte(ia)] ^ S2[byte(ia>>8)] ^ S3[byte(ia>>16)] ^ S4[byte(ia>>24)] + t2 + ic = ror(ic^(t1+k[0]), 1) + id = rol(id, 1) ^ (t2 + t1 + k[1]) + + t2 = S2[byte(id)] ^ S3[byte(id>>8)] ^ S4[byte(id>>16)] ^ S1[byte(id>>24)] + t1 = S1[byte(ic)] ^ S2[byte(ic>>8)] ^ S3[byte(ic>>16)] ^ S4[byte(ic>>24)] + t2 + ia = ror(ia^(t1+k[2]), 1) + ib = rol(ib, 1) ^ (t2 + t1 + k[3]) + } + + // Output with "undo last swap" + ta := ic ^ c.k[4] + tb := id ^ c.k[5] + tc := ia ^ c.k[6] + td := ib ^ c.k[7] + + store32l(dst[0:4], ta) + store32l(dst[4:8], tb) + store32l(dst[8:12], tc) + store32l(dst[12:16], td) +} + +// Decrypt decrypts a 16-byte block from src to dst, which may overlap. +func (c *Cipher) Decrypt(dst, src []byte) { + S1 := c.s[0] + S2 := c.s[1] + S3 := c.s[2] + S4 := c.s[3] + + // Load input + ta := load32l(src[0:4]) + tb := load32l(src[4:8]) + tc := load32l(src[8:12]) + td := load32l(src[12:16]) + + // Undo undo final swap + ia := tc ^ c.k[6] + ib := td ^ c.k[7] + ic := ta ^ c.k[4] + id := tb ^ c.k[5] + + for i := 8; i > 0; i-- { + k := c.k[4+i*4 : 8+i*4] + t2 := S2[byte(id)] ^ S3[byte(id>>8)] ^ S4[byte(id>>16)] ^ S1[byte(id>>24)] + t1 := S1[byte(ic)] ^ S2[byte(ic>>8)] ^ S3[byte(ic>>16)] ^ S4[byte(ic>>24)] + t2 + ia = rol(ia, 1) ^ (t1 + k[2]) + ib = ror(ib^(t2+t1+k[3]), 1) + + t2 = S2[byte(ib)] ^ S3[byte(ib>>8)] ^ S4[byte(ib>>16)] ^ S1[byte(ib>>24)] + t1 = S1[byte(ia)] ^ S2[byte(ia>>8)] ^ S3[byte(ia>>16)] ^ S4[byte(ia>>24)] + t2 + ic = rol(ic, 1) ^ (t1 + k[0]) + id = ror(id^(t2+t1+k[1]), 1) + } + + // Undo pre-whitening + ia ^= c.k[0] + ib ^= c.k[1] + ic ^= c.k[2] + id ^= c.k[3] + + store32l(dst[0:4], ia) + store32l(dst[4:8], ib) + store32l(dst[8:12], ic) + store32l(dst[12:16], id) +} diff --git a/src/pkg/crypto/twofish/twofish_test.go b/src/pkg/crypto/twofish/twofish_test.go new file mode 100644 index 000000000..303081f3f --- /dev/null +++ b/src/pkg/crypto/twofish/twofish_test.go @@ -0,0 +1,129 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package twofish + +import ( + "bytes" + "testing" +) + +var qbox = [2][4][16]byte{ + { + {0x8, 0x1, 0x7, 0xD, 0x6, 0xF, 0x3, 0x2, 0x0, 0xB, 0x5, 0x9, 0xE, 0xC, 0xA, 0x4}, + {0xE, 0xC, 0xB, 0x8, 0x1, 0x2, 0x3, 0x5, 0xF, 0x4, 0xA, 0x6, 0x7, 0x0, 0x9, 0xD}, + {0xB, 0xA, 0x5, 0xE, 0x6, 0xD, 0x9, 0x0, 0xC, 0x8, 0xF, 0x3, 0x2, 0x4, 0x7, 0x1}, + {0xD, 0x7, 0xF, 0x4, 0x1, 0x2, 0x6, 0xE, 0x9, 0xB, 0x3, 0x0, 0x8, 0x5, 0xC, 0xA}, + }, + { + {0x2, 0x8, 0xB, 0xD, 0xF, 0x7, 0x6, 0xE, 0x3, 0x1, 0x9, 0x4, 0x0, 0xA, 0xC, 0x5}, + {0x1, 0xE, 0x2, 0xB, 0x4, 0xC, 0x3, 0x7, 0x6, 0xD, 0xA, 0x5, 0xF, 0x9, 0x0, 0x8}, + {0x4, 0xC, 0x7, 0x5, 0x1, 0x6, 0x9, 0xA, 0x0, 0xE, 0xD, 0x8, 0x2, 0xB, 0x3, 0xF}, + {0xB, 0x9, 0x5, 0x1, 0xC, 0x3, 0xD, 0xE, 0x6, 0x4, 0x7, 0xF, 0x2, 0x0, 0x8, 0xA}, + }, +} + +// genSbox generates the variable sbox +func genSbox(qi int, x byte) byte { + a0, b0 := x/16, x%16 + for i := 0; i < 2; i++ { + a1 := a0 ^ b0 + b1 := (a0 ^ ((b0 << 3) | (b0 >> 1)) ^ (a0 << 3)) & 15 + a0 = qbox[qi][2*i][a1] + b0 = qbox[qi][2*i+1][b1] + } + return (b0 << 4) + a0 +} + +func TestSbox(t *testing.T) { + for n := range sbox { + for m := range sbox[n] { + if genSbox(n, byte(m)) != sbox[n][m] { + t.Errorf("#%d|%d: sbox value = %d want %d", n, m, sbox[n][m], genSbox(n, byte(m))) + } + } + } +} + +var testVectors = []struct { + key []byte + dec []byte + enc []byte +}{ + // These tests are extracted from LibTom + { + []byte{0x9F, 0x58, 0x9F, 0x5C, 0xF6, 0x12, 0x2C, 0x32, 0xB6, 0xBF, 0xEC, 0x2F, 0x2A, 0xE8, 0xC3, 0x5A}, + []byte{0xD4, 0x91, 0xDB, 0x16, 0xE7, 0xB1, 0xC3, 0x9E, 0x86, 0xCB, 0x08, 0x6B, 0x78, 0x9F, 0x54, 0x19}, + []byte{0x01, 0x9F, 0x98, 0x09, 0xDE, 0x17, 0x11, 0x85, 0x8F, 0xAA, 0xC3, 0xA3, 0xBA, 0x20, 0xFB, 0xC3}, + }, + { + []byte{0x88, 0xB2, 0xB2, 0x70, 0x6B, 0x10, 0x5E, 0x36, 0xB4, 0x46, 0xBB, 0x6D, 0x73, 0x1A, 0x1E, 0x88, + 0xEF, 0xA7, 0x1F, 0x78, 0x89, 0x65, 0xBD, 0x44}, + []byte{0x39, 0xDA, 0x69, 0xD6, 0xBA, 0x49, 0x97, 0xD5, 0x85, 0xB6, 0xDC, 0x07, 0x3C, 0xA3, 0x41, 0xB2}, + []byte{0x18, 0x2B, 0x02, 0xD8, 0x14, 0x97, 0xEA, 0x45, 0xF9, 0xDA, 0xAC, 0xDC, 0x29, 0x19, 0x3A, 0x65}, + }, + { + []byte{0xD4, 0x3B, 0xB7, 0x55, 0x6E, 0xA3, 0x2E, 0x46, 0xF2, 0xA2, 0x82, 0xB7, 0xD4, 0x5B, 0x4E, 0x0D, + 0x57, 0xFF, 0x73, 0x9D, 0x4D, 0xC9, 0x2C, 0x1B, 0xD7, 0xFC, 0x01, 0x70, 0x0C, 0xC8, 0x21, 0x6F}, + []byte{0x90, 0xAF, 0xE9, 0x1B, 0xB2, 0x88, 0x54, 0x4F, 0x2C, 0x32, 0xDC, 0x23, 0x9B, 0x26, 0x35, 0xE6}, + []byte{0x6C, 0xB4, 0x56, 0x1C, 0x40, 0xBF, 0x0A, 0x97, 0x05, 0x93, 0x1C, 0xB6, 0xD4, 0x08, 0xE7, 0xFA}, + }, + // These test are derived from http://www.schneier.com/code/ecb_ival.txt + { + []byte{0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00}, + []byte{0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00}, + []byte{0x9F, 0x58, 0x9F, 0x5C, 0xF6, 0x12, 0x2C, 0x32, 0xB6, 0xBF, 0xEC, 0x2F, 0x2A, 0xE8, 0xC3, 0x5A}, + }, + { + []byte{0x01, 0x23, 0x45, 0x67, 0x89, 0xAB, 0xCD, 0xEF, 0xFE, 0xDC, 0xBA, 0x98, 0x76, 0x54, 0x32, 0x10, + 0x00, 0x11, 0x22, 0x33, 0x44, 0x55, 0x66, 0x77, + }, + []byte{0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00}, + []byte{0xCF, 0xD1, 0xD2, 0xE5, 0xA9, 0xBE, 0x9C, 0xDF, 0x50, 0x1F, 0x13, 0xB8, 0x92, 0xBD, 0x22, 0x48}, + }, + { + []byte{0x01, 0x23, 0x45, 0x67, 0x89, 0xAB, 0xCD, 0xEF, 0xFE, 0xDC, 0xBA, 0x98, 0x76, 0x54, 0x32, 0x10, + 0x00, 0x11, 0x22, 0x33, 0x44, 0x55, 0x66, 0x77, 0x88, 0x99, 0xAA, 0xBB, 0xCC, 0xDD, 0xEE, 0xFF, + }, + []byte{0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00}, + []byte{0x37, 0x52, 0x7B, 0xE0, 0x05, 0x23, 0x34, 0xB8, 0x9F, 0x0C, 0xFC, 0xCA, 0xE8, 0x7C, 0xFA, 0x20}, + }, +} + +func TestCipher(t *testing.T) { + for n, tt := range testVectors { + // Test if the plaintext (dec) is encrypts to the given + // ciphertext (enc) using the given key. Test also if enc can + // be decrypted again into dec. + c, err := NewCipher(tt.key) + if err != nil { + t.Errorf("#%d: NewCipher: %v", n, err) + return + } + + buf := make([]byte, 16) + c.Encrypt(buf, tt.dec) + if !bytes.Equal(buf, tt.enc) { + t.Errorf("#%d: encrypt = %x want %x", n, buf, tt.enc) + } + c.Decrypt(buf, tt.enc) + if !bytes.Equal(buf, tt.dec) { + t.Errorf("#%d: decrypt = %x want %x", n, buf, tt.dec) + } + + // Test that 16 zero bytes, encrypted 1000 times then decrypted + // 1000 times results in zero bytes again. + zero := make([]byte, 16) + buf = make([]byte, 16) + for i := 0; i < 1000; i++ { + c.Encrypt(buf, buf) + } + for i := 0; i < 1000; i++ { + c.Decrypt(buf, buf) + } + if !bytes.Equal(buf, zero) { + t.Errorf("#%d: encrypt/decrypt 1000: have %x want %x", n, buf, zero) + } + } +} diff --git a/src/pkg/crypto/x509/Makefile b/src/pkg/crypto/x509/Makefile new file mode 100644 index 000000000..14ffd095f --- /dev/null +++ b/src/pkg/crypto/x509/Makefile @@ -0,0 +1,13 @@ +# Copyright 2009 The Go Authors. All rights reserved. +# Use of this source code is governed by a BSD-style +# license that can be found in the LICENSE file. + +include ../../../Make.inc + +TARG=crypto/x509 +GOFILES=\ + cert_pool.go\ + verify.go\ + x509.go\ + +include ../../../Make.pkg diff --git a/src/pkg/crypto/x509/cert_pool.go b/src/pkg/crypto/x509/cert_pool.go new file mode 100644 index 000000000..16cd92efc --- /dev/null +++ b/src/pkg/crypto/x509/cert_pool.go @@ -0,0 +1,106 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package x509 + +import ( + "crypto/x509/pkix" + "encoding/pem" + "strings" +) + +// Roots is a set of certificates. +type CertPool struct { + bySubjectKeyId map[string][]int + byName map[string][]int + certs []*Certificate +} + +// NewCertPool returns a new, empty CertPool. +func NewCertPool() *CertPool { + return &CertPool{ + make(map[string][]int), + make(map[string][]int), + nil, + } +} + +func nameToKey(name *pkix.Name) string { + return strings.Join(name.Country, ",") + "/" + strings.Join(name.Organization, ",") + "/" + strings.Join(name.OrganizationalUnit, ",") + "/" + name.CommonName +} + +// findVerifiedParents attempts to find certificates in s which have signed the +// given certificate. If no such certificate can be found or the signature +// doesn't match, it returns nil. +func (s *CertPool) findVerifiedParents(cert *Certificate) (parents []int) { + var candidates []int + + if len(cert.AuthorityKeyId) > 0 { + candidates = s.bySubjectKeyId[string(cert.AuthorityKeyId)] + } + if len(candidates) == 0 { + candidates = s.byName[nameToKey(&cert.Issuer)] + } + + for _, c := range candidates { + if cert.CheckSignatureFrom(s.certs[c]) == nil { + parents = append(parents, c) + } + } + + return +} + +// AddCert adds a certificate to a pool. +func (s *CertPool) AddCert(cert *Certificate) { + if cert == nil { + panic("adding nil Certificate to CertPool") + } + + // Check that the certificate isn't being added twice. + for _, c := range s.certs { + if c.Equal(cert) { + return + } + } + + n := len(s.certs) + s.certs = append(s.certs, cert) + + if len(cert.SubjectKeyId) > 0 { + keyId := string(cert.SubjectKeyId) + s.bySubjectKeyId[keyId] = append(s.bySubjectKeyId[keyId], n) + } + name := nameToKey(&cert.Subject) + s.byName[name] = append(s.byName[name], n) +} + +// AppendCertsFromPEM attempts to parse a series of PEM encoded root +// certificates. It appends any certificates found to s and returns true if any +// certificates were successfully parsed. +// +// On many Linux systems, /etc/ssl/cert.pem will contains the system wide set +// of root CAs in a format suitable for this function. +func (s *CertPool) AppendCertsFromPEM(pemCerts []byte) (ok bool) { + for len(pemCerts) > 0 { + var block *pem.Block + block, pemCerts = pem.Decode(pemCerts) + if block == nil { + break + } + if block.Type != "CERTIFICATE" || len(block.Headers) != 0 { + continue + } + + cert, err := ParseCertificate(block.Bytes) + if err != nil { + continue + } + + s.AddCert(cert) + ok = true + } + + return +} diff --git a/src/pkg/crypto/x509/pkix/Makefile b/src/pkg/crypto/x509/pkix/Makefile new file mode 100644 index 000000000..e29b74c01 --- /dev/null +++ b/src/pkg/crypto/x509/pkix/Makefile @@ -0,0 +1,11 @@ +# Copyright 2011 The Go Authors. All rights reserved. +# Use of this source code is governed by a BSD-style +# license that can be found in the LICENSE file. + +include ../../../../Make.inc + +TARG=crypto/x509/pkix +GOFILES=\ + pkix.go\ + +include ../../../../Make.pkg diff --git a/src/pkg/crypto/x509/pkix/pkix.go b/src/pkg/crypto/x509/pkix/pkix.go new file mode 100644 index 000000000..266fd557a --- /dev/null +++ b/src/pkg/crypto/x509/pkix/pkix.go @@ -0,0 +1,167 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Package pkix contains shared, low level structures used for ASN.1 parsing +// and serialization of X.509 certificates, CRL and OCSP. +package pkix + +import ( + "asn1" + "big" + "time" +) + +// AlgorithmIdentifier represents the ASN.1 structure of the same name. See RFC +// 5280, section 4.1.1.2. +type AlgorithmIdentifier struct { + Algorithm asn1.ObjectIdentifier + Parameters asn1.RawValue `asn1:"optional"` +} + +type RDNSequence []RelativeDistinguishedNameSET + +type RelativeDistinguishedNameSET []AttributeTypeAndValue + +type AttributeTypeAndValue struct { + Type asn1.ObjectIdentifier + Value interface{} +} + +// Extension represents the ASN.1 structure of the same name. See RFC +// 5280, section 4.2. +type Extension struct { + Id asn1.ObjectIdentifier + Critical bool `asn1:"optional"` + Value []byte +} + +// Name represents an X.509 distinguished name. This only includes the common +// elements of a DN. Additional elements in the name are ignored. +type Name struct { + Country, Organization, OrganizationalUnit []string + Locality, Province []string + StreetAddress, PostalCode []string + SerialNumber, CommonName string +} + +func (n *Name) FillFromRDNSequence(rdns *RDNSequence) { + for _, rdn := range *rdns { + if len(rdn) == 0 { + continue + } + atv := rdn[0] + value, ok := atv.Value.(string) + if !ok { + continue + } + + t := atv.Type + if len(t) == 4 && t[0] == 2 && t[1] == 5 && t[2] == 4 { + switch t[3] { + case 3: + n.CommonName = value + case 5: + n.SerialNumber = value + case 6: + n.Country = append(n.Country, value) + case 7: + n.Locality = append(n.Locality, value) + case 8: + n.Province = append(n.Province, value) + case 9: + n.StreetAddress = append(n.StreetAddress, value) + case 10: + n.Organization = append(n.Organization, value) + case 11: + n.OrganizationalUnit = append(n.OrganizationalUnit, value) + case 17: + n.PostalCode = append(n.PostalCode, value) + } + } + } +} + +var ( + oidCountry = []int{2, 5, 4, 6} + oidOrganization = []int{2, 5, 4, 10} + oidOrganizationalUnit = []int{2, 5, 4, 11} + oidCommonName = []int{2, 5, 4, 3} + oidSerialNumber = []int{2, 5, 4, 5} + oidLocality = []int{2, 5, 4, 7} + oidProvince = []int{2, 5, 4, 8} + oidStreetAddress = []int{2, 5, 4, 9} + oidPostalCode = []int{2, 5, 4, 17} +) + +// appendRDNs appends a relativeDistinguishedNameSET to the given RDNSequence +// and returns the new value. The relativeDistinguishedNameSET contains an +// attributeTypeAndValue for each of the given values. See RFC 5280, A.1, and +// search for AttributeTypeAndValue. +func appendRDNs(in RDNSequence, values []string, oid asn1.ObjectIdentifier) RDNSequence { + if len(values) == 0 { + return in + } + + s := make([]AttributeTypeAndValue, len(values)) + for i, value := range values { + s[i].Type = oid + s[i].Value = value + } + + return append(in, s) +} + +func (n Name) ToRDNSequence() (ret RDNSequence) { + ret = appendRDNs(ret, n.Country, oidCountry) + ret = appendRDNs(ret, n.Organization, oidOrganization) + ret = appendRDNs(ret, n.OrganizationalUnit, oidOrganizationalUnit) + ret = appendRDNs(ret, n.Locality, oidLocality) + ret = appendRDNs(ret, n.Province, oidProvince) + ret = appendRDNs(ret, n.StreetAddress, oidStreetAddress) + ret = appendRDNs(ret, n.PostalCode, oidPostalCode) + if len(n.CommonName) > 0 { + ret = appendRDNs(ret, []string{n.CommonName}, oidCommonName) + } + if len(n.SerialNumber) > 0 { + ret = appendRDNs(ret, []string{n.SerialNumber}, oidSerialNumber) + } + + return ret +} + +// CertificateList represents the ASN.1 structure of the same name. See RFC +// 5280, section 5.1. Use Certificate.CheckCRLSignature to verify the +// signature. +type CertificateList struct { + TBSCertList TBSCertificateList + SignatureAlgorithm AlgorithmIdentifier + SignatureValue asn1.BitString +} + +// HasExpired returns true iff currentTimeSeconds is past the expiry time of +// certList. +func (certList *CertificateList) HasExpired(currentTimeSeconds int64) bool { + return certList.TBSCertList.NextUpdate.Seconds() <= currentTimeSeconds +} + +// TBSCertificateList represents the ASN.1 structure of the same name. See RFC +// 5280, section 5.1. +type TBSCertificateList struct { + Raw asn1.RawContent + Version int `asn1:"optional,default:2"` + Signature AlgorithmIdentifier + Issuer RDNSequence + ThisUpdate *time.Time + NextUpdate *time.Time + RevokedCertificates []RevokedCertificate `asn1:"optional"` + Extensions []Extension `asn1:"tag:0,optional,explicit"` +} + +// RevokedCertificate represents the ASN.1 structure of the same name. See RFC +// 5280, section 5.1. +type RevokedCertificate struct { + SerialNumber *big.Int + RevocationTime *time.Time + Extensions []Extension `asn1:"optional"` +} diff --git a/src/pkg/crypto/x509/verify.go b/src/pkg/crypto/x509/verify.go new file mode 100644 index 000000000..4c0fecccb --- /dev/null +++ b/src/pkg/crypto/x509/verify.go @@ -0,0 +1,244 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package x509 + +import ( + "os" + "strings" + "time" +) + +type InvalidReason int + +const ( + // NotAuthorizedToSign results when a certificate is signed by another + // which isn't marked as a CA certificate. + NotAuthorizedToSign InvalidReason = iota + // Expired results when a certificate has expired, based on the time + // given in the VerifyOptions. + Expired + // CANotAuthorizedForThisName results when an intermediate or root + // certificate has a name constraint which doesn't include the name + // being checked. + CANotAuthorizedForThisName +) + +// CertificateInvalidError results when an odd error occurs. Users of this +// library probably want to handle all these errors uniformly. +type CertificateInvalidError struct { + Cert *Certificate + Reason InvalidReason +} + +func (e CertificateInvalidError) String() string { + switch e.Reason { + case NotAuthorizedToSign: + return "x509: certificate is not authorized to sign other other certificates" + case Expired: + return "x509: certificate has expired or is not yet valid" + case CANotAuthorizedForThisName: + return "x509: a root or intermediate certificate is not authorized to sign in this domain" + } + return "x509: unknown error" +} + +// HostnameError results when the set of authorized names doesn't match the +// requested name. +type HostnameError struct { + Certificate *Certificate + Host string +} + +func (h HostnameError) String() string { + var valid string + c := h.Certificate + if len(c.DNSNames) > 0 { + valid = strings.Join(c.DNSNames, ", ") + } else { + valid = c.Subject.CommonName + } + return "certificate is valid for " + valid + ", not " + h.Host +} + +// UnknownAuthorityError results when the certificate issuer is unknown +type UnknownAuthorityError struct { + cert *Certificate +} + +func (e UnknownAuthorityError) String() string { + return "x509: certificate signed by unknown authority" +} + +// VerifyOptions contains parameters for Certificate.Verify. It's a structure +// because other PKIX verification APIs have ended up needing many options. +type VerifyOptions struct { + DNSName string + Intermediates *CertPool + Roots *CertPool + CurrentTime int64 // if 0, the current system time is used. +} + +const ( + leafCertificate = iota + intermediateCertificate + rootCertificate +) + +// isValid performs validity checks on the c. +func (c *Certificate) isValid(certType int, opts *VerifyOptions) os.Error { + if opts.CurrentTime < c.NotBefore.Seconds() || + opts.CurrentTime > c.NotAfter.Seconds() { + return CertificateInvalidError{c, Expired} + } + + if len(c.PermittedDNSDomains) > 0 { + for _, domain := range c.PermittedDNSDomains { + if opts.DNSName == domain || + (strings.HasSuffix(opts.DNSName, domain) && + len(opts.DNSName) >= 1+len(domain) && + opts.DNSName[len(opts.DNSName)-len(domain)-1] == '.') { + continue + } + + return CertificateInvalidError{c, CANotAuthorizedForThisName} + } + } + + // KeyUsage status flags are ignored. From Engineering Security, Peter + // Gutmann: A European government CA marked its signing certificates as + // being valid for encryption only, but no-one noticed. Another + // European CA marked its signature keys as not being valid for + // signatures. A different CA marked its own trusted root certificate + // as being invalid for certificate signing. Another national CA + // distributed a certificate to be used to encrypt data for the + // country’s tax authority that was marked as only being usable for + // digital signatures but not for encryption. Yet another CA reversed + // the order of the bit flags in the keyUsage due to confusion over + // encoding endianness, essentially setting a random keyUsage in + // certificates that it issued. Another CA created a self-invalidating + // certificate by adding a certificate policy statement stipulating + // that the certificate had to be used strictly as specified in the + // keyUsage, and a keyUsage containing a flag indicating that the RSA + // encryption key could only be used for Diffie-Hellman key agreement. + + if certType == intermediateCertificate && (!c.BasicConstraintsValid || !c.IsCA) { + return CertificateInvalidError{c, NotAuthorizedToSign} + } + + return nil +} + +// Verify attempts to verify c by building one or more chains from c to a +// certificate in opts.roots, using certificates in opts.Intermediates if +// needed. If successful, it returns one or chains where the first element of +// the chain is c and the last element is from opts.Roots. +// +// WARNING: this doesn't do any revocation checking. +func (c *Certificate) Verify(opts VerifyOptions) (chains [][]*Certificate, err os.Error) { + if opts.CurrentTime == 0 { + opts.CurrentTime = time.Seconds() + } + err = c.isValid(leafCertificate, &opts) + if err != nil { + return + } + if len(opts.DNSName) > 0 { + err = c.VerifyHostname(opts.DNSName) + if err != nil { + return + } + } + return c.buildChains(make(map[int][][]*Certificate), []*Certificate{c}, &opts) +} + +func appendToFreshChain(chain []*Certificate, cert *Certificate) []*Certificate { + n := make([]*Certificate, len(chain)+1) + copy(n, chain) + n[len(chain)] = cert + return n +} + +func (c *Certificate) buildChains(cache map[int][][]*Certificate, currentChain []*Certificate, opts *VerifyOptions) (chains [][]*Certificate, err os.Error) { + for _, rootNum := range opts.Roots.findVerifiedParents(c) { + root := opts.Roots.certs[rootNum] + err = root.isValid(rootCertificate, opts) + if err != nil { + continue + } + chains = append(chains, appendToFreshChain(currentChain, root)) + } + +nextIntermediate: + for _, intermediateNum := range opts.Intermediates.findVerifiedParents(c) { + intermediate := opts.Intermediates.certs[intermediateNum] + for _, cert := range currentChain { + if cert == intermediate { + continue nextIntermediate + } + } + err = intermediate.isValid(intermediateCertificate, opts) + if err != nil { + continue + } + var childChains [][]*Certificate + childChains, ok := cache[intermediateNum] + if !ok { + childChains, err = intermediate.buildChains(cache, appendToFreshChain(currentChain, intermediate), opts) + cache[intermediateNum] = childChains + } + chains = append(chains, childChains...) + } + + if len(chains) > 0 { + err = nil + } + + if len(chains) == 0 && err == nil { + err = UnknownAuthorityError{c} + } + + return +} + +func matchHostnames(pattern, host string) bool { + if len(pattern) == 0 || len(host) == 0 { + return false + } + + patternParts := strings.Split(pattern, ".") + hostParts := strings.Split(host, ".") + + if len(patternParts) != len(hostParts) { + return false + } + + for i, patternPart := range patternParts { + if patternPart == "*" { + continue + } + if patternPart != hostParts[i] { + return false + } + } + + return true +} + +// VerifyHostname returns nil if c is a valid certificate for the named host. +// Otherwise it returns an os.Error describing the mismatch. +func (c *Certificate) VerifyHostname(h string) os.Error { + if len(c.DNSNames) > 0 { + for _, match := range c.DNSNames { + if matchHostnames(match, h) { + return nil + } + } + // If Subject Alt Name is given, we ignore the common name. + } else if matchHostnames(c.Subject.CommonName, h) { + return nil + } + + return HostnameError{c, h} +} diff --git a/src/pkg/crypto/x509/verify_test.go b/src/pkg/crypto/x509/verify_test.go new file mode 100644 index 000000000..111f60eb1 --- /dev/null +++ b/src/pkg/crypto/x509/verify_test.go @@ -0,0 +1,391 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package x509 + +import ( + "encoding/pem" + "os" + "strings" + "testing" +) + +type verifyTest struct { + leaf string + intermediates []string + roots []string + currentTime int64 + dnsName string + + errorCallback func(*testing.T, int, os.Error) bool + expectedChains [][]string +} + +var verifyTests = []verifyTest{ + { + leaf: googleLeaf, + intermediates: []string{thawteIntermediate}, + roots: []string{verisignRoot}, + currentTime: 1302726541, + dnsName: "www.google.com", + + expectedChains: [][]string{ + []string{"Google", "Thawte", "VeriSign"}, + }, + }, + { + leaf: googleLeaf, + intermediates: []string{thawteIntermediate}, + roots: []string{verisignRoot}, + currentTime: 1302726541, + dnsName: "www.example.com", + + errorCallback: expectHostnameError, + }, + { + leaf: googleLeaf, + intermediates: []string{thawteIntermediate}, + roots: []string{verisignRoot}, + currentTime: 1, + dnsName: "www.example.com", + + errorCallback: expectExpired, + }, + { + leaf: googleLeaf, + roots: []string{verisignRoot}, + currentTime: 1302726541, + dnsName: "www.google.com", + + errorCallback: expectAuthorityUnknown, + }, + { + leaf: googleLeaf, + intermediates: []string{verisignRoot, thawteIntermediate}, + roots: []string{verisignRoot}, + currentTime: 1302726541, + dnsName: "www.google.com", + + expectedChains: [][]string{ + []string{"Google", "Thawte", "VeriSign"}, + }, + }, + { + leaf: dnssecExpLeaf, + intermediates: []string{startComIntermediate}, + roots: []string{startComRoot}, + currentTime: 1302726541, + + expectedChains: [][]string{ + []string{"dnssec-exp", "StartCom Class 1", "StartCom Certification Authority"}, + }, + }, + { + leaf: dnssecExpLeaf, + intermediates: []string{startComIntermediate, startComRoot}, + roots: []string{startComRoot}, + currentTime: 1302726541, + + expectedChains: [][]string{ + []string{"dnssec-exp", "StartCom Class 1", "StartCom Certification Authority"}, + []string{"dnssec-exp", "StartCom Class 1", "StartCom Certification Authority", "StartCom Certification Authority"}, + }, + }, +} + +func expectHostnameError(t *testing.T, i int, err os.Error) (ok bool) { + if _, ok := err.(HostnameError); !ok { + t.Errorf("#%d: error was not a HostnameError: %s", i, err) + return false + } + return true +} + +func expectExpired(t *testing.T, i int, err os.Error) (ok bool) { + if inval, ok := err.(CertificateInvalidError); !ok || inval.Reason != Expired { + t.Errorf("#%d: error was not Expired: %s", i, err) + return false + } + return true +} + +func expectAuthorityUnknown(t *testing.T, i int, err os.Error) (ok bool) { + if _, ok := err.(UnknownAuthorityError); !ok { + t.Errorf("#%d: error was not UnknownAuthorityError: %s", i, err) + return false + } + return true +} + +func certificateFromPEM(pemBytes string) (*Certificate, os.Error) { + block, _ := pem.Decode([]byte(pemBytes)) + if block == nil { + return nil, os.NewError("failed to decode PEM") + } + return ParseCertificate(block.Bytes) +} + +func TestVerify(t *testing.T) { + for i, test := range verifyTests { + opts := VerifyOptions{ + Roots: NewCertPool(), + Intermediates: NewCertPool(), + DNSName: test.dnsName, + CurrentTime: test.currentTime, + } + + for j, root := range test.roots { + ok := opts.Roots.AppendCertsFromPEM([]byte(root)) + if !ok { + t.Errorf("#%d: failed to parse root #%d", i, j) + return + } + } + + for j, intermediate := range test.intermediates { + ok := opts.Intermediates.AppendCertsFromPEM([]byte(intermediate)) + if !ok { + t.Errorf("#%d: failed to parse intermediate #%d", i, j) + return + } + } + + leaf, err := certificateFromPEM(test.leaf) + if err != nil { + t.Errorf("#%d: failed to parse leaf: %s", i, err) + return + } + + chains, err := leaf.Verify(opts) + + if test.errorCallback == nil && err != nil { + t.Errorf("#%d: unexpected error: %s", i, err) + } + if test.errorCallback != nil { + if !test.errorCallback(t, i, err) { + return + } + } + + if len(chains) != len(test.expectedChains) { + t.Errorf("#%d: wanted %d chains, got %d", i, len(test.expectedChains), len(chains)) + } + + // We check that each returned chain matches a chain from + // expectedChains but an entry in expectedChains can't match + // two chains. + seenChains := make([]bool, len(chains)) + NextOutputChain: + for _, chain := range chains { + TryNextExpected: + for j, expectedChain := range test.expectedChains { + if seenChains[j] { + continue + } + if len(chain) != len(expectedChain) { + continue + } + for k, cert := range chain { + if strings.Index(nameToKey(&cert.Subject), expectedChain[k]) == -1 { + continue TryNextExpected + } + } + // we matched + seenChains[j] = true + continue NextOutputChain + } + t.Errorf("#%d: No expected chain matched %s", i, chainToDebugString(chain)) + } + } +} + +func chainToDebugString(chain []*Certificate) string { + var chainStr string + for _, cert := range chain { + if len(chainStr) > 0 { + chainStr += " -> " + } + chainStr += nameToKey(&cert.Subject) + } + return chainStr +} + +const verisignRoot = `-----BEGIN CERTIFICATE----- +MIICPDCCAaUCEHC65B0Q2Sk0tjjKewPMur8wDQYJKoZIhvcNAQECBQAwXzELMAkG +A1UEBhMCVVMxFzAVBgNVBAoTDlZlcmlTaWduLCBJbmMuMTcwNQYDVQQLEy5DbGFz +cyAzIFB1YmxpYyBQcmltYXJ5IENlcnRpZmljYXRpb24gQXV0aG9yaXR5MB4XDTk2 +MDEyOTAwMDAwMFoXDTI4MDgwMTIzNTk1OVowXzELMAkGA1UEBhMCVVMxFzAVBgNV +BAoTDlZlcmlTaWduLCBJbmMuMTcwNQYDVQQLEy5DbGFzcyAzIFB1YmxpYyBQcmlt +YXJ5IENlcnRpZmljYXRpb24gQXV0aG9yaXR5MIGfMA0GCSqGSIb3DQEBAQUAA4GN +ADCBiQKBgQDJXFme8huKARS0EN8EQNvjV69qRUCPhAwL0TPZ2RHP7gJYHyX3KqhE +BarsAx94f56TuZoAqiN91qyFomNFx3InzPRMxnVx0jnvT0Lwdd8KkMaOIG+YD/is +I19wKTakyYbnsZogy1Olhec9vn2a/iRFM9x2Fe0PonFkTGUugWhFpwIDAQABMA0G +CSqGSIb3DQEBAgUAA4GBALtMEivPLCYATxQT3ab7/AoRhIzzKBxnki98tsX63/Do +lbwdj2wsqFHMc9ikwFPwTtYmwHYBV4GSXiHx0bH/59AhWM1pF+NEHJwZRDmJXNyc +AA9WjQKZ7aKQRUzkuxCkPfAyAw7xzvjoyVGM5mKf5p/AfbdynMk2OmufTqj/ZA1k +-----END CERTIFICATE----- +` + +const thawteIntermediate = `-----BEGIN CERTIFICATE----- +MIIDIzCCAoygAwIBAgIEMAAAAjANBgkqhkiG9w0BAQUFADBfMQswCQYDVQQGEwJV +UzEXMBUGA1UEChMOVmVyaVNpZ24sIEluYy4xNzA1BgNVBAsTLkNsYXNzIDMgUHVi +bGljIFByaW1hcnkgQ2VydGlmaWNhdGlvbiBBdXRob3JpdHkwHhcNMDQwNTEzMDAw +MDAwWhcNMTQwNTEyMjM1OTU5WjBMMQswCQYDVQQGEwJaQTElMCMGA1UEChMcVGhh +d3RlIENvbnN1bHRpbmcgKFB0eSkgTHRkLjEWMBQGA1UEAxMNVGhhd3RlIFNHQyBD +QTCBnzANBgkqhkiG9w0BAQEFAAOBjQAwgYkCgYEA1NNn0I0Vf67NMf59HZGhPwtx +PKzMyGT7Y/wySweUvW+Aui/hBJPAM/wJMyPpC3QrccQDxtLN4i/1CWPN/0ilAL/g +5/OIty0y3pg25gqtAHvEZEo7hHUD8nCSfQ5i9SGraTaEMXWQ+L/HbIgbBpV8yeWo +3nWhLHpo39XKHIdYYBkCAwEAAaOB/jCB+zASBgNVHRMBAf8ECDAGAQH/AgEAMAsG +A1UdDwQEAwIBBjARBglghkgBhvhCAQEEBAMCAQYwKAYDVR0RBCEwH6QdMBsxGTAX +BgNVBAMTEFByaXZhdGVMYWJlbDMtMTUwMQYDVR0fBCowKDAmoCSgIoYgaHR0cDov +L2NybC52ZXJpc2lnbi5jb20vcGNhMy5jcmwwMgYIKwYBBQUHAQEEJjAkMCIGCCsG +AQUFBzABhhZodHRwOi8vb2NzcC50aGF3dGUuY29tMDQGA1UdJQQtMCsGCCsGAQUF +BwMBBggrBgEFBQcDAgYJYIZIAYb4QgQBBgpghkgBhvhFAQgBMA0GCSqGSIb3DQEB +BQUAA4GBAFWsY+reod3SkF+fC852vhNRj5PZBSvIG3dLrWlQoe7e3P3bB+noOZTc +q3J5Lwa/q4FwxKjt6lM07e8eU9kGx1Yr0Vz00YqOtCuxN5BICEIlxT6Ky3/rbwTR +bcV0oveifHtgPHfNDs5IAn8BL7abN+AqKjbc1YXWrOU/VG+WHgWv +-----END CERTIFICATE----- +` + +const googleLeaf = `-----BEGIN CERTIFICATE----- +MIIDITCCAoqgAwIBAgIQL9+89q6RUm0PmqPfQDQ+mjANBgkqhkiG9w0BAQUFADBM +MQswCQYDVQQGEwJaQTElMCMGA1UEChMcVGhhd3RlIENvbnN1bHRpbmcgKFB0eSkg +THRkLjEWMBQGA1UEAxMNVGhhd3RlIFNHQyBDQTAeFw0wOTEyMTgwMDAwMDBaFw0x +MTEyMTgyMzU5NTlaMGgxCzAJBgNVBAYTAlVTMRMwEQYDVQQIEwpDYWxpZm9ybmlh +MRYwFAYDVQQHFA1Nb3VudGFpbiBWaWV3MRMwEQYDVQQKFApHb29nbGUgSW5jMRcw +FQYDVQQDFA53d3cuZ29vZ2xlLmNvbTCBnzANBgkqhkiG9w0BAQEFAAOBjQAwgYkC +gYEA6PmGD5D6htffvXImttdEAoN4c9kCKO+IRTn7EOh8rqk41XXGOOsKFQebg+jN +gtXj9xVoRaELGYW84u+E593y17iYwqG7tcFR39SDAqc9BkJb4SLD3muFXxzW2k6L +05vuuWciKh0R73mkszeK9P4Y/bz5RiNQl/Os/CRGK1w7t0UCAwEAAaOB5zCB5DAM +BgNVHRMBAf8EAjAAMDYGA1UdHwQvMC0wK6ApoCeGJWh0dHA6Ly9jcmwudGhhd3Rl +LmNvbS9UaGF3dGVTR0NDQS5jcmwwKAYDVR0lBCEwHwYIKwYBBQUHAwEGCCsGAQUF +BwMCBglghkgBhvhCBAEwcgYIKwYBBQUHAQEEZjBkMCIGCCsGAQUFBzABhhZodHRw +Oi8vb2NzcC50aGF3dGUuY29tMD4GCCsGAQUFBzAChjJodHRwOi8vd3d3LnRoYXd0 +ZS5jb20vcmVwb3NpdG9yeS9UaGF3dGVfU0dDX0NBLmNydDANBgkqhkiG9w0BAQUF +AAOBgQCfQ89bxFApsb/isJr/aiEdLRLDLE5a+RLizrmCUi3nHX4adpaQedEkUjh5 +u2ONgJd8IyAPkU0Wueru9G2Jysa9zCRo1kNbzipYvzwY4OA8Ys+WAi0oR1A04Se6 +z5nRUP8pJcA2NhUzUnC+MY+f6H/nEQyNv4SgQhqAibAxWEEHXw== +-----END CERTIFICATE-----` + +const dnssecExpLeaf = `-----BEGIN CERTIFICATE----- +MIIGzTCCBbWgAwIBAgIDAdD6MA0GCSqGSIb3DQEBBQUAMIGMMQswCQYDVQQGEwJJ +TDEWMBQGA1UEChMNU3RhcnRDb20gTHRkLjErMCkGA1UECxMiU2VjdXJlIERpZ2l0 +YWwgQ2VydGlmaWNhdGUgU2lnbmluZzE4MDYGA1UEAxMvU3RhcnRDb20gQ2xhc3Mg +MSBQcmltYXJ5IEludGVybWVkaWF0ZSBTZXJ2ZXIgQ0EwHhcNMTAwNzA0MTQ1MjQ1 +WhcNMTEwNzA1MTA1NzA0WjCBwTEgMB4GA1UEDRMXMjIxMTM3LWxpOWE5dHhJRzZM +NnNyVFMxCzAJBgNVBAYTAlVTMR4wHAYDVQQKExVQZXJzb25hIE5vdCBWYWxpZGF0 +ZWQxKTAnBgNVBAsTIFN0YXJ0Q29tIEZyZWUgQ2VydGlmaWNhdGUgTWVtYmVyMRsw +GQYDVQQDExJ3d3cuZG5zc2VjLWV4cC5vcmcxKDAmBgkqhkiG9w0BCQEWGWhvc3Rt +YXN0ZXJAZG5zc2VjLWV4cC5vcmcwggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAwggEK +AoIBAQDEdF/22vaxrPbqpgVYMWi+alfpzBctpbfLBdPGuqOazJdCT0NbWcK8/+B4 +X6OlSOURNIlwLzhkmwVsWdVv6dVSaN7d4yI/fJkvgfDB9+au+iBJb6Pcz8ULBfe6 +D8HVvqKdORp6INzHz71z0sghxrQ0EAEkoWAZLh+kcn2ZHdcmZaBNUfjmGbyU6PRt +RjdqoP+owIaC1aktBN7zl4uO7cRjlYFdusINrh2kPP02KAx2W84xjxX1uyj6oS6e +7eBfvcwe8czW/N1rbE0CoR7h9+HnIrjnVG9RhBiZEiw3mUmF++Up26+4KTdRKbu3 ++BL4yMpfd66z0+zzqu+HkvyLpFn5AgMBAAGjggL/MIIC+zAJBgNVHRMEAjAAMAsG +A1UdDwQEAwIDqDATBgNVHSUEDDAKBggrBgEFBQcDATAdBgNVHQ4EFgQUy04I5guM +drzfh2JQaXhgV86+4jUwHwYDVR0jBBgwFoAU60I00Jiwq5/0G2sI98xkLu8OLEUw +LQYDVR0RBCYwJIISd3d3LmRuc3NlYy1leHAub3Jngg5kbnNzZWMtZXhwLm9yZzCC +AUIGA1UdIASCATkwggE1MIIBMQYLKwYBBAGBtTcBAgIwggEgMC4GCCsGAQUFBwIB +FiJodHRwOi8vd3d3LnN0YXJ0c3NsLmNvbS9wb2xpY3kucGRmMDQGCCsGAQUFBwIB +FihodHRwOi8vd3d3LnN0YXJ0c3NsLmNvbS9pbnRlcm1lZGlhdGUucGRmMIG3Bggr +BgEFBQcCAjCBqjAUFg1TdGFydENvbSBMdGQuMAMCAQEagZFMaW1pdGVkIExpYWJp +bGl0eSwgc2VlIHNlY3Rpb24gKkxlZ2FsIExpbWl0YXRpb25zKiBvZiB0aGUgU3Rh +cnRDb20gQ2VydGlmaWNhdGlvbiBBdXRob3JpdHkgUG9saWN5IGF2YWlsYWJsZSBh +dCBodHRwOi8vd3d3LnN0YXJ0c3NsLmNvbS9wb2xpY3kucGRmMGEGA1UdHwRaMFgw +KqAooCaGJGh0dHA6Ly93d3cuc3RhcnRzc2wuY29tL2NydDEtY3JsLmNybDAqoCig +JoYkaHR0cDovL2NybC5zdGFydHNzbC5jb20vY3J0MS1jcmwuY3JsMIGOBggrBgEF +BQcBAQSBgTB/MDkGCCsGAQUFBzABhi1odHRwOi8vb2NzcC5zdGFydHNzbC5jb20v +c3ViL2NsYXNzMS9zZXJ2ZXIvY2EwQgYIKwYBBQUHMAKGNmh0dHA6Ly93d3cuc3Rh +cnRzc2wuY29tL2NlcnRzL3N1Yi5jbGFzczEuc2VydmVyLmNhLmNydDAjBgNVHRIE +HDAahhhodHRwOi8vd3d3LnN0YXJ0c3NsLmNvbS8wDQYJKoZIhvcNAQEFBQADggEB +ACXj6SB59KRJPenn6gUdGEqcta97U769SATyiQ87i9er64qLwvIGLMa3o2Rcgl2Y +kghUeyLdN/EXyFBYA8L8uvZREPoc7EZukpT/ZDLXy9i2S0jkOxvF2fD/XLbcjGjM +iEYG1/6ASw0ri9C0k4oDDoJLCoeH9++yqF7SFCCMcDkJqiAGXNb4euDpa8vCCtEQ +CSS+ObZbfkreRt3cNCf5LfCXe9OsTnCfc8Cuq81c0oLaG+SmaLUQNBuToq8e9/Zm ++b+/a3RVjxmkV5OCcGVBxsXNDn54Q6wsdw0TBMcjwoEndzpLS7yWgFbbkq5ZiGpw +Qibb2+CfKuQ+WFV1GkVQmVA= +-----END CERTIFICATE-----` + +const startComIntermediate = `-----BEGIN CERTIFICATE----- +MIIGNDCCBBygAwIBAgIBGDANBgkqhkiG9w0BAQUFADB9MQswCQYDVQQGEwJJTDEW +MBQGA1UEChMNU3RhcnRDb20gTHRkLjErMCkGA1UECxMiU2VjdXJlIERpZ2l0YWwg +Q2VydGlmaWNhdGUgU2lnbmluZzEpMCcGA1UEAxMgU3RhcnRDb20gQ2VydGlmaWNh +dGlvbiBBdXRob3JpdHkwHhcNMDcxMDI0MjA1NDE3WhcNMTcxMDI0MjA1NDE3WjCB +jDELMAkGA1UEBhMCSUwxFjAUBgNVBAoTDVN0YXJ0Q29tIEx0ZC4xKzApBgNVBAsT +IlNlY3VyZSBEaWdpdGFsIENlcnRpZmljYXRlIFNpZ25pbmcxODA2BgNVBAMTL1N0 +YXJ0Q29tIENsYXNzIDEgUHJpbWFyeSBJbnRlcm1lZGlhdGUgU2VydmVyIENBMIIB +IjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAtonGrO8JUngHrJJj0PREGBiE +gFYfka7hh/oyULTTRwbw5gdfcA4Q9x3AzhA2NIVaD5Ksg8asWFI/ujjo/OenJOJA +pgh2wJJuniptTT9uYSAK21ne0n1jsz5G/vohURjXzTCm7QduO3CHtPn66+6CPAVv +kvek3AowHpNz/gfK11+AnSJYUq4G2ouHI2mw5CrY6oPSvfNx23BaKA+vWjhwRRI/ +ME3NO68X5Q/LoKldSKqxYVDLNM08XMML6BDAjJvwAwNi/rJsPnIO7hxDKslIDlc5 +xDEhyBDBLIf+VJVSH1I8MRKbf+fAoKVZ1eKPPvDVqOHXcDGpxLPPr21TLwb0pwID +AQABo4IBrTCCAakwDwYDVR0TAQH/BAUwAwEB/zAOBgNVHQ8BAf8EBAMCAQYwHQYD +VR0OBBYEFOtCNNCYsKuf9BtrCPfMZC7vDixFMB8GA1UdIwQYMBaAFE4L7xqkQFul +F2mHMMo0aEPQQa7yMGYGCCsGAQUFBwEBBFowWDAnBggrBgEFBQcwAYYbaHR0cDov +L29jc3Auc3RhcnRzc2wuY29tL2NhMC0GCCsGAQUFBzAChiFodHRwOi8vd3d3LnN0 +YXJ0c3NsLmNvbS9zZnNjYS5jcnQwWwYDVR0fBFQwUjAnoCWgI4YhaHR0cDovL3d3 +dy5zdGFydHNzbC5jb20vc2ZzY2EuY3JsMCegJaAjhiFodHRwOi8vY3JsLnN0YXJ0 +c3NsLmNvbS9zZnNjYS5jcmwwgYAGA1UdIAR5MHcwdQYLKwYBBAGBtTcBAgEwZjAu +BggrBgEFBQcCARYiaHR0cDovL3d3dy5zdGFydHNzbC5jb20vcG9saWN5LnBkZjA0 +BggrBgEFBQcCARYoaHR0cDovL3d3dy5zdGFydHNzbC5jb20vaW50ZXJtZWRpYXRl +LnBkZjANBgkqhkiG9w0BAQUFAAOCAgEAIQlJPqWIbuALi0jaMU2P91ZXouHTYlfp +tVbzhUV1O+VQHwSL5qBaPucAroXQ+/8gA2TLrQLhxpFy+KNN1t7ozD+hiqLjfDen +xk+PNdb01m4Ge90h2c9W/8swIkn+iQTzheWq8ecf6HWQTd35RvdCNPdFWAwRDYSw +xtpdPvkBnufh2lWVvnQce/xNFE+sflVHfXv0pQ1JHpXo9xLBzP92piVH0PN1Nb6X +t1gW66pceG/sUzCv6gRNzKkC4/C2BBL2MLERPZBOVmTX3DxDX3M570uvh+v2/miI +RHLq0gfGabDBoYvvF0nXYbFFSF87ICHpW7LM9NfpMfULFWE7epTj69m8f5SuauNi +YpaoZHy4h/OZMn6SolK+u/hlz8nyMPyLwcKmltdfieFcNID1j0cHL7SRv7Gifl9L +WtBbnySGBVFaaQNlQ0lxxeBvlDRr9hvYqbBMflPrj0jfyjO1SPo2ShpTpjMM0InN +SRXNiTE8kMBy12VLUjWKRhFEuT2OKGWmPnmeXAhEKa2wNREuIU640ucQPl2Eg7PD +wuTSxv0JS3QJ3fGz0xk+gA2iCxnwOOfFwq/iI9th4p1cbiCJSS4jarJiwUW0n6+L +p/EiO/h94pDQehn7Skzj0n1fSoMD7SfWI55rjbRZotnvbIIp3XUZPD9MEI3vu3Un +0q6Dp6jOW6c= +-----END CERTIFICATE-----` + +const startComRoot = `-----BEGIN CERTIFICATE----- +MIIHyTCCBbGgAwIBAgIBATANBgkqhkiG9w0BAQUFADB9MQswCQYDVQQGEwJJTDEW +MBQGA1UEChMNU3RhcnRDb20gTHRkLjErMCkGA1UECxMiU2VjdXJlIERpZ2l0YWwg +Q2VydGlmaWNhdGUgU2lnbmluZzEpMCcGA1UEAxMgU3RhcnRDb20gQ2VydGlmaWNh +dGlvbiBBdXRob3JpdHkwHhcNMDYwOTE3MTk0NjM2WhcNMzYwOTE3MTk0NjM2WjB9 +MQswCQYDVQQGEwJJTDEWMBQGA1UEChMNU3RhcnRDb20gTHRkLjErMCkGA1UECxMi +U2VjdXJlIERpZ2l0YWwgQ2VydGlmaWNhdGUgU2lnbmluZzEpMCcGA1UEAxMgU3Rh +cnRDb20gQ2VydGlmaWNhdGlvbiBBdXRob3JpdHkwggIiMA0GCSqGSIb3DQEBAQUA +A4ICDwAwggIKAoICAQDBiNsJvGxGfHiflXu1M5DycmLWwTYgIiRezul38kMKogZk +pMyONvg45iPwbm2xPN1yo4UcodM9tDMr0y+v/uqwQVlntsQGfQqedIXWeUyAN3rf +OQVSWff0G0ZDpNKFhdLDcfN1YjS6LIp/Ho/u7TTQEceWzVI9ujPW3U3eCztKS5/C +Ji/6tRYccjV3yjxd5srhJosaNnZcAdt0FCX+7bWgiA/deMotHweXMAEtcnn6RtYT +Kqi5pquDSR3l8u/d5AGOGAqPY1MWhWKpDhk6zLVmpsJrdAfkK+F2PrRt2PZE4XNi +HzvEvqBTViVsUQn3qqvKv3b9bZvzndu/PWa8DFaqr5hIlTpL36dYUNk4dalb6kMM +Av+Z6+hsTXBbKWWc3apdzK8BMewM69KN6Oqce+Zu9ydmDBpI125C4z/eIT574Q1w ++2OqqGwaVLRcJXrJosmLFqa7LH4XXgVNWG4SHQHuEhANxjJ/GP/89PrNbpHoNkm+ +Gkhpi8KWTRoSsmkXwQqQ1vp5Iki/untp+HDH+no32NgN0nZPV/+Qt+OR0t3vwmC3 +Zzrd/qqc8NSLf3Iizsafl7b4r4qgEKjZ+xjGtrVcUjyJthkqcwEKDwOzEmDyei+B +26Nu/yYwl/WL3YlXtq09s68rxbd2AvCl1iuahhQqcvbjM4xdCUsT37uMdBNSSwID +AQABo4ICUjCCAk4wDAYDVR0TBAUwAwEB/zALBgNVHQ8EBAMCAa4wHQYDVR0OBBYE +FE4L7xqkQFulF2mHMMo0aEPQQa7yMGQGA1UdHwRdMFswLKAqoCiGJmh0dHA6Ly9j +ZXJ0LnN0YXJ0Y29tLm9yZy9zZnNjYS1jcmwuY3JsMCugKaAnhiVodHRwOi8vY3Js +LnN0YXJ0Y29tLm9yZy9zZnNjYS1jcmwuY3JsMIIBXQYDVR0gBIIBVDCCAVAwggFM +BgsrBgEEAYG1NwEBATCCATswLwYIKwYBBQUHAgEWI2h0dHA6Ly9jZXJ0LnN0YXJ0 +Y29tLm9yZy9wb2xpY3kucGRmMDUGCCsGAQUFBwIBFilodHRwOi8vY2VydC5zdGFy +dGNvbS5vcmcvaW50ZXJtZWRpYXRlLnBkZjCB0AYIKwYBBQUHAgIwgcMwJxYgU3Rh +cnQgQ29tbWVyY2lhbCAoU3RhcnRDb20pIEx0ZC4wAwIBARqBl0xpbWl0ZWQgTGlh +YmlsaXR5LCByZWFkIHRoZSBzZWN0aW9uICpMZWdhbCBMaW1pdGF0aW9ucyogb2Yg +dGhlIFN0YXJ0Q29tIENlcnRpZmljYXRpb24gQXV0aG9yaXR5IFBvbGljeSBhdmFp +bGFibGUgYXQgaHR0cDovL2NlcnQuc3RhcnRjb20ub3JnL3BvbGljeS5wZGYwEQYJ +YIZIAYb4QgEBBAQDAgAHMDgGCWCGSAGG+EIBDQQrFilTdGFydENvbSBGcmVlIFNT +TCBDZXJ0aWZpY2F0aW9uIEF1dGhvcml0eTANBgkqhkiG9w0BAQUFAAOCAgEAFmyZ +9GYMNPXQhV59CuzaEE44HF7fpiUFS5Eyweg78T3dRAlbB0mKKctmArexmvclmAk8 +jhvh3TaHK0u7aNM5Zj2gJsfyOZEdUauCe37Vzlrk4gNXcGmXCPleWKYK34wGmkUW +FjgKXlf2Ysd6AgXmvB618p70qSmD+LIU424oh0TDkBreOKk8rENNZEXO3SipXPJz +ewT4F+irsfMuXGRuczE6Eri8sxHkfY+BUZo7jYn0TZNmezwD7dOaHZrzZVD1oNB1 +ny+v8OqCQ5j4aZyJecRDjkZy42Q2Eq/3JR44iZB3fsNrarnDy0RLrHiQi+fHLB5L +EUTINFInzQpdn4XBidUaePKVEFMy3YCEZnXZtWgo+2EuvoSoOMCZEoalHmdkrQYu +L6lwhceWD3yJZfWOQ1QOq92lgDmUYMA0yZZwLKMS9R9Ie70cfmu3nZD0Ijuu+Pwq +yvqCUqDvr0tVk+vBtfAii6w0TiYiBKGHLHVKt+V9E9e4DGTANtLJL4YSjCMJwRuC +O3NJo2pXh5Tl1njFmUNj403gdy3hZZlyaQQaRwnmDwFWJPsfvw55qVguucQJAX6V +um0ABj6y6koQOdjQK/W/7HW/lwLFCRsI3FU34oH7N4RDYiDK51ZLZer+bMEkkySh +NOsF/5oirpt9P/FlUQqmMGqz9IgcgA38corog14= +-----END CERTIFICATE-----` diff --git a/src/pkg/crypto/x509/x509.go b/src/pkg/crypto/x509/x509.go new file mode 100644 index 000000000..8fda47159 --- /dev/null +++ b/src/pkg/crypto/x509/x509.go @@ -0,0 +1,1073 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Package x509 parses X.509-encoded keys and certificates. +package x509 + +import ( + "asn1" + "big" + "bytes" + "crypto" + "crypto/dsa" + "crypto/rsa" + "crypto/sha1" + "crypto/x509/pkix" + "encoding/pem" + "io" + "os" + "time" +) + +// pkcs1PrivateKey is a structure which mirrors the PKCS#1 ASN.1 for an RSA private key. +type pkcs1PrivateKey struct { + Version int + N *big.Int + E int + D *big.Int + P *big.Int + Q *big.Int + // We ignore these values, if present, because rsa will calculate them. + Dp *big.Int `asn1:"optional"` + Dq *big.Int `asn1:"optional"` + Qinv *big.Int `asn1:"optional"` + + AdditionalPrimes []pkcs1AdditionalRSAPrime `asn1:"optional"` +} + +type pkcs1AdditionalRSAPrime struct { + Prime *big.Int + + // We ignore these values because rsa will calculate them. + Exp *big.Int + Coeff *big.Int +} + +// ParsePKCS1PrivateKey returns an RSA private key from its ASN.1 PKCS#1 DER encoded form. +func ParsePKCS1PrivateKey(der []byte) (key *rsa.PrivateKey, err os.Error) { + var priv pkcs1PrivateKey + rest, err := asn1.Unmarshal(der, &priv) + if len(rest) > 0 { + err = asn1.SyntaxError{"trailing data"} + return + } + if err != nil { + return + } + + if priv.Version > 1 { + return nil, os.NewError("x509: unsupported private key version") + } + + if priv.N.Sign() <= 0 || priv.D.Sign() <= 0 || priv.P.Sign() <= 0 || priv.Q.Sign() <= 0 { + return nil, os.NewError("private key contains zero or negative value") + } + + key = new(rsa.PrivateKey) + key.PublicKey = rsa.PublicKey{ + E: priv.E, + N: priv.N, + } + + key.D = priv.D + key.Primes = make([]*big.Int, 2+len(priv.AdditionalPrimes)) + key.Primes[0] = priv.P + key.Primes[1] = priv.Q + for i, a := range priv.AdditionalPrimes { + if a.Prime.Sign() <= 0 { + return nil, os.NewError("private key contains zero or negative prime") + } + key.Primes[i+2] = a.Prime + // We ignore the other two values because rsa will calculate + // them as needed. + } + + err = key.Validate() + if err != nil { + return nil, err + } + key.Precompute() + + return +} + +// MarshalPKCS1PrivateKey converts a private key to ASN.1 DER encoded form. +func MarshalPKCS1PrivateKey(key *rsa.PrivateKey) []byte { + key.Precompute() + + version := 0 + if len(key.Primes) > 2 { + version = 1 + } + + priv := pkcs1PrivateKey{ + Version: version, + N: key.N, + E: key.PublicKey.E, + D: key.D, + P: key.Primes[0], + Q: key.Primes[1], + Dp: key.Precomputed.Dp, + Dq: key.Precomputed.Dq, + Qinv: key.Precomputed.Qinv, + } + + priv.AdditionalPrimes = make([]pkcs1AdditionalRSAPrime, len(key.Precomputed.CRTValues)) + for i, values := range key.Precomputed.CRTValues { + priv.AdditionalPrimes[i].Prime = key.Primes[2+i] + priv.AdditionalPrimes[i].Exp = values.Exp + priv.AdditionalPrimes[i].Coeff = values.Coeff + } + + b, _ := asn1.Marshal(priv) + return b +} + +// These structures reflect the ASN.1 structure of X.509 certificates.: + +type certificate struct { + Raw asn1.RawContent + TBSCertificate tbsCertificate + SignatureAlgorithm pkix.AlgorithmIdentifier + SignatureValue asn1.BitString +} + +type tbsCertificate struct { + Raw asn1.RawContent + Version int `asn1:"optional,explicit,default:1,tag:0"` + SerialNumber *big.Int + SignatureAlgorithm pkix.AlgorithmIdentifier + Issuer pkix.RDNSequence + Validity validity + Subject pkix.RDNSequence + PublicKey publicKeyInfo + UniqueId asn1.BitString `asn1:"optional,tag:1"` + SubjectUniqueId asn1.BitString `asn1:"optional,tag:2"` + Extensions []pkix.Extension `asn1:"optional,explicit,tag:3"` +} + +type dsaAlgorithmParameters struct { + P, Q, G *big.Int +} + +type dsaSignature struct { + R, S *big.Int +} + +type validity struct { + NotBefore, NotAfter *time.Time +} + +type publicKeyInfo struct { + Raw asn1.RawContent + Algorithm pkix.AlgorithmIdentifier + PublicKey asn1.BitString +} + +// RFC 5280, 4.2.1.1 +type authKeyId struct { + Id []byte `asn1:"optional,tag:0"` +} + +type SignatureAlgorithm int + +const ( + UnknownSignatureAlgorithm SignatureAlgorithm = iota + MD2WithRSA + MD5WithRSA + SHA1WithRSA + SHA256WithRSA + SHA384WithRSA + SHA512WithRSA + DSAWithSHA1 + DSAWithSHA256 +) + +type PublicKeyAlgorithm int + +const ( + UnknownPublicKeyAlgorithm PublicKeyAlgorithm = iota + RSA + DSA +) + +// OIDs for signature algorithms +// +// pkcs-1 OBJECT IDENTIFIER ::= { +// iso(1) member-body(2) us(840) rsadsi(113549) pkcs(1) 1 } +// +// +// RFC 3279 2.2.1 RSA Signature Algorithms +// +// md2WithRSAEncryption OBJECT IDENTIFIER ::= { pkcs-1 2 } +// +// md5WithRSAEncryption OBJECT IDENTIFER ::= { pkcs-1 4 } +// +// sha-1WithRSAEncryption OBJECT IDENTIFIER ::= { pkcs-1 5 } +// +// dsaWithSha1 OBJECT IDENTIFIER ::= { +// iso(1) member-body(2) us(840) x9-57(10040) x9cm(4) 3 } +// +// +// RFC 4055 5 PKCS #1 Version 1.5 +// +// sha256WithRSAEncryption OBJECT IDENTIFIER ::= { pkcs-1 11 } +// +// sha384WithRSAEncryption OBJECT IDENTIFIER ::= { pkcs-1 12 } +// +// sha512WithRSAEncryption OBJECT IDENTIFIER ::= { pkcs-1 13 } +// +// +// RFC 5758 3.1 DSA Signature Algorithms +// +// dsaWithSha356 OBJECT IDENTIFER ::= { +// joint-iso-ccitt(2) country(16) us(840) organization(1) gov(101) +// algorithms(4) id-dsa-with-sha2(3) 2} +// +var ( + oidSignatureMD2WithRSA = asn1.ObjectIdentifier{1, 2, 840, 113549, 1, 1, 2} + oidSignatureMD5WithRSA = asn1.ObjectIdentifier{1, 2, 840, 113549, 1, 1, 4} + oidSignatureSHA1WithRSA = asn1.ObjectIdentifier{1, 2, 840, 113549, 1, 1, 5} + oidSignatureSHA256WithRSA = asn1.ObjectIdentifier{1, 2, 840, 113549, 1, 1, 11} + oidSignatureSHA384WithRSA = asn1.ObjectIdentifier{1, 2, 840, 113549, 1, 1, 12} + oidSignatureSHA512WithRSA = asn1.ObjectIdentifier{1, 2, 840, 113549, 1, 1, 13} + oidSignatureDSAWithSHA1 = asn1.ObjectIdentifier{1, 2, 840, 10040, 4, 3} + oidSignatureDSAWithSHA256 = asn1.ObjectIdentifier{2, 16, 840, 1, 101, 4, 3, 2} +) + +func getSignatureAlgorithmFromOID(oid asn1.ObjectIdentifier) SignatureAlgorithm { + switch { + case oid.Equal(oidSignatureMD2WithRSA): + return MD2WithRSA + case oid.Equal(oidSignatureMD5WithRSA): + return MD5WithRSA + case oid.Equal(oidSignatureSHA1WithRSA): + return SHA1WithRSA + case oid.Equal(oidSignatureSHA256WithRSA): + return SHA256WithRSA + case oid.Equal(oidSignatureSHA384WithRSA): + return SHA384WithRSA + case oid.Equal(oidSignatureSHA512WithRSA): + return SHA512WithRSA + case oid.Equal(oidSignatureDSAWithSHA1): + return DSAWithSHA1 + case oid.Equal(oidSignatureDSAWithSHA256): + return DSAWithSHA256 + } + return UnknownSignatureAlgorithm +} + +// RFC 3279, 2.3 Public Key Algorithms +// +// pkcs-1 OBJECT IDENTIFIER ::== { iso(1) member-body(2) us(840) +// rsadsi(113549) pkcs(1) 1 } +// +// rsaEncryption OBJECT IDENTIFIER ::== { pkcs1-1 1 } +// +// id-dsa OBJECT IDENTIFIER ::== { iso(1) member-body(2) us(840) +// x9-57(10040) x9cm(4) 1 } +var ( + oidPublicKeyRsa = asn1.ObjectIdentifier{1, 2, 840, 113549, 1, 1, 1} + oidPublicKeyDsa = asn1.ObjectIdentifier{1, 2, 840, 10040, 4, 1} +) + +func getPublicKeyAlgorithmFromOID(oid asn1.ObjectIdentifier) PublicKeyAlgorithm { + switch { + case oid.Equal(oidPublicKeyRsa): + return RSA + case oid.Equal(oidPublicKeyDsa): + return DSA + } + return UnknownPublicKeyAlgorithm +} + +// KeyUsage represents the set of actions that are valid for a given key. It's +// a bitmap of the KeyUsage* constants. +type KeyUsage int + +const ( + KeyUsageDigitalSignature KeyUsage = 1 << iota + KeyUsageContentCommitment + KeyUsageKeyEncipherment + KeyUsageDataEncipherment + KeyUsageKeyAgreement + KeyUsageCertSign + KeyUsageCRLSign + KeyUsageEncipherOnly + KeyUsageDecipherOnly +) + +// RFC 5280, 4.2.1.12 Extended Key Usage +// +// anyExtendedKeyUsage OBJECT IDENTIFIER ::= { id-ce-extKeyUsage 0 } +// +// id-kp OBJECT IDENTIFIER ::= { id-pkix 3 } +// +// id-kp-serverAuth OBJECT IDENTIFIER ::= { id-kp 1 } +// id-kp-clientAuth OBJECT IDENTIFIER ::= { id-kp 2 } +// id-kp-codeSigning OBJECT IDENTIFIER ::= { id-kp 3 } +// id-kp-emailProtection OBJECT IDENTIFIER ::= { id-kp 4 } +// id-kp-timeStamping OBJECT IDENTIFIER ::= { id-kp 8 } +// id-kp-OCSPSigning OBJECT IDENTIFIER ::= { id-kp 9 } +var ( + oidExtKeyUsageAny = asn1.ObjectIdentifier{2, 5, 29, 37, 0} + oidExtKeyUsageServerAuth = asn1.ObjectIdentifier{1, 3, 6, 1, 5, 5, 7, 3, 1} + oidExtKeyUsageClientAuth = asn1.ObjectIdentifier{1, 3, 6, 1, 5, 5, 7, 3, 2} + oidExtKeyUsageCodeSigning = asn1.ObjectIdentifier{1, 3, 6, 1, 5, 5, 7, 3, 3} + oidExtKeyUsageEmailProtection = asn1.ObjectIdentifier{1, 3, 6, 1, 5, 5, 7, 3, 4} + oidExtKeyUsageTimeStamping = asn1.ObjectIdentifier{1, 3, 6, 1, 5, 5, 7, 3, 8} + oidExtKeyUsageOCSPSigning = asn1.ObjectIdentifier{1, 3, 6, 1, 5, 5, 7, 3, 9} +) + +// ExtKeyUsage represents an extended set of actions that are valid for a given key. +// Each of the ExtKeyUsage* constants define a unique action. +type ExtKeyUsage int + +const ( + ExtKeyUsageAny ExtKeyUsage = iota + ExtKeyUsageServerAuth + ExtKeyUsageClientAuth + ExtKeyUsageCodeSigning + ExtKeyUsageEmailProtection + ExtKeyUsageTimeStamping + ExtKeyUsageOCSPSigning +) + +// A Certificate represents an X.509 certificate. +type Certificate struct { + Raw []byte // Complete ASN.1 DER content (certificate, signature algorithm and signature). + RawTBSCertificate []byte // Certificate part of raw ASN.1 DER content. + RawSubjectPublicKeyInfo []byte // DER encoded SubjectPublicKeyInfo. + + Signature []byte + SignatureAlgorithm SignatureAlgorithm + + PublicKeyAlgorithm PublicKeyAlgorithm + PublicKey interface{} + + Version int + SerialNumber *big.Int + Issuer pkix.Name + Subject pkix.Name + NotBefore, NotAfter *time.Time // Validity bounds. + KeyUsage KeyUsage + + ExtKeyUsage []ExtKeyUsage // Sequence of extended key usages. + UnknownExtKeyUsage []asn1.ObjectIdentifier // Encountered extended key usages unknown to this package. + + BasicConstraintsValid bool // if true then the next two fields are valid. + IsCA bool + MaxPathLen int + + SubjectKeyId []byte + AuthorityKeyId []byte + + // Subject Alternate Name values + DNSNames []string + EmailAddresses []string + + // Name constraints + PermittedDNSDomainsCritical bool // if true then the name constraints are marked critical. + PermittedDNSDomains []string + + PolicyIdentifiers []asn1.ObjectIdentifier +} + +// UnsupportedAlgorithmError results from attempting to perform an operation +// that involves algorithms that are not currently implemented. +type UnsupportedAlgorithmError struct{} + +func (UnsupportedAlgorithmError) String() string { + return "cannot verify signature: algorithm unimplemented" +} + +// ConstraintViolationError results when a requested usage is not permitted by +// a certificate. For example: checking a signature when the public key isn't a +// certificate signing key. +type ConstraintViolationError struct{} + +func (ConstraintViolationError) String() string { + return "invalid signature: parent certificate cannot sign this kind of certificate" +} + +func (c *Certificate) Equal(other *Certificate) bool { + return bytes.Equal(c.Raw, other.Raw) +} + +// CheckSignatureFrom verifies that the signature on c is a valid signature +// from parent. +func (c *Certificate) CheckSignatureFrom(parent *Certificate) (err os.Error) { + // RFC 5280, 4.2.1.9: + // "If the basic constraints extension is not present in a version 3 + // certificate, or the extension is present but the cA boolean is not + // asserted, then the certified public key MUST NOT be used to verify + // certificate signatures." + if parent.Version == 3 && !parent.BasicConstraintsValid || + parent.BasicConstraintsValid && !parent.IsCA { + return ConstraintViolationError{} + } + + if parent.KeyUsage != 0 && parent.KeyUsage&KeyUsageCertSign == 0 { + return ConstraintViolationError{} + } + + if parent.PublicKeyAlgorithm == UnknownPublicKeyAlgorithm { + return UnsupportedAlgorithmError{} + } + + // TODO(agl): don't ignore the path length constraint. + + return parent.CheckSignature(c.SignatureAlgorithm, c.RawTBSCertificate, c.Signature) +} + +// CheckSignature verifies that signature is a valid signature over signed from +// c's public key. +func (c *Certificate) CheckSignature(algo SignatureAlgorithm, signed, signature []byte) (err os.Error) { + var hashType crypto.Hash + + switch algo { + case SHA1WithRSA, DSAWithSHA1: + hashType = crypto.SHA1 + case SHA256WithRSA, DSAWithSHA256: + hashType = crypto.SHA256 + case SHA384WithRSA: + hashType = crypto.SHA384 + case SHA512WithRSA: + hashType = crypto.SHA512 + default: + return UnsupportedAlgorithmError{} + } + + h := hashType.New() + if h == nil { + return UnsupportedAlgorithmError{} + } + + h.Write(signed) + digest := h.Sum() + + switch pub := c.PublicKey.(type) { + case *rsa.PublicKey: + return rsa.VerifyPKCS1v15(pub, hashType, digest, signature) + case *dsa.PublicKey: + dsaSig := new(dsaSignature) + if _, err := asn1.Unmarshal(signature, dsaSig); err != nil { + return err + } + if dsaSig.R.Sign() <= 0 || dsaSig.S.Sign() <= 0 { + return os.NewError("DSA signature contained zero or negative values") + } + if !dsa.Verify(pub, digest, dsaSig.R, dsaSig.S) { + return os.NewError("DSA verification failure") + } + return + } + return UnsupportedAlgorithmError{} +} + +// CheckCRLSignature checks that the signature in crl is from c. +func (c *Certificate) CheckCRLSignature(crl *pkix.CertificateList) (err os.Error) { + algo := getSignatureAlgorithmFromOID(crl.SignatureAlgorithm.Algorithm) + return c.CheckSignature(algo, crl.TBSCertList.Raw, crl.SignatureValue.RightAlign()) +} + +type UnhandledCriticalExtension struct{} + +func (h UnhandledCriticalExtension) String() string { + return "unhandled critical extension" +} + +type basicConstraints struct { + IsCA bool `asn1:"optional"` + MaxPathLen int `asn1:"optional"` +} + +type rsaPublicKey struct { + N *big.Int + E int +} + +// RFC 5280 4.2.1.4 +type policyInformation struct { + Policy asn1.ObjectIdentifier + // policyQualifiers omitted +} + +// RFC 5280, 4.2.1.10 +type nameConstraints struct { + Permitted []generalSubtree `asn1:"optional,tag:0"` + Excluded []generalSubtree `asn1:"optional,tag:1"` +} + +type generalSubtree struct { + Name string `asn1:"tag:2,optional,ia5"` + Min int `asn1:"optional,tag:0"` + Max int `asn1:"optional,tag:1"` +} + +func parsePublicKey(algo PublicKeyAlgorithm, keyData *publicKeyInfo) (interface{}, os.Error) { + asn1Data := keyData.PublicKey.RightAlign() + switch algo { + case RSA: + p := new(rsaPublicKey) + _, err := asn1.Unmarshal(asn1Data, p) + if err != nil { + return nil, err + } + + pub := &rsa.PublicKey{ + E: p.E, + N: p.N, + } + return pub, nil + case DSA: + var p *big.Int + _, err := asn1.Unmarshal(asn1Data, &p) + if err != nil { + return nil, err + } + paramsData := keyData.Algorithm.Parameters.FullBytes + params := new(dsaAlgorithmParameters) + _, err = asn1.Unmarshal(paramsData, params) + if err != nil { + return nil, err + } + if p.Sign() <= 0 || params.P.Sign() <= 0 || params.Q.Sign() <= 0 || params.G.Sign() <= 0 { + return nil, os.NewError("zero or negative DSA parameter") + } + pub := &dsa.PublicKey{ + Parameters: dsa.Parameters{ + P: params.P, + Q: params.Q, + G: params.G, + }, + Y: p, + } + return pub, nil + default: + return nil, nil + } + panic("unreachable") +} + +func parseCertificate(in *certificate) (*Certificate, os.Error) { + out := new(Certificate) + out.Raw = in.Raw + out.RawTBSCertificate = in.TBSCertificate.Raw + out.RawSubjectPublicKeyInfo = in.TBSCertificate.PublicKey.Raw + + out.Signature = in.SignatureValue.RightAlign() + out.SignatureAlgorithm = + getSignatureAlgorithmFromOID(in.TBSCertificate.SignatureAlgorithm.Algorithm) + + out.PublicKeyAlgorithm = + getPublicKeyAlgorithmFromOID(in.TBSCertificate.PublicKey.Algorithm.Algorithm) + var err os.Error + out.PublicKey, err = parsePublicKey(out.PublicKeyAlgorithm, &in.TBSCertificate.PublicKey) + if err != nil { + return nil, err + } + + if in.TBSCertificate.SerialNumber.Sign() < 0 { + return nil, os.NewError("negative serial number") + } + + out.Version = in.TBSCertificate.Version + 1 + out.SerialNumber = in.TBSCertificate.SerialNumber + out.Issuer.FillFromRDNSequence(&in.TBSCertificate.Issuer) + out.Subject.FillFromRDNSequence(&in.TBSCertificate.Subject) + out.NotBefore = in.TBSCertificate.Validity.NotBefore + out.NotAfter = in.TBSCertificate.Validity.NotAfter + + for _, e := range in.TBSCertificate.Extensions { + if len(e.Id) == 4 && e.Id[0] == 2 && e.Id[1] == 5 && e.Id[2] == 29 { + switch e.Id[3] { + case 15: + // RFC 5280, 4.2.1.3 + var usageBits asn1.BitString + _, err := asn1.Unmarshal(e.Value, &usageBits) + + if err == nil { + var usage int + for i := 0; i < 9; i++ { + if usageBits.At(i) != 0 { + usage |= 1 << uint(i) + } + } + out.KeyUsage = KeyUsage(usage) + continue + } + case 19: + // RFC 5280, 4.2.1.9 + var constraints basicConstraints + _, err := asn1.Unmarshal(e.Value, &constraints) + + if err == nil { + out.BasicConstraintsValid = true + out.IsCA = constraints.IsCA + out.MaxPathLen = constraints.MaxPathLen + continue + } + case 17: + // RFC 5280, 4.2.1.6 + + // SubjectAltName ::= GeneralNames + // + // GeneralNames ::= SEQUENCE SIZE (1..MAX) OF GeneralName + // + // GeneralName ::= CHOICE { + // otherName [0] OtherName, + // rfc822Name [1] IA5String, + // dNSName [2] IA5String, + // x400Address [3] ORAddress, + // directoryName [4] Name, + // ediPartyName [5] EDIPartyName, + // uniformResourceIdentifier [6] IA5String, + // iPAddress [7] OCTET STRING, + // registeredID [8] OBJECT IDENTIFIER } + var seq asn1.RawValue + _, err := asn1.Unmarshal(e.Value, &seq) + if err != nil { + return nil, err + } + if !seq.IsCompound || seq.Tag != 16 || seq.Class != 0 { + return nil, asn1.StructuralError{"bad SAN sequence"} + } + + parsedName := false + + rest := seq.Bytes + for len(rest) > 0 { + var v asn1.RawValue + rest, err = asn1.Unmarshal(rest, &v) + if err != nil { + return nil, err + } + switch v.Tag { + case 1: + out.EmailAddresses = append(out.EmailAddresses, string(v.Bytes)) + parsedName = true + case 2: + out.DNSNames = append(out.DNSNames, string(v.Bytes)) + parsedName = true + } + } + + if parsedName { + continue + } + // If we didn't parse any of the names then we + // fall through to the critical check below. + + case 30: + // RFC 5280, 4.2.1.10 + + // NameConstraints ::= SEQUENCE { + // permittedSubtrees [0] GeneralSubtrees OPTIONAL, + // excludedSubtrees [1] GeneralSubtrees OPTIONAL } + // + // GeneralSubtrees ::= SEQUENCE SIZE (1..MAX) OF GeneralSubtree + // + // GeneralSubtree ::= SEQUENCE { + // base GeneralName, + // minimum [0] BaseDistance DEFAULT 0, + // maximum [1] BaseDistance OPTIONAL } + // + // BaseDistance ::= INTEGER (0..MAX) + + var constraints nameConstraints + _, err := asn1.Unmarshal(e.Value, &constraints) + if err != nil { + return nil, err + } + + if len(constraints.Excluded) > 0 && e.Critical { + return out, UnhandledCriticalExtension{} + } + + for _, subtree := range constraints.Permitted { + if subtree.Min > 0 || subtree.Max > 0 || len(subtree.Name) == 0 { + if e.Critical { + return out, UnhandledCriticalExtension{} + } + continue + } + out.PermittedDNSDomains = append(out.PermittedDNSDomains, subtree.Name) + } + continue + + case 35: + // RFC 5280, 4.2.1.1 + var a authKeyId + _, err = asn1.Unmarshal(e.Value, &a) + if err != nil { + return nil, err + } + out.AuthorityKeyId = a.Id + continue + + case 37: + // RFC 5280, 4.2.1.12. Extended Key Usage + + // id-ce-extKeyUsage OBJECT IDENTIFIER ::= { id-ce 37 } + // + // ExtKeyUsageSyntax ::= SEQUENCE SIZE (1..MAX) OF KeyPurposeId + // + // KeyPurposeId ::= OBJECT IDENTIFIER + + var keyUsage []asn1.ObjectIdentifier + _, err = asn1.Unmarshal(e.Value, &keyUsage) + if err != nil { + return nil, err + } + + for _, u := range keyUsage { + switch { + case u.Equal(oidExtKeyUsageAny): + out.ExtKeyUsage = append(out.ExtKeyUsage, ExtKeyUsageAny) + case u.Equal(oidExtKeyUsageServerAuth): + out.ExtKeyUsage = append(out.ExtKeyUsage, ExtKeyUsageServerAuth) + case u.Equal(oidExtKeyUsageClientAuth): + out.ExtKeyUsage = append(out.ExtKeyUsage, ExtKeyUsageClientAuth) + case u.Equal(oidExtKeyUsageCodeSigning): + out.ExtKeyUsage = append(out.ExtKeyUsage, ExtKeyUsageCodeSigning) + case u.Equal(oidExtKeyUsageEmailProtection): + out.ExtKeyUsage = append(out.ExtKeyUsage, ExtKeyUsageEmailProtection) + case u.Equal(oidExtKeyUsageTimeStamping): + out.ExtKeyUsage = append(out.ExtKeyUsage, ExtKeyUsageTimeStamping) + case u.Equal(oidExtKeyUsageOCSPSigning): + out.ExtKeyUsage = append(out.ExtKeyUsage, ExtKeyUsageOCSPSigning) + default: + out.UnknownExtKeyUsage = append(out.UnknownExtKeyUsage, u) + } + } + + continue + + case 14: + // RFC 5280, 4.2.1.2 + var keyid []byte + _, err = asn1.Unmarshal(e.Value, &keyid) + if err != nil { + return nil, err + } + out.SubjectKeyId = keyid + continue + + case 32: + // RFC 5280 4.2.1.4: Certificate Policies + var policies []policyInformation + if _, err = asn1.Unmarshal(e.Value, &policies); err != nil { + return nil, err + } + out.PolicyIdentifiers = make([]asn1.ObjectIdentifier, len(policies)) + for i, policy := range policies { + out.PolicyIdentifiers[i] = policy.Policy + } + } + } + + if e.Critical { + return out, UnhandledCriticalExtension{} + } + } + + return out, nil +} + +// ParseCertificate parses a single certificate from the given ASN.1 DER data. +func ParseCertificate(asn1Data []byte) (*Certificate, os.Error) { + var cert certificate + rest, err := asn1.Unmarshal(asn1Data, &cert) + if err != nil { + return nil, err + } + if len(rest) > 0 { + return nil, asn1.SyntaxError{"trailing data"} + } + + return parseCertificate(&cert) +} + +// ParseCertificates parses one or more certificates from the given ASN.1 DER +// data. The certificates must be concatenated with no intermediate padding. +func ParseCertificates(asn1Data []byte) ([]*Certificate, os.Error) { + var v []*certificate + + for len(asn1Data) > 0 { + cert := new(certificate) + var err os.Error + asn1Data, err = asn1.Unmarshal(asn1Data, cert) + if err != nil { + return nil, err + } + v = append(v, cert) + } + + ret := make([]*Certificate, len(v)) + for i, ci := range v { + cert, err := parseCertificate(ci) + if err != nil { + return nil, err + } + ret[i] = cert + } + + return ret, nil +} + +func reverseBitsInAByte(in byte) byte { + b1 := in>>4 | in<<4 + b2 := b1>>2&0x33 | b1<<2&0xcc + b3 := b2>>1&0x55 | b2<<1&0xaa + return b3 +} + +var ( + oidExtensionSubjectKeyId = []int{2, 5, 29, 14} + oidExtensionKeyUsage = []int{2, 5, 29, 15} + oidExtensionAuthorityKeyId = []int{2, 5, 29, 35} + oidExtensionBasicConstraints = []int{2, 5, 29, 19} + oidExtensionSubjectAltName = []int{2, 5, 29, 17} + oidExtensionCertificatePolicies = []int{2, 5, 29, 32} + oidExtensionNameConstraints = []int{2, 5, 29, 30} +) + +func buildExtensions(template *Certificate) (ret []pkix.Extension, err os.Error) { + ret = make([]pkix.Extension, 7 /* maximum number of elements. */ ) + n := 0 + + if template.KeyUsage != 0 { + ret[n].Id = oidExtensionKeyUsage + ret[n].Critical = true + + var a [2]byte + a[0] = reverseBitsInAByte(byte(template.KeyUsage)) + a[1] = reverseBitsInAByte(byte(template.KeyUsage >> 8)) + + l := 1 + if a[1] != 0 { + l = 2 + } + + ret[n].Value, err = asn1.Marshal(asn1.BitString{Bytes: a[0:l], BitLength: l * 8}) + if err != nil { + return + } + n++ + } + + if template.BasicConstraintsValid { + ret[n].Id = oidExtensionBasicConstraints + ret[n].Value, err = asn1.Marshal(basicConstraints{template.IsCA, template.MaxPathLen}) + ret[n].Critical = true + if err != nil { + return + } + n++ + } + + if len(template.SubjectKeyId) > 0 { + ret[n].Id = oidExtensionSubjectKeyId + ret[n].Value, err = asn1.Marshal(template.SubjectKeyId) + if err != nil { + return + } + n++ + } + + if len(template.AuthorityKeyId) > 0 { + ret[n].Id = oidExtensionAuthorityKeyId + ret[n].Value, err = asn1.Marshal(authKeyId{template.AuthorityKeyId}) + if err != nil { + return + } + n++ + } + + if len(template.DNSNames) > 0 { + ret[n].Id = oidExtensionSubjectAltName + rawValues := make([]asn1.RawValue, len(template.DNSNames)) + for i, name := range template.DNSNames { + rawValues[i] = asn1.RawValue{Tag: 2, Class: 2, Bytes: []byte(name)} + } + ret[n].Value, err = asn1.Marshal(rawValues) + if err != nil { + return + } + n++ + } + + if len(template.PolicyIdentifiers) > 0 { + ret[n].Id = oidExtensionCertificatePolicies + policies := make([]policyInformation, len(template.PolicyIdentifiers)) + for i, policy := range template.PolicyIdentifiers { + policies[i].Policy = policy + } + ret[n].Value, err = asn1.Marshal(policies) + if err != nil { + return + } + n++ + } + + if len(template.PermittedDNSDomains) > 0 { + ret[n].Id = oidExtensionNameConstraints + ret[n].Critical = template.PermittedDNSDomainsCritical + + var out nameConstraints + out.Permitted = make([]generalSubtree, len(template.PermittedDNSDomains)) + for i, permitted := range template.PermittedDNSDomains { + out.Permitted[i] = generalSubtree{Name: permitted} + } + ret[n].Value, err = asn1.Marshal(out) + if err != nil { + return + } + n++ + } + + // Adding another extension here? Remember to update the maximum number + // of elements in the make() at the top of the function. + + return ret[0:n], nil +} + +var ( + oidSHA1WithRSA = []int{1, 2, 840, 113549, 1, 1, 5} + oidRSA = []int{1, 2, 840, 113549, 1, 1, 1} +) + +// CreateSelfSignedCertificate creates a new certificate based on +// a template. The following members of template are used: SerialNumber, +// Subject, NotBefore, NotAfter, KeyUsage, BasicConstraintsValid, IsCA, +// MaxPathLen, SubjectKeyId, DNSNames, PermittedDNSDomainsCritical, +// PermittedDNSDomains. +// +// The certificate is signed by parent. If parent is equal to template then the +// certificate is self-signed. The parameter pub is the public key of the +// signee and priv is the private key of the signer. +// +// The returned slice is the certificate in DER encoding. +func CreateCertificate(rand io.Reader, template, parent *Certificate, pub *rsa.PublicKey, priv *rsa.PrivateKey) (cert []byte, err os.Error) { + asn1PublicKey, err := asn1.Marshal(rsaPublicKey{ + N: pub.N, + E: pub.E, + }) + if err != nil { + return + } + + if len(parent.SubjectKeyId) > 0 { + template.AuthorityKeyId = parent.SubjectKeyId + } + + extensions, err := buildExtensions(template) + if err != nil { + return + } + + encodedPublicKey := asn1.BitString{BitLength: len(asn1PublicKey) * 8, Bytes: asn1PublicKey} + c := tbsCertificate{ + Version: 2, + SerialNumber: template.SerialNumber, + SignatureAlgorithm: pkix.AlgorithmIdentifier{Algorithm: oidSHA1WithRSA}, + Issuer: parent.Subject.ToRDNSequence(), + Validity: validity{template.NotBefore, template.NotAfter}, + Subject: template.Subject.ToRDNSequence(), + PublicKey: publicKeyInfo{nil, pkix.AlgorithmIdentifier{Algorithm: oidRSA}, encodedPublicKey}, + Extensions: extensions, + } + + tbsCertContents, err := asn1.Marshal(c) + if err != nil { + return + } + + c.Raw = tbsCertContents + + h := sha1.New() + h.Write(tbsCertContents) + digest := h.Sum() + + signature, err := rsa.SignPKCS1v15(rand, priv, crypto.SHA1, digest) + if err != nil { + return + } + + cert, err = asn1.Marshal(certificate{ + nil, + c, + pkix.AlgorithmIdentifier{Algorithm: oidSHA1WithRSA}, + asn1.BitString{Bytes: signature, BitLength: len(signature) * 8}, + }) + return +} + +// pemCRLPrefix is the magic string that indicates that we have a PEM encoded +// CRL. +var pemCRLPrefix = []byte("-----BEGIN X509 CRL") +// pemType is the type of a PEM encoded CRL. +var pemType = "X509 CRL" + +// ParseCRL parses a CRL from the given bytes. It's often the case that PEM +// encoded CRLs will appear where they should be DER encoded, so this function +// will transparently handle PEM encoding as long as there isn't any leading +// garbage. +func ParseCRL(crlBytes []byte) (certList *pkix.CertificateList, err os.Error) { + if bytes.HasPrefix(crlBytes, pemCRLPrefix) { + block, _ := pem.Decode(crlBytes) + if block != nil && block.Type == pemType { + crlBytes = block.Bytes + } + } + return ParseDERCRL(crlBytes) +} + +// ParseDERCRL parses a DER encoded CRL from the given bytes. +func ParseDERCRL(derBytes []byte) (certList *pkix.CertificateList, err os.Error) { + certList = new(pkix.CertificateList) + _, err = asn1.Unmarshal(derBytes, certList) + if err != nil { + certList = nil + } + return +} + +// CreateCRL returns a DER encoded CRL, signed by this Certificate, that +// contains the given list of revoked certificates. +func (c *Certificate) CreateCRL(rand io.Reader, priv *rsa.PrivateKey, revokedCerts []pkix.RevokedCertificate, now, expiry *time.Time) (crlBytes []byte, err os.Error) { + tbsCertList := pkix.TBSCertificateList{ + Version: 2, + Signature: pkix.AlgorithmIdentifier{ + Algorithm: oidSignatureSHA1WithRSA, + }, + Issuer: c.Subject.ToRDNSequence(), + ThisUpdate: now, + NextUpdate: expiry, + RevokedCertificates: revokedCerts, + } + + tbsCertListContents, err := asn1.Marshal(tbsCertList) + if err != nil { + return + } + + h := sha1.New() + h.Write(tbsCertListContents) + digest := h.Sum() + + signature, err := rsa.SignPKCS1v15(rand, priv, crypto.SHA1, digest) + if err != nil { + return + } + + return asn1.Marshal(pkix.CertificateList{ + TBSCertList: tbsCertList, + SignatureAlgorithm: pkix.AlgorithmIdentifier{ + Algorithm: oidSignatureSHA1WithRSA, + }, + SignatureValue: asn1.BitString{Bytes: signature, BitLength: len(signature) * 8}, + }) +} diff --git a/src/pkg/crypto/x509/x509_test.go b/src/pkg/crypto/x509/x509_test.go new file mode 100644 index 000000000..dc216505e --- /dev/null +++ b/src/pkg/crypto/x509/x509_test.go @@ -0,0 +1,431 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package x509 + +import ( + "asn1" + "big" + "crypto/dsa" + "crypto/rand" + "crypto/rsa" + "crypto/x509/pkix" + "encoding/base64" + "encoding/hex" + "encoding/pem" + "testing" + "time" +) + +func TestParsePKCS1PrivateKey(t *testing.T) { + block, _ := pem.Decode([]byte(pemPrivateKey)) + priv, err := ParsePKCS1PrivateKey(block.Bytes) + if err != nil { + t.Errorf("Failed to parse private key: %s", err) + return + } + if priv.PublicKey.N.Cmp(rsaPrivateKey.PublicKey.N) != 0 || + priv.PublicKey.E != rsaPrivateKey.PublicKey.E || + priv.D.Cmp(rsaPrivateKey.D) != 0 || + priv.Primes[0].Cmp(rsaPrivateKey.Primes[0]) != 0 || + priv.Primes[1].Cmp(rsaPrivateKey.Primes[1]) != 0 { + t.Errorf("got:%+v want:%+v", priv, rsaPrivateKey) + } +} + +var pemPrivateKey = `-----BEGIN RSA PRIVATE KEY----- +MIIBOgIBAAJBALKZD0nEffqM1ACuak0bijtqE2QrI/KLADv7l3kK3ppMyCuLKoF0 +fd7Ai2KW5ToIwzFofvJcS/STa6HA5gQenRUCAwEAAQJBAIq9amn00aS0h/CrjXqu +/ThglAXJmZhOMPVn4eiu7/ROixi9sex436MaVeMqSNf7Ex9a8fRNfWss7Sqd9eWu +RTUCIQDasvGASLqmjeffBNLTXV2A5g4t+kLVCpsEIZAycV5GswIhANEPLmax0ME/ +EO+ZJ79TJKN5yiGBRsv5yvx5UiHxajEXAiAhAol5N4EUyq6I9w1rYdhPMGpLfk7A +IU2snfRJ6Nq2CQIgFrPsWRCkV+gOYcajD17rEqmuLrdIRexpg8N1DOSXoJ8CIGlS +tAboUGBxTDq3ZroNism3DaMIbKPyYrAqhKov1h5V +-----END RSA PRIVATE KEY----- +` + +func bigFromString(s string) *big.Int { + ret := new(big.Int) + ret.SetString(s, 10) + return ret +} + +func fromBase10(base10 string) *big.Int { + i := new(big.Int) + i.SetString(base10, 10) + return i +} + +func bigFromHexString(s string) *big.Int { + ret := new(big.Int) + ret.SetString(s, 16) + return ret +} + +var rsaPrivateKey = &rsa.PrivateKey{ + PublicKey: rsa.PublicKey{ + N: bigFromString("9353930466774385905609975137998169297361893554149986716853295022578535724979677252958524466350471210367835187480748268864277464700638583474144061408845077"), + E: 65537, + }, + D: bigFromString("7266398431328116344057699379749222532279343923819063639497049039389899328538543087657733766554155839834519529439851673014800261285757759040931985506583861"), + Primes: []*big.Int{ + bigFromString("98920366548084643601728869055592650835572950932266967461790948584315647051443"), + bigFromString("94560208308847015747498523884063394671606671904944666360068158221458669711639"), + }, +} + +func TestMarshalRSAPrivateKey(t *testing.T) { + priv := &rsa.PrivateKey{ + PublicKey: rsa.PublicKey{ + N: fromBase10("16346378922382193400538269749936049106320265317511766357599732575277382844051791096569333808598921852351577762718529818072849191122419410612033592401403764925096136759934497687765453905884149505175426053037420486697072448609022753683683718057795566811401938833367954642951433473337066311978821180526439641496973296037000052546108507805269279414789035461158073156772151892452251106173507240488993608650881929629163465099476849643165682709047462010581308719577053905787496296934240246311806555924593059995202856826239801816771116902778517096212527979497399966526283516447337775509777558018145573127308919204297111496233"), + E: 3, + }, + D: fromBase10("10897585948254795600358846499957366070880176878341177571733155050184921896034527397712889205732614568234385175145686545381899460748279607074689061600935843283397424506622998458510302603922766336783617368686090042765718290914099334449154829375179958369993407724946186243249568928237086215759259909861748642124071874879861299389874230489928271621259294894142840428407196932444474088857746123104978617098858619445675532587787023228852383149557470077802718705420275739737958953794088728369933811184572620857678792001136676902250566845618813972833750098806496641114644760255910789397593428910198080271317419213080834885003"), + Primes: []*big.Int{ + fromBase10("1025363189502892836833747188838978207017355117492483312747347695538428729137306368764177201532277413433182799108299960196606011786562992097313508180436744488171474690412562218914213688661311117337381958560443"), + fromBase10("3467903426626310123395340254094941045497208049900750380025518552334536945536837294961497712862519984786362199788654739924501424784631315081391467293694361474867825728031147665777546570788493758372218019373"), + fromBase10("4597024781409332673052708605078359346966325141767460991205742124888960305710298765592730135879076084498363772408626791576005136245060321874472727132746643162385746062759369754202494417496879741537284589047"), + }, + } + + derBytes := MarshalPKCS1PrivateKey(priv) + + priv2, err := ParsePKCS1PrivateKey(derBytes) + if err != nil { + t.Errorf("error parsing serialized key: %s", err) + return + } + if priv.PublicKey.N.Cmp(priv2.PublicKey.N) != 0 || + priv.PublicKey.E != priv2.PublicKey.E || + priv.D.Cmp(priv2.D) != 0 || + len(priv2.Primes) != 3 || + priv.Primes[0].Cmp(priv2.Primes[0]) != 0 || + priv.Primes[1].Cmp(priv2.Primes[1]) != 0 || + priv.Primes[2].Cmp(priv2.Primes[2]) != 0 { + t.Errorf("got:%+v want:%+v", priv, priv2) + } +} + +type matchHostnamesTest struct { + pattern, host string + ok bool +} + +var matchHostnamesTests = []matchHostnamesTest{ + {"a.b.c", "a.b.c", true}, + {"a.b.c", "b.b.c", false}, + {"", "b.b.c", false}, + {"a.b.c", "", false}, + {"example.com", "example.com", true}, + {"example.com", "www.example.com", false}, + {"*.example.com", "www.example.com", true}, + {"*.example.com", "xyz.www.example.com", false}, + {"*.*.example.com", "xyz.www.example.com", true}, + {"*.www.*.com", "xyz.www.example.com", true}, +} + +func TestMatchHostnames(t *testing.T) { + for i, test := range matchHostnamesTests { + r := matchHostnames(test.pattern, test.host) + if r != test.ok { + t.Errorf("#%d mismatch got: %t want: %t", i, r, test.ok) + } + } +} + +func TestCertificateParse(t *testing.T) { + s, _ := hex.DecodeString(certBytes) + certs, err := ParseCertificates(s) + if err != nil { + t.Error(err) + } + if len(certs) != 2 { + t.Errorf("Wrong number of certs: got %d want 2", len(certs)) + return + } + + err = certs[0].CheckSignatureFrom(certs[1]) + if err != nil { + t.Error(err) + } + + if err := certs[0].VerifyHostname("mail.google.com"); err != nil { + t.Error(err) + } +} + +var certBytes = "308203223082028ba00302010202106edf0d9499fd4533dd1297fc42a93be1300d06092a864886" + + "f70d0101050500304c310b3009060355040613025a4131253023060355040a131c546861777465" + + "20436f6e73756c74696e67202850747929204c74642e311630140603550403130d546861777465" + + "20534743204341301e170d3039303332353136343932395a170d3130303332353136343932395a" + + "3069310b3009060355040613025553311330110603550408130a43616c69666f726e6961311630" + + "140603550407130d4d6f756e7461696e205669657731133011060355040a130a476f6f676c6520" + + "496e63311830160603550403130f6d61696c2e676f6f676c652e636f6d30819f300d06092a8648" + + "86f70d010101050003818d0030818902818100c5d6f892fccaf5614b064149e80a2c9581a218ef" + + "41ec35bd7a58125ae76f9ea54ddc893abbeb029f6b73616bf0ffd868791fba7af9c4aebf3706ba" + + "3eeaeed27435b4ddcfb157c05f351d66aa87fee0de072d66d773affbd36ab78bef090e0cc861a9" + + "03ac90dd98b51c9c41566c017f0beec3bff391051ffba0f5cc6850ad2a590203010001a381e730" + + "81e430280603551d250421301f06082b0601050507030106082b06010505070302060960864801" + + "86f842040130360603551d1f042f302d302ba029a0278625687474703a2f2f63726c2e74686177" + + "74652e636f6d2f54686177746553474343412e63726c307206082b060105050701010466306430" + + "2206082b060105050730018616687474703a2f2f6f6373702e7468617774652e636f6d303e0608" + + "2b060105050730028632687474703a2f2f7777772e7468617774652e636f6d2f7265706f736974" + + "6f72792f5468617774655f5347435f43412e637274300c0603551d130101ff04023000300d0609" + + "2a864886f70d01010505000381810062f1f3050ebc105e497c7aedf87e24d2f4a986bb3b837bd1" + + "9b91ebcad98b065992f6bd2b49b7d6d3cb2e427a99d606c7b1d46352527fac39e6a8b6726de5bf" + + "70212a52cba07634a5e332011bd1868e78eb5e3c93cf03072276786f207494feaa0ed9d53b2110" + + "a76571f90209cdae884385c882587030ee15f33d761e2e45a6bc308203233082028ca003020102" + + "020430000002300d06092a864886f70d0101050500305f310b3009060355040613025553311730" + + "15060355040a130e566572695369676e2c20496e632e31373035060355040b132e436c61737320" + + "33205075626c6963205072696d6172792043657274696669636174696f6e20417574686f726974" + + "79301e170d3034303531333030303030305a170d3134303531323233353935395a304c310b3009" + + "060355040613025a4131253023060355040a131c54686177746520436f6e73756c74696e672028" + + "50747929204c74642e311630140603550403130d5468617774652053474320434130819f300d06" + + "092a864886f70d010101050003818d0030818902818100d4d367d08d157faecd31fe7d1d91a13f" + + "0b713cacccc864fb63fc324b0794bd6f80ba2fe10493c033fc093323e90b742b71c403c6d2cde2" + + "2ff50963cdff48a500bfe0e7f388b72d32de9836e60aad007bc4644a3b847503f270927d0e62f5" + + "21ab693684317590f8bfc76c881b06957cc9e5a8de75a12c7a68dfd5ca1c875860190203010001" + + "a381fe3081fb30120603551d130101ff040830060101ff020100300b0603551d0f040403020106" + + "301106096086480186f842010104040302010630280603551d110421301fa41d301b3119301706" + + "035504031310507269766174654c6162656c332d313530310603551d1f042a30283026a024a022" + + "8620687474703a2f2f63726c2e766572697369676e2e636f6d2f706361332e63726c303206082b" + + "0601050507010104263024302206082b060105050730018616687474703a2f2f6f6373702e7468" + + "617774652e636f6d30340603551d25042d302b06082b0601050507030106082b06010505070302" + + "06096086480186f8420401060a6086480186f845010801300d06092a864886f70d010105050003" + + "81810055ac63eadea1ddd2905f9f0bce76be13518f93d9052bc81b774bad6950a1eededcfddb07" + + "e9e83994dcab72792f06bfab8170c4a8edea5334edef1e53d906c7562bd15cf4d18a8eb42bb137" + + "9048084225c53e8acb7feb6f04d16dc574a2f7a27c7b603c77cd0ece48027f012fb69b37e02a2a" + + "36dcd585d6ace53f546f961e05af" + +func TestCreateSelfSignedCertificate(t *testing.T) { + random := rand.Reader + + block, _ := pem.Decode([]byte(pemPrivateKey)) + priv, err := ParsePKCS1PrivateKey(block.Bytes) + if err != nil { + t.Errorf("Failed to parse private key: %s", err) + return + } + + template := Certificate{ + SerialNumber: big.NewInt(1), + Subject: pkix.Name{ + CommonName: "test.example.com", + Organization: []string{"Acme Co"}, + }, + NotBefore: time.SecondsToUTC(1000), + NotAfter: time.SecondsToUTC(100000), + + SubjectKeyId: []byte{1, 2, 3, 4}, + KeyUsage: KeyUsageCertSign, + + BasicConstraintsValid: true, + IsCA: true, + DNSNames: []string{"test.example.com"}, + + PolicyIdentifiers: []asn1.ObjectIdentifier{[]int{1, 2, 3}}, + PermittedDNSDomains: []string{".example.com", "example.com"}, + } + + derBytes, err := CreateCertificate(random, &template, &template, &priv.PublicKey, priv) + if err != nil { + t.Errorf("Failed to create certificate: %s", err) + return + } + + cert, err := ParseCertificate(derBytes) + if err != nil { + t.Errorf("Failed to parse certificate: %s", err) + return + } + + if len(cert.PolicyIdentifiers) != 1 || !cert.PolicyIdentifiers[0].Equal(template.PolicyIdentifiers[0]) { + t.Errorf("Failed to parse policy identifiers: got:%#v want:%#v", cert.PolicyIdentifiers, template.PolicyIdentifiers) + } + + if len(cert.PermittedDNSDomains) != 2 || cert.PermittedDNSDomains[0] != ".example.com" || cert.PermittedDNSDomains[1] != "example.com" { + t.Errorf("Failed to parse name constraints: %#v", cert.PermittedDNSDomains) + } + + err = cert.CheckSignatureFrom(cert) + if err != nil { + t.Errorf("Signature verification failed: %s", err) + return + } +} + +// Self-signed certificate using DSA with SHA1 +var dsaCertPem = `-----BEGIN CERTIFICATE----- +MIIEDTCCA82gAwIBAgIJALHPghaoxeDhMAkGByqGSM44BAMweTELMAkGA1UEBhMC +VVMxCzAJBgNVBAgTAk5DMQ8wDQYDVQQHEwZOZXd0b24xFDASBgNVBAoTC0dvb2ds +ZSwgSW5jMRIwEAYDVQQDEwlKb24gQWxsaWUxIjAgBgkqhkiG9w0BCQEWE2pvbmFs +bGllQGdvb2dsZS5jb20wHhcNMTEwNTE0MDMwMTQ1WhcNMTEwNjEzMDMwMTQ1WjB5 +MQswCQYDVQQGEwJVUzELMAkGA1UECBMCTkMxDzANBgNVBAcTBk5ld3RvbjEUMBIG +A1UEChMLR29vZ2xlLCBJbmMxEjAQBgNVBAMTCUpvbiBBbGxpZTEiMCAGCSqGSIb3 +DQEJARYTam9uYWxsaWVAZ29vZ2xlLmNvbTCCAbcwggEsBgcqhkjOOAQBMIIBHwKB +gQC8hLUnQ7FpFYu4WXTj6DKvXvz8QrJkNJCVMTpKAT7uBpobk32S5RrPKXocd4gN +8lyGB9ggS03EVlEwXvSmO0DH2MQtke2jl9j1HLydClMf4sbx5V6TV9IFw505U1iW +jL7awRMgxge+FsudtJK254FjMFo03ZnOQ8ZJJ9E6AEDrlwIVAJpnBn9moyP11Ox5 +Asc/5dnjb6dPAoGBAJFHd4KVv1iTVCvEG6gGiYop5DJh28hUQcN9kul+2A0yPUSC +X93oN00P8Vh3eYgSaCWZsha7zDG53MrVJ0Zf6v/X/CoZNhLldeNOepivTRAzn+Rz +kKUYy5l1sxYLHQKF0UGNCXfFKZT0PCmgU+PWhYNBBMn6/cIh44vp85ideo5CA4GE +AAKBgFmifCafzeRaohYKXJgMGSEaggCVCRq5xdyDCat+wbOkjC4mfG01/um3G8u5 +LxasjlWRKTR/tcAL7t0QuokVyQaYdVypZXNaMtx1db7YBuHjj3aP+8JOQRI9xz8c +bp5NDJ5pISiFOv4p3GZfqZPcqckDt78AtkQrmnal2txhhjF6o4HeMIHbMB0GA1Ud +DgQWBBQVyyr7hO11ZFFpWX50298Sa3V+rzCBqwYDVR0jBIGjMIGggBQVyyr7hO11 +ZFFpWX50298Sa3V+r6F9pHsweTELMAkGA1UEBhMCVVMxCzAJBgNVBAgTAk5DMQ8w +DQYDVQQHEwZOZXd0b24xFDASBgNVBAoTC0dvb2dsZSwgSW5jMRIwEAYDVQQDEwlK +b24gQWxsaWUxIjAgBgkqhkiG9w0BCQEWE2pvbmFsbGllQGdvb2dsZS5jb22CCQCx +z4IWqMXg4TAMBgNVHRMEBTADAQH/MAkGByqGSM44BAMDLwAwLAIUPtn/5j8Q1jJI +7ggOIsgrhgUdjGQCFCsmDq1H11q9+9Wp9IMeGrTSKHIM +-----END CERTIFICATE----- +` + +func TestParseCertificateWithDsaPublicKey(t *testing.T) { + expectedKey := &dsa.PublicKey{ + Parameters: dsa.Parameters{ + P: bigFromHexString("00BC84B52743B169158BB85974E3E832AF5EFCFC42B264349095313A4A013EEE069A1B937D92E51ACF297A1C77880DF25C8607D8204B4DC45651305EF4A63B40C7D8C42D91EDA397D8F51CBC9D0A531FE2C6F1E55E9357D205C39D395358968CBEDAC11320C607BE16CB9DB492B6E78163305A34DD99CE43C64927D13A0040EB97"), + Q: bigFromHexString("009A67067F66A323F5D4EC7902C73FE5D9E36FA74F"), + G: bigFromHexString("009147778295BF5893542BC41BA806898A29E43261DBC85441C37D92E97ED80D323D44825FDDE8374D0FF15877798812682599B216BBCC31B9DCCAD527465FEAFFD7FC2A193612E575E34E7A98AF4D10339FE47390A518CB9975B3160B1D0285D1418D0977C52994F43C29A053E3D685834104C9FAFDC221E38BE9F3989D7A8E42"), + }, + Y: bigFromHexString("59A27C269FCDE45AA2160A5C980C19211A820095091AB9C5DC8309AB7EC1B3A48C2E267C6D35FEE9B71BCBB92F16AC8E559129347FB5C00BEEDD10BA8915C90698755CA965735A32DC7575BED806E1E38F768FFBC24E41123DC73F1C6E9E4D0C9E692128853AFE29DC665FA993DCA9C903B7BF00B6442B9A76A5DADC6186317A"), + } + pemBlock, _ := pem.Decode([]byte(dsaCertPem)) + cert, err := ParseCertificate(pemBlock.Bytes) + if err != nil { + t.Fatalf("Failed to parse certificate: %s", err) + } + if cert.PublicKeyAlgorithm != DSA { + t.Errorf("Parsed key algorithm was not DSA") + } + parsedKey, ok := cert.PublicKey.(*dsa.PublicKey) + if !ok { + t.Fatalf("Parsed key was not a DSA key: %s", err) + } + if expectedKey.Y.Cmp(parsedKey.Y) != 0 || + expectedKey.P.Cmp(parsedKey.P) != 0 || + expectedKey.Q.Cmp(parsedKey.Q) != 0 || + expectedKey.G.Cmp(parsedKey.G) != 0 { + t.Fatal("Parsed key differs from expected key") + } +} + +func TestParseCertificateWithDSASignatureAlgorithm(t *testing.T) { + pemBlock, _ := pem.Decode([]byte(dsaCertPem)) + cert, err := ParseCertificate(pemBlock.Bytes) + if err != nil { + t.Fatalf("Failed to parse certificate: %s", err) + } + if cert.SignatureAlgorithm != DSAWithSHA1 { + t.Errorf("Parsed signature algorithm was not DSAWithSHA1") + } +} + +func TestVerifyCertificateWithDSASignature(t *testing.T) { + pemBlock, _ := pem.Decode([]byte(dsaCertPem)) + cert, err := ParseCertificate(pemBlock.Bytes) + if err != nil { + t.Fatalf("Failed to parse certificate: %s", err) + } + // test cert is self-signed + if err = cert.CheckSignatureFrom(cert); err != nil { + t.Fatalf("DSA Certificate verfication failed: %s", err) + } +} + +const pemCertificate = `-----BEGIN CERTIFICATE----- +MIIB5DCCAZCgAwIBAgIBATALBgkqhkiG9w0BAQUwLTEQMA4GA1UEChMHQWNtZSBDbzEZMBcGA1UE +AxMQdGVzdC5leGFtcGxlLmNvbTAeFw03MDAxMDEwMDE2NDBaFw03MDAxMDIwMzQ2NDBaMC0xEDAO +BgNVBAoTB0FjbWUgQ28xGTAXBgNVBAMTEHRlc3QuZXhhbXBsZS5jb20wWjALBgkqhkiG9w0BAQED +SwAwSAJBALKZD0nEffqM1ACuak0bijtqE2QrI/KLADv7l3kK3ppMyCuLKoF0fd7Ai2KW5ToIwzFo +fvJcS/STa6HA5gQenRUCAwEAAaOBnjCBmzAOBgNVHQ8BAf8EBAMCAAQwDwYDVR0TAQH/BAUwAwEB +/zANBgNVHQ4EBgQEAQIDBDAPBgNVHSMECDAGgAQBAgMEMBsGA1UdEQQUMBKCEHRlc3QuZXhhbXBs +ZS5jb20wDwYDVR0gBAgwBjAEBgIqAzAqBgNVHR4EIzAhoB8wDoIMLmV4YW1wbGUuY29tMA2CC2V4 +YW1wbGUuY29tMAsGCSqGSIb3DQEBBQNBAHKZKoS1wEQOGhgklx4+/yFYQlnqwKXvar/ZecQvJwui +0seMQnwBhwdBkHfVIU2Fu5VUMRyxlf0ZNaDXcpU581k= +-----END CERTIFICATE-----` + +func TestCRLCreation(t *testing.T) { + block, _ := pem.Decode([]byte(pemPrivateKey)) + priv, _ := ParsePKCS1PrivateKey(block.Bytes) + block, _ = pem.Decode([]byte(pemCertificate)) + cert, _ := ParseCertificate(block.Bytes) + + now := time.SecondsToUTC(1000) + expiry := time.SecondsToUTC(10000) + + revokedCerts := []pkix.RevokedCertificate{ + { + SerialNumber: big.NewInt(1), + RevocationTime: now, + }, + { + SerialNumber: big.NewInt(42), + RevocationTime: now, + }, + } + + crlBytes, err := cert.CreateCRL(rand.Reader, priv, revokedCerts, now, expiry) + if err != nil { + t.Errorf("error creating CRL: %s", err) + } + + _, err = ParseDERCRL(crlBytes) + if err != nil { + t.Errorf("error reparsing CRL: %s", err) + } +} + +func fromBase64(in string) []byte { + out := make([]byte, base64.StdEncoding.DecodedLen(len(in))) + _, err := base64.StdEncoding.Decode(out, []byte(in)) + if err != nil { + panic("failed to base64 decode") + } + return out +} + +func TestParseDERCRL(t *testing.T) { + derBytes := fromBase64(derCRLBase64) + certList, err := ParseDERCRL(derBytes) + if err != nil { + t.Errorf("error parsing: %s", err) + return + } + numCerts := len(certList.TBSCertList.RevokedCertificates) + expected := 88 + if numCerts != expected { + t.Errorf("bad number of revoked certificates. got: %d want: %d", numCerts, expected) + } + + if certList.HasExpired(1302517272) { + t.Errorf("CRL has expired (but shouldn't have)") + } + + // Can't check the signature here without a package cycle. +} + +func TestParsePEMCRL(t *testing.T) { + pemBytes := fromBase64(pemCRLBase64) + certList, err := ParseCRL(pemBytes) + if err != nil { + t.Errorf("error parsing: %s", err) + return + } + numCerts := len(certList.TBSCertList.RevokedCertificates) + expected := 2 + if numCerts != expected { + t.Errorf("bad number of revoked certificates. got: %d want: %d", numCerts, expected) + } + + if certList.HasExpired(1302517272) { + t.Errorf("CRL has expired (but shouldn't have)") + } + + // Can't check the signature here without a package cycle. +} + +const derCRLBase64 = "MIINqzCCDJMCAQEwDQYJKoZIhvcNAQEFBQAwVjEZMBcGA1UEAxMQUEtJIEZJTk1FQ0NBTklDQTEVMBMGA1UEChMMRklOTUVDQ0FOSUNBMRUwEwYDVQQLEwxGSU5NRUNDQU5JQ0ExCzAJBgNVBAYTAklUFw0xMTA1MDQxNjU3NDJaFw0xMTA1MDQyMDU3NDJaMIIMBzAhAg4Ze1od49Lt1qIXBydAzhcNMDkwNzE2MDg0MzIyWjAAMCECDl0HSL9bcZ1Ci/UHJ0DPFw0wOTA3MTYwODQzMTNaMAAwIQIOESB9tVAmX3cY7QcnQNAXDTA5MDcxNjA4NDUyMlowADAhAg4S1tGAQ3mHt8uVBydA1RcNMDkwODA0MTUyNTIyWjAAMCECDlQ249Y7vtC25ScHJ0DWFw0wOTA4MDQxNTI1MzdaMAAwIQIOISMop3NkA4PfYwcnQNkXDTA5MDgwNDExMDAzNFowADAhAg56/BMoS29KEShTBydA2hcNMDkwODA0MTEwMTAzWjAAMCECDnBp/22HPH5CSWoHJ0DbFw0wOTA4MDQxMDU0NDlaMAAwIQIOV9IP+8CD8bK+XAcnQNwXDTA5MDgwNDEwNTcxN1owADAhAg4v5aRz0IxWqYiXBydA3RcNMDkwODA0MTA1NzQ1WjAAMCECDlOU34VzvZAybQwHJ0DeFw0wOTA4MDQxMDU4MjFaMAAwIAINO4CD9lluIxcwBydBAxcNMDkwNzIyMTUzMTU5WjAAMCECDgOllfO8Y1QA7/wHJ0ExFw0wOTA3MjQxMTQxNDNaMAAwIQIOJBX7jbiCdRdyjgcnQUQXDTA5MDkxNjA5MzAwOFowADAhAg5iYSAgmDrlH/RZBydBRRcNMDkwOTE2MDkzMDE3WjAAMCECDmu6k6srP3jcMaQHJ0FRFw0wOTA4MDQxMDU2NDBaMAAwIQIOX8aHlO0V+WVH4QcnQVMXDTA5MDgwNDEwNTcyOVowADAhAg5flK2rg3NnsRgDBydBzhcNMTEwMjAxMTUzMzQ2WjAAMCECDg35yJDL1jOPTgoHJ0HPFw0xMTAyMDExNTM0MjZaMAAwIQIOMyFJ6+e9iiGVBQcnQdAXDTA5MDkxODEzMjAwNVowADAhAg5Emb/Oykucmn8fBydB1xcNMDkwOTIxMTAxMDQ3WjAAMCECDjQKCncV+MnUavMHJ0HaFw0wOTA5MjIwODE1MjZaMAAwIQIOaxiFUt3dpd+tPwcnQfQXDTEwMDYxODA4NDI1MVowADAhAg5G7P8nO0tkrMt7BydB9RcNMTAwNjE4MDg0MjMwWjAAMCECDmTCC3SXhmDRst4HJ0H2Fw0wOTA5MjgxMjA3MjBaMAAwIQIOHoGhUr/pRwzTKgcnQfcXDTA5MDkyODEyMDcyNFowADAhAg50wrcrCiw8mQmPBydCBBcNMTAwMjE2MTMwMTA2WjAAMCECDifWmkvwyhEqwEcHJ0IFFw0xMDAyMTYxMzAxMjBaMAAwIQIOfgPmlW9fg+osNgcnQhwXDTEwMDQxMzA5NTIwMFowADAhAg4YHAGuA6LgCk7tBydCHRcNMTAwNDEzMDk1MTM4WjAAMCECDi1zH1bxkNJhokAHJ0IsFw0xMDA0MTMwOTU5MzBaMAAwIQIOMipNccsb/wo2fwcnQi0XDTEwMDQxMzA5NTkwMFowADAhAg46lCmvPl4GpP6ABydCShcNMTAwMTE5MDk1MjE3WjAAMCECDjaTcaj+wBpcGAsHJ0JLFw0xMDAxMTkwOTUyMzRaMAAwIQIOOMC13EOrBuxIOQcnQloXDTEwMDIwMTA5NDcwNVowADAhAg5KmZl+krz4RsmrBydCWxcNMTAwMjAxMDk0NjQwWjAAMCECDmLG3zQJ/fzdSsUHJ0JiFw0xMDAzMDEwOTUxNDBaMAAwIQIOP39ksgHdojf4owcnQmMXDTEwMDMwMTA5NTExN1owADAhAg4LDQzvWNRlD6v9BydCZBcNMTAwMzAxMDk0NjIyWjAAMCECDkmNfeclaFhIaaUHJ0JlFw0xMDAzMDEwOTQ2MDVaMAAwIQIOT/qWWfpH/m8NTwcnQpQXDTEwMDUxMTA5MTgyMVowADAhAg5m/ksYxvCEgJSvBydClRcNMTAwNTExMDkxODAxWjAAMCECDgvf3Ohq6JOPU9AHJ0KWFw0xMDA1MTEwOTIxMjNaMAAwIQIOKSPas10z4jNVIQcnQpcXDTEwMDUxMTA5MjEwMlowADAhAg4mCWmhoZ3lyKCDBydCohcNMTEwNDI4MTEwMjI1WjAAMCECDkeiyRsBMK0Gvr4HJ0KjFw0xMTA0MjgxMTAyMDdaMAAwIQIOa09b/nH2+55SSwcnQq4XDTExMDQwMTA4Mjk0NlowADAhAg5O7M7iq7gGplr1BydCrxcNMTEwNDAxMDgzMDE3WjAAMCECDjlT6mJxUjTvyogHJ0K1Fw0xMTAxMjcxNTQ4NTJaMAAwIQIODS/l4UUFLe21NAcnQrYXDTExMDEyNzE1NDgyOFowADAhAg5lPRA0XdOUF6lSBydDHhcNMTEwMTI4MTQzNTA1WjAAMCECDixKX4fFGGpENwgHJ0MfFw0xMTAxMjgxNDM1MzBaMAAwIQIORNBkqsPnpKTtbAcnQ08XDTEwMDkwOTA4NDg0MlowADAhAg5QL+EMM3lohedEBydDUBcNMTAwOTA5MDg0ODE5WjAAMCECDlhDnHK+HiTRAXcHJ0NUFw0xMDEwMTkxNjIxNDBaMAAwIQIOdBFqAzq/INz53gcnQ1UXDTEwMTAxOTE2MjA0NFowADAhAg4OjR7s8MgKles1BydDWhcNMTEwMTI3MTY1MzM2WjAAMCECDmfR/elHee+d0SoHJ0NbFw0xMTAxMjcxNjUzNTZaMAAwIQIOBTKv2ui+KFMI+wcnQ5YXDTEwMDkxNTEwMjE1N1owADAhAg49F3c/GSah+oRUBydDmxcNMTEwMTI3MTczMjMzWjAAMCECDggv4I61WwpKFMMHJ0OcFw0xMTAxMjcxNzMyNTVaMAAwIQIOXx/Y8sEvwS10LAcnQ6UXDTExMDEyODExMjkzN1owADAhAg5LSLbnVrSKaw/9BydDphcNMTEwMTI4MTEyOTIwWjAAMCECDmFFoCuhKUeACQQHJ0PfFw0xMTAxMTExMDE3MzdaMAAwIQIOQTDdFh2fSPF6AAcnQ+AXDTExMDExMTEwMTcxMFowADAhAg5B8AOXX61FpvbbBydD5RcNMTAxMDA2MTAxNDM2WjAAMCECDh41P2Gmi7PkwI4HJ0PmFw0xMDEwMDYxMDE2MjVaMAAwIQIOWUHGLQCd+Ale9gcnQ/0XDTExMDUwMjA3NTYxMFowADAhAg5Z2c9AYkikmgWOBydD/hcNMTEwNTAyMDc1NjM0WjAAMCECDmf/UD+/h8nf+74HJ0QVFw0xMTA0MTUwNzI4MzNaMAAwIQIOICvj4epy3MrqfwcnRBYXDTExMDQxNTA3Mjg1NlowADAhAg4bouRMfOYqgv4xBydEHxcNMTEwMzA4MTYyNDI1WjAAMCECDhebWHGoKiTp7pEHJ0QgFw0xMTAzMDgxNjI0NDhaMAAwIQIOX+qnxxAqJ8LtawcnRDcXDTExMDEzMTE1MTIyOFowADAhAg4j0fICqZ+wkOdqBydEOBcNMTEwMTMxMTUxMTQxWjAAMCECDhmXjsV4SUpWtAMHJ0RLFw0xMTAxMjgxMTI0MTJaMAAwIQIODno/w+zG43kkTwcnREwXDTExMDEyODExMjM1MlowADAhAg4b1gc88767Fr+LBydETxcNMTEwMTI4MTEwMjA4WjAAMCECDn+M3Pa1w2nyFeUHJ0RQFw0xMTAxMjgxMDU4NDVaMAAwIQIOaduoyIH61tqybAcnRJUXDTEwMTIxNTA5NDMyMlowADAhAg4nLqQPkyi3ESAKBydElhcNMTAxMjE1MDk0MzM2WjAAMCECDi504NIMH8578gQHJ0SbFw0xMTAyMTQxNDA1NDFaMAAwIQIOGuaM8PDaC5u1egcnRJwXDTExMDIxNDE0MDYwNFowADAhAg4ehYq/BXGnB5PWBydEnxcNMTEwMjA0MDgwOTUxWjAAMCECDkSD4eS4FxW5H20HJ0SgFw0xMTAyMDQwODA5MjVaMAAwIQIOOCcb6ilYObt1egcnRKEXDTExMDEyNjEwNDEyOVowADAhAg58tISWCCwFnKGnBydEohcNMTEwMjA0MDgxMzQyWjAAMCECDn5rjtabY/L/WL0HJ0TJFw0xMTAyMDQxMTAzNDFaMAAwDQYJKoZIhvcNAQEFBQADggEBAGnF2Gs0+LNiYCW1Ipm83OXQYP/bd5tFFRzyz3iepFqNfYs4D68/QihjFoRHQoXEB0OEe1tvaVnnPGnEOpi6krwekquMxo4H88B5SlyiFIqemCOIss0SxlCFs69LmfRYvPPvPEhoXtQ3ZThe0UvKG83GOklhvGl6OaiRf4Mt+m8zOT4Wox/j6aOBK6cw6qKCdmD+Yj1rrNqFGg1CnSWMoD6S6mwNgkzwdBUJZ22BwrzAAo4RHa2Uy3ef1FjwD0XtU5N3uDSxGGBEDvOe5z82rps3E22FpAA8eYl8kaXtmWqyvYU0epp4brGuTxCuBMCAsxt/OjIjeNNQbBGkwxgfYA0=" + +const pemCRLBase64 = "LS0tLS1CRUdJTiBYNTA5IENSTC0tLS0tDQpNSUlCOWpDQ0FWOENBUUV3RFFZSktvWklodmNOQVFFRkJRQXdiREVhTUJnR0ExVUVDaE1SVWxOQklGTmxZM1Z5DQphWFI1SUVsdVl5NHhIakFjQmdOVkJBTVRGVkpUUVNCUWRXSnNhV01nVW05dmRDQkRRU0IyTVRFdU1Dd0dDU3FHDQpTSWIzRFFFSkFSWWZjbk5oYTJWdmJuSnZiM1J6YVdkdVFISnpZWE5sWTNWeWFYUjVMbU52YlJjTk1URXdNakl6DQpNVGt5T0RNd1doY05NVEV3T0RJeU1Ua3lPRE13V2pDQmpEQktBaEVBckRxb2g5RkhKSFhUN09QZ3V1bjQrQmNODQpNRGt4TVRBeU1UUXlOekE1V2pBbU1Bb0dBMVVkRlFRRENnRUpNQmdHQTFVZEdBUVJHQTh5TURBNU1URXdNakUwDQpNalExTlZvd1BnSVJBTEd6blowOTVQQjVhQU9MUGc1N2ZNTVhEVEF5TVRBeU16RTBOVEF4TkZvd0dqQVlCZ05WDQpIUmdFRVJnUE1qQXdNakV3TWpNeE5EVXdNVFJhb0RBd0xqQWZCZ05WSFNNRUdEQVdnQlQxVERGNlVRTS9MTmVMDQpsNWx2cUhHUXEzZzltekFMQmdOVkhSUUVCQUlDQUlRd0RRWUpLb1pJaHZjTkFRRUZCUUFEZ1lFQUZVNUFzNk16DQpxNVBSc2lmYW9iUVBHaDFhSkx5QytNczVBZ2MwYld5QTNHQWR4dXI1U3BQWmVSV0NCamlQL01FSEJXSkNsQkhQDQpHUmNxNXlJZDNFakRrYUV5eFJhK2k2N0x6dmhJNmMyOUVlNks5cFNZd2ppLzdSVWhtbW5Qclh0VHhsTDBsckxyDQptUVFKNnhoRFJhNUczUUE0Q21VZHNITnZicnpnbUNZcHZWRT0NCi0tLS0tRU5EIFg1MDkgQ1JMLS0tLS0NCg0K" diff --git a/src/pkg/crypto/xtea/Makefile b/src/pkg/crypto/xtea/Makefile new file mode 100644 index 000000000..301621168 --- /dev/null +++ b/src/pkg/crypto/xtea/Makefile @@ -0,0 +1,12 @@ +# Copyright 2009 The Go Authors. All rights reserved. +# Use of this source code is governed by a BSD-style +# license that can be found in the LICENSE file. + +include ../../../Make.inc + +TARG=crypto/xtea +GOFILES=\ + cipher.go\ + block.go\ + +include ../../../Make.pkg diff --git a/src/pkg/crypto/xtea/block.go b/src/pkg/crypto/xtea/block.go new file mode 100644 index 000000000..bf5d24599 --- /dev/null +++ b/src/pkg/crypto/xtea/block.go @@ -0,0 +1,66 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +/* + Implementation adapted from Needham and Wheeler's paper: + http://www.cix.co.uk/~klockstone/xtea.pdf + + A precalculated look up table is used during encryption/decryption for values that are based purely on the key. +*/ + +package xtea + +// XTEA is based on 64 rounds. +const numRounds = 64 + +// blockToUint32 reads an 8 byte slice into two uint32s. +// The block is treated as big endian. +func blockToUint32(src []byte) (uint32, uint32) { + r0 := uint32(src[0])<<24 | uint32(src[1])<<16 | uint32(src[2])<<8 | uint32(src[3]) + r1 := uint32(src[4])<<24 | uint32(src[5])<<16 | uint32(src[6])<<8 | uint32(src[7]) + return r0, r1 +} + +// uint32ToBlock writes two uint32s into an 8 byte data block. +// Values are written as big endian. +func uint32ToBlock(v0, v1 uint32, dst []byte) { + dst[0] = byte(v0 >> 24) + dst[1] = byte(v0 >> 16) + dst[2] = byte(v0 >> 8) + dst[3] = byte(v0) + dst[4] = byte(v1 >> 24) + dst[5] = byte(v1 >> 16) + dst[6] = byte(v1 >> 8) + dst[7] = byte(v1 >> 0) +} + +// encryptBlock encrypts a single 8 byte block using XTEA. +func encryptBlock(c *Cipher, dst, src []byte) { + v0, v1 := blockToUint32(src) + + // Two rounds of XTEA applied per loop + for i := 0; i < numRounds; { + v0 += ((v1<<4 ^ v1>>5) + v1) ^ c.table[i] + i++ + v1 += ((v0<<4 ^ v0>>5) + v0) ^ c.table[i] + i++ + } + + uint32ToBlock(v0, v1, dst) +} + +// decryptBlock decrypt a single 8 byte block using XTEA. +func decryptBlock(c *Cipher, dst, src []byte) { + v0, v1 := blockToUint32(src) + + // Two rounds of XTEA applied per loop + for i := numRounds; i > 0; { + i-- + v1 -= ((v0<<4 ^ v0>>5) + v0) ^ c.table[i] + i-- + v0 -= ((v1<<4 ^ v1>>5) + v1) ^ c.table[i] + } + + uint32ToBlock(v0, v1, dst) +} diff --git a/src/pkg/crypto/xtea/cipher.go b/src/pkg/crypto/xtea/cipher.go new file mode 100644 index 000000000..b3fba3c84 --- /dev/null +++ b/src/pkg/crypto/xtea/cipher.go @@ -0,0 +1,92 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Package xtea implements XTEA encryption, as defined in Needham and Wheeler's +// 1997 technical report, "Tea extensions." +package xtea + +// For details, see http://www.cix.co.uk/~klockstone/xtea.pdf + +import ( + "os" + "strconv" +) + +// The XTEA block size in bytes. +const BlockSize = 8 + +// A Cipher is an instance of an XTEA cipher using a particular key. +// table contains a series of precalculated values that are used each round. +type Cipher struct { + table [64]uint32 +} + +type KeySizeError int + +func (k KeySizeError) String() string { + return "crypto/xtea: invalid key size " + strconv.Itoa(int(k)) +} + +// NewCipher creates and returns a new Cipher. +// The key argument should be the XTEA key. +// XTEA only supports 128 bit (16 byte) keys. +func NewCipher(key []byte) (*Cipher, os.Error) { + k := len(key) + switch k { + default: + return nil, KeySizeError(k) + case 16: + break + } + + c := new(Cipher) + initCipher(c, key) + + return c, nil +} + +// BlockSize returns the XTEA block size, 8 bytes. +// It is necessary to satisfy the Cipher interface in the +// package "crypto/cipher". +func (c *Cipher) BlockSize() int { return BlockSize } + +// Encrypt encrypts the 8 byte buffer src using the key and stores the result in dst. +// Note that for amounts of data larger than a block, +// it is not safe to just call Encrypt on successive blocks; +// instead, use an encryption mode like CBC (see crypto/cipher/cbc.go). +func (c *Cipher) Encrypt(dst, src []byte) { encryptBlock(c, dst, src) } + +// Decrypt decrypts the 8 byte buffer src using the key k and stores the result in dst. +func (c *Cipher) Decrypt(dst, src []byte) { decryptBlock(c, dst, src) } + +// Reset zeros the table, so that it will no longer appear in the process's memory. +func (c *Cipher) Reset() { + for i := 0; i < len(c.table); i++ { + c.table[i] = 0 + } +} + +// initCipher initializes the cipher context by creating a look up table +// of precalculated values that are based on the key. +func initCipher(c *Cipher, key []byte) { + // Load the key into four uint32s + var k [4]uint32 + for i := 0; i < len(k); i++ { + j := i << 2 // Multiply by 4 + k[i] = uint32(key[j+0])<<24 | uint32(key[j+1])<<16 | uint32(key[j+2])<<8 | uint32(key[j+3]) + } + + // Precalculate the table + const delta = 0x9E3779B9 + var sum uint32 = 0 + + // Two rounds of XTEA applied per loop + for i := 0; i < numRounds; { + c.table[i] = sum + k[sum&3] + i++ + sum += delta + c.table[i] = sum + k[(sum>>11)&3] + i++ + } +} diff --git a/src/pkg/crypto/xtea/xtea_test.go b/src/pkg/crypto/xtea/xtea_test.go new file mode 100644 index 000000000..217d96adc --- /dev/null +++ b/src/pkg/crypto/xtea/xtea_test.go @@ -0,0 +1,246 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package xtea + +import ( + "testing" +) + +// A sample test key for when we just want to initialize a cipher +var testKey = []byte{0x00, 0x11, 0x22, 0x33, 0x44, 0x55, 0x66, 0x77, 0x88, 0x99, 0xAA, 0xBB, 0xCC, 0xDD, 0xEE, 0xFF} + +// Test that the block size for XTEA is correct +func TestBlocksize(t *testing.T) { + if BlockSize != 8 { + t.Errorf("BlockSize constant - expected 8, got %d", BlockSize) + return + } + + c, err := NewCipher(testKey) + if err != nil { + t.Errorf("NewCipher(%d bytes) = %s", len(testKey), err) + return + } + + result := c.BlockSize() + if result != 8 { + t.Errorf("BlockSize function - expected 8, got %d", result) + return + } +} + +// A series of test values to confirm that the Cipher.table array was initialized correctly +var testTable = []uint32{ + 0x00112233, 0x6B1568B8, 0xE28CE030, 0xC5089E2D, 0xC5089E2D, 0x1EFBD3A2, 0xA7845C2A, 0x78EF0917, + 0x78EF0917, 0x172682D0, 0x5B6AC714, 0x822AC955, 0x3DE68511, 0xDC1DFECA, 0x2062430E, 0x3611343F, + 0xF1CCEFFB, 0x900469B4, 0xD448ADF8, 0x2E3BE36D, 0xB6C46BF5, 0x994029F2, 0x994029F2, 0xF3335F67, + 0x6AAAD6DF, 0x4D2694DC, 0x4D2694DC, 0xEB5E0E95, 0x2FA252D9, 0x4551440A, 0x121E10D6, 0xB0558A8F, + 0xE388BDC3, 0x0A48C004, 0xC6047BC0, 0x643BF579, 0xA88039BD, 0x02736F32, 0x8AFBF7BA, 0x5C66A4A7, + 0x5C66A4A7, 0xC76AEB2C, 0x3EE262A4, 0x215E20A1, 0x215E20A1, 0x7B515616, 0x03D9DE9E, 0x1988CFCF, + 0xD5448B8B, 0x737C0544, 0xB7C04988, 0xDE804BC9, 0x9A3C0785, 0x3873813E, 0x7CB7C582, 0xD6AAFAF7, + 0x4E22726F, 0x309E306C, 0x309E306C, 0x8A9165E1, 0x1319EE69, 0xF595AC66, 0xF595AC66, 0x4F88E1DB, +} + +// Test that the cipher context is initialized correctly +func TestCipherInit(t *testing.T) { + c, err := NewCipher(testKey) + if err != nil { + t.Errorf("NewCipher(%d bytes) = %s", len(testKey), err) + return + } + + for i := 0; i < len(c.table); i++ { + if c.table[i] != testTable[i] { + t.Errorf("NewCipher() failed to initialize Cipher.table[%d] correctly. Expected %08X, got %08X", i, testTable[i], c.table[i]) + break + } + } +} + +// Test that invalid key sizes return an error +func TestInvalidKeySize(t *testing.T) { + // Test a long key + key := []byte{ + 0x00, 0x11, 0x22, 0x33, 0x44, 0x55, 0x66, 0x77, 0x88, 0x99, 0xAA, 0xBB, 0xCC, 0xDD, 0xEE, 0xFF, + 0x00, 0x11, 0x22, 0x33, 0x44, 0x55, 0x66, 0x77, 0x88, 0x99, 0xAA, 0xBB, 0xCC, 0xDD, 0xEE, 0xFF, + } + + _, err := NewCipher(key) + if err == nil { + t.Errorf("Invalid key size %d didn't result in an error.", len(key)) + } + + // Test a short key + key = []byte{0x00, 0x11, 0x22, 0x33, 0x44, 0x55, 0x66, 0x77} + + _, err = NewCipher(key) + if err == nil { + t.Errorf("Invalid key size %d didn't result in an error.", len(key)) + } +} + +// Test that we can correctly decode some bytes we have encoded +func TestEncodeDecode(t *testing.T) { + original := []byte{0x01, 0x23, 0x45, 0x67, 0x89, 0xAB, 0xCD, 0xEF} + input := original + output := make([]byte, BlockSize) + + c, err := NewCipher(testKey) + if err != nil { + t.Errorf("NewCipher(%d bytes) = %s", len(testKey), err) + return + } + + // Encrypt the input block + c.Encrypt(output, input) + + // Check that the output does not match the input + differs := false + for i := 0; i < len(input); i++ { + if output[i] != input[i] { + differs = true + break + } + } + if differs == false { + t.Error("Cipher.Encrypt: Failed to encrypt the input block.") + return + } + + // Decrypt the block we just encrypted + input = output + output = make([]byte, BlockSize) + c.Decrypt(output, input) + + // Check that the output from decrypt matches our initial input + for i := 0; i < len(input); i++ { + if output[i] != original[i] { + t.Errorf("Decrypted byte %d differed. Expected %02X, got %02X\n", i, original[i], output[i]) + return + } + } +} + +// Test Vectors +type CryptTest struct { + key []byte + plainText []byte + cipherText []byte +} + +var CryptTests = []CryptTest{ + // These were sourced from http://www.freemedialibrary.com/index.php/XTEA_test_vectors + { + []byte{0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f}, + []byte{0x41, 0x42, 0x43, 0x44, 0x45, 0x46, 0x47, 0x48}, + []byte{0x49, 0x7d, 0xf3, 0xd0, 0x72, 0x61, 0x2c, 0xb5}, + }, + { + []byte{0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f}, + []byte{0x41, 0x41, 0x41, 0x41, 0x41, 0x41, 0x41, 0x41}, + []byte{0xe7, 0x8f, 0x2d, 0x13, 0x74, 0x43, 0x41, 0xd8}, + }, + { + []byte{0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f}, + []byte{0x5a, 0x5b, 0x6e, 0x27, 0x89, 0x48, 0xd7, 0x7f}, + []byte{0x41, 0x41, 0x41, 0x41, 0x41, 0x41, 0x41, 0x41}, + }, + { + []byte{0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00}, + []byte{0x41, 0x42, 0x43, 0x44, 0x45, 0x46, 0x47, 0x48}, + []byte{0xa0, 0x39, 0x05, 0x89, 0xf8, 0xb8, 0xef, 0xa5}, + }, + { + []byte{0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00}, + []byte{0x41, 0x41, 0x41, 0x41, 0x41, 0x41, 0x41, 0x41}, + []byte{0xed, 0x23, 0x37, 0x5a, 0x82, 0x1a, 0x8c, 0x2d}, + }, + { + []byte{0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00}, + []byte{0x70, 0xe1, 0x22, 0x5d, 0x6e, 0x4e, 0x76, 0x55}, + []byte{0x41, 0x41, 0x41, 0x41, 0x41, 0x41, 0x41, 0x41}, + }, + + // These vectors are from http://wiki.secondlife.com/wiki/XTEA_Strong_Encryption_Implementation#Bouncy_Castle_C.23_API + { + []byte{0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00}, + []byte{0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00}, + []byte{0xDE, 0xE9, 0xD4, 0xD8, 0xF7, 0x13, 0x1E, 0xD9}, + }, + { + []byte{0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00}, + []byte{0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08}, + []byte{0x06, 0x5C, 0x1B, 0x89, 0x75, 0xC6, 0xA8, 0x16}, + }, + { + []byte{0x01, 0x23, 0x45, 0x67, 0x12, 0x34, 0x56, 0x78, 0x23, 0x45, 0x67, 0x89, 0x34, 0x56, 0x78, 0x9A}, + []byte{0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00}, + []byte{0x1F, 0xF9, 0xA0, 0x26, 0x1A, 0xC6, 0x42, 0x64}, + }, + { + []byte{0x01, 0x23, 0x45, 0x67, 0x12, 0x34, 0x56, 0x78, 0x23, 0x45, 0x67, 0x89, 0x34, 0x56, 0x78, 0x9A}, + []byte{0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08}, + []byte{0x8C, 0x67, 0x15, 0x5B, 0x2E, 0xF9, 0x1E, 0xAD}, + }, +} + +// Test encryption +func TestCipherEncrypt(t *testing.T) { + for i, tt := range CryptTests { + c, err := NewCipher(tt.key) + if err != nil { + t.Errorf("NewCipher(%d bytes), vector %d = %s", len(tt.key), i, err) + continue + } + + out := make([]byte, len(tt.plainText)) + c.Encrypt(out, tt.plainText) + + for j := 0; j < len(out); j++ { + if out[j] != tt.cipherText[j] { + t.Errorf("Cipher.Encrypt %d: out[%d] = %02X, expected %02X", i, j, out[j], tt.cipherText[j]) + break + } + } + } +} + +// Test decryption +func TestCipherDecrypt(t *testing.T) { + for i, tt := range CryptTests { + c, err := NewCipher(tt.key) + if err != nil { + t.Errorf("NewCipher(%d bytes), vector %d = %s", len(tt.key), i, err) + continue + } + + out := make([]byte, len(tt.cipherText)) + c.Decrypt(out, tt.cipherText) + + for j := 0; j < len(out); j++ { + if out[j] != tt.plainText[j] { + t.Errorf("Cipher.Decrypt %d: out[%d] = %02X, expected %02X", i, j, out[j], tt.plainText[j]) + break + } + } + } +} + +// Test resetting the cipher context +func TestReset(t *testing.T) { + c, err := NewCipher(testKey) + if err != nil { + t.Errorf("NewCipher(%d bytes) = %s", len(testKey), err) + return + } + + c.Reset() + for i := 0; i < len(c.table); i++ { + if c.table[i] != 0 { + t.Errorf("Cipher.Reset: Failed to clear Cipher.table[%d]. expected 0, got %08X", i, c.table[i]) + return + } + } +} diff --git a/src/pkg/csv/Makefile b/src/pkg/csv/Makefile new file mode 100644 index 000000000..e364d51d2 --- /dev/null +++ b/src/pkg/csv/Makefile @@ -0,0 +1,12 @@ +# Copyright 2011 The Go Authors. All rights reserved. +# Use of this source code is governed by a BSD-style +# license that can be found in the LICENSE file. + +include ../../Make.inc + +TARG=csv +GOFILES=\ + reader.go\ + writer.go\ + +include ../../Make.pkg diff --git a/src/pkg/csv/reader.go b/src/pkg/csv/reader.go new file mode 100644 index 000000000..ea2c266a4 --- /dev/null +++ b/src/pkg/csv/reader.go @@ -0,0 +1,372 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Package csv reads and writes comma-separated values (CSV) files. +// +// A csv file contains zero or more records of one or more fields per record. +// Each record is separated by the newline character. The final record may +// optionally be followed by a newline character. +// +// field1,field2,field3 +// +// White space is considered part of a field. +// +// Carriage returns before newline characters are silently removed. +// +// Blank lines are ignored. A line with only whitespace characters (excluding +// the ending newline character) is not considered a blank line. +// +// Fields which start and stop with the quote character " are called +// quoted-fields. The beginning and ending quote are not part of the +// field. +// +// The source: +// +// normal string,"quoted-field" +// +// results in the fields +// +// {`normal string`, `quoted-field`} +// +// Within a quoted-field a quote character followed by a second quote +// character is considered a single quote. +// +// "the ""word"" is true","a ""quoted-field""" +// +// results in +// +// {`the "word" is true`, `a "quoted-field"`} +// +// Newlines and commas may be included in a quoted-field +// +// "Multi-line +// field","comma is ," +// +// results in +// +// {`Multi-line +// field`, `comma is ,`} +package csv + +import ( + "bufio" + "bytes" + "fmt" + "io" + "os" + "unicode" +) + +// A ParseError is returned for parsing errors. +// The first line is 1. The first column is 0. +type ParseError struct { + Line int // Line where the error occurred + Column int // Column (rune index) where the error occurred + Error os.Error // The actual error +} + +func (e *ParseError) String() string { + return fmt.Sprintf("line %d, column %d: %s", e.Line, e.Column, e.Error) +} + +// These are the errors that can be returned in ParseError.Error +var ( + ErrTrailingComma = os.NewError("extra delimiter at end of line") + ErrBareQuote = os.NewError("bare \" in non-quoted-field") + ErrQuote = os.NewError("extraneous \" in field") + ErrFieldCount = os.NewError("wrong number of fields in line") +) + +// A Reader reads records from a CSV-encoded file. +// +// As returned by NewReader, a Reader expects input conforming to RFC 4180. +// The exported fields can be changed to customize the details before the +// first call to Read or ReadAll. +// +// Comma is the field delimiter. It defaults to ','. +// +// Comment, if not 0, is the comment character. Lines beginning with the +// Comment character are ignored. +// +// If FieldsPerRecord is positive, Read requires each record to +// have the given number of fields. If FieldsPerRecord is 0, Read sets it to +// the number of fields in the first record, so that future records must +// have the same field count. +// +// If LazyQuotes is true, a quote may appear in an unquoted field and a +// non-doubled quote may appear in a quoted field. +// +// If TrailingComma is true, the last field may be an unquoted empty field. +// +// If TrimLeadingSpace is true, leading white space in a field is ignored. +type Reader struct { + Comma int // Field delimiter (set to ',' by NewReader) + Comment int // Comment character for start of line + FieldsPerRecord int // Number of expected fields per record + LazyQuotes bool // Allow lazy quotes + TrailingComma bool // Allow trailing comma + TrimLeadingSpace bool // Trim leading space + line int + column int + r *bufio.Reader + field bytes.Buffer +} + +// NewReader returns a new Reader that reads from r. +func NewReader(r io.Reader) *Reader { + return &Reader{ + Comma: ',', + r: bufio.NewReader(r), + } +} + +// error creates a new ParseError based on err. +func (r *Reader) error(err os.Error) os.Error { + return &ParseError{ + Line: r.line, + Column: r.column, + Error: err, + } +} + +// Read reads one record from r. The record is a slice of strings with each +// string representing one field. +func (r *Reader) Read() (record []string, err os.Error) { + for { + record, err = r.parseRecord() + if record != nil { + break + } + if err != nil { + return nil, err + } + } + + if r.FieldsPerRecord > 0 { + if len(record) != r.FieldsPerRecord { + r.column = 0 // report at start of record + return record, r.error(ErrFieldCount) + } + } else if r.FieldsPerRecord == 0 { + r.FieldsPerRecord = len(record) + } + return record, nil +} + +// ReadAll reads all the remaining records from r. +// Each record is a slice of fields. +func (r *Reader) ReadAll() (records [][]string, err os.Error) { + for { + record, err := r.Read() + if err == os.EOF { + return records, nil + } + if err != nil { + return nil, err + } + records = append(records, record) + } + panic("unreachable") +} + +// readRune reads one rune from r, folding \r\n to \n and keeping track +// of how far into the line we have read. r.column will point to the start +// of this rune, not the end of this rune. +func (r *Reader) readRune() (int, os.Error) { + rune, _, err := r.r.ReadRune() + + // Handle \r\n here. We make the simplifying assumption that + // anytime \r is followed by \n that it can be folded to \n. + // We will not detect files which contain both \r\n and bare \n. + if rune == '\r' { + rune, _, err = r.r.ReadRune() + if err == nil { + if rune != '\n' { + r.r.UnreadRune() + rune = '\r' + } + } + } + r.column++ + return rune, err +} + +// unreadRune puts the last rune read from r back. +func (r *Reader) unreadRune() { + r.r.UnreadRune() + r.column-- +} + +// skip reads runes up to and including the rune delim or until error. +func (r *Reader) skip(delim int) os.Error { + for { + rune, err := r.readRune() + if err != nil { + return err + } + if rune == delim { + return nil + } + } + panic("unreachable") +} + +// parseRecord reads and parses a single csv record from r. +func (r *Reader) parseRecord() (fields []string, err os.Error) { + // Each record starts on a new line. We increment our line + // number (lines start at 1, not 0) and set column to -1 + // so as we increment in readRune it points to the character we read. + r.line++ + r.column = -1 + + // Peek at the first rune. If it is an error we are done. + // If we are support comments and it is the comment character + // then skip to the end of line. + + rune, _, err := r.r.ReadRune() + if err != nil { + return nil, err + } + + if r.Comment != 0 && rune == r.Comment { + return nil, r.skip('\n') + } + r.r.UnreadRune() + + // At this point we have at least one field. + for { + haveField, delim, err := r.parseField() + if haveField { + fields = append(fields, r.field.String()) + } + if delim == '\n' || err == os.EOF { + return fields, err + } else if err != nil { + return nil, err + } + } + panic("unreachable") +} + +// parseField parses the next field in the record. The read field is +// located in r.field. Delim is the first character not part of the field +// (r.Comma or '\n'). +func (r *Reader) parseField() (haveField bool, delim int, err os.Error) { + r.field.Reset() + + rune, err := r.readRune() + if err != nil { + // If we have EOF and are not at the start of a line + // then we return the empty field. We have already + // checked for trailing commas if needed. + if err == os.EOF && r.column != 0 { + return true, 0, err + } + return false, 0, err + } + + if r.TrimLeadingSpace { + for unicode.IsSpace(rune) { + rune, err = r.readRune() + if err != nil { + return false, 0, err + } + } + } + + switch rune { + case r.Comma: + // will check below + + case '\n': + // We are a trailing empty field or a blank line + if r.column == 0 { + return false, rune, nil + } + return true, rune, nil + + case '"': + // quoted field + Quoted: + for { + rune, err = r.readRune() + if err != nil { + if err == os.EOF { + if r.LazyQuotes { + return true, 0, err + } + return false, 0, r.error(ErrQuote) + } + return false, 0, err + } + switch rune { + case '"': + rune, err = r.readRune() + if err != nil || rune == r.Comma { + break Quoted + } + if rune == '\n' { + return true, rune, nil + } + if rune != '"' { + if !r.LazyQuotes { + r.column-- + return false, 0, r.error(ErrQuote) + } + // accept the bare quote + r.field.WriteRune('"') + } + case '\n': + r.line++ + r.column = -1 + } + r.field.WriteRune(rune) + } + + default: + // unquoted field + for { + r.field.WriteRune(rune) + rune, err = r.readRune() + if err != nil || rune == r.Comma { + break + } + if rune == '\n' { + return true, rune, nil + } + if !r.LazyQuotes && rune == '"' { + return false, 0, r.error(ErrBareQuote) + } + } + } + + if err != nil { + if err == os.EOF { + return true, 0, err + } + return false, 0, err + } + + if !r.TrailingComma { + // We don't allow trailing commas. See if we + // are at the end of the line (being mindful + // of trimming spaces). + c := r.column + rune, err = r.readRune() + if r.TrimLeadingSpace { + for unicode.IsSpace(rune) { + rune, err = r.readRune() + if err != nil { + break + } + } + } + if err == os.EOF || rune == '\n' { + r.column = c // report the comma + return false, 0, r.error(ErrTrailingComma) + } + r.unreadRune() + } + return true, rune, nil +} diff --git a/src/pkg/csv/reader_test.go b/src/pkg/csv/reader_test.go new file mode 100644 index 000000000..0068bad1d --- /dev/null +++ b/src/pkg/csv/reader_test.go @@ -0,0 +1,265 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package csv + +import ( + "reflect" + "strings" + "testing" +) + +var readTests = []struct { + Name string + Input string + Output [][]string + UseFieldsPerRecord bool // false (default) means FieldsPerRecord is -1 + + // These fields are copied into the Reader + Comma int + Comment int + FieldsPerRecord int + LazyQuotes bool + TrailingComma bool + TrimLeadingSpace bool + + Error string + Line int // Expected error line if != 0 + Column int // Expected error column if line != 0 +}{ + { + Name: "Simple", + Input: "a,b,c\n", + Output: [][]string{{"a", "b", "c"}}, + }, + { + Name: "CRLF", + Input: "a,b\r\nc,d\r\n", + Output: [][]string{{"a", "b"}, {"c", "d"}}, + }, + { + Name: "BareCR", + Input: "a,b\rc,d\r\n", + Output: [][]string{{"a", "b\rc", "d"}}, + }, + { + Name: "RFC4180test", + UseFieldsPerRecord: true, + Input: `#field1,field2,field3 +"aaa","bb +b","ccc" +"a,a","b""bb","ccc" +zzz,yyy,xxx +`, + Output: [][]string{ + {"#field1", "field2", "field3"}, + {"aaa", "bb\nb", "ccc"}, + {"a,a", `b"bb`, "ccc"}, + {"zzz", "yyy", "xxx"}, + }, + }, + { + Name: "NoEOLTest", + Input: "a,b,c", + Output: [][]string{{"a", "b", "c"}}, + }, + { + Name: "Semicolon", + Comma: ';', + Input: "a;b;c\n", + Output: [][]string{{"a", "b", "c"}}, + }, + { + Name: "MultiLine", + Input: `"two +line","one line","three +line +field"`, + Output: [][]string{{"two\nline", "one line", "three\nline\nfield"}}, + }, + { + Name: "BlankLine", + Input: "a,b,c\n\nd,e,f\n\n", + Output: [][]string{ + {"a", "b", "c"}, + {"d", "e", "f"}, + }, + }, + { + Name: "TrimSpace", + Input: " a, b, c\n", + TrimLeadingSpace: true, + Output: [][]string{{"a", "b", "c"}}, + }, + { + Name: "LeadingSpace", + Input: " a, b, c\n", + Output: [][]string{{" a", " b", " c"}}, + }, + { + Name: "Comment", + Comment: '#', + Input: "#1,2,3\na,b,c\n#comment", + Output: [][]string{{"a", "b", "c"}}, + }, + { + Name: "NoComment", + Input: "#1,2,3\na,b,c", + Output: [][]string{{"#1", "2", "3"}, {"a", "b", "c"}}, + }, + { + Name: "LazyQuotes", + LazyQuotes: true, + Input: `a "word","1"2",a","b`, + Output: [][]string{{`a "word"`, `1"2`, `a"`, `b`}}, + }, + { + Name: "BareQuotes", + LazyQuotes: true, + Input: `a "word","1"2",a"`, + Output: [][]string{{`a "word"`, `1"2`, `a"`}}, + }, + { + Name: "BareDoubleQuotes", + LazyQuotes: true, + Input: `a""b,c`, + Output: [][]string{{`a""b`, `c`}}, + }, + { + Name: "BadDoubleQuotes", + Input: `a""b,c`, + Output: [][]string{{`a""b`, `c`}}, + Error: `bare " in non-quoted-field`, Line: 1, Column: 1, + }, + { + Name: "TrimQuote", + Input: ` "a"," b",c`, + TrimLeadingSpace: true, + Output: [][]string{{"a", " b", "c"}}, + }, + { + Name: "BadBareQuote", + Input: `a "word","b"`, + Error: `bare " in non-quoted-field`, Line: 1, Column: 2, + }, + { + Name: "BadTrailingQuote", + Input: `"a word",b"`, + Error: `bare " in non-quoted-field`, Line: 1, Column: 10, + }, + { + Name: "ExtraneousQuote", + Input: `"a "word","b"`, + Error: `extraneous " in field`, Line: 1, Column: 3, + }, + { + Name: "BadFieldCount", + UseFieldsPerRecord: true, + Input: "a,b,c\nd,e", + Error: "wrong number of fields", Line: 2, + }, + { + Name: "BadFieldCount1", + UseFieldsPerRecord: true, + FieldsPerRecord: 2, + Input: `a,b,c`, + Error: "wrong number of fields", Line: 1, + }, + { + Name: "FieldCount", + Input: "a,b,c\nd,e", + Output: [][]string{{"a", "b", "c"}, {"d", "e"}}, + }, + { + Name: "BadTrailingCommaEOF", + Input: "a,b,c,", + Error: "extra delimiter at end of line", Line: 1, Column: 5, + }, + { + Name: "BadTrailingCommaEOL", + Input: "a,b,c,\n", + Error: "extra delimiter at end of line", Line: 1, Column: 5, + }, + { + Name: "BadTrailingCommaSpaceEOF", + TrimLeadingSpace: true, + Input: "a,b,c, ", + Error: "extra delimiter at end of line", Line: 1, Column: 5, + }, + { + Name: "BadTrailingCommaSpaceEOL", + TrimLeadingSpace: true, + Input: "a,b,c, \n", + Error: "extra delimiter at end of line", Line: 1, Column: 5, + }, + { + Name: "BadTrailingCommaLine3", + TrimLeadingSpace: true, + Input: "a,b,c\nd,e,f\ng,hi,", + Error: "extra delimiter at end of line", Line: 3, Column: 4, + }, + { + Name: "NotTrailingComma3", + Input: "a,b,c, \n", + Output: [][]string{{"a", "b", "c", " "}}, + }, + { + Name: "CommaFieldTest", + TrailingComma: true, + Input: `x,y,z,w +x,y,z, +x,y,, +x,,, +,,, +"x","y","z","w" +"x","y","z","" +"x","y","","" +"x","","","" +"","","","" +`, + Output: [][]string{ + {"x", "y", "z", "w"}, + {"x", "y", "z", ""}, + {"x", "y", "", ""}, + {"x", "", "", ""}, + {"", "", "", ""}, + {"x", "y", "z", "w"}, + {"x", "y", "z", ""}, + {"x", "y", "", ""}, + {"x", "", "", ""}, + {"", "", "", ""}, + }, + }, +} + +func TestRead(t *testing.T) { + for _, tt := range readTests { + r := NewReader(strings.NewReader(tt.Input)) + r.Comment = tt.Comment + if tt.UseFieldsPerRecord { + r.FieldsPerRecord = tt.FieldsPerRecord + } else { + r.FieldsPerRecord = -1 + } + r.LazyQuotes = tt.LazyQuotes + r.TrailingComma = tt.TrailingComma + r.TrimLeadingSpace = tt.TrimLeadingSpace + if tt.Comma != 0 { + r.Comma = tt.Comma + } + out, err := r.ReadAll() + perr, _ := err.(*ParseError) + if tt.Error != "" { + if err == nil || !strings.Contains(err.String(), tt.Error) { + t.Errorf("%s: error %v, want error %q", tt.Name, err, tt.Error) + } else if tt.Line != 0 && (tt.Line != perr.Line || tt.Column != perr.Column) { + t.Errorf("%s: error at %d:%d expected %d:%d", tt.Name, perr.Line, perr.Column, tt.Line, tt.Column) + } + } else if err != nil { + t.Errorf("%s: unexpected error %v", tt.Name, err) + } else if !reflect.DeepEqual(out, tt.Output) { + t.Errorf("%s: out=%q want %q", tt.Name, out, tt.Output) + } + } +} diff --git a/src/pkg/csv/writer.go b/src/pkg/csv/writer.go new file mode 100644 index 000000000..ccf703f0f --- /dev/null +++ b/src/pkg/csv/writer.go @@ -0,0 +1,122 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package csv + +import ( + "bufio" + "io" + "os" + "strings" + "unicode" + "utf8" +) + +// A Writer writes records to a CSV encoded file. +// +// As returned by NewWriter, a Writer writes records terminated by a +// newline and uses ',' as the field delimiter. The exported fields can be +// changed to customize the details before the first call to Write or WriteAll. +// +// Comma is the field delimiter. +// +// If UseCRLF is true, the Writer ends each record with \r\n instead of \n. +type Writer struct { + Comma int // Field delimiter (set to to ',' by NewWriter) + UseCRLF bool // True to use \r\n as the line terminator + w *bufio.Writer +} + +// NewWriter returns a new Writer that writes to w. +func NewWriter(w io.Writer) *Writer { + return &Writer{ + Comma: ',', + w: bufio.NewWriter(w), + } +} + +// Writer writes a single CSV record to w along with any necessary quoting. +// A record is a slice of strings with each string being one field. +func (w *Writer) Write(record []string) (err os.Error) { + for n, field := range record { + if n > 0 { + if _, err = w.w.WriteRune(w.Comma); err != nil { + return + } + } + + // If we don't have to have a quoted field then just + // write out the field and continue to the next field. + if !w.fieldNeedsQuotes(field) { + if _, err = w.w.WriteString(field); err != nil { + return + } + continue + } + if err = w.w.WriteByte('"'); err != nil { + return + } + + for _, rune := range field { + switch rune { + case '"': + _, err = w.w.WriteString(`""`) + case '\r': + if !w.UseCRLF { + err = w.w.WriteByte('\r') + } + case '\n': + if w.UseCRLF { + _, err = w.w.WriteString("\r\n") + } else { + err = w.w.WriteByte('\n') + } + default: + _, err = w.w.WriteRune(rune) + } + if err != nil { + return + } + } + + if err = w.w.WriteByte('"'); err != nil { + return + } + } + if w.UseCRLF { + _, err = w.w.WriteString("\r\n") + } else { + err = w.w.WriteByte('\n') + } + return +} + +// Flush writes any buffered data to the underlying io.Writer. +func (w *Writer) Flush() { + w.w.Flush() +} + +// WriteAll writes multiple CSV records to w using Write and then calls Flush. +func (w *Writer) WriteAll(records [][]string) (err os.Error) { + for _, record := range records { + err = w.Write(record) + if err != nil { + break + } + } + w.Flush() + return nil +} + +// fieldNeedsQuotes returns true if our field must be enclosed in quotes. +// Empty fields, files with a Comma, fields with a quote or newline, and +// fields which start with a space must be enclosed in quotes. +func (w *Writer) fieldNeedsQuotes(field string) bool { + if len(field) == 0 || strings.IndexRune(field, w.Comma) >= 0 || strings.IndexAny(field, "\"\r\n") >= 0 { + return true + } + + rune, _ := utf8.DecodeRuneInString(field) + return unicode.IsSpace(rune) +} diff --git a/src/pkg/csv/writer_test.go b/src/pkg/csv/writer_test.go new file mode 100644 index 000000000..578959007 --- /dev/null +++ b/src/pkg/csv/writer_test.go @@ -0,0 +1,44 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package csv + +import ( + "bytes" + "testing" +) + +var writeTests = []struct { + Input [][]string + Output string + UseCRLF bool +}{ + {Input: [][]string{{"abc"}}, Output: "abc\n"}, + {Input: [][]string{{"abc"}}, Output: "abc\r\n", UseCRLF: true}, + {Input: [][]string{{`"abc"`}}, Output: `"""abc"""` + "\n"}, + {Input: [][]string{{`a"b`}}, Output: `"a""b"` + "\n"}, + {Input: [][]string{{`"a"b"`}}, Output: `"""a""b"""` + "\n"}, + {Input: [][]string{{" abc"}}, Output: `" abc"` + "\n"}, + {Input: [][]string{{"abc,def"}}, Output: `"abc,def"` + "\n"}, + {Input: [][]string{{"abc", "def"}}, Output: "abc,def\n"}, + {Input: [][]string{{"abc"}, {"def"}}, Output: "abc\ndef\n"}, + {Input: [][]string{{"abc\ndef"}}, Output: "\"abc\ndef\"\n"}, + {Input: [][]string{{"abc\ndef"}}, Output: "\"abc\r\ndef\"\r\n", UseCRLF: true}, +} + +func TestWrite(t *testing.T) { + for n, tt := range writeTests { + b := &bytes.Buffer{} + f := NewWriter(b) + f.UseCRLF = tt.UseCRLF + err := f.WriteAll(tt.Input) + if err != nil { + t.Errorf("Unexpected error: %s\n", err) + } + out := b.String() + if out != tt.Output { + t.Errorf("#%d: out=%q want %q", n, out, tt.Output) + } + } +} diff --git a/src/pkg/debug/dwarf/Makefile b/src/pkg/debug/dwarf/Makefile new file mode 100644 index 000000000..c4203188e --- /dev/null +++ b/src/pkg/debug/dwarf/Makefile @@ -0,0 +1,16 @@ +# Copyright 2009 The Go Authors. All rights reserved. +# Use of this source code is governed by a BSD-style +# license that can be found in the LICENSE file. + +include ../../../Make.inc + +TARG=debug/dwarf +GOFILES=\ + buf.go\ + const.go\ + entry.go\ + open.go\ + type.go\ + unit.go\ + +include ../../../Make.pkg diff --git a/src/pkg/debug/dwarf/buf.go b/src/pkg/debug/dwarf/buf.go new file mode 100644 index 000000000..2d29cebdd --- /dev/null +++ b/src/pkg/debug/dwarf/buf.go @@ -0,0 +1,154 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Buffered reading and decoding of DWARF data streams. + +package dwarf + +import ( + "encoding/binary" + "os" + "strconv" +) + +// Data buffer being decoded. +type buf struct { + dwarf *Data + order binary.ByteOrder + name string + off Offset + data []byte + addrsize int + err os.Error +} + +func makeBuf(d *Data, name string, off Offset, data []byte, addrsize int) buf { + return buf{d, d.order, name, off, data, addrsize, nil} +} + +func (b *buf) uint8() uint8 { + if len(b.data) < 1 { + b.error("underflow") + return 0 + } + val := b.data[0] + b.data = b.data[1:] + b.off++ + return val +} + +func (b *buf) bytes(n int) []byte { + if len(b.data) < n { + b.error("underflow") + return nil + } + data := b.data[0:n] + b.data = b.data[n:] + b.off += Offset(n) + return data +} + +func (b *buf) skip(n int) { b.bytes(n) } + +func (b *buf) string() string { + for i := 0; i < len(b.data); i++ { + if b.data[i] == 0 { + s := string(b.data[0:i]) + b.data = b.data[i+1:] + b.off += Offset(i + 1) + return s + } + } + b.error("underflow") + return "" +} + +func (b *buf) uint16() uint16 { + a := b.bytes(2) + if a == nil { + return 0 + } + return b.order.Uint16(a) +} + +func (b *buf) uint32() uint32 { + a := b.bytes(4) + if a == nil { + return 0 + } + return b.order.Uint32(a) +} + +func (b *buf) uint64() uint64 { + a := b.bytes(8) + if a == nil { + return 0 + } + return b.order.Uint64(a) +} + +// Read a varint, which is 7 bits per byte, little endian. +// the 0x80 bit means read another byte. +func (b *buf) varint() (c uint64, bits uint) { + for i := 0; i < len(b.data); i++ { + byte := b.data[i] + c |= uint64(byte&0x7F) << bits + bits += 7 + if byte&0x80 == 0 { + b.off += Offset(i + 1) + b.data = b.data[i+1:] + return c, bits + } + } + return 0, 0 +} + +// Unsigned int is just a varint. +func (b *buf) uint() uint64 { + x, _ := b.varint() + return x +} + +// Signed int is a sign-extended varint. +func (b *buf) int() int64 { + ux, bits := b.varint() + x := int64(ux) + if x&(1<<(bits-1)) != 0 { + x |= -1 << bits + } + return x +} + +// Address-sized uint. +func (b *buf) addr() uint64 { + switch b.addrsize { + case 1: + return uint64(b.uint8()) + case 2: + return uint64(b.uint16()) + case 4: + return uint64(b.uint32()) + case 8: + return uint64(b.uint64()) + } + b.error("unknown address size") + return 0 +} + +func (b *buf) error(s string) { + if b.err == nil { + b.data = nil + b.err = DecodeError{b.name, b.off, s} + } +} + +type DecodeError struct { + Name string + Offset Offset + Error string +} + +func (e DecodeError) String() string { + return "decoding dwarf section " + e.Name + " at offset 0x" + strconv.Itob64(int64(e.Offset), 16) + ": " + e.Error +} diff --git a/src/pkg/debug/dwarf/const.go b/src/pkg/debug/dwarf/const.go new file mode 100644 index 000000000..1a3fec155 --- /dev/null +++ b/src/pkg/debug/dwarf/const.go @@ -0,0 +1,433 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Constants + +package dwarf + +import "strconv" + +// An Attr identifies the attribute type in a DWARF Entry's Field. +type Attr uint32 + +const ( + AttrSibling Attr = 0x01 + AttrLocation Attr = 0x02 + AttrName Attr = 0x03 + AttrOrdering Attr = 0x09 + AttrByteSize Attr = 0x0B + AttrBitOffset Attr = 0x0C + AttrBitSize Attr = 0x0D + AttrStmtList Attr = 0x10 + AttrLowpc Attr = 0x11 + AttrHighpc Attr = 0x12 + AttrLanguage Attr = 0x13 + AttrDiscr Attr = 0x15 + AttrDiscrValue Attr = 0x16 + AttrVisibility Attr = 0x17 + AttrImport Attr = 0x18 + AttrStringLength Attr = 0x19 + AttrCommonRef Attr = 0x1A + AttrCompDir Attr = 0x1B + AttrConstValue Attr = 0x1C + AttrContainingType Attr = 0x1D + AttrDefaultValue Attr = 0x1E + AttrInline Attr = 0x20 + AttrIsOptional Attr = 0x21 + AttrLowerBound Attr = 0x22 + AttrProducer Attr = 0x25 + AttrPrototyped Attr = 0x27 + AttrReturnAddr Attr = 0x2A + AttrStartScope Attr = 0x2C + AttrStrideSize Attr = 0x2E + AttrUpperBound Attr = 0x2F + AttrAbstractOrigin Attr = 0x31 + AttrAccessibility Attr = 0x32 + AttrAddrClass Attr = 0x33 + AttrArtificial Attr = 0x34 + AttrBaseTypes Attr = 0x35 + AttrCalling Attr = 0x36 + AttrCount Attr = 0x37 + AttrDataMemberLoc Attr = 0x38 + AttrDeclColumn Attr = 0x39 + AttrDeclFile Attr = 0x3A + AttrDeclLine Attr = 0x3B + AttrDeclaration Attr = 0x3C + AttrDiscrList Attr = 0x3D + AttrEncoding Attr = 0x3E + AttrExternal Attr = 0x3F + AttrFrameBase Attr = 0x40 + AttrFriend Attr = 0x41 + AttrIdentifierCase Attr = 0x42 + AttrMacroInfo Attr = 0x43 + AttrNamelistItem Attr = 0x44 + AttrPriority Attr = 0x45 + AttrSegment Attr = 0x46 + AttrSpecification Attr = 0x47 + AttrStaticLink Attr = 0x48 + AttrType Attr = 0x49 + AttrUseLocation Attr = 0x4A + AttrVarParam Attr = 0x4B + AttrVirtuality Attr = 0x4C + AttrVtableElemLoc Attr = 0x4D + AttrAllocated Attr = 0x4E + AttrAssociated Attr = 0x4F + AttrDataLocation Attr = 0x50 + AttrStride Attr = 0x51 + AttrEntrypc Attr = 0x52 + AttrUseUTF8 Attr = 0x53 + AttrExtension Attr = 0x54 + AttrRanges Attr = 0x55 + AttrTrampoline Attr = 0x56 + AttrCallColumn Attr = 0x57 + AttrCallFile Attr = 0x58 + AttrCallLine Attr = 0x59 + AttrDescription Attr = 0x5A +) + +var attrNames = [...]string{ + AttrSibling: "Sibling", + AttrLocation: "Location", + AttrName: "Name", + AttrOrdering: "Ordering", + AttrByteSize: "ByteSize", + AttrBitOffset: "BitOffset", + AttrBitSize: "BitSize", + AttrStmtList: "StmtList", + AttrLowpc: "Lowpc", + AttrHighpc: "Highpc", + AttrLanguage: "Language", + AttrDiscr: "Discr", + AttrDiscrValue: "DiscrValue", + AttrVisibility: "Visibility", + AttrImport: "Import", + AttrStringLength: "StringLength", + AttrCommonRef: "CommonRef", + AttrCompDir: "CompDir", + AttrConstValue: "ConstValue", + AttrContainingType: "ContainingType", + AttrDefaultValue: "DefaultValue", + AttrInline: "Inline", + AttrIsOptional: "IsOptional", + AttrLowerBound: "LowerBound", + AttrProducer: "Producer", + AttrPrototyped: "Prototyped", + AttrReturnAddr: "ReturnAddr", + AttrStartScope: "StartScope", + AttrStrideSize: "StrideSize", + AttrUpperBound: "UpperBound", + AttrAbstractOrigin: "AbstractOrigin", + AttrAccessibility: "Accessibility", + AttrAddrClass: "AddrClass", + AttrArtificial: "Artificial", + AttrBaseTypes: "BaseTypes", + AttrCalling: "Calling", + AttrCount: "Count", + AttrDataMemberLoc: "DataMemberLoc", + AttrDeclColumn: "DeclColumn", + AttrDeclFile: "DeclFile", + AttrDeclLine: "DeclLine", + AttrDeclaration: "Declaration", + AttrDiscrList: "DiscrList", + AttrEncoding: "Encoding", + AttrExternal: "External", + AttrFrameBase: "FrameBase", + AttrFriend: "Friend", + AttrIdentifierCase: "IdentifierCase", + AttrMacroInfo: "MacroInfo", + AttrNamelistItem: "NamelistItem", + AttrPriority: "Priority", + AttrSegment: "Segment", + AttrSpecification: "Specification", + AttrStaticLink: "StaticLink", + AttrType: "Type", + AttrUseLocation: "UseLocation", + AttrVarParam: "VarParam", + AttrVirtuality: "Virtuality", + AttrVtableElemLoc: "VtableElemLoc", + AttrAllocated: "Allocated", + AttrAssociated: "Associated", + AttrDataLocation: "DataLocation", + AttrStride: "Stride", + AttrEntrypc: "Entrypc", + AttrUseUTF8: "UseUTF8", + AttrExtension: "Extension", + AttrRanges: "Ranges", + AttrTrampoline: "Trampoline", + AttrCallColumn: "CallColumn", + AttrCallFile: "CallFile", + AttrCallLine: "CallLine", + AttrDescription: "Description", +} + +func (a Attr) String() string { + if int(a) < len(attrNames) { + s := attrNames[a] + if s != "" { + return s + } + } + return strconv.Itoa(int(a)) +} + +func (a Attr) GoString() string { + if int(a) < len(attrNames) { + s := attrNames[a] + if s != "" { + return "dwarf.Attr" + s + } + } + return "dwarf.Attr(" + strconv.Itoa64(int64(a)) + ")" +} + +// A format is a DWARF data encoding format. +type format uint32 + +const ( + // value formats + formAddr format = 0x01 + formDwarfBlock2 format = 0x03 + formDwarfBlock4 format = 0x04 + formData2 format = 0x05 + formData4 format = 0x06 + formData8 format = 0x07 + formString format = 0x08 + formDwarfBlock format = 0x09 + formDwarfBlock1 format = 0x0A + formData1 format = 0x0B + formFlag format = 0x0C + formSdata format = 0x0D + formStrp format = 0x0E + formUdata format = 0x0F + formRefAddr format = 0x10 + formRef1 format = 0x11 + formRef2 format = 0x12 + formRef4 format = 0x13 + formRef8 format = 0x14 + formRefUdata format = 0x15 + formIndirect format = 0x16 +) + +// A Tag is the classification (the type) of an Entry. +type Tag uint32 + +const ( + TagArrayType Tag = 0x01 + TagClassType Tag = 0x02 + TagEntryPoint Tag = 0x03 + TagEnumerationType Tag = 0x04 + TagFormalParameter Tag = 0x05 + TagImportedDeclaration Tag = 0x08 + TagLabel Tag = 0x0A + TagLexDwarfBlock Tag = 0x0B + TagMember Tag = 0x0D + TagPointerType Tag = 0x0F + TagReferenceType Tag = 0x10 + TagCompileUnit Tag = 0x11 + TagStringType Tag = 0x12 + TagStructType Tag = 0x13 + TagSubroutineType Tag = 0x15 + TagTypedef Tag = 0x16 + TagUnionType Tag = 0x17 + TagUnspecifiedParameters Tag = 0x18 + TagVariant Tag = 0x19 + TagCommonDwarfBlock Tag = 0x1A + TagCommonInclusion Tag = 0x1B + TagInheritance Tag = 0x1C + TagInlinedSubroutine Tag = 0x1D + TagModule Tag = 0x1E + TagPtrToMemberType Tag = 0x1F + TagSetType Tag = 0x20 + TagSubrangeType Tag = 0x21 + TagWithStmt Tag = 0x22 + TagAccessDeclaration Tag = 0x23 + TagBaseType Tag = 0x24 + TagCatchDwarfBlock Tag = 0x25 + TagConstType Tag = 0x26 + TagConstant Tag = 0x27 + TagEnumerator Tag = 0x28 + TagFileType Tag = 0x29 + TagFriend Tag = 0x2A + TagNamelist Tag = 0x2B + TagNamelistItem Tag = 0x2C + TagPackedType Tag = 0x2D + TagSubprogram Tag = 0x2E + TagTemplateTypeParameter Tag = 0x2F + TagTemplateValueParameter Tag = 0x30 + TagThrownType Tag = 0x31 + TagTryDwarfBlock Tag = 0x32 + TagVariantPart Tag = 0x33 + TagVariable Tag = 0x34 + TagVolatileType Tag = 0x35 + TagDwarfProcedure Tag = 0x36 + TagRestrictType Tag = 0x37 + TagInterfaceType Tag = 0x38 + TagNamespace Tag = 0x39 + TagImportedModule Tag = 0x3A + TagUnspecifiedType Tag = 0x3B + TagPartialUnit Tag = 0x3C + TagImportedUnit Tag = 0x3D + TagMutableType Tag = 0x3E +) + +var tagNames = [...]string{ + TagArrayType: "ArrayType", + TagClassType: "ClassType", + TagEntryPoint: "EntryPoint", + TagEnumerationType: "EnumerationType", + TagFormalParameter: "FormalParameter", + TagImportedDeclaration: "ImportedDeclaration", + TagLabel: "Label", + TagLexDwarfBlock: "LexDwarfBlock", + TagMember: "Member", + TagPointerType: "PointerType", + TagReferenceType: "ReferenceType", + TagCompileUnit: "CompileUnit", + TagStringType: "StringType", + TagStructType: "StructType", + TagSubroutineType: "SubroutineType", + TagTypedef: "Typedef", + TagUnionType: "UnionType", + TagUnspecifiedParameters: "UnspecifiedParameters", + TagVariant: "Variant", + TagCommonDwarfBlock: "CommonDwarfBlock", + TagCommonInclusion: "CommonInclusion", + TagInheritance: "Inheritance", + TagInlinedSubroutine: "InlinedSubroutine", + TagModule: "Module", + TagPtrToMemberType: "PtrToMemberType", + TagSetType: "SetType", + TagSubrangeType: "SubrangeType", + TagWithStmt: "WithStmt", + TagAccessDeclaration: "AccessDeclaration", + TagBaseType: "BaseType", + TagCatchDwarfBlock: "CatchDwarfBlock", + TagConstType: "ConstType", + TagConstant: "Constant", + TagEnumerator: "Enumerator", + TagFileType: "FileType", + TagFriend: "Friend", + TagNamelist: "Namelist", + TagNamelistItem: "NamelistItem", + TagPackedType: "PackedType", + TagSubprogram: "Subprogram", + TagTemplateTypeParameter: "TemplateTypeParameter", + TagTemplateValueParameter: "TemplateValueParameter", + TagThrownType: "ThrownType", + TagTryDwarfBlock: "TryDwarfBlock", + TagVariantPart: "VariantPart", + TagVariable: "Variable", + TagVolatileType: "VolatileType", + TagDwarfProcedure: "DwarfProcedure", + TagRestrictType: "RestrictType", + TagInterfaceType: "InterfaceType", + TagNamespace: "Namespace", + TagImportedModule: "ImportedModule", + TagUnspecifiedType: "UnspecifiedType", + TagPartialUnit: "PartialUnit", + TagImportedUnit: "ImportedUnit", + TagMutableType: "MutableType", +} + +func (t Tag) String() string { + if int(t) < len(tagNames) { + s := tagNames[t] + if s != "" { + return s + } + } + return strconv.Itoa(int(t)) +} + +func (t Tag) GoString() string { + if int(t) < len(tagNames) { + s := tagNames[t] + if s != "" { + return "dwarf.Tag" + s + } + } + return "dwarf.Tag(" + strconv.Itoa64(int64(t)) + ")" +} + +// Location expression operators. +// The debug info encodes value locations like 8(R3) +// as a sequence of these op codes. +// This package does not implement full expressions; +// the opPlusUconst operator is expected by the type parser. +const ( + opAddr = 0x03 /* 1 op, const addr */ + opDeref = 0x06 + opConst1u = 0x08 /* 1 op, 1 byte const */ + opConst1s = 0x09 /* " signed */ + opConst2u = 0x0A /* 1 op, 2 byte const */ + opConst2s = 0x0B /* " signed */ + opConst4u = 0x0C /* 1 op, 4 byte const */ + opConst4s = 0x0D /* " signed */ + opConst8u = 0x0E /* 1 op, 8 byte const */ + opConst8s = 0x0F /* " signed */ + opConstu = 0x10 /* 1 op, LEB128 const */ + opConsts = 0x11 /* " signed */ + opDup = 0x12 + opDrop = 0x13 + opOver = 0x14 + opPick = 0x15 /* 1 op, 1 byte stack index */ + opSwap = 0x16 + opRot = 0x17 + opXderef = 0x18 + opAbs = 0x19 + opAnd = 0x1A + opDiv = 0x1B + opMinus = 0x1C + opMod = 0x1D + opMul = 0x1E + opNeg = 0x1F + opNot = 0x20 + opOr = 0x21 + opPlus = 0x22 + opPlusUconst = 0x23 /* 1 op, ULEB128 addend */ + opShl = 0x24 + opShr = 0x25 + opShra = 0x26 + opXor = 0x27 + opSkip = 0x2F /* 1 op, signed 2-byte constant */ + opBra = 0x28 /* 1 op, signed 2-byte constant */ + opEq = 0x29 + opGe = 0x2A + opGt = 0x2B + opLe = 0x2C + opLt = 0x2D + opNe = 0x2E + opLit0 = 0x30 + /* OpLitN = OpLit0 + N for N = 0..31 */ + opReg0 = 0x50 + /* OpRegN = OpReg0 + N for N = 0..31 */ + opBreg0 = 0x70 /* 1 op, signed LEB128 constant */ + /* OpBregN = OpBreg0 + N for N = 0..31 */ + opRegx = 0x90 /* 1 op, ULEB128 register */ + opFbreg = 0x91 /* 1 op, SLEB128 offset */ + opBregx = 0x92 /* 2 op, ULEB128 reg; SLEB128 off */ + opPiece = 0x93 /* 1 op, ULEB128 size of piece */ + opDerefSize = 0x94 /* 1-byte size of data retrieved */ + opXderefSize = 0x95 /* 1-byte size of data retrieved */ + opNop = 0x96 + /* next four new in Dwarf v3 */ + opPushObjAddr = 0x97 + opCall2 = 0x98 /* 2-byte offset of DIE */ + opCall4 = 0x99 /* 4-byte offset of DIE */ + opCallRef = 0x9A /* 4- or 8- byte offset of DIE */ + /* 0xE0-0xFF reserved for user-specific */ +) + +// Basic type encodings -- the value for AttrEncoding in a TagBaseType Entry. +const ( + encAddress = 0x01 + encBoolean = 0x02 + encComplexFloat = 0x03 + encFloat = 0x04 + encSigned = 0x05 + encSignedChar = 0x06 + encUnsigned = 0x07 + encUnsignedChar = 0x08 + encImaginaryFloat = 0x09 +) diff --git a/src/pkg/debug/dwarf/entry.go b/src/pkg/debug/dwarf/entry.go new file mode 100644 index 000000000..549e5c2cc --- /dev/null +++ b/src/pkg/debug/dwarf/entry.go @@ -0,0 +1,343 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// DWARF debug information entry parser. +// An entry is a sequence of data items of a given format. +// The first word in the entry is an index into what DWARF +// calls the ``abbreviation table.'' An abbreviation is really +// just a type descriptor: it's an array of attribute tag/value format pairs. + +package dwarf + +import "os" + +// a single entry's description: a sequence of attributes +type abbrev struct { + tag Tag + children bool + field []afield +} + +type afield struct { + attr Attr + fmt format +} + +// a map from entry format ids to their descriptions +type abbrevTable map[uint32]abbrev + +// ParseAbbrev returns the abbreviation table that starts at byte off +// in the .debug_abbrev section. +func (d *Data) parseAbbrev(off uint32) (abbrevTable, os.Error) { + if m, ok := d.abbrevCache[off]; ok { + return m, nil + } + + data := d.abbrev + if off > uint32(len(data)) { + data = nil + } else { + data = data[off:] + } + b := makeBuf(d, "abbrev", 0, data, 0) + + // Error handling is simplified by the buf getters + // returning an endless stream of 0s after an error. + m := make(abbrevTable) + for { + // Table ends with id == 0. + id := uint32(b.uint()) + if id == 0 { + break + } + + // Walk over attributes, counting. + n := 0 + b1 := b // Read from copy of b. + b1.uint() + b1.uint8() + for { + tag := b1.uint() + fmt := b1.uint() + if tag == 0 && fmt == 0 { + break + } + n++ + } + if b1.err != nil { + return nil, b1.err + } + + // Walk over attributes again, this time writing them down. + var a abbrev + a.tag = Tag(b.uint()) + a.children = b.uint8() != 0 + a.field = make([]afield, n) + for i := range a.field { + a.field[i].attr = Attr(b.uint()) + a.field[i].fmt = format(b.uint()) + } + b.uint() + b.uint() + + m[id] = a + } + if b.err != nil { + return nil, b.err + } + d.abbrevCache[off] = m + return m, nil +} + +// An entry is a sequence of attribute/value pairs. +type Entry struct { + Offset Offset // offset of Entry in DWARF info + Tag Tag // tag (kind of Entry) + Children bool // whether Entry is followed by children + Field []Field +} + +// A Field is a single attribute/value pair in an Entry. +type Field struct { + Attr Attr + Val interface{} +} + +// Val returns the value associated with attribute Attr in Entry, +// or nil if there is no such attribute. +// +// A common idiom is to merge the check for nil return with +// the check that the value has the expected dynamic type, as in: +// v, ok := e.Val(AttrSibling).(int64); +// +func (e *Entry) Val(a Attr) interface{} { + for _, f := range e.Field { + if f.Attr == a { + return f.Val + } + } + return nil +} + +// An Offset represents the location of an Entry within the DWARF info. +// (See Reader.Seek.) +type Offset uint32 + +// Entry reads a single entry from buf, decoding +// according to the given abbreviation table. +func (b *buf) entry(atab abbrevTable, ubase Offset) *Entry { + off := b.off + id := uint32(b.uint()) + if id == 0 { + return &Entry{} + } + a, ok := atab[id] + if !ok { + b.error("unknown abbreviation table index") + return nil + } + e := &Entry{ + Offset: off, + Tag: a.tag, + Children: a.children, + Field: make([]Field, len(a.field)), + } + for i := range e.Field { + e.Field[i].Attr = a.field[i].attr + fmt := a.field[i].fmt + if fmt == formIndirect { + fmt = format(b.uint()) + } + var val interface{} + switch fmt { + default: + b.error("unknown entry attr format") + + // address + case formAddr: + val = b.addr() + + // block + case formDwarfBlock1: + val = b.bytes(int(b.uint8())) + case formDwarfBlock2: + val = b.bytes(int(b.uint16())) + case formDwarfBlock4: + val = b.bytes(int(b.uint32())) + case formDwarfBlock: + val = b.bytes(int(b.uint())) + + // constant + case formData1: + val = int64(b.uint8()) + case formData2: + val = int64(b.uint16()) + case formData4: + val = int64(b.uint32()) + case formData8: + val = int64(b.uint64()) + case formSdata: + val = int64(b.int()) + case formUdata: + val = int64(b.uint()) + + // flag + case formFlag: + val = b.uint8() == 1 + + // reference to other entry + case formRefAddr: + val = Offset(b.addr()) + case formRef1: + val = Offset(b.uint8()) + ubase + case formRef2: + val = Offset(b.uint16()) + ubase + case formRef4: + val = Offset(b.uint32()) + ubase + case formRef8: + val = Offset(b.uint64()) + ubase + case formRefUdata: + val = Offset(b.uint()) + ubase + + // string + case formString: + val = b.string() + case formStrp: + off := b.uint32() // offset into .debug_str + if b.err != nil { + return nil + } + b1 := makeBuf(b.dwarf, "str", 0, b.dwarf.str, 0) + b1.skip(int(off)) + val = b1.string() + if b1.err != nil { + b.err = b1.err + return nil + } + } + e.Field[i].Val = val + } + if b.err != nil { + return nil + } + return e +} + +// A Reader allows reading Entry structures from a DWARF ``info'' section. +// The Entry structures are arranged in a tree. The Reader's Next function +// return successive entries from a pre-order traversal of the tree. +// If an entry has children, its Children field will be true, and the children +// follow, terminated by an Entry with Tag 0. +type Reader struct { + b buf + d *Data + err os.Error + unit int + lastChildren bool // .Children of last entry returned by Next + lastSibling Offset // .Val(AttrSibling) of last entry returned by Next +} + +// Reader returns a new Reader for Data. +// The reader is positioned at byte offset 0 in the DWARF ``info'' section. +func (d *Data) Reader() *Reader { + r := &Reader{d: d} + r.Seek(0) + return r +} + +// Seek positions the Reader at offset off in the encoded entry stream. +// Offset 0 can be used to denote the first entry. +func (r *Reader) Seek(off Offset) { + d := r.d + r.err = nil + r.lastChildren = false + if off == 0 { + if len(d.unit) == 0 { + return + } + u := &d.unit[0] + r.unit = 0 + r.b = makeBuf(r.d, "info", u.off, u.data, u.addrsize) + return + } + + // TODO(rsc): binary search (maybe a new package) + var i int + var u *unit + for i = range d.unit { + u = &d.unit[i] + if u.off <= off && off < u.off+Offset(len(u.data)) { + r.unit = i + r.b = makeBuf(r.d, "info", off, u.data[off-u.off:], u.addrsize) + return + } + } + r.err = os.NewError("offset out of range") +} + +// maybeNextUnit advances to the next unit if this one is finished. +func (r *Reader) maybeNextUnit() { + for len(r.b.data) == 0 && r.unit+1 < len(r.d.unit) { + r.unit++ + u := &r.d.unit[r.unit] + r.b = makeBuf(r.d, "info", u.off, u.data, u.addrsize) + } +} + +// Next reads the next entry from the encoded entry stream. +// It returns nil, nil when it reaches the end of the section. +// It returns an error if the current offset is invalid or the data at the +// offset cannot be decoded as a valid Entry. +func (r *Reader) Next() (*Entry, os.Error) { + if r.err != nil { + return nil, r.err + } + r.maybeNextUnit() + if len(r.b.data) == 0 { + return nil, nil + } + u := &r.d.unit[r.unit] + e := r.b.entry(u.atable, u.base) + if r.b.err != nil { + r.err = r.b.err + return nil, r.err + } + if e != nil { + r.lastChildren = e.Children + if r.lastChildren { + r.lastSibling, _ = e.Val(AttrSibling).(Offset) + } + } else { + r.lastChildren = false + } + return e, nil +} + +// SkipChildren skips over the child entries associated with +// the last Entry returned by Next. If that Entry did not have +// children or Next has not been called, SkipChildren is a no-op. +func (r *Reader) SkipChildren() { + if r.err != nil || !r.lastChildren { + return + } + + // If the last entry had a sibling attribute, + // that attribute gives the offset of the next + // sibling, so we can avoid decoding the + // child subtrees. + if r.lastSibling >= r.b.off { + r.Seek(r.lastSibling) + return + } + + for { + e, err := r.Next() + if err != nil || e == nil || e.Tag == 0 { + break + } + if e.Children { + r.SkipChildren() + } + } +} diff --git a/src/pkg/debug/dwarf/open.go b/src/pkg/debug/dwarf/open.go new file mode 100644 index 000000000..d9525f788 --- /dev/null +++ b/src/pkg/debug/dwarf/open.go @@ -0,0 +1,80 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Package dwarf provides access to DWARF debugging information loaded from +// executable files, as defined in the DWARF 2.0 Standard at +// http://dwarfstd.org/doc/dwarf-2.0.0.pdf +package dwarf + +import ( + "encoding/binary" + "os" +) + +// Data represents the DWARF debugging information +// loaded from an executable file (for example, an ELF or Mach-O executable). +type Data struct { + // raw data + abbrev []byte + aranges []byte + frame []byte + info []byte + line []byte + pubnames []byte + ranges []byte + str []byte + + // parsed data + abbrevCache map[uint32]abbrevTable + addrsize int + order binary.ByteOrder + typeCache map[Offset]Type + unit []unit +} + +// New returns a new Data object initialized from the given parameters. +// Clients should typically use [TODO(rsc): method to be named later] instead of calling +// New directly. +// +// The []byte arguments are the data from the corresponding debug section +// in the object file; for example, for an ELF object, abbrev is the contents of +// the ".debug_abbrev" section. +func New(abbrev, aranges, frame, info, line, pubnames, ranges, str []byte) (*Data, os.Error) { + d := &Data{ + abbrev: abbrev, + aranges: aranges, + frame: frame, + info: info, + line: line, + pubnames: pubnames, + ranges: ranges, + str: str, + abbrevCache: make(map[uint32]abbrevTable), + typeCache: make(map[Offset]Type), + } + + // Sniff .debug_info to figure out byte order. + // bytes 4:6 are the version, a tiny 16-bit number (1, 2, 3). + if len(d.info) < 6 { + return nil, DecodeError{"info", Offset(len(d.info)), "too short"} + } + x, y := d.info[4], d.info[5] + switch { + case x == 0 && y == 0: + return nil, DecodeError{"info", 4, "unsupported version 0"} + case x == 0: + d.order = binary.BigEndian + case y == 0: + d.order = binary.LittleEndian + default: + return nil, DecodeError{"info", 4, "cannot determine byte order"} + } + + u, err := d.parseUnits() + if err != nil { + return nil, err + } + d.unit = u + return d, nil +} diff --git a/src/pkg/debug/dwarf/testdata/typedef.c b/src/pkg/debug/dwarf/testdata/typedef.c new file mode 100644 index 000000000..664d021ce --- /dev/null +++ b/src/pkg/debug/dwarf/testdata/typedef.c @@ -0,0 +1,79 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +/* +Linux ELF: +gcc -gdwarf-2 -m64 -c typedef.c && gcc -gdwarf-2 -m64 -o typedef.elf typedef.o + +OS X Mach-O: +gcc -gdwarf-2 -m64 -c typedef.c -o typedef.macho +*/ +#include + +typedef volatile int* t_ptr_volatile_int; +typedef const char *t_ptr_const_char; +typedef long t_long; +typedef unsigned short t_ushort; +typedef int t_func_int_of_float_double(float, double); +typedef int (*t_ptr_func_int_of_float_double)(float, double); +typedef int (*t_ptr_func_int_of_float_complex)(float complex); +typedef int (*t_ptr_func_int_of_double_complex)(double complex); +typedef int (*t_ptr_func_int_of_long_double_complex)(long double complex); +typedef int *t_func_ptr_int_of_char_schar_uchar(char, signed char, unsigned char); +typedef void t_func_void_of_char(char); +typedef void t_func_void_of_void(void); +typedef void t_func_void_of_ptr_char_dots(char*, ...); +typedef struct my_struct { + volatile int vi; + char x : 1; + int y : 4; + long long array[40]; +} t_my_struct; +typedef union my_union { + volatile int vi; + char x : 1; + int y : 4; + long long array[40]; +} t_my_union; +typedef enum my_enum { + e1 = 1, + e2 = 2, + e3 = -5, + e4 = 1000000000000000LL, +} t_my_enum; + +typedef struct list t_my_list; +struct list { + short val; + t_my_list *next; +}; + +typedef struct tree { + struct tree *left, *right; + unsigned long long val; +} t_my_tree; + +t_ptr_volatile_int *a2; +t_ptr_const_char **a3a; +t_long *a4; +t_ushort *a5; +t_func_int_of_float_double *a6; +t_ptr_func_int_of_float_double *a7; +t_func_ptr_int_of_char_schar_uchar *a8; +t_func_void_of_char *a9; +t_func_void_of_void *a10; +t_func_void_of_ptr_char_dots *a11; +t_my_struct *a12; +t_my_union *a12a; +t_my_enum *a13; +t_my_list *a14; +t_my_tree *a15; +t_ptr_func_int_of_float_complex *a16; +t_ptr_func_int_of_double_complex *a17; +t_ptr_func_int_of_long_double_complex *a18; + +int main() +{ + return 0; +} diff --git a/src/pkg/debug/dwarf/testdata/typedef.elf b/src/pkg/debug/dwarf/testdata/typedef.elf new file mode 100755 index 000000000..44df8da9b Binary files /dev/null and b/src/pkg/debug/dwarf/testdata/typedef.elf differ diff --git a/src/pkg/debug/dwarf/testdata/typedef.macho b/src/pkg/debug/dwarf/testdata/typedef.macho new file mode 100644 index 000000000..41019c1e1 Binary files /dev/null and b/src/pkg/debug/dwarf/testdata/typedef.macho differ diff --git a/src/pkg/debug/dwarf/type.go b/src/pkg/debug/dwarf/type.go new file mode 100644 index 000000000..f35365ebe --- /dev/null +++ b/src/pkg/debug/dwarf/type.go @@ -0,0 +1,584 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// DWARF type information structures. +// The format is heavily biased toward C, but for simplicity +// the String methods use a pseudo-Go syntax. + +package dwarf + +import ( + "os" + "strconv" +) + +// A Type conventionally represents a pointer to any of the +// specific Type structures (CharType, StructType, etc.). +type Type interface { + Common() *CommonType + String() string + Size() int64 +} + +// A CommonType holds fields common to multiple types. +// If a field is not known or not applicable for a given type, +// the zero value is used. +type CommonType struct { + ByteSize int64 // size of value of this type, in bytes + Name string // name that can be used to refer to type +} + +func (c *CommonType) Common() *CommonType { return c } + +func (c *CommonType) Size() int64 { return c.ByteSize } + +// Basic types + +// A BasicType holds fields common to all basic types. +type BasicType struct { + CommonType + BitSize int64 + BitOffset int64 +} + +func (b *BasicType) Basic() *BasicType { return b } + +func (t *BasicType) String() string { + if t.Name != "" { + return t.Name + } + return "?" +} + +// A CharType represents a signed character type. +type CharType struct { + BasicType +} + +// A UcharType represents an unsigned character type. +type UcharType struct { + BasicType +} + +// An IntType represents a signed integer type. +type IntType struct { + BasicType +} + +// A UintType represents an unsigned integer type. +type UintType struct { + BasicType +} + +// A FloatType represents a floating point type. +type FloatType struct { + BasicType +} + +// A ComplexType represents a complex floating point type. +type ComplexType struct { + BasicType +} + +// A BoolType represents a boolean type. +type BoolType struct { + BasicType +} + +// An AddrType represents a machine address type. +type AddrType struct { + BasicType +} + +// qualifiers + +// A QualType represents a type that has the C/C++ "const", "restrict", or "volatile" qualifier. +type QualType struct { + CommonType + Qual string + Type Type +} + +func (t *QualType) String() string { return t.Qual + " " + t.Type.String() } + +func (t *QualType) Size() int64 { return t.Type.Size() } + +// An ArrayType represents a fixed size array type. +type ArrayType struct { + CommonType + Type Type + StrideBitSize int64 // if > 0, number of bits to hold each element + Count int64 // if == -1, an incomplete array, like char x[]. +} + +func (t *ArrayType) String() string { + return "[" + strconv.Itoa64(t.Count) + "]" + t.Type.String() +} + +func (t *ArrayType) Size() int64 { return t.Count * t.Type.Size() } + +// A VoidType represents the C void type. +type VoidType struct { + CommonType +} + +func (t *VoidType) String() string { return "void" } + +// A PtrType represents a pointer type. +type PtrType struct { + CommonType + Type Type +} + +func (t *PtrType) String() string { return "*" + t.Type.String() } + +// A StructType represents a struct, union, or C++ class type. +type StructType struct { + CommonType + StructName string + Kind string // "struct", "union", or "class". + Field []*StructField + Incomplete bool // if true, struct, union, class is declared but not defined +} + +// A StructField represents a field in a struct, union, or C++ class type. +type StructField struct { + Name string + Type Type + ByteOffset int64 + ByteSize int64 + BitOffset int64 // within the ByteSize bytes at ByteOffset + BitSize int64 // zero if not a bit field +} + +func (t *StructType) String() string { + if t.StructName != "" { + return t.Kind + " " + t.StructName + } + return t.Defn() +} + +func (t *StructType) Defn() string { + s := t.Kind + if t.StructName != "" { + s += " " + t.StructName + } + if t.Incomplete { + s += " /*incomplete*/" + return s + } + s += " {" + for i, f := range t.Field { + if i > 0 { + s += "; " + } + s += f.Name + " " + f.Type.String() + s += "@" + strconv.Itoa64(f.ByteOffset) + if f.BitSize > 0 { + s += " : " + strconv.Itoa64(f.BitSize) + s += "@" + strconv.Itoa64(f.BitOffset) + } + } + s += "}" + return s +} + +// An EnumType represents an enumerated type. +// The only indication of its native integer type is its ByteSize +// (inside CommonType). +type EnumType struct { + CommonType + EnumName string + Val []*EnumValue +} + +// An EnumValue represents a single enumeration value. +type EnumValue struct { + Name string + Val int64 +} + +func (t *EnumType) String() string { + s := "enum" + if t.EnumName != "" { + s += " " + t.EnumName + } + s += " {" + for i, v := range t.Val { + if i > 0 { + s += "; " + } + s += v.Name + "=" + strconv.Itoa64(v.Val) + } + s += "}" + return s +} + +// A FuncType represents a function type. +type FuncType struct { + CommonType + ReturnType Type + ParamType []Type +} + +func (t *FuncType) String() string { + s := "func(" + for i, t := range t.ParamType { + if i > 0 { + s += ", " + } + s += t.String() + } + s += ")" + if t.ReturnType != nil { + s += " " + t.ReturnType.String() + } + return s +} + +// A DotDotDotType represents the variadic ... function parameter. +type DotDotDotType struct { + CommonType +} + +func (t *DotDotDotType) String() string { return "..." } + +// A TypedefType represents a named type. +type TypedefType struct { + CommonType + Type Type +} + +func (t *TypedefType) String() string { return t.Name } + +func (t *TypedefType) Size() int64 { return t.Type.Size() } + +func (d *Data) Type(off Offset) (Type, os.Error) { + if t, ok := d.typeCache[off]; ok { + return t, nil + } + + r := d.Reader() + r.Seek(off) + e, err := r.Next() + if err != nil { + return nil, err + } + if e == nil || e.Offset != off { + return nil, DecodeError{"info", off, "no type at offset"} + } + + // Parse type from Entry. + // Must always set d.typeCache[off] before calling + // d.Type recursively, to handle circular types correctly. + var typ Type + + // Get next child; set err if error happens. + next := func() *Entry { + if !e.Children { + return nil + } + kid, err1 := r.Next() + if err1 != nil { + err = err1 + return nil + } + if kid == nil { + err = DecodeError{"info", r.b.off, "unexpected end of DWARF entries"} + return nil + } + if kid.Tag == 0 { + return nil + } + return kid + } + + // Get Type referred to by Entry's AttrType field. + // Set err if error happens. Not having a type is an error. + typeOf := func(e *Entry) Type { + toff, ok := e.Val(AttrType).(Offset) + if !ok { + // It appears that no Type means "void". + return new(VoidType) + } + var t Type + if t, err = d.Type(toff); err != nil { + return nil + } + return t + } + + switch e.Tag { + case TagArrayType: + // Multi-dimensional array. (DWARF v2 §5.4) + // Attributes: + // AttrType:subtype [required] + // AttrStrideSize: size in bits of each element of the array + // AttrByteSize: size of entire array + // Children: + // TagSubrangeType or TagEnumerationType giving one dimension. + // dimensions are in left to right order. + t := new(ArrayType) + typ = t + d.typeCache[off] = t + if t.Type = typeOf(e); err != nil { + goto Error + } + t.StrideBitSize, _ = e.Val(AttrStrideSize).(int64) + + // Accumulate dimensions, + ndim := 0 + for kid := next(); kid != nil; kid = next() { + // TODO(rsc): Can also be TagEnumerationType + // but haven't seen that in the wild yet. + switch kid.Tag { + case TagSubrangeType: + max, ok := kid.Val(AttrUpperBound).(int64) + if !ok { + max = -2 // Count == -1, as in x[]. + } + if ndim == 0 { + t.Count = max + 1 + } else { + // Multidimensional array. + // Create new array type underneath this one. + t.Type = &ArrayType{Type: t.Type, Count: max + 1} + } + ndim++ + case TagEnumerationType: + err = DecodeError{"info", kid.Offset, "cannot handle enumeration type as array bound"} + goto Error + } + } + if ndim == 0 { + // LLVM generates this for x[]. + t.Count = -1 + } + + case TagBaseType: + // Basic type. (DWARF v2 §5.1) + // Attributes: + // AttrName: name of base type in programming language of the compilation unit [required] + // AttrEncoding: encoding value for type (encFloat etc) [required] + // AttrByteSize: size of type in bytes [required] + // AttrBitOffset: for sub-byte types, size in bits + // AttrBitSize: for sub-byte types, bit offset of high order bit in the AttrByteSize bytes + name, _ := e.Val(AttrName).(string) + enc, ok := e.Val(AttrEncoding).(int64) + if !ok { + err = DecodeError{"info", e.Offset, "missing encoding attribute for " + name} + goto Error + } + switch enc { + default: + err = DecodeError{"info", e.Offset, "unrecognized encoding attribute value"} + goto Error + + case encAddress: + typ = new(AddrType) + case encBoolean: + typ = new(BoolType) + case encComplexFloat: + typ = new(ComplexType) + case encFloat: + typ = new(FloatType) + case encSigned: + typ = new(IntType) + case encUnsigned: + typ = new(UintType) + case encSignedChar: + typ = new(CharType) + case encUnsignedChar: + typ = new(UcharType) + } + d.typeCache[off] = typ + t := typ.(interface { + Basic() *BasicType + }).Basic() + t.Name = name + t.BitSize, _ = e.Val(AttrBitSize).(int64) + t.BitOffset, _ = e.Val(AttrBitOffset).(int64) + + case TagClassType, TagStructType, TagUnionType: + // Structure, union, or class type. (DWARF v2 §5.5) + // Attributes: + // AttrName: name of struct, union, or class + // AttrByteSize: byte size [required] + // AttrDeclaration: if true, struct/union/class is incomplete + // Children: + // TagMember to describe one member. + // AttrName: name of member [required] + // AttrType: type of member [required] + // AttrByteSize: size in bytes + // AttrBitOffset: bit offset within bytes for bit fields + // AttrBitSize: bit size for bit fields + // AttrDataMemberLoc: location within struct [required for struct, class] + // There is much more to handle C++, all ignored for now. + t := new(StructType) + typ = t + d.typeCache[off] = t + switch e.Tag { + case TagClassType: + t.Kind = "class" + case TagStructType: + t.Kind = "struct" + case TagUnionType: + t.Kind = "union" + } + t.StructName, _ = e.Val(AttrName).(string) + t.Incomplete = e.Val(AttrDeclaration) != nil + t.Field = make([]*StructField, 0, 8) + for kid := next(); kid != nil; kid = next() { + if kid.Tag == TagMember { + f := new(StructField) + if f.Type = typeOf(kid); err != nil { + goto Error + } + if loc, ok := kid.Val(AttrDataMemberLoc).([]byte); ok { + b := makeBuf(d, "location", 0, loc, d.addrsize) + if b.uint8() != opPlusUconst { + err = DecodeError{"info", kid.Offset, "unexpected opcode"} + goto Error + } + f.ByteOffset = int64(b.uint()) + if b.err != nil { + err = b.err + goto Error + } + } + f.Name, _ = kid.Val(AttrName).(string) + f.ByteSize, _ = kid.Val(AttrByteSize).(int64) + f.BitOffset, _ = kid.Val(AttrBitOffset).(int64) + f.BitSize, _ = kid.Val(AttrBitSize).(int64) + t.Field = append(t.Field, f) + } + } + + case TagConstType, TagVolatileType, TagRestrictType: + // Type modifier (DWARF v2 §5.2) + // Attributes: + // AttrType: subtype + t := new(QualType) + typ = t + d.typeCache[off] = t + if t.Type = typeOf(e); err != nil { + goto Error + } + switch e.Tag { + case TagConstType: + t.Qual = "const" + case TagRestrictType: + t.Qual = "restrict" + case TagVolatileType: + t.Qual = "volatile" + } + + case TagEnumerationType: + // Enumeration type (DWARF v2 §5.6) + // Attributes: + // AttrName: enum name if any + // AttrByteSize: bytes required to represent largest value + // Children: + // TagEnumerator: + // AttrName: name of constant + // AttrConstValue: value of constant + t := new(EnumType) + typ = t + d.typeCache[off] = t + t.EnumName, _ = e.Val(AttrName).(string) + t.Val = make([]*EnumValue, 0, 8) + for kid := next(); kid != nil; kid = next() { + if kid.Tag == TagEnumerator { + f := new(EnumValue) + f.Name, _ = kid.Val(AttrName).(string) + f.Val, _ = kid.Val(AttrConstValue).(int64) + n := len(t.Val) + if n >= cap(t.Val) { + val := make([]*EnumValue, n, n*2) + copy(val, t.Val) + t.Val = val + } + t.Val = t.Val[0 : n+1] + t.Val[n] = f + } + } + + case TagPointerType: + // Type modifier (DWARF v2 §5.2) + // Attributes: + // AttrType: subtype [not required! void* has no AttrType] + // AttrAddrClass: address class [ignored] + t := new(PtrType) + typ = t + d.typeCache[off] = t + if e.Val(AttrType) == nil { + t.Type = &VoidType{} + break + } + t.Type = typeOf(e) + + case TagSubroutineType: + // Subroutine type. (DWARF v2 §5.7) + // Attributes: + // AttrType: type of return value if any + // AttrName: possible name of type [ignored] + // AttrPrototyped: whether used ANSI C prototype [ignored] + // Children: + // TagFormalParameter: typed parameter + // AttrType: type of parameter + // TagUnspecifiedParameter: final ... + t := new(FuncType) + typ = t + d.typeCache[off] = t + if t.ReturnType = typeOf(e); err != nil { + goto Error + } + t.ParamType = make([]Type, 0, 8) + for kid := next(); kid != nil; kid = next() { + var tkid Type + switch kid.Tag { + default: + continue + case TagFormalParameter: + if tkid = typeOf(kid); err != nil { + goto Error + } + case TagUnspecifiedParameters: + tkid = &DotDotDotType{} + } + t.ParamType = append(t.ParamType, tkid) + } + + case TagTypedef: + // Typedef (DWARF v2 §5.3) + // Attributes: + // AttrName: name [required] + // AttrType: type definition [required] + t := new(TypedefType) + typ = t + d.typeCache[off] = t + t.Name, _ = e.Val(AttrName).(string) + t.Type = typeOf(e) + } + + if err != nil { + goto Error + } + + { + b, ok := e.Val(AttrByteSize).(int64) + if !ok { + b = -1 + } + typ.Common().ByteSize = b + } + return typ, nil + +Error: + // If the parse fails, take the type out of the cache + // so that the next call with this offset doesn't hit + // the cache and return success. + d.typeCache[off] = nil, false + return nil, err +} diff --git a/src/pkg/debug/dwarf/type_test.go b/src/pkg/debug/dwarf/type_test.go new file mode 100644 index 000000000..b9470a4fc --- /dev/null +++ b/src/pkg/debug/dwarf/type_test.go @@ -0,0 +1,111 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package dwarf_test + +import ( + . "debug/dwarf" + "debug/elf" + "debug/macho" + "testing" +) + +var typedefTests = map[string]string{ + "t_ptr_volatile_int": "*volatile int", + "t_ptr_const_char": "*const char", + "t_long": "long int", + "t_ushort": "short unsigned int", + "t_func_int_of_float_double": "func(float, double) int", + "t_ptr_func_int_of_float_double": "*func(float, double) int", + "t_ptr_func_int_of_float_complex": "*func(complex float) int", + "t_ptr_func_int_of_double_complex": "*func(complex double) int", + "t_ptr_func_int_of_long_double_complex": "*func(complex long double) int", + "t_func_ptr_int_of_char_schar_uchar": "func(char, signed char, unsigned char) *int", + "t_func_void_of_char": "func(char) void", + "t_func_void_of_void": "func() void", + "t_func_void_of_ptr_char_dots": "func(*char, ...) void", + "t_my_struct": "struct my_struct {vi volatile int@0; x char@4 : 1@7; y int@4 : 4@27; array [40]long long int@8}", + "t_my_union": "union my_union {vi volatile int@0; x char@0 : 1@7; y int@0 : 4@28; array [40]long long int@0}", + "t_my_enum": "enum my_enum {e1=1; e2=2; e3=-5; e4=1000000000000000}", + "t_my_list": "struct list {val short int@0; next *t_my_list@8}", + "t_my_tree": "struct tree {left *struct tree@0; right *struct tree@8; val long long unsigned int@16}", +} + +func elfData(t *testing.T, name string) *Data { + f, err := elf.Open(name) + if err != nil { + t.Fatal(err) + } + + d, err := f.DWARF() + if err != nil { + t.Fatal(err) + } + return d +} + +func machoData(t *testing.T, name string) *Data { + f, err := macho.Open(name) + if err != nil { + t.Fatal(err) + } + + d, err := f.DWARF() + if err != nil { + t.Fatal(err) + } + return d +} + +func TestTypedefsELF(t *testing.T) { testTypedefs(t, elfData(t, "testdata/typedef.elf")) } + +func TestTypedefsMachO(t *testing.T) { + testTypedefs(t, machoData(t, "testdata/typedef.macho")) +} + +func testTypedefs(t *testing.T, d *Data) { + r := d.Reader() + seen := make(map[string]bool) + for { + e, err := r.Next() + if err != nil { + t.Fatal("r.Next:", err) + } + if e == nil { + break + } + if e.Tag == TagTypedef { + typ, err := d.Type(e.Offset) + if err != nil { + t.Fatal("d.Type:", err) + } + t1 := typ.(*TypedefType) + var typstr string + if ts, ok := t1.Type.(*StructType); ok { + typstr = ts.Defn() + } else { + typstr = t1.Type.String() + } + + if want, ok := typedefTests[t1.Name]; ok { + if seen[t1.Name] { + t.Errorf("multiple definitions for %s", t1.Name) + } + seen[t1.Name] = true + if typstr != want { + t.Errorf("%s:\n\thave %s\n\twant %s", t1.Name, typstr, want) + } + } + } + if e.Tag != TagCompileUnit { + r.SkipChildren() + } + } + + for k := range typedefTests { + if !seen[k] { + t.Errorf("missing %s", k) + } + } +} diff --git a/src/pkg/debug/dwarf/unit.go b/src/pkg/debug/dwarf/unit.go new file mode 100644 index 000000000..02cb363b4 --- /dev/null +++ b/src/pkg/debug/dwarf/unit.go @@ -0,0 +1,62 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package dwarf + +import ( + "os" + "strconv" +) + +// DWARF debug info is split into a sequence of compilation units. +// Each unit has its own abbreviation table and address size. + +type unit struct { + base Offset // byte offset of header within the aggregate info + off Offset // byte offset of data within the aggregate info + data []byte + atable abbrevTable + addrsize int +} + +func (d *Data) parseUnits() ([]unit, os.Error) { + // Count units. + nunit := 0 + b := makeBuf(d, "info", 0, d.info, 0) + for len(b.data) > 0 { + b.skip(int(b.uint32())) + nunit++ + } + if b.err != nil { + return nil, b.err + } + + // Again, this time writing them down. + b = makeBuf(d, "info", 0, d.info, 0) + units := make([]unit, nunit) + for i := range units { + u := &units[i] + u.base = b.off + n := b.uint32() + if vers := b.uint16(); vers != 2 { + b.error("unsupported DWARF version " + strconv.Itoa(int(vers))) + break + } + atable, err := d.parseAbbrev(b.uint32()) + if err != nil { + if b.err == nil { + b.err = err + } + break + } + u.atable = atable + u.addrsize = int(b.uint8()) + u.off = b.off + u.data = b.bytes(int(n - (2 + 4 + 1))) + } + if b.err != nil { + return nil, b.err + } + return units, nil +} diff --git a/src/pkg/debug/elf/Makefile b/src/pkg/debug/elf/Makefile new file mode 100644 index 000000000..dd431f653 --- /dev/null +++ b/src/pkg/debug/elf/Makefile @@ -0,0 +1,12 @@ +# Copyright 2009 The Go Authors. All rights reserved. +# Use of this source code is governed by a BSD-style +# license that can be found in the LICENSE file. + +include ../../../Make.inc + +TARG=debug/elf +GOFILES=\ + elf.go\ + file.go\ + +include ../../../Make.pkg diff --git a/src/pkg/debug/elf/elf.go b/src/pkg/debug/elf/elf.go new file mode 100644 index 000000000..c71b230bd --- /dev/null +++ b/src/pkg/debug/elf/elf.go @@ -0,0 +1,1521 @@ +/* + * ELF constants and data structures + * + * Derived from: + * $FreeBSD: src/sys/sys/elf32.h,v 1.8.14.1 2005/12/30 22:13:58 marcel Exp $ + * $FreeBSD: src/sys/sys/elf64.h,v 1.10.14.1 2005/12/30 22:13:58 marcel Exp $ + * $FreeBSD: src/sys/sys/elf_common.h,v 1.15.8.1 2005/12/30 22:13:58 marcel Exp $ + * $FreeBSD: src/sys/alpha/include/elf.h,v 1.14 2003/09/25 01:10:22 peter Exp $ + * $FreeBSD: src/sys/amd64/include/elf.h,v 1.18 2004/08/03 08:21:48 dfr Exp $ + * $FreeBSD: src/sys/arm/include/elf.h,v 1.5.2.1 2006/06/30 21:42:52 cognet Exp $ + * $FreeBSD: src/sys/i386/include/elf.h,v 1.16 2004/08/02 19:12:17 dfr Exp $ + * $FreeBSD: src/sys/powerpc/include/elf.h,v 1.7 2004/11/02 09:47:01 ssouhlal Exp $ + * $FreeBSD: src/sys/sparc64/include/elf.h,v 1.12 2003/09/25 01:10:26 peter Exp $ + * + * Copyright (c) 1996-1998 John D. Polstra. All rights reserved. + * Copyright (c) 2001 David E. O'Brien + * Portions Copyright 2009 The Go Authors. All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + */ + +package elf + +import "strconv" + +/* + * Constants + */ + +// Indexes into the Header.Ident array. +const ( + EI_CLASS = 4 /* Class of machine. */ + EI_DATA = 5 /* Data format. */ + EI_VERSION = 6 /* ELF format version. */ + EI_OSABI = 7 /* Operating system / ABI identification */ + EI_ABIVERSION = 8 /* ABI version */ + EI_PAD = 9 /* Start of padding (per SVR4 ABI). */ + EI_NIDENT = 16 /* Size of e_ident array. */ +) + +// Initial magic number for ELF files. +const ELFMAG = "\177ELF" + +// Version is found in Header.Ident[EI_VERSION] and Header.Version. +type Version byte + +const ( + EV_NONE Version = 0 + EV_CURRENT Version = 1 +) + +var versionStrings = []intName{ + {0, "EV_NONE"}, + {1, "EV_CURRENT"}, +} + +func (i Version) String() string { return stringName(uint32(i), versionStrings, false) } +func (i Version) GoString() string { return stringName(uint32(i), versionStrings, true) } + +// Class is found in Header.Ident[EI_CLASS] and Header.Class. +type Class byte + +const ( + ELFCLASSNONE Class = 0 /* Unknown class. */ + ELFCLASS32 Class = 1 /* 32-bit architecture. */ + ELFCLASS64 Class = 2 /* 64-bit architecture. */ +) + +var classStrings = []intName{ + {0, "ELFCLASSNONE"}, + {1, "ELFCLASS32"}, + {2, "ELFCLASS64"}, +} + +func (i Class) String() string { return stringName(uint32(i), classStrings, false) } +func (i Class) GoString() string { return stringName(uint32(i), classStrings, true) } + +// Data is found in Header.Ident[EI_DATA] and Header.Data. +type Data byte + +const ( + ELFDATANONE Data = 0 /* Unknown data format. */ + ELFDATA2LSB Data = 1 /* 2's complement little-endian. */ + ELFDATA2MSB Data = 2 /* 2's complement big-endian. */ +) + +var dataStrings = []intName{ + {0, "ELFDATANONE"}, + {1, "ELFDATA2LSB"}, + {2, "ELFDATA2MSB"}, +} + +func (i Data) String() string { return stringName(uint32(i), dataStrings, false) } +func (i Data) GoString() string { return stringName(uint32(i), dataStrings, true) } + +// OSABI is found in Header.Ident[EI_OSABI] and Header.OSABI. +type OSABI byte + +const ( + ELFOSABI_NONE OSABI = 0 /* UNIX System V ABI */ + ELFOSABI_HPUX OSABI = 1 /* HP-UX operating system */ + ELFOSABI_NETBSD OSABI = 2 /* NetBSD */ + ELFOSABI_LINUX OSABI = 3 /* GNU/Linux */ + ELFOSABI_HURD OSABI = 4 /* GNU/Hurd */ + ELFOSABI_86OPEN OSABI = 5 /* 86Open common IA32 ABI */ + ELFOSABI_SOLARIS OSABI = 6 /* Solaris */ + ELFOSABI_AIX OSABI = 7 /* AIX */ + ELFOSABI_IRIX OSABI = 8 /* IRIX */ + ELFOSABI_FREEBSD OSABI = 9 /* FreeBSD */ + ELFOSABI_TRU64 OSABI = 10 /* TRU64 UNIX */ + ELFOSABI_MODESTO OSABI = 11 /* Novell Modesto */ + ELFOSABI_OPENBSD OSABI = 12 /* OpenBSD */ + ELFOSABI_OPENVMS OSABI = 13 /* Open VMS */ + ELFOSABI_NSK OSABI = 14 /* HP Non-Stop Kernel */ + ELFOSABI_ARM OSABI = 97 /* ARM */ + ELFOSABI_STANDALONE OSABI = 255 /* Standalone (embedded) application */ +) + +var osabiStrings = []intName{ + {0, "ELFOSABI_NONE"}, + {1, "ELFOSABI_HPUX"}, + {2, "ELFOSABI_NETBSD"}, + {3, "ELFOSABI_LINUX"}, + {4, "ELFOSABI_HURD"}, + {5, "ELFOSABI_86OPEN"}, + {6, "ELFOSABI_SOLARIS"}, + {7, "ELFOSABI_AIX"}, + {8, "ELFOSABI_IRIX"}, + {9, "ELFOSABI_FREEBSD"}, + {10, "ELFOSABI_TRU64"}, + {11, "ELFOSABI_MODESTO"}, + {12, "ELFOSABI_OPENBSD"}, + {13, "ELFOSABI_OPENVMS"}, + {14, "ELFOSABI_NSK"}, + {97, "ELFOSABI_ARM"}, + {255, "ELFOSABI_STANDALONE"}, +} + +func (i OSABI) String() string { return stringName(uint32(i), osabiStrings, false) } +func (i OSABI) GoString() string { return stringName(uint32(i), osabiStrings, true) } + +// Type is found in Header.Type. +type Type uint16 + +const ( + ET_NONE Type = 0 /* Unknown type. */ + ET_REL Type = 1 /* Relocatable. */ + ET_EXEC Type = 2 /* Executable. */ + ET_DYN Type = 3 /* Shared object. */ + ET_CORE Type = 4 /* Core file. */ + ET_LOOS Type = 0xfe00 /* First operating system specific. */ + ET_HIOS Type = 0xfeff /* Last operating system-specific. */ + ET_LOPROC Type = 0xff00 /* First processor-specific. */ + ET_HIPROC Type = 0xffff /* Last processor-specific. */ +) + +var typeStrings = []intName{ + {0, "ET_NONE"}, + {1, "ET_REL"}, + {2, "ET_EXEC"}, + {3, "ET_DYN"}, + {4, "ET_CORE"}, + {0xfe00, "ET_LOOS"}, + {0xfeff, "ET_HIOS"}, + {0xff00, "ET_LOPROC"}, + {0xffff, "ET_HIPROC"}, +} + +func (i Type) String() string { return stringName(uint32(i), typeStrings, false) } +func (i Type) GoString() string { return stringName(uint32(i), typeStrings, true) } + +// Machine is found in Header.Machine. +type Machine uint16 + +const ( + EM_NONE Machine = 0 /* Unknown machine. */ + EM_M32 Machine = 1 /* AT&T WE32100. */ + EM_SPARC Machine = 2 /* Sun SPARC. */ + EM_386 Machine = 3 /* Intel i386. */ + EM_68K Machine = 4 /* Motorola 68000. */ + EM_88K Machine = 5 /* Motorola 88000. */ + EM_860 Machine = 7 /* Intel i860. */ + EM_MIPS Machine = 8 /* MIPS R3000 Big-Endian only. */ + EM_S370 Machine = 9 /* IBM System/370. */ + EM_MIPS_RS3_LE Machine = 10 /* MIPS R3000 Little-Endian. */ + EM_PARISC Machine = 15 /* HP PA-RISC. */ + EM_VPP500 Machine = 17 /* Fujitsu VPP500. */ + EM_SPARC32PLUS Machine = 18 /* SPARC v8plus. */ + EM_960 Machine = 19 /* Intel 80960. */ + EM_PPC Machine = 20 /* PowerPC 32-bit. */ + EM_PPC64 Machine = 21 /* PowerPC 64-bit. */ + EM_S390 Machine = 22 /* IBM System/390. */ + EM_V800 Machine = 36 /* NEC V800. */ + EM_FR20 Machine = 37 /* Fujitsu FR20. */ + EM_RH32 Machine = 38 /* TRW RH-32. */ + EM_RCE Machine = 39 /* Motorola RCE. */ + EM_ARM Machine = 40 /* ARM. */ + EM_SH Machine = 42 /* Hitachi SH. */ + EM_SPARCV9 Machine = 43 /* SPARC v9 64-bit. */ + EM_TRICORE Machine = 44 /* Siemens TriCore embedded processor. */ + EM_ARC Machine = 45 /* Argonaut RISC Core. */ + EM_H8_300 Machine = 46 /* Hitachi H8/300. */ + EM_H8_300H Machine = 47 /* Hitachi H8/300H. */ + EM_H8S Machine = 48 /* Hitachi H8S. */ + EM_H8_500 Machine = 49 /* Hitachi H8/500. */ + EM_IA_64 Machine = 50 /* Intel IA-64 Processor. */ + EM_MIPS_X Machine = 51 /* Stanford MIPS-X. */ + EM_COLDFIRE Machine = 52 /* Motorola ColdFire. */ + EM_68HC12 Machine = 53 /* Motorola M68HC12. */ + EM_MMA Machine = 54 /* Fujitsu MMA. */ + EM_PCP Machine = 55 /* Siemens PCP. */ + EM_NCPU Machine = 56 /* Sony nCPU. */ + EM_NDR1 Machine = 57 /* Denso NDR1 microprocessor. */ + EM_STARCORE Machine = 58 /* Motorola Star*Core processor. */ + EM_ME16 Machine = 59 /* Toyota ME16 processor. */ + EM_ST100 Machine = 60 /* STMicroelectronics ST100 processor. */ + EM_TINYJ Machine = 61 /* Advanced Logic Corp. TinyJ processor. */ + EM_X86_64 Machine = 62 /* Advanced Micro Devices x86-64 */ + + /* Non-standard or deprecated. */ + EM_486 Machine = 6 /* Intel i486. */ + EM_MIPS_RS4_BE Machine = 10 /* MIPS R4000 Big-Endian */ + EM_ALPHA_STD Machine = 41 /* Digital Alpha (standard value). */ + EM_ALPHA Machine = 0x9026 /* Alpha (written in the absence of an ABI) */ +) + +var machineStrings = []intName{ + {0, "EM_NONE"}, + {1, "EM_M32"}, + {2, "EM_SPARC"}, + {3, "EM_386"}, + {4, "EM_68K"}, + {5, "EM_88K"}, + {7, "EM_860"}, + {8, "EM_MIPS"}, + {9, "EM_S370"}, + {10, "EM_MIPS_RS3_LE"}, + {15, "EM_PARISC"}, + {17, "EM_VPP500"}, + {18, "EM_SPARC32PLUS"}, + {19, "EM_960"}, + {20, "EM_PPC"}, + {21, "EM_PPC64"}, + {22, "EM_S390"}, + {36, "EM_V800"}, + {37, "EM_FR20"}, + {38, "EM_RH32"}, + {39, "EM_RCE"}, + {40, "EM_ARM"}, + {42, "EM_SH"}, + {43, "EM_SPARCV9"}, + {44, "EM_TRICORE"}, + {45, "EM_ARC"}, + {46, "EM_H8_300"}, + {47, "EM_H8_300H"}, + {48, "EM_H8S"}, + {49, "EM_H8_500"}, + {50, "EM_IA_64"}, + {51, "EM_MIPS_X"}, + {52, "EM_COLDFIRE"}, + {53, "EM_68HC12"}, + {54, "EM_MMA"}, + {55, "EM_PCP"}, + {56, "EM_NCPU"}, + {57, "EM_NDR1"}, + {58, "EM_STARCORE"}, + {59, "EM_ME16"}, + {60, "EM_ST100"}, + {61, "EM_TINYJ"}, + {62, "EM_X86_64"}, + + /* Non-standard or deprecated. */ + {6, "EM_486"}, + {10, "EM_MIPS_RS4_BE"}, + {41, "EM_ALPHA_STD"}, + {0x9026, "EM_ALPHA"}, +} + +func (i Machine) String() string { return stringName(uint32(i), machineStrings, false) } +func (i Machine) GoString() string { return stringName(uint32(i), machineStrings, true) } + +// Special section indices. +type SectionIndex int + +const ( + SHN_UNDEF SectionIndex = 0 /* Undefined, missing, irrelevant. */ + SHN_LORESERVE SectionIndex = 0xff00 /* First of reserved range. */ + SHN_LOPROC SectionIndex = 0xff00 /* First processor-specific. */ + SHN_HIPROC SectionIndex = 0xff1f /* Last processor-specific. */ + SHN_LOOS SectionIndex = 0xff20 /* First operating system-specific. */ + SHN_HIOS SectionIndex = 0xff3f /* Last operating system-specific. */ + SHN_ABS SectionIndex = 0xfff1 /* Absolute values. */ + SHN_COMMON SectionIndex = 0xfff2 /* Common data. */ + SHN_XINDEX SectionIndex = 0xffff /* Escape -- index stored elsewhere. */ + SHN_HIRESERVE SectionIndex = 0xffff /* Last of reserved range. */ +) + +var shnStrings = []intName{ + {0, "SHN_UNDEF"}, + {0xff00, "SHN_LOPROC"}, + {0xff20, "SHN_LOOS"}, + {0xfff1, "SHN_ABS"}, + {0xfff2, "SHN_COMMON"}, + {0xffff, "SHN_XINDEX"}, +} + +func (i SectionIndex) String() string { return stringName(uint32(i), shnStrings, false) } +func (i SectionIndex) GoString() string { return stringName(uint32(i), shnStrings, true) } + +// Section type. +type SectionType uint32 + +const ( + SHT_NULL SectionType = 0 /* inactive */ + SHT_PROGBITS SectionType = 1 /* program defined information */ + SHT_SYMTAB SectionType = 2 /* symbol table section */ + SHT_STRTAB SectionType = 3 /* string table section */ + SHT_RELA SectionType = 4 /* relocation section with addends */ + SHT_HASH SectionType = 5 /* symbol hash table section */ + SHT_DYNAMIC SectionType = 6 /* dynamic section */ + SHT_NOTE SectionType = 7 /* note section */ + SHT_NOBITS SectionType = 8 /* no space section */ + SHT_REL SectionType = 9 /* relocation section - no addends */ + SHT_SHLIB SectionType = 10 /* reserved - purpose unknown */ + SHT_DYNSYM SectionType = 11 /* dynamic symbol table section */ + SHT_INIT_ARRAY SectionType = 14 /* Initialization function pointers. */ + SHT_FINI_ARRAY SectionType = 15 /* Termination function pointers. */ + SHT_PREINIT_ARRAY SectionType = 16 /* Pre-initialization function ptrs. */ + SHT_GROUP SectionType = 17 /* Section group. */ + SHT_SYMTAB_SHNDX SectionType = 18 /* Section indexes (see SHN_XINDEX). */ + SHT_LOOS SectionType = 0x60000000 /* First of OS specific semantics */ + SHT_GNU_ATTRIBUTES SectionType = 0x6ffffff5 /* GNU object attributes */ + SHT_GNU_HASH SectionType = 0x6ffffff6 /* GNU hash table */ + SHT_GNU_LIBLIST SectionType = 0x6ffffff7 /* GNU prelink library list */ + SHT_GNU_VERDEF SectionType = 0x6ffffffd /* GNU version definition section */ + SHT_GNU_VERNEED SectionType = 0x6ffffffe /* GNU version needs section */ + SHT_GNU_VERSYM SectionType = 0x6fffffff /* GNU version symbol table */ + SHT_HIOS SectionType = 0x6fffffff /* Last of OS specific semantics */ + SHT_LOPROC SectionType = 0x70000000 /* reserved range for processor */ + SHT_HIPROC SectionType = 0x7fffffff /* specific section header types */ + SHT_LOUSER SectionType = 0x80000000 /* reserved range for application */ + SHT_HIUSER SectionType = 0xffffffff /* specific indexes */ +) + +var shtStrings = []intName{ + {0, "SHT_NULL"}, + {1, "SHT_PROGBITS"}, + {2, "SHT_SYMTAB"}, + {3, "SHT_STRTAB"}, + {4, "SHT_RELA"}, + {5, "SHT_HASH"}, + {6, "SHT_DYNAMIC"}, + {7, "SHT_NOTE"}, + {8, "SHT_NOBITS"}, + {9, "SHT_REL"}, + {10, "SHT_SHLIB"}, + {11, "SHT_DYNSYM"}, + {14, "SHT_INIT_ARRAY"}, + {15, "SHT_FINI_ARRAY"}, + {16, "SHT_PREINIT_ARRAY"}, + {17, "SHT_GROUP"}, + {18, "SHT_SYMTAB_SHNDX"}, + {0x60000000, "SHT_LOOS"}, + {0x6ffffff5, "SHT_GNU_ATTRIBUTES"}, + {0x6ffffff6, "SHT_GNU_HASH"}, + {0x6ffffff7, "SHT_GNU_LIBLIST"}, + {0x6ffffffd, "SHT_GNU_VERDEF"}, + {0x6ffffffe, "SHT_GNU_VERNEED"}, + {0x6fffffff, "SHT_GNU_VERSYM"}, + {0x70000000, "SHT_LOPROC"}, + {0x7fffffff, "SHT_HIPROC"}, + {0x80000000, "SHT_LOUSER"}, + {0xffffffff, "SHT_HIUSER"}, +} + +func (i SectionType) String() string { return stringName(uint32(i), shtStrings, false) } +func (i SectionType) GoString() string { return stringName(uint32(i), shtStrings, true) } + +// Section flags. +type SectionFlag uint32 + +const ( + SHF_WRITE SectionFlag = 0x1 /* Section contains writable data. */ + SHF_ALLOC SectionFlag = 0x2 /* Section occupies memory. */ + SHF_EXECINSTR SectionFlag = 0x4 /* Section contains instructions. */ + SHF_MERGE SectionFlag = 0x10 /* Section may be merged. */ + SHF_STRINGS SectionFlag = 0x20 /* Section contains strings. */ + SHF_INFO_LINK SectionFlag = 0x40 /* sh_info holds section index. */ + SHF_LINK_ORDER SectionFlag = 0x80 /* Special ordering requirements. */ + SHF_OS_NONCONFORMING SectionFlag = 0x100 /* OS-specific processing required. */ + SHF_GROUP SectionFlag = 0x200 /* Member of section group. */ + SHF_TLS SectionFlag = 0x400 /* Section contains TLS data. */ + SHF_MASKOS SectionFlag = 0x0ff00000 /* OS-specific semantics. */ + SHF_MASKPROC SectionFlag = 0xf0000000 /* Processor-specific semantics. */ +) + +var shfStrings = []intName{ + {0x1, "SHF_WRITE"}, + {0x2, "SHF_ALLOC"}, + {0x4, "SHF_EXECINSTR"}, + {0x10, "SHF_MERGE"}, + {0x20, "SHF_STRINGS"}, + {0x40, "SHF_INFO_LINK"}, + {0x80, "SHF_LINK_ORDER"}, + {0x100, "SHF_OS_NONCONFORMING"}, + {0x200, "SHF_GROUP"}, + {0x400, "SHF_TLS"}, +} + +func (i SectionFlag) String() string { return flagName(uint32(i), shfStrings, false) } +func (i SectionFlag) GoString() string { return flagName(uint32(i), shfStrings, true) } + +// Prog.Type +type ProgType int + +const ( + PT_NULL ProgType = 0 /* Unused entry. */ + PT_LOAD ProgType = 1 /* Loadable segment. */ + PT_DYNAMIC ProgType = 2 /* Dynamic linking information segment. */ + PT_INTERP ProgType = 3 /* Pathname of interpreter. */ + PT_NOTE ProgType = 4 /* Auxiliary information. */ + PT_SHLIB ProgType = 5 /* Reserved (not used). */ + PT_PHDR ProgType = 6 /* Location of program header itself. */ + PT_TLS ProgType = 7 /* Thread local storage segment */ + PT_LOOS ProgType = 0x60000000 /* First OS-specific. */ + PT_HIOS ProgType = 0x6fffffff /* Last OS-specific. */ + PT_LOPROC ProgType = 0x70000000 /* First processor-specific type. */ + PT_HIPROC ProgType = 0x7fffffff /* Last processor-specific type. */ +) + +var ptStrings = []intName{ + {0, "PT_NULL"}, + {1, "PT_LOAD"}, + {2, "PT_DYNAMIC"}, + {3, "PT_INTERP"}, + {4, "PT_NOTE"}, + {5, "PT_SHLIB"}, + {6, "PT_PHDR"}, + {7, "PT_TLS"}, + {0x60000000, "PT_LOOS"}, + {0x6fffffff, "PT_HIOS"}, + {0x70000000, "PT_LOPROC"}, + {0x7fffffff, "PT_HIPROC"}, +} + +func (i ProgType) String() string { return stringName(uint32(i), ptStrings, false) } +func (i ProgType) GoString() string { return stringName(uint32(i), ptStrings, true) } + +// Prog.Flag +type ProgFlag uint32 + +const ( + PF_X ProgFlag = 0x1 /* Executable. */ + PF_W ProgFlag = 0x2 /* Writable. */ + PF_R ProgFlag = 0x4 /* Readable. */ + PF_MASKOS ProgFlag = 0x0ff00000 /* Operating system-specific. */ + PF_MASKPROC ProgFlag = 0xf0000000 /* Processor-specific. */ +) + +var pfStrings = []intName{ + {0x1, "PF_X"}, + {0x2, "PF_W"}, + {0x4, "PF_R"}, +} + +func (i ProgFlag) String() string { return flagName(uint32(i), pfStrings, false) } +func (i ProgFlag) GoString() string { return flagName(uint32(i), pfStrings, true) } + +// Dyn.Tag +type DynTag int + +const ( + DT_NULL DynTag = 0 /* Terminating entry. */ + DT_NEEDED DynTag = 1 /* String table offset of a needed shared library. */ + DT_PLTRELSZ DynTag = 2 /* Total size in bytes of PLT relocations. */ + DT_PLTGOT DynTag = 3 /* Processor-dependent address. */ + DT_HASH DynTag = 4 /* Address of symbol hash table. */ + DT_STRTAB DynTag = 5 /* Address of string table. */ + DT_SYMTAB DynTag = 6 /* Address of symbol table. */ + DT_RELA DynTag = 7 /* Address of ElfNN_Rela relocations. */ + DT_RELASZ DynTag = 8 /* Total size of ElfNN_Rela relocations. */ + DT_RELAENT DynTag = 9 /* Size of each ElfNN_Rela relocation entry. */ + DT_STRSZ DynTag = 10 /* Size of string table. */ + DT_SYMENT DynTag = 11 /* Size of each symbol table entry. */ + DT_INIT DynTag = 12 /* Address of initialization function. */ + DT_FINI DynTag = 13 /* Address of finalization function. */ + DT_SONAME DynTag = 14 /* String table offset of shared object name. */ + DT_RPATH DynTag = 15 /* String table offset of library path. [sup] */ + DT_SYMBOLIC DynTag = 16 /* Indicates "symbolic" linking. [sup] */ + DT_REL DynTag = 17 /* Address of ElfNN_Rel relocations. */ + DT_RELSZ DynTag = 18 /* Total size of ElfNN_Rel relocations. */ + DT_RELENT DynTag = 19 /* Size of each ElfNN_Rel relocation. */ + DT_PLTREL DynTag = 20 /* Type of relocation used for PLT. */ + DT_DEBUG DynTag = 21 /* Reserved (not used). */ + DT_TEXTREL DynTag = 22 /* Indicates there may be relocations in non-writable segments. [sup] */ + DT_JMPREL DynTag = 23 /* Address of PLT relocations. */ + DT_BIND_NOW DynTag = 24 /* [sup] */ + DT_INIT_ARRAY DynTag = 25 /* Address of the array of pointers to initialization functions */ + DT_FINI_ARRAY DynTag = 26 /* Address of the array of pointers to termination functions */ + DT_INIT_ARRAYSZ DynTag = 27 /* Size in bytes of the array of initialization functions. */ + DT_FINI_ARRAYSZ DynTag = 28 /* Size in bytes of the array of terminationfunctions. */ + DT_RUNPATH DynTag = 29 /* String table offset of a null-terminated library search path string. */ + DT_FLAGS DynTag = 30 /* Object specific flag values. */ + DT_ENCODING DynTag = 32 /* Values greater than or equal to DT_ENCODING + and less than DT_LOOS follow the rules for + the interpretation of the d_un union + as follows: even == 'd_ptr', even == 'd_val' + or none */ + DT_PREINIT_ARRAY DynTag = 32 /* Address of the array of pointers to pre-initialization functions. */ + DT_PREINIT_ARRAYSZ DynTag = 33 /* Size in bytes of the array of pre-initialization functions. */ + DT_LOOS DynTag = 0x6000000d /* First OS-specific */ + DT_HIOS DynTag = 0x6ffff000 /* Last OS-specific */ + DT_VERSYM DynTag = 0x6ffffff0 + DT_VERNEED DynTag = 0x6ffffffe + DT_VERNEEDNUM DynTag = 0x6fffffff + DT_LOPROC DynTag = 0x70000000 /* First processor-specific type. */ + DT_HIPROC DynTag = 0x7fffffff /* Last processor-specific type. */ +) + +var dtStrings = []intName{ + {0, "DT_NULL"}, + {1, "DT_NEEDED"}, + {2, "DT_PLTRELSZ"}, + {3, "DT_PLTGOT"}, + {4, "DT_HASH"}, + {5, "DT_STRTAB"}, + {6, "DT_SYMTAB"}, + {7, "DT_RELA"}, + {8, "DT_RELASZ"}, + {9, "DT_RELAENT"}, + {10, "DT_STRSZ"}, + {11, "DT_SYMENT"}, + {12, "DT_INIT"}, + {13, "DT_FINI"}, + {14, "DT_SONAME"}, + {15, "DT_RPATH"}, + {16, "DT_SYMBOLIC"}, + {17, "DT_REL"}, + {18, "DT_RELSZ"}, + {19, "DT_RELENT"}, + {20, "DT_PLTREL"}, + {21, "DT_DEBUG"}, + {22, "DT_TEXTREL"}, + {23, "DT_JMPREL"}, + {24, "DT_BIND_NOW"}, + {25, "DT_INIT_ARRAY"}, + {26, "DT_FINI_ARRAY"}, + {27, "DT_INIT_ARRAYSZ"}, + {28, "DT_FINI_ARRAYSZ"}, + {29, "DT_RUNPATH"}, + {30, "DT_FLAGS"}, + {32, "DT_ENCODING"}, + {32, "DT_PREINIT_ARRAY"}, + {33, "DT_PREINIT_ARRAYSZ"}, + {0x6000000d, "DT_LOOS"}, + {0x6ffff000, "DT_HIOS"}, + {0x6ffffff0, "DT_VERSYM"}, + {0x6ffffffe, "DT_VERNEED"}, + {0x6fffffff, "DT_VERNEEDNUM"}, + {0x70000000, "DT_LOPROC"}, + {0x7fffffff, "DT_HIPROC"}, +} + +func (i DynTag) String() string { return stringName(uint32(i), dtStrings, false) } +func (i DynTag) GoString() string { return stringName(uint32(i), dtStrings, true) } + +// DT_FLAGS values. +type DynFlag int + +const ( + DF_ORIGIN DynFlag = 0x0001 /* Indicates that the object being loaded may + make reference to the + $ORIGIN substitution string */ + DF_SYMBOLIC DynFlag = 0x0002 /* Indicates "symbolic" linking. */ + DF_TEXTREL DynFlag = 0x0004 /* Indicates there may be relocations in non-writable segments. */ + DF_BIND_NOW DynFlag = 0x0008 /* Indicates that the dynamic linker should + process all relocations for the object + containing this entry before transferring + control to the program. */ + DF_STATIC_TLS DynFlag = 0x0010 /* Indicates that the shared object or + executable contains code using a static + thread-local storage scheme. */ +) + +var dflagStrings = []intName{ + {0x0001, "DF_ORIGIN"}, + {0x0002, "DF_SYMBOLIC"}, + {0x0004, "DF_TEXTREL"}, + {0x0008, "DF_BIND_NOW"}, + {0x0010, "DF_STATIC_TLS"}, +} + +func (i DynFlag) String() string { return flagName(uint32(i), dflagStrings, false) } +func (i DynFlag) GoString() string { return flagName(uint32(i), dflagStrings, true) } + +// NType values; used in core files. +type NType int + +const ( + NT_PRSTATUS NType = 1 /* Process status. */ + NT_FPREGSET NType = 2 /* Floating point registers. */ + NT_PRPSINFO NType = 3 /* Process state info. */ +) + +var ntypeStrings = []intName{ + {1, "NT_PRSTATUS"}, + {2, "NT_FPREGSET"}, + {3, "NT_PRPSINFO"}, +} + +func (i NType) String() string { return stringName(uint32(i), ntypeStrings, false) } +func (i NType) GoString() string { return stringName(uint32(i), ntypeStrings, true) } + +/* Symbol Binding - ELFNN_ST_BIND - st_info */ +type SymBind int + +const ( + STB_LOCAL SymBind = 0 /* Local symbol */ + STB_GLOBAL SymBind = 1 /* Global symbol */ + STB_WEAK SymBind = 2 /* like global - lower precedence */ + STB_LOOS SymBind = 10 /* Reserved range for operating system */ + STB_HIOS SymBind = 12 /* specific semantics. */ + STB_LOPROC SymBind = 13 /* reserved range for processor */ + STB_HIPROC SymBind = 15 /* specific semantics. */ +) + +var stbStrings = []intName{ + {0, "STB_LOCAL"}, + {1, "STB_GLOBAL"}, + {2, "STB_WEAK"}, + {10, "STB_LOOS"}, + {12, "STB_HIOS"}, + {13, "STB_LOPROC"}, + {15, "STB_HIPROC"}, +} + +func (i SymBind) String() string { return stringName(uint32(i), stbStrings, false) } +func (i SymBind) GoString() string { return stringName(uint32(i), stbStrings, true) } + +/* Symbol type - ELFNN_ST_TYPE - st_info */ +type SymType int + +const ( + STT_NOTYPE SymType = 0 /* Unspecified type. */ + STT_OBJECT SymType = 1 /* Data object. */ + STT_FUNC SymType = 2 /* Function. */ + STT_SECTION SymType = 3 /* Section. */ + STT_FILE SymType = 4 /* Source file. */ + STT_COMMON SymType = 5 /* Uninitialized common block. */ + STT_TLS SymType = 6 /* TLS object. */ + STT_LOOS SymType = 10 /* Reserved range for operating system */ + STT_HIOS SymType = 12 /* specific semantics. */ + STT_LOPROC SymType = 13 /* reserved range for processor */ + STT_HIPROC SymType = 15 /* specific semantics. */ +) + +var sttStrings = []intName{ + {0, "STT_NOTYPE"}, + {1, "STT_OBJECT"}, + {2, "STT_FUNC"}, + {3, "STT_SECTION"}, + {4, "STT_FILE"}, + {5, "STT_COMMON"}, + {6, "STT_TLS"}, + {10, "STT_LOOS"}, + {12, "STT_HIOS"}, + {13, "STT_LOPROC"}, + {15, "STT_HIPROC"}, +} + +func (i SymType) String() string { return stringName(uint32(i), sttStrings, false) } +func (i SymType) GoString() string { return stringName(uint32(i), sttStrings, true) } + +/* Symbol visibility - ELFNN_ST_VISIBILITY - st_other */ +type SymVis int + +const ( + STV_DEFAULT SymVis = 0x0 /* Default visibility (see binding). */ + STV_INTERNAL SymVis = 0x1 /* Special meaning in relocatable objects. */ + STV_HIDDEN SymVis = 0x2 /* Not visible. */ + STV_PROTECTED SymVis = 0x3 /* Visible but not preemptible. */ +) + +var stvStrings = []intName{ + {0x0, "STV_DEFAULT"}, + {0x1, "STV_INTERNAL"}, + {0x2, "STV_HIDDEN"}, + {0x3, "STV_PROTECTED"}, +} + +func (i SymVis) String() string { return stringName(uint32(i), stvStrings, false) } +func (i SymVis) GoString() string { return stringName(uint32(i), stvStrings, true) } + +/* + * Relocation types. + */ + +// Relocation types for x86-64. +type R_X86_64 int + +const ( + R_X86_64_NONE R_X86_64 = 0 /* No relocation. */ + R_X86_64_64 R_X86_64 = 1 /* Add 64 bit symbol value. */ + R_X86_64_PC32 R_X86_64 = 2 /* PC-relative 32 bit signed sym value. */ + R_X86_64_GOT32 R_X86_64 = 3 /* PC-relative 32 bit GOT offset. */ + R_X86_64_PLT32 R_X86_64 = 4 /* PC-relative 32 bit PLT offset. */ + R_X86_64_COPY R_X86_64 = 5 /* Copy data from shared object. */ + R_X86_64_GLOB_DAT R_X86_64 = 6 /* Set GOT entry to data address. */ + R_X86_64_JMP_SLOT R_X86_64 = 7 /* Set GOT entry to code address. */ + R_X86_64_RELATIVE R_X86_64 = 8 /* Add load address of shared object. */ + R_X86_64_GOTPCREL R_X86_64 = 9 /* Add 32 bit signed pcrel offset to GOT. */ + R_X86_64_32 R_X86_64 = 10 /* Add 32 bit zero extended symbol value */ + R_X86_64_32S R_X86_64 = 11 /* Add 32 bit sign extended symbol value */ + R_X86_64_16 R_X86_64 = 12 /* Add 16 bit zero extended symbol value */ + R_X86_64_PC16 R_X86_64 = 13 /* Add 16 bit signed extended pc relative symbol value */ + R_X86_64_8 R_X86_64 = 14 /* Add 8 bit zero extended symbol value */ + R_X86_64_PC8 R_X86_64 = 15 /* Add 8 bit signed extended pc relative symbol value */ + R_X86_64_DTPMOD64 R_X86_64 = 16 /* ID of module containing symbol */ + R_X86_64_DTPOFF64 R_X86_64 = 17 /* Offset in TLS block */ + R_X86_64_TPOFF64 R_X86_64 = 18 /* Offset in static TLS block */ + R_X86_64_TLSGD R_X86_64 = 19 /* PC relative offset to GD GOT entry */ + R_X86_64_TLSLD R_X86_64 = 20 /* PC relative offset to LD GOT entry */ + R_X86_64_DTPOFF32 R_X86_64 = 21 /* Offset in TLS block */ + R_X86_64_GOTTPOFF R_X86_64 = 22 /* PC relative offset to IE GOT entry */ + R_X86_64_TPOFF32 R_X86_64 = 23 /* Offset in static TLS block */ +) + +var rx86_64Strings = []intName{ + {0, "R_X86_64_NONE"}, + {1, "R_X86_64_64"}, + {2, "R_X86_64_PC32"}, + {3, "R_X86_64_GOT32"}, + {4, "R_X86_64_PLT32"}, + {5, "R_X86_64_COPY"}, + {6, "R_X86_64_GLOB_DAT"}, + {7, "R_X86_64_JMP_SLOT"}, + {8, "R_X86_64_RELATIVE"}, + {9, "R_X86_64_GOTPCREL"}, + {10, "R_X86_64_32"}, + {11, "R_X86_64_32S"}, + {12, "R_X86_64_16"}, + {13, "R_X86_64_PC16"}, + {14, "R_X86_64_8"}, + {15, "R_X86_64_PC8"}, + {16, "R_X86_64_DTPMOD64"}, + {17, "R_X86_64_DTPOFF64"}, + {18, "R_X86_64_TPOFF64"}, + {19, "R_X86_64_TLSGD"}, + {20, "R_X86_64_TLSLD"}, + {21, "R_X86_64_DTPOFF32"}, + {22, "R_X86_64_GOTTPOFF"}, + {23, "R_X86_64_TPOFF32"}, +} + +func (i R_X86_64) String() string { return stringName(uint32(i), rx86_64Strings, false) } +func (i R_X86_64) GoString() string { return stringName(uint32(i), rx86_64Strings, true) } + +// Relocation types for Alpha. +type R_ALPHA int + +const ( + R_ALPHA_NONE R_ALPHA = 0 /* No reloc */ + R_ALPHA_REFLONG R_ALPHA = 1 /* Direct 32 bit */ + R_ALPHA_REFQUAD R_ALPHA = 2 /* Direct 64 bit */ + R_ALPHA_GPREL32 R_ALPHA = 3 /* GP relative 32 bit */ + R_ALPHA_LITERAL R_ALPHA = 4 /* GP relative 16 bit w/optimization */ + R_ALPHA_LITUSE R_ALPHA = 5 /* Optimization hint for LITERAL */ + R_ALPHA_GPDISP R_ALPHA = 6 /* Add displacement to GP */ + R_ALPHA_BRADDR R_ALPHA = 7 /* PC+4 relative 23 bit shifted */ + R_ALPHA_HINT R_ALPHA = 8 /* PC+4 relative 16 bit shifted */ + R_ALPHA_SREL16 R_ALPHA = 9 /* PC relative 16 bit */ + R_ALPHA_SREL32 R_ALPHA = 10 /* PC relative 32 bit */ + R_ALPHA_SREL64 R_ALPHA = 11 /* PC relative 64 bit */ + R_ALPHA_OP_PUSH R_ALPHA = 12 /* OP stack push */ + R_ALPHA_OP_STORE R_ALPHA = 13 /* OP stack pop and store */ + R_ALPHA_OP_PSUB R_ALPHA = 14 /* OP stack subtract */ + R_ALPHA_OP_PRSHIFT R_ALPHA = 15 /* OP stack right shift */ + R_ALPHA_GPVALUE R_ALPHA = 16 + R_ALPHA_GPRELHIGH R_ALPHA = 17 + R_ALPHA_GPRELLOW R_ALPHA = 18 + R_ALPHA_IMMED_GP_16 R_ALPHA = 19 + R_ALPHA_IMMED_GP_HI32 R_ALPHA = 20 + R_ALPHA_IMMED_SCN_HI32 R_ALPHA = 21 + R_ALPHA_IMMED_BR_HI32 R_ALPHA = 22 + R_ALPHA_IMMED_LO32 R_ALPHA = 23 + R_ALPHA_COPY R_ALPHA = 24 /* Copy symbol at runtime */ + R_ALPHA_GLOB_DAT R_ALPHA = 25 /* Create GOT entry */ + R_ALPHA_JMP_SLOT R_ALPHA = 26 /* Create PLT entry */ + R_ALPHA_RELATIVE R_ALPHA = 27 /* Adjust by program base */ +) + +var ralphaStrings = []intName{ + {0, "R_ALPHA_NONE"}, + {1, "R_ALPHA_REFLONG"}, + {2, "R_ALPHA_REFQUAD"}, + {3, "R_ALPHA_GPREL32"}, + {4, "R_ALPHA_LITERAL"}, + {5, "R_ALPHA_LITUSE"}, + {6, "R_ALPHA_GPDISP"}, + {7, "R_ALPHA_BRADDR"}, + {8, "R_ALPHA_HINT"}, + {9, "R_ALPHA_SREL16"}, + {10, "R_ALPHA_SREL32"}, + {11, "R_ALPHA_SREL64"}, + {12, "R_ALPHA_OP_PUSH"}, + {13, "R_ALPHA_OP_STORE"}, + {14, "R_ALPHA_OP_PSUB"}, + {15, "R_ALPHA_OP_PRSHIFT"}, + {16, "R_ALPHA_GPVALUE"}, + {17, "R_ALPHA_GPRELHIGH"}, + {18, "R_ALPHA_GPRELLOW"}, + {19, "R_ALPHA_IMMED_GP_16"}, + {20, "R_ALPHA_IMMED_GP_HI32"}, + {21, "R_ALPHA_IMMED_SCN_HI32"}, + {22, "R_ALPHA_IMMED_BR_HI32"}, + {23, "R_ALPHA_IMMED_LO32"}, + {24, "R_ALPHA_COPY"}, + {25, "R_ALPHA_GLOB_DAT"}, + {26, "R_ALPHA_JMP_SLOT"}, + {27, "R_ALPHA_RELATIVE"}, +} + +func (i R_ALPHA) String() string { return stringName(uint32(i), ralphaStrings, false) } +func (i R_ALPHA) GoString() string { return stringName(uint32(i), ralphaStrings, true) } + +// Relocation types for ARM. +type R_ARM int + +const ( + R_ARM_NONE R_ARM = 0 /* No relocation. */ + R_ARM_PC24 R_ARM = 1 + R_ARM_ABS32 R_ARM = 2 + R_ARM_REL32 R_ARM = 3 + R_ARM_PC13 R_ARM = 4 + R_ARM_ABS16 R_ARM = 5 + R_ARM_ABS12 R_ARM = 6 + R_ARM_THM_ABS5 R_ARM = 7 + R_ARM_ABS8 R_ARM = 8 + R_ARM_SBREL32 R_ARM = 9 + R_ARM_THM_PC22 R_ARM = 10 + R_ARM_THM_PC8 R_ARM = 11 + R_ARM_AMP_VCALL9 R_ARM = 12 + R_ARM_SWI24 R_ARM = 13 + R_ARM_THM_SWI8 R_ARM = 14 + R_ARM_XPC25 R_ARM = 15 + R_ARM_THM_XPC22 R_ARM = 16 + R_ARM_COPY R_ARM = 20 /* Copy data from shared object. */ + R_ARM_GLOB_DAT R_ARM = 21 /* Set GOT entry to data address. */ + R_ARM_JUMP_SLOT R_ARM = 22 /* Set GOT entry to code address. */ + R_ARM_RELATIVE R_ARM = 23 /* Add load address of shared object. */ + R_ARM_GOTOFF R_ARM = 24 /* Add GOT-relative symbol address. */ + R_ARM_GOTPC R_ARM = 25 /* Add PC-relative GOT table address. */ + R_ARM_GOT32 R_ARM = 26 /* Add PC-relative GOT offset. */ + R_ARM_PLT32 R_ARM = 27 /* Add PC-relative PLT offset. */ + R_ARM_GNU_VTENTRY R_ARM = 100 + R_ARM_GNU_VTINHERIT R_ARM = 101 + R_ARM_RSBREL32 R_ARM = 250 + R_ARM_THM_RPC22 R_ARM = 251 + R_ARM_RREL32 R_ARM = 252 + R_ARM_RABS32 R_ARM = 253 + R_ARM_RPC24 R_ARM = 254 + R_ARM_RBASE R_ARM = 255 +) + +var rarmStrings = []intName{ + {0, "R_ARM_NONE"}, + {1, "R_ARM_PC24"}, + {2, "R_ARM_ABS32"}, + {3, "R_ARM_REL32"}, + {4, "R_ARM_PC13"}, + {5, "R_ARM_ABS16"}, + {6, "R_ARM_ABS12"}, + {7, "R_ARM_THM_ABS5"}, + {8, "R_ARM_ABS8"}, + {9, "R_ARM_SBREL32"}, + {10, "R_ARM_THM_PC22"}, + {11, "R_ARM_THM_PC8"}, + {12, "R_ARM_AMP_VCALL9"}, + {13, "R_ARM_SWI24"}, + {14, "R_ARM_THM_SWI8"}, + {15, "R_ARM_XPC25"}, + {16, "R_ARM_THM_XPC22"}, + {20, "R_ARM_COPY"}, + {21, "R_ARM_GLOB_DAT"}, + {22, "R_ARM_JUMP_SLOT"}, + {23, "R_ARM_RELATIVE"}, + {24, "R_ARM_GOTOFF"}, + {25, "R_ARM_GOTPC"}, + {26, "R_ARM_GOT32"}, + {27, "R_ARM_PLT32"}, + {100, "R_ARM_GNU_VTENTRY"}, + {101, "R_ARM_GNU_VTINHERIT"}, + {250, "R_ARM_RSBREL32"}, + {251, "R_ARM_THM_RPC22"}, + {252, "R_ARM_RREL32"}, + {253, "R_ARM_RABS32"}, + {254, "R_ARM_RPC24"}, + {255, "R_ARM_RBASE"}, +} + +func (i R_ARM) String() string { return stringName(uint32(i), rarmStrings, false) } +func (i R_ARM) GoString() string { return stringName(uint32(i), rarmStrings, true) } + +// Relocation types for 386. +type R_386 int + +const ( + R_386_NONE R_386 = 0 /* No relocation. */ + R_386_32 R_386 = 1 /* Add symbol value. */ + R_386_PC32 R_386 = 2 /* Add PC-relative symbol value. */ + R_386_GOT32 R_386 = 3 /* Add PC-relative GOT offset. */ + R_386_PLT32 R_386 = 4 /* Add PC-relative PLT offset. */ + R_386_COPY R_386 = 5 /* Copy data from shared object. */ + R_386_GLOB_DAT R_386 = 6 /* Set GOT entry to data address. */ + R_386_JMP_SLOT R_386 = 7 /* Set GOT entry to code address. */ + R_386_RELATIVE R_386 = 8 /* Add load address of shared object. */ + R_386_GOTOFF R_386 = 9 /* Add GOT-relative symbol address. */ + R_386_GOTPC R_386 = 10 /* Add PC-relative GOT table address. */ + R_386_TLS_TPOFF R_386 = 14 /* Negative offset in static TLS block */ + R_386_TLS_IE R_386 = 15 /* Absolute address of GOT for -ve static TLS */ + R_386_TLS_GOTIE R_386 = 16 /* GOT entry for negative static TLS block */ + R_386_TLS_LE R_386 = 17 /* Negative offset relative to static TLS */ + R_386_TLS_GD R_386 = 18 /* 32 bit offset to GOT (index,off) pair */ + R_386_TLS_LDM R_386 = 19 /* 32 bit offset to GOT (index,zero) pair */ + R_386_TLS_GD_32 R_386 = 24 /* 32 bit offset to GOT (index,off) pair */ + R_386_TLS_GD_PUSH R_386 = 25 /* pushl instruction for Sun ABI GD sequence */ + R_386_TLS_GD_CALL R_386 = 26 /* call instruction for Sun ABI GD sequence */ + R_386_TLS_GD_POP R_386 = 27 /* popl instruction for Sun ABI GD sequence */ + R_386_TLS_LDM_32 R_386 = 28 /* 32 bit offset to GOT (index,zero) pair */ + R_386_TLS_LDM_PUSH R_386 = 29 /* pushl instruction for Sun ABI LD sequence */ + R_386_TLS_LDM_CALL R_386 = 30 /* call instruction for Sun ABI LD sequence */ + R_386_TLS_LDM_POP R_386 = 31 /* popl instruction for Sun ABI LD sequence */ + R_386_TLS_LDO_32 R_386 = 32 /* 32 bit offset from start of TLS block */ + R_386_TLS_IE_32 R_386 = 33 /* 32 bit offset to GOT static TLS offset entry */ + R_386_TLS_LE_32 R_386 = 34 /* 32 bit offset within static TLS block */ + R_386_TLS_DTPMOD32 R_386 = 35 /* GOT entry containing TLS index */ + R_386_TLS_DTPOFF32 R_386 = 36 /* GOT entry containing TLS offset */ + R_386_TLS_TPOFF32 R_386 = 37 /* GOT entry of -ve static TLS offset */ +) + +var r386Strings = []intName{ + {0, "R_386_NONE"}, + {1, "R_386_32"}, + {2, "R_386_PC32"}, + {3, "R_386_GOT32"}, + {4, "R_386_PLT32"}, + {5, "R_386_COPY"}, + {6, "R_386_GLOB_DAT"}, + {7, "R_386_JMP_SLOT"}, + {8, "R_386_RELATIVE"}, + {9, "R_386_GOTOFF"}, + {10, "R_386_GOTPC"}, + {14, "R_386_TLS_TPOFF"}, + {15, "R_386_TLS_IE"}, + {16, "R_386_TLS_GOTIE"}, + {17, "R_386_TLS_LE"}, + {18, "R_386_TLS_GD"}, + {19, "R_386_TLS_LDM"}, + {24, "R_386_TLS_GD_32"}, + {25, "R_386_TLS_GD_PUSH"}, + {26, "R_386_TLS_GD_CALL"}, + {27, "R_386_TLS_GD_POP"}, + {28, "R_386_TLS_LDM_32"}, + {29, "R_386_TLS_LDM_PUSH"}, + {30, "R_386_TLS_LDM_CALL"}, + {31, "R_386_TLS_LDM_POP"}, + {32, "R_386_TLS_LDO_32"}, + {33, "R_386_TLS_IE_32"}, + {34, "R_386_TLS_LE_32"}, + {35, "R_386_TLS_DTPMOD32"}, + {36, "R_386_TLS_DTPOFF32"}, + {37, "R_386_TLS_TPOFF32"}, +} + +func (i R_386) String() string { return stringName(uint32(i), r386Strings, false) } +func (i R_386) GoString() string { return stringName(uint32(i), r386Strings, true) } + +// Relocation types for PowerPC. +type R_PPC int + +const ( + R_PPC_NONE R_PPC = 0 /* No relocation. */ + R_PPC_ADDR32 R_PPC = 1 + R_PPC_ADDR24 R_PPC = 2 + R_PPC_ADDR16 R_PPC = 3 + R_PPC_ADDR16_LO R_PPC = 4 + R_PPC_ADDR16_HI R_PPC = 5 + R_PPC_ADDR16_HA R_PPC = 6 + R_PPC_ADDR14 R_PPC = 7 + R_PPC_ADDR14_BRTAKEN R_PPC = 8 + R_PPC_ADDR14_BRNTAKEN R_PPC = 9 + R_PPC_REL24 R_PPC = 10 + R_PPC_REL14 R_PPC = 11 + R_PPC_REL14_BRTAKEN R_PPC = 12 + R_PPC_REL14_BRNTAKEN R_PPC = 13 + R_PPC_GOT16 R_PPC = 14 + R_PPC_GOT16_LO R_PPC = 15 + R_PPC_GOT16_HI R_PPC = 16 + R_PPC_GOT16_HA R_PPC = 17 + R_PPC_PLTREL24 R_PPC = 18 + R_PPC_COPY R_PPC = 19 + R_PPC_GLOB_DAT R_PPC = 20 + R_PPC_JMP_SLOT R_PPC = 21 + R_PPC_RELATIVE R_PPC = 22 + R_PPC_LOCAL24PC R_PPC = 23 + R_PPC_UADDR32 R_PPC = 24 + R_PPC_UADDR16 R_PPC = 25 + R_PPC_REL32 R_PPC = 26 + R_PPC_PLT32 R_PPC = 27 + R_PPC_PLTREL32 R_PPC = 28 + R_PPC_PLT16_LO R_PPC = 29 + R_PPC_PLT16_HI R_PPC = 30 + R_PPC_PLT16_HA R_PPC = 31 + R_PPC_SDAREL16 R_PPC = 32 + R_PPC_SECTOFF R_PPC = 33 + R_PPC_SECTOFF_LO R_PPC = 34 + R_PPC_SECTOFF_HI R_PPC = 35 + R_PPC_SECTOFF_HA R_PPC = 36 + R_PPC_TLS R_PPC = 67 + R_PPC_DTPMOD32 R_PPC = 68 + R_PPC_TPREL16 R_PPC = 69 + R_PPC_TPREL16_LO R_PPC = 70 + R_PPC_TPREL16_HI R_PPC = 71 + R_PPC_TPREL16_HA R_PPC = 72 + R_PPC_TPREL32 R_PPC = 73 + R_PPC_DTPREL16 R_PPC = 74 + R_PPC_DTPREL16_LO R_PPC = 75 + R_PPC_DTPREL16_HI R_PPC = 76 + R_PPC_DTPREL16_HA R_PPC = 77 + R_PPC_DTPREL32 R_PPC = 78 + R_PPC_GOT_TLSGD16 R_PPC = 79 + R_PPC_GOT_TLSGD16_LO R_PPC = 80 + R_PPC_GOT_TLSGD16_HI R_PPC = 81 + R_PPC_GOT_TLSGD16_HA R_PPC = 82 + R_PPC_GOT_TLSLD16 R_PPC = 83 + R_PPC_GOT_TLSLD16_LO R_PPC = 84 + R_PPC_GOT_TLSLD16_HI R_PPC = 85 + R_PPC_GOT_TLSLD16_HA R_PPC = 86 + R_PPC_GOT_TPREL16 R_PPC = 87 + R_PPC_GOT_TPREL16_LO R_PPC = 88 + R_PPC_GOT_TPREL16_HI R_PPC = 89 + R_PPC_GOT_TPREL16_HA R_PPC = 90 + R_PPC_EMB_NADDR32 R_PPC = 101 + R_PPC_EMB_NADDR16 R_PPC = 102 + R_PPC_EMB_NADDR16_LO R_PPC = 103 + R_PPC_EMB_NADDR16_HI R_PPC = 104 + R_PPC_EMB_NADDR16_HA R_PPC = 105 + R_PPC_EMB_SDAI16 R_PPC = 106 + R_PPC_EMB_SDA2I16 R_PPC = 107 + R_PPC_EMB_SDA2REL R_PPC = 108 + R_PPC_EMB_SDA21 R_PPC = 109 + R_PPC_EMB_MRKREF R_PPC = 110 + R_PPC_EMB_RELSEC16 R_PPC = 111 + R_PPC_EMB_RELST_LO R_PPC = 112 + R_PPC_EMB_RELST_HI R_PPC = 113 + R_PPC_EMB_RELST_HA R_PPC = 114 + R_PPC_EMB_BIT_FLD R_PPC = 115 + R_PPC_EMB_RELSDA R_PPC = 116 +) + +var rppcStrings = []intName{ + {0, "R_PPC_NONE"}, + {1, "R_PPC_ADDR32"}, + {2, "R_PPC_ADDR24"}, + {3, "R_PPC_ADDR16"}, + {4, "R_PPC_ADDR16_LO"}, + {5, "R_PPC_ADDR16_HI"}, + {6, "R_PPC_ADDR16_HA"}, + {7, "R_PPC_ADDR14"}, + {8, "R_PPC_ADDR14_BRTAKEN"}, + {9, "R_PPC_ADDR14_BRNTAKEN"}, + {10, "R_PPC_REL24"}, + {11, "R_PPC_REL14"}, + {12, "R_PPC_REL14_BRTAKEN"}, + {13, "R_PPC_REL14_BRNTAKEN"}, + {14, "R_PPC_GOT16"}, + {15, "R_PPC_GOT16_LO"}, + {16, "R_PPC_GOT16_HI"}, + {17, "R_PPC_GOT16_HA"}, + {18, "R_PPC_PLTREL24"}, + {19, "R_PPC_COPY"}, + {20, "R_PPC_GLOB_DAT"}, + {21, "R_PPC_JMP_SLOT"}, + {22, "R_PPC_RELATIVE"}, + {23, "R_PPC_LOCAL24PC"}, + {24, "R_PPC_UADDR32"}, + {25, "R_PPC_UADDR16"}, + {26, "R_PPC_REL32"}, + {27, "R_PPC_PLT32"}, + {28, "R_PPC_PLTREL32"}, + {29, "R_PPC_PLT16_LO"}, + {30, "R_PPC_PLT16_HI"}, + {31, "R_PPC_PLT16_HA"}, + {32, "R_PPC_SDAREL16"}, + {33, "R_PPC_SECTOFF"}, + {34, "R_PPC_SECTOFF_LO"}, + {35, "R_PPC_SECTOFF_HI"}, + {36, "R_PPC_SECTOFF_HA"}, + + {67, "R_PPC_TLS"}, + {68, "R_PPC_DTPMOD32"}, + {69, "R_PPC_TPREL16"}, + {70, "R_PPC_TPREL16_LO"}, + {71, "R_PPC_TPREL16_HI"}, + {72, "R_PPC_TPREL16_HA"}, + {73, "R_PPC_TPREL32"}, + {74, "R_PPC_DTPREL16"}, + {75, "R_PPC_DTPREL16_LO"}, + {76, "R_PPC_DTPREL16_HI"}, + {77, "R_PPC_DTPREL16_HA"}, + {78, "R_PPC_DTPREL32"}, + {79, "R_PPC_GOT_TLSGD16"}, + {80, "R_PPC_GOT_TLSGD16_LO"}, + {81, "R_PPC_GOT_TLSGD16_HI"}, + {82, "R_PPC_GOT_TLSGD16_HA"}, + {83, "R_PPC_GOT_TLSLD16"}, + {84, "R_PPC_GOT_TLSLD16_LO"}, + {85, "R_PPC_GOT_TLSLD16_HI"}, + {86, "R_PPC_GOT_TLSLD16_HA"}, + {87, "R_PPC_GOT_TPREL16"}, + {88, "R_PPC_GOT_TPREL16_LO"}, + {89, "R_PPC_GOT_TPREL16_HI"}, + {90, "R_PPC_GOT_TPREL16_HA"}, + + {101, "R_PPC_EMB_NADDR32"}, + {102, "R_PPC_EMB_NADDR16"}, + {103, "R_PPC_EMB_NADDR16_LO"}, + {104, "R_PPC_EMB_NADDR16_HI"}, + {105, "R_PPC_EMB_NADDR16_HA"}, + {106, "R_PPC_EMB_SDAI16"}, + {107, "R_PPC_EMB_SDA2I16"}, + {108, "R_PPC_EMB_SDA2REL"}, + {109, "R_PPC_EMB_SDA21"}, + {110, "R_PPC_EMB_MRKREF"}, + {111, "R_PPC_EMB_RELSEC16"}, + {112, "R_PPC_EMB_RELST_LO"}, + {113, "R_PPC_EMB_RELST_HI"}, + {114, "R_PPC_EMB_RELST_HA"}, + {115, "R_PPC_EMB_BIT_FLD"}, + {116, "R_PPC_EMB_RELSDA"}, +} + +func (i R_PPC) String() string { return stringName(uint32(i), rppcStrings, false) } +func (i R_PPC) GoString() string { return stringName(uint32(i), rppcStrings, true) } + +// Relocation types for SPARC. +type R_SPARC int + +const ( + R_SPARC_NONE R_SPARC = 0 + R_SPARC_8 R_SPARC = 1 + R_SPARC_16 R_SPARC = 2 + R_SPARC_32 R_SPARC = 3 + R_SPARC_DISP8 R_SPARC = 4 + R_SPARC_DISP16 R_SPARC = 5 + R_SPARC_DISP32 R_SPARC = 6 + R_SPARC_WDISP30 R_SPARC = 7 + R_SPARC_WDISP22 R_SPARC = 8 + R_SPARC_HI22 R_SPARC = 9 + R_SPARC_22 R_SPARC = 10 + R_SPARC_13 R_SPARC = 11 + R_SPARC_LO10 R_SPARC = 12 + R_SPARC_GOT10 R_SPARC = 13 + R_SPARC_GOT13 R_SPARC = 14 + R_SPARC_GOT22 R_SPARC = 15 + R_SPARC_PC10 R_SPARC = 16 + R_SPARC_PC22 R_SPARC = 17 + R_SPARC_WPLT30 R_SPARC = 18 + R_SPARC_COPY R_SPARC = 19 + R_SPARC_GLOB_DAT R_SPARC = 20 + R_SPARC_JMP_SLOT R_SPARC = 21 + R_SPARC_RELATIVE R_SPARC = 22 + R_SPARC_UA32 R_SPARC = 23 + R_SPARC_PLT32 R_SPARC = 24 + R_SPARC_HIPLT22 R_SPARC = 25 + R_SPARC_LOPLT10 R_SPARC = 26 + R_SPARC_PCPLT32 R_SPARC = 27 + R_SPARC_PCPLT22 R_SPARC = 28 + R_SPARC_PCPLT10 R_SPARC = 29 + R_SPARC_10 R_SPARC = 30 + R_SPARC_11 R_SPARC = 31 + R_SPARC_64 R_SPARC = 32 + R_SPARC_OLO10 R_SPARC = 33 + R_SPARC_HH22 R_SPARC = 34 + R_SPARC_HM10 R_SPARC = 35 + R_SPARC_LM22 R_SPARC = 36 + R_SPARC_PC_HH22 R_SPARC = 37 + R_SPARC_PC_HM10 R_SPARC = 38 + R_SPARC_PC_LM22 R_SPARC = 39 + R_SPARC_WDISP16 R_SPARC = 40 + R_SPARC_WDISP19 R_SPARC = 41 + R_SPARC_GLOB_JMP R_SPARC = 42 + R_SPARC_7 R_SPARC = 43 + R_SPARC_5 R_SPARC = 44 + R_SPARC_6 R_SPARC = 45 + R_SPARC_DISP64 R_SPARC = 46 + R_SPARC_PLT64 R_SPARC = 47 + R_SPARC_HIX22 R_SPARC = 48 + R_SPARC_LOX10 R_SPARC = 49 + R_SPARC_H44 R_SPARC = 50 + R_SPARC_M44 R_SPARC = 51 + R_SPARC_L44 R_SPARC = 52 + R_SPARC_REGISTER R_SPARC = 53 + R_SPARC_UA64 R_SPARC = 54 + R_SPARC_UA16 R_SPARC = 55 +) + +var rsparcStrings = []intName{ + {0, "R_SPARC_NONE"}, + {1, "R_SPARC_8"}, + {2, "R_SPARC_16"}, + {3, "R_SPARC_32"}, + {4, "R_SPARC_DISP8"}, + {5, "R_SPARC_DISP16"}, + {6, "R_SPARC_DISP32"}, + {7, "R_SPARC_WDISP30"}, + {8, "R_SPARC_WDISP22"}, + {9, "R_SPARC_HI22"}, + {10, "R_SPARC_22"}, + {11, "R_SPARC_13"}, + {12, "R_SPARC_LO10"}, + {13, "R_SPARC_GOT10"}, + {14, "R_SPARC_GOT13"}, + {15, "R_SPARC_GOT22"}, + {16, "R_SPARC_PC10"}, + {17, "R_SPARC_PC22"}, + {18, "R_SPARC_WPLT30"}, + {19, "R_SPARC_COPY"}, + {20, "R_SPARC_GLOB_DAT"}, + {21, "R_SPARC_JMP_SLOT"}, + {22, "R_SPARC_RELATIVE"}, + {23, "R_SPARC_UA32"}, + {24, "R_SPARC_PLT32"}, + {25, "R_SPARC_HIPLT22"}, + {26, "R_SPARC_LOPLT10"}, + {27, "R_SPARC_PCPLT32"}, + {28, "R_SPARC_PCPLT22"}, + {29, "R_SPARC_PCPLT10"}, + {30, "R_SPARC_10"}, + {31, "R_SPARC_11"}, + {32, "R_SPARC_64"}, + {33, "R_SPARC_OLO10"}, + {34, "R_SPARC_HH22"}, + {35, "R_SPARC_HM10"}, + {36, "R_SPARC_LM22"}, + {37, "R_SPARC_PC_HH22"}, + {38, "R_SPARC_PC_HM10"}, + {39, "R_SPARC_PC_LM22"}, + {40, "R_SPARC_WDISP16"}, + {41, "R_SPARC_WDISP19"}, + {42, "R_SPARC_GLOB_JMP"}, + {43, "R_SPARC_7"}, + {44, "R_SPARC_5"}, + {45, "R_SPARC_6"}, + {46, "R_SPARC_DISP64"}, + {47, "R_SPARC_PLT64"}, + {48, "R_SPARC_HIX22"}, + {49, "R_SPARC_LOX10"}, + {50, "R_SPARC_H44"}, + {51, "R_SPARC_M44"}, + {52, "R_SPARC_L44"}, + {53, "R_SPARC_REGISTER"}, + {54, "R_SPARC_UA64"}, + {55, "R_SPARC_UA16"}, +} + +func (i R_SPARC) String() string { return stringName(uint32(i), rsparcStrings, false) } +func (i R_SPARC) GoString() string { return stringName(uint32(i), rsparcStrings, true) } + +// Magic number for the elf trampoline, chosen wisely to be an immediate value. +const ARM_MAGIC_TRAMP_NUMBER = 0x5c000003 + +// ELF32 File header. +type Header32 struct { + Ident [EI_NIDENT]byte /* File identification. */ + Type uint16 /* File type. */ + Machine uint16 /* Machine architecture. */ + Version uint32 /* ELF format version. */ + Entry uint32 /* Entry point. */ + Phoff uint32 /* Program header file offset. */ + Shoff uint32 /* Section header file offset. */ + Flags uint32 /* Architecture-specific flags. */ + Ehsize uint16 /* Size of ELF header in bytes. */ + Phentsize uint16 /* Size of program header entry. */ + Phnum uint16 /* Number of program header entries. */ + Shentsize uint16 /* Size of section header entry. */ + Shnum uint16 /* Number of section header entries. */ + Shstrndx uint16 /* Section name strings section. */ +} + +// ELF32 Section header. +type Section32 struct { + Name uint32 /* Section name (index into the section header string table). */ + Type uint32 /* Section type. */ + Flags uint32 /* Section flags. */ + Addr uint32 /* Address in memory image. */ + Off uint32 /* Offset in file. */ + Size uint32 /* Size in bytes. */ + Link uint32 /* Index of a related section. */ + Info uint32 /* Depends on section type. */ + Addralign uint32 /* Alignment in bytes. */ + Entsize uint32 /* Size of each entry in section. */ +} + +// ELF32 Program header. +type Prog32 struct { + Type uint32 /* Entry type. */ + Off uint32 /* File offset of contents. */ + Vaddr uint32 /* Virtual address in memory image. */ + Paddr uint32 /* Physical address (not used). */ + Filesz uint32 /* Size of contents in file. */ + Memsz uint32 /* Size of contents in memory. */ + Flags uint32 /* Access permission flags. */ + Align uint32 /* Alignment in memory and file. */ +} + +// ELF32 Dynamic structure. The ".dynamic" section contains an array of them. +type Dyn32 struct { + Tag int32 /* Entry type. */ + Val uint32 /* Integer/Address value. */ +} + +/* + * Relocation entries. + */ + +// ELF32 Relocations that don't need an addend field. +type Rel32 struct { + Off uint32 /* Location to be relocated. */ + Info uint32 /* Relocation type and symbol index. */ +} + +// ELF32 Relocations that need an addend field. +type Rela32 struct { + Off uint32 /* Location to be relocated. */ + Info uint32 /* Relocation type and symbol index. */ + Addend int32 /* Addend. */ +} + +func R_SYM32(info uint32) uint32 { return uint32(info >> 8) } +func R_TYPE32(info uint32) uint32 { return uint32(info & 0xff) } +func R_INFO32(sym, typ uint32) uint32 { return sym<<8 | typ } + +// ELF32 Symbol. +type Sym32 struct { + Name uint32 + Value uint32 + Size uint32 + Info uint8 + Other uint8 + Shndx uint16 +} + +const Sym32Size = 16 + +func ST_BIND(info uint8) SymBind { return SymBind(info >> 4) } +func ST_TYPE(info uint8) SymType { return SymType(info & 0xF) } +func ST_INFO(bind SymBind, typ SymType) uint8 { + return uint8(bind)<<4 | uint8(typ)&0xf +} +func ST_VISIBILITY(other uint8) SymVis { return SymVis(other & 3) } + +/* + * ELF64 + */ + +// ELF64 file header. +type Header64 struct { + Ident [EI_NIDENT]byte /* File identification. */ + Type uint16 /* File type. */ + Machine uint16 /* Machine architecture. */ + Version uint32 /* ELF format version. */ + Entry uint64 /* Entry point. */ + Phoff uint64 /* Program header file offset. */ + Shoff uint64 /* Section header file offset. */ + Flags uint32 /* Architecture-specific flags. */ + Ehsize uint16 /* Size of ELF header in bytes. */ + Phentsize uint16 /* Size of program header entry. */ + Phnum uint16 /* Number of program header entries. */ + Shentsize uint16 /* Size of section header entry. */ + Shnum uint16 /* Number of section header entries. */ + Shstrndx uint16 /* Section name strings section. */ +} + +// ELF64 Section header. +type Section64 struct { + Name uint32 /* Section name (index into the section header string table). */ + Type uint32 /* Section type. */ + Flags uint64 /* Section flags. */ + Addr uint64 /* Address in memory image. */ + Off uint64 /* Offset in file. */ + Size uint64 /* Size in bytes. */ + Link uint32 /* Index of a related section. */ + Info uint32 /* Depends on section type. */ + Addralign uint64 /* Alignment in bytes. */ + Entsize uint64 /* Size of each entry in section. */ +} + +// ELF64 Program header. +type Prog64 struct { + Type uint32 /* Entry type. */ + Flags uint32 /* Access permission flags. */ + Off uint64 /* File offset of contents. */ + Vaddr uint64 /* Virtual address in memory image. */ + Paddr uint64 /* Physical address (not used). */ + Filesz uint64 /* Size of contents in file. */ + Memsz uint64 /* Size of contents in memory. */ + Align uint64 /* Alignment in memory and file. */ +} + +// ELF64 Dynamic structure. The ".dynamic" section contains an array of them. +type Dyn64 struct { + Tag int64 /* Entry type. */ + Val uint64 /* Integer/address value */ +} + +/* + * Relocation entries. + */ + +/* ELF64 relocations that don't need an addend field. */ +type Rel64 struct { + Off uint64 /* Location to be relocated. */ + Info uint64 /* Relocation type and symbol index. */ +} + +/* ELF64 relocations that need an addend field. */ +type Rela64 struct { + Off uint64 /* Location to be relocated. */ + Info uint64 /* Relocation type and symbol index. */ + Addend int64 /* Addend. */ +} + +func R_SYM64(info uint64) uint32 { return uint32(info >> 32) } +func R_TYPE64(info uint64) uint32 { return uint32(info) } +func R_INFO(sym, typ uint32) uint64 { return uint64(sym)<<32 | uint64(typ) } + +// ELF64 symbol table entries. +type Sym64 struct { + Name uint32 /* String table index of name. */ + Info uint8 /* Type and binding information. */ + Other uint8 /* Reserved (not used). */ + Shndx uint16 /* Section index of symbol. */ + Value uint64 /* Symbol value. */ + Size uint64 /* Size of associated object. */ +} + +const Sym64Size = 24 + +type intName struct { + i uint32 + s string +} + +func stringName(i uint32, names []intName, goSyntax bool) string { + for _, n := range names { + if n.i == i { + if goSyntax { + return "elf." + n.s + } + return n.s + } + } + + // second pass - look for smaller to add with. + // assume sorted already + for j := len(names) - 1; j >= 0; j-- { + n := names[j] + if n.i < i { + s := n.s + if goSyntax { + s = "elf." + s + } + return s + "+" + strconv.Uitoa64(uint64(i-n.i)) + } + } + + return strconv.Uitoa64(uint64(i)) +} + +func flagName(i uint32, names []intName, goSyntax bool) string { + s := "" + for _, n := range names { + if n.i&i == n.i { + if len(s) > 0 { + s += "+" + } + if goSyntax { + s += "elf." + } + s += n.s + i -= n.i + } + } + if len(s) == 0 { + return "0x" + strconv.Uitob64(uint64(i), 16) + } + if i != 0 { + s += "+0x" + strconv.Uitob64(uint64(i), 16) + } + return s +} diff --git a/src/pkg/debug/elf/elf_test.go b/src/pkg/debug/elf/elf_test.go new file mode 100644 index 000000000..67b961b5c --- /dev/null +++ b/src/pkg/debug/elf/elf_test.go @@ -0,0 +1,49 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package elf + +import ( + "fmt" + "testing" +) + +type nameTest struct { + val interface{} + str string +} + +var nameTests = []nameTest{ + {ELFOSABI_LINUX, "ELFOSABI_LINUX"}, + {ET_EXEC, "ET_EXEC"}, + {EM_860, "EM_860"}, + {SHN_LOPROC, "SHN_LOPROC"}, + {SHT_PROGBITS, "SHT_PROGBITS"}, + {SHF_MERGE + SHF_TLS, "SHF_MERGE+SHF_TLS"}, + {PT_LOAD, "PT_LOAD"}, + {PF_W + PF_R + 0x50, "PF_W+PF_R+0x50"}, + {DT_SYMBOLIC, "DT_SYMBOLIC"}, + {DF_BIND_NOW, "DF_BIND_NOW"}, + {NT_FPREGSET, "NT_FPREGSET"}, + {STB_GLOBAL, "STB_GLOBAL"}, + {STT_COMMON, "STT_COMMON"}, + {STV_HIDDEN, "STV_HIDDEN"}, + {R_X86_64_PC32, "R_X86_64_PC32"}, + {R_ALPHA_OP_PUSH, "R_ALPHA_OP_PUSH"}, + {R_ARM_THM_ABS5, "R_ARM_THM_ABS5"}, + {R_386_GOT32, "R_386_GOT32"}, + {R_PPC_GOT16_HI, "R_PPC_GOT16_HI"}, + {R_SPARC_GOT22, "R_SPARC_GOT22"}, + {ET_LOOS + 5, "ET_LOOS+5"}, + {ProgFlag(0x50), "0x50"}, +} + +func TestNames(t *testing.T) { + for i, tt := range nameTests { + s := fmt.Sprint(tt.val) + if s != tt.str { + t.Errorf("#%d: want %q have %q", i, s, tt.str) + } + } +} diff --git a/src/pkg/debug/elf/file.go b/src/pkg/debug/elf/file.go new file mode 100644 index 000000000..a0ddb1fc7 --- /dev/null +++ b/src/pkg/debug/elf/file.go @@ -0,0 +1,761 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Package elf implements access to ELF object files. +package elf + +import ( + "bytes" + "debug/dwarf" + "encoding/binary" + "fmt" + "io" + "os" +) + +// TODO: error reporting detail + +/* + * Internal ELF representation + */ + +// A FileHeader represents an ELF file header. +type FileHeader struct { + Class Class + Data Data + Version Version + OSABI OSABI + ABIVersion uint8 + ByteOrder binary.ByteOrder + Type Type + Machine Machine +} + +// A File represents an open ELF file. +type File struct { + FileHeader + Sections []*Section + Progs []*Prog + closer io.Closer + gnuNeed []verneed + gnuVersym []byte +} + +// A SectionHeader represents a single ELF section header. +type SectionHeader struct { + Name string + Type SectionType + Flags SectionFlag + Addr uint64 + Offset uint64 + Size uint64 + Link uint32 + Info uint32 + Addralign uint64 + Entsize uint64 +} + +// A Section represents a single section in an ELF file. +type Section struct { + SectionHeader + + // Embed ReaderAt for ReadAt method. + // Do not embed SectionReader directly + // to avoid having Read and Seek. + // If a client wants Read and Seek it must use + // Open() to avoid fighting over the seek offset + // with other clients. + io.ReaderAt + sr *io.SectionReader +} + +// Data reads and returns the contents of the ELF section. +func (s *Section) Data() ([]byte, os.Error) { + dat := make([]byte, s.sr.Size()) + n, err := s.sr.ReadAt(dat, 0) + return dat[0:n], err +} + +// stringTable reads and returns the string table given by the +// specified link value. +func (f *File) stringTable(link uint32) ([]byte, os.Error) { + if link <= 0 || link >= uint32(len(f.Sections)) { + return nil, os.NewError("section has invalid string table link") + } + return f.Sections[link].Data() +} + +// Open returns a new ReadSeeker reading the ELF section. +func (s *Section) Open() io.ReadSeeker { return io.NewSectionReader(s.sr, 0, 1<<63-1) } + +// A ProgHeader represents a single ELF program header. +type ProgHeader struct { + Type ProgType + Flags ProgFlag + Off uint64 + Vaddr uint64 + Paddr uint64 + Filesz uint64 + Memsz uint64 + Align uint64 +} + +// A Prog represents a single ELF program header in an ELF binary. +type Prog struct { + ProgHeader + + // Embed ReaderAt for ReadAt method. + // Do not embed SectionReader directly + // to avoid having Read and Seek. + // If a client wants Read and Seek it must use + // Open() to avoid fighting over the seek offset + // with other clients. + io.ReaderAt + sr *io.SectionReader +} + +// Open returns a new ReadSeeker reading the ELF program body. +func (p *Prog) Open() io.ReadSeeker { return io.NewSectionReader(p.sr, 0, 1<<63-1) } + +// A Symbol represents an entry in an ELF symbol table section. +type Symbol struct { + Name string + Info, Other byte + Section SectionIndex + Value, Size uint64 +} + +/* + * ELF reader + */ + +type FormatError struct { + off int64 + msg string + val interface{} +} + +func (e *FormatError) String() string { + msg := e.msg + if e.val != nil { + msg += fmt.Sprintf(" '%v' ", e.val) + } + msg += fmt.Sprintf("in record at byte %#x", e.off) + return msg +} + +// Open opens the named file using os.Open and prepares it for use as an ELF binary. +func Open(name string) (*File, os.Error) { + f, err := os.Open(name) + if err != nil { + return nil, err + } + ff, err := NewFile(f) + if err != nil { + f.Close() + return nil, err + } + ff.closer = f + return ff, nil +} + +// Close closes the File. +// If the File was created using NewFile directly instead of Open, +// Close has no effect. +func (f *File) Close() os.Error { + var err os.Error + if f.closer != nil { + err = f.closer.Close() + f.closer = nil + } + return err +} + +// SectionByType returns the first section in f with the +// given type, or nil if there is no such section. +func (f *File) SectionByType(typ SectionType) *Section { + for _, s := range f.Sections { + if s.Type == typ { + return s + } + } + return nil +} + +// NewFile creates a new File for accessing an ELF binary in an underlying reader. +// The ELF binary is expected to start at position 0 in the ReaderAt. +func NewFile(r io.ReaderAt) (*File, os.Error) { + sr := io.NewSectionReader(r, 0, 1<<63-1) + // Read and decode ELF identifier + var ident [16]uint8 + if _, err := r.ReadAt(ident[0:], 0); err != nil { + return nil, err + } + if ident[0] != '\x7f' || ident[1] != 'E' || ident[2] != 'L' || ident[3] != 'F' { + return nil, &FormatError{0, "bad magic number", ident[0:4]} + } + + f := new(File) + f.Class = Class(ident[EI_CLASS]) + switch f.Class { + case ELFCLASS32: + case ELFCLASS64: + // ok + default: + return nil, &FormatError{0, "unknown ELF class", f.Class} + } + + f.Data = Data(ident[EI_DATA]) + switch f.Data { + case ELFDATA2LSB: + f.ByteOrder = binary.LittleEndian + case ELFDATA2MSB: + f.ByteOrder = binary.BigEndian + default: + return nil, &FormatError{0, "unknown ELF data encoding", f.Data} + } + + f.Version = Version(ident[EI_VERSION]) + if f.Version != EV_CURRENT { + return nil, &FormatError{0, "unknown ELF version", f.Version} + } + + f.OSABI = OSABI(ident[EI_OSABI]) + f.ABIVersion = ident[EI_ABIVERSION] + + // Read ELF file header + var phoff int64 + var phentsize, phnum int + var shoff int64 + var shentsize, shnum, shstrndx int + shstrndx = -1 + switch f.Class { + case ELFCLASS32: + hdr := new(Header32) + sr.Seek(0, os.SEEK_SET) + if err := binary.Read(sr, f.ByteOrder, hdr); err != nil { + return nil, err + } + f.Type = Type(hdr.Type) + f.Machine = Machine(hdr.Machine) + if v := Version(hdr.Version); v != f.Version { + return nil, &FormatError{0, "mismatched ELF version", v} + } + phoff = int64(hdr.Phoff) + phentsize = int(hdr.Phentsize) + phnum = int(hdr.Phnum) + shoff = int64(hdr.Shoff) + shentsize = int(hdr.Shentsize) + shnum = int(hdr.Shnum) + shstrndx = int(hdr.Shstrndx) + case ELFCLASS64: + hdr := new(Header64) + sr.Seek(0, os.SEEK_SET) + if err := binary.Read(sr, f.ByteOrder, hdr); err != nil { + return nil, err + } + f.Type = Type(hdr.Type) + f.Machine = Machine(hdr.Machine) + if v := Version(hdr.Version); v != f.Version { + return nil, &FormatError{0, "mismatched ELF version", v} + } + phoff = int64(hdr.Phoff) + phentsize = int(hdr.Phentsize) + phnum = int(hdr.Phnum) + shoff = int64(hdr.Shoff) + shentsize = int(hdr.Shentsize) + shnum = int(hdr.Shnum) + shstrndx = int(hdr.Shstrndx) + } + if shstrndx < 0 || shstrndx >= shnum { + return nil, &FormatError{0, "invalid ELF shstrndx", shstrndx} + } + + // Read program headers + f.Progs = make([]*Prog, phnum) + for i := 0; i < phnum; i++ { + off := phoff + int64(i)*int64(phentsize) + sr.Seek(off, os.SEEK_SET) + p := new(Prog) + switch f.Class { + case ELFCLASS32: + ph := new(Prog32) + if err := binary.Read(sr, f.ByteOrder, ph); err != nil { + return nil, err + } + p.ProgHeader = ProgHeader{ + Type: ProgType(ph.Type), + Flags: ProgFlag(ph.Flags), + Off: uint64(ph.Off), + Vaddr: uint64(ph.Vaddr), + Paddr: uint64(ph.Paddr), + Filesz: uint64(ph.Filesz), + Memsz: uint64(ph.Memsz), + Align: uint64(ph.Align), + } + case ELFCLASS64: + ph := new(Prog64) + if err := binary.Read(sr, f.ByteOrder, ph); err != nil { + return nil, err + } + p.ProgHeader = ProgHeader{ + Type: ProgType(ph.Type), + Flags: ProgFlag(ph.Flags), + Off: uint64(ph.Off), + Vaddr: uint64(ph.Vaddr), + Paddr: uint64(ph.Paddr), + Filesz: uint64(ph.Filesz), + Memsz: uint64(ph.Memsz), + Align: uint64(ph.Align), + } + } + p.sr = io.NewSectionReader(r, int64(p.Off), int64(p.Filesz)) + p.ReaderAt = p.sr + f.Progs[i] = p + } + + // Read section headers + f.Sections = make([]*Section, shnum) + names := make([]uint32, shnum) + for i := 0; i < shnum; i++ { + off := shoff + int64(i)*int64(shentsize) + sr.Seek(off, os.SEEK_SET) + s := new(Section) + switch f.Class { + case ELFCLASS32: + sh := new(Section32) + if err := binary.Read(sr, f.ByteOrder, sh); err != nil { + return nil, err + } + names[i] = sh.Name + s.SectionHeader = SectionHeader{ + Type: SectionType(sh.Type), + Flags: SectionFlag(sh.Flags), + Addr: uint64(sh.Addr), + Offset: uint64(sh.Off), + Size: uint64(sh.Size), + Link: uint32(sh.Link), + Info: uint32(sh.Info), + Addralign: uint64(sh.Addralign), + Entsize: uint64(sh.Entsize), + } + case ELFCLASS64: + sh := new(Section64) + if err := binary.Read(sr, f.ByteOrder, sh); err != nil { + return nil, err + } + names[i] = sh.Name + s.SectionHeader = SectionHeader{ + Type: SectionType(sh.Type), + Flags: SectionFlag(sh.Flags), + Offset: uint64(sh.Off), + Size: uint64(sh.Size), + Addr: uint64(sh.Addr), + Link: uint32(sh.Link), + Info: uint32(sh.Info), + Addralign: uint64(sh.Addralign), + Entsize: uint64(sh.Entsize), + } + } + s.sr = io.NewSectionReader(r, int64(s.Offset), int64(s.Size)) + s.ReaderAt = s.sr + f.Sections[i] = s + } + + // Load section header string table. + shstrtab, err := f.Sections[shstrndx].Data() + if err != nil { + return nil, err + } + for i, s := range f.Sections { + var ok bool + s.Name, ok = getString(shstrtab, int(names[i])) + if !ok { + return nil, &FormatError{shoff + int64(i*shentsize), "bad section name index", names[i]} + } + } + + return f, nil +} + +// getSymbols returns a slice of Symbols from parsing the symbol table +// with the given type, along with the associated string table. +func (f *File) getSymbols(typ SectionType) ([]Symbol, []byte, os.Error) { + switch f.Class { + case ELFCLASS64: + return f.getSymbols64(typ) + + case ELFCLASS32: + return f.getSymbols32(typ) + } + + return nil, nil, os.NewError("not implemented") +} + +func (f *File) getSymbols32(typ SectionType) ([]Symbol, []byte, os.Error) { + symtabSection := f.SectionByType(typ) + if symtabSection == nil { + return nil, nil, os.NewError("no symbol section") + } + + data, err := symtabSection.Data() + if err != nil { + return nil, nil, os.NewError("cannot load symbol section") + } + symtab := bytes.NewBuffer(data) + if symtab.Len()%Sym32Size != 0 { + return nil, nil, os.NewError("length of symbol section is not a multiple of SymSize") + } + + strdata, err := f.stringTable(symtabSection.Link) + if err != nil { + return nil, nil, os.NewError("cannot load string table section") + } + + // The first entry is all zeros. + var skip [Sym32Size]byte + symtab.Read(skip[0:]) + + symbols := make([]Symbol, symtab.Len()/Sym32Size) + + i := 0 + var sym Sym32 + for symtab.Len() > 0 { + binary.Read(symtab, f.ByteOrder, &sym) + str, _ := getString(strdata, int(sym.Name)) + symbols[i].Name = str + symbols[i].Info = sym.Info + symbols[i].Other = sym.Other + symbols[i].Section = SectionIndex(sym.Shndx) + symbols[i].Value = uint64(sym.Value) + symbols[i].Size = uint64(sym.Size) + i++ + } + + return symbols, strdata, nil +} + +func (f *File) getSymbols64(typ SectionType) ([]Symbol, []byte, os.Error) { + symtabSection := f.SectionByType(typ) + if symtabSection == nil { + return nil, nil, os.NewError("no symbol section") + } + + data, err := symtabSection.Data() + if err != nil { + return nil, nil, os.NewError("cannot load symbol section") + } + symtab := bytes.NewBuffer(data) + if symtab.Len()%Sym64Size != 0 { + return nil, nil, os.NewError("length of symbol section is not a multiple of Sym64Size") + } + + strdata, err := f.stringTable(symtabSection.Link) + if err != nil { + return nil, nil, os.NewError("cannot load string table section") + } + + // The first entry is all zeros. + var skip [Sym64Size]byte + symtab.Read(skip[0:]) + + symbols := make([]Symbol, symtab.Len()/Sym64Size) + + i := 0 + var sym Sym64 + for symtab.Len() > 0 { + binary.Read(symtab, f.ByteOrder, &sym) + str, _ := getString(strdata, int(sym.Name)) + symbols[i].Name = str + symbols[i].Info = sym.Info + symbols[i].Other = sym.Other + symbols[i].Section = SectionIndex(sym.Shndx) + symbols[i].Value = sym.Value + symbols[i].Size = sym.Size + i++ + } + + return symbols, strdata, nil +} + +// getString extracts a string from an ELF string table. +func getString(section []byte, start int) (string, bool) { + if start < 0 || start >= len(section) { + return "", false + } + + for end := start; end < len(section); end++ { + if section[end] == 0 { + return string(section[start:end]), true + } + } + return "", false +} + +// Section returns a section with the given name, or nil if no such +// section exists. +func (f *File) Section(name string) *Section { + for _, s := range f.Sections { + if s.Name == name { + return s + } + } + return nil +} + +// applyRelocations applies relocations to dst. rels is a relocations section +// in RELA format. +func (f *File) applyRelocations(dst []byte, rels []byte) os.Error { + if f.Class == ELFCLASS64 && f.Machine == EM_X86_64 { + return f.applyRelocationsAMD64(dst, rels) + } + + return os.NewError("not implemented") +} + +func (f *File) applyRelocationsAMD64(dst []byte, rels []byte) os.Error { + if len(rels)%Sym64Size != 0 { + return os.NewError("length of relocation section is not a multiple of Sym64Size") + } + + symbols, _, err := f.getSymbols(SHT_SYMTAB) + if err != nil { + return err + } + + b := bytes.NewBuffer(rels) + var rela Rela64 + + for b.Len() > 0 { + binary.Read(b, f.ByteOrder, &rela) + symNo := rela.Info >> 32 + t := R_X86_64(rela.Info & 0xffff) + + if symNo >= uint64(len(symbols)) { + continue + } + sym := &symbols[symNo] + if SymType(sym.Info&0xf) != STT_SECTION { + // We don't handle non-section relocations for now. + continue + } + + switch t { + case R_X86_64_64: + if rela.Off+8 >= uint64(len(dst)) || rela.Addend < 0 { + continue + } + f.ByteOrder.PutUint64(dst[rela.Off:rela.Off+8], uint64(rela.Addend)) + case R_X86_64_32: + if rela.Off+4 >= uint64(len(dst)) || rela.Addend < 0 { + continue + } + f.ByteOrder.PutUint32(dst[rela.Off:rela.Off+4], uint32(rela.Addend)) + } + } + + return nil +} + +func (f *File) DWARF() (*dwarf.Data, os.Error) { + // There are many other DWARF sections, but these + // are the required ones, and the debug/dwarf package + // does not use the others, so don't bother loading them. + var names = [...]string{"abbrev", "info", "str"} + var dat [len(names)][]byte + for i, name := range names { + name = ".debug_" + name + s := f.Section(name) + if s == nil { + continue + } + b, err := s.Data() + if err != nil && uint64(len(b)) < s.Size { + return nil, err + } + dat[i] = b + } + + // If there's a relocation table for .debug_info, we have to process it + // now otherwise the data in .debug_info is invalid for x86-64 objects. + rela := f.Section(".rela.debug_info") + if rela != nil && rela.Type == SHT_RELA && f.Machine == EM_X86_64 { + data, err := rela.Data() + if err != nil { + return nil, err + } + err = f.applyRelocations(dat[1], data) + if err != nil { + return nil, err + } + } + + abbrev, info, str := dat[0], dat[1], dat[2] + return dwarf.New(abbrev, nil, nil, info, nil, nil, nil, str) +} + +// Symbols returns the symbol table for f. +func (f *File) Symbols() ([]Symbol, os.Error) { + sym, _, err := f.getSymbols(SHT_SYMTAB) + return sym, err +} + +type ImportedSymbol struct { + Name string + Version string + Library string +} + +// ImportedSymbols returns the names of all symbols +// referred to by the binary f that are expected to be +// satisfied by other libraries at dynamic load time. +// It does not return weak symbols. +func (f *File) ImportedSymbols() ([]ImportedSymbol, os.Error) { + sym, str, err := f.getSymbols(SHT_DYNSYM) + if err != nil { + return nil, err + } + f.gnuVersionInit(str) + var all []ImportedSymbol + for i, s := range sym { + if ST_BIND(s.Info) == STB_GLOBAL && s.Section == SHN_UNDEF { + all = append(all, ImportedSymbol{Name: s.Name}) + f.gnuVersion(i, &all[len(all)-1]) + } + } + return all, nil +} + +type verneed struct { + File string + Name string +} + +// gnuVersionInit parses the GNU version tables +// for use by calls to gnuVersion. +func (f *File) gnuVersionInit(str []byte) { + // Accumulate verneed information. + vn := f.SectionByType(SHT_GNU_VERNEED) + if vn == nil { + return + } + d, _ := vn.Data() + + var need []verneed + i := 0 + for { + if i+16 > len(d) { + break + } + vers := f.ByteOrder.Uint16(d[i : i+2]) + if vers != 1 { + break + } + cnt := f.ByteOrder.Uint16(d[i+2 : i+4]) + fileoff := f.ByteOrder.Uint32(d[i+4 : i+8]) + aux := f.ByteOrder.Uint32(d[i+8 : i+12]) + next := f.ByteOrder.Uint32(d[i+12 : i+16]) + file, _ := getString(str, int(fileoff)) + + var name string + j := i + int(aux) + for c := 0; c < int(cnt); c++ { + if j+16 > len(d) { + break + } + // hash := f.ByteOrder.Uint32(d[j:j+4]) + // flags := f.ByteOrder.Uint16(d[j+4:j+6]) + other := f.ByteOrder.Uint16(d[j+6 : j+8]) + nameoff := f.ByteOrder.Uint32(d[j+8 : j+12]) + next := f.ByteOrder.Uint32(d[j+12 : j+16]) + name, _ = getString(str, int(nameoff)) + ndx := int(other) + if ndx >= len(need) { + a := make([]verneed, 2*(ndx+1)) + copy(a, need) + need = a + } + + need[ndx] = verneed{file, name} + if next == 0 { + break + } + j += int(next) + } + + if next == 0 { + break + } + i += int(next) + } + + // Versym parallels symbol table, indexing into verneed. + vs := f.SectionByType(SHT_GNU_VERSYM) + if vs == nil { + return + } + d, _ = vs.Data() + + f.gnuNeed = need + f.gnuVersym = d +} + +// gnuVersion adds Library and Version information to sym, +// which came from offset i of the symbol table. +func (f *File) gnuVersion(i int, sym *ImportedSymbol) { + // Each entry is two bytes; skip undef entry at beginning. + i = (i + 1) * 2 + if i >= len(f.gnuVersym) { + return + } + j := int(f.ByteOrder.Uint16(f.gnuVersym[i:])) + if j < 2 || j >= len(f.gnuNeed) { + return + } + n := &f.gnuNeed[j] + sym.Library = n.File + sym.Version = n.Name +} + +// ImportedLibraries returns the names of all libraries +// referred to by the binary f that are expected to be +// linked with the binary at dynamic link time. +func (f *File) ImportedLibraries() ([]string, os.Error) { + ds := f.SectionByType(SHT_DYNAMIC) + if ds == nil { + // not dynamic, so no libraries + return nil, nil + } + d, err := ds.Data() + if err != nil { + return nil, err + } + str, err := f.stringTable(ds.Link) + if err != nil { + return nil, err + } + var all []string + for len(d) > 0 { + var tag DynTag + var value uint64 + switch f.Class { + case ELFCLASS32: + tag = DynTag(f.ByteOrder.Uint32(d[0:4])) + value = uint64(f.ByteOrder.Uint32(d[4:8])) + d = d[8:] + case ELFCLASS64: + tag = DynTag(f.ByteOrder.Uint64(d[0:8])) + value = f.ByteOrder.Uint64(d[8:16]) + d = d[16:] + } + if tag == DT_NEEDED { + s, ok := getString(str, int(value)) + if ok { + all = append(all, s) + } + } + } + + return all, nil +} diff --git a/src/pkg/debug/elf/file_test.go b/src/pkg/debug/elf/file_test.go new file mode 100644 index 000000000..451d3d514 --- /dev/null +++ b/src/pkg/debug/elf/file_test.go @@ -0,0 +1,241 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package elf + +import ( + "debug/dwarf" + "encoding/binary" + "net" + "os" + "reflect" + "runtime" + "testing" +) + +type fileTest struct { + file string + hdr FileHeader + sections []SectionHeader + progs []ProgHeader +} + +var fileTests = []fileTest{ + { + "testdata/gcc-386-freebsd-exec", + FileHeader{ELFCLASS32, ELFDATA2LSB, EV_CURRENT, ELFOSABI_FREEBSD, 0, binary.LittleEndian, ET_EXEC, EM_386}, + []SectionHeader{ + {"", SHT_NULL, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0}, + {".interp", SHT_PROGBITS, SHF_ALLOC, 0x80480d4, 0xd4, 0x15, 0x0, 0x0, 0x1, 0x0}, + {".hash", SHT_HASH, SHF_ALLOC, 0x80480ec, 0xec, 0x90, 0x3, 0x0, 0x4, 0x4}, + {".dynsym", SHT_DYNSYM, SHF_ALLOC, 0x804817c, 0x17c, 0x110, 0x4, 0x1, 0x4, 0x10}, + {".dynstr", SHT_STRTAB, SHF_ALLOC, 0x804828c, 0x28c, 0xbb, 0x0, 0x0, 0x1, 0x0}, + {".rel.plt", SHT_REL, SHF_ALLOC, 0x8048348, 0x348, 0x20, 0x3, 0x7, 0x4, 0x8}, + {".init", SHT_PROGBITS, SHF_ALLOC + SHF_EXECINSTR, 0x8048368, 0x368, 0x11, 0x0, 0x0, 0x4, 0x0}, + {".plt", SHT_PROGBITS, SHF_ALLOC + SHF_EXECINSTR, 0x804837c, 0x37c, 0x50, 0x0, 0x0, 0x4, 0x4}, + {".text", SHT_PROGBITS, SHF_ALLOC + SHF_EXECINSTR, 0x80483cc, 0x3cc, 0x180, 0x0, 0x0, 0x4, 0x0}, + {".fini", SHT_PROGBITS, SHF_ALLOC + SHF_EXECINSTR, 0x804854c, 0x54c, 0xc, 0x0, 0x0, 0x4, 0x0}, + {".rodata", SHT_PROGBITS, SHF_ALLOC, 0x8048558, 0x558, 0xa3, 0x0, 0x0, 0x1, 0x0}, + {".data", SHT_PROGBITS, SHF_WRITE + SHF_ALLOC, 0x80495fc, 0x5fc, 0xc, 0x0, 0x0, 0x4, 0x0}, + {".eh_frame", SHT_PROGBITS, SHF_ALLOC, 0x8049608, 0x608, 0x4, 0x0, 0x0, 0x4, 0x0}, + {".dynamic", SHT_DYNAMIC, SHF_WRITE + SHF_ALLOC, 0x804960c, 0x60c, 0x98, 0x4, 0x0, 0x4, 0x8}, + {".ctors", SHT_PROGBITS, SHF_WRITE + SHF_ALLOC, 0x80496a4, 0x6a4, 0x8, 0x0, 0x0, 0x4, 0x0}, + {".dtors", SHT_PROGBITS, SHF_WRITE + SHF_ALLOC, 0x80496ac, 0x6ac, 0x8, 0x0, 0x0, 0x4, 0x0}, + {".jcr", SHT_PROGBITS, SHF_WRITE + SHF_ALLOC, 0x80496b4, 0x6b4, 0x4, 0x0, 0x0, 0x4, 0x0}, + {".got", SHT_PROGBITS, SHF_WRITE + SHF_ALLOC, 0x80496b8, 0x6b8, 0x1c, 0x0, 0x0, 0x4, 0x4}, + {".bss", SHT_NOBITS, SHF_WRITE + SHF_ALLOC, 0x80496d4, 0x6d4, 0x20, 0x0, 0x0, 0x4, 0x0}, + {".comment", SHT_PROGBITS, 0x0, 0x0, 0x6d4, 0x12d, 0x0, 0x0, 0x1, 0x0}, + {".debug_aranges", SHT_PROGBITS, 0x0, 0x0, 0x801, 0x20, 0x0, 0x0, 0x1, 0x0}, + {".debug_pubnames", SHT_PROGBITS, 0x0, 0x0, 0x821, 0x1b, 0x0, 0x0, 0x1, 0x0}, + {".debug_info", SHT_PROGBITS, 0x0, 0x0, 0x83c, 0x11d, 0x0, 0x0, 0x1, 0x0}, + {".debug_abbrev", SHT_PROGBITS, 0x0, 0x0, 0x959, 0x41, 0x0, 0x0, 0x1, 0x0}, + {".debug_line", SHT_PROGBITS, 0x0, 0x0, 0x99a, 0x35, 0x0, 0x0, 0x1, 0x0}, + {".debug_frame", SHT_PROGBITS, 0x0, 0x0, 0x9d0, 0x30, 0x0, 0x0, 0x4, 0x0}, + {".debug_str", SHT_PROGBITS, 0x0, 0x0, 0xa00, 0xd, 0x0, 0x0, 0x1, 0x0}, + {".shstrtab", SHT_STRTAB, 0x0, 0x0, 0xa0d, 0xf8, 0x0, 0x0, 0x1, 0x0}, + {".symtab", SHT_SYMTAB, 0x0, 0x0, 0xfb8, 0x4b0, 0x1d, 0x38, 0x4, 0x10}, + {".strtab", SHT_STRTAB, 0x0, 0x0, 0x1468, 0x206, 0x0, 0x0, 0x1, 0x0}, + }, + []ProgHeader{ + {PT_PHDR, PF_R + PF_X, 0x34, 0x8048034, 0x8048034, 0xa0, 0xa0, 0x4}, + {PT_INTERP, PF_R, 0xd4, 0x80480d4, 0x80480d4, 0x15, 0x15, 0x1}, + {PT_LOAD, PF_R + PF_X, 0x0, 0x8048000, 0x8048000, 0x5fb, 0x5fb, 0x1000}, + {PT_LOAD, PF_R + PF_W, 0x5fc, 0x80495fc, 0x80495fc, 0xd8, 0xf8, 0x1000}, + {PT_DYNAMIC, PF_R + PF_W, 0x60c, 0x804960c, 0x804960c, 0x98, 0x98, 0x4}, + }, + }, + { + "testdata/gcc-amd64-linux-exec", + FileHeader{ELFCLASS64, ELFDATA2LSB, EV_CURRENT, ELFOSABI_NONE, 0, binary.LittleEndian, ET_EXEC, EM_X86_64}, + []SectionHeader{ + {"", SHT_NULL, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0}, + {".interp", SHT_PROGBITS, SHF_ALLOC, 0x400200, 0x200, 0x1c, 0x0, 0x0, 0x1, 0x0}, + {".note.ABI-tag", SHT_NOTE, SHF_ALLOC, 0x40021c, 0x21c, 0x20, 0x0, 0x0, 0x4, 0x0}, + {".hash", SHT_HASH, SHF_ALLOC, 0x400240, 0x240, 0x24, 0x5, 0x0, 0x8, 0x4}, + {".gnu.hash", SHT_LOOS + 268435446, SHF_ALLOC, 0x400268, 0x268, 0x1c, 0x5, 0x0, 0x8, 0x0}, + {".dynsym", SHT_DYNSYM, SHF_ALLOC, 0x400288, 0x288, 0x60, 0x6, 0x1, 0x8, 0x18}, + {".dynstr", SHT_STRTAB, SHF_ALLOC, 0x4002e8, 0x2e8, 0x3d, 0x0, 0x0, 0x1, 0x0}, + {".gnu.version", SHT_HIOS, SHF_ALLOC, 0x400326, 0x326, 0x8, 0x5, 0x0, 0x2, 0x2}, + {".gnu.version_r", SHT_LOOS + 268435454, SHF_ALLOC, 0x400330, 0x330, 0x20, 0x6, 0x1, 0x8, 0x0}, + {".rela.dyn", SHT_RELA, SHF_ALLOC, 0x400350, 0x350, 0x18, 0x5, 0x0, 0x8, 0x18}, + {".rela.plt", SHT_RELA, SHF_ALLOC, 0x400368, 0x368, 0x30, 0x5, 0xc, 0x8, 0x18}, + {".init", SHT_PROGBITS, SHF_ALLOC + SHF_EXECINSTR, 0x400398, 0x398, 0x18, 0x0, 0x0, 0x4, 0x0}, + {".plt", SHT_PROGBITS, SHF_ALLOC + SHF_EXECINSTR, 0x4003b0, 0x3b0, 0x30, 0x0, 0x0, 0x4, 0x10}, + {".text", SHT_PROGBITS, SHF_ALLOC + SHF_EXECINSTR, 0x4003e0, 0x3e0, 0x1b4, 0x0, 0x0, 0x10, 0x0}, + {".fini", SHT_PROGBITS, SHF_ALLOC + SHF_EXECINSTR, 0x400594, 0x594, 0xe, 0x0, 0x0, 0x4, 0x0}, + {".rodata", SHT_PROGBITS, SHF_ALLOC, 0x4005a4, 0x5a4, 0x11, 0x0, 0x0, 0x4, 0x0}, + {".eh_frame_hdr", SHT_PROGBITS, SHF_ALLOC, 0x4005b8, 0x5b8, 0x24, 0x0, 0x0, 0x4, 0x0}, + {".eh_frame", SHT_PROGBITS, SHF_ALLOC, 0x4005e0, 0x5e0, 0xa4, 0x0, 0x0, 0x8, 0x0}, + {".ctors", SHT_PROGBITS, SHF_WRITE + SHF_ALLOC, 0x600688, 0x688, 0x10, 0x0, 0x0, 0x8, 0x0}, + {".dtors", SHT_PROGBITS, SHF_WRITE + SHF_ALLOC, 0x600698, 0x698, 0x10, 0x0, 0x0, 0x8, 0x0}, + {".jcr", SHT_PROGBITS, SHF_WRITE + SHF_ALLOC, 0x6006a8, 0x6a8, 0x8, 0x0, 0x0, 0x8, 0x0}, + {".dynamic", SHT_DYNAMIC, SHF_WRITE + SHF_ALLOC, 0x6006b0, 0x6b0, 0x1a0, 0x6, 0x0, 0x8, 0x10}, + {".got", SHT_PROGBITS, SHF_WRITE + SHF_ALLOC, 0x600850, 0x850, 0x8, 0x0, 0x0, 0x8, 0x8}, + {".got.plt", SHT_PROGBITS, SHF_WRITE + SHF_ALLOC, 0x600858, 0x858, 0x28, 0x0, 0x0, 0x8, 0x8}, + {".data", SHT_PROGBITS, SHF_WRITE + SHF_ALLOC, 0x600880, 0x880, 0x18, 0x0, 0x0, 0x8, 0x0}, + {".bss", SHT_NOBITS, SHF_WRITE + SHF_ALLOC, 0x600898, 0x898, 0x8, 0x0, 0x0, 0x4, 0x0}, + {".comment", SHT_PROGBITS, 0x0, 0x0, 0x898, 0x126, 0x0, 0x0, 0x1, 0x0}, + {".debug_aranges", SHT_PROGBITS, 0x0, 0x0, 0x9c0, 0x90, 0x0, 0x0, 0x10, 0x0}, + {".debug_pubnames", SHT_PROGBITS, 0x0, 0x0, 0xa50, 0x25, 0x0, 0x0, 0x1, 0x0}, + {".debug_info", SHT_PROGBITS, 0x0, 0x0, 0xa75, 0x1a7, 0x0, 0x0, 0x1, 0x0}, + {".debug_abbrev", SHT_PROGBITS, 0x0, 0x0, 0xc1c, 0x6f, 0x0, 0x0, 0x1, 0x0}, + {".debug_line", SHT_PROGBITS, 0x0, 0x0, 0xc8b, 0x13f, 0x0, 0x0, 0x1, 0x0}, + {".debug_str", SHT_PROGBITS, SHF_MERGE + SHF_STRINGS, 0x0, 0xdca, 0xb1, 0x0, 0x0, 0x1, 0x1}, + {".debug_ranges", SHT_PROGBITS, 0x0, 0x0, 0xe80, 0x90, 0x0, 0x0, 0x10, 0x0}, + {".shstrtab", SHT_STRTAB, 0x0, 0x0, 0xf10, 0x149, 0x0, 0x0, 0x1, 0x0}, + {".symtab", SHT_SYMTAB, 0x0, 0x0, 0x19a0, 0x6f0, 0x24, 0x39, 0x8, 0x18}, + {".strtab", SHT_STRTAB, 0x0, 0x0, 0x2090, 0x1fc, 0x0, 0x0, 0x1, 0x0}, + }, + []ProgHeader{ + {PT_PHDR, PF_R + PF_X, 0x40, 0x400040, 0x400040, 0x1c0, 0x1c0, 0x8}, + {PT_INTERP, PF_R, 0x200, 0x400200, 0x400200, 0x1c, 0x1c, 1}, + {PT_LOAD, PF_R + PF_X, 0x0, 0x400000, 0x400000, 0x684, 0x684, 0x200000}, + {PT_LOAD, PF_R + PF_W, 0x688, 0x600688, 0x600688, 0x210, 0x218, 0x200000}, + {PT_DYNAMIC, PF_R + PF_W, 0x6b0, 0x6006b0, 0x6006b0, 0x1a0, 0x1a0, 0x8}, + {PT_NOTE, PF_R, 0x21c, 0x40021c, 0x40021c, 0x20, 0x20, 0x4}, + {PT_LOOS + 0x474E550, PF_R, 0x5b8, 0x4005b8, 0x4005b8, 0x24, 0x24, 0x4}, + {PT_LOOS + 0x474E551, PF_R + PF_W, 0x0, 0x0, 0x0, 0x0, 0x0, 0x8}, + }, + }, +} + +func TestOpen(t *testing.T) { + for i := range fileTests { + tt := &fileTests[i] + + f, err := Open(tt.file) + if err != nil { + t.Error(err) + continue + } + if !reflect.DeepEqual(f.FileHeader, tt.hdr) { + t.Errorf("open %s:\n\thave %#v\n\twant %#v\n", tt.file, f.FileHeader, tt.hdr) + continue + } + for i, s := range f.Sections { + if i >= len(tt.sections) { + break + } + sh := &tt.sections[i] + if !reflect.DeepEqual(&s.SectionHeader, sh) { + t.Errorf("open %s, section %d:\n\thave %#v\n\twant %#v\n", tt.file, i, &s.SectionHeader, sh) + } + } + for i, p := range f.Progs { + if i >= len(tt.progs) { + break + } + ph := &tt.progs[i] + if !reflect.DeepEqual(&p.ProgHeader, ph) { + t.Errorf("open %s, program %d:\n\thave %#v\n\twant %#v\n", tt.file, i, &p.ProgHeader, ph) + } + } + tn := len(tt.sections) + fn := len(f.Sections) + if tn != fn { + t.Errorf("open %s: len(Sections) = %d, want %d", tt.file, fn, tn) + } + tn = len(tt.progs) + fn = len(f.Progs) + if tn != fn { + t.Errorf("open %s: len(Progs) = %d, want %d", tt.file, fn, tn) + } + } +} + +type relocationTest struct { + file string + firstEntry *dwarf.Entry +} + +var relocationTests = []relocationTest{ + { + "testdata/go-relocation-test-gcc441-x86-64.obj", + &dwarf.Entry{Offset: 0xb, Tag: dwarf.TagCompileUnit, Children: true, Field: []dwarf.Field{{Attr: dwarf.AttrProducer, Val: "GNU C 4.4.1"}, {Attr: dwarf.AttrLanguage, Val: int64(1)}, {Attr: dwarf.AttrName, Val: "go-relocation-test.c"}, {Attr: dwarf.AttrCompDir, Val: "/tmp"}, {Attr: dwarf.AttrLowpc, Val: uint64(0x0)}, {Attr: dwarf.AttrHighpc, Val: uint64(0x6)}, {Attr: dwarf.AttrStmtList, Val: int64(0)}}}, + }, + { + "testdata/go-relocation-test-gcc441-x86.obj", + &dwarf.Entry{Offset: 0xb, Tag: dwarf.TagCompileUnit, Children: true, Field: []dwarf.Field{{Attr: dwarf.AttrProducer, Val: "GNU C 4.4.1"}, {Attr: dwarf.AttrLanguage, Val: int64(1)}, {Attr: dwarf.AttrName, Val: "t.c"}, {Attr: dwarf.AttrCompDir, Val: "/tmp"}, {Attr: dwarf.AttrLowpc, Val: uint64(0x0)}, {Attr: dwarf.AttrHighpc, Val: uint64(0x5)}, {Attr: dwarf.AttrStmtList, Val: int64(0)}}}, + }, + { + "testdata/go-relocation-test-gcc424-x86-64.obj", + &dwarf.Entry{Offset: 0xb, Tag: dwarf.TagCompileUnit, Children: true, Field: []dwarf.Field{{Attr: dwarf.AttrProducer, Val: "GNU C 4.2.4 (Ubuntu 4.2.4-1ubuntu4)"}, {Attr: dwarf.AttrLanguage, Val: int64(1)}, {Attr: dwarf.AttrName, Val: "go-relocation-test-gcc424.c"}, {Attr: dwarf.AttrCompDir, Val: "/tmp"}, {Attr: dwarf.AttrLowpc, Val: uint64(0x0)}, {Attr: dwarf.AttrHighpc, Val: uint64(0x6)}, {Attr: dwarf.AttrStmtList, Val: int64(0)}}}, + }, +} + +func TestDWARFRelocations(t *testing.T) { + for i, test := range relocationTests { + f, err := Open(test.file) + if err != nil { + t.Error(err) + continue + } + dwarf, err := f.DWARF() + if err != nil { + t.Error(err) + continue + } + reader := dwarf.Reader() + // Checking only the first entry is sufficient since it has + // many different strings. If the relocation had failed, all + // the string offsets would be zero and all the strings would + // end up being the same. + firstEntry, err := reader.Next() + if err != nil { + t.Error(err) + continue + } + + if !reflect.DeepEqual(test.firstEntry, firstEntry) { + t.Errorf("#%d: mismatch: got:%#v want:%#v", i, firstEntry, test.firstEntry) + continue + } + } +} + +func TestNoSectionOverlaps(t *testing.T) { + // Ensure 6l outputs sections without overlaps. + if runtime.GOOS != "linux" && runtime.GOOS != "freebsd" { + return // not ELF + } + _ = net.ResolveIPAddr // force dynamic linkage + f, err := Open(os.Args[0]) + if err != nil { + t.Error(err) + return + } + for i, si := range f.Sections { + sih := si.SectionHeader + for j, sj := range f.Sections { + sjh := sj.SectionHeader + if i == j || sjh.Type == SHT_NOBITS || sih.Offset == sjh.Offset && sih.Size == 0 { + continue + } + if sih.Offset >= sjh.Offset && sih.Offset < sjh.Offset+sjh.Size { + t.Errorf("ld produced ELF with section %s within %s: 0x%x <= 0x%x..0x%x < 0x%x", + sih.Name, sjh.Name, sjh.Offset, sih.Offset, sih.Offset+sih.Size, sjh.Offset+sjh.Size) + } + } + } +} diff --git a/src/pkg/debug/elf/testdata/gcc-386-freebsd-exec b/src/pkg/debug/elf/testdata/gcc-386-freebsd-exec new file mode 100755 index 000000000..7af9c58ca Binary files /dev/null and b/src/pkg/debug/elf/testdata/gcc-386-freebsd-exec differ diff --git a/src/pkg/debug/elf/testdata/gcc-amd64-linux-exec b/src/pkg/debug/elf/testdata/gcc-amd64-linux-exec new file mode 100755 index 000000000..c6cb1de28 Binary files /dev/null and b/src/pkg/debug/elf/testdata/gcc-amd64-linux-exec differ diff --git a/src/pkg/debug/elf/testdata/go-relocation-test-gcc424-x86-64.obj b/src/pkg/debug/elf/testdata/go-relocation-test-gcc424-x86-64.obj new file mode 100644 index 000000000..a7c6d6e56 Binary files /dev/null and b/src/pkg/debug/elf/testdata/go-relocation-test-gcc424-x86-64.obj differ diff --git a/src/pkg/debug/elf/testdata/go-relocation-test-gcc441-x86-64.obj b/src/pkg/debug/elf/testdata/go-relocation-test-gcc441-x86-64.obj new file mode 100644 index 000000000..2d37ab6e6 Binary files /dev/null and b/src/pkg/debug/elf/testdata/go-relocation-test-gcc441-x86-64.obj differ diff --git a/src/pkg/debug/elf/testdata/go-relocation-test-gcc441-x86.obj b/src/pkg/debug/elf/testdata/go-relocation-test-gcc441-x86.obj new file mode 100644 index 000000000..0d59fe303 Binary files /dev/null and b/src/pkg/debug/elf/testdata/go-relocation-test-gcc441-x86.obj differ diff --git a/src/pkg/debug/gosym/Makefile b/src/pkg/debug/gosym/Makefile new file mode 100644 index 000000000..4f420e729 --- /dev/null +++ b/src/pkg/debug/gosym/Makefile @@ -0,0 +1,19 @@ +# Copyright 2009 The Go Authors. All rights reserved. +# Use of this source code is governed by a BSD-style +# license that can be found in the LICENSE file. + +include ../../../Make.inc + +TARG=debug/gosym +GOFILES=\ + pclntab.go\ + symtab.go\ + +include ../../../Make.pkg + +test: make-pclinetest + +testshort: make-pclinetest + +make-pclinetest: + @if [ "`uname`-`uname -m`" = Linux-x86_64 -a $(GOARCH) = amd64 ]; then mkdir -p _test && $(AS) pclinetest.s && $(LD) -E main -o _test/pclinetest pclinetest.$O; fi diff --git a/src/pkg/debug/gosym/pclinetest.h b/src/pkg/debug/gosym/pclinetest.h new file mode 100644 index 000000000..a6c40e76c --- /dev/null +++ b/src/pkg/debug/gosym/pclinetest.h @@ -0,0 +1,7 @@ +// Empty include file to generate z symbols + + + + + +// EOF diff --git a/src/pkg/debug/gosym/pclinetest.s b/src/pkg/debug/gosym/pclinetest.s new file mode 100644 index 000000000..6305435b0 --- /dev/null +++ b/src/pkg/debug/gosym/pclinetest.s @@ -0,0 +1,58 @@ +TEXT linefrompc(SB),7,$0 // Each byte stores its line delta +BYTE $2; +BYTE $1; +BYTE $1; BYTE $0; +BYTE $1; BYTE $0; BYTE $0; +BYTE $1; BYTE $0; BYTE $0; BYTE $0; BYTE $0; +BYTE $1; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; +BYTE $1; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; +BYTE $1; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; +BYTE $1; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; +BYTE $1; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; +BYTE $1; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; +BYTE $1; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; +BYTE $1; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; +BYTE $1; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; +BYTE $1; +BYTE $1; +BYTE $1; BYTE $0; +BYTE $1; BYTE $0; BYTE $0; +BYTE $1; BYTE $0; BYTE $0; BYTE $0; BYTE $0; +BYTE $1; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; +BYTE $1; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; +BYTE $1; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; +BYTE $1; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; +#include "pclinetest.h" +BYTE $2; +#include "pclinetest.h" +BYTE $2; + +TEXT pcfromline(SB),7,$0 // Each record stores its line delta, then n, then n more bytes +BYTE $31; BYTE $0; +BYTE $1; BYTE $1; BYTE $0; +BYTE $1; BYTE $0; + +BYTE $2; BYTE $4; BYTE $0; BYTE $0; BYTE $0; BYTE $0; + + +#include "pclinetest.h" +BYTE $4; BYTE $0; + + +BYTE $3; BYTE $3; BYTE $0; BYTE $0; BYTE $0; +#include "pclinetest.h" + + +BYTE $4; BYTE $3; BYTE $0; BYTE $0; BYTE $0; + +TEXT main(SB),7,$0 + // Prevent GC of our test symbols + CALL linefrompc(SB) + CALL pcfromline(SB) + +// Keep the linker happy +TEXT main·main(SB),7,$0 + RET + +TEXT main·init(SB),7,$0 + RET diff --git a/src/pkg/debug/gosym/pclntab.go b/src/pkg/debug/gosym/pclntab.go new file mode 100644 index 000000000..9d7b0d15f --- /dev/null +++ b/src/pkg/debug/gosym/pclntab.go @@ -0,0 +1,82 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +/* + * Line tables + */ + +package gosym + +import "encoding/binary" + +type LineTable struct { + Data []byte + PC uint64 + Line int +} + +// TODO(rsc): Need to pull in quantum from architecture definition. +const quantum = 1 + +func (t *LineTable) parse(targetPC uint64, targetLine int) (b []byte, pc uint64, line int) { + // The PC/line table can be thought of as a sequence of + // * + // batches. Each update batch results in a (pc, line) pair, + // where line applies to every PC from pc up to but not + // including the pc of the next pair. + // + // Here we process each update individually, which simplifies + // the code, but makes the corner cases more confusing. + b, pc, line = t.Data, t.PC, t.Line + for pc <= targetPC && line != targetLine && len(b) > 0 { + code := b[0] + b = b[1:] + switch { + case code == 0: + if len(b) < 4 { + b = b[0:0] + break + } + val := binary.BigEndian.Uint32(b) + b = b[4:] + line += int(val) + case code <= 64: + line += int(code) + case code <= 128: + line -= int(code - 64) + default: + pc += quantum * uint64(code-128) + continue + } + pc += quantum + } + return b, pc, line +} + +func (t *LineTable) slice(pc uint64) *LineTable { + data, pc, line := t.parse(pc, -1) + return &LineTable{data, pc, line} +} + +func (t *LineTable) PCToLine(pc uint64) int { + _, _, line := t.parse(pc, -1) + return line +} + +func (t *LineTable) LineToPC(line int, maxpc uint64) uint64 { + _, pc, line1 := t.parse(maxpc, line) + if line1 != line { + return 0 + } + // Subtract quantum from PC to account for post-line increment + return pc - quantum +} + +// NewLineTable returns a new PC/line table +// corresponding to the encoded data. +// Text must be the start address of the +// corresponding text segment. +func NewLineTable(data []byte, text uint64) *LineTable { + return &LineTable{data, text, 0} +} diff --git a/src/pkg/debug/gosym/pclntab_test.go b/src/pkg/debug/gosym/pclntab_test.go new file mode 100644 index 000000000..c83e64eab --- /dev/null +++ b/src/pkg/debug/gosym/pclntab_test.go @@ -0,0 +1,204 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package gosym + +import ( + "debug/elf" + "os" + "testing" + "syscall" +) + +func dotest() bool { + // For now, only works on ELF platforms. + return syscall.OS == "linux" && os.Getenv("GOARCH") == "amd64" +} + +func getTable(t *testing.T) *Table { + f, tab := crack(os.Args[0], t) + f.Close() + return tab +} + +func crack(file string, t *testing.T) (*elf.File, *Table) { + // Open self + f, err := elf.Open(file) + if err != nil { + t.Fatal(err) + } + return parse(file, f, t) +} + +func parse(file string, f *elf.File, t *testing.T) (*elf.File, *Table) { + symdat, err := f.Section(".gosymtab").Data() + if err != nil { + f.Close() + t.Fatalf("reading %s gosymtab: %v", file, err) + } + pclndat, err := f.Section(".gopclntab").Data() + if err != nil { + f.Close() + t.Fatalf("reading %s gopclntab: %v", file, err) + } + + pcln := NewLineTable(pclndat, f.Section(".text").Addr) + tab, err := NewTable(symdat, pcln) + if err != nil { + f.Close() + t.Fatalf("parsing %s gosymtab: %v", file, err) + } + + return f, tab +} + +var goarch = os.Getenv("O") + +func TestLineFromAline(t *testing.T) { + if !dotest() { + return + } + + tab := getTable(t) + + // Find the sym package + pkg := tab.LookupFunc("debug/gosym.TestLineFromAline").Obj + if pkg == nil { + t.Fatalf("nil pkg") + } + + // Walk every absolute line and ensure that we hit every + // source line monotonically + lastline := make(map[string]int) + final := -1 + for i := 0; i < 10000; i++ { + path, line := pkg.lineFromAline(i) + // Check for end of object + if path == "" { + if final == -1 { + final = i - 1 + } + continue + } else if final != -1 { + t.Fatalf("reached end of package at absolute line %d, but absolute line %d mapped to %s:%d", final, i, path, line) + } + // It's okay to see files multiple times (e.g., sys.a) + if line == 1 { + lastline[path] = 1 + continue + } + // Check that the is the next line in path + ll, ok := lastline[path] + if !ok { + t.Errorf("file %s starts on line %d", path, line) + } else if line != ll+1 { + t.Errorf("expected next line of file %s to be %d, got %d", path, ll+1, line) + } + lastline[path] = line + } + if final == -1 { + t.Errorf("never reached end of object") + } +} + +func TestLineAline(t *testing.T) { + if !dotest() { + return + } + + tab := getTable(t) + + for _, o := range tab.Files { + // A source file can appear multiple times in a + // object. alineFromLine will always return alines in + // the first file, so track which lines we've seen. + found := make(map[string]int) + for i := 0; i < 1000; i++ { + path, line := o.lineFromAline(i) + if path == "" { + break + } + + // cgo files are full of 'Z' symbols, which we don't handle + if len(path) > 4 && path[len(path)-4:] == ".cgo" { + continue + } + + if minline, ok := found[path]; path != "" && ok { + if minline >= line { + // We've already covered this file + continue + } + } + found[path] = line + + a, err := o.alineFromLine(path, line) + if err != nil { + t.Errorf("absolute line %d in object %s maps to %s:%d, but mapping that back gives error %s", i, o.Paths[0].Name, path, line, err) + } else if a != i { + t.Errorf("absolute line %d in object %s maps to %s:%d, which maps back to absolute line %d\n", i, o.Paths[0].Name, path, line, a) + } + } + } +} + +func TestPCLine(t *testing.T) { + if !dotest() { + return + } + + f, tab := crack("_test/pclinetest", t) + text := f.Section(".text") + textdat, err := text.Data() + if err != nil { + t.Fatalf("reading .text: %v", err) + } + + // Test PCToLine + sym := tab.LookupFunc("linefrompc") + wantLine := 0 + for pc := sym.Entry; pc < sym.End; pc++ { + file, line, fn := tab.PCToLine(pc) + off := pc - text.Addr // TODO(rsc): should not need off; bug in 8g + wantLine += int(textdat[off]) + if fn == nil { + t.Errorf("failed to get line of PC %#x", pc) + } else if len(file) < 12 || file[len(file)-12:] != "pclinetest.s" || line != wantLine || fn != sym { + t.Errorf("expected %s:%d (%s) at PC %#x, got %s:%d (%s)", "pclinetest.s", wantLine, sym.Name, pc, file, line, fn.Name) + } + } + + // Test LineToPC + sym = tab.LookupFunc("pcfromline") + lookupline := -1 + wantLine = 0 + off := uint64(0) // TODO(rsc): should not need off; bug in 8g + for pc := sym.Value; pc < sym.End; pc += 2 + uint64(textdat[off]) { + file, line, fn := tab.PCToLine(pc) + off = pc - text.Addr + wantLine += int(textdat[off]) + if line != wantLine { + t.Errorf("expected line %d at PC %#x in pcfromline, got %d", wantLine, pc, line) + off = pc + 1 - text.Addr + continue + } + if lookupline == -1 { + lookupline = line + } + for ; lookupline <= line; lookupline++ { + pc2, fn2, err := tab.LineToPC(file, lookupline) + if lookupline != line { + // Should be nothing on this line + if err == nil { + t.Errorf("expected no PC at line %d, got %#x (%s)", lookupline, pc2, fn2.Name) + } + } else if err != nil { + t.Errorf("failed to get PC of line %d: %s", lookupline, err) + } else if pc != pc2 { + t.Errorf("expected PC %#x (%s) at line %d, got PC %#x (%s)", pc, fn.Name, line, pc2, fn2.Name) + } + } + off = pc + 1 - text.Addr + } +} diff --git a/src/pkg/debug/gosym/symtab.go b/src/pkg/debug/gosym/symtab.go new file mode 100644 index 000000000..dea460d71 --- /dev/null +++ b/src/pkg/debug/gosym/symtab.go @@ -0,0 +1,548 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Package gosym implements access to the Go symbol +// and line number tables embedded in Go binaries generated +// by the gc compilers. +package gosym + +// The table format is a variant of the format used in Plan 9's a.out +// format, documented at http://plan9.bell-labs.com/magic/man2html/6/a.out. +// The best reference for the differences between the Plan 9 format +// and the Go format is the runtime source, specifically ../../runtime/symtab.c. + +import ( + "encoding/binary" + "fmt" + "os" + "strconv" + "strings" +) + +/* + * Symbols + */ + +// A Sym represents a single symbol table entry. +type Sym struct { + Value uint64 + Type byte + Name string + GoType uint64 + // If this symbol if a function symbol, the corresponding Func + Func *Func +} + +// Static returns whether this symbol is static (not visible outside its file). +func (s *Sym) Static() bool { return s.Type >= 'a' } + +// PackageName returns the package part of the symbol name, +// or the empty string if there is none. +func (s *Sym) PackageName() string { + if i := strings.Index(s.Name, "."); i != -1 { + return s.Name[0:i] + } + return "" +} + +// ReceiverName returns the receiver type name of this symbol, +// or the empty string if there is none. +func (s *Sym) ReceiverName() string { + l := strings.Index(s.Name, ".") + r := strings.LastIndex(s.Name, ".") + if l == -1 || r == -1 || l == r { + return "" + } + return s.Name[l+1 : r] +} + +// BaseName returns the symbol name without the package or receiver name. +func (s *Sym) BaseName() string { + if i := strings.LastIndex(s.Name, "."); i != -1 { + return s.Name[i+1:] + } + return s.Name +} + +// A Func collects information about a single function. +type Func struct { + Entry uint64 + *Sym + End uint64 + Params []*Sym + Locals []*Sym + FrameSize int + LineTable *LineTable + Obj *Obj +} + +// An Obj represents a single object file. +type Obj struct { + Funcs []Func + Paths []Sym +} + +/* + * Symbol tables + */ + +// Table represents a Go symbol table. It stores all of the +// symbols decoded from the program and provides methods to translate +// between symbols, names, and addresses. +type Table struct { + Syms []Sym + Funcs []Func + Files map[string]*Obj + Objs []Obj + // textEnd uint64; +} + +type sym struct { + value uint32 + gotype uint32 + typ byte + name []byte +} + +func walksymtab(data []byte, fn func(sym) os.Error) os.Error { + var s sym + p := data + for len(p) >= 6 { + s.value = binary.BigEndian.Uint32(p[0:4]) + typ := p[4] + if typ&0x80 == 0 { + return &DecodingError{len(data) - len(p) + 4, "bad symbol type", typ} + } + typ &^= 0x80 + s.typ = typ + p = p[5:] + var i int + var nnul int + for i = 0; i < len(p); i++ { + if p[i] == 0 { + nnul = 1 + break + } + } + switch typ { + case 'z', 'Z': + p = p[i+nnul:] + for i = 0; i+2 <= len(p); i += 2 { + if p[i] == 0 && p[i+1] == 0 { + nnul = 2 + break + } + } + } + if i+nnul+4 > len(p) { + return &DecodingError{len(data), "unexpected EOF", nil} + } + s.name = p[0:i] + i += nnul + s.gotype = binary.BigEndian.Uint32(p[i : i+4]) + p = p[i+4:] + fn(s) + } + return nil +} + +// NewTable decodes the Go symbol table in data, +// returning an in-memory representation. +func NewTable(symtab []byte, pcln *LineTable) (*Table, os.Error) { + var n int + err := walksymtab(symtab, func(s sym) os.Error { + n++ + return nil + }) + if err != nil { + return nil, err + } + + var t Table + fname := make(map[uint16]string) + t.Syms = make([]Sym, 0, n) + nf := 0 + nz := 0 + lasttyp := uint8(0) + err = walksymtab(symtab, func(s sym) os.Error { + n := len(t.Syms) + t.Syms = t.Syms[0 : n+1] + ts := &t.Syms[n] + ts.Type = s.typ + ts.Value = uint64(s.value) + ts.GoType = uint64(s.gotype) + switch s.typ { + default: + // rewrite name to use . instead of · (c2 b7) + w := 0 + b := s.name + for i := 0; i < len(b); i++ { + if b[i] == 0xc2 && i+1 < len(b) && b[i+1] == 0xb7 { + i++ + b[i] = '.' + } + b[w] = b[i] + w++ + } + ts.Name = string(s.name[0:w]) + case 'z', 'Z': + if lasttyp != 'z' && lasttyp != 'Z' { + nz++ + } + for i := 0; i < len(s.name); i += 2 { + eltIdx := binary.BigEndian.Uint16(s.name[i : i+2]) + elt, ok := fname[eltIdx] + if !ok { + return &DecodingError{-1, "bad filename code", eltIdx} + } + if n := len(ts.Name); n > 0 && ts.Name[n-1] != '/' { + ts.Name += "/" + } + ts.Name += elt + } + } + switch s.typ { + case 'T', 't', 'L', 'l': + nf++ + case 'f': + fname[uint16(s.value)] = ts.Name + } + lasttyp = s.typ + return nil + }) + if err != nil { + return nil, err + } + + t.Funcs = make([]Func, 0, nf) + t.Objs = make([]Obj, 0, nz) + t.Files = make(map[string]*Obj) + + // Count text symbols and attach frame sizes, parameters, and + // locals to them. Also, find object file boundaries. + var obj *Obj + lastf := 0 + for i := 0; i < len(t.Syms); i++ { + sym := &t.Syms[i] + switch sym.Type { + case 'Z', 'z': // path symbol + // Finish the current object + if obj != nil { + obj.Funcs = t.Funcs[lastf:] + } + lastf = len(t.Funcs) + + // Start new object + n := len(t.Objs) + t.Objs = t.Objs[0 : n+1] + obj = &t.Objs[n] + + // Count & copy path symbols + var end int + for end = i + 1; end < len(t.Syms); end++ { + if c := t.Syms[end].Type; c != 'Z' && c != 'z' { + break + } + } + obj.Paths = t.Syms[i:end] + i = end - 1 // loop will i++ + + // Record file names + depth := 0 + for j := range obj.Paths { + s := &obj.Paths[j] + if s.Name == "" { + depth-- + } else { + if depth == 0 { + t.Files[s.Name] = obj + } + depth++ + } + } + + case 'T', 't', 'L', 'l': // text symbol + if n := len(t.Funcs); n > 0 { + t.Funcs[n-1].End = sym.Value + } + if sym.Name == "etext" { + continue + } + + // Count parameter and local (auto) syms + var np, na int + var end int + countloop: + for end = i + 1; end < len(t.Syms); end++ { + switch t.Syms[end].Type { + case 'T', 't', 'L', 'l', 'Z', 'z': + break countloop + case 'p': + np++ + case 'a': + na++ + } + } + + // Fill in the function symbol + n := len(t.Funcs) + t.Funcs = t.Funcs[0 : n+1] + fn := &t.Funcs[n] + sym.Func = fn + fn.Params = make([]*Sym, 0, np) + fn.Locals = make([]*Sym, 0, na) + fn.Sym = sym + fn.Entry = sym.Value + fn.Obj = obj + if pcln != nil { + fn.LineTable = pcln.slice(fn.Entry) + pcln = fn.LineTable + } + for j := i; j < end; j++ { + s := &t.Syms[j] + switch s.Type { + case 'm': + fn.FrameSize = int(s.Value) + case 'p': + n := len(fn.Params) + fn.Params = fn.Params[0 : n+1] + fn.Params[n] = s + case 'a': + n := len(fn.Locals) + fn.Locals = fn.Locals[0 : n+1] + fn.Locals[n] = s + } + } + i = end - 1 // loop will i++ + } + } + if obj != nil { + obj.Funcs = t.Funcs[lastf:] + } + return &t, nil +} + +// PCToFunc returns the function containing the program counter pc, +// or nil if there is no such function. +func (t *Table) PCToFunc(pc uint64) *Func { + funcs := t.Funcs + for len(funcs) > 0 { + m := len(funcs) / 2 + fn := &funcs[m] + switch { + case pc < fn.Entry: + funcs = funcs[0:m] + case fn.Entry <= pc && pc < fn.End: + return fn + default: + funcs = funcs[m+1:] + } + } + return nil +} + +// PCToLine looks up line number information for a program counter. +// If there is no information, it returns fn == nil. +func (t *Table) PCToLine(pc uint64) (file string, line int, fn *Func) { + if fn = t.PCToFunc(pc); fn == nil { + return + } + file, line = fn.Obj.lineFromAline(fn.LineTable.PCToLine(pc)) + return +} + +// LineToPC looks up the first program counter on the given line in +// the named file. Returns UnknownPathError or UnknownLineError if +// there is an error looking up this line. +func (t *Table) LineToPC(file string, line int) (pc uint64, fn *Func, err os.Error) { + obj, ok := t.Files[file] + if !ok { + return 0, nil, UnknownFileError(file) + } + abs, err := obj.alineFromLine(file, line) + if err != nil { + return + } + for i := range obj.Funcs { + f := &obj.Funcs[i] + pc := f.LineTable.LineToPC(abs, f.End) + if pc != 0 { + return pc, f, nil + } + } + return 0, nil, &UnknownLineError{file, line} +} + +// LookupSym returns the text, data, or bss symbol with the given name, +// or nil if no such symbol is found. +func (t *Table) LookupSym(name string) *Sym { + // TODO(austin) Maybe make a map + for i := range t.Syms { + s := &t.Syms[i] + switch s.Type { + case 'T', 't', 'L', 'l', 'D', 'd', 'B', 'b': + if s.Name == name { + return s + } + } + } + return nil +} + +// LookupFunc returns the text, data, or bss symbol with the given name, +// or nil if no such symbol is found. +func (t *Table) LookupFunc(name string) *Func { + for i := range t.Funcs { + f := &t.Funcs[i] + if f.Sym.Name == name { + return f + } + } + return nil +} + +// SymByAddr returns the text, data, or bss symbol starting at the given address. +// TODO(rsc): Allow lookup by any address within the symbol. +func (t *Table) SymByAddr(addr uint64) *Sym { + // TODO(austin) Maybe make a map + for i := range t.Syms { + s := &t.Syms[i] + switch s.Type { + case 'T', 't', 'L', 'l', 'D', 'd', 'B', 'b': + if s.Value == addr { + return s + } + } + } + return nil +} + +/* + * Object files + */ + +func (o *Obj) lineFromAline(aline int) (string, int) { + type stackEnt struct { + path string + start int + offset int + prev *stackEnt + } + + noPath := &stackEnt{"", 0, 0, nil} + tos := noPath + + // TODO(austin) I have no idea how 'Z' symbols work, except + // that they pop the stack. +pathloop: + for _, s := range o.Paths { + val := int(s.Value) + switch { + case val > aline: + break pathloop + + case val == 1: + // Start a new stack + tos = &stackEnt{s.Name, val, 0, noPath} + + case s.Name == "": + // Pop + if tos == noPath { + return "", 0 + } + tos.prev.offset += val - tos.start + tos = tos.prev + + default: + // Push + tos = &stackEnt{s.Name, val, 0, tos} + } + } + + if tos == noPath { + return "", 0 + } + return tos.path, aline - tos.start - tos.offset + 1 +} + +func (o *Obj) alineFromLine(path string, line int) (int, os.Error) { + if line < 1 { + return 0, &UnknownLineError{path, line} + } + + for i, s := range o.Paths { + // Find this path + if s.Name != path { + continue + } + + // Find this line at this stack level + depth := 0 + var incstart int + line += int(s.Value) + pathloop: + for _, s := range o.Paths[i:] { + val := int(s.Value) + switch { + case depth == 1 && val >= line: + return line - 1, nil + + case s.Name == "": + depth-- + if depth == 0 { + break pathloop + } else if depth == 1 { + line += val - incstart + } + + default: + if depth == 1 { + incstart = val + } + depth++ + } + } + return 0, &UnknownLineError{path, line} + } + return 0, UnknownFileError(path) +} + +/* + * Errors + */ + +// UnknownFileError represents a failure to find the specific file in +// the symbol table. +type UnknownFileError string + +func (e UnknownFileError) String() string { return "unknown file: " + string(e) } + +// UnknownLineError represents a failure to map a line to a program +// counter, either because the line is beyond the bounds of the file +// or because there is no code on the given line. +type UnknownLineError struct { + File string + Line int +} + +func (e *UnknownLineError) String() string { + return "no code at " + e.File + ":" + strconv.Itoa(e.Line) +} + +// DecodingError represents an error during the decoding of +// the symbol table. +type DecodingError struct { + off int + msg string + val interface{} +} + +func (e *DecodingError) String() string { + msg := e.msg + if e.val != nil { + msg += fmt.Sprintf(" '%v'", e.val) + } + msg += fmt.Sprintf(" at byte %#x", e.off) + return msg +} diff --git a/src/pkg/debug/macho/Makefile b/src/pkg/debug/macho/Makefile new file mode 100644 index 000000000..5fbbf1efe --- /dev/null +++ b/src/pkg/debug/macho/Makefile @@ -0,0 +1,12 @@ +# Copyright 2009 The Go Authors. All rights reserved. +# Use of this source code is governed by a BSD-style +# license that can be found in the LICENSE file. + +include ../../../Make.inc + +TARG=debug/macho +GOFILES=\ + macho.go\ + file.go\ + +include ../../../Make.pkg diff --git a/src/pkg/debug/macho/file.go b/src/pkg/debug/macho/file.go new file mode 100644 index 000000000..721a4c416 --- /dev/null +++ b/src/pkg/debug/macho/file.go @@ -0,0 +1,517 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Package macho implements access to Mach-O object files, as defined by +// http://developer.apple.com/mac/library/documentation/DeveloperTools/Conceptual/MachORuntime/Reference/reference.html. +package macho + +// High level access to low level data structures. + +import ( + "bytes" + "debug/dwarf" + "encoding/binary" + "fmt" + "io" + "os" +) + +// A File represents an open Mach-O file. +type File struct { + FileHeader + ByteOrder binary.ByteOrder + Loads []Load + Sections []*Section + + Symtab *Symtab + Dysymtab *Dysymtab + + closer io.Closer +} + +// A Load represents any Mach-O load command. +type Load interface { + Raw() []byte +} + +// A LoadBytes is the uninterpreted bytes of a Mach-O load command. +type LoadBytes []byte + +func (b LoadBytes) Raw() []byte { return b } + +// A SegmentHeader is the header for a Mach-O 32-bit or 64-bit load segment command. +type SegmentHeader struct { + Cmd LoadCmd + Len uint32 + Name string + Addr uint64 + Memsz uint64 + Offset uint64 + Filesz uint64 + Maxprot uint32 + Prot uint32 + Nsect uint32 + Flag uint32 +} + +// A Segment represents a Mach-O 32-bit or 64-bit load segment command. +type Segment struct { + LoadBytes + SegmentHeader + + // Embed ReaderAt for ReadAt method. + // Do not embed SectionReader directly + // to avoid having Read and Seek. + // If a client wants Read and Seek it must use + // Open() to avoid fighting over the seek offset + // with other clients. + io.ReaderAt + sr *io.SectionReader +} + +// Data reads and returns the contents of the segment. +func (s *Segment) Data() ([]byte, os.Error) { + dat := make([]byte, s.sr.Size()) + n, err := s.sr.ReadAt(dat, 0) + return dat[0:n], err +} + +// Open returns a new ReadSeeker reading the segment. +func (s *Segment) Open() io.ReadSeeker { return io.NewSectionReader(s.sr, 0, 1<<63-1) } + +type SectionHeader struct { + Name string + Seg string + Addr uint64 + Size uint64 + Offset uint32 + Align uint32 + Reloff uint32 + Nreloc uint32 + Flags uint32 +} + +type Section struct { + SectionHeader + + // Embed ReaderAt for ReadAt method. + // Do not embed SectionReader directly + // to avoid having Read and Seek. + // If a client wants Read and Seek it must use + // Open() to avoid fighting over the seek offset + // with other clients. + io.ReaderAt + sr *io.SectionReader +} + +// Data reads and returns the contents of the Mach-O section. +func (s *Section) Data() ([]byte, os.Error) { + dat := make([]byte, s.sr.Size()) + n, err := s.sr.ReadAt(dat, 0) + return dat[0:n], err +} + +// Open returns a new ReadSeeker reading the Mach-O section. +func (s *Section) Open() io.ReadSeeker { return io.NewSectionReader(s.sr, 0, 1<<63-1) } + +// A Dylib represents a Mach-O load dynamic library command. +type Dylib struct { + LoadBytes + Name string + Time uint32 + CurrentVersion uint32 + CompatVersion uint32 +} + +// A Symtab represents a Mach-O symbol table command. +type Symtab struct { + LoadBytes + SymtabCmd + Syms []Symbol +} + +// A Dysymtab represents a Mach-O dynamic symbol table command. +type Dysymtab struct { + LoadBytes + DysymtabCmd + IndirectSyms []uint32 // indices into Symtab.Syms +} + +/* + * Mach-O reader + */ + +type FormatError struct { + off int64 + msg string + val interface{} +} + +func (e *FormatError) String() string { + msg := e.msg + if e.val != nil { + msg += fmt.Sprintf(" '%v'", e.val) + } + msg += fmt.Sprintf(" in record at byte %#x", e.off) + return msg +} + +// Open opens the named file using os.Open and prepares it for use as a Mach-O binary. +func Open(name string) (*File, os.Error) { + f, err := os.Open(name) + if err != nil { + return nil, err + } + ff, err := NewFile(f) + if err != nil { + f.Close() + return nil, err + } + ff.closer = f + return ff, nil +} + +// Close closes the File. +// If the File was created using NewFile directly instead of Open, +// Close has no effect. +func (f *File) Close() os.Error { + var err os.Error + if f.closer != nil { + err = f.closer.Close() + f.closer = nil + } + return err +} + +// NewFile creates a new File for accessing a Mach-O binary in an underlying reader. +// The Mach-O binary is expected to start at position 0 in the ReaderAt. +func NewFile(r io.ReaderAt) (*File, os.Error) { + f := new(File) + sr := io.NewSectionReader(r, 0, 1<<63-1) + + // Read and decode Mach magic to determine byte order, size. + // Magic32 and Magic64 differ only in the bottom bit. + var ident [4]byte + if _, err := r.ReadAt(ident[0:], 0); err != nil { + return nil, err + } + be := binary.BigEndian.Uint32(ident[0:]) + le := binary.LittleEndian.Uint32(ident[0:]) + switch Magic32 &^ 1 { + case be &^ 1: + f.ByteOrder = binary.BigEndian + f.Magic = be + case le &^ 1: + f.ByteOrder = binary.LittleEndian + f.Magic = le + default: + return nil, &FormatError{0, "invalid magic number", nil} + } + + // Read entire file header. + if err := binary.Read(sr, f.ByteOrder, &f.FileHeader); err != nil { + return nil, err + } + + // Then load commands. + offset := int64(fileHeaderSize32) + if f.Magic == Magic64 { + offset = fileHeaderSize64 + } + dat := make([]byte, f.Cmdsz) + if _, err := r.ReadAt(dat, offset); err != nil { + return nil, err + } + f.Loads = make([]Load, f.Ncmd) + bo := f.ByteOrder + for i := range f.Loads { + // Each load command begins with uint32 command and length. + if len(dat) < 8 { + return nil, &FormatError{offset, "command block too small", nil} + } + cmd, siz := LoadCmd(bo.Uint32(dat[0:4])), bo.Uint32(dat[4:8]) + if siz < 8 || siz > uint32(len(dat)) { + return nil, &FormatError{offset, "invalid command block size", nil} + } + var cmddat []byte + cmddat, dat = dat[0:siz], dat[siz:] + offset += int64(siz) + var s *Segment + switch cmd { + default: + f.Loads[i] = LoadBytes(cmddat) + + case LoadCmdDylib: + var hdr DylibCmd + b := bytes.NewBuffer(cmddat) + if err := binary.Read(b, bo, &hdr); err != nil { + return nil, err + } + l := new(Dylib) + if hdr.Name >= uint32(len(cmddat)) { + return nil, &FormatError{offset, "invalid name in dynamic library command", hdr.Name} + } + l.Name = cstring(cmddat[hdr.Name:]) + l.Time = hdr.Time + l.CurrentVersion = hdr.CurrentVersion + l.CompatVersion = hdr.CompatVersion + l.LoadBytes = LoadBytes(cmddat) + f.Loads[i] = l + + case LoadCmdSymtab: + var hdr SymtabCmd + b := bytes.NewBuffer(cmddat) + if err := binary.Read(b, bo, &hdr); err != nil { + return nil, err + } + strtab := make([]byte, hdr.Strsize) + if _, err := r.ReadAt(strtab, int64(hdr.Stroff)); err != nil { + return nil, err + } + var symsz int + if f.Magic == Magic64 { + symsz = 16 + } else { + symsz = 12 + } + symdat := make([]byte, int(hdr.Nsyms)*symsz) + if _, err := r.ReadAt(symdat, int64(hdr.Symoff)); err != nil { + return nil, err + } + st, err := f.parseSymtab(symdat, strtab, cmddat, &hdr, offset) + if err != nil { + return nil, err + } + f.Loads[i] = st + f.Symtab = st + + case LoadCmdDysymtab: + var hdr DysymtabCmd + b := bytes.NewBuffer(cmddat) + if err := binary.Read(b, bo, &hdr); err != nil { + return nil, err + } + dat := make([]byte, hdr.Nindirectsyms*4) + if _, err := r.ReadAt(dat, int64(hdr.Indirectsymoff)); err != nil { + return nil, err + } + x := make([]uint32, hdr.Nindirectsyms) + if err := binary.Read(bytes.NewBuffer(dat), bo, x); err != nil { + return nil, err + } + st := new(Dysymtab) + st.LoadBytes = LoadBytes(cmddat) + st.DysymtabCmd = hdr + st.IndirectSyms = x + f.Loads[i] = st + f.Dysymtab = st + + case LoadCmdSegment: + var seg32 Segment32 + b := bytes.NewBuffer(cmddat) + if err := binary.Read(b, bo, &seg32); err != nil { + return nil, err + } + s = new(Segment) + s.LoadBytes = cmddat + s.Cmd = cmd + s.Len = siz + s.Name = cstring(seg32.Name[0:]) + s.Addr = uint64(seg32.Addr) + s.Memsz = uint64(seg32.Memsz) + s.Offset = uint64(seg32.Offset) + s.Filesz = uint64(seg32.Filesz) + s.Maxprot = seg32.Maxprot + s.Prot = seg32.Prot + s.Nsect = seg32.Nsect + s.Flag = seg32.Flag + f.Loads[i] = s + for i := 0; i < int(s.Nsect); i++ { + var sh32 Section32 + if err := binary.Read(b, bo, &sh32); err != nil { + return nil, err + } + sh := new(Section) + sh.Name = cstring(sh32.Name[0:]) + sh.Seg = cstring(sh32.Seg[0:]) + sh.Addr = uint64(sh32.Addr) + sh.Size = uint64(sh32.Size) + sh.Offset = sh32.Offset + sh.Align = sh32.Align + sh.Reloff = sh32.Reloff + sh.Nreloc = sh32.Nreloc + sh.Flags = sh32.Flags + f.pushSection(sh, r) + } + + case LoadCmdSegment64: + var seg64 Segment64 + b := bytes.NewBuffer(cmddat) + if err := binary.Read(b, bo, &seg64); err != nil { + return nil, err + } + s = new(Segment) + s.LoadBytes = cmddat + s.Cmd = cmd + s.Len = siz + s.Name = cstring(seg64.Name[0:]) + s.Addr = seg64.Addr + s.Memsz = seg64.Memsz + s.Offset = seg64.Offset + s.Filesz = seg64.Filesz + s.Maxprot = seg64.Maxprot + s.Prot = seg64.Prot + s.Nsect = seg64.Nsect + s.Flag = seg64.Flag + f.Loads[i] = s + for i := 0; i < int(s.Nsect); i++ { + var sh64 Section64 + if err := binary.Read(b, bo, &sh64); err != nil { + return nil, err + } + sh := new(Section) + sh.Name = cstring(sh64.Name[0:]) + sh.Seg = cstring(sh64.Seg[0:]) + sh.Addr = sh64.Addr + sh.Size = sh64.Size + sh.Offset = sh64.Offset + sh.Align = sh64.Align + sh.Reloff = sh64.Reloff + sh.Nreloc = sh64.Nreloc + sh.Flags = sh64.Flags + f.pushSection(sh, r) + } + } + if s != nil { + s.sr = io.NewSectionReader(r, int64(s.Offset), int64(s.Filesz)) + s.ReaderAt = s.sr + } + } + return f, nil +} + +func (f *File) parseSymtab(symdat, strtab, cmddat []byte, hdr *SymtabCmd, offset int64) (*Symtab, os.Error) { + bo := f.ByteOrder + symtab := make([]Symbol, hdr.Nsyms) + b := bytes.NewBuffer(symdat) + for i := range symtab { + var n Nlist64 + if f.Magic == Magic64 { + if err := binary.Read(b, bo, &n); err != nil { + return nil, err + } + } else { + var n32 Nlist32 + if err := binary.Read(b, bo, &n32); err != nil { + return nil, err + } + n.Name = n32.Name + n.Type = n32.Type + n.Sect = n32.Sect + n.Desc = n32.Desc + n.Value = uint64(n32.Value) + } + sym := &symtab[i] + if n.Name >= uint32(len(strtab)) { + return nil, &FormatError{offset, "invalid name in symbol table", n.Name} + } + sym.Name = cstring(strtab[n.Name:]) + sym.Type = n.Type + sym.Sect = n.Sect + sym.Desc = n.Desc + sym.Value = n.Value + } + st := new(Symtab) + st.LoadBytes = LoadBytes(cmddat) + st.Syms = symtab + return st, nil +} + +func (f *File) pushSection(sh *Section, r io.ReaderAt) { + f.Sections = append(f.Sections, sh) + sh.sr = io.NewSectionReader(r, int64(sh.Offset), int64(sh.Size)) + sh.ReaderAt = sh.sr +} + +func cstring(b []byte) string { + var i int + for i = 0; i < len(b) && b[i] != 0; i++ { + } + return string(b[0:i]) +} + +// Segment returns the first Segment with the given name, or nil if no such segment exists. +func (f *File) Segment(name string) *Segment { + for _, l := range f.Loads { + if s, ok := l.(*Segment); ok && s.Name == name { + return s + } + } + return nil +} + +// Section returns the first section with the given name, or nil if no such +// section exists. +func (f *File) Section(name string) *Section { + for _, s := range f.Sections { + if s.Name == name { + return s + } + } + return nil +} + +// DWARF returns the DWARF debug information for the Mach-O file. +func (f *File) DWARF() (*dwarf.Data, os.Error) { + // There are many other DWARF sections, but these + // are the required ones, and the debug/dwarf package + // does not use the others, so don't bother loading them. + var names = [...]string{"abbrev", "info", "str"} + var dat [len(names)][]byte + for i, name := range names { + name = "__debug_" + name + s := f.Section(name) + if s == nil { + return nil, os.NewError("missing Mach-O section " + name) + } + b, err := s.Data() + if err != nil && uint64(len(b)) < s.Size { + return nil, err + } + dat[i] = b + } + + abbrev, info, str := dat[0], dat[1], dat[2] + return dwarf.New(abbrev, nil, nil, info, nil, nil, nil, str) +} + +// ImportedSymbols returns the names of all symbols +// referred to by the binary f that are expected to be +// satisfied by other libraries at dynamic load time. +func (f *File) ImportedSymbols() ([]string, os.Error) { + if f.Dysymtab == nil || f.Symtab == nil { + return nil, &FormatError{0, "missing symbol table", nil} + } + + st := f.Symtab + dt := f.Dysymtab + var all []string + for _, s := range st.Syms[dt.Iundefsym : dt.Iundefsym+dt.Nundefsym] { + all = append(all, s.Name) + } + return all, nil +} + +// ImportedLibraries returns the paths of all libraries +// referred to by the binary f that are expected to be +// linked with the binary at dynamic link time. +func (f *File) ImportedLibraries() ([]string, os.Error) { + var all []string + for _, l := range f.Loads { + if lib, ok := l.(*Dylib); ok { + all = append(all, lib.Name) + } + } + return all, nil +} diff --git a/src/pkg/debug/macho/file_test.go b/src/pkg/debug/macho/file_test.go new file mode 100644 index 000000000..56d8a20be --- /dev/null +++ b/src/pkg/debug/macho/file_test.go @@ -0,0 +1,167 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package macho + +import ( + "reflect" + "testing" +) + +type fileTest struct { + file string + hdr FileHeader + segments []*SegmentHeader + sections []*SectionHeader +} + +var fileTests = []fileTest{ + { + "testdata/gcc-386-darwin-exec", + FileHeader{0xfeedface, Cpu386, 0x3, 0x2, 0xc, 0x3c0, 0x85}, + []*SegmentHeader{ + &SegmentHeader{LoadCmdSegment, 0x38, "__PAGEZERO", 0x0, 0x1000, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0}, + &SegmentHeader{LoadCmdSegment, 0xc0, "__TEXT", 0x1000, 0x1000, 0x0, 0x1000, 0x7, 0x5, 0x2, 0x0}, + &SegmentHeader{LoadCmdSegment, 0xc0, "__DATA", 0x2000, 0x1000, 0x1000, 0x1000, 0x7, 0x3, 0x2, 0x0}, + &SegmentHeader{LoadCmdSegment, 0x7c, "__IMPORT", 0x3000, 0x1000, 0x2000, 0x1000, 0x7, 0x7, 0x1, 0x0}, + &SegmentHeader{LoadCmdSegment, 0x38, "__LINKEDIT", 0x4000, 0x1000, 0x3000, 0x12c, 0x7, 0x1, 0x0, 0x0}, + nil, + nil, + nil, + nil, + nil, + nil, + nil, + }, + []*SectionHeader{ + &SectionHeader{"__text", "__TEXT", 0x1f68, 0x88, 0xf68, 0x2, 0x0, 0x0, 0x80000400}, + &SectionHeader{"__cstring", "__TEXT", 0x1ff0, 0xd, 0xff0, 0x0, 0x0, 0x0, 0x2}, + &SectionHeader{"__data", "__DATA", 0x2000, 0x14, 0x1000, 0x2, 0x0, 0x0, 0x0}, + &SectionHeader{"__dyld", "__DATA", 0x2014, 0x1c, 0x1014, 0x2, 0x0, 0x0, 0x0}, + &SectionHeader{"__jump_table", "__IMPORT", 0x3000, 0xa, 0x2000, 0x6, 0x0, 0x0, 0x4000008}, + }, + }, + { + "testdata/gcc-amd64-darwin-exec", + FileHeader{0xfeedfacf, CpuAmd64, 0x80000003, 0x2, 0xb, 0x568, 0x85}, + []*SegmentHeader{ + &SegmentHeader{LoadCmdSegment64, 0x48, "__PAGEZERO", 0x0, 0x100000000, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0}, + &SegmentHeader{LoadCmdSegment64, 0x1d8, "__TEXT", 0x100000000, 0x1000, 0x0, 0x1000, 0x7, 0x5, 0x5, 0x0}, + &SegmentHeader{LoadCmdSegment64, 0x138, "__DATA", 0x100001000, 0x1000, 0x1000, 0x1000, 0x7, 0x3, 0x3, 0x0}, + &SegmentHeader{LoadCmdSegment64, 0x48, "__LINKEDIT", 0x100002000, 0x1000, 0x2000, 0x140, 0x7, 0x1, 0x0, 0x0}, + nil, + nil, + nil, + nil, + nil, + nil, + nil, + }, + []*SectionHeader{ + &SectionHeader{"__text", "__TEXT", 0x100000f14, 0x6d, 0xf14, 0x2, 0x0, 0x0, 0x80000400}, + &SectionHeader{"__symbol_stub1", "__TEXT", 0x100000f81, 0xc, 0xf81, 0x0, 0x0, 0x0, 0x80000408}, + &SectionHeader{"__stub_helper", "__TEXT", 0x100000f90, 0x18, 0xf90, 0x2, 0x0, 0x0, 0x0}, + &SectionHeader{"__cstring", "__TEXT", 0x100000fa8, 0xd, 0xfa8, 0x0, 0x0, 0x0, 0x2}, + &SectionHeader{"__eh_frame", "__TEXT", 0x100000fb8, 0x48, 0xfb8, 0x3, 0x0, 0x0, 0x6000000b}, + &SectionHeader{"__data", "__DATA", 0x100001000, 0x1c, 0x1000, 0x3, 0x0, 0x0, 0x0}, + &SectionHeader{"__dyld", "__DATA", 0x100001020, 0x38, 0x1020, 0x3, 0x0, 0x0, 0x0}, + &SectionHeader{"__la_symbol_ptr", "__DATA", 0x100001058, 0x10, 0x1058, 0x2, 0x0, 0x0, 0x7}, + }, + }, + { + "testdata/gcc-amd64-darwin-exec-debug", + FileHeader{0xfeedfacf, CpuAmd64, 0x80000003, 0xa, 0x4, 0x5a0, 0}, + []*SegmentHeader{ + nil, + &SegmentHeader{LoadCmdSegment64, 0x1d8, "__TEXT", 0x100000000, 0x1000, 0x0, 0x0, 0x7, 0x5, 0x5, 0x0}, + &SegmentHeader{LoadCmdSegment64, 0x138, "__DATA", 0x100001000, 0x1000, 0x0, 0x0, 0x7, 0x3, 0x3, 0x0}, + &SegmentHeader{LoadCmdSegment64, 0x278, "__DWARF", 0x100002000, 0x1000, 0x1000, 0x1bc, 0x7, 0x3, 0x7, 0x0}, + }, + []*SectionHeader{ + &SectionHeader{"__text", "__TEXT", 0x100000f14, 0x0, 0x0, 0x2, 0x0, 0x0, 0x80000400}, + &SectionHeader{"__symbol_stub1", "__TEXT", 0x100000f81, 0x0, 0x0, 0x0, 0x0, 0x0, 0x80000408}, + &SectionHeader{"__stub_helper", "__TEXT", 0x100000f90, 0x0, 0x0, 0x2, 0x0, 0x0, 0x0}, + &SectionHeader{"__cstring", "__TEXT", 0x100000fa8, 0x0, 0x0, 0x0, 0x0, 0x0, 0x2}, + &SectionHeader{"__eh_frame", "__TEXT", 0x100000fb8, 0x0, 0x0, 0x3, 0x0, 0x0, 0x6000000b}, + &SectionHeader{"__data", "__DATA", 0x100001000, 0x0, 0x0, 0x3, 0x0, 0x0, 0x0}, + &SectionHeader{"__dyld", "__DATA", 0x100001020, 0x0, 0x0, 0x3, 0x0, 0x0, 0x0}, + &SectionHeader{"__la_symbol_ptr", "__DATA", 0x100001058, 0x0, 0x0, 0x2, 0x0, 0x0, 0x7}, + &SectionHeader{"__debug_abbrev", "__DWARF", 0x100002000, 0x36, 0x1000, 0x0, 0x0, 0x0, 0x0}, + &SectionHeader{"__debug_aranges", "__DWARF", 0x100002036, 0x30, 0x1036, 0x0, 0x0, 0x0, 0x0}, + &SectionHeader{"__debug_frame", "__DWARF", 0x100002066, 0x40, 0x1066, 0x0, 0x0, 0x0, 0x0}, + &SectionHeader{"__debug_info", "__DWARF", 0x1000020a6, 0x54, 0x10a6, 0x0, 0x0, 0x0, 0x0}, + &SectionHeader{"__debug_line", "__DWARF", 0x1000020fa, 0x47, 0x10fa, 0x0, 0x0, 0x0, 0x0}, + &SectionHeader{"__debug_pubnames", "__DWARF", 0x100002141, 0x1b, 0x1141, 0x0, 0x0, 0x0, 0x0}, + &SectionHeader{"__debug_str", "__DWARF", 0x10000215c, 0x60, 0x115c, 0x0, 0x0, 0x0, 0x0}, + }, + }, +} + +func TestOpen(t *testing.T) { + for i := range fileTests { + tt := &fileTests[i] + + f, err := Open(tt.file) + if err != nil { + t.Error(err) + continue + } + if !reflect.DeepEqual(f.FileHeader, tt.hdr) { + t.Errorf("open %s:\n\thave %#v\n\twant %#v\n", tt.file, f.FileHeader, tt.hdr) + continue + } + for i, l := range f.Loads { + if i >= len(tt.segments) { + break + } + sh := tt.segments[i] + s, ok := l.(*Segment) + if sh == nil { + if ok { + t.Errorf("open %s, section %d: skipping %#v\n", tt.file, i, &s.SegmentHeader) + } + continue + } + if !ok { + t.Errorf("open %s, section %d: not *Segment\n", tt.file, i) + continue + } + have := &s.SegmentHeader + want := sh + if !reflect.DeepEqual(have, want) { + t.Errorf("open %s, segment %d:\n\thave %#v\n\twant %#v\n", tt.file, i, have, want) + } + } + tn := len(tt.segments) + fn := len(f.Loads) + if tn != fn { + t.Errorf("open %s: len(Loads) = %d, want %d", tt.file, fn, tn) + } + + for i, sh := range f.Sections { + if i >= len(tt.sections) { + break + } + have := &sh.SectionHeader + want := tt.sections[i] + if !reflect.DeepEqual(have, want) { + t.Errorf("open %s, section %d:\n\thave %#v\n\twant %#v\n", tt.file, i, have, want) + } + } + tn = len(tt.sections) + fn = len(f.Sections) + if tn != fn { + t.Errorf("open %s: len(Sections) = %d, want %d", tt.file, fn, tn) + } + + } +} + +func TestOpenFailure(t *testing.T) { + filename := "file.go" // not a Mach-O file + _, err := Open(filename) // don't crash + if err == nil { + t.Errorf("open %s: succeeded unexpectedly", filename) + } +} diff --git a/src/pkg/debug/macho/macho.go b/src/pkg/debug/macho/macho.go new file mode 100644 index 000000000..1386f5acf --- /dev/null +++ b/src/pkg/debug/macho/macho.go @@ -0,0 +1,305 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Mach-O header data structures +// http://developer.apple.com/mac/library/documentation/DeveloperTools/Conceptual/MachORuntime/Reference/reference.html + +package macho + +import "strconv" + +// A FileHeader represents a Mach-O file header. +type FileHeader struct { + Magic uint32 + Cpu Cpu + SubCpu uint32 + Type Type + Ncmd uint32 + Cmdsz uint32 + Flags uint32 +} + +const ( + fileHeaderSize32 = 7 * 4 + fileHeaderSize64 = 8 * 4 +) + +const ( + Magic32 uint32 = 0xfeedface + Magic64 uint32 = 0xfeedfacf +) + +// A Type is a Mach-O file type, either an object or an executable. +type Type uint32 + +const ( + TypeObj Type = 1 + TypeExec Type = 2 +) + +// A Cpu is a Mach-O cpu type. +type Cpu uint32 + +const ( + Cpu386 Cpu = 7 + CpuAmd64 Cpu = Cpu386 + 1<<24 +) + +var cpuStrings = []intName{ + {uint32(Cpu386), "Cpu386"}, + {uint32(CpuAmd64), "CpuAmd64"}, +} + +func (i Cpu) String() string { return stringName(uint32(i), cpuStrings, false) } +func (i Cpu) GoString() string { return stringName(uint32(i), cpuStrings, true) } + +// A LoadCmd is a Mach-O load command. +type LoadCmd uint32 + +const ( + LoadCmdSegment LoadCmd = 1 + LoadCmdSymtab LoadCmd = 2 + LoadCmdThread LoadCmd = 4 + LoadCmdUnixThread LoadCmd = 5 // thread+stack + LoadCmdDysymtab LoadCmd = 11 + LoadCmdDylib LoadCmd = 12 + LoadCmdDylinker LoadCmd = 15 + LoadCmdSegment64 LoadCmd = 25 +) + +var cmdStrings = []intName{ + {uint32(LoadCmdSegment), "LoadCmdSegment"}, + {uint32(LoadCmdThread), "LoadCmdThread"}, + {uint32(LoadCmdUnixThread), "LoadCmdUnixThread"}, + {uint32(LoadCmdDylib), "LoadCmdDylib"}, + {uint32(LoadCmdSegment64), "LoadCmdSegment64"}, +} + +func (i LoadCmd) String() string { return stringName(uint32(i), cmdStrings, false) } +func (i LoadCmd) GoString() string { return stringName(uint32(i), cmdStrings, true) } + +// A Segment64 is a 64-bit Mach-O segment load command. +type Segment64 struct { + Cmd LoadCmd + Len uint32 + Name [16]byte + Addr uint64 + Memsz uint64 + Offset uint64 + Filesz uint64 + Maxprot uint32 + Prot uint32 + Nsect uint32 + Flag uint32 +} + +// A Segment32 is a 32-bit Mach-O segment load command. +type Segment32 struct { + Cmd LoadCmd + Len uint32 + Name [16]byte + Addr uint32 + Memsz uint32 + Offset uint32 + Filesz uint32 + Maxprot uint32 + Prot uint32 + Nsect uint32 + Flag uint32 +} + +// A DylibCmd is a Mach-O load dynamic library command. +type DylibCmd struct { + Cmd LoadCmd + Len uint32 + Name uint32 + Time uint32 + CurrentVersion uint32 + CompatVersion uint32 +} + +// A Section32 is a 32-bit Mach-O section header. +type Section32 struct { + Name [16]byte + Seg [16]byte + Addr uint32 + Size uint32 + Offset uint32 + Align uint32 + Reloff uint32 + Nreloc uint32 + Flags uint32 + Reserve1 uint32 + Reserve2 uint32 +} + +// A Section32 is a 64-bit Mach-O section header. +type Section64 struct { + Name [16]byte + Seg [16]byte + Addr uint64 + Size uint64 + Offset uint32 + Align uint32 + Reloff uint32 + Nreloc uint32 + Flags uint32 + Reserve1 uint32 + Reserve2 uint32 + Reserve3 uint32 +} + +// A SymtabCmd is a Mach-O symbol table command. +type SymtabCmd struct { + Cmd LoadCmd + Len uint32 + Symoff uint32 + Nsyms uint32 + Stroff uint32 + Strsize uint32 +} + +// A DysymtabCmd is a Mach-O dynamic symbol table command. +type DysymtabCmd struct { + Cmd LoadCmd + Len uint32 + Ilocalsym uint32 + Nlocalsym uint32 + Iextdefsym uint32 + Nextdefsym uint32 + Iundefsym uint32 + Nundefsym uint32 + Tocoffset uint32 + Ntoc uint32 + Modtaboff uint32 + Nmodtab uint32 + Extrefsymoff uint32 + Nextrefsyms uint32 + Indirectsymoff uint32 + Nindirectsyms uint32 + Extreloff uint32 + Nextrel uint32 + Locreloff uint32 + Nlocrel uint32 +} + +// An Nlist32 is a Mach-O 32-bit symbol table entry. +type Nlist32 struct { + Name uint32 + Type uint8 + Sect uint8 + Desc uint16 + Value uint32 +} + +// An Nlist64 is a Mach-O 64-bit symbol table entry. +type Nlist64 struct { + Name uint32 + Type uint8 + Sect uint8 + Desc uint16 + Value uint64 +} + +// A Symbol is a Mach-O 32-bit or 64-bit symbol table entry. +type Symbol struct { + Name string + Type uint8 + Sect uint8 + Desc uint16 + Value uint64 +} + +// A Thread is a Mach-O thread state command. +type Thread struct { + Cmd LoadCmd + Len uint32 + Type uint32 + Data []uint32 +} + +// Regs386 is the Mach-O 386 register structure. +type Regs386 struct { + AX uint32 + BX uint32 + CX uint32 + DX uint32 + DI uint32 + SI uint32 + BP uint32 + SP uint32 + SS uint32 + FLAGS uint32 + IP uint32 + CS uint32 + DS uint32 + ES uint32 + FS uint32 + GS uint32 +} + +// RegsAMD64 is the Mach-O AMD64 register structure. +type RegsAMD64 struct { + AX uint64 + BX uint64 + CX uint64 + DX uint64 + DI uint64 + SI uint64 + BP uint64 + SP uint64 + R8 uint64 + R9 uint64 + R10 uint64 + R11 uint64 + R12 uint64 + R13 uint64 + R14 uint64 + R15 uint64 + IP uint64 + FLAGS uint64 + CS uint64 + FS uint64 + GS uint64 +} + +type intName struct { + i uint32 + s string +} + +func stringName(i uint32, names []intName, goSyntax bool) string { + for _, n := range names { + if n.i == i { + if goSyntax { + return "macho." + n.s + } + return n.s + } + } + return strconv.Uitoa64(uint64(i)) +} + +func flagName(i uint32, names []intName, goSyntax bool) string { + s := "" + for _, n := range names { + if n.i&i == n.i { + if len(s) > 0 { + s += "+" + } + if goSyntax { + s += "macho." + } + s += n.s + i -= n.i + } + } + if len(s) == 0 { + return "0x" + strconv.Uitob64(uint64(i), 16) + } + if i != 0 { + s += "+0x" + strconv.Uitob64(uint64(i), 16) + } + return s +} diff --git a/src/pkg/debug/macho/testdata/gcc-386-darwin-exec b/src/pkg/debug/macho/testdata/gcc-386-darwin-exec new file mode 100755 index 000000000..03ba1bafa Binary files /dev/null and b/src/pkg/debug/macho/testdata/gcc-386-darwin-exec differ diff --git a/src/pkg/debug/macho/testdata/gcc-amd64-darwin-exec b/src/pkg/debug/macho/testdata/gcc-amd64-darwin-exec new file mode 100755 index 000000000..5155a5a26 Binary files /dev/null and b/src/pkg/debug/macho/testdata/gcc-amd64-darwin-exec differ diff --git a/src/pkg/debug/macho/testdata/gcc-amd64-darwin-exec-debug b/src/pkg/debug/macho/testdata/gcc-amd64-darwin-exec-debug new file mode 100644 index 000000000..a47d3aef7 Binary files /dev/null and b/src/pkg/debug/macho/testdata/gcc-amd64-darwin-exec-debug differ diff --git a/src/pkg/debug/macho/testdata/hello.c b/src/pkg/debug/macho/testdata/hello.c new file mode 100644 index 000000000..a689d3644 --- /dev/null +++ b/src/pkg/debug/macho/testdata/hello.c @@ -0,0 +1,8 @@ +#include + +int +main(void) +{ + printf("hello, world\n"); + return 0; +} diff --git a/src/pkg/debug/pe/Makefile b/src/pkg/debug/pe/Makefile new file mode 100644 index 000000000..998e6a418 --- /dev/null +++ b/src/pkg/debug/pe/Makefile @@ -0,0 +1,12 @@ +# Copyright 2009 The Go Authors. All rights reserved. +# Use of this source code is governed by a BSD-style +# license that can be found in the LICENSE file. + +include ../../../Make.inc + +TARG=debug/pe +GOFILES=\ + pe.go\ + file.go\ + +include ../../../Make.pkg diff --git a/src/pkg/debug/pe/file.go b/src/pkg/debug/pe/file.go new file mode 100644 index 000000000..d86d916f5 --- /dev/null +++ b/src/pkg/debug/pe/file.go @@ -0,0 +1,315 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Package pe implements access to PE (Microsoft Windows Portable Executable) files. +package pe + +import ( + "debug/dwarf" + "encoding/binary" + "fmt" + "io" + "os" + "strconv" +) + +// A File represents an open PE file. +type File struct { + FileHeader + Sections []*Section + + closer io.Closer +} + +type SectionHeader struct { + Name string + VirtualSize uint32 + VirtualAddress uint32 + Size uint32 + Offset uint32 + PointerToRelocations uint32 + PointerToLineNumbers uint32 + NumberOfRelocations uint16 + NumberOfLineNumbers uint16 + Characteristics uint32 +} + +type Section struct { + SectionHeader + + // Embed ReaderAt for ReadAt method. + // Do not embed SectionReader directly + // to avoid having Read and Seek. + // If a client wants Read and Seek it must use + // Open() to avoid fighting over the seek offset + // with other clients. + io.ReaderAt + sr *io.SectionReader +} + +type ImportDirectory struct { + OriginalFirstThunk uint32 + TimeDateStamp uint32 + ForwarderChain uint32 + Name uint32 + FirstThunk uint32 + + dll string +} + +// Data reads and returns the contents of the PE section. +func (s *Section) Data() ([]byte, os.Error) { + dat := make([]byte, s.sr.Size()) + n, err := s.sr.ReadAt(dat, 0) + return dat[0:n], err +} + +// Open returns a new ReadSeeker reading the PE section. +func (s *Section) Open() io.ReadSeeker { return io.NewSectionReader(s.sr, 0, 1<<63-1) } + +type FormatError struct { + off int64 + msg string + val interface{} +} + +func (e *FormatError) String() string { + msg := e.msg + if e.val != nil { + msg += fmt.Sprintf(" '%v'", e.val) + } + msg += fmt.Sprintf(" in record at byte %#x", e.off) + return msg +} + +// Open opens the named file using os.Open and prepares it for use as a PE binary. +func Open(name string) (*File, os.Error) { + f, err := os.Open(name) + if err != nil { + return nil, err + } + ff, err := NewFile(f) + if err != nil { + f.Close() + return nil, err + } + ff.closer = f + return ff, nil +} + +// Close closes the File. +// If the File was created using NewFile directly instead of Open, +// Close has no effect. +func (f *File) Close() os.Error { + var err os.Error + if f.closer != nil { + err = f.closer.Close() + f.closer = nil + } + return err +} + +// NewFile creates a new File for accessing a PE binary in an underlying reader. +func NewFile(r io.ReaderAt) (*File, os.Error) { + f := new(File) + sr := io.NewSectionReader(r, 0, 1<<63-1) + + var dosheader [96]byte + if _, err := r.ReadAt(dosheader[0:], 0); err != nil { + return nil, err + } + var base int64 + if dosheader[0] == 'M' && dosheader[1] == 'Z' { + var sign [4]byte + r.ReadAt(sign[0:], int64(dosheader[0x3c])) + if !(sign[0] == 'P' && sign[1] == 'E' && sign[2] == 0 && sign[3] == 0) { + return nil, os.NewError("Invalid PE File Format.") + } + base = int64(dosheader[0x3c]) + 4 + } else { + base = int64(0) + } + sr.Seek(base, os.SEEK_SET) + if err := binary.Read(sr, binary.LittleEndian, &f.FileHeader); err != nil { + return nil, err + } + if f.FileHeader.Machine != IMAGE_FILE_MACHINE_UNKNOWN && f.FileHeader.Machine != IMAGE_FILE_MACHINE_AMD64 && f.FileHeader.Machine != IMAGE_FILE_MACHINE_I386 { + return nil, os.NewError("Invalid PE File Format.") + } + // get symbol string table + sr.Seek(int64(f.FileHeader.PointerToSymbolTable+18*f.FileHeader.NumberOfSymbols), os.SEEK_SET) + var l uint32 + if err := binary.Read(sr, binary.LittleEndian, &l); err != nil { + return nil, err + } + ss := make([]byte, l) + if _, err := r.ReadAt(ss, int64(f.FileHeader.PointerToSymbolTable+18*f.FileHeader.NumberOfSymbols)); err != nil { + return nil, err + } + sr.Seek(base, os.SEEK_SET) + binary.Read(sr, binary.LittleEndian, &f.FileHeader) + sr.Seek(int64(f.FileHeader.SizeOfOptionalHeader), os.SEEK_CUR) //Skip OptionalHeader + f.Sections = make([]*Section, f.FileHeader.NumberOfSections) + for i := 0; i < int(f.FileHeader.NumberOfSections); i++ { + sh := new(SectionHeader32) + if err := binary.Read(sr, binary.LittleEndian, sh); err != nil { + return nil, err + } + var name string + if sh.Name[0] == '\x2F' { + si, _ := strconv.Atoi(cstring(sh.Name[1:])) + name, _ = getString(ss, si) + } else { + name = cstring(sh.Name[0:]) + } + s := new(Section) + s.SectionHeader = SectionHeader{ + Name: name, + VirtualSize: uint32(sh.VirtualSize), + VirtualAddress: uint32(sh.VirtualAddress), + Size: uint32(sh.SizeOfRawData), + Offset: uint32(sh.PointerToRawData), + PointerToRelocations: uint32(sh.PointerToRelocations), + PointerToLineNumbers: uint32(sh.PointerToLineNumbers), + NumberOfRelocations: uint16(sh.NumberOfRelocations), + NumberOfLineNumbers: uint16(sh.NumberOfLineNumbers), + Characteristics: uint32(sh.Characteristics), + } + s.sr = io.NewSectionReader(r, int64(s.SectionHeader.Offset), int64(s.SectionHeader.Size)) + s.ReaderAt = s.sr + f.Sections[i] = s + } + return f, nil +} + +func cstring(b []byte) string { + var i int + for i = 0; i < len(b) && b[i] != 0; i++ { + } + return string(b[0:i]) +} + +// getString extracts a string from symbol string table. +func getString(section []byte, start int) (string, bool) { + if start < 0 || start >= len(section) { + return "", false + } + + for end := start; end < len(section); end++ { + if section[end] == 0 { + return string(section[start:end]), true + } + } + return "", false +} + +// Section returns the first section with the given name, or nil if no such +// section exists. +func (f *File) Section(name string) *Section { + for _, s := range f.Sections { + if s.Name == name { + return s + } + } + return nil +} + +func (f *File) DWARF() (*dwarf.Data, os.Error) { + // There are many other DWARF sections, but these + // are the required ones, and the debug/dwarf package + // does not use the others, so don't bother loading them. + var names = [...]string{"abbrev", "info", "str"} + var dat [len(names)][]byte + for i, name := range names { + name = ".debug_" + name + s := f.Section(name) + if s == nil { + continue + } + b, err := s.Data() + if err != nil && uint32(len(b)) < s.Size { + return nil, err + } + dat[i] = b + } + + abbrev, info, str := dat[0], dat[1], dat[2] + return dwarf.New(abbrev, nil, nil, info, nil, nil, nil, str) +} + +// ImportedSymbols returns the names of all symbols +// referred to by the binary f that are expected to be +// satisfied by other libraries at dynamic load time. +// It does not return weak symbols. +func (f *File) ImportedSymbols() ([]string, os.Error) { + pe64 := f.Machine == IMAGE_FILE_MACHINE_AMD64 + ds := f.Section(".idata") + if ds == nil { + // not dynamic, so no libraries + return nil, nil + } + d, err := ds.Data() + if err != nil { + return nil, err + } + var ida []ImportDirectory + for len(d) > 0 { + var dt ImportDirectory + dt.OriginalFirstThunk = binary.LittleEndian.Uint32(d[0:4]) + dt.Name = binary.LittleEndian.Uint32(d[12:16]) + dt.FirstThunk = binary.LittleEndian.Uint32(d[16:20]) + d = d[20:] + if dt.OriginalFirstThunk == 0 { + break + } + ida = append(ida, dt) + } + names, _ := ds.Data() + var all []string + for _, dt := range ida { + dt.dll, _ = getString(names, int(dt.Name-ds.VirtualAddress)) + d, _ = ds.Data() + // seek to OriginalFirstThunk + d = d[dt.OriginalFirstThunk-ds.VirtualAddress:] + for len(d) > 0 { + if pe64 { // 64bit + va := binary.LittleEndian.Uint64(d[0:8]) + d = d[8:] + if va == 0 { + break + } + if va&0x8000000000000000 > 0 { // is Ordinal + // TODO add dynimport ordinal support. + } else { + fn, _ := getString(names, int(uint32(va)-ds.VirtualAddress+2)) + all = append(all, fn+":"+dt.dll) + } + } else { // 32bit + va := binary.LittleEndian.Uint32(d[0:4]) + d = d[4:] + if va == 0 { + break + } + if va&0x80000000 > 0 { // is Ordinal + // TODO add dynimport ordinal support. + //ord := va&0x0000FFFF + } else { + fn, _ := getString(names, int(va-ds.VirtualAddress+2)) + all = append(all, fn+":"+dt.dll) + } + } + } + } + + return all, nil +} + +// ImportedLibraries returns the names of all libraries +// referred to by the binary f that are expected to be +// linked with the binary at dynamic link time. +func (f *File) ImportedLibraries() ([]string, os.Error) { + // TODO + // cgo -dynimport don't use this for windows PE, so just return. + return nil, nil +} diff --git a/src/pkg/debug/pe/file_test.go b/src/pkg/debug/pe/file_test.go new file mode 100644 index 000000000..2c5c25b8c --- /dev/null +++ b/src/pkg/debug/pe/file_test.go @@ -0,0 +1,99 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package pe + +import ( + "reflect" + "testing" +) + +type fileTest struct { + file string + hdr FileHeader + sections []*SectionHeader +} + +var fileTests = []fileTest{ + { + "testdata/gcc-386-mingw-obj", + FileHeader{0x014c, 0x000c, 0x0, 0x64a, 0x1e, 0x0, 0x104}, + []*SectionHeader{ + &SectionHeader{".text", 0, 0, 36, 500, 1440, 0, 3, 0, 0x60300020}, + &SectionHeader{".data", 0, 0, 0, 0, 0, 0, 0, 0, 3224371264}, + &SectionHeader{".bss", 0, 0, 0, 0, 0, 0, 0, 0, 3224371328}, + &SectionHeader{".debug_abbrev", 0, 0, 137, 536, 0, 0, 0, 0, 0x42100000}, + &SectionHeader{".debug_info", 0, 0, 418, 673, 1470, 0, 7, 0, 1108344832}, + &SectionHeader{".debug_line", 0, 0, 128, 1091, 1540, 0, 1, 0, 1108344832}, + &SectionHeader{".rdata", 0, 0, 16, 1219, 0, 0, 0, 0, 1076887616}, + &SectionHeader{".debug_frame", 0, 0, 52, 1235, 1550, 0, 2, 0, 1110441984}, + &SectionHeader{".debug_loc", 0, 0, 56, 1287, 0, 0, 0, 0, 1108344832}, + &SectionHeader{".debug_pubnames", 0, 0, 27, 1343, 1570, 0, 1, 0, 1108344832}, + &SectionHeader{".debug_pubtypes", 0, 0, 38, 1370, 1580, 0, 1, 0, 1108344832}, + &SectionHeader{".debug_aranges", 0, 0, 32, 1408, 1590, 0, 2, 0, 1108344832}, + }, + }, + { + "testdata/gcc-386-mingw-exec", + FileHeader{0x014c, 0x000f, 0x4c6a1b60, 0x3c00, 0x282, 0xe0, 0x107}, + []*SectionHeader{ + &SectionHeader{Name: ".text", VirtualSize: 0xcd8, VirtualAddress: 0x1000, Size: 0xe00, Offset: 0x400, PointerToRelocations: 0x0, PointerToLineNumbers: 0x0, NumberOfRelocations: 0x0, NumberOfLineNumbers: 0x0, Characteristics: 0x60500060}, + &SectionHeader{Name: ".data", VirtualSize: 0x10, VirtualAddress: 0x2000, Size: 0x200, Offset: 0x1200, PointerToRelocations: 0x0, PointerToLineNumbers: 0x0, NumberOfRelocations: 0x0, NumberOfLineNumbers: 0x0, Characteristics: 0xc0300040}, + &SectionHeader{Name: ".rdata", VirtualSize: 0x120, VirtualAddress: 0x3000, Size: 0x200, Offset: 0x1400, PointerToRelocations: 0x0, PointerToLineNumbers: 0x0, NumberOfRelocations: 0x0, NumberOfLineNumbers: 0x0, Characteristics: 0x40300040}, + &SectionHeader{Name: ".bss", VirtualSize: 0xdc, VirtualAddress: 0x4000, Size: 0x0, Offset: 0x0, PointerToRelocations: 0x0, PointerToLineNumbers: 0x0, NumberOfRelocations: 0x0, NumberOfLineNumbers: 0x0, Characteristics: 0xc0400080}, + &SectionHeader{Name: ".idata", VirtualSize: 0x3c8, VirtualAddress: 0x5000, Size: 0x400, Offset: 0x1600, PointerToRelocations: 0x0, PointerToLineNumbers: 0x0, NumberOfRelocations: 0x0, NumberOfLineNumbers: 0x0, Characteristics: 0xc0300040}, + &SectionHeader{Name: ".CRT", VirtualSize: 0x18, VirtualAddress: 0x6000, Size: 0x200, Offset: 0x1a00, PointerToRelocations: 0x0, PointerToLineNumbers: 0x0, NumberOfRelocations: 0x0, NumberOfLineNumbers: 0x0, Characteristics: 0xc0300040}, + &SectionHeader{Name: ".tls", VirtualSize: 0x20, VirtualAddress: 0x7000, Size: 0x200, Offset: 0x1c00, PointerToRelocations: 0x0, PointerToLineNumbers: 0x0, NumberOfRelocations: 0x0, NumberOfLineNumbers: 0x0, Characteristics: 0xc0300040}, + &SectionHeader{Name: ".debug_aranges", VirtualSize: 0x20, VirtualAddress: 0x8000, Size: 0x200, Offset: 0x1e00, PointerToRelocations: 0x0, PointerToLineNumbers: 0x0, NumberOfRelocations: 0x0, NumberOfLineNumbers: 0x0, Characteristics: 0x42100000}, + &SectionHeader{Name: ".debug_pubnames", VirtualSize: 0x51, VirtualAddress: 0x9000, Size: 0x200, Offset: 0x2000, PointerToRelocations: 0x0, PointerToLineNumbers: 0x0, NumberOfRelocations: 0x0, NumberOfLineNumbers: 0x0, Characteristics: 0x42100000}, + &SectionHeader{Name: ".debug_pubtypes", VirtualSize: 0x91, VirtualAddress: 0xa000, Size: 0x200, Offset: 0x2200, PointerToRelocations: 0x0, PointerToLineNumbers: 0x0, NumberOfRelocations: 0x0, NumberOfLineNumbers: 0x0, Characteristics: 0x42100000}, + &SectionHeader{Name: ".debug_info", VirtualSize: 0xe22, VirtualAddress: 0xb000, Size: 0x1000, Offset: 0x2400, PointerToRelocations: 0x0, PointerToLineNumbers: 0x0, NumberOfRelocations: 0x0, NumberOfLineNumbers: 0x0, Characteristics: 0x42100000}, + &SectionHeader{Name: ".debug_abbrev", VirtualSize: 0x157, VirtualAddress: 0xc000, Size: 0x200, Offset: 0x3400, PointerToRelocations: 0x0, PointerToLineNumbers: 0x0, NumberOfRelocations: 0x0, NumberOfLineNumbers: 0x0, Characteristics: 0x42100000}, + &SectionHeader{Name: ".debug_line", VirtualSize: 0x144, VirtualAddress: 0xd000, Size: 0x200, Offset: 0x3600, PointerToRelocations: 0x0, PointerToLineNumbers: 0x0, NumberOfRelocations: 0x0, NumberOfLineNumbers: 0x0, Characteristics: 0x42100000}, + &SectionHeader{Name: ".debug_frame", VirtualSize: 0x34, VirtualAddress: 0xe000, Size: 0x200, Offset: 0x3800, PointerToRelocations: 0x0, PointerToLineNumbers: 0x0, NumberOfRelocations: 0x0, NumberOfLineNumbers: 0x0, Characteristics: 0x42300000}, + &SectionHeader{Name: ".debug_loc", VirtualSize: 0x38, VirtualAddress: 0xf000, Size: 0x200, Offset: 0x3a00, PointerToRelocations: 0x0, PointerToLineNumbers: 0x0, NumberOfRelocations: 0x0, NumberOfLineNumbers: 0x0, Characteristics: 0x42100000}, + }, + }, +} + +func TestOpen(t *testing.T) { + for i := range fileTests { + tt := &fileTests[i] + + f, err := Open(tt.file) + if err != nil { + t.Error(err) + continue + } + if !reflect.DeepEqual(f.FileHeader, tt.hdr) { + t.Errorf("open %s:\n\thave %#v\n\twant %#v\n", tt.file, f.FileHeader, tt.hdr) + continue + } + + for i, sh := range f.Sections { + if i >= len(tt.sections) { + break + } + have := &sh.SectionHeader + want := tt.sections[i] + if !reflect.DeepEqual(have, want) { + t.Errorf("open %s, section %d:\n\thave %#v\n\twant %#v\n", tt.file, i, have, want) + } + } + tn := len(tt.sections) + fn := len(f.Sections) + if tn != fn { + t.Errorf("open %s: len(Sections) = %d, want %d", tt.file, fn, tn) + } + + } +} + +func TestOpenFailure(t *testing.T) { + filename := "file.go" // not a PE file + _, err := Open(filename) // don't crash + if err == nil { + t.Errorf("open %s: succeeded unexpectedly", filename) + } +} diff --git a/src/pkg/debug/pe/pe.go b/src/pkg/debug/pe/pe.go new file mode 100644 index 000000000..b3dab739a --- /dev/null +++ b/src/pkg/debug/pe/pe.go @@ -0,0 +1,51 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package pe + +type FileHeader struct { + Machine uint16 + NumberOfSections uint16 + TimeDateStamp uint32 + PointerToSymbolTable uint32 + NumberOfSymbols uint32 + SizeOfOptionalHeader uint16 + Characteristics uint16 +} + +type SectionHeader32 struct { + Name [8]uint8 + VirtualSize uint32 + VirtualAddress uint32 + SizeOfRawData uint32 + PointerToRawData uint32 + PointerToRelocations uint32 + PointerToLineNumbers uint32 + NumberOfRelocations uint16 + NumberOfLineNumbers uint16 + Characteristics uint32 +} + +const ( + IMAGE_FILE_MACHINE_UNKNOWN = 0x0 + IMAGE_FILE_MACHINE_AM33 = 0x1d3 + IMAGE_FILE_MACHINE_AMD64 = 0x8664 + IMAGE_FILE_MACHINE_ARM = 0x1c0 + IMAGE_FILE_MACHINE_EBC = 0xebc + IMAGE_FILE_MACHINE_I386 = 0x14c + IMAGE_FILE_MACHINE_IA64 = 0x200 + IMAGE_FILE_MACHINE_M32R = 0x9041 + IMAGE_FILE_MACHINE_MIPS16 = 0x266 + IMAGE_FILE_MACHINE_MIPSFPU = 0x366 + IMAGE_FILE_MACHINE_MIPSFPU16 = 0x466 + IMAGE_FILE_MACHINE_POWERPC = 0x1f0 + IMAGE_FILE_MACHINE_POWERPCFP = 0x1f1 + IMAGE_FILE_MACHINE_R4000 = 0x166 + IMAGE_FILE_MACHINE_SH3 = 0x1a2 + IMAGE_FILE_MACHINE_SH3DSP = 0x1a3 + IMAGE_FILE_MACHINE_SH4 = 0x1a6 + IMAGE_FILE_MACHINE_SH5 = 0x1a8 + IMAGE_FILE_MACHINE_THUMB = 0x1c2 + IMAGE_FILE_MACHINE_WCEMIPSV2 = 0x169 +) diff --git a/src/pkg/debug/pe/testdata/gcc-386-mingw-exec b/src/pkg/debug/pe/testdata/gcc-386-mingw-exec new file mode 100644 index 000000000..4b808d043 Binary files /dev/null and b/src/pkg/debug/pe/testdata/gcc-386-mingw-exec differ diff --git a/src/pkg/debug/pe/testdata/gcc-386-mingw-obj b/src/pkg/debug/pe/testdata/gcc-386-mingw-obj new file mode 100644 index 000000000..0c84d898d Binary files /dev/null and b/src/pkg/debug/pe/testdata/gcc-386-mingw-obj differ diff --git a/src/pkg/debug/pe/testdata/hello.c b/src/pkg/debug/pe/testdata/hello.c new file mode 100644 index 000000000..a689d3644 --- /dev/null +++ b/src/pkg/debug/pe/testdata/hello.c @@ -0,0 +1,8 @@ +#include + +int +main(void) +{ + printf("hello, world\n"); + return 0; +} diff --git a/src/pkg/deps.bash b/src/pkg/deps.bash new file mode 100755 index 000000000..2095ec1d8 --- /dev/null +++ b/src/pkg/deps.bash @@ -0,0 +1,49 @@ +#!/usr/bin/env bash +# Copyright 2009 The Go Authors. All rights reserved. +# Use of this source code is governed by a BSD-style +# license that can be found in the LICENSE file. + +eval $(gomake --no-print-directory -f ../Make.inc go-env) + +OUT="Make.deps" +TMP="Make.deps.tmp" + +if [ -f $OUT ] && ! [ -w $OUT ]; then + echo "$0: $OUT is read-only; aborting." 1>&2 + exit 1 +fi + +# Get list of directories from Makefile +dirs=$(gomake --no-print-directory echo-dirs) +dirpat=$(echo $dirs C | awk '{ + for(i=1;i<=NF;i++){ + x=$i + gsub("/", "\\/", x) + printf("/^(%s)$/\n", x) + } +}') + +for dir in $dirs; do ( + cd $dir || exit 1 + + sources=$(sed -n 's/^[ ]*\([^ ]*\.go\)[ ]*\\*[ ]*$/\1/p' Makefile) + sources=$(echo $sources | sed 's/\$(GOOS)/'$GOOS'/g') + sources=$(echo $sources | sed 's/\$(GOARCH)/'$GOARCH'/g') + # /dev/null here means we get an empty dependency list if $sources is empty + # instead of listing every file in the directory. + sources=$(ls $sources /dev/null 2> /dev/null) # remove .s, .c, etc. + + deps=$( + sed -n '/^import.*"/p; /^import[ \t]*(/,/^)/p' $sources /dev/null | + cut -d '"' -f2 | + awk "$dirpat" | + grep -v "^$dir\$" | + sed 's/$/.install/' | + sed 's;^C\.install;runtime/cgo.install;' | + sort -u + ) + + echo $dir.install: $deps +) done > $TMP + +mv $TMP $OUT diff --git a/src/pkg/ebnf/Makefile b/src/pkg/ebnf/Makefile new file mode 100644 index 000000000..f5555d272 --- /dev/null +++ b/src/pkg/ebnf/Makefile @@ -0,0 +1,12 @@ +# Copyright 2009 The Go Authors. All rights reserved. +# Use of this source code is governed by a BSD-style +# license that can be found in the LICENSE file. + +include ../../Make.inc + +TARG=ebnf +GOFILES=\ + ebnf.go\ + parser.go\ + +include ../../Make.pkg diff --git a/src/pkg/ebnf/ebnf.go b/src/pkg/ebnf/ebnf.go new file mode 100644 index 000000000..69da11767 --- /dev/null +++ b/src/pkg/ebnf/ebnf.go @@ -0,0 +1,245 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Package ebnf is a library for EBNF grammars. The input is text ([]byte) +// satisfying the following grammar (represented itself in EBNF): +// +// Production = name "=" [ Expression ] "." . +// Expression = Alternative { "|" Alternative } . +// Alternative = Term { Term } . +// Term = name | token [ "…" token ] | Group | Option | Repetition . +// Group = "(" Expression ")" . +// Option = "[" Expression "]" . +// Repetition = "{" Expression "}" . +// +// A name is a Go identifier, a token is a Go string, and comments +// and white space follow the same rules as for the Go language. +// Production names starting with an uppercase Unicode letter denote +// non-terminal productions (i.e., productions which allow white-space +// and comments between tokens); all other production names denote +// lexical productions. +// +package ebnf + +import ( + "go/scanner" + "go/token" + "os" + "unicode" + "utf8" +) + +// ---------------------------------------------------------------------------- +// Internal representation + +type ( + // An Expression node represents a production expression. + Expression interface { + // Pos is the position of the first character of the syntactic construct + Pos() token.Pos + } + + // An Alternative node represents a non-empty list of alternative expressions. + Alternative []Expression // x | y | z + + // A Sequence node represents a non-empty list of sequential expressions. + Sequence []Expression // x y z + + // A Name node represents a production name. + Name struct { + StringPos token.Pos + String string + } + + // A Token node represents a literal. + Token struct { + StringPos token.Pos + String string + } + + // A List node represents a range of characters. + Range struct { + Begin, End *Token // begin ... end + } + + // A Group node represents a grouped expression. + Group struct { + Lparen token.Pos + Body Expression // (body) + } + + // An Option node represents an optional expression. + Option struct { + Lbrack token.Pos + Body Expression // [body] + } + + // A Repetition node represents a repeated expression. + Repetition struct { + Lbrace token.Pos + Body Expression // {body} + } + + // A Bad node stands for pieces of source code that lead to a parse error. + Bad struct { + TokPos token.Pos + Error string // parser error message + } + + // A Production node represents an EBNF production. + Production struct { + Name *Name + Expr Expression + } + + // A Grammar is a set of EBNF productions. The map + // is indexed by production name. + // + Grammar map[string]*Production +) + +func (x Alternative) Pos() token.Pos { return x[0].Pos() } // the parser always generates non-empty Alternative +func (x Sequence) Pos() token.Pos { return x[0].Pos() } // the parser always generates non-empty Sequences +func (x *Name) Pos() token.Pos { return x.StringPos } +func (x *Token) Pos() token.Pos { return x.StringPos } +func (x *Range) Pos() token.Pos { return x.Begin.Pos() } +func (x *Group) Pos() token.Pos { return x.Lparen } +func (x *Option) Pos() token.Pos { return x.Lbrack } +func (x *Repetition) Pos() token.Pos { return x.Lbrace } +func (x *Bad) Pos() token.Pos { return x.TokPos } +func (x *Production) Pos() token.Pos { return x.Name.Pos() } + +// ---------------------------------------------------------------------------- +// Grammar verification + +func isLexical(name string) bool { + ch, _ := utf8.DecodeRuneInString(name) + return !unicode.IsUpper(ch) +} + +type verifier struct { + fset *token.FileSet + scanner.ErrorVector + worklist []*Production + reached Grammar // set of productions reached from (and including) the root production + grammar Grammar +} + +func (v *verifier) error(pos token.Pos, msg string) { + v.Error(v.fset.Position(pos), msg) +} + +func (v *verifier) push(prod *Production) { + name := prod.Name.String + if _, found := v.reached[name]; !found { + v.worklist = append(v.worklist, prod) + v.reached[name] = prod + } +} + +func (v *verifier) verifyChar(x *Token) int { + s := x.String + if utf8.RuneCountInString(s) != 1 { + v.error(x.Pos(), "single char expected, found "+s) + return 0 + } + ch, _ := utf8.DecodeRuneInString(s) + return ch +} + +func (v *verifier) verifyExpr(expr Expression, lexical bool) { + switch x := expr.(type) { + case nil: + // empty expression + case Alternative: + for _, e := range x { + v.verifyExpr(e, lexical) + } + case Sequence: + for _, e := range x { + v.verifyExpr(e, lexical) + } + case *Name: + // a production with this name must exist; + // add it to the worklist if not yet processed + if prod, found := v.grammar[x.String]; found { + v.push(prod) + } else { + v.error(x.Pos(), "missing production "+x.String) + } + // within a lexical production references + // to non-lexical productions are invalid + if lexical && !isLexical(x.String) { + v.error(x.Pos(), "reference to non-lexical production "+x.String) + } + case *Token: + // nothing to do for now + case *Range: + i := v.verifyChar(x.Begin) + j := v.verifyChar(x.End) + if i >= j { + v.error(x.Pos(), "decreasing character range") + } + case *Group: + v.verifyExpr(x.Body, lexical) + case *Option: + v.verifyExpr(x.Body, lexical) + case *Repetition: + v.verifyExpr(x.Body, lexical) + default: + panic("unreachable") + } +} + +func (v *verifier) verify(fset *token.FileSet, grammar Grammar, start string) { + // find root production + root, found := grammar[start] + if !found { + // token.NoPos doesn't require a file set; + // ok to set v.fset only afterwards + v.error(token.NoPos, "no start production "+start) + return + } + + // initialize verifier + v.fset = fset + v.ErrorVector.Reset() + v.worklist = v.worklist[0:0] + v.reached = make(Grammar) + v.grammar = grammar + + // work through the worklist + v.push(root) + for { + n := len(v.worklist) - 1 + if n < 0 { + break + } + prod := v.worklist[n] + v.worklist = v.worklist[0:n] + v.verifyExpr(prod.Expr, isLexical(prod.Name.String)) + } + + // check if all productions were reached + if len(v.reached) < len(v.grammar) { + for name, prod := range v.grammar { + if _, found := v.reached[name]; !found { + v.error(prod.Pos(), name+" is unreachable") + } + } + } +} + +// Verify checks that: +// - all productions used are defined +// - all productions defined are used when beginning at start +// - lexical productions refer only to other lexical productions +// +// Position information is interpreted relative to the file set fset. +// +func Verify(fset *token.FileSet, grammar Grammar, start string) os.Error { + var v verifier + v.verify(fset, grammar, start) + return v.GetError(scanner.Sorted) +} diff --git a/src/pkg/ebnf/ebnf_test.go b/src/pkg/ebnf/ebnf_test.go new file mode 100644 index 000000000..b086facc3 --- /dev/null +++ b/src/pkg/ebnf/ebnf_test.go @@ -0,0 +1,87 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package ebnf + +import ( + "go/token" + "io/ioutil" + "testing" +) + +var fset = token.NewFileSet() + +var goodGrammars = []string{ + `Program = .`, + + `Program = foo . + foo = "foo" .`, + + `Program = "a" | "b" "c" .`, + + `Program = "a" … "z" .`, + + `Program = Song . + Song = { Note } . + Note = Do | (Re | Mi | Fa | So | La) | Ti . + Do = "c" . + Re = "d" . + Mi = "e" . + Fa = "f" . + So = "g" . + La = "a" . + Ti = ti . + ti = "b" .`, +} + +var badGrammars = []string{ + `Program = | .`, + `Program = | b .`, + `Program = a … b .`, + `Program = "a" … .`, + `Program = … "b" .`, + `Program = () .`, + `Program = [] .`, + `Program = {} .`, +} + +func checkGood(t *testing.T, filename string, src []byte) { + grammar, err := Parse(fset, filename, src) + if err != nil { + t.Errorf("Parse(%s) failed: %v", src, err) + } + if err = Verify(fset, grammar, "Program"); err != nil { + t.Errorf("Verify(%s) failed: %v", src, err) + } +} + +func checkBad(t *testing.T, filename string, src []byte) { + _, err := Parse(fset, filename, src) + if err == nil { + t.Errorf("Parse(%s) should have failed", src) + } +} + +func TestGrammars(t *testing.T) { + for _, src := range goodGrammars { + checkGood(t, "", []byte(src)) + } + for _, src := range badGrammars { + checkBad(t, "", []byte(src)) + } +} + +var files = []string{ +// TODO(gri) add some test files +} + +func TestFiles(t *testing.T) { + for _, filename := range files { + src, err := ioutil.ReadFile(filename) + if err != nil { + t.Fatal(err) + } + checkGood(t, filename, src) + } +} diff --git a/src/pkg/ebnf/parser.go b/src/pkg/ebnf/parser.go new file mode 100644 index 000000000..ef2fac000 --- /dev/null +++ b/src/pkg/ebnf/parser.go @@ -0,0 +1,197 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package ebnf + +import ( + "go/scanner" + "go/token" + "os" + "strconv" +) + +type parser struct { + fset *token.FileSet + scanner.ErrorVector + scanner scanner.Scanner + pos token.Pos // token position + tok token.Token // one token look-ahead + lit string // token literal +} + +func (p *parser) next() { + p.pos, p.tok, p.lit = p.scanner.Scan() + if p.tok.IsKeyword() { + // TODO Should keyword mapping always happen outside scanner? + // Or should there be a flag to scanner to enable keyword mapping? + p.tok = token.IDENT + } +} + +func (p *parser) error(pos token.Pos, msg string) { + p.Error(p.fset.Position(pos), msg) +} + +func (p *parser) errorExpected(pos token.Pos, msg string) { + msg = "expected " + msg + if pos == p.pos { + // the error happened at the current position; + // make the error message more specific + msg += ", found '" + p.tok.String() + "'" + if p.tok.IsLiteral() { + msg += " " + p.lit + } + } + p.error(pos, msg) +} + +func (p *parser) expect(tok token.Token) token.Pos { + pos := p.pos + if p.tok != tok { + p.errorExpected(pos, "'"+tok.String()+"'") + } + p.next() // make progress in any case + return pos +} + +func (p *parser) parseIdentifier() *Name { + pos := p.pos + name := p.lit + p.expect(token.IDENT) + return &Name{pos, name} +} + +func (p *parser) parseToken() *Token { + pos := p.pos + value := "" + if p.tok == token.STRING { + value, _ = strconv.Unquote(p.lit) + // Unquote may fail with an error, but only if the scanner found + // an illegal string in the first place. In this case the error + // has already been reported. + p.next() + } else { + p.expect(token.STRING) + } + return &Token{pos, value} +} + +// ParseTerm returns nil if no term was found. +func (p *parser) parseTerm() (x Expression) { + pos := p.pos + + switch p.tok { + case token.IDENT: + x = p.parseIdentifier() + + case token.STRING: + tok := p.parseToken() + x = tok + const ellipsis = "…" // U+2026, the horizontal ellipsis character + if p.tok == token.ILLEGAL && p.lit == ellipsis { + p.next() + x = &Range{tok, p.parseToken()} + } + + case token.LPAREN: + p.next() + x = &Group{pos, p.parseExpression()} + p.expect(token.RPAREN) + + case token.LBRACK: + p.next() + x = &Option{pos, p.parseExpression()} + p.expect(token.RBRACK) + + case token.LBRACE: + p.next() + x = &Repetition{pos, p.parseExpression()} + p.expect(token.RBRACE) + } + + return x +} + +func (p *parser) parseSequence() Expression { + var list Sequence + + for x := p.parseTerm(); x != nil; x = p.parseTerm() { + list = append(list, x) + } + + // no need for a sequence if list.Len() < 2 + switch len(list) { + case 0: + p.errorExpected(p.pos, "term") + return &Bad{p.pos, "term expected"} + case 1: + return list[0] + } + + return list +} + +func (p *parser) parseExpression() Expression { + var list Alternative + + for { + list = append(list, p.parseSequence()) + if p.tok != token.OR { + break + } + p.next() + } + // len(list) > 0 + + // no need for an Alternative node if list.Len() < 2 + if len(list) == 1 { + return list[0] + } + + return list +} + +func (p *parser) parseProduction() *Production { + name := p.parseIdentifier() + p.expect(token.ASSIGN) + var expr Expression + if p.tok != token.PERIOD { + expr = p.parseExpression() + } + p.expect(token.PERIOD) + return &Production{name, expr} +} + +func (p *parser) parse(fset *token.FileSet, filename string, src []byte) Grammar { + // initialize parser + p.fset = fset + p.ErrorVector.Reset() + p.scanner.Init(fset.AddFile(filename, fset.Base(), len(src)), src, p, scanner.AllowIllegalChars) + p.next() // initializes pos, tok, lit + + grammar := make(Grammar) + for p.tok != token.EOF { + prod := p.parseProduction() + name := prod.Name.String + if _, found := grammar[name]; !found { + grammar[name] = prod + } else { + p.error(prod.Pos(), name+" declared already") + } + } + + return grammar +} + +// Parse parses a set of EBNF productions from source src. +// It returns a set of productions. Errors are reported +// for incorrect syntax and if a production is declared +// more than once. Position information is recorded relative +// to the file set fset. +// +func Parse(fset *token.FileSet, filename string, src []byte) (Grammar, os.Error) { + var p parser + grammar := p.parse(fset, filename, src) + return grammar, p.GetError(scanner.Sorted) +} diff --git a/src/pkg/encoding/ascii85/Makefile b/src/pkg/encoding/ascii85/Makefile new file mode 100644 index 000000000..412383efd --- /dev/null +++ b/src/pkg/encoding/ascii85/Makefile @@ -0,0 +1,11 @@ +# Copyright 2009 The Go Authors. All rights reserved. +# Use of this source code is governed by a BSD-style +# license that can be found in the LICENSE file. + +include ../../../Make.inc + +TARG=encoding/ascii85 +GOFILES=\ + ascii85.go\ + +include ../../../Make.pkg diff --git a/src/pkg/encoding/ascii85/ascii85.go b/src/pkg/encoding/ascii85/ascii85.go new file mode 100644 index 000000000..ead0c2475 --- /dev/null +++ b/src/pkg/encoding/ascii85/ascii85.go @@ -0,0 +1,300 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Package ascii85 implements the ascii85 data encoding +// as used in the btoa tool and Adobe's PostScript and PDF document formats. +package ascii85 + +import ( + "io" + "os" + "strconv" +) + +/* + * Encoder + */ + +// Encode encodes src into at most MaxEncodedLen(len(src)) +// bytes of dst, returning the actual number of bytes written. +// +// The encoding handles 4-byte chunks, using a special encoding +// for the last fragment, so Encode is not appropriate for use on +// individual blocks of a large data stream. Use NewEncoder() instead. +// +// Often, ascii85-encoded data is wrapped in <~ and ~> symbols. +// Encode does not add these. +func Encode(dst, src []byte) int { + if len(src) == 0 { + return 0 + } + + n := 0 + for len(src) > 0 { + dst[0] = 0 + dst[1] = 0 + dst[2] = 0 + dst[3] = 0 + dst[4] = 0 + + // Unpack 4 bytes into uint32 to repack into base 85 5-byte. + var v uint32 + switch len(src) { + default: + v |= uint32(src[3]) + fallthrough + case 3: + v |= uint32(src[2]) << 8 + fallthrough + case 2: + v |= uint32(src[1]) << 16 + fallthrough + case 1: + v |= uint32(src[0]) << 24 + } + + // Special case: zero (!!!!!) shortens to z. + if v == 0 && len(src) >= 4 { + dst[0] = 'z' + dst = dst[1:] + n++ + continue + } + + // Otherwise, 5 base 85 digits starting at !. + for i := 4; i >= 0; i-- { + dst[i] = '!' + byte(v%85) + v /= 85 + } + + // If src was short, discard the low destination bytes. + m := 5 + if len(src) < 4 { + m -= 4 - len(src) + src = nil + } else { + src = src[4:] + } + dst = dst[m:] + n += m + } + return n +} + +// MaxEncodedLen returns the maximum length of an encoding of n source bytes. +func MaxEncodedLen(n int) int { return (n + 3) / 4 * 5 } + +// NewEncoder returns a new ascii85 stream encoder. Data written to +// the returned writer will be encoded and then written to w. +// Ascii85 encodings operate in 32-bit blocks; when finished +// writing, the caller must Close the returned encoder to flush any +// trailing partial block. +func NewEncoder(w io.Writer) io.WriteCloser { return &encoder{w: w} } + +type encoder struct { + err os.Error + w io.Writer + buf [4]byte // buffered data waiting to be encoded + nbuf int // number of bytes in buf + out [1024]byte // output buffer +} + +func (e *encoder) Write(p []byte) (n int, err os.Error) { + if e.err != nil { + return 0, e.err + } + + // Leading fringe. + if e.nbuf > 0 { + var i int + for i = 0; i < len(p) && e.nbuf < 4; i++ { + e.buf[e.nbuf] = p[i] + e.nbuf++ + } + n += i + p = p[i:] + if e.nbuf < 4 { + return + } + nout := Encode(e.out[0:], e.buf[0:]) + if _, e.err = e.w.Write(e.out[0:nout]); e.err != nil { + return n, e.err + } + e.nbuf = 0 + } + + // Large interior chunks. + for len(p) >= 4 { + nn := len(e.out) / 5 * 4 + if nn > len(p) { + nn = len(p) + } + nn -= nn % 4 + if nn > 0 { + nout := Encode(e.out[0:], p[0:nn]) + if _, e.err = e.w.Write(e.out[0:nout]); e.err != nil { + return n, e.err + } + } + n += nn + p = p[nn:] + } + + // Trailing fringe. + for i := 0; i < len(p); i++ { + e.buf[i] = p[i] + } + e.nbuf = len(p) + n += len(p) + return +} + +// Close flushes any pending output from the encoder. +// It is an error to call Write after calling Close. +func (e *encoder) Close() os.Error { + // If there's anything left in the buffer, flush it out + if e.err == nil && e.nbuf > 0 { + nout := Encode(e.out[0:], e.buf[0:e.nbuf]) + e.nbuf = 0 + _, e.err = e.w.Write(e.out[0:nout]) + } + return e.err +} + +/* + * Decoder + */ + +type CorruptInputError int64 + +func (e CorruptInputError) String() string { + return "illegal ascii85 data at input byte " + strconv.Itoa64(int64(e)) +} + +// Decode decodes src into dst, returning both the number +// of bytes written to dst and the number consumed from src. +// If src contains invalid ascii85 data, Decode will return the +// number of bytes successfully written and a CorruptInputError. +// Decode ignores space and control characters in src. +// Often, ascii85-encoded data is wrapped in <~ and ~> symbols. +// Decode expects these to have been stripped by the caller. +// +// If flush is true, Decode assumes that src represents the +// end of the input stream and processes it completely rather +// than wait for the completion of another 32-bit block. +// +// NewDecoder wraps an io.Reader interface around Decode. +// +func Decode(dst, src []byte, flush bool) (ndst, nsrc int, err os.Error) { + var v uint32 + var nb int + for i, b := range src { + if len(dst)-ndst < 4 { + return + } + switch { + case b <= ' ': + continue + case b == 'z' && nb == 0: + nb = 5 + v = 0 + case '!' <= b && b <= 'u': + v = v*85 + uint32(b-'!') + nb++ + default: + return 0, 0, CorruptInputError(i) + } + if nb == 5 { + nsrc = i + 1 + dst[ndst] = byte(v >> 24) + dst[ndst+1] = byte(v >> 16) + dst[ndst+2] = byte(v >> 8) + dst[ndst+3] = byte(v) + ndst += 4 + nb = 0 + v = 0 + } + } + if flush { + nsrc = len(src) + if nb > 0 { + // The number of output bytes in the last fragment + // is the number of leftover input bytes - 1: + // the extra byte provides enough bits to cover + // the inefficiency of the encoding for the block. + if nb == 1 { + return 0, 0, CorruptInputError(len(src)) + } + for i := nb; i < 5; i++ { + // The short encoding truncated the output value. + // We have to assume the worst case values (digit 84) + // in order to ensure that the top bits are correct. + v = v*85 + 84 + } + for i := 0; i < nb-1; i++ { + dst[ndst] = byte(v >> 24) + v <<= 8 + ndst++ + } + } + } + return +} + +// NewDecoder constructs a new ascii85 stream decoder. +func NewDecoder(r io.Reader) io.Reader { return &decoder{r: r} } + +type decoder struct { + err os.Error + readErr os.Error + r io.Reader + end bool // saw end of message + buf [1024]byte // leftover input + nbuf int + out []byte // leftover decoded output + outbuf [1024]byte +} + +func (d *decoder) Read(p []byte) (n int, err os.Error) { + if len(p) == 0 { + return 0, nil + } + if d.err != nil { + return 0, d.err + } + + for { + // Copy leftover output from last decode. + if len(d.out) > 0 { + n = copy(p, d.out) + d.out = d.out[n:] + return + } + + // Decode leftover input from last read. + var nn, nsrc, ndst int + if d.nbuf > 0 { + ndst, nsrc, d.err = Decode(d.outbuf[0:], d.buf[0:d.nbuf], d.readErr != nil) + if ndst > 0 { + d.out = d.outbuf[0:ndst] + d.nbuf = copy(d.buf[0:], d.buf[nsrc:d.nbuf]) + continue // copy out and return + } + } + + // Out of input, out of decoded output. Check errors. + if d.err != nil { + return 0, d.err + } + if d.readErr != nil { + d.err = d.readErr + return 0, d.err + } + + // Read more data. + nn, d.readErr = d.r.Read(d.buf[d.nbuf:]) + d.nbuf += nn + } + panic("unreachable") +} diff --git a/src/pkg/encoding/ascii85/ascii85_test.go b/src/pkg/encoding/ascii85/ascii85_test.go new file mode 100644 index 000000000..fdfeb889f --- /dev/null +++ b/src/pkg/encoding/ascii85/ascii85_test.go @@ -0,0 +1,188 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package ascii85 + +import ( + "bytes" + "io/ioutil" + "os" + "testing" +) + +type testpair struct { + decoded, encoded string +} + +var pairs = []testpair{ + // Wikipedia example + { + "Man is distinguished, not only by his reason, but by this singular passion from " + + "other animals, which is a lust of the mind, that by a perseverance of delight in " + + "the continued and indefatigable generation of knowledge, exceeds the short " + + "vehemence of any carnal pleasure.", + "9jqo^BlbD-BleB1DJ+*+F(f,q/0JhKFCj@.4Gp$d7F!,L7@<6@)/0JDEF@3BB/F*&OCAfu2/AKY\n" + + "i(DIb:@FD,*)+C]U=@3BN#EcYf8ATD3s@q?d$AftVqCh[NqF-FD5W8ARlolDIa\n" + + "l(DIduD.RTpAKYo'+CT/5+Cei#DII?(E,9)oF*2M7/c\n", + }, +} + +var bigtest = pairs[len(pairs)-1] + +func testEqual(t *testing.T, msg string, args ...interface{}) bool { + if args[len(args)-2] != args[len(args)-1] { + t.Errorf(msg, args...) + return false + } + return true +} + +func strip85(s string) string { + t := make([]byte, len(s)) + w := 0 + for r := 0; r < len(s); r++ { + c := s[r] + if c > ' ' { + t[w] = c + w++ + } + } + return string(t[0:w]) +} + +func TestEncode(t *testing.T) { + for _, p := range pairs { + buf := make([]byte, MaxEncodedLen(len(p.decoded))) + n := Encode(buf, []byte(p.decoded)) + buf = buf[0:n] + testEqual(t, "Encode(%q) = %q, want %q", p.decoded, strip85(string(buf)), strip85(p.encoded)) + } +} + +func TestEncoder(t *testing.T) { + for _, p := range pairs { + bb := &bytes.Buffer{} + encoder := NewEncoder(bb) + encoder.Write([]byte(p.decoded)) + encoder.Close() + testEqual(t, "Encode(%q) = %q, want %q", p.decoded, strip85(bb.String()), strip85(p.encoded)) + } +} + +func TestEncoderBuffering(t *testing.T) { + input := []byte(bigtest.decoded) + for bs := 1; bs <= 12; bs++ { + bb := &bytes.Buffer{} + encoder := NewEncoder(bb) + for pos := 0; pos < len(input); pos += bs { + end := pos + bs + if end > len(input) { + end = len(input) + } + n, err := encoder.Write(input[pos:end]) + testEqual(t, "Write(%q) gave error %v, want %v", input[pos:end], err, os.Error(nil)) + testEqual(t, "Write(%q) gave length %v, want %v", input[pos:end], n, end-pos) + } + err := encoder.Close() + testEqual(t, "Close gave error %v, want %v", err, os.Error(nil)) + testEqual(t, "Encoding/%d of %q = %q, want %q", bs, bigtest.decoded, strip85(bb.String()), strip85(bigtest.encoded)) + } +} + +func TestDecode(t *testing.T) { + for _, p := range pairs { + dbuf := make([]byte, 4*len(p.encoded)) + ndst, nsrc, err := Decode(dbuf, []byte(p.encoded), true) + testEqual(t, "Decode(%q) = error %v, want %v", p.encoded, err, os.Error(nil)) + testEqual(t, "Decode(%q) = nsrc %v, want %v", p.encoded, nsrc, len(p.encoded)) + testEqual(t, "Decode(%q) = ndst %v, want %v", p.encoded, ndst, len(p.decoded)) + testEqual(t, "Decode(%q) = %q, want %q", p.encoded, string(dbuf[0:ndst]), p.decoded) + } +} + +func TestDecoder(t *testing.T) { + for _, p := range pairs { + decoder := NewDecoder(bytes.NewBufferString(p.encoded)) + dbuf, err := ioutil.ReadAll(decoder) + if err != nil { + t.Fatal("Read failed", err) + } + testEqual(t, "Read from %q = length %v, want %v", p.encoded, len(dbuf), len(p.decoded)) + testEqual(t, "Decoding of %q = %q, want %q", p.encoded, string(dbuf), p.decoded) + if err != nil { + testEqual(t, "Read from %q = %v, want %v", p.encoded, err, os.EOF) + } + } +} + +func TestDecoderBuffering(t *testing.T) { + for bs := 1; bs <= 12; bs++ { + decoder := NewDecoder(bytes.NewBufferString(bigtest.encoded)) + buf := make([]byte, len(bigtest.decoded)+12) + var total int + for total = 0; total < len(bigtest.decoded); { + n, err := decoder.Read(buf[total : total+bs]) + testEqual(t, "Read from %q at pos %d = %d, %v, want _, %v", bigtest.encoded, total, n, err, os.Error(nil)) + total += n + } + testEqual(t, "Decoding/%d of %q = %q, want %q", bs, bigtest.encoded, string(buf[0:total]), bigtest.decoded) + } +} + +func TestDecodeCorrupt(t *testing.T) { + type corrupt struct { + e string + p int + } + examples := []corrupt{ + {"v", 0}, + {"!z!!!!!!!!!", 1}, + } + + for _, e := range examples { + dbuf := make([]byte, 4*len(e.e)) + _, _, err := Decode(dbuf, []byte(e.e), true) + switch err := err.(type) { + case CorruptInputError: + testEqual(t, "Corruption in %q at offset %v, want %v", e.e, int(err), e.p) + default: + t.Error("Decoder failed to detect corruption in", e) + } + } +} + +func TestBig(t *testing.T) { + n := 3*1000 + 1 + raw := make([]byte, n) + const alpha = "0123456789abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ" + for i := 0; i < n; i++ { + raw[i] = alpha[i%len(alpha)] + } + encoded := new(bytes.Buffer) + w := NewEncoder(encoded) + nn, err := w.Write(raw) + if nn != n || err != nil { + t.Fatalf("Encoder.Write(raw) = %d, %v want %d, nil", nn, err, n) + } + err = w.Close() + if err != nil { + t.Fatalf("Encoder.Close() = %v want nil", err) + } + decoded, err := ioutil.ReadAll(NewDecoder(encoded)) + if err != nil { + t.Fatalf("io.ReadAll(NewDecoder(...)): %v", err) + } + + if !bytes.Equal(raw, decoded) { + var i int + for i = 0; i < len(decoded) && i < len(raw); i++ { + if decoded[i] != raw[i] { + break + } + } + t.Errorf("Decode(Encode(%d-byte string)) failed at offset %d", n, i) + } +} diff --git a/src/pkg/encoding/base32/Makefile b/src/pkg/encoding/base32/Makefile new file mode 100644 index 000000000..c0e85b644 --- /dev/null +++ b/src/pkg/encoding/base32/Makefile @@ -0,0 +1,11 @@ +# Copyright 2009 The Go Authors. All rights reserved. +# Use of this source code is governed by a BSD-style +# license that can be found in the LICENSE file. + +include $(GOROOT)/src/Make.inc + +TARG=encoding/base32 +GOFILES=\ + base32.go\ + +include $(GOROOT)/src/Make.pkg diff --git a/src/pkg/encoding/base32/base32.go b/src/pkg/encoding/base32/base32.go new file mode 100644 index 000000000..acace30d6 --- /dev/null +++ b/src/pkg/encoding/base32/base32.go @@ -0,0 +1,368 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Package base32 implements base32 encoding as specified by RFC 4648. +package base32 + +import ( + "io" + "os" + "strconv" +) + +/* + * Encodings + */ + +// An Encoding is a radix 32 encoding/decoding scheme, defined by a +// 32-character alphabet. The most common is the "base32" encoding +// introduced for SASL GSSAPI and standardized in RFC 4648. +// The alternate "base32hex" encoding is used in DNSSEC. +type Encoding struct { + encode string + decodeMap [256]byte +} + +const encodeStd = "ABCDEFGHIJKLMNOPQRSTUVWXYZ234567" +const encodeHex = "0123456789ABCDEFGHIJKLMNOPQRSTUV" + +// NewEncoding returns a new Encoding defined by the given alphabet, +// which must be a 32-byte string. +func NewEncoding(encoder string) *Encoding { + e := new(Encoding) + e.encode = encoder + for i := 0; i < len(e.decodeMap); i++ { + e.decodeMap[i] = 0xFF + } + for i := 0; i < len(encoder); i++ { + e.decodeMap[encoder[i]] = byte(i) + } + return e +} + +// StdEncoding is the standard base32 encoding, as defined in +// RFC 4648. +var StdEncoding = NewEncoding(encodeStd) + +// HexEncoding is the ``Extended Hex Alphabet'' defined in RFC 4648. +// It is typically used in DNS. +var HexEncoding = NewEncoding(encodeHex) + +/* + * Encoder + */ + +// Encode encodes src using the encoding enc, writing +// EncodedLen(len(src)) bytes to dst. +// +// The encoding pads the output to a multiple of 8 bytes, +// so Encode is not appropriate for use on individual blocks +// of a large data stream. Use NewEncoder() instead. +func (enc *Encoding) Encode(dst, src []byte) { + if len(src) == 0 { + return + } + + for len(src) > 0 { + dst[0] = 0 + dst[1] = 0 + dst[2] = 0 + dst[3] = 0 + dst[4] = 0 + dst[5] = 0 + dst[6] = 0 + dst[7] = 0 + + // Unpack 8x 5-bit source blocks into a 5 byte + // destination quantum + switch len(src) { + default: + dst[7] |= src[4] & 0x1F + dst[6] |= src[4] >> 5 + fallthrough + case 4: + dst[6] |= (src[3] << 3) & 0x1F + dst[5] |= (src[3] >> 2) & 0x1F + dst[4] |= src[3] >> 7 + fallthrough + case 3: + dst[4] |= (src[2] << 1) & 0x1F + dst[3] |= (src[2] >> 4) & 0x1F + fallthrough + case 2: + dst[3] |= (src[1] << 4) & 0x1F + dst[2] |= (src[1] >> 1) & 0x1F + dst[1] |= (src[1] >> 6) & 0x1F + fallthrough + case 1: + dst[1] |= (src[0] << 2) & 0x1F + dst[0] |= src[0] >> 3 + } + + // Encode 5-bit blocks using the base32 alphabet + for j := 0; j < 8; j++ { + dst[j] = enc.encode[dst[j]] + } + + // Pad the final quantum + if len(src) < 5 { + dst[7] = '=' + if len(src) < 4 { + dst[6] = '=' + dst[5] = '=' + if len(src) < 3 { + dst[4] = '=' + if len(src) < 2 { + dst[3] = '=' + dst[2] = '=' + } + } + } + break + } + src = src[5:] + dst = dst[8:] + } +} + +type encoder struct { + err os.Error + enc *Encoding + w io.Writer + buf [5]byte // buffered data waiting to be encoded + nbuf int // number of bytes in buf + out [1024]byte // output buffer +} + +func (e *encoder) Write(p []byte) (n int, err os.Error) { + if e.err != nil { + return 0, e.err + } + + // Leading fringe. + if e.nbuf > 0 { + var i int + for i = 0; i < len(p) && e.nbuf < 5; i++ { + e.buf[e.nbuf] = p[i] + e.nbuf++ + } + n += i + p = p[i:] + if e.nbuf < 5 { + return + } + e.enc.Encode(e.out[0:], e.buf[0:]) + if _, e.err = e.w.Write(e.out[0:8]); e.err != nil { + return n, e.err + } + e.nbuf = 0 + } + + // Large interior chunks. + for len(p) >= 5 { + nn := len(e.out) / 8 * 5 + if nn > len(p) { + nn = len(p) + } + nn -= nn % 5 + if nn > 0 { + e.enc.Encode(e.out[0:], p[0:nn]) + if _, e.err = e.w.Write(e.out[0 : nn/5*8]); e.err != nil { + return n, e.err + } + } + n += nn + p = p[nn:] + } + + // Trailing fringe. + for i := 0; i < len(p); i++ { + e.buf[i] = p[i] + } + e.nbuf = len(p) + n += len(p) + return +} + +// Close flushes any pending output from the encoder. +// It is an error to call Write after calling Close. +func (e *encoder) Close() os.Error { + // If there's anything left in the buffer, flush it out + if e.err == nil && e.nbuf > 0 { + e.enc.Encode(e.out[0:], e.buf[0:e.nbuf]) + e.nbuf = 0 + _, e.err = e.w.Write(e.out[0:8]) + } + return e.err +} + +// NewEncoder returns a new base32 stream encoder. Data written to +// the returned writer will be encoded using enc and then written to w. +// Base32 encodings operate in 5-byte blocks; when finished +// writing, the caller must Close the returned encoder to flush any +// partially written blocks. +func NewEncoder(enc *Encoding, w io.Writer) io.WriteCloser { + return &encoder{enc: enc, w: w} +} + +// EncodedLen returns the length in bytes of the base32 encoding +// of an input buffer of length n. +func (enc *Encoding) EncodedLen(n int) int { return (n + 4) / 5 * 8 } + +/* + * Decoder + */ + +type CorruptInputError int64 + +func (e CorruptInputError) String() string { + return "illegal base32 data at input byte " + strconv.Itoa64(int64(e)) +} + +// decode is like Decode but returns an additional 'end' value, which +// indicates if end-of-message padding was encountered and thus any +// additional data is an error. decode also assumes len(src)%8==0, +// since it is meant for internal use. +func (enc *Encoding) decode(dst, src []byte) (n int, end bool, err os.Error) { + for i := 0; i < len(src)/8 && !end; i++ { + // Decode quantum using the base32 alphabet + var dbuf [8]byte + dlen := 8 + + // do the top bytes contain any data? + dbufloop: + for j := 0; j < 8; j++ { + in := src[i*8+j] + if in == '=' && j >= 2 && i == len(src)/8-1 { + // We've reached the end and there's + // padding, the rest should be padded + for k := j; k < 8; k++ { + if src[i*8+k] != '=' { + return n, false, CorruptInputError(i*8 + j) + } + } + dlen = j + end = true + break dbufloop + } + dbuf[j] = enc.decodeMap[in] + if dbuf[j] == 0xFF { + return n, false, CorruptInputError(i*8 + j) + } + } + + // Pack 8x 5-bit source blocks into 5 byte destination + // quantum + switch dlen { + case 7, 8: + dst[i*5+4] = dbuf[6]<<5 | dbuf[7] + fallthrough + case 6, 5: + dst[i*5+3] = dbuf[4]<<7 | dbuf[5]<<2 | dbuf[6]>>3 + fallthrough + case 4: + dst[i*5+2] = dbuf[3]<<4 | dbuf[4]>>1 + fallthrough + case 3: + dst[i*5+1] = dbuf[1]<<6 | dbuf[2]<<1 | dbuf[3]>>4 + fallthrough + case 2: + dst[i*5+0] = dbuf[0]<<3 | dbuf[1]>>2 + } + switch dlen { + case 2: + n += 1 + case 3, 4: + n += 2 + case 5: + n += 3 + case 6, 7: + n += 4 + case 8: + n += 5 + } + } + return n, end, nil +} + +// Decode decodes src using the encoding enc. It writes at most +// DecodedLen(len(src)) bytes to dst and returns the number of bytes +// written. If src contains invalid base32 data, it will return the +// number of bytes successfully written and CorruptInputError. +func (enc *Encoding) Decode(dst, src []byte) (n int, err os.Error) { + if len(src)%8 != 0 { + return 0, CorruptInputError(len(src) / 8 * 8) + } + + n, _, err = enc.decode(dst, src) + return +} + +type decoder struct { + err os.Error + enc *Encoding + r io.Reader + end bool // saw end of message + buf [1024]byte // leftover input + nbuf int + out []byte // leftover decoded output + outbuf [1024 / 8 * 5]byte +} + +func (d *decoder) Read(p []byte) (n int, err os.Error) { + if d.err != nil { + return 0, d.err + } + + // Use leftover decoded output from last read. + if len(d.out) > 0 { + n = copy(p, d.out) + d.out = d.out[n:] + return n, nil + } + + // Read a chunk. + nn := len(p) / 5 * 8 + if nn < 8 { + nn = 8 + } + if nn > len(d.buf) { + nn = len(d.buf) + } + nn, d.err = io.ReadAtLeast(d.r, d.buf[d.nbuf:nn], 8-d.nbuf) + d.nbuf += nn + if d.nbuf < 8 { + return 0, d.err + } + + // Decode chunk into p, or d.out and then p if p is too small. + nr := d.nbuf / 8 * 8 + nw := d.nbuf / 8 * 5 + if nw > len(p) { + nw, d.end, d.err = d.enc.decode(d.outbuf[0:], d.buf[0:nr]) + d.out = d.outbuf[0:nw] + n = copy(p, d.out) + d.out = d.out[n:] + } else { + n, d.end, d.err = d.enc.decode(p, d.buf[0:nr]) + } + d.nbuf -= nr + for i := 0; i < d.nbuf; i++ { + d.buf[i] = d.buf[i+nr] + } + + if d.err == nil { + d.err = err + } + return n, d.err +} + +// NewDecoder constructs a new base32 stream decoder. +func NewDecoder(enc *Encoding, r io.Reader) io.Reader { + return &decoder{enc: enc, r: r} +} + +// DecodedLen returns the maximum length in bytes of the decoded data +// corresponding to n bytes of base32-encoded data. +func (enc *Encoding) DecodedLen(n int) int { return n / 8 * 5 } diff --git a/src/pkg/encoding/base32/base32_test.go b/src/pkg/encoding/base32/base32_test.go new file mode 100644 index 000000000..3fa1c2b26 --- /dev/null +++ b/src/pkg/encoding/base32/base32_test.go @@ -0,0 +1,193 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package base32 + +import ( + "bytes" + "io/ioutil" + "os" + "testing" +) + +type testpair struct { + decoded, encoded string +} + +var pairs = []testpair{ + // RFC 4648 examples + {"", ""}, + {"f", "MY======"}, + {"fo", "MZXQ===="}, + {"foo", "MZXW6==="}, + {"foob", "MZXW6YQ="}, + {"fooba", "MZXW6YTB"}, + {"foobar", "MZXW6YTBOI======"}, + + // Wikipedia examples, converted to base32 + {"sure.", "ON2XEZJO"}, + {"sure", "ON2XEZI="}, + {"sur", "ON2XE==="}, + {"su", "ON2Q===="}, + {"leasure.", "NRSWC43VOJSS4==="}, + {"easure.", "MVQXG5LSMUXA===="}, + {"asure.", "MFZXK4TFFY======"}, + {"sure.", "ON2XEZJO"}, +} + +var bigtest = testpair{ + "Twas brillig, and the slithy toves", + "KR3WC4ZAMJZGS3DMNFTSYIDBNZSCA5DIMUQHG3DJORUHSIDUN53GK4Y=", +} + +func testEqual(t *testing.T, msg string, args ...interface{}) bool { + if args[len(args)-2] != args[len(args)-1] { + t.Errorf(msg, args...) + return false + } + return true +} + +func TestEncode(t *testing.T) { + for _, p := range pairs { + buf := make([]byte, StdEncoding.EncodedLen(len(p.decoded))) + StdEncoding.Encode(buf, []byte(p.decoded)) + testEqual(t, "Encode(%q) = %q, want %q", p.decoded, string(buf), p.encoded) + } +} + +func TestEncoder(t *testing.T) { + for _, p := range pairs { + bb := &bytes.Buffer{} + encoder := NewEncoder(StdEncoding, bb) + encoder.Write([]byte(p.decoded)) + encoder.Close() + testEqual(t, "Encode(%q) = %q, want %q", p.decoded, bb.String(), p.encoded) + } +} + +func TestEncoderBuffering(t *testing.T) { + input := []byte(bigtest.decoded) + for bs := 1; bs <= 12; bs++ { + bb := &bytes.Buffer{} + encoder := NewEncoder(StdEncoding, bb) + for pos := 0; pos < len(input); pos += bs { + end := pos + bs + if end > len(input) { + end = len(input) + } + n, err := encoder.Write(input[pos:end]) + testEqual(t, "Write(%q) gave error %v, want %v", input[pos:end], err, os.Error(nil)) + testEqual(t, "Write(%q) gave length %v, want %v", input[pos:end], n, end-pos) + } + err := encoder.Close() + testEqual(t, "Close gave error %v, want %v", err, os.Error(nil)) + testEqual(t, "Encoding/%d of %q = %q, want %q", bs, bigtest.decoded, bb.String(), bigtest.encoded) + } +} + +func TestDecode(t *testing.T) { + for _, p := range pairs { + dbuf := make([]byte, StdEncoding.DecodedLen(len(p.encoded))) + count, end, err := StdEncoding.decode(dbuf, []byte(p.encoded)) + testEqual(t, "Decode(%q) = error %v, want %v", p.encoded, err, os.Error(nil)) + testEqual(t, "Decode(%q) = length %v, want %v", p.encoded, count, len(p.decoded)) + if len(p.encoded) > 0 { + testEqual(t, "Decode(%q) = end %v, want %v", p.encoded, end, (p.encoded[len(p.encoded)-1] == '=')) + } + testEqual(t, "Decode(%q) = %q, want %q", p.encoded, + string(dbuf[0:count]), + p.decoded) + } +} + +func TestDecoder(t *testing.T) { + for _, p := range pairs { + decoder := NewDecoder(StdEncoding, bytes.NewBufferString(p.encoded)) + dbuf := make([]byte, StdEncoding.DecodedLen(len(p.encoded))) + count, err := decoder.Read(dbuf) + if err != nil && err != os.EOF { + t.Fatal("Read failed", err) + } + testEqual(t, "Read from %q = length %v, want %v", p.encoded, count, len(p.decoded)) + testEqual(t, "Decoding of %q = %q, want %q", p.encoded, string(dbuf[0:count]), p.decoded) + if err != os.EOF { + count, err = decoder.Read(dbuf) + } + testEqual(t, "Read from %q = %v, want %v", p.encoded, err, os.EOF) + } +} + +func TestDecoderBuffering(t *testing.T) { + for bs := 1; bs <= 12; bs++ { + decoder := NewDecoder(StdEncoding, bytes.NewBufferString(bigtest.encoded)) + buf := make([]byte, len(bigtest.decoded)+12) + var total int + for total = 0; total < len(bigtest.decoded); { + n, err := decoder.Read(buf[total : total+bs]) + testEqual(t, "Read from %q at pos %d = %d, %v, want _, %v", bigtest.encoded, total, n, err, os.Error(nil)) + total += n + } + testEqual(t, "Decoding/%d of %q = %q, want %q", bs, bigtest.encoded, string(buf[0:total]), bigtest.decoded) + } +} + +func TestDecodeCorrupt(t *testing.T) { + type corrupt struct { + e string + p int + } + examples := []corrupt{ + {"!!!!", 0}, + {"x===", 0}, + {"AA=A====", 2}, + {"AAA=AAAA", 3}, + {"MMMMMMMMM", 8}, + {"MMMMMM", 0}, + } + + for _, e := range examples { + dbuf := make([]byte, StdEncoding.DecodedLen(len(e.e))) + _, err := StdEncoding.Decode(dbuf, []byte(e.e)) + switch err := err.(type) { + case CorruptInputError: + testEqual(t, "Corruption in %q at offset %v, want %v", e.e, int(err), e.p) + default: + t.Error("Decoder failed to detect corruption in", e) + } + } +} + +func TestBig(t *testing.T) { + n := 3*1000 + 1 + raw := make([]byte, n) + const alpha = "0123456789abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ" + for i := 0; i < n; i++ { + raw[i] = alpha[i%len(alpha)] + } + encoded := new(bytes.Buffer) + w := NewEncoder(StdEncoding, encoded) + nn, err := w.Write(raw) + if nn != n || err != nil { + t.Fatalf("Encoder.Write(raw) = %d, %v want %d, nil", nn, err, n) + } + err = w.Close() + if err != nil { + t.Fatalf("Encoder.Close() = %v want nil", err) + } + decoded, err := ioutil.ReadAll(NewDecoder(StdEncoding, encoded)) + if err != nil { + t.Fatalf("ioutil.ReadAll(NewDecoder(...)): %v", err) + } + + if !bytes.Equal(raw, decoded) { + var i int + for i = 0; i < len(decoded) && i < len(raw); i++ { + if decoded[i] != raw[i] { + break + } + } + t.Errorf("Decode(Encode(%d-byte string)) failed at offset %d", n, i) + } +} diff --git a/src/pkg/encoding/base64/Makefile b/src/pkg/encoding/base64/Makefile new file mode 100644 index 000000000..2f54ed839 --- /dev/null +++ b/src/pkg/encoding/base64/Makefile @@ -0,0 +1,11 @@ +# Copyright 2009 The Go Authors. All rights reserved. +# Use of this source code is governed by a BSD-style +# license that can be found in the LICENSE file. + +include ../../../Make.inc + +TARG=encoding/base64 +GOFILES=\ + base64.go\ + +include ../../../Make.pkg diff --git a/src/pkg/encoding/base64/base64.go b/src/pkg/encoding/base64/base64.go new file mode 100644 index 000000000..c6b2a13e4 --- /dev/null +++ b/src/pkg/encoding/base64/base64.go @@ -0,0 +1,343 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Package base64 implements base64 encoding as specified by RFC 4648. +package base64 + +import ( + "io" + "os" + "strconv" +) + +/* + * Encodings + */ + +// An Encoding is a radix 64 encoding/decoding scheme, defined by a +// 64-character alphabet. The most common encoding is the "base64" +// encoding defined in RFC 4648 and used in MIME (RFC 2045) and PEM +// (RFC 1421). RFC 4648 also defines an alternate encoding, which is +// the standard encoding with - and _ substituted for + and /. +type Encoding struct { + encode string + decodeMap [256]byte +} + +const encodeStd = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/" +const encodeURL = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789-_" + +// NewEncoding returns a new Encoding defined by the given alphabet, +// which must be a 64-byte string. +func NewEncoding(encoder string) *Encoding { + e := new(Encoding) + e.encode = encoder + for i := 0; i < len(e.decodeMap); i++ { + e.decodeMap[i] = 0xFF + } + for i := 0; i < len(encoder); i++ { + e.decodeMap[encoder[i]] = byte(i) + } + return e +} + +// StdEncoding is the standard base64 encoding, as defined in +// RFC 4648. +var StdEncoding = NewEncoding(encodeStd) + +// URLEncoding is the alternate base64 encoding defined in RFC 4648. +// It is typically used in URLs and file names. +var URLEncoding = NewEncoding(encodeURL) + +/* + * Encoder + */ + +// Encode encodes src using the encoding enc, writing +// EncodedLen(len(src)) bytes to dst. +// +// The encoding pads the output to a multiple of 4 bytes, +// so Encode is not appropriate for use on individual blocks +// of a large data stream. Use NewEncoder() instead. +func (enc *Encoding) Encode(dst, src []byte) { + if len(src) == 0 { + return + } + + for len(src) > 0 { + dst[0] = 0 + dst[1] = 0 + dst[2] = 0 + dst[3] = 0 + + // Unpack 4x 6-bit source blocks into a 4 byte + // destination quantum + switch len(src) { + default: + dst[3] |= src[2] & 0x3F + dst[2] |= src[2] >> 6 + fallthrough + case 2: + dst[2] |= (src[1] << 2) & 0x3F + dst[1] |= src[1] >> 4 + fallthrough + case 1: + dst[1] |= (src[0] << 4) & 0x3F + dst[0] |= src[0] >> 2 + } + + // Encode 6-bit blocks using the base64 alphabet + for j := 0; j < 4; j++ { + dst[j] = enc.encode[dst[j]] + } + + // Pad the final quantum + if len(src) < 3 { + dst[3] = '=' + if len(src) < 2 { + dst[2] = '=' + } + break + } + + src = src[3:] + dst = dst[4:] + } +} + +// EncodeToString returns the base64 encoding of src. +func (enc *Encoding) EncodeToString(src []byte) string { + buf := make([]byte, enc.EncodedLen(len(src))) + enc.Encode(buf, src) + return string(buf) +} + +type encoder struct { + err os.Error + enc *Encoding + w io.Writer + buf [3]byte // buffered data waiting to be encoded + nbuf int // number of bytes in buf + out [1024]byte // output buffer +} + +func (e *encoder) Write(p []byte) (n int, err os.Error) { + if e.err != nil { + return 0, e.err + } + + // Leading fringe. + if e.nbuf > 0 { + var i int + for i = 0; i < len(p) && e.nbuf < 3; i++ { + e.buf[e.nbuf] = p[i] + e.nbuf++ + } + n += i + p = p[i:] + if e.nbuf < 3 { + return + } + e.enc.Encode(e.out[0:], e.buf[0:]) + if _, e.err = e.w.Write(e.out[0:4]); e.err != nil { + return n, e.err + } + e.nbuf = 0 + } + + // Large interior chunks. + for len(p) >= 3 { + nn := len(e.out) / 4 * 3 + if nn > len(p) { + nn = len(p) + } + nn -= nn % 3 + if nn > 0 { + e.enc.Encode(e.out[0:], p[0:nn]) + if _, e.err = e.w.Write(e.out[0 : nn/3*4]); e.err != nil { + return n, e.err + } + } + n += nn + p = p[nn:] + } + + // Trailing fringe. + for i := 0; i < len(p); i++ { + e.buf[i] = p[i] + } + e.nbuf = len(p) + n += len(p) + return +} + +// Close flushes any pending output from the encoder. +// It is an error to call Write after calling Close. +func (e *encoder) Close() os.Error { + // If there's anything left in the buffer, flush it out + if e.err == nil && e.nbuf > 0 { + e.enc.Encode(e.out[0:], e.buf[0:e.nbuf]) + e.nbuf = 0 + _, e.err = e.w.Write(e.out[0:4]) + } + return e.err +} + +// NewEncoder returns a new base64 stream encoder. Data written to +// the returned writer will be encoded using enc and then written to w. +// Base64 encodings operate in 4-byte blocks; when finished +// writing, the caller must Close the returned encoder to flush any +// partially written blocks. +func NewEncoder(enc *Encoding, w io.Writer) io.WriteCloser { + return &encoder{enc: enc, w: w} +} + +// EncodedLen returns the length in bytes of the base64 encoding +// of an input buffer of length n. +func (enc *Encoding) EncodedLen(n int) int { return (n + 2) / 3 * 4 } + +/* + * Decoder + */ + +type CorruptInputError int64 + +func (e CorruptInputError) String() string { + return "illegal base64 data at input byte " + strconv.Itoa64(int64(e)) +} + +// decode is like Decode but returns an additional 'end' value, which +// indicates if end-of-message padding was encountered and thus any +// additional data is an error. decode also assumes len(src)%4==0, +// since it is meant for internal use. +func (enc *Encoding) decode(dst, src []byte) (n int, end bool, err os.Error) { + for i := 0; i < len(src)/4 && !end; i++ { + // Decode quantum using the base64 alphabet + var dbuf [4]byte + dlen := 4 + + dbufloop: + for j := 0; j < 4; j++ { + in := src[i*4+j] + if in == '=' && j >= 2 && i == len(src)/4-1 { + // We've reached the end and there's + // padding + if src[i*4+3] != '=' { + return n, false, CorruptInputError(i*4 + 2) + } + dlen = j + end = true + break dbufloop + } + dbuf[j] = enc.decodeMap[in] + if dbuf[j] == 0xFF { + return n, false, CorruptInputError(i*4 + j) + } + } + + // Pack 4x 6-bit source blocks into 3 byte destination + // quantum + switch dlen { + case 4: + dst[i*3+2] = dbuf[2]<<6 | dbuf[3] + fallthrough + case 3: + dst[i*3+1] = dbuf[1]<<4 | dbuf[2]>>2 + fallthrough + case 2: + dst[i*3+0] = dbuf[0]<<2 | dbuf[1]>>4 + } + n += dlen - 1 + } + + return n, end, nil +} + +// Decode decodes src using the encoding enc. It writes at most +// DecodedLen(len(src)) bytes to dst and returns the number of bytes +// written. If src contains invalid base64 data, it will return the +// number of bytes successfully written and CorruptInputError. +func (enc *Encoding) Decode(dst, src []byte) (n int, err os.Error) { + if len(src)%4 != 0 { + return 0, CorruptInputError(len(src) / 4 * 4) + } + + n, _, err = enc.decode(dst, src) + return +} + +// DecodeString returns the bytes represented by the base64 string s. +func (enc *Encoding) DecodeString(s string) ([]byte, os.Error) { + dbuf := make([]byte, enc.DecodedLen(len(s))) + n, err := enc.Decode(dbuf, []byte(s)) + return dbuf[:n], err +} + +type decoder struct { + err os.Error + enc *Encoding + r io.Reader + end bool // saw end of message + buf [1024]byte // leftover input + nbuf int + out []byte // leftover decoded output + outbuf [1024 / 4 * 3]byte +} + +func (d *decoder) Read(p []byte) (n int, err os.Error) { + if d.err != nil { + return 0, d.err + } + + // Use leftover decoded output from last read. + if len(d.out) > 0 { + n = copy(p, d.out) + d.out = d.out[n:] + return n, nil + } + + // Read a chunk. + nn := len(p) / 3 * 4 + if nn < 4 { + nn = 4 + } + if nn > len(d.buf) { + nn = len(d.buf) + } + nn, d.err = io.ReadAtLeast(d.r, d.buf[d.nbuf:nn], 4-d.nbuf) + d.nbuf += nn + if d.nbuf < 4 { + return 0, d.err + } + + // Decode chunk into p, or d.out and then p if p is too small. + nr := d.nbuf / 4 * 4 + nw := d.nbuf / 4 * 3 + if nw > len(p) { + nw, d.end, d.err = d.enc.decode(d.outbuf[0:], d.buf[0:nr]) + d.out = d.outbuf[0:nw] + n = copy(p, d.out) + d.out = d.out[n:] + } else { + n, d.end, d.err = d.enc.decode(p, d.buf[0:nr]) + } + d.nbuf -= nr + for i := 0; i < d.nbuf; i++ { + d.buf[i] = d.buf[i+nr] + } + + if d.err == nil { + d.err = err + } + return n, d.err +} + +// NewDecoder constructs a new base64 stream decoder. +func NewDecoder(enc *Encoding, r io.Reader) io.Reader { + return &decoder{enc: enc, r: r} +} + +// DecodedLen returns the maximum length in bytes of the decoded data +// corresponding to n bytes of base64-encoded data. +func (enc *Encoding) DecodedLen(n int) int { return n / 4 * 3 } diff --git a/src/pkg/encoding/base64/base64_test.go b/src/pkg/encoding/base64/base64_test.go new file mode 100644 index 000000000..c163dae84 --- /dev/null +++ b/src/pkg/encoding/base64/base64_test.go @@ -0,0 +1,199 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package base64 + +import ( + "bytes" + "io/ioutil" + "os" + "testing" +) + +type testpair struct { + decoded, encoded string +} + +var pairs = []testpair{ + // RFC 3548 examples + {"\x14\xfb\x9c\x03\xd9\x7e", "FPucA9l+"}, + {"\x14\xfb\x9c\x03\xd9", "FPucA9k="}, + {"\x14\xfb\x9c\x03", "FPucAw=="}, + + // RFC 4648 examples + {"", ""}, + {"f", "Zg=="}, + {"fo", "Zm8="}, + {"foo", "Zm9v"}, + {"foob", "Zm9vYg=="}, + {"fooba", "Zm9vYmE="}, + {"foobar", "Zm9vYmFy"}, + + // Wikipedia examples + {"sure.", "c3VyZS4="}, + {"sure", "c3VyZQ=="}, + {"sur", "c3Vy"}, + {"su", "c3U="}, + {"leasure.", "bGVhc3VyZS4="}, + {"easure.", "ZWFzdXJlLg=="}, + {"asure.", "YXN1cmUu"}, + {"sure.", "c3VyZS4="}, +} + +var bigtest = testpair{ + "Twas brillig, and the slithy toves", + "VHdhcyBicmlsbGlnLCBhbmQgdGhlIHNsaXRoeSB0b3Zlcw==", +} + +func testEqual(t *testing.T, msg string, args ...interface{}) bool { + if args[len(args)-2] != args[len(args)-1] { + t.Errorf(msg, args...) + return false + } + return true +} + +func TestEncode(t *testing.T) { + for _, p := range pairs { + got := StdEncoding.EncodeToString([]byte(p.decoded)) + testEqual(t, "Encode(%q) = %q, want %q", p.decoded, got, p.encoded) + } +} + +func TestEncoder(t *testing.T) { + for _, p := range pairs { + bb := &bytes.Buffer{} + encoder := NewEncoder(StdEncoding, bb) + encoder.Write([]byte(p.decoded)) + encoder.Close() + testEqual(t, "Encode(%q) = %q, want %q", p.decoded, bb.String(), p.encoded) + } +} + +func TestEncoderBuffering(t *testing.T) { + input := []byte(bigtest.decoded) + for bs := 1; bs <= 12; bs++ { + bb := &bytes.Buffer{} + encoder := NewEncoder(StdEncoding, bb) + for pos := 0; pos < len(input); pos += bs { + end := pos + bs + if end > len(input) { + end = len(input) + } + n, err := encoder.Write(input[pos:end]) + testEqual(t, "Write(%q) gave error %v, want %v", input[pos:end], err, os.Error(nil)) + testEqual(t, "Write(%q) gave length %v, want %v", input[pos:end], n, end-pos) + } + err := encoder.Close() + testEqual(t, "Close gave error %v, want %v", err, os.Error(nil)) + testEqual(t, "Encoding/%d of %q = %q, want %q", bs, bigtest.decoded, bb.String(), bigtest.encoded) + } +} + +func TestDecode(t *testing.T) { + for _, p := range pairs { + dbuf := make([]byte, StdEncoding.DecodedLen(len(p.encoded))) + count, end, err := StdEncoding.decode(dbuf, []byte(p.encoded)) + testEqual(t, "Decode(%q) = error %v, want %v", p.encoded, err, os.Error(nil)) + testEqual(t, "Decode(%q) = length %v, want %v", p.encoded, count, len(p.decoded)) + if len(p.encoded) > 0 { + testEqual(t, "Decode(%q) = end %v, want %v", p.encoded, end, (p.encoded[len(p.encoded)-1] == '=')) + } + testEqual(t, "Decode(%q) = %q, want %q", p.encoded, string(dbuf[0:count]), p.decoded) + + dbuf, err = StdEncoding.DecodeString(p.encoded) + testEqual(t, "DecodeString(%q) = error %v, want %v", p.encoded, err, os.Error(nil)) + testEqual(t, "DecodeString(%q) = %q, want %q", string(dbuf), p.decoded) + } +} + +func TestDecoder(t *testing.T) { + for _, p := range pairs { + decoder := NewDecoder(StdEncoding, bytes.NewBufferString(p.encoded)) + dbuf := make([]byte, StdEncoding.DecodedLen(len(p.encoded))) + count, err := decoder.Read(dbuf) + if err != nil && err != os.EOF { + t.Fatal("Read failed", err) + } + testEqual(t, "Read from %q = length %v, want %v", p.encoded, count, len(p.decoded)) + testEqual(t, "Decoding of %q = %q, want %q", p.encoded, string(dbuf[0:count]), p.decoded) + if err != os.EOF { + count, err = decoder.Read(dbuf) + } + testEqual(t, "Read from %q = %v, want %v", p.encoded, err, os.EOF) + } +} + +func TestDecoderBuffering(t *testing.T) { + for bs := 1; bs <= 12; bs++ { + decoder := NewDecoder(StdEncoding, bytes.NewBufferString(bigtest.encoded)) + buf := make([]byte, len(bigtest.decoded)+12) + var total int + for total = 0; total < len(bigtest.decoded); { + n, err := decoder.Read(buf[total : total+bs]) + testEqual(t, "Read from %q at pos %d = %d, %v, want _, %v", bigtest.encoded, total, n, err, os.Error(nil)) + total += n + } + testEqual(t, "Decoding/%d of %q = %q, want %q", bs, bigtest.encoded, string(buf[0:total]), bigtest.decoded) + } +} + +func TestDecodeCorrupt(t *testing.T) { + type corrupt struct { + e string + p int + } + examples := []corrupt{ + {"!!!!", 0}, + {"x===", 1}, + {"AA=A", 2}, + {"AAA=AAAA", 3}, + {"AAAAA", 4}, + {"AAAAAA", 4}, + } + + for _, e := range examples { + dbuf := make([]byte, StdEncoding.DecodedLen(len(e.e))) + _, err := StdEncoding.Decode(dbuf, []byte(e.e)) + switch err := err.(type) { + case CorruptInputError: + testEqual(t, "Corruption in %q at offset %v, want %v", e.e, int(err), e.p) + default: + t.Error("Decoder failed to detect corruption in", e) + } + } +} + +func TestBig(t *testing.T) { + n := 3*1000 + 1 + raw := make([]byte, n) + const alpha = "0123456789abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ" + for i := 0; i < n; i++ { + raw[i] = alpha[i%len(alpha)] + } + encoded := new(bytes.Buffer) + w := NewEncoder(StdEncoding, encoded) + nn, err := w.Write(raw) + if nn != n || err != nil { + t.Fatalf("Encoder.Write(raw) = %d, %v want %d, nil", nn, err, n) + } + err = w.Close() + if err != nil { + t.Fatalf("Encoder.Close() = %v want nil", err) + } + decoded, err := ioutil.ReadAll(NewDecoder(StdEncoding, encoded)) + if err != nil { + t.Fatalf("ioutil.ReadAll(NewDecoder(...)): %v", err) + } + + if !bytes.Equal(raw, decoded) { + var i int + for i = 0; i < len(decoded) && i < len(raw); i++ { + if decoded[i] != raw[i] { + break + } + } + t.Errorf("Decode(Encode(%d-byte string)) failed at offset %d", n, i) + } +} diff --git a/src/pkg/encoding/binary/Makefile b/src/pkg/encoding/binary/Makefile new file mode 100644 index 000000000..dc46abe90 --- /dev/null +++ b/src/pkg/encoding/binary/Makefile @@ -0,0 +1,11 @@ +# Copyright 2009 The Go Authors. All rights reserved. +# Use of this source code is governed by a BSD-style +# license that can be found in the LICENSE file. + +include ../../../Make.inc + +TARG=encoding/binary +GOFILES=\ + binary.go\ + +include ../../../Make.pkg diff --git a/src/pkg/encoding/binary/binary.go b/src/pkg/encoding/binary/binary.go new file mode 100644 index 000000000..8e55cb23b --- /dev/null +++ b/src/pkg/encoding/binary/binary.go @@ -0,0 +1,498 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Package binary implements translation between +// unsigned integer values and byte sequences +// and the reading and writing of fixed-size values. +package binary + +import ( + "math" + "io" + "os" + "reflect" +) + +// A ByteOrder specifies how to convert byte sequences into +// 16-, 32-, or 64-bit unsigned integers. +type ByteOrder interface { + Uint16(b []byte) uint16 + Uint32(b []byte) uint32 + Uint64(b []byte) uint64 + PutUint16([]byte, uint16) + PutUint32([]byte, uint32) + PutUint64([]byte, uint64) + String() string +} + +// This is byte instead of struct{} so that it can be compared, +// allowing, e.g., order == binary.LittleEndian. +type unused byte + +// LittleEndian is the little-endian implementation of ByteOrder. +var LittleEndian littleEndian + +// BigEndian is the big-endian implementation of ByteOrder. +var BigEndian bigEndian + +type littleEndian unused + +func (littleEndian) Uint16(b []byte) uint16 { return uint16(b[0]) | uint16(b[1])<<8 } + +func (littleEndian) PutUint16(b []byte, v uint16) { + b[0] = byte(v) + b[1] = byte(v >> 8) +} + +func (littleEndian) Uint32(b []byte) uint32 { + return uint32(b[0]) | uint32(b[1])<<8 | uint32(b[2])<<16 | uint32(b[3])<<24 +} + +func (littleEndian) PutUint32(b []byte, v uint32) { + b[0] = byte(v) + b[1] = byte(v >> 8) + b[2] = byte(v >> 16) + b[3] = byte(v >> 24) +} + +func (littleEndian) Uint64(b []byte) uint64 { + return uint64(b[0]) | uint64(b[1])<<8 | uint64(b[2])<<16 | uint64(b[3])<<24 | + uint64(b[4])<<32 | uint64(b[5])<<40 | uint64(b[6])<<48 | uint64(b[7])<<56 +} + +func (littleEndian) PutUint64(b []byte, v uint64) { + b[0] = byte(v) + b[1] = byte(v >> 8) + b[2] = byte(v >> 16) + b[3] = byte(v >> 24) + b[4] = byte(v >> 32) + b[5] = byte(v >> 40) + b[6] = byte(v >> 48) + b[7] = byte(v >> 56) +} + +func (littleEndian) String() string { return "LittleEndian" } + +func (littleEndian) GoString() string { return "binary.LittleEndian" } + +type bigEndian unused + +func (bigEndian) Uint16(b []byte) uint16 { return uint16(b[1]) | uint16(b[0])<<8 } + +func (bigEndian) PutUint16(b []byte, v uint16) { + b[0] = byte(v >> 8) + b[1] = byte(v) +} + +func (bigEndian) Uint32(b []byte) uint32 { + return uint32(b[3]) | uint32(b[2])<<8 | uint32(b[1])<<16 | uint32(b[0])<<24 +} + +func (bigEndian) PutUint32(b []byte, v uint32) { + b[0] = byte(v >> 24) + b[1] = byte(v >> 16) + b[2] = byte(v >> 8) + b[3] = byte(v) +} + +func (bigEndian) Uint64(b []byte) uint64 { + return uint64(b[7]) | uint64(b[6])<<8 | uint64(b[5])<<16 | uint64(b[4])<<24 | + uint64(b[3])<<32 | uint64(b[2])<<40 | uint64(b[1])<<48 | uint64(b[0])<<56 +} + +func (bigEndian) PutUint64(b []byte, v uint64) { + b[0] = byte(v >> 56) + b[1] = byte(v >> 48) + b[2] = byte(v >> 40) + b[3] = byte(v >> 32) + b[4] = byte(v >> 24) + b[5] = byte(v >> 16) + b[6] = byte(v >> 8) + b[7] = byte(v) +} + +func (bigEndian) String() string { return "BigEndian" } + +func (bigEndian) GoString() string { return "binary.BigEndian" } + +// Read reads structured binary data from r into data. +// Data must be a pointer to a fixed-size value or a slice +// of fixed-size values. +// A fixed-size value is either a fixed-size arithmetic +// type (int8, uint8, int16, float32, complex64, ...) +// or an array or struct containing only fixed-size values. +// Bytes read from r are decoded using the specified byte order +// and written to successive fields of the data. +func Read(r io.Reader, order ByteOrder, data interface{}) os.Error { + // Fast path for basic types. + if n := intDestSize(data); n != 0 { + var b [8]byte + bs := b[:n] + if _, err := io.ReadFull(r, bs); err != nil { + return err + } + switch v := data.(type) { + case *int8: + *v = int8(b[0]) + case *uint8: + *v = b[0] + case *int16: + *v = int16(order.Uint16(bs)) + case *uint16: + *v = order.Uint16(bs) + case *int32: + *v = int32(order.Uint32(bs)) + case *uint32: + *v = order.Uint32(bs) + case *int64: + *v = int64(order.Uint64(bs)) + case *uint64: + *v = order.Uint64(bs) + } + return nil + } + + // Fallback to reflect-based. + var v reflect.Value + switch d := reflect.ValueOf(data); d.Kind() { + case reflect.Ptr: + v = d.Elem() + case reflect.Slice: + v = d + default: + return os.NewError("binary.Read: invalid type " + d.Type().String()) + } + size := TotalSize(v) + if size < 0 { + return os.NewError("binary.Read: invalid type " + v.Type().String()) + } + d := &decoder{order: order, buf: make([]byte, size)} + if _, err := io.ReadFull(r, d.buf); err != nil { + return err + } + d.value(v) + return nil +} + +// Write writes the binary representation of data into w. +// Data must be a fixed-size value or a pointer to +// a fixed-size value. +// A fixed-size value is either a fixed-size arithmetic +// type (int8, uint8, int16, float32, complex64, ...) +// or an array or struct containing only fixed-size values. +// Bytes written to w are encoded using the specified byte order +// and read from successive fields of the data. +func Write(w io.Writer, order ByteOrder, data interface{}) os.Error { + // Fast path for basic types. + var b [8]byte + var bs []byte + switch v := data.(type) { + case *int8: + bs = b[:1] + b[0] = byte(*v) + case int8: + bs = b[:1] + b[0] = byte(v) + case *uint8: + bs = b[:1] + b[0] = *v + case uint8: + bs = b[:1] + b[0] = byte(v) + case *int16: + bs = b[:2] + order.PutUint16(bs, uint16(*v)) + case int16: + bs = b[:2] + order.PutUint16(bs, uint16(v)) + case *uint16: + bs = b[:2] + order.PutUint16(bs, *v) + case uint16: + bs = b[:2] + order.PutUint16(bs, v) + case *int32: + bs = b[:4] + order.PutUint32(bs, uint32(*v)) + case int32: + bs = b[:4] + order.PutUint32(bs, uint32(v)) + case *uint32: + bs = b[:4] + order.PutUint32(bs, *v) + case uint32: + bs = b[:4] + order.PutUint32(bs, v) + case *int64: + bs = b[:8] + order.PutUint64(bs, uint64(*v)) + case int64: + bs = b[:8] + order.PutUint64(bs, uint64(v)) + case *uint64: + bs = b[:8] + order.PutUint64(bs, *v) + case uint64: + bs = b[:8] + order.PutUint64(bs, v) + } + if bs != nil { + _, err := w.Write(bs) + return err + } + v := reflect.Indirect(reflect.ValueOf(data)) + size := TotalSize(v) + if size < 0 { + return os.NewError("binary.Write: invalid type " + v.Type().String()) + } + buf := make([]byte, size) + e := &encoder{order: order, buf: buf} + e.value(v) + _, err := w.Write(buf) + return err +} + +func TotalSize(v reflect.Value) int { + if v.Kind() == reflect.Slice { + elem := sizeof(v.Type().Elem()) + if elem < 0 { + return -1 + } + return v.Len() * elem + } + return sizeof(v.Type()) +} + +func sizeof(t reflect.Type) int { + switch t.Kind() { + case reflect.Array: + n := sizeof(t.Elem()) + if n < 0 { + return -1 + } + return t.Len() * n + + case reflect.Struct: + sum := 0 + for i, n := 0, t.NumField(); i < n; i++ { + s := sizeof(t.Field(i).Type) + if s < 0 { + return -1 + } + sum += s + } + return sum + + case reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64, + reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64, + reflect.Float32, reflect.Float64, reflect.Complex64, reflect.Complex128: + return int(t.Size()) + } + return -1 +} + +type decoder struct { + order ByteOrder + buf []byte +} + +type encoder struct { + order ByteOrder + buf []byte +} + +func (d *decoder) uint8() uint8 { + x := d.buf[0] + d.buf = d.buf[1:] + return x +} + +func (e *encoder) uint8(x uint8) { + e.buf[0] = x + e.buf = e.buf[1:] +} + +func (d *decoder) uint16() uint16 { + x := d.order.Uint16(d.buf[0:2]) + d.buf = d.buf[2:] + return x +} + +func (e *encoder) uint16(x uint16) { + e.order.PutUint16(e.buf[0:2], x) + e.buf = e.buf[2:] +} + +func (d *decoder) uint32() uint32 { + x := d.order.Uint32(d.buf[0:4]) + d.buf = d.buf[4:] + return x +} + +func (e *encoder) uint32(x uint32) { + e.order.PutUint32(e.buf[0:4], x) + e.buf = e.buf[4:] +} + +func (d *decoder) uint64() uint64 { + x := d.order.Uint64(d.buf[0:8]) + d.buf = d.buf[8:] + return x +} + +func (e *encoder) uint64(x uint64) { + e.order.PutUint64(e.buf[0:8], x) + e.buf = e.buf[8:] +} + +func (d *decoder) int8() int8 { return int8(d.uint8()) } + +func (e *encoder) int8(x int8) { e.uint8(uint8(x)) } + +func (d *decoder) int16() int16 { return int16(d.uint16()) } + +func (e *encoder) int16(x int16) { e.uint16(uint16(x)) } + +func (d *decoder) int32() int32 { return int32(d.uint32()) } + +func (e *encoder) int32(x int32) { e.uint32(uint32(x)) } + +func (d *decoder) int64() int64 { return int64(d.uint64()) } + +func (e *encoder) int64(x int64) { e.uint64(uint64(x)) } + +func (d *decoder) value(v reflect.Value) { + switch v.Kind() { + case reflect.Array: + l := v.Len() + for i := 0; i < l; i++ { + d.value(v.Index(i)) + } + case reflect.Struct: + l := v.NumField() + for i := 0; i < l; i++ { + d.value(v.Field(i)) + } + + case reflect.Slice: + l := v.Len() + for i := 0; i < l; i++ { + d.value(v.Index(i)) + } + + case reflect.Int8: + v.SetInt(int64(d.int8())) + case reflect.Int16: + v.SetInt(int64(d.int16())) + case reflect.Int32: + v.SetInt(int64(d.int32())) + case reflect.Int64: + v.SetInt(d.int64()) + + case reflect.Uint8: + v.SetUint(uint64(d.uint8())) + case reflect.Uint16: + v.SetUint(uint64(d.uint16())) + case reflect.Uint32: + v.SetUint(uint64(d.uint32())) + case reflect.Uint64: + v.SetUint(d.uint64()) + + case reflect.Float32: + v.SetFloat(float64(math.Float32frombits(d.uint32()))) + case reflect.Float64: + v.SetFloat(math.Float64frombits(d.uint64())) + + case reflect.Complex64: + v.SetComplex(complex( + float64(math.Float32frombits(d.uint32())), + float64(math.Float32frombits(d.uint32())), + )) + case reflect.Complex128: + v.SetComplex(complex( + math.Float64frombits(d.uint64()), + math.Float64frombits(d.uint64()), + )) + } +} + +func (e *encoder) value(v reflect.Value) { + switch v.Kind() { + case reflect.Array: + l := v.Len() + for i := 0; i < l; i++ { + e.value(v.Index(i)) + } + case reflect.Struct: + l := v.NumField() + for i := 0; i < l; i++ { + e.value(v.Field(i)) + } + case reflect.Slice: + l := v.Len() + for i := 0; i < l; i++ { + e.value(v.Index(i)) + } + + case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64: + switch v.Type().Kind() { + case reflect.Int8: + e.int8(int8(v.Int())) + case reflect.Int16: + e.int16(int16(v.Int())) + case reflect.Int32: + e.int32(int32(v.Int())) + case reflect.Int64: + e.int64(v.Int()) + } + + case reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64, reflect.Uintptr: + switch v.Type().Kind() { + case reflect.Uint8: + e.uint8(uint8(v.Uint())) + case reflect.Uint16: + e.uint16(uint16(v.Uint())) + case reflect.Uint32: + e.uint32(uint32(v.Uint())) + case reflect.Uint64: + e.uint64(v.Uint()) + } + + case reflect.Float32, reflect.Float64: + switch v.Type().Kind() { + case reflect.Float32: + e.uint32(math.Float32bits(float32(v.Float()))) + case reflect.Float64: + e.uint64(math.Float64bits(v.Float())) + } + + case reflect.Complex64, reflect.Complex128: + switch v.Type().Kind() { + case reflect.Complex64: + x := v.Complex() + e.uint32(math.Float32bits(float32(real(x)))) + e.uint32(math.Float32bits(float32(imag(x)))) + case reflect.Complex128: + x := v.Complex() + e.uint64(math.Float64bits(real(x))) + e.uint64(math.Float64bits(imag(x))) + } + } +} + +// intDestSize returns the size of the integer that ptrType points to, +// or 0 if the type is not supported. +func intDestSize(ptrType interface{}) int { + switch ptrType.(type) { + case *int8, *uint8: + return 1 + case *int16, *uint16: + return 2 + case *int32, *uint32: + return 4 + case *int64, *uint64: + return 8 + } + return 0 +} diff --git a/src/pkg/encoding/binary/binary_test.go b/src/pkg/encoding/binary/binary_test.go new file mode 100644 index 000000000..b266996f6 --- /dev/null +++ b/src/pkg/encoding/binary/binary_test.go @@ -0,0 +1,235 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package binary + +import ( + "io" + "os" + "bytes" + "math" + "reflect" + "testing" +) + +type Struct struct { + Int8 int8 + Int16 int16 + Int32 int32 + Int64 int64 + Uint8 uint8 + Uint16 uint16 + Uint32 uint32 + Uint64 uint64 + Float32 float32 + Float64 float64 + Complex64 complex64 + Complex128 complex128 + Array [4]uint8 +} + +type T struct { + Int int + Uint uint + Uintptr uintptr + Array [4]int +} + +var s = Struct{ + 0x01, + 0x0203, + 0x04050607, + 0x08090a0b0c0d0e0f, + 0x10, + 0x1112, + 0x13141516, + 0x1718191a1b1c1d1e, + + math.Float32frombits(0x1f202122), + math.Float64frombits(0x232425262728292a), + complex( + math.Float32frombits(0x2b2c2d2e), + math.Float32frombits(0x2f303132), + ), + complex( + math.Float64frombits(0x333435363738393a), + math.Float64frombits(0x3b3c3d3e3f404142), + ), + + [4]uint8{0x43, 0x44, 0x45, 0x46}, +} + +var big = []byte{ + 1, + 2, 3, + 4, 5, 6, 7, + 8, 9, 10, 11, 12, 13, 14, 15, + 16, + 17, 18, + 19, 20, 21, 22, + 23, 24, 25, 26, 27, 28, 29, 30, + + 31, 32, 33, 34, + 35, 36, 37, 38, 39, 40, 41, 42, + 43, 44, 45, 46, 47, 48, 49, 50, + 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 62, 63, 64, 65, 66, + + 67, 68, 69, 70, +} + +var little = []byte{ + 1, + 3, 2, + 7, 6, 5, 4, + 15, 14, 13, 12, 11, 10, 9, 8, + 16, + 18, 17, + 22, 21, 20, 19, + 30, 29, 28, 27, 26, 25, 24, 23, + + 34, 33, 32, 31, + 42, 41, 40, 39, 38, 37, 36, 35, + 46, 45, 44, 43, 50, 49, 48, 47, + 58, 57, 56, 55, 54, 53, 52, 51, 66, 65, 64, 63, 62, 61, 60, 59, + + 67, 68, 69, 70, +} + +var src = []byte{1, 2, 3, 4, 5, 6, 7, 8} +var res = []int32{0x01020304, 0x05060708} + +func checkResult(t *testing.T, dir string, order, err os.Error, have, want interface{}) { + if err != nil { + t.Errorf("%v %v: %v", dir, order, err) + return + } + if !reflect.DeepEqual(have, want) { + t.Errorf("%v %v:\n\thave %+v\n\twant %+v", dir, order, have, want) + } +} + +func testRead(t *testing.T, order ByteOrder, b []byte, s1 interface{}) { + var s2 Struct + err := Read(bytes.NewBuffer(b), order, &s2) + checkResult(t, "Read", order, err, s2, s1) +} + +func testWrite(t *testing.T, order ByteOrder, b []byte, s1 interface{}) { + buf := new(bytes.Buffer) + err := Write(buf, order, s1) + checkResult(t, "Write", order, err, buf.Bytes(), b) +} + +func TestBigEndianRead(t *testing.T) { testRead(t, BigEndian, big, s) } + +func TestLittleEndianRead(t *testing.T) { testRead(t, LittleEndian, little, s) } + +func TestBigEndianWrite(t *testing.T) { testWrite(t, BigEndian, big, s) } + +func TestLittleEndianWrite(t *testing.T) { testWrite(t, LittleEndian, little, s) } + +func TestBigEndianPtrWrite(t *testing.T) { testWrite(t, BigEndian, big, &s) } + +func TestLittleEndianPtrWrite(t *testing.T) { testWrite(t, LittleEndian, little, &s) } + +func TestReadSlice(t *testing.T) { + slice := make([]int32, 2) + err := Read(bytes.NewBuffer(src), BigEndian, slice) + checkResult(t, "ReadSlice", BigEndian, err, slice, res) +} + +func TestWriteSlice(t *testing.T) { + buf := new(bytes.Buffer) + err := Write(buf, BigEndian, res) + checkResult(t, "WriteSlice", BigEndian, err, buf.Bytes(), src) +} + +func TestWriteT(t *testing.T) { + buf := new(bytes.Buffer) + ts := T{} + err := Write(buf, BigEndian, ts) + if err == nil { + t.Errorf("WriteT: have nil, want non-nil") + } + + tv := reflect.Indirect(reflect.ValueOf(ts)) + for i, n := 0, tv.NumField(); i < n; i++ { + err = Write(buf, BigEndian, tv.Field(i).Interface()) + if err == nil { + t.Errorf("WriteT.%v: have nil, want non-nil", tv.Field(i).Type()) + } + } +} + +type byteSliceReader struct { + remain []byte +} + +func (br *byteSliceReader) Read(p []byte) (int, os.Error) { + n := copy(p, br.remain) + br.remain = br.remain[n:] + return n, nil +} + +func BenchmarkRead(b *testing.B) { + var ls Struct + bsr := &byteSliceReader{} + var r io.Reader = bsr + + for i := 0; i < b.N; i++ { + bsr.remain = big + Read(r, BigEndian, &ls.Int8) + Read(r, BigEndian, &ls.Int16) + Read(r, BigEndian, &ls.Int32) + Read(r, BigEndian, &ls.Int64) + Read(r, BigEndian, &ls.Uint8) + Read(r, BigEndian, &ls.Uint16) + Read(r, BigEndian, &ls.Uint32) + Read(r, BigEndian, &ls.Uint64) + } + + want := s + want.Float32 = 0 + want.Float64 = 0 + want.Complex64 = 0 + want.Complex128 = 0 + for i := range want.Array { + want.Array[i] = 0 + } + if !reflect.DeepEqual(ls, want) { + panic("no match") + } +} + +func BenchmarkWrite(b *testing.B) { + buf := new(bytes.Buffer) + var w io.Writer = buf + + for i := 0; i < b.N; i++ { + buf.Reset() + Write(w, BigEndian, &s.Int8) + Write(w, BigEndian, &s.Int16) + Write(w, BigEndian, &s.Int32) + Write(w, BigEndian, &s.Int64) + Write(w, BigEndian, &s.Uint8) + Write(w, BigEndian, &s.Uint16) + Write(w, BigEndian, &s.Uint32) + Write(w, BigEndian, &s.Uint64) + Write(w, BigEndian, s.Int8) + Write(w, BigEndian, s.Int16) + Write(w, BigEndian, s.Int32) + Write(w, BigEndian, s.Int64) + Write(w, BigEndian, s.Uint8) + Write(w, BigEndian, s.Uint16) + Write(w, BigEndian, s.Uint32) + Write(w, BigEndian, s.Uint64) + } + + if !bytes.Equal(buf.Bytes()[:30], big[:30]) { + panic("first half doesn't match") + } + if !bytes.Equal(buf.Bytes()[30:], big[:30]) { + panic("second half doesn't match") + } +} diff --git a/src/pkg/encoding/git85/Makefile b/src/pkg/encoding/git85/Makefile new file mode 100644 index 000000000..fbd003454 --- /dev/null +++ b/src/pkg/encoding/git85/Makefile @@ -0,0 +1,11 @@ +# Copyright 2009 The Go Authors. All rights reserved. +# Use of this source code is governed by a BSD-style +# license that can be found in the LICENSE file. + +include ../../../Make.inc + +TARG=encoding/git85 +GOFILES=\ + git.go\ + +include ../../../Make.pkg diff --git a/src/pkg/encoding/git85/git.go b/src/pkg/encoding/git85/git.go new file mode 100644 index 000000000..6bb74f46c --- /dev/null +++ b/src/pkg/encoding/git85/git.go @@ -0,0 +1,277 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Package git85 implements the radix 85 data encoding +// used in the Git version control system. +package git85 + +import ( + "bytes" + "io" + "os" + "strconv" +) + +type CorruptInputError int64 + +func (e CorruptInputError) String() string { + return "illegal git85 data at input byte " + strconv.Itoa64(int64(e)) +} + +const encode = "0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz!#$%&()*+-;<=>?@^_`{|}~" + +// The decodings are 1+ the actual value, so that the +// default zero value can be used to mean "not valid". +var decode = [256]uint8{ + '0': 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, + 'A': 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, + 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, + 'a': 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, 49, + 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 62, + '!': 63, + '#': 64, 65, 66, 67, + '(': 68, 69, 70, 71, + '-': 72, + ';': 73, + '<': 74, 75, 76, 77, + '@': 78, + '^': 79, 80, 81, + '{': 82, 83, 84, 85, +} + +// Encode encodes src into EncodedLen(len(src)) +// bytes of dst. As a convenience, it returns the number +// of bytes written to dst, but this value is always EncodedLen(len(src)). +// Encode implements the radix 85 encoding used in the +// Git version control tool. +// +// The encoding splits src into chunks of at most 52 bytes +// and encodes each chunk on its own line. +func Encode(dst, src []byte) int { + ndst := 0 + for len(src) > 0 { + n := len(src) + if n > 52 { + n = 52 + } + if n <= 27 { + dst[ndst] = byte('A' + n - 1) + } else { + dst[ndst] = byte('a' + n - 26 - 1) + } + ndst++ + for i := 0; i < n; i += 4 { + var v uint32 + for j := 0; j < 4 && i+j < n; j++ { + v |= uint32(src[i+j]) << uint(24-j*8) + } + for j := 4; j >= 0; j-- { + dst[ndst+j] = encode[v%85] + v /= 85 + } + ndst += 5 + } + dst[ndst] = '\n' + ndst++ + src = src[n:] + } + return ndst +} + +// EncodedLen returns the length of an encoding of n source bytes. +func EncodedLen(n int) int { + if n == 0 { + return 0 + } + // 5 bytes per 4 bytes of input, rounded up. + // 2 extra bytes for each line of 52 src bytes, rounded up. + return (n+3)/4*5 + (n+51)/52*2 +} + +var newline = []byte{'\n'} + +// Decode decodes src into at most MaxDecodedLen(len(src)) +// bytes, returning the actual number of bytes written to dst. +// +// If Decode encounters invalid input, it returns a CorruptInputError. +// +func Decode(dst, src []byte) (n int, err os.Error) { + ndst := 0 + nsrc := 0 + for nsrc < len(src) { + var l int + switch ch := int(src[nsrc]); { + case 'A' <= ch && ch <= 'Z': + l = ch - 'A' + 1 + case 'a' <= ch && ch <= 'z': + l = ch - 'a' + 26 + 1 + default: + return ndst, CorruptInputError(nsrc) + } + if nsrc+1+l > len(src) { + return ndst, CorruptInputError(nsrc) + } + el := (l + 3) / 4 * 5 // encoded len + if nsrc+1+el+1 > len(src) || src[nsrc+1+el] != '\n' { + return ndst, CorruptInputError(nsrc) + } + line := src[nsrc+1 : nsrc+1+el] + for i := 0; i < el; i += 5 { + var v uint32 + for j := 0; j < 5; j++ { + ch := decode[line[i+j]] + if ch == 0 { + return ndst, CorruptInputError(nsrc + 1 + i + j) + } + v = v*85 + uint32(ch-1) + } + for j := 0; j < 4; j++ { + dst[ndst] = byte(v >> 24) + v <<= 8 + ndst++ + } + } + // Last fragment may have run too far (but there was room in dst). + // Back up. + if l%4 != 0 { + ndst -= 4 - l%4 + } + nsrc += 1 + el + 1 + } + return ndst, nil +} + +func MaxDecodedLen(n int) int { return n / 5 * 4 } + +// NewEncoder returns a new Git base85 stream encoder. Data written to +// the returned writer will be encoded and then written to w. +// The Git encoding operates on 52-byte blocks; when finished +// writing, the caller must Close the returned encoder to flush any +// partially written blocks. +func NewEncoder(w io.Writer) io.WriteCloser { return &encoder{w: w} } + +type encoder struct { + w io.Writer + err os.Error + buf [52]byte + nbuf int + out [1024]byte + nout int +} + +func (e *encoder) Write(p []byte) (n int, err os.Error) { + if e.err != nil { + return 0, e.err + } + + // Leading fringe. + if e.nbuf > 0 { + var i int + for i = 0; i < len(p) && e.nbuf < 52; i++ { + e.buf[e.nbuf] = p[i] + e.nbuf++ + } + n += i + p = p[i:] + if e.nbuf < 52 { + return + } + nout := Encode(e.out[0:], e.buf[0:]) + if _, e.err = e.w.Write(e.out[0:nout]); e.err != nil { + return n, e.err + } + e.nbuf = 0 + } + + // Large interior chunks. + for len(p) >= 52 { + nn := len(e.out) / (1 + 52/4*5 + 1) * 52 + if nn > len(p) { + nn = len(p) / 52 * 52 + } + if nn > 0 { + nout := Encode(e.out[0:], p[0:nn]) + if _, e.err = e.w.Write(e.out[0:nout]); e.err != nil { + return n, e.err + } + } + n += nn + p = p[nn:] + } + + // Trailing fringe. + for i := 0; i < len(p); i++ { + e.buf[i] = p[i] + } + e.nbuf = len(p) + n += len(p) + return +} + +func (e *encoder) Close() os.Error { + // If there's anything left in the buffer, flush it out + if e.err == nil && e.nbuf > 0 { + nout := Encode(e.out[0:], e.buf[0:e.nbuf]) + e.nbuf = 0 + _, e.err = e.w.Write(e.out[0:nout]) + } + return e.err +} + +// NewDecoder returns a new Git base85 stream decoder. +func NewDecoder(r io.Reader) io.Reader { return &decoder{r: r} } + +type decoder struct { + r io.Reader + err os.Error + readErr os.Error + buf [1024]byte + nbuf int + out []byte + outbuf [1024]byte + off int64 +} + +func (d *decoder) Read(p []byte) (n int, err os.Error) { + if len(p) == 0 { + return 0, nil + } + + for { + // Copy leftover output from last decode. + if len(d.out) > 0 { + n = copy(p, d.out) + d.out = d.out[n:] + return + } + + // Out of decoded output. Check errors. + if d.err != nil { + return 0, d.err + } + if d.readErr != nil { + d.err = d.readErr + return 0, d.err + } + + // Read and decode more input. + var nn int + nn, d.readErr = d.r.Read(d.buf[d.nbuf:]) + d.nbuf += nn + + // Send complete lines to Decode. + nl := bytes.LastIndex(d.buf[0:d.nbuf], newline) + if nl < 0 { + continue + } + nn, d.err = Decode(d.outbuf[0:], d.buf[0:nl+1]) + if e, ok := d.err.(CorruptInputError); ok { + d.err = CorruptInputError(int64(e) + d.off) + } + d.out = d.outbuf[0:nn] + d.nbuf = copy(d.buf[0:], d.buf[nl+1:d.nbuf]) + d.off += int64(nl + 1) + } + panic("unreachable") +} diff --git a/src/pkg/encoding/git85/git_test.go b/src/pkg/encoding/git85/git_test.go new file mode 100644 index 000000000..c76385c35 --- /dev/null +++ b/src/pkg/encoding/git85/git_test.go @@ -0,0 +1,194 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package git85 + +import ( + "bytes" + "io/ioutil" + "os" + "testing" +) + +type testpair struct { + decoded, encoded string +} + +func testEqual(t *testing.T, msg string, args ...interface{}) bool { + if args[len(args)-2] != args[len(args)-1] { + t.Errorf(msg, args...) + return false + } + return true +} + +func TestGitTable(t *testing.T) { + var saw [256]bool + for i, c := range encode { + if decode[c] != uint8(i+1) { + t.Errorf("decode['%c'] = %d, want %d", c, decode[c], i+1) + } + saw[c] = true + } + for i, b := range saw { + if !b && decode[i] != 0 { + t.Errorf("decode[%d] = %d, want 0", i, decode[i]) + } + } +} + +var gitPairs = []testpair{ + // Wikipedia example, adapted. + { + "Man is distinguished, not only by his reason, but by this singular passion from " + + "other animals, which is a lust of the mind, that by a perseverance of delight in " + + "the continued and indefatigable generation of knowledge, exceeds the short " + + "vehemence of any carnal pleasure.", + + "zO<`^zX>%ZCX>)XGZfA9Ab7*B`EFf-gbRchTYDO_b1WctXlY|;AZc?T\n" + + "zVIXXEb95kYW*~HEWgu;7Ze%PVbZB98AYyqSVIXj2a&u*NWpZI|V`U(3W*}r`Y-wj`\n" + + "zbRcPNAarPDAY*TCbZKsNWn>^>Ze$>7Ze(RV>IZ)PBC\n" + + "zZf|#NWn^b%EFfigV`XJzb0BnRWgv5CZ*p`Xc4cT~ZDnp_Wgu^6AYpEKAY);2ZeeU7\n" + + "IaBO8^b9HiME&u=k\n", + }, +} + +var gitBigtest = gitPairs[len(gitPairs)-1] + +func TestEncode(t *testing.T) { + for _, p := range gitPairs { + buf := make([]byte, EncodedLen(len(p.decoded))) + n := Encode(buf, []byte(p.decoded)) + if n != len(buf) { + t.Errorf("EncodedLen does not agree with Encode") + } + buf = buf[0:n] + testEqual(t, "Encode(%q) = %q, want %q", p.decoded, string(buf), p.encoded) + } +} + +func TestEncoder(t *testing.T) { + for _, p := range gitPairs { + bb := &bytes.Buffer{} + encoder := NewEncoder(bb) + encoder.Write([]byte(p.decoded)) + encoder.Close() + testEqual(t, "Encode(%q) = %q, want %q", p.decoded, bb.String(), p.encoded) + } +} + +func TestEncoderBuffering(t *testing.T) { + input := []byte(gitBigtest.decoded) + for bs := 1; bs <= 12; bs++ { + bb := &bytes.Buffer{} + encoder := NewEncoder(bb) + for pos := 0; pos < len(input); pos += bs { + end := pos + bs + if end > len(input) { + end = len(input) + } + n, err := encoder.Write(input[pos:end]) + testEqual(t, "Write(%q) gave error %v, want %v", input[pos:end], err, os.Error(nil)) + testEqual(t, "Write(%q) gave length %v, want %v", input[pos:end], n, end-pos) + } + err := encoder.Close() + testEqual(t, "Close gave error %v, want %v", err, os.Error(nil)) + testEqual(t, "Encoding/%d of %q = %q, want %q", bs, gitBigtest.decoded, bb.String(), gitBigtest.encoded) + } +} + +func TestDecode(t *testing.T) { + for _, p := range gitPairs { + dbuf := make([]byte, 4*len(p.encoded)) + ndst, err := Decode(dbuf, []byte(p.encoded)) + testEqual(t, "Decode(%q) = error %v, want %v", p.encoded, err, os.Error(nil)) + testEqual(t, "Decode(%q) = ndst %v, want %v", p.encoded, ndst, len(p.decoded)) + testEqual(t, "Decode(%q) = %q, want %q", p.encoded, string(dbuf[0:ndst]), p.decoded) + } +} + +func TestDecoder(t *testing.T) { + for _, p := range gitPairs { + decoder := NewDecoder(bytes.NewBufferString(p.encoded)) + dbuf, err := ioutil.ReadAll(decoder) + if err != nil { + t.Fatal("Read failed", err) + } + testEqual(t, "Read from %q = length %v, want %v", p.encoded, len(dbuf), len(p.decoded)) + testEqual(t, "Decoding of %q = %q, want %q", p.encoded, string(dbuf), p.decoded) + if err != nil { + testEqual(t, "Read from %q = %v, want %v", p.encoded, err, os.EOF) + } + } +} + +func TestDecoderBuffering(t *testing.T) { + for bs := 1; bs <= 12; bs++ { + decoder := NewDecoder(bytes.NewBufferString(gitBigtest.encoded)) + buf := make([]byte, len(gitBigtest.decoded)+12) + var total int + for total = 0; total < len(gitBigtest.decoded); { + n, err := decoder.Read(buf[total : total+bs]) + testEqual(t, "Read from %q at pos %d = %d, %v, want _, %v", gitBigtest.encoded, total, n, err, os.Error(nil)) + total += n + } + testEqual(t, "Decoding/%d of %q = %q, want %q", bs, gitBigtest.encoded, string(buf[0:total]), gitBigtest.decoded) + } +} + +func TestDecodeCorrupt(t *testing.T) { + type corrupt struct { + e string + p int + } + examples := []corrupt{ + {"v", 0}, + {"!z!!!!!!!!!", 0}, + } + + for _, e := range examples { + dbuf := make([]byte, 2*len(e.e)) + _, err := Decode(dbuf, []byte(e.e)) + switch err := err.(type) { + case CorruptInputError: + testEqual(t, "Corruption in %q at offset %v, want %v", e.e, int(err), e.p) + default: + t.Error("Decoder failed to detect corruption in", e) + } + } +} + +func TestGitBig(t *testing.T) { + n := 3*1000 + 1 + raw := make([]byte, n) + const alpha = "0123456789abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ" + for i := 0; i < n; i++ { + raw[i] = alpha[i%len(alpha)] + } + encoded := new(bytes.Buffer) + w := NewEncoder(encoded) + nn, err := w.Write(raw) + if nn != n || err != nil { + t.Fatalf("Encoder.Write(raw) = %d, %v want %d, nil", nn, err, n) + } + err = w.Close() + if err != nil { + t.Fatalf("Encoder.Close() = %v want nil", err) + } + decoded, err := ioutil.ReadAll(NewDecoder(encoded)) + if err != nil { + t.Fatalf("ioutil.ReadAll(NewDecoder(...)): %v", err) + } + + if !bytes.Equal(raw, decoded) { + var i int + for i = 0; i < len(decoded) && i < len(raw); i++ { + if decoded[i] != raw[i] { + break + } + } + t.Errorf("Decode(Encode(%d-byte string)) failed at offset %d", n, i) + } +} diff --git a/src/pkg/encoding/hex/Makefile b/src/pkg/encoding/hex/Makefile new file mode 100644 index 000000000..22049f43b --- /dev/null +++ b/src/pkg/encoding/hex/Makefile @@ -0,0 +1,11 @@ +# Copyright 2009 The Go Authors. All rights reserved. +# Use of this source code is governed by a BSD-style +# license that can be found in the LICENSE file. + +include ../../../Make.inc + +TARG=encoding/hex +GOFILES=\ + hex.go\ + +include ../../../Make.pkg diff --git a/src/pkg/encoding/hex/hex.go b/src/pkg/encoding/hex/hex.go new file mode 100644 index 000000000..e7ea8b0f2 --- /dev/null +++ b/src/pkg/encoding/hex/hex.go @@ -0,0 +1,216 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Package hex implements hexadecimal encoding and decoding. +package hex + +import ( + "bytes" + "io" + "os" + "strconv" +) + +const hextable = "0123456789abcdef" + +// EncodedLen returns the length of an encoding of n source bytes. +func EncodedLen(n int) int { return n * 2 } + +// Encode encodes src into EncodedLen(len(src)) +// bytes of dst. As a convenience, it returns the number +// of bytes written to dst, but this value is always EncodedLen(len(src)). +// Encode implements hexadecimal encoding. +func Encode(dst, src []byte) int { + for i, v := range src { + dst[i*2] = hextable[v>>4] + dst[i*2+1] = hextable[v&0x0f] + } + + return len(src) * 2 +} + +// OddLengthInputError results from decoding an odd length slice. +type OddLengthInputError struct{} + +func (OddLengthInputError) String() string { return "odd length hex string" } + +// InvalidHexCharError results from finding an invalid character in a hex string. +type InvalidHexCharError byte + +func (e InvalidHexCharError) String() string { + return "invalid hex char: " + strconv.Itoa(int(e)) +} + +func DecodedLen(x int) int { return x / 2 } + +// Decode decodes src into DecodedLen(len(src)) bytes, returning the actual +// number of bytes written to dst. +// +// If Decode encounters invalid input, it returns an OddLengthInputError or an +// InvalidHexCharError. +func Decode(dst, src []byte) (int, os.Error) { + if len(src)%2 == 1 { + return 0, OddLengthInputError{} + } + + for i := 0; i < len(src)/2; i++ { + a, ok := fromHexChar(src[i*2]) + if !ok { + return 0, InvalidHexCharError(src[i*2]) + } + b, ok := fromHexChar(src[i*2+1]) + if !ok { + return 0, InvalidHexCharError(src[i*2+1]) + } + dst[i] = (a << 4) | b + } + + return len(src) / 2, nil +} + +// fromHexChar converts a hex character into its value and a success flag. +func fromHexChar(c byte) (byte, bool) { + switch { + case '0' <= c && c <= '9': + return c - '0', true + case 'a' <= c && c <= 'f': + return c - 'a' + 10, true + case 'A' <= c && c <= 'F': + return c - 'A' + 10, true + } + + return 0, false +} + +// EncodeToString returns the hexadecimal encoding of src. +func EncodeToString(src []byte) string { + dst := make([]byte, EncodedLen(len(src))) + Encode(dst, src) + return string(dst) +} + +// DecodeString returns the bytes represented by the hexadecimal string s. +func DecodeString(s string) ([]byte, os.Error) { + src := []byte(s) + dst := make([]byte, DecodedLen(len(src))) + _, err := Decode(dst, src) + if err != nil { + return nil, err + } + return dst, nil +} + +// Dump returns a string that contains a hex dump of the given data. The format +// of the hex dump matches the output of `hexdump -C` on the command line. +func Dump(data []byte) string { + buf := bytes.NewBuffer(nil) + dumper := Dumper(buf) + dumper.Write(data) + dumper.Close() + return string(buf.Bytes()) +} + +// Dumper returns a WriteCloser that writes a hex dump of all written data to +// w. The format of the dump matches the output of `hexdump -C` on the command +// line. +func Dumper(w io.Writer) io.WriteCloser { + return &dumper{w: w} +} + +type dumper struct { + w io.Writer + rightChars [18]byte + buf [14]byte + used int // number of bytes in the current line + n uint // number of bytes, total +} + +func toChar(b byte) byte { + if b < 32 || b > 126 { + return '.' + } + return b +} + +func (h *dumper) Write(data []byte) (n int, err os.Error) { + // Output lines look like: + // 00000010 2e 2f 30 31 32 33 34 35 36 37 38 39 3a 3b 3c 3d |./0123456789:;<=| + // ^ offset ^ extra space ^ ASCII of line. + for i := range data { + if h.used == 0 { + // At the beginning of a line we print the current + // offset in hex. + h.buf[0] = byte(h.n >> 24) + h.buf[1] = byte(h.n >> 16) + h.buf[2] = byte(h.n >> 8) + h.buf[3] = byte(h.n) + Encode(h.buf[4:], h.buf[:4]) + h.buf[12] = ' ' + h.buf[13] = ' ' + _, err = h.w.Write(h.buf[4:]) + } + Encode(h.buf[:], data[i:i+1]) + h.buf[2] = ' ' + l := 3 + if h.used == 7 { + // There's an additional space after the 8th byte. + h.buf[3] = ' ' + l = 4 + } else if h.used == 15 { + // At the end of the line there's an extra space and + // the bar for the right column. + h.buf[3] = ' ' + h.buf[4] = '|' + l = 5 + } + _, err = h.w.Write(h.buf[:l]) + if err != nil { + return + } + n++ + h.rightChars[h.used] = toChar(data[i]) + h.used++ + h.n++ + if h.used == 16 { + h.rightChars[16] = '|' + h.rightChars[17] = '\n' + _, err = h.w.Write(h.rightChars[:]) + if err != nil { + return + } + h.used = 0 + } + } + return +} + +func (h *dumper) Close() (err os.Error) { + // See the comments in Write() for the details of this format. + if h.used == 0 { + return + } + h.buf[0] = ' ' + h.buf[1] = ' ' + h.buf[2] = ' ' + h.buf[3] = ' ' + h.buf[4] = '|' + nBytes := h.used + for h.used < 16 { + l := 3 + if h.used == 7 { + l = 4 + } else if h.used == 15 { + l = 5 + } + _, err = h.w.Write(h.buf[:l]) + if err != nil { + return + } + h.used++ + } + h.rightChars[nBytes] = '|' + h.rightChars[nBytes+1] = '\n' + _, err = h.w.Write(h.rightChars[:nBytes+2]) + return +} diff --git a/src/pkg/encoding/hex/hex_test.go b/src/pkg/encoding/hex/hex_test.go new file mode 100644 index 000000000..8e1838e51 --- /dev/null +++ b/src/pkg/encoding/hex/hex_test.go @@ -0,0 +1,192 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package hex + +import ( + "bytes" + "testing" +) + +type encodeTest struct { + in, out []byte +} + +var encodeTests = []encodeTest{ + {[]byte{}, []byte{}}, + {[]byte{0x01}, []byte{'0', '1'}}, + {[]byte{0xff}, []byte{'f', 'f'}}, + {[]byte{0xff, 00}, []byte{'f', 'f', '0', '0'}}, + {[]byte{0}, []byte{'0', '0'}}, + {[]byte{1}, []byte{'0', '1'}}, + {[]byte{2}, []byte{'0', '2'}}, + {[]byte{3}, []byte{'0', '3'}}, + {[]byte{4}, []byte{'0', '4'}}, + {[]byte{5}, []byte{'0', '5'}}, + {[]byte{6}, []byte{'0', '6'}}, + {[]byte{7}, []byte{'0', '7'}}, + {[]byte{8}, []byte{'0', '8'}}, + {[]byte{9}, []byte{'0', '9'}}, + {[]byte{10}, []byte{'0', 'a'}}, + {[]byte{11}, []byte{'0', 'b'}}, + {[]byte{12}, []byte{'0', 'c'}}, + {[]byte{13}, []byte{'0', 'd'}}, + {[]byte{14}, []byte{'0', 'e'}}, + {[]byte{15}, []byte{'0', 'f'}}, +} + +func TestEncode(t *testing.T) { + for i, test := range encodeTests { + dst := make([]byte, EncodedLen(len(test.in))) + n := Encode(dst, test.in) + if n != len(dst) { + t.Errorf("#%d: bad return value: got: %d want: %d", i, n, len(dst)) + } + if bytes.Compare(dst, test.out) != 0 { + t.Errorf("#%d: got: %#v want: %#v", i, dst, test.out) + } + } +} + +type decodeTest struct { + in, out []byte + ok bool +} + +var decodeTests = []decodeTest{ + {[]byte{}, []byte{}, true}, + {[]byte{'0'}, []byte{}, false}, + {[]byte{'0', 'g'}, []byte{}, false}, + {[]byte{'0', '\x01'}, []byte{}, false}, + {[]byte{'0', '0'}, []byte{0}, true}, + {[]byte{'0', '1'}, []byte{1}, true}, + {[]byte{'0', '2'}, []byte{2}, true}, + {[]byte{'0', '3'}, []byte{3}, true}, + {[]byte{'0', '4'}, []byte{4}, true}, + {[]byte{'0', '5'}, []byte{5}, true}, + {[]byte{'0', '6'}, []byte{6}, true}, + {[]byte{'0', '7'}, []byte{7}, true}, + {[]byte{'0', '8'}, []byte{8}, true}, + {[]byte{'0', '9'}, []byte{9}, true}, + {[]byte{'0', 'a'}, []byte{10}, true}, + {[]byte{'0', 'b'}, []byte{11}, true}, + {[]byte{'0', 'c'}, []byte{12}, true}, + {[]byte{'0', 'd'}, []byte{13}, true}, + {[]byte{'0', 'e'}, []byte{14}, true}, + {[]byte{'0', 'f'}, []byte{15}, true}, + {[]byte{'0', 'A'}, []byte{10}, true}, + {[]byte{'0', 'B'}, []byte{11}, true}, + {[]byte{'0', 'C'}, []byte{12}, true}, + {[]byte{'0', 'D'}, []byte{13}, true}, + {[]byte{'0', 'E'}, []byte{14}, true}, + {[]byte{'0', 'F'}, []byte{15}, true}, +} + +func TestDecode(t *testing.T) { + for i, test := range decodeTests { + dst := make([]byte, DecodedLen(len(test.in))) + n, err := Decode(dst, test.in) + if err == nil && n != len(dst) { + t.Errorf("#%d: bad return value: got:%d want:%d", i, n, len(dst)) + } + if test.ok != (err == nil) { + t.Errorf("#%d: unexpected err value: %s", i, err) + } + if err == nil && bytes.Compare(dst, test.out) != 0 { + t.Errorf("#%d: got: %#v want: %#v", i, dst, test.out) + } + } +} + +type encodeStringTest struct { + in []byte + out string +} + +var encodeStringTests = []encodeStringTest{ + {[]byte{}, ""}, + {[]byte{0}, "00"}, + {[]byte{0, 1}, "0001"}, + {[]byte{0, 1, 255}, "0001ff"}, +} + +func TestEncodeToString(t *testing.T) { + for i, test := range encodeStringTests { + s := EncodeToString(test.in) + if s != test.out { + t.Errorf("#%d got:%s want:%s", i, s, test.out) + } + } +} + +type decodeStringTest struct { + in string + out []byte + ok bool +} + +var decodeStringTests = []decodeStringTest{ + {"", []byte{}, true}, + {"0", []byte{}, false}, + {"00", []byte{0}, true}, + {"0\x01", []byte{}, false}, + {"0g", []byte{}, false}, + {"00ff00", []byte{0, 255, 0}, true}, + {"0000ff", []byte{0, 0, 255}, true}, +} + +func TestDecodeString(t *testing.T) { + for i, test := range decodeStringTests { + dst, err := DecodeString(test.in) + if test.ok != (err == nil) { + t.Errorf("#%d: unexpected err value: %s", i, err) + } + if err == nil && bytes.Compare(dst, test.out) != 0 { + t.Errorf("#%d: got: %#v want: #%v", i, dst, test.out) + } + } +} + +func TestDumper(t *testing.T) { + var in [40]byte + for i := range in { + in[i] = byte(i + 30) + } + + for stride := 1; stride < len(in); stride++ { + out := bytes.NewBuffer(nil) + dumper := Dumper(out) + done := 0 + for done < len(in) { + todo := done + stride + if todo > len(in) { + todo = len(in) + } + dumper.Write(in[done:todo]) + done = todo + } + + dumper.Close() + if !bytes.Equal(out.Bytes(), expectedHexDump) { + t.Errorf("stride: %d failed. got:\n%s\nwant:\n%s", stride, out.Bytes(), expectedHexDump) + } + } +} + +func TestDump(t *testing.T) { + var in [40]byte + for i := range in { + in[i] = byte(i + 30) + } + + out := []byte(Dump(in[:])) + if !bytes.Equal(out, expectedHexDump) { + t.Errorf("got:\n%s\nwant:\n%s", out, expectedHexDump) + } +} + +var expectedHexDump = []byte(`00000000 1e 1f 20 21 22 23 24 25 26 27 28 29 2a 2b 2c 2d |.. !"#$%&'()*+,-| +00000010 2e 2f 30 31 32 33 34 35 36 37 38 39 3a 3b 3c 3d |./0123456789:;<=| +00000020 3e 3f 40 41 42 43 44 45 |>?@ABCDE| +`) diff --git a/src/pkg/encoding/pem/Makefile b/src/pkg/encoding/pem/Makefile new file mode 100644 index 000000000..527670380 --- /dev/null +++ b/src/pkg/encoding/pem/Makefile @@ -0,0 +1,11 @@ +# Copyright 2009 The Go Authors. All rights reserved. +# Use of this source code is governed by a BSD-style +# license that can be found in the LICENSE file. + +include ../../../Make.inc + +TARG=encoding/pem +GOFILES=\ + pem.go\ + +include ../../../Make.pkg diff --git a/src/pkg/encoding/pem/pem.go b/src/pkg/encoding/pem/pem.go new file mode 100644 index 000000000..12689b57b --- /dev/null +++ b/src/pkg/encoding/pem/pem.go @@ -0,0 +1,258 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Package pem implements the PEM data encoding, which originated in Privacy +// Enhanced Mail. The most common use of PEM encoding today is in TLS keys and +// certificates. See RFC 1421. +package pem + +import ( + "bytes" + "encoding/base64" + "io" + "os" +) + +// A Block represents a PEM encoded structure. +// +// The encoded form is: +// -----BEGIN Type----- +// Headers +// base64-encoded Bytes +// -----END Type----- +// where Headers is a possibly empty sequence of Key: Value lines. +type Block struct { + Type string // The type, taken from the preamble (i.e. "RSA PRIVATE KEY"). + Headers map[string]string // Optional headers. + Bytes []byte // The decoded bytes of the contents. Typically a DER encoded ASN.1 structure. +} + +// getLine results the first \r\n or \n delineated line from the given byte +// array. The line does not include the \r\n or \n. The remainder of the byte +// array (also not including the new line bytes) is also returned and this will +// always be smaller than the original argument. +func getLine(data []byte) (line, rest []byte) { + i := bytes.Index(data, []byte{'\n'}) + var j int + if i < 0 { + i = len(data) + j = i + } else { + j = i + 1 + if i > 0 && data[i-1] == '\r' { + i-- + } + } + return data[0:i], data[j:] +} + +// removeWhitespace returns a copy of its input with all spaces, tab and +// newline characters removed. +func removeWhitespace(data []byte) []byte { + result := make([]byte, len(data)) + n := 0 + + for _, b := range data { + if b == ' ' || b == '\t' || b == '\r' || b == '\n' { + continue + } + result[n] = b + n++ + } + + return result[0:n] +} + +var pemStart = []byte("\n-----BEGIN ") +var pemEnd = []byte("\n-----END ") +var pemEndOfLine = []byte("-----") + +// Decode will find the next PEM formatted block (certificate, private key +// etc) in the input. It returns that block and the remainder of the input. If +// no PEM data is found, p is nil and the whole of the input is returned in +// rest. +func Decode(data []byte) (p *Block, rest []byte) { + // pemStart begins with a newline. However, at the very beginning of + // the byte array, we'll accept the start string without it. + rest = data + if bytes.HasPrefix(data, pemStart[1:]) { + rest = rest[len(pemStart)-1 : len(data)] + } else if i := bytes.Index(data, pemStart); i >= 0 { + rest = rest[i+len(pemStart) : len(data)] + } else { + return nil, data + } + + typeLine, rest := getLine(rest) + if !bytes.HasSuffix(typeLine, pemEndOfLine) { + return decodeError(data, rest) + } + typeLine = typeLine[0 : len(typeLine)-len(pemEndOfLine)] + + p = &Block{ + Headers: make(map[string]string), + Type: string(typeLine), + } + + for { + // This loop terminates because getLine's second result is + // always smaller than its argument. + if len(rest) == 0 { + return nil, data + } + line, next := getLine(rest) + + i := bytes.Index(line, []byte{':'}) + if i == -1 { + break + } + + // TODO(agl): need to cope with values that spread across lines. + key, val := line[0:i], line[i+1:] + key = bytes.TrimSpace(key) + val = bytes.TrimSpace(val) + p.Headers[string(key)] = string(val) + rest = next + } + + i := bytes.Index(rest, pemEnd) + if i < 0 { + return decodeError(data, rest) + } + base64Data := removeWhitespace(rest[0:i]) + + p.Bytes = make([]byte, base64.StdEncoding.DecodedLen(len(base64Data))) + n, err := base64.StdEncoding.Decode(p.Bytes, base64Data) + if err != nil { + return decodeError(data, rest) + } + p.Bytes = p.Bytes[0:n] + + _, rest = getLine(rest[i+len(pemEnd):]) + + return +} + +func decodeError(data, rest []byte) (*Block, []byte) { + // If we get here then we have rejected a likely looking, but + // ultimately invalid PEM block. We need to start over from a new + // position. We have consumed the preamble line and will have consumed + // any lines which could be header lines. However, a valid preamble + // line is not a valid header line, therefore we cannot have consumed + // the preamble line for the any subsequent block. Thus, we will always + // find any valid block, no matter what bytes precede it. + // + // For example, if the input is + // + // -----BEGIN MALFORMED BLOCK----- + // junk that may look like header lines + // or data lines, but no END line + // + // -----BEGIN ACTUAL BLOCK----- + // realdata + // -----END ACTUAL BLOCK----- + // + // we've failed to parse using the first BEGIN line + // and now will try again, using the second BEGIN line. + p, rest := Decode(rest) + if p == nil { + rest = data + } + return p, rest +} + +const pemLineLength = 64 + +type lineBreaker struct { + line [pemLineLength]byte + used int + out io.Writer +} + +func (l *lineBreaker) Write(b []byte) (n int, err os.Error) { + if l.used+len(b) < pemLineLength { + copy(l.line[l.used:], b) + l.used += len(b) + return len(b), nil + } + + n, err = l.out.Write(l.line[0:l.used]) + if err != nil { + return + } + excess := pemLineLength - l.used + l.used = 0 + + n, err = l.out.Write(b[0:excess]) + if err != nil { + return + } + + n, err = l.out.Write([]byte{'\n'}) + if err != nil { + return + } + + return l.Write(b[excess:]) +} + +func (l *lineBreaker) Close() (err os.Error) { + if l.used > 0 { + _, err = l.out.Write(l.line[0:l.used]) + if err != nil { + return + } + _, err = l.out.Write([]byte{'\n'}) + } + + return +} + +func Encode(out io.Writer, b *Block) (err os.Error) { + _, err = out.Write(pemStart[1:]) + if err != nil { + return + } + _, err = out.Write([]byte(b.Type + "-----\n")) + if err != nil { + return + } + + if len(b.Headers) > 0 { + for k, v := range b.Headers { + _, err = out.Write([]byte(k + ": " + v + "\n")) + if err != nil { + return + } + } + _, err = out.Write([]byte{'\n'}) + if err != nil { + return + } + } + + var breaker lineBreaker + breaker.out = out + + b64 := base64.NewEncoder(base64.StdEncoding, &breaker) + _, err = b64.Write(b.Bytes) + if err != nil { + return + } + b64.Close() + breaker.Close() + + _, err = out.Write(pemEnd[1:]) + if err != nil { + return + } + _, err = out.Write([]byte(b.Type + "-----\n")) + return +} + +func EncodeToMemory(b *Block) []byte { + buf := bytes.NewBuffer(nil) + Encode(buf, b) + return buf.Bytes() +} diff --git a/src/pkg/encoding/pem/pem_test.go b/src/pkg/encoding/pem/pem_test.go new file mode 100644 index 000000000..11efe5544 --- /dev/null +++ b/src/pkg/encoding/pem/pem_test.go @@ -0,0 +1,390 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package pem + +import ( + "bytes" + "reflect" + "testing" +) + +type GetLineTest struct { + in, out1, out2 string +} + +var getLineTests = []GetLineTest{ + {"abc", "abc", ""}, + {"abc\r", "abc\r", ""}, + {"abc\n", "abc", ""}, + {"abc\r\n", "abc", ""}, + {"abc\nd", "abc", "d"}, + {"abc\r\nd", "abc", "d"}, + {"\nabc", "", "abc"}, + {"\r\nabc", "", "abc"}, +} + +func TestGetLine(t *testing.T) { + for i, test := range getLineTests { + x, y := getLine([]byte(test.in)) + if string(x) != test.out1 || string(y) != test.out2 { + t.Errorf("#%d got:%+v,%+v want:%s,%s", i, x, y, test.out1, test.out2) + } + } +} + +func TestDecode(t *testing.T) { + result, remainder := Decode([]byte(pemData)) + if !reflect.DeepEqual(result, certificate) { + t.Errorf("#0 got:%#v want:%#v", result, certificate) + } + result, remainder = Decode(remainder) + if !reflect.DeepEqual(result, privateKey) { + t.Errorf("#1 got:%#v want:%#v", result, privateKey) + } + result, _ = Decode([]byte(pemPrivateKey)) + if !reflect.DeepEqual(result, privateKey2) { + t.Errorf("#2 got:%#v want:%#v", result, privateKey2) + } +} + +func TestEncode(t *testing.T) { + r := EncodeToMemory(privateKey2) + if string(r) != pemPrivateKey { + t.Errorf("got:%s want:%s", r, pemPrivateKey) + } +} + +type lineBreakerTest struct { + in, out string +} + +const sixtyFourCharString = "0123456789012345678901234567890123456789012345678901234567890123" + +var lineBreakerTests = []lineBreakerTest{ + {"", ""}, + {"a", "a\n"}, + {"ab", "ab\n"}, + {sixtyFourCharString, sixtyFourCharString + "\n"}, + {sixtyFourCharString + "X", sixtyFourCharString + "\nX\n"}, + {sixtyFourCharString + sixtyFourCharString, sixtyFourCharString + "\n" + sixtyFourCharString + "\n"}, +} + +func TestLineBreaker(t *testing.T) { + for i, test := range lineBreakerTests { + buf := bytes.NewBuffer(nil) + var breaker lineBreaker + breaker.out = buf + _, err := breaker.Write([]byte(test.in)) + if err != nil { + t.Errorf("#%d: error from Write: %s", i, err) + continue + } + err = breaker.Close() + if err != nil { + t.Errorf("#%d: error from Close: %s", i, err) + continue + } + + if string(buf.Bytes()) != test.out { + t.Errorf("#%d: got:%s want:%s", i, string(buf.Bytes()), test.out) + } + } + + for i, test := range lineBreakerTests { + buf := bytes.NewBuffer(nil) + var breaker lineBreaker + breaker.out = buf + + for i := 0; i < len(test.in); i++ { + _, err := breaker.Write([]byte(test.in[i : i+1])) + if err != nil { + t.Errorf("#%d: error from Write (byte by byte): %s", i, err) + continue + } + } + err := breaker.Close() + if err != nil { + t.Errorf("#%d: error from Close (byte by byte): %s", i, err) + continue + } + + if string(buf.Bytes()) != test.out { + t.Errorf("#%d: (byte by byte) got:%s want:%s", i, string(buf.Bytes()), test.out) + } + } +} + +var pemData = `verify return:0 +-----BEGIN CERTIFICATE----- +sdlfkjskldfj + -----BEGIN CERTIFICATE----- +--- +Certificate chain + 0 s:/C=AU/ST=Somewhere/L=Someplace/O=Foo Bar/CN=foo.example.com + i:/C=ZA/O=CA Inc./CN=CA Inc +-----BEGIN CERTIFICATE----- +testing +-----BEGIN CERTIFICATE----- +-----BEGIN CERTIFICATE----- +MIID6TCCA1ICAQEwDQYJKoZIhvcNAQEFBQAwgYsxCzAJBgNVBAYTAlVTMRMwEQYD +VQQIEwpDYWxpZm9ybmlhMRYwFAYDVQQHEw1TYW4gRnJhbmNpc2NvMRQwEgYDVQQK +EwtHb29nbGUgSW5jLjEMMAoGA1UECxMDRW5nMQwwCgYDVQQDEwNhZ2wxHTAbBgkq +hkiG9w0BCQEWDmFnbEBnb29nbGUuY29tMB4XDTA5MDkwOTIyMDU0M1oXDTEwMDkw +OTIyMDU0M1owajELMAkGA1UEBhMCQVUxEzARBgNVBAgTClNvbWUtU3RhdGUxITAf +BgNVBAoTGEludGVybmV0IFdpZGdpdHMgUHR5IEx0ZDEjMCEGA1UEAxMaZXVyb3Bh +LnNmby5jb3JwLmdvb2dsZS5jb20wggIiMA0GCSqGSIb3DQEBAQUAA4ICDwAwggIK +AoICAQC6pgYt7/EibBDumASF+S0qvqdL/f+nouJw2T1Qc8GmXF/iiUcrsgzh/Fd8 +pDhz/T96Qg9IyR4ztuc2MXrmPra+zAuSf5bevFReSqvpIt8Duv0HbDbcqs/XKPfB +uMDe+of7a9GCywvAZ4ZUJcp0thqD9fKTTjUWOBzHY1uNE4RitrhmJCrbBGXbJ249 +bvgmb7jgdInH2PU7PT55hujvOoIsQW2osXBFRur4pF1wmVh4W4lTLD6pjfIMUcML +ICHEXEN73PDic8KS3EtNYCwoIld+tpIBjE1QOb1KOyuJBNW6Esw9ALZn7stWdYcE +qAwvv20egN2tEXqj7Q4/1ccyPZc3PQgC3FJ8Be2mtllM+80qf4dAaQ/fWvCtOrQ5 +pnfe9juQvCo8Y0VGlFcrSys/MzSg9LJ/24jZVgzQved/Qupsp89wVidwIzjt+WdS +fyWfH0/v1aQLvu5cMYuW//C0W2nlYziL5blETntM8My2ybNARy3ICHxCBv2RNtPI +WQVm+E9/W5rwh2IJR4DHn2LHwUVmT/hHNTdBLl5Uhwr4Wc7JhE7AVqb14pVNz1lr +5jxsp//ncIwftb7mZQ3DF03Yna+jJhpzx8CQoeLT6aQCHyzmH68MrHHT4MALPyUs +Pomjn71GNTtDeWAXibjCgdL6iHACCF6Htbl0zGlG0OAK+bdn0QIDAQABMA0GCSqG +SIb3DQEBBQUAA4GBAOKnQDtqBV24vVqvesL5dnmyFpFPXBn3WdFfwD6DzEb21UVG +5krmJiu+ViipORJPGMkgoL6BjU21XI95VQbun5P8vvg8Z+FnFsvRFY3e1CCzAVQY +ZsUkLw2I7zI/dNlWdB8Xp7v+3w9sX5N3J/WuJ1KOO5m26kRlHQo7EzT3974g +-----END CERTIFICATE----- + 1 s:/C=ZA/O=Ca Inc./CN=CA Inc + +-----BEGIN RSA PRIVATE KEY----- +Proc-Type: 4,ENCRYPTED +DEK-Info: DES-EDE3-CBC,80C7C7A09690757A + +eQp5ZkH6CyHBz7BZfUPxyLCCmftsBJ7HlqGb8Ld21cSwnzWZ4/SIlhyrUtsfw7VR +2TTwA+odo9ex7GdxOTaH8oZFumIRoiEjHsk8U7Bhntp+ekkPP79xunnN7hb7hkhr +yGDQZgA7s2cQHQ71v3gwT2BACAft26jCjbM1wgNzBnJ8M0Rzn68YWqaPtdBu8qb/ +zVR5JB1mnqvTSbFsfF5yMc6o2WQ9jJCl6KypnMl+BpL+dlvdjYVK4l9lYsB1Hs3d ++zDBbWxos818zzhS8/y6eIfiSG27cqrbhURbmgiSfDXjncK4m/pLcQ7mmBL6mFOr +3Pj4jepzgOiFRL6MKE//h62fZvI1ErYr8VunHEykgKNhChDvb1RO6LEfqKBu+Ivw +TB6fBhW3TCLMnVPYVoYwA+fHNTmZZm8BEonlIMfI+KktjWUg4Oia+NI6vKcPpFox +hSnlGgCtvfEaq5/H4kHJp95eOpnFsLviw2seHNkz/LxJMRP1X428+DpYW/QD/0JU +tJSuC/q9FUHL6RI3u/Asrv8pCb4+D7i1jW/AMIdJTtycOGsbPxQA7yHMWujHmeb1 +BTiHcL3s3KrJu1vDVrshvxfnz71KTeNnZH8UbOqT5i7fPGyXtY1XJddcbI/Q6tXf +wHFsZc20TzSdsVLBtwksUacpbDogcEVMctnNrB8FIrB3vZEv9Q0Z1VeY7nmTpF+6 +a+z2P7acL7j6A6Pr3+q8P9CPiPC7zFonVzuVPyB8GchGR2hytyiOVpuD9+k8hcuw +ZWAaUoVtWIQ52aKS0p19G99hhb+IVANC4akkdHV4SP8i7MVNZhfUmg== +-----END RSA PRIVATE KEY-----` + +var certificate = &Block{Type: "CERTIFICATE", + Headers: map[string]string{}, + Bytes: []uint8{0x30, 0x82, 0x3, 0xe9, 0x30, 0x82, 0x3, 0x52, 0x2, 0x1, + 0x1, 0x30, 0xd, 0x6, 0x9, 0x2a, 0x86, 0x48, 0x86, 0xf7, 0xd, + 0x1, 0x1, 0x5, 0x5, 0x0, 0x30, 0x81, 0x8b, 0x31, 0xb, 0x30, + 0x9, 0x6, 0x3, 0x55, 0x4, 0x6, 0x13, 0x2, 0x55, 0x53, 0x31, + 0x13, 0x30, 0x11, 0x6, 0x3, 0x55, 0x4, 0x8, 0x13, 0xa, 0x43, + 0x61, 0x6c, 0x69, 0x66, 0x6f, 0x72, 0x6e, 0x69, 0x61, 0x31, + 0x16, 0x30, 0x14, 0x6, 0x3, 0x55, 0x4, 0x7, 0x13, 0xd, 0x53, + 0x61, 0x6e, 0x20, 0x46, 0x72, 0x61, 0x6e, 0x63, 0x69, 0x73, + 0x63, 0x6f, 0x31, 0x14, 0x30, 0x12, 0x6, 0x3, 0x55, 0x4, 0xa, + 0x13, 0xb, 0x47, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x20, 0x49, + 0x6e, 0x63, 0x2e, 0x31, 0xc, 0x30, 0xa, 0x6, 0x3, 0x55, 0x4, + 0xb, 0x13, 0x3, 0x45, 0x6e, 0x67, 0x31, 0xc, 0x30, 0xa, 0x6, + 0x3, 0x55, 0x4, 0x3, 0x13, 0x3, 0x61, 0x67, 0x6c, 0x31, 0x1d, + 0x30, 0x1b, 0x6, 0x9, 0x2a, 0x86, 0x48, 0x86, 0xf7, 0xd, 0x1, + 0x9, 0x1, 0x16, 0xe, 0x61, 0x67, 0x6c, 0x40, 0x67, 0x6f, 0x6f, + 0x67, 0x6c, 0x65, 0x2e, 0x63, 0x6f, 0x6d, 0x30, 0x1e, 0x17, + 0xd, 0x30, 0x39, 0x30, 0x39, 0x30, 0x39, 0x32, 0x32, 0x30, + 0x35, 0x34, 0x33, 0x5a, 0x17, 0xd, 0x31, 0x30, 0x30, 0x39, + 0x30, 0x39, 0x32, 0x32, 0x30, 0x35, 0x34, 0x33, 0x5a, 0x30, + 0x6a, 0x31, 0xb, 0x30, 0x9, 0x6, 0x3, 0x55, 0x4, 0x6, 0x13, + 0x2, 0x41, 0x55, 0x31, 0x13, 0x30, 0x11, 0x6, 0x3, 0x55, 0x4, + 0x8, 0x13, 0xa, 0x53, 0x6f, 0x6d, 0x65, 0x2d, 0x53, 0x74, 0x61, + 0x74, 0x65, 0x31, 0x21, 0x30, 0x1f, 0x6, 0x3, 0x55, 0x4, 0xa, + 0x13, 0x18, 0x49, 0x6e, 0x74, 0x65, 0x72, 0x6e, 0x65, 0x74, + 0x20, 0x57, 0x69, 0x64, 0x67, 0x69, 0x74, 0x73, 0x20, 0x50, + 0x74, 0x79, 0x20, 0x4c, 0x74, 0x64, 0x31, 0x23, 0x30, 0x21, + 0x6, 0x3, 0x55, 0x4, 0x3, 0x13, 0x1a, 0x65, 0x75, 0x72, 0x6f, + 0x70, 0x61, 0x2e, 0x73, 0x66, 0x6f, 0x2e, 0x63, 0x6f, 0x72, + 0x70, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x63, + 0x6f, 0x6d, 0x30, 0x82, 0x2, 0x22, 0x30, 0xd, 0x6, 0x9, 0x2a, + 0x86, 0x48, 0x86, 0xf7, 0xd, 0x1, 0x1, 0x1, 0x5, 0x0, 0x3, + 0x82, 0x2, 0xf, 0x0, 0x30, 0x82, 0x2, 0xa, 0x2, 0x82, 0x2, 0x1, + 0x0, 0xba, 0xa6, 0x6, 0x2d, 0xef, 0xf1, 0x22, 0x6c, 0x10, 0xee, + 0x98, 0x4, 0x85, 0xf9, 0x2d, 0x2a, 0xbe, 0xa7, 0x4b, 0xfd, + 0xff, 0xa7, 0xa2, 0xe2, 0x70, 0xd9, 0x3d, 0x50, 0x73, 0xc1, + 0xa6, 0x5c, 0x5f, 0xe2, 0x89, 0x47, 0x2b, 0xb2, 0xc, 0xe1, + 0xfc, 0x57, 0x7c, 0xa4, 0x38, 0x73, 0xfd, 0x3f, 0x7a, 0x42, + 0xf, 0x48, 0xc9, 0x1e, 0x33, 0xb6, 0xe7, 0x36, 0x31, 0x7a, + 0xe6, 0x3e, 0xb6, 0xbe, 0xcc, 0xb, 0x92, 0x7f, 0x96, 0xde, + 0xbc, 0x54, 0x5e, 0x4a, 0xab, 0xe9, 0x22, 0xdf, 0x3, 0xba, + 0xfd, 0x7, 0x6c, 0x36, 0xdc, 0xaa, 0xcf, 0xd7, 0x28, 0xf7, + 0xc1, 0xb8, 0xc0, 0xde, 0xfa, 0x87, 0xfb, 0x6b, 0xd1, 0x82, + 0xcb, 0xb, 0xc0, 0x67, 0x86, 0x54, 0x25, 0xca, 0x74, 0xb6, + 0x1a, 0x83, 0xf5, 0xf2, 0x93, 0x4e, 0x35, 0x16, 0x38, 0x1c, + 0xc7, 0x63, 0x5b, 0x8d, 0x13, 0x84, 0x62, 0xb6, 0xb8, 0x66, + 0x24, 0x2a, 0xdb, 0x4, 0x65, 0xdb, 0x27, 0x6e, 0x3d, 0x6e, + 0xf8, 0x26, 0x6f, 0xb8, 0xe0, 0x74, 0x89, 0xc7, 0xd8, 0xf5, + 0x3b, 0x3d, 0x3e, 0x79, 0x86, 0xe8, 0xef, 0x3a, 0x82, 0x2c, + 0x41, 0x6d, 0xa8, 0xb1, 0x70, 0x45, 0x46, 0xea, 0xf8, 0xa4, + 0x5d, 0x70, 0x99, 0x58, 0x78, 0x5b, 0x89, 0x53, 0x2c, 0x3e, + 0xa9, 0x8d, 0xf2, 0xc, 0x51, 0xc3, 0xb, 0x20, 0x21, 0xc4, 0x5c, + 0x43, 0x7b, 0xdc, 0xf0, 0xe2, 0x73, 0xc2, 0x92, 0xdc, 0x4b, + 0x4d, 0x60, 0x2c, 0x28, 0x22, 0x57, 0x7e, 0xb6, 0x92, 0x1, + 0x8c, 0x4d, 0x50, 0x39, 0xbd, 0x4a, 0x3b, 0x2b, 0x89, 0x4, + 0xd5, 0xba, 0x12, 0xcc, 0x3d, 0x0, 0xb6, 0x67, 0xee, 0xcb, + 0x56, 0x75, 0x87, 0x4, 0xa8, 0xc, 0x2f, 0xbf, 0x6d, 0x1e, 0x80, + 0xdd, 0xad, 0x11, 0x7a, 0xa3, 0xed, 0xe, 0x3f, 0xd5, 0xc7, + 0x32, 0x3d, 0x97, 0x37, 0x3d, 0x8, 0x2, 0xdc, 0x52, 0x7c, 0x5, + 0xed, 0xa6, 0xb6, 0x59, 0x4c, 0xfb, 0xcd, 0x2a, 0x7f, 0x87, + 0x40, 0x69, 0xf, 0xdf, 0x5a, 0xf0, 0xad, 0x3a, 0xb4, 0x39, + 0xa6, 0x77, 0xde, 0xf6, 0x3b, 0x90, 0xbc, 0x2a, 0x3c, 0x63, + 0x45, 0x46, 0x94, 0x57, 0x2b, 0x4b, 0x2b, 0x3f, 0x33, 0x34, + 0xa0, 0xf4, 0xb2, 0x7f, 0xdb, 0x88, 0xd9, 0x56, 0xc, 0xd0, + 0xbd, 0xe7, 0x7f, 0x42, 0xea, 0x6c, 0xa7, 0xcf, 0x70, 0x56, + 0x27, 0x70, 0x23, 0x38, 0xed, 0xf9, 0x67, 0x52, 0x7f, 0x25, + 0x9f, 0x1f, 0x4f, 0xef, 0xd5, 0xa4, 0xb, 0xbe, 0xee, 0x5c, + 0x31, 0x8b, 0x96, 0xff, 0xf0, 0xb4, 0x5b, 0x69, 0xe5, 0x63, + 0x38, 0x8b, 0xe5, 0xb9, 0x44, 0x4e, 0x7b, 0x4c, 0xf0, 0xcc, + 0xb6, 0xc9, 0xb3, 0x40, 0x47, 0x2d, 0xc8, 0x8, 0x7c, 0x42, 0x6, + 0xfd, 0x91, 0x36, 0xd3, 0xc8, 0x59, 0x5, 0x66, 0xf8, 0x4f, + 0x7f, 0x5b, 0x9a, 0xf0, 0x87, 0x62, 0x9, 0x47, 0x80, 0xc7, + 0x9f, 0x62, 0xc7, 0xc1, 0x45, 0x66, 0x4f, 0xf8, 0x47, 0x35, + 0x37, 0x41, 0x2e, 0x5e, 0x54, 0x87, 0xa, 0xf8, 0x59, 0xce, + 0xc9, 0x84, 0x4e, 0xc0, 0x56, 0xa6, 0xf5, 0xe2, 0x95, 0x4d, + 0xcf, 0x59, 0x6b, 0xe6, 0x3c, 0x6c, 0xa7, 0xff, 0xe7, 0x70, + 0x8c, 0x1f, 0xb5, 0xbe, 0xe6, 0x65, 0xd, 0xc3, 0x17, 0x4d, + 0xd8, 0x9d, 0xaf, 0xa3, 0x26, 0x1a, 0x73, 0xc7, 0xc0, 0x90, + 0xa1, 0xe2, 0xd3, 0xe9, 0xa4, 0x2, 0x1f, 0x2c, 0xe6, 0x1f, + 0xaf, 0xc, 0xac, 0x71, 0xd3, 0xe0, 0xc0, 0xb, 0x3f, 0x25, 0x2c, + 0x3e, 0x89, 0xa3, 0x9f, 0xbd, 0x46, 0x35, 0x3b, 0x43, 0x79, + 0x60, 0x17, 0x89, 0xb8, 0xc2, 0x81, 0xd2, 0xfa, 0x88, 0x70, + 0x2, 0x8, 0x5e, 0x87, 0xb5, 0xb9, 0x74, 0xcc, 0x69, 0x46, 0xd0, + 0xe0, 0xa, 0xf9, 0xb7, 0x67, 0xd1, 0x2, 0x3, 0x1, 0x0, 0x1, + 0x30, 0xd, 0x6, 0x9, 0x2a, 0x86, 0x48, 0x86, 0xf7, 0xd, 0x1, + 0x1, 0x5, 0x5, 0x0, 0x3, 0x81, 0x81, 0x0, 0xe2, 0xa7, 0x40, + 0x3b, 0x6a, 0x5, 0x5d, 0xb8, 0xbd, 0x5a, 0xaf, 0x7a, 0xc2, + 0xf9, 0x76, 0x79, 0xb2, 0x16, 0x91, 0x4f, 0x5c, 0x19, 0xf7, + 0x59, 0xd1, 0x5f, 0xc0, 0x3e, 0x83, 0xcc, 0x46, 0xf6, 0xd5, + 0x45, 0x46, 0xe6, 0x4a, 0xe6, 0x26, 0x2b, 0xbe, 0x56, 0x28, + 0xa9, 0x39, 0x12, 0x4f, 0x18, 0xc9, 0x20, 0xa0, 0xbe, 0x81, + 0x8d, 0x4d, 0xb5, 0x5c, 0x8f, 0x79, 0x55, 0x6, 0xee, 0x9f, + 0x93, 0xfc, 0xbe, 0xf8, 0x3c, 0x67, 0xe1, 0x67, 0x16, 0xcb, + 0xd1, 0x15, 0x8d, 0xde, 0xd4, 0x20, 0xb3, 0x1, 0x54, 0x18, + 0x66, 0xc5, 0x24, 0x2f, 0xd, 0x88, 0xef, 0x32, 0x3f, 0x74, + 0xd9, 0x56, 0x74, 0x1f, 0x17, 0xa7, 0xbb, 0xfe, 0xdf, 0xf, + 0x6c, 0x5f, 0x93, 0x77, 0x27, 0xf5, 0xae, 0x27, 0x52, 0x8e, + 0x3b, 0x99, 0xb6, 0xea, 0x44, 0x65, 0x1d, 0xa, 0x3b, 0x13, + 0x34, 0xf7, 0xf7, 0xbe, 0x20, + }, +} + +var privateKey = &Block{Type: "RSA PRIVATE KEY", + Headers: map[string]string{"DEK-Info": "DES-EDE3-CBC,80C7C7A09690757A", "Proc-Type": "4,ENCRYPTED"}, + Bytes: []uint8{0x79, 0xa, 0x79, 0x66, 0x41, 0xfa, 0xb, + 0x21, 0xc1, 0xcf, 0xb0, 0x59, 0x7d, 0x43, 0xf1, 0xc8, 0xb0, + 0x82, 0x99, 0xfb, 0x6c, 0x4, 0x9e, 0xc7, 0x96, 0xa1, 0x9b, + 0xf0, 0xb7, 0x76, 0xd5, 0xc4, 0xb0, 0x9f, 0x35, 0x99, 0xe3, + 0xf4, 0x88, 0x96, 0x1c, 0xab, 0x52, 0xdb, 0x1f, 0xc3, 0xb5, + 0x51, 0xd9, 0x34, 0xf0, 0x3, 0xea, 0x1d, 0xa3, 0xd7, 0xb1, + 0xec, 0x67, 0x71, 0x39, 0x36, 0x87, 0xf2, 0x86, 0x45, 0xba, + 0x62, 0x11, 0xa2, 0x21, 0x23, 0x1e, 0xc9, 0x3c, 0x53, 0xb0, + 0x61, 0x9e, 0xda, 0x7e, 0x7a, 0x49, 0xf, 0x3f, 0xbf, 0x71, + 0xba, 0x79, 0xcd, 0xee, 0x16, 0xfb, 0x86, 0x48, 0x6b, 0xc8, + 0x60, 0xd0, 0x66, 0x0, 0x3b, 0xb3, 0x67, 0x10, 0x1d, 0xe, + 0xf5, 0xbf, 0x78, 0x30, 0x4f, 0x60, 0x40, 0x8, 0x7, 0xed, + 0xdb, 0xa8, 0xc2, 0x8d, 0xb3, 0x35, 0xc2, 0x3, 0x73, 0x6, + 0x72, 0x7c, 0x33, 0x44, 0x73, 0x9f, 0xaf, 0x18, 0x5a, 0xa6, + 0x8f, 0xb5, 0xd0, 0x6e, 0xf2, 0xa6, 0xff, 0xcd, 0x54, 0x79, + 0x24, 0x1d, 0x66, 0x9e, 0xab, 0xd3, 0x49, 0xb1, 0x6c, 0x7c, + 0x5e, 0x72, 0x31, 0xce, 0xa8, 0xd9, 0x64, 0x3d, 0x8c, 0x90, + 0xa5, 0xe8, 0xac, 0xa9, 0x9c, 0xc9, 0x7e, 0x6, 0x92, 0xfe, + 0x76, 0x5b, 0xdd, 0x8d, 0x85, 0x4a, 0xe2, 0x5f, 0x65, 0x62, + 0xc0, 0x75, 0x1e, 0xcd, 0xdd, 0xfb, 0x30, 0xc1, 0x6d, 0x6c, + 0x68, 0xb3, 0xcd, 0x7c, 0xcf, 0x38, 0x52, 0xf3, 0xfc, 0xba, + 0x78, 0x87, 0xe2, 0x48, 0x6d, 0xbb, 0x72, 0xaa, 0xdb, 0x85, + 0x44, 0x5b, 0x9a, 0x8, 0x92, 0x7c, 0x35, 0xe3, 0x9d, 0xc2, + 0xb8, 0x9b, 0xfa, 0x4b, 0x71, 0xe, 0xe6, 0x98, 0x12, 0xfa, + 0x98, 0x53, 0xab, 0xdc, 0xf8, 0xf8, 0x8d, 0xea, 0x73, 0x80, + 0xe8, 0x85, 0x44, 0xbe, 0x8c, 0x28, 0x4f, 0xff, 0x87, 0xad, + 0x9f, 0x66, 0xf2, 0x35, 0x12, 0xb6, 0x2b, 0xf1, 0x5b, 0xa7, + 0x1c, 0x4c, 0xa4, 0x80, 0xa3, 0x61, 0xa, 0x10, 0xef, 0x6f, + 0x54, 0x4e, 0xe8, 0xb1, 0x1f, 0xa8, 0xa0, 0x6e, 0xf8, 0x8b, + 0xf0, 0x4c, 0x1e, 0x9f, 0x6, 0x15, 0xb7, 0x4c, 0x22, 0xcc, + 0x9d, 0x53, 0xd8, 0x56, 0x86, 0x30, 0x3, 0xe7, 0xc7, 0x35, + 0x39, 0x99, 0x66, 0x6f, 0x1, 0x12, 0x89, 0xe5, 0x20, 0xc7, + 0xc8, 0xf8, 0xa9, 0x2d, 0x8d, 0x65, 0x20, 0xe0, 0xe8, 0x9a, + 0xf8, 0xd2, 0x3a, 0xbc, 0xa7, 0xf, 0xa4, 0x5a, 0x31, 0x85, + 0x29, 0xe5, 0x1a, 0x0, 0xad, 0xbd, 0xf1, 0x1a, 0xab, 0x9f, + 0xc7, 0xe2, 0x41, 0xc9, 0xa7, 0xde, 0x5e, 0x3a, 0x99, 0xc5, + 0xb0, 0xbb, 0xe2, 0xc3, 0x6b, 0x1e, 0x1c, 0xd9, 0x33, 0xfc, + 0xbc, 0x49, 0x31, 0x13, 0xf5, 0x5f, 0x8d, 0xbc, 0xf8, 0x3a, + 0x58, 0x5b, 0xf4, 0x3, 0xff, 0x42, 0x54, 0xb4, 0x94, 0xae, + 0xb, 0xfa, 0xbd, 0x15, 0x41, 0xcb, 0xe9, 0x12, 0x37, 0xbb, + 0xf0, 0x2c, 0xae, 0xff, 0x29, 0x9, 0xbe, 0x3e, 0xf, 0xb8, + 0xb5, 0x8d, 0x6f, 0xc0, 0x30, 0x87, 0x49, 0x4e, 0xdc, 0x9c, + 0x38, 0x6b, 0x1b, 0x3f, 0x14, 0x0, 0xef, 0x21, 0xcc, 0x5a, + 0xe8, 0xc7, 0x99, 0xe6, 0xf5, 0x5, 0x38, 0x87, 0x70, 0xbd, + 0xec, 0xdc, 0xaa, 0xc9, 0xbb, 0x5b, 0xc3, 0x56, 0xbb, 0x21, + 0xbf, 0x17, 0xe7, 0xcf, 0xbd, 0x4a, 0x4d, 0xe3, 0x67, 0x64, + 0x7f, 0x14, 0x6c, 0xea, 0x93, 0xe6, 0x2e, 0xdf, 0x3c, 0x6c, + 0x97, 0xb5, 0x8d, 0x57, 0x25, 0xd7, 0x5c, 0x6c, 0x8f, 0xd0, + 0xea, 0xd5, 0xdf, 0xc0, 0x71, 0x6c, 0x65, 0xcd, 0xb4, 0x4f, + 0x34, 0x9d, 0xb1, 0x52, 0xc1, 0xb7, 0x9, 0x2c, 0x51, 0xa7, + 0x29, 0x6c, 0x3a, 0x20, 0x70, 0x45, 0x4c, 0x72, 0xd9, 0xcd, + 0xac, 0x1f, 0x5, 0x22, 0xb0, 0x77, 0xbd, 0x91, 0x2f, 0xf5, + 0xd, 0x19, 0xd5, 0x57, 0x98, 0xee, 0x79, 0x93, 0xa4, 0x5f, + 0xba, 0x6b, 0xec, 0xf6, 0x3f, 0xb6, 0x9c, 0x2f, 0xb8, 0xfa, + 0x3, 0xa3, 0xeb, 0xdf, 0xea, 0xbc, 0x3f, 0xd0, 0x8f, 0x88, + 0xf0, 0xbb, 0xcc, 0x5a, 0x27, 0x57, 0x3b, 0x95, 0x3f, 0x20, + 0x7c, 0x19, 0xc8, 0x46, 0x47, 0x68, 0x72, 0xb7, 0x28, 0x8e, + 0x56, 0x9b, 0x83, 0xf7, 0xe9, 0x3c, 0x85, 0xcb, 0xb0, 0x65, + 0x60, 0x1a, 0x52, 0x85, 0x6d, 0x58, 0x84, 0x39, 0xd9, 0xa2, + 0x92, 0xd2, 0x9d, 0x7d, 0x1b, 0xdf, 0x61, 0x85, 0xbf, 0x88, + 0x54, 0x3, 0x42, 0xe1, 0xa9, 0x24, 0x74, 0x75, 0x78, 0x48, + 0xff, 0x22, 0xec, 0xc5, 0x4d, 0x66, 0x17, 0xd4, 0x9a, + }, +} + +var privateKey2 = &Block{Type: "RSA PRIVATE KEY", + Headers: map[string]string{}, + Bytes: []uint8{0x30, 0x82, 0x1, 0x3a, 0x2, 0x1, 0x0, 0x2, + 0x41, 0x0, 0xb2, 0x99, 0xf, 0x49, 0xc4, 0x7d, 0xfa, 0x8c, + 0xd4, 0x0, 0xae, 0x6a, 0x4d, 0x1b, 0x8a, 0x3b, 0x6a, 0x13, + 0x64, 0x2b, 0x23, 0xf2, 0x8b, 0x0, 0x3b, 0xfb, 0x97, 0x79, + 0xa, 0xde, 0x9a, 0x4c, 0xc8, 0x2b, 0x8b, 0x2a, 0x81, 0x74, + 0x7d, 0xde, 0xc0, 0x8b, 0x62, 0x96, 0xe5, 0x3a, 0x8, 0xc3, + 0x31, 0x68, 0x7e, 0xf2, 0x5c, 0x4b, 0xf4, 0x93, 0x6b, 0xa1, + 0xc0, 0xe6, 0x4, 0x1e, 0x9d, 0x15, 0x2, 0x3, 0x1, 0x0, 0x1, + 0x2, 0x41, 0x0, 0x8a, 0xbd, 0x6a, 0x69, 0xf4, 0xd1, 0xa4, + 0xb4, 0x87, 0xf0, 0xab, 0x8d, 0x7a, 0xae, 0xfd, 0x38, 0x60, + 0x94, 0x5, 0xc9, 0x99, 0x98, 0x4e, 0x30, 0xf5, 0x67, 0xe1, + 0xe8, 0xae, 0xef, 0xf4, 0x4e, 0x8b, 0x18, 0xbd, 0xb1, 0xec, + 0x78, 0xdf, 0xa3, 0x1a, 0x55, 0xe3, 0x2a, 0x48, 0xd7, 0xfb, + 0x13, 0x1f, 0x5a, 0xf1, 0xf4, 0x4d, 0x7d, 0x6b, 0x2c, 0xed, + 0x2a, 0x9d, 0xf5, 0xe5, 0xae, 0x45, 0x35, 0x2, 0x21, 0x0, + 0xda, 0xb2, 0xf1, 0x80, 0x48, 0xba, 0xa6, 0x8d, 0xe7, 0xdf, + 0x4, 0xd2, 0xd3, 0x5d, 0x5d, 0x80, 0xe6, 0xe, 0x2d, 0xfa, + 0x42, 0xd5, 0xa, 0x9b, 0x4, 0x21, 0x90, 0x32, 0x71, 0x5e, + 0x46, 0xb3, 0x2, 0x21, 0x0, 0xd1, 0xf, 0x2e, 0x66, 0xb1, + 0xd0, 0xc1, 0x3f, 0x10, 0xef, 0x99, 0x27, 0xbf, 0x53, 0x24, + 0xa3, 0x79, 0xca, 0x21, 0x81, 0x46, 0xcb, 0xf9, 0xca, 0xfc, + 0x79, 0x52, 0x21, 0xf1, 0x6a, 0x31, 0x17, 0x2, 0x20, 0x21, + 0x2, 0x89, 0x79, 0x37, 0x81, 0x14, 0xca, 0xae, 0x88, 0xf7, + 0xd, 0x6b, 0x61, 0xd8, 0x4f, 0x30, 0x6a, 0x4b, 0x7e, 0x4e, + 0xc0, 0x21, 0x4d, 0xac, 0x9d, 0xf4, 0x49, 0xe8, 0xda, 0xb6, + 0x9, 0x2, 0x20, 0x16, 0xb3, 0xec, 0x59, 0x10, 0xa4, 0x57, + 0xe8, 0xe, 0x61, 0xc6, 0xa3, 0xf, 0x5e, 0xeb, 0x12, 0xa9, + 0xae, 0x2e, 0xb7, 0x48, 0x45, 0xec, 0x69, 0x83, 0xc3, 0x75, + 0xc, 0xe4, 0x97, 0xa0, 0x9f, 0x2, 0x20, 0x69, 0x52, 0xb4, + 0x6, 0xe8, 0x50, 0x60, 0x71, 0x4c, 0x3a, 0xb7, 0x66, 0xba, + 0xd, 0x8a, 0xc9, 0xb7, 0xd, 0xa3, 0x8, 0x6c, 0xa3, 0xf2, + 0x62, 0xb0, 0x2a, 0x84, 0xaa, 0x2f, 0xd6, 0x1e, 0x55, + }, +} + +var pemPrivateKey = `-----BEGIN RSA PRIVATE KEY----- +MIIBOgIBAAJBALKZD0nEffqM1ACuak0bijtqE2QrI/KLADv7l3kK3ppMyCuLKoF0 +fd7Ai2KW5ToIwzFofvJcS/STa6HA5gQenRUCAwEAAQJBAIq9amn00aS0h/CrjXqu +/ThglAXJmZhOMPVn4eiu7/ROixi9sex436MaVeMqSNf7Ex9a8fRNfWss7Sqd9eWu +RTUCIQDasvGASLqmjeffBNLTXV2A5g4t+kLVCpsEIZAycV5GswIhANEPLmax0ME/ +EO+ZJ79TJKN5yiGBRsv5yvx5UiHxajEXAiAhAol5N4EUyq6I9w1rYdhPMGpLfk7A +IU2snfRJ6Nq2CQIgFrPsWRCkV+gOYcajD17rEqmuLrdIRexpg8N1DOSXoJ8CIGlS +tAboUGBxTDq3ZroNism3DaMIbKPyYrAqhKov1h5V +-----END RSA PRIVATE KEY----- +` diff --git a/src/pkg/exec/Makefile b/src/pkg/exec/Makefile new file mode 100644 index 000000000..ba19d0e4d --- /dev/null +++ b/src/pkg/exec/Makefile @@ -0,0 +1,31 @@ +# Copyright 2009 The Go Authors. All rights reserved. +# Use of this source code is governed by a BSD-style +# license that can be found in the LICENSE file. + +include ../../Make.inc + +TARG=exec +GOFILES=\ + exec.go\ + +GOFILES_freebsd=\ + lp_unix.go\ + +GOFILES_darwin=\ + lp_unix.go\ + +GOFILES_linux=\ + lp_unix.go\ + +GOFILES_openbsd=\ + lp_unix.go\ + +GOFILES_windows=\ + lp_windows.go\ + +GOFILES_plan9=\ + lp_plan9.go\ + +GOFILES+=$(GOFILES_$(GOOS)) + +include ../../Make.pkg diff --git a/src/pkg/exec/exec.go b/src/pkg/exec/exec.go new file mode 100644 index 000000000..3b20f2008 --- /dev/null +++ b/src/pkg/exec/exec.go @@ -0,0 +1,377 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Package exec runs external commands. It wraps os.StartProcess to make it +// easier to remap stdin and stdout, connect I/O with pipes, and do other +// adjustments. +package exec + +import ( + "bytes" + "io" + "os" + "strconv" + "syscall" +) + +// Error records the name of a binary that failed to be be executed +// and the reason it failed. +type Error struct { + Name string + Error os.Error +} + +func (e *Error) String() string { + return "exec: " + strconv.Quote(e.Name) + ": " + e.Error.String() +} + +// Cmd represents an external command being prepared or run. +type Cmd struct { + // Path is the path of the command to run. + // + // This is the only field that must be set to a non-zero + // value. + Path string + + // Args holds command line arguments, including the command as Args[0]. + // If the Args field is empty or nil, Run uses {Path}. + // + // In typical use, both Path and Args are set by calling Command. + Args []string + + // Env specifies the environment of the process. + // If Env is nil, Run uses the current process's environment. + Env []string + + // Dir specifies the working directory of the command. + // If Dir is the empty string, Run runs the command in the + // calling process's current directory. + Dir string + + // Stdin specifies the process's standard input. + // If Stdin is nil, the process reads from DevNull. + Stdin io.Reader + + // Stdout and Stderr specify the process's standard output and error. + // + // If either is nil, Run connects the + // corresponding file descriptor to /dev/null. + // + // If Stdout and Stderr are are the same writer, at most one + // goroutine at a time will call Write. + Stdout io.Writer + Stderr io.Writer + + // SysProcAttr holds optional, operating system-specific attributes. + // Run passes it to os.StartProcess as the os.ProcAttr's Sys field. + SysProcAttr *syscall.SysProcAttr + + // Process is the underlying process, once started. + Process *os.Process + + err os.Error // last error (from LookPath, stdin, stdout, stderr) + finished bool // when Wait was called + childFiles []*os.File + closeAfterStart []io.Closer + closeAfterWait []io.Closer + goroutine []func() os.Error + errch chan os.Error // one send per goroutine +} + +// Command returns the Cmd struct to execute the named program with +// the given arguments. +// +// It sets Path and Args in the returned structure and zeroes the +// other fields. +// +// If name contains no path separators, Command uses LookPath to +// resolve the path to a complete name if possible. Otherwise it uses +// name directly. +// +// The returned Cmd's Args field is constructed from the command name +// followed by the elements of arg, so arg should not include the +// command name itself. For example, Command("echo", "hello") +func Command(name string, arg ...string) *Cmd { + aname, err := LookPath(name) + if err != nil { + aname = name + } + return &Cmd{ + Path: aname, + Args: append([]string{name}, arg...), + err: err, + } +} + +// interfaceEqual protects against panics from doing equality tests on +// two interfaces with non-comparable underlying types +func interfaceEqual(a, b interface{}) bool { + defer func() { + recover() + }() + return a == b +} + +func (c *Cmd) envv() []string { + if c.Env != nil { + return c.Env + } + return os.Environ() +} + +func (c *Cmd) argv() []string { + if len(c.Args) > 0 { + return c.Args + } + return []string{c.Path} +} + +func (c *Cmd) stdin() (f *os.File, err os.Error) { + if c.Stdin == nil { + f, err = os.Open(os.DevNull) + c.closeAfterStart = append(c.closeAfterStart, f) + return + } + + if f, ok := c.Stdin.(*os.File); ok { + return f, nil + } + + pr, pw, err := os.Pipe() + if err != nil { + return + } + + c.closeAfterStart = append(c.closeAfterStart, pr) + c.closeAfterWait = append(c.closeAfterWait, pw) + c.goroutine = append(c.goroutine, func() os.Error { + _, err := io.Copy(pw, c.Stdin) + if err1 := pw.Close(); err == nil { + err = err1 + } + return err + }) + return pr, nil +} + +func (c *Cmd) stdout() (f *os.File, err os.Error) { + return c.writerDescriptor(c.Stdout) +} + +func (c *Cmd) stderr() (f *os.File, err os.Error) { + if c.Stderr != nil && interfaceEqual(c.Stderr, c.Stdout) { + return c.childFiles[1], nil + } + return c.writerDescriptor(c.Stderr) +} + +func (c *Cmd) writerDescriptor(w io.Writer) (f *os.File, err os.Error) { + if w == nil { + f, err = os.OpenFile(os.DevNull, os.O_WRONLY, 0) + c.closeAfterStart = append(c.closeAfterStart, f) + return + } + + if f, ok := w.(*os.File); ok { + return f, nil + } + + pr, pw, err := os.Pipe() + if err != nil { + return + } + + c.closeAfterStart = append(c.closeAfterStart, pw) + c.closeAfterWait = append(c.closeAfterWait, pr) + c.goroutine = append(c.goroutine, func() os.Error { + _, err := io.Copy(w, pr) + return err + }) + return pw, nil +} + +// Run starts the specified command and waits for it to complete. +// +// The returned error is nil if the command runs, has no problems +// copying stdin, stdout, and stderr, and exits with a zero exit +// status. +// +// If the command fails to run or doesn't complete successfully, the +// error is of type *os.Waitmsg. Other error types may be +// returned for I/O problems. +func (c *Cmd) Run() os.Error { + if err := c.Start(); err != nil { + return err + } + return c.Wait() +} + +// Start starts the specified command but does not wait for it to complete. +func (c *Cmd) Start() os.Error { + if c.err != nil { + return c.err + } + if c.Process != nil { + return os.NewError("exec: already started") + } + + type F func(*Cmd) (*os.File, os.Error) + for _, setupFd := range []F{(*Cmd).stdin, (*Cmd).stdout, (*Cmd).stderr} { + fd, err := setupFd(c) + if err != nil { + return err + } + c.childFiles = append(c.childFiles, fd) + } + + var err os.Error + c.Process, err = os.StartProcess(c.Path, c.argv(), &os.ProcAttr{ + Dir: c.Dir, + Files: c.childFiles, + Env: c.envv(), + Sys: c.SysProcAttr, + }) + if err != nil { + return err + } + + for _, fd := range c.closeAfterStart { + fd.Close() + } + + c.errch = make(chan os.Error, len(c.goroutine)) + for _, fn := range c.goroutine { + go func(fn func() os.Error) { + c.errch <- fn() + }(fn) + } + + return nil +} + +// Wait waits for the command to exit. +// It must have been started by Start. +// +// The returned error is nil if the command runs, has no problems +// copying stdin, stdout, and stderr, and exits with a zero exit +// status. +// +// If the command fails to run or doesn't complete successfully, the +// error is of type *os.Waitmsg. Other error types may be +// returned for I/O problems. +func (c *Cmd) Wait() os.Error { + if c.Process == nil { + return os.NewError("exec: not started") + } + if c.finished { + return os.NewError("exec: Wait was already called") + } + c.finished = true + msg, err := c.Process.Wait(0) + + var copyError os.Error + for _ = range c.goroutine { + if err := <-c.errch; err != nil && copyError == nil { + copyError = err + } + } + + for _, fd := range c.closeAfterWait { + fd.Close() + } + + if err != nil { + return err + } else if !msg.Exited() || msg.ExitStatus() != 0 { + return msg + } + + return copyError +} + +// Output runs the command and returns its standard output. +func (c *Cmd) Output() ([]byte, os.Error) { + if c.Stdout != nil { + return nil, os.NewError("exec: Stdout already set") + } + var b bytes.Buffer + c.Stdout = &b + err := c.Run() + return b.Bytes(), err +} + +// CombinedOutput runs the command and returns its combined standard +// output and standard error. +func (c *Cmd) CombinedOutput() ([]byte, os.Error) { + if c.Stdout != nil { + return nil, os.NewError("exec: Stdout already set") + } + if c.Stderr != nil { + return nil, os.NewError("exec: Stderr already set") + } + var b bytes.Buffer + c.Stdout = &b + c.Stderr = &b + err := c.Run() + return b.Bytes(), err +} + +// StdinPipe returns a pipe that will be connected to the command's +// standard input when the command starts. +func (c *Cmd) StdinPipe() (io.WriteCloser, os.Error) { + if c.Stdin != nil { + return nil, os.NewError("exec: Stdin already set") + } + if c.Process != nil { + return nil, os.NewError("exec: StdinPipe after process started") + } + pr, pw, err := os.Pipe() + if err != nil { + return nil, err + } + c.Stdin = pr + c.closeAfterStart = append(c.closeAfterStart, pr) + c.closeAfterWait = append(c.closeAfterWait, pw) + return pw, nil +} + +// StdoutPipe returns a pipe that will be connected to the command's +// standard output when the command starts. +// The pipe will be closed automatically after Wait sees the command exit. +func (c *Cmd) StdoutPipe() (io.ReadCloser, os.Error) { + if c.Stdout != nil { + return nil, os.NewError("exec: Stdout already set") + } + if c.Process != nil { + return nil, os.NewError("exec: StdoutPipe after process started") + } + pr, pw, err := os.Pipe() + if err != nil { + return nil, err + } + c.Stdout = pw + c.closeAfterStart = append(c.closeAfterStart, pw) + c.closeAfterWait = append(c.closeAfterWait, pr) + return pr, nil +} + +// StderrPipe returns a pipe that will be connected to the command's +// standard error when the command starts. +// The pipe will be closed automatically after Wait sees the command exit. +func (c *Cmd) StderrPipe() (io.ReadCloser, os.Error) { + if c.Stderr != nil { + return nil, os.NewError("exec: Stderr already set") + } + if c.Process != nil { + return nil, os.NewError("exec: StderrPipe after process started") + } + pr, pw, err := os.Pipe() + if err != nil { + return nil, err + } + c.Stderr = pw + c.closeAfterStart = append(c.closeAfterStart, pw) + c.closeAfterWait = append(c.closeAfterWait, pr) + return pr, nil +} diff --git a/src/pkg/exec/exec_test.go b/src/pkg/exec/exec_test.go new file mode 100644 index 000000000..242120faa --- /dev/null +++ b/src/pkg/exec/exec_test.go @@ -0,0 +1,214 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package exec + +import ( + "bufio" + "bytes" + "fmt" + "io" + "testing" + "os" + "strconv" + "strings" +) + +func helperCommand(s ...string) *Cmd { + cs := []string{"-test.run=exec.TestHelperProcess", "--"} + cs = append(cs, s...) + cmd := Command(os.Args[0], cs...) + cmd.Env = append([]string{"GO_WANT_HELPER_PROCESS=1"}, os.Environ()...) + return cmd +} + +func TestEcho(t *testing.T) { + bs, err := helperCommand("echo", "foo bar", "baz").Output() + if err != nil { + t.Errorf("echo: %v", err) + } + if g, e := string(bs), "foo bar baz\n"; g != e { + t.Errorf("echo: want %q, got %q", e, g) + } +} + +func TestCatStdin(t *testing.T) { + // Cat, testing stdin and stdout. + input := "Input string\nLine 2" + p := helperCommand("cat") + p.Stdin = strings.NewReader(input) + bs, err := p.Output() + if err != nil { + t.Errorf("cat: %v", err) + } + s := string(bs) + if s != input { + t.Errorf("cat: want %q, got %q", input, s) + } +} + +func TestCatGoodAndBadFile(t *testing.T) { + // Testing combined output and error values. + bs, err := helperCommand("cat", "/bogus/file.foo", "exec_test.go").CombinedOutput() + if _, ok := err.(*os.Waitmsg); !ok { + t.Errorf("expected Waitmsg from cat combined; got %T: %v", err, err) + } + s := string(bs) + sp := strings.SplitN(s, "\n", 2) + if len(sp) != 2 { + t.Fatalf("expected two lines from cat; got %q", s) + } + errLine, body := sp[0], sp[1] + if !strings.HasPrefix(errLine, "Error: open /bogus/file.foo") { + t.Errorf("expected stderr to complain about file; got %q", errLine) + } + if !strings.Contains(body, "func TestHelperProcess(t *testing.T)") { + t.Errorf("expected test code; got %q (len %d)", body, len(body)) + } +} + +func TestNoExistBinary(t *testing.T) { + // Can't run a non-existent binary + err := Command("/no-exist-binary").Run() + if err == nil { + t.Error("expected error from /no-exist-binary") + } +} + +func TestExitStatus(t *testing.T) { + // Test that exit values are returned correctly + err := helperCommand("exit", "42").Run() + if werr, ok := err.(*os.Waitmsg); ok { + if s, e := werr.String(), "exit status 42"; s != e { + t.Errorf("from exit 42 got exit %q, want %q", s, e) + } + } else { + t.Fatalf("expected Waitmsg from exit 42; got %T: %v", err, err) + } +} + +func TestPipes(t *testing.T) { + check := func(what string, err os.Error) { + if err != nil { + t.Fatalf("%s: %v", what, err) + } + } + // Cat, testing stdin and stdout. + c := helperCommand("pipetest") + stdin, err := c.StdinPipe() + check("StdinPipe", err) + stdout, err := c.StdoutPipe() + check("StdoutPipe", err) + stderr, err := c.StderrPipe() + check("StderrPipe", err) + + outbr := bufio.NewReader(stdout) + errbr := bufio.NewReader(stderr) + line := func(what string, br *bufio.Reader) string { + line, _, err := br.ReadLine() + if err != nil { + t.Fatalf("%s: %v", what, err) + } + return string(line) + } + + err = c.Start() + check("Start", err) + + _, err = stdin.Write([]byte("O:I am output\n")) + check("first stdin Write", err) + if g, e := line("first output line", outbr), "O:I am output"; g != e { + t.Errorf("got %q, want %q", g, e) + } + + _, err = stdin.Write([]byte("E:I am error\n")) + check("second stdin Write", err) + if g, e := line("first error line", errbr), "E:I am error"; g != e { + t.Errorf("got %q, want %q", g, e) + } + + _, err = stdin.Write([]byte("O:I am output2\n")) + check("third stdin Write 3", err) + if g, e := line("second output line", outbr), "O:I am output2"; g != e { + t.Errorf("got %q, want %q", g, e) + } + + stdin.Close() + err = c.Wait() + check("Wait", err) +} + +// TestHelperProcess isn't a real test. It's used as a helper process +// for TestParameterRun. +func TestHelperProcess(*testing.T) { + if os.Getenv("GO_WANT_HELPER_PROCESS") != "1" { + return + } + defer os.Exit(0) + + args := os.Args + for len(args) > 0 { + if args[0] == "--" { + args = args[1:] + break + } + args = args[1:] + } + if len(args) == 0 { + fmt.Fprintf(os.Stderr, "No command\n") + os.Exit(2) + } + + cmd, args := args[0], args[1:] + switch cmd { + case "echo": + iargs := []interface{}{} + for _, s := range args { + iargs = append(iargs, s) + } + fmt.Println(iargs...) + case "cat": + if len(args) == 0 { + io.Copy(os.Stdout, os.Stdin) + return + } + exit := 0 + for _, fn := range args { + f, err := os.Open(fn) + if err != nil { + fmt.Fprintf(os.Stderr, "Error: %v\n", err) + exit = 2 + } else { + defer f.Close() + io.Copy(os.Stdout, f) + } + } + os.Exit(exit) + case "pipetest": + bufr := bufio.NewReader(os.Stdin) + for { + line, _, err := bufr.ReadLine() + if err == os.EOF { + break + } else if err != nil { + os.Exit(1) + } + if bytes.HasPrefix(line, []byte("O:")) { + os.Stdout.Write(line) + os.Stdout.Write([]byte{'\n'}) + } else if bytes.HasPrefix(line, []byte("E:")) { + os.Stderr.Write(line) + os.Stderr.Write([]byte{'\n'}) + } else { + os.Exit(1) + } + } + case "exit": + n, _ := strconv.Atoi(args[0]) + os.Exit(n) + default: + fmt.Fprintf(os.Stderr, "Unknown command %q\n", cmd) + os.Exit(2) + } +} diff --git a/src/pkg/exec/lp_plan9.go b/src/pkg/exec/lp_plan9.go new file mode 100644 index 000000000..e4751a4df --- /dev/null +++ b/src/pkg/exec/lp_plan9.go @@ -0,0 +1,51 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package exec + +import ( + "os" + "strings" +) + +// ErrNotFound is the error resulting if a path search failed to find an executable file. +var ErrNotFound = os.NewError("executable file not found in $path") + +func findExecutable(file string) os.Error { + d, err := os.Stat(file) + if err != nil { + return err + } + if d.IsRegular() && d.Permission()&0111 != 0 { + return nil + } + return os.EPERM +} + +// LookPath searches for an executable binary named file +// in the directories named by the path environment variable. +// If file begins with "/", "#", "./", or "../", it is tried +// directly and the path is not consulted. +func LookPath(file string) (string, os.Error) { + // skip the path lookup for these prefixes + skip := []string{"/", "#", "./", "../"} + + for _, p := range skip { + if strings.HasPrefix(file, p) { + err := findExecutable(file) + if err == nil { + return file, nil + } + return "", &Error{file, err} + } + } + + path := os.Getenv("path") + for _, dir := range strings.Split(path, "\000") { + if err := findExecutable(dir + "/" + file); err == nil { + return dir + "/" + file, nil + } + } + return "", &Error{file, ErrNotFound} +} diff --git a/src/pkg/exec/lp_test.go b/src/pkg/exec/lp_test.go new file mode 100644 index 000000000..77d8e848c --- /dev/null +++ b/src/pkg/exec/lp_test.go @@ -0,0 +1,33 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package exec + +import ( + "testing" +) + +var nonExistentPaths = []string{ + "some-non-existent-path", + "non-existent-path/slashed", +} + +func TestLookPathNotFound(t *testing.T) { + for _, name := range nonExistentPaths { + path, err := LookPath(name) + if err == nil { + t.Fatalf("LookPath found %q in $PATH", name) + } + if path != "" { + t.Fatalf("LookPath path == %q when err != nil", path) + } + perr, ok := err.(*Error) + if !ok { + t.Fatal("LookPath error is not an exec.Error") + } + if perr.Name != name { + t.Fatalf("want Error name %q, got %q", name, perr.Name) + } + } +} diff --git a/src/pkg/exec/lp_unix.go b/src/pkg/exec/lp_unix.go new file mode 100644 index 000000000..008fb11a8 --- /dev/null +++ b/src/pkg/exec/lp_unix.go @@ -0,0 +1,52 @@ +// Copyright 2010 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package exec + +import ( + "os" + "strings" +) + +// ErrNotFound is the error resulting if a path search failed to find an executable file. +var ErrNotFound = os.NewError("executable file not found in $PATH") + +func findExecutable(file string) os.Error { + d, err := os.Stat(file) + if err != nil { + return err + } + if d.IsRegular() && d.Permission()&0111 != 0 { + return nil + } + return os.EPERM +} + +// LookPath searches for an executable binary named file +// in the directories named by the PATH environment variable. +// If file contains a slash, it is tried directly and the PATH is not consulted. +func LookPath(file string) (string, os.Error) { + // NOTE(rsc): I wish we could use the Plan 9 behavior here + // (only bypass the path if file begins with / or ./ or ../) + // but that would not match all the Unix shells. + + if strings.Contains(file, "/") { + err := findExecutable(file) + if err == nil { + return file, nil + } + return "", &Error{file, err} + } + pathenv := os.Getenv("PATH") + for _, dir := range strings.Split(pathenv, ":") { + if dir == "" { + // Unix shell semantics: path element "" means "." + dir = "." + } + if err := findExecutable(dir + "/" + file); err == nil { + return dir + "/" + file, nil + } + } + return "", &Error{file, ErrNotFound} +} diff --git a/src/pkg/exec/lp_windows.go b/src/pkg/exec/lp_windows.go new file mode 100644 index 000000000..7581088eb --- /dev/null +++ b/src/pkg/exec/lp_windows.go @@ -0,0 +1,77 @@ +// Copyright 2010 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package exec + +import ( + "os" + "strings" +) + +// ErrNotFound is the error resulting if a path search failed to find an executable file. +var ErrNotFound = os.NewError("executable file not found in %PATH%") + +func chkStat(file string) os.Error { + d, err := os.Stat(file) + if err != nil { + return err + } + if d.IsRegular() { + return nil + } + return os.EPERM +} + +func findExecutable(file string, exts []string) (string, os.Error) { + if len(exts) == 0 { + return file, chkStat(file) + } + f := strings.ToLower(file) + for _, e := range exts { + if strings.HasSuffix(f, e) { + return file, chkStat(file) + } + } + for _, e := range exts { + if f := file + e; chkStat(f) == nil { + return f, nil + } + } + return ``, os.ENOENT +} + +func LookPath(file string) (f string, err os.Error) { + x := os.Getenv(`PATHEXT`) + if x == `` { + x = `.COM;.EXE;.BAT;.CMD` + } + exts := []string{} + for _, e := range strings.Split(strings.ToLower(x), `;`) { + if e == "" { + continue + } + if e[0] != '.' { + e = "." + e + } + exts = append(exts, e) + } + if strings.IndexAny(file, `:\/`) != -1 { + if f, err = findExecutable(file, exts); err == nil { + return + } + return ``, &Error{file, err} + } + if pathenv := os.Getenv(`PATH`); pathenv == `` { + if f, err = findExecutable(`.\`+file, exts); err == nil { + return + } + } else { + for _, dir := range strings.Split(pathenv, `;`) { + if f, err = findExecutable(dir+`\`+file, exts); err == nil { + return + } + } + } + return ``, &Error{file, ErrNotFound} +} diff --git a/src/pkg/exp/README b/src/pkg/exp/README new file mode 100644 index 000000000..e602e3ac9 --- /dev/null +++ b/src/pkg/exp/README @@ -0,0 +1,3 @@ +This directory tree contains experimental packages and +unfinished code that is subject to even more change than the +rest of the Go tree. diff --git a/src/pkg/exp/datafmt/Makefile b/src/pkg/exp/datafmt/Makefile new file mode 100644 index 000000000..aa9453897 --- /dev/null +++ b/src/pkg/exp/datafmt/Makefile @@ -0,0 +1,12 @@ +# Copyright 2009 The Go Authors. All rights reserved. +# Use of this source code is governed by a BSD-style +# license that can be found in the LICENSE file. + +include ../../../Make.inc + +TARG=exp/datafmt +GOFILES=\ + datafmt.go\ + parser.go\ + +include ../../../Make.pkg diff --git a/src/pkg/exp/datafmt/datafmt.go b/src/pkg/exp/datafmt/datafmt.go new file mode 100644 index 000000000..6d7e76442 --- /dev/null +++ b/src/pkg/exp/datafmt/datafmt.go @@ -0,0 +1,710 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +/* Package datafmt implements syntax-directed, type-driven formatting + of arbitrary data structures. Formatting a data structure consists of + two phases: first, a parser reads a format specification and builds a + "compiled" format. Then, the format can be applied repeatedly to + arbitrary values. Applying a format to a value evaluates to a []byte + containing the formatted value bytes, or nil. + + A format specification is a set of package declarations and format rules: + + Format = [ Entry { ";" Entry } [ ";" ] ] . + Entry = PackageDecl | FormatRule . + + (The syntax of a format specification is presented in the same EBNF + notation as used in the Go language specification. The syntax of white + space, comments, identifiers, and string literals is the same as in Go.) + + A package declaration binds a package name (such as 'ast') to a + package import path (such as '"go/ast"'). Each package used (in + a type name, see below) must be declared once before use. + + PackageDecl = PackageName ImportPath . + PackageName = identifier . + ImportPath = string . + + A format rule binds a rule name to a format expression. A rule name + may be a type name or one of the special names 'default' or '/'. + A type name may be the name of a predeclared type (for example, 'int', + 'float32', etc.), the package-qualified name of a user-defined type + (for example, 'ast.MapType'), or an identifier indicating the structure + of unnamed composite types ('array', 'chan', 'func', 'interface', 'map', + or 'ptr'). Each rule must have a unique name; rules can be declared in + any order. + + FormatRule = RuleName "=" Expression . + RuleName = TypeName | "default" | "/" . + TypeName = [ PackageName "." ] identifier . + + To format a value, the value's type name is used to select the format rule + (there is an override mechanism, see below). The format expression of the + selected rule specifies how the value is formatted. Each format expression, + when applied to a value, evaluates to a byte sequence or nil. + + In its most general form, a format expression is a list of alternatives, + each of which is a sequence of operands: + + Expression = [ Sequence ] { "|" [ Sequence ] } . + Sequence = Operand { Operand } . + + The formatted result produced by an expression is the result of the first + alternative sequence that evaluates to a non-nil result; if there is no + such alternative, the expression evaluates to nil. The result produced by + an operand sequence is the concatenation of the results of its operands. + If any operand in the sequence evaluates to nil, the entire sequence + evaluates to nil. + + There are five kinds of operands: + + Operand = Literal | Field | Group | Option | Repetition . + + Literals evaluate to themselves, with two substitutions. First, + %-formats expand in the manner of fmt.Printf, with the current value + passed as the parameter. Second, the current indentation (see below) + is inserted after every newline or form feed character. + + Literal = string . + + This table shows string literals applied to the value 42 and the + corresponding formatted result: + + "foo" foo + "%x" 2a + "x = %d" x = 42 + "%#x = %d" 0x2a = 42 + + A field operand is a field name optionally followed by an alternate + rule name. The field name may be an identifier or one of the special + names @ or *. + + Field = FieldName [ ":" RuleName ] . + FieldName = identifier | "@" | "*" . + + If the field name is an identifier, the current value must be a struct, + and there must be a field with that name in the struct. The same lookup + rules apply as in the Go language (for instance, the name of an anonymous + field is the unqualified type name). The field name denotes the field + value in the struct. If the field is not found, formatting is aborted + and an error message is returned. (TODO consider changing the semantics + such that if a field is not found, it evaluates to nil). + + The special name '@' denotes the current value. + + The meaning of the special name '*' depends on the type of the current + value: + + array, slice types array, slice element (inside {} only, see below) + interfaces value stored in interface + pointers value pointed to by pointer + + (Implementation restriction: channel, function and map types are not + supported due to missing reflection support). + + Fields are evaluated as follows: If the field value is nil, or an array + or slice element does not exist, the result is nil (see below for details + on array/slice elements). If the value is not nil the field value is + formatted (recursively) using the rule corresponding to its type name, + or the alternate rule name, if given. + + The following example shows a complete format specification for a + struct 'myPackage.Point'. Assume the package + + package myPackage // in directory myDir/myPackage + type Point struct { + name string; + x, y int; + } + + Applying the format specification + + myPackage "myDir/myPackage"; + int = "%d"; + hexInt = "0x%x"; + string = "---%s---"; + myPackage.Point = name "{" x ", " y:hexInt "}"; + + to the value myPackage.Point{"foo", 3, 15} results in + + ---foo---{3, 0xf} + + Finally, an operand may be a grouped, optional, or repeated expression. + A grouped expression ("group") groups a more complex expression (body) + so that it can be used in place of a single operand: + + Group = "(" [ Indentation ">>" ] Body ")" . + Indentation = Expression . + Body = Expression . + + A group body may be prefixed by an indentation expression followed by '>>'. + The indentation expression is applied to the current value like any other + expression and the result, if not nil, is appended to the current indentation + during the evaluation of the body (see also formatting state, below). + + An optional expression ("option") is enclosed in '[]' brackets. + + Option = "[" Body "]" . + + An option evaluates to its body, except that if the body evaluates to nil, + the option expression evaluates to an empty []byte. Thus an option's purpose + is to protect the expression containing the option from a nil operand. + + A repeated expression ("repetition") is enclosed in '{}' braces. + + Repetition = "{" Body [ "/" Separator ] "}" . + Separator = Expression . + + A repeated expression is evaluated as follows: The body is evaluated + repeatedly and its results are concatenated until the body evaluates + to nil. The result of the repetition is the (possibly empty) concatenation, + but it is never nil. An implicit index is supplied for the evaluation of + the body: that index is used to address elements of arrays or slices. If + the corresponding elements do not exist, the field denoting the element + evaluates to nil (which in turn may terminate the repetition). + + The body of a repetition may be followed by a '/' and a "separator" + expression. If the separator is present, it is invoked between repetitions + of the body. + + The following example shows a complete format specification for formatting + a slice of unnamed type. Applying the specification + + int = "%b"; + array = { * / ", " }; // array is the type name for an unnamed slice + + to the value '[]int{2, 3, 5, 7}' results in + + 10, 11, 101, 111 + + Default rule: If a format rule named 'default' is present, it is used for + formatting a value if no other rule was found. A common default rule is + + default = "%v" + + to provide default formatting for basic types without having to specify + a specific rule for each basic type. + + Global separator rule: If a format rule named '/' is present, it is + invoked with the current value between literals. If the separator + expression evaluates to nil, it is ignored. + + For instance, a global separator rule may be used to punctuate a sequence + of values with commas. The rules: + + default = "%v"; + / = ", "; + + will format an argument list by printing each one in its default format, + separated by a comma and a space. +*/ +package datafmt + +import ( + "bytes" + "fmt" + "go/token" + "io" + "os" + "reflect" + "runtime" +) + +// ---------------------------------------------------------------------------- +// Format representation + +// Custom formatters implement the Formatter function type. +// A formatter is invoked with the current formatting state, the +// value to format, and the rule name under which the formatter +// was installed (the same formatter function may be installed +// under different names). The formatter may access the current state +// to guide formatting and use State.Write to append to the state's +// output. +// +// A formatter must return a boolean value indicating if it evaluated +// to a non-nil value (true), or a nil value (false). +// +type Formatter func(state *State, value interface{}, ruleName string) bool + +// A FormatterMap is a set of custom formatters. +// It maps a rule name to a formatter function. +// +type FormatterMap map[string]Formatter + +// A parsed format expression is built from the following nodes. +// +type ( + expr interface{} + + alternatives []expr // x | y | z + + sequence []expr // x y z + + literal [][]byte // a list of string segments, possibly starting with '%' + + field struct { + fieldName string // including "@", "*" + ruleName string // "" if no rule name specified + } + + group struct { + indent, body expr // (indent >> body) + } + + option struct { + body expr // [body] + } + + repetition struct { + body, separator expr // {body / separator} + } + + custom struct { + ruleName string + fun Formatter + } +) + +// A Format is the result of parsing a format specification. +// The format may be applied repeatedly to format values. +// +type Format map[string]expr + +// ---------------------------------------------------------------------------- +// Formatting + +// An application-specific environment may be provided to Format.Apply; +// the environment is available inside custom formatters via State.Env(). +// Environments must implement copying; the Copy method must return an +// complete copy of the receiver. This is necessary so that the formatter +// can save and restore an environment (in case of an absent expression). +// +// If the Environment doesn't change during formatting (this is under +// control of the custom formatters), the Copy function can simply return +// the receiver, and thus can be very light-weight. +// +type Environment interface { + Copy() Environment +} + +// State represents the current formatting state. +// It is provided as argument to custom formatters. +// +type State struct { + fmt Format // format in use + env Environment // user-supplied environment + errors chan os.Error // not chan *Error (errors <- nil would be wrong!) + hasOutput bool // true after the first literal has been written + indent bytes.Buffer // current indentation + output bytes.Buffer // format output + linePos token.Position // position of line beginning (Column == 0) + default_ expr // possibly nil + separator expr // possibly nil +} + +func newState(fmt Format, env Environment, errors chan os.Error) *State { + s := new(State) + s.fmt = fmt + s.env = env + s.errors = errors + s.linePos = token.Position{Line: 1} + + // if we have a default rule, cache its expression for fast access + if x, found := fmt["default"]; found { + s.default_ = x + } + + // if we have a global separator rule, cache its expression for fast access + if x, found := fmt["/"]; found { + s.separator = x + } + + return s +} + +// Env returns the environment passed to Format.Apply. +func (s *State) Env() interface{} { return s.env } + +// LinePos returns the position of the current line beginning +// in the state's output buffer. Line numbers start at 1. +// +func (s *State) LinePos() token.Position { return s.linePos } + +// Pos returns the position of the next byte to be written to the +// output buffer. Line numbers start at 1. +// +func (s *State) Pos() token.Position { + offs := s.output.Len() + return token.Position{Line: s.linePos.Line, Column: offs - s.linePos.Offset, Offset: offs} +} + +// Write writes data to the output buffer, inserting the indentation +// string after each newline or form feed character. It cannot return an error. +// +func (s *State) Write(data []byte) (int, os.Error) { + n := 0 + i0 := 0 + for i, ch := range data { + if ch == '\n' || ch == '\f' { + // write text segment and indentation + n1, _ := s.output.Write(data[i0 : i+1]) + n2, _ := s.output.Write(s.indent.Bytes()) + n += n1 + n2 + i0 = i + 1 + s.linePos.Offset = s.output.Len() + s.linePos.Line++ + } + } + n3, _ := s.output.Write(data[i0:]) + return n + n3, nil +} + +type checkpoint struct { + env Environment + hasOutput bool + outputLen int + linePos token.Position +} + +func (s *State) save() checkpoint { + saved := checkpoint{nil, s.hasOutput, s.output.Len(), s.linePos} + if s.env != nil { + saved.env = s.env.Copy() + } + return saved +} + +func (s *State) restore(m checkpoint) { + s.env = m.env + s.output.Truncate(m.outputLen) +} + +func (s *State) error(msg string) { + s.errors <- os.NewError(msg) + runtime.Goexit() +} + +// TODO At the moment, unnamed types are simply mapped to the default +// names below. For instance, all unnamed arrays are mapped to +// 'array' which is not really sufficient. Eventually one may want +// to be able to specify rules for say an unnamed slice of T. +// + +func typename(typ reflect.Type) string { + switch typ.Kind() { + case reflect.Array: + return "array" + case reflect.Slice: + return "array" + case reflect.Chan: + return "chan" + case reflect.Func: + return "func" + case reflect.Interface: + return "interface" + case reflect.Map: + return "map" + case reflect.Ptr: + return "ptr" + } + return typ.String() +} + +func (s *State) getFormat(name string) expr { + if fexpr, found := s.fmt[name]; found { + return fexpr + } + + if s.default_ != nil { + return s.default_ + } + + s.error(fmt.Sprintf("no format rule for type: '%s'", name)) + return nil +} + +// eval applies a format expression fexpr to a value. If the expression +// evaluates internally to a non-nil []byte, that slice is appended to +// the state's output buffer and eval returns true. Otherwise, eval +// returns false and the state remains unchanged. +// +func (s *State) eval(fexpr expr, value reflect.Value, index int) bool { + // an empty format expression always evaluates + // to a non-nil (but empty) []byte + if fexpr == nil { + return true + } + + switch t := fexpr.(type) { + case alternatives: + // append the result of the first alternative that evaluates to + // a non-nil []byte to the state's output + mark := s.save() + for _, x := range t { + if s.eval(x, value, index) { + return true + } + s.restore(mark) + } + return false + + case sequence: + // append the result of all operands to the state's output + // unless a nil result is encountered + mark := s.save() + for _, x := range t { + if !s.eval(x, value, index) { + s.restore(mark) + return false + } + } + return true + + case literal: + // write separator, if any + if s.hasOutput { + // not the first literal + if s.separator != nil { + sep := s.separator // save current separator + s.separator = nil // and disable it (avoid recursion) + mark := s.save() + if !s.eval(sep, value, index) { + s.restore(mark) + } + s.separator = sep // enable it again + } + } + s.hasOutput = true + // write literal segments + for _, lit := range t { + if len(lit) > 1 && lit[0] == '%' { + // segment contains a %-format at the beginning + if lit[1] == '%' { + // "%%" is printed as a single "%" + s.Write(lit[1:]) + } else { + // use s instead of s.output to get indentation right + fmt.Fprintf(s, string(lit), value.Interface()) + } + } else { + // segment contains no %-formats + s.Write(lit) + } + } + return true // a literal never evaluates to nil + + case *field: + // determine field value + switch t.fieldName { + case "@": + // field value is current value + + case "*": + // indirection: operation is type-specific + switch v := value; v.Kind() { + case reflect.Array: + if v.Len() <= index { + return false + } + value = v.Index(index) + + case reflect.Slice: + if v.IsNil() || v.Len() <= index { + return false + } + value = v.Index(index) + + case reflect.Map: + s.error("reflection support for maps incomplete") + + case reflect.Ptr: + if v.IsNil() { + return false + } + value = v.Elem() + + case reflect.Interface: + if v.IsNil() { + return false + } + value = v.Elem() + + case reflect.Chan: + s.error("reflection support for chans incomplete") + + case reflect.Func: + s.error("reflection support for funcs incomplete") + + default: + s.error(fmt.Sprintf("error: * does not apply to `%s`", value.Type())) + } + + default: + // value is value of named field + var field reflect.Value + if sval := value; sval.Kind() == reflect.Struct { + field = sval.FieldByName(t.fieldName) + if !field.IsValid() { + // TODO consider just returning false in this case + s.error(fmt.Sprintf("error: no field `%s` in `%s`", t.fieldName, value.Type())) + } + } + value = field + } + + // determine rule + ruleName := t.ruleName + if ruleName == "" { + // no alternate rule name, value type determines rule + ruleName = typename(value.Type()) + } + fexpr = s.getFormat(ruleName) + + mark := s.save() + if !s.eval(fexpr, value, index) { + s.restore(mark) + return false + } + return true + + case *group: + // remember current indentation + indentLen := s.indent.Len() + + // update current indentation + mark := s.save() + s.eval(t.indent, value, index) + // if the indentation evaluates to nil, the state's output buffer + // didn't change - either way it's ok to append the difference to + // the current indentation + s.indent.Write(s.output.Bytes()[mark.outputLen:s.output.Len()]) + s.restore(mark) + + // format group body + mark = s.save() + b := true + if !s.eval(t.body, value, index) { + s.restore(mark) + b = false + } + + // reset indentation + s.indent.Truncate(indentLen) + return b + + case *option: + // evaluate the body and append the result to the state's output + // buffer unless the result is nil + mark := s.save() + if !s.eval(t.body, value, 0) { // TODO is 0 index correct? + s.restore(mark) + } + return true // an option never evaluates to nil + + case *repetition: + // evaluate the body and append the result to the state's output + // buffer until a result is nil + for i := 0; ; i++ { + mark := s.save() + // write separator, if any + if i > 0 && t.separator != nil { + // nil result from separator is ignored + mark := s.save() + if !s.eval(t.separator, value, i) { + s.restore(mark) + } + } + if !s.eval(t.body, value, i) { + s.restore(mark) + break + } + } + return true // a repetition never evaluates to nil + + case *custom: + // invoke the custom formatter to obtain the result + mark := s.save() + if !t.fun(s, value.Interface(), t.ruleName) { + s.restore(mark) + return false + } + return true + } + + panic("unreachable") + return false +} + +// Eval formats each argument according to the format +// f and returns the resulting []byte and os.Error. If +// an error occurred, the []byte contains the partially +// formatted result. An environment env may be passed +// in which is available in custom formatters through +// the state parameter. +// +func (f Format) Eval(env Environment, args ...interface{}) ([]byte, os.Error) { + if f == nil { + return nil, os.NewError("format is nil") + } + + errors := make(chan os.Error) + s := newState(f, env, errors) + + go func() { + for _, v := range args { + fld := reflect.ValueOf(v) + if !fld.IsValid() { + errors <- os.NewError("nil argument") + return + } + mark := s.save() + if !s.eval(s.getFormat(typename(fld.Type())), fld, 0) { // TODO is 0 index correct? + s.restore(mark) + } + } + errors <- nil // no errors + }() + + err := <-errors + return s.output.Bytes(), err +} + +// ---------------------------------------------------------------------------- +// Convenience functions + +// Fprint formats each argument according to the format f +// and writes to w. The result is the total number of bytes +// written and an os.Error, if any. +// +func (f Format) Fprint(w io.Writer, env Environment, args ...interface{}) (int, os.Error) { + data, err := f.Eval(env, args...) + if err != nil { + // TODO should we print partial result in case of error? + return 0, err + } + return w.Write(data) +} + +// Print formats each argument according to the format f +// and writes to standard output. The result is the total +// number of bytes written and an os.Error, if any. +// +func (f Format) Print(args ...interface{}) (int, os.Error) { + return f.Fprint(os.Stdout, nil, args...) +} + +// Sprint formats each argument according to the format f +// and returns the resulting string. If an error occurs +// during formatting, the result string contains the +// partially formatted result followed by an error message. +// +func (f Format) Sprint(args ...interface{}) string { + var buf bytes.Buffer + _, err := f.Fprint(&buf, nil, args...) + if err != nil { + var i interface{} = args + fmt.Fprintf(&buf, "--- Sprint(%s) failed: %v", fmt.Sprint(i), err) + } + return buf.String() +} diff --git a/src/pkg/exp/datafmt/datafmt_test.go b/src/pkg/exp/datafmt/datafmt_test.go new file mode 100644 index 000000000..87d071659 --- /dev/null +++ b/src/pkg/exp/datafmt/datafmt_test.go @@ -0,0 +1,330 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package datafmt + +import ( + "fmt" + "testing" + "go/token" +) + +var fset = token.NewFileSet() + +func parse(t *testing.T, form string, fmap FormatterMap) Format { + f, err := Parse(fset, "", []byte(form), fmap) + if err != nil { + t.Errorf("Parse(%s): %v", form, err) + return nil + } + return f +} + +func verify(t *testing.T, f Format, expected string, args ...interface{}) { + if f == nil { + return // allow other tests to run + } + result := f.Sprint(args...) + if result != expected { + t.Errorf( + "result : `%s`\nexpected: `%s`\n\n", + result, expected) + } +} + +func formatter(s *State, value interface{}, rule_name string) bool { + switch rule_name { + case "/": + fmt.Fprintf(s, "%d %d %d", s.Pos().Line, s.LinePos().Column, s.Pos().Column) + return true + case "blank": + s.Write([]byte{' '}) + return true + case "int": + if value.(int)&1 == 0 { + fmt.Fprint(s, "even ") + } else { + fmt.Fprint(s, "odd ") + } + return true + case "nil": + return false + case "testing.T": + s.Write([]byte("testing.T")) + return true + } + panic("unreachable") + return false +} + +func TestCustomFormatters(t *testing.T) { + fmap0 := FormatterMap{"/": formatter} + fmap1 := FormatterMap{"int": formatter, "blank": formatter, "nil": formatter} + fmap2 := FormatterMap{"testing.T": formatter} + + f := parse(t, `int=`, fmap0) + verify(t, f, ``, 1, 2, 3) + + f = parse(t, `int="#"`, nil) + verify(t, f, `###`, 1, 2, 3) + + f = parse(t, `int="#";string="%s"`, fmap0) + verify(t, f, "#1 0 1#1 0 7#1 0 13\n2 0 0foo2 0 8\n", 1, 2, 3, "\n", "foo", "\n") + + f = parse(t, ``, fmap1) + verify(t, f, `even odd even odd `, 0, 1, 2, 3) + + f = parse(t, `/ =@:blank; float64="#"`, fmap1) + verify(t, f, `# # #`, 0.0, 1.0, 2.0) + + f = parse(t, `float64=@:nil`, fmap1) + verify(t, f, ``, 0.0, 1.0, 2.0) + + f = parse(t, `testing "testing"; ptr=*`, fmap2) + verify(t, f, `testing.T`, t) + + // TODO needs more tests +} + +// ---------------------------------------------------------------------------- +// Formatting of basic and simple composite types + +func check(t *testing.T, form, expected string, args ...interface{}) { + f := parse(t, form, nil) + if f == nil { + return // allow other tests to run + } + result := f.Sprint(args...) + if result != expected { + t.Errorf( + "format : %s\nresult : `%s`\nexpected: `%s`\n\n", + form, result, expected) + } +} + +func TestBasicTypes(t *testing.T) { + check(t, ``, ``) + check(t, `bool=":%v"`, `:true:false`, true, false) + check(t, `int="%b %d %o 0x%x"`, `101010 42 52 0x2a`, 42) + + check(t, `int="%"`, `%`, 42) + check(t, `int="%%"`, `%`, 42) + check(t, `int="**%%**"`, `**%**`, 42) + check(t, `int="%%%%%%"`, `%%%`, 42) + check(t, `int="%%%d%%"`, `%42%`, 42) + + const i = -42 + const is = `-42` + check(t, `int ="%d"`, is, i) + check(t, `int8 ="%d"`, is, int8(i)) + check(t, `int16="%d"`, is, int16(i)) + check(t, `int32="%d"`, is, int32(i)) + check(t, `int64="%d"`, is, int64(i)) + + const u = 42 + const us = `42` + check(t, `uint ="%d"`, us, uint(u)) + check(t, `uint8 ="%d"`, us, uint8(u)) + check(t, `uint16="%d"`, us, uint16(u)) + check(t, `uint32="%d"`, us, uint32(u)) + check(t, `uint64="%d"`, us, uint64(u)) + + const f = 3.141592 + const fs = `3.141592` + check(t, `float64="%g"`, fs, f) + check(t, `float32="%g"`, fs, float32(f)) + check(t, `float64="%g"`, fs, float64(f)) +} + +func TestArrayTypes(t *testing.T) { + var a0 [10]int + check(t, `array="array";`, `array`, a0) + + a1 := [...]int{1, 2, 3} + check(t, `array="array";`, `array`, a1) + check(t, `array={*}; int="%d";`, `123`, a1) + check(t, `array={* / ", "}; int="%d";`, `1, 2, 3`, a1) + check(t, `array={* / *}; int="%d";`, `12233`, a1) + + a2 := []interface{}{42, "foo", 3.14} + check(t, `array={* / ", "}; interface=*; string="bar"; default="%v";`, `42, bar, 3.14`, a2) +} + +func TestChanTypes(t *testing.T) { + var c0 chan int + check(t, `chan="chan"`, `chan`, c0) + + c1 := make(chan int) + go func() { c1 <- 42 }() + check(t, `chan="chan"`, `chan`, c1) + // check(t, `chan=*`, `42`, c1); // reflection support for chans incomplete +} + +func TestFuncTypes(t *testing.T) { + var f0 func() int + check(t, `func="func"`, `func`, f0) + + f1 := func() int { return 42 } + check(t, `func="func"`, `func`, f1) + // check(t, `func=*`, `42`, f1); // reflection support for funcs incomplete +} + +func TestMapTypes(t *testing.T) { + var m0 map[string]int + check(t, `map="map"`, `map`, m0) + + m1 := map[string]int{} + check(t, `map="map"`, `map`, m1) + // check(t, `map=*`, ``, m1); // reflection support for maps incomplete +} + +func TestPointerTypes(t *testing.T) { + var p0 *int + check(t, `ptr="ptr"`, `ptr`, p0) + check(t, `ptr=*`, ``, p0) + check(t, `ptr=*|"nil"`, `nil`, p0) + + x := 99991 + p1 := &x + check(t, `ptr="ptr"`, `ptr`, p1) + check(t, `ptr=*; int="%d"`, `99991`, p1) +} + +func TestDefaultRule(t *testing.T) { + check(t, `default="%v"`, `42foo3.14`, 42, "foo", 3.14) + check(t, `default="%v"; int="%x"`, `abcdef`, 10, 11, 12, 13, 14, 15) + check(t, `default="%v"; int="%x"`, `ab**ef`, 10, 11, "**", 14, 15) + check(t, `default="%x"; int=@:default`, `abcdef`, 10, 11, 12, 13, 14, 15) +} + +func TestGlobalSeparatorRule(t *testing.T) { + check(t, `int="%d"; / ="-"`, `1-2-3-4`, 1, 2, 3, 4) + check(t, `int="%x%x"; / ="*"`, `aa*aa`, 10, 10) +} + +// ---------------------------------------------------------------------------- +// Formatting of a struct + +type T1 struct { + a int +} + +const F1 = `datafmt "datafmt";` + + `int = "%d";` + + `datafmt.T1 = "<" a ">";` + +func TestStruct1(t *testing.T) { check(t, F1, "<42>", T1{42}) } + +// ---------------------------------------------------------------------------- +// Formatting of a struct with an optional field (ptr) + +type T2 struct { + s string + p *T1 +} + +const F2a = F1 + + `string = "%s";` + + `ptr = *;` + + `datafmt.T2 = s ["-" p "-"];` + +const F2b = F1 + + `string = "%s";` + + `ptr = *;` + + `datafmt.T2 = s ("-" p "-" | "empty");` + +func TestStruct2(t *testing.T) { + check(t, F2a, "foo", T2{"foo", nil}) + check(t, F2a, "bar-<17>-", T2{"bar", &T1{17}}) + check(t, F2b, "fooempty", T2{"foo", nil}) +} + +// ---------------------------------------------------------------------------- +// Formatting of a struct with a repetitive field (slice) + +type T3 struct { + s string + a []int +} + +const F3a = `datafmt "datafmt";` + + `default = "%v";` + + `array = *;` + + `datafmt.T3 = s {" " a a / ","};` + +const F3b = `datafmt "datafmt";` + + `int = "%d";` + + `string = "%s";` + + `array = *;` + + `nil = ;` + + `empty = *:nil;` + + `datafmt.T3 = s [a:empty ": " {a / "-"}]` + +func TestStruct3(t *testing.T) { + check(t, F3a, "foo", T3{"foo", nil}) + check(t, F3a, "foo 00, 11, 22", T3{"foo", []int{0, 1, 2}}) + check(t, F3b, "bar", T3{"bar", nil}) + check(t, F3b, "bal: 2-3-5", T3{"bal", []int{2, 3, 5}}) +} + +// ---------------------------------------------------------------------------- +// Formatting of a struct with alternative field + +type T4 struct { + x *int + a []int +} + +const F4a = `datafmt "datafmt";` + + `int = "%d";` + + `ptr = *;` + + `array = *;` + + `nil = ;` + + `empty = *:nil;` + + `datafmt.T4 = "<" (x:empty x | "-") ">" ` + +const F4b = `datafmt "datafmt";` + + `int = "%d";` + + `ptr = *;` + + `array = *;` + + `nil = ;` + + `empty = *:nil;` + + `datafmt.T4 = "<" (a:empty {a / ", "} | "-") ">" ` + +func TestStruct4(t *testing.T) { + x := 7 + check(t, F4a, "<->", T4{nil, nil}) + check(t, F4a, "<7>", T4{&x, nil}) + check(t, F4b, "<->", T4{nil, nil}) + check(t, F4b, "<2, 3, 7>", T4{nil, []int{2, 3, 7}}) +} + +// ---------------------------------------------------------------------------- +// Formatting a struct (documentation example) + +type Point struct { + name string + x, y int +} + +const FPoint = `datafmt "datafmt";` + + `int = "%d";` + + `hexInt = "0x%x";` + + `string = "---%s---";` + + `datafmt.Point = name "{" x ", " y:hexInt "}";` + +func TestStructPoint(t *testing.T) { + p := Point{"foo", 3, 15} + check(t, FPoint, "---foo---{3, 0xf}", p) +} + +// ---------------------------------------------------------------------------- +// Formatting a slice (documentation example) + +const FSlice = `int = "%b";` + + `array = { * / ", " }` + +func TestSlice(t *testing.T) { check(t, FSlice, "10, 11, 101, 111", []int{2, 3, 5, 7}) } + +// TODO add more tests diff --git a/src/pkg/exp/datafmt/parser.go b/src/pkg/exp/datafmt/parser.go new file mode 100644 index 000000000..a2ddd3897 --- /dev/null +++ b/src/pkg/exp/datafmt/parser.go @@ -0,0 +1,368 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package datafmt + +import ( + "go/scanner" + "go/token" + "os" + "strconv" + "strings" +) + +// ---------------------------------------------------------------------------- +// Parsing + +type parser struct { + scanner.ErrorVector + scanner scanner.Scanner + file *token.File + pos token.Pos // token position + tok token.Token // one token look-ahead + lit string // token literal + + packs map[string]string // PackageName -> ImportPath + rules map[string]expr // RuleName -> Expression +} + +func (p *parser) next() { + p.pos, p.tok, p.lit = p.scanner.Scan() + switch p.tok { + case token.CHAN, token.FUNC, token.INTERFACE, token.MAP, token.STRUCT: + // Go keywords for composite types are type names + // returned by reflect. Accept them as identifiers. + p.tok = token.IDENT // p.lit is already set correctly + } +} + +func (p *parser) init(fset *token.FileSet, filename string, src []byte) { + p.ErrorVector.Reset() + p.file = fset.AddFile(filename, fset.Base(), len(src)) + p.scanner.Init(p.file, src, p, scanner.AllowIllegalChars) // return '@' as token.ILLEGAL w/o error message + p.next() // initializes pos, tok, lit + p.packs = make(map[string]string) + p.rules = make(map[string]expr) +} + +func (p *parser) error(pos token.Pos, msg string) { + p.Error(p.file.Position(pos), msg) +} + +func (p *parser) errorExpected(pos token.Pos, msg string) { + msg = "expected " + msg + if pos == p.pos { + // the error happened at the current position; + // make the error message more specific + msg += ", found '" + p.tok.String() + "'" + if p.tok.IsLiteral() { + msg += " " + p.lit + } + } + p.error(pos, msg) +} + +func (p *parser) expect(tok token.Token) token.Pos { + pos := p.pos + if p.tok != tok { + p.errorExpected(pos, "'"+tok.String()+"'") + } + p.next() // make progress in any case + return pos +} + +func (p *parser) parseIdentifier() string { + name := p.lit + p.expect(token.IDENT) + return name +} + +func (p *parser) parseTypeName() (string, bool) { + pos := p.pos + name, isIdent := p.parseIdentifier(), true + if p.tok == token.PERIOD { + // got a package name, lookup package + if importPath, found := p.packs[name]; found { + name = importPath + } else { + p.error(pos, "package not declared: "+name) + } + p.next() + name, isIdent = name+"."+p.parseIdentifier(), false + } + return name, isIdent +} + +// Parses a rule name and returns it. If the rule name is +// a package-qualified type name, the package name is resolved. +// The 2nd result value is true iff the rule name consists of a +// single identifier only (and thus could be a package name). +// +func (p *parser) parseRuleName() (string, bool) { + name, isIdent := "", false + switch p.tok { + case token.IDENT: + name, isIdent = p.parseTypeName() + case token.DEFAULT: + name = "default" + p.next() + case token.QUO: + name = "/" + p.next() + default: + p.errorExpected(p.pos, "rule name") + p.next() // make progress in any case + } + return name, isIdent +} + +func (p *parser) parseString() string { + s := "" + if p.tok == token.STRING { + s, _ = strconv.Unquote(p.lit) + // Unquote may fail with an error, but only if the scanner found + // an illegal string in the first place. In this case the error + // has already been reported. + p.next() + return s + } else { + p.expect(token.STRING) + } + return s +} + +func (p *parser) parseLiteral() literal { + s := []byte(p.parseString()) + + // A string literal may contain %-format specifiers. To simplify + // and speed up printing of the literal, split it into segments + // that start with "%" possibly followed by a last segment that + // starts with some other character. + var list []interface{} + i0 := 0 + for i := 0; i < len(s); i++ { + if s[i] == '%' && i+1 < len(s) { + // the next segment starts with a % format + if i0 < i { + // the current segment is not empty, split it off + list = append(list, s[i0:i]) + i0 = i + } + i++ // skip %; let loop skip over char after % + } + } + // the final segment may start with any character + // (it is empty iff the string is empty) + list = append(list, s[i0:]) + + // convert list into a literal + lit := make(literal, len(list)) + for i := 0; i < len(list); i++ { + lit[i] = list[i].([]byte) + } + + return lit +} + +func (p *parser) parseField() expr { + var fname string + switch p.tok { + case token.ILLEGAL: + if p.lit != "@" { + return nil + } + fname = "@" + p.next() + case token.MUL: + fname = "*" + p.next() + case token.IDENT: + fname = p.parseIdentifier() + default: + return nil + } + + var ruleName string + if p.tok == token.COLON { + p.next() + ruleName, _ = p.parseRuleName() + } + + return &field{fname, ruleName} +} + +func (p *parser) parseOperand() (x expr) { + switch p.tok { + case token.STRING: + x = p.parseLiteral() + + case token.LPAREN: + p.next() + x = p.parseExpression() + if p.tok == token.SHR { + p.next() + x = &group{x, p.parseExpression()} + } + p.expect(token.RPAREN) + + case token.LBRACK: + p.next() + x = &option{p.parseExpression()} + p.expect(token.RBRACK) + + case token.LBRACE: + p.next() + x = p.parseExpression() + var div expr + if p.tok == token.QUO { + p.next() + div = p.parseExpression() + } + x = &repetition{x, div} + p.expect(token.RBRACE) + + default: + x = p.parseField() // may be nil + } + + return x +} + +func (p *parser) parseSequence() expr { + var list []interface{} + + for x := p.parseOperand(); x != nil; x = p.parseOperand() { + list = append(list, x) + } + + // no need for a sequence if list.Len() < 2 + switch len(list) { + case 0: + return nil + case 1: + return list[0].(expr) + } + + // convert list into a sequence + seq := make(sequence, len(list)) + for i := 0; i < len(list); i++ { + seq[i] = list[i].(expr) + } + return seq +} + +func (p *parser) parseExpression() expr { + var list []interface{} + + for { + x := p.parseSequence() + if x != nil { + list = append(list, x) + } + if p.tok != token.OR { + break + } + p.next() + } + + // no need for an alternatives if list.Len() < 2 + switch len(list) { + case 0: + return nil + case 1: + return list[0].(expr) + } + + // convert list into a alternatives + alt := make(alternatives, len(list)) + for i := 0; i < len(list); i++ { + alt[i] = list[i].(expr) + } + return alt +} + +func (p *parser) parseFormat() { + for p.tok != token.EOF { + pos := p.pos + + name, isIdent := p.parseRuleName() + switch p.tok { + case token.STRING: + // package declaration + importPath := p.parseString() + + // add package declaration + if !isIdent { + p.error(pos, "illegal package name: "+name) + } else if _, found := p.packs[name]; !found { + p.packs[name] = importPath + } else { + p.error(pos, "package already declared: "+name) + } + + case token.ASSIGN: + // format rule + p.next() + x := p.parseExpression() + + // add rule + if _, found := p.rules[name]; !found { + p.rules[name] = x + } else { + p.error(pos, "format rule already declared: "+name) + } + + default: + p.errorExpected(p.pos, "package declaration or format rule") + p.next() // make progress in any case + } + + if p.tok == token.SEMICOLON { + p.next() + } else { + break + } + } + p.expect(token.EOF) +} + +func remap(p *parser, name string) string { + i := strings.Index(name, ".") + if i >= 0 { + packageName, suffix := name[0:i], name[i:] + // lookup package + if importPath, found := p.packs[packageName]; found { + name = importPath + suffix + } else { + var invalidPos token.Position + p.Error(invalidPos, "package not declared: "+packageName) + } + } + return name +} + +// Parse parses a set of format productions from source src. Custom +// formatters may be provided via a map of formatter functions. If +// there are no errors, the result is a Format and the error is nil. +// Otherwise the format is nil and a non-empty ErrorList is returned. +// +func Parse(fset *token.FileSet, filename string, src []byte, fmap FormatterMap) (Format, os.Error) { + // parse source + var p parser + p.init(fset, filename, src) + p.parseFormat() + + // add custom formatters, if any + for name, form := range fmap { + name = remap(&p, name) + if _, found := p.rules[name]; !found { + p.rules[name] = &custom{name, form} + } else { + var invalidPos token.Position + p.Error(invalidPos, "formatter already declared: "+name) + } + } + + return p.rules, p.GetError(scanner.NoMultiples) +} diff --git a/src/pkg/exp/gui/Makefile b/src/pkg/exp/gui/Makefile new file mode 100644 index 000000000..af065e4a5 --- /dev/null +++ b/src/pkg/exp/gui/Makefile @@ -0,0 +1,11 @@ +# Copyright 2009 The Go Authors. All rights reserved. +# Use of this source code is governed by a BSD-style +# license that can be found in the LICENSE file. + +include ../../../Make.inc + +TARG=exp/gui +GOFILES=\ + gui.go\ + +include ../../../Make.pkg diff --git a/src/pkg/exp/gui/gui.go b/src/pkg/exp/gui/gui.go new file mode 100644 index 000000000..171499186 --- /dev/null +++ b/src/pkg/exp/gui/gui.go @@ -0,0 +1,58 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Package gui defines a basic graphical user interface programming model. +package gui + +import ( + "image" + "image/draw" + "os" +) + +// A Window represents a single graphics window. +type Window interface { + // Screen returns an editable Image for the window. + Screen() draw.Image + // FlushImage flushes changes made to Screen() back to screen. + FlushImage() + // EventChan returns a channel carrying UI events such as key presses, + // mouse movements and window resizes. + EventChan() <-chan interface{} + // Close closes the window. + Close() os.Error +} + +// A KeyEvent is sent for a key press or release. +type KeyEvent struct { + // The value k represents key k being pressed. + // The value -k represents key k being released. + // The specific set of key values is not specified, + // but ordinary characters represent themselves. + Key int +} + +// A MouseEvent is sent for a button press or release or for a mouse movement. +type MouseEvent struct { + // Buttons is a bit mask of buttons: 1<<0 is left, 1<<1 middle, 1<<2 right. + // It represents button state and not necessarily the state delta: bit 0 + // being on means that the left mouse button is down, but does not imply + // that the same button was up in the previous MouseEvent. + Buttons int + // Loc is the location of the cursor. + Loc image.Point + // Nsec is the event's timestamp. + Nsec int64 +} + +// A ConfigEvent is sent each time the window's color model or size changes. +// The client should respond by calling Window.Screen to obtain a new image. +type ConfigEvent struct { + Config image.Config +} + +// An ErrEvent is sent when an error occurs. +type ErrEvent struct { + Err os.Error +} diff --git a/src/pkg/exp/gui/x11/Makefile b/src/pkg/exp/gui/x11/Makefile new file mode 100644 index 000000000..88cc1e23b --- /dev/null +++ b/src/pkg/exp/gui/x11/Makefile @@ -0,0 +1,12 @@ +# Copyright 2009 The Go Authors. All rights reserved. +# Use of this source code is governed by a BSD-style +# license that can be found in the LICENSE file. + +include ../../../../Make.inc + +TARG=exp/gui/x11 +GOFILES=\ + auth.go\ + conn.go\ + +include ../../../../Make.pkg diff --git a/src/pkg/exp/gui/x11/auth.go b/src/pkg/exp/gui/x11/auth.go new file mode 100644 index 000000000..d48936ac1 --- /dev/null +++ b/src/pkg/exp/gui/x11/auth.go @@ -0,0 +1,93 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package x11 + +import ( + "bufio" + "io" + "os" +) + +// readU16BE reads a big-endian uint16 from r, using b as a scratch buffer. +func readU16BE(r io.Reader, b []byte) (uint16, os.Error) { + _, err := io.ReadFull(r, b[0:2]) + if err != nil { + return 0, err + } + return uint16(b[0])<<8 + uint16(b[1]), nil +} + +// readStr reads a length-prefixed string from r, using b as a scratch buffer. +func readStr(r io.Reader, b []byte) (string, os.Error) { + n, err := readU16BE(r, b) + if err != nil { + return "", err + } + if int(n) > len(b) { + return "", os.NewError("Xauthority entry too long for buffer") + } + _, err = io.ReadFull(r, b[0:n]) + if err != nil { + return "", err + } + return string(b[0:n]), nil +} + +// readAuth reads the X authority file and returns the name/data pair for the display. +// displayStr is the "12" out of a $DISPLAY like ":12.0". +func readAuth(displayStr string) (name, data string, err os.Error) { + // b is a scratch buffer to use and should be at least 256 bytes long + // (i.e. it should be able to hold a hostname). + var b [256]byte + // As per /usr/include/X11/Xauth.h. + const familyLocal = 256 + + fn := os.Getenv("XAUTHORITY") + if fn == "" { + home := os.Getenv("HOME") + if home == "" { + err = os.NewError("Xauthority not found: $XAUTHORITY, $HOME not set") + return + } + fn = home + "/.Xauthority" + } + r, err := os.Open(fn) + if err != nil { + return + } + defer r.Close() + br := bufio.NewReader(r) + + hostname, err := os.Hostname() + if err != nil { + return + } + for { + family, err := readU16BE(br, b[0:2]) + if err != nil { + return + } + addr, err := readStr(br, b[0:]) + if err != nil { + return + } + disp, err := readStr(br, b[0:]) + if err != nil { + return + } + name0, err := readStr(br, b[0:]) + if err != nil { + return + } + data0, err := readStr(br, b[0:]) + if err != nil { + return + } + if family == familyLocal && addr == hostname && disp == displayStr { + return name0, data0, nil + } + } + panic("unreachable") +} diff --git a/src/pkg/exp/gui/x11/conn.go b/src/pkg/exp/gui/x11/conn.go new file mode 100644 index 000000000..1d237816a --- /dev/null +++ b/src/pkg/exp/gui/x11/conn.go @@ -0,0 +1,627 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Package x11 implements an X11 backend for the exp/gui package. +// +// The X protocol specification is at ftp://ftp.x.org/pub/X11R7.0/doc/PDF/proto.pdf. +// A summary of the wire format can be found in XCB's xproto.xml. +package x11 + +import ( + "bufio" + "exp/gui" + "image" + "image/draw" + "io" + "log" + "net" + "os" + "strconv" + "strings" + "time" +) + +type resID uint32 // X resource IDs. + +// TODO(nigeltao): Handle window resizes. +const ( + windowHeight = 600 + windowWidth = 800 +) + +const ( + keymapLo = 8 + keymapHi = 255 +) + +type conn struct { + c io.Closer + r *bufio.Reader + w *bufio.Writer + + gc, window, root, visual resID + + img *image.RGBA + eventc chan interface{} + mouseState gui.MouseEvent + + buf [256]byte // General purpose scratch buffer. + + flush chan bool + flushBuf0 [24]byte + flushBuf1 [4 * 1024]byte +} + +// writeSocket runs in its own goroutine, serving both FlushImage calls +// directly from the exp/gui client and indirectly from X expose events. +// It paints c.img to the X server via PutImage requests. +func (c *conn) writeSocket() { + defer c.c.Close() + for _ = range c.flush { + b := c.img.Bounds() + if b.Empty() { + continue + } + // Each X request has a 16-bit length (in terms of 4-byte units). To avoid going over + // this limit, we send PutImage for each row of the image, rather than trying to paint + // the entire image in one X request. This approach could easily be optimized (or the + // X protocol may have an escape sequence to delimit very large requests). + // TODO(nigeltao): See what XCB's xcb_put_image does in this situation. + units := 6 + b.Dx() + if units > 0xffff || b.Dy() > 0xffff { + log.Print("x11: window is too large for PutImage") + return + } + + c.flushBuf0[0] = 0x48 // PutImage opcode. + c.flushBuf0[1] = 0x02 // XCB_IMAGE_FORMAT_Z_PIXMAP. + c.flushBuf0[2] = uint8(units) + c.flushBuf0[3] = uint8(units >> 8) + setU32LE(c.flushBuf0[4:8], uint32(c.window)) + setU32LE(c.flushBuf0[8:12], uint32(c.gc)) + setU32LE(c.flushBuf0[12:16], 1<<16|uint32(b.Dx())) + c.flushBuf0[21] = 0x18 // depth = 24 bits. + + for y := b.Min.Y; y < b.Max.Y; y++ { + setU32LE(c.flushBuf0[16:20], uint32(y<<16)) + if _, err := c.w.Write(c.flushBuf0[:24]); err != nil { + if err != os.EOF { + log.Println("x11:", err.String()) + } + return + } + p := c.img.Pix[(y-b.Min.Y)*c.img.Stride:] + for x, dx := 0, 4*b.Dx(); x < dx; { + nx := dx - x + if nx > len(c.flushBuf1) { + nx = len(c.flushBuf1) &^ 3 + } + for i := 0; i < nx; i += 4 { + // X11's order is BGRX, not RGBA. + c.flushBuf1[i+0] = p[x+i+2] + c.flushBuf1[i+1] = p[x+i+1] + c.flushBuf1[i+2] = p[x+i+0] + } + x += nx + if _, err := c.w.Write(c.flushBuf1[:nx]); err != nil { + if err != os.EOF { + log.Println("x11:", err.String()) + } + return + } + } + } + if err := c.w.Flush(); err != nil { + if err != os.EOF { + log.Println("x11:", err.String()) + } + return + } + } +} + +func (c *conn) Screen() draw.Image { return c.img } + +func (c *conn) FlushImage() { + select { + case c.flush <- false: + // Flush notification sent. + default: + // Could not send. + // Flush notification must be pending already. + } +} + +func (c *conn) Close() os.Error { + // Shut down the writeSocket goroutine. This will close the socket to the + // X11 server, which will cause c.eventc to close. + close(c.flush) + for _ = range c.eventc { + // Drain the channel to allow the readSocket goroutine to shut down. + } + return nil +} + +func (c *conn) EventChan() <-chan interface{} { return c.eventc } + +// readSocket runs in its own goroutine, reading X events and sending gui +// events on c's EventChan. +func (c *conn) readSocket() { + var ( + keymap [256][]int + keysymsPerKeycode int + ) + defer close(c.eventc) + for { + // X events are always 32 bytes long. + if _, err := io.ReadFull(c.r, c.buf[:32]); err != nil { + if err != os.EOF { + c.eventc <- gui.ErrEvent{err} + } + return + } + switch c.buf[0] { + case 0x01: // Reply from a request (e.g. GetKeyboardMapping). + cookie := int(c.buf[3])<<8 | int(c.buf[2]) + if cookie != 1 { + // We issued only one request (GetKeyboardMapping) with a cookie of 1, + // so we shouldn't get any other reply from the X server. + c.eventc <- gui.ErrEvent{os.NewError("x11: unexpected cookie")} + return + } + keysymsPerKeycode = int(c.buf[1]) + b := make([]int, 256*keysymsPerKeycode) + for i := range keymap { + keymap[i] = b[i*keysymsPerKeycode : (i+1)*keysymsPerKeycode] + } + for i := keymapLo; i <= keymapHi; i++ { + m := keymap[i] + for j := range m { + u, err := readU32LE(c.r, c.buf[:4]) + if err != nil { + if err != os.EOF { + c.eventc <- gui.ErrEvent{err} + } + return + } + m[j] = int(u) + } + } + case 0x02, 0x03: // Key press, key release. + // X Keyboard Encoding is documented at http://tronche.com/gui/x/xlib/input/keyboard-encoding.html + // TODO(nigeltao): Do we need to implement the "MODE SWITCH / group modifier" feature + // or is that some no-longer-used X construct? + if keysymsPerKeycode < 2 { + // Either we haven't yet received the GetKeyboardMapping reply or + // the X server has sent one that's too short. + continue + } + keycode := int(c.buf[1]) + shift := int(c.buf[28]) & 0x01 + keysym := keymap[keycode][shift] + if keysym == 0 { + keysym = keymap[keycode][0] + } + // TODO(nigeltao): Should we send KeyEvents for Shift/Ctrl/Alt? Should Shift-A send + // the same int down the channel as the sent on just the A key? + // TODO(nigeltao): How should IME events (e.g. key presses that should generate CJK text) work? Or + // is that outside the scope of the gui.Window interface? + if c.buf[0] == 0x03 { + keysym = -keysym + } + c.eventc <- gui.KeyEvent{keysym} + case 0x04, 0x05: // Button press, button release. + mask := 1 << (c.buf[1] - 1) + if c.buf[0] == 0x04 { + c.mouseState.Buttons |= mask + } else { + c.mouseState.Buttons &^= mask + } + c.mouseState.Nsec = time.Nanoseconds() + c.eventc <- c.mouseState + case 0x06: // Motion notify. + c.mouseState.Loc.X = int(int16(c.buf[25])<<8 | int16(c.buf[24])) + c.mouseState.Loc.Y = int(int16(c.buf[27])<<8 | int16(c.buf[26])) + c.mouseState.Nsec = time.Nanoseconds() + c.eventc <- c.mouseState + case 0x0c: // Expose. + // A single user action could trigger multiple expose events (e.g. if moving another + // window with XShape'd rounded corners over our window). In that case, the X server will + // send a uint16 count (in bytes 16-17) of the number of additional expose events coming. + // We could parse each event for the (x, y, width, height) and maintain a minimal dirty + // rectangle, but for now, the simplest approach is to paint the entire window, when + // receiving the final event in the series. + if c.buf[17] == 0 && c.buf[16] == 0 { + // TODO(nigeltao): Should we ignore the very first expose event? A freshly mapped window + // will trigger expose, but until the first c.FlushImage call, there's probably nothing to + // paint but black. For an 800x600 window, at 4 bytes per pixel, each repaint writes about + // 2MB over the socket. + c.FlushImage() + } + // TODO(nigeltao): Should we listen to DestroyNotify (0x11) and ResizeRequest (0x19) events? + // What about EnterNotify (0x07) and LeaveNotify (0x08)? + } + } +} + +// connect connects to the X server given by the full X11 display name (e.g. +// ":12.0") and returns the connection as well as the portion of the full name +// that is the display number (e.g. "12"). +// Examples: +// connect(":1") // calls net.Dial("unix", "", "/tmp/.X11-unix/X1"), displayStr="1" +// connect("/tmp/launch-123/:0") // calls net.Dial("unix", "", "/tmp/launch-123/:0"), displayStr="0" +// connect("hostname:2.1") // calls net.Dial("tcp", "", "hostname:6002"), displayStr="2" +// connect("tcp/hostname:1.0") // calls net.Dial("tcp", "", "hostname:6001"), displayStr="1" +func connect(display string) (conn net.Conn, displayStr string, err os.Error) { + colonIdx := strings.LastIndex(display, ":") + if colonIdx < 0 { + return nil, "", os.NewError("bad display: " + display) + } + // Parse the section before the colon. + var protocol, host, socket string + if display[0] == '/' { + socket = display[:colonIdx] + } else { + if i := strings.LastIndex(display, "/"); i < 0 { + // The default protocol is TCP. + protocol = "tcp" + host = display[:colonIdx] + } else { + protocol = display[:i] + host = display[i+1 : colonIdx] + } + } + // Parse the section after the colon. + after := display[colonIdx+1:] + if after == "" { + return nil, "", os.NewError("bad display: " + display) + } + if i := strings.LastIndex(after, "."); i < 0 { + displayStr = after + } else { + displayStr = after[:i] + } + displayInt, err := strconv.Atoi(displayStr) + if err != nil || displayInt < 0 { + return nil, "", os.NewError("bad display: " + display) + } + // Make the connection. + if socket != "" { + conn, err = net.Dial("unix", socket+":"+displayStr) + } else if host != "" { + conn, err = net.Dial(protocol, host+":"+strconv.Itoa(6000+displayInt)) + } else { + conn, err = net.Dial("unix", "/tmp/.X11-unix/X"+displayStr) + } + if err != nil { + return nil, "", os.NewError("cannot connect to " + display + ": " + err.String()) + } + return +} + +// authenticate authenticates ourselves with the X server. +// displayStr is the "12" out of ":12.0". +func authenticate(w *bufio.Writer, displayStr string) os.Error { + key, value, err := readAuth(displayStr) + if err != nil { + return err + } + // Assume that the authentication protocol is "MIT-MAGIC-COOKIE-1". + if len(key) != 18 || len(value) != 16 { + return os.NewError("unsupported Xauth") + } + // 0x006c means little-endian. 0x000b, 0x0000 means X major version 11, minor version 0. + // 0x0012 and 0x0010 means the auth key and value have lengths 18 and 16. + // The final 0x0000 is padding, so that the string length is a multiple of 4. + _, err = io.WriteString(w, "\x6c\x00\x0b\x00\x00\x00\x12\x00\x10\x00\x00\x00") + if err != nil { + return err + } + _, err = io.WriteString(w, key) + if err != nil { + return err + } + // Again, the 0x0000 is padding. + _, err = io.WriteString(w, "\x00\x00") + if err != nil { + return err + } + _, err = io.WriteString(w, value) + if err != nil { + return err + } + err = w.Flush() + if err != nil { + return err + } + return nil +} + +// readU8 reads a uint8 from r, using b as a scratch buffer. +func readU8(r io.Reader, b []byte) (uint8, os.Error) { + _, err := io.ReadFull(r, b[:1]) + if err != nil { + return 0, err + } + return uint8(b[0]), nil +} + +// readU16LE reads a little-endian uint16 from r, using b as a scratch buffer. +func readU16LE(r io.Reader, b []byte) (uint16, os.Error) { + _, err := io.ReadFull(r, b[:2]) + if err != nil { + return 0, err + } + return uint16(b[0]) | uint16(b[1])<<8, nil +} + +// readU32LE reads a little-endian uint32 from r, using b as a scratch buffer. +func readU32LE(r io.Reader, b []byte) (uint32, os.Error) { + _, err := io.ReadFull(r, b[:4]) + if err != nil { + return 0, err + } + return uint32(b[0]) | uint32(b[1])<<8 | uint32(b[2])<<16 | uint32(b[3])<<24, nil +} + +// setU32LE sets b[:4] to be the little-endian representation of u. +func setU32LE(b []byte, u uint32) { + b[0] = byte((u >> 0) & 0xff) + b[1] = byte((u >> 8) & 0xff) + b[2] = byte((u >> 16) & 0xff) + b[3] = byte((u >> 24) & 0xff) +} + +// checkPixmapFormats checks that we have an agreeable X pixmap Format. +func checkPixmapFormats(r io.Reader, b []byte, n int) (agree bool, err os.Error) { + for i := 0; i < n; i++ { + _, err = io.ReadFull(r, b[:8]) + if err != nil { + return + } + // Byte 0 is depth, byte 1 is bits-per-pixel, byte 2 is scanline-pad, the rest (5) is padding. + if b[0] == 24 && b[1] == 32 { + agree = true + } + } + return +} + +// checkDepths checks that we have an agreeable X Depth (i.e. one that has an agreeable X VisualType). +func checkDepths(r io.Reader, b []byte, n int, visual uint32) (agree bool, err os.Error) { + for i := 0; i < n; i++ { + depth, err := readU16LE(r, b) + if err != nil { + return + } + depth &= 0xff + visualsLen, err := readU16LE(r, b) + if err != nil { + return + } + // Ignore 4 bytes of padding. + _, err = io.ReadFull(r, b[:4]) + if err != nil { + return + } + for j := 0; j < int(visualsLen); j++ { + // Read 24 bytes: visual(4), class(1), bits per rgb value(1), colormap entries(2), + // red mask(4), green mask(4), blue mask(4), padding(4). + v, err := readU32LE(r, b) + _, err = readU32LE(r, b) + rm, err := readU32LE(r, b) + gm, err := readU32LE(r, b) + bm, err := readU32LE(r, b) + _, err = readU32LE(r, b) + if err != nil { + return + } + if v == visual && rm == 0xff0000 && gm == 0xff00 && bm == 0xff && depth == 24 { + agree = true + } + } + } + return +} + +// checkScreens checks that we have an agreeable X Screen. +func checkScreens(r io.Reader, b []byte, n int) (root, visual uint32, err os.Error) { + for i := 0; i < n; i++ { + root0, err := readU32LE(r, b) + if err != nil { + return + } + // Ignore the next 7x4 bytes, which is: colormap, whitepixel, blackpixel, current input masks, + // width and height (pixels), width and height (mm), min and max installed maps. + _, err = io.ReadFull(r, b[:28]) + if err != nil { + return + } + visual0, err := readU32LE(r, b) + if err != nil { + return + } + // Next 4 bytes: backing stores, save unders, root depth, allowed depths length. + x, err := readU32LE(r, b) + if err != nil { + return + } + nDepths := int(x >> 24) + agree, err := checkDepths(r, b, nDepths, visual0) + if err != nil { + return + } + if agree && root == 0 { + root = root0 + visual = visual0 + } + } + return +} + +// handshake performs the protocol handshake with the X server, and ensures +// that the server provides a compatible Screen, Depth, etc. +func (c *conn) handshake() os.Error { + _, err := io.ReadFull(c.r, c.buf[:8]) + if err != nil { + return err + } + // Byte 0 should be 1 (success), bytes 2:6 should be 0xb0000000 (major/minor version 11.0). + if c.buf[0] != 1 || c.buf[2] != 11 || c.buf[3] != 0 || c.buf[4] != 0 || c.buf[5] != 0 { + return os.NewError("unsupported X version") + } + // Ignore the release number. + _, err = io.ReadFull(c.r, c.buf[:4]) + if err != nil { + return err + } + // Read the resource ID base. + resourceIdBase, err := readU32LE(c.r, c.buf[:4]) + if err != nil { + return err + } + // Read the resource ID mask. + resourceIdMask, err := readU32LE(c.r, c.buf[:4]) + if err != nil { + return err + } + if resourceIdMask < 256 { + return os.NewError("X resource ID mask is too small") + } + // Ignore the motion buffer size. + _, err = io.ReadFull(c.r, c.buf[:4]) + if err != nil { + return err + } + // Read the vendor length and round it up to a multiple of 4, + // for X11 protocol alignment reasons. + vendorLen, err := readU16LE(c.r, c.buf[:2]) + if err != nil { + return err + } + vendorLen = (vendorLen + 3) &^ 3 + // Read the maximum request length. + maxReqLen, err := readU16LE(c.r, c.buf[:2]) + if err != nil { + return err + } + if maxReqLen != 0xffff { + return os.NewError("unsupported X maximum request length") + } + // Read the roots length. + rootsLen, err := readU8(c.r, c.buf[:1]) + if err != nil { + return err + } + // Read the pixmap formats length. + pixmapFormatsLen, err := readU8(c.r, c.buf[:1]) + if err != nil { + return err + } + // Ignore some things that we don't care about (totaling 10 + vendorLen bytes): + // imageByteOrder(1), bitmapFormatBitOrder(1), bitmapFormatScanlineUnit(1) bitmapFormatScanlinePad(1), + // minKeycode(1), maxKeycode(1), padding(4), vendor (vendorLen). + if 10+int(vendorLen) > cap(c.buf) { + return os.NewError("unsupported X vendor") + } + _, err = io.ReadFull(c.r, c.buf[:10+int(vendorLen)]) + if err != nil { + return err + } + // Check that we have an agreeable pixmap format. + agree, err := checkPixmapFormats(c.r, c.buf[:8], int(pixmapFormatsLen)) + if err != nil { + return err + } + if !agree { + return os.NewError("unsupported X pixmap formats") + } + // Check that we have an agreeable screen. + root, visual, err := checkScreens(c.r, c.buf[:24], int(rootsLen)) + if err != nil { + return err + } + if root == 0 || visual == 0 { + return os.NewError("unsupported X screen") + } + c.gc = resID(resourceIdBase) + c.window = resID(resourceIdBase + 1) + c.root = resID(root) + c.visual = resID(visual) + return nil +} + +// NewWindow calls NewWindowDisplay with $DISPLAY. +func NewWindow() (gui.Window, os.Error) { + display := os.Getenv("DISPLAY") + if len(display) == 0 { + return nil, os.NewError("$DISPLAY not set") + } + return NewWindowDisplay(display) +} + +// NewWindowDisplay returns a new gui.Window, backed by a newly created and +// mapped X11 window. The X server to connect to is specified by the display +// string, such as ":1". +func NewWindowDisplay(display string) (gui.Window, os.Error) { + socket, displayStr, err := connect(display) + if err != nil { + return nil, err + } + c := new(conn) + c.c = socket + c.r = bufio.NewReader(socket) + c.w = bufio.NewWriter(socket) + err = authenticate(c.w, displayStr) + if err != nil { + return nil, err + } + err = c.handshake() + if err != nil { + return nil, err + } + + // Now that we're connected, show a window, via three X protocol messages. + // First, issue a GetKeyboardMapping request. This is the first request, and + // will be associated with a cookie of 1. + setU32LE(c.buf[0:4], 0x00020065) // 0x65 is the GetKeyboardMapping opcode, and the message is 2 x 4 bytes long. + setU32LE(c.buf[4:8], uint32((keymapHi-keymapLo+1)<<8|keymapLo)) + // Second, create a graphics context (GC). + setU32LE(c.buf[8:12], 0x00060037) // 0x37 is the CreateGC opcode, and the message is 6 x 4 bytes long. + setU32LE(c.buf[12:16], uint32(c.gc)) + setU32LE(c.buf[16:20], uint32(c.root)) + setU32LE(c.buf[20:24], 0x00010004) // Bit 2 is XCB_GC_FOREGROUND, bit 16 is XCB_GC_GRAPHICS_EXPOSURES. + setU32LE(c.buf[24:28], 0x00000000) // The Foreground is black. + setU32LE(c.buf[28:32], 0x00000000) // GraphicsExposures' value is unused. + // Third, create the window. + setU32LE(c.buf[32:36], 0x000a0001) // 0x01 is the CreateWindow opcode, and the message is 10 x 4 bytes long. + setU32LE(c.buf[36:40], uint32(c.window)) + setU32LE(c.buf[40:44], uint32(c.root)) + setU32LE(c.buf[44:48], 0x00000000) // Initial (x, y) is (0, 0). + setU32LE(c.buf[48:52], windowHeight<<16|windowWidth) + setU32LE(c.buf[52:56], 0x00010000) // Border width is 0, XCB_WINDOW_CLASS_INPUT_OUTPUT is 1. + setU32LE(c.buf[56:60], uint32(c.visual)) + setU32LE(c.buf[60:64], 0x00000802) // Bit 1 is XCB_CW_BACK_PIXEL, bit 11 is XCB_CW_EVENT_MASK. + setU32LE(c.buf[64:68], 0x00000000) // The Back-Pixel is black. + setU32LE(c.buf[68:72], 0x0000804f) // Key/button press and release, pointer motion, and expose event masks. + // Fourth, map the window. + setU32LE(c.buf[72:76], 0x00020008) // 0x08 is the MapWindow opcode, and the message is 2 x 4 bytes long. + setU32LE(c.buf[76:80], uint32(c.window)) + // Write the bytes. + _, err = c.w.Write(c.buf[:80]) + if err != nil { + return nil, err + } + err = c.w.Flush() + if err != nil { + return nil, err + } + + c.img = image.NewRGBA(windowWidth, windowHeight) + c.eventc = make(chan interface{}, 16) + c.flush = make(chan bool, 1) + go c.readSocket() + go c.writeSocket() + return c, nil +} diff --git a/src/pkg/exp/norm/Makefile b/src/pkg/exp/norm/Makefile new file mode 100644 index 000000000..a4dfb43f7 --- /dev/null +++ b/src/pkg/exp/norm/Makefile @@ -0,0 +1,44 @@ +# Copyright 2011 The Go Authors. All rights reserved. +# Use of this source code is governed by a BSD-style +# license that can be found in the LICENSE file. + +include ../../../Make.inc + +TARG=exp/norm +GOFILES=\ + composition.go\ + forminfo.go\ + normalize.go\ + tables.go\ + trie.go\ + +include ../../../Make.pkg + +CLEANFILES+=maketables maketesttables + +maketables: maketables.go triegen.go + $(GC) maketables.go triegen.go + $(LD) -o maketables maketables.$O + +maketesttables: maketesttables.go triegen.go + $(GC) maketesttables.go triegen.go + $(LD) -o maketesttables maketesttables.$O + +tables: maketables + ./maketables > tables.go + gofmt -w tables.go + +trietesttables: maketesttables + ./maketesttables > triedata_test.go + gofmt -w triedata_test.go + +# Build (but do not run) maketables during testing, +# just to make sure it still compiles. +testshort: maketables maketesttables + +# Downloads from www.unicode.org, so not part +# of standard test scripts. +test: testtables + +testtables: maketables + ./maketables -test -tables= diff --git a/src/pkg/exp/norm/composition.go b/src/pkg/exp/norm/composition.go new file mode 100644 index 000000000..b2d2abaf6 --- /dev/null +++ b/src/pkg/exp/norm/composition.go @@ -0,0 +1,344 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package norm + +import "utf8" + +const ( + maxCombiningChars = 30 + 2 // +2 to hold CGJ and Hangul overflow. + maxBackRunes = maxCombiningChars - 1 + maxNFCExpansion = 3 // NFC(0x1D160) + maxNFKCExpansion = 18 // NFKC(0xFDFA) + + maxRuneSizeInDecomp = 4 + // Need to multiply by 2 as we don't reuse byte buffer space for recombining. + maxByteBufferSize = 2 * maxRuneSizeInDecomp * maxCombiningChars // 256 +) + +// reorderBuffer is used to normalize a single segment. Characters inserted with +// insert() are decomposed and reordered based on CCC. The compose() method can +// be used to recombine characters. Note that the byte buffer does not hold +// the UTF-8 characters in order. Only the rune array is maintained in sorted +// order. flush() writes the resulting segment to a byte array. +type reorderBuffer struct { + rune [maxCombiningChars]runeInfo // Per character info. + byte [maxByteBufferSize]byte // UTF-8 buffer. Referenced by runeInfo.pos. + nrune int // Number of runeInfos. + nbyte uint8 // Number or bytes. + f formInfo +} + +// reset discards all characters from the buffer. +func (rb *reorderBuffer) reset() { + rb.nrune = 0 + rb.nbyte = 0 +} + +// flush appends the normalized segment to out and resets rb. +func (rb *reorderBuffer) flush(out []byte) []byte { + for i := 0; i < rb.nrune; i++ { + start := rb.rune[i].pos + end := start + rb.rune[i].size + out = append(out, rb.byte[start:end]...) + } + rb.reset() + return out +} + +// insertOrdered inserts a rune in the buffer, ordered by Canonical Combining Class. +// It returns false if the buffer is not large enough to hold the rune. +// It is used internally by insert. +func (rb *reorderBuffer) insertOrdered(info runeInfo) bool { + n := rb.nrune + if n >= maxCombiningChars { + return false + } + b := rb.rune[:] + cc := info.ccc + if cc > 0 { + // Find insertion position + move elements to make room. + for ; n > 0; n-- { + if b[n-1].ccc <= cc { + break + } + b[n] = b[n-1] + } + } + rb.nrune += 1 + pos := uint8(rb.nbyte) + rb.nbyte += info.size + info.pos = pos + b[n] = info + return true +} + +// insert inserts the given rune in the buffer ordered by CCC. +// It returns true if the buffer was large enough to hold the decomposed rune. +func (rb *reorderBuffer) insert(src []byte, info runeInfo) bool { + if info.size == 3 && isHangul(src) { + rune, _ := utf8.DecodeRune(src) + return rb.decomposeHangul(uint32(rune)) + } + pos := rb.nbyte + if info.flags.hasDecomposition() { + dcomp := rb.f.decompose(src) + for i := 0; i < len(dcomp); i += int(info.size) { + info = rb.f.info(dcomp[i:]) + if !rb.insertOrdered(info) { + return false + } + } + copy(rb.byte[pos:], dcomp) + } else { + if !rb.insertOrdered(info) { + return false + } + copy(rb.byte[pos:], src[:info.size]) + } + return true +} + +// insertString inserts the given rune in the buffer ordered by CCC. +// It returns true if the buffer was large enough to hold the decomposed rune. +func (rb *reorderBuffer) insertString(src string, info runeInfo) bool { + if info.size == 3 && isHangulString(src) { + rune, _ := utf8.DecodeRuneInString(src) + return rb.decomposeHangul(uint32(rune)) + } + pos := rb.nbyte + dcomp := rb.f.decomposeString(src) + dn := len(dcomp) + if dn != 0 { + for i := 0; i < dn; i += int(info.size) { + info = rb.f.info(dcomp[i:]) + if !rb.insertOrdered(info) { + return false + } + } + copy(rb.byte[pos:], dcomp) + } else { + if !rb.insertOrdered(info) { + return false + } + copy(rb.byte[pos:], src[:info.size]) + } + return true +} + +// appendRune inserts a rune at the end of the buffer. It is used for Hangul. +func (rb *reorderBuffer) appendRune(rune uint32) { + bn := rb.nbyte + sz := utf8.EncodeRune(rb.byte[bn:], int(rune)) + rb.nbyte += uint8(sz) + rb.rune[rb.nrune] = runeInfo{bn, uint8(sz), 0, 0} + rb.nrune++ +} + +// assignRune sets a rune at position pos. It is used for Hangul and recomposition. +func (rb *reorderBuffer) assignRune(pos int, rune uint32) { + bn := rb.nbyte + sz := utf8.EncodeRune(rb.byte[bn:], int(rune)) + rb.rune[pos] = runeInfo{bn, uint8(sz), 0, 0} + rb.nbyte += uint8(sz) +} + +// runeAt returns the rune at position n. It is used for Hangul and recomposition. +func (rb *reorderBuffer) runeAt(n int) uint32 { + inf := rb.rune[n] + rune, _ := utf8.DecodeRune(rb.byte[inf.pos : inf.pos+inf.size]) + return uint32(rune) +} + +// bytesAt returns the UTF-8 encoding of the rune at position n. +// It is used for Hangul and recomposition. +func (rb *reorderBuffer) bytesAt(n int) []byte { + inf := rb.rune[n] + return rb.byte[inf.pos : int(inf.pos)+int(inf.size)] +} + +// For Hangul we combine algorithmically, instead of using tables. +const ( + hangulBase = 0xAC00 // UTF-8(hangulBase) -> EA B0 80 + hangulBase0 = 0xEA + hangulBase1 = 0xB0 + hangulBase2 = 0x80 + + hangulEnd = hangulBase + jamoLVTCount // UTF-8(0xD7A4) -> ED 9E A4 + hangulEnd0 = 0xED + hangulEnd1 = 0x9E + hangulEnd2 = 0xA4 + + jamoLBase = 0x1100 // UTF-8(jamoLBase) -> E1 84 00 + jamoLBase0 = 0xE1 + jamoLBase1 = 0x84 + jamoLEnd = 0x1113 + jamoVBase = 0x1161 + jamoVEnd = 0x1176 + jamoTBase = 0x11A7 + jamoTEnd = 0x11C3 + + jamoTCount = 28 + jamoVCount = 21 + jamoVTCount = 21 * 28 + jamoLVTCount = 19 * 21 * 28 +) + +// Caller must verify that len(b) >= 3. +func isHangul(b []byte) bool { + b0 := b[0] + if b0 < hangulBase0 { + return false + } + b1 := b[1] + switch { + case b0 == hangulBase0: + return b1 >= hangulBase1 + case b0 < hangulEnd0: + return true + case b0 > hangulEnd0: + return false + case b1 < hangulEnd1: + return true + } + return b1 == hangulEnd1 && b[2] < hangulEnd2 +} + +// Caller must verify that len(b) >= 3. +func isHangulString(b string) bool { + b0 := b[0] + if b0 < hangulBase0 { + return false + } + b1 := b[1] + switch { + case b0 == hangulBase0: + return b1 >= hangulBase1 + case b0 < hangulEnd0: + return true + case b0 > hangulEnd0: + return false + case b1 < hangulEnd1: + return true + } + return b1 == hangulEnd1 && b[2] < hangulEnd2 +} + +// Caller must ensure len(b) >= 2. +func isJamoVT(b []byte) bool { + // True if (rune & 0xff00) == jamoLBase + return b[0] == jamoLBase0 && (b[1]&0xFC) == jamoLBase1 +} + +func isHangulWithoutJamoT(b []byte) bool { + c, _ := utf8.DecodeRune(b) + c -= hangulBase + return c < jamoLVTCount && c%jamoTCount == 0 +} + +// decomposeHangul algorithmically decomposes a Hangul rune into +// its Jamo components. +// See http://unicode.org/reports/tr15/#Hangul for details on decomposing Hangul. +func (rb *reorderBuffer) decomposeHangul(rune uint32) bool { + b := rb.rune[:] + n := rb.nrune + if n+3 > len(b) { + return false + } + rune -= hangulBase + x := rune % jamoTCount + rune /= jamoTCount + rb.appendRune(jamoLBase + rune/jamoVCount) + rb.appendRune(jamoVBase + rune%jamoVCount) + if x != 0 { + rb.appendRune(jamoTBase + x) + } + return true +} + +// combineHangul algorithmically combines Jamo character components into Hangul. +// See http://unicode.org/reports/tr15/#Hangul for details on combining Hangul. +func (rb *reorderBuffer) combineHangul() { + k := 1 + b := rb.rune[:] + bn := rb.nrune + for s, i := 0, 1; i < bn; i++ { + cccB := b[k-1].ccc + cccC := b[i].ccc + if cccB == 0 { + s = k - 1 + } + if s != k-1 && cccB >= cccC { + // b[i] is blocked by greater-equal cccX below it + b[k] = b[i] + k++ + } else { + l := rb.runeAt(s) // also used to compare to hangulBase + v := rb.runeAt(i) // also used to compare to jamoT + switch { + case jamoLBase <= l && l < jamoLEnd && + jamoVBase <= v && v < jamoVEnd: + // 11xx plus 116x to LV + rb.assignRune(s, hangulBase+ + (l-jamoLBase)*jamoVTCount+(v-jamoVBase)*jamoTCount) + case hangulBase <= l && l < hangulEnd && + jamoTBase < v && v < jamoTEnd && + ((l-hangulBase)%jamoTCount) == 0: + // ACxx plus 11Ax to LVT + rb.assignRune(s, l+v-jamoTBase) + default: + b[k] = b[i] + k++ + } + } + } + rb.nrune = k +} + +// compose recombines the runes in the buffer. +// It should only be used to recompose a single segment, as it will not +// handle alternations between Hangul and non-Hangul characters correctly. +func (rb *reorderBuffer) compose() { + // UAX #15, section X5 , including Corrigendum #5 + // "In any character sequence beginning with starter S, a character C is + // blocked from S if and only if there is some character B between S + // and C, and either B is a starter or it has the same or higher + // combining class as C." + k := 1 + b := rb.rune[:] + bn := rb.nrune + for s, i := 0, 1; i < bn; i++ { + if isJamoVT(rb.bytesAt(i)) { + // Redo from start in Hangul mode. Necessary to support + // U+320E..U+321E in NFKC mode. + rb.combineHangul() + return + } + ii := b[i] + // We can only use combineForward as a filter if we later + // get the info for the combined character. This is more + // expensive than using the filter. Using combinesBackward() + // is safe. + if ii.flags.combinesBackward() { + cccB := b[k-1].ccc + cccC := ii.ccc + blocked := false // b[i] blocked by starter or greater or equal CCC? + if cccB == 0 { + s = k - 1 + } else { + blocked = s != k-1 && cccB >= cccC + } + if !blocked { + combined := combine(rb.runeAt(s), rb.runeAt(i)) + if combined != 0 { + rb.assignRune(s, combined) + continue + } + } + } + b[k] = b[i] + k++ + } + rb.nrune = k +} diff --git a/src/pkg/exp/norm/composition_test.go b/src/pkg/exp/norm/composition_test.go new file mode 100644 index 000000000..195a0c1e8 --- /dev/null +++ b/src/pkg/exp/norm/composition_test.go @@ -0,0 +1,138 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package norm + +import "testing" + +// TestCase is used for most tests. +type TestCase struct { + in []int + out []int +} + +type insertFunc func(rb *reorderBuffer, rune int) bool + +func insert(rb *reorderBuffer, rune int) bool { + b := []byte(string(rune)) + return rb.insert(b, rb.f.info(b)) +} + +func insertString(rb *reorderBuffer, rune int) bool { + s := string(rune) + return rb.insertString(s, rb.f.infoString(s)) +} + +func runTests(t *testing.T, name string, rb *reorderBuffer, f insertFunc, tests []TestCase) { + for i, test := range tests { + rb.reset() + for j, rune := range test.in { + b := []byte(string(rune)) + if !rb.insert(b, rb.f.info(b)) { + t.Errorf("%s:%d: insert failed for rune %d", name, i, j) + } + } + if rb.f.composing { + rb.compose() + } + if rb.nrune != len(test.out) { + t.Errorf("%s:%d: length = %d; want %d", name, i, rb.nrune, len(test.out)) + continue + } + for j, want := range test.out { + found := int(rb.runeAt(j)) + if found != want { + t.Errorf("%s:%d: runeAt(%d) = %U; want %U", name, i, j, found, want) + } + } + } +} + +func TestFlush(t *testing.T) { + rb := &reorderBuffer{f: *formTable[NFC]} + out := make([]byte, 0) + + out = rb.flush(out) + if len(out) != 0 { + t.Errorf("wrote bytes on flush of empty buffer. (len(out) = %d)", len(out)) + } + + for _, r := range []int("world!") { + insert(rb, r) + } + + out = []byte("Hello ") + out = rb.flush(out) + want := "Hello world!" + if string(out) != want { + t.Errorf(`output after flush was "%s"; want "%s"`, string(out), want) + } + if rb.nrune != 0 { + t.Errorf("flush: non-null size of info buffer (rb.nrune == %d)", rb.nrune) + } + if rb.nbyte != 0 { + t.Errorf("flush: non-null size of byte buffer (rb.nbyte == %d)", rb.nbyte) + } +} + +var insertTests = []TestCase{ + {[]int{'a'}, []int{'a'}}, + {[]int{0x300}, []int{0x300}}, + {[]int{0x300, 0x316}, []int{0x316, 0x300}}, // CCC(0x300)==230; CCC(0x316)==220 + {[]int{0x316, 0x300}, []int{0x316, 0x300}}, + {[]int{0x41, 0x316, 0x300}, []int{0x41, 0x316, 0x300}}, + {[]int{0x41, 0x300, 0x316}, []int{0x41, 0x316, 0x300}}, + {[]int{0x300, 0x316, 0x41}, []int{0x316, 0x300, 0x41}}, + {[]int{0x41, 0x300, 0x40, 0x316}, []int{0x41, 0x300, 0x40, 0x316}}, +} + +func TestInsert(t *testing.T) { + rb := &reorderBuffer{f: *formTable[NFD]} + runTests(t, "TestInsert", rb, insert, insertTests) +} + +func TestInsertString(t *testing.T) { + rb := &reorderBuffer{f: *formTable[NFD]} + runTests(t, "TestInsertString", rb, insertString, insertTests) +} + +var decompositionNFDTest = []TestCase{ + {[]int{0xC0}, []int{0x41, 0x300}}, + {[]int{0xAC00}, []int{0x1100, 0x1161}}, + {[]int{0x01C4}, []int{0x01C4}}, + {[]int{0x320E}, []int{0x320E}}, + {[]int("음ẻ과"), []int{0x110B, 0x1173, 0x11B7, 0x65, 0x309, 0x1100, 0x116A}}, +} + +var decompositionNFKDTest = []TestCase{ + {[]int{0xC0}, []int{0x41, 0x300}}, + {[]int{0xAC00}, []int{0x1100, 0x1161}}, + {[]int{0x01C4}, []int{0x44, 0x5A, 0x030C}}, + {[]int{0x320E}, []int{0x28, 0x1100, 0x1161, 0x29}}, +} + +func TestDecomposition(t *testing.T) { + rb := &reorderBuffer{} + rb.f = *formTable[NFD] + runTests(t, "TestDecompositionNFD", rb, insert, decompositionNFDTest) + rb.f = *formTable[NFKD] + runTests(t, "TestDecompositionNFKD", rb, insert, decompositionNFKDTest) +} + +var compositionTest = []TestCase{ + {[]int{0x41, 0x300}, []int{0xC0}}, + {[]int{0x41, 0x316}, []int{0x41, 0x316}}, + {[]int{0x41, 0x300, 0x35D}, []int{0xC0, 0x35D}}, + {[]int{0x41, 0x316, 0x300}, []int{0xC0, 0x316}}, + // blocking starter + {[]int{0x41, 0x316, 0x40, 0x300}, []int{0x41, 0x316, 0x40, 0x300}}, + {[]int{0x1100, 0x1161}, []int{0xAC00}}, + // parenthesized Hangul, alternate between ASCII and Hangul. + {[]int{0x28, 0x1100, 0x1161, 0x29}, []int{0x28, 0xAC00, 0x29}}, +} + +func TestComposition(t *testing.T) { + rb := &reorderBuffer{f: *formTable[NFC]} + runTests(t, "TestComposition", rb, insert, compositionTest) +} diff --git a/src/pkg/exp/norm/forminfo.go b/src/pkg/exp/norm/forminfo.go new file mode 100644 index 000000000..ee3edb8ea --- /dev/null +++ b/src/pkg/exp/norm/forminfo.go @@ -0,0 +1,188 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package norm + +// This file contains Form-specific logic and wrappers for data in tables.go. + +type runeInfo struct { + pos uint8 // start position in reorderBuffer; used in composition.go + size uint8 // length of UTF-8 encoding of this rune + ccc uint8 // canonical combining class + flags qcInfo // quick check flags +} + +// functions dispatchable per form +type boundaryFunc func(f *formInfo, info runeInfo) bool +type lookupFunc func(b []byte) runeInfo +type lookupFuncString func(s string) runeInfo +type decompFunc func(b []byte) []byte +type decompFuncString func(s string) []byte + +// formInfo holds Form-specific functions and tables. +type formInfo struct { + form Form + + composing, compatibility bool // form type + + decompose decompFunc + decomposeString decompFuncString + info lookupFunc + infoString lookupFuncString + boundaryBefore boundaryFunc + boundaryAfter boundaryFunc +} + +var formTable []*formInfo + +func init() { + formTable = make([]*formInfo, 4) + + for i := range formTable { + f := &formInfo{} + formTable[i] = f + f.form = Form(i) + if Form(i) == NFKD || Form(i) == NFKC { + f.compatibility = true + f.decompose = decomposeNFKC + f.decomposeString = decomposeStringNFKC + f.info = lookupInfoNFKC + f.infoString = lookupInfoStringNFKC + } else { + f.decompose = decomposeNFC + f.decomposeString = decomposeStringNFC + f.info = lookupInfoNFC + f.infoString = lookupInfoStringNFC + } + if Form(i) == NFC || Form(i) == NFKC { + f.composing = true + f.boundaryBefore = compBoundaryBefore + f.boundaryAfter = compBoundaryAfter + } else { + f.boundaryBefore = decompBoundary + f.boundaryAfter = decompBoundary + } + } +} + +func decompBoundary(f *formInfo, info runeInfo) bool { + if info.ccc == 0 && info.flags.isYesD() { // Implies isHangul(b) == true + return true + } + // We assume that the CCC of the first character in a decomposition + // is always non-zero if different from info.ccc and that we can return + // false at this point. This is verified by maketables. + return false +} + +func compBoundaryBefore(f *formInfo, info runeInfo) bool { + if info.ccc == 0 && info.flags.isYesC() { + return true + } + // We assume that the CCC of the first character in a decomposition + // is always non-zero if different from info.ccc and that we can return + // false at this point. This is verified by maketables. + return false +} + +func compBoundaryAfter(f *formInfo, info runeInfo) bool { + // This misses values where the last char in a decomposition is a + // boundary such as Hangul with JamoT. + // TODO(mpvl): verify this does not lead to segments that do + // not fit in the reorderBuffer. + return info.flags.isInert() +} + +// We pack quick check data in 4 bits: +// 0: NFD_QC Yes (0) or No (1). No also means there is a decomposition. +// 1..2: NFC_QC Yes(00), No (01), or Maybe (11) +// 3: Combines forward (0 == false, 1 == true) +// +// When all 4 bits are zero, the character is inert, meaning it is never +// influenced by normalization. +// +// We pack the bits for both NFC/D and NFKC/D in one byte. +type qcInfo uint8 + +func (i qcInfo) isYesC() bool { return i&0x2 == 0 } +func (i qcInfo) isNoC() bool { return i&0x6 == 0x2 } +func (i qcInfo) isMaybe() bool { return i&0x4 != 0 } +func (i qcInfo) isYesD() bool { return i&0x1 == 0 } +func (i qcInfo) isNoD() bool { return i&0x1 != 0 } +func (i qcInfo) isInert() bool { return i&0xf == 0 } + +func (i qcInfo) combinesForward() bool { return i&0x8 != 0 } +func (i qcInfo) combinesBackward() bool { return i&0x4 != 0 } // == isMaybe +func (i qcInfo) hasDecomposition() bool { return i&0x1 != 0 } // == isNoD + +// Wrappers for tables.go + +// The 16-bit value of the decompostion tries is an index into a byte +// array of UTF-8 decomposition sequences. The first byte is the number +// of bytes in the decomposition (excluding this length byte). The actual +// sequence starts at the offset+1. +func decomposeNFC(b []byte) []byte { + p := nfcDecompTrie.lookupUnsafe(b) + n := decomps[p] + p++ + return decomps[p : p+uint16(n)] +} + +func decomposeNFKC(b []byte) []byte { + p := nfkcDecompTrie.lookupUnsafe(b) + n := decomps[p] + p++ + return decomps[p : p+uint16(n)] +} + +func decomposeStringNFC(s string) []byte { + p := nfcDecompTrie.lookupStringUnsafe(s) + n := decomps[p] + p++ + return decomps[p : p+uint16(n)] +} + +func decomposeStringNFKC(s string) []byte { + p := nfkcDecompTrie.lookupStringUnsafe(s) + n := decomps[p] + p++ + return decomps[p : p+uint16(n)] +} + +// Recomposition +// We use 32-bit keys instead of 64-bit for the two codepoint keys. +// This clips off the bits of three entries, but we know this will not +// result in a collision. In the unlikely event that changes to +// UnicodeData.txt introduce collisions, the compiler will catch it. +// Note that the recomposition map for NFC and NFKC are identical. + +// combine returns the combined rune or 0 if it doesn't exist. +func combine(a, b uint32) uint32 { + key := uint32(uint16(a))<<16 + uint32(uint16(b)) + return recompMap[key] +} + +// The 16-bit character info has the following bit layout: +// 0..7 CCC value. +// 8..11 qcInfo for NFC/NFD +// 12..15 qcInfo for NFKC/NFKD +func lookupInfoNFC(b []byte) runeInfo { + v, sz := charInfoTrie.lookup(b) + return runeInfo{0, uint8(sz), uint8(v), qcInfo(v >> 8)} +} + +func lookupInfoStringNFC(s string) runeInfo { + v, sz := charInfoTrie.lookupString(s) + return runeInfo{0, uint8(sz), uint8(v), qcInfo(v >> 8)} +} + +func lookupInfoNFKC(b []byte) runeInfo { + v, sz := charInfoTrie.lookup(b) + return runeInfo{0, uint8(sz), uint8(v), qcInfo(v >> 12)} +} + +func lookupInfoStringNFKC(s string) runeInfo { + v, sz := charInfoTrie.lookupString(s) + return runeInfo{0, uint8(sz), uint8(v), qcInfo(v >> 12)} +} diff --git a/src/pkg/exp/norm/maketables.go b/src/pkg/exp/norm/maketables.go new file mode 100644 index 000000000..e3e5700a6 --- /dev/null +++ b/src/pkg/exp/norm/maketables.go @@ -0,0 +1,855 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Normalization table generator. +// Data read from the web. + +package main + +import ( + "bufio" + "bytes" + "flag" + "fmt" + "http" + "io" + "log" + "os" + "regexp" + "strconv" + "strings" +) + +func main() { + flag.Parse() + loadUnicodeData() + loadCompositionExclusions() + completeCharFields(FCanonical) + completeCharFields(FCompatibility) + verifyComputed() + printChars() + makeTables() + testDerived() +} + +var url = flag.String("url", + "http://www.unicode.org/Public/6.0.0/ucd/", + "URL of Unicode database directory") +var tablelist = flag.String("tables", + "all", + "comma-separated list of which tables to generate; "+ + "can be 'decomp', 'recomp', 'info' and 'all'") +var test = flag.Bool("test", + false, + "test existing tables; can be used to compare web data with package data") +var verbose = flag.Bool("verbose", + false, + "write data to stdout as it is parsed") +var localFiles = flag.Bool("local", + false, + "data files have been copied to the current directory; for debugging only") + +var logger = log.New(os.Stderr, "", log.Lshortfile) + +// UnicodeData.txt has form: +// 0037;DIGIT SEVEN;Nd;0;EN;;7;7;7;N;;;;; +// 007A;LATIN SMALL LETTER Z;Ll;0;L;;;;;N;;;005A;;005A +// See http://unicode.org/reports/tr44/ for full explanation +// The fields: +const ( + FCodePoint = iota + FName + FGeneralCategory + FCanonicalCombiningClass + FBidiClass + FDecompMapping + FDecimalValue + FDigitValue + FNumericValue + FBidiMirrored + FUnicode1Name + FISOComment + FSimpleUppercaseMapping + FSimpleLowercaseMapping + FSimpleTitlecaseMapping + NumField + + MaxChar = 0x10FFFF // anything above this shouldn't exist +) + +// Quick Check properties of runes allow us to quickly +// determine whether a rune may occur in a normal form. +// For a given normal form, a rune may be guaranteed to occur +// verbatim (QC=Yes), may or may not combine with another +// rune (QC=Maybe), or may not occur (QC=No). +type QCResult int + +const ( + QCUnknown QCResult = iota + QCYes + QCNo + QCMaybe +) + +func (r QCResult) String() string { + switch r { + case QCYes: + return "Yes" + case QCNo: + return "No" + case QCMaybe: + return "Maybe" + } + return "***UNKNOWN***" +} + +const ( + FCanonical = iota // NFC or NFD + FCompatibility // NFKC or NFKD + FNumberOfFormTypes +) + +const ( + MComposed = iota // NFC or NFKC + MDecomposed // NFD or NFKD + MNumberOfModes +) + +// This contains only the properties we're interested in. +type Char struct { + name string + codePoint int // if zero, this index is not a valid code point. + ccc uint8 // canonical combining class + excludeInComp bool // from CompositionExclusions.txt + compatDecomp bool // it has a compatibility expansion + + forms [FNumberOfFormTypes]FormInfo // For FCanonical and FCompatibility + + state State +} + +var chars = make([]Char, MaxChar+1) + +func (c Char) String() string { + buf := new(bytes.Buffer) + + fmt.Fprintf(buf, "%U [%s]:\n", c.codePoint, c.name) + fmt.Fprintf(buf, " ccc: %v\n", c.ccc) + fmt.Fprintf(buf, " excludeInComp: %v\n", c.excludeInComp) + fmt.Fprintf(buf, " compatDecomp: %v\n", c.compatDecomp) + fmt.Fprintf(buf, " state: %v\n", c.state) + fmt.Fprintf(buf, " NFC:\n") + fmt.Fprint(buf, c.forms[FCanonical]) + fmt.Fprintf(buf, " NFKC:\n") + fmt.Fprint(buf, c.forms[FCompatibility]) + + return buf.String() +} + +// In UnicodeData.txt, some ranges are marked like this: +// 3400;;Lo;0;L;;;;;N;;;;; +// 4DB5;;Lo;0;L;;;;;N;;;;; +// parseCharacter keeps a state variable indicating the weirdness. +type State int + +const ( + SNormal State = iota // known to be zero for the type + SFirst + SLast + SMissing +) + +var lastChar int = 0 + +func (c Char) isValid() bool { + return c.codePoint != 0 && c.state != SMissing +} + +type FormInfo struct { + quickCheck [MNumberOfModes]QCResult // index: MComposed or MDecomposed + verified [MNumberOfModes]bool // index: MComposed or MDecomposed + + combinesForward bool // May combine with rune on the right + combinesBackward bool // May combine with rune on the left + isOneWay bool // Never appears in result + inDecomp bool // Some decompositions result in this char. + decomp Decomposition + expandedDecomp Decomposition +} + +func (f FormInfo) String() string { + buf := bytes.NewBuffer(make([]byte, 0)) + + fmt.Fprintf(buf, " quickCheck[C]: %v\n", f.quickCheck[MComposed]) + fmt.Fprintf(buf, " quickCheck[D]: %v\n", f.quickCheck[MDecomposed]) + fmt.Fprintf(buf, " cmbForward: %v\n", f.combinesForward) + fmt.Fprintf(buf, " cmbBackward: %v\n", f.combinesBackward) + fmt.Fprintf(buf, " isOneWay: %v\n", f.isOneWay) + fmt.Fprintf(buf, " inDecomp: %v\n", f.inDecomp) + fmt.Fprintf(buf, " decomposition: %v\n", f.decomp) + fmt.Fprintf(buf, " expandedDecomp: %v\n", f.expandedDecomp) + + return buf.String() +} + +type Decomposition []int + +func (d Decomposition) String() string { + return fmt.Sprintf("%.4X", d) +} + +func openReader(file string) (input io.ReadCloser) { + if *localFiles { + f, err := os.Open(file) + if err != nil { + logger.Fatal(err) + } + input = f + } else { + path := *url + file + resp, err := http.Get(path) + if err != nil { + logger.Fatal(err) + } + if resp.StatusCode != 200 { + logger.Fatal("bad GET status for "+file, resp.Status) + } + input = resp.Body + } + return +} + +func parseDecomposition(s string, skipfirst bool) (a []int, e os.Error) { + decomp := strings.Split(s, " ") + if len(decomp) > 0 && skipfirst { + decomp = decomp[1:] + } + for _, d := range decomp { + point, err := strconv.Btoui64(d, 16) + if err != nil { + return a, err + } + a = append(a, int(point)) + } + return a, nil +} + +func parseCharacter(line string) { + field := strings.Split(line, ";") + if len(field) != NumField { + logger.Fatalf("%5s: %d fields (expected %d)\n", line, len(field), NumField) + } + x, err := strconv.Btoui64(field[FCodePoint], 16) + point := int(x) + if err != nil { + logger.Fatalf("%.5s...: %s", line, err) + } + if point == 0 { + return // not interesting and we use 0 as unset + } + if point > MaxChar { + logger.Fatalf("%5s: Rune %X > MaxChar (%X)", line, point, MaxChar) + return + } + state := SNormal + switch { + case strings.Index(field[FName], ", First>") > 0: + state = SFirst + case strings.Index(field[FName], ", Last>") > 0: + state = SLast + } + firstChar := lastChar + 1 + lastChar = int(point) + if state != SLast { + firstChar = lastChar + } + x, err = strconv.Atoui64(field[FCanonicalCombiningClass]) + if err != nil { + logger.Fatalf("%U: bad ccc field: %s", int(x), err) + } + ccc := uint8(x) + decmap := field[FDecompMapping] + exp, e := parseDecomposition(decmap, false) + isCompat := false + if e != nil { + if len(decmap) > 0 { + exp, e = parseDecomposition(decmap, true) + if e != nil { + logger.Fatalf(`%U: bad decomp |%v|: "%s"`, int(x), decmap, e) + } + isCompat = true + } + } + for i := firstChar; i <= lastChar; i++ { + char := &chars[i] + char.name = field[FName] + char.codePoint = i + char.forms[FCompatibility].decomp = exp + if !isCompat { + char.forms[FCanonical].decomp = exp + } else { + char.compatDecomp = true + } + if len(decmap) > 0 { + char.forms[FCompatibility].decomp = exp + } + char.ccc = ccc + char.state = SMissing + if i == lastChar { + char.state = state + } + } + return +} + +func loadUnicodeData() { + f := openReader("UnicodeData.txt") + defer f.Close() + input := bufio.NewReader(f) + for { + line, err := input.ReadString('\n') + if err != nil { + if err == os.EOF { + break + } + logger.Fatal(err) + } + parseCharacter(line[0 : len(line)-1]) + } +} + +var singlePointRe = regexp.MustCompile(`^([0-9A-F]+) *$`) + +// CompositionExclusions.txt has form: +// 0958 # ... +// See http://unicode.org/reports/tr44/ for full explanation +func parseExclusion(line string) int { + comment := strings.Index(line, "#") + if comment >= 0 { + line = line[0:comment] + } + if len(line) == 0 { + return 0 + } + matches := singlePointRe.FindStringSubmatch(line) + if len(matches) != 2 { + logger.Fatalf("%s: %d matches (expected 1)\n", line, len(matches)) + } + point, err := strconv.Btoui64(matches[1], 16) + if err != nil { + logger.Fatalf("%.5s...: %s", line, err) + } + return int(point) +} + +func loadCompositionExclusions() { + f := openReader("CompositionExclusions.txt") + defer f.Close() + input := bufio.NewReader(f) + for { + line, err := input.ReadString('\n') + if err != nil { + if err == os.EOF { + break + } + logger.Fatal(err) + } + point := parseExclusion(line[0 : len(line)-1]) + if point == 0 { + continue + } + c := &chars[point] + if c.excludeInComp { + logger.Fatalf("%U: Duplicate entry in exclusions.", c.codePoint) + } + c.excludeInComp = true + } +} + +// hasCompatDecomp returns true if any of the recursive +// decompositions contains a compatibility expansion. +// In this case, the character may not occur in NFK*. +func hasCompatDecomp(rune int) bool { + c := &chars[rune] + if c.compatDecomp { + return true + } + for _, d := range c.forms[FCompatibility].decomp { + if hasCompatDecomp(d) { + return true + } + } + return false +} + +// Hangul related constants. +const ( + HangulBase = 0xAC00 + HangulEnd = 0xD7A4 // hangulBase + Jamo combinations (19 * 21 * 28) + + JamoLBase = 0x1100 + JamoLEnd = 0x1113 + JamoVBase = 0x1161 + JamoVEnd = 0x1176 + JamoTBase = 0x11A8 + JamoTEnd = 0x11C3 +) + +func isHangul(rune int) bool { + return HangulBase <= rune && rune < HangulEnd +} + +func ccc(rune int) uint8 { + return chars[rune].ccc +} + +// Insert a rune in a buffer, ordered by Canonical Combining Class. +func insertOrdered(b Decomposition, rune int) Decomposition { + n := len(b) + b = append(b, 0) + cc := ccc(rune) + if cc > 0 { + // Use bubble sort. + for ; n > 0; n-- { + if ccc(b[n-1]) <= cc { + break + } + b[n] = b[n-1] + } + } + b[n] = rune + return b +} + +// Recursively decompose. +func decomposeRecursive(form int, rune int, d Decomposition) Decomposition { + if isHangul(rune) { + return d + } + dcomp := chars[rune].forms[form].decomp + if len(dcomp) == 0 { + return insertOrdered(d, rune) + } + for _, c := range dcomp { + d = decomposeRecursive(form, c, d) + } + return d +} + +func completeCharFields(form int) { + // Phase 0: pre-expand decomposition. + for i := range chars { + f := &chars[i].forms[form] + if len(f.decomp) == 0 { + continue + } + exp := make(Decomposition, 0) + for _, c := range f.decomp { + exp = decomposeRecursive(form, c, exp) + } + f.expandedDecomp = exp + } + + // Phase 1: composition exclusion, mark decomposition. + for i := range chars { + c := &chars[i] + f := &c.forms[form] + + // Marks script-specific exclusions and version restricted. + f.isOneWay = c.excludeInComp + + // Singletons + f.isOneWay = f.isOneWay || len(f.decomp) == 1 + + // Non-starter decompositions + if len(f.decomp) > 1 { + chk := c.ccc != 0 || chars[f.decomp[0]].ccc != 0 + f.isOneWay = f.isOneWay || chk + } + + // Runes that decompose into more than two runes. + f.isOneWay = f.isOneWay || len(f.decomp) > 2 + + if form == FCompatibility { + f.isOneWay = f.isOneWay || hasCompatDecomp(c.codePoint) + } + + for _, rune := range f.decomp { + chars[rune].forms[form].inDecomp = true + } + } + + // Phase 2: forward and backward combining. + for i := range chars { + c := &chars[i] + f := &c.forms[form] + + if !f.isOneWay && len(f.decomp) == 2 { + f0 := &chars[f.decomp[0]].forms[form] + f1 := &chars[f.decomp[1]].forms[form] + if !f0.isOneWay { + f0.combinesForward = true + } + if !f1.isOneWay { + f1.combinesBackward = true + } + } + } + + // Phase 3: quick check values. + for i := range chars { + c := &chars[i] + f := &c.forms[form] + + switch { + case len(f.decomp) > 0: + f.quickCheck[MDecomposed] = QCNo + case isHangul(i): + f.quickCheck[MDecomposed] = QCNo + default: + f.quickCheck[MDecomposed] = QCYes + } + switch { + case f.isOneWay: + f.quickCheck[MComposed] = QCNo + case (i & 0xffff00) == JamoLBase: + f.quickCheck[MComposed] = QCYes + if JamoVBase <= i && i < JamoVEnd { + f.quickCheck[MComposed] = QCMaybe + f.combinesBackward = true + } + if JamoTBase <= i && i < JamoTEnd { + f.quickCheck[MComposed] = QCMaybe + f.combinesBackward = true + } + case !f.combinesBackward: + f.quickCheck[MComposed] = QCYes + default: + f.quickCheck[MComposed] = QCMaybe + } + } +} + +func printBytes(b []byte, name string) { + fmt.Printf("// %s: %d bytes\n", name, len(b)) + fmt.Printf("var %s = [...]byte {", name) + for i, c := range b { + switch { + case i%64 == 0: + fmt.Printf("\n// Bytes %x - %x\n", i, i+63) + case i%8 == 0: + fmt.Printf("\n") + } + fmt.Printf("0x%.2X, ", c) + } + fmt.Print("\n}\n\n") +} + +// See forminfo.go for format. +func makeEntry(f *FormInfo) uint16 { + e := uint16(0) + if f.combinesForward { + e |= 0x8 + } + if f.quickCheck[MDecomposed] == QCNo { + e |= 0x1 + } + switch f.quickCheck[MComposed] { + case QCYes: + case QCNo: + e |= 0x2 + case QCMaybe: + e |= 0x6 + default: + log.Fatalf("Illegal quickcheck value %d.", f.quickCheck[MComposed]) + } + return e +} + +// Bits +// 0..8: CCC +// 9..12: NF(C|D) qc bits. +// 13..16: NFK(C|D) qc bits. +func makeCharInfo(c Char) uint16 { + e := makeEntry(&c.forms[FCompatibility]) + e = e<<4 | makeEntry(&c.forms[FCanonical]) + e = e<<8 | uint16(c.ccc) + return e +} + +func printCharInfoTables() int { + // Quick Check + CCC trie. + t := newNode() + for i, char := range chars { + v := makeCharInfo(char) + if v != 0 { + t.insert(i, v) + } + } + return t.printTables("charInfo") +} + +func printDecompositionTables() int { + decompositions := bytes.NewBuffer(make([]byte, 0, 10000)) + size := 0 + + // Map decompositions + positionMap := make(map[string]uint16) + + // Store the uniqued decompositions in a byte buffer, + // preceded by their byte length. + for _, c := range chars { + for f := 0; f < 2; f++ { + d := c.forms[f].expandedDecomp + s := string([]int(d)) + if _, ok := positionMap[s]; !ok { + p := decompositions.Len() + decompositions.WriteByte(uint8(len(s))) + decompositions.WriteString(s) + positionMap[s] = uint16(p) + } + } + } + b := decompositions.Bytes() + printBytes(b, "decomps") + size += len(b) + + nfcT := newNode() + nfkcT := newNode() + for i, c := range chars { + d := c.forms[FCanonical].expandedDecomp + if len(d) != 0 { + nfcT.insert(i, positionMap[string([]int(d))]) + if ccc(c.codePoint) != ccc(d[0]) { + // We assume the lead ccc of a decomposition is !=0 in this case. + if ccc(d[0]) == 0 { + logger.Fatal("Expected differing CCC to be non-zero.") + } + } + } + d = c.forms[FCompatibility].expandedDecomp + if len(d) != 0 { + nfkcT.insert(i, positionMap[string([]int(d))]) + if ccc(c.codePoint) != ccc(d[0]) { + // We assume the lead ccc of a decomposition is !=0 in this case. + if ccc(d[0]) == 0 { + logger.Fatal("Expected differing CCC to be non-zero.") + } + } + } + } + size += nfcT.printTables("nfcDecomp") + size += nfkcT.printTables("nfkcDecomp") + return size +} + +func contains(sa []string, s string) bool { + for _, a := range sa { + if a == s { + return true + } + } + return false +} + +// Extract the version number from the URL. +func version() string { + // From http://www.unicode.org/standard/versions/#Version_Numbering: + // for the later Unicode versions, data files are located in + // versioned directories. + fields := strings.Split(*url, "/") + for _, f := range fields { + if match, _ := regexp.MatchString(`[0-9]\.[0-9]\.[0-9]`, f); match { + return f + } + } + logger.Fatal("unknown version") + return "Unknown" +} + +const fileHeader = `// Generated by running +// maketables --tables=%s --url=%s +// DO NOT EDIT + +package norm + +` + +func makeTables() { + size := 0 + if *tablelist == "" { + return + } + list := strings.Split(*tablelist, ",") + if *tablelist == "all" { + list = []string{"decomp", "recomp", "info"} + } + fmt.Printf(fileHeader, *tablelist, *url) + + fmt.Println("// Version is the Unicode edition from which the tables are derived.") + fmt.Printf("const Version = %q\n\n", version()) + + if contains(list, "decomp") { + size += printDecompositionTables() + } + + if contains(list, "recomp") { + // Note that we use 32 bit keys, instead of 64 bit. + // This clips the bits of three entries, but we know + // this won't cause a collision. The compiler will catch + // any changes made to UnicodeData.txt that introduces + // a collision. + // Note that the recomposition map for NFC and NFKC + // are identical. + + // Recomposition map + nrentries := 0 + for _, c := range chars { + f := c.forms[FCanonical] + if !f.isOneWay && len(f.decomp) > 0 { + nrentries++ + } + } + sz := nrentries * 8 + size += sz + fmt.Printf("// recompMap: %d bytes (entries only)\n", sz) + fmt.Println("var recompMap = map[uint32]uint32{") + for i, c := range chars { + f := c.forms[FCanonical] + d := f.decomp + if !f.isOneWay && len(d) > 0 { + key := uint32(uint16(d[0]))<<16 + uint32(uint16(d[1])) + fmt.Printf("0x%.8X: 0x%.4X,\n", key, i) + } + } + fmt.Printf("}\n\n") + } + + if contains(list, "info") { + size += printCharInfoTables() + } + fmt.Printf("// Total size of tables: %dKB (%d bytes)\n", (size+512)/1024, size) +} + +func printChars() { + if *verbose { + for _, c := range chars { + if !c.isValid() || c.state == SMissing { + continue + } + fmt.Println(c) + } + } +} + +// verifyComputed does various consistency tests. +func verifyComputed() { + for i, c := range chars { + for _, f := range c.forms { + isNo := (f.quickCheck[MDecomposed] == QCNo) + if (len(f.decomp) > 0) != isNo && !isHangul(i) { + log.Fatalf("%U: NF*D must be no if rune decomposes", i) + } + + isMaybe := f.quickCheck[MComposed] == QCMaybe + if f.combinesBackward != isMaybe { + log.Fatalf("%U: NF*C must be maybe if combinesBackward", i) + } + } + } +} + +var qcRe = regexp.MustCompile(`^([0-9A-F\.]+) *; (NF.*_QC); ([YNM]) #.*$`) + +// Use values in DerivedNormalizationProps.txt to compare against the +// values we computed. +// DerivedNormalizationProps.txt has form: +// 00C0..00C5 ; NFD_QC; N # ... +// 0374 ; NFD_QC; N # ... +// See http://unicode.org/reports/tr44/ for full explanation +func testDerived() { + if !*test { + return + } + f := openReader("DerivedNormalizationProps.txt") + defer f.Close() + input := bufio.NewReader(f) + for { + line, err := input.ReadString('\n') + if err != nil { + if err == os.EOF { + break + } + logger.Fatal(err) + } + qc := qcRe.FindStringSubmatch(line) + if qc == nil { + continue + } + rng := strings.Split(qc[1], "..") + i, err := strconv.Btoui64(rng[0], 16) + if err != nil { + log.Fatal(err) + } + j := i + if len(rng) > 1 { + j, err = strconv.Btoui64(rng[1], 16) + if err != nil { + log.Fatal(err) + } + } + var ftype, mode int + qt := strings.TrimSpace(qc[2]) + switch qt { + case "NFC_QC": + ftype, mode = FCanonical, MComposed + case "NFD_QC": + ftype, mode = FCanonical, MDecomposed + case "NFKC_QC": + ftype, mode = FCompatibility, MComposed + case "NFKD_QC": + ftype, mode = FCompatibility, MDecomposed + default: + log.Fatalf(`Unexpected quick check type "%s"`, qt) + } + var qr QCResult + switch qc[3] { + case "Y": + qr = QCYes + case "N": + qr = QCNo + case "M": + qr = QCMaybe + default: + log.Fatalf(`Unexpected quick check value "%s"`, qc[3]) + } + var lastFailed bool + // Verify current + for ; i <= j; i++ { + c := &chars[int(i)] + c.forms[ftype].verified[mode] = true + curqr := c.forms[ftype].quickCheck[mode] + if curqr != qr { + if !lastFailed { + logger.Printf("%s: %.4X..%.4X -- %s\n", + qt, int(i), int(j), line[0:50]) + } + logger.Printf("%U: FAILED %s (was %v need %v)\n", + int(i), qt, curqr, qr) + lastFailed = true + } + } + } + // Any unspecified value must be QCYes. Verify this. + for i, c := range chars { + for j, fd := range c.forms { + for k, qr := range fd.quickCheck { + if !fd.verified[k] && qr != QCYes { + m := "%U: FAIL F:%d M:%d (was %v need Yes) %s\n" + logger.Printf(m, i, j, k, qr, c.name) + } + } + } + } +} diff --git a/src/pkg/exp/norm/maketesttables.go b/src/pkg/exp/norm/maketesttables.go new file mode 100644 index 000000000..c5f6a6436 --- /dev/null +++ b/src/pkg/exp/norm/maketesttables.go @@ -0,0 +1,42 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Generate test data for trie code. + +package main + +import ( + "fmt" +) + +func main() { + printTestTables() +} + +// We take the smallest, largest and an arbitrary value for each +// of the UTF-8 sequence lengths. +var testRunes = []int{ + 0x01, 0x0C, 0x7F, // 1-byte sequences + 0x80, 0x100, 0x7FF, // 2-byte sequences + 0x800, 0x999, 0xFFFF, // 3-byte sequences + 0x10000, 0x10101, 0x10FFFF, // 4-byte sequences +} + +const fileHeader = `// Generated by running +// maketesttables +// DO NOT EDIT + +package norm + +` + +func printTestTables() { + fmt.Print(fileHeader) + fmt.Printf("var testRunes = %#v\n\n", testRunes) + t := newNode() + for i, r := range testRunes { + t.insert(r, uint16(i)) + } + t.printTables("testdata") +} diff --git a/src/pkg/exp/norm/norm_test.go b/src/pkg/exp/norm/norm_test.go new file mode 100644 index 000000000..12dacfcf3 --- /dev/null +++ b/src/pkg/exp/norm/norm_test.go @@ -0,0 +1,14 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package norm_test + +import ( + "testing" +) + +func TestPlaceHolder(t *testing.T) { + // Does nothing, just allows the Makefile to be canonical + // while waiting for the package itself to be written. +} diff --git a/src/pkg/exp/norm/normalize.go b/src/pkg/exp/norm/normalize.go new file mode 100644 index 000000000..e9d18dd9e --- /dev/null +++ b/src/pkg/exp/norm/normalize.go @@ -0,0 +1,99 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Package norm contains types and functions for normalizing Unicode strings. +package norm + +// A Form denotes a canonical representation of Unicode code points. +// The Unicode-defined normalization and equivalence forms are: +// +// NFC Unicode Normalization Form C +// NFD Unicode Normalization Form D +// NFKC Unicode Normalization Form KC +// NFKD Unicode Normalization Form KD +// +// For a Form f, this documentation uses the notation f(x) to mean +// the bytes or string x converted to the given form. +// A position n in x is called a boundary if conversion to the form can +// proceed independently on both sides: +// f(x) == append(f(x[0:n]), f(x[n:])...) +// +// References: http://unicode.org/reports/tr15/ and +// http://unicode.org/notes/tn5/. +type Form int + +const ( + NFC Form = iota + NFD + NFKC + NFKD +) + +// Bytes returns f(b). May return b if f(b) = b. +func (f Form) Bytes(b []byte) []byte { + panic("not implemented") +} + +// String returns f(s). +func (f Form) String(s string) string { + panic("not implemented") +} + +// IsNormal returns true if b == f(b). +func (f Form) IsNormal(b []byte) bool { + panic("not implemented") +} + +// IsNormalString returns true if s == f(s). +func (f Form) IsNormalString(s string) bool { + panic("not implemented") +} + +// Append returns f(append(out, b...)). +// The buffer out must be empty or equal to f(out). +func (f Form) Append(out, b []byte) []byte { + panic("not implemented") +} + +// AppendString returns f(append(out, []byte(s))). +// The buffer out must be empty or equal to f(out). +func (f Form) AppendString(out []byte, s string) []byte { + panic("not implemented") +} + +// QuickSpan returns a boundary n such that b[0:n] == f(b[0:n]). +// It is not guaranteed to return the largest such n. +func (f Form) QuickSpan(b []byte) int { + panic("not implemented") +} + +// QuickSpanString returns a boundary n such that b[0:n] == f(s[0:n]). +// It is not guaranteed to return the largest such n. +func (f Form) QuickSpanString(s string) int { + panic("not implemented") +} + +// FirstBoundary returns the position i of the first boundary in b. +// It returns len(b), false if b contains no boundaries. +func (f Form) FirstBoundary(b []byte) (i int, ok bool) { + panic("not implemented") +} + +// FirstBoundaryInString return the position i of the first boundary in s. +// It returns len(s), false if s contains no boundaries. +func (f Form) FirstBoundaryInString(s string) (i int, ok bool) { + panic("not implemented") +} + +// LastBoundaryIn returns the position i of the last boundary in b. +// It returns 0, false if b contains no boundary. +func (f Form) LastBoundary(b []byte) (i int, ok bool) { + panic("not implemented") +} + +// LastBoundaryInString returns the position i of the last boundary in s. +// It returns 0, false if s contains no boundary. +func (f Form) LastBoundaryInString(s string) (i int, ok bool) { + panic("not implemented") +} diff --git a/src/pkg/exp/norm/tables.go b/src/pkg/exp/norm/tables.go new file mode 100644 index 000000000..76995c2fa --- /dev/null +++ b/src/pkg/exp/norm/tables.go @@ -0,0 +1,6580 @@ +// Generated by running +// maketables --tables=all --url=http://www.unicode.org/Public/6.0.0/ucd/ +// DO NOT EDIT + +package norm + +// Version is the Unicode edition from which the tables are derived. +const Version = "6.0.0" + +// decomps: 17618 bytes +var decomps = [...]byte{ + // Bytes 0 - 3f + 0x00, 0x01, 0x20, 0x03, 0x20, 0xCC, 0x88, 0x01, + 0x61, 0x03, 0x20, 0xCC, 0x84, 0x01, 0x32, 0x01, + 0x33, 0x03, 0x20, 0xCC, 0x81, 0x02, 0xCE, 0xBC, + 0x03, 0x20, 0xCC, 0xA7, 0x01, 0x31, 0x01, 0x6F, + 0x05, 0x31, 0xE2, 0x81, 0x84, 0x34, 0x05, 0x31, + 0xE2, 0x81, 0x84, 0x32, 0x05, 0x33, 0xE2, 0x81, + 0x84, 0x34, 0x03, 0x41, 0xCC, 0x80, 0x03, 0x41, + 0xCC, 0x81, 0x03, 0x41, 0xCC, 0x82, 0x03, 0x41, + // Bytes 40 - 7f + 0xCC, 0x83, 0x03, 0x41, 0xCC, 0x88, 0x03, 0x41, + 0xCC, 0x8A, 0x03, 0x43, 0xCC, 0xA7, 0x03, 0x45, + 0xCC, 0x80, 0x03, 0x45, 0xCC, 0x81, 0x03, 0x45, + 0xCC, 0x82, 0x03, 0x45, 0xCC, 0x88, 0x03, 0x49, + 0xCC, 0x80, 0x03, 0x49, 0xCC, 0x81, 0x03, 0x49, + 0xCC, 0x82, 0x03, 0x49, 0xCC, 0x88, 0x03, 0x4E, + 0xCC, 0x83, 0x03, 0x4F, 0xCC, 0x80, 0x03, 0x4F, + 0xCC, 0x81, 0x03, 0x4F, 0xCC, 0x82, 0x03, 0x4F, + // Bytes 80 - bf + 0xCC, 0x83, 0x03, 0x4F, 0xCC, 0x88, 0x03, 0x55, + 0xCC, 0x80, 0x03, 0x55, 0xCC, 0x81, 0x03, 0x55, + 0xCC, 0x82, 0x03, 0x55, 0xCC, 0x88, 0x03, 0x59, + 0xCC, 0x81, 0x03, 0x61, 0xCC, 0x80, 0x03, 0x61, + 0xCC, 0x81, 0x03, 0x61, 0xCC, 0x82, 0x03, 0x61, + 0xCC, 0x83, 0x03, 0x61, 0xCC, 0x88, 0x03, 0x61, + 0xCC, 0x8A, 0x03, 0x63, 0xCC, 0xA7, 0x03, 0x65, + 0xCC, 0x80, 0x03, 0x65, 0xCC, 0x81, 0x03, 0x65, + // Bytes c0 - ff + 0xCC, 0x82, 0x03, 0x65, 0xCC, 0x88, 0x03, 0x69, + 0xCC, 0x80, 0x03, 0x69, 0xCC, 0x81, 0x03, 0x69, + 0xCC, 0x82, 0x03, 0x69, 0xCC, 0x88, 0x03, 0x6E, + 0xCC, 0x83, 0x03, 0x6F, 0xCC, 0x80, 0x03, 0x6F, + 0xCC, 0x81, 0x03, 0x6F, 0xCC, 0x82, 0x03, 0x6F, + 0xCC, 0x83, 0x03, 0x6F, 0xCC, 0x88, 0x03, 0x75, + 0xCC, 0x80, 0x03, 0x75, 0xCC, 0x81, 0x03, 0x75, + 0xCC, 0x82, 0x03, 0x75, 0xCC, 0x88, 0x03, 0x79, + // Bytes 100 - 13f + 0xCC, 0x81, 0x03, 0x79, 0xCC, 0x88, 0x03, 0x41, + 0xCC, 0x84, 0x03, 0x61, 0xCC, 0x84, 0x03, 0x41, + 0xCC, 0x86, 0x03, 0x61, 0xCC, 0x86, 0x03, 0x41, + 0xCC, 0xA8, 0x03, 0x61, 0xCC, 0xA8, 0x03, 0x43, + 0xCC, 0x81, 0x03, 0x63, 0xCC, 0x81, 0x03, 0x43, + 0xCC, 0x82, 0x03, 0x63, 0xCC, 0x82, 0x03, 0x43, + 0xCC, 0x87, 0x03, 0x63, 0xCC, 0x87, 0x03, 0x43, + 0xCC, 0x8C, 0x03, 0x63, 0xCC, 0x8C, 0x03, 0x44, + // Bytes 140 - 17f + 0xCC, 0x8C, 0x03, 0x64, 0xCC, 0x8C, 0x03, 0x45, + 0xCC, 0x84, 0x03, 0x65, 0xCC, 0x84, 0x03, 0x45, + 0xCC, 0x86, 0x03, 0x65, 0xCC, 0x86, 0x03, 0x45, + 0xCC, 0x87, 0x03, 0x65, 0xCC, 0x87, 0x03, 0x45, + 0xCC, 0xA8, 0x03, 0x65, 0xCC, 0xA8, 0x03, 0x45, + 0xCC, 0x8C, 0x03, 0x65, 0xCC, 0x8C, 0x03, 0x47, + 0xCC, 0x82, 0x03, 0x67, 0xCC, 0x82, 0x03, 0x47, + 0xCC, 0x86, 0x03, 0x67, 0xCC, 0x86, 0x03, 0x47, + // Bytes 180 - 1bf + 0xCC, 0x87, 0x03, 0x67, 0xCC, 0x87, 0x03, 0x47, + 0xCC, 0xA7, 0x03, 0x67, 0xCC, 0xA7, 0x03, 0x48, + 0xCC, 0x82, 0x03, 0x68, 0xCC, 0x82, 0x03, 0x49, + 0xCC, 0x83, 0x03, 0x69, 0xCC, 0x83, 0x03, 0x49, + 0xCC, 0x84, 0x03, 0x69, 0xCC, 0x84, 0x03, 0x49, + 0xCC, 0x86, 0x03, 0x69, 0xCC, 0x86, 0x03, 0x49, + 0xCC, 0xA8, 0x03, 0x69, 0xCC, 0xA8, 0x03, 0x49, + 0xCC, 0x87, 0x02, 0x49, 0x4A, 0x02, 0x69, 0x6A, + // Bytes 1c0 - 1ff + 0x03, 0x4A, 0xCC, 0x82, 0x03, 0x6A, 0xCC, 0x82, + 0x03, 0x4B, 0xCC, 0xA7, 0x03, 0x6B, 0xCC, 0xA7, + 0x03, 0x4C, 0xCC, 0x81, 0x03, 0x6C, 0xCC, 0x81, + 0x03, 0x4C, 0xCC, 0xA7, 0x03, 0x6C, 0xCC, 0xA7, + 0x03, 0x4C, 0xCC, 0x8C, 0x03, 0x6C, 0xCC, 0x8C, + 0x03, 0x4C, 0xC2, 0xB7, 0x03, 0x6C, 0xC2, 0xB7, + 0x03, 0x4E, 0xCC, 0x81, 0x03, 0x6E, 0xCC, 0x81, + 0x03, 0x4E, 0xCC, 0xA7, 0x03, 0x6E, 0xCC, 0xA7, + // Bytes 200 - 23f + 0x03, 0x4E, 0xCC, 0x8C, 0x03, 0x6E, 0xCC, 0x8C, + 0x03, 0xCA, 0xBC, 0x6E, 0x03, 0x4F, 0xCC, 0x84, + 0x03, 0x6F, 0xCC, 0x84, 0x03, 0x4F, 0xCC, 0x86, + 0x03, 0x6F, 0xCC, 0x86, 0x03, 0x4F, 0xCC, 0x8B, + 0x03, 0x6F, 0xCC, 0x8B, 0x03, 0x52, 0xCC, 0x81, + 0x03, 0x72, 0xCC, 0x81, 0x03, 0x52, 0xCC, 0xA7, + 0x03, 0x72, 0xCC, 0xA7, 0x03, 0x52, 0xCC, 0x8C, + 0x03, 0x72, 0xCC, 0x8C, 0x03, 0x53, 0xCC, 0x81, + // Bytes 240 - 27f + 0x03, 0x73, 0xCC, 0x81, 0x03, 0x53, 0xCC, 0x82, + 0x03, 0x73, 0xCC, 0x82, 0x03, 0x53, 0xCC, 0xA7, + 0x03, 0x73, 0xCC, 0xA7, 0x03, 0x53, 0xCC, 0x8C, + 0x03, 0x73, 0xCC, 0x8C, 0x03, 0x54, 0xCC, 0xA7, + 0x03, 0x74, 0xCC, 0xA7, 0x03, 0x54, 0xCC, 0x8C, + 0x03, 0x74, 0xCC, 0x8C, 0x03, 0x55, 0xCC, 0x83, + 0x03, 0x75, 0xCC, 0x83, 0x03, 0x55, 0xCC, 0x84, + 0x03, 0x75, 0xCC, 0x84, 0x03, 0x55, 0xCC, 0x86, + // Bytes 280 - 2bf + 0x03, 0x75, 0xCC, 0x86, 0x03, 0x55, 0xCC, 0x8A, + 0x03, 0x75, 0xCC, 0x8A, 0x03, 0x55, 0xCC, 0x8B, + 0x03, 0x75, 0xCC, 0x8B, 0x03, 0x55, 0xCC, 0xA8, + 0x03, 0x75, 0xCC, 0xA8, 0x03, 0x57, 0xCC, 0x82, + 0x03, 0x77, 0xCC, 0x82, 0x03, 0x59, 0xCC, 0x82, + 0x03, 0x79, 0xCC, 0x82, 0x03, 0x59, 0xCC, 0x88, + 0x03, 0x5A, 0xCC, 0x81, 0x03, 0x7A, 0xCC, 0x81, + 0x03, 0x5A, 0xCC, 0x87, 0x03, 0x7A, 0xCC, 0x87, + // Bytes 2c0 - 2ff + 0x03, 0x5A, 0xCC, 0x8C, 0x03, 0x7A, 0xCC, 0x8C, + 0x01, 0x73, 0x03, 0x4F, 0xCC, 0x9B, 0x03, 0x6F, + 0xCC, 0x9B, 0x03, 0x55, 0xCC, 0x9B, 0x03, 0x75, + 0xCC, 0x9B, 0x04, 0x44, 0x5A, 0xCC, 0x8C, 0x04, + 0x44, 0x7A, 0xCC, 0x8C, 0x04, 0x64, 0x7A, 0xCC, + 0x8C, 0x02, 0x4C, 0x4A, 0x02, 0x4C, 0x6A, 0x02, + 0x6C, 0x6A, 0x02, 0x4E, 0x4A, 0x02, 0x4E, 0x6A, + 0x02, 0x6E, 0x6A, 0x03, 0x41, 0xCC, 0x8C, 0x03, + // Bytes 300 - 33f + 0x61, 0xCC, 0x8C, 0x03, 0x49, 0xCC, 0x8C, 0x03, + 0x69, 0xCC, 0x8C, 0x03, 0x4F, 0xCC, 0x8C, 0x03, + 0x6F, 0xCC, 0x8C, 0x03, 0x55, 0xCC, 0x8C, 0x03, + 0x75, 0xCC, 0x8C, 0x05, 0x55, 0xCC, 0x88, 0xCC, + 0x84, 0x05, 0x75, 0xCC, 0x88, 0xCC, 0x84, 0x05, + 0x55, 0xCC, 0x88, 0xCC, 0x81, 0x05, 0x75, 0xCC, + 0x88, 0xCC, 0x81, 0x05, 0x55, 0xCC, 0x88, 0xCC, + 0x8C, 0x05, 0x75, 0xCC, 0x88, 0xCC, 0x8C, 0x05, + // Bytes 340 - 37f + 0x55, 0xCC, 0x88, 0xCC, 0x80, 0x05, 0x75, 0xCC, + 0x88, 0xCC, 0x80, 0x05, 0x41, 0xCC, 0x88, 0xCC, + 0x84, 0x05, 0x61, 0xCC, 0x88, 0xCC, 0x84, 0x05, + 0x41, 0xCC, 0x87, 0xCC, 0x84, 0x05, 0x61, 0xCC, + 0x87, 0xCC, 0x84, 0x04, 0xC3, 0x86, 0xCC, 0x84, + 0x04, 0xC3, 0xA6, 0xCC, 0x84, 0x03, 0x47, 0xCC, + 0x8C, 0x03, 0x67, 0xCC, 0x8C, 0x03, 0x4B, 0xCC, + 0x8C, 0x03, 0x6B, 0xCC, 0x8C, 0x03, 0x4F, 0xCC, + // Bytes 380 - 3bf + 0xA8, 0x03, 0x6F, 0xCC, 0xA8, 0x05, 0x4F, 0xCC, + 0xA8, 0xCC, 0x84, 0x05, 0x6F, 0xCC, 0xA8, 0xCC, + 0x84, 0x04, 0xC6, 0xB7, 0xCC, 0x8C, 0x04, 0xCA, + 0x92, 0xCC, 0x8C, 0x03, 0x6A, 0xCC, 0x8C, 0x02, + 0x44, 0x5A, 0x02, 0x44, 0x7A, 0x02, 0x64, 0x7A, + 0x03, 0x47, 0xCC, 0x81, 0x03, 0x67, 0xCC, 0x81, + 0x03, 0x4E, 0xCC, 0x80, 0x03, 0x6E, 0xCC, 0x80, + 0x05, 0x41, 0xCC, 0x8A, 0xCC, 0x81, 0x05, 0x61, + // Bytes 3c0 - 3ff + 0xCC, 0x8A, 0xCC, 0x81, 0x04, 0xC3, 0x86, 0xCC, + 0x81, 0x04, 0xC3, 0xA6, 0xCC, 0x81, 0x04, 0xC3, + 0x98, 0xCC, 0x81, 0x04, 0xC3, 0xB8, 0xCC, 0x81, + 0x03, 0x41, 0xCC, 0x8F, 0x03, 0x61, 0xCC, 0x8F, + 0x03, 0x41, 0xCC, 0x91, 0x03, 0x61, 0xCC, 0x91, + 0x03, 0x45, 0xCC, 0x8F, 0x03, 0x65, 0xCC, 0x8F, + 0x03, 0x45, 0xCC, 0x91, 0x03, 0x65, 0xCC, 0x91, + 0x03, 0x49, 0xCC, 0x8F, 0x03, 0x69, 0xCC, 0x8F, + // Bytes 400 - 43f + 0x03, 0x49, 0xCC, 0x91, 0x03, 0x69, 0xCC, 0x91, + 0x03, 0x4F, 0xCC, 0x8F, 0x03, 0x6F, 0xCC, 0x8F, + 0x03, 0x4F, 0xCC, 0x91, 0x03, 0x6F, 0xCC, 0x91, + 0x03, 0x52, 0xCC, 0x8F, 0x03, 0x72, 0xCC, 0x8F, + 0x03, 0x52, 0xCC, 0x91, 0x03, 0x72, 0xCC, 0x91, + 0x03, 0x55, 0xCC, 0x8F, 0x03, 0x75, 0xCC, 0x8F, + 0x03, 0x55, 0xCC, 0x91, 0x03, 0x75, 0xCC, 0x91, + 0x03, 0x53, 0xCC, 0xA6, 0x03, 0x73, 0xCC, 0xA6, + // Bytes 440 - 47f + 0x03, 0x54, 0xCC, 0xA6, 0x03, 0x74, 0xCC, 0xA6, + 0x03, 0x48, 0xCC, 0x8C, 0x03, 0x68, 0xCC, 0x8C, + 0x03, 0x41, 0xCC, 0x87, 0x03, 0x61, 0xCC, 0x87, + 0x03, 0x45, 0xCC, 0xA7, 0x03, 0x65, 0xCC, 0xA7, + 0x05, 0x4F, 0xCC, 0x88, 0xCC, 0x84, 0x05, 0x6F, + 0xCC, 0x88, 0xCC, 0x84, 0x05, 0x4F, 0xCC, 0x83, + 0xCC, 0x84, 0x05, 0x6F, 0xCC, 0x83, 0xCC, 0x84, + 0x03, 0x4F, 0xCC, 0x87, 0x03, 0x6F, 0xCC, 0x87, + // Bytes 480 - 4bf + 0x05, 0x4F, 0xCC, 0x87, 0xCC, 0x84, 0x05, 0x6F, + 0xCC, 0x87, 0xCC, 0x84, 0x03, 0x59, 0xCC, 0x84, + 0x03, 0x79, 0xCC, 0x84, 0x01, 0x68, 0x02, 0xC9, + 0xA6, 0x01, 0x6A, 0x01, 0x72, 0x02, 0xC9, 0xB9, + 0x02, 0xC9, 0xBB, 0x02, 0xCA, 0x81, 0x01, 0x77, + 0x01, 0x79, 0x03, 0x20, 0xCC, 0x86, 0x03, 0x20, + 0xCC, 0x87, 0x03, 0x20, 0xCC, 0x8A, 0x03, 0x20, + 0xCC, 0xA8, 0x03, 0x20, 0xCC, 0x83, 0x03, 0x20, + // Bytes 4c0 - 4ff + 0xCC, 0x8B, 0x02, 0xC9, 0xA3, 0x01, 0x6C, 0x01, + 0x78, 0x02, 0xCA, 0x95, 0x02, 0xCC, 0x80, 0x02, + 0xCC, 0x81, 0x02, 0xCC, 0x93, 0x04, 0xCC, 0x88, + 0xCC, 0x81, 0x02, 0xCA, 0xB9, 0x03, 0x20, 0xCD, + 0x85, 0x01, 0x3B, 0x04, 0xC2, 0xA8, 0xCC, 0x81, + 0x05, 0x20, 0xCC, 0x88, 0xCC, 0x81, 0x04, 0xCE, + 0x91, 0xCC, 0x81, 0x02, 0xC2, 0xB7, 0x04, 0xCE, + 0x95, 0xCC, 0x81, 0x04, 0xCE, 0x97, 0xCC, 0x81, + // Bytes 500 - 53f + 0x04, 0xCE, 0x99, 0xCC, 0x81, 0x04, 0xCE, 0x9F, + 0xCC, 0x81, 0x04, 0xCE, 0xA5, 0xCC, 0x81, 0x04, + 0xCE, 0xA9, 0xCC, 0x81, 0x06, 0xCE, 0xB9, 0xCC, + 0x88, 0xCC, 0x81, 0x04, 0xCE, 0x99, 0xCC, 0x88, + 0x04, 0xCE, 0xA5, 0xCC, 0x88, 0x04, 0xCE, 0xB1, + 0xCC, 0x81, 0x04, 0xCE, 0xB5, 0xCC, 0x81, 0x04, + 0xCE, 0xB7, 0xCC, 0x81, 0x04, 0xCE, 0xB9, 0xCC, + 0x81, 0x06, 0xCF, 0x85, 0xCC, 0x88, 0xCC, 0x81, + // Bytes 540 - 57f + 0x04, 0xCE, 0xB9, 0xCC, 0x88, 0x04, 0xCF, 0x85, + 0xCC, 0x88, 0x04, 0xCE, 0xBF, 0xCC, 0x81, 0x04, + 0xCF, 0x85, 0xCC, 0x81, 0x04, 0xCF, 0x89, 0xCC, + 0x81, 0x02, 0xCE, 0xB2, 0x02, 0xCE, 0xB8, 0x02, + 0xCE, 0xA5, 0x04, 0xCF, 0x92, 0xCC, 0x81, 0x04, + 0xCF, 0x92, 0xCC, 0x88, 0x02, 0xCF, 0x86, 0x02, + 0xCF, 0x80, 0x02, 0xCE, 0xBA, 0x02, 0xCF, 0x81, + 0x02, 0xCF, 0x82, 0x02, 0xCE, 0x98, 0x02, 0xCE, + // Bytes 580 - 5bf + 0xB5, 0x02, 0xCE, 0xA3, 0x04, 0xD0, 0x95, 0xCC, + 0x80, 0x04, 0xD0, 0x95, 0xCC, 0x88, 0x04, 0xD0, + 0x93, 0xCC, 0x81, 0x04, 0xD0, 0x86, 0xCC, 0x88, + 0x04, 0xD0, 0x9A, 0xCC, 0x81, 0x04, 0xD0, 0x98, + 0xCC, 0x80, 0x04, 0xD0, 0xA3, 0xCC, 0x86, 0x04, + 0xD0, 0x98, 0xCC, 0x86, 0x04, 0xD0, 0xB8, 0xCC, + 0x86, 0x04, 0xD0, 0xB5, 0xCC, 0x80, 0x04, 0xD0, + 0xB5, 0xCC, 0x88, 0x04, 0xD0, 0xB3, 0xCC, 0x81, + // Bytes 5c0 - 5ff + 0x04, 0xD1, 0x96, 0xCC, 0x88, 0x04, 0xD0, 0xBA, + 0xCC, 0x81, 0x04, 0xD0, 0xB8, 0xCC, 0x80, 0x04, + 0xD1, 0x83, 0xCC, 0x86, 0x04, 0xD1, 0xB4, 0xCC, + 0x8F, 0x04, 0xD1, 0xB5, 0xCC, 0x8F, 0x04, 0xD0, + 0x96, 0xCC, 0x86, 0x04, 0xD0, 0xB6, 0xCC, 0x86, + 0x04, 0xD0, 0x90, 0xCC, 0x86, 0x04, 0xD0, 0xB0, + 0xCC, 0x86, 0x04, 0xD0, 0x90, 0xCC, 0x88, 0x04, + 0xD0, 0xB0, 0xCC, 0x88, 0x04, 0xD0, 0x95, 0xCC, + // Bytes 600 - 63f + 0x86, 0x04, 0xD0, 0xB5, 0xCC, 0x86, 0x04, 0xD3, + 0x98, 0xCC, 0x88, 0x04, 0xD3, 0x99, 0xCC, 0x88, + 0x04, 0xD0, 0x96, 0xCC, 0x88, 0x04, 0xD0, 0xB6, + 0xCC, 0x88, 0x04, 0xD0, 0x97, 0xCC, 0x88, 0x04, + 0xD0, 0xB7, 0xCC, 0x88, 0x04, 0xD0, 0x98, 0xCC, + 0x84, 0x04, 0xD0, 0xB8, 0xCC, 0x84, 0x04, 0xD0, + 0x98, 0xCC, 0x88, 0x04, 0xD0, 0xB8, 0xCC, 0x88, + 0x04, 0xD0, 0x9E, 0xCC, 0x88, 0x04, 0xD0, 0xBE, + // Bytes 640 - 67f + 0xCC, 0x88, 0x04, 0xD3, 0xA8, 0xCC, 0x88, 0x04, + 0xD3, 0xA9, 0xCC, 0x88, 0x04, 0xD0, 0xAD, 0xCC, + 0x88, 0x04, 0xD1, 0x8D, 0xCC, 0x88, 0x04, 0xD0, + 0xA3, 0xCC, 0x84, 0x04, 0xD1, 0x83, 0xCC, 0x84, + 0x04, 0xD0, 0xA3, 0xCC, 0x88, 0x04, 0xD1, 0x83, + 0xCC, 0x88, 0x04, 0xD0, 0xA3, 0xCC, 0x8B, 0x04, + 0xD1, 0x83, 0xCC, 0x8B, 0x04, 0xD0, 0xA7, 0xCC, + 0x88, 0x04, 0xD1, 0x87, 0xCC, 0x88, 0x04, 0xD0, + // Bytes 680 - 6bf + 0xAB, 0xCC, 0x88, 0x04, 0xD1, 0x8B, 0xCC, 0x88, + 0x04, 0xD5, 0xA5, 0xD6, 0x82, 0x04, 0xD8, 0xA7, + 0xD9, 0x93, 0x04, 0xD8, 0xA7, 0xD9, 0x94, 0x04, + 0xD9, 0x88, 0xD9, 0x94, 0x04, 0xD8, 0xA7, 0xD9, + 0x95, 0x04, 0xD9, 0x8A, 0xD9, 0x94, 0x04, 0xD8, + 0xA7, 0xD9, 0xB4, 0x04, 0xD9, 0x88, 0xD9, 0xB4, + 0x04, 0xDB, 0x87, 0xD9, 0xB4, 0x04, 0xD9, 0x8A, + 0xD9, 0xB4, 0x04, 0xDB, 0x95, 0xD9, 0x94, 0x04, + // Bytes 6c0 - 6ff + 0xDB, 0x81, 0xD9, 0x94, 0x04, 0xDB, 0x92, 0xD9, + 0x94, 0x06, 0xE0, 0xA4, 0xA8, 0xE0, 0xA4, 0xBC, + 0x06, 0xE0, 0xA4, 0xB0, 0xE0, 0xA4, 0xBC, 0x06, + 0xE0, 0xA4, 0xB3, 0xE0, 0xA4, 0xBC, 0x06, 0xE0, + 0xA4, 0x95, 0xE0, 0xA4, 0xBC, 0x06, 0xE0, 0xA4, + 0x96, 0xE0, 0xA4, 0xBC, 0x06, 0xE0, 0xA4, 0x97, + 0xE0, 0xA4, 0xBC, 0x06, 0xE0, 0xA4, 0x9C, 0xE0, + 0xA4, 0xBC, 0x06, 0xE0, 0xA4, 0xA1, 0xE0, 0xA4, + // Bytes 700 - 73f + 0xBC, 0x06, 0xE0, 0xA4, 0xA2, 0xE0, 0xA4, 0xBC, + 0x06, 0xE0, 0xA4, 0xAB, 0xE0, 0xA4, 0xBC, 0x06, + 0xE0, 0xA4, 0xAF, 0xE0, 0xA4, 0xBC, 0x06, 0xE0, + 0xA7, 0x87, 0xE0, 0xA6, 0xBE, 0x06, 0xE0, 0xA7, + 0x87, 0xE0, 0xA7, 0x97, 0x06, 0xE0, 0xA6, 0xA1, + 0xE0, 0xA6, 0xBC, 0x06, 0xE0, 0xA6, 0xA2, 0xE0, + 0xA6, 0xBC, 0x06, 0xE0, 0xA6, 0xAF, 0xE0, 0xA6, + 0xBC, 0x06, 0xE0, 0xA8, 0xB2, 0xE0, 0xA8, 0xBC, + // Bytes 740 - 77f + 0x06, 0xE0, 0xA8, 0xB8, 0xE0, 0xA8, 0xBC, 0x06, + 0xE0, 0xA8, 0x96, 0xE0, 0xA8, 0xBC, 0x06, 0xE0, + 0xA8, 0x97, 0xE0, 0xA8, 0xBC, 0x06, 0xE0, 0xA8, + 0x9C, 0xE0, 0xA8, 0xBC, 0x06, 0xE0, 0xA8, 0xAB, + 0xE0, 0xA8, 0xBC, 0x06, 0xE0, 0xAD, 0x87, 0xE0, + 0xAD, 0x96, 0x06, 0xE0, 0xAD, 0x87, 0xE0, 0xAC, + 0xBE, 0x06, 0xE0, 0xAD, 0x87, 0xE0, 0xAD, 0x97, + 0x06, 0xE0, 0xAC, 0xA1, 0xE0, 0xAC, 0xBC, 0x06, + // Bytes 780 - 7bf + 0xE0, 0xAC, 0xA2, 0xE0, 0xAC, 0xBC, 0x06, 0xE0, + 0xAE, 0x92, 0xE0, 0xAF, 0x97, 0x06, 0xE0, 0xAF, + 0x86, 0xE0, 0xAE, 0xBE, 0x06, 0xE0, 0xAF, 0x87, + 0xE0, 0xAE, 0xBE, 0x06, 0xE0, 0xAF, 0x86, 0xE0, + 0xAF, 0x97, 0x06, 0xE0, 0xB1, 0x86, 0xE0, 0xB1, + 0x96, 0x06, 0xE0, 0xB2, 0xBF, 0xE0, 0xB3, 0x95, + 0x06, 0xE0, 0xB3, 0x86, 0xE0, 0xB3, 0x95, 0x06, + 0xE0, 0xB3, 0x86, 0xE0, 0xB3, 0x96, 0x06, 0xE0, + // Bytes 7c0 - 7ff + 0xB3, 0x86, 0xE0, 0xB3, 0x82, 0x09, 0xE0, 0xB3, + 0x86, 0xE0, 0xB3, 0x82, 0xE0, 0xB3, 0x95, 0x06, + 0xE0, 0xB5, 0x86, 0xE0, 0xB4, 0xBE, 0x06, 0xE0, + 0xB5, 0x87, 0xE0, 0xB4, 0xBE, 0x06, 0xE0, 0xB5, + 0x86, 0xE0, 0xB5, 0x97, 0x06, 0xE0, 0xB7, 0x99, + 0xE0, 0xB7, 0x8A, 0x06, 0xE0, 0xB7, 0x99, 0xE0, + 0xB7, 0x8F, 0x09, 0xE0, 0xB7, 0x99, 0xE0, 0xB7, + 0x8F, 0xE0, 0xB7, 0x8A, 0x06, 0xE0, 0xB7, 0x99, + // Bytes 800 - 83f + 0xE0, 0xB7, 0x9F, 0x06, 0xE0, 0xB9, 0x8D, 0xE0, + 0xB8, 0xB2, 0x06, 0xE0, 0xBB, 0x8D, 0xE0, 0xBA, + 0xB2, 0x06, 0xE0, 0xBA, 0xAB, 0xE0, 0xBA, 0x99, + 0x06, 0xE0, 0xBA, 0xAB, 0xE0, 0xBA, 0xA1, 0x03, + 0xE0, 0xBC, 0x8B, 0x06, 0xE0, 0xBD, 0x82, 0xE0, + 0xBE, 0xB7, 0x06, 0xE0, 0xBD, 0x8C, 0xE0, 0xBE, + 0xB7, 0x06, 0xE0, 0xBD, 0x91, 0xE0, 0xBE, 0xB7, + 0x06, 0xE0, 0xBD, 0x96, 0xE0, 0xBE, 0xB7, 0x06, + // Bytes 840 - 87f + 0xE0, 0xBD, 0x9B, 0xE0, 0xBE, 0xB7, 0x06, 0xE0, + 0xBD, 0x80, 0xE0, 0xBE, 0xB5, 0x06, 0xE0, 0xBD, + 0xB1, 0xE0, 0xBD, 0xB2, 0x06, 0xE0, 0xBD, 0xB1, + 0xE0, 0xBD, 0xB4, 0x06, 0xE0, 0xBE, 0xB2, 0xE0, + 0xBE, 0x80, 0x09, 0xE0, 0xBE, 0xB2, 0xE0, 0xBD, + 0xB1, 0xE0, 0xBE, 0x80, 0x06, 0xE0, 0xBE, 0xB3, + 0xE0, 0xBE, 0x80, 0x09, 0xE0, 0xBE, 0xB3, 0xE0, + 0xBD, 0xB1, 0xE0, 0xBE, 0x80, 0x06, 0xE0, 0xBD, + // Bytes 880 - 8bf + 0xB1, 0xE0, 0xBE, 0x80, 0x06, 0xE0, 0xBE, 0x92, + 0xE0, 0xBE, 0xB7, 0x06, 0xE0, 0xBE, 0x9C, 0xE0, + 0xBE, 0xB7, 0x06, 0xE0, 0xBE, 0xA1, 0xE0, 0xBE, + 0xB7, 0x06, 0xE0, 0xBE, 0xA6, 0xE0, 0xBE, 0xB7, + 0x06, 0xE0, 0xBE, 0xAB, 0xE0, 0xBE, 0xB7, 0x06, + 0xE0, 0xBE, 0x90, 0xE0, 0xBE, 0xB5, 0x06, 0xE1, + 0x80, 0xA5, 0xE1, 0x80, 0xAE, 0x03, 0xE1, 0x83, + 0x9C, 0x06, 0xE1, 0xAC, 0x85, 0xE1, 0xAC, 0xB5, + // Bytes 8c0 - 8ff + 0x06, 0xE1, 0xAC, 0x87, 0xE1, 0xAC, 0xB5, 0x06, + 0xE1, 0xAC, 0x89, 0xE1, 0xAC, 0xB5, 0x06, 0xE1, + 0xAC, 0x8B, 0xE1, 0xAC, 0xB5, 0x06, 0xE1, 0xAC, + 0x8D, 0xE1, 0xAC, 0xB5, 0x06, 0xE1, 0xAC, 0x91, + 0xE1, 0xAC, 0xB5, 0x06, 0xE1, 0xAC, 0xBA, 0xE1, + 0xAC, 0xB5, 0x06, 0xE1, 0xAC, 0xBC, 0xE1, 0xAC, + 0xB5, 0x06, 0xE1, 0xAC, 0xBE, 0xE1, 0xAC, 0xB5, + 0x06, 0xE1, 0xAC, 0xBF, 0xE1, 0xAC, 0xB5, 0x06, + // Bytes 900 - 93f + 0xE1, 0xAD, 0x82, 0xE1, 0xAC, 0xB5, 0x01, 0x41, + 0x02, 0xC3, 0x86, 0x01, 0x42, 0x01, 0x44, 0x01, + 0x45, 0x02, 0xC6, 0x8E, 0x01, 0x47, 0x01, 0x48, + 0x01, 0x49, 0x01, 0x4A, 0x01, 0x4B, 0x01, 0x4C, + 0x01, 0x4D, 0x01, 0x4E, 0x01, 0x4F, 0x02, 0xC8, + 0xA2, 0x01, 0x50, 0x01, 0x52, 0x01, 0x54, 0x01, + 0x55, 0x01, 0x57, 0x02, 0xC9, 0x90, 0x02, 0xC9, + 0x91, 0x03, 0xE1, 0xB4, 0x82, 0x01, 0x62, 0x01, + // Bytes 940 - 97f + 0x64, 0x01, 0x65, 0x02, 0xC9, 0x99, 0x02, 0xC9, + 0x9B, 0x02, 0xC9, 0x9C, 0x01, 0x67, 0x01, 0x6B, + 0x01, 0x6D, 0x02, 0xC5, 0x8B, 0x02, 0xC9, 0x94, + 0x03, 0xE1, 0xB4, 0x96, 0x03, 0xE1, 0xB4, 0x97, + 0x01, 0x70, 0x01, 0x74, 0x01, 0x75, 0x03, 0xE1, + 0xB4, 0x9D, 0x02, 0xC9, 0xAF, 0x01, 0x76, 0x03, + 0xE1, 0xB4, 0xA5, 0x02, 0xCE, 0xB3, 0x02, 0xCE, + 0xB4, 0x02, 0xCF, 0x87, 0x01, 0x69, 0x02, 0xD0, + // Bytes 980 - 9bf + 0xBD, 0x02, 0xC9, 0x92, 0x01, 0x63, 0x02, 0xC9, + 0x95, 0x02, 0xC3, 0xB0, 0x01, 0x66, 0x02, 0xC9, + 0x9F, 0x02, 0xC9, 0xA1, 0x02, 0xC9, 0xA5, 0x02, + 0xC9, 0xA8, 0x02, 0xC9, 0xA9, 0x02, 0xC9, 0xAA, + 0x03, 0xE1, 0xB5, 0xBB, 0x02, 0xCA, 0x9D, 0x02, + 0xC9, 0xAD, 0x03, 0xE1, 0xB6, 0x85, 0x02, 0xCA, + 0x9F, 0x02, 0xC9, 0xB1, 0x02, 0xC9, 0xB0, 0x02, + 0xC9, 0xB2, 0x02, 0xC9, 0xB3, 0x02, 0xC9, 0xB4, + // Bytes 9c0 - 9ff + 0x02, 0xC9, 0xB5, 0x02, 0xC9, 0xB8, 0x02, 0xCA, + 0x82, 0x02, 0xCA, 0x83, 0x02, 0xC6, 0xAB, 0x02, + 0xCA, 0x89, 0x02, 0xCA, 0x8A, 0x03, 0xE1, 0xB4, + 0x9C, 0x02, 0xCA, 0x8B, 0x02, 0xCA, 0x8C, 0x01, + 0x7A, 0x02, 0xCA, 0x90, 0x02, 0xCA, 0x91, 0x02, + 0xCA, 0x92, 0x03, 0x41, 0xCC, 0xA5, 0x03, 0x61, + 0xCC, 0xA5, 0x03, 0x42, 0xCC, 0x87, 0x03, 0x62, + 0xCC, 0x87, 0x03, 0x42, 0xCC, 0xA3, 0x03, 0x62, + // Bytes a00 - a3f + 0xCC, 0xA3, 0x03, 0x42, 0xCC, 0xB1, 0x03, 0x62, + 0xCC, 0xB1, 0x05, 0x43, 0xCC, 0xA7, 0xCC, 0x81, + 0x05, 0x63, 0xCC, 0xA7, 0xCC, 0x81, 0x03, 0x44, + 0xCC, 0x87, 0x03, 0x64, 0xCC, 0x87, 0x03, 0x44, + 0xCC, 0xA3, 0x03, 0x64, 0xCC, 0xA3, 0x03, 0x44, + 0xCC, 0xB1, 0x03, 0x64, 0xCC, 0xB1, 0x03, 0x44, + 0xCC, 0xA7, 0x03, 0x64, 0xCC, 0xA7, 0x03, 0x44, + 0xCC, 0xAD, 0x03, 0x64, 0xCC, 0xAD, 0x05, 0x45, + // Bytes a40 - a7f + 0xCC, 0x84, 0xCC, 0x80, 0x05, 0x65, 0xCC, 0x84, + 0xCC, 0x80, 0x05, 0x45, 0xCC, 0x84, 0xCC, 0x81, + 0x05, 0x65, 0xCC, 0x84, 0xCC, 0x81, 0x03, 0x45, + 0xCC, 0xAD, 0x03, 0x65, 0xCC, 0xAD, 0x03, 0x45, + 0xCC, 0xB0, 0x03, 0x65, 0xCC, 0xB0, 0x05, 0x45, + 0xCC, 0xA7, 0xCC, 0x86, 0x05, 0x65, 0xCC, 0xA7, + 0xCC, 0x86, 0x03, 0x46, 0xCC, 0x87, 0x03, 0x66, + 0xCC, 0x87, 0x03, 0x47, 0xCC, 0x84, 0x03, 0x67, + // Bytes a80 - abf + 0xCC, 0x84, 0x03, 0x48, 0xCC, 0x87, 0x03, 0x68, + 0xCC, 0x87, 0x03, 0x48, 0xCC, 0xA3, 0x03, 0x68, + 0xCC, 0xA3, 0x03, 0x48, 0xCC, 0x88, 0x03, 0x68, + 0xCC, 0x88, 0x03, 0x48, 0xCC, 0xA7, 0x03, 0x68, + 0xCC, 0xA7, 0x03, 0x48, 0xCC, 0xAE, 0x03, 0x68, + 0xCC, 0xAE, 0x03, 0x49, 0xCC, 0xB0, 0x03, 0x69, + 0xCC, 0xB0, 0x05, 0x49, 0xCC, 0x88, 0xCC, 0x81, + 0x05, 0x69, 0xCC, 0x88, 0xCC, 0x81, 0x03, 0x4B, + // Bytes ac0 - aff + 0xCC, 0x81, 0x03, 0x6B, 0xCC, 0x81, 0x03, 0x4B, + 0xCC, 0xA3, 0x03, 0x6B, 0xCC, 0xA3, 0x03, 0x4B, + 0xCC, 0xB1, 0x03, 0x6B, 0xCC, 0xB1, 0x03, 0x4C, + 0xCC, 0xA3, 0x03, 0x6C, 0xCC, 0xA3, 0x05, 0x4C, + 0xCC, 0xA3, 0xCC, 0x84, 0x05, 0x6C, 0xCC, 0xA3, + 0xCC, 0x84, 0x03, 0x4C, 0xCC, 0xB1, 0x03, 0x6C, + 0xCC, 0xB1, 0x03, 0x4C, 0xCC, 0xAD, 0x03, 0x6C, + 0xCC, 0xAD, 0x03, 0x4D, 0xCC, 0x81, 0x03, 0x6D, + // Bytes b00 - b3f + 0xCC, 0x81, 0x03, 0x4D, 0xCC, 0x87, 0x03, 0x6D, + 0xCC, 0x87, 0x03, 0x4D, 0xCC, 0xA3, 0x03, 0x6D, + 0xCC, 0xA3, 0x03, 0x4E, 0xCC, 0x87, 0x03, 0x6E, + 0xCC, 0x87, 0x03, 0x4E, 0xCC, 0xA3, 0x03, 0x6E, + 0xCC, 0xA3, 0x03, 0x4E, 0xCC, 0xB1, 0x03, 0x6E, + 0xCC, 0xB1, 0x03, 0x4E, 0xCC, 0xAD, 0x03, 0x6E, + 0xCC, 0xAD, 0x05, 0x4F, 0xCC, 0x83, 0xCC, 0x81, + 0x05, 0x6F, 0xCC, 0x83, 0xCC, 0x81, 0x05, 0x4F, + // Bytes b40 - b7f + 0xCC, 0x83, 0xCC, 0x88, 0x05, 0x6F, 0xCC, 0x83, + 0xCC, 0x88, 0x05, 0x4F, 0xCC, 0x84, 0xCC, 0x80, + 0x05, 0x6F, 0xCC, 0x84, 0xCC, 0x80, 0x05, 0x4F, + 0xCC, 0x84, 0xCC, 0x81, 0x05, 0x6F, 0xCC, 0x84, + 0xCC, 0x81, 0x03, 0x50, 0xCC, 0x81, 0x03, 0x70, + 0xCC, 0x81, 0x03, 0x50, 0xCC, 0x87, 0x03, 0x70, + 0xCC, 0x87, 0x03, 0x52, 0xCC, 0x87, 0x03, 0x72, + 0xCC, 0x87, 0x03, 0x52, 0xCC, 0xA3, 0x03, 0x72, + // Bytes b80 - bbf + 0xCC, 0xA3, 0x05, 0x52, 0xCC, 0xA3, 0xCC, 0x84, + 0x05, 0x72, 0xCC, 0xA3, 0xCC, 0x84, 0x03, 0x52, + 0xCC, 0xB1, 0x03, 0x72, 0xCC, 0xB1, 0x03, 0x53, + 0xCC, 0x87, 0x03, 0x73, 0xCC, 0x87, 0x03, 0x53, + 0xCC, 0xA3, 0x03, 0x73, 0xCC, 0xA3, 0x05, 0x53, + 0xCC, 0x81, 0xCC, 0x87, 0x05, 0x73, 0xCC, 0x81, + 0xCC, 0x87, 0x05, 0x53, 0xCC, 0x8C, 0xCC, 0x87, + 0x05, 0x73, 0xCC, 0x8C, 0xCC, 0x87, 0x05, 0x53, + // Bytes bc0 - bff + 0xCC, 0xA3, 0xCC, 0x87, 0x05, 0x73, 0xCC, 0xA3, + 0xCC, 0x87, 0x03, 0x54, 0xCC, 0x87, 0x03, 0x74, + 0xCC, 0x87, 0x03, 0x54, 0xCC, 0xA3, 0x03, 0x74, + 0xCC, 0xA3, 0x03, 0x54, 0xCC, 0xB1, 0x03, 0x74, + 0xCC, 0xB1, 0x03, 0x54, 0xCC, 0xAD, 0x03, 0x74, + 0xCC, 0xAD, 0x03, 0x55, 0xCC, 0xA4, 0x03, 0x75, + 0xCC, 0xA4, 0x03, 0x55, 0xCC, 0xB0, 0x03, 0x75, + 0xCC, 0xB0, 0x03, 0x55, 0xCC, 0xAD, 0x03, 0x75, + // Bytes c00 - c3f + 0xCC, 0xAD, 0x05, 0x55, 0xCC, 0x83, 0xCC, 0x81, + 0x05, 0x75, 0xCC, 0x83, 0xCC, 0x81, 0x05, 0x55, + 0xCC, 0x84, 0xCC, 0x88, 0x05, 0x75, 0xCC, 0x84, + 0xCC, 0x88, 0x03, 0x56, 0xCC, 0x83, 0x03, 0x76, + 0xCC, 0x83, 0x03, 0x56, 0xCC, 0xA3, 0x03, 0x76, + 0xCC, 0xA3, 0x03, 0x57, 0xCC, 0x80, 0x03, 0x77, + 0xCC, 0x80, 0x03, 0x57, 0xCC, 0x81, 0x03, 0x77, + 0xCC, 0x81, 0x03, 0x57, 0xCC, 0x88, 0x03, 0x77, + // Bytes c40 - c7f + 0xCC, 0x88, 0x03, 0x57, 0xCC, 0x87, 0x03, 0x77, + 0xCC, 0x87, 0x03, 0x57, 0xCC, 0xA3, 0x03, 0x77, + 0xCC, 0xA3, 0x03, 0x58, 0xCC, 0x87, 0x03, 0x78, + 0xCC, 0x87, 0x03, 0x58, 0xCC, 0x88, 0x03, 0x78, + 0xCC, 0x88, 0x03, 0x59, 0xCC, 0x87, 0x03, 0x79, + 0xCC, 0x87, 0x03, 0x5A, 0xCC, 0x82, 0x03, 0x7A, + 0xCC, 0x82, 0x03, 0x5A, 0xCC, 0xA3, 0x03, 0x7A, + 0xCC, 0xA3, 0x03, 0x5A, 0xCC, 0xB1, 0x03, 0x7A, + // Bytes c80 - cbf + 0xCC, 0xB1, 0x03, 0x68, 0xCC, 0xB1, 0x03, 0x74, + 0xCC, 0x88, 0x03, 0x77, 0xCC, 0x8A, 0x03, 0x79, + 0xCC, 0x8A, 0x03, 0x61, 0xCA, 0xBE, 0x04, 0xC5, + 0xBF, 0xCC, 0x87, 0x03, 0x41, 0xCC, 0xA3, 0x03, + 0x61, 0xCC, 0xA3, 0x03, 0x41, 0xCC, 0x89, 0x03, + 0x61, 0xCC, 0x89, 0x05, 0x41, 0xCC, 0x82, 0xCC, + 0x81, 0x05, 0x61, 0xCC, 0x82, 0xCC, 0x81, 0x05, + 0x41, 0xCC, 0x82, 0xCC, 0x80, 0x05, 0x61, 0xCC, + // Bytes cc0 - cff + 0x82, 0xCC, 0x80, 0x05, 0x41, 0xCC, 0x82, 0xCC, + 0x89, 0x05, 0x61, 0xCC, 0x82, 0xCC, 0x89, 0x05, + 0x41, 0xCC, 0x82, 0xCC, 0x83, 0x05, 0x61, 0xCC, + 0x82, 0xCC, 0x83, 0x05, 0x41, 0xCC, 0xA3, 0xCC, + 0x82, 0x05, 0x61, 0xCC, 0xA3, 0xCC, 0x82, 0x05, + 0x41, 0xCC, 0x86, 0xCC, 0x81, 0x05, 0x61, 0xCC, + 0x86, 0xCC, 0x81, 0x05, 0x41, 0xCC, 0x86, 0xCC, + 0x80, 0x05, 0x61, 0xCC, 0x86, 0xCC, 0x80, 0x05, + // Bytes d00 - d3f + 0x41, 0xCC, 0x86, 0xCC, 0x89, 0x05, 0x61, 0xCC, + 0x86, 0xCC, 0x89, 0x05, 0x41, 0xCC, 0x86, 0xCC, + 0x83, 0x05, 0x61, 0xCC, 0x86, 0xCC, 0x83, 0x05, + 0x41, 0xCC, 0xA3, 0xCC, 0x86, 0x05, 0x61, 0xCC, + 0xA3, 0xCC, 0x86, 0x03, 0x45, 0xCC, 0xA3, 0x03, + 0x65, 0xCC, 0xA3, 0x03, 0x45, 0xCC, 0x89, 0x03, + 0x65, 0xCC, 0x89, 0x03, 0x45, 0xCC, 0x83, 0x03, + 0x65, 0xCC, 0x83, 0x05, 0x45, 0xCC, 0x82, 0xCC, + // Bytes d40 - d7f + 0x81, 0x05, 0x65, 0xCC, 0x82, 0xCC, 0x81, 0x05, + 0x45, 0xCC, 0x82, 0xCC, 0x80, 0x05, 0x65, 0xCC, + 0x82, 0xCC, 0x80, 0x05, 0x45, 0xCC, 0x82, 0xCC, + 0x89, 0x05, 0x65, 0xCC, 0x82, 0xCC, 0x89, 0x05, + 0x45, 0xCC, 0x82, 0xCC, 0x83, 0x05, 0x65, 0xCC, + 0x82, 0xCC, 0x83, 0x05, 0x45, 0xCC, 0xA3, 0xCC, + 0x82, 0x05, 0x65, 0xCC, 0xA3, 0xCC, 0x82, 0x03, + 0x49, 0xCC, 0x89, 0x03, 0x69, 0xCC, 0x89, 0x03, + // Bytes d80 - dbf + 0x49, 0xCC, 0xA3, 0x03, 0x69, 0xCC, 0xA3, 0x03, + 0x4F, 0xCC, 0xA3, 0x03, 0x6F, 0xCC, 0xA3, 0x03, + 0x4F, 0xCC, 0x89, 0x03, 0x6F, 0xCC, 0x89, 0x05, + 0x4F, 0xCC, 0x82, 0xCC, 0x81, 0x05, 0x6F, 0xCC, + 0x82, 0xCC, 0x81, 0x05, 0x4F, 0xCC, 0x82, 0xCC, + 0x80, 0x05, 0x6F, 0xCC, 0x82, 0xCC, 0x80, 0x05, + 0x4F, 0xCC, 0x82, 0xCC, 0x89, 0x05, 0x6F, 0xCC, + 0x82, 0xCC, 0x89, 0x05, 0x4F, 0xCC, 0x82, 0xCC, + // Bytes dc0 - dff + 0x83, 0x05, 0x6F, 0xCC, 0x82, 0xCC, 0x83, 0x05, + 0x4F, 0xCC, 0xA3, 0xCC, 0x82, 0x05, 0x6F, 0xCC, + 0xA3, 0xCC, 0x82, 0x05, 0x4F, 0xCC, 0x9B, 0xCC, + 0x81, 0x05, 0x6F, 0xCC, 0x9B, 0xCC, 0x81, 0x05, + 0x4F, 0xCC, 0x9B, 0xCC, 0x80, 0x05, 0x6F, 0xCC, + 0x9B, 0xCC, 0x80, 0x05, 0x4F, 0xCC, 0x9B, 0xCC, + 0x89, 0x05, 0x6F, 0xCC, 0x9B, 0xCC, 0x89, 0x05, + 0x4F, 0xCC, 0x9B, 0xCC, 0x83, 0x05, 0x6F, 0xCC, + // Bytes e00 - e3f + 0x9B, 0xCC, 0x83, 0x05, 0x4F, 0xCC, 0x9B, 0xCC, + 0xA3, 0x05, 0x6F, 0xCC, 0x9B, 0xCC, 0xA3, 0x03, + 0x55, 0xCC, 0xA3, 0x03, 0x75, 0xCC, 0xA3, 0x03, + 0x55, 0xCC, 0x89, 0x03, 0x75, 0xCC, 0x89, 0x05, + 0x55, 0xCC, 0x9B, 0xCC, 0x81, 0x05, 0x75, 0xCC, + 0x9B, 0xCC, 0x81, 0x05, 0x55, 0xCC, 0x9B, 0xCC, + 0x80, 0x05, 0x75, 0xCC, 0x9B, 0xCC, 0x80, 0x05, + 0x55, 0xCC, 0x9B, 0xCC, 0x89, 0x05, 0x75, 0xCC, + // Bytes e40 - e7f + 0x9B, 0xCC, 0x89, 0x05, 0x55, 0xCC, 0x9B, 0xCC, + 0x83, 0x05, 0x75, 0xCC, 0x9B, 0xCC, 0x83, 0x05, + 0x55, 0xCC, 0x9B, 0xCC, 0xA3, 0x05, 0x75, 0xCC, + 0x9B, 0xCC, 0xA3, 0x03, 0x59, 0xCC, 0x80, 0x03, + 0x79, 0xCC, 0x80, 0x03, 0x59, 0xCC, 0xA3, 0x03, + 0x79, 0xCC, 0xA3, 0x03, 0x59, 0xCC, 0x89, 0x03, + 0x79, 0xCC, 0x89, 0x03, 0x59, 0xCC, 0x83, 0x03, + 0x79, 0xCC, 0x83, 0x04, 0xCE, 0xB1, 0xCC, 0x93, + // Bytes e80 - ebf + 0x04, 0xCE, 0xB1, 0xCC, 0x94, 0x06, 0xCE, 0xB1, + 0xCC, 0x93, 0xCC, 0x80, 0x06, 0xCE, 0xB1, 0xCC, + 0x94, 0xCC, 0x80, 0x06, 0xCE, 0xB1, 0xCC, 0x93, + 0xCC, 0x81, 0x06, 0xCE, 0xB1, 0xCC, 0x94, 0xCC, + 0x81, 0x06, 0xCE, 0xB1, 0xCC, 0x93, 0xCD, 0x82, + 0x06, 0xCE, 0xB1, 0xCC, 0x94, 0xCD, 0x82, 0x04, + 0xCE, 0x91, 0xCC, 0x93, 0x04, 0xCE, 0x91, 0xCC, + 0x94, 0x06, 0xCE, 0x91, 0xCC, 0x93, 0xCC, 0x80, + // Bytes ec0 - eff + 0x06, 0xCE, 0x91, 0xCC, 0x94, 0xCC, 0x80, 0x06, + 0xCE, 0x91, 0xCC, 0x93, 0xCC, 0x81, 0x06, 0xCE, + 0x91, 0xCC, 0x94, 0xCC, 0x81, 0x06, 0xCE, 0x91, + 0xCC, 0x93, 0xCD, 0x82, 0x06, 0xCE, 0x91, 0xCC, + 0x94, 0xCD, 0x82, 0x04, 0xCE, 0xB5, 0xCC, 0x93, + 0x04, 0xCE, 0xB5, 0xCC, 0x94, 0x06, 0xCE, 0xB5, + 0xCC, 0x93, 0xCC, 0x80, 0x06, 0xCE, 0xB5, 0xCC, + 0x94, 0xCC, 0x80, 0x06, 0xCE, 0xB5, 0xCC, 0x93, + // Bytes f00 - f3f + 0xCC, 0x81, 0x06, 0xCE, 0xB5, 0xCC, 0x94, 0xCC, + 0x81, 0x04, 0xCE, 0x95, 0xCC, 0x93, 0x04, 0xCE, + 0x95, 0xCC, 0x94, 0x06, 0xCE, 0x95, 0xCC, 0x93, + 0xCC, 0x80, 0x06, 0xCE, 0x95, 0xCC, 0x94, 0xCC, + 0x80, 0x06, 0xCE, 0x95, 0xCC, 0x93, 0xCC, 0x81, + 0x06, 0xCE, 0x95, 0xCC, 0x94, 0xCC, 0x81, 0x04, + 0xCE, 0xB7, 0xCC, 0x93, 0x04, 0xCE, 0xB7, 0xCC, + 0x94, 0x06, 0xCE, 0xB7, 0xCC, 0x93, 0xCC, 0x80, + // Bytes f40 - f7f + 0x06, 0xCE, 0xB7, 0xCC, 0x94, 0xCC, 0x80, 0x06, + 0xCE, 0xB7, 0xCC, 0x93, 0xCC, 0x81, 0x06, 0xCE, + 0xB7, 0xCC, 0x94, 0xCC, 0x81, 0x06, 0xCE, 0xB7, + 0xCC, 0x93, 0xCD, 0x82, 0x06, 0xCE, 0xB7, 0xCC, + 0x94, 0xCD, 0x82, 0x04, 0xCE, 0x97, 0xCC, 0x93, + 0x04, 0xCE, 0x97, 0xCC, 0x94, 0x06, 0xCE, 0x97, + 0xCC, 0x93, 0xCC, 0x80, 0x06, 0xCE, 0x97, 0xCC, + 0x94, 0xCC, 0x80, 0x06, 0xCE, 0x97, 0xCC, 0x93, + // Bytes f80 - fbf + 0xCC, 0x81, 0x06, 0xCE, 0x97, 0xCC, 0x94, 0xCC, + 0x81, 0x06, 0xCE, 0x97, 0xCC, 0x93, 0xCD, 0x82, + 0x06, 0xCE, 0x97, 0xCC, 0x94, 0xCD, 0x82, 0x04, + 0xCE, 0xB9, 0xCC, 0x93, 0x04, 0xCE, 0xB9, 0xCC, + 0x94, 0x06, 0xCE, 0xB9, 0xCC, 0x93, 0xCC, 0x80, + 0x06, 0xCE, 0xB9, 0xCC, 0x94, 0xCC, 0x80, 0x06, + 0xCE, 0xB9, 0xCC, 0x93, 0xCC, 0x81, 0x06, 0xCE, + 0xB9, 0xCC, 0x94, 0xCC, 0x81, 0x06, 0xCE, 0xB9, + // Bytes fc0 - fff + 0xCC, 0x93, 0xCD, 0x82, 0x06, 0xCE, 0xB9, 0xCC, + 0x94, 0xCD, 0x82, 0x04, 0xCE, 0x99, 0xCC, 0x93, + 0x04, 0xCE, 0x99, 0xCC, 0x94, 0x06, 0xCE, 0x99, + 0xCC, 0x93, 0xCC, 0x80, 0x06, 0xCE, 0x99, 0xCC, + 0x94, 0xCC, 0x80, 0x06, 0xCE, 0x99, 0xCC, 0x93, + 0xCC, 0x81, 0x06, 0xCE, 0x99, 0xCC, 0x94, 0xCC, + 0x81, 0x06, 0xCE, 0x99, 0xCC, 0x93, 0xCD, 0x82, + 0x06, 0xCE, 0x99, 0xCC, 0x94, 0xCD, 0x82, 0x04, + // Bytes 1000 - 103f + 0xCE, 0xBF, 0xCC, 0x93, 0x04, 0xCE, 0xBF, 0xCC, + 0x94, 0x06, 0xCE, 0xBF, 0xCC, 0x93, 0xCC, 0x80, + 0x06, 0xCE, 0xBF, 0xCC, 0x94, 0xCC, 0x80, 0x06, + 0xCE, 0xBF, 0xCC, 0x93, 0xCC, 0x81, 0x06, 0xCE, + 0xBF, 0xCC, 0x94, 0xCC, 0x81, 0x04, 0xCE, 0x9F, + 0xCC, 0x93, 0x04, 0xCE, 0x9F, 0xCC, 0x94, 0x06, + 0xCE, 0x9F, 0xCC, 0x93, 0xCC, 0x80, 0x06, 0xCE, + 0x9F, 0xCC, 0x94, 0xCC, 0x80, 0x06, 0xCE, 0x9F, + // Bytes 1040 - 107f + 0xCC, 0x93, 0xCC, 0x81, 0x06, 0xCE, 0x9F, 0xCC, + 0x94, 0xCC, 0x81, 0x04, 0xCF, 0x85, 0xCC, 0x93, + 0x04, 0xCF, 0x85, 0xCC, 0x94, 0x06, 0xCF, 0x85, + 0xCC, 0x93, 0xCC, 0x80, 0x06, 0xCF, 0x85, 0xCC, + 0x94, 0xCC, 0x80, 0x06, 0xCF, 0x85, 0xCC, 0x93, + 0xCC, 0x81, 0x06, 0xCF, 0x85, 0xCC, 0x94, 0xCC, + 0x81, 0x06, 0xCF, 0x85, 0xCC, 0x93, 0xCD, 0x82, + 0x06, 0xCF, 0x85, 0xCC, 0x94, 0xCD, 0x82, 0x04, + // Bytes 1080 - 10bf + 0xCE, 0xA5, 0xCC, 0x94, 0x06, 0xCE, 0xA5, 0xCC, + 0x94, 0xCC, 0x80, 0x06, 0xCE, 0xA5, 0xCC, 0x94, + 0xCC, 0x81, 0x06, 0xCE, 0xA5, 0xCC, 0x94, 0xCD, + 0x82, 0x04, 0xCF, 0x89, 0xCC, 0x93, 0x04, 0xCF, + 0x89, 0xCC, 0x94, 0x06, 0xCF, 0x89, 0xCC, 0x93, + 0xCC, 0x80, 0x06, 0xCF, 0x89, 0xCC, 0x94, 0xCC, + 0x80, 0x06, 0xCF, 0x89, 0xCC, 0x93, 0xCC, 0x81, + 0x06, 0xCF, 0x89, 0xCC, 0x94, 0xCC, 0x81, 0x06, + // Bytes 10c0 - 10ff + 0xCF, 0x89, 0xCC, 0x93, 0xCD, 0x82, 0x06, 0xCF, + 0x89, 0xCC, 0x94, 0xCD, 0x82, 0x04, 0xCE, 0xA9, + 0xCC, 0x93, 0x04, 0xCE, 0xA9, 0xCC, 0x94, 0x06, + 0xCE, 0xA9, 0xCC, 0x93, 0xCC, 0x80, 0x06, 0xCE, + 0xA9, 0xCC, 0x94, 0xCC, 0x80, 0x06, 0xCE, 0xA9, + 0xCC, 0x93, 0xCC, 0x81, 0x06, 0xCE, 0xA9, 0xCC, + 0x94, 0xCC, 0x81, 0x06, 0xCE, 0xA9, 0xCC, 0x93, + 0xCD, 0x82, 0x06, 0xCE, 0xA9, 0xCC, 0x94, 0xCD, + // Bytes 1100 - 113f + 0x82, 0x04, 0xCE, 0xB1, 0xCC, 0x80, 0x04, 0xCE, + 0xB5, 0xCC, 0x80, 0x04, 0xCE, 0xB7, 0xCC, 0x80, + 0x04, 0xCE, 0xB9, 0xCC, 0x80, 0x04, 0xCE, 0xBF, + 0xCC, 0x80, 0x04, 0xCF, 0x85, 0xCC, 0x80, 0x04, + 0xCF, 0x89, 0xCC, 0x80, 0x06, 0xCE, 0xB1, 0xCC, + 0x93, 0xCD, 0x85, 0x06, 0xCE, 0xB1, 0xCC, 0x94, + 0xCD, 0x85, 0x08, 0xCE, 0xB1, 0xCC, 0x93, 0xCC, + 0x80, 0xCD, 0x85, 0x08, 0xCE, 0xB1, 0xCC, 0x94, + // Bytes 1140 - 117f + 0xCC, 0x80, 0xCD, 0x85, 0x08, 0xCE, 0xB1, 0xCC, + 0x93, 0xCC, 0x81, 0xCD, 0x85, 0x08, 0xCE, 0xB1, + 0xCC, 0x94, 0xCC, 0x81, 0xCD, 0x85, 0x08, 0xCE, + 0xB1, 0xCC, 0x93, 0xCD, 0x82, 0xCD, 0x85, 0x08, + 0xCE, 0xB1, 0xCC, 0x94, 0xCD, 0x82, 0xCD, 0x85, + 0x06, 0xCE, 0x91, 0xCC, 0x93, 0xCD, 0x85, 0x06, + 0xCE, 0x91, 0xCC, 0x94, 0xCD, 0x85, 0x08, 0xCE, + 0x91, 0xCC, 0x93, 0xCC, 0x80, 0xCD, 0x85, 0x08, + // Bytes 1180 - 11bf + 0xCE, 0x91, 0xCC, 0x94, 0xCC, 0x80, 0xCD, 0x85, + 0x08, 0xCE, 0x91, 0xCC, 0x93, 0xCC, 0x81, 0xCD, + 0x85, 0x08, 0xCE, 0x91, 0xCC, 0x94, 0xCC, 0x81, + 0xCD, 0x85, 0x08, 0xCE, 0x91, 0xCC, 0x93, 0xCD, + 0x82, 0xCD, 0x85, 0x08, 0xCE, 0x91, 0xCC, 0x94, + 0xCD, 0x82, 0xCD, 0x85, 0x06, 0xCE, 0xB7, 0xCC, + 0x93, 0xCD, 0x85, 0x06, 0xCE, 0xB7, 0xCC, 0x94, + 0xCD, 0x85, 0x08, 0xCE, 0xB7, 0xCC, 0x93, 0xCC, + // Bytes 11c0 - 11ff + 0x80, 0xCD, 0x85, 0x08, 0xCE, 0xB7, 0xCC, 0x94, + 0xCC, 0x80, 0xCD, 0x85, 0x08, 0xCE, 0xB7, 0xCC, + 0x93, 0xCC, 0x81, 0xCD, 0x85, 0x08, 0xCE, 0xB7, + 0xCC, 0x94, 0xCC, 0x81, 0xCD, 0x85, 0x08, 0xCE, + 0xB7, 0xCC, 0x93, 0xCD, 0x82, 0xCD, 0x85, 0x08, + 0xCE, 0xB7, 0xCC, 0x94, 0xCD, 0x82, 0xCD, 0x85, + 0x06, 0xCE, 0x97, 0xCC, 0x93, 0xCD, 0x85, 0x06, + 0xCE, 0x97, 0xCC, 0x94, 0xCD, 0x85, 0x08, 0xCE, + // Bytes 1200 - 123f + 0x97, 0xCC, 0x93, 0xCC, 0x80, 0xCD, 0x85, 0x08, + 0xCE, 0x97, 0xCC, 0x94, 0xCC, 0x80, 0xCD, 0x85, + 0x08, 0xCE, 0x97, 0xCC, 0x93, 0xCC, 0x81, 0xCD, + 0x85, 0x08, 0xCE, 0x97, 0xCC, 0x94, 0xCC, 0x81, + 0xCD, 0x85, 0x08, 0xCE, 0x97, 0xCC, 0x93, 0xCD, + 0x82, 0xCD, 0x85, 0x08, 0xCE, 0x97, 0xCC, 0x94, + 0xCD, 0x82, 0xCD, 0x85, 0x06, 0xCF, 0x89, 0xCC, + 0x93, 0xCD, 0x85, 0x06, 0xCF, 0x89, 0xCC, 0x94, + // Bytes 1240 - 127f + 0xCD, 0x85, 0x08, 0xCF, 0x89, 0xCC, 0x93, 0xCC, + 0x80, 0xCD, 0x85, 0x08, 0xCF, 0x89, 0xCC, 0x94, + 0xCC, 0x80, 0xCD, 0x85, 0x08, 0xCF, 0x89, 0xCC, + 0x93, 0xCC, 0x81, 0xCD, 0x85, 0x08, 0xCF, 0x89, + 0xCC, 0x94, 0xCC, 0x81, 0xCD, 0x85, 0x08, 0xCF, + 0x89, 0xCC, 0x93, 0xCD, 0x82, 0xCD, 0x85, 0x08, + 0xCF, 0x89, 0xCC, 0x94, 0xCD, 0x82, 0xCD, 0x85, + 0x06, 0xCE, 0xA9, 0xCC, 0x93, 0xCD, 0x85, 0x06, + // Bytes 1280 - 12bf + 0xCE, 0xA9, 0xCC, 0x94, 0xCD, 0x85, 0x08, 0xCE, + 0xA9, 0xCC, 0x93, 0xCC, 0x80, 0xCD, 0x85, 0x08, + 0xCE, 0xA9, 0xCC, 0x94, 0xCC, 0x80, 0xCD, 0x85, + 0x08, 0xCE, 0xA9, 0xCC, 0x93, 0xCC, 0x81, 0xCD, + 0x85, 0x08, 0xCE, 0xA9, 0xCC, 0x94, 0xCC, 0x81, + 0xCD, 0x85, 0x08, 0xCE, 0xA9, 0xCC, 0x93, 0xCD, + 0x82, 0xCD, 0x85, 0x08, 0xCE, 0xA9, 0xCC, 0x94, + 0xCD, 0x82, 0xCD, 0x85, 0x04, 0xCE, 0xB1, 0xCC, + // Bytes 12c0 - 12ff + 0x86, 0x04, 0xCE, 0xB1, 0xCC, 0x84, 0x06, 0xCE, + 0xB1, 0xCC, 0x80, 0xCD, 0x85, 0x04, 0xCE, 0xB1, + 0xCD, 0x85, 0x06, 0xCE, 0xB1, 0xCC, 0x81, 0xCD, + 0x85, 0x04, 0xCE, 0xB1, 0xCD, 0x82, 0x06, 0xCE, + 0xB1, 0xCD, 0x82, 0xCD, 0x85, 0x04, 0xCE, 0x91, + 0xCC, 0x86, 0x04, 0xCE, 0x91, 0xCC, 0x84, 0x04, + 0xCE, 0x91, 0xCC, 0x80, 0x04, 0xCE, 0x91, 0xCD, + 0x85, 0x03, 0x20, 0xCC, 0x93, 0x02, 0xCE, 0xB9, + // Bytes 1300 - 133f + 0x03, 0x20, 0xCD, 0x82, 0x04, 0xC2, 0xA8, 0xCD, + 0x82, 0x05, 0x20, 0xCC, 0x88, 0xCD, 0x82, 0x06, + 0xCE, 0xB7, 0xCC, 0x80, 0xCD, 0x85, 0x04, 0xCE, + 0xB7, 0xCD, 0x85, 0x06, 0xCE, 0xB7, 0xCC, 0x81, + 0xCD, 0x85, 0x04, 0xCE, 0xB7, 0xCD, 0x82, 0x06, + 0xCE, 0xB7, 0xCD, 0x82, 0xCD, 0x85, 0x04, 0xCE, + 0x95, 0xCC, 0x80, 0x04, 0xCE, 0x97, 0xCC, 0x80, + 0x04, 0xCE, 0x97, 0xCD, 0x85, 0x05, 0xE1, 0xBE, + // Bytes 1340 - 137f + 0xBF, 0xCC, 0x80, 0x05, 0x20, 0xCC, 0x93, 0xCC, + 0x80, 0x05, 0xE1, 0xBE, 0xBF, 0xCC, 0x81, 0x05, + 0x20, 0xCC, 0x93, 0xCC, 0x81, 0x05, 0xE1, 0xBE, + 0xBF, 0xCD, 0x82, 0x05, 0x20, 0xCC, 0x93, 0xCD, + 0x82, 0x04, 0xCE, 0xB9, 0xCC, 0x86, 0x04, 0xCE, + 0xB9, 0xCC, 0x84, 0x06, 0xCE, 0xB9, 0xCC, 0x88, + 0xCC, 0x80, 0x04, 0xCE, 0xB9, 0xCD, 0x82, 0x06, + 0xCE, 0xB9, 0xCC, 0x88, 0xCD, 0x82, 0x04, 0xCE, + // Bytes 1380 - 13bf + 0x99, 0xCC, 0x86, 0x04, 0xCE, 0x99, 0xCC, 0x84, + 0x04, 0xCE, 0x99, 0xCC, 0x80, 0x05, 0xE1, 0xBF, + 0xBE, 0xCC, 0x80, 0x05, 0x20, 0xCC, 0x94, 0xCC, + 0x80, 0x05, 0xE1, 0xBF, 0xBE, 0xCC, 0x81, 0x05, + 0x20, 0xCC, 0x94, 0xCC, 0x81, 0x05, 0xE1, 0xBF, + 0xBE, 0xCD, 0x82, 0x05, 0x20, 0xCC, 0x94, 0xCD, + 0x82, 0x04, 0xCF, 0x85, 0xCC, 0x86, 0x04, 0xCF, + 0x85, 0xCC, 0x84, 0x06, 0xCF, 0x85, 0xCC, 0x88, + // Bytes 13c0 - 13ff + 0xCC, 0x80, 0x04, 0xCF, 0x81, 0xCC, 0x93, 0x04, + 0xCF, 0x81, 0xCC, 0x94, 0x04, 0xCF, 0x85, 0xCD, + 0x82, 0x06, 0xCF, 0x85, 0xCC, 0x88, 0xCD, 0x82, + 0x04, 0xCE, 0xA5, 0xCC, 0x86, 0x04, 0xCE, 0xA5, + 0xCC, 0x84, 0x04, 0xCE, 0xA5, 0xCC, 0x80, 0x04, + 0xCE, 0xA1, 0xCC, 0x94, 0x04, 0xC2, 0xA8, 0xCC, + 0x80, 0x05, 0x20, 0xCC, 0x88, 0xCC, 0x80, 0x01, + 0x60, 0x06, 0xCF, 0x89, 0xCC, 0x80, 0xCD, 0x85, + // Bytes 1400 - 143f + 0x04, 0xCF, 0x89, 0xCD, 0x85, 0x06, 0xCF, 0x89, + 0xCC, 0x81, 0xCD, 0x85, 0x04, 0xCF, 0x89, 0xCD, + 0x82, 0x06, 0xCF, 0x89, 0xCD, 0x82, 0xCD, 0x85, + 0x04, 0xCE, 0x9F, 0xCC, 0x80, 0x04, 0xCE, 0xA9, + 0xCC, 0x80, 0x04, 0xCE, 0xA9, 0xCD, 0x85, 0x02, + 0xC2, 0xB4, 0x03, 0x20, 0xCC, 0x94, 0x03, 0xE2, + 0x80, 0x82, 0x03, 0xE2, 0x80, 0x83, 0x03, 0xE2, + 0x80, 0x90, 0x03, 0x20, 0xCC, 0xB3, 0x01, 0x2E, + // Bytes 1440 - 147f + 0x02, 0x2E, 0x2E, 0x03, 0x2E, 0x2E, 0x2E, 0x06, + 0xE2, 0x80, 0xB2, 0xE2, 0x80, 0xB2, 0x09, 0xE2, + 0x80, 0xB2, 0xE2, 0x80, 0xB2, 0xE2, 0x80, 0xB2, + 0x06, 0xE2, 0x80, 0xB5, 0xE2, 0x80, 0xB5, 0x09, + 0xE2, 0x80, 0xB5, 0xE2, 0x80, 0xB5, 0xE2, 0x80, + 0xB5, 0x02, 0x21, 0x21, 0x03, 0x20, 0xCC, 0x85, + 0x02, 0x3F, 0x3F, 0x02, 0x3F, 0x21, 0x02, 0x21, + 0x3F, 0x0C, 0xE2, 0x80, 0xB2, 0xE2, 0x80, 0xB2, + // Bytes 1480 - 14bf + 0xE2, 0x80, 0xB2, 0xE2, 0x80, 0xB2, 0x01, 0x30, + 0x01, 0x34, 0x01, 0x35, 0x01, 0x36, 0x01, 0x37, + 0x01, 0x38, 0x01, 0x39, 0x01, 0x2B, 0x03, 0xE2, + 0x88, 0x92, 0x01, 0x3D, 0x01, 0x28, 0x01, 0x29, + 0x01, 0x6E, 0x02, 0x52, 0x73, 0x03, 0x61, 0x2F, + 0x63, 0x03, 0x61, 0x2F, 0x73, 0x01, 0x43, 0x03, + 0xC2, 0xB0, 0x43, 0x03, 0x63, 0x2F, 0x6F, 0x03, + 0x63, 0x2F, 0x75, 0x02, 0xC6, 0x90, 0x03, 0xC2, + // Bytes 14c0 - 14ff + 0xB0, 0x46, 0x02, 0xC4, 0xA7, 0x02, 0x4E, 0x6F, + 0x01, 0x51, 0x02, 0x53, 0x4D, 0x03, 0x54, 0x45, + 0x4C, 0x02, 0x54, 0x4D, 0x01, 0x5A, 0x02, 0xCE, + 0xA9, 0x01, 0x46, 0x02, 0xD7, 0x90, 0x02, 0xD7, + 0x91, 0x02, 0xD7, 0x92, 0x02, 0xD7, 0x93, 0x03, + 0x46, 0x41, 0x58, 0x02, 0xCE, 0x93, 0x02, 0xCE, + 0xA0, 0x03, 0xE2, 0x88, 0x91, 0x05, 0x31, 0xE2, + 0x81, 0x84, 0x37, 0x05, 0x31, 0xE2, 0x81, 0x84, + // Bytes 1500 - 153f + 0x39, 0x06, 0x31, 0xE2, 0x81, 0x84, 0x31, 0x30, + 0x05, 0x31, 0xE2, 0x81, 0x84, 0x33, 0x05, 0x32, + 0xE2, 0x81, 0x84, 0x33, 0x05, 0x31, 0xE2, 0x81, + 0x84, 0x35, 0x05, 0x32, 0xE2, 0x81, 0x84, 0x35, + 0x05, 0x33, 0xE2, 0x81, 0x84, 0x35, 0x05, 0x34, + 0xE2, 0x81, 0x84, 0x35, 0x05, 0x31, 0xE2, 0x81, + 0x84, 0x36, 0x05, 0x35, 0xE2, 0x81, 0x84, 0x36, + 0x05, 0x31, 0xE2, 0x81, 0x84, 0x38, 0x05, 0x33, + // Bytes 1540 - 157f + 0xE2, 0x81, 0x84, 0x38, 0x05, 0x35, 0xE2, 0x81, + 0x84, 0x38, 0x05, 0x37, 0xE2, 0x81, 0x84, 0x38, + 0x04, 0x31, 0xE2, 0x81, 0x84, 0x02, 0x49, 0x49, + 0x03, 0x49, 0x49, 0x49, 0x02, 0x49, 0x56, 0x01, + 0x56, 0x02, 0x56, 0x49, 0x03, 0x56, 0x49, 0x49, + 0x04, 0x56, 0x49, 0x49, 0x49, 0x02, 0x49, 0x58, + 0x01, 0x58, 0x02, 0x58, 0x49, 0x03, 0x58, 0x49, + 0x49, 0x02, 0x69, 0x69, 0x03, 0x69, 0x69, 0x69, + // Bytes 1580 - 15bf + 0x02, 0x69, 0x76, 0x02, 0x76, 0x69, 0x03, 0x76, + 0x69, 0x69, 0x04, 0x76, 0x69, 0x69, 0x69, 0x02, + 0x69, 0x78, 0x02, 0x78, 0x69, 0x03, 0x78, 0x69, + 0x69, 0x05, 0x30, 0xE2, 0x81, 0x84, 0x33, 0x05, + 0xE2, 0x86, 0x90, 0xCC, 0xB8, 0x05, 0xE2, 0x86, + 0x92, 0xCC, 0xB8, 0x05, 0xE2, 0x86, 0x94, 0xCC, + 0xB8, 0x05, 0xE2, 0x87, 0x90, 0xCC, 0xB8, 0x05, + 0xE2, 0x87, 0x94, 0xCC, 0xB8, 0x05, 0xE2, 0x87, + // Bytes 15c0 - 15ff + 0x92, 0xCC, 0xB8, 0x05, 0xE2, 0x88, 0x83, 0xCC, + 0xB8, 0x05, 0xE2, 0x88, 0x88, 0xCC, 0xB8, 0x05, + 0xE2, 0x88, 0x8B, 0xCC, 0xB8, 0x05, 0xE2, 0x88, + 0xA3, 0xCC, 0xB8, 0x05, 0xE2, 0x88, 0xA5, 0xCC, + 0xB8, 0x06, 0xE2, 0x88, 0xAB, 0xE2, 0x88, 0xAB, + 0x09, 0xE2, 0x88, 0xAB, 0xE2, 0x88, 0xAB, 0xE2, + 0x88, 0xAB, 0x06, 0xE2, 0x88, 0xAE, 0xE2, 0x88, + 0xAE, 0x09, 0xE2, 0x88, 0xAE, 0xE2, 0x88, 0xAE, + // Bytes 1600 - 163f + 0xE2, 0x88, 0xAE, 0x05, 0xE2, 0x88, 0xBC, 0xCC, + 0xB8, 0x05, 0xE2, 0x89, 0x83, 0xCC, 0xB8, 0x05, + 0xE2, 0x89, 0x85, 0xCC, 0xB8, 0x05, 0xE2, 0x89, + 0x88, 0xCC, 0xB8, 0x03, 0x3D, 0xCC, 0xB8, 0x05, + 0xE2, 0x89, 0xA1, 0xCC, 0xB8, 0x05, 0xE2, 0x89, + 0x8D, 0xCC, 0xB8, 0x03, 0x3C, 0xCC, 0xB8, 0x03, + 0x3E, 0xCC, 0xB8, 0x05, 0xE2, 0x89, 0xA4, 0xCC, + 0xB8, 0x05, 0xE2, 0x89, 0xA5, 0xCC, 0xB8, 0x05, + // Bytes 1640 - 167f + 0xE2, 0x89, 0xB2, 0xCC, 0xB8, 0x05, 0xE2, 0x89, + 0xB3, 0xCC, 0xB8, 0x05, 0xE2, 0x89, 0xB6, 0xCC, + 0xB8, 0x05, 0xE2, 0x89, 0xB7, 0xCC, 0xB8, 0x05, + 0xE2, 0x89, 0xBA, 0xCC, 0xB8, 0x05, 0xE2, 0x89, + 0xBB, 0xCC, 0xB8, 0x05, 0xE2, 0x8A, 0x82, 0xCC, + 0xB8, 0x05, 0xE2, 0x8A, 0x83, 0xCC, 0xB8, 0x05, + 0xE2, 0x8A, 0x86, 0xCC, 0xB8, 0x05, 0xE2, 0x8A, + 0x87, 0xCC, 0xB8, 0x05, 0xE2, 0x8A, 0xA2, 0xCC, + // Bytes 1680 - 16bf + 0xB8, 0x05, 0xE2, 0x8A, 0xA8, 0xCC, 0xB8, 0x05, + 0xE2, 0x8A, 0xA9, 0xCC, 0xB8, 0x05, 0xE2, 0x8A, + 0xAB, 0xCC, 0xB8, 0x05, 0xE2, 0x89, 0xBC, 0xCC, + 0xB8, 0x05, 0xE2, 0x89, 0xBD, 0xCC, 0xB8, 0x05, + 0xE2, 0x8A, 0x91, 0xCC, 0xB8, 0x05, 0xE2, 0x8A, + 0x92, 0xCC, 0xB8, 0x05, 0xE2, 0x8A, 0xB2, 0xCC, + 0xB8, 0x05, 0xE2, 0x8A, 0xB3, 0xCC, 0xB8, 0x05, + 0xE2, 0x8A, 0xB4, 0xCC, 0xB8, 0x05, 0xE2, 0x8A, + // Bytes 16c0 - 16ff + 0xB5, 0xCC, 0xB8, 0x03, 0xE3, 0x80, 0x88, 0x03, + 0xE3, 0x80, 0x89, 0x02, 0x31, 0x30, 0x02, 0x31, + 0x31, 0x02, 0x31, 0x32, 0x02, 0x31, 0x33, 0x02, + 0x31, 0x34, 0x02, 0x31, 0x35, 0x02, 0x31, 0x36, + 0x02, 0x31, 0x37, 0x02, 0x31, 0x38, 0x02, 0x31, + 0x39, 0x02, 0x32, 0x30, 0x03, 0x28, 0x31, 0x29, + 0x03, 0x28, 0x32, 0x29, 0x03, 0x28, 0x33, 0x29, + 0x03, 0x28, 0x34, 0x29, 0x03, 0x28, 0x35, 0x29, + // Bytes 1700 - 173f + 0x03, 0x28, 0x36, 0x29, 0x03, 0x28, 0x37, 0x29, + 0x03, 0x28, 0x38, 0x29, 0x03, 0x28, 0x39, 0x29, + 0x04, 0x28, 0x31, 0x30, 0x29, 0x04, 0x28, 0x31, + 0x31, 0x29, 0x04, 0x28, 0x31, 0x32, 0x29, 0x04, + 0x28, 0x31, 0x33, 0x29, 0x04, 0x28, 0x31, 0x34, + 0x29, 0x04, 0x28, 0x31, 0x35, 0x29, 0x04, 0x28, + 0x31, 0x36, 0x29, 0x04, 0x28, 0x31, 0x37, 0x29, + 0x04, 0x28, 0x31, 0x38, 0x29, 0x04, 0x28, 0x31, + // Bytes 1740 - 177f + 0x39, 0x29, 0x04, 0x28, 0x32, 0x30, 0x29, 0x02, + 0x31, 0x2E, 0x02, 0x32, 0x2E, 0x02, 0x33, 0x2E, + 0x02, 0x34, 0x2E, 0x02, 0x35, 0x2E, 0x02, 0x36, + 0x2E, 0x02, 0x37, 0x2E, 0x02, 0x38, 0x2E, 0x02, + 0x39, 0x2E, 0x03, 0x31, 0x30, 0x2E, 0x03, 0x31, + 0x31, 0x2E, 0x03, 0x31, 0x32, 0x2E, 0x03, 0x31, + 0x33, 0x2E, 0x03, 0x31, 0x34, 0x2E, 0x03, 0x31, + 0x35, 0x2E, 0x03, 0x31, 0x36, 0x2E, 0x03, 0x31, + // Bytes 1780 - 17bf + 0x37, 0x2E, 0x03, 0x31, 0x38, 0x2E, 0x03, 0x31, + 0x39, 0x2E, 0x03, 0x32, 0x30, 0x2E, 0x03, 0x28, + 0x61, 0x29, 0x03, 0x28, 0x62, 0x29, 0x03, 0x28, + 0x63, 0x29, 0x03, 0x28, 0x64, 0x29, 0x03, 0x28, + 0x65, 0x29, 0x03, 0x28, 0x66, 0x29, 0x03, 0x28, + 0x67, 0x29, 0x03, 0x28, 0x68, 0x29, 0x03, 0x28, + 0x69, 0x29, 0x03, 0x28, 0x6A, 0x29, 0x03, 0x28, + 0x6B, 0x29, 0x03, 0x28, 0x6C, 0x29, 0x03, 0x28, + // Bytes 17c0 - 17ff + 0x6D, 0x29, 0x03, 0x28, 0x6E, 0x29, 0x03, 0x28, + 0x6F, 0x29, 0x03, 0x28, 0x70, 0x29, 0x03, 0x28, + 0x71, 0x29, 0x03, 0x28, 0x72, 0x29, 0x03, 0x28, + 0x73, 0x29, 0x03, 0x28, 0x74, 0x29, 0x03, 0x28, + 0x75, 0x29, 0x03, 0x28, 0x76, 0x29, 0x03, 0x28, + 0x77, 0x29, 0x03, 0x28, 0x78, 0x29, 0x03, 0x28, + 0x79, 0x29, 0x03, 0x28, 0x7A, 0x29, 0x01, 0x53, + 0x01, 0x59, 0x01, 0x71, 0x0C, 0xE2, 0x88, 0xAB, + // Bytes 1800 - 183f + 0xE2, 0x88, 0xAB, 0xE2, 0x88, 0xAB, 0xE2, 0x88, + 0xAB, 0x03, 0x3A, 0x3A, 0x3D, 0x02, 0x3D, 0x3D, + 0x03, 0x3D, 0x3D, 0x3D, 0x05, 0xE2, 0xAB, 0x9D, + 0xCC, 0xB8, 0x03, 0xE2, 0xB5, 0xA1, 0x03, 0xE6, + 0xAF, 0x8D, 0x03, 0xE9, 0xBE, 0x9F, 0x03, 0xE4, + 0xB8, 0x80, 0x03, 0xE4, 0xB8, 0xA8, 0x03, 0xE4, + 0xB8, 0xB6, 0x03, 0xE4, 0xB8, 0xBF, 0x03, 0xE4, + 0xB9, 0x99, 0x03, 0xE4, 0xBA, 0x85, 0x03, 0xE4, + // Bytes 1840 - 187f + 0xBA, 0x8C, 0x03, 0xE4, 0xBA, 0xA0, 0x03, 0xE4, + 0xBA, 0xBA, 0x03, 0xE5, 0x84, 0xBF, 0x03, 0xE5, + 0x85, 0xA5, 0x03, 0xE5, 0x85, 0xAB, 0x03, 0xE5, + 0x86, 0x82, 0x03, 0xE5, 0x86, 0x96, 0x03, 0xE5, + 0x86, 0xAB, 0x03, 0xE5, 0x87, 0xA0, 0x03, 0xE5, + 0x87, 0xB5, 0x03, 0xE5, 0x88, 0x80, 0x03, 0xE5, + 0x8A, 0x9B, 0x03, 0xE5, 0x8B, 0xB9, 0x03, 0xE5, + 0x8C, 0x95, 0x03, 0xE5, 0x8C, 0x9A, 0x03, 0xE5, + // Bytes 1880 - 18bf + 0x8C, 0xB8, 0x03, 0xE5, 0x8D, 0x81, 0x03, 0xE5, + 0x8D, 0x9C, 0x03, 0xE5, 0x8D, 0xA9, 0x03, 0xE5, + 0x8E, 0x82, 0x03, 0xE5, 0x8E, 0xB6, 0x03, 0xE5, + 0x8F, 0x88, 0x03, 0xE5, 0x8F, 0xA3, 0x03, 0xE5, + 0x9B, 0x97, 0x03, 0xE5, 0x9C, 0x9F, 0x03, 0xE5, + 0xA3, 0xAB, 0x03, 0xE5, 0xA4, 0x82, 0x03, 0xE5, + 0xA4, 0x8A, 0x03, 0xE5, 0xA4, 0x95, 0x03, 0xE5, + 0xA4, 0xA7, 0x03, 0xE5, 0xA5, 0xB3, 0x03, 0xE5, + // Bytes 18c0 - 18ff + 0xAD, 0x90, 0x03, 0xE5, 0xAE, 0x80, 0x03, 0xE5, + 0xAF, 0xB8, 0x03, 0xE5, 0xB0, 0x8F, 0x03, 0xE5, + 0xB0, 0xA2, 0x03, 0xE5, 0xB0, 0xB8, 0x03, 0xE5, + 0xB1, 0xAE, 0x03, 0xE5, 0xB1, 0xB1, 0x03, 0xE5, + 0xB7, 0x9B, 0x03, 0xE5, 0xB7, 0xA5, 0x03, 0xE5, + 0xB7, 0xB1, 0x03, 0xE5, 0xB7, 0xBE, 0x03, 0xE5, + 0xB9, 0xB2, 0x03, 0xE5, 0xB9, 0xBA, 0x03, 0xE5, + 0xB9, 0xBF, 0x03, 0xE5, 0xBB, 0xB4, 0x03, 0xE5, + // Bytes 1900 - 193f + 0xBB, 0xBE, 0x03, 0xE5, 0xBC, 0x8B, 0x03, 0xE5, + 0xBC, 0x93, 0x03, 0xE5, 0xBD, 0x90, 0x03, 0xE5, + 0xBD, 0xA1, 0x03, 0xE5, 0xBD, 0xB3, 0x03, 0xE5, + 0xBF, 0x83, 0x03, 0xE6, 0x88, 0x88, 0x03, 0xE6, + 0x88, 0xB6, 0x03, 0xE6, 0x89, 0x8B, 0x03, 0xE6, + 0x94, 0xAF, 0x03, 0xE6, 0x94, 0xB4, 0x03, 0xE6, + 0x96, 0x87, 0x03, 0xE6, 0x96, 0x97, 0x03, 0xE6, + 0x96, 0xA4, 0x03, 0xE6, 0x96, 0xB9, 0x03, 0xE6, + // Bytes 1940 - 197f + 0x97, 0xA0, 0x03, 0xE6, 0x97, 0xA5, 0x03, 0xE6, + 0x9B, 0xB0, 0x03, 0xE6, 0x9C, 0x88, 0x03, 0xE6, + 0x9C, 0xA8, 0x03, 0xE6, 0xAC, 0xA0, 0x03, 0xE6, + 0xAD, 0xA2, 0x03, 0xE6, 0xAD, 0xB9, 0x03, 0xE6, + 0xAE, 0xB3, 0x03, 0xE6, 0xAF, 0x8B, 0x03, 0xE6, + 0xAF, 0x94, 0x03, 0xE6, 0xAF, 0x9B, 0x03, 0xE6, + 0xB0, 0x8F, 0x03, 0xE6, 0xB0, 0x94, 0x03, 0xE6, + 0xB0, 0xB4, 0x03, 0xE7, 0x81, 0xAB, 0x03, 0xE7, + // Bytes 1980 - 19bf + 0x88, 0xAA, 0x03, 0xE7, 0x88, 0xB6, 0x03, 0xE7, + 0x88, 0xBB, 0x03, 0xE7, 0x88, 0xBF, 0x03, 0xE7, + 0x89, 0x87, 0x03, 0xE7, 0x89, 0x99, 0x03, 0xE7, + 0x89, 0x9B, 0x03, 0xE7, 0x8A, 0xAC, 0x03, 0xE7, + 0x8E, 0x84, 0x03, 0xE7, 0x8E, 0x89, 0x03, 0xE7, + 0x93, 0x9C, 0x03, 0xE7, 0x93, 0xA6, 0x03, 0xE7, + 0x94, 0x98, 0x03, 0xE7, 0x94, 0x9F, 0x03, 0xE7, + 0x94, 0xA8, 0x03, 0xE7, 0x94, 0xB0, 0x03, 0xE7, + // Bytes 19c0 - 19ff + 0x96, 0x8B, 0x03, 0xE7, 0x96, 0x92, 0x03, 0xE7, + 0x99, 0xB6, 0x03, 0xE7, 0x99, 0xBD, 0x03, 0xE7, + 0x9A, 0xAE, 0x03, 0xE7, 0x9A, 0xBF, 0x03, 0xE7, + 0x9B, 0xAE, 0x03, 0xE7, 0x9F, 0x9B, 0x03, 0xE7, + 0x9F, 0xA2, 0x03, 0xE7, 0x9F, 0xB3, 0x03, 0xE7, + 0xA4, 0xBA, 0x03, 0xE7, 0xA6, 0xB8, 0x03, 0xE7, + 0xA6, 0xBE, 0x03, 0xE7, 0xA9, 0xB4, 0x03, 0xE7, + 0xAB, 0x8B, 0x03, 0xE7, 0xAB, 0xB9, 0x03, 0xE7, + // Bytes 1a00 - 1a3f + 0xB1, 0xB3, 0x03, 0xE7, 0xB3, 0xB8, 0x03, 0xE7, + 0xBC, 0xB6, 0x03, 0xE7, 0xBD, 0x91, 0x03, 0xE7, + 0xBE, 0x8A, 0x03, 0xE7, 0xBE, 0xBD, 0x03, 0xE8, + 0x80, 0x81, 0x03, 0xE8, 0x80, 0x8C, 0x03, 0xE8, + 0x80, 0x92, 0x03, 0xE8, 0x80, 0xB3, 0x03, 0xE8, + 0x81, 0xBF, 0x03, 0xE8, 0x82, 0x89, 0x03, 0xE8, + 0x87, 0xA3, 0x03, 0xE8, 0x87, 0xAA, 0x03, 0xE8, + 0x87, 0xB3, 0x03, 0xE8, 0x87, 0xBC, 0x03, 0xE8, + // Bytes 1a40 - 1a7f + 0x88, 0x8C, 0x03, 0xE8, 0x88, 0x9B, 0x03, 0xE8, + 0x88, 0x9F, 0x03, 0xE8, 0x89, 0xAE, 0x03, 0xE8, + 0x89, 0xB2, 0x03, 0xE8, 0x89, 0xB8, 0x03, 0xE8, + 0x99, 0x8D, 0x03, 0xE8, 0x99, 0xAB, 0x03, 0xE8, + 0xA1, 0x80, 0x03, 0xE8, 0xA1, 0x8C, 0x03, 0xE8, + 0xA1, 0xA3, 0x03, 0xE8, 0xA5, 0xBE, 0x03, 0xE8, + 0xA6, 0x8B, 0x03, 0xE8, 0xA7, 0x92, 0x03, 0xE8, + 0xA8, 0x80, 0x03, 0xE8, 0xB0, 0xB7, 0x03, 0xE8, + // Bytes 1a80 - 1abf + 0xB1, 0x86, 0x03, 0xE8, 0xB1, 0x95, 0x03, 0xE8, + 0xB1, 0xB8, 0x03, 0xE8, 0xB2, 0x9D, 0x03, 0xE8, + 0xB5, 0xA4, 0x03, 0xE8, 0xB5, 0xB0, 0x03, 0xE8, + 0xB6, 0xB3, 0x03, 0xE8, 0xBA, 0xAB, 0x03, 0xE8, + 0xBB, 0x8A, 0x03, 0xE8, 0xBE, 0x9B, 0x03, 0xE8, + 0xBE, 0xB0, 0x03, 0xE8, 0xBE, 0xB5, 0x03, 0xE9, + 0x82, 0x91, 0x03, 0xE9, 0x85, 0x89, 0x03, 0xE9, + 0x87, 0x86, 0x03, 0xE9, 0x87, 0x8C, 0x03, 0xE9, + // Bytes 1ac0 - 1aff + 0x87, 0x91, 0x03, 0xE9, 0x95, 0xB7, 0x03, 0xE9, + 0x96, 0x80, 0x03, 0xE9, 0x98, 0x9C, 0x03, 0xE9, + 0x9A, 0xB6, 0x03, 0xE9, 0x9A, 0xB9, 0x03, 0xE9, + 0x9B, 0xA8, 0x03, 0xE9, 0x9D, 0x91, 0x03, 0xE9, + 0x9D, 0x9E, 0x03, 0xE9, 0x9D, 0xA2, 0x03, 0xE9, + 0x9D, 0xA9, 0x03, 0xE9, 0x9F, 0x8B, 0x03, 0xE9, + 0x9F, 0xAD, 0x03, 0xE9, 0x9F, 0xB3, 0x03, 0xE9, + 0xA0, 0x81, 0x03, 0xE9, 0xA2, 0xA8, 0x03, 0xE9, + // Bytes 1b00 - 1b3f + 0xA3, 0x9B, 0x03, 0xE9, 0xA3, 0x9F, 0x03, 0xE9, + 0xA6, 0x96, 0x03, 0xE9, 0xA6, 0x99, 0x03, 0xE9, + 0xA6, 0xAC, 0x03, 0xE9, 0xAA, 0xA8, 0x03, 0xE9, + 0xAB, 0x98, 0x03, 0xE9, 0xAB, 0x9F, 0x03, 0xE9, + 0xAC, 0xA5, 0x03, 0xE9, 0xAC, 0xAF, 0x03, 0xE9, + 0xAC, 0xB2, 0x03, 0xE9, 0xAC, 0xBC, 0x03, 0xE9, + 0xAD, 0x9A, 0x03, 0xE9, 0xB3, 0xA5, 0x03, 0xE9, + 0xB9, 0xB5, 0x03, 0xE9, 0xB9, 0xBF, 0x03, 0xE9, + // Bytes 1b40 - 1b7f + 0xBA, 0xA5, 0x03, 0xE9, 0xBA, 0xBB, 0x03, 0xE9, + 0xBB, 0x83, 0x03, 0xE9, 0xBB, 0x8D, 0x03, 0xE9, + 0xBB, 0x91, 0x03, 0xE9, 0xBB, 0xB9, 0x03, 0xE9, + 0xBB, 0xBD, 0x03, 0xE9, 0xBC, 0x8E, 0x03, 0xE9, + 0xBC, 0x93, 0x03, 0xE9, 0xBC, 0xA0, 0x03, 0xE9, + 0xBC, 0xBB, 0x03, 0xE9, 0xBD, 0x8A, 0x03, 0xE9, + 0xBD, 0x92, 0x03, 0xE9, 0xBE, 0x8D, 0x03, 0xE9, + 0xBE, 0x9C, 0x03, 0xE9, 0xBE, 0xA0, 0x03, 0xE3, + // Bytes 1b80 - 1bbf + 0x80, 0x92, 0x03, 0xE5, 0x8D, 0x84, 0x03, 0xE5, + 0x8D, 0x85, 0x06, 0xE3, 0x81, 0x8B, 0xE3, 0x82, + 0x99, 0x06, 0xE3, 0x81, 0x8D, 0xE3, 0x82, 0x99, + 0x06, 0xE3, 0x81, 0x8F, 0xE3, 0x82, 0x99, 0x06, + 0xE3, 0x81, 0x91, 0xE3, 0x82, 0x99, 0x06, 0xE3, + 0x81, 0x93, 0xE3, 0x82, 0x99, 0x06, 0xE3, 0x81, + 0x95, 0xE3, 0x82, 0x99, 0x06, 0xE3, 0x81, 0x97, + 0xE3, 0x82, 0x99, 0x06, 0xE3, 0x81, 0x99, 0xE3, + // Bytes 1bc0 - 1bff + 0x82, 0x99, 0x06, 0xE3, 0x81, 0x9B, 0xE3, 0x82, + 0x99, 0x06, 0xE3, 0x81, 0x9D, 0xE3, 0x82, 0x99, + 0x06, 0xE3, 0x81, 0x9F, 0xE3, 0x82, 0x99, 0x06, + 0xE3, 0x81, 0xA1, 0xE3, 0x82, 0x99, 0x06, 0xE3, + 0x81, 0xA4, 0xE3, 0x82, 0x99, 0x06, 0xE3, 0x81, + 0xA6, 0xE3, 0x82, 0x99, 0x06, 0xE3, 0x81, 0xA8, + 0xE3, 0x82, 0x99, 0x06, 0xE3, 0x81, 0xAF, 0xE3, + 0x82, 0x99, 0x06, 0xE3, 0x81, 0xAF, 0xE3, 0x82, + // Bytes 1c00 - 1c3f + 0x9A, 0x06, 0xE3, 0x81, 0xB2, 0xE3, 0x82, 0x99, + 0x06, 0xE3, 0x81, 0xB2, 0xE3, 0x82, 0x9A, 0x06, + 0xE3, 0x81, 0xB5, 0xE3, 0x82, 0x99, 0x06, 0xE3, + 0x81, 0xB5, 0xE3, 0x82, 0x9A, 0x06, 0xE3, 0x81, + 0xB8, 0xE3, 0x82, 0x99, 0x06, 0xE3, 0x81, 0xB8, + 0xE3, 0x82, 0x9A, 0x06, 0xE3, 0x81, 0xBB, 0xE3, + 0x82, 0x99, 0x06, 0xE3, 0x81, 0xBB, 0xE3, 0x82, + 0x9A, 0x06, 0xE3, 0x81, 0x86, 0xE3, 0x82, 0x99, + // Bytes 1c40 - 1c7f + 0x04, 0x20, 0xE3, 0x82, 0x99, 0x04, 0x20, 0xE3, + 0x82, 0x9A, 0x06, 0xE3, 0x82, 0x9D, 0xE3, 0x82, + 0x99, 0x06, 0xE3, 0x82, 0x88, 0xE3, 0x82, 0x8A, + 0x06, 0xE3, 0x82, 0xAB, 0xE3, 0x82, 0x99, 0x06, + 0xE3, 0x82, 0xAD, 0xE3, 0x82, 0x99, 0x06, 0xE3, + 0x82, 0xAF, 0xE3, 0x82, 0x99, 0x06, 0xE3, 0x82, + 0xB1, 0xE3, 0x82, 0x99, 0x06, 0xE3, 0x82, 0xB3, + 0xE3, 0x82, 0x99, 0x06, 0xE3, 0x82, 0xB5, 0xE3, + // Bytes 1c80 - 1cbf + 0x82, 0x99, 0x06, 0xE3, 0x82, 0xB7, 0xE3, 0x82, + 0x99, 0x06, 0xE3, 0x82, 0xB9, 0xE3, 0x82, 0x99, + 0x06, 0xE3, 0x82, 0xBB, 0xE3, 0x82, 0x99, 0x06, + 0xE3, 0x82, 0xBD, 0xE3, 0x82, 0x99, 0x06, 0xE3, + 0x82, 0xBF, 0xE3, 0x82, 0x99, 0x06, 0xE3, 0x83, + 0x81, 0xE3, 0x82, 0x99, 0x06, 0xE3, 0x83, 0x84, + 0xE3, 0x82, 0x99, 0x06, 0xE3, 0x83, 0x86, 0xE3, + 0x82, 0x99, 0x06, 0xE3, 0x83, 0x88, 0xE3, 0x82, + // Bytes 1cc0 - 1cff + 0x99, 0x06, 0xE3, 0x83, 0x8F, 0xE3, 0x82, 0x99, + 0x06, 0xE3, 0x83, 0x8F, 0xE3, 0x82, 0x9A, 0x06, + 0xE3, 0x83, 0x92, 0xE3, 0x82, 0x99, 0x06, 0xE3, + 0x83, 0x92, 0xE3, 0x82, 0x9A, 0x06, 0xE3, 0x83, + 0x95, 0xE3, 0x82, 0x99, 0x06, 0xE3, 0x83, 0x95, + 0xE3, 0x82, 0x9A, 0x06, 0xE3, 0x83, 0x98, 0xE3, + 0x82, 0x99, 0x06, 0xE3, 0x83, 0x98, 0xE3, 0x82, + 0x9A, 0x06, 0xE3, 0x83, 0x9B, 0xE3, 0x82, 0x99, + // Bytes 1d00 - 1d3f + 0x06, 0xE3, 0x83, 0x9B, 0xE3, 0x82, 0x9A, 0x06, + 0xE3, 0x82, 0xA6, 0xE3, 0x82, 0x99, 0x06, 0xE3, + 0x83, 0xAF, 0xE3, 0x82, 0x99, 0x06, 0xE3, 0x83, + 0xB0, 0xE3, 0x82, 0x99, 0x06, 0xE3, 0x83, 0xB1, + 0xE3, 0x82, 0x99, 0x06, 0xE3, 0x83, 0xB2, 0xE3, + 0x82, 0x99, 0x06, 0xE3, 0x83, 0xBD, 0xE3, 0x82, + 0x99, 0x06, 0xE3, 0x82, 0xB3, 0xE3, 0x83, 0x88, + 0x03, 0xE1, 0x84, 0x80, 0x03, 0xE1, 0x84, 0x81, + // Bytes 1d40 - 1d7f + 0x03, 0xE1, 0x86, 0xAA, 0x03, 0xE1, 0x84, 0x82, + 0x03, 0xE1, 0x86, 0xAC, 0x03, 0xE1, 0x86, 0xAD, + 0x03, 0xE1, 0x84, 0x83, 0x03, 0xE1, 0x84, 0x84, + 0x03, 0xE1, 0x84, 0x85, 0x03, 0xE1, 0x86, 0xB0, + 0x03, 0xE1, 0x86, 0xB1, 0x03, 0xE1, 0x86, 0xB2, + 0x03, 0xE1, 0x86, 0xB3, 0x03, 0xE1, 0x86, 0xB4, + 0x03, 0xE1, 0x86, 0xB5, 0x03, 0xE1, 0x84, 0x9A, + 0x03, 0xE1, 0x84, 0x86, 0x03, 0xE1, 0x84, 0x87, + // Bytes 1d80 - 1dbf + 0x03, 0xE1, 0x84, 0x88, 0x03, 0xE1, 0x84, 0xA1, + 0x03, 0xE1, 0x84, 0x89, 0x03, 0xE1, 0x84, 0x8A, + 0x03, 0xE1, 0x84, 0x8B, 0x03, 0xE1, 0x84, 0x8C, + 0x03, 0xE1, 0x84, 0x8D, 0x03, 0xE1, 0x84, 0x8E, + 0x03, 0xE1, 0x84, 0x8F, 0x03, 0xE1, 0x84, 0x90, + 0x03, 0xE1, 0x84, 0x91, 0x03, 0xE1, 0x84, 0x92, + 0x03, 0xE1, 0x85, 0xA1, 0x03, 0xE1, 0x85, 0xA2, + 0x03, 0xE1, 0x85, 0xA3, 0x03, 0xE1, 0x85, 0xA4, + // Bytes 1dc0 - 1dff + 0x03, 0xE1, 0x85, 0xA5, 0x03, 0xE1, 0x85, 0xA6, + 0x03, 0xE1, 0x85, 0xA7, 0x03, 0xE1, 0x85, 0xA8, + 0x03, 0xE1, 0x85, 0xA9, 0x03, 0xE1, 0x85, 0xAA, + 0x03, 0xE1, 0x85, 0xAB, 0x03, 0xE1, 0x85, 0xAC, + 0x03, 0xE1, 0x85, 0xAD, 0x03, 0xE1, 0x85, 0xAE, + 0x03, 0xE1, 0x85, 0xAF, 0x03, 0xE1, 0x85, 0xB0, + 0x03, 0xE1, 0x85, 0xB1, 0x03, 0xE1, 0x85, 0xB2, + 0x03, 0xE1, 0x85, 0xB3, 0x03, 0xE1, 0x85, 0xB4, + // Bytes 1e00 - 1e3f + 0x03, 0xE1, 0x85, 0xB5, 0x03, 0xE1, 0x85, 0xA0, + 0x03, 0xE1, 0x84, 0x94, 0x03, 0xE1, 0x84, 0x95, + 0x03, 0xE1, 0x87, 0x87, 0x03, 0xE1, 0x87, 0x88, + 0x03, 0xE1, 0x87, 0x8C, 0x03, 0xE1, 0x87, 0x8E, + 0x03, 0xE1, 0x87, 0x93, 0x03, 0xE1, 0x87, 0x97, + 0x03, 0xE1, 0x87, 0x99, 0x03, 0xE1, 0x84, 0x9C, + 0x03, 0xE1, 0x87, 0x9D, 0x03, 0xE1, 0x87, 0x9F, + 0x03, 0xE1, 0x84, 0x9D, 0x03, 0xE1, 0x84, 0x9E, + // Bytes 1e40 - 1e7f + 0x03, 0xE1, 0x84, 0xA0, 0x03, 0xE1, 0x84, 0xA2, + 0x03, 0xE1, 0x84, 0xA3, 0x03, 0xE1, 0x84, 0xA7, + 0x03, 0xE1, 0x84, 0xA9, 0x03, 0xE1, 0x84, 0xAB, + 0x03, 0xE1, 0x84, 0xAC, 0x03, 0xE1, 0x84, 0xAD, + 0x03, 0xE1, 0x84, 0xAE, 0x03, 0xE1, 0x84, 0xAF, + 0x03, 0xE1, 0x84, 0xB2, 0x03, 0xE1, 0x84, 0xB6, + 0x03, 0xE1, 0x85, 0x80, 0x03, 0xE1, 0x85, 0x87, + 0x03, 0xE1, 0x85, 0x8C, 0x03, 0xE1, 0x87, 0xB1, + // Bytes 1e80 - 1ebf + 0x03, 0xE1, 0x87, 0xB2, 0x03, 0xE1, 0x85, 0x97, + 0x03, 0xE1, 0x85, 0x98, 0x03, 0xE1, 0x85, 0x99, + 0x03, 0xE1, 0x86, 0x84, 0x03, 0xE1, 0x86, 0x85, + 0x03, 0xE1, 0x86, 0x88, 0x03, 0xE1, 0x86, 0x91, + 0x03, 0xE1, 0x86, 0x92, 0x03, 0xE1, 0x86, 0x94, + 0x03, 0xE1, 0x86, 0x9E, 0x03, 0xE1, 0x86, 0xA1, + 0x03, 0xE4, 0xB8, 0x89, 0x03, 0xE5, 0x9B, 0x9B, + 0x03, 0xE4, 0xB8, 0x8A, 0x03, 0xE4, 0xB8, 0xAD, + // Bytes 1ec0 - 1eff + 0x03, 0xE4, 0xB8, 0x8B, 0x03, 0xE7, 0x94, 0xB2, + 0x03, 0xE4, 0xB8, 0x99, 0x03, 0xE4, 0xB8, 0x81, + 0x03, 0xE5, 0xA4, 0xA9, 0x03, 0xE5, 0x9C, 0xB0, + 0x05, 0x28, 0xE1, 0x84, 0x80, 0x29, 0x05, 0x28, + 0xE1, 0x84, 0x82, 0x29, 0x05, 0x28, 0xE1, 0x84, + 0x83, 0x29, 0x05, 0x28, 0xE1, 0x84, 0x85, 0x29, + 0x05, 0x28, 0xE1, 0x84, 0x86, 0x29, 0x05, 0x28, + 0xE1, 0x84, 0x87, 0x29, 0x05, 0x28, 0xE1, 0x84, + // Bytes 1f00 - 1f3f + 0x89, 0x29, 0x05, 0x28, 0xE1, 0x84, 0x8B, 0x29, + 0x05, 0x28, 0xE1, 0x84, 0x8C, 0x29, 0x05, 0x28, + 0xE1, 0x84, 0x8E, 0x29, 0x05, 0x28, 0xE1, 0x84, + 0x8F, 0x29, 0x05, 0x28, 0xE1, 0x84, 0x90, 0x29, + 0x05, 0x28, 0xE1, 0x84, 0x91, 0x29, 0x05, 0x28, + 0xE1, 0x84, 0x92, 0x29, 0x08, 0x28, 0xE1, 0x84, + 0x80, 0xE1, 0x85, 0xA1, 0x29, 0x08, 0x28, 0xE1, + 0x84, 0x82, 0xE1, 0x85, 0xA1, 0x29, 0x08, 0x28, + // Bytes 1f40 - 1f7f + 0xE1, 0x84, 0x83, 0xE1, 0x85, 0xA1, 0x29, 0x08, + 0x28, 0xE1, 0x84, 0x85, 0xE1, 0x85, 0xA1, 0x29, + 0x08, 0x28, 0xE1, 0x84, 0x86, 0xE1, 0x85, 0xA1, + 0x29, 0x08, 0x28, 0xE1, 0x84, 0x87, 0xE1, 0x85, + 0xA1, 0x29, 0x08, 0x28, 0xE1, 0x84, 0x89, 0xE1, + 0x85, 0xA1, 0x29, 0x08, 0x28, 0xE1, 0x84, 0x8B, + 0xE1, 0x85, 0xA1, 0x29, 0x08, 0x28, 0xE1, 0x84, + 0x8C, 0xE1, 0x85, 0xA1, 0x29, 0x08, 0x28, 0xE1, + // Bytes 1f80 - 1fbf + 0x84, 0x8E, 0xE1, 0x85, 0xA1, 0x29, 0x08, 0x28, + 0xE1, 0x84, 0x8F, 0xE1, 0x85, 0xA1, 0x29, 0x08, + 0x28, 0xE1, 0x84, 0x90, 0xE1, 0x85, 0xA1, 0x29, + 0x08, 0x28, 0xE1, 0x84, 0x91, 0xE1, 0x85, 0xA1, + 0x29, 0x08, 0x28, 0xE1, 0x84, 0x92, 0xE1, 0x85, + 0xA1, 0x29, 0x08, 0x28, 0xE1, 0x84, 0x8C, 0xE1, + 0x85, 0xAE, 0x29, 0x11, 0x28, 0xE1, 0x84, 0x8B, + 0xE1, 0x85, 0xA9, 0xE1, 0x84, 0x8C, 0xE1, 0x85, + // Bytes 1fc0 - 1fff + 0xA5, 0xE1, 0x86, 0xAB, 0x29, 0x0E, 0x28, 0xE1, + 0x84, 0x8B, 0xE1, 0x85, 0xA9, 0xE1, 0x84, 0x92, + 0xE1, 0x85, 0xAE, 0x29, 0x05, 0x28, 0xE4, 0xB8, + 0x80, 0x29, 0x05, 0x28, 0xE4, 0xBA, 0x8C, 0x29, + 0x05, 0x28, 0xE4, 0xB8, 0x89, 0x29, 0x05, 0x28, + 0xE5, 0x9B, 0x9B, 0x29, 0x05, 0x28, 0xE4, 0xBA, + 0x94, 0x29, 0x05, 0x28, 0xE5, 0x85, 0xAD, 0x29, + 0x05, 0x28, 0xE4, 0xB8, 0x83, 0x29, 0x05, 0x28, + // Bytes 2000 - 203f + 0xE5, 0x85, 0xAB, 0x29, 0x05, 0x28, 0xE4, 0xB9, + 0x9D, 0x29, 0x05, 0x28, 0xE5, 0x8D, 0x81, 0x29, + 0x05, 0x28, 0xE6, 0x9C, 0x88, 0x29, 0x05, 0x28, + 0xE7, 0x81, 0xAB, 0x29, 0x05, 0x28, 0xE6, 0xB0, + 0xB4, 0x29, 0x05, 0x28, 0xE6, 0x9C, 0xA8, 0x29, + 0x05, 0x28, 0xE9, 0x87, 0x91, 0x29, 0x05, 0x28, + 0xE5, 0x9C, 0x9F, 0x29, 0x05, 0x28, 0xE6, 0x97, + 0xA5, 0x29, 0x05, 0x28, 0xE6, 0xA0, 0xAA, 0x29, + // Bytes 2040 - 207f + 0x05, 0x28, 0xE6, 0x9C, 0x89, 0x29, 0x05, 0x28, + 0xE7, 0xA4, 0xBE, 0x29, 0x05, 0x28, 0xE5, 0x90, + 0x8D, 0x29, 0x05, 0x28, 0xE7, 0x89, 0xB9, 0x29, + 0x05, 0x28, 0xE8, 0xB2, 0xA1, 0x29, 0x05, 0x28, + 0xE7, 0xA5, 0x9D, 0x29, 0x05, 0x28, 0xE5, 0x8A, + 0xB4, 0x29, 0x05, 0x28, 0xE4, 0xBB, 0xA3, 0x29, + 0x05, 0x28, 0xE5, 0x91, 0xBC, 0x29, 0x05, 0x28, + 0xE5, 0xAD, 0xA6, 0x29, 0x05, 0x28, 0xE7, 0x9B, + // Bytes 2080 - 20bf + 0xA3, 0x29, 0x05, 0x28, 0xE4, 0xBC, 0x81, 0x29, + 0x05, 0x28, 0xE8, 0xB3, 0x87, 0x29, 0x05, 0x28, + 0xE5, 0x8D, 0x94, 0x29, 0x05, 0x28, 0xE7, 0xA5, + 0xAD, 0x29, 0x05, 0x28, 0xE4, 0xBC, 0x91, 0x29, + 0x05, 0x28, 0xE8, 0x87, 0xAA, 0x29, 0x05, 0x28, + 0xE8, 0x87, 0xB3, 0x29, 0x03, 0xE5, 0x95, 0x8F, + 0x03, 0xE5, 0xB9, 0xBC, 0x03, 0xE7, 0xAE, 0x8F, + 0x03, 0x50, 0x54, 0x45, 0x02, 0x32, 0x31, 0x02, + // Bytes 20c0 - 20ff + 0x32, 0x32, 0x02, 0x32, 0x33, 0x02, 0x32, 0x34, + 0x02, 0x32, 0x35, 0x02, 0x32, 0x36, 0x02, 0x32, + 0x37, 0x02, 0x32, 0x38, 0x02, 0x32, 0x39, 0x02, + 0x33, 0x30, 0x02, 0x33, 0x31, 0x02, 0x33, 0x32, + 0x02, 0x33, 0x33, 0x02, 0x33, 0x34, 0x02, 0x33, + 0x35, 0x06, 0xE1, 0x84, 0x80, 0xE1, 0x85, 0xA1, + 0x06, 0xE1, 0x84, 0x82, 0xE1, 0x85, 0xA1, 0x06, + 0xE1, 0x84, 0x83, 0xE1, 0x85, 0xA1, 0x06, 0xE1, + // Bytes 2100 - 213f + 0x84, 0x85, 0xE1, 0x85, 0xA1, 0x06, 0xE1, 0x84, + 0x86, 0xE1, 0x85, 0xA1, 0x06, 0xE1, 0x84, 0x87, + 0xE1, 0x85, 0xA1, 0x06, 0xE1, 0x84, 0x89, 0xE1, + 0x85, 0xA1, 0x06, 0xE1, 0x84, 0x8B, 0xE1, 0x85, + 0xA1, 0x06, 0xE1, 0x84, 0x8C, 0xE1, 0x85, 0xA1, + 0x06, 0xE1, 0x84, 0x8E, 0xE1, 0x85, 0xA1, 0x06, + 0xE1, 0x84, 0x8F, 0xE1, 0x85, 0xA1, 0x06, 0xE1, + 0x84, 0x90, 0xE1, 0x85, 0xA1, 0x06, 0xE1, 0x84, + // Bytes 2140 - 217f + 0x91, 0xE1, 0x85, 0xA1, 0x06, 0xE1, 0x84, 0x92, + 0xE1, 0x85, 0xA1, 0x0F, 0xE1, 0x84, 0x8E, 0xE1, + 0x85, 0xA1, 0xE1, 0x86, 0xB7, 0xE1, 0x84, 0x80, + 0xE1, 0x85, 0xA9, 0x0C, 0xE1, 0x84, 0x8C, 0xE1, + 0x85, 0xAE, 0xE1, 0x84, 0x8B, 0xE1, 0x85, 0xB4, + 0x06, 0xE1, 0x84, 0x8B, 0xE1, 0x85, 0xAE, 0x03, + 0xE4, 0xBA, 0x94, 0x03, 0xE5, 0x85, 0xAD, 0x03, + 0xE4, 0xB8, 0x83, 0x03, 0xE4, 0xB9, 0x9D, 0x03, + // Bytes 2180 - 21bf + 0xE6, 0xA0, 0xAA, 0x03, 0xE6, 0x9C, 0x89, 0x03, + 0xE7, 0xA4, 0xBE, 0x03, 0xE5, 0x90, 0x8D, 0x03, + 0xE7, 0x89, 0xB9, 0x03, 0xE8, 0xB2, 0xA1, 0x03, + 0xE7, 0xA5, 0x9D, 0x03, 0xE5, 0x8A, 0xB4, 0x03, + 0xE7, 0xA7, 0x98, 0x03, 0xE7, 0x94, 0xB7, 0x03, + 0xE9, 0x81, 0xA9, 0x03, 0xE5, 0x84, 0xAA, 0x03, + 0xE5, 0x8D, 0xB0, 0x03, 0xE6, 0xB3, 0xA8, 0x03, + 0xE9, 0xA0, 0x85, 0x03, 0xE4, 0xBC, 0x91, 0x03, + // Bytes 21c0 - 21ff + 0xE5, 0x86, 0x99, 0x03, 0xE6, 0xAD, 0xA3, 0x03, + 0xE5, 0xB7, 0xA6, 0x03, 0xE5, 0x8F, 0xB3, 0x03, + 0xE5, 0x8C, 0xBB, 0x03, 0xE5, 0xAE, 0x97, 0x03, + 0xE5, 0xAD, 0xA6, 0x03, 0xE7, 0x9B, 0xA3, 0x03, + 0xE4, 0xBC, 0x81, 0x03, 0xE8, 0xB3, 0x87, 0x03, + 0xE5, 0x8D, 0x94, 0x03, 0xE5, 0xA4, 0x9C, 0x02, + 0x33, 0x36, 0x02, 0x33, 0x37, 0x02, 0x33, 0x38, + 0x02, 0x33, 0x39, 0x02, 0x34, 0x30, 0x02, 0x34, + // Bytes 2200 - 223f + 0x31, 0x02, 0x34, 0x32, 0x02, 0x34, 0x33, 0x02, + 0x34, 0x34, 0x02, 0x34, 0x35, 0x02, 0x34, 0x36, + 0x02, 0x34, 0x37, 0x02, 0x34, 0x38, 0x02, 0x34, + 0x39, 0x02, 0x35, 0x30, 0x04, 0x31, 0xE6, 0x9C, + 0x88, 0x04, 0x32, 0xE6, 0x9C, 0x88, 0x04, 0x33, + 0xE6, 0x9C, 0x88, 0x04, 0x34, 0xE6, 0x9C, 0x88, + 0x04, 0x35, 0xE6, 0x9C, 0x88, 0x04, 0x36, 0xE6, + 0x9C, 0x88, 0x04, 0x37, 0xE6, 0x9C, 0x88, 0x04, + // Bytes 2240 - 227f + 0x38, 0xE6, 0x9C, 0x88, 0x04, 0x39, 0xE6, 0x9C, + 0x88, 0x05, 0x31, 0x30, 0xE6, 0x9C, 0x88, 0x05, + 0x31, 0x31, 0xE6, 0x9C, 0x88, 0x05, 0x31, 0x32, + 0xE6, 0x9C, 0x88, 0x02, 0x48, 0x67, 0x03, 0x65, + 0x72, 0x67, 0x02, 0x65, 0x56, 0x03, 0x4C, 0x54, + 0x44, 0x03, 0xE3, 0x82, 0xA2, 0x03, 0xE3, 0x82, + 0xA4, 0x03, 0xE3, 0x82, 0xA6, 0x03, 0xE3, 0x82, + 0xA8, 0x03, 0xE3, 0x82, 0xAA, 0x03, 0xE3, 0x82, + // Bytes 2280 - 22bf + 0xAB, 0x03, 0xE3, 0x82, 0xAD, 0x03, 0xE3, 0x82, + 0xAF, 0x03, 0xE3, 0x82, 0xB1, 0x03, 0xE3, 0x82, + 0xB3, 0x03, 0xE3, 0x82, 0xB5, 0x03, 0xE3, 0x82, + 0xB7, 0x03, 0xE3, 0x82, 0xB9, 0x03, 0xE3, 0x82, + 0xBB, 0x03, 0xE3, 0x82, 0xBD, 0x03, 0xE3, 0x82, + 0xBF, 0x03, 0xE3, 0x83, 0x81, 0x03, 0xE3, 0x83, + 0x84, 0x03, 0xE3, 0x83, 0x86, 0x03, 0xE3, 0x83, + 0x88, 0x03, 0xE3, 0x83, 0x8A, 0x03, 0xE3, 0x83, + // Bytes 22c0 - 22ff + 0x8B, 0x03, 0xE3, 0x83, 0x8C, 0x03, 0xE3, 0x83, + 0x8D, 0x03, 0xE3, 0x83, 0x8E, 0x03, 0xE3, 0x83, + 0x8F, 0x03, 0xE3, 0x83, 0x92, 0x03, 0xE3, 0x83, + 0x95, 0x03, 0xE3, 0x83, 0x98, 0x03, 0xE3, 0x83, + 0x9B, 0x03, 0xE3, 0x83, 0x9E, 0x03, 0xE3, 0x83, + 0x9F, 0x03, 0xE3, 0x83, 0xA0, 0x03, 0xE3, 0x83, + 0xA1, 0x03, 0xE3, 0x83, 0xA2, 0x03, 0xE3, 0x83, + 0xA4, 0x03, 0xE3, 0x83, 0xA6, 0x03, 0xE3, 0x83, + // Bytes 2300 - 233f + 0xA8, 0x03, 0xE3, 0x83, 0xA9, 0x03, 0xE3, 0x83, + 0xAA, 0x03, 0xE3, 0x83, 0xAB, 0x03, 0xE3, 0x83, + 0xAC, 0x03, 0xE3, 0x83, 0xAD, 0x03, 0xE3, 0x83, + 0xAF, 0x03, 0xE3, 0x83, 0xB0, 0x03, 0xE3, 0x83, + 0xB1, 0x03, 0xE3, 0x83, 0xB2, 0x0F, 0xE3, 0x82, + 0xA2, 0xE3, 0x83, 0x8F, 0xE3, 0x82, 0x9A, 0xE3, + 0x83, 0xBC, 0xE3, 0x83, 0x88, 0x0C, 0xE3, 0x82, + 0xA2, 0xE3, 0x83, 0xAB, 0xE3, 0x83, 0x95, 0xE3, + // Bytes 2340 - 237f + 0x82, 0xA1, 0x0F, 0xE3, 0x82, 0xA2, 0xE3, 0x83, + 0xB3, 0xE3, 0x83, 0x98, 0xE3, 0x82, 0x9A, 0xE3, + 0x82, 0xA2, 0x09, 0xE3, 0x82, 0xA2, 0xE3, 0x83, + 0xBC, 0xE3, 0x83, 0xAB, 0x0F, 0xE3, 0x82, 0xA4, + 0xE3, 0x83, 0x8B, 0xE3, 0x83, 0xB3, 0xE3, 0x82, + 0xAF, 0xE3, 0x82, 0x99, 0x09, 0xE3, 0x82, 0xA4, + 0xE3, 0x83, 0xB3, 0xE3, 0x83, 0x81, 0x09, 0xE3, + 0x82, 0xA6, 0xE3, 0x82, 0xA9, 0xE3, 0x83, 0xB3, + // Bytes 2380 - 23bf + 0x12, 0xE3, 0x82, 0xA8, 0xE3, 0x82, 0xB9, 0xE3, + 0x82, 0xAF, 0xE3, 0x83, 0xBC, 0xE3, 0x83, 0x88, + 0xE3, 0x82, 0x99, 0x0C, 0xE3, 0x82, 0xA8, 0xE3, + 0x83, 0xBC, 0xE3, 0x82, 0xAB, 0xE3, 0x83, 0xBC, + 0x09, 0xE3, 0x82, 0xAA, 0xE3, 0x83, 0xB3, 0xE3, + 0x82, 0xB9, 0x09, 0xE3, 0x82, 0xAA, 0xE3, 0x83, + 0xBC, 0xE3, 0x83, 0xA0, 0x09, 0xE3, 0x82, 0xAB, + 0xE3, 0x82, 0xA4, 0xE3, 0x83, 0xAA, 0x0C, 0xE3, + // Bytes 23c0 - 23ff + 0x82, 0xAB, 0xE3, 0x83, 0xA9, 0xE3, 0x83, 0x83, + 0xE3, 0x83, 0x88, 0x0C, 0xE3, 0x82, 0xAB, 0xE3, + 0x83, 0xAD, 0xE3, 0x83, 0xAA, 0xE3, 0x83, 0xBC, + 0x0C, 0xE3, 0x82, 0xAB, 0xE3, 0x82, 0x99, 0xE3, + 0x83, 0xAD, 0xE3, 0x83, 0xB3, 0x0C, 0xE3, 0x82, + 0xAB, 0xE3, 0x82, 0x99, 0xE3, 0x83, 0xB3, 0xE3, + 0x83, 0x9E, 0x0C, 0xE3, 0x82, 0xAD, 0xE3, 0x82, + 0x99, 0xE3, 0x82, 0xAB, 0xE3, 0x82, 0x99, 0x0C, + // Bytes 2400 - 243f + 0xE3, 0x82, 0xAD, 0xE3, 0x82, 0x99, 0xE3, 0x83, + 0x8B, 0xE3, 0x83, 0xBC, 0x0C, 0xE3, 0x82, 0xAD, + 0xE3, 0x83, 0xA5, 0xE3, 0x83, 0xAA, 0xE3, 0x83, + 0xBC, 0x12, 0xE3, 0x82, 0xAD, 0xE3, 0x82, 0x99, + 0xE3, 0x83, 0xAB, 0xE3, 0x82, 0xBF, 0xE3, 0x82, + 0x99, 0xE3, 0x83, 0xBC, 0x06, 0xE3, 0x82, 0xAD, + 0xE3, 0x83, 0xAD, 0x12, 0xE3, 0x82, 0xAD, 0xE3, + 0x83, 0xAD, 0xE3, 0x82, 0xAF, 0xE3, 0x82, 0x99, + // Bytes 2440 - 247f + 0xE3, 0x83, 0xA9, 0xE3, 0x83, 0xA0, 0x12, 0xE3, + 0x82, 0xAD, 0xE3, 0x83, 0xAD, 0xE3, 0x83, 0xA1, + 0xE3, 0x83, 0xBC, 0xE3, 0x83, 0x88, 0xE3, 0x83, + 0xAB, 0x0F, 0xE3, 0x82, 0xAD, 0xE3, 0x83, 0xAD, + 0xE3, 0x83, 0xAF, 0xE3, 0x83, 0x83, 0xE3, 0x83, + 0x88, 0x0C, 0xE3, 0x82, 0xAF, 0xE3, 0x82, 0x99, + 0xE3, 0x83, 0xA9, 0xE3, 0x83, 0xA0, 0x12, 0xE3, + 0x82, 0xAF, 0xE3, 0x82, 0x99, 0xE3, 0x83, 0xA9, + // Bytes 2480 - 24bf + 0xE3, 0x83, 0xA0, 0xE3, 0x83, 0x88, 0xE3, 0x83, + 0xB3, 0x12, 0xE3, 0x82, 0xAF, 0xE3, 0x83, 0xAB, + 0xE3, 0x82, 0xBB, 0xE3, 0x82, 0x99, 0xE3, 0x82, + 0xA4, 0xE3, 0x83, 0xAD, 0x0C, 0xE3, 0x82, 0xAF, + 0xE3, 0x83, 0xAD, 0xE3, 0x83, 0xBC, 0xE3, 0x83, + 0x8D, 0x09, 0xE3, 0x82, 0xB1, 0xE3, 0x83, 0xBC, + 0xE3, 0x82, 0xB9, 0x09, 0xE3, 0x82, 0xB3, 0xE3, + 0x83, 0xAB, 0xE3, 0x83, 0x8A, 0x0C, 0xE3, 0x82, + // Bytes 24c0 - 24ff + 0xB3, 0xE3, 0x83, 0xBC, 0xE3, 0x83, 0x9B, 0xE3, + 0x82, 0x9A, 0x0C, 0xE3, 0x82, 0xB5, 0xE3, 0x82, + 0xA4, 0xE3, 0x82, 0xAF, 0xE3, 0x83, 0xAB, 0x0F, + 0xE3, 0x82, 0xB5, 0xE3, 0x83, 0xB3, 0xE3, 0x83, + 0x81, 0xE3, 0x83, 0xBC, 0xE3, 0x83, 0xA0, 0x0F, + 0xE3, 0x82, 0xB7, 0xE3, 0x83, 0xAA, 0xE3, 0x83, + 0xB3, 0xE3, 0x82, 0xAF, 0xE3, 0x82, 0x99, 0x09, + 0xE3, 0x82, 0xBB, 0xE3, 0x83, 0xB3, 0xE3, 0x83, + // Bytes 2500 - 253f + 0x81, 0x09, 0xE3, 0x82, 0xBB, 0xE3, 0x83, 0xB3, + 0xE3, 0x83, 0x88, 0x0C, 0xE3, 0x82, 0xBF, 0xE3, + 0x82, 0x99, 0xE3, 0x83, 0xBC, 0xE3, 0x82, 0xB9, + 0x09, 0xE3, 0x83, 0x86, 0xE3, 0x82, 0x99, 0xE3, + 0x82, 0xB7, 0x09, 0xE3, 0x83, 0x88, 0xE3, 0x82, + 0x99, 0xE3, 0x83, 0xAB, 0x06, 0xE3, 0x83, 0x88, + 0xE3, 0x83, 0xB3, 0x06, 0xE3, 0x83, 0x8A, 0xE3, + 0x83, 0x8E, 0x09, 0xE3, 0x83, 0x8E, 0xE3, 0x83, + // Bytes 2540 - 257f + 0x83, 0xE3, 0x83, 0x88, 0x09, 0xE3, 0x83, 0x8F, + 0xE3, 0x82, 0xA4, 0xE3, 0x83, 0x84, 0x12, 0xE3, + 0x83, 0x8F, 0xE3, 0x82, 0x9A, 0xE3, 0x83, 0xBC, + 0xE3, 0x82, 0xBB, 0xE3, 0x83, 0xB3, 0xE3, 0x83, + 0x88, 0x0C, 0xE3, 0x83, 0x8F, 0xE3, 0x82, 0x9A, + 0xE3, 0x83, 0xBC, 0xE3, 0x83, 0x84, 0x0F, 0xE3, + 0x83, 0x8F, 0xE3, 0x82, 0x99, 0xE3, 0x83, 0xBC, + 0xE3, 0x83, 0xAC, 0xE3, 0x83, 0xAB, 0x12, 0xE3, + // Bytes 2580 - 25bf + 0x83, 0x92, 0xE3, 0x82, 0x9A, 0xE3, 0x82, 0xA2, + 0xE3, 0x82, 0xB9, 0xE3, 0x83, 0x88, 0xE3, 0x83, + 0xAB, 0x0C, 0xE3, 0x83, 0x92, 0xE3, 0x82, 0x9A, + 0xE3, 0x82, 0xAF, 0xE3, 0x83, 0xAB, 0x09, 0xE3, + 0x83, 0x92, 0xE3, 0x82, 0x9A, 0xE3, 0x82, 0xB3, + 0x09, 0xE3, 0x83, 0x92, 0xE3, 0x82, 0x99, 0xE3, + 0x83, 0xAB, 0x12, 0xE3, 0x83, 0x95, 0xE3, 0x82, + 0xA1, 0xE3, 0x83, 0xA9, 0xE3, 0x83, 0x83, 0xE3, + // Bytes 25c0 - 25ff + 0x83, 0x88, 0xE3, 0x82, 0x99, 0x0C, 0xE3, 0x83, + 0x95, 0xE3, 0x82, 0xA3, 0xE3, 0x83, 0xBC, 0xE3, + 0x83, 0x88, 0x12, 0xE3, 0x83, 0x95, 0xE3, 0x82, + 0x99, 0xE3, 0x83, 0x83, 0xE3, 0x82, 0xB7, 0xE3, + 0x82, 0xA7, 0xE3, 0x83, 0xAB, 0x09, 0xE3, 0x83, + 0x95, 0xE3, 0x83, 0xA9, 0xE3, 0x83, 0xB3, 0x0F, + 0xE3, 0x83, 0x98, 0xE3, 0x82, 0xAF, 0xE3, 0x82, + 0xBF, 0xE3, 0x83, 0xBC, 0xE3, 0x83, 0xAB, 0x09, + // Bytes 2600 - 263f + 0xE3, 0x83, 0x98, 0xE3, 0x82, 0x9A, 0xE3, 0x82, + 0xBD, 0x0C, 0xE3, 0x83, 0x98, 0xE3, 0x82, 0x9A, + 0xE3, 0x83, 0x8B, 0xE3, 0x83, 0x92, 0x09, 0xE3, + 0x83, 0x98, 0xE3, 0x83, 0xAB, 0xE3, 0x83, 0x84, + 0x0C, 0xE3, 0x83, 0x98, 0xE3, 0x82, 0x9A, 0xE3, + 0x83, 0xB3, 0xE3, 0x82, 0xB9, 0x0F, 0xE3, 0x83, + 0x98, 0xE3, 0x82, 0x9A, 0xE3, 0x83, 0xBC, 0xE3, + 0x82, 0xB7, 0xE3, 0x82, 0x99, 0x0C, 0xE3, 0x83, + // Bytes 2640 - 267f + 0x98, 0xE3, 0x82, 0x99, 0xE3, 0x83, 0xBC, 0xE3, + 0x82, 0xBF, 0x0F, 0xE3, 0x83, 0x9B, 0xE3, 0x82, + 0x9A, 0xE3, 0x82, 0xA4, 0xE3, 0x83, 0xB3, 0xE3, + 0x83, 0x88, 0x0C, 0xE3, 0x83, 0x9B, 0xE3, 0x82, + 0x99, 0xE3, 0x83, 0xAB, 0xE3, 0x83, 0x88, 0x06, + 0xE3, 0x83, 0x9B, 0xE3, 0x83, 0xB3, 0x0F, 0xE3, + 0x83, 0x9B, 0xE3, 0x82, 0x9A, 0xE3, 0x83, 0xB3, + 0xE3, 0x83, 0x88, 0xE3, 0x82, 0x99, 0x09, 0xE3, + // Bytes 2680 - 26bf + 0x83, 0x9B, 0xE3, 0x83, 0xBC, 0xE3, 0x83, 0xAB, + 0x09, 0xE3, 0x83, 0x9B, 0xE3, 0x83, 0xBC, 0xE3, + 0x83, 0xB3, 0x0C, 0xE3, 0x83, 0x9E, 0xE3, 0x82, + 0xA4, 0xE3, 0x82, 0xAF, 0xE3, 0x83, 0xAD, 0x09, + 0xE3, 0x83, 0x9E, 0xE3, 0x82, 0xA4, 0xE3, 0x83, + 0xAB, 0x09, 0xE3, 0x83, 0x9E, 0xE3, 0x83, 0x83, + 0xE3, 0x83, 0x8F, 0x09, 0xE3, 0x83, 0x9E, 0xE3, + 0x83, 0xAB, 0xE3, 0x82, 0xAF, 0x0F, 0xE3, 0x83, + // Bytes 26c0 - 26ff + 0x9E, 0xE3, 0x83, 0xB3, 0xE3, 0x82, 0xB7, 0xE3, + 0x83, 0xA7, 0xE3, 0x83, 0xB3, 0x0C, 0xE3, 0x83, + 0x9F, 0xE3, 0x82, 0xAF, 0xE3, 0x83, 0xAD, 0xE3, + 0x83, 0xB3, 0x06, 0xE3, 0x83, 0x9F, 0xE3, 0x83, + 0xAA, 0x12, 0xE3, 0x83, 0x9F, 0xE3, 0x83, 0xAA, + 0xE3, 0x83, 0x8F, 0xE3, 0x82, 0x99, 0xE3, 0x83, + 0xBC, 0xE3, 0x83, 0xAB, 0x09, 0xE3, 0x83, 0xA1, + 0xE3, 0x82, 0xAB, 0xE3, 0x82, 0x99, 0x0F, 0xE3, + // Bytes 2700 - 273f + 0x83, 0xA1, 0xE3, 0x82, 0xAB, 0xE3, 0x82, 0x99, + 0xE3, 0x83, 0x88, 0xE3, 0x83, 0xB3, 0x0C, 0xE3, + 0x83, 0xA1, 0xE3, 0x83, 0xBC, 0xE3, 0x83, 0x88, + 0xE3, 0x83, 0xAB, 0x0C, 0xE3, 0x83, 0xA4, 0xE3, + 0x83, 0xBC, 0xE3, 0x83, 0x88, 0xE3, 0x82, 0x99, + 0x09, 0xE3, 0x83, 0xA4, 0xE3, 0x83, 0xBC, 0xE3, + 0x83, 0xAB, 0x09, 0xE3, 0x83, 0xA6, 0xE3, 0x82, + 0xA2, 0xE3, 0x83, 0xB3, 0x0C, 0xE3, 0x83, 0xAA, + // Bytes 2740 - 277f + 0xE3, 0x83, 0x83, 0xE3, 0x83, 0x88, 0xE3, 0x83, + 0xAB, 0x06, 0xE3, 0x83, 0xAA, 0xE3, 0x83, 0xA9, + 0x0C, 0xE3, 0x83, 0xAB, 0xE3, 0x83, 0x92, 0xE3, + 0x82, 0x9A, 0xE3, 0x83, 0xBC, 0x0F, 0xE3, 0x83, + 0xAB, 0xE3, 0x83, 0xBC, 0xE3, 0x83, 0x95, 0xE3, + 0x82, 0x99, 0xE3, 0x83, 0xAB, 0x06, 0xE3, 0x83, + 0xAC, 0xE3, 0x83, 0xA0, 0x12, 0xE3, 0x83, 0xAC, + 0xE3, 0x83, 0xB3, 0xE3, 0x83, 0x88, 0xE3, 0x82, + // Bytes 2780 - 27bf + 0xB1, 0xE3, 0x82, 0x99, 0xE3, 0x83, 0xB3, 0x09, + 0xE3, 0x83, 0xAF, 0xE3, 0x83, 0x83, 0xE3, 0x83, + 0x88, 0x04, 0x30, 0xE7, 0x82, 0xB9, 0x04, 0x31, + 0xE7, 0x82, 0xB9, 0x04, 0x32, 0xE7, 0x82, 0xB9, + 0x04, 0x33, 0xE7, 0x82, 0xB9, 0x04, 0x34, 0xE7, + 0x82, 0xB9, 0x04, 0x35, 0xE7, 0x82, 0xB9, 0x04, + 0x36, 0xE7, 0x82, 0xB9, 0x04, 0x37, 0xE7, 0x82, + 0xB9, 0x04, 0x38, 0xE7, 0x82, 0xB9, 0x04, 0x39, + // Bytes 27c0 - 27ff + 0xE7, 0x82, 0xB9, 0x05, 0x31, 0x30, 0xE7, 0x82, + 0xB9, 0x05, 0x31, 0x31, 0xE7, 0x82, 0xB9, 0x05, + 0x31, 0x32, 0xE7, 0x82, 0xB9, 0x05, 0x31, 0x33, + 0xE7, 0x82, 0xB9, 0x05, 0x31, 0x34, 0xE7, 0x82, + 0xB9, 0x05, 0x31, 0x35, 0xE7, 0x82, 0xB9, 0x05, + 0x31, 0x36, 0xE7, 0x82, 0xB9, 0x05, 0x31, 0x37, + 0xE7, 0x82, 0xB9, 0x05, 0x31, 0x38, 0xE7, 0x82, + 0xB9, 0x05, 0x31, 0x39, 0xE7, 0x82, 0xB9, 0x05, + // Bytes 2800 - 283f + 0x32, 0x30, 0xE7, 0x82, 0xB9, 0x05, 0x32, 0x31, + 0xE7, 0x82, 0xB9, 0x05, 0x32, 0x32, 0xE7, 0x82, + 0xB9, 0x05, 0x32, 0x33, 0xE7, 0x82, 0xB9, 0x05, + 0x32, 0x34, 0xE7, 0x82, 0xB9, 0x03, 0x68, 0x50, + 0x61, 0x02, 0x64, 0x61, 0x02, 0x41, 0x55, 0x03, + 0x62, 0x61, 0x72, 0x02, 0x6F, 0x56, 0x02, 0x70, + 0x63, 0x02, 0x64, 0x6D, 0x03, 0x64, 0x6D, 0x32, + 0x03, 0x64, 0x6D, 0x33, 0x02, 0x49, 0x55, 0x06, + // Bytes 2840 - 287f + 0xE5, 0xB9, 0xB3, 0xE6, 0x88, 0x90, 0x06, 0xE6, + 0x98, 0xAD, 0xE5, 0x92, 0x8C, 0x06, 0xE5, 0xA4, + 0xA7, 0xE6, 0xAD, 0xA3, 0x06, 0xE6, 0x98, 0x8E, + 0xE6, 0xB2, 0xBB, 0x0C, 0xE6, 0xA0, 0xAA, 0xE5, + 0xBC, 0x8F, 0xE4, 0xBC, 0x9A, 0xE7, 0xA4, 0xBE, + 0x02, 0x70, 0x41, 0x02, 0x6E, 0x41, 0x03, 0xCE, + 0xBC, 0x41, 0x02, 0x6D, 0x41, 0x02, 0x6B, 0x41, + 0x02, 0x4B, 0x42, 0x02, 0x4D, 0x42, 0x02, 0x47, + // Bytes 2880 - 28bf + 0x42, 0x03, 0x63, 0x61, 0x6C, 0x04, 0x6B, 0x63, + 0x61, 0x6C, 0x02, 0x70, 0x46, 0x02, 0x6E, 0x46, + 0x03, 0xCE, 0xBC, 0x46, 0x03, 0xCE, 0xBC, 0x67, + 0x02, 0x6D, 0x67, 0x02, 0x6B, 0x67, 0x02, 0x48, + 0x7A, 0x03, 0x6B, 0x48, 0x7A, 0x03, 0x4D, 0x48, + 0x7A, 0x03, 0x47, 0x48, 0x7A, 0x03, 0x54, 0x48, + 0x7A, 0x03, 0xCE, 0xBC, 0x6C, 0x02, 0x6D, 0x6C, + 0x02, 0x64, 0x6C, 0x02, 0x6B, 0x6C, 0x02, 0x66, + // Bytes 28c0 - 28ff + 0x6D, 0x02, 0x6E, 0x6D, 0x03, 0xCE, 0xBC, 0x6D, + 0x02, 0x6D, 0x6D, 0x02, 0x63, 0x6D, 0x02, 0x6B, + 0x6D, 0x03, 0x6D, 0x6D, 0x32, 0x03, 0x63, 0x6D, + 0x32, 0x02, 0x6D, 0x32, 0x03, 0x6B, 0x6D, 0x32, + 0x03, 0x6D, 0x6D, 0x33, 0x03, 0x63, 0x6D, 0x33, + 0x02, 0x6D, 0x33, 0x03, 0x6B, 0x6D, 0x33, 0x05, + 0x6D, 0xE2, 0x88, 0x95, 0x73, 0x06, 0x6D, 0xE2, + 0x88, 0x95, 0x73, 0x32, 0x02, 0x50, 0x61, 0x03, + // Bytes 2900 - 293f + 0x6B, 0x50, 0x61, 0x03, 0x4D, 0x50, 0x61, 0x03, + 0x47, 0x50, 0x61, 0x03, 0x72, 0x61, 0x64, 0x07, + 0x72, 0x61, 0x64, 0xE2, 0x88, 0x95, 0x73, 0x08, + 0x72, 0x61, 0x64, 0xE2, 0x88, 0x95, 0x73, 0x32, + 0x02, 0x70, 0x73, 0x02, 0x6E, 0x73, 0x03, 0xCE, + 0xBC, 0x73, 0x02, 0x6D, 0x73, 0x02, 0x70, 0x56, + 0x02, 0x6E, 0x56, 0x03, 0xCE, 0xBC, 0x56, 0x02, + 0x6D, 0x56, 0x02, 0x6B, 0x56, 0x02, 0x4D, 0x56, + // Bytes 2940 - 297f + 0x02, 0x70, 0x57, 0x02, 0x6E, 0x57, 0x03, 0xCE, + 0xBC, 0x57, 0x02, 0x6D, 0x57, 0x02, 0x6B, 0x57, + 0x02, 0x4D, 0x57, 0x03, 0x6B, 0xCE, 0xA9, 0x03, + 0x4D, 0xCE, 0xA9, 0x04, 0x61, 0x2E, 0x6D, 0x2E, + 0x02, 0x42, 0x71, 0x02, 0x63, 0x63, 0x02, 0x63, + 0x64, 0x06, 0x43, 0xE2, 0x88, 0x95, 0x6B, 0x67, + 0x03, 0x43, 0x6F, 0x2E, 0x02, 0x64, 0x42, 0x02, + 0x47, 0x79, 0x02, 0x68, 0x61, 0x02, 0x48, 0x50, + // Bytes 2980 - 29bf + 0x02, 0x69, 0x6E, 0x02, 0x4B, 0x4B, 0x02, 0x4B, + 0x4D, 0x02, 0x6B, 0x74, 0x02, 0x6C, 0x6D, 0x02, + 0x6C, 0x6E, 0x03, 0x6C, 0x6F, 0x67, 0x02, 0x6C, + 0x78, 0x02, 0x6D, 0x62, 0x03, 0x6D, 0x69, 0x6C, + 0x03, 0x6D, 0x6F, 0x6C, 0x02, 0x50, 0x48, 0x04, + 0x70, 0x2E, 0x6D, 0x2E, 0x03, 0x50, 0x50, 0x4D, + 0x02, 0x50, 0x52, 0x02, 0x73, 0x72, 0x02, 0x53, + 0x76, 0x02, 0x57, 0x62, 0x05, 0x56, 0xE2, 0x88, + // Bytes 29c0 - 29ff + 0x95, 0x6D, 0x05, 0x41, 0xE2, 0x88, 0x95, 0x6D, + 0x04, 0x31, 0xE6, 0x97, 0xA5, 0x04, 0x32, 0xE6, + 0x97, 0xA5, 0x04, 0x33, 0xE6, 0x97, 0xA5, 0x04, + 0x34, 0xE6, 0x97, 0xA5, 0x04, 0x35, 0xE6, 0x97, + 0xA5, 0x04, 0x36, 0xE6, 0x97, 0xA5, 0x04, 0x37, + 0xE6, 0x97, 0xA5, 0x04, 0x38, 0xE6, 0x97, 0xA5, + 0x04, 0x39, 0xE6, 0x97, 0xA5, 0x05, 0x31, 0x30, + 0xE6, 0x97, 0xA5, 0x05, 0x31, 0x31, 0xE6, 0x97, + // Bytes 2a00 - 2a3f + 0xA5, 0x05, 0x31, 0x32, 0xE6, 0x97, 0xA5, 0x05, + 0x31, 0x33, 0xE6, 0x97, 0xA5, 0x05, 0x31, 0x34, + 0xE6, 0x97, 0xA5, 0x05, 0x31, 0x35, 0xE6, 0x97, + 0xA5, 0x05, 0x31, 0x36, 0xE6, 0x97, 0xA5, 0x05, + 0x31, 0x37, 0xE6, 0x97, 0xA5, 0x05, 0x31, 0x38, + 0xE6, 0x97, 0xA5, 0x05, 0x31, 0x39, 0xE6, 0x97, + 0xA5, 0x05, 0x32, 0x30, 0xE6, 0x97, 0xA5, 0x05, + 0x32, 0x31, 0xE6, 0x97, 0xA5, 0x05, 0x32, 0x32, + // Bytes 2a40 - 2a7f + 0xE6, 0x97, 0xA5, 0x05, 0x32, 0x33, 0xE6, 0x97, + 0xA5, 0x05, 0x32, 0x34, 0xE6, 0x97, 0xA5, 0x05, + 0x32, 0x35, 0xE6, 0x97, 0xA5, 0x05, 0x32, 0x36, + 0xE6, 0x97, 0xA5, 0x05, 0x32, 0x37, 0xE6, 0x97, + 0xA5, 0x05, 0x32, 0x38, 0xE6, 0x97, 0xA5, 0x05, + 0x32, 0x39, 0xE6, 0x97, 0xA5, 0x05, 0x33, 0x30, + 0xE6, 0x97, 0xA5, 0x05, 0x33, 0x31, 0xE6, 0x97, + 0xA5, 0x03, 0x67, 0x61, 0x6C, 0x03, 0xEA, 0x9D, + // Bytes 2a80 - 2abf + 0xAF, 0x03, 0xE8, 0xB1, 0x88, 0x03, 0xE6, 0x9B, + 0xB4, 0x03, 0xE8, 0xB3, 0x88, 0x03, 0xE6, 0xBB, + 0x91, 0x03, 0xE4, 0xB8, 0xB2, 0x03, 0xE5, 0x8F, + 0xA5, 0x03, 0xE5, 0xA5, 0x91, 0x03, 0xE5, 0x96, + 0x87, 0x03, 0xE5, 0xA5, 0x88, 0x03, 0xE6, 0x87, + 0xB6, 0x03, 0xE7, 0x99, 0xA9, 0x03, 0xE7, 0xBE, + 0x85, 0x03, 0xE8, 0x98, 0xBF, 0x03, 0xE8, 0x9E, + 0xBA, 0x03, 0xE8, 0xA3, 0xB8, 0x03, 0xE9, 0x82, + // Bytes 2ac0 - 2aff + 0x8F, 0x03, 0xE6, 0xA8, 0x82, 0x03, 0xE6, 0xB4, + 0x9B, 0x03, 0xE7, 0x83, 0x99, 0x03, 0xE7, 0x8F, + 0x9E, 0x03, 0xE8, 0x90, 0xBD, 0x03, 0xE9, 0x85, + 0xAA, 0x03, 0xE9, 0xA7, 0xB1, 0x03, 0xE4, 0xBA, + 0x82, 0x03, 0xE5, 0x8D, 0xB5, 0x03, 0xE6, 0xAC, + 0x84, 0x03, 0xE7, 0x88, 0x9B, 0x03, 0xE8, 0x98, + 0xAD, 0x03, 0xE9, 0xB8, 0x9E, 0x03, 0xE5, 0xB5, + 0x90, 0x03, 0xE6, 0xBF, 0xAB, 0x03, 0xE8, 0x97, + // Bytes 2b00 - 2b3f + 0x8D, 0x03, 0xE8, 0xA5, 0xA4, 0x03, 0xE6, 0x8B, + 0x89, 0x03, 0xE8, 0x87, 0x98, 0x03, 0xE8, 0xA0, + 0x9F, 0x03, 0xE5, 0xBB, 0x8A, 0x03, 0xE6, 0x9C, + 0x97, 0x03, 0xE6, 0xB5, 0xAA, 0x03, 0xE7, 0x8B, + 0xBC, 0x03, 0xE9, 0x83, 0x8E, 0x03, 0xE4, 0xBE, + 0x86, 0x03, 0xE5, 0x86, 0xB7, 0x03, 0xE5, 0x8B, + 0x9E, 0x03, 0xE6, 0x93, 0x84, 0x03, 0xE6, 0xAB, + 0x93, 0x03, 0xE7, 0x88, 0x90, 0x03, 0xE7, 0x9B, + // Bytes 2b40 - 2b7f + 0xA7, 0x03, 0xE8, 0x98, 0x86, 0x03, 0xE8, 0x99, + 0x9C, 0x03, 0xE8, 0xB7, 0xAF, 0x03, 0xE9, 0x9C, + 0xB2, 0x03, 0xE9, 0xAD, 0xAF, 0x03, 0xE9, 0xB7, + 0xBA, 0x03, 0xE7, 0xA2, 0x8C, 0x03, 0xE7, 0xA5, + 0xBF, 0x03, 0xE7, 0xB6, 0xA0, 0x03, 0xE8, 0x8F, + 0x89, 0x03, 0xE9, 0x8C, 0x84, 0x03, 0xE8, 0xAB, + 0x96, 0x03, 0xE5, 0xA3, 0x9F, 0x03, 0xE5, 0xBC, + 0x84, 0x03, 0xE7, 0xB1, 0xA0, 0x03, 0xE8, 0x81, + // Bytes 2b80 - 2bbf + 0xBE, 0x03, 0xE7, 0x89, 0xA2, 0x03, 0xE7, 0xA3, + 0x8A, 0x03, 0xE8, 0xB3, 0x82, 0x03, 0xE9, 0x9B, + 0xB7, 0x03, 0xE5, 0xA3, 0x98, 0x03, 0xE5, 0xB1, + 0xA2, 0x03, 0xE6, 0xA8, 0x93, 0x03, 0xE6, 0xB7, + 0x9A, 0x03, 0xE6, 0xBC, 0x8F, 0x03, 0xE7, 0xB4, + 0xAF, 0x03, 0xE7, 0xB8, 0xB7, 0x03, 0xE9, 0x99, + 0x8B, 0x03, 0xE5, 0x8B, 0x92, 0x03, 0xE8, 0x82, + 0x8B, 0x03, 0xE5, 0x87, 0x9C, 0x03, 0xE5, 0x87, + // Bytes 2bc0 - 2bff + 0x8C, 0x03, 0xE7, 0xA8, 0x9C, 0x03, 0xE7, 0xB6, + 0xBE, 0x03, 0xE8, 0x8F, 0xB1, 0x03, 0xE9, 0x99, + 0xB5, 0x03, 0xE8, 0xAE, 0x80, 0x03, 0xE6, 0x8B, + 0x8F, 0x03, 0xE8, 0xAB, 0xBE, 0x03, 0xE4, 0xB8, + 0xB9, 0x03, 0xE5, 0xAF, 0xA7, 0x03, 0xE6, 0x80, + 0x92, 0x03, 0xE7, 0x8E, 0x87, 0x03, 0xE7, 0x95, + 0xB0, 0x03, 0xE5, 0x8C, 0x97, 0x03, 0xE7, 0xA3, + 0xBB, 0x03, 0xE4, 0xBE, 0xBF, 0x03, 0xE5, 0xBE, + // Bytes 2c00 - 2c3f + 0xA9, 0x03, 0xE4, 0xB8, 0x8D, 0x03, 0xE6, 0xB3, + 0x8C, 0x03, 0xE6, 0x95, 0xB8, 0x03, 0xE7, 0xB4, + 0xA2, 0x03, 0xE5, 0x8F, 0x83, 0x03, 0xE5, 0xA1, + 0x9E, 0x03, 0xE7, 0x9C, 0x81, 0x03, 0xE8, 0x91, + 0x89, 0x03, 0xE8, 0xAA, 0xAA, 0x03, 0xE6, 0xAE, + 0xBA, 0x03, 0xE6, 0xB2, 0x88, 0x03, 0xE6, 0x8B, + 0xBE, 0x03, 0xE8, 0x8B, 0xA5, 0x03, 0xE6, 0x8E, + 0xA0, 0x03, 0xE7, 0x95, 0xA5, 0x03, 0xE4, 0xBA, + // Bytes 2c40 - 2c7f + 0xAE, 0x03, 0xE5, 0x85, 0xA9, 0x03, 0xE5, 0x87, + 0x89, 0x03, 0xE6, 0xA2, 0x81, 0x03, 0xE7, 0xB3, + 0xA7, 0x03, 0xE8, 0x89, 0xAF, 0x03, 0xE8, 0xAB, + 0x92, 0x03, 0xE9, 0x87, 0x8F, 0x03, 0xE5, 0x8B, + 0xB5, 0x03, 0xE5, 0x91, 0x82, 0x03, 0xE5, 0xBB, + 0xAC, 0x03, 0xE6, 0x97, 0x85, 0x03, 0xE6, 0xBF, + 0xBE, 0x03, 0xE7, 0xA4, 0xAA, 0x03, 0xE9, 0x96, + 0xAD, 0x03, 0xE9, 0xA9, 0xAA, 0x03, 0xE9, 0xBA, + // Bytes 2c80 - 2cbf + 0x97, 0x03, 0xE9, 0xBB, 0x8E, 0x03, 0xE6, 0x9B, + 0x86, 0x03, 0xE6, 0xAD, 0xB7, 0x03, 0xE8, 0xBD, + 0xA2, 0x03, 0xE5, 0xB9, 0xB4, 0x03, 0xE6, 0x86, + 0x90, 0x03, 0xE6, 0x88, 0x80, 0x03, 0xE6, 0x92, + 0x9A, 0x03, 0xE6, 0xBC, 0xA3, 0x03, 0xE7, 0x85, + 0x89, 0x03, 0xE7, 0x92, 0x89, 0x03, 0xE7, 0xA7, + 0x8A, 0x03, 0xE7, 0xB7, 0xB4, 0x03, 0xE8, 0x81, + 0xAF, 0x03, 0xE8, 0xBC, 0xA6, 0x03, 0xE8, 0x93, + // Bytes 2cc0 - 2cff + 0xAE, 0x03, 0xE9, 0x80, 0xA3, 0x03, 0xE9, 0x8D, + 0x8A, 0x03, 0xE5, 0x88, 0x97, 0x03, 0xE5, 0x8A, + 0xA3, 0x03, 0xE5, 0x92, 0xBD, 0x03, 0xE7, 0x83, + 0x88, 0x03, 0xE8, 0xA3, 0x82, 0x03, 0xE5, 0xBB, + 0x89, 0x03, 0xE5, 0xBF, 0xB5, 0x03, 0xE6, 0x8D, + 0xBB, 0x03, 0xE6, 0xAE, 0xAE, 0x03, 0xE7, 0xB0, + 0xBE, 0x03, 0xE7, 0x8D, 0xB5, 0x03, 0xE4, 0xBB, + 0xA4, 0x03, 0xE5, 0x9B, 0xB9, 0x03, 0xE5, 0xB6, + // Bytes 2d00 - 2d3f + 0xBA, 0x03, 0xE6, 0x80, 0x9C, 0x03, 0xE7, 0x8E, + 0xB2, 0x03, 0xE7, 0x91, 0xA9, 0x03, 0xE7, 0xBE, + 0x9A, 0x03, 0xE8, 0x81, 0x86, 0x03, 0xE9, 0x88, + 0xB4, 0x03, 0xE9, 0x9B, 0xB6, 0x03, 0xE9, 0x9D, + 0x88, 0x03, 0xE9, 0xA0, 0x98, 0x03, 0xE4, 0xBE, + 0x8B, 0x03, 0xE7, 0xA6, 0xAE, 0x03, 0xE9, 0x86, + 0xB4, 0x03, 0xE9, 0x9A, 0xB8, 0x03, 0xE6, 0x83, + 0xA1, 0x03, 0xE4, 0xBA, 0x86, 0x03, 0xE5, 0x83, + // Bytes 2d40 - 2d7f + 0x9A, 0x03, 0xE5, 0xAF, 0xAE, 0x03, 0xE5, 0xB0, + 0xBF, 0x03, 0xE6, 0x96, 0x99, 0x03, 0xE7, 0x87, + 0x8E, 0x03, 0xE7, 0x99, 0x82, 0x03, 0xE8, 0x93, + 0xBC, 0x03, 0xE9, 0x81, 0xBC, 0x03, 0xE6, 0x9A, + 0x88, 0x03, 0xE9, 0x98, 0xAE, 0x03, 0xE5, 0x8A, + 0x89, 0x03, 0xE6, 0x9D, 0xBB, 0x03, 0xE6, 0x9F, + 0xB3, 0x03, 0xE6, 0xB5, 0x81, 0x03, 0xE6, 0xBA, + 0x9C, 0x03, 0xE7, 0x90, 0x89, 0x03, 0xE7, 0x95, + // Bytes 2d80 - 2dbf + 0x99, 0x03, 0xE7, 0xA1, 0xAB, 0x03, 0xE7, 0xB4, + 0x90, 0x03, 0xE9, 0xA1, 0x9E, 0x03, 0xE6, 0x88, + 0xAE, 0x03, 0xE9, 0x99, 0xB8, 0x03, 0xE5, 0x80, + 0xAB, 0x03, 0xE5, 0xB4, 0x99, 0x03, 0xE6, 0xB7, + 0xAA, 0x03, 0xE8, 0xBC, 0xAA, 0x03, 0xE5, 0xBE, + 0x8B, 0x03, 0xE6, 0x85, 0x84, 0x03, 0xE6, 0xA0, + 0x97, 0x03, 0xE9, 0x9A, 0x86, 0x03, 0xE5, 0x88, + 0xA9, 0x03, 0xE5, 0x90, 0x8F, 0x03, 0xE5, 0xB1, + // Bytes 2dc0 - 2dff + 0xA5, 0x03, 0xE6, 0x98, 0x93, 0x03, 0xE6, 0x9D, + 0x8E, 0x03, 0xE6, 0xA2, 0xA8, 0x03, 0xE6, 0xB3, + 0xA5, 0x03, 0xE7, 0x90, 0x86, 0x03, 0xE7, 0x97, + 0xA2, 0x03, 0xE7, 0xBD, 0xB9, 0x03, 0xE8, 0xA3, + 0x8F, 0x03, 0xE8, 0xA3, 0xA1, 0x03, 0xE9, 0x9B, + 0xA2, 0x03, 0xE5, 0x8C, 0xBF, 0x03, 0xE6, 0xBA, + 0xBA, 0x03, 0xE5, 0x90, 0x9D, 0x03, 0xE7, 0x87, + 0x90, 0x03, 0xE7, 0x92, 0x98, 0x03, 0xE8, 0x97, + // Bytes 2e00 - 2e3f + 0xBA, 0x03, 0xE9, 0x9A, 0xA3, 0x03, 0xE9, 0xB1, + 0x97, 0x03, 0xE9, 0xBA, 0x9F, 0x03, 0xE6, 0x9E, + 0x97, 0x03, 0xE6, 0xB7, 0x8B, 0x03, 0xE8, 0x87, + 0xA8, 0x03, 0xE7, 0xAC, 0xA0, 0x03, 0xE7, 0xB2, + 0x92, 0x03, 0xE7, 0x8B, 0x80, 0x03, 0xE7, 0x82, + 0x99, 0x03, 0xE8, 0xAD, 0x98, 0x03, 0xE4, 0xBB, + 0x80, 0x03, 0xE8, 0x8C, 0xB6, 0x03, 0xE5, 0x88, + 0xBA, 0x03, 0xE5, 0x88, 0x87, 0x03, 0xE5, 0xBA, + // Bytes 2e40 - 2e7f + 0xA6, 0x03, 0xE6, 0x8B, 0x93, 0x03, 0xE7, 0xB3, + 0x96, 0x03, 0xE5, 0xAE, 0x85, 0x03, 0xE6, 0xB4, + 0x9E, 0x03, 0xE6, 0x9A, 0xB4, 0x03, 0xE8, 0xBC, + 0xBB, 0x03, 0xE9, 0x99, 0x8D, 0x03, 0xE5, 0xBB, + 0x93, 0x03, 0xE5, 0x85, 0x80, 0x03, 0xE5, 0x97, + 0x80, 0x03, 0xE5, 0xA1, 0x9A, 0x03, 0xE6, 0x99, + 0xB4, 0x03, 0xE5, 0x87, 0x9E, 0x03, 0xE7, 0x8C, + 0xAA, 0x03, 0xE7, 0x9B, 0x8A, 0x03, 0xE7, 0xA4, + // Bytes 2e80 - 2ebf + 0xBC, 0x03, 0xE7, 0xA5, 0x9E, 0x03, 0xE7, 0xA5, + 0xA5, 0x03, 0xE7, 0xA6, 0x8F, 0x03, 0xE9, 0x9D, + 0x96, 0x03, 0xE7, 0xB2, 0xBE, 0x03, 0xE8, 0x98, + 0x92, 0x03, 0xE8, 0xAB, 0xB8, 0x03, 0xE9, 0x80, + 0xB8, 0x03, 0xE9, 0x83, 0xBD, 0x03, 0xE9, 0xA3, + 0xAF, 0x03, 0xE9, 0xA3, 0xBC, 0x03, 0xE9, 0xA4, + 0xA8, 0x03, 0xE9, 0xB6, 0xB4, 0x03, 0xE4, 0xBE, + 0xAE, 0x03, 0xE5, 0x83, 0xA7, 0x03, 0xE5, 0x85, + // Bytes 2ec0 - 2eff + 0x8D, 0x03, 0xE5, 0x8B, 0x89, 0x03, 0xE5, 0x8B, + 0xA4, 0x03, 0xE5, 0x8D, 0x91, 0x03, 0xE5, 0x96, + 0x9D, 0x03, 0xE5, 0x98, 0x86, 0x03, 0xE5, 0x99, + 0xA8, 0x03, 0xE5, 0xA1, 0x80, 0x03, 0xE5, 0xA2, + 0xA8, 0x03, 0xE5, 0xB1, 0xA4, 0x03, 0xE6, 0x82, + 0x94, 0x03, 0xE6, 0x85, 0xA8, 0x03, 0xE6, 0x86, + 0x8E, 0x03, 0xE6, 0x87, 0xB2, 0x03, 0xE6, 0x95, + 0x8F, 0x03, 0xE6, 0x97, 0xA2, 0x03, 0xE6, 0x9A, + // Bytes 2f00 - 2f3f + 0x91, 0x03, 0xE6, 0xA2, 0x85, 0x03, 0xE6, 0xB5, + 0xB7, 0x03, 0xE6, 0xB8, 0x9A, 0x03, 0xE6, 0xBC, + 0xA2, 0x03, 0xE7, 0x85, 0xAE, 0x03, 0xE7, 0x88, + 0xAB, 0x03, 0xE7, 0x90, 0xA2, 0x03, 0xE7, 0xA2, + 0x91, 0x03, 0xE7, 0xA5, 0x89, 0x03, 0xE7, 0xA5, + 0x88, 0x03, 0xE7, 0xA5, 0x90, 0x03, 0xE7, 0xA5, + 0x96, 0x03, 0xE7, 0xA6, 0x8D, 0x03, 0xE7, 0xA6, + 0x8E, 0x03, 0xE7, 0xA9, 0x80, 0x03, 0xE7, 0xAA, + // Bytes 2f40 - 2f7f + 0x81, 0x03, 0xE7, 0xAF, 0x80, 0x03, 0xE7, 0xB8, + 0x89, 0x03, 0xE7, 0xB9, 0x81, 0x03, 0xE7, 0xBD, + 0xB2, 0x03, 0xE8, 0x80, 0x85, 0x03, 0xE8, 0x87, + 0xAD, 0x03, 0xE8, 0x89, 0xB9, 0x03, 0xE8, 0x91, + 0x97, 0x03, 0xE8, 0xA4, 0x90, 0x03, 0xE8, 0xA6, + 0x96, 0x03, 0xE8, 0xAC, 0x81, 0x03, 0xE8, 0xAC, + 0xB9, 0x03, 0xE8, 0xB3, 0x93, 0x03, 0xE8, 0xB4, + 0x88, 0x03, 0xE8, 0xBE, 0xB6, 0x03, 0xE9, 0x9B, + // Bytes 2f80 - 2fbf + 0xA3, 0x03, 0xE9, 0x9F, 0xBF, 0x03, 0xE9, 0xA0, + 0xBB, 0x03, 0xE6, 0x81, 0xB5, 0x04, 0xF0, 0xA4, + 0x8B, 0xAE, 0x03, 0xE8, 0x88, 0x98, 0x03, 0xE4, + 0xB8, 0xA6, 0x03, 0xE5, 0x86, 0xB5, 0x03, 0xE5, + 0x85, 0xA8, 0x03, 0xE4, 0xBE, 0x80, 0x03, 0xE5, + 0x85, 0x85, 0x03, 0xE5, 0x86, 0x80, 0x03, 0xE5, + 0x8B, 0x87, 0x03, 0xE5, 0x8B, 0xBA, 0x03, 0xE5, + 0x95, 0x95, 0x03, 0xE5, 0x96, 0x99, 0x03, 0xE5, + // Bytes 2fc0 - 2fff + 0x97, 0xA2, 0x03, 0xE5, 0xA2, 0xB3, 0x03, 0xE5, + 0xA5, 0x84, 0x03, 0xE5, 0xA5, 0x94, 0x03, 0xE5, + 0xA9, 0xA2, 0x03, 0xE5, 0xAC, 0xA8, 0x03, 0xE5, + 0xBB, 0x92, 0x03, 0xE5, 0xBB, 0x99, 0x03, 0xE5, + 0xBD, 0xA9, 0x03, 0xE5, 0xBE, 0xAD, 0x03, 0xE6, + 0x83, 0x98, 0x03, 0xE6, 0x85, 0x8E, 0x03, 0xE6, + 0x84, 0x88, 0x03, 0xE6, 0x85, 0xA0, 0x03, 0xE6, + 0x88, 0xB4, 0x03, 0xE6, 0x8F, 0x84, 0x03, 0xE6, + // Bytes 3000 - 303f + 0x90, 0x9C, 0x03, 0xE6, 0x91, 0x92, 0x03, 0xE6, + 0x95, 0x96, 0x03, 0xE6, 0x9C, 0x9B, 0x03, 0xE6, + 0x9D, 0x96, 0x03, 0xE6, 0xBB, 0x9B, 0x03, 0xE6, + 0xBB, 0x8B, 0x03, 0xE7, 0x80, 0x9E, 0x03, 0xE7, + 0x9E, 0xA7, 0x03, 0xE7, 0x88, 0xB5, 0x03, 0xE7, + 0x8A, 0xAF, 0x03, 0xE7, 0x91, 0xB1, 0x03, 0xE7, + 0x94, 0x86, 0x03, 0xE7, 0x94, 0xBB, 0x03, 0xE7, + 0x98, 0x9D, 0x03, 0xE7, 0x98, 0x9F, 0x03, 0xE7, + // Bytes 3040 - 307f + 0x9B, 0x9B, 0x03, 0xE7, 0x9B, 0xB4, 0x03, 0xE7, + 0x9D, 0x8A, 0x03, 0xE7, 0x9D, 0x80, 0x03, 0xE7, + 0xA3, 0x8C, 0x03, 0xE7, 0xAA, 0xB1, 0x03, 0xE7, + 0xB1, 0xBB, 0x03, 0xE7, 0xB5, 0x9B, 0x03, 0xE7, + 0xBC, 0xBE, 0x03, 0xE8, 0x8D, 0x92, 0x03, 0xE8, + 0x8F, 0xAF, 0x03, 0xE8, 0x9D, 0xB9, 0x03, 0xE8, + 0xA5, 0x81, 0x03, 0xE8, 0xA6, 0x86, 0x03, 0xE8, + 0xAA, 0xBF, 0x03, 0xE8, 0xAB, 0x8B, 0x03, 0xE8, + // Bytes 3080 - 30bf + 0xAB, 0xAD, 0x03, 0xE8, 0xAE, 0x8A, 0x03, 0xE8, + 0xBC, 0xB8, 0x03, 0xE9, 0x81, 0xB2, 0x03, 0xE9, + 0x86, 0x99, 0x03, 0xE9, 0x89, 0xB6, 0x03, 0xE9, + 0x99, 0xBC, 0x03, 0xE9, 0x9F, 0x9B, 0x03, 0xE9, + 0xA0, 0x8B, 0x03, 0xE9, 0xAC, 0x92, 0x04, 0xF0, + 0xA2, 0xA1, 0x8A, 0x04, 0xF0, 0xA2, 0xA1, 0x84, + 0x04, 0xF0, 0xA3, 0x8F, 0x95, 0x03, 0xE3, 0xAE, + 0x9D, 0x03, 0xE4, 0x80, 0x98, 0x03, 0xE4, 0x80, + // Bytes 30c0 - 30ff + 0xB9, 0x04, 0xF0, 0xA5, 0x89, 0x89, 0x04, 0xF0, + 0xA5, 0xB3, 0x90, 0x04, 0xF0, 0xA7, 0xBB, 0x93, + 0x03, 0xE9, 0xBD, 0x83, 0x03, 0xE9, 0xBE, 0x8E, + 0x02, 0x66, 0x66, 0x02, 0x66, 0x69, 0x02, 0x66, + 0x6C, 0x03, 0x66, 0x66, 0x69, 0x03, 0x66, 0x66, + 0x6C, 0x02, 0x73, 0x74, 0x04, 0xD5, 0xB4, 0xD5, + 0xB6, 0x04, 0xD5, 0xB4, 0xD5, 0xA5, 0x04, 0xD5, + 0xB4, 0xD5, 0xAB, 0x04, 0xD5, 0xBE, 0xD5, 0xB6, + // Bytes 3100 - 313f + 0x04, 0xD5, 0xB4, 0xD5, 0xAD, 0x04, 0xD7, 0x99, + 0xD6, 0xB4, 0x04, 0xD7, 0xB2, 0xD6, 0xB7, 0x02, + 0xD7, 0xA2, 0x02, 0xD7, 0x94, 0x02, 0xD7, 0x9B, + 0x02, 0xD7, 0x9C, 0x02, 0xD7, 0x9D, 0x02, 0xD7, + 0xA8, 0x02, 0xD7, 0xAA, 0x04, 0xD7, 0xA9, 0xD7, + 0x81, 0x04, 0xD7, 0xA9, 0xD7, 0x82, 0x06, 0xD7, + 0xA9, 0xD6, 0xBC, 0xD7, 0x81, 0x06, 0xD7, 0xA9, + 0xD6, 0xBC, 0xD7, 0x82, 0x04, 0xD7, 0x90, 0xD6, + // Bytes 3140 - 317f + 0xB7, 0x04, 0xD7, 0x90, 0xD6, 0xB8, 0x04, 0xD7, + 0x90, 0xD6, 0xBC, 0x04, 0xD7, 0x91, 0xD6, 0xBC, + 0x04, 0xD7, 0x92, 0xD6, 0xBC, 0x04, 0xD7, 0x93, + 0xD6, 0xBC, 0x04, 0xD7, 0x94, 0xD6, 0xBC, 0x04, + 0xD7, 0x95, 0xD6, 0xBC, 0x04, 0xD7, 0x96, 0xD6, + 0xBC, 0x04, 0xD7, 0x98, 0xD6, 0xBC, 0x04, 0xD7, + 0x99, 0xD6, 0xBC, 0x04, 0xD7, 0x9A, 0xD6, 0xBC, + 0x04, 0xD7, 0x9B, 0xD6, 0xBC, 0x04, 0xD7, 0x9C, + // Bytes 3180 - 31bf + 0xD6, 0xBC, 0x04, 0xD7, 0x9E, 0xD6, 0xBC, 0x04, + 0xD7, 0xA0, 0xD6, 0xBC, 0x04, 0xD7, 0xA1, 0xD6, + 0xBC, 0x04, 0xD7, 0xA3, 0xD6, 0xBC, 0x04, 0xD7, + 0xA4, 0xD6, 0xBC, 0x04, 0xD7, 0xA6, 0xD6, 0xBC, + 0x04, 0xD7, 0xA7, 0xD6, 0xBC, 0x04, 0xD7, 0xA8, + 0xD6, 0xBC, 0x04, 0xD7, 0xA9, 0xD6, 0xBC, 0x04, + 0xD7, 0xAA, 0xD6, 0xBC, 0x04, 0xD7, 0x95, 0xD6, + 0xB9, 0x04, 0xD7, 0x91, 0xD6, 0xBF, 0x04, 0xD7, + // Bytes 31c0 - 31ff + 0x9B, 0xD6, 0xBF, 0x04, 0xD7, 0xA4, 0xD6, 0xBF, + 0x04, 0xD7, 0x90, 0xD7, 0x9C, 0x02, 0xD9, 0xB1, + 0x02, 0xD9, 0xBB, 0x02, 0xD9, 0xBE, 0x02, 0xDA, + 0x80, 0x02, 0xD9, 0xBA, 0x02, 0xD9, 0xBF, 0x02, + 0xD9, 0xB9, 0x02, 0xDA, 0xA4, 0x02, 0xDA, 0xA6, + 0x02, 0xDA, 0x84, 0x02, 0xDA, 0x83, 0x02, 0xDA, + 0x86, 0x02, 0xDA, 0x87, 0x02, 0xDA, 0x8D, 0x02, + 0xDA, 0x8C, 0x02, 0xDA, 0x8E, 0x02, 0xDA, 0x88, + // Bytes 3200 - 323f + 0x02, 0xDA, 0x98, 0x02, 0xDA, 0x91, 0x02, 0xDA, + 0xA9, 0x02, 0xDA, 0xAF, 0x02, 0xDA, 0xB3, 0x02, + 0xDA, 0xB1, 0x02, 0xDA, 0xBA, 0x02, 0xDA, 0xBB, + 0x02, 0xDB, 0x81, 0x02, 0xDA, 0xBE, 0x02, 0xDB, + 0x92, 0x02, 0xDA, 0xAD, 0x02, 0xDB, 0x87, 0x02, + 0xDB, 0x86, 0x02, 0xDB, 0x88, 0x02, 0xDB, 0x8B, + 0x02, 0xDB, 0x85, 0x02, 0xDB, 0x89, 0x02, 0xDB, + 0x90, 0x02, 0xD9, 0x89, 0x06, 0xD9, 0x8A, 0xD9, + // Bytes 3240 - 327f + 0x94, 0xD8, 0xA7, 0x06, 0xD9, 0x8A, 0xD9, 0x94, + 0xDB, 0x95, 0x06, 0xD9, 0x8A, 0xD9, 0x94, 0xD9, + 0x88, 0x06, 0xD9, 0x8A, 0xD9, 0x94, 0xDB, 0x87, + 0x06, 0xD9, 0x8A, 0xD9, 0x94, 0xDB, 0x86, 0x06, + 0xD9, 0x8A, 0xD9, 0x94, 0xDB, 0x88, 0x06, 0xD9, + 0x8A, 0xD9, 0x94, 0xDB, 0x90, 0x06, 0xD9, 0x8A, + 0xD9, 0x94, 0xD9, 0x89, 0x02, 0xDB, 0x8C, 0x06, + 0xD9, 0x8A, 0xD9, 0x94, 0xD8, 0xAC, 0x06, 0xD9, + // Bytes 3280 - 32bf + 0x8A, 0xD9, 0x94, 0xD8, 0xAD, 0x06, 0xD9, 0x8A, + 0xD9, 0x94, 0xD9, 0x85, 0x06, 0xD9, 0x8A, 0xD9, + 0x94, 0xD9, 0x8A, 0x04, 0xD8, 0xA8, 0xD8, 0xAC, + 0x04, 0xD8, 0xA8, 0xD8, 0xAD, 0x04, 0xD8, 0xA8, + 0xD8, 0xAE, 0x04, 0xD8, 0xA8, 0xD9, 0x85, 0x04, + 0xD8, 0xA8, 0xD9, 0x89, 0x04, 0xD8, 0xA8, 0xD9, + 0x8A, 0x04, 0xD8, 0xAA, 0xD8, 0xAC, 0x04, 0xD8, + 0xAA, 0xD8, 0xAD, 0x04, 0xD8, 0xAA, 0xD8, 0xAE, + // Bytes 32c0 - 32ff + 0x04, 0xD8, 0xAA, 0xD9, 0x85, 0x04, 0xD8, 0xAA, + 0xD9, 0x89, 0x04, 0xD8, 0xAA, 0xD9, 0x8A, 0x04, + 0xD8, 0xAB, 0xD8, 0xAC, 0x04, 0xD8, 0xAB, 0xD9, + 0x85, 0x04, 0xD8, 0xAB, 0xD9, 0x89, 0x04, 0xD8, + 0xAB, 0xD9, 0x8A, 0x04, 0xD8, 0xAC, 0xD8, 0xAD, + 0x04, 0xD8, 0xAC, 0xD9, 0x85, 0x04, 0xD8, 0xAD, + 0xD8, 0xAC, 0x04, 0xD8, 0xAD, 0xD9, 0x85, 0x04, + 0xD8, 0xAE, 0xD8, 0xAC, 0x04, 0xD8, 0xAE, 0xD8, + // Bytes 3300 - 333f + 0xAD, 0x04, 0xD8, 0xAE, 0xD9, 0x85, 0x04, 0xD8, + 0xB3, 0xD8, 0xAC, 0x04, 0xD8, 0xB3, 0xD8, 0xAD, + 0x04, 0xD8, 0xB3, 0xD8, 0xAE, 0x04, 0xD8, 0xB3, + 0xD9, 0x85, 0x04, 0xD8, 0xB5, 0xD8, 0xAD, 0x04, + 0xD8, 0xB5, 0xD9, 0x85, 0x04, 0xD8, 0xB6, 0xD8, + 0xAC, 0x04, 0xD8, 0xB6, 0xD8, 0xAD, 0x04, 0xD8, + 0xB6, 0xD8, 0xAE, 0x04, 0xD8, 0xB6, 0xD9, 0x85, + 0x04, 0xD8, 0xB7, 0xD8, 0xAD, 0x04, 0xD8, 0xB7, + // Bytes 3340 - 337f + 0xD9, 0x85, 0x04, 0xD8, 0xB8, 0xD9, 0x85, 0x04, + 0xD8, 0xB9, 0xD8, 0xAC, 0x04, 0xD8, 0xB9, 0xD9, + 0x85, 0x04, 0xD8, 0xBA, 0xD8, 0xAC, 0x04, 0xD8, + 0xBA, 0xD9, 0x85, 0x04, 0xD9, 0x81, 0xD8, 0xAC, + 0x04, 0xD9, 0x81, 0xD8, 0xAD, 0x04, 0xD9, 0x81, + 0xD8, 0xAE, 0x04, 0xD9, 0x81, 0xD9, 0x85, 0x04, + 0xD9, 0x81, 0xD9, 0x89, 0x04, 0xD9, 0x81, 0xD9, + 0x8A, 0x04, 0xD9, 0x82, 0xD8, 0xAD, 0x04, 0xD9, + // Bytes 3380 - 33bf + 0x82, 0xD9, 0x85, 0x04, 0xD9, 0x82, 0xD9, 0x89, + 0x04, 0xD9, 0x82, 0xD9, 0x8A, 0x04, 0xD9, 0x83, + 0xD8, 0xA7, 0x04, 0xD9, 0x83, 0xD8, 0xAC, 0x04, + 0xD9, 0x83, 0xD8, 0xAD, 0x04, 0xD9, 0x83, 0xD8, + 0xAE, 0x04, 0xD9, 0x83, 0xD9, 0x84, 0x04, 0xD9, + 0x83, 0xD9, 0x85, 0x04, 0xD9, 0x83, 0xD9, 0x89, + 0x04, 0xD9, 0x83, 0xD9, 0x8A, 0x04, 0xD9, 0x84, + 0xD8, 0xAC, 0x04, 0xD9, 0x84, 0xD8, 0xAD, 0x04, + // Bytes 33c0 - 33ff + 0xD9, 0x84, 0xD8, 0xAE, 0x04, 0xD9, 0x84, 0xD9, + 0x85, 0x04, 0xD9, 0x84, 0xD9, 0x89, 0x04, 0xD9, + 0x84, 0xD9, 0x8A, 0x04, 0xD9, 0x85, 0xD8, 0xAC, + 0x04, 0xD9, 0x85, 0xD8, 0xAD, 0x04, 0xD9, 0x85, + 0xD8, 0xAE, 0x04, 0xD9, 0x85, 0xD9, 0x85, 0x04, + 0xD9, 0x85, 0xD9, 0x89, 0x04, 0xD9, 0x85, 0xD9, + 0x8A, 0x04, 0xD9, 0x86, 0xD8, 0xAC, 0x04, 0xD9, + 0x86, 0xD8, 0xAD, 0x04, 0xD9, 0x86, 0xD8, 0xAE, + // Bytes 3400 - 343f + 0x04, 0xD9, 0x86, 0xD9, 0x85, 0x04, 0xD9, 0x86, + 0xD9, 0x89, 0x04, 0xD9, 0x86, 0xD9, 0x8A, 0x04, + 0xD9, 0x87, 0xD8, 0xAC, 0x04, 0xD9, 0x87, 0xD9, + 0x85, 0x04, 0xD9, 0x87, 0xD9, 0x89, 0x04, 0xD9, + 0x87, 0xD9, 0x8A, 0x04, 0xD9, 0x8A, 0xD8, 0xAC, + 0x04, 0xD9, 0x8A, 0xD8, 0xAD, 0x04, 0xD9, 0x8A, + 0xD8, 0xAE, 0x04, 0xD9, 0x8A, 0xD9, 0x85, 0x04, + 0xD9, 0x8A, 0xD9, 0x89, 0x04, 0xD9, 0x8A, 0xD9, + // Bytes 3440 - 347f + 0x8A, 0x04, 0xD8, 0xB0, 0xD9, 0xB0, 0x04, 0xD8, + 0xB1, 0xD9, 0xB0, 0x04, 0xD9, 0x89, 0xD9, 0xB0, + 0x05, 0x20, 0xD9, 0x8C, 0xD9, 0x91, 0x05, 0x20, + 0xD9, 0x8D, 0xD9, 0x91, 0x05, 0x20, 0xD9, 0x8E, + 0xD9, 0x91, 0x05, 0x20, 0xD9, 0x8F, 0xD9, 0x91, + 0x05, 0x20, 0xD9, 0x90, 0xD9, 0x91, 0x05, 0x20, + 0xD9, 0x91, 0xD9, 0xB0, 0x06, 0xD9, 0x8A, 0xD9, + 0x94, 0xD8, 0xB1, 0x06, 0xD9, 0x8A, 0xD9, 0x94, + // Bytes 3480 - 34bf + 0xD8, 0xB2, 0x06, 0xD9, 0x8A, 0xD9, 0x94, 0xD9, + 0x86, 0x04, 0xD8, 0xA8, 0xD8, 0xB1, 0x04, 0xD8, + 0xA8, 0xD8, 0xB2, 0x04, 0xD8, 0xA8, 0xD9, 0x86, + 0x04, 0xD8, 0xAA, 0xD8, 0xB1, 0x04, 0xD8, 0xAA, + 0xD8, 0xB2, 0x04, 0xD8, 0xAA, 0xD9, 0x86, 0x04, + 0xD8, 0xAB, 0xD8, 0xB1, 0x04, 0xD8, 0xAB, 0xD8, + 0xB2, 0x04, 0xD8, 0xAB, 0xD9, 0x86, 0x04, 0xD9, + 0x85, 0xD8, 0xA7, 0x04, 0xD9, 0x86, 0xD8, 0xB1, + // Bytes 34c0 - 34ff + 0x04, 0xD9, 0x86, 0xD8, 0xB2, 0x04, 0xD9, 0x86, + 0xD9, 0x86, 0x04, 0xD9, 0x8A, 0xD8, 0xB1, 0x04, + 0xD9, 0x8A, 0xD8, 0xB2, 0x04, 0xD9, 0x8A, 0xD9, + 0x86, 0x06, 0xD9, 0x8A, 0xD9, 0x94, 0xD8, 0xAE, + 0x06, 0xD9, 0x8A, 0xD9, 0x94, 0xD9, 0x87, 0x04, + 0xD8, 0xA8, 0xD9, 0x87, 0x04, 0xD8, 0xAA, 0xD9, + 0x87, 0x04, 0xD8, 0xB5, 0xD8, 0xAE, 0x04, 0xD9, + 0x84, 0xD9, 0x87, 0x04, 0xD9, 0x86, 0xD9, 0x87, + // Bytes 3500 - 353f + 0x04, 0xD9, 0x87, 0xD9, 0xB0, 0x04, 0xD9, 0x8A, + 0xD9, 0x87, 0x04, 0xD8, 0xAB, 0xD9, 0x87, 0x04, + 0xD8, 0xB3, 0xD9, 0x87, 0x04, 0xD8, 0xB4, 0xD9, + 0x85, 0x04, 0xD8, 0xB4, 0xD9, 0x87, 0x06, 0xD9, + 0x80, 0xD9, 0x8E, 0xD9, 0x91, 0x06, 0xD9, 0x80, + 0xD9, 0x8F, 0xD9, 0x91, 0x06, 0xD9, 0x80, 0xD9, + 0x90, 0xD9, 0x91, 0x04, 0xD8, 0xB7, 0xD9, 0x89, + 0x04, 0xD8, 0xB7, 0xD9, 0x8A, 0x04, 0xD8, 0xB9, + // Bytes 3540 - 357f + 0xD9, 0x89, 0x04, 0xD8, 0xB9, 0xD9, 0x8A, 0x04, + 0xD8, 0xBA, 0xD9, 0x89, 0x04, 0xD8, 0xBA, 0xD9, + 0x8A, 0x04, 0xD8, 0xB3, 0xD9, 0x89, 0x04, 0xD8, + 0xB3, 0xD9, 0x8A, 0x04, 0xD8, 0xB4, 0xD9, 0x89, + 0x04, 0xD8, 0xB4, 0xD9, 0x8A, 0x04, 0xD8, 0xAD, + 0xD9, 0x89, 0x04, 0xD8, 0xAD, 0xD9, 0x8A, 0x04, + 0xD8, 0xAC, 0xD9, 0x89, 0x04, 0xD8, 0xAC, 0xD9, + 0x8A, 0x04, 0xD8, 0xAE, 0xD9, 0x89, 0x04, 0xD8, + // Bytes 3580 - 35bf + 0xAE, 0xD9, 0x8A, 0x04, 0xD8, 0xB5, 0xD9, 0x89, + 0x04, 0xD8, 0xB5, 0xD9, 0x8A, 0x04, 0xD8, 0xB6, + 0xD9, 0x89, 0x04, 0xD8, 0xB6, 0xD9, 0x8A, 0x04, + 0xD8, 0xB4, 0xD8, 0xAC, 0x04, 0xD8, 0xB4, 0xD8, + 0xAD, 0x04, 0xD8, 0xB4, 0xD8, 0xAE, 0x04, 0xD8, + 0xB4, 0xD8, 0xB1, 0x04, 0xD8, 0xB3, 0xD8, 0xB1, + 0x04, 0xD8, 0xB5, 0xD8, 0xB1, 0x04, 0xD8, 0xB6, + 0xD8, 0xB1, 0x04, 0xD8, 0xA7, 0xD9, 0x8B, 0x06, + // Bytes 35c0 - 35ff + 0xD8, 0xAA, 0xD8, 0xAC, 0xD9, 0x85, 0x06, 0xD8, + 0xAA, 0xD8, 0xAD, 0xD8, 0xAC, 0x06, 0xD8, 0xAA, + 0xD8, 0xAD, 0xD9, 0x85, 0x06, 0xD8, 0xAA, 0xD8, + 0xAE, 0xD9, 0x85, 0x06, 0xD8, 0xAA, 0xD9, 0x85, + 0xD8, 0xAC, 0x06, 0xD8, 0xAA, 0xD9, 0x85, 0xD8, + 0xAD, 0x06, 0xD8, 0xAA, 0xD9, 0x85, 0xD8, 0xAE, + 0x06, 0xD8, 0xAC, 0xD9, 0x85, 0xD8, 0xAD, 0x06, + 0xD8, 0xAD, 0xD9, 0x85, 0xD9, 0x8A, 0x06, 0xD8, + // Bytes 3600 - 363f + 0xAD, 0xD9, 0x85, 0xD9, 0x89, 0x06, 0xD8, 0xB3, + 0xD8, 0xAD, 0xD8, 0xAC, 0x06, 0xD8, 0xB3, 0xD8, + 0xAC, 0xD8, 0xAD, 0x06, 0xD8, 0xB3, 0xD8, 0xAC, + 0xD9, 0x89, 0x06, 0xD8, 0xB3, 0xD9, 0x85, 0xD8, + 0xAD, 0x06, 0xD8, 0xB3, 0xD9, 0x85, 0xD8, 0xAC, + 0x06, 0xD8, 0xB3, 0xD9, 0x85, 0xD9, 0x85, 0x06, + 0xD8, 0xB5, 0xD8, 0xAD, 0xD8, 0xAD, 0x06, 0xD8, + 0xB5, 0xD9, 0x85, 0xD9, 0x85, 0x06, 0xD8, 0xB4, + // Bytes 3640 - 367f + 0xD8, 0xAD, 0xD9, 0x85, 0x06, 0xD8, 0xB4, 0xD8, + 0xAC, 0xD9, 0x8A, 0x06, 0xD8, 0xB4, 0xD9, 0x85, + 0xD8, 0xAE, 0x06, 0xD8, 0xB4, 0xD9, 0x85, 0xD9, + 0x85, 0x06, 0xD8, 0xB6, 0xD8, 0xAD, 0xD9, 0x89, + 0x06, 0xD8, 0xB6, 0xD8, 0xAE, 0xD9, 0x85, 0x06, + 0xD8, 0xB7, 0xD9, 0x85, 0xD8, 0xAD, 0x06, 0xD8, + 0xB7, 0xD9, 0x85, 0xD9, 0x85, 0x06, 0xD8, 0xB7, + 0xD9, 0x85, 0xD9, 0x8A, 0x06, 0xD8, 0xB9, 0xD8, + // Bytes 3680 - 36bf + 0xAC, 0xD9, 0x85, 0x06, 0xD8, 0xB9, 0xD9, 0x85, + 0xD9, 0x85, 0x06, 0xD8, 0xB9, 0xD9, 0x85, 0xD9, + 0x89, 0x06, 0xD8, 0xBA, 0xD9, 0x85, 0xD9, 0x85, + 0x06, 0xD8, 0xBA, 0xD9, 0x85, 0xD9, 0x8A, 0x06, + 0xD8, 0xBA, 0xD9, 0x85, 0xD9, 0x89, 0x06, 0xD9, + 0x81, 0xD8, 0xAE, 0xD9, 0x85, 0x06, 0xD9, 0x82, + 0xD9, 0x85, 0xD8, 0xAD, 0x06, 0xD9, 0x82, 0xD9, + 0x85, 0xD9, 0x85, 0x06, 0xD9, 0x84, 0xD8, 0xAD, + // Bytes 36c0 - 36ff + 0xD9, 0x85, 0x06, 0xD9, 0x84, 0xD8, 0xAD, 0xD9, + 0x8A, 0x06, 0xD9, 0x84, 0xD8, 0xAD, 0xD9, 0x89, + 0x06, 0xD9, 0x84, 0xD8, 0xAC, 0xD8, 0xAC, 0x06, + 0xD9, 0x84, 0xD8, 0xAE, 0xD9, 0x85, 0x06, 0xD9, + 0x84, 0xD9, 0x85, 0xD8, 0xAD, 0x06, 0xD9, 0x85, + 0xD8, 0xAD, 0xD8, 0xAC, 0x06, 0xD9, 0x85, 0xD8, + 0xAD, 0xD9, 0x85, 0x06, 0xD9, 0x85, 0xD8, 0xAD, + 0xD9, 0x8A, 0x06, 0xD9, 0x85, 0xD8, 0xAC, 0xD8, + // Bytes 3700 - 373f + 0xAD, 0x06, 0xD9, 0x85, 0xD8, 0xAC, 0xD9, 0x85, + 0x06, 0xD9, 0x85, 0xD8, 0xAE, 0xD8, 0xAC, 0x06, + 0xD9, 0x85, 0xD8, 0xAE, 0xD9, 0x85, 0x06, 0xD9, + 0x85, 0xD8, 0xAC, 0xD8, 0xAE, 0x06, 0xD9, 0x87, + 0xD9, 0x85, 0xD8, 0xAC, 0x06, 0xD9, 0x87, 0xD9, + 0x85, 0xD9, 0x85, 0x06, 0xD9, 0x86, 0xD8, 0xAD, + 0xD9, 0x85, 0x06, 0xD9, 0x86, 0xD8, 0xAD, 0xD9, + 0x89, 0x06, 0xD9, 0x86, 0xD8, 0xAC, 0xD9, 0x85, + // Bytes 3740 - 377f + 0x06, 0xD9, 0x86, 0xD8, 0xAC, 0xD9, 0x89, 0x06, + 0xD9, 0x86, 0xD9, 0x85, 0xD9, 0x8A, 0x06, 0xD9, + 0x86, 0xD9, 0x85, 0xD9, 0x89, 0x06, 0xD9, 0x8A, + 0xD9, 0x85, 0xD9, 0x85, 0x06, 0xD8, 0xA8, 0xD8, + 0xAE, 0xD9, 0x8A, 0x06, 0xD8, 0xAA, 0xD8, 0xAC, + 0xD9, 0x8A, 0x06, 0xD8, 0xAA, 0xD8, 0xAC, 0xD9, + 0x89, 0x06, 0xD8, 0xAA, 0xD8, 0xAE, 0xD9, 0x8A, + 0x06, 0xD8, 0xAA, 0xD8, 0xAE, 0xD9, 0x89, 0x06, + // Bytes 3780 - 37bf + 0xD8, 0xAA, 0xD9, 0x85, 0xD9, 0x8A, 0x06, 0xD8, + 0xAA, 0xD9, 0x85, 0xD9, 0x89, 0x06, 0xD8, 0xAC, + 0xD9, 0x85, 0xD9, 0x8A, 0x06, 0xD8, 0xAC, 0xD8, + 0xAD, 0xD9, 0x89, 0x06, 0xD8, 0xAC, 0xD9, 0x85, + 0xD9, 0x89, 0x06, 0xD8, 0xB3, 0xD8, 0xAE, 0xD9, + 0x89, 0x06, 0xD8, 0xB5, 0xD8, 0xAD, 0xD9, 0x8A, + 0x06, 0xD8, 0xB4, 0xD8, 0xAD, 0xD9, 0x8A, 0x06, + 0xD8, 0xB6, 0xD8, 0xAD, 0xD9, 0x8A, 0x06, 0xD9, + // Bytes 37c0 - 37ff + 0x84, 0xD8, 0xAC, 0xD9, 0x8A, 0x06, 0xD9, 0x84, + 0xD9, 0x85, 0xD9, 0x8A, 0x06, 0xD9, 0x8A, 0xD8, + 0xAD, 0xD9, 0x8A, 0x06, 0xD9, 0x8A, 0xD8, 0xAC, + 0xD9, 0x8A, 0x06, 0xD9, 0x8A, 0xD9, 0x85, 0xD9, + 0x8A, 0x06, 0xD9, 0x85, 0xD9, 0x85, 0xD9, 0x8A, + 0x06, 0xD9, 0x82, 0xD9, 0x85, 0xD9, 0x8A, 0x06, + 0xD9, 0x86, 0xD8, 0xAD, 0xD9, 0x8A, 0x06, 0xD8, + 0xB9, 0xD9, 0x85, 0xD9, 0x8A, 0x06, 0xD9, 0x83, + // Bytes 3800 - 383f + 0xD9, 0x85, 0xD9, 0x8A, 0x06, 0xD9, 0x86, 0xD8, + 0xAC, 0xD8, 0xAD, 0x06, 0xD9, 0x85, 0xD8, 0xAE, + 0xD9, 0x8A, 0x06, 0xD9, 0x84, 0xD8, 0xAC, 0xD9, + 0x85, 0x06, 0xD9, 0x83, 0xD9, 0x85, 0xD9, 0x85, + 0x06, 0xD8, 0xAC, 0xD8, 0xAD, 0xD9, 0x8A, 0x06, + 0xD8, 0xAD, 0xD8, 0xAC, 0xD9, 0x8A, 0x06, 0xD9, + 0x85, 0xD8, 0xAC, 0xD9, 0x8A, 0x06, 0xD9, 0x81, + 0xD9, 0x85, 0xD9, 0x8A, 0x06, 0xD8, 0xA8, 0xD8, + // Bytes 3840 - 387f + 0xAD, 0xD9, 0x8A, 0x06, 0xD8, 0xB3, 0xD8, 0xAE, + 0xD9, 0x8A, 0x06, 0xD9, 0x86, 0xD8, 0xAC, 0xD9, + 0x8A, 0x06, 0xD8, 0xB5, 0xD9, 0x84, 0xDB, 0x92, + 0x06, 0xD9, 0x82, 0xD9, 0x84, 0xDB, 0x92, 0x08, + 0xD8, 0xA7, 0xD9, 0x84, 0xD9, 0x84, 0xD9, 0x87, + 0x08, 0xD8, 0xA7, 0xD9, 0x83, 0xD8, 0xA8, 0xD8, + 0xB1, 0x08, 0xD9, 0x85, 0xD8, 0xAD, 0xD9, 0x85, + 0xD8, 0xAF, 0x08, 0xD8, 0xB5, 0xD9, 0x84, 0xD8, + // Bytes 3880 - 38bf + 0xB9, 0xD9, 0x85, 0x08, 0xD8, 0xB1, 0xD8, 0xB3, + 0xD9, 0x88, 0xD9, 0x84, 0x08, 0xD8, 0xB9, 0xD9, + 0x84, 0xD9, 0x8A, 0xD9, 0x87, 0x08, 0xD9, 0x88, + 0xD8, 0xB3, 0xD9, 0x84, 0xD9, 0x85, 0x06, 0xD8, + 0xB5, 0xD9, 0x84, 0xD9, 0x89, 0x21, 0xD8, 0xB5, + 0xD9, 0x84, 0xD9, 0x89, 0x20, 0xD8, 0xA7, 0xD9, + 0x84, 0xD9, 0x84, 0xD9, 0x87, 0x20, 0xD8, 0xB9, + 0xD9, 0x84, 0xD9, 0x8A, 0xD9, 0x87, 0x20, 0xD9, + // Bytes 38c0 - 38ff + 0x88, 0xD8, 0xB3, 0xD9, 0x84, 0xD9, 0x85, 0x0F, + 0xD8, 0xAC, 0xD9, 0x84, 0x20, 0xD8, 0xAC, 0xD9, + 0x84, 0xD8, 0xA7, 0xD9, 0x84, 0xD9, 0x87, 0x08, + 0xD8, 0xB1, 0xDB, 0x8C, 0xD8, 0xA7, 0xD9, 0x84, + 0x01, 0x2C, 0x03, 0xE3, 0x80, 0x81, 0x03, 0xE3, + 0x80, 0x82, 0x01, 0x3A, 0x01, 0x21, 0x01, 0x3F, + 0x03, 0xE3, 0x80, 0x96, 0x03, 0xE3, 0x80, 0x97, + 0x03, 0xE2, 0x80, 0x94, 0x03, 0xE2, 0x80, 0x93, + // Bytes 3900 - 393f + 0x01, 0x5F, 0x01, 0x7B, 0x01, 0x7D, 0x03, 0xE3, + 0x80, 0x94, 0x03, 0xE3, 0x80, 0x95, 0x03, 0xE3, + 0x80, 0x90, 0x03, 0xE3, 0x80, 0x91, 0x03, 0xE3, + 0x80, 0x8A, 0x03, 0xE3, 0x80, 0x8B, 0x03, 0xE3, + 0x80, 0x8C, 0x03, 0xE3, 0x80, 0x8D, 0x03, 0xE3, + 0x80, 0x8E, 0x03, 0xE3, 0x80, 0x8F, 0x01, 0x5B, + 0x01, 0x5D, 0x01, 0x23, 0x01, 0x26, 0x01, 0x2A, + 0x01, 0x2D, 0x01, 0x3C, 0x01, 0x3E, 0x01, 0x5C, + // Bytes 3940 - 397f + 0x01, 0x24, 0x01, 0x25, 0x01, 0x40, 0x03, 0x20, + 0xD9, 0x8B, 0x04, 0xD9, 0x80, 0xD9, 0x8B, 0x03, + 0x20, 0xD9, 0x8C, 0x03, 0x20, 0xD9, 0x8D, 0x03, + 0x20, 0xD9, 0x8E, 0x04, 0xD9, 0x80, 0xD9, 0x8E, + 0x03, 0x20, 0xD9, 0x8F, 0x04, 0xD9, 0x80, 0xD9, + 0x8F, 0x03, 0x20, 0xD9, 0x90, 0x04, 0xD9, 0x80, + 0xD9, 0x90, 0x03, 0x20, 0xD9, 0x91, 0x04, 0xD9, + 0x80, 0xD9, 0x91, 0x03, 0x20, 0xD9, 0x92, 0x04, + // Bytes 3980 - 39bf + 0xD9, 0x80, 0xD9, 0x92, 0x02, 0xD8, 0xA1, 0x02, + 0xD8, 0xA7, 0x02, 0xD8, 0xA8, 0x02, 0xD8, 0xA9, + 0x02, 0xD8, 0xAA, 0x02, 0xD8, 0xAB, 0x02, 0xD8, + 0xAC, 0x02, 0xD8, 0xAD, 0x02, 0xD8, 0xAE, 0x02, + 0xD8, 0xAF, 0x02, 0xD8, 0xB0, 0x02, 0xD8, 0xB1, + 0x02, 0xD8, 0xB2, 0x02, 0xD8, 0xB3, 0x02, 0xD8, + 0xB4, 0x02, 0xD8, 0xB5, 0x02, 0xD8, 0xB6, 0x02, + 0xD8, 0xB7, 0x02, 0xD8, 0xB8, 0x02, 0xD8, 0xB9, + // Bytes 39c0 - 39ff + 0x02, 0xD8, 0xBA, 0x02, 0xD9, 0x81, 0x02, 0xD9, + 0x82, 0x02, 0xD9, 0x83, 0x02, 0xD9, 0x84, 0x02, + 0xD9, 0x85, 0x02, 0xD9, 0x86, 0x02, 0xD9, 0x87, + 0x02, 0xD9, 0x88, 0x02, 0xD9, 0x8A, 0x06, 0xD9, + 0x84, 0xD8, 0xA7, 0xD9, 0x93, 0x06, 0xD9, 0x84, + 0xD8, 0xA7, 0xD9, 0x94, 0x06, 0xD9, 0x84, 0xD8, + 0xA7, 0xD9, 0x95, 0x04, 0xD9, 0x84, 0xD8, 0xA7, + 0x01, 0x22, 0x01, 0x27, 0x01, 0x2F, 0x01, 0x5E, + // Bytes 3a00 - 3a3f + 0x01, 0x7C, 0x01, 0x7E, 0x03, 0xE2, 0xA6, 0x85, + 0x03, 0xE2, 0xA6, 0x86, 0x03, 0xE3, 0x83, 0xBB, + 0x03, 0xE3, 0x82, 0xA1, 0x03, 0xE3, 0x82, 0xA3, + 0x03, 0xE3, 0x82, 0xA5, 0x03, 0xE3, 0x82, 0xA7, + 0x03, 0xE3, 0x82, 0xA9, 0x03, 0xE3, 0x83, 0xA3, + 0x03, 0xE3, 0x83, 0xA5, 0x03, 0xE3, 0x83, 0xA7, + 0x03, 0xE3, 0x83, 0x83, 0x03, 0xE3, 0x83, 0xBC, + 0x03, 0xE3, 0x83, 0xB3, 0x03, 0xE3, 0x82, 0x99, + // Bytes 3a40 - 3a7f + 0x03, 0xE3, 0x82, 0x9A, 0x02, 0xC2, 0xA2, 0x02, + 0xC2, 0xA3, 0x02, 0xC2, 0xAC, 0x02, 0xC2, 0xA6, + 0x02, 0xC2, 0xA5, 0x03, 0xE2, 0x82, 0xA9, 0x03, + 0xE2, 0x94, 0x82, 0x03, 0xE2, 0x86, 0x90, 0x03, + 0xE2, 0x86, 0x91, 0x03, 0xE2, 0x86, 0x92, 0x03, + 0xE2, 0x86, 0x93, 0x03, 0xE2, 0x96, 0xA0, 0x03, + 0xE2, 0x97, 0x8B, 0x08, 0xF0, 0x91, 0x82, 0x99, + 0xF0, 0x91, 0x82, 0xBA, 0x08, 0xF0, 0x91, 0x82, + // Bytes 3a80 - 3abf + 0x9B, 0xF0, 0x91, 0x82, 0xBA, 0x08, 0xF0, 0x91, + 0x82, 0xA5, 0xF0, 0x91, 0x82, 0xBA, 0x08, 0xF0, + 0x9D, 0x85, 0x97, 0xF0, 0x9D, 0x85, 0xA5, 0x08, + 0xF0, 0x9D, 0x85, 0x98, 0xF0, 0x9D, 0x85, 0xA5, + 0x0C, 0xF0, 0x9D, 0x85, 0x98, 0xF0, 0x9D, 0x85, + 0xA5, 0xF0, 0x9D, 0x85, 0xAE, 0x0C, 0xF0, 0x9D, + 0x85, 0x98, 0xF0, 0x9D, 0x85, 0xA5, 0xF0, 0x9D, + 0x85, 0xAF, 0x0C, 0xF0, 0x9D, 0x85, 0x98, 0xF0, + // Bytes 3ac0 - 3aff + 0x9D, 0x85, 0xA5, 0xF0, 0x9D, 0x85, 0xB0, 0x0C, + 0xF0, 0x9D, 0x85, 0x98, 0xF0, 0x9D, 0x85, 0xA5, + 0xF0, 0x9D, 0x85, 0xB1, 0x0C, 0xF0, 0x9D, 0x85, + 0x98, 0xF0, 0x9D, 0x85, 0xA5, 0xF0, 0x9D, 0x85, + 0xB2, 0x08, 0xF0, 0x9D, 0x86, 0xB9, 0xF0, 0x9D, + 0x85, 0xA5, 0x08, 0xF0, 0x9D, 0x86, 0xBA, 0xF0, + 0x9D, 0x85, 0xA5, 0x0C, 0xF0, 0x9D, 0x86, 0xB9, + 0xF0, 0x9D, 0x85, 0xA5, 0xF0, 0x9D, 0x85, 0xAE, + // Bytes 3b00 - 3b3f + 0x0C, 0xF0, 0x9D, 0x86, 0xBA, 0xF0, 0x9D, 0x85, + 0xA5, 0xF0, 0x9D, 0x85, 0xAE, 0x0C, 0xF0, 0x9D, + 0x86, 0xB9, 0xF0, 0x9D, 0x85, 0xA5, 0xF0, 0x9D, + 0x85, 0xAF, 0x0C, 0xF0, 0x9D, 0x86, 0xBA, 0xF0, + 0x9D, 0x85, 0xA5, 0xF0, 0x9D, 0x85, 0xAF, 0x02, + 0xC4, 0xB1, 0x02, 0xC8, 0xB7, 0x02, 0xCE, 0x91, + 0x02, 0xCE, 0x92, 0x02, 0xCE, 0x94, 0x02, 0xCE, + 0x95, 0x02, 0xCE, 0x96, 0x02, 0xCE, 0x97, 0x02, + // Bytes 3b40 - 3b7f + 0xCE, 0x99, 0x02, 0xCE, 0x9A, 0x02, 0xCE, 0x9B, + 0x02, 0xCE, 0x9C, 0x02, 0xCE, 0x9D, 0x02, 0xCE, + 0x9E, 0x02, 0xCE, 0x9F, 0x02, 0xCE, 0xA1, 0x02, + 0xCE, 0xA4, 0x02, 0xCE, 0xA6, 0x02, 0xCE, 0xA7, + 0x02, 0xCE, 0xA8, 0x03, 0xE2, 0x88, 0x87, 0x02, + 0xCE, 0xB1, 0x02, 0xCE, 0xB6, 0x02, 0xCE, 0xB7, + 0x02, 0xCE, 0xBB, 0x02, 0xCE, 0xBD, 0x02, 0xCE, + 0xBE, 0x02, 0xCE, 0xBF, 0x02, 0xCF, 0x83, 0x02, + // Bytes 3b80 - 3bbf + 0xCF, 0x84, 0x02, 0xCF, 0x85, 0x02, 0xCF, 0x88, + 0x02, 0xCF, 0x89, 0x03, 0xE2, 0x88, 0x82, 0x02, + 0xCF, 0x9C, 0x02, 0xCF, 0x9D, 0x02, 0x30, 0x2E, + 0x02, 0x30, 0x2C, 0x02, 0x31, 0x2C, 0x02, 0x32, + 0x2C, 0x02, 0x33, 0x2C, 0x02, 0x34, 0x2C, 0x02, + 0x35, 0x2C, 0x02, 0x36, 0x2C, 0x02, 0x37, 0x2C, + 0x02, 0x38, 0x2C, 0x02, 0x39, 0x2C, 0x03, 0x28, + 0x41, 0x29, 0x03, 0x28, 0x42, 0x29, 0x03, 0x28, + // Bytes 3bc0 - 3bff + 0x43, 0x29, 0x03, 0x28, 0x44, 0x29, 0x03, 0x28, + 0x45, 0x29, 0x03, 0x28, 0x46, 0x29, 0x03, 0x28, + 0x47, 0x29, 0x03, 0x28, 0x48, 0x29, 0x03, 0x28, + 0x49, 0x29, 0x03, 0x28, 0x4A, 0x29, 0x03, 0x28, + 0x4B, 0x29, 0x03, 0x28, 0x4C, 0x29, 0x03, 0x28, + 0x4D, 0x29, 0x03, 0x28, 0x4E, 0x29, 0x03, 0x28, + 0x4F, 0x29, 0x03, 0x28, 0x50, 0x29, 0x03, 0x28, + 0x51, 0x29, 0x03, 0x28, 0x52, 0x29, 0x03, 0x28, + // Bytes 3c00 - 3c3f + 0x53, 0x29, 0x03, 0x28, 0x54, 0x29, 0x03, 0x28, + 0x55, 0x29, 0x03, 0x28, 0x56, 0x29, 0x03, 0x28, + 0x57, 0x29, 0x03, 0x28, 0x58, 0x29, 0x03, 0x28, + 0x59, 0x29, 0x03, 0x28, 0x5A, 0x29, 0x07, 0xE3, + 0x80, 0x94, 0x53, 0xE3, 0x80, 0x95, 0x02, 0x43, + 0x44, 0x02, 0x57, 0x5A, 0x02, 0x48, 0x56, 0x02, + 0x53, 0x44, 0x02, 0x53, 0x53, 0x03, 0x50, 0x50, + 0x56, 0x02, 0x57, 0x43, 0x02, 0x44, 0x4A, 0x06, + // Bytes 3c40 - 3c7f + 0xE3, 0x81, 0xBB, 0xE3, 0x81, 0x8B, 0x06, 0xE3, + 0x82, 0xB3, 0xE3, 0x82, 0xB3, 0x03, 0xE5, 0xAD, + 0x97, 0x03, 0xE5, 0x8F, 0x8C, 0x03, 0xE5, 0xA4, + 0x9A, 0x03, 0xE8, 0xA7, 0xA3, 0x03, 0xE4, 0xBA, + 0xA4, 0x03, 0xE6, 0x98, 0xA0, 0x03, 0xE7, 0x84, + 0xA1, 0x03, 0xE5, 0x89, 0x8D, 0x03, 0xE5, 0xBE, + 0x8C, 0x03, 0xE5, 0x86, 0x8D, 0x03, 0xE6, 0x96, + 0xB0, 0x03, 0xE5, 0x88, 0x9D, 0x03, 0xE7, 0xB5, + // Bytes 3c80 - 3cbf + 0x82, 0x03, 0xE8, 0xB2, 0xA9, 0x03, 0xE5, 0xA3, + 0xB0, 0x03, 0xE5, 0x90, 0xB9, 0x03, 0xE6, 0xBC, + 0x94, 0x03, 0xE6, 0x8A, 0x95, 0x03, 0xE6, 0x8D, + 0x95, 0x03, 0xE9, 0x81, 0x8A, 0x03, 0xE6, 0x8C, + 0x87, 0x03, 0xE6, 0x89, 0x93, 0x03, 0xE7, 0xA6, + 0x81, 0x03, 0xE7, 0xA9, 0xBA, 0x03, 0xE5, 0x90, + 0x88, 0x03, 0xE6, 0xBA, 0x80, 0x03, 0xE7, 0x94, + 0xB3, 0x03, 0xE5, 0x89, 0xB2, 0x03, 0xE5, 0x96, + // Bytes 3cc0 - 3cff + 0xB6, 0x09, 0xE3, 0x80, 0x94, 0xE6, 0x9C, 0xAC, + 0xE3, 0x80, 0x95, 0x09, 0xE3, 0x80, 0x94, 0xE4, + 0xB8, 0x89, 0xE3, 0x80, 0x95, 0x09, 0xE3, 0x80, + 0x94, 0xE4, 0xBA, 0x8C, 0xE3, 0x80, 0x95, 0x09, + 0xE3, 0x80, 0x94, 0xE5, 0xAE, 0x89, 0xE3, 0x80, + 0x95, 0x09, 0xE3, 0x80, 0x94, 0xE7, 0x82, 0xB9, + 0xE3, 0x80, 0x95, 0x09, 0xE3, 0x80, 0x94, 0xE6, + 0x89, 0x93, 0xE3, 0x80, 0x95, 0x09, 0xE3, 0x80, + // Bytes 3d00 - 3d3f + 0x94, 0xE7, 0x9B, 0x97, 0xE3, 0x80, 0x95, 0x09, + 0xE3, 0x80, 0x94, 0xE5, 0x8B, 0x9D, 0xE3, 0x80, + 0x95, 0x09, 0xE3, 0x80, 0x94, 0xE6, 0x95, 0x97, + 0xE3, 0x80, 0x95, 0x03, 0xE5, 0xBE, 0x97, 0x03, + 0xE5, 0x8F, 0xAF, 0x03, 0xE4, 0xB8, 0xBD, 0x03, + 0xE4, 0xB8, 0xB8, 0x03, 0xE4, 0xB9, 0x81, 0x04, + 0xF0, 0xA0, 0x84, 0xA2, 0x03, 0xE4, 0xBD, 0xA0, + 0x03, 0xE4, 0xBE, 0xBB, 0x03, 0xE5, 0x80, 0x82, + // Bytes 3d40 - 3d7f + 0x03, 0xE5, 0x81, 0xBA, 0x03, 0xE5, 0x82, 0x99, + 0x03, 0xE5, 0x83, 0x8F, 0x03, 0xE3, 0x92, 0x9E, + 0x04, 0xF0, 0xA0, 0x98, 0xBA, 0x03, 0xE5, 0x85, + 0x94, 0x03, 0xE5, 0x85, 0xA4, 0x03, 0xE5, 0x85, + 0xB7, 0x04, 0xF0, 0xA0, 0x94, 0x9C, 0x03, 0xE3, + 0x92, 0xB9, 0x03, 0xE5, 0x85, 0xA7, 0x04, 0xF0, + 0xA0, 0x95, 0x8B, 0x03, 0xE5, 0x86, 0x97, 0x03, + 0xE5, 0x86, 0xA4, 0x03, 0xE4, 0xBB, 0x8C, 0x03, + // Bytes 3d80 - 3dbf + 0xE5, 0x86, 0xAC, 0x04, 0xF0, 0xA9, 0x87, 0x9F, + 0x03, 0xE5, 0x88, 0x83, 0x03, 0xE3, 0x93, 0x9F, + 0x03, 0xE5, 0x88, 0xBB, 0x03, 0xE5, 0x89, 0x86, + 0x03, 0xE5, 0x89, 0xB7, 0x03, 0xE3, 0x94, 0x95, + 0x03, 0xE5, 0x8C, 0x85, 0x03, 0xE5, 0x8C, 0x86, + 0x03, 0xE5, 0x8D, 0x89, 0x03, 0xE5, 0x8D, 0x9A, + 0x03, 0xE5, 0x8D, 0xB3, 0x03, 0xE5, 0x8D, 0xBD, + 0x03, 0xE5, 0x8D, 0xBF, 0x04, 0xF0, 0xA0, 0xA8, + // Bytes 3dc0 - 3dff + 0xAC, 0x03, 0xE7, 0x81, 0xB0, 0x03, 0xE5, 0x8F, + 0x8A, 0x03, 0xE5, 0x8F, 0x9F, 0x04, 0xF0, 0xA0, + 0xAD, 0xA3, 0x03, 0xE5, 0x8F, 0xAB, 0x03, 0xE5, + 0x8F, 0xB1, 0x03, 0xE5, 0x90, 0x86, 0x03, 0xE5, + 0x92, 0x9E, 0x03, 0xE5, 0x90, 0xB8, 0x03, 0xE5, + 0x91, 0x88, 0x03, 0xE5, 0x91, 0xA8, 0x03, 0xE5, + 0x92, 0xA2, 0x03, 0xE5, 0x93, 0xB6, 0x03, 0xE5, + 0x94, 0x90, 0x03, 0xE5, 0x95, 0x93, 0x03, 0xE5, + // Bytes 3e00 - 3e3f + 0x95, 0xA3, 0x03, 0xE5, 0x96, 0x84, 0x03, 0xE5, + 0x96, 0xAB, 0x03, 0xE5, 0x96, 0xB3, 0x03, 0xE5, + 0x97, 0x82, 0x03, 0xE5, 0x9C, 0x96, 0x03, 0xE5, + 0x9C, 0x97, 0x03, 0xE5, 0x99, 0x91, 0x03, 0xE5, + 0x99, 0xB4, 0x03, 0xE5, 0xA3, 0xAE, 0x03, 0xE5, + 0x9F, 0x8E, 0x03, 0xE5, 0x9F, 0xB4, 0x03, 0xE5, + 0xA0, 0x8D, 0x03, 0xE5, 0x9E, 0x8B, 0x03, 0xE5, + 0xA0, 0xB2, 0x03, 0xE5, 0xA0, 0xB1, 0x03, 0xE5, + // Bytes 3e40 - 3e7f + 0xA2, 0xAC, 0x04, 0xF0, 0xA1, 0x93, 0xA4, 0x03, + 0xE5, 0xA3, 0xB2, 0x03, 0xE5, 0xA3, 0xB7, 0x03, + 0xE5, 0xA4, 0x86, 0x03, 0xE5, 0xA4, 0xA2, 0x03, + 0xE5, 0xA5, 0xA2, 0x04, 0xF0, 0xA1, 0x9A, 0xA8, + 0x04, 0xF0, 0xA1, 0x9B, 0xAA, 0x03, 0xE5, 0xA7, + 0xAC, 0x03, 0xE5, 0xA8, 0x9B, 0x03, 0xE5, 0xA8, + 0xA7, 0x03, 0xE5, 0xA7, 0x98, 0x03, 0xE5, 0xA9, + 0xA6, 0x03, 0xE3, 0x9B, 0xAE, 0x03, 0xE3, 0x9B, + // Bytes 3e80 - 3ebf + 0xBC, 0x03, 0xE5, 0xAC, 0x88, 0x03, 0xE5, 0xAC, + 0xBE, 0x04, 0xF0, 0xA1, 0xA7, 0x88, 0x03, 0xE5, + 0xAF, 0x83, 0x03, 0xE5, 0xAF, 0x98, 0x03, 0xE5, + 0xAF, 0xB3, 0x04, 0xF0, 0xA1, 0xAC, 0x98, 0x03, + 0xE5, 0xAF, 0xBF, 0x03, 0xE5, 0xB0, 0x86, 0x03, + 0xE5, 0xBD, 0x93, 0x03, 0xE3, 0x9E, 0x81, 0x03, + 0xE5, 0xB1, 0xA0, 0x03, 0xE5, 0xB3, 0x80, 0x03, + 0xE5, 0xB2, 0x8D, 0x04, 0xF0, 0xA1, 0xB7, 0xA4, + // Bytes 3ec0 - 3eff + 0x03, 0xE5, 0xB5, 0x83, 0x04, 0xF0, 0xA1, 0xB7, + 0xA6, 0x03, 0xE5, 0xB5, 0xAE, 0x03, 0xE5, 0xB5, + 0xAB, 0x03, 0xE5, 0xB5, 0xBC, 0x03, 0xE5, 0xB7, + 0xA1, 0x03, 0xE5, 0xB7, 0xA2, 0x03, 0xE3, 0xA0, + 0xAF, 0x03, 0xE5, 0xB7, 0xBD, 0x03, 0xE5, 0xB8, + 0xA8, 0x03, 0xE5, 0xB8, 0xBD, 0x03, 0xE5, 0xB9, + 0xA9, 0x03, 0xE3, 0xA1, 0xA2, 0x04, 0xF0, 0xA2, + 0x86, 0x83, 0x03, 0xE3, 0xA1, 0xBC, 0x03, 0xE5, + // Bytes 3f00 - 3f3f + 0xBA, 0xB0, 0x03, 0xE5, 0xBA, 0xB3, 0x03, 0xE5, + 0xBA, 0xB6, 0x04, 0xF0, 0xAA, 0x8E, 0x92, 0x04, + 0xF0, 0xA2, 0x8C, 0xB1, 0x03, 0xE8, 0x88, 0x81, + 0x03, 0xE5, 0xBC, 0xA2, 0x03, 0xE3, 0xA3, 0x87, + 0x04, 0xF0, 0xA3, 0x8A, 0xB8, 0x04, 0xF0, 0xA6, + 0x87, 0x9A, 0x03, 0xE5, 0xBD, 0xA2, 0x03, 0xE5, + 0xBD, 0xAB, 0x03, 0xE3, 0xA3, 0xA3, 0x03, 0xE5, + 0xBE, 0x9A, 0x03, 0xE5, 0xBF, 0x8D, 0x03, 0xE5, + // Bytes 3f40 - 3f7f + 0xBF, 0x97, 0x03, 0xE5, 0xBF, 0xB9, 0x03, 0xE6, + 0x82, 0x81, 0x03, 0xE3, 0xA4, 0xBA, 0x03, 0xE3, + 0xA4, 0x9C, 0x04, 0xF0, 0xA2, 0x9B, 0x94, 0x03, + 0xE6, 0x83, 0x87, 0x03, 0xE6, 0x85, 0x88, 0x03, + 0xE6, 0x85, 0x8C, 0x03, 0xE6, 0x85, 0xBA, 0x03, + 0xE6, 0x86, 0xB2, 0x03, 0xE6, 0x86, 0xA4, 0x03, + 0xE6, 0x86, 0xAF, 0x03, 0xE6, 0x87, 0x9E, 0x03, + 0xE6, 0x88, 0x90, 0x03, 0xE6, 0x88, 0x9B, 0x03, + // Bytes 3f80 - 3fbf + 0xE6, 0x89, 0x9D, 0x03, 0xE6, 0x8A, 0xB1, 0x03, + 0xE6, 0x8B, 0x94, 0x03, 0xE6, 0x8D, 0x90, 0x04, + 0xF0, 0xA2, 0xAC, 0x8C, 0x03, 0xE6, 0x8C, 0xBD, + 0x03, 0xE6, 0x8B, 0xBC, 0x03, 0xE6, 0x8D, 0xA8, + 0x03, 0xE6, 0x8E, 0x83, 0x03, 0xE6, 0x8F, 0xA4, + 0x04, 0xF0, 0xA2, 0xAF, 0xB1, 0x03, 0xE6, 0x90, + 0xA2, 0x03, 0xE6, 0x8F, 0x85, 0x03, 0xE6, 0x8E, + 0xA9, 0x03, 0xE3, 0xA8, 0xAE, 0x03, 0xE6, 0x91, + // Bytes 3fc0 - 3fff + 0xA9, 0x03, 0xE6, 0x91, 0xBE, 0x03, 0xE6, 0x92, + 0x9D, 0x03, 0xE6, 0x91, 0xB7, 0x03, 0xE3, 0xA9, + 0xAC, 0x03, 0xE6, 0x95, 0xAC, 0x04, 0xF0, 0xA3, + 0x80, 0x8A, 0x03, 0xE6, 0x97, 0xA3, 0x03, 0xE6, + 0x9B, 0xB8, 0x03, 0xE6, 0x99, 0x89, 0x03, 0xE3, + 0xAC, 0x99, 0x03, 0xE3, 0xAC, 0x88, 0x03, 0xE3, + 0xAB, 0xA4, 0x03, 0xE5, 0x86, 0x92, 0x03, 0xE5, + 0x86, 0x95, 0x03, 0xE6, 0x9C, 0x80, 0x03, 0xE6, + // Bytes 4000 - 403f + 0x9A, 0x9C, 0x03, 0xE8, 0x82, 0xAD, 0x03, 0xE4, + 0x8F, 0x99, 0x03, 0xE6, 0x9C, 0xA1, 0x03, 0xE6, + 0x9D, 0x9E, 0x03, 0xE6, 0x9D, 0x93, 0x04, 0xF0, + 0xA3, 0x8F, 0x83, 0x03, 0xE3, 0xAD, 0x89, 0x03, + 0xE6, 0x9F, 0xBA, 0x03, 0xE6, 0x9E, 0x85, 0x03, + 0xE6, 0xA1, 0x92, 0x04, 0xF0, 0xA3, 0x91, 0xAD, + 0x03, 0xE6, 0xA2, 0x8E, 0x03, 0xE6, 0xA0, 0x9F, + 0x03, 0xE6, 0xA4, 0x94, 0x03, 0xE6, 0xA5, 0x82, + // Bytes 4040 - 407f + 0x03, 0xE6, 0xA6, 0xA3, 0x03, 0xE6, 0xA7, 0xAA, + 0x03, 0xE6, 0xAA, 0xA8, 0x04, 0xF0, 0xA3, 0x9A, + 0xA3, 0x03, 0xE6, 0xAB, 0x9B, 0x03, 0xE3, 0xB0, + 0x98, 0x03, 0xE6, 0xAC, 0xA1, 0x04, 0xF0, 0xA3, + 0xA2, 0xA7, 0x03, 0xE6, 0xAD, 0x94, 0x03, 0xE3, + 0xB1, 0x8E, 0x03, 0xE6, 0xAD, 0xB2, 0x03, 0xE6, + 0xAE, 0x9F, 0x03, 0xE6, 0xAE, 0xBB, 0x04, 0xF0, + 0xA3, 0xAA, 0x8D, 0x04, 0xF0, 0xA1, 0xB4, 0x8B, + // Bytes 4080 - 40bf + 0x04, 0xF0, 0xA3, 0xAB, 0xBA, 0x03, 0xE6, 0xB1, + 0x8E, 0x04, 0xF0, 0xA3, 0xB2, 0xBC, 0x03, 0xE6, + 0xB2, 0xBF, 0x03, 0xE6, 0xB3, 0x8D, 0x03, 0xE6, + 0xB1, 0xA7, 0x03, 0xE6, 0xB4, 0x96, 0x03, 0xE6, + 0xB4, 0xBE, 0x03, 0xE6, 0xB5, 0xA9, 0x03, 0xE6, + 0xB5, 0xB8, 0x03, 0xE6, 0xB6, 0x85, 0x04, 0xF0, + 0xA3, 0xB4, 0x9E, 0x03, 0xE6, 0xB4, 0xB4, 0x03, + 0xE6, 0xB8, 0xAF, 0x03, 0xE6, 0xB9, 0xAE, 0x03, + // Bytes 40c0 - 40ff + 0xE3, 0xB4, 0xB3, 0x03, 0xE6, 0xBB, 0x87, 0x04, + 0xF0, 0xA3, 0xBB, 0x91, 0x03, 0xE6, 0xB7, 0xB9, + 0x03, 0xE6, 0xBD, 0xAE, 0x04, 0xF0, 0xA3, 0xBD, + 0x9E, 0x04, 0xF0, 0xA3, 0xBE, 0x8E, 0x03, 0xE6, + 0xBF, 0x86, 0x03, 0xE7, 0x80, 0xB9, 0x03, 0xE7, + 0x80, 0x9B, 0x03, 0xE3, 0xB6, 0x96, 0x03, 0xE7, + 0x81, 0x8A, 0x03, 0xE7, 0x81, 0xBD, 0x03, 0xE7, + 0x81, 0xB7, 0x03, 0xE7, 0x82, 0xAD, 0x04, 0xF0, + // Bytes 4100 - 413f + 0xA0, 0x94, 0xA5, 0x03, 0xE7, 0x85, 0x85, 0x04, + 0xF0, 0xA4, 0x89, 0xA3, 0x03, 0xE7, 0x86, 0x9C, + 0x04, 0xF0, 0xA4, 0x8E, 0xAB, 0x03, 0xE7, 0x88, + 0xA8, 0x03, 0xE7, 0x89, 0x90, 0x04, 0xF0, 0xA4, + 0x98, 0x88, 0x03, 0xE7, 0x8A, 0x80, 0x03, 0xE7, + 0x8A, 0x95, 0x04, 0xF0, 0xA4, 0x9C, 0xB5, 0x04, + 0xF0, 0xA4, 0xA0, 0x94, 0x03, 0xE7, 0x8D, 0xBA, + 0x03, 0xE7, 0x8E, 0x8B, 0x03, 0xE3, 0xBA, 0xAC, + // Bytes 4140 - 417f + 0x03, 0xE7, 0x8E, 0xA5, 0x03, 0xE3, 0xBA, 0xB8, + 0x03, 0xE7, 0x91, 0x87, 0x03, 0xE7, 0x91, 0x9C, + 0x03, 0xE7, 0x92, 0x85, 0x03, 0xE7, 0x93, 0x8A, + 0x03, 0xE3, 0xBC, 0x9B, 0x03, 0xE7, 0x94, 0xA4, + 0x04, 0xF0, 0xA4, 0xB0, 0xB6, 0x03, 0xE7, 0x94, + 0xBE, 0x04, 0xF0, 0xA4, 0xB2, 0x92, 0x04, 0xF0, + 0xA2, 0x86, 0x9F, 0x03, 0xE7, 0x98, 0x90, 0x04, + 0xF0, 0xA4, 0xBE, 0xA1, 0x04, 0xF0, 0xA4, 0xBE, + // Bytes 4180 - 41bf + 0xB8, 0x04, 0xF0, 0xA5, 0x81, 0x84, 0x03, 0xE3, + 0xBF, 0xBC, 0x03, 0xE4, 0x80, 0x88, 0x04, 0xF0, + 0xA5, 0x83, 0xB3, 0x04, 0xF0, 0xA5, 0x83, 0xB2, + 0x04, 0xF0, 0xA5, 0x84, 0x99, 0x04, 0xF0, 0xA5, + 0x84, 0xB3, 0x03, 0xE7, 0x9C, 0x9E, 0x03, 0xE7, + 0x9C, 0x9F, 0x03, 0xE7, 0x9E, 0x8B, 0x03, 0xE4, + 0x81, 0x86, 0x03, 0xE4, 0x82, 0x96, 0x04, 0xF0, + 0xA5, 0x90, 0x9D, 0x03, 0xE7, 0xA1, 0x8E, 0x03, + // Bytes 41c0 - 41ff + 0xE4, 0x83, 0xA3, 0x04, 0xF0, 0xA5, 0x98, 0xA6, + 0x04, 0xF0, 0xA5, 0x9A, 0x9A, 0x04, 0xF0, 0xA5, + 0x9B, 0x85, 0x03, 0xE7, 0xA7, 0xAB, 0x03, 0xE4, + 0x84, 0xAF, 0x03, 0xE7, 0xA9, 0x8A, 0x03, 0xE7, + 0xA9, 0x8F, 0x04, 0xF0, 0xA5, 0xA5, 0xBC, 0x04, + 0xF0, 0xA5, 0xAA, 0xA7, 0x03, 0xE7, 0xAB, 0xAE, + 0x03, 0xE4, 0x88, 0x82, 0x04, 0xF0, 0xA5, 0xAE, + 0xAB, 0x03, 0xE7, 0xAF, 0x86, 0x03, 0xE7, 0xAF, + // Bytes 4200 - 423f + 0x89, 0x03, 0xE4, 0x88, 0xA7, 0x04, 0xF0, 0xA5, + 0xB2, 0x80, 0x03, 0xE7, 0xB3, 0x92, 0x03, 0xE4, + 0x8A, 0xA0, 0x03, 0xE7, 0xB3, 0xA8, 0x03, 0xE7, + 0xB3, 0xA3, 0x03, 0xE7, 0xB4, 0x80, 0x04, 0xF0, + 0xA5, 0xBE, 0x86, 0x03, 0xE7, 0xB5, 0xA3, 0x03, + 0xE4, 0x8C, 0x81, 0x03, 0xE7, 0xB7, 0x87, 0x03, + 0xE7, 0xB8, 0x82, 0x03, 0xE7, 0xB9, 0x85, 0x03, + 0xE4, 0x8C, 0xB4, 0x04, 0xF0, 0xA6, 0x88, 0xA8, + // Bytes 4240 - 427f + 0x04, 0xF0, 0xA6, 0x89, 0x87, 0x03, 0xE4, 0x8D, + 0x99, 0x04, 0xF0, 0xA6, 0x8B, 0x99, 0x03, 0xE7, + 0xBD, 0xBA, 0x04, 0xF0, 0xA6, 0x8C, 0xBE, 0x03, + 0xE7, 0xBE, 0x95, 0x03, 0xE7, 0xBF, 0xBA, 0x04, + 0xF0, 0xA6, 0x93, 0x9A, 0x04, 0xF0, 0xA6, 0x94, + 0xA3, 0x03, 0xE8, 0x81, 0xA0, 0x04, 0xF0, 0xA6, + 0x96, 0xA8, 0x03, 0xE8, 0x81, 0xB0, 0x04, 0xF0, + 0xA3, 0x8D, 0x9F, 0x03, 0xE4, 0x8F, 0x95, 0x03, + // Bytes 4280 - 42bf + 0xE8, 0x82, 0xB2, 0x03, 0xE8, 0x84, 0x83, 0x03, + 0xE4, 0x90, 0x8B, 0x03, 0xE8, 0x84, 0xBE, 0x03, + 0xE5, 0xAA, 0xB5, 0x04, 0xF0, 0xA6, 0x9E, 0xA7, + 0x04, 0xF0, 0xA6, 0x9E, 0xB5, 0x04, 0xF0, 0xA3, + 0x8E, 0x93, 0x04, 0xF0, 0xA3, 0x8E, 0x9C, 0x03, + 0xE8, 0x88, 0x84, 0x03, 0xE8, 0xBE, 0x9E, 0x03, + 0xE4, 0x91, 0xAB, 0x03, 0xE8, 0x8A, 0x91, 0x03, + 0xE8, 0x8A, 0x8B, 0x03, 0xE8, 0x8A, 0x9D, 0x03, + // Bytes 42c0 - 42ff + 0xE5, 0x8A, 0xB3, 0x03, 0xE8, 0x8A, 0xB1, 0x03, + 0xE8, 0x8A, 0xB3, 0x03, 0xE8, 0x8A, 0xBD, 0x03, + 0xE8, 0x8B, 0xA6, 0x04, 0xF0, 0xA6, 0xAC, 0xBC, + 0x03, 0xE8, 0x8C, 0x9D, 0x03, 0xE8, 0x8D, 0xA3, + 0x03, 0xE8, 0x8E, 0xAD, 0x03, 0xE8, 0x8C, 0xA3, + 0x03, 0xE8, 0x8E, 0xBD, 0x03, 0xE8, 0x8F, 0xA7, + 0x03, 0xE8, 0x8D, 0x93, 0x03, 0xE8, 0x8F, 0x8A, + 0x03, 0xE8, 0x8F, 0x8C, 0x03, 0xE8, 0x8F, 0x9C, + // Bytes 4300 - 433f + 0x04, 0xF0, 0xA6, 0xB0, 0xB6, 0x04, 0xF0, 0xA6, + 0xB5, 0xAB, 0x04, 0xF0, 0xA6, 0xB3, 0x95, 0x03, + 0xE4, 0x94, 0xAB, 0x03, 0xE8, 0x93, 0xB1, 0x03, + 0xE8, 0x93, 0xB3, 0x03, 0xE8, 0x94, 0x96, 0x04, + 0xF0, 0xA7, 0x8F, 0x8A, 0x03, 0xE8, 0x95, 0xA4, + 0x04, 0xF0, 0xA6, 0xBC, 0xAC, 0x03, 0xE4, 0x95, + 0x9D, 0x03, 0xE4, 0x95, 0xA1, 0x04, 0xF0, 0xA6, + 0xBE, 0xB1, 0x04, 0xF0, 0xA7, 0x83, 0x92, 0x03, + // Bytes 4340 - 437f + 0xE4, 0x95, 0xAB, 0x03, 0xE8, 0x99, 0x90, 0x03, + 0xE8, 0x99, 0xA7, 0x03, 0xE8, 0x99, 0xA9, 0x03, + 0xE8, 0x9A, 0xA9, 0x03, 0xE8, 0x9A, 0x88, 0x03, + 0xE8, 0x9C, 0x8E, 0x03, 0xE8, 0x9B, 0xA2, 0x03, + 0xE8, 0x9C, 0xA8, 0x03, 0xE8, 0x9D, 0xAB, 0x03, + 0xE8, 0x9E, 0x86, 0x03, 0xE4, 0x97, 0x97, 0x03, + 0xE8, 0x9F, 0xA1, 0x03, 0xE8, 0xA0, 0x81, 0x03, + 0xE4, 0x97, 0xB9, 0x03, 0xE8, 0xA1, 0xA0, 0x04, + // Bytes 4380 - 43bf + 0xF0, 0xA7, 0x99, 0xA7, 0x03, 0xE8, 0xA3, 0x97, + 0x03, 0xE8, 0xA3, 0x9E, 0x03, 0xE4, 0x98, 0xB5, + 0x03, 0xE8, 0xA3, 0xBA, 0x03, 0xE3, 0x92, 0xBB, + 0x04, 0xF0, 0xA7, 0xA2, 0xAE, 0x04, 0xF0, 0xA7, + 0xA5, 0xA6, 0x03, 0xE4, 0x9A, 0xBE, 0x03, 0xE4, + 0x9B, 0x87, 0x03, 0xE8, 0xAA, 0xA0, 0x04, 0xF0, + 0xA7, 0xB2, 0xA8, 0x03, 0xE8, 0xB2, 0xAB, 0x03, + 0xE8, 0xB3, 0x81, 0x03, 0xE8, 0xB4, 0x9B, 0x03, + // Bytes 43c0 - 43ff + 0xE8, 0xB5, 0xB7, 0x04, 0xF0, 0xA7, 0xBC, 0xAF, + 0x04, 0xF0, 0xA0, 0xA0, 0x84, 0x03, 0xE8, 0xB7, + 0x8B, 0x03, 0xE8, 0xB6, 0xBC, 0x03, 0xE8, 0xB7, + 0xB0, 0x04, 0xF0, 0xA0, 0xA3, 0x9E, 0x03, 0xE8, + 0xBB, 0x94, 0x04, 0xF0, 0xA8, 0x97, 0x92, 0x04, + 0xF0, 0xA8, 0x97, 0xAD, 0x03, 0xE9, 0x82, 0x94, + 0x03, 0xE9, 0x83, 0xB1, 0x03, 0xE9, 0x84, 0x91, + 0x04, 0xF0, 0xA8, 0x9C, 0xAE, 0x03, 0xE9, 0x84, + // Bytes 4400 - 443f + 0x9B, 0x03, 0xE9, 0x88, 0xB8, 0x03, 0xE9, 0x8B, + 0x97, 0x03, 0xE9, 0x8B, 0x98, 0x03, 0xE9, 0x89, + 0xBC, 0x03, 0xE9, 0x8F, 0xB9, 0x03, 0xE9, 0x90, + 0x95, 0x04, 0xF0, 0xA8, 0xAF, 0xBA, 0x03, 0xE9, + 0x96, 0x8B, 0x03, 0xE4, 0xA6, 0x95, 0x03, 0xE9, + 0x96, 0xB7, 0x04, 0xF0, 0xA8, 0xB5, 0xB7, 0x03, + 0xE4, 0xA7, 0xA6, 0x03, 0xE9, 0x9B, 0x83, 0x03, + 0xE5, 0xB6, 0xB2, 0x03, 0xE9, 0x9C, 0xA3, 0x04, + // Bytes 4440 - 447f + 0xF0, 0xA9, 0x85, 0x85, 0x04, 0xF0, 0xA9, 0x88, + 0x9A, 0x03, 0xE4, 0xA9, 0xAE, 0x03, 0xE4, 0xA9, + 0xB6, 0x03, 0xE9, 0x9F, 0xA0, 0x04, 0xF0, 0xA9, + 0x90, 0x8A, 0x03, 0xE4, 0xAA, 0xB2, 0x04, 0xF0, + 0xA9, 0x92, 0x96, 0x03, 0xE9, 0xA0, 0xA9, 0x04, + 0xF0, 0xA9, 0x96, 0xB6, 0x03, 0xE9, 0xA3, 0xA2, + 0x03, 0xE4, 0xAC, 0xB3, 0x03, 0xE9, 0xA4, 0xA9, + 0x03, 0xE9, 0xA6, 0xA7, 0x03, 0xE9, 0xA7, 0x82, + // Bytes 4480 - 44bf + 0x03, 0xE9, 0xA7, 0xBE, 0x03, 0xE4, 0xAF, 0x8E, + 0x04, 0xF0, 0xA9, 0xAC, 0xB0, 0x03, 0xE9, 0xB1, + 0x80, 0x03, 0xE9, 0xB3, 0xBD, 0x03, 0xE4, 0xB3, + 0x8E, 0x03, 0xE4, 0xB3, 0xAD, 0x03, 0xE9, 0xB5, + 0xA7, 0x04, 0xF0, 0xAA, 0x83, 0x8E, 0x03, 0xE4, + 0xB3, 0xB8, 0x04, 0xF0, 0xAA, 0x84, 0x85, 0x04, + 0xF0, 0xAA, 0x88, 0x8E, 0x04, 0xF0, 0xAA, 0x8A, + 0x91, 0x03, 0xE4, 0xB5, 0x96, 0x03, 0xE9, 0xBB, + // Bytes 44c0 - 44ff + 0xBE, 0x03, 0xE9, 0xBC, 0x85, 0x03, 0xE9, 0xBC, + 0x8F, 0x03, 0xE9, 0xBC, 0x96, 0x04, 0xF0, 0xAA, + 0x98, 0x80, +} + +// nfcDecompValues: 4992 entries, 9984 bytes +// Block 2 is the null block. +var nfcDecompValues = [4992]uint16{ + // Block 0x0, offset 0x0 + // Block 0x1, offset 0x40 + // Block 0x2, offset 0x80 + // Block 0x3, offset 0xc0 + 0x00c0: 0x0032, 0x00c1: 0x0036, 0x00c2: 0x003a, 0x00c3: 0x003e, 0x00c4: 0x0042, 0x00c5: 0x0046, + 0x00c7: 0x004a, 0x00c8: 0x004e, 0x00c9: 0x0052, 0x00ca: 0x0056, 0x00cb: 0x005a, + 0x00cc: 0x005e, 0x00cd: 0x0062, 0x00ce: 0x0066, 0x00cf: 0x006a, 0x00d1: 0x006e, + 0x00d2: 0x0072, 0x00d3: 0x0076, 0x00d4: 0x007a, 0x00d5: 0x007e, 0x00d6: 0x0082, + 0x00d9: 0x0086, 0x00da: 0x008a, 0x00db: 0x008e, 0x00dc: 0x0092, 0x00dd: 0x0096, + 0x00e0: 0x009a, 0x00e1: 0x009e, 0x00e2: 0x00a2, 0x00e3: 0x00a6, + 0x00e4: 0x00aa, 0x00e5: 0x00ae, 0x00e7: 0x00b2, 0x00e8: 0x00b6, 0x00e9: 0x00ba, + 0x00ea: 0x00be, 0x00eb: 0x00c2, 0x00ec: 0x00c6, 0x00ed: 0x00ca, 0x00ee: 0x00ce, 0x00ef: 0x00d2, + 0x00f1: 0x00d6, 0x00f2: 0x00da, 0x00f3: 0x00de, 0x00f4: 0x00e2, 0x00f5: 0x00e6, + 0x00f6: 0x00ea, 0x00f9: 0x00ee, 0x00fa: 0x00f2, 0x00fb: 0x00f6, + 0x00fc: 0x00fa, 0x00fd: 0x00fe, 0x00ff: 0x0102, + // Block 0x4, offset 0x100 + 0x0100: 0x0106, 0x0101: 0x010a, 0x0102: 0x010e, 0x0103: 0x0112, 0x0104: 0x0116, 0x0105: 0x011a, + 0x0106: 0x011e, 0x0107: 0x0122, 0x0108: 0x0126, 0x0109: 0x012a, 0x010a: 0x012e, 0x010b: 0x0132, + 0x010c: 0x0136, 0x010d: 0x013a, 0x010e: 0x013e, 0x010f: 0x0142, + 0x0112: 0x0146, 0x0113: 0x014a, 0x0114: 0x014e, 0x0115: 0x0152, 0x0116: 0x0156, 0x0117: 0x015a, + 0x0118: 0x015e, 0x0119: 0x0162, 0x011a: 0x0166, 0x011b: 0x016a, 0x011c: 0x016e, 0x011d: 0x0172, + 0x011e: 0x0176, 0x011f: 0x017a, 0x0120: 0x017e, 0x0121: 0x0182, 0x0122: 0x0186, 0x0123: 0x018a, + 0x0124: 0x018e, 0x0125: 0x0192, 0x0128: 0x0196, 0x0129: 0x019a, + 0x012a: 0x019e, 0x012b: 0x01a2, 0x012c: 0x01a6, 0x012d: 0x01aa, 0x012e: 0x01ae, 0x012f: 0x01b2, + 0x0130: 0x01b6, 0x0134: 0x01c0, 0x0135: 0x01c4, + 0x0136: 0x01c8, 0x0137: 0x01cc, 0x0139: 0x01d0, 0x013a: 0x01d4, 0x013b: 0x01d8, + 0x013c: 0x01dc, 0x013d: 0x01e0, 0x013e: 0x01e4, + // Block 0x5, offset 0x140 + 0x0143: 0x01f0, 0x0144: 0x01f4, 0x0145: 0x01f8, + 0x0146: 0x01fc, 0x0147: 0x0200, 0x0148: 0x0204, + 0x014c: 0x020c, 0x014d: 0x0210, 0x014e: 0x0214, 0x014f: 0x0218, 0x0150: 0x021c, 0x0151: 0x0220, + 0x0154: 0x0224, 0x0155: 0x0228, 0x0156: 0x022c, 0x0157: 0x0230, + 0x0158: 0x0234, 0x0159: 0x0238, 0x015a: 0x023c, 0x015b: 0x0240, 0x015c: 0x0244, 0x015d: 0x0248, + 0x015e: 0x024c, 0x015f: 0x0250, 0x0160: 0x0254, 0x0161: 0x0258, 0x0162: 0x025c, 0x0163: 0x0260, + 0x0164: 0x0264, 0x0165: 0x0268, 0x0168: 0x026c, 0x0169: 0x0270, + 0x016a: 0x0274, 0x016b: 0x0278, 0x016c: 0x027c, 0x016d: 0x0280, 0x016e: 0x0284, 0x016f: 0x0288, + 0x0170: 0x028c, 0x0171: 0x0290, 0x0172: 0x0294, 0x0173: 0x0298, 0x0174: 0x029c, 0x0175: 0x02a0, + 0x0176: 0x02a4, 0x0177: 0x02a8, 0x0178: 0x02ac, 0x0179: 0x02b0, 0x017a: 0x02b4, 0x017b: 0x02b8, + 0x017c: 0x02bc, 0x017d: 0x02c0, 0x017e: 0x02c4, + // Block 0x6, offset 0x180 + 0x01a0: 0x02ca, 0x01a1: 0x02ce, + 0x01af: 0x02d2, + 0x01b0: 0x02d6, + // Block 0x7, offset 0x1c0 + 0x01cd: 0x02fb, 0x01ce: 0x02ff, 0x01cf: 0x0303, 0x01d0: 0x0307, 0x01d1: 0x030b, + 0x01d2: 0x030f, 0x01d3: 0x0313, 0x01d4: 0x0317, 0x01d5: 0x031b, 0x01d6: 0x0321, 0x01d7: 0x0327, + 0x01d8: 0x032d, 0x01d9: 0x0333, 0x01da: 0x0339, 0x01db: 0x033f, 0x01dc: 0x0345, + 0x01de: 0x034b, 0x01df: 0x0351, 0x01e0: 0x0357, 0x01e1: 0x035d, 0x01e2: 0x0363, 0x01e3: 0x0368, + 0x01e6: 0x036d, 0x01e7: 0x0371, 0x01e8: 0x0375, 0x01e9: 0x0379, + 0x01ea: 0x037d, 0x01eb: 0x0381, 0x01ec: 0x0385, 0x01ed: 0x038b, 0x01ee: 0x0391, 0x01ef: 0x0396, + 0x01f0: 0x039b, 0x01f4: 0x03a8, 0x01f5: 0x03ac, + 0x01f8: 0x03b0, 0x01f9: 0x03b4, 0x01fa: 0x03b8, 0x01fb: 0x03be, + 0x01fc: 0x03c4, 0x01fd: 0x03c9, 0x01fe: 0x03ce, 0x01ff: 0x03d3, + // Block 0x8, offset 0x200 + 0x0200: 0x03d8, 0x0201: 0x03dc, 0x0202: 0x03e0, 0x0203: 0x03e4, 0x0204: 0x03e8, 0x0205: 0x03ec, + 0x0206: 0x03f0, 0x0207: 0x03f4, 0x0208: 0x03f8, 0x0209: 0x03fc, 0x020a: 0x0400, 0x020b: 0x0404, + 0x020c: 0x0408, 0x020d: 0x040c, 0x020e: 0x0410, 0x020f: 0x0414, 0x0210: 0x0418, 0x0211: 0x041c, + 0x0212: 0x0420, 0x0213: 0x0424, 0x0214: 0x0428, 0x0215: 0x042c, 0x0216: 0x0430, 0x0217: 0x0434, + 0x0218: 0x0438, 0x0219: 0x043c, 0x021a: 0x0440, 0x021b: 0x0444, + 0x021e: 0x0448, 0x021f: 0x044c, + 0x0226: 0x0450, 0x0227: 0x0454, 0x0228: 0x0458, 0x0229: 0x045c, + 0x022a: 0x0460, 0x022b: 0x0466, 0x022c: 0x046c, 0x022d: 0x0472, 0x022e: 0x0478, 0x022f: 0x047c, + 0x0230: 0x0480, 0x0231: 0x0486, 0x0232: 0x048c, 0x0233: 0x0490, + // Block 0x9, offset 0x240 + 0x0240: 0x04cc, 0x0241: 0x04cf, 0x0243: 0x04d2, 0x0244: 0x04d5, + 0x0274: 0x04da, + 0x027e: 0x04e1, + // Block 0xa, offset 0x280 + 0x0285: 0x04e3, + 0x0286: 0x04ee, 0x0287: 0x04f3, 0x0288: 0x04f6, 0x0289: 0x04fb, 0x028a: 0x0500, + 0x028c: 0x0505, 0x028e: 0x050a, 0x028f: 0x050f, 0x0290: 0x0514, + 0x02aa: 0x051b, 0x02ab: 0x0520, 0x02ac: 0x0525, 0x02ad: 0x052a, 0x02ae: 0x052f, 0x02af: 0x0534, + 0x02b0: 0x0539, + // Block 0xb, offset 0x2c0 + 0x02ca: 0x0540, 0x02cb: 0x0545, + 0x02cc: 0x054a, 0x02cd: 0x054f, 0x02ce: 0x0554, + 0x02d3: 0x0562, 0x02d4: 0x0567, + // Block 0xc, offset 0x300 + 0x0300: 0x0584, 0x0301: 0x0589, 0x0303: 0x058e, + 0x0307: 0x0593, + 0x030c: 0x0598, 0x030d: 0x059d, 0x030e: 0x05a2, + 0x0319: 0x05a7, + 0x0339: 0x05ac, + // Block 0xd, offset 0x340 + 0x0350: 0x05b1, 0x0351: 0x05b6, + 0x0353: 0x05bb, 0x0357: 0x05c0, + 0x035c: 0x05c5, 0x035d: 0x05ca, + 0x035e: 0x05cf, + 0x0376: 0x05d4, 0x0377: 0x05d9, + // Block 0xe, offset 0x380 + 0x0381: 0x05de, 0x0382: 0x05e3, + 0x0390: 0x05e8, 0x0391: 0x05ed, + 0x0392: 0x05f2, 0x0393: 0x05f7, 0x0396: 0x05fc, 0x0397: 0x0601, + 0x039a: 0x0606, 0x039b: 0x060b, 0x039c: 0x0610, 0x039d: 0x0615, + 0x039e: 0x061a, 0x039f: 0x061f, 0x03a2: 0x0624, 0x03a3: 0x0629, + 0x03a4: 0x062e, 0x03a5: 0x0633, 0x03a6: 0x0638, 0x03a7: 0x063d, + 0x03aa: 0x0642, 0x03ab: 0x0647, 0x03ac: 0x064c, 0x03ad: 0x0651, 0x03ae: 0x0656, 0x03af: 0x065b, + 0x03b0: 0x0660, 0x03b1: 0x0665, 0x03b2: 0x066a, 0x03b3: 0x066f, 0x03b4: 0x0674, 0x03b5: 0x0679, + 0x03b8: 0x067e, 0x03b9: 0x0683, + // Block 0xf, offset 0x3c0 + 0x03e2: 0x068d, 0x03e3: 0x0692, + 0x03e4: 0x0697, 0x03e5: 0x069c, 0x03e6: 0x06a1, + // Block 0x10, offset 0x400 + 0x0400: 0x06ba, 0x0402: 0x06bf, + 0x0413: 0x06c4, + // Block 0x11, offset 0x440 + 0x0469: 0x06c9, + 0x0471: 0x06d0, 0x0474: 0x06d7, + // Block 0x12, offset 0x480 + 0x0498: 0x06de, 0x0499: 0x06e5, 0x049a: 0x06ec, 0x049b: 0x06f3, 0x049c: 0x06fa, 0x049d: 0x0701, + 0x049e: 0x0708, 0x049f: 0x070f, + // Block 0x13, offset 0x4c0 + 0x04cb: 0x0716, + 0x04cc: 0x071d, + 0x04dc: 0x0724, 0x04dd: 0x072b, + 0x04df: 0x0732, + // Block 0x14, offset 0x500 + 0x0533: 0x0739, + 0x0536: 0x0740, + // Block 0x15, offset 0x540 + 0x0559: 0x0747, 0x055a: 0x074e, 0x055b: 0x0755, + 0x055e: 0x075c, + // Block 0x16, offset 0x580 + 0x0588: 0x0763, 0x058b: 0x076a, + 0x058c: 0x0771, + 0x059c: 0x0778, 0x059d: 0x077f, + // Block 0x17, offset 0x5c0 + 0x05d4: 0x0786, + // Block 0x18, offset 0x600 + 0x060a: 0x078d, 0x060b: 0x0794, + 0x060c: 0x079b, + // Block 0x19, offset 0x640 + 0x0648: 0x07a2, + // Block 0x1a, offset 0x680 + 0x0680: 0x07a9, + 0x0687: 0x07b0, 0x0688: 0x07b7, 0x068a: 0x07be, 0x068b: 0x07c5, + // Block 0x1b, offset 0x6c0 + 0x06ca: 0x07cf, 0x06cb: 0x07d6, + 0x06cc: 0x07dd, + // Block 0x1c, offset 0x700 + 0x071a: 0x07e4, 0x071c: 0x07eb, 0x071d: 0x07f2, + 0x071e: 0x07fc, + // Block 0x1d, offset 0x740 + 0x0743: 0x0823, + 0x074d: 0x082a, + 0x0752: 0x0831, 0x0757: 0x0838, + 0x075c: 0x083f, + 0x0769: 0x0846, + 0x0773: 0x084d, 0x0775: 0x0854, + 0x0776: 0x085b, 0x0778: 0x086c, + // Block 0x1e, offset 0x780 + 0x0781: 0x087d, + 0x0793: 0x0884, + 0x079d: 0x088b, + 0x07a2: 0x0892, + 0x07a7: 0x0899, + 0x07ac: 0x08a0, + 0x07b9: 0x08a7, + // Block 0x1f, offset 0x7c0 + 0x07e6: 0x08ae, + // Block 0x20, offset 0x800 + 0x0806: 0x08b9, 0x0808: 0x08c0, 0x080a: 0x08c7, + 0x080c: 0x08ce, 0x080e: 0x08d5, + 0x0812: 0x08dc, + 0x083b: 0x08e3, + 0x083d: 0x08ea, + // Block 0x21, offset 0x840 + 0x0840: 0x08f1, 0x0841: 0x08f8, 0x0843: 0x08ff, + // Block 0x22, offset 0x880 + 0x0880: 0x09ea, 0x0881: 0x09ee, 0x0882: 0x09f2, 0x0883: 0x09f6, 0x0884: 0x09fa, 0x0885: 0x09fe, + 0x0886: 0x0a02, 0x0887: 0x0a06, 0x0888: 0x0a0a, 0x0889: 0x0a10, 0x088a: 0x0a16, 0x088b: 0x0a1a, + 0x088c: 0x0a1e, 0x088d: 0x0a22, 0x088e: 0x0a26, 0x088f: 0x0a2a, 0x0890: 0x0a2e, 0x0891: 0x0a32, + 0x0892: 0x0a36, 0x0893: 0x0a3a, 0x0894: 0x0a3e, 0x0895: 0x0a44, 0x0896: 0x0a4a, 0x0897: 0x0a50, + 0x0898: 0x0a56, 0x0899: 0x0a5a, 0x089a: 0x0a5e, 0x089b: 0x0a62, 0x089c: 0x0a66, 0x089d: 0x0a6c, + 0x089e: 0x0a72, 0x089f: 0x0a76, 0x08a0: 0x0a7a, 0x08a1: 0x0a7e, 0x08a2: 0x0a82, 0x08a3: 0x0a86, + 0x08a4: 0x0a8a, 0x08a5: 0x0a8e, 0x08a6: 0x0a92, 0x08a7: 0x0a96, 0x08a8: 0x0a9a, 0x08a9: 0x0a9e, + 0x08aa: 0x0aa2, 0x08ab: 0x0aa6, 0x08ac: 0x0aaa, 0x08ad: 0x0aae, 0x08ae: 0x0ab2, 0x08af: 0x0ab8, + 0x08b0: 0x0abe, 0x08b1: 0x0ac2, 0x08b2: 0x0ac6, 0x08b3: 0x0aca, 0x08b4: 0x0ace, 0x08b5: 0x0ad2, + 0x08b6: 0x0ad6, 0x08b7: 0x0ada, 0x08b8: 0x0ade, 0x08b9: 0x0ae4, 0x08ba: 0x0aea, 0x08bb: 0x0aee, + 0x08bc: 0x0af2, 0x08bd: 0x0af6, 0x08be: 0x0afa, 0x08bf: 0x0afe, + // Block 0x23, offset 0x8c0 + 0x08c0: 0x0b02, 0x08c1: 0x0b06, 0x08c2: 0x0b0a, 0x08c3: 0x0b0e, 0x08c4: 0x0b12, 0x08c5: 0x0b16, + 0x08c6: 0x0b1a, 0x08c7: 0x0b1e, 0x08c8: 0x0b22, 0x08c9: 0x0b26, 0x08ca: 0x0b2a, 0x08cb: 0x0b2e, + 0x08cc: 0x0b32, 0x08cd: 0x0b38, 0x08ce: 0x0b3e, 0x08cf: 0x0b44, 0x08d0: 0x0b4a, 0x08d1: 0x0b50, + 0x08d2: 0x0b56, 0x08d3: 0x0b5c, 0x08d4: 0x0b62, 0x08d5: 0x0b66, 0x08d6: 0x0b6a, 0x08d7: 0x0b6e, + 0x08d8: 0x0b72, 0x08d9: 0x0b76, 0x08da: 0x0b7a, 0x08db: 0x0b7e, 0x08dc: 0x0b82, 0x08dd: 0x0b88, + 0x08de: 0x0b8e, 0x08df: 0x0b92, 0x08e0: 0x0b96, 0x08e1: 0x0b9a, 0x08e2: 0x0b9e, 0x08e3: 0x0ba2, + 0x08e4: 0x0ba6, 0x08e5: 0x0bac, 0x08e6: 0x0bb2, 0x08e7: 0x0bb8, 0x08e8: 0x0bbe, 0x08e9: 0x0bc4, + 0x08ea: 0x0bca, 0x08eb: 0x0bce, 0x08ec: 0x0bd2, 0x08ed: 0x0bd6, 0x08ee: 0x0bda, 0x08ef: 0x0bde, + 0x08f0: 0x0be2, 0x08f1: 0x0be6, 0x08f2: 0x0bea, 0x08f3: 0x0bee, 0x08f4: 0x0bf2, 0x08f5: 0x0bf6, + 0x08f6: 0x0bfa, 0x08f7: 0x0bfe, 0x08f8: 0x0c02, 0x08f9: 0x0c08, 0x08fa: 0x0c0e, 0x08fb: 0x0c14, + 0x08fc: 0x0c1a, 0x08fd: 0x0c1e, 0x08fe: 0x0c22, 0x08ff: 0x0c26, + // Block 0x24, offset 0x900 + 0x0900: 0x0c2a, 0x0901: 0x0c2e, 0x0902: 0x0c32, 0x0903: 0x0c36, 0x0904: 0x0c3a, 0x0905: 0x0c3e, + 0x0906: 0x0c42, 0x0907: 0x0c46, 0x0908: 0x0c4a, 0x0909: 0x0c4e, 0x090a: 0x0c52, 0x090b: 0x0c56, + 0x090c: 0x0c5a, 0x090d: 0x0c5e, 0x090e: 0x0c62, 0x090f: 0x0c66, 0x0910: 0x0c6a, 0x0911: 0x0c6e, + 0x0912: 0x0c72, 0x0913: 0x0c76, 0x0914: 0x0c7a, 0x0915: 0x0c7e, 0x0916: 0x0c82, 0x0917: 0x0c86, + 0x0918: 0x0c8a, 0x0919: 0x0c8e, 0x091b: 0x0c96, + 0x0920: 0x0c9b, 0x0921: 0x0c9f, 0x0922: 0x0ca3, 0x0923: 0x0ca7, + 0x0924: 0x0cab, 0x0925: 0x0cb1, 0x0926: 0x0cb7, 0x0927: 0x0cbd, 0x0928: 0x0cc3, 0x0929: 0x0cc9, + 0x092a: 0x0ccf, 0x092b: 0x0cd5, 0x092c: 0x0cdb, 0x092d: 0x0ce1, 0x092e: 0x0ce7, 0x092f: 0x0ced, + 0x0930: 0x0cf3, 0x0931: 0x0cf9, 0x0932: 0x0cff, 0x0933: 0x0d05, 0x0934: 0x0d0b, 0x0935: 0x0d11, + 0x0936: 0x0d17, 0x0937: 0x0d1d, 0x0938: 0x0d23, 0x0939: 0x0d27, 0x093a: 0x0d2b, 0x093b: 0x0d2f, + 0x093c: 0x0d33, 0x093d: 0x0d37, 0x093e: 0x0d3b, 0x093f: 0x0d41, + // Block 0x25, offset 0x940 + 0x0940: 0x0d47, 0x0941: 0x0d4d, 0x0942: 0x0d53, 0x0943: 0x0d59, 0x0944: 0x0d5f, 0x0945: 0x0d65, + 0x0946: 0x0d6b, 0x0947: 0x0d71, 0x0948: 0x0d77, 0x0949: 0x0d7b, 0x094a: 0x0d7f, 0x094b: 0x0d83, + 0x094c: 0x0d87, 0x094d: 0x0d8b, 0x094e: 0x0d8f, 0x094f: 0x0d93, 0x0950: 0x0d97, 0x0951: 0x0d9d, + 0x0952: 0x0da3, 0x0953: 0x0da9, 0x0954: 0x0daf, 0x0955: 0x0db5, 0x0956: 0x0dbb, 0x0957: 0x0dc1, + 0x0958: 0x0dc7, 0x0959: 0x0dcd, 0x095a: 0x0dd3, 0x095b: 0x0dd9, 0x095c: 0x0ddf, 0x095d: 0x0de5, + 0x095e: 0x0deb, 0x095f: 0x0df1, 0x0960: 0x0df7, 0x0961: 0x0dfd, 0x0962: 0x0e03, 0x0963: 0x0e09, + 0x0964: 0x0e0f, 0x0965: 0x0e13, 0x0966: 0x0e17, 0x0967: 0x0e1b, 0x0968: 0x0e1f, 0x0969: 0x0e25, + 0x096a: 0x0e2b, 0x096b: 0x0e31, 0x096c: 0x0e37, 0x096d: 0x0e3d, 0x096e: 0x0e43, 0x096f: 0x0e49, + 0x0970: 0x0e4f, 0x0971: 0x0e55, 0x0972: 0x0e5b, 0x0973: 0x0e5f, 0x0974: 0x0e63, 0x0975: 0x0e67, + 0x0976: 0x0e6b, 0x0977: 0x0e6f, 0x0978: 0x0e73, 0x0979: 0x0e77, + // Block 0x26, offset 0x980 + 0x0980: 0x0e7b, 0x0981: 0x0e80, 0x0982: 0x0e85, 0x0983: 0x0e8c, 0x0984: 0x0e93, 0x0985: 0x0e9a, + 0x0986: 0x0ea1, 0x0987: 0x0ea8, 0x0988: 0x0eaf, 0x0989: 0x0eb4, 0x098a: 0x0eb9, 0x098b: 0x0ec0, + 0x098c: 0x0ec7, 0x098d: 0x0ece, 0x098e: 0x0ed5, 0x098f: 0x0edc, 0x0990: 0x0ee3, 0x0991: 0x0ee8, + 0x0992: 0x0eed, 0x0993: 0x0ef4, 0x0994: 0x0efb, 0x0995: 0x0f02, + 0x0998: 0x0f09, 0x0999: 0x0f0e, 0x099a: 0x0f13, 0x099b: 0x0f1a, 0x099c: 0x0f21, 0x099d: 0x0f28, + 0x09a0: 0x0f2f, 0x09a1: 0x0f34, 0x09a2: 0x0f39, 0x09a3: 0x0f40, + 0x09a4: 0x0f47, 0x09a5: 0x0f4e, 0x09a6: 0x0f55, 0x09a7: 0x0f5c, 0x09a8: 0x0f63, 0x09a9: 0x0f68, + 0x09aa: 0x0f6d, 0x09ab: 0x0f74, 0x09ac: 0x0f7b, 0x09ad: 0x0f82, 0x09ae: 0x0f89, 0x09af: 0x0f90, + 0x09b0: 0x0f97, 0x09b1: 0x0f9c, 0x09b2: 0x0fa1, 0x09b3: 0x0fa8, 0x09b4: 0x0faf, 0x09b5: 0x0fb6, + 0x09b6: 0x0fbd, 0x09b7: 0x0fc4, 0x09b8: 0x0fcb, 0x09b9: 0x0fd0, 0x09ba: 0x0fd5, 0x09bb: 0x0fdc, + 0x09bc: 0x0fe3, 0x09bd: 0x0fea, 0x09be: 0x0ff1, 0x09bf: 0x0ff8, + // Block 0x27, offset 0x9c0 + 0x09c0: 0x0fff, 0x09c1: 0x1004, 0x09c2: 0x1009, 0x09c3: 0x1010, 0x09c4: 0x1017, 0x09c5: 0x101e, + 0x09c8: 0x1025, 0x09c9: 0x102a, 0x09ca: 0x102f, 0x09cb: 0x1036, + 0x09cc: 0x103d, 0x09cd: 0x1044, 0x09d0: 0x104b, 0x09d1: 0x1050, + 0x09d2: 0x1055, 0x09d3: 0x105c, 0x09d4: 0x1063, 0x09d5: 0x106a, 0x09d6: 0x1071, 0x09d7: 0x1078, + 0x09d9: 0x107f, 0x09db: 0x1084, 0x09dd: 0x108b, + 0x09df: 0x1092, 0x09e0: 0x1099, 0x09e1: 0x109e, 0x09e2: 0x10a3, 0x09e3: 0x10aa, + 0x09e4: 0x10b1, 0x09e5: 0x10b8, 0x09e6: 0x10bf, 0x09e7: 0x10c6, 0x09e8: 0x10cd, 0x09e9: 0x10d2, + 0x09ea: 0x10d7, 0x09eb: 0x10de, 0x09ec: 0x10e5, 0x09ed: 0x10ec, 0x09ee: 0x10f3, 0x09ef: 0x10fa, + 0x09f0: 0x1101, 0x09f1: 0x0525, 0x09f2: 0x1106, 0x09f3: 0x052a, 0x09f4: 0x110b, 0x09f5: 0x052f, + 0x09f6: 0x1110, 0x09f7: 0x0534, 0x09f8: 0x1115, 0x09f9: 0x054a, 0x09fa: 0x111a, 0x09fb: 0x054f, + 0x09fc: 0x111f, 0x09fd: 0x0554, + // Block 0x28, offset 0xa00 + 0x0a00: 0x1124, 0x0a01: 0x112b, 0x0a02: 0x1132, 0x0a03: 0x113b, 0x0a04: 0x1144, 0x0a05: 0x114d, + 0x0a06: 0x1156, 0x0a07: 0x115f, 0x0a08: 0x1168, 0x0a09: 0x116f, 0x0a0a: 0x1176, 0x0a0b: 0x117f, + 0x0a0c: 0x1188, 0x0a0d: 0x1191, 0x0a0e: 0x119a, 0x0a0f: 0x11a3, 0x0a10: 0x11ac, 0x0a11: 0x11b3, + 0x0a12: 0x11ba, 0x0a13: 0x11c3, 0x0a14: 0x11cc, 0x0a15: 0x11d5, 0x0a16: 0x11de, 0x0a17: 0x11e7, + 0x0a18: 0x11f0, 0x0a19: 0x11f7, 0x0a1a: 0x11fe, 0x0a1b: 0x1207, 0x0a1c: 0x1210, 0x0a1d: 0x1219, + 0x0a1e: 0x1222, 0x0a1f: 0x122b, 0x0a20: 0x1234, 0x0a21: 0x123b, 0x0a22: 0x1242, 0x0a23: 0x124b, + 0x0a24: 0x1254, 0x0a25: 0x125d, 0x0a26: 0x1266, 0x0a27: 0x126f, 0x0a28: 0x1278, 0x0a29: 0x127f, + 0x0a2a: 0x1286, 0x0a2b: 0x128f, 0x0a2c: 0x1298, 0x0a2d: 0x12a1, 0x0a2e: 0x12aa, 0x0a2f: 0x12b3, + 0x0a30: 0x12bc, 0x0a31: 0x12c1, 0x0a32: 0x12c6, 0x0a33: 0x12cd, 0x0a34: 0x12d2, + 0x0a36: 0x12d9, 0x0a37: 0x12de, 0x0a38: 0x12e5, 0x0a39: 0x12ea, 0x0a3a: 0x12ef, 0x0a3b: 0x04ee, + 0x0a3c: 0x12f4, 0x0a3e: 0x12fd, + // Block 0x29, offset 0xa40 + 0x0a41: 0x1304, 0x0a42: 0x130f, 0x0a43: 0x1316, 0x0a44: 0x131b, + 0x0a46: 0x1322, 0x0a47: 0x1327, 0x0a48: 0x132e, 0x0a49: 0x04f6, 0x0a4a: 0x1333, 0x0a4b: 0x04fb, + 0x0a4c: 0x1338, 0x0a4d: 0x133d, 0x0a4e: 0x1349, 0x0a4f: 0x1355, 0x0a50: 0x1361, 0x0a51: 0x1366, + 0x0a52: 0x136b, 0x0a53: 0x0514, 0x0a56: 0x1372, 0x0a57: 0x1377, + 0x0a58: 0x137e, 0x0a59: 0x1383, 0x0a5a: 0x1388, 0x0a5b: 0x0500, 0x0a5d: 0x138d, + 0x0a5e: 0x1399, 0x0a5f: 0x13a5, 0x0a60: 0x13b1, 0x0a61: 0x13b6, 0x0a62: 0x13bb, 0x0a63: 0x0539, + 0x0a64: 0x13c2, 0x0a65: 0x13c7, 0x0a66: 0x13cc, 0x0a67: 0x13d1, 0x0a68: 0x13d8, 0x0a69: 0x13dd, + 0x0a6a: 0x13e2, 0x0a6b: 0x050a, 0x0a6c: 0x13e7, 0x0a6d: 0x13ec, 0x0a6e: 0x04e3, 0x0a6f: 0x13f7, + 0x0a72: 0x13f9, 0x0a73: 0x1400, 0x0a74: 0x1405, + 0x0a76: 0x140c, 0x0a77: 0x1411, 0x0a78: 0x1418, 0x0a79: 0x0505, 0x0a7a: 0x141d, 0x0a7b: 0x050f, + 0x0a7c: 0x1422, 0x0a7d: 0x1427, + // Block 0x2a, offset 0xa80 + 0x0a80: 0x142e, 0x0a81: 0x1432, + // Block 0x2b, offset 0xac0 + 0x0ae6: 0x14d6, + 0x0aea: 0x091c, 0x0aeb: 0x0046, + // Block 0x2c, offset 0xb00 + 0x0b1a: 0x159f, 0x0b1b: 0x15a5, + 0x0b2e: 0x15ab, + // Block 0x2d, offset 0xb40 + 0x0b4d: 0x15b1, 0x0b4e: 0x15b7, 0x0b4f: 0x15bd, + // Block 0x2e, offset 0xb80 + 0x0b84: 0x15c3, + 0x0b89: 0x15c9, + 0x0b8c: 0x15cf, + 0x0ba4: 0x15d5, 0x0ba6: 0x15db, + // Block 0x2f, offset 0xbc0 + 0x0bc1: 0x1603, 0x0bc4: 0x1609, + 0x0bc7: 0x160f, 0x0bc9: 0x1615, + 0x0be0: 0x161b, 0x0be2: 0x161f, + 0x0bed: 0x1625, 0x0bee: 0x162b, 0x0bef: 0x162f, + 0x0bf0: 0x1633, 0x0bf1: 0x1639, 0x0bf4: 0x163f, 0x0bf5: 0x1645, + 0x0bf8: 0x164b, 0x0bf9: 0x1651, + // Block 0x30, offset 0xc00 + 0x0c00: 0x1657, 0x0c01: 0x165d, 0x0c04: 0x1663, 0x0c05: 0x1669, + 0x0c08: 0x166f, 0x0c09: 0x1675, + 0x0c2c: 0x167b, 0x0c2d: 0x1681, 0x0c2e: 0x1687, 0x0c2f: 0x168d, + // Block 0x31, offset 0xc40 + 0x0c60: 0x1693, 0x0c61: 0x1699, 0x0c62: 0x169f, 0x0c63: 0x16a5, + 0x0c6a: 0x16ab, 0x0c6b: 0x16b1, 0x0c6c: 0x16b7, 0x0c6d: 0x16bd, + // Block 0x32, offset 0xc80 + 0x0ca9: 0x16c3, + 0x0caa: 0x16c7, + // Block 0x33, offset 0xcc0 + 0x0cdc: 0x1814, + // Block 0x34, offset 0xd00 + 0x0d0c: 0x1b8a, 0x0d0e: 0x1b91, 0x0d10: 0x1b98, + 0x0d12: 0x1b9f, 0x0d14: 0x1ba6, 0x0d16: 0x1bad, + 0x0d18: 0x1bb4, 0x0d1a: 0x1bbb, 0x0d1c: 0x1bc2, + 0x0d1e: 0x1bc9, 0x0d20: 0x1bd0, 0x0d22: 0x1bd7, + 0x0d25: 0x1bde, 0x0d27: 0x1be5, 0x0d29: 0x1bec, + 0x0d30: 0x1bf3, 0x0d31: 0x1bfa, 0x0d33: 0x1c01, 0x0d34: 0x1c08, + 0x0d36: 0x1c0f, 0x0d37: 0x1c16, 0x0d39: 0x1c1d, 0x0d3a: 0x1c24, + 0x0d3c: 0x1c2b, 0x0d3d: 0x1c32, + // Block 0x35, offset 0xd40 + 0x0d54: 0x1c39, + 0x0d5e: 0x1c4a, + 0x0d6c: 0x1c58, 0x0d6e: 0x1c5f, + 0x0d70: 0x1c66, 0x0d72: 0x1c6d, 0x0d74: 0x1c74, + 0x0d76: 0x1c7b, 0x0d78: 0x1c82, 0x0d7a: 0x1c89, + 0x0d7c: 0x1c90, 0x0d7e: 0x1c97, + // Block 0x36, offset 0xd80 + 0x0d80: 0x1c9e, 0x0d82: 0x1ca5, 0x0d85: 0x1cac, + 0x0d87: 0x1cb3, 0x0d89: 0x1cba, + 0x0d90: 0x1cc1, 0x0d91: 0x1cc8, + 0x0d93: 0x1ccf, 0x0d94: 0x1cd6, 0x0d96: 0x1cdd, 0x0d97: 0x1ce4, + 0x0d99: 0x1ceb, 0x0d9a: 0x1cf2, 0x0d9c: 0x1cf9, 0x0d9d: 0x1d00, + 0x0db4: 0x1d07, + 0x0db7: 0x1d0e, 0x0db8: 0x1d15, 0x0db9: 0x1d1c, 0x0dba: 0x1d23, + 0x0dbe: 0x1d2a, + // Block 0x37, offset 0xdc0 + 0x0dc0: 0x2a81, 0x0dc1: 0x2a85, 0x0dc2: 0x1a9e, 0x0dc3: 0x2a89, 0x0dc4: 0x2a8d, 0x0dc5: 0x2a91, + 0x0dc6: 0x2a95, 0x0dc7: 0x1b76, 0x0dc8: 0x1b76, 0x0dc9: 0x2a99, 0x0dca: 0x1abe, 0x0dcb: 0x2a9d, + 0x0dcc: 0x2aa1, 0x0dcd: 0x2aa5, 0x0dce: 0x2aa9, 0x0dcf: 0x2aad, 0x0dd0: 0x2ab1, 0x0dd1: 0x2ab5, + 0x0dd2: 0x2ab9, 0x0dd3: 0x2abd, 0x0dd4: 0x2ac1, 0x0dd5: 0x2ac5, 0x0dd6: 0x2ac9, 0x0dd7: 0x2acd, + 0x0dd8: 0x2ad1, 0x0dd9: 0x2ad5, 0x0dda: 0x2ad9, 0x0ddb: 0x2add, 0x0ddc: 0x2ae1, 0x0ddd: 0x2ae5, + 0x0dde: 0x2ae9, 0x0ddf: 0x2aed, 0x0de0: 0x2af1, 0x0de1: 0x2af5, 0x0de2: 0x2af9, 0x0de3: 0x2afd, + 0x0de4: 0x2b01, 0x0de5: 0x2b05, 0x0de6: 0x2b09, 0x0de7: 0x2b0d, 0x0de8: 0x2b11, 0x0de9: 0x2b15, + 0x0dea: 0x2b19, 0x0deb: 0x2b1d, 0x0dec: 0x2b21, 0x0ded: 0x2b25, 0x0dee: 0x2b29, 0x0def: 0x2b2d, + 0x0df0: 0x2b31, 0x0df1: 0x2b35, 0x0df2: 0x2b39, 0x0df3: 0x2b3d, 0x0df4: 0x1a16, 0x0df5: 0x2b41, + 0x0df6: 0x2b45, 0x0df7: 0x2b49, 0x0df8: 0x2b4d, 0x0df9: 0x2b51, 0x0dfa: 0x2b55, 0x0dfb: 0x2b59, + 0x0dfc: 0x2b5d, 0x0dfd: 0x2b61, 0x0dfe: 0x2b65, 0x0dff: 0x2b69, + // Block 0x38, offset 0xe00 + 0x0e00: 0x1b3a, 0x0e01: 0x2b6d, 0x0e02: 0x2b71, 0x0e03: 0x2b75, 0x0e04: 0x2b79, 0x0e05: 0x2b7d, + 0x0e06: 0x2b81, 0x0e07: 0x2b85, 0x0e08: 0x2b89, 0x0e09: 0x2b8d, 0x0e0a: 0x2b91, 0x0e0b: 0x2b95, + 0x0e0c: 0x2b99, 0x0e0d: 0x2b9d, 0x0e0e: 0x2ba1, 0x0e0f: 0x2ba5, 0x0e10: 0x2ba9, 0x0e11: 0x2bad, + 0x0e12: 0x2bb1, 0x0e13: 0x2bb5, 0x0e14: 0x2bb9, 0x0e15: 0x2bbd, 0x0e16: 0x2bc1, 0x0e17: 0x2bc5, + 0x0e18: 0x2bc9, 0x0e19: 0x2bcd, 0x0e1a: 0x2bd1, 0x0e1b: 0x2bd5, 0x0e1c: 0x2ac1, 0x0e1d: 0x2bd9, + 0x0e1e: 0x2bdd, 0x0e1f: 0x2be1, 0x0e20: 0x2be5, 0x0e21: 0x2be9, 0x0e22: 0x2bed, 0x0e23: 0x2bf1, + 0x0e24: 0x2bf5, 0x0e25: 0x2bf9, 0x0e26: 0x2bfd, 0x0e27: 0x2c01, 0x0e28: 0x2c05, 0x0e29: 0x2c09, + 0x0e2a: 0x2c0d, 0x0e2b: 0x2c11, 0x0e2c: 0x2c15, 0x0e2d: 0x2c19, 0x0e2e: 0x2c1d, 0x0e2f: 0x2c21, + 0x0e30: 0x2c25, 0x0e31: 0x1aa6, 0x0e32: 0x2c29, 0x0e33: 0x2c2d, 0x0e34: 0x2c31, 0x0e35: 0x2c35, + 0x0e36: 0x2c39, 0x0e37: 0x2c3d, 0x0e38: 0x2c41, 0x0e39: 0x2c45, 0x0e3a: 0x2c49, 0x0e3b: 0x2c4d, + 0x0e3c: 0x2c51, 0x0e3d: 0x2c55, 0x0e3e: 0x2c59, 0x0e3f: 0x2c5d, + // Block 0x39, offset 0xe40 + 0x0e40: 0x2c61, 0x0e41: 0x18ba, 0x0e42: 0x2c65, 0x0e43: 0x2c69, 0x0e44: 0x2c6d, 0x0e45: 0x2c71, + 0x0e46: 0x2c75, 0x0e47: 0x2c79, 0x0e48: 0x2c7d, 0x0e49: 0x2c81, 0x0e4a: 0x186e, 0x0e4b: 0x2c85, + 0x0e4c: 0x2c89, 0x0e4d: 0x2c8d, 0x0e4e: 0x2c91, 0x0e4f: 0x2c95, 0x0e50: 0x2c99, 0x0e51: 0x2c9d, + 0x0e52: 0x2ca1, 0x0e53: 0x2ca5, 0x0e54: 0x2ca9, 0x0e55: 0x2cad, 0x0e56: 0x2cb1, 0x0e57: 0x2cb5, + 0x0e58: 0x2cb9, 0x0e59: 0x2cbd, 0x0e5a: 0x2cc1, 0x0e5b: 0x2cc5, 0x0e5c: 0x2cc9, 0x0e5d: 0x2ccd, + 0x0e5e: 0x2cd1, 0x0e5f: 0x2cd5, 0x0e60: 0x2cd9, 0x0e61: 0x2c21, 0x0e62: 0x2cdd, 0x0e63: 0x2ce1, + 0x0e64: 0x2ce5, 0x0e65: 0x2ce9, 0x0e66: 0x2ced, 0x0e67: 0x2cf1, 0x0e68: 0x2cf5, 0x0e69: 0x2cf9, + 0x0e6a: 0x2be1, 0x0e6b: 0x2cfd, 0x0e6c: 0x2d01, 0x0e6d: 0x2d05, 0x0e6e: 0x2d09, 0x0e6f: 0x2d0d, + 0x0e70: 0x2d11, 0x0e71: 0x2d15, 0x0e72: 0x2d19, 0x0e73: 0x2d1d, 0x0e74: 0x2d21, 0x0e75: 0x2d25, + 0x0e76: 0x2d29, 0x0e77: 0x2d2d, 0x0e78: 0x2d31, 0x0e79: 0x2d35, 0x0e7a: 0x2d39, 0x0e7b: 0x2d3d, + 0x0e7c: 0x2d41, 0x0e7d: 0x2d45, 0x0e7e: 0x2d49, 0x0e7f: 0x2ac1, + // Block 0x3a, offset 0xe80 + 0x0e80: 0x2d4d, 0x0e81: 0x2d51, 0x0e82: 0x2d55, 0x0e83: 0x2d59, 0x0e84: 0x1b72, 0x0e85: 0x2d5d, + 0x0e86: 0x2d61, 0x0e87: 0x2d65, 0x0e88: 0x2d69, 0x0e89: 0x2d6d, 0x0e8a: 0x2d71, 0x0e8b: 0x2d75, + 0x0e8c: 0x2d79, 0x0e8d: 0x2d7d, 0x0e8e: 0x2d81, 0x0e8f: 0x2d85, 0x0e90: 0x2d89, 0x0e91: 0x2173, + 0x0e92: 0x2d8d, 0x0e93: 0x2d91, 0x0e94: 0x2d95, 0x0e95: 0x2d99, 0x0e96: 0x2d9d, 0x0e97: 0x2da1, + 0x0e98: 0x2da5, 0x0e99: 0x2da9, 0x0e9a: 0x2dad, 0x0e9b: 0x2be9, 0x0e9c: 0x2db1, 0x0e9d: 0x2db5, + 0x0e9e: 0x2db9, 0x0e9f: 0x2dbd, 0x0ea0: 0x2dc1, 0x0ea1: 0x2dc5, 0x0ea2: 0x2dc9, 0x0ea3: 0x2dcd, + 0x0ea4: 0x2dd1, 0x0ea5: 0x2dd5, 0x0ea6: 0x2dd9, 0x0ea7: 0x2ddd, 0x0ea8: 0x2de1, 0x0ea9: 0x1aba, + 0x0eaa: 0x2de5, 0x0eab: 0x2de9, 0x0eac: 0x2ded, 0x0ead: 0x2df1, 0x0eae: 0x2df5, 0x0eaf: 0x2df9, + 0x0eb0: 0x2dfd, 0x0eb1: 0x2e01, 0x0eb2: 0x2e05, 0x0eb3: 0x2e09, 0x0eb4: 0x2e0d, 0x0eb5: 0x2e11, + 0x0eb6: 0x2e15, 0x0eb7: 0x19f6, 0x0eb8: 0x2e19, 0x0eb9: 0x2e1d, 0x0eba: 0x2e21, 0x0ebb: 0x2e25, + 0x0ebc: 0x2e29, 0x0ebd: 0x2e2d, 0x0ebe: 0x2e31, 0x0ebf: 0x2e35, + // Block 0x3b, offset 0xec0 + 0x0ec0: 0x2e39, 0x0ec1: 0x2e3d, 0x0ec2: 0x2e41, 0x0ec3: 0x2e45, 0x0ec4: 0x2e49, 0x0ec5: 0x2e4d, + 0x0ec6: 0x2e51, 0x0ec7: 0x2e55, 0x0ec8: 0x1a62, 0x0ec9: 0x2e59, 0x0eca: 0x1a6e, 0x0ecb: 0x2e5d, + 0x0ecc: 0x2e61, 0x0ecd: 0x2e65, 0x0ed0: 0x2e69, + 0x0ed2: 0x2e6d, 0x0ed5: 0x2e71, 0x0ed6: 0x2e75, 0x0ed7: 0x2e79, + 0x0ed8: 0x2e7d, 0x0ed9: 0x2e81, 0x0eda: 0x2e85, 0x0edb: 0x2e89, 0x0edc: 0x2e8d, 0x0edd: 0x2e91, + 0x0ede: 0x1a12, 0x0ee0: 0x2e95, 0x0ee2: 0x2e99, + 0x0ee5: 0x2e9d, 0x0ee6: 0x2ea1, + 0x0eea: 0x2ea5, 0x0eeb: 0x2ea9, 0x0eec: 0x2ead, 0x0eed: 0x2eb1, + 0x0ef0: 0x2eb5, 0x0ef1: 0x2eb9, 0x0ef2: 0x2ebd, 0x0ef3: 0x2ec1, 0x0ef4: 0x2ec5, 0x0ef5: 0x2ec9, + 0x0ef6: 0x2ecd, 0x0ef7: 0x2ed1, 0x0ef8: 0x2ed5, 0x0ef9: 0x2ed9, 0x0efa: 0x2edd, 0x0efb: 0x2ee1, + 0x0efc: 0x18d6, 0x0efd: 0x2ee5, 0x0efe: 0x2ee9, 0x0eff: 0x2eed, + // Block 0x3c, offset 0xf00 + 0x0f00: 0x2ef1, 0x0f01: 0x2ef5, 0x0f02: 0x2ef9, 0x0f03: 0x2efd, 0x0f04: 0x2f01, 0x0f05: 0x2f05, + 0x0f06: 0x2f09, 0x0f07: 0x2f0d, 0x0f08: 0x2f11, 0x0f09: 0x2f15, 0x0f0a: 0x2f19, 0x0f0b: 0x2f1d, + 0x0f0c: 0x2187, 0x0f0d: 0x2f21, 0x0f0e: 0x2f25, 0x0f0f: 0x2f29, 0x0f10: 0x2f2d, 0x0f11: 0x2197, + 0x0f12: 0x2f31, 0x0f13: 0x2f35, 0x0f14: 0x2f39, 0x0f15: 0x2f3d, 0x0f16: 0x2f41, 0x0f17: 0x2cb1, + 0x0f18: 0x2f45, 0x0f19: 0x2f49, 0x0f1a: 0x2f4d, 0x0f1b: 0x2f51, 0x0f1c: 0x2f55, 0x0f1d: 0x2f59, + 0x0f1e: 0x2f59, 0x0f1f: 0x2f5d, 0x0f20: 0x2f61, 0x0f21: 0x2f65, 0x0f22: 0x2f69, 0x0f23: 0x2f6d, + 0x0f24: 0x2f71, 0x0f25: 0x2f75, 0x0f26: 0x2f79, 0x0f27: 0x2e9d, 0x0f28: 0x2f7d, 0x0f29: 0x2f81, + 0x0f2a: 0x2f85, 0x0f2b: 0x2f89, 0x0f2c: 0x2f8d, 0x0f2d: 0x2f92, + 0x0f30: 0x2f96, 0x0f31: 0x2f9a, 0x0f32: 0x2f9e, 0x0f33: 0x2fa2, 0x0f34: 0x2fa6, 0x0f35: 0x2faa, + 0x0f36: 0x2fae, 0x0f37: 0x2fb2, 0x0f38: 0x2ecd, 0x0f39: 0x2fb6, 0x0f3a: 0x2fba, 0x0f3b: 0x2fbe, + 0x0f3c: 0x2e69, 0x0f3d: 0x2fc2, 0x0f3e: 0x2fc6, 0x0f3f: 0x2fca, + // Block 0x3d, offset 0xf40 + 0x0f40: 0x2fce, 0x0f41: 0x2fd2, 0x0f42: 0x2fd6, 0x0f43: 0x2fda, 0x0f44: 0x2fde, 0x0f45: 0x2fe2, + 0x0f46: 0x2fe6, 0x0f47: 0x2fea, 0x0f48: 0x2fee, 0x0f49: 0x2eed, 0x0f4a: 0x2ff2, 0x0f4b: 0x2ef1, + 0x0f4c: 0x2ff6, 0x0f4d: 0x2ffa, 0x0f4e: 0x2ffe, 0x0f4f: 0x3002, 0x0f50: 0x3006, 0x0f51: 0x2e6d, + 0x0f52: 0x2b15, 0x0f53: 0x300a, 0x0f54: 0x300e, 0x0f55: 0x195a, 0x0f56: 0x2c25, 0x0f57: 0x2d71, + 0x0f58: 0x3012, 0x0f59: 0x3016, 0x0f5a: 0x2f0d, 0x0f5b: 0x301a, 0x0f5c: 0x2f11, 0x0f5d: 0x301e, + 0x0f5e: 0x3022, 0x0f5f: 0x3026, 0x0f60: 0x2e75, 0x0f61: 0x302a, 0x0f62: 0x302e, 0x0f63: 0x3032, + 0x0f64: 0x3036, 0x0f65: 0x303a, 0x0f66: 0x2e79, 0x0f67: 0x303e, 0x0f68: 0x3042, 0x0f69: 0x3046, + 0x0f6a: 0x304a, 0x0f6b: 0x304e, 0x0f6c: 0x3052, 0x0f6d: 0x2f41, 0x0f6e: 0x3056, 0x0f6f: 0x305a, + 0x0f70: 0x2cb1, 0x0f71: 0x305e, 0x0f72: 0x2f51, 0x0f73: 0x3062, 0x0f74: 0x3066, 0x0f75: 0x306a, + 0x0f76: 0x306e, 0x0f77: 0x3072, 0x0f78: 0x2f65, 0x0f79: 0x3076, 0x0f7a: 0x2e99, 0x0f7b: 0x307a, + 0x0f7c: 0x2f69, 0x0f7d: 0x2bd9, 0x0f7e: 0x307e, 0x0f7f: 0x2f6d, + // Block 0x3e, offset 0xf80 + 0x0f80: 0x3082, 0x0f81: 0x2f75, 0x0f82: 0x3086, 0x0f83: 0x308a, 0x0f84: 0x308e, 0x0f85: 0x3092, + 0x0f86: 0x3096, 0x0f87: 0x2f7d, 0x0f88: 0x2e8d, 0x0f89: 0x309a, 0x0f8a: 0x2f81, 0x0f8b: 0x309e, + 0x0f8c: 0x2f85, 0x0f8d: 0x30a2, 0x0f8e: 0x1b76, 0x0f8f: 0x30a6, 0x0f90: 0x30ab, 0x0f91: 0x30b0, + 0x0f92: 0x30b5, 0x0f93: 0x30b9, 0x0f94: 0x30bd, 0x0f95: 0x30c1, 0x0f96: 0x30c6, 0x0f97: 0x30cb, + 0x0f98: 0x30d0, 0x0f99: 0x30d4, + // Block 0x3f, offset 0xfc0 + 0x0fdd: 0x3105, + 0x0fdf: 0x310a, + 0x0fea: 0x3124, 0x0feb: 0x3129, 0x0fec: 0x312e, 0x0fed: 0x3135, 0x0fee: 0x313c, 0x0fef: 0x3141, + 0x0ff0: 0x3146, 0x0ff1: 0x314b, 0x0ff2: 0x3150, 0x0ff3: 0x3155, 0x0ff4: 0x315a, 0x0ff5: 0x315f, + 0x0ff6: 0x3164, 0x0ff8: 0x3169, 0x0ff9: 0x316e, 0x0ffa: 0x3173, 0x0ffb: 0x3178, + 0x0ffc: 0x317d, 0x0ffe: 0x3182, + // Block 0x40, offset 0x1000 + 0x1000: 0x3187, 0x1001: 0x318c, 0x1003: 0x3191, 0x1004: 0x3196, + 0x1006: 0x319b, 0x1007: 0x31a0, 0x1008: 0x31a5, 0x1009: 0x31aa, 0x100a: 0x31af, 0x100b: 0x31b4, + 0x100c: 0x31b9, 0x100d: 0x31be, 0x100e: 0x31c3, + // Block 0x41, offset 0x1040 + 0x105a: 0x3a73, 0x105c: 0x3a7c, + 0x106b: 0x3a85, + // Block 0x42, offset 0x1080 + 0x109e: 0x3a8e, 0x109f: 0x3a97, 0x10a0: 0x3aa0, 0x10a1: 0x3aad, 0x10a2: 0x3aba, 0x10a3: 0x3ac7, + 0x10a4: 0x3ad4, + // Block 0x43, offset 0x10c0 + 0x10fb: 0x3ae1, + 0x10fc: 0x3aea, 0x10fd: 0x3af3, 0x10fe: 0x3b00, 0x10ff: 0x3b0d, + // Block 0x44, offset 0x1100 + 0x1100: 0x3b1a, + // Block 0x45, offset 0x1140 + 0x1140: 0x3d23, 0x1141: 0x3d27, 0x1142: 0x3d2b, 0x1143: 0x3d2f, 0x1144: 0x3d34, 0x1145: 0x2eb5, + 0x1146: 0x3d38, 0x1147: 0x3d3c, 0x1148: 0x3d40, 0x1149: 0x3d44, 0x114a: 0x2eb9, 0x114b: 0x3d48, + 0x114c: 0x3d4c, 0x114d: 0x3d50, 0x114e: 0x2ebd, 0x114f: 0x3d55, 0x1150: 0x3d59, 0x1151: 0x3d5d, + 0x1152: 0x3d61, 0x1153: 0x3d66, 0x1154: 0x3d6a, 0x1155: 0x3c71, 0x1156: 0x3d6e, 0x1157: 0x3d73, + 0x1158: 0x3d77, 0x1159: 0x3d7b, 0x115a: 0x3d7f, 0x115b: 0x2f9a, 0x115c: 0x3d83, 0x115d: 0x1866, + 0x115e: 0x3d88, 0x115f: 0x3d8c, 0x1160: 0x3d90, 0x1161: 0x3d94, 0x1162: 0x3cb9, 0x1163: 0x3d98, + 0x1164: 0x3d9c, 0x1165: 0x2fae, 0x1166: 0x2ec1, 0x1167: 0x2ec5, 0x1168: 0x2fb2, 0x1169: 0x3da0, + 0x116a: 0x3da4, 0x116b: 0x2bf1, 0x116c: 0x3da8, 0x116d: 0x2ec9, 0x116e: 0x3dac, 0x116f: 0x3db0, + 0x1170: 0x3db4, 0x1171: 0x3db8, 0x1172: 0x3db8, 0x1173: 0x3db8, 0x1174: 0x3dbc, 0x1175: 0x3dc1, + 0x1176: 0x3dc5, 0x1177: 0x3dc9, 0x1178: 0x3dcd, 0x1179: 0x3dd2, 0x117a: 0x3dd6, 0x117b: 0x3dda, + 0x117c: 0x3dde, 0x117d: 0x3de2, 0x117e: 0x3de6, 0x117f: 0x3dea, + // Block 0x46, offset 0x1180 + 0x1180: 0x3dee, 0x1181: 0x3df2, 0x1182: 0x3df6, 0x1183: 0x3dfa, 0x1184: 0x3dfe, 0x1185: 0x3e02, + 0x1186: 0x3e02, 0x1187: 0x2fba, 0x1188: 0x3e06, 0x1189: 0x3e0a, 0x118a: 0x3e0e, 0x118b: 0x3e12, + 0x118c: 0x2ed1, 0x118d: 0x3e16, 0x118e: 0x3e1a, 0x118f: 0x3e1e, 0x1190: 0x2e39, 0x1191: 0x3e22, + 0x1192: 0x3e26, 0x1193: 0x3e2a, 0x1194: 0x3e2e, 0x1195: 0x3e32, 0x1196: 0x3e36, 0x1197: 0x3e3a, + 0x1198: 0x3e3e, 0x1199: 0x3e42, 0x119a: 0x3e47, 0x119b: 0x3e4b, 0x119c: 0x3e4f, 0x119d: 0x3c55, + 0x119e: 0x3e53, 0x119f: 0x3e57, 0x11a0: 0x3e5b, 0x11a1: 0x3e60, 0x11a2: 0x3e65, 0x11a3: 0x3e69, + 0x11a4: 0x3e6d, 0x11a5: 0x3e71, 0x11a6: 0x3e75, 0x11a7: 0x3e79, 0x11a8: 0x3e7d, 0x11a9: 0x3e81, + 0x11aa: 0x3e85, 0x11ab: 0x3e85, 0x11ac: 0x3e89, 0x11ad: 0x3e8e, 0x11ae: 0x3e92, 0x11af: 0x2be1, + 0x11b0: 0x3e96, 0x11b1: 0x3e9a, 0x11b2: 0x3e9f, 0x11b3: 0x3ea3, 0x11b4: 0x3ea7, 0x11b5: 0x18ce, + 0x11b6: 0x3eab, 0x11b7: 0x3eaf, 0x11b8: 0x18d6, 0x11b9: 0x3eb3, 0x11ba: 0x3eb7, 0x11bb: 0x3ebb, + 0x11bc: 0x3ec0, 0x11bd: 0x3ec4, 0x11be: 0x3ec9, 0x11bf: 0x3ecd, + // Block 0x47, offset 0x11c0 + 0x11c0: 0x3ed1, 0x11c1: 0x3ed5, 0x11c2: 0x3ed9, 0x11c3: 0x3edd, 0x11c4: 0x3ee1, 0x11c5: 0x3ee5, + 0x11c6: 0x3ee9, 0x11c7: 0x3eed, 0x11c8: 0x3ef1, 0x11c9: 0x3ef5, 0x11ca: 0x3efa, 0x11cb: 0x3efe, + 0x11cc: 0x3f02, 0x11cd: 0x3f06, 0x11ce: 0x2b11, 0x11cf: 0x3f0a, 0x11d0: 0x18fe, 0x11d1: 0x3f0f, + 0x11d2: 0x3f0f, 0x11d3: 0x3f14, 0x11d4: 0x3f18, 0x11d5: 0x3f18, 0x11d6: 0x3f1c, 0x11d7: 0x3f20, + 0x11d8: 0x3f25, 0x11d9: 0x3f2a, 0x11da: 0x3f2e, 0x11db: 0x3f32, 0x11dc: 0x3f36, 0x11dd: 0x3f3a, + 0x11de: 0x3f3e, 0x11df: 0x3f42, 0x11e0: 0x3f46, 0x11e1: 0x3f4a, 0x11e2: 0x3f4e, 0x11e3: 0x2ee5, + 0x11e4: 0x3f52, 0x11e5: 0x3f57, 0x11e6: 0x3f5b, 0x11e7: 0x3f5f, 0x11e8: 0x2fea, 0x11e9: 0x3f5f, + 0x11ea: 0x3f63, 0x11eb: 0x2eed, 0x11ec: 0x3f67, 0x11ed: 0x3f6b, 0x11ee: 0x3f6f, 0x11ef: 0x3f73, + 0x11f0: 0x2ef1, 0x11f1: 0x2aa5, 0x11f2: 0x3f77, 0x11f3: 0x3f7b, 0x11f4: 0x3f7f, 0x11f5: 0x3f83, + 0x11f6: 0x3f87, 0x11f7: 0x3f8b, 0x11f8: 0x3f8f, 0x11f9: 0x3f94, 0x11fa: 0x3f98, 0x11fb: 0x3f9c, + 0x11fc: 0x3fa0, 0x11fd: 0x3fa4, 0x11fe: 0x3fa8, 0x11ff: 0x3fad, + // Block 0x48, offset 0x1200 + 0x1200: 0x3fb1, 0x1201: 0x3fb5, 0x1202: 0x3fb9, 0x1203: 0x3fbd, 0x1204: 0x3fc1, 0x1205: 0x3fc5, + 0x1206: 0x3fc9, 0x1207: 0x3fcd, 0x1208: 0x2ef5, 0x1209: 0x3fd1, 0x120a: 0x3fd5, 0x120b: 0x3fda, + 0x120c: 0x3fde, 0x120d: 0x3fe2, 0x120e: 0x3fe6, 0x120f: 0x2efd, 0x1210: 0x3fea, 0x1211: 0x3fee, + 0x1212: 0x3ff2, 0x1213: 0x3ff6, 0x1214: 0x3ffa, 0x1215: 0x3ffe, 0x1216: 0x4002, 0x1217: 0x4006, + 0x1218: 0x2b15, 0x1219: 0x300a, 0x121a: 0x400a, 0x121b: 0x400e, 0x121c: 0x4012, 0x121d: 0x4016, + 0x121e: 0x401b, 0x121f: 0x401f, 0x1220: 0x4023, 0x1221: 0x4027, 0x1222: 0x2f01, 0x1223: 0x402b, + 0x1224: 0x4030, 0x1225: 0x4034, 0x1226: 0x4038, 0x1227: 0x30b5, 0x1228: 0x403c, 0x1229: 0x4040, + 0x122a: 0x4044, 0x122b: 0x4048, 0x122c: 0x404c, 0x122d: 0x4051, 0x122e: 0x4055, 0x122f: 0x4059, + 0x1230: 0x405d, 0x1231: 0x4062, 0x1232: 0x4066, 0x1233: 0x406a, 0x1234: 0x406e, 0x1235: 0x2c25, + 0x1236: 0x4072, 0x1237: 0x4076, 0x1238: 0x407b, 0x1239: 0x4080, 0x123a: 0x4085, 0x123b: 0x4089, + 0x123c: 0x408e, 0x123d: 0x4092, 0x123e: 0x4096, 0x123f: 0x409a, + // Block 0x49, offset 0x1240 + 0x1240: 0x409e, 0x1241: 0x2f05, 0x1242: 0x2d71, 0x1243: 0x40a2, 0x1244: 0x40a6, 0x1245: 0x40aa, + 0x1246: 0x40ae, 0x1247: 0x40b3, 0x1248: 0x40b7, 0x1249: 0x40bb, 0x124a: 0x40bf, 0x124b: 0x3016, + 0x124c: 0x40c3, 0x124d: 0x40c7, 0x124e: 0x40cc, 0x124f: 0x40d0, 0x1250: 0x40d4, 0x1251: 0x40d9, + 0x1252: 0x40de, 0x1253: 0x40e2, 0x1254: 0x301a, 0x1255: 0x40e6, 0x1256: 0x40ea, 0x1257: 0x40ee, + 0x1258: 0x40f2, 0x1259: 0x40f6, 0x125a: 0x40fa, 0x125b: 0x40fe, 0x125c: 0x4103, 0x125d: 0x4107, + 0x125e: 0x410c, 0x125f: 0x4110, 0x1260: 0x4115, 0x1261: 0x3022, 0x1262: 0x4119, 0x1263: 0x411d, + 0x1264: 0x4122, 0x1265: 0x4126, 0x1266: 0x412a, 0x1267: 0x412f, 0x1268: 0x4134, 0x1269: 0x4138, + 0x126a: 0x413c, 0x126b: 0x4140, 0x126c: 0x4144, 0x126d: 0x4144, 0x126e: 0x4148, 0x126f: 0x414c, + 0x1270: 0x302a, 0x1271: 0x4150, 0x1272: 0x4154, 0x1273: 0x4158, 0x1274: 0x415c, 0x1275: 0x4160, + 0x1276: 0x4165, 0x1277: 0x4169, 0x1278: 0x2bed, 0x1279: 0x416e, 0x127a: 0x4173, 0x127b: 0x4177, + 0x127c: 0x417c, 0x127d: 0x4181, 0x127e: 0x4186, 0x127f: 0x418a, + // Block 0x4a, offset 0x1280 + 0x1280: 0x3042, 0x1281: 0x418e, 0x1282: 0x4193, 0x1283: 0x4198, 0x1284: 0x419d, 0x1285: 0x41a2, + 0x1286: 0x41a6, 0x1287: 0x41a6, 0x1288: 0x3046, 0x1289: 0x30bd, 0x128a: 0x41aa, 0x128b: 0x41ae, + 0x128c: 0x41b2, 0x128d: 0x41b6, 0x128e: 0x41bb, 0x128f: 0x2b59, 0x1290: 0x304e, 0x1291: 0x41bf, + 0x1292: 0x41c3, 0x1293: 0x2f2d, 0x1294: 0x41c8, 0x1295: 0x41cd, 0x1296: 0x2e89, 0x1297: 0x41d2, + 0x1298: 0x41d6, 0x1299: 0x2f39, 0x129a: 0x41da, 0x129b: 0x41de, 0x129c: 0x41e2, 0x129d: 0x41e7, + 0x129e: 0x41e7, 0x129f: 0x41ec, 0x12a0: 0x41f0, 0x12a1: 0x41f4, 0x12a2: 0x41f9, 0x12a3: 0x41fd, + 0x12a4: 0x4201, 0x12a5: 0x4205, 0x12a6: 0x420a, 0x12a7: 0x420e, 0x12a8: 0x4212, 0x12a9: 0x4216, + 0x12aa: 0x421a, 0x12ab: 0x421e, 0x12ac: 0x4223, 0x12ad: 0x4227, 0x12ae: 0x422b, 0x12af: 0x422f, + 0x12b0: 0x4233, 0x12b1: 0x4237, 0x12b2: 0x423b, 0x12b3: 0x4240, 0x12b4: 0x4245, 0x12b5: 0x4249, + 0x12b6: 0x424e, 0x12b7: 0x4252, 0x12b8: 0x4257, 0x12b9: 0x425b, 0x12ba: 0x2f51, 0x12bb: 0x425f, + 0x12bc: 0x4264, 0x12bd: 0x4269, 0x12be: 0x426d, 0x12bf: 0x4272, + // Block 0x4b, offset 0x12c0 + 0x12c0: 0x4276, 0x12c1: 0x427b, 0x12c2: 0x427f, 0x12c3: 0x4283, 0x12c4: 0x4287, 0x12c5: 0x428b, + 0x12c6: 0x428f, 0x12c7: 0x4293, 0x12c8: 0x4298, 0x12c9: 0x429d, 0x12ca: 0x42a2, 0x12cb: 0x3f14, + 0x12cc: 0x42a7, 0x12cd: 0x42ab, 0x12ce: 0x42af, 0x12cf: 0x42b3, 0x12d0: 0x42b7, 0x12d1: 0x42bb, + 0x12d2: 0x42bf, 0x12d3: 0x42c3, 0x12d4: 0x42c7, 0x12d5: 0x42cb, 0x12d6: 0x42cf, 0x12d7: 0x42d3, + 0x12d8: 0x2c31, 0x12d9: 0x42d8, 0x12da: 0x42dc, 0x12db: 0x42e0, 0x12dc: 0x42e4, 0x12dd: 0x42e8, + 0x12de: 0x42ec, 0x12df: 0x2f5d, 0x12e0: 0x42f0, 0x12e1: 0x42f4, 0x12e2: 0x42f8, 0x12e3: 0x42fc, + 0x12e4: 0x4300, 0x12e5: 0x4305, 0x12e6: 0x430a, 0x12e7: 0x430f, 0x12e8: 0x4313, 0x12e9: 0x4317, + 0x12ea: 0x431b, 0x12eb: 0x431f, 0x12ec: 0x4324, 0x12ed: 0x4328, 0x12ee: 0x432d, 0x12ef: 0x4331, + 0x12f0: 0x4335, 0x12f1: 0x433a, 0x12f2: 0x433f, 0x12f3: 0x4343, 0x12f4: 0x2b45, 0x12f5: 0x4347, + 0x12f6: 0x434b, 0x12f7: 0x434f, 0x12f8: 0x4353, 0x12f9: 0x4357, 0x12fa: 0x435b, 0x12fb: 0x306a, + 0x12fc: 0x435f, 0x12fd: 0x4363, 0x12fe: 0x4367, 0x12ff: 0x436b, + // Block 0x4c, offset 0x1300 + 0x1300: 0x436f, 0x1301: 0x4373, 0x1302: 0x4377, 0x1303: 0x437b, 0x1304: 0x1a66, 0x1305: 0x437f, + 0x1306: 0x4384, 0x1307: 0x4388, 0x1308: 0x438c, 0x1309: 0x4390, 0x130a: 0x4394, 0x130b: 0x4398, + 0x130c: 0x439d, 0x130d: 0x43a2, 0x130e: 0x43a6, 0x130f: 0x43aa, 0x1310: 0x307e, 0x1311: 0x3082, + 0x1312: 0x1a82, 0x1313: 0x43ae, 0x1314: 0x43b3, 0x1315: 0x43b7, 0x1316: 0x43bb, 0x1317: 0x43bf, + 0x1318: 0x43c3, 0x1319: 0x43c8, 0x131a: 0x43cd, 0x131b: 0x43d1, 0x131c: 0x43d5, 0x131d: 0x43d9, + 0x131e: 0x43de, 0x131f: 0x3086, 0x1320: 0x43e2, 0x1321: 0x43e7, 0x1322: 0x43ec, 0x1323: 0x43f0, + 0x1324: 0x43f4, 0x1325: 0x43f8, 0x1326: 0x43fd, 0x1327: 0x4401, 0x1328: 0x4405, 0x1329: 0x4409, + 0x132a: 0x440d, 0x132b: 0x4411, 0x132c: 0x4415, 0x132d: 0x4419, 0x132e: 0x441e, 0x132f: 0x4422, + 0x1330: 0x4426, 0x1331: 0x442a, 0x1332: 0x442f, 0x1333: 0x4433, 0x1334: 0x4437, 0x1335: 0x443b, + 0x1336: 0x443f, 0x1337: 0x4444, 0x1338: 0x4449, 0x1339: 0x444d, 0x133a: 0x4451, 0x133b: 0x4455, + 0x133c: 0x445a, 0x133d: 0x445e, 0x133e: 0x309e, 0x133f: 0x309e, + // Block 0x4d, offset 0x1340 + 0x1340: 0x4463, 0x1341: 0x4467, 0x1342: 0x446c, 0x1343: 0x4470, 0x1344: 0x4474, 0x1345: 0x4478, + 0x1346: 0x447c, 0x1347: 0x4480, 0x1348: 0x4484, 0x1349: 0x4488, 0x134a: 0x30a2, 0x134b: 0x448d, + 0x134c: 0x4491, 0x134d: 0x4495, 0x134e: 0x4499, 0x134f: 0x449d, 0x1350: 0x44a1, 0x1351: 0x44a6, + 0x1352: 0x44aa, 0x1353: 0x44af, 0x1354: 0x44b4, 0x1355: 0x1b42, 0x1356: 0x44b9, 0x1357: 0x1b52, + 0x1358: 0x44bd, 0x1359: 0x44c1, 0x135a: 0x44c5, 0x135b: 0x44c9, 0x135c: 0x1b66, 0x135d: 0x44cd, +} + +// nfcDecompLookup: 832 bytes +// Block 0 is the null block. +var nfcDecompLookup = [832]uint8{ + // Block 0x0, offset 0x0 + // Block 0x1, offset 0x40 + // Block 0x2, offset 0x80 + // Block 0x3, offset 0xc0 + 0x0c3: 0x03, 0x0c4: 0x04, 0x0c5: 0x05, 0x0c6: 0x06, 0x0c7: 0x07, + 0x0c8: 0x08, 0x0cd: 0x09, 0x0ce: 0x0a, 0x0cf: 0x0b, + 0x0d0: 0x0c, 0x0d1: 0x0d, 0x0d3: 0x0e, + 0x0d8: 0x0f, 0x0db: 0x10, + 0x0e0: 0x04, 0x0e1: 0x05, 0x0e2: 0x06, 0x0e3: 0x07, + 0x0ef: 0x08, + 0x0f0: 0x0c, + // Block 0x4, offset 0x100 + 0x124: 0x11, 0x125: 0x12, 0x127: 0x13, + 0x128: 0x14, 0x129: 0x15, 0x12d: 0x16, 0x12e: 0x17, 0x12f: 0x18, + 0x131: 0x19, 0x133: 0x1a, 0x135: 0x1b, 0x137: 0x1c, + 0x13d: 0x1d, 0x13e: 0x1e, + // Block 0x5, offset 0x140 + 0x140: 0x1f, + 0x16c: 0x20, 0x16d: 0x21, + 0x178: 0x22, 0x179: 0x23, 0x17a: 0x24, 0x17b: 0x25, 0x17c: 0x26, 0x17d: 0x27, 0x17e: 0x28, 0x17f: 0x29, + // Block 0x6, offset 0x180 + 0x180: 0x2a, 0x184: 0x2b, 0x186: 0x2c, 0x187: 0x2d, + 0x188: 0x2e, 0x189: 0x2f, 0x18a: 0x30, 0x18b: 0x31, 0x18c: 0x32, + 0x1ab: 0x33, + // Block 0x7, offset 0x1c0 + 0x1c1: 0x34, 0x1c2: 0x35, 0x1c3: 0x36, + // Block 0x8, offset 0x200 + 0x224: 0x37, 0x225: 0x38, 0x226: 0x39, 0x227: 0x3a, + 0x228: 0x3b, 0x229: 0x3c, 0x22a: 0x3d, 0x22b: 0x3e, 0x22c: 0x3f, 0x22d: 0x40, + // Block 0x9, offset 0x240 + 0x242: 0x41, + // Block 0xa, offset 0x280 + 0x285: 0x42, 0x286: 0x43, 0x287: 0x44, + // Block 0xb, offset 0x2c0 + 0x2e0: 0x45, 0x2e1: 0x46, 0x2e2: 0x47, 0x2e3: 0x48, 0x2e4: 0x49, 0x2e5: 0x4a, 0x2e6: 0x4b, 0x2e7: 0x4c, + 0x2e8: 0x4d, + // Block 0xc, offset 0x300 + 0x311: 0x09, + 0x31d: 0x0a, + 0x32f: 0x0b, +} + +var nfcDecompTrie = trie{nfcDecompLookup[:], nfcDecompValues[:]} + +// nfkcDecompValues: 10176 entries, 20352 bytes +// Block 2 is the null block. +var nfkcDecompValues = [10176]uint16{ + // Block 0x0, offset 0x0 + // Block 0x1, offset 0x40 + // Block 0x2, offset 0x80 + // Block 0x3, offset 0xc0 + 0x00e0: 0x0001, + 0x00e8: 0x0003, + 0x00ea: 0x0007, 0x00ef: 0x0009, + 0x00f2: 0x000d, 0x00f3: 0x000f, 0x00f4: 0x0011, 0x00f5: 0x0015, + 0x00f8: 0x0018, 0x00f9: 0x001c, 0x00fa: 0x001e, + 0x00fc: 0x0020, 0x00fd: 0x0026, 0x00fe: 0x002c, + // Block 0x4, offset 0x100 + 0x0100: 0x0032, 0x0101: 0x0036, 0x0102: 0x003a, 0x0103: 0x003e, 0x0104: 0x0042, 0x0105: 0x0046, + 0x0107: 0x004a, 0x0108: 0x004e, 0x0109: 0x0052, 0x010a: 0x0056, 0x010b: 0x005a, + 0x010c: 0x005e, 0x010d: 0x0062, 0x010e: 0x0066, 0x010f: 0x006a, 0x0111: 0x006e, + 0x0112: 0x0072, 0x0113: 0x0076, 0x0114: 0x007a, 0x0115: 0x007e, 0x0116: 0x0082, + 0x0119: 0x0086, 0x011a: 0x008a, 0x011b: 0x008e, 0x011c: 0x0092, 0x011d: 0x0096, + 0x0120: 0x009a, 0x0121: 0x009e, 0x0122: 0x00a2, 0x0123: 0x00a6, + 0x0124: 0x00aa, 0x0125: 0x00ae, 0x0127: 0x00b2, 0x0128: 0x00b6, 0x0129: 0x00ba, + 0x012a: 0x00be, 0x012b: 0x00c2, 0x012c: 0x00c6, 0x012d: 0x00ca, 0x012e: 0x00ce, 0x012f: 0x00d2, + 0x0131: 0x00d6, 0x0132: 0x00da, 0x0133: 0x00de, 0x0134: 0x00e2, 0x0135: 0x00e6, + 0x0136: 0x00ea, 0x0139: 0x00ee, 0x013a: 0x00f2, 0x013b: 0x00f6, + 0x013c: 0x00fa, 0x013d: 0x00fe, 0x013f: 0x0102, + // Block 0x5, offset 0x140 + 0x0140: 0x0106, 0x0141: 0x010a, 0x0142: 0x010e, 0x0143: 0x0112, 0x0144: 0x0116, 0x0145: 0x011a, + 0x0146: 0x011e, 0x0147: 0x0122, 0x0148: 0x0126, 0x0149: 0x012a, 0x014a: 0x012e, 0x014b: 0x0132, + 0x014c: 0x0136, 0x014d: 0x013a, 0x014e: 0x013e, 0x014f: 0x0142, + 0x0152: 0x0146, 0x0153: 0x014a, 0x0154: 0x014e, 0x0155: 0x0152, 0x0156: 0x0156, 0x0157: 0x015a, + 0x0158: 0x015e, 0x0159: 0x0162, 0x015a: 0x0166, 0x015b: 0x016a, 0x015c: 0x016e, 0x015d: 0x0172, + 0x015e: 0x0176, 0x015f: 0x017a, 0x0160: 0x017e, 0x0161: 0x0182, 0x0162: 0x0186, 0x0163: 0x018a, + 0x0164: 0x018e, 0x0165: 0x0192, 0x0168: 0x0196, 0x0169: 0x019a, + 0x016a: 0x019e, 0x016b: 0x01a2, 0x016c: 0x01a6, 0x016d: 0x01aa, 0x016e: 0x01ae, 0x016f: 0x01b2, + 0x0170: 0x01b6, 0x0172: 0x01ba, 0x0173: 0x01bd, 0x0174: 0x01c0, 0x0175: 0x01c4, + 0x0176: 0x01c8, 0x0177: 0x01cc, 0x0179: 0x01d0, 0x017a: 0x01d4, 0x017b: 0x01d8, + 0x017c: 0x01dc, 0x017d: 0x01e0, 0x017e: 0x01e4, 0x017f: 0x01e8, + // Block 0x6, offset 0x180 + 0x0180: 0x01ec, 0x0183: 0x01f0, 0x0184: 0x01f4, 0x0185: 0x01f8, + 0x0186: 0x01fc, 0x0187: 0x0200, 0x0188: 0x0204, 0x0189: 0x0208, + 0x018c: 0x020c, 0x018d: 0x0210, 0x018e: 0x0214, 0x018f: 0x0218, 0x0190: 0x021c, 0x0191: 0x0220, + 0x0194: 0x0224, 0x0195: 0x0228, 0x0196: 0x022c, 0x0197: 0x0230, + 0x0198: 0x0234, 0x0199: 0x0238, 0x019a: 0x023c, 0x019b: 0x0240, 0x019c: 0x0244, 0x019d: 0x0248, + 0x019e: 0x024c, 0x019f: 0x0250, 0x01a0: 0x0254, 0x01a1: 0x0258, 0x01a2: 0x025c, 0x01a3: 0x0260, + 0x01a4: 0x0264, 0x01a5: 0x0268, 0x01a8: 0x026c, 0x01a9: 0x0270, + 0x01aa: 0x0274, 0x01ab: 0x0278, 0x01ac: 0x027c, 0x01ad: 0x0280, 0x01ae: 0x0284, 0x01af: 0x0288, + 0x01b0: 0x028c, 0x01b1: 0x0290, 0x01b2: 0x0294, 0x01b3: 0x0298, 0x01b4: 0x029c, 0x01b5: 0x02a0, + 0x01b6: 0x02a4, 0x01b7: 0x02a8, 0x01b8: 0x02ac, 0x01b9: 0x02b0, 0x01ba: 0x02b4, 0x01bb: 0x02b8, + 0x01bc: 0x02bc, 0x01bd: 0x02c0, 0x01be: 0x02c4, 0x01bf: 0x02c8, + // Block 0x7, offset 0x1c0 + 0x01e0: 0x02ca, 0x01e1: 0x02ce, + 0x01ef: 0x02d2, + 0x01f0: 0x02d6, + // Block 0x8, offset 0x200 + 0x0204: 0x02da, 0x0205: 0x02df, + 0x0206: 0x02e4, 0x0207: 0x02e9, 0x0208: 0x02ec, 0x0209: 0x02ef, 0x020a: 0x02f2, 0x020b: 0x02f5, + 0x020c: 0x02f8, 0x020d: 0x02fb, 0x020e: 0x02ff, 0x020f: 0x0303, 0x0210: 0x0307, 0x0211: 0x030b, + 0x0212: 0x030f, 0x0213: 0x0313, 0x0214: 0x0317, 0x0215: 0x031b, 0x0216: 0x0321, 0x0217: 0x0327, + 0x0218: 0x032d, 0x0219: 0x0333, 0x021a: 0x0339, 0x021b: 0x033f, 0x021c: 0x0345, + 0x021e: 0x034b, 0x021f: 0x0351, 0x0220: 0x0357, 0x0221: 0x035d, 0x0222: 0x0363, 0x0223: 0x0368, + 0x0226: 0x036d, 0x0227: 0x0371, 0x0228: 0x0375, 0x0229: 0x0379, + 0x022a: 0x037d, 0x022b: 0x0381, 0x022c: 0x0385, 0x022d: 0x038b, 0x022e: 0x0391, 0x022f: 0x0396, + 0x0230: 0x039b, 0x0231: 0x039f, 0x0232: 0x03a2, 0x0233: 0x03a5, 0x0234: 0x03a8, 0x0235: 0x03ac, + 0x0238: 0x03b0, 0x0239: 0x03b4, 0x023a: 0x03b8, 0x023b: 0x03be, + 0x023c: 0x03c4, 0x023d: 0x03c9, 0x023e: 0x03ce, 0x023f: 0x03d3, + // Block 0x9, offset 0x240 + 0x0240: 0x03d8, 0x0241: 0x03dc, 0x0242: 0x03e0, 0x0243: 0x03e4, 0x0244: 0x03e8, 0x0245: 0x03ec, + 0x0246: 0x03f0, 0x0247: 0x03f4, 0x0248: 0x03f8, 0x0249: 0x03fc, 0x024a: 0x0400, 0x024b: 0x0404, + 0x024c: 0x0408, 0x024d: 0x040c, 0x024e: 0x0410, 0x024f: 0x0414, 0x0250: 0x0418, 0x0251: 0x041c, + 0x0252: 0x0420, 0x0253: 0x0424, 0x0254: 0x0428, 0x0255: 0x042c, 0x0256: 0x0430, 0x0257: 0x0434, + 0x0258: 0x0438, 0x0259: 0x043c, 0x025a: 0x0440, 0x025b: 0x0444, + 0x025e: 0x0448, 0x025f: 0x044c, + 0x0266: 0x0450, 0x0267: 0x0454, 0x0268: 0x0458, 0x0269: 0x045c, + 0x026a: 0x0460, 0x026b: 0x0466, 0x026c: 0x046c, 0x026d: 0x0472, 0x026e: 0x0478, 0x026f: 0x047c, + 0x0270: 0x0480, 0x0271: 0x0486, 0x0272: 0x048c, 0x0273: 0x0490, + // Block 0xa, offset 0x280 + 0x02b0: 0x0494, 0x02b1: 0x0496, 0x02b2: 0x0499, 0x02b3: 0x049b, 0x02b4: 0x049d, 0x02b5: 0x04a0, + 0x02b6: 0x04a3, 0x02b7: 0x04a6, 0x02b8: 0x04a8, + // Block 0xb, offset 0x2c0 + 0x02d8: 0x04aa, 0x02d9: 0x04ae, 0x02da: 0x04b2, 0x02db: 0x04b6, 0x02dc: 0x04ba, 0x02dd: 0x04be, + 0x02e0: 0x04c2, 0x02e1: 0x04c5, 0x02e2: 0x02c8, 0x02e3: 0x04c7, + 0x02e4: 0x04c9, + // Block 0xc, offset 0x300 + 0x0300: 0x04cc, 0x0301: 0x04cf, 0x0303: 0x04d2, 0x0304: 0x04d5, + 0x0334: 0x04da, + 0x033a: 0x04dd, + 0x033e: 0x04e1, + // Block 0xd, offset 0x340 + 0x0344: 0x0011, 0x0345: 0x04e8, + 0x0346: 0x04ee, 0x0347: 0x04f3, 0x0348: 0x04f6, 0x0349: 0x04fb, 0x034a: 0x0500, + 0x034c: 0x0505, 0x034e: 0x050a, 0x034f: 0x050f, 0x0350: 0x0514, + 0x036a: 0x051b, 0x036b: 0x0520, 0x036c: 0x0525, 0x036d: 0x052a, 0x036e: 0x052f, 0x036f: 0x0534, + 0x0370: 0x0539, + // Block 0xe, offset 0x380 + 0x038a: 0x0540, 0x038b: 0x0545, + 0x038c: 0x054a, 0x038d: 0x054f, 0x038e: 0x0554, 0x0390: 0x0559, 0x0391: 0x055c, + 0x0392: 0x055f, 0x0393: 0x050a, 0x0394: 0x0520, 0x0395: 0x056c, 0x0396: 0x056f, + 0x03b0: 0x0572, 0x03b1: 0x0575, 0x03b2: 0x0578, 0x03b4: 0x057b, 0x03b5: 0x057e, + 0x03b9: 0x0581, + // Block 0xf, offset 0x3c0 + 0x03c0: 0x0584, 0x03c1: 0x0589, 0x03c3: 0x058e, + 0x03c7: 0x0593, + 0x03cc: 0x0598, 0x03cd: 0x059d, 0x03ce: 0x05a2, + 0x03d9: 0x05a7, + 0x03f9: 0x05ac, + // Block 0x10, offset 0x400 + 0x0410: 0x05b1, 0x0411: 0x05b6, + 0x0413: 0x05bb, 0x0417: 0x05c0, + 0x041c: 0x05c5, 0x041d: 0x05ca, + 0x041e: 0x05cf, + 0x0436: 0x05d4, 0x0437: 0x05d9, + // Block 0x11, offset 0x440 + 0x0441: 0x05de, 0x0442: 0x05e3, + 0x0450: 0x05e8, 0x0451: 0x05ed, + 0x0452: 0x05f2, 0x0453: 0x05f7, 0x0456: 0x05fc, 0x0457: 0x0601, + 0x045a: 0x0606, 0x045b: 0x060b, 0x045c: 0x0610, 0x045d: 0x0615, + 0x045e: 0x061a, 0x045f: 0x061f, 0x0462: 0x0624, 0x0463: 0x0629, + 0x0464: 0x062e, 0x0465: 0x0633, 0x0466: 0x0638, 0x0467: 0x063d, + 0x046a: 0x0642, 0x046b: 0x0647, 0x046c: 0x064c, 0x046d: 0x0651, 0x046e: 0x0656, 0x046f: 0x065b, + 0x0470: 0x0660, 0x0471: 0x0665, 0x0472: 0x066a, 0x0473: 0x066f, 0x0474: 0x0674, 0x0475: 0x0679, + 0x0478: 0x067e, 0x0479: 0x0683, + // Block 0x12, offset 0x480 + 0x0487: 0x0688, + // Block 0x13, offset 0x4c0 + 0x04e2: 0x068d, 0x04e3: 0x0692, + 0x04e4: 0x0697, 0x04e5: 0x069c, 0x04e6: 0x06a1, + // Block 0x14, offset 0x500 + 0x0535: 0x06a6, + 0x0536: 0x06ab, 0x0537: 0x06b0, 0x0538: 0x06b5, + // Block 0x15, offset 0x540 + 0x0540: 0x06ba, 0x0542: 0x06bf, + 0x0553: 0x06c4, + // Block 0x16, offset 0x580 + 0x05a9: 0x06c9, + 0x05b1: 0x06d0, 0x05b4: 0x06d7, + // Block 0x17, offset 0x5c0 + 0x05d8: 0x06de, 0x05d9: 0x06e5, 0x05da: 0x06ec, 0x05db: 0x06f3, 0x05dc: 0x06fa, 0x05dd: 0x0701, + 0x05de: 0x0708, 0x05df: 0x070f, + // Block 0x18, offset 0x600 + 0x060b: 0x0716, + 0x060c: 0x071d, + 0x061c: 0x0724, 0x061d: 0x072b, + 0x061f: 0x0732, + // Block 0x19, offset 0x640 + 0x0673: 0x0739, + 0x0676: 0x0740, + // Block 0x1a, offset 0x680 + 0x0699: 0x0747, 0x069a: 0x074e, 0x069b: 0x0755, + 0x069e: 0x075c, + // Block 0x1b, offset 0x6c0 + 0x06c8: 0x0763, 0x06cb: 0x076a, + 0x06cc: 0x0771, + 0x06dc: 0x0778, 0x06dd: 0x077f, + // Block 0x1c, offset 0x700 + 0x0714: 0x0786, + // Block 0x1d, offset 0x740 + 0x074a: 0x078d, 0x074b: 0x0794, + 0x074c: 0x079b, + // Block 0x1e, offset 0x780 + 0x0788: 0x07a2, + // Block 0x1f, offset 0x7c0 + 0x07c0: 0x07a9, + 0x07c7: 0x07b0, 0x07c8: 0x07b7, 0x07ca: 0x07be, 0x07cb: 0x07c5, + // Block 0x20, offset 0x800 + 0x080a: 0x07cf, 0x080b: 0x07d6, + 0x080c: 0x07dd, + // Block 0x21, offset 0x840 + 0x085a: 0x07e4, 0x085c: 0x07eb, 0x085d: 0x07f2, + 0x085e: 0x07fc, + // Block 0x22, offset 0x880 + 0x08b3: 0x0803, + // Block 0x23, offset 0x8c0 + 0x08f3: 0x080a, + // Block 0x24, offset 0x900 + 0x091c: 0x0811, 0x091d: 0x0818, + // Block 0x25, offset 0x940 + 0x094c: 0x081f, + // Block 0x26, offset 0x980 + 0x0983: 0x0823, + 0x098d: 0x082a, + 0x0992: 0x0831, 0x0997: 0x0838, + 0x099c: 0x083f, + 0x09a9: 0x0846, + 0x09b3: 0x084d, 0x09b5: 0x0854, + 0x09b6: 0x085b, 0x09b7: 0x0862, 0x09b8: 0x086c, 0x09b9: 0x0873, + // Block 0x27, offset 0x9c0 + 0x09c1: 0x087d, + 0x09d3: 0x0884, + 0x09dd: 0x088b, + 0x09e2: 0x0892, + 0x09e7: 0x0899, + 0x09ec: 0x08a0, + 0x09f9: 0x08a7, + // Block 0x28, offset 0xa00 + 0x0a26: 0x08ae, + // Block 0x29, offset 0xa40 + 0x0a7c: 0x08b5, + // Block 0x2a, offset 0xa80 + 0x0a86: 0x08b9, 0x0a88: 0x08c0, 0x0a8a: 0x08c7, + 0x0a8c: 0x08ce, 0x0a8e: 0x08d5, + 0x0a92: 0x08dc, + 0x0abb: 0x08e3, + 0x0abd: 0x08ea, + // Block 0x2b, offset 0xac0 + 0x0ac0: 0x08f1, 0x0ac1: 0x08f8, 0x0ac3: 0x08ff, + // Block 0x2c, offset 0xb00 + 0x0b2c: 0x0906, 0x0b2d: 0x0908, 0x0b2e: 0x090b, + 0x0b30: 0x090d, 0x0b31: 0x090f, 0x0b32: 0x0911, 0x0b33: 0x0914, 0x0b34: 0x0916, 0x0b35: 0x0918, + 0x0b36: 0x091a, 0x0b37: 0x091c, 0x0b38: 0x091e, 0x0b39: 0x0920, 0x0b3a: 0x0922, + 0x0b3c: 0x0924, 0x0b3d: 0x0926, 0x0b3e: 0x0929, 0x0b3f: 0x092b, + // Block 0x2d, offset 0xb40 + 0x0b40: 0x092d, 0x0b41: 0x092f, 0x0b42: 0x0931, 0x0b43: 0x0007, 0x0b44: 0x0933, 0x0b45: 0x0936, + 0x0b46: 0x0939, 0x0b47: 0x093d, 0x0b48: 0x093f, 0x0b49: 0x0941, 0x0b4a: 0x0943, 0x0b4b: 0x0946, + 0x0b4c: 0x0949, 0x0b4d: 0x094c, 0x0b4f: 0x094e, 0x0b50: 0x0950, 0x0b51: 0x0952, + 0x0b52: 0x001e, 0x0b53: 0x0955, 0x0b54: 0x0958, 0x0b55: 0x095c, 0x0b56: 0x0960, 0x0b57: 0x0962, + 0x0b58: 0x0964, 0x0b59: 0x0966, 0x0b5a: 0x096a, 0x0b5b: 0x096d, 0x0b5c: 0x096f, 0x0b5d: 0x0559, + 0x0b5e: 0x0973, 0x0b5f: 0x0976, 0x0b60: 0x056c, 0x0b61: 0x0979, 0x0b62: 0x097c, 0x0b63: 0x049b, + 0x0b64: 0x0964, 0x0b65: 0x096d, 0x0b66: 0x0559, 0x0b67: 0x0973, 0x0b68: 0x0575, 0x0b69: 0x056c, + 0x0b6a: 0x0979, + 0x0b78: 0x097e, + // Block 0x2e, offset 0xb80 + 0x0b9b: 0x0981, 0x0b9c: 0x0984, 0x0b9d: 0x0986, + 0x0b9e: 0x0989, 0x0b9f: 0x0949, 0x0ba0: 0x098c, 0x0ba1: 0x098e, 0x0ba2: 0x0991, 0x0ba3: 0x0994, + 0x0ba4: 0x0997, 0x0ba5: 0x099a, 0x0ba6: 0x099d, 0x0ba7: 0x09a0, 0x0ba8: 0x09a4, 0x0ba9: 0x09a7, + 0x0baa: 0x09aa, 0x0bab: 0x09ae, 0x0bac: 0x09b1, 0x0bad: 0x09b4, 0x0bae: 0x09b7, 0x0baf: 0x09ba, + 0x0bb0: 0x09bd, 0x0bb1: 0x09c0, 0x0bb2: 0x09c3, 0x0bb3: 0x09c6, 0x0bb4: 0x09c9, 0x0bb5: 0x09cc, + 0x0bb6: 0x09cf, 0x0bb7: 0x09d2, 0x0bb8: 0x09d5, 0x0bb9: 0x09d9, 0x0bba: 0x09dc, 0x0bbb: 0x09df, + 0x0bbc: 0x09e1, 0x0bbd: 0x09e4, 0x0bbe: 0x09e7, 0x0bbf: 0x055c, + // Block 0x2f, offset 0xbc0 + 0x0bc0: 0x09ea, 0x0bc1: 0x09ee, 0x0bc2: 0x09f2, 0x0bc3: 0x09f6, 0x0bc4: 0x09fa, 0x0bc5: 0x09fe, + 0x0bc6: 0x0a02, 0x0bc7: 0x0a06, 0x0bc8: 0x0a0a, 0x0bc9: 0x0a10, 0x0bca: 0x0a16, 0x0bcb: 0x0a1a, + 0x0bcc: 0x0a1e, 0x0bcd: 0x0a22, 0x0bce: 0x0a26, 0x0bcf: 0x0a2a, 0x0bd0: 0x0a2e, 0x0bd1: 0x0a32, + 0x0bd2: 0x0a36, 0x0bd3: 0x0a3a, 0x0bd4: 0x0a3e, 0x0bd5: 0x0a44, 0x0bd6: 0x0a4a, 0x0bd7: 0x0a50, + 0x0bd8: 0x0a56, 0x0bd9: 0x0a5a, 0x0bda: 0x0a5e, 0x0bdb: 0x0a62, 0x0bdc: 0x0a66, 0x0bdd: 0x0a6c, + 0x0bde: 0x0a72, 0x0bdf: 0x0a76, 0x0be0: 0x0a7a, 0x0be1: 0x0a7e, 0x0be2: 0x0a82, 0x0be3: 0x0a86, + 0x0be4: 0x0a8a, 0x0be5: 0x0a8e, 0x0be6: 0x0a92, 0x0be7: 0x0a96, 0x0be8: 0x0a9a, 0x0be9: 0x0a9e, + 0x0bea: 0x0aa2, 0x0beb: 0x0aa6, 0x0bec: 0x0aaa, 0x0bed: 0x0aae, 0x0bee: 0x0ab2, 0x0bef: 0x0ab8, + 0x0bf0: 0x0abe, 0x0bf1: 0x0ac2, 0x0bf2: 0x0ac6, 0x0bf3: 0x0aca, 0x0bf4: 0x0ace, 0x0bf5: 0x0ad2, + 0x0bf6: 0x0ad6, 0x0bf7: 0x0ada, 0x0bf8: 0x0ade, 0x0bf9: 0x0ae4, 0x0bfa: 0x0aea, 0x0bfb: 0x0aee, + 0x0bfc: 0x0af2, 0x0bfd: 0x0af6, 0x0bfe: 0x0afa, 0x0bff: 0x0afe, + // Block 0x30, offset 0xc00 + 0x0c00: 0x0b02, 0x0c01: 0x0b06, 0x0c02: 0x0b0a, 0x0c03: 0x0b0e, 0x0c04: 0x0b12, 0x0c05: 0x0b16, + 0x0c06: 0x0b1a, 0x0c07: 0x0b1e, 0x0c08: 0x0b22, 0x0c09: 0x0b26, 0x0c0a: 0x0b2a, 0x0c0b: 0x0b2e, + 0x0c0c: 0x0b32, 0x0c0d: 0x0b38, 0x0c0e: 0x0b3e, 0x0c0f: 0x0b44, 0x0c10: 0x0b4a, 0x0c11: 0x0b50, + 0x0c12: 0x0b56, 0x0c13: 0x0b5c, 0x0c14: 0x0b62, 0x0c15: 0x0b66, 0x0c16: 0x0b6a, 0x0c17: 0x0b6e, + 0x0c18: 0x0b72, 0x0c19: 0x0b76, 0x0c1a: 0x0b7a, 0x0c1b: 0x0b7e, 0x0c1c: 0x0b82, 0x0c1d: 0x0b88, + 0x0c1e: 0x0b8e, 0x0c1f: 0x0b92, 0x0c20: 0x0b96, 0x0c21: 0x0b9a, 0x0c22: 0x0b9e, 0x0c23: 0x0ba2, + 0x0c24: 0x0ba6, 0x0c25: 0x0bac, 0x0c26: 0x0bb2, 0x0c27: 0x0bb8, 0x0c28: 0x0bbe, 0x0c29: 0x0bc4, + 0x0c2a: 0x0bca, 0x0c2b: 0x0bce, 0x0c2c: 0x0bd2, 0x0c2d: 0x0bd6, 0x0c2e: 0x0bda, 0x0c2f: 0x0bde, + 0x0c30: 0x0be2, 0x0c31: 0x0be6, 0x0c32: 0x0bea, 0x0c33: 0x0bee, 0x0c34: 0x0bf2, 0x0c35: 0x0bf6, + 0x0c36: 0x0bfa, 0x0c37: 0x0bfe, 0x0c38: 0x0c02, 0x0c39: 0x0c08, 0x0c3a: 0x0c0e, 0x0c3b: 0x0c14, + 0x0c3c: 0x0c1a, 0x0c3d: 0x0c1e, 0x0c3e: 0x0c22, 0x0c3f: 0x0c26, + // Block 0x31, offset 0xc40 + 0x0c40: 0x0c2a, 0x0c41: 0x0c2e, 0x0c42: 0x0c32, 0x0c43: 0x0c36, 0x0c44: 0x0c3a, 0x0c45: 0x0c3e, + 0x0c46: 0x0c42, 0x0c47: 0x0c46, 0x0c48: 0x0c4a, 0x0c49: 0x0c4e, 0x0c4a: 0x0c52, 0x0c4b: 0x0c56, + 0x0c4c: 0x0c5a, 0x0c4d: 0x0c5e, 0x0c4e: 0x0c62, 0x0c4f: 0x0c66, 0x0c50: 0x0c6a, 0x0c51: 0x0c6e, + 0x0c52: 0x0c72, 0x0c53: 0x0c76, 0x0c54: 0x0c7a, 0x0c55: 0x0c7e, 0x0c56: 0x0c82, 0x0c57: 0x0c86, + 0x0c58: 0x0c8a, 0x0c59: 0x0c8e, 0x0c5a: 0x0c92, 0x0c5b: 0x0b9a, + 0x0c60: 0x0c9b, 0x0c61: 0x0c9f, 0x0c62: 0x0ca3, 0x0c63: 0x0ca7, + 0x0c64: 0x0cab, 0x0c65: 0x0cb1, 0x0c66: 0x0cb7, 0x0c67: 0x0cbd, 0x0c68: 0x0cc3, 0x0c69: 0x0cc9, + 0x0c6a: 0x0ccf, 0x0c6b: 0x0cd5, 0x0c6c: 0x0cdb, 0x0c6d: 0x0ce1, 0x0c6e: 0x0ce7, 0x0c6f: 0x0ced, + 0x0c70: 0x0cf3, 0x0c71: 0x0cf9, 0x0c72: 0x0cff, 0x0c73: 0x0d05, 0x0c74: 0x0d0b, 0x0c75: 0x0d11, + 0x0c76: 0x0d17, 0x0c77: 0x0d1d, 0x0c78: 0x0d23, 0x0c79: 0x0d27, 0x0c7a: 0x0d2b, 0x0c7b: 0x0d2f, + 0x0c7c: 0x0d33, 0x0c7d: 0x0d37, 0x0c7e: 0x0d3b, 0x0c7f: 0x0d41, + // Block 0x32, offset 0xc80 + 0x0c80: 0x0d47, 0x0c81: 0x0d4d, 0x0c82: 0x0d53, 0x0c83: 0x0d59, 0x0c84: 0x0d5f, 0x0c85: 0x0d65, + 0x0c86: 0x0d6b, 0x0c87: 0x0d71, 0x0c88: 0x0d77, 0x0c89: 0x0d7b, 0x0c8a: 0x0d7f, 0x0c8b: 0x0d83, + 0x0c8c: 0x0d87, 0x0c8d: 0x0d8b, 0x0c8e: 0x0d8f, 0x0c8f: 0x0d93, 0x0c90: 0x0d97, 0x0c91: 0x0d9d, + 0x0c92: 0x0da3, 0x0c93: 0x0da9, 0x0c94: 0x0daf, 0x0c95: 0x0db5, 0x0c96: 0x0dbb, 0x0c97: 0x0dc1, + 0x0c98: 0x0dc7, 0x0c99: 0x0dcd, 0x0c9a: 0x0dd3, 0x0c9b: 0x0dd9, 0x0c9c: 0x0ddf, 0x0c9d: 0x0de5, + 0x0c9e: 0x0deb, 0x0c9f: 0x0df1, 0x0ca0: 0x0df7, 0x0ca1: 0x0dfd, 0x0ca2: 0x0e03, 0x0ca3: 0x0e09, + 0x0ca4: 0x0e0f, 0x0ca5: 0x0e13, 0x0ca6: 0x0e17, 0x0ca7: 0x0e1b, 0x0ca8: 0x0e1f, 0x0ca9: 0x0e25, + 0x0caa: 0x0e2b, 0x0cab: 0x0e31, 0x0cac: 0x0e37, 0x0cad: 0x0e3d, 0x0cae: 0x0e43, 0x0caf: 0x0e49, + 0x0cb0: 0x0e4f, 0x0cb1: 0x0e55, 0x0cb2: 0x0e5b, 0x0cb3: 0x0e5f, 0x0cb4: 0x0e63, 0x0cb5: 0x0e67, + 0x0cb6: 0x0e6b, 0x0cb7: 0x0e6f, 0x0cb8: 0x0e73, 0x0cb9: 0x0e77, + // Block 0x33, offset 0xcc0 + 0x0cc0: 0x0e7b, 0x0cc1: 0x0e80, 0x0cc2: 0x0e85, 0x0cc3: 0x0e8c, 0x0cc4: 0x0e93, 0x0cc5: 0x0e9a, + 0x0cc6: 0x0ea1, 0x0cc7: 0x0ea8, 0x0cc8: 0x0eaf, 0x0cc9: 0x0eb4, 0x0cca: 0x0eb9, 0x0ccb: 0x0ec0, + 0x0ccc: 0x0ec7, 0x0ccd: 0x0ece, 0x0cce: 0x0ed5, 0x0ccf: 0x0edc, 0x0cd0: 0x0ee3, 0x0cd1: 0x0ee8, + 0x0cd2: 0x0eed, 0x0cd3: 0x0ef4, 0x0cd4: 0x0efb, 0x0cd5: 0x0f02, + 0x0cd8: 0x0f09, 0x0cd9: 0x0f0e, 0x0cda: 0x0f13, 0x0cdb: 0x0f1a, 0x0cdc: 0x0f21, 0x0cdd: 0x0f28, + 0x0ce0: 0x0f2f, 0x0ce1: 0x0f34, 0x0ce2: 0x0f39, 0x0ce3: 0x0f40, + 0x0ce4: 0x0f47, 0x0ce5: 0x0f4e, 0x0ce6: 0x0f55, 0x0ce7: 0x0f5c, 0x0ce8: 0x0f63, 0x0ce9: 0x0f68, + 0x0cea: 0x0f6d, 0x0ceb: 0x0f74, 0x0cec: 0x0f7b, 0x0ced: 0x0f82, 0x0cee: 0x0f89, 0x0cef: 0x0f90, + 0x0cf0: 0x0f97, 0x0cf1: 0x0f9c, 0x0cf2: 0x0fa1, 0x0cf3: 0x0fa8, 0x0cf4: 0x0faf, 0x0cf5: 0x0fb6, + 0x0cf6: 0x0fbd, 0x0cf7: 0x0fc4, 0x0cf8: 0x0fcb, 0x0cf9: 0x0fd0, 0x0cfa: 0x0fd5, 0x0cfb: 0x0fdc, + 0x0cfc: 0x0fe3, 0x0cfd: 0x0fea, 0x0cfe: 0x0ff1, 0x0cff: 0x0ff8, + // Block 0x34, offset 0xd00 + 0x0d00: 0x0fff, 0x0d01: 0x1004, 0x0d02: 0x1009, 0x0d03: 0x1010, 0x0d04: 0x1017, 0x0d05: 0x101e, + 0x0d08: 0x1025, 0x0d09: 0x102a, 0x0d0a: 0x102f, 0x0d0b: 0x1036, + 0x0d0c: 0x103d, 0x0d0d: 0x1044, 0x0d10: 0x104b, 0x0d11: 0x1050, + 0x0d12: 0x1055, 0x0d13: 0x105c, 0x0d14: 0x1063, 0x0d15: 0x106a, 0x0d16: 0x1071, 0x0d17: 0x1078, + 0x0d19: 0x107f, 0x0d1b: 0x1084, 0x0d1d: 0x108b, + 0x0d1f: 0x1092, 0x0d20: 0x1099, 0x0d21: 0x109e, 0x0d22: 0x10a3, 0x0d23: 0x10aa, + 0x0d24: 0x10b1, 0x0d25: 0x10b8, 0x0d26: 0x10bf, 0x0d27: 0x10c6, 0x0d28: 0x10cd, 0x0d29: 0x10d2, + 0x0d2a: 0x10d7, 0x0d2b: 0x10de, 0x0d2c: 0x10e5, 0x0d2d: 0x10ec, 0x0d2e: 0x10f3, 0x0d2f: 0x10fa, + 0x0d30: 0x1101, 0x0d31: 0x0525, 0x0d32: 0x1106, 0x0d33: 0x052a, 0x0d34: 0x110b, 0x0d35: 0x052f, + 0x0d36: 0x1110, 0x0d37: 0x0534, 0x0d38: 0x1115, 0x0d39: 0x054a, 0x0d3a: 0x111a, 0x0d3b: 0x054f, + 0x0d3c: 0x111f, 0x0d3d: 0x0554, + // Block 0x35, offset 0xd40 + 0x0d40: 0x1124, 0x0d41: 0x112b, 0x0d42: 0x1132, 0x0d43: 0x113b, 0x0d44: 0x1144, 0x0d45: 0x114d, + 0x0d46: 0x1156, 0x0d47: 0x115f, 0x0d48: 0x1168, 0x0d49: 0x116f, 0x0d4a: 0x1176, 0x0d4b: 0x117f, + 0x0d4c: 0x1188, 0x0d4d: 0x1191, 0x0d4e: 0x119a, 0x0d4f: 0x11a3, 0x0d50: 0x11ac, 0x0d51: 0x11b3, + 0x0d52: 0x11ba, 0x0d53: 0x11c3, 0x0d54: 0x11cc, 0x0d55: 0x11d5, 0x0d56: 0x11de, 0x0d57: 0x11e7, + 0x0d58: 0x11f0, 0x0d59: 0x11f7, 0x0d5a: 0x11fe, 0x0d5b: 0x1207, 0x0d5c: 0x1210, 0x0d5d: 0x1219, + 0x0d5e: 0x1222, 0x0d5f: 0x122b, 0x0d60: 0x1234, 0x0d61: 0x123b, 0x0d62: 0x1242, 0x0d63: 0x124b, + 0x0d64: 0x1254, 0x0d65: 0x125d, 0x0d66: 0x1266, 0x0d67: 0x126f, 0x0d68: 0x1278, 0x0d69: 0x127f, + 0x0d6a: 0x1286, 0x0d6b: 0x128f, 0x0d6c: 0x1298, 0x0d6d: 0x12a1, 0x0d6e: 0x12aa, 0x0d6f: 0x12b3, + 0x0d70: 0x12bc, 0x0d71: 0x12c1, 0x0d72: 0x12c6, 0x0d73: 0x12cd, 0x0d74: 0x12d2, + 0x0d76: 0x12d9, 0x0d77: 0x12de, 0x0d78: 0x12e5, 0x0d79: 0x12ea, 0x0d7a: 0x12ef, 0x0d7b: 0x04ee, + 0x0d7c: 0x12f4, 0x0d7d: 0x12f9, 0x0d7e: 0x12fd, 0x0d7f: 0x12f9, + // Block 0x36, offset 0xd80 + 0x0d80: 0x1300, 0x0d81: 0x1309, 0x0d82: 0x130f, 0x0d83: 0x1316, 0x0d84: 0x131b, + 0x0d86: 0x1322, 0x0d87: 0x1327, 0x0d88: 0x132e, 0x0d89: 0x04f6, 0x0d8a: 0x1333, 0x0d8b: 0x04fb, + 0x0d8c: 0x1338, 0x0d8d: 0x1343, 0x0d8e: 0x134f, 0x0d8f: 0x135b, 0x0d90: 0x1361, 0x0d91: 0x1366, + 0x0d92: 0x136b, 0x0d93: 0x0514, 0x0d96: 0x1372, 0x0d97: 0x1377, + 0x0d98: 0x137e, 0x0d99: 0x1383, 0x0d9a: 0x1388, 0x0d9b: 0x0500, 0x0d9d: 0x1393, + 0x0d9e: 0x139f, 0x0d9f: 0x13ab, 0x0da0: 0x13b1, 0x0da1: 0x13b6, 0x0da2: 0x13bb, 0x0da3: 0x0539, + 0x0da4: 0x13c2, 0x0da5: 0x13c7, 0x0da6: 0x13cc, 0x0da7: 0x13d1, 0x0da8: 0x13d8, 0x0da9: 0x13dd, + 0x0daa: 0x13e2, 0x0dab: 0x050a, 0x0dac: 0x13e7, 0x0dad: 0x13f1, 0x0dae: 0x04e8, 0x0daf: 0x13f7, + 0x0db2: 0x13f9, 0x0db3: 0x1400, 0x0db4: 0x1405, + 0x0db6: 0x140c, 0x0db7: 0x1411, 0x0db8: 0x1418, 0x0db9: 0x0505, 0x0dba: 0x141d, 0x0dbb: 0x050f, + 0x0dbc: 0x1422, 0x0dbd: 0x0011, 0x0dbe: 0x142a, + // Block 0x37, offset 0xdc0 + 0x0dc0: 0x0001, 0x0dc1: 0x0001, 0x0dc2: 0x0001, 0x0dc3: 0x0001, 0x0dc4: 0x0001, 0x0dc5: 0x0001, + 0x0dc6: 0x0001, 0x0dc7: 0x0001, 0x0dc8: 0x0001, 0x0dc9: 0x0001, 0x0dca: 0x0001, + 0x0dd1: 0x1436, + 0x0dd7: 0x143a, + 0x0de4: 0x143e, 0x0de5: 0x1440, 0x0de6: 0x1443, + 0x0def: 0x0001, + 0x0df3: 0x1447, 0x0df4: 0x144e, + 0x0df6: 0x1458, 0x0df7: 0x145f, + 0x0dfc: 0x1469, 0x0dfe: 0x146c, + // Block 0x38, offset 0xe00 + 0x0e07: 0x1470, 0x0e08: 0x1473, 0x0e09: 0x1476, + 0x0e17: 0x1479, + 0x0e1f: 0x0001, + 0x0e30: 0x1486, 0x0e31: 0x097c, 0x0e34: 0x1488, 0x0e35: 0x148a, + 0x0e36: 0x148c, 0x0e37: 0x148e, 0x0e38: 0x1490, 0x0e39: 0x1492, 0x0e3a: 0x1494, 0x0e3b: 0x1496, + 0x0e3c: 0x149a, 0x0e3d: 0x149c, 0x0e3e: 0x149e, 0x0e3f: 0x14a0, + // Block 0x39, offset 0xe40 + 0x0e40: 0x1486, 0x0e41: 0x001c, 0x0e42: 0x000d, 0x0e43: 0x000f, 0x0e44: 0x1488, 0x0e45: 0x148a, + 0x0e46: 0x148c, 0x0e47: 0x148e, 0x0e48: 0x1490, 0x0e49: 0x1492, 0x0e4a: 0x1494, 0x0e4b: 0x1496, + 0x0e4c: 0x149a, 0x0e4d: 0x149c, 0x0e4e: 0x149e, 0x0e50: 0x0007, 0x0e51: 0x0941, + 0x0e52: 0x001e, 0x0e53: 0x04c7, 0x0e54: 0x0943, 0x0e55: 0x0494, 0x0e56: 0x094e, 0x0e57: 0x04c5, + 0x0e58: 0x0950, 0x0e59: 0x14a0, 0x0e5a: 0x0960, 0x0e5b: 0x02c8, 0x0e5c: 0x0962, + 0x0e68: 0x14a2, + // Block 0x3a, offset 0xe80 + 0x0e80: 0x14a5, 0x0e81: 0x14a9, 0x0e82: 0x14ad, 0x0e83: 0x14af, 0x0e85: 0x14b3, + 0x0e86: 0x14b7, 0x0e87: 0x14bb, 0x0e89: 0x14be, 0x0e8a: 0x094c, 0x0e8b: 0x0916, + 0x0e8c: 0x0916, 0x0e8d: 0x0916, 0x0e8e: 0x0494, 0x0e8f: 0x14c2, 0x0e90: 0x0918, 0x0e91: 0x0918, + 0x0e92: 0x091e, 0x0e93: 0x04c5, 0x0e95: 0x0922, 0x0e96: 0x14c5, + 0x0e99: 0x0929, 0x0e9a: 0x14c8, 0x0e9b: 0x092b, 0x0e9c: 0x092b, 0x0e9d: 0x092b, + 0x0ea0: 0x14ca, 0x0ea1: 0x14cd, 0x0ea2: 0x14d1, + 0x0ea4: 0x14d4, 0x0ea6: 0x14d6, 0x0ea8: 0x14d4, + 0x0eaa: 0x091c, 0x0eab: 0x0046, 0x0eac: 0x090b, 0x0ead: 0x14ad, 0x0eaf: 0x0941, + 0x0eb0: 0x090f, 0x0eb1: 0x14d9, 0x0eb3: 0x0920, 0x0eb4: 0x001e, 0x0eb5: 0x14db, + 0x0eb6: 0x14de, 0x0eb7: 0x14e1, 0x0eb8: 0x14e4, 0x0eb9: 0x097c, 0x0ebb: 0x14e7, + 0x0ebc: 0x056f, 0x0ebd: 0x0973, 0x0ebe: 0x14eb, 0x0ebf: 0x14ee, + // Block 0x3b, offset 0xec0 + 0x0ec0: 0x14f1, 0x0ec5: 0x090d, + 0x0ec6: 0x093f, 0x0ec7: 0x0941, 0x0ec8: 0x097c, 0x0ec9: 0x0499, + 0x0ed0: 0x14f5, 0x0ed1: 0x14fb, + 0x0ed2: 0x1501, 0x0ed3: 0x1508, 0x0ed4: 0x150e, 0x0ed5: 0x1514, 0x0ed6: 0x151a, 0x0ed7: 0x1520, + 0x0ed8: 0x1526, 0x0ed9: 0x152c, 0x0eda: 0x1532, 0x0edb: 0x1538, 0x0edc: 0x153e, 0x0edd: 0x1544, + 0x0ede: 0x154a, 0x0edf: 0x1550, 0x0ee0: 0x0918, 0x0ee1: 0x1555, 0x0ee2: 0x1558, 0x0ee3: 0x155c, + 0x0ee4: 0x155f, 0x0ee5: 0x1561, 0x0ee6: 0x1564, 0x0ee7: 0x1568, 0x0ee8: 0x156d, 0x0ee9: 0x1570, + 0x0eea: 0x1572, 0x0eeb: 0x1575, 0x0eec: 0x091e, 0x0eed: 0x14ad, 0x0eee: 0x090d, 0x0eef: 0x0920, + 0x0ef0: 0x097c, 0x0ef1: 0x1579, 0x0ef2: 0x157c, 0x0ef3: 0x1580, 0x0ef4: 0x096d, 0x0ef5: 0x1583, + 0x0ef6: 0x1586, 0x0ef7: 0x158a, 0x0ef8: 0x158f, 0x0ef9: 0x04c7, 0x0efa: 0x1592, 0x0efb: 0x1595, + 0x0efc: 0x04c5, 0x0efd: 0x0984, 0x0efe: 0x093f, 0x0eff: 0x0950, + // Block 0x3c, offset 0xf00 + 0x0f09: 0x1599, + 0x0f1a: 0x159f, 0x0f1b: 0x15a5, + 0x0f2e: 0x15ab, + // Block 0x3d, offset 0xf40 + 0x0f4d: 0x15b1, 0x0f4e: 0x15b7, 0x0f4f: 0x15bd, + // Block 0x3e, offset 0xf80 + 0x0f84: 0x15c3, + 0x0f89: 0x15c9, + 0x0f8c: 0x15cf, + 0x0fa4: 0x15d5, 0x0fa6: 0x15db, + 0x0fac: 0x15e1, 0x0fad: 0x15e8, 0x0faf: 0x15f2, + 0x0fb0: 0x15f9, + // Block 0x3f, offset 0xfc0 + 0x0fc1: 0x1603, 0x0fc4: 0x1609, + 0x0fc7: 0x160f, 0x0fc9: 0x1615, + 0x0fe0: 0x161b, 0x0fe2: 0x161f, + 0x0fed: 0x1625, 0x0fee: 0x162b, 0x0fef: 0x162f, + 0x0ff0: 0x1633, 0x0ff1: 0x1639, 0x0ff4: 0x163f, 0x0ff5: 0x1645, + 0x0ff8: 0x164b, 0x0ff9: 0x1651, + // Block 0x40, offset 0x1000 + 0x1000: 0x1657, 0x1001: 0x165d, 0x1004: 0x1663, 0x1005: 0x1669, + 0x1008: 0x166f, 0x1009: 0x1675, + 0x102c: 0x167b, 0x102d: 0x1681, 0x102e: 0x1687, 0x102f: 0x168d, + // Block 0x41, offset 0x1040 + 0x1060: 0x1693, 0x1061: 0x1699, 0x1062: 0x169f, 0x1063: 0x16a5, + 0x106a: 0x16ab, 0x106b: 0x16b1, 0x106c: 0x16b7, 0x106d: 0x16bd, + // Block 0x42, offset 0x1080 + 0x10a9: 0x16c3, + 0x10aa: 0x16c7, + // Block 0x43, offset 0x10c0 + 0x10e0: 0x001c, 0x10e1: 0x000d, 0x10e2: 0x000f, 0x10e3: 0x1488, + 0x10e4: 0x148a, 0x10e5: 0x148c, 0x10e6: 0x148e, 0x10e7: 0x1490, 0x10e8: 0x1492, 0x10e9: 0x16cb, + 0x10ea: 0x16ce, 0x10eb: 0x16d1, 0x10ec: 0x16d4, 0x10ed: 0x16d7, 0x10ee: 0x16da, 0x10ef: 0x16dd, + 0x10f0: 0x16e0, 0x10f1: 0x16e3, 0x10f2: 0x16e6, 0x10f3: 0x16e9, 0x10f4: 0x16ec, 0x10f5: 0x16f0, + 0x10f6: 0x16f4, 0x10f7: 0x16f8, 0x10f8: 0x16fc, 0x10f9: 0x1700, 0x10fa: 0x1704, 0x10fb: 0x1708, + 0x10fc: 0x170c, 0x10fd: 0x1710, 0x10fe: 0x1715, 0x10ff: 0x171a, + // Block 0x44, offset 0x1100 + 0x1100: 0x171f, 0x1101: 0x1724, 0x1102: 0x1729, 0x1103: 0x172e, 0x1104: 0x1733, 0x1105: 0x1738, + 0x1106: 0x173d, 0x1107: 0x1742, 0x1108: 0x1747, 0x1109: 0x174a, 0x110a: 0x174d, 0x110b: 0x1750, + 0x110c: 0x1753, 0x110d: 0x1756, 0x110e: 0x1759, 0x110f: 0x175c, 0x1110: 0x175f, 0x1111: 0x1762, + 0x1112: 0x1766, 0x1113: 0x176a, 0x1114: 0x176e, 0x1115: 0x1772, 0x1116: 0x1776, 0x1117: 0x177a, + 0x1118: 0x177e, 0x1119: 0x1782, 0x111a: 0x1786, 0x111b: 0x178a, 0x111c: 0x178e, 0x111d: 0x1792, + 0x111e: 0x1796, 0x111f: 0x179a, 0x1120: 0x179e, 0x1121: 0x17a2, 0x1122: 0x17a6, 0x1123: 0x17aa, + 0x1124: 0x17ae, 0x1125: 0x17b2, 0x1126: 0x17b6, 0x1127: 0x17ba, 0x1128: 0x17be, 0x1129: 0x17c2, + 0x112a: 0x17c6, 0x112b: 0x17ca, 0x112c: 0x17ce, 0x112d: 0x17d2, 0x112e: 0x17d6, 0x112f: 0x17da, + 0x1130: 0x17de, 0x1131: 0x17e2, 0x1132: 0x17e6, 0x1133: 0x17ea, 0x1134: 0x17ee, 0x1135: 0x17f2, + 0x1136: 0x0906, 0x1137: 0x090b, 0x1138: 0x14ad, 0x1139: 0x090d, 0x113a: 0x090f, 0x113b: 0x14d9, + 0x113c: 0x0914, 0x113d: 0x0916, 0x113e: 0x0918, 0x113f: 0x091a, + // Block 0x45, offset 0x1140 + 0x1140: 0x091c, 0x1141: 0x091e, 0x1142: 0x0920, 0x1143: 0x0922, 0x1144: 0x0924, 0x1145: 0x0929, + 0x1146: 0x14c8, 0x1147: 0x092b, 0x1148: 0x17f6, 0x1149: 0x092d, 0x114a: 0x092f, 0x114b: 0x155f, + 0x114c: 0x0931, 0x114d: 0x1570, 0x114e: 0x17f8, 0x114f: 0x14d4, 0x1150: 0x0007, 0x1151: 0x093d, + 0x1152: 0x0984, 0x1153: 0x093f, 0x1154: 0x0941, 0x1155: 0x098c, 0x1156: 0x094c, 0x1157: 0x0494, + 0x1158: 0x097c, 0x1159: 0x0499, 0x115a: 0x094e, 0x115b: 0x04c5, 0x115c: 0x0950, 0x115d: 0x14a0, + 0x115e: 0x001e, 0x115f: 0x0960, 0x1160: 0x17fa, 0x1161: 0x049b, 0x1162: 0x02c8, 0x1163: 0x0962, + 0x1164: 0x0964, 0x1165: 0x096d, 0x1166: 0x04a6, 0x1167: 0x04c7, 0x1168: 0x04a8, 0x1169: 0x09df, + 0x116a: 0x1486, + // Block 0x46, offset 0x1180 + 0x118c: 0x17fc, + // Block 0x47, offset 0x11c0 + 0x11f4: 0x1809, 0x11f5: 0x180d, + 0x11f6: 0x1810, + // Block 0x48, offset 0x1200 + 0x121c: 0x1814, + // Block 0x49, offset 0x1240 + 0x127c: 0x0499, 0x127d: 0x155f, + // Block 0x4a, offset 0x1280 + 0x12af: 0x181a, + // Block 0x4b, offset 0x12c0 + 0x12df: 0x181e, + // Block 0x4c, offset 0x1300 + 0x1333: 0x1822, + // Block 0x4d, offset 0x1340 + 0x1340: 0x1826, 0x1341: 0x182a, 0x1342: 0x182e, 0x1343: 0x1832, 0x1344: 0x1836, 0x1345: 0x183a, + 0x1346: 0x183e, 0x1347: 0x1842, 0x1348: 0x1846, 0x1349: 0x184a, 0x134a: 0x184e, 0x134b: 0x1852, + 0x134c: 0x1856, 0x134d: 0x185a, 0x134e: 0x185e, 0x134f: 0x1862, 0x1350: 0x1866, 0x1351: 0x186a, + 0x1352: 0x186e, 0x1353: 0x1872, 0x1354: 0x1876, 0x1355: 0x187a, 0x1356: 0x187e, 0x1357: 0x1882, + 0x1358: 0x1886, 0x1359: 0x188a, 0x135a: 0x188e, 0x135b: 0x1892, 0x135c: 0x1896, 0x135d: 0x189a, + 0x135e: 0x189e, 0x135f: 0x18a2, 0x1360: 0x18a6, 0x1361: 0x18aa, 0x1362: 0x18ae, 0x1363: 0x18b2, + 0x1364: 0x18b6, 0x1365: 0x18ba, 0x1366: 0x18be, 0x1367: 0x18c2, 0x1368: 0x18c6, 0x1369: 0x18ca, + 0x136a: 0x18ce, 0x136b: 0x18d2, 0x136c: 0x18d6, 0x136d: 0x18da, 0x136e: 0x18de, 0x136f: 0x18e2, + 0x1370: 0x18e6, 0x1371: 0x18ea, 0x1372: 0x18ee, 0x1373: 0x18f2, 0x1374: 0x18f6, 0x1375: 0x18fa, + 0x1376: 0x18fe, 0x1377: 0x1902, 0x1378: 0x1906, 0x1379: 0x190a, 0x137a: 0x190e, 0x137b: 0x1912, + 0x137c: 0x1916, 0x137d: 0x191a, 0x137e: 0x191e, 0x137f: 0x1922, + // Block 0x4e, offset 0x1380 + 0x1380: 0x1926, 0x1381: 0x192a, 0x1382: 0x192e, 0x1383: 0x1932, 0x1384: 0x1936, 0x1385: 0x193a, + 0x1386: 0x193e, 0x1387: 0x1942, 0x1388: 0x1946, 0x1389: 0x194a, 0x138a: 0x194e, 0x138b: 0x1952, + 0x138c: 0x1956, 0x138d: 0x195a, 0x138e: 0x195e, 0x138f: 0x1962, 0x1390: 0x1966, 0x1391: 0x196a, + 0x1392: 0x196e, 0x1393: 0x1972, 0x1394: 0x1976, 0x1395: 0x197a, 0x1396: 0x197e, 0x1397: 0x1982, + 0x1398: 0x1986, 0x1399: 0x198a, 0x139a: 0x198e, 0x139b: 0x1992, 0x139c: 0x1996, 0x139d: 0x199a, + 0x139e: 0x199e, 0x139f: 0x19a2, 0x13a0: 0x19a6, 0x13a1: 0x19aa, 0x13a2: 0x19ae, 0x13a3: 0x19b2, + 0x13a4: 0x19b6, 0x13a5: 0x19ba, 0x13a6: 0x19be, 0x13a7: 0x19c2, 0x13a8: 0x19c6, 0x13a9: 0x19ca, + 0x13aa: 0x19ce, 0x13ab: 0x19d2, 0x13ac: 0x19d6, 0x13ad: 0x19da, 0x13ae: 0x19de, 0x13af: 0x19e2, + 0x13b0: 0x19e6, 0x13b1: 0x19ea, 0x13b2: 0x19ee, 0x13b3: 0x19f2, 0x13b4: 0x19f6, 0x13b5: 0x19fa, + 0x13b6: 0x19fe, 0x13b7: 0x1a02, 0x13b8: 0x1a06, 0x13b9: 0x1a0a, 0x13ba: 0x1a0e, 0x13bb: 0x1a12, + 0x13bc: 0x1a16, 0x13bd: 0x1a1a, 0x13be: 0x1a1e, 0x13bf: 0x1a22, + // Block 0x4f, offset 0x13c0 + 0x13c0: 0x1a26, 0x13c1: 0x1a2a, 0x13c2: 0x1a2e, 0x13c3: 0x1a32, 0x13c4: 0x1a36, 0x13c5: 0x1a3a, + 0x13c6: 0x1a3e, 0x13c7: 0x1a42, 0x13c8: 0x1a46, 0x13c9: 0x1a4a, 0x13ca: 0x1a4e, 0x13cb: 0x1a52, + 0x13cc: 0x1a56, 0x13cd: 0x1a5a, 0x13ce: 0x1a5e, 0x13cf: 0x1a62, 0x13d0: 0x1a66, 0x13d1: 0x1a6a, + 0x13d2: 0x1a6e, 0x13d3: 0x1a72, 0x13d4: 0x1a76, 0x13d5: 0x1a7a, 0x13d6: 0x1a7e, 0x13d7: 0x1a82, + 0x13d8: 0x1a86, 0x13d9: 0x1a8a, 0x13da: 0x1a8e, 0x13db: 0x1a92, 0x13dc: 0x1a96, 0x13dd: 0x1a9a, + 0x13de: 0x1a9e, 0x13df: 0x1aa2, 0x13e0: 0x1aa6, 0x13e1: 0x1aaa, 0x13e2: 0x1aae, 0x13e3: 0x1ab2, + 0x13e4: 0x1ab6, 0x13e5: 0x1aba, 0x13e6: 0x1abe, 0x13e7: 0x1ac2, 0x13e8: 0x1ac6, 0x13e9: 0x1aca, + 0x13ea: 0x1ace, 0x13eb: 0x1ad2, 0x13ec: 0x1ad6, 0x13ed: 0x1ada, 0x13ee: 0x1ade, 0x13ef: 0x1ae2, + 0x13f0: 0x1ae6, 0x13f1: 0x1aea, 0x13f2: 0x1aee, 0x13f3: 0x1af2, 0x13f4: 0x1af6, 0x13f5: 0x1afa, + 0x13f6: 0x1afe, 0x13f7: 0x1b02, 0x13f8: 0x1b06, 0x13f9: 0x1b0a, 0x13fa: 0x1b0e, 0x13fb: 0x1b12, + 0x13fc: 0x1b16, 0x13fd: 0x1b1a, 0x13fe: 0x1b1e, 0x13ff: 0x1b22, + // Block 0x50, offset 0x1400 + 0x1400: 0x1b26, 0x1401: 0x1b2a, 0x1402: 0x1b2e, 0x1403: 0x1b32, 0x1404: 0x1b36, 0x1405: 0x1b3a, + 0x1406: 0x1b3e, 0x1407: 0x1b42, 0x1408: 0x1b46, 0x1409: 0x1b4a, 0x140a: 0x1b4e, 0x140b: 0x1b52, + 0x140c: 0x1b56, 0x140d: 0x1b5a, 0x140e: 0x1b5e, 0x140f: 0x1b62, 0x1410: 0x1b66, 0x1411: 0x1b6a, + 0x1412: 0x1b6e, 0x1413: 0x1b72, 0x1414: 0x1b76, 0x1415: 0x1b7a, + // Block 0x51, offset 0x1440 + 0x1440: 0x0001, + 0x1476: 0x1b7e, 0x1478: 0x1882, 0x1479: 0x1b82, 0x147a: 0x1b86, + // Block 0x52, offset 0x1480 + 0x148c: 0x1b8a, 0x148e: 0x1b91, 0x1490: 0x1b98, + 0x1492: 0x1b9f, 0x1494: 0x1ba6, 0x1496: 0x1bad, + 0x1498: 0x1bb4, 0x149a: 0x1bbb, 0x149c: 0x1bc2, + 0x149e: 0x1bc9, 0x14a0: 0x1bd0, 0x14a2: 0x1bd7, + 0x14a5: 0x1bde, 0x14a7: 0x1be5, 0x14a9: 0x1bec, + 0x14b0: 0x1bf3, 0x14b1: 0x1bfa, 0x14b3: 0x1c01, 0x14b4: 0x1c08, + 0x14b6: 0x1c0f, 0x14b7: 0x1c16, 0x14b9: 0x1c1d, 0x14ba: 0x1c24, + 0x14bc: 0x1c2b, 0x14bd: 0x1c32, + // Block 0x53, offset 0x14c0 + 0x14d4: 0x1c39, + 0x14db: 0x1c40, 0x14dc: 0x1c45, + 0x14de: 0x1c4a, 0x14df: 0x1c51, + 0x14ec: 0x1c58, 0x14ee: 0x1c5f, + 0x14f0: 0x1c66, 0x14f2: 0x1c6d, 0x14f4: 0x1c74, + 0x14f6: 0x1c7b, 0x14f8: 0x1c82, 0x14fa: 0x1c89, + 0x14fc: 0x1c90, 0x14fe: 0x1c97, + // Block 0x54, offset 0x1500 + 0x1500: 0x1c9e, 0x1502: 0x1ca5, 0x1505: 0x1cac, + 0x1507: 0x1cb3, 0x1509: 0x1cba, + 0x1510: 0x1cc1, 0x1511: 0x1cc8, + 0x1513: 0x1ccf, 0x1514: 0x1cd6, 0x1516: 0x1cdd, 0x1517: 0x1ce4, + 0x1519: 0x1ceb, 0x151a: 0x1cf2, 0x151c: 0x1cf9, 0x151d: 0x1d00, + 0x1534: 0x1d07, + 0x1537: 0x1d0e, 0x1538: 0x1d15, 0x1539: 0x1d1c, 0x153a: 0x1d23, + 0x153e: 0x1d2a, 0x153f: 0x1d31, + // Block 0x55, offset 0x1540 + 0x1571: 0x1d38, 0x1572: 0x1d3c, 0x1573: 0x1d40, 0x1574: 0x1d44, 0x1575: 0x1d48, + 0x1576: 0x1d4c, 0x1577: 0x1d50, 0x1578: 0x1d54, 0x1579: 0x1d58, 0x157a: 0x1d5c, 0x157b: 0x1d60, + 0x157c: 0x1d64, 0x157d: 0x1d68, 0x157e: 0x1d6c, 0x157f: 0x1d70, + // Block 0x56, offset 0x1580 + 0x1580: 0x1d74, 0x1581: 0x1d78, 0x1582: 0x1d7c, 0x1583: 0x1d80, 0x1584: 0x1d84, 0x1585: 0x1d88, + 0x1586: 0x1d8c, 0x1587: 0x1d90, 0x1588: 0x1d94, 0x1589: 0x1d98, 0x158a: 0x1d9c, 0x158b: 0x1da0, + 0x158c: 0x1da4, 0x158d: 0x1da8, 0x158e: 0x1dac, 0x158f: 0x1db0, 0x1590: 0x1db4, 0x1591: 0x1db8, + 0x1592: 0x1dbc, 0x1593: 0x1dc0, 0x1594: 0x1dc4, 0x1595: 0x1dc8, 0x1596: 0x1dcc, 0x1597: 0x1dd0, + 0x1598: 0x1dd4, 0x1599: 0x1dd8, 0x159a: 0x1ddc, 0x159b: 0x1de0, 0x159c: 0x1de4, 0x159d: 0x1de8, + 0x159e: 0x1dec, 0x159f: 0x1df0, 0x15a0: 0x1df4, 0x15a1: 0x1df8, 0x15a2: 0x1dfc, 0x15a3: 0x1e00, + 0x15a4: 0x1e04, 0x15a5: 0x1e08, 0x15a6: 0x1e0c, 0x15a7: 0x1e10, 0x15a8: 0x1e14, 0x15a9: 0x1e18, + 0x15aa: 0x1e1c, 0x15ab: 0x1e20, 0x15ac: 0x1e24, 0x15ad: 0x1e28, 0x15ae: 0x1e2c, 0x15af: 0x1e30, + 0x15b0: 0x1e34, 0x15b1: 0x1e38, 0x15b2: 0x1e3c, 0x15b3: 0x1e40, 0x15b4: 0x1e44, 0x15b5: 0x1e48, + 0x15b6: 0x1e4c, 0x15b7: 0x1e50, 0x15b8: 0x1e54, 0x15b9: 0x1e58, 0x15ba: 0x1e5c, 0x15bb: 0x1e60, + 0x15bc: 0x1e64, 0x15bd: 0x1e68, 0x15be: 0x1e6c, 0x15bf: 0x1e70, + // Block 0x57, offset 0x15c0 + 0x15c0: 0x1e74, 0x15c1: 0x1e78, 0x15c2: 0x1e7c, 0x15c3: 0x1e80, 0x15c4: 0x1e84, 0x15c5: 0x1e88, + 0x15c6: 0x1e8c, 0x15c7: 0x1e90, 0x15c8: 0x1e94, 0x15c9: 0x1e98, 0x15ca: 0x1e9c, 0x15cb: 0x1ea0, + 0x15cc: 0x1ea4, 0x15cd: 0x1ea8, 0x15ce: 0x1eac, + 0x15d2: 0x1826, 0x15d3: 0x183e, 0x15d4: 0x1eb0, 0x15d5: 0x1eb4, 0x15d6: 0x1eb8, 0x15d7: 0x1ebc, + 0x15d8: 0x1ec0, 0x15d9: 0x1ec4, 0x15da: 0x1836, 0x15db: 0x1ec8, 0x15dc: 0x1ecc, 0x15dd: 0x1ed0, + 0x15de: 0x1ed4, 0x15df: 0x1846, + // Block 0x58, offset 0x1600 + 0x1600: 0x1ed8, 0x1601: 0x1ede, 0x1602: 0x1ee4, 0x1603: 0x1eea, 0x1604: 0x1ef0, 0x1605: 0x1ef6, + 0x1606: 0x1efc, 0x1607: 0x1f02, 0x1608: 0x1f08, 0x1609: 0x1f0e, 0x160a: 0x1f14, 0x160b: 0x1f1a, + 0x160c: 0x1f20, 0x160d: 0x1f26, 0x160e: 0x1f2c, 0x160f: 0x1f35, 0x1610: 0x1f3e, 0x1611: 0x1f47, + 0x1612: 0x1f50, 0x1613: 0x1f59, 0x1614: 0x1f62, 0x1615: 0x1f6b, 0x1616: 0x1f74, 0x1617: 0x1f7d, + 0x1618: 0x1f86, 0x1619: 0x1f8f, 0x161a: 0x1f98, 0x161b: 0x1fa1, 0x161c: 0x1faa, 0x161d: 0x1fb3, + 0x161e: 0x1fc5, 0x1620: 0x1fd4, 0x1621: 0x1fda, 0x1622: 0x1fe0, 0x1623: 0x1fe6, + 0x1624: 0x1fec, 0x1625: 0x1ff2, 0x1626: 0x1ff8, 0x1627: 0x1ffe, 0x1628: 0x2004, 0x1629: 0x200a, + 0x162a: 0x2010, 0x162b: 0x2016, 0x162c: 0x201c, 0x162d: 0x2022, 0x162e: 0x2028, 0x162f: 0x202e, + 0x1630: 0x2034, 0x1631: 0x203a, 0x1632: 0x2040, 0x1633: 0x2046, 0x1634: 0x204c, 0x1635: 0x2052, + 0x1636: 0x2058, 0x1637: 0x205e, 0x1638: 0x2064, 0x1639: 0x206a, 0x163a: 0x2070, 0x163b: 0x2076, + 0x163c: 0x207c, 0x163d: 0x2082, 0x163e: 0x2088, 0x163f: 0x208e, + // Block 0x59, offset 0x1640 + 0x1640: 0x2094, 0x1641: 0x209a, 0x1642: 0x20a0, 0x1643: 0x20a6, 0x1644: 0x20ac, 0x1645: 0x20b0, + 0x1646: 0x192e, 0x1647: 0x20b4, + 0x1650: 0x20b8, 0x1651: 0x20bc, + 0x1652: 0x20bf, 0x1653: 0x20c2, 0x1654: 0x20c5, 0x1655: 0x20c8, 0x1656: 0x20cb, 0x1657: 0x20ce, + 0x1658: 0x20d1, 0x1659: 0x20d4, 0x165a: 0x20d7, 0x165b: 0x20da, 0x165c: 0x20dd, 0x165d: 0x20e0, + 0x165e: 0x20e3, 0x165f: 0x20e6, 0x1660: 0x1d38, 0x1661: 0x1d44, 0x1662: 0x1d50, 0x1663: 0x1d58, + 0x1664: 0x1d78, 0x1665: 0x1d7c, 0x1666: 0x1d88, 0x1667: 0x1d90, 0x1668: 0x1d94, 0x1669: 0x1d9c, + 0x166a: 0x1da0, 0x166b: 0x1da4, 0x166c: 0x1da8, 0x166d: 0x1dac, 0x166e: 0x20e9, 0x166f: 0x20f0, + 0x1670: 0x20f7, 0x1671: 0x20fe, 0x1672: 0x2105, 0x1673: 0x210c, 0x1674: 0x2113, 0x1675: 0x211a, + 0x1676: 0x2121, 0x1677: 0x2128, 0x1678: 0x212f, 0x1679: 0x2136, 0x167a: 0x213d, 0x167b: 0x2144, + 0x167c: 0x214b, 0x167d: 0x215b, 0x167e: 0x2168, + // Block 0x5a, offset 0x1680 + 0x1680: 0x1826, 0x1681: 0x183e, 0x1682: 0x1eb0, 0x1683: 0x1eb4, 0x1684: 0x216f, 0x1685: 0x2173, + 0x1686: 0x2177, 0x1687: 0x1852, 0x1688: 0x217b, 0x1689: 0x1882, 0x168a: 0x194a, 0x168b: 0x197a, + 0x168c: 0x1976, 0x168d: 0x194e, 0x168e: 0x1abe, 0x168f: 0x18a2, 0x1690: 0x1942, 0x1691: 0x217f, + 0x1692: 0x2183, 0x1693: 0x2187, 0x1694: 0x218b, 0x1695: 0x218f, 0x1696: 0x2193, 0x1697: 0x2197, + 0x1698: 0x219b, 0x1699: 0x219f, 0x169a: 0x21a3, 0x169b: 0x18ba, 0x169c: 0x21a7, 0x169d: 0x21ab, + 0x169e: 0x21af, 0x169f: 0x21b3, 0x16a0: 0x21b7, 0x16a1: 0x21bb, 0x16a2: 0x21bf, 0x16a3: 0x21c3, + 0x16a4: 0x1eb8, 0x16a5: 0x1ebc, 0x16a6: 0x1ec0, 0x16a7: 0x21c7, 0x16a8: 0x21cb, 0x16a9: 0x21cf, + 0x16aa: 0x21d3, 0x16ab: 0x21d7, 0x16ac: 0x21db, 0x16ad: 0x21df, 0x16ae: 0x21e3, 0x16af: 0x21e7, + 0x16b0: 0x21eb, 0x16b1: 0x21ef, 0x16b2: 0x21f2, 0x16b3: 0x21f5, 0x16b4: 0x21f8, 0x16b5: 0x21fb, + 0x16b6: 0x21fe, 0x16b7: 0x2201, 0x16b8: 0x2204, 0x16b9: 0x2207, 0x16ba: 0x220a, 0x16bb: 0x220d, + 0x16bc: 0x2210, 0x16bd: 0x2213, 0x16be: 0x2216, 0x16bf: 0x2219, + // Block 0x5b, offset 0x16c0 + 0x16c0: 0x221c, 0x16c1: 0x2221, 0x16c2: 0x2226, 0x16c3: 0x222b, 0x16c4: 0x2230, 0x16c5: 0x2235, + 0x16c6: 0x223a, 0x16c7: 0x223f, 0x16c8: 0x2244, 0x16c9: 0x2249, 0x16ca: 0x224f, 0x16cb: 0x2255, + 0x16cc: 0x225b, 0x16cd: 0x225e, 0x16ce: 0x2262, 0x16cf: 0x2265, 0x16d0: 0x2269, 0x16d1: 0x226d, + 0x16d2: 0x2271, 0x16d3: 0x2275, 0x16d4: 0x2279, 0x16d5: 0x227d, 0x16d6: 0x2281, 0x16d7: 0x2285, + 0x16d8: 0x2289, 0x16d9: 0x228d, 0x16da: 0x2291, 0x16db: 0x2295, 0x16dc: 0x2299, 0x16dd: 0x229d, + 0x16de: 0x22a1, 0x16df: 0x22a5, 0x16e0: 0x22a9, 0x16e1: 0x22ad, 0x16e2: 0x22b1, 0x16e3: 0x22b5, + 0x16e4: 0x22b9, 0x16e5: 0x22bd, 0x16e6: 0x22c1, 0x16e7: 0x22c5, 0x16e8: 0x22c9, 0x16e9: 0x22cd, + 0x16ea: 0x22d1, 0x16eb: 0x22d5, 0x16ec: 0x22d9, 0x16ed: 0x22dd, 0x16ee: 0x22e1, 0x16ef: 0x22e5, + 0x16f0: 0x22e9, 0x16f1: 0x22ed, 0x16f2: 0x22f1, 0x16f3: 0x22f5, 0x16f4: 0x22f9, 0x16f5: 0x22fd, + 0x16f6: 0x2301, 0x16f7: 0x2305, 0x16f8: 0x2309, 0x16f9: 0x230d, 0x16fa: 0x2311, 0x16fb: 0x2315, + 0x16fc: 0x2319, 0x16fd: 0x231d, 0x16fe: 0x2321, + // Block 0x5c, offset 0x1700 + 0x1700: 0x2325, 0x1701: 0x2335, 0x1702: 0x2342, 0x1703: 0x2352, 0x1704: 0x235c, 0x1705: 0x236c, + 0x1706: 0x2376, 0x1707: 0x2380, 0x1708: 0x2393, 0x1709: 0x23a0, 0x170a: 0x23aa, 0x170b: 0x23b4, + 0x170c: 0x23be, 0x170d: 0x23cb, 0x170e: 0x23d8, 0x170f: 0x23e5, 0x1710: 0x23f2, 0x1711: 0x23ff, + 0x1712: 0x240c, 0x1713: 0x2419, 0x1714: 0x242c, 0x1715: 0x2433, 0x1716: 0x2446, 0x1717: 0x2459, + 0x1718: 0x2469, 0x1719: 0x2476, 0x171a: 0x2489, 0x171b: 0x249c, 0x171c: 0x24a9, 0x171d: 0x24b3, + 0x171e: 0x24bd, 0x171f: 0x24ca, 0x1720: 0x24d7, 0x1721: 0x24e7, 0x1722: 0x24f7, 0x1723: 0x2501, + 0x1724: 0x250b, 0x1725: 0x2518, 0x1726: 0x2522, 0x1727: 0x252c, 0x1728: 0x2533, 0x1729: 0x253a, + 0x172a: 0x2544, 0x172b: 0x254e, 0x172c: 0x2561, 0x172d: 0x256e, 0x172e: 0x257e, 0x172f: 0x2591, + 0x1730: 0x259e, 0x1731: 0x25a8, 0x1732: 0x25b2, 0x1733: 0x25c5, 0x1734: 0x25d2, 0x1735: 0x25e5, + 0x1736: 0x25ef, 0x1737: 0x25ff, 0x1738: 0x2609, 0x1739: 0x2616, 0x173a: 0x2620, 0x173b: 0x262d, + 0x173c: 0x263d, 0x173d: 0x264a, 0x173e: 0x265a, 0x173f: 0x2667, + // Block 0x5d, offset 0x1740 + 0x1740: 0x266e, 0x1741: 0x267e, 0x1742: 0x2688, 0x1743: 0x2692, 0x1744: 0x269f, 0x1745: 0x26a9, + 0x1746: 0x26b3, 0x1747: 0x26bd, 0x1748: 0x26cd, 0x1749: 0x26da, 0x174a: 0x26e1, 0x174b: 0x26f4, + 0x174c: 0x26fe, 0x174d: 0x270e, 0x174e: 0x271b, 0x174f: 0x2728, 0x1750: 0x2732, 0x1751: 0x273c, + 0x1752: 0x2749, 0x1753: 0x2750, 0x1754: 0x275d, 0x1755: 0x276d, 0x1756: 0x2774, 0x1757: 0x2787, + 0x1758: 0x2791, 0x1759: 0x2796, 0x175a: 0x279b, 0x175b: 0x27a0, 0x175c: 0x27a5, 0x175d: 0x27aa, + 0x175e: 0x27af, 0x175f: 0x27b4, 0x1760: 0x27b9, 0x1761: 0x27be, 0x1762: 0x27c3, 0x1763: 0x27c9, + 0x1764: 0x27cf, 0x1765: 0x27d5, 0x1766: 0x27db, 0x1767: 0x27e1, 0x1768: 0x27e7, 0x1769: 0x27ed, + 0x176a: 0x27f3, 0x176b: 0x27f9, 0x176c: 0x27ff, 0x176d: 0x2805, 0x176e: 0x280b, 0x176f: 0x2811, + 0x1770: 0x2817, 0x1771: 0x281d, 0x1772: 0x2821, 0x1773: 0x2824, 0x1774: 0x2827, 0x1775: 0x282b, + 0x1776: 0x282e, 0x1777: 0x2831, 0x1778: 0x2834, 0x1779: 0x2838, 0x177a: 0x283c, 0x177b: 0x283f, + 0x177c: 0x2846, 0x177d: 0x284d, 0x177e: 0x2854, 0x177f: 0x285b, + // Block 0x5e, offset 0x1780 + 0x1780: 0x2868, 0x1781: 0x286b, 0x1782: 0x286e, 0x1783: 0x2872, 0x1784: 0x2875, 0x1785: 0x2878, + 0x1786: 0x287b, 0x1787: 0x287e, 0x1788: 0x2881, 0x1789: 0x2885, 0x178a: 0x288a, 0x178b: 0x288d, + 0x178c: 0x2890, 0x178d: 0x2894, 0x178e: 0x2898, 0x178f: 0x289b, 0x1790: 0x289e, 0x1791: 0x28a1, + 0x1792: 0x28a5, 0x1793: 0x28a9, 0x1794: 0x28ad, 0x1795: 0x28b1, 0x1796: 0x28b5, 0x1797: 0x28b8, + 0x1798: 0x28bb, 0x1799: 0x28be, 0x179a: 0x28c1, 0x179b: 0x28c4, 0x179c: 0x28c8, 0x179d: 0x28cb, + 0x179e: 0x28ce, 0x179f: 0x28d1, 0x17a0: 0x28d5, 0x17a1: 0x28d9, 0x17a2: 0x28dc, 0x17a3: 0x28e0, + 0x17a4: 0x28e4, 0x17a5: 0x28e8, 0x17a6: 0x28eb, 0x17a7: 0x28ef, 0x17a8: 0x28f5, 0x17a9: 0x28fc, + 0x17aa: 0x28ff, 0x17ab: 0x2903, 0x17ac: 0x2907, 0x17ad: 0x290b, 0x17ae: 0x290f, 0x17af: 0x2917, + 0x17b0: 0x2920, 0x17b1: 0x2923, 0x17b2: 0x2926, 0x17b3: 0x292a, 0x17b4: 0x292d, 0x17b5: 0x2930, + 0x17b6: 0x2933, 0x17b7: 0x2937, 0x17b8: 0x293a, 0x17b9: 0x293d, 0x17ba: 0x2940, 0x17bb: 0x2943, + 0x17bc: 0x2946, 0x17bd: 0x294a, 0x17be: 0x294d, 0x17bf: 0x2950, + // Block 0x5f, offset 0x17c0 + 0x17c0: 0x2953, 0x17c1: 0x2957, 0x17c2: 0x295b, 0x17c3: 0x2960, 0x17c4: 0x2963, 0x17c5: 0x2966, + 0x17c6: 0x2969, 0x17c7: 0x2970, 0x17c8: 0x2974, 0x17c9: 0x2977, 0x17ca: 0x297a, 0x17cb: 0x297d, + 0x17cc: 0x2980, 0x17cd: 0x2983, 0x17ce: 0x2986, 0x17cf: 0x2989, 0x17d0: 0x298c, 0x17d1: 0x298f, + 0x17d2: 0x2992, 0x17d3: 0x2996, 0x17d4: 0x2999, 0x17d5: 0x299c, 0x17d6: 0x29a0, 0x17d7: 0x29a4, + 0x17d8: 0x29a7, 0x17d9: 0x29ac, 0x17da: 0x29b0, 0x17db: 0x29b3, 0x17dc: 0x29b6, 0x17dd: 0x29b9, + 0x17de: 0x29bc, 0x17df: 0x29c2, 0x17e0: 0x29c8, 0x17e1: 0x29cd, 0x17e2: 0x29d2, 0x17e3: 0x29d7, + 0x17e4: 0x29dc, 0x17e5: 0x29e1, 0x17e6: 0x29e6, 0x17e7: 0x29eb, 0x17e8: 0x29f0, 0x17e9: 0x29f5, + 0x17ea: 0x29fb, 0x17eb: 0x2a01, 0x17ec: 0x2a07, 0x17ed: 0x2a0d, 0x17ee: 0x2a13, 0x17ef: 0x2a19, + 0x17f0: 0x2a1f, 0x17f1: 0x2a25, 0x17f2: 0x2a2b, 0x17f3: 0x2a31, 0x17f4: 0x2a37, 0x17f5: 0x2a3d, + 0x17f6: 0x2a43, 0x17f7: 0x2a49, 0x17f8: 0x2a4f, 0x17f9: 0x2a55, 0x17fa: 0x2a5b, 0x17fb: 0x2a61, + 0x17fc: 0x2a67, 0x17fd: 0x2a6d, 0x17fe: 0x2a73, 0x17ff: 0x2a79, + // Block 0x60, offset 0x1800 + 0x1830: 0x2a7d, + // Block 0x61, offset 0x1840 + 0x1840: 0x2a81, 0x1841: 0x2a85, 0x1842: 0x1a9e, 0x1843: 0x2a89, 0x1844: 0x2a8d, 0x1845: 0x2a91, + 0x1846: 0x2a95, 0x1847: 0x1b76, 0x1848: 0x1b76, 0x1849: 0x2a99, 0x184a: 0x1abe, 0x184b: 0x2a9d, + 0x184c: 0x2aa1, 0x184d: 0x2aa5, 0x184e: 0x2aa9, 0x184f: 0x2aad, 0x1850: 0x2ab1, 0x1851: 0x2ab5, + 0x1852: 0x2ab9, 0x1853: 0x2abd, 0x1854: 0x2ac1, 0x1855: 0x2ac5, 0x1856: 0x2ac9, 0x1857: 0x2acd, + 0x1858: 0x2ad1, 0x1859: 0x2ad5, 0x185a: 0x2ad9, 0x185b: 0x2add, 0x185c: 0x2ae1, 0x185d: 0x2ae5, + 0x185e: 0x2ae9, 0x185f: 0x2aed, 0x1860: 0x2af1, 0x1861: 0x2af5, 0x1862: 0x2af9, 0x1863: 0x2afd, + 0x1864: 0x2b01, 0x1865: 0x2b05, 0x1866: 0x2b09, 0x1867: 0x2b0d, 0x1868: 0x2b11, 0x1869: 0x2b15, + 0x186a: 0x2b19, 0x186b: 0x2b1d, 0x186c: 0x2b21, 0x186d: 0x2b25, 0x186e: 0x2b29, 0x186f: 0x2b2d, + 0x1870: 0x2b31, 0x1871: 0x2b35, 0x1872: 0x2b39, 0x1873: 0x2b3d, 0x1874: 0x1a16, 0x1875: 0x2b41, + 0x1876: 0x2b45, 0x1877: 0x2b49, 0x1878: 0x2b4d, 0x1879: 0x2b51, 0x187a: 0x2b55, 0x187b: 0x2b59, + 0x187c: 0x2b5d, 0x187d: 0x2b61, 0x187e: 0x2b65, 0x187f: 0x2b69, + // Block 0x62, offset 0x1880 + 0x1880: 0x1b3a, 0x1881: 0x2b6d, 0x1882: 0x2b71, 0x1883: 0x2b75, 0x1884: 0x2b79, 0x1885: 0x2b7d, + 0x1886: 0x2b81, 0x1887: 0x2b85, 0x1888: 0x2b89, 0x1889: 0x2b8d, 0x188a: 0x2b91, 0x188b: 0x2b95, + 0x188c: 0x2b99, 0x188d: 0x2b9d, 0x188e: 0x2ba1, 0x188f: 0x2ba5, 0x1890: 0x2ba9, 0x1891: 0x2bad, + 0x1892: 0x2bb1, 0x1893: 0x2bb5, 0x1894: 0x2bb9, 0x1895: 0x2bbd, 0x1896: 0x2bc1, 0x1897: 0x2bc5, + 0x1898: 0x2bc9, 0x1899: 0x2bcd, 0x189a: 0x2bd1, 0x189b: 0x2bd5, 0x189c: 0x2ac1, 0x189d: 0x2bd9, + 0x189e: 0x2bdd, 0x189f: 0x2be1, 0x18a0: 0x2be5, 0x18a1: 0x2be9, 0x18a2: 0x2bed, 0x18a3: 0x2bf1, + 0x18a4: 0x2bf5, 0x18a5: 0x2bf9, 0x18a6: 0x2bfd, 0x18a7: 0x2c01, 0x18a8: 0x2c05, 0x18a9: 0x2c09, + 0x18aa: 0x2c0d, 0x18ab: 0x2c11, 0x18ac: 0x2c15, 0x18ad: 0x2c19, 0x18ae: 0x2c1d, 0x18af: 0x2c21, + 0x18b0: 0x2c25, 0x18b1: 0x1aa6, 0x18b2: 0x2c29, 0x18b3: 0x2c2d, 0x18b4: 0x2c31, 0x18b5: 0x2c35, + 0x18b6: 0x2c39, 0x18b7: 0x2c3d, 0x18b8: 0x2c41, 0x18b9: 0x2c45, 0x18ba: 0x2c49, 0x18bb: 0x2c4d, + 0x18bc: 0x2c51, 0x18bd: 0x2c55, 0x18be: 0x2c59, 0x18bf: 0x2c5d, + // Block 0x63, offset 0x18c0 + 0x18c0: 0x2c61, 0x18c1: 0x18ba, 0x18c2: 0x2c65, 0x18c3: 0x2c69, 0x18c4: 0x2c6d, 0x18c5: 0x2c71, + 0x18c6: 0x2c75, 0x18c7: 0x2c79, 0x18c8: 0x2c7d, 0x18c9: 0x2c81, 0x18ca: 0x186e, 0x18cb: 0x2c85, + 0x18cc: 0x2c89, 0x18cd: 0x2c8d, 0x18ce: 0x2c91, 0x18cf: 0x2c95, 0x18d0: 0x2c99, 0x18d1: 0x2c9d, + 0x18d2: 0x2ca1, 0x18d3: 0x2ca5, 0x18d4: 0x2ca9, 0x18d5: 0x2cad, 0x18d6: 0x2cb1, 0x18d7: 0x2cb5, + 0x18d8: 0x2cb9, 0x18d9: 0x2cbd, 0x18da: 0x2cc1, 0x18db: 0x2cc5, 0x18dc: 0x2cc9, 0x18dd: 0x2ccd, + 0x18de: 0x2cd1, 0x18df: 0x2cd5, 0x18e0: 0x2cd9, 0x18e1: 0x2c21, 0x18e2: 0x2cdd, 0x18e3: 0x2ce1, + 0x18e4: 0x2ce5, 0x18e5: 0x2ce9, 0x18e6: 0x2ced, 0x18e7: 0x2cf1, 0x18e8: 0x2cf5, 0x18e9: 0x2cf9, + 0x18ea: 0x2be1, 0x18eb: 0x2cfd, 0x18ec: 0x2d01, 0x18ed: 0x2d05, 0x18ee: 0x2d09, 0x18ef: 0x2d0d, + 0x18f0: 0x2d11, 0x18f1: 0x2d15, 0x18f2: 0x2d19, 0x18f3: 0x2d1d, 0x18f4: 0x2d21, 0x18f5: 0x2d25, + 0x18f6: 0x2d29, 0x18f7: 0x2d2d, 0x18f8: 0x2d31, 0x18f9: 0x2d35, 0x18fa: 0x2d39, 0x18fb: 0x2d3d, + 0x18fc: 0x2d41, 0x18fd: 0x2d45, 0x18fe: 0x2d49, 0x18ff: 0x2ac1, + // Block 0x64, offset 0x1900 + 0x1900: 0x2d4d, 0x1901: 0x2d51, 0x1902: 0x2d55, 0x1903: 0x2d59, 0x1904: 0x1b72, 0x1905: 0x2d5d, + 0x1906: 0x2d61, 0x1907: 0x2d65, 0x1908: 0x2d69, 0x1909: 0x2d6d, 0x190a: 0x2d71, 0x190b: 0x2d75, + 0x190c: 0x2d79, 0x190d: 0x2d7d, 0x190e: 0x2d81, 0x190f: 0x2d85, 0x1910: 0x2d89, 0x1911: 0x2173, + 0x1912: 0x2d8d, 0x1913: 0x2d91, 0x1914: 0x2d95, 0x1915: 0x2d99, 0x1916: 0x2d9d, 0x1917: 0x2da1, + 0x1918: 0x2da5, 0x1919: 0x2da9, 0x191a: 0x2dad, 0x191b: 0x2be9, 0x191c: 0x2db1, 0x191d: 0x2db5, + 0x191e: 0x2db9, 0x191f: 0x2dbd, 0x1920: 0x2dc1, 0x1921: 0x2dc5, 0x1922: 0x2dc9, 0x1923: 0x2dcd, + 0x1924: 0x2dd1, 0x1925: 0x2dd5, 0x1926: 0x2dd9, 0x1927: 0x2ddd, 0x1928: 0x2de1, 0x1929: 0x1aba, + 0x192a: 0x2de5, 0x192b: 0x2de9, 0x192c: 0x2ded, 0x192d: 0x2df1, 0x192e: 0x2df5, 0x192f: 0x2df9, + 0x1930: 0x2dfd, 0x1931: 0x2e01, 0x1932: 0x2e05, 0x1933: 0x2e09, 0x1934: 0x2e0d, 0x1935: 0x2e11, + 0x1936: 0x2e15, 0x1937: 0x19f6, 0x1938: 0x2e19, 0x1939: 0x2e1d, 0x193a: 0x2e21, 0x193b: 0x2e25, + 0x193c: 0x2e29, 0x193d: 0x2e2d, 0x193e: 0x2e31, 0x193f: 0x2e35, + // Block 0x65, offset 0x1940 + 0x1940: 0x2e39, 0x1941: 0x2e3d, 0x1942: 0x2e41, 0x1943: 0x2e45, 0x1944: 0x2e49, 0x1945: 0x2e4d, + 0x1946: 0x2e51, 0x1947: 0x2e55, 0x1948: 0x1a62, 0x1949: 0x2e59, 0x194a: 0x1a6e, 0x194b: 0x2e5d, + 0x194c: 0x2e61, 0x194d: 0x2e65, 0x1950: 0x2e69, + 0x1952: 0x2e6d, 0x1955: 0x2e71, 0x1956: 0x2e75, 0x1957: 0x2e79, + 0x1958: 0x2e7d, 0x1959: 0x2e81, 0x195a: 0x2e85, 0x195b: 0x2e89, 0x195c: 0x2e8d, 0x195d: 0x2e91, + 0x195e: 0x1a12, 0x1960: 0x2e95, 0x1962: 0x2e99, + 0x1965: 0x2e9d, 0x1966: 0x2ea1, + 0x196a: 0x2ea5, 0x196b: 0x2ea9, 0x196c: 0x2ead, 0x196d: 0x2eb1, + 0x1970: 0x2eb5, 0x1971: 0x2eb9, 0x1972: 0x2ebd, 0x1973: 0x2ec1, 0x1974: 0x2ec5, 0x1975: 0x2ec9, + 0x1976: 0x2ecd, 0x1977: 0x2ed1, 0x1978: 0x2ed5, 0x1979: 0x2ed9, 0x197a: 0x2edd, 0x197b: 0x2ee1, + 0x197c: 0x18d6, 0x197d: 0x2ee5, 0x197e: 0x2ee9, 0x197f: 0x2eed, + // Block 0x66, offset 0x1980 + 0x1980: 0x2ef1, 0x1981: 0x2ef5, 0x1982: 0x2ef9, 0x1983: 0x2efd, 0x1984: 0x2f01, 0x1985: 0x2f05, + 0x1986: 0x2f09, 0x1987: 0x2f0d, 0x1988: 0x2f11, 0x1989: 0x2f15, 0x198a: 0x2f19, 0x198b: 0x2f1d, + 0x198c: 0x2187, 0x198d: 0x2f21, 0x198e: 0x2f25, 0x198f: 0x2f29, 0x1990: 0x2f2d, 0x1991: 0x2197, + 0x1992: 0x2f31, 0x1993: 0x2f35, 0x1994: 0x2f39, 0x1995: 0x2f3d, 0x1996: 0x2f41, 0x1997: 0x2cb1, + 0x1998: 0x2f45, 0x1999: 0x2f49, 0x199a: 0x2f4d, 0x199b: 0x2f51, 0x199c: 0x2f55, 0x199d: 0x2f59, + 0x199e: 0x2f59, 0x199f: 0x2f5d, 0x19a0: 0x2f61, 0x19a1: 0x2f65, 0x19a2: 0x2f69, 0x19a3: 0x2f6d, + 0x19a4: 0x2f71, 0x19a5: 0x2f75, 0x19a6: 0x2f79, 0x19a7: 0x2e9d, 0x19a8: 0x2f7d, 0x19a9: 0x2f81, + 0x19aa: 0x2f85, 0x19ab: 0x2f89, 0x19ac: 0x2f8d, 0x19ad: 0x2f92, + 0x19b0: 0x2f96, 0x19b1: 0x2f9a, 0x19b2: 0x2f9e, 0x19b3: 0x2fa2, 0x19b4: 0x2fa6, 0x19b5: 0x2faa, + 0x19b6: 0x2fae, 0x19b7: 0x2fb2, 0x19b8: 0x2ecd, 0x19b9: 0x2fb6, 0x19ba: 0x2fba, 0x19bb: 0x2fbe, + 0x19bc: 0x2e69, 0x19bd: 0x2fc2, 0x19be: 0x2fc6, 0x19bf: 0x2fca, + // Block 0x67, offset 0x19c0 + 0x19c0: 0x2fce, 0x19c1: 0x2fd2, 0x19c2: 0x2fd6, 0x19c3: 0x2fda, 0x19c4: 0x2fde, 0x19c5: 0x2fe2, + 0x19c6: 0x2fe6, 0x19c7: 0x2fea, 0x19c8: 0x2fee, 0x19c9: 0x2eed, 0x19ca: 0x2ff2, 0x19cb: 0x2ef1, + 0x19cc: 0x2ff6, 0x19cd: 0x2ffa, 0x19ce: 0x2ffe, 0x19cf: 0x3002, 0x19d0: 0x3006, 0x19d1: 0x2e6d, + 0x19d2: 0x2b15, 0x19d3: 0x300a, 0x19d4: 0x300e, 0x19d5: 0x195a, 0x19d6: 0x2c25, 0x19d7: 0x2d71, + 0x19d8: 0x3012, 0x19d9: 0x3016, 0x19da: 0x2f0d, 0x19db: 0x301a, 0x19dc: 0x2f11, 0x19dd: 0x301e, + 0x19de: 0x3022, 0x19df: 0x3026, 0x19e0: 0x2e75, 0x19e1: 0x302a, 0x19e2: 0x302e, 0x19e3: 0x3032, + 0x19e4: 0x3036, 0x19e5: 0x303a, 0x19e6: 0x2e79, 0x19e7: 0x303e, 0x19e8: 0x3042, 0x19e9: 0x3046, + 0x19ea: 0x304a, 0x19eb: 0x304e, 0x19ec: 0x3052, 0x19ed: 0x2f41, 0x19ee: 0x3056, 0x19ef: 0x305a, + 0x19f0: 0x2cb1, 0x19f1: 0x305e, 0x19f2: 0x2f51, 0x19f3: 0x3062, 0x19f4: 0x3066, 0x19f5: 0x306a, + 0x19f6: 0x306e, 0x19f7: 0x3072, 0x19f8: 0x2f65, 0x19f9: 0x3076, 0x19fa: 0x2e99, 0x19fb: 0x307a, + 0x19fc: 0x2f69, 0x19fd: 0x2bd9, 0x19fe: 0x307e, 0x19ff: 0x2f6d, + // Block 0x68, offset 0x1a00 + 0x1a00: 0x3082, 0x1a01: 0x2f75, 0x1a02: 0x3086, 0x1a03: 0x308a, 0x1a04: 0x308e, 0x1a05: 0x3092, + 0x1a06: 0x3096, 0x1a07: 0x2f7d, 0x1a08: 0x2e8d, 0x1a09: 0x309a, 0x1a0a: 0x2f81, 0x1a0b: 0x309e, + 0x1a0c: 0x2f85, 0x1a0d: 0x30a2, 0x1a0e: 0x1b76, 0x1a0f: 0x30a6, 0x1a10: 0x30ab, 0x1a11: 0x30b0, + 0x1a12: 0x30b5, 0x1a13: 0x30b9, 0x1a14: 0x30bd, 0x1a15: 0x30c1, 0x1a16: 0x30c6, 0x1a17: 0x30cb, + 0x1a18: 0x30d0, 0x1a19: 0x30d4, + // Block 0x69, offset 0x1a40 + 0x1a40: 0x30d8, 0x1a41: 0x30db, 0x1a42: 0x30de, 0x1a43: 0x30e1, 0x1a44: 0x30e5, 0x1a45: 0x30e9, + 0x1a46: 0x30e9, + 0x1a53: 0x30ec, 0x1a54: 0x30f1, 0x1a55: 0x30f6, 0x1a56: 0x30fb, 0x1a57: 0x3100, + 0x1a5d: 0x3105, + 0x1a5f: 0x310a, 0x1a60: 0x310f, 0x1a61: 0x14db, 0x1a62: 0x14e4, 0x1a63: 0x3112, + 0x1a64: 0x3115, 0x1a65: 0x3118, 0x1a66: 0x311b, 0x1a67: 0x311e, 0x1a68: 0x3121, 0x1a69: 0x1494, + 0x1a6a: 0x3124, 0x1a6b: 0x3129, 0x1a6c: 0x312e, 0x1a6d: 0x3135, 0x1a6e: 0x313c, 0x1a6f: 0x3141, + 0x1a70: 0x3146, 0x1a71: 0x314b, 0x1a72: 0x3150, 0x1a73: 0x3155, 0x1a74: 0x315a, 0x1a75: 0x315f, + 0x1a76: 0x3164, 0x1a78: 0x3169, 0x1a79: 0x316e, 0x1a7a: 0x3173, 0x1a7b: 0x3178, + 0x1a7c: 0x317d, 0x1a7e: 0x3182, + // Block 0x6a, offset 0x1a80 + 0x1a80: 0x3187, 0x1a81: 0x318c, 0x1a83: 0x3191, 0x1a84: 0x3196, + 0x1a86: 0x319b, 0x1a87: 0x31a0, 0x1a88: 0x31a5, 0x1a89: 0x31aa, 0x1a8a: 0x31af, 0x1a8b: 0x31b4, + 0x1a8c: 0x31b9, 0x1a8d: 0x31be, 0x1a8e: 0x31c3, 0x1a8f: 0x31c8, 0x1a90: 0x31cd, 0x1a91: 0x31cd, + 0x1a92: 0x31d0, 0x1a93: 0x31d0, 0x1a94: 0x31d0, 0x1a95: 0x31d0, 0x1a96: 0x31d3, 0x1a97: 0x31d3, + 0x1a98: 0x31d3, 0x1a99: 0x31d3, 0x1a9a: 0x31d6, 0x1a9b: 0x31d6, 0x1a9c: 0x31d6, 0x1a9d: 0x31d6, + 0x1a9e: 0x31d9, 0x1a9f: 0x31d9, 0x1aa0: 0x31d9, 0x1aa1: 0x31d9, 0x1aa2: 0x31dc, 0x1aa3: 0x31dc, + 0x1aa4: 0x31dc, 0x1aa5: 0x31dc, 0x1aa6: 0x31df, 0x1aa7: 0x31df, 0x1aa8: 0x31df, 0x1aa9: 0x31df, + 0x1aaa: 0x31e2, 0x1aab: 0x31e2, 0x1aac: 0x31e2, 0x1aad: 0x31e2, 0x1aae: 0x31e5, 0x1aaf: 0x31e5, + 0x1ab0: 0x31e5, 0x1ab1: 0x31e5, 0x1ab2: 0x31e8, 0x1ab3: 0x31e8, 0x1ab4: 0x31e8, 0x1ab5: 0x31e8, + 0x1ab6: 0x31eb, 0x1ab7: 0x31eb, 0x1ab8: 0x31eb, 0x1ab9: 0x31eb, 0x1aba: 0x31ee, 0x1abb: 0x31ee, + 0x1abc: 0x31ee, 0x1abd: 0x31ee, 0x1abe: 0x31f1, 0x1abf: 0x31f1, + // Block 0x6b, offset 0x1ac0 + 0x1ac0: 0x31f1, 0x1ac1: 0x31f1, 0x1ac2: 0x31f4, 0x1ac3: 0x31f4, 0x1ac4: 0x31f7, 0x1ac5: 0x31f7, + 0x1ac6: 0x31fa, 0x1ac7: 0x31fa, 0x1ac8: 0x31fd, 0x1ac9: 0x31fd, 0x1aca: 0x3200, 0x1acb: 0x3200, + 0x1acc: 0x3203, 0x1acd: 0x3203, 0x1ace: 0x3206, 0x1acf: 0x3206, 0x1ad0: 0x3206, 0x1ad1: 0x3206, + 0x1ad2: 0x3209, 0x1ad3: 0x3209, 0x1ad4: 0x3209, 0x1ad5: 0x3209, 0x1ad6: 0x320c, 0x1ad7: 0x320c, + 0x1ad8: 0x320c, 0x1ad9: 0x320c, 0x1ada: 0x320f, 0x1adb: 0x320f, 0x1adc: 0x320f, 0x1add: 0x320f, + 0x1ade: 0x3212, 0x1adf: 0x3212, 0x1ae0: 0x3215, 0x1ae1: 0x3215, 0x1ae2: 0x3215, 0x1ae3: 0x3215, + 0x1ae4: 0x06ba, 0x1ae5: 0x06ba, 0x1ae6: 0x3218, 0x1ae7: 0x3218, 0x1ae8: 0x3218, 0x1ae9: 0x3218, + 0x1aea: 0x321b, 0x1aeb: 0x321b, 0x1aec: 0x321b, 0x1aed: 0x321b, 0x1aee: 0x321e, 0x1aef: 0x321e, + 0x1af0: 0x06c4, 0x1af1: 0x06c4, + // Block 0x6c, offset 0x1b00 + 0x1b13: 0x3221, 0x1b14: 0x3221, 0x1b15: 0x3221, 0x1b16: 0x3221, 0x1b17: 0x3224, + 0x1b18: 0x3224, 0x1b19: 0x3227, 0x1b1a: 0x3227, 0x1b1b: 0x322a, 0x1b1c: 0x322a, 0x1b1d: 0x06b0, + 0x1b1e: 0x322d, 0x1b1f: 0x322d, 0x1b20: 0x3230, 0x1b21: 0x3230, 0x1b22: 0x3233, 0x1b23: 0x3233, + 0x1b24: 0x3236, 0x1b25: 0x3236, 0x1b26: 0x3236, 0x1b27: 0x3236, 0x1b28: 0x3239, 0x1b29: 0x3239, + 0x1b2a: 0x323c, 0x1b2b: 0x323c, 0x1b2c: 0x3243, 0x1b2d: 0x3243, 0x1b2e: 0x324a, 0x1b2f: 0x324a, + 0x1b30: 0x3251, 0x1b31: 0x3251, 0x1b32: 0x3258, 0x1b33: 0x3258, 0x1b34: 0x325f, 0x1b35: 0x325f, + 0x1b36: 0x3266, 0x1b37: 0x3266, 0x1b38: 0x3266, 0x1b39: 0x326d, 0x1b3a: 0x326d, 0x1b3b: 0x326d, + 0x1b3c: 0x3274, 0x1b3d: 0x3274, 0x1b3e: 0x3274, 0x1b3f: 0x3274, + // Block 0x6d, offset 0x1b40 + 0x1b40: 0x3277, 0x1b41: 0x327e, 0x1b42: 0x3285, 0x1b43: 0x326d, 0x1b44: 0x328c, 0x1b45: 0x3293, + 0x1b46: 0x3298, 0x1b47: 0x329d, 0x1b48: 0x32a2, 0x1b49: 0x32a7, 0x1b4a: 0x32ac, 0x1b4b: 0x32b1, + 0x1b4c: 0x32b6, 0x1b4d: 0x32bb, 0x1b4e: 0x32c0, 0x1b4f: 0x32c5, 0x1b50: 0x32ca, 0x1b51: 0x32cf, + 0x1b52: 0x32d4, 0x1b53: 0x32d9, 0x1b54: 0x32de, 0x1b55: 0x32e3, 0x1b56: 0x32e8, 0x1b57: 0x32ed, + 0x1b58: 0x32f2, 0x1b59: 0x32f7, 0x1b5a: 0x32fc, 0x1b5b: 0x3301, 0x1b5c: 0x3306, 0x1b5d: 0x330b, + 0x1b5e: 0x3310, 0x1b5f: 0x3315, 0x1b60: 0x331a, 0x1b61: 0x331f, 0x1b62: 0x3324, 0x1b63: 0x3329, + 0x1b64: 0x332e, 0x1b65: 0x3333, 0x1b66: 0x3338, 0x1b67: 0x333d, 0x1b68: 0x3342, 0x1b69: 0x3347, + 0x1b6a: 0x334c, 0x1b6b: 0x3351, 0x1b6c: 0x3356, 0x1b6d: 0x335b, 0x1b6e: 0x3360, 0x1b6f: 0x3365, + 0x1b70: 0x336a, 0x1b71: 0x336f, 0x1b72: 0x3374, 0x1b73: 0x3379, 0x1b74: 0x337e, 0x1b75: 0x3383, + 0x1b76: 0x3388, 0x1b77: 0x338d, 0x1b78: 0x3392, 0x1b79: 0x3397, 0x1b7a: 0x339c, 0x1b7b: 0x33a1, + 0x1b7c: 0x33a6, 0x1b7d: 0x33ab, 0x1b7e: 0x33b0, 0x1b7f: 0x33b5, + // Block 0x6e, offset 0x1b80 + 0x1b80: 0x33ba, 0x1b81: 0x33bf, 0x1b82: 0x33c4, 0x1b83: 0x33c9, 0x1b84: 0x33ce, 0x1b85: 0x33d3, + 0x1b86: 0x33d8, 0x1b87: 0x33dd, 0x1b88: 0x33e2, 0x1b89: 0x33e7, 0x1b8a: 0x33ec, 0x1b8b: 0x33f1, + 0x1b8c: 0x33f6, 0x1b8d: 0x33fb, 0x1b8e: 0x3400, 0x1b8f: 0x3405, 0x1b90: 0x340a, 0x1b91: 0x340f, + 0x1b92: 0x3414, 0x1b93: 0x3419, 0x1b94: 0x341e, 0x1b95: 0x3423, 0x1b96: 0x3428, 0x1b97: 0x342d, + 0x1b98: 0x3432, 0x1b99: 0x3437, 0x1b9a: 0x343c, 0x1b9b: 0x3441, 0x1b9c: 0x3446, 0x1b9d: 0x344b, + 0x1b9e: 0x3450, 0x1b9f: 0x3456, 0x1ba0: 0x345c, 0x1ba1: 0x3462, 0x1ba2: 0x3468, 0x1ba3: 0x346e, + 0x1ba4: 0x3474, 0x1ba5: 0x347b, 0x1ba6: 0x3285, 0x1ba7: 0x3482, 0x1ba8: 0x326d, 0x1ba9: 0x328c, + 0x1baa: 0x3489, 0x1bab: 0x348e, 0x1bac: 0x32a2, 0x1bad: 0x3493, 0x1bae: 0x32a7, 0x1baf: 0x32ac, + 0x1bb0: 0x3498, 0x1bb1: 0x349d, 0x1bb2: 0x32c0, 0x1bb3: 0x34a2, 0x1bb4: 0x32c5, 0x1bb5: 0x32ca, + 0x1bb6: 0x34a7, 0x1bb7: 0x34ac, 0x1bb8: 0x32d4, 0x1bb9: 0x34b1, 0x1bba: 0x32d9, 0x1bbb: 0x32de, + 0x1bbc: 0x336f, 0x1bbd: 0x3374, 0x1bbe: 0x3383, 0x1bbf: 0x3388, + // Block 0x6f, offset 0x1bc0 + 0x1bc0: 0x338d, 0x1bc1: 0x33a1, 0x1bc2: 0x33a6, 0x1bc3: 0x33ab, 0x1bc4: 0x33b0, 0x1bc5: 0x33c4, + 0x1bc6: 0x33c9, 0x1bc7: 0x33ce, 0x1bc8: 0x34b6, 0x1bc9: 0x33e2, 0x1bca: 0x34bb, 0x1bcb: 0x34c0, + 0x1bcc: 0x3400, 0x1bcd: 0x34c5, 0x1bce: 0x3405, 0x1bcf: 0x340a, 0x1bd0: 0x344b, 0x1bd1: 0x34ca, + 0x1bd2: 0x34cf, 0x1bd3: 0x3432, 0x1bd4: 0x34d4, 0x1bd5: 0x3437, 0x1bd6: 0x343c, 0x1bd7: 0x3277, + 0x1bd8: 0x327e, 0x1bd9: 0x34d9, 0x1bda: 0x3285, 0x1bdb: 0x34e0, 0x1bdc: 0x3293, 0x1bdd: 0x3298, + 0x1bde: 0x329d, 0x1bdf: 0x32a2, 0x1be0: 0x34e7, 0x1be1: 0x32b1, 0x1be2: 0x32b6, 0x1be3: 0x32bb, + 0x1be4: 0x32c0, 0x1be5: 0x34ec, 0x1be6: 0x32d4, 0x1be7: 0x32e3, 0x1be8: 0x32e8, 0x1be9: 0x32ed, + 0x1bea: 0x32f2, 0x1beb: 0x32f7, 0x1bec: 0x3301, 0x1bed: 0x3306, 0x1bee: 0x330b, 0x1bef: 0x3310, + 0x1bf0: 0x3315, 0x1bf1: 0x331a, 0x1bf2: 0x34f1, 0x1bf3: 0x331f, 0x1bf4: 0x3324, 0x1bf5: 0x3329, + 0x1bf6: 0x332e, 0x1bf7: 0x3333, 0x1bf8: 0x3338, 0x1bf9: 0x3342, 0x1bfa: 0x3347, 0x1bfb: 0x334c, + 0x1bfc: 0x3351, 0x1bfd: 0x3356, 0x1bfe: 0x335b, 0x1bff: 0x3360, + // Block 0x70, offset 0x1c00 + 0x1c00: 0x3365, 0x1c01: 0x336a, 0x1c02: 0x3379, 0x1c03: 0x337e, 0x1c04: 0x3392, 0x1c05: 0x3397, + 0x1c06: 0x339c, 0x1c07: 0x33a1, 0x1c08: 0x33a6, 0x1c09: 0x33b5, 0x1c0a: 0x33ba, 0x1c0b: 0x33bf, + 0x1c0c: 0x33c4, 0x1c0d: 0x34f6, 0x1c0e: 0x33d3, 0x1c0f: 0x33d8, 0x1c10: 0x33dd, 0x1c11: 0x33e2, + 0x1c12: 0x33f1, 0x1c13: 0x33f6, 0x1c14: 0x33fb, 0x1c15: 0x3400, 0x1c16: 0x34fb, 0x1c17: 0x340f, + 0x1c18: 0x3414, 0x1c19: 0x3500, 0x1c1a: 0x3423, 0x1c1b: 0x3428, 0x1c1c: 0x342d, 0x1c1d: 0x3432, + 0x1c1e: 0x3505, 0x1c1f: 0x3285, 0x1c20: 0x34e0, 0x1c21: 0x32a2, 0x1c22: 0x34e7, 0x1c23: 0x32c0, + 0x1c24: 0x34ec, 0x1c25: 0x32d4, 0x1c26: 0x350a, 0x1c27: 0x3315, 0x1c28: 0x350f, 0x1c29: 0x3514, + 0x1c2a: 0x3519, 0x1c2b: 0x33a1, 0x1c2c: 0x33a6, 0x1c2d: 0x33c4, 0x1c2e: 0x3400, 0x1c2f: 0x34fb, + 0x1c30: 0x3432, 0x1c31: 0x3505, 0x1c32: 0x351e, 0x1c33: 0x3525, 0x1c34: 0x352c, 0x1c35: 0x3533, + 0x1c36: 0x3538, 0x1c37: 0x353d, 0x1c38: 0x3542, 0x1c39: 0x3547, 0x1c3a: 0x354c, 0x1c3b: 0x3551, + 0x1c3c: 0x3556, 0x1c3d: 0x355b, 0x1c3e: 0x3560, 0x1c3f: 0x3565, + // Block 0x71, offset 0x1c40 + 0x1c40: 0x356a, 0x1c41: 0x356f, 0x1c42: 0x3574, 0x1c43: 0x3579, 0x1c44: 0x357e, 0x1c45: 0x3583, + 0x1c46: 0x3588, 0x1c47: 0x358d, 0x1c48: 0x3592, 0x1c49: 0x3597, 0x1c4a: 0x359c, 0x1c4b: 0x35a1, + 0x1c4c: 0x3514, 0x1c4d: 0x35a6, 0x1c4e: 0x35ab, 0x1c4f: 0x35b0, 0x1c50: 0x35b5, 0x1c51: 0x3533, + 0x1c52: 0x3538, 0x1c53: 0x353d, 0x1c54: 0x3542, 0x1c55: 0x3547, 0x1c56: 0x354c, 0x1c57: 0x3551, + 0x1c58: 0x3556, 0x1c59: 0x355b, 0x1c5a: 0x3560, 0x1c5b: 0x3565, 0x1c5c: 0x356a, 0x1c5d: 0x356f, + 0x1c5e: 0x3574, 0x1c5f: 0x3579, 0x1c60: 0x357e, 0x1c61: 0x3583, 0x1c62: 0x3588, 0x1c63: 0x358d, + 0x1c64: 0x3592, 0x1c65: 0x3597, 0x1c66: 0x359c, 0x1c67: 0x35a1, 0x1c68: 0x3514, 0x1c69: 0x35a6, + 0x1c6a: 0x35ab, 0x1c6b: 0x35b0, 0x1c6c: 0x35b5, 0x1c6d: 0x3597, 0x1c6e: 0x359c, 0x1c6f: 0x35a1, + 0x1c70: 0x3514, 0x1c71: 0x350f, 0x1c72: 0x3519, 0x1c73: 0x333d, 0x1c74: 0x3306, 0x1c75: 0x330b, + 0x1c76: 0x3310, 0x1c77: 0x3597, 0x1c78: 0x359c, 0x1c79: 0x35a1, 0x1c7a: 0x333d, 0x1c7b: 0x3342, + 0x1c7c: 0x35ba, 0x1c7d: 0x35ba, + // Block 0x72, offset 0x1c80 + 0x1c90: 0x35bf, 0x1c91: 0x35c6, + 0x1c92: 0x35c6, 0x1c93: 0x35cd, 0x1c94: 0x35d4, 0x1c95: 0x35db, 0x1c96: 0x35e2, 0x1c97: 0x35e9, + 0x1c98: 0x35f0, 0x1c99: 0x35f0, 0x1c9a: 0x35f7, 0x1c9b: 0x35fe, 0x1c9c: 0x3605, 0x1c9d: 0x360c, + 0x1c9e: 0x3613, 0x1c9f: 0x361a, 0x1ca0: 0x361a, 0x1ca1: 0x3621, 0x1ca2: 0x3628, 0x1ca3: 0x3628, + 0x1ca4: 0x362f, 0x1ca5: 0x362f, 0x1ca6: 0x3636, 0x1ca7: 0x363d, 0x1ca8: 0x363d, 0x1ca9: 0x3644, + 0x1caa: 0x364b, 0x1cab: 0x364b, 0x1cac: 0x3652, 0x1cad: 0x3652, 0x1cae: 0x3659, 0x1caf: 0x3660, + 0x1cb0: 0x3660, 0x1cb1: 0x3667, 0x1cb2: 0x3667, 0x1cb3: 0x366e, 0x1cb4: 0x3675, 0x1cb5: 0x367c, + 0x1cb6: 0x3683, 0x1cb7: 0x3683, 0x1cb8: 0x368a, 0x1cb9: 0x3691, 0x1cba: 0x3698, 0x1cbb: 0x369f, + 0x1cbc: 0x36a6, 0x1cbd: 0x36a6, 0x1cbe: 0x36ad, 0x1cbf: 0x36b4, + // Block 0x73, offset 0x1cc0 + 0x1cc0: 0x36bb, 0x1cc1: 0x36c2, 0x1cc2: 0x36c9, 0x1cc3: 0x36d0, 0x1cc4: 0x36d0, 0x1cc5: 0x36d7, + 0x1cc6: 0x36d7, 0x1cc7: 0x36de, 0x1cc8: 0x36de, 0x1cc9: 0x36e5, 0x1cca: 0x36ec, 0x1ccb: 0x36f3, + 0x1ccc: 0x36fa, 0x1ccd: 0x3701, 0x1cce: 0x3708, 0x1ccf: 0x370f, + 0x1cd2: 0x3716, 0x1cd3: 0x371d, 0x1cd4: 0x3724, 0x1cd5: 0x372b, 0x1cd6: 0x3732, 0x1cd7: 0x3739, + 0x1cd8: 0x3739, 0x1cd9: 0x3740, 0x1cda: 0x3747, 0x1cdb: 0x374e, 0x1cdc: 0x3755, 0x1cdd: 0x3755, + 0x1cde: 0x375c, 0x1cdf: 0x3763, 0x1ce0: 0x376a, 0x1ce1: 0x3771, 0x1ce2: 0x3778, 0x1ce3: 0x377f, + 0x1ce4: 0x3786, 0x1ce5: 0x378d, 0x1ce6: 0x3794, 0x1ce7: 0x379b, 0x1ce8: 0x37a2, 0x1ce9: 0x37a9, + 0x1cea: 0x37b0, 0x1ceb: 0x37b7, 0x1cec: 0x37be, 0x1ced: 0x37c5, 0x1cee: 0x37cc, 0x1cef: 0x37d3, + 0x1cf0: 0x37da, 0x1cf1: 0x37e1, 0x1cf2: 0x37e8, 0x1cf3: 0x37ef, 0x1cf4: 0x36ad, 0x1cf5: 0x36bb, + 0x1cf6: 0x37f6, 0x1cf7: 0x37fd, 0x1cf8: 0x3804, 0x1cf9: 0x380b, 0x1cfa: 0x3812, 0x1cfb: 0x3819, + 0x1cfc: 0x3812, 0x1cfd: 0x3804, 0x1cfe: 0x3820, 0x1cff: 0x3827, + // Block 0x74, offset 0x1d00 + 0x1d00: 0x382e, 0x1d01: 0x3835, 0x1d02: 0x383c, 0x1d03: 0x3819, 0x1d04: 0x367c, 0x1d05: 0x3636, + 0x1d06: 0x3843, 0x1d07: 0x384a, + 0x1d30: 0x3851, 0x1d31: 0x3858, 0x1d32: 0x385f, 0x1d33: 0x3868, 0x1d34: 0x3871, 0x1d35: 0x387a, + 0x1d36: 0x3883, 0x1d37: 0x388c, 0x1d38: 0x3895, 0x1d39: 0x389e, 0x1d3a: 0x38a5, 0x1d3b: 0x38c7, + 0x1d3c: 0x38d7, + // Block 0x75, offset 0x1d40 + 0x1d50: 0x38e0, 0x1d51: 0x38e2, + 0x1d52: 0x38e6, 0x1d53: 0x38ea, 0x1d54: 0x04e1, 0x1d55: 0x38ec, 0x1d56: 0x38ee, 0x1d57: 0x38f0, + 0x1d58: 0x38f4, 0x1d59: 0x1443, + 0x1d70: 0x1440, 0x1d71: 0x38f8, 0x1d72: 0x38fc, 0x1d73: 0x3900, 0x1d74: 0x3900, 0x1d75: 0x149c, + 0x1d76: 0x149e, 0x1d77: 0x3902, 0x1d78: 0x3904, 0x1d79: 0x3906, 0x1d7a: 0x390a, 0x1d7b: 0x390e, + 0x1d7c: 0x3912, 0x1d7d: 0x3916, 0x1d7e: 0x391a, 0x1d7f: 0x16c3, + // Block 0x76, offset 0x1d80 + 0x1d80: 0x16c7, 0x1d81: 0x391e, 0x1d82: 0x3922, 0x1d83: 0x3926, 0x1d84: 0x392a, + 0x1d87: 0x392e, 0x1d88: 0x3930, 0x1d89: 0x146c, 0x1d8a: 0x146c, 0x1d8b: 0x146c, + 0x1d8c: 0x146c, 0x1d8d: 0x3900, 0x1d8e: 0x3900, 0x1d8f: 0x3900, 0x1d90: 0x38e0, 0x1d91: 0x38e2, + 0x1d92: 0x143e, 0x1d94: 0x04e1, 0x1d95: 0x38ea, 0x1d96: 0x38ee, 0x1d97: 0x38ec, + 0x1d98: 0x38f8, 0x1d99: 0x149c, 0x1d9a: 0x149e, 0x1d9b: 0x3902, 0x1d9c: 0x3904, 0x1d9d: 0x3906, + 0x1d9e: 0x390a, 0x1d9f: 0x3932, 0x1da0: 0x3934, 0x1da1: 0x3936, 0x1da2: 0x1494, 0x1da3: 0x3938, + 0x1da4: 0x393a, 0x1da5: 0x393c, 0x1da6: 0x149a, 0x1da8: 0x393e, 0x1da9: 0x3940, + 0x1daa: 0x3942, 0x1dab: 0x3944, + 0x1db0: 0x3946, 0x1db1: 0x394a, 0x1db2: 0x394f, 0x1db4: 0x3953, + 0x1db6: 0x3957, 0x1db7: 0x395b, 0x1db8: 0x3960, 0x1db9: 0x3964, 0x1dba: 0x3969, 0x1dbb: 0x396d, + 0x1dbc: 0x3972, 0x1dbd: 0x3976, 0x1dbe: 0x397b, 0x1dbf: 0x397f, + // Block 0x77, offset 0x1dc0 + 0x1dc0: 0x3984, 0x1dc1: 0x068d, 0x1dc2: 0x068d, 0x1dc3: 0x0692, 0x1dc4: 0x0692, 0x1dc5: 0x0697, + 0x1dc6: 0x0697, 0x1dc7: 0x069c, 0x1dc8: 0x069c, 0x1dc9: 0x06a1, 0x1dca: 0x06a1, 0x1dcb: 0x06a1, + 0x1dcc: 0x06a1, 0x1dcd: 0x3987, 0x1dce: 0x3987, 0x1dcf: 0x398a, 0x1dd0: 0x398a, 0x1dd1: 0x398a, + 0x1dd2: 0x398a, 0x1dd3: 0x398d, 0x1dd4: 0x398d, 0x1dd5: 0x3990, 0x1dd6: 0x3990, 0x1dd7: 0x3990, + 0x1dd8: 0x3990, 0x1dd9: 0x3993, 0x1dda: 0x3993, 0x1ddb: 0x3993, 0x1ddc: 0x3993, 0x1ddd: 0x3996, + 0x1dde: 0x3996, 0x1ddf: 0x3996, 0x1de0: 0x3996, 0x1de1: 0x3999, 0x1de2: 0x3999, 0x1de3: 0x3999, + 0x1de4: 0x3999, 0x1de5: 0x399c, 0x1de6: 0x399c, 0x1de7: 0x399c, 0x1de8: 0x399c, 0x1de9: 0x399f, + 0x1dea: 0x399f, 0x1deb: 0x39a2, 0x1dec: 0x39a2, 0x1ded: 0x39a5, 0x1dee: 0x39a5, 0x1def: 0x39a8, + 0x1df0: 0x39a8, 0x1df1: 0x39ab, 0x1df2: 0x39ab, 0x1df3: 0x39ab, 0x1df4: 0x39ab, 0x1df5: 0x39ae, + 0x1df6: 0x39ae, 0x1df7: 0x39ae, 0x1df8: 0x39ae, 0x1df9: 0x39b1, 0x1dfa: 0x39b1, 0x1dfb: 0x39b1, + 0x1dfc: 0x39b1, 0x1dfd: 0x39b4, 0x1dfe: 0x39b4, 0x1dff: 0x39b4, + // Block 0x78, offset 0x1e00 + 0x1e00: 0x39b4, 0x1e01: 0x39b7, 0x1e02: 0x39b7, 0x1e03: 0x39b7, 0x1e04: 0x39b7, 0x1e05: 0x39ba, + 0x1e06: 0x39ba, 0x1e07: 0x39ba, 0x1e08: 0x39ba, 0x1e09: 0x39bd, 0x1e0a: 0x39bd, 0x1e0b: 0x39bd, + 0x1e0c: 0x39bd, 0x1e0d: 0x39c0, 0x1e0e: 0x39c0, 0x1e0f: 0x39c0, 0x1e10: 0x39c0, 0x1e11: 0x39c3, + 0x1e12: 0x39c3, 0x1e13: 0x39c3, 0x1e14: 0x39c3, 0x1e15: 0x39c6, 0x1e16: 0x39c6, 0x1e17: 0x39c6, + 0x1e18: 0x39c6, 0x1e19: 0x39c9, 0x1e1a: 0x39c9, 0x1e1b: 0x39c9, 0x1e1c: 0x39c9, 0x1e1d: 0x39cc, + 0x1e1e: 0x39cc, 0x1e1f: 0x39cc, 0x1e20: 0x39cc, 0x1e21: 0x39cf, 0x1e22: 0x39cf, 0x1e23: 0x39cf, + 0x1e24: 0x39cf, 0x1e25: 0x39d2, 0x1e26: 0x39d2, 0x1e27: 0x39d2, 0x1e28: 0x39d2, 0x1e29: 0x39d5, + 0x1e2a: 0x39d5, 0x1e2b: 0x39d5, 0x1e2c: 0x39d5, 0x1e2d: 0x39d8, 0x1e2e: 0x39d8, 0x1e2f: 0x3239, + 0x1e30: 0x3239, 0x1e31: 0x39db, 0x1e32: 0x39db, 0x1e33: 0x39db, 0x1e34: 0x39db, 0x1e35: 0x39de, + 0x1e36: 0x39de, 0x1e37: 0x39e5, 0x1e38: 0x39e5, 0x1e39: 0x39ec, 0x1e3a: 0x39ec, 0x1e3b: 0x39f3, + 0x1e3c: 0x39f3, + // Block 0x79, offset 0x1e40 + 0x1e41: 0x38ec, 0x1e42: 0x39f8, 0x1e43: 0x3932, 0x1e44: 0x3940, 0x1e45: 0x3942, + 0x1e46: 0x3934, 0x1e47: 0x39fa, 0x1e48: 0x149c, 0x1e49: 0x149e, 0x1e4a: 0x3936, 0x1e4b: 0x1494, + 0x1e4c: 0x38e0, 0x1e4d: 0x3938, 0x1e4e: 0x143e, 0x1e4f: 0x39fc, 0x1e50: 0x1486, 0x1e51: 0x001c, + 0x1e52: 0x000d, 0x1e53: 0x000f, 0x1e54: 0x1488, 0x1e55: 0x148a, 0x1e56: 0x148c, 0x1e57: 0x148e, + 0x1e58: 0x1490, 0x1e59: 0x1492, 0x1e5a: 0x38ea, 0x1e5b: 0x04e1, 0x1e5c: 0x393a, 0x1e5d: 0x149a, + 0x1e5e: 0x393c, 0x1e5f: 0x38ee, 0x1e60: 0x3944, 0x1e61: 0x0906, 0x1e62: 0x090b, 0x1e63: 0x14ad, + 0x1e64: 0x090d, 0x1e65: 0x090f, 0x1e66: 0x14d9, 0x1e67: 0x0914, 0x1e68: 0x0916, 0x1e69: 0x0918, + 0x1e6a: 0x091a, 0x1e6b: 0x091c, 0x1e6c: 0x091e, 0x1e6d: 0x0920, 0x1e6e: 0x0922, 0x1e6f: 0x0924, + 0x1e70: 0x0929, 0x1e71: 0x14c8, 0x1e72: 0x092b, 0x1e73: 0x17f6, 0x1e74: 0x092d, 0x1e75: 0x092f, + 0x1e76: 0x155f, 0x1e77: 0x0931, 0x1e78: 0x1570, 0x1e79: 0x17f8, 0x1e7a: 0x14d4, 0x1e7b: 0x392e, + 0x1e7c: 0x393e, 0x1e7d: 0x3930, 0x1e7e: 0x39fe, 0x1e7f: 0x3900, + // Block 0x7a, offset 0x1e80 + 0x1e80: 0x13f7, 0x1e81: 0x0007, 0x1e82: 0x093d, 0x1e83: 0x0984, 0x1e84: 0x093f, 0x1e85: 0x0941, + 0x1e86: 0x098c, 0x1e87: 0x094c, 0x1e88: 0x0494, 0x1e89: 0x097c, 0x1e8a: 0x0499, 0x1e8b: 0x094e, + 0x1e8c: 0x04c5, 0x1e8d: 0x0950, 0x1e8e: 0x14a0, 0x1e8f: 0x001e, 0x1e90: 0x0960, 0x1e91: 0x17fa, + 0x1e92: 0x049b, 0x1e93: 0x02c8, 0x1e94: 0x0962, 0x1e95: 0x0964, 0x1e96: 0x096d, 0x1e97: 0x04a6, + 0x1e98: 0x04c7, 0x1e99: 0x04a8, 0x1e9a: 0x09df, 0x1e9b: 0x3902, 0x1e9c: 0x3a00, 0x1e9d: 0x3904, + 0x1e9e: 0x3a02, 0x1e9f: 0x3a04, 0x1ea0: 0x3a08, 0x1ea1: 0x38e6, 0x1ea2: 0x391e, 0x1ea3: 0x3922, + 0x1ea4: 0x38e2, 0x1ea5: 0x3a0c, 0x1ea6: 0x2321, 0x1ea7: 0x3a10, 0x1ea8: 0x3a14, 0x1ea9: 0x3a18, + 0x1eaa: 0x3a1c, 0x1eab: 0x3a20, 0x1eac: 0x3a24, 0x1ead: 0x3a28, 0x1eae: 0x3a2c, 0x1eaf: 0x3a30, + 0x1eb0: 0x3a34, 0x1eb1: 0x2269, 0x1eb2: 0x226d, 0x1eb3: 0x2271, 0x1eb4: 0x2275, 0x1eb5: 0x2279, + 0x1eb6: 0x227d, 0x1eb7: 0x2281, 0x1eb8: 0x2285, 0x1eb9: 0x2289, 0x1eba: 0x228d, 0x1ebb: 0x2291, + 0x1ebc: 0x2295, 0x1ebd: 0x2299, 0x1ebe: 0x229d, 0x1ebf: 0x22a1, + // Block 0x7b, offset 0x1ec0 + 0x1ec0: 0x22a5, 0x1ec1: 0x22a9, 0x1ec2: 0x22ad, 0x1ec3: 0x22b1, 0x1ec4: 0x22b5, 0x1ec5: 0x22b9, + 0x1ec6: 0x22bd, 0x1ec7: 0x22c1, 0x1ec8: 0x22c5, 0x1ec9: 0x22c9, 0x1eca: 0x22cd, 0x1ecb: 0x22d1, + 0x1ecc: 0x22d5, 0x1ecd: 0x22d9, 0x1ece: 0x22dd, 0x1ecf: 0x22e1, 0x1ed0: 0x22e5, 0x1ed1: 0x22e9, + 0x1ed2: 0x22ed, 0x1ed3: 0x22f1, 0x1ed4: 0x22f5, 0x1ed5: 0x22f9, 0x1ed6: 0x22fd, 0x1ed7: 0x2301, + 0x1ed8: 0x2305, 0x1ed9: 0x2309, 0x1eda: 0x230d, 0x1edb: 0x2311, 0x1edc: 0x2315, 0x1edd: 0x3a38, + 0x1ede: 0x3a3c, 0x1edf: 0x3a40, 0x1ee0: 0x1e04, 0x1ee1: 0x1d38, 0x1ee2: 0x1d3c, 0x1ee3: 0x1d40, + 0x1ee4: 0x1d44, 0x1ee5: 0x1d48, 0x1ee6: 0x1d4c, 0x1ee7: 0x1d50, 0x1ee8: 0x1d54, 0x1ee9: 0x1d58, + 0x1eea: 0x1d5c, 0x1eeb: 0x1d60, 0x1eec: 0x1d64, 0x1eed: 0x1d68, 0x1eee: 0x1d6c, 0x1eef: 0x1d70, + 0x1ef0: 0x1d74, 0x1ef1: 0x1d78, 0x1ef2: 0x1d7c, 0x1ef3: 0x1d80, 0x1ef4: 0x1d84, 0x1ef5: 0x1d88, + 0x1ef6: 0x1d8c, 0x1ef7: 0x1d90, 0x1ef8: 0x1d94, 0x1ef9: 0x1d98, 0x1efa: 0x1d9c, 0x1efb: 0x1da0, + 0x1efc: 0x1da4, 0x1efd: 0x1da8, 0x1efe: 0x1dac, + // Block 0x7c, offset 0x1f00 + 0x1f02: 0x1db0, 0x1f03: 0x1db4, 0x1f04: 0x1db8, 0x1f05: 0x1dbc, + 0x1f06: 0x1dc0, 0x1f07: 0x1dc4, 0x1f0a: 0x1dc8, 0x1f0b: 0x1dcc, + 0x1f0c: 0x1dd0, 0x1f0d: 0x1dd4, 0x1f0e: 0x1dd8, 0x1f0f: 0x1ddc, + 0x1f12: 0x1de0, 0x1f13: 0x1de4, 0x1f14: 0x1de8, 0x1f15: 0x1dec, 0x1f16: 0x1df0, 0x1f17: 0x1df4, + 0x1f1a: 0x1df8, 0x1f1b: 0x1dfc, 0x1f1c: 0x1e00, + 0x1f20: 0x3a44, 0x1f21: 0x3a47, 0x1f22: 0x3a4a, 0x1f23: 0x0009, + 0x1f24: 0x3a4d, 0x1f25: 0x3a50, 0x1f26: 0x3a53, 0x1f28: 0x3a57, 0x1f29: 0x3a5b, + 0x1f2a: 0x3a5f, 0x1f2b: 0x3a63, 0x1f2c: 0x3a67, 0x1f2d: 0x3a6b, 0x1f2e: 0x3a6f, + // Block 0x7d, offset 0x1f40 + 0x1f5a: 0x3a73, 0x1f5c: 0x3a7c, + 0x1f6b: 0x3a85, + // Block 0x7e, offset 0x1f80 + 0x1f9e: 0x3a8e, 0x1f9f: 0x3a97, 0x1fa0: 0x3aa0, 0x1fa1: 0x3aad, 0x1fa2: 0x3aba, 0x1fa3: 0x3ac7, + 0x1fa4: 0x3ad4, + // Block 0x7f, offset 0x1fc0 + 0x1ffb: 0x3ae1, + 0x1ffc: 0x3aea, 0x1ffd: 0x3af3, 0x1ffe: 0x3b00, 0x1fff: 0x3b0d, + // Block 0x80, offset 0x2000 + 0x2000: 0x3b1a, + // Block 0x81, offset 0x2040 + 0x2040: 0x0906, 0x2041: 0x090b, 0x2042: 0x14ad, 0x2043: 0x090d, 0x2044: 0x090f, 0x2045: 0x14d9, + 0x2046: 0x0914, 0x2047: 0x0916, 0x2048: 0x0918, 0x2049: 0x091a, 0x204a: 0x091c, 0x204b: 0x091e, + 0x204c: 0x0920, 0x204d: 0x0922, 0x204e: 0x0924, 0x204f: 0x0929, 0x2050: 0x14c8, 0x2051: 0x092b, + 0x2052: 0x17f6, 0x2053: 0x092d, 0x2054: 0x092f, 0x2055: 0x155f, 0x2056: 0x0931, 0x2057: 0x1570, + 0x2058: 0x17f8, 0x2059: 0x14d4, 0x205a: 0x0007, 0x205b: 0x093d, 0x205c: 0x0984, 0x205d: 0x093f, + 0x205e: 0x0941, 0x205f: 0x098c, 0x2060: 0x094c, 0x2061: 0x0494, 0x2062: 0x097c, 0x2063: 0x0499, + 0x2064: 0x094e, 0x2065: 0x04c5, 0x2066: 0x0950, 0x2067: 0x14a0, 0x2068: 0x001e, 0x2069: 0x0960, + 0x206a: 0x17fa, 0x206b: 0x049b, 0x206c: 0x02c8, 0x206d: 0x0962, 0x206e: 0x0964, 0x206f: 0x096d, + 0x2070: 0x04a6, 0x2071: 0x04c7, 0x2072: 0x04a8, 0x2073: 0x09df, 0x2074: 0x0906, 0x2075: 0x090b, + 0x2076: 0x14ad, 0x2077: 0x090d, 0x2078: 0x090f, 0x2079: 0x14d9, 0x207a: 0x0914, 0x207b: 0x0916, + 0x207c: 0x0918, 0x207d: 0x091a, 0x207e: 0x091c, 0x207f: 0x091e, + // Block 0x82, offset 0x2080 + 0x2080: 0x0920, 0x2081: 0x0922, 0x2082: 0x0924, 0x2083: 0x0929, 0x2084: 0x14c8, 0x2085: 0x092b, + 0x2086: 0x17f6, 0x2087: 0x092d, 0x2088: 0x092f, 0x2089: 0x155f, 0x208a: 0x0931, 0x208b: 0x1570, + 0x208c: 0x17f8, 0x208d: 0x14d4, 0x208e: 0x0007, 0x208f: 0x093d, 0x2090: 0x0984, 0x2091: 0x093f, + 0x2092: 0x0941, 0x2093: 0x098c, 0x2094: 0x094c, 0x2096: 0x097c, 0x2097: 0x0499, + 0x2098: 0x094e, 0x2099: 0x04c5, 0x209a: 0x0950, 0x209b: 0x14a0, 0x209c: 0x001e, 0x209d: 0x0960, + 0x209e: 0x17fa, 0x209f: 0x049b, 0x20a0: 0x02c8, 0x20a1: 0x0962, 0x20a2: 0x0964, 0x20a3: 0x096d, + 0x20a4: 0x04a6, 0x20a5: 0x04c7, 0x20a6: 0x04a8, 0x20a7: 0x09df, 0x20a8: 0x0906, 0x20a9: 0x090b, + 0x20aa: 0x14ad, 0x20ab: 0x090d, 0x20ac: 0x090f, 0x20ad: 0x14d9, 0x20ae: 0x0914, 0x20af: 0x0916, + 0x20b0: 0x0918, 0x20b1: 0x091a, 0x20b2: 0x091c, 0x20b3: 0x091e, 0x20b4: 0x0920, 0x20b5: 0x0922, + 0x20b6: 0x0924, 0x20b7: 0x0929, 0x20b8: 0x14c8, 0x20b9: 0x092b, 0x20ba: 0x17f6, 0x20bb: 0x092d, + 0x20bc: 0x092f, 0x20bd: 0x155f, 0x20be: 0x0931, 0x20bf: 0x1570, + // Block 0x83, offset 0x20c0 + 0x20c0: 0x17f8, 0x20c1: 0x14d4, 0x20c2: 0x0007, 0x20c3: 0x093d, 0x20c4: 0x0984, 0x20c5: 0x093f, + 0x20c6: 0x0941, 0x20c7: 0x098c, 0x20c8: 0x094c, 0x20c9: 0x0494, 0x20ca: 0x097c, 0x20cb: 0x0499, + 0x20cc: 0x094e, 0x20cd: 0x04c5, 0x20ce: 0x0950, 0x20cf: 0x14a0, 0x20d0: 0x001e, 0x20d1: 0x0960, + 0x20d2: 0x17fa, 0x20d3: 0x049b, 0x20d4: 0x02c8, 0x20d5: 0x0962, 0x20d6: 0x0964, 0x20d7: 0x096d, + 0x20d8: 0x04a6, 0x20d9: 0x04c7, 0x20da: 0x04a8, 0x20db: 0x09df, 0x20dc: 0x0906, + 0x20de: 0x14ad, 0x20df: 0x090d, 0x20e2: 0x0914, + 0x20e5: 0x091a, 0x20e6: 0x091c, 0x20e9: 0x0922, + 0x20ea: 0x0924, 0x20eb: 0x0929, 0x20ec: 0x14c8, 0x20ee: 0x17f6, 0x20ef: 0x092d, + 0x20f0: 0x092f, 0x20f1: 0x155f, 0x20f2: 0x0931, 0x20f3: 0x1570, 0x20f4: 0x17f8, 0x20f5: 0x14d4, + 0x20f6: 0x0007, 0x20f7: 0x093d, 0x20f8: 0x0984, 0x20f9: 0x093f, 0x20fb: 0x098c, + 0x20fd: 0x0494, 0x20fe: 0x097c, 0x20ff: 0x0499, + // Block 0x84, offset 0x2100 + 0x2100: 0x094e, 0x2101: 0x04c5, 0x2102: 0x0950, 0x2103: 0x14a0, 0x2105: 0x0960, + 0x2106: 0x17fa, 0x2107: 0x049b, 0x2108: 0x02c8, 0x2109: 0x0962, 0x210a: 0x0964, 0x210b: 0x096d, + 0x210c: 0x04a6, 0x210d: 0x04c7, 0x210e: 0x04a8, 0x210f: 0x09df, 0x2110: 0x0906, 0x2111: 0x090b, + 0x2112: 0x14ad, 0x2113: 0x090d, 0x2114: 0x090f, 0x2115: 0x14d9, 0x2116: 0x0914, 0x2117: 0x0916, + 0x2118: 0x0918, 0x2119: 0x091a, 0x211a: 0x091c, 0x211b: 0x091e, 0x211c: 0x0920, 0x211d: 0x0922, + 0x211e: 0x0924, 0x211f: 0x0929, 0x2120: 0x14c8, 0x2121: 0x092b, 0x2122: 0x17f6, 0x2123: 0x092d, + 0x2124: 0x092f, 0x2125: 0x155f, 0x2126: 0x0931, 0x2127: 0x1570, 0x2128: 0x17f8, 0x2129: 0x14d4, + 0x212a: 0x0007, 0x212b: 0x093d, 0x212c: 0x0984, 0x212d: 0x093f, 0x212e: 0x0941, 0x212f: 0x098c, + 0x2130: 0x094c, 0x2131: 0x0494, 0x2132: 0x097c, 0x2133: 0x0499, 0x2134: 0x094e, 0x2135: 0x04c5, + 0x2136: 0x0950, 0x2137: 0x14a0, 0x2138: 0x001e, 0x2139: 0x0960, 0x213a: 0x17fa, 0x213b: 0x049b, + 0x213c: 0x02c8, 0x213d: 0x0962, 0x213e: 0x0964, 0x213f: 0x096d, + // Block 0x85, offset 0x2140 + 0x2140: 0x04a6, 0x2141: 0x04c7, 0x2142: 0x04a8, 0x2143: 0x09df, 0x2144: 0x0906, 0x2145: 0x090b, + 0x2147: 0x090d, 0x2148: 0x090f, 0x2149: 0x14d9, 0x214a: 0x0914, + 0x214d: 0x091a, 0x214e: 0x091c, 0x214f: 0x091e, 0x2150: 0x0920, 0x2151: 0x0922, + 0x2152: 0x0924, 0x2153: 0x0929, 0x2154: 0x14c8, 0x2156: 0x17f6, 0x2157: 0x092d, + 0x2158: 0x092f, 0x2159: 0x155f, 0x215a: 0x0931, 0x215b: 0x1570, 0x215c: 0x17f8, + 0x215e: 0x0007, 0x215f: 0x093d, 0x2160: 0x0984, 0x2161: 0x093f, 0x2162: 0x0941, 0x2163: 0x098c, + 0x2164: 0x094c, 0x2165: 0x0494, 0x2166: 0x097c, 0x2167: 0x0499, 0x2168: 0x094e, 0x2169: 0x04c5, + 0x216a: 0x0950, 0x216b: 0x14a0, 0x216c: 0x001e, 0x216d: 0x0960, 0x216e: 0x17fa, 0x216f: 0x049b, + 0x2170: 0x02c8, 0x2171: 0x0962, 0x2172: 0x0964, 0x2173: 0x096d, 0x2174: 0x04a6, 0x2175: 0x04c7, + 0x2176: 0x04a8, 0x2177: 0x09df, 0x2178: 0x0906, 0x2179: 0x090b, 0x217b: 0x090d, + 0x217c: 0x090f, 0x217d: 0x14d9, 0x217e: 0x0914, + // Block 0x86, offset 0x2180 + 0x2180: 0x0918, 0x2181: 0x091a, 0x2182: 0x091c, 0x2183: 0x091e, 0x2184: 0x0920, + 0x2186: 0x0924, 0x218a: 0x17f6, 0x218b: 0x092d, + 0x218c: 0x092f, 0x218d: 0x155f, 0x218e: 0x0931, 0x218f: 0x1570, 0x2190: 0x17f8, + 0x2192: 0x0007, 0x2193: 0x093d, 0x2194: 0x0984, 0x2195: 0x093f, 0x2196: 0x0941, 0x2197: 0x098c, + 0x2198: 0x094c, 0x2199: 0x0494, 0x219a: 0x097c, 0x219b: 0x0499, 0x219c: 0x094e, 0x219d: 0x04c5, + 0x219e: 0x0950, 0x219f: 0x14a0, 0x21a0: 0x001e, 0x21a1: 0x0960, 0x21a2: 0x17fa, 0x21a3: 0x049b, + 0x21a4: 0x02c8, 0x21a5: 0x0962, 0x21a6: 0x0964, 0x21a7: 0x096d, 0x21a8: 0x04a6, 0x21a9: 0x04c7, + 0x21aa: 0x04a8, 0x21ab: 0x09df, 0x21ac: 0x0906, 0x21ad: 0x090b, 0x21ae: 0x14ad, 0x21af: 0x090d, + 0x21b0: 0x090f, 0x21b1: 0x14d9, 0x21b2: 0x0914, 0x21b3: 0x0916, 0x21b4: 0x0918, 0x21b5: 0x091a, + 0x21b6: 0x091c, 0x21b7: 0x091e, 0x21b8: 0x0920, 0x21b9: 0x0922, 0x21ba: 0x0924, 0x21bb: 0x0929, + 0x21bc: 0x14c8, 0x21bd: 0x092b, 0x21be: 0x17f6, 0x21bf: 0x092d, + // Block 0x87, offset 0x21c0 + 0x21c0: 0x092f, 0x21c1: 0x155f, 0x21c2: 0x0931, 0x21c3: 0x1570, 0x21c4: 0x17f8, 0x21c5: 0x14d4, + 0x21c6: 0x0007, 0x21c7: 0x093d, 0x21c8: 0x0984, 0x21c9: 0x093f, 0x21ca: 0x0941, 0x21cb: 0x098c, + 0x21cc: 0x094c, 0x21cd: 0x0494, 0x21ce: 0x097c, 0x21cf: 0x0499, 0x21d0: 0x094e, 0x21d1: 0x04c5, + 0x21d2: 0x0950, 0x21d3: 0x14a0, 0x21d4: 0x001e, 0x21d5: 0x0960, 0x21d6: 0x17fa, 0x21d7: 0x049b, + 0x21d8: 0x02c8, 0x21d9: 0x0962, 0x21da: 0x0964, 0x21db: 0x096d, 0x21dc: 0x04a6, 0x21dd: 0x04c7, + 0x21de: 0x04a8, 0x21df: 0x09df, 0x21e0: 0x0906, 0x21e1: 0x090b, 0x21e2: 0x14ad, 0x21e3: 0x090d, + 0x21e4: 0x090f, 0x21e5: 0x14d9, 0x21e6: 0x0914, 0x21e7: 0x0916, 0x21e8: 0x0918, 0x21e9: 0x091a, + 0x21ea: 0x091c, 0x21eb: 0x091e, 0x21ec: 0x0920, 0x21ed: 0x0922, 0x21ee: 0x0924, 0x21ef: 0x0929, + 0x21f0: 0x14c8, 0x21f1: 0x092b, 0x21f2: 0x17f6, 0x21f3: 0x092d, 0x21f4: 0x092f, 0x21f5: 0x155f, + 0x21f6: 0x0931, 0x21f7: 0x1570, 0x21f8: 0x17f8, 0x21f9: 0x14d4, 0x21fa: 0x0007, 0x21fb: 0x093d, + 0x21fc: 0x0984, 0x21fd: 0x093f, 0x21fe: 0x0941, 0x21ff: 0x098c, + // Block 0x88, offset 0x2200 + 0x2200: 0x094c, 0x2201: 0x0494, 0x2202: 0x097c, 0x2203: 0x0499, 0x2204: 0x094e, 0x2205: 0x04c5, + 0x2206: 0x0950, 0x2207: 0x14a0, 0x2208: 0x001e, 0x2209: 0x0960, 0x220a: 0x17fa, 0x220b: 0x049b, + 0x220c: 0x02c8, 0x220d: 0x0962, 0x220e: 0x0964, 0x220f: 0x096d, 0x2210: 0x04a6, 0x2211: 0x04c7, + 0x2212: 0x04a8, 0x2213: 0x09df, 0x2214: 0x0906, 0x2215: 0x090b, 0x2216: 0x14ad, 0x2217: 0x090d, + 0x2218: 0x090f, 0x2219: 0x14d9, 0x221a: 0x0914, 0x221b: 0x0916, 0x221c: 0x0918, 0x221d: 0x091a, + 0x221e: 0x091c, 0x221f: 0x091e, 0x2220: 0x0920, 0x2221: 0x0922, 0x2222: 0x0924, 0x2223: 0x0929, + 0x2224: 0x14c8, 0x2225: 0x092b, 0x2226: 0x17f6, 0x2227: 0x092d, 0x2228: 0x092f, 0x2229: 0x155f, + 0x222a: 0x0931, 0x222b: 0x1570, 0x222c: 0x17f8, 0x222d: 0x14d4, 0x222e: 0x0007, 0x222f: 0x093d, + 0x2230: 0x0984, 0x2231: 0x093f, 0x2232: 0x0941, 0x2233: 0x098c, 0x2234: 0x094c, 0x2235: 0x0494, + 0x2236: 0x097c, 0x2237: 0x0499, 0x2238: 0x094e, 0x2239: 0x04c5, 0x223a: 0x0950, 0x223b: 0x14a0, + 0x223c: 0x001e, 0x223d: 0x0960, 0x223e: 0x17fa, 0x223f: 0x049b, + // Block 0x89, offset 0x2240 + 0x2240: 0x02c8, 0x2241: 0x0962, 0x2242: 0x0964, 0x2243: 0x096d, 0x2244: 0x04a6, 0x2245: 0x04c7, + 0x2246: 0x04a8, 0x2247: 0x09df, 0x2248: 0x0906, 0x2249: 0x090b, 0x224a: 0x14ad, 0x224b: 0x090d, + 0x224c: 0x090f, 0x224d: 0x14d9, 0x224e: 0x0914, 0x224f: 0x0916, 0x2250: 0x0918, 0x2251: 0x091a, + 0x2252: 0x091c, 0x2253: 0x091e, 0x2254: 0x0920, 0x2255: 0x0922, 0x2256: 0x0924, 0x2257: 0x0929, + 0x2258: 0x14c8, 0x2259: 0x092b, 0x225a: 0x17f6, 0x225b: 0x092d, 0x225c: 0x092f, 0x225d: 0x155f, + 0x225e: 0x0931, 0x225f: 0x1570, 0x2260: 0x17f8, 0x2261: 0x14d4, 0x2262: 0x0007, 0x2263: 0x093d, + 0x2264: 0x0984, 0x2265: 0x093f, 0x2266: 0x0941, 0x2267: 0x098c, 0x2268: 0x094c, 0x2269: 0x0494, + 0x226a: 0x097c, 0x226b: 0x0499, 0x226c: 0x094e, 0x226d: 0x04c5, 0x226e: 0x0950, 0x226f: 0x14a0, + 0x2270: 0x001e, 0x2271: 0x0960, 0x2272: 0x17fa, 0x2273: 0x049b, 0x2274: 0x02c8, 0x2275: 0x0962, + 0x2276: 0x0964, 0x2277: 0x096d, 0x2278: 0x04a6, 0x2279: 0x04c7, 0x227a: 0x04a8, 0x227b: 0x09df, + 0x227c: 0x0906, 0x227d: 0x090b, 0x227e: 0x14ad, 0x227f: 0x090d, + // Block 0x8a, offset 0x2280 + 0x2280: 0x090f, 0x2281: 0x14d9, 0x2282: 0x0914, 0x2283: 0x0916, 0x2284: 0x0918, 0x2285: 0x091a, + 0x2286: 0x091c, 0x2287: 0x091e, 0x2288: 0x0920, 0x2289: 0x0922, 0x228a: 0x0924, 0x228b: 0x0929, + 0x228c: 0x14c8, 0x228d: 0x092b, 0x228e: 0x17f6, 0x228f: 0x092d, 0x2290: 0x092f, 0x2291: 0x155f, + 0x2292: 0x0931, 0x2293: 0x1570, 0x2294: 0x17f8, 0x2295: 0x14d4, 0x2296: 0x0007, 0x2297: 0x093d, + 0x2298: 0x0984, 0x2299: 0x093f, 0x229a: 0x0941, 0x229b: 0x098c, 0x229c: 0x094c, 0x229d: 0x0494, + 0x229e: 0x097c, 0x229f: 0x0499, 0x22a0: 0x094e, 0x22a1: 0x04c5, 0x22a2: 0x0950, 0x22a3: 0x14a0, + 0x22a4: 0x001e, 0x22a5: 0x0960, 0x22a6: 0x17fa, 0x22a7: 0x049b, 0x22a8: 0x02c8, 0x22a9: 0x0962, + 0x22aa: 0x0964, 0x22ab: 0x096d, 0x22ac: 0x04a6, 0x22ad: 0x04c7, 0x22ae: 0x04a8, 0x22af: 0x09df, + 0x22b0: 0x0906, 0x22b1: 0x090b, 0x22b2: 0x14ad, 0x22b3: 0x090d, 0x22b4: 0x090f, 0x22b5: 0x14d9, + 0x22b6: 0x0914, 0x22b7: 0x0916, 0x22b8: 0x0918, 0x22b9: 0x091a, 0x22ba: 0x091c, 0x22bb: 0x091e, + 0x22bc: 0x0920, 0x22bd: 0x0922, 0x22be: 0x0924, 0x22bf: 0x0929, + // Block 0x8b, offset 0x22c0 + 0x22c0: 0x14c8, 0x22c1: 0x092b, 0x22c2: 0x17f6, 0x22c3: 0x092d, 0x22c4: 0x092f, 0x22c5: 0x155f, + 0x22c6: 0x0931, 0x22c7: 0x1570, 0x22c8: 0x17f8, 0x22c9: 0x14d4, 0x22ca: 0x0007, 0x22cb: 0x093d, + 0x22cc: 0x0984, 0x22cd: 0x093f, 0x22ce: 0x0941, 0x22cf: 0x098c, 0x22d0: 0x094c, 0x22d1: 0x0494, + 0x22d2: 0x097c, 0x22d3: 0x0499, 0x22d4: 0x094e, 0x22d5: 0x04c5, 0x22d6: 0x0950, 0x22d7: 0x14a0, + 0x22d8: 0x001e, 0x22d9: 0x0960, 0x22da: 0x17fa, 0x22db: 0x049b, 0x22dc: 0x02c8, 0x22dd: 0x0962, + 0x22de: 0x0964, 0x22df: 0x096d, 0x22e0: 0x04a6, 0x22e1: 0x04c7, 0x22e2: 0x04a8, 0x22e3: 0x09df, + 0x22e4: 0x3b27, 0x22e5: 0x3b2a, 0x22e8: 0x3b2d, 0x22e9: 0x3b30, + 0x22ea: 0x14eb, 0x22eb: 0x3b33, 0x22ec: 0x3b36, 0x22ed: 0x3b39, 0x22ee: 0x3b3c, 0x22ef: 0x057b, + 0x22f0: 0x3b3f, 0x22f1: 0x3b42, 0x22f2: 0x3b45, 0x22f3: 0x3b48, 0x22f4: 0x3b4b, 0x22f5: 0x3b4e, + 0x22f6: 0x3b51, 0x22f7: 0x14ee, 0x22f8: 0x3b54, 0x22f9: 0x057b, 0x22fa: 0x0581, 0x22fb: 0x3b57, + 0x22fc: 0x055f, 0x22fd: 0x3b5a, 0x22fe: 0x3b5d, 0x22ff: 0x3b60, + // Block 0x8c, offset 0x2300 + 0x2300: 0x14d6, 0x2301: 0x3b63, 0x2302: 0x3b67, 0x2303: 0x0559, 0x2304: 0x0973, 0x2305: 0x0976, + 0x2306: 0x057e, 0x2307: 0x3b6a, 0x2308: 0x3b6d, 0x2309: 0x055c, 0x230a: 0x12fd, 0x230b: 0x0572, + 0x230c: 0x3b70, 0x230d: 0x0015, 0x230e: 0x3b73, 0x230f: 0x3b76, 0x2310: 0x3b79, 0x2311: 0x056f, + 0x2312: 0x0575, 0x2313: 0x0578, 0x2314: 0x3b7c, 0x2315: 0x3b7f, 0x2316: 0x3b82, 0x2317: 0x056c, + 0x2318: 0x0979, 0x2319: 0x3b85, 0x231a: 0x3b88, 0x231b: 0x3b8b, 0x231c: 0x057e, 0x231d: 0x055c, + 0x231e: 0x0572, 0x231f: 0x056c, 0x2320: 0x0575, 0x2321: 0x056f, 0x2322: 0x3b2d, 0x2323: 0x3b30, + 0x2324: 0x14eb, 0x2325: 0x3b33, 0x2326: 0x3b36, 0x2327: 0x3b39, 0x2328: 0x3b3c, 0x2329: 0x057b, + 0x232a: 0x3b3f, 0x232b: 0x3b42, 0x232c: 0x3b45, 0x232d: 0x3b48, 0x232e: 0x3b4b, 0x232f: 0x3b4e, + 0x2330: 0x3b51, 0x2331: 0x14ee, 0x2332: 0x3b54, 0x2333: 0x057b, 0x2334: 0x0581, 0x2335: 0x3b57, + 0x2336: 0x055f, 0x2337: 0x3b5a, 0x2338: 0x3b5d, 0x2339: 0x3b60, 0x233a: 0x14d6, 0x233b: 0x3b63, + 0x233c: 0x3b67, 0x233d: 0x0559, 0x233e: 0x0973, 0x233f: 0x0976, + // Block 0x8d, offset 0x2340 + 0x2340: 0x057e, 0x2341: 0x3b6a, 0x2342: 0x3b6d, 0x2343: 0x055c, 0x2344: 0x12fd, 0x2345: 0x0572, + 0x2346: 0x3b70, 0x2347: 0x0015, 0x2348: 0x3b73, 0x2349: 0x3b76, 0x234a: 0x3b79, 0x234b: 0x056f, + 0x234c: 0x0575, 0x234d: 0x0578, 0x234e: 0x3b7c, 0x234f: 0x3b7f, 0x2350: 0x3b82, 0x2351: 0x056c, + 0x2352: 0x0979, 0x2353: 0x3b85, 0x2354: 0x3b88, 0x2355: 0x3b8b, 0x2356: 0x057e, 0x2357: 0x055c, + 0x2358: 0x0572, 0x2359: 0x056c, 0x235a: 0x0575, 0x235b: 0x056f, 0x235c: 0x3b2d, 0x235d: 0x3b30, + 0x235e: 0x14eb, 0x235f: 0x3b33, 0x2360: 0x3b36, 0x2361: 0x3b39, 0x2362: 0x3b3c, 0x2363: 0x057b, + 0x2364: 0x3b3f, 0x2365: 0x3b42, 0x2366: 0x3b45, 0x2367: 0x3b48, 0x2368: 0x3b4b, 0x2369: 0x3b4e, + 0x236a: 0x3b51, 0x236b: 0x14ee, 0x236c: 0x3b54, 0x236d: 0x057b, 0x236e: 0x0581, 0x236f: 0x3b57, + 0x2370: 0x055f, 0x2371: 0x3b5a, 0x2372: 0x3b5d, 0x2373: 0x3b60, 0x2374: 0x14d6, 0x2375: 0x3b63, + 0x2376: 0x3b67, 0x2377: 0x0559, 0x2378: 0x0973, 0x2379: 0x0976, 0x237a: 0x057e, 0x237b: 0x3b6a, + 0x237c: 0x3b6d, 0x237d: 0x055c, 0x237e: 0x12fd, 0x237f: 0x0572, + // Block 0x8e, offset 0x2380 + 0x2380: 0x3b70, 0x2381: 0x0015, 0x2382: 0x3b73, 0x2383: 0x3b76, 0x2384: 0x3b79, 0x2385: 0x056f, + 0x2386: 0x0575, 0x2387: 0x0578, 0x2388: 0x3b7c, 0x2389: 0x3b7f, 0x238a: 0x3b82, 0x238b: 0x056c, + 0x238c: 0x0979, 0x238d: 0x3b85, 0x238e: 0x3b88, 0x238f: 0x3b8b, 0x2390: 0x057e, 0x2391: 0x055c, + 0x2392: 0x0572, 0x2393: 0x056c, 0x2394: 0x0575, 0x2395: 0x056f, 0x2396: 0x3b2d, 0x2397: 0x3b30, + 0x2398: 0x14eb, 0x2399: 0x3b33, 0x239a: 0x3b36, 0x239b: 0x3b39, 0x239c: 0x3b3c, 0x239d: 0x057b, + 0x239e: 0x3b3f, 0x239f: 0x3b42, 0x23a0: 0x3b45, 0x23a1: 0x3b48, 0x23a2: 0x3b4b, 0x23a3: 0x3b4e, + 0x23a4: 0x3b51, 0x23a5: 0x14ee, 0x23a6: 0x3b54, 0x23a7: 0x057b, 0x23a8: 0x0581, 0x23a9: 0x3b57, + 0x23aa: 0x055f, 0x23ab: 0x3b5a, 0x23ac: 0x3b5d, 0x23ad: 0x3b60, 0x23ae: 0x14d6, 0x23af: 0x3b63, + 0x23b0: 0x3b67, 0x23b1: 0x0559, 0x23b2: 0x0973, 0x23b3: 0x0976, 0x23b4: 0x057e, 0x23b5: 0x3b6a, + 0x23b6: 0x3b6d, 0x23b7: 0x055c, 0x23b8: 0x12fd, 0x23b9: 0x0572, 0x23ba: 0x3b70, 0x23bb: 0x0015, + 0x23bc: 0x3b73, 0x23bd: 0x3b76, 0x23be: 0x3b79, 0x23bf: 0x056f, + // Block 0x8f, offset 0x23c0 + 0x23c0: 0x0575, 0x23c1: 0x0578, 0x23c2: 0x3b7c, 0x23c3: 0x3b7f, 0x23c4: 0x3b82, 0x23c5: 0x056c, + 0x23c6: 0x0979, 0x23c7: 0x3b85, 0x23c8: 0x3b88, 0x23c9: 0x3b8b, 0x23ca: 0x057e, 0x23cb: 0x055c, + 0x23cc: 0x0572, 0x23cd: 0x056c, 0x23ce: 0x0575, 0x23cf: 0x056f, 0x23d0: 0x3b2d, 0x23d1: 0x3b30, + 0x23d2: 0x14eb, 0x23d3: 0x3b33, 0x23d4: 0x3b36, 0x23d5: 0x3b39, 0x23d6: 0x3b3c, 0x23d7: 0x057b, + 0x23d8: 0x3b3f, 0x23d9: 0x3b42, 0x23da: 0x3b45, 0x23db: 0x3b48, 0x23dc: 0x3b4b, 0x23dd: 0x3b4e, + 0x23de: 0x3b51, 0x23df: 0x14ee, 0x23e0: 0x3b54, 0x23e1: 0x057b, 0x23e2: 0x0581, 0x23e3: 0x3b57, + 0x23e4: 0x055f, 0x23e5: 0x3b5a, 0x23e6: 0x3b5d, 0x23e7: 0x3b60, 0x23e8: 0x14d6, 0x23e9: 0x3b63, + 0x23ea: 0x3b67, 0x23eb: 0x0559, 0x23ec: 0x0973, 0x23ed: 0x0976, 0x23ee: 0x057e, 0x23ef: 0x3b6a, + 0x23f0: 0x3b6d, 0x23f1: 0x055c, 0x23f2: 0x12fd, 0x23f3: 0x0572, 0x23f4: 0x3b70, 0x23f5: 0x0015, + 0x23f6: 0x3b73, 0x23f7: 0x3b76, 0x23f8: 0x3b79, 0x23f9: 0x056f, 0x23fa: 0x0575, 0x23fb: 0x0578, + 0x23fc: 0x3b7c, 0x23fd: 0x3b7f, 0x23fe: 0x3b82, 0x23ff: 0x056c, + // Block 0x90, offset 0x2400 + 0x2400: 0x0979, 0x2401: 0x3b85, 0x2402: 0x3b88, 0x2403: 0x3b8b, 0x2404: 0x057e, 0x2405: 0x055c, + 0x2406: 0x0572, 0x2407: 0x056c, 0x2408: 0x0575, 0x2409: 0x056f, 0x240a: 0x3b8f, 0x240b: 0x3b92, + 0x240e: 0x1486, 0x240f: 0x001c, 0x2410: 0x000d, 0x2411: 0x000f, + 0x2412: 0x1488, 0x2413: 0x148a, 0x2414: 0x148c, 0x2415: 0x148e, 0x2416: 0x1490, 0x2417: 0x1492, + 0x2418: 0x1486, 0x2419: 0x001c, 0x241a: 0x000d, 0x241b: 0x000f, 0x241c: 0x1488, 0x241d: 0x148a, + 0x241e: 0x148c, 0x241f: 0x148e, 0x2420: 0x1490, 0x2421: 0x1492, 0x2422: 0x1486, 0x2423: 0x001c, + 0x2424: 0x000d, 0x2425: 0x000f, 0x2426: 0x1488, 0x2427: 0x148a, 0x2428: 0x148c, 0x2429: 0x148e, + 0x242a: 0x1490, 0x242b: 0x1492, 0x242c: 0x1486, 0x242d: 0x001c, 0x242e: 0x000d, 0x242f: 0x000f, + 0x2430: 0x1488, 0x2431: 0x148a, 0x2432: 0x148c, 0x2433: 0x148e, 0x2434: 0x1490, 0x2435: 0x1492, + 0x2436: 0x1486, 0x2437: 0x001c, 0x2438: 0x000d, 0x2439: 0x000f, 0x243a: 0x1488, 0x243b: 0x148a, + 0x243c: 0x148c, 0x243d: 0x148e, 0x243e: 0x1490, 0x243f: 0x1492, + // Block 0x91, offset 0x2440 + 0x2440: 0x3b95, 0x2441: 0x3b98, 0x2442: 0x3b9b, 0x2443: 0x3b9e, 0x2444: 0x3ba1, 0x2445: 0x3ba4, + 0x2446: 0x3ba7, 0x2447: 0x3baa, 0x2448: 0x3bad, 0x2449: 0x3bb0, 0x244a: 0x3bb3, + 0x2450: 0x3bb6, 0x2451: 0x3bba, + 0x2452: 0x3bbe, 0x2453: 0x3bc2, 0x2454: 0x3bc6, 0x2455: 0x3bca, 0x2456: 0x3bce, 0x2457: 0x3bd2, + 0x2458: 0x3bd6, 0x2459: 0x3bda, 0x245a: 0x3bde, 0x245b: 0x3be2, 0x245c: 0x3be6, 0x245d: 0x3bea, + 0x245e: 0x3bee, 0x245f: 0x3bf2, 0x2460: 0x3bf6, 0x2461: 0x3bfa, 0x2462: 0x3bfe, 0x2463: 0x3c02, + 0x2464: 0x3c06, 0x2465: 0x3c0a, 0x2466: 0x3c0e, 0x2467: 0x3c12, 0x2468: 0x3c16, 0x2469: 0x3c1a, + 0x246a: 0x3c1e, 0x246b: 0x14ad, 0x246c: 0x092b, 0x246d: 0x3c26, 0x246e: 0x3c29, + 0x2470: 0x0906, 0x2471: 0x090b, 0x2472: 0x14ad, 0x2473: 0x090d, 0x2474: 0x090f, 0x2475: 0x14d9, + 0x2476: 0x0914, 0x2477: 0x0916, 0x2478: 0x0918, 0x2479: 0x091a, 0x247a: 0x091c, 0x247b: 0x091e, + 0x247c: 0x0920, 0x247d: 0x0922, 0x247e: 0x0924, 0x247f: 0x0929, + // Block 0x92, offset 0x2480 + 0x2480: 0x14c8, 0x2481: 0x092b, 0x2482: 0x17f6, 0x2483: 0x092d, 0x2484: 0x092f, 0x2485: 0x155f, + 0x2486: 0x0931, 0x2487: 0x1570, 0x2488: 0x17f8, 0x2489: 0x14d4, 0x248a: 0x3c2c, 0x248b: 0x293d, + 0x248c: 0x3c2f, 0x248d: 0x3c32, 0x248e: 0x3c35, 0x248f: 0x3c39, + // Block 0x93, offset 0x24c0 + 0x24d0: 0x3c3c, + // Block 0x94, offset 0x2500 + 0x2500: 0x3c3f, 0x2501: 0x3c46, 0x2502: 0x2291, + 0x2510: 0x1922, 0x2511: 0x3c4d, + 0x2512: 0x3c51, 0x2513: 0x1cb3, 0x2514: 0x183e, 0x2515: 0x3c55, 0x2516: 0x3c59, 0x2517: 0x1ed0, + 0x2518: 0x3c5d, 0x2519: 0x3c61, 0x251a: 0x3c65, 0x251b: 0x2d49, 0x251c: 0x3c69, 0x251d: 0x3c6d, + 0x251e: 0x3c71, 0x251f: 0x3c75, 0x2520: 0x3c79, 0x2521: 0x3c7d, 0x2522: 0x19b2, 0x2523: 0x3c81, + 0x2524: 0x3c85, 0x2525: 0x3c89, 0x2526: 0x3c8d, 0x2527: 0x3c91, 0x2528: 0x3c95, 0x2529: 0x1826, + 0x252a: 0x1eb0, 0x252b: 0x3c99, 0x252c: 0x21c7, 0x252d: 0x1ebc, 0x252e: 0x21cb, 0x252f: 0x3c9d, + 0x2530: 0x1a92, 0x2531: 0x3ca1, 0x2532: 0x3ca5, 0x2533: 0x3ca9, 0x2534: 0x3cad, 0x2535: 0x3cb1, + 0x2536: 0x2183, 0x2537: 0x194a, 0x2538: 0x3cb5, 0x2539: 0x3cb9, 0x253a: 0x3cbd, + // Block 0x95, offset 0x2540 + 0x2540: 0x3cc1, 0x2541: 0x3ccb, 0x2542: 0x3cd5, 0x2543: 0x3cdf, 0x2544: 0x3ce9, 0x2545: 0x3cf3, + 0x2546: 0x3cfd, 0x2547: 0x3d07, 0x2548: 0x3d11, + 0x2550: 0x3d1b, 0x2551: 0x3d1f, + // Block 0x96, offset 0x2580 + 0x2580: 0x3d23, 0x2581: 0x3d27, 0x2582: 0x3d2b, 0x2583: 0x3d2f, 0x2584: 0x3d34, 0x2585: 0x2eb5, + 0x2586: 0x3d38, 0x2587: 0x3d3c, 0x2588: 0x3d40, 0x2589: 0x3d44, 0x258a: 0x2eb9, 0x258b: 0x3d48, + 0x258c: 0x3d4c, 0x258d: 0x3d50, 0x258e: 0x2ebd, 0x258f: 0x3d55, 0x2590: 0x3d59, 0x2591: 0x3d5d, + 0x2592: 0x3d61, 0x2593: 0x3d66, 0x2594: 0x3d6a, 0x2595: 0x3c71, 0x2596: 0x3d6e, 0x2597: 0x3d73, + 0x2598: 0x3d77, 0x2599: 0x3d7b, 0x259a: 0x3d7f, 0x259b: 0x2f9a, 0x259c: 0x3d83, 0x259d: 0x1866, + 0x259e: 0x3d88, 0x259f: 0x3d8c, 0x25a0: 0x3d90, 0x25a1: 0x3d94, 0x25a2: 0x3cb9, 0x25a3: 0x3d98, + 0x25a4: 0x3d9c, 0x25a5: 0x2fae, 0x25a6: 0x2ec1, 0x25a7: 0x2ec5, 0x25a8: 0x2fb2, 0x25a9: 0x3da0, + 0x25aa: 0x3da4, 0x25ab: 0x2bf1, 0x25ac: 0x3da8, 0x25ad: 0x2ec9, 0x25ae: 0x3dac, 0x25af: 0x3db0, + 0x25b0: 0x3db4, 0x25b1: 0x3db8, 0x25b2: 0x3db8, 0x25b3: 0x3db8, 0x25b4: 0x3dbc, 0x25b5: 0x3dc1, + 0x25b6: 0x3dc5, 0x25b7: 0x3dc9, 0x25b8: 0x3dcd, 0x25b9: 0x3dd2, 0x25ba: 0x3dd6, 0x25bb: 0x3dda, + 0x25bc: 0x3dde, 0x25bd: 0x3de2, 0x25be: 0x3de6, 0x25bf: 0x3dea, + // Block 0x97, offset 0x25c0 + 0x25c0: 0x3dee, 0x25c1: 0x3df2, 0x25c2: 0x3df6, 0x25c3: 0x3dfa, 0x25c4: 0x3dfe, 0x25c5: 0x3e02, + 0x25c6: 0x3e02, 0x25c7: 0x2fba, 0x25c8: 0x3e06, 0x25c9: 0x3e0a, 0x25ca: 0x3e0e, 0x25cb: 0x3e12, + 0x25cc: 0x2ed1, 0x25cd: 0x3e16, 0x25ce: 0x3e1a, 0x25cf: 0x3e1e, 0x25d0: 0x2e39, 0x25d1: 0x3e22, + 0x25d2: 0x3e26, 0x25d3: 0x3e2a, 0x25d4: 0x3e2e, 0x25d5: 0x3e32, 0x25d6: 0x3e36, 0x25d7: 0x3e3a, + 0x25d8: 0x3e3e, 0x25d9: 0x3e42, 0x25da: 0x3e47, 0x25db: 0x3e4b, 0x25dc: 0x3e4f, 0x25dd: 0x3c55, + 0x25de: 0x3e53, 0x25df: 0x3e57, 0x25e0: 0x3e5b, 0x25e1: 0x3e60, 0x25e2: 0x3e65, 0x25e3: 0x3e69, + 0x25e4: 0x3e6d, 0x25e5: 0x3e71, 0x25e6: 0x3e75, 0x25e7: 0x3e79, 0x25e8: 0x3e7d, 0x25e9: 0x3e81, + 0x25ea: 0x3e85, 0x25eb: 0x3e85, 0x25ec: 0x3e89, 0x25ed: 0x3e8e, 0x25ee: 0x3e92, 0x25ef: 0x2be1, + 0x25f0: 0x3e96, 0x25f1: 0x3e9a, 0x25f2: 0x3e9f, 0x25f3: 0x3ea3, 0x25f4: 0x3ea7, 0x25f5: 0x18ce, + 0x25f6: 0x3eab, 0x25f7: 0x3eaf, 0x25f8: 0x18d6, 0x25f9: 0x3eb3, 0x25fa: 0x3eb7, 0x25fb: 0x3ebb, + 0x25fc: 0x3ec0, 0x25fd: 0x3ec4, 0x25fe: 0x3ec9, 0x25ff: 0x3ecd, + // Block 0x98, offset 0x2600 + 0x2600: 0x3ed1, 0x2601: 0x3ed5, 0x2602: 0x3ed9, 0x2603: 0x3edd, 0x2604: 0x3ee1, 0x2605: 0x3ee5, + 0x2606: 0x3ee9, 0x2607: 0x3eed, 0x2608: 0x3ef1, 0x2609: 0x3ef5, 0x260a: 0x3efa, 0x260b: 0x3efe, + 0x260c: 0x3f02, 0x260d: 0x3f06, 0x260e: 0x2b11, 0x260f: 0x3f0a, 0x2610: 0x18fe, 0x2611: 0x3f0f, + 0x2612: 0x3f0f, 0x2613: 0x3f14, 0x2614: 0x3f18, 0x2615: 0x3f18, 0x2616: 0x3f1c, 0x2617: 0x3f20, + 0x2618: 0x3f25, 0x2619: 0x3f2a, 0x261a: 0x3f2e, 0x261b: 0x3f32, 0x261c: 0x3f36, 0x261d: 0x3f3a, + 0x261e: 0x3f3e, 0x261f: 0x3f42, 0x2620: 0x3f46, 0x2621: 0x3f4a, 0x2622: 0x3f4e, 0x2623: 0x2ee5, + 0x2624: 0x3f52, 0x2625: 0x3f57, 0x2626: 0x3f5b, 0x2627: 0x3f5f, 0x2628: 0x2fea, 0x2629: 0x3f5f, + 0x262a: 0x3f63, 0x262b: 0x2eed, 0x262c: 0x3f67, 0x262d: 0x3f6b, 0x262e: 0x3f6f, 0x262f: 0x3f73, + 0x2630: 0x2ef1, 0x2631: 0x2aa5, 0x2632: 0x3f77, 0x2633: 0x3f7b, 0x2634: 0x3f7f, 0x2635: 0x3f83, + 0x2636: 0x3f87, 0x2637: 0x3f8b, 0x2638: 0x3f8f, 0x2639: 0x3f94, 0x263a: 0x3f98, 0x263b: 0x3f9c, + 0x263c: 0x3fa0, 0x263d: 0x3fa4, 0x263e: 0x3fa8, 0x263f: 0x3fad, + // Block 0x99, offset 0x2640 + 0x2640: 0x3fb1, 0x2641: 0x3fb5, 0x2642: 0x3fb9, 0x2643: 0x3fbd, 0x2644: 0x3fc1, 0x2645: 0x3fc5, + 0x2646: 0x3fc9, 0x2647: 0x3fcd, 0x2648: 0x2ef5, 0x2649: 0x3fd1, 0x264a: 0x3fd5, 0x264b: 0x3fda, + 0x264c: 0x3fde, 0x264d: 0x3fe2, 0x264e: 0x3fe6, 0x264f: 0x2efd, 0x2650: 0x3fea, 0x2651: 0x3fee, + 0x2652: 0x3ff2, 0x2653: 0x3ff6, 0x2654: 0x3ffa, 0x2655: 0x3ffe, 0x2656: 0x4002, 0x2657: 0x4006, + 0x2658: 0x2b15, 0x2659: 0x300a, 0x265a: 0x400a, 0x265b: 0x400e, 0x265c: 0x4012, 0x265d: 0x4016, + 0x265e: 0x401b, 0x265f: 0x401f, 0x2660: 0x4023, 0x2661: 0x4027, 0x2662: 0x2f01, 0x2663: 0x402b, + 0x2664: 0x4030, 0x2665: 0x4034, 0x2666: 0x4038, 0x2667: 0x30b5, 0x2668: 0x403c, 0x2669: 0x4040, + 0x266a: 0x4044, 0x266b: 0x4048, 0x266c: 0x404c, 0x266d: 0x4051, 0x266e: 0x4055, 0x266f: 0x4059, + 0x2670: 0x405d, 0x2671: 0x4062, 0x2672: 0x4066, 0x2673: 0x406a, 0x2674: 0x406e, 0x2675: 0x2c25, + 0x2676: 0x4072, 0x2677: 0x4076, 0x2678: 0x407b, 0x2679: 0x4080, 0x267a: 0x4085, 0x267b: 0x4089, + 0x267c: 0x408e, 0x267d: 0x4092, 0x267e: 0x4096, 0x267f: 0x409a, + // Block 0x9a, offset 0x2680 + 0x2680: 0x409e, 0x2681: 0x2f05, 0x2682: 0x2d71, 0x2683: 0x40a2, 0x2684: 0x40a6, 0x2685: 0x40aa, + 0x2686: 0x40ae, 0x2687: 0x40b3, 0x2688: 0x40b7, 0x2689: 0x40bb, 0x268a: 0x40bf, 0x268b: 0x3016, + 0x268c: 0x40c3, 0x268d: 0x40c7, 0x268e: 0x40cc, 0x268f: 0x40d0, 0x2690: 0x40d4, 0x2691: 0x40d9, + 0x2692: 0x40de, 0x2693: 0x40e2, 0x2694: 0x301a, 0x2695: 0x40e6, 0x2696: 0x40ea, 0x2697: 0x40ee, + 0x2698: 0x40f2, 0x2699: 0x40f6, 0x269a: 0x40fa, 0x269b: 0x40fe, 0x269c: 0x4103, 0x269d: 0x4107, + 0x269e: 0x410c, 0x269f: 0x4110, 0x26a0: 0x4115, 0x26a1: 0x3022, 0x26a2: 0x4119, 0x26a3: 0x411d, + 0x26a4: 0x4122, 0x26a5: 0x4126, 0x26a6: 0x412a, 0x26a7: 0x412f, 0x26a8: 0x4134, 0x26a9: 0x4138, + 0x26aa: 0x413c, 0x26ab: 0x4140, 0x26ac: 0x4144, 0x26ad: 0x4144, 0x26ae: 0x4148, 0x26af: 0x414c, + 0x26b0: 0x302a, 0x26b1: 0x4150, 0x26b2: 0x4154, 0x26b3: 0x4158, 0x26b4: 0x415c, 0x26b5: 0x4160, + 0x26b6: 0x4165, 0x26b7: 0x4169, 0x26b8: 0x2bed, 0x26b9: 0x416e, 0x26ba: 0x4173, 0x26bb: 0x4177, + 0x26bc: 0x417c, 0x26bd: 0x4181, 0x26be: 0x4186, 0x26bf: 0x418a, + // Block 0x9b, offset 0x26c0 + 0x26c0: 0x3042, 0x26c1: 0x418e, 0x26c2: 0x4193, 0x26c3: 0x4198, 0x26c4: 0x419d, 0x26c5: 0x41a2, + 0x26c6: 0x41a6, 0x26c7: 0x41a6, 0x26c8: 0x3046, 0x26c9: 0x30bd, 0x26ca: 0x41aa, 0x26cb: 0x41ae, + 0x26cc: 0x41b2, 0x26cd: 0x41b6, 0x26ce: 0x41bb, 0x26cf: 0x2b59, 0x26d0: 0x304e, 0x26d1: 0x41bf, + 0x26d2: 0x41c3, 0x26d3: 0x2f2d, 0x26d4: 0x41c8, 0x26d5: 0x41cd, 0x26d6: 0x2e89, 0x26d7: 0x41d2, + 0x26d8: 0x41d6, 0x26d9: 0x2f39, 0x26da: 0x41da, 0x26db: 0x41de, 0x26dc: 0x41e2, 0x26dd: 0x41e7, + 0x26de: 0x41e7, 0x26df: 0x41ec, 0x26e0: 0x41f0, 0x26e1: 0x41f4, 0x26e2: 0x41f9, 0x26e3: 0x41fd, + 0x26e4: 0x4201, 0x26e5: 0x4205, 0x26e6: 0x420a, 0x26e7: 0x420e, 0x26e8: 0x4212, 0x26e9: 0x4216, + 0x26ea: 0x421a, 0x26eb: 0x421e, 0x26ec: 0x4223, 0x26ed: 0x4227, 0x26ee: 0x422b, 0x26ef: 0x422f, + 0x26f0: 0x4233, 0x26f1: 0x4237, 0x26f2: 0x423b, 0x26f3: 0x4240, 0x26f4: 0x4245, 0x26f5: 0x4249, + 0x26f6: 0x424e, 0x26f7: 0x4252, 0x26f8: 0x4257, 0x26f9: 0x425b, 0x26fa: 0x2f51, 0x26fb: 0x425f, + 0x26fc: 0x4264, 0x26fd: 0x4269, 0x26fe: 0x426d, 0x26ff: 0x4272, + // Block 0x9c, offset 0x2700 + 0x2700: 0x4276, 0x2701: 0x427b, 0x2702: 0x427f, 0x2703: 0x4283, 0x2704: 0x4287, 0x2705: 0x428b, + 0x2706: 0x428f, 0x2707: 0x4293, 0x2708: 0x4298, 0x2709: 0x429d, 0x270a: 0x42a2, 0x270b: 0x3f14, + 0x270c: 0x42a7, 0x270d: 0x42ab, 0x270e: 0x42af, 0x270f: 0x42b3, 0x2710: 0x42b7, 0x2711: 0x42bb, + 0x2712: 0x42bf, 0x2713: 0x42c3, 0x2714: 0x42c7, 0x2715: 0x42cb, 0x2716: 0x42cf, 0x2717: 0x42d3, + 0x2718: 0x2c31, 0x2719: 0x42d8, 0x271a: 0x42dc, 0x271b: 0x42e0, 0x271c: 0x42e4, 0x271d: 0x42e8, + 0x271e: 0x42ec, 0x271f: 0x2f5d, 0x2720: 0x42f0, 0x2721: 0x42f4, 0x2722: 0x42f8, 0x2723: 0x42fc, + 0x2724: 0x4300, 0x2725: 0x4305, 0x2726: 0x430a, 0x2727: 0x430f, 0x2728: 0x4313, 0x2729: 0x4317, + 0x272a: 0x431b, 0x272b: 0x431f, 0x272c: 0x4324, 0x272d: 0x4328, 0x272e: 0x432d, 0x272f: 0x4331, + 0x2730: 0x4335, 0x2731: 0x433a, 0x2732: 0x433f, 0x2733: 0x4343, 0x2734: 0x2b45, 0x2735: 0x4347, + 0x2736: 0x434b, 0x2737: 0x434f, 0x2738: 0x4353, 0x2739: 0x4357, 0x273a: 0x435b, 0x273b: 0x306a, + 0x273c: 0x435f, 0x273d: 0x4363, 0x273e: 0x4367, 0x273f: 0x436b, + // Block 0x9d, offset 0x2740 + 0x2740: 0x436f, 0x2741: 0x4373, 0x2742: 0x4377, 0x2743: 0x437b, 0x2744: 0x1a66, 0x2745: 0x437f, + 0x2746: 0x4384, 0x2747: 0x4388, 0x2748: 0x438c, 0x2749: 0x4390, 0x274a: 0x4394, 0x274b: 0x4398, + 0x274c: 0x439d, 0x274d: 0x43a2, 0x274e: 0x43a6, 0x274f: 0x43aa, 0x2750: 0x307e, 0x2751: 0x3082, + 0x2752: 0x1a82, 0x2753: 0x43ae, 0x2754: 0x43b3, 0x2755: 0x43b7, 0x2756: 0x43bb, 0x2757: 0x43bf, + 0x2758: 0x43c3, 0x2759: 0x43c8, 0x275a: 0x43cd, 0x275b: 0x43d1, 0x275c: 0x43d5, 0x275d: 0x43d9, + 0x275e: 0x43de, 0x275f: 0x3086, 0x2760: 0x43e2, 0x2761: 0x43e7, 0x2762: 0x43ec, 0x2763: 0x43f0, + 0x2764: 0x43f4, 0x2765: 0x43f8, 0x2766: 0x43fd, 0x2767: 0x4401, 0x2768: 0x4405, 0x2769: 0x4409, + 0x276a: 0x440d, 0x276b: 0x4411, 0x276c: 0x4415, 0x276d: 0x4419, 0x276e: 0x441e, 0x276f: 0x4422, + 0x2770: 0x4426, 0x2771: 0x442a, 0x2772: 0x442f, 0x2773: 0x4433, 0x2774: 0x4437, 0x2775: 0x443b, + 0x2776: 0x443f, 0x2777: 0x4444, 0x2778: 0x4449, 0x2779: 0x444d, 0x277a: 0x4451, 0x277b: 0x4455, + 0x277c: 0x445a, 0x277d: 0x445e, 0x277e: 0x309e, 0x277f: 0x309e, + // Block 0x9e, offset 0x2780 + 0x2780: 0x4463, 0x2781: 0x4467, 0x2782: 0x446c, 0x2783: 0x4470, 0x2784: 0x4474, 0x2785: 0x4478, + 0x2786: 0x447c, 0x2787: 0x4480, 0x2788: 0x4484, 0x2789: 0x4488, 0x278a: 0x30a2, 0x278b: 0x448d, + 0x278c: 0x4491, 0x278d: 0x4495, 0x278e: 0x4499, 0x278f: 0x449d, 0x2790: 0x44a1, 0x2791: 0x44a6, + 0x2792: 0x44aa, 0x2793: 0x44af, 0x2794: 0x44b4, 0x2795: 0x1b42, 0x2796: 0x44b9, 0x2797: 0x1b52, + 0x2798: 0x44bd, 0x2799: 0x44c1, 0x279a: 0x44c5, 0x279b: 0x44c9, 0x279c: 0x1b66, 0x279d: 0x44cd, +} + +// nfkcDecompLookup: 960 bytes +// Block 0 is the null block. +var nfkcDecompLookup = [960]uint8{ + // Block 0x0, offset 0x0 + // Block 0x1, offset 0x40 + // Block 0x2, offset 0x80 + // Block 0x3, offset 0xc0 + 0x0c2: 0x03, 0x0c3: 0x04, 0x0c4: 0x05, 0x0c5: 0x06, 0x0c6: 0x07, 0x0c7: 0x08, + 0x0c8: 0x09, 0x0ca: 0x0a, 0x0cb: 0x0b, 0x0cd: 0x0c, 0x0ce: 0x0d, 0x0cf: 0x0e, + 0x0d0: 0x0f, 0x0d1: 0x10, 0x0d3: 0x11, 0x0d6: 0x12, + 0x0d8: 0x13, 0x0d9: 0x14, 0x0db: 0x15, + 0x0e0: 0x04, 0x0e1: 0x05, 0x0e2: 0x06, 0x0e3: 0x07, + 0x0ea: 0x08, 0x0ef: 0x09, + 0x0f0: 0x0e, + // Block 0x4, offset 0x100 + 0x124: 0x16, 0x125: 0x17, 0x127: 0x18, + 0x128: 0x19, 0x129: 0x1a, 0x12d: 0x1b, 0x12e: 0x1c, 0x12f: 0x1d, + 0x131: 0x1e, 0x133: 0x1f, 0x135: 0x20, 0x137: 0x21, + 0x138: 0x22, 0x13a: 0x23, 0x13b: 0x24, 0x13c: 0x25, 0x13d: 0x26, 0x13e: 0x27, + // Block 0x5, offset 0x140 + 0x140: 0x28, 0x143: 0x29, + 0x16c: 0x2a, 0x16d: 0x2b, + 0x174: 0x2c, 0x175: 0x2d, 0x176: 0x2e, + 0x178: 0x2f, 0x179: 0x30, 0x17a: 0x31, 0x17b: 0x32, 0x17c: 0x33, 0x17d: 0x34, 0x17e: 0x35, 0x17f: 0x36, + // Block 0x6, offset 0x180 + 0x180: 0x37, 0x181: 0x38, 0x182: 0x39, 0x184: 0x3a, 0x185: 0x3b, 0x186: 0x3c, 0x187: 0x3d, + 0x188: 0x3e, 0x189: 0x3f, 0x18a: 0x40, 0x18b: 0x41, 0x18c: 0x42, + 0x191: 0x43, 0x192: 0x44, 0x193: 0x45, + 0x1a8: 0x46, 0x1a9: 0x47, 0x1ab: 0x48, + 0x1b1: 0x49, 0x1b5: 0x4a, + 0x1ba: 0x4b, 0x1bb: 0x4c, 0x1bc: 0x4d, 0x1bd: 0x4e, 0x1be: 0x4f, 0x1bf: 0x50, + // Block 0x7, offset 0x1c0 + 0x1c0: 0x51, 0x1c1: 0x52, 0x1c2: 0x53, 0x1c3: 0x54, 0x1c4: 0x55, 0x1c5: 0x56, 0x1c6: 0x57, + 0x1c8: 0x58, 0x1c9: 0x59, 0x1ca: 0x5a, 0x1cb: 0x5b, 0x1cc: 0x5c, 0x1cd: 0x5d, 0x1ce: 0x5e, 0x1cf: 0x5f, + // Block 0x8, offset 0x200 + 0x21d: 0x60, + // Block 0x9, offset 0x240 + 0x264: 0x61, 0x265: 0x62, 0x266: 0x63, 0x267: 0x64, + 0x268: 0x65, 0x269: 0x66, 0x26a: 0x67, 0x26b: 0x68, 0x26c: 0x69, 0x26d: 0x6a, 0x26e: 0x6b, 0x26f: 0x6c, + 0x270: 0x6d, 0x271: 0x6e, 0x272: 0x6f, 0x273: 0x70, 0x274: 0x71, 0x275: 0x72, 0x276: 0x73, 0x277: 0x74, + 0x278: 0x75, 0x279: 0x76, 0x27a: 0x77, 0x27b: 0x78, 0x27c: 0x79, 0x27d: 0x7a, 0x27e: 0x7b, 0x27f: 0x7c, + // Block 0xa, offset 0x280 + 0x282: 0x7d, + // Block 0xb, offset 0x2c0 + 0x2c5: 0x7e, 0x2c6: 0x7f, 0x2c7: 0x80, + 0x2d0: 0x81, 0x2d1: 0x82, 0x2d2: 0x83, 0x2d3: 0x84, 0x2d4: 0x85, 0x2d5: 0x86, 0x2d6: 0x87, 0x2d7: 0x88, + 0x2d8: 0x89, 0x2d9: 0x8a, 0x2da: 0x8b, 0x2db: 0x8c, 0x2dc: 0x8d, 0x2dd: 0x8e, 0x2de: 0x8f, 0x2df: 0x90, + // Block 0xc, offset 0x300 + 0x304: 0x91, 0x305: 0x92, 0x306: 0x93, + 0x308: 0x94, 0x309: 0x95, + // Block 0xd, offset 0x340 + 0x360: 0x96, 0x361: 0x97, 0x362: 0x98, 0x363: 0x99, 0x364: 0x9a, 0x365: 0x9b, 0x366: 0x9c, 0x367: 0x9d, + 0x368: 0x9e, + // Block 0xe, offset 0x380 + 0x391: 0x0a, + 0x39d: 0x0b, 0x39f: 0x0c, + 0x3af: 0x0d, +} + +var nfkcDecompTrie = trie{nfkcDecompLookup[:], nfkcDecompValues[:]} + +// recompMap: 7448 bytes (entries only) +var recompMap = map[uint32]uint32{ + 0x00410300: 0x00C0, + 0x00410301: 0x00C1, + 0x00410302: 0x00C2, + 0x00410303: 0x00C3, + 0x00410308: 0x00C4, + 0x0041030A: 0x00C5, + 0x00430327: 0x00C7, + 0x00450300: 0x00C8, + 0x00450301: 0x00C9, + 0x00450302: 0x00CA, + 0x00450308: 0x00CB, + 0x00490300: 0x00CC, + 0x00490301: 0x00CD, + 0x00490302: 0x00CE, + 0x00490308: 0x00CF, + 0x004E0303: 0x00D1, + 0x004F0300: 0x00D2, + 0x004F0301: 0x00D3, + 0x004F0302: 0x00D4, + 0x004F0303: 0x00D5, + 0x004F0308: 0x00D6, + 0x00550300: 0x00D9, + 0x00550301: 0x00DA, + 0x00550302: 0x00DB, + 0x00550308: 0x00DC, + 0x00590301: 0x00DD, + 0x00610300: 0x00E0, + 0x00610301: 0x00E1, + 0x00610302: 0x00E2, + 0x00610303: 0x00E3, + 0x00610308: 0x00E4, + 0x0061030A: 0x00E5, + 0x00630327: 0x00E7, + 0x00650300: 0x00E8, + 0x00650301: 0x00E9, + 0x00650302: 0x00EA, + 0x00650308: 0x00EB, + 0x00690300: 0x00EC, + 0x00690301: 0x00ED, + 0x00690302: 0x00EE, + 0x00690308: 0x00EF, + 0x006E0303: 0x00F1, + 0x006F0300: 0x00F2, + 0x006F0301: 0x00F3, + 0x006F0302: 0x00F4, + 0x006F0303: 0x00F5, + 0x006F0308: 0x00F6, + 0x00750300: 0x00F9, + 0x00750301: 0x00FA, + 0x00750302: 0x00FB, + 0x00750308: 0x00FC, + 0x00790301: 0x00FD, + 0x00790308: 0x00FF, + 0x00410304: 0x0100, + 0x00610304: 0x0101, + 0x00410306: 0x0102, + 0x00610306: 0x0103, + 0x00410328: 0x0104, + 0x00610328: 0x0105, + 0x00430301: 0x0106, + 0x00630301: 0x0107, + 0x00430302: 0x0108, + 0x00630302: 0x0109, + 0x00430307: 0x010A, + 0x00630307: 0x010B, + 0x0043030C: 0x010C, + 0x0063030C: 0x010D, + 0x0044030C: 0x010E, + 0x0064030C: 0x010F, + 0x00450304: 0x0112, + 0x00650304: 0x0113, + 0x00450306: 0x0114, + 0x00650306: 0x0115, + 0x00450307: 0x0116, + 0x00650307: 0x0117, + 0x00450328: 0x0118, + 0x00650328: 0x0119, + 0x0045030C: 0x011A, + 0x0065030C: 0x011B, + 0x00470302: 0x011C, + 0x00670302: 0x011D, + 0x00470306: 0x011E, + 0x00670306: 0x011F, + 0x00470307: 0x0120, + 0x00670307: 0x0121, + 0x00470327: 0x0122, + 0x00670327: 0x0123, + 0x00480302: 0x0124, + 0x00680302: 0x0125, + 0x00490303: 0x0128, + 0x00690303: 0x0129, + 0x00490304: 0x012A, + 0x00690304: 0x012B, + 0x00490306: 0x012C, + 0x00690306: 0x012D, + 0x00490328: 0x012E, + 0x00690328: 0x012F, + 0x00490307: 0x0130, + 0x004A0302: 0x0134, + 0x006A0302: 0x0135, + 0x004B0327: 0x0136, + 0x006B0327: 0x0137, + 0x004C0301: 0x0139, + 0x006C0301: 0x013A, + 0x004C0327: 0x013B, + 0x006C0327: 0x013C, + 0x004C030C: 0x013D, + 0x006C030C: 0x013E, + 0x004E0301: 0x0143, + 0x006E0301: 0x0144, + 0x004E0327: 0x0145, + 0x006E0327: 0x0146, + 0x004E030C: 0x0147, + 0x006E030C: 0x0148, + 0x004F0304: 0x014C, + 0x006F0304: 0x014D, + 0x004F0306: 0x014E, + 0x006F0306: 0x014F, + 0x004F030B: 0x0150, + 0x006F030B: 0x0151, + 0x00520301: 0x0154, + 0x00720301: 0x0155, + 0x00520327: 0x0156, + 0x00720327: 0x0157, + 0x0052030C: 0x0158, + 0x0072030C: 0x0159, + 0x00530301: 0x015A, + 0x00730301: 0x015B, + 0x00530302: 0x015C, + 0x00730302: 0x015D, + 0x00530327: 0x015E, + 0x00730327: 0x015F, + 0x0053030C: 0x0160, + 0x0073030C: 0x0161, + 0x00540327: 0x0162, + 0x00740327: 0x0163, + 0x0054030C: 0x0164, + 0x0074030C: 0x0165, + 0x00550303: 0x0168, + 0x00750303: 0x0169, + 0x00550304: 0x016A, + 0x00750304: 0x016B, + 0x00550306: 0x016C, + 0x00750306: 0x016D, + 0x0055030A: 0x016E, + 0x0075030A: 0x016F, + 0x0055030B: 0x0170, + 0x0075030B: 0x0171, + 0x00550328: 0x0172, + 0x00750328: 0x0173, + 0x00570302: 0x0174, + 0x00770302: 0x0175, + 0x00590302: 0x0176, + 0x00790302: 0x0177, + 0x00590308: 0x0178, + 0x005A0301: 0x0179, + 0x007A0301: 0x017A, + 0x005A0307: 0x017B, + 0x007A0307: 0x017C, + 0x005A030C: 0x017D, + 0x007A030C: 0x017E, + 0x004F031B: 0x01A0, + 0x006F031B: 0x01A1, + 0x0055031B: 0x01AF, + 0x0075031B: 0x01B0, + 0x0041030C: 0x01CD, + 0x0061030C: 0x01CE, + 0x0049030C: 0x01CF, + 0x0069030C: 0x01D0, + 0x004F030C: 0x01D1, + 0x006F030C: 0x01D2, + 0x0055030C: 0x01D3, + 0x0075030C: 0x01D4, + 0x00DC0304: 0x01D5, + 0x00FC0304: 0x01D6, + 0x00DC0301: 0x01D7, + 0x00FC0301: 0x01D8, + 0x00DC030C: 0x01D9, + 0x00FC030C: 0x01DA, + 0x00DC0300: 0x01DB, + 0x00FC0300: 0x01DC, + 0x00C40304: 0x01DE, + 0x00E40304: 0x01DF, + 0x02260304: 0x01E0, + 0x02270304: 0x01E1, + 0x00C60304: 0x01E2, + 0x00E60304: 0x01E3, + 0x0047030C: 0x01E6, + 0x0067030C: 0x01E7, + 0x004B030C: 0x01E8, + 0x006B030C: 0x01E9, + 0x004F0328: 0x01EA, + 0x006F0328: 0x01EB, + 0x01EA0304: 0x01EC, + 0x01EB0304: 0x01ED, + 0x01B7030C: 0x01EE, + 0x0292030C: 0x01EF, + 0x006A030C: 0x01F0, + 0x00470301: 0x01F4, + 0x00670301: 0x01F5, + 0x004E0300: 0x01F8, + 0x006E0300: 0x01F9, + 0x00C50301: 0x01FA, + 0x00E50301: 0x01FB, + 0x00C60301: 0x01FC, + 0x00E60301: 0x01FD, + 0x00D80301: 0x01FE, + 0x00F80301: 0x01FF, + 0x0041030F: 0x0200, + 0x0061030F: 0x0201, + 0x00410311: 0x0202, + 0x00610311: 0x0203, + 0x0045030F: 0x0204, + 0x0065030F: 0x0205, + 0x00450311: 0x0206, + 0x00650311: 0x0207, + 0x0049030F: 0x0208, + 0x0069030F: 0x0209, + 0x00490311: 0x020A, + 0x00690311: 0x020B, + 0x004F030F: 0x020C, + 0x006F030F: 0x020D, + 0x004F0311: 0x020E, + 0x006F0311: 0x020F, + 0x0052030F: 0x0210, + 0x0072030F: 0x0211, + 0x00520311: 0x0212, + 0x00720311: 0x0213, + 0x0055030F: 0x0214, + 0x0075030F: 0x0215, + 0x00550311: 0x0216, + 0x00750311: 0x0217, + 0x00530326: 0x0218, + 0x00730326: 0x0219, + 0x00540326: 0x021A, + 0x00740326: 0x021B, + 0x0048030C: 0x021E, + 0x0068030C: 0x021F, + 0x00410307: 0x0226, + 0x00610307: 0x0227, + 0x00450327: 0x0228, + 0x00650327: 0x0229, + 0x00D60304: 0x022A, + 0x00F60304: 0x022B, + 0x00D50304: 0x022C, + 0x00F50304: 0x022D, + 0x004F0307: 0x022E, + 0x006F0307: 0x022F, + 0x022E0304: 0x0230, + 0x022F0304: 0x0231, + 0x00590304: 0x0232, + 0x00790304: 0x0233, + 0x00A80301: 0x0385, + 0x03910301: 0x0386, + 0x03950301: 0x0388, + 0x03970301: 0x0389, + 0x03990301: 0x038A, + 0x039F0301: 0x038C, + 0x03A50301: 0x038E, + 0x03A90301: 0x038F, + 0x03CA0301: 0x0390, + 0x03990308: 0x03AA, + 0x03A50308: 0x03AB, + 0x03B10301: 0x03AC, + 0x03B50301: 0x03AD, + 0x03B70301: 0x03AE, + 0x03B90301: 0x03AF, + 0x03CB0301: 0x03B0, + 0x03B90308: 0x03CA, + 0x03C50308: 0x03CB, + 0x03BF0301: 0x03CC, + 0x03C50301: 0x03CD, + 0x03C90301: 0x03CE, + 0x03D20301: 0x03D3, + 0x03D20308: 0x03D4, + 0x04150300: 0x0400, + 0x04150308: 0x0401, + 0x04130301: 0x0403, + 0x04060308: 0x0407, + 0x041A0301: 0x040C, + 0x04180300: 0x040D, + 0x04230306: 0x040E, + 0x04180306: 0x0419, + 0x04380306: 0x0439, + 0x04350300: 0x0450, + 0x04350308: 0x0451, + 0x04330301: 0x0453, + 0x04560308: 0x0457, + 0x043A0301: 0x045C, + 0x04380300: 0x045D, + 0x04430306: 0x045E, + 0x0474030F: 0x0476, + 0x0475030F: 0x0477, + 0x04160306: 0x04C1, + 0x04360306: 0x04C2, + 0x04100306: 0x04D0, + 0x04300306: 0x04D1, + 0x04100308: 0x04D2, + 0x04300308: 0x04D3, + 0x04150306: 0x04D6, + 0x04350306: 0x04D7, + 0x04D80308: 0x04DA, + 0x04D90308: 0x04DB, + 0x04160308: 0x04DC, + 0x04360308: 0x04DD, + 0x04170308: 0x04DE, + 0x04370308: 0x04DF, + 0x04180304: 0x04E2, + 0x04380304: 0x04E3, + 0x04180308: 0x04E4, + 0x04380308: 0x04E5, + 0x041E0308: 0x04E6, + 0x043E0308: 0x04E7, + 0x04E80308: 0x04EA, + 0x04E90308: 0x04EB, + 0x042D0308: 0x04EC, + 0x044D0308: 0x04ED, + 0x04230304: 0x04EE, + 0x04430304: 0x04EF, + 0x04230308: 0x04F0, + 0x04430308: 0x04F1, + 0x0423030B: 0x04F2, + 0x0443030B: 0x04F3, + 0x04270308: 0x04F4, + 0x04470308: 0x04F5, + 0x042B0308: 0x04F8, + 0x044B0308: 0x04F9, + 0x06270653: 0x0622, + 0x06270654: 0x0623, + 0x06480654: 0x0624, + 0x06270655: 0x0625, + 0x064A0654: 0x0626, + 0x06D50654: 0x06C0, + 0x06C10654: 0x06C2, + 0x06D20654: 0x06D3, + 0x0928093C: 0x0929, + 0x0930093C: 0x0931, + 0x0933093C: 0x0934, + 0x09C709BE: 0x09CB, + 0x09C709D7: 0x09CC, + 0x0B470B56: 0x0B48, + 0x0B470B3E: 0x0B4B, + 0x0B470B57: 0x0B4C, + 0x0B920BD7: 0x0B94, + 0x0BC60BBE: 0x0BCA, + 0x0BC70BBE: 0x0BCB, + 0x0BC60BD7: 0x0BCC, + 0x0C460C56: 0x0C48, + 0x0CBF0CD5: 0x0CC0, + 0x0CC60CD5: 0x0CC7, + 0x0CC60CD6: 0x0CC8, + 0x0CC60CC2: 0x0CCA, + 0x0CCA0CD5: 0x0CCB, + 0x0D460D3E: 0x0D4A, + 0x0D470D3E: 0x0D4B, + 0x0D460D57: 0x0D4C, + 0x0DD90DCA: 0x0DDA, + 0x0DD90DCF: 0x0DDC, + 0x0DDC0DCA: 0x0DDD, + 0x0DD90DDF: 0x0DDE, + 0x1025102E: 0x1026, + 0x1B051B35: 0x1B06, + 0x1B071B35: 0x1B08, + 0x1B091B35: 0x1B0A, + 0x1B0B1B35: 0x1B0C, + 0x1B0D1B35: 0x1B0E, + 0x1B111B35: 0x1B12, + 0x1B3A1B35: 0x1B3B, + 0x1B3C1B35: 0x1B3D, + 0x1B3E1B35: 0x1B40, + 0x1B3F1B35: 0x1B41, + 0x1B421B35: 0x1B43, + 0x00410325: 0x1E00, + 0x00610325: 0x1E01, + 0x00420307: 0x1E02, + 0x00620307: 0x1E03, + 0x00420323: 0x1E04, + 0x00620323: 0x1E05, + 0x00420331: 0x1E06, + 0x00620331: 0x1E07, + 0x00C70301: 0x1E08, + 0x00E70301: 0x1E09, + 0x00440307: 0x1E0A, + 0x00640307: 0x1E0B, + 0x00440323: 0x1E0C, + 0x00640323: 0x1E0D, + 0x00440331: 0x1E0E, + 0x00640331: 0x1E0F, + 0x00440327: 0x1E10, + 0x00640327: 0x1E11, + 0x0044032D: 0x1E12, + 0x0064032D: 0x1E13, + 0x01120300: 0x1E14, + 0x01130300: 0x1E15, + 0x01120301: 0x1E16, + 0x01130301: 0x1E17, + 0x0045032D: 0x1E18, + 0x0065032D: 0x1E19, + 0x00450330: 0x1E1A, + 0x00650330: 0x1E1B, + 0x02280306: 0x1E1C, + 0x02290306: 0x1E1D, + 0x00460307: 0x1E1E, + 0x00660307: 0x1E1F, + 0x00470304: 0x1E20, + 0x00670304: 0x1E21, + 0x00480307: 0x1E22, + 0x00680307: 0x1E23, + 0x00480323: 0x1E24, + 0x00680323: 0x1E25, + 0x00480308: 0x1E26, + 0x00680308: 0x1E27, + 0x00480327: 0x1E28, + 0x00680327: 0x1E29, + 0x0048032E: 0x1E2A, + 0x0068032E: 0x1E2B, + 0x00490330: 0x1E2C, + 0x00690330: 0x1E2D, + 0x00CF0301: 0x1E2E, + 0x00EF0301: 0x1E2F, + 0x004B0301: 0x1E30, + 0x006B0301: 0x1E31, + 0x004B0323: 0x1E32, + 0x006B0323: 0x1E33, + 0x004B0331: 0x1E34, + 0x006B0331: 0x1E35, + 0x004C0323: 0x1E36, + 0x006C0323: 0x1E37, + 0x1E360304: 0x1E38, + 0x1E370304: 0x1E39, + 0x004C0331: 0x1E3A, + 0x006C0331: 0x1E3B, + 0x004C032D: 0x1E3C, + 0x006C032D: 0x1E3D, + 0x004D0301: 0x1E3E, + 0x006D0301: 0x1E3F, + 0x004D0307: 0x1E40, + 0x006D0307: 0x1E41, + 0x004D0323: 0x1E42, + 0x006D0323: 0x1E43, + 0x004E0307: 0x1E44, + 0x006E0307: 0x1E45, + 0x004E0323: 0x1E46, + 0x006E0323: 0x1E47, + 0x004E0331: 0x1E48, + 0x006E0331: 0x1E49, + 0x004E032D: 0x1E4A, + 0x006E032D: 0x1E4B, + 0x00D50301: 0x1E4C, + 0x00F50301: 0x1E4D, + 0x00D50308: 0x1E4E, + 0x00F50308: 0x1E4F, + 0x014C0300: 0x1E50, + 0x014D0300: 0x1E51, + 0x014C0301: 0x1E52, + 0x014D0301: 0x1E53, + 0x00500301: 0x1E54, + 0x00700301: 0x1E55, + 0x00500307: 0x1E56, + 0x00700307: 0x1E57, + 0x00520307: 0x1E58, + 0x00720307: 0x1E59, + 0x00520323: 0x1E5A, + 0x00720323: 0x1E5B, + 0x1E5A0304: 0x1E5C, + 0x1E5B0304: 0x1E5D, + 0x00520331: 0x1E5E, + 0x00720331: 0x1E5F, + 0x00530307: 0x1E60, + 0x00730307: 0x1E61, + 0x00530323: 0x1E62, + 0x00730323: 0x1E63, + 0x015A0307: 0x1E64, + 0x015B0307: 0x1E65, + 0x01600307: 0x1E66, + 0x01610307: 0x1E67, + 0x1E620307: 0x1E68, + 0x1E630307: 0x1E69, + 0x00540307: 0x1E6A, + 0x00740307: 0x1E6B, + 0x00540323: 0x1E6C, + 0x00740323: 0x1E6D, + 0x00540331: 0x1E6E, + 0x00740331: 0x1E6F, + 0x0054032D: 0x1E70, + 0x0074032D: 0x1E71, + 0x00550324: 0x1E72, + 0x00750324: 0x1E73, + 0x00550330: 0x1E74, + 0x00750330: 0x1E75, + 0x0055032D: 0x1E76, + 0x0075032D: 0x1E77, + 0x01680301: 0x1E78, + 0x01690301: 0x1E79, + 0x016A0308: 0x1E7A, + 0x016B0308: 0x1E7B, + 0x00560303: 0x1E7C, + 0x00760303: 0x1E7D, + 0x00560323: 0x1E7E, + 0x00760323: 0x1E7F, + 0x00570300: 0x1E80, + 0x00770300: 0x1E81, + 0x00570301: 0x1E82, + 0x00770301: 0x1E83, + 0x00570308: 0x1E84, + 0x00770308: 0x1E85, + 0x00570307: 0x1E86, + 0x00770307: 0x1E87, + 0x00570323: 0x1E88, + 0x00770323: 0x1E89, + 0x00580307: 0x1E8A, + 0x00780307: 0x1E8B, + 0x00580308: 0x1E8C, + 0x00780308: 0x1E8D, + 0x00590307: 0x1E8E, + 0x00790307: 0x1E8F, + 0x005A0302: 0x1E90, + 0x007A0302: 0x1E91, + 0x005A0323: 0x1E92, + 0x007A0323: 0x1E93, + 0x005A0331: 0x1E94, + 0x007A0331: 0x1E95, + 0x00680331: 0x1E96, + 0x00740308: 0x1E97, + 0x0077030A: 0x1E98, + 0x0079030A: 0x1E99, + 0x017F0307: 0x1E9B, + 0x00410323: 0x1EA0, + 0x00610323: 0x1EA1, + 0x00410309: 0x1EA2, + 0x00610309: 0x1EA3, + 0x00C20301: 0x1EA4, + 0x00E20301: 0x1EA5, + 0x00C20300: 0x1EA6, + 0x00E20300: 0x1EA7, + 0x00C20309: 0x1EA8, + 0x00E20309: 0x1EA9, + 0x00C20303: 0x1EAA, + 0x00E20303: 0x1EAB, + 0x1EA00302: 0x1EAC, + 0x1EA10302: 0x1EAD, + 0x01020301: 0x1EAE, + 0x01030301: 0x1EAF, + 0x01020300: 0x1EB0, + 0x01030300: 0x1EB1, + 0x01020309: 0x1EB2, + 0x01030309: 0x1EB3, + 0x01020303: 0x1EB4, + 0x01030303: 0x1EB5, + 0x1EA00306: 0x1EB6, + 0x1EA10306: 0x1EB7, + 0x00450323: 0x1EB8, + 0x00650323: 0x1EB9, + 0x00450309: 0x1EBA, + 0x00650309: 0x1EBB, + 0x00450303: 0x1EBC, + 0x00650303: 0x1EBD, + 0x00CA0301: 0x1EBE, + 0x00EA0301: 0x1EBF, + 0x00CA0300: 0x1EC0, + 0x00EA0300: 0x1EC1, + 0x00CA0309: 0x1EC2, + 0x00EA0309: 0x1EC3, + 0x00CA0303: 0x1EC4, + 0x00EA0303: 0x1EC5, + 0x1EB80302: 0x1EC6, + 0x1EB90302: 0x1EC7, + 0x00490309: 0x1EC8, + 0x00690309: 0x1EC9, + 0x00490323: 0x1ECA, + 0x00690323: 0x1ECB, + 0x004F0323: 0x1ECC, + 0x006F0323: 0x1ECD, + 0x004F0309: 0x1ECE, + 0x006F0309: 0x1ECF, + 0x00D40301: 0x1ED0, + 0x00F40301: 0x1ED1, + 0x00D40300: 0x1ED2, + 0x00F40300: 0x1ED3, + 0x00D40309: 0x1ED4, + 0x00F40309: 0x1ED5, + 0x00D40303: 0x1ED6, + 0x00F40303: 0x1ED7, + 0x1ECC0302: 0x1ED8, + 0x1ECD0302: 0x1ED9, + 0x01A00301: 0x1EDA, + 0x01A10301: 0x1EDB, + 0x01A00300: 0x1EDC, + 0x01A10300: 0x1EDD, + 0x01A00309: 0x1EDE, + 0x01A10309: 0x1EDF, + 0x01A00303: 0x1EE0, + 0x01A10303: 0x1EE1, + 0x01A00323: 0x1EE2, + 0x01A10323: 0x1EE3, + 0x00550323: 0x1EE4, + 0x00750323: 0x1EE5, + 0x00550309: 0x1EE6, + 0x00750309: 0x1EE7, + 0x01AF0301: 0x1EE8, + 0x01B00301: 0x1EE9, + 0x01AF0300: 0x1EEA, + 0x01B00300: 0x1EEB, + 0x01AF0309: 0x1EEC, + 0x01B00309: 0x1EED, + 0x01AF0303: 0x1EEE, + 0x01B00303: 0x1EEF, + 0x01AF0323: 0x1EF0, + 0x01B00323: 0x1EF1, + 0x00590300: 0x1EF2, + 0x00790300: 0x1EF3, + 0x00590323: 0x1EF4, + 0x00790323: 0x1EF5, + 0x00590309: 0x1EF6, + 0x00790309: 0x1EF7, + 0x00590303: 0x1EF8, + 0x00790303: 0x1EF9, + 0x03B10313: 0x1F00, + 0x03B10314: 0x1F01, + 0x1F000300: 0x1F02, + 0x1F010300: 0x1F03, + 0x1F000301: 0x1F04, + 0x1F010301: 0x1F05, + 0x1F000342: 0x1F06, + 0x1F010342: 0x1F07, + 0x03910313: 0x1F08, + 0x03910314: 0x1F09, + 0x1F080300: 0x1F0A, + 0x1F090300: 0x1F0B, + 0x1F080301: 0x1F0C, + 0x1F090301: 0x1F0D, + 0x1F080342: 0x1F0E, + 0x1F090342: 0x1F0F, + 0x03B50313: 0x1F10, + 0x03B50314: 0x1F11, + 0x1F100300: 0x1F12, + 0x1F110300: 0x1F13, + 0x1F100301: 0x1F14, + 0x1F110301: 0x1F15, + 0x03950313: 0x1F18, + 0x03950314: 0x1F19, + 0x1F180300: 0x1F1A, + 0x1F190300: 0x1F1B, + 0x1F180301: 0x1F1C, + 0x1F190301: 0x1F1D, + 0x03B70313: 0x1F20, + 0x03B70314: 0x1F21, + 0x1F200300: 0x1F22, + 0x1F210300: 0x1F23, + 0x1F200301: 0x1F24, + 0x1F210301: 0x1F25, + 0x1F200342: 0x1F26, + 0x1F210342: 0x1F27, + 0x03970313: 0x1F28, + 0x03970314: 0x1F29, + 0x1F280300: 0x1F2A, + 0x1F290300: 0x1F2B, + 0x1F280301: 0x1F2C, + 0x1F290301: 0x1F2D, + 0x1F280342: 0x1F2E, + 0x1F290342: 0x1F2F, + 0x03B90313: 0x1F30, + 0x03B90314: 0x1F31, + 0x1F300300: 0x1F32, + 0x1F310300: 0x1F33, + 0x1F300301: 0x1F34, + 0x1F310301: 0x1F35, + 0x1F300342: 0x1F36, + 0x1F310342: 0x1F37, + 0x03990313: 0x1F38, + 0x03990314: 0x1F39, + 0x1F380300: 0x1F3A, + 0x1F390300: 0x1F3B, + 0x1F380301: 0x1F3C, + 0x1F390301: 0x1F3D, + 0x1F380342: 0x1F3E, + 0x1F390342: 0x1F3F, + 0x03BF0313: 0x1F40, + 0x03BF0314: 0x1F41, + 0x1F400300: 0x1F42, + 0x1F410300: 0x1F43, + 0x1F400301: 0x1F44, + 0x1F410301: 0x1F45, + 0x039F0313: 0x1F48, + 0x039F0314: 0x1F49, + 0x1F480300: 0x1F4A, + 0x1F490300: 0x1F4B, + 0x1F480301: 0x1F4C, + 0x1F490301: 0x1F4D, + 0x03C50313: 0x1F50, + 0x03C50314: 0x1F51, + 0x1F500300: 0x1F52, + 0x1F510300: 0x1F53, + 0x1F500301: 0x1F54, + 0x1F510301: 0x1F55, + 0x1F500342: 0x1F56, + 0x1F510342: 0x1F57, + 0x03A50314: 0x1F59, + 0x1F590300: 0x1F5B, + 0x1F590301: 0x1F5D, + 0x1F590342: 0x1F5F, + 0x03C90313: 0x1F60, + 0x03C90314: 0x1F61, + 0x1F600300: 0x1F62, + 0x1F610300: 0x1F63, + 0x1F600301: 0x1F64, + 0x1F610301: 0x1F65, + 0x1F600342: 0x1F66, + 0x1F610342: 0x1F67, + 0x03A90313: 0x1F68, + 0x03A90314: 0x1F69, + 0x1F680300: 0x1F6A, + 0x1F690300: 0x1F6B, + 0x1F680301: 0x1F6C, + 0x1F690301: 0x1F6D, + 0x1F680342: 0x1F6E, + 0x1F690342: 0x1F6F, + 0x03B10300: 0x1F70, + 0x03B50300: 0x1F72, + 0x03B70300: 0x1F74, + 0x03B90300: 0x1F76, + 0x03BF0300: 0x1F78, + 0x03C50300: 0x1F7A, + 0x03C90300: 0x1F7C, + 0x1F000345: 0x1F80, + 0x1F010345: 0x1F81, + 0x1F020345: 0x1F82, + 0x1F030345: 0x1F83, + 0x1F040345: 0x1F84, + 0x1F050345: 0x1F85, + 0x1F060345: 0x1F86, + 0x1F070345: 0x1F87, + 0x1F080345: 0x1F88, + 0x1F090345: 0x1F89, + 0x1F0A0345: 0x1F8A, + 0x1F0B0345: 0x1F8B, + 0x1F0C0345: 0x1F8C, + 0x1F0D0345: 0x1F8D, + 0x1F0E0345: 0x1F8E, + 0x1F0F0345: 0x1F8F, + 0x1F200345: 0x1F90, + 0x1F210345: 0x1F91, + 0x1F220345: 0x1F92, + 0x1F230345: 0x1F93, + 0x1F240345: 0x1F94, + 0x1F250345: 0x1F95, + 0x1F260345: 0x1F96, + 0x1F270345: 0x1F97, + 0x1F280345: 0x1F98, + 0x1F290345: 0x1F99, + 0x1F2A0345: 0x1F9A, + 0x1F2B0345: 0x1F9B, + 0x1F2C0345: 0x1F9C, + 0x1F2D0345: 0x1F9D, + 0x1F2E0345: 0x1F9E, + 0x1F2F0345: 0x1F9F, + 0x1F600345: 0x1FA0, + 0x1F610345: 0x1FA1, + 0x1F620345: 0x1FA2, + 0x1F630345: 0x1FA3, + 0x1F640345: 0x1FA4, + 0x1F650345: 0x1FA5, + 0x1F660345: 0x1FA6, + 0x1F670345: 0x1FA7, + 0x1F680345: 0x1FA8, + 0x1F690345: 0x1FA9, + 0x1F6A0345: 0x1FAA, + 0x1F6B0345: 0x1FAB, + 0x1F6C0345: 0x1FAC, + 0x1F6D0345: 0x1FAD, + 0x1F6E0345: 0x1FAE, + 0x1F6F0345: 0x1FAF, + 0x03B10306: 0x1FB0, + 0x03B10304: 0x1FB1, + 0x1F700345: 0x1FB2, + 0x03B10345: 0x1FB3, + 0x03AC0345: 0x1FB4, + 0x03B10342: 0x1FB6, + 0x1FB60345: 0x1FB7, + 0x03910306: 0x1FB8, + 0x03910304: 0x1FB9, + 0x03910300: 0x1FBA, + 0x03910345: 0x1FBC, + 0x00A80342: 0x1FC1, + 0x1F740345: 0x1FC2, + 0x03B70345: 0x1FC3, + 0x03AE0345: 0x1FC4, + 0x03B70342: 0x1FC6, + 0x1FC60345: 0x1FC7, + 0x03950300: 0x1FC8, + 0x03970300: 0x1FCA, + 0x03970345: 0x1FCC, + 0x1FBF0300: 0x1FCD, + 0x1FBF0301: 0x1FCE, + 0x1FBF0342: 0x1FCF, + 0x03B90306: 0x1FD0, + 0x03B90304: 0x1FD1, + 0x03CA0300: 0x1FD2, + 0x03B90342: 0x1FD6, + 0x03CA0342: 0x1FD7, + 0x03990306: 0x1FD8, + 0x03990304: 0x1FD9, + 0x03990300: 0x1FDA, + 0x1FFE0300: 0x1FDD, + 0x1FFE0301: 0x1FDE, + 0x1FFE0342: 0x1FDF, + 0x03C50306: 0x1FE0, + 0x03C50304: 0x1FE1, + 0x03CB0300: 0x1FE2, + 0x03C10313: 0x1FE4, + 0x03C10314: 0x1FE5, + 0x03C50342: 0x1FE6, + 0x03CB0342: 0x1FE7, + 0x03A50306: 0x1FE8, + 0x03A50304: 0x1FE9, + 0x03A50300: 0x1FEA, + 0x03A10314: 0x1FEC, + 0x00A80300: 0x1FED, + 0x1F7C0345: 0x1FF2, + 0x03C90345: 0x1FF3, + 0x03CE0345: 0x1FF4, + 0x03C90342: 0x1FF6, + 0x1FF60345: 0x1FF7, + 0x039F0300: 0x1FF8, + 0x03A90300: 0x1FFA, + 0x03A90345: 0x1FFC, + 0x21900338: 0x219A, + 0x21920338: 0x219B, + 0x21940338: 0x21AE, + 0x21D00338: 0x21CD, + 0x21D40338: 0x21CE, + 0x21D20338: 0x21CF, + 0x22030338: 0x2204, + 0x22080338: 0x2209, + 0x220B0338: 0x220C, + 0x22230338: 0x2224, + 0x22250338: 0x2226, + 0x223C0338: 0x2241, + 0x22430338: 0x2244, + 0x22450338: 0x2247, + 0x22480338: 0x2249, + 0x003D0338: 0x2260, + 0x22610338: 0x2262, + 0x224D0338: 0x226D, + 0x003C0338: 0x226E, + 0x003E0338: 0x226F, + 0x22640338: 0x2270, + 0x22650338: 0x2271, + 0x22720338: 0x2274, + 0x22730338: 0x2275, + 0x22760338: 0x2278, + 0x22770338: 0x2279, + 0x227A0338: 0x2280, + 0x227B0338: 0x2281, + 0x22820338: 0x2284, + 0x22830338: 0x2285, + 0x22860338: 0x2288, + 0x22870338: 0x2289, + 0x22A20338: 0x22AC, + 0x22A80338: 0x22AD, + 0x22A90338: 0x22AE, + 0x22AB0338: 0x22AF, + 0x227C0338: 0x22E0, + 0x227D0338: 0x22E1, + 0x22910338: 0x22E2, + 0x22920338: 0x22E3, + 0x22B20338: 0x22EA, + 0x22B30338: 0x22EB, + 0x22B40338: 0x22EC, + 0x22B50338: 0x22ED, + 0x304B3099: 0x304C, + 0x304D3099: 0x304E, + 0x304F3099: 0x3050, + 0x30513099: 0x3052, + 0x30533099: 0x3054, + 0x30553099: 0x3056, + 0x30573099: 0x3058, + 0x30593099: 0x305A, + 0x305B3099: 0x305C, + 0x305D3099: 0x305E, + 0x305F3099: 0x3060, + 0x30613099: 0x3062, + 0x30643099: 0x3065, + 0x30663099: 0x3067, + 0x30683099: 0x3069, + 0x306F3099: 0x3070, + 0x306F309A: 0x3071, + 0x30723099: 0x3073, + 0x3072309A: 0x3074, + 0x30753099: 0x3076, + 0x3075309A: 0x3077, + 0x30783099: 0x3079, + 0x3078309A: 0x307A, + 0x307B3099: 0x307C, + 0x307B309A: 0x307D, + 0x30463099: 0x3094, + 0x309D3099: 0x309E, + 0x30AB3099: 0x30AC, + 0x30AD3099: 0x30AE, + 0x30AF3099: 0x30B0, + 0x30B13099: 0x30B2, + 0x30B33099: 0x30B4, + 0x30B53099: 0x30B6, + 0x30B73099: 0x30B8, + 0x30B93099: 0x30BA, + 0x30BB3099: 0x30BC, + 0x30BD3099: 0x30BE, + 0x30BF3099: 0x30C0, + 0x30C13099: 0x30C2, + 0x30C43099: 0x30C5, + 0x30C63099: 0x30C7, + 0x30C83099: 0x30C9, + 0x30CF3099: 0x30D0, + 0x30CF309A: 0x30D1, + 0x30D23099: 0x30D3, + 0x30D2309A: 0x30D4, + 0x30D53099: 0x30D6, + 0x30D5309A: 0x30D7, + 0x30D83099: 0x30D9, + 0x30D8309A: 0x30DA, + 0x30DB3099: 0x30DC, + 0x30DB309A: 0x30DD, + 0x30A63099: 0x30F4, + 0x30EF3099: 0x30F7, + 0x30F03099: 0x30F8, + 0x30F13099: 0x30F9, + 0x30F23099: 0x30FA, + 0x30FD3099: 0x30FE, + 0x109910BA: 0x1109A, + 0x109B10BA: 0x1109C, + 0x10A510BA: 0x110AB, +} + +// charInfoValues: 10944 entries, 21888 bytes +// Block 2 is the null block. +var charInfoValues = [10944]uint16{ + // Block 0x0, offset 0x0 + 0x003c: 0x8800, 0x003d: 0x8800, 0x003e: 0x8800, + // Block 0x1, offset 0x40 + 0x0041: 0x8800, 0x0042: 0x8800, 0x0043: 0x8800, 0x0044: 0x8800, 0x0045: 0x8800, + 0x0046: 0x8800, 0x0047: 0x8800, 0x0048: 0x8800, 0x0049: 0x8800, 0x004a: 0x8800, 0x004b: 0x8800, + 0x004c: 0x8800, 0x004d: 0x8800, 0x004e: 0x8800, 0x004f: 0x8800, 0x0050: 0x8800, + 0x0052: 0x8800, 0x0053: 0x8800, 0x0054: 0x8800, 0x0055: 0x8800, 0x0056: 0x8800, 0x0057: 0x8800, + 0x0058: 0x8800, 0x0059: 0x8800, 0x005a: 0x8800, + 0x0061: 0x8800, 0x0062: 0x8800, 0x0063: 0x8800, + 0x0064: 0x8800, 0x0065: 0x8800, 0x0066: 0x8800, 0x0067: 0x8800, 0x0068: 0x8800, 0x0069: 0x8800, + 0x006a: 0x8800, 0x006b: 0x8800, 0x006c: 0x8800, 0x006d: 0x8800, 0x006e: 0x8800, 0x006f: 0x8800, + 0x0070: 0x8800, 0x0072: 0x8800, 0x0073: 0x8800, 0x0074: 0x8800, 0x0075: 0x8800, + 0x0076: 0x8800, 0x0077: 0x8800, 0x0078: 0x8800, 0x0079: 0x8800, 0x007a: 0x8800, + // Block 0x2, offset 0x80 + // Block 0x3, offset 0xc0 + 0x00e0: 0x3000, + 0x00e8: 0x3800, + 0x00ea: 0x3000, 0x00ef: 0x3000, + 0x00f2: 0x3000, 0x00f3: 0x3000, 0x00f4: 0x3000, 0x00f5: 0x3000, + 0x00f8: 0x3000, 0x00f9: 0x3000, 0x00fa: 0x3000, + 0x00fc: 0x3000, 0x00fd: 0x3000, 0x00fe: 0x3000, + // Block 0x4, offset 0x100 + 0x0100: 0x1100, 0x0101: 0x1100, 0x0102: 0x9900, 0x0103: 0x1100, 0x0104: 0x9900, 0x0105: 0x9900, + 0x0106: 0x8800, 0x0107: 0x9900, 0x0108: 0x1100, 0x0109: 0x1100, 0x010a: 0x9900, 0x010b: 0x1100, + 0x010c: 0x1100, 0x010d: 0x1100, 0x010e: 0x1100, 0x010f: 0x9900, 0x0111: 0x1100, + 0x0112: 0x1100, 0x0113: 0x1100, 0x0114: 0x9900, 0x0115: 0x9900, 0x0116: 0x9900, + 0x0118: 0x8800, 0x0119: 0x1100, 0x011a: 0x1100, 0x011b: 0x1100, 0x011c: 0x9900, 0x011d: 0x1100, + 0x0120: 0x1100, 0x0121: 0x1100, 0x0122: 0x9900, 0x0123: 0x1100, + 0x0124: 0x9900, 0x0125: 0x9900, 0x0126: 0x8800, 0x0127: 0x9900, 0x0128: 0x1100, 0x0129: 0x1100, + 0x012a: 0x9900, 0x012b: 0x1100, 0x012c: 0x1100, 0x012d: 0x1100, 0x012e: 0x1100, 0x012f: 0x9900, + 0x0131: 0x1100, 0x0132: 0x1100, 0x0133: 0x1100, 0x0134: 0x9900, 0x0135: 0x9900, + 0x0136: 0x9900, 0x0138: 0x8800, 0x0139: 0x1100, 0x013a: 0x1100, 0x013b: 0x1100, + 0x013c: 0x9900, 0x013d: 0x1100, 0x013f: 0x1100, + // Block 0x5, offset 0x140 + 0x0140: 0x1100, 0x0141: 0x1100, 0x0142: 0x9900, 0x0143: 0x9900, 0x0144: 0x1100, 0x0145: 0x1100, + 0x0146: 0x1100, 0x0147: 0x1100, 0x0148: 0x1100, 0x0149: 0x1100, 0x014a: 0x1100, 0x014b: 0x1100, + 0x014c: 0x1100, 0x014d: 0x1100, 0x014e: 0x1100, 0x014f: 0x1100, + 0x0152: 0x9900, 0x0153: 0x9900, 0x0154: 0x1100, 0x0155: 0x1100, 0x0156: 0x1100, 0x0157: 0x1100, + 0x0158: 0x1100, 0x0159: 0x1100, 0x015a: 0x1100, 0x015b: 0x1100, 0x015c: 0x1100, 0x015d: 0x1100, + 0x015e: 0x1100, 0x015f: 0x1100, 0x0160: 0x1100, 0x0161: 0x1100, 0x0162: 0x1100, 0x0163: 0x1100, + 0x0164: 0x1100, 0x0165: 0x1100, 0x0168: 0x1100, 0x0169: 0x1100, + 0x016a: 0x1100, 0x016b: 0x1100, 0x016c: 0x1100, 0x016d: 0x1100, 0x016e: 0x1100, 0x016f: 0x1100, + 0x0170: 0x1100, 0x0172: 0x3000, 0x0173: 0x3000, 0x0174: 0x1100, 0x0175: 0x1100, + 0x0176: 0x1100, 0x0177: 0x1100, 0x0179: 0x1100, 0x017a: 0x1100, 0x017b: 0x1100, + 0x017c: 0x1100, 0x017d: 0x1100, 0x017e: 0x1100, 0x017f: 0x3000, + // Block 0x6, offset 0x180 + 0x0180: 0x3000, 0x0183: 0x1100, 0x0184: 0x1100, 0x0185: 0x1100, + 0x0186: 0x1100, 0x0187: 0x1100, 0x0188: 0x1100, 0x0189: 0x3000, + 0x018c: 0x9900, 0x018d: 0x9900, 0x018e: 0x1100, 0x018f: 0x1100, 0x0190: 0x1100, 0x0191: 0x1100, + 0x0194: 0x1100, 0x0195: 0x1100, 0x0196: 0x1100, 0x0197: 0x1100, + 0x0198: 0x1100, 0x0199: 0x1100, 0x019a: 0x9900, 0x019b: 0x9900, 0x019c: 0x1100, 0x019d: 0x1100, + 0x019e: 0x1100, 0x019f: 0x1100, 0x01a0: 0x9900, 0x01a1: 0x9900, 0x01a2: 0x1100, 0x01a3: 0x1100, + 0x01a4: 0x1100, 0x01a5: 0x1100, 0x01a8: 0x9900, 0x01a9: 0x9900, + 0x01aa: 0x9900, 0x01ab: 0x9900, 0x01ac: 0x1100, 0x01ad: 0x1100, 0x01ae: 0x1100, 0x01af: 0x1100, + 0x01b0: 0x1100, 0x01b1: 0x1100, 0x01b2: 0x1100, 0x01b3: 0x1100, 0x01b4: 0x1100, 0x01b5: 0x1100, + 0x01b6: 0x1100, 0x01b7: 0x1100, 0x01b8: 0x1100, 0x01b9: 0x1100, 0x01ba: 0x1100, 0x01bb: 0x1100, + 0x01bc: 0x1100, 0x01bd: 0x1100, 0x01be: 0x1100, 0x01bf: 0x3800, + // Block 0x7, offset 0x1c0 + 0x01e0: 0x9900, 0x01e1: 0x9900, + 0x01ef: 0x9900, + 0x01f0: 0x9900, + 0x01f7: 0x8800, + // Block 0x8, offset 0x200 + 0x0204: 0x3000, 0x0205: 0x3000, + 0x0206: 0x3000, 0x0207: 0x3000, 0x0208: 0x3000, 0x0209: 0x3000, 0x020a: 0x3000, 0x020b: 0x3000, + 0x020c: 0x3000, 0x020d: 0x1100, 0x020e: 0x1100, 0x020f: 0x1100, 0x0210: 0x1100, 0x0211: 0x1100, + 0x0212: 0x1100, 0x0213: 0x1100, 0x0214: 0x1100, 0x0215: 0x1100, 0x0216: 0x1100, 0x0217: 0x1100, + 0x0218: 0x1100, 0x0219: 0x1100, 0x021a: 0x1100, 0x021b: 0x1100, 0x021c: 0x1100, + 0x021e: 0x1100, 0x021f: 0x1100, 0x0220: 0x1100, 0x0221: 0x1100, 0x0222: 0x1100, 0x0223: 0x1100, + 0x0226: 0x1100, 0x0227: 0x1100, 0x0228: 0x1100, 0x0229: 0x1100, + 0x022a: 0x9900, 0x022b: 0x9900, 0x022c: 0x1100, 0x022d: 0x1100, 0x022e: 0x1100, 0x022f: 0x1100, + 0x0230: 0x1100, 0x0231: 0x3000, 0x0232: 0x3000, 0x0233: 0x3000, 0x0234: 0x1100, 0x0235: 0x1100, + 0x0238: 0x1100, 0x0239: 0x1100, 0x023a: 0x1100, 0x023b: 0x1100, + 0x023c: 0x1100, 0x023d: 0x1100, 0x023e: 0x1100, 0x023f: 0x1100, + // Block 0x9, offset 0x240 + 0x0240: 0x1100, 0x0241: 0x1100, 0x0242: 0x1100, 0x0243: 0x1100, 0x0244: 0x1100, 0x0245: 0x1100, + 0x0246: 0x1100, 0x0247: 0x1100, 0x0248: 0x1100, 0x0249: 0x1100, 0x024a: 0x1100, 0x024b: 0x1100, + 0x024c: 0x1100, 0x024d: 0x1100, 0x024e: 0x1100, 0x024f: 0x1100, 0x0250: 0x1100, 0x0251: 0x1100, + 0x0252: 0x1100, 0x0253: 0x1100, 0x0254: 0x1100, 0x0255: 0x1100, 0x0256: 0x1100, 0x0257: 0x1100, + 0x0258: 0x1100, 0x0259: 0x1100, 0x025a: 0x1100, 0x025b: 0x1100, + 0x025e: 0x1100, 0x025f: 0x1100, + 0x0266: 0x9900, 0x0267: 0x9900, 0x0268: 0x9900, 0x0269: 0x9900, + 0x026a: 0x1100, 0x026b: 0x1100, 0x026c: 0x1100, 0x026d: 0x1100, 0x026e: 0x9900, 0x026f: 0x9900, + 0x0270: 0x1100, 0x0271: 0x1100, 0x0272: 0x1100, 0x0273: 0x1100, + // Block 0xa, offset 0x280 + 0x0292: 0x8800, + 0x02b0: 0x3000, 0x02b1: 0x3000, 0x02b2: 0x3000, 0x02b3: 0x3000, 0x02b4: 0x3000, 0x02b5: 0x3000, + 0x02b6: 0x3000, 0x02b7: 0x3000, 0x02b8: 0x3000, + // Block 0xb, offset 0x2c0 + 0x02d8: 0x3000, 0x02d9: 0x3000, 0x02da: 0x3000, 0x02db: 0x3000, 0x02dc: 0x3000, 0x02dd: 0x3000, + 0x02e0: 0x3000, 0x02e1: 0x3000, 0x02e2: 0x3000, 0x02e3: 0x3000, + 0x02e4: 0x3000, + // Block 0xc, offset 0x300 + 0x0300: 0x66e6, 0x0301: 0x66e6, 0x0302: 0x66e6, 0x0303: 0x66e6, 0x0304: 0x66e6, 0x0305: 0x00e6, + 0x0306: 0x66e6, 0x0307: 0x66e6, 0x0308: 0x66e6, 0x0309: 0x66e6, 0x030a: 0x66e6, 0x030b: 0x66e6, + 0x030c: 0x66e6, 0x030d: 0x00e6, 0x030e: 0x00e6, 0x030f: 0x66e6, 0x0310: 0x00e6, 0x0311: 0x66e6, + 0x0312: 0x00e6, 0x0313: 0x66e6, 0x0314: 0x66e6, 0x0315: 0x00e8, 0x0316: 0x00dc, 0x0317: 0x00dc, + 0x0318: 0x00dc, 0x0319: 0x00dc, 0x031a: 0x00e8, 0x031b: 0x66d8, 0x031c: 0x00dc, 0x031d: 0x00dc, + 0x031e: 0x00dc, 0x031f: 0x00dc, 0x0320: 0x00dc, 0x0321: 0x00ca, 0x0322: 0x00ca, 0x0323: 0x66dc, + 0x0324: 0x66dc, 0x0325: 0x66dc, 0x0326: 0x66dc, 0x0327: 0x66ca, 0x0328: 0x66ca, 0x0329: 0x00dc, + 0x032a: 0x00dc, 0x032b: 0x00dc, 0x032c: 0x00dc, 0x032d: 0x66dc, 0x032e: 0x66dc, 0x032f: 0x00dc, + 0x0330: 0x66dc, 0x0331: 0x66dc, 0x0332: 0x00dc, 0x0333: 0x00dc, 0x0334: 0x0001, 0x0335: 0x0001, + 0x0336: 0x0001, 0x0337: 0x0001, 0x0338: 0x6601, 0x0339: 0x00dc, 0x033a: 0x00dc, 0x033b: 0x00dc, + 0x033c: 0x00dc, 0x033d: 0x00e6, 0x033e: 0x00e6, 0x033f: 0x00e6, + // Block 0xd, offset 0x340 + 0x0340: 0x33e6, 0x0341: 0x33e6, 0x0342: 0x66e6, 0x0343: 0x33e6, 0x0344: 0x33e6, 0x0345: 0x66f0, + 0x0346: 0x00e6, 0x0347: 0x00dc, 0x0348: 0x00dc, 0x0349: 0x00dc, 0x034a: 0x00e6, 0x034b: 0x00e6, + 0x034c: 0x00e6, 0x034d: 0x00dc, 0x034e: 0x00dc, 0x0350: 0x00e6, 0x0351: 0x00e6, + 0x0352: 0x00e6, 0x0353: 0x00dc, 0x0354: 0x00dc, 0x0355: 0x00dc, 0x0356: 0x00dc, 0x0357: 0x00e6, + 0x0358: 0x00e8, 0x0359: 0x00dc, 0x035a: 0x00dc, 0x035b: 0x00e6, 0x035c: 0x00e9, 0x035d: 0x00ea, + 0x035e: 0x00ea, 0x035f: 0x00e9, 0x0360: 0x00ea, 0x0361: 0x00ea, 0x0362: 0x00e9, 0x0363: 0x00e6, + 0x0364: 0x00e6, 0x0365: 0x00e6, 0x0366: 0x00e6, 0x0367: 0x00e6, 0x0368: 0x00e6, 0x0369: 0x00e6, + 0x036a: 0x00e6, 0x036b: 0x00e6, 0x036c: 0x00e6, 0x036d: 0x00e6, 0x036e: 0x00e6, 0x036f: 0x00e6, + 0x0374: 0x3300, + 0x037a: 0x3000, + 0x037e: 0x3300, + // Block 0xe, offset 0x380 + 0x0384: 0x3000, 0x0385: 0x3100, + 0x0386: 0x1100, 0x0387: 0x3300, 0x0388: 0x1100, 0x0389: 0x1100, 0x038a: 0x1100, + 0x038c: 0x1100, 0x038e: 0x1100, 0x038f: 0x1100, 0x0390: 0x1100, 0x0391: 0x8800, + 0x0395: 0x8800, 0x0397: 0x8800, + 0x0399: 0x8800, + 0x039f: 0x8800, 0x03a1: 0x8800, + 0x03a5: 0x8800, 0x03a9: 0x8800, + 0x03aa: 0x1100, 0x03ab: 0x1100, 0x03ac: 0x9900, 0x03ad: 0x1100, 0x03ae: 0x9900, 0x03af: 0x1100, + 0x03b0: 0x1100, 0x03b1: 0x8800, 0x03b5: 0x8800, + 0x03b7: 0x8800, 0x03b9: 0x8800, + 0x03bf: 0x8800, + // Block 0xf, offset 0x3c0 + 0x03c1: 0x8800, 0x03c5: 0x8800, + 0x03c9: 0x8800, 0x03ca: 0x9900, 0x03cb: 0x9900, + 0x03cc: 0x1100, 0x03cd: 0x1100, 0x03ce: 0x9900, 0x03d0: 0x3000, 0x03d1: 0x3000, + 0x03d2: 0x3800, 0x03d3: 0x3100, 0x03d4: 0x3100, 0x03d5: 0x3000, 0x03d6: 0x3000, + 0x03f0: 0x3000, 0x03f1: 0x3000, 0x03f2: 0x3000, 0x03f4: 0x3000, 0x03f5: 0x3000, + 0x03f9: 0x3000, + // Block 0x10, offset 0x400 + 0x0400: 0x1100, 0x0401: 0x1100, 0x0403: 0x1100, + 0x0406: 0x8800, 0x0407: 0x1100, + 0x040c: 0x1100, 0x040d: 0x1100, 0x040e: 0x1100, 0x0410: 0x8800, + 0x0413: 0x8800, 0x0415: 0x8800, 0x0416: 0x8800, 0x0417: 0x8800, + 0x0418: 0x8800, 0x0419: 0x1100, 0x041a: 0x8800, + 0x041e: 0x8800, 0x0423: 0x8800, + 0x0427: 0x8800, + 0x042b: 0x8800, 0x042d: 0x8800, + 0x0430: 0x8800, 0x0433: 0x8800, 0x0435: 0x8800, + 0x0436: 0x8800, 0x0437: 0x8800, 0x0438: 0x8800, 0x0439: 0x1100, 0x043a: 0x8800, + 0x043e: 0x8800, + // Block 0x11, offset 0x440 + 0x0443: 0x8800, + 0x0447: 0x8800, 0x044b: 0x8800, + 0x044d: 0x8800, 0x0450: 0x1100, 0x0451: 0x1100, + 0x0453: 0x1100, 0x0456: 0x8800, 0x0457: 0x1100, + 0x045c: 0x1100, 0x045d: 0x1100, + 0x045e: 0x1100, + 0x0474: 0x8800, 0x0475: 0x8800, + 0x0476: 0x1100, 0x0477: 0x1100, + // Block 0x12, offset 0x480 + 0x0483: 0x00e6, 0x0484: 0x00e6, 0x0485: 0x00e6, + 0x0486: 0x00e6, 0x0487: 0x00e6, + // Block 0x13, offset 0x4c0 + 0x04c1: 0x1100, 0x04c2: 0x1100, + 0x04d0: 0x1100, 0x04d1: 0x1100, + 0x04d2: 0x1100, 0x04d3: 0x1100, 0x04d6: 0x1100, 0x04d7: 0x1100, + 0x04d8: 0x8800, 0x04d9: 0x8800, 0x04da: 0x1100, 0x04db: 0x1100, 0x04dc: 0x1100, 0x04dd: 0x1100, + 0x04de: 0x1100, 0x04df: 0x1100, 0x04e2: 0x1100, 0x04e3: 0x1100, + 0x04e4: 0x1100, 0x04e5: 0x1100, 0x04e6: 0x1100, 0x04e7: 0x1100, 0x04e8: 0x8800, 0x04e9: 0x8800, + 0x04ea: 0x1100, 0x04eb: 0x1100, 0x04ec: 0x1100, 0x04ed: 0x1100, 0x04ee: 0x1100, 0x04ef: 0x1100, + 0x04f0: 0x1100, 0x04f1: 0x1100, 0x04f2: 0x1100, 0x04f3: 0x1100, 0x04f4: 0x1100, 0x04f5: 0x1100, + 0x04f8: 0x1100, 0x04f9: 0x1100, + // Block 0x14, offset 0x500 + 0x0507: 0x3000, + 0x0511: 0x00dc, + 0x0512: 0x00e6, 0x0513: 0x00e6, 0x0514: 0x00e6, 0x0515: 0x00e6, 0x0516: 0x00dc, 0x0517: 0x00e6, + 0x0518: 0x00e6, 0x0519: 0x00e6, 0x051a: 0x00de, 0x051b: 0x00dc, 0x051c: 0x00e6, 0x051d: 0x00e6, + 0x051e: 0x00e6, 0x051f: 0x00e6, 0x0520: 0x00e6, 0x0521: 0x00e6, 0x0522: 0x00dc, 0x0523: 0x00dc, + 0x0524: 0x00dc, 0x0525: 0x00dc, 0x0526: 0x00dc, 0x0527: 0x00dc, 0x0528: 0x00e6, 0x0529: 0x00e6, + 0x052a: 0x00dc, 0x052b: 0x00e6, 0x052c: 0x00e6, 0x052d: 0x00de, 0x052e: 0x00e4, 0x052f: 0x00e6, + 0x0530: 0x000a, 0x0531: 0x000b, 0x0532: 0x000c, 0x0533: 0x000d, 0x0534: 0x000e, 0x0535: 0x000f, + 0x0536: 0x0010, 0x0537: 0x0011, 0x0538: 0x0012, 0x0539: 0x0013, 0x053a: 0x0013, 0x053b: 0x0014, + 0x053c: 0x0015, 0x053d: 0x0016, 0x053f: 0x0017, + // Block 0x15, offset 0x540 + 0x0541: 0x0018, 0x0542: 0x0019, 0x0544: 0x00e6, 0x0545: 0x00dc, + 0x0547: 0x0012, + // Block 0x16, offset 0x580 + 0x0590: 0x00e6, 0x0591: 0x00e6, + 0x0592: 0x00e6, 0x0593: 0x00e6, 0x0594: 0x00e6, 0x0595: 0x00e6, 0x0596: 0x00e6, 0x0597: 0x00e6, + 0x0598: 0x001e, 0x0599: 0x001f, 0x059a: 0x0020, + 0x05a2: 0x1100, 0x05a3: 0x1100, + 0x05a4: 0x1100, 0x05a5: 0x1100, 0x05a6: 0x1100, 0x05a7: 0x8800, + // Block 0x17, offset 0x5c0 + 0x05c8: 0x8800, 0x05ca: 0x8800, 0x05cb: 0x001b, + 0x05cc: 0x001c, 0x05cd: 0x001d, 0x05ce: 0x001e, 0x05cf: 0x001f, 0x05d0: 0x0020, 0x05d1: 0x0021, + 0x05d2: 0x0022, 0x05d3: 0x66e6, 0x05d4: 0x66e6, 0x05d5: 0x66dc, 0x05d6: 0x00dc, 0x05d7: 0x00e6, + 0x05d8: 0x00e6, 0x05d9: 0x00e6, 0x05da: 0x00e6, 0x05db: 0x00e6, 0x05dc: 0x00dc, 0x05dd: 0x00e6, + 0x05de: 0x00e6, 0x05df: 0x00dc, + 0x05f0: 0x0023, 0x05f5: 0x3000, + 0x05f6: 0x3000, 0x05f7: 0x3000, 0x05f8: 0x3000, + // Block 0x18, offset 0x600 + 0x0600: 0x1100, 0x0601: 0x8800, 0x0602: 0x1100, + 0x0612: 0x8800, 0x0613: 0x1100, 0x0615: 0x8800, 0x0616: 0x00e6, 0x0617: 0x00e6, + 0x0618: 0x00e6, 0x0619: 0x00e6, 0x061a: 0x00e6, 0x061b: 0x00e6, 0x061c: 0x00e6, + 0x061f: 0x00e6, 0x0620: 0x00e6, 0x0621: 0x00e6, 0x0622: 0x00e6, 0x0623: 0x00dc, + 0x0624: 0x00e6, 0x0627: 0x00e6, 0x0628: 0x00e6, + 0x062a: 0x00dc, 0x062b: 0x00e6, 0x062c: 0x00e6, 0x062d: 0x00dc, + // Block 0x19, offset 0x640 + 0x0651: 0x0024, + 0x0670: 0x00e6, 0x0671: 0x00dc, 0x0672: 0x00e6, 0x0673: 0x00e6, 0x0674: 0x00dc, 0x0675: 0x00e6, + 0x0676: 0x00e6, 0x0677: 0x00dc, 0x0678: 0x00dc, 0x0679: 0x00dc, 0x067a: 0x00e6, 0x067b: 0x00dc, + 0x067c: 0x00dc, 0x067d: 0x00e6, 0x067e: 0x00dc, 0x067f: 0x00e6, + // Block 0x1a, offset 0x680 + 0x0680: 0x00e6, 0x0681: 0x00e6, 0x0682: 0x00dc, 0x0683: 0x00e6, 0x0684: 0x00dc, 0x0685: 0x00e6, + 0x0686: 0x00dc, 0x0687: 0x00e6, 0x0688: 0x00dc, 0x0689: 0x00e6, 0x068a: 0x00e6, + // Block 0x1b, offset 0x6c0 + 0x06eb: 0x00e6, 0x06ec: 0x00e6, 0x06ed: 0x00e6, 0x06ee: 0x00e6, 0x06ef: 0x00e6, + 0x06f0: 0x00e6, 0x06f1: 0x00e6, 0x06f2: 0x00dc, 0x06f3: 0x00e6, + // Block 0x1c, offset 0x700 + 0x0716: 0x00e6, 0x0717: 0x00e6, + 0x0718: 0x00e6, 0x0719: 0x00e6, 0x071b: 0x00e6, 0x071c: 0x00e6, 0x071d: 0x00e6, + 0x071e: 0x00e6, 0x071f: 0x00e6, 0x0720: 0x00e6, 0x0721: 0x00e6, 0x0722: 0x00e6, 0x0723: 0x00e6, + 0x0725: 0x00e6, 0x0726: 0x00e6, 0x0727: 0x00e6, 0x0729: 0x00e6, + 0x072a: 0x00e6, 0x072b: 0x00e6, 0x072c: 0x00e6, 0x072d: 0x00e6, + // Block 0x1d, offset 0x740 + 0x0759: 0x00dc, 0x075a: 0x00dc, 0x075b: 0x00dc, + // Block 0x1e, offset 0x780 + 0x07a8: 0x8800, 0x07a9: 0x1100, + 0x07b0: 0x8800, 0x07b1: 0x1100, 0x07b3: 0x8800, 0x07b4: 0x1100, + 0x07bc: 0x6607, + // Block 0x1f, offset 0x7c0 + 0x07cd: 0x0009, 0x07d1: 0x00e6, + 0x07d2: 0x00dc, 0x07d3: 0x00e6, 0x07d4: 0x00e6, + 0x07d8: 0x3300, 0x07d9: 0x3300, 0x07da: 0x3300, 0x07db: 0x3300, 0x07dc: 0x3300, 0x07dd: 0x3300, + 0x07de: 0x3300, 0x07df: 0x3300, + // Block 0x20, offset 0x800 + 0x083c: 0x0007, 0x083e: 0x6600, + // Block 0x21, offset 0x840 + 0x0847: 0x8800, 0x084b: 0x1100, + 0x084c: 0x1100, 0x084d: 0x0009, + 0x0857: 0x6600, + 0x085c: 0x3300, 0x085d: 0x3300, + 0x085f: 0x3300, + // Block 0x22, offset 0x880 + 0x08b3: 0x3300, + 0x08b6: 0x3300, + 0x08bc: 0x0007, + // Block 0x23, offset 0x8c0 + 0x08cd: 0x0009, + 0x08d9: 0x3300, 0x08da: 0x3300, 0x08db: 0x3300, + 0x08de: 0x3300, + // Block 0x24, offset 0x900 + 0x093c: 0x0007, + // Block 0x25, offset 0x940 + 0x094d: 0x0009, + // Block 0x26, offset 0x980 + 0x0987: 0x8800, 0x0988: 0x1100, 0x098b: 0x1100, + 0x098c: 0x1100, 0x098d: 0x0009, + 0x0996: 0x6600, 0x0997: 0x6600, + 0x099c: 0x3300, 0x099d: 0x3300, + // Block 0x27, offset 0x9c0 + 0x09d2: 0x8800, 0x09d4: 0x1100, + 0x09fe: 0x6600, + // Block 0x28, offset 0xa00 + 0x0a06: 0x8800, 0x0a07: 0x8800, 0x0a0a: 0x1100, 0x0a0b: 0x1100, + 0x0a0c: 0x1100, 0x0a0d: 0x0009, + 0x0a17: 0x6600, + // Block 0x29, offset 0xa40 + 0x0a46: 0x8800, 0x0a48: 0x1100, + 0x0a4d: 0x0009, + 0x0a55: 0x0054, 0x0a56: 0x665b, + // Block 0x2a, offset 0xa80 + 0x0abc: 0x0007, 0x0abf: 0x8800, + // Block 0x2b, offset 0xac0 + 0x0ac0: 0x1100, 0x0ac2: 0x6600, + 0x0ac6: 0x8800, 0x0ac7: 0x1100, 0x0ac8: 0x1100, 0x0aca: 0x9900, 0x0acb: 0x1100, + 0x0acd: 0x0009, + 0x0ad5: 0x6600, 0x0ad6: 0x6600, + // Block 0x2c, offset 0xb00 + 0x0b3e: 0x6600, + // Block 0x2d, offset 0xb40 + 0x0b4a: 0x6609, + 0x0b4f: 0x6600, + 0x0b59: 0x8800, 0x0b5a: 0x1100, 0x0b5c: 0x9900, 0x0b5d: 0x1100, + 0x0b5e: 0x1100, 0x0b5f: 0x6600, + // Block 0x2e, offset 0xb80 + 0x0bb3: 0x3000, + 0x0bb8: 0x0067, 0x0bb9: 0x0067, 0x0bba: 0x0009, + // Block 0x2f, offset 0xbc0 + 0x0bc8: 0x006b, 0x0bc9: 0x006b, 0x0bca: 0x006b, 0x0bcb: 0x006b, + // Block 0x30, offset 0xc00 + 0x0c33: 0x3000, + 0x0c38: 0x0076, 0x0c39: 0x0076, + // Block 0x31, offset 0xc40 + 0x0c48: 0x007a, 0x0c49: 0x007a, 0x0c4a: 0x007a, 0x0c4b: 0x007a, + 0x0c5c: 0x3000, 0x0c5d: 0x3000, + // Block 0x32, offset 0xc80 + 0x0c8c: 0x3000, + 0x0c98: 0x00dc, 0x0c99: 0x00dc, + 0x0cb5: 0x00dc, + 0x0cb7: 0x00dc, 0x0cb9: 0x00d8, + // Block 0x33, offset 0xcc0 + 0x0cc3: 0x3300, + 0x0ccd: 0x3300, + 0x0cd2: 0x3300, 0x0cd7: 0x3300, + 0x0cdc: 0x3300, + 0x0ce9: 0x3300, + 0x0cf1: 0x0081, 0x0cf2: 0x0082, 0x0cf3: 0x3300, 0x0cf4: 0x0084, 0x0cf5: 0x3300, + 0x0cf6: 0x3300, 0x0cf7: 0x3000, 0x0cf8: 0x3300, 0x0cf9: 0x3000, 0x0cfa: 0x0082, 0x0cfb: 0x0082, + 0x0cfc: 0x0082, 0x0cfd: 0x0082, + // Block 0x34, offset 0xd00 + 0x0d00: 0x0082, 0x0d01: 0x3300, 0x0d02: 0x00e6, 0x0d03: 0x00e6, 0x0d04: 0x0009, + 0x0d06: 0x00e6, 0x0d07: 0x00e6, + 0x0d13: 0x3300, + 0x0d1d: 0x3300, + 0x0d22: 0x3300, + 0x0d27: 0x3300, + 0x0d2c: 0x3300, + 0x0d39: 0x3300, + // Block 0x35, offset 0xd40 + 0x0d46: 0x00dc, + // Block 0x36, offset 0xd80 + 0x0da5: 0x8800, 0x0da6: 0x1100, + 0x0dae: 0x6600, + 0x0db7: 0x0007, 0x0db9: 0x0009, 0x0dba: 0x0009, + // Block 0x37, offset 0xdc0 + 0x0dcd: 0x00dc, + // Block 0x38, offset 0xe00 + 0x0e3c: 0x3000, + // Block 0x39, offset 0xe40 + 0x0e61: 0x6600, 0x0e62: 0x6600, 0x0e63: 0x6600, + 0x0e64: 0x6600, 0x0e65: 0x6600, 0x0e66: 0x6600, 0x0e67: 0x6600, 0x0e68: 0x6600, 0x0e69: 0x6600, + 0x0e6a: 0x6600, 0x0e6b: 0x6600, 0x0e6c: 0x6600, 0x0e6d: 0x6600, 0x0e6e: 0x6600, 0x0e6f: 0x6600, + 0x0e70: 0x6600, 0x0e71: 0x6600, 0x0e72: 0x6600, 0x0e73: 0x6600, 0x0e74: 0x6600, 0x0e75: 0x6600, + // Block 0x3a, offset 0xe80 + 0x0ea8: 0x6600, 0x0ea9: 0x6600, + 0x0eaa: 0x6600, 0x0eab: 0x6600, 0x0eac: 0x6600, 0x0ead: 0x6600, 0x0eae: 0x6600, 0x0eaf: 0x6600, + 0x0eb0: 0x6600, 0x0eb1: 0x6600, 0x0eb2: 0x6600, 0x0eb3: 0x6600, 0x0eb4: 0x6600, 0x0eb5: 0x6600, + 0x0eb6: 0x6600, 0x0eb7: 0x6600, 0x0eb8: 0x6600, 0x0eb9: 0x6600, 0x0eba: 0x6600, 0x0ebb: 0x6600, + 0x0ebc: 0x6600, 0x0ebd: 0x6600, 0x0ebe: 0x6600, 0x0ebf: 0x6600, + // Block 0x3b, offset 0xec0 + 0x0ec0: 0x6600, 0x0ec1: 0x6600, 0x0ec2: 0x6600, + // Block 0x3c, offset 0xf00 + 0x0f1d: 0x00e6, + 0x0f1e: 0x00e6, 0x0f1f: 0x00e6, + // Block 0x3d, offset 0xf40 + 0x0f54: 0x0009, + 0x0f74: 0x0009, + // Block 0x3e, offset 0xf80 + 0x0f92: 0x0009, + 0x0f9d: 0x00e6, + // Block 0x3f, offset 0xfc0 + 0x0fe9: 0x00e4, + // Block 0x40, offset 0x1000 + 0x1039: 0x00de, 0x103a: 0x00e6, 0x103b: 0x00dc, + // Block 0x41, offset 0x1040 + 0x1057: 0x00e6, + 0x1058: 0x00dc, + // Block 0x42, offset 0x1080 + 0x10a0: 0x0009, + 0x10b5: 0x00e6, + 0x10b6: 0x00e6, 0x10b7: 0x00e6, 0x10b8: 0x00e6, 0x10b9: 0x00e6, 0x10ba: 0x00e6, 0x10bb: 0x00e6, + 0x10bc: 0x00e6, 0x10bf: 0x00dc, + // Block 0x43, offset 0x10c0 + 0x10c5: 0x8800, + 0x10c6: 0x1100, 0x10c7: 0x8800, 0x10c8: 0x1100, 0x10c9: 0x8800, 0x10ca: 0x1100, 0x10cb: 0x8800, + 0x10cc: 0x1100, 0x10cd: 0x8800, 0x10ce: 0x1100, 0x10d1: 0x8800, + 0x10d2: 0x1100, + 0x10f4: 0x0007, 0x10f5: 0x6600, + 0x10fa: 0x8800, 0x10fb: 0x1100, + 0x10fc: 0x8800, 0x10fd: 0x1100, 0x10fe: 0x8800, 0x10ff: 0x8800, + // Block 0x44, offset 0x1100 + 0x1100: 0x1100, 0x1101: 0x1100, 0x1102: 0x8800, 0x1103: 0x1100, 0x1104: 0x0009, + 0x112b: 0x00e6, 0x112c: 0x00dc, 0x112d: 0x00e6, 0x112e: 0x00e6, 0x112f: 0x00e6, + 0x1130: 0x00e6, 0x1131: 0x00e6, 0x1132: 0x00e6, 0x1133: 0x00e6, + // Block 0x45, offset 0x1140 + 0x116a: 0x0009, + // Block 0x46, offset 0x1180 + 0x11a6: 0x0007, + 0x11b2: 0x0009, 0x11b3: 0x0009, + // Block 0x47, offset 0x11c0 + 0x11f7: 0x0007, + // Block 0x48, offset 0x1200 + 0x1210: 0x00e6, 0x1211: 0x00e6, + 0x1212: 0x00e6, 0x1214: 0x0001, 0x1215: 0x00dc, 0x1216: 0x00dc, 0x1217: 0x00dc, + 0x1218: 0x00dc, 0x1219: 0x00dc, 0x121a: 0x00e6, 0x121b: 0x00e6, 0x121c: 0x00dc, 0x121d: 0x00dc, + 0x121e: 0x00dc, 0x121f: 0x00dc, 0x1220: 0x00e6, 0x1222: 0x0001, 0x1223: 0x0001, + 0x1224: 0x0001, 0x1225: 0x0001, 0x1226: 0x0001, 0x1227: 0x0001, 0x1228: 0x0001, + 0x122d: 0x00dc, + // Block 0x49, offset 0x1240 + 0x126c: 0x3000, 0x126d: 0x3000, 0x126e: 0x3000, + 0x1270: 0x3000, 0x1271: 0x3000, 0x1272: 0x3000, 0x1273: 0x3000, 0x1274: 0x3000, 0x1275: 0x3000, + 0x1276: 0x3000, 0x1277: 0x3000, 0x1278: 0x3000, 0x1279: 0x3000, 0x127a: 0x3000, + 0x127c: 0x3000, 0x127d: 0x3000, 0x127e: 0x3000, 0x127f: 0x3000, + // Block 0x4a, offset 0x1280 + 0x1280: 0x3000, 0x1281: 0x3000, 0x1282: 0x3000, 0x1283: 0x3000, 0x1284: 0x3000, 0x1285: 0x3000, + 0x1286: 0x3000, 0x1287: 0x3000, 0x1288: 0x3000, 0x1289: 0x3000, 0x128a: 0x3000, 0x128b: 0x3000, + 0x128c: 0x3000, 0x128d: 0x3000, 0x128f: 0x3000, 0x1290: 0x3000, 0x1291: 0x3000, + 0x1292: 0x3000, 0x1293: 0x3000, 0x1294: 0x3000, 0x1295: 0x3000, 0x1296: 0x3000, 0x1297: 0x3000, + 0x1298: 0x3000, 0x1299: 0x3000, 0x129a: 0x3000, 0x129b: 0x3000, 0x129c: 0x3000, 0x129d: 0x3000, + 0x129e: 0x3000, 0x129f: 0x3000, 0x12a0: 0x3000, 0x12a1: 0x3000, 0x12a2: 0x3000, 0x12a3: 0x3000, + 0x12a4: 0x3000, 0x12a5: 0x3000, 0x12a6: 0x3000, 0x12a7: 0x3000, 0x12a8: 0x3000, 0x12a9: 0x3000, + 0x12aa: 0x3000, + 0x12b8: 0x3000, + // Block 0x4b, offset 0x12c0 + 0x12db: 0x3000, 0x12dc: 0x3000, 0x12dd: 0x3000, + 0x12de: 0x3000, 0x12df: 0x3000, 0x12e0: 0x3000, 0x12e1: 0x3000, 0x12e2: 0x3000, 0x12e3: 0x3000, + 0x12e4: 0x3000, 0x12e5: 0x3000, 0x12e6: 0x3000, 0x12e7: 0x3000, 0x12e8: 0x3000, 0x12e9: 0x3000, + 0x12ea: 0x3000, 0x12eb: 0x3000, 0x12ec: 0x3000, 0x12ed: 0x3000, 0x12ee: 0x3000, 0x12ef: 0x3000, + 0x12f0: 0x3000, 0x12f1: 0x3000, 0x12f2: 0x3000, 0x12f3: 0x3000, 0x12f4: 0x3000, 0x12f5: 0x3000, + 0x12f6: 0x3000, 0x12f7: 0x3000, 0x12f8: 0x3000, 0x12f9: 0x3000, 0x12fa: 0x3000, 0x12fb: 0x3000, + 0x12fc: 0x3000, 0x12fd: 0x3000, 0x12fe: 0x3000, 0x12ff: 0x3000, + // Block 0x4c, offset 0x1300 + 0x1300: 0x00e6, 0x1301: 0x00e6, 0x1302: 0x00dc, 0x1303: 0x00e6, 0x1304: 0x00e6, 0x1305: 0x00e6, + 0x1306: 0x00e6, 0x1307: 0x00e6, 0x1308: 0x00e6, 0x1309: 0x00e6, 0x130a: 0x00dc, 0x130b: 0x00e6, + 0x130c: 0x00e6, 0x130d: 0x00ea, 0x130e: 0x00d6, 0x130f: 0x00dc, 0x1310: 0x00ca, 0x1311: 0x00e6, + 0x1312: 0x00e6, 0x1313: 0x00e6, 0x1314: 0x00e6, 0x1315: 0x00e6, 0x1316: 0x00e6, 0x1317: 0x00e6, + 0x1318: 0x00e6, 0x1319: 0x00e6, 0x131a: 0x00e6, 0x131b: 0x00e6, 0x131c: 0x00e6, 0x131d: 0x00e6, + 0x131e: 0x00e6, 0x131f: 0x00e6, 0x1320: 0x00e6, 0x1321: 0x00e6, 0x1322: 0x00e6, 0x1323: 0x00e6, + 0x1324: 0x00e6, 0x1325: 0x00e6, 0x1326: 0x00e6, + 0x133c: 0x00e9, 0x133d: 0x00dc, 0x133e: 0x00e6, 0x133f: 0x00dc, + // Block 0x4d, offset 0x1340 + 0x1340: 0x1100, 0x1341: 0x1100, 0x1342: 0x1100, 0x1343: 0x1100, 0x1344: 0x1100, 0x1345: 0x1100, + 0x1346: 0x1100, 0x1347: 0x1100, 0x1348: 0x1100, 0x1349: 0x1100, 0x134a: 0x1100, 0x134b: 0x1100, + 0x134c: 0x1100, 0x134d: 0x1100, 0x134e: 0x1100, 0x134f: 0x1100, 0x1350: 0x1100, 0x1351: 0x1100, + 0x1352: 0x1100, 0x1353: 0x1100, 0x1354: 0x1100, 0x1355: 0x1100, 0x1356: 0x1100, 0x1357: 0x1100, + 0x1358: 0x1100, 0x1359: 0x1100, 0x135a: 0x1100, 0x135b: 0x1100, 0x135c: 0x1100, 0x135d: 0x1100, + 0x135e: 0x1100, 0x135f: 0x1100, 0x1360: 0x1100, 0x1361: 0x1100, 0x1362: 0x1100, 0x1363: 0x1100, + 0x1364: 0x1100, 0x1365: 0x1100, 0x1366: 0x1100, 0x1367: 0x1100, 0x1368: 0x1100, 0x1369: 0x1100, + 0x136a: 0x1100, 0x136b: 0x1100, 0x136c: 0x1100, 0x136d: 0x1100, 0x136e: 0x1100, 0x136f: 0x1100, + 0x1370: 0x1100, 0x1371: 0x1100, 0x1372: 0x1100, 0x1373: 0x1100, 0x1374: 0x1100, 0x1375: 0x1100, + 0x1376: 0x9900, 0x1377: 0x9900, 0x1378: 0x1100, 0x1379: 0x1100, 0x137a: 0x1100, 0x137b: 0x1100, + 0x137c: 0x1100, 0x137d: 0x1100, 0x137e: 0x1100, 0x137f: 0x1100, + // Block 0x4e, offset 0x1380 + 0x1380: 0x1100, 0x1381: 0x1100, 0x1382: 0x1100, 0x1383: 0x1100, 0x1384: 0x1100, 0x1385: 0x1100, + 0x1386: 0x1100, 0x1387: 0x1100, 0x1388: 0x1100, 0x1389: 0x1100, 0x138a: 0x1100, 0x138b: 0x1100, + 0x138c: 0x1100, 0x138d: 0x1100, 0x138e: 0x1100, 0x138f: 0x1100, 0x1390: 0x1100, 0x1391: 0x1100, + 0x1392: 0x1100, 0x1393: 0x1100, 0x1394: 0x1100, 0x1395: 0x1100, 0x1396: 0x1100, 0x1397: 0x1100, + 0x1398: 0x1100, 0x1399: 0x1100, 0x139a: 0x9900, 0x139b: 0x9900, 0x139c: 0x1100, 0x139d: 0x1100, + 0x139e: 0x1100, 0x139f: 0x1100, 0x13a0: 0x1100, 0x13a1: 0x1100, 0x13a2: 0x9900, 0x13a3: 0x9900, + 0x13a4: 0x1100, 0x13a5: 0x1100, 0x13a6: 0x1100, 0x13a7: 0x1100, 0x13a8: 0x1100, 0x13a9: 0x1100, + 0x13aa: 0x1100, 0x13ab: 0x1100, 0x13ac: 0x1100, 0x13ad: 0x1100, 0x13ae: 0x1100, 0x13af: 0x1100, + 0x13b0: 0x1100, 0x13b1: 0x1100, 0x13b2: 0x1100, 0x13b3: 0x1100, 0x13b4: 0x1100, 0x13b5: 0x1100, + 0x13b6: 0x1100, 0x13b7: 0x1100, 0x13b8: 0x1100, 0x13b9: 0x1100, 0x13ba: 0x1100, 0x13bb: 0x1100, + 0x13bc: 0x1100, 0x13bd: 0x1100, 0x13be: 0x1100, 0x13bf: 0x1100, + // Block 0x4f, offset 0x13c0 + 0x13c0: 0x1100, 0x13c1: 0x1100, 0x13c2: 0x1100, 0x13c3: 0x1100, 0x13c4: 0x1100, 0x13c5: 0x1100, + 0x13c6: 0x1100, 0x13c7: 0x1100, 0x13c8: 0x1100, 0x13c9: 0x1100, 0x13ca: 0x1100, 0x13cb: 0x1100, + 0x13cc: 0x1100, 0x13cd: 0x1100, 0x13ce: 0x1100, 0x13cf: 0x1100, 0x13d0: 0x1100, 0x13d1: 0x1100, + 0x13d2: 0x1100, 0x13d3: 0x1100, 0x13d4: 0x1100, 0x13d5: 0x1100, 0x13d6: 0x1100, 0x13d7: 0x1100, + 0x13d8: 0x1100, 0x13d9: 0x1100, 0x13da: 0x3000, 0x13db: 0x3100, + 0x13e0: 0x9900, 0x13e1: 0x9900, 0x13e2: 0x1100, 0x13e3: 0x1100, + 0x13e4: 0x1100, 0x13e5: 0x1100, 0x13e6: 0x1100, 0x13e7: 0x1100, 0x13e8: 0x1100, 0x13e9: 0x1100, + 0x13ea: 0x1100, 0x13eb: 0x1100, 0x13ec: 0x1100, 0x13ed: 0x1100, 0x13ee: 0x1100, 0x13ef: 0x1100, + 0x13f0: 0x1100, 0x13f1: 0x1100, 0x13f2: 0x1100, 0x13f3: 0x1100, 0x13f4: 0x1100, 0x13f5: 0x1100, + 0x13f6: 0x1100, 0x13f7: 0x1100, 0x13f8: 0x9900, 0x13f9: 0x9900, 0x13fa: 0x1100, 0x13fb: 0x1100, + 0x13fc: 0x1100, 0x13fd: 0x1100, 0x13fe: 0x1100, 0x13ff: 0x1100, + // Block 0x50, offset 0x1400 + 0x1400: 0x1100, 0x1401: 0x1100, 0x1402: 0x1100, 0x1403: 0x1100, 0x1404: 0x1100, 0x1405: 0x1100, + 0x1406: 0x1100, 0x1407: 0x1100, 0x1408: 0x1100, 0x1409: 0x1100, 0x140a: 0x1100, 0x140b: 0x1100, + 0x140c: 0x9900, 0x140d: 0x9900, 0x140e: 0x1100, 0x140f: 0x1100, 0x1410: 0x1100, 0x1411: 0x1100, + 0x1412: 0x1100, 0x1413: 0x1100, 0x1414: 0x1100, 0x1415: 0x1100, 0x1416: 0x1100, 0x1417: 0x1100, + 0x1418: 0x1100, 0x1419: 0x1100, 0x141a: 0x1100, 0x141b: 0x1100, 0x141c: 0x1100, 0x141d: 0x1100, + 0x141e: 0x1100, 0x141f: 0x1100, 0x1420: 0x1100, 0x1421: 0x1100, 0x1422: 0x1100, 0x1423: 0x1100, + 0x1424: 0x1100, 0x1425: 0x1100, 0x1426: 0x1100, 0x1427: 0x1100, 0x1428: 0x1100, 0x1429: 0x1100, + 0x142a: 0x1100, 0x142b: 0x1100, 0x142c: 0x1100, 0x142d: 0x1100, 0x142e: 0x1100, 0x142f: 0x1100, + 0x1430: 0x1100, 0x1431: 0x1100, 0x1432: 0x1100, 0x1433: 0x1100, 0x1434: 0x1100, 0x1435: 0x1100, + 0x1436: 0x1100, 0x1437: 0x1100, 0x1438: 0x1100, 0x1439: 0x1100, + // Block 0x51, offset 0x1440 + 0x1440: 0x9900, 0x1441: 0x9900, 0x1442: 0x9900, 0x1443: 0x9900, 0x1444: 0x9900, 0x1445: 0x9900, + 0x1446: 0x9900, 0x1447: 0x9900, 0x1448: 0x9900, 0x1449: 0x9900, 0x144a: 0x9900, 0x144b: 0x9900, + 0x144c: 0x9900, 0x144d: 0x9900, 0x144e: 0x9900, 0x144f: 0x9900, 0x1450: 0x9900, 0x1451: 0x9900, + 0x1452: 0x1100, 0x1453: 0x1100, 0x1454: 0x1100, 0x1455: 0x1100, + 0x1458: 0x9900, 0x1459: 0x9900, 0x145a: 0x1100, 0x145b: 0x1100, 0x145c: 0x1100, 0x145d: 0x1100, + 0x1460: 0x9900, 0x1461: 0x9900, 0x1462: 0x9900, 0x1463: 0x9900, + 0x1464: 0x9900, 0x1465: 0x9900, 0x1466: 0x9900, 0x1467: 0x9900, 0x1468: 0x9900, 0x1469: 0x9900, + 0x146a: 0x9900, 0x146b: 0x9900, 0x146c: 0x9900, 0x146d: 0x9900, 0x146e: 0x9900, 0x146f: 0x9900, + 0x1470: 0x9900, 0x1471: 0x9900, 0x1472: 0x1100, 0x1473: 0x1100, 0x1474: 0x1100, 0x1475: 0x1100, + 0x1476: 0x1100, 0x1477: 0x1100, 0x1478: 0x9900, 0x1479: 0x9900, 0x147a: 0x1100, 0x147b: 0x1100, + 0x147c: 0x1100, 0x147d: 0x1100, 0x147e: 0x1100, 0x147f: 0x1100, + // Block 0x52, offset 0x1480 + 0x1480: 0x9900, 0x1481: 0x9900, 0x1482: 0x1100, 0x1483: 0x1100, 0x1484: 0x1100, 0x1485: 0x1100, + 0x1488: 0x9900, 0x1489: 0x9900, 0x148a: 0x1100, 0x148b: 0x1100, + 0x148c: 0x1100, 0x148d: 0x1100, 0x1490: 0x9900, 0x1491: 0x9900, + 0x1492: 0x1100, 0x1493: 0x1100, 0x1494: 0x1100, 0x1495: 0x1100, 0x1496: 0x1100, 0x1497: 0x1100, + 0x1499: 0x9900, 0x149b: 0x1100, 0x149d: 0x1100, + 0x149f: 0x1100, 0x14a0: 0x9900, 0x14a1: 0x9900, 0x14a2: 0x9900, 0x14a3: 0x9900, + 0x14a4: 0x9900, 0x14a5: 0x9900, 0x14a6: 0x9900, 0x14a7: 0x9900, 0x14a8: 0x9900, 0x14a9: 0x9900, + 0x14aa: 0x9900, 0x14ab: 0x9900, 0x14ac: 0x9900, 0x14ad: 0x9900, 0x14ae: 0x9900, 0x14af: 0x9900, + 0x14b0: 0x9900, 0x14b1: 0x3300, 0x14b2: 0x1100, 0x14b3: 0x3300, 0x14b4: 0x9900, 0x14b5: 0x3300, + 0x14b6: 0x1100, 0x14b7: 0x3300, 0x14b8: 0x1100, 0x14b9: 0x3300, 0x14ba: 0x1100, 0x14bb: 0x3300, + 0x14bc: 0x9900, 0x14bd: 0x3300, + // Block 0x53, offset 0x14c0 + 0x14c0: 0x1100, 0x14c1: 0x1100, 0x14c2: 0x1100, 0x14c3: 0x1100, 0x14c4: 0x1100, 0x14c5: 0x1100, + 0x14c6: 0x1100, 0x14c7: 0x1100, 0x14c8: 0x1100, 0x14c9: 0x1100, 0x14ca: 0x1100, 0x14cb: 0x1100, + 0x14cc: 0x1100, 0x14cd: 0x1100, 0x14ce: 0x1100, 0x14cf: 0x1100, 0x14d0: 0x1100, 0x14d1: 0x1100, + 0x14d2: 0x1100, 0x14d3: 0x1100, 0x14d4: 0x1100, 0x14d5: 0x1100, 0x14d6: 0x1100, 0x14d7: 0x1100, + 0x14d8: 0x1100, 0x14d9: 0x1100, 0x14da: 0x1100, 0x14db: 0x1100, 0x14dc: 0x1100, 0x14dd: 0x1100, + 0x14de: 0x1100, 0x14df: 0x1100, 0x14e0: 0x1100, 0x14e1: 0x1100, 0x14e2: 0x1100, 0x14e3: 0x1100, + 0x14e4: 0x1100, 0x14e5: 0x1100, 0x14e6: 0x1100, 0x14e7: 0x1100, 0x14e8: 0x1100, 0x14e9: 0x1100, + 0x14ea: 0x1100, 0x14eb: 0x1100, 0x14ec: 0x1100, 0x14ed: 0x1100, 0x14ee: 0x1100, 0x14ef: 0x1100, + 0x14f0: 0x1100, 0x14f1: 0x1100, 0x14f2: 0x1100, 0x14f3: 0x1100, 0x14f4: 0x1100, + 0x14f6: 0x9900, 0x14f7: 0x1100, 0x14f8: 0x1100, 0x14f9: 0x1100, 0x14fa: 0x1100, 0x14fb: 0x3300, + 0x14fc: 0x1100, 0x14fd: 0x3000, 0x14fe: 0x3300, 0x14ff: 0x3800, + // Block 0x54, offset 0x1500 + 0x1500: 0x3000, 0x1501: 0x3100, 0x1502: 0x1100, 0x1503: 0x1100, 0x1504: 0x1100, + 0x1506: 0x9900, 0x1507: 0x1100, 0x1508: 0x1100, 0x1509: 0x3300, 0x150a: 0x1100, 0x150b: 0x3300, + 0x150c: 0x1100, 0x150d: 0x3100, 0x150e: 0x3100, 0x150f: 0x3100, 0x1510: 0x1100, 0x1511: 0x1100, + 0x1512: 0x1100, 0x1513: 0x3300, 0x1516: 0x1100, 0x1517: 0x1100, + 0x1518: 0x1100, 0x1519: 0x1100, 0x151a: 0x1100, 0x151b: 0x3300, 0x151d: 0x3100, + 0x151e: 0x3100, 0x151f: 0x3100, 0x1520: 0x1100, 0x1521: 0x1100, 0x1522: 0x1100, 0x1523: 0x3300, + 0x1524: 0x1100, 0x1525: 0x1100, 0x1526: 0x1100, 0x1527: 0x1100, 0x1528: 0x1100, 0x1529: 0x1100, + 0x152a: 0x1100, 0x152b: 0x3300, 0x152c: 0x1100, 0x152d: 0x3100, 0x152e: 0x3300, 0x152f: 0x3300, + 0x1532: 0x1100, 0x1533: 0x1100, 0x1534: 0x1100, + 0x1536: 0x9900, 0x1537: 0x1100, 0x1538: 0x1100, 0x1539: 0x3300, 0x153a: 0x1100, 0x153b: 0x3300, + 0x153c: 0x1100, 0x153d: 0x3300, 0x153e: 0x3800, + // Block 0x55, offset 0x1540 + 0x1540: 0x3300, 0x1541: 0x3300, 0x1542: 0x3000, 0x1543: 0x3000, 0x1544: 0x3000, 0x1545: 0x3000, + 0x1546: 0x3000, 0x1547: 0x3000, 0x1548: 0x3000, 0x1549: 0x3000, 0x154a: 0x3000, + 0x1551: 0x3000, + 0x1557: 0x3000, + 0x1564: 0x3000, 0x1565: 0x3000, 0x1566: 0x3000, + 0x156f: 0x3000, + 0x1573: 0x3000, 0x1574: 0x3000, + 0x1576: 0x3000, 0x1577: 0x3000, + 0x157c: 0x3000, 0x157e: 0x3000, + // Block 0x56, offset 0x1580 + 0x1587: 0x3000, 0x1588: 0x3000, 0x1589: 0x3000, + 0x1597: 0x3000, + 0x159f: 0x3000, + 0x15b0: 0x3000, 0x15b1: 0x3000, 0x15b4: 0x3000, 0x15b5: 0x3000, + 0x15b6: 0x3000, 0x15b7: 0x3000, 0x15b8: 0x3000, 0x15b9: 0x3000, 0x15ba: 0x3000, 0x15bb: 0x3000, + 0x15bc: 0x3000, 0x15bd: 0x3000, 0x15be: 0x3000, 0x15bf: 0x3000, + // Block 0x57, offset 0x15c0 + 0x15c0: 0x3000, 0x15c1: 0x3000, 0x15c2: 0x3000, 0x15c3: 0x3000, 0x15c4: 0x3000, 0x15c5: 0x3000, + 0x15c6: 0x3000, 0x15c7: 0x3000, 0x15c8: 0x3000, 0x15c9: 0x3000, 0x15ca: 0x3000, 0x15cb: 0x3000, + 0x15cc: 0x3000, 0x15cd: 0x3000, 0x15ce: 0x3000, 0x15d0: 0x3000, 0x15d1: 0x3000, + 0x15d2: 0x3000, 0x15d3: 0x3000, 0x15d4: 0x3000, 0x15d5: 0x3000, 0x15d6: 0x3000, 0x15d7: 0x3000, + 0x15d8: 0x3000, 0x15d9: 0x3000, 0x15da: 0x3000, 0x15db: 0x3000, 0x15dc: 0x3000, + 0x15e8: 0x3000, + // Block 0x58, offset 0x1600 + 0x1610: 0x00e6, 0x1611: 0x00e6, + 0x1612: 0x0001, 0x1613: 0x0001, 0x1614: 0x00e6, 0x1615: 0x00e6, 0x1616: 0x00e6, 0x1617: 0x00e6, + 0x1618: 0x0001, 0x1619: 0x0001, 0x161a: 0x0001, 0x161b: 0x00e6, 0x161c: 0x00e6, + 0x1621: 0x00e6, + 0x1625: 0x0001, 0x1626: 0x0001, 0x1627: 0x00e6, 0x1628: 0x00dc, 0x1629: 0x00e6, + 0x162a: 0x0001, 0x162b: 0x0001, 0x162c: 0x00dc, 0x162d: 0x00dc, 0x162e: 0x00dc, 0x162f: 0x00dc, + 0x1630: 0x00e6, + // Block 0x59, offset 0x1640 + 0x1640: 0x3000, 0x1641: 0x3000, 0x1642: 0x3000, 0x1643: 0x3000, 0x1645: 0x3000, + 0x1646: 0x3000, 0x1647: 0x3000, 0x1649: 0x3000, 0x164a: 0x3000, 0x164b: 0x3000, + 0x164c: 0x3000, 0x164d: 0x3000, 0x164e: 0x3000, 0x164f: 0x3000, 0x1650: 0x3000, 0x1651: 0x3000, + 0x1652: 0x3000, 0x1653: 0x3000, 0x1655: 0x3000, 0x1656: 0x3000, + 0x1659: 0x3000, 0x165a: 0x3000, 0x165b: 0x3000, 0x165c: 0x3000, 0x165d: 0x3000, + 0x1660: 0x3000, 0x1661: 0x3000, 0x1662: 0x3000, + 0x1664: 0x3000, 0x1666: 0x3300, 0x1668: 0x3000, + 0x166a: 0x3300, 0x166b: 0x3300, 0x166c: 0x3000, 0x166d: 0x3000, 0x166f: 0x3000, + 0x1670: 0x3000, 0x1671: 0x3000, 0x1673: 0x3000, 0x1674: 0x3000, 0x1675: 0x3000, + 0x1676: 0x3000, 0x1677: 0x3000, 0x1678: 0x3000, 0x1679: 0x3000, 0x167b: 0x3000, + 0x167c: 0x3000, 0x167d: 0x3000, 0x167e: 0x3000, 0x167f: 0x3000, + // Block 0x5a, offset 0x1680 + 0x1680: 0x3000, 0x1685: 0x3000, + 0x1686: 0x3000, 0x1687: 0x3000, 0x1688: 0x3000, 0x1689: 0x3000, + 0x1690: 0x3000, 0x1691: 0x3000, + 0x1692: 0x3000, 0x1693: 0x3000, 0x1694: 0x3000, 0x1695: 0x3000, 0x1696: 0x3000, 0x1697: 0x3000, + 0x1698: 0x3000, 0x1699: 0x3000, 0x169a: 0x3000, 0x169b: 0x3000, 0x169c: 0x3000, 0x169d: 0x3000, + 0x169e: 0x3000, 0x169f: 0x3000, 0x16a0: 0x3000, 0x16a1: 0x3000, 0x16a2: 0x3000, 0x16a3: 0x3000, + 0x16a4: 0x3000, 0x16a5: 0x3000, 0x16a6: 0x3000, 0x16a7: 0x3000, 0x16a8: 0x3000, 0x16a9: 0x3000, + 0x16aa: 0x3000, 0x16ab: 0x3000, 0x16ac: 0x3000, 0x16ad: 0x3000, 0x16ae: 0x3000, 0x16af: 0x3000, + 0x16b0: 0x3000, 0x16b1: 0x3000, 0x16b2: 0x3000, 0x16b3: 0x3000, 0x16b4: 0x3000, 0x16b5: 0x3000, + 0x16b6: 0x3000, 0x16b7: 0x3000, 0x16b8: 0x3000, 0x16b9: 0x3000, 0x16ba: 0x3000, 0x16bb: 0x3000, + 0x16bc: 0x3000, 0x16bd: 0x3000, 0x16be: 0x3000, 0x16bf: 0x3000, + // Block 0x5b, offset 0x16c0 + 0x16c9: 0x3000, + 0x16d0: 0x8800, + 0x16d2: 0x8800, 0x16d4: 0x8800, + 0x16da: 0x1100, 0x16db: 0x1100, + 0x16ee: 0x1100, + // Block 0x5c, offset 0x1700 + 0x170d: 0x1100, 0x170e: 0x1100, 0x170f: 0x1100, 0x1710: 0x8800, + 0x1712: 0x8800, 0x1714: 0x8800, + // Block 0x5d, offset 0x1740 + 0x1743: 0x8800, 0x1744: 0x1100, + 0x1748: 0x8800, 0x1749: 0x1100, 0x174b: 0x8800, + 0x174c: 0x1100, + 0x1763: 0x8800, + 0x1764: 0x1100, 0x1765: 0x8800, 0x1766: 0x1100, + 0x176c: 0x3000, 0x176d: 0x3000, 0x176f: 0x3000, + 0x1770: 0x3000, + 0x177c: 0x8800, + // Block 0x5e, offset 0x1780 + 0x1781: 0x1100, 0x1783: 0x8800, 0x1784: 0x1100, 0x1785: 0x8800, + 0x1787: 0x1100, 0x1788: 0x8800, 0x1789: 0x1100, + 0x178d: 0x8800, + 0x17a0: 0x1100, 0x17a1: 0x8800, 0x17a2: 0x1100, + 0x17a4: 0x8800, 0x17a5: 0x8800, + 0x17ad: 0x1100, 0x17ae: 0x1100, 0x17af: 0x1100, + 0x17b0: 0x1100, 0x17b1: 0x1100, 0x17b2: 0x8800, 0x17b3: 0x8800, 0x17b4: 0x1100, 0x17b5: 0x1100, + 0x17b6: 0x8800, 0x17b7: 0x8800, 0x17b8: 0x1100, 0x17b9: 0x1100, 0x17ba: 0x8800, 0x17bb: 0x8800, + 0x17bc: 0x8800, 0x17bd: 0x8800, + // Block 0x5f, offset 0x17c0 + 0x17c0: 0x1100, 0x17c1: 0x1100, 0x17c2: 0x8800, 0x17c3: 0x8800, 0x17c4: 0x1100, 0x17c5: 0x1100, + 0x17c6: 0x8800, 0x17c7: 0x8800, 0x17c8: 0x1100, 0x17c9: 0x1100, + 0x17d1: 0x8800, + 0x17d2: 0x8800, + 0x17e2: 0x8800, + 0x17e8: 0x8800, 0x17e9: 0x8800, + 0x17eb: 0x8800, 0x17ec: 0x1100, 0x17ed: 0x1100, 0x17ee: 0x1100, 0x17ef: 0x1100, + 0x17f2: 0x8800, 0x17f3: 0x8800, 0x17f4: 0x8800, 0x17f5: 0x8800, + // Block 0x60, offset 0x1800 + 0x1820: 0x1100, 0x1821: 0x1100, 0x1822: 0x1100, 0x1823: 0x1100, + 0x182a: 0x1100, 0x182b: 0x1100, 0x182c: 0x1100, 0x182d: 0x1100, + // Block 0x61, offset 0x1840 + 0x1869: 0x3300, + 0x186a: 0x3300, + // Block 0x62, offset 0x1880 + 0x18a0: 0x3000, 0x18a1: 0x3000, 0x18a2: 0x3000, 0x18a3: 0x3000, + 0x18a4: 0x3000, 0x18a5: 0x3000, 0x18a6: 0x3000, 0x18a7: 0x3000, 0x18a8: 0x3000, 0x18a9: 0x3000, + 0x18aa: 0x3000, 0x18ab: 0x3000, 0x18ac: 0x3000, 0x18ad: 0x3000, 0x18ae: 0x3000, 0x18af: 0x3000, + 0x18b0: 0x3000, 0x18b1: 0x3000, 0x18b2: 0x3000, 0x18b3: 0x3000, 0x18b4: 0x3000, 0x18b5: 0x3000, + 0x18b6: 0x3000, 0x18b7: 0x3000, 0x18b8: 0x3000, 0x18b9: 0x3000, 0x18ba: 0x3000, 0x18bb: 0x3000, + 0x18bc: 0x3000, 0x18bd: 0x3000, 0x18be: 0x3000, 0x18bf: 0x3000, + // Block 0x63, offset 0x18c0 + 0x18c0: 0x3000, 0x18c1: 0x3000, 0x18c2: 0x3000, 0x18c3: 0x3000, 0x18c4: 0x3000, 0x18c5: 0x3000, + 0x18c6: 0x3000, 0x18c7: 0x3000, 0x18c8: 0x3000, 0x18c9: 0x3000, 0x18ca: 0x3000, 0x18cb: 0x3000, + 0x18cc: 0x3000, 0x18cd: 0x3000, 0x18ce: 0x3000, 0x18cf: 0x3000, 0x18d0: 0x3000, 0x18d1: 0x3000, + 0x18d2: 0x3000, 0x18d3: 0x3000, 0x18d4: 0x3000, 0x18d5: 0x3000, 0x18d6: 0x3000, 0x18d7: 0x3000, + 0x18d8: 0x3000, 0x18d9: 0x3000, 0x18da: 0x3000, 0x18db: 0x3000, 0x18dc: 0x3000, 0x18dd: 0x3000, + 0x18de: 0x3000, 0x18df: 0x3000, 0x18e0: 0x3000, 0x18e1: 0x3000, 0x18e2: 0x3000, 0x18e3: 0x3000, + 0x18e4: 0x3000, 0x18e5: 0x3000, 0x18e6: 0x3000, 0x18e7: 0x3000, 0x18e8: 0x3000, 0x18e9: 0x3000, + 0x18ea: 0x3000, 0x18eb: 0x3000, 0x18ec: 0x3000, 0x18ed: 0x3000, 0x18ee: 0x3000, 0x18ef: 0x3000, + 0x18f0: 0x3000, 0x18f1: 0x3000, 0x18f2: 0x3000, 0x18f3: 0x3000, 0x18f4: 0x3000, 0x18f5: 0x3000, + 0x18f6: 0x3000, 0x18f7: 0x3000, 0x18f8: 0x3000, 0x18f9: 0x3000, 0x18fa: 0x3000, 0x18fb: 0x3000, + 0x18fc: 0x3000, 0x18fd: 0x3000, 0x18fe: 0x3000, 0x18ff: 0x3000, + // Block 0x64, offset 0x1900 + 0x1900: 0x3000, 0x1901: 0x3000, 0x1902: 0x3000, 0x1903: 0x3000, 0x1904: 0x3000, 0x1905: 0x3000, + 0x1906: 0x3000, 0x1907: 0x3000, 0x1908: 0x3000, 0x1909: 0x3000, 0x190a: 0x3000, 0x190b: 0x3000, + 0x190c: 0x3000, 0x190d: 0x3000, 0x190e: 0x3000, 0x190f: 0x3000, 0x1910: 0x3000, 0x1911: 0x3000, + 0x1912: 0x3000, 0x1913: 0x3000, 0x1914: 0x3000, 0x1915: 0x3000, 0x1916: 0x3000, 0x1917: 0x3000, + 0x1918: 0x3000, 0x1919: 0x3000, 0x191a: 0x3000, 0x191b: 0x3000, 0x191c: 0x3000, 0x191d: 0x3000, + 0x191e: 0x3000, 0x191f: 0x3000, 0x1920: 0x3000, 0x1921: 0x3000, 0x1922: 0x3000, 0x1923: 0x3000, + 0x1924: 0x3000, 0x1925: 0x3000, 0x1926: 0x3000, 0x1927: 0x3000, 0x1928: 0x3000, 0x1929: 0x3000, + 0x192a: 0x3000, + // Block 0x65, offset 0x1940 + 0x194c: 0x3000, + // Block 0x66, offset 0x1980 + 0x19b4: 0x3000, 0x19b5: 0x3000, + 0x19b6: 0x3000, + // Block 0x67, offset 0x19c0 + 0x19dc: 0x3300, + // Block 0x68, offset 0x1a00 + 0x1a3c: 0x3000, 0x1a3d: 0x3000, + // Block 0x69, offset 0x1a40 + 0x1a6f: 0x00e6, + 0x1a70: 0x00e6, 0x1a71: 0x00e6, + // Block 0x6a, offset 0x1a80 + 0x1aaf: 0x3000, + 0x1abf: 0x0009, + // Block 0x6b, offset 0x1ac0 + 0x1ae0: 0x00e6, 0x1ae1: 0x00e6, 0x1ae2: 0x00e6, 0x1ae3: 0x00e6, + 0x1ae4: 0x00e6, 0x1ae5: 0x00e6, 0x1ae6: 0x00e6, 0x1ae7: 0x00e6, 0x1ae8: 0x00e6, 0x1ae9: 0x00e6, + 0x1aea: 0x00e6, 0x1aeb: 0x00e6, 0x1aec: 0x00e6, 0x1aed: 0x00e6, 0x1aee: 0x00e6, 0x1aef: 0x00e6, + 0x1af0: 0x00e6, 0x1af1: 0x00e6, 0x1af2: 0x00e6, 0x1af3: 0x00e6, 0x1af4: 0x00e6, 0x1af5: 0x00e6, + 0x1af6: 0x00e6, 0x1af7: 0x00e6, 0x1af8: 0x00e6, 0x1af9: 0x00e6, 0x1afa: 0x00e6, 0x1afb: 0x00e6, + 0x1afc: 0x00e6, 0x1afd: 0x00e6, 0x1afe: 0x00e6, 0x1aff: 0x00e6, + // Block 0x6c, offset 0x1b00 + 0x1b1f: 0x3000, + // Block 0x6d, offset 0x1b40 + 0x1b73: 0x3000, + // Block 0x6e, offset 0x1b80 + 0x1b80: 0x3000, 0x1b81: 0x3000, 0x1b82: 0x3000, 0x1b83: 0x3000, 0x1b84: 0x3000, 0x1b85: 0x3000, + 0x1b86: 0x3000, 0x1b87: 0x3000, 0x1b88: 0x3000, 0x1b89: 0x3000, 0x1b8a: 0x3000, 0x1b8b: 0x3000, + 0x1b8c: 0x3000, 0x1b8d: 0x3000, 0x1b8e: 0x3000, 0x1b8f: 0x3000, 0x1b90: 0x3000, 0x1b91: 0x3000, + 0x1b92: 0x3000, 0x1b93: 0x3000, 0x1b94: 0x3000, 0x1b95: 0x3000, + // Block 0x6f, offset 0x1bc0 + 0x1bc0: 0x3000, + 0x1bea: 0x00da, 0x1beb: 0x00e4, 0x1bec: 0x00e8, 0x1bed: 0x00de, 0x1bee: 0x00e0, 0x1bef: 0x00e0, + 0x1bf6: 0x3000, 0x1bf8: 0x3000, 0x1bf9: 0x3000, 0x1bfa: 0x3000, + // Block 0x70, offset 0x1c00 + 0x1c06: 0x8800, 0x1c0b: 0x8800, + 0x1c0c: 0x1100, 0x1c0d: 0x8800, 0x1c0e: 0x1100, 0x1c0f: 0x8800, 0x1c10: 0x1100, 0x1c11: 0x8800, + 0x1c12: 0x1100, 0x1c13: 0x8800, 0x1c14: 0x1100, 0x1c15: 0x8800, 0x1c16: 0x1100, 0x1c17: 0x8800, + 0x1c18: 0x1100, 0x1c19: 0x8800, 0x1c1a: 0x1100, 0x1c1b: 0x8800, 0x1c1c: 0x1100, 0x1c1d: 0x8800, + 0x1c1e: 0x1100, 0x1c1f: 0x8800, 0x1c20: 0x1100, 0x1c21: 0x8800, 0x1c22: 0x1100, + 0x1c24: 0x8800, 0x1c25: 0x1100, 0x1c26: 0x8800, 0x1c27: 0x1100, 0x1c28: 0x8800, 0x1c29: 0x1100, + 0x1c2f: 0x8800, + 0x1c30: 0x1100, 0x1c31: 0x1100, 0x1c32: 0x8800, 0x1c33: 0x1100, 0x1c34: 0x1100, 0x1c35: 0x8800, + 0x1c36: 0x1100, 0x1c37: 0x1100, 0x1c38: 0x8800, 0x1c39: 0x1100, 0x1c3a: 0x1100, 0x1c3b: 0x8800, + 0x1c3c: 0x1100, 0x1c3d: 0x1100, + // Block 0x71, offset 0x1c40 + 0x1c54: 0x1100, + 0x1c59: 0x6608, 0x1c5a: 0x6608, 0x1c5b: 0x3000, 0x1c5c: 0x3000, 0x1c5d: 0x8800, + 0x1c5e: 0x1100, 0x1c5f: 0x3000, + 0x1c66: 0x8800, + 0x1c6b: 0x8800, 0x1c6c: 0x1100, 0x1c6d: 0x8800, 0x1c6e: 0x1100, 0x1c6f: 0x8800, + 0x1c70: 0x1100, 0x1c71: 0x8800, 0x1c72: 0x1100, 0x1c73: 0x8800, 0x1c74: 0x1100, 0x1c75: 0x8800, + 0x1c76: 0x1100, 0x1c77: 0x8800, 0x1c78: 0x1100, 0x1c79: 0x8800, 0x1c7a: 0x1100, 0x1c7b: 0x8800, + 0x1c7c: 0x1100, 0x1c7d: 0x8800, 0x1c7e: 0x1100, 0x1c7f: 0x8800, + // Block 0x72, offset 0x1c80 + 0x1c80: 0x1100, 0x1c81: 0x8800, 0x1c82: 0x1100, 0x1c84: 0x8800, 0x1c85: 0x1100, + 0x1c86: 0x8800, 0x1c87: 0x1100, 0x1c88: 0x8800, 0x1c89: 0x1100, + 0x1c8f: 0x8800, 0x1c90: 0x1100, 0x1c91: 0x1100, + 0x1c92: 0x8800, 0x1c93: 0x1100, 0x1c94: 0x1100, 0x1c95: 0x8800, 0x1c96: 0x1100, 0x1c97: 0x1100, + 0x1c98: 0x8800, 0x1c99: 0x1100, 0x1c9a: 0x1100, 0x1c9b: 0x8800, 0x1c9c: 0x1100, 0x1c9d: 0x1100, + 0x1caf: 0x8800, + 0x1cb0: 0x8800, 0x1cb1: 0x8800, 0x1cb2: 0x8800, 0x1cb4: 0x1100, + 0x1cb7: 0x1100, 0x1cb8: 0x1100, 0x1cb9: 0x1100, 0x1cba: 0x1100, + 0x1cbd: 0x8800, 0x1cbe: 0x1100, 0x1cbf: 0x3000, + // Block 0x73, offset 0x1cc0 + 0x1cf1: 0x3000, 0x1cf2: 0x3000, 0x1cf3: 0x3000, 0x1cf4: 0x3000, 0x1cf5: 0x3000, + 0x1cf6: 0x3000, 0x1cf7: 0x3000, 0x1cf8: 0x3000, 0x1cf9: 0x3000, 0x1cfa: 0x3000, 0x1cfb: 0x3000, + 0x1cfc: 0x3000, 0x1cfd: 0x3000, 0x1cfe: 0x3000, 0x1cff: 0x3000, + // Block 0x74, offset 0x1d00 + 0x1d00: 0x3000, 0x1d01: 0x3000, 0x1d02: 0x3000, 0x1d03: 0x3000, 0x1d04: 0x3000, 0x1d05: 0x3000, + 0x1d06: 0x3000, 0x1d07: 0x3000, 0x1d08: 0x3000, 0x1d09: 0x3000, 0x1d0a: 0x3000, 0x1d0b: 0x3000, + 0x1d0c: 0x3000, 0x1d0d: 0x3000, 0x1d0e: 0x3000, + 0x1d12: 0x3000, 0x1d13: 0x3000, 0x1d14: 0x3000, 0x1d15: 0x3000, 0x1d16: 0x3000, 0x1d17: 0x3000, + 0x1d18: 0x3000, 0x1d19: 0x3000, 0x1d1a: 0x3000, 0x1d1b: 0x3000, 0x1d1c: 0x3000, 0x1d1d: 0x3000, + 0x1d1e: 0x3000, 0x1d1f: 0x3000, + // Block 0x75, offset 0x1d40 + 0x1d40: 0x3000, 0x1d41: 0x3000, 0x1d42: 0x3000, 0x1d43: 0x3000, 0x1d44: 0x3000, 0x1d45: 0x3000, + 0x1d46: 0x3000, 0x1d47: 0x3000, 0x1d48: 0x3000, 0x1d49: 0x3000, 0x1d4a: 0x3000, 0x1d4b: 0x3000, + 0x1d4c: 0x3000, 0x1d4d: 0x3000, 0x1d4e: 0x3000, 0x1d4f: 0x3000, 0x1d50: 0x3000, 0x1d51: 0x3000, + 0x1d52: 0x3000, 0x1d53: 0x3000, 0x1d54: 0x3000, 0x1d55: 0x3000, 0x1d56: 0x3000, 0x1d57: 0x3000, + 0x1d58: 0x3000, 0x1d59: 0x3000, 0x1d5a: 0x3000, 0x1d5b: 0x3000, 0x1d5c: 0x3000, 0x1d5d: 0x3000, + 0x1d5e: 0x3000, 0x1d60: 0x3000, 0x1d61: 0x3000, 0x1d62: 0x3000, 0x1d63: 0x3000, + 0x1d64: 0x3000, 0x1d65: 0x3000, 0x1d66: 0x3000, 0x1d67: 0x3000, 0x1d68: 0x3000, 0x1d69: 0x3000, + 0x1d6a: 0x3000, 0x1d6b: 0x3000, 0x1d6c: 0x3000, 0x1d6d: 0x3000, 0x1d6e: 0x3000, 0x1d6f: 0x3000, + 0x1d70: 0x3000, 0x1d71: 0x3000, 0x1d72: 0x3000, 0x1d73: 0x3000, 0x1d74: 0x3000, 0x1d75: 0x3000, + 0x1d76: 0x3000, 0x1d77: 0x3000, 0x1d78: 0x3000, 0x1d79: 0x3000, 0x1d7a: 0x3000, 0x1d7b: 0x3000, + 0x1d7c: 0x3000, 0x1d7d: 0x3000, 0x1d7e: 0x3000, 0x1d7f: 0x3000, + // Block 0x76, offset 0x1d80 + 0x1d80: 0x3000, 0x1d81: 0x3000, 0x1d82: 0x3000, 0x1d83: 0x3000, 0x1d84: 0x3000, 0x1d85: 0x3000, + 0x1d86: 0x3000, 0x1d87: 0x3000, + 0x1d90: 0x3000, 0x1d91: 0x3000, + 0x1d92: 0x3000, 0x1d93: 0x3000, 0x1d94: 0x3000, 0x1d95: 0x3000, 0x1d96: 0x3000, 0x1d97: 0x3000, + 0x1d98: 0x3000, 0x1d99: 0x3000, 0x1d9a: 0x3000, 0x1d9b: 0x3000, 0x1d9c: 0x3000, 0x1d9d: 0x3000, + 0x1d9e: 0x3000, 0x1d9f: 0x3000, 0x1da0: 0x3000, 0x1da1: 0x3000, 0x1da2: 0x3000, 0x1da3: 0x3000, + 0x1da4: 0x3000, 0x1da5: 0x3000, 0x1da6: 0x3000, 0x1da7: 0x3000, 0x1da8: 0x3000, 0x1da9: 0x3000, + 0x1daa: 0x3000, 0x1dab: 0x3000, 0x1dac: 0x3000, 0x1dad: 0x3000, 0x1dae: 0x3000, 0x1daf: 0x3000, + 0x1db0: 0x3000, 0x1db1: 0x3000, 0x1db2: 0x3000, 0x1db3: 0x3000, 0x1db4: 0x3000, 0x1db5: 0x3000, + 0x1db6: 0x3000, 0x1db7: 0x3000, 0x1db8: 0x3000, 0x1db9: 0x3000, 0x1dba: 0x3000, 0x1dbb: 0x3000, + 0x1dbc: 0x3000, 0x1dbd: 0x3000, 0x1dbe: 0x3000, + // Block 0x77, offset 0x1dc0 + 0x1dc0: 0x3000, 0x1dc1: 0x3000, 0x1dc2: 0x3000, 0x1dc3: 0x3000, 0x1dc4: 0x3000, 0x1dc5: 0x3000, + 0x1dc6: 0x3000, 0x1dc7: 0x3000, 0x1dc8: 0x3000, 0x1dc9: 0x3000, 0x1dca: 0x3000, 0x1dcb: 0x3000, + 0x1dcc: 0x3000, 0x1dcd: 0x3000, 0x1dce: 0x3000, 0x1dcf: 0x3000, 0x1dd0: 0x3000, 0x1dd1: 0x3000, + 0x1dd2: 0x3000, 0x1dd3: 0x3000, 0x1dd4: 0x3000, 0x1dd5: 0x3000, 0x1dd6: 0x3000, 0x1dd7: 0x3000, + 0x1dd8: 0x3000, 0x1dd9: 0x3000, 0x1dda: 0x3000, 0x1ddb: 0x3000, 0x1ddc: 0x3000, 0x1ddd: 0x3000, + 0x1dde: 0x3000, 0x1ddf: 0x3000, 0x1de0: 0x3000, 0x1de1: 0x3000, 0x1de2: 0x3000, 0x1de3: 0x3000, + 0x1de4: 0x3000, 0x1de5: 0x3000, 0x1de6: 0x3000, 0x1de7: 0x3000, 0x1de8: 0x3000, 0x1de9: 0x3000, + 0x1dea: 0x3000, 0x1deb: 0x3000, 0x1dec: 0x3000, 0x1ded: 0x3000, 0x1dee: 0x3000, 0x1def: 0x3000, + 0x1df0: 0x3000, 0x1df1: 0x3000, 0x1df2: 0x3000, 0x1df3: 0x3000, 0x1df4: 0x3000, 0x1df5: 0x3000, + 0x1df6: 0x3000, 0x1df7: 0x3000, 0x1df8: 0x3000, 0x1df9: 0x3000, 0x1dfa: 0x3000, 0x1dfb: 0x3000, + 0x1dfc: 0x3000, 0x1dfd: 0x3000, 0x1dfe: 0x3000, + // Block 0x78, offset 0x1e00 + 0x1e2f: 0x00e6, + 0x1e3c: 0x00e6, 0x1e3d: 0x00e6, + // Block 0x79, offset 0x1e40 + 0x1e70: 0x00e6, 0x1e71: 0x00e6, + // Block 0x7a, offset 0x1e80 + 0x1eb0: 0x3000, + // Block 0x7b, offset 0x1ec0 + 0x1ec6: 0x0009, + // Block 0x7c, offset 0x1f00 + 0x1f04: 0x0009, + 0x1f20: 0x00e6, 0x1f21: 0x00e6, 0x1f22: 0x00e6, 0x1f23: 0x00e6, + 0x1f24: 0x00e6, 0x1f25: 0x00e6, 0x1f26: 0x00e6, 0x1f27: 0x00e6, 0x1f28: 0x00e6, 0x1f29: 0x00e6, + 0x1f2a: 0x00e6, 0x1f2b: 0x00e6, 0x1f2c: 0x00e6, 0x1f2d: 0x00e6, 0x1f2e: 0x00e6, 0x1f2f: 0x00e6, + 0x1f30: 0x00e6, 0x1f31: 0x00e6, + // Block 0x7d, offset 0x1f40 + 0x1f6b: 0x00dc, 0x1f6c: 0x00dc, 0x1f6d: 0x00dc, + // Block 0x7e, offset 0x1f80 + 0x1f93: 0x0009, + // Block 0x7f, offset 0x1fc0 + 0x1ff3: 0x0007, + // Block 0x80, offset 0x2000 + 0x2000: 0x0009, + // Block 0x81, offset 0x2040 + 0x2070: 0x00e6, 0x2072: 0x00e6, 0x2073: 0x00e6, 0x2074: 0x00dc, + 0x2077: 0x00e6, 0x2078: 0x00e6, + 0x207e: 0x00e6, 0x207f: 0x00e6, + // Block 0x82, offset 0x2080 + 0x2081: 0x00e6, + // Block 0x83, offset 0x20c0 + 0x20ed: 0x0009, + // Block 0x84, offset 0x2100 + 0x2100: 0x1100, 0x2101: 0x1100, 0x2102: 0x1100, 0x2103: 0x1100, 0x2104: 0x1100, 0x2105: 0x1100, + 0x2106: 0x1100, 0x2107: 0x1100, 0x2108: 0x1100, 0x2109: 0x1100, 0x210a: 0x1100, 0x210b: 0x1100, + 0x210c: 0x1100, 0x210d: 0x1100, 0x210e: 0x1100, 0x210f: 0x1100, 0x2110: 0x1100, 0x2111: 0x1100, + 0x2112: 0x1100, 0x2113: 0x1100, 0x2114: 0x1100, 0x2115: 0x1100, 0x2116: 0x1100, 0x2117: 0x1100, + 0x2118: 0x1100, 0x2119: 0x1100, 0x211a: 0x1100, 0x211b: 0x1100, 0x211c: 0x1100, 0x211d: 0x1100, + 0x211e: 0x1100, 0x211f: 0x1100, 0x2120: 0x1100, 0x2121: 0x1100, 0x2122: 0x1100, 0x2123: 0x1100, + 0x2124: 0x1100, 0x2125: 0x1100, 0x2126: 0x1100, 0x2127: 0x1100, 0x2128: 0x1100, 0x2129: 0x1100, + 0x212a: 0x1100, 0x212b: 0x1100, 0x212c: 0x1100, 0x212d: 0x1100, 0x212e: 0x1100, 0x212f: 0x1100, + 0x2130: 0x1100, 0x2131: 0x1100, 0x2132: 0x1100, 0x2133: 0x1100, 0x2134: 0x1100, 0x2135: 0x1100, + 0x2136: 0x1100, 0x2137: 0x1100, 0x2138: 0x1100, 0x2139: 0x1100, 0x213a: 0x1100, 0x213b: 0x1100, + 0x213c: 0x1100, 0x213d: 0x1100, 0x213e: 0x1100, 0x213f: 0x1100, + // Block 0x85, offset 0x2140 + 0x2140: 0x1100, 0x2141: 0x1100, 0x2142: 0x1100, 0x2143: 0x1100, 0x2144: 0x1100, 0x2145: 0x1100, + 0x2146: 0x1100, 0x2147: 0x1100, 0x2148: 0x1100, 0x2149: 0x1100, 0x214a: 0x1100, 0x214b: 0x1100, + 0x214c: 0x1100, 0x214d: 0x1100, 0x214e: 0x1100, 0x214f: 0x1100, 0x2150: 0x1100, 0x2151: 0x1100, + 0x2152: 0x1100, 0x2153: 0x1100, 0x2154: 0x1100, 0x2155: 0x1100, 0x2156: 0x1100, 0x2157: 0x1100, + 0x2158: 0x1100, 0x2159: 0x1100, 0x215a: 0x1100, 0x215b: 0x1100, 0x215c: 0x1100, 0x215d: 0x1100, + 0x215e: 0x1100, 0x215f: 0x1100, 0x2160: 0x1100, 0x2161: 0x1100, 0x2162: 0x1100, 0x2163: 0x1100, + // Block 0x86, offset 0x2180 + 0x2180: 0x3300, 0x2181: 0x3300, 0x2182: 0x3300, 0x2183: 0x3300, 0x2184: 0x3300, 0x2185: 0x3300, + 0x2186: 0x3300, 0x2187: 0x3300, 0x2188: 0x3300, 0x2189: 0x3300, 0x218a: 0x3300, 0x218b: 0x3300, + 0x218c: 0x3300, 0x218d: 0x3300, 0x218e: 0x3300, 0x218f: 0x3300, 0x2190: 0x3300, 0x2191: 0x3300, + 0x2192: 0x3300, 0x2193: 0x3300, 0x2194: 0x3300, 0x2195: 0x3300, 0x2196: 0x3300, 0x2197: 0x3300, + 0x2198: 0x3300, 0x2199: 0x3300, 0x219a: 0x3300, 0x219b: 0x3300, 0x219c: 0x3300, 0x219d: 0x3300, + 0x219e: 0x3300, 0x219f: 0x3300, 0x21a0: 0x3300, 0x21a1: 0x3300, 0x21a2: 0x3300, 0x21a3: 0x3300, + 0x21a4: 0x3300, 0x21a5: 0x3300, 0x21a6: 0x3300, 0x21a7: 0x3300, 0x21a8: 0x3300, 0x21a9: 0x3300, + 0x21aa: 0x3300, 0x21ab: 0x3300, 0x21ac: 0x3300, 0x21ad: 0x3300, 0x21ae: 0x3300, 0x21af: 0x3300, + 0x21b0: 0x3300, 0x21b1: 0x3300, 0x21b2: 0x3300, 0x21b3: 0x3300, 0x21b4: 0x3300, 0x21b5: 0x3300, + 0x21b6: 0x3300, 0x21b7: 0x3300, 0x21b8: 0x3300, 0x21b9: 0x3300, 0x21ba: 0x3300, 0x21bb: 0x3300, + 0x21bc: 0x3300, 0x21bd: 0x3300, 0x21be: 0x3300, 0x21bf: 0x3300, + // Block 0x87, offset 0x21c0 + 0x21c0: 0x3300, 0x21c1: 0x3300, 0x21c2: 0x3300, 0x21c3: 0x3300, 0x21c4: 0x3300, 0x21c5: 0x3300, + 0x21c6: 0x3300, 0x21c7: 0x3300, 0x21c8: 0x3300, 0x21c9: 0x3300, 0x21ca: 0x3300, 0x21cb: 0x3300, + 0x21cc: 0x3300, 0x21cd: 0x3300, 0x21d0: 0x3300, + 0x21d2: 0x3300, 0x21d5: 0x3300, 0x21d6: 0x3300, 0x21d7: 0x3300, + 0x21d8: 0x3300, 0x21d9: 0x3300, 0x21da: 0x3300, 0x21db: 0x3300, 0x21dc: 0x3300, 0x21dd: 0x3300, + 0x21de: 0x3300, 0x21e0: 0x3300, 0x21e2: 0x3300, + 0x21e5: 0x3300, 0x21e6: 0x3300, + 0x21ea: 0x3300, 0x21eb: 0x3300, 0x21ec: 0x3300, 0x21ed: 0x3300, + 0x21f0: 0x3300, 0x21f1: 0x3300, 0x21f2: 0x3300, 0x21f3: 0x3300, 0x21f4: 0x3300, 0x21f5: 0x3300, + 0x21f6: 0x3300, 0x21f7: 0x3300, 0x21f8: 0x3300, 0x21f9: 0x3300, 0x21fa: 0x3300, 0x21fb: 0x3300, + 0x21fc: 0x3300, 0x21fd: 0x3300, 0x21fe: 0x3300, 0x21ff: 0x3300, + // Block 0x88, offset 0x2200 + 0x2200: 0x3300, 0x2201: 0x3300, 0x2202: 0x3300, 0x2203: 0x3300, 0x2204: 0x3300, 0x2205: 0x3300, + 0x2206: 0x3300, 0x2207: 0x3300, 0x2208: 0x3300, 0x2209: 0x3300, 0x220a: 0x3300, 0x220b: 0x3300, + 0x220c: 0x3300, 0x220d: 0x3300, 0x220e: 0x3300, 0x220f: 0x3300, 0x2210: 0x3300, 0x2211: 0x3300, + 0x2212: 0x3300, 0x2213: 0x3300, 0x2214: 0x3300, 0x2215: 0x3300, 0x2216: 0x3300, 0x2217: 0x3300, + 0x2218: 0x3300, 0x2219: 0x3300, 0x221a: 0x3300, 0x221b: 0x3300, 0x221c: 0x3300, 0x221d: 0x3300, + 0x221e: 0x3300, 0x221f: 0x3300, 0x2220: 0x3300, 0x2221: 0x3300, 0x2222: 0x3300, 0x2223: 0x3300, + 0x2224: 0x3300, 0x2225: 0x3300, 0x2226: 0x3300, 0x2227: 0x3300, 0x2228: 0x3300, 0x2229: 0x3300, + 0x222a: 0x3300, 0x222b: 0x3300, 0x222c: 0x3300, 0x222d: 0x3300, + 0x2230: 0x3300, 0x2231: 0x3300, 0x2232: 0x3300, 0x2233: 0x3300, 0x2234: 0x3300, 0x2235: 0x3300, + 0x2236: 0x3300, 0x2237: 0x3300, 0x2238: 0x3300, 0x2239: 0x3300, 0x223a: 0x3300, 0x223b: 0x3300, + 0x223c: 0x3300, 0x223d: 0x3300, 0x223e: 0x3300, 0x223f: 0x3300, + // Block 0x89, offset 0x2240 + 0x2240: 0x3300, 0x2241: 0x3300, 0x2242: 0x3300, 0x2243: 0x3300, 0x2244: 0x3300, 0x2245: 0x3300, + 0x2246: 0x3300, 0x2247: 0x3300, 0x2248: 0x3300, 0x2249: 0x3300, 0x224a: 0x3300, 0x224b: 0x3300, + 0x224c: 0x3300, 0x224d: 0x3300, 0x224e: 0x3300, 0x224f: 0x3300, 0x2250: 0x3300, 0x2251: 0x3300, + 0x2252: 0x3300, 0x2253: 0x3300, 0x2254: 0x3300, 0x2255: 0x3300, 0x2256: 0x3300, 0x2257: 0x3300, + 0x2258: 0x3300, 0x2259: 0x3300, + // Block 0x8a, offset 0x2280 + 0x2280: 0x3000, 0x2281: 0x3000, 0x2282: 0x3000, 0x2283: 0x3000, 0x2284: 0x3000, 0x2285: 0x3000, + 0x2286: 0x3000, + 0x2293: 0x3000, 0x2294: 0x3000, 0x2295: 0x3000, 0x2296: 0x3000, 0x2297: 0x3000, + 0x229d: 0x3300, + 0x229e: 0x001a, 0x229f: 0x3300, 0x22a0: 0x3000, 0x22a1: 0x3000, 0x22a2: 0x3000, 0x22a3: 0x3000, + 0x22a4: 0x3000, 0x22a5: 0x3000, 0x22a6: 0x3000, 0x22a7: 0x3000, 0x22a8: 0x3000, 0x22a9: 0x3000, + 0x22aa: 0x3300, 0x22ab: 0x3300, 0x22ac: 0x3300, 0x22ad: 0x3300, 0x22ae: 0x3300, 0x22af: 0x3300, + 0x22b0: 0x3300, 0x22b1: 0x3300, 0x22b2: 0x3300, 0x22b3: 0x3300, 0x22b4: 0x3300, 0x22b5: 0x3300, + 0x22b6: 0x3300, 0x22b8: 0x3300, 0x22b9: 0x3300, 0x22ba: 0x3300, 0x22bb: 0x3300, + 0x22bc: 0x3300, 0x22be: 0x3300, + // Block 0x8b, offset 0x22c0 + 0x22c0: 0x3300, 0x22c1: 0x3300, 0x22c3: 0x3300, 0x22c4: 0x3300, + 0x22c6: 0x3300, 0x22c7: 0x3300, 0x22c8: 0x3300, 0x22c9: 0x3300, 0x22ca: 0x3300, 0x22cb: 0x3300, + 0x22cc: 0x3300, 0x22cd: 0x3300, 0x22ce: 0x3300, 0x22cf: 0x3000, 0x22d0: 0x3000, 0x22d1: 0x3000, + 0x22d2: 0x3000, 0x22d3: 0x3000, 0x22d4: 0x3000, 0x22d5: 0x3000, 0x22d6: 0x3000, 0x22d7: 0x3000, + 0x22d8: 0x3000, 0x22d9: 0x3000, 0x22da: 0x3000, 0x22db: 0x3000, 0x22dc: 0x3000, 0x22dd: 0x3000, + 0x22de: 0x3000, 0x22df: 0x3000, 0x22e0: 0x3000, 0x22e1: 0x3000, 0x22e2: 0x3000, 0x22e3: 0x3000, + 0x22e4: 0x3000, 0x22e5: 0x3000, 0x22e6: 0x3000, 0x22e7: 0x3000, 0x22e8: 0x3000, 0x22e9: 0x3000, + 0x22ea: 0x3000, 0x22eb: 0x3000, 0x22ec: 0x3000, 0x22ed: 0x3000, 0x22ee: 0x3000, 0x22ef: 0x3000, + 0x22f0: 0x3000, 0x22f1: 0x3000, 0x22f2: 0x3000, 0x22f3: 0x3000, 0x22f4: 0x3000, 0x22f5: 0x3000, + 0x22f6: 0x3000, 0x22f7: 0x3000, 0x22f8: 0x3000, 0x22f9: 0x3000, 0x22fa: 0x3000, 0x22fb: 0x3000, + 0x22fc: 0x3000, 0x22fd: 0x3000, 0x22fe: 0x3000, 0x22ff: 0x3000, + // Block 0x8c, offset 0x2300 + 0x2300: 0x3000, 0x2301: 0x3000, 0x2302: 0x3000, 0x2303: 0x3000, 0x2304: 0x3000, 0x2305: 0x3000, + 0x2306: 0x3000, 0x2307: 0x3000, 0x2308: 0x3000, 0x2309: 0x3000, 0x230a: 0x3000, 0x230b: 0x3000, + 0x230c: 0x3000, 0x230d: 0x3000, 0x230e: 0x3000, 0x230f: 0x3000, 0x2310: 0x3000, 0x2311: 0x3000, + 0x2312: 0x3000, 0x2313: 0x3000, 0x2314: 0x3000, 0x2315: 0x3000, 0x2316: 0x3000, 0x2317: 0x3000, + 0x2318: 0x3000, 0x2319: 0x3000, 0x231a: 0x3000, 0x231b: 0x3000, 0x231c: 0x3000, 0x231d: 0x3000, + 0x231e: 0x3000, 0x231f: 0x3000, 0x2320: 0x3000, 0x2321: 0x3000, 0x2322: 0x3000, 0x2323: 0x3000, + 0x2324: 0x3000, 0x2325: 0x3000, 0x2326: 0x3000, 0x2327: 0x3000, 0x2328: 0x3000, 0x2329: 0x3000, + 0x232a: 0x3000, 0x232b: 0x3000, 0x232c: 0x3000, 0x232d: 0x3000, 0x232e: 0x3000, 0x232f: 0x3000, + 0x2330: 0x3000, 0x2331: 0x3000, + // Block 0x8d, offset 0x2340 + 0x2353: 0x3000, 0x2354: 0x3000, 0x2355: 0x3000, 0x2356: 0x3000, 0x2357: 0x3000, + 0x2358: 0x3000, 0x2359: 0x3000, 0x235a: 0x3000, 0x235b: 0x3000, 0x235c: 0x3000, 0x235d: 0x3000, + 0x235e: 0x3000, 0x235f: 0x3000, 0x2360: 0x3000, 0x2361: 0x3000, 0x2362: 0x3000, 0x2363: 0x3000, + 0x2364: 0x3000, 0x2365: 0x3000, 0x2366: 0x3000, 0x2367: 0x3000, 0x2368: 0x3000, 0x2369: 0x3000, + 0x236a: 0x3000, 0x236b: 0x3000, 0x236c: 0x3000, 0x236d: 0x3000, 0x236e: 0x3000, 0x236f: 0x3000, + 0x2370: 0x3000, 0x2371: 0x3000, 0x2372: 0x3000, 0x2373: 0x3000, 0x2374: 0x3000, 0x2375: 0x3000, + 0x2376: 0x3000, 0x2377: 0x3000, 0x2378: 0x3000, 0x2379: 0x3000, 0x237a: 0x3000, 0x237b: 0x3000, + 0x237c: 0x3000, 0x237d: 0x3000, 0x237e: 0x3000, 0x237f: 0x3000, + // Block 0x8e, offset 0x2380 + 0x2380: 0x3000, 0x2381: 0x3000, 0x2382: 0x3000, 0x2383: 0x3000, 0x2384: 0x3000, 0x2385: 0x3000, + 0x2386: 0x3000, 0x2387: 0x3000, 0x2388: 0x3000, 0x2389: 0x3000, 0x238a: 0x3000, 0x238b: 0x3000, + 0x238c: 0x3000, 0x238d: 0x3000, 0x238e: 0x3000, 0x238f: 0x3000, 0x2390: 0x3000, 0x2391: 0x3000, + 0x2392: 0x3000, 0x2393: 0x3000, 0x2394: 0x3000, 0x2395: 0x3000, 0x2396: 0x3000, 0x2397: 0x3000, + 0x2398: 0x3000, 0x2399: 0x3000, 0x239a: 0x3000, 0x239b: 0x3000, 0x239c: 0x3000, 0x239d: 0x3000, + 0x239e: 0x3000, 0x239f: 0x3000, 0x23a0: 0x3000, 0x23a1: 0x3000, 0x23a2: 0x3000, 0x23a3: 0x3000, + 0x23a4: 0x3000, 0x23a5: 0x3000, 0x23a6: 0x3000, 0x23a7: 0x3000, 0x23a8: 0x3000, 0x23a9: 0x3000, + 0x23aa: 0x3000, 0x23ab: 0x3000, 0x23ac: 0x3000, 0x23ad: 0x3000, 0x23ae: 0x3000, 0x23af: 0x3000, + 0x23b0: 0x3000, 0x23b1: 0x3000, 0x23b2: 0x3000, 0x23b3: 0x3000, 0x23b4: 0x3000, 0x23b5: 0x3000, + 0x23b6: 0x3000, 0x23b7: 0x3000, 0x23b8: 0x3000, 0x23b9: 0x3000, 0x23ba: 0x3000, 0x23bb: 0x3000, + 0x23bc: 0x3000, 0x23bd: 0x3000, + // Block 0x8f, offset 0x23c0 + 0x23d0: 0x3000, 0x23d1: 0x3000, + 0x23d2: 0x3000, 0x23d3: 0x3000, 0x23d4: 0x3000, 0x23d5: 0x3000, 0x23d6: 0x3000, 0x23d7: 0x3000, + 0x23d8: 0x3000, 0x23d9: 0x3000, 0x23da: 0x3000, 0x23db: 0x3000, 0x23dc: 0x3000, 0x23dd: 0x3000, + 0x23de: 0x3000, 0x23df: 0x3000, 0x23e0: 0x3000, 0x23e1: 0x3000, 0x23e2: 0x3000, 0x23e3: 0x3000, + 0x23e4: 0x3000, 0x23e5: 0x3000, 0x23e6: 0x3000, 0x23e7: 0x3000, 0x23e8: 0x3000, 0x23e9: 0x3000, + 0x23ea: 0x3000, 0x23eb: 0x3000, 0x23ec: 0x3000, 0x23ed: 0x3000, 0x23ee: 0x3000, 0x23ef: 0x3000, + 0x23f0: 0x3000, 0x23f1: 0x3000, 0x23f2: 0x3000, 0x23f3: 0x3000, 0x23f4: 0x3000, 0x23f5: 0x3000, + 0x23f6: 0x3000, 0x23f7: 0x3000, 0x23f8: 0x3000, 0x23f9: 0x3000, 0x23fa: 0x3000, 0x23fb: 0x3000, + 0x23fc: 0x3000, 0x23fd: 0x3000, 0x23fe: 0x3000, 0x23ff: 0x3000, + // Block 0x90, offset 0x2400 + 0x2400: 0x3000, 0x2401: 0x3000, 0x2402: 0x3000, 0x2403: 0x3000, 0x2404: 0x3000, 0x2405: 0x3000, + 0x2406: 0x3000, 0x2407: 0x3000, 0x2408: 0x3000, 0x2409: 0x3000, 0x240a: 0x3000, 0x240b: 0x3000, + 0x240c: 0x3000, 0x240d: 0x3000, 0x240e: 0x3000, 0x240f: 0x3000, + 0x2412: 0x3000, 0x2413: 0x3000, 0x2414: 0x3000, 0x2415: 0x3000, 0x2416: 0x3000, 0x2417: 0x3000, + 0x2418: 0x3000, 0x2419: 0x3000, 0x241a: 0x3000, 0x241b: 0x3000, 0x241c: 0x3000, 0x241d: 0x3000, + 0x241e: 0x3000, 0x241f: 0x3000, 0x2420: 0x3000, 0x2421: 0x3000, 0x2422: 0x3000, 0x2423: 0x3000, + 0x2424: 0x3000, 0x2425: 0x3000, 0x2426: 0x3000, 0x2427: 0x3000, 0x2428: 0x3000, 0x2429: 0x3000, + 0x242a: 0x3000, 0x242b: 0x3000, 0x242c: 0x3000, 0x242d: 0x3000, 0x242e: 0x3000, 0x242f: 0x3000, + 0x2430: 0x3000, 0x2431: 0x3000, 0x2432: 0x3000, 0x2433: 0x3000, 0x2434: 0x3000, 0x2435: 0x3000, + 0x2436: 0x3000, 0x2437: 0x3000, 0x2438: 0x3000, 0x2439: 0x3000, 0x243a: 0x3000, 0x243b: 0x3000, + 0x243c: 0x3000, 0x243d: 0x3000, 0x243e: 0x3000, 0x243f: 0x3000, + // Block 0x91, offset 0x2440 + 0x2440: 0x3000, 0x2441: 0x3000, 0x2442: 0x3000, 0x2443: 0x3000, 0x2444: 0x3000, 0x2445: 0x3000, + 0x2446: 0x3000, 0x2447: 0x3000, + 0x2470: 0x3000, 0x2471: 0x3000, 0x2472: 0x3000, 0x2473: 0x3000, 0x2474: 0x3000, 0x2475: 0x3000, + 0x2476: 0x3000, 0x2477: 0x3000, 0x2478: 0x3000, 0x2479: 0x3000, 0x247a: 0x3000, 0x247b: 0x3000, + 0x247c: 0x3000, + // Block 0x92, offset 0x2480 + 0x2490: 0x3000, 0x2491: 0x3000, + 0x2492: 0x3000, 0x2493: 0x3000, 0x2494: 0x3000, 0x2495: 0x3000, 0x2496: 0x3000, 0x2497: 0x3000, + 0x2498: 0x3000, 0x2499: 0x3000, + 0x24a0: 0x00e6, 0x24a1: 0x00e6, 0x24a2: 0x00e6, 0x24a3: 0x00e6, + 0x24a4: 0x00e6, 0x24a5: 0x00e6, 0x24a6: 0x00e6, + 0x24b0: 0x3000, 0x24b1: 0x3000, 0x24b2: 0x3000, 0x24b3: 0x3000, 0x24b4: 0x3000, 0x24b5: 0x3000, + 0x24b6: 0x3000, 0x24b7: 0x3000, 0x24b8: 0x3000, 0x24b9: 0x3000, 0x24ba: 0x3000, 0x24bb: 0x3000, + 0x24bc: 0x3000, 0x24bd: 0x3000, 0x24be: 0x3000, 0x24bf: 0x3000, + // Block 0x93, offset 0x24c0 + 0x24c0: 0x3000, 0x24c1: 0x3000, 0x24c2: 0x3000, 0x24c3: 0x3000, 0x24c4: 0x3000, + 0x24c7: 0x3000, 0x24c8: 0x3000, 0x24c9: 0x3000, 0x24ca: 0x3000, 0x24cb: 0x3000, + 0x24cc: 0x3000, 0x24cd: 0x3000, 0x24ce: 0x3000, 0x24cf: 0x3000, 0x24d0: 0x3000, 0x24d1: 0x3000, + 0x24d2: 0x3000, 0x24d4: 0x3000, 0x24d5: 0x3000, 0x24d6: 0x3000, 0x24d7: 0x3000, + 0x24d8: 0x3000, 0x24d9: 0x3000, 0x24da: 0x3000, 0x24db: 0x3000, 0x24dc: 0x3000, 0x24dd: 0x3000, + 0x24de: 0x3000, 0x24df: 0x3000, 0x24e0: 0x3000, 0x24e1: 0x3000, 0x24e2: 0x3000, 0x24e3: 0x3000, + 0x24e4: 0x3000, 0x24e5: 0x3000, 0x24e6: 0x3000, 0x24e8: 0x3000, 0x24e9: 0x3000, + 0x24ea: 0x3000, 0x24eb: 0x3000, + 0x24f0: 0x3000, 0x24f1: 0x3000, 0x24f2: 0x3000, 0x24f4: 0x3000, + 0x24f6: 0x3000, 0x24f7: 0x3000, 0x24f8: 0x3000, 0x24f9: 0x3000, 0x24fa: 0x3000, 0x24fb: 0x3000, + 0x24fc: 0x3000, 0x24fd: 0x3000, 0x24fe: 0x3000, 0x24ff: 0x3000, + // Block 0x94, offset 0x2500 + 0x2500: 0x3000, 0x2501: 0x3000, 0x2502: 0x3000, 0x2503: 0x3000, 0x2504: 0x3000, 0x2505: 0x3000, + 0x2506: 0x3000, 0x2507: 0x3000, 0x2508: 0x3000, 0x2509: 0x3000, 0x250a: 0x3000, 0x250b: 0x3000, + 0x250c: 0x3000, 0x250d: 0x3000, 0x250e: 0x3000, 0x250f: 0x3000, 0x2510: 0x3000, 0x2511: 0x3000, + 0x2512: 0x3000, 0x2513: 0x3000, 0x2514: 0x3000, 0x2515: 0x3000, 0x2516: 0x3000, 0x2517: 0x3000, + 0x2518: 0x3000, 0x2519: 0x3000, 0x251a: 0x3000, 0x251b: 0x3000, 0x251c: 0x3000, 0x251d: 0x3000, + 0x251e: 0x3000, 0x251f: 0x3000, 0x2520: 0x3000, 0x2521: 0x3000, 0x2522: 0x3000, 0x2523: 0x3000, + 0x2524: 0x3000, 0x2525: 0x3000, 0x2526: 0x3000, 0x2527: 0x3000, 0x2528: 0x3000, 0x2529: 0x3000, + 0x252a: 0x3000, 0x252b: 0x3000, 0x252c: 0x3000, 0x252d: 0x3000, 0x252e: 0x3000, 0x252f: 0x3000, + 0x2530: 0x3000, 0x2531: 0x3000, 0x2532: 0x3000, 0x2533: 0x3000, 0x2534: 0x3000, 0x2535: 0x3000, + 0x2536: 0x3000, 0x2537: 0x3000, 0x2538: 0x3000, 0x2539: 0x3000, 0x253a: 0x3000, 0x253b: 0x3000, + 0x253c: 0x3000, + // Block 0x95, offset 0x2540 + 0x2541: 0x3000, 0x2542: 0x3000, 0x2543: 0x3000, 0x2544: 0x3000, 0x2545: 0x3000, + 0x2546: 0x3000, 0x2547: 0x3000, 0x2548: 0x3000, 0x2549: 0x3000, 0x254a: 0x3000, 0x254b: 0x3000, + 0x254c: 0x3000, 0x254d: 0x3000, 0x254e: 0x3000, 0x254f: 0x3000, 0x2550: 0x3000, 0x2551: 0x3000, + 0x2552: 0x3000, 0x2553: 0x3000, 0x2554: 0x3000, 0x2555: 0x3000, 0x2556: 0x3000, 0x2557: 0x3000, + 0x2558: 0x3000, 0x2559: 0x3000, 0x255a: 0x3000, 0x255b: 0x3000, 0x255c: 0x3000, 0x255d: 0x3000, + 0x255e: 0x3000, 0x255f: 0x3000, 0x2560: 0x3000, 0x2561: 0x3000, 0x2562: 0x3000, 0x2563: 0x3000, + 0x2564: 0x3000, 0x2565: 0x3000, 0x2566: 0x3000, 0x2567: 0x3000, 0x2568: 0x3000, 0x2569: 0x3000, + 0x256a: 0x3000, 0x256b: 0x3000, 0x256c: 0x3000, 0x256d: 0x3000, 0x256e: 0x3000, 0x256f: 0x3000, + 0x2570: 0x3000, 0x2571: 0x3000, 0x2572: 0x3000, 0x2573: 0x3000, 0x2574: 0x3000, 0x2575: 0x3000, + 0x2576: 0x3000, 0x2577: 0x3000, 0x2578: 0x3000, 0x2579: 0x3000, 0x257a: 0x3000, 0x257b: 0x3000, + 0x257c: 0x3000, 0x257d: 0x3000, 0x257e: 0x3000, 0x257f: 0x3000, + // Block 0x96, offset 0x2580 + 0x2582: 0x3000, 0x2583: 0x3000, 0x2584: 0x3000, 0x2585: 0x3000, + 0x2586: 0x3000, 0x2587: 0x3000, 0x258a: 0x3000, 0x258b: 0x3000, + 0x258c: 0x3000, 0x258d: 0x3000, 0x258e: 0x3000, 0x258f: 0x3000, + 0x2592: 0x3000, 0x2593: 0x3000, 0x2594: 0x3000, 0x2595: 0x3000, 0x2596: 0x3000, 0x2597: 0x3000, + 0x259a: 0x3000, 0x259b: 0x3000, 0x259c: 0x3000, + 0x25a0: 0x3000, 0x25a1: 0x3000, 0x25a2: 0x3000, 0x25a3: 0x3000, + 0x25a4: 0x3000, 0x25a5: 0x3000, 0x25a6: 0x3000, 0x25a8: 0x3000, 0x25a9: 0x3000, + 0x25aa: 0x3000, 0x25ab: 0x3000, 0x25ac: 0x3000, 0x25ad: 0x3000, 0x25ae: 0x3000, + // Block 0x97, offset 0x25c0 + 0x25fd: 0x00dc, + // Block 0x98, offset 0x2600 + 0x260d: 0x00dc, 0x260f: 0x00e6, + 0x2638: 0x00e6, 0x2639: 0x0001, 0x263a: 0x00dc, + 0x263f: 0x0009, + // Block 0x99, offset 0x2640 + 0x2659: 0x8800, 0x265a: 0x1100, 0x265b: 0x8800, 0x265c: 0x1100, + 0x2665: 0x8800, + 0x266b: 0x1100, + 0x2679: 0x0009, 0x267a: 0x6607, + // Block 0x9a, offset 0x2680 + 0x269e: 0x3300, 0x269f: 0x3300, 0x26a0: 0x3300, 0x26a1: 0x3300, 0x26a2: 0x3300, 0x26a3: 0x3300, + 0x26a4: 0x3300, 0x26a5: 0x00d8, 0x26a6: 0x00d8, 0x26a7: 0x0001, 0x26a8: 0x0001, 0x26a9: 0x0001, + 0x26ad: 0x00e2, 0x26ae: 0x00d8, 0x26af: 0x00d8, + 0x26b0: 0x00d8, 0x26b1: 0x00d8, 0x26b2: 0x00d8, + 0x26bb: 0x00dc, + 0x26bc: 0x00dc, 0x26bd: 0x00dc, 0x26be: 0x00dc, 0x26bf: 0x00dc, + // Block 0x9b, offset 0x26c0 + 0x26c0: 0x00dc, 0x26c1: 0x00dc, 0x26c2: 0x00dc, 0x26c5: 0x00e6, + 0x26c6: 0x00e6, 0x26c7: 0x00e6, 0x26c8: 0x00e6, 0x26c9: 0x00e6, 0x26ca: 0x00dc, 0x26cb: 0x00dc, + 0x26ea: 0x00e6, 0x26eb: 0x00e6, 0x26ec: 0x00e6, 0x26ed: 0x00e6, + 0x26fb: 0x3300, + 0x26fc: 0x3300, 0x26fd: 0x3300, 0x26fe: 0x3300, 0x26ff: 0x3300, + // Block 0x9c, offset 0x2700 + 0x2700: 0x3300, + // Block 0x9d, offset 0x2740 + 0x2742: 0x00e6, 0x2743: 0x00e6, 0x2744: 0x00e6, + // Block 0x9e, offset 0x2780 + 0x2780: 0x3000, 0x2781: 0x3000, 0x2782: 0x3000, 0x2783: 0x3000, 0x2784: 0x3000, 0x2785: 0x3000, + 0x2786: 0x3000, 0x2787: 0x3000, 0x2788: 0x3000, 0x2789: 0x3000, 0x278a: 0x3000, 0x278b: 0x3000, + 0x278c: 0x3000, 0x278d: 0x3000, 0x278e: 0x3000, 0x278f: 0x3000, 0x2790: 0x3000, 0x2791: 0x3000, + 0x2792: 0x3000, 0x2793: 0x3000, 0x2794: 0x3000, 0x2796: 0x3000, 0x2797: 0x3000, + 0x2798: 0x3000, 0x2799: 0x3000, 0x279a: 0x3000, 0x279b: 0x3000, 0x279c: 0x3000, 0x279d: 0x3000, + 0x279e: 0x3000, 0x279f: 0x3000, 0x27a0: 0x3000, 0x27a1: 0x3000, 0x27a2: 0x3000, 0x27a3: 0x3000, + 0x27a4: 0x3000, 0x27a5: 0x3000, 0x27a6: 0x3000, 0x27a7: 0x3000, 0x27a8: 0x3000, 0x27a9: 0x3000, + 0x27aa: 0x3000, 0x27ab: 0x3000, 0x27ac: 0x3000, 0x27ad: 0x3000, 0x27ae: 0x3000, 0x27af: 0x3000, + 0x27b0: 0x3000, 0x27b1: 0x3000, 0x27b2: 0x3000, 0x27b3: 0x3000, 0x27b4: 0x3000, 0x27b5: 0x3000, + 0x27b6: 0x3000, 0x27b7: 0x3000, 0x27b8: 0x3000, 0x27b9: 0x3000, 0x27ba: 0x3000, 0x27bb: 0x3000, + 0x27bc: 0x3000, 0x27bd: 0x3000, 0x27be: 0x3000, 0x27bf: 0x3000, + // Block 0x9f, offset 0x27c0 + 0x27c0: 0x3000, 0x27c1: 0x3000, 0x27c2: 0x3000, 0x27c3: 0x3000, 0x27c4: 0x3000, 0x27c5: 0x3000, + 0x27c6: 0x3000, 0x27c7: 0x3000, 0x27c8: 0x3000, 0x27c9: 0x3000, 0x27ca: 0x3000, 0x27cb: 0x3000, + 0x27cc: 0x3000, 0x27cd: 0x3000, 0x27ce: 0x3000, 0x27cf: 0x3000, 0x27d0: 0x3000, 0x27d1: 0x3000, + 0x27d2: 0x3000, 0x27d3: 0x3000, 0x27d4: 0x3000, 0x27d5: 0x3000, 0x27d6: 0x3000, 0x27d7: 0x3000, + 0x27d8: 0x3000, 0x27d9: 0x3000, 0x27da: 0x3000, 0x27db: 0x3000, 0x27dc: 0x3000, + 0x27de: 0x3000, 0x27df: 0x3000, 0x27e2: 0x3000, + 0x27e5: 0x3000, 0x27e6: 0x3000, 0x27e9: 0x3000, + 0x27ea: 0x3000, 0x27eb: 0x3000, 0x27ec: 0x3000, 0x27ee: 0x3000, 0x27ef: 0x3000, + 0x27f0: 0x3000, 0x27f1: 0x3000, 0x27f2: 0x3000, 0x27f3: 0x3000, 0x27f4: 0x3000, 0x27f5: 0x3000, + 0x27f6: 0x3000, 0x27f7: 0x3000, 0x27f8: 0x3000, 0x27f9: 0x3000, 0x27fb: 0x3000, + 0x27fd: 0x3000, 0x27fe: 0x3000, 0x27ff: 0x3000, + // Block 0xa0, offset 0x2800 + 0x2800: 0x3000, 0x2801: 0x3000, 0x2802: 0x3000, 0x2803: 0x3000, 0x2805: 0x3000, + 0x2806: 0x3000, 0x2807: 0x3000, 0x2808: 0x3000, 0x2809: 0x3000, 0x280a: 0x3000, 0x280b: 0x3000, + 0x280c: 0x3000, 0x280d: 0x3000, 0x280e: 0x3000, 0x280f: 0x3000, 0x2810: 0x3000, 0x2811: 0x3000, + 0x2812: 0x3000, 0x2813: 0x3000, 0x2814: 0x3000, 0x2815: 0x3000, 0x2816: 0x3000, 0x2817: 0x3000, + 0x2818: 0x3000, 0x2819: 0x3000, 0x281a: 0x3000, 0x281b: 0x3000, 0x281c: 0x3000, 0x281d: 0x3000, + 0x281e: 0x3000, 0x281f: 0x3000, 0x2820: 0x3000, 0x2821: 0x3000, 0x2822: 0x3000, 0x2823: 0x3000, + 0x2824: 0x3000, 0x2825: 0x3000, 0x2826: 0x3000, 0x2827: 0x3000, 0x2828: 0x3000, 0x2829: 0x3000, + 0x282a: 0x3000, 0x282b: 0x3000, 0x282c: 0x3000, 0x282d: 0x3000, 0x282e: 0x3000, 0x282f: 0x3000, + 0x2830: 0x3000, 0x2831: 0x3000, 0x2832: 0x3000, 0x2833: 0x3000, 0x2834: 0x3000, 0x2835: 0x3000, + 0x2836: 0x3000, 0x2837: 0x3000, 0x2838: 0x3000, 0x2839: 0x3000, 0x283a: 0x3000, 0x283b: 0x3000, + 0x283c: 0x3000, 0x283d: 0x3000, 0x283e: 0x3000, 0x283f: 0x3000, + // Block 0xa1, offset 0x2840 + 0x2840: 0x3000, 0x2841: 0x3000, 0x2842: 0x3000, 0x2843: 0x3000, 0x2844: 0x3000, 0x2845: 0x3000, + 0x2847: 0x3000, 0x2848: 0x3000, 0x2849: 0x3000, 0x284a: 0x3000, + 0x284d: 0x3000, 0x284e: 0x3000, 0x284f: 0x3000, 0x2850: 0x3000, 0x2851: 0x3000, + 0x2852: 0x3000, 0x2853: 0x3000, 0x2854: 0x3000, 0x2856: 0x3000, 0x2857: 0x3000, + 0x2858: 0x3000, 0x2859: 0x3000, 0x285a: 0x3000, 0x285b: 0x3000, 0x285c: 0x3000, + 0x285e: 0x3000, 0x285f: 0x3000, 0x2860: 0x3000, 0x2861: 0x3000, 0x2862: 0x3000, 0x2863: 0x3000, + 0x2864: 0x3000, 0x2865: 0x3000, 0x2866: 0x3000, 0x2867: 0x3000, 0x2868: 0x3000, 0x2869: 0x3000, + 0x286a: 0x3000, 0x286b: 0x3000, 0x286c: 0x3000, 0x286d: 0x3000, 0x286e: 0x3000, 0x286f: 0x3000, + 0x2870: 0x3000, 0x2871: 0x3000, 0x2872: 0x3000, 0x2873: 0x3000, 0x2874: 0x3000, 0x2875: 0x3000, + 0x2876: 0x3000, 0x2877: 0x3000, 0x2878: 0x3000, 0x2879: 0x3000, 0x287b: 0x3000, + 0x287c: 0x3000, 0x287d: 0x3000, 0x287e: 0x3000, + // Block 0xa2, offset 0x2880 + 0x2880: 0x3000, 0x2881: 0x3000, 0x2882: 0x3000, 0x2883: 0x3000, 0x2884: 0x3000, + 0x2886: 0x3000, 0x288a: 0x3000, 0x288b: 0x3000, + 0x288c: 0x3000, 0x288d: 0x3000, 0x288e: 0x3000, 0x288f: 0x3000, 0x2890: 0x3000, + 0x2892: 0x3000, 0x2893: 0x3000, 0x2894: 0x3000, 0x2895: 0x3000, 0x2896: 0x3000, 0x2897: 0x3000, + 0x2898: 0x3000, 0x2899: 0x3000, 0x289a: 0x3000, 0x289b: 0x3000, 0x289c: 0x3000, 0x289d: 0x3000, + 0x289e: 0x3000, 0x289f: 0x3000, 0x28a0: 0x3000, 0x28a1: 0x3000, 0x28a2: 0x3000, 0x28a3: 0x3000, + 0x28a4: 0x3000, 0x28a5: 0x3000, 0x28a6: 0x3000, 0x28a7: 0x3000, 0x28a8: 0x3000, 0x28a9: 0x3000, + 0x28aa: 0x3000, 0x28ab: 0x3000, 0x28ac: 0x3000, 0x28ad: 0x3000, 0x28ae: 0x3000, 0x28af: 0x3000, + 0x28b0: 0x3000, 0x28b1: 0x3000, 0x28b2: 0x3000, 0x28b3: 0x3000, 0x28b4: 0x3000, 0x28b5: 0x3000, + 0x28b6: 0x3000, 0x28b7: 0x3000, 0x28b8: 0x3000, 0x28b9: 0x3000, 0x28ba: 0x3000, 0x28bb: 0x3000, + 0x28bc: 0x3000, 0x28bd: 0x3000, 0x28be: 0x3000, 0x28bf: 0x3000, + // Block 0xa3, offset 0x28c0 + 0x28c0: 0x3000, 0x28c1: 0x3000, 0x28c2: 0x3000, 0x28c3: 0x3000, 0x28c4: 0x3000, 0x28c5: 0x3000, + 0x28c6: 0x3000, 0x28c7: 0x3000, 0x28c8: 0x3000, 0x28c9: 0x3000, 0x28ca: 0x3000, 0x28cb: 0x3000, + 0x28cc: 0x3000, 0x28cd: 0x3000, 0x28ce: 0x3000, 0x28cf: 0x3000, 0x28d0: 0x3000, 0x28d1: 0x3000, + 0x28d2: 0x3000, 0x28d3: 0x3000, 0x28d4: 0x3000, 0x28d5: 0x3000, 0x28d6: 0x3000, 0x28d7: 0x3000, + 0x28d8: 0x3000, 0x28d9: 0x3000, 0x28da: 0x3000, 0x28db: 0x3000, 0x28dc: 0x3000, 0x28dd: 0x3000, + 0x28de: 0x3000, 0x28df: 0x3000, 0x28e0: 0x3000, 0x28e1: 0x3000, 0x28e2: 0x3000, 0x28e3: 0x3000, + 0x28e4: 0x3000, 0x28e5: 0x3000, 0x28e8: 0x3000, 0x28e9: 0x3000, + 0x28ea: 0x3000, 0x28eb: 0x3000, 0x28ec: 0x3000, 0x28ed: 0x3000, 0x28ee: 0x3000, 0x28ef: 0x3000, + 0x28f0: 0x3000, 0x28f1: 0x3000, 0x28f2: 0x3000, 0x28f3: 0x3000, 0x28f4: 0x3000, 0x28f5: 0x3000, + 0x28f6: 0x3000, 0x28f7: 0x3000, 0x28f8: 0x3000, 0x28f9: 0x3000, 0x28fa: 0x3000, 0x28fb: 0x3000, + 0x28fc: 0x3000, 0x28fd: 0x3000, 0x28fe: 0x3000, 0x28ff: 0x3000, + // Block 0xa4, offset 0x2900 + 0x2900: 0x3000, 0x2901: 0x3000, 0x2902: 0x3000, 0x2903: 0x3000, 0x2904: 0x3000, 0x2905: 0x3000, + 0x2906: 0x3000, 0x2907: 0x3000, 0x2908: 0x3000, 0x2909: 0x3000, 0x290a: 0x3000, 0x290b: 0x3000, + 0x290e: 0x3000, 0x290f: 0x3000, 0x2910: 0x3000, 0x2911: 0x3000, + 0x2912: 0x3000, 0x2913: 0x3000, 0x2914: 0x3000, 0x2915: 0x3000, 0x2916: 0x3000, 0x2917: 0x3000, + 0x2918: 0x3000, 0x2919: 0x3000, 0x291a: 0x3000, 0x291b: 0x3000, 0x291c: 0x3000, 0x291d: 0x3000, + 0x291e: 0x3000, 0x291f: 0x3000, 0x2920: 0x3000, 0x2921: 0x3000, 0x2922: 0x3000, 0x2923: 0x3000, + 0x2924: 0x3000, 0x2925: 0x3000, 0x2926: 0x3000, 0x2927: 0x3000, 0x2928: 0x3000, 0x2929: 0x3000, + 0x292a: 0x3000, 0x292b: 0x3000, 0x292c: 0x3000, 0x292d: 0x3000, 0x292e: 0x3000, 0x292f: 0x3000, + 0x2930: 0x3000, 0x2931: 0x3000, 0x2932: 0x3000, 0x2933: 0x3000, 0x2934: 0x3000, 0x2935: 0x3000, + 0x2936: 0x3000, 0x2937: 0x3000, 0x2938: 0x3000, 0x2939: 0x3000, 0x293a: 0x3000, 0x293b: 0x3000, + 0x293c: 0x3000, 0x293d: 0x3000, 0x293e: 0x3000, 0x293f: 0x3000, + // Block 0xa5, offset 0x2940 + 0x2940: 0x3000, 0x2941: 0x3000, 0x2942: 0x3000, 0x2943: 0x3000, 0x2944: 0x3000, 0x2945: 0x3000, + 0x2946: 0x3000, 0x2947: 0x3000, 0x2948: 0x3000, 0x2949: 0x3000, 0x294a: 0x3000, + 0x2950: 0x3000, 0x2951: 0x3000, + 0x2952: 0x3000, 0x2953: 0x3000, 0x2954: 0x3000, 0x2955: 0x3000, 0x2956: 0x3000, 0x2957: 0x3000, + 0x2958: 0x3000, 0x2959: 0x3000, 0x295a: 0x3000, 0x295b: 0x3000, 0x295c: 0x3000, 0x295d: 0x3000, + 0x295e: 0x3000, 0x295f: 0x3000, 0x2960: 0x3000, 0x2961: 0x3000, 0x2962: 0x3000, 0x2963: 0x3000, + 0x2964: 0x3000, 0x2965: 0x3000, 0x2966: 0x3000, 0x2967: 0x3000, 0x2968: 0x3000, 0x2969: 0x3000, + 0x296a: 0x3000, 0x296b: 0x3000, 0x296c: 0x3000, 0x296d: 0x3000, 0x296e: 0x3000, + 0x2970: 0x3000, 0x2971: 0x3000, 0x2972: 0x3000, 0x2973: 0x3000, 0x2974: 0x3000, 0x2975: 0x3000, + 0x2976: 0x3000, 0x2977: 0x3000, 0x2978: 0x3000, 0x2979: 0x3000, 0x297a: 0x3000, 0x297b: 0x3000, + 0x297c: 0x3000, 0x297d: 0x3000, 0x297e: 0x3000, 0x297f: 0x3000, + // Block 0xa6, offset 0x2980 + 0x2980: 0x3000, 0x2981: 0x3000, 0x2982: 0x3000, 0x2983: 0x3000, 0x2984: 0x3000, 0x2985: 0x3000, + 0x2986: 0x3000, 0x2987: 0x3000, 0x2988: 0x3000, 0x2989: 0x3000, 0x298a: 0x3000, 0x298b: 0x3000, + 0x298c: 0x3000, 0x298d: 0x3000, 0x298e: 0x3000, 0x298f: 0x3000, + // Block 0xa7, offset 0x29c0 + 0x29d0: 0x3000, + // Block 0xa8, offset 0x2a00 + 0x2a00: 0x3000, 0x2a01: 0x3000, 0x2a02: 0x3000, + 0x2a10: 0x3000, 0x2a11: 0x3000, + 0x2a12: 0x3000, 0x2a13: 0x3000, 0x2a14: 0x3000, 0x2a15: 0x3000, 0x2a16: 0x3000, 0x2a17: 0x3000, + 0x2a18: 0x3000, 0x2a19: 0x3000, 0x2a1a: 0x3000, 0x2a1b: 0x3000, 0x2a1c: 0x3000, 0x2a1d: 0x3000, + 0x2a1e: 0x3000, 0x2a1f: 0x3000, 0x2a20: 0x3000, 0x2a21: 0x3000, 0x2a22: 0x3000, 0x2a23: 0x3000, + 0x2a24: 0x3000, 0x2a25: 0x3000, 0x2a26: 0x3000, 0x2a27: 0x3000, 0x2a28: 0x3000, 0x2a29: 0x3000, + 0x2a2a: 0x3000, 0x2a2b: 0x3000, 0x2a2c: 0x3000, 0x2a2d: 0x3000, 0x2a2e: 0x3000, 0x2a2f: 0x3000, + 0x2a30: 0x3000, 0x2a31: 0x3000, 0x2a32: 0x3000, 0x2a33: 0x3000, 0x2a34: 0x3000, 0x2a35: 0x3000, + 0x2a36: 0x3000, 0x2a37: 0x3000, 0x2a38: 0x3000, 0x2a39: 0x3000, 0x2a3a: 0x3000, + // Block 0xa9, offset 0x2a40 + 0x2a40: 0x3000, 0x2a41: 0x3000, 0x2a42: 0x3000, 0x2a43: 0x3000, 0x2a44: 0x3000, 0x2a45: 0x3000, + 0x2a46: 0x3000, 0x2a47: 0x3000, 0x2a48: 0x3000, + 0x2a50: 0x3000, 0x2a51: 0x3000, + // Block 0xaa, offset 0x2a80 + 0x2a80: 0x3300, 0x2a81: 0x3300, 0x2a82: 0x3300, 0x2a83: 0x3300, 0x2a84: 0x3300, 0x2a85: 0x3300, + 0x2a86: 0x3300, 0x2a87: 0x3300, 0x2a88: 0x3300, 0x2a89: 0x3300, 0x2a8a: 0x3300, 0x2a8b: 0x3300, + 0x2a8c: 0x3300, 0x2a8d: 0x3300, 0x2a8e: 0x3300, 0x2a8f: 0x3300, 0x2a90: 0x3300, 0x2a91: 0x3300, + 0x2a92: 0x3300, 0x2a93: 0x3300, 0x2a94: 0x3300, 0x2a95: 0x3300, 0x2a96: 0x3300, 0x2a97: 0x3300, + 0x2a98: 0x3300, 0x2a99: 0x3300, 0x2a9a: 0x3300, 0x2a9b: 0x3300, 0x2a9c: 0x3300, 0x2a9d: 0x3300, +} + +// charInfoLookup: 1152 bytes +// Block 0 is the null block. +var charInfoLookup = [1152]uint8{ + // Block 0x0, offset 0x0 + // Block 0x1, offset 0x40 + // Block 0x2, offset 0x80 + // Block 0x3, offset 0xc0 + 0x0c2: 0x03, 0x0c3: 0x04, 0x0c4: 0x05, 0x0c5: 0x06, 0x0c6: 0x07, 0x0c7: 0x08, + 0x0c8: 0x09, 0x0ca: 0x0a, 0x0cb: 0x0b, 0x0cc: 0x0c, 0x0cd: 0x0d, 0x0ce: 0x0e, 0x0cf: 0x0f, + 0x0d0: 0x10, 0x0d1: 0x11, 0x0d2: 0x12, 0x0d3: 0x13, 0x0d6: 0x14, 0x0d7: 0x15, + 0x0d8: 0x16, 0x0d9: 0x17, 0x0db: 0x18, 0x0dc: 0x19, 0x0dd: 0x1a, 0x0df: 0x1b, + 0x0e0: 0x04, 0x0e1: 0x05, 0x0e2: 0x06, 0x0e3: 0x07, + 0x0ea: 0x08, 0x0eb: 0x09, 0x0ec: 0x09, 0x0ed: 0x0a, 0x0ef: 0x0b, + 0x0f0: 0x11, + // Block 0x4, offset 0x100 + 0x120: 0x1c, 0x121: 0x1d, 0x124: 0x1e, 0x125: 0x1f, 0x126: 0x20, 0x127: 0x21, + 0x128: 0x22, 0x129: 0x23, 0x12a: 0x24, 0x12b: 0x25, 0x12c: 0x20, 0x12d: 0x26, 0x12e: 0x27, 0x12f: 0x28, + 0x131: 0x29, 0x132: 0x2a, 0x133: 0x2b, 0x134: 0x2c, 0x135: 0x28, 0x137: 0x2d, + 0x138: 0x2e, 0x139: 0x2f, 0x13a: 0x30, 0x13b: 0x31, 0x13c: 0x32, 0x13d: 0x33, 0x13e: 0x34, 0x13f: 0x35, + // Block 0x5, offset 0x140 + 0x140: 0x36, 0x142: 0x37, 0x143: 0x38, 0x145: 0x39, 0x146: 0x3a, 0x147: 0x3b, + 0x14d: 0x3c, + 0x15c: 0x3d, 0x15f: 0x3e, + 0x162: 0x3f, 0x164: 0x40, + 0x168: 0x41, 0x169: 0x42, 0x16c: 0x43, 0x16d: 0x44, 0x16e: 0x45, 0x16f: 0x46, + 0x170: 0x47, 0x173: 0x48, 0x174: 0x49, 0x175: 0x4a, 0x176: 0x4b, 0x177: 0x4c, + 0x178: 0x4d, 0x179: 0x4e, 0x17a: 0x4f, 0x17b: 0x50, 0x17c: 0x51, 0x17d: 0x52, 0x17e: 0x53, 0x17f: 0x54, + // Block 0x6, offset 0x180 + 0x180: 0x55, 0x181: 0x56, 0x182: 0x57, 0x183: 0x58, 0x184: 0x59, 0x185: 0x5a, 0x186: 0x5b, 0x187: 0x5c, + 0x188: 0x5d, 0x189: 0x5e, 0x18a: 0x5f, 0x18b: 0x60, 0x18c: 0x61, + 0x191: 0x62, 0x192: 0x63, 0x193: 0x64, + 0x1a8: 0x65, 0x1a9: 0x66, 0x1ab: 0x67, + 0x1b1: 0x68, 0x1b3: 0x69, 0x1b5: 0x6a, 0x1b7: 0x6b, + 0x1ba: 0x6c, 0x1bb: 0x6d, 0x1bc: 0x63, 0x1bd: 0x63, 0x1be: 0x63, 0x1bf: 0x6e, + // Block 0x7, offset 0x1c0 + 0x1c0: 0x6f, 0x1c1: 0x70, 0x1c2: 0x71, 0x1c3: 0x72, 0x1c4: 0x73, 0x1c5: 0x63, 0x1c6: 0x74, + 0x1c8: 0x75, 0x1c9: 0x76, 0x1ca: 0x63, 0x1cb: 0x77, 0x1cc: 0x63, 0x1cd: 0x63, 0x1ce: 0x63, 0x1cf: 0x63, + // Block 0x8, offset 0x200 + 0x219: 0x78, 0x21b: 0x79, 0x21d: 0x7a, + 0x220: 0x7b, 0x223: 0x7c, 0x224: 0x7d, 0x225: 0x7e, 0x226: 0x7f, 0x227: 0x80, + 0x22a: 0x81, 0x22b: 0x82, 0x22f: 0x83, + 0x230: 0x84, 0x231: 0x84, 0x232: 0x84, 0x233: 0x84, 0x234: 0x84, 0x235: 0x84, 0x236: 0x84, 0x237: 0x84, + 0x238: 0x84, 0x239: 0x84, 0x23a: 0x84, 0x23b: 0x84, 0x23c: 0x84, 0x23d: 0x84, 0x23e: 0x84, 0x23f: 0x84, + // Block 0x9, offset 0x240 + 0x240: 0x84, 0x241: 0x84, 0x242: 0x84, 0x243: 0x84, 0x244: 0x84, 0x245: 0x84, 0x246: 0x84, 0x247: 0x84, + 0x248: 0x84, 0x249: 0x84, 0x24a: 0x84, 0x24b: 0x84, 0x24c: 0x84, 0x24d: 0x84, 0x24e: 0x84, 0x24f: 0x84, + 0x250: 0x84, 0x251: 0x84, 0x252: 0x84, 0x253: 0x84, 0x254: 0x84, 0x255: 0x84, 0x256: 0x84, 0x257: 0x84, + 0x258: 0x84, 0x259: 0x84, 0x25a: 0x84, 0x25b: 0x84, 0x25c: 0x84, 0x25d: 0x84, 0x25e: 0x84, 0x25f: 0x84, + 0x260: 0x84, 0x261: 0x84, 0x262: 0x84, 0x263: 0x84, 0x264: 0x84, 0x265: 0x84, 0x266: 0x84, 0x267: 0x84, + 0x268: 0x84, 0x269: 0x84, 0x26a: 0x84, 0x26b: 0x84, 0x26c: 0x84, 0x26d: 0x84, 0x26e: 0x84, 0x26f: 0x84, + 0x270: 0x84, 0x271: 0x84, 0x272: 0x84, 0x273: 0x84, 0x274: 0x84, 0x275: 0x84, 0x276: 0x84, 0x277: 0x84, + 0x278: 0x84, 0x279: 0x84, 0x27a: 0x84, 0x27b: 0x84, 0x27c: 0x84, 0x27d: 0x84, 0x27e: 0x84, 0x27f: 0x84, + // Block 0xa, offset 0x280 + 0x280: 0x84, 0x281: 0x84, 0x282: 0x84, 0x283: 0x84, 0x284: 0x84, 0x285: 0x84, 0x286: 0x84, 0x287: 0x84, + 0x288: 0x84, 0x289: 0x84, 0x28a: 0x84, 0x28b: 0x84, 0x28c: 0x84, 0x28d: 0x84, 0x28e: 0x84, 0x28f: 0x84, + 0x290: 0x84, 0x291: 0x84, 0x292: 0x84, 0x293: 0x84, 0x294: 0x84, 0x295: 0x84, 0x296: 0x84, 0x297: 0x84, + 0x298: 0x84, 0x299: 0x84, 0x29a: 0x84, 0x29b: 0x84, 0x29c: 0x84, 0x29d: 0x84, 0x29e: 0x85, + // Block 0xb, offset 0x2c0 + 0x2e4: 0x86, 0x2e5: 0x86, 0x2e6: 0x86, 0x2e7: 0x86, + 0x2e8: 0x87, 0x2e9: 0x88, 0x2ea: 0x86, 0x2eb: 0x89, 0x2ec: 0x8a, 0x2ed: 0x8b, 0x2ee: 0x8c, 0x2ef: 0x8d, + 0x2f0: 0x63, 0x2f1: 0x63, 0x2f2: 0x63, 0x2f3: 0x63, 0x2f4: 0x8e, 0x2f5: 0x8f, 0x2f6: 0x90, 0x2f7: 0x91, + 0x2f8: 0x92, 0x2f9: 0x93, 0x2fa: 0x63, 0x2fb: 0x94, 0x2fc: 0x95, 0x2fd: 0x63, 0x2fe: 0x77, 0x2ff: 0x96, + // Block 0xc, offset 0x300 + 0x307: 0x97, + 0x328: 0x98, + // Block 0xd, offset 0x340 + 0x341: 0x7b, 0x342: 0x99, + // Block 0xe, offset 0x380 + 0x385: 0x9a, 0x386: 0x9b, 0x387: 0x9c, + 0x389: 0x9d, + 0x390: 0x63, 0x391: 0x9e, 0x392: 0x9f, 0x393: 0xa0, 0x394: 0xa1, 0x395: 0xa2, 0x396: 0x63, 0x397: 0x63, + 0x398: 0x63, 0x399: 0x63, 0x39a: 0xa3, 0x39b: 0x63, 0x39c: 0x63, 0x39d: 0x63, 0x39e: 0x63, 0x39f: 0xa4, + // Block 0xf, offset 0x3c0 + 0x3c4: 0xa5, 0x3c5: 0xa6, 0x3c6: 0xa7, + 0x3c8: 0xa8, 0x3c9: 0xa9, + // Block 0x10, offset 0x400 + 0x420: 0x86, 0x421: 0x86, 0x422: 0x86, 0x423: 0x86, 0x424: 0x86, 0x425: 0x86, 0x426: 0x86, 0x427: 0x86, + 0x428: 0xaa, + // Block 0x11, offset 0x440 + 0x450: 0x0c, 0x451: 0x0d, + 0x45d: 0x0e, 0x45f: 0x0f, + 0x46f: 0x10, +} + +var charInfoTrie = trie{charInfoLookup[:], charInfoValues[:]} + +// Total size of tables: 78KB (80234 bytes) diff --git a/src/pkg/exp/norm/trie.go b/src/pkg/exp/norm/trie.go new file mode 100644 index 000000000..6b6540187 --- /dev/null +++ b/src/pkg/exp/norm/trie.go @@ -0,0 +1,234 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package norm + +type trie struct { + index []uint8 + values []uint16 +} + +const ( + t1 = 0x00 // 0000 0000 + tx = 0x80 // 1000 0000 + t2 = 0xC0 // 1100 0000 + t3 = 0xE0 // 1110 0000 + t4 = 0xF0 // 1111 0000 + t5 = 0xF8 // 1111 1000 + t6 = 0xFC // 1111 1100 + te = 0xFE // 1111 1110 + + maskx = 0x3F // 0011 1111 + mask2 = 0x1F // 0001 1111 + mask3 = 0x0F // 0000 1111 + mask4 = 0x07 // 0000 0111 +) + +// lookup returns the trie value for the first UTF-8 encoding in s and +// the width in bytes of this encoding. The size will be 0 if s does not +// hold enough bytes to complete the encoding. len(s) must be greater than 0. +func (t *trie) lookup(s []byte) (v uint16, sz int) { + c0 := s[0] + switch { + case c0 < tx: + return t.values[c0], 1 + case c0 < t2: + return 0, 1 + case c0 < t3: + if len(s) < 2 { + return 0, 0 + } + i := t.index[c0] + c1 := s[1] + if c1 < tx || t2 <= c1 { + return 0, 1 + } + o := uint16(i)<<6 + uint16(c1)&maskx + return t.values[o], 2 + case c0 < t4: + if len(s) < 3 { + return 0, 0 + } + i := t.index[c0] + c1 := s[1] + if c1 < tx || t2 <= c1 { + return 0, 1 + } + o := uint16(i)<<6 + uint16(c1)&maskx + i = t.index[o] + c2 := s[2] + if c2 < tx || t2 <= c2 { + return 0, 2 + } + o = uint16(i)<<6 + uint16(c2)&maskx + return t.values[o], 3 + case c0 < t5: + if len(s) < 4 { + return 0, 0 + } + i := t.index[c0] + c1 := s[1] + if c1 < tx || t2 <= c1 { + return 0, 1 + } + o := uint16(i)<<6 + uint16(c1)&maskx + i = t.index[o] + c2 := s[2] + if c2 < tx || t2 <= c2 { + return 0, 2 + } + o = uint16(i)<<6 + uint16(c2)&maskx + i = t.index[o] + c3 := s[3] + if c3 < tx || t2 <= c3 { + return 0, 3 + } + o = uint16(i)<<6 + uint16(c3)&maskx + return t.values[o], 4 + case c0 < t6: + if len(s) < 5 { + return 0, 0 + } + return 0, 5 + case c0 < te: + if len(s) < 6 { + return 0, 0 + } + return 0, 6 + } + // Illegal rune + return 0, 1 +} + +// lookupString returns the trie value for the first UTF-8 encoding in s and +// the width in bytes of this encoding. The size will be 0 if s does not +// hold enough bytes to complete the encoding. len(s) must be greater than 0. +func (t *trie) lookupString(s string) (v uint16, sz int) { + c0 := s[0] + switch { + case c0 < tx: + return t.values[c0], 1 + case c0 < t2: + return 0, 1 + case c0 < t3: + if len(s) < 2 { + return 0, 0 + } + i := t.index[c0] + c1 := s[1] + if c1 < tx || t2 <= c1 { + return 0, 1 + } + o := uint16(i)<<6 + uint16(c1)&maskx + return t.values[o], 2 + case c0 < t4: + if len(s) < 3 { + return 0, 0 + } + i := t.index[c0] + c1 := s[1] + if c1 < tx || t2 <= c1 { + return 0, 1 + } + o := uint16(i)<<6 + uint16(c1)&maskx + i = t.index[o] + c2 := s[2] + if c2 < tx || t2 <= c2 { + return 0, 2 + } + o = uint16(i)<<6 + uint16(c2)&maskx + return t.values[o], 3 + case c0 < t5: + if len(s) < 4 { + return 0, 0 + } + i := t.index[c0] + c1 := s[1] + if c1 < tx || t2 <= c1 { + return 0, 1 + } + o := uint16(i)<<6 + uint16(c1)&maskx + i = t.index[o] + c2 := s[2] + if c2 < tx || t2 <= c2 { + return 0, 2 + } + o = uint16(i)<<6 + uint16(c2)&maskx + i = t.index[o] + c3 := s[3] + if c3 < tx || t2 <= c3 { + return 0, 3 + } + o = uint16(i)<<6 + uint16(c3)&maskx + return t.values[o], 4 + case c0 < t6: + if len(s) < 5 { + return 0, 0 + } + return 0, 5 + case c0 < te: + if len(s) < 6 { + return 0, 0 + } + return 0, 6 + } + // Illegal rune + return 0, 1 +} + +// lookupUnsafe returns the trie value for the first UTF-8 encoding in s. +// s must hold a full encoding. +func (t *trie) lookupUnsafe(s []byte) uint16 { + c0 := s[0] + if c0 < tx { + return t.values[c0] + } + if c0 < t2 { + return 0 + } + i := t.index[c0] + o := uint16(i)<<6 + uint16(s[1])&maskx + if c0 < t3 { + return t.values[o] + } + i = t.index[o] + o = uint16(i)<<6 + uint16(s[2])&maskx + if c0 < t4 { + return t.values[o] + } + i = t.index[o] + o = uint16(i)<<6 + uint16(s[3])&maskx + if c0 < t5 { + return t.values[o] + } + return 0 +} + +// lookupStringUnsafe returns the trie value for the first UTF-8 encoding in s. +// s must hold a full encoding. +func (t *trie) lookupStringUnsafe(s string) uint16 { + c0 := s[0] + if c0 < tx { + return t.values[c0] + } + if c0 < t2 { + return 0 + } + i := t.index[c0] + o := uint16(i)<<6 + uint16(s[1])&maskx + if c0 < t3 { + return t.values[o] + } + i = t.index[o] + o = uint16(i)<<6 + uint16(s[2])&maskx + if c0 < t4 { + return t.values[o] + } + i = t.index[o] + o = uint16(i)<<6 + uint16(s[3])&maskx + if c0 < t5 { + return t.values[o] + } + return 0 +} diff --git a/src/pkg/exp/norm/trie_test.go b/src/pkg/exp/norm/trie_test.go new file mode 100644 index 000000000..ad87d972b --- /dev/null +++ b/src/pkg/exp/norm/trie_test.go @@ -0,0 +1,107 @@ +package norm + +import ( + "testing" + "utf8" +) + +// Test data is located in triedata_test.go; generated by maketesttables. +var testdata = testdataTrie + +// Test cases for illegal runes. +type trietest struct { + size int + bytes []byte +} + +var tests = []trietest{ + // illegal runes + {1, []byte{0x80}}, + {1, []byte{0xFF}}, + {1, []byte{t2, tx - 1}}, + {1, []byte{t2, t2}}, + {2, []byte{t3, tx, tx - 1}}, + {2, []byte{t3, tx, t2}}, + {1, []byte{t3, tx - 1, tx}}, + {3, []byte{t4, tx, tx, tx - 1}}, + {3, []byte{t4, tx, tx, t2}}, + {1, []byte{t4, t2, tx, tx - 1}}, + {2, []byte{t4, tx, t2, tx - 1}}, + + // short runes + {0, []byte{t2}}, + {0, []byte{t3, tx}}, + {0, []byte{t4, tx, tx}}, + {0, []byte{t5, tx, tx, tx}}, + {0, []byte{t6, tx, tx, tx, tx}}, +} + +func mkUtf8(rune int) ([]byte, int) { + var b [utf8.UTFMax]byte + sz := utf8.EncodeRune(b[:], rune) + return b[:sz], sz +} + +func TestLookup(t *testing.T) { + for i, tt := range testRunes { + b, szg := mkUtf8(tt) + v, szt := testdata.lookup(b) + if int(v) != i { + t.Errorf("lookup(%U): found value %#x, expected %#x", i, v, i) + } + if szt != szg { + t.Errorf("lookup(%U): found size %d, expected %d", i, szt, szg) + } + } + for i, tt := range tests { + v, sz := testdata.lookup(tt.bytes) + if int(v) != 0 { + t.Errorf("lookup of illegal rune, case %d: found value %#x, expected 0", i, v) + } + if sz != tt.size { + t.Errorf("lookup of illegal rune, case %d: found size %d, expected %d", i, sz, tt.size) + } + } +} + +func TestLookupUnsafe(t *testing.T) { + for i, tt := range testRunes { + b, _ := mkUtf8(tt) + v := testdata.lookupUnsafe(b) + if int(v) != i { + t.Errorf("lookupUnsafe(%U): found value %#x, expected %#x", i, v, i) + } + } +} + +func TestLookupString(t *testing.T) { + for i, tt := range testRunes { + b, szg := mkUtf8(tt) + v, szt := testdata.lookupString(string(b)) + if int(v) != i { + t.Errorf("lookup(%U): found value %#x, expected %#x", i, v, i) + } + if szt != szg { + t.Errorf("lookup(%U): found size %d, expected %d", i, szt, szg) + } + } + for i, tt := range tests { + v, sz := testdata.lookupString(string(tt.bytes)) + if int(v) != 0 { + t.Errorf("lookup of illegal rune, case %d: found value %#x, expected 0", i, v) + } + if sz != tt.size { + t.Errorf("lookup of illegal rune, case %d: found size %d, expected %d", i, sz, tt.size) + } + } +} + +func TestLookupStringUnsafe(t *testing.T) { + for i, tt := range testRunes { + b, _ := mkUtf8(tt) + v := testdata.lookupStringUnsafe(string(b)) + if int(v) != i { + t.Errorf("lookupUnsafe(%U): found value %#x, expected %#x", i, v, i) + } + } +} diff --git a/src/pkg/exp/norm/triedata_test.go b/src/pkg/exp/norm/triedata_test.go new file mode 100644 index 000000000..f886e6004 --- /dev/null +++ b/src/pkg/exp/norm/triedata_test.go @@ -0,0 +1,63 @@ +// Generated by running +// maketesttables +// DO NOT EDIT + +package norm + +var testRunes = []int{1, 12, 127, 128, 256, 2047, 2048, 2457, 65535, 65536, 65793, 1114111} + +// testdataValues: 768 entries, 1536 bytes +// Block 2 is the null block. +var testdataValues = [768]uint16{ + // Block 0x0, offset 0x0 + 0x000c: 0x0001, + // Block 0x1, offset 0x40 + 0x007f: 0x0002, + // Block 0x2, offset 0x80 + // Block 0x3, offset 0xc0 + 0x00c0: 0x0003, + // Block 0x4, offset 0x100 + 0x0100: 0x0004, + // Block 0x5, offset 0x140 + 0x017f: 0x0005, + // Block 0x6, offset 0x180 + 0x0180: 0x0006, + // Block 0x7, offset 0x1c0 + 0x01d9: 0x0007, + // Block 0x8, offset 0x200 + 0x023f: 0x0008, + // Block 0x9, offset 0x240 + 0x0240: 0x0009, + // Block 0xa, offset 0x280 + 0x0281: 0x000a, + // Block 0xb, offset 0x2c0 + 0x02ff: 0x000b, +} + +// testdataLookup: 640 bytes +// Block 0 is the null block. +var testdataLookup = [640]uint8{ + // Block 0x0, offset 0x0 + // Block 0x1, offset 0x40 + // Block 0x2, offset 0x80 + // Block 0x3, offset 0xc0 + 0x0c2: 0x03, 0x0c4: 0x04, + 0x0df: 0x05, + 0x0e0: 0x04, + 0x0ef: 0x05, + 0x0f0: 0x07, 0x0f4: 0x09, + // Block 0x4, offset 0x100 + 0x120: 0x06, 0x126: 0x07, + // Block 0x5, offset 0x140 + 0x17f: 0x08, + // Block 0x6, offset 0x180 + 0x180: 0x09, 0x184: 0x0a, + // Block 0x7, offset 0x1c0 + 0x1d0: 0x06, + // Block 0x8, offset 0x200 + 0x23f: 0x0b, + // Block 0x9, offset 0x240 + 0x24f: 0x08, +} + +var testdataTrie = trie{testdataLookup[:], testdataValues[:]} diff --git a/src/pkg/exp/norm/triegen.go b/src/pkg/exp/norm/triegen.go new file mode 100644 index 000000000..2b7eeee17 --- /dev/null +++ b/src/pkg/exp/norm/triegen.go @@ -0,0 +1,211 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Trie table generator. +// Used by make*tables tools to generate a go file with trie data structures +// for mapping UTF-8 to a 16-bit value. All but the last byte in a UTF-8 byte +// sequence are used to lookup offsets in the index table to be used for the +// next byte. The last byte is used to index into a table with 16-bit values. + +package main + +import ( + "fmt" + "hash/crc32" + "log" + "utf8" +) + +// Intermediate trie structure +type trieNode struct { + table [256]*trieNode + value uint16 + b byte + leaf bool +} + +func newNode() *trieNode { + return new(trieNode) +} + +func (n trieNode) String() string { + s := fmt.Sprint("trieNode{table: { non-nil at index: ") + for i, v := range n.table { + if v != nil { + s += fmt.Sprintf("%d, ", i) + } + } + s += fmt.Sprintf("}, value:%#x, b:%#x leaf:%v}", n.value, n.b, n.leaf) + return s +} + +func (n trieNode) isInternal() bool { + internal := true + for i := 0; i < 256; i++ { + if nn := n.table[i]; nn != nil { + if !internal && !nn.leaf { + log.Fatalf("triegen: isInternal: node contains both leaf and non-leaf children (%v)", n) + } + internal = internal && !nn.leaf + } + } + return internal +} + +func (n *trieNode) insert(rune int, value uint16) { + var p [utf8.UTFMax]byte + sz := utf8.EncodeRune(p[:], rune) + + for i := 0; i < sz; i++ { + if n.leaf { + log.Fatalf("triegen: insert: node (%#v) should not be a leaf", n) + } + nn := n.table[p[i]] + if nn == nil { + nn = newNode() + nn.b = p[i] + n.table[p[i]] = nn + } + n = nn + } + n.value = value + n.leaf = true +} + +type nodeIndex struct { + lookupBlocks []*trieNode + valueBlocks []*trieNode + + lookupBlockIdx map[uint32]uint16 + valueBlockIdx map[uint32]uint16 +} + +func newIndex() *nodeIndex { + index := &nodeIndex{} + index.lookupBlocks = make([]*trieNode, 0) + index.valueBlocks = make([]*trieNode, 0) + index.lookupBlockIdx = make(map[uint32]uint16) + index.valueBlockIdx = make(map[uint32]uint16) + return index +} + +func computeOffsets(index *nodeIndex, n *trieNode) uint16 { + if n.leaf { + return n.value + } + hasher := crc32.New(crc32.MakeTable(crc32.IEEE)) + // We only index continuation bytes. + for i := 0; i < 64; i++ { + var v uint16 = 0 + if nn := n.table[0x80+i]; nn != nil { + v = computeOffsets(index, nn) + } + hasher.Write([]byte{uint8(v >> 8), uint8(v)}) + } + h := hasher.Sum32() + if n.isInternal() { + v, ok := index.lookupBlockIdx[h] + if !ok { + v = uint16(len(index.lookupBlocks)) + index.lookupBlocks = append(index.lookupBlocks, n) + index.lookupBlockIdx[h] = v + } + n.value = v + } else { + v, ok := index.valueBlockIdx[h] + if !ok { + v = uint16(len(index.valueBlocks)) + index.valueBlocks = append(index.valueBlocks, n) + index.valueBlockIdx[h] = v + } + n.value = v + } + return n.value +} + +func printValueBlock(nr int, n *trieNode, offset int) { + boff := nr * 64 + fmt.Printf("\n// Block %#x, offset %#x", nr, boff) + var printnewline bool + for i := 0; i < 64; i++ { + if i%6 == 0 { + printnewline = true + } + v := uint16(0) + if nn := n.table[i+offset]; nn != nil { + v = nn.value + } + if v != 0 { + if printnewline { + fmt.Printf("\n") + printnewline = false + } + fmt.Printf("%#04x:%#04x, ", nr*64+i, v) + } + } +} + +func printLookupBlock(nr int, n *trieNode, offset int) { + boff := nr * 64 + fmt.Printf("\n// Block %#x, offset %#x", nr, boff) + var printnewline bool + for i := 0; i < 64; i++ { + if i%8 == 0 { + printnewline = true + } + v := uint16(0) + if nn := n.table[i+offset]; nn != nil { + v = nn.value + } + if v != 0 { + if printnewline { + fmt.Printf("\n") + printnewline = false + } + fmt.Printf("%#03x:%#02x, ", boff+i, v) + } + } +} + +// printTables returns the size in bytes of the generated tables. +func (t *trieNode) printTables(name string) int { + index := newIndex() + // Values for 7-bit ASCII are stored in first two block, followed by nil block. + index.valueBlocks = append(index.valueBlocks, nil, nil, nil) + // First byte of multi-byte UTF-8 codepoints are indexed in 4th block. + index.lookupBlocks = append(index.lookupBlocks, nil, nil, nil, nil) + // Index starter bytes of multi-byte UTF-8. + for i := 0xC0; i < 0x100; i++ { + if t.table[i] != nil { + computeOffsets(index, t.table[i]) + } + } + + nv := len(index.valueBlocks) * 64 + fmt.Printf("// %sValues: %d entries, %d bytes\n", name, nv, nv*2) + fmt.Printf("// Block 2 is the null block.\n") + fmt.Printf("var %sValues = [%d]uint16 {", name, nv) + printValueBlock(0, t, 0) + printValueBlock(1, t, 64) + printValueBlock(2, newNode(), 0) + for i := 3; i < len(index.valueBlocks); i++ { + printValueBlock(i, index.valueBlocks[i], 0x80) + } + fmt.Print("\n}\n\n") + + ni := len(index.lookupBlocks) * 64 + fmt.Printf("// %sLookup: %d bytes\n", name, ni) + fmt.Printf("// Block 0 is the null block.\n") + fmt.Printf("var %sLookup = [%d]uint8 {", name, ni) + printLookupBlock(0, newNode(), 0) + printLookupBlock(1, newNode(), 0) + printLookupBlock(2, newNode(), 0) + printLookupBlock(3, t, 0xC0) + for i := 4; i < len(index.lookupBlocks); i++ { + printLookupBlock(i, index.lookupBlocks[i], 0x80) + } + fmt.Print("\n}\n\n") + fmt.Printf("var %sTrie = trie{ %sLookup[:], %sValues[:] }\n\n", name, name, name) + return nv*2 + ni +} diff --git a/src/pkg/exp/regexp/Makefile b/src/pkg/exp/regexp/Makefile new file mode 100644 index 000000000..fc90df320 --- /dev/null +++ b/src/pkg/exp/regexp/Makefile @@ -0,0 +1,12 @@ +# Copyright 2011 The Go Authors. All rights reserved. +# Use of this source code is governed by a BSD-style +# license that can be found in the LICENSE file. + +include ../../../Make.inc + +TARG=exp/regexp +GOFILES=\ + exec.go\ + regexp.go\ + +include ../../../Make.pkg diff --git a/src/pkg/exp/regexp/all_test.go b/src/pkg/exp/regexp/all_test.go new file mode 100644 index 000000000..77f32ca1a --- /dev/null +++ b/src/pkg/exp/regexp/all_test.go @@ -0,0 +1,429 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package regexp + +import ( + "os" + "strings" + "testing" +) + +var good_re = []string{ + ``, + `.`, + `^.$`, + `a`, + `a*`, + `a+`, + `a?`, + `a|b`, + `a*|b*`, + `(a*|b)(c*|d)`, + `[a-z]`, + `[a-abc-c\-\]\[]`, + `[a-z]+`, + `[abc]`, + `[^1234]`, + `[^\n]`, + `\!\\`, +} + +/* +type stringError struct { + re string + err os.Error +} + +var bad_re = []stringError{ + {`*`, ErrBareClosure}, + {`+`, ErrBareClosure}, + {`?`, ErrBareClosure}, + {`(abc`, ErrUnmatchedLpar}, + {`abc)`, ErrUnmatchedRpar}, + {`x[a-z`, ErrUnmatchedLbkt}, + {`abc]`, ErrUnmatchedRbkt}, + {`[z-a]`, ErrBadRange}, + {`abc\`, ErrExtraneousBackslash}, + {`a**`, ErrBadClosure}, + {`a*+`, ErrBadClosure}, + {`a??`, ErrBadClosure}, + {`\x`, ErrBadBackslash}, +} +*/ + +func compileTest(t *testing.T, expr string, error os.Error) *Regexp { + re, err := Compile(expr) + if err != error { + t.Error("compiling `", expr, "`; unexpected error: ", err.String()) + } + return re +} + +func TestGoodCompile(t *testing.T) { + for i := 0; i < len(good_re); i++ { + compileTest(t, good_re[i], nil) + } +} + +/* +func TestBadCompile(t *testing.T) { + for i := 0; i < len(bad_re); i++ { + compileTest(t, bad_re[i].re, bad_re[i].err) + } +} +*/ + +func matchTest(t *testing.T, test *FindTest) { + re := compileTest(t, test.pat, nil) + if re == nil { + return + } + m := re.MatchString(test.text) + if m != (len(test.matches) > 0) { + t.Errorf("MatchString failure on %s: %t should be %t", test, m, len(test.matches) > 0) + } + // now try bytes + m = re.Match([]byte(test.text)) + if m != (len(test.matches) > 0) { + t.Errorf("Match failure on %s: %t should be %t", test, m, len(test.matches) > 0) + } +} + +func TestMatch(t *testing.T) { + for _, test := range findTests { + matchTest(t, &test) + } +} + +func matchFunctionTest(t *testing.T, test *FindTest) { + m, err := MatchString(test.pat, test.text) + if err == nil { + return + } + if m != (len(test.matches) > 0) { + t.Errorf("Match failure on %s: %t should be %t", test, m, len(test.matches) > 0) + } +} + +func TestMatchFunction(t *testing.T) { + for _, test := range findTests { + matchFunctionTest(t, &test) + } +} + +type ReplaceTest struct { + pattern, replacement, input, output string +} + +var replaceTests = []ReplaceTest{ + // Test empty input and/or replacement, with pattern that matches the empty string. + {"", "", "", ""}, + {"", "x", "", "x"}, + {"", "", "abc", "abc"}, + {"", "x", "abc", "xaxbxcx"}, + + // Test empty input and/or replacement, with pattern that does not match the empty string. + {"b", "", "", ""}, + {"b", "x", "", ""}, + {"b", "", "abc", "ac"}, + {"b", "x", "abc", "axc"}, + {"y", "", "", ""}, + {"y", "x", "", ""}, + {"y", "", "abc", "abc"}, + {"y", "x", "abc", "abc"}, + + // Multibyte characters -- verify that we don't try to match in the middle + // of a character. + {"[a-c]*", "x", "\u65e5", "x\u65e5x"}, + {"[^\u65e5]", "x", "abc\u65e5def", "xxx\u65e5xxx"}, + + // Start and end of a string. + {"^[a-c]*", "x", "abcdabc", "xdabc"}, + {"[a-c]*$", "x", "abcdabc", "abcdx"}, + {"^[a-c]*$", "x", "abcdabc", "abcdabc"}, + {"^[a-c]*", "x", "abc", "x"}, + {"[a-c]*$", "x", "abc", "x"}, + {"^[a-c]*$", "x", "abc", "x"}, + {"^[a-c]*", "x", "dabce", "xdabce"}, + {"[a-c]*$", "x", "dabce", "dabcex"}, + {"^[a-c]*$", "x", "dabce", "dabce"}, + {"^[a-c]*", "x", "", "x"}, + {"[a-c]*$", "x", "", "x"}, + {"^[a-c]*$", "x", "", "x"}, + + {"^[a-c]+", "x", "abcdabc", "xdabc"}, + {"[a-c]+$", "x", "abcdabc", "abcdx"}, + {"^[a-c]+$", "x", "abcdabc", "abcdabc"}, + {"^[a-c]+", "x", "abc", "x"}, + {"[a-c]+$", "x", "abc", "x"}, + {"^[a-c]+$", "x", "abc", "x"}, + {"^[a-c]+", "x", "dabce", "dabce"}, + {"[a-c]+$", "x", "dabce", "dabce"}, + {"^[a-c]+$", "x", "dabce", "dabce"}, + {"^[a-c]+", "x", "", ""}, + {"[a-c]+$", "x", "", ""}, + {"^[a-c]+$", "x", "", ""}, + + // Other cases. + {"abc", "def", "abcdefg", "defdefg"}, + {"bc", "BC", "abcbcdcdedef", "aBCBCdcdedef"}, + {"abc", "", "abcdabc", "d"}, + {"x", "xXx", "xxxXxxx", "xXxxXxxXxXxXxxXxxXx"}, + {"abc", "d", "", ""}, + {"abc", "d", "abc", "d"}, + {".+", "x", "abc", "x"}, + {"[a-c]*", "x", "def", "xdxexfx"}, + {"[a-c]+", "x", "abcbcdcdedef", "xdxdedef"}, + {"[a-c]*", "x", "abcbcdcdedef", "xdxdxexdxexfx"}, +} + +type ReplaceFuncTest struct { + pattern string + replacement func(string) string + input, output string +} + +var replaceFuncTests = []ReplaceFuncTest{ + {"[a-c]", func(s string) string { return "x" + s + "y" }, "defabcdef", "defxayxbyxcydef"}, + {"[a-c]+", func(s string) string { return "x" + s + "y" }, "defabcdef", "defxabcydef"}, + {"[a-c]*", func(s string) string { return "x" + s + "y" }, "defabcdef", "xydxyexyfxabcydxyexyfxy"}, +} + +func TestReplaceAll(t *testing.T) { + for _, tc := range replaceTests { + re, err := Compile(tc.pattern) + if err != nil { + t.Errorf("Unexpected error compiling %q: %v", tc.pattern, err) + continue + } + actual := re.ReplaceAllString(tc.input, tc.replacement) + if actual != tc.output { + t.Errorf("%q.Replace(%q,%q) = %q; want %q", + tc.pattern, tc.input, tc.replacement, actual, tc.output) + } + // now try bytes + actual = string(re.ReplaceAll([]byte(tc.input), []byte(tc.replacement))) + if actual != tc.output { + t.Errorf("%q.Replace(%q,%q) = %q; want %q", + tc.pattern, tc.input, tc.replacement, actual, tc.output) + } + } +} + +func TestReplaceAllFunc(t *testing.T) { + for _, tc := range replaceFuncTests { + re, err := Compile(tc.pattern) + if err != nil { + t.Errorf("Unexpected error compiling %q: %v", tc.pattern, err) + continue + } + actual := re.ReplaceAllStringFunc(tc.input, tc.replacement) + if actual != tc.output { + t.Errorf("%q.ReplaceFunc(%q,%q) = %q; want %q", + tc.pattern, tc.input, tc.replacement, actual, tc.output) + } + // now try bytes + actual = string(re.ReplaceAllFunc([]byte(tc.input), func(s []byte) []byte { return []byte(tc.replacement(string(s))) })) + if actual != tc.output { + t.Errorf("%q.ReplaceFunc(%q,%q) = %q; want %q", + tc.pattern, tc.input, tc.replacement, actual, tc.output) + } + } +} + +type MetaTest struct { + pattern, output, literal string + isLiteral bool +} + +var metaTests = []MetaTest{ + {``, ``, ``, true}, + {`foo`, `foo`, `foo`, true}, + {`foo\.\$`, `foo\\\.\\\$`, `foo.$`, true}, // has meta but no operator + {`foo.\$`, `foo\.\\\$`, `foo`, false}, // has escaped operators and real operators + {`!@#$%^&*()_+-=[{]}\|,<.>/?~`, `!@#\$%\^&\*\(\)_\+-=\[\{\]\}\\\|,<\.>/\?~`, `!@#`, false}, +} + +func TestQuoteMeta(t *testing.T) { + for _, tc := range metaTests { + // Verify that QuoteMeta returns the expected string. + quoted := QuoteMeta(tc.pattern) + if quoted != tc.output { + t.Errorf("QuoteMeta(`%s`) = `%s`; want `%s`", + tc.pattern, quoted, tc.output) + continue + } + + // Verify that the quoted string is in fact treated as expected + // by Compile -- i.e. that it matches the original, unquoted string. + if tc.pattern != "" { + re, err := Compile(quoted) + if err != nil { + t.Errorf("Unexpected error compiling QuoteMeta(`%s`): %v", tc.pattern, err) + continue + } + src := "abc" + tc.pattern + "def" + repl := "xyz" + replaced := re.ReplaceAllString(src, repl) + expected := "abcxyzdef" + if replaced != expected { + t.Errorf("QuoteMeta(`%s`).Replace(`%s`,`%s`) = `%s`; want `%s`", + tc.pattern, src, repl, replaced, expected) + } + } + } +} + +func TestLiteralPrefix(t *testing.T) { + for _, tc := range metaTests { + // Literal method needs to scan the pattern. + re := MustCompile(tc.pattern) + str, complete := re.LiteralPrefix() + if complete != tc.isLiteral { + t.Errorf("LiteralPrefix(`%s`) = %t; want %t", tc.pattern, complete, tc.isLiteral) + } + if str != tc.literal { + t.Errorf("LiteralPrefix(`%s`) = `%s`; want `%s`", tc.pattern, str, tc.literal) + } + } +} + +type numSubexpCase struct { + input string + expected int +} + +var numSubexpCases = []numSubexpCase{ + {``, 0}, + {`.*`, 0}, + {`abba`, 0}, + {`ab(b)a`, 1}, + {`ab(.*)a`, 1}, + {`(.*)ab(.*)a`, 2}, + {`(.*)(ab)(.*)a`, 3}, + {`(.*)((a)b)(.*)a`, 4}, + {`(.*)(\(ab)(.*)a`, 3}, + {`(.*)(\(a\)b)(.*)a`, 3}, +} + +func TestNumSubexp(t *testing.T) { + for _, c := range numSubexpCases { + re := MustCompile(c.input) + n := re.NumSubexp() + if n != c.expected { + t.Errorf("NumSubexp for %q returned %d, expected %d", c.input, n, c.expected) + } + } +} + +func BenchmarkLiteral(b *testing.B) { + x := strings.Repeat("x", 50) + "y" + b.StopTimer() + re := MustCompile("y") + b.StartTimer() + for i := 0; i < b.N; i++ { + if !re.MatchString(x) { + println("no match!") + break + } + } +} + +func BenchmarkNotLiteral(b *testing.B) { + x := strings.Repeat("x", 50) + "y" + b.StopTimer() + re := MustCompile(".y") + b.StartTimer() + for i := 0; i < b.N; i++ { + if !re.MatchString(x) { + println("no match!") + break + } + } +} + +func BenchmarkMatchClass(b *testing.B) { + b.StopTimer() + x := strings.Repeat("xxxx", 20) + "w" + re := MustCompile("[abcdw]") + b.StartTimer() + for i := 0; i < b.N; i++ { + if !re.MatchString(x) { + println("no match!") + break + } + } +} + +func BenchmarkMatchClass_InRange(b *testing.B) { + b.StopTimer() + // 'b' is between 'a' and 'c', so the charclass + // range checking is no help here. + x := strings.Repeat("bbbb", 20) + "c" + re := MustCompile("[ac]") + b.StartTimer() + for i := 0; i < b.N; i++ { + if !re.MatchString(x) { + println("no match!") + break + } + } +} + +func BenchmarkReplaceAll(b *testing.B) { + x := "abcdefghijklmnopqrstuvwxyz" + b.StopTimer() + re := MustCompile("[cjrw]") + b.StartTimer() + for i := 0; i < b.N; i++ { + re.ReplaceAllString(x, "") + } +} + +func BenchmarkAnchoredLiteralShortNonMatch(b *testing.B) { + b.StopTimer() + x := []byte("abcdefghijklmnopqrstuvwxyz") + re := MustCompile("^zbc(d|e)") + b.StartTimer() + for i := 0; i < b.N; i++ { + re.Match(x) + } +} + +func BenchmarkAnchoredLiteralLongNonMatch(b *testing.B) { + b.StopTimer() + x := []byte("abcdefghijklmnopqrstuvwxyz") + for i := 0; i < 15; i++ { + x = append(x, x...) + } + re := MustCompile("^zbc(d|e)") + b.StartTimer() + for i := 0; i < b.N; i++ { + re.Match(x) + } +} + +func BenchmarkAnchoredShortMatch(b *testing.B) { + b.StopTimer() + x := []byte("abcdefghijklmnopqrstuvwxyz") + re := MustCompile("^.bc(d|e)") + b.StartTimer() + for i := 0; i < b.N; i++ { + re.Match(x) + } +} + +func BenchmarkAnchoredLongMatch(b *testing.B) { + b.StopTimer() + x := []byte("abcdefghijklmnopqrstuvwxyz") + for i := 0; i < 15; i++ { + x = append(x, x...) + } + re := MustCompile("^.bc(d|e)") + b.StartTimer() + for i := 0; i < b.N; i++ { + re.Match(x) + } +} diff --git a/src/pkg/exp/regexp/exec.go b/src/pkg/exp/regexp/exec.go new file mode 100644 index 000000000..0670bb9b1 --- /dev/null +++ b/src/pkg/exp/regexp/exec.go @@ -0,0 +1,295 @@ +package regexp + +import "exp/regexp/syntax" + +// A queue is a 'sparse array' holding pending threads of execution. +// See http://research.swtch.com/2008/03/using-uninitialized-memory-for-fun-and.html +type queue struct { + sparse []uint32 + dense []entry +} + +// A entry is an entry on a queue. +// It holds both the instruction pc and the actual thread. +// Some queue entries are just place holders so that the machine +// knows it has considered that pc. Such entries have t == nil. +type entry struct { + pc uint32 + t *thread +} + +// A thread is the state of a single path through the machine: +// an instruction and a corresponding capture array. +// See http://swtch.com/~rsc/regexp/regexp2.html +type thread struct { + inst *syntax.Inst + cap []int +} + +// A machine holds all the state during an NFA simulation for p. +type machine struct { + re *Regexp // corresponding Regexp + p *syntax.Prog // compiled program + q0, q1 queue // two queues for runq, nextq + pool []*thread // pool of available threads + matched bool // whether a match was found + matchcap []int // capture information for the match +} + +// progMachine returns a new machine running the prog p. +func progMachine(p *syntax.Prog) *machine { + m := &machine{p: p} + n := len(m.p.Inst) + m.q0 = queue{make([]uint32, n), make([]entry, 0, n)} + m.q1 = queue{make([]uint32, n), make([]entry, 0, n)} + ncap := p.NumCap + if ncap < 2 { + ncap = 2 + } + m.matchcap = make([]int, ncap) + return m +} + +// alloc allocates a new thread with the given instruction. +// It uses the free pool if possible. +func (m *machine) alloc(i *syntax.Inst) *thread { + var t *thread + if n := len(m.pool); n > 0 { + t = m.pool[n-1] + m.pool = m.pool[:n-1] + } else { + t = new(thread) + t.cap = make([]int, cap(m.matchcap)) + } + t.cap = t.cap[:len(m.matchcap)] + t.inst = i + return t +} + +// free returns t to the free pool. +func (m *machine) free(t *thread) { + m.pool = append(m.pool, t) +} + +// match runs the machine over the input starting at pos. +// It reports whether a match was found. +// If so, m.matchcap holds the submatch information. +func (m *machine) match(i input, pos int) bool { + startCond := m.re.cond + if startCond == ^syntax.EmptyOp(0) { // impossible + return false + } + m.matched = false + for i := range m.matchcap { + m.matchcap[i] = -1 + } + runq, nextq := &m.q0, &m.q1 + rune, rune1 := endOfText, endOfText + width, width1 := 0, 0 + rune, width = i.step(pos) + if rune != endOfText { + rune1, width1 = i.step(pos + width) + } + // TODO: Let caller specify the initial flag setting. + // For now assume pos == 0 is beginning of text and + // pos != 0 is not even beginning of line. + // TODO: Word boundary. + var flag syntax.EmptyOp + if pos == 0 { + flag = syntax.EmptyBeginText | syntax.EmptyBeginLine + } + + // Update flag using lookahead rune. + if rune1 == '\n' { + flag |= syntax.EmptyEndLine + } + if rune1 == endOfText { + flag |= syntax.EmptyEndText + } + + for { + if len(runq.dense) == 0 { + if startCond&syntax.EmptyBeginText != 0 && pos != 0 { + // Anchored match, past beginning of text. + break + } + if m.matched { + // Have match; finished exploring alternatives. + break + } + if len(m.re.prefix) > 0 && rune1 != m.re.prefixRune && i.canCheckPrefix() { + // Match requires literal prefix; fast search for it. + advance := i.index(m.re, pos) + if advance < 0 { + break + } + pos += advance + rune, width = i.step(pos) + rune1, width1 = i.step(pos + width) + } + } + if !m.matched { + if len(m.matchcap) > 0 { + m.matchcap[0] = pos + } + m.add(runq, uint32(m.p.Start), pos, m.matchcap, flag) + } + // TODO: word boundary + flag = 0 + if rune == '\n' { + flag |= syntax.EmptyBeginLine + } + if rune1 == '\n' { + flag |= syntax.EmptyEndLine + } + if rune1 == endOfText { + flag |= syntax.EmptyEndText + } + m.step(runq, nextq, pos, pos+width, rune, flag) + if width == 0 { + break + } + pos += width + rune, width = rune1, width1 + if rune != endOfText { + rune1, width1 = i.step(pos + width) + } + runq, nextq = nextq, runq + } + m.clear(nextq) + return m.matched +} + +// clear frees all threads on the thread queue. +func (m *machine) clear(q *queue) { + for _, d := range q.dense { + if d.t != nil { + m.free(d.t) + } + } + q.dense = q.dense[:0] +} + +// step executes one step of the machine, running each of the threads +// on runq and appending new threads to nextq. +// The step processes the rune c (which may be endOfText), +// which starts at position pos and ends at nextPos. +// nextCond gives the setting for the empty-width flags after c. +func (m *machine) step(runq, nextq *queue, pos, nextPos, c int, nextCond syntax.EmptyOp) { + for j := 0; j < len(runq.dense); j++ { + d := &runq.dense[j] + t := d.t + if t == nil { + continue + } + /* + * If we support leftmost-longest matching: + if longest && matched && match[0] < t.cap[0] { + m.free(t) + continue + } + */ + + i := t.inst + switch i.Op { + default: + panic("bad inst") + + case syntax.InstMatch: + if len(t.cap) > 0 { + t.cap[1] = pos + copy(m.matchcap, t.cap) + } + m.matched = true + for _, d := range runq.dense[j+1:] { + if d.t != nil { + m.free(d.t) + } + } + runq.dense = runq.dense[:0] + + case syntax.InstRune: + if i.MatchRune(c) { + m.add(nextq, i.Out, nextPos, t.cap, nextCond) + } + } + m.free(t) + } + runq.dense = runq.dense[:0] +} + +// add adds an entry to q for pc, unless the q already has such an entry. +// It also recursively adds an entry for all instructions reachable from pc by following +// empty-width conditions satisfied by cond. pos gives the current position +// in the input. +func (m *machine) add(q *queue, pc uint32, pos int, cap []int, cond syntax.EmptyOp) { + if pc == 0 { + return + } + if j := q.sparse[pc]; j < uint32(len(q.dense)) && q.dense[j].pc == pc { + return + } + + j := len(q.dense) + q.dense = q.dense[:j+1] + d := &q.dense[j] + d.t = nil + d.pc = pc + q.sparse[pc] = uint32(j) + + i := &m.p.Inst[pc] + switch i.Op { + default: + panic("unhandled") + case syntax.InstFail: + // nothing + case syntax.InstAlt, syntax.InstAltMatch: + m.add(q, i.Out, pos, cap, cond) + m.add(q, i.Arg, pos, cap, cond) + case syntax.InstEmptyWidth: + if syntax.EmptyOp(i.Arg)&^cond == 0 { + m.add(q, i.Out, pos, cap, cond) + } + case syntax.InstNop: + m.add(q, i.Out, pos, cap, cond) + case syntax.InstCapture: + if int(i.Arg) < len(cap) { + opos := cap[i.Arg] + cap[i.Arg] = pos + m.add(q, i.Out, pos, cap, cond) + cap[i.Arg] = opos + } else { + m.add(q, i.Out, pos, cap, cond) + } + case syntax.InstMatch, syntax.InstRune: + t := m.alloc(i) + if len(t.cap) > 0 { + copy(t.cap, cap) + } + d.t = t + } +} + +// empty is a non-nil 0-element slice, +// so doExecute can avoid an allocation +// when 0 captures are requested from a successful match. +var empty = make([]int, 0) + +// doExecute finds the leftmost match in the input and returns +// the position of its subexpressions. +func (re *Regexp) doExecute(i input, pos int, ncap int) []int { + m := re.get() + m.matchcap = m.matchcap[:ncap] + if !m.match(i, pos) { + re.put(m) + return nil + } + if ncap == 0 { + re.put(m) + return empty // empty but not nil + } + cap := make([]int, ncap) + copy(cap, m.matchcap) + re.put(m) + return cap +} diff --git a/src/pkg/exp/regexp/find_test.go b/src/pkg/exp/regexp/find_test.go new file mode 100644 index 000000000..dddc3484c --- /dev/null +++ b/src/pkg/exp/regexp/find_test.go @@ -0,0 +1,472 @@ +// Copyright 2010 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package regexp + +import ( + "fmt" + "strings" + "testing" +) + +// For each pattern/text pair, what is the expected output of each function? +// We can derive the textual results from the indexed results, the non-submatch +// results from the submatched results, the single results from the 'all' results, +// and the byte results from the string results. Therefore the table includes +// only the FindAllStringSubmatchIndex result. +type FindTest struct { + pat string + text string + matches [][]int +} + +func (t FindTest) String() string { + return fmt.Sprintf("pat: %#q text: %#q", t.pat, t.text) +} + +var findTests = []FindTest{ + {``, ``, build(1, 0, 0)}, + {`^abcdefg`, "abcdefg", build(1, 0, 7)}, + {`a+`, "baaab", build(1, 1, 4)}, + {"abcd..", "abcdef", build(1, 0, 6)}, + {`a`, "a", build(1, 0, 1)}, + {`x`, "y", nil}, + {`b`, "abc", build(1, 1, 2)}, + {`.`, "a", build(1, 0, 1)}, + {`.*`, "abcdef", build(1, 0, 6)}, + {`^`, "abcde", build(1, 0, 0)}, + {`$`, "abcde", build(1, 5, 5)}, + {`^abcd$`, "abcd", build(1, 0, 4)}, + {`^bcd'`, "abcdef", nil}, + {`^abcd$`, "abcde", nil}, + {`a+`, "baaab", build(1, 1, 4)}, + {`a*`, "baaab", build(3, 0, 0, 1, 4, 5, 5)}, + {`[a-z]+`, "abcd", build(1, 0, 4)}, + {`[^a-z]+`, "ab1234cd", build(1, 2, 6)}, + {`[a\-\]z]+`, "az]-bcz", build(2, 0, 4, 6, 7)}, + {`[^\n]+`, "abcd\n", build(1, 0, 4)}, + {`[日本語]+`, "日本語日本語", build(1, 0, 18)}, + {`日本語+`, "日本語", build(1, 0, 9)}, + {`日本語+`, "日本語語語語", build(1, 0, 18)}, + {`()`, "", build(1, 0, 0, 0, 0)}, + {`(a)`, "a", build(1, 0, 1, 0, 1)}, + {`(.)(.)`, "日a", build(1, 0, 4, 0, 3, 3, 4)}, + {`(.*)`, "", build(1, 0, 0, 0, 0)}, + {`(.*)`, "abcd", build(1, 0, 4, 0, 4)}, + {`(..)(..)`, "abcd", build(1, 0, 4, 0, 2, 2, 4)}, + {`(([^xyz]*)(d))`, "abcd", build(1, 0, 4, 0, 4, 0, 3, 3, 4)}, + {`((a|b|c)*(d))`, "abcd", build(1, 0, 4, 0, 4, 2, 3, 3, 4)}, + {`(((a|b|c)*)(d))`, "abcd", build(1, 0, 4, 0, 4, 0, 3, 2, 3, 3, 4)}, + {`\a\f\n\r\t\v`, "\a\f\n\r\t\v", build(1, 0, 6)}, + {`[\a\f\n\r\t\v]+`, "\a\f\n\r\t\v", build(1, 0, 6)}, + + {`a*(|(b))c*`, "aacc", build(1, 0, 4, 2, 2, -1, -1)}, + {`(.*).*`, "ab", build(1, 0, 2, 0, 2)}, + {`[.]`, ".", build(1, 0, 1)}, + {`/$`, "/abc/", build(1, 4, 5)}, + {`/$`, "/abc", nil}, + + // multiple matches + {`.`, "abc", build(3, 0, 1, 1, 2, 2, 3)}, + {`(.)`, "abc", build(3, 0, 1, 0, 1, 1, 2, 1, 2, 2, 3, 2, 3)}, + {`.(.)`, "abcd", build(2, 0, 2, 1, 2, 2, 4, 3, 4)}, + {`ab*`, "abbaab", build(3, 0, 3, 3, 4, 4, 6)}, + {`a(b*)`, "abbaab", build(3, 0, 3, 1, 3, 3, 4, 4, 4, 4, 6, 5, 6)}, + + // fixed bugs + {`ab$`, "cab", build(1, 1, 3)}, + {`axxb$`, "axxcb", nil}, + {`data`, "daXY data", build(1, 5, 9)}, + {`da(.)a$`, "daXY data", build(1, 5, 9, 7, 8)}, + {`zx+`, "zzx", build(1, 1, 3)}, + + // can backslash-escape any punctuation + {`\!\"\#\$\%\&\'\(\)\*\+\,\-\.\/\:\;\<\=\>\?\@\[\\\]\^\_\{\|\}\~`, + `!"#$%&'()*+,-./:;<=>?@[\]^_{|}~`, build(1, 0, 31)}, + {`[\!\"\#\$\%\&\'\(\)\*\+\,\-\.\/\:\;\<\=\>\?\@\[\\\]\^\_\{\|\}\~]+`, + `!"#$%&'()*+,-./:;<=>?@[\]^_{|}~`, build(1, 0, 31)}, + {"\\`", "`", build(1, 0, 1)}, + {"[\\`]+", "`", build(1, 0, 1)}, + + // long set of matches (longer than startSize) + { + ".", + "qwertyuiopasdfghjklzxcvbnm1234567890", + build(36, 0, 1, 1, 2, 2, 3, 3, 4, 4, 5, 5, 6, 6, 7, 7, 8, 8, 9, 9, 10, + 10, 11, 11, 12, 12, 13, 13, 14, 14, 15, 15, 16, 16, 17, 17, 18, 18, 19, 19, 20, + 20, 21, 21, 22, 22, 23, 23, 24, 24, 25, 25, 26, 26, 27, 27, 28, 28, 29, 29, 30, + 30, 31, 31, 32, 32, 33, 33, 34, 34, 35, 35, 36), + }, +} + +// build is a helper to construct a [][]int by extracting n sequences from x. +// This represents n matches with len(x)/n submatches each. +func build(n int, x ...int) [][]int { + ret := make([][]int, n) + runLength := len(x) / n + j := 0 + for i := range ret { + ret[i] = make([]int, runLength) + copy(ret[i], x[j:]) + j += runLength + if j > len(x) { + panic("invalid build entry") + } + } + return ret +} + +// First the simple cases. + +func TestFind(t *testing.T) { + for _, test := range findTests { + re := MustCompile(test.pat) + if re.String() != test.pat { + t.Errorf("String() = `%s`; should be `%s`", re.String(), test.pat) + } + result := re.Find([]byte(test.text)) + switch { + case len(test.matches) == 0 && len(result) == 0: + // ok + case test.matches == nil && result != nil: + t.Errorf("expected no match; got one: %s", test) + case test.matches != nil && result == nil: + t.Errorf("expected match; got none: %s", test) + case test.matches != nil && result != nil: + expect := test.text[test.matches[0][0]:test.matches[0][1]] + if expect != string(result) { + t.Errorf("expected %q got %q: %s", expect, result, test) + } + } + } +} + +func TestFindString(t *testing.T) { + for _, test := range findTests { + result := MustCompile(test.pat).FindString(test.text) + switch { + case len(test.matches) == 0 && len(result) == 0: + // ok + case test.matches == nil && result != "": + t.Errorf("expected no match; got one: %s", test) + case test.matches != nil && result == "": + // Tricky because an empty result has two meanings: no match or empty match. + if test.matches[0][0] != test.matches[0][1] { + t.Errorf("expected match; got none: %s", test) + } + case test.matches != nil && result != "": + expect := test.text[test.matches[0][0]:test.matches[0][1]] + if expect != result { + t.Errorf("expected %q got %q: %s", expect, result, test) + } + } + } +} + +func testFindIndex(test *FindTest, result []int, t *testing.T) { + switch { + case len(test.matches) == 0 && len(result) == 0: + // ok + case test.matches == nil && result != nil: + t.Errorf("expected no match; got one: %s", test) + case test.matches != nil && result == nil: + t.Errorf("expected match; got none: %s", test) + case test.matches != nil && result != nil: + expect := test.matches[0] + if expect[0] != result[0] || expect[1] != result[1] { + t.Errorf("expected %v got %v: %s", expect, result, test) + } + } +} + +func TestFindIndex(t *testing.T) { + for _, test := range findTests { + testFindIndex(&test, MustCompile(test.pat).FindIndex([]byte(test.text)), t) + } +} + +func TestFindStringIndex(t *testing.T) { + for _, test := range findTests { + testFindIndex(&test, MustCompile(test.pat).FindStringIndex(test.text), t) + } +} + +func TestFindReaderIndex(t *testing.T) { + for _, test := range findTests { + testFindIndex(&test, MustCompile(test.pat).FindReaderIndex(strings.NewReader(test.text)), t) + } +} + +// Now come the simple All cases. + +func TestFindAll(t *testing.T) { + for _, test := range findTests { + result := MustCompile(test.pat).FindAll([]byte(test.text), -1) + switch { + case test.matches == nil && result == nil: + // ok + case test.matches == nil && result != nil: + t.Errorf("expected no match; got one: %s", test) + case test.matches != nil && result == nil: + t.Fatalf("expected match; got none: %s", test) + case test.matches != nil && result != nil: + if len(test.matches) != len(result) { + t.Errorf("expected %d matches; got %d: %s", len(test.matches), len(result), test) + continue + } + for k, e := range test.matches { + expect := test.text[e[0]:e[1]] + if expect != string(result[k]) { + t.Errorf("match %d: expected %q got %q: %s", k, expect, result[k], test) + } + } + } + } +} + +func TestFindAllString(t *testing.T) { + for _, test := range findTests { + result := MustCompile(test.pat).FindAllString(test.text, -1) + switch { + case test.matches == nil && result == nil: + // ok + case test.matches == nil && result != nil: + t.Errorf("expected no match; got one: %s", test) + case test.matches != nil && result == nil: + t.Errorf("expected match; got none: %s", test) + case test.matches != nil && result != nil: + if len(test.matches) != len(result) { + t.Errorf("expected %d matches; got %d: %s", len(test.matches), len(result), test) + continue + } + for k, e := range test.matches { + expect := test.text[e[0]:e[1]] + if expect != result[k] { + t.Errorf("expected %q got %q: %s", expect, result, test) + } + } + } + } +} + +func testFindAllIndex(test *FindTest, result [][]int, t *testing.T) { + switch { + case test.matches == nil && result == nil: + // ok + case test.matches == nil && result != nil: + t.Errorf("expected no match; got one: %s", test) + case test.matches != nil && result == nil: + t.Errorf("expected match; got none: %s", test) + case test.matches != nil && result != nil: + if len(test.matches) != len(result) { + t.Errorf("expected %d matches; got %d: %s", len(test.matches), len(result), test) + return + } + for k, e := range test.matches { + if e[0] != result[k][0] || e[1] != result[k][1] { + t.Errorf("match %d: expected %v got %v: %s", k, e, result[k], test) + } + } + } +} + +func TestFindAllIndex(t *testing.T) { + for _, test := range findTests { + testFindAllIndex(&test, MustCompile(test.pat).FindAllIndex([]byte(test.text), -1), t) + } +} + +func TestFindAllStringIndex(t *testing.T) { + for _, test := range findTests { + testFindAllIndex(&test, MustCompile(test.pat).FindAllStringIndex(test.text, -1), t) + } +} + +// Now come the Submatch cases. + +func testSubmatchBytes(test *FindTest, n int, submatches []int, result [][]byte, t *testing.T) { + if len(submatches) != len(result)*2 { + t.Errorf("match %d: expected %d submatches; got %d: %s", n, len(submatches)/2, len(result), test) + return + } + for k := 0; k < len(submatches); k += 2 { + if submatches[k] == -1 { + if result[k/2] != nil { + t.Errorf("match %d: expected nil got %q: %s", n, result, test) + } + continue + } + expect := test.text[submatches[k]:submatches[k+1]] + if expect != string(result[k/2]) { + t.Errorf("match %d: expected %q got %q: %s", n, expect, result, test) + return + } + } +} + +func TestFindSubmatch(t *testing.T) { + for _, test := range findTests { + result := MustCompile(test.pat).FindSubmatch([]byte(test.text)) + switch { + case test.matches == nil && result == nil: + // ok + case test.matches == nil && result != nil: + t.Errorf("expected no match; got one: %s", test) + case test.matches != nil && result == nil: + t.Errorf("expected match; got none: %s", test) + case test.matches != nil && result != nil: + testSubmatchBytes(&test, 0, test.matches[0], result, t) + } + } +} + +func testSubmatchString(test *FindTest, n int, submatches []int, result []string, t *testing.T) { + if len(submatches) != len(result)*2 { + t.Errorf("match %d: expected %d submatches; got %d: %s", n, len(submatches)/2, len(result), test) + return + } + for k := 0; k < len(submatches); k += 2 { + if submatches[k] == -1 { + if result[k/2] != "" { + t.Errorf("match %d: expected nil got %q: %s", n, result, test) + } + continue + } + expect := test.text[submatches[k]:submatches[k+1]] + if expect != result[k/2] { + t.Errorf("match %d: expected %q got %q: %s", n, expect, result, test) + return + } + } +} + +func TestFindStringSubmatch(t *testing.T) { + for _, test := range findTests { + result := MustCompile(test.pat).FindStringSubmatch(test.text) + switch { + case test.matches == nil && result == nil: + // ok + case test.matches == nil && result != nil: + t.Errorf("expected no match; got one: %s", test) + case test.matches != nil && result == nil: + t.Errorf("expected match; got none: %s", test) + case test.matches != nil && result != nil: + testSubmatchString(&test, 0, test.matches[0], result, t) + } + } +} + +func testSubmatchIndices(test *FindTest, n int, expect, result []int, t *testing.T) { + if len(expect) != len(result) { + t.Errorf("match %d: expected %d matches; got %d: %s", n, len(expect)/2, len(result)/2, test) + return + } + for k, e := range expect { + if e != result[k] { + t.Errorf("match %d: submatch error: expected %v got %v: %s", n, expect, result, test) + } + } +} + +func testFindSubmatchIndex(test *FindTest, result []int, t *testing.T) { + switch { + case test.matches == nil && result == nil: + // ok + case test.matches == nil && result != nil: + t.Errorf("expected no match; got one: %s", test) + case test.matches != nil && result == nil: + t.Errorf("expected match; got none: %s", test) + case test.matches != nil && result != nil: + testSubmatchIndices(test, 0, test.matches[0], result, t) + } +} + +func TestFindSubmatchIndex(t *testing.T) { + for _, test := range findTests { + testFindSubmatchIndex(&test, MustCompile(test.pat).FindSubmatchIndex([]byte(test.text)), t) + } +} + +func TestFindStringSubmatchIndex(t *testing.T) { + for _, test := range findTests { + testFindSubmatchIndex(&test, MustCompile(test.pat).FindStringSubmatchIndex(test.text), t) + } +} + +func TestFindReaderSubmatchIndex(t *testing.T) { + for _, test := range findTests { + testFindSubmatchIndex(&test, MustCompile(test.pat).FindReaderSubmatchIndex(strings.NewReader(test.text)), t) + } +} + +// Now come the monster AllSubmatch cases. + +func TestFindAllSubmatch(t *testing.T) { + for _, test := range findTests { + result := MustCompile(test.pat).FindAllSubmatch([]byte(test.text), -1) + switch { + case test.matches == nil && result == nil: + // ok + case test.matches == nil && result != nil: + t.Errorf("expected no match; got one: %s", test) + case test.matches != nil && result == nil: + t.Errorf("expected match; got none: %s", test) + case len(test.matches) != len(result): + t.Errorf("expected %d matches; got %d: %s", len(test.matches), len(result), test) + case test.matches != nil && result != nil: + for k, match := range test.matches { + testSubmatchBytes(&test, k, match, result[k], t) + } + } + } +} + +func TestFindAllStringSubmatch(t *testing.T) { + for _, test := range findTests { + result := MustCompile(test.pat).FindAllStringSubmatch(test.text, -1) + switch { + case test.matches == nil && result == nil: + // ok + case test.matches == nil && result != nil: + t.Errorf("expected no match; got one: %s", test) + case test.matches != nil && result == nil: + t.Errorf("expected match; got none: %s", test) + case len(test.matches) != len(result): + t.Errorf("expected %d matches; got %d: %s", len(test.matches), len(result), test) + case test.matches != nil && result != nil: + for k, match := range test.matches { + testSubmatchString(&test, k, match, result[k], t) + } + } + } +} + +func testFindAllSubmatchIndex(test *FindTest, result [][]int, t *testing.T) { + switch { + case test.matches == nil && result == nil: + // ok + case test.matches == nil && result != nil: + t.Errorf("expected no match; got one: %s", test) + case test.matches != nil && result == nil: + t.Errorf("expected match; got none: %s", test) + case len(test.matches) != len(result): + t.Errorf("expected %d matches; got %d: %s", len(test.matches), len(result), test) + case test.matches != nil && result != nil: + for k, match := range test.matches { + testSubmatchIndices(test, k, match, result[k], t) + } + } +} + +func TestFindAllSubmatchIndex(t *testing.T) { + for _, test := range findTests { + testFindAllSubmatchIndex(&test, MustCompile(test.pat).FindAllSubmatchIndex([]byte(test.text), -1), t) + } +} + +func TestFindAllStringSubmatchIndex(t *testing.T) { + for _, test := range findTests { + testFindAllSubmatchIndex(&test, MustCompile(test.pat).FindAllStringSubmatchIndex(test.text, -1), t) + } +} diff --git a/src/pkg/exp/regexp/regexp.go b/src/pkg/exp/regexp/regexp.go new file mode 100644 index 000000000..1b75900f8 --- /dev/null +++ b/src/pkg/exp/regexp/regexp.go @@ -0,0 +1,795 @@ +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Package regexp implements a simple regular expression library. +// +// The syntax of the regular expressions accepted is the same +// general syntax used by Perl, Python, and other languages. +// More precisely, it is the syntax accepted by RE2 and described at +// http://code.google.com/p/re2/wiki/Syntax, except for \C. +// +// All characters are UTF-8-encoded code points. +// +// There are 16 methods of Regexp that match a regular expression and identify +// the matched text. Their names are matched by this regular expression: +// +// Find(All)?(String)?(Submatch)?(Index)? +// +// If 'All' is present, the routine matches successive non-overlapping +// matches of the entire expression. Empty matches abutting a preceding +// match are ignored. The return value is a slice containing the successive +// return values of the corresponding non-'All' routine. These routines take +// an extra integer argument, n; if n >= 0, the function returns at most n +// matches/submatches. +// +// If 'String' is present, the argument is a string; otherwise it is a slice +// of bytes; return values are adjusted as appropriate. +// +// If 'Submatch' is present, the return value is a slice identifying the +// successive submatches of the expression. Submatches are matches of +// parenthesized subexpressions within the regular expression, numbered from +// left to right in order of opening parenthesis. Submatch 0 is the match of +// the entire expression, submatch 1 the match of the first parenthesized +// subexpression, and so on. +// +// If 'Index' is present, matches and submatches are identified by byte index +// pairs within the input string: result[2*n:2*n+1] identifies the indexes of +// the nth submatch. The pair for n==0 identifies the match of the entire +// expression. If 'Index' is not present, the match is identified by the +// text of the match/submatch. If an index is negative, it means that +// subexpression did not match any string in the input. +// +// There is also a subset of the methods that can be applied to text read +// from a RuneReader: +// +// MatchReader, FindReaderIndex, FindReaderSubmatchIndex +// +// This set may grow. Note that regular expression matches may need to +// examine text beyond the text returned by a match, so the methods that +// match text from a RuneReader may read arbitrarily far into the input +// before returning. +// +// (There are a few other methods that do not match this pattern.) +// +package regexp + +import ( + "bytes" + "exp/regexp/syntax" + "io" + "os" + "strings" + "sync" + "utf8" +) + +var debug = false + +// Error is the local type for a parsing error. +type Error string + +func (e Error) String() string { + return string(e) +} + +// Regexp is the representation of a compiled regular expression. +// The public interface is entirely through methods. +// A Regexp is safe for concurrent use by multiple goroutines. +type Regexp struct { + // read-only after Compile + expr string // as passed to Compile + prog *syntax.Prog // compiled program + prefix string // required prefix in unanchored matches + prefixBytes []byte // prefix, as a []byte + prefixComplete bool // prefix is the entire regexp + prefixRune int // first rune in prefix + cond syntax.EmptyOp // empty-width conditions required at start of match + + // cache of machines for running regexp + mu sync.Mutex + machine []*machine +} + +// String returns the source text used to compile the regular expression. +func (re *Regexp) String() string { + return re.expr +} + +// Compile parses a regular expression and returns, if successful, a Regexp +// object that can be used to match against text. +func Compile(expr string) (*Regexp, os.Error) { + re, err := syntax.Parse(expr, syntax.Perl) + if err != nil { + return nil, err + } + prog, err := syntax.Compile(re) + if err != nil { + return nil, err + } + regexp := &Regexp{ + expr: expr, + prog: prog, + } + regexp.prefix, regexp.prefixComplete = prog.Prefix() + if regexp.prefix != "" { + // TODO(rsc): Remove this allocation by adding + // IndexString to package bytes. + regexp.prefixBytes = []byte(regexp.prefix) + regexp.prefixRune, _ = utf8.DecodeRuneInString(regexp.prefix) + } + regexp.cond = prog.StartCond() + return regexp, nil +} + +// get returns a machine to use for matching re. +// It uses the re's machine cache if possible, to avoid +// unnecessary allocation. +func (re *Regexp) get() *machine { + re.mu.Lock() + if n := len(re.machine); n > 0 { + z := re.machine[n-1] + re.machine = re.machine[:n-1] + re.mu.Unlock() + return z + } + re.mu.Unlock() + z := progMachine(re.prog) + z.re = re + return z +} + +// put returns a machine to the re's machine cache. +// There is no attempt to limit the size of the cache, so it will +// grow to the maximum number of simultaneous matches +// run using re. (The cache empties when re gets garbage collected.) +func (re *Regexp) put(z *machine) { + re.mu.Lock() + re.machine = append(re.machine, z) + re.mu.Unlock() +} + +// MustCompile is like Compile but panics if the expression cannot be parsed. +// It simplifies safe initialization of global variables holding compiled regular +// expressions. +func MustCompile(str string) *Regexp { + regexp, error := Compile(str) + if error != nil { + panic(`regexp: compiling "` + str + `": ` + error.String()) + } + return regexp +} + +// NumSubexp returns the number of parenthesized subexpressions in this Regexp. +func (re *Regexp) NumSubexp() int { + // NumCap/2 because captures count ( and ) separately. + // -1 because NumCap counts $0 but NumSubexp does not. + return re.prog.NumCap/2 - 1 +} + +const endOfText = -1 + +// input abstracts different representations of the input text. It provides +// one-character lookahead. +type input interface { + step(pos int) (rune int, width int) // advance one rune + canCheckPrefix() bool // can we look ahead without losing info? + hasPrefix(re *Regexp) bool + index(re *Regexp, pos int) int +} + +// inputString scans a string. +type inputString struct { + str string +} + +func newInputString(str string) *inputString { + return &inputString{str: str} +} + +func (i *inputString) step(pos int) (int, int) { + if pos < len(i.str) { + return utf8.DecodeRuneInString(i.str[pos:len(i.str)]) + } + return endOfText, 0 +} + +func (i *inputString) canCheckPrefix() bool { + return true +} + +func (i *inputString) hasPrefix(re *Regexp) bool { + return strings.HasPrefix(i.str, re.prefix) +} + +func (i *inputString) index(re *Regexp, pos int) int { + return strings.Index(i.str[pos:], re.prefix) +} + +// inputBytes scans a byte slice. +type inputBytes struct { + str []byte +} + +func newInputBytes(str []byte) *inputBytes { + return &inputBytes{str: str} +} + +func (i *inputBytes) step(pos int) (int, int) { + if pos < len(i.str) { + return utf8.DecodeRune(i.str[pos:len(i.str)]) + } + return endOfText, 0 +} + +func (i *inputBytes) canCheckPrefix() bool { + return true +} + +func (i *inputBytes) hasPrefix(re *Regexp) bool { + return bytes.HasPrefix(i.str, re.prefixBytes) +} + +func (i *inputBytes) index(re *Regexp, pos int) int { + return bytes.Index(i.str[pos:], re.prefixBytes) +} + +// inputReader scans a RuneReader. +type inputReader struct { + r io.RuneReader + atEOT bool + pos int +} + +func newInputReader(r io.RuneReader) *inputReader { + return &inputReader{r: r} +} + +func (i *inputReader) step(pos int) (int, int) { + if !i.atEOT && pos != i.pos { + return endOfText, 0 + + } + r, w, err := i.r.ReadRune() + if err != nil { + i.atEOT = true + return endOfText, 0 + } + i.pos += w + return r, w +} + +func (i *inputReader) canCheckPrefix() bool { + return false +} + +func (i *inputReader) hasPrefix(re *Regexp) bool { + return false +} + +func (i *inputReader) index(re *Regexp, pos int) int { + return -1 +} + +// LiteralPrefix returns a literal string that must begin any match +// of the regular expression re. It returns the boolean true if the +// literal string comprises the entire regular expression. +func (re *Regexp) LiteralPrefix() (prefix string, complete bool) { + return re.prefix, re.prefixComplete +} + +// MatchReader returns whether the Regexp matches the text read by the +// RuneReader. The return value is a boolean: true for match, false for no +// match. +func (re *Regexp) MatchReader(r io.RuneReader) bool { + return re.doExecute(newInputReader(r), 0, 0) != nil +} + +// MatchString returns whether the Regexp matches the string s. +// The return value is a boolean: true for match, false for no match. +func (re *Regexp) MatchString(s string) bool { + return re.doExecute(newInputString(s), 0, 0) != nil +} + +// Match returns whether the Regexp matches the byte slice b. +// The return value is a boolean: true for match, false for no match. +func (re *Regexp) Match(b []byte) bool { + return re.doExecute(newInputBytes(b), 0, 0) != nil +} + +// MatchReader checks whether a textual regular expression matches the text +// read by the RuneReader. More complicated queries need to use Compile and +// the full Regexp interface. +func MatchReader(pattern string, r io.RuneReader) (matched bool, error os.Error) { + re, err := Compile(pattern) + if err != nil { + return false, err + } + return re.MatchReader(r), nil +} + +// MatchString checks whether a textual regular expression +// matches a string. More complicated queries need +// to use Compile and the full Regexp interface. +func MatchString(pattern string, s string) (matched bool, error os.Error) { + re, err := Compile(pattern) + if err != nil { + return false, err + } + return re.MatchString(s), nil +} + +// Match checks whether a textual regular expression +// matches a byte slice. More complicated queries need +// to use Compile and the full Regexp interface. +func Match(pattern string, b []byte) (matched bool, error os.Error) { + re, err := Compile(pattern) + if err != nil { + return false, err + } + return re.Match(b), nil +} + +// ReplaceAllString returns a copy of src in which all matches for the Regexp +// have been replaced by repl. No support is provided for expressions +// (e.g. \1 or $1) in the replacement string. +func (re *Regexp) ReplaceAllString(src, repl string) string { + return re.ReplaceAllStringFunc(src, func(string) string { return repl }) +} + +// ReplaceAllStringFunc returns a copy of src in which all matches for the +// Regexp have been replaced by the return value of of function repl (whose +// first argument is the matched string). No support is provided for +// expressions (e.g. \1 or $1) in the replacement string. +func (re *Regexp) ReplaceAllStringFunc(src string, repl func(string) string) string { + lastMatchEnd := 0 // end position of the most recent match + searchPos := 0 // position where we next look for a match + buf := new(bytes.Buffer) + for searchPos <= len(src) { + a := re.doExecute(newInputString(src), searchPos, 2) + if len(a) == 0 { + break // no more matches + } + + // Copy the unmatched characters before this match. + io.WriteString(buf, src[lastMatchEnd:a[0]]) + + // Now insert a copy of the replacement string, but not for a + // match of the empty string immediately after another match. + // (Otherwise, we get double replacement for patterns that + // match both empty and nonempty strings.) + if a[1] > lastMatchEnd || a[0] == 0 { + io.WriteString(buf, repl(src[a[0]:a[1]])) + } + lastMatchEnd = a[1] + + // Advance past this match; always advance at least one character. + _, width := utf8.DecodeRuneInString(src[searchPos:]) + if searchPos+width > a[1] { + searchPos += width + } else if searchPos+1 > a[1] { + // This clause is only needed at the end of the input + // string. In that case, DecodeRuneInString returns width=0. + searchPos++ + } else { + searchPos = a[1] + } + } + + // Copy the unmatched characters after the last match. + io.WriteString(buf, src[lastMatchEnd:]) + + return buf.String() +} + +// ReplaceAll returns a copy of src in which all matches for the Regexp +// have been replaced by repl. No support is provided for expressions +// (e.g. \1 or $1) in the replacement text. +func (re *Regexp) ReplaceAll(src, repl []byte) []byte { + return re.ReplaceAllFunc(src, func([]byte) []byte { return repl }) +} + +// ReplaceAllFunc returns a copy of src in which all matches for the +// Regexp have been replaced by the return value of of function repl (whose +// first argument is the matched []byte). No support is provided for +// expressions (e.g. \1 or $1) in the replacement string. +func (re *Regexp) ReplaceAllFunc(src []byte, repl func([]byte) []byte) []byte { + lastMatchEnd := 0 // end position of the most recent match + searchPos := 0 // position where we next look for a match + buf := new(bytes.Buffer) + for searchPos <= len(src) { + a := re.doExecute(newInputBytes(src), searchPos, 2) + if len(a) == 0 { + break // no more matches + } + + // Copy the unmatched characters before this match. + buf.Write(src[lastMatchEnd:a[0]]) + + // Now insert a copy of the replacement string, but not for a + // match of the empty string immediately after another match. + // (Otherwise, we get double replacement for patterns that + // match both empty and nonempty strings.) + if a[1] > lastMatchEnd || a[0] == 0 { + buf.Write(repl(src[a[0]:a[1]])) + } + lastMatchEnd = a[1] + + // Advance past this match; always advance at least one character. + _, width := utf8.DecodeRune(src[searchPos:]) + if searchPos+width > a[1] { + searchPos += width + } else if searchPos+1 > a[1] { + // This clause is only needed at the end of the input + // string. In that case, DecodeRuneInString returns width=0. + searchPos++ + } else { + searchPos = a[1] + } + } + + // Copy the unmatched characters after the last match. + buf.Write(src[lastMatchEnd:]) + + return buf.Bytes() +} + +var specialBytes = []byte(`\.+*?()|[]{}^$`) + +func special(b byte) bool { + return bytes.IndexByte(specialBytes, b) >= 0 +} + +// QuoteMeta returns a string that quotes all regular expression metacharacters +// inside the argument text; the returned string is a regular expression matching +// the literal text. For example, QuoteMeta(`[foo]`) returns `\[foo\]`. +func QuoteMeta(s string) string { + b := make([]byte, 2*len(s)) + + // A byte loop is correct because all metacharacters are ASCII. + j := 0 + for i := 0; i < len(s); i++ { + if special(s[i]) { + b[j] = '\\' + j++ + } + b[j] = s[i] + j++ + } + return string(b[0:j]) +} + +// Find matches in slice b if b is non-nil, otherwise find matches in string s. +func (re *Regexp) allMatches(s string, b []byte, n int, deliver func([]int)) { + var end int + if b == nil { + end = len(s) + } else { + end = len(b) + } + + for pos, i, prevMatchEnd := 0, 0, -1; i < n && pos <= end; { + var in input + if b == nil { + in = newInputString(s) + } else { + in = newInputBytes(b) + } + matches := re.doExecute(in, pos, re.prog.NumCap) + if len(matches) == 0 { + break + } + + accept := true + if matches[1] == pos { + // We've found an empty match. + if matches[0] == prevMatchEnd { + // We don't allow an empty match right + // after a previous match, so ignore it. + accept = false + } + var width int + // TODO: use step() + if b == nil { + _, width = utf8.DecodeRuneInString(s[pos:end]) + } else { + _, width = utf8.DecodeRune(b[pos:end]) + } + if width > 0 { + pos += width + } else { + pos = end + 1 + } + } else { + pos = matches[1] + } + prevMatchEnd = matches[1] + + if accept { + deliver(matches) + i++ + } + } +} + +// Find returns a slice holding the text of the leftmost match in b of the regular expression. +// A return value of nil indicates no match. +func (re *Regexp) Find(b []byte) []byte { + a := re.doExecute(newInputBytes(b), 0, 2) + if a == nil { + return nil + } + return b[a[0]:a[1]] +} + +// FindIndex returns a two-element slice of integers defining the location of +// the leftmost match in b of the regular expression. The match itself is at +// b[loc[0]:loc[1]]. +// A return value of nil indicates no match. +func (re *Regexp) FindIndex(b []byte) (loc []int) { + a := re.doExecute(newInputBytes(b), 0, 2) + if a == nil { + return nil + } + return a[0:2] +} + +// FindString returns a string holding the text of the leftmost match in s of the regular +// expression. If there is no match, the return value is an empty string, +// but it will also be empty if the regular expression successfully matches +// an empty string. Use FindStringIndex or FindStringSubmatch if it is +// necessary to distinguish these cases. +func (re *Regexp) FindString(s string) string { + a := re.doExecute(newInputString(s), 0, 2) + if a == nil { + return "" + } + return s[a[0]:a[1]] +} + +// FindStringIndex returns a two-element slice of integers defining the +// location of the leftmost match in s of the regular expression. The match +// itself is at s[loc[0]:loc[1]]. +// A return value of nil indicates no match. +func (re *Regexp) FindStringIndex(s string) []int { + a := re.doExecute(newInputString(s), 0, 2) + if a == nil { + return nil + } + return a[0:2] +} + +// FindReaderIndex returns a two-element slice of integers defining the +// location of the leftmost match of the regular expression in text read from +// the RuneReader. The match itself is at s[loc[0]:loc[1]]. A return +// value of nil indicates no match. +func (re *Regexp) FindReaderIndex(r io.RuneReader) []int { + a := re.doExecute(newInputReader(r), 0, 2) + if a == nil { + return nil + } + return a[0:2] +} + +// FindSubmatch returns a slice of slices holding the text of the leftmost +// match of the regular expression in b and the matches, if any, of its +// subexpressions, as defined by the 'Submatch' descriptions in the package +// comment. +// A return value of nil indicates no match. +func (re *Regexp) FindSubmatch(b []byte) [][]byte { + a := re.doExecute(newInputBytes(b), 0, re.prog.NumCap) + if a == nil { + return nil + } + ret := make([][]byte, len(a)/2) + for i := range ret { + if a[2*i] >= 0 { + ret[i] = b[a[2*i]:a[2*i+1]] + } + } + return ret +} + +// FindSubmatchIndex returns a slice holding the index pairs identifying the +// leftmost match of the regular expression in b and the matches, if any, of +// its subexpressions, as defined by the 'Submatch' and 'Index' descriptions +// in the package comment. +// A return value of nil indicates no match. +func (re *Regexp) FindSubmatchIndex(b []byte) []int { + return re.doExecute(newInputBytes(b), 0, re.prog.NumCap) +} + +// FindStringSubmatch returns a slice of strings holding the text of the +// leftmost match of the regular expression in s and the matches, if any, of +// its subexpressions, as defined by the 'Submatch' description in the +// package comment. +// A return value of nil indicates no match. +func (re *Regexp) FindStringSubmatch(s string) []string { + a := re.doExecute(newInputString(s), 0, re.prog.NumCap) + if a == nil { + return nil + } + ret := make([]string, len(a)/2) + for i := range ret { + if a[2*i] >= 0 { + ret[i] = s[a[2*i]:a[2*i+1]] + } + } + return ret +} + +// FindStringSubmatchIndex returns a slice holding the index pairs +// identifying the leftmost match of the regular expression in s and the +// matches, if any, of its subexpressions, as defined by the 'Submatch' and +// 'Index' descriptions in the package comment. +// A return value of nil indicates no match. +func (re *Regexp) FindStringSubmatchIndex(s string) []int { + return re.doExecute(newInputString(s), 0, re.prog.NumCap) +} + +// FindReaderSubmatchIndex returns a slice holding the index pairs +// identifying the leftmost match of the regular expression of text read by +// the RuneReader, and the matches, if any, of its subexpressions, as defined +// by the 'Submatch' and 'Index' descriptions in the package comment. A +// return value of nil indicates no match. +func (re *Regexp) FindReaderSubmatchIndex(r io.RuneReader) []int { + return re.doExecute(newInputReader(r), 0, re.prog.NumCap) +} + +const startSize = 10 // The size at which to start a slice in the 'All' routines. + +// FindAll is the 'All' version of Find; it returns a slice of all successive +// matches of the expression, as defined by the 'All' description in the +// package comment. +// A return value of nil indicates no match. +func (re *Regexp) FindAll(b []byte, n int) [][]byte { + if n < 0 { + n = len(b) + 1 + } + result := make([][]byte, 0, startSize) + re.allMatches("", b, n, func(match []int) { + result = append(result, b[match[0]:match[1]]) + }) + if len(result) == 0 { + return nil + } + return result +} + +// FindAllIndex is the 'All' version of FindIndex; it returns a slice of all +// successive matches of the expression, as defined by the 'All' description +// in the package comment. +// A return value of nil indicates no match. +func (re *Regexp) FindAllIndex(b []byte, n int) [][]int { + if n < 0 { + n = len(b) + 1 + } + result := make([][]int, 0, startSize) + re.allMatches("", b, n, func(match []int) { + result = append(result, match[0:2]) + }) + if len(result) == 0 { + return nil + } + return result +} + +// FindAllString is the 'All' version of FindString; it returns a slice of all +// successive matches of the expression, as defined by the 'All' description +// in the package comment. +// A return value of nil indicates no match. +func (re *Regexp) FindAllString(s string, n int) []string { + if n < 0 { + n = len(s) + 1 + } + result := make([]string, 0, startSize) + re.allMatches(s, nil, n, func(match []int) { + result = append(result, s[match[0]:match[1]]) + }) + if len(result) == 0 { + return nil + } + return result +} + +// FindAllStringIndex is the 'All' version of FindStringIndex; it returns a +// slice of all successive matches of the expression, as defined by the 'All' +// description in the package comment. +// A return value of nil indicates no match. +func (re *Regexp) FindAllStringIndex(s string, n int) [][]int { + if n < 0 { + n = len(s) + 1 + } + result := make([][]int, 0, startSize) + re.allMatches(s, nil, n, func(match []int) { + result = append(result, match[0:2]) + }) + if len(result) == 0 { + return nil + } + return result +} + +// FindAllSubmatch is the 'All' version of FindSubmatch; it returns a slice +// of all successive matches of the expression, as defined by the 'All' +// description in the package comment. +// A return value of nil indicates no match. +func (re *Regexp) FindAllSubmatch(b []byte, n int) [][][]byte { + if n < 0 { + n = len(b) + 1 + } + result := make([][][]byte, 0, startSize) + re.allMatches("", b, n, func(match []int) { + slice := make([][]byte, len(match)/2) + for j := range slice { + if match[2*j] >= 0 { + slice[j] = b[match[2*j]:match[2*j+1]] + } + } + result = append(result, slice) + }) + if len(result) == 0 { + return nil + } + return result +} + +// FindAllSubmatchIndex is the 'All' version of FindSubmatchIndex; it returns +// a slice of all successive matches of the expression, as defined by the +// 'All' description in the package comment. +// A return value of nil indicates no match. +func (re *Regexp) FindAllSubmatchIndex(b []byte, n int) [][]int { + if n < 0 { + n = len(b) + 1 + } + result := make([][]int, 0, startSize) + re.allMatches("", b, n, func(match []int) { + result = append(result, match) + }) + if len(result) == 0 { + return nil + } + return result +} + +// FindAllStringSubmatch is the 'All' version of FindStringSubmatch; it +// returns a slice of all successive matches of the expression, as defined by +// the 'All' description in the package comment. +// A return value of nil indicates no match. +func (re *Regexp) FindAllStringSubmatch(s string, n int) [][]string { + if n < 0 { + n = len(s) + 1 + } + result := make([][]string, 0, startSize) + re.allMatches(s, nil, n, func(match []int) { + slice := make([]string, len(match)/2) + for j := range slice { + if match[2*j] >= 0 { + slice[j] = s[match[2*j]:match[2*j+1]] + } + } + result = append(result, slice) + }) + if len(result) == 0 { + return nil + } + return result +} + +// FindAllStringSubmatchIndex is the 'All' version of +// FindStringSubmatchIndex; it returns a slice of all successive matches of +// the expression, as defined by the 'All' description in the package +// comment. +// A return value of nil indicates no match. +func (re *Regexp) FindAllStringSubmatchIndex(s string, n int) [][]int { + if n < 0 { + n = len(s) + 1 + } + result := make([][]int, 0, startSize) + re.allMatches(s, nil, n, func(match []int) { + result = append(result, match) + }) + if len(result) == 0 { + return nil + } + return result +} diff --git a/src/pkg/exp/regexp/syntax/Makefile b/src/pkg/exp/regexp/syntax/Makefile new file mode 100644 index 000000000..97d4ad6ca --- /dev/null +++ b/src/pkg/exp/regexp/syntax/Makefile @@ -0,0 +1,16 @@ +# Copyright 2011 The Go Authors. All rights reserved. +# Use of this source code is governed by a BSD-style +# license that can be found in the LICENSE file. + +include ../../../../Make.inc + +TARG=exp/regexp/syntax +GOFILES=\ + compile.go\ + parse.go\ + perl_groups.go\ + prog.go\ + regexp.go\ + simplify.go\ + +include ../../../../Make.pkg diff --git a/src/pkg/exp/regexp/syntax/compile.go b/src/pkg/exp/regexp/syntax/compile.go new file mode 100644 index 000000000..5ea2425c3 --- /dev/null +++ b/src/pkg/exp/regexp/syntax/compile.go @@ -0,0 +1,269 @@ +package syntax + +import ( + "os" + "unicode" +) + +// A patchList is a list of instruction pointers that need to be filled in (patched). +// Because the pointers haven't been filled in yet, we can reuse their storage +// to hold the list. It's kind of sleazy, but works well in practice. +// See http://swtch.com/~rsc/regexp/regexp1.html for inspiration. +// +// These aren't really pointers: they're integers, so we can reinterpret them +// this way without using package unsafe. A value l denotes +// p.inst[l>>1].Out (l&1==0) or .Arg (l&1==1). +// l == 0 denotes the empty list, okay because we start every program +// with a fail instruction, so we'll never want to point at its output link. +type patchList uint32 + +func (l patchList) next(p *Prog) patchList { + i := &p.Inst[l>>1] + if l&1 == 0 { + return patchList(i.Out) + } + return patchList(i.Arg) +} + +func (l patchList) patch(p *Prog, val uint32) { + for l != 0 { + i := &p.Inst[l>>1] + if l&1 == 0 { + l = patchList(i.Out) + i.Out = val + } else { + l = patchList(i.Arg) + i.Arg = val + } + } +} + +func (l1 patchList) append(p *Prog, l2 patchList) patchList { + if l1 == 0 { + return l2 + } + if l2 == 0 { + return l1 + } + + last := l1 + for { + next := last.next(p) + if next == 0 { + break + } + last = next + } + + i := &p.Inst[last>>1] + if last&1 == 0 { + i.Out = uint32(l2) + } else { + i.Arg = uint32(l2) + } + return l1 +} + +// A frag represents a compiled program fragment. +type frag struct { + i uint32 // index of first instruction + out patchList // where to record end instruction +} + +type compiler struct { + p *Prog +} + +// Compile compiles the regexp into a program to be executed. +func Compile(re *Regexp) (*Prog, os.Error) { + var c compiler + c.init() + f := c.compile(re) + f.out.patch(c.p, c.inst(InstMatch).i) + c.p.Start = int(f.i) + return c.p, nil +} + +func (c *compiler) init() { + c.p = new(Prog) + c.p.NumCap = 2 // implicit ( and ) for whole match $0 + c.inst(InstFail) +} + +var anyRuneNotNL = []int{0, '\n' - 1, '\n' - 1, unicode.MaxRune} +var anyRune = []int{0, unicode.MaxRune} + +func (c *compiler) compile(re *Regexp) frag { + switch re.Op { + case OpNoMatch: + return c.fail() + case OpEmptyMatch: + return c.nop() + case OpLiteral: + if len(re.Rune) == 0 { + return c.nop() + } + var f frag + for j := range re.Rune { + f1 := c.rune(re.Rune[j : j+1]) + if j == 0 { + f = f1 + } else { + f = c.cat(f, f1) + } + } + return f + case OpCharClass: + return c.rune(re.Rune) + case OpAnyCharNotNL: + return c.rune(anyRuneNotNL) + case OpAnyChar: + return c.rune(anyRune) + case OpBeginLine: + return c.empty(EmptyBeginLine) + case OpEndLine: + return c.empty(EmptyEndLine) + case OpBeginText: + return c.empty(EmptyBeginText) + case OpEndText: + return c.empty(EmptyEndText) + case OpWordBoundary: + return c.empty(EmptyWordBoundary) + case OpNoWordBoundary: + return c.empty(EmptyNoWordBoundary) + case OpCapture: + bra := c.cap(uint32(re.Cap << 1)) + sub := c.compile(re.Sub[0]) + ket := c.cap(uint32(re.Cap<<1 | 1)) + return c.cat(c.cat(bra, sub), ket) + case OpStar: + return c.star(c.compile(re.Sub[0]), re.Flags&NonGreedy != 0) + case OpPlus: + return c.plus(c.compile(re.Sub[0]), re.Flags&NonGreedy != 0) + case OpQuest: + return c.quest(c.compile(re.Sub[0]), re.Flags&NonGreedy != 0) + case OpConcat: + if len(re.Sub) == 0 { + return c.nop() + } + var f frag + for i, sub := range re.Sub { + if i == 0 { + f = c.compile(sub) + } else { + f = c.cat(f, c.compile(sub)) + } + } + return f + case OpAlternate: + var f frag + for _, sub := range re.Sub { + f = c.alt(f, c.compile(sub)) + } + return f + } + panic("regexp: unhandled case in compile") +} + +func (c *compiler) inst(op InstOp) frag { + // TODO: impose length limit + f := frag{i: uint32(len(c.p.Inst))} + c.p.Inst = append(c.p.Inst, Inst{Op: op}) + return f +} + +func (c *compiler) nop() frag { + f := c.inst(InstNop) + f.out = patchList(f.i << 1) + return f +} + +func (c *compiler) fail() frag { + return frag{} +} + +func (c *compiler) cap(arg uint32) frag { + f := c.inst(InstCapture) + f.out = patchList(f.i << 1) + c.p.Inst[f.i].Arg = arg + + if c.p.NumCap < int(arg)+1 { + c.p.NumCap = int(arg) + 1 + } + return f +} + +func (c *compiler) cat(f1, f2 frag) frag { + // concat of failure is failure + if f1.i == 0 || f2.i == 0 { + return frag{} + } + + // TODO: elide nop + + f1.out.patch(c.p, f2.i) + return frag{f1.i, f2.out} +} + +func (c *compiler) alt(f1, f2 frag) frag { + // alt of failure is other + if f1.i == 0 { + return f2 + } + if f2.i == 0 { + return f1 + } + + f := c.inst(InstAlt) + i := &c.p.Inst[f.i] + i.Out = f1.i + i.Arg = f2.i + f.out = f1.out.append(c.p, f2.out) + return f +} + +func (c *compiler) quest(f1 frag, nongreedy bool) frag { + f := c.inst(InstAlt) + i := &c.p.Inst[f.i] + if nongreedy { + i.Arg = f1.i + f.out = patchList(f.i << 1) + } else { + i.Out = f1.i + f.out = patchList(f.i<<1 | 1) + } + f.out = f.out.append(c.p, f1.out) + return f +} + +func (c *compiler) star(f1 frag, nongreedy bool) frag { + f := c.inst(InstAlt) + i := &c.p.Inst[f.i] + if nongreedy { + i.Arg = f1.i + f.out = patchList(f.i << 1) + } else { + i.Out = f1.i + f.out = patchList(f.i<<1 | 1) + } + f1.out.patch(c.p, f.i) + return f +} + +func (c *compiler) plus(f1 frag, nongreedy bool) frag { + return frag{f1.i, c.star(f1, nongreedy).out} +} + +func (c *compiler) empty(op EmptyOp) frag { + f := c.inst(InstEmptyWidth) + c.p.Inst[f.i].Arg = uint32(op) + f.out = patchList(f.i << 1) + return f +} + +func (c *compiler) rune(rune []int) frag { + f := c.inst(InstRune) + c.p.Inst[f.i].Rune = rune + f.out = patchList(f.i << 1) + return f +} diff --git a/src/pkg/exp/regexp/syntax/make_perl_groups.pl b/src/pkg/exp/regexp/syntax/make_perl_groups.pl new file mode 100755 index 000000000..6d1b84b10 --- /dev/null +++ b/src/pkg/exp/regexp/syntax/make_perl_groups.pl @@ -0,0 +1,103 @@ +#!/usr/bin/perl +# Copyright 2008 The Go Authors. All rights reserved. +# Use of this source code is governed by a BSD-style +# license that can be found in the LICENSE file. + +# Modified version of RE2's make_perl_groups.pl. + +# Generate table entries giving character ranges +# for POSIX/Perl character classes. Rather than +# figure out what the definition is, it is easier to ask +# Perl about each letter from 0-128 and write down +# its answer. + +@posixclasses = ( + "[:alnum:]", + "[:alpha:]", + "[:ascii:]", + "[:blank:]", + "[:cntrl:]", + "[:digit:]", + "[:graph:]", + "[:lower:]", + "[:print:]", + "[:punct:]", + "[:space:]", + "[:upper:]", + "[:word:]", + "[:xdigit:]", +); + +@perlclasses = ( + "\\d", + "\\s", + "\\w", +); + +sub ComputeClass($) { + my @ranges; + my ($class) = @_; + my $regexp = "[$class]"; + my $start = -1; + for (my $i=0; $i<=129; $i++) { + if ($i == 129) { $i = 256; } + if ($i <= 128 && chr($i) =~ $regexp) { + if ($start < 0) { + $start = $i; + } + } else { + if ($start >= 0) { + push @ranges, [$start, $i-1]; + } + $start = -1; + } + } + return @ranges; +} + +sub PrintClass($$@) { + my ($cname, $name, @ranges) = @_; + print "var code$cname = []int{ /* $name */\n"; + for (my $i=0; $i<@ranges; $i++) { + my @a = @{$ranges[$i]}; + printf "\t0x%x, 0x%x,\n", $a[0], $a[1]; + } + print "}\n\n"; + my $n = @ranges; + $negname = $name; + if ($negname =~ /:/) { + $negname =~ s/:/:^/; + } else { + $negname =~ y/a-z/A-Z/; + } + return "\t`$name`: {+1, code$cname},\n" . + "\t`$negname`: {-1, code$cname},\n"; +} + +my $gen = 0; + +sub PrintClasses($@) { + my ($cname, @classes) = @_; + my @entries; + foreach my $cl (@classes) { + my @ranges = ComputeClass($cl); + push @entries, PrintClass(++$gen, $cl, @ranges); + } + print "var ${cname}Group = map[string]charGroup{\n"; + foreach my $e (@entries) { + print $e; + } + print "}\n"; + my $count = @entries; +} + +print <perl_groups.go + +package syntax + +EOF + +PrintClasses("perl", @perlclasses); +PrintClasses("posix", @posixclasses); diff --git a/src/pkg/exp/regexp/syntax/parse.go b/src/pkg/exp/regexp/syntax/parse.go new file mode 100644 index 000000000..4eed18268 --- /dev/null +++ b/src/pkg/exp/regexp/syntax/parse.go @@ -0,0 +1,1797 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package syntax + +import ( + "os" + "sort" + "strings" + "unicode" + "utf8" +) + +// An Error describes a failure to parse a regular expression +// and gives the offending expression. +type Error struct { + Code ErrorCode + Expr string +} + +func (e *Error) String() string { + return "error parsing regexp: " + e.Code.String() + ": `" + e.Expr + "`" +} + +// An ErrorCode describes a failure to parse a regular expression. +type ErrorCode string + +const ( + // Unexpected error + ErrInternalError ErrorCode = "regexp/syntax: internal error" + + // Parse errors + ErrInvalidCharClass ErrorCode = "invalid character class" + ErrInvalidCharRange ErrorCode = "invalid character class range" + ErrInvalidEscape ErrorCode = "invalid escape sequence" + ErrInvalidNamedCapture ErrorCode = "invalid named capture" + ErrInvalidPerlOp ErrorCode = "invalid or unsupported Perl syntax" + ErrInvalidRepeatOp ErrorCode = "invalid nested repetition operator" + ErrInvalidRepeatSize ErrorCode = "invalid repeat count" + ErrInvalidUTF8 ErrorCode = "invalid UTF-8" + ErrMissingBracket ErrorCode = "missing closing ]" + ErrMissingParen ErrorCode = "missing closing )" + ErrMissingRepeatArgument ErrorCode = "missing argument to repetition operator" + ErrTrailingBackslash ErrorCode = "trailing backslash at end of expression" +) + +func (e ErrorCode) String() string { + return string(e) +} + +// Flags control the behavior of the parser and record information about regexp context. +type Flags uint16 + +const ( + FoldCase Flags = 1 << iota // case-insensitive match + Literal // treat pattern as literal string + ClassNL // allow character classes like [^a-z] and [[:space:]] to match newline + DotNL // allow . to match newline + OneLine // treat ^ and $ as only matching at beginning and end of text + NonGreedy // make repetition operators default to non-greedy + PerlX // allow Perl extensions + UnicodeGroups // allow \p{Han}, \P{Han} for Unicode group and negation + WasDollar // regexp OpEndText was $, not \z + Simple // regexp contains no counted repetition + + MatchNL = ClassNL | DotNL + + Perl = ClassNL | OneLine | PerlX | UnicodeGroups // as close to Perl as possible + POSIX Flags = 0 // POSIX syntax +) + +// Pseudo-ops for parsing stack. +const ( + opLeftParen = opPseudo + iota + opVerticalBar +) + +type parser struct { + flags Flags // parse mode flags + stack []*Regexp // stack of parsed expressions + free *Regexp + numCap int // number of capturing groups seen + wholeRegexp string + tmpClass []int // temporary char class work space +} + +func (p *parser) newRegexp(op Op) *Regexp { + re := p.free + if re != nil { + p.free = re.Sub0[0] + *re = Regexp{} + } else { + re = new(Regexp) + } + re.Op = op + return re +} + +func (p *parser) reuse(re *Regexp) { + re.Sub0[0] = p.free + p.free = re +} + +// Parse stack manipulation. + +// push pushes the regexp re onto the parse stack and returns the regexp. +func (p *parser) push(re *Regexp) *Regexp { + if re.Op == OpCharClass && len(re.Rune) == 2 && re.Rune[0] == re.Rune[1] { + // Single rune. + if p.maybeConcat(re.Rune[0], p.flags&^FoldCase) { + return nil + } + re.Op = OpLiteral + re.Rune = re.Rune[:1] + re.Flags = p.flags &^ FoldCase + } else if re.Op == OpCharClass && len(re.Rune) == 4 && + re.Rune[0] == re.Rune[1] && re.Rune[2] == re.Rune[3] && + unicode.SimpleFold(re.Rune[0]) == re.Rune[2] && + unicode.SimpleFold(re.Rune[2]) == re.Rune[0] || + re.Op == OpCharClass && len(re.Rune) == 2 && + re.Rune[0]+1 == re.Rune[1] && + unicode.SimpleFold(re.Rune[0]) == re.Rune[1] && + unicode.SimpleFold(re.Rune[1]) == re.Rune[0] { + // Case-insensitive rune like [Aa] or [Δδ]. + if p.maybeConcat(re.Rune[0], p.flags|FoldCase) { + return nil + } + + // Rewrite as (case-insensitive) literal. + re.Op = OpLiteral + re.Rune = re.Rune[:1] + re.Flags = p.flags | FoldCase + } else { + // Incremental concatenation. + p.maybeConcat(-1, 0) + } + + p.stack = append(p.stack, re) + return re +} + +// maybeConcat implements incremental concatenation +// of literal runes into string nodes. The parser calls this +// before each push, so only the top fragment of the stack +// might need processing. Since this is called before a push, +// the topmost literal is no longer subject to operators like * +// (Otherwise ab* would turn into (ab)*.) +// If r >= 0 and there's a node left over, maybeConcat uses it +// to push r with the given flags. +// maybeConcat reports whether r was pushed. +func (p *parser) maybeConcat(r int, flags Flags) bool { + n := len(p.stack) + if n < 2 { + return false + } + + re1 := p.stack[n-1] + re2 := p.stack[n-2] + if re1.Op != OpLiteral || re2.Op != OpLiteral || re1.Flags&FoldCase != re2.Flags&FoldCase { + return false + } + + // Push re1 into re2. + re2.Rune = append(re2.Rune, re1.Rune...) + + // Reuse re1 if possible. + if r >= 0 { + re1.Rune = re1.Rune0[:1] + re1.Rune[0] = r + re1.Flags = flags + return true + } + + p.stack = p.stack[:n-1] + p.reuse(re1) + return false // did not push r +} + +// newLiteral returns a new OpLiteral Regexp with the given flags +func (p *parser) newLiteral(r int, flags Flags) *Regexp { + re := p.newRegexp(OpLiteral) + re.Flags = flags + re.Rune0[0] = r + re.Rune = re.Rune0[:1] + return re +} + +// literal pushes a literal regexp for the rune r on the stack +// and returns that regexp. +func (p *parser) literal(r int) { + p.push(p.newLiteral(r, p.flags)) +} + +// op pushes a regexp with the given op onto the stack +// and returns that regexp. +func (p *parser) op(op Op) *Regexp { + re := p.newRegexp(op) + re.Flags = p.flags + return p.push(re) +} + +// repeat replaces the top stack element with itself repeated +// according to op. +func (p *parser) repeat(op Op, min, max int, opstr, t, lastRepeat string) (string, os.Error) { + flags := p.flags + if p.flags&PerlX != 0 { + if len(t) > 0 && t[0] == '?' { + t = t[1:] + flags ^= NonGreedy + } + if lastRepeat != "" { + // In Perl it is not allowed to stack repetition operators: + // a** is a syntax error, not a doubled star, and a++ means + // something else entirely, which we don't support! + return "", &Error{ErrInvalidRepeatOp, lastRepeat[:len(lastRepeat)-len(t)]} + } + } + n := len(p.stack) + if n == 0 { + return "", &Error{ErrMissingRepeatArgument, opstr} + } + sub := p.stack[n-1] + re := p.newRegexp(op) + re.Min = min + re.Max = max + re.Flags = flags + re.Sub = re.Sub0[:1] + re.Sub[0] = sub + p.stack[n-1] = re + return t, nil +} + +// concat replaces the top of the stack (above the topmost '|' or '(') with its concatenation. +func (p *parser) concat() *Regexp { + p.maybeConcat(-1, 0) + + // Scan down to find pseudo-operator | or (. + i := len(p.stack) + for i > 0 && p.stack[i-1].Op < opPseudo { + i-- + } + subs := p.stack[i:] + p.stack = p.stack[:i] + + // Empty concatenation is special case. + if len(subs) == 0 { + return p.push(p.newRegexp(OpEmptyMatch)) + } + + return p.push(p.collapse(subs, OpConcat)) +} + +// alternate replaces the top of the stack (above the topmost '(') with its alternation. +func (p *parser) alternate() *Regexp { + // Scan down to find pseudo-operator (. + // There are no | above (. + i := len(p.stack) + for i > 0 && p.stack[i-1].Op < opPseudo { + i-- + } + subs := p.stack[i:] + p.stack = p.stack[:i] + + // Make sure top class is clean. + // All the others already are (see swapVerticalBar). + if len(subs) > 0 { + cleanAlt(subs[len(subs)-1]) + } + + // Empty alternate is special case + // (shouldn't happen but easy to handle). + if len(subs) == 0 { + return p.push(p.newRegexp(OpNoMatch)) + } + + return p.push(p.collapse(subs, OpAlternate)) +} + +// cleanAlt cleans re for eventual inclusion in an alternation. +func cleanAlt(re *Regexp) { + switch re.Op { + case OpCharClass: + re.Rune = cleanClass(&re.Rune) + if len(re.Rune) == 2 && re.Rune[0] == 0 && re.Rune[1] == unicode.MaxRune { + re.Rune = nil + re.Op = OpAnyChar + return + } + if len(re.Rune) == 4 && re.Rune[0] == 0 && re.Rune[1] == '\n'-1 && re.Rune[2] == '\n'+1 && re.Rune[3] == unicode.MaxRune { + re.Rune = nil + re.Op = OpAnyCharNotNL + return + } + if cap(re.Rune)-len(re.Rune) > 100 { + // re.Rune will not grow any more. + // Make a copy or inline to reclaim storage. + re.Rune = append(re.Rune0[:0], re.Rune...) + } + } +} + +// collapse returns the result of applying op to sub. +// If sub contains op nodes, they all get hoisted up +// so that there is never a concat of a concat or an +// alternate of an alternate. +func (p *parser) collapse(subs []*Regexp, op Op) *Regexp { + if len(subs) == 1 { + return subs[0] + } + re := p.newRegexp(op) + re.Sub = re.Sub0[:0] + for _, sub := range subs { + if sub.Op == op { + re.Sub = append(re.Sub, sub.Sub...) + p.reuse(sub) + } else { + re.Sub = append(re.Sub, sub) + } + } + if op == OpAlternate { + re.Sub = p.factor(re.Sub, re.Flags) + if len(re.Sub) == 1 { + old := re + re = re.Sub[0] + p.reuse(old) + } + } + return re +} + +// factor factors common prefixes from the alternation list sub. +// It returns a replacement list that reuses the same storage and +// frees (passes to p.reuse) any removed *Regexps. +// +// For example, +// ABC|ABD|AEF|BCX|BCY +// simplifies by literal prefix extraction to +// A(B(C|D)|EF)|BC(X|Y) +// which simplifies by character class introduction to +// A(B[CD]|EF)|BC[XY] +// +func (p *parser) factor(sub []*Regexp, flags Flags) []*Regexp { + if len(sub) < 2 { + return sub + } + + // Round 1: Factor out common literal prefixes. + var str []int + var strflags Flags + start := 0 + out := sub[:0] + for i := 0; i <= len(sub); i++ { + // Invariant: the Regexps that were in sub[0:start] have been + // used or marked for reuse, and the slice space has been reused + // for out (len(out) <= start). + // + // Invariant: sub[start:i] consists of regexps that all begin + // with str as modified by strflags. + var istr []int + var iflags Flags + if i < len(sub) { + istr, iflags = p.leadingString(sub[i]) + if iflags == strflags { + same := 0 + for same < len(str) && same < len(istr) && str[same] == istr[same] { + same++ + } + if same > 0 { + // Matches at least one rune in current range. + // Keep going around. + str = str[:same] + continue + } + } + } + + // Found end of a run with common leading literal string: + // sub[start:i] all begin with str[0:len(str)], but sub[i] + // does not even begin with str[0]. + // + // Factor out common string and append factored expression to out. + if i == start { + // Nothing to do - run of length 0. + } else if i == start+1 { + // Just one: don't bother factoring. + out = append(out, sub[start]) + } else { + // Construct factored form: prefix(suffix1|suffix2|...) + prefix := p.newRegexp(OpLiteral) + prefix.Flags = strflags + prefix.Rune = append(prefix.Rune[:0], str...) + + for j := start; j < i; j++ { + sub[j] = p.removeLeadingString(sub[j], len(str)) + } + suffix := p.collapse(sub[start:i], OpAlternate) // recurse + + re := p.newRegexp(OpConcat) + re.Sub = append(re.Sub[:0], prefix, suffix) + out = append(out, re) + } + + // Prepare for next iteration. + start = i + str = istr + strflags = iflags + } + sub = out + + // Round 2: Factor out common complex prefixes, + // just the first piece of each concatenation, + // whatever it is. This is good enough a lot of the time. + start = 0 + out = sub[:0] + var first *Regexp + for i := 0; i <= len(sub); i++ { + // Invariant: the Regexps that were in sub[0:start] have been + // used or marked for reuse, and the slice space has been reused + // for out (len(out) <= start). + // + // Invariant: sub[start:i] consists of regexps that all begin + // with str as modified by strflags. + var ifirst *Regexp + if i < len(sub) { + ifirst = p.leadingRegexp(sub[i]) + if first != nil && first.Equal(ifirst) { + continue + } + } + + // Found end of a run with common leading regexp: + // sub[start:i] all begin with first but sub[i] does not. + // + // Factor out common regexp and append factored expression to out. + if i == start { + // Nothing to do - run of length 0. + } else if i == start+1 { + // Just one: don't bother factoring. + out = append(out, sub[start]) + } else { + // Construct factored form: prefix(suffix1|suffix2|...) + prefix := first + + for j := start; j < i; j++ { + reuse := j != start // prefix came from sub[start] + sub[j] = p.removeLeadingRegexp(sub[j], reuse) + } + suffix := p.collapse(sub[start:i], OpAlternate) // recurse + + re := p.newRegexp(OpConcat) + re.Sub = append(re.Sub[:0], prefix, suffix) + out = append(out, re) + } + + // Prepare for next iteration. + start = i + first = ifirst + } + sub = out + + // Round 3: Collapse runs of single literals into character classes. + start = 0 + out = sub[:0] + for i := 0; i <= len(sub); i++ { + // Invariant: the Regexps that were in sub[0:start] have been + // used or marked for reuse, and the slice space has been reused + // for out (len(out) <= start). + // + // Invariant: sub[start:i] consists of regexps that are either + // literal runes or character classes. + if i < len(sub) && isCharClass(sub[i]) { + continue + } + + // sub[i] is not a char or char class; + // emit char class for sub[start:i]... + if i == start { + // Nothing to do - run of length 0. + } else if i == start+1 { + out = append(out, sub[start]) + } else { + // Make new char class. + // Start with most complex regexp in sub[start]. + max := start + for j := start + 1; j < i; j++ { + if sub[max].Op < sub[j].Op || sub[max].Op == sub[j].Op && len(sub[max].Rune) < len(sub[j].Rune) { + max = j + } + } + sub[start], sub[max] = sub[max], sub[start] + + for j := start + 1; j < i; j++ { + mergeCharClass(sub[start], sub[j]) + p.reuse(sub[j]) + } + cleanAlt(sub[start]) + out = append(out, sub[start]) + } + + // ... and then emit sub[i]. + if i < len(sub) { + out = append(out, sub[i]) + } + start = i + 1 + } + sub = out + + // Round 4: Collapse runs of empty matches into a single empty match. + start = 0 + out = sub[:0] + for i := range sub { + if i+1 < len(sub) && sub[i].Op == OpEmptyMatch && sub[i+1].Op == OpEmptyMatch { + continue + } + out = append(out, sub[i]) + } + sub = out + + return sub +} + +// leadingString returns the leading literal string that re begins with. +// The string refers to storage in re or its children. +func (p *parser) leadingString(re *Regexp) ([]int, Flags) { + if re.Op == OpConcat && len(re.Sub) > 0 { + re = re.Sub[0] + } + if re.Op != OpLiteral { + return nil, 0 + } + return re.Rune, re.Flags & FoldCase +} + +// removeLeadingString removes the first n leading runes +// from the beginning of re. It returns the replacement for re. +func (p *parser) removeLeadingString(re *Regexp, n int) *Regexp { + if re.Op == OpConcat && len(re.Sub) > 0 { + // Removing a leading string in a concatenation + // might simplify the concatenation. + sub := re.Sub[0] + sub = p.removeLeadingString(sub, n) + re.Sub[0] = sub + if sub.Op == OpEmptyMatch { + p.reuse(sub) + switch len(re.Sub) { + case 0, 1: + // Impossible but handle. + re.Op = OpEmptyMatch + re.Sub = nil + case 2: + old := re + re = re.Sub[1] + p.reuse(old) + default: + copy(re.Sub, re.Sub[1:]) + re.Sub = re.Sub[:len(re.Sub)-1] + } + } + return re + } + + if re.Op == OpLiteral { + re.Rune = re.Rune[:copy(re.Rune, re.Rune[n:])] + if len(re.Rune) == 0 { + re.Op = OpEmptyMatch + } + } + return re +} + +// leadingRegexp returns the leading regexp that re begins with. +// The regexp refers to storage in re or its children. +func (p *parser) leadingRegexp(re *Regexp) *Regexp { + if re.Op == OpEmptyMatch { + return nil + } + if re.Op == OpConcat && len(re.Sub) > 0 { + sub := re.Sub[0] + if sub.Op == OpEmptyMatch { + return nil + } + return sub + } + return re +} + +// removeLeadingRegexp removes the leading regexp in re. +// It returns the replacement for re. +// If reuse is true, it passes the removed regexp (if no longer needed) to p.reuse. +func (p *parser) removeLeadingRegexp(re *Regexp, reuse bool) *Regexp { + if re.Op == OpConcat && len(re.Sub) > 0 { + if reuse { + p.reuse(re.Sub[0]) + } + re.Sub = re.Sub[:copy(re.Sub, re.Sub[1:])] + switch len(re.Sub) { + case 0: + re.Op = OpEmptyMatch + re.Sub = nil + case 1: + old := re + re = re.Sub[0] + p.reuse(old) + } + return re + } + re.Op = OpEmptyMatch + return re +} + +func literalRegexp(s string, flags Flags) *Regexp { + re := &Regexp{Op: OpLiteral} + re.Flags = flags + re.Rune = re.Rune0[:0] // use local storage for small strings + for _, c := range s { + if len(re.Rune) >= cap(re.Rune) { + // string is too long to fit in Rune0. let Go handle it + re.Rune = []int(s) + break + } + re.Rune = append(re.Rune, c) + } + return re +} + +// Parsing. + +func Parse(s string, flags Flags) (*Regexp, os.Error) { + if flags&Literal != 0 { + // Trivial parser for literal string. + if err := checkUTF8(s); err != nil { + return nil, err + } + return literalRegexp(s, flags), nil + } + + // Otherwise, must do real work. + var ( + p parser + err os.Error + c int + op Op + lastRepeat string + min, max int + ) + p.flags = flags + p.wholeRegexp = s + t := s + for t != "" { + repeat := "" + BigSwitch: + switch t[0] { + default: + if c, t, err = nextRune(t); err != nil { + return nil, err + } + p.literal(c) + + case '(': + if p.flags&PerlX != 0 && len(t) >= 2 && t[1] == '?' { + // Flag changes and non-capturing groups. + if t, err = p.parsePerlFlags(t); err != nil { + return nil, err + } + break + } + p.numCap++ + p.op(opLeftParen).Cap = p.numCap + t = t[1:] + case '|': + if err = p.parseVerticalBar(); err != nil { + return nil, err + } + t = t[1:] + case ')': + if err = p.parseRightParen(); err != nil { + return nil, err + } + t = t[1:] + case '^': + if p.flags&OneLine != 0 { + p.op(OpBeginText) + } else { + p.op(OpBeginLine) + } + t = t[1:] + case '$': + if p.flags&OneLine != 0 { + p.op(OpEndText).Flags |= WasDollar + } else { + p.op(OpEndLine) + } + t = t[1:] + case '.': + if p.flags&DotNL != 0 { + p.op(OpAnyChar) + } else { + p.op(OpAnyCharNotNL) + } + t = t[1:] + case '[': + if t, err = p.parseClass(t); err != nil { + return nil, err + } + case '*', '+', '?': + switch t[0] { + case '*': + op = OpStar + case '+': + op = OpPlus + case '?': + op = OpQuest + } + if t, err = p.repeat(op, min, max, t[:1], t[1:], lastRepeat); err != nil { + return nil, err + } + case '{': + op = OpRepeat + min, max, tt, ok := p.parseRepeat(t) + if !ok { + // If the repeat cannot be parsed, { is a literal. + p.literal('{') + t = t[1:] + break + } + if t, err = p.repeat(op, min, max, t[:len(t)-len(tt)], tt, lastRepeat); err != nil { + return nil, err + } + case '\\': + if p.flags&PerlX != 0 && len(t) >= 2 { + switch t[1] { + case 'A': + p.op(OpBeginText) + t = t[2:] + break BigSwitch + case 'b': + p.op(OpWordBoundary) + t = t[2:] + break BigSwitch + case 'B': + p.op(OpNoWordBoundary) + t = t[2:] + break BigSwitch + case 'C': + // any byte; not supported + return nil, &Error{ErrInvalidEscape, t[:2]} + case 'Q': + // \Q ... \E: the ... is always literals + var lit string + if i := strings.Index(t, `\E`); i < 0 { + lit = t[2:] + t = "" + } else { + lit = t[2:i] + t = t[i+2:] + } + p.push(literalRegexp(lit, p.flags)) + break BigSwitch + case 'z': + p.op(OpEndText) + t = t[2:] + break BigSwitch + } + } + + re := p.newRegexp(OpCharClass) + re.Flags = p.flags + + // Look for Unicode character group like \p{Han} + if len(t) >= 2 && (t[1] == 'p' || t[1] == 'P') { + r, rest, err := p.parseUnicodeClass(t, re.Rune0[:0]) + if err != nil { + return nil, err + } + if r != nil { + re.Rune = r + t = rest + p.push(re) + break BigSwitch + } + } + + // Perl character class escape. + if r, rest := p.parsePerlClassEscape(t, re.Rune0[:0]); r != nil { + re.Rune = r + t = rest + p.push(re) + break BigSwitch + } + p.reuse(re) + + // Ordinary single-character escape. + if c, t, err = p.parseEscape(t); err != nil { + return nil, err + } + p.literal(c) + } + lastRepeat = repeat + } + + p.concat() + if p.swapVerticalBar() { + // pop vertical bar + p.stack = p.stack[:len(p.stack)-1] + } + p.alternate() + + n := len(p.stack) + if n != 1 { + return nil, &Error{ErrMissingParen, s} + } + return p.stack[0], nil +} + +// parseRepeat parses {min} (max=min) or {min,} (max=-1) or {min,max}. +// If s is not of that form, it returns ok == false. +func (p *parser) parseRepeat(s string) (min, max int, rest string, ok bool) { + if s == "" || s[0] != '{' { + return + } + s = s[1:] + if min, s, ok = p.parseInt(s); !ok { + return + } + if s == "" { + return + } + if s[0] != ',' { + max = min + } else { + s = s[1:] + if s == "" { + return + } + if s[0] == '}' { + max = -1 + } else if max, s, ok = p.parseInt(s); !ok { + return + } + } + if s == "" || s[0] != '}' { + return + } + rest = s[1:] + ok = true + return +} + +// parsePerlFlags parses a Perl flag setting or non-capturing group or both, +// like (?i) or (?: or (?i:. It removes the prefix from s and updates the parse state. +// The caller must have ensured that s begins with "(?". +func (p *parser) parsePerlFlags(s string) (rest string, err os.Error) { + t := s + + // Check for named captures, first introduced in Python's regexp library. + // As usual, there are three slightly different syntaxes: + // + // (?Pexpr) the original, introduced by Python + // (?expr) the .NET alteration, adopted by Perl 5.10 + // (?'name'expr) another .NET alteration, adopted by Perl 5.10 + // + // Perl 5.10 gave in and implemented the Python version too, + // but they claim that the last two are the preferred forms. + // PCRE and languages based on it (specifically, PHP and Ruby) + // support all three as well. EcmaScript 4 uses only the Python form. + // + // In both the open source world (via Code Search) and the + // Google source tree, (?Pname) is the dominant form, + // so that's the one we implement. One is enough. + if len(t) > 4 && t[2] == 'P' && t[3] == '<' { + // Pull out name. + end := strings.IndexRune(t, '>') + if end < 0 { + if err = checkUTF8(t); err != nil { + return "", err + } + return "", &Error{ErrInvalidNamedCapture, s} + } + + capture := t[:end+1] // "(?P" + name := t[4:end] // "name" + if err = checkUTF8(name); err != nil { + return "", err + } + if !isValidCaptureName(name) { + return "", &Error{ErrInvalidNamedCapture, capture} + } + + // Like ordinary capture, but named. + p.numCap++ + re := p.op(opLeftParen) + re.Cap = p.numCap + re.Name = name + return t[end+1:], nil + } + + // Non-capturing group. Might also twiddle Perl flags. + var c int + t = t[2:] // skip (? + flags := p.flags + sign := +1 + sawFlag := false +Loop: + for t != "" { + if c, t, err = nextRune(t); err != nil { + return "", err + } + switch c { + default: + break Loop + + // Flags. + case 'i': + flags |= FoldCase + sawFlag = true + case 'm': + flags &^= OneLine + sawFlag = true + case 's': + flags |= DotNL + sawFlag = true + case 'U': + flags |= NonGreedy + sawFlag = true + + // Switch to negation. + case '-': + if sign < 0 { + break Loop + } + sign = -1 + // Invert flags so that | above turn into &^ and vice versa. + // We'll invert flags again before using it below. + flags = ^flags + sawFlag = false + + // End of flags, starting group or not. + case ':', ')': + if sign < 0 { + if !sawFlag { + break Loop + } + flags = ^flags + } + if c == ':' { + // Open new group + p.op(opLeftParen) + } + p.flags = flags + return t, nil + } + } + + return "", &Error{ErrInvalidPerlOp, s[:len(s)-len(t)]} +} + +// isValidCaptureName reports whether name +// is a valid capture name: [A-Za-z0-9_]+. +// PCRE limits names to 32 bytes. +// Python rejects names starting with digits. +// We don't enforce either of those. +func isValidCaptureName(name string) bool { + if name == "" { + return false + } + for _, c := range name { + if c != '_' && !isalnum(c) { + return false + } + } + return true +} + +// parseInt parses a decimal integer. +func (p *parser) parseInt(s string) (n int, rest string, ok bool) { + if s == "" || s[0] < '0' || '9' < s[0] { + return + } + // Disallow leading zeros. + if len(s) >= 2 && s[0] == '0' && '0' <= s[1] && s[1] <= '9' { + return + } + for s != "" && '0' <= s[0] && s[0] <= '9' { + // Avoid overflow. + if n >= 1e8 { + return + } + n = n*10 + int(s[0]) - '0' + s = s[1:] + } + rest = s + ok = true + return +} + +// can this be represented as a character class? +// single-rune literal string, char class, ., and .|\n. +func isCharClass(re *Regexp) bool { + return re.Op == OpLiteral && len(re.Rune) == 1 || + re.Op == OpCharClass || + re.Op == OpAnyCharNotNL || + re.Op == OpAnyChar +} + +// does re match r? +func matchRune(re *Regexp, r int) bool { + switch re.Op { + case OpLiteral: + return len(re.Rune) == 1 && re.Rune[0] == r + case OpCharClass: + for i := 0; i < len(re.Rune); i += 2 { + if re.Rune[i] <= r && r <= re.Rune[i+1] { + return true + } + } + return false + case OpAnyCharNotNL: + return r != '\n' + case OpAnyChar: + return true + } + return false +} + +// parseVerticalBar handles a | in the input. +func (p *parser) parseVerticalBar() os.Error { + p.concat() + + // The concatenation we just parsed is on top of the stack. + // If it sits above an opVerticalBar, swap it below + // (things below an opVerticalBar become an alternation). + // Otherwise, push a new vertical bar. + if !p.swapVerticalBar() { + p.op(opVerticalBar) + } + + return nil +} + +// mergeCharClass makes dst = dst|src. +// The caller must ensure that dst.Op >= src.Op, +// to reduce the amount of copying. +func mergeCharClass(dst, src *Regexp) { + switch dst.Op { + case OpAnyChar: + // src doesn't add anything. + case OpAnyCharNotNL: + // src might add \n + if matchRune(src, '\n') { + dst.Op = OpAnyChar + } + case OpCharClass: + // src is simpler, so either literal or char class + if src.Op == OpLiteral { + dst.Rune = appendRange(dst.Rune, src.Rune[0], src.Rune[0]) + } else { + dst.Rune = appendClass(dst.Rune, src.Rune) + } + case OpLiteral: + // both literal + if src.Rune[0] == dst.Rune[0] { + break + } + dst.Op = OpCharClass + dst.Rune = append(dst.Rune, dst.Rune[0]) + dst.Rune = appendRange(dst.Rune, src.Rune[0], src.Rune[0]) + } +} + +// If the top of the stack is an element followed by an opVerticalBar +// swapVerticalBar swaps the two and returns true. +// Otherwise it returns false. +func (p *parser) swapVerticalBar() bool { + // If above and below vertical bar are literal or char class, + // can merge into a single char class. + n := len(p.stack) + if n >= 3 && p.stack[n-2].Op == opVerticalBar && isCharClass(p.stack[n-1]) && isCharClass(p.stack[n-3]) { + re1 := p.stack[n-1] + re3 := p.stack[n-3] + // Make re3 the more complex of the two. + if re1.Op > re3.Op { + re1, re3 = re3, re1 + p.stack[n-3] = re3 + } + mergeCharClass(re3, re1) + p.reuse(re1) + p.stack = p.stack[:n-1] + return true + } + + if n >= 2 { + re1 := p.stack[n-1] + re2 := p.stack[n-2] + if re2.Op == opVerticalBar { + if n >= 3 { + // Now out of reach. + // Clean opportunistically. + cleanAlt(p.stack[n-3]) + } + p.stack[n-2] = re1 + p.stack[n-1] = re2 + return true + } + } + return false +} + +// parseRightParen handles a ) in the input. +func (p *parser) parseRightParen() os.Error { + p.concat() + if p.swapVerticalBar() { + // pop vertical bar + p.stack = p.stack[:len(p.stack)-1] + } + p.alternate() + + n := len(p.stack) + if n < 2 { + return &Error{ErrInternalError, ""} + } + re1 := p.stack[n-1] + re2 := p.stack[n-2] + p.stack = p.stack[:n-2] + if re2.Op != opLeftParen { + return &Error{ErrMissingParen, p.wholeRegexp} + } + if re2.Cap == 0 { + // Just for grouping. + p.push(re1) + } else { + re2.Op = OpCapture + re2.Sub = re2.Sub0[:1] + re2.Sub[0] = re1 + p.push(re2) + } + return nil +} + +// parseEscape parses an escape sequence at the beginning of s +// and returns the rune. +func (p *parser) parseEscape(s string) (r int, rest string, err os.Error) { + t := s[1:] + if t == "" { + return 0, "", &Error{ErrTrailingBackslash, ""} + } + c, t, err := nextRune(t) + if err != nil { + return 0, "", err + } + +Switch: + switch c { + default: + if c < utf8.RuneSelf && !isalnum(c) { + // Escaped non-word characters are always themselves. + // PCRE is not quite so rigorous: it accepts things like + // \q, but we don't. We once rejected \_, but too many + // programs and people insist on using it, so allow \_. + return c, t, nil + } + + // Octal escapes. + case '1', '2', '3', '4', '5', '6', '7': + // Single non-zero digit is a backreference; not supported + if t == "" || t[0] < '0' || t[0] > '7' { + break + } + fallthrough + case '0': + // Consume up to three octal digits; already have one. + r = c - '0' + for i := 1; i < 3; i++ { + if t == "" || t[0] < '0' || t[0] > '7' { + break + } + r = r*8 + int(t[0]) - '0' + t = t[1:] + } + return r, t, nil + + // Hexadecimal escapes. + case 'x': + if t == "" { + break + } + if c, t, err = nextRune(t); err != nil { + return 0, "", err + } + if c == '{' { + // Any number of digits in braces. + // Perl accepts any text at all; it ignores all text + // after the first non-hex digit. We require only hex digits, + // and at least one. + nhex := 0 + r = 0 + for { + if t == "" { + break Switch + } + if c, t, err = nextRune(t); err != nil { + return 0, "", err + } + if c == '}' { + break + } + v := unhex(c) + if v < 0 { + break Switch + } + r = r*16 + v + if r > unicode.MaxRune { + break Switch + } + nhex++ + } + if nhex == 0 { + break Switch + } + return r, t, nil + } + + // Easy case: two hex digits. + x := unhex(c) + if c, t, err = nextRune(t); err != nil { + return 0, "", err + } + y := unhex(c) + if x < 0 || y < 0 { + break + } + return x*16 + y, t, nil + + // C escapes. There is no case 'b', to avoid misparsing + // the Perl word-boundary \b as the C backspace \b + // when in POSIX mode. In Perl, /\b/ means word-boundary + // but /[\b]/ means backspace. We don't support that. + // If you want a backspace, embed a literal backspace + // character or use \x08. + case 'a': + return '\a', t, err + case 'f': + return '\f', t, err + case 'n': + return '\n', t, err + case 'r': + return '\r', t, err + case 't': + return '\t', t, err + case 'v': + return '\v', t, err + } + return 0, "", &Error{ErrInvalidEscape, s[:len(s)-len(t)]} +} + +// parseClassChar parses a character class character at the beginning of s +// and returns it. +func (p *parser) parseClassChar(s, wholeClass string) (r int, rest string, err os.Error) { + if s == "" { + return 0, "", &Error{Code: ErrMissingBracket, Expr: wholeClass} + } + + // Allow regular escape sequences even though + // many need not be escaped in this context. + if s[0] == '\\' { + return p.parseEscape(s) + } + + return nextRune(s) +} + +type charGroup struct { + sign int + class []int +} + +// parsePerlClassEscape parses a leading Perl character class escape like \d +// from the beginning of s. If one is present, it appends the characters to r +// and returns the new slice r and the remainder of the string. +func (p *parser) parsePerlClassEscape(s string, r []int) (out []int, rest string) { + if p.flags&PerlX == 0 || len(s) < 2 || s[0] != '\\' { + return + } + g := perlGroup[s[0:2]] + if g.sign == 0 { + return + } + return p.appendGroup(r, g), s[2:] +} + +// parseNamedClass parses a leading POSIX named character class like [:alnum:] +// from the beginning of s. If one is present, it appends the characters to r +// and returns the new slice r and the remainder of the string. +func (p *parser) parseNamedClass(s string, r []int) (out []int, rest string, err os.Error) { + if len(s) < 2 || s[0] != '[' || s[1] != ':' { + return + } + + i := strings.Index(s[2:], ":]") + if i < 0 { + return + } + i += 2 + name, s := s[0:i+2], s[i+2:] + g := posixGroup[name] + if g.sign == 0 { + return nil, "", &Error{ErrInvalidCharRange, name} + } + return p.appendGroup(r, g), s, nil +} + +func (p *parser) appendGroup(r []int, g charGroup) []int { + if p.flags&FoldCase == 0 { + if g.sign < 0 { + r = appendNegatedClass(r, g.class) + } else { + r = appendClass(r, g.class) + } + } else { + tmp := p.tmpClass[:0] + tmp = appendFoldedClass(tmp, g.class) + p.tmpClass = tmp + tmp = cleanClass(&p.tmpClass) + if g.sign < 0 { + r = appendNegatedClass(r, tmp) + } else { + r = appendClass(r, tmp) + } + } + return r +} + +// unicodeTable returns the unicode.RangeTable identified by name +// and the table of additional fold-equivalent code points. +func unicodeTable(name string) (*unicode.RangeTable, *unicode.RangeTable) { + if t := unicode.Categories[name]; t != nil { + return t, unicode.FoldCategory[name] + } + if t := unicode.Scripts[name]; t != nil { + return t, unicode.FoldScript[name] + } + return nil, nil +} + +// parseUnicodeClass parses a leading Unicode character class like \p{Han} +// from the beginning of s. If one is present, it appends the characters to r +// and returns the new slice r and the remainder of the string. +func (p *parser) parseUnicodeClass(s string, r []int) (out []int, rest string, err os.Error) { + if p.flags&UnicodeGroups == 0 || len(s) < 2 || s[0] != '\\' || s[1] != 'p' && s[1] != 'P' { + return + } + + // Committed to parse or return error. + sign := +1 + if s[1] == 'P' { + sign = -1 + } + t := s[2:] + c, t, err := nextRune(t) + if err != nil { + return + } + var seq, name string + if c != '{' { + // Single-letter name. + seq = s[:len(s)-len(t)] + name = seq[2:] + } else { + // Name is in braces. + end := strings.IndexRune(s, '}') + if end < 0 { + if err = checkUTF8(s); err != nil { + return + } + return nil, "", &Error{ErrInvalidCharRange, s} + } + seq, t = s[:end+1], s[end+1:] + name = s[3:end] + if err = checkUTF8(name); err != nil { + return + } + } + + // Group can have leading negation too. \p{^Han} == \P{Han}, \P{^Han} == \p{Han}. + if name != "" && name[0] == '^' { + sign = -sign + name = name[1:] + } + + tab, fold := unicodeTable(name) + if tab == nil { + return nil, "", &Error{ErrInvalidCharRange, seq} + } + + if p.flags&FoldCase == 0 || fold == nil { + if sign > 0 { + r = appendTable(r, tab) + } else { + r = appendNegatedTable(r, tab) + } + } else { + // Merge and clean tab and fold in a temporary buffer. + // This is necessary for the negative case and just tidy + // for the positive case. + tmp := p.tmpClass[:0] + tmp = appendTable(tmp, tab) + tmp = appendTable(tmp, fold) + p.tmpClass = tmp + tmp = cleanClass(&p.tmpClass) + if sign > 0 { + r = appendClass(r, tmp) + } else { + r = appendNegatedClass(r, tmp) + } + } + return r, t, nil +} + +// parseClass parses a character class at the beginning of s +// and pushes it onto the parse stack. +func (p *parser) parseClass(s string) (rest string, err os.Error) { + t := s[1:] // chop [ + re := p.newRegexp(OpCharClass) + re.Flags = p.flags + re.Rune = re.Rune0[:0] + + sign := +1 + if t != "" && t[0] == '^' { + sign = -1 + t = t[1:] + + // If character class does not match \n, add it here, + // so that negation later will do the right thing. + if p.flags&ClassNL == 0 { + re.Rune = append(re.Rune, '\n', '\n') + } + } + + class := re.Rune + first := true // ] and - are okay as first char in class + for t == "" || t[0] != ']' || first { + // POSIX: - is only okay unescaped as first or last in class. + // Perl: - is okay anywhere. + if t != "" && t[0] == '-' && p.flags&PerlX == 0 && !first && (len(t) == 1 || t[1] != ']') { + _, size := utf8.DecodeRuneInString(t[1:]) + return "", &Error{Code: ErrInvalidCharRange, Expr: t[:1+size]} + } + first = false + + // Look for POSIX [:alnum:] etc. + if len(t) > 2 && t[0] == '[' && t[1] == ':' { + nclass, nt, err := p.parseNamedClass(t, class) + if err != nil { + return "", err + } + if nclass != nil { + class, t = nclass, nt + continue + } + } + + // Look for Unicode character group like \p{Han}. + nclass, nt, err := p.parseUnicodeClass(t, class) + if err != nil { + return "", err + } + if nclass != nil { + class, t = nclass, nt + continue + } + + // Look for Perl character class symbols (extension). + if nclass, nt := p.parsePerlClassEscape(t, class); nclass != nil { + class, t = nclass, nt + continue + } + + // Single character or simple range. + rng := t + var lo, hi int + if lo, t, err = p.parseClassChar(t, s); err != nil { + return "", err + } + hi = lo + // [a-] means (a|-) so check for final ]. + if len(t) >= 2 && t[0] == '-' && t[1] != ']' { + t = t[1:] + if hi, t, err = p.parseClassChar(t, s); err != nil { + return "", err + } + if hi < lo { + rng = rng[:len(rng)-len(t)] + return "", &Error{Code: ErrInvalidCharRange, Expr: rng} + } + } + if p.flags&FoldCase == 0 { + class = appendRange(class, lo, hi) + } else { + class = appendFoldedRange(class, lo, hi) + } + } + t = t[1:] // chop ] + + // Use &re.Rune instead of &class to avoid allocation. + re.Rune = class + class = cleanClass(&re.Rune) + if sign < 0 { + class = negateClass(class) + } + re.Rune = class + p.push(re) + return t, nil +} + +// cleanClass sorts the ranges (pairs of elements of r), +// merges them, and eliminates duplicates. +func cleanClass(rp *[]int) []int { + + // Sort by lo increasing, hi decreasing to break ties. + sort.Sort(ranges{rp}) + + r := *rp + if len(r) < 2 { + return r + } + + // Merge abutting, overlapping. + w := 2 // write index + for i := 2; i < len(r); i += 2 { + lo, hi := r[i], r[i+1] + if lo <= r[w-1]+1 { + // merge with previous range + if hi > r[w-1] { + r[w-1] = hi + } + continue + } + // new disjoint range + r[w] = lo + r[w+1] = hi + w += 2 + } + + return r[:w] +} + +// appendRange returns the result of appending the range lo-hi to the class r. +func appendRange(r []int, lo, hi int) []int { + // Expand last range or next to last range if it overlaps or abuts. + // Checking two ranges helps when appending case-folded + // alphabets, so that one range can be expanding A-Z and the + // other expanding a-z. + n := len(r) + for i := 2; i <= 4; i += 2 { // twice, using i=2, i=4 + if n >= i { + rlo, rhi := r[n-i], r[n-i+1] + if lo <= rhi+1 && rlo <= hi+1 { + if lo < rlo { + r[n-i] = lo + } + if hi > rhi { + r[n-i+1] = hi + } + return r + } + } + } + + return append(r, lo, hi) +} + +const ( + // minimum and maximum runes involved in folding. + // checked during test. + minFold = 0x0041 + maxFold = 0x1044f +) + +// appendFoldedRange returns the result of appending the range lo-hi +// and its case folding-equivalent runes to the class r. +func appendFoldedRange(r []int, lo, hi int) []int { + // Optimizations. + if lo <= minFold && hi >= maxFold { + // Range is full: folding can't add more. + return appendRange(r, lo, hi) + } + if hi < minFold || lo > maxFold { + // Range is outside folding possibilities. + return appendRange(r, lo, hi) + } + if lo < minFold { + // [lo, minFold-1] needs no folding. + r = appendRange(r, lo, minFold-1) + lo = minFold + } + if hi > maxFold { + // [maxFold+1, hi] needs no folding. + r = appendRange(r, maxFold+1, hi) + hi = maxFold + } + + // Brute force. Depend on appendRange to coalesce ranges on the fly. + for c := lo; c <= hi; c++ { + r = appendRange(r, c, c) + f := unicode.SimpleFold(c) + for f != c { + r = appendRange(r, f, f) + f = unicode.SimpleFold(f) + } + } + return r +} + +// appendClass returns the result of appending the class x to the class r. +// It assume x is clean. +func appendClass(r []int, x []int) []int { + for i := 0; i < len(x); i += 2 { + r = appendRange(r, x[i], x[i+1]) + } + return r +} + +// appendFolded returns the result of appending the case folding of the class x to the class r. +func appendFoldedClass(r []int, x []int) []int { + for i := 0; i < len(x); i += 2 { + r = appendFoldedRange(r, x[i], x[i+1]) + } + return r +} + +// appendNegatedClass returns the result of appending the negation of the class x to the class r. +// It assumes x is clean. +func appendNegatedClass(r []int, x []int) []int { + nextLo := 0 + for i := 0; i < len(x); i += 2 { + lo, hi := x[i], x[i+1] + if nextLo <= lo-1 { + r = appendRange(r, nextLo, lo-1) + } + nextLo = hi + 1 + } + if nextLo <= unicode.MaxRune { + r = appendRange(r, nextLo, unicode.MaxRune) + } + return r +} + +// appendTable returns the result of appending x to the class r. +func appendTable(r []int, x *unicode.RangeTable) []int { + for _, xr := range x.R16 { + lo, hi, stride := int(xr.Lo), int(xr.Hi), int(xr.Stride) + if stride == 1 { + r = appendRange(r, lo, hi) + continue + } + for c := lo; c <= hi; c += stride { + r = appendRange(r, c, c) + } + } + for _, xr := range x.R32 { + lo, hi, stride := int(xr.Lo), int(xr.Hi), int(xr.Stride) + if stride == 1 { + r = appendRange(r, lo, hi) + continue + } + for c := lo; c <= hi; c += stride { + r = appendRange(r, c, c) + } + } + return r +} + +// appendNegatedTable returns the result of appending the negation of x to the class r. +func appendNegatedTable(r []int, x *unicode.RangeTable) []int { + nextLo := 0 // lo end of next class to add + for _, xr := range x.R16 { + lo, hi, stride := int(xr.Lo), int(xr.Hi), int(xr.Stride) + if stride == 1 { + if nextLo <= lo-1 { + r = appendRange(r, nextLo, lo-1) + } + nextLo = hi + 1 + continue + } + for c := lo; c <= hi; c += stride { + if nextLo <= c-1 { + r = appendRange(r, nextLo, c-1) + } + nextLo = c + 1 + } + } + for _, xr := range x.R32 { + lo, hi, stride := int(xr.Lo), int(xr.Hi), int(xr.Stride) + if stride == 1 { + if nextLo <= lo-1 { + r = appendRange(r, nextLo, lo-1) + } + nextLo = hi + 1 + continue + } + for c := lo; c <= hi; c += stride { + if nextLo <= c-1 { + r = appendRange(r, nextLo, c-1) + } + nextLo = c + 1 + } + } + if nextLo <= unicode.MaxRune { + r = appendRange(r, nextLo, unicode.MaxRune) + } + return r +} + +// negateClass overwrites r and returns r's negation. +// It assumes the class r is already clean. +func negateClass(r []int) []int { + nextLo := 0 // lo end of next class to add + w := 0 // write index + for i := 0; i < len(r); i += 2 { + lo, hi := r[i], r[i+1] + if nextLo <= lo-1 { + r[w] = nextLo + r[w+1] = lo - 1 + w += 2 + } + nextLo = hi + 1 + } + r = r[:w] + if nextLo <= unicode.MaxRune { + // It's possible for the negation to have one more + // range - this one - than the original class, so use append. + r = append(r, nextLo, unicode.MaxRune) + } + return r +} + +// ranges implements sort.Interface on a []rune. +// The choice of receiver type definition is strange +// but avoids an allocation since we already have +// a *[]int. +type ranges struct { + p *[]int +} + +func (ra ranges) Less(i, j int) bool { + p := *ra.p + i *= 2 + j *= 2 + return p[i] < p[j] || p[i] == p[j] && p[i+1] > p[j+1] +} + +func (ra ranges) Len() int { + return len(*ra.p) / 2 +} + +func (ra ranges) Swap(i, j int) { + p := *ra.p + i *= 2 + j *= 2 + p[i], p[i+1], p[j], p[j+1] = p[j], p[j+1], p[i], p[i+1] +} + +func checkUTF8(s string) os.Error { + for s != "" { + rune, size := utf8.DecodeRuneInString(s) + if rune == utf8.RuneError && size == 1 { + return &Error{Code: ErrInvalidUTF8, Expr: s} + } + s = s[size:] + } + return nil +} + +func nextRune(s string) (c int, t string, err os.Error) { + c, size := utf8.DecodeRuneInString(s) + if c == utf8.RuneError && size == 1 { + return 0, "", &Error{Code: ErrInvalidUTF8, Expr: s} + } + return c, s[size:], nil +} + +func isalnum(c int) bool { + return '0' <= c && c <= '9' || 'A' <= c && c <= 'Z' || 'a' <= c && c <= 'z' +} + +func unhex(c int) int { + if '0' <= c && c <= '9' { + return c - '0' + } + if 'a' <= c && c <= 'f' { + return c - 'a' + 10 + } + if 'A' <= c && c <= 'F' { + return c - 'A' + 10 + } + return -1 +} diff --git a/src/pkg/exp/regexp/syntax/parse_test.go b/src/pkg/exp/regexp/syntax/parse_test.go new file mode 100644 index 000000000..779b9afde --- /dev/null +++ b/src/pkg/exp/regexp/syntax/parse_test.go @@ -0,0 +1,350 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package syntax + +import ( + "bytes" + "fmt" + "testing" + "unicode" +) + +var parseTests = []struct { + Regexp string + Dump string +}{ + // Base cases + {`a`, `lit{a}`}, + {`a.`, `cat{lit{a}dot{}}`}, + {`a.b`, `cat{lit{a}dot{}lit{b}}`}, + {`ab`, `str{ab}`}, + {`a.b.c`, `cat{lit{a}dot{}lit{b}dot{}lit{c}}`}, + {`abc`, `str{abc}`}, + {`a|^`, `alt{lit{a}bol{}}`}, + {`a|b`, `cc{0x61-0x62}`}, + {`(a)`, `cap{lit{a}}`}, + {`(a)|b`, `alt{cap{lit{a}}lit{b}}`}, + {`a*`, `star{lit{a}}`}, + {`a+`, `plus{lit{a}}`}, + {`a?`, `que{lit{a}}`}, + {`a{2}`, `rep{2,2 lit{a}}`}, + {`a{2,3}`, `rep{2,3 lit{a}}`}, + {`a{2,}`, `rep{2,-1 lit{a}}`}, + {`a*?`, `nstar{lit{a}}`}, + {`a+?`, `nplus{lit{a}}`}, + {`a??`, `nque{lit{a}}`}, + {`a{2}?`, `nrep{2,2 lit{a}}`}, + {`a{2,3}?`, `nrep{2,3 lit{a}}`}, + {`a{2,}?`, `nrep{2,-1 lit{a}}`}, + {``, `emp{}`}, + {`|`, `emp{}`}, // alt{emp{}emp{}} but got factored + {`|x|`, `alt{emp{}lit{x}emp{}}`}, + {`.`, `dot{}`}, + {`^`, `bol{}`}, + {`$`, `eol{}`}, + {`\|`, `lit{|}`}, + {`\(`, `lit{(}`}, + {`\)`, `lit{)}`}, + {`\*`, `lit{*}`}, + {`\+`, `lit{+}`}, + {`\?`, `lit{?}`}, + {`{`, `lit{{}`}, + {`}`, `lit{}}`}, + {`\.`, `lit{.}`}, + {`\^`, `lit{^}`}, + {`\$`, `lit{$}`}, + {`\\`, `lit{\}`}, + {`[ace]`, `cc{0x61 0x63 0x65}`}, + {`[abc]`, `cc{0x61-0x63}`}, + {`[a-z]`, `cc{0x61-0x7a}`}, + {`[a]`, `lit{a}`}, + {`\-`, `lit{-}`}, + {`-`, `lit{-}`}, + {`\_`, `lit{_}`}, + {`abc`, `str{abc}`}, + {`abc|def`, `alt{str{abc}str{def}}`}, + {`abc|def|ghi`, `alt{str{abc}str{def}str{ghi}}`}, + + // Posix and Perl extensions + {`[[:lower:]]`, `cc{0x61-0x7a}`}, + {`[a-z]`, `cc{0x61-0x7a}`}, + {`[^[:lower:]]`, `cc{0x0-0x60 0x7b-0x10ffff}`}, + {`[[:^lower:]]`, `cc{0x0-0x60 0x7b-0x10ffff}`}, + {`(?i)[[:lower:]]`, `cc{0x41-0x5a 0x61-0x7a 0x17f 0x212a}`}, + {`(?i)[a-z]`, `cc{0x41-0x5a 0x61-0x7a 0x17f 0x212a}`}, + {`(?i)[^[:lower:]]`, `cc{0x0-0x40 0x5b-0x60 0x7b-0x17e 0x180-0x2129 0x212b-0x10ffff}`}, + {`(?i)[[:^lower:]]`, `cc{0x0-0x40 0x5b-0x60 0x7b-0x17e 0x180-0x2129 0x212b-0x10ffff}`}, + {`\d`, `cc{0x30-0x39}`}, + {`\D`, `cc{0x0-0x2f 0x3a-0x10ffff}`}, + {`\s`, `cc{0x9-0xa 0xc-0xd 0x20}`}, + {`\S`, `cc{0x0-0x8 0xb 0xe-0x1f 0x21-0x10ffff}`}, + {`\w`, `cc{0x30-0x39 0x41-0x5a 0x5f 0x61-0x7a}`}, + {`\W`, `cc{0x0-0x2f 0x3a-0x40 0x5b-0x5e 0x60 0x7b-0x10ffff}`}, + {`(?i)\w`, `cc{0x30-0x39 0x41-0x5a 0x5f 0x61-0x7a 0x17f 0x212a}`}, + {`(?i)\W`, `cc{0x0-0x2f 0x3a-0x40 0x5b-0x5e 0x60 0x7b-0x17e 0x180-0x2129 0x212b-0x10ffff}`}, + {`[^\\]`, `cc{0x0-0x5b 0x5d-0x10ffff}`}, + // { `\C`, `byte{}` }, // probably never + + // Unicode, negatives, and a double negative. + {`\p{Braille}`, `cc{0x2800-0x28ff}`}, + {`\P{Braille}`, `cc{0x0-0x27ff 0x2900-0x10ffff}`}, + {`\p{^Braille}`, `cc{0x0-0x27ff 0x2900-0x10ffff}`}, + {`\P{^Braille}`, `cc{0x2800-0x28ff}`}, + {`\pZ`, `cc{0x20 0xa0 0x1680 0x180e 0x2000-0x200a 0x2028-0x2029 0x202f 0x205f 0x3000}`}, + {`[\p{Braille}]`, `cc{0x2800-0x28ff}`}, + {`[\P{Braille}]`, `cc{0x0-0x27ff 0x2900-0x10ffff}`}, + {`[\p{^Braille}]`, `cc{0x0-0x27ff 0x2900-0x10ffff}`}, + {`[\P{^Braille}]`, `cc{0x2800-0x28ff}`}, + {`[\pZ]`, `cc{0x20 0xa0 0x1680 0x180e 0x2000-0x200a 0x2028-0x2029 0x202f 0x205f 0x3000}`}, + {`\p{Lu}`, mkCharClass(unicode.IsUpper)}, + {`[\p{Lu}]`, mkCharClass(unicode.IsUpper)}, + {`(?i)[\p{Lu}]`, mkCharClass(isUpperFold)}, + + // Hex, octal. + {`[\012-\234]\141`, `cat{cc{0xa-0x9c}lit{a}}`}, + {`[\x{41}-\x7a]\x61`, `cat{cc{0x41-0x7a}lit{a}}`}, + + // More interesting regular expressions. + {`a{,2}`, `str{a{,2}}`}, + {`\.\^\$\\`, `str{.^$\}`}, + {`[a-zABC]`, `cc{0x41-0x43 0x61-0x7a}`}, + {`[^a]`, `cc{0x0-0x60 0x62-0x10ffff}`}, + {`[α-ε☺]`, `cc{0x3b1-0x3b5 0x263a}`}, // utf-8 + {`a*{`, `cat{star{lit{a}}lit{{}}`}, + + // Test precedences + {`(?:ab)*`, `star{str{ab}}`}, + {`(ab)*`, `star{cap{str{ab}}}`}, + {`ab|cd`, `alt{str{ab}str{cd}}`}, + {`a(b|c)d`, `cat{lit{a}cap{cc{0x62-0x63}}lit{d}}`}, + + // Test flattening. + {`(?:a)`, `lit{a}`}, + {`(?:ab)(?:cd)`, `str{abcd}`}, + {`(?:a+b+)(?:c+d+)`, `cat{plus{lit{a}}plus{lit{b}}plus{lit{c}}plus{lit{d}}}`}, + {`(?:a+|b+)|(?:c+|d+)`, `alt{plus{lit{a}}plus{lit{b}}plus{lit{c}}plus{lit{d}}}`}, + {`(?:a|b)|(?:c|d)`, `cc{0x61-0x64}`}, + {`a|.`, `dot{}`}, + {`.|a`, `dot{}`}, + {`(?:[abc]|A|Z|hello|world)`, `alt{cc{0x41 0x5a 0x61-0x63}str{hello}str{world}}`}, + {`(?:[abc]|A|Z)`, `cc{0x41 0x5a 0x61-0x63}`}, + + // Test Perl quoted literals + {`\Q+|*?{[\E`, `str{+|*?{[}`}, + {`\Q+\E+`, `plus{lit{+}}`}, + {`\Q\\E`, `lit{\}`}, + {`\Q\\\E`, `str{\\}`}, + + // Test Perl \A and \z + {`(?m)^`, `bol{}`}, + {`(?m)$`, `eol{}`}, + {`(?-m)^`, `bot{}`}, + {`(?-m)$`, `eot{}`}, + {`(?m)\A`, `bot{}`}, + {`(?m)\z`, `eot{\z}`}, + {`(?-m)\A`, `bot{}`}, + {`(?-m)\z`, `eot{\z}`}, + + // Test named captures + {`(?Pa)`, `cap{name:lit{a}}`}, + + // Case-folded literals + {`[Aa]`, `litfold{A}`}, + {`[\x{100}\x{101}]`, `litfold{Ā}`}, + {`[Δδ]`, `litfold{Δ}`}, + + // Strings + {`abcde`, `str{abcde}`}, + {`[Aa][Bb]cd`, `cat{strfold{AB}str{cd}}`}, + + // Factoring. + {`abc|abd|aef|bcx|bcy`, `alt{cat{lit{a}alt{cat{lit{b}cc{0x63-0x64}}str{ef}}}cat{str{bc}cc{0x78-0x79}}}`}, + {`ax+y|ax+z|ay+w`, `cat{lit{a}alt{cat{plus{lit{x}}cc{0x79-0x7a}}cat{plus{lit{y}}lit{w}}}}`}, +} + +const testFlags = MatchNL | PerlX | UnicodeGroups + +// Test Parse -> Dump. +func TestParseDump(t *testing.T) { + for _, tt := range parseTests { + re, err := Parse(tt.Regexp, testFlags) + if err != nil { + t.Errorf("Parse(%#q): %v", tt.Regexp, err) + continue + } + d := dump(re) + if d != tt.Dump { + t.Errorf("Parse(%#q).Dump() = %#q want %#q", tt.Regexp, d, tt.Dump) + } + } +} + +// dump prints a string representation of the regexp showing +// the structure explicitly. +func dump(re *Regexp) string { + var b bytes.Buffer + dumpRegexp(&b, re) + return b.String() +} + +var opNames = []string{ + OpNoMatch: "no", + OpEmptyMatch: "emp", + OpLiteral: "lit", + OpCharClass: "cc", + OpAnyCharNotNL: "dnl", + OpAnyChar: "dot", + OpBeginLine: "bol", + OpEndLine: "eol", + OpBeginText: "bot", + OpEndText: "eot", + OpWordBoundary: "wb", + OpNoWordBoundary: "nwb", + OpCapture: "cap", + OpStar: "star", + OpPlus: "plus", + OpQuest: "que", + OpRepeat: "rep", + OpConcat: "cat", + OpAlternate: "alt", +} + +// dumpRegexp writes an encoding of the syntax tree for the regexp re to b. +// It is used during testing to distinguish between parses that might print +// the same using re's String method. +func dumpRegexp(b *bytes.Buffer, re *Regexp) { + if int(re.Op) >= len(opNames) || opNames[re.Op] == "" { + fmt.Fprintf(b, "op%d", re.Op) + } else { + switch re.Op { + default: + b.WriteString(opNames[re.Op]) + case OpStar, OpPlus, OpQuest, OpRepeat: + if re.Flags&NonGreedy != 0 { + b.WriteByte('n') + } + b.WriteString(opNames[re.Op]) + case OpLiteral: + if len(re.Rune) > 1 { + b.WriteString("str") + } else { + b.WriteString("lit") + } + if re.Flags&FoldCase != 0 { + for _, r := range re.Rune { + if unicode.SimpleFold(r) != r { + b.WriteString("fold") + break + } + } + } + } + } + b.WriteByte('{') + switch re.Op { + case OpEndText: + if re.Flags&WasDollar == 0 { + b.WriteString(`\z`) + } + case OpLiteral: + for _, r := range re.Rune { + b.WriteRune(r) + } + case OpConcat, OpAlternate: + for _, sub := range re.Sub { + dumpRegexp(b, sub) + } + case OpStar, OpPlus, OpQuest: + dumpRegexp(b, re.Sub[0]) + case OpRepeat: + fmt.Fprintf(b, "%d,%d ", re.Min, re.Max) + dumpRegexp(b, re.Sub[0]) + case OpCapture: + if re.Name != "" { + b.WriteString(re.Name) + b.WriteByte(':') + } + dumpRegexp(b, re.Sub[0]) + case OpCharClass: + sep := "" + for i := 0; i < len(re.Rune); i += 2 { + b.WriteString(sep) + sep = " " + lo, hi := re.Rune[i], re.Rune[i+1] + if lo == hi { + fmt.Fprintf(b, "%#x", lo) + } else { + fmt.Fprintf(b, "%#x-%#x", lo, hi) + } + } + } + b.WriteByte('}') +} + +func mkCharClass(f func(int) bool) string { + re := &Regexp{Op: OpCharClass} + lo := -1 + for i := 0; i <= unicode.MaxRune; i++ { + if f(i) { + if lo < 0 { + lo = i + } + } else { + if lo >= 0 { + re.Rune = append(re.Rune, lo, i-1) + lo = -1 + } + } + } + if lo >= 0 { + re.Rune = append(re.Rune, lo, unicode.MaxRune) + } + return dump(re) +} + +func isUpperFold(rune int) bool { + if unicode.IsUpper(rune) { + return true + } + c := unicode.SimpleFold(rune) + for c != rune { + if unicode.IsUpper(c) { + return true + } + c = unicode.SimpleFold(c) + } + return false +} + +func TestFoldConstants(t *testing.T) { + last := -1 + for i := 0; i <= unicode.MaxRune; i++ { + if unicode.SimpleFold(i) == i { + continue + } + if last == -1 && minFold != i { + t.Errorf("minFold=%#U should be %#U", minFold, i) + } + last = i + } + if maxFold != last { + t.Errorf("maxFold=%#U should be %#U", maxFold, last) + } +} + +func TestAppendRangeCollapse(t *testing.T) { + // AppendRange should collapse each of the new ranges + // into the earlier ones (it looks back two ranges), so that + // the slice never grows very large. + // Note that we are not calling cleanClass. + var r []int + for i := 'A'; i <= 'Z'; i++ { + r = appendRange(r, i, i) + r = appendRange(r, i+'a'-'A', i+'a'-'A') + } + if string(r) != "AZaz" { + t.Errorf("appendRange interlaced A-Z a-z = %s, want AZaz", string(r)) + } +} diff --git a/src/pkg/exp/regexp/syntax/perl_groups.go b/src/pkg/exp/regexp/syntax/perl_groups.go new file mode 100644 index 000000000..05b392c40 --- /dev/null +++ b/src/pkg/exp/regexp/syntax/perl_groups.go @@ -0,0 +1,130 @@ +// GENERATED BY make_perl_groups.pl; DO NOT EDIT. +// make_perl_groups.pl >perl_groups.go + +package syntax + +var code1 = []int{ /* \d */ + 0x30, 0x39, +} + +var code2 = []int{ /* \s */ + 0x9, 0xa, + 0xc, 0xd, + 0x20, 0x20, +} + +var code3 = []int{ /* \w */ + 0x30, 0x39, + 0x41, 0x5a, + 0x5f, 0x5f, + 0x61, 0x7a, +} + +var perlGroup = map[string]charGroup{ + `\d`: {+1, code1}, + `\D`: {-1, code1}, + `\s`: {+1, code2}, + `\S`: {-1, code2}, + `\w`: {+1, code3}, + `\W`: {-1, code3}, +} +var code4 = []int{ /* [:alnum:] */ + 0x30, 0x39, + 0x41, 0x5a, + 0x61, 0x7a, +} + +var code5 = []int{ /* [:alpha:] */ + 0x41, 0x5a, + 0x61, 0x7a, +} + +var code6 = []int{ /* [:ascii:] */ + 0x0, 0x7f, +} + +var code7 = []int{ /* [:blank:] */ + 0x9, 0x9, + 0x20, 0x20, +} + +var code8 = []int{ /* [:cntrl:] */ + 0x0, 0x1f, + 0x7f, 0x7f, +} + +var code9 = []int{ /* [:digit:] */ + 0x30, 0x39, +} + +var code10 = []int{ /* [:graph:] */ + 0x21, 0x7e, +} + +var code11 = []int{ /* [:lower:] */ + 0x61, 0x7a, +} + +var code12 = []int{ /* [:print:] */ + 0x20, 0x7e, +} + +var code13 = []int{ /* [:punct:] */ + 0x21, 0x2f, + 0x3a, 0x40, + 0x5b, 0x60, + 0x7b, 0x7e, +} + +var code14 = []int{ /* [:space:] */ + 0x9, 0xd, + 0x20, 0x20, +} + +var code15 = []int{ /* [:upper:] */ + 0x41, 0x5a, +} + +var code16 = []int{ /* [:word:] */ + 0x30, 0x39, + 0x41, 0x5a, + 0x5f, 0x5f, + 0x61, 0x7a, +} + +var code17 = []int{ /* [:xdigit:] */ + 0x30, 0x39, + 0x41, 0x46, + 0x61, 0x66, +} + +var posixGroup = map[string]charGroup{ + `[:alnum:]`: {+1, code4}, + `[:^alnum:]`: {-1, code4}, + `[:alpha:]`: {+1, code5}, + `[:^alpha:]`: {-1, code5}, + `[:ascii:]`: {+1, code6}, + `[:^ascii:]`: {-1, code6}, + `[:blank:]`: {+1, code7}, + `[:^blank:]`: {-1, code7}, + `[:cntrl:]`: {+1, code8}, + `[:^cntrl:]`: {-1, code8}, + `[:digit:]`: {+1, code9}, + `[:^digit:]`: {-1, code9}, + `[:graph:]`: {+1, code10}, + `[:^graph:]`: {-1, code10}, + `[:lower:]`: {+1, code11}, + `[:^lower:]`: {-1, code11}, + `[:print:]`: {+1, code12}, + `[:^print:]`: {-1, code12}, + `[:punct:]`: {+1, code13}, + `[:^punct:]`: {-1, code13}, + `[:space:]`: {+1, code14}, + `[:^space:]`: {-1, code14}, + `[:upper:]`: {+1, code15}, + `[:^upper:]`: {-1, code15}, + `[:word:]`: {+1, code16}, + `[:^word:]`: {-1, code16}, + `[:xdigit:]`: {+1, code17}, + `[:^xdigit:]`: {-1, code17}, +} diff --git a/src/pkg/exp/regexp/syntax/prog.go b/src/pkg/exp/regexp/syntax/prog.go new file mode 100644 index 000000000..bf85b720d --- /dev/null +++ b/src/pkg/exp/regexp/syntax/prog.go @@ -0,0 +1,237 @@ +package syntax + +import ( + "bytes" + "strconv" +) + +// Compiled program. +// May not belong in this package, but convenient for now. + +// A Prog is a compiled regular expression program. +type Prog struct { + Inst []Inst + Start int // index of start instruction + NumCap int // number of InstCapture insts in re +} + +// An InstOp is an instruction opcode. +type InstOp uint8 + +const ( + InstAlt InstOp = iota + InstAltMatch + InstCapture + InstEmptyWidth + InstMatch + InstFail + InstNop + InstRune +) + +// An EmptyOp specifies a kind or mixture of zero-width assertions. +type EmptyOp uint8 + +const ( + EmptyBeginLine EmptyOp = 1 << iota + EmptyEndLine + EmptyBeginText + EmptyEndText + EmptyWordBoundary + EmptyNoWordBoundary +) + +// An Inst is a single instruction in a regular expression program. +type Inst struct { + Op InstOp + Out uint32 // all but InstMatch, InstFail + Arg uint32 // InstAlt, InstAltMatch, InstCapture, InstEmptyWidth + Rune []int +} + +func (p *Prog) String() string { + var b bytes.Buffer + dumpProg(&b, p) + return b.String() +} + +// skipNop follows any no-op or capturing instructions +// and returns the resulting pc. +func (p *Prog) skipNop(pc uint32) *Inst { + i := &p.Inst[pc] + for i.Op == InstNop || i.Op == InstCapture { + pc = i.Out + i = &p.Inst[pc] + } + return i +} + +// Prefix returns a literal string that all matches for the +// regexp must start with. Complete is true if the prefix +// is the entire match. +func (p *Prog) Prefix() (prefix string, complete bool) { + i := p.skipNop(uint32(p.Start)) + + // Avoid allocation of buffer if prefix is empty. + if i.Op != InstRune || len(i.Rune) != 1 { + return "", i.Op == InstMatch + } + + // Have prefix; gather characters. + var buf bytes.Buffer + for i.Op == InstRune && len(i.Rune) == 1 { + buf.WriteRune(i.Rune[0]) + i = p.skipNop(i.Out) + } + return buf.String(), i.Op == InstMatch +} + +// StartCond returns the leading empty-width conditions that must +// be true in any match. It returns ^EmptyOp(0) if no matches are possible. +func (p *Prog) StartCond() EmptyOp { + var flag EmptyOp + pc := uint32(p.Start) + i := &p.Inst[pc] +Loop: + for { + switch i.Op { + case InstEmptyWidth: + flag |= EmptyOp(i.Arg) + case InstFail: + return ^EmptyOp(0) + case InstCapture, InstNop: + // skip + default: + break Loop + } + pc = i.Out + i = &p.Inst[pc] + } + return flag +} + +// MatchRune returns true if the instruction matches (and consumes) r. +// It should only be called when i.Op == InstRune. +func (i *Inst) MatchRune(r int) bool { + rune := i.Rune + + // Special case: single-rune slice is from literal string, not char class. + // TODO: Case folding. + if len(rune) == 1 { + return r == rune[0] + } + + // Peek at the first few pairs. + // Should handle ASCII well. + for j := 0; j < len(rune) && j <= 8; j += 2 { + if r < rune[j] { + return false + } + if r <= rune[j+1] { + return true + } + } + + // Otherwise binary search. + lo := 0 + hi := len(rune) / 2 + for lo < hi { + m := lo + (hi-lo)/2 + if c := rune[2*m]; c <= r { + if r <= rune[2*m+1] { + return true + } + lo = m + 1 + } else { + hi = m + } + } + return false +} + +// As per re2's Prog::IsWordChar. Determines whether rune is an ASCII word char. +// Since we act on runes, it would be easy to support Unicode here. +func wordRune(rune int) bool { + return rune == '_' || + ('A' <= rune && rune <= 'Z') || + ('a' <= rune && rune <= 'z') || + ('0' <= rune && rune <= '9') +} + +// MatchEmptyWidth returns true if the instruction matches +// an empty string between the runes before and after. +// It should only be called when i.Op == InstEmptyWidth. +func (i *Inst) MatchEmptyWidth(before int, after int) bool { + switch EmptyOp(i.Arg) { + case EmptyBeginLine: + return before == '\n' || before == -1 + case EmptyEndLine: + return after == '\n' || after == -1 + case EmptyBeginText: + return before == -1 + case EmptyEndText: + return after == -1 + case EmptyWordBoundary: + return wordRune(before) != wordRune(after) + case EmptyNoWordBoundary: + return wordRune(before) == wordRune(after) + } + panic("unknown empty width arg") +} + +func (i *Inst) String() string { + var b bytes.Buffer + dumpInst(&b, i) + return b.String() +} + +func bw(b *bytes.Buffer, args ...string) { + for _, s := range args { + b.WriteString(s) + } +} + +func dumpProg(b *bytes.Buffer, p *Prog) { + for j := range p.Inst { + i := &p.Inst[j] + pc := strconv.Itoa(j) + if len(pc) < 3 { + b.WriteString(" "[len(pc):]) + } + if j == p.Start { + pc += "*" + } + bw(b, pc, "\t") + dumpInst(b, i) + bw(b, "\n") + } +} + +func u32(i uint32) string { + return strconv.Uitoa64(uint64(i)) +} + +func dumpInst(b *bytes.Buffer, i *Inst) { + switch i.Op { + case InstAlt: + bw(b, "alt -> ", u32(i.Out), ", ", u32(i.Arg)) + case InstAltMatch: + bw(b, "altmatch -> ", u32(i.Out), ", ", u32(i.Arg)) + case InstCapture: + bw(b, "cap ", u32(i.Arg), " -> ", u32(i.Out)) + case InstEmptyWidth: + bw(b, "empty ", u32(i.Arg), " -> ", u32(i.Out)) + case InstMatch: + bw(b, "match") + case InstFail: + bw(b, "fail") + case InstNop: + bw(b, "nop -> ", u32(i.Out)) + case InstRune: + if i.Rune == nil { + // shouldn't happen + bw(b, "rune ") + } + bw(b, "rune ", strconv.QuoteToASCII(string(i.Rune)), " -> ", u32(i.Out)) + } +} diff --git a/src/pkg/exp/regexp/syntax/prog_test.go b/src/pkg/exp/regexp/syntax/prog_test.go new file mode 100644 index 000000000..7be4281c2 --- /dev/null +++ b/src/pkg/exp/regexp/syntax/prog_test.go @@ -0,0 +1,91 @@ +package syntax + +import ( + "testing" +) + +var compileTests = []struct { + Regexp string + Prog string +}{ + {"a", ` 0 fail + 1* rune "a" -> 2 + 2 match +`}, + {"[A-M][n-z]", ` 0 fail + 1* rune "AM" -> 2 + 2 rune "nz" -> 3 + 3 match +`}, + {"", ` 0 fail + 1* nop -> 2 + 2 match +`}, + {"a?", ` 0 fail + 1 rune "a" -> 3 + 2* alt -> 1, 3 + 3 match +`}, + {"a??", ` 0 fail + 1 rune "a" -> 3 + 2* alt -> 3, 1 + 3 match +`}, + {"a+", ` 0 fail + 1* rune "a" -> 2 + 2 alt -> 1, 3 + 3 match +`}, + {"a+?", ` 0 fail + 1* rune "a" -> 2 + 2 alt -> 3, 1 + 3 match +`}, + {"a*", ` 0 fail + 1 rune "a" -> 2 + 2* alt -> 1, 3 + 3 match +`}, + {"a*?", ` 0 fail + 1 rune "a" -> 2 + 2* alt -> 3, 1 + 3 match +`}, + {"a+b+", ` 0 fail + 1* rune "a" -> 2 + 2 alt -> 1, 3 + 3 rune "b" -> 4 + 4 alt -> 3, 5 + 5 match +`}, + {"(a+)(b+)", ` 0 fail + 1* cap 2 -> 2 + 2 rune "a" -> 3 + 3 alt -> 2, 4 + 4 cap 3 -> 5 + 5 cap 4 -> 6 + 6 rune "b" -> 7 + 7 alt -> 6, 8 + 8 cap 5 -> 9 + 9 match +`}, + {"a+|b+", ` 0 fail + 1 rune "a" -> 2 + 2 alt -> 1, 6 + 3 rune "b" -> 4 + 4 alt -> 3, 6 + 5* alt -> 1, 3 + 6 match +`}, +} + +func TestCompile(t *testing.T) { + for _, tt := range compileTests { + re, _ := Parse(tt.Regexp, Perl) + p, _ := Compile(re) + s := p.String() + if s != tt.Prog { + t.Errorf("compiled %#q:\n--- have\n%s---\n--- want\n%s---", tt.Regexp, s, tt.Prog) + } + } +} diff --git a/src/pkg/exp/regexp/syntax/regexp.go b/src/pkg/exp/regexp/syntax/regexp.go new file mode 100644 index 000000000..00a4addef --- /dev/null +++ b/src/pkg/exp/regexp/syntax/regexp.go @@ -0,0 +1,284 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Package syntax parses regular expressions into syntax trees. +// WORK IN PROGRESS. +package syntax + +// Note to implementers: +// In this package, re is always a *Regexp and r is always a rune. + +import ( + "bytes" + "strconv" + "strings" + "unicode" +) + +// A Regexp is a node in a regular expression syntax tree. +type Regexp struct { + Op Op // operator + Flags Flags + Sub []*Regexp // subexpressions, if any + Sub0 [1]*Regexp // storage for short Sub + Rune []int // matched runes, for OpLiteral, OpCharClass + Rune0 [2]int // storage for short Rune + Min, Max int // min, max for OpRepeat + Cap int // capturing index, for OpCapture + Name string // capturing name, for OpCapture +} + +// An Op is a single regular expression operator. +type Op uint8 + +// Operators are listed in precedence order, tightest binding to weakest. +// Character class operators are listed simplest to most complex +// (OpLiteral, OpCharClass, OpAnyCharNotNL, OpAnyChar). + +const ( + OpNoMatch Op = 1 + iota // matches no strings + OpEmptyMatch // matches empty string + OpLiteral // matches Runes sequence + OpCharClass // matches Runes interpreted as range pair list + OpAnyCharNotNL // matches any character + OpAnyChar // matches any character + OpBeginLine // matches empty string at beginning of line + OpEndLine // matches empty string at end of line + OpBeginText // matches empty string at beginning of text + OpEndText // matches empty string at end of text + OpWordBoundary // matches word boundary `\b` + OpNoWordBoundary // matches word non-boundary `\B` + OpCapture // capturing subexpression with index Cap, optional name Name + OpStar // matches Sub[0] zero or more times + OpPlus // matches Sub[0] one or more times + OpQuest // matches Sub[0] zero or one times + OpRepeat // matches Sub[0] at least Min times, at most Max (Max == -1 is no limit) + OpConcat // matches concatenation of Subs + OpAlternate // matches alternation of Subs +) + +const opPseudo Op = 128 // where pseudo-ops start + +// Equal returns true if x and y have identical structure. +func (x *Regexp) Equal(y *Regexp) bool { + if x == nil || y == nil { + return x == y + } + if x.Op != y.Op { + return false + } + switch x.Op { + case OpEndText: + // The parse flags remember whether this is \z or \Z. + if x.Flags&WasDollar != y.Flags&WasDollar { + return false + } + + case OpLiteral, OpCharClass: + if len(x.Rune) != len(y.Rune) { + return false + } + for i, r := range x.Rune { + if r != y.Rune[i] { + return false + } + } + + case OpAlternate, OpConcat: + if len(x.Sub) != len(y.Sub) { + return false + } + for i, sub := range x.Sub { + if !sub.Equal(y.Sub[i]) { + return false + } + } + + case OpStar, OpPlus, OpQuest: + if x.Flags&NonGreedy != y.Flags&NonGreedy || !x.Sub[0].Equal(y.Sub[0]) { + return false + } + + case OpRepeat: + if x.Flags&NonGreedy != y.Flags&NonGreedy || x.Min != y.Min || x.Max != y.Max || !x.Sub[0].Equal(y.Sub[0]) { + return false + } + + case OpCapture: + if x.Cap != y.Cap || x.Name != y.Name || !x.Sub[0].Equal(y.Sub[0]) { + return false + } + } + return true +} + +// writeRegexp writes the Perl syntax for the regular expression re to b. +func writeRegexp(b *bytes.Buffer, re *Regexp) { + switch re.Op { + default: + b.WriteString("") + case OpNoMatch: + b.WriteString(`[^\x00-\x{10FFFF}]`) + case OpEmptyMatch: + b.WriteString(`(?:)`) + case OpLiteral: + if re.Flags&FoldCase != 0 { + b.WriteString(`(?i:`) + } + for _, r := range re.Rune { + escape(b, r, false) + } + if re.Flags&FoldCase != 0 { + b.WriteString(`)`) + } + case OpCharClass: + if len(re.Rune)%2 != 0 { + b.WriteString(`[invalid char class]`) + break + } + b.WriteRune('[') + if len(re.Rune) == 0 { + b.WriteString(`^\x00-\x{10FFFF}`) + } else if re.Rune[0] == 0 && re.Rune[len(re.Rune)-1] == unicode.MaxRune { + // Contains 0 and MaxRune. Probably a negated class. + // Print the gaps. + b.WriteRune('^') + for i := 1; i < len(re.Rune)-1; i += 2 { + lo, hi := re.Rune[i]+1, re.Rune[i+1]-1 + escape(b, lo, lo == '-') + if lo != hi { + b.WriteRune('-') + escape(b, hi, hi == '-') + } + } + } else { + for i := 0; i < len(re.Rune); i += 2 { + lo, hi := re.Rune[i], re.Rune[i+1] + escape(b, lo, lo == '-') + if lo != hi { + b.WriteRune('-') + escape(b, hi, hi == '-') + } + } + } + b.WriteRune(']') + case OpAnyCharNotNL: + b.WriteString(`[^\n]`) + case OpAnyChar: + b.WriteRune('.') + case OpBeginLine: + b.WriteRune('^') + case OpEndLine: + b.WriteRune('$') + case OpBeginText: + b.WriteString(`\A`) + case OpEndText: + b.WriteString(`\z`) + case OpWordBoundary: + b.WriteString(`\b`) + case OpNoWordBoundary: + b.WriteString(`\B`) + case OpCapture: + if re.Name != "" { + b.WriteString(`(?P<`) + b.WriteString(re.Name) + b.WriteRune('>') + } else { + b.WriteRune('(') + } + if re.Sub[0].Op != OpEmptyMatch { + writeRegexp(b, re.Sub[0]) + } + b.WriteRune(')') + case OpStar, OpPlus, OpQuest, OpRepeat: + if sub := re.Sub[0]; sub.Op > OpCapture { + b.WriteString(`(?:`) + writeRegexp(b, sub) + b.WriteString(`)`) + } else { + writeRegexp(b, sub) + } + switch re.Op { + case OpStar: + b.WriteRune('*') + case OpPlus: + b.WriteRune('+') + case OpQuest: + b.WriteRune('?') + case OpRepeat: + b.WriteRune('{') + b.WriteString(strconv.Itoa(re.Min)) + if re.Max != re.Min { + b.WriteRune(',') + if re.Max >= 0 { + b.WriteString(strconv.Itoa(re.Max)) + } + } + b.WriteRune('}') + } + case OpConcat: + for _, sub := range re.Sub { + if sub.Op == OpAlternate { + b.WriteString(`(?:`) + writeRegexp(b, sub) + b.WriteString(`)`) + } else { + writeRegexp(b, sub) + } + } + case OpAlternate: + for i, sub := range re.Sub { + if i > 0 { + b.WriteRune('|') + } + writeRegexp(b, sub) + } + } +} + +func (re *Regexp) String() string { + var b bytes.Buffer + writeRegexp(&b, re) + return b.String() +} + +const meta = `\.+*?()|[]{}^$` + +func escape(b *bytes.Buffer, r int, force bool) { + if unicode.IsPrint(r) { + if strings.IndexRune(meta, r) >= 0 || force { + b.WriteRune('\\') + } + b.WriteRune(r) + return + } + + switch r { + case '\a': + b.WriteString(`\a`) + case '\f': + b.WriteString(`\f`) + case '\n': + b.WriteString(`\n`) + case '\r': + b.WriteString(`\r`) + case '\t': + b.WriteString(`\t`) + case '\v': + b.WriteString(`\v`) + default: + if r < 0x100 { + b.WriteString(`\x`) + s := strconv.Itob(r, 16) + if len(s) == 1 { + b.WriteRune('0') + } + b.WriteString(s) + break + } + b.WriteString(`\x{`) + b.WriteString(strconv.Itob(r, 16)) + b.WriteString(`}`) + } +} diff --git a/src/pkg/exp/regexp/syntax/simplify.go b/src/pkg/exp/regexp/syntax/simplify.go new file mode 100644 index 000000000..72390417b --- /dev/null +++ b/src/pkg/exp/regexp/syntax/simplify.go @@ -0,0 +1,151 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package syntax + +// Simplify returns a regexp equivalent to re but without counted repetitions +// and with various other simplifications, such as rewriting /(?:a+)+/ to /a+/. +// The resulting regexp will execute correctly but its string representation +// will not produce the same parse tree, because capturing parentheses +// may have been duplicated or removed. For example, the simplified form +// for /(x){1,2}/ is /(x)(x)?/ but both parentheses capture as $1. +// The returned regexp may share structure with or be the original. +func (re *Regexp) Simplify() *Regexp { + if re == nil { + return nil + } + switch re.Op { + case OpCapture, OpConcat, OpAlternate: + // Simplify children, building new Regexp if children change. + nre := re + for i, sub := range re.Sub { + nsub := sub.Simplify() + if nre == re && nsub != sub { + // Start a copy. + nre = new(Regexp) + *nre = *re + nre.Rune = nil + nre.Sub = append(nre.Sub0[:0], re.Sub[:i]...) + } + if nre != re { + nre.Sub = append(nre.Sub, nsub) + } + } + return nre + + case OpStar, OpPlus, OpQuest: + sub := re.Sub[0].Simplify() + return simplify1(re.Op, re.Flags, sub, re) + + case OpRepeat: + // Special special case: x{0} matches the empty string + // and doesn't even need to consider x. + if re.Min == 0 && re.Max == 0 { + return &Regexp{Op: OpEmptyMatch} + } + + // The fun begins. + sub := re.Sub[0].Simplify() + + // x{n,} means at least n matches of x. + if re.Max == -1 { + // Special case: x{0,} is x*. + if re.Min == 0 { + return simplify1(OpStar, re.Flags, sub, nil) + } + + // Special case: x{1,} is x+. + if re.Min == 1 { + return simplify1(OpPlus, re.Flags, sub, nil) + } + + // General case: x{4,} is xxxx+. + nre := &Regexp{Op: OpConcat} + nre.Sub = nre.Sub0[:0] + for i := 0; i < re.Min-1; i++ { + nre.Sub = append(nre.Sub, sub) + } + nre.Sub = append(nre.Sub, simplify1(OpPlus, re.Flags, sub, nil)) + return nre + } + + // Special case x{0} handled above. + + // Special case: x{1} is just x. + if re.Min == 1 && re.Max == 1 { + return sub + } + + // General case: x{n,m} means n copies of x and m copies of x? + // The machine will do less work if we nest the final m copies, + // so that x{2,5} = xx(x(x(x)?)?)? + + // Build leading prefix: xx. + var prefix *Regexp + if re.Min > 0 { + prefix = &Regexp{Op: OpConcat} + prefix.Sub = prefix.Sub0[:0] + for i := 0; i < re.Min; i++ { + prefix.Sub = append(prefix.Sub, sub) + } + } + + // Build and attach suffix: (x(x(x)?)?)? + if re.Max > re.Min { + suffix := simplify1(OpQuest, re.Flags, sub, nil) + for i := re.Min + 1; i < re.Max; i++ { + nre2 := &Regexp{Op: OpConcat} + nre2.Sub = append(nre2.Sub0[:0], sub, suffix) + suffix = simplify1(OpQuest, re.Flags, nre2, nil) + } + if prefix == nil { + return suffix + } + prefix.Sub = append(prefix.Sub, suffix) + } + if prefix != nil { + return prefix + } + + // Some degenerate case like min > max or min < max < 0. + // Handle as impossible match. + return &Regexp{Op: OpNoMatch} + } + + return re +} + +// simplify1 implements Simplify for the unary OpStar, +// OpPlus, and OpQuest operators. It returns the simple regexp +// equivalent to +// +// Regexp{Op: op, Flags: flags, Sub: {sub}} +// +// under the assumption that sub is already simple, and +// without first allocating that structure. If the regexp +// to be returned turns out to be equivalent to re, simplify1 +// returns re instead. +// +// simplify1 is factored out of Simplify because the implementation +// for other operators generates these unary expressions. +// Letting them call simplify1 makes sure the expressions they +// generate are simple. +func simplify1(op Op, flags Flags, sub, re *Regexp) *Regexp { + // Special case: repeat the empty string as much as + // you want, but it's still the empty string. + if sub.Op == OpEmptyMatch { + return sub + } + // The operators are idempotent if the flags match. + if op == sub.Op && flags&NonGreedy == sub.Flags&NonGreedy { + return sub + } + if re != nil && re.Op == op && re.Flags&NonGreedy == flags&NonGreedy && sub == re.Sub[0] { + return re + } + + re = &Regexp{Op: op, Flags: flags} + re.Sub = append(re.Sub0[:0], sub) + return re +} diff --git a/src/pkg/exp/regexp/syntax/simplify_test.go b/src/pkg/exp/regexp/syntax/simplify_test.go new file mode 100644 index 000000000..c8cec2183 --- /dev/null +++ b/src/pkg/exp/regexp/syntax/simplify_test.go @@ -0,0 +1,151 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package syntax + +import "testing" + +var simplifyTests = []struct { + Regexp string + Simple string +}{ + // Already-simple constructs + {`a`, `a`}, + {`ab`, `ab`}, + {`a|b`, `[a-b]`}, + {`ab|cd`, `ab|cd`}, + {`(ab)*`, `(ab)*`}, + {`(ab)+`, `(ab)+`}, + {`(ab)?`, `(ab)?`}, + {`.`, `.`}, + {`^`, `^`}, + {`$`, `$`}, + {`[ac]`, `[ac]`}, + {`[^ac]`, `[^ac]`}, + + // Posix character classes + {`[[:alnum:]]`, `[0-9A-Za-z]`}, + {`[[:alpha:]]`, `[A-Za-z]`}, + {`[[:blank:]]`, `[\t ]`}, + {`[[:cntrl:]]`, `[\x00-\x1f\x7f]`}, + {`[[:digit:]]`, `[0-9]`}, + {`[[:graph:]]`, `[!-~]`}, + {`[[:lower:]]`, `[a-z]`}, + {`[[:print:]]`, `[ -~]`}, + {`[[:punct:]]`, "[!-/:-@\\[-`\\{-~]"}, + {`[[:space:]]`, `[\t-\r ]`}, + {`[[:upper:]]`, `[A-Z]`}, + {`[[:xdigit:]]`, `[0-9A-Fa-f]`}, + + // Perl character classes + {`\d`, `[0-9]`}, + {`\s`, `[\t-\n\f-\r ]`}, + {`\w`, `[0-9A-Z_a-z]`}, + {`\D`, `[^0-9]`}, + {`\S`, `[^\t-\n\f-\r ]`}, + {`\W`, `[^0-9A-Z_a-z]`}, + {`[\d]`, `[0-9]`}, + {`[\s]`, `[\t-\n\f-\r ]`}, + {`[\w]`, `[0-9A-Z_a-z]`}, + {`[\D]`, `[^0-9]`}, + {`[\S]`, `[^\t-\n\f-\r ]`}, + {`[\W]`, `[^0-9A-Z_a-z]`}, + + // Posix repetitions + {`a{1}`, `a`}, + {`a{2}`, `aa`}, + {`a{5}`, `aaaaa`}, + {`a{0,1}`, `a?`}, + // The next three are illegible because Simplify inserts (?:) + // parens instead of () parens to avoid creating extra + // captured subexpressions. The comments show a version with fewer parens. + {`(a){0,2}`, `(?:(a)(a)?)?`}, // (aa?)? + {`(a){0,4}`, `(?:(a)(?:(a)(?:(a)(a)?)?)?)?`}, // (a(a(aa?)?)?)? + {`(a){2,6}`, `(a)(a)(?:(a)(?:(a)(?:(a)(a)?)?)?)?`}, // aa(a(a(aa?)?)?)? + {`a{0,2}`, `(?:aa?)?`}, // (aa?)? + {`a{0,4}`, `(?:a(?:a(?:aa?)?)?)?`}, // (a(a(aa?)?)?)? + {`a{2,6}`, `aa(?:a(?:a(?:aa?)?)?)?`}, // aa(a(a(aa?)?)?)? + {`a{0,}`, `a*`}, + {`a{1,}`, `a+`}, + {`a{2,}`, `aa+`}, + {`a{5,}`, `aaaaa+`}, + + // Test that operators simplify their arguments. + {`(?:a{1,}){1,}`, `a+`}, + {`(a{1,}b{1,})`, `(a+b+)`}, + {`a{1,}|b{1,}`, `a+|b+`}, + {`(?:a{1,})*`, `(?:a+)*`}, + {`(?:a{1,})+`, `a+`}, + {`(?:a{1,})?`, `(?:a+)?`}, + {``, `(?:)`}, + {`a{0}`, `(?:)`}, + + // Character class simplification + {`[ab]`, `[a-b]`}, + {`[a-za-za-z]`, `[a-z]`}, + {`[A-Za-zA-Za-z]`, `[A-Za-z]`}, + {`[ABCDEFGH]`, `[A-H]`}, + {`[AB-CD-EF-GH]`, `[A-H]`}, + {`[W-ZP-XE-R]`, `[E-Z]`}, + {`[a-ee-gg-m]`, `[a-m]`}, + {`[a-ea-ha-m]`, `[a-m]`}, + {`[a-ma-ha-e]`, `[a-m]`}, + {`[a-zA-Z0-9 -~]`, `[ -~]`}, + + // Empty character classes + {`[^[:cntrl:][:^cntrl:]]`, `[^\x00-\x{10FFFF}]`}, + + // Full character classes + {`[[:cntrl:][:^cntrl:]]`, `.`}, + + // Unicode case folding. + {`(?i)A`, `(?i:A)`}, + {`(?i)a`, `(?i:a)`}, + {`(?i)[A]`, `(?i:A)`}, + {`(?i)[a]`, `(?i:A)`}, + {`(?i)K`, `(?i:K)`}, + {`(?i)k`, `(?i:k)`}, + {`(?i)\x{212a}`, "(?i:\u212A)"}, + {`(?i)[K]`, "[Kk\u212A]"}, + {`(?i)[k]`, "[Kk\u212A]"}, + {`(?i)[\x{212a}]`, "[Kk\u212A]"}, + {`(?i)[a-z]`, "[A-Za-z\u017F\u212A]"}, + {`(?i)[\x00-\x{FFFD}]`, "[\\x00-\uFFFD]"}, + {`(?i)[\x00-\x{10FFFF}]`, `.`}, + + // Empty string as a regular expression. + // The empty string must be preserved inside parens in order + // to make submatches work right, so these tests are less + // interesting than they might otherwise be. String inserts + // explicit (?:) in place of non-parenthesized empty strings, + // to make them easier to spot for other parsers. + {`(a|b|)`, `([a-b]|(?:))`}, + {`(|)`, `()`}, + {`a()`, `a()`}, + {`(()|())`, `(()|())`}, + {`(a|)`, `(a|(?:))`}, + {`ab()cd()`, `ab()cd()`}, + {`()`, `()`}, + {`()*`, `()*`}, + {`()+`, `()+`}, + {`()?`, `()?`}, + {`(){0}`, `(?:)`}, + {`(){1}`, `()`}, + {`(){1,}`, `()+`}, + {`(){0,2}`, `(?:()()?)?`}, +} + +func TestSimplify(t *testing.T) { + for _, tt := range simplifyTests { + re, err := Parse(tt.Regexp, MatchNL|Perl&^OneLine) + if err != nil { + t.Errorf("Parse(%#q) = error %v", tt.Regexp, err) + continue + } + s := re.Simplify().String() + if s != tt.Simple { + t.Errorf("Simplify(%#q) = %#q, want %#q", tt.Regexp, s, tt.Simple) + } + } +} diff --git a/src/pkg/exp/template/html/Makefile b/src/pkg/exp/template/html/Makefile new file mode 100644 index 000000000..2f107da11 --- /dev/null +++ b/src/pkg/exp/template/html/Makefile @@ -0,0 +1,11 @@ +# Copyright 2011 The Go Authors. All rights reserved. +# Use of this source code is governed by a BSD-style +# license that can be found in the LICENSE file. + +include ../../../../Make.inc + +TARG=exp/template/html +GOFILES=\ + escape.go + +include ../../../../Make.pkg diff --git a/src/pkg/exp/template/html/context.go b/src/pkg/exp/template/html/context.go new file mode 100644 index 000000000..411006883 --- /dev/null +++ b/src/pkg/exp/template/html/context.go @@ -0,0 +1,98 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package html + +import ( + "fmt" +) + +// context describes the state an HTML parser must be in when it reaches the +// portion of HTML produced by evaluating a particular template node. +// +// The zero value of type context is the start context for a template that +// produces an HTML fragment as defined at +// http://www.w3.org/TR/html5/the-end.html#parsing-html-fragments +// where the context element is null. +type context struct { + state state + delim delim +} + +func (c context) String() string { + return fmt.Sprintf("context{state: %s, delim: %s", c.state, c.delim) +} + +// eq is true if the two contexts are identical field-wise. +func (c context) eq(d context) bool { + return c.state == d.state && c.delim == d.delim +} + +// state describes a high-level HTML parser state. +// +// It bounds the top of the element stack, and by extension the HTML +// insertion mode, but also contains state that does not correspond to +// anything in the HTML5 parsing algorithm because a single token +// production in the HTML grammar may contain embedded actions in a template. +// For instance, the quoted HTML attribute produced by +//
+// is a single token in HTML's grammar but in a template spans several nodes. +type state uint8 + +const ( + // statePCDATA is parsed character data. An HTML parser is in + // this state when its parse position is outside an HTML tag, + // directive, comment, and special element body. + statePCDATA state = iota + // stateTag occurs before an HTML attribute or the end of a tag. + stateTag + // stateURI occurs inside an HTML attribute whose content is a URI. + stateURI + // stateError is an infectious error state outside any valid + // HTML/CSS/JS construct. + stateError +) + +var stateNames = [...]string{ + statePCDATA: "statePCDATA", + stateTag: "stateTag", + stateURI: "stateURI", + stateError: "stateError", +} + +func (s state) String() string { + if uint(s) < uint(len(stateNames)) { + return stateNames[s] + } + return fmt.Sprintf("illegal state %d", uint(s)) +} + +// delim is the delimiter that will end the current HTML attribute. +type delim uint8 + +const ( + // delimNone occurs outside any attribute. + delimNone delim = iota + // delimDoubleQuote occurs when a double quote (") closes the attribute. + delimDoubleQuote + // delimSingleQuote occurs when a single quote (') closes the attribute. + delimSingleQuote + // delimSpaceOrTagEnd occurs when a space or right angle bracket (>) + // closes the attribute. + delimSpaceOrTagEnd +) + +var delimNames = [...]string{ + delimNone: "delimNone", + delimDoubleQuote: "delimDoubleQuote", + delimSingleQuote: "delimSingleQuote", + delimSpaceOrTagEnd: "delimSpaceOrTagEnd", +} + +func (d delim) String() string { + if uint(d) < uint(len(delimNames)) { + return delimNames[d] + } + return fmt.Sprintf("illegal delim %d", uint(d)) +} diff --git a/src/pkg/exp/template/html/escape.go b/src/pkg/exp/template/html/escape.go new file mode 100644 index 000000000..e0e87b98d --- /dev/null +++ b/src/pkg/exp/template/html/escape.go @@ -0,0 +1,105 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Package html is a specialization of exp/template that automates the +// construction of safe HTML output. +// At the moment, the escaping is naive. All dynamic content is assumed to be +// plain text interpolated in an HTML PCDATA context. +package html + +import ( + "template" + "template/parse" +) + +// Escape rewrites each action in the template to guarantee the output is +// HTML-escaped. +func Escape(t *template.Template) { + // If the parser shares trees based on common-subexpression + // joining then we will need to avoid multiply escaping the same action. + escapeListNode(t.Tree.Root) +} + +// escapeNode dispatches to escape helpers by type. +func escapeNode(node parse.Node) { + switch n := node.(type) { + case *parse.ListNode: + escapeListNode(n) + case *parse.TextNode: + // Nothing to do. + case *parse.ActionNode: + escapeActionNode(n) + case *parse.IfNode: + escapeIfNode(n) + case *parse.RangeNode: + escapeRangeNode(n) + case *parse.TemplateNode: + // Nothing to do. + case *parse.WithNode: + escapeWithNode(n) + default: + panic("handling for " + node.String() + " not implemented") + // TODO: Handle other inner node types. + } +} + +// escapeListNode recursively escapes its input's children. +func escapeListNode(node *parse.ListNode) { + if node == nil { + return + } + children := node.Nodes + for _, child := range children { + escapeNode(child) + } +} + +// escapeActionNode adds a pipeline call to the end that escapes the result +// of the expression before it is interpolated into the template output. +func escapeActionNode(node *parse.ActionNode) { + pipe := node.Pipe + + cmds := pipe.Cmds + nCmds := len(cmds) + + // If it already has an escaping command, do not interfere. + if nCmds != 0 { + if lastCmd := cmds[nCmds-1]; len(lastCmd.Args) != 0 { + // TODO: Recognize url and js as escaping functions once + // we have enough context to know whether additional + // escaping is necessary. + if arg, ok := lastCmd.Args[0].(*parse.IdentifierNode); ok && arg.Ident == "html" { + return + } + } + } + + htmlEscapeCommand := parse.CommandNode{ + NodeType: parse.NodeCommand, + Args: []parse.Node{parse.NewIdentifier("html")}, + } + + node.Pipe.Cmds = append(node.Pipe.Cmds, &htmlEscapeCommand) +} + +// escapeIfNode recursively escapes the if and then clauses but leaves the +// condition unchanged. +func escapeIfNode(node *parse.IfNode) { + escapeListNode(node.List) + escapeListNode(node.ElseList) +} + +// escapeRangeNode recursively escapes the loop body and else clause but +// leaves the series unchanged. +func escapeRangeNode(node *parse.RangeNode) { + escapeListNode(node.List) + escapeListNode(node.ElseList) +} + +// escapeWithNode recursively escapes the scope body and else clause but +// leaves the pipeline unchanged. +func escapeWithNode(node *parse.WithNode) { + escapeListNode(node.List) + escapeListNode(node.ElseList) +} diff --git a/src/pkg/exp/template/html/escape_test.go b/src/pkg/exp/template/html/escape_test.go new file mode 100644 index 000000000..345a752a8 --- /dev/null +++ b/src/pkg/exp/template/html/escape_test.go @@ -0,0 +1,75 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package html + +import ( + "bytes" + "template" + "testing" +) + +type data struct { + F, T bool + C, G, H string + A, E []string +} + +var testData = data{ + F: false, + T: true, + C: "", + G: "", + H: "", + A: []string{"", ""}, + E: []string{}, +} + +type testCase struct { + name string + input string + output string +} + +var testCases = []testCase{ + {"if", "{{if .T}}Hello{{end}}, {{.C}}!", "Hello, <Cincinatti>!"}, + {"else", "{{if .F}}{{.H}}{{else}}{{.G}}{{end}}!", "<Goodbye>!"}, + {"overescaping", "Hello, {{.C | html}}!", "Hello, <Cincinatti>!"}, + {"assignment", "{{if $x := .H}}{{$x}}{{end}}", "<Hello>"}, + {"withBody", "{{with .H}}{{.}}{{end}}", "<Hello>"}, + {"withElse", "{{with .E}}{{.}}{{else}}{{.H}}{{end}}", "<Hello>"}, + {"rangeBody", "{{range .A}}{{.}}{{end}}", "<a><b>"}, + {"rangeElse", "{{range .E}}{{.}}{{else}}{{.H}}{{end}}", "<Hello>"}, + {"nonStringValue", "{{.T}}", "true"}, + {"constant", ``, ``}, +} + +func TestAutoesc(t *testing.T) { + for _, testCase := range testCases { + name := testCase.name + tmpl := template.New(name) + tmpl, err := tmpl.Parse(testCase.input) + if err != nil { + t.Errorf("%s: failed to parse template: %s", name, err) + continue + } + + Escape(tmpl) + + buffer := new(bytes.Buffer) + + err = tmpl.Execute(buffer, testData) + if err != nil { + t.Errorf("%s: template execution failed: %s", name, err) + continue + } + + output := testCase.output + actual := buffer.String() + if output != actual { + t.Errorf("%s: escaped output: %q != %q", + name, output, actual) + } + } +} diff --git a/src/pkg/exp/wingui/Makefile b/src/pkg/exp/wingui/Makefile new file mode 100644 index 000000000..00f7d234d --- /dev/null +++ b/src/pkg/exp/wingui/Makefile @@ -0,0 +1,29 @@ +# Copyright 2011 The Go Authors. All rights reserved. +# Use of this source code is governed by a BSD-style +# license that can be found in the LICENSE file. + +GOOS=windows + +include ../../../Make.inc + +LD:=$(LD) -Hwindowsgui + +TARG=wingui + +GOFILES=\ + gui.go\ + winapi.go\ + zwinapi.go\ + +include ../../../Make.cmd + +zwinapi.go: winapi.go + $(GOROOT)/src/pkg/syscall/mksyscall_windows.pl $< \ + | sed 's/^package.*syscall$$/package main/' \ + | sed '/^import/a \ + import "syscall"' \ + | sed 's/Syscall/syscall.Syscall/' \ + | sed 's/NewLazyDLL/syscall.NewLazyDLL/' \ + | sed 's/EINVAL/syscall.EINVAL/' \ + | gofmt \ + > $@ diff --git a/src/pkg/exp/wingui/gui.go b/src/pkg/exp/wingui/gui.go new file mode 100644 index 000000000..cf392934c --- /dev/null +++ b/src/pkg/exp/wingui/gui.go @@ -0,0 +1,153 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package main + +import ( + "fmt" + "syscall" + "os" + "unsafe" +) + +// some help functions + +func abortf(format string, a ...interface{}) { + fmt.Fprintf(os.Stdout, format, a...) + os.Exit(1) +} + +func abortErrNo(funcname string, err int) { + abortf("%s failed: %d %s\n", funcname, err, syscall.Errstr(err)) +} + +// global vars + +var ( + mh uint32 + bh uint32 +) + +// WinProc called by windows to notify us of all windows events we might be interested in. +func WndProc(hwnd, msg uint32, wparam, lparam int32) uintptr { + var rc int32 + switch msg { + case WM_CREATE: + var e int + // CreateWindowEx + bh, e = CreateWindowEx( + 0, + syscall.StringToUTF16Ptr("button"), + syscall.StringToUTF16Ptr("Quit"), + WS_CHILD|WS_VISIBLE|BS_DEFPUSHBUTTON, + 75, 70, 140, 25, + hwnd, 1, mh, 0) + if e != 0 { + abortErrNo("CreateWindowEx", e) + } + fmt.Printf("button handle is %x\n", bh) + rc = DefWindowProc(hwnd, msg, wparam, lparam) + case WM_COMMAND: + switch uint32(lparam) { + case bh: + e := PostMessage(hwnd, WM_CLOSE, 0, 0) + if e != 0 { + abortErrNo("PostMessage", e) + } + default: + rc = DefWindowProc(hwnd, msg, wparam, lparam) + } + case WM_CLOSE: + DestroyWindow(hwnd) + case WM_DESTROY: + PostQuitMessage(0) + default: + rc = DefWindowProc(hwnd, msg, wparam, lparam) + } + //fmt.Printf("WndProc(0x%08x, %d, 0x%08x, 0x%08x) (%d)\n", hwnd, msg, wparam, lparam, rc) + return uintptr(rc) +} + +func rungui() int { + var e int + + // GetModuleHandle + mh, e = GetModuleHandle(nil) + if e != 0 { + abortErrNo("GetModuleHandle", e) + } + + // Get icon we're going to use. + myicon, e := LoadIcon(0, IDI_APPLICATION) + if e != 0 { + abortErrNo("LoadIcon", e) + } + + // Get cursor we're going to use. + mycursor, e := LoadCursor(0, IDC_ARROW) + if e != 0 { + abortErrNo("LoadCursor", e) + } + + // Create callback + wproc := syscall.NewCallback(WndProc) + + // RegisterClassEx + wcname := syscall.StringToUTF16Ptr("myWindowClass") + var wc Wndclassex + wc.Size = uint32(unsafe.Sizeof(wc)) + wc.WndProc = wproc + wc.Instance = mh + wc.Icon = myicon + wc.Cursor = mycursor + wc.Background = COLOR_BTNFACE + 1 + wc.MenuName = nil + wc.ClassName = wcname + wc.IconSm = myicon + if _, e := RegisterClassEx(&wc); e != 0 { + abortErrNo("RegisterClassEx", e) + } + + // CreateWindowEx + wh, e := CreateWindowEx( + WS_EX_CLIENTEDGE, + wcname, + syscall.StringToUTF16Ptr("My window"), + WS_OVERLAPPEDWINDOW, + CW_USEDEFAULT, CW_USEDEFAULT, 300, 200, + 0, 0, mh, 0) + if e != 0 { + abortErrNo("CreateWindowEx", e) + } + fmt.Printf("main window handle is %x\n", wh) + + // ShowWindow + ShowWindow(wh, SW_SHOWDEFAULT) + + // UpdateWindow + if e := UpdateWindow(wh); e != 0 { + abortErrNo("UpdateWindow", e) + } + + // Process all windows messages until WM_QUIT. + var m Msg + for { + r, e := GetMessage(&m, 0, 0, 0) + if e != 0 { + abortErrNo("GetMessage", e) + } + if r == 0 { + // WM_QUIT received -> get out + break + } + TranslateMessage(&m) + DispatchMessage(&m) + } + return int(m.Wparam) +} + +func main() { + rc := rungui() + os.Exit(rc) +} diff --git a/src/pkg/exp/wingui/winapi.go b/src/pkg/exp/wingui/winapi.go new file mode 100644 index 000000000..31b57a2cc --- /dev/null +++ b/src/pkg/exp/wingui/winapi.go @@ -0,0 +1,131 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package main + +import ( + "unsafe" +) + +type Wndclassex struct { + Size uint32 + Style uint32 + WndProc uintptr + ClsExtra int32 + WndExtra int32 + Instance uint32 + Icon uint32 + Cursor uint32 + Background uint32 + MenuName *uint16 + ClassName *uint16 + IconSm uint32 +} + +type Point struct { + X int32 + Y int32 +} + +type Msg struct { + Hwnd uint32 + Message uint32 + Wparam int32 + Lparam int32 + Time uint32 + Pt Point +} + +const ( + // Window styles + WS_OVERLAPPED = 0 + WS_POPUP = 0x80000000 + WS_CHILD = 0x40000000 + WS_MINIMIZE = 0x20000000 + WS_VISIBLE = 0x10000000 + WS_DISABLED = 0x8000000 + WS_CLIPSIBLINGS = 0x4000000 + WS_CLIPCHILDREN = 0x2000000 + WS_MAXIMIZE = 0x1000000 + WS_CAPTION = WS_BORDER | WS_DLGFRAME + WS_BORDER = 0x800000 + WS_DLGFRAME = 0x400000 + WS_VSCROLL = 0x200000 + WS_HSCROLL = 0x100000 + WS_SYSMENU = 0x80000 + WS_THICKFRAME = 0x40000 + WS_GROUP = 0x20000 + WS_TABSTOP = 0x10000 + WS_MINIMIZEBOX = 0x20000 + WS_MAXIMIZEBOX = 0x10000 + WS_TILED = WS_OVERLAPPED + WS_ICONIC = WS_MINIMIZE + WS_SIZEBOX = WS_THICKFRAME + // Common Window Styles + WS_OVERLAPPEDWINDOW = WS_OVERLAPPED | WS_CAPTION | WS_SYSMENU | WS_THICKFRAME | WS_MINIMIZEBOX | WS_MAXIMIZEBOX + WS_TILEDWINDOW = WS_OVERLAPPEDWINDOW + WS_POPUPWINDOW = WS_POPUP | WS_BORDER | WS_SYSMENU + WS_CHILDWINDOW = WS_CHILD + + WS_EX_CLIENTEDGE = 0x200 + + // Some windows messages + WM_CREATE = 1 + WM_DESTROY = 2 + WM_CLOSE = 16 + WM_COMMAND = 273 + + // Some button control styles + BS_DEFPUSHBUTTON = 1 + + // Some color constants + COLOR_WINDOW = 5 + COLOR_BTNFACE = 15 + + // Default window position + CW_USEDEFAULT = 0x80000000 - 0x100000000 + + // Show window default style + SW_SHOWDEFAULT = 10 +) + +var ( + // Some globally known cursors + IDC_ARROW = MakeIntResource(32512) + IDC_IBEAM = MakeIntResource(32513) + IDC_WAIT = MakeIntResource(32514) + IDC_CROSS = MakeIntResource(32515) + + // Some globally known icons + IDI_APPLICATION = MakeIntResource(32512) + IDI_HAND = MakeIntResource(32513) + IDI_QUESTION = MakeIntResource(32514) + IDI_EXCLAMATION = MakeIntResource(32515) + IDI_ASTERISK = MakeIntResource(32516) + IDI_WINLOGO = MakeIntResource(32517) + IDI_WARNING = IDI_EXCLAMATION + IDI_ERROR = IDI_HAND + IDI_INFORMATION = IDI_ASTERISK +) + +//sys GetModuleHandle(modname *uint16) (handle uint32, errno int) = GetModuleHandleW +//sys RegisterClassEx(wndclass *Wndclassex) (atom uint16, errno int) = user32.RegisterClassExW +//sys CreateWindowEx(exstyle uint32, classname *uint16, windowname *uint16, style uint32, x int32, y int32, width int32, height int32, wndparent uint32, menu uint32, instance uint32, param uintptr) (hwnd uint32, errno int) = user32.CreateWindowExW +//sys DefWindowProc(hwnd uint32, msg uint32, wparam int32, lparam int32) (lresult int32) = user32.DefWindowProcW +//sys DestroyWindow(hwnd uint32) (errno int) = user32.DestroyWindow +//sys PostQuitMessage(exitcode int32) = user32.PostQuitMessage +//sys ShowWindow(hwnd uint32, cmdshow int32) (wasvisible bool) = user32.ShowWindow +//sys UpdateWindow(hwnd uint32) (errno int) = user32.UpdateWindow +//sys GetMessage(msg *Msg, hwnd uint32, MsgFilterMin uint32, MsgFilterMax uint32) (ret int32, errno int) [failretval==-1] = user32.GetMessageW +//sys TranslateMessage(msg *Msg) (done bool) = user32.TranslateMessage +//sys DispatchMessage(msg *Msg) (ret int32) = user32.DispatchMessageW +//sys LoadIcon(instance uint32, iconname *uint16) (icon uint32, errno int) = user32.LoadIconW +//sys LoadCursor(instance uint32, cursorname *uint16) (cursor uint32, errno int) = user32.LoadCursorW +//sys SetCursor(cursor uint32) (precursor uint32, errno int) = user32.SetCursor +//sys SendMessage(hwnd uint32, msg uint32, wparam int32, lparam int32) (lresult int32) = user32.SendMessageW +//sys PostMessage(hwnd uint32, msg uint32, wparam int32, lparam int32) (errno int) = user32.PostMessageW + +func MakeIntResource(id uint16) *uint16 { + return (*uint16)(unsafe.Pointer(uintptr(id))) +} diff --git a/src/pkg/exp/wingui/zwinapi.go b/src/pkg/exp/wingui/zwinapi.go new file mode 100644 index 000000000..4c009dd69 --- /dev/null +++ b/src/pkg/exp/wingui/zwinapi.go @@ -0,0 +1,211 @@ +// mksyscall_windows.pl winapi.go +// MACHINE GENERATED BY THE COMMAND ABOVE; DO NOT EDIT + +package main + +import "unsafe" +import "syscall" + +var ( + modkernel32 = syscall.NewLazyDLL("kernel32.dll") + moduser32 = syscall.NewLazyDLL("user32.dll") + + procGetModuleHandleW = modkernel32.NewProc("GetModuleHandleW") + procRegisterClassExW = moduser32.NewProc("RegisterClassExW") + procCreateWindowExW = moduser32.NewProc("CreateWindowExW") + procDefWindowProcW = moduser32.NewProc("DefWindowProcW") + procDestroyWindow = moduser32.NewProc("DestroyWindow") + procPostQuitMessage = moduser32.NewProc("PostQuitMessage") + procShowWindow = moduser32.NewProc("ShowWindow") + procUpdateWindow = moduser32.NewProc("UpdateWindow") + procGetMessageW = moduser32.NewProc("GetMessageW") + procTranslateMessage = moduser32.NewProc("TranslateMessage") + procDispatchMessageW = moduser32.NewProc("DispatchMessageW") + procLoadIconW = moduser32.NewProc("LoadIconW") + procLoadCursorW = moduser32.NewProc("LoadCursorW") + procSetCursor = moduser32.NewProc("SetCursor") + procSendMessageW = moduser32.NewProc("SendMessageW") + procPostMessageW = moduser32.NewProc("PostMessageW") +) + +func GetModuleHandle(modname *uint16) (handle uint32, errno int) { + r0, _, e1 := syscall.Syscall(procGetModuleHandleW.Addr(), 1, uintptr(unsafe.Pointer(modname)), 0, 0) + handle = uint32(r0) + if handle == 0 { + if e1 != 0 { + errno = int(e1) + } else { + errno = syscall.EINVAL + } + } else { + errno = 0 + } + return +} + +func RegisterClassEx(wndclass *Wndclassex) (atom uint16, errno int) { + r0, _, e1 := syscall.Syscall(procRegisterClassExW.Addr(), 1, uintptr(unsafe.Pointer(wndclass)), 0, 0) + atom = uint16(r0) + if atom == 0 { + if e1 != 0 { + errno = int(e1) + } else { + errno = syscall.EINVAL + } + } else { + errno = 0 + } + return +} + +func CreateWindowEx(exstyle uint32, classname *uint16, windowname *uint16, style uint32, x int32, y int32, width int32, height int32, wndparent uint32, menu uint32, instance uint32, param uintptr) (hwnd uint32, errno int) { + r0, _, e1 := syscall.Syscall12(procCreateWindowExW.Addr(), 12, uintptr(exstyle), uintptr(unsafe.Pointer(classname)), uintptr(unsafe.Pointer(windowname)), uintptr(style), uintptr(x), uintptr(y), uintptr(width), uintptr(height), uintptr(wndparent), uintptr(menu), uintptr(instance), uintptr(param)) + hwnd = uint32(r0) + if hwnd == 0 { + if e1 != 0 { + errno = int(e1) + } else { + errno = syscall.EINVAL + } + } else { + errno = 0 + } + return +} + +func DefWindowProc(hwnd uint32, msg uint32, wparam int32, lparam int32) (lresult int32) { + r0, _, _ := syscall.Syscall6(procDefWindowProcW.Addr(), 4, uintptr(hwnd), uintptr(msg), uintptr(wparam), uintptr(lparam), 0, 0) + lresult = int32(r0) + return +} + +func DestroyWindow(hwnd uint32) (errno int) { + r1, _, e1 := syscall.Syscall(procDestroyWindow.Addr(), 1, uintptr(hwnd), 0, 0) + if int(r1) == 0 { + if e1 != 0 { + errno = int(e1) + } else { + errno = syscall.EINVAL + } + } else { + errno = 0 + } + return +} + +func PostQuitMessage(exitcode int32) { + syscall.Syscall(procPostQuitMessage.Addr(), 1, uintptr(exitcode), 0, 0) + return +} + +func ShowWindow(hwnd uint32, cmdshow int32) (wasvisible bool) { + r0, _, _ := syscall.Syscall(procShowWindow.Addr(), 2, uintptr(hwnd), uintptr(cmdshow), 0) + wasvisible = bool(r0 != 0) + return +} + +func UpdateWindow(hwnd uint32) (errno int) { + r1, _, e1 := syscall.Syscall(procUpdateWindow.Addr(), 1, uintptr(hwnd), 0, 0) + if int(r1) == 0 { + if e1 != 0 { + errno = int(e1) + } else { + errno = syscall.EINVAL + } + } else { + errno = 0 + } + return +} + +func GetMessage(msg *Msg, hwnd uint32, MsgFilterMin uint32, MsgFilterMax uint32) (ret int32, errno int) { + r0, _, e1 := syscall.Syscall6(procGetMessageW.Addr(), 4, uintptr(unsafe.Pointer(msg)), uintptr(hwnd), uintptr(MsgFilterMin), uintptr(MsgFilterMax), 0, 0) + ret = int32(r0) + if ret == -1 { + if e1 != 0 { + errno = int(e1) + } else { + errno = syscall.EINVAL + } + } else { + errno = 0 + } + return +} + +func TranslateMessage(msg *Msg) (done bool) { + r0, _, _ := syscall.Syscall(procTranslateMessage.Addr(), 1, uintptr(unsafe.Pointer(msg)), 0, 0) + done = bool(r0 != 0) + return +} + +func DispatchMessage(msg *Msg) (ret int32) { + r0, _, _ := syscall.Syscall(procDispatchMessageW.Addr(), 1, uintptr(unsafe.Pointer(msg)), 0, 0) + ret = int32(r0) + return +} + +func LoadIcon(instance uint32, iconname *uint16) (icon uint32, errno int) { + r0, _, e1 := syscall.Syscall(procLoadIconW.Addr(), 2, uintptr(instance), uintptr(unsafe.Pointer(iconname)), 0) + icon = uint32(r0) + if icon == 0 { + if e1 != 0 { + errno = int(e1) + } else { + errno = syscall.EINVAL + } + } else { + errno = 0 + } + return +} + +func LoadCursor(instance uint32, cursorname *uint16) (cursor uint32, errno int) { + r0, _, e1 := syscall.Syscall(procLoadCursorW.Addr(), 2, uintptr(instance), uintptr(unsafe.Pointer(cursorname)), 0) + cursor = uint32(r0) + if cursor == 0 { + if e1 != 0 { + errno = int(e1) + } else { + errno = syscall.EINVAL + } + } else { + errno = 0 + } + return +} + +func SetCursor(cursor uint32) (precursor uint32, errno int) { + r0, _, e1 := syscall.Syscall(procSetCursor.Addr(), 1, uintptr(cursor), 0, 0) + precursor = uint32(r0) + if precursor == 0 { + if e1 != 0 { + errno = int(e1) + } else { + errno = syscall.EINVAL + } + } else { + errno = 0 + } + return +} + +func SendMessage(hwnd uint32, msg uint32, wparam int32, lparam int32) (lresult int32) { + r0, _, _ := syscall.Syscall6(procSendMessageW.Addr(), 4, uintptr(hwnd), uintptr(msg), uintptr(wparam), uintptr(lparam), 0, 0) + lresult = int32(r0) + return +} + +func PostMessage(hwnd uint32, msg uint32, wparam int32, lparam int32) (errno int) { + r1, _, e1 := syscall.Syscall6(procPostMessageW.Addr(), 4, uintptr(hwnd), uintptr(msg), uintptr(wparam), uintptr(lparam), 0, 0) + if int(r1) == 0 { + if e1 != 0 { + errno = int(e1) + } else { + errno = syscall.EINVAL + } + } else { + errno = 0 + } + return +} diff --git a/src/pkg/expvar/Makefile b/src/pkg/expvar/Makefile new file mode 100644 index 000000000..5619630d1 --- /dev/null +++ b/src/pkg/expvar/Makefile @@ -0,0 +1,11 @@ +# Copyright 2009 The Go Authors. All rights reserved. +# Use of this source code is governed by a BSD-style +# license that can be found in the LICENSE file. + +include ../../Make.inc + +TARG=expvar +GOFILES=\ + expvar.go\ + +include ../../Make.pkg diff --git a/src/pkg/expvar/expvar.go b/src/pkg/expvar/expvar.go new file mode 100644 index 000000000..7b733faf6 --- /dev/null +++ b/src/pkg/expvar/expvar.go @@ -0,0 +1,287 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Package expvar provides a standardized interface to public variables, such +// as operation counters in servers. It exposes these variables via HTTP at +// /debug/vars in JSON format. +// +// Operations to set or modify these public variables are atomic. +// +// In addition to adding the HTTP handler, this package registers the +// following variables: +// +// cmdline os.Args +// memstats runtime.Memstats +// +// The package is sometimes only imported for the side effect of +// registering its HTTP handler and the above variables. To use it +// this way, simply link this package into your program: +// import _ "expvar" +// +package expvar + +import ( + "bytes" + "fmt" + "http" + "json" + "log" + "os" + "runtime" + "strconv" + "sync" +) + +// Var is an abstract type for all exported variables. +type Var interface { + String() string +} + +// Int is a 64-bit integer variable that satisfies the Var interface. +type Int struct { + i int64 + mu sync.Mutex +} + +func (v *Int) String() string { return strconv.Itoa64(v.i) } + +func (v *Int) Add(delta int64) { + v.mu.Lock() + defer v.mu.Unlock() + v.i += delta +} + +func (v *Int) Set(value int64) { + v.mu.Lock() + defer v.mu.Unlock() + v.i = value +} + +// Float is a 64-bit float variable that satisfies the Var interface. +type Float struct { + f float64 + mu sync.Mutex +} + +func (v *Float) String() string { return strconv.Ftoa64(v.f, 'g', -1) } + +// Add adds delta to v. +func (v *Float) Add(delta float64) { + v.mu.Lock() + defer v.mu.Unlock() + v.f += delta +} + +// Set sets v to value. +func (v *Float) Set(value float64) { + v.mu.Lock() + defer v.mu.Unlock() + v.f = value +} + +// Map is a string-to-Var map variable that satisfies the Var interface. +type Map struct { + m map[string]Var + mu sync.Mutex +} + +// KeyValue represents a single entry in a Map. +type KeyValue struct { + Key string + Value Var +} + +func (v *Map) String() string { + v.mu.Lock() + defer v.mu.Unlock() + b := new(bytes.Buffer) + fmt.Fprintf(b, "{") + first := true + for key, val := range v.m { + if !first { + fmt.Fprintf(b, ", ") + } + fmt.Fprintf(b, "\"%s\": %v", key, val.String()) + first = false + } + fmt.Fprintf(b, "}") + return b.String() +} + +func (v *Map) Init() *Map { + v.m = make(map[string]Var) + return v +} + +func (v *Map) Get(key string) Var { + v.mu.Lock() + defer v.mu.Unlock() + return v.m[key] +} + +func (v *Map) Set(key string, av Var) { + v.mu.Lock() + defer v.mu.Unlock() + v.m[key] = av +} + +func (v *Map) Add(key string, delta int64) { + v.mu.Lock() + defer v.mu.Unlock() + av, ok := v.m[key] + if !ok { + av = new(Int) + v.m[key] = av + } + + // Add to Int; ignore otherwise. + if iv, ok := av.(*Int); ok { + iv.Add(delta) + } +} + +// AddFloat adds delta to the *Float value stored under the given map key. +func (v *Map) AddFloat(key string, delta float64) { + v.mu.Lock() + defer v.mu.Unlock() + av, ok := v.m[key] + if !ok { + av = new(Float) + v.m[key] = av + } + + // Add to Float; ignore otherwise. + if iv, ok := av.(*Float); ok { + iv.Add(delta) + } +} + +// TODO(rsc): Make sure map access in separate thread is safe. +func (v *Map) iterate(c chan<- KeyValue) { + for k, v := range v.m { + c <- KeyValue{k, v} + } + close(c) +} + +func (v *Map) Iter() <-chan KeyValue { + c := make(chan KeyValue) + go v.iterate(c) + return c +} + +// String is a string variable, and satisfies the Var interface. +type String struct { + s string +} + +func (v *String) String() string { return strconv.Quote(v.s) } + +func (v *String) Set(value string) { v.s = value } + +// Func implements Var by calling the function +// and formatting the returned value using JSON. +type Func func() interface{} + +func (f Func) String() string { + v, _ := json.Marshal(f()) + return string(v) +} + +// All published variables. +var vars map[string]Var = make(map[string]Var) +var mutex sync.Mutex + +// Publish declares an named exported variable. This should be called from a +// package's init function when it creates its Vars. If the name is already +// registered then this will log.Panic. +func Publish(name string, v Var) { + mutex.Lock() + defer mutex.Unlock() + if _, existing := vars[name]; existing { + log.Panicln("Reuse of exported var name:", name) + } + vars[name] = v +} + +// Get retrieves a named exported variable. +func Get(name string) Var { + return vars[name] +} + +// RemoveAll removes all exported variables. +// This is for tests; don't call this on a real server. +func RemoveAll() { + mutex.Lock() + defer mutex.Unlock() + vars = make(map[string]Var) +} + +// Convenience functions for creating new exported variables. + +func NewInt(name string) *Int { + v := new(Int) + Publish(name, v) + return v +} + +func NewFloat(name string) *Float { + v := new(Float) + Publish(name, v) + return v +} + +func NewMap(name string) *Map { + v := new(Map).Init() + Publish(name, v) + return v +} + +func NewString(name string) *String { + v := new(String) + Publish(name, v) + return v +} + +// TODO(rsc): Make sure map access in separate thread is safe. +func iterate(c chan<- KeyValue) { + for k, v := range vars { + c <- KeyValue{k, v} + } + close(c) +} + +func Iter() <-chan KeyValue { + c := make(chan KeyValue) + go iterate(c) + return c +} + +func expvarHandler(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "application/json; charset=utf-8") + fmt.Fprintf(w, "{\n") + first := true + for name, value := range vars { + if !first { + fmt.Fprintf(w, ",\n") + } + first = false + fmt.Fprintf(w, "%q: %s", name, value) + } + fmt.Fprintf(w, "\n}\n") +} + +func cmdline() interface{} { + return os.Args +} + +func memstats() interface{} { + return runtime.MemStats +} + +func init() { + http.Handle("/debug/vars", http.HandlerFunc(expvarHandler)) + Publish("cmdline", Func(cmdline)) + Publish("memstats", Func(memstats)) +} diff --git a/src/pkg/expvar/expvar_test.go b/src/pkg/expvar/expvar_test.go new file mode 100644 index 000000000..8f7a48168 --- /dev/null +++ b/src/pkg/expvar/expvar_test.go @@ -0,0 +1,128 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package expvar + +import ( + "json" + "testing" +) + +func TestInt(t *testing.T) { + reqs := NewInt("requests") + if reqs.i != 0 { + t.Errorf("reqs.i = %v, want 0", reqs.i) + } + if reqs != Get("requests").(*Int) { + t.Errorf("Get() failed.") + } + + reqs.Add(1) + reqs.Add(3) + if reqs.i != 4 { + t.Errorf("reqs.i = %v, want 4", reqs.i) + } + + if s := reqs.String(); s != "4" { + t.Errorf("reqs.String() = %q, want \"4\"", s) + } + + reqs.Set(-2) + if reqs.i != -2 { + t.Errorf("reqs.i = %v, want -2", reqs.i) + } +} + +func TestFloat(t *testing.T) { + reqs := NewFloat("requests-float") + if reqs.f != 0.0 { + t.Errorf("reqs.f = %v, want 0", reqs.f) + } + if reqs != Get("requests-float").(*Float) { + t.Errorf("Get() failed.") + } + + reqs.Add(1.5) + reqs.Add(1.25) + if reqs.f != 2.75 { + t.Errorf("reqs.f = %v, want 2.75", reqs.f) + } + + if s := reqs.String(); s != "2.75" { + t.Errorf("reqs.String() = %q, want \"4.64\"", s) + } + + reqs.Add(-2) + if reqs.f != 0.75 { + t.Errorf("reqs.f = %v, want 0.75", reqs.f) + } +} + +func TestString(t *testing.T) { + name := NewString("my-name") + if name.s != "" { + t.Errorf("name.s = %q, want \"\"", name.s) + } + + name.Set("Mike") + if name.s != "Mike" { + t.Errorf("name.s = %q, want \"Mike\"", name.s) + } + + if s := name.String(); s != "\"Mike\"" { + t.Errorf("reqs.String() = %q, want \"\"Mike\"\"", s) + } +} + +func TestMapCounter(t *testing.T) { + colors := NewMap("bike-shed-colors") + + colors.Add("red", 1) + colors.Add("red", 2) + colors.Add("blue", 4) + colors.AddFloat("green", 4.125) + if x := colors.m["red"].(*Int).i; x != 3 { + t.Errorf("colors.m[\"red\"] = %v, want 3", x) + } + if x := colors.m["blue"].(*Int).i; x != 4 { + t.Errorf("colors.m[\"blue\"] = %v, want 4", x) + } + if x := colors.m["green"].(*Float).f; x != 4.125 { + t.Errorf("colors.m[\"green\"] = %v, want 3.14", x) + } + + // colors.String() should be '{"red":3, "blue":4}', + // though the order of red and blue could vary. + s := colors.String() + var j interface{} + err := json.Unmarshal([]byte(s), &j) + if err != nil { + t.Errorf("colors.String() isn't valid JSON: %v", err) + } + m, ok := j.(map[string]interface{}) + if !ok { + t.Error("colors.String() didn't produce a map.") + } + red := m["red"] + x, ok := red.(float64) + if !ok { + t.Error("red.Kind() is not a number.") + } + if x != 3 { + t.Errorf("red = %v, want 3", x) + } +} + +func TestFunc(t *testing.T) { + var x interface{} = []string{"a", "b"} + f := Func(func() interface{} { return x }) + if s, exp := f.String(), `["a","b"]`; s != exp { + t.Errorf(`f.String() = %q, want %q`, s, exp) + } + + x = 17 + if s, exp := f.String(), `17`; s != exp { + t.Errorf(`f.String() = %q, want %q`, s, exp) + } +} diff --git a/src/pkg/flag/Makefile b/src/pkg/flag/Makefile new file mode 100644 index 000000000..3408ca474 --- /dev/null +++ b/src/pkg/flag/Makefile @@ -0,0 +1,11 @@ +# Copyright 2009 The Go Authors. All rights reserved. +# Use of this source code is governed by a BSD-style +# license that can be found in the LICENSE file. + +include ../../Make.inc + +TARG=flag +GOFILES=\ + flag.go\ + +include ../../Make.pkg diff --git a/src/pkg/flag/export_test.go b/src/pkg/flag/export_test.go new file mode 100644 index 000000000..7b190807a --- /dev/null +++ b/src/pkg/flag/export_test.go @@ -0,0 +1,22 @@ +// Copyright 2010 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package flag + +import "os" + +// Additional routines compiled into the package only during testing. + +// ResetForTesting clears all flag state and sets the usage function as directed. +// After calling ResetForTesting, parse errors in flag handling will not +// exit the program. +func ResetForTesting(usage func()) { + commandLine = NewFlagSet(os.Args[0], ContinueOnError) + Usage = usage +} + +// CommandLine returns the default FlagSet. +func CommandLine() *FlagSet { + return commandLine +} diff --git a/src/pkg/flag/flag.go b/src/pkg/flag/flag.go new file mode 100644 index 000000000..01bbc3770 --- /dev/null +++ b/src/pkg/flag/flag.go @@ -0,0 +1,705 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +/* + Package flag implements command-line flag parsing. + + Usage: + + Define flags using flag.String(), Bool(), Int(), etc. Example: + import "flag" + var ip *int = flag.Int("flagname", 1234, "help message for flagname") + If you like, you can bind the flag to a variable using the Var() functions. + var flagvar int + func init() { + flag.IntVar(&flagvar, "flagname", 1234, "help message for flagname") + } + Or you can create custom flags that satisfy the Value interface (with + pointer receivers) and couple them to flag parsing by + flag.Var(&flagVal, "name", "help message for flagname") + For such flags, the default value is just the initial value of the variable. + + After all flags are defined, call + flag.Parse() + to parse the command line into the defined flags. + + Flags may then be used directly. If you're using the flags themselves, + they are all pointers; if you bind to variables, they're values. + fmt.Println("ip has value ", *ip); + fmt.Println("flagvar has value ", flagvar); + + After parsing, the arguments after the flag are available as the + slice flag.Args() or individually as flag.Arg(i). + The arguments are indexed from 0 up to flag.NArg(). + + Command line flag syntax: + -flag + -flag=x + -flag x // non-boolean flags only + One or two minus signs may be used; they are equivalent. + The last form is not permitted for boolean flags because the + meaning of the command + cmd -x * + will change if there is a file called 0, false, etc. You must + use the -flag=false form to turn off a boolean flag. + + Flag parsing stops just before the first non-flag argument + ("-" is a non-flag argument) or after the terminator "--". + + Integer flags accept 1234, 0664, 0x1234 and may be negative. + Boolean flags may be 1, 0, t, f, true, false, TRUE, FALSE, True, False. + + The default set of command-line flags is controlled by + top-level functions. The FlagSet type allows one to define + independent sets of flags, such as to implement subcommands + in a command-line interface. The methods of FlagSet are + analogous to the top-level functions for the command-line + flag set. +*/ +package flag + +import ( + "fmt" + "os" + "sort" + "strconv" +) + +// ErrHelp is the error returned if the flag -help is invoked but no such flag is defined. +var ErrHelp = os.NewError("flag: help requested") + +// -- Bool Value +type boolValue bool + +func newBoolValue(val bool, p *bool) *boolValue { + *p = val + return (*boolValue)(p) +} + +func (b *boolValue) Set(s string) bool { + v, err := strconv.Atob(s) + *b = boolValue(v) + return err == nil +} + +func (b *boolValue) String() string { return fmt.Sprintf("%v", *b) } + +// -- Int Value +type intValue int + +func newIntValue(val int, p *int) *intValue { + *p = val + return (*intValue)(p) +} + +func (i *intValue) Set(s string) bool { + v, err := strconv.Btoi64(s, 0) + *i = intValue(v) + return err == nil +} + +func (i *intValue) String() string { return fmt.Sprintf("%v", *i) } + +// -- Int64 Value +type int64Value int64 + +func newInt64Value(val int64, p *int64) *int64Value { + *p = val + return (*int64Value)(p) +} + +func (i *int64Value) Set(s string) bool { + v, err := strconv.Btoi64(s, 0) + *i = int64Value(v) + return err == nil +} + +func (i *int64Value) String() string { return fmt.Sprintf("%v", *i) } + +// -- Uint Value +type uintValue uint + +func newUintValue(val uint, p *uint) *uintValue { + *p = val + return (*uintValue)(p) +} + +func (i *uintValue) Set(s string) bool { + v, err := strconv.Btoui64(s, 0) + *i = uintValue(v) + return err == nil +} + +func (i *uintValue) String() string { return fmt.Sprintf("%v", *i) } + +// -- uint64 Value +type uint64Value uint64 + +func newUint64Value(val uint64, p *uint64) *uint64Value { + *p = val + return (*uint64Value)(p) +} + +func (i *uint64Value) Set(s string) bool { + v, err := strconv.Btoui64(s, 0) + *i = uint64Value(v) + return err == nil +} + +func (i *uint64Value) String() string { return fmt.Sprintf("%v", *i) } + +// -- string Value +type stringValue string + +func newStringValue(val string, p *string) *stringValue { + *p = val + return (*stringValue)(p) +} + +func (s *stringValue) Set(val string) bool { + *s = stringValue(val) + return true +} + +func (s *stringValue) String() string { return fmt.Sprintf("%s", *s) } + +// -- Float64 Value +type float64Value float64 + +func newFloat64Value(val float64, p *float64) *float64Value { + *p = val + return (*float64Value)(p) +} + +func (f *float64Value) Set(s string) bool { + v, err := strconv.Atof64(s) + *f = float64Value(v) + return err == nil +} + +func (f *float64Value) String() string { return fmt.Sprintf("%v", *f) } + +// Value is the interface to the dynamic value stored in a flag. +// (The default value is represented as a string.) +type Value interface { + String() string + Set(string) bool +} + +// ErrorHandling defines how to handle flag parsing errors. +type ErrorHandling int + +const ( + ContinueOnError ErrorHandling = iota + ExitOnError + PanicOnError +) + +// A FlagSet represents a set of defined flags. +type FlagSet struct { + // Usage is the function called when an error occurs while parsing flags. + // The field is a function (not a method) that may be changed to point to + // a custom error handler. + Usage func() + + name string + actual map[string]*Flag + formal map[string]*Flag + args []string // arguments after flags + exitOnError bool // does the program exit if there's an error? + errorHandling ErrorHandling +} + +// A Flag represents the state of a flag. +type Flag struct { + Name string // name as it appears on command line + Usage string // help message + Value Value // value as set + DefValue string // default value (as text); for usage message +} + +// sortFlags returns the flags as a slice in lexicographical sorted order. +func sortFlags(flags map[string]*Flag) []*Flag { + list := make(sort.StringSlice, len(flags)) + i := 0 + for _, f := range flags { + list[i] = f.Name + i++ + } + list.Sort() + result := make([]*Flag, len(list)) + for i, name := range list { + result[i] = flags[name] + } + return result +} + +// VisitAll visits the flags in lexicographical order, calling fn for each. +// It visits all flags, even those not set. +func (f *FlagSet) VisitAll(fn func(*Flag)) { + for _, flag := range sortFlags(f.formal) { + fn(flag) + } +} + +// VisitAll visits the command-line flags in lexicographical order, calling +// fn for each. It visits all flags, even those not set. +func VisitAll(fn func(*Flag)) { + commandLine.VisitAll(fn) +} + +// Visit visits the flags in lexicographical order, calling fn for each. +// It visits only those flags that have been set. +func (f *FlagSet) Visit(fn func(*Flag)) { + for _, flag := range sortFlags(f.actual) { + fn(flag) + } +} + +// Visit visits the command-line flags in lexicographical order, calling fn +// for each. It visits only those flags that have been set. +func Visit(fn func(*Flag)) { + commandLine.Visit(fn) +} + +// Lookup returns the Flag structure of the named flag, returning nil if none exists. +func (f *FlagSet) Lookup(name string) *Flag { + return f.formal[name] +} + +// Lookup returns the Flag structure of the named command-line flag, +// returning nil if none exists. +func Lookup(name string) *Flag { + return commandLine.formal[name] +} + +// Set sets the value of the named flag. It returns true if the set succeeded; false if +// there is no such flag defined. +func (f *FlagSet) Set(name, value string) bool { + flag, ok := f.formal[name] + if !ok { + return false + } + ok = flag.Value.Set(value) + if !ok { + return false + } + f.actual[name] = flag + return true +} + +// Set sets the value of the named command-line flag. It returns true if the +// set succeeded; false if there is no such flag defined. +func Set(name, value string) bool { + return commandLine.Set(name, value) +} + +// PrintDefaults prints to standard error the default values of all defined flags in the set. +func (f *FlagSet) PrintDefaults() { + f.VisitAll(func(f *Flag) { + format := " -%s=%s: %s\n" + if _, ok := f.Value.(*stringValue); ok { + // put quotes on the value + format = " -%s=%q: %s\n" + } + fmt.Fprintf(os.Stderr, format, f.Name, f.DefValue, f.Usage) + }) +} + +// PrintDefaults prints to standard error the default values of all defined command-line flags. +func PrintDefaults() { + commandLine.PrintDefaults() +} + +// defaultUsage is the default function to print a usage message. +func defaultUsage(f *FlagSet) { + fmt.Fprintf(os.Stderr, "Usage of %s:\n", f.name) + f.PrintDefaults() +} + +// Usage prints to standard error a usage message documenting all defined command-line flags. +// The function is a variable that may be changed to point to a custom function. +var Usage = func() { + defaultUsage(commandLine) +} + +// NFlag returns the number of flags that have been set. +func (f *FlagSet) NFlag() int { return len(f.actual) } + +// NFlag returns the number of command-line flags that have been set. +func NFlag() int { return len(commandLine.actual) } + +// Arg returns the i'th argument. Arg(0) is the first remaining argument +// after flags have been processed. +func (f *FlagSet) Arg(i int) string { + if i < 0 || i >= len(f.args) { + return "" + } + return f.args[i] +} + +// Arg returns the i'th command-line argument. Arg(0) is the first remaining argument +// after flags have been processed. +func Arg(i int) string { + return commandLine.Arg(i) +} + +// NArg is the number of arguments remaining after flags have been processed. +func (f *FlagSet) NArg() int { return len(f.args) } + +// NArg is the number of arguments remaining after flags have been processed. +func NArg() int { return len(commandLine.args) } + +// Args returns the non-flag arguments. +func (f *FlagSet) Args() []string { return f.args } + +// Args returns the non-flag command-line arguments. +func Args() []string { return commandLine.args } + +// BoolVar defines a bool flag with specified name, default value, and usage string. +// The argument p points to a bool variable in which to store the value of the flag. +func (f *FlagSet) BoolVar(p *bool, name string, value bool, usage string) { + f.Var(newBoolValue(value, p), name, usage) +} + +// BoolVar defines a bool flag with specified name, default value, and usage string. +// The argument p points to a bool variable in which to store the value of the flag. +func BoolVar(p *bool, name string, value bool, usage string) { + commandLine.Var(newBoolValue(value, p), name, usage) +} + +// Bool defines a bool flag with specified name, default value, and usage string. +// The return value is the address of a bool variable that stores the value of the flag. +func (f *FlagSet) Bool(name string, value bool, usage string) *bool { + p := new(bool) + f.BoolVar(p, name, value, usage) + return p +} + +// Bool defines a bool flag with specified name, default value, and usage string. +// The return value is the address of a bool variable that stores the value of the flag. +func Bool(name string, value bool, usage string) *bool { + return commandLine.Bool(name, value, usage) +} + +// IntVar defines an int flag with specified name, default value, and usage string. +// The argument p points to an int variable in which to store the value of the flag. +func (f *FlagSet) IntVar(p *int, name string, value int, usage string) { + f.Var(newIntValue(value, p), name, usage) +} + +// IntVar defines an int flag with specified name, default value, and usage string. +// The argument p points to an int variable in which to store the value of the flag. +func IntVar(p *int, name string, value int, usage string) { + commandLine.Var(newIntValue(value, p), name, usage) +} + +// Int defines an int flag with specified name, default value, and usage string. +// The return value is the address of an int variable that stores the value of the flag. +func (f *FlagSet) Int(name string, value int, usage string) *int { + p := new(int) + f.IntVar(p, name, value, usage) + return p +} + +// Int defines an int flag with specified name, default value, and usage string. +// The return value is the address of an int variable that stores the value of the flag. +func Int(name string, value int, usage string) *int { + return commandLine.Int(name, value, usage) +} + +// Int64Var defines an int64 flag with specified name, default value, and usage string. +// The argument p points to an int64 variable in which to store the value of the flag. +func (f *FlagSet) Int64Var(p *int64, name string, value int64, usage string) { + f.Var(newInt64Value(value, p), name, usage) +} + +// Int64Var defines an int64 flag with specified name, default value, and usage string. +// The argument p points to an int64 variable in which to store the value of the flag. +func Int64Var(p *int64, name string, value int64, usage string) { + commandLine.Var(newInt64Value(value, p), name, usage) +} + +// Int64 defines an int64 flag with specified name, default value, and usage string. +// The return value is the address of an int64 variable that stores the value of the flag. +func (f *FlagSet) Int64(name string, value int64, usage string) *int64 { + p := new(int64) + f.Int64Var(p, name, value, usage) + return p +} + +// Int64 defines an int64 flag with specified name, default value, and usage string. +// The return value is the address of an int64 variable that stores the value of the flag. +func Int64(name string, value int64, usage string) *int64 { + return commandLine.Int64(name, value, usage) +} + +// UintVar defines a uint flag with specified name, default value, and usage string. +// The argument p points to a uint variable in which to store the value of the flag. +func (f *FlagSet) UintVar(p *uint, name string, value uint, usage string) { + f.Var(newUintValue(value, p), name, usage) +} + +// UintVar defines a uint flag with specified name, default value, and usage string. +// The argument p points to a uint variable in which to store the value of the flag. +func UintVar(p *uint, name string, value uint, usage string) { + commandLine.Var(newUintValue(value, p), name, usage) +} + +// Uint defines a uint flag with specified name, default value, and usage string. +// The return value is the address of a uint variable that stores the value of the flag. +func (f *FlagSet) Uint(name string, value uint, usage string) *uint { + p := new(uint) + f.UintVar(p, name, value, usage) + return p +} + +// Uint defines a uint flag with specified name, default value, and usage string. +// The return value is the address of a uint variable that stores the value of the flag. +func Uint(name string, value uint, usage string) *uint { + return commandLine.Uint(name, value, usage) +} + +// Uint64Var defines a uint64 flag with specified name, default value, and usage string. +// The argument p points to a uint64 variable in which to store the value of the flag. +func (f *FlagSet) Uint64Var(p *uint64, name string, value uint64, usage string) { + f.Var(newUint64Value(value, p), name, usage) +} + +// Uint64Var defines a uint64 flag with specified name, default value, and usage string. +// The argument p points to a uint64 variable in which to store the value of the flag. +func Uint64Var(p *uint64, name string, value uint64, usage string) { + commandLine.Var(newUint64Value(value, p), name, usage) +} + +// Uint64 defines a uint64 flag with specified name, default value, and usage string. +// The return value is the address of a uint64 variable that stores the value of the flag. +func (f *FlagSet) Uint64(name string, value uint64, usage string) *uint64 { + p := new(uint64) + f.Uint64Var(p, name, value, usage) + return p +} + +// Uint64 defines a uint64 flag with specified name, default value, and usage string. +// The return value is the address of a uint64 variable that stores the value of the flag. +func Uint64(name string, value uint64, usage string) *uint64 { + return commandLine.Uint64(name, value, usage) +} + +// StringVar defines a string flag with specified name, default value, and usage string. +// The argument p points to a string variable in which to store the value of the flag. +func (f *FlagSet) StringVar(p *string, name string, value string, usage string) { + f.Var(newStringValue(value, p), name, usage) +} + +// StringVar defines a string flag with specified name, default value, and usage string. +// The argument p points to a string variable in which to store the value of the flag. +func StringVar(p *string, name string, value string, usage string) { + commandLine.Var(newStringValue(value, p), name, usage) +} + +// String defines a string flag with specified name, default value, and usage string. +// The return value is the address of a string variable that stores the value of the flag. +func (f *FlagSet) String(name string, value string, usage string) *string { + p := new(string) + f.StringVar(p, name, value, usage) + return p +} + +// String defines a string flag with specified name, default value, and usage string. +// The return value is the address of a string variable that stores the value of the flag. +func String(name string, value string, usage string) *string { + return commandLine.String(name, value, usage) +} + +// Float64Var defines a float64 flag with specified name, default value, and usage string. +// The argument p points to a float64 variable in which to store the value of the flag. +func (f *FlagSet) Float64Var(p *float64, name string, value float64, usage string) { + f.Var(newFloat64Value(value, p), name, usage) +} + +// Float64Var defines a float64 flag with specified name, default value, and usage string. +// The argument p points to a float64 variable in which to store the value of the flag. +func Float64Var(p *float64, name string, value float64, usage string) { + commandLine.Var(newFloat64Value(value, p), name, usage) +} + +// Float64 defines a float64 flag with specified name, default value, and usage string. +// The return value is the address of a float64 variable that stores the value of the flag. +func (f *FlagSet) Float64(name string, value float64, usage string) *float64 { + p := new(float64) + f.Float64Var(p, name, value, usage) + return p +} + +// Float64 defines an int flag with specified name, default value, and usage string. +// The return value is the address of a float64 variable that stores the value of the flag. +func Float64(name string, value float64, usage string) *float64 { + return commandLine.Float64(name, value, usage) +} + +// Var defines a flag with the specified name and usage string. The type and +// value of the flag are represented by the first argument, of type Value, which +// typically holds a user-defined implementation of Value. For instance, the +// caller could create a flag that turns a comma-separated string into a slice +// of strings by giving the slice the methods of Value; in particular, Set would +// decompose the comma-separated string into the slice. +func (f *FlagSet) Var(value Value, name string, usage string) { + // Remember the default value as a string; it won't change. + flag := &Flag{name, usage, value, value.String()} + _, alreadythere := f.formal[name] + if alreadythere { + fmt.Fprintf(os.Stderr, "%s flag redefined: %s\n", f.name, name) + panic("flag redefinition") // Happens only if flags are declared with identical names + } + f.formal[name] = flag +} + +// Var defines a flag with the specified name and usage string. The type and +// value of the flag are represented by the first argument, of type Value, which +// typically holds a user-defined implementation of Value. For instance, the +// caller could create a flag that turns a comma-separated string into a slice +// of strings by giving the slice the methods of Value; in particular, Set would +// decompose the comma-separated string into the slice. +func Var(value Value, name string, usage string) { + commandLine.Var(value, name, usage) +} + +// failf prints to standard error a formatted error and usage message and +// returns the error. +func (f *FlagSet) failf(format string, a ...interface{}) os.Error { + err := fmt.Errorf(format, a...) + fmt.Fprintln(os.Stderr, err) + f.usage() + return err +} + +// usage calls the Usage method for the flag set, or the usage function if +// the flag set is commandLine. +func (f *FlagSet) usage() { + if f == commandLine { + Usage() + } else { + f.Usage() + } +} + +// parseOne parses one flag. It returns whether a flag was seen. +func (f *FlagSet) parseOne() (bool, os.Error) { + if len(f.args) == 0 { + return false, nil + } + s := f.args[0] + if len(s) == 0 || s[0] != '-' || len(s) == 1 { + return false, nil + } + num_minuses := 1 + if s[1] == '-' { + num_minuses++ + if len(s) == 2 { // "--" terminates the flags + f.args = f.args[1:] + return false, nil + } + } + name := s[num_minuses:] + if len(name) == 0 || name[0] == '-' || name[0] == '=' { + return false, f.failf("bad flag syntax: %s", s) + } + + // it's a flag. does it have an argument? + f.args = f.args[1:] + has_value := false + value := "" + for i := 1; i < len(name); i++ { // equals cannot be first + if name[i] == '=' { + value = name[i+1:] + has_value = true + name = name[0:i] + break + } + } + m := f.formal + flag, alreadythere := m[name] // BUG + if !alreadythere { + if name == "help" || name == "h" { // special case for nice help message. + f.usage() + return false, ErrHelp + } + return false, f.failf("flag provided but not defined: -%s", name) + } + if fv, ok := flag.Value.(*boolValue); ok { // special case: doesn't need an arg + if has_value { + if !fv.Set(value) { + f.failf("invalid boolean value %q for flag: -%s", value, name) + } + } else { + fv.Set("true") + } + } else { + // It must have a value, which might be the next argument. + if !has_value && len(f.args) > 0 { + // value is the next arg + has_value = true + value, f.args = f.args[0], f.args[1:] + } + if !has_value { + return false, f.failf("flag needs an argument: -%s", name) + } + ok = flag.Value.Set(value) + if !ok { + return false, f.failf("invalid value %q for flag: -%s", value, name) + } + } + f.actual[name] = flag + return true, nil +} + +// Parse parses flag definitions from the argument list, which should not +// include the command name. Must be called after all flags in the FlagSet +// are defined and before flags are accessed by the program. +// The return value will be ErrHelp if -help was set but not defined. +func (f *FlagSet) Parse(arguments []string) os.Error { + f.args = arguments + for { + seen, err := f.parseOne() + if seen { + continue + } + if err == nil { + break + } + switch f.errorHandling { + case ContinueOnError: + return err + case ExitOnError: + os.Exit(2) + case PanicOnError: + panic(err) + } + } + return nil +} + +// Parse parses the command-line flags from os.Args[1:]. Must be called +// after all flags are defined and before flags are accessed by the program. +func Parse() { + // Ignore errors; commandLine is set for ExitOnError. + commandLine.Parse(os.Args[1:]) +} + +// The default set of command-line flags, parsed from os.Args. +var commandLine = NewFlagSet(os.Args[0], ExitOnError) + +// NewFlagSet returns a new, empty flag set with the specified name and +// error handling property. +func NewFlagSet(name string, errorHandling ErrorHandling) *FlagSet { + f := &FlagSet{ + name: name, + actual: make(map[string]*Flag), + formal: make(map[string]*Flag), + errorHandling: errorHandling, + } + f.Usage = func() { defaultUsage(f) } + return f +} diff --git a/src/pkg/flag/flag_test.go b/src/pkg/flag/flag_test.go new file mode 100644 index 000000000..63d0a9fc8 --- /dev/null +++ b/src/pkg/flag/flag_test.go @@ -0,0 +1,255 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package flag_test + +import ( + . "flag" + "fmt" + "os" + "sort" + "testing" +) + +var ( + test_bool = Bool("test_bool", false, "bool value") + test_int = Int("test_int", 0, "int value") + test_int64 = Int64("test_int64", 0, "int64 value") + test_uint = Uint("test_uint", 0, "uint value") + test_uint64 = Uint64("test_uint64", 0, "uint64 value") + test_string = String("test_string", "0", "string value") + test_float64 = Float64("test_float64", 0, "float64 value") +) + +func boolString(s string) string { + if s == "0" { + return "false" + } + return "true" +} + +func TestEverything(t *testing.T) { + m := make(map[string]*Flag) + desired := "0" + visitor := func(f *Flag) { + if len(f.Name) > 5 && f.Name[0:5] == "test_" { + m[f.Name] = f + ok := false + switch { + case f.Value.String() == desired: + ok = true + case f.Name == "test_bool" && f.Value.String() == boolString(desired): + ok = true + } + if !ok { + t.Error("Visit: bad value", f.Value.String(), "for", f.Name) + } + } + } + VisitAll(visitor) + if len(m) != 7 { + t.Error("VisitAll misses some flags") + for k, v := range m { + t.Log(k, *v) + } + } + m = make(map[string]*Flag) + Visit(visitor) + if len(m) != 0 { + t.Errorf("Visit sees unset flags") + for k, v := range m { + t.Log(k, *v) + } + } + // Now set all flags + Set("test_bool", "true") + Set("test_int", "1") + Set("test_int64", "1") + Set("test_uint", "1") + Set("test_uint64", "1") + Set("test_string", "1") + Set("test_float64", "1") + desired = "1" + Visit(visitor) + if len(m) != 7 { + t.Error("Visit fails after set") + for k, v := range m { + t.Log(k, *v) + } + } + // Now test they're visited in sort order. + var flagNames []string + Visit(func(f *Flag) { flagNames = append(flagNames, f.Name) }) + if !sort.StringsAreSorted(flagNames) { + t.Errorf("flag names not sorted: %v", flagNames) + } +} + +func TestUsage(t *testing.T) { + called := false + ResetForTesting(func() { called = true }) + if CommandLine().Parse([]string{"-x"}) == nil { + t.Error("parse did not fail for unknown flag") + } + if !called { + t.Error("did not call Usage for unknown flag") + } +} + +func testParse(f *FlagSet, t *testing.T) { + boolFlag := f.Bool("bool", false, "bool value") + bool2Flag := f.Bool("bool2", false, "bool2 value") + intFlag := f.Int("int", 0, "int value") + int64Flag := f.Int64("int64", 0, "int64 value") + uintFlag := f.Uint("uint", 0, "uint value") + uint64Flag := f.Uint64("uint64", 0, "uint64 value") + stringFlag := f.String("string", "0", "string value") + float64Flag := f.Float64("float64", 0, "float64 value") + extra := "one-extra-argument" + args := []string{ + "-bool", + "-bool2=true", + "--int", "22", + "--int64", "0x23", + "-uint", "24", + "--uint64", "25", + "-string", "hello", + "-float64", "2718e28", + extra, + } + if err := f.Parse(args); err != nil { + t.Fatal(err) + } + if *boolFlag != true { + t.Error("bool flag should be true, is ", *boolFlag) + } + if *bool2Flag != true { + t.Error("bool2 flag should be true, is ", *bool2Flag) + } + if *intFlag != 22 { + t.Error("int flag should be 22, is ", *intFlag) + } + if *int64Flag != 0x23 { + t.Error("int64 flag should be 0x23, is ", *int64Flag) + } + if *uintFlag != 24 { + t.Error("uint flag should be 24, is ", *uintFlag) + } + if *uint64Flag != 25 { + t.Error("uint64 flag should be 25, is ", *uint64Flag) + } + if *stringFlag != "hello" { + t.Error("string flag should be `hello`, is ", *stringFlag) + } + if *float64Flag != 2718e28 { + t.Error("float64 flag should be 2718e28, is ", *float64Flag) + } + if len(f.Args()) != 1 { + t.Error("expected one argument, got", len(f.Args())) + } else if f.Args()[0] != extra { + t.Errorf("expected argument %q got %q", extra, f.Args()[0]) + } +} + +func TestParse(t *testing.T) { + ResetForTesting(func() { t.Error("bad parse") }) + testParse(CommandLine(), t) +} + +func TestFlagSetParse(t *testing.T) { + testParse(NewFlagSet("test", ContinueOnError), t) +} + +// Declare a user-defined flag type. +type flagVar []string + +func (f *flagVar) String() string { + return fmt.Sprint([]string(*f)) +} + +func (f *flagVar) Set(value string) bool { + *f = append(*f, value) + return true +} + +func TestUserDefined(t *testing.T) { + flags := NewFlagSet("test", ContinueOnError) + var v flagVar + flags.Var(&v, "v", "usage") + if err := flags.Parse([]string{"-v", "1", "-v", "2", "-v=3"}); err != nil { + t.Error(err) + } + if len(v) != 3 { + t.Fatal("expected 3 args; got ", len(v)) + } + expect := "[1 2 3]" + if v.String() != expect { + t.Errorf("expected value %q got %q", expect, v.String()) + } +} + +// This tests that one can reset the flags. This still works but not well, and is +// superseded by FlagSet. +func TestChangingArgs(t *testing.T) { + ResetForTesting(func() { t.Fatal("bad parse") }) + oldArgs := os.Args + defer func() { os.Args = oldArgs }() + os.Args = []string{"cmd", "-before", "subcmd", "-after", "args"} + before := Bool("before", false, "") + if err := CommandLine().Parse(os.Args[1:]); err != nil { + t.Fatal(err) + } + cmd := Arg(0) + os.Args = Args() + after := Bool("after", false, "") + Parse() + args := Args() + + if !*before || cmd != "subcmd" || !*after || len(args) != 1 || args[0] != "args" { + t.Fatalf("expected true subcmd true [args] got %v %v %v %v", *before, cmd, *after, args) + } +} + +// Test that -help invokes the usage message and returns ErrHelp. +func TestHelp(t *testing.T) { + var helpCalled = false + fs := NewFlagSet("help test", ContinueOnError) + fs.Usage = func() { helpCalled = true } + var flag bool + fs.BoolVar(&flag, "flag", false, "regular flag") + // Regular flag invocation should work + err := fs.Parse([]string{"-flag=true"}) + if err != nil { + t.Fatal("expected no error; got ", err) + } + if !flag { + t.Error("flag was not set by -flag") + } + if helpCalled { + t.Error("help called for regular flag") + helpCalled = false // reset for next test + } + // Help flag should work as expected. + err = fs.Parse([]string{"-help"}) + if err == nil { + t.Fatal("error expected") + } + if err != ErrHelp { + t.Fatal("expected ErrHelp; got ", err) + } + if !helpCalled { + t.Fatal("help was not called") + } + // If we define a help flag, that should override. + var help bool + fs.BoolVar(&help, "help", false, "help flag") + helpCalled = false + err = fs.Parse([]string{"-help"}) + if err != nil { + t.Fatal("expected no error for defined -help; got ", err) + } + if helpCalled { + t.Fatal("help was called; should not have been for defined help flag") + } +} diff --git a/src/pkg/fmt/Makefile b/src/pkg/fmt/Makefile new file mode 100644 index 000000000..44b48bc67 --- /dev/null +++ b/src/pkg/fmt/Makefile @@ -0,0 +1,14 @@ +# Copyright 2009 The Go Authors. All rights reserved. +# Use of this source code is governed by a BSD-style +# license that can be found in the LICENSE file. + +include ../../Make.inc + +TARG=fmt +GOFILES=\ + doc.go\ + format.go\ + print.go\ + scan.go\ + +include ../../Make.pkg diff --git a/src/pkg/fmt/doc.go b/src/pkg/fmt/doc.go new file mode 100644 index 000000000..35a11e19f --- /dev/null +++ b/src/pkg/fmt/doc.go @@ -0,0 +1,181 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +/* + Package fmt implements formatted I/O with functions analogous + to C's printf and scanf. The format 'verbs' are derived from C's but + are simpler. + + Printing: + + The verbs: + + General: + %v the value in a default format. + when printing structs, the plus flag (%+v) adds field names + %#v a Go-syntax representation of the value + %T a Go-syntax representation of the type of the value + %% a literal percent sign; consumes no value + + Boolean: + %t the word true or false + Integer: + %b base 2 + %c the character represented by the corresponding Unicode code point + %d base 10 + %o base 8 + %q a single-quoted character literal safely escaped with Go syntax. + %x base 16, with lower-case letters for a-f + %X base 16, with upper-case letters for A-F + %U Unicode format: U+1234; same as "U+%04X" + Floating-point and complex constituents: + %b decimalless scientific notation with exponent a power + of two, in the manner of strconv.Ftoa32, e.g. -123456p-78 + %e scientific notation, e.g. -1234.456e+78 + %E scientific notation, e.g. -1234.456E+78 + %f decimal point but no exponent, e.g. 123.456 + %g whichever of %e or %f produces more compact output + %G whichever of %E or %f produces more compact output + String and slice of bytes: + %s the uninterpreted bytes of the string or slice + %q a double-quoted string safely escaped with Go syntax + %x base 16, lower-case, two characters per byte + %X base 16, upper-case, two characters per byte + Pointer: + %p base 16 notation, with leading 0x + + There is no 'u' flag. Integers are printed unsigned if they have unsigned type. + Similarly, there is no need to specify the size of the operand (int8, int64). + + The width and precision control formatting and are in units of Unicode + code points. (This differs from C's printf where the units are numbers + of bytes.) Either or both of the flags may be replaced with the + character '*', causing their values to be obtained from the next + operand, which must be of type int. + + For numeric values, width sets the width of the field and precision + sets the number of places after the decimal, if appropriate. For + example, the format %6.2f prints 123.45. + + For strings, width is the minimum number of characters to output, + padding with spaces if necessary, and precision is the maximum + number of characters to output, truncating if necessary. + + Other flags: + + always print a sign for numeric values; + guarantee ASCII-only output for %q (%+q) + - pad with spaces on the right rather than the left (left-justify the field) + # alternate format: add leading 0 for octal (%#o), 0x for hex (%#x); + 0X for hex (%#X); suppress 0x for %p (%#p); + print a raw (backquoted) string if possible for %q (%#q); + write e.g. U+0078 'x' if the character is printable for %U (%#U). + ' ' (space) leave a space for elided sign in numbers (% d); + put spaces between bytes printing strings or slices in hex (% x, % X) + 0 pad with leading zeros rather than spaces + + For each Printf-like function, there is also a Print function + that takes no format and is equivalent to saying %v for every + operand. Another variant Println inserts blanks between + operands and appends a newline. + + Regardless of the verb, if an operand is an interface value, + the internal concrete value is used, not the interface itself. + Thus: + var i interface{} = 23 + fmt.Printf("%v\n", i) + will print 23. + + If an operand implements interface Formatter, that interface + can be used for fine control of formatting. + + If an operand implements method String() string that method + will be used to convert the object to a string, which will then + be formatted as required by the verb (if any). To avoid + recursion in cases such as + type X int + func (x X) String() string { return Sprintf("%d", x) } + cast the value before recurring: + func (x X) String() string { return Sprintf("%d", int(x)) } + + Format errors: + + If an invalid argument is given for a verb, such as providing + a string to %d, the generated string will contain a + description of the problem, as in these examples: + + Wrong type or unknown verb: %!verb(type=value) + Printf("%d", hi): %!d(string=hi) + Too many arguments: %!(EXTRA type=value) + Printf("hi", "guys"): hi%!(EXTRA string=guys) + Too few arguments: %!verb(MISSING) + Printf("hi%d"): hi %!d(MISSING) + Non-int for width or precision: %!(BADWIDTH) or %!(BADPREC) + Printf("%*s", 4.5, "hi"): %!(BADWIDTH)hi + Printf("%.*s", 4.5, "hi"): %!(BADPREC)hi + + All errors begin with the string "%!" followed sometimes + by a single character (the verb) and end with a parenthesized + description. + + Scanning: + + An analogous set of functions scans formatted text to yield + values. Scan, Scanf and Scanln read from os.Stdin; Fscan, + Fscanf and Fscanln read from a specified os.Reader; Sscan, + Sscanf and Sscanln read from an argument string. Scanln, + Fscanln and Sscanln stop scanning at a newline and require that + the items be followed by one; Sscanf, Fscanf and Sscanf require + newlines in the input to match newlines in the format; the other + routines treat newlines as spaces. + + Scanf, Fscanf, and Sscanf parse the arguments according to a + format string, analogous to that of Printf. For example, %x + will scan an integer as a hexadecimal number, and %v will scan + the default representation format for the value. + + The formats behave analogously to those of Printf with the + following exceptions: + + %p is not implemented + %T is not implemented + %e %E %f %F %g %G are all equivalent and scan any floating point or complex value + %s and %v on strings scan a space-delimited token + + The familiar base-setting prefixes 0 (octal) and 0x + (hexadecimal) are accepted when scanning integers without a + format or with the %v verb. + + Width is interpreted in the input text (%5s means at most + five runes of input will be read to scan a string) but there + is no syntax for scanning with a precision (no %5.2f, just + %5f). + + When scanning with a format, all non-empty runs of space + characters (except newline) are equivalent to a single + space in both the format and the input. With that proviso, + text in the format string must match the input text; scanning + stops if it does not, with the return value of the function + indicating the number of arguments scanned. + + In all the scanning functions, if an operand implements method + Scan (that is, it implements the Scanner interface) that + method will be used to scan the text for that operand. Also, + if the number of arguments scanned is less than the number of + arguments provided, an error is returned. + + All arguments to be scanned must be either pointers to basic + types or implementations of the Scanner interface. + + Note: Fscan etc. can read one character (rune) past the input + they return, which means that a loop calling a scan routine + may skip some of the input. This is usually a problem only + when there is no space between input values. If the reader + provided to Fscan implements ReadRune, that method will be used + to read characters. If the reader also implements UnreadRune, + that method will be used to save the character and successive + calls will not lose data. To attach ReadRune and UnreadRune + methods to a reader without that capability, use + bufio.NewReader. +*/ +package fmt diff --git a/src/pkg/fmt/fmt_test.go b/src/pkg/fmt/fmt_test.go new file mode 100644 index 000000000..1142c9f8a --- /dev/null +++ b/src/pkg/fmt/fmt_test.go @@ -0,0 +1,746 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package fmt_test + +import ( + . "fmt" + "io" + "math" + "runtime" // for the malloc count test only + "strings" + "testing" +) + +type ( + renamedBool bool + renamedInt int + renamedInt8 int8 + renamedInt16 int16 + renamedInt32 int32 + renamedInt64 int64 + renamedUint uint + renamedUint8 uint8 + renamedUint16 uint16 + renamedUint32 uint32 + renamedUint64 uint64 + renamedUintptr uintptr + renamedString string + renamedBytes []byte + renamedFloat32 float32 + renamedFloat64 float64 + renamedComplex64 complex64 + renamedComplex128 complex128 +) + +func TestFmtInterface(t *testing.T) { + var i1 interface{} + i1 = "abc" + s := Sprintf("%s", i1) + if s != "abc" { + t.Errorf(`Sprintf("%%s", empty("abc")) = %q want %q`, s, "abc") + } +} + +const b32 uint32 = 1<<32 - 1 +const b64 uint64 = 1<<64 - 1 + +var array = []int{1, 2, 3, 4, 5} +var iarray = []interface{}{1, "hello", 2.5, nil} + +type A struct { + i int + j uint + s string + x []int +} + +type I int + +func (i I) String() string { return Sprintf("<%d>", int(i)) } + +type B struct { + i I + j int +} + +type C struct { + i int + B +} + +type F int + +func (f F) Format(s State, c int) { + Fprintf(s, "<%c=F(%d)>", c, int(f)) +} + +type G int + +func (g G) GoString() string { + return Sprintf("GoString(%d)", int(g)) +} + +type S struct { + f F // a struct field that Formats + g G // a struct field that GoStrings +} + +// A type with a String method with pointer receiver for testing %p +type P int + +var pValue P + +func (p *P) String() string { + return "String(p)" +} + +var b byte + +var fmttests = []struct { + fmt string + val interface{} + out string +}{ + {"%d", 12345, "12345"}, + {"%v", 12345, "12345"}, + {"%t", true, "true"}, + + // basic string + {"%s", "abc", "abc"}, + {"%x", "abc", "616263"}, + {"%x", "xyz", "78797a"}, + {"%X", "xyz", "78797A"}, + {"%q", "abc", `"abc"`}, + + // basic bytes + {"%s", []byte("abc"), "abc"}, + {"%x", []byte("abc"), "616263"}, + {"% x", []byte("abc\xff"), "61 62 63 ff"}, + {"% X", []byte("abc\xff"), "61 62 63 FF"}, + {"%x", []byte("xyz"), "78797a"}, + {"%X", []byte("xyz"), "78797A"}, + {"%q", []byte("abc"), `"abc"`}, + + // escaped strings + {"%#q", `abc`, "`abc`"}, + {"%#q", `"`, "`\"`"}, + {"1 %#q", `\n`, "1 `\\n`"}, + {"2 %#q", "\n", `2 "\n"`}, + {"%q", `"`, `"\""`}, + {"%q", "\a\b\f\r\n\t\v", `"\a\b\f\r\n\t\v"`}, + {"%q", "abc\xffdef", `"abc\xffdef"`}, + {"%q", "\u263a", `"☺"`}, + {"%+q", "\u263a", `"\u263a"`}, + {"%q", "\U0010ffff", `"\U0010ffff"`}, + + // escaped characters + {"%q", 'x', `'x'`}, + {"%q", 0, `'\x00'`}, + {"%q", '\n', `'\n'`}, + {"%q", '\u0e00', `'\u0e00'`}, // not a printable rune. + {"%q", '\U000c2345', `'\U000c2345'`}, // not a printable rune. + {"%q", int64(0x7FFFFFFF), `%!q(int64=2147483647)`}, + {"%q", uint64(0xFFFFFFFF), `%!q(uint64=4294967295)`}, + {"%q", '"', `'"'`}, + {"%q", '\'', `'\''`}, + {"%q", "\u263a", `"☺"`}, + {"%+q", "\u263a", `"\u263a"`}, + + // width + {"%5s", "abc", " abc"}, + {"%2s", "\u263a", " ☺"}, + {"%-5s", "abc", "abc "}, + {"%-8q", "abc", `"abc" `}, + {"%05s", "abc", "00abc"}, + {"%08q", "abc", `000"abc"`}, + {"%5s", "abcdefghijklmnopqrstuvwxyz", "abcdefghijklmnopqrstuvwxyz"}, + {"%.5s", "abcdefghijklmnopqrstuvwxyz", "abcde"}, + {"%.5s", "日本語日本語", "日本語日本"}, + {"%.5s", []byte("日本語日本語"), "日本語日本"}, + {"%.5q", "abcdefghijklmnopqrstuvwxyz", `"abcde"`}, + {"%.3q", "日本語日本語", `"日本語"`}, + {"%.3q", []byte("日本語日本語"), `"日本語"`}, + {"%10.1q", "日本語日本語", ` "日"`}, + + // integers + {"%d", 12345, "12345"}, + {"%d", -12345, "-12345"}, + {"%10d", 12345, " 12345"}, + {"%10d", -12345, " -12345"}, + {"%+10d", 12345, " +12345"}, + {"%010d", 12345, "0000012345"}, + {"%010d", -12345, "-000012345"}, + {"%-10d", 12345, "12345 "}, + {"%010.3d", 1, " 001"}, + {"%010.3d", -1, " -001"}, + {"%+d", 12345, "+12345"}, + {"%+d", -12345, "-12345"}, + {"%+d", 0, "+0"}, + {"% d", 0, " 0"}, + {"% d", 12345, " 12345"}, + {"%.0d", 0, ""}, + {"%.d", 0, ""}, + + // unicode format + {"%U", 0x1, "U+0001"}, + {"%U", uint(0x1), "U+0001"}, + {"%.8U", 0x2, "U+00000002"}, + {"%U", 0x1234, "U+1234"}, + {"%U", 0x12345, "U+12345"}, + {"%10.6U", 0xABC, " U+000ABC"}, + {"%-10.6U", 0xABC, "U+000ABC "}, + {"%U", '\n', `U+000A`}, + {"%#U", '\n', `U+000A`}, + {"%U", 'x', `U+0078`}, + {"%#U", 'x', `U+0078 'x'`}, + {"%U", '\u263a', `U+263A`}, + {"%#U", '\u263a', `U+263A '☺'`}, + + // floats + {"%+.3e", 0.0, "+0.000e+00"}, + {"%+.3e", 1.0, "+1.000e+00"}, + {"%+.3f", -1.0, "-1.000"}, + {"% .3E", -1.0, "-1.000E+00"}, + {"% .3e", 1.0, " 1.000e+00"}, + {"%+.3g", 0.0, "+0"}, + {"%+.3g", 1.0, "+1"}, + {"%+.3g", -1.0, "-1"}, + {"% .3g", -1.0, "-1"}, + {"% .3g", 1.0, " 1"}, + + // complex values + {"%+.3e", 0i, "(+0.000e+00+0.000e+00i)"}, + {"%+.3f", 0i, "(+0.000+0.000i)"}, + {"%+.3g", 0i, "(+0+0i)"}, + {"%+.3e", 1 + 2i, "(+1.000e+00+2.000e+00i)"}, + {"%+.3f", 1 + 2i, "(+1.000+2.000i)"}, + {"%+.3g", 1 + 2i, "(+1+2i)"}, + {"%.3e", 0i, "(0.000e+00+0.000e+00i)"}, + {"%.3f", 0i, "(0.000+0.000i)"}, + {"%.3g", 0i, "(0+0i)"}, + {"%.3e", 1 + 2i, "(1.000e+00+2.000e+00i)"}, + {"%.3f", 1 + 2i, "(1.000+2.000i)"}, + {"%.3g", 1 + 2i, "(1+2i)"}, + {"%.3e", -1 - 2i, "(-1.000e+00-2.000e+00i)"}, + {"%.3f", -1 - 2i, "(-1.000-2.000i)"}, + {"%.3g", -1 - 2i, "(-1-2i)"}, + {"% .3E", -1 - 2i, "(-1.000E+00-2.000E+00i)"}, + {"%+.3g", complex64(1 + 2i), "(+1+2i)"}, + {"%+.3g", complex128(1 + 2i), "(+1+2i)"}, + + // erroneous formats + {"", 2, "%!(EXTRA int=2)"}, + {"%d", "hello", "%!d(string=hello)"}, + + // old test/fmt_test.go + {"%d", 1234, "1234"}, + {"%d", -1234, "-1234"}, + {"%d", uint(1234), "1234"}, + {"%d", uint32(b32), "4294967295"}, + {"%d", uint64(b64), "18446744073709551615"}, + {"%o", 01234, "1234"}, + {"%#o", 01234, "01234"}, + {"%o", uint32(b32), "37777777777"}, + {"%o", uint64(b64), "1777777777777777777777"}, + {"%x", 0x1234abcd, "1234abcd"}, + {"%#x", 0x1234abcd, "0x1234abcd"}, + {"%x", b32 - 0x1234567, "fedcba98"}, + {"%X", 0x1234abcd, "1234ABCD"}, + {"%X", b32 - 0x1234567, "FEDCBA98"}, + {"%#X", 0, "0X0"}, + {"%x", b64, "ffffffffffffffff"}, + {"%b", 7, "111"}, + {"%b", b64, "1111111111111111111111111111111111111111111111111111111111111111"}, + {"%b", -6, "-110"}, + {"%e", 1.0, "1.000000e+00"}, + {"%e", 1234.5678e3, "1.234568e+06"}, + {"%e", 1234.5678e-8, "1.234568e-05"}, + {"%e", -7.0, "-7.000000e+00"}, + {"%e", -1e-9, "-1.000000e-09"}, + {"%f", 1234.5678e3, "1234567.800000"}, + {"%f", 1234.5678e-8, "0.000012"}, + {"%f", -7.0, "-7.000000"}, + {"%f", -1e-9, "-0.000000"}, + {"%g", 1234.5678e3, "1.2345678e+06"}, + {"%g", float32(1234.5678e3), "1.2345678e+06"}, + {"%g", 1234.5678e-8, "1.2345678e-05"}, + {"%g", -7.0, "-7"}, + {"%g", -1e-9, "-1e-09"}, + {"%g", float32(-1e-9), "-1e-09"}, + {"%E", 1.0, "1.000000E+00"}, + {"%E", 1234.5678e3, "1.234568E+06"}, + {"%E", 1234.5678e-8, "1.234568E-05"}, + {"%E", -7.0, "-7.000000E+00"}, + {"%E", -1e-9, "-1.000000E-09"}, + {"%G", 1234.5678e3, "1.2345678E+06"}, + {"%G", float32(1234.5678e3), "1.2345678E+06"}, + {"%G", 1234.5678e-8, "1.2345678E-05"}, + {"%G", -7.0, "-7"}, + {"%G", -1e-9, "-1E-09"}, + {"%G", float32(-1e-9), "-1E-09"}, + {"%c", 'x', "x"}, + {"%c", 0xe4, "ä"}, + {"%c", 0x672c, "本"}, + {"%c", '日', "日"}, + {"%20.8d", 1234, " 00001234"}, + {"%20.8d", -1234, " -00001234"}, + {"%20d", 1234, " 1234"}, + {"%-20.8d", 1234, "00001234 "}, + {"%-20.8d", -1234, "-00001234 "}, + {"%-#20.8x", 0x1234abc, "0x01234abc "}, + {"%-#20.8X", 0x1234abc, "0X01234ABC "}, + {"%-#20.8o", 01234, "00001234 "}, + {"%.20b", 7, "00000000000000000111"}, + {"%20.5s", "qwertyuiop", " qwert"}, + {"%.5s", "qwertyuiop", "qwert"}, + {"%-20.5s", "qwertyuiop", "qwert "}, + {"%20c", 'x', " x"}, + {"%-20c", 'x', "x "}, + {"%20.6e", 1.2345e3, " 1.234500e+03"}, + {"%20.6e", 1.2345e-3, " 1.234500e-03"}, + {"%20e", 1.2345e3, " 1.234500e+03"}, + {"%20e", 1.2345e-3, " 1.234500e-03"}, + {"%20.8e", 1.2345e3, " 1.23450000e+03"}, + {"%20f", 1.23456789e3, " 1234.567890"}, + {"%20f", 1.23456789e-3, " 0.001235"}, + {"%20f", 12345678901.23456789, " 12345678901.234568"}, + {"%-20f", 1.23456789e3, "1234.567890 "}, + {"%20.8f", 1.23456789e3, " 1234.56789000"}, + {"%20.8f", 1.23456789e-3, " 0.00123457"}, + {"%g", 1.23456789e3, "1234.56789"}, + {"%g", 1.23456789e-3, "0.00123456789"}, + {"%g", 1.23456789e20, "1.23456789e+20"}, + {"%20e", math.Inf(1), " +Inf"}, + {"%-20f", math.Inf(-1), "-Inf "}, + {"%20g", math.NaN(), " NaN"}, + + // arrays + {"%v", array, "[1 2 3 4 5]"}, + {"%v", iarray, "[1 hello 2.5 ]"}, + {"%v", &array, "&[1 2 3 4 5]"}, + {"%v", &iarray, "&[1 hello 2.5 ]"}, + + // complexes with %v + {"%v", 1 + 2i, "(1+2i)"}, + {"%v", complex64(1 + 2i), "(1+2i)"}, + {"%v", complex128(1 + 2i), "(1+2i)"}, + + // structs + {"%v", A{1, 2, "a", []int{1, 2}}, `{1 2 a [1 2]}`}, + {"%+v", A{1, 2, "a", []int{1, 2}}, `{i:1 j:2 s:a x:[1 2]}`}, + + // +v on structs with Stringable items + {"%+v", B{1, 2}, `{i:<1> j:2}`}, + {"%+v", C{1, B{2, 3}}, `{i:1 B:{i:<2> j:3}}`}, + + // q on Stringable items + {"%s", I(23), `<23>`}, + {"%q", I(23), `"<23>"`}, + {"%x", I(23), `3c32333e`}, + {"%d", I(23), `%!d(string=<23>)`}, + + // go syntax + {"%#v", A{1, 2, "a", []int{1, 2}}, `fmt_test.A{i:1, j:0x2, s:"a", x:[]int{1, 2}}`}, + {"%#v", &b, "(*uint8)(0xPTR)"}, + {"%#v", TestFmtInterface, "(func(*testing.T))(0xPTR)"}, + {"%#v", make(chan int), "(chan int)(0xPTR)"}, + {"%#v", uint64(1<<64 - 1), "0xffffffffffffffff"}, + {"%#v", 1000000000, "1000000000"}, + {"%#v", map[string]int{"a": 1, "b": 2}, `map[string] int{"a":1, "b":2}`}, + {"%#v", map[string]B{"a": {1, 2}, "b": {3, 4}}, `map[string] fmt_test.B{"a":fmt_test.B{i:1, j:2}, "b":fmt_test.B{i:3, j:4}}`}, + {"%#v", []string{"a", "b"}, `[]string{"a", "b"}`}, + + // slices with other formats + {"%#x", []int{1, 2, 15}, `[0x1 0x2 0xf]`}, + {"%x", []int{1, 2, 15}, `[1 2 f]`}, + {"%d", []int{1, 2, 15}, `[1 2 15]`}, + {"%d", []byte{1, 2, 15}, `[1 2 15]`}, + {"%q", []string{"a", "b"}, `["a" "b"]`}, + + // renamings + {"%v", renamedBool(true), "true"}, + {"%d", renamedBool(true), "%!d(fmt_test.renamedBool=true)"}, + {"%o", renamedInt(8), "10"}, + {"%d", renamedInt8(-9), "-9"}, + {"%v", renamedInt16(10), "10"}, + {"%v", renamedInt32(-11), "-11"}, + {"%X", renamedInt64(255), "FF"}, + {"%v", renamedUint(13), "13"}, + {"%o", renamedUint8(14), "16"}, + {"%X", renamedUint16(15), "F"}, + {"%d", renamedUint32(16), "16"}, + {"%X", renamedUint64(17), "11"}, + {"%o", renamedUintptr(18), "22"}, + {"%x", renamedString("thing"), "7468696e67"}, + {"%d", renamedBytes([]byte{1, 2, 15}), `[1 2 15]`}, + {"%q", renamedBytes([]byte("hello")), `"hello"`}, + {"%v", renamedFloat32(22), "22"}, + {"%v", renamedFloat64(33), "33"}, + {"%v", renamedComplex64(3 + 4i), "(3+4i)"}, + {"%v", renamedComplex128(4 - 3i), "(4-3i)"}, + + // Formatter + {"%x", F(1), ""}, + {"%x", G(2), "2"}, + {"%+v", S{F(4), G(5)}, "{f: g:5}"}, + + // GoStringer + {"%#v", G(6), "GoString(6)"}, + {"%#v", S{F(7), G(8)}, "fmt_test.S{f:, g:GoString(8)}"}, + + // %T + {"%T", (4 - 3i), "complex128"}, + {"%T", renamedComplex128(4 - 3i), "fmt_test.renamedComplex128"}, + {"%T", intVal, "int"}, + {"%6T", &intVal, " *int"}, + + // %p + {"p0=%p", new(int), "p0=0xPTR"}, + {"p1=%s", &pValue, "p1=String(p)"}, // String method... + {"p2=%p", &pValue, "p2=0xPTR"}, // ... not called with %p + {"p4=%#p", new(int), "p4=PTR"}, + + // %p on non-pointers + {"%p", make(chan int), "0xPTR"}, + {"%p", make(map[int]int), "0xPTR"}, + {"%p", make([]int, 1), "0xPTR"}, + {"%p", 27, "%!p(int=27)"}, // not a pointer at all + + // erroneous things + {"%s %", "hello", "hello %!(NOVERB)"}, + {"%s %.2", "hello", "hello %!(NOVERB)"}, + {"%d", "hello", "%!d(string=hello)"}, + {"no args", "hello", "no args%!(EXTRA string=hello)"}, + {"%s", nil, "%!s()"}, + {"%T", nil, ""}, + {"%-1", 100, "%!(NOVERB)%!(EXTRA int=100)"}, +} + +func TestSprintf(t *testing.T) { + for _, tt := range fmttests { + s := Sprintf(tt.fmt, tt.val) + if i := strings.Index(tt.out, "PTR"); i >= 0 { + j := i + for ; j < len(s); j++ { + c := s[j] + if (c < '0' || c > '9') && (c < 'a' || c > 'f') && (c < 'A' || c > 'F') { + break + } + } + s = s[0:i] + "PTR" + s[j:] + } + if s != tt.out { + if _, ok := tt.val.(string); ok { + // Don't requote the already-quoted strings. + // It's too confusing to read the errors. + t.Errorf("Sprintf(%q, %q) = <%s> want <%s>", tt.fmt, tt.val, s, tt.out) + } else { + t.Errorf("Sprintf(%q, %v) = %q want %q", tt.fmt, tt.val, s, tt.out) + } + } + } +} + +func BenchmarkSprintfEmpty(b *testing.B) { + for i := 0; i < b.N; i++ { + Sprintf("") + } +} + +func BenchmarkSprintfString(b *testing.B) { + for i := 0; i < b.N; i++ { + Sprintf("%s", "hello") + } +} + +func BenchmarkSprintfInt(b *testing.B) { + for i := 0; i < b.N; i++ { + Sprintf("%d", 5) + } +} + +func BenchmarkSprintfIntInt(b *testing.B) { + for i := 0; i < b.N; i++ { + Sprintf("%d %d", 5, 6) + } +} + +func BenchmarkSprintfPrefixedInt(b *testing.B) { + for i := 0; i < b.N; i++ { + Sprintf("This is some meaningless prefix text that needs to be scanned %d", 6) + } +} + +func TestCountMallocs(t *testing.T) { + if testing.Short() { + return + } + runtime.UpdateMemStats() + mallocs := 0 - runtime.MemStats.Mallocs + for i := 0; i < 100; i++ { + Sprintf("") + } + runtime.UpdateMemStats() + mallocs += runtime.MemStats.Mallocs + Printf("mallocs per Sprintf(\"\"): %d\n", mallocs/100) + runtime.UpdateMemStats() + mallocs = 0 - runtime.MemStats.Mallocs + for i := 0; i < 100; i++ { + Sprintf("xxx") + } + runtime.UpdateMemStats() + mallocs += runtime.MemStats.Mallocs + Printf("mallocs per Sprintf(\"xxx\"): %d\n", mallocs/100) + runtime.UpdateMemStats() + mallocs = 0 - runtime.MemStats.Mallocs + for i := 0; i < 100; i++ { + Sprintf("%x", i) + } + runtime.UpdateMemStats() + mallocs += runtime.MemStats.Mallocs + Printf("mallocs per Sprintf(\"%%x\"): %d\n", mallocs/100) + runtime.UpdateMemStats() + mallocs = 0 - runtime.MemStats.Mallocs + for i := 0; i < 100; i++ { + Sprintf("%x %x", i, i) + } + runtime.UpdateMemStats() + mallocs += runtime.MemStats.Mallocs + Printf("mallocs per Sprintf(\"%%x %%x\"): %d\n", mallocs/100) +} + +type flagPrinter struct{} + +func (*flagPrinter) Format(f State, c int) { + s := "%" + for i := 0; i < 128; i++ { + if f.Flag(i) { + s += string(i) + } + } + if w, ok := f.Width(); ok { + s += Sprintf("%d", w) + } + if p, ok := f.Precision(); ok { + s += Sprintf(".%d", p) + } + s += string(c) + io.WriteString(f, "["+s+"]") +} + +var flagtests = []struct { + in string + out string +}{ + {"%a", "[%a]"}, + {"%-a", "[%-a]"}, + {"%+a", "[%+a]"}, + {"%#a", "[%#a]"}, + {"% a", "[% a]"}, + {"%0a", "[%0a]"}, + {"%1.2a", "[%1.2a]"}, + {"%-1.2a", "[%-1.2a]"}, + {"%+1.2a", "[%+1.2a]"}, + {"%-+1.2a", "[%+-1.2a]"}, + {"%-+1.2abc", "[%+-1.2a]bc"}, + {"%-1.2abc", "[%-1.2a]bc"}, +} + +func TestFlagParser(t *testing.T) { + var flagprinter flagPrinter + for _, tt := range flagtests { + s := Sprintf(tt.in, &flagprinter) + if s != tt.out { + t.Errorf("Sprintf(%q, &flagprinter) => %q, want %q", tt.in, s, tt.out) + } + } +} + +func TestStructPrinter(t *testing.T) { + var s struct { + a string + b string + c int + } + s.a = "abc" + s.b = "def" + s.c = 123 + var tests = []struct { + fmt string + out string + }{ + {"%v", "{abc def 123}"}, + {"%+v", "{a:abc b:def c:123}"}, + } + for _, tt := range tests { + out := Sprintf(tt.fmt, s) + if out != tt.out { + t.Errorf("Sprintf(%q, &s) = %q, want %q", tt.fmt, out, tt.out) + } + } +} + +// Check map printing using substrings so we don't depend on the print order. +func presentInMap(s string, a []string, t *testing.T) { + for i := 0; i < len(a); i++ { + loc := strings.Index(s, a[i]) + if loc < 0 { + t.Errorf("map print: expected to find %q in %q", a[i], s) + } + // make sure the match ends here + loc += len(a[i]) + if loc >= len(s) || (s[loc] != ' ' && s[loc] != ']') { + t.Errorf("map print: %q not properly terminated in %q", a[i], s) + } + } +} + +func TestMapPrinter(t *testing.T) { + m0 := make(map[int]string) + s := Sprint(m0) + if s != "map[]" { + t.Errorf("empty map printed as %q not %q", s, "map[]") + } + m1 := map[int]string{1: "one", 2: "two", 3: "three"} + a := []string{"1:one", "2:two", "3:three"} + presentInMap(Sprintf("%v", m1), a, t) + presentInMap(Sprint(m1), a, t) +} + +func TestEmptyMap(t *testing.T) { + const emptyMapStr = "map[]" + var m map[string]int + s := Sprint(m) + if s != emptyMapStr { + t.Errorf("nil map printed as %q not %q", s, emptyMapStr) + } + m = make(map[string]int) + s = Sprint(m) + if s != emptyMapStr { + t.Errorf("empty map printed as %q not %q", s, emptyMapStr) + } +} + +// Check that Sprint (and hence Print, Fprint) puts spaces in the right places, +// that is, between arg pairs in which neither is a string. +func TestBlank(t *testing.T) { + got := Sprint("<", 1, ">:", 1, 2, 3, "!") + expect := "<1>:1 2 3!" + if got != expect { + t.Errorf("got %q expected %q", got, expect) + } +} + +// Check that Sprintln (and hence Println, Fprintln) puts spaces in the right places, +// that is, between all arg pairs. +func TestBlankln(t *testing.T) { + got := Sprintln("<", 1, ">:", 1, 2, 3, "!") + expect := "< 1 >: 1 2 3 !\n" + if got != expect { + t.Errorf("got %q expected %q", got, expect) + } +} + +// Check Formatter with Sprint, Sprintln, Sprintf +func TestFormatterPrintln(t *testing.T) { + f := F(1) + expect := "\n" + s := Sprint(f, "\n") + if s != expect { + t.Errorf("Sprint wrong with Formatter: expected %q got %q", expect, s) + } + s = Sprintln(f) + if s != expect { + t.Errorf("Sprintln wrong with Formatter: expected %q got %q", expect, s) + } + s = Sprintf("%v\n", f) + if s != expect { + t.Errorf("Sprintf wrong with Formatter: expected %q got %q", expect, s) + } +} + +func args(a ...interface{}) []interface{} { return a } + +var startests = []struct { + fmt string + in []interface{} + out string +}{ + {"%*d", args(4, 42), " 42"}, + {"%.*d", args(4, 42), "0042"}, + {"%*.*d", args(8, 4, 42), " 0042"}, + {"%0*d", args(4, 42), "0042"}, + {"%-*d", args(4, 42), "42 "}, + + // erroneous + {"%*d", args(nil, 42), "%!(BADWIDTH)42"}, + {"%.*d", args(nil, 42), "%!(BADPREC)42"}, + {"%*d", args(5, "foo"), "%!d(string= foo)"}, + {"%*% %d", args(20, 5), "% 5"}, + {"%*", args(4), "%!(NOVERB)"}, + {"%*d", args(int32(4), 42), "%!(BADWIDTH)42"}, +} + +func TestWidthAndPrecision(t *testing.T) { + for _, tt := range startests { + s := Sprintf(tt.fmt, tt.in...) + if s != tt.out { + t.Errorf("%q: got %q expected %q", tt.fmt, s, tt.out) + } + } +} + +// A type that panics in String. +type Panic struct { + message interface{} +} + +// Value receiver. +func (p Panic) GoString() string { + panic(p.message) +} + +// Value receiver. +func (p Panic) String() string { + panic(p.message) +} + +// A type that panics in Format. +type PanicF struct { + message interface{} +} + +// Value receiver. +func (p PanicF) Format(f State, c int) { + panic(p.message) +} + +var panictests = []struct { + fmt string + in interface{} + out string +}{ + // String + {"%d", (*Panic)(nil), ""}, // nil pointer special case + {"%d", Panic{io.ErrUnexpectedEOF}, "%d(PANIC=unexpected EOF)"}, + {"%d", Panic{3}, "%d(PANIC=3)"}, + // GoString + {"%#v", (*Panic)(nil), ""}, // nil pointer special case + {"%#v", Panic{io.ErrUnexpectedEOF}, "%v(PANIC=unexpected EOF)"}, + {"%#v", Panic{3}, "%v(PANIC=3)"}, + // Format + {"%s", (*PanicF)(nil), ""}, // nil pointer special case + {"%s", PanicF{io.ErrUnexpectedEOF}, "%s(PANIC=unexpected EOF)"}, + {"%s", PanicF{3}, "%s(PANIC=3)"}, +} + +func TestPanics(t *testing.T) { + for _, tt := range panictests { + s := Sprintf(tt.fmt, tt.in) + if s != tt.out { + t.Errorf("%q: got %q expected %q", tt.fmt, s, tt.out) + } + } +} diff --git a/src/pkg/fmt/format.go b/src/pkg/fmt/format.go new file mode 100644 index 000000000..24b15a286 --- /dev/null +++ b/src/pkg/fmt/format.go @@ -0,0 +1,452 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package fmt + +import ( + "bytes" + "strconv" + "unicode" + "utf8" +) + +const ( + nByte = 64 + + ldigits = "0123456789abcdef" + udigits = "0123456789ABCDEF" +) + +const ( + signed = true + unsigned = false +) + +var padZeroBytes = make([]byte, nByte) +var padSpaceBytes = make([]byte, nByte) + +var newline = []byte{'\n'} + +func init() { + for i := 0; i < nByte; i++ { + padZeroBytes[i] = '0' + padSpaceBytes[i] = ' ' + } +} + +// A fmt is the raw formatter used by Printf etc. +// It prints into a bytes.Buffer that must be set up externally. +type fmt struct { + intbuf [nByte]byte + buf *bytes.Buffer + // width, precision + wid int + prec int + // flags + widPresent bool + precPresent bool + minus bool + plus bool + sharp bool + space bool + unicode bool + uniQuote bool // Use 'x'= prefix for %U if printable. + zero bool +} + +func (f *fmt) clearflags() { + f.wid = 0 + f.widPresent = false + f.prec = 0 + f.precPresent = false + f.minus = false + f.plus = false + f.sharp = false + f.space = false + f.unicode = false + f.uniQuote = false + f.zero = false +} + +func (f *fmt) init(buf *bytes.Buffer) { + f.buf = buf + f.clearflags() +} + +// Compute left and right padding widths (only one will be non-zero). +func (f *fmt) computePadding(width int) (padding []byte, leftWidth, rightWidth int) { + left := !f.minus + w := f.wid + if w < 0 { + left = false + w = -w + } + w -= width + if w > 0 { + if left && f.zero { + return padZeroBytes, w, 0 + } + if left { + return padSpaceBytes, w, 0 + } else { + // can't be zero padding on the right + return padSpaceBytes, 0, w + } + } + return +} + +// Generate n bytes of padding. +func (f *fmt) writePadding(n int, padding []byte) { + for n > 0 { + m := n + if m > nByte { + m = nByte + } + f.buf.Write(padding[0:m]) + n -= m + } +} + +// Append b to f.buf, padded on left (w > 0) or right (w < 0 or f.minus) +// clear flags afterwards. +func (f *fmt) pad(b []byte) { + var padding []byte + var left, right int + if f.widPresent && f.wid != 0 { + padding, left, right = f.computePadding(len(b)) + } + if left > 0 { + f.writePadding(left, padding) + } + f.buf.Write(b) + if right > 0 { + f.writePadding(right, padding) + } +} + +// append s to buf, padded on left (w > 0) or right (w < 0 or f.minus). +// clear flags afterwards. +func (f *fmt) padString(s string) { + var padding []byte + var left, right int + if f.widPresent && f.wid != 0 { + padding, left, right = f.computePadding(utf8.RuneCountInString(s)) + } + if left > 0 { + f.writePadding(left, padding) + } + f.buf.WriteString(s) + if right > 0 { + f.writePadding(right, padding) + } +} + +func putint(buf []byte, base, val uint64, digits string) int { + i := len(buf) - 1 + for val >= base { + buf[i] = digits[val%base] + i-- + val /= base + } + buf[i] = digits[val] + return i - 1 +} + +// fmt_boolean formats a boolean. +func (f *fmt) fmt_boolean(v bool) { + if v { + f.padString("true") + } else { + f.padString("false") + } +} + +// integer; interprets prec but not wid. Once formatted, result is sent to pad() +// and then flags are cleared. +func (f *fmt) integer(a int64, base uint64, signedness bool, digits string) { + // precision of 0 and value of 0 means "print nothing" + if f.precPresent && f.prec == 0 && a == 0 { + return + } + + var buf []byte = f.intbuf[0:] + negative := signedness == signed && a < 0 + if negative { + a = -a + } + + // two ways to ask for extra leading zero digits: %.3d or %03d. + // apparently the first cancels the second. + prec := 0 + if f.precPresent { + prec = f.prec + f.zero = false + } else if f.zero && f.widPresent && !f.minus && f.wid > 0 { + prec = f.wid + if negative || f.plus || f.space { + prec-- // leave room for sign + } + } + + // format a into buf, ending at buf[i]. (printing is easier right-to-left.) + // a is made into unsigned ua. we could make things + // marginally faster by splitting the 32-bit case out into a separate + // block but it's not worth the duplication, so ua has 64 bits. + i := len(f.intbuf) + ua := uint64(a) + for ua >= base { + i-- + buf[i] = digits[ua%base] + ua /= base + } + i-- + buf[i] = digits[ua] + for i > 0 && prec > nByte-i { + i-- + buf[i] = '0' + } + + // Various prefixes: 0x, -, etc. + if f.sharp { + switch base { + case 8: + if buf[i] != '0' { + i-- + buf[i] = '0' + } + case 16: + i-- + buf[i] = 'x' + digits[10] - 'a' + i-- + buf[i] = '0' + } + } + if f.unicode { + i-- + buf[i] = '+' + i-- + buf[i] = 'U' + } + + if negative { + i-- + buf[i] = '-' + } else if f.plus { + i-- + buf[i] = '+' + } else if f.space { + i-- + buf[i] = ' ' + } + + // If we want a quoted char for %#U, move the data up to make room. + if f.unicode && f.uniQuote && a >= 0 && a <= unicode.MaxRune && unicode.IsPrint(int(a)) { + runeWidth := utf8.RuneLen(int(a)) + width := 1 + 1 + runeWidth + 1 // space, quote, rune, quote + copy(buf[i-width:], buf[i:]) // guaranteed to have enough room. + i -= width + // Now put " 'x'" at the end. + j := len(buf) - width + buf[j] = ' ' + j++ + buf[j] = '\'' + j++ + utf8.EncodeRune(buf[j:], int(a)) + j += runeWidth + buf[j] = '\'' + } + + f.pad(buf[i:]) +} + +// truncate truncates the string to the specified precision, if present. +func (f *fmt) truncate(s string) string { + if f.precPresent && f.prec < utf8.RuneCountInString(s) { + n := f.prec + for i := range s { + if n == 0 { + s = s[:i] + break + } + n-- + } + } + return s +} + +// fmt_s formats a string. +func (f *fmt) fmt_s(s string) { + s = f.truncate(s) + f.padString(s) +} + +// fmt_sx formats a string as a hexadecimal encoding of its bytes. +func (f *fmt) fmt_sx(s string) { + t := "" + for i := 0; i < len(s); i++ { + if i > 0 && f.space { + t += " " + } + v := s[i] + t += string(ldigits[v>>4]) + t += string(ldigits[v&0xF]) + } + f.padString(t) +} + +// fmt_sX formats a string as an uppercase hexadecimal encoding of its bytes. +func (f *fmt) fmt_sX(s string) { + t := "" + for i := 0; i < len(s); i++ { + if i > 0 && f.space { + t += " " + } + v := s[i] + t += string(udigits[v>>4]) + t += string(udigits[v&0xF]) + } + f.padString(t) +} + +// fmt_q formats a string as a double-quoted, escaped Go string constant. +func (f *fmt) fmt_q(s string) { + s = f.truncate(s) + var quoted string + if f.sharp && strconv.CanBackquote(s) { + quoted = "`" + s + "`" + } else { + if f.plus { + quoted = strconv.QuoteToASCII(s) + } else { + quoted = strconv.Quote(s) + } + } + f.padString(quoted) +} + +// fmt_qc formats the integer as a single-quoted, escaped Go character constant. +// If the character is not valid Unicode, it will print '\ufffd'. +func (f *fmt) fmt_qc(c int64) { + var quoted string + if f.plus { + quoted = strconv.QuoteRuneToASCII(int(c)) + } else { + quoted = strconv.QuoteRune(int(c)) + } + f.padString(quoted) +} + +// floating-point + +func doPrec(f *fmt, def int) int { + if f.precPresent { + return f.prec + } + return def +} + +// Add a plus sign or space to the floating-point string representation if missing and required. +func (f *fmt) plusSpace(s string) { + if s[0] != '-' { + if f.plus { + s = "+" + s + } else if f.space { + s = " " + s + } + } + f.padString(s) +} + +// fmt_e64 formats a float64 in the form -1.23e+12. +func (f *fmt) fmt_e64(v float64) { f.plusSpace(strconv.Ftoa64(v, 'e', doPrec(f, 6))) } + +// fmt_E64 formats a float64 in the form -1.23E+12. +func (f *fmt) fmt_E64(v float64) { f.plusSpace(strconv.Ftoa64(v, 'E', doPrec(f, 6))) } + +// fmt_f64 formats a float64 in the form -1.23. +func (f *fmt) fmt_f64(v float64) { f.plusSpace(strconv.Ftoa64(v, 'f', doPrec(f, 6))) } + +// fmt_g64 formats a float64 in the 'f' or 'e' form according to size. +func (f *fmt) fmt_g64(v float64) { f.plusSpace(strconv.Ftoa64(v, 'g', doPrec(f, -1))) } + +// fmt_g64 formats a float64 in the 'f' or 'E' form according to size. +func (f *fmt) fmt_G64(v float64) { f.plusSpace(strconv.Ftoa64(v, 'G', doPrec(f, -1))) } + +// fmt_fb64 formats a float64 in the form -123p3 (exponent is power of 2). +func (f *fmt) fmt_fb64(v float64) { f.plusSpace(strconv.Ftoa64(v, 'b', 0)) } + +// float32 +// cannot defer to float64 versions +// because it will get rounding wrong in corner cases. + +// fmt_e32 formats a float32 in the form -1.23e+12. +func (f *fmt) fmt_e32(v float32) { f.plusSpace(strconv.Ftoa32(v, 'e', doPrec(f, 6))) } + +// fmt_E32 formats a float32 in the form -1.23E+12. +func (f *fmt) fmt_E32(v float32) { f.plusSpace(strconv.Ftoa32(v, 'E', doPrec(f, 6))) } + +// fmt_f32 formats a float32 in the form -1.23. +func (f *fmt) fmt_f32(v float32) { f.plusSpace(strconv.Ftoa32(v, 'f', doPrec(f, 6))) } + +// fmt_g32 formats a float32 in the 'f' or 'e' form according to size. +func (f *fmt) fmt_g32(v float32) { f.plusSpace(strconv.Ftoa32(v, 'g', doPrec(f, -1))) } + +// fmt_G32 formats a float32 in the 'f' or 'E' form according to size. +func (f *fmt) fmt_G32(v float32) { f.plusSpace(strconv.Ftoa32(v, 'G', doPrec(f, -1))) } + +// fmt_fb32 formats a float32 in the form -123p3 (exponent is power of 2). +func (f *fmt) fmt_fb32(v float32) { f.padString(strconv.Ftoa32(v, 'b', 0)) } + +// fmt_c64 formats a complex64 according to the verb. +func (f *fmt) fmt_c64(v complex64, verb int) { + f.buf.WriteByte('(') + r := real(v) + for i := 0; ; i++ { + switch verb { + case 'e': + f.fmt_e32(r) + case 'E': + f.fmt_E32(r) + case 'f': + f.fmt_f32(r) + case 'g': + f.fmt_g32(r) + case 'G': + f.fmt_G32(r) + } + if i != 0 { + break + } + f.plus = true + r = imag(v) + } + f.buf.Write(irparenBytes) +} + +// fmt_c128 formats a complex128 according to the verb. +func (f *fmt) fmt_c128(v complex128, verb int) { + f.buf.WriteByte('(') + r := real(v) + for i := 0; ; i++ { + switch verb { + case 'e': + f.fmt_e64(r) + case 'E': + f.fmt_E64(r) + case 'f': + f.fmt_f64(r) + case 'g': + f.fmt_g64(r) + case 'G': + f.fmt_G64(r) + } + if i != 0 { + break + } + f.plus = true + r = imag(v) + } + f.buf.Write(irparenBytes) +} diff --git a/src/pkg/fmt/print.go b/src/pkg/fmt/print.go new file mode 100644 index 000000000..738734908 --- /dev/null +++ b/src/pkg/fmt/print.go @@ -0,0 +1,996 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package fmt + +import ( + "bytes" + "io" + "os" + "reflect" + "unicode" + "utf8" +) + +// Some constants in the form of bytes, to avoid string overhead. +// Needlessly fastidious, I suppose. +var ( + commaSpaceBytes = []byte(", ") + nilAngleBytes = []byte("") + nilParenBytes = []byte("(nil)") + nilBytes = []byte("nil") + mapBytes = []byte("map[") + missingBytes = []byte("(MISSING)") + panicBytes = []byte("(PANIC=") + extraBytes = []byte("%!(EXTRA ") + irparenBytes = []byte("i)") + bytesBytes = []byte("[]byte{") + widthBytes = []byte("%!(BADWIDTH)") + precBytes = []byte("%!(BADPREC)") + noVerbBytes = []byte("%!(NOVERB)") +) + +// State represents the printer state passed to custom formatters. +// It provides access to the io.Writer interface plus information about +// the flags and options for the operand's format specifier. +type State interface { + // Write is the function to call to emit formatted output to be printed. + Write(b []byte) (ret int, err os.Error) + // Width returns the value of the width option and whether it has been set. + Width() (wid int, ok bool) + // Precision returns the value of the precision option and whether it has been set. + Precision() (prec int, ok bool) + + // Flag returns whether the flag c, a character, has been set. + Flag(c int) bool +} + +// Formatter is the interface implemented by values with a custom formatter. +// The implementation of Format may call Sprintf or Fprintf(f) etc. +// to generate its output. +type Formatter interface { + Format(f State, c int) +} + +// Stringer is implemented by any value that has a String method, +// which defines the ``native'' format for that value. +// The String method is used to print values passed as an operand +// to a %s or %v format or to an unformatted printer such as Print. +type Stringer interface { + String() string +} + +// GoStringer is implemented by any value that has a GoString method, +// which defines the Go syntax for that value. +// The GoString method is used to print values passed as an operand +// to a %#v format. +type GoStringer interface { + GoString() string +} + +type pp struct { + n int + panicking bool + buf bytes.Buffer + runeBuf [utf8.UTFMax]byte + fmt fmt +} + +// A cache holds a set of reusable objects. +// The buffered channel holds the currently available objects. +// If more are needed, the cache creates them by calling new. +type cache struct { + saved chan interface{} + new func() interface{} +} + +func (c *cache) put(x interface{}) { + select { + case c.saved <- x: + // saved in cache + default: + // discard + } +} + +func (c *cache) get() interface{} { + select { + case x := <-c.saved: + return x // reused from cache + default: + return c.new() + } + panic("not reached") +} + +func newCache(f func() interface{}) *cache { + return &cache{make(chan interface{}, 100), f} +} + +var ppFree = newCache(func() interface{} { return new(pp) }) + +// Allocate a new pp struct or grab a cached one. +func newPrinter() *pp { + p := ppFree.get().(*pp) + p.panicking = false + p.fmt.init(&p.buf) + return p +} + +// Save used pp structs in ppFree; avoids an allocation per invocation. +func (p *pp) free() { + // Don't hold on to pp structs with large buffers. + if cap(p.buf.Bytes()) > 1024 { + return + } + p.buf.Reset() + ppFree.put(p) +} + +func (p *pp) Width() (wid int, ok bool) { return p.fmt.wid, p.fmt.widPresent } + +func (p *pp) Precision() (prec int, ok bool) { return p.fmt.prec, p.fmt.precPresent } + +func (p *pp) Flag(b int) bool { + switch b { + case '-': + return p.fmt.minus + case '+': + return p.fmt.plus + case '#': + return p.fmt.sharp + case ' ': + return p.fmt.space + case '0': + return p.fmt.zero + } + return false +} + +func (p *pp) add(c int) { + p.buf.WriteRune(c) +} + +// Implement Write so we can call Fprintf on a pp (through State), for +// recursive use in custom verbs. +func (p *pp) Write(b []byte) (ret int, err os.Error) { + return p.buf.Write(b) +} + +// These routines end in 'f' and take a format string. + +// Fprintf formats according to a format specifier and writes to w. +// It returns the number of bytes written and any write error encountered. +func Fprintf(w io.Writer, format string, a ...interface{}) (n int, err os.Error) { + p := newPrinter() + p.doPrintf(format, a) + n64, err := p.buf.WriteTo(w) + p.free() + return int(n64), err +} + +// Printf formats according to a format specifier and writes to standard output. +// It returns the number of bytes written and any write error encountered. +func Printf(format string, a ...interface{}) (n int, err os.Error) { + return Fprintf(os.Stdout, format, a...) +} + +// Sprintf formats according to a format specifier and returns the resulting string. +func Sprintf(format string, a ...interface{}) string { + p := newPrinter() + p.doPrintf(format, a) + s := p.buf.String() + p.free() + return s +} + +// Errorf formats according to a format specifier and returns the string +// converted to an os.ErrorString, which satisfies the os.Error interface. +func Errorf(format string, a ...interface{}) os.Error { + return os.NewError(Sprintf(format, a...)) +} + +// These routines do not take a format string + +// Fprint formats using the default formats for its operands and writes to w. +// Spaces are added between operands when neither is a string. +// It returns the number of bytes written and any write error encountered. +func Fprint(w io.Writer, a ...interface{}) (n int, err os.Error) { + p := newPrinter() + p.doPrint(a, false, false) + n64, err := p.buf.WriteTo(w) + p.free() + return int(n64), err +} + +// Print formats using the default formats for its operands and writes to standard output. +// Spaces are added between operands when neither is a string. +// It returns the number of bytes written and any write error encountered. +func Print(a ...interface{}) (n int, err os.Error) { + return Fprint(os.Stdout, a...) +} + +// Sprint formats using the default formats for its operands and returns the resulting string. +// Spaces are added between operands when neither is a string. +func Sprint(a ...interface{}) string { + p := newPrinter() + p.doPrint(a, false, false) + s := p.buf.String() + p.free() + return s +} + +// These routines end in 'ln', do not take a format string, +// always add spaces between operands, and add a newline +// after the last operand. + +// Fprintln formats using the default formats for its operands and writes to w. +// Spaces are always added between operands and a newline is appended. +// It returns the number of bytes written and any write error encountered. +func Fprintln(w io.Writer, a ...interface{}) (n int, err os.Error) { + p := newPrinter() + p.doPrint(a, true, true) + n64, err := p.buf.WriteTo(w) + p.free() + return int(n64), err +} + +// Println formats using the default formats for its operands and writes to standard output. +// Spaces are always added between operands and a newline is appended. +// It returns the number of bytes written and any write error encountered. +func Println(a ...interface{}) (n int, err os.Error) { + return Fprintln(os.Stdout, a...) +} + +// Sprintln formats using the default formats for its operands and returns the resulting string. +// Spaces are always added between operands and a newline is appended. +func Sprintln(a ...interface{}) string { + p := newPrinter() + p.doPrint(a, true, true) + s := p.buf.String() + p.free() + return s +} + +// Get the i'th arg of the struct value. +// If the arg itself is an interface, return a value for +// the thing inside the interface, not the interface itself. +func getField(v reflect.Value, i int) reflect.Value { + val := v.Field(i) + if i := val; i.Kind() == reflect.Interface { + if inter := i.Interface(); inter != nil { + return reflect.ValueOf(inter) + } + } + return val +} + +// Convert ASCII to integer. n is 0 (and got is false) if no number present. +func parsenum(s string, start, end int) (num int, isnum bool, newi int) { + if start >= end { + return 0, false, end + } + for newi = start; newi < end && '0' <= s[newi] && s[newi] <= '9'; newi++ { + num = num*10 + int(s[newi]-'0') + isnum = true + } + return +} + +func (p *pp) unknownType(v interface{}) { + if v == nil { + p.buf.Write(nilAngleBytes) + return + } + p.buf.WriteByte('?') + p.buf.WriteString(reflect.TypeOf(v).String()) + p.buf.WriteByte('?') +} + +func (p *pp) badVerb(verb int, val interface{}) { + p.add('%') + p.add('!') + p.add(verb) + p.add('(') + if val == nil { + p.buf.Write(nilAngleBytes) + } else { + p.buf.WriteString(reflect.TypeOf(val).String()) + p.add('=') + p.printField(val, 'v', false, false, 0) + } + p.add(')') +} + +func (p *pp) fmtBool(v bool, verb int, value interface{}) { + switch verb { + case 't', 'v': + p.fmt.fmt_boolean(v) + default: + p.badVerb(verb, value) + } +} + +// fmtC formats a rune for the 'c' format. +func (p *pp) fmtC(c int64) { + rune := int(c) // Check for overflow. + if int64(rune) != c { + rune = utf8.RuneError + } + w := utf8.EncodeRune(p.runeBuf[0:utf8.UTFMax], rune) + p.fmt.pad(p.runeBuf[0:w]) +} + +func (p *pp) fmtInt64(v int64, verb int, value interface{}) { + switch verb { + case 'b': + p.fmt.integer(v, 2, signed, ldigits) + case 'c': + p.fmtC(v) + case 'd', 'v': + p.fmt.integer(v, 10, signed, ldigits) + case 'o': + p.fmt.integer(v, 8, signed, ldigits) + case 'q': + if 0 <= v && v <= unicode.MaxRune { + p.fmt.fmt_qc(v) + } else { + p.badVerb(verb, value) + } + case 'x': + p.fmt.integer(v, 16, signed, ldigits) + case 'U': + p.fmtUnicode(v) + case 'X': + p.fmt.integer(v, 16, signed, udigits) + default: + p.badVerb(verb, value) + } +} + +// fmt0x64 formats a uint64 in hexadecimal and prefixes it with 0x or +// not, as requested, by temporarily setting the sharp flag. +func (p *pp) fmt0x64(v uint64, leading0x bool) { + sharp := p.fmt.sharp + p.fmt.sharp = leading0x + p.fmt.integer(int64(v), 16, unsigned, ldigits) + p.fmt.sharp = sharp +} + +// fmtUnicode formats a uint64 in U+1234 form by +// temporarily turning on the unicode flag and tweaking the precision. +func (p *pp) fmtUnicode(v int64) { + precPresent := p.fmt.precPresent + sharp := p.fmt.sharp + p.fmt.sharp = false + prec := p.fmt.prec + if !precPresent { + // If prec is already set, leave it alone; otherwise 4 is minimum. + p.fmt.prec = 4 + p.fmt.precPresent = true + } + p.fmt.unicode = true // turn on U+ + p.fmt.uniQuote = sharp + p.fmt.integer(int64(v), 16, unsigned, udigits) + p.fmt.unicode = false + p.fmt.uniQuote = false + p.fmt.prec = prec + p.fmt.precPresent = precPresent + p.fmt.sharp = sharp +} + +func (p *pp) fmtUint64(v uint64, verb int, goSyntax bool, value interface{}) { + switch verb { + case 'b': + p.fmt.integer(int64(v), 2, unsigned, ldigits) + case 'c': + p.fmtC(int64(v)) + case 'd': + p.fmt.integer(int64(v), 10, unsigned, ldigits) + case 'v': + if goSyntax { + p.fmt0x64(v, true) + } else { + p.fmt.integer(int64(v), 10, unsigned, ldigits) + } + case 'o': + p.fmt.integer(int64(v), 8, unsigned, ldigits) + case 'q': + if 0 <= v && v <= unicode.MaxRune { + p.fmt.fmt_qc(int64(v)) + } else { + p.badVerb(verb, value) + } + case 'x': + p.fmt.integer(int64(v), 16, unsigned, ldigits) + case 'X': + p.fmt.integer(int64(v), 16, unsigned, udigits) + case 'U': + p.fmtUnicode(int64(v)) + default: + p.badVerb(verb, value) + } +} + +func (p *pp) fmtFloat32(v float32, verb int, value interface{}) { + switch verb { + case 'b': + p.fmt.fmt_fb32(v) + case 'e': + p.fmt.fmt_e32(v) + case 'E': + p.fmt.fmt_E32(v) + case 'f': + p.fmt.fmt_f32(v) + case 'g', 'v': + p.fmt.fmt_g32(v) + case 'G': + p.fmt.fmt_G32(v) + default: + p.badVerb(verb, value) + } +} + +func (p *pp) fmtFloat64(v float64, verb int, value interface{}) { + switch verb { + case 'b': + p.fmt.fmt_fb64(v) + case 'e': + p.fmt.fmt_e64(v) + case 'E': + p.fmt.fmt_E64(v) + case 'f': + p.fmt.fmt_f64(v) + case 'g', 'v': + p.fmt.fmt_g64(v) + case 'G': + p.fmt.fmt_G64(v) + default: + p.badVerb(verb, value) + } +} + +func (p *pp) fmtComplex64(v complex64, verb int, value interface{}) { + switch verb { + case 'e', 'E', 'f', 'F', 'g', 'G': + p.fmt.fmt_c64(v, verb) + case 'v': + p.fmt.fmt_c64(v, 'g') + default: + p.badVerb(verb, value) + } +} + +func (p *pp) fmtComplex128(v complex128, verb int, value interface{}) { + switch verb { + case 'e', 'E', 'f', 'F', 'g', 'G': + p.fmt.fmt_c128(v, verb) + case 'v': + p.fmt.fmt_c128(v, 'g') + default: + p.badVerb(verb, value) + } +} + +func (p *pp) fmtString(v string, verb int, goSyntax bool, value interface{}) { + switch verb { + case 'v': + if goSyntax { + p.fmt.fmt_q(v) + } else { + p.fmt.fmt_s(v) + } + case 's': + p.fmt.fmt_s(v) + case 'x': + p.fmt.fmt_sx(v) + case 'X': + p.fmt.fmt_sX(v) + case 'q': + p.fmt.fmt_q(v) + default: + p.badVerb(verb, value) + } +} + +func (p *pp) fmtBytes(v []byte, verb int, goSyntax bool, depth int, value interface{}) { + if verb == 'v' || verb == 'd' { + if goSyntax { + p.buf.Write(bytesBytes) + } else { + p.buf.WriteByte('[') + } + for i, c := range v { + if i > 0 { + if goSyntax { + p.buf.Write(commaSpaceBytes) + } else { + p.buf.WriteByte(' ') + } + } + p.printField(c, 'v', p.fmt.plus, goSyntax, depth+1) + } + if goSyntax { + p.buf.WriteByte('}') + } else { + p.buf.WriteByte(']') + } + return + } + s := string(v) + switch verb { + case 's': + p.fmt.fmt_s(s) + case 'x': + p.fmt.fmt_sx(s) + case 'X': + p.fmt.fmt_sX(s) + case 'q': + p.fmt.fmt_q(s) + default: + p.badVerb(verb, value) + } +} + +func (p *pp) fmtPointer(field interface{}, value reflect.Value, verb int, goSyntax bool) { + var u uintptr + switch value.Kind() { + case reflect.Chan, reflect.Func, reflect.Map, reflect.Ptr, reflect.Slice, reflect.UnsafePointer: + u = value.Pointer() + default: + p.badVerb(verb, field) + return + } + if goSyntax { + p.add('(') + p.buf.WriteString(reflect.TypeOf(field).String()) + p.add(')') + p.add('(') + if u == 0 { + p.buf.Write(nilBytes) + } else { + p.fmt0x64(uint64(u), true) + } + p.add(')') + } else { + p.fmt0x64(uint64(u), !p.fmt.sharp) + } +} + +var ( + intBits = reflect.TypeOf(0).Bits() + floatBits = reflect.TypeOf(0.0).Bits() + complexBits = reflect.TypeOf(1i).Bits() + uintptrBits = reflect.TypeOf(uintptr(0)).Bits() +) + +func (p *pp) catchPanic(val interface{}, verb int) { + if err := recover(); err != nil { + // If it's a nil pointer, just say "". The likeliest causes are a + // Stringer that fails to guard against nil or a nil pointer for a + // value receiver, and in either case, "" is a nice result. + if v := reflect.ValueOf(val); v.Kind() == reflect.Ptr && v.IsNil() { + p.buf.Write(nilAngleBytes) + return + } + // Otherwise print a concise panic message. Most of the time the panic + // value will print itself nicely. + if p.panicking { + // Nested panics; the recursion in printField cannot succeed. + panic(err) + } + p.buf.WriteByte('%') + p.add(verb) + p.buf.Write(panicBytes) + p.panicking = true + p.printField(err, 'v', false, false, 0) + p.panicking = false + p.buf.WriteByte(')') + } +} + +func (p *pp) printField(field interface{}, verb int, plus, goSyntax bool, depth int) (wasString bool) { + if field == nil { + if verb == 'T' || verb == 'v' { + p.buf.Write(nilAngleBytes) + } else { + p.badVerb(verb, field) + } + return false + } + + // Special processing considerations. + // %T (the value's type) and %p (its address) are special; we always do them first. + switch verb { + case 'T': + p.printField(reflect.TypeOf(field).String(), 's', false, false, 0) + return false + case 'p': + p.fmtPointer(field, reflect.ValueOf(field), verb, goSyntax) + return false + } + // Is it a Formatter? + if formatter, ok := field.(Formatter); ok { + defer p.catchPanic(field, verb) + formatter.Format(p, verb) + return false // this value is not a string + + } + // Must not touch flags before Formatter looks at them. + if plus { + p.fmt.plus = false + } + // If we're doing Go syntax and the field knows how to supply it, take care of it now. + if goSyntax { + p.fmt.sharp = false + if stringer, ok := field.(GoStringer); ok { + defer p.catchPanic(field, verb) + // Print the result of GoString unadorned. + p.fmtString(stringer.GoString(), 's', false, field) + return false // this value is not a string + } + } else { + // Is it a Stringer? + if stringer, ok := field.(Stringer); ok { + defer p.catchPanic(field, verb) + p.printField(stringer.String(), verb, plus, false, depth) + return false // this value is not a string + } + } + + // Some types can be done without reflection. + switch f := field.(type) { + case bool: + p.fmtBool(f, verb, field) + return false + case float32: + p.fmtFloat32(f, verb, field) + return false + case float64: + p.fmtFloat64(f, verb, field) + return false + case complex64: + p.fmtComplex64(complex64(f), verb, field) + return false + case complex128: + p.fmtComplex128(f, verb, field) + return false + case int: + p.fmtInt64(int64(f), verb, field) + return false + case int8: + p.fmtInt64(int64(f), verb, field) + return false + case int16: + p.fmtInt64(int64(f), verb, field) + return false + case int32: + p.fmtInt64(int64(f), verb, field) + return false + case int64: + p.fmtInt64(f, verb, field) + return false + case uint: + p.fmtUint64(uint64(f), verb, goSyntax, field) + return false + case uint8: + p.fmtUint64(uint64(f), verb, goSyntax, field) + return false + case uint16: + p.fmtUint64(uint64(f), verb, goSyntax, field) + return false + case uint32: + p.fmtUint64(uint64(f), verb, goSyntax, field) + return false + case uint64: + p.fmtUint64(f, verb, goSyntax, field) + return false + case uintptr: + p.fmtUint64(uint64(f), verb, goSyntax, field) + return false + case string: + p.fmtString(f, verb, goSyntax, field) + return verb == 's' || verb == 'v' + case []byte: + p.fmtBytes(f, verb, goSyntax, depth, field) + return verb == 's' + } + + // Need to use reflection + value := reflect.ValueOf(field) + +BigSwitch: + switch f := value; f.Kind() { + case reflect.Bool: + p.fmtBool(f.Bool(), verb, field) + case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64: + p.fmtInt64(f.Int(), verb, field) + case reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64, reflect.Uintptr: + p.fmtUint64(uint64(f.Uint()), verb, goSyntax, field) + case reflect.Float32, reflect.Float64: + if f.Type().Size() == 4 { + p.fmtFloat32(float32(f.Float()), verb, field) + } else { + p.fmtFloat64(float64(f.Float()), verb, field) + } + case reflect.Complex64, reflect.Complex128: + if f.Type().Size() == 8 { + p.fmtComplex64(complex64(f.Complex()), verb, field) + } else { + p.fmtComplex128(complex128(f.Complex()), verb, field) + } + case reflect.String: + p.fmtString(f.String(), verb, goSyntax, field) + case reflect.Map: + if goSyntax { + p.buf.WriteString(f.Type().String()) + p.buf.WriteByte('{') + } else { + p.buf.Write(mapBytes) + } + keys := f.MapKeys() + for i, key := range keys { + if i > 0 { + if goSyntax { + p.buf.Write(commaSpaceBytes) + } else { + p.buf.WriteByte(' ') + } + } + p.printField(key.Interface(), verb, plus, goSyntax, depth+1) + p.buf.WriteByte(':') + p.printField(f.MapIndex(key).Interface(), verb, plus, goSyntax, depth+1) + } + if goSyntax { + p.buf.WriteByte('}') + } else { + p.buf.WriteByte(']') + } + case reflect.Struct: + if goSyntax { + p.buf.WriteString(reflect.TypeOf(field).String()) + } + p.add('{') + v := f + t := v.Type() + for i := 0; i < v.NumField(); i++ { + if i > 0 { + if goSyntax { + p.buf.Write(commaSpaceBytes) + } else { + p.buf.WriteByte(' ') + } + } + if plus || goSyntax { + if f := t.Field(i); f.Name != "" { + p.buf.WriteString(f.Name) + p.buf.WriteByte(':') + } + } + p.printField(getField(v, i).Interface(), verb, plus, goSyntax, depth+1) + } + p.buf.WriteByte('}') + case reflect.Interface: + value := f.Elem() + if !value.IsValid() { + if goSyntax { + p.buf.WriteString(reflect.TypeOf(field).String()) + p.buf.Write(nilParenBytes) + } else { + p.buf.Write(nilAngleBytes) + } + } else { + return p.printField(value.Interface(), verb, plus, goSyntax, depth+1) + } + case reflect.Array, reflect.Slice: + // Byte slices are special. + if f.Type().Elem().Kind() == reflect.Uint8 { + // We know it's a slice of bytes, but we also know it does not have static type + // []byte, or it would have been caught above. Therefore we cannot convert + // it directly in the (slightly) obvious way: f.Interface().([]byte); it doesn't have + // that type, and we can't write an expression of the right type and do a + // conversion because we don't have a static way to write the right type. + // So we build a slice by hand. This is a rare case but it would be nice + // if reflection could help a little more. + bytes := make([]byte, f.Len()) + for i := range bytes { + bytes[i] = byte(f.Index(i).Uint()) + } + p.fmtBytes(bytes, verb, goSyntax, depth, field) + return verb == 's' + } + if goSyntax { + p.buf.WriteString(reflect.TypeOf(field).String()) + p.buf.WriteByte('{') + } else { + p.buf.WriteByte('[') + } + for i := 0; i < f.Len(); i++ { + if i > 0 { + if goSyntax { + p.buf.Write(commaSpaceBytes) + } else { + p.buf.WriteByte(' ') + } + } + p.printField(f.Index(i).Interface(), verb, plus, goSyntax, depth+1) + } + if goSyntax { + p.buf.WriteByte('}') + } else { + p.buf.WriteByte(']') + } + case reflect.Ptr: + v := f.Pointer() + // pointer to array or slice or struct? ok at top level + // but not embedded (avoid loops) + if v != 0 && depth == 0 { + switch a := f.Elem(); a.Kind() { + case reflect.Array, reflect.Slice: + p.buf.WriteByte('&') + p.printField(a.Interface(), verb, plus, goSyntax, depth+1) + break BigSwitch + case reflect.Struct: + p.buf.WriteByte('&') + p.printField(a.Interface(), verb, plus, goSyntax, depth+1) + break BigSwitch + } + } + if goSyntax { + p.buf.WriteByte('(') + p.buf.WriteString(reflect.TypeOf(field).String()) + p.buf.WriteByte(')') + p.buf.WriteByte('(') + if v == 0 { + p.buf.Write(nilBytes) + } else { + p.fmt0x64(uint64(v), true) + } + p.buf.WriteByte(')') + break + } + if v == 0 { + p.buf.Write(nilAngleBytes) + break + } + p.fmt0x64(uint64(v), true) + case reflect.Chan, reflect.Func, reflect.UnsafePointer: + p.fmtPointer(field, value, verb, goSyntax) + default: + p.unknownType(f) + } + return false +} + +// intFromArg gets the fieldnumth element of a. On return, isInt reports whether the argument has type int. +func intFromArg(a []interface{}, end, i, fieldnum int) (num int, isInt bool, newi, newfieldnum int) { + newi, newfieldnum = end, fieldnum + if i < end && fieldnum < len(a) { + num, isInt = a[fieldnum].(int) + newi, newfieldnum = i+1, fieldnum+1 + } + return +} + +func (p *pp) doPrintf(format string, a []interface{}) { + end := len(format) + fieldnum := 0 // we process one field per non-trivial format + for i := 0; i < end; { + lasti := i + for i < end && format[i] != '%' { + i++ + } + if i > lasti { + p.buf.WriteString(format[lasti:i]) + } + if i >= end { + // done processing format string + break + } + + // Process one verb + i++ + // flags and widths + p.fmt.clearflags() + F: + for ; i < end; i++ { + switch format[i] { + case '#': + p.fmt.sharp = true + case '0': + p.fmt.zero = true + case '+': + p.fmt.plus = true + case '-': + p.fmt.minus = true + case ' ': + p.fmt.space = true + default: + break F + } + } + // do we have width? + if i < end && format[i] == '*' { + p.fmt.wid, p.fmt.widPresent, i, fieldnum = intFromArg(a, end, i, fieldnum) + if !p.fmt.widPresent { + p.buf.Write(widthBytes) + } + } else { + p.fmt.wid, p.fmt.widPresent, i = parsenum(format, i, end) + } + // do we have precision? + if i < end && format[i] == '.' { + if format[i+1] == '*' { + p.fmt.prec, p.fmt.precPresent, i, fieldnum = intFromArg(a, end, i+1, fieldnum) + if !p.fmt.precPresent { + p.buf.Write(precBytes) + } + } else { + p.fmt.prec, p.fmt.precPresent, i = parsenum(format, i+1, end) + if !p.fmt.precPresent { + p.fmt.prec = 0 + p.fmt.precPresent = true + } + } + } + if i >= end { + p.buf.Write(noVerbBytes) + continue + } + c, w := utf8.DecodeRuneInString(format[i:]) + i += w + // percent is special - absorbs no operand + if c == '%' { + p.buf.WriteByte('%') // We ignore width and prec. + continue + } + if fieldnum >= len(a) { // out of operands + p.buf.WriteByte('%') + p.add(c) + p.buf.Write(missingBytes) + continue + } + field := a[fieldnum] + fieldnum++ + + goSyntax := c == 'v' && p.fmt.sharp + plus := c == 'v' && p.fmt.plus + p.printField(field, c, plus, goSyntax, 0) + } + + if fieldnum < len(a) { + p.buf.Write(extraBytes) + for ; fieldnum < len(a); fieldnum++ { + field := a[fieldnum] + if field != nil { + p.buf.WriteString(reflect.TypeOf(field).String()) + p.buf.WriteByte('=') + } + p.printField(field, 'v', false, false, 0) + if fieldnum+1 < len(a) { + p.buf.Write(commaSpaceBytes) + } + } + p.buf.WriteByte(')') + } +} + +func (p *pp) doPrint(a []interface{}, addspace, addnewline bool) { + prevString := false + for fieldnum := 0; fieldnum < len(a); fieldnum++ { + p.fmt.clearflags() + // always add spaces if we're doing println + field := a[fieldnum] + if fieldnum > 0 { + isString := field != nil && reflect.TypeOf(field).Kind() == reflect.String + if addspace || !isString && !prevString { + p.buf.WriteByte(' ') + } + } + prevString = p.printField(field, 'v', false, false, 0) + } + if addnewline { + p.buf.WriteByte('\n') + } +} diff --git a/src/pkg/fmt/scan.go b/src/pkg/fmt/scan.go new file mode 100644 index 000000000..259451d02 --- /dev/null +++ b/src/pkg/fmt/scan.go @@ -0,0 +1,1112 @@ +// Copyright 2010 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package fmt + +import ( + "bytes" + "io" + "math" + "os" + "reflect" + "strconv" + "strings" + "unicode" + "utf8" +) + +// runeUnreader is the interface to something that can unread runes. +// If the object provided to Scan does not satisfy this interface, +// a local buffer will be used to back up the input, but its contents +// will be lost when Scan returns. +type runeUnreader interface { + UnreadRune() os.Error +} + +// ScanState represents the scanner state passed to custom scanners. +// Scanners may do rune-at-a-time scanning or ask the ScanState +// to discover the next space-delimited token. +type ScanState interface { + // ReadRune reads the next rune (Unicode code point) from the input. + // If invoked during Scanln, Fscanln, or Sscanln, ReadRune() will + // return EOF after returning the first '\n' or when reading beyond + // the specified width. + ReadRune() (rune int, size int, err os.Error) + // UnreadRune causes the next call to ReadRune to return the same rune. + UnreadRune() os.Error + // SkipSpace skips space in the input. Newlines are treated as space + // unless the scan operation is Scanln, Fscanln or Sscanln, in which case + // a newline is treated as EOF. + SkipSpace() + // Token skips space in the input if skipSpace is true, then returns the + // run of Unicode code points c satisfying f(c). If f is nil, + // !unicode.IsSpace(c) is used; that is, the token will hold non-space + // characters. Newlines are treated as space unless the scan operation + // is Scanln, Fscanln or Sscanln, in which case a newline is treated as + // EOF. The returned slice points to shared data that may be overwritten + // by the next call to Token, a call to a Scan function using the ScanState + // as input, or when the calling Scan method returns. + Token(skipSpace bool, f func(int) bool) (token []byte, err os.Error) + // Width returns the value of the width option and whether it has been set. + // The unit is Unicode code points. + Width() (wid int, ok bool) + // Because ReadRune is implemented by the interface, Read should never be + // called by the scanning routines and a valid implementation of + // ScanState may choose always to return an error from Read. + Read(buf []byte) (n int, err os.Error) +} + +// Scanner is implemented by any value that has a Scan method, which scans +// the input for the representation of a value and stores the result in the +// receiver, which must be a pointer to be useful. The Scan method is called +// for any argument to Scan, Scanf, or Scanln that implements it. +type Scanner interface { + Scan(state ScanState, verb int) os.Error +} + +// Scan scans text read from standard input, storing successive +// space-separated values into successive arguments. Newlines count +// as space. It returns the number of items successfully scanned. +// If that is less than the number of arguments, err will report why. +func Scan(a ...interface{}) (n int, err os.Error) { + return Fscan(os.Stdin, a...) +} + +// Scanln is similar to Scan, but stops scanning at a newline and +// after the final item there must be a newline or EOF. +func Scanln(a ...interface{}) (n int, err os.Error) { + return Fscanln(os.Stdin, a...) +} + +// Scanf scans text read from standard input, storing successive +// space-separated values into successive arguments as determined by +// the format. It returns the number of items successfully scanned. +func Scanf(format string, a ...interface{}) (n int, err os.Error) { + return Fscanf(os.Stdin, format, a...) +} + +// Sscan scans the argument string, storing successive space-separated +// values into successive arguments. Newlines count as space. It +// returns the number of items successfully scanned. If that is less +// than the number of arguments, err will report why. +func Sscan(str string, a ...interface{}) (n int, err os.Error) { + return Fscan(strings.NewReader(str), a...) +} + +// Sscanln is similar to Sscan, but stops scanning at a newline and +// after the final item there must be a newline or EOF. +func Sscanln(str string, a ...interface{}) (n int, err os.Error) { + return Fscanln(strings.NewReader(str), a...) +} + +// Sscanf scans the argument string, storing successive space-separated +// values into successive arguments as determined by the format. It +// returns the number of items successfully parsed. +func Sscanf(str string, format string, a ...interface{}) (n int, err os.Error) { + return Fscanf(strings.NewReader(str), format, a...) +} + +// Fscan scans text read from r, storing successive space-separated +// values into successive arguments. Newlines count as space. It +// returns the number of items successfully scanned. If that is less +// than the number of arguments, err will report why. +func Fscan(r io.Reader, a ...interface{}) (n int, err os.Error) { + s, old := newScanState(r, true, false) + n, err = s.doScan(a) + s.free(old) + return +} + +// Fscanln is similar to Fscan, but stops scanning at a newline and +// after the final item there must be a newline or EOF. +func Fscanln(r io.Reader, a ...interface{}) (n int, err os.Error) { + s, old := newScanState(r, false, true) + n, err = s.doScan(a) + s.free(old) + return +} + +// Fscanf scans text read from r, storing successive space-separated +// values into successive arguments as determined by the format. It +// returns the number of items successfully parsed. +func Fscanf(r io.Reader, format string, a ...interface{}) (n int, err os.Error) { + s, old := newScanState(r, false, false) + n, err = s.doScanf(format, a) + s.free(old) + return +} + +// scanError represents an error generated by the scanning software. +// It's used as a unique signature to identify such errors when recovering. +type scanError struct { + err os.Error +} + +const eof = -1 + +// ss is the internal implementation of ScanState. +type ss struct { + rr io.RuneReader // where to read input + buf bytes.Buffer // token accumulator + peekRune int // one-rune lookahead + prevRune int // last rune returned by ReadRune + count int // runes consumed so far. + atEOF bool // already read EOF + ssave +} + +// ssave holds the parts of ss that need to be +// saved and restored on recursive scans. +type ssave struct { + validSave bool // is or was a part of an actual ss. + nlIsEnd bool // whether newline terminates scan + nlIsSpace bool // whether newline counts as white space + fieldLimit int // max value of ss.count for this field; fieldLimit <= limit + limit int // max value of ss.count. + maxWid int // width of this field. +} + +// The Read method is only in ScanState so that ScanState +// satisfies io.Reader. It will never be called when used as +// intended, so there is no need to make it actually work. +func (s *ss) Read(buf []byte) (n int, err os.Error) { + return 0, os.NewError("ScanState's Read should not be called. Use ReadRune") +} + +func (s *ss) ReadRune() (rune int, size int, err os.Error) { + if s.peekRune >= 0 { + s.count++ + rune = s.peekRune + size = utf8.RuneLen(rune) + s.prevRune = rune + s.peekRune = -1 + return + } + if s.atEOF || s.nlIsEnd && s.prevRune == '\n' || s.count >= s.fieldLimit { + err = os.EOF + return + } + + rune, size, err = s.rr.ReadRune() + if err == nil { + s.count++ + s.prevRune = rune + } else if err == os.EOF { + s.atEOF = true + } + return +} + +func (s *ss) Width() (wid int, ok bool) { + if s.maxWid == hugeWid { + return 0, false + } + return s.maxWid, true +} + +// The public method returns an error; this private one panics. +// If getRune reaches EOF, the return value is EOF (-1). +func (s *ss) getRune() (rune int) { + rune, _, err := s.ReadRune() + if err != nil { + if err == os.EOF { + return eof + } + s.error(err) + } + return +} + +// mustReadRune turns os.EOF into a panic(io.ErrUnexpectedEOF). +// It is called in cases such as string scanning where an EOF is a +// syntax error. +func (s *ss) mustReadRune() (rune int) { + rune = s.getRune() + if rune == eof { + s.error(io.ErrUnexpectedEOF) + } + return +} + +func (s *ss) UnreadRune() os.Error { + if u, ok := s.rr.(runeUnreader); ok { + u.UnreadRune() + } else { + s.peekRune = s.prevRune + } + s.prevRune = -1 + s.count-- + return nil +} + +func (s *ss) error(err os.Error) { + panic(scanError{err}) +} + +func (s *ss) errorString(err string) { + panic(scanError{os.NewError(err)}) +} + +func (s *ss) Token(skipSpace bool, f func(int) bool) (tok []byte, err os.Error) { + defer func() { + if e := recover(); e != nil { + if se, ok := e.(scanError); ok { + err = se.err + } else { + panic(e) + } + } + }() + if f == nil { + f = notSpace + } + s.buf.Reset() + tok = s.token(skipSpace, f) + return +} + +// notSpace is the default scanning function used in Token. +func notSpace(r int) bool { + return !unicode.IsSpace(r) +} + +// skipSpace provides Scan() methods the ability to skip space and newline characters +// in keeping with the current scanning mode set by format strings and Scan()/Scanln(). +func (s *ss) SkipSpace() { + s.skipSpace(false) +} + +// readRune is a structure to enable reading UTF-8 encoded code points +// from an io.Reader. It is used if the Reader given to the scanner does +// not already implement io.RuneReader. +type readRune struct { + reader io.Reader + buf [utf8.UTFMax]byte // used only inside ReadRune + pending int // number of bytes in pendBuf; only >0 for bad UTF-8 + pendBuf [utf8.UTFMax]byte // bytes left over +} + +// readByte returns the next byte from the input, which may be +// left over from a previous read if the UTF-8 was ill-formed. +func (r *readRune) readByte() (b byte, err os.Error) { + if r.pending > 0 { + b = r.pendBuf[0] + copy(r.pendBuf[0:], r.pendBuf[1:]) + r.pending-- + return + } + _, err = r.reader.Read(r.pendBuf[0:1]) + return r.pendBuf[0], err +} + +// unread saves the bytes for the next read. +func (r *readRune) unread(buf []byte) { + copy(r.pendBuf[r.pending:], buf) + r.pending += len(buf) +} + +// ReadRune returns the next UTF-8 encoded code point from the +// io.Reader inside r. +func (r *readRune) ReadRune() (rune int, size int, err os.Error) { + r.buf[0], err = r.readByte() + if err != nil { + return 0, 0, err + } + if r.buf[0] < utf8.RuneSelf { // fast check for common ASCII case + rune = int(r.buf[0]) + return + } + var n int + for n = 1; !utf8.FullRune(r.buf[0:n]); n++ { + r.buf[n], err = r.readByte() + if err != nil { + if err == os.EOF { + err = nil + break + } + return + } + } + rune, size = utf8.DecodeRune(r.buf[0:n]) + if size < n { // an error + r.unread(r.buf[size:n]) + } + return +} + +var ssFree = newCache(func() interface{} { return new(ss) }) + +// Allocate a new ss struct or grab a cached one. +func newScanState(r io.Reader, nlIsSpace, nlIsEnd bool) (s *ss, old ssave) { + // If the reader is a *ss, then we've got a recursive + // call to Scan, so re-use the scan state. + s, ok := r.(*ss) + if ok { + old = s.ssave + s.limit = s.fieldLimit + s.nlIsEnd = nlIsEnd || s.nlIsEnd + s.nlIsSpace = nlIsSpace + return + } + + s = ssFree.get().(*ss) + if rr, ok := r.(io.RuneReader); ok { + s.rr = rr + } else { + s.rr = &readRune{reader: r} + } + s.nlIsSpace = nlIsSpace + s.nlIsEnd = nlIsEnd + s.prevRune = -1 + s.peekRune = -1 + s.atEOF = false + s.limit = hugeWid + s.fieldLimit = hugeWid + s.maxWid = hugeWid + s.validSave = true + return +} + +// Save used ss structs in ssFree; avoid an allocation per invocation. +func (s *ss) free(old ssave) { + // If it was used recursively, just restore the old state. + if old.validSave { + s.ssave = old + return + } + // Don't hold on to ss structs with large buffers. + if cap(s.buf.Bytes()) > 1024 { + return + } + s.buf.Reset() + s.rr = nil + ssFree.put(s) +} + +// skipSpace skips spaces and maybe newlines. +func (s *ss) skipSpace(stopAtNewline bool) { + for { + rune := s.getRune() + if rune == eof { + return + } + if rune == '\n' { + if stopAtNewline { + break + } + if s.nlIsSpace { + continue + } + s.errorString("unexpected newline") + return + } + if !unicode.IsSpace(rune) { + s.UnreadRune() + break + } + } +} + +// token returns the next space-delimited string from the input. It +// skips white space. For Scanln, it stops at newlines. For Scan, +// newlines are treated as spaces. +func (s *ss) token(skipSpace bool, f func(int) bool) []byte { + if skipSpace { + s.skipSpace(false) + } + // read until white space or newline + for { + rune := s.getRune() + if rune == eof { + break + } + if !f(rune) { + s.UnreadRune() + break + } + s.buf.WriteRune(rune) + } + return s.buf.Bytes() +} + +// typeError indicates that the type of the operand did not match the format +func (s *ss) typeError(field interface{}, expected string) { + s.errorString("expected field of type pointer to " + expected + "; found " + reflect.TypeOf(field).String()) +} + +var complexError = os.NewError("syntax error scanning complex number") +var boolError = os.NewError("syntax error scanning boolean") + +// consume reads the next rune in the input and reports whether it is in the ok string. +// If accept is true, it puts the character into the input token. +func (s *ss) consume(ok string, accept bool) bool { + rune := s.getRune() + if rune == eof { + return false + } + if strings.IndexRune(ok, rune) >= 0 { + if accept { + s.buf.WriteRune(rune) + } + return true + } + if rune != eof && accept { + s.UnreadRune() + } + return false +} + +// peek reports whether the next character is in the ok string, without consuming it. +func (s *ss) peek(ok string) bool { + rune := s.getRune() + if rune != eof { + s.UnreadRune() + } + return strings.IndexRune(ok, rune) >= 0 +} + +func (s *ss) notEOF() { + // Guarantee there is data to be read. + if rune := s.getRune(); rune == eof { + panic(os.EOF) + } + s.UnreadRune() +} + +// accept checks the next rune in the input. If it's a byte (sic) in the string, it puts it in the +// buffer and returns true. Otherwise it return false. +func (s *ss) accept(ok string) bool { + return s.consume(ok, true) +} + +// okVerb verifies that the verb is present in the list, setting s.err appropriately if not. +func (s *ss) okVerb(verb int, okVerbs, typ string) bool { + for _, v := range okVerbs { + if v == verb { + return true + } + } + s.errorString("bad verb %" + string(verb) + " for " + typ) + return false +} + +// scanBool returns the value of the boolean represented by the next token. +func (s *ss) scanBool(verb int) bool { + s.skipSpace(false) + s.notEOF() + if !s.okVerb(verb, "tv", "boolean") { + return false + } + // Syntax-checking a boolean is annoying. We're not fastidious about case. + switch s.getRune() { + case '0': + return false + case '1': + return true + case 't', 'T': + if s.accept("rR") && (!s.accept("uU") || !s.accept("eE")) { + s.error(boolError) + } + return true + case 'f', 'F': + if s.accept("aL") && (!s.accept("lL") || !s.accept("sS") || !s.accept("eE")) { + s.error(boolError) + } + return false + } + return false +} + +// Numerical elements +const ( + binaryDigits = "01" + octalDigits = "01234567" + decimalDigits = "0123456789" + hexadecimalDigits = "0123456789aAbBcCdDeEfF" + sign = "+-" + period = "." + exponent = "eEp" +) + +// getBase returns the numeric base represented by the verb and its digit string. +func (s *ss) getBase(verb int) (base int, digits string) { + s.okVerb(verb, "bdoUxXv", "integer") // sets s.err + base = 10 + digits = decimalDigits + switch verb { + case 'b': + base = 2 + digits = binaryDigits + case 'o': + base = 8 + digits = octalDigits + case 'x', 'X', 'U': + base = 16 + digits = hexadecimalDigits + } + return +} + +// scanNumber returns the numerical string with specified digits starting here. +func (s *ss) scanNumber(digits string, haveDigits bool) string { + if !haveDigits { + s.notEOF() + if !s.accept(digits) { + s.errorString("expected integer") + } + } + for s.accept(digits) { + } + return s.buf.String() +} + +// scanRune returns the next rune value in the input. +func (s *ss) scanRune(bitSize int) int64 { + s.notEOF() + rune := int64(s.getRune()) + n := uint(bitSize) + x := (rune << (64 - n)) >> (64 - n) + if x != rune { + s.errorString("overflow on character value " + string(rune)) + } + return rune +} + +// scanBasePrefix reports whether the integer begins with a 0 or 0x, +// and returns the base, digit string, and whether a zero was found. +// It is called only if the verb is %v. +func (s *ss) scanBasePrefix() (base int, digits string, found bool) { + if !s.peek("0") { + return 10, decimalDigits, false + } + s.accept("0") + found = true // We've put a digit into the token buffer. + // Special cases for '0' && '0x' + base, digits = 8, octalDigits + if s.peek("xX") { + s.consume("xX", false) + base, digits = 16, hexadecimalDigits + } + return +} + +// scanInt returns the value of the integer represented by the next +// token, checking for overflow. Any error is stored in s.err. +func (s *ss) scanInt(verb int, bitSize int) int64 { + if verb == 'c' { + return s.scanRune(bitSize) + } + s.skipSpace(false) + s.notEOF() + base, digits := s.getBase(verb) + haveDigits := false + if verb == 'U' { + if !s.consume("U", false) || !s.consume("+", false) { + s.errorString("bad unicode format ") + } + } else { + s.accept(sign) // If there's a sign, it will be left in the token buffer. + if verb == 'v' { + base, digits, haveDigits = s.scanBasePrefix() + } + } + tok := s.scanNumber(digits, haveDigits) + i, err := strconv.Btoi64(tok, base) + if err != nil { + s.error(err) + } + n := uint(bitSize) + x := (i << (64 - n)) >> (64 - n) + if x != i { + s.errorString("integer overflow on token " + tok) + } + return i +} + +// scanUint returns the value of the unsigned integer represented +// by the next token, checking for overflow. Any error is stored in s.err. +func (s *ss) scanUint(verb int, bitSize int) uint64 { + if verb == 'c' { + return uint64(s.scanRune(bitSize)) + } + s.skipSpace(false) + s.notEOF() + base, digits := s.getBase(verb) + haveDigits := false + if verb == 'U' { + if !s.consume("U", false) || !s.consume("+", false) { + s.errorString("bad unicode format ") + } + } else if verb == 'v' { + base, digits, haveDigits = s.scanBasePrefix() + } + tok := s.scanNumber(digits, haveDigits) + i, err := strconv.Btoui64(tok, base) + if err != nil { + s.error(err) + } + n := uint(bitSize) + x := (i << (64 - n)) >> (64 - n) + if x != i { + s.errorString("unsigned integer overflow on token " + tok) + } + return i +} + +// floatToken returns the floating-point number starting here, no longer than swid +// if the width is specified. It's not rigorous about syntax because it doesn't check that +// we have at least some digits, but Atof will do that. +func (s *ss) floatToken() string { + s.buf.Reset() + // NaN? + if s.accept("nN") && s.accept("aA") && s.accept("nN") { + return s.buf.String() + } + // leading sign? + s.accept(sign) + // Inf? + if s.accept("iI") && s.accept("nN") && s.accept("fF") { + return s.buf.String() + } + // digits? + for s.accept(decimalDigits) { + } + // decimal point? + if s.accept(period) { + // fraction? + for s.accept(decimalDigits) { + } + } + // exponent? + if s.accept(exponent) { + // leading sign? + s.accept(sign) + // digits? + for s.accept(decimalDigits) { + } + } + return s.buf.String() +} + +// complexTokens returns the real and imaginary parts of the complex number starting here. +// The number might be parenthesized and has the format (N+Ni) where N is a floating-point +// number and there are no spaces within. +func (s *ss) complexTokens() (real, imag string) { + // TODO: accept N and Ni independently? + parens := s.accept("(") + real = s.floatToken() + s.buf.Reset() + // Must now have a sign. + if !s.accept("+-") { + s.error(complexError) + } + // Sign is now in buffer + imagSign := s.buf.String() + imag = s.floatToken() + if !s.accept("i") { + s.error(complexError) + } + if parens && !s.accept(")") { + s.error(complexError) + } + return real, imagSign + imag +} + +// convertFloat converts the string to a float64value. +func (s *ss) convertFloat(str string, n int) float64 { + if p := strings.Index(str, "p"); p >= 0 { + // Atof doesn't handle power-of-2 exponents, + // but they're easy to evaluate. + f, err := strconv.AtofN(str[:p], n) + if err != nil { + // Put full string into error. + if e, ok := err.(*strconv.NumError); ok { + e.Num = str + } + s.error(err) + } + n, err := strconv.Atoi(str[p+1:]) + if err != nil { + // Put full string into error. + if e, ok := err.(*strconv.NumError); ok { + e.Num = str + } + s.error(err) + } + return math.Ldexp(f, n) + } + f, err := strconv.AtofN(str, n) + if err != nil { + s.error(err) + } + return f +} + +// convertComplex converts the next token to a complex128 value. +// The atof argument is a type-specific reader for the underlying type. +// If we're reading complex64, atof will parse float32s and convert them +// to float64's to avoid reproducing this code for each complex type. +func (s *ss) scanComplex(verb int, n int) complex128 { + if !s.okVerb(verb, floatVerbs, "complex") { + return 0 + } + s.skipSpace(false) + s.notEOF() + sreal, simag := s.complexTokens() + real := s.convertFloat(sreal, n/2) + imag := s.convertFloat(simag, n/2) + return complex(real, imag) +} + +// convertString returns the string represented by the next input characters. +// The format of the input is determined by the verb. +func (s *ss) convertString(verb int) (str string) { + if !s.okVerb(verb, "svqx", "string") { + return "" + } + s.skipSpace(false) + s.notEOF() + switch verb { + case 'q': + str = s.quotedString() + case 'x': + str = s.hexString() + default: + str = string(s.token(true, notSpace)) // %s and %v just return the next word + } + return +} + +// quotedString returns the double- or back-quoted string represented by the next input characters. +func (s *ss) quotedString() string { + s.notEOF() + quote := s.getRune() + switch quote { + case '`': + // Back-quoted: Anything goes until EOF or back quote. + for { + rune := s.mustReadRune() + if rune == quote { + break + } + s.buf.WriteRune(rune) + } + return s.buf.String() + case '"': + // Double-quoted: Include the quotes and let strconv.Unquote do the backslash escapes. + s.buf.WriteRune(quote) + for { + rune := s.mustReadRune() + s.buf.WriteRune(rune) + if rune == '\\' { + // In a legal backslash escape, no matter how long, only the character + // immediately after the escape can itself be a backslash or quote. + // Thus we only need to protect the first character after the backslash. + rune := s.mustReadRune() + s.buf.WriteRune(rune) + } else if rune == '"' { + break + } + } + result, err := strconv.Unquote(s.buf.String()) + if err != nil { + s.error(err) + } + return result + default: + s.errorString("expected quoted string") + } + return "" +} + +// hexDigit returns the value of the hexadecimal digit +func (s *ss) hexDigit(digit int) int { + switch digit { + case '0', '1', '2', '3', '4', '5', '6', '7', '8', '9': + return digit - '0' + case 'a', 'b', 'c', 'd', 'e', 'f': + return 10 + digit - 'a' + case 'A', 'B', 'C', 'D', 'E', 'F': + return 10 + digit - 'A' + } + s.errorString("Scan: illegal hex digit") + return 0 +} + +// hexByte returns the next hex-encoded (two-character) byte from the input. +// There must be either two hexadecimal digits or a space character in the input. +func (s *ss) hexByte() (b byte, ok bool) { + rune1 := s.getRune() + if rune1 == eof { + return + } + if unicode.IsSpace(rune1) { + s.UnreadRune() + return + } + rune2 := s.mustReadRune() + return byte(s.hexDigit(rune1)<<4 | s.hexDigit(rune2)), true +} + +// hexString returns the space-delimited hexpair-encoded string. +func (s *ss) hexString() string { + s.notEOF() + for { + b, ok := s.hexByte() + if !ok { + break + } + s.buf.WriteByte(b) + } + if s.buf.Len() == 0 { + s.errorString("Scan: no hex data for %x string") + return "" + } + return s.buf.String() +} + +const floatVerbs = "beEfFgGv" + +const hugeWid = 1 << 30 + +// scanOne scans a single value, deriving the scanner from the type of the argument. +func (s *ss) scanOne(verb int, field interface{}) { + s.buf.Reset() + var err os.Error + // If the parameter has its own Scan method, use that. + if v, ok := field.(Scanner); ok { + err = v.Scan(s, verb) + if err != nil { + if err == os.EOF { + err = io.ErrUnexpectedEOF + } + s.error(err) + } + return + } + + switch v := field.(type) { + case *bool: + *v = s.scanBool(verb) + case *complex64: + *v = complex64(s.scanComplex(verb, 64)) + case *complex128: + *v = s.scanComplex(verb, 128) + case *int: + *v = int(s.scanInt(verb, intBits)) + case *int8: + *v = int8(s.scanInt(verb, 8)) + case *int16: + *v = int16(s.scanInt(verb, 16)) + case *int32: + *v = int32(s.scanInt(verb, 32)) + case *int64: + *v = s.scanInt(verb, 64) + case *uint: + *v = uint(s.scanUint(verb, intBits)) + case *uint8: + *v = uint8(s.scanUint(verb, 8)) + case *uint16: + *v = uint16(s.scanUint(verb, 16)) + case *uint32: + *v = uint32(s.scanUint(verb, 32)) + case *uint64: + *v = s.scanUint(verb, 64) + case *uintptr: + *v = uintptr(s.scanUint(verb, uintptrBits)) + // Floats are tricky because you want to scan in the precision of the result, not + // scan in high precision and convert, in order to preserve the correct error condition. + case *float32: + if s.okVerb(verb, floatVerbs, "float32") { + s.skipSpace(false) + s.notEOF() + *v = float32(s.convertFloat(s.floatToken(), 32)) + } + case *float64: + if s.okVerb(verb, floatVerbs, "float64") { + s.skipSpace(false) + s.notEOF() + *v = s.convertFloat(s.floatToken(), 64) + } + case *string: + *v = s.convertString(verb) + case *[]byte: + // We scan to string and convert so we get a copy of the data. + // If we scanned to bytes, the slice would point at the buffer. + *v = []byte(s.convertString(verb)) + default: + val := reflect.ValueOf(v) + ptr := val + if ptr.Kind() != reflect.Ptr { + s.errorString("Scan: type not a pointer: " + val.Type().String()) + return + } + switch v := ptr.Elem(); v.Kind() { + case reflect.Bool: + v.SetBool(s.scanBool(verb)) + case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64: + v.SetInt(s.scanInt(verb, v.Type().Bits())) + case reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64, reflect.Uintptr: + v.SetUint(s.scanUint(verb, v.Type().Bits())) + case reflect.String: + v.SetString(s.convertString(verb)) + case reflect.Slice: + // For now, can only handle (renamed) []byte. + typ := v.Type() + if typ.Elem().Kind() != reflect.Uint8 { + s.errorString("Scan: can't handle type: " + val.Type().String()) + } + str := s.convertString(verb) + v.Set(reflect.MakeSlice(typ, len(str), len(str))) + for i := 0; i < len(str); i++ { + v.Index(i).SetUint(uint64(str[i])) + } + case reflect.Float32, reflect.Float64: + s.skipSpace(false) + s.notEOF() + v.SetFloat(s.convertFloat(s.floatToken(), v.Type().Bits())) + case reflect.Complex64, reflect.Complex128: + v.SetComplex(s.scanComplex(verb, v.Type().Bits())) + default: + s.errorString("Scan: can't handle type: " + val.Type().String()) + } + } +} + +// errorHandler turns local panics into error returns. +func errorHandler(errp *os.Error) { + if e := recover(); e != nil { + if se, ok := e.(scanError); ok { // catch local error + *errp = se.err + } else if eof, ok := e.(os.Error); ok && eof == os.EOF { // out of input + *errp = eof + } else { + panic(e) + } + } +} + +// doScan does the real work for scanning without a format string. +func (s *ss) doScan(a []interface{}) (numProcessed int, err os.Error) { + defer errorHandler(&err) + for _, field := range a { + s.scanOne('v', field) + numProcessed++ + } + // Check for newline if required. + if !s.nlIsSpace { + for { + rune := s.getRune() + if rune == '\n' || rune == eof { + break + } + if !unicode.IsSpace(rune) { + s.errorString("Scan: expected newline") + break + } + } + } + return +} + +// advance determines whether the next characters in the input match +// those of the format. It returns the number of bytes (sic) consumed +// in the format. Newlines included, all runs of space characters in +// either input or format behave as a single space. This routine also +// handles the %% case. If the return value is zero, either format +// starts with a % (with no following %) or the input is empty. +// If it is negative, the input did not match the string. +func (s *ss) advance(format string) (i int) { + for i < len(format) { + fmtc, w := utf8.DecodeRuneInString(format[i:]) + if fmtc == '%' { + // %% acts like a real percent + nextc, _ := utf8.DecodeRuneInString(format[i+w:]) // will not match % if string is empty + if nextc != '%' { + return + } + i += w // skip the first % + } + sawSpace := false + for unicode.IsSpace(fmtc) && i < len(format) { + sawSpace = true + i += w + fmtc, w = utf8.DecodeRuneInString(format[i:]) + } + if sawSpace { + // There was space in the format, so there should be space (EOF) + // in the input. + inputc := s.getRune() + if inputc == eof { + return + } + if !unicode.IsSpace(inputc) { + // Space in format but not in input: error + s.errorString("expected space in input to match format") + } + s.skipSpace(true) + continue + } + inputc := s.mustReadRune() + if fmtc != inputc { + s.UnreadRune() + return -1 + } + i += w + } + return +} + +// doScanf does the real work when scanning with a format string. +// At the moment, it handles only pointers to basic types. +func (s *ss) doScanf(format string, a []interface{}) (numProcessed int, err os.Error) { + defer errorHandler(&err) + end := len(format) - 1 + // We process one item per non-trivial format + for i := 0; i <= end; { + w := s.advance(format[i:]) + if w > 0 { + i += w + continue + } + // Either we failed to advance, we have a percent character, or we ran out of input. + if format[i] != '%' { + // Can't advance format. Why not? + if w < 0 { + s.errorString("input does not match format") + } + // Otherwise at EOF; "too many operands" error handled below + break + } + i++ // % is one byte + + // do we have 20 (width)? + var widPresent bool + s.maxWid, widPresent, i = parsenum(format, i, end) + if !widPresent { + s.maxWid = hugeWid + } + s.fieldLimit = s.limit + if f := s.count + s.maxWid; f < s.fieldLimit { + s.fieldLimit = f + } + + c, w := utf8.DecodeRuneInString(format[i:]) + i += w + + if numProcessed >= len(a) { // out of operands + s.errorString("too few operands for format %" + format[i-w:]) + break + } + field := a[numProcessed] + + s.scanOne(c, field) + numProcessed++ + s.fieldLimit = s.limit + } + if numProcessed < len(a) { + s.errorString("too many operands") + } + return +} diff --git a/src/pkg/fmt/scan_test.go b/src/pkg/fmt/scan_test.go new file mode 100644 index 000000000..3f06e5725 --- /dev/null +++ b/src/pkg/fmt/scan_test.go @@ -0,0 +1,923 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package fmt_test + +import ( + "bufio" + "bytes" + . "fmt" + "io" + "math" + "os" + "reflect" + "regexp" + "strings" + "testing" + "utf8" +) + +type ScanTest struct { + text string + in interface{} + out interface{} +} + +type ScanfTest struct { + format string + text string + in interface{} + out interface{} +} + +type ScanfMultiTest struct { + format string + text string + in []interface{} + out []interface{} + err string +} + +var ( + boolVal bool + intVal int + int8Val int8 + int16Val int16 + int32Val int32 + int64Val int64 + uintVal uint + uint8Val uint8 + uint16Val uint16 + uint32Val uint32 + uint64Val uint64 + float32Val float32 + float64Val float64 + stringVal string + stringVal1 string + bytesVal []byte + complex64Val complex64 + complex128Val complex128 + renamedBoolVal renamedBool + renamedIntVal renamedInt + renamedInt8Val renamedInt8 + renamedInt16Val renamedInt16 + renamedInt32Val renamedInt32 + renamedInt64Val renamedInt64 + renamedUintVal renamedUint + renamedUint8Val renamedUint8 + renamedUint16Val renamedUint16 + renamedUint32Val renamedUint32 + renamedUint64Val renamedUint64 + renamedUintptrVal renamedUintptr + renamedStringVal renamedString + renamedBytesVal renamedBytes + renamedFloat32Val renamedFloat32 + renamedFloat64Val renamedFloat64 + renamedComplex64Val renamedComplex64 + renamedComplex128Val renamedComplex128 +) + +type FloatTest struct { + text string + in float64 + out float64 +} + +// Xs accepts any non-empty run of the verb character +type Xs string + +func (x *Xs) Scan(state ScanState, verb int) os.Error { + tok, err := state.Token(true, func(r int) bool { return r == verb }) + if err != nil { + return err + } + s := string(tok) + if !regexp.MustCompile("^" + string(verb) + "+$").MatchString(s) { + return os.NewError("syntax error for xs") + } + *x = Xs(s) + return nil +} + +var xVal Xs + +// IntString accepts an integer followed immediately by a string. +// It tests the embedding of a scan within a scan. +type IntString struct { + i int + s string +} + +func (s *IntString) Scan(state ScanState, verb int) os.Error { + if _, err := Fscan(state, &s.i); err != nil { + return err + } + + tok, err := state.Token(true, nil) + if err != nil { + return err + } + s.s = string(tok) + return nil +} + +var intStringVal IntString + +// myStringReader implements Read but not ReadRune, allowing us to test our readRune wrapper +// type that creates something that can read runes given only Read(). +type myStringReader struct { + r *strings.Reader +} + +func (s *myStringReader) Read(p []byte) (n int, err os.Error) { + return s.r.Read(p) +} + +func newReader(s string) *myStringReader { + return &myStringReader{strings.NewReader(s)} +} + +var scanTests = []ScanTest{ + // Basic types + {"T\n", &boolVal, true}, // boolean test vals toggle to be sure they are written + {"F\n", &boolVal, false}, // restored to zero value + {"21\n", &intVal, 21}, + {"0\n", &intVal, 0}, + {"000\n", &intVal, 0}, + {"0x10\n", &intVal, 0x10}, + {"-0x10\n", &intVal, -0x10}, + {"0377\n", &intVal, 0377}, + {"-0377\n", &intVal, -0377}, + {"0\n", &uintVal, uint(0)}, + {"000\n", &uintVal, uint(0)}, + {"0x10\n", &uintVal, uint(0x10)}, + {"0377\n", &uintVal, uint(0377)}, + {"22\n", &int8Val, int8(22)}, + {"23\n", &int16Val, int16(23)}, + {"24\n", &int32Val, int32(24)}, + {"25\n", &int64Val, int64(25)}, + {"127\n", &int8Val, int8(127)}, + {"-21\n", &intVal, -21}, + {"-22\n", &int8Val, int8(-22)}, + {"-23\n", &int16Val, int16(-23)}, + {"-24\n", &int32Val, int32(-24)}, + {"-25\n", &int64Val, int64(-25)}, + {"-128\n", &int8Val, int8(-128)}, + {"+21\n", &intVal, +21}, + {"+22\n", &int8Val, int8(+22)}, + {"+23\n", &int16Val, int16(+23)}, + {"+24\n", &int32Val, int32(+24)}, + {"+25\n", &int64Val, int64(+25)}, + {"+127\n", &int8Val, int8(+127)}, + {"26\n", &uintVal, uint(26)}, + {"27\n", &uint8Val, uint8(27)}, + {"28\n", &uint16Val, uint16(28)}, + {"29\n", &uint32Val, uint32(29)}, + {"30\n", &uint64Val, uint64(30)}, + {"255\n", &uint8Val, uint8(255)}, + {"32767\n", &int16Val, int16(32767)}, + {"2.3\n", &float64Val, 2.3}, + {"2.3e1\n", &float32Val, float32(2.3e1)}, + {"2.3e2\n", &float64Val, 2.3e2}, + {"2.3p2\n", &float64Val, 2.3 * 4}, + {"2.3p+2\n", &float64Val, 2.3 * 4}, + {"2.3p+66\n", &float64Val, 2.3 * (1 << 32) * (1 << 32) * 4}, + {"2.3p-66\n", &float64Val, 2.3 / ((1 << 32) * (1 << 32) * 4)}, + {"2.35\n", &stringVal, "2.35"}, + {"2345678\n", &bytesVal, []byte("2345678")}, + {"(3.4e1-2i)\n", &complex128Val, 3.4e1 - 2i}, + {"-3.45e1-3i\n", &complex64Val, complex64(-3.45e1 - 3i)}, + {"-.45e1-1e2i\n", &complex128Val, complex128(-.45e1 - 100i)}, + {"hello\n", &stringVal, "hello"}, + + // Renamed types + {"true\n", &renamedBoolVal, renamedBool(true)}, + {"F\n", &renamedBoolVal, renamedBool(false)}, + {"101\n", &renamedIntVal, renamedInt(101)}, + {"102\n", &renamedIntVal, renamedInt(102)}, + {"103\n", &renamedUintVal, renamedUint(103)}, + {"104\n", &renamedUintVal, renamedUint(104)}, + {"105\n", &renamedInt8Val, renamedInt8(105)}, + {"106\n", &renamedInt16Val, renamedInt16(106)}, + {"107\n", &renamedInt32Val, renamedInt32(107)}, + {"108\n", &renamedInt64Val, renamedInt64(108)}, + {"109\n", &renamedUint8Val, renamedUint8(109)}, + {"110\n", &renamedUint16Val, renamedUint16(110)}, + {"111\n", &renamedUint32Val, renamedUint32(111)}, + {"112\n", &renamedUint64Val, renamedUint64(112)}, + {"113\n", &renamedUintptrVal, renamedUintptr(113)}, + {"114\n", &renamedStringVal, renamedString("114")}, + {"115\n", &renamedBytesVal, renamedBytes([]byte("115"))}, + + // Custom scanners. + {" vvv ", &xVal, Xs("vvv")}, + {" 1234hello", &intStringVal, IntString{1234, "hello"}}, + + // Fixed bugs + {"2147483648\n", &int64Val, int64(2147483648)}, // was: integer overflow +} + +var scanfTests = []ScanfTest{ + {"%v", "TRUE\n", &boolVal, true}, + {"%t", "false\n", &boolVal, false}, + {"%v", "-71\n", &intVal, -71}, + {"%v", "0377\n", &intVal, 0377}, + {"%v", "0x44\n", &intVal, 0x44}, + {"%d", "72\n", &intVal, 72}, + {"%c", "a\n", &intVal, 'a'}, + {"%c", "\u5072\n", &intVal, 0x5072}, + {"%c", "\u1234\n", &intVal, '\u1234'}, + {"%d", "73\n", &int8Val, int8(73)}, + {"%d", "+74\n", &int16Val, int16(74)}, + {"%d", "75\n", &int32Val, int32(75)}, + {"%d", "76\n", &int64Val, int64(76)}, + {"%b", "1001001\n", &intVal, 73}, + {"%o", "075\n", &intVal, 075}, + {"%x", "a75\n", &intVal, 0xa75}, + {"%v", "71\n", &uintVal, uint(71)}, + {"%d", "72\n", &uintVal, uint(72)}, + {"%d", "73\n", &uint8Val, uint8(73)}, + {"%d", "74\n", &uint16Val, uint16(74)}, + {"%d", "75\n", &uint32Val, uint32(75)}, + {"%d", "76\n", &uint64Val, uint64(76)}, + {"%b", "1001001\n", &uintVal, uint(73)}, + {"%o", "075\n", &uintVal, uint(075)}, + {"%x", "a75\n", &uintVal, uint(0xa75)}, + {"%x", "A75\n", &uintVal, uint(0xa75)}, + {"%U", "U+1234\n", &intVal, int(0x1234)}, + {"%U", "U+4567\n", &uintVal, uint(0x4567)}, + + // Strings + {"%s", "using-%s\n", &stringVal, "using-%s"}, + {"%x", "7573696e672d2578\n", &stringVal, "using-%x"}, + {"%q", `"quoted\twith\\do\u0075bl\x65s"` + "\n", &stringVal, "quoted\twith\\doubles"}, + {"%q", "`quoted with backs`\n", &stringVal, "quoted with backs"}, + + // Byte slices + {"%s", "bytes-%s\n", &bytesVal, []byte("bytes-%s")}, + {"%x", "62797465732d2578\n", &bytesVal, []byte("bytes-%x")}, + {"%q", `"bytes\rwith\vdo\u0075bl\x65s"` + "\n", &bytesVal, []byte("bytes\rwith\vdoubles")}, + {"%q", "`bytes with backs`\n", &bytesVal, []byte("bytes with backs")}, + + // Renamed types + {"%v\n", "true\n", &renamedBoolVal, renamedBool(true)}, + {"%t\n", "F\n", &renamedBoolVal, renamedBool(false)}, + {"%v", "101\n", &renamedIntVal, renamedInt(101)}, + {"%c", "\u0101\n", &renamedIntVal, renamedInt('\u0101')}, + {"%o", "0146\n", &renamedIntVal, renamedInt(102)}, + {"%v", "103\n", &renamedUintVal, renamedUint(103)}, + {"%d", "104\n", &renamedUintVal, renamedUint(104)}, + {"%d", "105\n", &renamedInt8Val, renamedInt8(105)}, + {"%d", "106\n", &renamedInt16Val, renamedInt16(106)}, + {"%d", "107\n", &renamedInt32Val, renamedInt32(107)}, + {"%d", "108\n", &renamedInt64Val, renamedInt64(108)}, + {"%x", "6D\n", &renamedUint8Val, renamedUint8(109)}, + {"%o", "0156\n", &renamedUint16Val, renamedUint16(110)}, + {"%d", "111\n", &renamedUint32Val, renamedUint32(111)}, + {"%d", "112\n", &renamedUint64Val, renamedUint64(112)}, + {"%d", "113\n", &renamedUintptrVal, renamedUintptr(113)}, + {"%s", "114\n", &renamedStringVal, renamedString("114")}, + {"%q", "\"1155\"\n", &renamedBytesVal, renamedBytes([]byte("1155"))}, + {"%g", "116e1\n", &renamedFloat32Val, renamedFloat32(116e1)}, + {"%g", "-11.7e+1", &renamedFloat64Val, renamedFloat64(-11.7e+1)}, + {"%g", "11+6e1i\n", &renamedComplex64Val, renamedComplex64(11 + 6e1i)}, + {"%g", "-11.+7e+1i", &renamedComplex128Val, renamedComplex128(-11. + 7e+1i)}, + + // Interesting formats + {"here is\tthe value:%d", "here is the\tvalue:118\n", &intVal, 118}, + {"%% %%:%d", "% %:119\n", &intVal, 119}, + + // Corner cases + {"%x", "FFFFFFFF\n", &uint32Val, uint32(0xFFFFFFFF)}, + + // Custom scanner. + {"%s", " sss ", &xVal, Xs("sss")}, + {"%2s", "sssss", &xVal, Xs("ss")}, + + // Fixed bugs + {"%d\n", "27\n", &intVal, 27}, // ok + {"%d\n", "28 \n", &intVal, 28}, // was: "unexpected newline" + {"%v", "0", &intVal, 0}, // was: "EOF"; 0 was taken as base prefix and not counted. + {"%v", "0", &uintVal, uint(0)}, // was: "EOF"; 0 was taken as base prefix and not counted. +} + +var overflowTests = []ScanTest{ + {"128", &int8Val, 0}, + {"32768", &int16Val, 0}, + {"-129", &int8Val, 0}, + {"-32769", &int16Val, 0}, + {"256", &uint8Val, 0}, + {"65536", &uint16Val, 0}, + {"1e100", &float32Val, 0}, + {"1e500", &float64Val, 0}, + {"(1e100+0i)", &complex64Val, 0}, + {"(1+1e100i)", &complex64Val, 0}, + {"(1-1e500i)", &complex128Val, 0}, +} + +var i, j, k int +var f float64 +var s, t string +var c complex128 +var x, y Xs +var z IntString + +var multiTests = []ScanfMultiTest{ + {"", "", nil, nil, ""}, + {"%d", "23", args(&i), args(23), ""}, + {"%2s%3s", "22333", args(&s, &t), args("22", "333"), ""}, + {"%2d%3d", "44555", args(&i, &j), args(44, 555), ""}, + {"%2d.%3d", "66.777", args(&i, &j), args(66, 777), ""}, + {"%d, %d", "23, 18", args(&i, &j), args(23, 18), ""}, + {"%3d22%3d", "33322333", args(&i, &j), args(333, 333), ""}, + {"%6vX=%3fY", "3+2iX=2.5Y", args(&c, &f), args((3 + 2i), 2.5), ""}, + {"%d%s", "123abc", args(&i, &s), args(123, "abc"), ""}, + {"%c%c%c", "2\u50c2X", args(&i, &j, &k), args('2', '\u50c2', 'X'), ""}, + + // Custom scanners. + {"%e%f", "eefffff", args(&x, &y), args(Xs("ee"), Xs("fffff")), ""}, + {"%4v%s", "12abcd", args(&z, &s), args(IntString{12, "ab"}, "cd"), ""}, + + // Errors + {"%t", "23 18", args(&i), nil, "bad verb"}, + {"%d %d %d", "23 18", args(&i, &j), args(23, 18), "too few operands"}, + {"%d %d", "23 18 27", args(&i, &j, &k), args(23, 18), "too many operands"}, + {"%c", "\u0100", args(&int8Val), nil, "overflow"}, + {"X%d", "10X", args(&intVal), nil, "input does not match format"}, + + // Bad UTF-8: should see every byte. + {"%c%c%c", "\xc2X\xc2", args(&i, &j, &k), args(utf8.RuneError, 'X', utf8.RuneError), ""}, +} + +func testScan(name string, t *testing.T, scan func(r io.Reader, a ...interface{}) (int, os.Error)) { + for _, test := range scanTests { + var r io.Reader + if name == "StringReader" { + r = strings.NewReader(test.text) + } else { + r = newReader(test.text) + } + n, err := scan(r, test.in) + if err != nil { + m := "" + if n > 0 { + m = Sprintf(" (%d fields ok)", n) + } + t.Errorf("%s got error scanning %q: %s%s", name, test.text, err, m) + continue + } + if n != 1 { + t.Errorf("%s count error on entry %q: got %d", name, test.text, n) + continue + } + // The incoming value may be a pointer + v := reflect.ValueOf(test.in) + if p := v; p.Kind() == reflect.Ptr { + v = p.Elem() + } + val := v.Interface() + if !reflect.DeepEqual(val, test.out) { + t.Errorf("%s scanning %q: expected %v got %v, type %T", name, test.text, test.out, val, val) + } + } +} + +func TestScan(t *testing.T) { + testScan("StringReader", t, Fscan) +} + +func TestMyReaderScan(t *testing.T) { + testScan("myStringReader", t, Fscan) +} + +func TestScanln(t *testing.T) { + testScan("StringReader", t, Fscanln) +} + +func TestMyReaderScanln(t *testing.T) { + testScan("myStringReader", t, Fscanln) +} + +func TestScanf(t *testing.T) { + for _, test := range scanfTests { + n, err := Sscanf(test.text, test.format, test.in) + if err != nil { + t.Errorf("got error scanning (%q, %q): %s", test.format, test.text, err) + continue + } + if n != 1 { + t.Errorf("count error on entry (%q, %q): got %d", test.format, test.text, n) + continue + } + // The incoming value may be a pointer + v := reflect.ValueOf(test.in) + if p := v; p.Kind() == reflect.Ptr { + v = p.Elem() + } + val := v.Interface() + if !reflect.DeepEqual(val, test.out) { + t.Errorf("scanning (%q, %q): expected %v got %v, type %T", test.format, test.text, test.out, val, val) + } + } +} + +func TestScanOverflow(t *testing.T) { + // different machines and different types report errors with different strings. + re := regexp.MustCompile("overflow|too large|out of range|not representable") + for _, test := range overflowTests { + _, err := Sscan(test.text, test.in) + if err == nil { + t.Errorf("expected overflow scanning %q", test.text) + continue + } + if !re.MatchString(err.String()) { + t.Errorf("expected overflow error scanning %q: %s", test.text, err) + } + } +} + +func verifyNaN(str string, t *testing.T) { + var f float64 + var f32 float32 + var f64 float64 + text := str + " " + str + " " + str + n, err := Fscan(strings.NewReader(text), &f, &f32, &f64) + if err != nil { + t.Errorf("got error scanning %q: %s", text, err) + } + if n != 3 { + t.Errorf("count error scanning %q: got %d", text, n) + } + if !math.IsNaN(float64(f)) || !math.IsNaN(float64(f32)) || !math.IsNaN(f64) { + t.Errorf("didn't get NaNs scanning %q: got %g %g %g", text, f, f32, f64) + } +} + +func TestNaN(t *testing.T) { + for _, s := range []string{"nan", "NAN", "NaN"} { + verifyNaN(s, t) + } +} + +func verifyInf(str string, t *testing.T) { + var f float64 + var f32 float32 + var f64 float64 + text := str + " " + str + " " + str + n, err := Fscan(strings.NewReader(text), &f, &f32, &f64) + if err != nil { + t.Errorf("got error scanning %q: %s", text, err) + } + if n != 3 { + t.Errorf("count error scanning %q: got %d", text, n) + } + sign := 1 + if str[0] == '-' { + sign = -1 + } + if !math.IsInf(float64(f), sign) || !math.IsInf(float64(f32), sign) || !math.IsInf(f64, sign) { + t.Errorf("didn't get right Infs scanning %q: got %g %g %g", text, f, f32, f64) + } +} + +func TestInf(t *testing.T) { + for _, s := range []string{"inf", "+inf", "-inf", "INF", "-INF", "+INF", "Inf", "-Inf", "+Inf"} { + verifyInf(s, t) + } +} + +func testScanfMulti(name string, t *testing.T) { + sliceType := reflect.TypeOf(make([]interface{}, 1)) + for _, test := range multiTests { + var r io.Reader + if name == "StringReader" { + r = strings.NewReader(test.text) + } else { + r = newReader(test.text) + } + n, err := Fscanf(r, test.format, test.in...) + if err != nil { + if test.err == "" { + t.Errorf("got error scanning (%q, %q): %q", test.format, test.text, err) + } else if strings.Index(err.String(), test.err) < 0 { + t.Errorf("got wrong error scanning (%q, %q): %q; expected %q", test.format, test.text, err, test.err) + } + continue + } + if test.err != "" { + t.Errorf("expected error %q error scanning (%q, %q)", test.err, test.format, test.text) + } + if n != len(test.out) { + t.Errorf("count error on entry (%q, %q): expected %d got %d", test.format, test.text, len(test.out), n) + continue + } + // Convert the slice of pointers into a slice of values + resultVal := reflect.MakeSlice(sliceType, n, n) + for i := 0; i < n; i++ { + v := reflect.ValueOf(test.in[i]).Elem() + resultVal.Index(i).Set(v) + } + result := resultVal.Interface() + if !reflect.DeepEqual(result, test.out) { + t.Errorf("scanning (%q, %q): expected %v got %v", test.format, test.text, test.out, result) + } + } +} + +func TestScanfMulti(t *testing.T) { + testScanfMulti("StringReader", t) +} + +func TestMyReaderScanfMulti(t *testing.T) { + testScanfMulti("myStringReader", t) +} + +func TestScanMultiple(t *testing.T) { + var a int + var s string + n, err := Sscan("123abc", &a, &s) + if n != 2 { + t.Errorf("Sscan count error: expected 2: got %d", n) + } + if err != nil { + t.Errorf("Sscan expected no error; got %s", err) + } + if a != 123 || s != "abc" { + t.Errorf("Sscan wrong values: got (%d %q) expected (123 \"abc\")", a, s) + } + n, err = Sscan("asdf", &s, &a) + if n != 1 { + t.Errorf("Sscan count error: expected 1: got %d", n) + } + if err == nil { + t.Errorf("Sscan expected error; got none: %s", err) + } + if s != "asdf" { + t.Errorf("Sscan wrong values: got %q expected \"asdf\"", s) + } +} + +// Empty strings are not valid input when scanning a string. +func TestScanEmpty(t *testing.T) { + var s1, s2 string + n, err := Sscan("abc", &s1, &s2) + if n != 1 { + t.Errorf("Sscan count error: expected 1: got %d", n) + } + if err == nil { + t.Error("Sscan expected error; got none") + } + if s1 != "abc" { + t.Errorf("Sscan wrong values: got %q expected \"abc\"", s1) + } + n, err = Sscan("", &s1, &s2) + if n != 0 { + t.Errorf("Sscan count error: expected 0: got %d", n) + } + if err == nil { + t.Error("Sscan expected error; got none") + } + // Quoted empty string is OK. + n, err = Sscanf(`""`, "%q", &s1) + if n != 1 { + t.Errorf("Sscanf count error: expected 1: got %d", n) + } + if err != nil { + t.Errorf("Sscanf expected no error with quoted string; got %s", err) + } +} + +func TestScanNotPointer(t *testing.T) { + r := strings.NewReader("1") + var a int + _, err := Fscan(r, a) + if err == nil { + t.Error("expected error scanning non-pointer") + } else if strings.Index(err.String(), "pointer") < 0 { + t.Errorf("expected pointer error scanning non-pointer, got: %s", err) + } +} + +func TestScanlnNoNewline(t *testing.T) { + var a int + _, err := Sscanln("1 x\n", &a) + if err == nil { + t.Error("expected error scanning string missing newline") + } else if strings.Index(err.String(), "newline") < 0 { + t.Errorf("expected newline error scanning string missing newline, got: %s", err) + } +} + +func TestScanlnWithMiddleNewline(t *testing.T) { + r := strings.NewReader("123\n456\n") + var a, b int + _, err := Fscanln(r, &a, &b) + if err == nil { + t.Error("expected error scanning string with extra newline") + } else if strings.Index(err.String(), "newline") < 0 { + t.Errorf("expected newline error scanning string with extra newline, got: %s", err) + } +} + +// Special Reader that counts reads at end of file. +type eofCounter struct { + reader *strings.Reader + eofCount int +} + +func (ec *eofCounter) Read(b []byte) (n int, err os.Error) { + n, err = ec.reader.Read(b) + if n == 0 { + ec.eofCount++ + } + return +} + +// Verify that when we scan, we see at most EOF once per call to a Scan function, +// and then only when it's really an EOF +func TestEOF(t *testing.T) { + ec := &eofCounter{strings.NewReader("123\n"), 0} + var a int + n, err := Fscanln(ec, &a) + if err != nil { + t.Error("unexpected error", err) + } + if n != 1 { + t.Error("expected to scan one item, got", n) + } + if ec.eofCount != 0 { + t.Error("expected zero EOFs", ec.eofCount) + ec.eofCount = 0 // reset for next test + } + n, err = Fscanln(ec, &a) + if err == nil { + t.Error("expected error scanning empty string") + } + if n != 0 { + t.Error("expected to scan zero items, got", n) + } + if ec.eofCount != 1 { + t.Error("expected one EOF, got", ec.eofCount) + } +} + +// Verify that we see an EOF error if we run out of input. +// This was a buglet: we used to get "expected integer". +func TestEOFAtEndOfInput(t *testing.T) { + var i, j int + n, err := Sscanf("23", "%d %d", &i, &j) + if n != 1 || i != 23 { + t.Errorf("Sscanf expected one value of 23; got %d %d", n, i) + } + if err != os.EOF { + t.Errorf("Sscanf expected EOF; got %q", err) + } + n, err = Sscan("234", &i, &j) + if n != 1 || i != 234 { + t.Errorf("Sscan expected one value of 234; got %d %d", n, i) + } + if err != os.EOF { + t.Errorf("Sscan expected EOF; got %q", err) + } + // Trailing space is tougher. + n, err = Sscan("234 ", &i, &j) + if n != 1 || i != 234 { + t.Errorf("Sscan expected one value of 234; got %d %d", n, i) + } + if err != os.EOF { + t.Errorf("Sscan expected EOF; got %q", err) + } +} + +var eofTests = []struct { + format string + v interface{} +}{ + {"%s", &stringVal}, + {"%q", &stringVal}, + {"%x", &stringVal}, + {"%v", &stringVal}, + {"%v", &bytesVal}, + {"%v", &intVal}, + {"%v", &uintVal}, + {"%v", &boolVal}, + {"%v", &float32Val}, + {"%v", &complex64Val}, + {"%v", &renamedStringVal}, + {"%v", &renamedBytesVal}, + {"%v", &renamedIntVal}, + {"%v", &renamedUintVal}, + {"%v", &renamedBoolVal}, + {"%v", &renamedFloat32Val}, + {"%v", &renamedComplex64Val}, +} + +func TestEOFAllTypes(t *testing.T) { + for i, test := range eofTests { + if _, err := Sscanf("", test.format, test.v); err != os.EOF { + t.Errorf("#%d: %s %T not eof on empty string: %s", i, test.format, test.v, err) + } + if _, err := Sscanf(" ", test.format, test.v); err != os.EOF { + t.Errorf("#%d: %s %T not eof on trailing blanks: %s", i, test.format, test.v, err) + } + } +} + +// Verify that, at least when using bufio, successive calls to Fscan do not lose runes. +func TestUnreadRuneWithBufio(t *testing.T) { + r := bufio.NewReader(strings.NewReader("123αb")) + var i int + var a string + n, err := Fscanf(r, "%d", &i) + if n != 1 || err != nil { + t.Errorf("reading int expected one item, no errors; got %d %q", n, err) + } + if i != 123 { + t.Errorf("expected 123; got %d", i) + } + n, err = Fscanf(r, "%s", &a) + if n != 1 || err != nil { + t.Errorf("reading string expected one item, no errors; got %d %q", n, err) + } + if a != "αb" { + t.Errorf("expected αb; got %q", a) + } +} + +type TwoLines string + +// Attempt to read two lines into the object. Scanln should prevent this +// because it stops at newline; Scan and Scanf should be fine. +func (t *TwoLines) Scan(state ScanState, verb int) os.Error { + chars := make([]int, 0, 100) + for nlCount := 0; nlCount < 2; { + c, _, err := state.ReadRune() + if err != nil { + return err + } + chars = append(chars, c) + if c == '\n' { + nlCount++ + } + } + *t = TwoLines(string(chars)) + return nil +} + +func TestMultiLine(t *testing.T) { + input := "abc\ndef\n" + // Sscan should work + var tscan TwoLines + n, err := Sscan(input, &tscan) + if n != 1 { + t.Errorf("Sscan: expected 1 item; got %d", n) + } + if err != nil { + t.Errorf("Sscan: expected no error; got %s", err) + } + if string(tscan) != input { + t.Errorf("Sscan: expected %q; got %q", input, tscan) + } + // Sscanf should work + var tscanf TwoLines + n, err = Sscanf(input, "%s", &tscanf) + if n != 1 { + t.Errorf("Sscanf: expected 1 item; got %d", n) + } + if err != nil { + t.Errorf("Sscanf: expected no error; got %s", err) + } + if string(tscanf) != input { + t.Errorf("Sscanf: expected %q; got %q", input, tscanf) + } + // Sscanln should not work + var tscanln TwoLines + n, err = Sscanln(input, &tscanln) + if n != 0 { + t.Errorf("Sscanln: expected 0 items; got %d: %q", n, tscanln) + } + if err == nil { + t.Error("Sscanln: expected error; got none") + } else if err != io.ErrUnexpectedEOF { + t.Errorf("Sscanln: expected io.ErrUnexpectedEOF (ha!); got %s", err) + } +} + +// RecursiveInt accepts an string matching %d.%d.%d.... +// and parses it into a linked list. +// It allows us to benchmark recursive descent style scanners. +type RecursiveInt struct { + i int + next *RecursiveInt +} + +func (r *RecursiveInt) Scan(state ScanState, verb int) (err os.Error) { + _, err = Fscan(state, &r.i) + if err != nil { + return + } + next := new(RecursiveInt) + _, err = Fscanf(state, ".%v", next) + if err != nil { + if err == os.NewError("input does not match format") || err == io.ErrUnexpectedEOF { + err = nil + } + return + } + r.next = next + return +} + +// Perform the same scanning task as RecursiveInt.Scan +// but without recurring through scanner, so we can compare +// performance more directly. +func scanInts(r *RecursiveInt, b *bytes.Buffer) (err os.Error) { + r.next = nil + _, err = Fscan(b, &r.i) + if err != nil { + return + } + var c int + c, _, err = b.ReadRune() + if err != nil { + if err == os.EOF { + err = nil + } + return + } + if c != '.' { + return + } + next := new(RecursiveInt) + err = scanInts(next, b) + if err == nil { + r.next = next + } + return +} + +func makeInts(n int) []byte { + var buf bytes.Buffer + Fprintf(&buf, "1") + for i := 1; i < n; i++ { + Fprintf(&buf, ".%d", i+1) + } + return buf.Bytes() +} + +func TestScanInts(t *testing.T) { + testScanInts(t, scanInts) + testScanInts(t, func(r *RecursiveInt, b *bytes.Buffer) (err os.Error) { + _, err = Fscan(b, r) + return + }) +} + +// 800 is small enough to not overflow the stack when using gccgo on a +// platform that does not support split stack. +const intCount = 800 + +func testScanInts(t *testing.T, scan func(*RecursiveInt, *bytes.Buffer) os.Error) { + r := new(RecursiveInt) + ints := makeInts(intCount) + buf := bytes.NewBuffer(ints) + err := scan(r, buf) + if err != nil { + t.Error("unexpected error", err) + } + i := 1 + for ; r != nil; r = r.next { + if r.i != i { + t.Fatalf("bad scan: expected %d got %d", i, r.i) + } + i++ + } + if i-1 != intCount { + t.Fatalf("bad scan count: expected %d got %d", intCount, i-1) + } +} + +func BenchmarkScanInts(b *testing.B) { + b.ResetTimer() + ints := makeInts(intCount) + var r RecursiveInt + for i := b.N - 1; i >= 0; i-- { + buf := bytes.NewBuffer(ints) + b.StartTimer() + scanInts(&r, buf) + b.StopTimer() + } +} + +func BenchmarkScanRecursiveInt(b *testing.B) { + b.ResetTimer() + ints := makeInts(intCount) + var r RecursiveInt + for i := b.N - 1; i >= 0; i-- { + buf := bytes.NewBuffer(ints) + b.StartTimer() + Fscan(buf, &r) + b.StopTimer() + } +} diff --git a/src/pkg/fmt/stringer_test.go b/src/pkg/fmt/stringer_test.go new file mode 100644 index 000000000..0ca3f522d --- /dev/null +++ b/src/pkg/fmt/stringer_test.go @@ -0,0 +1,61 @@ +// Copyright 2010 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package fmt_test + +import ( + . "fmt" + "testing" +) + +type TI int +type TI8 int8 +type TI16 int16 +type TI32 int32 +type TI64 int64 +type TU uint +type TU8 uint8 +type TU16 uint16 +type TU32 uint32 +type TU64 uint64 +type TUI uintptr +type TF float64 +type TF32 float32 +type TF64 float64 +type TB bool +type TS string + +func (v TI) String() string { return Sprintf("I: %d", int(v)) } +func (v TI8) String() string { return Sprintf("I8: %d", int8(v)) } +func (v TI16) String() string { return Sprintf("I16: %d", int16(v)) } +func (v TI32) String() string { return Sprintf("I32: %d", int32(v)) } +func (v TI64) String() string { return Sprintf("I64: %d", int64(v)) } +func (v TU) String() string { return Sprintf("U: %d", uint(v)) } +func (v TU8) String() string { return Sprintf("U8: %d", uint8(v)) } +func (v TU16) String() string { return Sprintf("U16: %d", uint16(v)) } +func (v TU32) String() string { return Sprintf("U32: %d", uint32(v)) } +func (v TU64) String() string { return Sprintf("U64: %d", uint64(v)) } +func (v TUI) String() string { return Sprintf("UI: %d", uintptr(v)) } +func (v TF) String() string { return Sprintf("F: %f", float64(v)) } +func (v TF32) String() string { return Sprintf("F32: %f", float32(v)) } +func (v TF64) String() string { return Sprintf("F64: %f", float64(v)) } +func (v TB) String() string { return Sprintf("B: %t", bool(v)) } +func (v TS) String() string { return Sprintf("S: %q", string(v)) } + +func check(t *testing.T, got, want string) { + if got != want { + t.Error(got, "!=", want) + } +} + +func TestStringer(t *testing.T) { + s := Sprintf("%v %v %v %v %v", TI(0), TI8(1), TI16(2), TI32(3), TI64(4)) + check(t, s, "I: 0 I8: 1 I16: 2 I32: 3 I64: 4") + s = Sprintf("%v %v %v %v %v %v", TU(5), TU8(6), TU16(7), TU32(8), TU64(9), TUI(10)) + check(t, s, "U: 5 U8: 6 U16: 7 U32: 8 U64: 9 UI: 10") + s = Sprintf("%v %v %v", TF(1.0), TF32(2.0), TF64(3.0)) + check(t, s, "F: 1.000000 F32: 2.000000 F64: 3.000000") + s = Sprintf("%v %v", TB(true), TS("x")) + check(t, s, "B: true S: \"x\"") +} diff --git a/src/pkg/go/ast/Makefile b/src/pkg/go/ast/Makefile new file mode 100644 index 000000000..40be10208 --- /dev/null +++ b/src/pkg/go/ast/Makefile @@ -0,0 +1,16 @@ +# Copyright 2009 The Go Authors. All rights reserved. +# Use of this source code is governed by a BSD-style +# license that can be found in the LICENSE file. + +include ../../../Make.inc + +TARG=go/ast +GOFILES=\ + ast.go\ + filter.go\ + print.go\ + resolve.go\ + scope.go\ + walk.go\ + +include ../../../Make.pkg diff --git a/src/pkg/go/ast/ast.go b/src/pkg/go/ast/ast.go new file mode 100644 index 000000000..22bd5ee22 --- /dev/null +++ b/src/pkg/go/ast/ast.go @@ -0,0 +1,914 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Package ast declares the types used to represent syntax trees for Go +// packages. +// +package ast + +import ( + "go/token" + "unicode" + "utf8" +) + +// ---------------------------------------------------------------------------- +// Interfaces +// +// There are 3 main classes of nodes: Expressions and type nodes, +// statement nodes, and declaration nodes. The node names usually +// match the corresponding Go spec production names to which they +// correspond. The node fields correspond to the individual parts +// of the respective productions. +// +// All nodes contain position information marking the beginning of +// the corresponding source text segment; it is accessible via the +// Pos accessor method. Nodes may contain additional position info +// for language constructs where comments may be found between parts +// of the construct (typically any larger, parenthesized subpart). +// That position information is needed to properly position comments +// when printing the construct. + +// All node types implement the Node interface. +type Node interface { + Pos() token.Pos // position of first character belonging to the node + End() token.Pos // position of first character immediately after the node +} + +// All expression nodes implement the Expr interface. +type Expr interface { + Node + exprNode() +} + +// All statement nodes implement the Stmt interface. +type Stmt interface { + Node + stmtNode() +} + +// All declaration nodes implement the Decl interface. +type Decl interface { + Node + declNode() +} + +// ---------------------------------------------------------------------------- +// Comments + +// A Comment node represents a single //-style or /*-style comment. +type Comment struct { + Slash token.Pos // position of "/" starting the comment + Text string // comment text (excluding '\n' for //-style comments) +} + +func (c *Comment) Pos() token.Pos { return c.Slash } +func (c *Comment) End() token.Pos { return token.Pos(int(c.Slash) + len(c.Text)) } + +// A CommentGroup represents a sequence of comments +// with no other tokens and no empty lines between. +// +type CommentGroup struct { + List []*Comment // len(List) > 0 +} + +func (g *CommentGroup) Pos() token.Pos { return g.List[0].Pos() } +func (g *CommentGroup) End() token.Pos { return g.List[len(g.List)-1].End() } + +// ---------------------------------------------------------------------------- +// Expressions and types + +// A Field represents a Field declaration list in a struct type, +// a method list in an interface type, or a parameter/result declaration +// in a signature. +// +type Field struct { + Doc *CommentGroup // associated documentation; or nil + Names []*Ident // field/method/parameter names; or nil if anonymous field + Type Expr // field/method/parameter type + Tag *BasicLit // field tag; or nil + Comment *CommentGroup // line comments; or nil +} + +func (f *Field) Pos() token.Pos { + if len(f.Names) > 0 { + return f.Names[0].Pos() + } + return f.Type.Pos() +} + +func (f *Field) End() token.Pos { + if f.Tag != nil { + return f.Tag.End() + } + return f.Type.End() +} + +// A FieldList represents a list of Fields, enclosed by parentheses or braces. +type FieldList struct { + Opening token.Pos // position of opening parenthesis/brace, if any + List []*Field // field list; or nil + Closing token.Pos // position of closing parenthesis/brace, if any +} + +func (f *FieldList) Pos() token.Pos { + if f.Opening.IsValid() { + return f.Opening + } + // the list should not be empty in this case; + // be conservative and guard against bad ASTs + if len(f.List) > 0 { + return f.List[0].Pos() + } + return token.NoPos +} + +func (f *FieldList) End() token.Pos { + if f.Closing.IsValid() { + return f.Closing + 1 + } + // the list should not be empty in this case; + // be conservative and guard against bad ASTs + if n := len(f.List); n > 0 { + return f.List[n-1].End() + } + return token.NoPos +} + +// NumFields returns the number of (named and anonymous fields) in a FieldList. +func (f *FieldList) NumFields() int { + n := 0 + if f != nil { + for _, g := range f.List { + m := len(g.Names) + if m == 0 { + m = 1 // anonymous field + } + n += m + } + } + return n +} + +// An expression is represented by a tree consisting of one +// or more of the following concrete expression nodes. +// +type ( + // A BadExpr node is a placeholder for expressions containing + // syntax errors for which no correct expression nodes can be + // created. + // + BadExpr struct { + From, To token.Pos // position range of bad expression + } + + // An Ident node represents an identifier. + Ident struct { + NamePos token.Pos // identifier position + Name string // identifier name + Obj *Object // denoted object; or nil + } + + // An Ellipsis node stands for the "..." type in a + // parameter list or the "..." length in an array type. + // + Ellipsis struct { + Ellipsis token.Pos // position of "..." + Elt Expr // ellipsis element type (parameter lists only); or nil + } + + // A BasicLit node represents a literal of basic type. + BasicLit struct { + ValuePos token.Pos // literal position + Kind token.Token // token.INT, token.FLOAT, token.IMAG, token.CHAR, or token.STRING + Value string // literal string; e.g. 42, 0x7f, 3.14, 1e-9, 2.4i, 'a', '\x7f', "foo" or `\m\n\o` + } + + // A FuncLit node represents a function literal. + FuncLit struct { + Type *FuncType // function type + Body *BlockStmt // function body + } + + // A CompositeLit node represents a composite literal. + CompositeLit struct { + Type Expr // literal type; or nil + Lbrace token.Pos // position of "{" + Elts []Expr // list of composite elements; or nil + Rbrace token.Pos // position of "}" + } + + // A ParenExpr node represents a parenthesized expression. + ParenExpr struct { + Lparen token.Pos // position of "(" + X Expr // parenthesized expression + Rparen token.Pos // position of ")" + } + + // A SelectorExpr node represents an expression followed by a selector. + SelectorExpr struct { + X Expr // expression + Sel *Ident // field selector + } + + // An IndexExpr node represents an expression followed by an index. + IndexExpr struct { + X Expr // expression + Lbrack token.Pos // position of "[" + Index Expr // index expression + Rbrack token.Pos // position of "]" + } + + // An SliceExpr node represents an expression followed by slice indices. + SliceExpr struct { + X Expr // expression + Lbrack token.Pos // position of "[" + Low Expr // begin of slice range; or nil + High Expr // end of slice range; or nil + Rbrack token.Pos // position of "]" + } + + // A TypeAssertExpr node represents an expression followed by a + // type assertion. + // + TypeAssertExpr struct { + X Expr // expression + Type Expr // asserted type; nil means type switch X.(type) + } + + // A CallExpr node represents an expression followed by an argument list. + CallExpr struct { + Fun Expr // function expression + Lparen token.Pos // position of "(" + Args []Expr // function arguments; or nil + Ellipsis token.Pos // position of "...", if any + Rparen token.Pos // position of ")" + } + + // A StarExpr node represents an expression of the form "*" Expression. + // Semantically it could be a unary "*" expression, or a pointer type. + // + StarExpr struct { + Star token.Pos // position of "*" + X Expr // operand + } + + // A UnaryExpr node represents a unary expression. + // Unary "*" expressions are represented via StarExpr nodes. + // + UnaryExpr struct { + OpPos token.Pos // position of Op + Op token.Token // operator + X Expr // operand + } + + // A BinaryExpr node represents a binary expression. + BinaryExpr struct { + X Expr // left operand + OpPos token.Pos // position of Op + Op token.Token // operator + Y Expr // right operand + } + + // A KeyValueExpr node represents (key : value) pairs + // in composite literals. + // + KeyValueExpr struct { + Key Expr + Colon token.Pos // position of ":" + Value Expr + } +) + +// The direction of a channel type is indicated by one +// of the following constants. +// +type ChanDir int + +const ( + SEND ChanDir = 1 << iota + RECV +) + +// A type is represented by a tree consisting of one +// or more of the following type-specific expression +// nodes. +// +type ( + // An ArrayType node represents an array or slice type. + ArrayType struct { + Lbrack token.Pos // position of "[" + Len Expr // Ellipsis node for [...]T array types, nil for slice types + Elt Expr // element type + } + + // A StructType node represents a struct type. + StructType struct { + Struct token.Pos // position of "struct" keyword + Fields *FieldList // list of field declarations + Incomplete bool // true if (source) fields are missing in the Fields list + } + + // Pointer types are represented via StarExpr nodes. + + // A FuncType node represents a function type. + FuncType struct { + Func token.Pos // position of "func" keyword + Params *FieldList // (incoming) parameters; or nil + Results *FieldList // (outgoing) results; or nil + } + + // An InterfaceType node represents an interface type. + InterfaceType struct { + Interface token.Pos // position of "interface" keyword + Methods *FieldList // list of methods + Incomplete bool // true if (source) methods are missing in the Methods list + } + + // A MapType node represents a map type. + MapType struct { + Map token.Pos // position of "map" keyword + Key Expr + Value Expr + } + + // A ChanType node represents a channel type. + ChanType struct { + Begin token.Pos // position of "chan" keyword or "<-" (whichever comes first) + Dir ChanDir // channel direction + Value Expr // value type + } +) + +// Pos and End implementations for expression/type nodes. +// +func (x *BadExpr) Pos() token.Pos { return x.From } +func (x *Ident) Pos() token.Pos { return x.NamePos } +func (x *Ellipsis) Pos() token.Pos { return x.Ellipsis } +func (x *BasicLit) Pos() token.Pos { return x.ValuePos } +func (x *FuncLit) Pos() token.Pos { return x.Type.Pos() } +func (x *CompositeLit) Pos() token.Pos { + if x.Type != nil { + return x.Type.Pos() + } + return x.Lbrace +} +func (x *ParenExpr) Pos() token.Pos { return x.Lparen } +func (x *SelectorExpr) Pos() token.Pos { return x.X.Pos() } +func (x *IndexExpr) Pos() token.Pos { return x.X.Pos() } +func (x *SliceExpr) Pos() token.Pos { return x.X.Pos() } +func (x *TypeAssertExpr) Pos() token.Pos { return x.X.Pos() } +func (x *CallExpr) Pos() token.Pos { return x.Fun.Pos() } +func (x *StarExpr) Pos() token.Pos { return x.Star } +func (x *UnaryExpr) Pos() token.Pos { return x.OpPos } +func (x *BinaryExpr) Pos() token.Pos { return x.X.Pos() } +func (x *KeyValueExpr) Pos() token.Pos { return x.Key.Pos() } +func (x *ArrayType) Pos() token.Pos { return x.Lbrack } +func (x *StructType) Pos() token.Pos { return x.Struct } +func (x *FuncType) Pos() token.Pos { return x.Func } +func (x *InterfaceType) Pos() token.Pos { return x.Interface } +func (x *MapType) Pos() token.Pos { return x.Map } +func (x *ChanType) Pos() token.Pos { return x.Begin } + +func (x *BadExpr) End() token.Pos { return x.To } +func (x *Ident) End() token.Pos { return token.Pos(int(x.NamePos) + len(x.Name)) } +func (x *Ellipsis) End() token.Pos { + if x.Elt != nil { + return x.Elt.End() + } + return x.Ellipsis + 3 // len("...") +} +func (x *BasicLit) End() token.Pos { return token.Pos(int(x.ValuePos) + len(x.Value)) } +func (x *FuncLit) End() token.Pos { return x.Body.End() } +func (x *CompositeLit) End() token.Pos { return x.Rbrace + 1 } +func (x *ParenExpr) End() token.Pos { return x.Rparen + 1 } +func (x *SelectorExpr) End() token.Pos { return x.Sel.End() } +func (x *IndexExpr) End() token.Pos { return x.Rbrack + 1 } +func (x *SliceExpr) End() token.Pos { return x.Rbrack + 1 } +func (x *TypeAssertExpr) End() token.Pos { + if x.Type != nil { + return x.Type.End() + } + return x.X.End() +} +func (x *CallExpr) End() token.Pos { return x.Rparen + 1 } +func (x *StarExpr) End() token.Pos { return x.X.End() } +func (x *UnaryExpr) End() token.Pos { return x.X.End() } +func (x *BinaryExpr) End() token.Pos { return x.Y.End() } +func (x *KeyValueExpr) End() token.Pos { return x.Value.End() } +func (x *ArrayType) End() token.Pos { return x.Elt.End() } +func (x *StructType) End() token.Pos { return x.Fields.End() } +func (x *FuncType) End() token.Pos { + if x.Results != nil { + return x.Results.End() + } + return x.Params.End() +} +func (x *InterfaceType) End() token.Pos { return x.Methods.End() } +func (x *MapType) End() token.Pos { return x.Value.End() } +func (x *ChanType) End() token.Pos { return x.Value.End() } + +// exprNode() ensures that only expression/type nodes can be +// assigned to an ExprNode. +// +func (x *BadExpr) exprNode() {} +func (x *Ident) exprNode() {} +func (x *Ellipsis) exprNode() {} +func (x *BasicLit) exprNode() {} +func (x *FuncLit) exprNode() {} +func (x *CompositeLit) exprNode() {} +func (x *ParenExpr) exprNode() {} +func (x *SelectorExpr) exprNode() {} +func (x *IndexExpr) exprNode() {} +func (x *SliceExpr) exprNode() {} +func (x *TypeAssertExpr) exprNode() {} +func (x *CallExpr) exprNode() {} +func (x *StarExpr) exprNode() {} +func (x *UnaryExpr) exprNode() {} +func (x *BinaryExpr) exprNode() {} +func (x *KeyValueExpr) exprNode() {} + +func (x *ArrayType) exprNode() {} +func (x *StructType) exprNode() {} +func (x *FuncType) exprNode() {} +func (x *InterfaceType) exprNode() {} +func (x *MapType) exprNode() {} +func (x *ChanType) exprNode() {} + +// ---------------------------------------------------------------------------- +// Convenience functions for Idents + +var noPos token.Pos + +// NewIdent creates a new Ident without position. +// Useful for ASTs generated by code other than the Go parser. +// +func NewIdent(name string) *Ident { return &Ident{noPos, name, nil} } + +// IsExported returns whether name is an exported Go symbol +// (i.e., whether it begins with an uppercase letter). +// +func IsExported(name string) bool { + ch, _ := utf8.DecodeRuneInString(name) + return unicode.IsUpper(ch) +} + +// IsExported returns whether id is an exported Go symbol +// (i.e., whether it begins with an uppercase letter). +// +func (id *Ident) IsExported() bool { return IsExported(id.Name) } + +func (id *Ident) String() string { + if id != nil { + return id.Name + } + return "" +} + +// ---------------------------------------------------------------------------- +// Statements + +// A statement is represented by a tree consisting of one +// or more of the following concrete statement nodes. +// +type ( + // A BadStmt node is a placeholder for statements containing + // syntax errors for which no correct statement nodes can be + // created. + // + BadStmt struct { + From, To token.Pos // position range of bad statement + } + + // A DeclStmt node represents a declaration in a statement list. + DeclStmt struct { + Decl Decl + } + + // An EmptyStmt node represents an empty statement. + // The "position" of the empty statement is the position + // of the immediately preceding semicolon. + // + EmptyStmt struct { + Semicolon token.Pos // position of preceding ";" + } + + // A LabeledStmt node represents a labeled statement. + LabeledStmt struct { + Label *Ident + Colon token.Pos // position of ":" + Stmt Stmt + } + + // An ExprStmt node represents a (stand-alone) expression + // in a statement list. + // + ExprStmt struct { + X Expr // expression + } + + // A SendStmt node represents a send statement. + SendStmt struct { + Chan Expr + Arrow token.Pos // position of "<-" + Value Expr + } + + // An IncDecStmt node represents an increment or decrement statement. + IncDecStmt struct { + X Expr + TokPos token.Pos // position of Tok + Tok token.Token // INC or DEC + } + + // An AssignStmt node represents an assignment or + // a short variable declaration. + // + AssignStmt struct { + Lhs []Expr + TokPos token.Pos // position of Tok + Tok token.Token // assignment token, DEFINE + Rhs []Expr + } + + // A GoStmt node represents a go statement. + GoStmt struct { + Go token.Pos // position of "go" keyword + Call *CallExpr + } + + // A DeferStmt node represents a defer statement. + DeferStmt struct { + Defer token.Pos // position of "defer" keyword + Call *CallExpr + } + + // A ReturnStmt node represents a return statement. + ReturnStmt struct { + Return token.Pos // position of "return" keyword + Results []Expr // result expressions; or nil + } + + // A BranchStmt node represents a break, continue, goto, + // or fallthrough statement. + // + BranchStmt struct { + TokPos token.Pos // position of Tok + Tok token.Token // keyword token (BREAK, CONTINUE, GOTO, FALLTHROUGH) + Label *Ident // label name; or nil + } + + // A BlockStmt node represents a braced statement list. + BlockStmt struct { + Lbrace token.Pos // position of "{" + List []Stmt + Rbrace token.Pos // position of "}" + } + + // An IfStmt node represents an if statement. + IfStmt struct { + If token.Pos // position of "if" keyword + Init Stmt // initialization statement; or nil + Cond Expr // condition + Body *BlockStmt + Else Stmt // else branch; or nil + } + + // A CaseClause represents a case of an expression or type switch statement. + CaseClause struct { + Case token.Pos // position of "case" or "default" keyword + List []Expr // list of expressions or types; nil means default case + Colon token.Pos // position of ":" + Body []Stmt // statement list; or nil + } + + // A SwitchStmt node represents an expression switch statement. + SwitchStmt struct { + Switch token.Pos // position of "switch" keyword + Init Stmt // initialization statement; or nil + Tag Expr // tag expression; or nil + Body *BlockStmt // CaseClauses only + } + + // An TypeSwitchStmt node represents a type switch statement. + TypeSwitchStmt struct { + Switch token.Pos // position of "switch" keyword + Init Stmt // initialization statement; or nil + Assign Stmt // x := y.(type) or y.(type) + Body *BlockStmt // CaseClauses only + } + + // A CommClause node represents a case of a select statement. + CommClause struct { + Case token.Pos // position of "case" or "default" keyword + Comm Stmt // send or receive statement; nil means default case + Colon token.Pos // position of ":" + Body []Stmt // statement list; or nil + } + + // An SelectStmt node represents a select statement. + SelectStmt struct { + Select token.Pos // position of "select" keyword + Body *BlockStmt // CommClauses only + } + + // A ForStmt represents a for statement. + ForStmt struct { + For token.Pos // position of "for" keyword + Init Stmt // initialization statement; or nil + Cond Expr // condition; or nil + Post Stmt // post iteration statement; or nil + Body *BlockStmt + } + + // A RangeStmt represents a for statement with a range clause. + RangeStmt struct { + For token.Pos // position of "for" keyword + Key, Value Expr // Value may be nil + TokPos token.Pos // position of Tok + Tok token.Token // ASSIGN, DEFINE + X Expr // value to range over + Body *BlockStmt + } +) + +// Pos and End implementations for statement nodes. +// +func (s *BadStmt) Pos() token.Pos { return s.From } +func (s *DeclStmt) Pos() token.Pos { return s.Decl.Pos() } +func (s *EmptyStmt) Pos() token.Pos { return s.Semicolon } +func (s *LabeledStmt) Pos() token.Pos { return s.Label.Pos() } +func (s *ExprStmt) Pos() token.Pos { return s.X.Pos() } +func (s *SendStmt) Pos() token.Pos { return s.Chan.Pos() } +func (s *IncDecStmt) Pos() token.Pos { return s.X.Pos() } +func (s *AssignStmt) Pos() token.Pos { return s.Lhs[0].Pos() } +func (s *GoStmt) Pos() token.Pos { return s.Go } +func (s *DeferStmt) Pos() token.Pos { return s.Defer } +func (s *ReturnStmt) Pos() token.Pos { return s.Return } +func (s *BranchStmt) Pos() token.Pos { return s.TokPos } +func (s *BlockStmt) Pos() token.Pos { return s.Lbrace } +func (s *IfStmt) Pos() token.Pos { return s.If } +func (s *CaseClause) Pos() token.Pos { return s.Case } +func (s *SwitchStmt) Pos() token.Pos { return s.Switch } +func (s *TypeSwitchStmt) Pos() token.Pos { return s.Switch } +func (s *CommClause) Pos() token.Pos { return s.Case } +func (s *SelectStmt) Pos() token.Pos { return s.Select } +func (s *ForStmt) Pos() token.Pos { return s.For } +func (s *RangeStmt) Pos() token.Pos { return s.For } + +func (s *BadStmt) End() token.Pos { return s.To } +func (s *DeclStmt) End() token.Pos { return s.Decl.End() } +func (s *EmptyStmt) End() token.Pos { + return s.Semicolon + 1 /* len(";") */ +} +func (s *LabeledStmt) End() token.Pos { return s.Stmt.End() } +func (s *ExprStmt) End() token.Pos { return s.X.End() } +func (s *SendStmt) End() token.Pos { return s.Value.End() } +func (s *IncDecStmt) End() token.Pos { + return s.TokPos + 2 /* len("++") */ +} +func (s *AssignStmt) End() token.Pos { return s.Rhs[len(s.Rhs)-1].End() } +func (s *GoStmt) End() token.Pos { return s.Call.End() } +func (s *DeferStmt) End() token.Pos { return s.Call.End() } +func (s *ReturnStmt) End() token.Pos { + if n := len(s.Results); n > 0 { + return s.Results[n-1].End() + } + return s.Return + 6 // len("return") +} +func (s *BranchStmt) End() token.Pos { + if s.Label != nil { + return s.Label.End() + } + return token.Pos(int(s.TokPos) + len(s.Tok.String())) +} +func (s *BlockStmt) End() token.Pos { return s.Rbrace + 1 } +func (s *IfStmt) End() token.Pos { + if s.Else != nil { + return s.Else.End() + } + return s.Body.End() +} +func (s *CaseClause) End() token.Pos { + if n := len(s.Body); n > 0 { + return s.Body[n-1].End() + } + return s.Colon + 1 +} +func (s *SwitchStmt) End() token.Pos { return s.Body.End() } +func (s *TypeSwitchStmt) End() token.Pos { return s.Body.End() } +func (s *CommClause) End() token.Pos { + if n := len(s.Body); n > 0 { + return s.Body[n-1].End() + } + return s.Colon + 1 +} +func (s *SelectStmt) End() token.Pos { return s.Body.End() } +func (s *ForStmt) End() token.Pos { return s.Body.End() } +func (s *RangeStmt) End() token.Pos { return s.Body.End() } + +// stmtNode() ensures that only statement nodes can be +// assigned to a StmtNode. +// +func (s *BadStmt) stmtNode() {} +func (s *DeclStmt) stmtNode() {} +func (s *EmptyStmt) stmtNode() {} +func (s *LabeledStmt) stmtNode() {} +func (s *ExprStmt) stmtNode() {} +func (s *SendStmt) stmtNode() {} +func (s *IncDecStmt) stmtNode() {} +func (s *AssignStmt) stmtNode() {} +func (s *GoStmt) stmtNode() {} +func (s *DeferStmt) stmtNode() {} +func (s *ReturnStmt) stmtNode() {} +func (s *BranchStmt) stmtNode() {} +func (s *BlockStmt) stmtNode() {} +func (s *IfStmt) stmtNode() {} +func (s *CaseClause) stmtNode() {} +func (s *SwitchStmt) stmtNode() {} +func (s *TypeSwitchStmt) stmtNode() {} +func (s *CommClause) stmtNode() {} +func (s *SelectStmt) stmtNode() {} +func (s *ForStmt) stmtNode() {} +func (s *RangeStmt) stmtNode() {} + +// ---------------------------------------------------------------------------- +// Declarations + +// A Spec node represents a single (non-parenthesized) import, +// constant, type, or variable declaration. +// +type ( + // The Spec type stands for any of *ImportSpec, *ValueSpec, and *TypeSpec. + Spec interface { + Node + specNode() + } + + // An ImportSpec node represents a single package import. + ImportSpec struct { + Doc *CommentGroup // associated documentation; or nil + Name *Ident // local package name (including "."); or nil + Path *BasicLit // import path + Comment *CommentGroup // line comments; or nil + } + + // A ValueSpec node represents a constant or variable declaration + // (ConstSpec or VarSpec production). + // + ValueSpec struct { + Doc *CommentGroup // associated documentation; or nil + Names []*Ident // value names (len(Names) > 0) + Type Expr // value type; or nil + Values []Expr // initial values; or nil + Comment *CommentGroup // line comments; or nil + } + + // A TypeSpec node represents a type declaration (TypeSpec production). + TypeSpec struct { + Doc *CommentGroup // associated documentation; or nil + Name *Ident // type name + Type Expr // *Ident, *ParenExpr, *SelectorExpr, *StarExpr, or any of the *XxxTypes + Comment *CommentGroup // line comments; or nil + } +) + +// Pos and End implementations for spec nodes. +// +func (s *ImportSpec) Pos() token.Pos { + if s.Name != nil { + return s.Name.Pos() + } + return s.Path.Pos() +} +func (s *ValueSpec) Pos() token.Pos { return s.Names[0].Pos() } +func (s *TypeSpec) Pos() token.Pos { return s.Name.Pos() } + +func (s *ImportSpec) End() token.Pos { return s.Path.End() } +func (s *ValueSpec) End() token.Pos { + if n := len(s.Values); n > 0 { + return s.Values[n-1].End() + } + if s.Type != nil { + return s.Type.End() + } + return s.Names[len(s.Names)-1].End() +} +func (s *TypeSpec) End() token.Pos { return s.Type.End() } + +// specNode() ensures that only spec nodes can be +// assigned to a Spec. +// +func (s *ImportSpec) specNode() {} +func (s *ValueSpec) specNode() {} +func (s *TypeSpec) specNode() {} + +// A declaration is represented by one of the following declaration nodes. +// +type ( + // A BadDecl node is a placeholder for declarations containing + // syntax errors for which no correct declaration nodes can be + // created. + // + BadDecl struct { + From, To token.Pos // position range of bad declaration + } + + // A GenDecl node (generic declaration node) represents an import, + // constant, type or variable declaration. A valid Lparen position + // (Lparen.Line > 0) indicates a parenthesized declaration. + // + // Relationship between Tok value and Specs element type: + // + // token.IMPORT *ImportSpec + // token.CONST *ValueSpec + // token.TYPE *TypeSpec + // token.VAR *ValueSpec + // + GenDecl struct { + Doc *CommentGroup // associated documentation; or nil + TokPos token.Pos // position of Tok + Tok token.Token // IMPORT, CONST, TYPE, VAR + Lparen token.Pos // position of '(', if any + Specs []Spec + Rparen token.Pos // position of ')', if any + } + + // A FuncDecl node represents a function declaration. + FuncDecl struct { + Doc *CommentGroup // associated documentation; or nil + Recv *FieldList // receiver (methods); or nil (functions) + Name *Ident // function/method name + Type *FuncType // position of Func keyword, parameters and results + Body *BlockStmt // function body; or nil (forward declaration) + } +) + +// Pos and End implementations for declaration nodes. +// +func (d *BadDecl) Pos() token.Pos { return d.From } +func (d *GenDecl) Pos() token.Pos { return d.TokPos } +func (d *FuncDecl) Pos() token.Pos { return d.Type.Pos() } + +func (d *BadDecl) End() token.Pos { return d.To } +func (d *GenDecl) End() token.Pos { + if d.Rparen.IsValid() { + return d.Rparen + 1 + } + return d.Specs[0].End() +} +func (d *FuncDecl) End() token.Pos { + if d.Body != nil { + return d.Body.End() + } + return d.Type.End() +} + +// declNode() ensures that only declaration nodes can be +// assigned to a DeclNode. +// +func (d *BadDecl) declNode() {} +func (d *GenDecl) declNode() {} +func (d *FuncDecl) declNode() {} + +// ---------------------------------------------------------------------------- +// Files and packages + +// A File node represents a Go source file. +// +// The Comments list contains all comments in the source file in order of +// appearance, including the comments that are pointed to from other nodes +// via Doc and Comment fields. +// +type File struct { + Doc *CommentGroup // associated documentation; or nil + Package token.Pos // position of "package" keyword + Name *Ident // package name + Decls []Decl // top-level declarations; or nil + Scope *Scope // package scope (this file only) + Imports []*ImportSpec // imports in this file + Unresolved []*Ident // unresolved identifiers in this file + Comments []*CommentGroup // list of all comments in the source file +} + +func (f *File) Pos() token.Pos { return f.Package } +func (f *File) End() token.Pos { + if n := len(f.Decls); n > 0 { + return f.Decls[n-1].End() + } + return f.Name.End() +} + +// A Package node represents a set of source files +// collectively building a Go package. +// +type Package struct { + Name string // package name + Scope *Scope // package scope across all files + Imports map[string]*Object // map of package id -> package object + Files map[string]*File // Go source files by filename +} + +func (p *Package) Pos() token.Pos { return token.NoPos } +func (p *Package) End() token.Pos { return token.NoPos } diff --git a/src/pkg/go/ast/filter.go b/src/pkg/go/ast/filter.go new file mode 100644 index 000000000..26733430d --- /dev/null +++ b/src/pkg/go/ast/filter.go @@ -0,0 +1,475 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package ast + +import "go/token" + +// ---------------------------------------------------------------------------- +// Export filtering + +func identListExports(list []*Ident) []*Ident { + j := 0 + for _, x := range list { + if x.IsExported() { + list[j] = x + j++ + } + } + return list[0:j] +} + +// fieldName assumes that x is the type of an anonymous field and +// returns the corresponding field name. If x is not an acceptable +// anonymous field, the result is nil. +// +func fieldName(x Expr) *Ident { + switch t := x.(type) { + case *Ident: + return t + case *SelectorExpr: + if _, ok := t.X.(*Ident); ok { + return t.Sel + } + case *StarExpr: + return fieldName(t.X) + } + return nil +} + +func fieldListExports(fields *FieldList) (removedFields bool) { + if fields == nil { + return + } + list := fields.List + j := 0 + for _, f := range list { + exported := false + if len(f.Names) == 0 { + // anonymous field + // (Note that a non-exported anonymous field + // may still refer to a type with exported + // fields, so this is not absolutely correct. + // However, this cannot be done w/o complete + // type information.) + name := fieldName(f.Type) + exported = name != nil && name.IsExported() + } else { + n := len(f.Names) + f.Names = identListExports(f.Names) + if len(f.Names) < n { + removedFields = true + } + exported = len(f.Names) > 0 + } + if exported { + typeExports(f.Type) + list[j] = f + j++ + } + } + if j < len(list) { + removedFields = true + } + fields.List = list[0:j] + return +} + +func paramListExports(fields *FieldList) { + if fields == nil { + return + } + for _, f := range fields.List { + typeExports(f.Type) + } +} + +func typeExports(typ Expr) { + switch t := typ.(type) { + case *ArrayType: + typeExports(t.Elt) + case *StructType: + if fieldListExports(t.Fields) { + t.Incomplete = true + } + case *FuncType: + paramListExports(t.Params) + paramListExports(t.Results) + case *InterfaceType: + if fieldListExports(t.Methods) { + t.Incomplete = true + } + case *MapType: + typeExports(t.Key) + typeExports(t.Value) + case *ChanType: + typeExports(t.Value) + } +} + +func specExports(spec Spec) bool { + switch s := spec.(type) { + case *ValueSpec: + s.Names = identListExports(s.Names) + if len(s.Names) > 0 { + typeExports(s.Type) + return true + } + case *TypeSpec: + if s.Name.IsExported() { + typeExports(s.Type) + return true + } + } + return false +} + +func specListExports(list []Spec) []Spec { + j := 0 + for _, s := range list { + if specExports(s) { + list[j] = s + j++ + } + } + return list[0:j] +} + +func declExports(decl Decl) bool { + switch d := decl.(type) { + case *GenDecl: + d.Specs = specListExports(d.Specs) + return len(d.Specs) > 0 + case *FuncDecl: + d.Body = nil // strip body + return d.Name.IsExported() + } + return false +} + +// FileExports trims the AST for a Go source file in place such that only +// exported nodes remain: all top-level identifiers which are not exported +// and their associated information (such as type, initial value, or function +// body) are removed. Non-exported fields and methods of exported types are +// stripped, and the function bodies of exported functions are set to nil. +// The File.comments list is not changed. +// +// FileExports returns true if there is an exported declaration; it returns +// false otherwise. +// +func FileExports(src *File) bool { + j := 0 + for _, d := range src.Decls { + if declExports(d) { + src.Decls[j] = d + j++ + } + } + src.Decls = src.Decls[0:j] + return j > 0 +} + +// PackageExports trims the AST for a Go package in place such that only +// exported nodes remain. The pkg.Files list is not changed, so that file +// names and top-level package comments don't get lost. +// +// PackageExports returns true if there is an exported declaration; it +// returns false otherwise. +// +func PackageExports(pkg *Package) bool { + hasExports := false + for _, f := range pkg.Files { + if FileExports(f) { + hasExports = true + } + } + return hasExports +} + +// ---------------------------------------------------------------------------- +// General filtering + +type Filter func(string) bool + +func filterIdentList(list []*Ident, f Filter) []*Ident { + j := 0 + for _, x := range list { + if f(x.Name) { + list[j] = x + j++ + } + } + return list[0:j] +} + +func filterFieldList(fields *FieldList, filter Filter) (removedFields bool) { + if fields == nil { + return false + } + list := fields.List + j := 0 + for _, f := range list { + keepField := false + if len(f.Names) == 0 { + // anonymous field + name := fieldName(f.Type) + keepField = name != nil && filter(name.Name) + } else { + n := len(f.Names) + f.Names = filterIdentList(f.Names, filter) + if len(f.Names) < n { + removedFields = true + } + keepField = len(f.Names) > 0 + } + if keepField { + list[j] = f + j++ + } + } + if j < len(list) { + removedFields = true + } + fields.List = list[0:j] + return +} + +func filterSpec(spec Spec, f Filter) bool { + switch s := spec.(type) { + case *ValueSpec: + s.Names = filterIdentList(s.Names, f) + return len(s.Names) > 0 + case *TypeSpec: + if f(s.Name.Name) { + return true + } + switch t := s.Type.(type) { + case *StructType: + if filterFieldList(t.Fields, f) { + t.Incomplete = true + } + return len(t.Fields.List) > 0 + case *InterfaceType: + if filterFieldList(t.Methods, f) { + t.Incomplete = true + } + return len(t.Methods.List) > 0 + } + } + return false +} + +func filterSpecList(list []Spec, f Filter) []Spec { + j := 0 + for _, s := range list { + if filterSpec(s, f) { + list[j] = s + j++ + } + } + return list[0:j] +} + +// FilterDecl trims the AST for a Go declaration in place by removing +// all names (including struct field and interface method names, but +// not from parameter lists) that don't pass through the filter f. +// +// FilterDecl returns true if there are any declared names left after +// filtering; it returns false otherwise. +// +func FilterDecl(decl Decl, f Filter) bool { + switch d := decl.(type) { + case *GenDecl: + d.Specs = filterSpecList(d.Specs, f) + return len(d.Specs) > 0 + case *FuncDecl: + return f(d.Name.Name) + } + return false +} + +// FilterFile trims the AST for a Go file in place by removing all +// names from top-level declarations (including struct field and +// interface method names, but not from parameter lists) that don't +// pass through the filter f. If the declaration is empty afterwards, +// the declaration is removed from the AST. +// The File.comments list is not changed. +// +// FilterFile returns true if there are any top-level declarations +// left after filtering; it returns false otherwise. +// +func FilterFile(src *File, f Filter) bool { + j := 0 + for _, d := range src.Decls { + if FilterDecl(d, f) { + src.Decls[j] = d + j++ + } + } + src.Decls = src.Decls[0:j] + return j > 0 +} + +// FilterPackage trims the AST for a Go package in place by removing all +// names from top-level declarations (including struct field and +// interface method names, but not from parameter lists) that don't +// pass through the filter f. If the declaration is empty afterwards, +// the declaration is removed from the AST. +// The pkg.Files list is not changed, so that file names and top-level +// package comments don't get lost. +// +// FilterPackage returns true if there are any top-level declarations +// left after filtering; it returns false otherwise. +// +func FilterPackage(pkg *Package, f Filter) bool { + hasDecls := false + for _, src := range pkg.Files { + if FilterFile(src, f) { + hasDecls = true + } + } + return hasDecls +} + +// ---------------------------------------------------------------------------- +// Merging of package files + +// The MergeMode flags control the behavior of MergePackageFiles. +type MergeMode uint + +const ( + // If set, duplicate function declarations are excluded. + FilterFuncDuplicates MergeMode = 1 << iota + // If set, comments that are not associated with a specific + // AST node (as Doc or Comment) are excluded. + FilterUnassociatedComments +) + +// separator is an empty //-style comment that is interspersed between +// different comment groups when they are concatenated into a single group +// +var separator = &Comment{noPos, "//"} + +// MergePackageFiles creates a file AST by merging the ASTs of the +// files belonging to a package. The mode flags control merging behavior. +// +func MergePackageFiles(pkg *Package, mode MergeMode) *File { + // Count the number of package docs, comments and declarations across + // all package files. + ndocs := 0 + ncomments := 0 + ndecls := 0 + for _, f := range pkg.Files { + if f.Doc != nil { + ndocs += len(f.Doc.List) + 1 // +1 for separator + } + ncomments += len(f.Comments) + ndecls += len(f.Decls) + } + + // Collect package comments from all package files into a single + // CommentGroup - the collected package documentation. The order + // is unspecified. In general there should be only one file with + // a package comment; but it's better to collect extra comments + // than drop them on the floor. + var doc *CommentGroup + var pos token.Pos + if ndocs > 0 { + list := make([]*Comment, ndocs-1) // -1: no separator before first group + i := 0 + for _, f := range pkg.Files { + if f.Doc != nil { + if i > 0 { + // not the first group - add separator + list[i] = separator + i++ + } + for _, c := range f.Doc.List { + list[i] = c + i++ + } + if f.Package > pos { + // Keep the maximum package clause position as + // position for the package clause of the merged + // files. + pos = f.Package + } + } + } + doc = &CommentGroup{list} + } + + // Collect declarations from all package files. + var decls []Decl + if ndecls > 0 { + decls = make([]Decl, ndecls) + funcs := make(map[string]int) // map of global function name -> decls index + i := 0 // current index + n := 0 // number of filtered entries + for _, f := range pkg.Files { + for _, d := range f.Decls { + if mode&FilterFuncDuplicates != 0 { + // A language entity may be declared multiple + // times in different package files; only at + // build time declarations must be unique. + // For now, exclude multiple declarations of + // functions - keep the one with documentation. + // + // TODO(gri): Expand this filtering to other + // entities (const, type, vars) if + // multiple declarations are common. + if f, isFun := d.(*FuncDecl); isFun { + name := f.Name.Name + if j, exists := funcs[name]; exists { + // function declared already + if decls[j] != nil && decls[j].(*FuncDecl).Doc == nil { + // existing declaration has no documentation; + // ignore the existing declaration + decls[j] = nil + } else { + // ignore the new declaration + d = nil + } + n++ // filtered an entry + } else { + funcs[name] = i + } + } + } + decls[i] = d + i++ + } + } + + // Eliminate nil entries from the decls list if entries were + // filtered. We do this using a 2nd pass in order to not disturb + // the original declaration order in the source (otherwise, this + // would also invalidate the monotonically increasing position + // info within a single file). + if n > 0 { + i = 0 + for _, d := range decls { + if d != nil { + decls[i] = d + i++ + } + } + decls = decls[0:i] + } + } + + // Collect comments from all package files. + var comments []*CommentGroup + if mode&FilterUnassociatedComments == 0 { + comments = make([]*CommentGroup, ncomments) + i := 0 + for _, f := range pkg.Files { + i += copy(comments[i:], f.Comments) + } + } + + // TODO(gri) need to compute pkgScope and unresolved identifiers! + // TODO(gri) need to compute imports! + return &File{doc, pos, NewIdent(pkg.Name), decls, nil, nil, nil, comments} +} diff --git a/src/pkg/go/ast/print.go b/src/pkg/go/ast/print.go new file mode 100644 index 000000000..62a30481d --- /dev/null +++ b/src/pkg/go/ast/print.go @@ -0,0 +1,224 @@ +// Copyright 2010 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// This file contains printing suppport for ASTs. + +package ast + +import ( + "fmt" + "go/token" + "io" + "os" + "reflect" +) + +// A FieldFilter may be provided to Fprint to control the output. +type FieldFilter func(name string, value reflect.Value) bool + +// NotNilFilter returns true for field values that are not nil; +// it returns false otherwise. +func NotNilFilter(_ string, v reflect.Value) bool { + switch v.Kind() { + case reflect.Chan, reflect.Func, reflect.Interface, reflect.Map, reflect.Ptr, reflect.Slice: + return !v.IsNil() + } + return true +} + +// Fprint prints the (sub-)tree starting at AST node x to w. +// If fset != nil, position information is interpreted relative +// to that file set. Otherwise positions are printed as integer +// values (file set specific offsets). +// +// A non-nil FieldFilter f may be provided to control the output: +// struct fields for which f(fieldname, fieldvalue) is true are +// are printed; all others are filtered from the output. +// +func Fprint(w io.Writer, fset *token.FileSet, x interface{}, f FieldFilter) (n int, err os.Error) { + // setup printer + p := printer{ + output: w, + fset: fset, + filter: f, + ptrmap: make(map[interface{}]int), + last: '\n', // force printing of line number on first line + } + + // install error handler + defer func() { + n = p.written + if e := recover(); e != nil { + err = e.(localError).err // re-panics if it's not a localError + } + }() + + // print x + if x == nil { + p.printf("nil\n") + return + } + p.print(reflect.ValueOf(x)) + p.printf("\n") + + return +} + +// Print prints x to standard output, skipping nil fields. +// Print(fset, x) is the same as Fprint(os.Stdout, fset, x, NotNilFilter). +func Print(fset *token.FileSet, x interface{}) (int, os.Error) { + return Fprint(os.Stdout, fset, x, NotNilFilter) +} + +type printer struct { + output io.Writer + fset *token.FileSet + filter FieldFilter + ptrmap map[interface{}]int // *T -> line number + written int // number of bytes written to output + indent int // current indentation level + last byte // the last byte processed by Write + line int // current line number +} + +var indent = []byte(". ") + +func (p *printer) Write(data []byte) (n int, err os.Error) { + var m int + for i, b := range data { + // invariant: data[0:n] has been written + if b == '\n' { + m, err = p.output.Write(data[n : i+1]) + n += m + if err != nil { + return + } + p.line++ + } else if p.last == '\n' { + _, err = fmt.Fprintf(p.output, "%6d ", p.line) + if err != nil { + return + } + for j := p.indent; j > 0; j-- { + _, err = p.output.Write(indent) + if err != nil { + return + } + } + } + p.last = b + } + m, err = p.output.Write(data[n:]) + n += m + return +} + +// localError wraps locally caught os.Errors so we can distinguish +// them from genuine panics which we don't want to return as errors. +type localError struct { + err os.Error +} + +// printf is a convenience wrapper that takes care of print errors. +func (p *printer) printf(format string, args ...interface{}) { + n, err := fmt.Fprintf(p, format, args...) + p.written += n + if err != nil { + panic(localError{err}) + } +} + +// Implementation note: Print is written for AST nodes but could be +// used to print arbitrary data structures; such a version should +// probably be in a different package. +// +// Note: This code detects (some) cycles created via pointers but +// not cycles that are created via slices or maps containing the +// same slice or map. Code for general data structures probably +// should catch those as well. + +func (p *printer) print(x reflect.Value) { + if !NotNilFilter("", x) { + p.printf("nil") + return + } + + switch x.Kind() { + case reflect.Interface: + p.print(x.Elem()) + + case reflect.Map: + p.printf("%s (len = %d) {\n", x.Type().String(), x.Len()) + p.indent++ + for _, key := range x.MapKeys() { + p.print(key) + p.printf(": ") + p.print(x.MapIndex(key)) + p.printf("\n") + } + p.indent-- + p.printf("}") + + case reflect.Ptr: + p.printf("*") + // type-checked ASTs may contain cycles - use ptrmap + // to keep track of objects that have been printed + // already and print the respective line number instead + ptr := x.Interface() + if line, exists := p.ptrmap[ptr]; exists { + p.printf("(obj @ %d)", line) + } else { + p.ptrmap[ptr] = p.line + p.print(x.Elem()) + } + + case reflect.Slice: + if s, ok := x.Interface().([]byte); ok { + p.printf("%#q", s) + return + } + p.printf("%s (len = %d) {\n", x.Type().String(), x.Len()) + p.indent++ + for i, n := 0, x.Len(); i < n; i++ { + p.printf("%d: ", i) + p.print(x.Index(i)) + p.printf("\n") + } + p.indent-- + p.printf("}") + + case reflect.Struct: + p.printf("%s {\n", x.Type().String()) + p.indent++ + t := x.Type() + for i, n := 0, t.NumField(); i < n; i++ { + name := t.Field(i).Name + value := x.Field(i) + if p.filter == nil || p.filter(name, value) { + p.printf("%s: ", name) + p.print(value) + p.printf("\n") + } + } + p.indent-- + p.printf("}") + + default: + v := x.Interface() + switch v := v.(type) { + case string: + // print strings in quotes + p.printf("%q", v) + return + case token.Pos: + // position values can be printed nicely if we have a file set + if p.fset != nil { + p.printf("%s", p.fset.Position(v)) + return + } + } + // default + p.printf("%v", v) + } +} diff --git a/src/pkg/go/ast/print_test.go b/src/pkg/go/ast/print_test.go new file mode 100644 index 000000000..f4e8f7a78 --- /dev/null +++ b/src/pkg/go/ast/print_test.go @@ -0,0 +1,77 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package ast + +import ( + "bytes" + "strings" + "testing" +) + +var tests = []struct { + x interface{} // x is printed as s + s string +}{ + // basic types + {nil, "0 nil"}, + {true, "0 true"}, + {42, "0 42"}, + {3.14, "0 3.14"}, + {1 + 2.718i, "0 (1+2.718i)"}, + {"foobar", "0 \"foobar\""}, + + // maps + {map[string]int{"a": 1, "b": 2}, + `0 map[string] int (len = 2) { + 1 . "a": 1 + 2 . "b": 2 + 3 }`}, + + // pointers + {new(int), "0 *0"}, + + // slices + {[]int{1, 2, 3}, + `0 []int (len = 3) { + 1 . 0: 1 + 2 . 1: 2 + 3 . 2: 3 + 4 }`}, + + // structs + {struct{ x, y int }{42, 991}, + `0 struct { x int; y int } { + 1 . x: 42 + 2 . y: 991 + 3 }`}, +} + +// Split s into lines, trim whitespace from all lines, and return +// the concatenated non-empty lines. +func trim(s string) string { + lines := strings.Split(s, "\n") + i := 0 + for _, line := range lines { + line = strings.TrimSpace(line) + if line != "" { + lines[i] = line + i++ + } + } + return strings.Join(lines[0:i], "\n") +} + +func TestPrint(t *testing.T) { + var buf bytes.Buffer + for _, test := range tests { + buf.Reset() + if _, err := Fprint(&buf, nil, test.x, nil); err != nil { + t.Errorf("Fprint failed: %s", err) + } + if s, ts := trim(buf.String()), trim(test.s); s != ts { + t.Errorf("got:\n%s\nexpected:\n%s\n", s, ts) + } + } +} diff --git a/src/pkg/go/ast/resolve.go b/src/pkg/go/ast/resolve.go new file mode 100644 index 000000000..3927a799e --- /dev/null +++ b/src/pkg/go/ast/resolve.go @@ -0,0 +1,174 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// This file implements NewPackage. + +package ast + +import ( + "fmt" + "go/scanner" + "go/token" + "os" + "strconv" +) + +type pkgBuilder struct { + scanner.ErrorVector + fset *token.FileSet +} + +func (p *pkgBuilder) error(pos token.Pos, msg string) { + p.Error(p.fset.Position(pos), msg) +} + +func (p *pkgBuilder) errorf(pos token.Pos, format string, args ...interface{}) { + p.error(pos, fmt.Sprintf(format, args...)) +} + +func (p *pkgBuilder) declare(scope, altScope *Scope, obj *Object) { + alt := scope.Insert(obj) + if alt == nil && altScope != nil { + // see if there is a conflicting declaration in altScope + alt = altScope.Lookup(obj.Name) + } + if alt != nil { + prevDecl := "" + if pos := alt.Pos(); pos.IsValid() { + prevDecl = fmt.Sprintf("\n\tprevious declaration at %s", p.fset.Position(pos)) + } + p.error(obj.Pos(), fmt.Sprintf("%s redeclared in this block%s", obj.Name, prevDecl)) + } +} + +func resolve(scope *Scope, ident *Ident) bool { + for ; scope != nil; scope = scope.Outer { + if obj := scope.Lookup(ident.Name); obj != nil { + ident.Obj = obj + return true + } + } + return false +} + +// An Importer resolves import paths to package Objects. +// The imports map records the packages already imported, +// indexed by package id (canonical import path). +// An Importer must determine the canonical import path and +// check the map to see if it is already present in the imports map. +// If so, the Importer can return the map entry. Otherwise, the +// Importer should load the package data for the given path into +// a new *Object (pkg), record pkg in the imports map, and then +// return pkg. +type Importer func(imports map[string]*Object, path string) (pkg *Object, err os.Error) + +// NewPackage creates a new Package node from a set of File nodes. It resolves +// unresolved identifiers across files and updates each file's Unresolved list +// accordingly. If a non-nil importer and universe scope are provided, they are +// used to resolve identifiers not declared in any of the package files. Any +// remaining unresolved identifiers are reported as undeclared. If the files +// belong to different packages, one package name is selected and files with +// different package names are reported and then ignored. +// The result is a package node and a scanner.ErrorList if there were errors. +// +func NewPackage(fset *token.FileSet, files map[string]*File, importer Importer, universe *Scope) (*Package, os.Error) { + var p pkgBuilder + p.fset = fset + + // complete package scope + pkgName := "" + pkgScope := NewScope(universe) + for _, file := range files { + // package names must match + switch name := file.Name.Name; { + case pkgName == "": + pkgName = name + case name != pkgName: + p.errorf(file.Package, "package %s; expected %s", name, pkgName) + continue // ignore this file + } + + // collect top-level file objects in package scope + for _, obj := range file.Scope.Objects { + p.declare(pkgScope, nil, obj) + } + } + + // package global mapping of imported package ids to package objects + imports := make(map[string]*Object) + + // complete file scopes with imports and resolve identifiers + for _, file := range files { + // ignore file if it belongs to a different package + // (error has already been reported) + if file.Name.Name != pkgName { + continue + } + + // build file scope by processing all imports + importErrors := false + fileScope := NewScope(pkgScope) + for _, spec := range file.Imports { + if importer == nil { + importErrors = true + continue + } + path, _ := strconv.Unquote(string(spec.Path.Value)) + pkg, err := importer(imports, path) + if err != nil { + p.errorf(spec.Path.Pos(), "could not import %s (%s)", path, err) + importErrors = true + continue + } + // TODO(gri) If a local package name != "." is provided, + // global identifier resolution could proceed even if the + // import failed. Consider adjusting the logic here a bit. + + // local name overrides imported package name + name := pkg.Name + if spec.Name != nil { + name = spec.Name.Name + } + + // add import to file scope + if name == "." { + // merge imported scope with file scope + for _, obj := range pkg.Data.(*Scope).Objects { + p.declare(fileScope, pkgScope, obj) + } + } else { + // declare imported package object in file scope + // (do not re-use pkg in the file scope but create + // a new object instead; the Decl field is different + // for different files) + obj := NewObj(Pkg, name) + obj.Decl = spec + obj.Data = pkg.Data + p.declare(fileScope, pkgScope, obj) + } + } + + // resolve identifiers + if importErrors { + // don't use the universe scope without correct imports + // (objects in the universe may be shadowed by imports; + // with missing imports, identifiers might get resolved + // incorrectly to universe objects) + pkgScope.Outer = nil + } + i := 0 + for _, ident := range file.Unresolved { + if !resolve(fileScope, ident) { + p.errorf(ident.Pos(), "undeclared name: %s", ident.Name) + file.Unresolved[i] = ident + i++ + } + + } + file.Unresolved = file.Unresolved[0:i] + pkgScope.Outer = universe // reset universe scope + } + + return &Package{pkgName, pkgScope, imports, files}, p.GetError(scanner.Sorted) +} diff --git a/src/pkg/go/ast/scope.go b/src/pkg/go/ast/scope.go new file mode 100644 index 000000000..92e366980 --- /dev/null +++ b/src/pkg/go/ast/scope.go @@ -0,0 +1,156 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// This file implements scopes and the objects they contain. + +package ast + +import ( + "bytes" + "fmt" + "go/token" +) + +// A Scope maintains the set of named language entities declared +// in the scope and a link to the immediately surrounding (outer) +// scope. +// +type Scope struct { + Outer *Scope + Objects map[string]*Object +} + +// NewScope creates a new scope nested in the outer scope. +func NewScope(outer *Scope) *Scope { + const n = 4 // initial scope capacity + return &Scope{outer, make(map[string]*Object, n)} +} + +// Lookup returns the object with the given name if it is +// found in scope s, otherwise it returns nil. Outer scopes +// are ignored. +// +func (s *Scope) Lookup(name string) *Object { + return s.Objects[name] +} + +// Insert attempts to insert a named object obj into the scope s. +// If the scope already contains an object alt with the same name, +// Insert leaves the scope unchanged and returns alt. Otherwise +// it inserts obj and returns nil." +// +func (s *Scope) Insert(obj *Object) (alt *Object) { + if alt = s.Objects[obj.Name]; alt == nil { + s.Objects[obj.Name] = obj + } + return +} + +// Debugging support +func (s *Scope) String() string { + var buf bytes.Buffer + fmt.Fprintf(&buf, "scope %p {", s) + if s != nil && len(s.Objects) > 0 { + fmt.Fprintln(&buf) + for _, obj := range s.Objects { + fmt.Fprintf(&buf, "\t%s %s\n", obj.Kind, obj.Name) + } + } + fmt.Fprintf(&buf, "}\n") + return buf.String() +} + +// ---------------------------------------------------------------------------- +// Objects + +// TODO(gri) Consider replacing the Object struct with an interface +// and a corresponding set of object implementations. + +// An Object describes a named language entity such as a package, +// constant, type, variable, function (incl. methods), or label. +// +// The Data fields contains object-specific data: +// +// Kind Data type Data value +// Pkg *Scope package scope +// Con int iota for the respective declaration +// Con != nil constant value +// +type Object struct { + Kind ObjKind + Name string // declared name + Decl interface{} // corresponding Field, XxxSpec, FuncDecl, or LabeledStmt; or nil + Data interface{} // object-specific data; or nil + Type interface{} // place holder for type information; may be nil +} + +// NewObj creates a new object of a given kind and name. +func NewObj(kind ObjKind, name string) *Object { + return &Object{Kind: kind, Name: name} +} + +// Pos computes the source position of the declaration of an object name. +// The result may be an invalid position if it cannot be computed +// (obj.Decl may be nil or not correct). +func (obj *Object) Pos() token.Pos { + name := obj.Name + switch d := obj.Decl.(type) { + case *Field: + for _, n := range d.Names { + if n.Name == name { + return n.Pos() + } + } + case *ImportSpec: + if d.Name != nil && d.Name.Name == name { + return d.Name.Pos() + } + return d.Path.Pos() + case *ValueSpec: + for _, n := range d.Names { + if n.Name == name { + return n.Pos() + } + } + case *TypeSpec: + if d.Name.Name == name { + return d.Name.Pos() + } + case *FuncDecl: + if d.Name.Name == name { + return d.Name.Pos() + } + case *LabeledStmt: + if d.Label.Name == name { + return d.Label.Pos() + } + } + return token.NoPos +} + +// ObKind describes what an object represents. +type ObjKind int + +// The list of possible Object kinds. +const ( + Bad ObjKind = iota // for error handling + Pkg // package + Con // constant + Typ // type + Var // variable + Fun // function or method + Lbl // label +) + +var objKindStrings = [...]string{ + Bad: "bad", + Pkg: "package", + Con: "const", + Typ: "type", + Var: "var", + Fun: "func", + Lbl: "label", +} + +func (kind ObjKind) String() string { return objKindStrings[kind] } diff --git a/src/pkg/go/ast/walk.go b/src/pkg/go/ast/walk.go new file mode 100644 index 000000000..181cfd149 --- /dev/null +++ b/src/pkg/go/ast/walk.go @@ -0,0 +1,382 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package ast + +import "fmt" + +// A Visitor's Visit method is invoked for each node encountered by Walk. +// If the result visitor w is not nil, Walk visits each of the children +// of node with the visitor w, followed by a call of w.Visit(nil). +type Visitor interface { + Visit(node Node) (w Visitor) +} + +// Helper functions for common node lists. They may be empty. + +func walkIdentList(v Visitor, list []*Ident) { + for _, x := range list { + Walk(v, x) + } +} + +func walkExprList(v Visitor, list []Expr) { + for _, x := range list { + Walk(v, x) + } +} + +func walkStmtList(v Visitor, list []Stmt) { + for _, x := range list { + Walk(v, x) + } +} + +func walkDeclList(v Visitor, list []Decl) { + for _, x := range list { + Walk(v, x) + } +} + +// TODO(gri): Investigate if providing a closure to Walk leads to +// simpler use (and may help eliminate Inspect in turn). + +// Walk traverses an AST in depth-first order: It starts by calling +// v.Visit(node); node must not be nil. If the visitor w returned by +// v.Visit(node) is not nil, Walk is invoked recursively with visitor +// w for each of the non-nil children of node, followed by a call of +// w.Visit(nil). +// +func Walk(v Visitor, node Node) { + if v = v.Visit(node); v == nil { + return + } + + // walk children + // (the order of the cases matches the order + // of the corresponding node types in ast.go) + switch n := node.(type) { + // Comments and fields + case *Comment: + // nothing to do + + case *CommentGroup: + for _, c := range n.List { + Walk(v, c) + } + + case *Field: + if n.Doc != nil { + Walk(v, n.Doc) + } + walkIdentList(v, n.Names) + Walk(v, n.Type) + if n.Tag != nil { + Walk(v, n.Tag) + } + if n.Comment != nil { + Walk(v, n.Comment) + } + + case *FieldList: + for _, f := range n.List { + Walk(v, f) + } + + // Expressions + case *BadExpr, *Ident, *BasicLit: + // nothing to do + + case *Ellipsis: + if n.Elt != nil { + Walk(v, n.Elt) + } + + case *FuncLit: + Walk(v, n.Type) + Walk(v, n.Body) + + case *CompositeLit: + if n.Type != nil { + Walk(v, n.Type) + } + walkExprList(v, n.Elts) + + case *ParenExpr: + Walk(v, n.X) + + case *SelectorExpr: + Walk(v, n.X) + Walk(v, n.Sel) + + case *IndexExpr: + Walk(v, n.X) + Walk(v, n.Index) + + case *SliceExpr: + Walk(v, n.X) + if n.Low != nil { + Walk(v, n.Low) + } + if n.High != nil { + Walk(v, n.High) + } + + case *TypeAssertExpr: + Walk(v, n.X) + if n.Type != nil { + Walk(v, n.Type) + } + + case *CallExpr: + Walk(v, n.Fun) + walkExprList(v, n.Args) + + case *StarExpr: + Walk(v, n.X) + + case *UnaryExpr: + Walk(v, n.X) + + case *BinaryExpr: + Walk(v, n.X) + Walk(v, n.Y) + + case *KeyValueExpr: + Walk(v, n.Key) + Walk(v, n.Value) + + // Types + case *ArrayType: + if n.Len != nil { + Walk(v, n.Len) + } + Walk(v, n.Elt) + + case *StructType: + Walk(v, n.Fields) + + case *FuncType: + Walk(v, n.Params) + if n.Results != nil { + Walk(v, n.Results) + } + + case *InterfaceType: + Walk(v, n.Methods) + + case *MapType: + Walk(v, n.Key) + Walk(v, n.Value) + + case *ChanType: + Walk(v, n.Value) + + // Statements + case *BadStmt: + // nothing to do + + case *DeclStmt: + Walk(v, n.Decl) + + case *EmptyStmt: + // nothing to do + + case *LabeledStmt: + Walk(v, n.Label) + Walk(v, n.Stmt) + + case *ExprStmt: + Walk(v, n.X) + + case *SendStmt: + Walk(v, n.Chan) + Walk(v, n.Value) + + case *IncDecStmt: + Walk(v, n.X) + + case *AssignStmt: + walkExprList(v, n.Lhs) + walkExprList(v, n.Rhs) + + case *GoStmt: + Walk(v, n.Call) + + case *DeferStmt: + Walk(v, n.Call) + + case *ReturnStmt: + walkExprList(v, n.Results) + + case *BranchStmt: + if n.Label != nil { + Walk(v, n.Label) + } + + case *BlockStmt: + walkStmtList(v, n.List) + + case *IfStmt: + if n.Init != nil { + Walk(v, n.Init) + } + Walk(v, n.Cond) + Walk(v, n.Body) + if n.Else != nil { + Walk(v, n.Else) + } + + case *CaseClause: + walkExprList(v, n.List) + walkStmtList(v, n.Body) + + case *SwitchStmt: + if n.Init != nil { + Walk(v, n.Init) + } + if n.Tag != nil { + Walk(v, n.Tag) + } + Walk(v, n.Body) + + case *TypeSwitchStmt: + if n.Init != nil { + Walk(v, n.Init) + } + Walk(v, n.Assign) + Walk(v, n.Body) + + case *CommClause: + if n.Comm != nil { + Walk(v, n.Comm) + } + walkStmtList(v, n.Body) + + case *SelectStmt: + Walk(v, n.Body) + + case *ForStmt: + if n.Init != nil { + Walk(v, n.Init) + } + if n.Cond != nil { + Walk(v, n.Cond) + } + if n.Post != nil { + Walk(v, n.Post) + } + Walk(v, n.Body) + + case *RangeStmt: + Walk(v, n.Key) + if n.Value != nil { + Walk(v, n.Value) + } + Walk(v, n.X) + Walk(v, n.Body) + + // Declarations + case *ImportSpec: + if n.Doc != nil { + Walk(v, n.Doc) + } + if n.Name != nil { + Walk(v, n.Name) + } + Walk(v, n.Path) + if n.Comment != nil { + Walk(v, n.Comment) + } + + case *ValueSpec: + if n.Doc != nil { + Walk(v, n.Doc) + } + walkIdentList(v, n.Names) + if n.Type != nil { + Walk(v, n.Type) + } + walkExprList(v, n.Values) + if n.Comment != nil { + Walk(v, n.Comment) + } + + case *TypeSpec: + if n.Doc != nil { + Walk(v, n.Doc) + } + Walk(v, n.Name) + Walk(v, n.Type) + if n.Comment != nil { + Walk(v, n.Comment) + } + + case *BadDecl: + // nothing to do + + case *GenDecl: + if n.Doc != nil { + Walk(v, n.Doc) + } + for _, s := range n.Specs { + Walk(v, s) + } + + case *FuncDecl: + if n.Doc != nil { + Walk(v, n.Doc) + } + if n.Recv != nil { + Walk(v, n.Recv) + } + Walk(v, n.Name) + Walk(v, n.Type) + if n.Body != nil { + Walk(v, n.Body) + } + + // Files and packages + case *File: + if n.Doc != nil { + Walk(v, n.Doc) + } + Walk(v, n.Name) + walkDeclList(v, n.Decls) + for _, g := range n.Comments { + Walk(v, g) + } + // don't walk n.Comments - they have been + // visited already through the individual + // nodes + + case *Package: + for _, f := range n.Files { + Walk(v, f) + } + + default: + fmt.Printf("ast.Walk: unexpected node type %T", n) + panic("ast.Walk") + } + + v.Visit(nil) +} + +type inspector func(Node) bool + +func (f inspector) Visit(node Node) Visitor { + if f(node) { + return f + } + return nil +} + +// Inspect traverses an AST in depth-first order: It starts by calling +// f(node); node must not be nil. If f returns true, Inspect invokes f +// for all the non-nil children of node, recursively. +// +func Inspect(node Node, f func(Node) bool) { + Walk(inspector(f), node) +} diff --git a/src/pkg/go/build/Makefile b/src/pkg/go/build/Makefile new file mode 100644 index 000000000..349e00e80 --- /dev/null +++ b/src/pkg/go/build/Makefile @@ -0,0 +1,22 @@ +# Copyright 2009 The Go Authors. All rights reserved. +# Use of this source code is governed by a BSD-style +# license that can be found in the LICENSE file. + +include ../../../Make.inc + +TARG=go/build +GOFILES=\ + build.go\ + dir.go\ + path.go\ + syslist.go\ + +CLEANFILES+=syslist.go pkgtest/_obj cmdtest/_obj cgotest/_obj + +include ../../../Make.pkg + +syslist.go: ../../../Make.inc Makefile + echo '// Generated automatically by make.' >$@ + echo 'package build' >>$@ + echo 'const goosList = "$(GOOS_LIST)"' >>$@ + echo 'const goarchList = "$(GOARCH_LIST)"' >>$@ diff --git a/src/pkg/go/build/build.go b/src/pkg/go/build/build.go new file mode 100644 index 000000000..97f92bfb6 --- /dev/null +++ b/src/pkg/go/build/build.go @@ -0,0 +1,444 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Package build provides tools for building Go packages. +package build + +import ( + "bytes" + "exec" + "fmt" + "os" + "path/filepath" + "regexp" + "runtime" + "strings" +) + +// Build produces a build Script for the given package. +func Build(tree *Tree, pkg string, info *DirInfo) (*Script, os.Error) { + s := &Script{} + b := &build{ + script: s, + path: filepath.Join(tree.SrcDir(), pkg), + } + b.obj = b.abs("_obj") + string(filepath.Separator) + + b.goarch = runtime.GOARCH + if g := os.Getenv("GOARCH"); g != "" { + b.goarch = g + } + var err os.Error + b.arch, err = ArchChar(b.goarch) + if err != nil { + return nil, err + } + + // add import object files to list of Inputs + for _, pkg := range info.Imports { + t, p, err := FindTree(pkg) + if err != nil && err != ErrNotFound { + // FindTree should always be able to suggest an import + // path and tree. The path must be malformed + // (for example, an absolute or relative path). + return nil, os.NewError("build: invalid import: " + pkg) + } + s.addInput(filepath.Join(t.PkgDir(), p+".a")) + } + + // .go files to be built with gc + gofiles := b.abss(info.GoFiles...) + s.addInput(gofiles...) + + var ofiles []string // object files to be linked or packed + + // make build directory + b.mkdir(b.obj) + s.addIntermediate(b.obj) + + // cgo + if len(info.CgoFiles) > 0 { + cgoFiles := b.abss(info.CgoFiles...) + s.addInput(cgoFiles...) + cgoCFiles := b.abss(info.CFiles...) + s.addInput(cgoCFiles...) + outGo, outObj := b.cgo(cgoFiles, cgoCFiles) + gofiles = append(gofiles, outGo...) + ofiles = append(ofiles, outObj...) + s.addIntermediate(outGo...) + s.addIntermediate(outObj...) + } + + // compile + if len(gofiles) > 0 { + ofile := b.obj + "_go_." + b.arch + b.gc(ofile, gofiles...) + ofiles = append(ofiles, ofile) + s.addIntermediate(ofile) + } + + // assemble + for _, sfile := range info.SFiles { + ofile := b.obj + sfile[:len(sfile)-1] + b.arch + sfile = b.abs(sfile) + s.addInput(sfile) + b.asm(ofile, sfile) + ofiles = append(ofiles, ofile) + s.addIntermediate(ofile) + } + + if len(ofiles) == 0 { + return nil, os.NewError("make: no object files to build") + } + + // choose target file + var targ string + if info.IsCommand() { + // use the last part of the import path as binary name + _, bin := filepath.Split(pkg) + if runtime.GOOS == "windows" { + bin += ".exe" + } + targ = filepath.Join(tree.BinDir(), bin) + } else { + targ = filepath.Join(tree.PkgDir(), pkg+".a") + } + + // make target directory + targDir, _ := filepath.Split(targ) + b.mkdir(targDir) + + // link binary or pack object + if info.IsCommand() { + b.ld(targ, ofiles...) + } else { + b.gopack(targ, ofiles...) + } + s.Output = append(s.Output, targ) + + return b.script, nil +} + +// A Script describes the build process for a Go package. +// The Input, Intermediate, and Output fields are lists of absolute paths. +type Script struct { + Cmd []*Cmd + Input []string + Intermediate []string + Output []string +} + +func (s *Script) addInput(file ...string) { + s.Input = append(s.Input, file...) +} + +func (s *Script) addIntermediate(file ...string) { + s.Intermediate = append(s.Intermediate, file...) +} + +// Run runs the Script's Cmds in order. +func (s *Script) Run() os.Error { + for _, c := range s.Cmd { + if err := c.Run(); err != nil { + return err + } + } + return nil +} + +// Stale returns true if the build's inputs are newer than its outputs. +func (s *Script) Stale() bool { + var latest int64 + // get latest mtime of outputs + for _, file := range s.Output { + fi, err := os.Stat(file) + if err != nil { + // any error reading output files means stale + return true + } + if m := fi.Mtime_ns; m > latest { + latest = m + } + } + for _, file := range s.Input { + fi, err := os.Stat(file) + if err != nil || fi.Mtime_ns > latest { + // any error reading input files means stale + // (attempt to rebuild to figure out why) + return true + } + } + return false +} + +// Clean removes the Script's Intermediate files. +// It tries to remove every file and returns the first error it encounters. +func (s *Script) Clean() (err os.Error) { + // Reverse order so that directories get removed after the files they contain. + for i := len(s.Intermediate) - 1; i >= 0; i-- { + if e := os.Remove(s.Intermediate[i]); err == nil { + err = e + } + } + return +} + +// Nuke removes the Script's Intermediate and Output files. +// It tries to remove every file and returns the first error it encounters. +func (s *Script) Nuke() (err os.Error) { + // Reverse order so that directories get removed after the files they contain. + for i := len(s.Output) - 1; i >= 0; i-- { + if e := os.Remove(s.Output[i]); err == nil { + err = e + } + } + if e := s.Clean(); err == nil { + err = e + } + return +} + +// A Cmd describes an individual build command. +type Cmd struct { + Args []string // command-line + Stdout string // write standard output to this file, "" is passthrough + Dir string // working directory + Env []string // environment + Input []string // file paths (dependencies) + Output []string // file paths +} + +func (c *Cmd) String() string { + return strings.Join(c.Args, " ") +} + +// Run executes the Cmd. +func (c *Cmd) Run() os.Error { + if c.Args[0] == "mkdir" { + for _, p := range c.Output { + if err := os.MkdirAll(p, 0777); err != nil { + return fmt.Errorf("command %q: %v", c, err) + } + } + return nil + } + out := new(bytes.Buffer) + cmd := exec.Command(c.Args[0], c.Args[1:]...) + cmd.Dir = c.Dir + cmd.Env = c.Env + cmd.Stdout = out + cmd.Stderr = out + if c.Stdout != "" { + f, err := os.Create(c.Stdout) + if err != nil { + return err + } + defer f.Close() + cmd.Stdout = f + } + if err := cmd.Run(); err != nil { + return fmt.Errorf("command %q: %v\n%v", c, err, out) + } + return nil +} + +// ArchChar returns the architecture character for the given goarch. +// For example, ArchChar("amd64") returns "6". +func ArchChar(goarch string) (string, os.Error) { + switch goarch { + case "386": + return "8", nil + case "amd64": + return "6", nil + case "arm": + return "5", nil + } + return "", os.NewError("unsupported GOARCH " + goarch) +} + +type build struct { + script *Script + path string + obj string + goarch string + arch string +} + +func (b *build) abs(file string) string { + if filepath.IsAbs(file) { + return file + } + return filepath.Join(b.path, file) +} + +func (b *build) abss(file ...string) []string { + s := make([]string, len(file)) + for i, f := range file { + s[i] = b.abs(f) + } + return s +} + +func (b *build) add(c Cmd) { + b.script.Cmd = append(b.script.Cmd, &c) +} + +func (b *build) mkdir(name string) { + b.add(Cmd{ + Args: []string{"mkdir", "-p", name}, + Output: []string{name}, + }) +} + +func (b *build) gc(ofile string, gofiles ...string) { + gc := b.arch + "g" + args := append([]string{gc, "-o", ofile}, gcImportArgs...) + args = append(args, gofiles...) + b.add(Cmd{ + Args: args, + Input: gofiles, + Output: []string{ofile}, + }) +} + +func (b *build) asm(ofile string, sfile string) { + asm := b.arch + "a" + b.add(Cmd{ + Args: []string{asm, "-o", ofile, sfile}, + Input: []string{sfile}, + Output: []string{ofile}, + }) +} + +func (b *build) ld(targ string, ofiles ...string) { + ld := b.arch + "l" + args := append([]string{ld, "-o", targ}, ldImportArgs...) + args = append(args, ofiles...) + b.add(Cmd{ + Args: args, + Input: ofiles, + Output: []string{targ}, + }) +} + +func (b *build) gopack(targ string, ofiles ...string) { + b.add(Cmd{ + Args: append([]string{"gopack", "grc", targ}, ofiles...), + Input: ofiles, + Output: []string{targ}, + }) +} + +func (b *build) cc(ofile string, cfiles ...string) { + cc := b.arch + "c" + dir := fmt.Sprintf("%s_%s", runtime.GOOS, runtime.GOARCH) + inc := filepath.Join(runtime.GOROOT(), "pkg", dir) + args := []string{cc, "-FVw", "-I", inc, "-o", ofile} + b.add(Cmd{ + Args: append(args, cfiles...), + Input: cfiles, + Output: []string{ofile}, + }) +} + +func (b *build) gccCompile(ofile, cfile string) { + b.add(Cmd{ + Args: b.gccArgs("-o", ofile, "-c", cfile), + Input: []string{cfile}, + Output: []string{ofile}, + }) +} + +func (b *build) gccLink(ofile string, ofiles ...string) { + b.add(Cmd{ + Args: append(b.gccArgs("-o", ofile), ofiles...), + Input: ofiles, + Output: []string{ofile}, + }) +} + +func (b *build) gccArgs(args ...string) []string { + // TODO(adg): HOST_CC + a := []string{"gcc", "-I", b.path, "-g", "-fPIC", "-O2"} + switch b.arch { + case "8": + a = append(a, "-m32") + case "6": + a = append(a, "-m64") + } + return append(a, args...) +} + +var cgoRe = regexp.MustCompile(`[/\\:]`) + +func (b *build) cgo(cgofiles, cgocfiles []string) (outGo, outObj []string) { + // cgo + // TODO(adg): CGOPKGPATH + // TODO(adg): CGO_FLAGS + gofiles := []string{b.obj + "_cgo_gotypes.go"} + cfiles := []string{b.obj + "_cgo_main.c", b.obj + "_cgo_export.c"} + for _, fn := range cgofiles { + f := b.obj + cgoRe.ReplaceAllString(fn[:len(fn)-2], "_") + gofiles = append(gofiles, f+"cgo1.go") + cfiles = append(cfiles, f+"cgo2.c") + } + defunC := b.obj + "_cgo_defun.c" + output := append([]string{defunC}, cfiles...) + output = append(output, gofiles...) + b.add(Cmd{ + Args: append([]string{"cgo", "--"}, cgofiles...), + Dir: b.path, + Env: append(os.Environ(), "GOARCH="+b.goarch), + Input: cgofiles, + Output: output, + }) + outGo = append(outGo, gofiles...) + exportH := filepath.Join(b.path, "_cgo_export.h") + b.script.addIntermediate(defunC, exportH, b.obj+"_cgo_flags") + b.script.addIntermediate(cfiles...) + + // cc _cgo_defun.c + defunObj := b.obj + "_cgo_defun." + b.arch + b.cc(defunObj, defunC) + outObj = append(outObj, defunObj) + + // gcc + linkobj := make([]string, 0, len(cfiles)) + for _, cfile := range cfiles { + ofile := cfile[:len(cfile)-1] + "o" + b.gccCompile(ofile, cfile) + linkobj = append(linkobj, ofile) + if !strings.HasSuffix(ofile, "_cgo_main.o") { + outObj = append(outObj, ofile) + } else { + b.script.addIntermediate(ofile) + } + } + for _, cfile := range cgocfiles { + ofile := b.obj + cgoRe.ReplaceAllString(cfile[:len(cfile)-1], "_") + "o" + b.gccCompile(ofile, cfile) + linkobj = append(linkobj, ofile) + outObj = append(outObj, ofile) + } + dynObj := b.obj + "_cgo_.o" + b.gccLink(dynObj, linkobj...) + b.script.addIntermediate(dynObj) + + // cgo -dynimport + importC := b.obj + "_cgo_import.c" + b.add(Cmd{ + Args: []string{"cgo", "-dynimport", dynObj}, + Stdout: importC, + Input: []string{dynObj}, + Output: []string{importC}, + }) + b.script.addIntermediate(importC) + + // cc _cgo_import.ARCH + importObj := b.obj + "_cgo_import." + b.arch + b.cc(importObj, importC) + outObj = append(outObj, importObj) + + return +} diff --git a/src/pkg/go/build/build_test.go b/src/pkg/go/build/build_test.go new file mode 100644 index 000000000..e59d87672 --- /dev/null +++ b/src/pkg/go/build/build_test.go @@ -0,0 +1,61 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package build + +import ( + "exec" + "path/filepath" + "testing" +) + +var buildPkgs = []string{ + "go/build/pkgtest", + "go/build/cmdtest", + "go/build/cgotest", +} + +const cmdtestOutput = "3" + +func TestBuild(t *testing.T) { + for _, pkg := range buildPkgs { + tree := Path[0] // Goroot + dir := filepath.Join(tree.SrcDir(), pkg) + + info, err := ScanDir(dir, true) + if err != nil { + t.Error("ScanDir:", err) + continue + } + + s, err := Build(tree, pkg, info) + if err != nil { + t.Error("Build:", err) + continue + } + + if err := s.Run(); err != nil { + t.Error("Run:", err) + continue + } + + if pkg == "go/build/cmdtest" { + bin := s.Output[0] + b, err := exec.Command(bin).CombinedOutput() + if err != nil { + t.Errorf("exec: %s: %v", bin, err) + continue + } + if string(b) != cmdtestOutput { + t.Errorf("cmdtest output: %s want: %s", b, cmdtestOutput) + } + } + + defer func(s *Script) { + if err := s.Nuke(); err != nil { + t.Errorf("nuking: %v", err) + } + }(s) + } +} diff --git a/src/pkg/go/build/cgotest/cgotest.c b/src/pkg/go/build/cgotest/cgotest.c new file mode 100644 index 000000000..b13acb227 --- /dev/null +++ b/src/pkg/go/build/cgotest/cgotest.c @@ -0,0 +1,9 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +int +Add(int x, int y, int *sum) +{ + sum = x+y; +} diff --git a/src/pkg/go/build/cgotest/cgotest.go b/src/pkg/go/build/cgotest/cgotest.go new file mode 100644 index 000000000..93bbf0688 --- /dev/null +++ b/src/pkg/go/build/cgotest/cgotest.go @@ -0,0 +1,19 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package cgotest + +/* +char* greeting = "hello, world"; +*/ +// #include "cgotest.h" +import "C" +import "unsafe" + +var Greeting = C.GoString(C.greeting) + +func DoAdd(x, y int) (sum int) { + C.Add(C.int(x), C.int(y), (*C.int)(unsafe.Pointer(&sum))) + return +} diff --git a/src/pkg/go/build/cgotest/cgotest.h b/src/pkg/go/build/cgotest/cgotest.h new file mode 100644 index 000000000..9c73643b6 --- /dev/null +++ b/src/pkg/go/build/cgotest/cgotest.h @@ -0,0 +1,5 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +extern int Add(int, int, int *); diff --git a/src/pkg/go/build/cmdtest/main.go b/src/pkg/go/build/cmdtest/main.go new file mode 100644 index 000000000..bed4f485a --- /dev/null +++ b/src/pkg/go/build/cmdtest/main.go @@ -0,0 +1,12 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package main + +import "go/build/pkgtest" + +func main() { + pkgtest.Foo() + print(int(pkgtest.Sqrt(9))) +} diff --git a/src/pkg/go/build/dir.go b/src/pkg/go/build/dir.go new file mode 100644 index 000000000..e0000b534 --- /dev/null +++ b/src/pkg/go/build/dir.go @@ -0,0 +1,172 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package build + +import ( + "go/parser" + "go/token" + "log" + "os" + "path/filepath" + "strconv" + "strings" + "runtime" +) + +type DirInfo struct { + GoFiles []string // .go files in dir (excluding CgoFiles) + CgoFiles []string // .go files that import "C" + CFiles []string // .c files in dir + SFiles []string // .s files in dir + Imports []string // All packages imported by goFiles + PkgName string // Name of package in dir +} + +func (d *DirInfo) IsCommand() bool { + return d.PkgName == "main" +} + +// ScanDir returns a structure with details about the Go content found +// in the given directory. The file lists exclude: +// +// - files in package main (unless allowMain is true) +// - files in package documentation +// - files ending in _test.go +// - files starting with _ or . +// +// Only files that satisfy the goodOSArch function are included. +func ScanDir(dir string, allowMain bool) (info *DirInfo, err os.Error) { + f, err := os.Open(dir) + if err != nil { + return nil, err + } + dirs, err := f.Readdir(-1) + f.Close() + if err != nil { + return nil, err + } + + var di DirInfo + imported := make(map[string]bool) + fset := token.NewFileSet() + for i := range dirs { + d := &dirs[i] + if strings.HasPrefix(d.Name, "_") || + strings.HasPrefix(d.Name, ".") { + continue + } + if !goodOSArch(d.Name) { + continue + } + + switch filepath.Ext(d.Name) { + case ".go": + if strings.HasSuffix(d.Name, "_test.go") { + continue + } + case ".c": + di.CFiles = append(di.CFiles, d.Name) + continue + case ".s": + di.SFiles = append(di.SFiles, d.Name) + continue + default: + continue + } + + filename := filepath.Join(dir, d.Name) + pf, err := parser.ParseFile(fset, filename, nil, parser.ImportsOnly) + if err != nil { + return nil, err + } + s := string(pf.Name.Name) + if s == "main" && !allowMain { + continue + } + if s == "documentation" { + continue + } + if di.PkgName == "" { + di.PkgName = s + } else if di.PkgName != s { + // Only if all files in the directory are in package main + // do we return PkgName=="main". + // A mix of main and another package reverts + // to the original (allowMain=false) behaviour. + if s == "main" || di.PkgName == "main" { + return ScanDir(dir, false) + } + return nil, os.NewError("multiple package names in " + dir) + } + isCgo := false + for _, spec := range pf.Imports { + quoted := string(spec.Path.Value) + path, err := strconv.Unquote(quoted) + if err != nil { + log.Panicf("%s: parser returned invalid quoted string: <%s>", filename, quoted) + } + imported[path] = true + if path == "C" { + isCgo = true + } + } + if isCgo { + di.CgoFiles = append(di.CgoFiles, d.Name) + } else { + di.GoFiles = append(di.GoFiles, d.Name) + } + } + di.Imports = make([]string, len(imported)) + i := 0 + for p := range imported { + di.Imports[i] = p + i++ + } + return &di, nil +} + +// goodOSArch returns false if the filename contains a $GOOS or $GOARCH +// suffix which does not match the current system. +// The recognized filename formats are: +// +// name_$(GOOS).* +// name_$(GOARCH).* +// name_$(GOOS)_$(GOARCH).* +// +func goodOSArch(filename string) bool { + if dot := strings.Index(filename, "."); dot != -1 { + filename = filename[:dot] + } + l := strings.Split(filename, "_") + n := len(l) + if n == 0 { + return true + } + if good, known := goodOS[l[n-1]]; known { + return good + } + if good, known := goodArch[l[n-1]]; known { + if !good || n < 2 { + return false + } + good, known = goodOS[l[n-2]] + return good || !known + } + return true +} + +var goodOS = make(map[string]bool) +var goodArch = make(map[string]bool) + +func init() { + goodOS = make(map[string]bool) + goodArch = make(map[string]bool) + for _, v := range strings.Fields(goosList) { + goodOS[v] = v == runtime.GOOS + } + for _, v := range strings.Fields(goarchList) { + goodArch[v] = v == runtime.GOARCH + } +} diff --git a/src/pkg/go/build/path.go b/src/pkg/go/build/path.go new file mode 100644 index 000000000..e39b5f8fa --- /dev/null +++ b/src/pkg/go/build/path.go @@ -0,0 +1,182 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package build + +import ( + "fmt" + "log" + "os" + "path/filepath" + "runtime" +) + +// Path is a validated list of Trees derived from $GOROOT and $GOPATH at init. +var Path []*Tree + +// Tree describes a Go source tree, either $GOROOT or one from $GOPATH. +type Tree struct { + Path string + Goroot bool +} + +func newTree(p string) (*Tree, os.Error) { + if !filepath.IsAbs(p) { + return nil, os.NewError("must be absolute") + } + ep, err := filepath.EvalSymlinks(p) + if err != nil { + return nil, err + } + return &Tree{Path: ep}, nil +} + +// SrcDir returns the tree's package source directory. +func (t *Tree) SrcDir() string { + if t.Goroot { + return filepath.Join(t.Path, "src", "pkg") + } + return filepath.Join(t.Path, "src") +} + +// PkgDir returns the tree's package object directory. +func (t *Tree) PkgDir() string { + goos, goarch := runtime.GOOS, runtime.GOARCH + if e := os.Getenv("GOOS"); e != "" { + goos = e + } + if e := os.Getenv("GOARCH"); e != "" { + goarch = e + } + return filepath.Join(t.Path, "pkg", goos+"_"+goarch) +} + +// BinDir returns the tree's binary executable directory. +func (t *Tree) BinDir() string { + if t.Goroot { + if gobin := os.Getenv("GOBIN"); gobin != "" { + return gobin + } + } + return filepath.Join(t.Path, "bin") +} + +// HasSrc returns whether the given package's +// source can be found inside this Tree. +func (t *Tree) HasSrc(pkg string) bool { + fi, err := os.Stat(filepath.Join(t.SrcDir(), pkg)) + if err != nil { + return false + } + return fi.IsDirectory() +} + +// HasPkg returns whether the given package's +// object file can be found inside this Tree. +func (t *Tree) HasPkg(pkg string) bool { + fi, err := os.Stat(filepath.Join(t.PkgDir(), pkg+".a")) + if err != nil { + return false + } + return fi.IsRegular() + // TODO(adg): check object version is consistent +} + +var ( + ErrNotFound = os.NewError("go/build: package could not be found locally") + ErrTreeNotFound = os.NewError("go/build: no valid GOROOT or GOPATH could be found") +) + +// FindTree takes an import or filesystem path and returns the +// tree where the package source should be and the package import path. +func FindTree(path string) (tree *Tree, pkg string, err os.Error) { + if isLocalPath(path) { + if path, err = filepath.Abs(path); err != nil { + return + } + if path, err = filepath.EvalSymlinks(path); err != nil { + return + } + for _, t := range Path { + tpath := t.SrcDir() + string(filepath.Separator) + if !filepath.HasPrefix(path, tpath) { + continue + } + tree = t + pkg = path[len(tpath):] + return + } + err = fmt.Errorf("path %q not inside a GOPATH", path) + return + } + tree = defaultTree + pkg = path + for _, t := range Path { + if t.HasSrc(pkg) { + tree = t + return + } + } + if tree == nil { + err = ErrTreeNotFound + } else { + err = ErrNotFound + } + return +} + +// isLocalPath returns whether the given path is local (/foo ./foo ../foo . ..) +// Windows paths that starts with drive letter (c:\foo c:foo) are considered local. +func isLocalPath(s string) bool { + const sep = string(filepath.Separator) + return s == "." || s == ".." || + filepath.HasPrefix(s, sep) || + filepath.HasPrefix(s, "."+sep) || filepath.HasPrefix(s, ".."+sep) || + filepath.VolumeName(s) != "" +} + +var ( + // argument lists used by the build's gc and ld methods + gcImportArgs []string + ldImportArgs []string + + // default tree for remote packages + defaultTree *Tree +) + +// set up Path: parse and validate GOROOT and GOPATH variables +func init() { + root := runtime.GOROOT() + t, err := newTree(root) + if err != nil { + log.Printf("go/build: invalid GOROOT %q: %v", root, err) + } else { + t.Goroot = true + Path = []*Tree{t} + } + + for _, p := range filepath.SplitList(os.Getenv("GOPATH")) { + if p == "" { + continue + } + t, err := newTree(p) + if err != nil { + log.Printf("go/build: invalid GOPATH %q: %v", p, err) + continue + } + Path = append(Path, t) + gcImportArgs = append(gcImportArgs, "-I", t.PkgDir()) + ldImportArgs = append(ldImportArgs, "-L", t.PkgDir()) + + // select first GOPATH entry as default + if defaultTree == nil { + defaultTree = t + } + } + + // use GOROOT if no valid GOPATH specified + if defaultTree == nil && len(Path) > 0 { + defaultTree = Path[0] + } +} diff --git a/src/pkg/go/build/pkgtest/pkgtest.go b/src/pkg/go/build/pkgtest/pkgtest.go new file mode 100644 index 000000000..9322f5ebd --- /dev/null +++ b/src/pkg/go/build/pkgtest/pkgtest.go @@ -0,0 +1,9 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package pkgtest + +func Foo() {} + +func Sqrt(x float64) float64 diff --git a/src/pkg/go/build/pkgtest/sqrt_386.s b/src/pkg/go/build/pkgtest/sqrt_386.s new file mode 100644 index 000000000..d0a428d52 --- /dev/null +++ b/src/pkg/go/build/pkgtest/sqrt_386.s @@ -0,0 +1,10 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// func Sqrt(x float64) float64 +TEXT ·Sqrt(SB),7,$0 + FMOVD x+0(FP),F0 + FSQRT + FMOVDP F0,r+8(FP) + RET diff --git a/src/pkg/go/build/pkgtest/sqrt_amd64.s b/src/pkg/go/build/pkgtest/sqrt_amd64.s new file mode 100644 index 000000000..f5b329e70 --- /dev/null +++ b/src/pkg/go/build/pkgtest/sqrt_amd64.s @@ -0,0 +1,9 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// func Sqrt(x float64) float64 +TEXT ·Sqrt(SB),7,$0 + SQRTSD x+0(FP), X0 + MOVSD X0, r+8(FP) + RET diff --git a/src/pkg/go/build/pkgtest/sqrt_arm.s b/src/pkg/go/build/pkgtest/sqrt_arm.s new file mode 100644 index 000000000..befbb8a89 --- /dev/null +++ b/src/pkg/go/build/pkgtest/sqrt_arm.s @@ -0,0 +1,10 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// func Sqrt(x float64) float64 +TEXT ·Sqrt(SB),7,$0 + MOVD x+0(FP),F0 + SQRTD F0,F0 + MOVD F0,r+8(FP) + RET diff --git a/src/pkg/go/build/syslist_test.go b/src/pkg/go/build/syslist_test.go new file mode 100644 index 000000000..eb0e5dcb6 --- /dev/null +++ b/src/pkg/go/build/syslist_test.go @@ -0,0 +1,62 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package build + +import ( + "runtime" + "testing" +) + +var ( + thisOS = runtime.GOOS + thisArch = runtime.GOARCH + otherOS = anotherOS() + otherArch = anotherArch() +) + +func anotherOS() string { + if thisOS != "darwin" { + return "darwin" + } + return "linux" +} + +func anotherArch() string { + if thisArch != "amd64" { + return "amd64" + } + return "386" +} + +type GoodFileTest struct { + name string + result bool +} + +var tests = []GoodFileTest{ + {"file.go", true}, + {"file.c", true}, + {"file_foo.go", true}, + {"file_" + thisArch + ".go", true}, + {"file_" + otherArch + ".go", false}, + {"file_" + thisOS + ".go", true}, + {"file_" + otherOS + ".go", false}, + {"file_" + thisOS + "_" + thisArch + ".go", true}, + {"file_" + otherOS + "_" + thisArch + ".go", false}, + {"file_" + thisOS + "_" + otherArch + ".go", false}, + {"file_" + otherOS + "_" + otherArch + ".go", false}, + {"file_foo_" + thisArch + ".go", true}, + {"file_foo_" + otherArch + ".go", false}, + {"file_" + thisOS + ".c", true}, + {"file_" + otherOS + ".c", false}, +} + +func TestGoodOSArch(t *testing.T) { + for _, test := range tests { + if goodOSArch(test.name) != test.result { + t.Fatalf("goodOSArch(%q) != %v", test.name, test.result) + } + } +} diff --git a/src/pkg/go/doc/Makefile b/src/pkg/go/doc/Makefile new file mode 100644 index 000000000..a5152c793 --- /dev/null +++ b/src/pkg/go/doc/Makefile @@ -0,0 +1,12 @@ +# Copyright 2009 The Go Authors. All rights reserved. +# Use of this source code is governed by a BSD-style +# license that can be found in the LICENSE file. + +include ../../../Make.inc + +TARG=go/doc +GOFILES=\ + comment.go\ + doc.go\ + +include ../../../Make.pkg diff --git a/src/pkg/go/doc/comment.go b/src/pkg/go/doc/comment.go new file mode 100644 index 000000000..e1989226b --- /dev/null +++ b/src/pkg/go/doc/comment.go @@ -0,0 +1,345 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Godoc comment extraction and comment -> HTML formatting. + +package doc + +import ( + "go/ast" + "io" + "regexp" + "strings" + "template" // for HTMLEscape +) + +func isWhitespace(ch byte) bool { return ch == ' ' || ch == '\t' || ch == '\n' || ch == '\r' } + +func stripTrailingWhitespace(s string) string { + i := len(s) + for i > 0 && isWhitespace(s[i-1]) { + i-- + } + return s[0:i] +} + +// CommentText returns the text of comment, +// with the comment markers - //, /*, and */ - removed. +func CommentText(comment *ast.CommentGroup) string { + if comment == nil { + return "" + } + comments := make([]string, len(comment.List)) + for i, c := range comment.List { + comments[i] = string(c.Text) + } + + lines := make([]string, 0, 10) // most comments are less than 10 lines + for _, c := range comments { + // Remove comment markers. + // The parser has given us exactly the comment text. + switch c[1] { + case '/': + //-style comment + c = c[2:] + // Remove leading space after //, if there is one. + // TODO(gri) This appears to be necessary in isolated + // cases (bignum.RatFromString) - why? + if len(c) > 0 && c[0] == ' ' { + c = c[1:] + } + case '*': + /*-style comment */ + c = c[2 : len(c)-2] + } + + // Split on newlines. + cl := strings.Split(c, "\n") + + // Walk lines, stripping trailing white space and adding to list. + for _, l := range cl { + lines = append(lines, stripTrailingWhitespace(l)) + } + } + + // Remove leading blank lines; convert runs of + // interior blank lines to a single blank line. + n := 0 + for _, line := range lines { + if line != "" || n > 0 && lines[n-1] != "" { + lines[n] = line + n++ + } + } + lines = lines[0:n] + + // Add final "" entry to get trailing newline from Join. + if n > 0 && lines[n-1] != "" { + lines = append(lines, "") + } + + return strings.Join(lines, "\n") +} + +// Split bytes into lines. +func split(text []byte) [][]byte { + // count lines + n := 0 + last := 0 + for i, c := range text { + if c == '\n' { + last = i + 1 + n++ + } + } + if last < len(text) { + n++ + } + + // split + out := make([][]byte, n) + last = 0 + n = 0 + for i, c := range text { + if c == '\n' { + out[n] = text[last : i+1] + last = i + 1 + n++ + } + } + if last < len(text) { + out[n] = text[last:] + } + + return out +} + +var ( + ldquo = []byte("“") + rdquo = []byte("”") +) + +// Escape comment text for HTML. If nice is set, +// also turn `` into “ and '' into ”. +func commentEscape(w io.Writer, s []byte, nice bool) { + last := 0 + if nice { + for i := 0; i < len(s)-1; i++ { + ch := s[i] + if ch == s[i+1] && (ch == '`' || ch == '\'') { + template.HTMLEscape(w, s[last:i]) + last = i + 2 + switch ch { + case '`': + w.Write(ldquo) + case '\'': + w.Write(rdquo) + } + i++ // loop will add one more + } + } + } + template.HTMLEscape(w, s[last:]) +} + +const ( + // Regexp for Go identifiers + identRx = `[a-zA-Z_][a-zA-Z_0-9]*` // TODO(gri) ASCII only for now - fix this + + // Regexp for URLs + protocol = `(https?|ftp|file|gopher|mailto|news|nntp|telnet|wais|prospero):` + hostPart = `[a-zA-Z0-9_@\-]+` + filePart = `[a-zA-Z0-9_?%#~&/\-+=]+` + urlRx = protocol + `//` + // http:// + hostPart + `([.:]` + hostPart + `)*/?` + // //www.google.com:8080/ + filePart + `([:.,]` + filePart + `)*` +) + +var matchRx = regexp.MustCompile(`(` + identRx + `)|(` + urlRx + `)`) + +var ( + html_a = []byte(``) + html_enda = []byte("") + html_i = []byte("") + html_endi = []byte("") + html_p = []byte("

\n") + html_endp = []byte("

\n") + html_pre = []byte("
")
+	html_endpre = []byte("
\n") +) + +// Emphasize and escape a line of text for HTML. URLs are converted into links; +// if the URL also appears in the words map, the link is taken from the map (if +// the corresponding map value is the empty string, the URL is not converted +// into a link). Go identifiers that appear in the words map are italicized; if +// the corresponding map value is not the empty string, it is considered a URL +// and the word is converted into a link. If nice is set, the remaining text's +// appearance is improved where it makes sense (e.g., `` is turned into “ +// and '' into ”). +func emphasize(w io.Writer, line []byte, words map[string]string, nice bool) { + for { + m := matchRx.FindSubmatchIndex(line) + if m == nil { + break + } + // m >= 6 (two parenthesized sub-regexps in matchRx, 1st one is identRx) + + // write text before match + commentEscape(w, line[0:m[0]], nice) + + // analyze match + match := line[m[0]:m[1]] + url := "" + italics := false + if words != nil { + url, italics = words[string(match)] + } + if m[2] < 0 { + // didn't match against first parenthesized sub-regexp; must be match against urlRx + if !italics { + // no alternative URL in words list, use match instead + url = string(match) + } + italics = false // don't italicize URLs + } + + // write match + if len(url) > 0 { + w.Write(html_a) + template.HTMLEscape(w, []byte(url)) + w.Write(html_aq) + } + if italics { + w.Write(html_i) + } + commentEscape(w, match, nice) + if italics { + w.Write(html_endi) + } + if len(url) > 0 { + w.Write(html_enda) + } + + // advance + line = line[m[1]:] + } + commentEscape(w, line, nice) +} + +func indentLen(s []byte) int { + i := 0 + for i < len(s) && (s[i] == ' ' || s[i] == '\t') { + i++ + } + return i +} + +func isBlank(s []byte) bool { return len(s) == 0 || (len(s) == 1 && s[0] == '\n') } + +func commonPrefix(a, b []byte) []byte { + i := 0 + for i < len(a) && i < len(b) && a[i] == b[i] { + i++ + } + return a[0:i] +} + +func unindent(block [][]byte) { + if len(block) == 0 { + return + } + + // compute maximum common white prefix + prefix := block[0][0:indentLen(block[0])] + for _, line := range block { + if !isBlank(line) { + prefix = commonPrefix(prefix, line[0:indentLen(line)]) + } + } + n := len(prefix) + + // remove + for i, line := range block { + if !isBlank(line) { + block[i] = line[n:] + } + } +} + +// Convert comment text to formatted HTML. +// The comment was prepared by DocReader, +// so it is known not to have leading, trailing blank lines +// nor to have trailing spaces at the end of lines. +// The comment markers have already been removed. +// +// Turn each run of multiple \n into

. +// Turn each run of indented lines into a

 block without indent.
+//
+// URLs in the comment text are converted into links; if the URL also appears
+// in the words map, the link is taken from the map (if the corresponding map
+// value is the empty string, the URL is not converted into a link).
+//
+// Go identifiers that appear in the words map are italicized; if the corresponding
+// map value is not the empty string, it is considered a URL and the word is converted
+// into a link.
+func ToHTML(w io.Writer, s []byte, words map[string]string) {
+	inpara := false
+
+	close := func() {
+		if inpara {
+			w.Write(html_endp)
+			inpara = false
+		}
+	}
+	open := func() {
+		if !inpara {
+			w.Write(html_p)
+			inpara = true
+		}
+	}
+
+	lines := split(s)
+	unindent(lines)
+	for i := 0; i < len(lines); {
+		line := lines[i]
+		if isBlank(line) {
+			// close paragraph
+			close()
+			i++
+			continue
+		}
+		if indentLen(line) > 0 {
+			// close paragraph
+			close()
+
+			// count indented or blank lines
+			j := i + 1
+			for j < len(lines) && (isBlank(lines[j]) || indentLen(lines[j]) > 0) {
+				j++
+			}
+			// but not trailing blank lines
+			for j > i && isBlank(lines[j-1]) {
+				j--
+			}
+			block := lines[i:j]
+			i = j
+
+			unindent(block)
+
+			// put those lines in a pre block
+			w.Write(html_pre)
+			for _, line := range block {
+				emphasize(w, line, nil, false) // no nice text formatting
+			}
+			w.Write(html_endpre)
+			continue
+		}
+		// open paragraph
+		open()
+		emphasize(w, lines[i], words, true) // nice text formatting
+		i++
+	}
+	close()
+}
diff --git a/src/pkg/go/doc/doc.go b/src/pkg/go/doc/doc.go
new file mode 100644
index 000000000..c7fed9784
--- /dev/null
+++ b/src/pkg/go/doc/doc.go
@@ -0,0 +1,641 @@
+// Copyright 2009 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// Package doc extracts source code documentation from a Go AST.
+package doc
+
+import (
+	"go/ast"
+	"go/token"
+	"regexp"
+	"sort"
+)
+
+// ----------------------------------------------------------------------------
+
+type typeDoc struct {
+	// len(decl.Specs) == 1, and the element type is *ast.TypeSpec
+	// if the type declaration hasn't been seen yet, decl is nil
+	decl *ast.GenDecl
+	// values, factory functions, and methods associated with the type
+	values    []*ast.GenDecl // consts and vars
+	factories map[string]*ast.FuncDecl
+	methods   map[string]*ast.FuncDecl
+}
+
+// docReader accumulates documentation for a single package.
+// It modifies the AST: Comments (declaration documentation)
+// that have been collected by the DocReader are set to nil
+// in the respective AST nodes so that they are not printed
+// twice (once when printing the documentation and once when
+// printing the corresponding AST node).
+//
+type docReader struct {
+	doc     *ast.CommentGroup // package documentation, if any
+	pkgName string
+	values  []*ast.GenDecl // consts and vars
+	types   map[string]*typeDoc
+	funcs   map[string]*ast.FuncDecl
+	bugs    []*ast.CommentGroup
+}
+
+func (doc *docReader) init(pkgName string) {
+	doc.pkgName = pkgName
+	doc.types = make(map[string]*typeDoc)
+	doc.funcs = make(map[string]*ast.FuncDecl)
+}
+
+func (doc *docReader) addDoc(comments *ast.CommentGroup) {
+	if doc.doc == nil {
+		// common case: just one package comment
+		doc.doc = comments
+		return
+	}
+
+	// More than one package comment: Usually there will be only
+	// one file with a package comment, but it's better to collect
+	// all comments than drop them on the floor.
+	// (This code isn't particularly clever - no amortized doubling is
+	// used - but this situation occurs rarely and is not time-critical.)
+	n1 := len(doc.doc.List)
+	n2 := len(comments.List)
+	list := make([]*ast.Comment, n1+1+n2) // + 1 for separator line
+	copy(list, doc.doc.List)
+	list[n1] = &ast.Comment{token.NoPos, "//"} // separator line
+	copy(list[n1+1:], comments.List)
+	doc.doc = &ast.CommentGroup{list}
+}
+
+func (doc *docReader) addType(decl *ast.GenDecl) {
+	spec := decl.Specs[0].(*ast.TypeSpec)
+	typ := doc.lookupTypeDoc(spec.Name.Name)
+	// typ should always be != nil since declared types
+	// are always named - be conservative and check
+	if typ != nil {
+		// a type should be added at most once, so typ.decl
+		// should be nil - if it isn't, simply overwrite it
+		typ.decl = decl
+	}
+}
+
+func (doc *docReader) lookupTypeDoc(name string) *typeDoc {
+	if name == "" {
+		return nil // no type docs for anonymous types
+	}
+	if tdoc, found := doc.types[name]; found {
+		return tdoc
+	}
+	// type wasn't found - add one without declaration
+	tdoc := &typeDoc{nil, nil, make(map[string]*ast.FuncDecl), make(map[string]*ast.FuncDecl)}
+	doc.types[name] = tdoc
+	return tdoc
+}
+
+func baseTypeName(typ ast.Expr) string {
+	switch t := typ.(type) {
+	case *ast.Ident:
+		// if the type is not exported, the effect to
+		// a client is as if there were no type name
+		if t.IsExported() {
+			return t.Name
+		}
+	case *ast.StarExpr:
+		return baseTypeName(t.X)
+	}
+	return ""
+}
+
+func (doc *docReader) addValue(decl *ast.GenDecl) {
+	// determine if decl should be associated with a type
+	// Heuristic: For each typed entry, determine the type name, if any.
+	//            If there is exactly one type name that is sufficiently
+	//            frequent, associate the decl with the respective type.
+	domName := ""
+	domFreq := 0
+	prev := ""
+	for _, s := range decl.Specs {
+		if v, ok := s.(*ast.ValueSpec); ok {
+			name := ""
+			switch {
+			case v.Type != nil:
+				// a type is present; determine its name
+				name = baseTypeName(v.Type)
+			case decl.Tok == token.CONST:
+				// no type is present but we have a constant declaration;
+				// use the previous type name (w/o more type information
+				// we cannot handle the case of unnamed variables with
+				// initializer expressions except for some trivial cases)
+				name = prev
+			}
+			if name != "" {
+				// entry has a named type
+				if domName != "" && domName != name {
+					// more than one type name - do not associate
+					// with any type
+					domName = ""
+					break
+				}
+				domName = name
+				domFreq++
+			}
+			prev = name
+		}
+	}
+
+	// determine values list
+	const threshold = 0.75
+	values := &doc.values
+	if domName != "" && domFreq >= int(float64(len(decl.Specs))*threshold) {
+		// typed entries are sufficiently frequent
+		typ := doc.lookupTypeDoc(domName)
+		if typ != nil {
+			values = &typ.values // associate with that type
+		}
+	}
+
+	*values = append(*values, decl)
+}
+
+// Helper function to set the table entry for function f. Makes sure that
+// at least one f with associated documentation is stored in table, if there
+// are multiple f's with the same name.
+func setFunc(table map[string]*ast.FuncDecl, f *ast.FuncDecl) {
+	name := f.Name.Name
+	if g, exists := table[name]; exists && g.Doc != nil {
+		// a function with the same name has already been registered;
+		// since it has documentation, assume f is simply another
+		// implementation and ignore it
+		// TODO(gri) consider collecting all functions, or at least
+		//           all comments
+		return
+	}
+	// function doesn't exist or has no documentation; use f
+	table[name] = f
+}
+
+func (doc *docReader) addFunc(fun *ast.FuncDecl) {
+	name := fun.Name.Name
+
+	// determine if it should be associated with a type
+	if fun.Recv != nil {
+		// method
+		typ := doc.lookupTypeDoc(baseTypeName(fun.Recv.List[0].Type))
+		if typ != nil {
+			// exported receiver type
+			setFunc(typ.methods, fun)
+		}
+		// otherwise don't show the method
+		// TODO(gri): There may be exported methods of non-exported types
+		// that can be called because of exported values (consts, vars, or
+		// function results) of that type. Could determine if that is the
+		// case and then show those methods in an appropriate section.
+		return
+	}
+
+	// perhaps a factory function
+	// determine result type, if any
+	if fun.Type.Results.NumFields() >= 1 {
+		res := fun.Type.Results.List[0]
+		if len(res.Names) <= 1 {
+			// exactly one (named or anonymous) result associated
+			// with the first type in result signature (there may
+			// be more than one result)
+			tname := baseTypeName(res.Type)
+			typ := doc.lookupTypeDoc(tname)
+			if typ != nil {
+				// named and exported result type
+
+				// Work-around for failure of heuristic: In package os
+				// too many functions are considered factory functions
+				// for the Error type. Eliminate manually for now as
+				// this appears to be the only important case in the
+				// current library where the heuristic fails.
+				if doc.pkgName == "os" && tname == "Error" &&
+					name != "NewError" && name != "NewSyscallError" {
+					// not a factory function for os.Error
+					setFunc(doc.funcs, fun) // treat as ordinary function
+					return
+				}
+
+				setFunc(typ.factories, fun)
+				return
+			}
+		}
+	}
+
+	// ordinary function
+	setFunc(doc.funcs, fun)
+}
+
+func (doc *docReader) addDecl(decl ast.Decl) {
+	switch d := decl.(type) {
+	case *ast.GenDecl:
+		if len(d.Specs) > 0 {
+			switch d.Tok {
+			case token.CONST, token.VAR:
+				// constants and variables are always handled as a group
+				doc.addValue(d)
+			case token.TYPE:
+				// types are handled individually
+				for _, spec := range d.Specs {
+					// make a (fake) GenDecl node for this TypeSpec
+					// (we need to do this here - as opposed to just
+					// for printing - so we don't lose the GenDecl
+					// documentation)
+					//
+					// TODO(gri): Consider just collecting the TypeSpec
+					// node (and copy in the GenDecl.doc if there is no
+					// doc in the TypeSpec - this is currently done in
+					// makeTypeDocs below). Simpler data structures, but
+					// would lose GenDecl documentation if the TypeSpec
+					// has documentation as well.
+					doc.addType(&ast.GenDecl{d.Doc, d.Pos(), token.TYPE, token.NoPos, []ast.Spec{spec}, token.NoPos})
+					// A new GenDecl node is created, no need to nil out d.Doc.
+				}
+			}
+		}
+	case *ast.FuncDecl:
+		doc.addFunc(d)
+	}
+}
+
+func copyCommentList(list []*ast.Comment) []*ast.Comment {
+	return append([]*ast.Comment(nil), list...)
+}
+
+var (
+	bug_markers = regexp.MustCompile("^/[/*][ \t]*BUG\\(.*\\):[ \t]*") // BUG(uid):
+	bug_content = regexp.MustCompile("[^ \n\r\t]+")                    // at least one non-whitespace char
+)
+
+// addFile adds the AST for a source file to the docReader.
+// Adding the same AST multiple times is a no-op.
+//
+func (doc *docReader) addFile(src *ast.File) {
+	// add package documentation
+	if src.Doc != nil {
+		doc.addDoc(src.Doc)
+		src.Doc = nil // doc consumed - remove from ast.File node
+	}
+
+	// add all declarations
+	for _, decl := range src.Decls {
+		doc.addDecl(decl)
+	}
+
+	// collect BUG(...) comments
+	for _, c := range src.Comments {
+		text := c.List[0].Text
+		if m := bug_markers.FindStringIndex(text); m != nil {
+			// found a BUG comment; maybe empty
+			if btxt := text[m[1]:]; bug_content.MatchString(btxt) {
+				// non-empty BUG comment; collect comment without BUG prefix
+				list := copyCommentList(c.List)
+				list[0].Text = text[m[1]:]
+				doc.bugs = append(doc.bugs, &ast.CommentGroup{list})
+			}
+		}
+	}
+	src.Comments = nil // consumed unassociated comments - remove from ast.File node
+}
+
+func NewFileDoc(file *ast.File) *PackageDoc {
+	var r docReader
+	r.init(file.Name.Name)
+	r.addFile(file)
+	return r.newDoc("", nil)
+}
+
+func NewPackageDoc(pkg *ast.Package, importpath string) *PackageDoc {
+	var r docReader
+	r.init(pkg.Name)
+	filenames := make([]string, len(pkg.Files))
+	i := 0
+	for filename, f := range pkg.Files {
+		r.addFile(f)
+		filenames[i] = filename
+		i++
+	}
+	return r.newDoc(importpath, filenames)
+}
+
+// ----------------------------------------------------------------------------
+// Conversion to external representation
+
+// ValueDoc is the documentation for a group of declared
+// values, either vars or consts.
+//
+type ValueDoc struct {
+	Doc   string
+	Decl  *ast.GenDecl
+	order int
+}
+
+type sortValueDoc []*ValueDoc
+
+func (p sortValueDoc) Len() int      { return len(p) }
+func (p sortValueDoc) Swap(i, j int) { p[i], p[j] = p[j], p[i] }
+
+func declName(d *ast.GenDecl) string {
+	if len(d.Specs) != 1 {
+		return ""
+	}
+
+	switch v := d.Specs[0].(type) {
+	case *ast.ValueSpec:
+		return v.Names[0].Name
+	case *ast.TypeSpec:
+		return v.Name.Name
+	}
+
+	return ""
+}
+
+func (p sortValueDoc) Less(i, j int) bool {
+	// sort by name
+	// pull blocks (name = "") up to top
+	// in original order
+	if ni, nj := declName(p[i].Decl), declName(p[j].Decl); ni != nj {
+		return ni < nj
+	}
+	return p[i].order < p[j].order
+}
+
+func makeValueDocs(list []*ast.GenDecl, tok token.Token) []*ValueDoc {
+	d := make([]*ValueDoc, len(list)) // big enough in any case
+	n := 0
+	for i, decl := range list {
+		if decl.Tok == tok {
+			d[n] = &ValueDoc{CommentText(decl.Doc), decl, i}
+			n++
+			decl.Doc = nil // doc consumed - removed from AST
+		}
+	}
+	d = d[0:n]
+	sort.Sort(sortValueDoc(d))
+	return d
+}
+
+// FuncDoc is the documentation for a func declaration,
+// either a top-level function or a method function.
+//
+type FuncDoc struct {
+	Doc  string
+	Recv ast.Expr // TODO(rsc): Would like string here
+	Name string
+	Decl *ast.FuncDecl
+}
+
+type sortFuncDoc []*FuncDoc
+
+func (p sortFuncDoc) Len() int           { return len(p) }
+func (p sortFuncDoc) Swap(i, j int)      { p[i], p[j] = p[j], p[i] }
+func (p sortFuncDoc) Less(i, j int) bool { return p[i].Name < p[j].Name }
+
+func makeFuncDocs(m map[string]*ast.FuncDecl) []*FuncDoc {
+	d := make([]*FuncDoc, len(m))
+	i := 0
+	for _, f := range m {
+		doc := new(FuncDoc)
+		doc.Doc = CommentText(f.Doc)
+		f.Doc = nil // doc consumed - remove from ast.FuncDecl node
+		if f.Recv != nil {
+			doc.Recv = f.Recv.List[0].Type
+		}
+		doc.Name = f.Name.Name
+		doc.Decl = f
+		d[i] = doc
+		i++
+	}
+	sort.Sort(sortFuncDoc(d))
+	return d
+}
+
+// TypeDoc is the documentation for a declared type.
+// Consts and Vars are sorted lists of constants and variables of (mostly) that type.
+// Factories is a sorted list of factory functions that return that type.
+// Methods is a sorted list of method functions on that type.
+type TypeDoc struct {
+	Doc       string
+	Type      *ast.TypeSpec
+	Consts    []*ValueDoc
+	Vars      []*ValueDoc
+	Factories []*FuncDoc
+	Methods   []*FuncDoc
+	Decl      *ast.GenDecl
+	order     int
+}
+
+type sortTypeDoc []*TypeDoc
+
+func (p sortTypeDoc) Len() int      { return len(p) }
+func (p sortTypeDoc) Swap(i, j int) { p[i], p[j] = p[j], p[i] }
+func (p sortTypeDoc) Less(i, j int) bool {
+	// sort by name
+	// pull blocks (name = "") up to top
+	// in original order
+	if ni, nj := p[i].Type.Name.Name, p[j].Type.Name.Name; ni != nj {
+		return ni < nj
+	}
+	return p[i].order < p[j].order
+}
+
+// NOTE(rsc): This would appear not to be correct for type ( )
+// blocks, but the doc extractor above has split them into
+// individual declarations.
+func (doc *docReader) makeTypeDocs(m map[string]*typeDoc) []*TypeDoc {
+	d := make([]*TypeDoc, len(m))
+	i := 0
+	for _, old := range m {
+		// all typeDocs should have a declaration associated with
+		// them after processing an entire package - be conservative
+		// and check
+		if decl := old.decl; decl != nil {
+			typespec := decl.Specs[0].(*ast.TypeSpec)
+			t := new(TypeDoc)
+			doc := typespec.Doc
+			typespec.Doc = nil // doc consumed - remove from ast.TypeSpec node
+			if doc == nil {
+				// no doc associated with the spec, use the declaration doc, if any
+				doc = decl.Doc
+			}
+			decl.Doc = nil // doc consumed - remove from ast.Decl node
+			t.Doc = CommentText(doc)
+			t.Type = typespec
+			t.Consts = makeValueDocs(old.values, token.CONST)
+			t.Vars = makeValueDocs(old.values, token.VAR)
+			t.Factories = makeFuncDocs(old.factories)
+			t.Methods = makeFuncDocs(old.methods)
+			t.Decl = old.decl
+			t.order = i
+			d[i] = t
+			i++
+		} else {
+			// no corresponding type declaration found - move any associated
+			// values, factory functions, and methods back to the top-level
+			// so that they are not lost (this should only happen if a package
+			// file containing the explicit type declaration is missing or if
+			// an unqualified type name was used after a "." import)
+			// 1) move values
+			doc.values = append(doc.values, old.values...)
+			// 2) move factory functions
+			for name, f := range old.factories {
+				doc.funcs[name] = f
+			}
+			// 3) move methods
+			for name, f := range old.methods {
+				// don't overwrite functions with the same name
+				if _, found := doc.funcs[name]; !found {
+					doc.funcs[name] = f
+				}
+			}
+		}
+	}
+	d = d[0:i] // some types may have been ignored
+	sort.Sort(sortTypeDoc(d))
+	return d
+}
+
+func makeBugDocs(list []*ast.CommentGroup) []string {
+	d := make([]string, len(list))
+	for i, g := range list {
+		d[i] = CommentText(g)
+	}
+	return d
+}
+
+// PackageDoc is the documentation for an entire package.
+//
+type PackageDoc struct {
+	PackageName string
+	ImportPath  string
+	Filenames   []string
+	Doc         string
+	Consts      []*ValueDoc
+	Types       []*TypeDoc
+	Vars        []*ValueDoc
+	Funcs       []*FuncDoc
+	Bugs        []string
+}
+
+// newDoc returns the accumulated documentation for the package.
+//
+func (doc *docReader) newDoc(importpath string, filenames []string) *PackageDoc {
+	p := new(PackageDoc)
+	p.PackageName = doc.pkgName
+	p.ImportPath = importpath
+	sort.Strings(filenames)
+	p.Filenames = filenames
+	p.Doc = CommentText(doc.doc)
+	// makeTypeDocs may extend the list of doc.values and
+	// doc.funcs and thus must be called before any other
+	// function consuming those lists
+	p.Types = doc.makeTypeDocs(doc.types)
+	p.Consts = makeValueDocs(doc.values, token.CONST)
+	p.Vars = makeValueDocs(doc.values, token.VAR)
+	p.Funcs = makeFuncDocs(doc.funcs)
+	p.Bugs = makeBugDocs(doc.bugs)
+	return p
+}
+
+// ----------------------------------------------------------------------------
+// Filtering by name
+
+type Filter func(string) bool
+
+func matchFields(fields *ast.FieldList, f Filter) bool {
+	if fields != nil {
+		for _, field := range fields.List {
+			for _, name := range field.Names {
+				if f(name.Name) {
+					return true
+				}
+			}
+		}
+	}
+	return false
+}
+
+func matchDecl(d *ast.GenDecl, f Filter) bool {
+	for _, d := range d.Specs {
+		switch v := d.(type) {
+		case *ast.ValueSpec:
+			for _, name := range v.Names {
+				if f(name.Name) {
+					return true
+				}
+			}
+		case *ast.TypeSpec:
+			if f(v.Name.Name) {
+				return true
+			}
+			switch t := v.Type.(type) {
+			case *ast.StructType:
+				if matchFields(t.Fields, f) {
+					return true
+				}
+			case *ast.InterfaceType:
+				if matchFields(t.Methods, f) {
+					return true
+				}
+			}
+		}
+	}
+	return false
+}
+
+func filterValueDocs(a []*ValueDoc, f Filter) []*ValueDoc {
+	w := 0
+	for _, vd := range a {
+		if matchDecl(vd.Decl, f) {
+			a[w] = vd
+			w++
+		}
+	}
+	return a[0:w]
+}
+
+func filterFuncDocs(a []*FuncDoc, f Filter) []*FuncDoc {
+	w := 0
+	for _, fd := range a {
+		if f(fd.Name) {
+			a[w] = fd
+			w++
+		}
+	}
+	return a[0:w]
+}
+
+func filterTypeDocs(a []*TypeDoc, f Filter) []*TypeDoc {
+	w := 0
+	for _, td := range a {
+		n := 0 // number of matches
+		if matchDecl(td.Decl, f) {
+			n = 1
+		} else {
+			// type name doesn't match, but we may have matching consts, vars, factories or methods
+			td.Consts = filterValueDocs(td.Consts, f)
+			td.Vars = filterValueDocs(td.Vars, f)
+			td.Factories = filterFuncDocs(td.Factories, f)
+			td.Methods = filterFuncDocs(td.Methods, f)
+			n += len(td.Consts) + len(td.Vars) + len(td.Factories) + len(td.Methods)
+		}
+		if n > 0 {
+			a[w] = td
+			w++
+		}
+	}
+	return a[0:w]
+}
+
+// Filter eliminates documentation for names that don't pass through the filter f.
+// TODO: Recognize "Type.Method" as a name.
+//
+func (p *PackageDoc) Filter(f Filter) {
+	p.Consts = filterValueDocs(p.Consts, f)
+	p.Vars = filterValueDocs(p.Vars, f)
+	p.Types = filterTypeDocs(p.Types, f)
+	p.Funcs = filterFuncDocs(p.Funcs, f)
+	p.Doc = "" // don't show top-level package doc
+}
diff --git a/src/pkg/go/parser/Makefile b/src/pkg/go/parser/Makefile
new file mode 100644
index 000000000..d301f41eb
--- /dev/null
+++ b/src/pkg/go/parser/Makefile
@@ -0,0 +1,12 @@
+# Copyright 2009 The Go Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style
+# license that can be found in the LICENSE file.
+
+include ../../../Make.inc
+
+TARG=go/parser
+GOFILES=\
+	interface.go\
+	parser.go\
+
+include ../../../Make.pkg
diff --git a/src/pkg/go/parser/interface.go b/src/pkg/go/parser/interface.go
new file mode 100644
index 000000000..4f980fc65
--- /dev/null
+++ b/src/pkg/go/parser/interface.go
@@ -0,0 +1,214 @@
+// Copyright 2009 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// This file contains the exported entry points for invoking the parser.
+
+package parser
+
+import (
+	"bytes"
+	"go/ast"
+	"go/scanner"
+	"go/token"
+	"io"
+	"io/ioutil"
+	"os"
+	"path/filepath"
+)
+
+// If src != nil, readSource converts src to a []byte if possible;
+// otherwise it returns an error. If src == nil, readSource returns
+// the result of reading the file specified by filename.
+//
+func readSource(filename string, src interface{}) ([]byte, os.Error) {
+	if src != nil {
+		switch s := src.(type) {
+		case string:
+			return []byte(s), nil
+		case []byte:
+			return s, nil
+		case *bytes.Buffer:
+			// is io.Reader, but src is already available in []byte form
+			if s != nil {
+				return s.Bytes(), nil
+			}
+		case io.Reader:
+			var buf bytes.Buffer
+			_, err := io.Copy(&buf, s)
+			if err != nil {
+				return nil, err
+			}
+			return buf.Bytes(), nil
+		default:
+			return nil, os.NewError("invalid source")
+		}
+	}
+
+	return ioutil.ReadFile(filename)
+}
+
+func (p *parser) errors() os.Error {
+	mode := scanner.Sorted
+	if p.mode&SpuriousErrors == 0 {
+		mode = scanner.NoMultiples
+	}
+	return p.GetError(mode)
+}
+
+// ParseExpr parses a Go expression and returns the corresponding
+// AST node. The fset, filename, and src arguments have the same interpretation
+// as for ParseFile. If there is an error, the result expression
+// may be nil or contain a partial AST.
+//
+func ParseExpr(fset *token.FileSet, filename string, src interface{}) (ast.Expr, os.Error) {
+	data, err := readSource(filename, src)
+	if err != nil {
+		return nil, err
+	}
+
+	var p parser
+	p.init(fset, filename, data, 0)
+	x := p.parseRhs()
+	if p.tok == token.SEMICOLON {
+		p.next() // consume automatically inserted semicolon, if any
+	}
+	p.expect(token.EOF)
+
+	return x, p.errors()
+}
+
+// ParseStmtList parses a list of Go statements and returns the list
+// of corresponding AST nodes. The fset, filename, and src arguments have the same
+// interpretation as for ParseFile. If there is an error, the node
+// list may be nil or contain partial ASTs.
+//
+func ParseStmtList(fset *token.FileSet, filename string, src interface{}) ([]ast.Stmt, os.Error) {
+	data, err := readSource(filename, src)
+	if err != nil {
+		return nil, err
+	}
+
+	var p parser
+	p.init(fset, filename, data, 0)
+	list := p.parseStmtList()
+	p.expect(token.EOF)
+
+	return list, p.errors()
+}
+
+// ParseDeclList parses a list of Go declarations and returns the list
+// of corresponding AST nodes. The fset, filename, and src arguments have the same
+// interpretation as for ParseFile. If there is an error, the node
+// list may be nil or contain partial ASTs.
+//
+func ParseDeclList(fset *token.FileSet, filename string, src interface{}) ([]ast.Decl, os.Error) {
+	data, err := readSource(filename, src)
+	if err != nil {
+		return nil, err
+	}
+
+	var p parser
+	p.init(fset, filename, data, 0)
+	list := p.parseDeclList()
+	p.expect(token.EOF)
+
+	return list, p.errors()
+}
+
+// ParseFile parses the source code of a single Go source file and returns
+// the corresponding ast.File node. The source code may be provided via
+// the filename of the source file, or via the src parameter.
+//
+// If src != nil, ParseFile parses the source from src and the filename is
+// only used when recording position information. The type of the argument
+// for the src parameter must be string, []byte, or io.Reader.
+//
+// If src == nil, ParseFile parses the file specified by filename.
+//
+// The mode parameter controls the amount of source text parsed and other
+// optional parser functionality. Position information is recorded in the
+// file set fset.
+//
+// If the source couldn't be read, the returned AST is nil and the error
+// indicates the specific failure. If the source was read but syntax
+// errors were found, the result is a partial AST (with ast.BadX nodes
+// representing the fragments of erroneous source code). Multiple errors
+// are returned via a scanner.ErrorList which is sorted by file position.
+//
+func ParseFile(fset *token.FileSet, filename string, src interface{}, mode uint) (*ast.File, os.Error) {
+	data, err := readSource(filename, src)
+	if err != nil {
+		return nil, err
+	}
+
+	var p parser
+	p.init(fset, filename, data, mode)
+	file := p.parseFile() // parseFile reads to EOF
+
+	return file, p.errors()
+}
+
+// ParseFiles calls ParseFile for each file in the filenames list and returns
+// a map of package name -> package AST with all the packages found. The mode
+// bits are passed to ParseFile unchanged. Position information is recorded
+// in the file set fset.
+//
+// Files with parse errors are ignored. In this case the map of packages may
+// be incomplete (missing packages and/or incomplete packages) and the first
+// error encountered is returned.
+//
+func ParseFiles(fset *token.FileSet, filenames []string, mode uint) (pkgs map[string]*ast.Package, first os.Error) {
+	pkgs = make(map[string]*ast.Package)
+	for _, filename := range filenames {
+		if src, err := ParseFile(fset, filename, nil, mode); err == nil {
+			name := src.Name.Name
+			pkg, found := pkgs[name]
+			if !found {
+				// TODO(gri) Use NewPackage here; reconsider ParseFiles API.
+				pkg = &ast.Package{name, nil, nil, make(map[string]*ast.File)}
+				pkgs[name] = pkg
+			}
+			pkg.Files[filename] = src
+		} else if first == nil {
+			first = err
+		}
+	}
+	return
+}
+
+// ParseDir calls ParseFile for the files in the directory specified by path and
+// returns a map of package name -> package AST with all the packages found. If
+// filter != nil, only the files with os.FileInfo entries passing through the filter
+// are considered. The mode bits are passed to ParseFile unchanged. Position
+// information is recorded in the file set fset.
+//
+// If the directory couldn't be read, a nil map and the respective error are
+// returned. If a parse error occurred, a non-nil but incomplete map and the
+// error are returned.
+//
+func ParseDir(fset *token.FileSet, path string, filter func(*os.FileInfo) bool, mode uint) (map[string]*ast.Package, os.Error) {
+	fd, err := os.Open(path)
+	if err != nil {
+		return nil, err
+	}
+	defer fd.Close()
+
+	list, err := fd.Readdir(-1)
+	if err != nil {
+		return nil, err
+	}
+
+	filenames := make([]string, len(list))
+	n := 0
+	for i := 0; i < len(list); i++ {
+		d := &list[i]
+		if filter == nil || filter(d) {
+			filenames[n] = filepath.Join(path, d.Name)
+			n++
+		}
+	}
+	filenames = filenames[0:n]
+
+	return ParseFiles(fset, filenames, mode)
+}
diff --git a/src/pkg/go/parser/parser.go b/src/pkg/go/parser/parser.go
new file mode 100644
index 000000000..9c14d1667
--- /dev/null
+++ b/src/pkg/go/parser/parser.go
@@ -0,0 +1,2161 @@
+// Copyright 2009 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// Package parser implements a parser for Go source files. Input may be
+// provided in a variety of forms (see the various Parse* functions); the
+// output is an abstract syntax tree (AST) representing the Go source. The
+// parser is invoked through one of the Parse* functions.
+//
+package parser
+
+import (
+	"fmt"
+	"go/ast"
+	"go/scanner"
+	"go/token"
+)
+
+// The mode parameter to the Parse* functions is a set of flags (or 0).
+// They control the amount of source code parsed and other optional
+// parser functionality.
+//
+const (
+	PackageClauseOnly uint = 1 << iota // parsing stops after package clause
+	ImportsOnly                        // parsing stops after import declarations
+	ParseComments                      // parse comments and add them to AST
+	Trace                              // print a trace of parsed productions
+	DeclarationErrors                  // report declaration errors
+	SpuriousErrors                     // report all (not just the first) errors per line
+)
+
+// The parser structure holds the parser's internal state.
+type parser struct {
+	file *token.File
+	scanner.ErrorVector
+	scanner scanner.Scanner
+
+	// Tracing/debugging
+	mode   uint // parsing mode
+	trace  bool // == (mode & Trace != 0)
+	indent uint // indentation used for tracing output
+
+	// Comments
+	comments    []*ast.CommentGroup
+	leadComment *ast.CommentGroup // last lead comment
+	lineComment *ast.CommentGroup // last line comment
+
+	// Next token
+	pos token.Pos   // token position
+	tok token.Token // one token look-ahead
+	lit string      // token literal
+
+	// Non-syntactic parser control
+	exprLev int // < 0: in control clause, >= 0: in expression
+
+	// Ordinary identifier scopes
+	pkgScope   *ast.Scope        // pkgScope.Outer == nil
+	topScope   *ast.Scope        // top-most scope; may be pkgScope
+	unresolved []*ast.Ident      // unresolved identifiers
+	imports    []*ast.ImportSpec // list of imports
+
+	// Label scope
+	// (maintained by open/close LabelScope)
+	labelScope  *ast.Scope     // label scope for current function
+	targetStack [][]*ast.Ident // stack of unresolved labels
+}
+
+// scannerMode returns the scanner mode bits given the parser's mode bits.
+func scannerMode(mode uint) uint {
+	var m uint = scanner.InsertSemis
+	if mode&ParseComments != 0 {
+		m |= scanner.ScanComments
+	}
+	return m
+}
+
+func (p *parser) init(fset *token.FileSet, filename string, src []byte, mode uint) {
+	p.file = fset.AddFile(filename, fset.Base(), len(src))
+	p.scanner.Init(p.file, src, p, scannerMode(mode))
+
+	p.mode = mode
+	p.trace = mode&Trace != 0 // for convenience (p.trace is used frequently)
+
+	p.next()
+
+	// set up the pkgScope here (as opposed to in parseFile) because
+	// there are other parser entry points (ParseExpr, etc.)
+	p.openScope()
+	p.pkgScope = p.topScope
+
+	// for the same reason, set up a label scope
+	p.openLabelScope()
+}
+
+// ----------------------------------------------------------------------------
+// Scoping support
+
+func (p *parser) openScope() {
+	p.topScope = ast.NewScope(p.topScope)
+}
+
+func (p *parser) closeScope() {
+	p.topScope = p.topScope.Outer
+}
+
+func (p *parser) openLabelScope() {
+	p.labelScope = ast.NewScope(p.labelScope)
+	p.targetStack = append(p.targetStack, nil)
+}
+
+func (p *parser) closeLabelScope() {
+	// resolve labels
+	n := len(p.targetStack) - 1
+	scope := p.labelScope
+	for _, ident := range p.targetStack[n] {
+		ident.Obj = scope.Lookup(ident.Name)
+		if ident.Obj == nil && p.mode&DeclarationErrors != 0 {
+			p.error(ident.Pos(), fmt.Sprintf("label %s undefined", ident.Name))
+		}
+	}
+	// pop label scope
+	p.targetStack = p.targetStack[0:n]
+	p.labelScope = p.labelScope.Outer
+}
+
+func (p *parser) declare(decl, data interface{}, scope *ast.Scope, kind ast.ObjKind, idents ...*ast.Ident) {
+	for _, ident := range idents {
+		assert(ident.Obj == nil, "identifier already declared or resolved")
+		obj := ast.NewObj(kind, ident.Name)
+		// remember the corresponding declaration for redeclaration
+		// errors and global variable resolution/typechecking phase
+		obj.Decl = decl
+		obj.Data = data
+		ident.Obj = obj
+		if ident.Name != "_" {
+			if alt := scope.Insert(obj); alt != nil && p.mode&DeclarationErrors != 0 {
+				prevDecl := ""
+				if pos := alt.Pos(); pos.IsValid() {
+					prevDecl = fmt.Sprintf("\n\tprevious declaration at %s", p.file.Position(pos))
+				}
+				p.error(ident.Pos(), fmt.Sprintf("%s redeclared in this block%s", ident.Name, prevDecl))
+			}
+		}
+	}
+}
+
+func (p *parser) shortVarDecl(idents []*ast.Ident) {
+	// Go spec: A short variable declaration may redeclare variables
+	// provided they were originally declared in the same block with
+	// the same type, and at least one of the non-blank variables is new.
+	n := 0 // number of new variables
+	for _, ident := range idents {
+		assert(ident.Obj == nil, "identifier already declared or resolved")
+		obj := ast.NewObj(ast.Var, ident.Name)
+		// short var declarations cannot have redeclaration errors
+		// and are not global => no need to remember the respective
+		// declaration
+		ident.Obj = obj
+		if ident.Name != "_" {
+			if alt := p.topScope.Insert(obj); alt != nil {
+				ident.Obj = alt // redeclaration
+			} else {
+				n++ // new declaration
+			}
+		}
+	}
+	if n == 0 && p.mode&DeclarationErrors != 0 {
+		p.error(idents[0].Pos(), "no new variables on left side of :=")
+	}
+}
+
+// The unresolved object is a sentinel to mark identifiers that have been added
+// to the list of unresolved identifiers. The sentinel is only used for verifying
+// internal consistency.
+var unresolved = new(ast.Object)
+
+func (p *parser) resolve(x ast.Expr) {
+	// nothing to do if x is not an identifier or the blank identifier
+	ident, _ := x.(*ast.Ident)
+	if ident == nil {
+		return
+	}
+	assert(ident.Obj == nil, "identifier already declared or resolved")
+	if ident.Name == "_" {
+		return
+	}
+	// try to resolve the identifier
+	for s := p.topScope; s != nil; s = s.Outer {
+		if obj := s.Lookup(ident.Name); obj != nil {
+			ident.Obj = obj
+			return
+		}
+	}
+	// all local scopes are known, so any unresolved identifier
+	// must be found either in the file scope, package scope
+	// (perhaps in another file), or universe scope --- collect
+	// them so that they can be resolved later
+	ident.Obj = unresolved
+	p.unresolved = append(p.unresolved, ident)
+}
+
+// ----------------------------------------------------------------------------
+// Parsing support
+
+func (p *parser) printTrace(a ...interface{}) {
+	const dots = ". . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . " +
+		". . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . "
+	const n = uint(len(dots))
+	pos := p.file.Position(p.pos)
+	fmt.Printf("%5d:%3d: ", pos.Line, pos.Column)
+	i := 2 * p.indent
+	for ; i > n; i -= n {
+		fmt.Print(dots)
+	}
+	fmt.Print(dots[0:i])
+	fmt.Println(a...)
+}
+
+func trace(p *parser, msg string) *parser {
+	p.printTrace(msg, "(")
+	p.indent++
+	return p
+}
+
+// Usage pattern: defer un(trace(p, "..."));
+func un(p *parser) {
+	p.indent--
+	p.printTrace(")")
+}
+
+// Advance to the next token.
+func (p *parser) next0() {
+	// Because of one-token look-ahead, print the previous token
+	// when tracing as it provides a more readable output. The
+	// very first token (!p.pos.IsValid()) is not initialized
+	// (it is token.ILLEGAL), so don't print it .
+	if p.trace && p.pos.IsValid() {
+		s := p.tok.String()
+		switch {
+		case p.tok.IsLiteral():
+			p.printTrace(s, p.lit)
+		case p.tok.IsOperator(), p.tok.IsKeyword():
+			p.printTrace("\"" + s + "\"")
+		default:
+			p.printTrace(s)
+		}
+	}
+
+	p.pos, p.tok, p.lit = p.scanner.Scan()
+}
+
+// Consume a comment and return it and the line on which it ends.
+func (p *parser) consumeComment() (comment *ast.Comment, endline int) {
+	// /*-style comments may end on a different line than where they start.
+	// Scan the comment for '\n' chars and adjust endline accordingly.
+	endline = p.file.Line(p.pos)
+	if p.lit[1] == '*' {
+		// don't use range here - no need to decode Unicode code points
+		for i := 0; i < len(p.lit); i++ {
+			if p.lit[i] == '\n' {
+				endline++
+			}
+		}
+	}
+
+	comment = &ast.Comment{p.pos, p.lit}
+	p.next0()
+
+	return
+}
+
+// Consume a group of adjacent comments, add it to the parser's
+// comments list, and return it together with the line at which
+// the last comment in the group ends. An empty line or non-comment
+// token terminates a comment group.
+//
+func (p *parser) consumeCommentGroup() (comments *ast.CommentGroup, endline int) {
+	var list []*ast.Comment
+	endline = p.file.Line(p.pos)
+	for p.tok == token.COMMENT && endline+1 >= p.file.Line(p.pos) {
+		var comment *ast.Comment
+		comment, endline = p.consumeComment()
+		list = append(list, comment)
+	}
+
+	// add comment group to the comments list
+	comments = &ast.CommentGroup{list}
+	p.comments = append(p.comments, comments)
+
+	return
+}
+
+// Advance to the next non-comment token. In the process, collect
+// any comment groups encountered, and remember the last lead and
+// and line comments.
+//
+// A lead comment is a comment group that starts and ends in a
+// line without any other tokens and that is followed by a non-comment
+// token on the line immediately after the comment group.
+//
+// A line comment is a comment group that follows a non-comment
+// token on the same line, and that has no tokens after it on the line
+// where it ends.
+//
+// Lead and line comments may be considered documentation that is
+// stored in the AST.
+//
+func (p *parser) next() {
+	p.leadComment = nil
+	p.lineComment = nil
+	line := p.file.Line(p.pos) // current line
+	p.next0()
+
+	if p.tok == token.COMMENT {
+		var comment *ast.CommentGroup
+		var endline int
+
+		if p.file.Line(p.pos) == line {
+			// The comment is on same line as the previous token; it
+			// cannot be a lead comment but may be a line comment.
+			comment, endline = p.consumeCommentGroup()
+			if p.file.Line(p.pos) != endline {
+				// The next token is on a different line, thus
+				// the last comment group is a line comment.
+				p.lineComment = comment
+			}
+		}
+
+		// consume successor comments, if any
+		endline = -1
+		for p.tok == token.COMMENT {
+			comment, endline = p.consumeCommentGroup()
+		}
+
+		if endline+1 == p.file.Line(p.pos) {
+			// The next token is following on the line immediately after the
+			// comment group, thus the last comment group is a lead comment.
+			p.leadComment = comment
+		}
+	}
+}
+
+func (p *parser) error(pos token.Pos, msg string) {
+	p.Error(p.file.Position(pos), msg)
+}
+
+func (p *parser) errorExpected(pos token.Pos, msg string) {
+	msg = "expected " + msg
+	if pos == p.pos {
+		// the error happened at the current position;
+		// make the error message more specific
+		if p.tok == token.SEMICOLON && p.lit[0] == '\n' {
+			msg += ", found newline"
+		} else {
+			msg += ", found '" + p.tok.String() + "'"
+			if p.tok.IsLiteral() {
+				msg += " " + p.lit
+			}
+		}
+	}
+	p.error(pos, msg)
+}
+
+func (p *parser) expect(tok token.Token) token.Pos {
+	pos := p.pos
+	if p.tok != tok {
+		p.errorExpected(pos, "'"+tok.String()+"'")
+	}
+	p.next() // make progress
+	return pos
+}
+
+func (p *parser) expectSemi() {
+	if p.tok != token.RPAREN && p.tok != token.RBRACE {
+		p.expect(token.SEMICOLON)
+	}
+}
+
+func assert(cond bool, msg string) {
+	if !cond {
+		panic("go/parser internal error: " + msg)
+	}
+}
+
+// ----------------------------------------------------------------------------
+// Identifiers
+
+func (p *parser) parseIdent() *ast.Ident {
+	pos := p.pos
+	name := "_"
+	if p.tok == token.IDENT {
+		name = p.lit
+		p.next()
+	} else {
+		p.expect(token.IDENT) // use expect() error handling
+	}
+	return &ast.Ident{pos, name, nil}
+}
+
+func (p *parser) parseIdentList() (list []*ast.Ident) {
+	if p.trace {
+		defer un(trace(p, "IdentList"))
+	}
+
+	list = append(list, p.parseIdent())
+	for p.tok == token.COMMA {
+		p.next()
+		list = append(list, p.parseIdent())
+	}
+
+	return
+}
+
+// ----------------------------------------------------------------------------
+// Common productions
+
+// If lhs is set, result list elements which are identifiers are not resolved.
+func (p *parser) parseExprList(lhs bool) (list []ast.Expr) {
+	if p.trace {
+		defer un(trace(p, "ExpressionList"))
+	}
+
+	list = append(list, p.checkExpr(p.parseExpr(lhs)))
+	for p.tok == token.COMMA {
+		p.next()
+		list = append(list, p.checkExpr(p.parseExpr(lhs)))
+	}
+
+	return
+}
+
+func (p *parser) parseLhsList() []ast.Expr {
+	list := p.parseExprList(true)
+	switch p.tok {
+	case token.DEFINE:
+		// lhs of a short variable declaration
+		p.shortVarDecl(p.makeIdentList(list))
+	case token.COLON:
+		// lhs of a label declaration or a communication clause of a select
+		// statement (parseLhsList is not called when parsing the case clause
+		// of a switch statement):
+		// - labels are declared by the caller of parseLhsList
+		// - for communication clauses, if there is a stand-alone identifier
+		//   followed by a colon, we have a syntax error; there is no need
+		//   to resolve the identifier in that case
+	default:
+		// identifiers must be declared elsewhere
+		for _, x := range list {
+			p.resolve(x)
+		}
+	}
+	return list
+}
+
+func (p *parser) parseRhsList() []ast.Expr {
+	return p.parseExprList(false)
+}
+
+// ----------------------------------------------------------------------------
+// Types
+
+func (p *parser) parseType() ast.Expr {
+	if p.trace {
+		defer un(trace(p, "Type"))
+	}
+
+	typ := p.tryType()
+
+	if typ == nil {
+		pos := p.pos
+		p.errorExpected(pos, "type")
+		p.next() // make progress
+		return &ast.BadExpr{pos, p.pos}
+	}
+
+	return typ
+}
+
+// If the result is an identifier, it is not resolved.
+func (p *parser) parseTypeName() ast.Expr {
+	if p.trace {
+		defer un(trace(p, "TypeName"))
+	}
+
+	ident := p.parseIdent()
+	// don't resolve ident yet - it may be a parameter or field name
+
+	if p.tok == token.PERIOD {
+		// ident is a package name
+		p.next()
+		p.resolve(ident)
+		sel := p.parseIdent()
+		return &ast.SelectorExpr{ident, sel}
+	}
+
+	return ident
+}
+
+func (p *parser) parseArrayType(ellipsisOk bool) ast.Expr {
+	if p.trace {
+		defer un(trace(p, "ArrayType"))
+	}
+
+	lbrack := p.expect(token.LBRACK)
+	var len ast.Expr
+	if ellipsisOk && p.tok == token.ELLIPSIS {
+		len = &ast.Ellipsis{p.pos, nil}
+		p.next()
+	} else if p.tok != token.RBRACK {
+		len = p.parseRhs()
+	}
+	p.expect(token.RBRACK)
+	elt := p.parseType()
+
+	return &ast.ArrayType{lbrack, len, elt}
+}
+
+func (p *parser) makeIdentList(list []ast.Expr) []*ast.Ident {
+	idents := make([]*ast.Ident, len(list))
+	for i, x := range list {
+		ident, isIdent := x.(*ast.Ident)
+		if !isIdent {
+			pos := x.(ast.Expr).Pos()
+			p.errorExpected(pos, "identifier")
+			ident = &ast.Ident{pos, "_", nil}
+		}
+		idents[i] = ident
+	}
+	return idents
+}
+
+func (p *parser) parseFieldDecl(scope *ast.Scope) *ast.Field {
+	if p.trace {
+		defer un(trace(p, "FieldDecl"))
+	}
+
+	doc := p.leadComment
+
+	// fields
+	list, typ := p.parseVarList(false)
+
+	// optional tag
+	var tag *ast.BasicLit
+	if p.tok == token.STRING {
+		tag = &ast.BasicLit{p.pos, p.tok, p.lit}
+		p.next()
+	}
+
+	// analyze case
+	var idents []*ast.Ident
+	if typ != nil {
+		// IdentifierList Type
+		idents = p.makeIdentList(list)
+	} else {
+		// ["*"] TypeName (AnonymousField)
+		typ = list[0] // we always have at least one element
+		p.resolve(typ)
+		if n := len(list); n > 1 || !isTypeName(deref(typ)) {
+			pos := typ.Pos()
+			p.errorExpected(pos, "anonymous field")
+			typ = &ast.BadExpr{pos, list[n-1].End()}
+		}
+	}
+
+	p.expectSemi() // call before accessing p.linecomment
+
+	field := &ast.Field{doc, idents, typ, tag, p.lineComment}
+	p.declare(field, nil, scope, ast.Var, idents...)
+
+	return field
+}
+
+func (p *parser) parseStructType() *ast.StructType {
+	if p.trace {
+		defer un(trace(p, "StructType"))
+	}
+
+	pos := p.expect(token.STRUCT)
+	lbrace := p.expect(token.LBRACE)
+	scope := ast.NewScope(nil) // struct scope
+	var list []*ast.Field
+	for p.tok == token.IDENT || p.tok == token.MUL || p.tok == token.LPAREN {
+		// a field declaration cannot start with a '(' but we accept
+		// it here for more robust parsing and better error messages
+		// (parseFieldDecl will check and complain if necessary)
+		list = append(list, p.parseFieldDecl(scope))
+	}
+	rbrace := p.expect(token.RBRACE)
+
+	// TODO(gri): store struct scope in AST
+	return &ast.StructType{pos, &ast.FieldList{lbrace, list, rbrace}, false}
+}
+
+func (p *parser) parsePointerType() *ast.StarExpr {
+	if p.trace {
+		defer un(trace(p, "PointerType"))
+	}
+
+	star := p.expect(token.MUL)
+	base := p.parseType()
+
+	return &ast.StarExpr{star, base}
+}
+
+func (p *parser) tryVarType(isParam bool) ast.Expr {
+	if isParam && p.tok == token.ELLIPSIS {
+		pos := p.pos
+		p.next()
+		typ := p.tryIdentOrType(isParam) // don't use parseType so we can provide better error message
+		if typ == nil {
+			p.error(pos, "'...' parameter is missing type")
+			typ = &ast.BadExpr{pos, p.pos}
+		}
+		if p.tok != token.RPAREN {
+			p.error(pos, "can use '...' with last parameter type only")
+		}
+		return &ast.Ellipsis{pos, typ}
+	}
+	return p.tryIdentOrType(false)
+}
+
+func (p *parser) parseVarType(isParam bool) ast.Expr {
+	typ := p.tryVarType(isParam)
+	if typ == nil {
+		pos := p.pos
+		p.errorExpected(pos, "type")
+		p.next() // make progress
+		typ = &ast.BadExpr{pos, p.pos}
+	}
+	return typ
+}
+
+func (p *parser) parseVarList(isParam bool) (list []ast.Expr, typ ast.Expr) {
+	if p.trace {
+		defer un(trace(p, "VarList"))
+	}
+
+	// a list of identifiers looks like a list of type names
+	for {
+		// parseVarType accepts any type (including parenthesized ones)
+		// even though the syntax does not permit them here: we
+		// accept them all for more robust parsing and complain
+		// afterwards
+		list = append(list, p.parseVarType(isParam))
+		if p.tok != token.COMMA {
+			break
+		}
+		p.next()
+	}
+
+	// if we had a list of identifiers, it must be followed by a type
+	typ = p.tryVarType(isParam)
+	if typ != nil {
+		p.resolve(typ)
+	}
+
+	return
+}
+
+func (p *parser) parseParameterList(scope *ast.Scope, ellipsisOk bool) (params []*ast.Field) {
+	if p.trace {
+		defer un(trace(p, "ParameterList"))
+	}
+
+	list, typ := p.parseVarList(ellipsisOk)
+	if typ != nil {
+		// IdentifierList Type
+		idents := p.makeIdentList(list)
+		field := &ast.Field{nil, idents, typ, nil, nil}
+		params = append(params, field)
+		// Go spec: The scope of an identifier denoting a function
+		// parameter or result variable is the function body.
+		p.declare(field, nil, scope, ast.Var, idents...)
+		if p.tok == token.COMMA {
+			p.next()
+		}
+
+		for p.tok != token.RPAREN && p.tok != token.EOF {
+			idents := p.parseIdentList()
+			typ := p.parseVarType(ellipsisOk)
+			field := &ast.Field{nil, idents, typ, nil, nil}
+			params = append(params, field)
+			// Go spec: The scope of an identifier denoting a function
+			// parameter or result variable is the function body.
+			p.declare(field, nil, scope, ast.Var, idents...)
+			if p.tok != token.COMMA {
+				break
+			}
+			p.next()
+		}
+
+	} else {
+		// Type { "," Type } (anonymous parameters)
+		params = make([]*ast.Field, len(list))
+		for i, x := range list {
+			p.resolve(x)
+			params[i] = &ast.Field{Type: x}
+		}
+	}
+
+	return
+}
+
+func (p *parser) parseParameters(scope *ast.Scope, ellipsisOk bool) *ast.FieldList {
+	if p.trace {
+		defer un(trace(p, "Parameters"))
+	}
+
+	var params []*ast.Field
+	lparen := p.expect(token.LPAREN)
+	if p.tok != token.RPAREN {
+		params = p.parseParameterList(scope, ellipsisOk)
+	}
+	rparen := p.expect(token.RPAREN)
+
+	return &ast.FieldList{lparen, params, rparen}
+}
+
+func (p *parser) parseResult(scope *ast.Scope) *ast.FieldList {
+	if p.trace {
+		defer un(trace(p, "Result"))
+	}
+
+	if p.tok == token.LPAREN {
+		return p.parseParameters(scope, false)
+	}
+
+	typ := p.tryType()
+	if typ != nil {
+		list := make([]*ast.Field, 1)
+		list[0] = &ast.Field{Type: typ}
+		return &ast.FieldList{List: list}
+	}
+
+	return nil
+}
+
+func (p *parser) parseSignature(scope *ast.Scope) (params, results *ast.FieldList) {
+	if p.trace {
+		defer un(trace(p, "Signature"))
+	}
+
+	params = p.parseParameters(scope, true)
+	results = p.parseResult(scope)
+
+	return
+}
+
+func (p *parser) parseFuncType() (*ast.FuncType, *ast.Scope) {
+	if p.trace {
+		defer un(trace(p, "FuncType"))
+	}
+
+	pos := p.expect(token.FUNC)
+	scope := ast.NewScope(p.topScope) // function scope
+	params, results := p.parseSignature(scope)
+
+	return &ast.FuncType{pos, params, results}, scope
+}
+
+func (p *parser) parseMethodSpec(scope *ast.Scope) *ast.Field {
+	if p.trace {
+		defer un(trace(p, "MethodSpec"))
+	}
+
+	doc := p.leadComment
+	var idents []*ast.Ident
+	var typ ast.Expr
+	x := p.parseTypeName()
+	if ident, isIdent := x.(*ast.Ident); isIdent && p.tok == token.LPAREN {
+		// method
+		idents = []*ast.Ident{ident}
+		scope := ast.NewScope(nil) // method scope
+		params, results := p.parseSignature(scope)
+		typ = &ast.FuncType{token.NoPos, params, results}
+	} else {
+		// embedded interface
+		typ = x
+		p.resolve(typ)
+	}
+	p.expectSemi() // call before accessing p.linecomment
+
+	spec := &ast.Field{doc, idents, typ, nil, p.lineComment}
+	p.declare(spec, nil, scope, ast.Fun, idents...)
+
+	return spec
+}
+
+func (p *parser) parseInterfaceType() *ast.InterfaceType {
+	if p.trace {
+		defer un(trace(p, "InterfaceType"))
+	}
+
+	pos := p.expect(token.INTERFACE)
+	lbrace := p.expect(token.LBRACE)
+	scope := ast.NewScope(nil) // interface scope
+	var list []*ast.Field
+	for p.tok == token.IDENT {
+		list = append(list, p.parseMethodSpec(scope))
+	}
+	rbrace := p.expect(token.RBRACE)
+
+	// TODO(gri): store interface scope in AST
+	return &ast.InterfaceType{pos, &ast.FieldList{lbrace, list, rbrace}, false}
+}
+
+func (p *parser) parseMapType() *ast.MapType {
+	if p.trace {
+		defer un(trace(p, "MapType"))
+	}
+
+	pos := p.expect(token.MAP)
+	p.expect(token.LBRACK)
+	key := p.parseType()
+	p.expect(token.RBRACK)
+	value := p.parseType()
+
+	return &ast.MapType{pos, key, value}
+}
+
+func (p *parser) parseChanType() *ast.ChanType {
+	if p.trace {
+		defer un(trace(p, "ChanType"))
+	}
+
+	pos := p.pos
+	dir := ast.SEND | ast.RECV
+	if p.tok == token.CHAN {
+		p.next()
+		if p.tok == token.ARROW {
+			p.next()
+			dir = ast.SEND
+		}
+	} else {
+		p.expect(token.ARROW)
+		p.expect(token.CHAN)
+		dir = ast.RECV
+	}
+	value := p.parseType()
+
+	return &ast.ChanType{pos, dir, value}
+}
+
+// If the result is an identifier, it is not resolved.
+func (p *parser) tryIdentOrType(ellipsisOk bool) ast.Expr {
+	switch p.tok {
+	case token.IDENT:
+		return p.parseTypeName()
+	case token.LBRACK:
+		return p.parseArrayType(ellipsisOk)
+	case token.STRUCT:
+		return p.parseStructType()
+	case token.MUL:
+		return p.parsePointerType()
+	case token.FUNC:
+		typ, _ := p.parseFuncType()
+		return typ
+	case token.INTERFACE:
+		return p.parseInterfaceType()
+	case token.MAP:
+		return p.parseMapType()
+	case token.CHAN, token.ARROW:
+		return p.parseChanType()
+	case token.LPAREN:
+		lparen := p.pos
+		p.next()
+		typ := p.parseType()
+		rparen := p.expect(token.RPAREN)
+		return &ast.ParenExpr{lparen, typ, rparen}
+	}
+
+	// no type found
+	return nil
+}
+
+func (p *parser) tryType() ast.Expr {
+	typ := p.tryIdentOrType(false)
+	if typ != nil {
+		p.resolve(typ)
+	}
+	return typ
+}
+
+// ----------------------------------------------------------------------------
+// Blocks
+
+func (p *parser) parseStmtList() (list []ast.Stmt) {
+	if p.trace {
+		defer un(trace(p, "StatementList"))
+	}
+
+	for p.tok != token.CASE && p.tok != token.DEFAULT && p.tok != token.RBRACE && p.tok != token.EOF {
+		list = append(list, p.parseStmt())
+	}
+
+	return
+}
+
+func (p *parser) parseBody(scope *ast.Scope) *ast.BlockStmt {
+	if p.trace {
+		defer un(trace(p, "Body"))
+	}
+
+	lbrace := p.expect(token.LBRACE)
+	p.topScope = scope // open function scope
+	p.openLabelScope()
+	list := p.parseStmtList()
+	p.closeLabelScope()
+	p.closeScope()
+	rbrace := p.expect(token.RBRACE)
+
+	return &ast.BlockStmt{lbrace, list, rbrace}
+}
+
+func (p *parser) parseBlockStmt() *ast.BlockStmt {
+	if p.trace {
+		defer un(trace(p, "BlockStmt"))
+	}
+
+	lbrace := p.expect(token.LBRACE)
+	p.openScope()
+	list := p.parseStmtList()
+	p.closeScope()
+	rbrace := p.expect(token.RBRACE)
+
+	return &ast.BlockStmt{lbrace, list, rbrace}
+}
+
+// ----------------------------------------------------------------------------
+// Expressions
+
+func (p *parser) parseFuncTypeOrLit() ast.Expr {
+	if p.trace {
+		defer un(trace(p, "FuncTypeOrLit"))
+	}
+
+	typ, scope := p.parseFuncType()
+	if p.tok != token.LBRACE {
+		// function type only
+		return typ
+	}
+
+	p.exprLev++
+	body := p.parseBody(scope)
+	p.exprLev--
+
+	return &ast.FuncLit{typ, body}
+}
+
+// parseOperand may return an expression or a raw type (incl. array
+// types of the form [...]T. Callers must verify the result.
+// If lhs is set and the result is an identifier, it is not resolved.
+//
+func (p *parser) parseOperand(lhs bool) ast.Expr {
+	if p.trace {
+		defer un(trace(p, "Operand"))
+	}
+
+	switch p.tok {
+	case token.IDENT:
+		x := p.parseIdent()
+		if !lhs {
+			p.resolve(x)
+		}
+		return x
+
+	case token.INT, token.FLOAT, token.IMAG, token.CHAR, token.STRING:
+		x := &ast.BasicLit{p.pos, p.tok, p.lit}
+		p.next()
+		return x
+
+	case token.LPAREN:
+		lparen := p.pos
+		p.next()
+		p.exprLev++
+		x := p.parseRhsOrType() // types may be parenthesized: (some type)
+		p.exprLev--
+		rparen := p.expect(token.RPAREN)
+		return &ast.ParenExpr{lparen, x, rparen}
+
+	case token.FUNC:
+		return p.parseFuncTypeOrLit()
+
+	default:
+		if typ := p.tryIdentOrType(true); typ != nil {
+			// could be type for composite literal or conversion
+			_, isIdent := typ.(*ast.Ident)
+			assert(!isIdent, "type cannot be identifier")
+			return typ
+		}
+	}
+
+	pos := p.pos
+	p.errorExpected(pos, "operand")
+	p.next() // make progress
+	return &ast.BadExpr{pos, p.pos}
+}
+
+func (p *parser) parseSelector(x ast.Expr) ast.Expr {
+	if p.trace {
+		defer un(trace(p, "Selector"))
+	}
+
+	sel := p.parseIdent()
+
+	return &ast.SelectorExpr{x, sel}
+}
+
+func (p *parser) parseTypeAssertion(x ast.Expr) ast.Expr {
+	if p.trace {
+		defer un(trace(p, "TypeAssertion"))
+	}
+
+	p.expect(token.LPAREN)
+	var typ ast.Expr
+	if p.tok == token.TYPE {
+		// type switch: typ == nil
+		p.next()
+	} else {
+		typ = p.parseType()
+	}
+	p.expect(token.RPAREN)
+
+	return &ast.TypeAssertExpr{x, typ}
+}
+
+func (p *parser) parseIndexOrSlice(x ast.Expr) ast.Expr {
+	if p.trace {
+		defer un(trace(p, "IndexOrSlice"))
+	}
+
+	lbrack := p.expect(token.LBRACK)
+	p.exprLev++
+	var low, high ast.Expr
+	isSlice := false
+	if p.tok != token.COLON {
+		low = p.parseRhs()
+	}
+	if p.tok == token.COLON {
+		isSlice = true
+		p.next()
+		if p.tok != token.RBRACK {
+			high = p.parseRhs()
+		}
+	}
+	p.exprLev--
+	rbrack := p.expect(token.RBRACK)
+
+	if isSlice {
+		return &ast.SliceExpr{x, lbrack, low, high, rbrack}
+	}
+	return &ast.IndexExpr{x, lbrack, low, rbrack}
+}
+
+func (p *parser) parseCallOrConversion(fun ast.Expr) *ast.CallExpr {
+	if p.trace {
+		defer un(trace(p, "CallOrConversion"))
+	}
+
+	lparen := p.expect(token.LPAREN)
+	p.exprLev++
+	var list []ast.Expr
+	var ellipsis token.Pos
+	for p.tok != token.RPAREN && p.tok != token.EOF && !ellipsis.IsValid() {
+		list = append(list, p.parseRhsOrType()) // builtins may expect a type: make(some type, ...)
+		if p.tok == token.ELLIPSIS {
+			ellipsis = p.pos
+			p.next()
+		}
+		if p.tok != token.COMMA {
+			break
+		}
+		p.next()
+	}
+	p.exprLev--
+	rparen := p.expect(token.RPAREN)
+
+	return &ast.CallExpr{fun, lparen, list, ellipsis, rparen}
+}
+
+func (p *parser) parseElement(keyOk bool) ast.Expr {
+	if p.trace {
+		defer un(trace(p, "Element"))
+	}
+
+	if p.tok == token.LBRACE {
+		return p.parseLiteralValue(nil)
+	}
+
+	x := p.checkExpr(p.parseExpr(keyOk)) // don't resolve if map key
+	if keyOk {
+		if p.tok == token.COLON {
+			colon := p.pos
+			p.next()
+			return &ast.KeyValueExpr{x, colon, p.parseElement(false)}
+		}
+		p.resolve(x) // not a map key
+	}
+
+	return x
+}
+
+func (p *parser) parseElementList() (list []ast.Expr) {
+	if p.trace {
+		defer un(trace(p, "ElementList"))
+	}
+
+	for p.tok != token.RBRACE && p.tok != token.EOF {
+		list = append(list, p.parseElement(true))
+		if p.tok != token.COMMA {
+			break
+		}
+		p.next()
+	}
+
+	return
+}
+
+func (p *parser) parseLiteralValue(typ ast.Expr) ast.Expr {
+	if p.trace {
+		defer un(trace(p, "LiteralValue"))
+	}
+
+	lbrace := p.expect(token.LBRACE)
+	var elts []ast.Expr
+	p.exprLev++
+	if p.tok != token.RBRACE {
+		elts = p.parseElementList()
+	}
+	p.exprLev--
+	rbrace := p.expect(token.RBRACE)
+	return &ast.CompositeLit{typ, lbrace, elts, rbrace}
+}
+
+// checkExpr checks that x is an expression (and not a type).
+func (p *parser) checkExpr(x ast.Expr) ast.Expr {
+	switch t := unparen(x).(type) {
+	case *ast.BadExpr:
+	case *ast.Ident:
+	case *ast.BasicLit:
+	case *ast.FuncLit:
+	case *ast.CompositeLit:
+	case *ast.ParenExpr:
+		panic("unreachable")
+	case *ast.SelectorExpr:
+	case *ast.IndexExpr:
+	case *ast.SliceExpr:
+	case *ast.TypeAssertExpr:
+		// If t.Type == nil we have a type assertion of the form
+		// y.(type), which is only allowed in type switch expressions.
+		// It's hard to exclude those but for the case where we are in
+		// a type switch. Instead be lenient and test this in the type
+		// checker.
+	case *ast.CallExpr:
+	case *ast.StarExpr:
+	case *ast.UnaryExpr:
+	case *ast.BinaryExpr:
+	default:
+		// all other nodes are not proper expressions
+		p.errorExpected(x.Pos(), "expression")
+		x = &ast.BadExpr{x.Pos(), x.End()}
+	}
+	return x
+}
+
+// isTypeName returns true iff x is a (qualified) TypeName.
+func isTypeName(x ast.Expr) bool {
+	switch t := x.(type) {
+	case *ast.BadExpr:
+	case *ast.Ident:
+	case *ast.SelectorExpr:
+		_, isIdent := t.X.(*ast.Ident)
+		return isIdent
+	default:
+		return false // all other nodes are not type names
+	}
+	return true
+}
+
+// isLiteralType returns true iff x is a legal composite literal type.
+func isLiteralType(x ast.Expr) bool {
+	switch t := x.(type) {
+	case *ast.BadExpr:
+	case *ast.Ident:
+	case *ast.SelectorExpr:
+		_, isIdent := t.X.(*ast.Ident)
+		return isIdent
+	case *ast.ArrayType:
+	case *ast.StructType:
+	case *ast.MapType:
+	default:
+		return false // all other nodes are not legal composite literal types
+	}
+	return true
+}
+
+// If x is of the form *T, deref returns T, otherwise it returns x.
+func deref(x ast.Expr) ast.Expr {
+	if p, isPtr := x.(*ast.StarExpr); isPtr {
+		x = p.X
+	}
+	return x
+}
+
+// If x is of the form (T), unparen returns unparen(T), otherwise it returns x.
+func unparen(x ast.Expr) ast.Expr {
+	if p, isParen := x.(*ast.ParenExpr); isParen {
+		x = unparen(p.X)
+	}
+	return x
+}
+
+// checkExprOrType checks that x is an expression or a type
+// (and not a raw type such as [...]T).
+//
+func (p *parser) checkExprOrType(x ast.Expr) ast.Expr {
+	switch t := unparen(x).(type) {
+	case *ast.ParenExpr:
+		panic("unreachable")
+	case *ast.UnaryExpr:
+	case *ast.ArrayType:
+		if len, isEllipsis := t.Len.(*ast.Ellipsis); isEllipsis {
+			p.error(len.Pos(), "expected array length, found '...'")
+			x = &ast.BadExpr{x.Pos(), x.End()}
+		}
+	}
+
+	// all other nodes are expressions or types
+	return x
+}
+
+// If lhs is set and the result is an identifier, it is not resolved.
+func (p *parser) parsePrimaryExpr(lhs bool) ast.Expr {
+	if p.trace {
+		defer un(trace(p, "PrimaryExpr"))
+	}
+
+	x := p.parseOperand(lhs)
+L:
+	for {
+		switch p.tok {
+		case token.PERIOD:
+			p.next()
+			if lhs {
+				p.resolve(x)
+			}
+			switch p.tok {
+			case token.IDENT:
+				x = p.parseSelector(p.checkExpr(x))
+			case token.LPAREN:
+				x = p.parseTypeAssertion(p.checkExpr(x))
+			default:
+				pos := p.pos
+				p.next() // make progress
+				p.errorExpected(pos, "selector or type assertion")
+				x = &ast.BadExpr{pos, p.pos}
+			}
+		case token.LBRACK:
+			if lhs {
+				p.resolve(x)
+			}
+			x = p.parseIndexOrSlice(p.checkExpr(x))
+		case token.LPAREN:
+			if lhs {
+				p.resolve(x)
+			}
+			x = p.parseCallOrConversion(p.checkExprOrType(x))
+		case token.LBRACE:
+			if isLiteralType(x) && (p.exprLev >= 0 || !isTypeName(x)) {
+				if lhs {
+					p.resolve(x)
+				}
+				x = p.parseLiteralValue(x)
+			} else {
+				break L
+			}
+		default:
+			break L
+		}
+		lhs = false // no need to try to resolve again
+	}
+
+	return x
+}
+
+// If lhs is set and the result is an identifier, it is not resolved.
+func (p *parser) parseUnaryExpr(lhs bool) ast.Expr {
+	if p.trace {
+		defer un(trace(p, "UnaryExpr"))
+	}
+
+	switch p.tok {
+	case token.ADD, token.SUB, token.NOT, token.XOR, token.AND:
+		pos, op := p.pos, p.tok
+		p.next()
+		x := p.parseUnaryExpr(false)
+		return &ast.UnaryExpr{pos, op, p.checkExpr(x)}
+
+	case token.ARROW:
+		// channel type or receive expression
+		pos := p.pos
+		p.next()
+		if p.tok == token.CHAN {
+			p.next()
+			value := p.parseType()
+			return &ast.ChanType{pos, ast.RECV, value}
+		}
+
+		x := p.parseUnaryExpr(false)
+		return &ast.UnaryExpr{pos, token.ARROW, p.checkExpr(x)}
+
+	case token.MUL:
+		// pointer type or unary "*" expression
+		pos := p.pos
+		p.next()
+		x := p.parseUnaryExpr(false)
+		return &ast.StarExpr{pos, p.checkExprOrType(x)}
+	}
+
+	return p.parsePrimaryExpr(lhs)
+}
+
+// If lhs is set and the result is an identifier, it is not resolved.
+func (p *parser) parseBinaryExpr(lhs bool, prec1 int) ast.Expr {
+	if p.trace {
+		defer un(trace(p, "BinaryExpr"))
+	}
+
+	x := p.parseUnaryExpr(lhs)
+	for prec := p.tok.Precedence(); prec >= prec1; prec-- {
+		for p.tok.Precedence() == prec {
+			pos, op := p.pos, p.tok
+			p.next()
+			if lhs {
+				p.resolve(x)
+				lhs = false
+			}
+			y := p.parseBinaryExpr(false, prec+1)
+			x = &ast.BinaryExpr{p.checkExpr(x), pos, op, p.checkExpr(y)}
+		}
+	}
+
+	return x
+}
+
+// If lhs is set and the result is an identifier, it is not resolved.
+// The result may be a type or even a raw type ([...]int). Callers must
+// check the result (using checkExpr or checkExprOrType), depending on
+// context.
+func (p *parser) parseExpr(lhs bool) ast.Expr {
+	if p.trace {
+		defer un(trace(p, "Expression"))
+	}
+
+	return p.parseBinaryExpr(lhs, token.LowestPrec+1)
+}
+
+func (p *parser) parseRhs() ast.Expr {
+	return p.checkExpr(p.parseExpr(false))
+}
+
+func (p *parser) parseRhsOrType() ast.Expr {
+	return p.checkExprOrType(p.parseExpr(false))
+}
+
+// ----------------------------------------------------------------------------
+// Statements
+
+// Parsing modes for parseSimpleStmt.
+const (
+	basic = iota
+	labelOk
+	rangeOk
+)
+
+// parseSimpleStmt returns true as 2nd result if it parsed the assignment
+// of a range clause (with mode == rangeOk). The returned statement is an
+// assignment with a right-hand side that is a single unary expression of
+// the form "range x". No guarantees are given for the left-hand side.
+func (p *parser) parseSimpleStmt(mode int) (ast.Stmt, bool) {
+	if p.trace {
+		defer un(trace(p, "SimpleStmt"))
+	}
+
+	x := p.parseLhsList()
+
+	switch p.tok {
+	case
+		token.DEFINE, token.ASSIGN, token.ADD_ASSIGN,
+		token.SUB_ASSIGN, token.MUL_ASSIGN, token.QUO_ASSIGN,
+		token.REM_ASSIGN, token.AND_ASSIGN, token.OR_ASSIGN,
+		token.XOR_ASSIGN, token.SHL_ASSIGN, token.SHR_ASSIGN, token.AND_NOT_ASSIGN:
+		// assignment statement, possibly part of a range clause
+		pos, tok := p.pos, p.tok
+		p.next()
+		var y []ast.Expr
+		isRange := false
+		if mode == rangeOk && p.tok == token.RANGE && (tok == token.DEFINE || tok == token.ASSIGN) {
+			pos := p.pos
+			p.next()
+			y = []ast.Expr{&ast.UnaryExpr{pos, token.RANGE, p.parseRhs()}}
+			isRange = true
+		} else {
+			y = p.parseRhsList()
+		}
+		return &ast.AssignStmt{x, pos, tok, y}, isRange
+	}
+
+	if len(x) > 1 {
+		p.errorExpected(x[0].Pos(), "1 expression")
+		// continue with first expression
+	}
+
+	switch p.tok {
+	case token.COLON:
+		// labeled statement
+		colon := p.pos
+		p.next()
+		if label, isIdent := x[0].(*ast.Ident); mode == labelOk && isIdent {
+			// Go spec: The scope of a label is the body of the function
+			// in which it is declared and excludes the body of any nested
+			// function.
+			stmt := &ast.LabeledStmt{label, colon, p.parseStmt()}
+			p.declare(stmt, nil, p.labelScope, ast.Lbl, label)
+			return stmt, false
+		}
+		// The label declaration typically starts at x[0].Pos(), but the label
+		// declaration may be erroneous due to a token after that position (and
+		// before the ':'). If SpuriousErrors is not set, the (only) error re-
+		// ported for the line is the illegal label error instead of the token
+		// before the ':' that caused the problem. Thus, use the (latest) colon
+		// position for error reporting.
+		p.error(colon, "illegal label declaration")
+		return &ast.BadStmt{x[0].Pos(), colon + 1}, false
+
+	case token.ARROW:
+		// send statement
+		arrow := p.pos
+		p.next() // consume "<-"
+		y := p.parseRhs()
+		return &ast.SendStmt{x[0], arrow, y}, false
+
+	case token.INC, token.DEC:
+		// increment or decrement
+		s := &ast.IncDecStmt{x[0], p.pos, p.tok}
+		p.next() // consume "++" or "--"
+		return s, false
+	}
+
+	// expression
+	return &ast.ExprStmt{x[0]}, false
+}
+
+func (p *parser) parseCallExpr() *ast.CallExpr {
+	x := p.parseRhsOrType() // could be a conversion: (some type)(x)
+	if call, isCall := x.(*ast.CallExpr); isCall {
+		return call
+	}
+	p.errorExpected(x.Pos(), "function/method call")
+	return nil
+}
+
+func (p *parser) parseGoStmt() ast.Stmt {
+	if p.trace {
+		defer un(trace(p, "GoStmt"))
+	}
+
+	pos := p.expect(token.GO)
+	call := p.parseCallExpr()
+	p.expectSemi()
+	if call == nil {
+		return &ast.BadStmt{pos, pos + 2} // len("go")
+	}
+
+	return &ast.GoStmt{pos, call}
+}
+
+func (p *parser) parseDeferStmt() ast.Stmt {
+	if p.trace {
+		defer un(trace(p, "DeferStmt"))
+	}
+
+	pos := p.expect(token.DEFER)
+	call := p.parseCallExpr()
+	p.expectSemi()
+	if call == nil {
+		return &ast.BadStmt{pos, pos + 5} // len("defer")
+	}
+
+	return &ast.DeferStmt{pos, call}
+}
+
+func (p *parser) parseReturnStmt() *ast.ReturnStmt {
+	if p.trace {
+		defer un(trace(p, "ReturnStmt"))
+	}
+
+	pos := p.pos
+	p.expect(token.RETURN)
+	var x []ast.Expr
+	if p.tok != token.SEMICOLON && p.tok != token.RBRACE {
+		x = p.parseRhsList()
+	}
+	p.expectSemi()
+
+	return &ast.ReturnStmt{pos, x}
+}
+
+func (p *parser) parseBranchStmt(tok token.Token) *ast.BranchStmt {
+	if p.trace {
+		defer un(trace(p, "BranchStmt"))
+	}
+
+	pos := p.expect(tok)
+	var label *ast.Ident
+	if tok != token.FALLTHROUGH && p.tok == token.IDENT {
+		label = p.parseIdent()
+		// add to list of unresolved targets
+		n := len(p.targetStack) - 1
+		p.targetStack[n] = append(p.targetStack[n], label)
+	}
+	p.expectSemi()
+
+	return &ast.BranchStmt{pos, tok, label}
+}
+
+func (p *parser) makeExpr(s ast.Stmt) ast.Expr {
+	if s == nil {
+		return nil
+	}
+	if es, isExpr := s.(*ast.ExprStmt); isExpr {
+		return p.checkExpr(es.X)
+	}
+	p.error(s.Pos(), "expected condition, found simple statement")
+	return &ast.BadExpr{s.Pos(), s.End()}
+}
+
+func (p *parser) parseIfStmt() *ast.IfStmt {
+	if p.trace {
+		defer un(trace(p, "IfStmt"))
+	}
+
+	pos := p.expect(token.IF)
+	p.openScope()
+	defer p.closeScope()
+
+	var s ast.Stmt
+	var x ast.Expr
+	{
+		prevLev := p.exprLev
+		p.exprLev = -1
+		if p.tok == token.SEMICOLON {
+			p.next()
+			x = p.parseRhs()
+		} else {
+			s, _ = p.parseSimpleStmt(basic)
+			if p.tok == token.SEMICOLON {
+				p.next()
+				x = p.parseRhs()
+			} else {
+				x = p.makeExpr(s)
+				s = nil
+			}
+		}
+		p.exprLev = prevLev
+	}
+
+	body := p.parseBlockStmt()
+	var else_ ast.Stmt
+	if p.tok == token.ELSE {
+		p.next()
+		else_ = p.parseStmt()
+	} else {
+		p.expectSemi()
+	}
+
+	return &ast.IfStmt{pos, s, x, body, else_}
+}
+
+func (p *parser) parseTypeList() (list []ast.Expr) {
+	if p.trace {
+		defer un(trace(p, "TypeList"))
+	}
+
+	list = append(list, p.parseType())
+	for p.tok == token.COMMA {
+		p.next()
+		list = append(list, p.parseType())
+	}
+
+	return
+}
+
+func (p *parser) parseCaseClause(exprSwitch bool) *ast.CaseClause {
+	if p.trace {
+		defer un(trace(p, "CaseClause"))
+	}
+
+	pos := p.pos
+	var list []ast.Expr
+	if p.tok == token.CASE {
+		p.next()
+		if exprSwitch {
+			list = p.parseRhsList()
+		} else {
+			list = p.parseTypeList()
+		}
+	} else {
+		p.expect(token.DEFAULT)
+	}
+
+	colon := p.expect(token.COLON)
+	p.openScope()
+	body := p.parseStmtList()
+	p.closeScope()
+
+	return &ast.CaseClause{pos, list, colon, body}
+}
+
+func isExprSwitch(s ast.Stmt) bool {
+	if s == nil {
+		return true
+	}
+	if e, ok := s.(*ast.ExprStmt); ok {
+		if a, ok := e.X.(*ast.TypeAssertExpr); ok {
+			return a.Type != nil // regular type assertion
+		}
+		return true
+	}
+	return false
+}
+
+func (p *parser) parseSwitchStmt() ast.Stmt {
+	if p.trace {
+		defer un(trace(p, "SwitchStmt"))
+	}
+
+	pos := p.expect(token.SWITCH)
+	p.openScope()
+	defer p.closeScope()
+
+	var s1, s2 ast.Stmt
+	if p.tok != token.LBRACE {
+		prevLev := p.exprLev
+		p.exprLev = -1
+		if p.tok != token.SEMICOLON {
+			s2, _ = p.parseSimpleStmt(basic)
+		}
+		if p.tok == token.SEMICOLON {
+			p.next()
+			s1 = s2
+			s2 = nil
+			if p.tok != token.LBRACE {
+				s2, _ = p.parseSimpleStmt(basic)
+			}
+		}
+		p.exprLev = prevLev
+	}
+
+	exprSwitch := isExprSwitch(s2)
+	lbrace := p.expect(token.LBRACE)
+	var list []ast.Stmt
+	for p.tok == token.CASE || p.tok == token.DEFAULT {
+		list = append(list, p.parseCaseClause(exprSwitch))
+	}
+	rbrace := p.expect(token.RBRACE)
+	p.expectSemi()
+	body := &ast.BlockStmt{lbrace, list, rbrace}
+
+	if exprSwitch {
+		return &ast.SwitchStmt{pos, s1, p.makeExpr(s2), body}
+	}
+	// type switch
+	// TODO(gri): do all the checks!
+	return &ast.TypeSwitchStmt{pos, s1, s2, body}
+}
+
+func (p *parser) parseCommClause() *ast.CommClause {
+	if p.trace {
+		defer un(trace(p, "CommClause"))
+	}
+
+	p.openScope()
+	pos := p.pos
+	var comm ast.Stmt
+	if p.tok == token.CASE {
+		p.next()
+		lhs := p.parseLhsList()
+		if p.tok == token.ARROW {
+			// SendStmt
+			if len(lhs) > 1 {
+				p.errorExpected(lhs[0].Pos(), "1 expression")
+				// continue with first expression
+			}
+			arrow := p.pos
+			p.next()
+			rhs := p.parseRhs()
+			comm = &ast.SendStmt{lhs[0], arrow, rhs}
+		} else {
+			// RecvStmt
+			pos := p.pos
+			tok := p.tok
+			var rhs ast.Expr
+			if tok == token.ASSIGN || tok == token.DEFINE {
+				// RecvStmt with assignment
+				if len(lhs) > 2 {
+					p.errorExpected(lhs[0].Pos(), "1 or 2 expressions")
+					// continue with first two expressions
+					lhs = lhs[0:2]
+				}
+				p.next()
+				rhs = p.parseRhs()
+			} else {
+				// rhs must be single receive operation
+				if len(lhs) > 1 {
+					p.errorExpected(lhs[0].Pos(), "1 expression")
+					// continue with first expression
+				}
+				rhs = lhs[0]
+				lhs = nil // there is no lhs
+			}
+			if lhs != nil {
+				comm = &ast.AssignStmt{lhs, pos, tok, []ast.Expr{rhs}}
+			} else {
+				comm = &ast.ExprStmt{rhs}
+			}
+		}
+	} else {
+		p.expect(token.DEFAULT)
+	}
+
+	colon := p.expect(token.COLON)
+	body := p.parseStmtList()
+	p.closeScope()
+
+	return &ast.CommClause{pos, comm, colon, body}
+}
+
+func (p *parser) parseSelectStmt() *ast.SelectStmt {
+	if p.trace {
+		defer un(trace(p, "SelectStmt"))
+	}
+
+	pos := p.expect(token.SELECT)
+	lbrace := p.expect(token.LBRACE)
+	var list []ast.Stmt
+	for p.tok == token.CASE || p.tok == token.DEFAULT {
+		list = append(list, p.parseCommClause())
+	}
+	rbrace := p.expect(token.RBRACE)
+	p.expectSemi()
+	body := &ast.BlockStmt{lbrace, list, rbrace}
+
+	return &ast.SelectStmt{pos, body}
+}
+
+func (p *parser) parseForStmt() ast.Stmt {
+	if p.trace {
+		defer un(trace(p, "ForStmt"))
+	}
+
+	pos := p.expect(token.FOR)
+	p.openScope()
+	defer p.closeScope()
+
+	var s1, s2, s3 ast.Stmt
+	var isRange bool
+	if p.tok != token.LBRACE {
+		prevLev := p.exprLev
+		p.exprLev = -1
+		if p.tok != token.SEMICOLON {
+			s2, isRange = p.parseSimpleStmt(rangeOk)
+		}
+		if !isRange && p.tok == token.SEMICOLON {
+			p.next()
+			s1 = s2
+			s2 = nil
+			if p.tok != token.SEMICOLON {
+				s2, _ = p.parseSimpleStmt(basic)
+			}
+			p.expectSemi()
+			if p.tok != token.LBRACE {
+				s3, _ = p.parseSimpleStmt(basic)
+			}
+		}
+		p.exprLev = prevLev
+	}
+
+	body := p.parseBlockStmt()
+	p.expectSemi()
+
+	if isRange {
+		as := s2.(*ast.AssignStmt)
+		// check lhs
+		var key, value ast.Expr
+		switch len(as.Lhs) {
+		case 2:
+			key, value = as.Lhs[0], as.Lhs[1]
+		case 1:
+			key = as.Lhs[0]
+		default:
+			p.errorExpected(as.Lhs[0].Pos(), "1 or 2 expressions")
+			return &ast.BadStmt{pos, body.End()}
+		}
+		// parseSimpleStmt returned a right-hand side that
+		// is a single unary expression of the form "range x"
+		x := as.Rhs[0].(*ast.UnaryExpr).X
+		return &ast.RangeStmt{pos, key, value, as.TokPos, as.Tok, x, body}
+	}
+
+	// regular for statement
+	return &ast.ForStmt{pos, s1, p.makeExpr(s2), s3, body}
+}
+
+func (p *parser) parseStmt() (s ast.Stmt) {
+	if p.trace {
+		defer un(trace(p, "Statement"))
+	}
+
+	switch p.tok {
+	case token.CONST, token.TYPE, token.VAR:
+		s = &ast.DeclStmt{p.parseDecl()}
+	case
+		// tokens that may start a top-level expression
+		token.IDENT, token.INT, token.FLOAT, token.CHAR, token.STRING, token.FUNC, token.LPAREN, // operand
+		token.LBRACK, token.STRUCT, // composite type
+		token.MUL, token.AND, token.ARROW, token.ADD, token.SUB, token.XOR: // unary operators
+		s, _ = p.parseSimpleStmt(labelOk)
+		// because of the required look-ahead, labeled statements are
+		// parsed by parseSimpleStmt - don't expect a semicolon after
+		// them
+		if _, isLabeledStmt := s.(*ast.LabeledStmt); !isLabeledStmt {
+			p.expectSemi()
+		}
+	case token.GO:
+		s = p.parseGoStmt()
+	case token.DEFER:
+		s = p.parseDeferStmt()
+	case token.RETURN:
+		s = p.parseReturnStmt()
+	case token.BREAK, token.CONTINUE, token.GOTO, token.FALLTHROUGH:
+		s = p.parseBranchStmt(p.tok)
+	case token.LBRACE:
+		s = p.parseBlockStmt()
+		p.expectSemi()
+	case token.IF:
+		s = p.parseIfStmt()
+	case token.SWITCH:
+		s = p.parseSwitchStmt()
+	case token.SELECT:
+		s = p.parseSelectStmt()
+	case token.FOR:
+		s = p.parseForStmt()
+	case token.SEMICOLON:
+		s = &ast.EmptyStmt{p.pos}
+		p.next()
+	case token.RBRACE:
+		// a semicolon may be omitted before a closing "}"
+		s = &ast.EmptyStmt{p.pos}
+	default:
+		// no statement found
+		pos := p.pos
+		p.errorExpected(pos, "statement")
+		p.next() // make progress
+		s = &ast.BadStmt{pos, p.pos}
+	}
+
+	return
+}
+
+// ----------------------------------------------------------------------------
+// Declarations
+
+type parseSpecFunction func(p *parser, doc *ast.CommentGroup, iota int) ast.Spec
+
+func parseImportSpec(p *parser, doc *ast.CommentGroup, _ int) ast.Spec {
+	if p.trace {
+		defer un(trace(p, "ImportSpec"))
+	}
+
+	var ident *ast.Ident
+	switch p.tok {
+	case token.PERIOD:
+		ident = &ast.Ident{p.pos, ".", nil}
+		p.next()
+	case token.IDENT:
+		ident = p.parseIdent()
+	}
+
+	var path *ast.BasicLit
+	if p.tok == token.STRING {
+		path = &ast.BasicLit{p.pos, p.tok, p.lit}
+		p.next()
+	} else {
+		p.expect(token.STRING) // use expect() error handling
+	}
+	p.expectSemi() // call before accessing p.linecomment
+
+	// collect imports
+	spec := &ast.ImportSpec{doc, ident, path, p.lineComment}
+	p.imports = append(p.imports, spec)
+
+	return spec
+}
+
+func parseConstSpec(p *parser, doc *ast.CommentGroup, iota int) ast.Spec {
+	if p.trace {
+		defer un(trace(p, "ConstSpec"))
+	}
+
+	idents := p.parseIdentList()
+	typ := p.tryType()
+	var values []ast.Expr
+	if typ != nil || p.tok == token.ASSIGN || iota == 0 {
+		p.expect(token.ASSIGN)
+		values = p.parseRhsList()
+	}
+	p.expectSemi() // call before accessing p.linecomment
+
+	// Go spec: The scope of a constant or variable identifier declared inside
+	// a function begins at the end of the ConstSpec or VarSpec and ends at
+	// the end of the innermost containing block.
+	// (Global identifiers are resolved in a separate phase after parsing.)
+	spec := &ast.ValueSpec{doc, idents, typ, values, p.lineComment}
+	p.declare(spec, iota, p.topScope, ast.Con, idents...)
+
+	return spec
+}
+
+func parseTypeSpec(p *parser, doc *ast.CommentGroup, _ int) ast.Spec {
+	if p.trace {
+		defer un(trace(p, "TypeSpec"))
+	}
+
+	ident := p.parseIdent()
+
+	// Go spec: The scope of a type identifier declared inside a function begins
+	// at the identifier in the TypeSpec and ends at the end of the innermost
+	// containing block.
+	// (Global identifiers are resolved in a separate phase after parsing.)
+	spec := &ast.TypeSpec{doc, ident, nil, nil}
+	p.declare(spec, nil, p.topScope, ast.Typ, ident)
+
+	spec.Type = p.parseType()
+	p.expectSemi() // call before accessing p.linecomment
+	spec.Comment = p.lineComment
+
+	return spec
+}
+
+func parseVarSpec(p *parser, doc *ast.CommentGroup, _ int) ast.Spec {
+	if p.trace {
+		defer un(trace(p, "VarSpec"))
+	}
+
+	idents := p.parseIdentList()
+	typ := p.tryType()
+	var values []ast.Expr
+	if typ == nil || p.tok == token.ASSIGN {
+		p.expect(token.ASSIGN)
+		values = p.parseRhsList()
+	}
+	p.expectSemi() // call before accessing p.linecomment
+
+	// Go spec: The scope of a constant or variable identifier declared inside
+	// a function begins at the end of the ConstSpec or VarSpec and ends at
+	// the end of the innermost containing block.
+	// (Global identifiers are resolved in a separate phase after parsing.)
+	spec := &ast.ValueSpec{doc, idents, typ, values, p.lineComment}
+	p.declare(spec, nil, p.topScope, ast.Var, idents...)
+
+	return spec
+}
+
+func (p *parser) parseGenDecl(keyword token.Token, f parseSpecFunction) *ast.GenDecl {
+	if p.trace {
+		defer un(trace(p, "GenDecl("+keyword.String()+")"))
+	}
+
+	doc := p.leadComment
+	pos := p.expect(keyword)
+	var lparen, rparen token.Pos
+	var list []ast.Spec
+	if p.tok == token.LPAREN {
+		lparen = p.pos
+		p.next()
+		for iota := 0; p.tok != token.RPAREN && p.tok != token.EOF; iota++ {
+			list = append(list, f(p, p.leadComment, iota))
+		}
+		rparen = p.expect(token.RPAREN)
+		p.expectSemi()
+	} else {
+		list = append(list, f(p, nil, 0))
+	}
+
+	return &ast.GenDecl{doc, pos, keyword, lparen, list, rparen}
+}
+
+func (p *parser) parseReceiver(scope *ast.Scope) *ast.FieldList {
+	if p.trace {
+		defer un(trace(p, "Receiver"))
+	}
+
+	pos := p.pos
+	par := p.parseParameters(scope, false)
+
+	// must have exactly one receiver
+	if par.NumFields() != 1 {
+		p.errorExpected(pos, "exactly one receiver")
+		// TODO determine a better range for BadExpr below
+		par.List = []*ast.Field{&ast.Field{Type: &ast.BadExpr{pos, pos}}}
+		return par
+	}
+
+	// recv type must be of the form ["*"] identifier
+	recv := par.List[0]
+	base := deref(recv.Type)
+	if _, isIdent := base.(*ast.Ident); !isIdent {
+		p.errorExpected(base.Pos(), "(unqualified) identifier")
+		par.List = []*ast.Field{&ast.Field{Type: &ast.BadExpr{recv.Pos(), recv.End()}}}
+	}
+
+	return par
+}
+
+func (p *parser) parseFuncDecl() *ast.FuncDecl {
+	if p.trace {
+		defer un(trace(p, "FunctionDecl"))
+	}
+
+	doc := p.leadComment
+	pos := p.expect(token.FUNC)
+	scope := ast.NewScope(p.topScope) // function scope
+
+	var recv *ast.FieldList
+	if p.tok == token.LPAREN {
+		recv = p.parseReceiver(scope)
+	}
+
+	ident := p.parseIdent()
+
+	params, results := p.parseSignature(scope)
+
+	var body *ast.BlockStmt
+	if p.tok == token.LBRACE {
+		body = p.parseBody(scope)
+	}
+	p.expectSemi()
+
+	decl := &ast.FuncDecl{doc, recv, ident, &ast.FuncType{pos, params, results}, body}
+	if recv == nil {
+		// Go spec: The scope of an identifier denoting a constant, type,
+		// variable, or function (but not method) declared at top level
+		// (outside any function) is the package block.
+		//
+		// init() functions cannot be referred to and there may
+		// be more than one - don't put them in the pkgScope
+		if ident.Name != "init" {
+			p.declare(decl, nil, p.pkgScope, ast.Fun, ident)
+		}
+	}
+
+	return decl
+}
+
+func (p *parser) parseDecl() ast.Decl {
+	if p.trace {
+		defer un(trace(p, "Declaration"))
+	}
+
+	var f parseSpecFunction
+	switch p.tok {
+	case token.CONST:
+		f = parseConstSpec
+
+	case token.TYPE:
+		f = parseTypeSpec
+
+	case token.VAR:
+		f = parseVarSpec
+
+	case token.FUNC:
+		return p.parseFuncDecl()
+
+	default:
+		pos := p.pos
+		p.errorExpected(pos, "declaration")
+		p.next() // make progress
+		decl := &ast.BadDecl{pos, p.pos}
+		return decl
+	}
+
+	return p.parseGenDecl(p.tok, f)
+}
+
+func (p *parser) parseDeclList() (list []ast.Decl) {
+	if p.trace {
+		defer un(trace(p, "DeclList"))
+	}
+
+	for p.tok != token.EOF {
+		list = append(list, p.parseDecl())
+	}
+
+	return
+}
+
+// ----------------------------------------------------------------------------
+// Source files
+
+func (p *parser) parseFile() *ast.File {
+	if p.trace {
+		defer un(trace(p, "File"))
+	}
+
+	// package clause
+	doc := p.leadComment
+	pos := p.expect(token.PACKAGE)
+	// Go spec: The package clause is not a declaration;
+	// the package name does not appear in any scope.
+	ident := p.parseIdent()
+	if ident.Name == "_" {
+		p.error(p.pos, "invalid package name _")
+	}
+	p.expectSemi()
+
+	var decls []ast.Decl
+
+	// Don't bother parsing the rest if we had errors already.
+	// Likely not a Go source file at all.
+
+	if p.ErrorCount() == 0 && p.mode&PackageClauseOnly == 0 {
+		// import decls
+		for p.tok == token.IMPORT {
+			decls = append(decls, p.parseGenDecl(token.IMPORT, parseImportSpec))
+		}
+
+		if p.mode&ImportsOnly == 0 {
+			// rest of package body
+			for p.tok != token.EOF {
+				decls = append(decls, p.parseDecl())
+			}
+		}
+	}
+
+	assert(p.topScope == p.pkgScope, "imbalanced scopes")
+
+	// resolve global identifiers within the same file
+	i := 0
+	for _, ident := range p.unresolved {
+		// i <= index for current ident
+		assert(ident.Obj == unresolved, "object already resolved")
+		ident.Obj = p.pkgScope.Lookup(ident.Name) // also removes unresolved sentinel
+		if ident.Obj == nil {
+			p.unresolved[i] = ident
+			i++
+		}
+	}
+
+	return &ast.File{doc, pos, ident, decls, p.pkgScope, p.imports, p.unresolved[0:i], p.comments}
+}
diff --git a/src/pkg/go/parser/parser_test.go b/src/pkg/go/parser/parser_test.go
new file mode 100644
index 000000000..39a78e515
--- /dev/null
+++ b/src/pkg/go/parser/parser_test.go
@@ -0,0 +1,130 @@
+// Copyright 2009 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package parser
+
+import (
+	"go/token"
+	"os"
+	"testing"
+)
+
+var fset = token.NewFileSet()
+
+var illegalInputs = []interface{}{
+	nil,
+	3.14,
+	[]byte(nil),
+	"foo!",
+	`package p; func f() { if /* should have condition */ {} };`,
+	`package p; func f() { if ; /* should have condition */ {} };`,
+	`package p; func f() { if f(); /* should have condition */ {} };`,
+	`package p; const c; /* should have constant value */`,
+	`package p; func f() { if _ = range x; true {} };`,
+	`package p; func f() { switch _ = range x; true {} };`,
+	`package p; func f() { for _ = range x ; ; {} };`,
+	`package p; func f() { for ; ; _ = range x {} };`,
+	`package p; func f() { for ; _ = range x ; {} };`,
+	`package p; var a = [1]int; /* illegal expression */`,
+	`package p; var a = [...]int; /* illegal expression */`,
+	`package p; var a = struct{} /* illegal expression */`,
+	`package p; var a = func(); /* illegal expression */`,
+	`package p; var a = interface{} /* illegal expression */`,
+	`package p; var a = []int /* illegal expression */`,
+	`package p; var a = map[int]int /* illegal expression */`,
+	`package p; var a = chan int; /* illegal expression */`,
+	`package p; var a = []int{[]int}; /* illegal expression */`,
+	`package p; var a = ([]int); /* illegal expression */`,
+	`package p; var a = a[[]int:[]int]; /* illegal expression */`,
+	`package p; var a = <- chan int; /* illegal expression */`,
+	`package p; func f() { select { case _ <- chan int: } };`,
+}
+
+func TestParseIllegalInputs(t *testing.T) {
+	for _, src := range illegalInputs {
+		_, err := ParseFile(fset, "", src, 0)
+		if err == nil {
+			t.Errorf("ParseFile(%v) should have failed", src)
+		}
+	}
+}
+
+var validPrograms = []interface{}{
+	"package p\n",
+	`package p;`,
+	`package p; import "fmt"; func f() { fmt.Println("Hello, World!") };`,
+	`package p; func f() { if f(T{}) {} };`,
+	`package p; func f() { _ = (<-chan int)(x) };`,
+	`package p; func f() { _ = (<-chan <-chan int)(x) };`,
+	`package p; func f(func() func() func());`,
+	`package p; func f(...T);`,
+	`package p; func f(float, ...int);`,
+	`package p; func f(x int, a ...int) { f(0, a...); f(1, a...,) };`,
+	`package p; type T []int; var a []bool; func f() { if a[T{42}[0]] {} };`,
+	`package p; type T []int; func g(int) bool { return true }; func f() { if g(T{42}[0]) {} };`,
+	`package p; type T []int; func f() { for _ = range []int{T{42}[0]} {} };`,
+	`package p; var a = T{{1, 2}, {3, 4}}`,
+	`package p; func f() { select { case <- c: case c <- d: case c <- <- d: case <-c <- d: } };`,
+	`package p; func f() { select { case x := (<-c): } };`,
+	`package p; func f() { if ; true {} };`,
+	`package p; func f() { switch ; {} };`,
+	`package p; func f() { for _ = range "foo" + "bar" {} };`,
+}
+
+func TestParseValidPrograms(t *testing.T) {
+	for _, src := range validPrograms {
+		_, err := ParseFile(fset, "", src, 0)
+		if err != nil {
+			t.Errorf("ParseFile(%q): %v", src, err)
+		}
+	}
+}
+
+var validFiles = []string{
+	"parser.go",
+	"parser_test.go",
+}
+
+func TestParse3(t *testing.T) {
+	for _, filename := range validFiles {
+		_, err := ParseFile(fset, filename, nil, DeclarationErrors)
+		if err != nil {
+			t.Errorf("ParseFile(%s): %v", filename, err)
+		}
+	}
+}
+
+func nameFilter(filename string) bool {
+	switch filename {
+	case "parser.go":
+	case "interface.go":
+	case "parser_test.go":
+	default:
+		return false
+	}
+	return true
+}
+
+func dirFilter(f *os.FileInfo) bool { return nameFilter(f.Name) }
+
+func TestParse4(t *testing.T) {
+	path := "."
+	pkgs, err := ParseDir(fset, path, dirFilter, 0)
+	if err != nil {
+		t.Fatalf("ParseDir(%s): %v", path, err)
+	}
+	if len(pkgs) != 1 {
+		t.Errorf("incorrect number of packages: %d", len(pkgs))
+	}
+	pkg := pkgs["parser"]
+	if pkg == nil {
+		t.Errorf(`package "parser" not found`)
+		return
+	}
+	for filename := range pkg.Files {
+		if !nameFilter(filename) {
+			t.Errorf("unexpected package file: %s", filename)
+		}
+	}
+}
diff --git a/src/pkg/go/printer/Makefile b/src/pkg/go/printer/Makefile
new file mode 100644
index 000000000..6a71efc93
--- /dev/null
+++ b/src/pkg/go/printer/Makefile
@@ -0,0 +1,12 @@
+# Copyright 2009 The Go Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style
+# license that can be found in the LICENSE file.
+
+include ../../../Make.inc
+
+TARG=go/printer
+GOFILES=\
+	printer.go\
+	nodes.go\
+
+include ../../../Make.pkg
diff --git a/src/pkg/go/printer/nodes.go b/src/pkg/go/printer/nodes.go
new file mode 100644
index 000000000..9cd975ec1
--- /dev/null
+++ b/src/pkg/go/printer/nodes.go
@@ -0,0 +1,1510 @@
+// Copyright 2009 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// This file implements printing of AST nodes; specifically
+// expressions, statements, declarations, and files. It uses
+// the print functionality implemented in printer.go.
+
+package printer
+
+import (
+	"bytes"
+	"go/ast"
+	"go/token"
+)
+
+// Other formatting issues:
+// - better comment formatting for /*-style comments at the end of a line (e.g. a declaration)
+//   when the comment spans multiple lines; if such a comment is just two lines, formatting is
+//   not idempotent
+// - formatting of expression lists
+// - should use blank instead of tab to separate one-line function bodies from
+//   the function header unless there is a group of consecutive one-liners
+
+// ----------------------------------------------------------------------------
+// Common AST nodes.
+
+// Print as many newlines as necessary (but at least min newlines) to get to
+// the current line. ws is printed before the first line break. If newSection
+// is set, the first line break is printed as formfeed. Returns true if any
+// line break was printed; returns false otherwise.
+//
+// TODO(gri): linebreak may add too many lines if the next statement at "line"
+//            is preceded by comments because the computation of n assumes
+//            the current position before the comment and the target position
+//            after the comment. Thus, after interspersing such comments, the
+//            space taken up by them is not considered to reduce the number of
+//            linebreaks. At the moment there is no easy way to know about
+//            future (not yet interspersed) comments in this function.
+//
+func (p *printer) linebreak(line, min int, ws whiteSpace, newSection bool) (printedBreak bool) {
+	n := p.nlines(line-p.pos.Line, min)
+	if n > 0 {
+		p.print(ws)
+		if newSection {
+			p.print(formfeed)
+			n--
+		}
+		for ; n > 0; n-- {
+			p.print(newline)
+		}
+		printedBreak = true
+	}
+	return
+}
+
+// setComment sets g as the next comment if g != nil and if node comments
+// are enabled - this mode is used when printing source code fragments such
+// as exports only. It assumes that there are no other pending comments to
+// intersperse.
+func (p *printer) setComment(g *ast.CommentGroup) {
+	if g == nil || !p.useNodeComments {
+		return
+	}
+	if p.comments == nil {
+		// initialize p.comments lazily
+		p.comments = make([]*ast.CommentGroup, 1)
+	} else if p.cindex < len(p.comments) {
+		// for some reason there are pending comments; this
+		// should never happen - handle gracefully and flush
+		// all comments up to g, ignore anything after that
+		p.flush(p.fset.Position(g.List[0].Pos()), token.ILLEGAL)
+	}
+	p.comments[0] = g
+	p.cindex = 0
+}
+
+type exprListMode uint
+
+const (
+	blankStart exprListMode = 1 << iota // print a blank before a non-empty list
+	blankEnd                            // print a blank after a non-empty list
+	commaSep                            // elements are separated by commas
+	commaTerm                           // list is optionally terminated by a comma
+	noIndent                            // no extra indentation in multi-line lists
+	periodSep                           // elements are separated by periods
+)
+
+// Sets multiLine to true if the identifier list spans multiple lines.
+// If indent is set, a multi-line identifier list is indented after the
+// first linebreak encountered.
+func (p *printer) identList(list []*ast.Ident, indent bool, multiLine *bool) {
+	// convert into an expression list so we can re-use exprList formatting
+	xlist := make([]ast.Expr, len(list))
+	for i, x := range list {
+		xlist[i] = x
+	}
+	mode := commaSep
+	if !indent {
+		mode |= noIndent
+	}
+	p.exprList(token.NoPos, xlist, 1, mode, multiLine, token.NoPos)
+}
+
+// Print a list of expressions. If the list spans multiple
+// source lines, the original line breaks are respected between
+// expressions. Sets multiLine to true if the list spans multiple
+// lines.
+//
+// TODO(gri) Consider rewriting this to be independent of []ast.Expr
+//           so that we can use the algorithm for any kind of list
+//           (e.g., pass list via a channel over which to range).
+func (p *printer) exprList(prev0 token.Pos, list []ast.Expr, depth int, mode exprListMode, multiLine *bool, next0 token.Pos) {
+	if len(list) == 0 {
+		return
+	}
+
+	if mode&blankStart != 0 {
+		p.print(blank)
+	}
+
+	prev := p.fset.Position(prev0)
+	next := p.fset.Position(next0)
+	line := p.fset.Position(list[0].Pos()).Line
+	endLine := p.fset.Position(list[len(list)-1].End()).Line
+
+	if prev.IsValid() && prev.Line == line && line == endLine {
+		// all list entries on a single line
+		for i, x := range list {
+			if i > 0 {
+				if mode&commaSep != 0 {
+					p.print(token.COMMA)
+				}
+				p.print(blank)
+			}
+			p.expr0(x, depth, multiLine)
+		}
+		if mode&blankEnd != 0 {
+			p.print(blank)
+		}
+		return
+	}
+
+	// list entries span multiple lines;
+	// use source code positions to guide line breaks
+
+	// don't add extra indentation if noIndent is set;
+	// i.e., pretend that the first line is already indented
+	ws := ignore
+	if mode&noIndent == 0 {
+		ws = indent
+	}
+
+	// the first linebreak is always a formfeed since this section must not
+	// depend on any previous formatting
+	prevBreak := -1 // index of last expression that was followed by a linebreak
+	if prev.IsValid() && prev.Line < line && p.linebreak(line, 0, ws, true) {
+		ws = ignore
+		*multiLine = true
+		prevBreak = 0
+	}
+
+	// initialize expression/key size: a zero value indicates expr/key doesn't fit on a single line
+	size := 0
+
+	// print all list elements
+	for i, x := range list {
+		prevLine := line
+		line = p.fset.Position(x.Pos()).Line
+
+		// determine if the next linebreak, if any, needs to use formfeed:
+		// in general, use the entire node size to make the decision; for
+		// key:value expressions, use the key size
+		// TODO(gri) for a better result, should probably incorporate both
+		//           the key and the node size into the decision process
+		useFF := true
+
+		// determine element size: all bets are off if we don't have
+		// position information for the previous and next token (likely
+		// generated code - simply ignore the size in this case by setting
+		// it to 0)
+		prevSize := size
+		const infinity = 1e6 // larger than any source line
+		size = p.nodeSize(x, infinity)
+		pair, isPair := x.(*ast.KeyValueExpr)
+		if size <= infinity && prev.IsValid() && next.IsValid() {
+			// x fits on a single line
+			if isPair {
+				size = p.nodeSize(pair.Key, infinity) // size <= infinity
+			}
+		} else {
+			// size too large or we don't have good layout information
+			size = 0
+		}
+
+		// if the previous line and the current line had single-
+		// line-expressions and the key sizes are small or the
+		// the ratio between the key sizes does not exceed a
+		// threshold, align columns and do not use formfeed
+		if prevSize > 0 && size > 0 {
+			const smallSize = 20
+			if prevSize <= smallSize && size <= smallSize {
+				useFF = false
+			} else {
+				const r = 4 // threshold
+				ratio := float64(size) / float64(prevSize)
+				useFF = ratio <= 1/r || r <= ratio
+			}
+		}
+
+		if i > 0 {
+			switch {
+			case mode&commaSep != 0:
+				p.print(token.COMMA)
+			case mode&periodSep != 0:
+				p.print(token.PERIOD)
+			}
+			needsBlank := mode&periodSep == 0 // period-separated list elements don't need a blank
+			if prevLine < line && prevLine > 0 && line > 0 {
+				// lines are broken using newlines so comments remain aligned
+				// unless forceFF is set or there are multiple expressions on
+				// the same line in which case formfeed is used
+				if p.linebreak(line, 0, ws, useFF || prevBreak+1 < i) {
+					ws = ignore
+					*multiLine = true
+					prevBreak = i
+					needsBlank = false // we got a line break instead
+				}
+			}
+			if needsBlank {
+				p.print(blank)
+			}
+		}
+
+		if isPair && size > 0 && len(list) > 1 {
+			// we have a key:value expression that fits onto one line and
+			// is in a list with more then one entry: use a column for the
+			// key such that consecutive entries can align if possible
+			p.expr(pair.Key, multiLine)
+			p.print(pair.Colon, token.COLON, vtab)
+			p.expr(pair.Value, multiLine)
+		} else {
+			p.expr0(x, depth, multiLine)
+		}
+	}
+
+	if mode&commaTerm != 0 && next.IsValid() && p.pos.Line < next.Line {
+		// print a terminating comma if the next token is on a new line
+		p.print(token.COMMA)
+		if ws == ignore && mode&noIndent == 0 {
+			// unindent if we indented
+			p.print(unindent)
+		}
+		p.print(formfeed) // terminating comma needs a line break to look good
+		return
+	}
+
+	if mode&blankEnd != 0 {
+		p.print(blank)
+	}
+
+	if ws == ignore && mode&noIndent == 0 {
+		// unindent if we indented
+		p.print(unindent)
+	}
+}
+
+// Sets multiLine to true if the the parameter list spans multiple lines.
+func (p *printer) parameters(fields *ast.FieldList, multiLine *bool) {
+	p.print(fields.Opening, token.LPAREN)
+	if len(fields.List) > 0 {
+		var prevLine, line int
+		for i, par := range fields.List {
+			if i > 0 {
+				p.print(token.COMMA)
+				if len(par.Names) > 0 {
+					line = p.fset.Position(par.Names[0].Pos()).Line
+				} else {
+					line = p.fset.Position(par.Type.Pos()).Line
+				}
+				if 0 < prevLine && prevLine < line && p.linebreak(line, 0, ignore, true) {
+					*multiLine = true
+				} else {
+					p.print(blank)
+				}
+			}
+			if len(par.Names) > 0 {
+				p.identList(par.Names, false, multiLine)
+				p.print(blank)
+			}
+			p.expr(par.Type, multiLine)
+			prevLine = p.fset.Position(par.Type.Pos()).Line
+		}
+	}
+	p.print(fields.Closing, token.RPAREN)
+}
+
+// Sets multiLine to true if the signature spans multiple lines.
+func (p *printer) signature(params, result *ast.FieldList, multiLine *bool) {
+	p.parameters(params, multiLine)
+	n := result.NumFields()
+	if n > 0 {
+		p.print(blank)
+		if n == 1 && result.List[0].Names == nil {
+			// single anonymous result; no ()'s
+			p.expr(result.List[0].Type, multiLine)
+			return
+		}
+		p.parameters(result, multiLine)
+	}
+}
+
+func identListSize(list []*ast.Ident, maxSize int) (size int) {
+	for i, x := range list {
+		if i > 0 {
+			size += 2 // ", "
+		}
+		size += len(x.Name)
+		if size >= maxSize {
+			break
+		}
+	}
+	return
+}
+
+func (p *printer) isOneLineFieldList(list []*ast.Field) bool {
+	if len(list) != 1 {
+		return false // allow only one field
+	}
+	f := list[0]
+	if f.Tag != nil || f.Comment != nil {
+		return false // don't allow tags or comments
+	}
+	// only name(s) and type
+	const maxSize = 30 // adjust as appropriate, this is an approximate value
+	namesSize := identListSize(f.Names, maxSize)
+	if namesSize > 0 {
+		namesSize = 1 // blank between names and types
+	}
+	typeSize := p.nodeSize(f.Type, maxSize)
+	return namesSize+typeSize <= maxSize
+}
+
+func (p *printer) setLineComment(text string) {
+	p.setComment(&ast.CommentGroup{[]*ast.Comment{&ast.Comment{token.NoPos, text}}})
+}
+
+func (p *printer) fieldList(fields *ast.FieldList, isStruct, isIncomplete bool) {
+	lbrace := fields.Opening
+	list := fields.List
+	rbrace := fields.Closing
+	srcIsOneLine := lbrace.IsValid() && rbrace.IsValid() && p.fset.Position(lbrace).Line == p.fset.Position(rbrace).Line
+
+	if !isIncomplete && !p.commentBefore(p.fset.Position(rbrace)) && srcIsOneLine {
+		// possibly a one-line struct/interface
+		if len(list) == 0 {
+			// no blank between keyword and {} in this case
+			p.print(lbrace, token.LBRACE, rbrace, token.RBRACE)
+			return
+		} else if isStruct && p.isOneLineFieldList(list) { // for now ignore interfaces
+			// small enough - print on one line
+			// (don't use identList and ignore source line breaks)
+			p.print(lbrace, token.LBRACE, blank)
+			f := list[0]
+			for i, x := range f.Names {
+				if i > 0 {
+					p.print(token.COMMA, blank)
+				}
+				p.expr(x, ignoreMultiLine)
+			}
+			if len(f.Names) > 0 {
+				p.print(blank)
+			}
+			p.expr(f.Type, ignoreMultiLine)
+			p.print(blank, rbrace, token.RBRACE)
+			return
+		}
+	}
+
+	// at least one entry or incomplete
+	p.print(blank, lbrace, token.LBRACE, indent, formfeed)
+	if isStruct {
+
+		sep := vtab
+		if len(list) == 1 {
+			sep = blank
+		}
+		var ml bool
+		for i, f := range list {
+			if i > 0 {
+				p.linebreak(p.fset.Position(f.Pos()).Line, 1, ignore, ml)
+			}
+			ml = false
+			extraTabs := 0
+			p.setComment(f.Doc)
+			if len(f.Names) > 0 {
+				// named fields
+				p.identList(f.Names, false, &ml)
+				p.print(sep)
+				p.expr(f.Type, &ml)
+				extraTabs = 1
+			} else {
+				// anonymous field
+				p.expr(f.Type, &ml)
+				extraTabs = 2
+			}
+			if f.Tag != nil {
+				if len(f.Names) > 0 && sep == vtab {
+					p.print(sep)
+				}
+				p.print(sep)
+				p.expr(f.Tag, &ml)
+				extraTabs = 0
+			}
+			if f.Comment != nil {
+				for ; extraTabs > 0; extraTabs-- {
+					p.print(sep)
+				}
+				p.setComment(f.Comment)
+			}
+		}
+		if isIncomplete {
+			if len(list) > 0 {
+				p.print(formfeed)
+			}
+			p.flush(p.fset.Position(rbrace), token.RBRACE) // make sure we don't lose the last line comment
+			p.setLineComment("// contains filtered or unexported fields")
+		}
+
+	} else { // interface
+
+		var ml bool
+		for i, f := range list {
+			if i > 0 {
+				p.linebreak(p.fset.Position(f.Pos()).Line, 1, ignore, ml)
+			}
+			ml = false
+			p.setComment(f.Doc)
+			if ftyp, isFtyp := f.Type.(*ast.FuncType); isFtyp {
+				// method
+				p.expr(f.Names[0], &ml)
+				p.signature(ftyp.Params, ftyp.Results, &ml)
+			} else {
+				// embedded interface
+				p.expr(f.Type, &ml)
+			}
+			p.setComment(f.Comment)
+		}
+		if isIncomplete {
+			if len(list) > 0 {
+				p.print(formfeed)
+			}
+			p.flush(p.fset.Position(rbrace), token.RBRACE) // make sure we don't lose the last line comment
+			p.setLineComment("// contains filtered or unexported methods")
+		}
+
+	}
+	p.print(unindent, formfeed, rbrace, token.RBRACE)
+}
+
+// ----------------------------------------------------------------------------
+// Expressions
+
+func walkBinary(e *ast.BinaryExpr) (has4, has5 bool, maxProblem int) {
+	switch e.Op.Precedence() {
+	case 4:
+		has4 = true
+	case 5:
+		has5 = true
+	}
+
+	switch l := e.X.(type) {
+	case *ast.BinaryExpr:
+		if l.Op.Precedence() < e.Op.Precedence() {
+			// parens will be inserted.
+			// pretend this is an *ast.ParenExpr and do nothing.
+			break
+		}
+		h4, h5, mp := walkBinary(l)
+		has4 = has4 || h4
+		has5 = has5 || h5
+		if maxProblem < mp {
+			maxProblem = mp
+		}
+	}
+
+	switch r := e.Y.(type) {
+	case *ast.BinaryExpr:
+		if r.Op.Precedence() <= e.Op.Precedence() {
+			// parens will be inserted.
+			// pretend this is an *ast.ParenExpr and do nothing.
+			break
+		}
+		h4, h5, mp := walkBinary(r)
+		has4 = has4 || h4
+		has5 = has5 || h5
+		if maxProblem < mp {
+			maxProblem = mp
+		}
+
+	case *ast.StarExpr:
+		if e.Op == token.QUO { // `*/`
+			maxProblem = 5
+		}
+
+	case *ast.UnaryExpr:
+		switch e.Op.String() + r.Op.String() {
+		case "/*", "&&", "&^":
+			maxProblem = 5
+		case "++", "--":
+			if maxProblem < 4 {
+				maxProblem = 4
+			}
+		}
+	}
+	return
+}
+
+func cutoff(e *ast.BinaryExpr, depth int) int {
+	has4, has5, maxProblem := walkBinary(e)
+	if maxProblem > 0 {
+		return maxProblem + 1
+	}
+	if has4 && has5 {
+		if depth == 1 {
+			return 5
+		}
+		return 4
+	}
+	if depth == 1 {
+		return 6
+	}
+	return 4
+}
+
+func diffPrec(expr ast.Expr, prec int) int {
+	x, ok := expr.(*ast.BinaryExpr)
+	if !ok || prec != x.Op.Precedence() {
+		return 1
+	}
+	return 0
+}
+
+func reduceDepth(depth int) int {
+	depth--
+	if depth < 1 {
+		depth = 1
+	}
+	return depth
+}
+
+// Format the binary expression: decide the cutoff and then format.
+// Let's call depth == 1 Normal mode, and depth > 1 Compact mode.
+// (Algorithm suggestion by Russ Cox.)
+//
+// The precedences are:
+//	5             *  /  %  <<  >>  &  &^
+//	4             +  -  |  ^
+//	3             ==  !=  <  <=  >  >=
+//	2             &&
+//	1             ||
+//
+// The only decision is whether there will be spaces around levels 4 and 5.
+// There are never spaces at level 6 (unary), and always spaces at levels 3 and below.
+//
+// To choose the cutoff, look at the whole expression but excluding primary
+// expressions (function calls, parenthesized exprs), and apply these rules:
+//
+//	1) If there is a binary operator with a right side unary operand
+//	   that would clash without a space, the cutoff must be (in order):
+//
+//		/*	6
+//		&&	6
+//		&^	6
+//		++	5
+//		--	5
+//
+//         (Comparison operators always have spaces around them.)
+//
+//	2) If there is a mix of level 5 and level 4 operators, then the cutoff
+//	   is 5 (use spaces to distinguish precedence) in Normal mode
+//	   and 4 (never use spaces) in Compact mode.
+//
+//	3) If there are no level 4 operators or no level 5 operators, then the
+//	   cutoff is 6 (always use spaces) in Normal mode
+//	   and 4 (never use spaces) in Compact mode.
+//
+// Sets multiLine to true if the binary expression spans multiple lines.
+func (p *printer) binaryExpr(x *ast.BinaryExpr, prec1, cutoff, depth int, multiLine *bool) {
+	prec := x.Op.Precedence()
+	if prec < prec1 {
+		// parenthesis needed
+		// Note: The parser inserts an ast.ParenExpr node; thus this case
+		//       can only occur if the AST is created in a different way.
+		p.print(token.LPAREN)
+		p.expr0(x, reduceDepth(depth), multiLine) // parentheses undo one level of depth
+		p.print(token.RPAREN)
+		return
+	}
+
+	printBlank := prec < cutoff
+
+	ws := indent
+	p.expr1(x.X, prec, depth+diffPrec(x.X, prec), multiLine)
+	if printBlank {
+		p.print(blank)
+	}
+	xline := p.pos.Line // before the operator (it may be on the next line!)
+	yline := p.fset.Position(x.Y.Pos()).Line
+	p.print(x.OpPos, x.Op)
+	if xline != yline && xline > 0 && yline > 0 {
+		// at least one line break, but respect an extra empty line
+		// in the source
+		if p.linebreak(yline, 1, ws, true) {
+			ws = ignore
+			*multiLine = true
+			printBlank = false // no blank after line break
+		}
+	}
+	if printBlank {
+		p.print(blank)
+	}
+	p.expr1(x.Y, prec+1, depth+1, multiLine)
+	if ws == ignore {
+		p.print(unindent)
+	}
+}
+
+func isBinary(expr ast.Expr) bool {
+	_, ok := expr.(*ast.BinaryExpr)
+	return ok
+}
+
+// If the expression contains one or more selector expressions, splits it into
+// two expressions at the rightmost period. Writes entire expr to suffix when
+// selector isn't found. Rewrites AST nodes for calls, index expressions and
+// type assertions, all of which may be found in selector chains, to make them
+// parts of the chain.
+func splitSelector(expr ast.Expr) (body, suffix ast.Expr) {
+	switch x := expr.(type) {
+	case *ast.SelectorExpr:
+		body, suffix = x.X, x.Sel
+		return
+	case *ast.CallExpr:
+		body, suffix = splitSelector(x.Fun)
+		if body != nil {
+			suffix = &ast.CallExpr{suffix, x.Lparen, x.Args, x.Ellipsis, x.Rparen}
+			return
+		}
+	case *ast.IndexExpr:
+		body, suffix = splitSelector(x.X)
+		if body != nil {
+			suffix = &ast.IndexExpr{suffix, x.Lbrack, x.Index, x.Rbrack}
+			return
+		}
+	case *ast.SliceExpr:
+		body, suffix = splitSelector(x.X)
+		if body != nil {
+			suffix = &ast.SliceExpr{suffix, x.Lbrack, x.Low, x.High, x.Rbrack}
+			return
+		}
+	case *ast.TypeAssertExpr:
+		body, suffix = splitSelector(x.X)
+		if body != nil {
+			suffix = &ast.TypeAssertExpr{suffix, x.Type}
+			return
+		}
+	}
+	suffix = expr
+	return
+}
+
+// Convert an expression into an expression list split at the periods of
+// selector expressions.
+func selectorExprList(expr ast.Expr) (list []ast.Expr) {
+	// split expression
+	for expr != nil {
+		var suffix ast.Expr
+		expr, suffix = splitSelector(expr)
+		list = append(list, suffix)
+	}
+
+	// reverse list
+	for i, j := 0, len(list)-1; i < j; i, j = i+1, j-1 {
+		list[i], list[j] = list[j], list[i]
+	}
+
+	return
+}
+
+// Sets multiLine to true if the expression spans multiple lines.
+func (p *printer) expr1(expr ast.Expr, prec1, depth int, multiLine *bool) {
+	p.print(expr.Pos())
+
+	switch x := expr.(type) {
+	case *ast.BadExpr:
+		p.print("BadExpr")
+
+	case *ast.Ident:
+		p.print(x)
+
+	case *ast.BinaryExpr:
+		if depth < 1 {
+			p.internalError("depth < 1:", depth)
+			depth = 1
+		}
+		p.binaryExpr(x, prec1, cutoff(x, depth), depth, multiLine)
+
+	case *ast.KeyValueExpr:
+		p.expr(x.Key, multiLine)
+		p.print(x.Colon, token.COLON, blank)
+		p.expr(x.Value, multiLine)
+
+	case *ast.StarExpr:
+		const prec = token.UnaryPrec
+		if prec < prec1 {
+			// parenthesis needed
+			p.print(token.LPAREN)
+			p.print(token.MUL)
+			p.expr(x.X, multiLine)
+			p.print(token.RPAREN)
+		} else {
+			// no parenthesis needed
+			p.print(token.MUL)
+			p.expr(x.X, multiLine)
+		}
+
+	case *ast.UnaryExpr:
+		const prec = token.UnaryPrec
+		if prec < prec1 {
+			// parenthesis needed
+			p.print(token.LPAREN)
+			p.expr(x, multiLine)
+			p.print(token.RPAREN)
+		} else {
+			// no parenthesis needed
+			p.print(x.Op)
+			if x.Op == token.RANGE {
+				// TODO(gri) Remove this code if it cannot be reached.
+				p.print(blank)
+			}
+			p.expr1(x.X, prec, depth, multiLine)
+		}
+
+	case *ast.BasicLit:
+		p.print(x)
+
+	case *ast.FuncLit:
+		p.expr(x.Type, multiLine)
+		p.funcBody(x.Body, p.distance(x.Type.Pos(), p.pos), true, multiLine)
+
+	case *ast.ParenExpr:
+		if _, hasParens := x.X.(*ast.ParenExpr); hasParens {
+			// don't print parentheses around an already parenthesized expression
+			// TODO(gri) consider making this more general and incorporate precedence levels
+			p.expr0(x.X, reduceDepth(depth), multiLine) // parentheses undo one level of depth
+		} else {
+			p.print(token.LPAREN)
+			p.expr0(x.X, reduceDepth(depth), multiLine) // parentheses undo one level of depth
+			p.print(x.Rparen, token.RPAREN)
+		}
+
+	case *ast.SelectorExpr:
+		parts := selectorExprList(expr)
+		p.exprList(token.NoPos, parts, depth, periodSep, multiLine, token.NoPos)
+
+	case *ast.TypeAssertExpr:
+		p.expr1(x.X, token.HighestPrec, depth, multiLine)
+		p.print(token.PERIOD, token.LPAREN)
+		if x.Type != nil {
+			p.expr(x.Type, multiLine)
+		} else {
+			p.print(token.TYPE)
+		}
+		p.print(token.RPAREN)
+
+	case *ast.IndexExpr:
+		// TODO(gri): should treat[] like parentheses and undo one level of depth
+		p.expr1(x.X, token.HighestPrec, 1, multiLine)
+		p.print(x.Lbrack, token.LBRACK)
+		p.expr0(x.Index, depth+1, multiLine)
+		p.print(x.Rbrack, token.RBRACK)
+
+	case *ast.SliceExpr:
+		// TODO(gri): should treat[] like parentheses and undo one level of depth
+		p.expr1(x.X, token.HighestPrec, 1, multiLine)
+		p.print(x.Lbrack, token.LBRACK)
+		if x.Low != nil {
+			p.expr0(x.Low, depth+1, multiLine)
+		}
+		// blanks around ":" if both sides exist and either side is a binary expression
+		if depth <= 1 && x.Low != nil && x.High != nil && (isBinary(x.Low) || isBinary(x.High)) {
+			p.print(blank, token.COLON, blank)
+		} else {
+			p.print(token.COLON)
+		}
+		if x.High != nil {
+			p.expr0(x.High, depth+1, multiLine)
+		}
+		p.print(x.Rbrack, token.RBRACK)
+
+	case *ast.CallExpr:
+		if len(x.Args) > 1 {
+			depth++
+		}
+		p.expr1(x.Fun, token.HighestPrec, depth, multiLine)
+		p.print(x.Lparen, token.LPAREN)
+		p.exprList(x.Lparen, x.Args, depth, commaSep|commaTerm, multiLine, x.Rparen)
+		if x.Ellipsis.IsValid() {
+			p.print(x.Ellipsis, token.ELLIPSIS)
+		}
+		p.print(x.Rparen, token.RPAREN)
+
+	case *ast.CompositeLit:
+		// composite literal elements that are composite literals themselves may have the type omitted
+		if x.Type != nil {
+			p.expr1(x.Type, token.HighestPrec, depth, multiLine)
+		}
+		p.print(x.Lbrace, token.LBRACE)
+		p.exprList(x.Lbrace, x.Elts, 1, commaSep|commaTerm, multiLine, x.Rbrace)
+		// do not insert extra line breaks because of comments before
+		// the closing '}' as it might break the code if there is no
+		// trailing ','
+		p.print(noExtraLinebreak, x.Rbrace, token.RBRACE, noExtraLinebreak)
+
+	case *ast.Ellipsis:
+		p.print(token.ELLIPSIS)
+		if x.Elt != nil {
+			p.expr(x.Elt, multiLine)
+		}
+
+	case *ast.ArrayType:
+		p.print(token.LBRACK)
+		if x.Len != nil {
+			p.expr(x.Len, multiLine)
+		}
+		p.print(token.RBRACK)
+		p.expr(x.Elt, multiLine)
+
+	case *ast.StructType:
+		p.print(token.STRUCT)
+		p.fieldList(x.Fields, true, x.Incomplete)
+
+	case *ast.FuncType:
+		p.print(token.FUNC)
+		p.signature(x.Params, x.Results, multiLine)
+
+	case *ast.InterfaceType:
+		p.print(token.INTERFACE)
+		p.fieldList(x.Methods, false, x.Incomplete)
+
+	case *ast.MapType:
+		p.print(token.MAP, token.LBRACK)
+		p.expr(x.Key, multiLine)
+		p.print(token.RBRACK)
+		p.expr(x.Value, multiLine)
+
+	case *ast.ChanType:
+		switch x.Dir {
+		case ast.SEND | ast.RECV:
+			p.print(token.CHAN)
+		case ast.RECV:
+			p.print(token.ARROW, token.CHAN)
+		case ast.SEND:
+			p.print(token.CHAN, token.ARROW)
+		}
+		p.print(blank)
+		p.expr(x.Value, multiLine)
+
+	default:
+		panic("unreachable")
+	}
+
+	return
+}
+
+func (p *printer) expr0(x ast.Expr, depth int, multiLine *bool) {
+	p.expr1(x, token.LowestPrec, depth, multiLine)
+}
+
+// Sets multiLine to true if the expression spans multiple lines.
+func (p *printer) expr(x ast.Expr, multiLine *bool) {
+	const depth = 1
+	p.expr1(x, token.LowestPrec, depth, multiLine)
+}
+
+// ----------------------------------------------------------------------------
+// Statements
+
+// Print the statement list indented, but without a newline after the last statement.
+// Extra line breaks between statements in the source are respected but at most one
+// empty line is printed between statements.
+func (p *printer) stmtList(list []ast.Stmt, _indent int, nextIsRBrace bool) {
+	// TODO(gri): fix _indent code
+	if _indent > 0 {
+		p.print(indent)
+	}
+	var multiLine bool
+	for i, s := range list {
+		// _indent == 0 only for lists of switch/select case clauses;
+		// in those cases each clause is a new section
+		p.linebreak(p.fset.Position(s.Pos()).Line, 1, ignore, i == 0 || _indent == 0 || multiLine)
+		multiLine = false
+		p.stmt(s, nextIsRBrace && i == len(list)-1, &multiLine)
+	}
+	if _indent > 0 {
+		p.print(unindent)
+	}
+}
+
+// block prints an *ast.BlockStmt; it always spans at least two lines.
+func (p *printer) block(s *ast.BlockStmt, indent int) {
+	p.print(s.Pos(), token.LBRACE)
+	p.stmtList(s.List, indent, true)
+	p.linebreak(p.fset.Position(s.Rbrace).Line, 1, ignore, true)
+	p.print(s.Rbrace, token.RBRACE)
+}
+
+func isTypeName(x ast.Expr) bool {
+	switch t := x.(type) {
+	case *ast.Ident:
+		return true
+	case *ast.SelectorExpr:
+		return isTypeName(t.X)
+	}
+	return false
+}
+
+func stripParens(x ast.Expr) ast.Expr {
+	if px, strip := x.(*ast.ParenExpr); strip {
+		// parentheses must not be stripped if there are any
+		// unparenthesized composite literals starting with
+		// a type name
+		ast.Inspect(px.X, func(node ast.Node) bool {
+			switch x := node.(type) {
+			case *ast.ParenExpr:
+				// parentheses protect enclosed composite literals
+				return false
+			case *ast.CompositeLit:
+				if isTypeName(x.Type) {
+					strip = false // do not strip parentheses
+				}
+				return false
+			}
+			// in all other cases, keep inspecting
+			return true
+		})
+		if strip {
+			return stripParens(px.X)
+		}
+	}
+	return x
+}
+
+func (p *printer) controlClause(isForStmt bool, init ast.Stmt, expr ast.Expr, post ast.Stmt) {
+	p.print(blank)
+	needsBlank := false
+	if init == nil && post == nil {
+		// no semicolons required
+		if expr != nil {
+			p.expr(stripParens(expr), ignoreMultiLine)
+			needsBlank = true
+		}
+	} else {
+		// all semicolons required
+		// (they are not separators, print them explicitly)
+		if init != nil {
+			p.stmt(init, false, ignoreMultiLine)
+		}
+		p.print(token.SEMICOLON, blank)
+		if expr != nil {
+			p.expr(stripParens(expr), ignoreMultiLine)
+			needsBlank = true
+		}
+		if isForStmt {
+			p.print(token.SEMICOLON, blank)
+			needsBlank = false
+			if post != nil {
+				p.stmt(post, false, ignoreMultiLine)
+				needsBlank = true
+			}
+		}
+	}
+	if needsBlank {
+		p.print(blank)
+	}
+}
+
+// Sets multiLine to true if the statements spans multiple lines.
+func (p *printer) stmt(stmt ast.Stmt, nextIsRBrace bool, multiLine *bool) {
+	p.print(stmt.Pos())
+
+	switch s := stmt.(type) {
+	case *ast.BadStmt:
+		p.print("BadStmt")
+
+	case *ast.DeclStmt:
+		p.decl(s.Decl, multiLine)
+
+	case *ast.EmptyStmt:
+		// nothing to do
+
+	case *ast.LabeledStmt:
+		// a "correcting" unindent immediately following a line break
+		// is applied before the line break if there is no comment
+		// between (see writeWhitespace)
+		p.print(unindent)
+		p.expr(s.Label, multiLine)
+		p.print(s.Colon, token.COLON, indent)
+		if e, isEmpty := s.Stmt.(*ast.EmptyStmt); isEmpty {
+			if !nextIsRBrace {
+				p.print(newline, e.Pos(), token.SEMICOLON)
+				break
+			}
+		} else {
+			p.linebreak(p.fset.Position(s.Stmt.Pos()).Line, 1, ignore, true)
+		}
+		p.stmt(s.Stmt, nextIsRBrace, multiLine)
+
+	case *ast.ExprStmt:
+		const depth = 1
+		p.expr0(s.X, depth, multiLine)
+
+	case *ast.SendStmt:
+		const depth = 1
+		p.expr0(s.Chan, depth, multiLine)
+		p.print(blank, s.Arrow, token.ARROW, blank)
+		p.expr0(s.Value, depth, multiLine)
+
+	case *ast.IncDecStmt:
+		const depth = 1
+		p.expr0(s.X, depth+1, multiLine)
+		p.print(s.TokPos, s.Tok)
+
+	case *ast.AssignStmt:
+		var depth = 1
+		if len(s.Lhs) > 1 && len(s.Rhs) > 1 {
+			depth++
+		}
+		p.exprList(s.Pos(), s.Lhs, depth, commaSep, multiLine, s.TokPos)
+		p.print(blank, s.TokPos, s.Tok)
+		p.exprList(s.TokPos, s.Rhs, depth, blankStart|commaSep, multiLine, token.NoPos)
+
+	case *ast.GoStmt:
+		p.print(token.GO, blank)
+		p.expr(s.Call, multiLine)
+
+	case *ast.DeferStmt:
+		p.print(token.DEFER, blank)
+		p.expr(s.Call, multiLine)
+
+	case *ast.ReturnStmt:
+		p.print(token.RETURN)
+		if s.Results != nil {
+			p.exprList(s.Pos(), s.Results, 1, blankStart|commaSep, multiLine, token.NoPos)
+		}
+
+	case *ast.BranchStmt:
+		p.print(s.Tok)
+		if s.Label != nil {
+			p.print(blank)
+			p.expr(s.Label, multiLine)
+		}
+
+	case *ast.BlockStmt:
+		p.block(s, 1)
+		*multiLine = true
+
+	case *ast.IfStmt:
+		p.print(token.IF)
+		p.controlClause(false, s.Init, s.Cond, nil)
+		p.block(s.Body, 1)
+		*multiLine = true
+		if s.Else != nil {
+			p.print(blank, token.ELSE, blank)
+			switch s.Else.(type) {
+			case *ast.BlockStmt, *ast.IfStmt:
+				p.stmt(s.Else, nextIsRBrace, ignoreMultiLine)
+			default:
+				p.print(token.LBRACE, indent, formfeed)
+				p.stmt(s.Else, true, ignoreMultiLine)
+				p.print(unindent, formfeed, token.RBRACE)
+			}
+		}
+
+	case *ast.CaseClause:
+		if s.List != nil {
+			p.print(token.CASE)
+			p.exprList(s.Pos(), s.List, 1, blankStart|commaSep, multiLine, s.Colon)
+		} else {
+			p.print(token.DEFAULT)
+		}
+		p.print(s.Colon, token.COLON)
+		p.stmtList(s.Body, 1, nextIsRBrace)
+
+	case *ast.SwitchStmt:
+		p.print(token.SWITCH)
+		p.controlClause(false, s.Init, s.Tag, nil)
+		p.block(s.Body, 0)
+		*multiLine = true
+
+	case *ast.TypeSwitchStmt:
+		p.print(token.SWITCH)
+		if s.Init != nil {
+			p.print(blank)
+			p.stmt(s.Init, false, ignoreMultiLine)
+			p.print(token.SEMICOLON)
+		}
+		p.print(blank)
+		p.stmt(s.Assign, false, ignoreMultiLine)
+		p.print(blank)
+		p.block(s.Body, 0)
+		*multiLine = true
+
+	case *ast.CommClause:
+		if s.Comm != nil {
+			p.print(token.CASE, blank)
+			p.stmt(s.Comm, false, ignoreMultiLine)
+		} else {
+			p.print(token.DEFAULT)
+		}
+		p.print(s.Colon, token.COLON)
+		p.stmtList(s.Body, 1, nextIsRBrace)
+
+	case *ast.SelectStmt:
+		p.print(token.SELECT, blank)
+		body := s.Body
+		if len(body.List) == 0 && !p.commentBefore(p.fset.Position(body.Rbrace)) {
+			// print empty select statement w/o comments on one line
+			p.print(body.Lbrace, token.LBRACE, body.Rbrace, token.RBRACE)
+		} else {
+			p.block(body, 0)
+			*multiLine = true
+		}
+
+	case *ast.ForStmt:
+		p.print(token.FOR)
+		p.controlClause(true, s.Init, s.Cond, s.Post)
+		p.block(s.Body, 1)
+		*multiLine = true
+
+	case *ast.RangeStmt:
+		p.print(token.FOR, blank)
+		p.expr(s.Key, multiLine)
+		if s.Value != nil {
+			p.print(token.COMMA, blank)
+			p.expr(s.Value, multiLine)
+		}
+		p.print(blank, s.TokPos, s.Tok, blank, token.RANGE, blank)
+		p.expr(stripParens(s.X), multiLine)
+		p.print(blank)
+		p.block(s.Body, 1)
+		*multiLine = true
+
+	default:
+		panic("unreachable")
+	}
+
+	return
+}
+
+// ----------------------------------------------------------------------------
+// Declarations
+
+// The keepTypeColumn function determines if the type column of a series of
+// consecutive const or var declarations must be kept, or if initialization
+// values (V) can be placed in the type column (T) instead. The i'th entry
+// in the result slice is true if the type column in spec[i] must be kept.
+//
+// For example, the declaration:
+//
+//	const (
+//		foobar int = 42 // comment
+//		x          = 7  // comment
+//		foo
+//              bar = 991
+//	)
+//
+// leads to the type/values matrix below. A run of value columns (V) can
+// be moved into the type column if there is no type for any of the values
+// in that column (we only move entire columns so that they align properly).
+//
+//	matrix        formatted     result
+//                    matrix
+//	T  V    ->    T  V     ->   true      there is a T and so the type
+//	-  V          -  V          true      column must be kept
+//	-  -          -  -          false
+//	-  V          V  -          false     V is moved into T column
+//
+func keepTypeColumn(specs []ast.Spec) []bool {
+	m := make([]bool, len(specs))
+
+	populate := func(i, j int, keepType bool) {
+		if keepType {
+			for ; i < j; i++ {
+				m[i] = true
+			}
+		}
+	}
+
+	i0 := -1 // if i0 >= 0 we are in a run and i0 is the start of the run
+	var keepType bool
+	for i, s := range specs {
+		t := s.(*ast.ValueSpec)
+		if t.Values != nil {
+			if i0 < 0 {
+				// start of a run of ValueSpecs with non-nil Values
+				i0 = i
+				keepType = false
+			}
+		} else {
+			if i0 >= 0 {
+				// end of a run
+				populate(i0, i, keepType)
+				i0 = -1
+			}
+		}
+		if t.Type != nil {
+			keepType = true
+		}
+	}
+	if i0 >= 0 {
+		// end of a run
+		populate(i0, len(specs), keepType)
+	}
+
+	return m
+}
+
+func (p *printer) valueSpec(s *ast.ValueSpec, keepType, doIndent bool, multiLine *bool) {
+	p.setComment(s.Doc)
+	p.identList(s.Names, doIndent, multiLine) // always present
+	extraTabs := 3
+	if s.Type != nil || keepType {
+		p.print(vtab)
+		extraTabs--
+	}
+	if s.Type != nil {
+		p.expr(s.Type, multiLine)
+	}
+	if s.Values != nil {
+		p.print(vtab, token.ASSIGN)
+		p.exprList(token.NoPos, s.Values, 1, blankStart|commaSep, multiLine, token.NoPos)
+		extraTabs--
+	}
+	if s.Comment != nil {
+		for ; extraTabs > 0; extraTabs-- {
+			p.print(vtab)
+		}
+		p.setComment(s.Comment)
+	}
+}
+
+// The parameter n is the number of specs in the group. If doIndent is set,
+// multi-line identifier lists in the spec are indented when the first
+// linebreak is encountered.
+// Sets multiLine to true if the spec spans multiple lines.
+//
+func (p *printer) spec(spec ast.Spec, n int, doIndent bool, multiLine *bool) {
+	switch s := spec.(type) {
+	case *ast.ImportSpec:
+		p.setComment(s.Doc)
+		if s.Name != nil {
+			p.expr(s.Name, multiLine)
+			p.print(blank)
+		}
+		p.expr(s.Path, multiLine)
+		p.setComment(s.Comment)
+
+	case *ast.ValueSpec:
+		if n != 1 {
+			p.internalError("expected n = 1; got", n)
+		}
+		p.setComment(s.Doc)
+		p.identList(s.Names, doIndent, multiLine) // always present
+		if s.Type != nil {
+			p.print(blank)
+			p.expr(s.Type, multiLine)
+		}
+		if s.Values != nil {
+			p.print(blank, token.ASSIGN)
+			p.exprList(token.NoPos, s.Values, 1, blankStart|commaSep, multiLine, token.NoPos)
+		}
+		p.setComment(s.Comment)
+
+	case *ast.TypeSpec:
+		p.setComment(s.Doc)
+		p.expr(s.Name, multiLine)
+		if n == 1 {
+			p.print(blank)
+		} else {
+			p.print(vtab)
+		}
+		p.expr(s.Type, multiLine)
+		p.setComment(s.Comment)
+
+	default:
+		panic("unreachable")
+	}
+}
+
+// Sets multiLine to true if the declaration spans multiple lines.
+func (p *printer) genDecl(d *ast.GenDecl, multiLine *bool) {
+	p.setComment(d.Doc)
+	p.print(d.Pos(), d.Tok, blank)
+
+	if d.Lparen.IsValid() {
+		// group of parenthesized declarations
+		p.print(d.Lparen, token.LPAREN)
+		if n := len(d.Specs); n > 0 {
+			p.print(indent, formfeed)
+			if n > 1 && (d.Tok == token.CONST || d.Tok == token.VAR) {
+				// two or more grouped const/var declarations:
+				// determine if the type column must be kept
+				keepType := keepTypeColumn(d.Specs)
+				var ml bool
+				for i, s := range d.Specs {
+					if i > 0 {
+						p.linebreak(p.fset.Position(s.Pos()).Line, 1, ignore, ml)
+					}
+					ml = false
+					p.valueSpec(s.(*ast.ValueSpec), keepType[i], false, &ml)
+				}
+			} else {
+				var ml bool
+				for i, s := range d.Specs {
+					if i > 0 {
+						p.linebreak(p.fset.Position(s.Pos()).Line, 1, ignore, ml)
+					}
+					ml = false
+					p.spec(s, n, false, &ml)
+				}
+			}
+			p.print(unindent, formfeed)
+			*multiLine = true
+		}
+		p.print(d.Rparen, token.RPAREN)
+
+	} else {
+		// single declaration
+		p.spec(d.Specs[0], 1, true, multiLine)
+	}
+}
+
+// nodeSize determines the size of n in chars after formatting.
+// The result is <= maxSize if the node fits on one line with at
+// most maxSize chars and the formatted output doesn't contain
+// any control chars. Otherwise, the result is > maxSize.
+//
+func (p *printer) nodeSize(n ast.Node, maxSize int) (size int) {
+	// nodeSize invokes the printer, which may invoke nodeSize
+	// recursively. For deep composite literal nests, this can
+	// lead to an exponential algorithm. Remember previous
+	// results to prune the recursion (was issue 1628).
+	if size, found := p.nodeSizes[n]; found {
+		return size
+	}
+
+	size = maxSize + 1 // assume n doesn't fit
+	p.nodeSizes[n] = size
+
+	// nodeSize computation must be independent of particular
+	// style so that we always get the same decision; print
+	// in RawFormat
+	cfg := Config{Mode: RawFormat}
+	var buf bytes.Buffer
+	if _, err := cfg.fprint(&buf, p.fset, n, p.nodeSizes); err != nil {
+		return
+	}
+	if buf.Len() <= maxSize {
+		for _, ch := range buf.Bytes() {
+			if ch < ' ' {
+				return
+			}
+		}
+		size = buf.Len() // n fits
+		p.nodeSizes[n] = size
+	}
+	return
+}
+
+func (p *printer) isOneLineFunc(b *ast.BlockStmt, headerSize int) bool {
+	pos1 := b.Pos()
+	pos2 := b.Rbrace
+	if pos1.IsValid() && pos2.IsValid() && p.fset.Position(pos1).Line != p.fset.Position(pos2).Line {
+		// opening and closing brace are on different lines - don't make it a one-liner
+		return false
+	}
+	if len(b.List) > 5 || p.commentBefore(p.fset.Position(pos2)) {
+		// too many statements or there is a comment inside - don't make it a one-liner
+		return false
+	}
+	// otherwise, estimate body size
+	const maxSize = 100
+	bodySize := 0
+	for i, s := range b.List {
+		if i > 0 {
+			bodySize += 2 // space for a semicolon and blank
+		}
+		bodySize += p.nodeSize(s, maxSize)
+	}
+	return headerSize+bodySize <= maxSize
+}
+
+// Sets multiLine to true if the function body spans multiple lines.
+func (p *printer) funcBody(b *ast.BlockStmt, headerSize int, isLit bool, multiLine *bool) {
+	if b == nil {
+		return
+	}
+
+	if p.isOneLineFunc(b, headerSize) {
+		sep := vtab
+		if isLit {
+			sep = blank
+		}
+		p.print(sep, b.Lbrace, token.LBRACE)
+		if len(b.List) > 0 {
+			p.print(blank)
+			for i, s := range b.List {
+				if i > 0 {
+					p.print(token.SEMICOLON, blank)
+				}
+				p.stmt(s, i == len(b.List)-1, ignoreMultiLine)
+			}
+			p.print(blank)
+		}
+		p.print(b.Rbrace, token.RBRACE)
+		return
+	}
+
+	p.print(blank)
+	p.block(b, 1)
+	*multiLine = true
+}
+
+// distance returns the column difference between from and to if both
+// are on the same line; if they are on different lines (or unknown)
+// the result is infinity.
+func (p *printer) distance(from0 token.Pos, to token.Position) int {
+	from := p.fset.Position(from0)
+	if from.IsValid() && to.IsValid() && from.Line == to.Line {
+		return to.Column - from.Column
+	}
+	return infinity
+}
+
+// Sets multiLine to true if the declaration spans multiple lines.
+func (p *printer) funcDecl(d *ast.FuncDecl, multiLine *bool) {
+	p.setComment(d.Doc)
+	p.print(d.Pos(), token.FUNC, blank)
+	if d.Recv != nil {
+		p.parameters(d.Recv, multiLine) // method: print receiver
+		p.print(blank)
+	}
+	p.expr(d.Name, multiLine)
+	p.signature(d.Type.Params, d.Type.Results, multiLine)
+	p.funcBody(d.Body, p.distance(d.Pos(), p.pos), false, multiLine)
+}
+
+// Sets multiLine to true if the declaration spans multiple lines.
+func (p *printer) decl(decl ast.Decl, multiLine *bool) {
+	switch d := decl.(type) {
+	case *ast.BadDecl:
+		p.print(d.Pos(), "BadDecl")
+	case *ast.GenDecl:
+		p.genDecl(d, multiLine)
+	case *ast.FuncDecl:
+		p.funcDecl(d, multiLine)
+	default:
+		panic("unreachable")
+	}
+}
+
+// ----------------------------------------------------------------------------
+// Files
+
+func declToken(decl ast.Decl) (tok token.Token) {
+	tok = token.ILLEGAL
+	switch d := decl.(type) {
+	case *ast.GenDecl:
+		tok = d.Tok
+	case *ast.FuncDecl:
+		tok = token.FUNC
+	}
+	return
+}
+
+func (p *printer) file(src *ast.File) {
+	p.setComment(src.Doc)
+	p.print(src.Pos(), token.PACKAGE, blank)
+	p.expr(src.Name, ignoreMultiLine)
+
+	if len(src.Decls) > 0 {
+		tok := token.ILLEGAL
+		for _, d := range src.Decls {
+			prev := tok
+			tok = declToken(d)
+			// if the declaration token changed (e.g., from CONST to TYPE)
+			// print an empty line between top-level declarations
+			min := 1
+			if prev != tok {
+				min = 2
+			}
+			p.linebreak(p.fset.Position(d.Pos()).Line, min, ignore, false)
+			p.decl(d, ignoreMultiLine)
+		}
+	}
+
+	p.print(newline)
+}
diff --git a/src/pkg/go/printer/performance_test.go b/src/pkg/go/printer/performance_test.go
new file mode 100644
index 000000000..84fb2808e
--- /dev/null
+++ b/src/pkg/go/printer/performance_test.go
@@ -0,0 +1,58 @@
+// Copyright 2009 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// This file implements a simple printer performance benchmark:
+// gotest -bench=BenchmarkPrint 
+
+package printer
+
+import (
+	"bytes"
+	"go/ast"
+	"go/parser"
+	"io"
+	"io/ioutil"
+	"log"
+	"testing"
+)
+
+var testfile *ast.File
+
+func testprint(out io.Writer, file *ast.File) {
+	if _, err := (&Config{TabIndent | UseSpaces, 8}).Fprint(out, fset, file); err != nil {
+		log.Fatalf("print error: %s", err)
+	}
+}
+
+// cannot initialize in init because (printer) Fprint launches goroutines.
+func initialize() {
+	const filename = "testdata/parser.go"
+
+	src, err := ioutil.ReadFile(filename)
+	if err != nil {
+		log.Fatalf("%s", err)
+	}
+
+	file, err := parser.ParseFile(fset, filename, src, parser.ParseComments)
+	if err != nil {
+		log.Fatalf("%s", err)
+	}
+
+	var buf bytes.Buffer
+	testprint(&buf, file)
+	if !bytes.Equal(buf.Bytes(), src) {
+		log.Fatalf("print error: %s not idempotent", filename)
+	}
+
+	testfile = file
+}
+
+func BenchmarkPrint(b *testing.B) {
+	if testfile == nil {
+		initialize()
+	}
+	for i := 0; i < b.N; i++ {
+		testprint(ioutil.Discard, testfile)
+	}
+}
diff --git a/src/pkg/go/printer/printer.go b/src/pkg/go/printer/printer.go
new file mode 100644
index 000000000..871fefa0c
--- /dev/null
+++ b/src/pkg/go/printer/printer.go
@@ -0,0 +1,1012 @@
+// Copyright 2009 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// Package printer implements printing of AST nodes.
+package printer
+
+import (
+	"bytes"
+	"fmt"
+	"go/ast"
+	"go/token"
+	"io"
+	"os"
+	"path/filepath"
+	"runtime"
+	"tabwriter"
+)
+
+const debug = false // enable for debugging
+
+
+type whiteSpace int
+
+const (
+	ignore   = whiteSpace(0)
+	blank    = whiteSpace(' ')
+	vtab     = whiteSpace('\v')
+	newline  = whiteSpace('\n')
+	formfeed = whiteSpace('\f')
+	indent   = whiteSpace('>')
+	unindent = whiteSpace('<')
+)
+
+var (
+	esc       = []byte{tabwriter.Escape}
+	htab      = []byte{'\t'}
+	htabs     = []byte("\t\t\t\t\t\t\t\t")
+	newlines  = []byte("\n\n\n\n\n\n\n\n") // more than the max determined by nlines
+	formfeeds = []byte("\f\f\f\f\f\f\f\f") // more than the max determined by nlines
+)
+
+// Special positions
+var noPos token.Position // use noPos when a position is needed but not known
+var infinity = 1 << 30
+
+// Use ignoreMultiLine if the multiLine information is not important.
+var ignoreMultiLine = new(bool)
+
+// A pmode value represents the current printer mode.
+type pmode int
+
+const (
+	inLiteral pmode = 1 << iota
+	noExtraLinebreak
+)
+
+type printer struct {
+	// Configuration (does not change after initialization)
+	output io.Writer
+	Config
+	fset   *token.FileSet
+	errors chan os.Error
+
+	// Current state
+	written int         // number of bytes written
+	indent  int         // current indentation
+	mode    pmode       // current printer mode
+	lastTok token.Token // the last token printed (token.ILLEGAL if it's whitespace)
+
+	// Reused buffers
+	wsbuf  []whiteSpace // delayed white space
+	litbuf bytes.Buffer // for creation of escaped literals and comments
+
+	// The (possibly estimated) position in the generated output;
+	// in AST space (i.e., pos is set whenever a token position is
+	// known accurately, and updated dependending on what has been
+	// written).
+	pos token.Position
+
+	// The value of pos immediately after the last item has been
+	// written using writeItem.
+	last token.Position
+
+	// The list of all source comments, in order of appearance.
+	comments        []*ast.CommentGroup // may be nil
+	cindex          int                 // current comment index
+	useNodeComments bool                // if not set, ignore lead and line comments of nodes
+
+	// Cache of already computed node sizes.
+	nodeSizes map[ast.Node]int
+}
+
+func (p *printer) init(output io.Writer, cfg *Config, fset *token.FileSet, nodeSizes map[ast.Node]int) {
+	p.output = output
+	p.Config = *cfg
+	p.fset = fset
+	p.errors = make(chan os.Error)
+	p.wsbuf = make([]whiteSpace, 0, 16) // whitespace sequences are short
+	p.nodeSizes = nodeSizes
+}
+
+func (p *printer) internalError(msg ...interface{}) {
+	if debug {
+		fmt.Print(p.pos.String() + ": ")
+		fmt.Println(msg...)
+		panic("go/printer")
+	}
+}
+
+// escape escapes string s by bracketing it with tabwriter.Escape.
+// Escaped strings pass through tabwriter unchanged. (Note that
+// valid Go programs cannot contain tabwriter.Escape bytes since
+// they do not appear in legal UTF-8 sequences).
+//
+func (p *printer) escape(s string) string {
+	p.litbuf.Reset()
+	p.litbuf.WriteByte(tabwriter.Escape)
+	p.litbuf.WriteString(s)
+	p.litbuf.WriteByte(tabwriter.Escape)
+	return p.litbuf.String()
+}
+
+// nlines returns the adjusted number of linebreaks given the desired number
+// of breaks n such that min <= result <= max.
+//
+func (p *printer) nlines(n, min int) int {
+	const max = 2 // max. number of newlines
+	switch {
+	case n < min:
+		return min
+	case n > max:
+		return max
+	}
+	return n
+}
+
+// write0 writes raw (uninterpreted) data to p.output and handles errors.
+// write0 does not indent after newlines, and does not HTML-escape or update p.pos.
+//
+func (p *printer) write0(data []byte) {
+	if len(data) > 0 {
+		n, err := p.output.Write(data)
+		p.written += n
+		if err != nil {
+			p.errors <- err
+			runtime.Goexit()
+		}
+	}
+}
+
+// write interprets data and writes it to p.output. It inserts indentation
+// after a line break unless in a tabwriter escape sequence.
+// It updates p.pos as a side-effect.
+//
+func (p *printer) write(data []byte) {
+	i0 := 0
+	for i, b := range data {
+		switch b {
+		case '\n', '\f':
+			// write segment ending in b
+			p.write0(data[i0 : i+1])
+
+			// update p.pos
+			p.pos.Offset += i + 1 - i0
+			p.pos.Line++
+			p.pos.Column = 1
+
+			if p.mode&inLiteral == 0 {
+				// write indentation
+				// use "hard" htabs - indentation columns
+				// must not be discarded by the tabwriter
+				j := p.indent
+				for ; j > len(htabs); j -= len(htabs) {
+					p.write0(htabs)
+				}
+				p.write0(htabs[0:j])
+
+				// update p.pos
+				p.pos.Offset += p.indent
+				p.pos.Column += p.indent
+			}
+
+			// next segment start
+			i0 = i + 1
+
+		case tabwriter.Escape:
+			p.mode ^= inLiteral
+
+			// ignore escape chars introduced by printer - they are
+			// invisible and must not affect p.pos (was issue #1089)
+			p.pos.Offset--
+			p.pos.Column--
+		}
+	}
+
+	// write remaining segment
+	p.write0(data[i0:])
+
+	// update p.pos
+	d := len(data) - i0
+	p.pos.Offset += d
+	p.pos.Column += d
+}
+
+func (p *printer) writeNewlines(n int, useFF bool) {
+	if n > 0 {
+		n = p.nlines(n, 0)
+		if useFF {
+			p.write(formfeeds[0:n])
+		} else {
+			p.write(newlines[0:n])
+		}
+	}
+}
+
+// writeItem writes data at position pos. data is the text corresponding to
+// a single lexical token, but may also be comment text. pos is the actual
+// (or at least very accurately estimated) position of the data in the original
+// source text. writeItem updates p.last to the position immediately following
+// the data.
+//
+func (p *printer) writeItem(pos token.Position, data string) {
+	if pos.IsValid() {
+		// continue with previous position if we don't have a valid pos
+		if p.last.IsValid() && p.last.Filename != pos.Filename {
+			// the file has changed - reset state
+			// (used when printing merged ASTs of different files
+			// e.g., the result of ast.MergePackageFiles)
+			p.indent = 0
+			p.mode = 0
+			p.wsbuf = p.wsbuf[0:0]
+		}
+		p.pos = pos
+	}
+	if debug {
+		// do not update p.pos - use write0
+		_, filename := filepath.Split(pos.Filename)
+		p.write0([]byte(fmt.Sprintf("[%s:%d:%d]", filename, pos.Line, pos.Column)))
+	}
+	p.write([]byte(data))
+	p.last = p.pos
+}
+
+// writeCommentPrefix writes the whitespace before a comment.
+// If there is any pending whitespace, it consumes as much of
+// it as is likely to help position the comment nicely.
+// pos is the comment position, next the position of the item
+// after all pending comments, prev is the previous comment in
+// a group of comments (or nil), and isKeyword indicates if the
+// next item is a keyword.
+//
+func (p *printer) writeCommentPrefix(pos, next token.Position, prev *ast.Comment, isKeyword bool) {
+	if p.written == 0 {
+		// the comment is the first item to be printed - don't write any whitespace
+		return
+	}
+
+	if pos.IsValid() && pos.Filename != p.last.Filename {
+		// comment in a different file - separate with newlines (writeNewlines will limit the number)
+		p.writeNewlines(10, true)
+		return
+	}
+
+	if pos.Line == p.last.Line && (prev == nil || prev.Text[1] != '/') {
+		// comment on the same line as last item:
+		// separate with at least one separator
+		hasSep := false
+		if prev == nil {
+			// first comment of a comment group
+			j := 0
+			for i, ch := range p.wsbuf {
+				switch ch {
+				case blank:
+					// ignore any blanks before a comment
+					p.wsbuf[i] = ignore
+					continue
+				case vtab:
+					// respect existing tabs - important
+					// for proper formatting of commented structs
+					hasSep = true
+					continue
+				case indent:
+					// apply pending indentation
+					continue
+				}
+				j = i
+				break
+			}
+			p.writeWhitespace(j)
+		}
+		// make sure there is at least one separator
+		if !hasSep {
+			if pos.Line == next.Line {
+				// next item is on the same line as the comment
+				// (which must be a /*-style comment): separate
+				// with a blank instead of a tab
+				p.write([]byte{' '})
+			} else {
+				p.write(htab)
+			}
+		}
+
+	} else {
+		// comment on a different line:
+		// separate with at least one line break
+		if prev == nil {
+			// first comment of a comment group
+			j := 0
+			for i, ch := range p.wsbuf {
+				switch ch {
+				case blank, vtab:
+					// ignore any horizontal whitespace before line breaks
+					p.wsbuf[i] = ignore
+					continue
+				case indent:
+					// apply pending indentation
+					continue
+				case unindent:
+					// if the next token is a keyword, apply the outdent
+					// if it appears that the comment is aligned with the
+					// keyword; otherwise assume the outdent is part of a
+					// closing block and stop (this scenario appears with
+					// comments before a case label where the comments
+					// apply to the next case instead of the current one)
+					if isKeyword && pos.Column == next.Column {
+						continue
+					}
+				case newline, formfeed:
+					// TODO(gri): may want to keep formfeed info in some cases
+					p.wsbuf[i] = ignore
+				}
+				j = i
+				break
+			}
+			p.writeWhitespace(j)
+		}
+		// use formfeeds to break columns before a comment;
+		// this is analogous to using formfeeds to separate
+		// individual lines of /*-style comments - but make
+		// sure there is at least one line break if the previous
+		// comment was a line comment
+		n := pos.Line - p.last.Line // if !pos.IsValid(), pos.Line == 0, and n will be 0
+		if n <= 0 && prev != nil && prev.Text[1] == '/' {
+			n = 1
+		}
+		p.writeNewlines(n, true)
+	}
+}
+
+// TODO(gri): It should be possible to convert the code below from using
+//            []byte to string and in the process eliminate some conversions.
+
+// Split comment text into lines
+func split(text []byte) [][]byte {
+	// count lines (comment text never ends in a newline)
+	n := 1
+	for _, c := range text {
+		if c == '\n' {
+			n++
+		}
+	}
+
+	// split
+	lines := make([][]byte, n)
+	n = 0
+	i := 0
+	for j, c := range text {
+		if c == '\n' {
+			lines[n] = text[i:j] // exclude newline
+			i = j + 1            // discard newline
+			n++
+		}
+	}
+	lines[n] = text[i:]
+
+	return lines
+}
+
+func isBlank(s []byte) bool {
+	for _, b := range s {
+		if b > ' ' {
+			return false
+		}
+	}
+	return true
+}
+
+func commonPrefix(a, b []byte) []byte {
+	i := 0
+	for i < len(a) && i < len(b) && a[i] == b[i] && (a[i] <= ' ' || a[i] == '*') {
+		i++
+	}
+	return a[0:i]
+}
+
+func stripCommonPrefix(lines [][]byte) {
+	if len(lines) < 2 {
+		return // at most one line - nothing to do
+	}
+	// len(lines) >= 2
+
+	// The heuristic in this function tries to handle a few
+	// common patterns of /*-style comments: Comments where
+	// the opening /* and closing */ are aligned and the
+	// rest of the comment text is aligned and indented with
+	// blanks or tabs, cases with a vertical "line of stars"
+	// on the left, and cases where the closing */ is on the
+	// same line as the last comment text.
+
+	// Compute maximum common white prefix of all but the first,
+	// last, and blank lines, and replace blank lines with empty
+	// lines (the first line starts with /* and has no prefix).
+	// In case of two-line comments, consider the last line for
+	// the prefix computation since otherwise the prefix would
+	// be empty.
+	//
+	// Note that the first and last line are never empty (they
+	// contain the opening /* and closing */ respectively) and
+	// thus they can be ignored by the blank line check.
+	var prefix []byte
+	if len(lines) > 2 {
+		for i, line := range lines[1 : len(lines)-1] {
+			switch {
+			case isBlank(line):
+				lines[1+i] = nil // range starts at line 1
+			case prefix == nil:
+				prefix = commonPrefix(line, line)
+			default:
+				prefix = commonPrefix(prefix, line)
+			}
+		}
+	} else { // len(lines) == 2
+		line := lines[1]
+		prefix = commonPrefix(line, line)
+	}
+
+	/*
+	 * Check for vertical "line of stars" and correct prefix accordingly.
+	 */
+	lineOfStars := false
+	if i := bytes.Index(prefix, []byte{'*'}); i >= 0 {
+		// Line of stars present.
+		if i > 0 && prefix[i-1] == ' ' {
+			i-- // remove trailing blank from prefix so stars remain aligned
+		}
+		prefix = prefix[0:i]
+		lineOfStars = true
+	} else {
+		// No line of stars present.
+		// Determine the white space on the first line after the /*
+		// and before the beginning of the comment text, assume two
+		// blanks instead of the /* unless the first character after
+		// the /* is a tab. If the first comment line is empty but
+		// for the opening /*, assume up to 3 blanks or a tab. This
+		// whitespace may be found as suffix in the common prefix.
+		first := lines[0]
+		if isBlank(first[2:]) {
+			// no comment text on the first line:
+			// reduce prefix by up to 3 blanks or a tab
+			// if present - this keeps comment text indented
+			// relative to the /* and */'s if it was indented
+			// in the first place
+			i := len(prefix)
+			for n := 0; n < 3 && i > 0 && prefix[i-1] == ' '; n++ {
+				i--
+			}
+			if i == len(prefix) && i > 0 && prefix[i-1] == '\t' {
+				i--
+			}
+			prefix = prefix[0:i]
+		} else {
+			// comment text on the first line
+			suffix := make([]byte, len(first))
+			n := 2 // start after opening /*
+			for n < len(first) && first[n] <= ' ' {
+				suffix[n] = first[n]
+				n++
+			}
+			if n > 2 && suffix[2] == '\t' {
+				// assume the '\t' compensates for the /*
+				suffix = suffix[2:n]
+			} else {
+				// otherwise assume two blanks
+				suffix[0], suffix[1] = ' ', ' '
+				suffix = suffix[0:n]
+			}
+			// Shorten the computed common prefix by the length of
+			// suffix, if it is found as suffix of the prefix.
+			if bytes.HasSuffix(prefix, suffix) {
+				prefix = prefix[0 : len(prefix)-len(suffix)]
+			}
+		}
+	}
+
+	// Handle last line: If it only contains a closing */, align it
+	// with the opening /*, otherwise align the text with the other
+	// lines.
+	last := lines[len(lines)-1]
+	closing := []byte("*/")
+	i := bytes.Index(last, closing)
+	if isBlank(last[0:i]) {
+		// last line only contains closing */
+		var sep []byte
+		if lineOfStars {
+			// insert an aligning blank
+			sep = []byte{' '}
+		}
+		lines[len(lines)-1] = bytes.Join([][]byte{prefix, closing}, sep)
+	} else {
+		// last line contains more comment text - assume
+		// it is aligned like the other lines
+		prefix = commonPrefix(prefix, last)
+	}
+
+	// Remove the common prefix from all but the first and empty lines.
+	for i, line := range lines[1:] {
+		if len(line) != 0 {
+			lines[1+i] = line[len(prefix):] // range starts at line 1
+		}
+	}
+}
+
+func (p *printer) writeComment(comment *ast.Comment) {
+	text := comment.Text
+
+	// shortcut common case of //-style comments
+	if text[1] == '/' {
+		p.writeItem(p.fset.Position(comment.Pos()), p.escape(text))
+		return
+	}
+
+	// for /*-style comments, print line by line and let the
+	// write function take care of the proper indentation
+	lines := split([]byte(text))
+	stripCommonPrefix(lines)
+
+	// write comment lines, separated by formfeed,
+	// without a line break after the last line
+	linebreak := formfeeds[0:1]
+	pos := p.fset.Position(comment.Pos())
+	for i, line := range lines {
+		if i > 0 {
+			p.write(linebreak)
+			pos = p.pos
+		}
+		if len(line) > 0 {
+			p.writeItem(pos, p.escape(string(line)))
+		}
+	}
+}
+
+// writeCommentSuffix writes a line break after a comment if indicated
+// and processes any leftover indentation information. If a line break
+// is needed, the kind of break (newline vs formfeed) depends on the
+// pending whitespace. writeCommentSuffix returns true if a pending
+// formfeed was dropped from the whitespace buffer.
+//
+func (p *printer) writeCommentSuffix(needsLinebreak bool) (droppedFF bool) {
+	for i, ch := range p.wsbuf {
+		switch ch {
+		case blank, vtab:
+			// ignore trailing whitespace
+			p.wsbuf[i] = ignore
+		case indent, unindent:
+			// don't lose indentation information
+		case newline, formfeed:
+			// if we need a line break, keep exactly one
+			// but remember if we dropped any formfeeds
+			if needsLinebreak {
+				needsLinebreak = false
+			} else {
+				if ch == formfeed {
+					droppedFF = true
+				}
+				p.wsbuf[i] = ignore
+			}
+		}
+	}
+	p.writeWhitespace(len(p.wsbuf))
+
+	// make sure we have a line break
+	if needsLinebreak {
+		p.write([]byte{'\n'})
+	}
+
+	return
+}
+
+// intersperseComments consumes all comments that appear before the next token
+// tok and prints it together with the buffered whitespace (i.e., the whitespace
+// that needs to be written before the next token). A heuristic is used to mix
+// the comments and whitespace. intersperseComments returns true if a pending
+// formfeed was dropped from the whitespace buffer.
+//
+func (p *printer) intersperseComments(next token.Position, tok token.Token) (droppedFF bool) {
+	var last *ast.Comment
+	for ; p.commentBefore(next); p.cindex++ {
+		for _, c := range p.comments[p.cindex].List {
+			p.writeCommentPrefix(p.fset.Position(c.Pos()), next, last, tok.IsKeyword())
+			p.writeComment(c)
+			last = c
+		}
+	}
+
+	if last != nil {
+		if last.Text[1] == '*' && p.fset.Position(last.Pos()).Line == next.Line {
+			// the last comment is a /*-style comment and the next item
+			// follows on the same line: separate with an extra blank
+			p.write([]byte{' '})
+		}
+		// ensure that there is a line break after a //-style comment,
+		// before a closing '}' unless explicitly disabled, or at eof
+		needsLinebreak :=
+			last.Text[1] == '/' ||
+				tok == token.RBRACE && p.mode&noExtraLinebreak == 0 ||
+				tok == token.EOF
+		return p.writeCommentSuffix(needsLinebreak)
+	}
+
+	// no comment was written - we should never reach here since
+	// intersperseComments should not be called in that case
+	p.internalError("intersperseComments called without pending comments")
+	return false
+}
+
+// whiteWhitespace writes the first n whitespace entries.
+func (p *printer) writeWhitespace(n int) {
+	// write entries
+	var data [1]byte
+	for i := 0; i < n; i++ {
+		switch ch := p.wsbuf[i]; ch {
+		case ignore:
+			// ignore!
+		case indent:
+			p.indent++
+		case unindent:
+			p.indent--
+			if p.indent < 0 {
+				p.internalError("negative indentation:", p.indent)
+				p.indent = 0
+			}
+		case newline, formfeed:
+			// A line break immediately followed by a "correcting"
+			// unindent is swapped with the unindent - this permits
+			// proper label positioning. If a comment is between
+			// the line break and the label, the unindent is not
+			// part of the comment whitespace prefix and the comment
+			// will be positioned correctly indented.
+			if i+1 < n && p.wsbuf[i+1] == unindent {
+				// Use a formfeed to terminate the current section.
+				// Otherwise, a long label name on the next line leading
+				// to a wide column may increase the indentation column
+				// of lines before the label; effectively leading to wrong
+				// indentation.
+				p.wsbuf[i], p.wsbuf[i+1] = unindent, formfeed
+				i-- // do it again
+				continue
+			}
+			fallthrough
+		default:
+			data[0] = byte(ch)
+			p.write(data[0:])
+		}
+	}
+
+	// shift remaining entries down
+	i := 0
+	for ; n < len(p.wsbuf); n++ {
+		p.wsbuf[i] = p.wsbuf[n]
+		i++
+	}
+	p.wsbuf = p.wsbuf[0:i]
+}
+
+// ----------------------------------------------------------------------------
+// Printing interface
+
+
+func mayCombine(prev token.Token, next byte) (b bool) {
+	switch prev {
+	case token.INT:
+		b = next == '.' // 1.
+	case token.ADD:
+		b = next == '+' // ++
+	case token.SUB:
+		b = next == '-' // --
+	case token.QUO:
+		b = next == '*' // /*
+	case token.LSS:
+		b = next == '-' || next == '<' // <- or <<
+	case token.AND:
+		b = next == '&' || next == '^' // && or &^
+	}
+	return
+}
+
+// print prints a list of "items" (roughly corresponding to syntactic
+// tokens, but also including whitespace and formatting information).
+// It is the only print function that should be called directly from
+// any of the AST printing functions in nodes.go.
+//
+// Whitespace is accumulated until a non-whitespace token appears. Any
+// comments that need to appear before that token are printed first,
+// taking into account the amount and structure of any pending white-
+// space for best comment placement. Then, any leftover whitespace is
+// printed, followed by the actual token.
+//
+func (p *printer) print(args ...interface{}) {
+	for _, f := range args {
+		next := p.pos // estimated position of next item
+		var data string
+		var tok token.Token
+
+		switch x := f.(type) {
+		case pmode:
+			// toggle printer mode
+			p.mode ^= x
+		case whiteSpace:
+			if x == ignore {
+				// don't add ignore's to the buffer; they
+				// may screw up "correcting" unindents (see
+				// LabeledStmt)
+				break
+			}
+			i := len(p.wsbuf)
+			if i == cap(p.wsbuf) {
+				// Whitespace sequences are very short so this should
+				// never happen. Handle gracefully (but possibly with
+				// bad comment placement) if it does happen.
+				p.writeWhitespace(i)
+				i = 0
+			}
+			p.wsbuf = p.wsbuf[0 : i+1]
+			p.wsbuf[i] = x
+		case *ast.Ident:
+			data = x.Name
+			tok = token.IDENT
+		case *ast.BasicLit:
+			data = p.escape(x.Value)
+			tok = x.Kind
+		case token.Token:
+			s := x.String()
+			if mayCombine(p.lastTok, s[0]) {
+				// the previous and the current token must be
+				// separated by a blank otherwise they combine
+				// into a different incorrect token sequence
+				// (except for token.INT followed by a '.' this
+				// should never happen because it is taken care
+				// of via binary expression formatting)
+				if len(p.wsbuf) != 0 {
+					p.internalError("whitespace buffer not empty")
+				}
+				p.wsbuf = p.wsbuf[0:1]
+				p.wsbuf[0] = ' '
+			}
+			data = s
+			tok = x
+		case token.Pos:
+			if x.IsValid() {
+				next = p.fset.Position(x) // accurate position of next item
+			}
+			tok = p.lastTok
+		default:
+			fmt.Fprintf(os.Stderr, "print: unsupported argument type %T\n", f)
+			panic("go/printer type")
+		}
+		p.lastTok = tok
+		p.pos = next
+
+		if data != "" {
+			droppedFF := p.flush(next, tok)
+
+			// intersperse extra newlines if present in the source
+			// (don't do this in flush as it will cause extra newlines
+			// at the end of a file) - use formfeeds if we dropped one
+			// before
+			p.writeNewlines(next.Line-p.pos.Line, droppedFF)
+
+			p.writeItem(next, data)
+		}
+	}
+}
+
+// commentBefore returns true iff the current comment occurs
+// before the next position in the source code.
+//
+func (p *printer) commentBefore(next token.Position) bool {
+	return p.cindex < len(p.comments) && p.fset.Position(p.comments[p.cindex].List[0].Pos()).Offset < next.Offset
+}
+
+// Flush prints any pending comments and whitespace occurring
+// textually before the position of the next token tok. Flush
+// returns true if a pending formfeed character was dropped
+// from the whitespace buffer as a result of interspersing
+// comments.
+//
+func (p *printer) flush(next token.Position, tok token.Token) (droppedFF bool) {
+	if p.commentBefore(next) {
+		// if there are comments before the next item, intersperse them
+		droppedFF = p.intersperseComments(next, tok)
+	} else {
+		// otherwise, write any leftover whitespace
+		p.writeWhitespace(len(p.wsbuf))
+	}
+	return
+}
+
+// ----------------------------------------------------------------------------
+// Trimmer
+
+// A trimmer is an io.Writer filter for stripping tabwriter.Escape
+// characters, trailing blanks and tabs, and for converting formfeed
+// and vtab characters into newlines and htabs (in case no tabwriter
+// is used). Text bracketed by tabwriter.Escape characters is passed
+// through unchanged.
+//
+type trimmer struct {
+	output io.Writer
+	state  int
+	space  bytes.Buffer
+}
+
+// trimmer is implemented as a state machine.
+// It can be in one of the following states:
+const (
+	inSpace  = iota // inside space
+	inEscape        // inside text bracketed by tabwriter.Escapes
+	inText          // inside text
+)
+
+// Design note: It is tempting to eliminate extra blanks occurring in
+//              whitespace in this function as it could simplify some
+//              of the blanks logic in the node printing functions.
+//              However, this would mess up any formatting done by
+//              the tabwriter.
+
+func (p *trimmer) Write(data []byte) (n int, err os.Error) {
+	// invariants:
+	// p.state == inSpace:
+	//	p.space is unwritten
+	// p.state == inEscape, inText:
+	//	data[m:n] is unwritten
+	m := 0
+	var b byte
+	for n, b = range data {
+		if b == '\v' {
+			b = '\t' // convert to htab
+		}
+		switch p.state {
+		case inSpace:
+			switch b {
+			case '\t', ' ':
+				p.space.WriteByte(b) // WriteByte returns no errors
+			case '\n', '\f':
+				p.space.Reset()                        // discard trailing space
+				_, err = p.output.Write(newlines[0:1]) // write newline
+			case tabwriter.Escape:
+				_, err = p.output.Write(p.space.Bytes())
+				p.state = inEscape
+				m = n + 1 // +1: skip tabwriter.Escape
+			default:
+				_, err = p.output.Write(p.space.Bytes())
+				p.state = inText
+				m = n
+			}
+		case inEscape:
+			if b == tabwriter.Escape {
+				_, err = p.output.Write(data[m:n])
+				p.state = inSpace
+				p.space.Reset()
+			}
+		case inText:
+			switch b {
+			case '\t', ' ':
+				_, err = p.output.Write(data[m:n])
+				p.state = inSpace
+				p.space.Reset()
+				p.space.WriteByte(b) // WriteByte returns no errors
+			case '\n', '\f':
+				_, err = p.output.Write(data[m:n])
+				p.state = inSpace
+				p.space.Reset()
+				_, err = p.output.Write(newlines[0:1]) // write newline
+			case tabwriter.Escape:
+				_, err = p.output.Write(data[m:n])
+				p.state = inEscape
+				m = n + 1 // +1: skip tabwriter.Escape
+			}
+		default:
+			panic("unreachable")
+		}
+		if err != nil {
+			return
+		}
+	}
+	n = len(data)
+
+	switch p.state {
+	case inEscape, inText:
+		_, err = p.output.Write(data[m:n])
+		p.state = inSpace
+		p.space.Reset()
+	}
+
+	return
+}
+
+// ----------------------------------------------------------------------------
+// Public interface
+
+// General printing is controlled with these Config.Mode flags.
+const (
+	RawFormat uint = 1 << iota // do not use a tabwriter; if set, UseSpaces is ignored
+	TabIndent                  // use tabs for indentation independent of UseSpaces
+	UseSpaces                  // use spaces instead of tabs for alignment
+)
+
+// A Config node controls the output of Fprint.
+type Config struct {
+	Mode     uint // default: 0
+	Tabwidth int  // default: 8
+}
+
+// fprint implements Fprint and takes a nodesSizes map for setting up the printer state.
+func (cfg *Config) fprint(output io.Writer, fset *token.FileSet, node interface{}, nodeSizes map[ast.Node]int) (int, os.Error) {
+	// redirect output through a trimmer to eliminate trailing whitespace
+	// (Input to a tabwriter must be untrimmed since trailing tabs provide
+	// formatting information. The tabwriter could provide trimming
+	// functionality but no tabwriter is used when RawFormat is set.)
+	output = &trimmer{output: output}
+
+	// setup tabwriter if needed and redirect output
+	var tw *tabwriter.Writer
+	if cfg.Mode&RawFormat == 0 {
+		minwidth := cfg.Tabwidth
+
+		padchar := byte('\t')
+		if cfg.Mode&UseSpaces != 0 {
+			padchar = ' '
+		}
+
+		twmode := tabwriter.DiscardEmptyColumns
+		if cfg.Mode&TabIndent != 0 {
+			minwidth = 0
+			twmode |= tabwriter.TabIndent
+		}
+
+		tw = tabwriter.NewWriter(output, minwidth, cfg.Tabwidth, 1, padchar, twmode)
+		output = tw
+	}
+
+	// setup printer and print node
+	var p printer
+	p.init(output, cfg, fset, nodeSizes)
+	go func() {
+		switch n := node.(type) {
+		case ast.Expr:
+			p.useNodeComments = true
+			p.expr(n, ignoreMultiLine)
+		case ast.Stmt:
+			p.useNodeComments = true
+			// A labeled statement will un-indent to position the
+			// label. Set indent to 1 so we don't get indent "underflow".
+			if _, labeledStmt := n.(*ast.LabeledStmt); labeledStmt {
+				p.indent = 1
+			}
+			p.stmt(n, false, ignoreMultiLine)
+		case ast.Decl:
+			p.useNodeComments = true
+			p.decl(n, ignoreMultiLine)
+		case ast.Spec:
+			p.useNodeComments = true
+			p.spec(n, 1, false, ignoreMultiLine)
+		case *ast.File:
+			p.comments = n.Comments
+			p.useNodeComments = n.Comments == nil
+			p.file(n)
+		default:
+			p.errors <- fmt.Errorf("printer.Fprint: unsupported node type %T", n)
+			runtime.Goexit()
+		}
+		p.flush(token.Position{Offset: infinity, Line: infinity}, token.EOF)
+		p.errors <- nil // no errors
+	}()
+	err := <-p.errors // wait for completion of goroutine
+
+	// flush tabwriter, if any
+	if tw != nil {
+		tw.Flush() // ignore errors
+	}
+
+	return p.written, err
+}
+
+// Fprint "pretty-prints" an AST node to output and returns the number
+// of bytes written and an error (if any) for a given configuration cfg.
+// Position information is interpreted relative to the file set fset.
+// The node type must be *ast.File, or assignment-compatible to ast.Expr,
+// ast.Decl, ast.Spec, or ast.Stmt.
+//
+func (cfg *Config) Fprint(output io.Writer, fset *token.FileSet, node interface{}) (int, os.Error) {
+	return cfg.fprint(output, fset, node, make(map[ast.Node]int))
+}
+
+// Fprint "pretty-prints" an AST node to output.
+// It calls Config.Fprint with default settings.
+//
+func Fprint(output io.Writer, fset *token.FileSet, node interface{}) os.Error {
+	_, err := (&Config{Tabwidth: 8}).Fprint(output, fset, node) // don't care about number of bytes written
+	return err
+}
diff --git a/src/pkg/go/printer/printer_test.go b/src/pkg/go/printer/printer_test.go
new file mode 100644
index 000000000..ff2d906b5
--- /dev/null
+++ b/src/pkg/go/printer/printer_test.go
@@ -0,0 +1,194 @@
+// Copyright 2009 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package printer
+
+import (
+	"bytes"
+	"flag"
+	"io/ioutil"
+	"go/ast"
+	"go/parser"
+	"go/token"
+	"path/filepath"
+	"testing"
+	"time"
+)
+
+const (
+	dataDir  = "testdata"
+	tabwidth = 8
+)
+
+var update = flag.Bool("update", false, "update golden files")
+
+var fset = token.NewFileSet()
+
+func lineString(text []byte, i int) string {
+	i0 := i
+	for i < len(text) && text[i] != '\n' {
+		i++
+	}
+	return string(text[i0:i])
+}
+
+type checkMode uint
+
+const (
+	export checkMode = 1 << iota
+	rawFormat
+)
+
+func runcheck(t *testing.T, source, golden string, mode checkMode) {
+	// parse source
+	prog, err := parser.ParseFile(fset, source, nil, parser.ParseComments)
+	if err != nil {
+		t.Error(err)
+		return
+	}
+
+	// filter exports if necessary
+	if mode&export != 0 {
+		ast.FileExports(prog) // ignore result
+		prog.Comments = nil   // don't print comments that are not in AST
+	}
+
+	// determine printer configuration
+	cfg := Config{Tabwidth: tabwidth}
+	if mode&rawFormat != 0 {
+		cfg.Mode |= RawFormat
+	}
+
+	// format source
+	var buf bytes.Buffer
+	if _, err := cfg.Fprint(&buf, fset, prog); err != nil {
+		t.Error(err)
+	}
+	res := buf.Bytes()
+
+	// update golden files if necessary
+	if *update {
+		if err := ioutil.WriteFile(golden, res, 0644); err != nil {
+			t.Error(err)
+		}
+		return
+	}
+
+	// get golden
+	gld, err := ioutil.ReadFile(golden)
+	if err != nil {
+		t.Error(err)
+		return
+	}
+
+	// compare lengths
+	if len(res) != len(gld) {
+		t.Errorf("len = %d, expected %d (= len(%s))", len(res), len(gld), golden)
+	}
+
+	// compare contents
+	for i, line, offs := 0, 1, 0; i < len(res) && i < len(gld); i++ {
+		ch := res[i]
+		if ch != gld[i] {
+			t.Errorf("%s:%d:%d: %s", source, line, i-offs+1, lineString(res, offs))
+			t.Errorf("%s:%d:%d: %s", golden, line, i-offs+1, lineString(gld, offs))
+			t.Error()
+			return
+		}
+		if ch == '\n' {
+			line++
+			offs = i + 1
+		}
+	}
+}
+
+func check(t *testing.T, source, golden string, mode checkMode) {
+	// start a timer to produce a time-out signal
+	tc := make(chan int)
+	go func() {
+		time.Sleep(10e9) // plenty of a safety margin, even for very slow machines
+		tc <- 0
+	}()
+
+	// run the test
+	cc := make(chan int)
+	go func() {
+		runcheck(t, source, golden, mode)
+		cc <- 0
+	}()
+
+	// wait for the first finisher
+	select {
+	case <-tc:
+		// test running past time out
+		t.Errorf("%s: running too slowly", source)
+	case <-cc:
+		// test finished within alloted time margin
+	}
+}
+
+type entry struct {
+	source, golden string
+	mode           checkMode
+}
+
+// Use gotest -update to create/update the respective golden files.
+var data = []entry{
+	{"empty.input", "empty.golden", 0},
+	{"comments.input", "comments.golden", 0},
+	{"comments.input", "comments.x", export},
+	{"linebreaks.input", "linebreaks.golden", 0},
+	{"expressions.input", "expressions.golden", 0},
+	{"expressions.input", "expressions.raw", rawFormat},
+	{"declarations.input", "declarations.golden", 0},
+	{"statements.input", "statements.golden", 0},
+	{"slow.input", "slow.golden", 0},
+}
+
+func TestFiles(t *testing.T) {
+	for i, e := range data {
+		source := filepath.Join(dataDir, e.source)
+		golden := filepath.Join(dataDir, e.golden)
+		check(t, source, golden, e.mode)
+		// TODO(gri) check that golden is idempotent
+		//check(t, golden, golden, e.mode)
+		if testing.Short() && i >= 3 {
+			break
+		}
+	}
+}
+
+// TestLineComments, using a simple test case, checks that consequtive line
+// comments are properly terminated with a newline even if the AST position
+// information is incorrect.
+//
+func TestLineComments(t *testing.T) {
+	const src = `// comment 1
+	// comment 2
+	// comment 3
+	package main
+	`
+
+	fset := token.NewFileSet()
+	ast1, err1 := parser.ParseFile(fset, "", src, parser.ParseComments)
+	if err1 != nil {
+		panic(err1)
+	}
+
+	var buf bytes.Buffer
+	fset = token.NewFileSet() // use the wrong file set
+	Fprint(&buf, fset, ast1)
+
+	nlines := 0
+	for _, ch := range buf.Bytes() {
+		if ch == '\n' {
+			nlines++
+		}
+	}
+
+	const expected = 3
+	if nlines < expected {
+		t.Errorf("got %d, expected %d\n", nlines, expected)
+	}
+}
diff --git a/src/pkg/go/printer/testdata/comments.golden b/src/pkg/go/printer/testdata/comments.golden
new file mode 100644
index 000000000..7b332252c
--- /dev/null
+++ b/src/pkg/go/printer/testdata/comments.golden
@@ -0,0 +1,473 @@
+// Copyright 2009 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// This is a package for testing comment placement by go/printer.
+//
+package main
+
+import "fmt"	// fmt
+
+const c0 = 0	// zero
+const (
+	c1	= iota	// c1
+	c2		// c2
+)
+
+// Alignment of comments in declarations>
+const (
+	_	T	= iota	// comment
+	_			// comment
+	_			// comment
+	_	= iota + 10
+	_	// comments
+
+	_		= 10	// comment
+	_	T	= 20	// comment
+)
+
+const (
+	_____	= iota	// foo
+	_		// bar
+	_	= 0	// bal
+	_		// bat
+)
+
+const (
+	_	T	= iota	// comment
+	_			// comment
+	_			// comment
+	_	= iota + 10
+	_		// comment
+	_		= 10
+	_		= 20	// comment
+	_	T	= 0	// comment
+)
+
+// The SZ struct; it is empty.
+type SZ struct{}
+
+// The S0 struct; no field is exported.
+type S0 struct {
+	int
+	x, y, z	int	// 3 unexported fields
+}
+
+// The S1 struct; some fields are not exported.
+type S1 struct {
+	S0
+	A, B, C	float	// 3 exported fields
+	D, b, c	int	// 2 unexported fields
+}
+
+// The S2 struct; all fields are exported.
+type S2 struct {
+	S1
+	A, B, C	float	// 3 exported fields
+}
+
+// The IZ interface; it is empty.
+type SZ interface{}
+
+// The I0 interface; no method is exported.
+type I0 interface {
+	f(x int) int	// unexported method
+}
+
+// The I1 interface; some methods are not exported.
+type I1 interface {
+	I0
+	F(x float) float	// exported methods
+	g(x int) int		// unexported method
+}
+
+// The I2 interface; all methods are exported.
+type I2 interface {
+	I0
+	F(x float) float	// exported method
+	G(x float) float	// exported method
+}
+
+// The S3 struct; all comments except for the last one must appear in the export.
+type S3 struct {
+	// lead comment for F1
+	F1	int	// line comment for F1
+	// lead comment for F2
+	F2	int	// line comment for F2
+	f3	int	// f3 is not exported
+}
+
+// This comment group should be separated
+// with a newline from the next comment
+// group.
+
+// This comment should NOT be associated with the next declaration.
+
+var x int	// x
+var ()
+
+// This comment SHOULD be associated with the next declaration.
+func f0() {
+	const pi = 3.14	// pi
+	var s1 struct{}	/* an empty struct */	/* foo */
+	// a struct constructor
+	// --------------------
+	var s2 struct{} = struct{}{}
+	x := pi
+}
+//
+// NO SPACE HERE
+//
+func f1() {
+	f0()
+	/* 1 */
+	// 2
+	/* 3 */
+	/* 4 */
+	f0()
+}
+
+func _() {
+	// this comment should be properly indented
+}
+
+func _(x int) int {
+	if x < 0 {	// the tab printed before this comment's // must not affect the remaining lines
+		return -x	// this statement should be properly indented
+	}
+	if x < 0 {	/* the tab printed before this comment's /* must not affect the remaining lines */
+		return -x	// this statement should be properly indented
+	}
+	return x
+}
+
+func typeswitch(x interface{}) {
+	switch v := x.(type) {
+	case bool, int, float:
+	case string:
+	default:
+	}
+
+	switch x.(type) {
+	}
+
+	switch v0, ok := x.(int); v := x.(type) {
+	}
+
+	switch v0, ok := x.(int); x.(type) {
+	case byte:	// this comment should be on the same line as the keyword
+		// this comment should be normally indented
+		_ = 0
+	case bool, int, float:
+		// this comment should be indented
+	case string:
+	default:
+		// this comment should be indented
+	}
+	// this comment should not be indented
+}
+
+func _() {
+	/* freestanding comment
+	   aligned		line
+	   aligned line
+	*/
+}
+
+func _() {
+	/* freestanding comment
+	   aligned		line
+	   aligned line
+	*/
+}
+
+func _() {
+	/* freestanding comment
+	   aligned		line
+	   aligned line */
+}
+
+func _() {
+	/*	freestanding comment
+		aligned		line
+		aligned line
+	*/
+}
+
+func _() {
+	/*	freestanding comment
+		aligned		line
+		aligned line
+	*/
+}
+
+func _() {
+	/*	freestanding comment
+		aligned		line
+		aligned line */
+}
+
+func _() {
+	/*
+	   freestanding comment
+	   aligned		line
+	   aligned line
+	*/
+}
+
+func _() {
+	/*
+	   freestanding comment
+	   aligned		line
+	   aligned line
+	*/
+}
+
+func _() {
+	/*
+	   freestanding comment
+	   aligned		line
+	   aligned line */
+}
+
+func _() {
+	/*
+		freestanding comment
+		aligned		line
+		aligned line
+	*/
+}
+
+func _() {
+	/*
+		freestanding comment
+		aligned		line
+		aligned line
+	*/
+}
+
+func _() {
+	/*
+		freestanding comment
+		aligned		line
+		aligned line */
+}
+
+func _() {
+	/* freestanding comment
+	   aligned line
+	*/
+}
+
+func _() {
+	/* freestanding comment
+	   aligned line
+	*/
+}
+
+func _() {
+	/* freestanding comment
+	   aligned line */
+}
+
+func _() {
+	/*	freestanding comment
+		aligned line
+	*/
+}
+
+func _() {
+	/*	freestanding comment
+		aligned line
+	*/
+}
+
+func _() {
+	/*	freestanding comment
+		aligned line */
+}
+
+func _() {
+	/*
+	   freestanding comment
+	   aligned line
+	*/
+}
+
+func _() {
+	/*
+	   freestanding comment
+	   aligned line
+	*/
+}
+
+func _() {
+	/*
+	   freestanding comment
+	   aligned line */
+}
+
+func _() {
+	/*
+		freestanding comment
+		aligned line
+	*/
+}
+
+func _() {
+	/*
+		freestanding comment
+		aligned line
+	*/
+}
+
+func _() {
+	/*
+		freestanding comment
+		aligned line */
+}
+
+/*
+ * line
+ * of
+ * stars
+ */
+
+/* another line
+ * of
+ * stars */
+
+/*	and another line
+ *	of
+ *	stars */
+
+/* a line of
+ * stars */
+
+/*	and another line of
+ *	stars */
+
+/* a line of stars
+ */
+
+/*	and another line of
+ */
+
+/* a line of stars
+ */
+
+/*	and another line of
+ */
+
+/*
+aligned in middle
+here
+        not here
+*/
+
+/*
+blank line in middle:
+
+with no leading spaces on blank line.
+*/
+
+/*
+   aligned in middle
+   here
+           not here
+*/
+
+/*
+	blank line in middle:
+
+	with no leading spaces on blank line.
+*/
+
+func _() {
+	/*
+	 * line
+	 * of
+	 * stars
+	 */
+
+	/*
+		aligned in middle
+		here
+			not here
+	*/
+
+	/*
+		blank line in middle:
+
+		with no leading spaces on blank line.
+	*/
+}
+
+// Some interesting interspersed comments
+func _( /* this */ x /* is */ /* an */ int) {
+}
+
+func _( /* no params */ )	{}
+
+func _() {
+	f( /* no args */ )
+}
+
+func ( /* comment1 */ T /* comment2 */ ) _()	{}
+
+func _() { /* one-line functions with comments are formatted as multi-line functions */
+}
+
+func _() {
+	_ = 0
+	/* closing curly brace should be on new line */
+}
+
+func _() {
+	_ = []int{0, 1 /* don't introduce a newline after this comment - was issue 1365 */ }
+}
+
+// Comments immediately adjacent to punctuation (for which the go/printer
+// may only have estimated position information) must remain after the punctuation.
+func _() {
+	_ = T{
+		1,	// comment after comma
+		2,	/* comment after comma */
+		3,	// comment after comma
+	}
+	_ = T{
+		1,	// comment after comma
+		2,	/* comment after comma */
+		3,	// comment after comma
+	}
+	_ = T{
+		/* comment before literal */ 1,
+		2,	/* comment before comma - ok to move after comma */
+		3,	/* comment before comma - ok to move after comma */
+	}
+
+	for i = 0;	// comment after semicolon
+	i < 9;		/* comment after semicolon */
+	i++ {		// comment after opening curly brace
+	}
+
+	// TODO(gri) the last comment in this example should be aligned */
+	for i = 0;	// comment after semicolon
+	i < 9;		/* comment before semicolon - ok to move after semicolon */
+	i++ /* comment before opening curly brace */ {
+	}
+}
+
+// Line comments with tabs
+func _() {
+	var finput *bufio.Reader	// input file
+	var stderr *bufio.Writer
+	var ftable *bufio.Writer	// y.go file
+	var foutput *bufio.Writer	// y.output file
+
+	var oflag string	// -o [y.go]		- y.go file
+	var vflag string	// -v [y.output]	- y.output file
+	var lflag bool		// -l			- disable line directives
+}
+
+/* This comment is the last entry in this file. It must be printed and should be followed by a newline */
diff --git a/src/pkg/go/printer/testdata/comments.input b/src/pkg/go/printer/testdata/comments.input
new file mode 100644
index 000000000..2a9a86b68
--- /dev/null
+++ b/src/pkg/go/printer/testdata/comments.input
@@ -0,0 +1,483 @@
+// Copyright 2009 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// This is a package for testing comment placement by go/printer.
+//
+package main
+
+import "fmt"  // fmt
+
+const c0 = 0  // zero
+const (
+	c1 = iota  // c1
+	c2  // c2
+)
+
+// Alignment of comments in declarations>
+const (
+	_ T = iota  // comment
+	_  // comment
+	_  // comment
+	_ = iota+10
+	_  // comments
+
+	_ = 10  // comment
+	_ T = 20  // comment
+)
+
+const (
+	_____ = iota // foo
+	_ // bar
+	_  = 0    // bal
+	_ // bat
+)
+
+const (
+	_ T = iota // comment
+	_ // comment
+	_ // comment
+	_ = iota + 10
+	_ // comment
+	_ = 10
+	_ = 20 // comment
+	_ T = 0 // comment
+)
+
+// The SZ struct; it is empty.
+type SZ struct {}
+
+// The S0 struct; no field is exported.
+type S0 struct {
+	int
+	x, y, z int  // 3 unexported fields
+}
+
+// The S1 struct; some fields are not exported.
+type S1 struct {
+	S0
+	A, B, C float  // 3 exported fields
+	D, b, c int  // 2 unexported fields
+}
+
+// The S2 struct; all fields are exported.
+type S2 struct {
+	S1
+	A, B, C float  // 3 exported fields
+}
+
+// The IZ interface; it is empty.
+type SZ interface {}
+
+// The I0 interface; no method is exported.
+type I0 interface {
+	f(x int) int  // unexported method
+}
+
+// The I1 interface; some methods are not exported.
+type I1 interface {
+	I0
+	F(x float) float  // exported methods
+	g(x int) int  // unexported method
+}
+
+// The I2 interface; all methods are exported.
+type I2 interface {
+	I0
+	F(x float) float  // exported method
+	G(x float) float  // exported method
+}
+
+// The S3 struct; all comments except for the last one must appear in the export.
+type S3 struct {
+	// lead comment for F1
+	F1 int // line comment for F1
+	// lead comment for F2
+	F2 int // line comment for F2
+	f3 int // f3 is not exported
+}
+
+// This comment group should be separated
+// with a newline from the next comment
+// group.
+
+// This comment should NOT be associated with the next declaration.
+
+var x int  // x
+var ()
+
+
+// This comment SHOULD be associated with the next declaration.
+func f0() {
+	const pi = 3.14  // pi
+	var s1 struct {}  /* an empty struct */ /* foo */
+	// a struct constructor
+	// --------------------
+	var s2 struct {} = struct {}{}
+	x := pi
+}
+//
+// NO SPACE HERE
+//
+func f1() {
+	f0()
+	/* 1 */
+	// 2
+	/* 3 */
+	/* 4 */
+	f0()
+}
+
+
+func _() {
+	// this comment should be properly indented
+}
+
+
+func _(x int) int {
+	if x < 0 {  // the tab printed before this comment's // must not affect the remaining lines
+		return -x  // this statement should be properly indented
+	}
+	if x < 0 {  /* the tab printed before this comment's /* must not affect the remaining lines */
+		return -x  // this statement should be properly indented
+	}
+	return x
+}
+
+
+func typeswitch(x interface{}) {
+	switch v := x.(type) {
+	case bool, int, float:
+	case string:
+	default:
+	}
+
+	switch x.(type) {
+	}
+
+	switch v0, ok := x.(int); v := x.(type) {
+	}
+
+	switch v0, ok := x.(int); x.(type) {
+	case byte:  // this comment should be on the same line as the keyword
+		// this comment should be normally indented
+		_ = 0
+	case bool, int, float:
+		// this comment should be indented
+	case string:
+	default:
+		// this comment should be indented
+	}
+	// this comment should not be indented
+}
+
+func _() {
+	/* freestanding comment
+	   aligned		line
+	   aligned line
+	*/
+}
+
+func _() {
+	/* freestanding comment
+	   aligned		line
+	   aligned line
+	   */
+}
+
+func _() {
+	/* freestanding comment
+	   aligned		line
+	   aligned line */
+}
+
+func _() {
+	/*	freestanding comment
+		aligned		line
+		aligned line
+	*/
+}
+
+func _() {
+	/*	freestanding comment
+		aligned		line
+		aligned line
+		*/
+}
+
+func _() {
+	/*	freestanding comment
+		aligned		line
+		aligned line */
+}
+
+
+func _() {
+	/*
+	   freestanding comment
+	   aligned		line
+	   aligned line
+	*/
+}
+
+func _() {
+	/*
+	   freestanding comment
+	   aligned		line
+	   aligned line
+	   */
+}
+
+func _() {
+	/*
+	   freestanding comment
+	   aligned		line
+	   aligned line */
+}
+
+func _() {
+	/*
+		freestanding comment
+		aligned		line
+		aligned line
+	*/
+}
+
+func _() {
+	/*
+		freestanding comment
+		aligned		line
+		aligned line
+		*/
+}
+
+func _() {
+	/*
+		freestanding comment
+		aligned		line
+		aligned line */
+}
+
+func _() {
+	/* freestanding comment
+	   aligned line
+	*/
+}
+
+func _() {
+	/* freestanding comment
+	   aligned line
+	   */
+}
+
+func _() {
+	/* freestanding comment
+	   aligned line */
+}
+
+func _() {
+	/*	freestanding comment
+		aligned line
+	*/
+}
+
+func _() {
+	/*	freestanding comment
+		aligned line
+		*/
+}
+
+func _() {
+	/*	freestanding comment
+		aligned line */
+}
+
+
+func _() {
+	/*
+	   freestanding comment
+	   aligned line
+	*/
+}
+
+func _() {
+	/*
+	   freestanding comment
+	   aligned line
+	   */
+}
+
+func _() {
+	/*
+	   freestanding comment
+	   aligned line */
+}
+
+func _() {
+	/*
+		freestanding comment
+		aligned line
+	*/
+}
+
+func _() {
+	/*
+		freestanding comment
+		aligned line
+		*/
+}
+
+func _() {
+	/*
+		freestanding comment
+		aligned line */
+}
+
+/*
+ * line
+ * of
+ * stars
+ */
+
+/* another line
+ * of
+ * stars */
+
+/*	and another line
+ *	of
+ *	stars */
+
+/* a line of
+ * stars */
+
+/*	and another line of
+ *	stars */
+
+/* a line of stars
+*/
+
+/*	and another line of
+*/
+
+/* a line of stars
+ */
+
+/*	and another line of
+ */
+
+/*
+aligned in middle
+here
+        not here
+*/
+
+/*
+blank line in middle:
+
+with no leading spaces on blank line.
+*/
+
+/*
+   aligned in middle
+   here
+           not here
+*/
+
+/*
+	blank line in middle:
+
+	with no leading spaces on blank line.
+*/
+
+func _() {
+	/*
+	 * line
+	 * of
+	 * stars
+	 */
+
+	/*
+	aligned in middle
+	here
+		not here
+	*/
+
+	/*
+	blank line in middle:
+
+	with no leading spaces on blank line.
+*/
+}
+
+
+// Some interesting interspersed comments
+func _(/* this */x/* is *//* an */ int) {
+}
+
+func _(/* no params */) {}
+
+func _() {
+	f(/* no args */)
+}
+
+func (/* comment1 */ T /* comment2 */) _() {}
+
+func _() { /* one-line functions with comments are formatted as multi-line functions */ }
+
+func _() {
+	_ = 0
+	/* closing curly brace should be on new line */ }
+
+func _() {
+	_ = []int{0, 1 /* don't introduce a newline after this comment - was issue 1365 */}
+}
+
+
+// Comments immediately adjacent to punctuation (for which the go/printer
+// may only have estimated position information) must remain after the punctuation.
+func _() {
+	_ = T{
+		1,    // comment after comma
+		2,    /* comment after comma */
+		3  ,  // comment after comma
+	}
+	_ = T{
+		1  ,// comment after comma
+		2  ,/* comment after comma */
+		3,// comment after comma
+	}
+	_ = T{
+		/* comment before literal */1,
+		2/* comment before comma - ok to move after comma */,
+		3  /* comment before comma - ok to move after comma */  ,
+	}
+
+	for
+		i=0;// comment after semicolon
+		i<9;/* comment after semicolon */
+		i++{// comment after opening curly brace
+	}
+
+	// TODO(gri) the last comment in this example should be aligned */
+	for
+		i=0;// comment after semicolon
+		i<9/* comment before semicolon - ok to move after semicolon */;
+		i++ /* comment before opening curly brace */ {
+	}
+}
+
+
+// Line comments with tabs
+func _() {
+var	finput		*bufio.Reader			// input file
+var	stderr		*bufio.Writer
+var	ftable		*bufio.Writer			// y.go file
+var	foutput		*bufio.Writer			// y.output file
+
+var	oflag		string				// -o [y.go]		- y.go file
+var	vflag		string				// -v [y.output]	- y.output file
+var	lflag		bool				// -l			- disable line directives
+}
+
+
+/* This comment is the last entry in this file. It must be printed and should be followed by a newline */
diff --git a/src/pkg/go/printer/testdata/comments.x b/src/pkg/go/printer/testdata/comments.x
new file mode 100644
index 000000000..ae7729286
--- /dev/null
+++ b/src/pkg/go/printer/testdata/comments.x
@@ -0,0 +1,56 @@
+// This is a package for testing comment placement by go/printer.
+//
+package main
+
+// The SZ struct; it is empty.
+type SZ struct{}
+
+// The S0 struct; no field is exported.
+type S0 struct {
+	// contains filtered or unexported fields
+}
+
+// The S1 struct; some fields are not exported.
+type S1 struct {
+	S0
+	A, B, C	float	// 3 exported fields
+	D	int	// 2 unexported fields
+	// contains filtered or unexported fields
+}
+
+// The S2 struct; all fields are exported.
+type S2 struct {
+	S1
+	A, B, C	float	// 3 exported fields
+}
+
+// The IZ interface; it is empty.
+type SZ interface{}
+
+// The I0 interface; no method is exported.
+type I0 interface {
+	// contains filtered or unexported methods
+}
+
+// The I1 interface; some methods are not exported.
+type I1 interface {
+	I0
+	F(x float) float	// exported methods
+	// contains filtered or unexported methods
+}
+
+// The I2 interface; all methods are exported.
+type I2 interface {
+	I0
+	F(x float) float	// exported method
+	G(x float) float	// exported method
+}
+
+// The S3 struct; all comments except for the last one must appear in the export.
+type S3 struct {
+	// lead comment for F1
+	F1	int	// line comment for F1
+	// lead comment for F2
+	F2	int	// line comment for F2
+	// contains filtered or unexported fields
+}
diff --git a/src/pkg/go/printer/testdata/declarations.golden b/src/pkg/go/printer/testdata/declarations.golden
new file mode 100644
index 000000000..970533e8c
--- /dev/null
+++ b/src/pkg/go/printer/testdata/declarations.golden
@@ -0,0 +1,747 @@
+// Copyright 2009 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package imports
+
+import "io"
+
+import (
+	_ "io"
+)
+
+import _ "io"
+
+import (
+	"io"
+	"io"
+	"io"
+)
+
+import (
+	"io"
+	aLongRename "io"
+
+	b "io"
+)
+
+import (
+	"unrenamed"
+	renamed "renameMe"
+	. "io"
+	_ "io"
+	"io"
+	. "os"
+)
+
+// no newlines between consecutive single imports, but
+// respect extra line breaks in the source (at most one empty line)
+import _ "io"
+import _ "io"
+import _ "io"
+
+import _ "os"
+import _ "os"
+import _ "os"
+
+import _ "fmt"
+import _ "fmt"
+import _ "fmt"
+
+import "foo"	// a comment
+import "bar"	// a comment
+
+import (
+	_ "foo"
+	// a comment
+	"bar"
+	"foo"	// a comment
+	"bar"	// a comment
+)
+
+// comments + renames
+import (
+	"unrenamed"	// a comment
+	renamed "renameMe"
+	. "io"		/* a comment */
+	_ "io/ioutil"	// a comment
+	"io"		// testing alignment
+	. "os"
+	// a comment
+)
+
+// a case that caused problems in the past (comment placement)
+import (
+	. "fmt"
+	"io"
+	"malloc"	// for the malloc count test only
+	"math"
+	"strings"
+	"testing"
+)
+
+// more import examples
+import (
+	"xxx"
+	"much longer name"	// comment
+	"short name"		// comment
+)
+
+import (
+	_ "xxx"
+	"much longer name"	// comment
+)
+
+import (
+	mymath "math"
+	"/foo/bar/long_package_path"	// a comment
+)
+
+import (
+	"package_a"	// comment
+	"package_b"
+	my_better_c "package_c"	// comment
+	"package_d"		// comment
+	my_e "package_e"	// comment
+
+	"package_a"	// comment
+	"package_bb"
+	"package_ccc"	// comment
+	"package_dddd"	// comment
+)
+
+// at least one empty line between declarations of different kind
+import _ "io"
+
+var _ int
+
+// printing of constant literals
+const (
+	_	= "foobar"
+	_	= "a۰۱۸"
+	_	= "foo६४"
+	_	= "bar9876"
+	_	= 0
+	_	= 1
+	_	= 123456789012345678890
+	_	= 01234567
+	_	= 0xcafebabe
+	_	= 0.
+	_	= .0
+	_	= 3.14159265
+	_	= 1e0
+	_	= 1e+100
+	_	= 1e-100
+	_	= 2.71828e-1000
+	_	= 0i
+	_	= 1i
+	_	= 012345678901234567889i
+	_	= 123456789012345678890i
+	_	= 0.i
+	_	= .0i
+	_	= 3.14159265i
+	_	= 1e0i
+	_	= 1e+100i
+	_	= 1e-100i
+	_	= 2.71828e-1000i
+	_	= 'a'
+	_	= '\000'
+	_	= '\xFF'
+	_	= '\uff16'
+	_	= '\U0000ff16'
+	_	= `foobar`
+	_	= `foo
+---
+---
+bar`
+)
+
+func _() {
+	type _ int
+	type _ *int
+	type _ []int
+	type _ map[string]int
+	type _ chan int
+	type _ func() int
+
+	var _ int
+	var _ *int
+	var _ []int
+	var _ map[string]int
+	var _ chan int
+	var _ func() int
+
+	type _ struct{}
+	type _ *struct{}
+	type _ []struct{}
+	type _ map[string]struct{}
+	type _ chan struct{}
+	type _ func() struct{}
+
+	type _ interface{}
+	type _ *interface{}
+	type _ []interface{}
+	type _ map[string]interface{}
+	type _ chan interface{}
+	type _ func() interface{}
+
+	var _ struct{}
+	var _ *struct{}
+	var _ []struct{}
+	var _ map[string]struct{}
+	var _ chan struct{}
+	var _ func() struct{}
+
+	var _ interface{}
+	var _ *interface{}
+	var _ []interface{}
+	var _ map[string]interface{}
+	var _ chan interface{}
+	var _ func() interface{}
+}
+
+// don't lose blank lines in grouped declarations
+const (
+	_	int	= 0
+	_	float	= 1
+
+	_	string	= "foo"
+
+	_	= iota
+	_
+
+	// a comment
+	_
+
+	_
+)
+
+type (
+	_	int
+	_	struct{}
+
+	_	interface{}
+
+	// a comment
+	_	map[string]int
+)
+
+var (
+	_	int	= 0
+	_	float	= 1
+
+	_	string	= "foo"
+
+	_	bool
+
+	// a comment
+	_	bool
+)
+
+// don't lose blank lines in this struct
+type _ struct {
+	String	struct {
+		Str, Len int
+	}
+	Slice	struct {
+		Array, Len, Cap int
+	}
+	Eface	struct {
+		Typ, Ptr int
+	}
+
+	UncommonType	struct {
+		Name, PkgPath int
+	}
+	CommonType	struct {
+		Size, Hash, Alg, Align, FieldAlign, String, UncommonType int
+	}
+	Type	struct {
+		Typ, Ptr int
+	}
+	StructField	struct {
+		Name, PkgPath, Typ, Tag, Offset int
+	}
+	StructType	struct {
+		Fields int
+	}
+	PtrType	struct {
+		Elem int
+	}
+	SliceType	struct {
+		Elem int
+	}
+	ArrayType	struct {
+		Elem, Len int
+	}
+
+	Stktop	struct {
+		Stackguard, Stackbase, Gobuf int
+	}
+	Gobuf	struct {
+		Sp, Pc, G int
+	}
+	G	struct {
+		Stackbase, Sched, Status, Alllink int
+	}
+}
+
+// no tabs for single or ungrouped decls
+func _() {
+	const xxxxxx = 0
+	type x int
+	var xxx int
+	var yyyy float = 3.14
+	var zzzzz = "bar"
+
+	const (
+		xxxxxx = 0
+	)
+	type (
+		x int
+	)
+	var (
+		xxx int
+	)
+	var (
+		yyyy float = 3.14
+	)
+	var (
+		zzzzz = "bar"
+	)
+}
+
+// tabs for multiple or grouped decls
+func _() {
+	// no entry has a type
+	const (
+		zzzzzz	= 1
+		z	= 2
+		zzz	= 3
+	)
+	// some entries have a type
+	const (
+		xxxxxx			= 1
+		x			= 2
+		xxx			= 3
+		yyyyyyyy	float	= iota
+		yyyy			= "bar"
+		yyy
+		yy	= 2
+	)
+}
+
+func _() {
+	// no entry has a type
+	var (
+		zzzzzz	= 1
+		z	= 2
+		zzz	= 3
+	)
+	// no entry has a value
+	var (
+		_	int
+		_	float
+		_	string
+
+		_	int	// comment
+		_	float	// comment
+		_	string	// comment
+	)
+	// some entries have a type
+	var (
+		xxxxxx		int
+		x		float
+		xxx		string
+		yyyyyyyy	int	= 1234
+		y		float	= 3.14
+		yyyy			= "bar"
+		yyy		string	= "foo"
+	)
+	// mixed entries - all comments should be aligned
+	var (
+		a, b, c			int
+		x			= 10
+		d			int			// comment
+		y				= 20		// comment
+		f, ff, fff, ffff	int	= 0, 1, 2, 3	// comment
+	)
+	// respect original line breaks
+	var _ = []T{
+		T{0x20, "Telugu"},
+	}
+	var _ = []T{
+		// respect original line breaks
+		T{0x20, "Telugu"},
+	}
+}
+
+func _() {
+	type (
+		xxxxxx	int
+		x	float
+		xxx	string
+		xxxxx	[]x
+		xx	struct{}
+		xxxxxxx	struct {
+			_, _	int
+			_	float
+		}
+		xxxx	chan<- string
+	)
+}
+
+// alignment of "=" in consecutive lines (extended example from issue 1414)
+const (
+	umax	uint	= ^uint(0)		// maximum value for a uint
+	bpu		= 1 << (5 + umax>>63)	// bits per uint
+	foo
+	bar	= -1
+)
+
+// typical enum
+const (
+	a	MyType	= iota
+	abcd
+	b
+	c
+	def
+)
+
+// excerpt from godoc.go
+var (
+	goroot		= flag.String("goroot", runtime.GOROOT(), "Go root directory")
+	testDir		= flag.String("testdir", "", "Go root subdirectory - for testing only (faster startups)")
+	pkgPath		= flag.String("path", "", "additional package directories (colon-separated)")
+	filter		= flag.String("filter", "", "filter file containing permitted package directory paths")
+	filterMin	= flag.Int("filter_minutes", 0, "filter file update interval in minutes; disabled if <= 0")
+	filterDelay	delayTime	// actual filter update interval in minutes; usually filterDelay == filterMin, but filterDelay may back off exponentially
+)
+
+// formatting of structs
+type _ struct{}
+
+type _ struct { /* this comment should be visible */
+}
+
+type _ struct {
+	// this comment should be visible and properly indented
+}
+
+type _ struct {	// this comment must not change indentation
+	f			int
+	f, ff, fff, ffff	int
+}
+
+type _ struct {
+	string
+}
+
+type _ struct {
+	string	// comment
+}
+
+type _ struct {
+	string "tag"
+}
+
+type _ struct {
+	string "tag"	// comment
+}
+
+type _ struct {
+	f int
+}
+
+type _ struct {
+	f int	// comment
+}
+
+type _ struct {
+	f int "tag"
+}
+
+type _ struct {
+	f int "tag"	// comment
+}
+
+type _ struct {
+	bool
+	a, b, c			int
+	int			"tag"
+	ES				// comment
+	float			"tag"	// comment
+	f			int	// comment
+	f, ff, fff, ffff	int	// comment
+	g			float	"tag"
+	h			float	"tag"	// comment
+}
+
+type _ struct {
+	a, b,
+	c, d	int	// this line should be indented
+	u, v, w, x	float	// this line should be indented
+	p, q,
+	r, s	float	// this line should be indented
+}
+
+// difficult cases
+type _ struct {
+	bool		// comment
+	text	[]byte	// comment
+}
+
+// formatting of interfaces
+type EI interface{}
+
+type _ interface {
+	EI
+}
+
+type _ interface {
+	f()
+	fffff()
+}
+
+type _ interface {
+	EI
+	f()
+	fffffg()
+}
+
+type _ interface {	// this comment must not change indentation
+	EI				// here's a comment
+	f()				// no blank between identifier and ()
+	fffff()				// no blank between identifier and ()
+	gggggggggggg(x, y, z int)	// hurray
+}
+
+// formatting of variable declarations
+func _() {
+	type day struct {
+		n		int
+		short, long	string
+	}
+	var (
+		Sunday		= day{0, "SUN", "Sunday"}
+		Monday		= day{1, "MON", "Monday"}
+		Tuesday		= day{2, "TUE", "Tuesday"}
+		Wednesday	= day{3, "WED", "Wednesday"}
+		Thursday	= day{4, "THU", "Thursday"}
+		Friday		= day{5, "FRI", "Friday"}
+		Saturday	= day{6, "SAT", "Saturday"}
+	)
+}
+
+// formatting of multi-line variable declarations
+var a1, b1, c1 int	// all on one line
+
+var a2, b2,
+	c2 int	// this line should be indented
+
+var (
+	a3, b3,
+	c3, d3	int	// this line should be indented
+	a4, b4, c4	int	// this line should be indented
+)
+
+func _() {
+	var privateKey2 = &Block{Type:	"RSA PRIVATE KEY",
+		Headers:	map[string]string{},
+		Bytes: []uint8{0x30, 0x82, 0x1, 0x3a, 0x2, 0x1, 0x0, 0x2,
+			0x41, 0x0, 0xb2, 0x99, 0xf, 0x49, 0xc4, 0x7d, 0xfa, 0x8c,
+			0xd4, 0x0, 0xae, 0x6a, 0x4d, 0x1b, 0x8a, 0x3b, 0x6a, 0x13,
+			0x64, 0x2b, 0x23, 0xf2, 0x8b, 0x0, 0x3b, 0xfb, 0x97, 0x79,
+		},
+	}
+}
+
+func _() {
+	var Universe = Scope{
+		Names: map[string]*Ident{
+			// basic types
+			"bool":		nil,
+			"byte":		nil,
+			"int8":		nil,
+			"int16":	nil,
+			"int32":	nil,
+			"int64":	nil,
+			"uint8":	nil,
+			"uint16":	nil,
+			"uint32":	nil,
+			"uint64":	nil,
+			"float32":	nil,
+			"float64":	nil,
+			"string":	nil,
+
+			// convenience types
+			"int":		nil,
+			"uint":		nil,
+			"uintptr":	nil,
+			"float":	nil,
+
+			// constants
+			"false":	nil,
+			"true":		nil,
+			"iota":		nil,
+			"nil":		nil,
+
+			// functions
+			"cap":		nil,
+			"len":		nil,
+			"new":		nil,
+			"make":		nil,
+			"panic":	nil,
+			"panicln":	nil,
+			"print":	nil,
+			"println":	nil,
+		},
+	}
+}
+
+// alignment of map composite entries
+var _ = map[int]int{
+	// small key sizes: always align even if size ratios are large
+	a:			a,
+	abcdefghabcdefgh:	a,
+	ab:			a,
+	abc:			a,
+	abcdefgabcdefg:		a,
+	abcd:			a,
+	abcde:			a,
+	abcdef:			a,
+
+	// mixed key sizes: align when key sizes change within accepted ratio
+	abcdefgh:		a,
+	abcdefghabcdefg:	a,
+	abcdefghij:		a,
+	abcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghij:	a,	// outlier - do not align with previous line
+	abcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghij:		a,	// align with previous line
+
+	ab:	a,	// do not align with previous line
+	abcde:	a,	// align with previous line
+}
+
+func _() {
+	var _ = T{
+		a,	// must introduce trailing comma
+	}
+}
+
+// formatting of function results
+func _() func()				{}
+func _() func(int)			{ return nil }
+func _() func(int) int			{ return nil }
+func _() func(int) func(int) func()	{ return nil }
+
+// formatting of consecutive single-line functions
+func _()	{}
+func _()	{}
+func _()	{}
+
+func _()	{}	// an empty line before this function
+func _()	{}
+func _()	{}
+
+func _()		{ f(1, 2, 3) }
+func _(x int) int	{ y := x; return y + 1 }
+func _() int		{ type T struct{}; var x T; return x }
+
+// these must remain multi-line since they are multi-line in the source
+func _() {
+	f(1, 2, 3)
+}
+func _(x int) int {
+	y := x
+	return y + 1
+}
+func _() int {
+	type T struct{}
+	var x T
+	return x
+}
+
+// making function declarations safe for new semicolon rules
+func _() { /* multi-line func because of comment */
+}
+
+func _() {
+	/* multi-line func because block is on multiple lines */
+}
+
+// ellipsis parameters
+func _(...int)
+func _(...*int)
+func _(...[]int)
+func _(...struct{})
+func _(bool, ...interface{})
+func _(bool, ...func())
+func _(bool, ...func(...int))
+func _(bool, ...map[string]int)
+func _(bool, ...chan int)
+
+func _(b bool, x ...int)
+func _(b bool, x ...*int)
+func _(b bool, x ...[]int)
+func _(b bool, x ...struct{})
+func _(x ...interface{})
+func _(x ...func())
+func _(x ...func(...int))
+func _(x ...map[string]int)
+func _(x ...chan int)
+
+// these parameter lists must remain multi-line since they are multi-line in the source
+func _(bool,
+int) {
+}
+func _(x bool,
+y int) {
+}
+func _(x,
+y bool) {
+}
+func _(bool,	// comment
+int) {
+}
+func _(x bool,	// comment
+y int) {
+}
+func _(x,	// comment
+y bool) {
+}
+func _(bool,	// comment
+// comment
+int) {
+}
+func _(x bool,	// comment
+// comment
+y int) {
+}
+func _(x,	// comment
+// comment
+y bool) {
+}
+func _(bool,
+// comment
+int) {
+}
+func _(x bool,
+// comment
+y int) {
+}
+func _(x,
+// comment
+y bool) {
+}
+func _(x,	// comment
+y,	// comment
+z bool) {
+}
+func _(x,	// comment
+y,	// comment
+z bool) {
+}
+func _(x int,	// comment
+y float,	// comment
+z bool) {
+}
diff --git a/src/pkg/go/printer/testdata/declarations.input b/src/pkg/go/printer/testdata/declarations.input
new file mode 100644
index 000000000..c6134096b
--- /dev/null
+++ b/src/pkg/go/printer/testdata/declarations.input
@@ -0,0 +1,757 @@
+// Copyright 2009 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package imports
+
+import "io"
+
+import (
+	_ "io"
+)
+
+import _ "io"
+
+import (
+	"io"
+	"io"
+	"io"
+)
+
+import (
+	"io"
+	aLongRename "io"
+
+	b "io"
+)
+
+import (
+       "unrenamed"
+       renamed "renameMe"
+       . "io"
+       _ "io"
+       "io"
+       . "os"
+)
+
+// no newlines between consecutive single imports, but
+// respect extra line breaks in the source (at most one empty line)
+import _ "io"
+import _ "io"
+import _ "io"
+
+import _ "os"
+import _ "os"
+import _ "os"
+
+
+import _ "fmt"
+import _ "fmt"
+import _ "fmt"
+
+import "foo"  // a comment
+import "bar"  // a comment
+
+import (
+	_ "foo"
+	// a comment
+	"bar"
+	"foo"  // a comment
+	"bar"  // a comment
+)
+
+// comments + renames
+import (
+       "unrenamed" // a comment
+       renamed "renameMe"
+       . "io" /* a comment */
+       _ "io/ioutil" // a comment
+       "io" // testing alignment
+       . "os"
+       // a comment
+)
+
+// a case that caused problems in the past (comment placement)
+import (
+	. "fmt"
+	"io"
+	"malloc"	// for the malloc count test only
+	"math"
+	"strings"
+	"testing"
+)
+
+// more import examples
+import (
+	"xxx"
+	"much longer name" // comment
+	"short name" // comment
+)
+
+import (
+	_ "xxx"
+	"much longer name" // comment
+)
+
+import (
+	mymath "math"
+	"/foo/bar/long_package_path" // a comment
+)
+
+import (
+	"package_a" // comment
+	"package_b"
+	my_better_c "package_c" // comment
+	"package_d" // comment
+	my_e "package_e" // comment
+
+	"package_a"    // comment
+	"package_bb"
+	"package_ccc"  // comment
+	"package_dddd" // comment
+)
+
+// at least one empty line between declarations of different kind
+import _ "io"
+var _ int
+
+
+// printing of constant literals
+const (
+	_ = "foobar"
+	_ = "a۰۱۸"
+	_ = "foo६४"
+	_ = "bar9876"
+	_ = 0
+	_ = 1
+	_ = 123456789012345678890
+	_ = 01234567
+	_ = 0xcafebabe
+	_ = 0.
+	_ = .0
+	_ = 3.14159265
+	_ = 1e0
+	_ = 1e+100
+	_ = 1e-100
+	_ = 2.71828e-1000
+	_ = 0i
+	_ = 1i
+	_ = 012345678901234567889i
+	_ = 123456789012345678890i
+	_ = 0.i
+	_ = .0i
+	_ = 3.14159265i
+	_ = 1e0i
+	_ = 1e+100i
+	_ = 1e-100i
+	_ = 2.71828e-1000i
+	_ = 'a'
+	_ = '\000'
+	_ = '\xFF'
+	_ = '\uff16'
+	_ = '\U0000ff16'
+	_ = `foobar`
+	_ = `foo
+---
+---
+bar`
+)
+
+
+func _() {
+	type _ int
+	type _ *int
+	type _ []int
+	type _ map[string]int
+	type _ chan int
+	type _ func() int
+
+	var _ int
+	var _ *int
+	var _ []int
+	var _ map[string]int
+	var _ chan int
+	var _ func() int
+
+	type _ struct{}
+	type _ *struct{}
+	type _ []struct{}
+	type _ map[string]struct{}
+	type _ chan struct{}
+	type _ func() struct{}
+
+	type _ interface{}
+	type _ *interface{}
+	type _ []interface{}
+	type _ map[string]interface{}
+	type _ chan interface{}
+	type _ func() interface{}
+
+	var _ struct{}
+	var _ *struct{}
+	var _ []struct{}
+	var _ map[string]struct{}
+	var _ chan struct{}
+	var _ func() struct{}
+
+	var _ interface{}
+	var _ *interface{}
+	var _ []interface{}
+	var _ map[string]interface{}
+	var _ chan interface{}
+	var _ func() interface{}
+}
+
+
+// don't lose blank lines in grouped declarations
+const (
+	_ int = 0
+	_ float = 1
+
+	_ string = "foo"
+
+	_ = iota
+	_
+	
+	// a comment
+	_
+
+	_
+)
+
+
+type (
+	_ int
+	_ struct {}
+	
+	_ interface{}
+	
+	// a comment
+	_ map[string]int
+)
+
+
+var (
+	_ int = 0
+	_ float = 1
+
+	_ string = "foo"
+
+	_ bool
+	
+	// a comment
+	_ bool
+)
+
+
+// don't lose blank lines in this struct
+type _ struct {
+	String struct {
+		Str, Len int
+	}
+	Slice struct {
+		Array, Len, Cap int
+	}
+	Eface struct {
+		Typ, Ptr int
+	}
+
+	UncommonType struct {
+		Name, PkgPath int
+	}
+	CommonType struct {
+		Size, Hash, Alg, Align, FieldAlign, String, UncommonType int
+	}
+	Type struct {
+		Typ, Ptr int
+	}
+	StructField struct {
+		Name, PkgPath, Typ, Tag, Offset int
+	}
+	StructType struct {
+		Fields int
+	}
+	PtrType struct {
+		Elem int
+	}
+	SliceType struct {
+		Elem int
+	}
+	ArrayType struct {
+		Elem, Len int
+	}
+
+	Stktop struct {
+		Stackguard, Stackbase, Gobuf int
+	}
+	Gobuf struct {
+		Sp, Pc, G int
+	}
+	G struct {
+		Stackbase, Sched, Status, Alllink int
+	}
+}
+
+
+// no tabs for single or ungrouped decls
+func _() {
+	const xxxxxx = 0
+	type x int
+	var xxx int
+	var yyyy float = 3.14
+	var zzzzz = "bar"
+
+	const (
+		xxxxxx = 0
+	)
+	type (
+		x int
+	)
+	var (
+		xxx int
+	)
+	var (
+		yyyy float = 3.14
+	)
+	var (
+		zzzzz = "bar"
+	)
+}
+
+// tabs for multiple or grouped decls
+func _() {
+	// no entry has a type
+	const (
+		zzzzzz = 1
+		z = 2
+		zzz = 3
+	)
+	// some entries have a type
+	const (
+		xxxxxx = 1
+		x = 2
+		xxx = 3
+		yyyyyyyy float = iota
+		yyyy = "bar"
+		yyy
+		yy = 2
+	)
+}
+
+func _() {
+	// no entry has a type
+	var (
+		zzzzzz = 1
+		z = 2
+		zzz = 3
+	)
+	// no entry has a value
+	var (
+		_ int
+		_ float
+		_ string
+
+		_ int  // comment
+		_ float  // comment
+		_ string  // comment
+	)
+	// some entries have a type
+	var (
+		xxxxxx int
+		x float
+		xxx string
+		yyyyyyyy int = 1234
+		y float = 3.14
+		yyyy = "bar"
+		yyy string = "foo"
+	)
+	// mixed entries - all comments should be aligned
+	var (
+		a, b, c int
+		x = 10
+		d int  // comment
+		y = 20  // comment
+		f, ff, fff, ffff int = 0, 1, 2, 3  // comment
+	)
+	// respect original line breaks
+	var _ = []T {
+		T{0x20,	"Telugu"},
+	}
+	var _ = []T {
+		// respect original line breaks
+		T{0x20,	"Telugu"},
+	}
+}
+
+func _() {
+	type (
+		xxxxxx int
+		x float
+		xxx string
+		xxxxx []x
+		xx struct{}
+		xxxxxxx struct {
+			_, _ int
+			_ float
+		}
+		xxxx chan<- string
+	)
+}
+
+// alignment of "=" in consecutive lines (extended example from issue 1414)
+const (
+	umax uint                  = ^uint(0) // maximum value for a uint
+	bpu  = 1 << (5 + umax>>63)            // bits per uint
+	foo
+	bar  = -1
+)
+
+// typical enum
+const (
+	a MyType = iota
+	abcd
+	b
+	c
+	def
+)
+
+// excerpt from godoc.go
+var (
+	goroot = flag.String("goroot", runtime.GOROOT(), "Go root directory")
+	testDir = flag.String("testdir", "", "Go root subdirectory - for testing only (faster startups)")
+	pkgPath = flag.String("path", "", "additional package directories (colon-separated)")
+	filter = flag.String("filter", "", "filter file containing permitted package directory paths")
+	filterMin = flag.Int("filter_minutes", 0, "filter file update interval in minutes; disabled if <= 0")
+	filterDelay delayTime // actual filter update interval in minutes; usually filterDelay == filterMin, but filterDelay may back off exponentially
+)
+
+
+// formatting of structs
+type _ struct{}
+
+type _ struct{ /* this comment should be visible */ }
+
+type _ struct{
+	// this comment should be visible and properly indented
+}
+
+type _ struct {  // this comment must not change indentation
+	f int
+	f, ff, fff, ffff int
+}
+
+type _ struct {
+	string
+}
+
+type _ struct {
+	string  // comment
+}
+
+type _ struct {
+	string "tag"
+}
+
+type _ struct {
+	string "tag"  // comment
+}
+
+type _ struct {
+	f int
+}
+
+type _ struct {
+	f int  // comment
+}
+
+type _ struct {
+	f int "tag"
+}
+
+type _ struct {
+	f int "tag"  // comment
+}
+
+type _ struct {
+	bool
+	a, b, c int
+	int "tag"
+	ES // comment
+	float "tag"  // comment
+	f int  // comment
+	f, ff, fff, ffff int  // comment
+	g float "tag"
+	h float "tag"  // comment
+}
+
+type _ struct { a, b,
+c, d int  // this line should be indented
+u, v, w, x float // this line should be indented
+p, q,
+r, s float // this line should be indented
+}
+
+
+// difficult cases
+type _ struct {
+	bool  // comment
+	text []byte  // comment
+}
+
+
+// formatting of interfaces
+type EI interface{}
+
+type _ interface {
+	EI
+}
+
+type _ interface {
+	f()
+	fffff()
+}
+
+type _ interface {
+	EI
+	f()
+	fffffg()
+}
+
+type _ interface {  // this comment must not change indentation
+	EI  // here's a comment
+	f()  // no blank between identifier and ()
+	fffff()  // no blank between identifier and ()
+	gggggggggggg(x, y, z int) ()  // hurray
+}
+
+
+// formatting of variable declarations
+func _() {
+	type day struct { n int; short, long string }
+	var (
+		Sunday = day{ 0, "SUN", "Sunday" }
+		Monday = day{ 1, "MON", "Monday" }
+		Tuesday = day{ 2, "TUE", "Tuesday" }
+		Wednesday = day{ 3, "WED", "Wednesday" }
+		Thursday = day{ 4, "THU", "Thursday" }
+		Friday = day{ 5, "FRI", "Friday" }
+		Saturday = day{ 6, "SAT", "Saturday" }
+	)
+}
+
+
+// formatting of multi-line variable declarations
+var a1, b1, c1 int  // all on one line
+
+var a2, b2,
+c2 int  // this line should be indented
+
+var (a3, b3,
+c3, d3 int  // this line should be indented
+a4, b4, c4 int  // this line should be indented
+)
+
+
+func _() {
+	var privateKey2 = &Block{Type: "RSA PRIVATE KEY",
+					Headers: map[string]string{},
+					Bytes: []uint8{0x30, 0x82, 0x1, 0x3a, 0x2, 0x1, 0x0, 0x2,
+			0x41, 0x0, 0xb2, 0x99, 0xf, 0x49, 0xc4, 0x7d, 0xfa, 0x8c,
+			0xd4, 0x0, 0xae, 0x6a, 0x4d, 0x1b, 0x8a, 0x3b, 0x6a, 0x13,
+			0x64, 0x2b, 0x23, 0xf2, 0x8b, 0x0, 0x3b, 0xfb, 0x97, 0x79,
+		},
+	}
+}
+
+
+func _() {
+	var Universe = Scope {
+		Names: map[string]*Ident {
+			// basic types
+			"bool": nil,
+			"byte": nil,
+			"int8": nil,
+			"int16": nil,
+			"int32": nil,
+			"int64": nil,
+			"uint8": nil,
+			"uint16": nil,
+			"uint32": nil,
+			"uint64": nil,
+			"float32": nil,
+			"float64": nil,
+			"string": nil,
+
+			// convenience types
+			"int": nil,
+			"uint": nil,
+			"uintptr": nil,
+			"float": nil,
+
+			// constants
+			"false": nil,
+			"true": nil,
+			"iota": nil,
+			"nil": nil,
+
+			// functions
+			"cap": nil,
+			"len": nil,
+			"new": nil,
+			"make": nil,
+			"panic": nil,
+			"panicln": nil,
+			"print": nil,
+			"println": nil,
+		},
+	}
+}
+
+
+// alignment of map composite entries
+var _ = map[int]int{
+	// small key sizes: always align even if size ratios are large
+	a: a,
+	abcdefghabcdefgh: a,
+	ab: a,
+	abc: a,
+	abcdefgabcdefg: a,
+	abcd: a,
+	abcde: a,
+	abcdef: a,
+
+	// mixed key sizes: align when key sizes change within accepted ratio
+	abcdefgh: a,
+	abcdefghabcdefg: a,
+	abcdefghij: a,
+	abcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghij: a, // outlier - do not align with previous line
+	abcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghij: a, // align with previous line
+
+	ab: a, // do not align with previous line
+	abcde: a, // align with previous line
+}
+
+
+func _() {
+	var _ = T{
+		a,	// must introduce trailing comma
+	}
+}
+
+
+// formatting of function results
+func _() func() {}
+func _() func(int) { return nil }
+func _() func(int) int { return nil }
+func _() func(int) func(int) func() { return nil }
+
+
+// formatting of consecutive single-line functions
+func _() {}
+func _() {}
+func _() {}
+
+func _() {}  // an empty line before this function
+func _() {}
+func _() {}
+
+func _() { f(1, 2, 3) }
+func _(x int) int { y := x; return y+1 }
+func _() int { type T struct{}; var x T; return x }
+
+// these must remain multi-line since they are multi-line in the source
+func _() {
+	f(1, 2, 3)
+}
+func _(x int) int {
+	y := x; return y+1
+}
+func _() int {
+	type T struct{}; var x T; return x
+}
+
+
+// making function declarations safe for new semicolon rules
+func _() { /* multi-line func because of comment */ }
+
+func _() {
+/* multi-line func because block is on multiple lines */ }
+
+
+// ellipsis parameters
+func _(...int)
+func _(...*int)
+func _(...[]int)
+func _(...struct{})
+func _(bool, ...interface{})
+func _(bool, ...func())
+func _(bool, ...func(...int))
+func _(bool, ...map[string]int)
+func _(bool, ...chan int)
+
+func _(b bool, x ...int)
+func _(b bool, x ...*int)
+func _(b bool, x ...[]int)
+func _(b bool, x ...struct{})
+func _(x ...interface{})
+func _(x ...func())
+func _(x ...func(...int))
+func _(x ...map[string]int)
+func _(x ...chan int)
+
+
+// these parameter lists must remain multi-line since they are multi-line in the source
+func _(bool,
+int) {
+}
+func _(x bool,
+y int) {
+}
+func _(x,
+y bool) {
+}
+func _(bool, // comment
+int) {
+}
+func _(x bool, // comment
+y int) {
+}
+func _(x, // comment
+y bool) {
+}
+func _(bool, // comment
+// comment
+int) {
+}
+func _(x bool, // comment
+// comment
+y int) {
+}
+func _(x, // comment
+// comment
+y bool) {
+}
+func _(bool,
+// comment
+int) {
+}
+func _(x bool,
+// comment
+y int) {
+}
+func _(x,
+// comment
+y bool) {
+}
+func _(x, // comment
+y,// comment
+z bool) {
+}
+func _(x, // comment
+	y,// comment
+	z bool) {
+}
+func _(x int,	// comment
+	y float,	// comment
+	z bool) {
+}
diff --git a/src/pkg/go/printer/testdata/empty.golden b/src/pkg/go/printer/testdata/empty.golden
new file mode 100644
index 000000000..a055f4758
--- /dev/null
+++ b/src/pkg/go/printer/testdata/empty.golden
@@ -0,0 +1,5 @@
+// a comment at the beginning of the file
+
+package empty
+
+// a comment at the end of the file
diff --git a/src/pkg/go/printer/testdata/empty.input b/src/pkg/go/printer/testdata/empty.input
new file mode 100644
index 000000000..a055f4758
--- /dev/null
+++ b/src/pkg/go/printer/testdata/empty.input
@@ -0,0 +1,5 @@
+// a comment at the beginning of the file
+
+package empty
+
+// a comment at the end of the file
diff --git a/src/pkg/go/printer/testdata/expressions.golden b/src/pkg/go/printer/testdata/expressions.golden
new file mode 100644
index 000000000..d0cf24ad6
--- /dev/null
+++ b/src/pkg/go/printer/testdata/expressions.golden
@@ -0,0 +1,627 @@
+// Copyright 2009 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package expressions
+
+type T struct {
+	x, y, z int
+}
+
+var (
+	a, b, c, d, e						int
+	under_bar						int
+	longIdentifier1, longIdentifier2, longIdentifier3	int
+	t0, t1, t2						T
+	s							string
+	p							*int
+)
+
+func _() {
+	// no spaces around simple or parenthesized expressions
+	_ = (a + 0)
+	_ = a + b
+	_ = a + b + c
+	_ = a + b - c
+	_ = a - b - c
+	_ = a + (b * c)
+	_ = a + (b / c)
+	_ = a - (b % c)
+	_ = 1 + a
+	_ = a + 1
+	_ = a + b + 1
+	_ = s[a]
+	_ = s[a:]
+	_ = s[:b]
+	_ = s[1:2]
+	_ = s[a:b]
+	_ = s[0:len(s)]
+	_ = s[0] << 1
+	_ = (s[0] << 1) & 0xf
+	_ = s[0]<<2 | s[1]>>4
+	_ = "foo" + s
+	_ = s + "foo"
+	_ = 'a' + 'b'
+	_ = len(s) / 2
+	_ = len(t0.x) / a
+
+	// spaces around expressions of different precedence or expressions containing spaces
+	_ = a + -b
+	_ = a - ^b
+	_ = a / *p
+	_ = a + b*c
+	_ = 1 + b*c
+	_ = a + 2*c
+	_ = a + c*2
+	_ = 1 + 2*3
+	_ = s[1 : 2*3]
+	_ = s[a : b-c]
+	_ = s[0:]
+	_ = s[a+b]
+	_ = s[:b-c]
+	_ = s[a+b:]
+	_ = a[a< b
+	_ = a >= b
+	_ = a < b
+	_ = a <= b
+	_ = a < b && c > d
+	_ = a < b || c > d
+
+	// spaces around "long" operands
+	_ = a + longIdentifier1
+	_ = longIdentifier1 + a
+	_ = longIdentifier1 + longIdentifier2*longIdentifier3
+	_ = s + "a longer string"
+
+	// some selected cases
+	_ = a + t0.x
+	_ = a + t0.x + t1.x*t2.x
+	_ = a + b + c + d + e + 2*3
+	_ = a + b + c + 2*3 + d + e
+	_ = (a + b + c) * 2
+	_ = a - b + c - d + (a + b + c) + d&e
+	_ = under_bar - 1
+	_ = Open(dpath+"/file", O_WRONLY|O_CREAT, 0666)
+	_ = int(c0&_Mask4)<<18 | int(c1&_Maskx)<<12 | int(c2&_Maskx)<<6 | int(c3&_Maskx)
+
+	// the parser does not restrict expressions that may appear as statements
+	true
+	42
+	"foo"
+	x
+	(x)
+	a + b
+	a + b + c
+	a + (b * c)
+	a + (b / c)
+	1 + a
+	a + 1
+	s[a]
+	x << 1
+	(s[0] << 1) & 0xf
+	"foo" + s
+	x == y
+	x < y || z > 42
+}
+
+func _() {
+	_ = a + b
+	_ = a + b + c
+	_ = a + b*c
+	_ = a + (b * c)
+	_ = (a + b) * c
+	_ = a + (b * c * d)
+	_ = a + (b*c + d)
+
+	_ = 1 << x
+	_ = -1 << x
+	_ = 1<>4
+
+	b.buf = b.buf[0 : b.off+m+n]
+	b.buf = b.buf[0 : b.off+m*n]
+	f(b.buf[0 : b.off+m+n])
+
+	signed += ' ' * 8
+	tw.octal(header[148:155], chksum)
+
+	_ = x > 0 && i >= 0
+
+	x1, x0 := x>>w2, x&m2
+	z0 = t1<>w2) >> w2
+	q1, r1 := x1/d1, x1%d1
+	r1 = r1*b2 | x0>>w2
+	x1 = (x1 << z) | (x0 >> (uint(w) - z))
+	x1 = x1<>(uint(w)-z)
+
+	_ = buf[0 : len(buf)+1]
+	_ = buf[0 : n+1]
+
+	a, b = b, a
+	a = b + c
+	a = b*c + d
+	_ = a*b + c
+	_ = a - b - c
+	_ = a - (b - c)
+	_ = a - b*c
+	_ = a - (b * c)
+	_ = a * b / c
+	_ = a / *b
+	_ = x[a|^b]
+	_ = x[a / *b]
+	_ = a & ^b
+	_ = a + +b
+	_ = a - -b
+	_ = x[a*-b]
+	_ = x[a + +b]
+	_ = x ^ y ^ z
+	_ = b[a>>24] ^ b[(a>>16)&0xFF] ^ b[(a>>8)&0xFF] ^ b[a&0xFF]
+	_ = len(longVariableName) * 2
+
+	_ = token(matchType + xlength<> 4
+	_ = "foo"+s
+	_ = s+"foo"
+	_ = 'a'+'b'
+	_ = len(s)/2
+	_ = len(t0.x)/a
+
+	// spaces around expressions of different precedence or expressions containing spaces
+	_ = a + -b
+	_ = a - ^b
+	_ = a / *p
+	_ = a + b*c
+	_ = 1 + b*c
+	_ = a + 2*c
+	_ = a + c*2
+	_ = 1 + 2*3
+	_ = s[1 : 2*3]
+	_ = s[a : b-c]
+	_ = s[0:]
+	_ = s[a+b]
+	_ = s[: b-c]
+	_ = s[a+b :]
+	_ = a[a< b
+	_ = a >= b
+	_ = a < b
+	_ = a <= b
+	_ = a < b && c > d
+	_ = a < b || c > d
+
+	// spaces around "long" operands
+	_ = a + longIdentifier1
+	_ = longIdentifier1 + a
+	_ = longIdentifier1 + longIdentifier2 * longIdentifier3
+	_ = s + "a longer string"
+
+	// some selected cases
+	_ = a + t0.x
+	_ = a + t0.x + t1.x * t2.x
+	_ = a + b + c + d + e + 2*3
+	_ = a + b + c + 2*3 + d + e
+	_ = (a+b+c)*2
+	_ = a - b + c - d + (a+b+c) + d&e
+	_ = under_bar-1
+	_ = Open(dpath + "/file", O_WRONLY | O_CREAT, 0666)
+	_ = int(c0&_Mask4)<<18 | int(c1&_Maskx)<<12 | int(c2&_Maskx)<<6 | int(c3&_Maskx)
+
+	// the parser does not restrict expressions that may appear as statements
+	true
+	42
+	"foo"
+	x
+	(x)
+	a+b
+	a+b+c
+	a+(b*c)
+	a+(b/c)
+	1+a
+	a+1
+	s[a]
+	x<<1
+	(s[0]<<1)&0xf
+	"foo"+s
+	x == y
+	x < y || z > 42
+}
+
+
+func _() {
+	_ = a+b
+	_ = a+b+c
+	_ = a+b*c
+	_ = a+(b*c)
+	_ = (a+b)*c
+	_ = a+(b*c*d)
+	_ = a+(b*c+d)
+
+	_ = 1<>4
+
+	b.buf = b.buf[0:b.off+m+n]
+	b.buf = b.buf[0:b.off+m*n]
+	f(b.buf[0:b.off+m+n])
+
+	signed += ' '*8
+	tw.octal(header[148:155], chksum)
+
+	_ = x > 0 && i >= 0
+
+	x1, x0 := x>>w2, x&m2
+	z0 = t1<>w2)>>w2
+	q1, r1 := x1/d1, x1%d1
+	r1 = r1*b2 | x0>>w2
+	x1 = (x1<>(uint(w)-z))
+	x1 = x1<>(uint(w)-z)
+
+	_ = buf[0:len(buf)+1]
+	_ = buf[0:n+1]
+
+	a,b = b,a
+	a = b+c
+	a = b*c+d
+	_ = a*b+c
+	_ = a-b-c
+	_ = a-(b-c)
+	_ = a-b*c
+	_ = a-(b*c)
+	_ = a*b/c
+	_ = a/ *b
+	_ = x[a|^b]
+	_ = x[a/ *b]
+	_ = a& ^b
+	_ = a+ +b
+	_ = a- -b
+	_ = x[a*-b]
+	_ = x[a+ +b]
+	_ = x^y^z
+	_ = b[a>>24] ^ b[(a>>16)&0xFF] ^ b[(a>>8)&0xFF] ^ b[a&0xFF]
+	_ = len(longVariableName)*2
+
+	_ = token(matchType + xlength<>4
+	_ = "foo" + s
+	_ = s + "foo"
+	_ = 'a' + 'b'
+	_ = len(s) / 2
+	_ = len(t0.x) / a
+
+	// spaces around expressions of different precedence or expressions containing spaces
+	_ = a + -b
+	_ = a - ^b
+	_ = a / *p
+	_ = a + b*c
+	_ = 1 + b*c
+	_ = a + 2*c
+	_ = a + c*2
+	_ = 1 + 2*3
+	_ = s[1 : 2*3]
+	_ = s[a : b-c]
+	_ = s[0:]
+	_ = s[a+b]
+	_ = s[:b-c]
+	_ = s[a+b:]
+	_ = a[a< b
+	_ = a >= b
+	_ = a < b
+	_ = a <= b
+	_ = a < b && c > d
+	_ = a < b || c > d
+
+	// spaces around "long" operands
+	_ = a + longIdentifier1
+	_ = longIdentifier1 + a
+	_ = longIdentifier1 + longIdentifier2*longIdentifier3
+	_ = s + "a longer string"
+
+	// some selected cases
+	_ = a + t0.x
+	_ = a + t0.x + t1.x*t2.x
+	_ = a + b + c + d + e + 2*3
+	_ = a + b + c + 2*3 + d + e
+	_ = (a + b + c) * 2
+	_ = a - b + c - d + (a + b + c) + d&e
+	_ = under_bar - 1
+	_ = Open(dpath+"/file", O_WRONLY|O_CREAT, 0666)
+	_ = int(c0&_Mask4)<<18 | int(c1&_Maskx)<<12 | int(c2&_Maskx)<<6 | int(c3&_Maskx)
+
+	// the parser does not restrict expressions that may appear as statements
+	true
+	42
+	"foo"
+	x
+	(x)
+	a + b
+	a + b + c
+	a + (b * c)
+	a + (b / c)
+	1 + a
+	a + 1
+	s[a]
+	x << 1
+	(s[0] << 1) & 0xf
+	"foo" + s
+	x == y
+	x < y || z > 42
+}
+
+func _() {
+	_ = a + b
+	_ = a + b + c
+	_ = a + b*c
+	_ = a + (b * c)
+	_ = (a + b) * c
+	_ = a + (b * c * d)
+	_ = a + (b*c + d)
+
+	_ = 1 << x
+	_ = -1 << x
+	_ = 1<>4
+
+	b.buf = b.buf[0 : b.off+m+n]
+	b.buf = b.buf[0 : b.off+m*n]
+	f(b.buf[0 : b.off+m+n])
+
+	signed += ' ' * 8
+	tw.octal(header[148:155], chksum)
+
+	_ = x > 0 && i >= 0
+
+	x1, x0 := x>>w2, x&m2
+	z0 = t1<>w2) >> w2
+	q1, r1 := x1/d1, x1%d1
+	r1 = r1*b2 | x0>>w2
+	x1 = (x1 << z) | (x0 >> (uint(w) - z))
+	x1 = x1<>(uint(w)-z)
+
+	_ = buf[0 : len(buf)+1]
+	_ = buf[0 : n+1]
+
+	a, b = b, a
+	a = b + c
+	a = b*c + d
+	_ = a*b + c
+	_ = a - b - c
+	_ = a - (b - c)
+	_ = a - b*c
+	_ = a - (b * c)
+	_ = a * b / c
+	_ = a / *b
+	_ = x[a|^b]
+	_ = x[a / *b]
+	_ = a & ^b
+	_ = a + +b
+	_ = a - -b
+	_ = x[a*-b]
+	_ = x[a + +b]
+	_ = x ^ y ^ z
+	_ = b[a>>24] ^ b[(a>>16)&0xFF] ^ b[(a>>8)&0xFF] ^ b[a&0xFF]
+	_ = len(longVariableName) * 2
+
+	_ = token(matchType + xlength< /tmp/16gig.txt
+	//   tar -b 1 -c -f- /tmp/16gig.txt | dd bs=512 count=8 > writer-big.tar
+	&writerTest{
+		file:	"testdata/writer-big.tar",
+		entries: []*writerTestEntry{
+			&writerTestEntry{
+				header: &Header{
+					Name:		"tmp/16gig.txt",
+					Mode:		0640,
+					Uid:		73025,
+					Gid:		5000,
+					Size:		16 << 30,
+					Mtime:		1254699560,
+					Typeflag:	'0',
+					Uname:		"dsymonds",
+					Gname:		"eng",
+				},
+				// no contents
+			},
+		},
+	},
+}
+
+type untarTest struct {
+	file	string
+	headers	[]*Header
+}
+
+var untarTests = []*untarTest{
+	&untarTest{
+		file:	"testdata/gnu.tar",
+		headers: []*Header{
+			&Header{
+				Name:		"small.txt",
+				Mode:		0640,
+				Uid:		73025,
+				Gid:		5000,
+				Size:		5,
+				Mtime:		1244428340,
+				Typeflag:	'0',
+				Uname:		"dsymonds",
+				Gname:		"eng",
+			},
+			&Header{
+				Name:		"small2.txt",
+				Mode:		0640,
+				Uid:		73025,
+				Gid:		5000,
+				Size:		11,
+				Mtime:		1244436044,
+				Typeflag:	'0',
+				Uname:		"dsymonds",
+				Gname:		"eng",
+			},
+		},
+	},
+	&untarTest{
+		file:	"testdata/star.tar",
+		headers: []*Header{
+			&Header{
+				Name:		"small.txt",
+				Mode:		0640,
+				Uid:		73025,
+				Gid:		5000,
+				Size:		5,
+				Mtime:		1244592783,
+				Typeflag:	'0',
+				Uname:		"dsymonds",
+				Gname:		"eng",
+				Atime:		1244592783,
+				Ctime:		1244592783,
+			},
+			&Header{
+				Name:		"small2.txt",
+				Mode:		0640,
+				Uid:		73025,
+				Gid:		5000,
+				Size:		11,
+				Mtime:		1244592783,
+				Typeflag:	'0',
+				Uname:		"dsymonds",
+				Gname:		"eng",
+				Atime:		1244592783,
+				Ctime:		1244592783,
+			},
+		},
+	},
+	&untarTest{
+		file:	"testdata/v7.tar",
+		headers: []*Header{
+			&Header{
+				Name:		"small.txt",
+				Mode:		0444,
+				Uid:		73025,
+				Gid:		5000,
+				Size:		5,
+				Mtime:		1244593104,
+				Typeflag:	'\x00',
+			},
+			&Header{
+				Name:		"small2.txt",
+				Mode:		0444,
+				Uid:		73025,
+				Gid:		5000,
+				Size:		11,
+				Mtime:		1244593104,
+				Typeflag:	'\x00',
+			},
+		},
+	},
+}
+
+var facts = map[int]string{
+	0:	"1",
+	1:	"1",
+	2:	"2",
+	10:	"3628800",
+	20:	"2432902008176640000",
+	100: "933262154439441526816992388562667004907159682643816214685929" +
+		"638952175999932299156089414639761565182862536979208272237582" +
+		"51185210916864000000000000000000000000",
+}
+
+func usage() {
+	fmt.Fprintf(os.Stderr,
+		// TODO(gri): the 2nd string of this string list should not be indented
+		"usage: godoc package [name ...]\n"+
+			"	godoc -http=:6060\n")
+	flag.PrintDefaults()
+	os.Exit(2)
+}
+
+func TestReader(t *testing.T) {
+testLoop:
+	for i, test := range untarTests {
+		f, err := os.Open(test.file, os.O_RDONLY, 0444)
+		if err != nil {
+			t.Errorf("test %d: Unexpected error: %v", i, err)
+			continue
+		}
+		tr := NewReader(f)
+		for j, header := range test.headers {
+			hdr, err := tr.Next()
+			if err != nil || hdr == nil {
+				t.Errorf("test %d, entry %d: Didn't get entry: %v", i, j, err)
+				f.Close()
+				continue testLoop
+			}
+			if !reflect.DeepEqual(hdr, header) {
+				t.Errorf("test %d, entry %d: Incorrect header:\nhave %+v\nwant %+v",
+					i, j, *hdr, *header)
+			}
+		}
+		hdr, err := tr.Next()
+		if hdr != nil || err != nil {
+			t.Errorf("test %d: Unexpected entry or error: hdr=%v err=%v", i, err)
+		}
+		f.Close()
+	}
+}
+
+// There should be exactly one linebreak after this comment.
diff --git a/src/pkg/go/printer/testdata/linebreaks.input b/src/pkg/go/printer/testdata/linebreaks.input
new file mode 100644
index 000000000..457b491e6
--- /dev/null
+++ b/src/pkg/go/printer/testdata/linebreaks.input
@@ -0,0 +1,223 @@
+// Copyright 2009 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package linebreaks
+
+import (
+	"bytes"
+	"fmt"
+	"io"
+	"os"
+	"reflect"
+	"strings"
+	"testing"
+)
+
+type writerTestEntry struct {
+	header *Header
+	contents string
+}
+
+type writerTest struct {
+	file string  // filename of expected output
+	entries []*writerTestEntry
+}
+
+var writerTests = []*writerTest{
+	&writerTest{
+		file: "testdata/writer.tar",
+		entries: []*writerTestEntry{
+			&writerTestEntry{
+				header: &Header{
+					Name: "small.txt",
+					Mode: 0640,
+					Uid: 73025,
+					Gid: 5000,
+					Size: 5,
+					Mtime: 1246508266,
+					Typeflag: '0',
+					Uname: "dsymonds",
+					Gname: "eng",
+				},
+				contents: "Kilts",
+			},
+			&writerTestEntry{
+				header: &Header{
+					Name: "small2.txt",
+					Mode: 0640,
+					Uid: 73025,
+					Gid: 5000,
+					Size: 11,
+					Mtime: 1245217492,
+					Typeflag: '0',
+					Uname: "dsymonds",
+					Gname: "eng",
+				},
+				contents: "Google.com\n",
+			},
+		},
+	},
+	// The truncated test file was produced using these commands:
+	//   dd if=/dev/zero bs=1048576 count=16384 > /tmp/16gig.txt
+	//   tar -b 1 -c -f- /tmp/16gig.txt | dd bs=512 count=8 > writer-big.tar
+	&writerTest{
+		file: "testdata/writer-big.tar",
+		entries: []*writerTestEntry{
+			&writerTestEntry{
+				header: &Header{
+					Name: "tmp/16gig.txt",
+					Mode: 0640,
+					Uid: 73025,
+					Gid: 5000,
+					Size: 16 << 30,
+					Mtime: 1254699560,
+					Typeflag: '0',
+					Uname: "dsymonds",
+					Gname: "eng",
+				},
+				// no contents
+			},
+		},
+	},
+}
+
+type untarTest struct {
+	file string
+	headers []*Header
+}
+
+var untarTests = []*untarTest{
+	&untarTest{
+		file: "testdata/gnu.tar",
+		headers: []*Header{
+			&Header{
+				Name: "small.txt",
+				Mode: 0640,
+				Uid: 73025,
+				Gid: 5000,
+				Size: 5,
+				Mtime: 1244428340,
+				Typeflag: '0',
+				Uname: "dsymonds",
+				Gname: "eng",
+			},
+			&Header{
+				Name: "small2.txt",
+				Mode: 0640,
+				Uid: 73025,
+				Gid: 5000,
+				Size: 11,
+				Mtime: 1244436044,
+				Typeflag: '0',
+				Uname: "dsymonds",
+				Gname: "eng",
+			},
+		},
+	},
+	&untarTest{
+		file: "testdata/star.tar",
+		headers: []*Header{
+			&Header{
+				Name: "small.txt",
+				Mode: 0640,
+				Uid: 73025,
+				Gid: 5000,
+				Size: 5,
+				Mtime: 1244592783,
+				Typeflag: '0',
+				Uname: "dsymonds",
+				Gname: "eng",
+				Atime: 1244592783,
+				Ctime: 1244592783,
+			},
+			&Header{
+				Name: "small2.txt",
+				Mode: 0640,
+				Uid: 73025,
+				Gid: 5000,
+				Size: 11,
+				Mtime: 1244592783,
+				Typeflag: '0',
+				Uname: "dsymonds",
+				Gname: "eng",
+				Atime: 1244592783,
+				Ctime: 1244592783,
+			},
+		},
+	},
+	&untarTest{
+		file: "testdata/v7.tar",
+		headers: []*Header{
+			&Header{
+				Name: "small.txt",
+				Mode: 0444,
+				Uid: 73025,
+				Gid: 5000,
+				Size: 5,
+				Mtime: 1244593104,
+				Typeflag: '\x00',
+			},
+			&Header{
+				Name: "small2.txt",
+				Mode: 0444,
+				Uid: 73025,
+				Gid: 5000,
+				Size: 11,
+				Mtime: 1244593104,
+				Typeflag: '\x00',
+			},
+		},
+	},
+}
+
+var facts = map[int] string {
+	0: "1",
+	1: "1",
+	2: "2",
+	10: "3628800",
+	20: "2432902008176640000",
+	100: "933262154439441526816992388562667004907159682643816214685929" +
+		"638952175999932299156089414639761565182862536979208272237582" +
+		"51185210916864000000000000000000000000",
+}
+
+func usage() {
+	fmt.Fprintf(os.Stderr,
+		// TODO(gri): the 2nd string of this string list should not be indented
+		"usage: godoc package [name ...]\n" +
+		"	godoc -http=:6060\n")
+	flag.PrintDefaults()
+	os.Exit(2)
+}
+
+func TestReader(t *testing.T) {
+testLoop:
+	for i, test := range untarTests {
+		f, err := os.Open(test.file, os.O_RDONLY, 0444)
+		if err != nil {
+			t.Errorf("test %d: Unexpected error: %v", i, err)
+			continue
+		}
+		tr := NewReader(f)
+		for j, header := range test.headers {
+			hdr, err := tr.Next()
+			if err != nil || hdr == nil {
+				t.Errorf("test %d, entry %d: Didn't get entry: %v", i, j, err)
+				f.Close()
+				continue testLoop
+			}
+			if !reflect.DeepEqual(hdr, header) {
+				t.Errorf("test %d, entry %d: Incorrect header:\nhave %+v\nwant %+v",
+					 i, j, *hdr, *header)
+			}
+		}
+		hdr, err := tr.Next()
+		if hdr != nil || err != nil {
+			t.Errorf("test %d: Unexpected entry or error: hdr=%v err=%v", i, err)
+		}
+		f.Close()
+	}
+}
+
+// There should be exactly one linebreak after this comment.
diff --git a/src/pkg/go/printer/testdata/parser.go b/src/pkg/go/printer/testdata/parser.go
new file mode 100644
index 000000000..2d27af499
--- /dev/null
+++ b/src/pkg/go/printer/testdata/parser.go
@@ -0,0 +1,2153 @@
+// Copyright 2009 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// Package parser implements a parser for Go source files. Input may be
+// provided in a variety of forms (see the various Parse* functions); the
+// output is an abstract syntax tree (AST) representing the Go source. The
+// parser is invoked through one of the Parse* functions.
+//
+package parser
+
+import (
+	"fmt"
+	"go/ast"
+	"go/scanner"
+	"go/token"
+)
+
+// The mode parameter to the Parse* functions is a set of flags (or 0).
+// They control the amount of source code parsed and other optional
+// parser functionality.
+//
+const (
+	PackageClauseOnly uint = 1 << iota // parsing stops after package clause
+	ImportsOnly                        // parsing stops after import declarations
+	ParseComments                      // parse comments and add them to AST
+	Trace                              // print a trace of parsed productions
+	DeclarationErrors                  // report declaration errors
+)
+
+// The parser structure holds the parser's internal state.
+type parser struct {
+	file *token.File
+	scanner.ErrorVector
+	scanner scanner.Scanner
+
+	// Tracing/debugging
+	mode   uint // parsing mode
+	trace  bool // == (mode & Trace != 0)
+	indent uint // indentation used for tracing output
+
+	// Comments
+	comments    []*ast.CommentGroup
+	leadComment *ast.CommentGroup // last lead comment
+	lineComment *ast.CommentGroup // last line comment
+
+	// Next token
+	pos token.Pos   // token position
+	tok token.Token // one token look-ahead
+	lit string      // token literal
+
+	// Non-syntactic parser control
+	exprLev int // < 0: in control clause, >= 0: in expression
+
+	// Ordinary identifer scopes
+	pkgScope   *ast.Scope        // pkgScope.Outer == nil
+	topScope   *ast.Scope        // top-most scope; may be pkgScope
+	unresolved []*ast.Ident      // unresolved identifiers
+	imports    []*ast.ImportSpec // list of imports
+
+	// Label scope
+	// (maintained by open/close LabelScope)
+	labelScope  *ast.Scope     // label scope for current function
+	targetStack [][]*ast.Ident // stack of unresolved labels
+}
+
+// scannerMode returns the scanner mode bits given the parser's mode bits.
+func scannerMode(mode uint) uint {
+	var m uint = scanner.InsertSemis
+	if mode&ParseComments != 0 {
+		m |= scanner.ScanComments
+	}
+	return m
+}
+
+func (p *parser) init(fset *token.FileSet, filename string, src []byte, mode uint) {
+	p.file = fset.AddFile(filename, fset.Base(), len(src))
+	p.scanner.Init(p.file, src, p, scannerMode(mode))
+
+	p.mode = mode
+	p.trace = mode&Trace != 0 // for convenience (p.trace is used frequently)
+
+	p.next()
+
+	// set up the pkgScope here (as opposed to in parseFile) because
+	// there are other parser entry points (ParseExpr, etc.)
+	p.openScope()
+	p.pkgScope = p.topScope
+
+	// for the same reason, set up a label scope
+	p.openLabelScope()
+}
+
+// ----------------------------------------------------------------------------
+// Scoping support
+
+func (p *parser) openScope() {
+	p.topScope = ast.NewScope(p.topScope)
+}
+
+func (p *parser) closeScope() {
+	p.topScope = p.topScope.Outer
+}
+
+func (p *parser) openLabelScope() {
+	p.labelScope = ast.NewScope(p.labelScope)
+	p.targetStack = append(p.targetStack, nil)
+}
+
+func (p *parser) closeLabelScope() {
+	// resolve labels
+	n := len(p.targetStack) - 1
+	scope := p.labelScope
+	for _, ident := range p.targetStack[n] {
+		ident.Obj = scope.Lookup(ident.Name)
+		if ident.Obj == nil && p.mode&DeclarationErrors != 0 {
+			p.error(ident.Pos(), fmt.Sprintf("label %s undefined", ident.Name))
+		}
+	}
+	// pop label scope
+	p.targetStack = p.targetStack[0:n]
+	p.labelScope = p.labelScope.Outer
+}
+
+func (p *parser) declare(decl interface{}, scope *ast.Scope, kind ast.ObjKind, idents ...*ast.Ident) {
+	for _, ident := range idents {
+		assert(ident.Obj == nil, "identifier already declared or resolved")
+		if ident.Name != "_" {
+			obj := ast.NewObj(kind, ident.Name)
+			// remember the corresponding declaration for redeclaration
+			// errors and global variable resolution/typechecking phase
+			obj.Decl = decl
+			if alt := scope.Insert(obj); alt != nil && p.mode&DeclarationErrors != 0 {
+				prevDecl := ""
+				if pos := alt.Pos(); pos.IsValid() {
+					prevDecl = fmt.Sprintf("\n\tprevious declaration at %s", p.file.Position(pos))
+				}
+				p.error(ident.Pos(), fmt.Sprintf("%s redeclared in this block%s", ident.Name, prevDecl))
+			}
+			ident.Obj = obj
+		}
+	}
+}
+
+func (p *parser) shortVarDecl(idents []*ast.Ident) {
+	// Go spec: A short variable declaration may redeclare variables
+	// provided they were originally declared in the same block with
+	// the same type, and at least one of the non-blank variables is new.
+	n := 0 // number of new variables
+	for _, ident := range idents {
+		assert(ident.Obj == nil, "identifier already declared or resolved")
+		if ident.Name != "_" {
+			obj := ast.NewObj(ast.Var, ident.Name)
+			// short var declarations cannot have redeclaration errors
+			// and are not global => no need to remember the respective
+			// declaration
+			alt := p.topScope.Insert(obj)
+			if alt == nil {
+				n++ // new declaration
+				alt = obj
+			}
+			ident.Obj = alt
+		}
+	}
+	if n == 0 && p.mode&DeclarationErrors != 0 {
+		p.error(idents[0].Pos(), "no new variables on left side of :=")
+	}
+}
+
+// The unresolved object is a sentinel to mark identifiers that have been added
+// to the list of unresolved identifiers. The sentinel is only used for verifying
+// internal consistency.
+var unresolved = new(ast.Object)
+
+func (p *parser) resolve(x ast.Expr) {
+	// nothing to do if x is not an identifier or the blank identifier
+	ident, _ := x.(*ast.Ident)
+	if ident == nil {
+		return
+	}
+	assert(ident.Obj == nil, "identifier already declared or resolved")
+	if ident.Name == "_" {
+		return
+	}
+	// try to resolve the identifier
+	for s := p.topScope; s != nil; s = s.Outer {
+		if obj := s.Lookup(ident.Name); obj != nil {
+			ident.Obj = obj
+			return
+		}
+	}
+	// all local scopes are known, so any unresolved identifier
+	// must be found either in the file scope, package scope
+	// (perhaps in another file), or universe scope --- collect
+	// them so that they can be resolved later
+	ident.Obj = unresolved
+	p.unresolved = append(p.unresolved, ident)
+}
+
+// ----------------------------------------------------------------------------
+// Parsing support
+
+func (p *parser) printTrace(a ...interface{}) {
+	const dots = ". . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . " +
+		". . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . "
+	const n = uint(len(dots))
+	pos := p.file.Position(p.pos)
+	fmt.Printf("%5d:%3d: ", pos.Line, pos.Column)
+	i := 2 * p.indent
+	for ; i > n; i -= n {
+		fmt.Print(dots)
+	}
+	fmt.Print(dots[0:i])
+	fmt.Println(a...)
+}
+
+func trace(p *parser, msg string) *parser {
+	p.printTrace(msg, "(")
+	p.indent++
+	return p
+}
+
+// Usage pattern: defer un(trace(p, "..."));
+func un(p *parser) {
+	p.indent--
+	p.printTrace(")")
+}
+
+// Advance to the next token.
+func (p *parser) next0() {
+	// Because of one-token look-ahead, print the previous token
+	// when tracing as it provides a more readable output. The
+	// very first token (!p.pos.IsValid()) is not initialized
+	// (it is token.ILLEGAL), so don't print it .
+	if p.trace && p.pos.IsValid() {
+		s := p.tok.String()
+		switch {
+		case p.tok.IsLiteral():
+			p.printTrace(s, p.lit)
+		case p.tok.IsOperator(), p.tok.IsKeyword():
+			p.printTrace("\"" + s + "\"")
+		default:
+			p.printTrace(s)
+		}
+	}
+
+	p.pos, p.tok, p.lit = p.scanner.Scan()
+}
+
+// Consume a comment and return it and the line on which it ends.
+func (p *parser) consumeComment() (comment *ast.Comment, endline int) {
+	// /*-style comments may end on a different line than where they start.
+	// Scan the comment for '\n' chars and adjust endline accordingly.
+	endline = p.file.Line(p.pos)
+	if p.lit[1] == '*' {
+		// don't use range here - no need to decode Unicode code points
+		for i := 0; i < len(p.lit); i++ {
+			if p.lit[i] == '\n' {
+				endline++
+			}
+		}
+	}
+
+	comment = &ast.Comment{p.pos, p.lit}
+	p.next0()
+
+	return
+}
+
+// Consume a group of adjacent comments, add it to the parser's
+// comments list, and return it together with the line at which
+// the last comment in the group ends. An empty line or non-comment
+// token terminates a comment group.
+//
+func (p *parser) consumeCommentGroup() (comments *ast.CommentGroup, endline int) {
+	var list []*ast.Comment
+	endline = p.file.Line(p.pos)
+	for p.tok == token.COMMENT && endline+1 >= p.file.Line(p.pos) {
+		var comment *ast.Comment
+		comment, endline = p.consumeComment()
+		list = append(list, comment)
+	}
+
+	// add comment group to the comments list
+	comments = &ast.CommentGroup{list}
+	p.comments = append(p.comments, comments)
+
+	return
+}
+
+// Advance to the next non-comment token. In the process, collect
+// any comment groups encountered, and remember the last lead and
+// and line comments.
+//
+// A lead comment is a comment group that starts and ends in a
+// line without any other tokens and that is followed by a non-comment
+// token on the line immediately after the comment group.
+//
+// A line comment is a comment group that follows a non-comment
+// token on the same line, and that has no tokens after it on the line
+// where it ends.
+//
+// Lead and line comments may be considered documentation that is
+// stored in the AST.
+//
+func (p *parser) next() {
+	p.leadComment = nil
+	p.lineComment = nil
+	line := p.file.Line(p.pos) // current line
+	p.next0()
+
+	if p.tok == token.COMMENT {
+		var comment *ast.CommentGroup
+		var endline int
+
+		if p.file.Line(p.pos) == line {
+			// The comment is on same line as the previous token; it
+			// cannot be a lead comment but may be a line comment.
+			comment, endline = p.consumeCommentGroup()
+			if p.file.Line(p.pos) != endline {
+				// The next token is on a different line, thus
+				// the last comment group is a line comment.
+				p.lineComment = comment
+			}
+		}
+
+		// consume successor comments, if any
+		endline = -1
+		for p.tok == token.COMMENT {
+			comment, endline = p.consumeCommentGroup()
+		}
+
+		if endline+1 == p.file.Line(p.pos) {
+			// The next token is following on the line immediately after the
+			// comment group, thus the last comment group is a lead comment.
+			p.leadComment = comment
+		}
+	}
+}
+
+func (p *parser) error(pos token.Pos, msg string) {
+	p.Error(p.file.Position(pos), msg)
+}
+
+func (p *parser) errorExpected(pos token.Pos, msg string) {
+	msg = "expected " + msg
+	if pos == p.pos {
+		// the error happened at the current position;
+		// make the error message more specific
+		if p.tok == token.SEMICOLON && p.lit[0] == '\n' {
+			msg += ", found newline"
+		} else {
+			msg += ", found '" + p.tok.String() + "'"
+			if p.tok.IsLiteral() {
+				msg += " " + p.lit
+			}
+		}
+	}
+	p.error(pos, msg)
+}
+
+func (p *parser) expect(tok token.Token) token.Pos {
+	pos := p.pos
+	if p.tok != tok {
+		p.errorExpected(pos, "'"+tok.String()+"'")
+	}
+	p.next() // make progress
+	return pos
+}
+
+func (p *parser) expectSemi() {
+	if p.tok != token.RPAREN && p.tok != token.RBRACE {
+		p.expect(token.SEMICOLON)
+	}
+}
+
+func assert(cond bool, msg string) {
+	if !cond {
+		panic("go/parser internal error: " + msg)
+	}
+}
+
+// ----------------------------------------------------------------------------
+// Identifiers
+
+func (p *parser) parseIdent() *ast.Ident {
+	pos := p.pos
+	name := "_"
+	if p.tok == token.IDENT {
+		name = p.lit
+		p.next()
+	} else {
+		p.expect(token.IDENT) // use expect() error handling
+	}
+	return &ast.Ident{pos, name, nil}
+}
+
+func (p *parser) parseIdentList() (list []*ast.Ident) {
+	if p.trace {
+		defer un(trace(p, "IdentList"))
+	}
+
+	list = append(list, p.parseIdent())
+	for p.tok == token.COMMA {
+		p.next()
+		list = append(list, p.parseIdent())
+	}
+
+	return
+}
+
+// ----------------------------------------------------------------------------
+// Common productions
+
+// If lhs is set, result list elements which are identifiers are not resolved.
+func (p *parser) parseExprList(lhs bool) (list []ast.Expr) {
+	if p.trace {
+		defer un(trace(p, "ExpressionList"))
+	}
+
+	list = append(list, p.parseExpr(lhs))
+	for p.tok == token.COMMA {
+		p.next()
+		list = append(list, p.parseExpr(lhs))
+	}
+
+	return
+}
+
+func (p *parser) parseLhsList() []ast.Expr {
+	list := p.parseExprList(true)
+	switch p.tok {
+	case token.DEFINE:
+		// lhs of a short variable declaration
+		p.shortVarDecl(p.makeIdentList(list))
+	case token.COLON:
+		// lhs of a label declaration or a communication clause of a select
+		// statement (parseLhsList is not called when parsing the case clause
+		// of a switch statement):
+		// - labels are declared by the caller of parseLhsList
+		// - for communication clauses, if there is a stand-alone identifier
+		//   followed by a colon, we have a syntax error; there is no need
+		//   to resolve the identifier in that case
+	default:
+		// identifiers must be declared elsewhere
+		for _, x := range list {
+			p.resolve(x)
+		}
+	}
+	return list
+}
+
+func (p *parser) parseRhsList() []ast.Expr {
+	return p.parseExprList(false)
+}
+
+// ----------------------------------------------------------------------------
+// Types
+
+func (p *parser) parseType() ast.Expr {
+	if p.trace {
+		defer un(trace(p, "Type"))
+	}
+
+	typ := p.tryType()
+
+	if typ == nil {
+		pos := p.pos
+		p.errorExpected(pos, "type")
+		p.next() // make progress
+		return &ast.BadExpr{pos, p.pos}
+	}
+
+	return typ
+}
+
+// If the result is an identifier, it is not resolved.
+func (p *parser) parseTypeName() ast.Expr {
+	if p.trace {
+		defer un(trace(p, "TypeName"))
+	}
+
+	ident := p.parseIdent()
+	// don't resolve ident yet - it may be a parameter or field name
+
+	if p.tok == token.PERIOD {
+		// ident is a package name
+		p.next()
+		p.resolve(ident)
+		sel := p.parseIdent()
+		return &ast.SelectorExpr{ident, sel}
+	}
+
+	return ident
+}
+
+func (p *parser) parseArrayType(ellipsisOk bool) ast.Expr {
+	if p.trace {
+		defer un(trace(p, "ArrayType"))
+	}
+
+	lbrack := p.expect(token.LBRACK)
+	var len ast.Expr
+	if ellipsisOk && p.tok == token.ELLIPSIS {
+		len = &ast.Ellipsis{p.pos, nil}
+		p.next()
+	} else if p.tok != token.RBRACK {
+		len = p.parseRhs()
+	}
+	p.expect(token.RBRACK)
+	elt := p.parseType()
+
+	return &ast.ArrayType{lbrack, len, elt}
+}
+
+func (p *parser) makeIdentList(list []ast.Expr) []*ast.Ident {
+	idents := make([]*ast.Ident, len(list))
+	for i, x := range list {
+		ident, isIdent := x.(*ast.Ident)
+		if !isIdent {
+			pos := x.(ast.Expr).Pos()
+			p.errorExpected(pos, "identifier")
+			ident = &ast.Ident{pos, "_", nil}
+		}
+		idents[i] = ident
+	}
+	return idents
+}
+
+func (p *parser) parseFieldDecl(scope *ast.Scope) *ast.Field {
+	if p.trace {
+		defer un(trace(p, "FieldDecl"))
+	}
+
+	doc := p.leadComment
+
+	// fields
+	list, typ := p.parseVarList(false)
+
+	// optional tag
+	var tag *ast.BasicLit
+	if p.tok == token.STRING {
+		tag = &ast.BasicLit{p.pos, p.tok, p.lit}
+		p.next()
+	}
+
+	// analyze case
+	var idents []*ast.Ident
+	if typ != nil {
+		// IdentifierList Type
+		idents = p.makeIdentList(list)
+	} else {
+		// ["*"] TypeName (AnonymousField)
+		typ = list[0] // we always have at least one element
+		p.resolve(typ)
+		if n := len(list); n > 1 || !isTypeName(deref(typ)) {
+			pos := typ.Pos()
+			p.errorExpected(pos, "anonymous field")
+			typ = &ast.BadExpr{pos, list[n-1].End()}
+		}
+	}
+
+	p.expectSemi() // call before accessing p.linecomment
+
+	field := &ast.Field{doc, idents, typ, tag, p.lineComment}
+	p.declare(field, scope, ast.Var, idents...)
+
+	return field
+}
+
+func (p *parser) parseStructType() *ast.StructType {
+	if p.trace {
+		defer un(trace(p, "StructType"))
+	}
+
+	pos := p.expect(token.STRUCT)
+	lbrace := p.expect(token.LBRACE)
+	scope := ast.NewScope(nil) // struct scope
+	var list []*ast.Field
+	for p.tok == token.IDENT || p.tok == token.MUL || p.tok == token.LPAREN {
+		// a field declaration cannot start with a '(' but we accept
+		// it here for more robust parsing and better error messages
+		// (parseFieldDecl will check and complain if necessary)
+		list = append(list, p.parseFieldDecl(scope))
+	}
+	rbrace := p.expect(token.RBRACE)
+
+	// TODO(gri): store struct scope in AST
+	return &ast.StructType{pos, &ast.FieldList{lbrace, list, rbrace}, false}
+}
+
+func (p *parser) parsePointerType() *ast.StarExpr {
+	if p.trace {
+		defer un(trace(p, "PointerType"))
+	}
+
+	star := p.expect(token.MUL)
+	base := p.parseType()
+
+	return &ast.StarExpr{star, base}
+}
+
+func (p *parser) tryVarType(isParam bool) ast.Expr {
+	if isParam && p.tok == token.ELLIPSIS {
+		pos := p.pos
+		p.next()
+		typ := p.tryIdentOrType(isParam) // don't use parseType so we can provide better error message
+		if typ == nil {
+			p.error(pos, "'...' parameter is missing type")
+			typ = &ast.BadExpr{pos, p.pos}
+		}
+		if p.tok != token.RPAREN {
+			p.error(pos, "can use '...' with last parameter type only")
+		}
+		return &ast.Ellipsis{pos, typ}
+	}
+	return p.tryIdentOrType(false)
+}
+
+func (p *parser) parseVarType(isParam bool) ast.Expr {
+	typ := p.tryVarType(isParam)
+	if typ == nil {
+		pos := p.pos
+		p.errorExpected(pos, "type")
+		p.next() // make progress
+		typ = &ast.BadExpr{pos, p.pos}
+	}
+	return typ
+}
+
+func (p *parser) parseVarList(isParam bool) (list []ast.Expr, typ ast.Expr) {
+	if p.trace {
+		defer un(trace(p, "VarList"))
+	}
+
+	// a list of identifiers looks like a list of type names
+	for {
+		// parseVarType accepts any type (including parenthesized ones)
+		// even though the syntax does not permit them here: we
+		// accept them all for more robust parsing and complain
+		// afterwards
+		list = append(list, p.parseVarType(isParam))
+		if p.tok != token.COMMA {
+			break
+		}
+		p.next()
+	}
+
+	// if we had a list of identifiers, it must be followed by a type
+	typ = p.tryVarType(isParam)
+	if typ != nil {
+		p.resolve(typ)
+	}
+
+	return
+}
+
+func (p *parser) parseParameterList(scope *ast.Scope, ellipsisOk bool) (params []*ast.Field) {
+	if p.trace {
+		defer un(trace(p, "ParameterList"))
+	}
+
+	list, typ := p.parseVarList(ellipsisOk)
+	if typ != nil {
+		// IdentifierList Type
+		idents := p.makeIdentList(list)
+		field := &ast.Field{nil, idents, typ, nil, nil}
+		params = append(params, field)
+		// Go spec: The scope of an identifier denoting a function
+		// parameter or result variable is the function body.
+		p.declare(field, scope, ast.Var, idents...)
+		if p.tok == token.COMMA {
+			p.next()
+		}
+
+		for p.tok != token.RPAREN && p.tok != token.EOF {
+			idents := p.parseIdentList()
+			typ := p.parseVarType(ellipsisOk)
+			field := &ast.Field{nil, idents, typ, nil, nil}
+			params = append(params, field)
+			// Go spec: The scope of an identifier denoting a function
+			// parameter or result variable is the function body.
+			p.declare(field, scope, ast.Var, idents...)
+			if p.tok != token.COMMA {
+				break
+			}
+			p.next()
+		}
+
+	} else {
+		// Type { "," Type } (anonymous parameters)
+		params = make([]*ast.Field, len(list))
+		for i, x := range list {
+			p.resolve(x)
+			params[i] = &ast.Field{Type: x}
+		}
+	}
+
+	return
+}
+
+func (p *parser) parseParameters(scope *ast.Scope, ellipsisOk bool) *ast.FieldList {
+	if p.trace {
+		defer un(trace(p, "Parameters"))
+	}
+
+	var params []*ast.Field
+	lparen := p.expect(token.LPAREN)
+	if p.tok != token.RPAREN {
+		params = p.parseParameterList(scope, ellipsisOk)
+	}
+	rparen := p.expect(token.RPAREN)
+
+	return &ast.FieldList{lparen, params, rparen}
+}
+
+func (p *parser) parseResult(scope *ast.Scope) *ast.FieldList {
+	if p.trace {
+		defer un(trace(p, "Result"))
+	}
+
+	if p.tok == token.LPAREN {
+		return p.parseParameters(scope, false)
+	}
+
+	typ := p.tryType()
+	if typ != nil {
+		list := make([]*ast.Field, 1)
+		list[0] = &ast.Field{Type: typ}
+		return &ast.FieldList{List: list}
+	}
+
+	return nil
+}
+
+func (p *parser) parseSignature(scope *ast.Scope) (params, results *ast.FieldList) {
+	if p.trace {
+		defer un(trace(p, "Signature"))
+	}
+
+	params = p.parseParameters(scope, true)
+	results = p.parseResult(scope)
+
+	return
+}
+
+func (p *parser) parseFuncType() (*ast.FuncType, *ast.Scope) {
+	if p.trace {
+		defer un(trace(p, "FuncType"))
+	}
+
+	pos := p.expect(token.FUNC)
+	scope := ast.NewScope(p.topScope) // function scope
+	params, results := p.parseSignature(scope)
+
+	return &ast.FuncType{pos, params, results}, scope
+}
+
+func (p *parser) parseMethodSpec(scope *ast.Scope) *ast.Field {
+	if p.trace {
+		defer un(trace(p, "MethodSpec"))
+	}
+
+	doc := p.leadComment
+	var idents []*ast.Ident
+	var typ ast.Expr
+	x := p.parseTypeName()
+	if ident, isIdent := x.(*ast.Ident); isIdent && p.tok == token.LPAREN {
+		// method
+		idents = []*ast.Ident{ident}
+		scope := ast.NewScope(nil) // method scope
+		params, results := p.parseSignature(scope)
+		typ = &ast.FuncType{token.NoPos, params, results}
+	} else {
+		// embedded interface
+		typ = x
+	}
+	p.expectSemi() // call before accessing p.linecomment
+
+	spec := &ast.Field{doc, idents, typ, nil, p.lineComment}
+	p.declare(spec, scope, ast.Fun, idents...)
+
+	return spec
+}
+
+func (p *parser) parseInterfaceType() *ast.InterfaceType {
+	if p.trace {
+		defer un(trace(p, "InterfaceType"))
+	}
+
+	pos := p.expect(token.INTERFACE)
+	lbrace := p.expect(token.LBRACE)
+	scope := ast.NewScope(nil) // interface scope
+	var list []*ast.Field
+	for p.tok == token.IDENT {
+		list = append(list, p.parseMethodSpec(scope))
+	}
+	rbrace := p.expect(token.RBRACE)
+
+	// TODO(gri): store interface scope in AST
+	return &ast.InterfaceType{pos, &ast.FieldList{lbrace, list, rbrace}, false}
+}
+
+func (p *parser) parseMapType() *ast.MapType {
+	if p.trace {
+		defer un(trace(p, "MapType"))
+	}
+
+	pos := p.expect(token.MAP)
+	p.expect(token.LBRACK)
+	key := p.parseType()
+	p.expect(token.RBRACK)
+	value := p.parseType()
+
+	return &ast.MapType{pos, key, value}
+}
+
+func (p *parser) parseChanType() *ast.ChanType {
+	if p.trace {
+		defer un(trace(p, "ChanType"))
+	}
+
+	pos := p.pos
+	dir := ast.SEND | ast.RECV
+	if p.tok == token.CHAN {
+		p.next()
+		if p.tok == token.ARROW {
+			p.next()
+			dir = ast.SEND
+		}
+	} else {
+		p.expect(token.ARROW)
+		p.expect(token.CHAN)
+		dir = ast.RECV
+	}
+	value := p.parseType()
+
+	return &ast.ChanType{pos, dir, value}
+}
+
+// If the result is an identifier, it is not resolved.
+func (p *parser) tryIdentOrType(ellipsisOk bool) ast.Expr {
+	switch p.tok {
+	case token.IDENT:
+		return p.parseTypeName()
+	case token.LBRACK:
+		return p.parseArrayType(ellipsisOk)
+	case token.STRUCT:
+		return p.parseStructType()
+	case token.MUL:
+		return p.parsePointerType()
+	case token.FUNC:
+		typ, _ := p.parseFuncType()
+		return typ
+	case token.INTERFACE:
+		return p.parseInterfaceType()
+	case token.MAP:
+		return p.parseMapType()
+	case token.CHAN, token.ARROW:
+		return p.parseChanType()
+	case token.LPAREN:
+		lparen := p.pos
+		p.next()
+		typ := p.parseType()
+		rparen := p.expect(token.RPAREN)
+		return &ast.ParenExpr{lparen, typ, rparen}
+	}
+
+	// no type found
+	return nil
+}
+
+func (p *parser) tryType() ast.Expr {
+	typ := p.tryIdentOrType(false)
+	if typ != nil {
+		p.resolve(typ)
+	}
+	return typ
+}
+
+// ----------------------------------------------------------------------------
+// Blocks
+
+func (p *parser) parseStmtList() (list []ast.Stmt) {
+	if p.trace {
+		defer un(trace(p, "StatementList"))
+	}
+
+	for p.tok != token.CASE && p.tok != token.DEFAULT && p.tok != token.RBRACE && p.tok != token.EOF {
+		list = append(list, p.parseStmt())
+	}
+
+	return
+}
+
+func (p *parser) parseBody(scope *ast.Scope) *ast.BlockStmt {
+	if p.trace {
+		defer un(trace(p, "Body"))
+	}
+
+	lbrace := p.expect(token.LBRACE)
+	p.topScope = scope // open function scope
+	p.openLabelScope()
+	list := p.parseStmtList()
+	p.closeLabelScope()
+	p.closeScope()
+	rbrace := p.expect(token.RBRACE)
+
+	return &ast.BlockStmt{lbrace, list, rbrace}
+}
+
+func (p *parser) parseBlockStmt() *ast.BlockStmt {
+	if p.trace {
+		defer un(trace(p, "BlockStmt"))
+	}
+
+	lbrace := p.expect(token.LBRACE)
+	p.openScope()
+	list := p.parseStmtList()
+	p.closeScope()
+	rbrace := p.expect(token.RBRACE)
+
+	return &ast.BlockStmt{lbrace, list, rbrace}
+}
+
+// ----------------------------------------------------------------------------
+// Expressions
+
+func (p *parser) parseFuncTypeOrLit() ast.Expr {
+	if p.trace {
+		defer un(trace(p, "FuncTypeOrLit"))
+	}
+
+	typ, scope := p.parseFuncType()
+	if p.tok != token.LBRACE {
+		// function type only
+		return typ
+	}
+
+	p.exprLev++
+	body := p.parseBody(scope)
+	p.exprLev--
+
+	return &ast.FuncLit{typ, body}
+}
+
+// parseOperand may return an expression or a raw type (incl. array
+// types of the form [...]T. Callers must verify the result.
+// If lhs is set and the result is an identifier, it is not resolved.
+//
+func (p *parser) parseOperand(lhs bool) ast.Expr {
+	if p.trace {
+		defer un(trace(p, "Operand"))
+	}
+
+	switch p.tok {
+	case token.IDENT:
+		x := p.parseIdent()
+		if !lhs {
+			p.resolve(x)
+		}
+		return x
+
+	case token.INT, token.FLOAT, token.IMAG, token.CHAR, token.STRING:
+		x := &ast.BasicLit{p.pos, p.tok, p.lit}
+		p.next()
+		return x
+
+	case token.LPAREN:
+		lparen := p.pos
+		p.next()
+		p.exprLev++
+		x := p.parseRhs()
+		p.exprLev--
+		rparen := p.expect(token.RPAREN)
+		return &ast.ParenExpr{lparen, x, rparen}
+
+	case token.FUNC:
+		return p.parseFuncTypeOrLit()
+
+	default:
+		if typ := p.tryIdentOrType(true); typ != nil {
+			// could be type for composite literal or conversion
+			_, isIdent := typ.(*ast.Ident)
+			assert(!isIdent, "type cannot be identifier")
+			return typ
+		}
+	}
+
+	pos := p.pos
+	p.errorExpected(pos, "operand")
+	p.next() // make progress
+	return &ast.BadExpr{pos, p.pos}
+}
+
+func (p *parser) parseSelector(x ast.Expr) ast.Expr {
+	if p.trace {
+		defer un(trace(p, "Selector"))
+	}
+
+	sel := p.parseIdent()
+
+	return &ast.SelectorExpr{x, sel}
+}
+
+func (p *parser) parseTypeAssertion(x ast.Expr) ast.Expr {
+	if p.trace {
+		defer un(trace(p, "TypeAssertion"))
+	}
+
+	p.expect(token.LPAREN)
+	var typ ast.Expr
+	if p.tok == token.TYPE {
+		// type switch: typ == nil
+		p.next()
+	} else {
+		typ = p.parseType()
+	}
+	p.expect(token.RPAREN)
+
+	return &ast.TypeAssertExpr{x, typ}
+}
+
+func (p *parser) parseIndexOrSlice(x ast.Expr) ast.Expr {
+	if p.trace {
+		defer un(trace(p, "IndexOrSlice"))
+	}
+
+	lbrack := p.expect(token.LBRACK)
+	p.exprLev++
+	var low, high ast.Expr
+	isSlice := false
+	if p.tok != token.COLON {
+		low = p.parseRhs()
+	}
+	if p.tok == token.COLON {
+		isSlice = true
+		p.next()
+		if p.tok != token.RBRACK {
+			high = p.parseRhs()
+		}
+	}
+	p.exprLev--
+	rbrack := p.expect(token.RBRACK)
+
+	if isSlice {
+		return &ast.SliceExpr{x, lbrack, low, high, rbrack}
+	}
+	return &ast.IndexExpr{x, lbrack, low, rbrack}
+}
+
+func (p *parser) parseCallOrConversion(fun ast.Expr) *ast.CallExpr {
+	if p.trace {
+		defer un(trace(p, "CallOrConversion"))
+	}
+
+	lparen := p.expect(token.LPAREN)
+	p.exprLev++
+	var list []ast.Expr
+	var ellipsis token.Pos
+	for p.tok != token.RPAREN && p.tok != token.EOF && !ellipsis.IsValid() {
+		list = append(list, p.parseRhs())
+		if p.tok == token.ELLIPSIS {
+			ellipsis = p.pos
+			p.next()
+		}
+		if p.tok != token.COMMA {
+			break
+		}
+		p.next()
+	}
+	p.exprLev--
+	rparen := p.expect(token.RPAREN)
+
+	return &ast.CallExpr{fun, lparen, list, ellipsis, rparen}
+}
+
+func (p *parser) parseElement(keyOk bool) ast.Expr {
+	if p.trace {
+		defer un(trace(p, "Element"))
+	}
+
+	if p.tok == token.LBRACE {
+		return p.parseLiteralValue(nil)
+	}
+
+	x := p.parseExpr(keyOk) // don't resolve if map key
+	if keyOk {
+		if p.tok == token.COLON {
+			colon := p.pos
+			p.next()
+			return &ast.KeyValueExpr{x, colon, p.parseElement(false)}
+		}
+		p.resolve(x) // not a map key
+	}
+
+	return x
+}
+
+func (p *parser) parseElementList() (list []ast.Expr) {
+	if p.trace {
+		defer un(trace(p, "ElementList"))
+	}
+
+	for p.tok != token.RBRACE && p.tok != token.EOF {
+		list = append(list, p.parseElement(true))
+		if p.tok != token.COMMA {
+			break
+		}
+		p.next()
+	}
+
+	return
+}
+
+func (p *parser) parseLiteralValue(typ ast.Expr) ast.Expr {
+	if p.trace {
+		defer un(trace(p, "LiteralValue"))
+	}
+
+	lbrace := p.expect(token.LBRACE)
+	var elts []ast.Expr
+	p.exprLev++
+	if p.tok != token.RBRACE {
+		elts = p.parseElementList()
+	}
+	p.exprLev--
+	rbrace := p.expect(token.RBRACE)
+	return &ast.CompositeLit{typ, lbrace, elts, rbrace}
+}
+
+// checkExpr checks that x is an expression (and not a type).
+func (p *parser) checkExpr(x ast.Expr) ast.Expr {
+	switch t := unparen(x).(type) {
+	case *ast.BadExpr:
+	case *ast.Ident:
+	case *ast.BasicLit:
+	case *ast.FuncLit:
+	case *ast.CompositeLit:
+	case *ast.ParenExpr:
+		panic("unreachable")
+	case *ast.SelectorExpr:
+	case *ast.IndexExpr:
+	case *ast.SliceExpr:
+	case *ast.TypeAssertExpr:
+		if t.Type == nil {
+			// the form X.(type) is only allowed in type switch expressions
+			p.errorExpected(x.Pos(), "expression")
+			x = &ast.BadExpr{x.Pos(), x.End()}
+		}
+	case *ast.CallExpr:
+	case *ast.StarExpr:
+	case *ast.UnaryExpr:
+		if t.Op == token.RANGE {
+			// the range operator is only allowed at the top of a for statement
+			p.errorExpected(x.Pos(), "expression")
+			x = &ast.BadExpr{x.Pos(), x.End()}
+		}
+	case *ast.BinaryExpr:
+	default:
+		// all other nodes are not proper expressions
+		p.errorExpected(x.Pos(), "expression")
+		x = &ast.BadExpr{x.Pos(), x.End()}
+	}
+	return x
+}
+
+// isTypeName returns true iff x is a (qualified) TypeName.
+func isTypeName(x ast.Expr) bool {
+	switch t := x.(type) {
+	case *ast.BadExpr:
+	case *ast.Ident:
+	case *ast.SelectorExpr:
+		_, isIdent := t.X.(*ast.Ident)
+		return isIdent
+	default:
+		return false // all other nodes are not type names
+	}
+	return true
+}
+
+// isLiteralType returns true iff x is a legal composite literal type.
+func isLiteralType(x ast.Expr) bool {
+	switch t := x.(type) {
+	case *ast.BadExpr:
+	case *ast.Ident:
+	case *ast.SelectorExpr:
+		_, isIdent := t.X.(*ast.Ident)
+		return isIdent
+	case *ast.ArrayType:
+	case *ast.StructType:
+	case *ast.MapType:
+	default:
+		return false // all other nodes are not legal composite literal types
+	}
+	return true
+}
+
+// If x is of the form *T, deref returns T, otherwise it returns x.
+func deref(x ast.Expr) ast.Expr {
+	if p, isPtr := x.(*ast.StarExpr); isPtr {
+		x = p.X
+	}
+	return x
+}
+
+// If x is of the form (T), unparen returns unparen(T), otherwise it returns x.
+func unparen(x ast.Expr) ast.Expr {
+	if p, isParen := x.(*ast.ParenExpr); isParen {
+		x = unparen(p.X)
+	}
+	return x
+}
+
+// checkExprOrType checks that x is an expression or a type
+// (and not a raw type such as [...]T).
+//
+func (p *parser) checkExprOrType(x ast.Expr) ast.Expr {
+	switch t := unparen(x).(type) {
+	case *ast.ParenExpr:
+		panic("unreachable")
+	case *ast.UnaryExpr:
+		if t.Op == token.RANGE {
+			// the range operator is only allowed at the top of a for statement
+			p.errorExpected(x.Pos(), "expression")
+			x = &ast.BadExpr{x.Pos(), x.End()}
+		}
+	case *ast.ArrayType:
+		if len, isEllipsis := t.Len.(*ast.Ellipsis); isEllipsis {
+			p.error(len.Pos(), "expected array length, found '...'")
+			x = &ast.BadExpr{x.Pos(), x.End()}
+		}
+	}
+
+	// all other nodes are expressions or types
+	return x
+}
+
+// If lhs is set and the result is an identifier, it is not resolved.
+func (p *parser) parsePrimaryExpr(lhs bool) ast.Expr {
+	if p.trace {
+		defer un(trace(p, "PrimaryExpr"))
+	}
+
+	x := p.parseOperand(lhs)
+L:
+	for {
+		switch p.tok {
+		case token.PERIOD:
+			p.next()
+			if lhs {
+				p.resolve(x)
+			}
+			switch p.tok {
+			case token.IDENT:
+				x = p.parseSelector(p.checkExpr(x))
+			case token.LPAREN:
+				x = p.parseTypeAssertion(p.checkExpr(x))
+			default:
+				pos := p.pos
+				p.next() // make progress
+				p.errorExpected(pos, "selector or type assertion")
+				x = &ast.BadExpr{pos, p.pos}
+			}
+		case token.LBRACK:
+			if lhs {
+				p.resolve(x)
+			}
+			x = p.parseIndexOrSlice(p.checkExpr(x))
+		case token.LPAREN:
+			if lhs {
+				p.resolve(x)
+			}
+			x = p.parseCallOrConversion(p.checkExprOrType(x))
+		case token.LBRACE:
+			if isLiteralType(x) && (p.exprLev >= 0 || !isTypeName(x)) {
+				if lhs {
+					p.resolve(x)
+				}
+				x = p.parseLiteralValue(x)
+			} else {
+				break L
+			}
+		default:
+			break L
+		}
+		lhs = false // no need to try to resolve again
+	}
+
+	return x
+}
+
+// If lhs is set and the result is an identifier, it is not resolved.
+func (p *parser) parseUnaryExpr(lhs bool) ast.Expr {
+	if p.trace {
+		defer un(trace(p, "UnaryExpr"))
+	}
+
+	switch p.tok {
+	case token.ADD, token.SUB, token.NOT, token.XOR, token.AND, token.RANGE:
+		pos, op := p.pos, p.tok
+		p.next()
+		x := p.parseUnaryExpr(false)
+		return &ast.UnaryExpr{pos, op, p.checkExpr(x)}
+
+	case token.ARROW:
+		// channel type or receive expression
+		pos := p.pos
+		p.next()
+		if p.tok == token.CHAN {
+			p.next()
+			value := p.parseType()
+			return &ast.ChanType{pos, ast.RECV, value}
+		}
+
+		x := p.parseUnaryExpr(false)
+		return &ast.UnaryExpr{pos, token.ARROW, p.checkExpr(x)}
+
+	case token.MUL:
+		// pointer type or unary "*" expression
+		pos := p.pos
+		p.next()
+		x := p.parseUnaryExpr(false)
+		return &ast.StarExpr{pos, p.checkExprOrType(x)}
+	}
+
+	return p.parsePrimaryExpr(lhs)
+}
+
+// If lhs is set and the result is an identifier, it is not resolved.
+func (p *parser) parseBinaryExpr(lhs bool, prec1 int) ast.Expr {
+	if p.trace {
+		defer un(trace(p, "BinaryExpr"))
+	}
+
+	x := p.parseUnaryExpr(lhs)
+	for prec := p.tok.Precedence(); prec >= prec1; prec-- {
+		for p.tok.Precedence() == prec {
+			pos, op := p.pos, p.tok
+			p.next()
+			if lhs {
+				p.resolve(x)
+				lhs = false
+			}
+			y := p.parseBinaryExpr(false, prec+1)
+			x = &ast.BinaryExpr{p.checkExpr(x), pos, op, p.checkExpr(y)}
+		}
+	}
+
+	return x
+}
+
+// If lhs is set and the result is an identifier, it is not resolved.
+// TODO(gri): parseExpr may return a type or even a raw type ([..]int) -
+//            should reject when a type/raw type is obviously not allowed
+func (p *parser) parseExpr(lhs bool) ast.Expr {
+	if p.trace {
+		defer un(trace(p, "Expression"))
+	}
+
+	return p.parseBinaryExpr(lhs, token.LowestPrec+1)
+}
+
+func (p *parser) parseRhs() ast.Expr {
+	return p.parseExpr(false)
+}
+
+// ----------------------------------------------------------------------------
+// Statements
+
+func (p *parser) parseSimpleStmt(labelOk bool) ast.Stmt {
+	if p.trace {
+		defer un(trace(p, "SimpleStmt"))
+	}
+
+	x := p.parseLhsList()
+
+	switch p.tok {
+	case
+		token.DEFINE, token.ASSIGN, token.ADD_ASSIGN,
+		token.SUB_ASSIGN, token.MUL_ASSIGN, token.QUO_ASSIGN,
+		token.REM_ASSIGN, token.AND_ASSIGN, token.OR_ASSIGN,
+		token.XOR_ASSIGN, token.SHL_ASSIGN, token.SHR_ASSIGN, token.AND_NOT_ASSIGN:
+		// assignment statement
+		pos, tok := p.pos, p.tok
+		p.next()
+		y := p.parseRhsList()
+		return &ast.AssignStmt{x, pos, tok, y}
+	}
+
+	if len(x) > 1 {
+		p.errorExpected(x[0].Pos(), "1 expression")
+		// continue with first expression
+	}
+
+	switch p.tok {
+	case token.COLON:
+		// labeled statement
+		colon := p.pos
+		p.next()
+		if label, isIdent := x[0].(*ast.Ident); labelOk && isIdent {
+			// Go spec: The scope of a label is the body of the function
+			// in which it is declared and excludes the body of any nested
+			// function.
+			stmt := &ast.LabeledStmt{label, colon, p.parseStmt()}
+			p.declare(stmt, p.labelScope, ast.Lbl, label)
+			return stmt
+		}
+		p.error(x[0].Pos(), "illegal label declaration")
+		return &ast.BadStmt{x[0].Pos(), colon + 1}
+
+	case token.ARROW:
+		// send statement
+		arrow := p.pos
+		p.next() // consume "<-"
+		y := p.parseRhs()
+		return &ast.SendStmt{x[0], arrow, y}
+
+	case token.INC, token.DEC:
+		// increment or decrement
+		s := &ast.IncDecStmt{x[0], p.pos, p.tok}
+		p.next() // consume "++" or "--"
+		return s
+	}
+
+	// expression
+	return &ast.ExprStmt{x[0]}
+}
+
+func (p *parser) parseCallExpr() *ast.CallExpr {
+	x := p.parseRhs()
+	if call, isCall := x.(*ast.CallExpr); isCall {
+		return call
+	}
+	p.errorExpected(x.Pos(), "function/method call")
+	return nil
+}
+
+func (p *parser) parseGoStmt() ast.Stmt {
+	if p.trace {
+		defer un(trace(p, "GoStmt"))
+	}
+
+	pos := p.expect(token.GO)
+	call := p.parseCallExpr()
+	p.expectSemi()
+	if call == nil {
+		return &ast.BadStmt{pos, pos + 2} // len("go")
+	}
+
+	return &ast.GoStmt{pos, call}
+}
+
+func (p *parser) parseDeferStmt() ast.Stmt {
+	if p.trace {
+		defer un(trace(p, "DeferStmt"))
+	}
+
+	pos := p.expect(token.DEFER)
+	call := p.parseCallExpr()
+	p.expectSemi()
+	if call == nil {
+		return &ast.BadStmt{pos, pos + 5} // len("defer")
+	}
+
+	return &ast.DeferStmt{pos, call}
+}
+
+func (p *parser) parseReturnStmt() *ast.ReturnStmt {
+	if p.trace {
+		defer un(trace(p, "ReturnStmt"))
+	}
+
+	pos := p.pos
+	p.expect(token.RETURN)
+	var x []ast.Expr
+	if p.tok != token.SEMICOLON && p.tok != token.RBRACE {
+		x = p.parseRhsList()
+	}
+	p.expectSemi()
+
+	return &ast.ReturnStmt{pos, x}
+}
+
+func (p *parser) parseBranchStmt(tok token.Token) *ast.BranchStmt {
+	if p.trace {
+		defer un(trace(p, "BranchStmt"))
+	}
+
+	pos := p.expect(tok)
+	var label *ast.Ident
+	if tok != token.FALLTHROUGH && p.tok == token.IDENT {
+		label = p.parseIdent()
+		// add to list of unresolved targets
+		n := len(p.targetStack) - 1
+		p.targetStack[n] = append(p.targetStack[n], label)
+	}
+	p.expectSemi()
+
+	return &ast.BranchStmt{pos, tok, label}
+}
+
+func (p *parser) makeExpr(s ast.Stmt) ast.Expr {
+	if s == nil {
+		return nil
+	}
+	if es, isExpr := s.(*ast.ExprStmt); isExpr {
+		return p.checkExpr(es.X)
+	}
+	p.error(s.Pos(), "expected condition, found simple statement")
+	return &ast.BadExpr{s.Pos(), s.End()}
+}
+
+func (p *parser) parseIfStmt() *ast.IfStmt {
+	if p.trace {
+		defer un(trace(p, "IfStmt"))
+	}
+
+	pos := p.expect(token.IF)
+	p.openScope()
+	defer p.closeScope()
+
+	var s ast.Stmt
+	var x ast.Expr
+	{
+		prevLev := p.exprLev
+		p.exprLev = -1
+		if p.tok == token.SEMICOLON {
+			p.next()
+			x = p.parseRhs()
+		} else {
+			s = p.parseSimpleStmt(false)
+			if p.tok == token.SEMICOLON {
+				p.next()
+				x = p.parseRhs()
+			} else {
+				x = p.makeExpr(s)
+				s = nil
+			}
+		}
+		p.exprLev = prevLev
+	}
+
+	body := p.parseBlockStmt()
+	var else_ ast.Stmt
+	if p.tok == token.ELSE {
+		p.next()
+		else_ = p.parseStmt()
+	} else {
+		p.expectSemi()
+	}
+
+	return &ast.IfStmt{pos, s, x, body, else_}
+}
+
+func (p *parser) parseTypeList() (list []ast.Expr) {
+	if p.trace {
+		defer un(trace(p, "TypeList"))
+	}
+
+	list = append(list, p.parseType())
+	for p.tok == token.COMMA {
+		p.next()
+		list = append(list, p.parseType())
+	}
+
+	return
+}
+
+func (p *parser) parseCaseClause(exprSwitch bool) *ast.CaseClause {
+	if p.trace {
+		defer un(trace(p, "CaseClause"))
+	}
+
+	pos := p.pos
+	var list []ast.Expr
+	if p.tok == token.CASE {
+		p.next()
+		if exprSwitch {
+			list = p.parseRhsList()
+		} else {
+			list = p.parseTypeList()
+		}
+	} else {
+		p.expect(token.DEFAULT)
+	}
+
+	colon := p.expect(token.COLON)
+	p.openScope()
+	body := p.parseStmtList()
+	p.closeScope()
+
+	return &ast.CaseClause{pos, list, colon, body}
+}
+
+func isExprSwitch(s ast.Stmt) bool {
+	if s == nil {
+		return true
+	}
+	if e, ok := s.(*ast.ExprStmt); ok {
+		if a, ok := e.X.(*ast.TypeAssertExpr); ok {
+			return a.Type != nil // regular type assertion
+		}
+		return true
+	}
+	return false
+}
+
+func (p *parser) parseSwitchStmt() ast.Stmt {
+	if p.trace {
+		defer un(trace(p, "SwitchStmt"))
+	}
+
+	pos := p.expect(token.SWITCH)
+	p.openScope()
+	defer p.closeScope()
+
+	var s1, s2 ast.Stmt
+	if p.tok != token.LBRACE {
+		prevLev := p.exprLev
+		p.exprLev = -1
+		if p.tok != token.SEMICOLON {
+			s2 = p.parseSimpleStmt(false)
+		}
+		if p.tok == token.SEMICOLON {
+			p.next()
+			s1 = s2
+			s2 = nil
+			if p.tok != token.LBRACE {
+				s2 = p.parseSimpleStmt(false)
+			}
+		}
+		p.exprLev = prevLev
+	}
+
+	exprSwitch := isExprSwitch(s2)
+	lbrace := p.expect(token.LBRACE)
+	var list []ast.Stmt
+	for p.tok == token.CASE || p.tok == token.DEFAULT {
+		list = append(list, p.parseCaseClause(exprSwitch))
+	}
+	rbrace := p.expect(token.RBRACE)
+	p.expectSemi()
+	body := &ast.BlockStmt{lbrace, list, rbrace}
+
+	if exprSwitch {
+		return &ast.SwitchStmt{pos, s1, p.makeExpr(s2), body}
+	}
+	// type switch
+	// TODO(gri): do all the checks!
+	return &ast.TypeSwitchStmt{pos, s1, s2, body}
+}
+
+func (p *parser) parseCommClause() *ast.CommClause {
+	if p.trace {
+		defer un(trace(p, "CommClause"))
+	}
+
+	p.openScope()
+	pos := p.pos
+	var comm ast.Stmt
+	if p.tok == token.CASE {
+		p.next()
+		lhs := p.parseLhsList()
+		if p.tok == token.ARROW {
+			// SendStmt
+			if len(lhs) > 1 {
+				p.errorExpected(lhs[0].Pos(), "1 expression")
+				// continue with first expression
+			}
+			arrow := p.pos
+			p.next()
+			rhs := p.parseRhs()
+			comm = &ast.SendStmt{lhs[0], arrow, rhs}
+		} else {
+			// RecvStmt
+			pos := p.pos
+			tok := p.tok
+			var rhs ast.Expr
+			if tok == token.ASSIGN || tok == token.DEFINE {
+				// RecvStmt with assignment
+				if len(lhs) > 2 {
+					p.errorExpected(lhs[0].Pos(), "1 or 2 expressions")
+					// continue with first two expressions
+					lhs = lhs[0:2]
+				}
+				p.next()
+				rhs = p.parseRhs()
+			} else {
+				// rhs must be single receive operation
+				if len(lhs) > 1 {
+					p.errorExpected(lhs[0].Pos(), "1 expression")
+					// continue with first expression
+				}
+				rhs = lhs[0]
+				lhs = nil // there is no lhs
+			}
+			if x, isUnary := rhs.(*ast.UnaryExpr); !isUnary || x.Op != token.ARROW {
+				p.errorExpected(rhs.Pos(), "send or receive operation")
+				rhs = &ast.BadExpr{rhs.Pos(), rhs.End()}
+			}
+			if lhs != nil {
+				comm = &ast.AssignStmt{lhs, pos, tok, []ast.Expr{rhs}}
+			} else {
+				comm = &ast.ExprStmt{rhs}
+			}
+		}
+	} else {
+		p.expect(token.DEFAULT)
+	}
+
+	colon := p.expect(token.COLON)
+	body := p.parseStmtList()
+	p.closeScope()
+
+	return &ast.CommClause{pos, comm, colon, body}
+}
+
+func (p *parser) parseSelectStmt() *ast.SelectStmt {
+	if p.trace {
+		defer un(trace(p, "SelectStmt"))
+	}
+
+	pos := p.expect(token.SELECT)
+	lbrace := p.expect(token.LBRACE)
+	var list []ast.Stmt
+	for p.tok == token.CASE || p.tok == token.DEFAULT {
+		list = append(list, p.parseCommClause())
+	}
+	rbrace := p.expect(token.RBRACE)
+	p.expectSemi()
+	body := &ast.BlockStmt{lbrace, list, rbrace}
+
+	return &ast.SelectStmt{pos, body}
+}
+
+func (p *parser) parseForStmt() ast.Stmt {
+	if p.trace {
+		defer un(trace(p, "ForStmt"))
+	}
+
+	pos := p.expect(token.FOR)
+	p.openScope()
+	defer p.closeScope()
+
+	var s1, s2, s3 ast.Stmt
+	if p.tok != token.LBRACE {
+		prevLev := p.exprLev
+		p.exprLev = -1
+		if p.tok != token.SEMICOLON {
+			s2 = p.parseSimpleStmt(false)
+		}
+		if p.tok == token.SEMICOLON {
+			p.next()
+			s1 = s2
+			s2 = nil
+			if p.tok != token.SEMICOLON {
+				s2 = p.parseSimpleStmt(false)
+			}
+			p.expectSemi()
+			if p.tok != token.LBRACE {
+				s3 = p.parseSimpleStmt(false)
+			}
+		}
+		p.exprLev = prevLev
+	}
+
+	body := p.parseBlockStmt()
+	p.expectSemi()
+
+	if as, isAssign := s2.(*ast.AssignStmt); isAssign {
+		// possibly a for statement with a range clause; check assignment operator
+		if as.Tok != token.ASSIGN && as.Tok != token.DEFINE {
+			p.errorExpected(as.TokPos, "'=' or ':='")
+			return &ast.BadStmt{pos, body.End()}
+		}
+		// check lhs
+		var key, value ast.Expr
+		switch len(as.Lhs) {
+		case 2:
+			key, value = as.Lhs[0], as.Lhs[1]
+		case 1:
+			key = as.Lhs[0]
+		default:
+			p.errorExpected(as.Lhs[0].Pos(), "1 or 2 expressions")
+			return &ast.BadStmt{pos, body.End()}
+		}
+		// check rhs
+		if len(as.Rhs) != 1 {
+			p.errorExpected(as.Rhs[0].Pos(), "1 expression")
+			return &ast.BadStmt{pos, body.End()}
+		}
+		if rhs, isUnary := as.Rhs[0].(*ast.UnaryExpr); isUnary && rhs.Op == token.RANGE {
+			// rhs is range expression
+			// (any short variable declaration was handled by parseSimpleStat above)
+			return &ast.RangeStmt{pos, key, value, as.TokPos, as.Tok, rhs.X, body}
+		}
+		p.errorExpected(s2.Pos(), "range clause")
+		return &ast.BadStmt{pos, body.End()}
+	}
+
+	// regular for statement
+	return &ast.ForStmt{pos, s1, p.makeExpr(s2), s3, body}
+}
+
+func (p *parser) parseStmt() (s ast.Stmt) {
+	if p.trace {
+		defer un(trace(p, "Statement"))
+	}
+
+	switch p.tok {
+	case token.CONST, token.TYPE, token.VAR:
+		s = &ast.DeclStmt{p.parseDecl()}
+	case
+		// tokens that may start a top-level expression
+		token.IDENT, token.INT, token.FLOAT, token.CHAR, token.STRING, token.FUNC, token.LPAREN, // operand
+		token.LBRACK, token.STRUCT, // composite type
+		token.MUL, token.AND, token.ARROW, token.ADD, token.SUB, token.XOR: // unary operators
+		s = p.parseSimpleStmt(true)
+		// because of the required look-ahead, labeled statements are
+		// parsed by parseSimpleStmt - don't expect a semicolon after
+		// them
+		if _, isLabeledStmt := s.(*ast.LabeledStmt); !isLabeledStmt {
+			p.expectSemi()
+		}
+	case token.GO:
+		s = p.parseGoStmt()
+	case token.DEFER:
+		s = p.parseDeferStmt()
+	case token.RETURN:
+		s = p.parseReturnStmt()
+	case token.BREAK, token.CONTINUE, token.GOTO, token.FALLTHROUGH:
+		s = p.parseBranchStmt(p.tok)
+	case token.LBRACE:
+		s = p.parseBlockStmt()
+		p.expectSemi()
+	case token.IF:
+		s = p.parseIfStmt()
+	case token.SWITCH:
+		s = p.parseSwitchStmt()
+	case token.SELECT:
+		s = p.parseSelectStmt()
+	case token.FOR:
+		s = p.parseForStmt()
+	case token.SEMICOLON:
+		s = &ast.EmptyStmt{p.pos}
+		p.next()
+	case token.RBRACE:
+		// a semicolon may be omitted before a closing "}"
+		s = &ast.EmptyStmt{p.pos}
+	default:
+		// no statement found
+		pos := p.pos
+		p.errorExpected(pos, "statement")
+		p.next() // make progress
+		s = &ast.BadStmt{pos, p.pos}
+	}
+
+	return
+}
+
+// ----------------------------------------------------------------------------
+// Declarations
+
+type parseSpecFunction func(p *parser, doc *ast.CommentGroup, iota int) ast.Spec
+
+func parseImportSpec(p *parser, doc *ast.CommentGroup, _ int) ast.Spec {
+	if p.trace {
+		defer un(trace(p, "ImportSpec"))
+	}
+
+	var ident *ast.Ident
+	switch p.tok {
+	case token.PERIOD:
+		ident = &ast.Ident{p.pos, ".", nil}
+		p.next()
+	case token.IDENT:
+		ident = p.parseIdent()
+	}
+
+	var path *ast.BasicLit
+	if p.tok == token.STRING {
+		path = &ast.BasicLit{p.pos, p.tok, p.lit}
+		p.next()
+	} else {
+		p.expect(token.STRING) // use expect() error handling
+	}
+	p.expectSemi() // call before accessing p.linecomment
+
+	// collect imports
+	spec := &ast.ImportSpec{doc, ident, path, p.lineComment}
+	p.imports = append(p.imports, spec)
+
+	return spec
+}
+
+func parseConstSpec(p *parser, doc *ast.CommentGroup, iota int) ast.Spec {
+	if p.trace {
+		defer un(trace(p, "ConstSpec"))
+	}
+
+	idents := p.parseIdentList()
+	typ := p.tryType()
+	var values []ast.Expr
+	if typ != nil || p.tok == token.ASSIGN || iota == 0 {
+		p.expect(token.ASSIGN)
+		values = p.parseRhsList()
+	}
+	p.expectSemi() // call before accessing p.linecomment
+
+	// Go spec: The scope of a constant or variable identifier declared inside
+	// a function begins at the end of the ConstSpec or VarSpec and ends at
+	// the end of the innermost containing block.
+	// (Global identifiers are resolved in a separate phase after parsing.)
+	spec := &ast.ValueSpec{doc, idents, typ, values, p.lineComment}
+	p.declare(spec, p.topScope, ast.Con, idents...)
+
+	return spec
+}
+
+func parseTypeSpec(p *parser, doc *ast.CommentGroup, _ int) ast.Spec {
+	if p.trace {
+		defer un(trace(p, "TypeSpec"))
+	}
+
+	ident := p.parseIdent()
+
+	// Go spec: The scope of a type identifier declared inside a function begins
+	// at the identifier in the TypeSpec and ends at the end of the innermost
+	// containing block.
+	// (Global identifiers are resolved in a separate phase after parsing.)
+	spec := &ast.TypeSpec{doc, ident, nil, nil}
+	p.declare(spec, p.topScope, ast.Typ, ident)
+
+	spec.Type = p.parseType()
+	p.expectSemi() // call before accessing p.linecomment
+	spec.Comment = p.lineComment
+
+	return spec
+}
+
+func parseVarSpec(p *parser, doc *ast.CommentGroup, _ int) ast.Spec {
+	if p.trace {
+		defer un(trace(p, "VarSpec"))
+	}
+
+	idents := p.parseIdentList()
+	typ := p.tryType()
+	var values []ast.Expr
+	if typ == nil || p.tok == token.ASSIGN {
+		p.expect(token.ASSIGN)
+		values = p.parseRhsList()
+	}
+	p.expectSemi() // call before accessing p.linecomment
+
+	// Go spec: The scope of a constant or variable identifier declared inside
+	// a function begins at the end of the ConstSpec or VarSpec and ends at
+	// the end of the innermost containing block.
+	// (Global identifiers are resolved in a separate phase after parsing.)
+	spec := &ast.ValueSpec{doc, idents, typ, values, p.lineComment}
+	p.declare(spec, p.topScope, ast.Var, idents...)
+
+	return spec
+}
+
+func (p *parser) parseGenDecl(keyword token.Token, f parseSpecFunction) *ast.GenDecl {
+	if p.trace {
+		defer un(trace(p, "GenDecl("+keyword.String()+")"))
+	}
+
+	doc := p.leadComment
+	pos := p.expect(keyword)
+	var lparen, rparen token.Pos
+	var list []ast.Spec
+	if p.tok == token.LPAREN {
+		lparen = p.pos
+		p.next()
+		for iota := 0; p.tok != token.RPAREN && p.tok != token.EOF; iota++ {
+			list = append(list, f(p, p.leadComment, iota))
+		}
+		rparen = p.expect(token.RPAREN)
+		p.expectSemi()
+	} else {
+		list = append(list, f(p, nil, 0))
+	}
+
+	return &ast.GenDecl{doc, pos, keyword, lparen, list, rparen}
+}
+
+func (p *parser) parseReceiver(scope *ast.Scope) *ast.FieldList {
+	if p.trace {
+		defer un(trace(p, "Receiver"))
+	}
+
+	pos := p.pos
+	par := p.parseParameters(scope, false)
+
+	// must have exactly one receiver
+	if par.NumFields() != 1 {
+		p.errorExpected(pos, "exactly one receiver")
+		// TODO determine a better range for BadExpr below
+		par.List = []*ast.Field{&ast.Field{Type: &ast.BadExpr{pos, pos}}}
+		return par
+	}
+
+	// recv type must be of the form ["*"] identifier
+	recv := par.List[0]
+	base := deref(recv.Type)
+	if _, isIdent := base.(*ast.Ident); !isIdent {
+		p.errorExpected(base.Pos(), "(unqualified) identifier")
+		par.List = []*ast.Field{&ast.Field{Type: &ast.BadExpr{recv.Pos(), recv.End()}}}
+	}
+
+	return par
+}
+
+func (p *parser) parseFuncDecl() *ast.FuncDecl {
+	if p.trace {
+		defer un(trace(p, "FunctionDecl"))
+	}
+
+	doc := p.leadComment
+	pos := p.expect(token.FUNC)
+	scope := ast.NewScope(p.topScope) // function scope
+
+	var recv *ast.FieldList
+	if p.tok == token.LPAREN {
+		recv = p.parseReceiver(scope)
+	}
+
+	ident := p.parseIdent()
+
+	params, results := p.parseSignature(scope)
+
+	var body *ast.BlockStmt
+	if p.tok == token.LBRACE {
+		body = p.parseBody(scope)
+	}
+	p.expectSemi()
+
+	decl := &ast.FuncDecl{doc, recv, ident, &ast.FuncType{pos, params, results}, body}
+	if recv == nil {
+		// Go spec: The scope of an identifier denoting a constant, type,
+		// variable, or function (but not method) declared at top level
+		// (outside any function) is the package block.
+		//
+		// init() functions cannot be referred to and there may
+		// be more than one - don't put them in the pkgScope
+		if ident.Name != "init" {
+			p.declare(decl, p.pkgScope, ast.Fun, ident)
+		}
+	}
+
+	return decl
+}
+
+func (p *parser) parseDecl() ast.Decl {
+	if p.trace {
+		defer un(trace(p, "Declaration"))
+	}
+
+	var f parseSpecFunction
+	switch p.tok {
+	case token.CONST:
+		f = parseConstSpec
+
+	case token.TYPE:
+		f = parseTypeSpec
+
+	case token.VAR:
+		f = parseVarSpec
+
+	case token.FUNC:
+		return p.parseFuncDecl()
+
+	default:
+		pos := p.pos
+		p.errorExpected(pos, "declaration")
+		p.next() // make progress
+		decl := &ast.BadDecl{pos, p.pos}
+		return decl
+	}
+
+	return p.parseGenDecl(p.tok, f)
+}
+
+func (p *parser) parseDeclList() (list []ast.Decl) {
+	if p.trace {
+		defer un(trace(p, "DeclList"))
+	}
+
+	for p.tok != token.EOF {
+		list = append(list, p.parseDecl())
+	}
+
+	return
+}
+
+// ----------------------------------------------------------------------------
+// Source files
+
+func (p *parser) parseFile() *ast.File {
+	if p.trace {
+		defer un(trace(p, "File"))
+	}
+
+	// package clause
+	doc := p.leadComment
+	pos := p.expect(token.PACKAGE)
+	// Go spec: The package clause is not a declaration;
+	// the package name does not appear in any scope.
+	ident := p.parseIdent()
+	if ident.Name == "_" {
+		p.error(p.pos, "invalid package name _")
+	}
+	p.expectSemi()
+
+	var decls []ast.Decl
+
+	// Don't bother parsing the rest if we had errors already.
+	// Likely not a Go source file at all.
+
+	if p.ErrorCount() == 0 && p.mode&PackageClauseOnly == 0 {
+		// import decls
+		for p.tok == token.IMPORT {
+			decls = append(decls, p.parseGenDecl(token.IMPORT, parseImportSpec))
+		}
+
+		if p.mode&ImportsOnly == 0 {
+			// rest of package body
+			for p.tok != token.EOF {
+				decls = append(decls, p.parseDecl())
+			}
+		}
+	}
+
+	assert(p.topScope == p.pkgScope, "imbalanced scopes")
+
+	// resolve global identifiers within the same file
+	i := 0
+	for _, ident := range p.unresolved {
+		// i <= index for current ident
+		assert(ident.Obj == unresolved, "object already resolved")
+		ident.Obj = p.pkgScope.Lookup(ident.Name) // also removes unresolved sentinel
+		if ident.Obj == nil {
+			p.unresolved[i] = ident
+			i++
+		}
+	}
+
+	// TODO(gri): store p.imports in AST
+	return &ast.File{doc, pos, ident, decls, p.pkgScope, p.imports, p.unresolved[0:i], p.comments}
+}
diff --git a/src/pkg/go/printer/testdata/slow.golden b/src/pkg/go/printer/testdata/slow.golden
new file mode 100644
index 000000000..43a15cb1d
--- /dev/null
+++ b/src/pkg/go/printer/testdata/slow.golden
@@ -0,0 +1,85 @@
+// Copyright 2011 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package deepequal_test
+
+import (
+	"testing"
+	"google3/spam/archer/frontend/deepequal"
+)
+
+func TestTwoNilValues(t *testing.T) {
+	if err := deepequal.Check(nil, nil); err != nil {
+		t.Errorf("expected nil, saw %v", err)
+	}
+}
+
+type Foo struct {
+	bar	*Bar
+	bang	*Bar
+}
+
+type Bar struct {
+	baz	*Baz
+	foo	[]*Foo
+}
+
+type Baz struct {
+	entries		map[int]interface{}
+	whatever	string
+}
+
+func newFoo() *Foo {
+	return &Foo{bar: &Bar{baz: &Baz{
+		entries: map[int]interface{}{
+			42:	&Foo{},
+			21:	&Bar{},
+			11:	&Baz{whatever: "it's just a test"}}}},
+		bang: &Bar{foo: []*Foo{
+			&Foo{bar: &Bar{baz: &Baz{
+				entries: map[int]interface{}{
+					43:	&Foo{},
+					22:	&Bar{},
+					13:	&Baz{whatever: "this is nuts"}}}},
+				bang: &Bar{foo: []*Foo{
+					&Foo{bar: &Bar{baz: &Baz{
+						entries: map[int]interface{}{
+							61:	&Foo{},
+							71:	&Bar{},
+							11:	&Baz{whatever: "no, it's Go"}}}},
+						bang: &Bar{foo: []*Foo{
+							&Foo{bar: &Bar{baz: &Baz{
+								entries: map[int]interface{}{
+									0:	&Foo{},
+									-2:	&Bar{},
+									-11:	&Baz{whatever: "we need to go deeper"}}}},
+								bang: &Bar{foo: []*Foo{
+									&Foo{bar: &Bar{baz: &Baz{
+										entries: map[int]interface{}{
+											-2:	&Foo{},
+											-5:	&Bar{},
+											-7:	&Baz{whatever: "are you serious?"}}}},
+										bang:	&Bar{foo: []*Foo{}}},
+									&Foo{bar: &Bar{baz: &Baz{
+										entries: map[int]interface{}{
+											-100:	&Foo{},
+											50:	&Bar{},
+											20:	&Baz{whatever: "na, not really ..."}}}},
+										bang:	&Bar{foo: []*Foo{}}}}}}}}},
+					&Foo{bar: &Bar{baz: &Baz{
+						entries: map[int]interface{}{
+							2:	&Foo{},
+							1:	&Bar{},
+							-1:	&Baz{whatever: "... it's just a test."}}}},
+						bang:	&Bar{foo: []*Foo{}}}}}}}}}
+}
+
+func TestElaborate(t *testing.T) {
+	a := newFoo()
+	b := newFoo()
+
+	if err := deepequal.Check(a, b); err != nil {
+		t.Errorf("expected nil, saw %v", err)
+	}
+}
diff --git a/src/pkg/go/printer/testdata/slow.input b/src/pkg/go/printer/testdata/slow.input
new file mode 100644
index 000000000..0e5a23d88
--- /dev/null
+++ b/src/pkg/go/printer/testdata/slow.input
@@ -0,0 +1,85 @@
+// Copyright 2011 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package deepequal_test
+
+import (
+        "testing"
+        "google3/spam/archer/frontend/deepequal"
+)
+
+func TestTwoNilValues(t *testing.T) {
+        if err := deepequal.Check(nil, nil); err != nil {
+                t.Errorf("expected nil, saw %v", err)
+        }
+}
+
+type Foo struct {
+        bar *Bar
+        bang *Bar
+}
+
+type Bar struct {
+        baz *Baz
+        foo []*Foo
+}
+
+type Baz struct {
+        entries  map[int]interface{}
+        whatever string
+}
+
+func newFoo() (*Foo) {
+return &Foo{bar: &Bar{ baz: &Baz{
+entries: map[int]interface{}{
+42: &Foo{},
+21: &Bar{},
+11: &Baz{ whatever: "it's just a test" }}}},
+        bang: &Bar{foo: []*Foo{
+&Foo{bar: &Bar{ baz: &Baz{
+entries: map[int]interface{}{
+43: &Foo{},
+22: &Bar{},
+13: &Baz{ whatever: "this is nuts" }}}},
+        bang: &Bar{foo: []*Foo{
+&Foo{bar: &Bar{ baz: &Baz{
+entries: map[int]interface{}{
+61: &Foo{},
+71: &Bar{},
+11: &Baz{ whatever: "no, it's Go" }}}},
+        bang: &Bar{foo: []*Foo{
+&Foo{bar: &Bar{ baz: &Baz{
+entries: map[int]interface{}{
+0: &Foo{},
+-2: &Bar{},
+-11: &Baz{ whatever: "we need to go deeper" }}}},
+        bang: &Bar{foo: []*Foo{
+&Foo{bar: &Bar{ baz: &Baz{
+entries: map[int]interface{}{
+-2: &Foo{},
+-5: &Bar{},
+-7: &Baz{ whatever: "are you serious?" }}}},
+        bang: &Bar{foo: []*Foo{}}},
+&Foo{bar: &Bar{ baz: &Baz{
+entries: map[int]interface{}{
+-100: &Foo{},
+50: &Bar{},
+20: &Baz{ whatever: "na, not really ..." }}}},
+        bang: &Bar{foo: []*Foo{}}}}}}}}},
+&Foo{bar: &Bar{ baz: &Baz{
+entries: map[int]interface{}{
+2: &Foo{},
+1: &Bar{},
+-1: &Baz{ whatever: "... it's just a test." }}}},
+        bang: &Bar{foo: []*Foo{}}}}}}}}}
+}
+
+func TestElaborate(t *testing.T) {
+        a := newFoo()
+        b := newFoo()
+
+        if err := deepequal.Check(a, b); err != nil {
+                t.Errorf("expected nil, saw %v", err)
+        }
+}
diff --git a/src/pkg/go/printer/testdata/statements.golden b/src/pkg/go/printer/testdata/statements.golden
new file mode 100644
index 000000000..a6d85107f
--- /dev/null
+++ b/src/pkg/go/printer/testdata/statements.golden
@@ -0,0 +1,412 @@
+// Copyright 2009 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package statements
+
+var expr bool
+
+func use(x interface{})	{}
+
+// Formatting of if-statement headers.
+func _() {
+	if true {
+	}
+	if true {
+	}	// no semicolon printed
+	if expr {
+	}
+	if expr {
+	}	// no semicolon printed
+	if expr {
+	}	// no parens printed
+	if expr {
+	}	// no semicolon and parens printed
+	if x := expr; true {
+		use(x)
+	}
+	if x := expr; expr {
+		use(x)
+	}
+}
+
+// Formatting of switch-statement headers.
+func _() {
+	switch {
+	}
+	switch {
+	}	// no semicolon printed
+	switch expr {
+	}
+	switch expr {
+	}	// no semicolon printed
+	switch expr {
+	}	// no parens printed
+	switch expr {
+	}	// no semicolon and parens printed
+	switch x := expr; {
+	default:
+		use(
+			x)
+	}
+	switch x := expr; expr {
+	default:
+		use(x)
+	}
+}
+
+// Formatting of switch statement bodies.
+func _() {
+	switch {
+	}
+
+	switch x := 0; x {
+	case 1:
+		use(x)
+		use(x)	// followed by an empty line
+
+	case 2:	// followed by an empty line
+
+		use(x)	// followed by an empty line
+
+	case 3:	// no empty lines
+		use(x)
+		use(x)
+	}
+
+	switch x {
+	case 0:
+		use(x)
+	case 1:	// this comment should have no effect on the previous or next line
+		use(x)
+	}
+
+	switch x := 0; x {
+	case 1:
+		x = 0
+		// this comment should be indented
+	case 2:
+		x = 0
+	// this comment should not be indented, it is aligned with the next case
+	case 3:
+		x = 0
+		/* indented comment
+		   aligned
+		   aligned
+		*/
+		// bla
+		/* and more */
+	case 4:
+		x = 0
+	/* not indented comment
+	   aligned
+	   aligned
+	*/
+	// bla
+	/* and more */
+	case 5:
+	}
+}
+
+// Formatting of selected select statements.
+func _() {
+	select {}
+	select { /* this comment should not be tab-aligned because the closing } is on the same line */
+	}
+	select {	/* this comment should be tab-aligned */
+	}
+	select {	// this comment should be tab-aligned
+	}
+	select {
+	case <-c:
+	}
+}
+
+// Formatting of for-statement headers.
+func _() {
+	for {
+	}
+	for expr {
+	}
+	for expr {
+	}	// no parens printed
+	for {
+	}	// no semicolons printed
+	for x := expr; ; {
+		use(x)
+	}
+	for expr {
+	}	// no semicolons printed
+	for expr {
+	}	// no semicolons and parens printed
+	for ; ; expr = false {
+	}
+	for x := expr; expr; {
+		use(x)
+	}
+	for x := expr; ; expr = false {
+		use(x)
+	}
+	for ; expr; expr = false {
+	}
+	for x := expr; expr; expr = false {
+		use(x)
+	}
+	for x := range []int{} {
+		use(x)
+	}
+	for x := range []int{} {
+		use(x)
+	}	// no parens printed
+}
+
+// Don't remove mandatory parentheses around composite literals in control clauses.
+func _() {
+	// strip parentheses - no composite literals or composite literals don't start with a type name
+	if x {
+	}
+	if x {
+	}
+	if []T{} {
+	}
+	if []T{} {
+	}
+	if []T{} {
+	}
+
+	for x {
+	}
+	for x {
+	}
+	for []T{} {
+	}
+	for []T{} {
+	}
+	for []T{} {
+	}
+
+	switch x {
+	}
+	switch x {
+	}
+	switch []T{} {
+	}
+	switch []T{} {
+	}
+
+	for _ = range []T{T{42}} {
+	}
+
+	// leave parentheses - composite literals start with a type name
+	if (T{}) {
+	}
+	if (T{}) {
+	}
+	if (T{}) {
+	}
+
+	for (T{}) {
+	}
+	for (T{}) {
+	}
+	for (T{}) {
+	}
+
+	switch (T{}) {
+	}
+	switch (T{}) {
+	}
+
+	for _ = range (T1{T{42}}) {
+	}
+
+	if x == (T{42}[0]) {
+	}
+	if (x == T{42}[0]) {
+	}
+	if x == (T{42}[0]) {
+	}
+	if x == (T{42}[0]) {
+	}
+	if x == (T{42}[0]) {
+	}
+	if x == a+b*(T{42}[0]) {
+	}
+	if (x == a+b*T{42}[0]) {
+	}
+	if x == a+b*(T{42}[0]) {
+	}
+	if x == a+(b * (T{42}[0])) {
+	}
+	if x == a+b*(T{42}[0]) {
+	}
+	if (a + b*(T{42}[0])) == x {
+	}
+	if (a + b*(T{42}[0])) == x {
+	}
+
+	if struct{ x bool }{false}.x {
+	}
+	if (struct{ x bool }{false}.x) == false {
+	}
+	if struct{ x bool }{false}.x == false {
+	}
+}
+
+// Extra empty lines inside functions. Do respect source code line
+// breaks between statement boundaries but print at most one empty
+// line at a time.
+func _() {
+
+	const _ = 0
+
+	const _ = 1
+	type _ int
+	type _ float
+
+	var _ = 0
+	var x = 1
+
+	// Each use(x) call below should have at most one empty line before and after.
+	// Known bug: The first use call may have more than one empty line before
+	//            (see go/printer/nodes.go, func linebreak).
+
+
+	use(x)
+
+	if x < x {
+
+		use(x)
+
+	} else {
+
+		use(x)
+
+	}
+}
+
+// Formatting around labels.
+func _() {
+L:
+}
+
+func _() {
+	// this comment should be indented
+L:	// no semicolon needed
+}
+
+func _() {
+	switch 0 {
+	case 0:
+	L0:
+		;	// semicolon required
+	case 1:
+	L1:
+		;	// semicolon required
+	default:
+	L2:	// no semicolon needed
+	}
+}
+
+func _() {
+	f()
+L1:
+	f()
+L2:
+	;
+L3:
+}
+
+func _() {
+	// this comment should be indented
+L:
+}
+
+func _() {
+L:
+	_ = 0
+}
+
+func _() {
+	// this comment should be indented
+L:
+	_ = 0
+}
+
+func _() {
+	for {
+	L1:
+		_ = 0
+	L2:
+		_ = 0
+	}
+}
+
+func _() {
+	// this comment should be indented
+	for {
+	L1:
+		_ = 0
+	L2:
+		_ = 0
+	}
+}
+
+func _() {
+	if true {
+		_ = 0
+	}
+	_ = 0	// the indentation here should not be affected by the long label name
+AnOverlongLabel:
+	_ = 0
+
+	if true {
+		_ = 0
+	}
+	_ = 0
+
+L:
+	_ = 0
+}
+
+func _() {
+	for {
+		goto L
+	}
+L:
+
+	MoreCode()
+}
+
+func _() {
+	for {
+		goto L
+	}
+L:	// A comment on the same line as the label, followed by a single empty line.
+	// Known bug: There may be more than one empty line before MoreCode()
+	//            (see go/printer/nodes.go, func linebreak).
+
+
+	MoreCode()
+}
+
+func _() {
+	for {
+		goto L
+	}
+L:
+
+	// There should be a single empty line before this comment.
+	MoreCode()
+}
+
+func _() {
+	for {
+		goto AVeryLongLabelThatShouldNotAffectFormatting
+	}
+AVeryLongLabelThatShouldNotAffectFormatting:
+	// There should be a single empty line after this comment.
+
+	// There should be a single empty line before this comment.
+	MoreCode()
+}
diff --git a/src/pkg/go/printer/testdata/statements.input b/src/pkg/go/printer/testdata/statements.input
new file mode 100644
index 000000000..86a753c5a
--- /dev/null
+++ b/src/pkg/go/printer/testdata/statements.input
@@ -0,0 +1,351 @@
+// Copyright 2009 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package statements
+
+var expr bool
+
+func use(x interface{}) {}
+
+// Formatting of if-statement headers.
+func _() {
+	if true {}
+	if; true {}  // no semicolon printed
+	if expr{}
+	if;expr{}  // no semicolon printed
+	if (expr){}  // no parens printed
+	if;((expr)){}  // no semicolon and parens printed
+	if x:=expr;true{
+	use(x)}
+	if x:=expr; expr {use(x)}
+}
+
+
+// Formatting of switch-statement headers.
+func _() {
+	switch {}
+	switch;{}  // no semicolon printed
+	switch expr {}
+	switch;expr{}  // no semicolon printed
+	switch (expr) {}  // no parens printed
+	switch;((expr)){}  // no semicolon and parens printed
+	switch x := expr; { default:use(
+x)
+	}
+	switch x := expr; expr {default:use(x)}
+}
+
+
+// Formatting of switch statement bodies.
+func _() {
+	switch {
+	}
+
+	switch x := 0; x {
+	case 1:
+		use(x)
+		use(x)  // followed by an empty line
+
+	case 2:  // followed by an empty line
+
+		use(x)  // followed by an empty line
+
+	case 3:  // no empty lines
+		use(x)
+		use(x)
+	}
+
+	switch x {
+	case 0:
+		use(x)
+	case 1:  // this comment should have no effect on the previous or next line
+		use(x)
+	}
+
+	switch x := 0; x {
+	case 1:
+		x = 0
+		// this comment should be indented
+	case 2:
+		x = 0
+	// this comment should not be indented, it is aligned with the next case
+	case 3:
+		x = 0
+		/* indented comment
+		   aligned
+		   aligned
+		*/
+		// bla
+		/* and more */
+	case 4:
+		x = 0
+	/* not indented comment
+	   aligned
+	   aligned
+	*/
+	// bla
+	/* and more */
+	case 5:
+	}
+}
+
+
+// Formatting of selected select statements.
+func _() {
+	select {
+	}
+	select { /* this comment should not be tab-aligned because the closing } is on the same line */ }
+	select { /* this comment should be tab-aligned */
+	}
+	select { // this comment should be tab-aligned
+	}
+	select { case <-c: }
+}
+
+
+// Formatting of for-statement headers.
+func _() {
+	for{}
+	for expr {}
+	for (expr) {}  // no parens printed
+	for;;{}  // no semicolons printed
+	for x :=expr;; {use( x)}
+	for; expr;{}  // no semicolons printed
+	for; ((expr));{}  // no semicolons and parens printed
+	for; ; expr = false {}
+	for x :=expr; expr; {use(x)}
+	for x := expr;; expr=false {use(x)}
+	for;expr;expr =false {
+	}
+	for x := expr;expr;expr = false { use(x) }
+	for x := range []int{} { use(x) }
+	for x := range (([]int{})) { use(x) }  // no parens printed
+}
+
+
+// Don't remove mandatory parentheses around composite literals in control clauses.
+func _() {
+	// strip parentheses - no composite literals or composite literals don't start with a type name
+	if (x) {}
+	if (((x))) {}
+	if ([]T{}) {}
+	if (([]T{})) {}
+	if ; (((([]T{})))) {}
+
+	for (x) {}
+	for (((x))) {}
+	for ([]T{}) {}
+	for (([]T{})) {}
+	for ; (((([]T{})))) ; {}
+
+	switch (x) {}
+	switch (((x))) {}
+	switch ([]T{}) {}
+	switch ; (((([]T{})))) {}
+
+	for _ = range ((([]T{T{42}}))) {}
+
+	// leave parentheses - composite literals start with a type name
+	if (T{}) {}
+	if ((T{})) {}
+	if ; ((((T{})))) {}
+
+	for (T{}) {}
+	for ((T{})) {}
+	for ; ((((T{})))) ; {}
+
+	switch (T{}) {}
+	switch ; ((((T{})))) {}
+
+	for _ = range (((T1{T{42}}))) {}
+
+	if x == (T{42}[0]) {}
+	if (x == T{42}[0]) {}
+	if (x == (T{42}[0])) {}
+	if (x == (((T{42}[0])))) {}
+	if (((x == (T{42}[0])))) {}
+	if x == a + b*(T{42}[0]) {}
+	if (x == a + b*T{42}[0]) {}
+	if (x == a + b*(T{42}[0])) {}
+	if (x == a + ((b * (T{42}[0])))) {}
+	if (((x == a + b * (T{42}[0])))) {}
+	if (((a + b * (T{42}[0])) == x)) {}
+	if (((a + b * (T{42}[0])))) == x {}
+
+	if (struct{x bool}{false}.x) {}
+	if (struct{x bool}{false}.x) == false {}
+	if (struct{x bool}{false}.x == false) {}
+}
+
+
+// Extra empty lines inside functions. Do respect source code line
+// breaks between statement boundaries but print at most one empty
+// line at a time.
+func _() {
+
+	const _ = 0
+
+	const _ = 1
+	type _ int
+	type _ float
+
+	var _ = 0
+	var x = 1
+
+	// Each use(x) call below should have at most one empty line before and after.
+	// Known bug: The first use call may have more than one empty line before
+	//            (see go/printer/nodes.go, func linebreak).
+
+
+
+	use(x)
+
+	if x < x {
+
+		use(x)
+
+	} else {
+
+		use(x)
+
+	}
+}
+
+
+// Formatting around labels.
+func _() {
+	L:
+}
+
+
+func _() {
+	// this comment should be indented
+	L: ;  // no semicolon needed
+}
+
+
+func _() {
+	switch 0 {
+	case 0:
+		L0: ;  // semicolon required
+	case 1:
+		L1: ;  // semicolon required
+	default:
+		L2: ;  // no semicolon needed
+	}
+}
+
+
+func _() {
+	f()
+L1:
+	f()
+L2:
+	;
+L3:
+}
+
+
+func _() {
+	// this comment should be indented
+	L:
+}
+
+
+func _() {
+	L: _ = 0
+}
+
+
+func _() {
+	// this comment should be indented
+	L: _ = 0
+}
+
+
+func _() {
+	for {
+	L1: _ = 0
+	L2:
+		_ = 0
+	}
+}
+
+
+func _() {
+		// this comment should be indented
+	for {
+	L1: _ = 0
+	L2:
+		_ = 0
+	}
+}
+
+
+func _() {
+	if true {
+		_ = 0
+	}
+	_ = 0  // the indentation here should not be affected by the long label name
+AnOverlongLabel:
+	_ = 0
+	
+	if true {
+		_ = 0
+	}
+	_ = 0
+
+L:	_ = 0
+}
+
+
+func _() {
+	for {
+		goto L
+	}
+L:
+
+	MoreCode()
+}
+
+
+func _() {
+	for {
+		goto L
+	}
+L:	// A comment on the same line as the label, followed by a single empty line.
+	// Known bug: There may be more than one empty line before MoreCode()
+	//            (see go/printer/nodes.go, func linebreak).
+
+
+
+
+	MoreCode()
+}
+
+
+func _() {
+	for {
+		goto L
+	}
+L:
+
+
+
+
+	// There should be a single empty line before this comment.
+	MoreCode()
+}
+
+
+func _() {
+	for {
+		goto AVeryLongLabelThatShouldNotAffectFormatting
+	}
+AVeryLongLabelThatShouldNotAffectFormatting:
+	// There should be a single empty line after this comment.
+
+	// There should be a single empty line before this comment.
+	MoreCode()
+}
diff --git a/src/pkg/go/scanner/Makefile b/src/pkg/go/scanner/Makefile
new file mode 100644
index 000000000..453faac00
--- /dev/null
+++ b/src/pkg/go/scanner/Makefile
@@ -0,0 +1,12 @@
+# Copyright 2009 The Go Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style
+# license that can be found in the LICENSE file.
+
+include ../../../Make.inc
+
+TARG=go/scanner
+GOFILES=\
+	errors.go\
+	scanner.go\
+
+include ../../../Make.pkg
diff --git a/src/pkg/go/scanner/errors.go b/src/pkg/go/scanner/errors.go
new file mode 100644
index 000000000..a0927e416
--- /dev/null
+++ b/src/pkg/go/scanner/errors.go
@@ -0,0 +1,168 @@
+// Copyright 2009 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package scanner
+
+import (
+	"fmt"
+	"go/token"
+	"io"
+	"os"
+	"sort"
+)
+
+// An implementation of an ErrorHandler may be provided to the Scanner.
+// If a syntax error is encountered and a handler was installed, Error
+// is called with a position and an error message. The position points
+// to the beginning of the offending token.
+//
+type ErrorHandler interface {
+	Error(pos token.Position, msg string)
+}
+
+// ErrorVector implements the ErrorHandler interface. It maintains a list
+// of errors which can be retrieved with GetErrorList and GetError. The
+// zero value for an ErrorVector is an empty ErrorVector ready to use.
+//
+// A common usage pattern is to embed an ErrorVector alongside a
+// scanner in a data structure that uses the scanner. By passing a
+// reference to an ErrorVector to the scanner's Init call, default
+// error handling is obtained.
+//
+type ErrorVector struct {
+	errors []*Error
+}
+
+// Reset resets an ErrorVector to no errors.
+func (h *ErrorVector) Reset() { h.errors = h.errors[:0] }
+
+// ErrorCount returns the number of errors collected.
+func (h *ErrorVector) ErrorCount() int { return len(h.errors) }
+
+// Within ErrorVector, an error is represented by an Error node. The
+// position Pos, if valid, points to the beginning of the offending
+// token, and the error condition is described by Msg.
+//
+type Error struct {
+	Pos token.Position
+	Msg string
+}
+
+func (e *Error) String() string {
+	if e.Pos.Filename != "" || e.Pos.IsValid() {
+		// don't print ""
+		// TODO(gri) reconsider the semantics of Position.IsValid
+		return e.Pos.String() + ": " + e.Msg
+	}
+	return e.Msg
+}
+
+// An ErrorList is a (possibly sorted) list of Errors.
+type ErrorList []*Error
+
+// ErrorList implements the sort Interface.
+func (p ErrorList) Len() int      { return len(p) }
+func (p ErrorList) Swap(i, j int) { p[i], p[j] = p[j], p[i] }
+
+func (p ErrorList) Less(i, j int) bool {
+	e := &p[i].Pos
+	f := &p[j].Pos
+	// Note that it is not sufficient to simply compare file offsets because
+	// the offsets do not reflect modified line information (through //line
+	// comments).
+	if e.Filename < f.Filename {
+		return true
+	}
+	if e.Filename == f.Filename {
+		if e.Line < f.Line {
+			return true
+		}
+		if e.Line == f.Line {
+			return e.Column < f.Column
+		}
+	}
+	return false
+}
+
+func (p ErrorList) String() string {
+	switch len(p) {
+	case 0:
+		return "unspecified error"
+	case 1:
+		return p[0].String()
+	}
+	return fmt.Sprintf("%s (and %d more errors)", p[0].String(), len(p)-1)
+}
+
+// These constants control the construction of the ErrorList
+// returned by GetErrors.
+//
+const (
+	Raw         = iota // leave error list unchanged
+	Sorted             // sort error list by file, line, and column number
+	NoMultiples        // sort error list and leave only the first error per line
+)
+
+// GetErrorList returns the list of errors collected by an ErrorVector.
+// The construction of the ErrorList returned is controlled by the mode
+// parameter. If there are no errors, the result is nil.
+//
+func (h *ErrorVector) GetErrorList(mode int) ErrorList {
+	if len(h.errors) == 0 {
+		return nil
+	}
+
+	list := make(ErrorList, len(h.errors))
+	copy(list, h.errors)
+
+	if mode >= Sorted {
+		sort.Sort(list)
+	}
+
+	if mode >= NoMultiples {
+		var last token.Position // initial last.Line is != any legal error line
+		i := 0
+		for _, e := range list {
+			if e.Pos.Filename != last.Filename || e.Pos.Line != last.Line {
+				last = e.Pos
+				list[i] = e
+				i++
+			}
+		}
+		list = list[0:i]
+	}
+
+	return list
+}
+
+// GetError is like GetErrorList, but it returns an os.Error instead
+// so that a nil result can be assigned to an os.Error variable and
+// remains nil.
+//
+func (h *ErrorVector) GetError(mode int) os.Error {
+	if len(h.errors) == 0 {
+		return nil
+	}
+
+	return h.GetErrorList(mode)
+}
+
+// ErrorVector implements the ErrorHandler interface.
+func (h *ErrorVector) Error(pos token.Position, msg string) {
+	h.errors = append(h.errors, &Error{pos, msg})
+}
+
+// PrintError is a utility function that prints a list of errors to w,
+// one error per line, if the err parameter is an ErrorList. Otherwise
+// it prints the err string.
+//
+func PrintError(w io.Writer, err os.Error) {
+	if list, ok := err.(ErrorList); ok {
+		for _, e := range list {
+			fmt.Fprintf(w, "%s\n", e)
+		}
+	} else {
+		fmt.Fprintf(w, "%s\n", err)
+	}
+}
diff --git a/src/pkg/go/scanner/scanner.go b/src/pkg/go/scanner/scanner.go
new file mode 100644
index 000000000..7f3dd2373
--- /dev/null
+++ b/src/pkg/go/scanner/scanner.go
@@ -0,0 +1,670 @@
+// Copyright 2009 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// Package scanner implements a scanner for Go source text. Takes a []byte as
+// source which can then be tokenized through repeated calls to the Scan
+// function. Typical use:
+//
+//	var s Scanner
+//	fset := token.NewFileSet()  // position information is relative to fset
+//      file := fset.AddFile(filename, fset.Base(), len(src))  // register file
+//	s.Init(file, src, nil /* no error handler */, 0)
+//	for {
+//		pos, tok, lit := s.Scan()
+//		if tok == token.EOF {
+//			break
+//		}
+//		// do something here with pos, tok, and lit
+//	}
+//
+package scanner
+
+import (
+	"bytes"
+	"fmt"
+	"go/token"
+	"path/filepath"
+	"strconv"
+	"unicode"
+	"utf8"
+)
+
+// A Scanner holds the scanner's internal state while processing
+// a given text.  It can be allocated as part of another data
+// structure but must be initialized via Init before use.
+//
+type Scanner struct {
+	// immutable state
+	file *token.File  // source file handle
+	dir  string       // directory portion of file.Name()
+	src  []byte       // source
+	err  ErrorHandler // error reporting; or nil
+	mode uint         // scanning mode
+
+	// scanning state
+	ch         int  // current character
+	offset     int  // character offset
+	rdOffset   int  // reading offset (position after current character)
+	lineOffset int  // current line offset
+	insertSemi bool // insert a semicolon before next newline
+
+	// public state - ok to modify
+	ErrorCount int // number of errors encountered
+}
+
+// Read the next Unicode char into S.ch.
+// S.ch < 0 means end-of-file.
+//
+func (S *Scanner) next() {
+	if S.rdOffset < len(S.src) {
+		S.offset = S.rdOffset
+		if S.ch == '\n' {
+			S.lineOffset = S.offset
+			S.file.AddLine(S.offset)
+		}
+		r, w := int(S.src[S.rdOffset]), 1
+		switch {
+		case r == 0:
+			S.error(S.offset, "illegal character NUL")
+		case r >= 0x80:
+			// not ASCII
+			r, w = utf8.DecodeRune(S.src[S.rdOffset:])
+			if r == utf8.RuneError && w == 1 {
+				S.error(S.offset, "illegal UTF-8 encoding")
+			}
+		}
+		S.rdOffset += w
+		S.ch = r
+	} else {
+		S.offset = len(S.src)
+		if S.ch == '\n' {
+			S.lineOffset = S.offset
+			S.file.AddLine(S.offset)
+		}
+		S.ch = -1 // eof
+	}
+}
+
+// The mode parameter to the Init function is a set of flags (or 0).
+// They control scanner behavior.
+//
+const (
+	ScanComments      = 1 << iota // return comments as COMMENT tokens
+	AllowIllegalChars             // do not report an error for illegal chars
+	InsertSemis                   // automatically insert semicolons
+)
+
+// Init prepares the scanner S to tokenize the text src by setting the
+// scanner at the beginning of src. The scanner uses the file set file
+// for position information and it adds line information for each line.
+// It is ok to re-use the same file when re-scanning the same file as
+// line information which is already present is ignored. Init causes a
+// panic if the file size does not match the src size.
+//
+// Calls to Scan will use the error handler err if they encounter a
+// syntax error and err is not nil. Also, for each error encountered,
+// the Scanner field ErrorCount is incremented by one. The mode parameter
+// determines how comments, illegal characters, and semicolons are handled.
+//
+// Note that Init may call err if there is an error in the first character
+// of the file.
+//
+func (S *Scanner) Init(file *token.File, src []byte, err ErrorHandler, mode uint) {
+	// Explicitly initialize all fields since a scanner may be reused.
+	if file.Size() != len(src) {
+		panic("file size does not match src len")
+	}
+	S.file = file
+	S.dir, _ = filepath.Split(file.Name())
+	S.src = src
+	S.err = err
+	S.mode = mode
+
+	S.ch = ' '
+	S.offset = 0
+	S.rdOffset = 0
+	S.lineOffset = 0
+	S.insertSemi = false
+	S.ErrorCount = 0
+
+	S.next()
+}
+
+func (S *Scanner) error(offs int, msg string) {
+	if S.err != nil {
+		S.err.Error(S.file.Position(S.file.Pos(offs)), msg)
+	}
+	S.ErrorCount++
+}
+
+var prefix = []byte("//line ")
+
+func (S *Scanner) interpretLineComment(text []byte) {
+	if bytes.HasPrefix(text, prefix) {
+		// get filename and line number, if any
+		if i := bytes.LastIndex(text, []byte{':'}); i > 0 {
+			if line, err := strconv.Atoi(string(text[i+1:])); err == nil && line > 0 {
+				// valid //line filename:line comment;
+				filename := filepath.Clean(string(text[len(prefix):i]))
+				if !filepath.IsAbs(filename) {
+					// make filename relative to current directory
+					filename = filepath.Join(S.dir, filename)
+				}
+				// update scanner position
+				S.file.AddLineInfo(S.lineOffset, filename, line-1) // -1 since comment applies to next line
+			}
+		}
+	}
+}
+
+func (S *Scanner) scanComment() {
+	// initial '/' already consumed; S.ch == '/' || S.ch == '*'
+	offs := S.offset - 1 // position of initial '/'
+
+	if S.ch == '/' {
+		//-style comment
+		S.next()
+		for S.ch != '\n' && S.ch >= 0 {
+			S.next()
+		}
+		if offs == S.lineOffset {
+			// comment starts at the beginning of the current line
+			S.interpretLineComment(S.src[offs:S.offset])
+		}
+		return
+	}
+
+	/*-style comment */
+	S.next()
+	for S.ch >= 0 {
+		ch := S.ch
+		S.next()
+		if ch == '*' && S.ch == '/' {
+			S.next()
+			return
+		}
+	}
+
+	S.error(offs, "comment not terminated")
+}
+
+func (S *Scanner) findLineEnd() bool {
+	// initial '/' already consumed
+
+	defer func(offs int) {
+		// reset scanner state to where it was upon calling findLineEnd
+		S.ch = '/'
+		S.offset = offs
+		S.rdOffset = offs + 1
+		S.next() // consume initial '/' again
+	}(S.offset - 1)
+
+	// read ahead until a newline, EOF, or non-comment token is found
+	for S.ch == '/' || S.ch == '*' {
+		if S.ch == '/' {
+			//-style comment always contains a newline
+			return true
+		}
+		/*-style comment: look for newline */
+		S.next()
+		for S.ch >= 0 {
+			ch := S.ch
+			if ch == '\n' {
+				return true
+			}
+			S.next()
+			if ch == '*' && S.ch == '/' {
+				S.next()
+				break
+			}
+		}
+		S.skipWhitespace() // S.insertSemi is set
+		if S.ch < 0 || S.ch == '\n' {
+			return true
+		}
+		if S.ch != '/' {
+			// non-comment token
+			return false
+		}
+		S.next() // consume '/'
+	}
+
+	return false
+}
+
+func isLetter(ch int) bool {
+	return 'a' <= ch && ch <= 'z' || 'A' <= ch && ch <= 'Z' || ch == '_' || ch >= 0x80 && unicode.IsLetter(ch)
+}
+
+func isDigit(ch int) bool {
+	return '0' <= ch && ch <= '9' || ch >= 0x80 && unicode.IsDigit(ch)
+}
+
+func (S *Scanner) scanIdentifier() token.Token {
+	offs := S.offset
+	for isLetter(S.ch) || isDigit(S.ch) {
+		S.next()
+	}
+	return token.Lookup(S.src[offs:S.offset])
+}
+
+func digitVal(ch int) int {
+	switch {
+	case '0' <= ch && ch <= '9':
+		return ch - '0'
+	case 'a' <= ch && ch <= 'f':
+		return ch - 'a' + 10
+	case 'A' <= ch && ch <= 'F':
+		return ch - 'A' + 10
+	}
+	return 16 // larger than any legal digit val
+}
+
+func (S *Scanner) scanMantissa(base int) {
+	for digitVal(S.ch) < base {
+		S.next()
+	}
+}
+
+func (S *Scanner) scanNumber(seenDecimalPoint bool) token.Token {
+	// digitVal(S.ch) < 10
+	tok := token.INT
+
+	if seenDecimalPoint {
+		tok = token.FLOAT
+		S.scanMantissa(10)
+		goto exponent
+	}
+
+	if S.ch == '0' {
+		// int or float
+		offs := S.offset
+		S.next()
+		if S.ch == 'x' || S.ch == 'X' {
+			// hexadecimal int
+			S.next()
+			S.scanMantissa(16)
+			if S.offset-offs <= 2 {
+				// only scanned "0x" or "0X"
+				S.error(offs, "illegal hexadecimal number")
+			}
+		} else {
+			// octal int or float
+			seenDecimalDigit := false
+			S.scanMantissa(8)
+			if S.ch == '8' || S.ch == '9' {
+				// illegal octal int or float
+				seenDecimalDigit = true
+				S.scanMantissa(10)
+			}
+			if S.ch == '.' || S.ch == 'e' || S.ch == 'E' || S.ch == 'i' {
+				goto fraction
+			}
+			// octal int
+			if seenDecimalDigit {
+				S.error(offs, "illegal octal number")
+			}
+		}
+		goto exit
+	}
+
+	// decimal int or float
+	S.scanMantissa(10)
+
+fraction:
+	if S.ch == '.' {
+		tok = token.FLOAT
+		S.next()
+		S.scanMantissa(10)
+	}
+
+exponent:
+	if S.ch == 'e' || S.ch == 'E' {
+		tok = token.FLOAT
+		S.next()
+		if S.ch == '-' || S.ch == '+' {
+			S.next()
+		}
+		S.scanMantissa(10)
+	}
+
+	if S.ch == 'i' {
+		tok = token.IMAG
+		S.next()
+	}
+
+exit:
+	return tok
+}
+
+func (S *Scanner) scanEscape(quote int) {
+	offs := S.offset
+
+	var i, base, max uint32
+	switch S.ch {
+	case 'a', 'b', 'f', 'n', 'r', 't', 'v', '\\', quote:
+		S.next()
+		return
+	case '0', '1', '2', '3', '4', '5', '6', '7':
+		i, base, max = 3, 8, 255
+	case 'x':
+		S.next()
+		i, base, max = 2, 16, 255
+	case 'u':
+		S.next()
+		i, base, max = 4, 16, unicode.MaxRune
+	case 'U':
+		S.next()
+		i, base, max = 8, 16, unicode.MaxRune
+	default:
+		S.next() // always make progress
+		S.error(offs, "unknown escape sequence")
+		return
+	}
+
+	var x uint32
+	for ; i > 0 && S.ch != quote && S.ch >= 0; i-- {
+		d := uint32(digitVal(S.ch))
+		if d >= base {
+			S.error(S.offset, "illegal character in escape sequence")
+			break
+		}
+		x = x*base + d
+		S.next()
+	}
+	// in case of an error, consume remaining chars
+	for ; i > 0 && S.ch != quote && S.ch >= 0; i-- {
+		S.next()
+	}
+	if x > max || 0xd800 <= x && x < 0xe000 {
+		S.error(offs, "escape sequence is invalid Unicode code point")
+	}
+}
+
+func (S *Scanner) scanChar() {
+	// '\'' opening already consumed
+	offs := S.offset - 1
+
+	n := 0
+	for S.ch != '\'' {
+		ch := S.ch
+		n++
+		S.next()
+		if ch == '\n' || ch < 0 {
+			S.error(offs, "character literal not terminated")
+			n = 1
+			break
+		}
+		if ch == '\\' {
+			S.scanEscape('\'')
+		}
+	}
+
+	S.next()
+
+	if n != 1 {
+		S.error(offs, "illegal character literal")
+	}
+}
+
+func (S *Scanner) scanString() {
+	// '"' opening already consumed
+	offs := S.offset - 1
+
+	for S.ch != '"' {
+		ch := S.ch
+		S.next()
+		if ch == '\n' || ch < 0 {
+			S.error(offs, "string not terminated")
+			break
+		}
+		if ch == '\\' {
+			S.scanEscape('"')
+		}
+	}
+
+	S.next()
+}
+
+func (S *Scanner) scanRawString() {
+	// '`' opening already consumed
+	offs := S.offset - 1
+
+	for S.ch != '`' {
+		ch := S.ch
+		S.next()
+		if ch < 0 {
+			S.error(offs, "string not terminated")
+			break
+		}
+	}
+
+	S.next()
+}
+
+func (S *Scanner) skipWhitespace() {
+	for S.ch == ' ' || S.ch == '\t' || S.ch == '\n' && !S.insertSemi || S.ch == '\r' {
+		S.next()
+	}
+}
+
+// Helper functions for scanning multi-byte tokens such as >> += >>= .
+// Different routines recognize different length tok_i based on matches
+// of ch_i. If a token ends in '=', the result is tok1 or tok3
+// respectively. Otherwise, the result is tok0 if there was no other
+// matching character, or tok2 if the matching character was ch2.
+
+func (S *Scanner) switch2(tok0, tok1 token.Token) token.Token {
+	if S.ch == '=' {
+		S.next()
+		return tok1
+	}
+	return tok0
+}
+
+func (S *Scanner) switch3(tok0, tok1 token.Token, ch2 int, tok2 token.Token) token.Token {
+	if S.ch == '=' {
+		S.next()
+		return tok1
+	}
+	if S.ch == ch2 {
+		S.next()
+		return tok2
+	}
+	return tok0
+}
+
+func (S *Scanner) switch4(tok0, tok1 token.Token, ch2 int, tok2, tok3 token.Token) token.Token {
+	if S.ch == '=' {
+		S.next()
+		return tok1
+	}
+	if S.ch == ch2 {
+		S.next()
+		if S.ch == '=' {
+			S.next()
+			return tok3
+		}
+		return tok2
+	}
+	return tok0
+}
+
+// Scan scans the next token and returns the token position,
+// the token, and the literal string corresponding to the
+// token. The source end is indicated by token.EOF.
+//
+// If the returned token is token.SEMICOLON, the corresponding
+// literal string is ";" if the semicolon was present in the source,
+// and "\n" if the semicolon was inserted because of a newline or
+// at EOF.
+//
+// For more tolerant parsing, Scan will return a valid token if
+// possible even if a syntax error was encountered. Thus, even
+// if the resulting token sequence contains no illegal tokens,
+// a client may not assume that no error occurred. Instead it
+// must check the scanner's ErrorCount or the number of calls
+// of the error handler, if there was one installed.
+//
+// Scan adds line information to the file added to the file
+// set with Init. Token positions are relative to that file
+// and thus relative to the file set.
+//
+func (S *Scanner) Scan() (token.Pos, token.Token, string) {
+scanAgain:
+	S.skipWhitespace()
+
+	// current token start
+	insertSemi := false
+	offs := S.offset
+	tok := token.ILLEGAL
+
+	// determine token value
+	switch ch := S.ch; {
+	case isLetter(ch):
+		tok = S.scanIdentifier()
+		switch tok {
+		case token.IDENT, token.BREAK, token.CONTINUE, token.FALLTHROUGH, token.RETURN:
+			insertSemi = true
+		}
+	case digitVal(ch) < 10:
+		insertSemi = true
+		tok = S.scanNumber(false)
+	default:
+		S.next() // always make progress
+		switch ch {
+		case -1:
+			if S.insertSemi {
+				S.insertSemi = false // EOF consumed
+				return S.file.Pos(offs), token.SEMICOLON, "\n"
+			}
+			tok = token.EOF
+		case '\n':
+			// we only reach here if S.insertSemi was
+			// set in the first place and exited early
+			// from S.skipWhitespace()
+			S.insertSemi = false // newline consumed
+			return S.file.Pos(offs), token.SEMICOLON, "\n"
+		case '"':
+			insertSemi = true
+			tok = token.STRING
+			S.scanString()
+		case '\'':
+			insertSemi = true
+			tok = token.CHAR
+			S.scanChar()
+		case '`':
+			insertSemi = true
+			tok = token.STRING
+			S.scanRawString()
+		case ':':
+			tok = S.switch2(token.COLON, token.DEFINE)
+		case '.':
+			if digitVal(S.ch) < 10 {
+				insertSemi = true
+				tok = S.scanNumber(true)
+			} else if S.ch == '.' {
+				S.next()
+				if S.ch == '.' {
+					S.next()
+					tok = token.ELLIPSIS
+				}
+			} else {
+				tok = token.PERIOD
+			}
+		case ',':
+			tok = token.COMMA
+		case ';':
+			tok = token.SEMICOLON
+		case '(':
+			tok = token.LPAREN
+		case ')':
+			insertSemi = true
+			tok = token.RPAREN
+		case '[':
+			tok = token.LBRACK
+		case ']':
+			insertSemi = true
+			tok = token.RBRACK
+		case '{':
+			tok = token.LBRACE
+		case '}':
+			insertSemi = true
+			tok = token.RBRACE
+		case '+':
+			tok = S.switch3(token.ADD, token.ADD_ASSIGN, '+', token.INC)
+			if tok == token.INC {
+				insertSemi = true
+			}
+		case '-':
+			tok = S.switch3(token.SUB, token.SUB_ASSIGN, '-', token.DEC)
+			if tok == token.DEC {
+				insertSemi = true
+			}
+		case '*':
+			tok = S.switch2(token.MUL, token.MUL_ASSIGN)
+		case '/':
+			if S.ch == '/' || S.ch == '*' {
+				// comment
+				if S.insertSemi && S.findLineEnd() {
+					// reset position to the beginning of the comment
+					S.ch = '/'
+					S.offset = offs
+					S.rdOffset = offs + 1
+					S.insertSemi = false // newline consumed
+					return S.file.Pos(offs), token.SEMICOLON, "\n"
+				}
+				S.scanComment()
+				if S.mode&ScanComments == 0 {
+					// skip comment
+					S.insertSemi = false // newline consumed
+					goto scanAgain
+				}
+				tok = token.COMMENT
+			} else {
+				tok = S.switch2(token.QUO, token.QUO_ASSIGN)
+			}
+		case '%':
+			tok = S.switch2(token.REM, token.REM_ASSIGN)
+		case '^':
+			tok = S.switch2(token.XOR, token.XOR_ASSIGN)
+		case '<':
+			if S.ch == '-' {
+				S.next()
+				tok = token.ARROW
+			} else {
+				tok = S.switch4(token.LSS, token.LEQ, '<', token.SHL, token.SHL_ASSIGN)
+			}
+		case '>':
+			tok = S.switch4(token.GTR, token.GEQ, '>', token.SHR, token.SHR_ASSIGN)
+		case '=':
+			tok = S.switch2(token.ASSIGN, token.EQL)
+		case '!':
+			tok = S.switch2(token.NOT, token.NEQ)
+		case '&':
+			if S.ch == '^' {
+				S.next()
+				tok = S.switch2(token.AND_NOT, token.AND_NOT_ASSIGN)
+			} else {
+				tok = S.switch3(token.AND, token.AND_ASSIGN, '&', token.LAND)
+			}
+		case '|':
+			tok = S.switch3(token.OR, token.OR_ASSIGN, '|', token.LOR)
+		default:
+			if S.mode&AllowIllegalChars == 0 {
+				S.error(offs, fmt.Sprintf("illegal character %#U", ch))
+			}
+			insertSemi = S.insertSemi // preserve insertSemi info
+		}
+	}
+
+	if S.mode&InsertSemis != 0 {
+		S.insertSemi = insertSemi
+	}
+
+	// TODO(gri): The scanner API should change such that the literal string
+	//            is only valid if an actual literal was scanned. This will
+	//            permit a more efficient implementation.
+	return S.file.Pos(offs), tok, string(S.src[offs:S.offset])
+}
diff --git a/src/pkg/go/scanner/scanner_test.go b/src/pkg/go/scanner/scanner_test.go
new file mode 100644
index 000000000..eb9e1cb81
--- /dev/null
+++ b/src/pkg/go/scanner/scanner_test.go
@@ -0,0 +1,672 @@
+// Copyright 2009 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package scanner
+
+import (
+	"go/token"
+	"os"
+	"path/filepath"
+	"runtime"
+	"testing"
+)
+
+var fset = token.NewFileSet()
+
+const /* class */ (
+	special = iota
+	literal
+	operator
+	keyword
+)
+
+func tokenclass(tok token.Token) int {
+	switch {
+	case tok.IsLiteral():
+		return literal
+	case tok.IsOperator():
+		return operator
+	case tok.IsKeyword():
+		return keyword
+	}
+	return special
+}
+
+type elt struct {
+	tok   token.Token
+	lit   string
+	class int
+}
+
+var tokens = [...]elt{
+	// Special tokens
+	{token.COMMENT, "/* a comment */", special},
+	{token.COMMENT, "// a comment \n", special},
+
+	// Identifiers and basic type literals
+	{token.IDENT, "foobar", literal},
+	{token.IDENT, "a۰۱۸", literal},
+	{token.IDENT, "foo६४", literal},
+	{token.IDENT, "bar9876", literal},
+	{token.INT, "0", literal},
+	{token.INT, "1", literal},
+	{token.INT, "123456789012345678890", literal},
+	{token.INT, "01234567", literal},
+	{token.INT, "0xcafebabe", literal},
+	{token.FLOAT, "0.", literal},
+	{token.FLOAT, ".0", literal},
+	{token.FLOAT, "3.14159265", literal},
+	{token.FLOAT, "1e0", literal},
+	{token.FLOAT, "1e+100", literal},
+	{token.FLOAT, "1e-100", literal},
+	{token.FLOAT, "2.71828e-1000", literal},
+	{token.IMAG, "0i", literal},
+	{token.IMAG, "1i", literal},
+	{token.IMAG, "012345678901234567889i", literal},
+	{token.IMAG, "123456789012345678890i", literal},
+	{token.IMAG, "0.i", literal},
+	{token.IMAG, ".0i", literal},
+	{token.IMAG, "3.14159265i", literal},
+	{token.IMAG, "1e0i", literal},
+	{token.IMAG, "1e+100i", literal},
+	{token.IMAG, "1e-100i", literal},
+	{token.IMAG, "2.71828e-1000i", literal},
+	{token.CHAR, "'a'", literal},
+	{token.CHAR, "'\\000'", literal},
+	{token.CHAR, "'\\xFF'", literal},
+	{token.CHAR, "'\\uff16'", literal},
+	{token.CHAR, "'\\U0000ff16'", literal},
+	{token.STRING, "`foobar`", literal},
+	{token.STRING, "`" + `foo
+	                        bar` +
+		"`",
+		literal,
+	},
+
+	// Operators and delimiters
+	{token.ADD, "+", operator},
+	{token.SUB, "-", operator},
+	{token.MUL, "*", operator},
+	{token.QUO, "/", operator},
+	{token.REM, "%", operator},
+
+	{token.AND, "&", operator},
+	{token.OR, "|", operator},
+	{token.XOR, "^", operator},
+	{token.SHL, "<<", operator},
+	{token.SHR, ">>", operator},
+	{token.AND_NOT, "&^", operator},
+
+	{token.ADD_ASSIGN, "+=", operator},
+	{token.SUB_ASSIGN, "-=", operator},
+	{token.MUL_ASSIGN, "*=", operator},
+	{token.QUO_ASSIGN, "/=", operator},
+	{token.REM_ASSIGN, "%=", operator},
+
+	{token.AND_ASSIGN, "&=", operator},
+	{token.OR_ASSIGN, "|=", operator},
+	{token.XOR_ASSIGN, "^=", operator},
+	{token.SHL_ASSIGN, "<<=", operator},
+	{token.SHR_ASSIGN, ">>=", operator},
+	{token.AND_NOT_ASSIGN, "&^=", operator},
+
+	{token.LAND, "&&", operator},
+	{token.LOR, "||", operator},
+	{token.ARROW, "<-", operator},
+	{token.INC, "++", operator},
+	{token.DEC, "--", operator},
+
+	{token.EQL, "==", operator},
+	{token.LSS, "<", operator},
+	{token.GTR, ">", operator},
+	{token.ASSIGN, "=", operator},
+	{token.NOT, "!", operator},
+
+	{token.NEQ, "!=", operator},
+	{token.LEQ, "<=", operator},
+	{token.GEQ, ">=", operator},
+	{token.DEFINE, ":=", operator},
+	{token.ELLIPSIS, "...", operator},
+
+	{token.LPAREN, "(", operator},
+	{token.LBRACK, "[", operator},
+	{token.LBRACE, "{", operator},
+	{token.COMMA, ",", operator},
+	{token.PERIOD, ".", operator},
+
+	{token.RPAREN, ")", operator},
+	{token.RBRACK, "]", operator},
+	{token.RBRACE, "}", operator},
+	{token.SEMICOLON, ";", operator},
+	{token.COLON, ":", operator},
+
+	// Keywords
+	{token.BREAK, "break", keyword},
+	{token.CASE, "case", keyword},
+	{token.CHAN, "chan", keyword},
+	{token.CONST, "const", keyword},
+	{token.CONTINUE, "continue", keyword},
+
+	{token.DEFAULT, "default", keyword},
+	{token.DEFER, "defer", keyword},
+	{token.ELSE, "else", keyword},
+	{token.FALLTHROUGH, "fallthrough", keyword},
+	{token.FOR, "for", keyword},
+
+	{token.FUNC, "func", keyword},
+	{token.GO, "go", keyword},
+	{token.GOTO, "goto", keyword},
+	{token.IF, "if", keyword},
+	{token.IMPORT, "import", keyword},
+
+	{token.INTERFACE, "interface", keyword},
+	{token.MAP, "map", keyword},
+	{token.PACKAGE, "package", keyword},
+	{token.RANGE, "range", keyword},
+	{token.RETURN, "return", keyword},
+
+	{token.SELECT, "select", keyword},
+	{token.STRUCT, "struct", keyword},
+	{token.SWITCH, "switch", keyword},
+	{token.TYPE, "type", keyword},
+	{token.VAR, "var", keyword},
+}
+
+const whitespace = "  \t  \n\n\n" // to separate tokens
+
+type testErrorHandler struct {
+	t *testing.T
+}
+
+func (h *testErrorHandler) Error(pos token.Position, msg string) {
+	h.t.Errorf("Error() called (msg = %s)", msg)
+}
+
+func newlineCount(s string) int {
+	n := 0
+	for i := 0; i < len(s); i++ {
+		if s[i] == '\n' {
+			n++
+		}
+	}
+	return n
+}
+
+func checkPos(t *testing.T, lit string, p token.Pos, expected token.Position) {
+	pos := fset.Position(p)
+	if pos.Filename != expected.Filename {
+		t.Errorf("bad filename for %q: got %s, expected %s", lit, pos.Filename, expected.Filename)
+	}
+	if pos.Offset != expected.Offset {
+		t.Errorf("bad position for %q: got %d, expected %d", lit, pos.Offset, expected.Offset)
+	}
+	if pos.Line != expected.Line {
+		t.Errorf("bad line for %q: got %d, expected %d", lit, pos.Line, expected.Line)
+	}
+	if pos.Column != expected.Column {
+		t.Errorf("bad column for %q: got %d, expected %d", lit, pos.Column, expected.Column)
+	}
+}
+
+// Verify that calling Scan() provides the correct results.
+func TestScan(t *testing.T) {
+	// make source
+	var src string
+	for _, e := range tokens {
+		src += e.lit + whitespace
+	}
+	src_linecount := newlineCount(src)
+	whitespace_linecount := newlineCount(whitespace)
+
+	// verify scan
+	var s Scanner
+	s.Init(fset.AddFile("", fset.Base(), len(src)), []byte(src), &testErrorHandler{t}, ScanComments)
+	index := 0
+	epos := token.Position{"", 0, 1, 1} // expected position
+	for {
+		pos, tok, lit := s.Scan()
+		e := elt{token.EOF, "", special}
+		if index < len(tokens) {
+			e = tokens[index]
+		}
+		if tok == token.EOF {
+			lit = ""
+			epos.Line = src_linecount
+			epos.Column = 2
+		}
+		checkPos(t, lit, pos, epos)
+		if tok != e.tok {
+			t.Errorf("bad token for %q: got %s, expected %s", lit, tok.String(), e.tok.String())
+		}
+		if e.tok.IsLiteral() && lit != e.lit {
+			t.Errorf("bad literal for %q: got %q, expected %q", lit, lit, e.lit)
+		}
+		if tokenclass(tok) != e.class {
+			t.Errorf("bad class for %q: got %d, expected %d", lit, tokenclass(tok), e.class)
+		}
+		epos.Offset += len(lit) + len(whitespace)
+		epos.Line += newlineCount(lit) + whitespace_linecount
+		if tok == token.COMMENT && lit[1] == '/' {
+			// correct for unaccounted '/n' in //-style comment
+			epos.Offset++
+			epos.Line++
+		}
+		index++
+		if tok == token.EOF {
+			break
+		}
+	}
+	if s.ErrorCount != 0 {
+		t.Errorf("found %d errors", s.ErrorCount)
+	}
+}
+
+func checkSemi(t *testing.T, line string, mode uint) {
+	var S Scanner
+	file := fset.AddFile("TestSemis", fset.Base(), len(line))
+	S.Init(file, []byte(line), nil, mode)
+	pos, tok, lit := S.Scan()
+	for tok != token.EOF {
+		if tok == token.ILLEGAL {
+			// the illegal token literal indicates what
+			// kind of semicolon literal to expect
+			semiLit := "\n"
+			if lit[0] == '#' {
+				semiLit = ";"
+			}
+			// next token must be a semicolon
+			semiPos := file.Position(pos)
+			semiPos.Offset++
+			semiPos.Column++
+			pos, tok, lit = S.Scan()
+			if tok == token.SEMICOLON {
+				if lit != semiLit {
+					t.Errorf(`bad literal for %q: got %q, expected %q`, line, lit, semiLit)
+				}
+				checkPos(t, line, pos, semiPos)
+			} else {
+				t.Errorf("bad token for %q: got %s, expected ;", line, tok.String())
+			}
+		} else if tok == token.SEMICOLON {
+			t.Errorf("bad token for %q: got ;, expected no ;", line)
+		}
+		pos, tok, lit = S.Scan()
+	}
+}
+
+var lines = []string{
+	// # indicates a semicolon present in the source
+	// $ indicates an automatically inserted semicolon
+	"",
+	"#;",
+	"foo$\n",
+	"123$\n",
+	"1.2$\n",
+	"'x'$\n",
+	`"x"` + "$\n",
+	"`x`$\n",
+
+	"+\n",
+	"-\n",
+	"*\n",
+	"/\n",
+	"%\n",
+
+	"&\n",
+	"|\n",
+	"^\n",
+	"<<\n",
+	">>\n",
+	"&^\n",
+
+	"+=\n",
+	"-=\n",
+	"*=\n",
+	"/=\n",
+	"%=\n",
+
+	"&=\n",
+	"|=\n",
+	"^=\n",
+	"<<=\n",
+	">>=\n",
+	"&^=\n",
+
+	"&&\n",
+	"||\n",
+	"<-\n",
+	"++$\n",
+	"--$\n",
+
+	"==\n",
+	"<\n",
+	">\n",
+	"=\n",
+	"!\n",
+
+	"!=\n",
+	"<=\n",
+	">=\n",
+	":=\n",
+	"...\n",
+
+	"(\n",
+	"[\n",
+	"{\n",
+	",\n",
+	".\n",
+
+	")$\n",
+	"]$\n",
+	"}$\n",
+	"#;\n",
+	":\n",
+
+	"break$\n",
+	"case\n",
+	"chan\n",
+	"const\n",
+	"continue$\n",
+
+	"default\n",
+	"defer\n",
+	"else\n",
+	"fallthrough$\n",
+	"for\n",
+
+	"func\n",
+	"go\n",
+	"goto\n",
+	"if\n",
+	"import\n",
+
+	"interface\n",
+	"map\n",
+	"package\n",
+	"range\n",
+	"return$\n",
+
+	"select\n",
+	"struct\n",
+	"switch\n",
+	"type\n",
+	"var\n",
+
+	"foo$//comment\n",
+	"foo$//comment",
+	"foo$/*comment*/\n",
+	"foo$/*\n*/",
+	"foo$/*comment*/    \n",
+	"foo$/*\n*/    ",
+
+	"foo    $// comment\n",
+	"foo    $// comment",
+	"foo    $/*comment*/\n",
+	"foo    $/*\n*/",
+	"foo    $/*  */ /* \n */ bar$/**/\n",
+	"foo    $/*0*/ /*1*/ /*2*/\n",
+
+	"foo    $/*comment*/    \n",
+	"foo    $/*0*/ /*1*/ /*2*/    \n",
+	"foo	$/**/ /*-------------*/       /*----\n*/bar       $/*  \n*/baa$\n",
+	"foo    $/* an EOF terminates a line */",
+	"foo    $/* an EOF terminates a line */ /*",
+	"foo    $/* an EOF terminates a line */ //",
+
+	"package main$\n\nfunc main() {\n\tif {\n\t\treturn /* */ }$\n}$\n",
+	"package main$",
+}
+
+func TestSemis(t *testing.T) {
+	for _, line := range lines {
+		checkSemi(t, line, AllowIllegalChars|InsertSemis)
+		checkSemi(t, line, AllowIllegalChars|InsertSemis|ScanComments)
+
+		// if the input ended in newlines, the input must tokenize the
+		// same with or without those newlines
+		for i := len(line) - 1; i >= 0 && line[i] == '\n'; i-- {
+			checkSemi(t, line[0:i], AllowIllegalChars|InsertSemis)
+			checkSemi(t, line[0:i], AllowIllegalChars|InsertSemis|ScanComments)
+		}
+	}
+}
+
+type segment struct {
+	srcline  string // a line of source text
+	filename string // filename for current token
+	line     int    // line number for current token
+}
+
+var segments = []segment{
+	// exactly one token per line since the test consumes one token per segment
+	{"  line1", filepath.Join("dir", "TestLineComments"), 1},
+	{"\nline2", filepath.Join("dir", "TestLineComments"), 2},
+	{"\nline3  //line File1.go:100", filepath.Join("dir", "TestLineComments"), 3}, // bad line comment, ignored
+	{"\nline4", filepath.Join("dir", "TestLineComments"), 4},
+	{"\n//line File1.go:100\n  line100", filepath.Join("dir", "File1.go"), 100},
+	{"\n//line File2.go:200\n  line200", filepath.Join("dir", "File2.go"), 200},
+	{"\n//line :1\n  line1", "dir", 1},
+	{"\n//line foo:42\n  line42", filepath.Join("dir", "foo"), 42},
+	{"\n //line foo:42\n  line44", filepath.Join("dir", "foo"), 44},           // bad line comment, ignored
+	{"\n//line foo 42\n  line46", filepath.Join("dir", "foo"), 46},            // bad line comment, ignored
+	{"\n//line foo:42 extra text\n  line48", filepath.Join("dir", "foo"), 48}, // bad line comment, ignored
+	{"\n//line ./foo:42\n  line42", filepath.Join("dir", "foo"), 42},
+	{"\n//line a/b/c/File1.go:100\n  line100", filepath.Join("dir", "a", "b", "c", "File1.go"), 100},
+}
+
+var unixsegments = []segment{
+	{"\n//line /bar:42\n  line42", "/bar", 42},
+}
+
+var winsegments = []segment{
+	{"\n//line c:\\bar:42\n  line42", "c:\\bar", 42},
+	{"\n//line c:\\dir\\File1.go:100\n  line100", "c:\\dir\\File1.go", 100},
+}
+
+// Verify that comments of the form "//line filename:line" are interpreted correctly.
+func TestLineComments(t *testing.T) {
+	segs := segments
+	if runtime.GOOS == "windows" {
+		segs = append(segs, winsegments...)
+	} else {
+		segs = append(segs, unixsegments...)
+	}
+
+	// make source
+	var src string
+	for _, e := range segs {
+		src += e.srcline
+	}
+
+	// verify scan
+	var S Scanner
+	file := fset.AddFile(filepath.Join("dir", "TestLineComments"), fset.Base(), len(src))
+	S.Init(file, []byte(src), nil, 0)
+	for _, s := range segs {
+		p, _, lit := S.Scan()
+		pos := file.Position(p)
+		checkPos(t, lit, p, token.Position{s.filename, pos.Offset, s.line, pos.Column})
+	}
+
+	if S.ErrorCount != 0 {
+		t.Errorf("found %d errors", S.ErrorCount)
+	}
+}
+
+// Verify that initializing the same scanner more then once works correctly.
+func TestInit(t *testing.T) {
+	var s Scanner
+
+	// 1st init
+	src1 := "if true { }"
+	f1 := fset.AddFile("src1", fset.Base(), len(src1))
+	s.Init(f1, []byte(src1), nil, 0)
+	if f1.Size() != len(src1) {
+		t.Errorf("bad file size: got %d, expected %d", f1.Size(), len(src1))
+	}
+	s.Scan()              // if
+	s.Scan()              // true
+	_, tok, _ := s.Scan() // {
+	if tok != token.LBRACE {
+		t.Errorf("bad token: got %s, expected %s", tok.String(), token.LBRACE)
+	}
+
+	// 2nd init
+	src2 := "go true { ]"
+	f2 := fset.AddFile("src2", fset.Base(), len(src2))
+	s.Init(f2, []byte(src2), nil, 0)
+	if f2.Size() != len(src2) {
+		t.Errorf("bad file size: got %d, expected %d", f2.Size(), len(src2))
+	}
+	_, tok, _ = s.Scan() // go
+	if tok != token.GO {
+		t.Errorf("bad token: got %s, expected %s", tok.String(), token.GO)
+	}
+
+	if s.ErrorCount != 0 {
+		t.Errorf("found %d errors", s.ErrorCount)
+	}
+}
+
+func TestIllegalChars(t *testing.T) {
+	var s Scanner
+
+	const src = "*?*$*@*"
+	file := fset.AddFile("", fset.Base(), len(src))
+	s.Init(file, []byte(src), &testErrorHandler{t}, AllowIllegalChars)
+	for offs, ch := range src {
+		pos, tok, lit := s.Scan()
+		if poffs := file.Offset(pos); poffs != offs {
+			t.Errorf("bad position for %s: got %d, expected %d", lit, poffs, offs)
+		}
+		if tok == token.ILLEGAL && lit != string(ch) {
+			t.Errorf("bad token: got %s, expected %s", lit, string(ch))
+		}
+	}
+
+	if s.ErrorCount != 0 {
+		t.Errorf("found %d errors", s.ErrorCount)
+	}
+}
+
+func TestStdErrorHander(t *testing.T) {
+	const src = "@\n" + // illegal character, cause an error
+		"@ @\n" + // two errors on the same line
+		"//line File2:20\n" +
+		"@\n" + // different file, but same line
+		"//line File2:1\n" +
+		"@ @\n" + // same file, decreasing line number
+		"//line File1:1\n" +
+		"@ @ @" // original file, line 1 again
+
+	v := new(ErrorVector)
+	var s Scanner
+	s.Init(fset.AddFile("File1", fset.Base(), len(src)), []byte(src), v, 0)
+	for {
+		if _, tok, _ := s.Scan(); tok == token.EOF {
+			break
+		}
+	}
+
+	list := v.GetErrorList(Raw)
+	if len(list) != 9 {
+		t.Errorf("found %d raw errors, expected 9", len(list))
+		PrintError(os.Stderr, list)
+	}
+
+	list = v.GetErrorList(Sorted)
+	if len(list) != 9 {
+		t.Errorf("found %d sorted errors, expected 9", len(list))
+		PrintError(os.Stderr, list)
+	}
+
+	list = v.GetErrorList(NoMultiples)
+	if len(list) != 4 {
+		t.Errorf("found %d one-per-line errors, expected 4", len(list))
+		PrintError(os.Stderr, list)
+	}
+
+	if v.ErrorCount() != s.ErrorCount {
+		t.Errorf("found %d errors, expected %d", v.ErrorCount(), s.ErrorCount)
+	}
+}
+
+type errorCollector struct {
+	cnt int            // number of errors encountered
+	msg string         // last error message encountered
+	pos token.Position // last error position encountered
+}
+
+func (h *errorCollector) Error(pos token.Position, msg string) {
+	h.cnt++
+	h.msg = msg
+	h.pos = pos
+}
+
+func checkError(t *testing.T, src string, tok token.Token, pos int, err string) {
+	var s Scanner
+	var h errorCollector
+	s.Init(fset.AddFile("", fset.Base(), len(src)), []byte(src), &h, ScanComments)
+	_, tok0, _ := s.Scan()
+	_, tok1, _ := s.Scan()
+	if tok0 != tok {
+		t.Errorf("%q: got %s, expected %s", src, tok0, tok)
+	}
+	if tok1 != token.EOF {
+		t.Errorf("%q: got %s, expected EOF", src, tok1)
+	}
+	cnt := 0
+	if err != "" {
+		cnt = 1
+	}
+	if h.cnt != cnt {
+		t.Errorf("%q: got cnt %d, expected %d", src, h.cnt, cnt)
+	}
+	if h.msg != err {
+		t.Errorf("%q: got msg %q, expected %q", src, h.msg, err)
+	}
+	if h.pos.Offset != pos {
+		t.Errorf("%q: got offset %d, expected %d", src, h.pos.Offset, pos)
+	}
+}
+
+var errors = []struct {
+	src string
+	tok token.Token
+	pos int
+	err string
+}{
+	{"\a", token.ILLEGAL, 0, "illegal character U+0007"},
+	{`#`, token.ILLEGAL, 0, "illegal character U+0023 '#'"},
+	{`…`, token.ILLEGAL, 0, "illegal character U+2026 '…'"},
+	{`' '`, token.CHAR, 0, ""},
+	{`''`, token.CHAR, 0, "illegal character literal"},
+	{`'\8'`, token.CHAR, 2, "unknown escape sequence"},
+	{`'\08'`, token.CHAR, 3, "illegal character in escape sequence"},
+	{`'\x0g'`, token.CHAR, 4, "illegal character in escape sequence"},
+	{`'\Uffffffff'`, token.CHAR, 2, "escape sequence is invalid Unicode code point"},
+	{`'`, token.CHAR, 0, "character literal not terminated"},
+	{`""`, token.STRING, 0, ""},
+	{`"`, token.STRING, 0, "string not terminated"},
+	{"``", token.STRING, 0, ""},
+	{"`", token.STRING, 0, "string not terminated"},
+	{"/**/", token.COMMENT, 0, ""},
+	{"/*", token.COMMENT, 0, "comment not terminated"},
+	{"077", token.INT, 0, ""},
+	{"078.", token.FLOAT, 0, ""},
+	{"07801234567.", token.FLOAT, 0, ""},
+	{"078e0", token.FLOAT, 0, ""},
+	{"078", token.INT, 0, "illegal octal number"},
+	{"07800000009", token.INT, 0, "illegal octal number"},
+	{"0x", token.INT, 0, "illegal hexadecimal number"},
+	{"0X", token.INT, 0, "illegal hexadecimal number"},
+	{"\"abc\x00def\"", token.STRING, 4, "illegal character NUL"},
+	{"\"abc\x80def\"", token.STRING, 4, "illegal UTF-8 encoding"},
+}
+
+func TestScanErrors(t *testing.T) {
+	for _, e := range errors {
+		checkError(t, e.src, e.tok, e.pos, e.err)
+	}
+}
diff --git a/src/pkg/go/token/Makefile b/src/pkg/go/token/Makefile
new file mode 100644
index 000000000..4a4e64dc8
--- /dev/null
+++ b/src/pkg/go/token/Makefile
@@ -0,0 +1,12 @@
+# Copyright 2009 The Go Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style
+# license that can be found in the LICENSE file.
+
+include ../../../Make.inc
+
+TARG=go/token
+GOFILES=\
+	position.go\
+	token.go\
+
+include ../../../Make.pkg
diff --git a/src/pkg/go/token/position.go b/src/pkg/go/token/position.go
new file mode 100644
index 000000000..c559e19f8
--- /dev/null
+++ b/src/pkg/go/token/position.go
@@ -0,0 +1,424 @@
+// Copyright 2010 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// TODO(gri) consider making this a separate package outside the go directory.
+
+package token
+
+import (
+	"fmt"
+	"sort"
+	"sync"
+)
+
+// Position describes an arbitrary source position
+// including the file, line, and column location.
+// A Position is valid if the line number is > 0.
+//
+type Position struct {
+	Filename string // filename, if any
+	Offset   int    // offset, starting at 0
+	Line     int    // line number, starting at 1
+	Column   int    // column number, starting at 1 (character count)
+}
+
+// IsValid returns true if the position is valid.
+func (pos *Position) IsValid() bool { return pos.Line > 0 }
+
+// String returns a string in one of several forms:
+//
+//	file:line:column    valid position with file name
+//	line:column         valid position without file name
+//	file                invalid position with file name
+//	-                   invalid position without file name
+//
+func (pos Position) String() string {
+	s := pos.Filename
+	if pos.IsValid() {
+		if s != "" {
+			s += ":"
+		}
+		s += fmt.Sprintf("%d:%d", pos.Line, pos.Column)
+	}
+	if s == "" {
+		s = "-"
+	}
+	return s
+}
+
+// Pos is a compact encoding of a source position within a file set.
+// It can be converted into a Position for a more convenient, but much
+// larger, representation.
+//
+// The Pos value for a given file is a number in the range [base, base+size],
+// where base and size are specified when adding the file to the file set via
+// AddFile.
+//
+// To create the Pos value for a specific source offset, first add
+// the respective file to the current file set (via FileSet.AddFile)
+// and then call File.Pos(offset) for that file. Given a Pos value p
+// for a specific file set fset, the corresponding Position value is
+// obtained by calling fset.Position(p).
+//
+// Pos values can be compared directly with the usual comparison operators:
+// If two Pos values p and q are in the same file, comparing p and q is
+// equivalent to comparing the respective source file offsets. If p and q
+// are in different files, p < q is true if the file implied by p was added
+// to the respective file set before the file implied by q.
+//
+type Pos int
+
+// The zero value for Pos is NoPos; there is no file and line information
+// associated with it, and NoPos().IsValid() is false. NoPos is always
+// smaller than any other Pos value. The corresponding Position value
+// for NoPos is the zero value for Position.
+// 
+const NoPos Pos = 0
+
+// IsValid returns true if the position is valid.
+func (p Pos) IsValid() bool {
+	return p != NoPos
+}
+
+func searchFiles(a []*File, x int) int {
+	return sort.Search(len(a), func(i int) bool { return a[i].base > x }) - 1
+}
+
+func (s *FileSet) file(p Pos) *File {
+	if f := s.last; f != nil && f.base <= int(p) && int(p) <= f.base+f.size {
+		return f
+	}
+	if i := searchFiles(s.files, int(p)); i >= 0 {
+		f := s.files[i]
+		// f.base <= int(p) by definition of searchFiles
+		if int(p) <= f.base+f.size {
+			s.last = f
+			return f
+		}
+	}
+	return nil
+}
+
+// File returns the file which contains the position p.
+// If no such file is found (for instance for p == NoPos),
+// the result is nil.
+//
+func (s *FileSet) File(p Pos) (f *File) {
+	if p != NoPos {
+		s.mutex.RLock()
+		f = s.file(p)
+		s.mutex.RUnlock()
+	}
+	return
+}
+
+func (f *File) position(p Pos) (pos Position) {
+	offset := int(p) - f.base
+	pos.Offset = offset
+	pos.Filename, pos.Line, pos.Column = f.info(offset)
+	return
+}
+
+// Position converts a Pos in the fileset into a general Position.
+func (s *FileSet) Position(p Pos) (pos Position) {
+	if p != NoPos {
+		// TODO(gri) consider optimizing the case where p
+		//           is in the last file added, or perhaps
+		//           looked at - will eliminate one level
+		//           of search
+		s.mutex.RLock()
+		if f := s.file(p); f != nil {
+			pos = f.position(p)
+		}
+		s.mutex.RUnlock()
+	}
+	return
+}
+
+type lineInfo struct {
+	offset   int
+	filename string
+	line     int
+}
+
+// AddLineInfo adds alternative file and line number information for
+// a given file offset. The offset must be larger than the offset for
+// the previously added alternative line info and smaller than the
+// file size; otherwise the information is ignored.
+//
+// AddLineInfo is typically used to register alternative position
+// information for //line filename:line comments in source files.
+//
+func (f *File) AddLineInfo(offset int, filename string, line int) {
+	f.set.mutex.Lock()
+	if i := len(f.infos); i == 0 || f.infos[i-1].offset < offset && offset < f.size {
+		f.infos = append(f.infos, lineInfo{offset, filename, line})
+	}
+	f.set.mutex.Unlock()
+}
+
+// A File is a handle for a file belonging to a FileSet.
+// A File has a name, size, and line offset table.
+//
+type File struct {
+	set  *FileSet
+	name string // file name as provided to AddFile
+	base int    // Pos value range for this file is [base...base+size]
+	size int    // file size as provided to AddFile
+
+	// lines and infos are protected by set.mutex
+	lines []int
+	infos []lineInfo
+}
+
+// Name returns the file name of file f as registered with AddFile.
+func (f *File) Name() string {
+	return f.name
+}
+
+// Base returns the base offset of file f as registered with AddFile.
+func (f *File) Base() int {
+	return f.base
+}
+
+// Size returns the size of file f as registered with AddFile.
+func (f *File) Size() int {
+	return f.size
+}
+
+// LineCount returns the number of lines in file f.
+func (f *File) LineCount() int {
+	f.set.mutex.RLock()
+	n := len(f.lines)
+	f.set.mutex.RUnlock()
+	return n
+}
+
+// AddLine adds the line offset for a new line.
+// The line offset must be larger than the offset for the previous line
+// and smaller than the file size; otherwise the line offset is ignored.
+//
+func (f *File) AddLine(offset int) {
+	f.set.mutex.Lock()
+	if i := len(f.lines); (i == 0 || f.lines[i-1] < offset) && offset < f.size {
+		f.lines = append(f.lines, offset)
+	}
+	f.set.mutex.Unlock()
+}
+
+// SetLines sets the line offsets for a file and returns true if successful.
+// The line offsets are the offsets of the first character of each line;
+// for instance for the content "ab\nc\n" the line offsets are {0, 3}.
+// An empty file has an empty line offset table.
+// Each line offset must be larger than the offset for the previous line
+// and smaller than the file size; otherwise SetLines fails and returns
+// false.
+//
+func (f *File) SetLines(lines []int) bool {
+	// verify validity of lines table
+	size := f.size
+	for i, offset := range lines {
+		if i > 0 && offset <= lines[i-1] || size <= offset {
+			return false
+		}
+	}
+
+	// set lines table
+	f.set.mutex.Lock()
+	f.lines = lines
+	f.set.mutex.Unlock()
+	return true
+}
+
+// SetLinesForContent sets the line offsets for the given file content.
+func (f *File) SetLinesForContent(content []byte) {
+	var lines []int
+	line := 0
+	for offset, b := range content {
+		if line >= 0 {
+			lines = append(lines, line)
+		}
+		line = -1
+		if b == '\n' {
+			line = offset + 1
+		}
+	}
+
+	// set lines table
+	f.set.mutex.Lock()
+	f.lines = lines
+	f.set.mutex.Unlock()
+}
+
+// Pos returns the Pos value for the given file offset;
+// the offset must be <= f.Size().
+// f.Pos(f.Offset(p)) == p.
+//
+func (f *File) Pos(offset int) Pos {
+	if offset > f.size {
+		panic("illegal file offset")
+	}
+	return Pos(f.base + offset)
+}
+
+// Offset returns the offset for the given file position p;
+// p must be a valid Pos value in that file.
+// f.Offset(f.Pos(offset)) == offset.
+//
+func (f *File) Offset(p Pos) int {
+	if int(p) < f.base || int(p) > f.base+f.size {
+		panic("illegal Pos value")
+	}
+	return int(p) - f.base
+}
+
+// Line returns the line number for the given file position p;
+// p must be a Pos value in that file or NoPos.
+//
+func (f *File) Line(p Pos) int {
+	// TODO(gri) this can be implemented much more efficiently
+	return f.Position(p).Line
+}
+
+// Position returns the Position value for the given file position p;
+// p must be a Pos value in that file or NoPos.
+//
+func (f *File) Position(p Pos) (pos Position) {
+	if p != NoPos {
+		if int(p) < f.base || int(p) > f.base+f.size {
+			panic("illegal Pos value")
+		}
+		pos = f.position(p)
+	}
+	return
+}
+
+func searchInts(a []int, x int) int {
+	// This function body is a manually inlined version of:
+	//
+	//   return sort.Search(len(a), func(i int) bool { return a[i] > x }) - 1
+	//
+	// With better compiler optimizations, this may not be needed in the
+	// future, but at the moment this change improves the go/printer
+	// benchmark performance by ~30%. This has a direct impact on the
+	// speed of gofmt and thus seems worthwhile (2011-04-29).
+	i, j := 0, len(a)
+	for i < j {
+		h := i + (j-i)/2 // avoid overflow when computing h
+		// i ≤ h < j
+		if a[h] <= x {
+			i = h + 1
+		} else {
+			j = h
+		}
+	}
+	return i - 1
+}
+
+func searchLineInfos(a []lineInfo, x int) int {
+	return sort.Search(len(a), func(i int) bool { return a[i].offset > x }) - 1
+}
+
+// info returns the file name, line, and column number for a file offset.
+func (f *File) info(offset int) (filename string, line, column int) {
+	filename = f.name
+	if i := searchInts(f.lines, offset); i >= 0 {
+		line, column = i+1, offset-f.lines[i]+1
+	}
+	if len(f.infos) > 0 {
+		// almost no files have extra line infos
+		if i := searchLineInfos(f.infos, offset); i >= 0 {
+			alt := &f.infos[i]
+			filename = alt.filename
+			if i := searchInts(f.lines, alt.offset); i >= 0 {
+				line += alt.line - i - 1
+			}
+		}
+	}
+	return
+}
+
+// A FileSet represents a set of source files.
+// Methods of file sets are synchronized; multiple goroutines
+// may invoke them concurrently.
+//
+type FileSet struct {
+	mutex sync.RWMutex // protects the file set
+	base  int          // base offset for the next file
+	files []*File      // list of files in the order added to the set
+	last  *File        // cache of last file looked up
+}
+
+// NewFileSet creates a new file set.
+func NewFileSet() *FileSet {
+	s := new(FileSet)
+	s.base = 1 // 0 == NoPos
+	return s
+}
+
+// Base returns the minimum base offset that must be provided to
+// AddFile when adding the next file.
+//
+func (s *FileSet) Base() int {
+	s.mutex.RLock()
+	b := s.base
+	s.mutex.RUnlock()
+	return b
+
+}
+
+// AddFile adds a new file with a given filename, base offset, and file size
+// to the file set s and returns the file. Multiple files may have the same
+// name. The base offset must not be smaller than the FileSet's Base(), and
+// size must not be negative.
+//
+// Adding the file will set the file set's Base() value to base + size + 1
+// as the minimum base value for the next file. The following relationship
+// exists between a Pos value p for a given file offset offs:
+//
+//	int(p) = base + offs
+//
+// with offs in the range [0, size] and thus p in the range [base, base+size].
+// For convenience, File.Pos may be used to create file-specific position
+// values from a file offset.
+//
+func (s *FileSet) AddFile(filename string, base, size int) *File {
+	s.mutex.Lock()
+	defer s.mutex.Unlock()
+	if base < s.base || size < 0 {
+		panic("illegal base or size")
+	}
+	// base >= s.base && size >= 0
+	f := &File{s, filename, base, size, []int{0}, nil}
+	base += size + 1 // +1 because EOF also has a position
+	if base < 0 {
+		panic("token.Pos offset overflow (> 2G of source code in file set)")
+	}
+	// add the file to the file set
+	s.base = base
+	s.files = append(s.files, f)
+	s.last = f
+	return f
+}
+
+// Files returns the files added to the file set.
+func (s *FileSet) Files() <-chan *File {
+	ch := make(chan *File)
+	go func() {
+		for i := 0; ; i++ {
+			var f *File
+			s.mutex.RLock()
+			if i < len(s.files) {
+				f = s.files[i]
+			}
+			s.mutex.RUnlock()
+			if f == nil {
+				break
+			}
+			ch <- f
+		}
+		close(ch)
+	}()
+	return ch
+}
diff --git a/src/pkg/go/token/position_test.go b/src/pkg/go/token/position_test.go
new file mode 100644
index 000000000..30bec5991
--- /dev/null
+++ b/src/pkg/go/token/position_test.go
@@ -0,0 +1,180 @@
+// Copyright 2010 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package token
+
+import (
+	"fmt"
+	"testing"
+)
+
+func checkPos(t *testing.T, msg string, p, q Position) {
+	if p.Filename != q.Filename {
+		t.Errorf("%s: expected filename = %q; got %q", msg, q.Filename, p.Filename)
+	}
+	if p.Offset != q.Offset {
+		t.Errorf("%s: expected offset = %d; got %d", msg, q.Offset, p.Offset)
+	}
+	if p.Line != q.Line {
+		t.Errorf("%s: expected line = %d; got %d", msg, q.Line, p.Line)
+	}
+	if p.Column != q.Column {
+		t.Errorf("%s: expected column = %d; got %d", msg, q.Column, p.Column)
+	}
+}
+
+func TestNoPos(t *testing.T) {
+	if NoPos.IsValid() {
+		t.Errorf("NoPos should not be valid")
+	}
+	var fset *FileSet
+	checkPos(t, "nil NoPos", fset.Position(NoPos), Position{})
+	fset = NewFileSet()
+	checkPos(t, "fset NoPos", fset.Position(NoPos), Position{})
+}
+
+var tests = []struct {
+	filename string
+	source   []byte // may be nil
+	size     int
+	lines    []int
+}{
+	{"a", []byte{}, 0, []int{}},
+	{"b", []byte("01234"), 5, []int{0}},
+	{"c", []byte("\n\n\n\n\n\n\n\n\n"), 9, []int{0, 1, 2, 3, 4, 5, 6, 7, 8}},
+	{"d", nil, 100, []int{0, 5, 10, 20, 30, 70, 71, 72, 80, 85, 90, 99}},
+	{"e", nil, 777, []int{0, 80, 100, 120, 130, 180, 267, 455, 500, 567, 620}},
+	{"f", []byte("package p\n\nimport \"fmt\""), 23, []int{0, 10, 11}},
+	{"g", []byte("package p\n\nimport \"fmt\"\n"), 24, []int{0, 10, 11}},
+	{"h", []byte("package p\n\nimport \"fmt\"\n "), 25, []int{0, 10, 11, 24}},
+}
+
+func linecol(lines []int, offs int) (int, int) {
+	prevLineOffs := 0
+	for line, lineOffs := range lines {
+		if offs < lineOffs {
+			return line, offs - prevLineOffs + 1
+		}
+		prevLineOffs = lineOffs
+	}
+	return len(lines), offs - prevLineOffs + 1
+}
+
+func verifyPositions(t *testing.T, fset *FileSet, f *File, lines []int) {
+	for offs := 0; offs < f.Size(); offs++ {
+		p := f.Pos(offs)
+		offs2 := f.Offset(p)
+		if offs2 != offs {
+			t.Errorf("%s, Offset: expected offset %d; got %d", f.Name(), offs, offs2)
+		}
+		line, col := linecol(lines, offs)
+		msg := fmt.Sprintf("%s (offs = %d, p = %d)", f.Name(), offs, p)
+		checkPos(t, msg, f.Position(f.Pos(offs)), Position{f.Name(), offs, line, col})
+		checkPos(t, msg, fset.Position(p), Position{f.Name(), offs, line, col})
+	}
+}
+
+func makeTestSource(size int, lines []int) []byte {
+	src := make([]byte, size)
+	for _, offs := range lines {
+		if offs > 0 {
+			src[offs-1] = '\n'
+		}
+	}
+	return src
+}
+
+func TestPositions(t *testing.T) {
+	const delta = 7 // a non-zero base offset increment
+	fset := NewFileSet()
+	for _, test := range tests {
+		// verify consistency of test case
+		if test.source != nil && len(test.source) != test.size {
+			t.Errorf("%s: inconsistent test case: expected file size %d; got %d", test.filename, test.size, len(test.source))
+		}
+
+		// add file and verify name and size
+		f := fset.AddFile(test.filename, fset.Base()+delta, test.size)
+		if f.Name() != test.filename {
+			t.Errorf("expected filename %q; got %q", test.filename, f.Name())
+		}
+		if f.Size() != test.size {
+			t.Errorf("%s: expected file size %d; got %d", f.Name(), test.size, f.Size())
+		}
+		if fset.File(f.Pos(0)) != f {
+			t.Errorf("%s: f.Pos(0) was not found in f", f.Name())
+		}
+
+		// add lines individually and verify all positions
+		for i, offset := range test.lines {
+			f.AddLine(offset)
+			if f.LineCount() != i+1 {
+				t.Errorf("%s, AddLine: expected line count %d; got %d", f.Name(), i+1, f.LineCount())
+			}
+			// adding the same offset again should be ignored
+			f.AddLine(offset)
+			if f.LineCount() != i+1 {
+				t.Errorf("%s, AddLine: expected unchanged line count %d; got %d", f.Name(), i+1, f.LineCount())
+			}
+			verifyPositions(t, fset, f, test.lines[0:i+1])
+		}
+
+		// add lines with SetLines and verify all positions
+		if ok := f.SetLines(test.lines); !ok {
+			t.Errorf("%s: SetLines failed", f.Name())
+		}
+		if f.LineCount() != len(test.lines) {
+			t.Errorf("%s, SetLines: expected line count %d; got %d", f.Name(), len(test.lines), f.LineCount())
+		}
+		verifyPositions(t, fset, f, test.lines)
+
+		// add lines with SetLinesForContent and verify all positions
+		src := test.source
+		if src == nil {
+			// no test source available - create one from scratch
+			src = makeTestSource(test.size, test.lines)
+		}
+		f.SetLinesForContent(src)
+		if f.LineCount() != len(test.lines) {
+			t.Errorf("%s, SetLinesForContent: expected line count %d; got %d", f.Name(), len(test.lines), f.LineCount())
+		}
+		verifyPositions(t, fset, f, test.lines)
+	}
+}
+
+func TestLineInfo(t *testing.T) {
+	fset := NewFileSet()
+	f := fset.AddFile("foo", fset.Base(), 500)
+	lines := []int{0, 42, 77, 100, 210, 220, 277, 300, 333, 401}
+	// add lines individually and provide alternative line information
+	for _, offs := range lines {
+		f.AddLine(offs)
+		f.AddLineInfo(offs, "bar", 42)
+	}
+	// verify positions for all offsets
+	for offs := 0; offs <= f.Size(); offs++ {
+		p := f.Pos(offs)
+		_, col := linecol(lines, offs)
+		msg := fmt.Sprintf("%s (offs = %d, p = %d)", f.Name(), offs, p)
+		checkPos(t, msg, f.Position(f.Pos(offs)), Position{"bar", offs, 42, col})
+		checkPos(t, msg, fset.Position(p), Position{"bar", offs, 42, col})
+	}
+}
+
+func TestFiles(t *testing.T) {
+	fset := NewFileSet()
+	for i, test := range tests {
+		fset.AddFile(test.filename, fset.Base(), test.size)
+		j := 0
+		for g := range fset.Files() {
+			if g.Name() != tests[j].filename {
+				t.Errorf("expected filename = %s; got %s", tests[j].filename, g.Name())
+			}
+			j++
+		}
+		if j != i+1 {
+			t.Errorf("expected %d files; got %d", i+1, j)
+		}
+	}
+}
diff --git a/src/pkg/go/token/token.go b/src/pkg/go/token/token.go
new file mode 100644
index 000000000..557374052
--- /dev/null
+++ b/src/pkg/go/token/token.go
@@ -0,0 +1,310 @@
+// Copyright 2009 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// Package token defines constants representing the lexical tokens of the Go
+// programming language and basic operations on tokens (printing, predicates).
+//
+package token
+
+import "strconv"
+
+// Token is the set of lexical tokens of the Go programming language.
+type Token int
+
+// The list of tokens.
+const (
+	// Special tokens
+	ILLEGAL Token = iota
+	EOF
+	COMMENT
+
+	literal_beg
+	// Identifiers and basic type literals
+	// (these tokens stand for classes of literals)
+	IDENT  // main
+	INT    // 12345
+	FLOAT  // 123.45
+	IMAG   // 123.45i
+	CHAR   // 'a'
+	STRING // "abc"
+	literal_end
+
+	operator_beg
+	// Operators and delimiters
+	ADD // +
+	SUB // -
+	MUL // *
+	QUO // /
+	REM // %
+
+	AND     // &
+	OR      // |
+	XOR     // ^
+	SHL     // <<
+	SHR     // >>
+	AND_NOT // &^
+
+	ADD_ASSIGN // +=
+	SUB_ASSIGN // -=
+	MUL_ASSIGN // *=
+	QUO_ASSIGN // /=
+	REM_ASSIGN // %=
+
+	AND_ASSIGN     // &=
+	OR_ASSIGN      // |=
+	XOR_ASSIGN     // ^=
+	SHL_ASSIGN     // <<=
+	SHR_ASSIGN     // >>=
+	AND_NOT_ASSIGN // &^=
+
+	LAND  // &&
+	LOR   // ||
+	ARROW // <-
+	INC   // ++
+	DEC   // --
+
+	EQL    // ==
+	LSS    // <
+	GTR    // >
+	ASSIGN // =
+	NOT    // !
+
+	NEQ      // !=
+	LEQ      // <=
+	GEQ      // >=
+	DEFINE   // :=
+	ELLIPSIS // ...
+
+	LPAREN // (
+	LBRACK // [
+	LBRACE // {
+	COMMA  // ,
+	PERIOD // .
+
+	RPAREN    // )
+	RBRACK    // ]
+	RBRACE    // }
+	SEMICOLON // ;
+	COLON     // :
+	operator_end
+
+	keyword_beg
+	// Keywords
+	BREAK
+	CASE
+	CHAN
+	CONST
+	CONTINUE
+
+	DEFAULT
+	DEFER
+	ELSE
+	FALLTHROUGH
+	FOR
+
+	FUNC
+	GO
+	GOTO
+	IF
+	IMPORT
+
+	INTERFACE
+	MAP
+	PACKAGE
+	RANGE
+	RETURN
+
+	SELECT
+	STRUCT
+	SWITCH
+	TYPE
+	VAR
+	keyword_end
+)
+
+var tokens = [...]string{
+	ILLEGAL: "ILLEGAL",
+
+	EOF:     "EOF",
+	COMMENT: "COMMENT",
+
+	IDENT:  "IDENT",
+	INT:    "INT",
+	FLOAT:  "FLOAT",
+	IMAG:   "IMAG",
+	CHAR:   "CHAR",
+	STRING: "STRING",
+
+	ADD: "+",
+	SUB: "-",
+	MUL: "*",
+	QUO: "/",
+	REM: "%",
+
+	AND:     "&",
+	OR:      "|",
+	XOR:     "^",
+	SHL:     "<<",
+	SHR:     ">>",
+	AND_NOT: "&^",
+
+	ADD_ASSIGN: "+=",
+	SUB_ASSIGN: "-=",
+	MUL_ASSIGN: "*=",
+	QUO_ASSIGN: "/=",
+	REM_ASSIGN: "%=",
+
+	AND_ASSIGN:     "&=",
+	OR_ASSIGN:      "|=",
+	XOR_ASSIGN:     "^=",
+	SHL_ASSIGN:     "<<=",
+	SHR_ASSIGN:     ">>=",
+	AND_NOT_ASSIGN: "&^=",
+
+	LAND:  "&&",
+	LOR:   "||",
+	ARROW: "<-",
+	INC:   "++",
+	DEC:   "--",
+
+	EQL:    "==",
+	LSS:    "<",
+	GTR:    ">",
+	ASSIGN: "=",
+	NOT:    "!",
+
+	NEQ:      "!=",
+	LEQ:      "<=",
+	GEQ:      ">=",
+	DEFINE:   ":=",
+	ELLIPSIS: "...",
+
+	LPAREN: "(",
+	LBRACK: "[",
+	LBRACE: "{",
+	COMMA:  ",",
+	PERIOD: ".",
+
+	RPAREN:    ")",
+	RBRACK:    "]",
+	RBRACE:    "}",
+	SEMICOLON: ";",
+	COLON:     ":",
+
+	BREAK:    "break",
+	CASE:     "case",
+	CHAN:     "chan",
+	CONST:    "const",
+	CONTINUE: "continue",
+
+	DEFAULT:     "default",
+	DEFER:       "defer",
+	ELSE:        "else",
+	FALLTHROUGH: "fallthrough",
+	FOR:         "for",
+
+	FUNC:   "func",
+	GO:     "go",
+	GOTO:   "goto",
+	IF:     "if",
+	IMPORT: "import",
+
+	INTERFACE: "interface",
+	MAP:       "map",
+	PACKAGE:   "package",
+	RANGE:     "range",
+	RETURN:    "return",
+
+	SELECT: "select",
+	STRUCT: "struct",
+	SWITCH: "switch",
+	TYPE:   "type",
+	VAR:    "var",
+}
+
+// String returns the string corresponding to the token tok.
+// For operators, delimiters, and keywords the string is the actual
+// token character sequence (e.g., for the token ADD, the string is
+// "+"). For all other tokens the string corresponds to the token
+// constant name (e.g. for the token IDENT, the string is "IDENT").
+//
+func (tok Token) String() string {
+	s := ""
+	if 0 <= tok && tok < Token(len(tokens)) {
+		s = tokens[tok]
+	}
+	if s == "" {
+		s = "token(" + strconv.Itoa(int(tok)) + ")"
+	}
+	return s
+}
+
+// A set of constants for precedence-based expression parsing.
+// Non-operators have lowest precedence, followed by operators
+// starting with precedence 1 up to unary operators. The highest
+// precedence corresponds serves as "catch-all" precedence for
+// selector, indexing, and other operator and delimiter tokens.
+//
+const (
+	LowestPrec  = 0 // non-operators
+	UnaryPrec   = 6
+	HighestPrec = 7
+)
+
+// Precedence returns the operator precedence of the binary
+// operator op. If op is not a binary operator, the result
+// is LowestPrecedence.
+//
+func (op Token) Precedence() int {
+	switch op {
+	case LOR:
+		return 1
+	case LAND:
+		return 2
+	case EQL, NEQ, LSS, LEQ, GTR, GEQ:
+		return 3
+	case ADD, SUB, OR, XOR:
+		return 4
+	case MUL, QUO, REM, SHL, SHR, AND, AND_NOT:
+		return 5
+	}
+	return LowestPrec
+}
+
+var keywords map[string]Token
+
+func init() {
+	keywords = make(map[string]Token)
+	for i := keyword_beg + 1; i < keyword_end; i++ {
+		keywords[tokens[i]] = i
+	}
+}
+
+// Lookup maps an identifier to its keyword token or IDENT (if not a keyword).
+//
+func Lookup(ident []byte) Token {
+	// TODO Maps with []byte key are illegal because []byte does not
+	//      support == . Should find a more efficient solution eventually.
+	if tok, is_keyword := keywords[string(ident)]; is_keyword {
+		return tok
+	}
+	return IDENT
+}
+
+// Predicates
+
+// IsLiteral returns true for tokens corresponding to identifiers
+// and basic type literals; returns false otherwise.
+//
+func (tok Token) IsLiteral() bool { return literal_beg < tok && tok < literal_end }
+
+// IsOperator returns true for tokens corresponding to operators and
+// delimiters; returns false otherwise.
+//
+func (tok Token) IsOperator() bool { return operator_beg < tok && tok < operator_end }
+
+// IsKeyword returns true for tokens corresponding to keywords;
+// returns false otherwise.
+//
+func (tok Token) IsKeyword() bool { return keyword_beg < tok && tok < keyword_end }
diff --git a/src/pkg/go/typechecker/Makefile b/src/pkg/go/typechecker/Makefile
new file mode 100644
index 000000000..83af3ef4e
--- /dev/null
+++ b/src/pkg/go/typechecker/Makefile
@@ -0,0 +1,14 @@
+# Copyright 2010 The Go Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style
+# license that can be found in the LICENSE file.
+
+include ../../../Make.inc
+
+TARG=go/typechecker
+GOFILES=\
+	scope.go\
+	type.go\
+	typechecker.go\
+	universe.go\
+
+include ../../../Make.pkg
diff --git a/src/pkg/go/typechecker/scope.go b/src/pkg/go/typechecker/scope.go
new file mode 100644
index 000000000..d73d1a450
--- /dev/null
+++ b/src/pkg/go/typechecker/scope.go
@@ -0,0 +1,69 @@
+// Copyright 2010 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// DEPRECATED FILE - WILL GO AWAY EVENTUALLY.
+//
+// Scope handling is now done in go/parser.
+// The functionality here is only present to
+// keep the typechecker running for now.
+
+package typechecker
+
+import "go/ast"
+
+func (tc *typechecker) openScope() *ast.Scope {
+	tc.topScope = ast.NewScope(tc.topScope)
+	return tc.topScope
+}
+
+func (tc *typechecker) closeScope() {
+	tc.topScope = tc.topScope.Outer
+}
+
+// declInScope declares an object of a given kind and name in scope and sets the object's Decl and N fields.
+// It returns the newly allocated object. If an object with the same name already exists in scope, an error
+// is reported and the object is not inserted.
+func (tc *typechecker) declInScope(scope *ast.Scope, kind ast.ObjKind, name *ast.Ident, decl interface{}, n int) *ast.Object {
+	obj := ast.NewObj(kind, name.Name)
+	obj.Decl = decl
+	//obj.N = n
+	name.Obj = obj
+	if name.Name != "_" {
+		if alt := scope.Insert(obj); alt != nil {
+			tc.Errorf(name.Pos(), "%s already declared at %s", name.Name, tc.fset.Position(alt.Pos()).String())
+		}
+	}
+	return obj
+}
+
+// decl is the same as declInScope(tc.topScope, ...)
+func (tc *typechecker) decl(kind ast.ObjKind, name *ast.Ident, decl interface{}, n int) *ast.Object {
+	return tc.declInScope(tc.topScope, kind, name, decl, n)
+}
+
+// find returns the object with the given name if visible in the current scope hierarchy.
+// If no such object is found, an error is reported and a bad object is returned instead.
+func (tc *typechecker) find(name *ast.Ident) (obj *ast.Object) {
+	for s := tc.topScope; s != nil && obj == nil; s = s.Outer {
+		obj = s.Lookup(name.Name)
+	}
+	if obj == nil {
+		tc.Errorf(name.Pos(), "%s not declared", name.Name)
+		obj = ast.NewObj(ast.Bad, name.Name)
+	}
+	name.Obj = obj
+	return
+}
+
+// findField returns the object with the given name if visible in the type's scope.
+// If no such object is found, an error is reported and a bad object is returned instead.
+func (tc *typechecker) findField(typ *Type, name *ast.Ident) (obj *ast.Object) {
+	// TODO(gri) This is simplistic at the moment and ignores anonymous fields.
+	obj = typ.Scope.Lookup(name.Name)
+	if obj == nil {
+		tc.Errorf(name.Pos(), "%s not declared", name.Name)
+		obj = ast.NewObj(ast.Bad, name.Name)
+	}
+	return
+}
diff --git a/src/pkg/go/typechecker/testdata/test0.src b/src/pkg/go/typechecker/testdata/test0.src
new file mode 100644
index 000000000..4e317f214
--- /dev/null
+++ b/src/pkg/go/typechecker/testdata/test0.src
@@ -0,0 +1,94 @@
+// Copyright 2010 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// type declarations
+
+package P0
+
+type (
+	B bool
+	I int32
+	A [10]P
+	T struct {
+		x, y P
+	}
+	P *T
+	R *R
+	F func(A) I
+	Y interface {
+		f(A) I
+	}
+	S []P
+	M map[I]F
+	C chan<- I
+)
+
+type (
+	a/* ERROR "illegal cycle" */ a
+	a/* ERROR "already declared" */ int
+
+	b/* ERROR "illegal cycle" */ c
+	c d
+	d e
+	e b /* ERROR "not a type" */
+
+	t *t
+
+	U V
+	V W
+	W *U
+
+	P1 *S2
+	P2 P1
+
+	S1 struct {
+		a, b, c int
+		u, v, a/* ERROR "already declared" */ float
+	}
+	S2/* ERROR "illegal cycle" */ struct {
+		x S2
+	}
+
+	L1 []L1
+	L2 []int
+
+	A1 [10]int
+	A2/* ERROR "illegal cycle" */ [10]A2
+	A3/* ERROR "illegal cycle" */ [10]struct {
+		x A4
+	}
+	A4 [10]A3
+
+	F1 func()
+	F2 func(x, y, z float)
+	F3 func(x, y, x /* ERROR "already declared" */ float)
+	F4 func() (x, y, x /* ERROR "already declared" */ float)
+	F5 func(x int) (x /* ERROR "already declared" */ float)
+
+	I1 interface{}
+	I2 interface {
+		m1()
+	}
+	I3 interface {
+		m1()
+		m1 /* ERROR "already declared" */ ()
+	}
+	I4 interface {
+		m1(x, y, x /* ERROR "already declared" */ float)
+		m2() (x, y, x /* ERROR "already declared" */ float)
+		m3(x int) (x /* ERROR "already declared" */ float)
+	}
+	I5 interface {
+		m1(I5)
+	}
+
+	C1 chan int
+	C2 <-chan int
+	C3 chan<- C3
+
+	M1 map[Last]string
+	M2 map[string]M2
+
+	Last int
+)
diff --git a/src/pkg/go/typechecker/testdata/test1.src b/src/pkg/go/typechecker/testdata/test1.src
new file mode 100644
index 000000000..b5531fb9f
--- /dev/null
+++ b/src/pkg/go/typechecker/testdata/test1.src
@@ -0,0 +1,13 @@
+// Copyright 2010 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// const and var declarations
+
+package P1
+
+const (
+	c1 = 0
+	c2     int = 0
+	c3, c4 = 0
+)
diff --git a/src/pkg/go/typechecker/testdata/test3.src b/src/pkg/go/typechecker/testdata/test3.src
new file mode 100644
index 000000000..2e1a9fa8f
--- /dev/null
+++ b/src/pkg/go/typechecker/testdata/test3.src
@@ -0,0 +1,41 @@
+// Copyright 2010 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package P3
+
+// function and method signatures
+
+func _()                                        {}
+func _()                                        {}
+func _(x, x /* ERROR "already declared" */ int) {}
+
+func f()                                 {}
+func f /* ERROR "already declared" */ () {}
+
+func (*foo /* ERROR "invalid receiver" */ ) m() {}
+func (bar /* ERROR "not a type" */ ) m()        {}
+
+func f1(x, _, _ int) (_, _ float)                              {}
+func f2(x, y, x /* ERROR "already declared" */ int)            {}
+func f3(x, y int) (a, b, x /* ERROR "already declared" */ int) {}
+
+func (x *T) m1()                                 {}
+func (x *T) m1 /* ERROR "already declared" */ () {}
+func (x T) m1 /* ERROR "already declared" */ ()  {}
+func (T) m1 /* ERROR "already declared" */ ()    {}
+
+func (x *T) m2(u, x /* ERROR "already declared" */ int)               {}
+func (x *T) m3(a, b, c int) (u, x /* ERROR "already declared" */ int) {}
+// The following are disabled for now because the typechecker
+// in in the process of being rewritten and cannot handle them
+// at the moment
+//func (T) _(x, x /* "already declared" */ int)                   {}
+//func (T) _() (x, x /* "already declared" */ int)                {}
+
+//func (PT) _() {}
+
+var bar int
+
+type T struct{}
+type PT (T)
diff --git a/src/pkg/go/typechecker/testdata/test4.src b/src/pkg/go/typechecker/testdata/test4.src
new file mode 100644
index 000000000..94d3558f9
--- /dev/null
+++ b/src/pkg/go/typechecker/testdata/test4.src
@@ -0,0 +1,11 @@
+// Copyright 2010 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// Constant declarations
+
+package P4
+
+const (
+	c0 = 0
+)
diff --git a/src/pkg/go/typechecker/type.go b/src/pkg/go/typechecker/type.go
new file mode 100644
index 000000000..1b88eb54b
--- /dev/null
+++ b/src/pkg/go/typechecker/type.go
@@ -0,0 +1,118 @@
+// Copyright 2010 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package typechecker
+
+import "go/ast"
+
+// A Type represents a Go type.
+type Type struct {
+	Form     Form
+	Obj      *ast.Object // corresponding type name, or nil
+	Scope    *ast.Scope  // fields and methods, always present
+	N        uint        // basic type id, array length, number of function results, or channel direction
+	Key, Elt *Type       // map key and array, pointer, slice, map or channel element
+	Params   *ast.Scope  // function (receiver, input and result) parameters, tuple expressions (results of function calls), or nil
+	Expr     ast.Expr    // corresponding AST expression
+}
+
+// NewType creates a new type of a given form.
+func NewType(form Form) *Type {
+	return &Type{Form: form, Scope: ast.NewScope(nil)}
+}
+
+// Form describes the form of a type.
+type Form int
+
+// The list of possible type forms.
+const (
+	BadType    Form = iota // for error handling
+	Unresolved             // type not fully setup
+	Basic
+	Array
+	Struct
+	Pointer
+	Function
+	Method
+	Interface
+	Slice
+	Map
+	Channel
+	Tuple
+)
+
+var formStrings = [...]string{
+	BadType:    "badType",
+	Unresolved: "unresolved",
+	Basic:      "basic",
+	Array:      "array",
+	Struct:     "struct",
+	Pointer:    "pointer",
+	Function:   "function",
+	Method:     "method",
+	Interface:  "interface",
+	Slice:      "slice",
+	Map:        "map",
+	Channel:    "channel",
+	Tuple:      "tuple",
+}
+
+func (form Form) String() string { return formStrings[form] }
+
+// The list of basic type id's.
+const (
+	Bool = iota
+	Byte
+	Uint
+	Int
+	Float
+	Complex
+	Uintptr
+	String
+
+	Uint8
+	Uint16
+	Uint32
+	Uint64
+
+	Int8
+	Int16
+	Int32
+	Int64
+
+	Float32
+	Float64
+
+	Complex64
+	Complex128
+
+	// TODO(gri) ideal types are missing
+)
+
+var BasicTypes = map[uint]string{
+	Bool:    "bool",
+	Byte:    "byte",
+	Uint:    "uint",
+	Int:     "int",
+	Float:   "float",
+	Complex: "complex",
+	Uintptr: "uintptr",
+	String:  "string",
+
+	Uint8:  "uint8",
+	Uint16: "uint16",
+	Uint32: "uint32",
+	Uint64: "uint64",
+
+	Int8:  "int8",
+	Int16: "int16",
+	Int32: "int32",
+	Int64: "int64",
+
+	Float32: "float32",
+	Float64: "float64",
+
+	Complex64:  "complex64",
+	Complex128: "complex128",
+}
diff --git a/src/pkg/go/typechecker/typechecker.go b/src/pkg/go/typechecker/typechecker.go
new file mode 100644
index 000000000..24480165b
--- /dev/null
+++ b/src/pkg/go/typechecker/typechecker.go
@@ -0,0 +1,468 @@
+// Copyright 2010 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// DEPRECATED PACKAGE - SEE go/types INSTEAD.
+// This package implements typechecking of a Go AST.
+// The result of the typecheck is an augmented AST
+// with object and type information for each identifier.
+//
+package typechecker
+
+import (
+	"fmt"
+	"go/ast"
+	"go/token"
+	"go/scanner"
+	"os"
+)
+
+// TODO(gri) don't report errors for objects/types that are marked as bad.
+
+
+const debug = true // set for debugging output
+
+// An importer takes an import path and returns the data describing the
+// respective package's exported interface. The data format is TBD.
+//
+type Importer func(path string) ([]byte, os.Error)
+
+// CheckPackage typechecks a package and augments the AST by setting
+// *ast.Object, *ast.Type, and *ast.Scope fields accordingly. If an
+// importer is provided, it is used to handle imports, otherwise they
+// are ignored (likely leading to typechecking errors).
+//
+// If errors are reported, the AST may be incompletely augmented (fields
+// may be nil) or contain incomplete object, type, or scope information.
+//
+func CheckPackage(fset *token.FileSet, pkg *ast.Package, importer Importer) os.Error {
+	var tc typechecker
+	tc.fset = fset
+	tc.importer = importer
+	tc.checkPackage(pkg)
+	return tc.GetError(scanner.Sorted)
+}
+
+// CheckFile typechecks a single file, but otherwise behaves like
+// CheckPackage. If the complete package consists of more than just
+// one file, the file may not typecheck without errors.
+//
+func CheckFile(fset *token.FileSet, file *ast.File, importer Importer) os.Error {
+	// create a single-file dummy package
+	pkg := &ast.Package{file.Name.Name, nil, nil, map[string]*ast.File{fset.Position(file.Name.NamePos).Filename: file}}
+	return CheckPackage(fset, pkg, importer)
+}
+
+// ----------------------------------------------------------------------------
+// Typechecker state
+
+type typechecker struct {
+	fset *token.FileSet
+	scanner.ErrorVector
+	importer Importer
+	globals  []*ast.Object        // list of global objects
+	topScope *ast.Scope           // current top-most scope
+	cyclemap map[*ast.Object]bool // for cycle detection
+	iota     int                  // current value of iota
+}
+
+func (tc *typechecker) Errorf(pos token.Pos, format string, args ...interface{}) {
+	tc.Error(tc.fset.Position(pos), fmt.Sprintf(format, args...))
+}
+
+func assert(pred bool) {
+	if !pred {
+		panic("internal error")
+	}
+}
+
+/*
+Typechecking is done in several phases:
+
+phase 1: declare all global objects; also collect all function and method declarations
+	- all objects have kind, name, decl fields; the decl field permits
+	  quick lookup of an object's declaration
+	- constant objects have an iota value
+	- type objects have unresolved types with empty scopes, all others have nil types
+	- report global double declarations
+
+phase 2: bind methods to their receiver base types
+	- receiver base types must be declared in the package, thus for
+	  each method a corresponding (unresolved) type must exist
+	- report method double declarations and errors with base types
+
+phase 3: resolve all global objects
+	- sequentially iterate through all objects in the global scope
+	- resolve types for all unresolved types and assign types to
+	  all attached methods
+	- assign types to all other objects, possibly by evaluating
+	  constant and initializer expressions
+	- resolution may recurse; a cyclemap is used to detect cycles
+	- report global typing errors
+
+phase 4: sequentially typecheck function and method bodies
+	- all global objects are declared and have types and values;
+	  all methods have types
+	- sequentially process statements in each body; any object
+	  referred to must be fully defined at this point
+	- report local typing errors
+*/
+
+func (tc *typechecker) checkPackage(pkg *ast.Package) {
+	// setup package scope
+	tc.topScope = Universe
+	tc.openScope()
+	defer tc.closeScope()
+
+	// TODO(gri) there's no file scope at the moment since we ignore imports
+
+	// phase 1: declare all global objects; also collect all function and method declarations
+	var funcs []*ast.FuncDecl
+	for _, file := range pkg.Files {
+		for _, decl := range file.Decls {
+			tc.declGlobal(decl)
+			if f, isFunc := decl.(*ast.FuncDecl); isFunc {
+				funcs = append(funcs, f)
+			}
+		}
+	}
+
+	// phase 2: bind methods to their receiver base types
+	for _, m := range funcs {
+		if m.Recv != nil {
+			tc.bindMethod(m)
+		}
+	}
+
+	// phase 3: resolve all global objects
+	tc.cyclemap = make(map[*ast.Object]bool)
+	for _, obj := range tc.globals {
+		tc.resolve(obj)
+	}
+	assert(len(tc.cyclemap) == 0)
+
+	// 4: sequentially typecheck function and method bodies
+	for _, f := range funcs {
+		ftype, _ := f.Name.Obj.Type.(*Type)
+		tc.checkBlock(f.Body.List, ftype)
+	}
+
+	pkg.Scope = tc.topScope
+}
+
+func (tc *typechecker) declGlobal(global ast.Decl) {
+	switch d := global.(type) {
+	case *ast.BadDecl:
+		// ignore
+
+	case *ast.GenDecl:
+		iota := 0
+		var prev *ast.ValueSpec
+		for _, spec := range d.Specs {
+			switch s := spec.(type) {
+			case *ast.ImportSpec:
+				// TODO(gri) imports go into file scope
+			case *ast.ValueSpec:
+				switch d.Tok {
+				case token.CONST:
+					if s.Values == nil {
+						// create a new spec with type and values from the previous one
+						if prev != nil {
+							s = &ast.ValueSpec{s.Doc, s.Names, prev.Type, prev.Values, s.Comment}
+						} else {
+							// TODO(gri) this should probably go into the const decl code
+							tc.Errorf(s.Pos(), "missing initializer for const %s", s.Names[0].Name)
+						}
+					}
+					for _, name := range s.Names {
+						tc.globals = append(tc.globals, tc.decl(ast.Con, name, s, iota))
+					}
+				case token.VAR:
+					for _, name := range s.Names {
+						tc.globals = append(tc.globals, tc.decl(ast.Var, name, s, 0))
+					}
+				default:
+					panic("unreachable")
+				}
+				prev = s
+				iota++
+			case *ast.TypeSpec:
+				obj := tc.decl(ast.Typ, s.Name, s, 0)
+				tc.globals = append(tc.globals, obj)
+				// give all type objects an unresolved type so
+				// that we can collect methods in the type scope
+				typ := NewType(Unresolved)
+				obj.Type = typ
+				typ.Obj = obj
+			default:
+				panic("unreachable")
+			}
+		}
+
+	case *ast.FuncDecl:
+		if d.Recv == nil {
+			tc.globals = append(tc.globals, tc.decl(ast.Fun, d.Name, d, 0))
+		}
+
+	default:
+		panic("unreachable")
+	}
+}
+
+// If x is of the form *T, deref returns T, otherwise it returns x.
+func deref(x ast.Expr) ast.Expr {
+	if p, isPtr := x.(*ast.StarExpr); isPtr {
+		x = p.X
+	}
+	return x
+}
+
+func (tc *typechecker) bindMethod(method *ast.FuncDecl) {
+	// a method is declared in the receiver base type's scope
+	var scope *ast.Scope
+	base := deref(method.Recv.List[0].Type)
+	if name, isIdent := base.(*ast.Ident); isIdent {
+		// if base is not an *ast.Ident, we had a syntax
+		// error and the parser reported an error already
+		obj := tc.topScope.Lookup(name.Name)
+		if obj == nil {
+			tc.Errorf(name.Pos(), "invalid receiver: %s is not declared in this package", name.Name)
+		} else if obj.Kind != ast.Typ {
+			tc.Errorf(name.Pos(), "invalid receiver: %s is not a type", name.Name)
+		} else {
+			typ := obj.Type.(*Type)
+			assert(typ.Form == Unresolved)
+			scope = typ.Scope
+		}
+	}
+	if scope == nil {
+		// no receiver type found; use a dummy scope
+		// (we still want to type-check the method
+		// body, so make sure there is a name object
+		// and type)
+		// TODO(gri) should we record the scope so
+		// that we don't lose the receiver for type-
+		// checking of the method body?
+		scope = ast.NewScope(nil)
+	}
+	tc.declInScope(scope, ast.Fun, method.Name, method, 0)
+}
+
+func (tc *typechecker) resolve(obj *ast.Object) {
+	// check for declaration cycles
+	if tc.cyclemap[obj] {
+		tc.Errorf(obj.Pos(), "illegal cycle in declaration of %s", obj.Name)
+		obj.Kind = ast.Bad
+		return
+	}
+	tc.cyclemap[obj] = true
+	defer func() {
+		tc.cyclemap[obj] = false, false
+	}()
+
+	// resolve non-type objects
+	typ, _ := obj.Type.(*Type)
+	if typ == nil {
+		switch obj.Kind {
+		case ast.Bad:
+			// ignore
+
+		case ast.Con:
+			tc.declConst(obj)
+
+		case ast.Var:
+			tc.declVar(obj)
+			obj.Type = tc.typeFor(nil, obj.Decl.(*ast.ValueSpec).Type, false)
+
+		case ast.Fun:
+			obj.Type = NewType(Function)
+			t := obj.Decl.(*ast.FuncDecl).Type
+			tc.declSignature(obj.Type.(*Type), nil, t.Params, t.Results)
+
+		default:
+			// type objects have non-nil types when resolve is called
+			if debug {
+				fmt.Printf("kind = %s\n", obj.Kind)
+			}
+			panic("unreachable")
+		}
+		return
+	}
+
+	// resolve type objects
+	if typ.Form == Unresolved {
+		tc.typeFor(typ, typ.Obj.Decl.(*ast.TypeSpec).Type, false)
+
+		// provide types for all methods
+		for _, obj := range typ.Scope.Objects {
+			if obj.Kind == ast.Fun {
+				assert(obj.Type == nil)
+				obj.Type = NewType(Method)
+				f := obj.Decl.(*ast.FuncDecl)
+				t := f.Type
+				tc.declSignature(obj.Type.(*Type), f.Recv, t.Params, t.Results)
+			}
+		}
+	}
+}
+
+func (tc *typechecker) checkBlock(body []ast.Stmt, ftype *Type) {
+	tc.openScope()
+	defer tc.closeScope()
+
+	// inject function/method parameters into block scope, if any
+	if ftype != nil {
+		for _, par := range ftype.Params.Objects {
+			if par.Name != "_" {
+				alt := tc.topScope.Insert(par)
+				assert(alt == nil) // ftype has no double declarations
+			}
+		}
+	}
+
+	for _, stmt := range body {
+		tc.checkStmt(stmt)
+	}
+}
+
+// ----------------------------------------------------------------------------
+// Types
+
+// unparen removes parentheses around x, if any.
+func unparen(x ast.Expr) ast.Expr {
+	if ux, hasParens := x.(*ast.ParenExpr); hasParens {
+		return unparen(ux.X)
+	}
+	return x
+}
+
+func (tc *typechecker) declFields(scope *ast.Scope, fields *ast.FieldList, ref bool) (n uint) {
+	if fields != nil {
+		for _, f := range fields.List {
+			typ := tc.typeFor(nil, f.Type, ref)
+			for _, name := range f.Names {
+				fld := tc.declInScope(scope, ast.Var, name, f, 0)
+				fld.Type = typ
+				n++
+			}
+		}
+	}
+	return n
+}
+
+func (tc *typechecker) declSignature(typ *Type, recv, params, results *ast.FieldList) {
+	assert((typ.Form == Method) == (recv != nil))
+	typ.Params = ast.NewScope(nil)
+	tc.declFields(typ.Params, recv, true)
+	tc.declFields(typ.Params, params, true)
+	typ.N = tc.declFields(typ.Params, results, true)
+}
+
+func (tc *typechecker) typeFor(def *Type, x ast.Expr, ref bool) (typ *Type) {
+	x = unparen(x)
+
+	// type name
+	if t, isIdent := x.(*ast.Ident); isIdent {
+		obj := tc.find(t)
+
+		if obj.Kind != ast.Typ {
+			tc.Errorf(t.Pos(), "%s is not a type", t.Name)
+			if def == nil {
+				typ = NewType(BadType)
+			} else {
+				typ = def
+				typ.Form = BadType
+			}
+			typ.Expr = x
+			return
+		}
+
+		if !ref {
+			tc.resolve(obj) // check for cycles even if type resolved
+		}
+		typ = obj.Type.(*Type)
+
+		if def != nil {
+			// new type declaration: copy type structure
+			def.Form = typ.Form
+			def.N = typ.N
+			def.Key, def.Elt = typ.Key, typ.Elt
+			def.Params = typ.Params
+			def.Expr = x
+			typ = def
+		}
+		return
+	}
+
+	// type literal
+	typ = def
+	if typ == nil {
+		typ = NewType(BadType)
+	}
+	typ.Expr = x
+
+	switch t := x.(type) {
+	case *ast.SelectorExpr:
+		if debug {
+			fmt.Println("qualified identifier unimplemented")
+		}
+		typ.Form = BadType
+
+	case *ast.StarExpr:
+		typ.Form = Pointer
+		typ.Elt = tc.typeFor(nil, t.X, true)
+
+	case *ast.ArrayType:
+		if t.Len != nil {
+			typ.Form = Array
+			// TODO(gri) compute the real length
+			// (this may call resolve recursively)
+			(*typ).N = 42
+		} else {
+			typ.Form = Slice
+		}
+		typ.Elt = tc.typeFor(nil, t.Elt, t.Len == nil)
+
+	case *ast.StructType:
+		typ.Form = Struct
+		tc.declFields(typ.Scope, t.Fields, false)
+
+	case *ast.FuncType:
+		typ.Form = Function
+		tc.declSignature(typ, nil, t.Params, t.Results)
+
+	case *ast.InterfaceType:
+		typ.Form = Interface
+		tc.declFields(typ.Scope, t.Methods, true)
+
+	case *ast.MapType:
+		typ.Form = Map
+		typ.Key = tc.typeFor(nil, t.Key, true)
+		typ.Elt = tc.typeFor(nil, t.Value, true)
+
+	case *ast.ChanType:
+		typ.Form = Channel
+		typ.N = uint(t.Dir)
+		typ.Elt = tc.typeFor(nil, t.Value, true)
+
+	default:
+		if debug {
+			fmt.Printf("x is %T\n", x)
+		}
+		panic("unreachable")
+	}
+
+	return
+}
+
+// ----------------------------------------------------------------------------
+// TODO(gri) implement these place holders
+
+func (tc *typechecker) declConst(*ast.Object) {
+}
+
+func (tc *typechecker) declVar(*ast.Object) {
+}
+
+func (tc *typechecker) checkStmt(ast.Stmt) {
+}
diff --git a/src/pkg/go/typechecker/typechecker_test.go b/src/pkg/go/typechecker/typechecker_test.go
new file mode 100644
index 000000000..4bad4499a
--- /dev/null
+++ b/src/pkg/go/typechecker/typechecker_test.go
@@ -0,0 +1,163 @@
+// Copyright 2010 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// This file implements a simple typechecker test harness. Packages found
+// in the testDir directory are typechecked. Error messages reported by
+// the typechecker are compared against the error messages expected for
+// the test files.
+//
+// Expected errors are indicated in the test files by putting a comment
+// of the form /* ERROR "rx" */ immediately following an offending token.
+// The harness will verify that an error matching the regular expression
+// rx is reported at that source position. Consecutive comments may be
+// used to indicate multiple errors for the same token position.
+//
+// For instance, the following test file indicates that a "not declared"
+// error should be reported for the undeclared variable x:
+//
+//	package P0
+//	func f() {
+//		_ = x /* ERROR "not declared" */ + 1
+//	}
+// 
+// If the -pkg flag is set, only packages with package names matching
+// the regular expression provided via the flag value are tested.
+
+package typechecker
+
+import (
+	"flag"
+	"fmt"
+	"go/ast"
+	"go/parser"
+	"go/scanner"
+	"go/token"
+	"io/ioutil"
+	"os"
+	"regexp"
+	"sort"
+	"strings"
+	"testing"
+)
+
+const testDir = "./testdata" // location of test packages
+
+var fset = token.NewFileSet()
+
+var (
+	pkgPat = flag.String("pkg", ".*", "regular expression to select test packages by package name")
+	trace  = flag.Bool("trace", false, "print package names")
+)
+
+// ERROR comments must be of the form /* ERROR "rx" */ and rx is
+// a regular expression that matches the expected error message.
+var errRx = regexp.MustCompile(`^/\* *ERROR *"([^"]*)" *\*/$`)
+
+// expectedErrors collects the regular expressions of ERROR comments
+// found in the package files of pkg and returns them in sorted order
+// (by filename and position).
+func expectedErrors(t *testing.T, pkg *ast.Package) (list scanner.ErrorList) {
+	// scan all package files
+	for filename := range pkg.Files {
+		src, err := ioutil.ReadFile(filename)
+		if err != nil {
+			t.Fatalf("expectedErrors(%s): %v", pkg.Name, err)
+		}
+
+		var s scanner.Scanner
+		file := fset.AddFile(filename, fset.Base(), len(src))
+		s.Init(file, src, nil, scanner.ScanComments)
+		var prev token.Pos // position of last non-comment token
+	loop:
+		for {
+			pos, tok, lit := s.Scan()
+			switch tok {
+			case token.EOF:
+				break loop
+			case token.COMMENT:
+				s := errRx.FindStringSubmatch(lit)
+				if len(s) == 2 {
+					list = append(list, &scanner.Error{fset.Position(prev), string(s[1])})
+				}
+			default:
+				prev = pos
+			}
+		}
+	}
+	sort.Sort(list) // multiple files may not be sorted
+	return
+}
+
+func testFilter(f *os.FileInfo) bool {
+	return strings.HasSuffix(f.Name, ".src") && f.Name[0] != '.'
+}
+
+func checkError(t *testing.T, expected, found *scanner.Error) {
+	rx, err := regexp.Compile(expected.Msg)
+	if err != nil {
+		t.Errorf("%s: %v", expected.Pos, err)
+		return
+	}
+
+	match := rx.MatchString(found.Msg)
+
+	if expected.Pos.Offset != found.Pos.Offset {
+		if match {
+			t.Errorf("%s: expected error should have been at %s", expected.Pos, found.Pos)
+		} else {
+			t.Errorf("%s: error matching %q expected", expected.Pos, expected.Msg)
+			return
+		}
+	}
+
+	if !match {
+		t.Errorf("%s: %q does not match %q", expected.Pos, expected.Msg, found.Msg)
+	}
+}
+
+func TestTypeCheck(t *testing.T) {
+	flag.Parse()
+	pkgRx, err := regexp.Compile(*pkgPat)
+	if err != nil {
+		t.Fatalf("illegal flag value %q: %s", *pkgPat, err)
+	}
+
+	pkgs, err := parser.ParseDir(fset, testDir, testFilter, 0)
+	if err != nil {
+		scanner.PrintError(os.Stderr, err)
+		t.Fatalf("packages in %s contain syntax errors", testDir)
+	}
+
+	for _, pkg := range pkgs {
+		if !pkgRx.MatchString(pkg.Name) {
+			continue // only test selected packages
+		}
+
+		if *trace {
+			fmt.Println(pkg.Name)
+		}
+
+		xlist := expectedErrors(t, pkg)
+		err := CheckPackage(fset, pkg, nil)
+		if err != nil {
+			if elist, ok := err.(scanner.ErrorList); ok {
+				// verify that errors match
+				for i := 0; i < len(xlist) && i < len(elist); i++ {
+					checkError(t, xlist[i], elist[i])
+				}
+				// the correct number or errors must have been found
+				if len(xlist) != len(elist) {
+					fmt.Fprintf(os.Stderr, "%s\n", pkg.Name)
+					scanner.PrintError(os.Stderr, elist)
+					fmt.Fprintln(os.Stderr)
+					t.Errorf("TypeCheck(%s): %d errors expected but %d reported", pkg.Name, len(xlist), len(elist))
+				}
+			} else {
+				t.Errorf("TypeCheck(%s): %v", pkg.Name, err)
+			}
+		} else if len(xlist) > 0 {
+			t.Errorf("TypeCheck(%s): %d errors expected but 0 reported", pkg.Name, len(xlist))
+		}
+	}
+}
diff --git a/src/pkg/go/typechecker/universe.go b/src/pkg/go/typechecker/universe.go
new file mode 100644
index 000000000..81c14a05e
--- /dev/null
+++ b/src/pkg/go/typechecker/universe.go
@@ -0,0 +1,36 @@
+// Copyright 2010 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package typechecker
+
+import "go/ast"
+
+// TODO(gri) should this be in package ast?
+
+// The Universe scope contains all predeclared identifiers.
+var Universe *ast.Scope
+
+func def(obj *ast.Object) {
+	alt := Universe.Insert(obj)
+	if alt != nil {
+		panic("object declared twice")
+	}
+}
+
+func init() {
+	Universe = ast.NewScope(nil)
+
+	// basic types
+	for n, name := range BasicTypes {
+		typ := NewType(Basic)
+		typ.N = n
+		obj := ast.NewObj(ast.Typ, name)
+		obj.Type = typ
+		typ.Obj = obj
+		def(obj)
+	}
+
+	// built-in functions
+	// TODO(gri) implement this
+}
diff --git a/src/pkg/go/types/Makefile b/src/pkg/go/types/Makefile
new file mode 100644
index 000000000..4ca707c73
--- /dev/null
+++ b/src/pkg/go/types/Makefile
@@ -0,0 +1,16 @@
+# Copyright 2010 The Go Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style
+# license that can be found in the LICENSE file.
+
+include ../../../Make.inc
+
+TARG=go/types
+GOFILES=\
+	check.go\
+	const.go\
+	exportdata.go\
+	gcimporter.go\
+	types.go\
+	universe.go\
+
+include ../../../Make.pkg
diff --git a/src/pkg/go/types/check.go b/src/pkg/go/types/check.go
new file mode 100644
index 000000000..87e3e93da
--- /dev/null
+++ b/src/pkg/go/types/check.go
@@ -0,0 +1,226 @@
+// Copyright 2011 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// This file implements the Check function, which typechecks a package.
+
+package types
+
+import (
+	"fmt"
+	"go/ast"
+	"go/scanner"
+	"go/token"
+	"os"
+	"strconv"
+)
+
+const debug = false
+
+type checker struct {
+	fset *token.FileSet
+	scanner.ErrorVector
+	types map[ast.Expr]Type
+}
+
+func (c *checker) errorf(pos token.Pos, format string, args ...interface{}) string {
+	msg := fmt.Sprintf(format, args...)
+	c.Error(c.fset.Position(pos), msg)
+	return msg
+}
+
+// collectFields collects struct fields tok = token.STRUCT), interface methods
+// (tok = token.INTERFACE), and function arguments/results (tok = token.FUNC).
+func (c *checker) collectFields(tok token.Token, list *ast.FieldList, cycleOk bool) (fields ObjList, tags []string, isVariadic bool) {
+	if list != nil {
+		for _, field := range list.List {
+			ftype := field.Type
+			if t, ok := ftype.(*ast.Ellipsis); ok {
+				ftype = t.Elt
+				isVariadic = true
+			}
+			typ := c.makeType(ftype, cycleOk)
+			tag := ""
+			if field.Tag != nil {
+				assert(field.Tag.Kind == token.STRING)
+				tag, _ = strconv.Unquote(field.Tag.Value)
+			}
+			if len(field.Names) > 0 {
+				// named fields
+				for _, name := range field.Names {
+					obj := name.Obj
+					obj.Type = typ
+					fields = append(fields, obj)
+					if tok == token.STRUCT {
+						tags = append(tags, tag)
+					}
+				}
+			} else {
+				// anonymous field
+				switch tok {
+				case token.STRUCT:
+					tags = append(tags, tag)
+					fallthrough
+				case token.FUNC:
+					obj := ast.NewObj(ast.Var, "")
+					obj.Type = typ
+					fields = append(fields, obj)
+				case token.INTERFACE:
+					utyp := Underlying(typ)
+					if typ, ok := utyp.(*Interface); ok {
+						// TODO(gri) This is not good enough. Check for double declarations!
+						fields = append(fields, typ.Methods...)
+					} else if _, ok := utyp.(*Bad); !ok {
+						// if utyp is Bad, don't complain (the root cause was reported before)
+						c.errorf(ftype.Pos(), "interface contains embedded non-interface type")
+					}
+				default:
+					panic("unreachable")
+				}
+			}
+		}
+	}
+	return
+}
+
+// makeType makes a new type for an AST type specification x or returns
+// the type referred to by a type name x. If cycleOk is set, a type may
+// refer to itself directly or indirectly; otherwise cycles are errors.
+//
+func (c *checker) makeType(x ast.Expr, cycleOk bool) (typ Type) {
+	if debug {
+		fmt.Printf("makeType (cycleOk = %v)\n", cycleOk)
+		ast.Print(c.fset, x)
+		defer func() {
+			fmt.Printf("-> %T %v\n\n", typ, typ)
+		}()
+	}
+
+	switch t := x.(type) {
+	case *ast.BadExpr:
+		return &Bad{}
+
+	case *ast.Ident:
+		// type name
+		obj := t.Obj
+		if obj == nil {
+			// unresolved identifier (error has been reported before)
+			return &Bad{Msg: "unresolved identifier"}
+		}
+		if obj.Kind != ast.Typ {
+			msg := c.errorf(t.Pos(), "%s is not a type", t.Name)
+			return &Bad{Msg: msg}
+		}
+		c.checkObj(obj, cycleOk)
+		if !cycleOk && obj.Type.(*Name).Underlying == nil {
+			// TODO(gri) Enable this message again once its position
+			// is independent of the underlying map implementation.
+			// msg := c.errorf(obj.Pos(), "illegal cycle in declaration of %s", obj.Name)
+			msg := "illegal cycle"
+			return &Bad{Msg: msg}
+		}
+		return obj.Type.(Type)
+
+	case *ast.ParenExpr:
+		return c.makeType(t.X, cycleOk)
+
+	case *ast.SelectorExpr:
+		// qualified identifier
+		// TODO (gri) eventually, this code belongs to expression
+		//            type checking - here for the time being
+		if ident, ok := t.X.(*ast.Ident); ok {
+			if obj := ident.Obj; obj != nil {
+				if obj.Kind != ast.Pkg {
+					msg := c.errorf(ident.Pos(), "%s is not a package", obj.Name)
+					return &Bad{Msg: msg}
+				}
+				// TODO(gri) we have a package name but don't
+				// have the mapping from package name to package
+				// scope anymore (created in ast.NewPackage).
+				return &Bad{} // for now
+			}
+		}
+		// TODO(gri) can this really happen (the parser should have excluded this)?
+		msg := c.errorf(t.Pos(), "expected qualified identifier")
+		return &Bad{Msg: msg}
+
+	case *ast.StarExpr:
+		return &Pointer{Base: c.makeType(t.X, true)}
+
+	case *ast.ArrayType:
+		if t.Len != nil {
+			// TODO(gri) compute length
+			return &Array{Elt: c.makeType(t.Elt, cycleOk)}
+		}
+		return &Slice{Elt: c.makeType(t.Elt, true)}
+
+	case *ast.StructType:
+		fields, tags, _ := c.collectFields(token.STRUCT, t.Fields, cycleOk)
+		return &Struct{Fields: fields, Tags: tags}
+
+	case *ast.FuncType:
+		params, _, _ := c.collectFields(token.FUNC, t.Params, true)
+		results, _, isVariadic := c.collectFields(token.FUNC, t.Results, true)
+		return &Func{Recv: nil, Params: params, Results: results, IsVariadic: isVariadic}
+
+	case *ast.InterfaceType:
+		methods, _, _ := c.collectFields(token.INTERFACE, t.Methods, cycleOk)
+		methods.Sort()
+		return &Interface{Methods: methods}
+
+	case *ast.MapType:
+		return &Map{Key: c.makeType(t.Key, true), Elt: c.makeType(t.Key, true)}
+
+	case *ast.ChanType:
+		return &Chan{Dir: t.Dir, Elt: c.makeType(t.Value, true)}
+	}
+
+	panic(fmt.Sprintf("unreachable (%T)", x))
+}
+
+// checkObj type checks an object.
+func (c *checker) checkObj(obj *ast.Object, ref bool) {
+	if obj.Type != nil {
+		// object has already been type checked
+		return
+	}
+
+	switch obj.Kind {
+	case ast.Bad:
+		// ignore
+
+	case ast.Con:
+		// TODO(gri) complete this
+
+	case ast.Typ:
+		typ := &Name{Obj: obj}
+		obj.Type = typ // "mark" object so recursion terminates
+		typ.Underlying = Underlying(c.makeType(obj.Decl.(*ast.TypeSpec).Type, ref))
+
+	case ast.Var:
+		// TODO(gri) complete this
+
+	case ast.Fun:
+		// TODO(gri) complete this
+
+	default:
+		panic("unreachable")
+	}
+}
+
+// Check typechecks a package.
+// It augments the AST by assigning types to all ast.Objects and returns a map
+// of types for all expression nodes in statements, and a scanner.ErrorList if
+// there are errors.
+//
+func Check(fset *token.FileSet, pkg *ast.Package) (types map[ast.Expr]Type, err os.Error) {
+	var c checker
+	c.fset = fset
+	c.types = make(map[ast.Expr]Type)
+
+	for _, obj := range pkg.Scope.Objects {
+		c.checkObj(obj, false)
+	}
+
+	return c.types, c.GetError(scanner.NoMultiples)
+}
diff --git a/src/pkg/go/types/check_test.go b/src/pkg/go/types/check_test.go
new file mode 100644
index 000000000..8be653fcb
--- /dev/null
+++ b/src/pkg/go/types/check_test.go
@@ -0,0 +1,215 @@
+// Copyright 2011 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// This file implements a typechecker test harness. The packages specified
+// in tests are typechecked. Error messages reported by the typechecker are
+// compared against the error messages expected in the test files.
+//
+// Expected errors are indicated in the test files by putting a comment
+// of the form /* ERROR "rx" */ immediately following an offending token.
+// The harness will verify that an error matching the regular expression
+// rx is reported at that source position. Consecutive comments may be
+// used to indicate multiple errors for the same token position.
+//
+// For instance, the following test file indicates that a "not declared"
+// error should be reported for the undeclared variable x:
+//
+//	package p
+//	func f() {
+//		_ = x /* ERROR "not declared" */ + 1
+//	}
+
+package types
+
+import (
+	"fmt"
+	"go/ast"
+	"go/parser"
+	"go/scanner"
+	"go/token"
+	"io/ioutil"
+	"os"
+	"regexp"
+	"testing"
+)
+
+// The test filenames do not end in .go so that they are invisible
+// to gofmt since they contain comments that must not change their
+// positions relative to surrounding tokens.
+
+var tests = []struct {
+	name  string
+	files []string
+}{
+	{"test0", []string{"testdata/test0.src"}},
+}
+
+var fset = token.NewFileSet()
+
+// TODO(gri) This functionality should be in token.Fileset.
+func getFile(filename string) *token.File {
+	for f := range fset.Files() {
+		if f.Name() == filename {
+			return f
+		}
+	}
+	return nil
+}
+
+// TODO(gri) This functionality should be in token.Fileset.
+func getPos(filename string, offset int) token.Pos {
+	if f := getFile(filename); f != nil {
+		return f.Pos(offset)
+	}
+	return token.NoPos
+}
+
+// TODO(gri) Need to revisit parser interface. We should be able to use parser.ParseFiles
+//           or a similar function instead.
+func parseFiles(t *testing.T, testname string, filenames []string) (map[string]*ast.File, os.Error) {
+	files := make(map[string]*ast.File)
+	var errors scanner.ErrorList
+	for _, filename := range filenames {
+		if _, exists := files[filename]; exists {
+			t.Fatalf("%s: duplicate file %s", testname, filename)
+		}
+		file, err := parser.ParseFile(fset, filename, nil, parser.DeclarationErrors)
+		if file == nil {
+			t.Fatalf("%s: could not parse file %s", testname, filename)
+		}
+		files[filename] = file
+		if err != nil {
+			// if the parser returns a non-scanner.ErrorList error
+			// the file couldn't be read in the first place and
+			// file == nil; in that case we shouldn't reach here
+			errors = append(errors, err.(scanner.ErrorList)...)
+		}
+
+	}
+	return files, errors
+}
+
+// ERROR comments must be of the form /* ERROR "rx" */ and rx is
+// a regular expression that matches the expected error message.
+//
+var errRx = regexp.MustCompile(`^/\* *ERROR *"([^"]*)" *\*/$`)
+
+// expectedErrors collects the regular expressions of ERROR comments found
+// in files and returns them as a map of error positions to error messages.
+//
+func expectedErrors(t *testing.T, testname string, files map[string]*ast.File) map[token.Pos]string {
+	errors := make(map[token.Pos]string)
+	for filename := range files {
+		src, err := ioutil.ReadFile(filename)
+		if err != nil {
+			t.Fatalf("%s: could not read %s", testname, filename)
+		}
+
+		var s scanner.Scanner
+		// file was parsed already - do not add it again to the file
+		// set otherwise the position information returned here will
+		// not match the position information collected by the parser
+		s.Init(getFile(filename), src, nil, scanner.ScanComments)
+		var prev token.Pos // position of last non-comment token
+
+	scanFile:
+		for {
+			pos, tok, lit := s.Scan()
+			switch tok {
+			case token.EOF:
+				break scanFile
+			case token.COMMENT:
+				s := errRx.FindStringSubmatch(lit)
+				if len(s) == 2 {
+					errors[prev] = string(s[1])
+				}
+			default:
+				prev = pos
+			}
+		}
+	}
+	return errors
+}
+
+func eliminate(t *testing.T, expected map[token.Pos]string, errors os.Error) {
+	if errors == nil {
+		return
+	}
+	for _, error := range errors.(scanner.ErrorList) {
+		// error.Pos is a token.Position, but we want
+		// a token.Pos so we can do a map lookup
+		// TODO(gri) Need to move scanner.Errors over
+		//           to use token.Pos and file set info.
+		pos := getPos(error.Pos.Filename, error.Pos.Offset)
+		if msg, found := expected[pos]; found {
+			// we expect a message at pos; check if it matches
+			rx, err := regexp.Compile(msg)
+			if err != nil {
+				t.Errorf("%s: %v", error.Pos, err)
+				continue
+			}
+			if match := rx.MatchString(error.Msg); !match {
+				t.Errorf("%s: %q does not match %q", error.Pos, error.Msg, msg)
+				continue
+			}
+			// we have a match - eliminate this error
+			expected[pos] = "", false
+		} else {
+			// To keep in mind when analyzing failed test output:
+			// If the same error position occurs multiple times in errors,
+			// this message will be triggered (because the first error at
+			// the position removes this position from the expected errors).
+			t.Errorf("%s: no (multiple?) error expected, but found: %s", error.Pos, error.Msg)
+		}
+	}
+}
+
+func check(t *testing.T, testname string, testfiles []string) {
+	// TODO(gri) Eventually all these different phases should be
+	//           subsumed into a single function call that takes
+	//           a set of files and creates a fully resolved and
+	//           type-checked AST.
+
+	files, err := parseFiles(t, testname, testfiles)
+
+	// we are expecting the following errors
+	// (collect these after parsing the files so that
+	// they are found in the file set)
+	errors := expectedErrors(t, testname, files)
+
+	// verify errors returned by the parser
+	eliminate(t, errors, err)
+
+	// verify errors returned after resolving identifiers
+	pkg, err := ast.NewPackage(fset, files, GcImporter, Universe)
+	eliminate(t, errors, err)
+
+	// verify errors returned by the typechecker
+	_, err = Check(fset, pkg)
+	eliminate(t, errors, err)
+
+	// there should be no expected errors left
+	if len(errors) > 0 {
+		t.Errorf("%s: %d errors not reported:", testname, len(errors))
+		for pos, msg := range errors {
+			t.Errorf("%s: %s\n", fset.Position(pos), msg)
+		}
+	}
+}
+
+func TestCheck(t *testing.T) {
+	// For easy debugging w/o changing the testing code,
+	// if there is a local test file, only test that file.
+	const testfile = "test.go"
+	if fi, err := os.Stat(testfile); err == nil && fi.IsRegular() {
+		fmt.Printf("WARNING: Testing only %s (remove it to run all tests)\n", testfile)
+		check(t, testfile, []string{testfile})
+		return
+	}
+
+	// Otherwise, run all the tests.
+	for _, test := range tests {
+		check(t, test.name, test.files)
+	}
+}
diff --git a/src/pkg/go/types/const.go b/src/pkg/go/types/const.go
new file mode 100644
index 000000000..1ef95d9f9
--- /dev/null
+++ b/src/pkg/go/types/const.go
@@ -0,0 +1,332 @@
+// Copyright 2011 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// This file implements operations on ideal constants.
+
+package types
+
+import (
+	"big"
+	"go/token"
+	"strconv"
+)
+
+// TODO(gri) Consider changing the API so Const is an interface
+//           and operations on consts don't have to type switch.
+
+// A Const implements an ideal constant Value.
+// The zero value z for a Const is not a valid constant value.
+type Const struct {
+	// representation of constant values:
+	// ideal bool     ->  bool
+	// ideal int      ->  *big.Int
+	// ideal float    ->  *big.Rat
+	// ideal complex  ->  cmplx
+	// ideal string   ->  string
+	val interface{}
+}
+
+// Representation of complex values.
+type cmplx struct {
+	re, im *big.Rat
+}
+
+func assert(cond bool) {
+	if !cond {
+		panic("go/types internal error: assertion failed")
+	}
+}
+
+// MakeConst makes an ideal constant from a literal
+// token and the corresponding literal string.
+func MakeConst(tok token.Token, lit string) Const {
+	switch tok {
+	case token.INT:
+		var x big.Int
+		_, ok := x.SetString(lit, 0)
+		assert(ok)
+		return Const{&x}
+	case token.FLOAT:
+		var y big.Rat
+		_, ok := y.SetString(lit)
+		assert(ok)
+		return Const{&y}
+	case token.IMAG:
+		assert(lit[len(lit)-1] == 'i')
+		var im big.Rat
+		_, ok := im.SetString(lit[0 : len(lit)-1])
+		assert(ok)
+		return Const{cmplx{big.NewRat(0, 1), &im}}
+	case token.CHAR:
+		assert(lit[0] == '\'' && lit[len(lit)-1] == '\'')
+		code, _, _, err := strconv.UnquoteChar(lit[1:len(lit)-1], '\'')
+		assert(err == nil)
+		return Const{big.NewInt(int64(code))}
+	case token.STRING:
+		s, err := strconv.Unquote(lit)
+		assert(err == nil)
+		return Const{s}
+	}
+	panic("unreachable")
+}
+
+// MakeZero returns the zero constant for the given type.
+func MakeZero(typ *Type) Const {
+	// TODO(gri) fix this
+	return Const{0}
+}
+
+// Match attempts to match the internal constant representations of x and y.
+// If the attempt is successful, the result is the values of x and y,
+// if necessary converted to have the same internal representation; otherwise
+// the results are invalid.
+func (x Const) Match(y Const) (u, v Const) {
+	switch a := x.val.(type) {
+	case bool:
+		if _, ok := y.val.(bool); ok {
+			u, v = x, y
+		}
+	case *big.Int:
+		switch y.val.(type) {
+		case *big.Int:
+			u, v = x, y
+		case *big.Rat:
+			var z big.Rat
+			z.SetInt(a)
+			u, v = Const{&z}, y
+		case cmplx:
+			var z big.Rat
+			z.SetInt(a)
+			u, v = Const{cmplx{&z, big.NewRat(0, 1)}}, y
+		}
+	case *big.Rat:
+		switch y.val.(type) {
+		case *big.Int:
+			v, u = y.Match(x)
+		case *big.Rat:
+			u, v = x, y
+		case cmplx:
+			u, v = Const{cmplx{a, big.NewRat(0, 0)}}, y
+		}
+	case cmplx:
+		switch y.val.(type) {
+		case *big.Int, *big.Rat:
+			v, u = y.Match(x)
+		case cmplx:
+			u, v = x, y
+		}
+	case string:
+		if _, ok := y.val.(string); ok {
+			u, v = x, y
+		}
+	default:
+		panic("unreachable")
+	}
+	return
+}
+
+// Convert attempts to convert the constant x to a given type.
+// If the attempt is successful, the result is the new constant;
+// otherwise the result is invalid.
+func (x Const) Convert(typ *Type) Const {
+	// TODO(gri) implement this
+	switch x := x.val.(type) {
+	case bool:
+	case *big.Int:
+	case *big.Rat:
+	case cmplx:
+	case string:
+	}
+	return x
+}
+
+func (x Const) String() string {
+	switch x := x.val.(type) {
+	case bool:
+		if x {
+			return "true"
+		}
+		return "false"
+	case *big.Int:
+		return x.String()
+	case *big.Rat:
+		return x.FloatString(10) // 10 digits of precision after decimal point seems fine
+	case cmplx:
+		// TODO(gri) don't print 0 components
+		return x.re.FloatString(10) + " + " + x.im.FloatString(10) + "i"
+	case string:
+		return x
+	}
+	panic("unreachable")
+}
+
+func (x Const) UnaryOp(op token.Token) Const {
+	panic("unimplemented")
+}
+
+func (x Const) BinaryOp(op token.Token, y Const) Const {
+	var z interface{}
+	switch x := x.val.(type) {
+	case bool:
+		z = binaryBoolOp(x, op, y.val.(bool))
+	case *big.Int:
+		z = binaryIntOp(x, op, y.val.(*big.Int))
+	case *big.Rat:
+		z = binaryFloatOp(x, op, y.val.(*big.Rat))
+	case cmplx:
+		z = binaryCmplxOp(x, op, y.val.(cmplx))
+	case string:
+		z = binaryStringOp(x, op, y.val.(string))
+	default:
+		panic("unreachable")
+	}
+	return Const{z}
+}
+
+func binaryBoolOp(x bool, op token.Token, y bool) interface{} {
+	switch op {
+	case token.EQL:
+		return x == y
+	case token.NEQ:
+		return x != y
+	}
+	panic("unreachable")
+}
+
+func binaryIntOp(x *big.Int, op token.Token, y *big.Int) interface{} {
+	var z big.Int
+	switch op {
+	case token.ADD:
+		return z.Add(x, y)
+	case token.SUB:
+		return z.Sub(x, y)
+	case token.MUL:
+		return z.Mul(x, y)
+	case token.QUO:
+		return z.Quo(x, y)
+	case token.REM:
+		return z.Rem(x, y)
+	case token.AND:
+		return z.And(x, y)
+	case token.OR:
+		return z.Or(x, y)
+	case token.XOR:
+		return z.Xor(x, y)
+	case token.AND_NOT:
+		return z.AndNot(x, y)
+	case token.SHL:
+		panic("unimplemented")
+	case token.SHR:
+		panic("unimplemented")
+	case token.EQL:
+		return x.Cmp(y) == 0
+	case token.NEQ:
+		return x.Cmp(y) != 0
+	case token.LSS:
+		return x.Cmp(y) < 0
+	case token.LEQ:
+		return x.Cmp(y) <= 0
+	case token.GTR:
+		return x.Cmp(y) > 0
+	case token.GEQ:
+		return x.Cmp(y) >= 0
+	}
+	panic("unreachable")
+}
+
+func binaryFloatOp(x *big.Rat, op token.Token, y *big.Rat) interface{} {
+	var z big.Rat
+	switch op {
+	case token.ADD:
+		return z.Add(x, y)
+	case token.SUB:
+		return z.Sub(x, y)
+	case token.MUL:
+		return z.Mul(x, y)
+	case token.QUO:
+		return z.Quo(x, y)
+	case token.EQL:
+		return x.Cmp(y) == 0
+	case token.NEQ:
+		return x.Cmp(y) != 0
+	case token.LSS:
+		return x.Cmp(y) < 0
+	case token.LEQ:
+		return x.Cmp(y) <= 0
+	case token.GTR:
+		return x.Cmp(y) > 0
+	case token.GEQ:
+		return x.Cmp(y) >= 0
+	}
+	panic("unreachable")
+}
+
+func binaryCmplxOp(x cmplx, op token.Token, y cmplx) interface{} {
+	a, b := x.re, x.im
+	c, d := y.re, y.im
+	switch op {
+	case token.ADD:
+		// (a+c) + i(b+d)
+		var re, im big.Rat
+		re.Add(a, c)
+		im.Add(b, d)
+		return cmplx{&re, &im}
+	case token.SUB:
+		// (a-c) + i(b-d)
+		var re, im big.Rat
+		re.Sub(a, c)
+		im.Sub(b, d)
+		return cmplx{&re, &im}
+	case token.MUL:
+		// (ac-bd) + i(bc+ad)
+		var ac, bd, bc, ad big.Rat
+		ac.Mul(a, c)
+		bd.Mul(b, d)
+		bc.Mul(b, c)
+		ad.Mul(a, d)
+		var re, im big.Rat
+		re.Sub(&ac, &bd)
+		im.Add(&bc, &ad)
+		return cmplx{&re, &im}
+	case token.QUO:
+		// (ac+bd)/s + i(bc-ad)/s, with s = cc + dd
+		var ac, bd, bc, ad, s big.Rat
+		ac.Mul(a, c)
+		bd.Mul(b, d)
+		bc.Mul(b, c)
+		ad.Mul(a, d)
+		s.Add(c.Mul(c, c), d.Mul(d, d))
+		var re, im big.Rat
+		re.Add(&ac, &bd)
+		re.Quo(&re, &s)
+		im.Sub(&bc, &ad)
+		im.Quo(&im, &s)
+		return cmplx{&re, &im}
+	case token.EQL:
+		return a.Cmp(c) == 0 && b.Cmp(d) == 0
+	case token.NEQ:
+		return a.Cmp(c) != 0 || b.Cmp(d) != 0
+	}
+	panic("unreachable")
+}
+
+func binaryStringOp(x string, op token.Token, y string) interface{} {
+	switch op {
+	case token.ADD:
+		return x + y
+	case token.EQL:
+		return x == y
+	case token.NEQ:
+		return x != y
+	case token.LSS:
+		return x < y
+	case token.LEQ:
+		return x <= y
+	case token.GTR:
+		return x > y
+	case token.GEQ:
+		return x >= y
+	}
+	panic("unreachable")
+}
diff --git a/src/pkg/go/types/exportdata.go b/src/pkg/go/types/exportdata.go
new file mode 100644
index 000000000..383520320
--- /dev/null
+++ b/src/pkg/go/types/exportdata.go
@@ -0,0 +1,132 @@
+// Copyright 2011 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// This file implements ExportData.
+
+package types
+
+import (
+	"bufio"
+	"fmt"
+	"io"
+	"os"
+	"strconv"
+	"strings"
+)
+
+func readGopackHeader(buf *bufio.Reader) (name string, size int, err os.Error) {
+	// See $GOROOT/include/ar.h.
+	hdr := make([]byte, 64+12+6+6+8+10+2)
+	_, err = io.ReadFull(buf, hdr)
+	if err != nil {
+		return
+	}
+	if trace {
+		fmt.Printf("header: %s", hdr)
+	}
+	s := strings.TrimSpace(string(hdr[64+12+6+6+8:][:10]))
+	size, err = strconv.Atoi(s)
+	if err != nil || hdr[len(hdr)-2] != '`' || hdr[len(hdr)-1] != '\n' {
+		err = os.NewError("invalid archive header")
+		return
+	}
+	name = strings.TrimSpace(string(hdr[:64]))
+	return
+}
+
+type dataReader struct {
+	*bufio.Reader
+	io.Closer
+}
+
+// ExportData returns a readCloser positioned at the beginning of the
+// export data section of the given object/archive file, or an error.
+// It is the caller's responsibility to close the readCloser.
+//
+func ExportData(filename string) (rc io.ReadCloser, err os.Error) {
+	file, err := os.Open(filename)
+	if err != nil {
+		return
+	}
+
+	defer func() {
+		if err != nil {
+			file.Close()
+			// Add file name to error.
+			err = fmt.Errorf("reading export data: %s: %v", filename, err)
+		}
+	}()
+
+	buf := bufio.NewReader(file)
+
+	// Read first line to make sure this is an object file.
+	line, err := buf.ReadSlice('\n')
+	if err != nil {
+		return
+	}
+	if string(line) == "!\n" {
+		// Archive file.  Scan to __.PKGDEF, which should
+		// be second archive entry.
+		var name string
+		var size int
+
+		// First entry should be __.SYMDEF.
+		// Read and discard.
+		if name, size, err = readGopackHeader(buf); err != nil {
+			return
+		}
+		if name != "__.SYMDEF" {
+			err = os.NewError("go archive does not begin with __.SYMDEF")
+			return
+		}
+		const block = 4096
+		tmp := make([]byte, block)
+		for size > 0 {
+			n := size
+			if n > block {
+				n = block
+			}
+			_, err = io.ReadFull(buf, tmp[:n])
+			if err != nil {
+				return
+			}
+			size -= n
+		}
+
+		// Second entry should be __.PKGDEF.
+		if name, size, err = readGopackHeader(buf); err != nil {
+			return
+		}
+		if name != "__.PKGDEF" {
+			err = os.NewError("go archive is missing __.PKGDEF")
+			return
+		}
+
+		// Read first line of __.PKGDEF data, so that line
+		// is once again the first line of the input.
+		line, err = buf.ReadSlice('\n')
+		if err != nil {
+			return
+		}
+	}
+
+	// Now at __.PKGDEF in archive or still at beginning of file.
+	// Either way, line should begin with "go object ".
+	if !strings.HasPrefix(string(line), "go object ") {
+		err = os.NewError("not a go object file")
+		return
+	}
+
+	// Skip over object header to export data.
+	// Begins after first line with $$.
+	for line[0] != '$' {
+		line, err = buf.ReadSlice('\n')
+		if err != nil {
+			return
+		}
+	}
+
+	rc = &dataReader{buf, file}
+	return
+}
diff --git a/src/pkg/go/types/gcimporter.go b/src/pkg/go/types/gcimporter.go
new file mode 100644
index 000000000..6ab1806b6
--- /dev/null
+++ b/src/pkg/go/types/gcimporter.go
@@ -0,0 +1,799 @@
+// Copyright 2011 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// This file implements an ast.Importer for gc generated object files.
+// TODO(gri) Eventually move this into a separate package outside types.
+
+package types
+
+import (
+	"big"
+	"fmt"
+	"go/ast"
+	"go/token"
+	"io"
+	"os"
+	"path/filepath"
+	"runtime"
+	"scanner"
+	"strconv"
+)
+
+const trace = false // set to true for debugging
+
+var (
+	pkgRoot = filepath.Join(runtime.GOROOT(), "pkg", runtime.GOOS+"_"+runtime.GOARCH)
+	pkgExts = [...]string{".a", ".5", ".6", ".8"}
+)
+
+// findPkg returns the filename and package id for an import path.
+// If no file was found, an empty filename is returned.
+func findPkg(path string) (filename, id string) {
+	if len(path) == 0 {
+		return
+	}
+
+	id = path
+	var noext string
+	switch path[0] {
+	default:
+		// "x" -> "$GOROOT/pkg/$GOOS_$GOARCH/x.ext", "x"
+		noext = filepath.Join(pkgRoot, path)
+
+	case '.':
+		// "./x" -> "/this/directory/x.ext", "/this/directory/x"
+		cwd, err := os.Getwd()
+		if err != nil {
+			return
+		}
+		noext = filepath.Join(cwd, path)
+		id = noext
+
+	case '/':
+		// "/x" -> "/x.ext", "/x"
+		noext = path
+	}
+
+	// try extensions
+	for _, ext := range pkgExts {
+		filename = noext + ext
+		if f, err := os.Stat(filename); err == nil && f.IsRegular() {
+			return
+		}
+	}
+
+	filename = "" // not found
+	return
+}
+
+// gcParser parses the exports inside a gc compiler-produced
+// object/archive file and populates its scope with the results.
+type gcParser struct {
+	scanner scanner.Scanner
+	tok     int                    // current token
+	lit     string                 // literal string; only valid for Ident, Int, String tokens
+	id      string                 // package id of imported package
+	imports map[string]*ast.Object // package id -> package object
+}
+
+func (p *gcParser) init(filename, id string, src io.Reader, imports map[string]*ast.Object) {
+	p.scanner.Init(src)
+	p.scanner.Error = func(_ *scanner.Scanner, msg string) { p.error(msg) }
+	p.scanner.Mode = scanner.ScanIdents | scanner.ScanInts | scanner.ScanStrings | scanner.ScanComments | scanner.SkipComments
+	p.scanner.Whitespace = 1<<'\t' | 1<<' '
+	p.scanner.Filename = filename // for good error messages
+	p.next()
+	p.id = id
+	p.imports = imports
+}
+
+func (p *gcParser) next() {
+	p.tok = p.scanner.Scan()
+	switch p.tok {
+	case scanner.Ident, scanner.Int, scanner.String:
+		p.lit = p.scanner.TokenText()
+	default:
+		p.lit = ""
+	}
+	if trace {
+		fmt.Printf("%s: %q -> %q\n", scanner.TokenString(p.tok), p.scanner.TokenText(), p.lit)
+	}
+}
+
+// GcImporter implements the ast.Importer signature.
+func GcImporter(imports map[string]*ast.Object, path string) (pkg *ast.Object, err os.Error) {
+	if path == "unsafe" {
+		return Unsafe, nil
+	}
+
+	defer func() {
+		if r := recover(); r != nil {
+			err = r.(importError) // will re-panic if r is not an importError
+			if trace {
+				panic(err) // force a stack trace
+			}
+		}
+	}()
+
+	filename, id := findPkg(path)
+	if filename == "" {
+		err = os.NewError("can't find import: " + id)
+		return
+	}
+
+	if pkg = imports[id]; pkg != nil {
+		return // package was imported before
+	}
+
+	buf, err := ExportData(filename)
+	if err != nil {
+		return
+	}
+	defer buf.Close()
+
+	if trace {
+		fmt.Printf("importing %s (%s)\n", id, filename)
+	}
+
+	var p gcParser
+	p.init(filename, id, buf, imports)
+	pkg = p.parseExport()
+	return
+}
+
+// ----------------------------------------------------------------------------
+// Error handling
+
+// Internal errors are boxed as importErrors.
+type importError struct {
+	pos scanner.Position
+	err os.Error
+}
+
+func (e importError) String() string {
+	return fmt.Sprintf("import error %s (byte offset = %d): %s", e.pos, e.pos.Offset, e.err)
+}
+
+func (p *gcParser) error(err interface{}) {
+	if s, ok := err.(string); ok {
+		err = os.NewError(s)
+	}
+	// panic with a runtime.Error if err is not an os.Error
+	panic(importError{p.scanner.Pos(), err.(os.Error)})
+}
+
+func (p *gcParser) errorf(format string, args ...interface{}) {
+	p.error(fmt.Sprintf(format, args...))
+}
+
+func (p *gcParser) expect(tok int) string {
+	lit := p.lit
+	if p.tok != tok {
+		p.errorf("expected %q, got %q (%q)", scanner.TokenString(tok), scanner.TokenString(p.tok), lit)
+	}
+	p.next()
+	return lit
+}
+
+func (p *gcParser) expectSpecial(tok string) {
+	sep := 'x' // not white space
+	i := 0
+	for i < len(tok) && p.tok == int(tok[i]) && sep > ' ' {
+		sep = p.scanner.Peek() // if sep <= ' ', there is white space before the next token
+		p.next()
+		i++
+	}
+	if i < len(tok) {
+		p.errorf("expected %q, got %q", tok, tok[0:i])
+	}
+}
+
+func (p *gcParser) expectKeyword(keyword string) {
+	lit := p.expect(scanner.Ident)
+	if lit != keyword {
+		p.errorf("expected keyword %s, got %q", keyword, lit)
+	}
+}
+
+// ----------------------------------------------------------------------------
+// Import declarations
+
+// ImportPath = string_lit .
+//
+func (p *gcParser) parsePkgId() *ast.Object {
+	id, err := strconv.Unquote(p.expect(scanner.String))
+	if err != nil {
+		p.error(err)
+	}
+
+	switch id {
+	case "":
+		// id == "" stands for the imported package id
+		// (only known at time of package installation)
+		id = p.id
+	case "unsafe":
+		// package unsafe is not in the imports map - handle explicitly
+		return Unsafe
+	}
+
+	pkg := p.imports[id]
+	if pkg == nil {
+		scope = ast.NewScope(nil)
+		pkg = ast.NewObj(ast.Pkg, "")
+		pkg.Data = scope
+		p.imports[id] = pkg
+	}
+
+	return pkg
+}
+
+// dotIdentifier = ( ident | '·' ) { ident | int | '·' } .
+func (p *gcParser) parseDotIdent() string {
+	ident := ""
+	if p.tok != scanner.Int {
+		sep := 'x' // not white space
+		for (p.tok == scanner.Ident || p.tok == scanner.Int || p.tok == '·') && sep > ' ' {
+			ident += p.lit
+			sep = p.scanner.Peek() // if sep <= ' ', there is white space before the next token
+			p.next()
+		}
+	}
+	if ident == "" {
+		p.expect(scanner.Ident) // use expect() for error handling
+	}
+	return ident
+}
+
+// ExportedName = ImportPath "." dotIdentifier .
+//
+func (p *gcParser) parseExportedName(kind ast.ObjKind) *ast.Object {
+	pkg := p.parsePkgId()
+	p.expect('.')
+	name := p.parseDotIdent()
+
+	// a type may have been declared before - if it exists
+	// already in the respective package scope, return that
+	// type
+	scope := pkg.Data.(*ast.Scope)
+	if kind == ast.Typ {
+		if obj := scope.Lookup(name); obj != nil {
+			assert(obj.Kind == ast.Typ)
+			return obj
+		}
+	}
+
+	// any other object must be a newly declared object -
+	// create it and insert it into the package scope
+	obj := ast.NewObj(kind, name)
+	if scope.Insert(obj) != nil {
+		p.errorf("already declared: %s", obj.Name)
+	}
+
+	// a new type object is a named type and may be referred
+	// to before the underlying type is known - set it up
+	if kind == ast.Typ {
+		obj.Type = &Name{Obj: obj}
+	}
+
+	return obj
+}
+
+// ----------------------------------------------------------------------------
+// Types
+
+// BasicType = identifier .
+//
+func (p *gcParser) parseBasicType() Type {
+	obj := Universe.Lookup(p.expect(scanner.Ident))
+	if obj == nil || obj.Kind != ast.Typ {
+		p.errorf("not a basic type: %s", obj.Name)
+	}
+	return obj.Type.(Type)
+}
+
+// ArrayType = "[" int_lit "]" Type .
+//
+func (p *gcParser) parseArrayType() Type {
+	// "[" already consumed and lookahead known not to be "]"
+	lit := p.expect(scanner.Int)
+	p.expect(']')
+	elt := p.parseType()
+	n, err := strconv.Atoui64(lit)
+	if err != nil {
+		p.error(err)
+	}
+	return &Array{Len: n, Elt: elt}
+}
+
+// MapType = "map" "[" Type "]" Type .
+//
+func (p *gcParser) parseMapType() Type {
+	p.expectKeyword("map")
+	p.expect('[')
+	key := p.parseType()
+	p.expect(']')
+	elt := p.parseType()
+	return &Map{Key: key, Elt: elt}
+}
+
+// Name = identifier | "?" .
+//
+func (p *gcParser) parseName() (name string) {
+	switch p.tok {
+	case scanner.Ident:
+		name = p.lit
+		p.next()
+	case '?':
+		// anonymous
+		p.next()
+	default:
+		p.error("name expected")
+	}
+	return
+}
+
+// Field = Name Type [ ":" string_lit ] .
+//
+func (p *gcParser) parseField() (fld *ast.Object, tag string) {
+	name := p.parseName()
+	ftyp := p.parseType()
+	if name == "" {
+		// anonymous field - ftyp must be T or *T and T must be a type name
+		if _, ok := Deref(ftyp).(*Name); !ok {
+			p.errorf("anonymous field expected")
+		}
+	}
+	if p.tok == ':' {
+		p.next()
+		tag = p.expect(scanner.String)
+	}
+	fld = ast.NewObj(ast.Var, name)
+	fld.Type = ftyp
+	return
+}
+
+// StructType = "struct" "{" [ FieldList ] "}" .
+// FieldList  = Field { ";" Field } .
+//
+func (p *gcParser) parseStructType() Type {
+	var fields []*ast.Object
+	var tags []string
+
+	parseField := func() {
+		fld, tag := p.parseField()
+		fields = append(fields, fld)
+		tags = append(tags, tag)
+	}
+
+	p.expectKeyword("struct")
+	p.expect('{')
+	if p.tok != '}' {
+		parseField()
+		for p.tok == ';' {
+			p.next()
+			parseField()
+		}
+	}
+	p.expect('}')
+
+	return &Struct{Fields: fields, Tags: tags}
+}
+
+// Parameter = ( identifier | "?" ) [ "..." ] Type [ ":" string_lit ] .
+//
+func (p *gcParser) parseParameter() (par *ast.Object, isVariadic bool) {
+	name := p.parseName()
+	if name == "" {
+		name = "_" // cannot access unnamed identifiers
+	}
+	if p.tok == '.' {
+		p.expectSpecial("...")
+		isVariadic = true
+	}
+	ptyp := p.parseType()
+	// ignore argument tag
+	if p.tok == ':' {
+		p.next()
+		p.expect(scanner.String)
+	}
+	par = ast.NewObj(ast.Var, name)
+	par.Type = ptyp
+	return
+}
+
+// Parameters    = "(" [ ParameterList ] ")" .
+// ParameterList = { Parameter "," } Parameter .
+//
+func (p *gcParser) parseParameters() (list []*ast.Object, isVariadic bool) {
+	parseParameter := func() {
+		par, variadic := p.parseParameter()
+		list = append(list, par)
+		if variadic {
+			if isVariadic {
+				p.error("... not on final argument")
+			}
+			isVariadic = true
+		}
+	}
+
+	p.expect('(')
+	if p.tok != ')' {
+		parseParameter()
+		for p.tok == ',' {
+			p.next()
+			parseParameter()
+		}
+	}
+	p.expect(')')
+
+	return
+}
+
+// Signature = Parameters [ Result ] .
+// Result    = Type | Parameters .
+//
+func (p *gcParser) parseSignature() *Func {
+	params, isVariadic := p.parseParameters()
+
+	// optional result type
+	var results []*ast.Object
+	switch p.tok {
+	case scanner.Ident, scanner.String, '[', '*', '<':
+		// single, unnamed result
+		result := ast.NewObj(ast.Var, "_")
+		result.Type = p.parseType()
+		results = []*ast.Object{result}
+	case '(':
+		// named or multiple result(s)
+		var variadic bool
+		results, variadic = p.parseParameters()
+		if variadic {
+			p.error("... not permitted on result type")
+		}
+	}
+
+	return &Func{Params: params, Results: results, IsVariadic: isVariadic}
+}
+
+// MethodSpec = identifier Signature .
+//
+func (p *gcParser) parseMethodSpec() *ast.Object {
+	if p.tok == scanner.Ident {
+		p.expect(scanner.Ident)
+	} else {
+		// TODO(gri) should this be parseExportedName here?
+		p.parsePkgId()
+		p.expect('.')
+		p.parseDotIdent()
+	}
+	p.parseSignature()
+
+	// TODO(gri) compute method object
+	return ast.NewObj(ast.Fun, "_")
+}
+
+// InterfaceType = "interface" "{" [ MethodList ] "}" .
+// MethodList    = MethodSpec { ";" MethodSpec } .
+//
+func (p *gcParser) parseInterfaceType() Type {
+	var methods ObjList
+
+	parseMethod := func() {
+		meth := p.parseMethodSpec()
+		methods = append(methods, meth)
+	}
+
+	p.expectKeyword("interface")
+	p.expect('{')
+	if p.tok != '}' {
+		parseMethod()
+		for p.tok == ';' {
+			p.next()
+			parseMethod()
+		}
+	}
+	p.expect('}')
+
+	methods.Sort()
+	return &Interface{Methods: methods}
+}
+
+// ChanType = ( "chan" [ "<-" ] | "<-" "chan" ) Type .
+//
+func (p *gcParser) parseChanType() Type {
+	dir := ast.SEND | ast.RECV
+	if p.tok == scanner.Ident {
+		p.expectKeyword("chan")
+		if p.tok == '<' {
+			p.expectSpecial("<-")
+			dir = ast.SEND
+		}
+	} else {
+		p.expectSpecial("<-")
+		p.expectKeyword("chan")
+		dir = ast.RECV
+	}
+	elt := p.parseType()
+	return &Chan{Dir: dir, Elt: elt}
+}
+
+// Type =
+//	BasicType | TypeName | ArrayType | SliceType | StructType |
+//      PointerType | FuncType | InterfaceType | MapType | ChanType |
+//      "(" Type ")" .
+// BasicType = ident .
+// TypeName = ExportedName .
+// SliceType = "[" "]" Type .
+// PointerType = "*" Type .
+// FuncType = "func" Signature .
+//
+func (p *gcParser) parseType() Type {
+	switch p.tok {
+	case scanner.Ident:
+		switch p.lit {
+		default:
+			return p.parseBasicType()
+		case "struct":
+			return p.parseStructType()
+		case "func":
+			// FuncType
+			p.next()
+			return p.parseSignature()
+		case "interface":
+			return p.parseInterfaceType()
+		case "map":
+			return p.parseMapType()
+		case "chan":
+			return p.parseChanType()
+		}
+	case scanner.String:
+		// TypeName
+		return p.parseExportedName(ast.Typ).Type.(Type)
+	case '[':
+		p.next() // look ahead
+		if p.tok == ']' {
+			// SliceType
+			p.next()
+			return &Slice{Elt: p.parseType()}
+		}
+		return p.parseArrayType()
+	case '*':
+		// PointerType
+		p.next()
+		return &Pointer{Base: p.parseType()}
+	case '<':
+		return p.parseChanType()
+	case '(':
+		// "(" Type ")"
+		p.next()
+		typ := p.parseType()
+		p.expect(')')
+		return typ
+	}
+	p.errorf("expected type, got %s (%q)", scanner.TokenString(p.tok), p.lit)
+	return nil
+}
+
+// ----------------------------------------------------------------------------
+// Declarations
+
+// ImportDecl = "import" identifier string_lit .
+//
+func (p *gcParser) parseImportDecl() {
+	p.expectKeyword("import")
+	// The identifier has no semantic meaning in the import data.
+	// It exists so that error messages can print the real package
+	// name: binary.ByteOrder instead of "encoding/binary".ByteOrder.
+	name := p.expect(scanner.Ident)
+	pkg := p.parsePkgId()
+	assert(pkg.Name == "" || pkg.Name == name)
+	pkg.Name = name
+}
+
+// int_lit = [ "+" | "-" ] { "0" ... "9" } .
+//
+func (p *gcParser) parseInt() (sign, val string) {
+	switch p.tok {
+	case '-':
+		p.next()
+		sign = "-"
+	case '+':
+		p.next()
+	}
+	val = p.expect(scanner.Int)
+	return
+}
+
+// number = int_lit [ "p" int_lit ] .
+//
+func (p *gcParser) parseNumber() Const {
+	// mantissa
+	sign, val := p.parseInt()
+	mant, ok := new(big.Int).SetString(sign+val, 10)
+	assert(ok)
+
+	if p.lit == "p" {
+		// exponent (base 2)
+		p.next()
+		sign, val = p.parseInt()
+		exp, err := strconv.Atoui(val)
+		if err != nil {
+			p.error(err)
+		}
+		if sign == "-" {
+			denom := big.NewInt(1)
+			denom.Lsh(denom, exp)
+			return Const{new(big.Rat).SetFrac(mant, denom)}
+		}
+		if exp > 0 {
+			mant.Lsh(mant, exp)
+		}
+		return Const{new(big.Rat).SetInt(mant)}
+	}
+
+	return Const{mant}
+}
+
+// ConstDecl   = "const" ExportedName [ Type ] "=" Literal .
+// Literal     = bool_lit | int_lit | float_lit | complex_lit | string_lit .
+// bool_lit    = "true" | "false" .
+// complex_lit = "(" float_lit "+" float_lit ")" .
+// string_lit  = `"` { unicode_char } `"` .
+//
+func (p *gcParser) parseConstDecl() {
+	p.expectKeyword("const")
+	obj := p.parseExportedName(ast.Con)
+	var x Const
+	var typ Type
+	if p.tok != '=' {
+		obj.Type = p.parseType()
+	}
+	p.expect('=')
+	switch p.tok {
+	case scanner.Ident:
+		// bool_lit
+		if p.lit != "true" && p.lit != "false" {
+			p.error("expected true or false")
+		}
+		x = Const{p.lit == "true"}
+		typ = Bool.Underlying
+		p.next()
+	case '-', scanner.Int:
+		// int_lit
+		x = p.parseNumber()
+		typ = Int.Underlying
+		if _, ok := x.val.(*big.Rat); ok {
+			typ = Float64.Underlying
+		}
+	case '(':
+		// complex_lit
+		p.next()
+		re := p.parseNumber()
+		p.expect('+')
+		im := p.parseNumber()
+		p.expect(')')
+		x = Const{cmplx{re.val.(*big.Rat), im.val.(*big.Rat)}}
+		typ = Complex128.Underlying
+	case scanner.String:
+		// string_lit
+		x = MakeConst(token.STRING, p.lit)
+		p.next()
+		typ = String.Underlying
+	default:
+		p.error("expected literal")
+	}
+	if obj.Type == nil {
+		obj.Type = typ
+	}
+	obj.Data = x
+}
+
+// TypeDecl = "type" ExportedName Type .
+//
+func (p *gcParser) parseTypeDecl() {
+	p.expectKeyword("type")
+	obj := p.parseExportedName(ast.Typ)
+
+	// The type object may have been imported before and thus already
+	// have a type associated with it. We still need to parse the type
+	// structure, but throw it away if the object already has a type.
+	// This ensures that all imports refer to the same type object for
+	// a given type declaration.
+	typ := p.parseType()
+
+	if name := obj.Type.(*Name); name.Underlying == nil {
+		assert(Underlying(typ) == typ)
+		name.Underlying = typ
+	}
+}
+
+// VarDecl = "var" ExportedName Type .
+//
+func (p *gcParser) parseVarDecl() {
+	p.expectKeyword("var")
+	obj := p.parseExportedName(ast.Var)
+	obj.Type = p.parseType()
+}
+
+// FuncDecl = "func" ExportedName Signature .
+//
+func (p *gcParser) parseFuncDecl() {
+	// "func" already consumed
+	obj := p.parseExportedName(ast.Fun)
+	obj.Type = p.parseSignature()
+}
+
+// MethodDecl = "func" Receiver identifier Signature .
+// Receiver   = "(" ( identifier | "?" ) [ "*" ] ExportedName ")" .
+//
+func (p *gcParser) parseMethodDecl() {
+	// "func" already consumed
+	p.expect('(')
+	p.parseParameter() // receiver
+	p.expect(')')
+	p.expect(scanner.Ident)
+	p.parseSignature()
+}
+
+// Decl = [ ImportDecl | ConstDecl | TypeDecl | VarDecl | FuncDecl | MethodDecl ] "\n" .
+//
+func (p *gcParser) parseDecl() {
+	switch p.lit {
+	case "import":
+		p.parseImportDecl()
+	case "const":
+		p.parseConstDecl()
+	case "type":
+		p.parseTypeDecl()
+	case "var":
+		p.parseVarDecl()
+	case "func":
+		p.next() // look ahead
+		if p.tok == '(' {
+			p.parseMethodDecl()
+		} else {
+			p.parseFuncDecl()
+		}
+	}
+	p.expect('\n')
+}
+
+// ----------------------------------------------------------------------------
+// Export
+
+// Export        = "PackageClause { Decl } "$$" .
+// PackageClause = "package" identifier [ "safe" ] "\n" .
+//
+func (p *gcParser) parseExport() *ast.Object {
+	p.expectKeyword("package")
+	name := p.expect(scanner.Ident)
+	if p.tok != '\n' {
+		// A package is safe if it was compiled with the -u flag,
+		// which disables the unsafe package.
+		// TODO(gri) remember "safe" package
+		p.expectKeyword("safe")
+	}
+	p.expect('\n')
+
+	assert(p.imports[p.id] == nil)
+	pkg := ast.NewObj(ast.Pkg, name)
+	pkg.Data = ast.NewScope(nil)
+	p.imports[p.id] = pkg
+
+	for p.tok != '$' && p.tok != scanner.EOF {
+		p.parseDecl()
+	}
+
+	if ch := p.scanner.Peek(); p.tok != '$' || ch != '$' {
+		// don't call next()/expect() since reading past the
+		// export data may cause scanner errors (e.g. NUL chars)
+		p.errorf("expected '$$', got %s %c", scanner.TokenString(p.tok), ch)
+	}
+
+	if n := p.scanner.ErrorCount; n != 0 {
+		p.errorf("expected no scanner errors, got %d", n)
+	}
+
+	return pkg
+}
diff --git a/src/pkg/go/types/gcimporter_test.go b/src/pkg/go/types/gcimporter_test.go
new file mode 100644
index 000000000..ec87f5d51
--- /dev/null
+++ b/src/pkg/go/types/gcimporter_test.go
@@ -0,0 +1,101 @@
+// Copyright 2011 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package types
+
+import (
+	"exec"
+	"go/ast"
+	"io/ioutil"
+	"path/filepath"
+	"runtime"
+	"strings"
+	"testing"
+	"time"
+)
+
+var gcName, gcPath string // compiler name and path
+
+func init() {
+	// determine compiler
+	switch runtime.GOARCH {
+	case "386":
+		gcName = "8g"
+	case "amd64":
+		gcName = "6g"
+	case "arm":
+		gcName = "5g"
+	default:
+		gcName = "unknown-GOARCH-compiler"
+		gcPath = gcName
+		return
+	}
+	gcPath, _ = exec.LookPath(gcName)
+}
+
+func compile(t *testing.T, dirname, filename string) {
+	cmd := exec.Command(gcPath, filename)
+	cmd.Dir = dirname
+	out, err := cmd.CombinedOutput()
+	if err != nil {
+		t.Errorf("%s %s failed: %s", gcName, filename, err)
+		return
+	}
+	t.Logf("%s", string(out))
+}
+
+// Use the same global imports map for all tests. The effect is
+// as if all tested packages were imported into a single package.
+var imports = make(map[string]*ast.Object)
+
+func testPath(t *testing.T, path string) bool {
+	_, err := GcImporter(imports, path)
+	if err != nil {
+		t.Errorf("testPath(%s): %s", path, err)
+		return false
+	}
+	return true
+}
+
+const maxTime = 3e9 // maximum allotted testing time in ns
+
+func testDir(t *testing.T, dir string, endTime int64) (nimports int) {
+	dirname := filepath.Join(pkgRoot, dir)
+	list, err := ioutil.ReadDir(dirname)
+	if err != nil {
+		t.Errorf("testDir(%s): %s", dirname, err)
+	}
+	for _, f := range list {
+		if time.Nanoseconds() >= endTime {
+			t.Log("testing time used up")
+			return
+		}
+		switch {
+		case f.IsRegular():
+			// try extensions
+			for _, ext := range pkgExts {
+				if strings.HasSuffix(f.Name, ext) {
+					name := f.Name[0 : len(f.Name)-len(ext)] // remove extension
+					if testPath(t, filepath.Join(dir, name)) {
+						nimports++
+					}
+				}
+			}
+		case f.IsDirectory():
+			nimports += testDir(t, filepath.Join(dir, f.Name), endTime)
+		}
+	}
+	return
+}
+
+func TestGcImport(t *testing.T) {
+	compile(t, "testdata", "exports.go")
+
+	nimports := 0
+	if testPath(t, "./testdata/exports") {
+		nimports++
+	}
+	nimports += testDir(t, "", time.Nanoseconds()+maxTime) // installed packages
+	t.Logf("tested %d imports", nimports)
+}
diff --git a/src/pkg/go/types/testdata/exports.go b/src/pkg/go/types/testdata/exports.go
new file mode 100644
index 000000000..ed63bf9ad
--- /dev/null
+++ b/src/pkg/go/types/testdata/exports.go
@@ -0,0 +1,84 @@
+// Copyright 2011 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// This file is used to generate an object file which
+// serves as test file for gcimporter_test.go.
+
+package exports
+
+import (
+	"go/ast"
+)
+
+const (
+	C0 int = 0
+	C1     = 3.14159265
+	C2     = 2.718281828i
+	C3     = -123.456e-789
+	C4     = +123.456E+789
+	C5     = 1234i
+	C6     = "foo\n"
+	C7     = `bar\n`
+)
+
+type (
+	T1  int
+	T2  [10]int
+	T3  []int
+	T4  *int
+	T5  chan int
+	T6a chan<- int
+	T6b chan (<-chan int)
+	T6c chan<- (chan int)
+	T7  <-chan *ast.File
+	T8  struct{}
+	T9  struct {
+		a    int
+		b, c float32
+		d    []string `go:"tag"`
+	}
+	T10 struct {
+		T8
+		T9
+		_ *T10
+	}
+	T11 map[int]string
+	T12 interface{}
+	T13 interface {
+		m1()
+		m2(int) float32
+	}
+	T14 interface {
+		T12
+		T13
+		m3(x ...struct{}) []T9
+	}
+	T15 func()
+	T16 func(int)
+	T17 func(x int)
+	T18 func() float32
+	T19 func() (x float32)
+	T20 func(...interface{})
+	T21 struct{ next *T21 }
+	T22 struct{ link *T23 }
+	T23 struct{ link *T22 }
+	T24 *T24
+	T25 *T26
+	T26 *T27
+	T27 *T25
+	T28 func(T28) T28
+)
+
+var (
+	V0 int
+	V1 = -991.0
+)
+
+func F1()         {}
+func F2(x int)    {}
+func F3() int     { return 0 }
+func F4() float32 { return 0 }
+func F5(a, b, c int, u, v, w struct{ x, y T1 }, more ...interface{}) (p, q, r chan<- T10)
+
+func (p *T1) M1()
diff --git a/src/pkg/go/types/testdata/test0.src b/src/pkg/go/types/testdata/test0.src
new file mode 100644
index 000000000..84a1abe27
--- /dev/null
+++ b/src/pkg/go/types/testdata/test0.src
@@ -0,0 +1,154 @@
+// Copyright 2011 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// type declarations
+
+package test0
+
+import "unsafe"
+
+const pi = 3.1415
+
+type (
+	N undeclared /* ERROR "undeclared" */
+	B bool
+	I int32
+	A [10]P
+	T struct {
+		x, y P
+	}
+	P *T
+	R (*R)
+	F func(A) I
+	Y interface {
+		f(A) I
+	}
+	S [](((P)))
+	M map[I]F
+	C chan<- I
+)
+
+
+type (
+	p1 pi /* ERROR "not a package" */ .foo
+	p2 unsafe.Pointer
+)
+
+
+type (
+	Pi pi /* ERROR "not a type" */
+
+	a /* DISABLED "illegal cycle" */ a
+	a /* ERROR "redeclared" */ int
+
+	// where the cycle error appears depends on the
+	// order in which declarations are processed
+	// (which depends on the order in which a map
+	// is iterated through)
+	b c
+	c /* DISABLED "illegal cycle" */ d
+	d e
+	e b
+
+	t *t
+
+	U V
+	V *W
+	W U
+
+	P1 *S2
+	P2 P1
+
+	S0 struct {
+	}
+	S1 struct {
+		a, b, c int
+		u, v, a /* ERROR "redeclared" */ float32
+	}
+	S2 struct {
+		U // anonymous field
+		// TODO(gri) recognize double-declaration below
+		// U /* ERROR "redeclared" */ int
+	}
+	S3 struct {
+		x S2
+	}
+	S4/* DISABLED "illegal cycle" */ struct {
+		S4
+	}
+	S5 struct {
+		S6
+	}
+	S6 /* DISABLED "illegal cycle" */ struct {
+		field S7
+	}
+	S7 struct {
+		S5
+	}
+
+	L1 []L1
+	L2 []int
+
+	A1 [10]int
+	A2 /* DISABLED "illegal cycle" */ [10]A2
+	A3 /* DISABLED "illegal cycle" */ [10]struct {
+		x A4
+	}
+	A4 [10]A3
+
+	F1 func()
+	F2 func(x, y, z float32)
+	F3 func(x, y, x /* ERROR "redeclared" */ float32)
+	F4 func() (x, y, x /* ERROR "redeclared" */ float32)
+	F5 func(x int) (x /* ERROR "redeclared" */ float32)
+	F6 func(x ...int)
+
+	I1 interface{}
+	I2 interface {
+		m1()
+	}
+	I3 interface {
+		m1()
+		m1 /* ERROR "redeclared" */ ()
+	}
+	I4 interface {
+		m1(x, y, x /* ERROR "redeclared" */ float32)
+		m2() (x, y, x /* ERROR "redeclared" */ float32)
+		m3(x int) (x /* ERROR "redeclared" */ float32)
+	}
+	I5 interface {
+		m1(I5)
+	}
+	I6 interface {
+		S0 /* ERROR "non-interface" */
+	}
+	I7 interface {
+		I1
+		I1
+	}
+	I8 /* DISABLED "illegal cycle" */ interface {
+		I8
+	}
+	I9 /* DISABLED "illegal cycle" */ interface {
+		I10
+	}
+	I10 interface {
+		I11
+	}
+	I11 interface {
+		I9
+	}
+
+	C1 chan int
+	C2 <-chan int
+	C3 chan<- C3
+	C4 chan C5
+	C5 chan C6
+	C6 chan C4
+
+	M1 map[Last]string
+	M2 map[string]M2
+
+	Last int
+)
diff --git a/src/pkg/go/types/types.go b/src/pkg/go/types/types.go
new file mode 100644
index 000000000..3aa896892
--- /dev/null
+++ b/src/pkg/go/types/types.go
@@ -0,0 +1,255 @@
+// Copyright 2011 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// PACKAGE UNDER CONSTRUCTION. ANY AND ALL PARTS MAY CHANGE.
+// Package types declares the types used to represent Go types.
+//
+package types
+
+import (
+	"go/ast"
+	"sort"
+)
+
+// All types implement the Type interface.
+type Type interface {
+	isType()
+}
+
+// All concrete types embed ImplementsType which
+// ensures that all types implement the Type interface.
+type ImplementsType struct{}
+
+func (t *ImplementsType) isType() {}
+
+// A Bad type is a non-nil placeholder type when we don't know a type.
+type Bad struct {
+	ImplementsType
+	Msg string // for better error reporting/debugging
+}
+
+// A Basic represents a (unnamed) basic type.
+type Basic struct {
+	ImplementsType
+	// TODO(gri) need a field specifying the exact basic type
+}
+
+// An Array represents an array type [Len]Elt.
+type Array struct {
+	ImplementsType
+	Len uint64
+	Elt Type
+}
+
+// A Slice represents a slice type []Elt.
+type Slice struct {
+	ImplementsType
+	Elt Type
+}
+
+// A Struct represents a struct type struct{...}.
+// Anonymous fields are represented by objects with empty names.
+type Struct struct {
+	ImplementsType
+	Fields ObjList  // struct fields; or nil
+	Tags   []string // corresponding tags; or nil
+	// TODO(gri) This type needs some rethinking:
+	// - at the moment anonymous fields are marked with "" object names,
+	//   and their names have to be reconstructed
+	// - there is no scope for fast lookup (but the parser creates one)
+}
+
+// A Pointer represents a pointer type *Base.
+type Pointer struct {
+	ImplementsType
+	Base Type
+}
+
+// A Func represents a function type func(...) (...).
+// Unnamed parameters are represented by objects with empty names.
+type Func struct {
+	ImplementsType
+	Recv       *ast.Object // nil if not a method
+	Params     ObjList     // (incoming) parameters from left to right; or nil
+	Results    ObjList     // (outgoing) results from left to right; or nil
+	IsVariadic bool        // true if the last parameter's type is of the form ...T
+}
+
+// An Interface represents an interface type interface{...}.
+type Interface struct {
+	ImplementsType
+	Methods ObjList // interface methods sorted by name; or nil
+}
+
+// A Map represents a map type map[Key]Elt.
+type Map struct {
+	ImplementsType
+	Key, Elt Type
+}
+
+// A Chan represents a channel type chan Elt, <-chan Elt, or chan<-Elt.
+type Chan struct {
+	ImplementsType
+	Dir ast.ChanDir
+	Elt Type
+}
+
+// A Name represents a named type as declared in a type declaration.
+type Name struct {
+	ImplementsType
+	Underlying Type        // nil if not fully declared
+	Obj        *ast.Object // corresponding declared object
+	// TODO(gri) need to remember fields and methods.
+}
+
+// If typ is a pointer type, Deref returns the pointer's base type;
+// otherwise it returns typ.
+func Deref(typ Type) Type {
+	if typ, ok := typ.(*Pointer); ok {
+		return typ.Base
+	}
+	return typ
+}
+
+// Underlying returns the underlying type of a type.
+func Underlying(typ Type) Type {
+	if typ, ok := typ.(*Name); ok {
+		utyp := typ.Underlying
+		if _, ok := utyp.(*Basic); !ok {
+			return utyp
+		}
+		// the underlying type of a type name referring
+		// to an (untyped) basic type is the basic type
+		// name
+	}
+	return typ
+}
+
+// An ObjList represents an ordered (in some fashion) list of objects.
+type ObjList []*ast.Object
+
+// ObjList implements sort.Interface.
+func (list ObjList) Len() int           { return len(list) }
+func (list ObjList) Less(i, j int) bool { return list[i].Name < list[j].Name }
+func (list ObjList) Swap(i, j int)      { list[i], list[j] = list[j], list[i] }
+
+// Sort sorts an object list by object name.
+func (list ObjList) Sort() { sort.Sort(list) }
+
+// identicalTypes returns true if both lists a and b have the
+// same length and corresponding objects have identical types.
+func identicalTypes(a, b ObjList) bool {
+	if len(a) == len(b) {
+		for i, x := range a {
+			y := b[i]
+			if !Identical(x.Type.(Type), y.Type.(Type)) {
+				return false
+			}
+		}
+		return true
+	}
+	return false
+}
+
+// Identical returns true if two types are identical.
+func Identical(x, y Type) bool {
+	if x == y {
+		return true
+	}
+
+	switch x := x.(type) {
+	case *Bad:
+		// A Bad type is always identical to any other type
+		// (to avoid spurious follow-up errors).
+		return true
+
+	case *Basic:
+		if y, ok := y.(*Basic); ok {
+			panic("unimplemented")
+			_ = y
+		}
+
+	case *Array:
+		// Two array types are identical if they have identical element types
+		// and the same array length.
+		if y, ok := y.(*Array); ok {
+			return x.Len == y.Len && Identical(x.Elt, y.Elt)
+		}
+
+	case *Slice:
+		// Two slice types are identical if they have identical element types.
+		if y, ok := y.(*Slice); ok {
+			return Identical(x.Elt, y.Elt)
+		}
+
+	case *Struct:
+		// Two struct types are identical if they have the same sequence of fields,
+		// and if corresponding fields have the same names, and identical types,
+		// and identical tags. Two anonymous fields are considered to have the same
+		// name. Lower-case field names from different packages are always different.
+		if y, ok := y.(*Struct); ok {
+			// TODO(gri) handle structs from different packages
+			if identicalTypes(x.Fields, y.Fields) {
+				for i, f := range x.Fields {
+					g := y.Fields[i]
+					if f.Name != g.Name || x.Tags[i] != y.Tags[i] {
+						return false
+					}
+				}
+				return true
+			}
+		}
+
+	case *Pointer:
+		// Two pointer types are identical if they have identical base types.
+		if y, ok := y.(*Pointer); ok {
+			return Identical(x.Base, y.Base)
+		}
+
+	case *Func:
+		// Two function types are identical if they have the same number of parameters
+		// and result values, corresponding parameter and result types are identical,
+		// and either both functions are variadic or neither is. Parameter and result
+		// names are not required to match.
+		if y, ok := y.(*Func); ok {
+			return identicalTypes(x.Params, y.Params) &&
+				identicalTypes(x.Results, y.Results) &&
+				x.IsVariadic == y.IsVariadic
+		}
+
+	case *Interface:
+		// Two interface types are identical if they have the same set of methods with
+		// the same names and identical function types. Lower-case method names from
+		// different packages are always different. The order of the methods is irrelevant.
+		if y, ok := y.(*Interface); ok {
+			return identicalTypes(x.Methods, y.Methods) // methods are sorted
+		}
+
+	case *Map:
+		// Two map types are identical if they have identical key and value types.
+		if y, ok := y.(*Map); ok {
+			return Identical(x.Key, y.Key) && Identical(x.Elt, y.Elt)
+		}
+
+	case *Chan:
+		// Two channel types are identical if they have identical value types
+		// and the same direction.
+		if y, ok := y.(*Chan); ok {
+			return x.Dir == y.Dir && Identical(x.Elt, y.Elt)
+		}
+
+	case *Name:
+		// Two named types are identical if their type names originate
+		// in the same type declaration.
+		if y, ok := y.(*Name); ok {
+			return x.Obj == y.Obj ||
+				// permit bad objects to be equal to avoid
+				// follow up errors
+				x.Obj != nil && x.Obj.Kind == ast.Bad ||
+				y.Obj != nil && y.Obj.Kind == ast.Bad
+		}
+	}
+
+	return false
+}
diff --git a/src/pkg/go/types/universe.go b/src/pkg/go/types/universe.go
new file mode 100644
index 000000000..6ae88e5f9
--- /dev/null
+++ b/src/pkg/go/types/universe.go
@@ -0,0 +1,108 @@
+// Copyright 2011 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// FILE UNDER CONSTRUCTION. ANY AND ALL PARTS MAY CHANGE.
+// This file implements the universe and unsafe package scopes.
+
+package types
+
+import "go/ast"
+
+var (
+	scope    *ast.Scope // current scope to use for initialization
+	Universe *ast.Scope
+	Unsafe   *ast.Object // package unsafe
+)
+
+func define(kind ast.ObjKind, name string) *ast.Object {
+	obj := ast.NewObj(kind, name)
+	if scope.Insert(obj) != nil {
+		panic("types internal error: double declaration")
+	}
+	return obj
+}
+
+func defType(name string) *Name {
+	obj := define(ast.Typ, name)
+	typ := &Name{Underlying: &Basic{}, Obj: obj}
+	obj.Type = typ
+	return typ
+}
+
+func defConst(name string) {
+	obj := define(ast.Con, name)
+	_ = obj // TODO(gri) fill in other properties
+}
+
+func defFun(name string) {
+	obj := define(ast.Fun, name)
+	_ = obj // TODO(gri) fill in other properties
+}
+
+var (
+	Bool,
+	Int,
+	Float64,
+	Complex128,
+	String *Name
+)
+
+func init() {
+	scope = ast.NewScope(nil)
+	Universe = scope
+
+	Bool = defType("bool")
+	defType("byte") // TODO(gri) should be an alias for uint8
+	defType("complex64")
+	Complex128 = defType("complex128")
+	defType("float32")
+	Float64 = defType("float64")
+	defType("int8")
+	defType("int16")
+	defType("int32")
+	defType("int64")
+	String = defType("string")
+	defType("uint8")
+	defType("uint16")
+	defType("uint32")
+	defType("uint64")
+	Int = defType("int")
+	defType("uint")
+	defType("uintptr")
+
+	defConst("true")
+	defConst("false")
+	defConst("iota")
+	defConst("nil")
+
+	defFun("append")
+	defFun("cap")
+	defFun("close")
+	defFun("complex")
+	defFun("copy")
+	defFun("imag")
+	defFun("len")
+	defFun("make")
+	defFun("new")
+	defFun("panic")
+	defFun("print")
+	defFun("println")
+	defFun("real")
+	defFun("recover")
+
+	scope = ast.NewScope(nil)
+	Unsafe = ast.NewObj(ast.Pkg, "unsafe")
+	Unsafe.Data = scope
+
+	defType("Pointer")
+
+	defFun("Alignof")
+	defFun("New")
+	defFun("NewArray")
+	defFun("Offsetof")
+	defFun("Reflect")
+	defFun("Sizeof")
+	defFun("Typeof")
+	defFun("Unreflect")
+}
diff --git a/src/pkg/gob/Makefile b/src/pkg/gob/Makefile
new file mode 100644
index 000000000..68007c189
--- /dev/null
+++ b/src/pkg/gob/Makefile
@@ -0,0 +1,25 @@
+# Copyright 2009 The Go Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style
+# license that can be found in the LICENSE file.
+
+include ../../Make.inc
+
+TARG=gob
+GOFILES=\
+	decode.go\
+	decoder.go\
+	doc.go\
+	encode.go\
+	encoder.go\
+	error.go\
+	type.go\
+
+include ../../Make.pkg
+
+# Help for debugging. Requires adding debug.go to the gob package as well.
+
+dump:	dump.$O
+	$(LD) -o dump $<
+
+dump.$O:	dump.go
+	$(GC) $<
diff --git a/src/pkg/gob/codec_test.go b/src/pkg/gob/codec_test.go
new file mode 100644
index 000000000..a5fb91cda
--- /dev/null
+++ b/src/pkg/gob/codec_test.go
@@ -0,0 +1,1406 @@
+// Copyright 2009 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package gob
+
+import (
+	"bytes"
+	"math"
+	"os"
+	"reflect"
+	"strings"
+	"testing"
+	"unsafe"
+)
+
+// Guarantee encoding format by comparing some encodings to hand-written values
+type EncodeT struct {
+	x uint64
+	b []byte
+}
+
+var encodeT = []EncodeT{
+	{0x00, []byte{0x00}},
+	{0x0F, []byte{0x0F}},
+	{0xFF, []byte{0xFF, 0xFF}},
+	{0xFFFF, []byte{0xFE, 0xFF, 0xFF}},
+	{0xFFFFFF, []byte{0xFD, 0xFF, 0xFF, 0xFF}},
+	{0xFFFFFFFF, []byte{0xFC, 0xFF, 0xFF, 0xFF, 0xFF}},
+	{0xFFFFFFFFFF, []byte{0xFB, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF}},
+	{0xFFFFFFFFFFFF, []byte{0xFA, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF}},
+	{0xFFFFFFFFFFFFFF, []byte{0xF9, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF}},
+	{0xFFFFFFFFFFFFFFFF, []byte{0xF8, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF}},
+	{0x1111, []byte{0xFE, 0x11, 0x11}},
+	{0x1111111111111111, []byte{0xF8, 0x11, 0x11, 0x11, 0x11, 0x11, 0x11, 0x11, 0x11}},
+	{0x8888888888888888, []byte{0xF8, 0x88, 0x88, 0x88, 0x88, 0x88, 0x88, 0x88, 0x88}},
+	{1 << 63, []byte{0xF8, 0x80, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00}},
+}
+
+// testError is meant to be used as a deferred function to turn a panic(gobError) into a
+// plain test.Error call.
+func testError(t *testing.T) {
+	if e := recover(); e != nil {
+		t.Error(e.(gobError).Error) // Will re-panic if not one of our errors, such as a runtime error.
+	}
+	return
+}
+
+// Test basic encode/decode routines for unsigned integers
+func TestUintCodec(t *testing.T) {
+	defer testError(t)
+	b := new(bytes.Buffer)
+	encState := newEncoderState(b)
+	for _, tt := range encodeT {
+		b.Reset()
+		encState.encodeUint(tt.x)
+		if !bytes.Equal(tt.b, b.Bytes()) {
+			t.Errorf("encodeUint: %#x encode: expected % x got % x", tt.x, tt.b, b.Bytes())
+		}
+	}
+	decState := newDecodeState(b)
+	for u := uint64(0); ; u = (u + 1) * 7 {
+		b.Reset()
+		encState.encodeUint(u)
+		v := decState.decodeUint()
+		if u != v {
+			t.Errorf("Encode/Decode: sent %#x received %#x", u, v)
+		}
+		if u&(1<<63) != 0 {
+			break
+		}
+	}
+}
+
+func verifyInt(i int64, t *testing.T) {
+	defer testError(t)
+	var b = new(bytes.Buffer)
+	encState := newEncoderState(b)
+	encState.encodeInt(i)
+	decState := newDecodeState(b)
+	decState.buf = make([]byte, 8)
+	j := decState.decodeInt()
+	if i != j {
+		t.Errorf("Encode/Decode: sent %#x received %#x", uint64(i), uint64(j))
+	}
+}
+
+// Test basic encode/decode routines for signed integers
+func TestIntCodec(t *testing.T) {
+	for u := uint64(0); ; u = (u + 1) * 7 {
+		// Do positive and negative values
+		i := int64(u)
+		verifyInt(i, t)
+		verifyInt(-i, t)
+		verifyInt(^i, t)
+		if u&(1<<63) != 0 {
+			break
+		}
+	}
+	verifyInt(-1<<63, t) // a tricky case
+}
+
+// The result of encoding a true boolean with field number 7
+var boolResult = []byte{0x07, 0x01}
+// The result of encoding a number 17 with field number 7
+var signedResult = []byte{0x07, 2 * 17}
+var unsignedResult = []byte{0x07, 17}
+var floatResult = []byte{0x07, 0xFE, 0x31, 0x40}
+// The result of encoding a number 17+19i with field number 7
+var complexResult = []byte{0x07, 0xFE, 0x31, 0x40, 0xFE, 0x33, 0x40}
+// The result of encoding "hello" with field number 7
+var bytesResult = []byte{0x07, 0x05, 'h', 'e', 'l', 'l', 'o'}
+
+func newDecodeState(buf *bytes.Buffer) *decoderState {
+	d := new(decoderState)
+	d.b = buf
+	d.buf = make([]byte, uint64Size)
+	return d
+}
+
+func newEncoderState(b *bytes.Buffer) *encoderState {
+	b.Reset()
+	state := &encoderState{enc: nil, b: b}
+	state.fieldnum = -1
+	return state
+}
+
+// Test instruction execution for encoding.
+// Do not run the machine yet; instead do individual instructions crafted by hand.
+func TestScalarEncInstructions(t *testing.T) {
+	var b = new(bytes.Buffer)
+
+	// bool
+	{
+		data := struct{ a bool }{true}
+		instr := &encInstr{encBool, 6, 0, 0}
+		state := newEncoderState(b)
+		instr.op(instr, state, unsafe.Pointer(&data))
+		if !bytes.Equal(boolResult, b.Bytes()) {
+			t.Errorf("bool enc instructions: expected % x got % x", boolResult, b.Bytes())
+		}
+	}
+
+	// int
+	{
+		b.Reset()
+		data := struct{ a int }{17}
+		instr := &encInstr{encInt, 6, 0, 0}
+		state := newEncoderState(b)
+		instr.op(instr, state, unsafe.Pointer(&data))
+		if !bytes.Equal(signedResult, b.Bytes()) {
+			t.Errorf("int enc instructions: expected % x got % x", signedResult, b.Bytes())
+		}
+	}
+
+	// uint
+	{
+		b.Reset()
+		data := struct{ a uint }{17}
+		instr := &encInstr{encUint, 6, 0, 0}
+		state := newEncoderState(b)
+		instr.op(instr, state, unsafe.Pointer(&data))
+		if !bytes.Equal(unsignedResult, b.Bytes()) {
+			t.Errorf("uint enc instructions: expected % x got % x", unsignedResult, b.Bytes())
+		}
+	}
+
+	// int8
+	{
+		b.Reset()
+		data := struct{ a int8 }{17}
+		instr := &encInstr{encInt8, 6, 0, 0}
+		state := newEncoderState(b)
+		instr.op(instr, state, unsafe.Pointer(&data))
+		if !bytes.Equal(signedResult, b.Bytes()) {
+			t.Errorf("int8 enc instructions: expected % x got % x", signedResult, b.Bytes())
+		}
+	}
+
+	// uint8
+	{
+		b.Reset()
+		data := struct{ a uint8 }{17}
+		instr := &encInstr{encUint8, 6, 0, 0}
+		state := newEncoderState(b)
+		instr.op(instr, state, unsafe.Pointer(&data))
+		if !bytes.Equal(unsignedResult, b.Bytes()) {
+			t.Errorf("uint8 enc instructions: expected % x got % x", unsignedResult, b.Bytes())
+		}
+	}
+
+	// int16
+	{
+		b.Reset()
+		data := struct{ a int16 }{17}
+		instr := &encInstr{encInt16, 6, 0, 0}
+		state := newEncoderState(b)
+		instr.op(instr, state, unsafe.Pointer(&data))
+		if !bytes.Equal(signedResult, b.Bytes()) {
+			t.Errorf("int16 enc instructions: expected % x got % x", signedResult, b.Bytes())
+		}
+	}
+
+	// uint16
+	{
+		b.Reset()
+		data := struct{ a uint16 }{17}
+		instr := &encInstr{encUint16, 6, 0, 0}
+		state := newEncoderState(b)
+		instr.op(instr, state, unsafe.Pointer(&data))
+		if !bytes.Equal(unsignedResult, b.Bytes()) {
+			t.Errorf("uint16 enc instructions: expected % x got % x", unsignedResult, b.Bytes())
+		}
+	}
+
+	// int32
+	{
+		b.Reset()
+		data := struct{ a int32 }{17}
+		instr := &encInstr{encInt32, 6, 0, 0}
+		state := newEncoderState(b)
+		instr.op(instr, state, unsafe.Pointer(&data))
+		if !bytes.Equal(signedResult, b.Bytes()) {
+			t.Errorf("int32 enc instructions: expected % x got % x", signedResult, b.Bytes())
+		}
+	}
+
+	// uint32
+	{
+		b.Reset()
+		data := struct{ a uint32 }{17}
+		instr := &encInstr{encUint32, 6, 0, 0}
+		state := newEncoderState(b)
+		instr.op(instr, state, unsafe.Pointer(&data))
+		if !bytes.Equal(unsignedResult, b.Bytes()) {
+			t.Errorf("uint32 enc instructions: expected % x got % x", unsignedResult, b.Bytes())
+		}
+	}
+
+	// int64
+	{
+		b.Reset()
+		data := struct{ a int64 }{17}
+		instr := &encInstr{encInt64, 6, 0, 0}
+		state := newEncoderState(b)
+		instr.op(instr, state, unsafe.Pointer(&data))
+		if !bytes.Equal(signedResult, b.Bytes()) {
+			t.Errorf("int64 enc instructions: expected % x got % x", signedResult, b.Bytes())
+		}
+	}
+
+	// uint64
+	{
+		b.Reset()
+		data := struct{ a uint64 }{17}
+		instr := &encInstr{encUint64, 6, 0, 0}
+		state := newEncoderState(b)
+		instr.op(instr, state, unsafe.Pointer(&data))
+		if !bytes.Equal(unsignedResult, b.Bytes()) {
+			t.Errorf("uint64 enc instructions: expected % x got % x", unsignedResult, b.Bytes())
+		}
+	}
+
+	// float32
+	{
+		b.Reset()
+		data := struct{ a float32 }{17}
+		instr := &encInstr{encFloat32, 6, 0, 0}
+		state := newEncoderState(b)
+		instr.op(instr, state, unsafe.Pointer(&data))
+		if !bytes.Equal(floatResult, b.Bytes()) {
+			t.Errorf("float32 enc instructions: expected % x got % x", floatResult, b.Bytes())
+		}
+	}
+
+	// float64
+	{
+		b.Reset()
+		data := struct{ a float64 }{17}
+		instr := &encInstr{encFloat64, 6, 0, 0}
+		state := newEncoderState(b)
+		instr.op(instr, state, unsafe.Pointer(&data))
+		if !bytes.Equal(floatResult, b.Bytes()) {
+			t.Errorf("float64 enc instructions: expected % x got % x", floatResult, b.Bytes())
+		}
+	}
+
+	// bytes == []uint8
+	{
+		b.Reset()
+		data := struct{ a []byte }{[]byte("hello")}
+		instr := &encInstr{encUint8Array, 6, 0, 0}
+		state := newEncoderState(b)
+		instr.op(instr, state, unsafe.Pointer(&data))
+		if !bytes.Equal(bytesResult, b.Bytes()) {
+			t.Errorf("bytes enc instructions: expected % x got % x", bytesResult, b.Bytes())
+		}
+	}
+
+	// string
+	{
+		b.Reset()
+		data := struct{ a string }{"hello"}
+		instr := &encInstr{encString, 6, 0, 0}
+		state := newEncoderState(b)
+		instr.op(instr, state, unsafe.Pointer(&data))
+		if !bytes.Equal(bytesResult, b.Bytes()) {
+			t.Errorf("string enc instructions: expected % x got % x", bytesResult, b.Bytes())
+		}
+	}
+}
+
+func execDec(typ string, instr *decInstr, state *decoderState, t *testing.T, p unsafe.Pointer) {
+	defer testError(t)
+	v := int(state.decodeUint())
+	if v+state.fieldnum != 6 {
+		t.Fatalf("decoding field number %d, got %d", 6, v+state.fieldnum)
+	}
+	instr.op(instr, state, decIndirect(p, instr.indir))
+	state.fieldnum = 6
+}
+
+func newDecodeStateFromData(data []byte) *decoderState {
+	b := bytes.NewBuffer(data)
+	state := newDecodeState(b)
+	state.fieldnum = -1
+	return state
+}
+
+// Test instruction execution for decoding.
+// Do not run the machine yet; instead do individual instructions crafted by hand.
+func TestScalarDecInstructions(t *testing.T) {
+	ovfl := os.NewError("overflow")
+
+	// bool
+	{
+		var data struct {
+			a bool
+		}
+		instr := &decInstr{decBool, 6, 0, 0, ovfl}
+		state := newDecodeStateFromData(boolResult)
+		execDec("bool", instr, state, t, unsafe.Pointer(&data))
+		if data.a != true {
+			t.Errorf("bool a = %v not true", data.a)
+		}
+	}
+	// int
+	{
+		var data struct {
+			a int
+		}
+		instr := &decInstr{decOpTable[reflect.Int], 6, 0, 0, ovfl}
+		state := newDecodeStateFromData(signedResult)
+		execDec("int", instr, state, t, unsafe.Pointer(&data))
+		if data.a != 17 {
+			t.Errorf("int a = %v not 17", data.a)
+		}
+	}
+
+	// uint
+	{
+		var data struct {
+			a uint
+		}
+		instr := &decInstr{decOpTable[reflect.Uint], 6, 0, 0, ovfl}
+		state := newDecodeStateFromData(unsignedResult)
+		execDec("uint", instr, state, t, unsafe.Pointer(&data))
+		if data.a != 17 {
+			t.Errorf("uint a = %v not 17", data.a)
+		}
+	}
+
+	// int8
+	{
+		var data struct {
+			a int8
+		}
+		instr := &decInstr{decInt8, 6, 0, 0, ovfl}
+		state := newDecodeStateFromData(signedResult)
+		execDec("int8", instr, state, t, unsafe.Pointer(&data))
+		if data.a != 17 {
+			t.Errorf("int8 a = %v not 17", data.a)
+		}
+	}
+
+	// uint8
+	{
+		var data struct {
+			a uint8
+		}
+		instr := &decInstr{decUint8, 6, 0, 0, ovfl}
+		state := newDecodeStateFromData(unsignedResult)
+		execDec("uint8", instr, state, t, unsafe.Pointer(&data))
+		if data.a != 17 {
+			t.Errorf("uint8 a = %v not 17", data.a)
+		}
+	}
+
+	// int16
+	{
+		var data struct {
+			a int16
+		}
+		instr := &decInstr{decInt16, 6, 0, 0, ovfl}
+		state := newDecodeStateFromData(signedResult)
+		execDec("int16", instr, state, t, unsafe.Pointer(&data))
+		if data.a != 17 {
+			t.Errorf("int16 a = %v not 17", data.a)
+		}
+	}
+
+	// uint16
+	{
+		var data struct {
+			a uint16
+		}
+		instr := &decInstr{decUint16, 6, 0, 0, ovfl}
+		state := newDecodeStateFromData(unsignedResult)
+		execDec("uint16", instr, state, t, unsafe.Pointer(&data))
+		if data.a != 17 {
+			t.Errorf("uint16 a = %v not 17", data.a)
+		}
+	}
+
+	// int32
+	{
+		var data struct {
+			a int32
+		}
+		instr := &decInstr{decInt32, 6, 0, 0, ovfl}
+		state := newDecodeStateFromData(signedResult)
+		execDec("int32", instr, state, t, unsafe.Pointer(&data))
+		if data.a != 17 {
+			t.Errorf("int32 a = %v not 17", data.a)
+		}
+	}
+
+	// uint32
+	{
+		var data struct {
+			a uint32
+		}
+		instr := &decInstr{decUint32, 6, 0, 0, ovfl}
+		state := newDecodeStateFromData(unsignedResult)
+		execDec("uint32", instr, state, t, unsafe.Pointer(&data))
+		if data.a != 17 {
+			t.Errorf("uint32 a = %v not 17", data.a)
+		}
+	}
+
+	// uintptr
+	{
+		var data struct {
+			a uintptr
+		}
+		instr := &decInstr{decOpTable[reflect.Uintptr], 6, 0, 0, ovfl}
+		state := newDecodeStateFromData(unsignedResult)
+		execDec("uintptr", instr, state, t, unsafe.Pointer(&data))
+		if data.a != 17 {
+			t.Errorf("uintptr a = %v not 17", data.a)
+		}
+	}
+
+	// int64
+	{
+		var data struct {
+			a int64
+		}
+		instr := &decInstr{decInt64, 6, 0, 0, ovfl}
+		state := newDecodeStateFromData(signedResult)
+		execDec("int64", instr, state, t, unsafe.Pointer(&data))
+		if data.a != 17 {
+			t.Errorf("int64 a = %v not 17", data.a)
+		}
+	}
+
+	// uint64
+	{
+		var data struct {
+			a uint64
+		}
+		instr := &decInstr{decUint64, 6, 0, 0, ovfl}
+		state := newDecodeStateFromData(unsignedResult)
+		execDec("uint64", instr, state, t, unsafe.Pointer(&data))
+		if data.a != 17 {
+			t.Errorf("uint64 a = %v not 17", data.a)
+		}
+	}
+
+	// float32
+	{
+		var data struct {
+			a float32
+		}
+		instr := &decInstr{decFloat32, 6, 0, 0, ovfl}
+		state := newDecodeStateFromData(floatResult)
+		execDec("float32", instr, state, t, unsafe.Pointer(&data))
+		if data.a != 17 {
+			t.Errorf("float32 a = %v not 17", data.a)
+		}
+	}
+
+	// float64
+	{
+		var data struct {
+			a float64
+		}
+		instr := &decInstr{decFloat64, 6, 0, 0, ovfl}
+		state := newDecodeStateFromData(floatResult)
+		execDec("float64", instr, state, t, unsafe.Pointer(&data))
+		if data.a != 17 {
+			t.Errorf("float64 a = %v not 17", data.a)
+		}
+	}
+
+	// complex64
+	{
+		var data struct {
+			a complex64
+		}
+		instr := &decInstr{decOpTable[reflect.Complex64], 6, 0, 0, ovfl}
+		state := newDecodeStateFromData(complexResult)
+		execDec("complex", instr, state, t, unsafe.Pointer(&data))
+		if data.a != 17+19i {
+			t.Errorf("complex a = %v not 17+19i", data.a)
+		}
+	}
+
+	// complex128
+	{
+		var data struct {
+			a complex128
+		}
+		instr := &decInstr{decOpTable[reflect.Complex128], 6, 0, 0, ovfl}
+		state := newDecodeStateFromData(complexResult)
+		execDec("complex", instr, state, t, unsafe.Pointer(&data))
+		if data.a != 17+19i {
+			t.Errorf("complex a = %v not 17+19i", data.a)
+		}
+	}
+
+	// bytes == []uint8
+	{
+		var data struct {
+			a []byte
+		}
+		instr := &decInstr{decUint8Array, 6, 0, 0, ovfl}
+		state := newDecodeStateFromData(bytesResult)
+		execDec("bytes", instr, state, t, unsafe.Pointer(&data))
+		if string(data.a) != "hello" {
+			t.Errorf(`bytes a = %q not "hello"`, string(data.a))
+		}
+	}
+
+	// string
+	{
+		var data struct {
+			a string
+		}
+		instr := &decInstr{decString, 6, 0, 0, ovfl}
+		state := newDecodeStateFromData(bytesResult)
+		execDec("bytes", instr, state, t, unsafe.Pointer(&data))
+		if data.a != "hello" {
+			t.Errorf(`bytes a = %q not "hello"`, data.a)
+		}
+	}
+}
+
+func TestEndToEnd(t *testing.T) {
+	type T2 struct {
+		T string
+	}
+	s1 := "string1"
+	s2 := "string2"
+	type T1 struct {
+		A, B, C  int
+		M        map[string]*float64
+		EmptyMap map[string]int // to check that we receive a non-nil map.
+		N        *[3]float64
+		Strs     *[2]string
+		Int64s   *[]int64
+		RI       complex64
+		S        string
+		Y        []byte
+		T        *T2
+	}
+	pi := 3.14159
+	e := 2.71828
+	t1 := &T1{
+		A:        17,
+		B:        18,
+		C:        -5,
+		M:        map[string]*float64{"pi": &pi, "e": &e},
+		EmptyMap: make(map[string]int),
+		N:        &[3]float64{1.5, 2.5, 3.5},
+		Strs:     &[2]string{s1, s2},
+		Int64s:   &[]int64{77, 89, 123412342134},
+		RI:       17 - 23i,
+		S:        "Now is the time",
+		Y:        []byte("hello, sailor"),
+		T:        &T2{"this is T2"},
+	}
+	b := new(bytes.Buffer)
+	err := NewEncoder(b).Encode(t1)
+	if err != nil {
+		t.Error("encode:", err)
+	}
+	var _t1 T1
+	err = NewDecoder(b).Decode(&_t1)
+	if err != nil {
+		t.Fatal("decode:", err)
+	}
+	if !reflect.DeepEqual(t1, &_t1) {
+		t.Errorf("encode expected %v got %v", *t1, _t1)
+	}
+	// Be absolutely sure the received map is non-nil.
+	if t1.EmptyMap == nil {
+		t.Errorf("nil map sent")
+	}
+	if _t1.EmptyMap == nil {
+		t.Errorf("nil map received")
+	}
+}
+
+func TestOverflow(t *testing.T) {
+	type inputT struct {
+		Maxi int64
+		Mini int64
+		Maxu uint64
+		Maxf float64
+		Minf float64
+		Maxc complex128
+		Minc complex128
+	}
+	var it inputT
+	var err os.Error
+	b := new(bytes.Buffer)
+	enc := NewEncoder(b)
+	dec := NewDecoder(b)
+
+	// int8
+	b.Reset()
+	it = inputT{
+		Maxi: math.MaxInt8 + 1,
+	}
+	type outi8 struct {
+		Maxi int8
+		Mini int8
+	}
+	var o1 outi8
+	enc.Encode(it)
+	err = dec.Decode(&o1)
+	if err == nil || err.String() != `value for "Maxi" out of range` {
+		t.Error("wrong overflow error for int8:", err)
+	}
+	it = inputT{
+		Mini: math.MinInt8 - 1,
+	}
+	b.Reset()
+	enc.Encode(it)
+	err = dec.Decode(&o1)
+	if err == nil || err.String() != `value for "Mini" out of range` {
+		t.Error("wrong underflow error for int8:", err)
+	}
+
+	// int16
+	b.Reset()
+	it = inputT{
+		Maxi: math.MaxInt16 + 1,
+	}
+	type outi16 struct {
+		Maxi int16
+		Mini int16
+	}
+	var o2 outi16
+	enc.Encode(it)
+	err = dec.Decode(&o2)
+	if err == nil || err.String() != `value for "Maxi" out of range` {
+		t.Error("wrong overflow error for int16:", err)
+	}
+	it = inputT{
+		Mini: math.MinInt16 - 1,
+	}
+	b.Reset()
+	enc.Encode(it)
+	err = dec.Decode(&o2)
+	if err == nil || err.String() != `value for "Mini" out of range` {
+		t.Error("wrong underflow error for int16:", err)
+	}
+
+	// int32
+	b.Reset()
+	it = inputT{
+		Maxi: math.MaxInt32 + 1,
+	}
+	type outi32 struct {
+		Maxi int32
+		Mini int32
+	}
+	var o3 outi32
+	enc.Encode(it)
+	err = dec.Decode(&o3)
+	if err == nil || err.String() != `value for "Maxi" out of range` {
+		t.Error("wrong overflow error for int32:", err)
+	}
+	it = inputT{
+		Mini: math.MinInt32 - 1,
+	}
+	b.Reset()
+	enc.Encode(it)
+	err = dec.Decode(&o3)
+	if err == nil || err.String() != `value for "Mini" out of range` {
+		t.Error("wrong underflow error for int32:", err)
+	}
+
+	// uint8
+	b.Reset()
+	it = inputT{
+		Maxu: math.MaxUint8 + 1,
+	}
+	type outu8 struct {
+		Maxu uint8
+	}
+	var o4 outu8
+	enc.Encode(it)
+	err = dec.Decode(&o4)
+	if err == nil || err.String() != `value for "Maxu" out of range` {
+		t.Error("wrong overflow error for uint8:", err)
+	}
+
+	// uint16
+	b.Reset()
+	it = inputT{
+		Maxu: math.MaxUint16 + 1,
+	}
+	type outu16 struct {
+		Maxu uint16
+	}
+	var o5 outu16
+	enc.Encode(it)
+	err = dec.Decode(&o5)
+	if err == nil || err.String() != `value for "Maxu" out of range` {
+		t.Error("wrong overflow error for uint16:", err)
+	}
+
+	// uint32
+	b.Reset()
+	it = inputT{
+		Maxu: math.MaxUint32 + 1,
+	}
+	type outu32 struct {
+		Maxu uint32
+	}
+	var o6 outu32
+	enc.Encode(it)
+	err = dec.Decode(&o6)
+	if err == nil || err.String() != `value for "Maxu" out of range` {
+		t.Error("wrong overflow error for uint32:", err)
+	}
+
+	// float32
+	b.Reset()
+	it = inputT{
+		Maxf: math.MaxFloat32 * 2,
+	}
+	type outf32 struct {
+		Maxf float32
+		Minf float32
+	}
+	var o7 outf32
+	enc.Encode(it)
+	err = dec.Decode(&o7)
+	if err == nil || err.String() != `value for "Maxf" out of range` {
+		t.Error("wrong overflow error for float32:", err)
+	}
+
+	// complex64
+	b.Reset()
+	it = inputT{
+		Maxc: complex(math.MaxFloat32*2, math.MaxFloat32*2),
+	}
+	type outc64 struct {
+		Maxc complex64
+		Minc complex64
+	}
+	var o8 outc64
+	enc.Encode(it)
+	err = dec.Decode(&o8)
+	if err == nil || err.String() != `value for "Maxc" out of range` {
+		t.Error("wrong overflow error for complex64:", err)
+	}
+}
+
+func TestNesting(t *testing.T) {
+	type RT struct {
+		A    string
+		Next *RT
+	}
+	rt := new(RT)
+	rt.A = "level1"
+	rt.Next = new(RT)
+	rt.Next.A = "level2"
+	b := new(bytes.Buffer)
+	NewEncoder(b).Encode(rt)
+	var drt RT
+	dec := NewDecoder(b)
+	err := dec.Decode(&drt)
+	if err != nil {
+		t.Fatal("decoder error:", err)
+	}
+	if drt.A != rt.A {
+		t.Errorf("nesting: encode expected %v got %v", *rt, drt)
+	}
+	if drt.Next == nil {
+		t.Errorf("nesting: recursion failed")
+	}
+	if drt.Next.A != rt.Next.A {
+		t.Errorf("nesting: encode expected %v got %v", *rt.Next, *drt.Next)
+	}
+}
+
+// These three structures have the same data with different indirections
+type T0 struct {
+	A int
+	B int
+	C int
+	D int
+}
+type T1 struct {
+	A int
+	B *int
+	C **int
+	D ***int
+}
+type T2 struct {
+	A ***int
+	B **int
+	C *int
+	D int
+}
+
+func TestAutoIndirection(t *testing.T) {
+	// First transfer t1 into t0
+	var t1 T1
+	t1.A = 17
+	t1.B = new(int)
+	*t1.B = 177
+	t1.C = new(*int)
+	*t1.C = new(int)
+	**t1.C = 1777
+	t1.D = new(**int)
+	*t1.D = new(*int)
+	**t1.D = new(int)
+	***t1.D = 17777
+	b := new(bytes.Buffer)
+	enc := NewEncoder(b)
+	enc.Encode(t1)
+	dec := NewDecoder(b)
+	var t0 T0
+	dec.Decode(&t0)
+	if t0.A != 17 || t0.B != 177 || t0.C != 1777 || t0.D != 17777 {
+		t.Errorf("t1->t0: expected {17 177 1777 17777}; got %v", t0)
+	}
+
+	// Now transfer t2 into t0
+	var t2 T2
+	t2.D = 17777
+	t2.C = new(int)
+	*t2.C = 1777
+	t2.B = new(*int)
+	*t2.B = new(int)
+	**t2.B = 177
+	t2.A = new(**int)
+	*t2.A = new(*int)
+	**t2.A = new(int)
+	***t2.A = 17
+	b.Reset()
+	enc.Encode(t2)
+	t0 = T0{}
+	dec.Decode(&t0)
+	if t0.A != 17 || t0.B != 177 || t0.C != 1777 || t0.D != 17777 {
+		t.Errorf("t2->t0 expected {17 177 1777 17777}; got %v", t0)
+	}
+
+	// Now transfer t0 into t1
+	t0 = T0{17, 177, 1777, 17777}
+	b.Reset()
+	enc.Encode(t0)
+	t1 = T1{}
+	dec.Decode(&t1)
+	if t1.A != 17 || *t1.B != 177 || **t1.C != 1777 || ***t1.D != 17777 {
+		t.Errorf("t0->t1 expected {17 177 1777 17777}; got {%d %d %d %d}", t1.A, *t1.B, **t1.C, ***t1.D)
+	}
+
+	// Now transfer t0 into t2
+	b.Reset()
+	enc.Encode(t0)
+	t2 = T2{}
+	dec.Decode(&t2)
+	if ***t2.A != 17 || **t2.B != 177 || *t2.C != 1777 || t2.D != 17777 {
+		t.Errorf("t0->t2 expected {17 177 1777 17777}; got {%d %d %d %d}", ***t2.A, **t2.B, *t2.C, t2.D)
+	}
+
+	// Now do t2 again but without pre-allocated pointers.
+	b.Reset()
+	enc.Encode(t0)
+	***t2.A = 0
+	**t2.B = 0
+	*t2.C = 0
+	t2.D = 0
+	dec.Decode(&t2)
+	if ***t2.A != 17 || **t2.B != 177 || *t2.C != 1777 || t2.D != 17777 {
+		t.Errorf("t0->t2 expected {17 177 1777 17777}; got {%d %d %d %d}", ***t2.A, **t2.B, *t2.C, t2.D)
+	}
+}
+
+type RT0 struct {
+	A int
+	B string
+	C float64
+}
+type RT1 struct {
+	C      float64
+	B      string
+	A      int
+	NotSet string
+}
+
+func TestReorderedFields(t *testing.T) {
+	var rt0 RT0
+	rt0.A = 17
+	rt0.B = "hello"
+	rt0.C = 3.14159
+	b := new(bytes.Buffer)
+	NewEncoder(b).Encode(rt0)
+	dec := NewDecoder(b)
+	var rt1 RT1
+	// Wire type is RT0, local type is RT1.
+	err := dec.Decode(&rt1)
+	if err != nil {
+		t.Fatal("decode error:", err)
+	}
+	if rt0.A != rt1.A || rt0.B != rt1.B || rt0.C != rt1.C {
+		t.Errorf("rt1->rt0: expected %v; got %v", rt0, rt1)
+	}
+}
+
+// Like an RT0 but with fields we'll ignore on the decode side.
+type IT0 struct {
+	A        int64
+	B        string
+	Ignore_d []int
+	Ignore_e [3]float64
+	Ignore_f bool
+	Ignore_g string
+	Ignore_h []byte
+	Ignore_i *RT1
+	Ignore_m map[string]int
+	C        float64
+}
+
+func TestIgnoredFields(t *testing.T) {
+	var it0 IT0
+	it0.A = 17
+	it0.B = "hello"
+	it0.C = 3.14159
+	it0.Ignore_d = []int{1, 2, 3}
+	it0.Ignore_e[0] = 1.0
+	it0.Ignore_e[1] = 2.0
+	it0.Ignore_e[2] = 3.0
+	it0.Ignore_f = true
+	it0.Ignore_g = "pay no attention"
+	it0.Ignore_h = []byte("to the curtain")
+	it0.Ignore_i = &RT1{3.1, "hi", 7, "hello"}
+	it0.Ignore_m = map[string]int{"one": 1, "two": 2}
+
+	b := new(bytes.Buffer)
+	NewEncoder(b).Encode(it0)
+	dec := NewDecoder(b)
+	var rt1 RT1
+	// Wire type is IT0, local type is RT1.
+	err := dec.Decode(&rt1)
+	if err != nil {
+		t.Error("error: ", err)
+	}
+	if int(it0.A) != rt1.A || it0.B != rt1.B || it0.C != rt1.C {
+		t.Errorf("rt0->rt1: expected %v; got %v", it0, rt1)
+	}
+}
+
+func TestBadRecursiveType(t *testing.T) {
+	type Rec ***Rec
+	var rec Rec
+	b := new(bytes.Buffer)
+	err := NewEncoder(b).Encode(&rec)
+	if err == nil {
+		t.Error("expected error; got none")
+	} else if strings.Index(err.String(), "recursive") < 0 {
+		t.Error("expected recursive type error; got", err)
+	}
+	// Can't test decode easily because we can't encode one, so we can't pass one to a Decoder.
+}
+
+type Bad0 struct {
+	CH chan int
+	C  float64
+}
+
+func TestInvalidField(t *testing.T) {
+	var bad0 Bad0
+	bad0.CH = make(chan int)
+	b := new(bytes.Buffer)
+	dummyEncoder := new(Encoder) // sufficient for this purpose.
+	dummyEncoder.encode(b, reflect.ValueOf(&bad0), userType(reflect.TypeOf(&bad0)))
+	if err := dummyEncoder.err; err == nil {
+		t.Error("expected error; got none")
+	} else if strings.Index(err.String(), "type") < 0 {
+		t.Error("expected type error; got", err)
+	}
+}
+
+type Indirect struct {
+	A ***[3]int
+	S ***[]int
+	M ****map[string]int
+}
+
+type Direct struct {
+	A [3]int
+	S []int
+	M map[string]int
+}
+
+func TestIndirectSliceMapArray(t *testing.T) {
+	// Marshal indirect, unmarshal to direct.
+	i := new(Indirect)
+	i.A = new(**[3]int)
+	*i.A = new(*[3]int)
+	**i.A = new([3]int)
+	***i.A = [3]int{1, 2, 3}
+	i.S = new(**[]int)
+	*i.S = new(*[]int)
+	**i.S = new([]int)
+	***i.S = []int{4, 5, 6}
+	i.M = new(***map[string]int)
+	*i.M = new(**map[string]int)
+	**i.M = new(*map[string]int)
+	***i.M = new(map[string]int)
+	****i.M = map[string]int{"one": 1, "two": 2, "three": 3}
+	b := new(bytes.Buffer)
+	NewEncoder(b).Encode(i)
+	dec := NewDecoder(b)
+	var d Direct
+	err := dec.Decode(&d)
+	if err != nil {
+		t.Error("error: ", err)
+	}
+	if len(d.A) != 3 || d.A[0] != 1 || d.A[1] != 2 || d.A[2] != 3 {
+		t.Errorf("indirect to direct: d.A is %v not %v", d.A, ***i.A)
+	}
+	if len(d.S) != 3 || d.S[0] != 4 || d.S[1] != 5 || d.S[2] != 6 {
+		t.Errorf("indirect to direct: d.S is %v not %v", d.S, ***i.S)
+	}
+	if len(d.M) != 3 || d.M["one"] != 1 || d.M["two"] != 2 || d.M["three"] != 3 {
+		t.Errorf("indirect to direct: d.M is %v not %v", d.M, ***i.M)
+	}
+	// Marshal direct, unmarshal to indirect.
+	d.A = [3]int{11, 22, 33}
+	d.S = []int{44, 55, 66}
+	d.M = map[string]int{"four": 4, "five": 5, "six": 6}
+	i = new(Indirect)
+	b.Reset()
+	NewEncoder(b).Encode(d)
+	dec = NewDecoder(b)
+	err = dec.Decode(&i)
+	if err != nil {
+		t.Fatal("error: ", err)
+	}
+	if len(***i.A) != 3 || (***i.A)[0] != 11 || (***i.A)[1] != 22 || (***i.A)[2] != 33 {
+		t.Errorf("direct to indirect: ***i.A is %v not %v", ***i.A, d.A)
+	}
+	if len(***i.S) != 3 || (***i.S)[0] != 44 || (***i.S)[1] != 55 || (***i.S)[2] != 66 {
+		t.Errorf("direct to indirect: ***i.S is %v not %v", ***i.S, ***i.S)
+	}
+	if len(****i.M) != 3 || (****i.M)["four"] != 4 || (****i.M)["five"] != 5 || (****i.M)["six"] != 6 {
+		t.Errorf("direct to indirect: ****i.M is %v not %v", ****i.M, d.M)
+	}
+}
+
+// An interface with several implementations
+type Squarer interface {
+	Square() int
+}
+
+type Int int
+
+func (i Int) Square() int {
+	return int(i * i)
+}
+
+type Float float64
+
+func (f Float) Square() int {
+	return int(f * f)
+}
+
+type Vector []int
+
+func (v Vector) Square() int {
+	sum := 0
+	for _, x := range v {
+		sum += x * x
+	}
+	return sum
+}
+
+type Point struct {
+	X, Y int
+}
+
+func (p Point) Square() int {
+	return p.X*p.X + p.Y*p.Y
+}
+
+// A struct with interfaces in it.
+type InterfaceItem struct {
+	I             int
+	Sq1, Sq2, Sq3 Squarer
+	F             float64
+	Sq            []Squarer
+}
+
+// The same struct without interfaces
+type NoInterfaceItem struct {
+	I int
+	F float64
+}
+
+func TestInterface(t *testing.T) {
+	iVal := Int(3)
+	fVal := Float(5)
+	// Sending a Vector will require that the receiver define a type in the middle of
+	// receiving the value for item2.
+	vVal := Vector{1, 2, 3}
+	b := new(bytes.Buffer)
+	item1 := &InterfaceItem{1, iVal, fVal, vVal, 11.5, []Squarer{iVal, fVal, nil, vVal}}
+	// Register the types.
+	Register(Int(0))
+	Register(Float(0))
+	Register(Vector{})
+	err := NewEncoder(b).Encode(item1)
+	if err != nil {
+		t.Error("expected no encode error; got", err)
+	}
+
+	item2 := InterfaceItem{}
+	err = NewDecoder(b).Decode(&item2)
+	if err != nil {
+		t.Fatal("decode:", err)
+	}
+	if item2.I != item1.I {
+		t.Error("normal int did not decode correctly")
+	}
+	if item2.Sq1 == nil || item2.Sq1.Square() != iVal.Square() {
+		t.Error("Int did not decode correctly")
+	}
+	if item2.Sq2 == nil || item2.Sq2.Square() != fVal.Square() {
+		t.Error("Float did not decode correctly")
+	}
+	if item2.Sq3 == nil || item2.Sq3.Square() != vVal.Square() {
+		t.Error("Vector did not decode correctly")
+	}
+	if item2.F != item1.F {
+		t.Error("normal float did not decode correctly")
+	}
+	// Now check that we received a slice of Squarers correctly, including a nil element
+	if len(item1.Sq) != len(item2.Sq) {
+		t.Fatalf("[]Squarer length wrong: got %d; expected %d", len(item2.Sq), len(item1.Sq))
+	}
+	for i, v1 := range item1.Sq {
+		v2 := item2.Sq[i]
+		if v1 == nil || v2 == nil {
+			if v1 != nil || v2 != nil {
+				t.Errorf("item %d inconsistent nils", i)
+			}
+			continue
+			if v1.Square() != v2.Square() {
+				t.Errorf("item %d inconsistent values: %v %v", i, v1, v2)
+			}
+		}
+	}
+}
+
+// A struct with all basic types, stored in interfaces.
+type BasicInterfaceItem struct {
+	Int, Int8, Int16, Int32, Int64      interface{}
+	Uint, Uint8, Uint16, Uint32, Uint64 interface{}
+	Float32, Float64                    interface{}
+	Complex64, Complex128               interface{}
+	Bool                                interface{}
+	String                              interface{}
+	Bytes                               interface{}
+}
+
+func TestInterfaceBasic(t *testing.T) {
+	b := new(bytes.Buffer)
+	item1 := &BasicInterfaceItem{
+		int(1), int8(1), int16(1), int32(1), int64(1),
+		uint(1), uint8(1), uint16(1), uint32(1), uint64(1),
+		float32(1), 1.0,
+		complex64(1i), complex128(1i),
+		true,
+		"hello",
+		[]byte("sailor"),
+	}
+	err := NewEncoder(b).Encode(item1)
+	if err != nil {
+		t.Error("expected no encode error; got", err)
+	}
+
+	item2 := &BasicInterfaceItem{}
+	err = NewDecoder(b).Decode(&item2)
+	if err != nil {
+		t.Fatal("decode:", err)
+	}
+	if !reflect.DeepEqual(item1, item2) {
+		t.Errorf("encode expected %v got %v", item1, item2)
+	}
+	// Hand check a couple for correct types.
+	if v, ok := item2.Bool.(bool); !ok || !v {
+		t.Error("boolean should be true")
+	}
+	if v, ok := item2.String.(string); !ok || v != item1.String.(string) {
+		t.Errorf("string should be %v is %v", item1.String, v)
+	}
+}
+
+type String string
+
+type PtrInterfaceItem struct {
+	Str1 interface{} // basic
+	Str2 interface{} // derived
+}
+
+// We'll send pointers; should receive values.
+// Also check that we can register T but send *T.
+func TestInterfacePointer(t *testing.T) {
+	b := new(bytes.Buffer)
+	str1 := "howdy"
+	str2 := String("kiddo")
+	item1 := &PtrInterfaceItem{
+		&str1,
+		&str2,
+	}
+	// Register the type.
+	Register(str2)
+	err := NewEncoder(b).Encode(item1)
+	if err != nil {
+		t.Error("expected no encode error; got", err)
+	}
+
+	item2 := &PtrInterfaceItem{}
+	err = NewDecoder(b).Decode(&item2)
+	if err != nil {
+		t.Fatal("decode:", err)
+	}
+	// Hand test for correct types and values.
+	if v, ok := item2.Str1.(string); !ok || v != str1 {
+		t.Errorf("basic string failed: %q should be %q", v, str1)
+	}
+	if v, ok := item2.Str2.(String); !ok || v != str2 {
+		t.Errorf("derived type String failed: %q should be %q", v, str2)
+	}
+}
+
+func TestIgnoreInterface(t *testing.T) {
+	iVal := Int(3)
+	fVal := Float(5)
+	// Sending a Point will require that the receiver define a type in the middle of
+	// receiving the value for item2.
+	pVal := Point{2, 3}
+	b := new(bytes.Buffer)
+	item1 := &InterfaceItem{1, iVal, fVal, pVal, 11.5, nil}
+	// Register the types.
+	Register(Int(0))
+	Register(Float(0))
+	Register(Point{})
+	err := NewEncoder(b).Encode(item1)
+	if err != nil {
+		t.Error("expected no encode error; got", err)
+	}
+
+	item2 := NoInterfaceItem{}
+	err = NewDecoder(b).Decode(&item2)
+	if err != nil {
+		t.Fatal("decode:", err)
+	}
+	if item2.I != item1.I {
+		t.Error("normal int did not decode correctly")
+	}
+	if item2.F != item2.F {
+		t.Error("normal float did not decode correctly")
+	}
+}
+
+type U struct {
+	A int
+	B string
+	c float64
+	D uint
+}
+
+func TestUnexportedFields(t *testing.T) {
+	var u0 U
+	u0.A = 17
+	u0.B = "hello"
+	u0.c = 3.14159
+	u0.D = 23
+	b := new(bytes.Buffer)
+	NewEncoder(b).Encode(u0)
+	dec := NewDecoder(b)
+	var u1 U
+	u1.c = 1234.
+	err := dec.Decode(&u1)
+	if err != nil {
+		t.Fatal("decode error:", err)
+	}
+	if u0.A != u0.A || u0.B != u1.B || u0.D != u1.D {
+		t.Errorf("u1->u0: expected %v; got %v", u0, u1)
+	}
+	if u1.c != 1234. {
+		t.Error("u1.c modified")
+	}
+}
+
+var singletons = []interface{}{
+	true,
+	7,
+	3.2,
+	"hello",
+	[3]int{11, 22, 33},
+	[]float32{0.5, 0.25, 0.125},
+	map[string]int{"one": 1, "two": 2},
+}
+
+func TestDebugSingleton(t *testing.T) {
+	if debugFunc == nil {
+		return
+	}
+	b := new(bytes.Buffer)
+	// Accumulate a number of values and print them out all at once.
+	for _, x := range singletons {
+		err := NewEncoder(b).Encode(x)
+		if err != nil {
+			t.Fatal("encode:", err)
+		}
+	}
+	debugFunc(b)
+}
+
+// A type that won't be defined in the gob until we send it in an interface value.
+type OnTheFly struct {
+	A int
+}
+
+type DT struct {
+	//	X OnTheFly
+	A     int
+	B     string
+	C     float64
+	I     interface{}
+	J     interface{}
+	I_nil interface{}
+	M     map[string]int
+	T     [3]int
+	S     []string
+}
+
+func TestDebugStruct(t *testing.T) {
+	if debugFunc == nil {
+		return
+	}
+	Register(OnTheFly{})
+	var dt DT
+	dt.A = 17
+	dt.B = "hello"
+	dt.C = 3.14159
+	dt.I = 271828
+	dt.J = OnTheFly{3}
+	dt.I_nil = nil
+	dt.M = map[string]int{"one": 1, "two": 2}
+	dt.T = [3]int{11, 22, 33}
+	dt.S = []string{"hi", "joe"}
+	b := new(bytes.Buffer)
+	err := NewEncoder(b).Encode(dt)
+	if err != nil {
+		t.Fatal("encode:", err)
+	}
+	debugBuffer := bytes.NewBuffer(b.Bytes())
+	dt2 := &DT{}
+	err = NewDecoder(b).Decode(&dt2)
+	if err != nil {
+		t.Error("decode:", err)
+	}
+	debugFunc(debugBuffer)
+}
diff --git a/src/pkg/gob/debug.go b/src/pkg/gob/debug.go
new file mode 100644
index 000000000..ce8a6ff5e
--- /dev/null
+++ b/src/pkg/gob/debug.go
@@ -0,0 +1,686 @@
+package gob
+
+// This file is not normally included in the gob package.  Used only for debugging the package itself.
+// Add debug.go to the files listed in the Makefile to add Debug to the gob package.
+// Except for reading uints, it is an implementation of a reader that is independent of
+// the one implemented by Decoder.
+
+import (
+	"bytes"
+	"fmt"
+	"io"
+	"os"
+	"strings"
+	"sync"
+)
+
+var dumpBytes = false // If true, print the remaining bytes in the input buffer at each item.
+
+// Init installs the debugging facility. If this file is not compiled in the
+// package, the tests in codec_test.go are no-ops.
+func init() {
+	debugFunc = Debug
+}
+
+var (
+	blanks = bytes.Repeat([]byte{' '}, 3*10)
+	empty  = []byte(": \n")
+	tabs   = strings.Repeat("\t", 100)
+)
+
+// tab indents itself when printed.
+type tab int
+
+func (t tab) String() string {
+	n := int(t)
+	if n > len(tabs) {
+		n = len(tabs)
+	}
+	return tabs[0:n]
+}
+
+func (t tab) print() {
+	fmt.Fprint(os.Stderr, t)
+}
+
+// A peekReader wraps an io.Reader, allowing one to peek ahead to see
+// what's coming without stealing the data from the client of the Reader.
+type peekReader struct {
+	r    io.Reader
+	data []byte // read-ahead data
+}
+
+// newPeekReader returns a peekReader that wraps r.
+func newPeekReader(r io.Reader) *peekReader {
+	return &peekReader{r: r}
+}
+
+// Read is the usual method. It will first take data that has been read ahead.
+func (p *peekReader) Read(b []byte) (n int, err os.Error) {
+	if len(p.data) == 0 {
+		return p.r.Read(b)
+	}
+	// Satisfy what's possible from the read-ahead data.
+	n = copy(b, p.data)
+	// Move data down to beginning of slice, to avoid endless growth
+	copy(p.data, p.data[n:])
+	p.data = p.data[:len(p.data)-n]
+	return
+}
+
+// peek returns as many bytes as possible from the unread
+// portion of the stream, up to the length of b.
+func (p *peekReader) peek(b []byte) (n int, err os.Error) {
+	if len(p.data) > 0 {
+		n = copy(b, p.data)
+		if n == len(b) {
+			return
+		}
+		b = b[n:]
+	}
+	if len(b) == 0 {
+		return
+	}
+	m, e := io.ReadFull(p.r, b)
+	if m > 0 {
+		p.data = append(p.data, b[:m]...)
+	}
+	n += m
+	if e == io.ErrUnexpectedEOF {
+		// That means m > 0 but we reached EOF. If we got data
+		// we won't complain about not being able to peek enough.
+		if n > 0 {
+			e = nil
+		} else {
+			e = os.EOF
+		}
+	}
+	return n, e
+}
+
+type debugger struct {
+	mutex          sync.Mutex
+	remain         int  // the number of bytes known to remain in the input
+	remainingKnown bool // the value of 'remain' is valid
+	r              *peekReader
+	wireType       map[typeId]*wireType
+	tmp            []byte // scratch space for decoding uints.
+}
+
+// dump prints the next nBytes of the input.
+// It arranges to print the output aligned from call to
+// call, to make it easy to see what has been consumed.
+func (deb *debugger) dump(format string, args ...interface{}) {
+	if !dumpBytes {
+		return
+	}
+	fmt.Fprintf(os.Stderr, format+" ", args...)
+	if !deb.remainingKnown {
+		return
+	}
+	if deb.remain < 0 {
+		fmt.Fprintf(os.Stderr, "remaining byte count is negative! %d\n", deb.remain)
+		return
+	}
+	data := make([]byte, deb.remain)
+	n, _ := deb.r.peek(data)
+	if n == 0 {
+		os.Stderr.Write(empty)
+		return
+	}
+	b := new(bytes.Buffer)
+	fmt.Fprintf(b, "[%d]{\n", deb.remain)
+	// Blanks until first byte
+	lineLength := 0
+	if n := len(data); n%10 != 0 {
+		lineLength = 10 - n%10
+		fmt.Fprintf(b, "\t%s", blanks[:lineLength*3])
+	}
+	// 10 bytes per line
+	for len(data) > 0 {
+		if lineLength == 0 {
+			fmt.Fprint(b, "\t")
+		}
+		m := 10 - lineLength
+		lineLength = 0
+		if m > len(data) {
+			m = len(data)
+		}
+		fmt.Fprintf(b, "% x\n", data[:m])
+		data = data[m:]
+	}
+	fmt.Fprint(b, "}\n")
+	os.Stderr.Write(b.Bytes())
+}
+
+// Debug prints a human-readable representation of the gob data read from r.
+func Debug(r io.Reader) {
+	err := debug(r)
+	if err != nil {
+		fmt.Fprintf(os.Stderr, "gob debug: %s\n", err)
+	}
+}
+
+// debug implements Debug, but catches panics and returns
+// them as errors to be printed by Debug.
+func debug(r io.Reader) (err os.Error) {
+	defer catchError(&err)
+	fmt.Fprintln(os.Stderr, "Start of debugging")
+	deb := &debugger{
+		r:        newPeekReader(r),
+		wireType: make(map[typeId]*wireType),
+		tmp:      make([]byte, 16),
+	}
+	if b, ok := r.(*bytes.Buffer); ok {
+		deb.remain = b.Len()
+		deb.remainingKnown = true
+	}
+	deb.gobStream()
+	return
+}
+
+// note that we've consumed some bytes
+func (deb *debugger) consumed(n int) {
+	if deb.remainingKnown {
+		deb.remain -= n
+	}
+}
+
+// int64 decodes and returns the next integer, which must be present.
+// Don't call this if you could be at EOF.
+func (deb *debugger) int64() int64 {
+	return toInt(deb.uint64())
+}
+
+// uint64 returns and decodes the next unsigned integer, which must be present.
+// Don't call this if you could be at EOF.
+// TODO: handle errors better.
+func (deb *debugger) uint64() uint64 {
+	n, w, err := decodeUintReader(deb.r, deb.tmp)
+	if err != nil {
+		errorf("debug: read error: %s", err)
+	}
+	deb.consumed(w)
+	return n
+}
+
+// GobStream:
+//	DelimitedMessage* (until EOF)
+func (deb *debugger) gobStream() {
+	// Make sure we're single-threaded through here.
+	deb.mutex.Lock()
+	defer deb.mutex.Unlock()
+
+	for deb.delimitedMessage(0) {
+	}
+}
+
+// DelimitedMessage:
+//	uint(lengthOfMessage) Message
+func (deb *debugger) delimitedMessage(indent tab) bool {
+	for {
+		n := deb.loadBlock(true)
+		if n < 0 {
+			return false
+		}
+		deb.dump("Delimited message of length %d", n)
+		deb.message(indent)
+	}
+	return true
+}
+
+// loadBlock preps us to read a message
+// of the length specified next in the input. It returns
+// the length of the block. The argument tells whether
+// an EOF is acceptable now.  If it is and one is found,
+// the return value is negative.
+func (deb *debugger) loadBlock(eofOK bool) int {
+	n64, w, err := decodeUintReader(deb.r, deb.tmp) // deb.uint64 will error at EOF
+	if err != nil {
+		if eofOK && err == os.EOF {
+			return -1
+		}
+		errorf("debug: unexpected error: %s", err)
+	}
+	deb.consumed(w)
+	n := int(n64)
+	if n < 0 {
+		errorf("huge value for message length: %d", n64)
+	}
+	return int(n)
+}
+
+// Message:
+//	TypeSequence TypedValue
+// TypeSequence
+//	(TypeDefinition DelimitedTypeDefinition*)?
+// DelimitedTypeDefinition:
+//	uint(lengthOfTypeDefinition) TypeDefinition
+// TypedValue:
+//	int(typeId) Value
+func (deb *debugger) message(indent tab) bool {
+	for {
+		// Convert the uint64 to a signed integer typeId
+		uid := deb.int64()
+		id := typeId(uid)
+		deb.dump("type id=%d", id)
+		if id < 0 {
+			deb.typeDefinition(indent, -id)
+			n := deb.loadBlock(false)
+			deb.dump("Message of length %d", n)
+			continue
+		} else {
+			deb.value(indent, id)
+			break
+		}
+	}
+	return true
+}
+
+// Helper methods to make it easy to scan a type descriptor.
+
+// common returns the CommonType at the input point.
+func (deb *debugger) common() CommonType {
+	fieldNum := -1
+	name := ""
+	id := typeId(0)
+	for {
+		delta := deb.delta(-1)
+		if delta == 0 {
+			break
+		}
+		fieldNum += delta
+		switch fieldNum {
+		case 0:
+			name = deb.string()
+		case 1:
+			// Id typeId
+			id = deb.typeId()
+		default:
+			errorf("corrupted CommonType")
+		}
+	}
+	return CommonType{name, id}
+}
+
+// uint returns the unsigned int at the input point, as a uint (not uint64).
+func (deb *debugger) uint() uint {
+	return uint(deb.uint64())
+}
+
+// int returns the signed int at the input point, as an int (not int64).
+func (deb *debugger) int() int {
+	return int(deb.int64())
+}
+
+// typeId returns the type id at the input point.
+func (deb *debugger) typeId() typeId {
+	return typeId(deb.int64())
+}
+
+// string returns the string at the input point.
+func (deb *debugger) string() string {
+	x := int(deb.uint64())
+	b := make([]byte, x)
+	nb, _ := deb.r.Read(b)
+	if nb != x {
+		errorf("corrupted type")
+	}
+	deb.consumed(nb)
+	return string(b)
+}
+
+// delta returns the field delta at the input point.  The expect argument,
+// if non-negative, identifies what the value should be.
+func (deb *debugger) delta(expect int) int {
+	delta := int(deb.uint64())
+	if delta < 0 || (expect >= 0 && delta != expect) {
+		errorf("decode: corrupted type: delta %d expected %d", delta, expect)
+	}
+	return delta
+}
+
+// TypeDefinition:
+//	[int(-typeId) (already read)] encodingOfWireType
+func (deb *debugger) typeDefinition(indent tab, id typeId) {
+	deb.dump("type definition for id %d", id)
+	// Encoding is of a wireType. Decode the structure as usual
+	fieldNum := -1
+	wire := new(wireType)
+	// A wireType defines a single field.
+	delta := deb.delta(-1)
+	fieldNum += delta
+	switch fieldNum {
+	case 0: // array type, one field of {{Common}, elem, length}
+		// Field number 0 is CommonType
+		deb.delta(1)
+		com := deb.common()
+		// Field number 1 is type Id of elem
+		deb.delta(1)
+		id := deb.typeId()
+		// Field number 3 is length
+		deb.delta(1)
+		length := deb.int()
+		wire.ArrayT = &arrayType{com, id, length}
+
+	case 1: // slice type, one field of {{Common}, elem}
+		// Field number 0 is CommonType
+		deb.delta(1)
+		com := deb.common()
+		// Field number 1 is type Id of elem
+		deb.delta(1)
+		id := deb.typeId()
+		wire.SliceT = &sliceType{com, id}
+
+	case 2: // struct type, one field of {{Common}, []fieldType}
+		// Field number 0 is CommonType
+		deb.delta(1)
+		com := deb.common()
+		// Field number 1 is slice of FieldType
+		deb.delta(1)
+		numField := int(deb.uint())
+		field := make([]*fieldType, numField)
+		for i := 0; i < numField; i++ {
+			field[i] = new(fieldType)
+			deb.delta(1) // field 0 of fieldType: name
+			field[i].Name = deb.string()
+			deb.delta(1) // field 1 of fieldType: id
+			field[i].Id = deb.typeId()
+			deb.delta(0) // end of fieldType
+		}
+		wire.StructT = &structType{com, field}
+
+	case 3: // map type, one field of {{Common}, key, elem}
+		// Field number 0 is CommonType
+		deb.delta(1)
+		com := deb.common()
+		// Field number 1 is type Id of key
+		deb.delta(1)
+		keyId := deb.typeId()
+		// Field number 2 is type Id of elem
+		deb.delta(1)
+		elemId := deb.typeId()
+		wire.MapT = &mapType{com, keyId, elemId}
+	case 4: // GobEncoder type, one field of {{Common}}
+		// Field number 0 is CommonType
+		deb.delta(1)
+		com := deb.common()
+		wire.GobEncoderT = &gobEncoderType{com}
+	default:
+		errorf("bad field in type %d", fieldNum)
+	}
+	deb.printWireType(indent, wire)
+	deb.delta(0) // end inner type (arrayType, etc.)
+	deb.delta(0) // end wireType
+	// Remember we've seen this type.
+	deb.wireType[id] = wire
+}
+
+// Value:
+//	SingletonValue | StructValue
+func (deb *debugger) value(indent tab, id typeId) {
+	wire, ok := deb.wireType[id]
+	if ok && wire.StructT != nil {
+		deb.structValue(indent, id)
+	} else {
+		deb.singletonValue(indent, id)
+	}
+}
+
+// SingletonValue:
+//	uint(0) FieldValue
+func (deb *debugger) singletonValue(indent tab, id typeId) {
+	deb.dump("Singleton value")
+	// is it a builtin type?
+	wire := deb.wireType[id]
+	_, ok := builtinIdToType[id]
+	if !ok && wire == nil {
+		errorf("type id %d not defined", id)
+	}
+	m := deb.uint64()
+	if m != 0 {
+		errorf("expected zero; got %d", m)
+	}
+	deb.fieldValue(indent, id)
+}
+
+// InterfaceValue:
+//	NilInterfaceValue | NonNilInterfaceValue
+func (deb *debugger) interfaceValue(indent tab) {
+	deb.dump("Start of interface value")
+	if nameLen := deb.uint64(); nameLen == 0 {
+		deb.nilInterfaceValue(indent)
+	} else {
+		deb.nonNilInterfaceValue(indent, int(nameLen))
+	}
+}
+
+// NilInterfaceValue:
+//	uint(0) [already read]
+func (deb *debugger) nilInterfaceValue(indent tab) int {
+	fmt.Fprintf(os.Stderr, "%snil interface\n", indent)
+	return 0
+}
+
+// NonNilInterfaceValue:
+//	ConcreteTypeName TypeSequence InterfaceContents
+// ConcreteTypeName:
+//	uint(lengthOfName) [already read=n] name
+// InterfaceContents:
+//	int(concreteTypeId) DelimitedValue
+// DelimitedValue:
+//	uint(length) Value
+func (deb *debugger) nonNilInterfaceValue(indent tab, nameLen int) {
+	// ConcreteTypeName
+	b := make([]byte, nameLen)
+	deb.r.Read(b) // TODO: CHECK THESE READS!!
+	deb.consumed(nameLen)
+	name := string(b)
+
+	for {
+		id := deb.typeId()
+		if id < 0 {
+			deb.typeDefinition(indent, -id)
+			n := deb.loadBlock(false)
+			deb.dump("Nested message of length %d", n)
+		} else {
+			// DelimitedValue
+			x := deb.uint64() // in case we want to ignore the value; we don't.
+			fmt.Fprintf(os.Stderr, "%sinterface value, type %q id=%d; valueLength %d\n", indent, name, id, x)
+			deb.value(indent, id)
+			break
+		}
+	}
+}
+
+// printCommonType prints a common type; used by printWireType.
+func (deb *debugger) printCommonType(indent tab, kind string, common *CommonType) {
+	indent.print()
+	fmt.Fprintf(os.Stderr, "%s %q id=%d\n", kind, common.Name, common.Id)
+}
+
+// printWireType prints the contents of a wireType.
+func (deb *debugger) printWireType(indent tab, wire *wireType) {
+	fmt.Fprintf(os.Stderr, "%stype definition {\n", indent)
+	indent++
+	switch {
+	case wire.ArrayT != nil:
+		deb.printCommonType(indent, "array", &wire.ArrayT.CommonType)
+		fmt.Fprintf(os.Stderr, "%slen %d\n", indent+1, wire.ArrayT.Len)
+		fmt.Fprintf(os.Stderr, "%selemid %d\n", indent+1, wire.ArrayT.Elem)
+	case wire.MapT != nil:
+		deb.printCommonType(indent, "map", &wire.MapT.CommonType)
+		fmt.Fprintf(os.Stderr, "%skey id=%d\n", indent+1, wire.MapT.Key)
+		fmt.Fprintf(os.Stderr, "%selem id=%d\n", indent+1, wire.MapT.Elem)
+	case wire.SliceT != nil:
+		deb.printCommonType(indent, "slice", &wire.SliceT.CommonType)
+		fmt.Fprintf(os.Stderr, "%selem id=%d\n", indent+1, wire.SliceT.Elem)
+	case wire.StructT != nil:
+		deb.printCommonType(indent, "struct", &wire.StructT.CommonType)
+		for i, field := range wire.StructT.Field {
+			fmt.Fprintf(os.Stderr, "%sfield %d:\t%s\tid=%d\n", indent+1, i, field.Name, field.Id)
+		}
+	case wire.GobEncoderT != nil:
+		deb.printCommonType(indent, "GobEncoder", &wire.GobEncoderT.CommonType)
+	}
+	indent--
+	fmt.Fprintf(os.Stderr, "%s}\n", indent)
+}
+
+// fieldValue prints a value of any type, such as a struct field.
+// FieldValue:
+//	builtinValue | ArrayValue | MapValue | SliceValue | StructValue | InterfaceValue
+func (deb *debugger) fieldValue(indent tab, id typeId) {
+	_, ok := builtinIdToType[id]
+	if ok {
+		if id == tInterface {
+			deb.interfaceValue(indent)
+		} else {
+			deb.printBuiltin(indent, id)
+		}
+		return
+	}
+	wire, ok := deb.wireType[id]
+	if !ok {
+		errorf("type id %d not defined", id)
+	}
+	switch {
+	case wire.ArrayT != nil:
+		deb.arrayValue(indent, wire)
+	case wire.MapT != nil:
+		deb.mapValue(indent, wire)
+	case wire.SliceT != nil:
+		deb.sliceValue(indent, wire)
+	case wire.StructT != nil:
+		deb.structValue(indent, id)
+	case wire.GobEncoderT != nil:
+		deb.gobEncoderValue(indent, id)
+	default:
+		panic("bad wire type for field")
+	}
+}
+
+// printBuiltin prints a value not of a fundamental type, that is,
+// one whose type is known to gobs at bootstrap time.
+func (deb *debugger) printBuiltin(indent tab, id typeId) {
+	switch id {
+	case tBool:
+		x := deb.int64()
+		if x == 0 {
+			fmt.Fprintf(os.Stderr, "%sfalse\n", indent)
+		} else {
+			fmt.Fprintf(os.Stderr, "%strue\n", indent)
+		}
+	case tInt:
+		x := deb.int64()
+		fmt.Fprintf(os.Stderr, "%s%d\n", indent, x)
+	case tUint:
+		x := deb.int64()
+		fmt.Fprintf(os.Stderr, "%s%d\n", indent, x)
+	case tFloat:
+		x := deb.uint64()
+		fmt.Fprintf(os.Stderr, "%s%g\n", indent, floatFromBits(x))
+	case tComplex:
+		r := deb.uint64()
+		i := deb.uint64()
+		fmt.Fprintf(os.Stderr, "%s%g+%gi\n", indent, floatFromBits(r), floatFromBits(i))
+	case tBytes:
+		x := int(deb.uint64())
+		b := make([]byte, x)
+		deb.r.Read(b)
+		deb.consumed(x)
+		fmt.Fprintf(os.Stderr, "%s{% x}=%q\n", indent, b, b)
+	case tString:
+		x := int(deb.uint64())
+		b := make([]byte, x)
+		deb.r.Read(b)
+		deb.consumed(x)
+		fmt.Fprintf(os.Stderr, "%s%q\n", indent, b)
+	default:
+		panic("unknown builtin")
+	}
+}
+
+// ArrayValue:
+//	uint(n) FieldValue*n
+func (deb *debugger) arrayValue(indent tab, wire *wireType) {
+	elemId := wire.ArrayT.Elem
+	u := deb.uint64()
+	length := int(u)
+	for i := 0; i < length; i++ {
+		deb.fieldValue(indent, elemId)
+	}
+	if length != wire.ArrayT.Len {
+		fmt.Fprintf(os.Stderr, "%s(wrong length for array: %d should be %d)\n", indent, length, wire.ArrayT.Len)
+	}
+}
+
+// MapValue:
+//	uint(n) (FieldValue FieldValue)*n  [n (key, value) pairs]
+func (deb *debugger) mapValue(indent tab, wire *wireType) {
+	keyId := wire.MapT.Key
+	elemId := wire.MapT.Elem
+	u := deb.uint64()
+	length := int(u)
+	for i := 0; i < length; i++ {
+		deb.fieldValue(indent+1, keyId)
+		deb.fieldValue(indent+1, elemId)
+	}
+}
+
+// SliceValue:
+//	uint(n) (n FieldValue)
+func (deb *debugger) sliceValue(indent tab, wire *wireType) {
+	elemId := wire.SliceT.Elem
+	u := deb.uint64()
+	length := int(u)
+	deb.dump("Start of slice of length %d", length)
+
+	for i := 0; i < length; i++ {
+		deb.fieldValue(indent, elemId)
+	}
+}
+
+// StructValue:
+//	(uint(fieldDelta) FieldValue)*
+func (deb *debugger) structValue(indent tab, id typeId) {
+	deb.dump("Start of struct value of %q id=%d\n<<\n", id.name(), id)
+	fmt.Fprintf(os.Stderr, "%s%s struct {\n", indent, id.name())
+	wire, ok := deb.wireType[id]
+	if !ok {
+		errorf("type id %d not defined", id)
+	}
+	strct := wire.StructT
+	fieldNum := -1
+	indent++
+	for {
+		delta := deb.uint64()
+		if delta == 0 { // struct terminator is zero delta fieldnum
+			break
+		}
+		fieldNum += int(delta)
+		if fieldNum < 0 || fieldNum >= len(strct.Field) {
+			deb.dump("field number out of range: prevField=%d delta=%d", fieldNum-int(delta), delta)
+			break
+		}
+		fmt.Fprintf(os.Stderr, "%sfield %d:\t%s\n", indent, fieldNum, wire.StructT.Field[fieldNum].Name)
+		deb.fieldValue(indent+1, strct.Field[fieldNum].Id)
+	}
+	indent--
+	fmt.Fprintf(os.Stderr, "%s} // end %s struct\n", indent, id.name())
+	deb.dump(">> End of struct value of type %d %q", id, id.name())
+}
+
+// GobEncoderValue:
+//	uint(n) byte*n
+func (deb *debugger) gobEncoderValue(indent tab, id typeId) {
+	len := deb.uint64()
+	deb.dump("GobEncoder value of %q id=%d, length %d\n", id.name(), id, len)
+	fmt.Fprintf(os.Stderr, "%s%s (implements GobEncoder)\n", indent, id.name())
+	data := make([]byte, len)
+	_, err := deb.r.Read(data)
+	if err != nil {
+		errorf("gobEncoder data read: %s", err)
+	}
+	fmt.Fprintf(os.Stderr, "%s[% .2x]\n", indent+1, data)
+}
diff --git a/src/pkg/gob/decode.go b/src/pkg/gob/decode.go
new file mode 100644
index 000000000..bf7cb95f2
--- /dev/null
+++ b/src/pkg/gob/decode.go
@@ -0,0 +1,1275 @@
+// Copyright 2009 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package gob
+
+// TODO(rsc): When garbage collector changes, revisit
+// the allocations in this file that use unsafe.Pointer.
+
+import (
+	"bytes"
+	"io"
+	"math"
+	"os"
+	"reflect"
+	"unsafe"
+)
+
+var (
+	errBadUint = os.NewError("gob: encoded unsigned integer out of range")
+	errBadType = os.NewError("gob: unknown type id or corrupted data")
+	errRange   = os.NewError("gob: bad data: field numbers out of bounds")
+)
+
+// decoderState is the execution state of an instance of the decoder. A new state
+// is created for nested objects.
+type decoderState struct {
+	dec *Decoder
+	// The buffer is stored with an extra indirection because it may be replaced
+	// if we load a type during decode (when reading an interface value).
+	b        *bytes.Buffer
+	fieldnum int // the last field number read.
+	buf      []byte
+	next     *decoderState // for free list
+}
+
+// We pass the bytes.Buffer separately for easier testing of the infrastructure
+// without requiring a full Decoder.
+func (dec *Decoder) newDecoderState(buf *bytes.Buffer) *decoderState {
+	d := dec.freeList
+	if d == nil {
+		d = new(decoderState)
+		d.dec = dec
+		d.buf = make([]byte, uint64Size)
+	} else {
+		dec.freeList = d.next
+	}
+	d.b = buf
+	return d
+}
+
+func (dec *Decoder) freeDecoderState(d *decoderState) {
+	d.next = dec.freeList
+	dec.freeList = d
+}
+
+func overflow(name string) os.Error {
+	return os.NewError(`value for "` + name + `" out of range`)
+}
+
+// decodeUintReader reads an encoded unsigned integer from an io.Reader.
+// Used only by the Decoder to read the message length.
+func decodeUintReader(r io.Reader, buf []byte) (x uint64, width int, err os.Error) {
+	width = 1
+	_, err = r.Read(buf[0:width])
+	if err != nil {
+		return
+	}
+	b := buf[0]
+	if b <= 0x7f {
+		return uint64(b), width, nil
+	}
+	nb := -int(int8(b))
+	if nb > uint64Size {
+		err = errBadUint
+		return
+	}
+	var n int
+	n, err = io.ReadFull(r, buf[0:nb])
+	if err != nil {
+		if err == os.EOF {
+			err = io.ErrUnexpectedEOF
+		}
+		return
+	}
+	// Could check that the high byte is zero but it's not worth it.
+	for i := 0; i < n; i++ {
+		x <<= 8
+		x |= uint64(buf[i])
+		width++
+	}
+	return
+}
+
+// decodeUint reads an encoded unsigned integer from state.r.
+// Does not check for overflow.
+func (state *decoderState) decodeUint() (x uint64) {
+	b, err := state.b.ReadByte()
+	if err != nil {
+		error(err)
+	}
+	if b <= 0x7f {
+		return uint64(b)
+	}
+	nb := -int(int8(b))
+	if nb > uint64Size {
+		error(errBadUint)
+	}
+	n, err := state.b.Read(state.buf[0:nb])
+	if err != nil {
+		error(err)
+	}
+	// Don't need to check error; it's safe to loop regardless.
+	// Could check that the high byte is zero but it's not worth it.
+	for i := 0; i < n; i++ {
+		x <<= 8
+		x |= uint64(state.buf[i])
+	}
+	return x
+}
+
+// decodeInt reads an encoded signed integer from state.r.
+// Does not check for overflow.
+func (state *decoderState) decodeInt() int64 {
+	x := state.decodeUint()
+	if x&1 != 0 {
+		return ^int64(x >> 1)
+	}
+	return int64(x >> 1)
+}
+
+// decOp is the signature of a decoding operator for a given type.
+type decOp func(i *decInstr, state *decoderState, p unsafe.Pointer)
+
+// The 'instructions' of the decoding machine
+type decInstr struct {
+	op     decOp
+	field  int      // field number of the wire type
+	indir  int      // how many pointer indirections to reach the value in the struct
+	offset uintptr  // offset in the structure of the field to encode
+	ovfl   os.Error // error message for overflow/underflow (for arrays, of the elements)
+}
+
+// Since the encoder writes no zeros, if we arrive at a decoder we have
+// a value to extract and store.  The field number has already been read
+// (it's how we knew to call this decoder).
+// Each decoder is responsible for handling any indirections associated
+// with the data structure.  If any pointer so reached is nil, allocation must
+// be done.
+
+// Walk the pointer hierarchy, allocating if we find a nil.  Stop one before the end.
+func decIndirect(p unsafe.Pointer, indir int) unsafe.Pointer {
+	for ; indir > 1; indir-- {
+		if *(*unsafe.Pointer)(p) == nil {
+			// Allocation required
+			*(*unsafe.Pointer)(p) = unsafe.Pointer(new(unsafe.Pointer))
+		}
+		p = *(*unsafe.Pointer)(p)
+	}
+	return p
+}
+
+// ignoreUint discards a uint value with no destination.
+func ignoreUint(i *decInstr, state *decoderState, p unsafe.Pointer) {
+	state.decodeUint()
+}
+
+// ignoreTwoUints discards a uint value with no destination. It's used to skip
+// complex values.
+func ignoreTwoUints(i *decInstr, state *decoderState, p unsafe.Pointer) {
+	state.decodeUint()
+	state.decodeUint()
+}
+
+// decBool decodes a uint and stores it as a boolean through p.
+func decBool(i *decInstr, state *decoderState, p unsafe.Pointer) {
+	if i.indir > 0 {
+		if *(*unsafe.Pointer)(p) == nil {
+			*(*unsafe.Pointer)(p) = unsafe.Pointer(new(bool))
+		}
+		p = *(*unsafe.Pointer)(p)
+	}
+	*(*bool)(p) = state.decodeUint() != 0
+}
+
+// decInt8 decodes an integer and stores it as an int8 through p.
+func decInt8(i *decInstr, state *decoderState, p unsafe.Pointer) {
+	if i.indir > 0 {
+		if *(*unsafe.Pointer)(p) == nil {
+			*(*unsafe.Pointer)(p) = unsafe.Pointer(new(int8))
+		}
+		p = *(*unsafe.Pointer)(p)
+	}
+	v := state.decodeInt()
+	if v < math.MinInt8 || math.MaxInt8 < v {
+		error(i.ovfl)
+	} else {
+		*(*int8)(p) = int8(v)
+	}
+}
+
+// decUint8 decodes an unsigned integer and stores it as a uint8 through p.
+func decUint8(i *decInstr, state *decoderState, p unsafe.Pointer) {
+	if i.indir > 0 {
+		if *(*unsafe.Pointer)(p) == nil {
+			*(*unsafe.Pointer)(p) = unsafe.Pointer(new(uint8))
+		}
+		p = *(*unsafe.Pointer)(p)
+	}
+	v := state.decodeUint()
+	if math.MaxUint8 < v {
+		error(i.ovfl)
+	} else {
+		*(*uint8)(p) = uint8(v)
+	}
+}
+
+// decInt16 decodes an integer and stores it as an int16 through p.
+func decInt16(i *decInstr, state *decoderState, p unsafe.Pointer) {
+	if i.indir > 0 {
+		if *(*unsafe.Pointer)(p) == nil {
+			*(*unsafe.Pointer)(p) = unsafe.Pointer(new(int16))
+		}
+		p = *(*unsafe.Pointer)(p)
+	}
+	v := state.decodeInt()
+	if v < math.MinInt16 || math.MaxInt16 < v {
+		error(i.ovfl)
+	} else {
+		*(*int16)(p) = int16(v)
+	}
+}
+
+// decUint16 decodes an unsigned integer and stores it as a uint16 through p.
+func decUint16(i *decInstr, state *decoderState, p unsafe.Pointer) {
+	if i.indir > 0 {
+		if *(*unsafe.Pointer)(p) == nil {
+			*(*unsafe.Pointer)(p) = unsafe.Pointer(new(uint16))
+		}
+		p = *(*unsafe.Pointer)(p)
+	}
+	v := state.decodeUint()
+	if math.MaxUint16 < v {
+		error(i.ovfl)
+	} else {
+		*(*uint16)(p) = uint16(v)
+	}
+}
+
+// decInt32 decodes an integer and stores it as an int32 through p.
+func decInt32(i *decInstr, state *decoderState, p unsafe.Pointer) {
+	if i.indir > 0 {
+		if *(*unsafe.Pointer)(p) == nil {
+			*(*unsafe.Pointer)(p) = unsafe.Pointer(new(int32))
+		}
+		p = *(*unsafe.Pointer)(p)
+	}
+	v := state.decodeInt()
+	if v < math.MinInt32 || math.MaxInt32 < v {
+		error(i.ovfl)
+	} else {
+		*(*int32)(p) = int32(v)
+	}
+}
+
+// decUint32 decodes an unsigned integer and stores it as a uint32 through p.
+func decUint32(i *decInstr, state *decoderState, p unsafe.Pointer) {
+	if i.indir > 0 {
+		if *(*unsafe.Pointer)(p) == nil {
+			*(*unsafe.Pointer)(p) = unsafe.Pointer(new(uint32))
+		}
+		p = *(*unsafe.Pointer)(p)
+	}
+	v := state.decodeUint()
+	if math.MaxUint32 < v {
+		error(i.ovfl)
+	} else {
+		*(*uint32)(p) = uint32(v)
+	}
+}
+
+// decInt64 decodes an integer and stores it as an int64 through p.
+func decInt64(i *decInstr, state *decoderState, p unsafe.Pointer) {
+	if i.indir > 0 {
+		if *(*unsafe.Pointer)(p) == nil {
+			*(*unsafe.Pointer)(p) = unsafe.Pointer(new(int64))
+		}
+		p = *(*unsafe.Pointer)(p)
+	}
+	*(*int64)(p) = int64(state.decodeInt())
+}
+
+// decUint64 decodes an unsigned integer and stores it as a uint64 through p.
+func decUint64(i *decInstr, state *decoderState, p unsafe.Pointer) {
+	if i.indir > 0 {
+		if *(*unsafe.Pointer)(p) == nil {
+			*(*unsafe.Pointer)(p) = unsafe.Pointer(new(uint64))
+		}
+		p = *(*unsafe.Pointer)(p)
+	}
+	*(*uint64)(p) = uint64(state.decodeUint())
+}
+
+// Floating-point numbers are transmitted as uint64s holding the bits
+// of the underlying representation.  They are sent byte-reversed, with
+// the exponent end coming out first, so integer floating point numbers
+// (for example) transmit more compactly.  This routine does the
+// unswizzling.
+func floatFromBits(u uint64) float64 {
+	var v uint64
+	for i := 0; i < 8; i++ {
+		v <<= 8
+		v |= u & 0xFF
+		u >>= 8
+	}
+	return math.Float64frombits(v)
+}
+
+// storeFloat32 decodes an unsigned integer, treats it as a 32-bit floating-point
+// number, and stores it through p. It's a helper function for float32 and complex64.
+func storeFloat32(i *decInstr, state *decoderState, p unsafe.Pointer) {
+	v := floatFromBits(state.decodeUint())
+	av := v
+	if av < 0 {
+		av = -av
+	}
+	// +Inf is OK in both 32- and 64-bit floats.  Underflow is always OK.
+	if math.MaxFloat32 < av && av <= math.MaxFloat64 {
+		error(i.ovfl)
+	} else {
+		*(*float32)(p) = float32(v)
+	}
+}
+
+// decFloat32 decodes an unsigned integer, treats it as a 32-bit floating-point
+// number, and stores it through p.
+func decFloat32(i *decInstr, state *decoderState, p unsafe.Pointer) {
+	if i.indir > 0 {
+		if *(*unsafe.Pointer)(p) == nil {
+			*(*unsafe.Pointer)(p) = unsafe.Pointer(new(float32))
+		}
+		p = *(*unsafe.Pointer)(p)
+	}
+	storeFloat32(i, state, p)
+}
+
+// decFloat64 decodes an unsigned integer, treats it as a 64-bit floating-point
+// number, and stores it through p.
+func decFloat64(i *decInstr, state *decoderState, p unsafe.Pointer) {
+	if i.indir > 0 {
+		if *(*unsafe.Pointer)(p) == nil {
+			*(*unsafe.Pointer)(p) = unsafe.Pointer(new(float64))
+		}
+		p = *(*unsafe.Pointer)(p)
+	}
+	*(*float64)(p) = floatFromBits(uint64(state.decodeUint()))
+}
+
+// decComplex64 decodes a pair of unsigned integers, treats them as a
+// pair of floating point numbers, and stores them as a complex64 through p.
+// The real part comes first.
+func decComplex64(i *decInstr, state *decoderState, p unsafe.Pointer) {
+	if i.indir > 0 {
+		if *(*unsafe.Pointer)(p) == nil {
+			*(*unsafe.Pointer)(p) = unsafe.Pointer(new(complex64))
+		}
+		p = *(*unsafe.Pointer)(p)
+	}
+	storeFloat32(i, state, p)
+	storeFloat32(i, state, unsafe.Pointer(uintptr(p)+unsafe.Sizeof(float32(0))))
+}
+
+// decComplex128 decodes a pair of unsigned integers, treats them as a
+// pair of floating point numbers, and stores them as a complex128 through p.
+// The real part comes first.
+func decComplex128(i *decInstr, state *decoderState, p unsafe.Pointer) {
+	if i.indir > 0 {
+		if *(*unsafe.Pointer)(p) == nil {
+			*(*unsafe.Pointer)(p) = unsafe.Pointer(new(complex128))
+		}
+		p = *(*unsafe.Pointer)(p)
+	}
+	real := floatFromBits(uint64(state.decodeUint()))
+	imag := floatFromBits(uint64(state.decodeUint()))
+	*(*complex128)(p) = complex(real, imag)
+}
+
+// decUint8Array decodes byte array and stores through p a slice header
+// describing the data.
+// uint8 arrays are encoded as an unsigned count followed by the raw bytes.
+func decUint8Array(i *decInstr, state *decoderState, p unsafe.Pointer) {
+	if i.indir > 0 {
+		if *(*unsafe.Pointer)(p) == nil {
+			*(*unsafe.Pointer)(p) = unsafe.Pointer(new([]uint8))
+		}
+		p = *(*unsafe.Pointer)(p)
+	}
+	b := make([]uint8, state.decodeUint())
+	state.b.Read(b)
+	*(*[]uint8)(p) = b
+}
+
+// decString decodes byte array and stores through p a string header
+// describing the data.
+// Strings are encoded as an unsigned count followed by the raw bytes.
+func decString(i *decInstr, state *decoderState, p unsafe.Pointer) {
+	if i.indir > 0 {
+		if *(*unsafe.Pointer)(p) == nil {
+			*(*unsafe.Pointer)(p) = unsafe.Pointer(new(string))
+		}
+		p = *(*unsafe.Pointer)(p)
+	}
+	b := make([]byte, state.decodeUint())
+	state.b.Read(b)
+	// It would be a shame to do the obvious thing here,
+	//	*(*string)(p) = string(b)
+	// because we've already allocated the storage and this would
+	// allocate again and copy.  So we do this ugly hack, which is even
+	// even more unsafe than it looks as it depends the memory
+	// representation of a string matching the beginning of the memory
+	// representation of a byte slice (a byte slice is longer).
+	*(*string)(p) = *(*string)(unsafe.Pointer(&b))
+}
+
+// ignoreUint8Array skips over the data for a byte slice value with no destination.
+func ignoreUint8Array(i *decInstr, state *decoderState, p unsafe.Pointer) {
+	b := make([]byte, state.decodeUint())
+	state.b.Read(b)
+}
+
+// Execution engine
+
+// The encoder engine is an array of instructions indexed by field number of the incoming
+// decoder.  It is executed with random access according to field number.
+type decEngine struct {
+	instr    []decInstr
+	numInstr int // the number of active instructions
+}
+
+// allocate makes sure storage is available for an object of underlying type rtyp
+// that is indir levels of indirection through p.
+func allocate(rtyp reflect.Type, p uintptr, indir int) uintptr {
+	if indir == 0 {
+		return p
+	}
+	up := unsafe.Pointer(p)
+	if indir > 1 {
+		up = decIndirect(up, indir)
+	}
+	if *(*unsafe.Pointer)(up) == nil {
+		// Allocate object.
+		*(*unsafe.Pointer)(up) = unsafe.New(rtyp)
+	}
+	return *(*uintptr)(up)
+}
+
+// decodeSingle decodes a top-level value that is not a struct and stores it through p.
+// Such values are preceded by a zero, making them have the memory layout of a
+// struct field (although with an illegal field number).
+func (dec *Decoder) decodeSingle(engine *decEngine, ut *userTypeInfo, p uintptr) (err os.Error) {
+	indir := ut.indir
+	if ut.isGobDecoder {
+		indir = int(ut.decIndir)
+	}
+	p = allocate(ut.base, p, indir)
+	state := dec.newDecoderState(&dec.buf)
+	state.fieldnum = singletonField
+	basep := p
+	delta := int(state.decodeUint())
+	if delta != 0 {
+		errorf("decode: corrupted data: non-zero delta for singleton")
+	}
+	instr := &engine.instr[singletonField]
+	ptr := unsafe.Pointer(basep) // offset will be zero
+	if instr.indir > 1 {
+		ptr = decIndirect(ptr, instr.indir)
+	}
+	instr.op(instr, state, ptr)
+	dec.freeDecoderState(state)
+	return nil
+}
+
+// decodeSingle decodes a top-level struct and stores it through p.
+// Indir is for the value, not the type.  At the time of the call it may
+// differ from ut.indir, which was computed when the engine was built.
+// This state cannot arise for decodeSingle, which is called directly
+// from the user's value, not from the innards of an engine.
+func (dec *Decoder) decodeStruct(engine *decEngine, ut *userTypeInfo, p uintptr, indir int) {
+	p = allocate(ut.base, p, indir)
+	state := dec.newDecoderState(&dec.buf)
+	state.fieldnum = -1
+	basep := p
+	for state.b.Len() > 0 {
+		delta := int(state.decodeUint())
+		if delta < 0 {
+			errorf("decode: corrupted data: negative delta")
+		}
+		if delta == 0 { // struct terminator is zero delta fieldnum
+			break
+		}
+		fieldnum := state.fieldnum + delta
+		if fieldnum >= len(engine.instr) {
+			error(errRange)
+			break
+		}
+		instr := &engine.instr[fieldnum]
+		p := unsafe.Pointer(basep + instr.offset)
+		if instr.indir > 1 {
+			p = decIndirect(p, instr.indir)
+		}
+		instr.op(instr, state, p)
+		state.fieldnum = fieldnum
+	}
+	dec.freeDecoderState(state)
+}
+
+// ignoreStruct discards the data for a struct with no destination.
+func (dec *Decoder) ignoreStruct(engine *decEngine) {
+	state := dec.newDecoderState(&dec.buf)
+	state.fieldnum = -1
+	for state.b.Len() > 0 {
+		delta := int(state.decodeUint())
+		if delta < 0 {
+			errorf("ignore decode: corrupted data: negative delta")
+		}
+		if delta == 0 { // struct terminator is zero delta fieldnum
+			break
+		}
+		fieldnum := state.fieldnum + delta
+		if fieldnum >= len(engine.instr) {
+			error(errRange)
+		}
+		instr := &engine.instr[fieldnum]
+		instr.op(instr, state, unsafe.Pointer(nil))
+		state.fieldnum = fieldnum
+	}
+	dec.freeDecoderState(state)
+}
+
+// ignoreSingle discards the data for a top-level non-struct value with no
+// destination. It's used when calling Decode with a nil value.
+func (dec *Decoder) ignoreSingle(engine *decEngine) {
+	state := dec.newDecoderState(&dec.buf)
+	state.fieldnum = singletonField
+	delta := int(state.decodeUint())
+	if delta != 0 {
+		errorf("decode: corrupted data: non-zero delta for singleton")
+	}
+	instr := &engine.instr[singletonField]
+	instr.op(instr, state, unsafe.Pointer(nil))
+	dec.freeDecoderState(state)
+}
+
+// decodeArrayHelper does the work for decoding arrays and slices.
+func (dec *Decoder) decodeArrayHelper(state *decoderState, p uintptr, elemOp decOp, elemWid uintptr, length, elemIndir int, ovfl os.Error) {
+	instr := &decInstr{elemOp, 0, elemIndir, 0, ovfl}
+	for i := 0; i < length; i++ {
+		up := unsafe.Pointer(p)
+		if elemIndir > 1 {
+			up = decIndirect(up, elemIndir)
+		}
+		elemOp(instr, state, up)
+		p += uintptr(elemWid)
+	}
+}
+
+// decodeArray decodes an array and stores it through p, that is, p points to the zeroth element.
+// The length is an unsigned integer preceding the elements.  Even though the length is redundant
+// (it's part of the type), it's a useful check and is included in the encoding.
+func (dec *Decoder) decodeArray(atyp reflect.Type, state *decoderState, p uintptr, elemOp decOp, elemWid uintptr, length, indir, elemIndir int, ovfl os.Error) {
+	if indir > 0 {
+		p = allocate(atyp, p, 1) // All but the last level has been allocated by dec.Indirect
+	}
+	if n := state.decodeUint(); n != uint64(length) {
+		errorf("length mismatch in decodeArray")
+	}
+	dec.decodeArrayHelper(state, p, elemOp, elemWid, length, elemIndir, ovfl)
+}
+
+// decodeIntoValue is a helper for map decoding.  Since maps are decoded using reflection,
+// unlike the other items we can't use a pointer directly.
+func decodeIntoValue(state *decoderState, op decOp, indir int, v reflect.Value, ovfl os.Error) reflect.Value {
+	instr := &decInstr{op, 0, indir, 0, ovfl}
+	up := unsafe.Pointer(unsafeAddr(v))
+	if indir > 1 {
+		up = decIndirect(up, indir)
+	}
+	op(instr, state, up)
+	return v
+}
+
+// decodeMap decodes a map and stores its header through p.
+// Maps are encoded as a length followed by key:value pairs.
+// Because the internals of maps are not visible to us, we must
+// use reflection rather than pointer magic.
+func (dec *Decoder) decodeMap(mtyp reflect.Type, state *decoderState, p uintptr, keyOp, elemOp decOp, indir, keyIndir, elemIndir int, ovfl os.Error) {
+	if indir > 0 {
+		p = allocate(mtyp, p, 1) // All but the last level has been allocated by dec.Indirect
+	}
+	up := unsafe.Pointer(p)
+	if *(*unsafe.Pointer)(up) == nil { // maps are represented as a pointer in the runtime
+		// Allocate map.
+		*(*unsafe.Pointer)(up) = unsafe.Pointer(reflect.MakeMap(mtyp).Pointer())
+	}
+	// Maps cannot be accessed by moving addresses around the way
+	// that slices etc. can.  We must recover a full reflection value for
+	// the iteration.
+	v := reflect.ValueOf(unsafe.Unreflect(mtyp, unsafe.Pointer(p)))
+	n := int(state.decodeUint())
+	for i := 0; i < n; i++ {
+		key := decodeIntoValue(state, keyOp, keyIndir, allocValue(mtyp.Key()), ovfl)
+		elem := decodeIntoValue(state, elemOp, elemIndir, allocValue(mtyp.Elem()), ovfl)
+		v.SetMapIndex(key, elem)
+	}
+}
+
+// ignoreArrayHelper does the work for discarding arrays and slices.
+func (dec *Decoder) ignoreArrayHelper(state *decoderState, elemOp decOp, length int) {
+	instr := &decInstr{elemOp, 0, 0, 0, os.NewError("no error")}
+	for i := 0; i < length; i++ {
+		elemOp(instr, state, nil)
+	}
+}
+
+// ignoreArray discards the data for an array value with no destination.
+func (dec *Decoder) ignoreArray(state *decoderState, elemOp decOp, length int) {
+	if n := state.decodeUint(); n != uint64(length) {
+		errorf("length mismatch in ignoreArray")
+	}
+	dec.ignoreArrayHelper(state, elemOp, length)
+}
+
+// ignoreMap discards the data for a map value with no destination.
+func (dec *Decoder) ignoreMap(state *decoderState, keyOp, elemOp decOp) {
+	n := int(state.decodeUint())
+	keyInstr := &decInstr{keyOp, 0, 0, 0, os.NewError("no error")}
+	elemInstr := &decInstr{elemOp, 0, 0, 0, os.NewError("no error")}
+	for i := 0; i < n; i++ {
+		keyOp(keyInstr, state, nil)
+		elemOp(elemInstr, state, nil)
+	}
+}
+
+// decodeSlice decodes a slice and stores the slice header through p.
+// Slices are encoded as an unsigned length followed by the elements.
+func (dec *Decoder) decodeSlice(atyp reflect.Type, state *decoderState, p uintptr, elemOp decOp, elemWid uintptr, indir, elemIndir int, ovfl os.Error) {
+	n := int(uintptr(state.decodeUint()))
+	if indir > 0 {
+		up := unsafe.Pointer(p)
+		if *(*unsafe.Pointer)(up) == nil {
+			// Allocate the slice header.
+			*(*unsafe.Pointer)(up) = unsafe.Pointer(new([]unsafe.Pointer))
+		}
+		p = *(*uintptr)(up)
+	}
+	// Allocate storage for the slice elements, that is, the underlying array.
+	// Always write a header at p.
+	hdrp := (*reflect.SliceHeader)(unsafe.Pointer(p))
+	hdrp.Data = uintptr(unsafe.NewArray(atyp.Elem(), n))
+	hdrp.Len = n
+	hdrp.Cap = n
+	dec.decodeArrayHelper(state, hdrp.Data, elemOp, elemWid, n, elemIndir, ovfl)
+}
+
+// ignoreSlice skips over the data for a slice value with no destination.
+func (dec *Decoder) ignoreSlice(state *decoderState, elemOp decOp) {
+	dec.ignoreArrayHelper(state, elemOp, int(state.decodeUint()))
+}
+
+// setInterfaceValue sets an interface value to a concrete value,
+// but first it checks that the assignment will succeed.
+func setInterfaceValue(ivalue reflect.Value, value reflect.Value) {
+	if !value.Type().AssignableTo(ivalue.Type()) {
+		errorf("cannot assign value of type %s to %s", value.Type(), ivalue.Type())
+	}
+	ivalue.Set(value)
+}
+
+// decodeInterface decodes an interface value and stores it through p.
+// Interfaces are encoded as the name of a concrete type followed by a value.
+// If the name is empty, the value is nil and no value is sent.
+func (dec *Decoder) decodeInterface(ityp reflect.Type, state *decoderState, p uintptr, indir int) {
+	// Create a writable interface reflect.Value.  We need one even for the nil case.
+	ivalue := allocValue(ityp)
+	// Read the name of the concrete type.
+	b := make([]byte, state.decodeUint())
+	state.b.Read(b)
+	name := string(b)
+	if name == "" {
+		// Copy the representation of the nil interface value to the target.
+		// This is horribly unsafe and special.
+		*(*[2]uintptr)(unsafe.Pointer(p)) = ivalue.InterfaceData()
+		return
+	}
+	// The concrete type must be registered.
+	typ, ok := nameToConcreteType[name]
+	if !ok {
+		errorf("name not registered for interface: %q", name)
+	}
+	// Read the type id of the concrete value.
+	concreteId := dec.decodeTypeSequence(true)
+	if concreteId < 0 {
+		error(dec.err)
+	}
+	// Byte count of value is next; we don't care what it is (it's there
+	// in case we want to ignore the value by skipping it completely).
+	state.decodeUint()
+	// Read the concrete value.
+	value := allocValue(typ)
+	dec.decodeValue(concreteId, value)
+	if dec.err != nil {
+		error(dec.err)
+	}
+	// Allocate the destination interface value.
+	if indir > 0 {
+		p = allocate(ityp, p, 1) // All but the last level has been allocated by dec.Indirect
+	}
+	// Assign the concrete value to the interface.
+	// Tread carefully; it might not satisfy the interface.
+	setInterfaceValue(ivalue, value)
+	// Copy the representation of the interface value to the target.
+	// This is horribly unsafe and special.
+	*(*[2]uintptr)(unsafe.Pointer(p)) = ivalue.InterfaceData()
+}
+
+// ignoreInterface discards the data for an interface value with no destination.
+func (dec *Decoder) ignoreInterface(state *decoderState) {
+	// Read the name of the concrete type.
+	b := make([]byte, state.decodeUint())
+	_, err := state.b.Read(b)
+	if err != nil {
+		error(err)
+	}
+	id := dec.decodeTypeSequence(true)
+	if id < 0 {
+		error(dec.err)
+	}
+	// At this point, the decoder buffer contains a delimited value. Just toss it.
+	state.b.Next(int(state.decodeUint()))
+}
+
+// decodeGobDecoder decodes something implementing the GobDecoder interface.
+// The data is encoded as a byte slice.
+func (dec *Decoder) decodeGobDecoder(state *decoderState, v reflect.Value) {
+	// Read the bytes for the value.
+	b := make([]byte, state.decodeUint())
+	_, err := state.b.Read(b)
+	if err != nil {
+		error(err)
+	}
+	// We know it's a GobDecoder, so just call the method directly.
+	err = v.Interface().(GobDecoder).GobDecode(b)
+	if err != nil {
+		error(err)
+	}
+}
+
+// ignoreGobDecoder discards the data for a GobDecoder value with no destination.
+func (dec *Decoder) ignoreGobDecoder(state *decoderState) {
+	// Read the bytes for the value.
+	b := make([]byte, state.decodeUint())
+	_, err := state.b.Read(b)
+	if err != nil {
+		error(err)
+	}
+}
+
+// Index by Go types.
+var decOpTable = [...]decOp{
+	reflect.Bool:       decBool,
+	reflect.Int8:       decInt8,
+	reflect.Int16:      decInt16,
+	reflect.Int32:      decInt32,
+	reflect.Int64:      decInt64,
+	reflect.Uint8:      decUint8,
+	reflect.Uint16:     decUint16,
+	reflect.Uint32:     decUint32,
+	reflect.Uint64:     decUint64,
+	reflect.Float32:    decFloat32,
+	reflect.Float64:    decFloat64,
+	reflect.Complex64:  decComplex64,
+	reflect.Complex128: decComplex128,
+	reflect.String:     decString,
+}
+
+// Indexed by gob types.  tComplex will be added during type.init().
+var decIgnoreOpMap = map[typeId]decOp{
+	tBool:    ignoreUint,
+	tInt:     ignoreUint,
+	tUint:    ignoreUint,
+	tFloat:   ignoreUint,
+	tBytes:   ignoreUint8Array,
+	tString:  ignoreUint8Array,
+	tComplex: ignoreTwoUints,
+}
+
+// decOpFor returns the decoding op for the base type under rt and
+// the indirection count to reach it.
+func (dec *Decoder) decOpFor(wireId typeId, rt reflect.Type, name string, inProgress map[reflect.Type]*decOp) (*decOp, int) {
+	ut := userType(rt)
+	// If the type implements GobEncoder, we handle it without further processing.
+	if ut.isGobDecoder {
+		return dec.gobDecodeOpFor(ut)
+	}
+	// If this type is already in progress, it's a recursive type (e.g. map[string]*T).
+	// Return the pointer to the op we're already building.
+	if opPtr := inProgress[rt]; opPtr != nil {
+		return opPtr, ut.indir
+	}
+	typ := ut.base
+	indir := ut.indir
+	var op decOp
+	k := typ.Kind()
+	if int(k) < len(decOpTable) {
+		op = decOpTable[k]
+	}
+	if op == nil {
+		inProgress[rt] = &op
+		// Special cases
+		switch t := typ; t.Kind() {
+		case reflect.Array:
+			name = "element of " + name
+			elemId := dec.wireType[wireId].ArrayT.Elem
+			elemOp, elemIndir := dec.decOpFor(elemId, t.Elem(), name, inProgress)
+			ovfl := overflow(name)
+			op = func(i *decInstr, state *decoderState, p unsafe.Pointer) {
+				state.dec.decodeArray(t, state, uintptr(p), *elemOp, t.Elem().Size(), t.Len(), i.indir, elemIndir, ovfl)
+			}
+
+		case reflect.Map:
+			name = "element of " + name
+			keyId := dec.wireType[wireId].MapT.Key
+			elemId := dec.wireType[wireId].MapT.Elem
+			keyOp, keyIndir := dec.decOpFor(keyId, t.Key(), name, inProgress)
+			elemOp, elemIndir := dec.decOpFor(elemId, t.Elem(), name, inProgress)
+			ovfl := overflow(name)
+			op = func(i *decInstr, state *decoderState, p unsafe.Pointer) {
+				up := unsafe.Pointer(p)
+				state.dec.decodeMap(t, state, uintptr(up), *keyOp, *elemOp, i.indir, keyIndir, elemIndir, ovfl)
+			}
+
+		case reflect.Slice:
+			name = "element of " + name
+			if t.Elem().Kind() == reflect.Uint8 {
+				op = decUint8Array
+				break
+			}
+			var elemId typeId
+			if tt, ok := builtinIdToType[wireId]; ok {
+				elemId = tt.(*sliceType).Elem
+			} else {
+				elemId = dec.wireType[wireId].SliceT.Elem
+			}
+			elemOp, elemIndir := dec.decOpFor(elemId, t.Elem(), name, inProgress)
+			ovfl := overflow(name)
+			op = func(i *decInstr, state *decoderState, p unsafe.Pointer) {
+				state.dec.decodeSlice(t, state, uintptr(p), *elemOp, t.Elem().Size(), i.indir, elemIndir, ovfl)
+			}
+
+		case reflect.Struct:
+			// Generate a closure that calls out to the engine for the nested type.
+			enginePtr, err := dec.getDecEnginePtr(wireId, userType(typ))
+			if err != nil {
+				error(err)
+			}
+			op = func(i *decInstr, state *decoderState, p unsafe.Pointer) {
+				// indirect through enginePtr to delay evaluation for recursive structs.
+				dec.decodeStruct(*enginePtr, userType(typ), uintptr(p), i.indir)
+			}
+		case reflect.Interface:
+			op = func(i *decInstr, state *decoderState, p unsafe.Pointer) {
+				state.dec.decodeInterface(t, state, uintptr(p), i.indir)
+			}
+		}
+	}
+	if op == nil {
+		errorf("decode can't handle type %s", rt.String())
+	}
+	return &op, indir
+}
+
+// decIgnoreOpFor returns the decoding op for a field that has no destination.
+func (dec *Decoder) decIgnoreOpFor(wireId typeId) decOp {
+	op, ok := decIgnoreOpMap[wireId]
+	if !ok {
+		if wireId == tInterface {
+			// Special case because it's a method: the ignored item might
+			// define types and we need to record their state in the decoder.
+			op = func(i *decInstr, state *decoderState, p unsafe.Pointer) {
+				state.dec.ignoreInterface(state)
+			}
+			return op
+		}
+		// Special cases
+		wire := dec.wireType[wireId]
+		switch {
+		case wire == nil:
+			errorf("bad data: undefined type %s", wireId.string())
+		case wire.ArrayT != nil:
+			elemId := wire.ArrayT.Elem
+			elemOp := dec.decIgnoreOpFor(elemId)
+			op = func(i *decInstr, state *decoderState, p unsafe.Pointer) {
+				state.dec.ignoreArray(state, elemOp, wire.ArrayT.Len)
+			}
+
+		case wire.MapT != nil:
+			keyId := dec.wireType[wireId].MapT.Key
+			elemId := dec.wireType[wireId].MapT.Elem
+			keyOp := dec.decIgnoreOpFor(keyId)
+			elemOp := dec.decIgnoreOpFor(elemId)
+			op = func(i *decInstr, state *decoderState, p unsafe.Pointer) {
+				state.dec.ignoreMap(state, keyOp, elemOp)
+			}
+
+		case wire.SliceT != nil:
+			elemId := wire.SliceT.Elem
+			elemOp := dec.decIgnoreOpFor(elemId)
+			op = func(i *decInstr, state *decoderState, p unsafe.Pointer) {
+				state.dec.ignoreSlice(state, elemOp)
+			}
+
+		case wire.StructT != nil:
+			// Generate a closure that calls out to the engine for the nested type.
+			enginePtr, err := dec.getIgnoreEnginePtr(wireId)
+			if err != nil {
+				error(err)
+			}
+			op = func(i *decInstr, state *decoderState, p unsafe.Pointer) {
+				// indirect through enginePtr to delay evaluation for recursive structs
+				state.dec.ignoreStruct(*enginePtr)
+			}
+
+		case wire.GobEncoderT != nil:
+			op = func(i *decInstr, state *decoderState, p unsafe.Pointer) {
+				state.dec.ignoreGobDecoder(state)
+			}
+		}
+	}
+	if op == nil {
+		errorf("bad data: ignore can't handle type %s", wireId.string())
+	}
+	return op
+}
+
+// gobDecodeOpFor returns the op for a type that is known to implement
+// GobDecoder.
+func (dec *Decoder) gobDecodeOpFor(ut *userTypeInfo) (*decOp, int) {
+	rcvrType := ut.user
+	if ut.decIndir == -1 {
+		rcvrType = reflect.PtrTo(rcvrType)
+	} else if ut.decIndir > 0 {
+		for i := int8(0); i < ut.decIndir; i++ {
+			rcvrType = rcvrType.Elem()
+		}
+	}
+	var op decOp
+	op = func(i *decInstr, state *decoderState, p unsafe.Pointer) {
+		// Caller has gotten us to within one indirection of our value.
+		if i.indir > 0 {
+			if *(*unsafe.Pointer)(p) == nil {
+				*(*unsafe.Pointer)(p) = unsafe.New(ut.base)
+			}
+		}
+		// Now p is a pointer to the base type.  Do we need to climb out to
+		// get to the receiver type?
+		var v reflect.Value
+		if ut.decIndir == -1 {
+			v = reflect.ValueOf(unsafe.Unreflect(rcvrType, unsafe.Pointer(&p)))
+		} else {
+			v = reflect.ValueOf(unsafe.Unreflect(rcvrType, p))
+		}
+		state.dec.decodeGobDecoder(state, v)
+	}
+	return &op, int(ut.indir)
+
+}
+
+// compatibleType asks: Are these two gob Types compatible?
+// Answers the question for basic types, arrays, maps and slices, plus
+// GobEncoder/Decoder pairs.
+// Structs are considered ok; fields will be checked later.
+func (dec *Decoder) compatibleType(fr reflect.Type, fw typeId, inProgress map[reflect.Type]typeId) bool {
+	if rhs, ok := inProgress[fr]; ok {
+		return rhs == fw
+	}
+	inProgress[fr] = fw
+	ut := userType(fr)
+	wire, ok := dec.wireType[fw]
+	// If fr is a GobDecoder, the wire type must be GobEncoder.
+	// And if fr is not a GobDecoder, the wire type must not be either.
+	if ut.isGobDecoder != (ok && wire.GobEncoderT != nil) { // the parentheses look odd but are correct.
+		return false
+	}
+	if ut.isGobDecoder { // This test trumps all others.
+		return true
+	}
+	switch t := ut.base; t.Kind() {
+	default:
+		// chan, etc: cannot handle.
+		return false
+	case reflect.Bool:
+		return fw == tBool
+	case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64:
+		return fw == tInt
+	case reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64, reflect.Uintptr:
+		return fw == tUint
+	case reflect.Float32, reflect.Float64:
+		return fw == tFloat
+	case reflect.Complex64, reflect.Complex128:
+		return fw == tComplex
+	case reflect.String:
+		return fw == tString
+	case reflect.Interface:
+		return fw == tInterface
+	case reflect.Array:
+		if !ok || wire.ArrayT == nil {
+			return false
+		}
+		array := wire.ArrayT
+		return t.Len() == array.Len && dec.compatibleType(t.Elem(), array.Elem, inProgress)
+	case reflect.Map:
+		if !ok || wire.MapT == nil {
+			return false
+		}
+		MapType := wire.MapT
+		return dec.compatibleType(t.Key(), MapType.Key, inProgress) && dec.compatibleType(t.Elem(), MapType.Elem, inProgress)
+	case reflect.Slice:
+		// Is it an array of bytes?
+		if t.Elem().Kind() == reflect.Uint8 {
+			return fw == tBytes
+		}
+		// Extract and compare element types.
+		var sw *sliceType
+		if tt, ok := builtinIdToType[fw]; ok {
+			sw = tt.(*sliceType)
+		} else {
+			sw = dec.wireType[fw].SliceT
+		}
+		elem := userType(t.Elem()).base
+		return sw != nil && dec.compatibleType(elem, sw.Elem, inProgress)
+	case reflect.Struct:
+		return true
+	}
+	return true
+}
+
+// typeString returns a human-readable description of the type identified by remoteId.
+func (dec *Decoder) typeString(remoteId typeId) string {
+	if t := idToType[remoteId]; t != nil {
+		// globally known type.
+		return t.string()
+	}
+	return dec.wireType[remoteId].string()
+}
+
+// compileSingle compiles the decoder engine for a non-struct top-level value, including
+// GobDecoders.
+func (dec *Decoder) compileSingle(remoteId typeId, ut *userTypeInfo) (engine *decEngine, err os.Error) {
+	rt := ut.base
+	if ut.isGobDecoder {
+		rt = ut.user
+	}
+	engine = new(decEngine)
+	engine.instr = make([]decInstr, 1) // one item
+	name := rt.String()                // best we can do
+	if !dec.compatibleType(rt, remoteId, make(map[reflect.Type]typeId)) {
+		return nil, os.NewError("gob: wrong type received for local value " + name + ": " + dec.typeString(remoteId))
+	}
+	op, indir := dec.decOpFor(remoteId, rt, name, make(map[reflect.Type]*decOp))
+	ovfl := os.NewError(`value for "` + name + `" out of range`)
+	engine.instr[singletonField] = decInstr{*op, singletonField, indir, 0, ovfl}
+	engine.numInstr = 1
+	return
+}
+
+// compileIgnoreSingle compiles the decoder engine for a non-struct top-level value that will be discarded.
+func (dec *Decoder) compileIgnoreSingle(remoteId typeId) (engine *decEngine, err os.Error) {
+	engine = new(decEngine)
+	engine.instr = make([]decInstr, 1) // one item
+	op := dec.decIgnoreOpFor(remoteId)
+	ovfl := overflow(dec.typeString(remoteId))
+	engine.instr[0] = decInstr{op, 0, 0, 0, ovfl}
+	engine.numInstr = 1
+	return
+}
+
+// compileDec compiles the decoder engine for a value.  If the value is not a struct,
+// it calls out to compileSingle.
+func (dec *Decoder) compileDec(remoteId typeId, ut *userTypeInfo) (engine *decEngine, err os.Error) {
+	rt := ut.base
+	srt := rt
+	if srt.Kind() != reflect.Struct ||
+		ut.isGobDecoder {
+		return dec.compileSingle(remoteId, ut)
+	}
+	var wireStruct *structType
+	// Builtin types can come from global pool; the rest must be defined by the decoder.
+	// Also we know we're decoding a struct now, so the client must have sent one.
+	if t, ok := builtinIdToType[remoteId]; ok {
+		wireStruct, _ = t.(*structType)
+	} else {
+		wire := dec.wireType[remoteId]
+		if wire == nil {
+			error(errBadType)
+		}
+		wireStruct = wire.StructT
+	}
+	if wireStruct == nil {
+		errorf("type mismatch in decoder: want struct type %s; got non-struct", rt.String())
+	}
+	engine = new(decEngine)
+	engine.instr = make([]decInstr, len(wireStruct.Field))
+	seen := make(map[reflect.Type]*decOp)
+	// Loop over the fields of the wire type.
+	for fieldnum := 0; fieldnum < len(wireStruct.Field); fieldnum++ {
+		wireField := wireStruct.Field[fieldnum]
+		if wireField.Name == "" {
+			errorf("empty name for remote field of type %s", wireStruct.Name)
+		}
+		ovfl := overflow(wireField.Name)
+		// Find the field of the local type with the same name.
+		localField, present := srt.FieldByName(wireField.Name)
+		// TODO(r): anonymous names
+		if !present || !isExported(wireField.Name) {
+			op := dec.decIgnoreOpFor(wireField.Id)
+			engine.instr[fieldnum] = decInstr{op, fieldnum, 0, 0, ovfl}
+			continue
+		}
+		if !dec.compatibleType(localField.Type, wireField.Id, make(map[reflect.Type]typeId)) {
+			errorf("wrong type (%s) for received field %s.%s", localField.Type, wireStruct.Name, wireField.Name)
+		}
+		op, indir := dec.decOpFor(wireField.Id, localField.Type, localField.Name, seen)
+		engine.instr[fieldnum] = decInstr{*op, fieldnum, indir, uintptr(localField.Offset), ovfl}
+		engine.numInstr++
+	}
+	return
+}
+
+// getDecEnginePtr returns the engine for the specified type.
+func (dec *Decoder) getDecEnginePtr(remoteId typeId, ut *userTypeInfo) (enginePtr **decEngine, err os.Error) {
+	rt := ut.base
+	decoderMap, ok := dec.decoderCache[rt]
+	if !ok {
+		decoderMap = make(map[typeId]**decEngine)
+		dec.decoderCache[rt] = decoderMap
+	}
+	if enginePtr, ok = decoderMap[remoteId]; !ok {
+		// To handle recursive types, mark this engine as underway before compiling.
+		enginePtr = new(*decEngine)
+		decoderMap[remoteId] = enginePtr
+		*enginePtr, err = dec.compileDec(remoteId, ut)
+		if err != nil {
+			decoderMap[remoteId] = nil, false
+		}
+	}
+	return
+}
+
+// emptyStruct is the type we compile into when ignoring a struct value.
+type emptyStruct struct{}
+
+var emptyStructType = reflect.TypeOf(emptyStruct{})
+
+// getDecEnginePtr returns the engine for the specified type when the value is to be discarded.
+func (dec *Decoder) getIgnoreEnginePtr(wireId typeId) (enginePtr **decEngine, err os.Error) {
+	var ok bool
+	if enginePtr, ok = dec.ignorerCache[wireId]; !ok {
+		// To handle recursive types, mark this engine as underway before compiling.
+		enginePtr = new(*decEngine)
+		dec.ignorerCache[wireId] = enginePtr
+		wire := dec.wireType[wireId]
+		if wire != nil && wire.StructT != nil {
+			*enginePtr, err = dec.compileDec(wireId, userType(emptyStructType))
+		} else {
+			*enginePtr, err = dec.compileIgnoreSingle(wireId)
+		}
+		if err != nil {
+			dec.ignorerCache[wireId] = nil, false
+		}
+	}
+	return
+}
+
+// decodeValue decodes the data stream representing a value and stores it in val.
+func (dec *Decoder) decodeValue(wireId typeId, val reflect.Value) {
+	defer catchError(&dec.err)
+	// If the value is nil, it means we should just ignore this item.
+	if !val.IsValid() {
+		dec.decodeIgnoredValue(wireId)
+		return
+	}
+	// Dereference down to the underlying struct type.
+	ut := userType(val.Type())
+	base := ut.base
+	var enginePtr **decEngine
+	enginePtr, dec.err = dec.getDecEnginePtr(wireId, ut)
+	if dec.err != nil {
+		return
+	}
+	engine := *enginePtr
+	if st := base; st.Kind() == reflect.Struct && !ut.isGobDecoder {
+		if engine.numInstr == 0 && st.NumField() > 0 && len(dec.wireType[wireId].StructT.Field) > 0 {
+			name := base.Name()
+			errorf("type mismatch: no fields matched compiling decoder for %s", name)
+		}
+		dec.decodeStruct(engine, ut, uintptr(unsafeAddr(val)), ut.indir)
+	} else {
+		dec.decodeSingle(engine, ut, uintptr(unsafeAddr(val)))
+	}
+}
+
+// decodeIgnoredValue decodes the data stream representing a value of the specified type and discards it.
+func (dec *Decoder) decodeIgnoredValue(wireId typeId) {
+	var enginePtr **decEngine
+	enginePtr, dec.err = dec.getIgnoreEnginePtr(wireId)
+	if dec.err != nil {
+		return
+	}
+	wire := dec.wireType[wireId]
+	if wire != nil && wire.StructT != nil {
+		dec.ignoreStruct(*enginePtr)
+	} else {
+		dec.ignoreSingle(*enginePtr)
+	}
+}
+
+func init() {
+	var iop, uop decOp
+	switch reflect.TypeOf(int(0)).Bits() {
+	case 32:
+		iop = decInt32
+		uop = decUint32
+	case 64:
+		iop = decInt64
+		uop = decUint64
+	default:
+		panic("gob: unknown size of int/uint")
+	}
+	decOpTable[reflect.Int] = iop
+	decOpTable[reflect.Uint] = uop
+
+	// Finally uintptr
+	switch reflect.TypeOf(uintptr(0)).Bits() {
+	case 32:
+		uop = decUint32
+	case 64:
+		uop = decUint64
+	default:
+		panic("gob: unknown size of uintptr")
+	}
+	decOpTable[reflect.Uintptr] = uop
+}
+
+// Gob assumes it can call UnsafeAddr on any Value
+// in order to get a pointer it can copy data from.
+// Values that have just been created and do not point
+// into existing structs or slices cannot be addressed,
+// so simulate it by returning a pointer to a copy.
+// Each call allocates once.
+func unsafeAddr(v reflect.Value) uintptr {
+	if v.CanAddr() {
+		return v.UnsafeAddr()
+	}
+	x := reflect.New(v.Type()).Elem()
+	x.Set(v)
+	return x.UnsafeAddr()
+}
+
+// Gob depends on being able to take the address
+// of zeroed Values it creates, so use this wrapper instead
+// of the standard reflect.Zero.
+// Each call allocates once.
+func allocValue(t reflect.Type) reflect.Value {
+	return reflect.New(t).Elem()
+}
diff --git a/src/pkg/gob/decoder.go b/src/pkg/gob/decoder.go
new file mode 100644
index 000000000..281947132
--- /dev/null
+++ b/src/pkg/gob/decoder.go
@@ -0,0 +1,202 @@
+// Copyright 2009 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package gob
+
+import (
+	"bufio"
+	"bytes"
+	"io"
+	"os"
+	"reflect"
+	"sync"
+)
+
+// A Decoder manages the receipt of type and data information read from the
+// remote side of a connection.
+type Decoder struct {
+	mutex        sync.Mutex                              // each item must be received atomically
+	r            io.Reader                               // source of the data
+	buf          bytes.Buffer                            // buffer for more efficient i/o from r
+	wireType     map[typeId]*wireType                    // map from remote ID to local description
+	decoderCache map[reflect.Type]map[typeId]**decEngine // cache of compiled engines
+	ignorerCache map[typeId]**decEngine                  // ditto for ignored objects
+	freeList     *decoderState                           // list of free decoderStates; avoids reallocation
+	countBuf     []byte                                  // used for decoding integers while parsing messages
+	tmp          []byte                                  // temporary storage for i/o; saves reallocating
+	err          os.Error
+}
+
+// NewDecoder returns a new decoder that reads from the io.Reader.
+func NewDecoder(r io.Reader) *Decoder {
+	dec := new(Decoder)
+	dec.r = bufio.NewReader(r)
+	dec.wireType = make(map[typeId]*wireType)
+	dec.decoderCache = make(map[reflect.Type]map[typeId]**decEngine)
+	dec.ignorerCache = make(map[typeId]**decEngine)
+	dec.countBuf = make([]byte, 9) // counts may be uint64s (unlikely!), require 9 bytes
+
+	return dec
+}
+
+// recvType loads the definition of a type.
+func (dec *Decoder) recvType(id typeId) {
+	// Have we already seen this type?  That's an error
+	if id < firstUserId || dec.wireType[id] != nil {
+		dec.err = os.NewError("gob: duplicate type received")
+		return
+	}
+
+	// Type:
+	wire := new(wireType)
+	dec.decodeValue(tWireType, reflect.ValueOf(wire))
+	if dec.err != nil {
+		return
+	}
+	// Remember we've seen this type.
+	dec.wireType[id] = wire
+}
+
+// recvMessage reads the next count-delimited item from the input. It is the converse
+// of Encoder.writeMessage. It returns false on EOF or other error reading the message.
+func (dec *Decoder) recvMessage() bool {
+	// Read a count.
+	nbytes, _, err := decodeUintReader(dec.r, dec.countBuf)
+	if err != nil {
+		dec.err = err
+		return false
+	}
+	dec.readMessage(int(nbytes))
+	return dec.err == nil
+}
+
+// readMessage reads the next nbytes bytes from the input.
+func (dec *Decoder) readMessage(nbytes int) {
+	// Allocate the buffer.
+	if cap(dec.tmp) < nbytes {
+		dec.tmp = make([]byte, nbytes+100) // room to grow
+	}
+	dec.tmp = dec.tmp[:nbytes]
+
+	// Read the data
+	_, dec.err = io.ReadFull(dec.r, dec.tmp)
+	if dec.err != nil {
+		if dec.err == os.EOF {
+			dec.err = io.ErrUnexpectedEOF
+		}
+		return
+	}
+	dec.buf.Write(dec.tmp)
+}
+
+// toInt turns an encoded uint64 into an int, according to the marshaling rules.
+func toInt(x uint64) int64 {
+	i := int64(x >> 1)
+	if x&1 != 0 {
+		i = ^i
+	}
+	return i
+}
+
+func (dec *Decoder) nextInt() int64 {
+	n, _, err := decodeUintReader(&dec.buf, dec.countBuf)
+	if err != nil {
+		dec.err = err
+	}
+	return toInt(n)
+}
+
+func (dec *Decoder) nextUint() uint64 {
+	n, _, err := decodeUintReader(&dec.buf, dec.countBuf)
+	if err != nil {
+		dec.err = err
+	}
+	return n
+}
+
+// decodeTypeSequence parses:
+// TypeSequence
+//	(TypeDefinition DelimitedTypeDefinition*)?
+// and returns the type id of the next value.  It returns -1 at
+// EOF.  Upon return, the remainder of dec.buf is the value to be
+// decoded.  If this is an interface value, it can be ignored by
+// simply resetting that buffer.
+func (dec *Decoder) decodeTypeSequence(isInterface bool) typeId {
+	for dec.err == nil {
+		if dec.buf.Len() == 0 {
+			if !dec.recvMessage() {
+				break
+			}
+		}
+		// Receive a type id.
+		id := typeId(dec.nextInt())
+		if id >= 0 {
+			// Value follows.
+			return id
+		}
+		// Type definition for (-id) follows.
+		dec.recvType(-id)
+		// When decoding an interface, after a type there may be a
+		// DelimitedValue still in the buffer.  Skip its count.
+		// (Alternatively, the buffer is empty and the byte count
+		// will be absorbed by recvMessage.)
+		if dec.buf.Len() > 0 {
+			if !isInterface {
+				dec.err = os.NewError("extra data in buffer")
+				break
+			}
+			dec.nextUint()
+		}
+	}
+	return -1
+}
+
+// Decode reads the next value from the connection and stores
+// it in the data represented by the empty interface value.
+// If e is nil, the value will be discarded. Otherwise,
+// the value underlying e must be a pointer to the
+// correct type for the next data item received.
+func (dec *Decoder) Decode(e interface{}) os.Error {
+	if e == nil {
+		return dec.DecodeValue(reflect.Value{})
+	}
+	value := reflect.ValueOf(e)
+	// If e represents a value as opposed to a pointer, the answer won't
+	// get back to the caller.  Make sure it's a pointer.
+	if value.Type().Kind() != reflect.Ptr {
+		dec.err = os.NewError("gob: attempt to decode into a non-pointer")
+		return dec.err
+	}
+	return dec.DecodeValue(value)
+}
+
+// DecodeValue reads the next value from the connection.
+// If v is the zero reflect.Value (v.Kind() == Invalid), DecodeValue discards the value.
+// Otherwise, it stores the value into v.  In that case, v must represent
+// a non-nil pointer to data or be an assignable reflect.Value (v.CanSet())
+func (dec *Decoder) DecodeValue(v reflect.Value) os.Error {
+	if v.IsValid() {
+		if v.Kind() == reflect.Ptr && !v.IsNil() {
+			// That's okay, we'll store through the pointer.
+		} else if !v.CanSet() {
+			return os.NewError("gob: DecodeValue of unassignable value")
+		}
+	}
+	// Make sure we're single-threaded through here.
+	dec.mutex.Lock()
+	defer dec.mutex.Unlock()
+
+	dec.buf.Reset() // In case data lingers from previous invocation.
+	dec.err = nil
+	id := dec.decodeTypeSequence(false)
+	if dec.err == nil {
+		dec.decodeValue(id, v)
+	}
+	return dec.err
+}
+
+// If debug.go is compiled into the program , debugFunc prints a human-readable
+// representation of the gob data read from r by calling that file's Debug function.
+// Otherwise it is nil.
+var debugFunc func(io.Reader)
diff --git a/src/pkg/gob/doc.go b/src/pkg/gob/doc.go
new file mode 100644
index 000000000..35d882afb
--- /dev/null
+++ b/src/pkg/gob/doc.go
@@ -0,0 +1,360 @@
+// Copyright 2009 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+/*
+Package gob manages streams of gobs - binary values exchanged between an
+Encoder (transmitter) and a Decoder (receiver).  A typical use is transporting
+arguments and results of remote procedure calls (RPCs) such as those provided by
+package "rpc".
+
+A stream of gobs is self-describing.  Each data item in the stream is preceded by
+a specification of its type, expressed in terms of a small set of predefined
+types.  Pointers are not transmitted, but the things they point to are
+transmitted; that is, the values are flattened.  Recursive types work fine, but
+recursive values (data with cycles) are problematic.  This may change.
+
+To use gobs, create an Encoder and present it with a series of data items as
+values or addresses that can be dereferenced to values.  The Encoder makes sure
+all type information is sent before it is needed.  At the receive side, a
+Decoder retrieves values from the encoded stream and unpacks them into local
+variables.
+
+The source and destination values/types need not correspond exactly.  For structs,
+fields (identified by name) that are in the source but absent from the receiving
+variable will be ignored.  Fields that are in the receiving variable but missing
+from the transmitted type or value will be ignored in the destination.  If a field
+with the same name is present in both, their types must be compatible. Both the
+receiver and transmitter will do all necessary indirection and dereferencing to
+convert between gobs and actual Go values.  For instance, a gob type that is
+schematically,
+
+	struct { A, B int }
+
+can be sent from or received into any of these Go types:
+
+	struct { A, B int }	// the same
+	*struct { A, B int }	// extra indirection of the struct
+	struct { *A, **B int }	// extra indirection of the fields
+	struct { A, B int64 }	// different concrete value type; see below
+
+It may also be received into any of these:
+
+	struct { A, B int }	// the same
+	struct { B, A int }	// ordering doesn't matter; matching is by name
+	struct { A, B, C int }	// extra field (C) ignored
+	struct { B int }	// missing field (A) ignored; data will be dropped
+	struct { B, C int }	// missing field (A) ignored; extra field (C) ignored.
+
+Attempting to receive into these types will draw a decode error:
+
+	struct { A int; B uint }	// change of signedness for B
+	struct { A int; B float }	// change of type for B
+	struct { }			// no field names in common
+	struct { C, D int }		// no field names in common
+
+Integers are transmitted two ways: arbitrary precision signed integers or
+arbitrary precision unsigned integers.  There is no int8, int16 etc.
+discrimination in the gob format; there are only signed and unsigned integers.  As
+described below, the transmitter sends the value in a variable-length encoding;
+the receiver accepts the value and stores it in the destination variable.
+Floating-point numbers are always sent using IEEE-754 64-bit precision (see
+below).
+
+Signed integers may be received into any signed integer variable: int, int16, etc.;
+unsigned integers may be received into any unsigned integer variable; and floating
+point values may be received into any floating point variable.  However,
+the destination variable must be able to represent the value or the decode
+operation will fail.
+
+Structs, arrays and slices are also supported.  Strings and arrays of bytes are
+supported with a special, efficient representation (see below).
+
+Functions and channels cannot be sent in a gob.  Attempting
+to encode a value that contains one will fail.
+
+The rest of this comment documents the encoding, details that are not important
+for most users.  Details are presented bottom-up.
+
+An unsigned integer is sent one of two ways.  If it is less than 128, it is sent
+as a byte with that value.  Otherwise it is sent as a minimal-length big-endian
+(high byte first) byte stream holding the value, preceded by one byte holding the
+byte count, negated.  Thus 0 is transmitted as (00), 7 is transmitted as (07) and
+256 is transmitted as (FE 01 00).
+
+A boolean is encoded within an unsigned integer: 0 for false, 1 for true.
+
+A signed integer, i, is encoded within an unsigned integer, u.  Within u, bits 1
+upward contain the value; bit 0 says whether they should be complemented upon
+receipt.  The encode algorithm looks like this:
+
+	uint u;
+	if i < 0 {
+		u = (^i << 1) | 1	// complement i, bit 0 is 1
+	} else {
+		u = (i << 1)	// do not complement i, bit 0 is 0
+	}
+	encodeUnsigned(u)
+
+The low bit is therefore analogous to a sign bit, but making it the complement bit
+instead guarantees that the largest negative integer is not a special case.  For
+example, -129=^128=(^256>>1) encodes as (FE 01 01).
+
+Floating-point numbers are always sent as a representation of a float64 value.
+That value is converted to a uint64 using math.Float64bits.  The uint64 is then
+byte-reversed and sent as a regular unsigned integer.  The byte-reversal means the
+exponent and high-precision part of the mantissa go first.  Since the low bits are
+often zero, this can save encoding bytes.  For instance, 17.0 is encoded in only
+three bytes (FE 31 40).
+
+Strings and slices of bytes are sent as an unsigned count followed by that many
+uninterpreted bytes of the value.
+
+All other slices and arrays are sent as an unsigned count followed by that many
+elements using the standard gob encoding for their type, recursively.
+
+Maps are sent as an unsigned count followed by that man key, element
+pairs. Empty but non-nil maps are sent, so if the sender has allocated
+a map, the receiver will allocate a map even no elements are
+transmitted.
+
+Structs are sent as a sequence of (field number, field value) pairs.  The field
+value is sent using the standard gob encoding for its type, recursively.  If a
+field has the zero value for its type, it is omitted from the transmission.  The
+field number is defined by the type of the encoded struct: the first field of the
+encoded type is field 0, the second is field 1, etc.  When encoding a value, the
+field numbers are delta encoded for efficiency and the fields are always sent in
+order of increasing field number; the deltas are therefore unsigned.  The
+initialization for the delta encoding sets the field number to -1, so an unsigned
+integer field 0 with value 7 is transmitted as unsigned delta = 1, unsigned value
+= 7 or (01 07).  Finally, after all the fields have been sent a terminating mark
+denotes the end of the struct.  That mark is a delta=0 value, which has
+representation (00).
+
+Interface types are not checked for compatibility; all interface types are
+treated, for transmission, as members of a single "interface" type, analogous to
+int or []byte - in effect they're all treated as interface{}.  Interface values
+are transmitted as a string identifying the concrete type being sent (a name
+that must be pre-defined by calling Register), followed by a byte count of the
+length of the following data (so the value can be skipped if it cannot be
+stored), followed by the usual encoding of concrete (dynamic) value stored in
+the interface value.  (A nil interface value is identified by the empty string
+and transmits no value.) Upon receipt, the decoder verifies that the unpacked
+concrete item satisfies the interface of the receiving variable.
+
+The representation of types is described below.  When a type is defined on a given
+connection between an Encoder and Decoder, it is assigned a signed integer type
+id.  When Encoder.Encode(v) is called, it makes sure there is an id assigned for
+the type of v and all its elements and then it sends the pair (typeid, encoded-v)
+where typeid is the type id of the encoded type of v and encoded-v is the gob
+encoding of the value v.
+
+To define a type, the encoder chooses an unused, positive type id and sends the
+pair (-type id, encoded-type) where encoded-type is the gob encoding of a wireType
+description, constructed from these types:
+
+	type wireType struct {
+		ArrayT  *ArrayType
+		SliceT  *SliceType
+		StructT *StructType
+		MapT    *MapType
+	}
+	type ArrayType struct {
+		CommonType
+		Elem typeId
+		Len  int
+	}
+	type CommonType struct {
+		Name string // the name of the struct type
+		Id  int    // the id of the type, repeated so it's inside the type
+	}
+	type SliceType struct {
+		CommonType
+		Elem typeId
+	}
+	type StructType struct {
+		CommonType
+		Field []*fieldType // the fields of the struct.
+	}
+	type FieldType struct {
+		Name string // the name of the field.
+		Id   int    // the type id of the field, which must be already defined
+	}
+	type MapType struct {
+		CommonType
+		Key  typeId
+		Elem typeId
+	}
+
+If there are nested type ids, the types for all inner type ids must be defined
+before the top-level type id is used to describe an encoded-v.
+
+For simplicity in setup, the connection is defined to understand these types a
+priori, as well as the basic gob types int, uint, etc.  Their ids are:
+
+	bool        1
+	int         2
+	uint        3
+	float       4
+	[]byte      5
+	string      6
+	complex     7
+	interface   8
+	// gap for reserved ids.
+	WireType    16
+	ArrayType   17
+	CommonType  18
+	SliceType   19
+	StructType  20
+	FieldType   21
+	// 22 is slice of fieldType.
+	MapType     23
+
+Finally, each message created by a call to Encode is preceded by an encoded
+unsigned integer count of the number of bytes remaining in the message.  After
+the initial type name, interface values are wrapped the same way; in effect, the
+interface value acts like a recursive invocation of Encode.
+
+In summary, a gob stream looks like
+
+	(byteCount (-type id, encoding of a wireType)* (type id, encoding of a value))*
+
+where * signifies zero or more repetitions and the type id of a value must
+be predefined or be defined before the value in the stream.
+*/
+package gob
+
+/*
+Grammar:
+
+Tokens starting with a lower case letter are terminals; int(n)
+and uint(n) represent the signed/unsigned encodings of the value n.
+
+GobStream:
+	DelimitedMessage*
+DelimitedMessage:
+	uint(lengthOfMessage) Message
+Message:
+	TypeSequence TypedValue
+TypeSequence
+	(TypeDefinition DelimitedTypeDefinition*)?
+DelimitedTypeDefinition:
+	uint(lengthOfTypeDefinition) TypeDefinition
+TypedValue:
+	int(typeId) Value
+TypeDefinition:
+	int(-typeId) encodingOfWireType
+Value:
+	SingletonValue | StructValue
+SingletonValue:
+	uint(0) FieldValue
+FieldValue:
+	builtinValue | ArrayValue | MapValue | SliceValue | StructValue | InterfaceValue
+InterfaceValue:
+	NilInterfaceValue | NonNilInterfaceValue
+NilInterfaceValue:
+	uint(0)
+NonNilInterfaceValue:
+	ConcreteTypeName TypeSequence InterfaceContents
+ConcreteTypeName:
+	uint(lengthOfName) [already read=n] name
+InterfaceContents:
+	int(concreteTypeId) DelimitedValue
+DelimitedValue:
+	uint(length) Value
+ArrayValue:
+	uint(n) FieldValue*n [n elements]
+MapValue:
+	uint(n) (FieldValue FieldValue)*n  [n (key, value) pairs]
+SliceValue:
+	uint(n) FieldValue*n [n elements]
+StructValue:
+	(uint(fieldDelta) FieldValue)*
+*/
+
+/*
+For implementers and the curious, here is an encoded example.  Given
+	type Point struct {X, Y int}
+and the value
+	p := Point{22, 33}
+the bytes transmitted that encode p will be:
+	1f ff 81 03 01 01 05 50 6f 69 6e 74 01 ff 82 00
+	01 02 01 01 58 01 04 00 01 01 59 01 04 00 00 00
+	07 ff 82 01 2c 01 42 00
+They are determined as follows.
+
+Since this is the first transmission of type Point, the type descriptor
+for Point itself must be sent before the value.  This is the first type
+we've sent on this Encoder, so it has type id 65 (0 through 64 are
+reserved).
+
+	1f	// This item (a type descriptor) is 31 bytes long.
+	ff 81	// The negative of the id for the type we're defining, -65.
+		// This is one byte (indicated by FF = -1) followed by
+		// ^-65<<1 | 1.  The low 1 bit signals to complement the
+		// rest upon receipt.
+
+	// Now we send a type descriptor, which is itself a struct (wireType).
+	// The type of wireType itself is known (it's built in, as is the type of
+	// all its components), so we just need to send a *value* of type wireType
+	// that represents type "Point".
+	// Here starts the encoding of that value.
+	// Set the field number implicitly to -1; this is done at the beginning
+	// of every struct, including nested structs.
+	03	// Add 3 to field number; now 2 (wireType.structType; this is a struct).
+		// structType starts with an embedded commonType, which appears
+		// as a regular structure here too.
+	01	// add 1 to field number (now 0); start of embedded commonType.
+	01	// add 1 to field number (now 0, the name of the type)
+	05	// string is (unsigned) 5 bytes long
+	50 6f 69 6e 74	// wireType.structType.commonType.name = "Point"
+	01	// add 1 to field number (now 1, the id of the type)
+	ff 82	// wireType.structType.commonType._id = 65
+	00	// end of embedded wiretype.structType.commonType struct
+	01	// add 1 to field number (now 1, the field array in wireType.structType)
+	02	// There are two fields in the type (len(structType.field))
+	01	// Start of first field structure; add 1 to get field number 0: field[0].name
+	01	// 1 byte
+	58	// structType.field[0].name = "X"
+	01	// Add 1 to get field number 1: field[0].id
+	04	// structType.field[0].typeId is 2 (signed int).
+	00	// End of structType.field[0]; start structType.field[1]; set field number to -1.
+	01	// Add 1 to get field number 0: field[1].name
+	01	// 1 byte
+	59	// structType.field[1].name = "Y"
+	01	// Add 1 to get field number 1: field[0].id
+	04	// struct.Type.field[1].typeId is 2 (signed int).
+	00	// End of structType.field[1]; end of structType.field.
+	00	// end of wireType.structType structure
+	00	// end of wireType structure
+
+Now we can send the Point value.  Again the field number resets to -1:
+
+	07	// this value is 7 bytes long
+	ff 82	// the type number, 65 (1 byte (-FF) followed by 65<<1)
+	01	// add one to field number, yielding field 0
+	2c	// encoding of signed "22" (0x22 = 44 = 22<<1); Point.x = 22
+	01	// add one to field number, yielding field 1
+	42	// encoding of signed "33" (0x42 = 66 = 33<<1); Point.y = 33
+	00	// end of structure
+
+The type encoding is long and fairly intricate but we send it only once.
+If p is transmitted a second time, the type is already known so the
+output will be just:
+
+	07 ff 82 01 2c 01 42 00
+
+A single non-struct value at top level is transmitted like a field with
+delta tag 0.  For instance, a signed integer with value 3 presented as
+the argument to Encode will emit:
+
+	03 04 00 06
+
+Which represents:
+
+	03	// this value is 3 bytes long
+	04	// the type number, 2, represents an integer
+	00	// tag delta 0
+	06	// value 3
+
+*/
diff --git a/src/pkg/gob/dump.go b/src/pkg/gob/dump.go
new file mode 100644
index 000000000..1555f0fbb
--- /dev/null
+++ b/src/pkg/gob/dump.go
@@ -0,0 +1,22 @@
+package main
+
+// Need to compile package gob with debug.go to build this program.
+
+import (
+	"fmt"
+	"gob"
+	"os"
+)
+
+func main() {
+	var err os.Error
+	file := os.Stdin
+	if len(os.Args) > 1 {
+		file, err = os.Open(os.Args[1])
+		if err != nil {
+			fmt.Fprintf(os.Stderr, "dump: %s\n", err)
+			os.Exit(1)
+		}
+	}
+	gob.Debug(file)
+}
diff --git a/src/pkg/gob/encode.go b/src/pkg/gob/encode.go
new file mode 100644
index 000000000..317014efd
--- /dev/null
+++ b/src/pkg/gob/encode.go
@@ -0,0 +1,717 @@
+// Copyright 2009 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package gob
+
+import (
+	"bytes"
+	"math"
+	"reflect"
+	"unsafe"
+)
+
+const uint64Size = int(unsafe.Sizeof(uint64(0)))
+
+// encoderState is the global execution state of an instance of the encoder.
+// Field numbers are delta encoded and always increase. The field
+// number is initialized to -1 so 0 comes out as delta(1). A delta of
+// 0 terminates the structure.
+type encoderState struct {
+	enc      *Encoder
+	b        *bytes.Buffer
+	sendZero bool                 // encoding an array element or map key/value pair; send zero values
+	fieldnum int                  // the last field number written.
+	buf      [1 + uint64Size]byte // buffer used by the encoder; here to avoid allocation.
+	next     *encoderState        // for free list
+}
+
+func (enc *Encoder) newEncoderState(b *bytes.Buffer) *encoderState {
+	e := enc.freeList
+	if e == nil {
+		e = new(encoderState)
+		e.enc = enc
+	} else {
+		enc.freeList = e.next
+	}
+	e.sendZero = false
+	e.fieldnum = 0
+	e.b = b
+	return e
+}
+
+func (enc *Encoder) freeEncoderState(e *encoderState) {
+	e.next = enc.freeList
+	enc.freeList = e
+}
+
+// Unsigned integers have a two-state encoding.  If the number is less
+// than 128 (0 through 0x7F), its value is written directly.
+// Otherwise the value is written in big-endian byte order preceded
+// by the byte length, negated.
+
+// encodeUint writes an encoded unsigned integer to state.b.
+func (state *encoderState) encodeUint(x uint64) {
+	if x <= 0x7F {
+		err := state.b.WriteByte(uint8(x))
+		if err != nil {
+			error(err)
+		}
+		return
+	}
+	var n, m int
+	m = uint64Size
+	for n = 1; x > 0; n++ {
+		state.buf[m] = uint8(x)
+		x >>= 8
+		m--
+	}
+	state.buf[m] = uint8(-(n - 1))
+	n, err := state.b.Write(state.buf[m : uint64Size+1])
+	if err != nil {
+		error(err)
+	}
+}
+
+// encodeInt writes an encoded signed integer to state.w.
+// The low bit of the encoding says whether to bit complement the (other bits of the)
+// uint to recover the int.
+func (state *encoderState) encodeInt(i int64) {
+	var x uint64
+	if i < 0 {
+		x = uint64(^i<<1) | 1
+	} else {
+		x = uint64(i << 1)
+	}
+	state.encodeUint(uint64(x))
+}
+
+// encOp is the signature of an encoding operator for a given type.
+type encOp func(i *encInstr, state *encoderState, p unsafe.Pointer)
+
+// The 'instructions' of the encoding machine
+type encInstr struct {
+	op     encOp
+	field  int     // field number
+	indir  int     // how many pointer indirections to reach the value in the struct
+	offset uintptr // offset in the structure of the field to encode
+}
+
+// update emits a field number and updates the state to record its value for delta encoding.
+// If the instruction pointer is nil, it does nothing
+func (state *encoderState) update(instr *encInstr) {
+	if instr != nil {
+		state.encodeUint(uint64(instr.field - state.fieldnum))
+		state.fieldnum = instr.field
+	}
+}
+
+// Each encoder for a composite is responsible for handling any
+// indirections associated with the elements of the data structure.
+// If any pointer so reached is nil, no bytes are written.  If the
+// data item is zero, no bytes are written.  Single values - ints,
+// strings etc. - are indirected before calling their encoders.
+// Otherwise, the output (for a scalar) is the field number, as an
+// encoded integer, followed by the field data in its appropriate
+// format.
+
+// encIndirect dereferences p indir times and returns the result.
+func encIndirect(p unsafe.Pointer, indir int) unsafe.Pointer {
+	for ; indir > 0; indir-- {
+		p = *(*unsafe.Pointer)(p)
+		if p == nil {
+			return unsafe.Pointer(nil)
+		}
+	}
+	return p
+}
+
+// encBool encodes the bool with address p as an unsigned 0 or 1.
+func encBool(i *encInstr, state *encoderState, p unsafe.Pointer) {
+	b := *(*bool)(p)
+	if b || state.sendZero {
+		state.update(i)
+		if b {
+			state.encodeUint(1)
+		} else {
+			state.encodeUint(0)
+		}
+	}
+}
+
+// encInt encodes the int with address p.
+func encInt(i *encInstr, state *encoderState, p unsafe.Pointer) {
+	v := int64(*(*int)(p))
+	if v != 0 || state.sendZero {
+		state.update(i)
+		state.encodeInt(v)
+	}
+}
+
+// encUint encodes the uint with address p.
+func encUint(i *encInstr, state *encoderState, p unsafe.Pointer) {
+	v := uint64(*(*uint)(p))
+	if v != 0 || state.sendZero {
+		state.update(i)
+		state.encodeUint(v)
+	}
+}
+
+// encInt8 encodes the int8 with address p.
+func encInt8(i *encInstr, state *encoderState, p unsafe.Pointer) {
+	v := int64(*(*int8)(p))
+	if v != 0 || state.sendZero {
+		state.update(i)
+		state.encodeInt(v)
+	}
+}
+
+// encUint8 encodes the uint8 with address p.
+func encUint8(i *encInstr, state *encoderState, p unsafe.Pointer) {
+	v := uint64(*(*uint8)(p))
+	if v != 0 || state.sendZero {
+		state.update(i)
+		state.encodeUint(v)
+	}
+}
+
+// encInt16 encodes the int16 with address p.
+func encInt16(i *encInstr, state *encoderState, p unsafe.Pointer) {
+	v := int64(*(*int16)(p))
+	if v != 0 || state.sendZero {
+		state.update(i)
+		state.encodeInt(v)
+	}
+}
+
+// encUint16 encodes the uint16 with address p.
+func encUint16(i *encInstr, state *encoderState, p unsafe.Pointer) {
+	v := uint64(*(*uint16)(p))
+	if v != 0 || state.sendZero {
+		state.update(i)
+		state.encodeUint(v)
+	}
+}
+
+// encInt32 encodes the int32 with address p.
+func encInt32(i *encInstr, state *encoderState, p unsafe.Pointer) {
+	v := int64(*(*int32)(p))
+	if v != 0 || state.sendZero {
+		state.update(i)
+		state.encodeInt(v)
+	}
+}
+
+// encUint encodes the uint32 with address p.
+func encUint32(i *encInstr, state *encoderState, p unsafe.Pointer) {
+	v := uint64(*(*uint32)(p))
+	if v != 0 || state.sendZero {
+		state.update(i)
+		state.encodeUint(v)
+	}
+}
+
+// encInt64 encodes the int64 with address p.
+func encInt64(i *encInstr, state *encoderState, p unsafe.Pointer) {
+	v := *(*int64)(p)
+	if v != 0 || state.sendZero {
+		state.update(i)
+		state.encodeInt(v)
+	}
+}
+
+// encInt64 encodes the uint64 with address p.
+func encUint64(i *encInstr, state *encoderState, p unsafe.Pointer) {
+	v := *(*uint64)(p)
+	if v != 0 || state.sendZero {
+		state.update(i)
+		state.encodeUint(v)
+	}
+}
+
+// encUintptr encodes the uintptr with address p.
+func encUintptr(i *encInstr, state *encoderState, p unsafe.Pointer) {
+	v := uint64(*(*uintptr)(p))
+	if v != 0 || state.sendZero {
+		state.update(i)
+		state.encodeUint(v)
+	}
+}
+
+// floatBits returns a uint64 holding the bits of a floating-point number.
+// Floating-point numbers are transmitted as uint64s holding the bits
+// of the underlying representation.  They are sent byte-reversed, with
+// the exponent end coming out first, so integer floating point numbers
+// (for example) transmit more compactly.  This routine does the
+// swizzling.
+func floatBits(f float64) uint64 {
+	u := math.Float64bits(f)
+	var v uint64
+	for i := 0; i < 8; i++ {
+		v <<= 8
+		v |= u & 0xFF
+		u >>= 8
+	}
+	return v
+}
+
+// encFloat32 encodes the float32 with address p.
+func encFloat32(i *encInstr, state *encoderState, p unsafe.Pointer) {
+	f := *(*float32)(p)
+	if f != 0 || state.sendZero {
+		v := floatBits(float64(f))
+		state.update(i)
+		state.encodeUint(v)
+	}
+}
+
+// encFloat64 encodes the float64 with address p.
+func encFloat64(i *encInstr, state *encoderState, p unsafe.Pointer) {
+	f := *(*float64)(p)
+	if f != 0 || state.sendZero {
+		state.update(i)
+		v := floatBits(f)
+		state.encodeUint(v)
+	}
+}
+
+// encComplex64 encodes the complex64 with address p.
+// Complex numbers are just a pair of floating-point numbers, real part first.
+func encComplex64(i *encInstr, state *encoderState, p unsafe.Pointer) {
+	c := *(*complex64)(p)
+	if c != 0+0i || state.sendZero {
+		rpart := floatBits(float64(real(c)))
+		ipart := floatBits(float64(imag(c)))
+		state.update(i)
+		state.encodeUint(rpart)
+		state.encodeUint(ipart)
+	}
+}
+
+// encComplex128 encodes the complex128 with address p.
+func encComplex128(i *encInstr, state *encoderState, p unsafe.Pointer) {
+	c := *(*complex128)(p)
+	if c != 0+0i || state.sendZero {
+		rpart := floatBits(real(c))
+		ipart := floatBits(imag(c))
+		state.update(i)
+		state.encodeUint(rpart)
+		state.encodeUint(ipart)
+	}
+}
+
+// encUint8Array encodes the byte slice whose header has address p.
+// Byte arrays are encoded as an unsigned count followed by the raw bytes.
+func encUint8Array(i *encInstr, state *encoderState, p unsafe.Pointer) {
+	b := *(*[]byte)(p)
+	if len(b) > 0 || state.sendZero {
+		state.update(i)
+		state.encodeUint(uint64(len(b)))
+		state.b.Write(b)
+	}
+}
+
+// encString encodes the string whose header has address p.
+// Strings are encoded as an unsigned count followed by the raw bytes.
+func encString(i *encInstr, state *encoderState, p unsafe.Pointer) {
+	s := *(*string)(p)
+	if len(s) > 0 || state.sendZero {
+		state.update(i)
+		state.encodeUint(uint64(len(s)))
+		state.b.WriteString(s)
+	}
+}
+
+// encStructTerminator encodes the end of an encoded struct
+// as delta field number of 0.
+func encStructTerminator(i *encInstr, state *encoderState, p unsafe.Pointer) {
+	state.encodeUint(0)
+}
+
+// Execution engine
+
+// encEngine an array of instructions indexed by field number of the encoding
+// data, typically a struct.  It is executed top to bottom, walking the struct.
+type encEngine struct {
+	instr []encInstr
+}
+
+const singletonField = 0
+
+// encodeSingle encodes a single top-level non-struct value.
+func (enc *Encoder) encodeSingle(b *bytes.Buffer, engine *encEngine, basep uintptr) {
+	state := enc.newEncoderState(b)
+	state.fieldnum = singletonField
+	// There is no surrounding struct to frame the transmission, so we must
+	// generate data even if the item is zero.  To do this, set sendZero.
+	state.sendZero = true
+	instr := &engine.instr[singletonField]
+	p := unsafe.Pointer(basep) // offset will be zero
+	if instr.indir > 0 {
+		if p = encIndirect(p, instr.indir); p == nil {
+			return
+		}
+	}
+	instr.op(instr, state, p)
+	enc.freeEncoderState(state)
+}
+
+// encodeStruct encodes a single struct value.
+func (enc *Encoder) encodeStruct(b *bytes.Buffer, engine *encEngine, basep uintptr) {
+	state := enc.newEncoderState(b)
+	state.fieldnum = -1
+	for i := 0; i < len(engine.instr); i++ {
+		instr := &engine.instr[i]
+		p := unsafe.Pointer(basep + instr.offset)
+		if instr.indir > 0 {
+			if p = encIndirect(p, instr.indir); p == nil {
+				continue
+			}
+		}
+		instr.op(instr, state, p)
+	}
+	enc.freeEncoderState(state)
+}
+
+// encodeArray encodes the array whose 0th element is at p.
+func (enc *Encoder) encodeArray(b *bytes.Buffer, p uintptr, op encOp, elemWid uintptr, elemIndir int, length int) {
+	state := enc.newEncoderState(b)
+	state.fieldnum = -1
+	state.sendZero = true
+	state.encodeUint(uint64(length))
+	for i := 0; i < length; i++ {
+		elemp := p
+		up := unsafe.Pointer(elemp)
+		if elemIndir > 0 {
+			if up = encIndirect(up, elemIndir); up == nil {
+				errorf("encodeArray: nil element")
+			}
+			elemp = uintptr(up)
+		}
+		op(nil, state, unsafe.Pointer(elemp))
+		p += uintptr(elemWid)
+	}
+	enc.freeEncoderState(state)
+}
+
+// encodeReflectValue is a helper for maps. It encodes the value v.
+func encodeReflectValue(state *encoderState, v reflect.Value, op encOp, indir int) {
+	for i := 0; i < indir && v.IsValid(); i++ {
+		v = reflect.Indirect(v)
+	}
+	if !v.IsValid() {
+		errorf("encodeReflectValue: nil element")
+	}
+	op(nil, state, unsafe.Pointer(unsafeAddr(v)))
+}
+
+// encodeMap encodes a map as unsigned count followed by key:value pairs.
+// Because map internals are not exposed, we must use reflection rather than
+// addresses.
+func (enc *Encoder) encodeMap(b *bytes.Buffer, mv reflect.Value, keyOp, elemOp encOp, keyIndir, elemIndir int) {
+	state := enc.newEncoderState(b)
+	state.fieldnum = -1
+	state.sendZero = true
+	keys := mv.MapKeys()
+	state.encodeUint(uint64(len(keys)))
+	for _, key := range keys {
+		encodeReflectValue(state, key, keyOp, keyIndir)
+		encodeReflectValue(state, mv.MapIndex(key), elemOp, elemIndir)
+	}
+	enc.freeEncoderState(state)
+}
+
+// encodeInterface encodes the interface value iv.
+// To send an interface, we send a string identifying the concrete type, followed
+// by the type identifier (which might require defining that type right now), followed
+// by the concrete value.  A nil value gets sent as the empty string for the name,
+// followed by no value.
+func (enc *Encoder) encodeInterface(b *bytes.Buffer, iv reflect.Value) {
+	state := enc.newEncoderState(b)
+	state.fieldnum = -1
+	state.sendZero = true
+	if iv.IsNil() {
+		state.encodeUint(0)
+		return
+	}
+
+	ut := userType(iv.Elem().Type())
+	name, ok := concreteTypeToName[ut.base]
+	if !ok {
+		errorf("type not registered for interface: %s", ut.base)
+	}
+	// Send the name.
+	state.encodeUint(uint64(len(name)))
+	_, err := state.b.WriteString(name)
+	if err != nil {
+		error(err)
+	}
+	// Define the type id if necessary.
+	enc.sendTypeDescriptor(enc.writer(), state, ut)
+	// Send the type id.
+	enc.sendTypeId(state, ut)
+	// Encode the value into a new buffer.  Any nested type definitions
+	// should be written to b, before the encoded value.
+	enc.pushWriter(b)
+	data := new(bytes.Buffer)
+	enc.encode(data, iv.Elem(), ut)
+	if enc.err != nil {
+		error(enc.err)
+	}
+	enc.popWriter()
+	enc.writeMessage(b, data)
+	if enc.err != nil {
+		error(err)
+	}
+	enc.freeEncoderState(state)
+}
+
+// isZero returns whether the value is the zero of its type.
+func isZero(val reflect.Value) bool {
+	switch val.Kind() {
+	case reflect.Array, reflect.Map, reflect.Slice, reflect.String:
+		return val.Len() == 0
+	case reflect.Bool:
+		return !val.Bool()
+	case reflect.Complex64, reflect.Complex128:
+		return val.Complex() == 0
+	case reflect.Chan, reflect.Func, reflect.Ptr:
+		return val.IsNil()
+	case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64:
+		return val.Int() == 0
+	case reflect.Float32, reflect.Float64:
+		return val.Float() == 0
+	case reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64, reflect.Uintptr:
+		return val.Uint() == 0
+	}
+	panic("unknown type in isZero " + val.Type().String())
+}
+
+// encGobEncoder encodes a value that implements the GobEncoder interface.
+// The data is sent as a byte array.
+func (enc *Encoder) encodeGobEncoder(b *bytes.Buffer, v reflect.Value) {
+	// TODO: should we catch panics from the called method?
+	// We know it's a GobEncoder, so just call the method directly.
+	data, err := v.Interface().(GobEncoder).GobEncode()
+	if err != nil {
+		error(err)
+	}
+	state := enc.newEncoderState(b)
+	state.fieldnum = -1
+	state.encodeUint(uint64(len(data)))
+	state.b.Write(data)
+	enc.freeEncoderState(state)
+}
+
+var encOpTable = [...]encOp{
+	reflect.Bool:       encBool,
+	reflect.Int:        encInt,
+	reflect.Int8:       encInt8,
+	reflect.Int16:      encInt16,
+	reflect.Int32:      encInt32,
+	reflect.Int64:      encInt64,
+	reflect.Uint:       encUint,
+	reflect.Uint8:      encUint8,
+	reflect.Uint16:     encUint16,
+	reflect.Uint32:     encUint32,
+	reflect.Uint64:     encUint64,
+	reflect.Uintptr:    encUintptr,
+	reflect.Float32:    encFloat32,
+	reflect.Float64:    encFloat64,
+	reflect.Complex64:  encComplex64,
+	reflect.Complex128: encComplex128,
+	reflect.String:     encString,
+}
+
+// encOpFor returns (a pointer to) the encoding op for the base type under rt and
+// the indirection count to reach it.
+func (enc *Encoder) encOpFor(rt reflect.Type, inProgress map[reflect.Type]*encOp) (*encOp, int) {
+	ut := userType(rt)
+	// If the type implements GobEncoder, we handle it without further processing.
+	if ut.isGobEncoder {
+		return enc.gobEncodeOpFor(ut)
+	}
+	// If this type is already in progress, it's a recursive type (e.g. map[string]*T).
+	// Return the pointer to the op we're already building.
+	if opPtr := inProgress[rt]; opPtr != nil {
+		return opPtr, ut.indir
+	}
+	typ := ut.base
+	indir := ut.indir
+	k := typ.Kind()
+	var op encOp
+	if int(k) < len(encOpTable) {
+		op = encOpTable[k]
+	}
+	if op == nil {
+		inProgress[rt] = &op
+		// Special cases
+		switch t := typ; t.Kind() {
+		case reflect.Slice:
+			if t.Elem().Kind() == reflect.Uint8 {
+				op = encUint8Array
+				break
+			}
+			// Slices have a header; we decode it to find the underlying array.
+			elemOp, indir := enc.encOpFor(t.Elem(), inProgress)
+			op = func(i *encInstr, state *encoderState, p unsafe.Pointer) {
+				slice := (*reflect.SliceHeader)(p)
+				if !state.sendZero && slice.Len == 0 {
+					return
+				}
+				state.update(i)
+				state.enc.encodeArray(state.b, slice.Data, *elemOp, t.Elem().Size(), indir, int(slice.Len))
+			}
+		case reflect.Array:
+			// True arrays have size in the type.
+			elemOp, indir := enc.encOpFor(t.Elem(), inProgress)
+			op = func(i *encInstr, state *encoderState, p unsafe.Pointer) {
+				state.update(i)
+				state.enc.encodeArray(state.b, uintptr(p), *elemOp, t.Elem().Size(), indir, t.Len())
+			}
+		case reflect.Map:
+			keyOp, keyIndir := enc.encOpFor(t.Key(), inProgress)
+			elemOp, elemIndir := enc.encOpFor(t.Elem(), inProgress)
+			op = func(i *encInstr, state *encoderState, p unsafe.Pointer) {
+				// Maps cannot be accessed by moving addresses around the way
+				// that slices etc. can.  We must recover a full reflection value for
+				// the iteration.
+				v := reflect.ValueOf(unsafe.Unreflect(t, unsafe.Pointer(p)))
+				mv := reflect.Indirect(v)
+				// We send zero-length (but non-nil) maps because the
+				// receiver might want to use the map.  (Maps don't use append.)
+				if !state.sendZero && mv.IsNil() {
+					return
+				}
+				state.update(i)
+				state.enc.encodeMap(state.b, mv, *keyOp, *elemOp, keyIndir, elemIndir)
+			}
+		case reflect.Struct:
+			// Generate a closure that calls out to the engine for the nested type.
+			enc.getEncEngine(userType(typ))
+			info := mustGetTypeInfo(typ)
+			op = func(i *encInstr, state *encoderState, p unsafe.Pointer) {
+				state.update(i)
+				// indirect through info to delay evaluation for recursive structs
+				state.enc.encodeStruct(state.b, info.encoder, uintptr(p))
+			}
+		case reflect.Interface:
+			op = func(i *encInstr, state *encoderState, p unsafe.Pointer) {
+				// Interfaces transmit the name and contents of the concrete
+				// value they contain.
+				v := reflect.ValueOf(unsafe.Unreflect(t, unsafe.Pointer(p)))
+				iv := reflect.Indirect(v)
+				if !state.sendZero && (!iv.IsValid() || iv.IsNil()) {
+					return
+				}
+				state.update(i)
+				state.enc.encodeInterface(state.b, iv)
+			}
+		}
+	}
+	if op == nil {
+		errorf("can't happen: encode type %s", rt.String())
+	}
+	return &op, indir
+}
+
+// gobEncodeOpFor returns the op for a type that is known to implement
+// GobEncoder.
+func (enc *Encoder) gobEncodeOpFor(ut *userTypeInfo) (*encOp, int) {
+	rt := ut.user
+	if ut.encIndir == -1 {
+		rt = reflect.PtrTo(rt)
+	} else if ut.encIndir > 0 {
+		for i := int8(0); i < ut.encIndir; i++ {
+			rt = rt.Elem()
+		}
+	}
+	var op encOp
+	op = func(i *encInstr, state *encoderState, p unsafe.Pointer) {
+		var v reflect.Value
+		if ut.encIndir == -1 {
+			// Need to climb up one level to turn value into pointer.
+			v = reflect.ValueOf(unsafe.Unreflect(rt, unsafe.Pointer(&p)))
+		} else {
+			v = reflect.ValueOf(unsafe.Unreflect(rt, p))
+		}
+		if !state.sendZero && isZero(v) {
+			return
+		}
+		state.update(i)
+		state.enc.encodeGobEncoder(state.b, v)
+	}
+	return &op, int(ut.encIndir) // encIndir: op will get called with p == address of receiver.
+}
+
+// compileEnc returns the engine to compile the type.
+func (enc *Encoder) compileEnc(ut *userTypeInfo) *encEngine {
+	srt := ut.base
+	engine := new(encEngine)
+	seen := make(map[reflect.Type]*encOp)
+	rt := ut.base
+	if ut.isGobEncoder {
+		rt = ut.user
+	}
+	if !ut.isGobEncoder &&
+		srt.Kind() == reflect.Struct {
+		for fieldNum, wireFieldNum := 0, 0; fieldNum < srt.NumField(); fieldNum++ {
+			f := srt.Field(fieldNum)
+			if !isExported(f.Name) {
+				continue
+			}
+			op, indir := enc.encOpFor(f.Type, seen)
+			engine.instr = append(engine.instr, encInstr{*op, wireFieldNum, indir, uintptr(f.Offset)})
+			wireFieldNum++
+		}
+		if srt.NumField() > 0 && len(engine.instr) == 0 {
+			errorf("type %s has no exported fields", rt)
+		}
+		engine.instr = append(engine.instr, encInstr{encStructTerminator, 0, 0, 0})
+	} else {
+		engine.instr = make([]encInstr, 1)
+		op, indir := enc.encOpFor(rt, seen)
+		engine.instr[0] = encInstr{*op, singletonField, indir, 0} // offset is zero
+	}
+	return engine
+}
+
+// getEncEngine returns the engine to compile the type.
+// typeLock must be held (or we're in initialization and guaranteed single-threaded).
+func (enc *Encoder) getEncEngine(ut *userTypeInfo) *encEngine {
+	info, err1 := getTypeInfo(ut)
+	if err1 != nil {
+		error(err1)
+	}
+	if info.encoder == nil {
+		// mark this engine as underway before compiling to handle recursive types.
+		info.encoder = new(encEngine)
+		info.encoder = enc.compileEnc(ut)
+	}
+	return info.encoder
+}
+
+// lockAndGetEncEngine is a function that locks and compiles.
+// This lets us hold the lock only while compiling, not when encoding.
+func (enc *Encoder) lockAndGetEncEngine(ut *userTypeInfo) *encEngine {
+	typeLock.Lock()
+	defer typeLock.Unlock()
+	return enc.getEncEngine(ut)
+}
+
+func (enc *Encoder) encode(b *bytes.Buffer, value reflect.Value, ut *userTypeInfo) {
+	defer catchError(&enc.err)
+	engine := enc.lockAndGetEncEngine(ut)
+	indir := ut.indir
+	if ut.isGobEncoder {
+		indir = int(ut.encIndir)
+	}
+	for i := 0; i < indir; i++ {
+		value = reflect.Indirect(value)
+	}
+	if !ut.isGobEncoder && value.Type().Kind() == reflect.Struct {
+		enc.encodeStruct(b, engine, unsafeAddr(value))
+	} else {
+		enc.encodeSingle(b, engine, unsafeAddr(value))
+	}
+}
diff --git a/src/pkg/gob/encoder.go b/src/pkg/gob/encoder.go
new file mode 100644
index 000000000..96101d92b
--- /dev/null
+++ b/src/pkg/gob/encoder.go
@@ -0,0 +1,243 @@
+// Copyright 2009 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package gob
+
+import (
+	"bytes"
+	"io"
+	"os"
+	"reflect"
+	"sync"
+)
+
+// An Encoder manages the transmission of type and data information to the
+// other side of a connection.
+type Encoder struct {
+	mutex      sync.Mutex              // each item must be sent atomically
+	w          []io.Writer             // where to send the data
+	sent       map[reflect.Type]typeId // which types we've already sent
+	countState *encoderState           // stage for writing counts
+	freeList   *encoderState           // list of free encoderStates; avoids reallocation
+	buf        []byte                  // for collecting the output.
+	byteBuf    bytes.Buffer            // buffer for top-level encoderState
+	err        os.Error
+}
+
+// NewEncoder returns a new encoder that will transmit on the io.Writer.
+func NewEncoder(w io.Writer) *Encoder {
+	enc := new(Encoder)
+	enc.w = []io.Writer{w}
+	enc.sent = make(map[reflect.Type]typeId)
+	enc.countState = enc.newEncoderState(new(bytes.Buffer))
+	return enc
+}
+
+// writer() returns the innermost writer the encoder is using
+func (enc *Encoder) writer() io.Writer {
+	return enc.w[len(enc.w)-1]
+}
+
+// pushWriter adds a writer to the encoder.
+func (enc *Encoder) pushWriter(w io.Writer) {
+	enc.w = append(enc.w, w)
+}
+
+// popWriter pops the innermost writer.
+func (enc *Encoder) popWriter() {
+	enc.w = enc.w[0 : len(enc.w)-1]
+}
+
+func (enc *Encoder) badType(rt reflect.Type) {
+	enc.setError(os.NewError("gob: can't encode type " + rt.String()))
+}
+
+func (enc *Encoder) setError(err os.Error) {
+	if enc.err == nil { // remember the first.
+		enc.err = err
+	}
+}
+
+// writeMessage sends the data item preceded by a unsigned count of its length.
+func (enc *Encoder) writeMessage(w io.Writer, b *bytes.Buffer) {
+	enc.countState.encodeUint(uint64(b.Len()))
+	// Build the buffer.
+	countLen := enc.countState.b.Len()
+	total := countLen + b.Len()
+	if total > len(enc.buf) {
+		enc.buf = make([]byte, total+1000) // extra for growth
+	}
+	// Place the length before the data.
+	// TODO(r): avoid the extra copy here.
+	enc.countState.b.Read(enc.buf[0:countLen])
+	// Now the data.
+	b.Read(enc.buf[countLen:total])
+	// Write the data.
+	_, err := w.Write(enc.buf[0:total])
+	if err != nil {
+		enc.setError(err)
+	}
+}
+
+// sendActualType sends the requested type, without further investigation, unless
+// it's been sent before.
+func (enc *Encoder) sendActualType(w io.Writer, state *encoderState, ut *userTypeInfo, actual reflect.Type) (sent bool) {
+	if _, alreadySent := enc.sent[actual]; alreadySent {
+		return false
+	}
+	typeLock.Lock()
+	info, err := getTypeInfo(ut)
+	typeLock.Unlock()
+	if err != nil {
+		enc.setError(err)
+		return
+	}
+	// Send the pair (-id, type)
+	// Id:
+	state.encodeInt(-int64(info.id))
+	// Type:
+	enc.encode(state.b, reflect.ValueOf(info.wire), wireTypeUserInfo)
+	enc.writeMessage(w, state.b)
+	if enc.err != nil {
+		return
+	}
+
+	// Remember we've sent this type, both what the user gave us and the base type.
+	enc.sent[ut.base] = info.id
+	if ut.user != ut.base {
+		enc.sent[ut.user] = info.id
+	}
+	// Now send the inner types
+	switch st := actual; st.Kind() {
+	case reflect.Struct:
+		for i := 0; i < st.NumField(); i++ {
+			enc.sendType(w, state, st.Field(i).Type)
+		}
+	case reflect.Array, reflect.Slice:
+		enc.sendType(w, state, st.Elem())
+	case reflect.Map:
+		enc.sendType(w, state, st.Key())
+		enc.sendType(w, state, st.Elem())
+	}
+	return true
+}
+
+// sendType sends the type info to the other side, if necessary. 
+func (enc *Encoder) sendType(w io.Writer, state *encoderState, origt reflect.Type) (sent bool) {
+	ut := userType(origt)
+	if ut.isGobEncoder {
+		// The rules are different: regardless of the underlying type's representation,
+		// we need to tell the other side that this exact type is a GobEncoder.
+		return enc.sendActualType(w, state, ut, ut.user)
+	}
+
+	// It's a concrete value, so drill down to the base type.
+	switch rt := ut.base; rt.Kind() {
+	default:
+		// Basic types and interfaces do not need to be described.
+		return
+	case reflect.Slice:
+		// If it's []uint8, don't send; it's considered basic.
+		if rt.Elem().Kind() == reflect.Uint8 {
+			return
+		}
+		// Otherwise we do send.
+		break
+	case reflect.Array:
+		// arrays must be sent so we know their lengths and element types.
+		break
+	case reflect.Map:
+		// maps must be sent so we know their lengths and key/value types.
+		break
+	case reflect.Struct:
+		// structs must be sent so we know their fields.
+		break
+	case reflect.Chan, reflect.Func:
+		// Probably a bad field in a struct.
+		enc.badType(rt)
+		return
+	}
+
+	return enc.sendActualType(w, state, ut, ut.base)
+}
+
+// Encode transmits the data item represented by the empty interface value,
+// guaranteeing that all necessary type information has been transmitted first.
+func (enc *Encoder) Encode(e interface{}) os.Error {
+	return enc.EncodeValue(reflect.ValueOf(e))
+}
+
+// sendTypeDescriptor makes sure the remote side knows about this type.
+// It will send a descriptor if this is the first time the type has been
+// sent.
+func (enc *Encoder) sendTypeDescriptor(w io.Writer, state *encoderState, ut *userTypeInfo) {
+	// Make sure the type is known to the other side.
+	// First, have we already sent this type?
+	rt := ut.base
+	if ut.isGobEncoder {
+		rt = ut.user
+	}
+	if _, alreadySent := enc.sent[rt]; !alreadySent {
+		// No, so send it.
+		sent := enc.sendType(w, state, rt)
+		if enc.err != nil {
+			return
+		}
+		// If the type info has still not been transmitted, it means we have
+		// a singleton basic type (int, []byte etc.) at top level.  We don't
+		// need to send the type info but we do need to update enc.sent.
+		if !sent {
+			typeLock.Lock()
+			info, err := getTypeInfo(ut)
+			typeLock.Unlock()
+			if err != nil {
+				enc.setError(err)
+				return
+			}
+			enc.sent[rt] = info.id
+		}
+	}
+}
+
+// sendTypeId sends the id, which must have already been defined.
+func (enc *Encoder) sendTypeId(state *encoderState, ut *userTypeInfo) {
+	// Identify the type of this top-level value.
+	state.encodeInt(int64(enc.sent[ut.base]))
+}
+
+// EncodeValue transmits the data item represented by the reflection value,
+// guaranteeing that all necessary type information has been transmitted first.
+func (enc *Encoder) EncodeValue(value reflect.Value) os.Error {
+	// Make sure we're single-threaded through here, so multiple
+	// goroutines can share an encoder.
+	enc.mutex.Lock()
+	defer enc.mutex.Unlock()
+
+	// Remove any nested writers remaining due to previous errors.
+	enc.w = enc.w[0:1]
+
+	ut, err := validUserType(value.Type())
+	if err != nil {
+		return err
+	}
+
+	enc.err = nil
+	enc.byteBuf.Reset()
+	state := enc.newEncoderState(&enc.byteBuf)
+
+	enc.sendTypeDescriptor(enc.writer(), state, ut)
+	enc.sendTypeId(state, ut)
+	if enc.err != nil {
+		return enc.err
+	}
+
+	// Encode the object.
+	enc.encode(state.b, value, ut)
+	if enc.err == nil {
+		enc.writeMessage(enc.writer(), state.b)
+	}
+
+	enc.freeEncoderState(state)
+	return enc.err
+}
diff --git a/src/pkg/gob/encoder_test.go b/src/pkg/gob/encoder_test.go
new file mode 100644
index 000000000..f5ee423cb
--- /dev/null
+++ b/src/pkg/gob/encoder_test.go
@@ -0,0 +1,580 @@
+// Copyright 2009 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package gob
+
+import (
+	"bytes"
+	"fmt"
+	"io"
+	"os"
+	"reflect"
+	"strings"
+	"testing"
+)
+
+type ET2 struct {
+	X string
+}
+
+type ET1 struct {
+	A    int
+	Et2  *ET2
+	Next *ET1
+}
+
+// Like ET1 but with a different name for a field
+type ET3 struct {
+	A             int
+	Et2           *ET2
+	DifferentNext *ET1
+}
+
+// Like ET1 but with a different type for a field
+type ET4 struct {
+	A    int
+	Et2  float64
+	Next int
+}
+
+func TestEncoderDecoder(t *testing.T) {
+	b := new(bytes.Buffer)
+	enc := NewEncoder(b)
+	et1 := new(ET1)
+	et1.A = 7
+	et1.Et2 = new(ET2)
+	err := enc.Encode(et1)
+	if err != nil {
+		t.Error("encoder fail:", err)
+	}
+	dec := NewDecoder(b)
+	newEt1 := new(ET1)
+	err = dec.Decode(newEt1)
+	if err != nil {
+		t.Fatal("error decoding ET1:", err)
+	}
+
+	if !reflect.DeepEqual(et1, newEt1) {
+		t.Fatalf("invalid data for et1: expected %+v; got %+v", *et1, *newEt1)
+	}
+	if b.Len() != 0 {
+		t.Error("not at eof;", b.Len(), "bytes left")
+	}
+
+	enc.Encode(et1)
+	newEt1 = new(ET1)
+	err = dec.Decode(newEt1)
+	if err != nil {
+		t.Fatal("round 2: error decoding ET1:", err)
+	}
+	if !reflect.DeepEqual(et1, newEt1) {
+		t.Fatalf("round 2: invalid data for et1: expected %+v; got %+v", *et1, *newEt1)
+	}
+	if b.Len() != 0 {
+		t.Error("round 2: not at eof;", b.Len(), "bytes left")
+	}
+
+	// Now test with a running encoder/decoder pair that we recognize a type mismatch.
+	err = enc.Encode(et1)
+	if err != nil {
+		t.Error("round 3: encoder fail:", err)
+	}
+	newEt2 := new(ET2)
+	err = dec.Decode(newEt2)
+	if err == nil {
+		t.Fatal("round 3: expected `bad type' error decoding ET2")
+	}
+}
+
+// Run one value through the encoder/decoder, but use the wrong type.
+// Input is always an ET1; we compare it to whatever is under 'e'.
+func badTypeCheck(e interface{}, shouldFail bool, msg string, t *testing.T) {
+	b := new(bytes.Buffer)
+	enc := NewEncoder(b)
+	et1 := new(ET1)
+	et1.A = 7
+	et1.Et2 = new(ET2)
+	err := enc.Encode(et1)
+	if err != nil {
+		t.Error("encoder fail:", err)
+	}
+	dec := NewDecoder(b)
+	err = dec.Decode(e)
+	if shouldFail && err == nil {
+		t.Error("expected error for", msg)
+	}
+	if !shouldFail && err != nil {
+		t.Error("unexpected error for", msg, err)
+	}
+}
+
+// Test that we recognize a bad type the first time.
+func TestWrongTypeDecoder(t *testing.T) {
+	badTypeCheck(new(ET2), true, "no fields in common", t)
+	badTypeCheck(new(ET3), false, "different name of field", t)
+	badTypeCheck(new(ET4), true, "different type of field", t)
+}
+
+func corruptDataCheck(s string, err os.Error, t *testing.T) {
+	b := bytes.NewBufferString(s)
+	dec := NewDecoder(b)
+	err1 := dec.Decode(new(ET2))
+	if err1 != err {
+		t.Errorf("from %q expected error %s; got %s", s, err, err1)
+	}
+}
+
+// Check that we survive bad data.
+func TestBadData(t *testing.T) {
+	corruptDataCheck("", os.EOF, t)
+	corruptDataCheck("\x7Fhi", io.ErrUnexpectedEOF, t)
+	corruptDataCheck("\x03now is the time for all good men", errBadType, t)
+}
+
+// Types not supported by the Encoder.
+var unsupportedValues = []interface{}{
+	make(chan int),
+	func(a int) bool { return true },
+}
+
+func TestUnsupported(t *testing.T) {
+	var b bytes.Buffer
+	enc := NewEncoder(&b)
+	for _, v := range unsupportedValues {
+		err := enc.Encode(v)
+		if err == nil {
+			t.Errorf("expected error for %T; got none", v)
+		}
+	}
+}
+
+func encAndDec(in, out interface{}) os.Error {
+	b := new(bytes.Buffer)
+	enc := NewEncoder(b)
+	err := enc.Encode(in)
+	if err != nil {
+		return err
+	}
+	dec := NewDecoder(b)
+	err = dec.Decode(out)
+	if err != nil {
+		return err
+	}
+	return nil
+}
+
+func TestTypeToPtrType(t *testing.T) {
+	// Encode a T, decode a *T
+	type Type0 struct {
+		A int
+	}
+	t0 := Type0{7}
+	t0p := new(Type0)
+	if err := encAndDec(t0, t0p); err != nil {
+		t.Error(err)
+	}
+}
+
+func TestPtrTypeToType(t *testing.T) {
+	// Encode a *T, decode a T
+	type Type1 struct {
+		A uint
+	}
+	t1p := &Type1{17}
+	var t1 Type1
+	if err := encAndDec(t1, t1p); err != nil {
+		t.Error(err)
+	}
+}
+
+func TestTypeToPtrPtrPtrPtrType(t *testing.T) {
+	type Type2 struct {
+		A ****float64
+	}
+	t2 := Type2{}
+	t2.A = new(***float64)
+	*t2.A = new(**float64)
+	**t2.A = new(*float64)
+	***t2.A = new(float64)
+	****t2.A = 27.4
+	t2pppp := new(***Type2)
+	if err := encAndDec(t2, t2pppp); err != nil {
+		t.Fatal(err)
+	}
+	if ****(****t2pppp).A != ****t2.A {
+		t.Errorf("wrong value after decode: %g not %g", ****(****t2pppp).A, ****t2.A)
+	}
+}
+
+func TestSlice(t *testing.T) {
+	type Type3 struct {
+		A []string
+	}
+	t3p := &Type3{[]string{"hello", "world"}}
+	var t3 Type3
+	if err := encAndDec(t3, t3p); err != nil {
+		t.Error(err)
+	}
+}
+
+func TestValueError(t *testing.T) {
+	// Encode a *T, decode a T
+	type Type4 struct {
+		A int
+	}
+	t4p := &Type4{3}
+	var t4 Type4 // note: not a pointer.
+	if err := encAndDec(t4p, t4); err == nil || strings.Index(err.String(), "pointer") < 0 {
+		t.Error("expected error about pointer; got", err)
+	}
+}
+
+func TestArray(t *testing.T) {
+	type Type5 struct {
+		A [3]string
+		B [3]byte
+	}
+	type Type6 struct {
+		A [2]string // can't hold t5.a
+	}
+	t5 := Type5{[3]string{"hello", ",", "world"}, [3]byte{1, 2, 3}}
+	var t5p Type5
+	if err := encAndDec(t5, &t5p); err != nil {
+		t.Error(err)
+	}
+	var t6 Type6
+	if err := encAndDec(t5, &t6); err == nil {
+		t.Error("should fail with mismatched array sizes")
+	}
+}
+
+func TestRecursiveMapType(t *testing.T) {
+	type recursiveMap map[string]recursiveMap
+	r1 := recursiveMap{"A": recursiveMap{"B": nil, "C": nil}, "D": nil}
+	r2 := make(recursiveMap)
+	if err := encAndDec(r1, &r2); err != nil {
+		t.Error(err)
+	}
+}
+
+func TestRecursiveSliceType(t *testing.T) {
+	type recursiveSlice []recursiveSlice
+	r1 := recursiveSlice{0: recursiveSlice{0: nil}, 1: nil}
+	r2 := make(recursiveSlice, 0)
+	if err := encAndDec(r1, &r2); err != nil {
+		t.Error(err)
+	}
+}
+
+// Regression test for bug: must send zero values inside arrays
+func TestDefaultsInArray(t *testing.T) {
+	type Type7 struct {
+		B []bool
+		I []int
+		S []string
+		F []float64
+	}
+	t7 := Type7{
+		[]bool{false, false, true},
+		[]int{0, 0, 1},
+		[]string{"hi", "", "there"},
+		[]float64{0, 0, 1},
+	}
+	var t7p Type7
+	if err := encAndDec(t7, &t7p); err != nil {
+		t.Error(err)
+	}
+}
+
+var testInt int
+var testFloat32 float32
+var testString string
+var testSlice []string
+var testMap map[string]int
+var testArray [7]int
+
+type SingleTest struct {
+	in  interface{}
+	out interface{}
+	err string
+}
+
+var singleTests = []SingleTest{
+	{17, &testInt, ""},
+	{float32(17.5), &testFloat32, ""},
+	{"bike shed", &testString, ""},
+	{[]string{"bike", "shed", "paint", "color"}, &testSlice, ""},
+	{map[string]int{"seven": 7, "twelve": 12}, &testMap, ""},
+	{[7]int{4, 55, 0, 0, 0, 0, 0}, &testArray, ""}, // case that once triggered a bug
+	{[7]int{4, 55, 1, 44, 22, 66, 1234}, &testArray, ""},
+
+	// Decode errors
+	{172, &testFloat32, "wrong type"},
+}
+
+func TestSingletons(t *testing.T) {
+	b := new(bytes.Buffer)
+	enc := NewEncoder(b)
+	dec := NewDecoder(b)
+	for _, test := range singleTests {
+		b.Reset()
+		err := enc.Encode(test.in)
+		if err != nil {
+			t.Errorf("error encoding %v: %s", test.in, err)
+			continue
+		}
+		err = dec.Decode(test.out)
+		switch {
+		case err != nil && test.err == "":
+			t.Errorf("error decoding %v: %s", test.in, err)
+			continue
+		case err == nil && test.err != "":
+			t.Errorf("expected error decoding %v: %s", test.in, test.err)
+			continue
+		case err != nil && test.err != "":
+			if strings.Index(err.String(), test.err) < 0 {
+				t.Errorf("wrong error decoding %v: wanted %s, got %v", test.in, test.err, err)
+			}
+			continue
+		}
+		// Get rid of the pointer in the rhs
+		val := reflect.ValueOf(test.out).Elem().Interface()
+		if !reflect.DeepEqual(test.in, val) {
+			t.Errorf("decoding singleton: expected %v got %v", test.in, val)
+		}
+	}
+}
+
+func TestStructNonStruct(t *testing.T) {
+	type Struct struct {
+		A string
+	}
+	type NonStruct string
+	s := Struct{"hello"}
+	var sp Struct
+	if err := encAndDec(s, &sp); err != nil {
+		t.Error(err)
+	}
+	var ns NonStruct
+	if err := encAndDec(s, &ns); err == nil {
+		t.Error("should get error for struct/non-struct")
+	} else if strings.Index(err.String(), "type") < 0 {
+		t.Error("for struct/non-struct expected type error; got", err)
+	}
+	// Now try the other way
+	var nsp NonStruct
+	if err := encAndDec(ns, &nsp); err != nil {
+		t.Error(err)
+	}
+	if err := encAndDec(ns, &s); err == nil {
+		t.Error("should get error for non-struct/struct")
+	} else if strings.Index(err.String(), "type") < 0 {
+		t.Error("for non-struct/struct expected type error; got", err)
+	}
+}
+
+type interfaceIndirectTestI interface {
+	F() bool
+}
+
+type interfaceIndirectTestT struct{}
+
+func (this *interfaceIndirectTestT) F() bool {
+	return true
+}
+
+// A version of a bug reported on golang-nuts.  Also tests top-level
+// slice of interfaces.  The issue was registering *T caused T to be
+// stored as the concrete type.
+func TestInterfaceIndirect(t *testing.T) {
+	Register(&interfaceIndirectTestT{})
+	b := new(bytes.Buffer)
+	w := []interfaceIndirectTestI{&interfaceIndirectTestT{}}
+	err := NewEncoder(b).Encode(w)
+	if err != nil {
+		t.Fatal("encode error:", err)
+	}
+
+	var r []interfaceIndirectTestI
+	err = NewDecoder(b).Decode(&r)
+	if err != nil {
+		t.Fatal("decode error:", err)
+	}
+}
+
+// Now follow various tests that decode into things that can't represent the
+// encoded value, all of which should be legal.
+
+// Also, when the ignored object contains an interface value, it may define
+// types. Make sure that skipping the value still defines the types by using
+// the encoder/decoder pair to send a value afterwards.  If an interface
+// is sent, its type in the test is always NewType0, so this checks that the
+// encoder and decoder don't skew with respect to type definitions.
+
+type Struct0 struct {
+	I interface{}
+}
+
+type NewType0 struct {
+	S string
+}
+
+type ignoreTest struct {
+	in, out interface{}
+}
+
+var ignoreTests = []ignoreTest{
+	// Decode normal struct into an empty struct
+	{&struct{ A int }{23}, &struct{}{}},
+	// Decode normal struct into a nil.
+	{&struct{ A int }{23}, nil},
+	// Decode singleton string into a nil.
+	{"hello, world", nil},
+	// Decode singleton slice into a nil.
+	{[]int{1, 2, 3, 4}, nil},
+	// Decode struct containing an interface into a nil.
+	{&Struct0{&NewType0{"value0"}}, nil},
+	// Decode singleton slice of interfaces into a nil.
+	{[]interface{}{"hi", &NewType0{"value1"}, 23}, nil},
+}
+
+func TestDecodeIntoNothing(t *testing.T) {
+	Register(new(NewType0))
+	for i, test := range ignoreTests {
+		b := new(bytes.Buffer)
+		enc := NewEncoder(b)
+		err := enc.Encode(test.in)
+		if err != nil {
+			t.Errorf("%d: encode error %s:", i, err)
+			continue
+		}
+		dec := NewDecoder(b)
+		err = dec.Decode(test.out)
+		if err != nil {
+			t.Errorf("%d: decode error: %s", i, err)
+			continue
+		}
+		// Now see if the encoder and decoder are in a consistent state.
+		str := fmt.Sprintf("Value %d", i)
+		err = enc.Encode(&NewType0{str})
+		if err != nil {
+			t.Fatalf("%d: NewType0 encode error: %s", i, err)
+		}
+		ns := new(NewType0)
+		err = dec.Decode(ns)
+		if err != nil {
+			t.Fatalf("%d: NewType0 decode error: %s", i, err)
+		}
+		if ns.S != str {
+			t.Fatalf("%d: expected %q got %q", i, str, ns.S)
+		}
+	}
+}
+
+// Another bug from golang-nuts, involving nested interfaces.
+type Bug0Outer struct {
+	Bug0Field interface{}
+}
+
+type Bug0Inner struct {
+	A int
+}
+
+func TestNestedInterfaces(t *testing.T) {
+	var buf bytes.Buffer
+	e := NewEncoder(&buf)
+	d := NewDecoder(&buf)
+	Register(new(Bug0Outer))
+	Register(new(Bug0Inner))
+	f := &Bug0Outer{&Bug0Outer{&Bug0Inner{7}}}
+	var v interface{} = f
+	err := e.Encode(&v)
+	if err != nil {
+		t.Fatal("Encode:", err)
+	}
+	err = d.Decode(&v)
+	if err != nil {
+		t.Fatal("Decode:", err)
+	}
+	// Make sure it decoded correctly.
+	outer1, ok := v.(*Bug0Outer)
+	if !ok {
+		t.Fatalf("v not Bug0Outer: %T", v)
+	}
+	outer2, ok := outer1.Bug0Field.(*Bug0Outer)
+	if !ok {
+		t.Fatalf("v.Bug0Field not Bug0Outer: %T", outer1.Bug0Field)
+	}
+	inner, ok := outer2.Bug0Field.(*Bug0Inner)
+	if !ok {
+		t.Fatalf("v.Bug0Field.Bug0Field not Bug0Inner: %T", outer2.Bug0Field)
+	}
+	if inner.A != 7 {
+		t.Fatalf("final value %d; expected %d", inner.A, 7)
+	}
+}
+
+// The bugs keep coming. We forgot to send map subtypes before the map.
+
+type Bug1Elem struct {
+	Name string
+	Id   int
+}
+
+type Bug1StructMap map[string]Bug1Elem
+
+func bug1EncDec(in Bug1StructMap, out *Bug1StructMap) os.Error {
+	return nil
+}
+
+func TestMapBug1(t *testing.T) {
+	in := make(Bug1StructMap)
+	in["val1"] = Bug1Elem{"elem1", 1}
+	in["val2"] = Bug1Elem{"elem2", 2}
+
+	b := new(bytes.Buffer)
+	enc := NewEncoder(b)
+	err := enc.Encode(in)
+	if err != nil {
+		t.Fatal("encode:", err)
+	}
+	dec := NewDecoder(b)
+	out := make(Bug1StructMap)
+	err = dec.Decode(&out)
+	if err != nil {
+		t.Fatal("decode:", err)
+	}
+	if !reflect.DeepEqual(in, out) {
+		t.Errorf("mismatch: %v %v", in, out)
+	}
+}
+
+func TestGobMapInterfaceEncode(t *testing.T) {
+	m := map[string]interface{}{
+		"up": uintptr(0),
+		"i0": []int{-1},
+		"i1": []int8{-1},
+		"i2": []int16{-1},
+		"i3": []int32{-1},
+		"i4": []int64{-1},
+		"u0": []uint{1},
+		"u1": []uint8{1},
+		"u2": []uint16{1},
+		"u3": []uint32{1},
+		"u4": []uint64{1},
+		"f0": []float32{1},
+		"f1": []float64{1},
+		"c0": []complex64{complex(2, -2)},
+		"c1": []complex128{complex(2, float64(-2))},
+		"us": []uintptr{0},
+		"bo": []bool{false},
+		"st": []string{"s"},
+	}
+	buf := bytes.NewBuffer(nil)
+	enc := NewEncoder(buf)
+	err := enc.Encode(m)
+	if err != nil {
+		t.Errorf("gob.Encode map: %s", err)
+	}
+}
diff --git a/src/pkg/gob/error.go b/src/pkg/gob/error.go
new file mode 100644
index 000000000..bfd38fc16
--- /dev/null
+++ b/src/pkg/gob/error.go
@@ -0,0 +1,42 @@
+// Copyright 2009 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package gob
+
+import (
+	"fmt"
+	"os"
+)
+
+// Errors in decoding and encoding are handled using panic and recover.
+// Panics caused by user error (that is, everything except run-time panics
+// such as "index out of bounds" errors) do not leave the file that caused
+// them, but are instead turned into plain os.Error returns.  Encoding and
+// decoding functions and methods that do not return an os.Error either use
+// panic to report an error or are guaranteed error-free.
+
+// A gobError wraps an os.Error and is used to distinguish errors (panics) generated in this package.
+type gobError struct {
+	os.Error
+}
+
+// errorf is like error but takes Printf-style arguments to construct an os.Error.
+// It always prefixes the message with "gob: ".
+func errorf(format string, args ...interface{}) {
+	error(fmt.Errorf("gob: "+format, args...))
+}
+
+// error wraps the argument error and uses it as the argument to panic.
+func error(err os.Error) {
+	panic(gobError{Error: err})
+}
+
+// catchError is meant to be used as a deferred function to turn a panic(gobError) into a
+// plain os.Error.  It overwrites the error return of the function that deferred its call.
+func catchError(err *os.Error) {
+	if e := recover(); e != nil {
+		*err = e.(gobError).Error // Will re-panic if not one of our errors, such as a runtime error.
+	}
+	return
+}
diff --git a/src/pkg/gob/gobencdec_test.go b/src/pkg/gob/gobencdec_test.go
new file mode 100644
index 000000000..371a43c8f
--- /dev/null
+++ b/src/pkg/gob/gobencdec_test.go
@@ -0,0 +1,490 @@
+// Copyright 20011 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// This file contains tests of the GobEncoder/GobDecoder support.
+
+package gob
+
+import (
+	"bytes"
+	"fmt"
+	"os"
+	"strings"
+	"testing"
+)
+
+// Types that implement the GobEncoder/Decoder interfaces.
+
+type ByteStruct struct {
+	a byte // not an exported field
+}
+
+type StringStruct struct {
+	s string // not an exported field
+}
+
+type ArrayStruct struct {
+	a [8192]byte // not an exported field
+}
+
+type Gobber int
+
+type ValueGobber string // encodes with a value, decodes with a pointer.
+
+// The relevant methods
+
+func (g *ByteStruct) GobEncode() ([]byte, os.Error) {
+	b := make([]byte, 3)
+	b[0] = g.a
+	b[1] = g.a + 1
+	b[2] = g.a + 2
+	return b, nil
+}
+
+func (g *ByteStruct) GobDecode(data []byte) os.Error {
+	if g == nil {
+		return os.NewError("NIL RECEIVER")
+	}
+	// Expect N sequential-valued bytes.
+	if len(data) == 0 {
+		return os.EOF
+	}
+	g.a = data[0]
+	for i, c := range data {
+		if c != g.a+byte(i) {
+			return os.NewError("invalid data sequence")
+		}
+	}
+	return nil
+}
+
+func (g *StringStruct) GobEncode() ([]byte, os.Error) {
+	return []byte(g.s), nil
+}
+
+func (g *StringStruct) GobDecode(data []byte) os.Error {
+	// Expect N sequential-valued bytes.
+	if len(data) == 0 {
+		return os.EOF
+	}
+	a := data[0]
+	for i, c := range data {
+		if c != a+byte(i) {
+			return os.NewError("invalid data sequence")
+		}
+	}
+	g.s = string(data)
+	return nil
+}
+
+func (a *ArrayStruct) GobEncode() ([]byte, os.Error) {
+	return a.a[:], nil
+}
+
+func (a *ArrayStruct) GobDecode(data []byte) os.Error {
+	if len(data) != len(a.a) {
+		return os.NewError("wrong length in array decode")
+	}
+	copy(a.a[:], data)
+	return nil
+}
+
+func (g *Gobber) GobEncode() ([]byte, os.Error) {
+	return []byte(fmt.Sprintf("VALUE=%d", *g)), nil
+}
+
+func (g *Gobber) GobDecode(data []byte) os.Error {
+	_, err := fmt.Sscanf(string(data), "VALUE=%d", (*int)(g))
+	return err
+}
+
+func (v ValueGobber) GobEncode() ([]byte, os.Error) {
+	return []byte(fmt.Sprintf("VALUE=%s", v)), nil
+}
+
+func (v *ValueGobber) GobDecode(data []byte) os.Error {
+	_, err := fmt.Sscanf(string(data), "VALUE=%s", (*string)(v))
+	return err
+}
+
+// Structs that include GobEncodable fields.
+
+type GobTest0 struct {
+	X int // guarantee we have  something in common with GobTest*
+	G *ByteStruct
+}
+
+type GobTest1 struct {
+	X int // guarantee we have  something in common with GobTest*
+	G *StringStruct
+}
+
+type GobTest2 struct {
+	X int    // guarantee we have  something in common with GobTest*
+	G string // not a GobEncoder - should give us errors
+}
+
+type GobTest3 struct {
+	X int // guarantee we have  something in common with GobTest*
+	G *Gobber
+}
+
+type GobTest4 struct {
+	X int // guarantee we have  something in common with GobTest*
+	V ValueGobber
+}
+
+type GobTest5 struct {
+	X int // guarantee we have  something in common with GobTest*
+	V *ValueGobber
+}
+
+type GobTestIgnoreEncoder struct {
+	X int // guarantee we have  something in common with GobTest*
+}
+
+type GobTestValueEncDec struct {
+	X int          // guarantee we have  something in common with GobTest*
+	G StringStruct // not a pointer.
+}
+
+type GobTestIndirectEncDec struct {
+	X int             // guarantee we have  something in common with GobTest*
+	G ***StringStruct // indirections to the receiver.
+}
+
+type GobTestArrayEncDec struct {
+	X int         // guarantee we have  something in common with GobTest*
+	A ArrayStruct // not a pointer.
+}
+
+type GobTestIndirectArrayEncDec struct {
+	X int            // guarantee we have  something in common with GobTest*
+	A ***ArrayStruct // indirections to a large receiver.
+}
+
+func TestGobEncoderField(t *testing.T) {
+	b := new(bytes.Buffer)
+	// First a field that's a structure.
+	enc := NewEncoder(b)
+	err := enc.Encode(GobTest0{17, &ByteStruct{'A'}})
+	if err != nil {
+		t.Fatal("encode error:", err)
+	}
+	dec := NewDecoder(b)
+	x := new(GobTest0)
+	err = dec.Decode(x)
+	if err != nil {
+		t.Fatal("decode error:", err)
+	}
+	if x.G.a != 'A' {
+		t.Errorf("expected 'A' got %c", x.G.a)
+	}
+	// Now a field that's not a structure.
+	b.Reset()
+	gobber := Gobber(23)
+	err = enc.Encode(GobTest3{17, &gobber})
+	if err != nil {
+		t.Fatal("encode error:", err)
+	}
+	y := new(GobTest3)
+	err = dec.Decode(y)
+	if err != nil {
+		t.Fatal("decode error:", err)
+	}
+	if *y.G != 23 {
+		t.Errorf("expected '23 got %d", *y.G)
+	}
+}
+
+// Even though the field is a value, we can still take its address
+// and should be able to call the methods.
+func TestGobEncoderValueField(t *testing.T) {
+	b := new(bytes.Buffer)
+	// First a field that's a structure.
+	enc := NewEncoder(b)
+	err := enc.Encode(GobTestValueEncDec{17, StringStruct{"HIJKL"}})
+	if err != nil {
+		t.Fatal("encode error:", err)
+	}
+	dec := NewDecoder(b)
+	x := new(GobTestValueEncDec)
+	err = dec.Decode(x)
+	if err != nil {
+		t.Fatal("decode error:", err)
+	}
+	if x.G.s != "HIJKL" {
+		t.Errorf("expected `HIJKL` got %s", x.G.s)
+	}
+}
+
+// GobEncode/Decode should work even if the value is
+// more indirect than the receiver.
+func TestGobEncoderIndirectField(t *testing.T) {
+	b := new(bytes.Buffer)
+	// First a field that's a structure.
+	enc := NewEncoder(b)
+	s := &StringStruct{"HIJKL"}
+	sp := &s
+	err := enc.Encode(GobTestIndirectEncDec{17, &sp})
+	if err != nil {
+		t.Fatal("encode error:", err)
+	}
+	dec := NewDecoder(b)
+	x := new(GobTestIndirectEncDec)
+	err = dec.Decode(x)
+	if err != nil {
+		t.Fatal("decode error:", err)
+	}
+	if (***x.G).s != "HIJKL" {
+		t.Errorf("expected `HIJKL` got %s", (***x.G).s)
+	}
+}
+
+// Test with a large field with methods.
+func TestGobEncoderArrayField(t *testing.T) {
+	b := new(bytes.Buffer)
+	enc := NewEncoder(b)
+	var a GobTestArrayEncDec
+	a.X = 17
+	for i := range a.A.a {
+		a.A.a[i] = byte(i)
+	}
+	err := enc.Encode(a)
+	if err != nil {
+		t.Fatal("encode error:", err)
+	}
+	dec := NewDecoder(b)
+	x := new(GobTestArrayEncDec)
+	err = dec.Decode(x)
+	if err != nil {
+		t.Fatal("decode error:", err)
+	}
+	for i, v := range x.A.a {
+		if v != byte(i) {
+			t.Errorf("expected %x got %x", byte(i), v)
+			break
+		}
+	}
+}
+
+// Test an indirection to a large field with methods.
+func TestGobEncoderIndirectArrayField(t *testing.T) {
+	b := new(bytes.Buffer)
+	enc := NewEncoder(b)
+	var a GobTestIndirectArrayEncDec
+	a.X = 17
+	var array ArrayStruct
+	ap := &array
+	app := &ap
+	a.A = &app
+	for i := range array.a {
+		array.a[i] = byte(i)
+	}
+	err := enc.Encode(a)
+	if err != nil {
+		t.Fatal("encode error:", err)
+	}
+	dec := NewDecoder(b)
+	x := new(GobTestIndirectArrayEncDec)
+	err = dec.Decode(x)
+	if err != nil {
+		t.Fatal("decode error:", err)
+	}
+	for i, v := range (***x.A).a {
+		if v != byte(i) {
+			t.Errorf("expected %x got %x", byte(i), v)
+			break
+		}
+	}
+}
+
+// As long as the fields have the same name and implement the
+// interface, we can cross-connect them.  Not sure it's useful
+// and may even be bad but it works and it's hard to prevent
+// without exposing the contents of the object, which would
+// defeat the purpose.
+func TestGobEncoderFieldsOfDifferentType(t *testing.T) {
+	// first, string in field to byte in field
+	b := new(bytes.Buffer)
+	enc := NewEncoder(b)
+	err := enc.Encode(GobTest1{17, &StringStruct{"ABC"}})
+	if err != nil {
+		t.Fatal("encode error:", err)
+	}
+	dec := NewDecoder(b)
+	x := new(GobTest0)
+	err = dec.Decode(x)
+	if err != nil {
+		t.Fatal("decode error:", err)
+	}
+	if x.G.a != 'A' {
+		t.Errorf("expected 'A' got %c", x.G.a)
+	}
+	// now the other direction, byte in field to string in field
+	b.Reset()
+	err = enc.Encode(GobTest0{17, &ByteStruct{'X'}})
+	if err != nil {
+		t.Fatal("encode error:", err)
+	}
+	y := new(GobTest1)
+	err = dec.Decode(y)
+	if err != nil {
+		t.Fatal("decode error:", err)
+	}
+	if y.G.s != "XYZ" {
+		t.Fatalf("expected `XYZ` got %c", y.G.s)
+	}
+}
+
+// Test that we can encode a value and decode into a pointer.
+func TestGobEncoderValueEncoder(t *testing.T) {
+	// first, string in field to byte in field
+	b := new(bytes.Buffer)
+	enc := NewEncoder(b)
+	err := enc.Encode(GobTest4{17, ValueGobber("hello")})
+	if err != nil {
+		t.Fatal("encode error:", err)
+	}
+	dec := NewDecoder(b)
+	x := new(GobTest5)
+	err = dec.Decode(x)
+	if err != nil {
+		t.Fatal("decode error:", err)
+	}
+	if *x.V != "hello" {
+		t.Errorf("expected `hello` got %s", x.V)
+	}
+}
+
+func TestGobEncoderFieldTypeError(t *testing.T) {
+	// GobEncoder to non-decoder: error
+	b := new(bytes.Buffer)
+	enc := NewEncoder(b)
+	err := enc.Encode(GobTest1{17, &StringStruct{"ABC"}})
+	if err != nil {
+		t.Fatal("encode error:", err)
+	}
+	dec := NewDecoder(b)
+	x := &GobTest2{}
+	err = dec.Decode(x)
+	if err == nil {
+		t.Fatal("expected decode error for mismatched fields (encoder to non-decoder)")
+	}
+	if strings.Index(err.String(), "type") < 0 {
+		t.Fatal("expected type error; got", err)
+	}
+	// Non-encoder to GobDecoder: error
+	b.Reset()
+	err = enc.Encode(GobTest2{17, "ABC"})
+	if err != nil {
+		t.Fatal("encode error:", err)
+	}
+	y := &GobTest1{}
+	err = dec.Decode(y)
+	if err == nil {
+		t.Fatal("expected decode error for mismatched fields (non-encoder to decoder)")
+	}
+	if strings.Index(err.String(), "type") < 0 {
+		t.Fatal("expected type error; got", err)
+	}
+}
+
+// Even though ByteStruct is a struct, it's treated as a singleton at the top level.
+func TestGobEncoderStructSingleton(t *testing.T) {
+	b := new(bytes.Buffer)
+	enc := NewEncoder(b)
+	err := enc.Encode(&ByteStruct{'A'})
+	if err != nil {
+		t.Fatal("encode error:", err)
+	}
+	dec := NewDecoder(b)
+	x := new(ByteStruct)
+	err = dec.Decode(x)
+	if err != nil {
+		t.Fatal("decode error:", err)
+	}
+	if x.a != 'A' {
+		t.Errorf("expected 'A' got %c", x.a)
+	}
+}
+
+func TestGobEncoderNonStructSingleton(t *testing.T) {
+	b := new(bytes.Buffer)
+	enc := NewEncoder(b)
+	err := enc.Encode(Gobber(1234))
+	if err != nil {
+		t.Fatal("encode error:", err)
+	}
+	dec := NewDecoder(b)
+	var x Gobber
+	err = dec.Decode(&x)
+	if err != nil {
+		t.Fatal("decode error:", err)
+	}
+	if x != 1234 {
+		t.Errorf("expected 1234 got %c", x)
+	}
+}
+
+func TestGobEncoderIgnoreStructField(t *testing.T) {
+	b := new(bytes.Buffer)
+	// First a field that's a structure.
+	enc := NewEncoder(b)
+	err := enc.Encode(GobTest0{17, &ByteStruct{'A'}})
+	if err != nil {
+		t.Fatal("encode error:", err)
+	}
+	dec := NewDecoder(b)
+	x := new(GobTestIgnoreEncoder)
+	err = dec.Decode(x)
+	if err != nil {
+		t.Fatal("decode error:", err)
+	}
+	if x.X != 17 {
+		t.Errorf("expected 17 got %c", x.X)
+	}
+}
+
+func TestGobEncoderIgnoreNonStructField(t *testing.T) {
+	b := new(bytes.Buffer)
+	// First a field that's a structure.
+	enc := NewEncoder(b)
+	gobber := Gobber(23)
+	err := enc.Encode(GobTest3{17, &gobber})
+	if err != nil {
+		t.Fatal("encode error:", err)
+	}
+	dec := NewDecoder(b)
+	x := new(GobTestIgnoreEncoder)
+	err = dec.Decode(x)
+	if err != nil {
+		t.Fatal("decode error:", err)
+	}
+	if x.X != 17 {
+		t.Errorf("expected 17 got %c", x.X)
+	}
+}
+
+func TestGobEncoderIgnoreNilEncoder(t *testing.T) {
+	b := new(bytes.Buffer)
+	// First a field that's a structure.
+	enc := NewEncoder(b)
+	err := enc.Encode(GobTest0{X: 18}) // G is nil
+	if err != nil {
+		t.Fatal("encode error:", err)
+	}
+	dec := NewDecoder(b)
+	x := new(GobTest0)
+	err = dec.Decode(x)
+	if err != nil {
+		t.Fatal("decode error:", err)
+	}
+	if x.X != 18 {
+		t.Errorf("expected x.X = 18, got %v", x.X)
+	}
+	if x.G != nil {
+		t.Errorf("expected x.G = nil, got %v", x.G)
+	}
+}
diff --git a/src/pkg/gob/timing_test.go b/src/pkg/gob/timing_test.go
new file mode 100644
index 000000000..2a2be7336
--- /dev/null
+++ b/src/pkg/gob/timing_test.go
@@ -0,0 +1,94 @@
+// Copyright 2011 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package gob
+
+import (
+	"bytes"
+	"fmt"
+	"io"
+	"os"
+	"runtime"
+	"testing"
+)
+
+type Bench struct {
+	A int
+	B float64
+	C string
+	D []byte
+}
+
+func benchmarkEndToEnd(r io.Reader, w io.Writer, b *testing.B) {
+	b.StopTimer()
+	enc := NewEncoder(w)
+	dec := NewDecoder(r)
+	bench := &Bench{7, 3.2, "now is the time", []byte("for all good men")}
+	b.StartTimer()
+	for i := 0; i < b.N; i++ {
+		if enc.Encode(bench) != nil {
+			panic("encode error")
+		}
+		if dec.Decode(bench) != nil {
+			panic("decode error")
+		}
+	}
+}
+
+func BenchmarkEndToEndPipe(b *testing.B) {
+	r, w, err := os.Pipe()
+	if err != nil {
+		panic("can't get pipe:" + err.String())
+	}
+	benchmarkEndToEnd(r, w, b)
+}
+
+func BenchmarkEndToEndByteBuffer(b *testing.B) {
+	var buf bytes.Buffer
+	benchmarkEndToEnd(&buf, &buf, b)
+}
+
+func TestCountEncodeMallocs(t *testing.T) {
+	var buf bytes.Buffer
+	enc := NewEncoder(&buf)
+	bench := &Bench{7, 3.2, "now is the time", []byte("for all good men")}
+	runtime.UpdateMemStats()
+	mallocs := 0 - runtime.MemStats.Mallocs
+	const count = 1000
+	for i := 0; i < count; i++ {
+		err := enc.Encode(bench)
+		if err != nil {
+			t.Fatal("encode:", err)
+		}
+	}
+	runtime.UpdateMemStats()
+	mallocs += runtime.MemStats.Mallocs
+	fmt.Printf("mallocs per encode of type Bench: %d\n", mallocs/count)
+}
+
+func TestCountDecodeMallocs(t *testing.T) {
+	var buf bytes.Buffer
+	enc := NewEncoder(&buf)
+	bench := &Bench{7, 3.2, "now is the time", []byte("for all good men")}
+	const count = 1000
+	for i := 0; i < count; i++ {
+		err := enc.Encode(bench)
+		if err != nil {
+			t.Fatal("encode:", err)
+		}
+	}
+	dec := NewDecoder(&buf)
+	runtime.UpdateMemStats()
+	mallocs := 0 - runtime.MemStats.Mallocs
+	for i := 0; i < count; i++ {
+		*bench = Bench{}
+		err := dec.Decode(&bench)
+		if err != nil {
+			t.Fatal("decode:", err)
+		}
+	}
+	runtime.UpdateMemStats()
+	mallocs += runtime.MemStats.Mallocs
+	fmt.Printf("mallocs per decode of type Bench: %d\n", mallocs/count)
+}
diff --git a/src/pkg/gob/type.go b/src/pkg/gob/type.go
new file mode 100644
index 000000000..b2f716c4b
--- /dev/null
+++ b/src/pkg/gob/type.go
@@ -0,0 +1,786 @@
+// Copyright 2009 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package gob
+
+import (
+	"fmt"
+	"os"
+	"reflect"
+	"sync"
+	"unicode"
+	"utf8"
+)
+
+// userTypeInfo stores the information associated with a type the user has handed
+// to the package.  It's computed once and stored in a map keyed by reflection
+// type.
+type userTypeInfo struct {
+	user         reflect.Type // the type the user handed us
+	base         reflect.Type // the base type after all indirections
+	indir        int          // number of indirections to reach the base type
+	isGobEncoder bool         // does the type implement GobEncoder?
+	isGobDecoder bool         // does the type implement GobDecoder?
+	encIndir     int8         // number of indirections to reach the receiver type; may be negative
+	decIndir     int8         // number of indirections to reach the receiver type; may be negative
+}
+
+var (
+	// Protected by an RWMutex because we read it a lot and write
+	// it only when we see a new type, typically when compiling.
+	userTypeLock  sync.RWMutex
+	userTypeCache = make(map[reflect.Type]*userTypeInfo)
+)
+
+// validType returns, and saves, the information associated with user-provided type rt.
+// If the user type is not valid, err will be non-nil.  To be used when the error handler
+// is not set up.
+func validUserType(rt reflect.Type) (ut *userTypeInfo, err os.Error) {
+	userTypeLock.RLock()
+	ut = userTypeCache[rt]
+	userTypeLock.RUnlock()
+	if ut != nil {
+		return
+	}
+	// Now set the value under the write lock.
+	userTypeLock.Lock()
+	defer userTypeLock.Unlock()
+	if ut = userTypeCache[rt]; ut != nil {
+		// Lost the race; not a problem.
+		return
+	}
+	ut = new(userTypeInfo)
+	ut.base = rt
+	ut.user = rt
+	// A type that is just a cycle of pointers (such as type T *T) cannot
+	// be represented in gobs, which need some concrete data.  We use a
+	// cycle detection algorithm from Knuth, Vol 2, Section 3.1, Ex 6,
+	// pp 539-540.  As we step through indirections, run another type at
+	// half speed. If they meet up, there's a cycle.
+	slowpoke := ut.base // walks half as fast as ut.base
+	for {
+		pt := ut.base
+		if pt.Kind() != reflect.Ptr {
+			break
+		}
+		ut.base = pt.Elem()
+		if ut.base == slowpoke { // ut.base lapped slowpoke
+			// recursive pointer type.
+			return nil, os.NewError("can't represent recursive pointer type " + ut.base.String())
+		}
+		if ut.indir%2 == 0 {
+			slowpoke = slowpoke.Elem()
+		}
+		ut.indir++
+	}
+	ut.isGobEncoder, ut.encIndir = implementsInterface(ut.user, gobEncoderInterfaceType)
+	ut.isGobDecoder, ut.decIndir = implementsInterface(ut.user, gobDecoderInterfaceType)
+	userTypeCache[rt] = ut
+	return
+}
+
+var (
+	gobEncoderInterfaceType = reflect.TypeOf((*GobEncoder)(nil)).Elem()
+	gobDecoderInterfaceType = reflect.TypeOf((*GobDecoder)(nil)).Elem()
+)
+
+// implementsInterface reports whether the type implements the
+// gobEncoder/gobDecoder interface.
+// It also returns the number of indirections required to get to the
+// implementation.
+func implementsInterface(typ, gobEncDecType reflect.Type) (success bool, indir int8) {
+	if typ == nil {
+		return
+	}
+	rt := typ
+	// The type might be a pointer and we need to keep
+	// dereferencing to the base type until we find an implementation.
+	for {
+		if rt.Implements(gobEncDecType) {
+			return true, indir
+		}
+		if p := rt; p.Kind() == reflect.Ptr {
+			indir++
+			if indir > 100 { // insane number of indirections
+				return false, 0
+			}
+			rt = p.Elem()
+			continue
+		}
+		break
+	}
+	// No luck yet, but if this is a base type (non-pointer), the pointer might satisfy.
+	if typ.Kind() != reflect.Ptr {
+		// Not a pointer, but does the pointer work?
+		if reflect.PtrTo(typ).Implements(gobEncDecType) {
+			return true, -1
+		}
+	}
+	return false, 0
+}
+
+// userType returns, and saves, the information associated with user-provided type rt.
+// If the user type is not valid, it calls error.
+func userType(rt reflect.Type) *userTypeInfo {
+	ut, err := validUserType(rt)
+	if err != nil {
+		error(err)
+	}
+	return ut
+}
+// A typeId represents a gob Type as an integer that can be passed on the wire.
+// Internally, typeIds are used as keys to a map to recover the underlying type info.
+type typeId int32
+
+var nextId typeId       // incremented for each new type we build
+var typeLock sync.Mutex // set while building a type
+const firstUserId = 64  // lowest id number granted to user
+
+type gobType interface {
+	id() typeId
+	setId(id typeId)
+	name() string
+	string() string // not public; only for debugging
+	safeString(seen map[typeId]bool) string
+}
+
+var types = make(map[reflect.Type]gobType)
+var idToType = make(map[typeId]gobType)
+var builtinIdToType map[typeId]gobType // set in init() after builtins are established
+
+func setTypeId(typ gobType) {
+	nextId++
+	typ.setId(nextId)
+	idToType[nextId] = typ
+}
+
+func (t typeId) gobType() gobType {
+	if t == 0 {
+		return nil
+	}
+	return idToType[t]
+}
+
+// string returns the string representation of the type associated with the typeId.
+func (t typeId) string() string {
+	if t.gobType() == nil {
+		return ""
+	}
+	return t.gobType().string()
+}
+
+// Name returns the name of the type associated with the typeId.
+func (t typeId) name() string {
+	if t.gobType() == nil {
+		return ""
+	}
+	return t.gobType().name()
+}
+
+// Common elements of all types.
+type CommonType struct {
+	Name string
+	Id   typeId
+}
+
+func (t *CommonType) id() typeId { return t.Id }
+
+func (t *CommonType) setId(id typeId) { t.Id = id }
+
+func (t *CommonType) string() string { return t.Name }
+
+func (t *CommonType) safeString(seen map[typeId]bool) string {
+	return t.Name
+}
+
+func (t *CommonType) name() string { return t.Name }
+
+// Create and check predefined types
+// The string for tBytes is "bytes" not "[]byte" to signify its specialness.
+
+var (
+	// Primordial types, needed during initialization.
+	// Always passed as pointers so the interface{} type
+	// goes through without losing its interfaceness.
+	tBool      = bootstrapType("bool", (*bool)(nil), 1)
+	tInt       = bootstrapType("int", (*int)(nil), 2)
+	tUint      = bootstrapType("uint", (*uint)(nil), 3)
+	tFloat     = bootstrapType("float", (*float64)(nil), 4)
+	tBytes     = bootstrapType("bytes", (*[]byte)(nil), 5)
+	tString    = bootstrapType("string", (*string)(nil), 6)
+	tComplex   = bootstrapType("complex", (*complex128)(nil), 7)
+	tInterface = bootstrapType("interface", (*interface{})(nil), 8)
+	// Reserve some Ids for compatible expansion
+	tReserved7 = bootstrapType("_reserved1", (*struct{ r7 int })(nil), 9)
+	tReserved6 = bootstrapType("_reserved1", (*struct{ r6 int })(nil), 10)
+	tReserved5 = bootstrapType("_reserved1", (*struct{ r5 int })(nil), 11)
+	tReserved4 = bootstrapType("_reserved1", (*struct{ r4 int })(nil), 12)
+	tReserved3 = bootstrapType("_reserved1", (*struct{ r3 int })(nil), 13)
+	tReserved2 = bootstrapType("_reserved1", (*struct{ r2 int })(nil), 14)
+	tReserved1 = bootstrapType("_reserved1", (*struct{ r1 int })(nil), 15)
+)
+
+// Predefined because it's needed by the Decoder
+var tWireType = mustGetTypeInfo(reflect.TypeOf(wireType{})).id
+var wireTypeUserInfo *userTypeInfo // userTypeInfo of (*wireType)
+
+func init() {
+	// Some magic numbers to make sure there are no surprises.
+	checkId(16, tWireType)
+	checkId(17, mustGetTypeInfo(reflect.TypeOf(arrayType{})).id)
+	checkId(18, mustGetTypeInfo(reflect.TypeOf(CommonType{})).id)
+	checkId(19, mustGetTypeInfo(reflect.TypeOf(sliceType{})).id)
+	checkId(20, mustGetTypeInfo(reflect.TypeOf(structType{})).id)
+	checkId(21, mustGetTypeInfo(reflect.TypeOf(fieldType{})).id)
+	checkId(23, mustGetTypeInfo(reflect.TypeOf(mapType{})).id)
+
+	builtinIdToType = make(map[typeId]gobType)
+	for k, v := range idToType {
+		builtinIdToType[k] = v
+	}
+
+	// Move the id space upwards to allow for growth in the predefined world
+	// without breaking existing files.
+	if nextId > firstUserId {
+		panic(fmt.Sprintln("nextId too large:", nextId))
+	}
+	nextId = firstUserId
+	registerBasics()
+	wireTypeUserInfo = userType(reflect.TypeOf((*wireType)(nil)))
+}
+
+// Array type
+type arrayType struct {
+	CommonType
+	Elem typeId
+	Len  int
+}
+
+func newArrayType(name string) *arrayType {
+	a := &arrayType{CommonType{Name: name}, 0, 0}
+	return a
+}
+
+func (a *arrayType) init(elem gobType, len int) {
+	// Set our type id before evaluating the element's, in case it's our own.
+	setTypeId(a)
+	a.Elem = elem.id()
+	a.Len = len
+}
+
+func (a *arrayType) safeString(seen map[typeId]bool) string {
+	if seen[a.Id] {
+		return a.Name
+	}
+	seen[a.Id] = true
+	return fmt.Sprintf("[%d]%s", a.Len, a.Elem.gobType().safeString(seen))
+}
+
+func (a *arrayType) string() string { return a.safeString(make(map[typeId]bool)) }
+
+// GobEncoder type (something that implements the GobEncoder interface)
+type gobEncoderType struct {
+	CommonType
+}
+
+func newGobEncoderType(name string) *gobEncoderType {
+	g := &gobEncoderType{CommonType{Name: name}}
+	setTypeId(g)
+	return g
+}
+
+func (g *gobEncoderType) safeString(seen map[typeId]bool) string {
+	return g.Name
+}
+
+func (g *gobEncoderType) string() string { return g.Name }
+
+// Map type
+type mapType struct {
+	CommonType
+	Key  typeId
+	Elem typeId
+}
+
+func newMapType(name string) *mapType {
+	m := &mapType{CommonType{Name: name}, 0, 0}
+	return m
+}
+
+func (m *mapType) init(key, elem gobType) {
+	// Set our type id before evaluating the element's, in case it's our own.
+	setTypeId(m)
+	m.Key = key.id()
+	m.Elem = elem.id()
+}
+
+func (m *mapType) safeString(seen map[typeId]bool) string {
+	if seen[m.Id] {
+		return m.Name
+	}
+	seen[m.Id] = true
+	key := m.Key.gobType().safeString(seen)
+	elem := m.Elem.gobType().safeString(seen)
+	return fmt.Sprintf("map[%s]%s", key, elem)
+}
+
+func (m *mapType) string() string { return m.safeString(make(map[typeId]bool)) }
+
+// Slice type
+type sliceType struct {
+	CommonType
+	Elem typeId
+}
+
+func newSliceType(name string) *sliceType {
+	s := &sliceType{CommonType{Name: name}, 0}
+	return s
+}
+
+func (s *sliceType) init(elem gobType) {
+	// Set our type id before evaluating the element's, in case it's our own.
+	setTypeId(s)
+	s.Elem = elem.id()
+}
+
+func (s *sliceType) safeString(seen map[typeId]bool) string {
+	if seen[s.Id] {
+		return s.Name
+	}
+	seen[s.Id] = true
+	return fmt.Sprintf("[]%s", s.Elem.gobType().safeString(seen))
+}
+
+func (s *sliceType) string() string { return s.safeString(make(map[typeId]bool)) }
+
+// Struct type
+type fieldType struct {
+	Name string
+	Id   typeId
+}
+
+type structType struct {
+	CommonType
+	Field []*fieldType
+}
+
+func (s *structType) safeString(seen map[typeId]bool) string {
+	if s == nil {
+		return ""
+	}
+	if _, ok := seen[s.Id]; ok {
+		return s.Name
+	}
+	seen[s.Id] = true
+	str := s.Name + " = struct { "
+	for _, f := range s.Field {
+		str += fmt.Sprintf("%s %s; ", f.Name, f.Id.gobType().safeString(seen))
+	}
+	str += "}"
+	return str
+}
+
+func (s *structType) string() string { return s.safeString(make(map[typeId]bool)) }
+
+func newStructType(name string) *structType {
+	s := &structType{CommonType{Name: name}, nil}
+	// For historical reasons we set the id here rather than init.
+	// See the comment in newTypeObject for details.
+	setTypeId(s)
+	return s
+}
+
+// newTypeObject allocates a gobType for the reflection type rt.
+// Unless ut represents a GobEncoder, rt should be the base type
+// of ut.
+// This is only called from the encoding side. The decoding side
+// works through typeIds and userTypeInfos alone.
+func newTypeObject(name string, ut *userTypeInfo, rt reflect.Type) (gobType, os.Error) {
+	// Does this type implement GobEncoder?
+	if ut.isGobEncoder {
+		return newGobEncoderType(name), nil
+	}
+	var err os.Error
+	var type0, type1 gobType
+	defer func() {
+		if err != nil {
+			types[rt] = nil, false
+		}
+	}()
+	// Install the top-level type before the subtypes (e.g. struct before
+	// fields) so recursive types can be constructed safely.
+	switch t := rt; t.Kind() {
+	// All basic types are easy: they are predefined.
+	case reflect.Bool:
+		return tBool.gobType(), nil
+
+	case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64:
+		return tInt.gobType(), nil
+
+	case reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64, reflect.Uintptr:
+		return tUint.gobType(), nil
+
+	case reflect.Float32, reflect.Float64:
+		return tFloat.gobType(), nil
+
+	case reflect.Complex64, reflect.Complex128:
+		return tComplex.gobType(), nil
+
+	case reflect.String:
+		return tString.gobType(), nil
+
+	case reflect.Interface:
+		return tInterface.gobType(), nil
+
+	case reflect.Array:
+		at := newArrayType(name)
+		types[rt] = at
+		type0, err = getBaseType("", t.Elem())
+		if err != nil {
+			return nil, err
+		}
+		// Historical aside:
+		// For arrays, maps, and slices, we set the type id after the elements
+		// are constructed. This is to retain the order of type id allocation after
+		// a fix made to handle recursive types, which changed the order in
+		// which types are built.  Delaying the setting in this way preserves
+		// type ids while allowing recursive types to be described. Structs,
+		// done below, were already handling recursion correctly so they
+		// assign the top-level id before those of the field.
+		at.init(type0, t.Len())
+		return at, nil
+
+	case reflect.Map:
+		mt := newMapType(name)
+		types[rt] = mt
+		type0, err = getBaseType("", t.Key())
+		if err != nil {
+			return nil, err
+		}
+		type1, err = getBaseType("", t.Elem())
+		if err != nil {
+			return nil, err
+		}
+		mt.init(type0, type1)
+		return mt, nil
+
+	case reflect.Slice:
+		// []byte == []uint8 is a special case
+		if t.Elem().Kind() == reflect.Uint8 {
+			return tBytes.gobType(), nil
+		}
+		st := newSliceType(name)
+		types[rt] = st
+		type0, err = getBaseType(t.Elem().Name(), t.Elem())
+		if err != nil {
+			return nil, err
+		}
+		st.init(type0)
+		return st, nil
+
+	case reflect.Struct:
+		st := newStructType(name)
+		types[rt] = st
+		idToType[st.id()] = st
+		for i := 0; i < t.NumField(); i++ {
+			f := t.Field(i)
+			if !isExported(f.Name) {
+				continue
+			}
+			typ := userType(f.Type).base
+			tname := typ.Name()
+			if tname == "" {
+				t := userType(f.Type).base
+				tname = t.String()
+			}
+			gt, err := getBaseType(tname, f.Type)
+			if err != nil {
+				return nil, err
+			}
+			st.Field = append(st.Field, &fieldType{f.Name, gt.id()})
+		}
+		return st, nil
+
+	default:
+		return nil, os.NewError("gob NewTypeObject can't handle type: " + rt.String())
+	}
+	return nil, nil
+}
+
+// isExported reports whether this is an exported - upper case - name.
+func isExported(name string) bool {
+	rune, _ := utf8.DecodeRuneInString(name)
+	return unicode.IsUpper(rune)
+}
+
+// getBaseType returns the Gob type describing the given reflect.Type's base type.
+// typeLock must be held.
+func getBaseType(name string, rt reflect.Type) (gobType, os.Error) {
+	ut := userType(rt)
+	return getType(name, ut, ut.base)
+}
+
+// getType returns the Gob type describing the given reflect.Type.
+// Should be called only when handling GobEncoders/Decoders,
+// which may be pointers.  All other types are handled through the
+// base type, never a pointer.
+// typeLock must be held.
+func getType(name string, ut *userTypeInfo, rt reflect.Type) (gobType, os.Error) {
+	typ, present := types[rt]
+	if present {
+		return typ, nil
+	}
+	typ, err := newTypeObject(name, ut, rt)
+	if err == nil {
+		types[rt] = typ
+	}
+	return typ, err
+}
+
+func checkId(want, got typeId) {
+	if want != got {
+		fmt.Fprintf(os.Stderr, "checkId: %d should be %d\n", int(got), int(want))
+		panic("bootstrap type wrong id: " + got.name() + " " + got.string() + " not " + want.string())
+	}
+}
+
+// used for building the basic types; called only from init().  the incoming
+// interface always refers to a pointer.
+func bootstrapType(name string, e interface{}, expect typeId) typeId {
+	rt := reflect.TypeOf(e).Elem()
+	_, present := types[rt]
+	if present {
+		panic("bootstrap type already present: " + name + ", " + rt.String())
+	}
+	typ := &CommonType{Name: name}
+	types[rt] = typ
+	setTypeId(typ)
+	checkId(expect, nextId)
+	userType(rt) // might as well cache it now
+	return nextId
+}
+
+// Representation of the information we send and receive about this type.
+// Each value we send is preceded by its type definition: an encoded int.
+// However, the very first time we send the value, we first send the pair
+// (-id, wireType).
+// For bootstrapping purposes, we assume that the recipient knows how
+// to decode a wireType; it is exactly the wireType struct here, interpreted
+// using the gob rules for sending a structure, except that we assume the
+// ids for wireType and structType etc. are known.  The relevant pieces
+// are built in encode.go's init() function.
+// To maintain binary compatibility, if you extend this type, always put
+// the new fields last.
+type wireType struct {
+	ArrayT      *arrayType
+	SliceT      *sliceType
+	StructT     *structType
+	MapT        *mapType
+	GobEncoderT *gobEncoderType
+}
+
+func (w *wireType) string() string {
+	const unknown = "unknown type"
+	if w == nil {
+		return unknown
+	}
+	switch {
+	case w.ArrayT != nil:
+		return w.ArrayT.Name
+	case w.SliceT != nil:
+		return w.SliceT.Name
+	case w.StructT != nil:
+		return w.StructT.Name
+	case w.MapT != nil:
+		return w.MapT.Name
+	case w.GobEncoderT != nil:
+		return w.GobEncoderT.Name
+	}
+	return unknown
+}
+
+type typeInfo struct {
+	id      typeId
+	encoder *encEngine
+	wire    *wireType
+}
+
+var typeInfoMap = make(map[reflect.Type]*typeInfo) // protected by typeLock
+
+// typeLock must be held.
+func getTypeInfo(ut *userTypeInfo) (*typeInfo, os.Error) {
+	rt := ut.base
+	if ut.isGobEncoder {
+		// We want the user type, not the base type.
+		rt = ut.user
+	}
+	info, ok := typeInfoMap[rt]
+	if ok {
+		return info, nil
+	}
+	info = new(typeInfo)
+	gt, err := getBaseType(rt.Name(), rt)
+	if err != nil {
+		return nil, err
+	}
+	info.id = gt.id()
+
+	if ut.isGobEncoder {
+		userType, err := getType(rt.Name(), ut, rt)
+		if err != nil {
+			return nil, err
+		}
+		info.wire = &wireType{GobEncoderT: userType.id().gobType().(*gobEncoderType)}
+		typeInfoMap[ut.user] = info
+		return info, nil
+	}
+
+	t := info.id.gobType()
+	switch typ := rt; typ.Kind() {
+	case reflect.Array:
+		info.wire = &wireType{ArrayT: t.(*arrayType)}
+	case reflect.Map:
+		info.wire = &wireType{MapT: t.(*mapType)}
+	case reflect.Slice:
+		// []byte == []uint8 is a special case handled separately
+		if typ.Elem().Kind() != reflect.Uint8 {
+			info.wire = &wireType{SliceT: t.(*sliceType)}
+		}
+	case reflect.Struct:
+		info.wire = &wireType{StructT: t.(*structType)}
+	}
+	typeInfoMap[rt] = info
+	return info, nil
+}
+
+// Called only when a panic is acceptable and unexpected.
+func mustGetTypeInfo(rt reflect.Type) *typeInfo {
+	t, err := getTypeInfo(userType(rt))
+	if err != nil {
+		panic("getTypeInfo: " + err.String())
+	}
+	return t
+}
+
+// GobEncoder is the interface describing data that provides its own
+// representation for encoding values for transmission to a GobDecoder.
+// A type that implements GobEncoder and GobDecoder has complete
+// control over the representation of its data and may therefore
+// contain things such as private fields, channels, and functions,
+// which are not usually transmissible in gob streams.
+//
+// Note: Since gobs can be stored permanently, It is good design
+// to guarantee the encoding used by a GobEncoder is stable as the
+// software evolves.  For instance, it might make sense for GobEncode
+// to include a version number in the encoding.
+type GobEncoder interface {
+	// GobEncode returns a byte slice representing the encoding of the
+	// receiver for transmission to a GobDecoder, usually of the same
+	// concrete type.
+	GobEncode() ([]byte, os.Error)
+}
+
+// GobDecoder is the interface describing data that provides its own
+// routine for decoding transmitted values sent by a GobEncoder.
+type GobDecoder interface {
+	// GobDecode overwrites the receiver, which must be a pointer,
+	// with the value represented by the byte slice, which was written
+	// by GobEncode, usually for the same concrete type.
+	GobDecode([]byte) os.Error
+}
+
+var (
+	nameToConcreteType = make(map[string]reflect.Type)
+	concreteTypeToName = make(map[reflect.Type]string)
+)
+
+// RegisterName is like Register but uses the provided name rather than the
+// type's default.
+func RegisterName(name string, value interface{}) {
+	if name == "" {
+		// reserved for nil
+		panic("attempt to register empty name")
+	}
+	base := userType(reflect.TypeOf(value)).base
+	// Check for incompatible duplicates.
+	if t, ok := nameToConcreteType[name]; ok && t != base {
+		panic("gob: registering duplicate types for " + name)
+	}
+	if n, ok := concreteTypeToName[base]; ok && n != name {
+		panic("gob: registering duplicate names for " + base.String())
+	}
+	// Store the name and type provided by the user....
+	nameToConcreteType[name] = reflect.TypeOf(value)
+	// but the flattened type in the type table, since that's what decode needs.
+	concreteTypeToName[base] = name
+}
+
+// Register records a type, identified by a value for that type, under its
+// internal type name.  That name will identify the concrete type of a value
+// sent or received as an interface variable.  Only types that will be
+// transferred as implementations of interface values need to be registered.
+// Expecting to be used only during initialization, it panics if the mapping
+// between types and names is not a bijection.
+func Register(value interface{}) {
+	// Default to printed representation for unnamed types
+	rt := reflect.TypeOf(value)
+	name := rt.String()
+
+	// But for named types (or pointers to them), qualify with import path.
+	// Dereference one pointer looking for a named type.
+	star := ""
+	if rt.Name() == "" {
+		if pt := rt; pt.Kind() == reflect.Ptr {
+			star = "*"
+			rt = pt
+		}
+	}
+	if rt.Name() != "" {
+		if rt.PkgPath() == "" {
+			name = star + rt.Name()
+		} else {
+			name = star + rt.PkgPath() + "." + rt.Name()
+		}
+	}
+
+	RegisterName(name, value)
+}
+
+func registerBasics() {
+	Register(int(0))
+	Register(int8(0))
+	Register(int16(0))
+	Register(int32(0))
+	Register(int64(0))
+	Register(uint(0))
+	Register(uint8(0))
+	Register(uint16(0))
+	Register(uint32(0))
+	Register(uint64(0))
+	Register(float32(0))
+	Register(float64(0))
+	Register(complex64(0i))
+	Register(complex128(0i))
+	Register(uintptr(0))
+	Register(false)
+	Register("")
+	Register([]byte(nil))
+	Register([]int(nil))
+	Register([]int8(nil))
+	Register([]int16(nil))
+	Register([]int32(nil))
+	Register([]int64(nil))
+	Register([]uint(nil))
+	Register([]uint8(nil))
+	Register([]uint16(nil))
+	Register([]uint32(nil))
+	Register([]uint64(nil))
+	Register([]float32(nil))
+	Register([]float64(nil))
+	Register([]complex64(nil))
+	Register([]complex128(nil))
+	Register([]uintptr(nil))
+	Register([]bool(nil))
+	Register([]string(nil))
+}
diff --git a/src/pkg/gob/type_test.go b/src/pkg/gob/type_test.go
new file mode 100644
index 000000000..411ffb797
--- /dev/null
+++ b/src/pkg/gob/type_test.go
@@ -0,0 +1,153 @@
+// Copyright 2009 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package gob
+
+import (
+	"reflect"
+	"testing"
+)
+
+type typeT struct {
+	id  typeId
+	str string
+}
+
+var basicTypes = []typeT{
+	{tBool, "bool"},
+	{tInt, "int"},
+	{tUint, "uint"},
+	{tFloat, "float"},
+	{tBytes, "bytes"},
+	{tString, "string"},
+}
+
+func getTypeUnlocked(name string, rt reflect.Type) gobType {
+	typeLock.Lock()
+	defer typeLock.Unlock()
+	t, err := getBaseType(name, rt)
+	if err != nil {
+		panic("getTypeUnlocked: " + err.String())
+	}
+	return t
+}
+
+// Sanity checks
+func TestBasic(t *testing.T) {
+	for _, tt := range basicTypes {
+		if tt.id.string() != tt.str {
+			t.Errorf("checkType: expected %q got %s", tt.str, tt.id.string())
+		}
+		if tt.id == 0 {
+			t.Errorf("id for %q is zero", tt.str)
+		}
+	}
+}
+
+// Reregister some basic types to check registration is idempotent.
+func TestReregistration(t *testing.T) {
+	newtyp := getTypeUnlocked("int", reflect.TypeOf(int(0)))
+	if newtyp != tInt.gobType() {
+		t.Errorf("reregistration of %s got new type", newtyp.string())
+	}
+	newtyp = getTypeUnlocked("uint", reflect.TypeOf(uint(0)))
+	if newtyp != tUint.gobType() {
+		t.Errorf("reregistration of %s got new type", newtyp.string())
+	}
+	newtyp = getTypeUnlocked("string", reflect.TypeOf("hello"))
+	if newtyp != tString.gobType() {
+		t.Errorf("reregistration of %s got new type", newtyp.string())
+	}
+}
+
+func TestArrayType(t *testing.T) {
+	var a3 [3]int
+	a3int := getTypeUnlocked("foo", reflect.TypeOf(a3))
+	newa3int := getTypeUnlocked("bar", reflect.TypeOf(a3))
+	if a3int != newa3int {
+		t.Errorf("second registration of [3]int creates new type")
+	}
+	var a4 [4]int
+	a4int := getTypeUnlocked("goo", reflect.TypeOf(a4))
+	if a3int == a4int {
+		t.Errorf("registration of [3]int creates same type as [4]int")
+	}
+	var b3 [3]bool
+	a3bool := getTypeUnlocked("", reflect.TypeOf(b3))
+	if a3int == a3bool {
+		t.Errorf("registration of [3]bool creates same type as [3]int")
+	}
+	str := a3bool.string()
+	expected := "[3]bool"
+	if str != expected {
+		t.Errorf("array printed as %q; expected %q", str, expected)
+	}
+}
+
+func TestSliceType(t *testing.T) {
+	var s []int
+	sint := getTypeUnlocked("slice", reflect.TypeOf(s))
+	var news []int
+	newsint := getTypeUnlocked("slice1", reflect.TypeOf(news))
+	if sint != newsint {
+		t.Errorf("second registration of []int creates new type")
+	}
+	var b []bool
+	sbool := getTypeUnlocked("", reflect.TypeOf(b))
+	if sbool == sint {
+		t.Errorf("registration of []bool creates same type as []int")
+	}
+	str := sbool.string()
+	expected := "[]bool"
+	if str != expected {
+		t.Errorf("slice printed as %q; expected %q", str, expected)
+	}
+}
+
+func TestMapType(t *testing.T) {
+	var m map[string]int
+	mapStringInt := getTypeUnlocked("map", reflect.TypeOf(m))
+	var newm map[string]int
+	newMapStringInt := getTypeUnlocked("map1", reflect.TypeOf(newm))
+	if mapStringInt != newMapStringInt {
+		t.Errorf("second registration of map[string]int creates new type")
+	}
+	var b map[string]bool
+	mapStringBool := getTypeUnlocked("", reflect.TypeOf(b))
+	if mapStringBool == mapStringInt {
+		t.Errorf("registration of map[string]bool creates same type as map[string]int")
+	}
+	str := mapStringBool.string()
+	expected := "map[string]bool"
+	if str != expected {
+		t.Errorf("map printed as %q; expected %q", str, expected)
+	}
+}
+
+type Bar struct {
+	X string
+}
+
+// This structure has pointers and refers to itself, making it a good test case.
+type Foo struct {
+	A int
+	B int32 // will become int
+	C string
+	D []byte
+	E *float64    // will become float64
+	F ****float64 // will become float64
+	G *Bar
+	H *Bar // should not interpolate the definition of Bar again
+	I *Foo // will not explode
+}
+
+func TestStructType(t *testing.T) {
+	sstruct := getTypeUnlocked("Foo", reflect.TypeOf(Foo{}))
+	str := sstruct.string()
+	// If we can print it correctly, we built it correctly.
+	expected := "Foo = struct { A int; B int; C string; D bytes; E float; F float; G Bar = struct { X string; }; H Bar; I Foo; }"
+	if str != expected {
+		t.Errorf("struct printed as %q; expected %q", str, expected)
+	}
+}
diff --git a/src/pkg/hash/Makefile b/src/pkg/hash/Makefile
new file mode 100644
index 000000000..56071cb33
--- /dev/null
+++ b/src/pkg/hash/Makefile
@@ -0,0 +1,11 @@
+# Copyright 2009 The Go Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style
+# license that can be found in the LICENSE file.
+
+include ../../Make.inc
+
+TARG=hash
+GOFILES=\
+	hash.go\
+
+include ../../Make.pkg
diff --git a/src/pkg/hash/adler32/Makefile b/src/pkg/hash/adler32/Makefile
new file mode 100644
index 000000000..38ce537ba
--- /dev/null
+++ b/src/pkg/hash/adler32/Makefile
@@ -0,0 +1,11 @@
+# Copyright 2009 The Go Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style
+# license that can be found in the LICENSE file.
+
+include ../../../Make.inc
+
+TARG=hash/adler32
+GOFILES=\
+	adler32.go\
+
+include ../../../Make.pkg
diff --git a/src/pkg/hash/adler32/adler32.go b/src/pkg/hash/adler32/adler32.go
new file mode 100644
index 000000000..84943d9ae
--- /dev/null
+++ b/src/pkg/hash/adler32/adler32.go
@@ -0,0 +1,88 @@
+// Copyright 2009 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// Package adler32 implements the Adler-32 checksum.
+// Defined in RFC 1950:
+//	Adler-32 is composed of two sums accumulated per byte: s1 is
+//	the sum of all bytes, s2 is the sum of all s1 values. Both sums
+//	are done modulo 65521. s1 is initialized to 1, s2 to zero.  The
+//	Adler-32 checksum is stored as s2*65536 + s1 in most-
+//	significant-byte first (network) order.
+package adler32
+
+import (
+	"hash"
+	"os"
+)
+
+const (
+	mod = 65521
+)
+
+// The size of an Adler-32 checksum in bytes.
+const Size = 4
+
+// digest represents the partial evaluation of a checksum.
+type digest struct {
+	// invariant: (a < mod && b < mod) || a <= b
+	// invariant: a + b + 255 <= 0xffffffff
+	a, b uint32
+}
+
+func (d *digest) Reset() { d.a, d.b = 1, 0 }
+
+// New returns a new hash.Hash32 computing the Adler-32 checksum.
+func New() hash.Hash32 {
+	d := new(digest)
+	d.Reset()
+	return d
+}
+
+func (d *digest) Size() int { return Size }
+
+// Add p to the running checksum a, b.
+func update(a, b uint32, p []byte) (aa, bb uint32) {
+	for _, pi := range p {
+		a += uint32(pi)
+		b += a
+		// invariant: a <= b
+		if b > (0xffffffff-255)/2 {
+			a %= mod
+			b %= mod
+			// invariant: a < mod && b < mod
+		} else {
+			// invariant: a + b + 255 <= 2 * b + 255 <= 0xffffffff
+		}
+	}
+	return a, b
+}
+
+// Return the 32-bit checksum corresponding to a, b.
+func finish(a, b uint32) uint32 {
+	if b >= mod {
+		a %= mod
+		b %= mod
+	}
+	return b<<16 | a
+}
+
+func (d *digest) Write(p []byte) (nn int, err os.Error) {
+	d.a, d.b = update(d.a, d.b, p)
+	return len(p), nil
+}
+
+func (d *digest) Sum32() uint32 { return finish(d.a, d.b) }
+
+func (d *digest) Sum() []byte {
+	p := make([]byte, 4)
+	s := d.Sum32()
+	p[0] = byte(s >> 24)
+	p[1] = byte(s >> 16)
+	p[2] = byte(s >> 8)
+	p[3] = byte(s)
+	return p
+}
+
+// Checksum returns the Adler-32 checksum of data.
+func Checksum(data []byte) uint32 { return finish(update(1, 0, data)) }
diff --git a/src/pkg/hash/adler32/adler32_test.go b/src/pkg/hash/adler32/adler32_test.go
new file mode 100644
index 000000000..01f931c68
--- /dev/null
+++ b/src/pkg/hash/adler32/adler32_test.go
@@ -0,0 +1,77 @@
+// Copyright 2009 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package adler32
+
+import (
+	"bytes"
+	"io"
+	"testing"
+)
+
+type _Adler32Test struct {
+	out uint32
+	in  string
+}
+
+var golden = []_Adler32Test{
+	{0x1, ""},
+	{0x620062, "a"},
+	{0x12600c4, "ab"},
+	{0x24d0127, "abc"},
+	{0x3d8018b, "abcd"},
+	{0x5c801f0, "abcde"},
+	{0x81e0256, "abcdef"},
+	{0xadb02bd, "abcdefg"},
+	{0xe000325, "abcdefgh"},
+	{0x118e038e, "abcdefghi"},
+	{0x158603f8, "abcdefghij"},
+	{0x3f090f02, "Discard medicine more than two years old."},
+	{0x46d81477, "He who has a shady past knows that nice guys finish last."},
+	{0x40ee0ee1, "I wouldn't marry him with a ten foot pole."},
+	{0x16661315, "Free! Free!/A trip/to Mars/for 900/empty jars/Burma Shave"},
+	{0x5b2e1480, "The days of the digital watch are numbered.  -Tom Stoppard"},
+	{0x8c3c09ea, "Nepal premier won't resign."},
+	{0x45ac18fd, "For every action there is an equal and opposite government program."},
+	{0x53c61462, "His money is twice tainted: 'taint yours and 'taint mine."},
+	{0x7e511e63, "There is no reason for any individual to have a computer in their home. -Ken Olsen, 1977"},
+	{0xe4801a6a, "It's a tiny change to the code and not completely disgusting. - Bob Manchek"},
+	{0x61b507df, "size:  a.out:  bad magic"},
+	{0xb8631171, "The major problem is with sendmail.  -Mark Horton"},
+	{0x8b5e1904, "Give me a rock, paper and scissors and I will move the world.  CCFestoon"},
+	{0x7cc6102b, "If the enemy is within range, then so are you."},
+	{0x700318e7, "It's well we cannot hear the screams/That we create in others' dreams."},
+	{0x1e601747, "You remind me of a TV show, but that's all right: I watch it anyway."},
+	{0xb55b0b09, "C is as portable as Stonehedge!!"},
+	{0x39111dd0, "Even if I could be Shakespeare, I think I should still choose to be Faraday. - A. Huxley"},
+	{0x91dd304f, "The fugacity of a constituent in a mixture of gases at a given temperature is proportional to its mole fraction.  Lewis-Randall Rule"},
+	{0x2e5d1316, "How can you write a big system without C++?  -Paul Glick"},
+	{0xd0201df6, "'Invariant assertions' is the most elegant programming technique!  -Tom Szymanski"},
+}
+
+func TestGolden(t *testing.T) {
+	for i := 0; i < len(golden); i++ {
+		g := golden[i]
+		c := New()
+		io.WriteString(c, g.in)
+		s := c.Sum32()
+		if s != g.out {
+			t.Errorf("adler32(%s) = 0x%x want 0x%x", g.in, s, g.out)
+			t.FailNow()
+		}
+	}
+}
+
+func BenchmarkGolden(b *testing.B) {
+	b.StopTimer()
+	c := New()
+	var buf bytes.Buffer
+	for _, g := range golden {
+		buf.Write([]byte(g.in))
+	}
+	b.StartTimer()
+	for i := 0; i < b.N; i++ {
+		c.Write(buf.Bytes())
+	}
+}
diff --git a/src/pkg/hash/crc32/Makefile b/src/pkg/hash/crc32/Makefile
new file mode 100644
index 000000000..af8a64cf2
--- /dev/null
+++ b/src/pkg/hash/crc32/Makefile
@@ -0,0 +1,20 @@
+# Copyright 2009 The Go Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style
+# license that can be found in the LICENSE file.
+
+include ../../../Make.inc
+
+TARG=hash/crc32
+
+ifeq ($(GOARCH), amd64)
+	ARCH_GOFILES=crc32_amd64.go
+	OFILES=crc32_amd64.6
+else
+	ARCH_GOFILES=crc32_generic.go
+endif
+
+GOFILES=\
+	crc32.go\
+	$(ARCH_GOFILES)
+
+include ../../../Make.pkg
diff --git a/src/pkg/hash/crc32/crc32.go b/src/pkg/hash/crc32/crc32.go
new file mode 100644
index 000000000..0245b1ee8
--- /dev/null
+++ b/src/pkg/hash/crc32/crc32.go
@@ -0,0 +1,139 @@
+// Copyright 2009 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// Package crc32 implements the 32-bit cyclic redundancy check, or CRC-32,
+// checksum. See http://en.wikipedia.org/wiki/Cyclic_redundancy_check for
+// information.
+package crc32
+
+import (
+	"hash"
+	"os"
+	"sync"
+)
+
+// The size of a CRC-32 checksum in bytes.
+const Size = 4
+
+// Predefined polynomials.
+const (
+	// Far and away the most common CRC-32 polynomial.
+	// Used by ethernet (IEEE 802.3), v.42, fddi, gzip, zip, png, mpeg-2, ...
+	IEEE = 0xedb88320
+
+	// Castagnoli's polynomial, used in iSCSI.
+	// Has better error detection characteristics than IEEE.
+	// http://dx.doi.org/10.1109/26.231911
+	Castagnoli = 0x82f63b78
+
+	// Koopman's polynomial.
+	// Also has better error detection characteristics than IEEE.
+	// http://dx.doi.org/10.1109/DSN.2002.1028931
+	Koopman = 0xeb31d82e
+)
+
+// Table is a 256-word table representing the polynomial for efficient processing.
+type Table [256]uint32
+
+// castagnoliTable points to a lazily initialized Table for the Castagnoli
+// polynomial. MakeTable will always return this value when asked to make a
+// Castagnoli table so we can compare against it to find when the caller is
+// using this polynomial.
+var castagnoliTable *Table
+var castagnoliOnce sync.Once
+
+func castagnoliInit() {
+	castagnoliTable = makeTable(Castagnoli)
+}
+
+// IEEETable is the table for the IEEE polynomial.
+var IEEETable = makeTable(IEEE)
+
+// MakeTable returns the Table constructed from the specified polynomial.
+func MakeTable(poly uint32) *Table {
+	switch poly {
+	case IEEE:
+		return IEEETable
+	case Castagnoli:
+		castagnoliOnce.Do(castagnoliInit)
+		return castagnoliTable
+	}
+	return makeTable(poly)
+}
+
+// makeTable returns the Table constructed from the specified polynomial.
+func makeTable(poly uint32) *Table {
+	t := new(Table)
+	for i := 0; i < 256; i++ {
+		crc := uint32(i)
+		for j := 0; j < 8; j++ {
+			if crc&1 == 1 {
+				crc = (crc >> 1) ^ poly
+			} else {
+				crc >>= 1
+			}
+		}
+		t[i] = crc
+	}
+	return t
+}
+
+// digest represents the partial evaluation of a checksum.
+type digest struct {
+	crc uint32
+	tab *Table
+}
+
+// New creates a new hash.Hash32 computing the CRC-32 checksum
+// using the polynomial represented by the Table.
+func New(tab *Table) hash.Hash32 { return &digest{0, tab} }
+
+// NewIEEE creates a new hash.Hash32 computing the CRC-32 checksum
+// using the IEEE polynomial.
+func NewIEEE() hash.Hash32 { return New(IEEETable) }
+
+func (d *digest) Size() int { return Size }
+
+func (d *digest) Reset() { d.crc = 0 }
+
+func update(crc uint32, tab *Table, p []byte) uint32 {
+	crc = ^crc
+	for _, v := range p {
+		crc = tab[byte(crc)^v] ^ (crc >> 8)
+	}
+	return ^crc
+}
+
+// Update returns the result of adding the bytes in p to the crc.
+func Update(crc uint32, tab *Table, p []byte) uint32 {
+	if tab == castagnoliTable {
+		return updateCastagnoli(crc, p)
+	}
+	return update(crc, tab, p)
+}
+
+func (d *digest) Write(p []byte) (n int, err os.Error) {
+	d.crc = Update(d.crc, d.tab, p)
+	return len(p), nil
+}
+
+func (d *digest) Sum32() uint32 { return d.crc }
+
+func (d *digest) Sum() []byte {
+	p := make([]byte, 4)
+	s := d.Sum32()
+	p[0] = byte(s >> 24)
+	p[1] = byte(s >> 16)
+	p[2] = byte(s >> 8)
+	p[3] = byte(s)
+	return p
+}
+
+// Checksum returns the CRC-32 checksum of data
+// using the polynomial represented by the Table.
+func Checksum(data []byte, tab *Table) uint32 { return Update(0, tab, data) }
+
+// ChecksumIEEE returns the CRC-32 checksum of data
+// using the IEEE polynomial.
+func ChecksumIEEE(data []byte) uint32 { return update(0, IEEETable, data) }
diff --git a/src/pkg/hash/crc32/crc32_amd64.go b/src/pkg/hash/crc32/crc32_amd64.go
new file mode 100644
index 000000000..83349bc6c
--- /dev/null
+++ b/src/pkg/hash/crc32/crc32_amd64.go
@@ -0,0 +1,25 @@
+// Copyright 2011 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package crc32
+
+// This file contains the code to call the SSE 4.2 version of the Castagnoli
+// CRC.
+
+// haveSSE42 is defined in crc_amd64.s and uses CPUID to test for SSE 4.2
+// support.
+func haveSSE42() bool
+
+// castagnoliSSE42 is defined in crc_amd64.s and uses the SSE4.2 CRC32
+// instruction.
+func castagnoliSSE42(uint32, []byte) uint32
+
+var sse42 = haveSSE42()
+
+func updateCastagnoli(crc uint32, p []byte) uint32 {
+	if sse42 {
+		return castagnoliSSE42(crc, p)
+	}
+	return update(crc, castagnoliTable, p)
+}
diff --git a/src/pkg/hash/crc32/crc32_amd64.s b/src/pkg/hash/crc32/crc32_amd64.s
new file mode 100644
index 000000000..a9e5317e1
--- /dev/null
+++ b/src/pkg/hash/crc32/crc32_amd64.s
@@ -0,0 +1,62 @@
+// Copyright 2011 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// func castagnoliSSE42(crc uint32, p []byte) uint32
+TEXT ·castagnoliSSE42(SB),7,$0
+	MOVL crc+0(FP), AX  // CRC value
+	MOVQ p+8(FP), SI  // data pointer
+	MOVL p+16(FP), CX  // len(p)
+
+	NOTL AX
+
+	/* If there's less than 8 bytes to process, we do it byte-by-byte. */
+	CMPL CX, $8
+	JL cleanup
+
+	/* Process individual bytes until the input is 8-byte aligned. */
+startup:
+	MOVQ SI, BX
+	ANDQ $7, BX
+	JZ aligned
+
+	CRC32B (SI), AX
+	DECL CX
+	INCQ SI
+	JMP startup
+
+aligned:
+	/* The input is now 8-byte aligned and we can process 8-byte chunks. */
+	CMPL CX, $8
+	JL cleanup
+
+	CRC32Q (SI), AX
+	ADDQ $8, SI
+	SUBQ $8, CX
+	JMP aligned
+
+cleanup:
+	/* We may have some bytes left over that we process one at a time. */
+	CMPL CX, $0
+	JE done
+
+	CRC32B (SI), AX
+	INCQ SI
+	DECQ CX
+	JMP cleanup
+
+done:
+	NOTL AX
+	MOVL AX, ret+24(FP)
+	RET
+
+// func haveSSE42() bool
+TEXT ·haveSSE42(SB),7,$0
+	XORQ AX, AX
+	INCL AX
+	CPUID
+	SHRQ $20, CX
+	ANDQ $1, CX
+	MOVB CX, ret+0(FP)
+	RET
+
diff --git a/src/pkg/hash/crc32/crc32_generic.go b/src/pkg/hash/crc32/crc32_generic.go
new file mode 100644
index 000000000..27aabd903
--- /dev/null
+++ b/src/pkg/hash/crc32/crc32_generic.go
@@ -0,0 +1,12 @@
+// Copyright 2011 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package crc32
+
+// The file contains the generic version of updateCastagnoli which just calls
+// the software implementation.
+
+func updateCastagnoli(crc uint32, p []byte) uint32 {
+	return update(crc, castagnoliTable, p)
+}
diff --git a/src/pkg/hash/crc32/crc32_test.go b/src/pkg/hash/crc32/crc32_test.go
new file mode 100644
index 000000000..7e82dd755
--- /dev/null
+++ b/src/pkg/hash/crc32/crc32_test.go
@@ -0,0 +1,97 @@
+// Copyright 2009 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package crc32
+
+import (
+	"io"
+	"testing"
+)
+
+type test struct {
+	ieee, castagnoli uint32
+	in               string
+}
+
+var golden = []test{
+	{0x0, 0x0, ""},
+	{0xe8b7be43, 0xc1d04330, "a"},
+	{0x9e83486d, 0xe2a22936, "ab"},
+	{0x352441c2, 0x364b3fb7, "abc"},
+	{0xed82cd11, 0x92c80a31, "abcd"},
+	{0x8587d865, 0xc450d697, "abcde"},
+	{0x4b8e39ef, 0x53bceff1, "abcdef"},
+	{0x312a6aa6, 0xe627f441, "abcdefg"},
+	{0xaeef2a50, 0xa9421b7, "abcdefgh"},
+	{0x8da988af, 0x2ddc99fc, "abcdefghi"},
+	{0x3981703a, 0xe6599437, "abcdefghij"},
+	{0x6b9cdfe7, 0xb2cc01fe, "Discard medicine more than two years old."},
+	{0xc90ef73f, 0xe28207f, "He who has a shady past knows that nice guys finish last."},
+	{0xb902341f, 0xbe93f964, "I wouldn't marry him with a ten foot pole."},
+	{0x42080e8, 0x9e3be0c3, "Free! Free!/A trip/to Mars/for 900/empty jars/Burma Shave"},
+	{0x154c6d11, 0xf505ef04, "The days of the digital watch are numbered.  -Tom Stoppard"},
+	{0x4c418325, 0x85d3dc82, "Nepal premier won't resign."},
+	{0x33955150, 0xc5142380, "For every action there is an equal and opposite government program."},
+	{0x26216a4b, 0x75eb77dd, "His money is twice tainted: 'taint yours and 'taint mine."},
+	{0x1abbe45e, 0x91ebe9f7, "There is no reason for any individual to have a computer in their home. -Ken Olsen, 1977"},
+	{0xc89a94f7, 0xf0b1168e, "It's a tiny change to the code and not completely disgusting. - Bob Manchek"},
+	{0xab3abe14, 0x572b74e2, "size:  a.out:  bad magic"},
+	{0xbab102b6, 0x8a58a6d5, "The major problem is with sendmail.  -Mark Horton"},
+	{0x999149d7, 0x9c426c50, "Give me a rock, paper and scissors and I will move the world.  CCFestoon"},
+	{0x6d52a33c, 0x735400a4, "If the enemy is within range, then so are you."},
+	{0x90631e8d, 0xbec49c95, "It's well we cannot hear the screams/That we create in others' dreams."},
+	{0x78309130, 0xa95a2079, "You remind me of a TV show, but that's all right: I watch it anyway."},
+	{0x7d0a377f, 0xde2e65c5, "C is as portable as Stonehedge!!"},
+	{0x8c79fd79, 0x297a88ed, "Even if I could be Shakespeare, I think I should still choose to be Faraday. - A. Huxley"},
+	{0xa20b7167, 0x66ed1d8b, "The fugacity of a constituent in a mixture of gases at a given temperature is proportional to its mole fraction.  Lewis-Randall Rule"},
+	{0x8e0bb443, 0xdcded527, "How can you write a big system without C++?  -Paul Glick"},
+}
+
+func TestGolden(t *testing.T) {
+	castagnoliTab := MakeTable(Castagnoli)
+
+	for _, g := range golden {
+		ieee := NewIEEE()
+		io.WriteString(ieee, g.in)
+		s := ieee.Sum32()
+		if s != g.ieee {
+			t.Errorf("IEEE(%s) = 0x%x want 0x%x", g.in, s, g.ieee)
+		}
+
+		castagnoli := New(castagnoliTab)
+		io.WriteString(castagnoli, g.in)
+		s = castagnoli.Sum32()
+		if s != g.castagnoli {
+			t.Errorf("Castagnoli(%s) = 0x%x want 0x%x", g.in, s, g.castagnoli)
+		}
+
+		if len(g.in) > 0 {
+			// The SSE4.2 implementation of this has code to deal
+			// with misaligned data so we ensure that we test that
+			// too.
+			castagnoli = New(castagnoliTab)
+			io.WriteString(castagnoli, g.in[:1])
+			io.WriteString(castagnoli, g.in[1:])
+			s = castagnoli.Sum32()
+			if s != g.castagnoli {
+				t.Errorf("Castagnoli[misaligned](%s) = 0x%x want 0x%x", g.in, s, g.castagnoli)
+			}
+		}
+	}
+}
+
+func BenchmarkCrc32KB(b *testing.B) {
+	b.StopTimer()
+	data := make([]uint8, 1024)
+	for i := 0; i < 1024; i++ {
+		data[i] = uint8(i)
+	}
+	c := NewIEEE()
+	b.StartTimer()
+	b.SetBytes(int64(len(data)))
+
+	for i := 0; i < b.N; i++ {
+		c.Write(data)
+	}
+}
diff --git a/src/pkg/hash/crc64/Makefile b/src/pkg/hash/crc64/Makefile
new file mode 100644
index 000000000..5f6c3de69
--- /dev/null
+++ b/src/pkg/hash/crc64/Makefile
@@ -0,0 +1,11 @@
+# Copyright 2009 The Go Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style
+# license that can be found in the LICENSE file.
+
+include ../../../Make.inc
+
+TARG=hash/crc64
+GOFILES=\
+	crc64.go\
+
+include ../../../Make.pkg
diff --git a/src/pkg/hash/crc64/crc64.go b/src/pkg/hash/crc64/crc64.go
new file mode 100644
index 000000000..ae37e781c
--- /dev/null
+++ b/src/pkg/hash/crc64/crc64.go
@@ -0,0 +1,97 @@
+// Copyright 2009 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// Package crc64 implements the 64-bit cyclic redundancy check, or CRC-64,
+// checksum. See http://en.wikipedia.org/wiki/Cyclic_redundancy_check for
+// information.
+package crc64
+
+import (
+	"hash"
+	"os"
+)
+
+// The size of a CRC-64 checksum in bytes.
+const Size = 8
+
+// Predefined polynomials.
+const (
+	// The ISO polynomial, defined in ISO 3309 and used in HDLC.
+	ISO = 0xD800000000000000
+
+	// The ECMA polynomial, defined in ECMA 182.
+	ECMA = 0xC96C5795D7870F42
+)
+
+// Table is a 256-word table representing the polynomial for efficient processing.
+type Table [256]uint64
+
+// MakeTable returns the Table constructed from the specified polynomial.
+func MakeTable(poly uint64) *Table {
+	t := new(Table)
+	for i := 0; i < 256; i++ {
+		crc := uint64(i)
+		for j := 0; j < 8; j++ {
+			if crc&1 == 1 {
+				crc = (crc >> 1) ^ poly
+			} else {
+				crc >>= 1
+			}
+		}
+		t[i] = crc
+	}
+	return t
+}
+
+// digest represents the partial evaluation of a checksum.
+type digest struct {
+	crc uint64
+	tab *Table
+}
+
+// New creates a new hash.Hash64 computing the CRC-64 checksum
+// using the polynomial represented by the Table.
+func New(tab *Table) hash.Hash64 { return &digest{0, tab} }
+
+func (d *digest) Size() int { return Size }
+
+func (d *digest) Reset() { d.crc = 0 }
+
+func update(crc uint64, tab *Table, p []byte) uint64 {
+	crc = ^crc
+	for _, v := range p {
+		crc = tab[byte(crc)^v] ^ (crc >> 8)
+	}
+	return ^crc
+}
+
+// Update returns the result of adding the bytes in p to the crc.
+func Update(crc uint64, tab *Table, p []byte) uint64 {
+	return update(crc, tab, p)
+}
+
+func (d *digest) Write(p []byte) (n int, err os.Error) {
+	d.crc = update(d.crc, d.tab, p)
+	return len(p), nil
+}
+
+func (d *digest) Sum64() uint64 { return d.crc }
+
+func (d *digest) Sum() []byte {
+	p := make([]byte, 8)
+	s := d.Sum64()
+	p[0] = byte(s >> 56)
+	p[1] = byte(s >> 48)
+	p[2] = byte(s >> 40)
+	p[3] = byte(s >> 32)
+	p[4] = byte(s >> 24)
+	p[5] = byte(s >> 16)
+	p[6] = byte(s >> 8)
+	p[7] = byte(s)
+	return p
+}
+
+// Checksum returns the CRC-64 checksum of data
+// using the polynomial represented by the Table.
+func Checksum(data []byte, tab *Table) uint64 { return update(0, tab, data) }
diff --git a/src/pkg/hash/crc64/crc64_test.go b/src/pkg/hash/crc64/crc64_test.go
new file mode 100644
index 000000000..e932524e0
--- /dev/null
+++ b/src/pkg/hash/crc64/crc64_test.go
@@ -0,0 +1,78 @@
+// Copyright 2009 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package crc64
+
+import (
+	"io"
+	"testing"
+)
+
+type test struct {
+	out uint64
+	in  string
+}
+
+var golden = []test{
+	{0x0, ""},
+	{0x3420000000000000, "a"},
+	{0x36c4200000000000, "ab"},
+	{0x3776c42000000000, "abc"},
+	{0x336776c420000000, "abcd"},
+	{0x32d36776c4200000, "abcde"},
+	{0x3002d36776c42000, "abcdef"},
+	{0x31b002d36776c420, "abcdefg"},
+	{0xe21b002d36776c4, "abcdefgh"},
+	{0x8b6e21b002d36776, "abcdefghi"},
+	{0x7f5b6e21b002d367, "abcdefghij"},
+	{0x8ec0e7c835bf9cdf, "Discard medicine more than two years old."},
+	{0xc7db1759e2be5ab4, "He who has a shady past knows that nice guys finish last."},
+	{0xfbf9d9603a6fa020, "I wouldn't marry him with a ten foot pole."},
+	{0xeafc4211a6daa0ef, "Free! Free!/A trip/to Mars/for 900/empty jars/Burma Shave"},
+	{0x3e05b21c7a4dc4da, "The days of the digital watch are numbered.  -Tom Stoppard"},
+	{0x5255866ad6ef28a6, "Nepal premier won't resign."},
+	{0x8a79895be1e9c361, "For every action there is an equal and opposite government program."},
+	{0x8878963a649d4916, "His money is twice tainted: 'taint yours and 'taint mine."},
+	{0xa7b9d53ea87eb82f, "There is no reason for any individual to have a computer in their home. -Ken Olsen, 1977"},
+	{0xdb6805c0966a2f9c, "It's a tiny change to the code and not completely disgusting. - Bob Manchek"},
+	{0xf3553c65dacdadd2, "size:  a.out:  bad magic"},
+	{0x9d5e034087a676b9, "The major problem is with sendmail.  -Mark Horton"},
+	{0xa6db2d7f8da96417, "Give me a rock, paper and scissors and I will move the world.  CCFestoon"},
+	{0x325e00cd2fe819f9, "If the enemy is within range, then so are you."},
+	{0x88c6600ce58ae4c6, "It's well we cannot hear the screams/That we create in others' dreams."},
+	{0x28c4a3f3b769e078, "You remind me of a TV show, but that's all right: I watch it anyway."},
+	{0xa698a34c9d9f1dca, "C is as portable as Stonehedge!!"},
+	{0xf6c1e2a8c26c5cfc, "Even if I could be Shakespeare, I think I should still choose to be Faraday. - A. Huxley"},
+	{0xd402559dfe9b70c, "The fugacity of a constituent in a mixture of gases at a given temperature is proportional to its mole fraction.  Lewis-Randall Rule"},
+	{0xdb6efff26aa94946, "How can you write a big system without C++?  -Paul Glick"},
+}
+
+var tab = MakeTable(ISO)
+
+func TestGolden(t *testing.T) {
+	for i := 0; i < len(golden); i++ {
+		g := golden[i]
+		c := New(tab)
+		io.WriteString(c, g.in)
+		s := c.Sum64()
+		if s != g.out {
+			t.Errorf("crc64(%s) = 0x%x want 0x%x", g.in, s, g.out)
+			t.FailNow()
+		}
+	}
+}
+
+func BenchmarkCrc64KB(b *testing.B) {
+	b.StopTimer()
+	data := make([]uint8, 1024)
+	for i := 0; i < 1024; i++ {
+		data[i] = uint8(i)
+	}
+	c := New(tab)
+	b.StartTimer()
+
+	for i := 0; i < b.N; i++ {
+		c.Write(data)
+	}
+}
diff --git a/src/pkg/hash/fnv/Makefile b/src/pkg/hash/fnv/Makefile
new file mode 100644
index 000000000..4c8a4ecf0
--- /dev/null
+++ b/src/pkg/hash/fnv/Makefile
@@ -0,0 +1,11 @@
+# Copyright 2011 The Go Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style
+# license that can be found in the LICENSE file.
+
+include ../../../Make.inc
+
+TARG=hash/fnv
+GOFILES=\
+	fnv.go\
+
+include ../../../Make.pkg
diff --git a/src/pkg/hash/fnv/fnv.go b/src/pkg/hash/fnv/fnv.go
new file mode 100644
index 000000000..3ff7d7c75
--- /dev/null
+++ b/src/pkg/hash/fnv/fnv.go
@@ -0,0 +1,131 @@
+// Copyright 2011 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// Package fnv implements FNV-1 and FNV-1a, non-cryptographic hash functions
+// created by Glenn Fowler, Landon Curt Noll, and Phong Vo.
+// See http://isthe.com/chongo/tech/comp/fnv/.
+package fnv
+
+import (
+	"encoding/binary"
+	"hash"
+	"os"
+)
+
+type (
+	sum32  uint32
+	sum32a uint32
+	sum64  uint64
+	sum64a uint64
+)
+
+const (
+	offset32 = 2166136261
+	offset64 = 14695981039346656037
+	prime32  = 16777619
+	prime64  = 1099511628211
+)
+
+// New32 returns a new 32-bit FNV-1 hash.Hash.
+func New32() hash.Hash32 {
+	var s sum32 = offset32
+	return &s
+}
+
+// New32a returns a new 32-bit FNV-1a hash.Hash.
+func New32a() hash.Hash32 {
+	var s sum32a = offset32
+	return &s
+}
+
+// New64 returns a new 64-bit FNV-1 hash.Hash.
+func New64() hash.Hash64 {
+	var s sum64 = offset64
+	return &s
+}
+
+// New64a returns a new 64-bit FNV-1a hash.Hash.
+func New64a() hash.Hash64 {
+	var s sum64a = offset64
+	return &s
+}
+
+func (s *sum32) Reset()  { *s = offset32 }
+func (s *sum32a) Reset() { *s = offset32 }
+func (s *sum64) Reset()  { *s = offset64 }
+func (s *sum64a) Reset() { *s = offset64 }
+
+func (s *sum32) Sum32() uint32  { return uint32(*s) }
+func (s *sum32a) Sum32() uint32 { return uint32(*s) }
+func (s *sum64) Sum64() uint64  { return uint64(*s) }
+func (s *sum64a) Sum64() uint64 { return uint64(*s) }
+
+func (s *sum32) Write(data []byte) (int, os.Error) {
+	hash := *s
+	for _, c := range data {
+		hash *= prime32
+		hash ^= sum32(c)
+	}
+	*s = hash
+	return len(data), nil
+}
+
+func (s *sum32a) Write(data []byte) (int, os.Error) {
+	hash := *s
+	for _, c := range data {
+		hash ^= sum32a(c)
+		hash *= prime32
+	}
+	*s = hash
+	return len(data), nil
+}
+
+func (s *sum64) Write(data []byte) (int, os.Error) {
+	hash := *s
+	for _, c := range data {
+		hash *= prime64
+		hash ^= sum64(c)
+	}
+	*s = hash
+	return len(data), nil
+}
+
+func (s *sum64a) Write(data []byte) (int, os.Error) {
+	hash := *s
+	for _, c := range data {
+		hash ^= sum64a(c)
+		hash *= prime64
+	}
+	*s = hash
+	return len(data), nil
+}
+
+func (s *sum32) Size() int  { return 4 }
+func (s *sum32a) Size() int { return 4 }
+func (s *sum64) Size() int  { return 8 }
+func (s *sum64a) Size() int { return 8 }
+
+func (s *sum32) Sum() []byte {
+	a := make([]byte, 4)
+	binary.BigEndian.PutUint32(a, uint32(*s))
+	return a
+}
+
+func (s *sum32a) Sum() []byte {
+	a := make([]byte, 4)
+	binary.BigEndian.PutUint32(a, uint32(*s))
+	return a
+}
+
+func (s *sum64) Sum() []byte {
+	a := make([]byte, 8)
+	binary.BigEndian.PutUint64(a, uint64(*s))
+	return a
+}
+
+func (s *sum64a) Sum() []byte {
+	a := make([]byte, 8)
+	binary.BigEndian.PutUint64(a, uint64(*s))
+	return a
+}
diff --git a/src/pkg/hash/fnv/fnv_test.go b/src/pkg/hash/fnv/fnv_test.go
new file mode 100644
index 000000000..429230c80
--- /dev/null
+++ b/src/pkg/hash/fnv/fnv_test.go
@@ -0,0 +1,167 @@
+// Copyright 2011 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package fnv
+
+import (
+	"bytes"
+	"encoding/binary"
+	"hash"
+	"testing"
+)
+
+const testDataSize = 40
+
+type golden struct {
+	sum  []byte
+	text string
+}
+
+var golden32 = []golden{
+	{[]byte{0x81, 0x1c, 0x9d, 0xc5}, ""},
+	{[]byte{0x05, 0x0c, 0x5d, 0x7e}, "a"},
+	{[]byte{0x70, 0x77, 0x2d, 0x38}, "ab"},
+	{[]byte{0x43, 0x9c, 0x2f, 0x4b}, "abc"},
+}
+
+var golden32a = []golden{
+	{[]byte{0x81, 0x1c, 0x9d, 0xc5}, ""},
+	{[]byte{0xe4, 0x0c, 0x29, 0x2c}, "a"},
+	{[]byte{0x4d, 0x25, 0x05, 0xca}, "ab"},
+	{[]byte{0x1a, 0x47, 0xe9, 0x0b}, "abc"},
+}
+
+var golden64 = []golden{
+	{[]byte{0xcb, 0xf2, 0x9c, 0xe4, 0x84, 0x22, 0x23, 0x25}, ""},
+	{[]byte{0xaf, 0x63, 0xbd, 0x4c, 0x86, 0x01, 0xb7, 0xbe}, "a"},
+	{[]byte{0x08, 0x32, 0x67, 0x07, 0xb4, 0xeb, 0x37, 0xb8}, "ab"},
+	{[]byte{0xd8, 0xdc, 0xca, 0x18, 0x6b, 0xaf, 0xad, 0xcb}, "abc"},
+}
+
+var golden64a = []golden{
+	{[]byte{0xcb, 0xf2, 0x9c, 0xe4, 0x84, 0x22, 0x23, 0x25}, ""},
+	{[]byte{0xaf, 0x63, 0xdc, 0x4c, 0x86, 0x01, 0xec, 0x8c}, "a"},
+	{[]byte{0x08, 0x9c, 0x44, 0x07, 0xb5, 0x45, 0x98, 0x6a}, "ab"},
+	{[]byte{0xe7, 0x1f, 0xa2, 0x19, 0x05, 0x41, 0x57, 0x4b}, "abc"},
+}
+
+func TestGolden32(t *testing.T) {
+	testGolden(t, New32(), golden32)
+}
+
+func TestGolden32a(t *testing.T) {
+	testGolden(t, New32a(), golden32a)
+}
+
+func TestGolden64(t *testing.T) {
+	testGolden(t, New64(), golden64)
+}
+
+func TestGolden64a(t *testing.T) {
+	testGolden(t, New64a(), golden64a)
+}
+
+func testGolden(t *testing.T, hash hash.Hash, gold []golden) {
+	for _, g := range gold {
+		hash.Reset()
+		done, error := hash.Write([]byte(g.text))
+		if error != nil {
+			t.Fatalf("write error: %s", error)
+		}
+		if done != len(g.text) {
+			t.Fatalf("wrote only %d out of %d bytes", done, len(g.text))
+		}
+		if actual := hash.Sum(); !bytes.Equal(g.sum, actual) {
+			t.Errorf("hash(%q) = 0x%x want 0x%x", g.text, actual, g.sum)
+		}
+	}
+}
+
+func TestIntegrity32(t *testing.T) {
+	testIntegrity(t, New32())
+}
+
+func TestIntegrity32a(t *testing.T) {
+	testIntegrity(t, New32a())
+}
+
+func TestIntegrity64(t *testing.T) {
+	testIntegrity(t, New64())
+}
+
+func TestIntegrity64a(t *testing.T) {
+	testIntegrity(t, New64a())
+}
+
+func testIntegrity(t *testing.T, h hash.Hash) {
+	data := []byte{'1', '2', 3, 4, 5}
+	h.Write(data)
+	sum := h.Sum()
+
+	if size := h.Size(); size != len(sum) {
+		t.Fatalf("Size()=%d but len(Sum())=%d", size, len(sum))
+	}
+
+	if a := h.Sum(); !bytes.Equal(sum, a) {
+		t.Fatalf("first Sum()=0x%x, second Sum()=0x%x", sum, a)
+	}
+
+	h.Reset()
+	h.Write(data)
+	if a := h.Sum(); !bytes.Equal(sum, a) {
+		t.Fatalf("Sum()=0x%x, but after Reset() Sum()=0x%x", sum, a)
+	}
+
+	h.Reset()
+	h.Write(data[:2])
+	h.Write(data[2:])
+	if a := h.Sum(); !bytes.Equal(sum, a) {
+		t.Fatalf("Sum()=0x%x, but with partial writes, Sum()=0x%x", sum, a)
+	}
+
+	switch h.Size() {
+	case 4:
+		sum32 := h.(hash.Hash32).Sum32()
+		if sum32 != binary.BigEndian.Uint32(sum) {
+			t.Fatalf("Sum()=0x%x, but Sum32()=0x%x", sum, sum32)
+		}
+	case 8:
+		sum64 := h.(hash.Hash64).Sum64()
+		if sum64 != binary.BigEndian.Uint64(sum) {
+			t.Fatalf("Sum()=0x%x, but Sum64()=0x%x", sum, sum64)
+		}
+	}
+}
+
+func Benchmark32(b *testing.B) {
+	benchmark(b, New32())
+}
+
+func Benchmark32a(b *testing.B) {
+	benchmark(b, New32a())
+}
+
+func Benchmark64(b *testing.B) {
+	benchmark(b, New64())
+}
+
+func Benchmark64a(b *testing.B) {
+	benchmark(b, New64a())
+}
+
+func benchmark(b *testing.B, h hash.Hash) {
+	b.ResetTimer()
+	b.SetBytes(testDataSize)
+	data := make([]byte, testDataSize)
+	for i := range data {
+		data[i] = byte(i + 'a')
+	}
+
+	b.StartTimer()
+	for todo := b.N; todo != 0; todo-- {
+		h.Reset()
+		h.Write(data)
+		h.Sum()
+	}
+}
diff --git a/src/pkg/hash/hash.go b/src/pkg/hash/hash.go
new file mode 100644
index 000000000..3536c0b6a
--- /dev/null
+++ b/src/pkg/hash/hash.go
@@ -0,0 +1,37 @@
+// Copyright 2009 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// Package hash provides interfaces for hash functions.
+package hash
+
+import "io"
+
+// Hash is the common interface implemented by all hash functions.
+type Hash interface {
+	// Write adds more data to the running hash.
+	// It never returns an error.
+	io.Writer
+
+	// Sum returns the current hash, without changing the
+	// underlying hash state.
+	Sum() []byte
+
+	// Reset resets the hash to one with zero bytes written.
+	Reset()
+
+	// Size returns the number of bytes Sum will return.
+	Size() int
+}
+
+// Hash32 is the common interface implemented by all 32-bit hash functions.
+type Hash32 interface {
+	Hash
+	Sum32() uint32
+}
+
+// Hash64 is the common interface implemented by all 64-bit hash functions.
+type Hash64 interface {
+	Hash
+	Sum64() uint64
+}
diff --git a/src/pkg/hash/test_cases.txt b/src/pkg/hash/test_cases.txt
new file mode 100644
index 000000000..26d3ccc05
--- /dev/null
+++ b/src/pkg/hash/test_cases.txt
@@ -0,0 +1,31 @@
+
+a
+ab
+abc
+abcd
+abcde
+abcdef
+abcdefg
+abcdefgh
+abcdefghi
+abcdefghij
+Discard medicine more than two years old.
+He who has a shady past knows that nice guys finish last.
+I wouldn't marry him with a ten foot pole.
+Free! Free!/A trip/to Mars/for 900/empty jars/Burma Shave
+The days of the digital watch are numbered.  -Tom Stoppard
+Nepal premier won't resign.
+For every action there is an equal and opposite government program.
+His money is twice tainted: 'taint yours and 'taint mine.
+There is no reason for any individual to have a computer in their home. -Ken Olsen, 1977
+It's a tiny change to the code and not completely disgusting. - Bob Manchek
+size:  a.out:  bad magic
+The major problem is with sendmail.  -Mark Horton
+Give me a rock, paper and scissors and I will move the world.  CCFestoon
+If the enemy is within range, then so are you.
+It's well we cannot hear the screams/That we create in others' dreams.
+You remind me of a TV show, but that's all right: I watch it anyway.
+C is as portable as Stonehedge!!
+Even if I could be Shakespeare, I think I should still choose to be Faraday. - A. Huxley
+The fugacity of a constituent in a mixture of gases at a given temperature is proportional to its mole fraction.  Lewis-Randall Rule
+How can you write a big system without C++?  -Paul Glick
diff --git a/src/pkg/hash/test_gen.awk b/src/pkg/hash/test_gen.awk
new file mode 100644
index 000000000..804f78679
--- /dev/null
+++ b/src/pkg/hash/test_gen.awk
@@ -0,0 +1,14 @@
+# Copyright 2009 The Go Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style
+# license that can be found in the LICENSE file.
+
+# awk -f test_gen.awk test_cases.txt
+# generates test case table.
+# edit next line to set particular reference implementation and name.
+BEGIN { cmd = "echo -n `9 sha1sum`"; name = "Sha1Test" }
+{
+	printf("\t%s{ \"", name);
+	printf("%s", $0) |cmd;
+	close(cmd);
+	printf("\", \"%s\" },\n", $0);
+}
diff --git a/src/pkg/html/Makefile b/src/pkg/html/Makefile
new file mode 100644
index 000000000..28dc1a3f5
--- /dev/null
+++ b/src/pkg/html/Makefile
@@ -0,0 +1,17 @@
+# Copyright 2010 The Go Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style
+# license that can be found in the LICENSE file.
+
+include ../../Make.inc
+
+TARG=html
+GOFILES=\
+	const.go\
+	doc.go\
+	entity.go\
+	escape.go\
+	node.go\
+	parse.go\
+	token.go\
+
+include ../../Make.pkg
diff --git a/src/pkg/html/const.go b/src/pkg/html/const.go
new file mode 100644
index 000000000..9078d2601
--- /dev/null
+++ b/src/pkg/html/const.go
@@ -0,0 +1,90 @@
+// Copyright 2011 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package html
+
+// Section 11.2.3.2 of the HTML5 specification says "The following elements
+// have varying levels of special parsing rules".
+// http://www.whatwg.org/specs/web-apps/current-work/multipage/parsing.html#the-stack-of-open-elements
+var isSpecialElement = map[string]bool{
+	"address":    true,
+	"applet":     true,
+	"area":       true,
+	"article":    true,
+	"aside":      true,
+	"base":       true,
+	"basefont":   true,
+	"bgsound":    true,
+	"blockquote": true,
+	"body":       true,
+	"br":         true,
+	"button":     true,
+	"caption":    true,
+	"center":     true,
+	"col":        true,
+	"colgroup":   true,
+	"command":    true,
+	"dd":         true,
+	"details":    true,
+	"dir":        true,
+	"div":        true,
+	"dl":         true,
+	"dt":         true,
+	"embed":      true,
+	"fieldset":   true,
+	"figcaption": true,
+	"figure":     true,
+	"footer":     true,
+	"form":       true,
+	"frame":      true,
+	"frameset":   true,
+	"h1":         true,
+	"h2":         true,
+	"h3":         true,
+	"h4":         true,
+	"h5":         true,
+	"h6":         true,
+	"head":       true,
+	"header":     true,
+	"hgroup":     true,
+	"hr":         true,
+	"html":       true,
+	"iframe":     true,
+	"img":        true,
+	"input":      true,
+	"isindex":    true,
+	"li":         true,
+	"link":       true,
+	"listing":    true,
+	"marquee":    true,
+	"menu":       true,
+	"meta":       true,
+	"nav":        true,
+	"noembed":    true,
+	"noframes":   true,
+	"noscript":   true,
+	"object":     true,
+	"ol":         true,
+	"p":          true,
+	"param":      true,
+	"plaintext":  true,
+	"pre":        true,
+	"script":     true,
+	"section":    true,
+	"select":     true,
+	"style":      true,
+	"summary":    true,
+	"table":      true,
+	"tbody":      true,
+	"td":         true,
+	"textarea":   true,
+	"tfoot":      true,
+	"th":         true,
+	"thead":      true,
+	"title":      true,
+	"tr":         true,
+	"ul":         true,
+	"wbr":        true,
+	"xmp":        true,
+}
diff --git a/src/pkg/html/doc.go b/src/pkg/html/doc.go
new file mode 100644
index 000000000..5bc063086
--- /dev/null
+++ b/src/pkg/html/doc.go
@@ -0,0 +1,110 @@
+// Copyright 2010 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+/*
+Package html implements an HTML5-compliant tokenizer and parser.
+INCOMPLETE.
+
+Tokenization is done by creating a Tokenizer for an io.Reader r. It is the
+caller's responsibility to ensure that r provides UTF-8 encoded HTML.
+
+	z := html.NewTokenizer(r)
+
+Given a Tokenizer z, the HTML is tokenized by repeatedly calling z.Next(),
+which parses the next token and returns its type, or an error:
+
+	for {
+		tt := z.Next()
+		if tt == html.ErrorToken {
+			// ...
+			return ...
+		}
+		// Process the current token.
+	}
+
+There are two APIs for retrieving the current token. The high-level API is to
+call Token; the low-level API is to call Text or TagName / TagAttr. Both APIs
+allow optionally calling Raw after Next but before Token, Text, TagName, or
+TagAttr. In EBNF notation, the valid call sequence per token is:
+
+	Next {Raw} [ Token | Text | TagName {TagAttr} ]
+
+Token returns an independent data structure that completely describes a token.
+Entities (such as "<") are unescaped, tag names and attribute keys are
+lower-cased, and attributes are collected into a []Attribute. For example:
+
+	for {
+		if z.Next() == html.ErrorToken {
+			// Returning os.EOF indicates success.
+			return z.Error()
+		}
+		emitToken(z.Token())
+	}
+
+The low-level API performs fewer allocations and copies, but the contents of
+the []byte values returned by Text, TagName and TagAttr may change on the next
+call to Next. For example, to extract an HTML page's anchor text:
+
+	depth := 0
+	for {
+		tt := z.Next()
+		switch tt {
+		case ErrorToken:
+			return z.Error()
+		case TextToken:
+			if depth > 0 {
+				// emitBytes should copy the []byte it receives,
+				// if it doesn't process it immediately.
+				emitBytes(z.Text())
+			}
+		case StartTagToken, EndTagToken:
+			tn, _ := z.TagName()
+			if len(tn) == 1 && tn[0] == 'a' {
+				if tt == StartTag {
+					depth++
+				} else {
+					depth--
+				}
+			}
+		}
+	}
+
+A Tokenizer typically skips over HTML comments. To return comment tokens, set
+Tokenizer.ReturnComments to true before looping over calls to Next.
+
+Parsing is done by calling Parse with an io.Reader, which returns the root of
+the parse tree (the document element) as a *Node. It is the caller's
+responsibility to ensure that the Reader provides UTF-8 encoded HTML. For
+example, to process each anchor node in depth-first order:
+
+	doc, err := html.Parse(r)
+	if err != nil {
+		// ...
+	}
+	var f func(*html.Node)
+	f = func(n *html.Node) {
+		if n.Type == html.ElementNode && n.Data == "a" {
+			// Do something with n...
+		}
+		for _, c := range n.Child {
+			f(c)
+		}
+	}
+	f(doc)
+
+The relevant specifications include:
+http://www.whatwg.org/specs/web-apps/current-work/multipage/syntax.html and
+http://www.whatwg.org/specs/web-apps/current-work/multipage/tokenization.html
+*/
+package html
+
+// The tokenization algorithm implemented by this package is not a line-by-line
+// transliteration of the relatively verbose state-machine in the WHATWG
+// specification. A more direct approach is used instead, where the program
+// counter implies the state, such as whether it is tokenizing a tag or a text
+// node. Specification compliance is verified by checking expected and actual
+// outputs over a test suite rather than aiming for algorithmic fidelity.
+
+// TODO(nigeltao): Does a DOM API belong in this package or a separate one?
+// TODO(nigeltao): How does parsing interact with a JavaScript engine?
diff --git a/src/pkg/html/entity.go b/src/pkg/html/entity.go
new file mode 100644
index 000000000..21263e22d
--- /dev/null
+++ b/src/pkg/html/entity.go
@@ -0,0 +1,2253 @@
+// Copyright 2010 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package html
+
+// All entities that do not end with ';' are 6 or fewer bytes long.
+const longestEntityWithoutSemicolon = 6
+
+// entity is a map from HTML entity names to their values. The semicolon matters:
+// http://www.whatwg.org/specs/web-apps/current-work/multipage/named-character-references.html
+// lists both "amp" and "amp;" as two separate entries.
+//
+// Note that the HTML5 list is larger than the HTML4 list at
+// http://www.w3.org/TR/html4/sgml/entities.html
+var entity = map[string]int{
+	"AElig;":                           '\U000000C6',
+	"AMP;":                             '\U00000026',
+	"Aacute;":                          '\U000000C1',
+	"Abreve;":                          '\U00000102',
+	"Acirc;":                           '\U000000C2',
+	"Acy;":                             '\U00000410',
+	"Afr;":                             '\U0001D504',
+	"Agrave;":                          '\U000000C0',
+	"Alpha;":                           '\U00000391',
+	"Amacr;":                           '\U00000100',
+	"And;":                             '\U00002A53',
+	"Aogon;":                           '\U00000104',
+	"Aopf;":                            '\U0001D538',
+	"ApplyFunction;":                   '\U00002061',
+	"Aring;":                           '\U000000C5',
+	"Ascr;":                            '\U0001D49C',
+	"Assign;":                          '\U00002254',
+	"Atilde;":                          '\U000000C3',
+	"Auml;":                            '\U000000C4',
+	"Backslash;":                       '\U00002216',
+	"Barv;":                            '\U00002AE7',
+	"Barwed;":                          '\U00002306',
+	"Bcy;":                             '\U00000411',
+	"Because;":                         '\U00002235',
+	"Bernoullis;":                      '\U0000212C',
+	"Beta;":                            '\U00000392',
+	"Bfr;":                             '\U0001D505',
+	"Bopf;":                            '\U0001D539',
+	"Breve;":                           '\U000002D8',
+	"Bscr;":                            '\U0000212C',
+	"Bumpeq;":                          '\U0000224E',
+	"CHcy;":                            '\U00000427',
+	"COPY;":                            '\U000000A9',
+	"Cacute;":                          '\U00000106',
+	"Cap;":                             '\U000022D2',
+	"CapitalDifferentialD;":            '\U00002145',
+	"Cayleys;":                         '\U0000212D',
+	"Ccaron;":                          '\U0000010C',
+	"Ccedil;":                          '\U000000C7',
+	"Ccirc;":                           '\U00000108',
+	"Cconint;":                         '\U00002230',
+	"Cdot;":                            '\U0000010A',
+	"Cedilla;":                         '\U000000B8',
+	"CenterDot;":                       '\U000000B7',
+	"Cfr;":                             '\U0000212D',
+	"Chi;":                             '\U000003A7',
+	"CircleDot;":                       '\U00002299',
+	"CircleMinus;":                     '\U00002296',
+	"CirclePlus;":                      '\U00002295',
+	"CircleTimes;":                     '\U00002297',
+	"ClockwiseContourIntegral;":        '\U00002232',
+	"CloseCurlyDoubleQuote;":           '\U0000201D',
+	"CloseCurlyQuote;":                 '\U00002019',
+	"Colon;":                           '\U00002237',
+	"Colone;":                          '\U00002A74',
+	"Congruent;":                       '\U00002261',
+	"Conint;":                          '\U0000222F',
+	"ContourIntegral;":                 '\U0000222E',
+	"Copf;":                            '\U00002102',
+	"Coproduct;":                       '\U00002210',
+	"CounterClockwiseContourIntegral;": '\U00002233',
+	"Cross;":                           '\U00002A2F',
+	"Cscr;":                            '\U0001D49E',
+	"Cup;":                             '\U000022D3',
+	"CupCap;":                          '\U0000224D',
+	"DD;":                              '\U00002145',
+	"DDotrahd;":                        '\U00002911',
+	"DJcy;":                            '\U00000402',
+	"DScy;":                            '\U00000405',
+	"DZcy;":                            '\U0000040F',
+	"Dagger;":                          '\U00002021',
+	"Darr;":                            '\U000021A1',
+	"Dashv;":                           '\U00002AE4',
+	"Dcaron;":                          '\U0000010E',
+	"Dcy;":                             '\U00000414',
+	"Del;":                             '\U00002207',
+	"Delta;":                           '\U00000394',
+	"Dfr;":                             '\U0001D507',
+	"DiacriticalAcute;":                '\U000000B4',
+	"DiacriticalDot;":                  '\U000002D9',
+	"DiacriticalDoubleAcute;":          '\U000002DD',
+	"DiacriticalGrave;":                '\U00000060',
+	"DiacriticalTilde;":                '\U000002DC',
+	"Diamond;":                         '\U000022C4',
+	"DifferentialD;":                   '\U00002146',
+	"Dopf;":                            '\U0001D53B',
+	"Dot;":                             '\U000000A8',
+	"DotDot;":                          '\U000020DC',
+	"DotEqual;":                        '\U00002250',
+	"DoubleContourIntegral;":           '\U0000222F',
+	"DoubleDot;":                       '\U000000A8',
+	"DoubleDownArrow;":                 '\U000021D3',
+	"DoubleLeftArrow;":                 '\U000021D0',
+	"DoubleLeftRightArrow;":            '\U000021D4',
+	"DoubleLeftTee;":                   '\U00002AE4',
+	"DoubleLongLeftArrow;":             '\U000027F8',
+	"DoubleLongLeftRightArrow;":        '\U000027FA',
+	"DoubleLongRightArrow;":            '\U000027F9',
+	"DoubleRightArrow;":                '\U000021D2',
+	"DoubleRightTee;":                  '\U000022A8',
+	"DoubleUpArrow;":                   '\U000021D1',
+	"DoubleUpDownArrow;":               '\U000021D5',
+	"DoubleVerticalBar;":               '\U00002225',
+	"DownArrow;":                       '\U00002193',
+	"DownArrowBar;":                    '\U00002913',
+	"DownArrowUpArrow;":                '\U000021F5',
+	"DownBreve;":                       '\U00000311',
+	"DownLeftRightVector;":             '\U00002950',
+	"DownLeftTeeVector;":               '\U0000295E',
+	"DownLeftVector;":                  '\U000021BD',
+	"DownLeftVectorBar;":               '\U00002956',
+	"DownRightTeeVector;":              '\U0000295F',
+	"DownRightVector;":                 '\U000021C1',
+	"DownRightVectorBar;":              '\U00002957',
+	"DownTee;":                         '\U000022A4',
+	"DownTeeArrow;":                    '\U000021A7',
+	"Downarrow;":                       '\U000021D3',
+	"Dscr;":                            '\U0001D49F',
+	"Dstrok;":                          '\U00000110',
+	"ENG;":                             '\U0000014A',
+	"ETH;":                             '\U000000D0',
+	"Eacute;":                          '\U000000C9',
+	"Ecaron;":                          '\U0000011A',
+	"Ecirc;":                           '\U000000CA',
+	"Ecy;":                             '\U0000042D',
+	"Edot;":                            '\U00000116',
+	"Efr;":                             '\U0001D508',
+	"Egrave;":                          '\U000000C8',
+	"Element;":                         '\U00002208',
+	"Emacr;":                           '\U00000112',
+	"EmptySmallSquare;":                '\U000025FB',
+	"EmptyVerySmallSquare;":            '\U000025AB',
+	"Eogon;":                           '\U00000118',
+	"Eopf;":                            '\U0001D53C',
+	"Epsilon;":                         '\U00000395',
+	"Equal;":                           '\U00002A75',
+	"EqualTilde;":                      '\U00002242',
+	"Equilibrium;":                     '\U000021CC',
+	"Escr;":                            '\U00002130',
+	"Esim;":                            '\U00002A73',
+	"Eta;":                             '\U00000397',
+	"Euml;":                            '\U000000CB',
+	"Exists;":                          '\U00002203',
+	"ExponentialE;":                    '\U00002147',
+	"Fcy;":                             '\U00000424',
+	"Ffr;":                             '\U0001D509',
+	"FilledSmallSquare;":               '\U000025FC',
+	"FilledVerySmallSquare;":           '\U000025AA',
+	"Fopf;":                            '\U0001D53D',
+	"ForAll;":                          '\U00002200',
+	"Fouriertrf;":                      '\U00002131',
+	"Fscr;":                            '\U00002131',
+	"GJcy;":                            '\U00000403',
+	"GT;":                              '\U0000003E',
+	"Gamma;":                           '\U00000393',
+	"Gammad;":                          '\U000003DC',
+	"Gbreve;":                          '\U0000011E',
+	"Gcedil;":                          '\U00000122',
+	"Gcirc;":                           '\U0000011C',
+	"Gcy;":                             '\U00000413',
+	"Gdot;":                            '\U00000120',
+	"Gfr;":                             '\U0001D50A',
+	"Gg;":                              '\U000022D9',
+	"Gopf;":                            '\U0001D53E',
+	"GreaterEqual;":                    '\U00002265',
+	"GreaterEqualLess;":                '\U000022DB',
+	"GreaterFullEqual;":                '\U00002267',
+	"GreaterGreater;":                  '\U00002AA2',
+	"GreaterLess;":                     '\U00002277',
+	"GreaterSlantEqual;":               '\U00002A7E',
+	"GreaterTilde;":                    '\U00002273',
+	"Gscr;":                            '\U0001D4A2',
+	"Gt;":                              '\U0000226B',
+	"HARDcy;":                          '\U0000042A',
+	"Hacek;":                           '\U000002C7',
+	"Hat;":                             '\U0000005E',
+	"Hcirc;":                           '\U00000124',
+	"Hfr;":                             '\U0000210C',
+	"HilbertSpace;":                    '\U0000210B',
+	"Hopf;":                            '\U0000210D',
+	"HorizontalLine;":                  '\U00002500',
+	"Hscr;":                            '\U0000210B',
+	"Hstrok;":                          '\U00000126',
+	"HumpDownHump;":                    '\U0000224E',
+	"HumpEqual;":                       '\U0000224F',
+	"IEcy;":                            '\U00000415',
+	"IJlig;":                           '\U00000132',
+	"IOcy;":                            '\U00000401',
+	"Iacute;":                          '\U000000CD',
+	"Icirc;":                           '\U000000CE',
+	"Icy;":                             '\U00000418',
+	"Idot;":                            '\U00000130',
+	"Ifr;":                             '\U00002111',
+	"Igrave;":                          '\U000000CC',
+	"Im;":                              '\U00002111',
+	"Imacr;":                           '\U0000012A',
+	"ImaginaryI;":                      '\U00002148',
+	"Implies;":                         '\U000021D2',
+	"Int;":                             '\U0000222C',
+	"Integral;":                        '\U0000222B',
+	"Intersection;":                    '\U000022C2',
+	"InvisibleComma;":                  '\U00002063',
+	"InvisibleTimes;":                  '\U00002062',
+	"Iogon;":                           '\U0000012E',
+	"Iopf;":                            '\U0001D540',
+	"Iota;":                            '\U00000399',
+	"Iscr;":                            '\U00002110',
+	"Itilde;":                          '\U00000128',
+	"Iukcy;":                           '\U00000406',
+	"Iuml;":                            '\U000000CF',
+	"Jcirc;":                           '\U00000134',
+	"Jcy;":                             '\U00000419',
+	"Jfr;":                             '\U0001D50D',
+	"Jopf;":                            '\U0001D541',
+	"Jscr;":                            '\U0001D4A5',
+	"Jsercy;":                          '\U00000408',
+	"Jukcy;":                           '\U00000404',
+	"KHcy;":                            '\U00000425',
+	"KJcy;":                            '\U0000040C',
+	"Kappa;":                           '\U0000039A',
+	"Kcedil;":                          '\U00000136',
+	"Kcy;":                             '\U0000041A',
+	"Kfr;":                             '\U0001D50E',
+	"Kopf;":                            '\U0001D542',
+	"Kscr;":                            '\U0001D4A6',
+	"LJcy;":                            '\U00000409',
+	"LT;":                              '\U0000003C',
+	"Lacute;":                          '\U00000139',
+	"Lambda;":                          '\U0000039B',
+	"Lang;":                            '\U000027EA',
+	"Laplacetrf;":                      '\U00002112',
+	"Larr;":                            '\U0000219E',
+	"Lcaron;":                          '\U0000013D',
+	"Lcedil;":                          '\U0000013B',
+	"Lcy;":                             '\U0000041B',
+	"LeftAngleBracket;":                '\U000027E8',
+	"LeftArrow;":                       '\U00002190',
+	"LeftArrowBar;":                    '\U000021E4',
+	"LeftArrowRightArrow;":             '\U000021C6',
+	"LeftCeiling;":                     '\U00002308',
+	"LeftDoubleBracket;":               '\U000027E6',
+	"LeftDownTeeVector;":               '\U00002961',
+	"LeftDownVector;":                  '\U000021C3',
+	"LeftDownVectorBar;":               '\U00002959',
+	"LeftFloor;":                       '\U0000230A',
+	"LeftRightArrow;":                  '\U00002194',
+	"LeftRightVector;":                 '\U0000294E',
+	"LeftTee;":                         '\U000022A3',
+	"LeftTeeArrow;":                    '\U000021A4',
+	"LeftTeeVector;":                   '\U0000295A',
+	"LeftTriangle;":                    '\U000022B2',
+	"LeftTriangleBar;":                 '\U000029CF',
+	"LeftTriangleEqual;":               '\U000022B4',
+	"LeftUpDownVector;":                '\U00002951',
+	"LeftUpTeeVector;":                 '\U00002960',
+	"LeftUpVector;":                    '\U000021BF',
+	"LeftUpVectorBar;":                 '\U00002958',
+	"LeftVector;":                      '\U000021BC',
+	"LeftVectorBar;":                   '\U00002952',
+	"Leftarrow;":                       '\U000021D0',
+	"Leftrightarrow;":                  '\U000021D4',
+	"LessEqualGreater;":                '\U000022DA',
+	"LessFullEqual;":                   '\U00002266',
+	"LessGreater;":                     '\U00002276',
+	"LessLess;":                        '\U00002AA1',
+	"LessSlantEqual;":                  '\U00002A7D',
+	"LessTilde;":                       '\U00002272',
+	"Lfr;":                             '\U0001D50F',
+	"Ll;":                              '\U000022D8',
+	"Lleftarrow;":                      '\U000021DA',
+	"Lmidot;":                          '\U0000013F',
+	"LongLeftArrow;":                   '\U000027F5',
+	"LongLeftRightArrow;":              '\U000027F7',
+	"LongRightArrow;":                  '\U000027F6',
+	"Longleftarrow;":                   '\U000027F8',
+	"Longleftrightarrow;":              '\U000027FA',
+	"Longrightarrow;":                  '\U000027F9',
+	"Lopf;":                            '\U0001D543',
+	"LowerLeftArrow;":                  '\U00002199',
+	"LowerRightArrow;":                 '\U00002198',
+	"Lscr;":                            '\U00002112',
+	"Lsh;":                             '\U000021B0',
+	"Lstrok;":                          '\U00000141',
+	"Lt;":                              '\U0000226A',
+	"Map;":                             '\U00002905',
+	"Mcy;":                             '\U0000041C',
+	"MediumSpace;":                     '\U0000205F',
+	"Mellintrf;":                       '\U00002133',
+	"Mfr;":                             '\U0001D510',
+	"MinusPlus;":                       '\U00002213',
+	"Mopf;":                            '\U0001D544',
+	"Mscr;":                            '\U00002133',
+	"Mu;":                              '\U0000039C',
+	"NJcy;":                            '\U0000040A',
+	"Nacute;":                          '\U00000143',
+	"Ncaron;":                          '\U00000147',
+	"Ncedil;":                          '\U00000145',
+	"Ncy;":                             '\U0000041D',
+	"NegativeMediumSpace;":             '\U0000200B',
+	"NegativeThickSpace;":              '\U0000200B',
+	"NegativeThinSpace;":               '\U0000200B',
+	"NegativeVeryThinSpace;":           '\U0000200B',
+	"NestedGreaterGreater;":            '\U0000226B',
+	"NestedLessLess;":                  '\U0000226A',
+	"NewLine;":                         '\U0000000A',
+	"Nfr;":                             '\U0001D511',
+	"NoBreak;":                         '\U00002060',
+	"NonBreakingSpace;":                '\U000000A0',
+	"Nopf;":                            '\U00002115',
+	"Not;":                             '\U00002AEC',
+	"NotCongruent;":                    '\U00002262',
+	"NotCupCap;":                       '\U0000226D',
+	"NotDoubleVerticalBar;":            '\U00002226',
+	"NotElement;":                      '\U00002209',
+	"NotEqual;":                        '\U00002260',
+	"NotExists;":                       '\U00002204',
+	"NotGreater;":                      '\U0000226F',
+	"NotGreaterEqual;":                 '\U00002271',
+	"NotGreaterLess;":                  '\U00002279',
+	"NotGreaterTilde;":                 '\U00002275',
+	"NotLeftTriangle;":                 '\U000022EA',
+	"NotLeftTriangleEqual;":            '\U000022EC',
+	"NotLess;":                         '\U0000226E',
+	"NotLessEqual;":                    '\U00002270',
+	"NotLessGreater;":                  '\U00002278',
+	"NotLessTilde;":                    '\U00002274',
+	"NotPrecedes;":                     '\U00002280',
+	"NotPrecedesSlantEqual;":           '\U000022E0',
+	"NotReverseElement;":               '\U0000220C',
+	"NotRightTriangle;":                '\U000022EB',
+	"NotRightTriangleEqual;":           '\U000022ED',
+	"NotSquareSubsetEqual;":            '\U000022E2',
+	"NotSquareSupersetEqual;":          '\U000022E3',
+	"NotSubsetEqual;":                  '\U00002288',
+	"NotSucceeds;":                     '\U00002281',
+	"NotSucceedsSlantEqual;":           '\U000022E1',
+	"NotSupersetEqual;":                '\U00002289',
+	"NotTilde;":                        '\U00002241',
+	"NotTildeEqual;":                   '\U00002244',
+	"NotTildeFullEqual;":               '\U00002247',
+	"NotTildeTilde;":                   '\U00002249',
+	"NotVerticalBar;":                  '\U00002224',
+	"Nscr;":                            '\U0001D4A9',
+	"Ntilde;":                          '\U000000D1',
+	"Nu;":                              '\U0000039D',
+	"OElig;":                           '\U00000152',
+	"Oacute;":                          '\U000000D3',
+	"Ocirc;":                           '\U000000D4',
+	"Ocy;":                             '\U0000041E',
+	"Odblac;":                          '\U00000150',
+	"Ofr;":                             '\U0001D512',
+	"Ograve;":                          '\U000000D2',
+	"Omacr;":                           '\U0000014C',
+	"Omega;":                           '\U000003A9',
+	"Omicron;":                         '\U0000039F',
+	"Oopf;":                            '\U0001D546',
+	"OpenCurlyDoubleQuote;":            '\U0000201C',
+	"OpenCurlyQuote;":                  '\U00002018',
+	"Or;":                              '\U00002A54',
+	"Oscr;":                            '\U0001D4AA',
+	"Oslash;":                          '\U000000D8',
+	"Otilde;":                          '\U000000D5',
+	"Otimes;":                          '\U00002A37',
+	"Ouml;":                            '\U000000D6',
+	"OverBar;":                         '\U0000203E',
+	"OverBrace;":                       '\U000023DE',
+	"OverBracket;":                     '\U000023B4',
+	"OverParenthesis;":                 '\U000023DC',
+	"PartialD;":                        '\U00002202',
+	"Pcy;":                             '\U0000041F',
+	"Pfr;":                             '\U0001D513',
+	"Phi;":                             '\U000003A6',
+	"Pi;":                              '\U000003A0',
+	"PlusMinus;":                       '\U000000B1',
+	"Poincareplane;":                   '\U0000210C',
+	"Popf;":                            '\U00002119',
+	"Pr;":                              '\U00002ABB',
+	"Precedes;":                        '\U0000227A',
+	"PrecedesEqual;":                   '\U00002AAF',
+	"PrecedesSlantEqual;":              '\U0000227C',
+	"PrecedesTilde;":                   '\U0000227E',
+	"Prime;":                           '\U00002033',
+	"Product;":                         '\U0000220F',
+	"Proportion;":                      '\U00002237',
+	"Proportional;":                    '\U0000221D',
+	"Pscr;":                            '\U0001D4AB',
+	"Psi;":                             '\U000003A8',
+	"QUOT;":                            '\U00000022',
+	"Qfr;":                             '\U0001D514',
+	"Qopf;":                            '\U0000211A',
+	"Qscr;":                            '\U0001D4AC',
+	"RBarr;":                           '\U00002910',
+	"REG;":                             '\U000000AE',
+	"Racute;":                          '\U00000154',
+	"Rang;":                            '\U000027EB',
+	"Rarr;":                            '\U000021A0',
+	"Rarrtl;":                          '\U00002916',
+	"Rcaron;":                          '\U00000158',
+	"Rcedil;":                          '\U00000156',
+	"Rcy;":                             '\U00000420',
+	"Re;":                              '\U0000211C',
+	"ReverseElement;":                  '\U0000220B',
+	"ReverseEquilibrium;":              '\U000021CB',
+	"ReverseUpEquilibrium;":            '\U0000296F',
+	"Rfr;":                             '\U0000211C',
+	"Rho;":                             '\U000003A1',
+	"RightAngleBracket;":               '\U000027E9',
+	"RightArrow;":                      '\U00002192',
+	"RightArrowBar;":                   '\U000021E5',
+	"RightArrowLeftArrow;":             '\U000021C4',
+	"RightCeiling;":                    '\U00002309',
+	"RightDoubleBracket;":              '\U000027E7',
+	"RightDownTeeVector;":              '\U0000295D',
+	"RightDownVector;":                 '\U000021C2',
+	"RightDownVectorBar;":              '\U00002955',
+	"RightFloor;":                      '\U0000230B',
+	"RightTee;":                        '\U000022A2',
+	"RightTeeArrow;":                   '\U000021A6',
+	"RightTeeVector;":                  '\U0000295B',
+	"RightTriangle;":                   '\U000022B3',
+	"RightTriangleBar;":                '\U000029D0',
+	"RightTriangleEqual;":              '\U000022B5',
+	"RightUpDownVector;":               '\U0000294F',
+	"RightUpTeeVector;":                '\U0000295C',
+	"RightUpVector;":                   '\U000021BE',
+	"RightUpVectorBar;":                '\U00002954',
+	"RightVector;":                     '\U000021C0',
+	"RightVectorBar;":                  '\U00002953',
+	"Rightarrow;":                      '\U000021D2',
+	"Ropf;":                            '\U0000211D',
+	"RoundImplies;":                    '\U00002970',
+	"Rrightarrow;":                     '\U000021DB',
+	"Rscr;":                            '\U0000211B',
+	"Rsh;":                             '\U000021B1',
+	"RuleDelayed;":                     '\U000029F4',
+	"SHCHcy;":                          '\U00000429',
+	"SHcy;":                            '\U00000428',
+	"SOFTcy;":                          '\U0000042C',
+	"Sacute;":                          '\U0000015A',
+	"Sc;":                              '\U00002ABC',
+	"Scaron;":                          '\U00000160',
+	"Scedil;":                          '\U0000015E',
+	"Scirc;":                           '\U0000015C',
+	"Scy;":                             '\U00000421',
+	"Sfr;":                             '\U0001D516',
+	"ShortDownArrow;":                  '\U00002193',
+	"ShortLeftArrow;":                  '\U00002190',
+	"ShortRightArrow;":                 '\U00002192',
+	"ShortUpArrow;":                    '\U00002191',
+	"Sigma;":                           '\U000003A3',
+	"SmallCircle;":                     '\U00002218',
+	"Sopf;":                            '\U0001D54A',
+	"Sqrt;":                            '\U0000221A',
+	"Square;":                          '\U000025A1',
+	"SquareIntersection;":              '\U00002293',
+	"SquareSubset;":                    '\U0000228F',
+	"SquareSubsetEqual;":               '\U00002291',
+	"SquareSuperset;":                  '\U00002290',
+	"SquareSupersetEqual;":             '\U00002292',
+	"SquareUnion;":                     '\U00002294',
+	"Sscr;":                            '\U0001D4AE',
+	"Star;":                            '\U000022C6',
+	"Sub;":                             '\U000022D0',
+	"Subset;":                          '\U000022D0',
+	"SubsetEqual;":                     '\U00002286',
+	"Succeeds;":                        '\U0000227B',
+	"SucceedsEqual;":                   '\U00002AB0',
+	"SucceedsSlantEqual;":              '\U0000227D',
+	"SucceedsTilde;":                   '\U0000227F',
+	"SuchThat;":                        '\U0000220B',
+	"Sum;":                             '\U00002211',
+	"Sup;":                             '\U000022D1',
+	"Superset;":                        '\U00002283',
+	"SupersetEqual;":                   '\U00002287',
+	"Supset;":                          '\U000022D1',
+	"THORN;":                           '\U000000DE',
+	"TRADE;":                           '\U00002122',
+	"TSHcy;":                           '\U0000040B',
+	"TScy;":                            '\U00000426',
+	"Tab;":                             '\U00000009',
+	"Tau;":                             '\U000003A4',
+	"Tcaron;":                          '\U00000164',
+	"Tcedil;":                          '\U00000162',
+	"Tcy;":                             '\U00000422',
+	"Tfr;":                             '\U0001D517',
+	"Therefore;":                       '\U00002234',
+	"Theta;":                           '\U00000398',
+	"ThinSpace;":                       '\U00002009',
+	"Tilde;":                           '\U0000223C',
+	"TildeEqual;":                      '\U00002243',
+	"TildeFullEqual;":                  '\U00002245',
+	"TildeTilde;":                      '\U00002248',
+	"Topf;":                            '\U0001D54B',
+	"TripleDot;":                       '\U000020DB',
+	"Tscr;":                            '\U0001D4AF',
+	"Tstrok;":                          '\U00000166',
+	"Uacute;":                          '\U000000DA',
+	"Uarr;":                            '\U0000219F',
+	"Uarrocir;":                        '\U00002949',
+	"Ubrcy;":                           '\U0000040E',
+	"Ubreve;":                          '\U0000016C',
+	"Ucirc;":                           '\U000000DB',
+	"Ucy;":                             '\U00000423',
+	"Udblac;":                          '\U00000170',
+	"Ufr;":                             '\U0001D518',
+	"Ugrave;":                          '\U000000D9',
+	"Umacr;":                           '\U0000016A',
+	"UnderBar;":                        '\U0000005F',
+	"UnderBrace;":                      '\U000023DF',
+	"UnderBracket;":                    '\U000023B5',
+	"UnderParenthesis;":                '\U000023DD',
+	"Union;":                           '\U000022C3',
+	"UnionPlus;":                       '\U0000228E',
+	"Uogon;":                           '\U00000172',
+	"Uopf;":                            '\U0001D54C',
+	"UpArrow;":                         '\U00002191',
+	"UpArrowBar;":                      '\U00002912',
+	"UpArrowDownArrow;":                '\U000021C5',
+	"UpDownArrow;":                     '\U00002195',
+	"UpEquilibrium;":                   '\U0000296E',
+	"UpTee;":                           '\U000022A5',
+	"UpTeeArrow;":                      '\U000021A5',
+	"Uparrow;":                         '\U000021D1',
+	"Updownarrow;":                     '\U000021D5',
+	"UpperLeftArrow;":                  '\U00002196',
+	"UpperRightArrow;":                 '\U00002197',
+	"Upsi;":                            '\U000003D2',
+	"Upsilon;":                         '\U000003A5',
+	"Uring;":                           '\U0000016E',
+	"Uscr;":                            '\U0001D4B0',
+	"Utilde;":                          '\U00000168',
+	"Uuml;":                            '\U000000DC',
+	"VDash;":                           '\U000022AB',
+	"Vbar;":                            '\U00002AEB',
+	"Vcy;":                             '\U00000412',
+	"Vdash;":                           '\U000022A9',
+	"Vdashl;":                          '\U00002AE6',
+	"Vee;":                             '\U000022C1',
+	"Verbar;":                          '\U00002016',
+	"Vert;":                            '\U00002016',
+	"VerticalBar;":                     '\U00002223',
+	"VerticalLine;":                    '\U0000007C',
+	"VerticalSeparator;":               '\U00002758',
+	"VerticalTilde;":                   '\U00002240',
+	"VeryThinSpace;":                   '\U0000200A',
+	"Vfr;":                             '\U0001D519',
+	"Vopf;":                            '\U0001D54D',
+	"Vscr;":                            '\U0001D4B1',
+	"Vvdash;":                          '\U000022AA',
+	"Wcirc;":                           '\U00000174',
+	"Wedge;":                           '\U000022C0',
+	"Wfr;":                             '\U0001D51A',
+	"Wopf;":                            '\U0001D54E',
+	"Wscr;":                            '\U0001D4B2',
+	"Xfr;":                             '\U0001D51B',
+	"Xi;":                              '\U0000039E',
+	"Xopf;":                            '\U0001D54F',
+	"Xscr;":                            '\U0001D4B3',
+	"YAcy;":                            '\U0000042F',
+	"YIcy;":                            '\U00000407',
+	"YUcy;":                            '\U0000042E',
+	"Yacute;":                          '\U000000DD',
+	"Ycirc;":                           '\U00000176',
+	"Ycy;":                             '\U0000042B',
+	"Yfr;":                             '\U0001D51C',
+	"Yopf;":                            '\U0001D550',
+	"Yscr;":                            '\U0001D4B4',
+	"Yuml;":                            '\U00000178',
+	"ZHcy;":                            '\U00000416',
+	"Zacute;":                          '\U00000179',
+	"Zcaron;":                          '\U0000017D',
+	"Zcy;":                             '\U00000417',
+	"Zdot;":                            '\U0000017B',
+	"ZeroWidthSpace;":                  '\U0000200B',
+	"Zeta;":                            '\U00000396',
+	"Zfr;":                             '\U00002128',
+	"Zopf;":                            '\U00002124',
+	"Zscr;":                            '\U0001D4B5',
+	"aacute;":                          '\U000000E1',
+	"abreve;":                          '\U00000103',
+	"ac;":                              '\U0000223E',
+	"acd;":                             '\U0000223F',
+	"acirc;":                           '\U000000E2',
+	"acute;":                           '\U000000B4',
+	"acy;":                             '\U00000430',
+	"aelig;":                           '\U000000E6',
+	"af;":                              '\U00002061',
+	"afr;":                             '\U0001D51E',
+	"agrave;":                          '\U000000E0',
+	"alefsym;":                         '\U00002135',
+	"aleph;":                           '\U00002135',
+	"alpha;":                           '\U000003B1',
+	"amacr;":                           '\U00000101',
+	"amalg;":                           '\U00002A3F',
+	"amp;":                             '\U00000026',
+	"and;":                             '\U00002227',
+	"andand;":                          '\U00002A55',
+	"andd;":                            '\U00002A5C',
+	"andslope;":                        '\U00002A58',
+	"andv;":                            '\U00002A5A',
+	"ang;":                             '\U00002220',
+	"ange;":                            '\U000029A4',
+	"angle;":                           '\U00002220',
+	"angmsd;":                          '\U00002221',
+	"angmsdaa;":                        '\U000029A8',
+	"angmsdab;":                        '\U000029A9',
+	"angmsdac;":                        '\U000029AA',
+	"angmsdad;":                        '\U000029AB',
+	"angmsdae;":                        '\U000029AC',
+	"angmsdaf;":                        '\U000029AD',
+	"angmsdag;":                        '\U000029AE',
+	"angmsdah;":                        '\U000029AF',
+	"angrt;":                           '\U0000221F',
+	"angrtvb;":                         '\U000022BE',
+	"angrtvbd;":                        '\U0000299D',
+	"angsph;":                          '\U00002222',
+	"angst;":                           '\U000000C5',
+	"angzarr;":                         '\U0000237C',
+	"aogon;":                           '\U00000105',
+	"aopf;":                            '\U0001D552',
+	"ap;":                              '\U00002248',
+	"apE;":                             '\U00002A70',
+	"apacir;":                          '\U00002A6F',
+	"ape;":                             '\U0000224A',
+	"apid;":                            '\U0000224B',
+	"apos;":                            '\U00000027',
+	"approx;":                          '\U00002248',
+	"approxeq;":                        '\U0000224A',
+	"aring;":                           '\U000000E5',
+	"ascr;":                            '\U0001D4B6',
+	"ast;":                             '\U0000002A',
+	"asymp;":                           '\U00002248',
+	"asympeq;":                         '\U0000224D',
+	"atilde;":                          '\U000000E3',
+	"auml;":                            '\U000000E4',
+	"awconint;":                        '\U00002233',
+	"awint;":                           '\U00002A11',
+	"bNot;":                            '\U00002AED',
+	"backcong;":                        '\U0000224C',
+	"backepsilon;":                     '\U000003F6',
+	"backprime;":                       '\U00002035',
+	"backsim;":                         '\U0000223D',
+	"backsimeq;":                       '\U000022CD',
+	"barvee;":                          '\U000022BD',
+	"barwed;":                          '\U00002305',
+	"barwedge;":                        '\U00002305',
+	"bbrk;":                            '\U000023B5',
+	"bbrktbrk;":                        '\U000023B6',
+	"bcong;":                           '\U0000224C',
+	"bcy;":                             '\U00000431',
+	"bdquo;":                           '\U0000201E',
+	"becaus;":                          '\U00002235',
+	"because;":                         '\U00002235',
+	"bemptyv;":                         '\U000029B0',
+	"bepsi;":                           '\U000003F6',
+	"bernou;":                          '\U0000212C',
+	"beta;":                            '\U000003B2',
+	"beth;":                            '\U00002136',
+	"between;":                         '\U0000226C',
+	"bfr;":                             '\U0001D51F',
+	"bigcap;":                          '\U000022C2',
+	"bigcirc;":                         '\U000025EF',
+	"bigcup;":                          '\U000022C3',
+	"bigodot;":                         '\U00002A00',
+	"bigoplus;":                        '\U00002A01',
+	"bigotimes;":                       '\U00002A02',
+	"bigsqcup;":                        '\U00002A06',
+	"bigstar;":                         '\U00002605',
+	"bigtriangledown;":                 '\U000025BD',
+	"bigtriangleup;":                   '\U000025B3',
+	"biguplus;":                        '\U00002A04',
+	"bigvee;":                          '\U000022C1',
+	"bigwedge;":                        '\U000022C0',
+	"bkarow;":                          '\U0000290D',
+	"blacklozenge;":                    '\U000029EB',
+	"blacksquare;":                     '\U000025AA',
+	"blacktriangle;":                   '\U000025B4',
+	"blacktriangledown;":               '\U000025BE',
+	"blacktriangleleft;":               '\U000025C2',
+	"blacktriangleright;":              '\U000025B8',
+	"blank;":                           '\U00002423',
+	"blk12;":                           '\U00002592',
+	"blk14;":                           '\U00002591',
+	"blk34;":                           '\U00002593',
+	"block;":                           '\U00002588',
+	"bnot;":                            '\U00002310',
+	"bopf;":                            '\U0001D553',
+	"bot;":                             '\U000022A5',
+	"bottom;":                          '\U000022A5',
+	"bowtie;":                          '\U000022C8',
+	"boxDL;":                           '\U00002557',
+	"boxDR;":                           '\U00002554',
+	"boxDl;":                           '\U00002556',
+	"boxDr;":                           '\U00002553',
+	"boxH;":                            '\U00002550',
+	"boxHD;":                           '\U00002566',
+	"boxHU;":                           '\U00002569',
+	"boxHd;":                           '\U00002564',
+	"boxHu;":                           '\U00002567',
+	"boxUL;":                           '\U0000255D',
+	"boxUR;":                           '\U0000255A',
+	"boxUl;":                           '\U0000255C',
+	"boxUr;":                           '\U00002559',
+	"boxV;":                            '\U00002551',
+	"boxVH;":                           '\U0000256C',
+	"boxVL;":                           '\U00002563',
+	"boxVR;":                           '\U00002560',
+	"boxVh;":                           '\U0000256B',
+	"boxVl;":                           '\U00002562',
+	"boxVr;":                           '\U0000255F',
+	"boxbox;":                          '\U000029C9',
+	"boxdL;":                           '\U00002555',
+	"boxdR;":                           '\U00002552',
+	"boxdl;":                           '\U00002510',
+	"boxdr;":                           '\U0000250C',
+	"boxh;":                            '\U00002500',
+	"boxhD;":                           '\U00002565',
+	"boxhU;":                           '\U00002568',
+	"boxhd;":                           '\U0000252C',
+	"boxhu;":                           '\U00002534',
+	"boxminus;":                        '\U0000229F',
+	"boxplus;":                         '\U0000229E',
+	"boxtimes;":                        '\U000022A0',
+	"boxuL;":                           '\U0000255B',
+	"boxuR;":                           '\U00002558',
+	"boxul;":                           '\U00002518',
+	"boxur;":                           '\U00002514',
+	"boxv;":                            '\U00002502',
+	"boxvH;":                           '\U0000256A',
+	"boxvL;":                           '\U00002561',
+	"boxvR;":                           '\U0000255E',
+	"boxvh;":                           '\U0000253C',
+	"boxvl;":                           '\U00002524',
+	"boxvr;":                           '\U0000251C',
+	"bprime;":                          '\U00002035',
+	"breve;":                           '\U000002D8',
+	"brvbar;":                          '\U000000A6',
+	"bscr;":                            '\U0001D4B7',
+	"bsemi;":                           '\U0000204F',
+	"bsim;":                            '\U0000223D',
+	"bsime;":                           '\U000022CD',
+	"bsol;":                            '\U0000005C',
+	"bsolb;":                           '\U000029C5',
+	"bsolhsub;":                        '\U000027C8',
+	"bull;":                            '\U00002022',
+	"bullet;":                          '\U00002022',
+	"bump;":                            '\U0000224E',
+	"bumpE;":                           '\U00002AAE',
+	"bumpe;":                           '\U0000224F',
+	"bumpeq;":                          '\U0000224F',
+	"cacute;":                          '\U00000107',
+	"cap;":                             '\U00002229',
+	"capand;":                          '\U00002A44',
+	"capbrcup;":                        '\U00002A49',
+	"capcap;":                          '\U00002A4B',
+	"capcup;":                          '\U00002A47',
+	"capdot;":                          '\U00002A40',
+	"caret;":                           '\U00002041',
+	"caron;":                           '\U000002C7',
+	"ccaps;":                           '\U00002A4D',
+	"ccaron;":                          '\U0000010D',
+	"ccedil;":                          '\U000000E7',
+	"ccirc;":                           '\U00000109',
+	"ccups;":                           '\U00002A4C',
+	"ccupssm;":                         '\U00002A50',
+	"cdot;":                            '\U0000010B',
+	"cedil;":                           '\U000000B8',
+	"cemptyv;":                         '\U000029B2',
+	"cent;":                            '\U000000A2',
+	"centerdot;":                       '\U000000B7',
+	"cfr;":                             '\U0001D520',
+	"chcy;":                            '\U00000447',
+	"check;":                           '\U00002713',
+	"checkmark;":                       '\U00002713',
+	"chi;":                             '\U000003C7',
+	"cir;":                             '\U000025CB',
+	"cirE;":                            '\U000029C3',
+	"circ;":                            '\U000002C6',
+	"circeq;":                          '\U00002257',
+	"circlearrowleft;":                 '\U000021BA',
+	"circlearrowright;":                '\U000021BB',
+	"circledR;":                        '\U000000AE',
+	"circledS;":                        '\U000024C8',
+	"circledast;":                      '\U0000229B',
+	"circledcirc;":                     '\U0000229A',
+	"circleddash;":                     '\U0000229D',
+	"cire;":                            '\U00002257',
+	"cirfnint;":                        '\U00002A10',
+	"cirmid;":                          '\U00002AEF',
+	"cirscir;":                         '\U000029C2',
+	"clubs;":                           '\U00002663',
+	"clubsuit;":                        '\U00002663',
+	"colon;":                           '\U0000003A',
+	"colone;":                          '\U00002254',
+	"coloneq;":                         '\U00002254',
+	"comma;":                           '\U0000002C',
+	"commat;":                          '\U00000040',
+	"comp;":                            '\U00002201',
+	"compfn;":                          '\U00002218',
+	"complement;":                      '\U00002201',
+	"complexes;":                       '\U00002102',
+	"cong;":                            '\U00002245',
+	"congdot;":                         '\U00002A6D',
+	"conint;":                          '\U0000222E',
+	"copf;":                            '\U0001D554',
+	"coprod;":                          '\U00002210',
+	"copy;":                            '\U000000A9',
+	"copysr;":                          '\U00002117',
+	"crarr;":                           '\U000021B5',
+	"cross;":                           '\U00002717',
+	"cscr;":                            '\U0001D4B8',
+	"csub;":                            '\U00002ACF',
+	"csube;":                           '\U00002AD1',
+	"csup;":                            '\U00002AD0',
+	"csupe;":                           '\U00002AD2',
+	"ctdot;":                           '\U000022EF',
+	"cudarrl;":                         '\U00002938',
+	"cudarrr;":                         '\U00002935',
+	"cuepr;":                           '\U000022DE',
+	"cuesc;":                           '\U000022DF',
+	"cularr;":                          '\U000021B6',
+	"cularrp;":                         '\U0000293D',
+	"cup;":                             '\U0000222A',
+	"cupbrcap;":                        '\U00002A48',
+	"cupcap;":                          '\U00002A46',
+	"cupcup;":                          '\U00002A4A',
+	"cupdot;":                          '\U0000228D',
+	"cupor;":                           '\U00002A45',
+	"curarr;":                          '\U000021B7',
+	"curarrm;":                         '\U0000293C',
+	"curlyeqprec;":                     '\U000022DE',
+	"curlyeqsucc;":                     '\U000022DF',
+	"curlyvee;":                        '\U000022CE',
+	"curlywedge;":                      '\U000022CF',
+	"curren;":                          '\U000000A4',
+	"curvearrowleft;":                  '\U000021B6',
+	"curvearrowright;":                 '\U000021B7',
+	"cuvee;":                           '\U000022CE',
+	"cuwed;":                           '\U000022CF',
+	"cwconint;":                        '\U00002232',
+	"cwint;":                           '\U00002231',
+	"cylcty;":                          '\U0000232D',
+	"dArr;":                            '\U000021D3',
+	"dHar;":                            '\U00002965',
+	"dagger;":                          '\U00002020',
+	"daleth;":                          '\U00002138',
+	"darr;":                            '\U00002193',
+	"dash;":                            '\U00002010',
+	"dashv;":                           '\U000022A3',
+	"dbkarow;":                         '\U0000290F',
+	"dblac;":                           '\U000002DD',
+	"dcaron;":                          '\U0000010F',
+	"dcy;":                             '\U00000434',
+	"dd;":                              '\U00002146',
+	"ddagger;":                         '\U00002021',
+	"ddarr;":                           '\U000021CA',
+	"ddotseq;":                         '\U00002A77',
+	"deg;":                             '\U000000B0',
+	"delta;":                           '\U000003B4',
+	"demptyv;":                         '\U000029B1',
+	"dfisht;":                          '\U0000297F',
+	"dfr;":                             '\U0001D521',
+	"dharl;":                           '\U000021C3',
+	"dharr;":                           '\U000021C2',
+	"diam;":                            '\U000022C4',
+	"diamond;":                         '\U000022C4',
+	"diamondsuit;":                     '\U00002666',
+	"diams;":                           '\U00002666',
+	"die;":                             '\U000000A8',
+	"digamma;":                         '\U000003DD',
+	"disin;":                           '\U000022F2',
+	"div;":                             '\U000000F7',
+	"divide;":                          '\U000000F7',
+	"divideontimes;":                   '\U000022C7',
+	"divonx;":                          '\U000022C7',
+	"djcy;":                            '\U00000452',
+	"dlcorn;":                          '\U0000231E',
+	"dlcrop;":                          '\U0000230D',
+	"dollar;":                          '\U00000024',
+	"dopf;":                            '\U0001D555',
+	"dot;":                             '\U000002D9',
+	"doteq;":                           '\U00002250',
+	"doteqdot;":                        '\U00002251',
+	"dotminus;":                        '\U00002238',
+	"dotplus;":                         '\U00002214',
+	"dotsquare;":                       '\U000022A1',
+	"doublebarwedge;":                  '\U00002306',
+	"downarrow;":                       '\U00002193',
+	"downdownarrows;":                  '\U000021CA',
+	"downharpoonleft;":                 '\U000021C3',
+	"downharpoonright;":                '\U000021C2',
+	"drbkarow;":                        '\U00002910',
+	"drcorn;":                          '\U0000231F',
+	"drcrop;":                          '\U0000230C',
+	"dscr;":                            '\U0001D4B9',
+	"dscy;":                            '\U00000455',
+	"dsol;":                            '\U000029F6',
+	"dstrok;":                          '\U00000111',
+	"dtdot;":                           '\U000022F1',
+	"dtri;":                            '\U000025BF',
+	"dtrif;":                           '\U000025BE',
+	"duarr;":                           '\U000021F5',
+	"duhar;":                           '\U0000296F',
+	"dwangle;":                         '\U000029A6',
+	"dzcy;":                            '\U0000045F',
+	"dzigrarr;":                        '\U000027FF',
+	"eDDot;":                           '\U00002A77',
+	"eDot;":                            '\U00002251',
+	"eacute;":                          '\U000000E9',
+	"easter;":                          '\U00002A6E',
+	"ecaron;":                          '\U0000011B',
+	"ecir;":                            '\U00002256',
+	"ecirc;":                           '\U000000EA',
+	"ecolon;":                          '\U00002255',
+	"ecy;":                             '\U0000044D',
+	"edot;":                            '\U00000117',
+	"ee;":                              '\U00002147',
+	"efDot;":                           '\U00002252',
+	"efr;":                             '\U0001D522',
+	"eg;":                              '\U00002A9A',
+	"egrave;":                          '\U000000E8',
+	"egs;":                             '\U00002A96',
+	"egsdot;":                          '\U00002A98',
+	"el;":                              '\U00002A99',
+	"elinters;":                        '\U000023E7',
+	"ell;":                             '\U00002113',
+	"els;":                             '\U00002A95',
+	"elsdot;":                          '\U00002A97',
+	"emacr;":                           '\U00000113',
+	"empty;":                           '\U00002205',
+	"emptyset;":                        '\U00002205',
+	"emptyv;":                          '\U00002205',
+	"emsp;":                            '\U00002003',
+	"emsp13;":                          '\U00002004',
+	"emsp14;":                          '\U00002005',
+	"eng;":                             '\U0000014B',
+	"ensp;":                            '\U00002002',
+	"eogon;":                           '\U00000119',
+	"eopf;":                            '\U0001D556',
+	"epar;":                            '\U000022D5',
+	"eparsl;":                          '\U000029E3',
+	"eplus;":                           '\U00002A71',
+	"epsi;":                            '\U000003B5',
+	"epsilon;":                         '\U000003B5',
+	"epsiv;":                           '\U000003F5',
+	"eqcirc;":                          '\U00002256',
+	"eqcolon;":                         '\U00002255',
+	"eqsim;":                           '\U00002242',
+	"eqslantgtr;":                      '\U00002A96',
+	"eqslantless;":                     '\U00002A95',
+	"equals;":                          '\U0000003D',
+	"equest;":                          '\U0000225F',
+	"equiv;":                           '\U00002261',
+	"equivDD;":                         '\U00002A78',
+	"eqvparsl;":                        '\U000029E5',
+	"erDot;":                           '\U00002253',
+	"erarr;":                           '\U00002971',
+	"escr;":                            '\U0000212F',
+	"esdot;":                           '\U00002250',
+	"esim;":                            '\U00002242',
+	"eta;":                             '\U000003B7',
+	"eth;":                             '\U000000F0',
+	"euml;":                            '\U000000EB',
+	"euro;":                            '\U000020AC',
+	"excl;":                            '\U00000021',
+	"exist;":                           '\U00002203',
+	"expectation;":                     '\U00002130',
+	"exponentiale;":                    '\U00002147',
+	"fallingdotseq;":                   '\U00002252',
+	"fcy;":                             '\U00000444',
+	"female;":                          '\U00002640',
+	"ffilig;":                          '\U0000FB03',
+	"fflig;":                           '\U0000FB00',
+	"ffllig;":                          '\U0000FB04',
+	"ffr;":                             '\U0001D523',
+	"filig;":                           '\U0000FB01',
+	"flat;":                            '\U0000266D',
+	"fllig;":                           '\U0000FB02',
+	"fltns;":                           '\U000025B1',
+	"fnof;":                            '\U00000192',
+	"fopf;":                            '\U0001D557',
+	"forall;":                          '\U00002200',
+	"fork;":                            '\U000022D4',
+	"forkv;":                           '\U00002AD9',
+	"fpartint;":                        '\U00002A0D',
+	"frac12;":                          '\U000000BD',
+	"frac13;":                          '\U00002153',
+	"frac14;":                          '\U000000BC',
+	"frac15;":                          '\U00002155',
+	"frac16;":                          '\U00002159',
+	"frac18;":                          '\U0000215B',
+	"frac23;":                          '\U00002154',
+	"frac25;":                          '\U00002156',
+	"frac34;":                          '\U000000BE',
+	"frac35;":                          '\U00002157',
+	"frac38;":                          '\U0000215C',
+	"frac45;":                          '\U00002158',
+	"frac56;":                          '\U0000215A',
+	"frac58;":                          '\U0000215D',
+	"frac78;":                          '\U0000215E',
+	"frasl;":                           '\U00002044',
+	"frown;":                           '\U00002322',
+	"fscr;":                            '\U0001D4BB',
+	"gE;":                              '\U00002267',
+	"gEl;":                             '\U00002A8C',
+	"gacute;":                          '\U000001F5',
+	"gamma;":                           '\U000003B3',
+	"gammad;":                          '\U000003DD',
+	"gap;":                             '\U00002A86',
+	"gbreve;":                          '\U0000011F',
+	"gcirc;":                           '\U0000011D',
+	"gcy;":                             '\U00000433',
+	"gdot;":                            '\U00000121',
+	"ge;":                              '\U00002265',
+	"gel;":                             '\U000022DB',
+	"geq;":                             '\U00002265',
+	"geqq;":                            '\U00002267',
+	"geqslant;":                        '\U00002A7E',
+	"ges;":                             '\U00002A7E',
+	"gescc;":                           '\U00002AA9',
+	"gesdot;":                          '\U00002A80',
+	"gesdoto;":                         '\U00002A82',
+	"gesdotol;":                        '\U00002A84',
+	"gesles;":                          '\U00002A94',
+	"gfr;":                             '\U0001D524',
+	"gg;":                              '\U0000226B',
+	"ggg;":                             '\U000022D9',
+	"gimel;":                           '\U00002137',
+	"gjcy;":                            '\U00000453',
+	"gl;":                              '\U00002277',
+	"glE;":                             '\U00002A92',
+	"gla;":                             '\U00002AA5',
+	"glj;":                             '\U00002AA4',
+	"gnE;":                             '\U00002269',
+	"gnap;":                            '\U00002A8A',
+	"gnapprox;":                        '\U00002A8A',
+	"gne;":                             '\U00002A88',
+	"gneq;":                            '\U00002A88',
+	"gneqq;":                           '\U00002269',
+	"gnsim;":                           '\U000022E7',
+	"gopf;":                            '\U0001D558',
+	"grave;":                           '\U00000060',
+	"gscr;":                            '\U0000210A',
+	"gsim;":                            '\U00002273',
+	"gsime;":                           '\U00002A8E',
+	"gsiml;":                           '\U00002A90',
+	"gt;":                              '\U0000003E',
+	"gtcc;":                            '\U00002AA7',
+	"gtcir;":                           '\U00002A7A',
+	"gtdot;":                           '\U000022D7',
+	"gtlPar;":                          '\U00002995',
+	"gtquest;":                         '\U00002A7C',
+	"gtrapprox;":                       '\U00002A86',
+	"gtrarr;":                          '\U00002978',
+	"gtrdot;":                          '\U000022D7',
+	"gtreqless;":                       '\U000022DB',
+	"gtreqqless;":                      '\U00002A8C',
+	"gtrless;":                         '\U00002277',
+	"gtrsim;":                          '\U00002273',
+	"hArr;":                            '\U000021D4',
+	"hairsp;":                          '\U0000200A',
+	"half;":                            '\U000000BD',
+	"hamilt;":                          '\U0000210B',
+	"hardcy;":                          '\U0000044A',
+	"harr;":                            '\U00002194',
+	"harrcir;":                         '\U00002948',
+	"harrw;":                           '\U000021AD',
+	"hbar;":                            '\U0000210F',
+	"hcirc;":                           '\U00000125',
+	"hearts;":                          '\U00002665',
+	"heartsuit;":                       '\U00002665',
+	"hellip;":                          '\U00002026',
+	"hercon;":                          '\U000022B9',
+	"hfr;":                             '\U0001D525',
+	"hksearow;":                        '\U00002925',
+	"hkswarow;":                        '\U00002926',
+	"hoarr;":                           '\U000021FF',
+	"homtht;":                          '\U0000223B',
+	"hookleftarrow;":                   '\U000021A9',
+	"hookrightarrow;":                  '\U000021AA',
+	"hopf;":                            '\U0001D559',
+	"horbar;":                          '\U00002015',
+	"hscr;":                            '\U0001D4BD',
+	"hslash;":                          '\U0000210F',
+	"hstrok;":                          '\U00000127',
+	"hybull;":                          '\U00002043',
+	"hyphen;":                          '\U00002010',
+	"iacute;":                          '\U000000ED',
+	"ic;":                              '\U00002063',
+	"icirc;":                           '\U000000EE',
+	"icy;":                             '\U00000438',
+	"iecy;":                            '\U00000435',
+	"iexcl;":                           '\U000000A1',
+	"iff;":                             '\U000021D4',
+	"ifr;":                             '\U0001D526',
+	"igrave;":                          '\U000000EC',
+	"ii;":                              '\U00002148',
+	"iiiint;":                          '\U00002A0C',
+	"iiint;":                           '\U0000222D',
+	"iinfin;":                          '\U000029DC',
+	"iiota;":                           '\U00002129',
+	"ijlig;":                           '\U00000133',
+	"imacr;":                           '\U0000012B',
+	"image;":                           '\U00002111',
+	"imagline;":                        '\U00002110',
+	"imagpart;":                        '\U00002111',
+	"imath;":                           '\U00000131',
+	"imof;":                            '\U000022B7',
+	"imped;":                           '\U000001B5',
+	"in;":                              '\U00002208',
+	"incare;":                          '\U00002105',
+	"infin;":                           '\U0000221E',
+	"infintie;":                        '\U000029DD',
+	"inodot;":                          '\U00000131',
+	"int;":                             '\U0000222B',
+	"intcal;":                          '\U000022BA',
+	"integers;":                        '\U00002124',
+	"intercal;":                        '\U000022BA',
+	"intlarhk;":                        '\U00002A17',
+	"intprod;":                         '\U00002A3C',
+	"iocy;":                            '\U00000451',
+	"iogon;":                           '\U0000012F',
+	"iopf;":                            '\U0001D55A',
+	"iota;":                            '\U000003B9',
+	"iprod;":                           '\U00002A3C',
+	"iquest;":                          '\U000000BF',
+	"iscr;":                            '\U0001D4BE',
+	"isin;":                            '\U00002208',
+	"isinE;":                           '\U000022F9',
+	"isindot;":                         '\U000022F5',
+	"isins;":                           '\U000022F4',
+	"isinsv;":                          '\U000022F3',
+	"isinv;":                           '\U00002208',
+	"it;":                              '\U00002062',
+	"itilde;":                          '\U00000129',
+	"iukcy;":                           '\U00000456',
+	"iuml;":                            '\U000000EF',
+	"jcirc;":                           '\U00000135',
+	"jcy;":                             '\U00000439',
+	"jfr;":                             '\U0001D527',
+	"jmath;":                           '\U00000237',
+	"jopf;":                            '\U0001D55B',
+	"jscr;":                            '\U0001D4BF',
+	"jsercy;":                          '\U00000458',
+	"jukcy;":                           '\U00000454',
+	"kappa;":                           '\U000003BA',
+	"kappav;":                          '\U000003F0',
+	"kcedil;":                          '\U00000137',
+	"kcy;":                             '\U0000043A',
+	"kfr;":                             '\U0001D528',
+	"kgreen;":                          '\U00000138',
+	"khcy;":                            '\U00000445',
+	"kjcy;":                            '\U0000045C',
+	"kopf;":                            '\U0001D55C',
+	"kscr;":                            '\U0001D4C0',
+	"lAarr;":                           '\U000021DA',
+	"lArr;":                            '\U000021D0',
+	"lAtail;":                          '\U0000291B',
+	"lBarr;":                           '\U0000290E',
+	"lE;":                              '\U00002266',
+	"lEg;":                             '\U00002A8B',
+	"lHar;":                            '\U00002962',
+	"lacute;":                          '\U0000013A',
+	"laemptyv;":                        '\U000029B4',
+	"lagran;":                          '\U00002112',
+	"lambda;":                          '\U000003BB',
+	"lang;":                            '\U000027E8',
+	"langd;":                           '\U00002991',
+	"langle;":                          '\U000027E8',
+	"lap;":                             '\U00002A85',
+	"laquo;":                           '\U000000AB',
+	"larr;":                            '\U00002190',
+	"larrb;":                           '\U000021E4',
+	"larrbfs;":                         '\U0000291F',
+	"larrfs;":                          '\U0000291D',
+	"larrhk;":                          '\U000021A9',
+	"larrlp;":                          '\U000021AB',
+	"larrpl;":                          '\U00002939',
+	"larrsim;":                         '\U00002973',
+	"larrtl;":                          '\U000021A2',
+	"lat;":                             '\U00002AAB',
+	"latail;":                          '\U00002919',
+	"late;":                            '\U00002AAD',
+	"lbarr;":                           '\U0000290C',
+	"lbbrk;":                           '\U00002772',
+	"lbrace;":                          '\U0000007B',
+	"lbrack;":                          '\U0000005B',
+	"lbrke;":                           '\U0000298B',
+	"lbrksld;":                         '\U0000298F',
+	"lbrkslu;":                         '\U0000298D',
+	"lcaron;":                          '\U0000013E',
+	"lcedil;":                          '\U0000013C',
+	"lceil;":                           '\U00002308',
+	"lcub;":                            '\U0000007B',
+	"lcy;":                             '\U0000043B',
+	"ldca;":                            '\U00002936',
+	"ldquo;":                           '\U0000201C',
+	"ldquor;":                          '\U0000201E',
+	"ldrdhar;":                         '\U00002967',
+	"ldrushar;":                        '\U0000294B',
+	"ldsh;":                            '\U000021B2',
+	"le;":                              '\U00002264',
+	"leftarrow;":                       '\U00002190',
+	"leftarrowtail;":                   '\U000021A2',
+	"leftharpoondown;":                 '\U000021BD',
+	"leftharpoonup;":                   '\U000021BC',
+	"leftleftarrows;":                  '\U000021C7',
+	"leftrightarrow;":                  '\U00002194',
+	"leftrightarrows;":                 '\U000021C6',
+	"leftrightharpoons;":               '\U000021CB',
+	"leftrightsquigarrow;":             '\U000021AD',
+	"leftthreetimes;":                  '\U000022CB',
+	"leg;":                             '\U000022DA',
+	"leq;":                             '\U00002264',
+	"leqq;":                            '\U00002266',
+	"leqslant;":                        '\U00002A7D',
+	"les;":                             '\U00002A7D',
+	"lescc;":                           '\U00002AA8',
+	"lesdot;":                          '\U00002A7F',
+	"lesdoto;":                         '\U00002A81',
+	"lesdotor;":                        '\U00002A83',
+	"lesges;":                          '\U00002A93',
+	"lessapprox;":                      '\U00002A85',
+	"lessdot;":                         '\U000022D6',
+	"lesseqgtr;":                       '\U000022DA',
+	"lesseqqgtr;":                      '\U00002A8B',
+	"lessgtr;":                         '\U00002276',
+	"lesssim;":                         '\U00002272',
+	"lfisht;":                          '\U0000297C',
+	"lfloor;":                          '\U0000230A',
+	"lfr;":                             '\U0001D529',
+	"lg;":                              '\U00002276',
+	"lgE;":                             '\U00002A91',
+	"lhard;":                           '\U000021BD',
+	"lharu;":                           '\U000021BC',
+	"lharul;":                          '\U0000296A',
+	"lhblk;":                           '\U00002584',
+	"ljcy;":                            '\U00000459',
+	"ll;":                              '\U0000226A',
+	"llarr;":                           '\U000021C7',
+	"llcorner;":                        '\U0000231E',
+	"llhard;":                          '\U0000296B',
+	"lltri;":                           '\U000025FA',
+	"lmidot;":                          '\U00000140',
+	"lmoust;":                          '\U000023B0',
+	"lmoustache;":                      '\U000023B0',
+	"lnE;":                             '\U00002268',
+	"lnap;":                            '\U00002A89',
+	"lnapprox;":                        '\U00002A89',
+	"lne;":                             '\U00002A87',
+	"lneq;":                            '\U00002A87',
+	"lneqq;":                           '\U00002268',
+	"lnsim;":                           '\U000022E6',
+	"loang;":                           '\U000027EC',
+	"loarr;":                           '\U000021FD',
+	"lobrk;":                           '\U000027E6',
+	"longleftarrow;":                   '\U000027F5',
+	"longleftrightarrow;":              '\U000027F7',
+	"longmapsto;":                      '\U000027FC',
+	"longrightarrow;":                  '\U000027F6',
+	"looparrowleft;":                   '\U000021AB',
+	"looparrowright;":                  '\U000021AC',
+	"lopar;":                           '\U00002985',
+	"lopf;":                            '\U0001D55D',
+	"loplus;":                          '\U00002A2D',
+	"lotimes;":                         '\U00002A34',
+	"lowast;":                          '\U00002217',
+	"lowbar;":                          '\U0000005F',
+	"loz;":                             '\U000025CA',
+	"lozenge;":                         '\U000025CA',
+	"lozf;":                            '\U000029EB',
+	"lpar;":                            '\U00000028',
+	"lparlt;":                          '\U00002993',
+	"lrarr;":                           '\U000021C6',
+	"lrcorner;":                        '\U0000231F',
+	"lrhar;":                           '\U000021CB',
+	"lrhard;":                          '\U0000296D',
+	"lrm;":                             '\U0000200E',
+	"lrtri;":                           '\U000022BF',
+	"lsaquo;":                          '\U00002039',
+	"lscr;":                            '\U0001D4C1',
+	"lsh;":                             '\U000021B0',
+	"lsim;":                            '\U00002272',
+	"lsime;":                           '\U00002A8D',
+	"lsimg;":                           '\U00002A8F',
+	"lsqb;":                            '\U0000005B',
+	"lsquo;":                           '\U00002018',
+	"lsquor;":                          '\U0000201A',
+	"lstrok;":                          '\U00000142',
+	"lt;":                              '\U0000003C',
+	"ltcc;":                            '\U00002AA6',
+	"ltcir;":                           '\U00002A79',
+	"ltdot;":                           '\U000022D6',
+	"lthree;":                          '\U000022CB',
+	"ltimes;":                          '\U000022C9',
+	"ltlarr;":                          '\U00002976',
+	"ltquest;":                         '\U00002A7B',
+	"ltrPar;":                          '\U00002996',
+	"ltri;":                            '\U000025C3',
+	"ltrie;":                           '\U000022B4',
+	"ltrif;":                           '\U000025C2',
+	"lurdshar;":                        '\U0000294A',
+	"luruhar;":                         '\U00002966',
+	"mDDot;":                           '\U0000223A',
+	"macr;":                            '\U000000AF',
+	"male;":                            '\U00002642',
+	"malt;":                            '\U00002720',
+	"maltese;":                         '\U00002720',
+	"map;":                             '\U000021A6',
+	"mapsto;":                          '\U000021A6',
+	"mapstodown;":                      '\U000021A7',
+	"mapstoleft;":                      '\U000021A4',
+	"mapstoup;":                        '\U000021A5',
+	"marker;":                          '\U000025AE',
+	"mcomma;":                          '\U00002A29',
+	"mcy;":                             '\U0000043C',
+	"mdash;":                           '\U00002014',
+	"measuredangle;":                   '\U00002221',
+	"mfr;":                             '\U0001D52A',
+	"mho;":                             '\U00002127',
+	"micro;":                           '\U000000B5',
+	"mid;":                             '\U00002223',
+	"midast;":                          '\U0000002A',
+	"midcir;":                          '\U00002AF0',
+	"middot;":                          '\U000000B7',
+	"minus;":                           '\U00002212',
+	"minusb;":                          '\U0000229F',
+	"minusd;":                          '\U00002238',
+	"minusdu;":                         '\U00002A2A',
+	"mlcp;":                            '\U00002ADB',
+	"mldr;":                            '\U00002026',
+	"mnplus;":                          '\U00002213',
+	"models;":                          '\U000022A7',
+	"mopf;":                            '\U0001D55E',
+	"mp;":                              '\U00002213',
+	"mscr;":                            '\U0001D4C2',
+	"mstpos;":                          '\U0000223E',
+	"mu;":                              '\U000003BC',
+	"multimap;":                        '\U000022B8',
+	"mumap;":                           '\U000022B8',
+	"nLeftarrow;":                      '\U000021CD',
+	"nLeftrightarrow;":                 '\U000021CE',
+	"nRightarrow;":                     '\U000021CF',
+	"nVDash;":                          '\U000022AF',
+	"nVdash;":                          '\U000022AE',
+	"nabla;":                           '\U00002207',
+	"nacute;":                          '\U00000144',
+	"nap;":                             '\U00002249',
+	"napos;":                           '\U00000149',
+	"napprox;":                         '\U00002249',
+	"natur;":                           '\U0000266E',
+	"natural;":                         '\U0000266E',
+	"naturals;":                        '\U00002115',
+	"nbsp;":                            '\U000000A0',
+	"ncap;":                            '\U00002A43',
+	"ncaron;":                          '\U00000148',
+	"ncedil;":                          '\U00000146',
+	"ncong;":                           '\U00002247',
+	"ncup;":                            '\U00002A42',
+	"ncy;":                             '\U0000043D',
+	"ndash;":                           '\U00002013',
+	"ne;":                              '\U00002260',
+	"neArr;":                           '\U000021D7',
+	"nearhk;":                          '\U00002924',
+	"nearr;":                           '\U00002197',
+	"nearrow;":                         '\U00002197',
+	"nequiv;":                          '\U00002262',
+	"nesear;":                          '\U00002928',
+	"nexist;":                          '\U00002204',
+	"nexists;":                         '\U00002204',
+	"nfr;":                             '\U0001D52B',
+	"nge;":                             '\U00002271',
+	"ngeq;":                            '\U00002271',
+	"ngsim;":                           '\U00002275',
+	"ngt;":                             '\U0000226F',
+	"ngtr;":                            '\U0000226F',
+	"nhArr;":                           '\U000021CE',
+	"nharr;":                           '\U000021AE',
+	"nhpar;":                           '\U00002AF2',
+	"ni;":                              '\U0000220B',
+	"nis;":                             '\U000022FC',
+	"nisd;":                            '\U000022FA',
+	"niv;":                             '\U0000220B',
+	"njcy;":                            '\U0000045A',
+	"nlArr;":                           '\U000021CD',
+	"nlarr;":                           '\U0000219A',
+	"nldr;":                            '\U00002025',
+	"nle;":                             '\U00002270',
+	"nleftarrow;":                      '\U0000219A',
+	"nleftrightarrow;":                 '\U000021AE',
+	"nleq;":                            '\U00002270',
+	"nless;":                           '\U0000226E',
+	"nlsim;":                           '\U00002274',
+	"nlt;":                             '\U0000226E',
+	"nltri;":                           '\U000022EA',
+	"nltrie;":                          '\U000022EC',
+	"nmid;":                            '\U00002224',
+	"nopf;":                            '\U0001D55F',
+	"not;":                             '\U000000AC',
+	"notin;":                           '\U00002209',
+	"notinva;":                         '\U00002209',
+	"notinvb;":                         '\U000022F7',
+	"notinvc;":                         '\U000022F6',
+	"notni;":                           '\U0000220C',
+	"notniva;":                         '\U0000220C',
+	"notnivb;":                         '\U000022FE',
+	"notnivc;":                         '\U000022FD',
+	"npar;":                            '\U00002226',
+	"nparallel;":                       '\U00002226',
+	"npolint;":                         '\U00002A14',
+	"npr;":                             '\U00002280',
+	"nprcue;":                          '\U000022E0',
+	"nprec;":                           '\U00002280',
+	"nrArr;":                           '\U000021CF',
+	"nrarr;":                           '\U0000219B',
+	"nrightarrow;":                     '\U0000219B',
+	"nrtri;":                           '\U000022EB',
+	"nrtrie;":                          '\U000022ED',
+	"nsc;":                             '\U00002281',
+	"nsccue;":                          '\U000022E1',
+	"nscr;":                            '\U0001D4C3',
+	"nshortmid;":                       '\U00002224',
+	"nshortparallel;":                  '\U00002226',
+	"nsim;":                            '\U00002241',
+	"nsime;":                           '\U00002244',
+	"nsimeq;":                          '\U00002244',
+	"nsmid;":                           '\U00002224',
+	"nspar;":                           '\U00002226',
+	"nsqsube;":                         '\U000022E2',
+	"nsqsupe;":                         '\U000022E3',
+	"nsub;":                            '\U00002284',
+	"nsube;":                           '\U00002288',
+	"nsubseteq;":                       '\U00002288',
+	"nsucc;":                           '\U00002281',
+	"nsup;":                            '\U00002285',
+	"nsupe;":                           '\U00002289',
+	"nsupseteq;":                       '\U00002289',
+	"ntgl;":                            '\U00002279',
+	"ntilde;":                          '\U000000F1',
+	"ntlg;":                            '\U00002278',
+	"ntriangleleft;":                   '\U000022EA',
+	"ntrianglelefteq;":                 '\U000022EC',
+	"ntriangleright;":                  '\U000022EB',
+	"ntrianglerighteq;":                '\U000022ED',
+	"nu;":                              '\U000003BD',
+	"num;":                             '\U00000023',
+	"numero;":                          '\U00002116',
+	"numsp;":                           '\U00002007',
+	"nvDash;":                          '\U000022AD',
+	"nvHarr;":                          '\U00002904',
+	"nvdash;":                          '\U000022AC',
+	"nvinfin;":                         '\U000029DE',
+	"nvlArr;":                          '\U00002902',
+	"nvrArr;":                          '\U00002903',
+	"nwArr;":                           '\U000021D6',
+	"nwarhk;":                          '\U00002923',
+	"nwarr;":                           '\U00002196',
+	"nwarrow;":                         '\U00002196',
+	"nwnear;":                          '\U00002927',
+	"oS;":                              '\U000024C8',
+	"oacute;":                          '\U000000F3',
+	"oast;":                            '\U0000229B',
+	"ocir;":                            '\U0000229A',
+	"ocirc;":                           '\U000000F4',
+	"ocy;":                             '\U0000043E',
+	"odash;":                           '\U0000229D',
+	"odblac;":                          '\U00000151',
+	"odiv;":                            '\U00002A38',
+	"odot;":                            '\U00002299',
+	"odsold;":                          '\U000029BC',
+	"oelig;":                           '\U00000153',
+	"ofcir;":                           '\U000029BF',
+	"ofr;":                             '\U0001D52C',
+	"ogon;":                            '\U000002DB',
+	"ograve;":                          '\U000000F2',
+	"ogt;":                             '\U000029C1',
+	"ohbar;":                           '\U000029B5',
+	"ohm;":                             '\U000003A9',
+	"oint;":                            '\U0000222E',
+	"olarr;":                           '\U000021BA',
+	"olcir;":                           '\U000029BE',
+	"olcross;":                         '\U000029BB',
+	"oline;":                           '\U0000203E',
+	"olt;":                             '\U000029C0',
+	"omacr;":                           '\U0000014D',
+	"omega;":                           '\U000003C9',
+	"omicron;":                         '\U000003BF',
+	"omid;":                            '\U000029B6',
+	"ominus;":                          '\U00002296',
+	"oopf;":                            '\U0001D560',
+	"opar;":                            '\U000029B7',
+	"operp;":                           '\U000029B9',
+	"oplus;":                           '\U00002295',
+	"or;":                              '\U00002228',
+	"orarr;":                           '\U000021BB',
+	"ord;":                             '\U00002A5D',
+	"order;":                           '\U00002134',
+	"orderof;":                         '\U00002134',
+	"ordf;":                            '\U000000AA',
+	"ordm;":                            '\U000000BA',
+	"origof;":                          '\U000022B6',
+	"oror;":                            '\U00002A56',
+	"orslope;":                         '\U00002A57',
+	"orv;":                             '\U00002A5B',
+	"oscr;":                            '\U00002134',
+	"oslash;":                          '\U000000F8',
+	"osol;":                            '\U00002298',
+	"otilde;":                          '\U000000F5',
+	"otimes;":                          '\U00002297',
+	"otimesas;":                        '\U00002A36',
+	"ouml;":                            '\U000000F6',
+	"ovbar;":                           '\U0000233D',
+	"par;":                             '\U00002225',
+	"para;":                            '\U000000B6',
+	"parallel;":                        '\U00002225',
+	"parsim;":                          '\U00002AF3',
+	"parsl;":                           '\U00002AFD',
+	"part;":                            '\U00002202',
+	"pcy;":                             '\U0000043F',
+	"percnt;":                          '\U00000025',
+	"period;":                          '\U0000002E',
+	"permil;":                          '\U00002030',
+	"perp;":                            '\U000022A5',
+	"pertenk;":                         '\U00002031',
+	"pfr;":                             '\U0001D52D',
+	"phi;":                             '\U000003C6',
+	"phiv;":                            '\U000003D5',
+	"phmmat;":                          '\U00002133',
+	"phone;":                           '\U0000260E',
+	"pi;":                              '\U000003C0',
+	"pitchfork;":                       '\U000022D4',
+	"piv;":                             '\U000003D6',
+	"planck;":                          '\U0000210F',
+	"planckh;":                         '\U0000210E',
+	"plankv;":                          '\U0000210F',
+	"plus;":                            '\U0000002B',
+	"plusacir;":                        '\U00002A23',
+	"plusb;":                           '\U0000229E',
+	"pluscir;":                         '\U00002A22',
+	"plusdo;":                          '\U00002214',
+	"plusdu;":                          '\U00002A25',
+	"pluse;":                           '\U00002A72',
+	"plusmn;":                          '\U000000B1',
+	"plussim;":                         '\U00002A26',
+	"plustwo;":                         '\U00002A27',
+	"pm;":                              '\U000000B1',
+	"pointint;":                        '\U00002A15',
+	"popf;":                            '\U0001D561',
+	"pound;":                           '\U000000A3',
+	"pr;":                              '\U0000227A',
+	"prE;":                             '\U00002AB3',
+	"prap;":                            '\U00002AB7',
+	"prcue;":                           '\U0000227C',
+	"pre;":                             '\U00002AAF',
+	"prec;":                            '\U0000227A',
+	"precapprox;":                      '\U00002AB7',
+	"preccurlyeq;":                     '\U0000227C',
+	"preceq;":                          '\U00002AAF',
+	"precnapprox;":                     '\U00002AB9',
+	"precneqq;":                        '\U00002AB5',
+	"precnsim;":                        '\U000022E8',
+	"precsim;":                         '\U0000227E',
+	"prime;":                           '\U00002032',
+	"primes;":                          '\U00002119',
+	"prnE;":                            '\U00002AB5',
+	"prnap;":                           '\U00002AB9',
+	"prnsim;":                          '\U000022E8',
+	"prod;":                            '\U0000220F',
+	"profalar;":                        '\U0000232E',
+	"profline;":                        '\U00002312',
+	"profsurf;":                        '\U00002313',
+	"prop;":                            '\U0000221D',
+	"propto;":                          '\U0000221D',
+	"prsim;":                           '\U0000227E',
+	"prurel;":                          '\U000022B0',
+	"pscr;":                            '\U0001D4C5',
+	"psi;":                             '\U000003C8',
+	"puncsp;":                          '\U00002008',
+	"qfr;":                             '\U0001D52E',
+	"qint;":                            '\U00002A0C',
+	"qopf;":                            '\U0001D562',
+	"qprime;":                          '\U00002057',
+	"qscr;":                            '\U0001D4C6',
+	"quaternions;":                     '\U0000210D',
+	"quatint;":                         '\U00002A16',
+	"quest;":                           '\U0000003F',
+	"questeq;":                         '\U0000225F',
+	"quot;":                            '\U00000022',
+	"rAarr;":                           '\U000021DB',
+	"rArr;":                            '\U000021D2',
+	"rAtail;":                          '\U0000291C',
+	"rBarr;":                           '\U0000290F',
+	"rHar;":                            '\U00002964',
+	"racute;":                          '\U00000155',
+	"radic;":                           '\U0000221A',
+	"raemptyv;":                        '\U000029B3',
+	"rang;":                            '\U000027E9',
+	"rangd;":                           '\U00002992',
+	"range;":                           '\U000029A5',
+	"rangle;":                          '\U000027E9',
+	"raquo;":                           '\U000000BB',
+	"rarr;":                            '\U00002192',
+	"rarrap;":                          '\U00002975',
+	"rarrb;":                           '\U000021E5',
+	"rarrbfs;":                         '\U00002920',
+	"rarrc;":                           '\U00002933',
+	"rarrfs;":                          '\U0000291E',
+	"rarrhk;":                          '\U000021AA',
+	"rarrlp;":                          '\U000021AC',
+	"rarrpl;":                          '\U00002945',
+	"rarrsim;":                         '\U00002974',
+	"rarrtl;":                          '\U000021A3',
+	"rarrw;":                           '\U0000219D',
+	"ratail;":                          '\U0000291A',
+	"ratio;":                           '\U00002236',
+	"rationals;":                       '\U0000211A',
+	"rbarr;":                           '\U0000290D',
+	"rbbrk;":                           '\U00002773',
+	"rbrace;":                          '\U0000007D',
+	"rbrack;":                          '\U0000005D',
+	"rbrke;":                           '\U0000298C',
+	"rbrksld;":                         '\U0000298E',
+	"rbrkslu;":                         '\U00002990',
+	"rcaron;":                          '\U00000159',
+	"rcedil;":                          '\U00000157',
+	"rceil;":                           '\U00002309',
+	"rcub;":                            '\U0000007D',
+	"rcy;":                             '\U00000440',
+	"rdca;":                            '\U00002937',
+	"rdldhar;":                         '\U00002969',
+	"rdquo;":                           '\U0000201D',
+	"rdquor;":                          '\U0000201D',
+	"rdsh;":                            '\U000021B3',
+	"real;":                            '\U0000211C',
+	"realine;":                         '\U0000211B',
+	"realpart;":                        '\U0000211C',
+	"reals;":                           '\U0000211D',
+	"rect;":                            '\U000025AD',
+	"reg;":                             '\U000000AE',
+	"rfisht;":                          '\U0000297D',
+	"rfloor;":                          '\U0000230B',
+	"rfr;":                             '\U0001D52F',
+	"rhard;":                           '\U000021C1',
+	"rharu;":                           '\U000021C0',
+	"rharul;":                          '\U0000296C',
+	"rho;":                             '\U000003C1',
+	"rhov;":                            '\U000003F1',
+	"rightarrow;":                      '\U00002192',
+	"rightarrowtail;":                  '\U000021A3',
+	"rightharpoondown;":                '\U000021C1',
+	"rightharpoonup;":                  '\U000021C0',
+	"rightleftarrows;":                 '\U000021C4',
+	"rightleftharpoons;":               '\U000021CC',
+	"rightrightarrows;":                '\U000021C9',
+	"rightsquigarrow;":                 '\U0000219D',
+	"rightthreetimes;":                 '\U000022CC',
+	"ring;":                            '\U000002DA',
+	"risingdotseq;":                    '\U00002253',
+	"rlarr;":                           '\U000021C4',
+	"rlhar;":                           '\U000021CC',
+	"rlm;":                             '\U0000200F',
+	"rmoust;":                          '\U000023B1',
+	"rmoustache;":                      '\U000023B1',
+	"rnmid;":                           '\U00002AEE',
+	"roang;":                           '\U000027ED',
+	"roarr;":                           '\U000021FE',
+	"robrk;":                           '\U000027E7',
+	"ropar;":                           '\U00002986',
+	"ropf;":                            '\U0001D563',
+	"roplus;":                          '\U00002A2E',
+	"rotimes;":                         '\U00002A35',
+	"rpar;":                            '\U00000029',
+	"rpargt;":                          '\U00002994',
+	"rppolint;":                        '\U00002A12',
+	"rrarr;":                           '\U000021C9',
+	"rsaquo;":                          '\U0000203A',
+	"rscr;":                            '\U0001D4C7',
+	"rsh;":                             '\U000021B1',
+	"rsqb;":                            '\U0000005D',
+	"rsquo;":                           '\U00002019',
+	"rsquor;":                          '\U00002019',
+	"rthree;":                          '\U000022CC',
+	"rtimes;":                          '\U000022CA',
+	"rtri;":                            '\U000025B9',
+	"rtrie;":                           '\U000022B5',
+	"rtrif;":                           '\U000025B8',
+	"rtriltri;":                        '\U000029CE',
+	"ruluhar;":                         '\U00002968',
+	"rx;":                              '\U0000211E',
+	"sacute;":                          '\U0000015B',
+	"sbquo;":                           '\U0000201A',
+	"sc;":                              '\U0000227B',
+	"scE;":                             '\U00002AB4',
+	"scap;":                            '\U00002AB8',
+	"scaron;":                          '\U00000161',
+	"sccue;":                           '\U0000227D',
+	"sce;":                             '\U00002AB0',
+	"scedil;":                          '\U0000015F',
+	"scirc;":                           '\U0000015D',
+	"scnE;":                            '\U00002AB6',
+	"scnap;":                           '\U00002ABA',
+	"scnsim;":                          '\U000022E9',
+	"scpolint;":                        '\U00002A13',
+	"scsim;":                           '\U0000227F',
+	"scy;":                             '\U00000441',
+	"sdot;":                            '\U000022C5',
+	"sdotb;":                           '\U000022A1',
+	"sdote;":                           '\U00002A66',
+	"seArr;":                           '\U000021D8',
+	"searhk;":                          '\U00002925',
+	"searr;":                           '\U00002198',
+	"searrow;":                         '\U00002198',
+	"sect;":                            '\U000000A7',
+	"semi;":                            '\U0000003B',
+	"seswar;":                          '\U00002929',
+	"setminus;":                        '\U00002216',
+	"setmn;":                           '\U00002216',
+	"sext;":                            '\U00002736',
+	"sfr;":                             '\U0001D530',
+	"sfrown;":                          '\U00002322',
+	"sharp;":                           '\U0000266F',
+	"shchcy;":                          '\U00000449',
+	"shcy;":                            '\U00000448',
+	"shortmid;":                        '\U00002223',
+	"shortparallel;":                   '\U00002225',
+	"shy;":                             '\U000000AD',
+	"sigma;":                           '\U000003C3',
+	"sigmaf;":                          '\U000003C2',
+	"sigmav;":                          '\U000003C2',
+	"sim;":                             '\U0000223C',
+	"simdot;":                          '\U00002A6A',
+	"sime;":                            '\U00002243',
+	"simeq;":                           '\U00002243',
+	"simg;":                            '\U00002A9E',
+	"simgE;":                           '\U00002AA0',
+	"siml;":                            '\U00002A9D',
+	"simlE;":                           '\U00002A9F',
+	"simne;":                           '\U00002246',
+	"simplus;":                         '\U00002A24',
+	"simrarr;":                         '\U00002972',
+	"slarr;":                           '\U00002190',
+	"smallsetminus;":                   '\U00002216',
+	"smashp;":                          '\U00002A33',
+	"smeparsl;":                        '\U000029E4',
+	"smid;":                            '\U00002223',
+	"smile;":                           '\U00002323',
+	"smt;":                             '\U00002AAA',
+	"smte;":                            '\U00002AAC',
+	"softcy;":                          '\U0000044C',
+	"sol;":                             '\U0000002F',
+	"solb;":                            '\U000029C4',
+	"solbar;":                          '\U0000233F',
+	"sopf;":                            '\U0001D564',
+	"spades;":                          '\U00002660',
+	"spadesuit;":                       '\U00002660',
+	"spar;":                            '\U00002225',
+	"sqcap;":                           '\U00002293',
+	"sqcup;":                           '\U00002294',
+	"sqsub;":                           '\U0000228F',
+	"sqsube;":                          '\U00002291',
+	"sqsubset;":                        '\U0000228F',
+	"sqsubseteq;":                      '\U00002291',
+	"sqsup;":                           '\U00002290',
+	"sqsupe;":                          '\U00002292',
+	"sqsupset;":                        '\U00002290',
+	"sqsupseteq;":                      '\U00002292',
+	"squ;":                             '\U000025A1',
+	"square;":                          '\U000025A1',
+	"squarf;":                          '\U000025AA',
+	"squf;":                            '\U000025AA',
+	"srarr;":                           '\U00002192',
+	"sscr;":                            '\U0001D4C8',
+	"ssetmn;":                          '\U00002216',
+	"ssmile;":                          '\U00002323',
+	"sstarf;":                          '\U000022C6',
+	"star;":                            '\U00002606',
+	"starf;":                           '\U00002605',
+	"straightepsilon;":                 '\U000003F5',
+	"straightphi;":                     '\U000003D5',
+	"strns;":                           '\U000000AF',
+	"sub;":                             '\U00002282',
+	"subE;":                            '\U00002AC5',
+	"subdot;":                          '\U00002ABD',
+	"sube;":                            '\U00002286',
+	"subedot;":                         '\U00002AC3',
+	"submult;":                         '\U00002AC1',
+	"subnE;":                           '\U00002ACB',
+	"subne;":                           '\U0000228A',
+	"subplus;":                         '\U00002ABF',
+	"subrarr;":                         '\U00002979',
+	"subset;":                          '\U00002282',
+	"subseteq;":                        '\U00002286',
+	"subseteqq;":                       '\U00002AC5',
+	"subsetneq;":                       '\U0000228A',
+	"subsetneqq;":                      '\U00002ACB',
+	"subsim;":                          '\U00002AC7',
+	"subsub;":                          '\U00002AD5',
+	"subsup;":                          '\U00002AD3',
+	"succ;":                            '\U0000227B',
+	"succapprox;":                      '\U00002AB8',
+	"succcurlyeq;":                     '\U0000227D',
+	"succeq;":                          '\U00002AB0',
+	"succnapprox;":                     '\U00002ABA',
+	"succneqq;":                        '\U00002AB6',
+	"succnsim;":                        '\U000022E9',
+	"succsim;":                         '\U0000227F',
+	"sum;":                             '\U00002211',
+	"sung;":                            '\U0000266A',
+	"sup;":                             '\U00002283',
+	"sup1;":                            '\U000000B9',
+	"sup2;":                            '\U000000B2',
+	"sup3;":                            '\U000000B3',
+	"supE;":                            '\U00002AC6',
+	"supdot;":                          '\U00002ABE',
+	"supdsub;":                         '\U00002AD8',
+	"supe;":                            '\U00002287',
+	"supedot;":                         '\U00002AC4',
+	"suphsol;":                         '\U000027C9',
+	"suphsub;":                         '\U00002AD7',
+	"suplarr;":                         '\U0000297B',
+	"supmult;":                         '\U00002AC2',
+	"supnE;":                           '\U00002ACC',
+	"supne;":                           '\U0000228B',
+	"supplus;":                         '\U00002AC0',
+	"supset;":                          '\U00002283',
+	"supseteq;":                        '\U00002287',
+	"supseteqq;":                       '\U00002AC6',
+	"supsetneq;":                       '\U0000228B',
+	"supsetneqq;":                      '\U00002ACC',
+	"supsim;":                          '\U00002AC8',
+	"supsub;":                          '\U00002AD4',
+	"supsup;":                          '\U00002AD6',
+	"swArr;":                           '\U000021D9',
+	"swarhk;":                          '\U00002926',
+	"swarr;":                           '\U00002199',
+	"swarrow;":                         '\U00002199',
+	"swnwar;":                          '\U0000292A',
+	"szlig;":                           '\U000000DF',
+	"target;":                          '\U00002316',
+	"tau;":                             '\U000003C4',
+	"tbrk;":                            '\U000023B4',
+	"tcaron;":                          '\U00000165',
+	"tcedil;":                          '\U00000163',
+	"tcy;":                             '\U00000442',
+	"tdot;":                            '\U000020DB',
+	"telrec;":                          '\U00002315',
+	"tfr;":                             '\U0001D531',
+	"there4;":                          '\U00002234',
+	"therefore;":                       '\U00002234',
+	"theta;":                           '\U000003B8',
+	"thetasym;":                        '\U000003D1',
+	"thetav;":                          '\U000003D1',
+	"thickapprox;":                     '\U00002248',
+	"thicksim;":                        '\U0000223C',
+	"thinsp;":                          '\U00002009',
+	"thkap;":                           '\U00002248',
+	"thksim;":                          '\U0000223C',
+	"thorn;":                           '\U000000FE',
+	"tilde;":                           '\U000002DC',
+	"times;":                           '\U000000D7',
+	"timesb;":                          '\U000022A0',
+	"timesbar;":                        '\U00002A31',
+	"timesd;":                          '\U00002A30',
+	"tint;":                            '\U0000222D',
+	"toea;":                            '\U00002928',
+	"top;":                             '\U000022A4',
+	"topbot;":                          '\U00002336',
+	"topcir;":                          '\U00002AF1',
+	"topf;":                            '\U0001D565',
+	"topfork;":                         '\U00002ADA',
+	"tosa;":                            '\U00002929',
+	"tprime;":                          '\U00002034',
+	"trade;":                           '\U00002122',
+	"triangle;":                        '\U000025B5',
+	"triangledown;":                    '\U000025BF',
+	"triangleleft;":                    '\U000025C3',
+	"trianglelefteq;":                  '\U000022B4',
+	"triangleq;":                       '\U0000225C',
+	"triangleright;":                   '\U000025B9',
+	"trianglerighteq;":                 '\U000022B5',
+	"tridot;":                          '\U000025EC',
+	"trie;":                            '\U0000225C',
+	"triminus;":                        '\U00002A3A',
+	"triplus;":                         '\U00002A39',
+	"trisb;":                           '\U000029CD',
+	"tritime;":                         '\U00002A3B',
+	"trpezium;":                        '\U000023E2',
+	"tscr;":                            '\U0001D4C9',
+	"tscy;":                            '\U00000446',
+	"tshcy;":                           '\U0000045B',
+	"tstrok;":                          '\U00000167',
+	"twixt;":                           '\U0000226C',
+	"twoheadleftarrow;":                '\U0000219E',
+	"twoheadrightarrow;":               '\U000021A0',
+	"uArr;":                            '\U000021D1',
+	"uHar;":                            '\U00002963',
+	"uacute;":                          '\U000000FA',
+	"uarr;":                            '\U00002191',
+	"ubrcy;":                           '\U0000045E',
+	"ubreve;":                          '\U0000016D',
+	"ucirc;":                           '\U000000FB',
+	"ucy;":                             '\U00000443',
+	"udarr;":                           '\U000021C5',
+	"udblac;":                          '\U00000171',
+	"udhar;":                           '\U0000296E',
+	"ufisht;":                          '\U0000297E',
+	"ufr;":                             '\U0001D532',
+	"ugrave;":                          '\U000000F9',
+	"uharl;":                           '\U000021BF',
+	"uharr;":                           '\U000021BE',
+	"uhblk;":                           '\U00002580',
+	"ulcorn;":                          '\U0000231C',
+	"ulcorner;":                        '\U0000231C',
+	"ulcrop;":                          '\U0000230F',
+	"ultri;":                           '\U000025F8',
+	"umacr;":                           '\U0000016B',
+	"uml;":                             '\U000000A8',
+	"uogon;":                           '\U00000173',
+	"uopf;":                            '\U0001D566',
+	"uparrow;":                         '\U00002191',
+	"updownarrow;":                     '\U00002195',
+	"upharpoonleft;":                   '\U000021BF',
+	"upharpoonright;":                  '\U000021BE',
+	"uplus;":                           '\U0000228E',
+	"upsi;":                            '\U000003C5',
+	"upsih;":                           '\U000003D2',
+	"upsilon;":                         '\U000003C5',
+	"upuparrows;":                      '\U000021C8',
+	"urcorn;":                          '\U0000231D',
+	"urcorner;":                        '\U0000231D',
+	"urcrop;":                          '\U0000230E',
+	"uring;":                           '\U0000016F',
+	"urtri;":                           '\U000025F9',
+	"uscr;":                            '\U0001D4CA',
+	"utdot;":                           '\U000022F0',
+	"utilde;":                          '\U00000169',
+	"utri;":                            '\U000025B5',
+	"utrif;":                           '\U000025B4',
+	"uuarr;":                           '\U000021C8',
+	"uuml;":                            '\U000000FC',
+	"uwangle;":                         '\U000029A7',
+	"vArr;":                            '\U000021D5',
+	"vBar;":                            '\U00002AE8',
+	"vBarv;":                           '\U00002AE9',
+	"vDash;":                           '\U000022A8',
+	"vangrt;":                          '\U0000299C',
+	"varepsilon;":                      '\U000003F5',
+	"varkappa;":                        '\U000003F0',
+	"varnothing;":                      '\U00002205',
+	"varphi;":                          '\U000003D5',
+	"varpi;":                           '\U000003D6',
+	"varpropto;":                       '\U0000221D',
+	"varr;":                            '\U00002195',
+	"varrho;":                          '\U000003F1',
+	"varsigma;":                        '\U000003C2',
+	"vartheta;":                        '\U000003D1',
+	"vartriangleleft;":                 '\U000022B2',
+	"vartriangleright;":                '\U000022B3',
+	"vcy;":                             '\U00000432',
+	"vdash;":                           '\U000022A2',
+	"vee;":                             '\U00002228',
+	"veebar;":                          '\U000022BB',
+	"veeeq;":                           '\U0000225A',
+	"vellip;":                          '\U000022EE',
+	"verbar;":                          '\U0000007C',
+	"vert;":                            '\U0000007C',
+	"vfr;":                             '\U0001D533',
+	"vltri;":                           '\U000022B2',
+	"vopf;":                            '\U0001D567',
+	"vprop;":                           '\U0000221D',
+	"vrtri;":                           '\U000022B3',
+	"vscr;":                            '\U0001D4CB',
+	"vzigzag;":                         '\U0000299A',
+	"wcirc;":                           '\U00000175',
+	"wedbar;":                          '\U00002A5F',
+	"wedge;":                           '\U00002227',
+	"wedgeq;":                          '\U00002259',
+	"weierp;":                          '\U00002118',
+	"wfr;":                             '\U0001D534',
+	"wopf;":                            '\U0001D568',
+	"wp;":                              '\U00002118',
+	"wr;":                              '\U00002240',
+	"wreath;":                          '\U00002240',
+	"wscr;":                            '\U0001D4CC',
+	"xcap;":                            '\U000022C2',
+	"xcirc;":                           '\U000025EF',
+	"xcup;":                            '\U000022C3',
+	"xdtri;":                           '\U000025BD',
+	"xfr;":                             '\U0001D535',
+	"xhArr;":                           '\U000027FA',
+	"xharr;":                           '\U000027F7',
+	"xi;":                              '\U000003BE',
+	"xlArr;":                           '\U000027F8',
+	"xlarr;":                           '\U000027F5',
+	"xmap;":                            '\U000027FC',
+	"xnis;":                            '\U000022FB',
+	"xodot;":                           '\U00002A00',
+	"xopf;":                            '\U0001D569',
+	"xoplus;":                          '\U00002A01',
+	"xotime;":                          '\U00002A02',
+	"xrArr;":                           '\U000027F9',
+	"xrarr;":                           '\U000027F6',
+	"xscr;":                            '\U0001D4CD',
+	"xsqcup;":                          '\U00002A06',
+	"xuplus;":                          '\U00002A04',
+	"xutri;":                           '\U000025B3',
+	"xvee;":                            '\U000022C1',
+	"xwedge;":                          '\U000022C0',
+	"yacute;":                          '\U000000FD',
+	"yacy;":                            '\U0000044F',
+	"ycirc;":                           '\U00000177',
+	"ycy;":                             '\U0000044B',
+	"yen;":                             '\U000000A5',
+	"yfr;":                             '\U0001D536',
+	"yicy;":                            '\U00000457',
+	"yopf;":                            '\U0001D56A',
+	"yscr;":                            '\U0001D4CE',
+	"yucy;":                            '\U0000044E',
+	"yuml;":                            '\U000000FF',
+	"zacute;":                          '\U0000017A',
+	"zcaron;":                          '\U0000017E',
+	"zcy;":                             '\U00000437',
+	"zdot;":                            '\U0000017C',
+	"zeetrf;":                          '\U00002128',
+	"zeta;":                            '\U000003B6',
+	"zfr;":                             '\U0001D537',
+	"zhcy;":                            '\U00000436',
+	"zigrarr;":                         '\U000021DD',
+	"zopf;":                            '\U0001D56B',
+	"zscr;":                            '\U0001D4CF',
+	"zwj;":                             '\U0000200D',
+	"zwnj;":                            '\U0000200C',
+	"AElig":                            '\U000000C6',
+	"AMP":                              '\U00000026',
+	"Aacute":                           '\U000000C1',
+	"Acirc":                            '\U000000C2',
+	"Agrave":                           '\U000000C0',
+	"Aring":                            '\U000000C5',
+	"Atilde":                           '\U000000C3',
+	"Auml":                             '\U000000C4',
+	"COPY":                             '\U000000A9',
+	"Ccedil":                           '\U000000C7',
+	"ETH":                              '\U000000D0',
+	"Eacute":                           '\U000000C9',
+	"Ecirc":                            '\U000000CA',
+	"Egrave":                           '\U000000C8',
+	"Euml":                             '\U000000CB',
+	"GT":                               '\U0000003E',
+	"Iacute":                           '\U000000CD',
+	"Icirc":                            '\U000000CE',
+	"Igrave":                           '\U000000CC',
+	"Iuml":                             '\U000000CF',
+	"LT":                               '\U0000003C',
+	"Ntilde":                           '\U000000D1',
+	"Oacute":                           '\U000000D3',
+	"Ocirc":                            '\U000000D4',
+	"Ograve":                           '\U000000D2',
+	"Oslash":                           '\U000000D8',
+	"Otilde":                           '\U000000D5',
+	"Ouml":                             '\U000000D6',
+	"QUOT":                             '\U00000022',
+	"REG":                              '\U000000AE',
+	"THORN":                            '\U000000DE',
+	"Uacute":                           '\U000000DA',
+	"Ucirc":                            '\U000000DB',
+	"Ugrave":                           '\U000000D9',
+	"Uuml":                             '\U000000DC',
+	"Yacute":                           '\U000000DD',
+	"aacute":                           '\U000000E1',
+	"acirc":                            '\U000000E2',
+	"acute":                            '\U000000B4',
+	"aelig":                            '\U000000E6',
+	"agrave":                           '\U000000E0',
+	"amp":                              '\U00000026',
+	"aring":                            '\U000000E5',
+	"atilde":                           '\U000000E3',
+	"auml":                             '\U000000E4',
+	"brvbar":                           '\U000000A6',
+	"ccedil":                           '\U000000E7',
+	"cedil":                            '\U000000B8',
+	"cent":                             '\U000000A2',
+	"copy":                             '\U000000A9',
+	"curren":                           '\U000000A4',
+	"deg":                              '\U000000B0',
+	"divide":                           '\U000000F7',
+	"eacute":                           '\U000000E9',
+	"ecirc":                            '\U000000EA',
+	"egrave":                           '\U000000E8',
+	"eth":                              '\U000000F0',
+	"euml":                             '\U000000EB',
+	"frac12":                           '\U000000BD',
+	"frac14":                           '\U000000BC',
+	"frac34":                           '\U000000BE',
+	"gt":                               '\U0000003E',
+	"iacute":                           '\U000000ED',
+	"icirc":                            '\U000000EE',
+	"iexcl":                            '\U000000A1',
+	"igrave":                           '\U000000EC',
+	"iquest":                           '\U000000BF',
+	"iuml":                             '\U000000EF',
+	"laquo":                            '\U000000AB',
+	"lt":                               '\U0000003C',
+	"macr":                             '\U000000AF',
+	"micro":                            '\U000000B5',
+	"middot":                           '\U000000B7',
+	"nbsp":                             '\U000000A0',
+	"not":                              '\U000000AC',
+	"ntilde":                           '\U000000F1',
+	"oacute":                           '\U000000F3',
+	"ocirc":                            '\U000000F4',
+	"ograve":                           '\U000000F2',
+	"ordf":                             '\U000000AA',
+	"ordm":                             '\U000000BA',
+	"oslash":                           '\U000000F8',
+	"otilde":                           '\U000000F5',
+	"ouml":                             '\U000000F6',
+	"para":                             '\U000000B6',
+	"plusmn":                           '\U000000B1',
+	"pound":                            '\U000000A3',
+	"quot":                             '\U00000022',
+	"raquo":                            '\U000000BB',
+	"reg":                              '\U000000AE',
+	"sect":                             '\U000000A7',
+	"shy":                              '\U000000AD',
+	"sup1":                             '\U000000B9',
+	"sup2":                             '\U000000B2',
+	"sup3":                             '\U000000B3',
+	"szlig":                            '\U000000DF',
+	"thorn":                            '\U000000FE',
+	"times":                            '\U000000D7',
+	"uacute":                           '\U000000FA',
+	"ucirc":                            '\U000000FB',
+	"ugrave":                           '\U000000F9',
+	"uml":                              '\U000000A8',
+	"uuml":                             '\U000000FC',
+	"yacute":                           '\U000000FD',
+	"yen":                              '\U000000A5',
+	"yuml":                             '\U000000FF',
+}
+
+// HTML entities that are two unicode codepoints.
+var entity2 = map[string][2]int{
+	// TODO(nigeltao): Handle replacements that are wider than their names.
+	// "nLt;":                     {'\u226A', '\u20D2'},
+	// "nGt;":                     {'\u226B', '\u20D2'},
+	"NotEqualTilde;":           {'\u2242', '\u0338'},
+	"NotGreaterFullEqual;":     {'\u2267', '\u0338'},
+	"NotGreaterGreater;":       {'\u226B', '\u0338'},
+	"NotGreaterSlantEqual;":    {'\u2A7E', '\u0338'},
+	"NotHumpDownHump;":         {'\u224E', '\u0338'},
+	"NotHumpEqual;":            {'\u224F', '\u0338'},
+	"NotLeftTriangleBar;":      {'\u29CF', '\u0338'},
+	"NotLessLess;":             {'\u226A', '\u0338'},
+	"NotLessSlantEqual;":       {'\u2A7D', '\u0338'},
+	"NotNestedGreaterGreater;": {'\u2AA2', '\u0338'},
+	"NotNestedLessLess;":       {'\u2AA1', '\u0338'},
+	"NotPrecedesEqual;":        {'\u2AAF', '\u0338'},
+	"NotRightTriangleBar;":     {'\u29D0', '\u0338'},
+	"NotSquareSubset;":         {'\u228F', '\u0338'},
+	"NotSquareSuperset;":       {'\u2290', '\u0338'},
+	"NotSubset;":               {'\u2282', '\u20D2'},
+	"NotSucceedsEqual;":        {'\u2AB0', '\u0338'},
+	"NotSucceedsTilde;":        {'\u227F', '\u0338'},
+	"NotSuperset;":             {'\u2283', '\u20D2'},
+	"ThickSpace;":              {'\u205F', '\u200A'},
+	"acE;":                     {'\u223E', '\u0333'},
+	"bne;":                     {'\u003D', '\u20E5'},
+	"bnequiv;":                 {'\u2261', '\u20E5'},
+	"caps;":                    {'\u2229', '\uFE00'},
+	"cups;":                    {'\u222A', '\uFE00'},
+	"fjlig;":                   {'\u0066', '\u006A'},
+	"gesl;":                    {'\u22DB', '\uFE00'},
+	"gvertneqq;":               {'\u2269', '\uFE00'},
+	"gvnE;":                    {'\u2269', '\uFE00'},
+	"lates;":                   {'\u2AAD', '\uFE00'},
+	"lesg;":                    {'\u22DA', '\uFE00'},
+	"lvertneqq;":               {'\u2268', '\uFE00'},
+	"lvnE;":                    {'\u2268', '\uFE00'},
+	"nGg;":                     {'\u22D9', '\u0338'},
+	"nGtv;":                    {'\u226B', '\u0338'},
+	"nLl;":                     {'\u22D8', '\u0338'},
+	"nLtv;":                    {'\u226A', '\u0338'},
+	"nang;":                    {'\u2220', '\u20D2'},
+	"napE;":                    {'\u2A70', '\u0338'},
+	"napid;":                   {'\u224B', '\u0338'},
+	"nbump;":                   {'\u224E', '\u0338'},
+	"nbumpe;":                  {'\u224F', '\u0338'},
+	"ncongdot;":                {'\u2A6D', '\u0338'},
+	"nedot;":                   {'\u2250', '\u0338'},
+	"nesim;":                   {'\u2242', '\u0338'},
+	"ngE;":                     {'\u2267', '\u0338'},
+	"ngeqq;":                   {'\u2267', '\u0338'},
+	"ngeqslant;":               {'\u2A7E', '\u0338'},
+	"nges;":                    {'\u2A7E', '\u0338'},
+	"nlE;":                     {'\u2266', '\u0338'},
+	"nleqq;":                   {'\u2266', '\u0338'},
+	"nleqslant;":               {'\u2A7D', '\u0338'},
+	"nles;":                    {'\u2A7D', '\u0338'},
+	"notinE;":                  {'\u22F9', '\u0338'},
+	"notindot;":                {'\u22F5', '\u0338'},
+	"nparsl;":                  {'\u2AFD', '\u20E5'},
+	"npart;":                   {'\u2202', '\u0338'},
+	"npre;":                    {'\u2AAF', '\u0338'},
+	"npreceq;":                 {'\u2AAF', '\u0338'},
+	"nrarrc;":                  {'\u2933', '\u0338'},
+	"nrarrw;":                  {'\u219D', '\u0338'},
+	"nsce;":                    {'\u2AB0', '\u0338'},
+	"nsubE;":                   {'\u2AC5', '\u0338'},
+	"nsubset;":                 {'\u2282', '\u20D2'},
+	"nsubseteqq;":              {'\u2AC5', '\u0338'},
+	"nsucceq;":                 {'\u2AB0', '\u0338'},
+	"nsupE;":                   {'\u2AC6', '\u0338'},
+	"nsupset;":                 {'\u2283', '\u20D2'},
+	"nsupseteqq;":              {'\u2AC6', '\u0338'},
+	"nvap;":                    {'\u224D', '\u20D2'},
+	"nvge;":                    {'\u2265', '\u20D2'},
+	"nvgt;":                    {'\u003E', '\u20D2'},
+	"nvle;":                    {'\u2264', '\u20D2'},
+	"nvlt;":                    {'\u003C', '\u20D2'},
+	"nvltrie;":                 {'\u22B4', '\u20D2'},
+	"nvrtrie;":                 {'\u22B5', '\u20D2'},
+	"nvsim;":                   {'\u223C', '\u20D2'},
+	"race;":                    {'\u223D', '\u0331'},
+	"smtes;":                   {'\u2AAC', '\uFE00'},
+	"sqcaps;":                  {'\u2293', '\uFE00'},
+	"sqcups;":                  {'\u2294', '\uFE00'},
+	"varsubsetneq;":            {'\u228A', '\uFE00'},
+	"varsubsetneqq;":           {'\u2ACB', '\uFE00'},
+	"varsupsetneq;":            {'\u228B', '\uFE00'},
+	"varsupsetneqq;":           {'\u2ACC', '\uFE00'},
+	"vnsub;":                   {'\u2282', '\u20D2'},
+	"vnsup;":                   {'\u2283', '\u20D2'},
+	"vsubnE;":                  {'\u2ACB', '\uFE00'},
+	"vsubne;":                  {'\u228A', '\uFE00'},
+	"vsupnE;":                  {'\u2ACC', '\uFE00'},
+	"vsupne;":                  {'\u228B', '\uFE00'},
+}
diff --git a/src/pkg/html/entity_test.go b/src/pkg/html/entity_test.go
new file mode 100644
index 000000000..2cf49d61d
--- /dev/null
+++ b/src/pkg/html/entity_test.go
@@ -0,0 +1,29 @@
+// Copyright 2010 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package html
+
+import (
+	"testing"
+	"utf8"
+)
+
+func TestEntityLength(t *testing.T) {
+	// We verify that the length of UTF-8 encoding of each value is <= 1 + len(key).
+	// The +1 comes from the leading "&". This property implies that the length of
+	// unescaped text is <= the length of escaped text.
+	for k, v := range entity {
+		if 1+len(k) < utf8.RuneLen(v) {
+			t.Error("escaped entity &" + k + " is shorter than its UTF-8 encoding " + string(v))
+		}
+		if len(k) > longestEntityWithoutSemicolon && k[len(k)-1] != ';' {
+			t.Errorf("entity name %s is %d characters, but longestEntityWithoutSemicolon=%d", k, len(k), longestEntityWithoutSemicolon)
+		}
+	}
+	for k, v := range entity2 {
+		if 1+len(k) < utf8.RuneLen(v[0])+utf8.RuneLen(v[1]) {
+			t.Error("escaped entity &" + k + " is shorter than its UTF-8 encoding " + string(v[0]) + string(v[1]))
+		}
+	}
+}
diff --git a/src/pkg/html/escape.go b/src/pkg/html/escape.go
new file mode 100644
index 000000000..0de97c5ac
--- /dev/null
+++ b/src/pkg/html/escape.go
@@ -0,0 +1,238 @@
+// Copyright 2010 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package html
+
+import (
+	"bytes"
+	"strings"
+	"utf8"
+)
+
+// These replacements permit compatibility with old numeric entities that 
+// assumed Windows-1252 encoding.
+// http://www.whatwg.org/specs/web-apps/current-work/multipage/tokenization.html#consume-a-character-reference
+var replacementTable = [...]int{
+	'\u20AC', // First entry is what 0x80 should be replaced with.
+	'\u0081',
+	'\u201A',
+	'\u0192',
+	'\u201E',
+	'\u2026',
+	'\u2020',
+	'\u2021',
+	'\u02C6',
+	'\u2030',
+	'\u0160',
+	'\u2039',
+	'\u0152',
+	'\u008D',
+	'\u017D',
+	'\u008F',
+	'\u0090',
+	'\u2018',
+	'\u2019',
+	'\u201C',
+	'\u201D',
+	'\u2022',
+	'\u2013',
+	'\u2014',
+	'\u02DC',
+	'\u2122',
+	'\u0161',
+	'\u203A',
+	'\u0153',
+	'\u009D',
+	'\u017E',
+	'\u0178', // Last entry is 0x9F.
+	// 0x00->'\uFFFD' is handled programmatically. 
+	// 0x0D->'\u000D' is a no-op.
+}
+
+// unescapeEntity reads an entity like "<" from b[src:] and writes the
+// corresponding "<" to b[dst:], returning the incremented dst and src cursors.
+// Precondition: b[src] == '&' && dst <= src.
+// attribute should be true if parsing an attribute value.
+func unescapeEntity(b []byte, dst, src int, attribute bool) (dst1, src1 int) {
+	// http://www.whatwg.org/specs/web-apps/current-work/multipage/tokenization.html#consume-a-character-reference
+
+	// i starts at 1 because we already know that s[0] == '&'.
+	i, s := 1, b[src:]
+
+	if len(s) <= 1 {
+		b[dst] = b[src]
+		return dst + 1, src + 1
+	}
+
+	if s[i] == '#' {
+		if len(s) <= 3 { // We need to have at least "&#.".
+			b[dst] = b[src]
+			return dst + 1, src + 1
+		}
+		i++
+		c := s[i]
+		hex := false
+		if c == 'x' || c == 'X' {
+			hex = true
+			i++
+		}
+
+		x := 0
+		for i < len(s) {
+			c = s[i]
+			i++
+			if hex {
+				if '0' <= c && c <= '9' {
+					x = 16*x + int(c) - '0'
+					continue
+				} else if 'a' <= c && c <= 'f' {
+					x = 16*x + int(c) - 'a' + 10
+					continue
+				} else if 'A' <= c && c <= 'F' {
+					x = 16*x + int(c) - 'A' + 10
+					continue
+				}
+			} else if '0' <= c && c <= '9' {
+				x = 10*x + int(c) - '0'
+				continue
+			}
+			if c != ';' {
+				i--
+			}
+			break
+		}
+
+		if i <= 3 { // No characters matched.
+			b[dst] = b[src]
+			return dst + 1, src + 1
+		}
+
+		if 0x80 <= x && x <= 0x9F {
+			// Replace characters from Windows-1252 with UTF-8 equivalents.
+			x = replacementTable[x-0x80]
+		} else if x == 0 || (0xD800 <= x && x <= 0xDFFF) || x > 0x10FFFF {
+			// Replace invalid characters with the replacement character.
+			x = '\uFFFD'
+		}
+
+		return dst + utf8.EncodeRune(b[dst:], x), src + i
+	}
+
+	// Consume the maximum number of characters possible, with the
+	// consumed characters matching one of the named references.
+
+	for i < len(s) {
+		c := s[i]
+		i++
+		// Lower-cased characters are more common in entities, so we check for them first.
+		if 'a' <= c && c <= 'z' || 'A' <= c && c <= 'Z' || '0' <= c && c <= '9' {
+			continue
+		}
+		if c != ';' {
+			i--
+		}
+		break
+	}
+
+	entityName := string(s[1:i])
+	if entityName == "" {
+		// No-op.
+	} else if attribute && entityName[len(entityName)-1] != ';' && len(s) > i && s[i] == '=' {
+		// No-op.
+	} else if x := entity[entityName]; x != 0 {
+		return dst + utf8.EncodeRune(b[dst:], x), src + i
+	} else if x := entity2[entityName]; x[0] != 0 {
+		dst1 := dst + utf8.EncodeRune(b[dst:], x[0])
+		return dst1 + utf8.EncodeRune(b[dst1:], x[1]), src + i
+	} else if !attribute {
+		maxLen := len(entityName) - 1
+		if maxLen > longestEntityWithoutSemicolon {
+			maxLen = longestEntityWithoutSemicolon
+		}
+		for j := maxLen; j > 1; j-- {
+			if x := entity[entityName[:j]]; x != 0 {
+				return dst + utf8.EncodeRune(b[dst:], x), src + j + 1
+			}
+		}
+	}
+
+	dst1, src1 = dst+i, src+i
+	copy(b[dst:dst1], b[src:src1])
+	return dst1, src1
+}
+
+// unescape unescapes b's entities in-place, so that "a<b" becomes "a"`
+
+func escape(buf *bytes.Buffer, s string) {
+	i := strings.IndexAny(s, escapedChars)
+	for i != -1 {
+		buf.WriteString(s[0:i])
+		var esc string
+		switch s[i] {
+		case '&':
+			esc = "&"
+		case '\'':
+			esc = "'"
+		case '<':
+			esc = "<"
+		case '>':
+			esc = ">"
+		case '"':
+			esc = """
+		default:
+			panic("unrecognized escape character")
+		}
+		s = s[i+1:]
+		buf.WriteString(esc)
+		i = strings.IndexAny(s, escapedChars)
+	}
+	buf.WriteString(s)
+}
+
+// EscapeString escapes special characters like "<" to become "<". It
+// escapes only five such characters: amp, apos, lt, gt and quot.
+// UnescapeString(EscapeString(s)) == s always holds, but the converse isn't
+// always true.
+func EscapeString(s string) string {
+	if strings.IndexAny(s, escapedChars) == -1 {
+		return s
+	}
+	buf := bytes.NewBuffer(nil)
+	escape(buf, s)
+	return buf.String()
+}
+
+// UnescapeString unescapes entities like "<" to become "<". It unescapes a
+// larger range of entities than EscapeString escapes. For example, "á"
+// unescapes to "á", as does "á" and "&xE1;".
+// UnescapeString(EscapeString(s)) == s always holds, but the converse isn't
+// always true.
+func UnescapeString(s string) string {
+	for _, c := range s {
+		if c == '&' {
+			return string(unescape([]byte(s)))
+		}
+	}
+	return s
+}
diff --git a/src/pkg/html/node.go b/src/pkg/html/node.go
new file mode 100644
index 000000000..4ecfd6ca2
--- /dev/null
+++ b/src/pkg/html/node.go
@@ -0,0 +1,147 @@
+// Copyright 2011 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package html
+
+// A NodeType is the type of a Node.
+type NodeType int
+
+const (
+	ErrorNode NodeType = iota
+	TextNode
+	DocumentNode
+	ElementNode
+	CommentNode
+	DoctypeNode
+	scopeMarkerNode
+)
+
+// Section 11.2.3.3 says "scope markers are inserted when entering applet
+// elements, buttons, object elements, marquees, table cells, and table
+// captions, and are used to prevent formatting from 'leaking'".
+var scopeMarker = Node{Type: scopeMarkerNode}
+
+// A Node consists of a NodeType and some Data (tag name for element nodes,
+// content for text) and are part of a tree of Nodes. Element nodes may also
+// contain a slice of Attributes. Data is unescaped, so that it looks like
+// "a 0 {
+		return (*s)[i-1]
+	}
+	return nil
+}
+
+// index returns the index of the top-most occurence of n in the stack, or -1
+// if n is not present.
+func (s *nodeStack) index(n *Node) int {
+	for i := len(*s) - 1; i >= 0; i-- {
+		if (*s)[i] == n {
+			return i
+		}
+	}
+	return -1
+}
+
+// insert inserts a node at the given index.
+func (s *nodeStack) insert(i int, n *Node) {
+	(*s) = append(*s, nil)
+	copy((*s)[i+1:], (*s)[i:])
+	(*s)[i] = n
+}
+
+// remove removes a node from the stack. It is a no-op if n is not present.
+func (s *nodeStack) remove(n *Node) {
+	i := s.index(n)
+	if i == -1 {
+		return
+	}
+	copy((*s)[i:], (*s)[i+1:])
+	j := len(*s) - 1
+	(*s)[j] = nil
+	*s = (*s)[:j]
+}
+
+// forTag returns the top-most element node with the given tag.
+func (s *nodeStack) forTag(tag string) *Node {
+	for i := len(*s) - 1; i >= 0; i-- {
+		n := (*s)[i]
+		if n.Type == ElementNode && n.Data == tag {
+			return n
+		}
+	}
+	return nil
+}
diff --git a/src/pkg/html/parse.go b/src/pkg/html/parse.go
new file mode 100644
index 000000000..519ebe587
--- /dev/null
+++ b/src/pkg/html/parse.go
@@ -0,0 +1,800 @@
+// Copyright 2010 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package html
+
+import (
+	"io"
+	"os"
+)
+
+// A parser implements the HTML5 parsing algorithm:
+// http://www.whatwg.org/specs/web-apps/current-work/multipage/tokenization.html#tree-construction
+type parser struct {
+	// tokenizer provides the tokens for the parser.
+	tokenizer *Tokenizer
+	// tok is the most recently read token.
+	tok Token
+	// Self-closing tags like 
are re-interpreted as a two-token sequence: + //
followed by . hasSelfClosingToken is true if we have just read + // the synthetic start tag and the next one due is the matching end tag. + hasSelfClosingToken bool + // doc is the document root element. + doc *Node + // The stack of open elements (section 11.2.3.2) and active formatting + // elements (section 11.2.3.3). + oe, afe nodeStack + // Element pointers (section 11.2.3.4). + head, form *Node + // Other parsing state flags (section 11.2.3.5). + scripting, framesetOK bool +} + +func (p *parser) top() *Node { + if n := p.oe.top(); n != nil { + return n + } + return p.doc +} + +// stopTags for use in popUntil. These come from section 11.2.3.2. +var ( + defaultScopeStopTags = []string{"applet", "caption", "html", "table", "td", "th", "marquee", "object"} + listItemScopeStopTags = []string{"applet", "caption", "html", "table", "td", "th", "marquee", "object", "ol", "ul"} + buttonScopeStopTags = []string{"applet", "caption", "html", "table", "td", "th", "marquee", "object", "button"} + tableScopeStopTags = []string{"html", "table"} +) + +// popUntil pops the stack of open elements at the highest element whose tag +// is in matchTags, provided there is no higher element in stopTags. It returns +// whether or not there was such an element. If there was not, popUntil leaves +// the stack unchanged. +// +// For example, if the stack was: +// ["html", "body", "font", "table", "b", "i", "u"] +// then popUntil([]string{"html, "table"}, "font") would return false, but +// popUntil([]string{"html, "table"}, "i") would return true and the resultant +// stack would be: +// ["html", "body", "font", "table", "b"] +// +// If an element's tag is in both stopTags and matchTags, then the stack will +// be popped and the function returns true (provided, of course, there was no +// higher element in the stack that was also in stopTags). For example, +// popUntil([]string{"html, "table"}, "table") would return true and leave: +// ["html", "body", "font"] +func (p *parser) popUntil(stopTags []string, matchTags ...string) bool { + for i := len(p.oe) - 1; i >= 0; i-- { + tag := p.oe[i].Data + for _, t := range matchTags { + if t == tag { + p.oe = p.oe[:i] + return true + } + } + for _, t := range stopTags { + if t == tag { + return false + } + } + } + return false +} + +// addChild adds a child node n to the top element, and pushes n onto the stack +// of open elements if it is an element node. +func (p *parser) addChild(n *Node) { + p.top().Add(n) + if n.Type == ElementNode { + p.oe = append(p.oe, n) + } +} + +// addText adds text to the preceding node if it is a text node, or else it +// calls addChild with a new text node. +func (p *parser) addText(text string) { + // TODO: distinguish whitespace text from others. + t := p.top() + if i := len(t.Child); i > 0 && t.Child[i-1].Type == TextNode { + t.Child[i-1].Data += text + return + } + p.addChild(&Node{ + Type: TextNode, + Data: text, + }) +} + +// addElement calls addChild with an element node. +func (p *parser) addElement(tag string, attr []Attribute) { + p.addChild(&Node{ + Type: ElementNode, + Data: tag, + Attr: attr, + }) +} + +// Section 11.2.3.3. +func (p *parser) addFormattingElement(tag string, attr []Attribute) { + p.addElement(tag, attr) + p.afe = append(p.afe, p.top()) + // TODO. +} + +// Section 11.2.3.3. +func (p *parser) clearActiveFormattingElements() { + for { + n := p.afe.pop() + if len(p.afe) == 0 || n.Type == scopeMarkerNode { + return + } + } +} + +// Section 11.2.3.3. +func (p *parser) reconstructActiveFormattingElements() { + n := p.afe.top() + if n == nil { + return + } + if n.Type == scopeMarkerNode || p.oe.index(n) != -1 { + return + } + i := len(p.afe) - 1 + for n.Type != scopeMarkerNode && p.oe.index(n) == -1 { + if i == 0 { + i = -1 + break + } + i-- + n = p.afe[i] + } + for { + i++ + n = p.afe[i] + p.addChild(n.clone()) + p.afe[i] = n + if i == len(p.afe)-1 { + break + } + } +} + +// read reads the next token. This is usually from the tokenizer, but it may +// be the synthesized end tag implied by a self-closing tag. +func (p *parser) read() os.Error { + if p.hasSelfClosingToken { + p.hasSelfClosingToken = false + p.tok.Type = EndTagToken + p.tok.Attr = nil + return nil + } + p.tokenizer.Next() + p.tok = p.tokenizer.Token() + switch p.tok.Type { + case ErrorToken: + return p.tokenizer.Error() + case SelfClosingTagToken: + p.hasSelfClosingToken = true + p.tok.Type = StartTagToken + } + return nil +} + +// Section 11.2.4. +func (p *parser) acknowledgeSelfClosingTag() { + p.hasSelfClosingToken = false +} + +// An insertion mode (section 11.2.3.1) is the state transition function from +// a particular state in the HTML5 parser's state machine. It updates the +// parser's fields depending on parser.token (where ErrorToken means EOF). In +// addition to returning the next insertionMode state, it also returns whether +// the token was consumed. +type insertionMode func(*parser) (insertionMode, bool) + +// useTheRulesFor runs the delegate insertionMode over p, returning the actual +// insertionMode unless the delegate caused a state transition. +// Section 11.2.3.1, "using the rules for". +func useTheRulesFor(p *parser, actual, delegate insertionMode) (insertionMode, bool) { + im, consumed := delegate(p) + if im != delegate { + return im, consumed + } + return actual, consumed +} + +// Section 11.2.5.4.1. +func initialIM(p *parser) (insertionMode, bool) { + if p.tok.Type == DoctypeToken { + p.addChild(&Node{ + Type: DoctypeNode, + Data: p.tok.Data, + }) + return beforeHTMLIM, true + } + // TODO: set "quirks mode"? It's defined in the DOM spec instead of HTML5 proper, + // and so switching on "quirks mode" might belong in a different package. + return beforeHTMLIM, false +} + +// Section 11.2.5.4.2. +func beforeHTMLIM(p *parser) (insertionMode, bool) { + var ( + add bool + attr []Attribute + implied bool + ) + switch p.tok.Type { + case ErrorToken: + implied = true + case TextToken: + // TODO: distinguish whitespace text from others. + implied = true + case StartTagToken: + if p.tok.Data == "html" { + add = true + attr = p.tok.Attr + } else { + implied = true + } + case EndTagToken: + switch p.tok.Data { + case "head", "body", "html", "br": + implied = true + default: + // Ignore the token. + } + } + if add || implied { + p.addElement("html", attr) + } + return beforeHeadIM, !implied +} + +// Section 11.2.5.4.3. +func beforeHeadIM(p *parser) (insertionMode, bool) { + var ( + add bool + attr []Attribute + implied bool + ) + switch p.tok.Type { + case ErrorToken: + implied = true + case TextToken: + // TODO: distinguish whitespace text from others. + implied = true + case StartTagToken: + switch p.tok.Data { + case "head": + add = true + attr = p.tok.Attr + case "html": + return useTheRulesFor(p, beforeHeadIM, inBodyIM) + default: + implied = true + } + case EndTagToken: + switch p.tok.Data { + case "head", "body", "html", "br": + implied = true + default: + // Ignore the token. + } + } + if add || implied { + p.addElement("head", attr) + } + return inHeadIM, !implied +} + +// Section 11.2.5.4.4. +func inHeadIM(p *parser) (insertionMode, bool) { + var ( + pop bool + implied bool + ) + switch p.tok.Type { + case ErrorToken, TextToken: + implied = true + case StartTagToken: + switch p.tok.Data { + case "meta": + // TODO. + case "script": + // TODO. + default: + implied = true + } + case EndTagToken: + if p.tok.Data == "head" { + pop = true + } + // TODO. + } + if pop || implied { + n := p.oe.pop() + if n.Data != "head" { + panic("html: bad parser state") + } + return afterHeadIM, !implied + } + return inHeadIM, !implied +} + +// Section 11.2.5.4.6. +func afterHeadIM(p *parser) (insertionMode, bool) { + var ( + add bool + attr []Attribute + framesetOK bool + implied bool + ) + switch p.tok.Type { + case ErrorToken, TextToken: + implied = true + framesetOK = true + case StartTagToken: + switch p.tok.Data { + case "html": + // TODO. + case "body": + add = true + attr = p.tok.Attr + framesetOK = false + case "frameset": + // TODO. + case "base", "basefont", "bgsound", "link", "meta", "noframes", "script", "style", "title": + // TODO. + case "head": + // TODO. + default: + implied = true + framesetOK = true + } + case EndTagToken: + // TODO. + } + if add || implied { + p.addElement("body", attr) + p.framesetOK = framesetOK + } + return inBodyIM, !implied +} + +// Section 11.2.5.4.7. +func inBodyIM(p *parser) (insertionMode, bool) { + var endP bool + switch p.tok.Type { + case TextToken: + p.reconstructActiveFormattingElements() + p.addText(p.tok.Data) + p.framesetOK = false + case StartTagToken: + switch p.tok.Data { + case "address", "article", "aside", "blockquote", "center", "details", "dir", "div", "dl", "fieldset", "figcaption", "figure", "footer", "header", "hgroup", "menu", "nav", "ol", "p", "section", "summary", "ul": + // TODO: Do the proper "does the stack of open elements has a p element in button scope" algorithm in section 11.2.3.2. + n := p.top() + if n.Type == ElementNode && n.Data == "p" { + endP = true + } else { + p.addElement(p.tok.Data, p.tok.Attr) + } + case "h1", "h2", "h3", "h4", "h5", "h6": + // TODO: auto-insert

if necessary. + switch n := p.top(); n.Data { + case "h1", "h2", "h3", "h4", "h5", "h6": + p.oe.pop() + } + p.addElement(p.tok.Data, p.tok.Attr) + case "a": + if n := p.afe.forTag("a"); n != nil { + p.inBodyEndTagFormatting("a") + p.oe.remove(n) + p.afe.remove(n) + } + p.reconstructActiveFormattingElements() + p.addFormattingElement(p.tok.Data, p.tok.Attr) + case "b", "big", "code", "em", "font", "i", "s", "small", "strike", "strong", "tt", "u": + p.reconstructActiveFormattingElements() + p.addFormattingElement(p.tok.Data, p.tok.Attr) + case "area", "br", "embed", "img", "input", "keygen", "wbr": + p.reconstructActiveFormattingElements() + p.addElement(p.tok.Data, p.tok.Attr) + p.oe.pop() + p.acknowledgeSelfClosingTag() + p.framesetOK = false + case "table": + // TODO: auto-insert

if necessary, depending on quirks mode. + p.addElement(p.tok.Data, p.tok.Attr) + p.framesetOK = false + return inTableIM, true + case "hr": + // TODO: auto-insert

if necessary. + p.addElement(p.tok.Data, p.tok.Attr) + p.oe.pop() + p.acknowledgeSelfClosingTag() + p.framesetOK = false + default: + // TODO. + p.addElement(p.tok.Data, p.tok.Attr) + } + case EndTagToken: + switch p.tok.Data { + case "body": + // TODO: autoclose the stack of open elements. + return afterBodyIM, true + case "a", "b", "big", "code", "em", "font", "i", "nobr", "s", "small", "strike", "strong", "tt", "u": + p.inBodyEndTagFormatting(p.tok.Data) + default: + // TODO: any other end tag + if p.tok.Data == p.top().Data { + p.oe.pop() + } + } + } + if endP { + // TODO: do the proper algorithm. + n := p.oe.pop() + if n.Type != ElementNode || n.Data != "p" { + panic("unreachable") + } + } + return inBodyIM, !endP +} + +func (p *parser) inBodyEndTagFormatting(tag string) { + // This is the "adoption agency" algorithm, described at + // http://www.whatwg.org/specs/web-apps/current-work/multipage/tokenization.html#adoptionAgency + + // TODO: this is a fairly literal line-by-line translation of that algorithm. + // Once the code successfully parses the comprehensive test suite, we should + // refactor this code to be more idiomatic. + + // Steps 1-3. The outer loop. + for i := 0; i < 8; i++ { + // Step 4. Find the formatting element. + var formattingElement *Node + for j := len(p.afe) - 1; j >= 0; j-- { + if p.afe[j].Type == scopeMarkerNode { + break + } + if p.afe[j].Data == tag { + formattingElement = p.afe[j] + break + } + } + if formattingElement == nil { + return + } + feIndex := p.oe.index(formattingElement) + if feIndex == -1 { + p.afe.remove(formattingElement) + return + } + + // Steps 5-6. Find the furthest block. + var furthestBlock *Node + for _, e := range p.oe[feIndex:] { + if isSpecialElement[e.Data] { + furthestBlock = e + break + } + } + if furthestBlock == nil { + e := p.oe.pop() + for e != formattingElement { + e = p.oe.pop() + } + p.afe.remove(e) + return + } + + // Steps 7-8. Find the common ancestor and bookmark node. + commonAncestor := p.oe[feIndex-1] + bookmark := p.afe.index(formattingElement) + + // Step 9. The inner loop. Find the lastNode to reparent. + lastNode := furthestBlock + node := furthestBlock + x := p.oe.index(node) + // Steps 9.1-9.3. + for j := 0; j < 3; j++ { + // Step 9.4. + x-- + node = p.oe[x] + // Step 9.5. + if p.afe.index(node) == -1 { + p.oe.remove(node) + continue + } + // Step 9.6. + if node == formattingElement { + break + } + // Step 9.7. + clone := node.clone() + p.afe[p.afe.index(node)] = clone + p.oe[p.oe.index(node)] = clone + node = clone + // Step 9.8. + if lastNode == furthestBlock { + bookmark = p.afe.index(node) + 1 + } + // Step 9.9. + if lastNode.Parent != nil { + lastNode.Parent.Remove(lastNode) + } + node.Add(lastNode) + // Step 9.10. + lastNode = node + } + + // Step 10. Reparent lastNode to the common ancestor, + // or for misnested table nodes, to the foster parent. + if lastNode.Parent != nil { + lastNode.Parent.Remove(lastNode) + } + switch commonAncestor.Data { + case "table", "tbody", "tfoot", "thead", "tr": + // TODO: fix up misnested table nodes; find the foster parent. + fallthrough + default: + commonAncestor.Add(lastNode) + } + + // Steps 11-13. Reparent nodes from the furthest block's children + // to a clone of the formatting element. + clone := formattingElement.clone() + reparentChildren(clone, furthestBlock) + furthestBlock.Add(clone) + + // Step 14. Fix up the list of active formatting elements. + p.afe.remove(formattingElement) + p.afe.insert(bookmark, clone) + + // Step 15. Fix up the stack of open elements. + p.oe.remove(formattingElement) + p.oe.insert(p.oe.index(furthestBlock)+1, clone) + } +} + +// Section 11.2.5.4.9. +func inTableIM(p *parser) (insertionMode, bool) { + var ( + add bool + data string + attr []Attribute + consumed bool + ) + switch p.tok.Type { + case ErrorToken: + // Stop parsing. + return nil, true + case TextToken: + // TODO. + case StartTagToken: + switch p.tok.Data { + case "tbody", "tfoot", "thead": + add = true + data = p.tok.Data + attr = p.tok.Attr + consumed = true + case "td", "th", "tr": + add = true + data = "tbody" + default: + // TODO. + } + case EndTagToken: + switch p.tok.Data { + case "table": + if p.popUntil(tableScopeStopTags, "table") { + // TODO: "reset the insertion mode appropriately" as per 11.2.3.1. + return inBodyIM, false + } + // Ignore the token. + return inTableIM, true + case "body", "caption", "col", "colgroup", "html", "tbody", "td", "tfoot", "th", "thead", "tr": + // Ignore the token. + return inTableIM, true + } + } + if add { + // TODO: clear the stack back to a table context. + p.addElement(data, attr) + return inTableBodyIM, consumed + } + // TODO: return useTheRulesFor(inTableIM, inBodyIM, p) unless etc. etc. foster parenting. + return inTableIM, true +} + +// Section 11.2.5.4.13. +func inTableBodyIM(p *parser) (insertionMode, bool) { + var ( + add bool + data string + attr []Attribute + consumed bool + ) + switch p.tok.Type { + case ErrorToken: + // TODO. + case TextToken: + // TODO. + case StartTagToken: + switch p.tok.Data { + case "tr": + add = true + data = p.tok.Data + attr = p.tok.Attr + consumed = true + case "td", "th": + add = true + data = "tr" + consumed = false + default: + // TODO. + } + case EndTagToken: + switch p.tok.Data { + case "table": + if p.popUntil(tableScopeStopTags, "tbody", "thead", "tfoot") { + return inTableIM, false + } + // Ignore the token. + return inTableBodyIM, true + case "body", "caption", "col", "colgroup", "html", "td", "th", "tr": + // Ignore the token. + return inTableBodyIM, true + } + } + if add { + // TODO: clear the stack back to a table body context. + p.addElement(data, attr) + return inRowIM, consumed + } + return useTheRulesFor(p, inTableBodyIM, inTableIM) +} + +// Section 11.2.5.4.14. +func inRowIM(p *parser) (insertionMode, bool) { + switch p.tok.Type { + case ErrorToken: + // TODO. + case TextToken: + // TODO. + case StartTagToken: + switch p.tok.Data { + case "td", "th": + // TODO: clear the stack back to a table row context. + p.addElement(p.tok.Data, p.tok.Attr) + p.afe = append(p.afe, &scopeMarker) + return inCellIM, true + default: + // TODO. + } + case EndTagToken: + switch p.tok.Data { + case "tr": + // TODO. + case "table": + if p.popUntil(tableScopeStopTags, "tr") { + return inTableBodyIM, false + } + // Ignore the token. + return inRowIM, true + case "tbody", "tfoot", "thead": + // TODO. + case "body", "caption", "col", "colgroup", "html", "td", "th": + // Ignore the token. + return inRowIM, true + default: + // TODO. + } + } + return useTheRulesFor(p, inRowIM, inTableIM) +} + +// Section 11.2.5.4.15. +func inCellIM(p *parser) (insertionMode, bool) { + var ( + closeTheCellAndReprocess bool + ) + switch p.tok.Type { + case StartTagToken: + switch p.tok.Data { + case "caption", "col", "colgroup", "tbody", "td", "tfoot", "th", "thead", "tr": + // TODO: check for "td" or "th" in table scope. + closeTheCellAndReprocess = true + } + case EndTagToken: + switch p.tok.Data { + case "td", "th": + // TODO. + case "body", "caption", "col", "colgroup", "html": + // TODO. + case "table", "tbody", "tfoot", "thead", "tr": + // TODO: check for matching element in table scope. + closeTheCellAndReprocess = true + } + } + if closeTheCellAndReprocess { + if p.popUntil(tableScopeStopTags, "td") || p.popUntil(tableScopeStopTags, "th") { + p.clearActiveFormattingElements() + return inRowIM, false + } + } + return useTheRulesFor(p, inCellIM, inBodyIM) +} + +// Section 11.2.5.4.18. +func afterBodyIM(p *parser) (insertionMode, bool) { + switch p.tok.Type { + case ErrorToken: + // TODO. + case TextToken: + // TODO. + case StartTagToken: + // TODO. + case EndTagToken: + switch p.tok.Data { + case "html": + // TODO: autoclose the stack of open elements. + return afterAfterBodyIM, true + default: + // TODO. + } + } + return afterBodyIM, true +} + +// Section 11.2.5.4.21. +func afterAfterBodyIM(p *parser) (insertionMode, bool) { + switch p.tok.Type { + case ErrorToken: + // Stop parsing. + return nil, true + case TextToken: + // TODO. + case StartTagToken: + if p.tok.Data == "html" { + return useTheRulesFor(p, afterAfterBodyIM, inBodyIM) + } + } + return inBodyIM, false +} + +// Parse returns the parse tree for the HTML from the given Reader. +// The input is assumed to be UTF-8 encoded. +func Parse(r io.Reader) (*Node, os.Error) { + p := &parser{ + tokenizer: NewTokenizer(r), + doc: &Node{ + Type: DocumentNode, + }, + scripting: true, + framesetOK: true, + } + // Iterate until EOF. Any other error will cause an early return. + im, consumed := initialIM, true + for { + if consumed { + if err := p.read(); err != nil { + if err == os.EOF { + break + } + return nil, err + } + } + im, consumed = im(p) + } + // Loop until the final token (the ErrorToken signifying EOF) is consumed. + for { + if im, consumed = im(p); consumed { + break + } + } + return p.doc, nil +} diff --git a/src/pkg/html/parse_test.go b/src/pkg/html/parse_test.go new file mode 100644 index 000000000..7d918d250 --- /dev/null +++ b/src/pkg/html/parse_test.go @@ -0,0 +1,156 @@ +// Copyright 2010 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package html + +import ( + "bufio" + "bytes" + "fmt" + "io" + "io/ioutil" + "os" + "strings" + "testing" +) + +func pipeErr(err os.Error) io.Reader { + pr, pw := io.Pipe() + pw.CloseWithError(err) + return pr +} + +func readDat(filename string, c chan io.Reader) { + f, err := os.Open("testdata/webkit/" + filename) + if err != nil { + c <- pipeErr(err) + return + } + defer f.Close() + + // Loop through the lines of the file. Each line beginning with "#" denotes + // a new section, which is returned as a separate io.Reader. + r := bufio.NewReader(f) + var pw *io.PipeWriter + for { + line, err := r.ReadSlice('\n') + if err != nil { + if pw != nil { + pw.CloseWithError(err) + pw = nil + } else { + c <- pipeErr(err) + } + return + } + if len(line) == 0 { + continue + } + if line[0] == '#' { + if pw != nil { + pw.Close() + } + var pr *io.PipeReader + pr, pw = io.Pipe() + c <- pr + continue + } + if line[0] != '|' { + // Strip the trailing '\n'. + line = line[:len(line)-1] + } + if pw != nil { + if _, err := pw.Write(line); err != nil { + pw.CloseWithError(err) + pw = nil + } + } + } +} + +func dumpLevel(w io.Writer, n *Node, level int) os.Error { + io.WriteString(w, "| ") + for i := 0; i < level; i++ { + io.WriteString(w, " ") + } + switch n.Type { + case ErrorNode: + return os.NewError("unexpected ErrorNode") + case DocumentNode: + return os.NewError("unexpected DocumentNode") + case ElementNode: + fmt.Fprintf(w, "<%s>", EscapeString(n.Data)) + case TextNode: + fmt.Fprintf(w, "%q", EscapeString(n.Data)) + case CommentNode: + return os.NewError("COMMENT") + case DoctypeNode: + fmt.Fprintf(w, "", EscapeString(n.Data)) + case scopeMarkerNode: + return os.NewError("unexpected scopeMarkerNode") + default: + return os.NewError("unknown node type") + } + io.WriteString(w, "\n") + for _, c := range n.Child { + if err := dumpLevel(w, c, level+1); err != nil { + return err + } + } + return nil +} + +func dump(n *Node) (string, os.Error) { + if n == nil || len(n.Child) == 0 { + return "", nil + } + b := bytes.NewBuffer(nil) + for _, child := range n.Child { + if err := dumpLevel(b, child, 0); err != nil { + return "", err + } + } + return b.String(), nil +} + +func TestParser(t *testing.T) { + // TODO(nigeltao): Process all the .dat files, not just the first one. + filenames := []string{ + "tests1.dat", + } + for _, filename := range filenames { + rc := make(chan io.Reader) + go readDat(filename, rc) + // TODO(nigeltao): Process all test cases, not just a subset. + for i := 0; i < 25; i++ { + // Parse the #data section. + b, err := ioutil.ReadAll(<-rc) + if err != nil { + t.Fatal(err) + } + text := string(b) + doc, err := Parse(strings.NewReader(text)) + if err != nil { + t.Fatal(err) + } + actual, err := dump(doc) + if err != nil { + t.Fatal(err) + } + // Skip the #error section. + if _, err := io.Copy(ioutil.Discard, <-rc); err != nil { + t.Fatal(err) + } + // Compare the parsed tree to the #document section. + b, err = ioutil.ReadAll(<-rc) + if err != nil { + t.Fatal(err) + } + expected := string(b) + if actual != expected { + t.Errorf("%s test #%d %q, actual vs expected:\n----\n%s----\n%s----", filename, i, text, actual, expected) + } + } + } +} diff --git a/src/pkg/html/testdata/webkit/README b/src/pkg/html/testdata/webkit/README new file mode 100644 index 000000000..9b4c2d8be --- /dev/null +++ b/src/pkg/html/testdata/webkit/README @@ -0,0 +1,28 @@ +The *.dat files in this directory are copied from The WebKit Open Source +Project, specifically $WEBKITROOT/LayoutTests/html5lib/resources. +WebKit is licensed under a BSD style license. +http://webkit.org/coding/bsd-license.html says: + +Copyright (C) 2009 Apple Inc. All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + +1. Redistributions of source code must retain the above copyright notice, +this list of conditions and the following disclaimer. + +2. Redistributions in binary form must reproduce the above copyright notice, +this list of conditions and the following disclaimer in the documentation +and/or other materials provided with the distribution. + +THIS SOFTWARE IS PROVIDED BY APPLE INC. AND ITS CONTRIBUTORS "AS IS" AND ANY +EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +DISCLAIMED. IN NO EVENT SHALL APPLE INC. OR ITS CONTRIBUTORS BE LIABLE FOR ANY +DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON +ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + diff --git a/src/pkg/html/testdata/webkit/adoption01.dat b/src/pkg/html/testdata/webkit/adoption01.dat new file mode 100644 index 000000000..787e1b01e --- /dev/null +++ b/src/pkg/html/testdata/webkit/adoption01.dat @@ -0,0 +1,194 @@ +#data +

+#errors +#document +| +| +| +| +|

+| + +#data +1

23

+#errors +#document +| +| +| +| +| "1" +|

+| +| "2" +| "3" + +#data +1 +#errors +#document +| +| +| +| +| "1" +| +#errors +Line: 1 Col: 7 Unexpected start tag (table). Expected DOCTYPE. +Line: 1 Col: 20 Unexpected end tag (strong) in table context caused voodoo mode. +Line: 1 Col: 20 End tag (strong) violates step 1, paragraph 1 of the adoption agency algorithm. +Line: 1 Col: 24 Unexpected end tag (b) in table context caused voodoo mode. +Line: 1 Col: 24 End tag (b) violates step 1, paragraph 1 of the adoption agency algorithm. +Line: 1 Col: 29 Unexpected end tag (em) in table context caused voodoo mode. +Line: 1 Col: 29 End tag (em) violates step 1, paragraph 1 of the adoption agency algorithm. +Line: 1 Col: 33 Unexpected end tag (i) in table context caused voodoo mode. +Line: 1 Col: 33 End tag (i) violates step 1, paragraph 1 of the adoption agency algorithm. +Line: 1 Col: 37 Unexpected end tag (u) in table context caused voodoo mode. +Line: 1 Col: 37 End tag (u) violates step 1, paragraph 1 of the adoption agency algorithm. +Line: 1 Col: 46 Unexpected end tag (strike) in table context caused voodoo mode. +Line: 1 Col: 46 End tag (strike) violates step 1, paragraph 1 of the adoption agency algorithm. +Line: 1 Col: 50 Unexpected end tag (s) in table context caused voodoo mode. +Line: 1 Col: 50 End tag (s) violates step 1, paragraph 1 of the adoption agency algorithm. +Line: 1 Col: 58 Unexpected end tag (blink) in table context caused voodoo mode. +Line: 1 Col: 58 Unexpected end tag (blink). Ignored. +Line: 1 Col: 63 Unexpected end tag (tt) in table context caused voodoo mode. +Line: 1 Col: 63 End tag (tt) violates step 1, paragraph 1 of the adoption agency algorithm. +Line: 1 Col: 69 Unexpected end tag (pre) in table context caused voodoo mode. +Line: 1 Col: 69 End tag (pre) seen too early. Expected other end tag. +Line: 1 Col: 75 Unexpected end tag (big) in table context caused voodoo mode. +Line: 1 Col: 75 End tag (big) violates step 1, paragraph 1 of the adoption agency algorithm. +Line: 1 Col: 83 Unexpected end tag (small) in table context caused voodoo mode. +Line: 1 Col: 83 End tag (small) violates step 1, paragraph 1 of the adoption agency algorithm. +Line: 1 Col: 90 Unexpected end tag (font) in table context caused voodoo mode. +Line: 1 Col: 90 End tag (font) violates step 1, paragraph 1 of the adoption agency algorithm. +Line: 1 Col: 99 Unexpected end tag (select) in table context caused voodoo mode. +Line: 1 Col: 99 Unexpected end tag (select). Ignored. +Line: 1 Col: 104 Unexpected end tag (h1) in table context caused voodoo mode. +Line: 1 Col: 104 End tag (h1) seen too early. Expected other end tag. +Line: 1 Col: 109 Unexpected end tag (h2) in table context caused voodoo mode. +Line: 1 Col: 109 End tag (h2) seen too early. Expected other end tag. +Line: 1 Col: 114 Unexpected end tag (h3) in table context caused voodoo mode. +Line: 1 Col: 114 End tag (h3) seen too early. Expected other end tag. +Line: 1 Col: 119 Unexpected end tag (h4) in table context caused voodoo mode. +Line: 1 Col: 119 End tag (h4) seen too early. Expected other end tag. +Line: 1 Col: 124 Unexpected end tag (h5) in table context caused voodoo mode. +Line: 1 Col: 124 End tag (h5) seen too early. Expected other end tag. +Line: 1 Col: 129 Unexpected end tag (h6) in table context caused voodoo mode. +Line: 1 Col: 129 End tag (h6) seen too early. Expected other end tag. +Line: 1 Col: 136 Unexpected end tag (body) in the table row phase. Ignored. +Line: 1 Col: 141 Unexpected end tag (br) in table context caused voodoo mode. +Line: 1 Col: 141 Unexpected end tag (br). Treated as br element. +Line: 1 Col: 145 Unexpected end tag (a) in table context caused voodoo mode. +Line: 1 Col: 145 End tag (a) violates step 1, paragraph 1 of the adoption agency algorithm. +Line: 1 Col: 151 Unexpected end tag (img) in table context caused voodoo mode. +Line: 1 Col: 151 This element (img) has no end tag. +Line: 1 Col: 159 Unexpected end tag (title) in table context caused voodoo mode. +Line: 1 Col: 159 Unexpected end tag (title). Ignored. +Line: 1 Col: 166 Unexpected end tag (span) in table context caused voodoo mode. +Line: 1 Col: 166 Unexpected end tag (span). Ignored. +Line: 1 Col: 174 Unexpected end tag (style) in table context caused voodoo mode. +Line: 1 Col: 174 Unexpected end tag (style). Ignored. +Line: 1 Col: 183 Unexpected end tag (script) in table context caused voodoo mode. +Line: 1 Col: 183 Unexpected end tag (script). Ignored. +Line: 1 Col: 196 Unexpected end tag (th). Ignored. +Line: 1 Col: 201 Unexpected end tag (td). Ignored. +Line: 1 Col: 206 Unexpected end tag (tr). Ignored. +Line: 1 Col: 214 This element (frame) has no end tag. +Line: 1 Col: 221 This element (area) has no end tag. +Line: 1 Col: 228 Unexpected end tag (link). Ignored. +Line: 1 Col: 236 This element (param) has no end tag. +Line: 1 Col: 241 This element (hr) has no end tag. +Line: 1 Col: 249 This element (input) has no end tag. +Line: 1 Col: 255 Unexpected end tag (col). Ignored. +Line: 1 Col: 262 Unexpected end tag (base). Ignored. +Line: 1 Col: 269 Unexpected end tag (meta). Ignored. +Line: 1 Col: 280 This element (basefont) has no end tag. +Line: 1 Col: 290 This element (bgsound) has no end tag. +Line: 1 Col: 298 This element (embed) has no end tag. +Line: 1 Col: 307 This element (spacer) has no end tag. +Line: 1 Col: 311 Unexpected end tag (p). Ignored. +Line: 1 Col: 316 End tag (dd) seen too early. Expected other end tag. +Line: 1 Col: 321 End tag (dt) seen too early. Expected other end tag. +Line: 1 Col: 331 Unexpected end tag (caption). Ignored. +Line: 1 Col: 342 Unexpected end tag (colgroup). Ignored. +Line: 1 Col: 350 Unexpected end tag (tbody). Ignored. +Line: 1 Col: 358 Unexpected end tag (tfoot). Ignored. +Line: 1 Col: 366 Unexpected end tag (thead). Ignored. +Line: 1 Col: 376 End tag (address) seen too early. Expected other end tag. +Line: 1 Col: 389 End tag (blockquote) seen too early. Expected other end tag. +Line: 1 Col: 398 End tag (center) seen too early. Expected other end tag. +Line: 1 Col: 404 Unexpected end tag (dir). Ignored. +Line: 1 Col: 410 End tag (div) seen too early. Expected other end tag. +Line: 1 Col: 415 End tag (dl) seen too early. Expected other end tag. +Line: 1 Col: 426 End tag (fieldset) seen too early. Expected other end tag. +Line: 1 Col: 436 End tag (listing) seen too early. Expected other end tag. +Line: 1 Col: 443 End tag (menu) seen too early. Expected other end tag. +Line: 1 Col: 448 End tag (ol) seen too early. Expected other end tag. +Line: 1 Col: 453 End tag (ul) seen too early. Expected other end tag. +Line: 1 Col: 458 End tag (li) seen too early. Expected other end tag. +Line: 1 Col: 465 End tag (nobr) violates step 1, paragraph 1 of the adoption agency algorithm. +Line: 1 Col: 471 This element (wbr) has no end tag. +Line: 1 Col: 487 End tag (button) seen too early. Expected other end tag. +Line: 1 Col: 497 End tag (marquee) seen too early. Expected other end tag. +Line: 1 Col: 506 End tag (object) seen too early. Expected other end tag. +Line: 1 Col: 524 Unexpected end tag (html). Ignored. +Line: 1 Col: 524 Unexpected end tag (frameset). Ignored. +Line: 1 Col: 531 Unexpected end tag (head). Ignored. +Line: 1 Col: 540 Unexpected end tag (iframe). Ignored. +Line: 1 Col: 548 This element (image) has no end tag. +Line: 1 Col: 558 This element (isindex) has no end tag. +Line: 1 Col: 568 Unexpected end tag (noembed). Ignored. +Line: 1 Col: 579 Unexpected end tag (noframes). Ignored. +Line: 1 Col: 590 Unexpected end tag (noscript). Ignored. +Line: 1 Col: 601 Unexpected end tag (optgroup). Ignored. +Line: 1 Col: 610 Unexpected end tag (option). Ignored. +Line: 1 Col: 622 Unexpected end tag (plaintext). Ignored. +Line: 1 Col: 633 Unexpected end tag (textarea). Ignored. +#document +| +| +| +|
+| +| +| +|

+ +#data + +#errors +Line: 1 Col: 10 Unexpected start tag (frameset). Expected DOCTYPE. +Line: 1 Col: 10 Expected closing tag. Unexpected end of file. +#document +| +| +| diff --git a/src/pkg/html/testdata/webkit/tests10.dat b/src/pkg/html/testdata/webkit/tests10.dat new file mode 100644 index 000000000..4f8df86f2 --- /dev/null +++ b/src/pkg/html/testdata/webkit/tests10.dat @@ -0,0 +1,799 @@ +#data + +#errors +#document +| +| +| +| +| + +#data +a +#errors +29: Bogus comment +#document +| +| +| +| +| +| + +#data + +#errors +#document +| +| +| +| +| + +#data + +#errors +35: Stray “svg” start tag. +42: Stray end tag “svg” +#document +| +| +| +| +| +#errors +43: Stray “svg” start tag. +50: Stray end tag “svg” +#document +| +| +| +| +|

+#errors +34: Start tag “svg” seen in “table”. +41: Stray end tag “svg”. +#document +| +| +| +| +| +| + +#data +
foo
+#errors +34: Start tag “svg” seen in “table”. +46: Stray end tag “g”. +53: Stray end tag “svg”. +#document +| +| +| +| +| +| +| "foo" +| + +#data +
foobar
+#errors +34: Start tag “svg” seen in “table”. +46: Stray end tag “g”. +58: Stray end tag “g”. +65: Stray end tag “svg”. +#document +| +| +| +| +| +| +| "foo" +| +| "bar" +| + +#data +
foobar
+#errors +41: Start tag “svg” seen in “table”. +53: Stray end tag “g”. +65: Stray end tag “g”. +72: Stray end tag “svg”. +#document +| +| +| +| +| +| +| "foo" +| +| "bar" +| +| + +#data +
foobar
+#errors +45: Start tag “svg” seen in “table”. +57: Stray end tag “g”. +69: Stray end tag “g”. +76: Stray end tag “svg”. +#document +| +| +| +| +| +| +| "foo" +| +| "bar" +| +| +| + +#data +
foobar
+#errors +#document +| +| +| +| +| +| +| +|
+| +| +| "foo" +| +| "bar" + +#data +
foobar

baz

+#errors +#document +| +| +| +| +| +| +| +|
+| +| +| "foo" +| +| "bar" +|

+| "baz" + +#data +
foobar

baz

+#errors +#document +| +| +| +| +| +|
+| +| +| "foo" +| +| "bar" +|

+| "baz" + +#data +
foobar

baz

quux +#errors +70: HTML start tag “p” in a foreign namespace context. +81: “table” closed but “caption” was still open. +#document +| +| +| +| +| +|
+| +| +| "foo" +| +| "bar" +|

+| "baz" +|

+| "quux" + +#data +
foobarbaz

quux +#errors +78: “table” closed but “caption” was still open. +78: Unclosed elements on stack. +#document +| +| +| +| +| +|
+| +| +| "foo" +| +| "bar" +| "baz" +|

+| "quux" + +#data +foobar

baz

quux +#errors +44: Start tag “svg” seen in “table”. +56: Stray end tag “g”. +68: Stray end tag “g”. +71: HTML start tag “p” in a foreign namespace context. +71: Start tag “p” seen in “table”. +#document +| +| +| +| +| +| +| "foo" +| +| "bar" +|

+| "baz" +| +| +|

+| "quux" + +#data +

quux +#errors +50: Stray “svg” start tag. +54: Stray “g” start tag. +62: Stray end tag “g” +66: Stray “g” start tag. +74: Stray end tag “g” +77: Stray “p” start tag. +88: “table” end tag with “select” open. +#document +| +| +| +| +| +| +| +|
+|

quux +#errors +36: Start tag “select” seen in “table”. +42: Stray “svg” start tag. +46: Stray “g” start tag. +54: Stray end tag “g” +58: Stray “g” start tag. +66: Stray end tag “g” +69: Stray “p” start tag. +80: “table” end tag with “select” open. +#document +| +| +| +| +| +|

+| "quux" + +#data +foobar

baz +#errors +41: Stray “svg” start tag. +68: HTML start tag “p” in a foreign namespace context. +#document +| +| +| +| +| +| +| "foo" +| +| "bar" +|

+| "baz" + +#data +foobar

baz +#errors +34: Stray “svg” start tag. +61: HTML start tag “p” in a foreign namespace context. +#document +| +| +| +| +| +| +| "foo" +| +| "bar" +|

+| "baz" + +#data +

+#errors +31: Stray “svg” start tag. +35: Stray “g” start tag. +40: Stray end tag “g” +44: Stray “g” start tag. +49: Stray end tag “g” +52: Stray “p” start tag. +58: Stray “span” start tag. +58: End of file seen and there were open elements. +#document +| +| +| +| + +#data +

+#errors +42: Stray “svg” start tag. +46: Stray “g” start tag. +51: Stray end tag “g” +55: Stray “g” start tag. +60: Stray end tag “g” +63: Stray “p” start tag. +69: Stray “span” start tag. +#document +| +| +| +| + +#data + +#errors +#document +| +| +| +| +| xlink:href="foo" +| +| xlink href="foo" + +#data + +#errors +#document +| +| +| +| +| xlink:href="foo" +| xml:lang="en" +| +| +| xlink href="foo" +| xml lang="en" + +#data + +#errors +#document +| +| +| +| +| xlink:href="foo" +| xml:lang="en" +| +| +| xlink href="foo" +| xml lang="en" + +#data +bar +#errors +#document +| +| +| +| +| xlink:href="foo" +| xml:lang="en" +| +| +| xlink href="foo" +| xml lang="en" +| "bar" + +#data + +#errors +#document +| +| +| +| + +#data +

a +#errors +#document +| +| +| +|
+| +| "a" + +#data +
a +#errors +#document +| +| +| +|
+| +| +| "a" + +#data +
+#errors +#document +| +| +| +|
+| +| +| + +#data +
a +#errors +#document +| +| +| +|
+| +| +| +| +| "a" + +#data +

a +#errors +#document +| +| +| +|

+| +| +| +|

+| "a" + +#data +
    a +#errors +40: HTML start tag “ul” in a foreign namespace context. +41: End of file in a foreign namespace context. +#document +| +| +| +| +| +| +|
    +| +|
      +| "a" + +#data +
        a +#errors +35: HTML start tag “ul” in a foreign namespace context. +36: End of file in a foreign namespace context. +#document +| +| +| +| +| +| +| +|
          +| "a" + +#data +

          +#errors +#document +| +| +| +| +|

          +| +| +|

          + +#data +

          +#errors +#document +| +| +| +| +|

          +| +| +|

          + +#data +

          +#errors +#document +| +| +| +|

          +| +| +| +|

          +|

          + +#data +
          +#errors +#document +| +| +| +| +| +|
          +| +|
          +| +| + +#data +
          +#errors +#document +| +| +| +| +| +| +| +|
          +|
          +| + +#data + +#errors +#document +| +| +| +| +| +| + +#data +

+#errors +#document +| +| +| +| +|
+| +| + +#data + +#errors +#document +| +| +| +| +| +| + +#data + +#errors +#document +| +| +| +| +| +| + +#data + +#errors +#document +| +| +| +| +| +| + +#data + +#errors +#document +| +| +| +| +| +| + +#data + +#errors +#document +| +| +| +| +| +| + +#data + +#errors +#document +| +| +| +| +| +| + +#data + +#errors +#document +| +| +| +| +| +| + +#data + +#errors +#document +| +| +| +| +| +| + +#data + +#errors +#document +| +| +| +| +| +| + +#data + +#errors +#document +| +| +| +| +| +| + +#data + +#errors +#document +| +| +| +| +| +| +| + +#data +
+#errors +#document +| +| +| +| +| +| +| +|
+| +| +| +| +| + +#data + +#errors +#document +| +| +| +| +| +| +| +| +| +| +| +| +| +| diff --git a/src/pkg/html/testdata/webkit/tests11.dat b/src/pkg/html/testdata/webkit/tests11.dat new file mode 100644 index 000000000..638cde479 --- /dev/null +++ b/src/pkg/html/testdata/webkit/tests11.dat @@ -0,0 +1,482 @@ +#data + +#errors +#document +| +| +| +| +| +| attributeName="" +| attributeType="" +| baseFrequency="" +| baseProfile="" +| calcMode="" +| clipPathUnits="" +| contentScriptType="" +| contentStyleType="" +| diffuseConstant="" +| edgeMode="" +| externalResourcesRequired="" +| filterRes="" +| filterUnits="" +| glyphRef="" +| gradientTransform="" +| gradientUnits="" +| kernelMatrix="" +| kernelUnitLength="" +| keyPoints="" +| keySplines="" +| keyTimes="" +| lengthAdjust="" +| limitingConeAngle="" +| markerHeight="" +| markerUnits="" +| markerWidth="" +| maskContentUnits="" +| maskUnits="" +| numOctaves="" +| pathLength="" +| patternContentUnits="" +| patternTransform="" +| patternUnits="" +| pointsAtX="" +| pointsAtY="" +| pointsAtZ="" +| preserveAlpha="" +| preserveAspectRatio="" +| primitiveUnits="" +| refX="" +| refY="" +| repeatCount="" +| repeatDur="" +| requiredExtensions="" +| requiredFeatures="" +| specularConstant="" +| specularExponent="" +| spreadMethod="" +| startOffset="" +| stdDeviation="" +| stitchTiles="" +| surfaceScale="" +| systemLanguage="" +| tableValues="" +| targetX="" +| targetY="" +| textLength="" +| viewBox="" +| viewTarget="" +| xChannelSelector="" +| yChannelSelector="" +| zoomAndPan="" + +#data + +#errors +#document +| +| +| +| +| +| attributeName="" +| attributeType="" +| baseFrequency="" +| baseProfile="" +| calcMode="" +| clipPathUnits="" +| contentScriptType="" +| contentStyleType="" +| diffuseConstant="" +| edgeMode="" +| externalResourcesRequired="" +| filterRes="" +| filterUnits="" +| glyphRef="" +| gradientTransform="" +| gradientUnits="" +| kernelMatrix="" +| kernelUnitLength="" +| keyPoints="" +| keySplines="" +| keyTimes="" +| lengthAdjust="" +| limitingConeAngle="" +| markerHeight="" +| markerUnits="" +| markerWidth="" +| maskContentUnits="" +| maskUnits="" +| numOctaves="" +| pathLength="" +| patternContentUnits="" +| patternTransform="" +| patternUnits="" +| pointsAtX="" +| pointsAtY="" +| pointsAtZ="" +| preserveAlpha="" +| preserveAspectRatio="" +| primitiveUnits="" +| refX="" +| refY="" +| repeatCount="" +| repeatDur="" +| requiredExtensions="" +| requiredFeatures="" +| specularConstant="" +| specularExponent="" +| spreadMethod="" +| startOffset="" +| stdDeviation="" +| stitchTiles="" +| surfaceScale="" +| systemLanguage="" +| tableValues="" +| targetX="" +| targetY="" +| textLength="" +| viewBox="" +| viewTarget="" +| xChannelSelector="" +| yChannelSelector="" +| zoomAndPan="" + +#data + +#errors +#document +| +| +| +| +| +| attributeName="" +| attributeType="" +| baseFrequency="" +| baseProfile="" +| calcMode="" +| clipPathUnits="" +| contentScriptType="" +| contentStyleType="" +| diffuseConstant="" +| edgeMode="" +| externalResourcesRequired="" +| filterRes="" +| filterUnits="" +| glyphRef="" +| gradientTransform="" +| gradientUnits="" +| kernelMatrix="" +| kernelUnitLength="" +| keyPoints="" +| keySplines="" +| keyTimes="" +| lengthAdjust="" +| limitingConeAngle="" +| markerHeight="" +| markerUnits="" +| markerWidth="" +| maskContentUnits="" +| maskUnits="" +| numOctaves="" +| pathLength="" +| patternContentUnits="" +| patternTransform="" +| patternUnits="" +| pointsAtX="" +| pointsAtY="" +| pointsAtZ="" +| preserveAlpha="" +| preserveAspectRatio="" +| primitiveUnits="" +| refX="" +| refY="" +| repeatCount="" +| repeatDur="" +| requiredExtensions="" +| requiredFeatures="" +| specularConstant="" +| specularExponent="" +| spreadMethod="" +| startOffset="" +| stdDeviation="" +| stitchTiles="" +| surfaceScale="" +| systemLanguage="" +| tableValues="" +| targetX="" +| targetY="" +| textLength="" +| viewBox="" +| viewTarget="" +| xChannelSelector="" +| yChannelSelector="" +| zoomAndPan="" + +#data + +#errors +#document +| +| +| +| +| +| attributename="" +| attributetype="" +| basefrequency="" +| baseprofile="" +| calcmode="" +| clippathunits="" +| contentscripttype="" +| contentstyletype="" +| diffuseconstant="" +| edgemode="" +| externalresourcesrequired="" +| filterres="" +| filterunits="" +| glyphref="" +| gradienttransform="" +| gradientunits="" +| kernelmatrix="" +| kernelunitlength="" +| keypoints="" +| keysplines="" +| keytimes="" +| lengthadjust="" +| limitingconeangle="" +| markerheight="" +| markerunits="" +| markerwidth="" +| maskcontentunits="" +| maskunits="" +| numoctaves="" +| pathlength="" +| patterncontentunits="" +| patterntransform="" +| patternunits="" +| pointsatx="" +| pointsaty="" +| pointsatz="" +| preservealpha="" +| preserveaspectratio="" +| primitiveunits="" +| refx="" +| refy="" +| repeatcount="" +| repeatdur="" +| requiredextensions="" +| requiredfeatures="" +| specularconstant="" +| specularexponent="" +| spreadmethod="" +| startoffset="" +| stddeviation="" +| stitchtiles="" +| surfacescale="" +| systemlanguage="" +| tablevalues="" +| targetx="" +| targety="" +| textlength="" +| viewbox="" +| viewtarget="" +| xchannelselector="" +| ychannelselector="" +| zoomandpan="" + +#data + +#errors +#document +| +| +| +| +| +| +| +| +| +| +| +| +| +| +| +| +| +| +| +| +| +| +| +| +| +| +| +| +| +| +| +| +| +| +| +| +| +| +| +| +| + +#data + +#errors +#document +| +| +| +| +| +| +| +| +| +| +| +| +| +| +| +| +| +| +| +| +| +| +| +| +| +| +| +| +| +| +| +| +| +| +| +| +| +| +| +| +| + +#data + +#errors +#document +| +| +| +| +| +| +| +| +| +| +| +| +| +| +| +| +| +| +| +| +| +| +| +| +| +| +| +| +| +| +| +| +| +| +| +| +| +| +| +| +| + +#data + +#errors +#document +| +| +| +| +| +| +| +| +| +| +| +| +| +| +| +| +| +| +| +| +| +| +| +| +| +| +| +| +| +| +| +| +| +| +| +| +| +| +| +| +| + +#data + +#errors +#document +| +| +| +| +| +| diff --git a/src/pkg/html/testdata/webkit/tests12.dat b/src/pkg/html/testdata/webkit/tests12.dat new file mode 100644 index 000000000..63107d277 --- /dev/null +++ b/src/pkg/html/testdata/webkit/tests12.dat @@ -0,0 +1,62 @@ +#data +

foobazeggs

spam

quuxbar +#errors +#document +| +| +| +| +|

+| "foo" +| +| +| +| "baz" +| +| +| +| +| "eggs" +| +| +|

+| "spam" +| +| +| +|
+| +| +| "quux" +| "bar" + +#data +foobazeggs

spam
quuxbar +#errors +#document +| +| +| +| +| "foo" +| +| +| +| "baz" +| +| +| +| +| "eggs" +| +| +|

+| "spam" +| +| +| +|
+| +| +| "quux" +| "bar" diff --git a/src/pkg/html/testdata/webkit/tests14.dat b/src/pkg/html/testdata/webkit/tests14.dat new file mode 100644 index 000000000..b8713f885 --- /dev/null +++ b/src/pkg/html/testdata/webkit/tests14.dat @@ -0,0 +1,74 @@ +#data + +#errors +#document +| +| +| +| +| + +#data + +#errors +#document +| +| +| +| +| +| + +#data + +#errors +15: Unexpected start tag html +#document +| +| +| abc:def="gh" +| +| +| + +#data + +#errors +15: Unexpected start tag html +#document +| +| +| xml:lang="bar" +| +| + +#data + +#errors +#document +| +| +| 123="456" +| +| + +#data + +#errors +#document +| +| +| 123="456" +| 789="012" +| +| + +#data + +#errors +#document +| +| +| +| +| 789="012" diff --git a/src/pkg/html/testdata/webkit/tests15.dat b/src/pkg/html/testdata/webkit/tests15.dat new file mode 100644 index 000000000..6ce1c0d16 --- /dev/null +++ b/src/pkg/html/testdata/webkit/tests15.dat @@ -0,0 +1,208 @@ +#data +

X +#errors +Line: 1 Col: 31 Unexpected end tag (p). Ignored. +Line: 1 Col: 36 Expected closing tag. Unexpected end of file. +#document +| +| +| +| +|

+| +| +| +| +| +| +| " " +|

+| "X" + +#data +

+

X +#errors +Line: 1 Col: 3 Unexpected start tag (p). Expected DOCTYPE. +Line: 1 Col: 16 Unexpected end tag (p). Ignored. +Line: 2 Col: 4 Expected closing tag. Unexpected end of file. +#document +| +| +| +|

+| +| +| +| +| +| +| " +" +|

+| "X" + +#data + +#errors +Line: 1 Col: 22 Unexpected end tag (html) after the (implied) root element. +#document +| +| +| +| +| " " + +#data + +#errors +Line: 1 Col: 22 Unexpected end tag (body) after the (implied) root element. +#document +| +| +| +| +| + +#data + +#errors +Line: 1 Col: 6 Unexpected start tag (html). Expected DOCTYPE. +Line: 1 Col: 13 Unexpected end tag (html) after the (implied) root element. +#document +| +| +| +| + +#data +X +#errors +Line: 1 Col: 22 Unexpected end tag (body) after the (implied) root element. +#document +| +| +| +| +| +| "X" + +#data +<!doctype html><table> X<meta></table> +#errors +Line: 1 Col: 24 Unexpected non-space characters in table context caused voodoo mode. +Line: 1 Col: 30 Unexpected start tag (meta) in table context caused voodoo mode. +#document +| <!DOCTYPE html> +| <html> +| <head> +| <body> +| " X" +| <meta> +| <table> + +#data +<!doctype html><table> x</table> +#errors +Line: 1 Col: 24 Unexpected non-space characters in table context caused voodoo mode. +#document +| <!DOCTYPE html> +| <html> +| <head> +| <body> +| " x" +| <table> + +#data +<!doctype html><table> x </table> +#errors +Line: 1 Col: 25 Unexpected non-space characters in table context caused voodoo mode. +#document +| <!DOCTYPE html> +| <html> +| <head> +| <body> +| " x " +| <table> + +#data +<!doctype html><table><tr> x</table> +#errors +Line: 1 Col: 28 Unexpected non-space characters in table context caused voodoo mode. +#document +| <!DOCTYPE html> +| <html> +| <head> +| <body> +| " x" +| <table> +| <tbody> +| <tr> + +#data +<!doctype html><table>X<style> <tr>x </style> </table> +#errors +Line: 1 Col: 23 Unexpected non-space characters in table context caused voodoo mode. +#document +| <!DOCTYPE html> +| <html> +| <head> +| <body> +| "X" +| <table> +| <style> +| " <tr>x " +| " " + +#data +<!doctype html><div><table><a>foo</a> <tr><td>bar</td> </tr></table></div> +#errors +Line: 1 Col: 30 Unexpected start tag (a) in table context caused voodoo mode. +Line: 1 Col: 37 Unexpected end tag (a) in table context caused voodoo mode. +#document +| <!DOCTYPE html> +| <html> +| <head> +| <body> +| <div> +| <a> +| "foo" +| <table> +| " " +| <tbody> +| <tr> +| <td> +| "bar" +| " " + +#data +<frame></frame></frame><frameset><frame><frameset><frame></frameset><noframes></frameset><noframes> +#errors +6: Start tag seen without seeing a doctype first. Expected “<!DOCTYPE html>”. +13: Stray start tag “frame”. +21: Stray end tag “frame”. +29: Stray end tag “frame”. +39: “frameset” start tag after “body” already open. +105: End of file seen inside an [R]CDATA element. +105: End of file seen and there were open elements. +XXX: These errors are wrong, please fix me! +#document +| <html> +| <head> +| <frameset> +| <frame> +| <frameset> +| <frame> +| <noframes> +| "</frameset><noframes>" + +#data +<!DOCTYPE html><object></html> +#errors +1: Expected closing tag. Unexpected end of file +#document +| <!DOCTYPE html> +| <html> +| <head> +| <body> +| <object> diff --git a/src/pkg/html/testdata/webkit/tests16.dat b/src/pkg/html/testdata/webkit/tests16.dat new file mode 100644 index 000000000..937dba9f4 --- /dev/null +++ b/src/pkg/html/testdata/webkit/tests16.dat @@ -0,0 +1,2277 @@ +#data +<!doctype html><script> +#errors +Line: 1 Col: 23 Unexpected end of file. Expected end tag (script). +#document +| <!DOCTYPE html> +| <html> +| <head> +| <script> +| <body> + +#data +<!doctype html><script>a +#errors +Line: 1 Col: 24 Unexpected end of file. Expected end tag (script). +#document +| <!DOCTYPE html> +| <html> +| <head> +| <script> +| "a" +| <body> + +#data +<!doctype html><script>< +#errors +Line: 1 Col: 24 Unexpected end of file. Expected end tag (script). +#document +| <!DOCTYPE html> +| <html> +| <head> +| <script> +| "<" +| <body> + +#data +<!doctype html><script></ +#errors +Line: 1 Col: 25 Unexpected end of file. Expected end tag (script). +#document +| <!DOCTYPE html> +| <html> +| <head> +| <script> +| "</" +| <body> + +#data +<!doctype html><script></S +#errors +Line: 1 Col: 26 Unexpected end of file. Expected end tag (script). +#document +| <!DOCTYPE html> +| <html> +| <head> +| <script> +| "</S" +| <body> + +#data +<!doctype html><script></SC +#errors +Line: 1 Col: 27 Unexpected end of file. Expected end tag (script). +#document +| <!DOCTYPE html> +| <html> +| <head> +| <script> +| "</SC" +| <body> + +#data +<!doctype html><script></SCR +#errors +Line: 1 Col: 28 Unexpected end of file. Expected end tag (script). +#document +| <!DOCTYPE html> +| <html> +| <head> +| <script> +| "</SCR" +| <body> + +#data +<!doctype html><script></SCRI +#errors +Line: 1 Col: 29 Unexpected end of file. Expected end tag (script). +#document +| <!DOCTYPE html> +| <html> +| <head> +| <script> +| "</SCRI" +| <body> + +#data +<!doctype html><script></SCRIP +#errors +Line: 1 Col: 30 Unexpected end of file. Expected end tag (script). +#document +| <!DOCTYPE html> +| <html> +| <head> +| <script> +| "</SCRIP" +| <body> + +#data +<!doctype html><script></SCRIPT +#errors +Line: 1 Col: 31 Unexpected end of file. Expected end tag (script). +#document +| <!DOCTYPE html> +| <html> +| <head> +| <script> +| "</SCRIPT" +| <body> + +#data +<!doctype html><script></SCRIPT +#errors +Line: 1 Col: 32 Unexpected end of file. Expected end tag (script). +#document +| <!DOCTYPE html> +| <html> +| <head> +| <script> +| <body> + +#data +<!doctype html><script></s +#errors +Line: 1 Col: 26 Unexpected end of file. Expected end tag (script). +#document +| <!DOCTYPE html> +| <html> +| <head> +| <script> +| "</s" +| <body> + +#data +<!doctype html><script></sc +#errors +Line: 1 Col: 27 Unexpected end of file. Expected end tag (script). +#document +| <!DOCTYPE html> +| <html> +| <head> +| <script> +| "</sc" +| <body> + +#data +<!doctype html><script></scr +#errors +Line: 1 Col: 28 Unexpected end of file. Expected end tag (script). +#document +| <!DOCTYPE html> +| <html> +| <head> +| <script> +| "</scr" +| <body> + +#data +<!doctype html><script></scri +#errors +Line: 1 Col: 29 Unexpected end of file. Expected end tag (script). +#document +| <!DOCTYPE html> +| <html> +| <head> +| <script> +| "</scri" +| <body> + +#data +<!doctype html><script></scrip +#errors +Line: 1 Col: 30 Unexpected end of file. Expected end tag (script). +#document +| <!DOCTYPE html> +| <html> +| <head> +| <script> +| "</scrip" +| <body> + +#data +<!doctype html><script></script +#errors +Line: 1 Col: 31 Unexpected end of file. Expected end tag (script). +#document +| <!DOCTYPE html> +| <html> +| <head> +| <script> +| "</script" +| <body> + +#data +<!doctype html><script></script +#errors +Line: 1 Col: 32 Unexpected end of file. Expected end tag (script). +#document +| <!DOCTYPE html> +| <html> +| <head> +| <script> +| <body> + +#data +<!doctype html><script><! +#errors +Line: 1 Col: 25 Unexpected end of file. Expected end tag (script). +#document +| <!DOCTYPE html> +| <html> +| <head> +| <script> +| "<!" +| <body> + +#data +<!doctype html><script><!a +#errors +Line: 1 Col: 26 Unexpected end of file. Expected end tag (script). +#document +| <!DOCTYPE html> +| <html> +| <head> +| <script> +| "<!a" +| <body> + +#data +<!doctype html><script><!- +#errors +Line: 1 Col: 26 Unexpected end of file. Expected end tag (script). +#document +| <!DOCTYPE html> +| <html> +| <head> +| <script> +| "<!-" +| <body> + +#data +<!doctype html><script><!-a +#errors +Line: 1 Col: 27 Unexpected end of file. Expected end tag (script). +#document +| <!DOCTYPE html> +| <html> +| <head> +| <script> +| "<!-a" +| <body> + +#data +<!doctype html><script><!-- +#errors +Line: 1 Col: 27 Unexpected end of file. Expected end tag (script). +#document +| <!DOCTYPE html> +| <html> +| <head> +| <script> +| "<!--" +| <body> + +#data +<!doctype html><script><!--a +#errors +Line: 1 Col: 28 Unexpected end of file. Expected end tag (script). +#document +| <!DOCTYPE html> +| <html> +| <head> +| <script> +| "<!--a" +| <body> + +#data +<!doctype html><script><!--< +#errors +Line: 1 Col: 28 Unexpected end of file. Expected end tag (script). +#document +| <!DOCTYPE html> +| <html> +| <head> +| <script> +| "<!--<" +| <body> + +#data +<!doctype html><script><!--<a +#errors +Line: 1 Col: 29 Unexpected end of file. Expected end tag (script). +#document +| <!DOCTYPE html> +| <html> +| <head> +| <script> +| "<!--<a" +| <body> + +#data +<!doctype html><script><!--</ +#errors +Line: 1 Col: 27 Unexpected end of file. Expected end tag (script). +#document +| <!DOCTYPE html> +| <html> +| <head> +| <script> +| "<!--</" +| <body> + +#data +<!doctype html><script><!--</script +#errors +Line: 1 Col: 35 Unexpected end of file. Expected end tag (script). +#document +| <!DOCTYPE html> +| <html> +| <head> +| <script> +| "<!--</script" +| <body> + +#data +<!doctype html><script><!--</script +#errors +Line: 1 Col: 36 Unexpected end of file. Expected end tag (script). +#document +| <!DOCTYPE html> +| <html> +| <head> +| <script> +| "<!--" +| <body> + +#data +<!doctype html><script><!--<s +#errors +Line: 1 Col: 29 Unexpected end of file. Expected end tag (script). +#document +| <!DOCTYPE html> +| <html> +| <head> +| <script> +| "<!--<s" +| <body> + +#data +<!doctype html><script><!--<script +#errors +Line: 1 Col: 34 Unexpected end of file. Expected end tag (script). +#document +| <!DOCTYPE html> +| <html> +| <head> +| <script> +| "<!--<script" +| <body> + +#data +<!doctype html><script><!--<script +#errors +Line: 1 Col: 35 Unexpected end of file. Expected end tag (script). +#document +| <!DOCTYPE html> +| <html> +| <head> +| <script> +| "<!--<script " +| <body> + +#data +<!doctype html><script><!--<script < +#errors +Line: 1 Col: 36 Unexpected end of file. Expected end tag (script). +#document +| <!DOCTYPE html> +| <html> +| <head> +| <script> +| "<!--<script <" +| <body> + +#data +<!doctype html><script><!--<script <a +#errors +Line: 1 Col: 37 Unexpected end of file. Expected end tag (script). +#document +| <!DOCTYPE html> +| <html> +| <head> +| <script> +| "<!--<script <a" +| <body> + +#data +<!doctype html><script><!--<script </ +#errors +Line: 1 Col: 37 Unexpected end of file. Expected end tag (script). +#document +| <!DOCTYPE html> +| <html> +| <head> +| <script> +| "<!--<script </" +| <body> + +#data +<!doctype html><script><!--<script </s +#errors +Line: 1 Col: 38 Unexpected end of file. Expected end tag (script). +#document +| <!DOCTYPE html> +| <html> +| <head> +| <script> +| "<!--<script </s" +| <body> + +#data +<!doctype html><script><!--<script </script +#errors +Line: 1 Col: 43 Unexpected end of file. Expected end tag (script). +#document +| <!DOCTYPE html> +| <html> +| <head> +| <script> +| "<!--<script </script" +| <body> + +#data +<!doctype html><script><!--<script </scripta +#errors +Line: 1 Col: 44 Unexpected end of file. Expected end tag (script). +#document +| <!DOCTYPE html> +| <html> +| <head> +| <script> +| "<!--<script </scripta" +| <body> + +#data +<!doctype html><script><!--<script </script +#errors +Line: 1 Col: 44 Unexpected end of file. Expected end tag (script). +#document +| <!DOCTYPE html> +| <html> +| <head> +| <script> +| "<!--<script </script " +| <body> + +#data +<!doctype html><script><!--<script </script> +#errors +Line: 1 Col: 44 Unexpected end of file. Expected end tag (script). +#document +| <!DOCTYPE html> +| <html> +| <head> +| <script> +| "<!--<script </script>" +| <body> + +#data +<!doctype html><script><!--<script </script/ +#errors +Line: 1 Col: 44 Unexpected end of file. Expected end tag (script). +#document +| <!DOCTYPE html> +| <html> +| <head> +| <script> +| "<!--<script </script/" +| <body> + +#data +<!doctype html><script><!--<script </script < +#errors +Line: 1 Col: 45 Unexpected end of file. Expected end tag (script). +#document +| <!DOCTYPE html> +| <html> +| <head> +| <script> +| "<!--<script </script <" +| <body> + +#data +<!doctype html><script><!--<script </script <a +#errors +Line: 1 Col: 46 Unexpected end of file. Expected end tag (script). +#document +| <!DOCTYPE html> +| <html> +| <head> +| <script> +| "<!--<script </script <a" +| <body> + +#data +<!doctype html><script><!--<script </script </ +#errors +Line: 1 Col: 46 Unexpected end of file. Expected end tag (script). +#document +| <!DOCTYPE html> +| <html> +| <head> +| <script> +| "<!--<script </script </" +| <body> + +#data +<!doctype html><script><!--<script </script </script +#errors +Line: 1 Col: 52 Unexpected end of file. Expected end tag (script). +#document +| <!DOCTYPE html> +| <html> +| <head> +| <script> +| "<!--<script </script </script" +| <body> + +#data +<!doctype html><script><!--<script </script </script +#errors +Line: 1 Col: 53 Unexpected end of file. Expected end tag (script). +#document +| <!DOCTYPE html> +| <html> +| <head> +| <script> +| "<!--<script </script " +| <body> + +#data +<!doctype html><script><!--<script </script </script/ +#errors +Line: 1 Col: 53 Unexpected end of file. Expected end tag (script). +#document +| <!DOCTYPE html> +| <html> +| <head> +| <script> +| "<!--<script </script " +| <body> + +#data +<!doctype html><script><!--<script </script </script> +#errors +#document +| <!DOCTYPE html> +| <html> +| <head> +| <script> +| "<!--<script </script " +| <body> + +#data +<!doctype html><script><!--<script - +#errors +Line: 1 Col: 36 Unexpected end of file. Expected end tag (script). +#document +| <!DOCTYPE html> +| <html> +| <head> +| <script> +| "<!--<script -" +| <body> + +#data +<!doctype html><script><!--<script -a +#errors +Line: 1 Col: 37 Unexpected end of file. Expected end tag (script). +#document +| <!DOCTYPE html> +| <html> +| <head> +| <script> +| "<!--<script -a" +| <body> + +#data +<!doctype html><script><!--<script -< +#errors +Line: 1 Col: 37 Unexpected end of file. Expected end tag (script). +#document +| <!DOCTYPE html> +| <html> +| <head> +| <script> +| "<!--<script -<" +| <body> + +#data +<!doctype html><script><!--<script -- +#errors +Line: 1 Col: 37 Unexpected end of file. Expected end tag (script). +#document +| <!DOCTYPE html> +| <html> +| <head> +| <script> +| "<!--<script --" +| <body> + +#data +<!doctype html><script><!--<script --a +#errors +Line: 1 Col: 38 Unexpected end of file. Expected end tag (script). +#document +| <!DOCTYPE html> +| <html> +| <head> +| <script> +| "<!--<script --a" +| <body> + +#data +<!doctype html><script><!--<script --< +#errors +Line: 1 Col: 38 Unexpected end of file. Expected end tag (script). +#document +| <!DOCTYPE html> +| <html> +| <head> +| <script> +| "<!--<script --<" +| <body> + +#data +<!doctype html><script><!--<script --> +#errors +Line: 1 Col: 38 Unexpected end of file. Expected end tag (script). +#document +| <!DOCTYPE html> +| <html> +| <head> +| <script> +| "<!--<script -->" +| <body> + +#data +<!doctype html><script><!--<script -->< +#errors +Line: 1 Col: 39 Unexpected end of file. Expected end tag (script). +#document +| <!DOCTYPE html> +| <html> +| <head> +| <script> +| "<!--<script --><" +| <body> + +#data +<!doctype html><script><!--<script --></ +#errors +Line: 1 Col: 40 Unexpected end of file. Expected end tag (script). +#document +| <!DOCTYPE html> +| <html> +| <head> +| <script> +| "<!--<script --></" +| <body> + +#data +<!doctype html><script><!--<script --></script +#errors +Line: 1 Col: 46 Unexpected end of file. Expected end tag (script). +#document +| <!DOCTYPE html> +| <html> +| <head> +| <script> +| "<!--<script --></script" +| <body> + +#data +<!doctype html><script><!--<script --></script +#errors +Line: 1 Col: 47 Unexpected end of file. Expected end tag (script). +#document +| <!DOCTYPE html> +| <html> +| <head> +| <script> +| "<!--<script -->" +| <body> + +#data +<!doctype html><script><!--<script --></script/ +#errors +Line: 1 Col: 47 Unexpected end of file. Expected end tag (script). +#document +| <!DOCTYPE html> +| <html> +| <head> +| <script> +| "<!--<script -->" +| <body> + +#data +<!doctype html><script><!--<script --></script> +#errors +#document +| <!DOCTYPE html> +| <html> +| <head> +| <script> +| "<!--<script -->" +| <body> + +#data +<!doctype html><script><!--<script><\/script>--></script> +#errors +#document +| <!DOCTYPE html> +| <html> +| <head> +| <script> +| "<!--<script><\/script>-->" +| <body> + +#data +<!doctype html><script><!--<script></scr'+'ipt>--></script> +#errors +#document +| <!DOCTYPE html> +| <html> +| <head> +| <script> +| "<!--<script></scr'+'ipt>-->" +| <body> + +#data +<!doctype html><script><!--<script></script><script></script></script> +#errors +#document +| <!DOCTYPE html> +| <html> +| <head> +| <script> +| "<!--<script></script><script></script>" +| <body> + +#data +<!doctype html><script><!--<script></script><script></script>--><!--</script> +#errors +#document +| <!DOCTYPE html> +| <html> +| <head> +| <script> +| "<!--<script></script><script></script>--><!--" +| <body> + +#data +<!doctype html><script><!--<script></script><script></script>-- ></script> +#errors +#document +| <!DOCTYPE html> +| <html> +| <head> +| <script> +| "<!--<script></script><script></script>-- >" +| <body> + +#data +<!doctype html><script><!--<script></script><script></script>- -></script> +#errors +#document +| <!DOCTYPE html> +| <html> +| <head> +| <script> +| "<!--<script></script><script></script>- ->" +| <body> + +#data +<!doctype html><script><!--<script></script><script></script>- - ></script> +#errors +#document +| <!DOCTYPE html> +| <html> +| <head> +| <script> +| "<!--<script></script><script></script>- - >" +| <body> + +#data +<!doctype html><script><!--<script></script><script></script>-></script> +#errors +#document +| <!DOCTYPE html> +| <html> +| <head> +| <script> +| "<!--<script></script><script></script>->" +| <body> + +#data +<!doctype html><script><!--<script>--!></script>X +#errors +Line: 1 Col: 49 Unexpected end of file. Expected end tag (script). +#document +| <!DOCTYPE html> +| <html> +| <head> +| <script> +| "<!--<script>--!></script>X" +| <body> + +#data +<!doctype html><script><!--<scr'+'ipt></script>--></script> +#errors +Line: 1 Col: 59 Unexpected end tag (script). +#document +| <!DOCTYPE html> +| <html> +| <head> +| <script> +| "<!--<scr'+'ipt>" +| <body> +| "-->" + +#data +<!doctype html><script><!--<script></scr'+'ipt></script>X +#errors +Line: 1 Col: 57 Unexpected end of file. Expected end tag (script). +#document +| <!DOCTYPE html> +| <html> +| <head> +| <script> +| "<!--<script></scr'+'ipt></script>X" +| <body> + +#data +<!doctype html><style><!--<style></style>--></style> +#errors +Line: 1 Col: 52 Unexpected end tag (style). +#document +| <!DOCTYPE html> +| <html> +| <head> +| <style> +| "<!--<style>" +| <body> +| "-->" + +#data +<!doctype html><style><!--</style>X +#errors +#document +| <!DOCTYPE html> +| <html> +| <head> +| <style> +| "<!--" +| <body> +| "X" + +#data +<!doctype html><style><!--...</style>...--></style> +#errors +Line: 1 Col: 51 Unexpected end tag (style). +#document +| <!DOCTYPE html> +| <html> +| <head> +| <style> +| "<!--..." +| <body> +| "...-->" + +#data +<!doctype html><style><!--<br><html xmlns:v="urn:schemas-microsoft-com:vml"><!--[if !mso]><style></style>X +#errors +#document +| <!DOCTYPE html> +| <html> +| <head> +| <style> +| "<!--<br><html xmlns:v="urn:schemas-microsoft-com:vml"><!--[if !mso]><style>" +| <body> +| "X" + +#data +<!doctype html><style><!--...<style><!--...--!></style>--></style> +#errors +Line: 1 Col: 66 Unexpected end tag (style). +#document +| <!DOCTYPE html> +| <html> +| <head> +| <style> +| "<!--...<style><!--...--!>" +| <body> +| "-->" + +#data +<!doctype html><style><!--...</style><!-- --><style>@import ...</style> +#errors +#document +| <!DOCTYPE html> +| <html> +| <head> +| <style> +| "<!--..." +| <!-- --> +| <style> +| "@import ..." +| <body> + +#data +<!doctype html><style>...<style><!--...</style><!-- --></style> +#errors +Line: 1 Col: 63 Unexpected end tag (style). +#document +| <!DOCTYPE html> +| <html> +| <head> +| <style> +| "...<style><!--..." +| <!-- --> +| <body> + +#data +<!doctype html><style>...<!--[if IE]><style>...</style>X +#errors +#document +| <!DOCTYPE html> +| <html> +| <head> +| <style> +| "...<!--[if IE]><style>..." +| <body> +| "X" + +#data +<!doctype html><title><!--<title>--> +#errors +Line: 1 Col: 52 Unexpected end tag (title). +#document +| +| +| +| +| "<!--<title>" +| <body> +| "-->" + +#data +<!doctype html><title></title> +#errors +#document +| +| +| +| +| "" +| + +#data +foo/title><link></head><body>X +#errors +Line: 1 Col: 52 Unexpected end of file. Expected end tag (title). +#document +| <!DOCTYPE html> +| <html> +| <head> +| <title> +| "foo/title><link></head><body>X" +| <body> + +#data +<!doctype html><noscript><!--<noscript></noscript>--></noscript> +#errors +Line: 1 Col: 64 Unexpected end tag (noscript). +#document +| <!DOCTYPE html> +| <html> +| <head> +| <noscript> +| "<!--<noscript>" +| <body> +| "-->" + +#data +<!doctype html><noscript><!--</noscript>X<noscript>--></noscript> +#errors +#document +| <!DOCTYPE html> +| <html> +| <head> +| <noscript> +| "<!--" +| <body> +| "X" +| <noscript> +| "-->" + +#data +<!doctype html><noscript><iframe></noscript>X +#errors +#document +| <!DOCTYPE html> +| <html> +| <head> +| <noscript> +| "<iframe>" +| <body> +| "X" + +#data +<!doctype html><noframes><!--<noframes></noframes>--></noframes> +#errors +Line: 1 Col: 64 Unexpected end tag (noframes). +#document +| <!DOCTYPE html> +| <html> +| <head> +| <noframes> +| "<!--<noframes>" +| <body> +| "-->" + +#data +<!doctype html><noframes><body><script><!--...</script></body></noframes></html> +#errors +#document +| <!DOCTYPE html> +| <html> +| <head> +| <noframes> +| "<body><script><!--...</script></body>" +| <body> + +#data +<!doctype html><textarea><!--<textarea></textarea>--></textarea> +#errors +Line: 1 Col: 64 Unexpected end tag (textarea). +#document +| <!DOCTYPE html> +| <html> +| <head> +| <body> +| <textarea> +| "<!--<textarea>" +| "-->" + +#data +<!doctype html><textarea></textarea></textarea> +#errors +#document +| <!DOCTYPE html> +| <html> +| <head> +| <body> +| <textarea> +| "</textarea>" + +#data +<!doctype html><iframe><!--<iframe></iframe>--></iframe> +#errors +Line: 1 Col: 56 Unexpected end tag (iframe). +#document +| <!DOCTYPE html> +| <html> +| <head> +| <body> +| <iframe> +| "<!--<iframe>" +| "-->" + +#data +<!doctype html><iframe>...<!--X->...<!--/X->...</iframe> +#errors +#document +| <!DOCTYPE html> +| <html> +| <head> +| <body> +| <iframe> +| "...<!--X->...<!--/X->..." + +#data +<!doctype html><xmp><!--<xmp></xmp>--></xmp> +#errors +Line: 1 Col: 44 Unexpected end tag (xmp). +#document +| <!DOCTYPE html> +| <html> +| <head> +| <body> +| <xmp> +| "<!--<xmp>" +| "-->" + +#data +<!doctype html><noembed><!--<noembed></noembed>--></noembed> +#errors +Line: 1 Col: 60 Unexpected end tag (noembed). +#document +| <!DOCTYPE html> +| <html> +| <head> +| <body> +| <noembed> +| "<!--<noembed>" +| "-->" + +#data +<script> +#errors +Line: 1 Col: 8 Unexpected start tag (script). Expected DOCTYPE. +Line: 1 Col: 8 Unexpected end of file. Expected end tag (script). +#document +| <html> +| <head> +| <script> +| <body> + +#data +<script>a +#errors +Line: 1 Col: 8 Unexpected start tag (script). Expected DOCTYPE. +Line: 1 Col: 9 Unexpected end of file. Expected end tag (script). +#document +| <html> +| <head> +| <script> +| "a" +| <body> + +#data +<script>< +#errors +Line: 1 Col: 8 Unexpected start tag (script). Expected DOCTYPE. +Line: 1 Col: 9 Unexpected end of file. Expected end tag (script). +#document +| <html> +| <head> +| <script> +| "<" +| <body> + +#data +<script></ +#errors +Line: 1 Col: 8 Unexpected start tag (script). Expected DOCTYPE. +Line: 1 Col: 10 Unexpected end of file. Expected end tag (script). +#document +| <html> +| <head> +| <script> +| "</" +| <body> + +#data +<script></S +#errors +Line: 1 Col: 8 Unexpected start tag (script). Expected DOCTYPE. +Line: 1 Col: 11 Unexpected end of file. Expected end tag (script). +#document +| <html> +| <head> +| <script> +| "</S" +| <body> + +#data +<script></SC +#errors +Line: 1 Col: 8 Unexpected start tag (script). Expected DOCTYPE. +Line: 1 Col: 12 Unexpected end of file. Expected end tag (script). +#document +| <html> +| <head> +| <script> +| "</SC" +| <body> + +#data +<script></SCR +#errors +Line: 1 Col: 8 Unexpected start tag (script). Expected DOCTYPE. +Line: 1 Col: 13 Unexpected end of file. Expected end tag (script). +#document +| <html> +| <head> +| <script> +| "</SCR" +| <body> + +#data +<script></SCRI +#errors +Line: 1 Col: 8 Unexpected start tag (script). Expected DOCTYPE. +Line: 1 Col: 14 Unexpected end of file. Expected end tag (script). +#document +| <html> +| <head> +| <script> +| "</SCRI" +| <body> + +#data +<script></SCRIP +#errors +Line: 1 Col: 8 Unexpected start tag (script). Expected DOCTYPE. +Line: 1 Col: 15 Unexpected end of file. Expected end tag (script). +#document +| <html> +| <head> +| <script> +| "</SCRIP" +| <body> + +#data +<script></SCRIPT +#errors +Line: 1 Col: 8 Unexpected start tag (script). Expected DOCTYPE. +Line: 1 Col: 16 Unexpected end of file. Expected end tag (script). +#document +| <html> +| <head> +| <script> +| "</SCRIPT" +| <body> + +#data +<script></SCRIPT +#errors +Line: 1 Col: 8 Unexpected start tag (script). Expected DOCTYPE. +Line: 1 Col: 17 Unexpected end of file. Expected end tag (script). +#document +| <html> +| <head> +| <script> +| <body> + +#data +<script></s +#errors +Line: 1 Col: 8 Unexpected start tag (script). Expected DOCTYPE. +Line: 1 Col: 11 Unexpected end of file. Expected end tag (script). +#document +| <html> +| <head> +| <script> +| "</s" +| <body> + +#data +<script></sc +#errors +Line: 1 Col: 8 Unexpected start tag (script). Expected DOCTYPE. +Line: 1 Col: 12 Unexpected end of file. Expected end tag (script). +#document +| <html> +| <head> +| <script> +| "</sc" +| <body> + +#data +<script></scr +#errors +Line: 1 Col: 8 Unexpected start tag (script). Expected DOCTYPE. +Line: 1 Col: 13 Unexpected end of file. Expected end tag (script). +#document +| <html> +| <head> +| <script> +| "</scr" +| <body> + +#data +<script></scri +#errors +Line: 1 Col: 8 Unexpected start tag (script). Expected DOCTYPE. +Line: 1 Col: 14 Unexpected end of file. Expected end tag (script). +#document +| <html> +| <head> +| <script> +| "</scri" +| <body> + +#data +<script></scrip +#errors +Line: 1 Col: 8 Unexpected start tag (script). Expected DOCTYPE. +Line: 1 Col: 15 Unexpected end of file. Expected end tag (script). +#document +| <html> +| <head> +| <script> +| "</scrip" +| <body> + +#data +<script></script +#errors +Line: 1 Col: 8 Unexpected start tag (script). Expected DOCTYPE. +Line: 1 Col: 16 Unexpected end of file. Expected end tag (script). +#document +| <html> +| <head> +| <script> +| "</script" +| <body> + +#data +<script></script +#errors +Line: 1 Col: 8 Unexpected start tag (script). Expected DOCTYPE. +Line: 1 Col: 17 Unexpected end of file. Expected end tag (script). +#document +| <html> +| <head> +| <script> +| <body> + +#data +<script><! +#errors +Line: 1 Col: 8 Unexpected start tag (script). Expected DOCTYPE. +Line: 1 Col: 10 Unexpected end of file. Expected end tag (script). +#document +| <html> +| <head> +| <script> +| "<!" +| <body> + +#data +<script><!a +#errors +Line: 1 Col: 8 Unexpected start tag (script). Expected DOCTYPE. +Line: 1 Col: 11 Unexpected end of file. Expected end tag (script). +#document +| <html> +| <head> +| <script> +| "<!a" +| <body> + +#data +<script><!- +#errors +Line: 1 Col: 8 Unexpected start tag (script). Expected DOCTYPE. +Line: 1 Col: 11 Unexpected end of file. Expected end tag (script). +#document +| <html> +| <head> +| <script> +| "<!-" +| <body> + +#data +<script><!-a +#errors +Line: 1 Col: 8 Unexpected start tag (script). Expected DOCTYPE. +Line: 1 Col: 12 Unexpected end of file. Expected end tag (script). +#document +| <html> +| <head> +| <script> +| "<!-a" +| <body> + +#data +<script><!-- +#errors +Line: 1 Col: 8 Unexpected start tag (script). Expected DOCTYPE. +Line: 1 Col: 12 Unexpected end of file. Expected end tag (script). +#document +| <html> +| <head> +| <script> +| "<!--" +| <body> + +#data +<script><!--a +#errors +Line: 1 Col: 8 Unexpected start tag (script). Expected DOCTYPE. +Line: 1 Col: 13 Unexpected end of file. Expected end tag (script). +#document +| <html> +| <head> +| <script> +| "<!--a" +| <body> + +#data +<script><!--< +#errors +Line: 1 Col: 8 Unexpected start tag (script). Expected DOCTYPE. +Line: 1 Col: 13 Unexpected end of file. Expected end tag (script). +#document +| <html> +| <head> +| <script> +| "<!--<" +| <body> + +#data +<script><!--<a +#errors +Line: 1 Col: 8 Unexpected start tag (script). Expected DOCTYPE. +Line: 1 Col: 14 Unexpected end of file. Expected end tag (script). +#document +| <html> +| <head> +| <script> +| "<!--<a" +| <body> + +#data +<script><!--</ +#errors +Line: 1 Col: 8 Unexpected start tag (script). Expected DOCTYPE. +Line: 1 Col: 14 Unexpected end of file. Expected end tag (script). +#document +| <html> +| <head> +| <script> +| "<!--</" +| <body> + +#data +<script><!--</script +#errors +Line: 1 Col: 8 Unexpected start tag (script). Expected DOCTYPE. +Line: 1 Col: 20 Unexpected end of file. Expected end tag (script). +#document +| <html> +| <head> +| <script> +| "<!--</script" +| <body> + +#data +<script><!--</script +#errors +Line: 1 Col: 8 Unexpected start tag (script). Expected DOCTYPE. +Line: 1 Col: 21 Unexpected end of file. Expected end tag (script). +#document +| <html> +| <head> +| <script> +| "<!--" +| <body> + +#data +<script><!--<s +#errors +Line: 1 Col: 8 Unexpected start tag (script). Expected DOCTYPE. +Line: 1 Col: 14 Unexpected end of file. Expected end tag (script). +#document +| <html> +| <head> +| <script> +| "<!--<s" +| <body> + +#data +<script><!--<script +#errors +Line: 1 Col: 8 Unexpected start tag (script). Expected DOCTYPE. +Line: 1 Col: 19 Unexpected end of file. Expected end tag (script). +#document +| <html> +| <head> +| <script> +| "<!--<script" +| <body> + +#data +<script><!--<script +#errors +Line: 1 Col: 8 Unexpected start tag (script). Expected DOCTYPE. +Line: 1 Col: 20 Unexpected end of file. Expected end tag (script). +#document +| <html> +| <head> +| <script> +| "<!--<script " +| <body> + +#data +<script><!--<script < +#errors +Line: 1 Col: 8 Unexpected start tag (script). Expected DOCTYPE. +Line: 1 Col: 21 Unexpected end of file. Expected end tag (script). +#document +| <html> +| <head> +| <script> +| "<!--<script <" +| <body> + +#data +<script><!--<script <a +#errors +Line: 1 Col: 8 Unexpected start tag (script). Expected DOCTYPE. +Line: 1 Col: 22 Unexpected end of file. Expected end tag (script). +#document +| <html> +| <head> +| <script> +| "<!--<script <a" +| <body> + +#data +<script><!--<script </ +#errors +Line: 1 Col: 8 Unexpected start tag (script). Expected DOCTYPE. +Line: 1 Col: 22 Unexpected end of file. Expected end tag (script). +#document +| <html> +| <head> +| <script> +| "<!--<script </" +| <body> + +#data +<script><!--<script </s +#errors +Line: 1 Col: 8 Unexpected start tag (script). Expected DOCTYPE. +Line: 1 Col: 23 Unexpected end of file. Expected end tag (script). +#document +| <html> +| <head> +| <script> +| "<!--<script </s" +| <body> + +#data +<script><!--<script </script +#errors +Line: 1 Col: 8 Unexpected start tag (script). Expected DOCTYPE. +Line: 1 Col: 28 Unexpected end of file. Expected end tag (script). +#document +| <html> +| <head> +| <script> +| "<!--<script </script" +| <body> + +#data +<script><!--<script </scripta +#errors +Line: 1 Col: 8 Unexpected start tag (script). Expected DOCTYPE. +Line: 1 Col: 29 Unexpected end of file. Expected end tag (script). +#document +| <html> +| <head> +| <script> +| "<!--<script </scripta" +| <body> + +#data +<script><!--<script </script +#errors +Line: 1 Col: 8 Unexpected start tag (script). Expected DOCTYPE. +Line: 1 Col: 29 Unexpected end of file. Expected end tag (script). +#document +| <html> +| <head> +| <script> +| "<!--<script </script " +| <body> + +#data +<script><!--<script </script> +#errors +Line: 1 Col: 8 Unexpected start tag (script). Expected DOCTYPE. +Line: 1 Col: 29 Unexpected end of file. Expected end tag (script). +#document +| <html> +| <head> +| <script> +| "<!--<script </script>" +| <body> + +#data +<script><!--<script </script/ +#errors +Line: 1 Col: 8 Unexpected start tag (script). Expected DOCTYPE. +Line: 1 Col: 29 Unexpected end of file. Expected end tag (script). +#document +| <html> +| <head> +| <script> +| "<!--<script </script/" +| <body> + +#data +<script><!--<script </script < +#errors +Line: 1 Col: 8 Unexpected start tag (script). Expected DOCTYPE. +Line: 1 Col: 30 Unexpected end of file. Expected end tag (script). +#document +| <html> +| <head> +| <script> +| "<!--<script </script <" +| <body> + +#data +<script><!--<script </script <a +#errors +Line: 1 Col: 8 Unexpected start tag (script). Expected DOCTYPE. +Line: 1 Col: 31 Unexpected end of file. Expected end tag (script). +#document +| <html> +| <head> +| <script> +| "<!--<script </script <a" +| <body> + +#data +<script><!--<script </script </ +#errors +Line: 1 Col: 8 Unexpected start tag (script). Expected DOCTYPE. +Line: 1 Col: 31 Unexpected end of file. Expected end tag (script). +#document +| <html> +| <head> +| <script> +| "<!--<script </script </" +| <body> + +#data +<script><!--<script </script </script +#errors +Line: 1 Col: 8 Unexpected start tag (script). Expected DOCTYPE. +Line: 1 Col: 38 Unexpected end of file. Expected end tag (script). +#document +| <html> +| <head> +| <script> +| "<!--<script </script </script" +| <body> + +#data +<script><!--<script </script </script +#errors +Line: 1 Col: 8 Unexpected start tag (script). Expected DOCTYPE. +Line: 1 Col: 38 Unexpected end of file. Expected end tag (script). +#document +| <html> +| <head> +| <script> +| "<!--<script </script " +| <body> + +#data +<script><!--<script </script </script/ +#errors +Line: 1 Col: 8 Unexpected start tag (script). Expected DOCTYPE. +Line: 1 Col: 38 Unexpected end of file. Expected end tag (script). +#document +| <html> +| <head> +| <script> +| "<!--<script </script " +| <body> + +#data +<script><!--<script </script </script> +#errors +Line: 1 Col: 8 Unexpected start tag (script). Expected DOCTYPE. +#document +| <html> +| <head> +| <script> +| "<!--<script </script " +| <body> + +#data +<script><!--<script - +#errors +Line: 1 Col: 8 Unexpected start tag (script). Expected DOCTYPE. +Line: 1 Col: 21 Unexpected end of file. Expected end tag (script). +#document +| <html> +| <head> +| <script> +| "<!--<script -" +| <body> + +#data +<script><!--<script -a +#errors +Line: 1 Col: 8 Unexpected start tag (script). Expected DOCTYPE. +Line: 1 Col: 22 Unexpected end of file. Expected end tag (script). +#document +| <html> +| <head> +| <script> +| "<!--<script -a" +| <body> + +#data +<script><!--<script -- +#errors +Line: 1 Col: 8 Unexpected start tag (script). Expected DOCTYPE. +Line: 1 Col: 22 Unexpected end of file. Expected end tag (script). +#document +| <html> +| <head> +| <script> +| "<!--<script --" +| <body> + +#data +<script><!--<script --a +#errors +Line: 1 Col: 8 Unexpected start tag (script). Expected DOCTYPE. +Line: 1 Col: 23 Unexpected end of file. Expected end tag (script). +#document +| <html> +| <head> +| <script> +| "<!--<script --a" +| <body> + +#data +<script><!--<script --> +#errors +Line: 1 Col: 8 Unexpected start tag (script). Expected DOCTYPE. +Line: 1 Col: 23 Unexpected end of file. Expected end tag (script). +#document +| <html> +| <head> +| <script> +| "<!--<script -->" +| <body> + +#data +<script><!--<script -->< +#errors +Line: 1 Col: 8 Unexpected start tag (script). Expected DOCTYPE. +Line: 1 Col: 24 Unexpected end of file. Expected end tag (script). +#document +| <html> +| <head> +| <script> +| "<!--<script --><" +| <body> + +#data +<script><!--<script --></ +#errors +Line: 1 Col: 8 Unexpected start tag (script). Expected DOCTYPE. +Line: 1 Col: 25 Unexpected end of file. Expected end tag (script). +#document +| <html> +| <head> +| <script> +| "<!--<script --></" +| <body> + +#data +<script><!--<script --></script +#errors +Line: 1 Col: 8 Unexpected start tag (script). Expected DOCTYPE. +Line: 1 Col: 31 Unexpected end of file. Expected end tag (script). +#document +| <html> +| <head> +| <script> +| "<!--<script --></script" +| <body> + +#data +<script><!--<script --></script +#errors +Line: 1 Col: 8 Unexpected start tag (script). Expected DOCTYPE. +Line: 1 Col: 32 Unexpected end of file. Expected end tag (script). +#document +| <html> +| <head> +| <script> +| "<!--<script -->" +| <body> + +#data +<script><!--<script --></script/ +#errors +Line: 1 Col: 8 Unexpected start tag (script). Expected DOCTYPE. +Line: 1 Col: 32 Unexpected end of file. Expected end tag (script). +#document +| <html> +| <head> +| <script> +| "<!--<script -->" +| <body> + +#data +<script><!--<script --></script> +#errors +Line: 1 Col: 8 Unexpected start tag (script). Expected DOCTYPE. +#document +| <html> +| <head> +| <script> +| "<!--<script -->" +| <body> + +#data +<script><!--<script><\/script>--></script> +#errors +Line: 1 Col: 8 Unexpected start tag (script). Expected DOCTYPE. +#document +| <html> +| <head> +| <script> +| "<!--<script><\/script>-->" +| <body> + +#data +<script><!--<script></scr'+'ipt>--></script> +#errors +Line: 1 Col: 8 Unexpected start tag (script). Expected DOCTYPE. +#document +| <html> +| <head> +| <script> +| "<!--<script></scr'+'ipt>-->" +| <body> + +#data +<script><!--<script></script><script></script></script> +#errors +Line: 1 Col: 8 Unexpected start tag (script). Expected DOCTYPE. +#document +| <html> +| <head> +| <script> +| "<!--<script></script><script></script>" +| <body> + +#data +<script><!--<script></script><script></script>--><!--</script> +#errors +Line: 1 Col: 8 Unexpected start tag (script). Expected DOCTYPE. +#document +| <html> +| <head> +| <script> +| "<!--<script></script><script></script>--><!--" +| <body> + +#data +<script><!--<script></script><script></script>-- ></script> +#errors +Line: 1 Col: 8 Unexpected start tag (script). Expected DOCTYPE. +#document +| <html> +| <head> +| <script> +| "<!--<script></script><script></script>-- >" +| <body> + +#data +<script><!--<script></script><script></script>- -></script> +#errors +Line: 1 Col: 8 Unexpected start tag (script). Expected DOCTYPE. +#document +| <html> +| <head> +| <script> +| "<!--<script></script><script></script>- ->" +| <body> + +#data +<script><!--<script></script><script></script>- - ></script> +#errors +Line: 1 Col: 8 Unexpected start tag (script). Expected DOCTYPE. +#document +| <html> +| <head> +| <script> +| "<!--<script></script><script></script>- - >" +| <body> + +#data +<script><!--<script></script><script></script>-></script> +#errors +Line: 1 Col: 8 Unexpected start tag (script). Expected DOCTYPE. +#document +| <html> +| <head> +| <script> +| "<!--<script></script><script></script>->" +| <body> + +#data +<script><!--<script>--!></script>X +#errors +Line: 1 Col: 8 Unexpected start tag (script). Expected DOCTYPE. +Line: 1 Col: 34 Unexpected end of file. Expected end tag (script). +#document +| <html> +| <head> +| <script> +| "<!--<script>--!></script>X" +| <body> + +#data +<script><!--<scr'+'ipt></script>--></script> +#errors +Line: 1 Col: 8 Unexpected start tag (script). Expected DOCTYPE. +Line: 1 Col: 44 Unexpected end tag (script). +#document +| <html> +| <head> +| <script> +| "<!--<scr'+'ipt>" +| <body> +| "-->" + +#data +<script><!--<script></scr'+'ipt></script>X +#errors +Line: 1 Col: 8 Unexpected start tag (script). Expected DOCTYPE. +Line: 1 Col: 42 Unexpected end of file. Expected end tag (script). +#document +| <html> +| <head> +| <script> +| "<!--<script></scr'+'ipt></script>X" +| <body> + +#data +<style><!--<style></style>--></style> +#errors +Line: 1 Col: 7 Unexpected start tag (style). Expected DOCTYPE. +Line: 1 Col: 37 Unexpected end tag (style). +#document +| <html> +| <head> +| <style> +| "<!--<style>" +| <body> +| "-->" + +#data +<style><!--</style>X +#errors +Line: 1 Col: 7 Unexpected start tag (style). Expected DOCTYPE. +#document +| <html> +| <head> +| <style> +| "<!--" +| <body> +| "X" + +#data +<style><!--...</style>...--></style> +#errors +Line: 1 Col: 7 Unexpected start tag (style). Expected DOCTYPE. +Line: 1 Col: 36 Unexpected end tag (style). +#document +| <html> +| <head> +| <style> +| "<!--..." +| <body> +| "...-->" + +#data +<style><!--<br><html xmlns:v="urn:schemas-microsoft-com:vml"><!--[if !mso]><style></style>X +#errors +Line: 1 Col: 7 Unexpected start tag (style). Expected DOCTYPE. +#document +| <html> +| <head> +| <style> +| "<!--<br><html xmlns:v="urn:schemas-microsoft-com:vml"><!--[if !mso]><style>" +| <body> +| "X" + +#data +<style><!--...<style><!--...--!></style>--></style> +#errors +Line: 1 Col: 7 Unexpected start tag (style). Expected DOCTYPE. +Line: 1 Col: 51 Unexpected end tag (style). +#document +| <html> +| <head> +| <style> +| "<!--...<style><!--...--!>" +| <body> +| "-->" + +#data +<style><!--...</style><!-- --><style>@import ...</style> +#errors +Line: 1 Col: 7 Unexpected start tag (style). Expected DOCTYPE. +#document +| <html> +| <head> +| <style> +| "<!--..." +| <!-- --> +| <style> +| "@import ..." +| <body> + +#data +<style>...<style><!--...</style><!-- --></style> +#errors +Line: 1 Col: 7 Unexpected start tag (style). Expected DOCTYPE. +Line: 1 Col: 48 Unexpected end tag (style). +#document +| <html> +| <head> +| <style> +| "...<style><!--..." +| <!-- --> +| <body> + +#data +<style>...<!--[if IE]><style>...</style>X +#errors +Line: 1 Col: 7 Unexpected start tag (style). Expected DOCTYPE. +#document +| <html> +| <head> +| <style> +| "...<!--[if IE]><style>..." +| <body> +| "X" + +#data +<title><!--<title>--> +#errors +Line: 1 Col: 7 Unexpected start tag (title). Expected DOCTYPE. +Line: 1 Col: 37 Unexpected end tag (title). +#document +| +| +| +| "<!--<title>" +| <body> +| "-->" + +#data +<title></title> +#errors +Line: 1 Col: 7 Unexpected start tag (title). Expected DOCTYPE. +#document +| +| +| +| "" +| + +#data +foo/title><link></head><body>X +#errors +Line: 1 Col: 7 Unexpected start tag (title). Expected DOCTYPE. +Line: 1 Col: 37 Unexpected end of file. Expected end tag (title). +#document +| <html> +| <head> +| <title> +| "foo/title><link></head><body>X" +| <body> + +#data +<noscript><!--<noscript></noscript>--></noscript> +#errors +Line: 1 Col: 10 Unexpected start tag (noscript). Expected DOCTYPE. +Line: 1 Col: 49 Unexpected end tag (noscript). +#document +| <html> +| <head> +| <noscript> +| "<!--<noscript>" +| <body> +| "-->" + +#data +<noscript><!--</noscript>X<noscript>--></noscript> +#errors +Line: 1 Col: 10 Unexpected start tag (noscript). Expected DOCTYPE. +#document +| <html> +| <head> +| <noscript> +| "<!--" +| <body> +| "X" +| <noscript> +| "-->" + +#data +<noscript><iframe></noscript>X +#errors +Line: 1 Col: 10 Unexpected start tag (noscript). Expected DOCTYPE. +#document +| <html> +| <head> +| <noscript> +| "<iframe>" +| <body> +| "X" + +#data +<noframes><!--<noframes></noframes>--></noframes> +#errors +Line: 1 Col: 10 Unexpected start tag (noframes). Expected DOCTYPE. +Line: 1 Col: 49 Unexpected end tag (noframes). +#document +| <html> +| <head> +| <noframes> +| "<!--<noframes>" +| <body> +| "-->" + +#data +<noframes><body><script><!--...</script></body></noframes></html> +#errors +Line: 1 Col: 10 Unexpected start tag (noframes). Expected DOCTYPE. +#document +| <html> +| <head> +| <noframes> +| "<body><script><!--...</script></body>" +| <body> + +#data +<textarea><!--<textarea></textarea>--></textarea> +#errors +Line: 1 Col: 10 Unexpected start tag (textarea). Expected DOCTYPE. +Line: 1 Col: 49 Unexpected end tag (textarea). +#document +| <html> +| <head> +| <body> +| <textarea> +| "<!--<textarea>" +| "-->" + +#data +<textarea></textarea></textarea> +#errors +Line: 1 Col: 10 Unexpected start tag (textarea). Expected DOCTYPE. +#document +| <html> +| <head> +| <body> +| <textarea> +| "</textarea>" + +#data +<iframe><!--<iframe></iframe>--></iframe> +#errors +Line: 1 Col: 8 Unexpected start tag (iframe). Expected DOCTYPE. +Line: 1 Col: 41 Unexpected end tag (iframe). +#document +| <html> +| <head> +| <body> +| <iframe> +| "<!--<iframe>" +| "-->" + +#data +<iframe>...<!--X->...<!--/X->...</iframe> +#errors +Line: 1 Col: 8 Unexpected start tag (iframe). Expected DOCTYPE. +#document +| <html> +| <head> +| <body> +| <iframe> +| "...<!--X->...<!--/X->..." + +#data +<xmp><!--<xmp></xmp>--></xmp> +#errors +Line: 1 Col: 5 Unexpected start tag (xmp). Expected DOCTYPE. +Line: 1 Col: 29 Unexpected end tag (xmp). +#document +| <html> +| <head> +| <body> +| <xmp> +| "<!--<xmp>" +| "-->" + +#data +<noembed><!--<noembed></noembed>--></noembed> +#errors +Line: 1 Col: 9 Unexpected start tag (noembed). Expected DOCTYPE. +Line: 1 Col: 45 Unexpected end tag (noembed). +#document +| <html> +| <head> +| <body> +| <noembed> +| "<!--<noembed>" +| "-->" + +#data +<!doctype html><table> + +#errors +Line 2 Col 0 Unexpected end of file. Expected table content. +#document +| <!DOCTYPE html> +| <html> +| <head> +| <body> +| <table> +| " +" + +#data +<!doctype html><table><td><span><font></span><span> +#errors +Line 1 Col 26 Unexpected table cell start tag (td) in the table body phase. +Line 1 Col 45 Unexpected end tag (span). +Line 1 Col 51 Expected closing tag. Unexpected end of file. +#document +| <!DOCTYPE html> +| <html> +| <head> +| <body> +| <table> +| <tbody> +| <tr> +| <td> +| <span> +| <font> +| <font> +| <span> + +#data +<!doctype html><form><table></form><form></table></form> +#errors +35: Stray end tag “form”. +41: Start tag “form” seen in “table”. +#document +| <!DOCTYPE html> +| <html> +| <head> +| <body> +| <form> +| <table> +| <form> diff --git a/src/pkg/html/testdata/webkit/tests17.dat b/src/pkg/html/testdata/webkit/tests17.dat new file mode 100644 index 000000000..7b555f888 --- /dev/null +++ b/src/pkg/html/testdata/webkit/tests17.dat @@ -0,0 +1,153 @@ +#data +<!doctype html><table><tbody><select><tr> +#errors +#document +| <!DOCTYPE html> +| <html> +| <head> +| <body> +| <select> +| <table> +| <tbody> +| <tr> + +#data +<!doctype html><table><tr><select><td> +#errors +#document +| <!DOCTYPE html> +| <html> +| <head> +| <body> +| <select> +| <table> +| <tbody> +| <tr> +| <td> + +#data +<!doctype html><table><tr><td><select><td> +#errors +#document +| <!DOCTYPE html> +| <html> +| <head> +| <body> +| <table> +| <tbody> +| <tr> +| <td> +| <select> +| <td> + +#data +<!doctype html><table><tr><th><select><td> +#errors +#document +| <!DOCTYPE html> +| <html> +| <head> +| <body> +| <table> +| <tbody> +| <tr> +| <th> +| <select> +| <td> + +#data +<!doctype html><table><caption><select><tr> +#errors +#document +| <!DOCTYPE html> +| <html> +| <head> +| <body> +| <table> +| <caption> +| <select> +| <tbody> +| <tr> + +#data +<!doctype html><select><tr> +#errors +#document +| <!DOCTYPE html> +| <html> +| <head> +| <body> +| <select> + +#data +<!doctype html><select><td> +#errors +#document +| <!DOCTYPE html> +| <html> +| <head> +| <body> +| <select> + +#data +<!doctype html><select><th> +#errors +#document +| <!DOCTYPE html> +| <html> +| <head> +| <body> +| <select> + +#data +<!doctype html><select><tbody> +#errors +#document +| <!DOCTYPE html> +| <html> +| <head> +| <body> +| <select> + +#data +<!doctype html><select><thead> +#errors +#document +| <!DOCTYPE html> +| <html> +| <head> +| <body> +| <select> + +#data +<!doctype html><select><tfoot> +#errors +#document +| <!DOCTYPE html> +| <html> +| <head> +| <body> +| <select> + +#data +<!doctype html><select><caption> +#errors +#document +| <!DOCTYPE html> +| <html> +| <head> +| <body> +| <select> + +#data +<!doctype html><table><tr></table>a +#errors +#document +| <!DOCTYPE html> +| <html> +| <head> +| <body> +| <table> +| <tbody> +| <tr> +| "a" diff --git a/src/pkg/html/testdata/webkit/tests18.dat b/src/pkg/html/testdata/webkit/tests18.dat new file mode 100644 index 000000000..680e1f068 --- /dev/null +++ b/src/pkg/html/testdata/webkit/tests18.dat @@ -0,0 +1,269 @@ +#data +<!doctype html><plaintext></plaintext> +#errors +#document +| <!DOCTYPE html> +| <html> +| <head> +| <body> +| <plaintext> +| "</plaintext>" + +#data +<!doctype html><table><plaintext></plaintext> +#errors +#document +| <!DOCTYPE html> +| <html> +| <head> +| <body> +| <plaintext> +| "</plaintext>" +| <table> + +#data +<!doctype html><table><tbody><plaintext></plaintext> +#errors +#document +| <!DOCTYPE html> +| <html> +| <head> +| <body> +| <plaintext> +| "</plaintext>" +| <table> +| <tbody> + +#data +<!doctype html><table><tbody><tr><plaintext></plaintext> +#errors +#document +| <!DOCTYPE html> +| <html> +| <head> +| <body> +| <plaintext> +| "</plaintext>" +| <table> +| <tbody> +| <tr> + +#data +<!doctype html><table><tbody><tr><plaintext></plaintext> +#errors +#document +| <!DOCTYPE html> +| <html> +| <head> +| <body> +| <plaintext> +| "</plaintext>" +| <table> +| <tbody> +| <tr> + +#data +<!doctype html><table><td><plaintext></plaintext> +#errors +#document +| <!DOCTYPE html> +| <html> +| <head> +| <body> +| <table> +| <tbody> +| <tr> +| <td> +| <plaintext> +| "</plaintext>" + +#data +<!doctype html><table><caption><plaintext></plaintext> +#errors +#document +| <!DOCTYPE html> +| <html> +| <head> +| <body> +| <table> +| <caption> +| <plaintext> +| "</plaintext>" + +#data +<!doctype html><table><tr><style></script></style>abc +#errors +#document +| <!DOCTYPE html> +| <html> +| <head> +| <body> +| "abc" +| <table> +| <tbody> +| <tr> +| <style> +| "</script>" + +#data +<!doctype html><table><tr><script></style></script>abc +#errors +#document +| <!DOCTYPE html> +| <html> +| <head> +| <body> +| "abc" +| <table> +| <tbody> +| <tr> +| <script> +| "</style>" + +#data +<!doctype html><table><caption><style></script></style>abc +#errors +#document +| <!DOCTYPE html> +| <html> +| <head> +| <body> +| <table> +| <caption> +| <style> +| "</script>" +| "abc" + +#data +<!doctype html><table><td><style></script></style>abc +#errors +#document +| <!DOCTYPE html> +| <html> +| <head> +| <body> +| <table> +| <tbody> +| <tr> +| <td> +| <style> +| "</script>" +| "abc" + +#data +<!doctype html><select><script></style></script>abc +#errors +#document +| <!DOCTYPE html> +| <html> +| <head> +| <body> +| <select> +| <script> +| "</style>" +| "abc" + +#data +<!doctype html><table><select><script></style></script>abc +#errors +#document +| <!DOCTYPE html> +| <html> +| <head> +| <body> +| <select> +| <script> +| "</style>" +| "abc" +| <table> + +#data +<!doctype html><table><tr><select><script></style></script>abc +#errors +#document +| <!DOCTYPE html> +| <html> +| <head> +| <body> +| <select> +| <script> +| "</style>" +| "abc" +| <table> +| <tbody> +| <tr> + +#data +<!doctype html><frameset></frameset><noframes>abc +#errors +#document +| <!DOCTYPE html> +| <html> +| <head> +| <frameset> +| <noframes> +| "abc" + +#data +<!doctype html><frameset></frameset><noframes>abc</noframes><!--abc--> +#errors +#document +| <!DOCTYPE html> +| <html> +| <head> +| <frameset> +| <noframes> +| "abc" +| <!-- abc --> + +#data +<!doctype html><frameset></frameset></html><noframes>abc +#errors +#document +| <!DOCTYPE html> +| <html> +| <head> +| <frameset> +| <noframes> +| "abc" + +#data +<!doctype html><frameset></frameset></html><noframes>abc</noframes><!--abc--> +#errors +#document +| <!DOCTYPE html> +| <html> +| <head> +| <frameset> +| <noframes> +| "abc" +| <!-- abc --> + +#data +<!doctype html><table><tr></tbody><tfoot> +#errors +#document +| <!DOCTYPE html> +| <html> +| <head> +| <body> +| <table> +| <tbody> +| <tr> +| <tfoot> + +#data +<!doctype html><table><td><svg></svg>abc<td> +#errors +#document +| <!DOCTYPE html> +| <html> +| <head> +| <body> +| <table> +| <tbody> +| <tr> +| <td> +| <svg svg> +| "abc" +| <td> diff --git a/src/pkg/html/testdata/webkit/tests19.dat b/src/pkg/html/testdata/webkit/tests19.dat new file mode 100644 index 000000000..06222f5b9 --- /dev/null +++ b/src/pkg/html/testdata/webkit/tests19.dat @@ -0,0 +1,1220 @@ +#data +<!doctype html><math><mn DefinitionUrl="foo"> +#errors +#document +| <!DOCTYPE html> +| <html> +| <head> +| <body> +| <math math> +| <math mn> +| definitionURL="foo" + +#data +<!doctype html><html></p><!--foo--> +#errors +#document +| <!DOCTYPE html> +| <html> +| <!-- foo --> +| <head> +| <body> + +#data +<!doctype html><head></head></p><!--foo--> +#errors +#document +| <!DOCTYPE html> +| <html> +| <head> +| <!-- foo --> +| <body> + +#data +<!doctype html><body><p><pre> +#errors +#document +| <!DOCTYPE html> +| <html> +| <head> +| <body> +| <p> +| <pre> + +#data +<!doctype html><body><p><listing> +#errors +#document +| <!DOCTYPE html> +| <html> +| <head> +| <body> +| <p> +| <listing> + +#data +<!doctype html><p><plaintext> +#errors +#document +| <!DOCTYPE html> +| <html> +| <head> +| <body> +| <p> +| <plaintext> + +#data +<!doctype html><p><h1> +#errors +#document +| <!DOCTYPE html> +| <html> +| <head> +| <body> +| <p> +| <h1> + +#data +<!doctype html><form><isindex> +#errors +#document +| <!DOCTYPE html> +| <html> +| <head> +| <body> +| <form> + +#data +<!doctype html><isindex action="POST"> +#errors +#document +| <!DOCTYPE html> +| <html> +| <head> +| <body> +| <form> +| action="POST" +| <hr> +| <label> +| "This is a searchable index. Enter search keywords: " +| <input> +| name="isindex" +| <hr> + +#data +<!doctype html><isindex prompt="this is isindex"> +#errors +#document +| <!DOCTYPE html> +| <html> +| <head> +| <body> +| <form> +| <hr> +| <label> +| "this is isindex" +| <input> +| name="isindex" +| <hr> + +#data +<!doctype html><isindex type="hidden"> +#errors +#document +| <!DOCTYPE html> +| <html> +| <head> +| <body> +| <form> +| <hr> +| <label> +| "This is a searchable index. Enter search keywords: " +| <input> +| name="isindex" +| type="hidden" +| <hr> + +#data +<!doctype html><isindex name="foo"> +#errors +#document +| <!DOCTYPE html> +| <html> +| <head> +| <body> +| <form> +| <hr> +| <label> +| "This is a searchable index. Enter search keywords: " +| <input> +| name="isindex" +| <hr> + +#data +<!doctype html><ruby><p><rp> +#errors +#document +| <!DOCTYPE html> +| <html> +| <head> +| <body> +| <ruby> +| <p> +| <rp> + +#data +<!doctype html><ruby><div><span><rp> +#errors +#document +| <!DOCTYPE html> +| <html> +| <head> +| <body> +| <ruby> +| <div> +| <span> +| <rp> + +#data +<!doctype html><ruby><div><p><rp> +#errors +#document +| <!DOCTYPE html> +| <html> +| <head> +| <body> +| <ruby> +| <div> +| <p> +| <rp> + +#data +<!doctype html><ruby><p><rt> +#errors +#document +| <!DOCTYPE html> +| <html> +| <head> +| <body> +| <ruby> +| <p> +| <rt> + +#data +<!doctype html><ruby><div><span><rt> +#errors +#document +| <!DOCTYPE html> +| <html> +| <head> +| <body> +| <ruby> +| <div> +| <span> +| <rt> + +#data +<!doctype html><ruby><div><p><rt> +#errors +#document +| <!DOCTYPE html> +| <html> +| <head> +| <body> +| <ruby> +| <div> +| <p> +| <rt> + +#data +<!doctype html><math/><foo> +#errors +#document +| <!DOCTYPE html> +| <html> +| <head> +| <body> +| <math math> +| <foo> + +#data +<!doctype html><svg/><foo> +#errors +#document +| <!DOCTYPE html> +| <html> +| <head> +| <body> +| <svg svg> +| <foo> + +#data +<!doctype html><div></body><!--foo--> +#errors +#document +| <!DOCTYPE html> +| <html> +| <head> +| <body> +| <div> +| <!-- foo --> + +#data +<!doctype html><h1><div><h3><span></h1>foo +#errors +#document +| <!DOCTYPE html> +| <html> +| <head> +| <body> +| <h1> +| <div> +| <h3> +| <span> +| "foo" + +#data +<!doctype html><p></h3>foo +#errors +#document +| <!DOCTYPE html> +| <html> +| <head> +| <body> +| <p> +| "foo" + +#data +<!doctype html><h3><li>abc</h2>foo +#errors +#document +| <!DOCTYPE html> +| <html> +| <head> +| <body> +| <h3> +| <li> +| "abc" +| "foo" + +#data +<!doctype html><table>abc<!--foo--> +#errors +#document +| <!DOCTYPE html> +| <html> +| <head> +| <body> +| "abc" +| <table> +| <!-- foo --> + +#data +<!doctype html><table> <!--foo--> +#errors +#document +| <!DOCTYPE html> +| <html> +| <head> +| <body> +| <table> +| " " +| <!-- foo --> + +#data +<!doctype html><table> b <!--foo--> +#errors +#document +| <!DOCTYPE html> +| <html> +| <head> +| <body> +| " b " +| <table> +| <!-- foo --> + +#data +<!doctype html><select><option><option> +#errors +#document +| <!DOCTYPE html> +| <html> +| <head> +| <body> +| <select> +| <option> +| <option> + +#data +<!doctype html><select><option></optgroup> +#errors +#document +| <!DOCTYPE html> +| <html> +| <head> +| <body> +| <select> +| <option> + +#data +<!doctype html><select><option></optgroup> +#errors +#document +| <!DOCTYPE html> +| <html> +| <head> +| <body> +| <select> +| <option> + +#data +<!doctype html><p><math><mi><p><h1> +#errors +#document +| <!DOCTYPE html> +| <html> +| <head> +| <body> +| <p> +| <math math> +| <math mi> +| <p> +| <h1> + +#data +<!doctype html><p><math><mo><p><h1> +#errors +#document +| <!DOCTYPE html> +| <html> +| <head> +| <body> +| <p> +| <math math> +| <math mo> +| <p> +| <h1> + +#data +<!doctype html><p><math><mn><p><h1> +#errors +#document +| <!DOCTYPE html> +| <html> +| <head> +| <body> +| <p> +| <math math> +| <math mn> +| <p> +| <h1> + +#data +<!doctype html><p><math><ms><p><h1> +#errors +#document +| <!DOCTYPE html> +| <html> +| <head> +| <body> +| <p> +| <math math> +| <math ms> +| <p> +| <h1> + +#data +<!doctype html><p><math><mtext><p><h1> +#errors +#document +| <!DOCTYPE html> +| <html> +| <head> +| <body> +| <p> +| <math math> +| <math mtext> +| <p> +| <h1> + +#data +<!doctype html><frameset></noframes> +#errors +#document +| <!DOCTYPE html> +| <html> +| <head> +| <frameset> + +#data +<!doctype html><html c=d><body></html><html a=b> +#errors +#document +| <!DOCTYPE html> +| <html> +| a="b" +| c="d" +| <head> +| <body> + +#data +<!doctype html><html c=d><frameset></frameset></html><html a=b> +#errors +#document +| <!DOCTYPE html> +| <html> +| a="b" +| c="d" +| <head> +| <frameset> + +#data +<!doctype html><html><frameset></frameset></html><!--foo--> +#errors +#document +| <!DOCTYPE html> +| <html> +| <head> +| <frameset> +| <!-- foo --> + +#data +<!doctype html><html><frameset></frameset></html> +#errors +#document +| <!DOCTYPE html> +| <html> +| <head> +| <frameset> +| " " + +#data +<!doctype html><html><frameset></frameset></html>abc +#errors +#document +| <!DOCTYPE html> +| <html> +| <head> +| <frameset> + +#data +<!doctype html><html><frameset></frameset></html><p> +#errors +#document +| <!DOCTYPE html> +| <html> +| <head> +| <frameset> + +#data +<!doctype html><html><frameset></frameset></html></p> +#errors +#document +| <!DOCTYPE html> +| <html> +| <head> +| <frameset> + +#data +<html><frameset></frameset></html><!doctype html> +#errors +#document +| <html> +| <head> +| <frameset> + +#data +<!doctype html><body><frameset> +#errors +#document +| <!DOCTYPE html> +| <html> +| <head> +| <body> + +#data +<!doctype html><p><frameset><frame> +#errors +#document +| <!DOCTYPE html> +| <html> +| <head> +| <frameset> +| <frame> + +#data +<!doctype html><p>a<frameset> +#errors +#document +| <!DOCTYPE html> +| <html> +| <head> +| <body> +| <p> +| "a" + +#data +<!doctype html><p> <frameset><frame> +#errors +#document +| <!DOCTYPE html> +| <html> +| <head> +| <frameset> +| <frame> + +#data +<!doctype html><pre><frameset> +#errors +#document +| <!DOCTYPE html> +| <html> +| <head> +| <body> +| <pre> + +#data +<!doctype html><listing><frameset> +#errors +#document +| <!DOCTYPE html> +| <html> +| <head> +| <body> +| <listing> + +#data +<!doctype html><li><frameset> +#errors +#document +| <!DOCTYPE html> +| <html> +| <head> +| <body> +| <li> + +#data +<!doctype html><dd><frameset> +#errors +#document +| <!DOCTYPE html> +| <html> +| <head> +| <body> +| <dd> + +#data +<!doctype html><dt><frameset> +#errors +#document +| <!DOCTYPE html> +| <html> +| <head> +| <body> +| <dt> + +#data +<!doctype html><button><frameset> +#errors +#document +| <!DOCTYPE html> +| <html> +| <head> +| <body> +| <button> + +#data +<!doctype html><applet><frameset> +#errors +#document +| <!DOCTYPE html> +| <html> +| <head> +| <body> +| <applet> + +#data +<!doctype html><marquee><frameset> +#errors +#document +| <!DOCTYPE html> +| <html> +| <head> +| <body> +| <marquee> + +#data +<!doctype html><object><frameset> +#errors +#document +| <!DOCTYPE html> +| <html> +| <head> +| <body> +| <object> + +#data +<!doctype html><table><frameset> +#errors +#document +| <!DOCTYPE html> +| <html> +| <head> +| <body> +| <table> + +#data +<!doctype html><area><frameset> +#errors +#document +| <!DOCTYPE html> +| <html> +| <head> +| <body> +| <area> + +#data +<!doctype html><basefont><frameset> +#errors +#document +| <!DOCTYPE html> +| <html> +| <head> +| <basefont> +| <frameset> + +#data +<!doctype html><bgsound><frameset> +#errors +#document +| <!DOCTYPE html> +| <html> +| <head> +| <bgsound> +| <frameset> + +#data +<!doctype html><br><frameset> +#errors +#document +| <!DOCTYPE html> +| <html> +| <head> +| <body> +| <br> + +#data +<!doctype html><embed><frameset> +#errors +#document +| <!DOCTYPE html> +| <html> +| <head> +| <body> +| <embed> + +#data +<!doctype html><img><frameset> +#errors +#document +| <!DOCTYPE html> +| <html> +| <head> +| <body> +| <img> + +#data +<!doctype html><input><frameset> +#errors +#document +| <!DOCTYPE html> +| <html> +| <head> +| <body> +| <input> + +#data +<!doctype html><keygen><frameset> +#errors +#document +| <!DOCTYPE html> +| <html> +| <head> +| <body> +| <keygen> + +#data +<!doctype html><wbr><frameset> +#errors +#document +| <!DOCTYPE html> +| <html> +| <head> +| <body> +| <wbr> + +#data +<!doctype html><hr><frameset> +#errors +#document +| <!DOCTYPE html> +| <html> +| <head> +| <body> +| <hr> + +#data +<!doctype html><textarea></textarea><frameset> +#errors +#document +| <!DOCTYPE html> +| <html> +| <head> +| <body> +| <textarea> + +#data +<!doctype html><xmp></xmp><frameset> +#errors +#document +| <!DOCTYPE html> +| <html> +| <head> +| <body> +| <xmp> + +#data +<!doctype html><iframe></iframe><frameset> +#errors +#document +| <!DOCTYPE html> +| <html> +| <head> +| <body> +| <iframe> + +#data +<!doctype html><select></select><frameset> +#errors +#document +| <!DOCTYPE html> +| <html> +| <head> +| <body> +| <select> + +#data +<!doctype html><svg></svg><frameset><frame> +#errors +#document +| <!DOCTYPE html> +| <html> +| <head> +| <frameset> +| <frame> + +#data +<!doctype html><math></math><frameset><frame> +#errors +#document +| <!DOCTYPE html> +| <html> +| <head> +| <frameset> +| <frame> + +#data +<!doctype html><svg><foreignObject><div> <frameset><frame> +#errors +#document +| <!DOCTYPE html> +| <html> +| <head> +| <frameset> +| <frame> + +#data +<!doctype html><svg>a</svg><frameset><frame> +#errors +#document +| <!DOCTYPE html> +| <html> +| <head> +| <body> +| <svg svg> +| "a" + +#data +<!doctype html><svg> </svg><frameset><frame> +#errors +#document +| <!DOCTYPE html> +| <html> +| <head> +| <frameset> +| <frame> + +#data +<html>aaa<frameset></frameset> +#errors +#document +| <html> +| <head> +| <body> +| "aaa" + +#data +<html> a <frameset></frameset> +#errors +#document +| <html> +| <head> +| <body> +| "a " + +#data +<!doctype html><div><frameset> +#errors +#document +| <!DOCTYPE html> +| <html> +| <head> +| <frameset> + +#data +<!doctype html><div><body><frameset> +#errors +#document +| <!DOCTYPE html> +| <html> +| <head> +| <body> +| <div> + +#data +<!doctype html><p><math></p>a +#errors +#document +| <!DOCTYPE html> +| <html> +| <head> +| <body> +| <p> +| <math math> +| "a" + +#data +<!doctype html><p><math><mn><span></p>a +#errors +#document +| <!DOCTYPE html> +| <html> +| <head> +| <body> +| <p> +| <math math> +| <math mn> +| <span> +| <p> +| "a" + +#data +<!doctype html><math></html> +#errors +#document +| <!DOCTYPE html> +| <html> +| <head> +| <body> +| <math math> + +#data +<!doctype html><meta charset="ascii"> +#errors +#document +| <!DOCTYPE html> +| <html> +| <head> +| <meta> +| charset="ascii" +| <body> + +#data +<!doctype html><meta http-equiv="content-type" content="text/html;charset=ascii"> +#errors +#document +| <!DOCTYPE html> +| <html> +| <head> +| <meta> +| content="text/html;charset=ascii" +| http-equiv="content-type" +| <body> + +#data +<!doctype html><head><!--aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa--><meta charset="utf8"> +#errors +#document +| <!DOCTYPE html> +| <html> +| <head> +| <!-- aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa --> +| <meta> +| charset="utf8" +| <body> + +#data +<!doctype html><html a=b><head></head><html c=d> +#errors +#document +| <!DOCTYPE html> +| <html> +| a="b" +| c="d" +| <head> +| <body> + +#data +<!doctype html><image/> +#errors +#document +| <!DOCTYPE html> +| <html> +| <head> +| <body> +| <img> + +#data +<!doctype html>a<i>b<table>c<b>d</i>e</b>f +#errors +#document +| <!DOCTYPE html> +| <html> +| <head> +| <body> +| "a" +| <i> +| "bc" +| <b> +| "de" +| "f" +| <table> + +#data +<!doctype html><table><i>a<b>b<div>c<a>d</i>e</b>f +#errors +#document +| <!DOCTYPE html> +| <html> +| <head> +| <body> +| <i> +| "a" +| <b> +| "b" +| <b> +| <div> +| <b> +| <i> +| "c" +| <a> +| "d" +| <a> +| "e" +| <a> +| "f" +| <table> + +#data +<!doctype html><i>a<b>b<div>c<a>d</i>e</b>f +#errors +#document +| <!DOCTYPE html> +| <html> +| <head> +| <body> +| <i> +| "a" +| <b> +| "b" +| <b> +| <div> +| <b> +| <i> +| "c" +| <a> +| "d" +| <a> +| "e" +| <a> +| "f" + +#data +<!doctype html><table><i>a<b>b<div>c</i> +#errors +#document +| <!DOCTYPE html> +| <html> +| <head> +| <body> +| <i> +| "a" +| <b> +| "b" +| <b> +| <div> +| <i> +| "c" +| <table> + +#data +<!doctype html><table><i>a<b>b<div>c<a>d</i>e</b>f +#errors +#document +| <!DOCTYPE html> +| <html> +| <head> +| <body> +| <i> +| "a" +| <b> +| "b" +| <b> +| <div> +| <b> +| <i> +| "c" +| <a> +| "d" +| <a> +| "e" +| <a> +| "f" +| <table> + +#data +<!doctype html><table><i>a<div>b<tr>c<b>d</i>e +#errors +#document +| <!DOCTYPE html> +| <html> +| <head> +| <body> +| <i> +| "a" +| <div> +| "b" +| <i> +| "c" +| <b> +| "d" +| <b> +| "e" +| <table> +| <tbody> +| <tr> + +#data +<!doctype html><table><td><table><i>a<div>b<b>c</i>d +#errors +#document +| <!DOCTYPE html> +| <html> +| <head> +| <body> +| <table> +| <tbody> +| <tr> +| <td> +| <i> +| "a" +| <div> +| <i> +| "b" +| <b> +| "c" +| <b> +| "d" +| <table> + +#data +<!doctype html><body><bgsound> +#errors +#document +| <!DOCTYPE html> +| <html> +| <head> +| <body> +| <bgsound> + +#data +<!doctype html><body><basefont> +#errors +#document +| <!DOCTYPE html> +| <html> +| <head> +| <body> +| <basefont> + +#data +<!doctype html><a><b></a><basefont> +#errors +#document +| <!DOCTYPE html> +| <html> +| <head> +| <body> +| <a> +| <b> +| <basefont> + +#data +<!doctype html><a><b></a><bgsound> +#errors +#document +| <!DOCTYPE html> +| <html> +| <head> +| <body> +| <a> +| <b> +| <bgsound> + +#data +<!doctype html><figcaption><article></figcaption>a +#errors +#document +| <!DOCTYPE html> +| <html> +| <head> +| <body> +| <figcaption> +| <article> +| "a" + +#data +<!doctype html><summary><article></summary>a +#errors +#document +| <!DOCTYPE html> +| <html> +| <head> +| <body> +| <summary> +| <article> +| "a" + +#data +<!doctype html><p><a><plaintext>b +#errors +#document +| <!DOCTYPE html> +| <html> +| <head> +| <body> +| <p> +| <a> +| <plaintext> +| <a> +| "b" diff --git a/src/pkg/html/testdata/webkit/tests2.dat b/src/pkg/html/testdata/webkit/tests2.dat new file mode 100644 index 000000000..60d859221 --- /dev/null +++ b/src/pkg/html/testdata/webkit/tests2.dat @@ -0,0 +1,763 @@ +#data +<!DOCTYPE html>Test +#errors +#document +| <!DOCTYPE html> +| <html> +| <head> +| <body> +| "Test" + +#data +<textarea>test</div>test +#errors +Line: 1 Col: 10 Unexpected start tag (textarea). Expected DOCTYPE. +Line: 1 Col: 24 Expected closing tag. Unexpected end of file. +#document +| <html> +| <head> +| <body> +| <textarea> +| "test</div>test" + +#data +<table><td> +#errors +Line: 1 Col: 7 Unexpected start tag (table). Expected DOCTYPE. +Line: 1 Col: 11 Unexpected table cell start tag (td) in the table body phase. +Line: 1 Col: 11 Expected closing tag. Unexpected end of file. +#document +| <html> +| <head> +| <body> +| <table> +| <tbody> +| <tr> +| <td> + +#data +<table><td>test</tbody></table> +#errors +Line: 1 Col: 7 Unexpected start tag (table). Expected DOCTYPE. +Line: 1 Col: 11 Unexpected table cell start tag (td) in the table body phase. +#document +| <html> +| <head> +| <body> +| <table> +| <tbody> +| <tr> +| <td> +| "test" + +#data +<frame>test +#errors +Line: 1 Col: 7 Unexpected start tag (frame). Expected DOCTYPE. +Line: 1 Col: 7 Unexpected start tag frame. Ignored. +#document +| <html> +| <head> +| <body> +| "test" + +#data +<!DOCTYPE html><frameset>test +#errors +Line: 1 Col: 29 Unepxected characters in the frameset phase. Characters ignored. +Line: 1 Col: 29 Expected closing tag. Unexpected end of file. +#document +| <!DOCTYPE html> +| <html> +| <head> +| <frameset> + +#data +<!DOCTYPE html><frameset><!DOCTYPE html> +#errors +Line: 1 Col: 40 Unexpected DOCTYPE. Ignored. +Line: 1 Col: 40 Expected closing tag. Unexpected end of file. +#document +| <!DOCTYPE html> +| <html> +| <head> +| <frameset> + +#data +<!DOCTYPE html><font><p><b>test</font> +#errors +Line: 1 Col: 38 End tag (font) violates step 1, paragraph 3 of the adoption agency algorithm. +Line: 1 Col: 38 End tag (font) violates step 1, paragraph 3 of the adoption agency algorithm. +#document +| <!DOCTYPE html> +| <html> +| <head> +| <body> +| <font> +| <p> +| <font> +| <b> +| "test" + +#data +<!DOCTYPE html><dt><div><dd> +#errors +Line: 1 Col: 28 Missing end tag (div, dt). +#document +| <!DOCTYPE html> +| <html> +| <head> +| <body> +| <dt> +| <div> +| <dd> + +#data +<script></x +#errors +Line: 1 Col: 8 Unexpected start tag (script). Expected DOCTYPE. +Line: 1 Col: 11 Unexpected end of file. Expected end tag (script). +#document +| <html> +| <head> +| <script> +| "</x" +| <body> + +#data +<table><plaintext><td> +#errors +Line: 1 Col: 7 Unexpected start tag (table). Expected DOCTYPE. +Line: 1 Col: 18 Unexpected start tag (plaintext) in table context caused voodoo mode. +Line: 1 Col: 22 Unexpected end of file. Expected table content. +#document +| <html> +| <head> +| <body> +| <plaintext> +| "<td>" +| <table> + +#data +<plaintext></plaintext> +#errors +Line: 1 Col: 11 Unexpected start tag (plaintext). Expected DOCTYPE. +Line: 1 Col: 23 Expected closing tag. Unexpected end of file. +#document +| <html> +| <head> +| <body> +| <plaintext> +| "</plaintext>" + +#data +<!DOCTYPE html><table><tr>TEST +#errors +Line: 1 Col: 30 Unexpected non-space characters in table context caused voodoo mode. +Line: 1 Col: 30 Unexpected end of file. Expected table content. +#document +| <!DOCTYPE html> +| <html> +| <head> +| <body> +| "TEST" +| <table> +| <tbody> +| <tr> + +#data +<!DOCTYPE html><body t1=1><body t2=2><body t3=3 t4=4> +#errors +Line: 1 Col: 37 Unexpected start tag (body). +Line: 1 Col: 53 Unexpected start tag (body). +#document +| <!DOCTYPE html> +| <html> +| <head> +| <body> +| t1="1" +| t2="2" +| t3="3" +| t4="4" + +#data +</b test +#errors +Line: 1 Col: 8 Unexpected end of file in attribute name. +Line: 1 Col: 8 End tag contains unexpected attributes. +Line: 1 Col: 8 Unexpected end tag (b). Expected DOCTYPE. +Line: 1 Col: 8 Unexpected end tag (b) after the (implied) root element. +#document +| <html> +| <head> +| <body> + +#data +<!DOCTYPE html></b test<b &=&>X +#errors +Line: 1 Col: 32 Named entity didn't end with ';'. +Line: 1 Col: 33 End tag contains unexpected attributes. +Line: 1 Col: 33 Unexpected end tag (b) after the (implied) root element. +#document +| <!DOCTYPE html> +| <html> +| <head> +| <body> +| "X" + +#data +<!doctypehtml><scrIPt type=text/x-foobar;baz>X</SCRipt +#errors +Line: 1 Col: 9 No space after literal string 'DOCTYPE'. +Line: 1 Col: 54 Unexpected end of file in the tag name. +#document +| <!DOCTYPE html> +| <html> +| <head> +| <script> +| type="text/x-foobar;baz" +| "X</SCRipt" +| <body> + +#data +& +#errors +Line: 1 Col: 1 Unexpected non-space characters. Expected DOCTYPE. +#document +| <html> +| <head> +| <body> +| "&" + +#data +&# +#errors +Line: 1 Col: 1 Numeric entity expected. Got end of file instead. +Line: 1 Col: 1 Unexpected non-space characters. Expected DOCTYPE. +#document +| <html> +| <head> +| <body> +| "&#" + +#data +&#X +#errors +Line: 1 Col: 3 Numeric entity expected but none found. +Line: 1 Col: 3 Unexpected non-space characters. Expected DOCTYPE. +#document +| <html> +| <head> +| <body> +| "&#X" + +#data +&#x +#errors +Line: 1 Col: 3 Numeric entity expected but none found. +Line: 1 Col: 3 Unexpected non-space characters. Expected DOCTYPE. +#document +| <html> +| <head> +| <body> +| "&#x" + +#data +- +#errors +Line: 1 Col: 4 Numeric entity didn't end with ';'. +Line: 1 Col: 4 Unexpected non-space characters. Expected DOCTYPE. +#document +| <html> +| <head> +| <body> +| "-" + +#data +&x-test +#errors +Line: 1 Col: 1 Named entity expected. Got none. +Line: 1 Col: 1 Unexpected non-space characters. Expected DOCTYPE. +#document +| <html> +| <head> +| <body> +| "&x-test" + +#data +<!doctypehtml><p><li> +#errors +Line: 1 Col: 9 No space after literal string 'DOCTYPE'. +#document +| <!DOCTYPE html> +| <html> +| <head> +| <body> +| <p> +| <li> + +#data +<!doctypehtml><p><dt> +#errors +Line: 1 Col: 9 No space after literal string 'DOCTYPE'. +#document +| <!DOCTYPE html> +| <html> +| <head> +| <body> +| <p> +| <dt> + +#data +<!doctypehtml><p><dd> +#errors +Line: 1 Col: 9 No space after literal string 'DOCTYPE'. +#document +| <!DOCTYPE html> +| <html> +| <head> +| <body> +| <p> +| <dd> + +#data +<!doctypehtml><p><form> +#errors +Line: 1 Col: 9 No space after literal string 'DOCTYPE'. +Line: 1 Col: 23 Expected closing tag. Unexpected end of file. +#document +| <!DOCTYPE html> +| <html> +| <head> +| <body> +| <p> +| <form> + +#data +<!DOCTYPE html><p></P>X +#errors +#document +| <!DOCTYPE html> +| <html> +| <head> +| <body> +| <p> +| "X" + +#data +& +#errors +Line: 1 Col: 4 Named entity didn't end with ';'. +Line: 1 Col: 4 Unexpected non-space characters. Expected DOCTYPE. +#document +| <html> +| <head> +| <body> +| "&" + +#data +&AMp; +#errors +Line: 1 Col: 1 Named entity expected. Got none. +Line: 1 Col: 1 Unexpected non-space characters. Expected DOCTYPE. +#document +| <html> +| <head> +| <body> +| "&AMp;" + +#data +<!DOCTYPE html><html><head></head><body><thisISasillyTESTelementNameToMakeSureCrazyTagNamesArePARSEDcorrectLY> +#errors +Line: 1 Col: 110 Expected closing tag. Unexpected end of file. +#document +| <!DOCTYPE html> +| <html> +| <head> +| <body> +| <thisisasillytestelementnametomakesurecrazytagnamesareparsedcorrectly> + +#data +<!DOCTYPE html>X</body>X +#errors +Line: 1 Col: 24 Unexpected non-space characters in the after body phase. +#document +| <!DOCTYPE html> +| <html> +| <head> +| <body> +| "XX" + +#data +<!DOCTYPE html><!-- X +#errors +Line: 1 Col: 21 Unexpected end of file in comment. +#document +| <!DOCTYPE html> +| <!-- X --> +| <html> +| <head> +| <body> + +#data +<!DOCTYPE html><table><caption>test TEST</caption><td>test +#errors +Line: 1 Col: 54 Unexpected table cell start tag (td) in the table body phase. +Line: 1 Col: 58 Expected closing tag. Unexpected end of file. +#document +| <!DOCTYPE html> +| <html> +| <head> +| <body> +| <table> +| <caption> +| "test TEST" +| <tbody> +| <tr> +| <td> +| "test" + +#data +<!DOCTYPE html><select><option><optgroup> +#errors +Line: 1 Col: 41 Expected closing tag. Unexpected end of file. +#document +| <!DOCTYPE html> +| <html> +| <head> +| <body> +| <select> +| <option> +| <optgroup> + +#data +<!DOCTYPE html><select><optgroup><option></optgroup><option><select><option> +#errors +Line: 1 Col: 68 Unexpected select start tag in the select phase treated as select end tag. +Line: 1 Col: 76 Expected closing tag. Unexpected end of file. +#document +| <!DOCTYPE html> +| <html> +| <head> +| <body> +| <select> +| <optgroup> +| <option> +| <option> +| <option> + +#data +<!DOCTYPE html><select><optgroup><option><optgroup> +#errors +Line: 1 Col: 51 Expected closing tag. Unexpected end of file. +#document +| <!DOCTYPE html> +| <html> +| <head> +| <body> +| <select> +| <optgroup> +| <option> +| <optgroup> + +#data +<!DOCTYPE html><datalist><option>foo</datalist>bar +#errors +#document +| <!DOCTYPE html> +| <html> +| <head> +| <body> +| <datalist> +| <option> +| "foo" +| "bar" + +#data +<!DOCTYPE html><font><input><input></font> +#errors +#document +| <!DOCTYPE html> +| <html> +| <head> +| <body> +| <font> +| <input> +| <input> + +#data +<!DOCTYPE html><!-- XXX - XXX --> +#errors +#document +| <!DOCTYPE html> +| <!-- XXX - XXX --> +| <html> +| <head> +| <body> + +#data +<!DOCTYPE html><!-- XXX - XXX +#errors +Line: 1 Col: 29 Unexpected end of file in comment (-) +#document +| <!DOCTYPE html> +| <!-- XXX - XXX --> +| <html> +| <head> +| <body> + +#data +<!DOCTYPE html><!-- XXX - XXX - XXX --> +#errors +#document +| <!DOCTYPE html> +| <!-- XXX - XXX - XXX --> +| <html> +| <head> +| <body> + +#data +<isindex test=x name=x> +#errors +Line: 1 Col: 23 Unexpected start tag (isindex). Expected DOCTYPE. +Line: 1 Col: 23 Unexpected start tag isindex. Don't use it! +#document +| <html> +| <head> +| <body> +| <form> +| <hr> +| <label> +| "This is a searchable index. Enter search keywords: " +| <input> +| name="isindex" +| test="x" +| <hr> + +#data +test +test +#errors +Line: 2 Col: 4 Unexpected non-space characters. Expected DOCTYPE. +#document +| <html> +| <head> +| <body> +| "test +test" + +#data +<!DOCTYPE html><body><title>test</body> +#errors +#document +| +| +| +| +| +| "test</body>" + +#data +<!DOCTYPE html><body><title>X +#errors +#document +| +| +| +| +| +| "X" +| <meta> +| name="z" +| <link> +| rel="foo" +| <style> +| " +x { content:"</style" } " + +#data +<!DOCTYPE html><select><optgroup></optgroup></select> +#errors +#document +| <!DOCTYPE html> +| <html> +| <head> +| <body> +| <select> +| <optgroup> + +#data + + +#errors +Line: 2 Col: 1 Unexpected End of file. Expected DOCTYPE. +#document +| <html> +| <head> +| <body> + +#data +<!DOCTYPE html> <html> +#errors +#document +| <!DOCTYPE html> +| <html> +| <head> +| <body> + +#data +<!DOCTYPE html><script> +</script> <title>x +#errors +#document +| +| +| +| +#errors +Line: 1 Col: 6 Unexpected start tag (head). Expected DOCTYPE. +Line: 1 Col: 21 Unexpected start tag (script) that can be in head. Moved. +#document +| +| +| +#errors +Line: 1 Col: 6 Unexpected start tag (head). Expected DOCTYPE. +Line: 1 Col: 28 Unexpected start tag (style) that can be in head. Moved. +#document +| +| +| +#errors +Line: 1 Col: 6 Unexpected start tag (head). Expected DOCTYPE. +#document +| +| +| +| +| "x" +| x +#errors +Line: 1 Col: 7 Unexpected start tag (style). Expected DOCTYPE. +Line: 1 Col: 22 Unexpected end of file. Expected end tag (style). +#document +| +| +| --> x +#errors +Line: 1 Col: 7 Unexpected start tag (style). Expected DOCTYPE. +#document +| +| +| x +#errors +Line: 1 Col: 7 Unexpected start tag (style). Expected DOCTYPE. +#document +| +| +| x +#errors +Line: 1 Col: 7 Unexpected start tag (style). Expected DOCTYPE. +#document +| +| +| x +#errors +Line: 1 Col: 7 Unexpected start tag (style). Expected DOCTYPE. +#document +| +| +|

+#errors +#document +| +| +| +| +| +| ddd +#errors +#document +| +| +| +#errors +#document +| +| +| +| +|
  • +| +|